summaryrefslogtreecommitdiffstats
path: root/js/src/jit
diff options
context:
space:
mode:
Diffstat (limited to 'js/src/jit')
-rw-r--r--js/src/jit/ABIArgGenerator.h109
-rw-r--r--js/src/jit/ABIFunctionList-inl.h251
-rw-r--r--js/src/jit/ABIFunctions.h73
-rw-r--r--js/src/jit/AliasAnalysis.cpp322
-rw-r--r--js/src/jit/AliasAnalysis.h64
-rw-r--r--js/src/jit/AlignmentMaskAnalysis.cpp97
-rw-r--r--js/src/jit/AlignmentMaskAnalysis.h27
-rw-r--r--js/src/jit/Assembler.h32
-rw-r--r--js/src/jit/AtomicOp.h98
-rw-r--r--js/src/jit/AtomicOperations.h352
-rw-r--r--js/src/jit/AutoWritableJitCode.h88
-rw-r--r--js/src/jit/BacktrackingAllocator.cpp4660
-rw-r--r--js/src/jit/BacktrackingAllocator.h798
-rw-r--r--js/src/jit/Bailouts.cpp336
-rw-r--r--js/src/jit/Bailouts.h227
-rw-r--r--js/src/jit/BaselineBailouts.cpp2058
-rw-r--r--js/src/jit/BaselineCacheIRCompiler.cpp3475
-rw-r--r--js/src/jit/BaselineCacheIRCompiler.h138
-rw-r--r--js/src/jit/BaselineCodeGen.cpp6746
-rw-r--r--js/src/jit/BaselineCodeGen.h522
-rw-r--r--js/src/jit/BaselineDebugModeOSR.cpp561
-rw-r--r--js/src/jit/BaselineDebugModeOSR.h30
-rw-r--r--js/src/jit/BaselineFrame-inl.h131
-rw-r--r--js/src/jit/BaselineFrame.cpp180
-rw-r--r--js/src/jit/BaselineFrame.h373
-rw-r--r--js/src/jit/BaselineFrameInfo-inl.h50
-rw-r--r--js/src/jit/BaselineFrameInfo.cpp239
-rw-r--r--js/src/jit/BaselineFrameInfo.h434
-rw-r--r--js/src/jit/BaselineIC.cpp2495
-rw-r--r--js/src/jit/BaselineIC.h439
-rw-r--r--js/src/jit/BaselineICList.h49
-rw-r--r--js/src/jit/BaselineJIT.cpp956
-rw-r--r--js/src/jit/BaselineJIT.h595
-rw-r--r--js/src/jit/BitSet.cpp111
-rw-r--r--js/src/jit/BitSet.h167
-rw-r--r--js/src/jit/BytecodeAnalysis.cpp283
-rw-r--r--js/src/jit/BytecodeAnalysis.h84
-rw-r--r--js/src/jit/CacheIR.cpp12698
-rw-r--r--js/src/jit/CacheIR.h523
-rw-r--r--js/src/jit/CacheIRCloner.h82
-rw-r--r--js/src/jit/CacheIRCompiler.cpp9573
-rw-r--r--js/src/jit/CacheIRCompiler.h1310
-rw-r--r--js/src/jit/CacheIRGenerator.h893
-rw-r--r--js/src/jit/CacheIRHealth.cpp416
-rw-r--r--js/src/jit/CacheIRHealth.h109
-rw-r--r--js/src/jit/CacheIROps.yaml3016
-rw-r--r--js/src/jit/CacheIRReader.h155
-rw-r--r--js/src/jit/CacheIRSpewer.cpp441
-rw-r--r--js/src/jit/CacheIRSpewer.h116
-rw-r--r--js/src/jit/CacheIRWriter.h638
-rw-r--r--js/src/jit/CalleeToken.h66
-rw-r--r--js/src/jit/CodeGenerator.cpp18002
-rw-r--r--js/src/jit/CodeGenerator.h427
-rw-r--r--js/src/jit/CompactBuffer.h254
-rw-r--r--js/src/jit/CompileInfo.h382
-rw-r--r--js/src/jit/CompileWrappers.cpp235
-rw-r--r--js/src/jit/CompileWrappers.h178
-rw-r--r--js/src/jit/Disassemble.cpp109
-rw-r--r--js/src/jit/Disassemble.h24
-rw-r--r--js/src/jit/EdgeCaseAnalysis.cpp50
-rw-r--r--js/src/jit/EdgeCaseAnalysis.h28
-rw-r--r--js/src/jit/EffectiveAddressAnalysis.cpp256
-rw-r--r--js/src/jit/EffectiveAddressAnalysis.h33
-rw-r--r--js/src/jit/ExecutableAllocator.cpp329
-rw-r--r--js/src/jit/ExecutableAllocator.h205
-rw-r--r--js/src/jit/FixedList.h99
-rw-r--r--js/src/jit/FlushICache.cpp132
-rw-r--r--js/src/jit/FlushICache.h92
-rw-r--r--js/src/jit/FoldLinearArithConstants.cpp112
-rw-r--r--js/src/jit/FoldLinearArithConstants.h21
-rw-r--r--js/src/jit/GenerateAtomicOperations.py873
-rw-r--r--js/src/jit/GenerateCacheIRFiles.py539
-rw-r--r--js/src/jit/GenerateLIRFiles.py298
-rw-r--r--js/src/jit/GenerateMIRFiles.py403
-rw-r--r--js/src/jit/ICState.h212
-rw-r--r--js/src/jit/ICStubSpace.h66
-rw-r--r--js/src/jit/InlinableNatives.cpp297
-rw-r--r--js/src/jit/InlinableNatives.h236
-rw-r--r--js/src/jit/InlineList.h590
-rw-r--r--js/src/jit/InlineScriptTree-inl.h67
-rw-r--r--js/src/jit/InlineScriptTree.h112
-rw-r--r--js/src/jit/InstructionReordering.cpp248
-rw-r--r--js/src/jit/InstructionReordering.h20
-rw-r--r--js/src/jit/Invalidation.h59
-rw-r--r--js/src/jit/Ion.cpp2560
-rw-r--r--js/src/jit/Ion.h158
-rw-r--r--js/src/jit/IonAnalysis.cpp4666
-rw-r--r--js/src/jit/IonAnalysis.h189
-rw-r--r--js/src/jit/IonCacheIRCompiler.cpp2093
-rw-r--r--js/src/jit/IonCacheIRCompiler.h89
-rw-r--r--js/src/jit/IonCompileTask.cpp203
-rw-r--r--js/src/jit/IonCompileTask.h89
-rw-r--r--js/src/jit/IonIC.cpp722
-rw-r--r--js/src/jit/IonIC.h664
-rw-r--r--js/src/jit/IonOptimizationLevels.cpp139
-rw-r--r--js/src/jit/IonOptimizationLevels.h194
-rw-r--r--js/src/jit/IonScript.h590
-rw-r--r--js/src/jit/IonTypes.h1090
-rw-r--r--js/src/jit/JSJitFrameIter-inl.h65
-rw-r--r--js/src/jit/JSJitFrameIter.cpp775
-rw-r--r--js/src/jit/JSJitFrameIter.h803
-rw-r--r--js/src/jit/JSONSpewer.cpp287
-rw-r--r--js/src/jit/JSONSpewer.h48
-rw-r--r--js/src/jit/Jit.cpp206
-rw-r--r--js/src/jit/Jit.h39
-rw-r--r--js/src/jit/JitAllocPolicy.h179
-rw-r--r--js/src/jit/JitCode.h171
-rw-r--r--js/src/jit/JitCommon.h53
-rw-r--r--js/src/jit/JitContext.cpp161
-rw-r--r--js/src/jit/JitContext.h168
-rw-r--r--js/src/jit/JitFrames-inl.h32
-rw-r--r--js/src/jit/JitFrames.cpp2528
-rw-r--r--js/src/jit/JitFrames.h741
-rw-r--r--js/src/jit/JitOptions.cpp393
-rw-r--r--js/src/jit/JitOptions.h174
-rw-r--r--js/src/jit/JitRealm.h161
-rw-r--r--js/src/jit/JitRuntime.h415
-rw-r--r--js/src/jit/JitScript-inl.h43
-rw-r--r--js/src/jit/JitScript.cpp735
-rw-r--r--js/src/jit/JitScript.h576
-rw-r--r--js/src/jit/JitSpewer.cpp657
-rw-r--r--js/src/jit/JitSpewer.h284
-rw-r--r--js/src/jit/JitZone.h182
-rw-r--r--js/src/jit/JitcodeMap.cpp1145
-rw-r--r--js/src/jit/JitcodeMap.h808
-rw-r--r--js/src/jit/Jitdump.h78
-rw-r--r--js/src/jit/KnownClass.cpp109
-rw-r--r--js/src/jit/KnownClass.h36
-rw-r--r--js/src/jit/LICM.cpp367
-rw-r--r--js/src/jit/LICM.h23
-rw-r--r--js/src/jit/LIR.cpp732
-rw-r--r--js/src/jit/LIR.h1992
-rw-r--r--js/src/jit/LIROps.yaml3866
-rw-r--r--js/src/jit/Label.cpp29
-rw-r--r--js/src/jit/Label.h106
-rw-r--r--js/src/jit/Linker.cpp79
-rw-r--r--js/src/jit/Linker.h52
-rw-r--r--js/src/jit/Lowering.cpp7004
-rw-r--r--js/src/jit/Lowering.h88
-rw-r--r--js/src/jit/MIR.cpp7195
-rw-r--r--js/src/jit/MIR.h11344
-rw-r--r--js/src/jit/MIRGenerator.h176
-rw-r--r--js/src/jit/MIRGraph.cpp1251
-rw-r--r--js/src/jit/MIRGraph.h875
-rw-r--r--js/src/jit/MIROps.yaml2972
-rw-r--r--js/src/jit/MachineState.h110
-rw-r--r--js/src/jit/MacroAssembler-inl.h1072
-rw-r--r--js/src/jit/MacroAssembler.cpp6222
-rw-r--r--js/src/jit/MacroAssembler.h5522
-rw-r--r--js/src/jit/MoveEmitter.h30
-rw-r--r--js/src/jit/MoveResolver.cpp443
-rw-r--r--js/src/jit/MoveResolver.h308
-rw-r--r--js/src/jit/PcScriptCache.h88
-rw-r--r--js/src/jit/PerfSpewer.cpp719
-rw-r--r--js/src/jit/PerfSpewer.h112
-rw-r--r--js/src/jit/ProcessExecutableMemory.cpp846
-rw-r--r--js/src/jit/ProcessExecutableMemory.h109
-rw-r--r--js/src/jit/RangeAnalysis.cpp3697
-rw-r--r--js/src/jit/RangeAnalysis.h683
-rw-r--r--js/src/jit/ReciprocalMulConstants.cpp94
-rw-r--r--js/src/jit/ReciprocalMulConstants.h33
-rw-r--r--js/src/jit/Recover.cpp2126
-rw-r--r--js/src/jit/Recover.h973
-rw-r--r--js/src/jit/RegisterAllocator.cpp669
-rw-r--r--js/src/jit/RegisterAllocator.h313
-rw-r--r--js/src/jit/RegisterSets.h1333
-rw-r--r--js/src/jit/Registers.h297
-rw-r--r--js/src/jit/RematerializedFrame-inl.h23
-rw-r--r--js/src/jit/RematerializedFrame.cpp221
-rw-r--r--js/src/jit/RematerializedFrame.h219
-rw-r--r--js/src/jit/SafepointIndex-inl.h22
-rw-r--r--js/src/jit/SafepointIndex.cpp20
-rw-r--r--js/src/jit/SafepointIndex.h76
-rw-r--r--js/src/jit/Safepoints.cpp559
-rw-r--r--js/src/jit/Safepoints.h129
-rw-r--r--js/src/jit/ScalarReplacement.cpp3071
-rw-r--r--js/src/jit/ScalarReplacement.h22
-rw-r--r--js/src/jit/ScalarTypeUtils.h41
-rw-r--r--js/src/jit/ScriptFromCalleeToken.h33
-rw-r--r--js/src/jit/SharedICHelpers-inl.h34
-rw-r--r--js/src/jit/SharedICHelpers.h34
-rw-r--r--js/src/jit/SharedICRegisters.h36
-rw-r--r--js/src/jit/ShuffleAnalysis.cpp747
-rw-r--r--js/src/jit/ShuffleAnalysis.h147
-rw-r--r--js/src/jit/Simulator.h30
-rw-r--r--js/src/jit/Sink.cpp255
-rw-r--r--js/src/jit/Sink.h22
-rw-r--r--js/src/jit/Snapshots.cpp605
-rw-r--r--js/src/jit/Snapshots.h529
-rw-r--r--js/src/jit/StackSlotAllocator.h133
-rw-r--r--js/src/jit/TemplateObject-inl.h126
-rw-r--r--js/src/jit/TemplateObject.h77
-rw-r--r--js/src/jit/Trampoline.cpp253
-rw-r--r--js/src/jit/TrialInlining.cpp851
-rw-r--r--js/src/jit/TrialInlining.h187
-rw-r--r--js/src/jit/TypeData.h54
-rw-r--r--js/src/jit/TypePolicy.cpp1139
-rw-r--r--js/src/jit/TypePolicy.h543
-rw-r--r--js/src/jit/VMFunctionList-inl.h371
-rw-r--r--js/src/jit/VMFunctions.cpp2834
-rw-r--r--js/src/jit/VMFunctions.h704
-rw-r--r--js/src/jit/ValueNumbering.cpp1335
-rw-r--r--js/src/jit/ValueNumbering.h123
-rw-r--r--js/src/jit/WarpBuilder.cpp3589
-rw-r--r--js/src/jit/WarpBuilder.h323
-rw-r--r--js/src/jit/WarpBuilderShared.cpp99
-rw-r--r--js/src/jit/WarpBuilderShared.h409
-rw-r--r--js/src/jit/WarpCacheIRTranspiler.cpp5574
-rw-r--r--js/src/jit/WarpCacheIRTranspiler.h33
-rw-r--r--js/src/jit/WarpOracle.cpp1171
-rw-r--r--js/src/jit/WarpOracle.h61
-rw-r--r--js/src/jit/WarpSnapshot.cpp402
-rw-r--r--js/src/jit/WarpSnapshot.h624
-rw-r--r--js/src/jit/WasmBCE.cpp139
-rw-r--r--js/src/jit/WasmBCE.h33
-rw-r--r--js/src/jit/XrayJitInfo.cpp17
-rw-r--r--js/src/jit/arm/Architecture-arm.cpp540
-rw-r--r--js/src/jit/arm/Architecture-arm.h735
-rw-r--r--js/src/jit/arm/Assembler-arm.cpp2832
-rw-r--r--js/src/jit/arm/Assembler-arm.h2291
-rw-r--r--js/src/jit/arm/CodeGenerator-arm.cpp3154
-rw-r--r--js/src/jit/arm/CodeGenerator-arm.h172
-rw-r--r--js/src/jit/arm/DoubleEntryTable.tbl257
-rw-r--r--js/src/jit/arm/LIR-arm.h511
-rw-r--r--js/src/jit/arm/Lowering-arm.cpp1223
-rw-r--r--js/src/jit/arm/Lowering-arm.h118
-rw-r--r--js/src/jit/arm/MacroAssembler-arm-inl.h2582
-rw-r--r--js/src/jit/arm/MacroAssembler-arm.cpp6375
-rw-r--r--js/src/jit/arm/MacroAssembler-arm.h1402
-rw-r--r--js/src/jit/arm/MoveEmitter-arm.cpp413
-rw-r--r--js/src/jit/arm/MoveEmitter-arm.h70
-rw-r--r--js/src/jit/arm/SharedICHelpers-arm-inl.h79
-rw-r--r--js/src/jit/arm/SharedICHelpers-arm.h80
-rw-r--r--js/src/jit/arm/SharedICRegisters-arm.h52
-rw-r--r--js/src/jit/arm/Simulator-arm.cpp5449
-rw-r--r--js/src/jit/arm/Simulator-arm.h632
-rw-r--r--js/src/jit/arm/Trampoline-arm.cpp829
-rw-r--r--js/src/jit/arm/disasm/Constants-arm.cpp117
-rw-r--r--js/src/jit/arm/disasm/Constants-arm.h684
-rw-r--r--js/src/jit/arm/disasm/Disasm-arm.cpp2031
-rw-r--r--js/src/jit/arm/disasm/Disasm-arm.h141
-rw-r--r--js/src/jit/arm/gen-double-encoder-table.py35
-rw-r--r--js/src/jit/arm/llvm-compiler-rt/arm/aeabi_idivmod.S27
-rw-r--r--js/src/jit/arm/llvm-compiler-rt/arm/aeabi_uidivmod.S28
-rw-r--r--js/src/jit/arm/llvm-compiler-rt/assembly.h67
-rw-r--r--js/src/jit/arm64/Architecture-arm64.cpp129
-rw-r--r--js/src/jit/arm64/Architecture-arm64.h775
-rw-r--r--js/src/jit/arm64/Assembler-arm64.cpp609
-rw-r--r--js/src/jit/arm64/Assembler-arm64.h788
-rw-r--r--js/src/jit/arm64/CodeGenerator-arm64.cpp4248
-rw-r--r--js/src/jit/arm64/CodeGenerator-arm64.h135
-rw-r--r--js/src/jit/arm64/LIR-arm64.h373
-rw-r--r--js/src/jit/arm64/Lowering-arm64.cpp1444
-rw-r--r--js/src/jit/arm64/Lowering-arm64.h135
-rw-r--r--js/src/jit/arm64/MacroAssembler-arm64-inl.h4052
-rw-r--r--js/src/jit/arm64/MacroAssembler-arm64.cpp3140
-rw-r--r--js/src/jit/arm64/MacroAssembler-arm64.h2335
-rw-r--r--js/src/jit/arm64/MoveEmitter-arm64.cpp329
-rw-r--r--js/src/jit/arm64/MoveEmitter-arm64.h99
-rw-r--r--js/src/jit/arm64/SharedICHelpers-arm64-inl.h79
-rw-r--r--js/src/jit/arm64/SharedICHelpers-arm64.h82
-rw-r--r--js/src/jit/arm64/SharedICRegisters-arm64.h51
-rw-r--r--js/src/jit/arm64/Trampoline-arm64.cpp838
-rw-r--r--js/src/jit/arm64/vixl/.clang-format4
-rw-r--r--js/src/jit/arm64/vixl/Assembler-vixl.cpp5145
-rw-r--r--js/src/jit/arm64/vixl/Assembler-vixl.h4327
-rw-r--r--js/src/jit/arm64/vixl/CompilerIntrinsics-vixl.h179
-rw-r--r--js/src/jit/arm64/vixl/Constants-vixl.h2694
-rw-r--r--js/src/jit/arm64/vixl/Cpu-Features-vixl.cpp231
-rw-r--r--js/src/jit/arm64/vixl/Cpu-Features-vixl.h397
-rw-r--r--js/src/jit/arm64/vixl/Cpu-vixl.cpp252
-rw-r--r--js/src/jit/arm64/vixl/Cpu-vixl.h241
-rw-r--r--js/src/jit/arm64/vixl/Debugger-vixl.cpp1535
-rw-r--r--js/src/jit/arm64/vixl/Debugger-vixl.h117
-rw-r--r--js/src/jit/arm64/vixl/Decoder-vixl.cpp878
-rw-r--r--js/src/jit/arm64/vixl/Decoder-vixl.h275
-rw-r--r--js/src/jit/arm64/vixl/Disasm-vixl.cpp3519
-rw-r--r--js/src/jit/arm64/vixl/Disasm-vixl.h181
-rw-r--r--js/src/jit/arm64/vixl/Globals-vixl.h272
-rw-r--r--js/src/jit/arm64/vixl/Instructions-vixl.cpp642
-rw-r--r--js/src/jit/arm64/vixl/Instructions-vixl.h805
-rw-r--r--js/src/jit/arm64/vixl/Instrument-vixl.cpp844
-rw-r--r--js/src/jit/arm64/vixl/Instrument-vixl.h109
-rw-r--r--js/src/jit/arm64/vixl/Logic-vixl.cpp4738
-rw-r--r--js/src/jit/arm64/vixl/MacroAssembler-vixl.cpp2027
-rw-r--r--js/src/jit/arm64/vixl/MacroAssembler-vixl.h2516
-rw-r--r--js/src/jit/arm64/vixl/MozAssembler-vixl.cpp610
-rw-r--r--js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h356
-rw-r--r--js/src/jit/arm64/vixl/MozCachingDecoder.h179
-rw-r--r--js/src/jit/arm64/vixl/MozCpu-vixl.cpp226
-rw-r--r--js/src/jit/arm64/vixl/MozInstructions-vixl.cpp211
-rw-r--r--js/src/jit/arm64/vixl/MozSimulator-vixl.cpp1239
-rw-r--r--js/src/jit/arm64/vixl/Platform-vixl.h39
-rw-r--r--js/src/jit/arm64/vixl/README.md7
-rw-r--r--js/src/jit/arm64/vixl/Simulator-Constants-vixl.h140
-rw-r--r--js/src/jit/arm64/vixl/Simulator-vixl.cpp3993
-rw-r--r--js/src/jit/arm64/vixl/Simulator-vixl.h2571
-rw-r--r--js/src/jit/arm64/vixl/Utils-vixl.cpp555
-rw-r--r--js/src/jit/arm64/vixl/Utils-vixl.h1283
-rw-r--r--js/src/jit/loong64/Architecture-loong64.cpp87
-rw-r--r--js/src/jit/loong64/Architecture-loong64.h524
-rw-r--r--js/src/jit/loong64/Assembler-loong64.cpp2478
-rw-r--r--js/src/jit/loong64/Assembler-loong64.h1879
-rw-r--r--js/src/jit/loong64/CodeGenerator-loong64.cpp2765
-rw-r--r--js/src/jit/loong64/CodeGenerator-loong64.h209
-rw-r--r--js/src/jit/loong64/LIR-loong64.h399
-rw-r--r--js/src/jit/loong64/Lowering-loong64.cpp1087
-rw-r--r--js/src/jit/loong64/Lowering-loong64.h110
-rw-r--r--js/src/jit/loong64/MacroAssembler-loong64-inl.h2120
-rw-r--r--js/src/jit/loong64/MacroAssembler-loong64.cpp5388
-rw-r--r--js/src/jit/loong64/MacroAssembler-loong64.h1042
-rw-r--r--js/src/jit/loong64/MoveEmitter-loong64.cpp326
-rw-r--r--js/src/jit/loong64/MoveEmitter-loong64.h76
-rw-r--r--js/src/jit/loong64/SharedICHelpers-loong64-inl.h83
-rw-r--r--js/src/jit/loong64/SharedICHelpers-loong64.h91
-rw-r--r--js/src/jit/loong64/SharedICRegisters-loong64.h42
-rw-r--r--js/src/jit/loong64/Simulator-loong64.cpp5220
-rw-r--r--js/src/jit/loong64/Simulator-loong64.h650
-rw-r--r--js/src/jit/loong64/Trampoline-loong64.cpp833
-rw-r--r--js/src/jit/mips-shared/Architecture-mips-shared.cpp121
-rw-r--r--js/src/jit/mips-shared/Architecture-mips-shared.h341
-rw-r--r--js/src/jit/mips-shared/Assembler-mips-shared.cpp2094
-rw-r--r--js/src/jit/mips-shared/Assembler-mips-shared.h1495
-rw-r--r--js/src/jit/mips-shared/AtomicOperations-mips-shared.h521
-rw-r--r--js/src/jit/mips-shared/BaselineIC-mips-shared.cpp37
-rw-r--r--js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp2440
-rw-r--r--js/src/jit/mips-shared/CodeGenerator-mips-shared.h157
-rw-r--r--js/src/jit/mips-shared/LIR-mips-shared.h360
-rw-r--r--js/src/jit/mips-shared/Lowering-mips-shared.cpp1023
-rw-r--r--js/src/jit/mips-shared/Lowering-mips-shared.h89
-rw-r--r--js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h1312
-rw-r--r--js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp3355
-rw-r--r--js/src/jit/mips-shared/MacroAssembler-mips-shared.h258
-rw-r--r--js/src/jit/mips-shared/MoveEmitter-mips-shared.cpp207
-rw-r--r--js/src/jit/mips-shared/MoveEmitter-mips-shared.h73
-rw-r--r--js/src/jit/mips-shared/SharedICHelpers-mips-shared-inl.h82
-rw-r--r--js/src/jit/mips-shared/SharedICHelpers-mips-shared.h88
-rw-r--r--js/src/jit/mips32/Architecture-mips32.cpp94
-rw-r--r--js/src/jit/mips32/Architecture-mips32.h282
-rw-r--r--js/src/jit/mips32/Assembler-mips32.cpp369
-rw-r--r--js/src/jit/mips32/Assembler-mips32.h265
-rw-r--r--js/src/jit/mips32/CodeGenerator-mips32.cpp507
-rw-r--r--js/src/jit/mips32/CodeGenerator-mips32.h60
-rw-r--r--js/src/jit/mips32/LIR-mips32.h197
-rw-r--r--js/src/jit/mips32/Lowering-mips32.cpp257
-rw-r--r--js/src/jit/mips32/Lowering-mips32.h54
-rw-r--r--js/src/jit/mips32/MacroAssembler-mips32-inl.h1022
-rw-r--r--js/src/jit/mips32/MacroAssembler-mips32.cpp2824
-rw-r--r--js/src/jit/mips32/MacroAssembler-mips32.h829
-rw-r--r--js/src/jit/mips32/MoveEmitter-mips32.cpp152
-rw-r--r--js/src/jit/mips32/MoveEmitter-mips32.h31
-rw-r--r--js/src/jit/mips32/SharedICRegisters-mips32.h42
-rw-r--r--js/src/jit/mips32/Simulator-mips32.cpp3631
-rw-r--r--js/src/jit/mips32/Simulator-mips32.h526
-rw-r--r--js/src/jit/mips32/Trampoline-mips32.cpp942
-rw-r--r--js/src/jit/mips64/Architecture-mips64.cpp88
-rw-r--r--js/src/jit/mips64/Architecture-mips64.h233
-rw-r--r--js/src/jit/mips64/Assembler-mips64.cpp371
-rw-r--r--js/src/jit/mips64/Assembler-mips64.h288
-rw-r--r--js/src/jit/mips64/CodeGenerator-mips64.cpp574
-rw-r--r--js/src/jit/mips64/CodeGenerator-mips64.h65
-rw-r--r--js/src/jit/mips64/LIR-mips64.h147
-rw-r--r--js/src/jit/mips64/Lowering-mips64.cpp201
-rw-r--r--js/src/jit/mips64/Lowering-mips64.h56
-rw-r--r--js/src/jit/mips64/MacroAssembler-mips64-inl.h824
-rw-r--r--js/src/jit/mips64/MacroAssembler-mips64.cpp2851
-rw-r--r--js/src/jit/mips64/MacroAssembler-mips64.h846
-rw-r--r--js/src/jit/mips64/MoveEmitter-mips64.cpp149
-rw-r--r--js/src/jit/mips64/MoveEmitter-mips64.h31
-rw-r--r--js/src/jit/mips64/SharedICRegisters-mips64.h45
-rw-r--r--js/src/jit/mips64/Simulator-mips64.cpp4386
-rw-r--r--js/src/jit/mips64/Simulator-mips64.h536
-rw-r--r--js/src/jit/mips64/Trampoline-mips64.cpp870
-rw-r--r--js/src/jit/moz.build269
-rw-r--r--js/src/jit/none/Architecture-none.h171
-rw-r--r--js/src/jit/none/Assembler-none.h210
-rw-r--r--js/src/jit/none/CodeGenerator-none.h78
-rw-r--r--js/src/jit/none/LIR-none.h111
-rw-r--r--js/src/jit/none/Lowering-none.h130
-rw-r--r--js/src/jit/none/MacroAssembler-none.h458
-rw-r--r--js/src/jit/none/MoveEmitter-none.h32
-rw-r--r--js/src/jit/none/SharedICHelpers-none-inl.h31
-rw-r--r--js/src/jit/none/SharedICHelpers-none.h32
-rw-r--r--js/src/jit/none/SharedICRegisters-none.h32
-rw-r--r--js/src/jit/none/Trampoline-none.cpp43
-rw-r--r--js/src/jit/shared/Architecture-shared.h18
-rw-r--r--js/src/jit/shared/Assembler-shared.cpp74
-rw-r--r--js/src/jit/shared/Assembler-shared.h709
-rw-r--r--js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h453
-rw-r--r--js/src/jit/shared/AtomicOperations-feeling-lucky.h16
-rw-r--r--js/src/jit/shared/AtomicOperations-shared-jit.cpp176
-rw-r--r--js/src/jit/shared/AtomicOperations-shared-jit.h490
-rw-r--r--js/src/jit/shared/CodeGenerator-shared-inl.h342
-rw-r--r--js/src/jit/shared/CodeGenerator-shared.cpp983
-rw-r--r--js/src/jit/shared/CodeGenerator-shared.h488
-rw-r--r--js/src/jit/shared/Disassembler-shared.cpp248
-rw-r--r--js/src/jit/shared/Disassembler-shared.h182
-rw-r--r--js/src/jit/shared/IonAssemblerBuffer.h438
-rw-r--r--js/src/jit/shared/IonAssemblerBufferWithConstantPools.h1197
-rw-r--r--js/src/jit/shared/LIR-shared.h4183
-rw-r--r--js/src/jit/shared/Lowering-shared-inl.h890
-rw-r--r--js/src/jit/shared/Lowering-shared.cpp319
-rw-r--r--js/src/jit/shared/Lowering-shared.h371
-rw-r--r--js/src/jit/wasm32/Architecture-wasm32.h174
-rw-r--r--js/src/jit/wasm32/Assembler-wasm32.h228
-rw-r--r--js/src/jit/wasm32/CodeGenerator-wasm32.cpp254
-rw-r--r--js/src/jit/wasm32/CodeGenerator-wasm32.h76
-rw-r--r--js/src/jit/wasm32/LIR-wasm32.h109
-rw-r--r--js/src/jit/wasm32/Lowering-wasm32.h128
-rw-r--r--js/src/jit/wasm32/MacroAssembler-wasm32-inl.h1176
-rw-r--r--js/src/jit/wasm32/MacroAssembler-wasm32.cpp502
-rw-r--r--js/src/jit/wasm32/MacroAssembler-wasm32.h533
-rw-r--r--js/src/jit/wasm32/MoveEmitter-wasm32.h30
-rw-r--r--js/src/jit/wasm32/SharedICHelpers-wasm32-inl.h32
-rw-r--r--js/src/jit/wasm32/SharedICHelpers-wasm32.h30
-rw-r--r--js/src/jit/wasm32/SharedICRegisters-wasm32.h36
-rw-r--r--js/src/jit/wasm32/Trampoline-wasm32.cpp46
-rw-r--r--js/src/jit/x64/Assembler-x64.cpp246
-rw-r--r--js/src/jit/x64/Assembler-x64.h1244
-rw-r--r--js/src/jit/x64/BaseAssembler-x64.h1373
-rw-r--r--js/src/jit/x64/CodeGenerator-x64.cpp984
-rw-r--r--js/src/jit/x64/CodeGenerator-x64.h41
-rw-r--r--js/src/jit/x64/LIR-x64.h170
-rw-r--r--js/src/jit/x64/Lowering-x64.cpp565
-rw-r--r--js/src/jit/x64/Lowering-x64.h70
-rw-r--r--js/src/jit/x64/MacroAssembler-x64-inl.h1099
-rw-r--r--js/src/jit/x64/MacroAssembler-x64.cpp1746
-rw-r--r--js/src/jit/x64/MacroAssembler-x64.h1219
-rw-r--r--js/src/jit/x64/SharedICHelpers-x64-inl.h80
-rw-r--r--js/src/jit/x64/SharedICHelpers-x64.h70
-rw-r--r--js/src/jit/x64/SharedICRegisters-x64.h33
-rw-r--r--js/src/jit/x64/Trampoline-x64.cpp886
-rw-r--r--js/src/jit/x86-shared/Architecture-x86-shared.cpp93
-rw-r--r--js/src/jit/x86-shared/Architecture-x86-shared.h468
-rw-r--r--js/src/jit/x86-shared/Assembler-x86-shared.cpp355
-rw-r--r--js/src/jit/x86-shared/Assembler-x86-shared.h4887
-rw-r--r--js/src/jit/x86-shared/AssemblerBuffer-x86-shared.cpp57
-rw-r--r--js/src/jit/x86-shared/AssemblerBuffer-x86-shared.h256
-rw-r--r--js/src/jit/x86-shared/BaseAssembler-x86-shared.h6460
-rw-r--r--js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp3888
-rw-r--r--js/src/jit/x86-shared/CodeGenerator-x86-shared.h189
-rw-r--r--js/src/jit/x86-shared/Constants-x86-shared.h326
-rw-r--r--js/src/jit/x86-shared/Encoding-x86-shared.h508
-rw-r--r--js/src/jit/x86-shared/LIR-x86-shared.h304
-rw-r--r--js/src/jit/x86-shared/Lowering-x86-shared.cpp1869
-rw-r--r--js/src/jit/x86-shared/Lowering-x86-shared.h78
-rw-r--r--js/src/jit/x86-shared/MacroAssembler-x86-shared-SIMD.cpp1511
-rw-r--r--js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h3403
-rw-r--r--js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp2154
-rw-r--r--js/src/jit/x86-shared/MacroAssembler-x86-shared.h1004
-rw-r--r--js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp528
-rw-r--r--js/src/jit/x86-shared/MoveEmitter-x86-shared.h83
-rw-r--r--js/src/jit/x86-shared/Patching-x86-shared.h113
-rw-r--r--js/src/jit/x86/Assembler-x86.cpp85
-rw-r--r--js/src/jit/x86/Assembler-x86.h1074
-rw-r--r--js/src/jit/x86/BaseAssembler-x86.h190
-rw-r--r--js/src/jit/x86/CodeGenerator-x86.cpp1509
-rw-r--r--js/src/jit/x86/CodeGenerator-x86.h49
-rw-r--r--js/src/jit/x86/LIR-x86.h308
-rw-r--r--js/src/jit/x86/Lowering-x86.cpp840
-rw-r--r--js/src/jit/x86/Lowering-x86.h79
-rw-r--r--js/src/jit/x86/MacroAssembler-x86-inl.h1386
-rw-r--r--js/src/jit/x86/MacroAssembler-x86.cpp1828
-rw-r--r--js/src/jit/x86/MacroAssembler-x86.h1140
-rw-r--r--js/src/jit/x86/SharedICHelpers-x86-inl.h77
-rw-r--r--js/src/jit/x86/SharedICHelpers-x86.h70
-rw-r--r--js/src/jit/x86/SharedICRegisters-x86.h36
-rw-r--r--js/src/jit/x86/Trampoline-x86.cpp794
468 files changed, 424565 insertions, 0 deletions
diff --git a/js/src/jit/ABIArgGenerator.h b/js/src/jit/ABIArgGenerator.h
new file mode 100644
index 0000000000..d78a21e242
--- /dev/null
+++ b/js/src/jit/ABIArgGenerator.h
@@ -0,0 +1,109 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ABIArgGenerator_h
+#define jit_ABIArgGenerator_h
+
+#include "mozilla/Assertions.h"
+
+#include <stdint.h>
+
+#include "jit/Assembler.h"
+#include "jit/IonTypes.h"
+#include "jit/RegisterSets.h"
+#include "wasm/WasmFrame.h"
+
+namespace js::jit {
+
+static inline MIRType ToMIRType(MIRType t) { return t; }
+
+static inline MIRType ToMIRType(ABIArgType argType) {
+ switch (argType) {
+ case ArgType_General:
+ return MIRType::Pointer;
+ case ArgType_Float64:
+ return MIRType::Double;
+ case ArgType_Float32:
+ return MIRType::Float32;
+ case ArgType_Int32:
+ return MIRType::Int32;
+ case ArgType_Int64:
+ return MIRType::Int64;
+ default:
+ break;
+ }
+ MOZ_CRASH("unexpected argType");
+}
+
+template <class VecT, class ABIArgGeneratorT>
+class ABIArgIterBase {
+ ABIArgGeneratorT gen_;
+ const VecT& types_;
+ unsigned i_;
+
+ void settle() {
+ if (!done()) gen_.next(ToMIRType(types_[i_]));
+ }
+
+ public:
+ explicit ABIArgIterBase(const VecT& types) : types_(types), i_(0) {
+ settle();
+ }
+ void operator++(int) {
+ MOZ_ASSERT(!done());
+ i_++;
+ settle();
+ }
+ bool done() const { return i_ == types_.length(); }
+
+ ABIArg* operator->() {
+ MOZ_ASSERT(!done());
+ return &gen_.current();
+ }
+ ABIArg& operator*() {
+ MOZ_ASSERT(!done());
+ return gen_.current();
+ }
+
+ unsigned index() const {
+ MOZ_ASSERT(!done());
+ return i_;
+ }
+ MIRType mirType() const {
+ MOZ_ASSERT(!done());
+ return ToMIRType(types_[i_]);
+ }
+ uint32_t stackBytesConsumedSoFar() const {
+ return gen_.stackBytesConsumedSoFar();
+ }
+};
+
+// This is not an alias because we want to allow class template argument
+// deduction.
+template <class VecT>
+class ABIArgIter : public ABIArgIterBase<VecT, ABIArgGenerator> {
+ public:
+ explicit ABIArgIter(const VecT& types)
+ : ABIArgIterBase<VecT, ABIArgGenerator>(types) {}
+};
+
+class WasmABIArgGenerator : public ABIArgGenerator {
+ public:
+ WasmABIArgGenerator() {
+ increaseStackOffset(wasm::FrameWithInstances::sizeOfInstanceFields());
+ }
+};
+
+template <class VecT>
+class WasmABIArgIter : public ABIArgIterBase<VecT, WasmABIArgGenerator> {
+ public:
+ explicit WasmABIArgIter(const VecT& types)
+ : ABIArgIterBase<VecT, WasmABIArgGenerator>(types) {}
+};
+
+} // namespace js::jit
+
+#endif /* jit_ABIArgGenerator_h */
diff --git a/js/src/jit/ABIFunctionList-inl.h b/js/src/jit/ABIFunctionList-inl.h
new file mode 100644
index 0000000000..82009774a6
--- /dev/null
+++ b/js/src/jit/ABIFunctionList-inl.h
@@ -0,0 +1,251 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ABIFunctionList_inl_h
+#define jit_ABIFunctionList_inl_h
+
+#include "jslibmath.h" // js::NumberMod
+#include "jsmath.h" // js::ecmaPow, js::ecmaHypot, js::hypot3, js::hypot4,
+ // js::ecmaAtan2, js::UnaryMathFunctionType, js::powi
+#include "jsnum.h" // js::StringToNumberPure, js::Int32ToStringPure,
+ // js::NumberToStringPure
+
+#include "builtin/Array.h" // js::ArrayShiftMoveElements
+#include "builtin/MapObject.h" // js::MapIteratorObject::next,
+ // js::SetIteratorObject::next
+#include "builtin/Object.h" // js::ObjectClassToString
+#include "builtin/RegExp.h" // js::RegExpPrototypeOptimizableRaw,
+ // js::RegExpInstanceOptimizableRaw
+#include "builtin/TestingFunctions.h" // js::FuzzilliHash*
+
+#include "irregexp/RegExpAPI.h"
+// js::irregexp::CaseInsensitiveCompareNonUnicode,
+// js::irregexp::CaseInsensitiveCompareUnicode,
+// js::irregexp::GrowBacktrackStack,
+// js::irregexp::IsCharacterInRangeArray
+
+#include "jit/ABIFunctions.h"
+#include "jit/Bailouts.h" // js::jit::FinishBailoutToBaseline, js::jit::Bailout,
+ // js::jit::InvalidationBailout
+
+#include "jit/Ion.h" // js::jit::LazyLinkTopActivation
+#include "jit/JitFrames.h" // HandleException
+#include "jit/VMFunctions.h" // Rest of js::jit::* functions.
+
+#include "js/CallArgs.h" // JSNative
+#include "js/Conversions.h" // JS::ToInt32
+// JSJitGetterOp, JSJitSetterOp, JSJitMethodOp
+#include "js/experimental/JitInfo.h"
+#include "js/Utility.h" // js_free
+
+#include "proxy/Proxy.h" // js::ProxyGetProperty
+
+#include "vm/ArgumentsObject.h" // js::ArgumentsObject::finishForIonPure
+#include "vm/Interpreter.h" // js::TypeOfObject
+#include "vm/NativeObject.h" // js::NativeObject
+#include "vm/RegExpShared.h" // js::ExecuteRegExpAtomRaw
+#include "wasm/WasmBuiltins.h" // js::wasm::*
+
+#include "builtin/Boolean-inl.h" // js::EmulatesUndefined
+
+namespace js {
+namespace jit {
+
+// List of all ABI functions to be used with callWithABI. Each entry stores
+// the fully qualified name of the C++ function. This list must be sorted.
+#if JS_GC_PROBES
+# define ABIFUNCTION_JS_GC_PROBES_LIST(_) _(js::jit::TraceCreateObject)
+#else
+# define ABIFUNCTION_JS_GC_PROBES_LIST(_)
+#endif
+
+#if defined(JS_CODEGEN_ARM)
+# define ABIFUNCTION_JS_CODEGEN_ARM_LIST(_) \
+ _(__aeabi_idivmod) \
+ _(__aeabi_uidivmod)
+#else
+# define ABIFUNCTION_JS_CODEGEN_ARM_LIST(_)
+#endif
+
+#ifdef WASM_CODEGEN_DEBUG
+# define ABIFUNCTION_WASM_CODEGEN_DEBUG_LIST(_) \
+ _(js::wasm::PrintF32) \
+ _(js::wasm::PrintF64) \
+ _(js::wasm::PrintI32) \
+ _(js::wasm::PrintPtr) \
+ _(js::wasm::PrintText)
+#else
+# define ABIFUNCTION_WASM_CODEGEN_DEBUG_LIST(_)
+#endif
+
+#ifdef FUZZING_JS_FUZZILLI
+# define ABIFUNCTION_FUZZILLI_LIST(_) _(js::FuzzilliHashBigInt)
+#else
+# define ABIFUNCTION_FUZZILLI_LIST(_)
+#endif
+
+#define ABIFUNCTION_LIST(_) \
+ ABIFUNCTION_JS_GC_PROBES_LIST(_) \
+ ABIFUNCTION_JS_CODEGEN_ARM_LIST(_) \
+ ABIFUNCTION_WASM_CODEGEN_DEBUG_LIST(_) \
+ _(js::ArgumentsObject::finishForIonPure) \
+ _(js::ArgumentsObject::finishInlineForIonPure) \
+ _(js::ArrayShiftMoveElements) \
+ _(js::ecmaAtan2) \
+ _(js::ecmaHypot) \
+ _(js::ecmaPow) \
+ _(js::EmulatesUndefined) \
+ _(js::ExecuteRegExpAtomRaw) \
+ _(js_free) \
+ _(js::hypot3) \
+ _(js::hypot4) \
+ _(js::Int32ToStringPure) \
+ _(js::irregexp::CaseInsensitiveCompareNonUnicode) \
+ _(js::irregexp::CaseInsensitiveCompareUnicode) \
+ _(js::irregexp::GrowBacktrackStack) \
+ _(js::irregexp::IsCharacterInRangeArray) \
+ _(js::jit::AllocateAndInitTypedArrayBuffer) \
+ _(js::jit::AllocateBigIntNoGC) \
+ _(js::jit::AllocateFatInlineString) \
+ _(js::jit::AllocateDependentString) \
+ _(js::jit::ArrayPushDensePure) \
+ _(js::jit::AssertMapObjectHash) \
+ _(js::jit::AssertSetObjectHash) \
+ _(js::jit::AssertValidBigIntPtr) \
+ _(js::jit::AssertValidObjectPtr) \
+ _(js::jit::AssertValidStringPtr) \
+ _(js::jit::AssertValidSymbolPtr) \
+ _(js::jit::AssertValidValue) \
+ _(js::jit::AssumeUnreachable) \
+ _(js::jit::AtomicsStore64) \
+ _(js::jit::AtomizeStringNoGC) \
+ _(js::jit::Bailout) \
+ _(js::jit::BigIntNumberEqual<EqualityKind::Equal>) \
+ _(js::jit::BigIntNumberEqual<EqualityKind::NotEqual>) \
+ _(js::jit::BigIntNumberCompare<ComparisonKind::LessThan>) \
+ _(js::jit::NumberBigIntCompare<ComparisonKind::LessThan>) \
+ _(js::jit::NumberBigIntCompare<ComparisonKind::GreaterThanOrEqual>) \
+ _(js::jit::BigIntNumberCompare<ComparisonKind::GreaterThanOrEqual>) \
+ _(js::jit::CreateMatchResultFallbackFunc) \
+ _(js::jit::EqualStringsHelperPure) \
+ _(js::jit::FinishBailoutToBaseline) \
+ _(js::jit::FrameIsDebuggeeCheck) \
+ _(js::jit::GetContextSensitiveInterpreterStub) \
+ _(js::jit::GetIndexFromString) \
+ _(js::jit::GetInt32FromStringPure) \
+ _(js::jit::GetNativeDataPropertyByValuePure) \
+ _(js::jit::GetNativeDataPropertyPure) \
+ _(js::jit::GetNativeDataPropertyPureFallback) \
+ _(js::jit::GlobalHasLiveOnDebuggerStatement) \
+ _(js::jit::HandleCodeCoverageAtPC) \
+ _(js::jit::HandleCodeCoverageAtPrologue) \
+ _(js::jit::HandleException) \
+ _(js::jit::HasNativeDataPropertyPure<false>) \
+ _(js::jit::HasNativeDataPropertyPure<true>) \
+ _(js::jit::HasNativeElementPure) \
+ _(js::jit::InitBaselineFrameForOsr) \
+ _(js::jit::InvalidationBailout) \
+ _(js::jit::InvokeFromInterpreterStub) \
+ _(js::jit::LazyLinkTopActivation) \
+ _(js::jit::LinearizeForCharAccessPure) \
+ _(js::jit::ObjectHasGetterSetterPure) \
+ _(js::jit::ObjectIsCallable) \
+ _(js::jit::ObjectIsConstructor) \
+ _(js::jit::PostGlobalWriteBarrier) \
+ _(js::jit::PostWriteBarrier) \
+ _(js::jit::PostWriteElementBarrier<IndexInBounds::Yes>) \
+ _(js::jit::PostWriteElementBarrier<IndexInBounds::Maybe>) \
+ _(js::jit::Printf0) \
+ _(js::jit::Printf1) \
+ _(js::jit::SetNativeDataPropertyPure) \
+ _(js::jit::StringFromCharCodeNoGC) \
+ _(js::jit::TypeOfNameObject) \
+ _(js::jit::WrapObjectPure) \
+ ABIFUNCTION_FUZZILLI_LIST(_) \
+ _(js::MapIteratorObject::next) \
+ _(js::NativeObject::addDenseElementPure) \
+ _(js::NativeObject::growSlotsPure) \
+ _(js::NumberMod) \
+ _(js::NumberToStringPure) \
+ _(js::ObjectClassToString) \
+ _(js::powi) \
+ _(js::ProxyGetProperty) \
+ _(js::RegExpInstanceOptimizableRaw) \
+ _(js::RegExpPrototypeOptimizableRaw) \
+ _(js::SetIteratorObject::next) \
+ _(js::StringToNumberPure) \
+ _(js::TypeOfObject)
+
+// List of all ABI functions to be used with callWithABI, which are
+// overloaded. Each entry stores the fully qualified name of the C++ function,
+// followed by the signature of the function to be called. When the function
+// is not overloaded, you should prefer adding the function to
+// ABIFUNCTION_LIST instead. This list must be sorted with the name of the C++
+// function.
+#define ABIFUNCTION_AND_TYPE_LIST(_) _(JS::ToInt32, int32_t (*)(double))
+
+// List of all ABI function signature which are using a computed function
+// pointer instead of a statically known function pointer.
+#define ABIFUNCTIONSIG_LIST(_) \
+ _(AtomicsCompareExchangeFn) \
+ _(AtomicsReadWriteModifyFn) \
+ _(bool (*)(BigInt*, BigInt*)) \
+ _(bool (*)(BigInt*, double)) \
+ _(bool (*)(double, BigInt*)) \
+ _(float (*)(float)) \
+ _(JSJitGetterOp) \
+ _(JSJitMethodOp) \
+ _(JSJitSetterOp) \
+ _(JSNative) \
+ _(js::UnaryMathFunctionType) \
+ _(void (*)(js::gc::StoreBuffer*, js::gc::Cell**)) \
+ _(void (*)(JSRuntime * rt, JSObject * *objp)) \
+ _(void (*)(JSRuntime * rt, JSString * *stringp)) \
+ _(void (*)(JSRuntime * rt, Shape * *shapep)) \
+ _(void (*)(JSRuntime * rt, Value * vp))
+
+// GCC warns when the signature does not have matching attributes (for example
+// [[nodiscard]]). Squelch this warning to avoid a GCC-only footgun.
+#if MOZ_IS_GCC
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wignored-attributes"
+#endif
+
+// Note: the use of ::fp instead of fp is intentional to enforce use of
+// fully-qualified names in the list above.
+#define DEF_TEMPLATE(fp) \
+ template <> \
+ struct ABIFunctionData<decltype(&(::fp)), ::fp> { \
+ static constexpr bool registered = true; \
+ };
+ABIFUNCTION_LIST(DEF_TEMPLATE)
+#undef DEF_TEMPLATE
+
+#define DEF_TEMPLATE(fp, ...) \
+ template <> \
+ struct ABIFunctionData<__VA_ARGS__, ::fp> { \
+ static constexpr bool registered = true; \
+ };
+ABIFUNCTION_AND_TYPE_LIST(DEF_TEMPLATE)
+#undef DEF_TEMPLATE
+
+// Define a known list of function signatures.
+#define DEF_TEMPLATE(...) \
+ template <> \
+ struct ABIFunctionSignatureData<__VA_ARGS__> { \
+ static constexpr bool registered = true; \
+ };
+ABIFUNCTIONSIG_LIST(DEF_TEMPLATE)
+#undef DEF_TEMPLATE
+
+#if MOZ_IS_GCC
+# pragma GCC diagnostic pop
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_VMFunctionList_inl_h
diff --git a/js/src/jit/ABIFunctions.h b/js/src/jit/ABIFunctions.h
new file mode 100644
index 0000000000..d6bd15555f
--- /dev/null
+++ b/js/src/jit/ABIFunctions.h
@@ -0,0 +1,73 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ABIFunctions_h
+#define jit_ABIFunctions_h
+
+#include "jstypes.h" // JS_FUNC_TO_DATA_PTR
+
+namespace js {
+namespace jit {
+
+// This class is used to ensure that all known targets of callWithABI are
+// registered here. Otherwise, this would raise a static assertion at compile
+// time.
+template <typename Sig, Sig fun>
+struct ABIFunctionData {
+ static const bool registered = false;
+};
+
+template <typename Sig, Sig fun>
+struct ABIFunction {
+ void* address() const { return JS_FUNC_TO_DATA_PTR(void*, fun); }
+
+ // If this assertion fails, you are likely in the context of a
+ // `callWithABI<Sig, fn>()` call. This error indicates that ABIFunction has
+ // not been specialized for `<Sig, fn>` by the time of this call.
+ //
+ // This can be fixed by adding the function signature to either
+ // ABIFUNCTION_LIST or ABIFUNCTION_AND_TYPE_LIST (if overloaded) within
+ // `ABIFunctionList-inl.h` and to add an `#include` statement of this header
+ // in the file which is making the call to `callWithABI<Sig, fn>()`.
+ static_assert(ABIFunctionData<Sig, fun>::registered,
+ "ABI function is not registered.");
+};
+
+template <typename Sig>
+struct ABIFunctionSignatureData {
+ static const bool registered = false;
+};
+
+template <typename Sig>
+struct ABIFunctionSignature {
+ void* address(Sig fun) const { return JS_FUNC_TO_DATA_PTR(void*, fun); }
+
+ // If this assertion fails, you are likely in the context of a
+ // `DynamicFunction<Sig>(fn)` call. This error indicates that
+ // ABIFunctionSignature has not been specialized for `Sig` by the time of this
+ // call.
+ //
+ // This can be fixed by adding the function signature to ABIFUNCTIONSIG_LIST
+ // within `ABIFunctionList-inl.h` and to add an `#include` statement of this
+ // header in the file which is making the call to `DynamicFunction<Sig>(fn)`.
+ static_assert(ABIFunctionSignatureData<Sig>::registered,
+ "ABI function signature is not registered.");
+};
+
+// This is a structure created to ensure that the dynamically computed
+// function pointer is well typed.
+//
+// It is meant to be created only through DynamicFunction function calls. In
+// extremelly rare cases, such as VMFunctions, it might be produced as a result
+// of GetVMFunctionTarget.
+struct DynFn {
+ void* address;
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_VMFunctions_h */
diff --git a/js/src/jit/AliasAnalysis.cpp b/js/src/jit/AliasAnalysis.cpp
new file mode 100644
index 0000000000..b843dae62e
--- /dev/null
+++ b/js/src/jit/AliasAnalysis.cpp
@@ -0,0 +1,322 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/AliasAnalysis.h"
+
+#include "jit/JitSpewer.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+#include "vm/Printer.h"
+
+using namespace js;
+using namespace js::jit;
+
+namespace js {
+namespace jit {
+
+class LoopAliasInfo : public TempObject {
+ private:
+ LoopAliasInfo* outer_;
+ MBasicBlock* loopHeader_;
+ MInstructionVector invariantLoads_;
+
+ public:
+ LoopAliasInfo(TempAllocator& alloc, LoopAliasInfo* outer,
+ MBasicBlock* loopHeader)
+ : outer_(outer), loopHeader_(loopHeader), invariantLoads_(alloc) {}
+
+ MBasicBlock* loopHeader() const { return loopHeader_; }
+ LoopAliasInfo* outer() const { return outer_; }
+ bool addInvariantLoad(MInstruction* ins) {
+ return invariantLoads_.append(ins);
+ }
+ const MInstructionVector& invariantLoads() const { return invariantLoads_; }
+ MInstruction* firstInstruction() const { return *loopHeader_->begin(); }
+};
+
+} // namespace jit
+} // namespace js
+
+void AliasAnalysis::spewDependencyList() {
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_AliasSummaries)) {
+ Fprinter& print = JitSpewPrinter();
+ JitSpewHeader(JitSpew_AliasSummaries);
+ print.printf("Dependency list for other passes:\n");
+
+ for (ReversePostorderIterator block(graph_.rpoBegin());
+ block != graph_.rpoEnd(); block++) {
+ for (MInstructionIterator def(block->begin()),
+ end(block->begin(block->lastIns()));
+ def != end; ++def) {
+ if (!def->dependency()) {
+ continue;
+ }
+ if (!def->getAliasSet().isLoad()) {
+ continue;
+ }
+
+ JitSpewHeader(JitSpew_AliasSummaries);
+ print.printf(" ");
+ MDefinition::PrintOpcodeName(print, def->op());
+ print.printf("%u marked depending on ", def->id());
+ MDefinition::PrintOpcodeName(print, def->dependency()->op());
+ print.printf("%u\n", def->dependency()->id());
+ }
+ }
+ }
+#endif
+}
+
+// Whether there might be a path from src to dest, excluding loop backedges.
+// This is approximate and really ought to depend on precomputed reachability
+// information.
+static inline bool BlockMightReach(MBasicBlock* src, MBasicBlock* dest) {
+ while (src->id() <= dest->id()) {
+ if (src == dest) {
+ return true;
+ }
+ switch (src->numSuccessors()) {
+ case 0:
+ return false;
+ case 1: {
+ MBasicBlock* successor = src->getSuccessor(0);
+ if (successor->id() <= src->id()) {
+ return true; // Don't iloop.
+ }
+ src = successor;
+ break;
+ }
+ default:
+ return true;
+ }
+ }
+ return false;
+}
+
+static void IonSpewDependency(MInstruction* load, MInstruction* store,
+ const char* verb, const char* reason) {
+#ifdef JS_JITSPEW
+ if (!JitSpewEnabled(JitSpew_Alias)) {
+ return;
+ }
+
+ JitSpewHeader(JitSpew_Alias);
+ Fprinter& out = JitSpewPrinter();
+ out.printf(" Load ");
+ load->printName(out);
+ out.printf(" %s on store ", verb);
+ store->printName(out);
+ out.printf(" (%s)\n", reason);
+#endif
+}
+
+static void IonSpewAliasInfo(const char* pre, MInstruction* ins,
+ const char* post) {
+#ifdef JS_JITSPEW
+ if (!JitSpewEnabled(JitSpew_Alias)) {
+ return;
+ }
+
+ JitSpewHeader(JitSpew_Alias);
+ Fprinter& out = JitSpewPrinter();
+ out.printf(" %s ", pre);
+ ins->printName(out);
+ out.printf(" %s\n", post);
+#endif
+}
+
+// [SMDOC] IonMonkey Alias Analysis
+//
+// This pass annotates every load instruction with the last store instruction
+// on which it depends. The algorithm is optimistic in that it ignores explicit
+// dependencies and only considers loads and stores.
+//
+// Loads inside loops only have an implicit dependency on a store before the
+// loop header if no instruction inside the loop body aliases it. To calculate
+// this efficiently, we maintain a list of maybe-invariant loads and the
+// combined alias set for all stores inside the loop. When we see the loop's
+// backedge, this information is used to mark every load we wrongly assumed to
+// be loop invariant as having an implicit dependency on the last instruction of
+// the loop header, so that it's never moved before the loop header.
+//
+// The algorithm depends on the invariant that both control instructions and
+// effectful instructions (stores) are never hoisted.
+bool AliasAnalysis::analyze() {
+ JitSpew(JitSpew_Alias, "Begin");
+ Vector<MInstructionVector, AliasSet::NumCategories, JitAllocPolicy> stores(
+ alloc());
+
+ // Initialize to the first instruction.
+ MInstruction* firstIns = *graph_.entryBlock()->begin();
+ for (unsigned i = 0; i < AliasSet::NumCategories; i++) {
+ MInstructionVector defs(alloc());
+ if (!defs.append(firstIns)) {
+ return false;
+ }
+ if (!stores.append(std::move(defs))) {
+ return false;
+ }
+ }
+
+ // Type analysis may have inserted new instructions. Since this pass depends
+ // on the instruction number ordering, all instructions are renumbered.
+ uint32_t newId = 0;
+
+ for (ReversePostorderIterator block(graph_.rpoBegin());
+ block != graph_.rpoEnd(); block++) {
+ if (mir->shouldCancel("Alias Analysis (main loop)")) {
+ return false;
+ }
+
+ if (block->isLoopHeader()) {
+ JitSpew(JitSpew_Alias, "Processing loop header %u", block->id());
+ loop_ = new (alloc().fallible()) LoopAliasInfo(alloc(), loop_, *block);
+ if (!loop_) {
+ return false;
+ }
+ }
+
+ for (MPhiIterator def(block->phisBegin()), end(block->phisEnd());
+ def != end; ++def) {
+ def->setId(newId++);
+ }
+
+ for (MInstructionIterator def(block->begin()),
+ end(block->begin(block->lastIns()));
+ def != end; ++def) {
+ def->setId(newId++);
+
+ AliasSet set = def->getAliasSet();
+ if (set.isNone()) {
+ continue;
+ }
+
+ // For the purposes of alias analysis, all recoverable operations
+ // are treated as effect free as the memory represented by these
+ // operations cannot be aliased by others.
+ if (def->canRecoverOnBailout()) {
+ continue;
+ }
+
+ if (set.isStore()) {
+ for (AliasSetIterator iter(set); iter; iter++) {
+ if (!stores[*iter].append(*def)) {
+ return false;
+ }
+ }
+
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_Alias)) {
+ JitSpewHeader(JitSpew_Alias);
+ Fprinter& out = JitSpewPrinter();
+ out.printf("Processing store ");
+ def->printName(out);
+ out.printf(" (flags %x)\n", set.flags());
+ }
+#endif
+ } else {
+ // Find the most recent store on which this instruction depends.
+ MInstruction* lastStore = firstIns;
+
+ for (AliasSetIterator iter(set); iter; iter++) {
+ MInstructionVector& aliasedStores = stores[*iter];
+ for (int i = aliasedStores.length() - 1; i >= 0; i--) {
+ MInstruction* store = aliasedStores[i];
+ if (def->mightAlias(store) != MDefinition::AliasType::NoAlias &&
+ BlockMightReach(store->block(), *block)) {
+ if (lastStore->id() < store->id()) {
+ lastStore = store;
+ }
+ break;
+ }
+ }
+ }
+
+ def->setDependency(lastStore);
+ IonSpewDependency(*def, lastStore, "depends", "");
+
+ // If the last store was before the current loop, we assume this load
+ // is loop invariant. If a later instruction writes to the same
+ // location, we will fix this at the end of the loop.
+ if (loop_ && lastStore->id() < loop_->firstInstruction()->id()) {
+ if (!loop_->addInvariantLoad(*def)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ // Renumber the last instruction, as the analysis depends on this and the
+ // order.
+ block->lastIns()->setId(newId++);
+
+ if (block->isLoopBackedge()) {
+ MOZ_ASSERT(loop_->loopHeader() == block->loopHeaderOfBackedge());
+ JitSpew(JitSpew_Alias, "Processing loop backedge %u (header %u)",
+ block->id(), loop_->loopHeader()->id());
+ LoopAliasInfo* outerLoop = loop_->outer();
+ MInstruction* firstLoopIns = *loop_->loopHeader()->begin();
+
+ const MInstructionVector& invariant = loop_->invariantLoads();
+
+ for (unsigned i = 0; i < invariant.length(); i++) {
+ MInstruction* ins = invariant[i];
+ AliasSet set = ins->getAliasSet();
+ MOZ_ASSERT(set.isLoad());
+
+ bool hasAlias = false;
+ for (AliasSetIterator iter(set); iter; iter++) {
+ MInstructionVector& aliasedStores = stores[*iter];
+ for (int i = aliasedStores.length() - 1;; i--) {
+ MInstruction* store = aliasedStores[i];
+ if (store->id() < firstLoopIns->id()) {
+ break;
+ }
+ if (ins->mightAlias(store) != MDefinition::AliasType::NoAlias) {
+ hasAlias = true;
+ IonSpewDependency(ins, store, "aliases", "store in loop body");
+ break;
+ }
+ }
+ if (hasAlias) {
+ break;
+ }
+ }
+
+ if (hasAlias) {
+ // This instruction depends on stores inside the loop body. Mark it as
+ // having a dependency on the last instruction of the loop header. The
+ // last instruction is a control instruction and these are never
+ // hoisted.
+ MControlInstruction* controlIns = loop_->loopHeader()->lastIns();
+ IonSpewDependency(ins, controlIns, "depends",
+ "due to stores in loop body");
+ ins->setDependency(controlIns);
+ } else {
+ IonSpewAliasInfo("Load", ins,
+ "does not depend on any stores in this loop");
+
+ if (outerLoop &&
+ ins->dependency()->id() < outerLoop->firstInstruction()->id()) {
+ IonSpewAliasInfo("Load", ins, "may be invariant in outer loop");
+ if (!outerLoop->addInvariantLoad(ins)) {
+ return false;
+ }
+ }
+ }
+ }
+ loop_ = loop_->outer();
+ }
+ }
+
+ spewDependencyList();
+
+ MOZ_ASSERT(loop_ == nullptr);
+ return true;
+}
diff --git a/js/src/jit/AliasAnalysis.h b/js/src/jit/AliasAnalysis.h
new file mode 100644
index 0000000000..49ddaee47c
--- /dev/null
+++ b/js/src/jit/AliasAnalysis.h
@@ -0,0 +1,64 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_AliasAnalysis_h
+#define jit_AliasAnalysis_h
+
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+
+namespace js {
+namespace jit {
+
+class LoopAliasInfo;
+
+class AliasAnalysis {
+ MIRGenerator* mir;
+ MIRGraph& graph_;
+ LoopAliasInfo* loop_;
+
+ void spewDependencyList();
+
+ TempAllocator& alloc() const { return graph_.alloc(); }
+
+ public:
+ AliasAnalysis(MIRGenerator* mir, MIRGraph& graph)
+ : mir(mir), graph_(graph), loop_(nullptr) {}
+
+ [[nodiscard]] bool analyze();
+};
+
+// Iterates over the flags in an AliasSet.
+class AliasSetIterator {
+ private:
+ uint32_t flags;
+ unsigned pos;
+
+ public:
+ explicit AliasSetIterator(AliasSet set) : flags(set.flags()), pos(0) {
+ while (flags && (flags & 1) == 0) {
+ flags >>= 1;
+ pos++;
+ }
+ }
+ AliasSetIterator& operator++(int) {
+ do {
+ flags >>= 1;
+ pos++;
+ } while (flags && (flags & 1) == 0);
+ return *this;
+ }
+ explicit operator bool() const { return !!flags; }
+ unsigned operator*() const {
+ MOZ_ASSERT(pos < AliasSet::NumCategories);
+ return pos;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_AliasAnalysis_h */
diff --git a/js/src/jit/AlignmentMaskAnalysis.cpp b/js/src/jit/AlignmentMaskAnalysis.cpp
new file mode 100644
index 0000000000..5b19b0861c
--- /dev/null
+++ b/js/src/jit/AlignmentMaskAnalysis.cpp
@@ -0,0 +1,97 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/AlignmentMaskAnalysis.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+
+using namespace js;
+using namespace jit;
+
+static bool IsAlignmentMask(uint32_t m) {
+ // Test whether m is just leading ones and trailing zeros.
+ return (-m & ~m) == 0;
+}
+
+static void AnalyzeAsmHeapAddress(MDefinition* ptr, MIRGraph& graph) {
+ // Fold (a+i)&m to (a&m)+i, provided that this doesn't change the result,
+ // since the users of the BitAnd include heap accesses. This will expose
+ // the redundancy for GVN when expressions like this:
+ // a&m
+ // (a+1)&m,
+ // (a+2)&m,
+ // are transformed into this:
+ // a&m
+ // (a&m)+1
+ // (a&m)+2
+ // and it will allow the constants to be folded by the
+ // EffectiveAddressAnalysis pass.
+ //
+ // Putting the add on the outside might seem like it exposes other users of
+ // the expression to the possibility of i32 overflow, if we aren't in wasm
+ // and they aren't naturally truncating. However, since we use MAdd::New
+ // with MIRType::Int32, we make sure that the value is truncated, just as it
+ // would be by the MBitAnd.
+
+ MOZ_ASSERT(IsCompilingWasm());
+
+ if (!ptr->isBitAnd()) {
+ return;
+ }
+
+ MDefinition* lhs = ptr->toBitAnd()->getOperand(0);
+ MDefinition* rhs = ptr->toBitAnd()->getOperand(1);
+ if (lhs->isConstant()) {
+ std::swap(lhs, rhs);
+ }
+ if (!lhs->isAdd() || !rhs->isConstant()) {
+ return;
+ }
+
+ MDefinition* op0 = lhs->toAdd()->getOperand(0);
+ MDefinition* op1 = lhs->toAdd()->getOperand(1);
+ if (op0->isConstant()) {
+ std::swap(op0, op1);
+ }
+ if (!op1->isConstant()) {
+ return;
+ }
+
+ uint32_t i = op1->toConstant()->toInt32();
+ uint32_t m = rhs->toConstant()->toInt32();
+ if (!IsAlignmentMask(m) || (i & m) != i) {
+ return;
+ }
+
+ // The pattern was matched! Produce the replacement expression.
+ MInstruction* and_ = MBitAnd::New(graph.alloc(), op0, rhs, MIRType::Int32);
+ ptr->block()->insertBefore(ptr->toBitAnd(), and_);
+ auto* add = MAdd::New(graph.alloc(), and_, op1, TruncateKind::Truncate);
+ ptr->block()->insertBefore(ptr->toBitAnd(), add);
+ ptr->replaceAllUsesWith(add);
+ ptr->block()->discard(ptr->toBitAnd());
+}
+
+bool AlignmentMaskAnalysis::analyze() {
+ for (ReversePostorderIterator block(graph_.rpoBegin());
+ block != graph_.rpoEnd(); block++) {
+ for (MInstructionIterator i = block->begin(); i != block->end(); i++) {
+ if (!graph_.alloc().ensureBallast()) {
+ return false;
+ }
+
+ // Note that we don't check for MWasmCompareExchangeHeap
+ // or MWasmAtomicBinopHeap, because the backend and the OOB
+ // mechanism don't support non-zero offsets for them yet.
+ if (i->isAsmJSLoadHeap()) {
+ AnalyzeAsmHeapAddress(i->toAsmJSLoadHeap()->base(), graph_);
+ } else if (i->isAsmJSStoreHeap()) {
+ AnalyzeAsmHeapAddress(i->toAsmJSStoreHeap()->base(), graph_);
+ }
+ }
+ }
+ return true;
+}
diff --git a/js/src/jit/AlignmentMaskAnalysis.h b/js/src/jit/AlignmentMaskAnalysis.h
new file mode 100644
index 0000000000..4e46ca97d3
--- /dev/null
+++ b/js/src/jit/AlignmentMaskAnalysis.h
@@ -0,0 +1,27 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_AlignmentMaskAnalysis_h
+#define jit_AlignmentMaskAnalysis_h
+
+namespace js {
+namespace jit {
+
+class MIRGraph;
+
+class AlignmentMaskAnalysis {
+ MIRGraph& graph_;
+
+ public:
+ explicit AlignmentMaskAnalysis(MIRGraph& graph) : graph_(graph) {}
+
+ [[nodiscard]] bool analyze();
+};
+
+} /* namespace jit */
+} /* namespace js */
+
+#endif /* jit_AlignmentMaskAnalysis_h */
diff --git a/js/src/jit/Assembler.h b/js/src/jit/Assembler.h
new file mode 100644
index 0000000000..2cc432491c
--- /dev/null
+++ b/js/src/jit/Assembler.h
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Assembler_h
+#define jit_Assembler_h
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/Assembler-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/Assembler-x64.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/Assembler-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/Assembler-arm64.h"
+#elif defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/Assembler-mips32.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/Assembler-mips64.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/Assembler-loong64.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/Assembler-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/Assembler-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+#endif /* jit_Assembler_h */
diff --git a/js/src/jit/AtomicOp.h b/js/src/jit/AtomicOp.h
new file mode 100644
index 0000000000..90edb631cb
--- /dev/null
+++ b/js/src/jit/AtomicOp.h
@@ -0,0 +1,98 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_AtomicOp_h
+#define jit_AtomicOp_h
+
+namespace js {
+namespace jit {
+
+// Types of atomic operation, shared by MIR and LIR.
+
+enum AtomicOp {
+ AtomicFetchAddOp,
+ AtomicFetchSubOp,
+ AtomicFetchAndOp,
+ AtomicFetchOrOp,
+ AtomicFetchXorOp
+};
+
+// Memory barrier types, shared by MIR and LIR.
+//
+// MembarSynchronizing is here because some platforms can make the
+// distinction (DSB vs DMB on ARM, SYNC vs parameterized SYNC on MIPS)
+// but there's been no reason to use it yet.
+
+enum MemoryBarrierBits {
+ MembarLoadLoad = 1,
+ MembarLoadStore = 2,
+ MembarStoreStore = 4,
+ MembarStoreLoad = 8,
+
+ MembarSynchronizing = 16,
+
+ // For validity testing
+ MembarNobits = 0,
+ MembarAllbits = 31,
+};
+
+static inline constexpr MemoryBarrierBits operator|(MemoryBarrierBits a,
+ MemoryBarrierBits b) {
+ return MemoryBarrierBits(int(a) | int(b));
+}
+
+static inline constexpr MemoryBarrierBits operator&(MemoryBarrierBits a,
+ MemoryBarrierBits b) {
+ return MemoryBarrierBits(int(a) & int(b));
+}
+
+static inline constexpr MemoryBarrierBits operator~(MemoryBarrierBits a) {
+ return MemoryBarrierBits(~int(a));
+}
+
+// Standard barrier bits for a full barrier.
+static constexpr MemoryBarrierBits MembarFull =
+ MembarLoadLoad | MembarLoadStore | MembarStoreLoad | MembarStoreStore;
+
+// Standard sets of barrier bits for atomic loads and stores.
+// See http://gee.cs.oswego.edu/dl/jmm/cookbook.html for more.
+static constexpr MemoryBarrierBits MembarBeforeLoad = MembarNobits;
+static constexpr MemoryBarrierBits MembarAfterLoad =
+ MembarLoadLoad | MembarLoadStore;
+static constexpr MemoryBarrierBits MembarBeforeStore = MembarStoreStore;
+static constexpr MemoryBarrierBits MembarAfterStore = MembarStoreLoad;
+
+struct Synchronization {
+ const MemoryBarrierBits barrierBefore;
+ const MemoryBarrierBits barrierAfter;
+
+ constexpr Synchronization(MemoryBarrierBits before, MemoryBarrierBits after)
+ : barrierBefore(before), barrierAfter(after) {}
+
+ static Synchronization None() {
+ return Synchronization(MemoryBarrierBits(MembarNobits),
+ MemoryBarrierBits(MembarNobits));
+ }
+
+ static Synchronization Full() {
+ return Synchronization(MembarFull, MembarFull);
+ }
+
+ static Synchronization Load() {
+ return Synchronization(MembarBeforeLoad, MembarAfterLoad);
+ }
+
+ static Synchronization Store() {
+ return Synchronization(MembarBeforeStore, MembarAfterStore);
+ }
+
+ bool isNone() const { return (barrierBefore | barrierAfter) == MembarNobits; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_AtomicOp_h */
diff --git a/js/src/jit/AtomicOperations.h b/js/src/jit/AtomicOperations.h
new file mode 100644
index 0000000000..8ad2839b36
--- /dev/null
+++ b/js/src/jit/AtomicOperations.h
@@ -0,0 +1,352 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_AtomicOperations_h
+#define jit_AtomicOperations_h
+
+#include "mozilla/Types.h"
+
+#include <string.h>
+
+#include "jit/AtomicOperationsGenerated.h"
+#include "vm/SharedMem.h"
+
+namespace js {
+namespace jit {
+
+/*
+ * [SMDOC] Atomic Operations
+ *
+ * The atomic operations layer defines types and functions for
+ * JIT-compatible atomic operation.
+ *
+ * The fundamental constraints on the functions are:
+ *
+ * - That their realization here MUST be compatible with code the JIT
+ * generates for its Atomics operations, so that an atomic access
+ * from the interpreter or runtime - from any C++ code - really is
+ * atomic relative to a concurrent, compatible atomic access from
+ * jitted code. That is, these primitives expose JIT-compatible
+ * atomicity functionality to C++.
+ *
+ * - That accesses may race without creating C++ undefined behavior:
+ * atomic accesses (marked "SeqCst") may race with non-atomic
+ * accesses (marked "SafeWhenRacy"); overlapping but non-matching,
+ * and hence incompatible, atomic accesses may race; and non-atomic
+ * accesses may race. The effects of races need not be predictable,
+ * so garbage can be produced by a read or written by a write, but
+ * the effects must be benign: the program must continue to run, and
+ * only the memory in the union of addresses named in the racing
+ * accesses may be affected.
+ *
+ * The compatibility constraint means that if the JIT makes dynamic
+ * decisions about how to implement atomic operations then
+ * corresponding dynamic decisions MUST be made in the implementations
+ * of the functions below.
+ *
+ * The safe-for-races constraint means that by and large, it is hard
+ * to implement these primitives in C++. See "Implementation notes"
+ * below.
+ *
+ * The "SeqCst" suffix on operations means "sequentially consistent"
+ * and means such a function's operation must have "sequentially
+ * consistent" memory ordering. See mfbt/Atomics.h for an explanation
+ * of this memory ordering.
+ *
+ * Note that a "SafeWhenRacy" access does not provide the atomicity of
+ * a "relaxed atomic" access: it can read or write garbage if there's
+ * a race.
+ *
+ *
+ * Implementation notes.
+ *
+ * It's not a requirement that these functions be inlined; performance
+ * is not a great concern. On some platforms these functions may call
+ * functions that use inline assembly. See GenerateAtomicOperations.py.
+ *
+ * In principle these functions will not be written in C++, thus
+ * making races defined behavior if all racy accesses from C++ go via
+ * these functions. (Jitted code will always be safe for races and
+ * provides the same guarantees as these functions.)
+ *
+ * The appropriate implementations will be platform-specific and
+ * there are some obvious implementation strategies to choose
+ * from, sometimes a combination is appropriate:
+ *
+ * - generating the code at run-time with the JIT;
+ * - hand-written assembler (maybe inline); or
+ * - using special compiler intrinsics or directives.
+ *
+ * Trusting the compiler not to generate code that blows up on a
+ * race definitely won't work in the presence of TSan, or even of
+ * optimizing compilers in seemingly-"innocuous" conditions. (See
+ * https://www.usenix.org/legacy/event/hotpar11/tech/final_files/Boehm.pdf
+ * for details.)
+ */
+class AtomicOperations {
+ // The following functions are defined for T = int8_t, uint8_t,
+ // int16_t, uint16_t, int32_t, uint32_t, int64_t, and uint64_t.
+
+ // Atomically read *addr.
+ template <typename T>
+ static inline T loadSeqCst(T* addr);
+
+ // Atomically store val in *addr.
+ template <typename T>
+ static inline void storeSeqCst(T* addr, T val);
+
+ // Atomically store val in *addr and return the old value of *addr.
+ template <typename T>
+ static inline T exchangeSeqCst(T* addr, T val);
+
+ // Atomically check that *addr contains oldval and if so replace it
+ // with newval, in any case returning the old contents of *addr.
+ template <typename T>
+ static inline T compareExchangeSeqCst(T* addr, T oldval, T newval);
+
+ // Atomically add, subtract, bitwise-AND, bitwise-OR, or bitwise-XOR
+ // val into *addr and return the old value of *addr.
+ template <typename T>
+ static inline T fetchAddSeqCst(T* addr, T val);
+
+ template <typename T>
+ static inline T fetchSubSeqCst(T* addr, T val);
+
+ template <typename T>
+ static inline T fetchAndSeqCst(T* addr, T val);
+
+ template <typename T>
+ static inline T fetchOrSeqCst(T* addr, T val);
+
+ template <typename T>
+ static inline T fetchXorSeqCst(T* addr, T val);
+
+ // The SafeWhenRacy functions are to be used when C++ code has to access
+ // memory without synchronization and can't guarantee that there won't be a
+ // race on the access. But they are access-atomic for integer data so long
+ // as any racing writes are of the same size and to the same address.
+
+ // Defined for all the integral types as well as for float32 and float64,
+ // but not access-atomic for floats, nor for int64 and uint64 on 32-bit
+ // platforms.
+ template <typename T>
+ static inline T loadSafeWhenRacy(T* addr);
+
+ // Defined for all the integral types as well as for float32 and float64,
+ // but not access-atomic for floats, nor for int64 and uint64 on 32-bit
+ // platforms.
+ template <typename T>
+ static inline void storeSafeWhenRacy(T* addr, T val);
+
+ // Replacement for memcpy(). No access-atomicity guarantees.
+ static inline void memcpySafeWhenRacy(void* dest, const void* src,
+ size_t nbytes);
+
+ // Replacement for memmove(). No access-atomicity guarantees.
+ static inline void memmoveSafeWhenRacy(void* dest, const void* src,
+ size_t nbytes);
+
+ public:
+ // Test lock-freedom for any int32 value. This implements the
+ // Atomics::isLockFree() operation in the ECMAScript Shared Memory and
+ // Atomics specification, as follows:
+ //
+ // 4-byte accesses are always lock free (in the spec).
+ // 1-, 2-, and 8-byte accesses are always lock free (in SpiderMonkey).
+ //
+ // There is no lock-freedom for JS for any other values on any platform.
+ static constexpr inline bool isLockfreeJS(int32_t n);
+
+ // If the return value is true then the templated functions below are
+ // supported for int64_t and uint64_t. If the return value is false then
+ // those functions will MOZ_CRASH. The value of this call does not change
+ // during execution.
+ static inline bool hasAtomic8();
+
+ // If the return value is true then hasAtomic8() is true and the atomic
+ // operations are indeed lock-free. The value of this call does not change
+ // during execution.
+ static inline bool isLockfree8();
+
+ // Execute a full memory barrier (LoadLoad+LoadStore+StoreLoad+StoreStore).
+ static inline void fenceSeqCst();
+
+ // All clients should use the APIs that take SharedMem pointers.
+ // See above for semantics and acceptable types.
+
+ template <typename T>
+ static T loadSeqCst(SharedMem<T*> addr) {
+ return loadSeqCst(addr.unwrap());
+ }
+
+ template <typename T>
+ static void storeSeqCst(SharedMem<T*> addr, T val) {
+ return storeSeqCst(addr.unwrap(), val);
+ }
+
+ template <typename T>
+ static T exchangeSeqCst(SharedMem<T*> addr, T val) {
+ return exchangeSeqCst(addr.unwrap(), val);
+ }
+
+ template <typename T>
+ static T compareExchangeSeqCst(SharedMem<T*> addr, T oldval, T newval) {
+ return compareExchangeSeqCst(addr.unwrap(), oldval, newval);
+ }
+
+ template <typename T>
+ static T fetchAddSeqCst(SharedMem<T*> addr, T val) {
+ return fetchAddSeqCst(addr.unwrap(), val);
+ }
+
+ template <typename T>
+ static T fetchSubSeqCst(SharedMem<T*> addr, T val) {
+ return fetchSubSeqCst(addr.unwrap(), val);
+ }
+
+ template <typename T>
+ static T fetchAndSeqCst(SharedMem<T*> addr, T val) {
+ return fetchAndSeqCst(addr.unwrap(), val);
+ }
+
+ template <typename T>
+ static T fetchOrSeqCst(SharedMem<T*> addr, T val) {
+ return fetchOrSeqCst(addr.unwrap(), val);
+ }
+
+ template <typename T>
+ static T fetchXorSeqCst(SharedMem<T*> addr, T val) {
+ return fetchXorSeqCst(addr.unwrap(), val);
+ }
+
+ template <typename T>
+ static T loadSafeWhenRacy(SharedMem<T*> addr) {
+ return loadSafeWhenRacy(addr.unwrap());
+ }
+
+ template <typename T>
+ static void storeSafeWhenRacy(SharedMem<T*> addr, T val) {
+ return storeSafeWhenRacy(addr.unwrap(), val);
+ }
+
+ template <typename T>
+ static void memcpySafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src,
+ size_t nbytes) {
+ memcpySafeWhenRacy(dest.template cast<void*>().unwrap(),
+ src.template cast<void*>().unwrap(), nbytes);
+ }
+
+ template <typename T>
+ static void memcpySafeWhenRacy(SharedMem<T*> dest, T* src, size_t nbytes) {
+ memcpySafeWhenRacy(dest.template cast<void*>().unwrap(),
+ static_cast<void*>(src), nbytes);
+ }
+
+ template <typename T>
+ static void memcpySafeWhenRacy(T* dest, SharedMem<T*> src, size_t nbytes) {
+ memcpySafeWhenRacy(static_cast<void*>(dest),
+ src.template cast<void*>().unwrap(), nbytes);
+ }
+
+ template <typename T>
+ static void memmoveSafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src,
+ size_t nbytes) {
+ memmoveSafeWhenRacy(dest.template cast<void*>().unwrap(),
+ src.template cast<void*>().unwrap(), nbytes);
+ }
+
+ static void memsetSafeWhenRacy(SharedMem<uint8_t*> dest, int value,
+ size_t nbytes) {
+ uint8_t buf[1024];
+ size_t iterations = nbytes / sizeof(buf);
+ size_t tail = nbytes % sizeof(buf);
+ size_t offs = 0;
+ if (iterations > 0) {
+ memset(buf, value, sizeof(buf));
+ while (iterations--) {
+ memcpySafeWhenRacy(dest + offs, SharedMem<uint8_t*>::unshared(buf),
+ sizeof(buf));
+ offs += sizeof(buf);
+ }
+ } else {
+ memset(buf, value, tail);
+ }
+ memcpySafeWhenRacy(dest + offs, SharedMem<uint8_t*>::unshared(buf), tail);
+ }
+
+ template <typename T>
+ static void podCopySafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src,
+ size_t nelem) {
+ memcpySafeWhenRacy(dest, src, nelem * sizeof(T));
+ }
+
+ template <typename T>
+ static void podMoveSafeWhenRacy(SharedMem<T*> dest, SharedMem<T*> src,
+ size_t nelem) {
+ memmoveSafeWhenRacy(dest, src, nelem * sizeof(T));
+ }
+};
+
+constexpr inline bool AtomicOperations::isLockfreeJS(int32_t size) {
+ // Keep this in sync with atomicIsLockFreeJS() in jit/MacroAssembler.cpp.
+
+ switch (size) {
+ case 1:
+ return true;
+ case 2:
+ return true;
+ case 4:
+ // The spec requires Atomics.isLockFree(4) to return true.
+ return true;
+ case 8:
+ return true;
+ default:
+ return false;
+ }
+}
+
+} // namespace jit
+} // namespace js
+
+// As explained above, our atomic operations are not portable even in principle,
+// so we must include platform+compiler specific definitions here.
+//
+// x86, x64, arm, and arm64 are maintained by Mozilla. All other platform
+// setups are by platform maintainers' request and are not maintained by
+// Mozilla.
+//
+// If you are using a platform+compiler combination that causes an error below
+// (and if the problem isn't just that the compiler uses a different name for a
+// known architecture), you have basically three options:
+//
+// - find an already-supported compiler for the platform and use that instead
+//
+// - write your own support code for the platform+compiler and create a new
+// case below
+//
+// - include jit/shared/AtomicOperations-feeling-lucky.h in a case for the
+// platform below, if you have a gcc-compatible compiler and truly feel
+// lucky. You may have to add a little code to that file, too.
+//
+// Simulators are confusing. These atomic primitives must be compatible with
+// the code that the JIT emits, but of course for an ARM simulator running on
+// x86 the primitives here will be for x86, not for ARM, while the JIT emits ARM
+// code. Our ARM simulator solves that the easy way: by using these primitives
+// to implement its atomic operations. For other simulators there may need to
+// be special cases below to provide simulator-compatible primitives, for
+// example, for our ARM64 simulator the primitives could in principle
+// participate in the memory exclusivity monitors implemented by the simulator.
+// Such a solution is likely to be difficult.
+
+#ifdef JS_HAVE_GENERATED_ATOMIC_OPS
+# include "jit/shared/AtomicOperations-shared-jit.h"
+#elif defined(JS_SIMULATOR_MIPS32) || defined(__mips__)
+# include "jit/mips-shared/AtomicOperations-mips-shared.h"
+#else
+# include "jit/shared/AtomicOperations-feeling-lucky.h"
+#endif
+
+#endif // jit_AtomicOperations_h
diff --git a/js/src/jit/AutoWritableJitCode.h b/js/src/jit/AutoWritableJitCode.h
new file mode 100644
index 0000000000..ab5b35a54f
--- /dev/null
+++ b/js/src/jit/AutoWritableJitCode.h
@@ -0,0 +1,88 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_AutoWritableJitCode_h
+#define jit_AutoWritableJitCode_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/TimeStamp.h"
+
+#include <stddef.h>
+
+#include "jit/ExecutableAllocator.h"
+#include "jit/JitCode.h"
+#include "jit/ProcessExecutableMemory.h"
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+#include "vm/Runtime.h"
+
+namespace js::jit {
+
+// This class ensures JIT code is executable on its destruction. Creators
+// must call makeWritable(), and not attempt to write to the buffer if it fails.
+//
+// AutoWritableJitCodeFallible may only fail to make code writable; it cannot
+// fail to make JIT code executable (because the creating code has no chance to
+// recover from a failed destructor).
+class MOZ_RAII AutoWritableJitCodeFallible {
+ JSRuntime* rt_;
+ void* addr_;
+ size_t size_;
+
+ public:
+ AutoWritableJitCodeFallible(JSRuntime* rt, void* addr, size_t size)
+ : rt_(rt), addr_(addr), size_(size) {
+ rt_->toggleAutoWritableJitCodeActive(true);
+ }
+
+ AutoWritableJitCodeFallible(void* addr, size_t size)
+ : AutoWritableJitCodeFallible(TlsContext.get()->runtime(), addr, size) {}
+
+ explicit AutoWritableJitCodeFallible(JitCode* code)
+ : AutoWritableJitCodeFallible(code->runtimeFromMainThread(), code->raw(),
+ code->bufferSize()) {}
+
+ [[nodiscard]] bool makeWritable() {
+ return ExecutableAllocator::makeWritable(addr_, size_);
+ }
+
+ ~AutoWritableJitCodeFallible() {
+ mozilla::TimeStamp startTime = mozilla::TimeStamp::Now();
+ auto timer = mozilla::MakeScopeExit([&] {
+ if (Realm* realm = rt_->mainContextFromOwnThread()->realm()) {
+ realm->timers.protectTime += mozilla::TimeStamp::Now() - startTime;
+ }
+ });
+
+ if (!ExecutableAllocator::makeExecutableAndFlushICache(addr_, size_)) {
+ MOZ_CRASH();
+ }
+ rt_->toggleAutoWritableJitCodeActive(false);
+ }
+};
+
+// Infallible variant of AutoWritableJitCodeFallible, ensures writable during
+// construction
+class MOZ_RAII AutoWritableJitCode : private AutoWritableJitCodeFallible {
+ public:
+ AutoWritableJitCode(JSRuntime* rt, void* addr, size_t size)
+ : AutoWritableJitCodeFallible(rt, addr, size) {
+ MOZ_RELEASE_ASSERT(makeWritable());
+ }
+
+ AutoWritableJitCode(void* addr, size_t size)
+ : AutoWritableJitCode(TlsContext.get()->runtime(), addr, size) {}
+
+ explicit AutoWritableJitCode(JitCode* code)
+ : AutoWritableJitCode(code->runtimeFromMainThread(), code->raw(),
+ code->bufferSize()) {}
+};
+
+} // namespace js::jit
+
+#endif /* jit_AutoWritableJitCode_h */
diff --git a/js/src/jit/BacktrackingAllocator.cpp b/js/src/jit/BacktrackingAllocator.cpp
new file mode 100644
index 0000000000..30273fa85c
--- /dev/null
+++ b/js/src/jit/BacktrackingAllocator.cpp
@@ -0,0 +1,4660 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Documentation. Code starts about 670 lines down from here. //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+// [SMDOC] An overview of Ion's register allocator
+//
+// The intent of this documentation is to give maintainers a map with which to
+// navigate the allocator. As further understanding is obtained, it should be
+// added to this overview.
+//
+// Where possible, invariants are stated and are marked "(INVAR)". Many
+// details are omitted because their workings are currently unknown. In
+// particular, this overview doesn't explain how Intel-style "modify" (tied)
+// operands are handled. Facts or invariants that are speculative -- believed
+// to be true, but not verified at the time of writing -- are marked "(SPEC)".
+//
+// The various concepts are interdependent, so a single forwards reading of the
+// following won't make much sense. Many concepts are explained only after
+// they are mentioned.
+//
+// Where possible examples are shown. Without those the description is
+// excessively abstract.
+//
+// Names of the form ::name mean BacktrackingAllocator::name.
+//
+// The description falls into two sections:
+//
+// * Section 1: A tour of the data structures
+// * Section 2: The core allocation loop, and bundle splitting
+//
+// The allocator sometimes produces poor allocations, with excessive spilling
+// and register-to-register moves (bugs 1752520, bug 1714280 bug 1746596).
+// Work in bug 1752582 shows we can get better quality allocations from this
+// framework without having to make any large (conceptual) changes, by having
+// better splitting heuristics.
+//
+// At https://bugzilla.mozilla.org/show_bug.cgi?id=1758274#c17
+// (https://bugzilla.mozilla.org/attachment.cgi?id=9288467) is a document
+// written at the same time as these comments. It describes some improvements
+// we could make to our splitting heuristics, particularly in the presence of
+// loops and calls, and shows why the current implementation sometimes produces
+// excessive spilling. It builds on the commentary in this SMDOC.
+//
+//
+// Top level pipeline
+// ~~~~~~~~~~~~~~~~~~
+// There are three major phases in allocation. They run sequentially, at a
+// per-function granularity.
+//
+// (1) Liveness analysis and bundle formation
+// (2) Bundle allocation and last-chance allocation
+// (3) Rewriting the function to create MoveGroups and to "install"
+// the allocation
+//
+// The input language (LIR) is in SSA form, and phases (1) and (3) depend on
+// that SSAness. Without it the allocator wouldn't work.
+//
+// The top level function is ::go. The phases are divided into functions as
+// follows:
+//
+// (1) ::buildLivenessInfo, ::mergeAndQueueRegisters
+// (2) ::processBundle, ::tryAllocatingRegistersForSpillBundles,
+// ::pickStackSlots
+// (3) ::createMoveGroupsFromLiveRangeTransitions, ::installAllocationsInLIR,
+// ::populateSafepoints, ::annotateMoveGroups
+//
+// The code in this file is structured as much as possible in the same sequence
+// as flow through the pipeline. Hence, top level function ::go is right at
+// the end. Where a function depends on helper function(s), the helpers appear
+// first.
+//
+//
+// ========================================================================
+// ==== ====
+// ==== Section 1: A tour of the data structures ====
+// ==== ====
+// ========================================================================
+//
+// Here are the key data structures necessary for understanding what follows.
+//
+// Some basic data structures
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// CodePosition
+// ------------
+// A CodePosition is an unsigned 32-bit int that indicates an instruction index
+// in the incoming LIR. Each LIR actually has two code positions, one to
+// denote the "input" point (where, one might imagine, the operands are read,
+// at least useAtStart ones) and the "output" point, where operands are
+// written. Eg:
+//
+// Block 0 [successor 2] [successor 1]
+// 2-3 WasmParameter [def v1<g>:r14]
+// 4-5 WasmCall [use v1:F:r14]
+// 6-7 WasmLoadTls [def v2<o>] [use v1:R]
+// 8-9 WasmNullConstant [def v3<o>]
+// 10-11 Compare [def v4<i>] [use v2:R] [use v3:A]
+// 12-13 TestIAndBranch [use v4:R]
+//
+// So for example the WasmLoadTls insn has its input CodePosition as 6 and
+// output point as 7. Input points are even numbered, output points are odd
+// numbered. CodePositions 0 and 1 never appear, because LIR instruction IDs
+// start at 1. Indeed, CodePosition 0 is assumed to be invalid and hence is
+// used as a marker for "unusual conditions" in various places.
+//
+// Phi nodes exist in the instruction stream too. They always appear at the
+// start of blocks (of course) (SPEC), but their start and end points are
+// printed for the group as a whole. This is to emphasise that they are really
+// parallel assignments and that printing them sequentially would misleadingly
+// imply that they are executed sequentially. Example:
+//
+// Block 6 [successor 7] [successor 8]
+// 56-59 Phi [def v19<o>] [use v2:A] [use v5:A] [use v13:A]
+// 56-59 Phi [def v20<o>] [use v7:A] [use v14:A] [use v12:A]
+// 60-61 WasmLoadSlot [def v21<o>] [use v1:R]
+// 62-63 Compare [def v22<i>] [use v20:R] [use v21:A]
+// 64-65 TestIAndBranch [use v22:R]
+//
+// See that both Phis are printed with limits 56-59, even though they are
+// stored in the LIR array like regular LIRs and so have code points 56-57 and
+// 58-59 in reality.
+//
+// The process of allocation adds MoveGroup LIRs to the function. Each
+// incoming LIR has its own private list of MoveGroups (actually, 3 lists; two
+// for moves that conceptually take place before the instruction, and one for
+// moves after it). Hence the CodePositions for LIRs (the "62-63", etc, above)
+// do not change as a result of allocation.
+//
+// Virtual registers (vregs) in LIR
+// --------------------------------
+// The MIR from which the LIR is derived, is standard SSA. That SSAness is
+// carried through into the LIR (SPEC). In the examples here, LIR SSA names
+// (virtual registers, a.k.a. vregs) are printed as "v<number>". v0 never
+// appears and is presumed to be a special value, perhaps "invalid" (SPEC).
+//
+// The allocator core has a type VirtualRegister, but this is private to the
+// allocator and not part of the LIR. It carries far more information than
+// merely the name of the vreg. The allocator creates one VirtualRegister
+// structure for each vreg in the LIR.
+//
+// LDefinition and LUse
+// --------------------
+// These are part of the incoming LIR. Each LIR instruction defines zero or
+// more values, and contains one LDefinition for each defined value (SPEC).
+// Each instruction has zero or more input operands, each of which has its own
+// LUse (SPEC).
+//
+// Both LDefinition and LUse hold both a virtual register name and, in general,
+// a real (physical) register identity. The incoming LIR has the real register
+// fields unset, except in places where the incoming LIR has fixed register
+// constraints (SPEC). Phase 3 of allocation will visit all of the
+// LDefinitions and LUses so as to write into the real register fields the
+// decisions made by the allocator. For LUses, this is done by overwriting the
+// complete LUse with a different LAllocation, for example LStackSlot. That's
+// possible because LUse is a child class of LAllocation.
+//
+// This action of reading and then later updating LDefinition/LUses is the core
+// of the allocator's interface to the outside world.
+//
+// To make visiting of LDefinitions/LUses possible, the allocator doesn't work
+// with LDefinition and LUse directly. Rather it has pointers to them
+// (VirtualRegister::def_, UsePosition::use_). Hence Phase 3 can modify the
+// LIR in-place.
+//
+// (INVARs, all SPEC):
+//
+// - The collective VirtualRegister::def_ values should be unique, and there
+// should be a 1:1 mapping between the VirtualRegister::def_ values and the
+// LDefinitions in the LIR. (So that the LIR LDefinition has exactly one
+// VirtualRegister::def_ to track it). But only for the valid LDefinitions.
+// If isBogusTemp() is true, the definition is invalid and doesn't have a
+// vreg.
+//
+// - The same for uses: there must be a 1:1 correspondence between the
+// CodePosition::use_ values and the LIR LUses.
+//
+// - The allocation process must preserve these 1:1 mappings. That implies
+// (weaker) that the number of VirtualRegisters and of UsePositions must
+// remain constant through allocation. (Eg: losing them would mean that some
+// LIR def or use would necessarily not get annotated with its final
+// allocation decision. Duplicating them would lead to the possibility of
+// conflicting allocation decisions.)
+//
+// Other comments regarding LIR
+// ----------------------------
+// The incoming LIR is structured into basic blocks and a CFG, as one would
+// expect. These (insns, block boundaries, block edges etc) are available
+// through the BacktrackingAllocator object. They are important for Phases 1
+// and 3 but not for Phase 2.
+//
+// Phase 3 "rewrites" the input LIR so as to "install" the final allocation.
+// It has to insert MoveGroup instructions, but that isn't done by pushing them
+// into the instruction array. Rather, each LIR has 3 auxiliary sets of
+// MoveGroups (SPEC): two that "happen" conceptually before the LIR, and one
+// that happens after it. The rewriter inserts MoveGroups into one of these 3
+// sets, and later code generation phases presumably insert the sets (suitably
+// translated) into the final machine code (SPEC).
+//
+//
+// Key data structures: LiveRange, VirtualRegister and LiveBundle
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// These three have central roles in allocation. Of them, LiveRange is the
+// most central. VirtualRegister is conceptually important throughout, but
+// appears less frequently in the allocator code. LiveBundle is important only
+// in Phase 2 (where it is central) and at the end of Phase 1, but plays no
+// role in Phase 3.
+//
+// It's important to understand that LiveRange and VirtualRegister correspond
+// to concepts visible in the incoming LIR, which is in SSA form. LiveBundle
+// by comparison is related to neither the structure of LIR nor its SSA
+// properties. Instead, LiveBundle is an essentially adminstrative structure
+// used to accelerate allocation and to implement a crude form of
+// move-coalescing.
+//
+// VirtualRegisters and LiveRanges are (almost) static throughout the process,
+// because they reflect aspects of the incoming LIR, which does not change.
+// LiveBundles by contrast come and go; they are created, but may be split up
+// into new bundles, and old ones abandoned.
+//
+// Each LiveRange is a member of two different linked lists, chained through
+// fields registerLink and bundleLink.
+//
+// A VirtualRegister (described in detail below) has a list of LiveRanges that
+// it "owns". These are chained through LiveRange::registerLink.
+//
+// A LiveBundle (also described below) also has a list LiveRanges that it
+// "owns", chained through LiveRange::bundleLink.
+//
+// Hence each LiveRange is "owned" by one VirtualRegister and one LiveBundle.
+// LiveRanges may have their owning LiveBundle changed as a result of
+// splitting. By contrast a LiveRange stays with its "owning" VirtualRegister
+// for ever.
+//
+// A few LiveRanges have no VirtualRegister. This is used to implement
+// register spilling for calls. Each physical register that's not preserved
+// across a call has a small range that covers the call. It is
+// ::buildLivenessInfo that adds these small ranges.
+//
+// Iterating over every VirtualRegister in the system is a common operation and
+// is straightforward because (somewhat redundantly?) the LIRGraph knows the
+// number of vregs, and more importantly because BacktrackingAllocator::vregs
+// is a vector of all VirtualRegisters. By contrast iterating over every
+// LiveBundle in the system is more complex because there is no top-level
+// registry of them. It is still possible though. See ::dumpLiveRangesByVReg
+// and ::dumpLiveRangesByBundle for example code.
+//
+// LiveRange
+// ---------
+// Fundamentally, a LiveRange (often written just "range") is a request for
+// storage of a LIR vreg for some contiguous sequence of LIRs. A LiveRange
+// generally covers only a fraction of a vreg's overall lifetime, so multiple
+// LiveRanges are generally needed to cover the whole lifetime.
+//
+// A LiveRange contains (amongst other things):
+//
+// * the vreg for which it is for, as a VirtualRegister*
+//
+// * the range of CodePositions for which it is for, as a LiveRange::Range
+//
+// * auxiliary information:
+//
+// - a boolean that indicates whether this LiveRange defines a value for the
+// vreg. If so, that definition is regarded as taking place at the first
+// CodePoint of the range.
+//
+// - a linked list of uses of the vreg within this range. Each use is a pair
+// of a CodePosition and an LUse*. (INVAR): the uses are maintained in
+// increasing order of CodePosition. Multiple uses at the same
+// CodePosition are permitted, since that is necessary to represent an LIR
+// that uses the same vreg in more than one of its operand slots.
+//
+// Some important facts about LiveRanges are best illustrated with examples:
+//
+// v25 75-82 { 75_def:R 78_v25:A 82_v25:A }
+//
+// This LiveRange is for vreg v25. The value is defined at CodePosition 75,
+// with the LIR requiring that it be in a register. It is used twice at
+// positions 78 and 82, both with no constraints (A meaning "any"). The range
+// runs from position 75 to 82 inclusive. Note however that LiveRange::Range
+// uses non-inclusive range ends; hence its .to field would be 83, not 82.
+//
+// v26 84-85 { 85_v26:R }
+//
+// This LiveRange is for vreg v26. Here, there's only a single use of it at
+// position 85. Presumably it is defined in some other LiveRange.
+//
+// v19 74-79 { }
+//
+// This LiveRange is for vreg v19. There is no def and no uses, so at first
+// glance this seems redundant. But it isn't: it still expresses a request for
+// storage for v19 across 74-79, because Phase 1 regards v19 as being live in
+// this range (meaning: having a value that, if changed in this range, would
+// cause the program to fail).
+//
+// Other points:
+//
+// * (INVAR) Each vreg/VirtualRegister has at least one LiveRange.
+//
+// * (INVAR) Exactly one LiveRange of a vreg gives a definition for the value.
+// All other LiveRanges must consist only of uses (including zero uses, for a
+// "flow-though" range as mentioned above). This requirement follows from
+// the fact that LIR is in SSA form.
+//
+// * It follows from this, that the LiveRanges for a VirtualRegister must form
+// a tree, where the parent-child relationship is "control flows directly
+// from a parent LiveRange (anywhere in the LiveRange) to a child LiveRange
+// (start)". The entire tree carries only one value. This is a use of
+// SSAness in the allocator which is fundamental: without SSA input, this
+// design would not work.
+//
+// The root node (LiveRange) in the tree must be one that defines the value,
+// and all other nodes must only use or be flow-throughs for the value. It's
+// OK for LiveRanges in the tree to overlap, providing that there is a unique
+// root node -- otherwise it would be unclear which LiveRange provides the
+// value.
+//
+// The function ::createMoveGroupsFromLiveRangeTransitions runs after all
+// LiveBundles have been allocated. It visits each VirtualRegister tree in
+// turn. For every parent->child edge in a tree, it creates a MoveGroup that
+// copies the value from the parent into the child -- this is how the
+// allocator decides where to put MoveGroups. There are various other
+// details not described here.
+//
+// * It's important to understand that a LiveRange carries no meaning about
+// control flow beyond that implied by the SSA (hence, dominance)
+// relationship between a def and its uses. In particular, there's no
+// implication that execution "flowing into" the start of the range implies
+// that it will "flow out" of the end. Or that any particular use will or
+// will not be executed.
+//
+// * (very SPEC) Indeed, even if a range has a def, there's no implication that
+// a use later in the range will have been immediately preceded by execution
+// of the def. It could be that the def is executed, flow jumps somewhere
+// else, and later jumps back into the middle of the range, where there are
+// then some uses.
+//
+// * Uses of a vreg by a phi node are not mentioned in the use list of a
+// LiveRange. The reasons for this are unknown, but it is speculated that
+// this is because we don't need to know about phi uses where we use the list
+// of positions. See comments on VirtualRegister::usedByPhi_.
+//
+// * Similarly, a definition of a vreg by a phi node is not regarded as being a
+// definition point (why not?), at least as the view of
+// LiveRange::hasDefinition_.
+//
+// * LiveRanges that nevertheless include a phi-defined value have their first
+// point set to the first of the block of phis, even if the var isn't defined
+// by that specific phi. Eg:
+//
+// Block 6 [successor 7] [successor 8]
+// 56-59 Phi [def v19<o>] [use v2:A] [use v5:A] [use v13:A]
+// 56-59 Phi [def v20<o>] [use v7:A] [use v14:A] [use v12:A]
+// 60-61 WasmLoadSlot [def v21<o>] [use v1:R]
+// 62-63 Compare [def v22<i>] [use v20:R] [use v21:A]
+//
+// The relevant live range for v20 is
+//
+// v20 56-65 { 63_v20:R }
+//
+// Observe that it starts at 56, not 58.
+//
+// VirtualRegister
+// ---------------
+// Each VirtualRegister is associated with an SSA value created by the LIR.
+// Fundamentally it is a container to hold all of the LiveRanges that together
+// indicate where the value must be kept live. This is a linked list beginning
+// at VirtualRegister::ranges_, and which, as described above, is chained
+// through LiveRange::registerLink. The set of LiveRanges must logically form
+// a tree, rooted at the LiveRange which defines the value.
+//
+// For adminstrative convenience, the linked list must contain the LiveRanges
+// in order of increasing start point.
+//
+// There are various auxiliary fields, most importantly the LIR node and the
+// associated LDefinition that define the value.
+//
+// It is OK, and quite common, for LiveRanges of a VirtualRegister to overlap.
+// The effect will be that, in an overlapped area, there are two storage
+// locations holding the value. This is OK -- although wasteful of storage
+// resources -- because the SSAness means the value must be the same in both
+// locations. Hence there's no questions like "which LiveRange holds the most
+// up-to-date value?", since it's all just one value anyway.
+//
+// Note by contrast, it is *not* OK for the LiveRanges of a LiveBundle to
+// overlap.
+//
+// LiveBundle
+// ----------
+// Similar to VirtualRegister, a LiveBundle is also, fundamentally, a container
+// for a set of LiveRanges. The set is stored as a linked list, rooted at
+// LiveBundle::ranges_ and chained through LiveRange::bundleLink.
+//
+// However, the similarity ends there:
+//
+// * The LiveRanges in a LiveBundle absolutely must not overlap. They must
+// indicate disjoint sets of CodePositions, and must be stored in the list in
+// order of increasing CodePosition. Because of the no-overlap requirement,
+// these ranges form a total ordering regardless of whether one uses the
+// LiveRange::Range::from_ or ::to_ fields for comparison.
+//
+// * The LiveRanges in a LiveBundle can otherwise be entirely arbitrary and
+// unrelated. They can be from different VirtualRegisters and can have no
+// particular mutual significance w.r.t. the SSAness or structure of the
+// input LIR.
+//
+// LiveBundles are the fundamental unit of allocation. The allocator attempts
+// to find a single storage location that will work for all LiveRanges in the
+// bundle. That's why the ranges must not overlap. If no such location can be
+// found, the allocator may decide to split the bundle into multiple smaller
+// bundles. Each of those may be allocated independently.
+//
+// The other really important field is LiveBundle::alloc_, indicating the
+// chosen storage location.
+//
+// Here's an example, for a LiveBundle before it has been allocated:
+//
+// LB2(parent=none v3 8-21 { 16_v3:A } ## v3 24-25 { 25_v3:F:xmm0 })
+//
+// LB merely indicates "LiveBundle", and the 2 is the debugId_ value (see
+// below). This bundle has two LiveRanges
+//
+// v3 8-21 { 16_v3:A }
+// v3 24-25 { 25_v3:F:xmm0 }
+//
+// both of which (coincidentally) are for the same VirtualRegister, v3.The
+// second LiveRange has a fixed use in `xmm0`, whilst the first one doesn't
+// care (A meaning "any location") so the allocator *could* choose `xmm0` for
+// the bundle as a whole.
+//
+// One might ask: why bother with LiveBundle at all? After all, it would be
+// possible to get correct allocations by allocating each LiveRange
+// individually, then leaving ::createMoveGroupsFromLiveRangeTransitions to add
+// MoveGroups to join up LiveRanges that form each SSA value tree (that is,
+// LiveRanges belonging to each VirtualRegister).
+//
+// There are two reasons:
+//
+// (1) By putting multiple LiveRanges into each LiveBundle, we can end up with
+// many fewer LiveBundles than LiveRanges. Since the cost of allocating a
+// LiveBundle is substantially less than the cost of allocating each of its
+// LiveRanges individually, the allocator will run faster.
+//
+// (2) It provides a crude form of move-coalescing. There are situations where
+// it would be beneficial, although not mandatory, to have two LiveRanges
+// assigned to the same storage unit. Most importantly: (a) LiveRanges
+// that form all of the inputs, and the output, of a phi node. (b)
+// LiveRanges for the output and first-operand values in the case where we
+// are targetting Intel-style instructions.
+//
+// In such cases, if the bundle can be allocated as-is, then no extra moves
+// are necessary. If not (and the bundle is split), then
+// ::createMoveGroupsFromLiveRangeTransitions will later fix things up by
+// inserting MoveGroups in the right places.
+//
+// Merging of LiveRanges into LiveBundles is done in Phase 1, by
+// ::mergeAndQueueRegisters, after liveness analysis (which generates only
+// LiveRanges).
+//
+// For the bundle mentioned above, viz
+//
+// LB2(parent=none v3 8-21 { 16_v3:A } ## v3 24-25 { 25_v3:F:xmm0 })
+//
+// the allocator did not in the end choose to allocate it to `xmm0`. Instead
+// it was split into two bundles, LB6 (a "spill parent", or root node in the
+// trees described above), and LB9, a leaf node, that points to its parent LB6:
+//
+// LB6(parent=none v3 8-21 %xmm1.s { 16_v3:A } ## v3 24-25 %xmm1.s { })
+// LB9(parent=LB6 v3 24-25 %xmm0.s { 25_v3:F:rax })
+//
+// Note that both bundles now have an allocation, and that is printed,
+// redundantly, for each LiveRange in the bundle -- hence the repeated
+// `%xmm1.s` in the lines above. Since all LiveRanges in a LiveBundle must be
+// allocated to the same storage location, we never expect to see output like
+// this
+//
+// LB6(parent=none v3 8-21 %xmm1.s { 16_v3:A } ## v3 24-25 %xmm2.s { })
+//
+// and that is in any case impossible, since a LiveRange doesn't have an
+// LAllocation field. Instead it has a pointer to its LiveBundle, and the
+// LAllocation lives in the LiveBundle.
+//
+// For the resulting allocation (LB6 and LB9), all the LiveRanges are use-only
+// or flow-through. There are no definitions. But they are all for the same
+// VirtualReg, v3, so they all have the same value. An important question is
+// where they "get their value from". The answer is that
+// ::createMoveGroupsFromLiveRangeTransitions will have to insert suitable
+// MoveGroups so that each LiveRange for v3 can "acquire" the value from a
+// previously-executed LiveRange, except for the range that defines it. The
+// defining LiveRange is not visible here; either it is in some other
+// LiveBundle, not shown, or (more likely) the value is defined by a phi-node,
+// in which case, as mentioned previously, it is not shown as having an
+// explicit definition in any LiveRange.
+//
+// LiveBundles also have a `SpillSet* spill_` field (see below) and a
+// `LiveBundle* spillParent_`. The latter is used to ensure that all bundles
+// originating from an "original" bundle share the same spill slot. The
+// `spillParent_` pointers can be thought of creating a 1-level tree, where
+// each node points at its parent. Since the tree can be only 1-level, the
+// following invariant (INVAR) must be maintained:
+//
+// * if a LiveBundle has a non-null spillParent_ field, then it is a leaf node,
+// and no other LiveBundle can point at this one.
+//
+// * else (it has a null spillParent_ field) it is a root node, and so other
+// LiveBundles may point at it.
+//
+// When compiled with JS_JITSPEW, LiveBundle has a 32-bit `debugId_` field.
+// This is used only for debug printing, and makes it easier to see
+// parent-child relationships induced by the `spillParent_` pointers.
+//
+// The "life cycle" of LiveBundles is described in Section 2 below.
+//
+//
+// Secondary data structures: SpillSet, Requirement
+// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+//
+// SpillSet
+// --------
+// A SpillSet is a container for a set of LiveBundles that have been spilled,
+// all of which are assigned the same spill slot. The set is represented as a
+// vector of points to LiveBundles. SpillSet also contains the identity of the
+// spill slot (its LAllocation).
+//
+// A LiveBundle, if it is to be spilled, has a pointer to the relevant
+// SpillSet, and the SpillSet in turn has a pointer back to the LiveBundle in
+// its vector thereof. So LiveBundles (that are to be spilled) and SpillSets
+// point at each other.
+//
+// (SPEC) LiveBundles that are not to be spilled (or for which the decision has
+// yet to be made, have their SpillSet pointers as null. (/SPEC)
+//
+// Requirement
+// -----------
+// Requirements are used transiently during the main allocation loop. It
+// summarises the set of constraints on storage location (must be any register,
+// must be this specific register, must be stack, etc) for a LiveBundle. This
+// is so that the main allocation loop knows what kind of storage location it
+// must choose in order to satisfy all of the defs and uses within the bundle.
+//
+// What Requirement provides is (a) a partially ordered set of locations, and
+// (b) a constraint-merging method `merge`.
+//
+// Requirement needs a rewrite (and, in fact, that has already happened in
+// un-landed code in bug 1758274) for the following reasons:
+//
+// * it's potentially buggy (bug 1761654), although that doesn't currently
+// affect us, for reasons which are unclear.
+//
+// * the partially ordered set has a top point, meaning "no constraint", but it
+// doesn't have a corresponding bottom point, meaning "impossible
+// constraints". (So it's a partially ordered set, but not a lattice). This
+// leads to awkward coding in some places, which would be simplified if there
+// were an explicit way to indicate "impossible constraint".
+//
+//
+// Some ASCII art
+// ~~~~~~~~~~~~~~
+//
+// Here's some not-very-good ASCII art that tries to summarise the data
+// structures that persist for the entire allocation of a function:
+//
+// BacktrackingAllocator
+// |
+// (vregs)
+// |
+// v
+// |
+// VirtualRegister -->--(ins)--> LNode
+// | | `->--(def)--> LDefinition
+// v ^
+// | |
+// (ranges) |
+// | (vreg)
+// `--v->--. | ,-->--v-->-------------->--v-->--. ,--NULL
+// \ | / \ /
+// LiveRange LiveRange LiveRange
+// / | \ / \.
+// ,--b->--' / `-->--b-->--' `--NULL
+// | (bundle)
+// ^ /
+// | v
+// (ranges) /
+// | /
+// LiveBundle --s-->- LiveBundle
+// | \ / |
+// | \ / |
+// (spill) ^ ^ (spill)
+// | \ / |
+// v \ / ^
+// | (list) |
+// \ | /
+// `--->---> SpillSet <--'
+//
+// --b-- LiveRange::bundleLink: links in the list of LiveRanges that belong to
+// a LiveBundle
+//
+// --v-- LiveRange::registerLink: links in the list of LiveRanges that belong
+// to a VirtualRegister
+//
+// --s-- LiveBundle::spillParent: a possible link to my "spill parent bundle"
+//
+//
+// * LiveRange is in the center. Each LiveRange is a member of two different
+// linked lists, the --b-- list and the --v-- list.
+//
+// * VirtualRegister has a pointer `ranges` that points to the start of its
+// --v-- list of LiveRanges.
+//
+// * LiveBundle has a pointer `ranges` that points to the start of its --b--
+// list of LiveRanges.
+//
+// * LiveRange points back at both its owning VirtualRegister (`vreg`) and its
+// owning LiveBundle (`bundle`).
+//
+// * LiveBundle has a pointer --s-- `spillParent`, which may be null, to its
+// conceptual "spill parent bundle", as discussed in detail above.
+//
+// * LiveBundle has a pointer `spill` to its SpillSet.
+//
+// * SpillSet has a vector `list` of pointers back to the LiveBundles that
+// point at it.
+//
+// * VirtualRegister has pointers `ins` to the LNode that defines the value and
+// `def` to the LDefinition within that LNode.
+//
+// * BacktrackingAllocator has a vector `vregs` of pointers to all the
+// VirtualRegisters for the function. There is no equivalent top-level table
+// of all the LiveBundles for the function.
+//
+// Note that none of these pointers are "owning" in the C++-storage-management
+// sense. Rather, everything is stored in single arena which is freed when
+// compilation of the function is complete. For this reason,
+// BacktrackingAllocator.{h,cpp} is almost completely free of the usual C++
+// storage-management artefacts one would normally expect to see.
+//
+//
+// ========================================================================
+// ==== ====
+// ==== Section 2: The core allocation loop, and bundle splitting ====
+// ==== ====
+// ========================================================================
+//
+// Phase 1 of the allocator (described at the start of this SMDOC) computes
+// live ranges, merges them into bundles, and places the bundles in a priority
+// queue ::allocationQueue, ordered by what ::computePriority computes.
+//
+//
+// The allocation loops
+// ~~~~~~~~~~~~~~~~~~~~
+// The core of the allocation machinery consists of two loops followed by a
+// call to ::pickStackSlots. The latter is uninteresting. The two loops live
+// in ::go and are documented in detail there.
+//
+//
+// Bundle splitting
+// ~~~~~~~~~~~~~~~~
+// If the first of the two abovementioned loops cannot find a register for a
+// bundle, either directly or as a result of evicting conflicting bundles, then
+// it will have to either split or spill the bundle. The entry point to the
+// split/spill subsystem is ::chooseBundleSplit. See comments there.
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// End of documentation //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+#include "jit/BacktrackingAllocator.h"
+
+#include <algorithm>
+
+#include "jit/BitSet.h"
+#include "jit/CompileInfo.h"
+#include "js/Printf.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+
+// This is a big, complex file. Code is grouped into various sections, each
+// preceded by a box comment. Sections not marked as "Misc helpers" are
+// pretty much the top level flow, and are presented roughly in the same order
+// in which the allocation pipeline operates. BacktrackingAllocator::go,
+// right at the end of the file, is a good starting point.
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Misc helpers: linked-list management //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+static inline bool SortBefore(UsePosition* a, UsePosition* b) {
+ return a->pos <= b->pos;
+}
+
+static inline bool SortBefore(LiveRange::BundleLink* a,
+ LiveRange::BundleLink* b) {
+ LiveRange* rangea = LiveRange::get(a);
+ LiveRange* rangeb = LiveRange::get(b);
+ MOZ_ASSERT(!rangea->intersects(rangeb));
+ return rangea->from() < rangeb->from();
+}
+
+static inline bool SortBefore(LiveRange::RegisterLink* a,
+ LiveRange::RegisterLink* b) {
+ return LiveRange::get(a)->from() <= LiveRange::get(b)->from();
+}
+
+template <typename T>
+static inline void InsertSortedList(InlineForwardList<T>& list, T* value) {
+ if (list.empty()) {
+ list.pushFront(value);
+ return;
+ }
+
+ if (SortBefore(list.back(), value)) {
+ list.pushBack(value);
+ return;
+ }
+
+ T* prev = nullptr;
+ for (InlineForwardListIterator<T> iter = list.begin(); iter; iter++) {
+ if (SortBefore(value, *iter)) {
+ break;
+ }
+ prev = *iter;
+ }
+
+ if (prev) {
+ list.insertAfter(prev, value);
+ } else {
+ list.pushFront(value);
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Misc helpers: methods for class SpillSet //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+void SpillSet::setAllocation(LAllocation alloc) {
+ for (size_t i = 0; i < numSpilledBundles(); i++) {
+ spilledBundle(i)->setAllocation(alloc);
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Misc helpers: methods for class LiveRange //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+static size_t SpillWeightFromUsePolicy(LUse::Policy policy) {
+ switch (policy) {
+ case LUse::ANY:
+ return 1000;
+
+ case LUse::REGISTER:
+ case LUse::FIXED:
+ return 2000;
+
+ default:
+ return 0;
+ }
+}
+
+inline void LiveRange::noteAddedUse(UsePosition* use) {
+ LUse::Policy policy = use->usePolicy();
+ usesSpillWeight_ += SpillWeightFromUsePolicy(policy);
+ if (policy == LUse::FIXED) {
+ ++numFixedUses_;
+ }
+}
+
+inline void LiveRange::noteRemovedUse(UsePosition* use) {
+ LUse::Policy policy = use->usePolicy();
+ usesSpillWeight_ -= SpillWeightFromUsePolicy(policy);
+ if (policy == LUse::FIXED) {
+ --numFixedUses_;
+ }
+ MOZ_ASSERT_IF(!hasUses(), !usesSpillWeight_ && !numFixedUses_);
+}
+
+void LiveRange::addUse(UsePosition* use) {
+ MOZ_ASSERT(covers(use->pos));
+ InsertSortedList(uses_, use);
+ noteAddedUse(use);
+}
+
+UsePosition* LiveRange::popUse() {
+ UsePosition* ret = uses_.popFront();
+ noteRemovedUse(ret);
+ return ret;
+}
+
+void LiveRange::tryToMoveDefAndUsesInto(LiveRange* other) {
+ MOZ_ASSERT(&other->vreg() == &vreg());
+ MOZ_ASSERT(this != other);
+
+ // Move over all uses which fit in |other|'s boundaries.
+ for (UsePositionIterator iter = usesBegin(); iter;) {
+ UsePosition* use = *iter;
+ if (other->covers(use->pos)) {
+ uses_.removeAndIncrement(iter);
+ noteRemovedUse(use);
+ other->addUse(use);
+ } else {
+ iter++;
+ }
+ }
+
+ // Distribute the definition to |other| as well, if possible.
+ if (hasDefinition() && from() == other->from()) {
+ other->setHasDefinition();
+ }
+}
+
+bool LiveRange::contains(LiveRange* other) const {
+ return from() <= other->from() && to() >= other->to();
+}
+
+void LiveRange::intersect(LiveRange* other, Range* pre, Range* inside,
+ Range* post) const {
+ MOZ_ASSERT(pre->empty() && inside->empty() && post->empty());
+
+ CodePosition innerFrom = from();
+ if (from() < other->from()) {
+ if (to() < other->from()) {
+ *pre = range_;
+ return;
+ }
+ *pre = Range(from(), other->from());
+ innerFrom = other->from();
+ }
+
+ CodePosition innerTo = to();
+ if (to() > other->to()) {
+ if (from() >= other->to()) {
+ *post = range_;
+ return;
+ }
+ *post = Range(other->to(), to());
+ innerTo = other->to();
+ }
+
+ if (innerFrom != innerTo) {
+ *inside = Range(innerFrom, innerTo);
+ }
+}
+
+bool LiveRange::intersects(LiveRange* other) const {
+ Range pre, inside, post;
+ intersect(other, &pre, &inside, &post);
+ return !inside.empty();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Misc helpers: methods for class LiveBundle //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+#ifdef DEBUG
+size_t LiveBundle::numRanges() const {
+ size_t count = 0;
+ for (LiveRange::BundleLinkIterator iter = rangesBegin(); iter; iter++) {
+ count++;
+ }
+ return count;
+}
+#endif
+
+LiveRange* LiveBundle::rangeFor(CodePosition pos) const {
+ for (LiveRange::BundleLinkIterator iter = rangesBegin(); iter; iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ if (range->covers(pos)) {
+ return range;
+ }
+ }
+ return nullptr;
+}
+
+void LiveBundle::addRange(LiveRange* range) {
+ MOZ_ASSERT(!range->bundle());
+ range->setBundle(this);
+ InsertSortedList(ranges_, &range->bundleLink);
+}
+
+bool LiveBundle::addRange(TempAllocator& alloc, VirtualRegister* vreg,
+ CodePosition from, CodePosition to) {
+ LiveRange* range = LiveRange::FallibleNew(alloc, vreg, from, to);
+ if (!range) {
+ return false;
+ }
+ addRange(range);
+ return true;
+}
+
+bool LiveBundle::addRangeAndDistributeUses(TempAllocator& alloc,
+ LiveRange* oldRange,
+ CodePosition from, CodePosition to) {
+ LiveRange* range = LiveRange::FallibleNew(alloc, &oldRange->vreg(), from, to);
+ if (!range) {
+ return false;
+ }
+ addRange(range);
+ oldRange->tryToMoveDefAndUsesInto(range);
+ return true;
+}
+
+LiveRange* LiveBundle::popFirstRange() {
+ LiveRange::BundleLinkIterator iter = rangesBegin();
+ if (!iter) {
+ return nullptr;
+ }
+
+ LiveRange* range = LiveRange::get(*iter);
+ ranges_.removeAt(iter);
+
+ range->setBundle(nullptr);
+ return range;
+}
+
+void LiveBundle::removeRange(LiveRange* range) {
+ for (LiveRange::BundleLinkIterator iter = rangesBegin(); iter; iter++) {
+ LiveRange* existing = LiveRange::get(*iter);
+ if (existing == range) {
+ ranges_.removeAt(iter);
+ return;
+ }
+ }
+ MOZ_CRASH();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Misc helpers: methods for class VirtualRegister //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+bool VirtualRegister::addInitialRange(TempAllocator& alloc, CodePosition from,
+ CodePosition to, size_t* numRanges) {
+ MOZ_ASSERT(from < to);
+
+ // Mark [from,to) as a live range for this register during the initial
+ // liveness analysis, coalescing with any existing overlapping ranges.
+
+ // On some pathological graphs there might be a huge number of different
+ // live ranges. Allow non-overlapping live range to be merged if the
+ // number of ranges exceeds the cap below.
+ static const size_t CoalesceLimit = 100000;
+
+ LiveRange* prev = nullptr;
+ LiveRange* merged = nullptr;
+ for (LiveRange::RegisterLinkIterator iter(rangesBegin()); iter;) {
+ LiveRange* existing = LiveRange::get(*iter);
+
+ if (from > existing->to() && *numRanges < CoalesceLimit) {
+ // The new range should go after this one.
+ prev = existing;
+ iter++;
+ continue;
+ }
+
+ if (to.next() < existing->from()) {
+ // The new range should go before this one.
+ break;
+ }
+
+ if (!merged) {
+ // This is the first old range we've found that overlaps the new
+ // range. Extend this one to cover its union with the new range.
+ merged = existing;
+
+ if (from < existing->from()) {
+ existing->setFrom(from);
+ }
+ if (to > existing->to()) {
+ existing->setTo(to);
+ }
+
+ // Continue searching to see if any other old ranges can be
+ // coalesced with the new merged range.
+ iter++;
+ continue;
+ }
+
+ // Coalesce this range into the previous range we merged into.
+ MOZ_ASSERT(existing->from() >= merged->from());
+ if (existing->to() > merged->to()) {
+ merged->setTo(existing->to());
+ }
+
+ MOZ_ASSERT(!existing->hasDefinition());
+ existing->tryToMoveDefAndUsesInto(merged);
+ MOZ_ASSERT(!existing->hasUses());
+
+ ranges_.removeAndIncrement(iter);
+ }
+
+ if (!merged) {
+ // The new range does not overlap any existing range for the vreg.
+ LiveRange* range = LiveRange::FallibleNew(alloc, this, from, to);
+ if (!range) {
+ return false;
+ }
+
+ if (prev) {
+ ranges_.insertAfter(&prev->registerLink, &range->registerLink);
+ } else {
+ ranges_.pushFront(&range->registerLink);
+ }
+
+ (*numRanges)++;
+ }
+
+ return true;
+}
+
+void VirtualRegister::addInitialUse(UsePosition* use) {
+ LiveRange::get(*rangesBegin())->addUse(use);
+}
+
+void VirtualRegister::setInitialDefinition(CodePosition from) {
+ LiveRange* first = LiveRange::get(*rangesBegin());
+ MOZ_ASSERT(from >= first->from());
+ first->setFrom(from);
+ first->setHasDefinition();
+}
+
+LiveRange* VirtualRegister::rangeFor(CodePosition pos,
+ bool preferRegister /* = false */) const {
+ LiveRange* found = nullptr;
+ for (LiveRange::RegisterLinkIterator iter = rangesBegin(); iter; iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ if (range->covers(pos)) {
+ if (!preferRegister || range->bundle()->allocation().isRegister()) {
+ return range;
+ }
+ if (!found) {
+ found = range;
+ }
+ }
+ }
+ return found;
+}
+
+void VirtualRegister::addRange(LiveRange* range) {
+ InsertSortedList(ranges_, &range->registerLink);
+}
+
+void VirtualRegister::removeRange(LiveRange* range) {
+ for (LiveRange::RegisterLinkIterator iter = rangesBegin(); iter; iter++) {
+ LiveRange* existing = LiveRange::get(*iter);
+ if (existing == range) {
+ ranges_.removeAt(iter);
+ return;
+ }
+ }
+ MOZ_CRASH();
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Misc helpers: queries about uses //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+static inline LDefinition* FindReusingDefOrTemp(LNode* node,
+ LAllocation* alloc) {
+ if (node->isPhi()) {
+ MOZ_ASSERT(node->toPhi()->numDefs() == 1);
+ MOZ_ASSERT(node->toPhi()->getDef(0)->policy() !=
+ LDefinition::MUST_REUSE_INPUT);
+ return nullptr;
+ }
+
+ LInstruction* ins = node->toInstruction();
+
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ LDefinition* def = ins->getDef(i);
+ if (def->policy() == LDefinition::MUST_REUSE_INPUT &&
+ ins->getOperand(def->getReusedInput()) == alloc) {
+ return def;
+ }
+ }
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ LDefinition* def = ins->getTemp(i);
+ if (def->policy() == LDefinition::MUST_REUSE_INPUT &&
+ ins->getOperand(def->getReusedInput()) == alloc) {
+ return def;
+ }
+ }
+ return nullptr;
+}
+
+bool BacktrackingAllocator::isReusedInput(LUse* use, LNode* ins,
+ bool considerCopy) {
+ if (LDefinition* def = FindReusingDefOrTemp(ins, use)) {
+ return considerCopy || !vregs[def->virtualRegister()].mustCopyInput();
+ }
+ return false;
+}
+
+bool BacktrackingAllocator::isRegisterUse(UsePosition* use, LNode* ins,
+ bool considerCopy) {
+ switch (use->usePolicy()) {
+ case LUse::ANY:
+ return isReusedInput(use->use(), ins, considerCopy);
+
+ case LUse::REGISTER:
+ case LUse::FIXED:
+ return true;
+
+ default:
+ return false;
+ }
+}
+
+bool BacktrackingAllocator::isRegisterDefinition(LiveRange* range) {
+ if (!range->hasDefinition()) {
+ return false;
+ }
+
+ VirtualRegister& reg = range->vreg();
+ if (reg.ins()->isPhi()) {
+ return false;
+ }
+
+ if (reg.def()->policy() == LDefinition::FIXED &&
+ !reg.def()->output()->isRegister()) {
+ return false;
+ }
+
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Misc helpers: atomic LIR groups //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+// The following groupings contain implicit (invisible to ::buildLivenessInfo)
+// value flows, and therefore no split points may be requested inside them.
+// This is an otherwise unstated part of the contract between LIR generation
+// and the allocator.
+//
+// (1) (any insn) ; OsiPoint
+//
+// [Further group definitions and supporting code to come, pending rework
+// of the wasm atomic-group situation.]
+
+CodePosition RegisterAllocator::minimalDefEnd(LNode* ins) const {
+ // Compute the shortest interval that captures vregs defined by ins.
+ // Watch for instructions that are followed by an OSI point.
+ // If moves are introduced between the instruction and the OSI point then
+ // safepoint information for the instruction may be incorrect.
+ while (true) {
+ LNode* next = insData[ins->id() + 1];
+ if (!next->isOsiPoint()) {
+ break;
+ }
+ ins = next;
+ }
+
+ return outputOf(ins);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Misc helpers: computation of bundle priorities and spill weights //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+size_t BacktrackingAllocator::computePriority(LiveBundle* bundle) {
+ // The priority of a bundle is its total length, so that longer lived
+ // bundles will be processed before shorter ones (even if the longer ones
+ // have a low spill weight). See processBundle().
+ size_t lifetimeTotal = 0;
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ lifetimeTotal += range->to() - range->from();
+ }
+
+ return lifetimeTotal;
+}
+
+bool BacktrackingAllocator::minimalDef(LiveRange* range, LNode* ins) {
+ // Whether this is a minimal range capturing a definition at ins.
+ return (range->to() <= minimalDefEnd(ins).next()) &&
+ ((!ins->isPhi() && range->from() == inputOf(ins)) ||
+ range->from() == outputOf(ins));
+}
+
+bool BacktrackingAllocator::minimalUse(LiveRange* range, UsePosition* use) {
+ // Whether this is a minimal range capturing |use|.
+ LNode* ins = insData[use->pos];
+ return (range->from() == inputOf(ins)) &&
+ (range->to() ==
+ (use->use()->usedAtStart() ? outputOf(ins) : outputOf(ins).next()));
+}
+
+bool BacktrackingAllocator::minimalBundle(LiveBundle* bundle, bool* pfixed) {
+ LiveRange::BundleLinkIterator iter = bundle->rangesBegin();
+ LiveRange* range = LiveRange::get(*iter);
+
+ if (!range->hasVreg()) {
+ *pfixed = true;
+ return true;
+ }
+
+ // If a bundle contains multiple ranges, splitAtAllRegisterUses will split
+ // each range into a separate bundle.
+ if (++iter) {
+ return false;
+ }
+
+ if (range->hasDefinition()) {
+ VirtualRegister& reg = range->vreg();
+ if (pfixed) {
+ *pfixed = reg.def()->policy() == LDefinition::FIXED &&
+ reg.def()->output()->isRegister();
+ }
+ return minimalDef(range, reg.ins());
+ }
+
+ bool fixed = false, minimal = false, multiple = false;
+
+ for (UsePositionIterator iter = range->usesBegin(); iter; iter++) {
+ if (iter != range->usesBegin()) {
+ multiple = true;
+ }
+
+ switch (iter->usePolicy()) {
+ case LUse::FIXED:
+ if (fixed) {
+ return false;
+ }
+ fixed = true;
+ if (minimalUse(range, *iter)) {
+ minimal = true;
+ }
+ break;
+
+ case LUse::REGISTER:
+ if (minimalUse(range, *iter)) {
+ minimal = true;
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ // If a range contains a fixed use and at least one other use,
+ // splitAtAllRegisterUses will split each use into a different bundle.
+ if (multiple && fixed) {
+ minimal = false;
+ }
+
+ if (pfixed) {
+ *pfixed = fixed;
+ }
+ return minimal;
+}
+
+size_t BacktrackingAllocator::computeSpillWeight(LiveBundle* bundle) {
+ // Minimal bundles have an extremely high spill weight, to ensure they
+ // can evict any other bundles and be allocated to a register.
+ bool fixed;
+ if (minimalBundle(bundle, &fixed)) {
+ return fixed ? 2000000 : 1000000;
+ }
+
+ size_t usesTotal = 0;
+ fixed = false;
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ if (range->hasDefinition()) {
+ VirtualRegister& reg = range->vreg();
+ if (reg.def()->policy() == LDefinition::FIXED &&
+ reg.def()->output()->isRegister()) {
+ usesTotal += 2000;
+ fixed = true;
+ } else if (!reg.ins()->isPhi()) {
+ usesTotal += 2000;
+ }
+ }
+
+ usesTotal += range->usesSpillWeight();
+ if (range->numFixedUses() > 0) {
+ fixed = true;
+ }
+ }
+
+ // Bundles with fixed uses are given a higher spill weight, since they must
+ // be allocated to a specific register.
+ if (testbed && fixed) {
+ usesTotal *= 2;
+ }
+
+ // Compute spill weight as a use density, lowering the weight for long
+ // lived bundles with relatively few uses.
+ size_t lifetimeTotal = computePriority(bundle);
+ return lifetimeTotal ? usesTotal / lifetimeTotal : 0;
+}
+
+size_t BacktrackingAllocator::maximumSpillWeight(
+ const LiveBundleVector& bundles) {
+ size_t maxWeight = 0;
+ for (size_t i = 0; i < bundles.length(); i++) {
+ maxWeight = std::max(maxWeight, computeSpillWeight(bundles[i]));
+ }
+ return maxWeight;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Initialization of the allocator //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+// This function pre-allocates and initializes as much global state as possible
+// to avoid littering the algorithms with memory management cruft.
+bool BacktrackingAllocator::init() {
+ if (!RegisterAllocator::init()) {
+ return false;
+ }
+
+ liveIn = mir->allocate<BitSet>(graph.numBlockIds());
+ if (!liveIn) {
+ return false;
+ }
+
+ size_t numVregs = graph.numVirtualRegisters();
+ if (!vregs.init(mir->alloc(), numVregs)) {
+ return false;
+ }
+ for (uint32_t i = 0; i < numVregs; i++) {
+ new (&vregs[i]) VirtualRegister();
+ }
+
+ // Build virtual register objects.
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ if (mir->shouldCancel("Create data structures (main loop)")) {
+ return false;
+ }
+
+ LBlock* block = graph.getBlock(i);
+ for (LInstructionIterator ins = block->begin(); ins != block->end();
+ ins++) {
+ if (mir->shouldCancel("Create data structures (inner loop 1)")) {
+ return false;
+ }
+
+ for (size_t j = 0; j < ins->numDefs(); j++) {
+ LDefinition* def = ins->getDef(j);
+ if (def->isBogusTemp()) {
+ continue;
+ }
+ vreg(def).init(*ins, def, /* isTemp = */ false);
+ }
+
+ for (size_t j = 0; j < ins->numTemps(); j++) {
+ LDefinition* def = ins->getTemp(j);
+ if (def->isBogusTemp()) {
+ continue;
+ }
+ vreg(def).init(*ins, def, /* isTemp = */ true);
+ }
+ }
+ for (size_t j = 0; j < block->numPhis(); j++) {
+ LPhi* phi = block->getPhi(j);
+ LDefinition* def = phi->getDef(0);
+ vreg(def).init(phi, def, /* isTemp = */ false);
+ }
+ }
+
+ LiveRegisterSet remainingRegisters(allRegisters_.asLiveSet());
+ while (!remainingRegisters.emptyGeneral()) {
+ AnyRegister reg = AnyRegister(remainingRegisters.takeAnyGeneral());
+ registers[reg.code()].allocatable = true;
+ }
+ while (!remainingRegisters.emptyFloat()) {
+ AnyRegister reg =
+ AnyRegister(remainingRegisters.takeAnyFloat<RegTypeName::Any>());
+ registers[reg.code()].allocatable = true;
+ }
+
+ LifoAlloc* lifoAlloc = mir->alloc().lifoAlloc();
+ for (size_t i = 0; i < AnyRegister::Total; i++) {
+ registers[i].reg = AnyRegister::FromCode(i);
+ registers[i].allocations.setAllocator(lifoAlloc);
+ }
+
+ hotcode.setAllocator(lifoAlloc);
+ callRanges.setAllocator(lifoAlloc);
+
+ // Partition the graph into hot and cold sections, for helping to make
+ // splitting decisions. Since we don't have any profiling data this is a
+ // crapshoot, so just mark the bodies of inner loops as hot and everything
+ // else as cold.
+
+ LBlock* backedge = nullptr;
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ LBlock* block = graph.getBlock(i);
+
+ // If we see a loop header, mark the backedge so we know when we have
+ // hit the end of the loop. Don't process the loop immediately, so that
+ // if there is an inner loop we will ignore the outer backedge.
+ if (block->mir()->isLoopHeader()) {
+ backedge = block->mir()->backedge()->lir();
+ }
+
+ if (block == backedge) {
+ LBlock* header = block->mir()->loopHeaderOfBackedge()->lir();
+ LiveRange* range = LiveRange::FallibleNew(
+ alloc(), nullptr, entryOf(header), exitOf(block).next());
+ if (!range || !hotcode.insert(range)) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Liveness analysis //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+// Helper for ::buildLivenessInfo
+bool BacktrackingAllocator::addInitialFixedRange(AnyRegister reg,
+ CodePosition from,
+ CodePosition to) {
+ LiveRange* range = LiveRange::FallibleNew(alloc(), nullptr, from, to);
+ return range && registers[reg.code()].allocations.insert(range);
+}
+
+// Helper for ::buildLivenessInfo
+#ifdef DEBUG
+// Returns true iff ins has a def/temp reusing the input allocation.
+static bool IsInputReused(LInstruction* ins, LUse* use) {
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ if (ins->getDef(i)->policy() == LDefinition::MUST_REUSE_INPUT &&
+ ins->getOperand(ins->getDef(i)->getReusedInput())->toUse() == use) {
+ return true;
+ }
+ }
+
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ if (ins->getTemp(i)->policy() == LDefinition::MUST_REUSE_INPUT &&
+ ins->getOperand(ins->getTemp(i)->getReusedInput())->toUse() == use) {
+ return true;
+ }
+ }
+
+ return false;
+}
+#endif
+
+/*
+ * This function builds up liveness ranges for all virtual registers
+ * defined in the function.
+ *
+ * The algorithm is based on the one published in:
+ *
+ * Wimmer, Christian, and Michael Franz. "Linear Scan Register Allocation on
+ * SSA Form." Proceedings of the International Symposium on Code Generation
+ * and Optimization. Toronto, Ontario, Canada, ACM. 2010. 170-79. PDF.
+ *
+ * The algorithm operates on blocks ordered such that dominators of a block
+ * are before the block itself, and such that all blocks of a loop are
+ * contiguous. It proceeds backwards over the instructions in this order,
+ * marking registers live at their uses, ending their live ranges at
+ * definitions, and recording which registers are live at the top of every
+ * block. To deal with loop backedges, registers live at the beginning of
+ * a loop gain a range covering the entire loop.
+ */
+bool BacktrackingAllocator::buildLivenessInfo() {
+ JitSpew(JitSpew_RegAlloc, "Beginning liveness analysis");
+
+ Vector<MBasicBlock*, 1, SystemAllocPolicy> loopWorkList;
+ BitSet loopDone(graph.numBlockIds());
+ if (!loopDone.init(alloc())) {
+ return false;
+ }
+
+ size_t numRanges = 0;
+
+ for (size_t i = graph.numBlocks(); i > 0; i--) {
+ if (mir->shouldCancel("Build Liveness Info (main loop)")) {
+ return false;
+ }
+
+ LBlock* block = graph.getBlock(i - 1);
+ MBasicBlock* mblock = block->mir();
+
+ BitSet& live = liveIn[mblock->id()];
+ new (&live) BitSet(graph.numVirtualRegisters());
+ if (!live.init(alloc())) {
+ return false;
+ }
+
+ // Propagate liveIn from our successors to us.
+ for (size_t i = 0; i < mblock->lastIns()->numSuccessors(); i++) {
+ MBasicBlock* successor = mblock->lastIns()->getSuccessor(i);
+ // Skip backedges, as we fix them up at the loop header.
+ if (mblock->id() < successor->id()) {
+ live.insertAll(liveIn[successor->id()]);
+ }
+ }
+
+ // Add successor phis.
+ if (mblock->successorWithPhis()) {
+ LBlock* phiSuccessor = mblock->successorWithPhis()->lir();
+ for (unsigned int j = 0; j < phiSuccessor->numPhis(); j++) {
+ LPhi* phi = phiSuccessor->getPhi(j);
+ LAllocation* use = phi->getOperand(mblock->positionInPhiSuccessor());
+ uint32_t reg = use->toUse()->virtualRegister();
+ live.insert(reg);
+ vreg(use).setUsedByPhi();
+ }
+ }
+
+ // Registers are assumed alive for the entire block, a define shortens
+ // the range to the point of definition.
+ for (BitSet::Iterator liveRegId(live); liveRegId; ++liveRegId) {
+ if (!vregs[*liveRegId].addInitialRange(alloc(), entryOf(block),
+ exitOf(block).next(), &numRanges))
+ return false;
+ }
+
+ // Shorten the front end of ranges for live variables to their point of
+ // definition, if found.
+ for (LInstructionReverseIterator ins = block->rbegin();
+ ins != block->rend(); ins++) {
+ // Calls may clobber registers, so force a spill and reload around the
+ // callsite.
+ if (ins->isCall()) {
+ for (AnyRegisterIterator iter(allRegisters_.asLiveSet()); iter.more();
+ ++iter) {
+ bool found = false;
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ if (ins->getDef(i)->isFixed() &&
+ ins->getDef(i)->output()->aliases(LAllocation(*iter))) {
+ found = true;
+ break;
+ }
+ }
+ // If this register doesn't have an explicit def above, mark
+ // it as clobbered by the call unless it is actually
+ // call-preserved.
+ if (!found && !ins->isCallPreserved(*iter)) {
+ if (!addInitialFixedRange(*iter, outputOf(*ins),
+ outputOf(*ins).next())) {
+ return false;
+ }
+ }
+ }
+
+ CallRange* callRange = new (alloc().fallible())
+ CallRange(outputOf(*ins), outputOf(*ins).next());
+ if (!callRange) {
+ return false;
+ }
+
+ callRangesList.pushFront(callRange);
+ if (!callRanges.insert(callRange)) {
+ return false;
+ }
+ }
+
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ LDefinition* def = ins->getDef(i);
+ if (def->isBogusTemp()) {
+ continue;
+ }
+
+ CodePosition from = outputOf(*ins);
+
+ if (def->policy() == LDefinition::MUST_REUSE_INPUT) {
+ // MUST_REUSE_INPUT is implemented by allocating an output
+ // register and moving the input to it. Register hints are
+ // used to avoid unnecessary moves. We give the input an
+ // LUse::ANY policy to avoid allocating a register for the
+ // input.
+ LUse* inputUse = ins->getOperand(def->getReusedInput())->toUse();
+ MOZ_ASSERT(inputUse->policy() == LUse::REGISTER);
+ MOZ_ASSERT(inputUse->usedAtStart());
+ *inputUse = LUse(inputUse->virtualRegister(), LUse::ANY,
+ /* usedAtStart = */ true);
+ }
+
+ if (!vreg(def).addInitialRange(alloc(), from, from.next(),
+ &numRanges)) {
+ return false;
+ }
+ vreg(def).setInitialDefinition(from);
+ live.remove(def->virtualRegister());
+ }
+
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ LDefinition* temp = ins->getTemp(i);
+ if (temp->isBogusTemp()) {
+ continue;
+ }
+
+ // Normally temps are considered to cover both the input
+ // and output of the associated instruction. In some cases
+ // though we want to use a fixed register as both an input
+ // and clobbered register in the instruction, so watch for
+ // this and shorten the temp to cover only the output.
+ CodePosition from = inputOf(*ins);
+ if (temp->policy() == LDefinition::FIXED) {
+ AnyRegister reg = temp->output()->toRegister();
+ for (LInstruction::InputIterator alloc(**ins); alloc.more();
+ alloc.next()) {
+ if (alloc->isUse()) {
+ LUse* use = alloc->toUse();
+ if (use->isFixedRegister()) {
+ if (GetFixedRegister(vreg(use).def(), use) == reg) {
+ from = outputOf(*ins);
+ }
+ }
+ }
+ }
+ }
+
+ // * For non-call instructions, temps cover both the input and output,
+ // so temps never alias uses (even at-start uses) or defs.
+ // * For call instructions, temps only cover the input (the output is
+ // used for the force-spill ranges added above). This means temps
+ // still don't alias uses but they can alias the (fixed) defs. For now
+ // we conservatively require temps to have a fixed register for call
+ // instructions to prevent a footgun.
+ MOZ_ASSERT_IF(ins->isCall(), temp->policy() == LDefinition::FIXED);
+ CodePosition to =
+ ins->isCall() ? outputOf(*ins) : outputOf(*ins).next();
+
+ if (!vreg(temp).addInitialRange(alloc(), from, to, &numRanges)) {
+ return false;
+ }
+ vreg(temp).setInitialDefinition(from);
+ }
+
+ DebugOnly<bool> hasUseRegister = false;
+ DebugOnly<bool> hasUseRegisterAtStart = false;
+
+ for (LInstruction::InputIterator inputAlloc(**ins); inputAlloc.more();
+ inputAlloc.next()) {
+ if (inputAlloc->isUse()) {
+ LUse* use = inputAlloc->toUse();
+
+ // Call uses should always be at-start, since calls use all
+ // registers.
+ MOZ_ASSERT_IF(ins->isCall() && !inputAlloc.isSnapshotInput(),
+ use->usedAtStart());
+
+#ifdef DEBUG
+ // If there are both useRegisterAtStart(x) and useRegister(y)
+ // uses, we may assign the same register to both operands
+ // (bug 772830). Don't allow this for now.
+ if (use->policy() == LUse::REGISTER) {
+ if (use->usedAtStart()) {
+ if (!IsInputReused(*ins, use)) {
+ hasUseRegisterAtStart = true;
+ }
+ } else {
+ hasUseRegister = true;
+ }
+ }
+ MOZ_ASSERT(!(hasUseRegister && hasUseRegisterAtStart));
+#endif
+
+ // Don't treat RECOVERED_INPUT uses as keeping the vreg alive.
+ if (use->policy() == LUse::RECOVERED_INPUT) {
+ continue;
+ }
+
+ CodePosition to = use->usedAtStart() ? inputOf(*ins) : outputOf(*ins);
+ if (use->isFixedRegister()) {
+ LAllocation reg(AnyRegister::FromCode(use->registerCode()));
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ LDefinition* def = ins->getDef(i);
+ if (def->policy() == LDefinition::FIXED &&
+ *def->output() == reg) {
+ to = inputOf(*ins);
+ }
+ }
+ }
+
+ if (!vreg(use).addInitialRange(alloc(), entryOf(block), to.next(),
+ &numRanges)) {
+ return false;
+ }
+ UsePosition* usePosition =
+ new (alloc().fallible()) UsePosition(use, to);
+ if (!usePosition) {
+ return false;
+ }
+ vreg(use).addInitialUse(usePosition);
+ live.insert(use->virtualRegister());
+ }
+ }
+ }
+
+ // Phis have simultaneous assignment semantics at block begin, so at
+ // the beginning of the block we can be sure that liveIn does not
+ // contain any phi outputs.
+ for (unsigned int i = 0; i < block->numPhis(); i++) {
+ LDefinition* def = block->getPhi(i)->getDef(0);
+ if (live.contains(def->virtualRegister())) {
+ live.remove(def->virtualRegister());
+ } else {
+ // This is a dead phi, so add a dummy range over all phis. This
+ // can go away if we have an earlier dead code elimination pass.
+ CodePosition entryPos = entryOf(block);
+ if (!vreg(def).addInitialRange(alloc(), entryPos, entryPos.next(),
+ &numRanges)) {
+ return false;
+ }
+ }
+ }
+
+ if (mblock->isLoopHeader()) {
+ // A divergence from the published algorithm is required here, as
+ // our block order does not guarantee that blocks of a loop are
+ // contiguous. As a result, a single live range spanning the
+ // loop is not possible. Additionally, we require liveIn in a later
+ // pass for resolution, so that must also be fixed up here.
+ MBasicBlock* loopBlock = mblock->backedge();
+ while (true) {
+ // Blocks must already have been visited to have a liveIn set.
+ MOZ_ASSERT(loopBlock->id() >= mblock->id());
+
+ // Add a range for this entire loop block
+ CodePosition from = entryOf(loopBlock->lir());
+ CodePosition to = exitOf(loopBlock->lir()).next();
+
+ for (BitSet::Iterator liveRegId(live); liveRegId; ++liveRegId) {
+ if (!vregs[*liveRegId].addInitialRange(alloc(), from, to,
+ &numRanges)) {
+ return false;
+ }
+ }
+
+ // Fix up the liveIn set.
+ liveIn[loopBlock->id()].insertAll(live);
+
+ // Make sure we don't visit this node again
+ loopDone.insert(loopBlock->id());
+
+ // If this is the loop header, any predecessors are either the
+ // backedge or out of the loop, so skip any predecessors of
+ // this block
+ if (loopBlock != mblock) {
+ for (size_t i = 0; i < loopBlock->numPredecessors(); i++) {
+ MBasicBlock* pred = loopBlock->getPredecessor(i);
+ if (loopDone.contains(pred->id())) {
+ continue;
+ }
+ if (!loopWorkList.append(pred)) {
+ return false;
+ }
+ }
+ }
+
+ // Terminate loop if out of work.
+ if (loopWorkList.empty()) {
+ break;
+ }
+
+ // Grab the next block off the work list, skipping any OSR block.
+ MBasicBlock* osrBlock = graph.mir().osrBlock();
+ while (!loopWorkList.empty()) {
+ loopBlock = loopWorkList.popCopy();
+ if (loopBlock != osrBlock) {
+ break;
+ }
+ }
+
+ // If end is reached without finding a non-OSR block, then no more work
+ // items were found.
+ if (loopBlock == osrBlock) {
+ MOZ_ASSERT(loopWorkList.empty());
+ break;
+ }
+ }
+
+ // Clear the done set for other loops
+ loopDone.clear();
+ }
+
+ MOZ_ASSERT_IF(!mblock->numPredecessors(), live.empty());
+ }
+
+ JitSpew(JitSpew_RegAlloc, "Completed liveness analysis");
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Merging and queueing of LiveRange groups //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+// Helper for ::tryMergeBundles
+static bool IsArgumentSlotDefinition(LDefinition* def) {
+ return def->policy() == LDefinition::FIXED && def->output()->isArgument();
+}
+
+// Helper for ::tryMergeBundles
+static bool IsThisSlotDefinition(LDefinition* def) {
+ return IsArgumentSlotDefinition(def) &&
+ def->output()->toArgument()->index() <
+ THIS_FRAME_ARGSLOT + sizeof(Value);
+}
+
+// Helper for ::tryMergeBundles
+static bool HasStackPolicy(LDefinition* def) {
+ return def->policy() == LDefinition::STACK;
+}
+
+// Helper for ::tryMergeBundles
+static bool CanMergeTypesInBundle(LDefinition::Type a, LDefinition::Type b) {
+ // Fast path for the common case.
+ if (a == b) {
+ return true;
+ }
+
+ // Only merge if the sizes match, so that we don't get confused about the
+ // width of spill slots.
+ return StackSlotAllocator::width(a) == StackSlotAllocator::width(b);
+}
+
+// Helper for ::tryMergeReusedRegister
+bool BacktrackingAllocator::tryMergeBundles(LiveBundle* bundle0,
+ LiveBundle* bundle1) {
+ // See if bundle0 and bundle1 can be merged together.
+ if (bundle0 == bundle1) {
+ return true;
+ }
+
+ // Get a representative virtual register from each bundle.
+ VirtualRegister& reg0 = bundle0->firstRange()->vreg();
+ VirtualRegister& reg1 = bundle1->firstRange()->vreg();
+
+ MOZ_ASSERT(CanMergeTypesInBundle(reg0.type(), reg1.type()));
+ MOZ_ASSERT(reg0.isCompatible(reg1));
+
+ // Registers which might spill to the frame's |this| slot can only be
+ // grouped with other such registers. The frame's |this| slot must always
+ // hold the |this| value, as required by JitFrame tracing and by the Ion
+ // constructor calling convention.
+ if (IsThisSlotDefinition(reg0.def()) || IsThisSlotDefinition(reg1.def())) {
+ if (*reg0.def()->output() != *reg1.def()->output()) {
+ return true;
+ }
+ }
+
+ // Registers which might spill to the frame's argument slots can only be
+ // grouped with other such registers if the frame might access those
+ // arguments through a lazy arguments object or rest parameter.
+ if (IsArgumentSlotDefinition(reg0.def()) ||
+ IsArgumentSlotDefinition(reg1.def())) {
+ if (graph.mir().entryBlock()->info().mayReadFrameArgsDirectly()) {
+ if (*reg0.def()->output() != *reg1.def()->output()) {
+ return true;
+ }
+ }
+ }
+
+ // When we make a call to a WebAssembly function that returns multiple
+ // results, some of those results can go on the stack. The callee is passed a
+ // pointer to this stack area, which is represented as having policy
+ // LDefinition::STACK (with type LDefinition::STACKRESULTS). Individual
+ // results alias parts of the stack area with a value-appropriate type, but
+ // policy LDefinition::STACK. This aliasing between allocations makes it
+ // unsound to merge anything with a LDefinition::STACK policy.
+ if (HasStackPolicy(reg0.def()) || HasStackPolicy(reg1.def())) {
+ return true;
+ }
+
+ // Limit the number of times we compare ranges if there are many ranges in
+ // one of the bundles, to avoid quadratic behavior.
+ static const size_t MAX_RANGES = 200;
+
+ // Make sure that ranges in the bundles do not overlap.
+ LiveRange::BundleLinkIterator iter0 = bundle0->rangesBegin(),
+ iter1 = bundle1->rangesBegin();
+ size_t count = 0;
+ while (iter0 && iter1) {
+ if (++count >= MAX_RANGES) {
+ return true;
+ }
+
+ LiveRange* range0 = LiveRange::get(*iter0);
+ LiveRange* range1 = LiveRange::get(*iter1);
+
+ if (range0->from() >= range1->to()) {
+ iter1++;
+ } else if (range1->from() >= range0->to()) {
+ iter0++;
+ } else {
+ return true;
+ }
+ }
+
+ // Move all ranges from bundle1 into bundle0.
+ while (LiveRange* range = bundle1->popFirstRange()) {
+ bundle0->addRange(range);
+ }
+
+ return true;
+}
+
+// Helper for ::mergeAndQueueRegisters
+void BacktrackingAllocator::allocateStackDefinition(VirtualRegister& reg) {
+ LInstruction* ins = reg.ins()->toInstruction();
+ if (reg.def()->type() == LDefinition::STACKRESULTS) {
+ LStackArea alloc(ins->toInstruction());
+ stackSlotAllocator.allocateStackArea(&alloc);
+ reg.def()->setOutput(alloc);
+ } else {
+ // Because the definitions are visited in order, the area has been allocated
+ // before we reach this result, so we know the operand is an LStackArea.
+ const LUse* use = ins->getOperand(0)->toUse();
+ VirtualRegister& area = vregs[use->virtualRegister()];
+ const LStackArea* areaAlloc = area.def()->output()->toStackArea();
+ reg.def()->setOutput(areaAlloc->resultAlloc(ins, reg.def()));
+ }
+}
+
+// Helper for ::mergeAndQueueRegisters
+bool BacktrackingAllocator::tryMergeReusedRegister(VirtualRegister& def,
+ VirtualRegister& input) {
+ // def is a vreg which reuses input for its output physical register. Try
+ // to merge ranges for def with those of input if possible, as avoiding
+ // copies before def's instruction is crucial for generated code quality
+ // (MUST_REUSE_INPUT is used for all arithmetic on x86/x64).
+
+ if (def.rangeFor(inputOf(def.ins()))) {
+ MOZ_ASSERT(def.isTemp());
+ def.setMustCopyInput();
+ return true;
+ }
+
+ if (!CanMergeTypesInBundle(def.type(), input.type())) {
+ def.setMustCopyInput();
+ return true;
+ }
+
+ LiveRange* inputRange = input.rangeFor(outputOf(def.ins()));
+ if (!inputRange) {
+ // The input is not live after the instruction, either in a safepoint
+ // for the instruction or in subsequent code. The input and output
+ // can thus be in the same group.
+ return tryMergeBundles(def.firstBundle(), input.firstBundle());
+ }
+
+ // Avoid merging in very large live ranges as merging has non-linear
+ // complexity. The cutoff value is hard to gauge. 1M was chosen because it
+ // is "large" and yet usefully caps compile time on AutoCad-for-the-web to
+ // something reasonable on a 2017-era desktop system.
+ const uint32_t RANGE_SIZE_CUTOFF = 1000000;
+ if (inputRange->to() - inputRange->from() > RANGE_SIZE_CUTOFF) {
+ def.setMustCopyInput();
+ return true;
+ }
+
+ // The input is live afterwards, either in future instructions or in a
+ // safepoint for the reusing instruction. This is impossible to satisfy
+ // without copying the input.
+ //
+ // It may or may not be better to split the input into two bundles at the
+ // point of the definition, which may permit merging. One case where it is
+ // definitely better to split is if the input never has any register uses
+ // after the instruction. Handle this splitting eagerly.
+
+ LBlock* block = def.ins()->block();
+
+ // The input's lifetime must end within the same block as the definition,
+ // otherwise it could live on in phis elsewhere.
+ if (inputRange != input.lastRange() || inputRange->to() > exitOf(block)) {
+ def.setMustCopyInput();
+ return true;
+ }
+
+ // If we already split the input for some other register, don't make a
+ // third bundle.
+ if (inputRange->bundle() != input.firstRange()->bundle()) {
+ def.setMustCopyInput();
+ return true;
+ }
+
+ // If the input will start out in memory then adding a separate bundle for
+ // memory uses after the def won't help.
+ if (input.def()->isFixed() && !input.def()->output()->isRegister()) {
+ def.setMustCopyInput();
+ return true;
+ }
+
+ // The input cannot have register or reused uses after the definition.
+ for (UsePositionIterator iter = inputRange->usesBegin(); iter; iter++) {
+ if (iter->pos <= inputOf(def.ins())) {
+ continue;
+ }
+
+ LUse* use = iter->use();
+ if (FindReusingDefOrTemp(insData[iter->pos], use)) {
+ def.setMustCopyInput();
+ return true;
+ }
+ if (iter->usePolicy() != LUse::ANY &&
+ iter->usePolicy() != LUse::KEEPALIVE) {
+ def.setMustCopyInput();
+ return true;
+ }
+ }
+
+ LiveRange* preRange = LiveRange::FallibleNew(
+ alloc(), &input, inputRange->from(), outputOf(def.ins()));
+ if (!preRange) {
+ return false;
+ }
+
+ // The new range starts at reg's input position, which means it overlaps
+ // with the old range at one position. This is what we want, because we
+ // need to copy the input before the instruction.
+ LiveRange* postRange = LiveRange::FallibleNew(
+ alloc(), &input, inputOf(def.ins()), inputRange->to());
+ if (!postRange) {
+ return false;
+ }
+
+ inputRange->tryToMoveDefAndUsesInto(preRange);
+ inputRange->tryToMoveDefAndUsesInto(postRange);
+ MOZ_ASSERT(!inputRange->hasUses());
+
+ JitSpewIfEnabled(JitSpew_RegAlloc,
+ " splitting reused input at %u to try to help grouping",
+ inputOf(def.ins()).bits());
+
+ LiveBundle* firstBundle = inputRange->bundle();
+ input.removeRange(inputRange);
+ input.addRange(preRange);
+ input.addRange(postRange);
+
+ firstBundle->removeRange(inputRange);
+ firstBundle->addRange(preRange);
+
+ // The new range goes in a separate bundle, where it will be spilled during
+ // allocation.
+ LiveBundle* secondBundle = LiveBundle::FallibleNew(alloc(), nullptr, nullptr);
+ if (!secondBundle) {
+ return false;
+ }
+ secondBundle->addRange(postRange);
+
+ return tryMergeBundles(def.firstBundle(), input.firstBundle());
+}
+
+bool BacktrackingAllocator::mergeAndQueueRegisters() {
+ MOZ_ASSERT(!vregs[0u].hasRanges());
+
+ // Create a bundle for each register containing all its ranges.
+ for (size_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+ if (!reg.hasRanges()) {
+ continue;
+ }
+
+ LiveBundle* bundle = LiveBundle::FallibleNew(alloc(), nullptr, nullptr);
+ if (!bundle) {
+ return false;
+ }
+ for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ bundle->addRange(range);
+ }
+ }
+
+ // If there is an OSR block, merge parameters in that block with the
+ // corresponding parameters in the initial block.
+ if (MBasicBlock* osr = graph.mir().osrBlock()) {
+ size_t original = 1;
+ for (LInstructionIterator iter = osr->lir()->begin();
+ iter != osr->lir()->end(); iter++) {
+ if (iter->isParameter()) {
+ for (size_t i = 0; i < iter->numDefs(); i++) {
+ DebugOnly<bool> found = false;
+ VirtualRegister& paramVreg = vreg(iter->getDef(i));
+ for (; original < paramVreg.vreg(); original++) {
+ VirtualRegister& originalVreg = vregs[original];
+ if (*originalVreg.def()->output() == *iter->getDef(i)->output()) {
+ MOZ_ASSERT(originalVreg.ins()->isParameter());
+ if (!tryMergeBundles(originalVreg.firstBundle(),
+ paramVreg.firstBundle())) {
+ return false;
+ }
+ found = true;
+ break;
+ }
+ }
+ MOZ_ASSERT(found);
+ }
+ }
+ }
+ }
+
+ // Try to merge registers with their reused inputs.
+ for (size_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+ if (!reg.hasRanges()) {
+ continue;
+ }
+
+ if (reg.def()->policy() == LDefinition::MUST_REUSE_INPUT) {
+ LUse* use = reg.ins()
+ ->toInstruction()
+ ->getOperand(reg.def()->getReusedInput())
+ ->toUse();
+ if (!tryMergeReusedRegister(reg, vreg(use))) {
+ return false;
+ }
+ }
+ }
+
+ // Try to merge phis with their inputs.
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ LBlock* block = graph.getBlock(i);
+ for (size_t j = 0; j < block->numPhis(); j++) {
+ LPhi* phi = block->getPhi(j);
+ VirtualRegister& outputVreg = vreg(phi->getDef(0));
+ for (size_t k = 0, kend = phi->numOperands(); k < kend; k++) {
+ VirtualRegister& inputVreg = vreg(phi->getOperand(k)->toUse());
+ if (!tryMergeBundles(inputVreg.firstBundle(),
+ outputVreg.firstBundle())) {
+ return false;
+ }
+ }
+ }
+ }
+
+ // Add all bundles to the allocation queue, and create spill sets for them.
+ for (size_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+
+ // Eagerly allocate stack result areas and their component stack results.
+ if (reg.def() && reg.def()->policy() == LDefinition::STACK) {
+ allocateStackDefinition(reg);
+ }
+
+ for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ LiveBundle* bundle = range->bundle();
+ if (range == bundle->firstRange()) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ SpillSet* spill = SpillSet::New(alloc());
+ if (!spill) {
+ return false;
+ }
+ bundle->setSpillSet(spill);
+
+ size_t priority = computePriority(bundle);
+ if (!allocationQueue.insert(QueueItem(bundle, priority))) {
+ return false;
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Code for the splitting/spilling subsystem begins here. //
+// //
+// The code that follows is structured in the following sequence: //
+// //
+// (1) Routines that are helpers for ::splitAt. //
+// (2) ::splitAt itself, which implements splitting decisions. //
+// (3) heuristic routines (eg ::splitAcrossCalls), which decide where //
+// splits should be made. They then call ::splitAt to perform the //
+// chosen split. //
+// (4) The top level driver, ::chooseBundleSplit. //
+// //
+// There are further comments on ::splitAt and ::chooseBundleSplit below. //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Implementation of splitting decisions, but not the making of those //
+// decisions: various helper functions //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+bool BacktrackingAllocator::updateVirtualRegisterListsThenRequeueBundles(
+ LiveBundle* bundle, const LiveBundleVector& newBundles) {
+#ifdef DEBUG
+ if (newBundles.length() == 1) {
+ LiveBundle* newBundle = newBundles[0];
+ if (newBundle->numRanges() == bundle->numRanges() &&
+ computePriority(newBundle) == computePriority(bundle)) {
+ bool different = false;
+ LiveRange::BundleLinkIterator oldRanges = bundle->rangesBegin();
+ LiveRange::BundleLinkIterator newRanges = newBundle->rangesBegin();
+ while (oldRanges) {
+ LiveRange* oldRange = LiveRange::get(*oldRanges);
+ LiveRange* newRange = LiveRange::get(*newRanges);
+ if (oldRange->from() != newRange->from() ||
+ oldRange->to() != newRange->to()) {
+ different = true;
+ break;
+ }
+ oldRanges++;
+ newRanges++;
+ }
+
+ // This is likely to trigger an infinite loop in register allocation. This
+ // can be the result of invalid register constraints, making regalloc
+ // impossible; consider relaxing those.
+ MOZ_ASSERT(different,
+ "Split results in the same bundle with the same priority");
+ }
+ }
+#endif
+
+ if (JitSpewEnabled(JitSpew_RegAlloc)) {
+ JitSpew(JitSpew_RegAlloc, " .. into:");
+ for (size_t i = 0; i < newBundles.length(); i++) {
+ JitSpew(JitSpew_RegAlloc, " %s", newBundles[i]->toString().get());
+ }
+ }
+
+ // Remove all ranges in the old bundle from their register's list.
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ range->vreg().removeRange(range);
+ }
+
+ // Add all ranges in the new bundles to their register's list.
+ for (size_t i = 0; i < newBundles.length(); i++) {
+ LiveBundle* newBundle = newBundles[i];
+ for (LiveRange::BundleLinkIterator iter = newBundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ range->vreg().addRange(range);
+ }
+ }
+
+ // Queue the new bundles for register assignment.
+ for (size_t i = 0; i < newBundles.length(); i++) {
+ LiveBundle* newBundle = newBundles[i];
+ size_t priority = computePriority(newBundle);
+ if (!allocationQueue.insert(QueueItem(newBundle, priority))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// Helper for ::splitAt
+// When splitting a bundle according to a list of split positions, return
+// whether a use or range at |pos| should use a different bundle than the last
+// position this was called for.
+static bool UseNewBundle(const SplitPositionVector& splitPositions,
+ CodePosition pos, size_t* activeSplitPosition) {
+ if (splitPositions.empty()) {
+ // When the split positions are empty we are splitting at all uses.
+ return true;
+ }
+
+ if (*activeSplitPosition == splitPositions.length()) {
+ // We've advanced past all split positions.
+ return false;
+ }
+
+ if (splitPositions[*activeSplitPosition] > pos) {
+ // We haven't gotten to the next split position yet.
+ return false;
+ }
+
+ // We've advanced past the next split position, find the next one which we
+ // should split at.
+ while (*activeSplitPosition < splitPositions.length() &&
+ splitPositions[*activeSplitPosition] <= pos) {
+ (*activeSplitPosition)++;
+ }
+ return true;
+}
+
+// Helper for ::splitAt
+static bool HasPrecedingRangeSharingVreg(LiveBundle* bundle, LiveRange* range) {
+ MOZ_ASSERT(range->bundle() == bundle);
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* prevRange = LiveRange::get(*iter);
+ if (prevRange == range) {
+ return false;
+ }
+ if (&prevRange->vreg() == &range->vreg()) {
+ return true;
+ }
+ }
+
+ MOZ_CRASH();
+}
+
+// Helper for ::splitAt
+static bool HasFollowingRangeSharingVreg(LiveBundle* bundle, LiveRange* range) {
+ MOZ_ASSERT(range->bundle() == bundle);
+
+ bool foundRange = false;
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* prevRange = LiveRange::get(*iter);
+ if (foundRange && &prevRange->vreg() == &range->vreg()) {
+ return true;
+ }
+ if (prevRange == range) {
+ foundRange = true;
+ }
+ }
+
+ MOZ_ASSERT(foundRange);
+ return false;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Implementation of splitting decisions, but not the making of those //
+// decisions: //
+// ::splitAt //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+// ::splitAt
+// ---------
+// It would be nice to be able to interpret ::splitAt as simply performing
+// whatever split the heuristic routines decide on. Unfortunately it
+// tries to "improve" on the initial locations, which as
+// https://bugzilla.mozilla.org/show_bug.cgi?id=1758274#c17 shows, often
+// leads to excessive spilling. So there is no clean distinction between
+// policy (where to split, computed by the heuristic routines) and
+// implementation (done by ::splitAt).
+//
+// ::splitAt -- creation of spill parent bundles
+// ---------------------------------------------
+// To understand what ::splitAt does, we must refer back to Section 1's
+// description of LiveBundle::spillParent_.
+//
+// Initially (as created by Phase 1), all bundles have `spillParent_` being
+// NULL. If ::splitAt is asked to split such a bundle, it will first create a
+// "spill bundle" or "spill parent" bundle. This is a copy of the original,
+// with two changes:
+//
+// * all register uses have been removed, so that only stack-compatible uses
+// remain.
+//
+// * for all LiveRanges in the bundle that define a register, the start point
+// of the range is moved one CodePosition forwards, thusly:
+//
+// from = minimalDefEnd(insData[from]).next();
+//
+// The reason for the latter relates to the idea described in Section 1, that
+// all LiveRanges for any given VirtualRegister must form a tree rooted at the
+// defining LiveRange. If the spill-bundle definition range start points are
+// the same as those in the original bundle, then we will end up with two roots
+// for the tree, and it is then unclear which one should supply "the value".
+//
+// Putting the spill-bundle start point one CodePosition further along causes
+// the range containing the register def (after splitting) to still be the
+// defining point. ::createMoveGroupsFromLiveRangeTransitions will observe the
+// equivalent spill-bundle range starting one point later and add a MoveGroup
+// to move the value into it. Since the spill bundle is intended to be stack
+// resident, the effect is to force creation of the MoveGroup that will
+// actually spill this value onto the stack.
+//
+// If the bundle provided to ::splitAt already has a spill parent, then
+// ::splitAt doesn't create a new spill parent. This situation will happen
+// when the bundle to be split was itself created by splitting. The effect is
+// that *all* bundles created from an "original bundle" share the same spill
+// parent, and hence they will share the same spill slot, which guarantees that
+// all the spilled fragments of a VirtualRegister share the same spill slot,
+// which means we'll never have to move a VirtualRegister between different
+// spill slots during its lifetime.
+//
+// ::splitAt -- creation of other bundles
+// --------------------------------------
+// With the spill parent bundle question out of the way, ::splitAt then goes on
+// to create the remaining new bundles, using near-incomprehensible logic
+// steered by `UseNewBundle`.
+//
+// This supposedly splits the bundle at the positions given by the
+// `SplitPositionVector` parameter to ::splitAt, putting them in a temporary
+// vector `newBundles`. Whether it really splits at the requested positions or
+// not is hard to say; more important is what happens next.
+//
+// ::splitAt -- "improvement" ("filtering") of the split bundles
+// -------------------------------------------------------------
+// ::splitAt now tries to reduce the length of the LiveRanges that make up the
+// new bundles (not including the "spill parent"). I assume this is to remove
+// sections of the bundles that carry no useful value (eg, extending after the
+// last using a range), thereby removing the demand for registers in those
+// parts. This does however mean that ::splitAt is no longer really splitting
+// where the heuristic routines wanted, and that can lead to a big increase in
+// spilling in loops, as
+// https://bugzilla.mozilla.org/show_bug.cgi?id=1758274#c17 describes.
+//
+// ::splitAt -- meaning of the incoming `SplitPositionVector`
+// ----------------------------------------------------------
+// ::splitAt has one last mystery which is important to document. The split
+// positions are specified as CodePositions, but this leads to ambiguity
+// because, in a sequence of N (LIR) instructions, there are 2N valid
+// CodePositions. For example:
+//
+// 6-7 WasmLoadTls [def v2<o>] [use v1:R]
+// 8-9 WasmNullConstant [def v3<o>]
+//
+// Consider splitting the range for `v2`, which starts at CodePosition 7.
+// What's the difference between saying "split it at 7" and "split it at 8" ?
+// Not much really, since in both cases what we intend is for the range to be
+// split in between the two instructions.
+//
+// Hence I believe the semantics is:
+//
+// * splitting at an even numbered CodePosition (eg, 8), which is an input-side
+// position, means "split before the instruction containing this position".
+//
+// * splitting at an odd numbered CodePositin (eg, 7), which is an output-side
+// position, means "split after the instruction containing this position".
+//
+// Hence in the example, we could specify either 7 or 8 to mean the same
+// placement of the split. Well, almost true, but actually:
+//
+// (SPEC) specifying 8 means
+//
+// "split between these two insns, and any resulting MoveGroup goes in the
+// list to be emitted before the start of the second insn"
+//
+// (SPEC) specifying 7 means
+//
+// "split between these two insns, and any resulting MoveGroup goes in the
+// list to be emitted after the end of the first insn"
+//
+// In most cases we don't care on which "side of the valley" the MoveGroup ends
+// up, in which case we can use either convention.
+//
+// (SPEC) I believe these semantics are implied by the logic in
+// ::createMoveGroupsFromLiveRangeTransitions. They are certainly not
+// documented anywhere in the code.
+
+bool BacktrackingAllocator::splitAt(LiveBundle* bundle,
+ const SplitPositionVector& splitPositions) {
+ // Split the bundle at the given split points. Register uses which have no
+ // intervening split points are consolidated into the same bundle. If the
+ // list of split points is empty, then all register uses are placed in
+ // minimal bundles.
+
+ // splitPositions should be sorted.
+ for (size_t i = 1; i < splitPositions.length(); ++i) {
+ MOZ_ASSERT(splitPositions[i - 1] < splitPositions[i]);
+ }
+
+ // We don't need to create a new spill bundle if there already is one.
+ bool spillBundleIsNew = false;
+ LiveBundle* spillBundle = bundle->spillParent();
+ if (!spillBundle) {
+ spillBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(), nullptr);
+ if (!spillBundle) {
+ return false;
+ }
+ spillBundleIsNew = true;
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ CodePosition from = range->from();
+ if (isRegisterDefinition(range)) {
+ from = minimalDefEnd(insData[from]).next();
+ }
+
+ if (from < range->to()) {
+ if (!spillBundle->addRange(alloc(), &range->vreg(), from,
+ range->to())) {
+ return false;
+ }
+
+ if (range->hasDefinition() && !isRegisterDefinition(range)) {
+ spillBundle->lastRange()->setHasDefinition();
+ }
+ }
+ }
+ }
+
+ LiveBundleVector newBundles;
+
+ // The bundle which ranges are currently being added to.
+ LiveBundle* activeBundle =
+ LiveBundle::FallibleNew(alloc(), bundle->spillSet(), spillBundle);
+ if (!activeBundle || !newBundles.append(activeBundle)) {
+ return false;
+ }
+
+ // State for use by UseNewBundle.
+ size_t activeSplitPosition = 0;
+
+ // Make new bundles according to the split positions, and distribute ranges
+ // and uses to them.
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ if (UseNewBundle(splitPositions, range->from(), &activeSplitPosition)) {
+ activeBundle =
+ LiveBundle::FallibleNew(alloc(), bundle->spillSet(), spillBundle);
+ if (!activeBundle || !newBundles.append(activeBundle)) {
+ return false;
+ }
+ }
+
+ LiveRange* activeRange = LiveRange::FallibleNew(alloc(), &range->vreg(),
+ range->from(), range->to());
+ if (!activeRange) {
+ return false;
+ }
+ activeBundle->addRange(activeRange);
+
+ if (isRegisterDefinition(range)) {
+ activeRange->setHasDefinition();
+ }
+
+ while (range->hasUses()) {
+ UsePosition* use = range->popUse();
+ LNode* ins = insData[use->pos];
+
+ // Any uses of a register that appear before its definition has
+ // finished must be associated with the range for that definition.
+ if (isRegisterDefinition(range) &&
+ use->pos <= minimalDefEnd(insData[range->from()])) {
+ activeRange->addUse(use);
+ } else if (isRegisterUse(use, ins)) {
+ // Place this register use into a different bundle from the
+ // last one if there are any split points between the two uses.
+ // UseNewBundle always returns true if we are splitting at all
+ // register uses, but we can still reuse the last range and
+ // bundle if they have uses at the same position, except when
+ // either use is fixed (the two uses might require incompatible
+ // registers.)
+ if (UseNewBundle(splitPositions, use->pos, &activeSplitPosition) &&
+ (!activeRange->hasUses() ||
+ activeRange->usesBegin()->pos != use->pos ||
+ activeRange->usesBegin()->usePolicy() == LUse::FIXED ||
+ use->usePolicy() == LUse::FIXED)) {
+ activeBundle =
+ LiveBundle::FallibleNew(alloc(), bundle->spillSet(), spillBundle);
+ if (!activeBundle || !newBundles.append(activeBundle)) {
+ return false;
+ }
+ activeRange = LiveRange::FallibleNew(alloc(), &range->vreg(),
+ range->from(), range->to());
+ if (!activeRange) {
+ return false;
+ }
+ activeBundle->addRange(activeRange);
+ }
+
+ activeRange->addUse(use);
+ } else {
+ MOZ_ASSERT(spillBundleIsNew);
+ spillBundle->rangeFor(use->pos)->addUse(use);
+ }
+ }
+ }
+
+ LiveBundleVector filteredBundles;
+
+ // Trim the ends of ranges in each new bundle when there are no other
+ // earlier or later ranges in the same bundle with the same vreg.
+ for (size_t i = 0; i < newBundles.length(); i++) {
+ LiveBundle* bundle = newBundles[i];
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ if (!range->hasDefinition()) {
+ if (!HasPrecedingRangeSharingVreg(bundle, range)) {
+ if (range->hasUses()) {
+ UsePosition* use = *range->usesBegin();
+ range->setFrom(inputOf(insData[use->pos]));
+ } else {
+ bundle->removeRangeAndIncrementIterator(iter);
+ continue;
+ }
+ }
+ }
+
+ if (!HasFollowingRangeSharingVreg(bundle, range)) {
+ if (range->hasUses()) {
+ UsePosition* use = range->lastUse();
+ range->setTo(use->pos.next());
+ } else if (range->hasDefinition()) {
+ range->setTo(minimalDefEnd(insData[range->from()]).next());
+ } else {
+ bundle->removeRangeAndIncrementIterator(iter);
+ continue;
+ }
+ }
+
+ iter++;
+ }
+
+ if (bundle->hasRanges() && !filteredBundles.append(bundle)) {
+ return false;
+ }
+ }
+
+ if (spillBundleIsNew && !filteredBundles.append(spillBundle)) {
+ return false;
+ }
+
+ return updateVirtualRegisterListsThenRequeueBundles(bundle, filteredBundles);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Creation of splitting decisions, but not their implementation: //
+// ::splitAcrossCalls //
+// ::trySplitAcrossHotcode //
+// ::trySplitAfterLastRegisterUse //
+// ::trySplitBeforeFirstRegisterUse //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+bool BacktrackingAllocator::splitAcrossCalls(LiveBundle* bundle) {
+ // Split the bundle to separate register uses and non-register uses and
+ // allow the vreg to be spilled across its range.
+
+ // Find the locations of all calls in the bundle's range.
+ SplitPositionVector callPositions;
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ CallRange searchRange(range->from(), range->to());
+ CallRange* callRange;
+ if (!callRanges.contains(&searchRange, &callRange)) {
+ // There are no calls inside this range.
+ continue;
+ }
+ MOZ_ASSERT(range->covers(callRange->range.from));
+
+ // The search above returns an arbitrary call within the range. Walk
+ // backwards to find the first call in the range.
+ for (CallRangeList::reverse_iterator riter =
+ callRangesList.rbegin(callRange);
+ riter != callRangesList.rend(); ++riter) {
+ CodePosition pos = riter->range.from;
+ if (range->covers(pos)) {
+ callRange = *riter;
+ } else {
+ break;
+ }
+ }
+
+ // Add all call positions within the range, by walking forwards.
+ for (CallRangeList::iterator iter = callRangesList.begin(callRange);
+ iter != callRangesList.end(); ++iter) {
+ CodePosition pos = iter->range.from;
+ if (!range->covers(pos)) {
+ break;
+ }
+
+ // Calls at the beginning of the range are ignored; there is no splitting
+ // to do.
+ if (range->covers(pos.previous())) {
+ MOZ_ASSERT_IF(callPositions.length(), pos > callPositions.back());
+ if (!callPositions.append(pos)) {
+ return false;
+ }
+ }
+ }
+ }
+ MOZ_ASSERT(callPositions.length());
+
+#ifdef JS_JITSPEW
+ JitSpewStart(JitSpew_RegAlloc, " .. split across calls at ");
+ for (size_t i = 0; i < callPositions.length(); ++i) {
+ JitSpewCont(JitSpew_RegAlloc, "%s%u", i != 0 ? ", " : "",
+ callPositions[i].bits());
+ }
+ JitSpewFin(JitSpew_RegAlloc);
+#endif
+
+ return splitAt(bundle, callPositions);
+}
+
+bool BacktrackingAllocator::trySplitAcrossHotcode(LiveBundle* bundle,
+ bool* success) {
+ // If this bundle has portions that are hot and portions that are cold,
+ // split it at the boundaries between hot and cold code.
+
+ LiveRange* hotRange = nullptr;
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ if (hotcode.contains(range, &hotRange)) {
+ break;
+ }
+ }
+
+ // Don't split if there is no hot code in the bundle.
+ if (!hotRange) {
+ JitSpew(JitSpew_RegAlloc, " .. bundle does not contain hot code");
+ return true;
+ }
+
+ // Don't split if there is no cold code in the bundle.
+ bool coldCode = false;
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ if (!hotRange->contains(range)) {
+ coldCode = true;
+ break;
+ }
+ }
+ if (!coldCode) {
+ JitSpew(JitSpew_RegAlloc, " .. bundle does not contain cold code");
+ return true;
+ }
+
+ JitSpewIfEnabled(JitSpew_RegAlloc, " .. split across hot range %s",
+ hotRange->toString().get());
+
+ // Tweak the splitting method when compiling wasm code to look at actual
+ // uses within the hot/cold code. This heuristic is in place as the below
+ // mechanism regresses several asm.js tests. Hopefully this will be fixed
+ // soon and this special case removed. See bug 948838.
+ if (compilingWasm()) {
+ SplitPositionVector splitPositions;
+ if (!splitPositions.append(hotRange->from()) ||
+ !splitPositions.append(hotRange->to())) {
+ return false;
+ }
+ *success = true;
+ return splitAt(bundle, splitPositions);
+ }
+
+ LiveBundle* hotBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(),
+ bundle->spillParent());
+ if (!hotBundle) {
+ return false;
+ }
+ LiveBundle* preBundle = nullptr;
+ LiveBundle* postBundle = nullptr;
+ LiveBundle* coldBundle = nullptr;
+
+ if (testbed) {
+ coldBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(),
+ bundle->spillParent());
+ if (!coldBundle) {
+ return false;
+ }
+ }
+
+ // Accumulate the ranges of hot and cold code in the bundle. Note that
+ // we are only comparing with the single hot range found, so the cold code
+ // may contain separate hot ranges.
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ LiveRange::Range hot, coldPre, coldPost;
+ range->intersect(hotRange, &coldPre, &hot, &coldPost);
+
+ if (!hot.empty()) {
+ if (!hotBundle->addRangeAndDistributeUses(alloc(), range, hot.from,
+ hot.to)) {
+ return false;
+ }
+ }
+
+ if (!coldPre.empty()) {
+ if (testbed) {
+ if (!coldBundle->addRangeAndDistributeUses(alloc(), range, coldPre.from,
+ coldPre.to)) {
+ return false;
+ }
+ } else {
+ if (!preBundle) {
+ preBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(),
+ bundle->spillParent());
+ if (!preBundle) {
+ return false;
+ }
+ }
+ if (!preBundle->addRangeAndDistributeUses(alloc(), range, coldPre.from,
+ coldPre.to)) {
+ return false;
+ }
+ }
+ }
+
+ if (!coldPost.empty()) {
+ if (testbed) {
+ if (!coldBundle->addRangeAndDistributeUses(
+ alloc(), range, coldPost.from, coldPost.to)) {
+ return false;
+ }
+ } else {
+ if (!postBundle) {
+ postBundle = LiveBundle::FallibleNew(alloc(), bundle->spillSet(),
+ bundle->spillParent());
+ if (!postBundle) {
+ return false;
+ }
+ }
+ if (!postBundle->addRangeAndDistributeUses(
+ alloc(), range, coldPost.from, coldPost.to)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ MOZ_ASSERT(hotBundle->numRanges() != 0);
+
+ LiveBundleVector newBundles;
+ if (!newBundles.append(hotBundle)) {
+ return false;
+ }
+
+ if (testbed) {
+ MOZ_ASSERT(coldBundle->numRanges() != 0);
+ if (!newBundles.append(coldBundle)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(preBundle || postBundle);
+ if (preBundle && !newBundles.append(preBundle)) {
+ return false;
+ }
+ if (postBundle && !newBundles.append(postBundle)) {
+ return false;
+ }
+ }
+
+ *success = true;
+ return updateVirtualRegisterListsThenRequeueBundles(bundle, newBundles);
+}
+
+bool BacktrackingAllocator::trySplitAfterLastRegisterUse(LiveBundle* bundle,
+ LiveBundle* conflict,
+ bool* success) {
+ // If this bundle's later uses do not require it to be in a register,
+ // split it after the last use which does require a register. If conflict
+ // is specified, only consider register uses before the conflict starts.
+
+ CodePosition lastRegisterFrom, lastRegisterTo, lastUse;
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ // If the range defines a register, consider that a register use for
+ // our purposes here.
+ if (isRegisterDefinition(range)) {
+ CodePosition spillStart = minimalDefEnd(insData[range->from()]).next();
+ if (!conflict || spillStart < conflict->firstRange()->from()) {
+ lastUse = lastRegisterFrom = range->from();
+ lastRegisterTo = spillStart;
+ }
+ }
+
+ for (UsePositionIterator iter(range->usesBegin()); iter; iter++) {
+ LNode* ins = insData[iter->pos];
+
+ // Uses in the bundle should be sorted.
+ MOZ_ASSERT(iter->pos >= lastUse);
+ lastUse = inputOf(ins);
+
+ if (!conflict || outputOf(ins) < conflict->firstRange()->from()) {
+ if (isRegisterUse(*iter, ins, /* considerCopy = */ true)) {
+ lastRegisterFrom = inputOf(ins);
+ lastRegisterTo = iter->pos.next();
+ }
+ }
+ }
+ }
+
+ // Can't trim non-register uses off the end by splitting.
+ if (!lastRegisterFrom.bits()) {
+ JitSpew(JitSpew_RegAlloc, " .. bundle has no register uses");
+ return true;
+ }
+ if (lastUse < lastRegisterTo) {
+ JitSpew(JitSpew_RegAlloc, " .. bundle's last use is a register use");
+ return true;
+ }
+
+ JitSpewIfEnabled(JitSpew_RegAlloc, " .. split after last register use at %u",
+ lastRegisterTo.bits());
+
+ SplitPositionVector splitPositions;
+ if (!splitPositions.append(lastRegisterTo)) {
+ return false;
+ }
+ *success = true;
+ return splitAt(bundle, splitPositions);
+}
+
+bool BacktrackingAllocator::trySplitBeforeFirstRegisterUse(LiveBundle* bundle,
+ LiveBundle* conflict,
+ bool* success) {
+ // If this bundle's earlier uses do not require it to be in a register,
+ // split it before the first use which does require a register. If conflict
+ // is specified, only consider register uses after the conflict ends.
+
+ if (isRegisterDefinition(bundle->firstRange())) {
+ JitSpew(JitSpew_RegAlloc, " .. bundle is defined by a register");
+ return true;
+ }
+ if (!bundle->firstRange()->hasDefinition()) {
+ JitSpew(JitSpew_RegAlloc, " .. bundle does not have definition");
+ return true;
+ }
+
+ CodePosition firstRegisterFrom;
+
+ CodePosition conflictEnd;
+ if (conflict) {
+ for (LiveRange::BundleLinkIterator iter = conflict->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ if (range->to() > conflictEnd) {
+ conflictEnd = range->to();
+ }
+ }
+ }
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ if (!conflict || range->from() > conflictEnd) {
+ if (range->hasDefinition() && isRegisterDefinition(range)) {
+ firstRegisterFrom = range->from();
+ break;
+ }
+ }
+
+ for (UsePositionIterator iter(range->usesBegin()); iter; iter++) {
+ LNode* ins = insData[iter->pos];
+
+ if (!conflict || outputOf(ins) >= conflictEnd) {
+ if (isRegisterUse(*iter, ins, /* considerCopy = */ true)) {
+ firstRegisterFrom = inputOf(ins);
+ break;
+ }
+ }
+ }
+ if (firstRegisterFrom.bits()) {
+ break;
+ }
+ }
+
+ if (!firstRegisterFrom.bits()) {
+ // Can't trim non-register uses off the beginning by splitting.
+ JitSpew(JitSpew_RegAlloc, " bundle has no register uses");
+ return true;
+ }
+
+ JitSpewIfEnabled(JitSpew_RegAlloc,
+ " .. split before first register use at %u",
+ firstRegisterFrom.bits());
+
+ SplitPositionVector splitPositions;
+ if (!splitPositions.append(firstRegisterFrom)) {
+ return false;
+ }
+ *success = true;
+ return splitAt(bundle, splitPositions);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// The top level driver for the splitting machinery //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+// ::chooseBundleSplit
+// -------------------
+// If the first allocation loop (in ::go) can't allocate a bundle, it hands it
+// off to ::chooseBundleSplit, which is the "entry point" of the bundle-split
+// machinery. This tries four heuristics in turn, to see if any can split the
+// bundle:
+//
+// * ::trySplitAcrossHotcode
+// * ::splitAcrossCalls (in some cases)
+// * ::trySplitBeforeFirstRegisterUse
+// * ::trySplitAfterLastRegisterUse
+//
+// These routines have similar structure: they try to decide on one or more
+// CodePositions at which to split the bundle, using whatever heuristics they
+// have to hand. If suitable CodePosition(s) are found, they are put into a
+// `SplitPositionVector`, and the bundle and the vector are handed off to
+// ::splitAt, which performs the split (at least in theory) at the chosen
+// positions. It also arranges for the new bundles to be added to
+// ::allocationQueue.
+//
+// ::trySplitAcrossHotcode has a special case for JS -- it modifies the
+// bundle(s) itself, rather than using ::splitAt.
+//
+// If none of the heuristic routines apply, then ::splitAt is called with an
+// empty vector of split points. This is interpreted to mean "split at all
+// register uses". When combined with how ::splitAt works, the effect is to
+// spill the bundle.
+
+bool BacktrackingAllocator::chooseBundleSplit(LiveBundle* bundle, bool fixed,
+ LiveBundle* conflict) {
+ bool success = false;
+
+ JitSpew(JitSpew_RegAlloc, " Splitting %s ..", bundle->toString().get());
+
+ if (!trySplitAcrossHotcode(bundle, &success)) {
+ return false;
+ }
+ if (success) {
+ return true;
+ }
+
+ if (fixed) {
+ return splitAcrossCalls(bundle);
+ }
+
+ if (!trySplitBeforeFirstRegisterUse(bundle, conflict, &success)) {
+ return false;
+ }
+ if (success) {
+ return true;
+ }
+
+ if (!trySplitAfterLastRegisterUse(bundle, conflict, &success)) {
+ return false;
+ }
+ if (success) {
+ return true;
+ }
+
+ // Split at all register uses.
+ SplitPositionVector emptyPositions;
+ return splitAt(bundle, emptyPositions);
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Bundle allocation //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+static const size_t MAX_ATTEMPTS = 2;
+
+bool BacktrackingAllocator::computeRequirement(LiveBundle* bundle,
+ Requirement* requirement,
+ Requirement* hint) {
+ // Set any requirement or hint on bundle according to its definition and
+ // uses. Return false if there are conflicting requirements which will
+ // require the bundle to be split.
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ VirtualRegister& reg = range->vreg();
+
+ if (range->hasDefinition()) {
+ // Deal with any definition constraints/hints.
+ LDefinition::Policy policy = reg.def()->policy();
+ if (policy == LDefinition::FIXED || policy == LDefinition::STACK) {
+ // Fixed and stack policies get a FIXED requirement. (In the stack
+ // case, the allocation should have been performed already by
+ // mergeAndQueueRegisters.)
+ JitSpewIfEnabled(JitSpew_RegAlloc,
+ " Requirement %s, fixed by definition",
+ reg.def()->output()->toString().get());
+ if (!requirement->merge(Requirement(*reg.def()->output()))) {
+ return false;
+ }
+ } else if (reg.ins()->isPhi()) {
+ // Phis don't have any requirements, but they should prefer their
+ // input allocations. This is captured by the group hints above.
+ } else {
+ // Non-phis get a REGISTER requirement.
+ if (!requirement->merge(Requirement(Requirement::REGISTER))) {
+ return false;
+ }
+ }
+ }
+
+ // Search uses for requirements.
+ for (UsePositionIterator iter = range->usesBegin(); iter; iter++) {
+ LUse::Policy policy = iter->usePolicy();
+ if (policy == LUse::FIXED) {
+ AnyRegister required = GetFixedRegister(reg.def(), iter->use());
+
+ JitSpewIfEnabled(JitSpew_RegAlloc, " Requirement %s, due to use at %u",
+ required.name(), iter->pos.bits());
+
+ // If there are multiple fixed registers which the bundle is
+ // required to use, fail. The bundle will need to be split before
+ // it can be allocated.
+ if (!requirement->merge(Requirement(LAllocation(required)))) {
+ return false;
+ }
+ } else if (policy == LUse::REGISTER) {
+ if (!requirement->merge(Requirement(Requirement::REGISTER))) {
+ return false;
+ }
+ } else if (policy == LUse::ANY) {
+ // ANY differs from KEEPALIVE by actively preferring a register.
+ if (!hint->merge(Requirement(Requirement::REGISTER))) {
+ return false;
+ }
+ }
+
+ // The only case of STACK use policies is individual stack results using
+ // their containing stack result area, which is given a fixed allocation
+ // above.
+ MOZ_ASSERT_IF(policy == LUse::STACK,
+ requirement->kind() == Requirement::FIXED);
+ MOZ_ASSERT_IF(policy == LUse::STACK,
+ requirement->allocation().isStackArea());
+ }
+ }
+
+ return true;
+}
+
+bool BacktrackingAllocator::tryAllocateRegister(PhysicalRegister& r,
+ LiveBundle* bundle,
+ bool* success, bool* pfixed,
+ LiveBundleVector& conflicting) {
+ *success = false;
+
+ if (!r.allocatable) {
+ return true;
+ }
+
+ LiveBundleVector aliasedConflicting;
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ // All ranges in the bundle must be compatible with the physical register.
+ MOZ_ASSERT(range->vreg().isCompatible(r.reg));
+
+ for (size_t a = 0; a < r.reg.numAliased(); a++) {
+ PhysicalRegister& rAlias = registers[r.reg.aliased(a).code()];
+ LiveRange* existing;
+ if (!rAlias.allocations.contains(range, &existing)) {
+ continue;
+ }
+ if (existing->hasVreg()) {
+ MOZ_ASSERT(existing->bundle()->allocation().toRegister() == rAlias.reg);
+ bool duplicate = false;
+ for (size_t i = 0; i < aliasedConflicting.length(); i++) {
+ if (aliasedConflicting[i] == existing->bundle()) {
+ duplicate = true;
+ break;
+ }
+ }
+ if (!duplicate && !aliasedConflicting.append(existing->bundle())) {
+ return false;
+ }
+ } else {
+ JitSpewIfEnabled(JitSpew_RegAlloc, " %s collides with fixed use %s",
+ rAlias.reg.name(), existing->toString().get());
+ *pfixed = true;
+ return true;
+ }
+ }
+ }
+
+ if (!aliasedConflicting.empty()) {
+ // One or more aliased registers is allocated to another bundle
+ // overlapping this one. Keep track of the conflicting set, and in the
+ // case of multiple conflicting sets keep track of the set with the
+ // lowest maximum spill weight.
+
+ // The #ifdef guards against "unused variable 'existing'" bustage.
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_RegAlloc)) {
+ if (aliasedConflicting.length() == 1) {
+ LiveBundle* existing = aliasedConflicting[0];
+ JitSpew(JitSpew_RegAlloc, " %s collides with %s [weight %zu]",
+ r.reg.name(), existing->toString().get(),
+ computeSpillWeight(existing));
+ } else {
+ JitSpew(JitSpew_RegAlloc, " %s collides with the following",
+ r.reg.name());
+ for (size_t i = 0; i < aliasedConflicting.length(); i++) {
+ LiveBundle* existing = aliasedConflicting[i];
+ JitSpew(JitSpew_RegAlloc, " %s [weight %zu]",
+ existing->toString().get(), computeSpillWeight(existing));
+ }
+ }
+ }
+#endif
+
+ if (conflicting.empty()) {
+ conflicting = std::move(aliasedConflicting);
+ } else {
+ if (maximumSpillWeight(aliasedConflicting) <
+ maximumSpillWeight(conflicting)) {
+ conflicting = std::move(aliasedConflicting);
+ }
+ }
+ return true;
+ }
+
+ JitSpewIfEnabled(JitSpew_RegAlloc, " allocated to %s", r.reg.name());
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ if (!r.allocations.insert(range)) {
+ return false;
+ }
+ }
+
+ bundle->setAllocation(LAllocation(r.reg));
+ *success = true;
+ return true;
+}
+
+bool BacktrackingAllocator::tryAllocateAnyRegister(
+ LiveBundle* bundle, bool* success, bool* pfixed,
+ LiveBundleVector& conflicting) {
+ // Search for any available register which the bundle can be allocated to.
+
+ LDefinition::Type type = bundle->firstRange()->vreg().type();
+
+ if (LDefinition::isFloatReg(type)) {
+ for (size_t i = AnyRegister::FirstFloatReg; i < AnyRegister::Total; i++) {
+ if (!LDefinition::isFloatRegCompatible(type, registers[i].reg.fpu())) {
+ continue;
+ }
+ if (!tryAllocateRegister(registers[i], bundle, success, pfixed,
+ conflicting)) {
+ return false;
+ }
+ if (*success) {
+ break;
+ }
+ }
+ return true;
+ }
+
+ for (size_t i = 0; i < AnyRegister::FirstFloatReg; i++) {
+ if (!tryAllocateRegister(registers[i], bundle, success, pfixed,
+ conflicting)) {
+ return false;
+ }
+ if (*success) {
+ break;
+ }
+ }
+ return true;
+}
+
+bool BacktrackingAllocator::evictBundle(LiveBundle* bundle) {
+ JitSpewIfEnabled(JitSpew_RegAlloc,
+ " Evicting %s [priority %zu] [weight %zu]",
+ bundle->toString().get(), computePriority(bundle),
+ computeSpillWeight(bundle));
+
+ AnyRegister reg(bundle->allocation().toRegister());
+ PhysicalRegister& physical = registers[reg.code()];
+ MOZ_ASSERT(physical.reg == reg && physical.allocatable);
+
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ physical.allocations.remove(range);
+ }
+
+ bundle->setAllocation(LAllocation());
+
+ size_t priority = computePriority(bundle);
+ return allocationQueue.insert(QueueItem(bundle, priority));
+}
+
+bool BacktrackingAllocator::tryAllocateFixed(LiveBundle* bundle,
+ Requirement requirement,
+ bool* success, bool* pfixed,
+ LiveBundleVector& conflicting) {
+ // Spill bundles which are required to be in a certain stack slot.
+ if (!requirement.allocation().isRegister()) {
+ JitSpew(JitSpew_RegAlloc, " stack allocation requirement");
+ bundle->setAllocation(requirement.allocation());
+ *success = true;
+ return true;
+ }
+
+ AnyRegister reg = requirement.allocation().toRegister();
+ return tryAllocateRegister(registers[reg.code()], bundle, success, pfixed,
+ conflicting);
+}
+
+bool BacktrackingAllocator::tryAllocateNonFixed(LiveBundle* bundle,
+ Requirement requirement,
+ Requirement hint, bool* success,
+ bool* pfixed,
+ LiveBundleVector& conflicting) {
+ // If we want, but do not require a bundle to be in a specific register,
+ // only look at that register for allocating and evict or spill if it is
+ // not available. Picking a separate register may be even worse than
+ // spilling, as it will still necessitate moves and will tie up more
+ // registers than if we spilled.
+ if (hint.kind() == Requirement::FIXED) {
+ AnyRegister reg = hint.allocation().toRegister();
+ if (!tryAllocateRegister(registers[reg.code()], bundle, success, pfixed,
+ conflicting)) {
+ return false;
+ }
+ if (*success) {
+ return true;
+ }
+ }
+
+ // Spill bundles which have no hint or register requirement.
+ if (requirement.kind() == Requirement::NONE &&
+ hint.kind() != Requirement::REGISTER) {
+ JitSpew(JitSpew_RegAlloc,
+ " postponed spill (no hint or register requirement)");
+ if (!spilledBundles.append(bundle)) {
+ return false;
+ }
+ *success = true;
+ return true;
+ }
+
+ if (conflicting.empty() || minimalBundle(bundle)) {
+ if (!tryAllocateAnyRegister(bundle, success, pfixed, conflicting)) {
+ return false;
+ }
+ if (*success) {
+ return true;
+ }
+ }
+
+ // Spill bundles which have no register requirement if they didn't get
+ // allocated.
+ if (requirement.kind() == Requirement::NONE) {
+ JitSpew(JitSpew_RegAlloc, " postponed spill (no register requirement)");
+ if (!spilledBundles.append(bundle)) {
+ return false;
+ }
+ *success = true;
+ return true;
+ }
+
+ // We failed to allocate this bundle.
+ MOZ_ASSERT(!*success);
+ return true;
+}
+
+bool BacktrackingAllocator::processBundle(MIRGenerator* mir,
+ LiveBundle* bundle) {
+ JitSpewIfEnabled(JitSpew_RegAlloc,
+ "Allocating %s [priority %zu] [weight %zu]",
+ bundle->toString().get(), computePriority(bundle),
+ computeSpillWeight(bundle));
+
+ // A bundle can be processed by doing any of the following:
+ //
+ // - Assigning the bundle a register. The bundle cannot overlap any other
+ // bundle allocated for that physical register.
+ //
+ // - Spilling the bundle, provided it has no register uses.
+ //
+ // - Splitting the bundle into two or more bundles which cover the original
+ // one. The new bundles are placed back onto the priority queue for later
+ // processing.
+ //
+ // - Evicting one or more existing allocated bundles, and then doing one
+ // of the above operations. Evicted bundles are placed back on the
+ // priority queue. Any evicted bundles must have a lower spill weight
+ // than the bundle being processed.
+ //
+ // As long as this structure is followed, termination is guaranteed.
+ // In general, we want to minimize the amount of bundle splitting (which
+ // generally necessitates spills), so allocate longer lived, lower weight
+ // bundles first and evict and split them later if they prevent allocation
+ // for higher weight bundles.
+
+ Requirement requirement, hint;
+ bool canAllocate = computeRequirement(bundle, &requirement, &hint);
+
+ bool fixed;
+ LiveBundleVector conflicting;
+ for (size_t attempt = 0;; attempt++) {
+ if (mir->shouldCancel("Backtracking Allocation (processBundle loop)")) {
+ return false;
+ }
+
+ if (canAllocate) {
+ bool success = false;
+ fixed = false;
+ conflicting.clear();
+
+ // Ok, let's try allocating for this bundle.
+ if (requirement.kind() == Requirement::FIXED) {
+ if (!tryAllocateFixed(bundle, requirement, &success, &fixed,
+ conflicting)) {
+ return false;
+ }
+ } else {
+ if (!tryAllocateNonFixed(bundle, requirement, hint, &success, &fixed,
+ conflicting)) {
+ return false;
+ }
+ }
+
+ // If that worked, we're done!
+ if (success) {
+ return true;
+ }
+
+ // If that didn't work, but we have one or more non-fixed bundles
+ // known to be conflicting, maybe we can evict them and try again.
+ if ((attempt < MAX_ATTEMPTS || minimalBundle(bundle)) && !fixed &&
+ !conflicting.empty() &&
+ maximumSpillWeight(conflicting) < computeSpillWeight(bundle)) {
+ for (size_t i = 0; i < conflicting.length(); i++) {
+ if (!evictBundle(conflicting[i])) {
+ return false;
+ }
+ }
+ continue;
+ }
+ }
+
+ // A minimal bundle cannot be split any further. If we try to split it
+ // it at this point we will just end up with the same bundle and will
+ // enter an infinite loop. Weights and the initial live ranges must
+ // be constructed so that any minimal bundle is allocatable.
+ MOZ_ASSERT(!minimalBundle(bundle));
+
+ LiveBundle* conflict = conflicting.empty() ? nullptr : conflicting[0];
+ return chooseBundleSplit(bundle, canAllocate && fixed, conflict);
+ }
+}
+
+// Helper for ::tryAllocatingRegistersForSpillBundles
+bool BacktrackingAllocator::spill(LiveBundle* bundle) {
+ JitSpew(JitSpew_RegAlloc, " Spilling bundle");
+ MOZ_ASSERT(bundle->allocation().isBogus());
+
+ if (LiveBundle* spillParent = bundle->spillParent()) {
+ JitSpew(JitSpew_RegAlloc, " Using existing spill bundle");
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ LiveRange* parentRange = spillParent->rangeFor(range->from());
+ MOZ_ASSERT(parentRange->contains(range));
+ MOZ_ASSERT(&range->vreg() == &parentRange->vreg());
+ range->tryToMoveDefAndUsesInto(parentRange);
+ MOZ_ASSERT(!range->hasUses());
+ range->vreg().removeRange(range);
+ }
+ return true;
+ }
+
+ return bundle->spillSet()->addSpilledBundle(bundle);
+}
+
+bool BacktrackingAllocator::tryAllocatingRegistersForSpillBundles() {
+ for (auto it = spilledBundles.begin(); it != spilledBundles.end(); it++) {
+ LiveBundle* bundle = *it;
+ LiveBundleVector conflicting;
+ bool fixed = false;
+ bool success = false;
+
+ if (mir->shouldCancel("Backtracking Try Allocating Spilled Bundles")) {
+ return false;
+ }
+
+ JitSpewIfEnabled(JitSpew_RegAlloc, "Spill or allocate %s",
+ bundle->toString().get());
+
+ if (!tryAllocateAnyRegister(bundle, &success, &fixed, conflicting)) {
+ return false;
+ }
+
+ // If the bundle still has no register, spill the bundle.
+ if (!success && !spill(bundle)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Rewriting of the LIR after bundle processing is done: //
+// ::pickStackSlots //
+// ::createMoveGroupsFromLiveRangeTransitions //
+// ::installAllocationsInLIR //
+// ::populateSafepoints //
+// ::annotateMoveGroups //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+// Helper for ::pickStackSlot
+bool BacktrackingAllocator::insertAllRanges(LiveRangeSet& set,
+ LiveBundle* bundle) {
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ if (!set.insert(range)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Helper for ::pickStackSlots
+bool BacktrackingAllocator::pickStackSlot(SpillSet* spillSet) {
+ // Look through all ranges that have been spilled in this set for a
+ // register definition which is fixed to a stack or argument slot. If we
+ // find one, use it for all bundles that have been spilled. tryMergeBundles
+ // makes sure this reuse is possible when an initial bundle contains ranges
+ // from multiple virtual registers.
+ for (size_t i = 0; i < spillSet->numSpilledBundles(); i++) {
+ LiveBundle* bundle = spillSet->spilledBundle(i);
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ if (range->hasDefinition()) {
+ LDefinition* def = range->vreg().def();
+ if (def->policy() == LDefinition::FIXED) {
+ MOZ_ASSERT(!def->output()->isRegister());
+ MOZ_ASSERT(!def->output()->isStackSlot());
+ spillSet->setAllocation(*def->output());
+ return true;
+ }
+ }
+ }
+ }
+
+ LDefinition::Type type =
+ spillSet->spilledBundle(0)->firstRange()->vreg().type();
+
+ SpillSlotList* slotList;
+ switch (StackSlotAllocator::width(type)) {
+ case 4:
+ slotList = &normalSlots;
+ break;
+ case 8:
+ slotList = &doubleSlots;
+ break;
+ case 16:
+ slotList = &quadSlots;
+ break;
+ default:
+ MOZ_CRASH("Bad width");
+ }
+
+ // Maximum number of existing spill slots we will look at before giving up
+ // and allocating a new slot.
+ static const size_t MAX_SEARCH_COUNT = 10;
+
+ size_t searches = 0;
+ SpillSlot* stop = nullptr;
+ while (!slotList->empty()) {
+ SpillSlot* spillSlot = *slotList->begin();
+ if (!stop) {
+ stop = spillSlot;
+ } else if (stop == spillSlot) {
+ // We looked through every slot in the list.
+ break;
+ }
+
+ bool success = true;
+ for (size_t i = 0; i < spillSet->numSpilledBundles(); i++) {
+ LiveBundle* bundle = spillSet->spilledBundle(i);
+ for (LiveRange::BundleLinkIterator iter = bundle->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ LiveRange* existing;
+ if (spillSlot->allocated.contains(range, &existing)) {
+ success = false;
+ break;
+ }
+ }
+ if (!success) {
+ break;
+ }
+ }
+ if (success) {
+ // We can reuse this physical stack slot for the new bundles.
+ // Update the allocated ranges for the slot.
+ for (size_t i = 0; i < spillSet->numSpilledBundles(); i++) {
+ LiveBundle* bundle = spillSet->spilledBundle(i);
+ if (!insertAllRanges(spillSlot->allocated, bundle)) {
+ return false;
+ }
+ }
+ spillSet->setAllocation(spillSlot->alloc);
+ return true;
+ }
+
+ // On a miss, move the spill to the end of the list. This will cause us
+ // to make fewer attempts to allocate from slots with a large and
+ // highly contended range.
+ slotList->popFront();
+ slotList->pushBack(spillSlot);
+
+ if (++searches == MAX_SEARCH_COUNT) {
+ break;
+ }
+ }
+
+ // We need a new physical stack slot.
+ uint32_t stackSlot = stackSlotAllocator.allocateSlot(type);
+
+ SpillSlot* spillSlot =
+ new (alloc().fallible()) SpillSlot(stackSlot, alloc().lifoAlloc());
+ if (!spillSlot) {
+ return false;
+ }
+
+ for (size_t i = 0; i < spillSet->numSpilledBundles(); i++) {
+ LiveBundle* bundle = spillSet->spilledBundle(i);
+ if (!insertAllRanges(spillSlot->allocated, bundle)) {
+ return false;
+ }
+ }
+
+ spillSet->setAllocation(spillSlot->alloc);
+
+ slotList->pushFront(spillSlot);
+ return true;
+}
+
+bool BacktrackingAllocator::pickStackSlots() {
+ for (size_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+
+ if (mir->shouldCancel("Backtracking Pick Stack Slots")) {
+ return false;
+ }
+
+ for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+ LiveBundle* bundle = range->bundle();
+
+ if (bundle->allocation().isBogus()) {
+ if (!pickStackSlot(bundle->spillSet())) {
+ return false;
+ }
+ MOZ_ASSERT(!bundle->allocation().isBogus());
+ }
+ }
+ }
+
+ return true;
+}
+
+// Helper for ::createMoveGroupsFromLiveRangeTransitions
+bool BacktrackingAllocator::moveAtEdge(LBlock* predecessor, LBlock* successor,
+ LiveRange* from, LiveRange* to,
+ LDefinition::Type type) {
+ if (successor->mir()->numPredecessors() > 1) {
+ MOZ_ASSERT(predecessor->mir()->numSuccessors() == 1);
+ return moveAtExit(predecessor, from, to, type);
+ }
+
+ return moveAtEntry(successor, from, to, type);
+}
+
+// Helper for ::createMoveGroupsFromLiveRangeTransitions
+bool BacktrackingAllocator::deadRange(LiveRange* range) {
+ // Check for direct uses of this range.
+ if (range->hasUses() || range->hasDefinition()) {
+ return false;
+ }
+
+ CodePosition start = range->from();
+ LNode* ins = insData[start];
+ if (start == entryOf(ins->block())) {
+ return false;
+ }
+
+ VirtualRegister& reg = range->vreg();
+
+ // Check if there are later ranges for this vreg.
+ LiveRange::RegisterLinkIterator iter = reg.rangesBegin(range);
+ for (iter++; iter; iter++) {
+ LiveRange* laterRange = LiveRange::get(*iter);
+ if (laterRange->from() > range->from()) {
+ return false;
+ }
+ }
+
+ // Check if this range ends at a loop backedge.
+ LNode* last = insData[range->to().previous()];
+ if (last->isGoto() &&
+ last->toGoto()->target()->id() < last->block()->mir()->id()) {
+ return false;
+ }
+
+ // Check if there are phis which this vreg flows to.
+ if (reg.usedByPhi()) {
+ return false;
+ }
+
+ return true;
+}
+
+bool BacktrackingAllocator::createMoveGroupsFromLiveRangeTransitions() {
+ // Add moves to handle changing assignments for vregs over their lifetime.
+ JitSpew(JitSpew_RegAlloc, "ResolveControlFlow: begin");
+
+ JitSpew(JitSpew_RegAlloc,
+ " ResolveControlFlow: adding MoveGroups within blocks");
+
+ // Look for places where a register's assignment changes in the middle of a
+ // basic block.
+ MOZ_ASSERT(!vregs[0u].hasRanges());
+ for (size_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+
+ if (mir->shouldCancel(
+ "Backtracking Resolve Control Flow (vreg outer loop)")) {
+ return false;
+ }
+
+ for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ if (mir->shouldCancel(
+ "Backtracking Resolve Control Flow (vreg inner loop)")) {
+ return false;
+ }
+
+ // Remove ranges which will never be used.
+ if (deadRange(range)) {
+ reg.removeRangeAndIncrement(iter);
+ continue;
+ }
+
+ // The range which defines the register does not have a predecessor
+ // to add moves from.
+ if (range->hasDefinition()) {
+ iter++;
+ continue;
+ }
+
+ // Ignore ranges that start at block boundaries. We will handle
+ // these in the next phase.
+ CodePosition start = range->from();
+ LNode* ins = insData[start];
+ if (start == entryOf(ins->block())) {
+ iter++;
+ continue;
+ }
+
+ // If we already saw a range which covers the start of this range
+ // and has the same allocation, we don't need an explicit move at
+ // the start of this range.
+ bool skip = false;
+ for (LiveRange::RegisterLinkIterator prevIter = reg.rangesBegin();
+ prevIter != iter; prevIter++) {
+ LiveRange* prevRange = LiveRange::get(*prevIter);
+ if (prevRange->covers(start) && prevRange->bundle()->allocation() ==
+ range->bundle()->allocation()) {
+ skip = true;
+ break;
+ }
+ }
+ if (skip) {
+ iter++;
+ continue;
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ LiveRange* predecessorRange =
+ reg.rangeFor(start.previous(), /* preferRegister = */ true);
+ if (start.subpos() == CodePosition::INPUT) {
+ JitSpew(JitSpew_RegAlloc, " moveInput (%s) <- (%s)",
+ range->toString().get(), predecessorRange->toString().get());
+ if (!moveInput(ins->toInstruction(), predecessorRange, range,
+ reg.type())) {
+ return false;
+ }
+ } else {
+ JitSpew(JitSpew_RegAlloc, " (moveAfter)");
+ if (!moveAfter(ins->toInstruction(), predecessorRange, range,
+ reg.type())) {
+ return false;
+ }
+ }
+
+ iter++;
+ }
+ }
+
+ JitSpew(JitSpew_RegAlloc,
+ " ResolveControlFlow: adding MoveGroups for phi nodes");
+
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ if (mir->shouldCancel("Backtracking Resolve Control Flow (block loop)")) {
+ return false;
+ }
+
+ LBlock* successor = graph.getBlock(i);
+ MBasicBlock* mSuccessor = successor->mir();
+ if (mSuccessor->numPredecessors() < 1) {
+ continue;
+ }
+
+ // Resolve phis to moves.
+ for (size_t j = 0; j < successor->numPhis(); j++) {
+ LPhi* phi = successor->getPhi(j);
+ MOZ_ASSERT(phi->numDefs() == 1);
+ LDefinition* def = phi->getDef(0);
+ VirtualRegister& reg = vreg(def);
+ LiveRange* to = reg.rangeFor(entryOf(successor));
+ MOZ_ASSERT(to);
+
+ for (size_t k = 0; k < mSuccessor->numPredecessors(); k++) {
+ LBlock* predecessor = mSuccessor->getPredecessor(k)->lir();
+ MOZ_ASSERT(predecessor->mir()->numSuccessors() == 1);
+
+ LAllocation* input = phi->getOperand(k);
+ LiveRange* from = vreg(input).rangeFor(exitOf(predecessor),
+ /* preferRegister = */ true);
+ MOZ_ASSERT(from);
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ // Note: we have to use moveAtEdge both here and below (for edge
+ // resolution) to avoid conflicting moves. See bug 1493900.
+ JitSpew(JitSpew_RegAlloc, " (moveAtEdge#1)");
+ if (!moveAtEdge(predecessor, successor, from, to, def->type())) {
+ return false;
+ }
+ }
+ }
+ }
+
+ JitSpew(JitSpew_RegAlloc,
+ " ResolveControlFlow: adding MoveGroups to fix conflicted edges");
+
+ // Add moves to resolve graph edges with different allocations at their
+ // source and target.
+ for (size_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+ for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;
+ iter++) {
+ LiveRange* targetRange = LiveRange::get(*iter);
+
+ size_t firstBlockId = insData[targetRange->from()]->block()->mir()->id();
+ if (!targetRange->covers(entryOf(graph.getBlock(firstBlockId)))) {
+ firstBlockId++;
+ }
+ for (size_t id = firstBlockId; id < graph.numBlocks(); id++) {
+ LBlock* successor = graph.getBlock(id);
+ if (!targetRange->covers(entryOf(successor))) {
+ break;
+ }
+
+ BitSet& live = liveIn[id];
+ if (!live.contains(i)) {
+ continue;
+ }
+
+ for (size_t j = 0; j < successor->mir()->numPredecessors(); j++) {
+ LBlock* predecessor = successor->mir()->getPredecessor(j)->lir();
+ if (targetRange->covers(exitOf(predecessor))) {
+ continue;
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ JitSpew(JitSpew_RegAlloc, " (moveAtEdge#2)");
+ LiveRange* from = reg.rangeFor(exitOf(predecessor), true);
+ if (!moveAtEdge(predecessor, successor, from, targetRange,
+ reg.type())) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+
+ JitSpew(JitSpew_RegAlloc, "ResolveControlFlow: end");
+ return true;
+}
+
+// Helper for ::addLiveRegistersForRange
+size_t BacktrackingAllocator::findFirstNonCallSafepoint(CodePosition from) {
+ size_t i = 0;
+ for (; i < graph.numNonCallSafepoints(); i++) {
+ const LInstruction* ins = graph.getNonCallSafepoint(i);
+ if (from <= inputOf(ins)) {
+ break;
+ }
+ }
+ return i;
+}
+
+// Helper for ::installAllocationsInLIR
+void BacktrackingAllocator::addLiveRegistersForRange(VirtualRegister& reg,
+ LiveRange* range) {
+ // Fill in the live register sets for all non-call safepoints.
+ LAllocation a = range->bundle()->allocation();
+ if (!a.isRegister()) {
+ return;
+ }
+
+ // Don't add output registers to the safepoint.
+ CodePosition start = range->from();
+ if (range->hasDefinition() && !reg.isTemp()) {
+#ifdef CHECK_OSIPOINT_REGISTERS
+ // We don't add the output register to the safepoint,
+ // but it still might get added as one of the inputs.
+ // So eagerly add this reg to the safepoint clobbered registers.
+ if (reg.ins()->isInstruction()) {
+ if (LSafepoint* safepoint = reg.ins()->toInstruction()->safepoint()) {
+ safepoint->addClobberedRegister(a.toRegister());
+ }
+ }
+#endif
+ start = start.next();
+ }
+
+ size_t i = findFirstNonCallSafepoint(start);
+ for (; i < graph.numNonCallSafepoints(); i++) {
+ LInstruction* ins = graph.getNonCallSafepoint(i);
+ CodePosition pos = inputOf(ins);
+
+ // Safepoints are sorted, so we can shortcut out of this loop
+ // if we go out of range.
+ if (range->to() <= pos) {
+ break;
+ }
+
+ MOZ_ASSERT(range->covers(pos));
+
+ LSafepoint* safepoint = ins->safepoint();
+ safepoint->addLiveRegister(a.toRegister());
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ if (reg.isTemp()) {
+ safepoint->addClobberedRegister(a.toRegister());
+ }
+#endif
+ }
+}
+
+// Helper for ::installAllocationsInLIR
+static inline size_t NumReusingDefs(LInstruction* ins) {
+ size_t num = 0;
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ LDefinition* def = ins->getDef(i);
+ if (def->policy() == LDefinition::MUST_REUSE_INPUT) {
+ num++;
+ }
+ }
+ return num;
+}
+
+bool BacktrackingAllocator::installAllocationsInLIR() {
+ JitSpew(JitSpew_RegAlloc, "Installing Allocations");
+
+ MOZ_ASSERT(!vregs[0u].hasRanges());
+ for (size_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+
+ if (mir->shouldCancel("Backtracking Install Allocations (main loop)")) {
+ return false;
+ }
+
+ for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ if (range->hasDefinition()) {
+ reg.def()->setOutput(range->bundle()->allocation());
+ if (reg.ins()->recoversInput()) {
+ LSnapshot* snapshot = reg.ins()->toInstruction()->snapshot();
+ for (size_t i = 0; i < snapshot->numEntries(); i++) {
+ LAllocation* entry = snapshot->getEntry(i);
+ if (entry->isUse() &&
+ entry->toUse()->policy() == LUse::RECOVERED_INPUT) {
+ *entry = *reg.def()->output();
+ }
+ }
+ }
+ }
+
+ for (UsePositionIterator iter(range->usesBegin()); iter; iter++) {
+ LAllocation* alloc = iter->use();
+ *alloc = range->bundle()->allocation();
+
+ // For any uses which feed into MUST_REUSE_INPUT definitions,
+ // add copies if the use and def have different allocations.
+ LNode* ins = insData[iter->pos];
+ if (LDefinition* def = FindReusingDefOrTemp(ins, alloc)) {
+ LiveRange* outputRange = vreg(def).rangeFor(outputOf(ins));
+ LAllocation res = outputRange->bundle()->allocation();
+ LAllocation sourceAlloc = range->bundle()->allocation();
+
+ if (res != *alloc) {
+ if (!this->alloc().ensureBallast()) {
+ return false;
+ }
+ if (NumReusingDefs(ins->toInstruction()) <= 1) {
+ LMoveGroup* group = getInputMoveGroup(ins->toInstruction());
+ if (!group->addAfter(sourceAlloc, res, reg.type())) {
+ return false;
+ }
+ } else {
+ LMoveGroup* group = getFixReuseMoveGroup(ins->toInstruction());
+ if (!group->add(sourceAlloc, res, reg.type())) {
+ return false;
+ }
+ }
+ *alloc = res;
+ }
+ }
+ }
+
+ addLiveRegistersForRange(reg, range);
+ }
+ }
+
+ graph.setLocalSlotsSize(stackSlotAllocator.stackHeight());
+ return true;
+}
+
+// Helper for ::populateSafepoints
+size_t BacktrackingAllocator::findFirstSafepoint(CodePosition pos,
+ size_t startFrom) {
+ size_t i = startFrom;
+ for (; i < graph.numSafepoints(); i++) {
+ LInstruction* ins = graph.getSafepoint(i);
+ if (pos <= inputOf(ins)) {
+ break;
+ }
+ }
+ return i;
+}
+
+// Helper for ::populateSafepoints
+static inline bool IsNunbox(VirtualRegister& reg) {
+#ifdef JS_NUNBOX32
+ return reg.type() == LDefinition::TYPE || reg.type() == LDefinition::PAYLOAD;
+#else
+ return false;
+#endif
+}
+
+// Helper for ::populateSafepoints
+static inline bool IsSlotsOrElements(VirtualRegister& reg) {
+ return reg.type() == LDefinition::SLOTS;
+}
+
+// Helper for ::populateSafepoints
+static inline bool IsTraceable(VirtualRegister& reg) {
+ if (reg.type() == LDefinition::OBJECT) {
+ return true;
+ }
+#ifdef JS_PUNBOX64
+ if (reg.type() == LDefinition::BOX) {
+ return true;
+ }
+#endif
+ if (reg.type() == LDefinition::STACKRESULTS) {
+ MOZ_ASSERT(reg.def());
+ const LStackArea* alloc = reg.def()->output()->toStackArea();
+ for (auto iter = alloc->results(); iter; iter.next()) {
+ if (iter.isGcPointer()) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+bool BacktrackingAllocator::populateSafepoints() {
+ JitSpew(JitSpew_RegAlloc, "Populating Safepoints");
+
+ size_t firstSafepoint = 0;
+
+ MOZ_ASSERT(!vregs[0u].def());
+ for (uint32_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+
+ if (!reg.def() ||
+ (!IsTraceable(reg) && !IsSlotsOrElements(reg) && !IsNunbox(reg))) {
+ continue;
+ }
+
+ firstSafepoint = findFirstSafepoint(inputOf(reg.ins()), firstSafepoint);
+ if (firstSafepoint >= graph.numSafepoints()) {
+ break;
+ }
+
+ for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ for (size_t j = firstSafepoint; j < graph.numSafepoints(); j++) {
+ LInstruction* ins = graph.getSafepoint(j);
+
+ if (!range->covers(inputOf(ins))) {
+ if (inputOf(ins) >= range->to()) {
+ break;
+ }
+ continue;
+ }
+
+ // Include temps but not instruction outputs. Also make sure
+ // MUST_REUSE_INPUT is not used with gcthings or nunboxes, or
+ // we would have to add the input reg to this safepoint.
+ if (ins == reg.ins() && !reg.isTemp()) {
+ DebugOnly<LDefinition*> def = reg.def();
+ MOZ_ASSERT_IF(def->policy() == LDefinition::MUST_REUSE_INPUT,
+ def->type() == LDefinition::GENERAL ||
+ def->type() == LDefinition::INT32 ||
+ def->type() == LDefinition::FLOAT32 ||
+ def->type() == LDefinition::DOUBLE ||
+ def->type() == LDefinition::SIMD128);
+ continue;
+ }
+
+ LSafepoint* safepoint = ins->safepoint();
+
+ LAllocation a = range->bundle()->allocation();
+ if (a.isGeneralReg() && ins->isCall()) {
+ continue;
+ }
+
+ switch (reg.type()) {
+ case LDefinition::OBJECT:
+ if (!safepoint->addGcPointer(a)) {
+ return false;
+ }
+ break;
+ case LDefinition::SLOTS:
+ if (!safepoint->addSlotsOrElementsPointer(a)) {
+ return false;
+ }
+ break;
+ case LDefinition::STACKRESULTS: {
+ MOZ_ASSERT(a.isStackArea());
+ for (auto iter = a.toStackArea()->results(); iter; iter.next()) {
+ if (iter.isGcPointer()) {
+ if (!safepoint->addGcPointer(iter.alloc())) {
+ return false;
+ }
+ }
+ }
+ break;
+ }
+#ifdef JS_NUNBOX32
+ case LDefinition::TYPE:
+ if (!safepoint->addNunboxType(i, a)) {
+ return false;
+ }
+ break;
+ case LDefinition::PAYLOAD:
+ if (!safepoint->addNunboxPayload(i, a)) {
+ return false;
+ }
+ break;
+#else
+ case LDefinition::BOX:
+ if (!safepoint->addBoxedValue(a)) {
+ return false;
+ }
+ break;
+#endif
+ default:
+ MOZ_CRASH("Bad register type");
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+bool BacktrackingAllocator::annotateMoveGroups() {
+ // Annotate move groups in the LIR graph with any register that is not
+ // allocated at that point and can be used as a scratch register. This is
+ // only required for x86, as other platforms always have scratch registers
+ // available for use.
+#ifdef JS_CODEGEN_X86
+ LiveRange* range = LiveRange::FallibleNew(alloc(), nullptr, CodePosition(),
+ CodePosition().next());
+ if (!range) {
+ return false;
+ }
+
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ if (mir->shouldCancel("Backtracking Annotate Move Groups")) {
+ return false;
+ }
+
+ LBlock* block = graph.getBlock(i);
+ LInstruction* last = nullptr;
+ for (LInstructionIterator iter = block->begin(); iter != block->end();
+ ++iter) {
+ if (iter->isMoveGroup()) {
+ CodePosition from = last ? outputOf(last) : entryOf(block);
+ range->setTo(from.next());
+ range->setFrom(from);
+
+ for (size_t i = 0; i < AnyRegister::Total; i++) {
+ PhysicalRegister& reg = registers[i];
+ if (reg.reg.isFloat() || !reg.allocatable) {
+ continue;
+ }
+
+ // This register is unavailable for use if (a) it is in use
+ // by some live range immediately before the move group,
+ // or (b) it is an operand in one of the group's moves. The
+ // latter case handles live ranges which end immediately
+ // before the move group or start immediately after.
+ // For (b) we need to consider move groups immediately
+ // preceding or following this one.
+
+ if (iter->toMoveGroup()->uses(reg.reg.gpr())) {
+ continue;
+ }
+ bool found = false;
+ LInstructionIterator niter(iter);
+ for (niter++; niter != block->end(); niter++) {
+ if (niter->isMoveGroup()) {
+ if (niter->toMoveGroup()->uses(reg.reg.gpr())) {
+ found = true;
+ break;
+ }
+ } else {
+ break;
+ }
+ }
+ if (iter != block->begin()) {
+ LInstructionIterator riter(iter);
+ do {
+ riter--;
+ if (riter->isMoveGroup()) {
+ if (riter->toMoveGroup()->uses(reg.reg.gpr())) {
+ found = true;
+ break;
+ }
+ } else {
+ break;
+ }
+ } while (riter != block->begin());
+ }
+
+ LiveRange* existing;
+ if (found || reg.allocations.contains(range, &existing)) {
+ continue;
+ }
+
+ iter->toMoveGroup()->setScratchRegister(reg.reg.gpr());
+ break;
+ }
+ } else {
+ last = *iter;
+ }
+ }
+ }
+#endif
+
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Debug-printing support //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+#ifdef JS_JITSPEW
+
+UniqueChars LiveRange::toString() const {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+
+ UniqueChars buf = JS_smprintf("v%u %u-%u", hasVreg() ? vreg().vreg() : 0,
+ from().bits(), to().bits() - 1);
+
+ if (buf && bundle() && !bundle()->allocation().isBogus()) {
+ buf = JS_sprintf_append(std::move(buf), " %s",
+ bundle()->allocation().toString().get());
+ }
+
+ buf = JS_sprintf_append(std::move(buf), " {");
+
+ if (buf && hasDefinition()) {
+ buf = JS_sprintf_append(std::move(buf), " %u_def", from().bits());
+ if (hasVreg()) {
+ // If the definition has a fixed requirement, print it too.
+ const LDefinition* def = vreg().def();
+ LDefinition::Policy policy = def->policy();
+ if (policy == LDefinition::FIXED || policy == LDefinition::STACK) {
+ if (buf) {
+ buf = JS_sprintf_append(std::move(buf), ":F:%s",
+ def->output()->toString().get());
+ }
+ }
+ }
+ }
+
+ for (UsePositionIterator iter = usesBegin(); buf && iter; iter++) {
+ buf = JS_sprintf_append(std::move(buf), " %u_%s", iter->pos.bits(),
+ iter->use()->toString().get());
+ }
+
+ buf = JS_sprintf_append(std::move(buf), " }");
+
+ if (!buf) {
+ oomUnsafe.crash("LiveRange::toString()");
+ }
+
+ return buf;
+}
+
+UniqueChars LiveBundle::toString() const {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+
+ UniqueChars buf = JS_smprintf("LB%u(", debugId());
+
+ if (buf) {
+ if (spillParent()) {
+ buf = JS_sprintf_append(std::move(buf), "parent=LB%u",
+ spillParent()->debugId());
+ } else {
+ buf = JS_sprintf_append(std::move(buf), "parent=none");
+ }
+ }
+
+ for (LiveRange::BundleLinkIterator iter = rangesBegin(); buf && iter;
+ iter++) {
+ if (buf) {
+ buf = JS_sprintf_append(std::move(buf), "%s %s",
+ (iter == rangesBegin()) ? "" : " ##",
+ LiveRange::get(*iter)->toString().get());
+ }
+ }
+
+ if (buf) {
+ buf = JS_sprintf_append(std::move(buf), ")");
+ }
+
+ if (!buf) {
+ oomUnsafe.crash("LiveBundle::toString()");
+ }
+
+ return buf;
+}
+
+void BacktrackingAllocator::dumpLiveRangesByVReg(const char* who) {
+ MOZ_ASSERT(!vregs[0u].hasRanges());
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc, "Live ranges by virtual register (%s):", who);
+
+ for (uint32_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ JitSpewHeader(JitSpew_RegAlloc);
+ JitSpewCont(JitSpew_RegAlloc, " ");
+ VirtualRegister& reg = vregs[i];
+ for (LiveRange::RegisterLinkIterator iter = reg.rangesBegin(); iter;
+ iter++) {
+ if (iter != reg.rangesBegin()) {
+ JitSpewCont(JitSpew_RegAlloc, " ## ");
+ }
+ JitSpewCont(JitSpew_RegAlloc, "%s",
+ LiveRange::get(*iter)->toString().get());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ }
+}
+
+void BacktrackingAllocator::dumpLiveRangesByBundle(const char* who) {
+ MOZ_ASSERT(!vregs[0u].hasRanges());
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc, "Live ranges by bundle (%s):", who);
+
+ for (uint32_t i = 1; i < graph.numVirtualRegisters(); i++) {
+ VirtualRegister& reg = vregs[i];
+ for (LiveRange::RegisterLinkIterator baseIter = reg.rangesBegin(); baseIter;
+ baseIter++) {
+ LiveRange* range = LiveRange::get(*baseIter);
+ LiveBundle* bundle = range->bundle();
+ if (range == bundle->firstRange()) {
+ JitSpew(JitSpew_RegAlloc, " %s", bundle->toString().get());
+ }
+ }
+ }
+}
+
+void BacktrackingAllocator::dumpAllocations() {
+ JitSpew(JitSpew_RegAlloc, "Allocations:");
+
+ dumpLiveRangesByBundle("in dumpAllocations()");
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc, "Allocations by physical register:");
+
+ for (size_t i = 0; i < AnyRegister::Total; i++) {
+ if (registers[i].allocatable && !registers[i].allocations.empty()) {
+ JitSpewHeader(JitSpew_RegAlloc);
+ JitSpewCont(JitSpew_RegAlloc, " %s:", AnyRegister::FromCode(i).name());
+ bool first = true;
+ LiveRangeSet::Iter lrIter(&registers[i].allocations);
+ while (lrIter.hasMore()) {
+ LiveRange* range = lrIter.next();
+ if (first) {
+ first = false;
+ } else {
+ fprintf(stderr, " /");
+ }
+ fprintf(stderr, " %s", range->toString().get());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ }
+ }
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+}
+
+#endif // JS_JITSPEW
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+// Top level of the register allocation machinery //
+// //
+///////////////////////////////////////////////////////////////////////////////
+
+bool BacktrackingAllocator::go() {
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc, "Beginning register allocation");
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ if (JitSpewEnabled(JitSpew_RegAlloc)) {
+ dumpInstructions("(Pre-allocation LIR)");
+ }
+
+ if (!init()) {
+ return false;
+ }
+
+ if (!buildLivenessInfo()) {
+ return false;
+ }
+
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_RegAlloc)) {
+ dumpLiveRangesByVReg("after liveness analysis");
+ }
+#endif
+
+ if (!allocationQueue.reserve(graph.numVirtualRegisters() * 3 / 2)) {
+ return false;
+ }
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc, "Beginning grouping and queueing registers");
+ if (!mergeAndQueueRegisters()) {
+ return false;
+ }
+ JitSpew(JitSpew_RegAlloc, "Completed grouping and queueing registers");
+
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_RegAlloc)) {
+ dumpLiveRangesByBundle("after grouping/queueing regs");
+ }
+#endif
+
+ // There now follow two allocation loops, which are really the heart of the
+ // allocator. First, the "main" allocation loop. This does almost all of
+ // the allocation work, by repeatedly pulling bundles out of
+ // ::allocationQueue and calling ::processBundle on it, until there are no
+ // bundles left in the queue. Note that ::processBundle can add new smaller
+ // bundles to the queue if it needs to split or spill a bundle.
+ //
+ // For each bundle in turn pulled out of ::allocationQueue, ::processBundle:
+ //
+ // * calls ::computeRequirement to discover the overall constraint for the
+ // bundle.
+ //
+ // * tries to find a register for it, by calling either ::tryAllocateFixed or
+ // ::tryAllocateNonFixed.
+ //
+ // * if that fails, but ::tryAllocateFixed / ::tryAllocateNonFixed indicate
+ // that there is some other bundle with lower spill weight that can be
+ // evicted, then that bundle is evicted (hence, put back into
+ // ::allocationQueue), and we try again.
+ //
+ // * at most MAX_ATTEMPTS may be made.
+ //
+ // * If that still fails to find a register, then the bundle is handed off to
+ // ::chooseBundleSplit. That will choose to either split the bundle,
+ // yielding multiple pieces which are put back into ::allocationQueue, or
+ // it will spill the bundle. Note that the same mechanism applies to both;
+ // there's no clear boundary between splitting and spilling, because
+ // spilling can be interpreted as an extreme form of splitting.
+ //
+ // ::processBundle and its callees contains much gnarly and logic which isn't
+ // easy to understand, particularly in the area of how eviction candidates
+ // are chosen. But it works well enough, and tinkering doesn't seem to
+ // improve the resulting allocations. More important is the splitting logic,
+ // because that controls where spill/reload instructions are placed.
+ //
+ // Eventually ::allocationQueue becomes empty, and each LiveBundle has either
+ // been allocated a register or is marked for spilling. In the latter case
+ // it will have been added to ::spilledBundles.
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc, "Beginning main allocation loop");
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+
+ // Allocate, spill and split bundles until finished.
+ while (!allocationQueue.empty()) {
+ if (mir->shouldCancel("Backtracking Allocation")) {
+ return false;
+ }
+
+ QueueItem item = allocationQueue.removeHighest();
+ if (!processBundle(mir, item.bundle)) {
+ return false;
+ }
+ }
+
+ // And here's the second allocation loop (hidden inside
+ // ::tryAllocatingRegistersForSpillBundles). It makes one last attempt to
+ // find a register for each spill bundle. There's no attempt to free up
+ // registers by eviction. In at least 99% of cases this attempt fails, in
+ // which case the bundle is handed off to ::spill. The lucky remaining 1%
+ // get a register. Unfortunately this scheme interacts badly with the
+ // splitting strategy, leading to excessive register-to-register copying in
+ // some very simple cases. See bug 1752520.
+ //
+ // A modest but probably worthwhile amount of allocation time can be saved by
+ // making ::tryAllocatingRegistersForSpillBundles use specialised versions of
+ // ::tryAllocateAnyRegister and its callees, that don't bother to create sets
+ // of conflicting bundles. Creating those sets is expensive and, here,
+ // pointless, since we're not going to do any eviction based on them. This
+ // refinement is implemented in the un-landed patch at bug 1758274 comment
+ // 15.
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc,
+ "Main allocation loop complete; "
+ "beginning spill-bundle allocation loop");
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+
+ if (!tryAllocatingRegistersForSpillBundles()) {
+ return false;
+ }
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc, "Spill-bundle allocation loop complete");
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+
+ if (!pickStackSlots()) {
+ return false;
+ }
+
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_RegAlloc)) {
+ dumpAllocations();
+ }
+#endif
+
+ if (!createMoveGroupsFromLiveRangeTransitions()) {
+ return false;
+ }
+
+ if (!installAllocationsInLIR()) {
+ return false;
+ }
+
+ if (!populateSafepoints()) {
+ return false;
+ }
+
+ if (!annotateMoveGroups()) {
+ return false;
+ }
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ if (JitSpewEnabled(JitSpew_RegAlloc)) {
+ dumpInstructions("(Post-allocation LIR)");
+ }
+
+ JitSpew(JitSpew_RegAlloc, "Finished register allocation");
+
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// //
+///////////////////////////////////////////////////////////////////////////////
diff --git a/js/src/jit/BacktrackingAllocator.h b/js/src/jit/BacktrackingAllocator.h
new file mode 100644
index 0000000000..3c3d701f6c
--- /dev/null
+++ b/js/src/jit/BacktrackingAllocator.h
@@ -0,0 +1,798 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BacktrackingAllocator_h
+#define jit_BacktrackingAllocator_h
+
+#include "mozilla/Array.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/Attributes.h"
+
+#include "ds/AvlTree.h"
+#include "ds/PriorityQueue.h"
+#include "jit/RegisterAllocator.h"
+#include "jit/StackSlotAllocator.h"
+
+// Gives better traces in Nightly/debug builds (could be EARLY_BETA_OR_EARLIER)
+#if defined(NIGHTLY_BUILD) || defined(DEBUG)
+# define AVOID_INLINE_FOR_DEBUGGING MOZ_NEVER_INLINE
+#else
+# define AVOID_INLINE_FOR_DEBUGGING
+#endif
+
+// Backtracking priority queue based register allocator based on that described
+// in the following blog post:
+//
+// http://blog.llvm.org/2011/09/greedy-register-allocation-in-llvm-30.html
+
+namespace js {
+namespace jit {
+
+class Requirement {
+ public:
+ enum Kind { NONE, REGISTER, FIXED };
+
+ Requirement() : kind_(NONE) {}
+
+ explicit Requirement(Kind kind) : kind_(kind) {
+ // FIXED has a dedicated constructor.
+ MOZ_ASSERT(kind != FIXED);
+ }
+
+ explicit Requirement(LAllocation fixed) : kind_(FIXED), allocation_(fixed) {
+ MOZ_ASSERT(!fixed.isBogus() && !fixed.isUse());
+ }
+
+ Kind kind() const { return kind_; }
+
+ LAllocation allocation() const {
+ MOZ_ASSERT(!allocation_.isBogus() && !allocation_.isUse());
+ return allocation_;
+ }
+
+ [[nodiscard]] bool merge(const Requirement& newRequirement) {
+ // Merge newRequirement with any existing requirement, returning false
+ // if the new and old requirements conflict.
+
+ if (newRequirement.kind() == Requirement::FIXED) {
+ if (kind() == Requirement::FIXED) {
+ return newRequirement.allocation() == allocation();
+ }
+ *this = newRequirement;
+ return true;
+ }
+
+ MOZ_ASSERT(newRequirement.kind() == Requirement::REGISTER);
+ if (kind() == Requirement::FIXED) {
+ return allocation().isRegister();
+ }
+
+ *this = newRequirement;
+ return true;
+ }
+
+ private:
+ Kind kind_;
+ LAllocation allocation_;
+};
+
+struct UsePosition : public TempObject,
+ public InlineForwardListNode<UsePosition> {
+ private:
+ // A UsePosition is an LUse* with a CodePosition. UsePosition also has an
+ // optimization that allows access to the associated LUse::Policy without
+ // dereferencing memory: the policy is encoded in the low bits of the LUse*.
+ //
+ // Note however that because LUse* is uintptr_t-aligned, on 32-bit systems
+ // there are only 4 encodable values, for more than 4 use policies; in that
+ // case we allocate the common LUse::ANY, LUse::REGISTER, and LUse::FIXED use
+ // policies to tags, and use tag 0x3 to indicate that dereferencing the LUse
+ // is necessary to get the policy (KEEPALIVE or STACK, in that case).
+ uintptr_t use_;
+ static_assert(LUse::ANY < 0x3,
+ "LUse::ANY can be represented in low tag on 32-bit systems");
+ static_assert(LUse::REGISTER < 0x3,
+ "LUse::REGISTER can be represented in tag on 32-bit systems");
+ static_assert(LUse::FIXED < 0x3,
+ "LUse::FIXED can be represented in tag on 32-bit systems");
+
+ static constexpr uintptr_t PolicyMask = sizeof(uintptr_t) - 1;
+ static constexpr uintptr_t UseMask = ~PolicyMask;
+
+ void setUse(LUse* use) {
+ // RECOVERED_INPUT is used by snapshots and ignored when building the
+ // liveness information. Thus we can safely assume that no such value
+ // would be seen.
+ MOZ_ASSERT(use->policy() != LUse::RECOVERED_INPUT);
+
+ uintptr_t policyBits = use->policy();
+#ifndef JS_64BIT
+ // On a 32-bit machine, LUse::KEEPALIVE and LUse::STACK are accessed by
+ // dereferencing the use pointer.
+ if (policyBits >= PolicyMask) {
+ policyBits = PolicyMask;
+ }
+#endif
+ use_ = uintptr_t(use) | policyBits;
+ MOZ_ASSERT(use->policy() == usePolicy());
+ }
+
+ public:
+ CodePosition pos;
+
+ LUse* use() const { return reinterpret_cast<LUse*>(use_ & UseMask); }
+
+ LUse::Policy usePolicy() const {
+ uintptr_t bits = use_ & PolicyMask;
+#ifndef JS_64BIT
+ // On 32-bit machines, reach out to memory if it's LUse::KEEPALIVE or
+ // LUse::STACK.
+ if (bits == PolicyMask) {
+ return use()->policy();
+ }
+#endif
+ LUse::Policy policy = LUse::Policy(bits);
+ MOZ_ASSERT(use()->policy() == policy);
+ return policy;
+ }
+
+ UsePosition(LUse* use, CodePosition pos) : pos(pos) {
+ // Verify that the usedAtStart() flag is consistent with the
+ // subposition. For now ignore fixed registers, because they
+ // are handled specially around calls.
+ MOZ_ASSERT_IF(!use->isFixedRegister(),
+ pos.subpos() == (use->usedAtStart() ? CodePosition::INPUT
+ : CodePosition::OUTPUT));
+ setUse(use);
+ }
+};
+
+using UsePositionIterator = InlineForwardListIterator<UsePosition>;
+
+// Backtracking allocator data structures overview.
+//
+// LiveRange: A continuous range of positions where a virtual register is live.
+// LiveBundle: A set of LiveRanges which do not overlap.
+// VirtualRegister: A set of all LiveRanges used for some LDefinition.
+//
+// The allocator first performs a liveness ananlysis on the LIR graph which
+// constructs LiveRanges for each VirtualRegister, determining where the
+// registers are live.
+//
+// The ranges are then bundled together according to heuristics, and placed on
+// the allocation queue.
+//
+// As bundles are removed from the allocation queue, we attempt to find a
+// physical register or stack slot allocation for all ranges in the removed
+// bundle, possibly evicting already-allocated bundles. See processBundle()
+// for details.
+//
+// If we are not able to allocate a bundle, it is split according to heuristics
+// into two or more smaller bundles which cover all the ranges of the original.
+// These smaller bundles are then allocated independently.
+
+class LiveBundle;
+class VirtualRegister;
+
+class LiveRange : public TempObject {
+ public:
+ // Linked lists are used to keep track of the ranges in each LiveBundle and
+ // VirtualRegister. Since a LiveRange may be in two lists simultaneously, use
+ // these auxiliary classes to keep things straight.
+ class BundleLink : public InlineForwardListNode<BundleLink> {};
+ class RegisterLink : public InlineForwardListNode<RegisterLink> {};
+
+ using BundleLinkIterator = InlineForwardListIterator<BundleLink>;
+ using RegisterLinkIterator = InlineForwardListIterator<RegisterLink>;
+
+ // Links in the lists in LiveBundle and VirtualRegister.
+ BundleLink bundleLink;
+ RegisterLink registerLink;
+
+ static LiveRange* get(BundleLink* link) {
+ return reinterpret_cast<LiveRange*>(reinterpret_cast<uint8_t*>(link) -
+ offsetof(LiveRange, bundleLink));
+ }
+ static LiveRange* get(RegisterLink* link) {
+ return reinterpret_cast<LiveRange*>(reinterpret_cast<uint8_t*>(link) -
+ offsetof(LiveRange, registerLink));
+ }
+
+ struct Range {
+ // The beginning of this range, inclusive.
+ CodePosition from;
+
+ // The end of this range, exclusive.
+ CodePosition to;
+
+ Range() = default;
+
+ Range(CodePosition from, CodePosition to) : from(from), to(to) {
+ MOZ_ASSERT(!empty());
+ }
+
+ bool empty() {
+ MOZ_ASSERT(from <= to);
+ return from == to;
+ }
+ };
+
+ private:
+ // The virtual register this range is for, or nullptr if this does not have a
+ // virtual register (for example, it is in the callRanges bundle).
+ VirtualRegister* vreg_;
+
+ // The bundle containing this range, null if liveness information is being
+ // constructed and we haven't started allocating bundles yet.
+ LiveBundle* bundle_;
+
+ // The code positions in this range.
+ Range range_;
+
+ // All uses of the virtual register in this range, ordered by location.
+ InlineForwardList<UsePosition> uses_;
+
+ // Total spill weight that calculate from all the uses' policy. Because the
+ // use's policy can't be changed after initialization, we can update the
+ // weight whenever a use is added to or remove from this range. This way, we
+ // don't need to iterate all the uses every time computeSpillWeight() is
+ // called.
+ size_t usesSpillWeight_;
+
+ // Number of uses that have policy LUse::FIXED.
+ uint32_t numFixedUses_;
+
+ // Whether this range contains the virtual register's definition.
+ bool hasDefinition_;
+
+ LiveRange(VirtualRegister* vreg, Range range)
+ : vreg_(vreg),
+ bundle_(nullptr),
+ range_(range),
+ usesSpillWeight_(0),
+ numFixedUses_(0),
+ hasDefinition_(false)
+
+ {
+ MOZ_ASSERT(!range.empty());
+ }
+
+ void noteAddedUse(UsePosition* use);
+ void noteRemovedUse(UsePosition* use);
+
+ public:
+ static LiveRange* FallibleNew(TempAllocator& alloc, VirtualRegister* vreg,
+ CodePosition from, CodePosition to) {
+ return new (alloc.fallible()) LiveRange(vreg, Range(from, to));
+ }
+
+ VirtualRegister& vreg() const {
+ MOZ_ASSERT(hasVreg());
+ return *vreg_;
+ }
+ bool hasVreg() const { return vreg_ != nullptr; }
+
+ LiveBundle* bundle() const { return bundle_; }
+
+ CodePosition from() const { return range_.from; }
+ CodePosition to() const { return range_.to; }
+ bool covers(CodePosition pos) const { return pos >= from() && pos < to(); }
+
+ // Whether this range wholly contains other.
+ bool contains(LiveRange* other) const;
+
+ // Intersect this range with other, returning the subranges of this
+ // that are before, inside, or after other.
+ void intersect(LiveRange* other, Range* pre, Range* inside,
+ Range* post) const;
+
+ // Whether this range has any intersection with other.
+ bool intersects(LiveRange* other) const;
+
+ UsePositionIterator usesBegin() const { return uses_.begin(); }
+ UsePosition* lastUse() const { return uses_.back(); }
+ bool hasUses() const { return !!usesBegin(); }
+ UsePosition* popUse();
+
+ bool hasDefinition() const { return hasDefinition_; }
+
+ void setFrom(CodePosition from) {
+ range_.from = from;
+ MOZ_ASSERT(!range_.empty());
+ }
+ void setTo(CodePosition to) {
+ range_.to = to;
+ MOZ_ASSERT(!range_.empty());
+ }
+
+ void setBundle(LiveBundle* bundle) { bundle_ = bundle; }
+
+ void addUse(UsePosition* use);
+ void tryToMoveDefAndUsesInto(LiveRange* other);
+
+ void setHasDefinition() {
+ MOZ_ASSERT(!hasDefinition_);
+ hasDefinition_ = true;
+ }
+
+ size_t usesSpillWeight() { return usesSpillWeight_; }
+ uint32_t numFixedUses() { return numFixedUses_; }
+
+#ifdef JS_JITSPEW
+ // Return a string describing this range.
+ UniqueChars toString() const;
+#endif
+
+ // Comparator for use in range splay trees.
+ static int compare(LiveRange* v0, LiveRange* v1) {
+ // LiveRange includes 'from' but excludes 'to'.
+ if (v0->to() <= v1->from()) {
+ return -1;
+ }
+ if (v0->from() >= v1->to()) {
+ return 1;
+ }
+ return 0;
+ }
+};
+
+// Tracks information about bundles that should all be spilled to the same
+// physical location. At the beginning of allocation, each bundle has its own
+// spill set. As bundles are split, the new smaller bundles continue to use the
+// same spill set.
+class SpillSet : public TempObject {
+ // All bundles with this spill set which have been spilled. All bundles in
+ // this list will be given the same physical slot.
+ Vector<LiveBundle*, 1, JitAllocPolicy> list_;
+
+ explicit SpillSet(TempAllocator& alloc) : list_(alloc) {}
+
+ public:
+ static SpillSet* New(TempAllocator& alloc) {
+ return new (alloc) SpillSet(alloc);
+ }
+
+ [[nodiscard]] bool addSpilledBundle(LiveBundle* bundle) {
+ return list_.append(bundle);
+ }
+ size_t numSpilledBundles() const { return list_.length(); }
+ LiveBundle* spilledBundle(size_t i) const { return list_[i]; }
+
+ void setAllocation(LAllocation alloc);
+};
+
+#ifdef JS_JITSPEW
+// See comment on LiveBundle::debugId_ just below. This needs to be atomic
+// because TSan automation runs on debug builds will otherwise (correctly)
+// report a race.
+static mozilla::Atomic<uint32_t> LiveBundle_debugIdCounter =
+ mozilla::Atomic<uint32_t>{0};
+#endif
+
+// A set of live ranges which are all pairwise disjoint. The register allocator
+// attempts to find allocations for an entire bundle, and if it fails the
+// bundle will be broken into smaller ones which are allocated independently.
+class LiveBundle : public TempObject {
+ // Set to use if this bundle or one it is split into is spilled.
+ SpillSet* spill_;
+
+ // All the ranges in this set, ordered by location.
+ InlineForwardList<LiveRange::BundleLink> ranges_;
+
+ // Allocation to use for ranges in this set, bogus if unallocated or spilled
+ // and not yet given a physical stack slot.
+ LAllocation alloc_;
+
+ // Bundle which entirely contains this one and has no register uses. This
+ // may or may not be spilled by the allocator, but it can be spilled and
+ // will not be split.
+ LiveBundle* spillParent_;
+
+#ifdef JS_JITSPEW
+ // This is used only for debug-printing bundles. It gives them an
+ // identifiable identity in the debug output, which they otherwise wouldn't
+ // have.
+ uint32_t debugId_;
+#endif
+
+ LiveBundle(SpillSet* spill, LiveBundle* spillParent)
+ : spill_(spill), spillParent_(spillParent) {
+#ifdef JS_JITSPEW
+ debugId_ = LiveBundle_debugIdCounter++;
+#endif
+ }
+
+ public:
+ static LiveBundle* FallibleNew(TempAllocator& alloc, SpillSet* spill,
+ LiveBundle* spillParent) {
+ return new (alloc.fallible()) LiveBundle(spill, spillParent);
+ }
+
+ SpillSet* spillSet() const { return spill_; }
+ void setSpillSet(SpillSet* spill) { spill_ = spill; }
+
+ LiveRange::BundleLinkIterator rangesBegin() const { return ranges_.begin(); }
+ bool hasRanges() const { return !!rangesBegin(); }
+ LiveRange* firstRange() const { return LiveRange::get(*rangesBegin()); }
+ LiveRange* lastRange() const { return LiveRange::get(ranges_.back()); }
+ LiveRange* rangeFor(CodePosition pos) const;
+ void removeRange(LiveRange* range);
+ void removeRangeAndIncrementIterator(LiveRange::BundleLinkIterator& iter) {
+ ranges_.removeAndIncrement(iter);
+ }
+ void addRange(LiveRange* range);
+ [[nodiscard]] bool addRange(TempAllocator& alloc, VirtualRegister* vreg,
+ CodePosition from, CodePosition to);
+ [[nodiscard]] bool addRangeAndDistributeUses(TempAllocator& alloc,
+ LiveRange* oldRange,
+ CodePosition from,
+ CodePosition to);
+ LiveRange* popFirstRange();
+#ifdef DEBUG
+ size_t numRanges() const;
+#endif
+
+ LAllocation allocation() const { return alloc_; }
+ void setAllocation(LAllocation alloc) { alloc_ = alloc; }
+
+ LiveBundle* spillParent() const { return spillParent_; }
+
+#ifdef JS_JITSPEW
+ uint32_t debugId() const { return debugId_; }
+
+ // Return a string describing this bundle.
+ UniqueChars toString() const;
+#endif
+};
+
+// Information about the allocation for a virtual register.
+class VirtualRegister {
+ // Instruction which defines this register.
+ LNode* ins_ = nullptr;
+
+ // Definition in the instruction for this register.
+ LDefinition* def_ = nullptr;
+
+ // All live ranges for this register. These may overlap each other, and are
+ // ordered by their start position.
+ InlineForwardList<LiveRange::RegisterLink> ranges_;
+
+ // Whether def_ is a temp or an output.
+ bool isTemp_ = false;
+
+ // Whether this vreg is an input for some phi. This use is not reflected in
+ // any range on the vreg.
+ bool usedByPhi_ = false;
+
+ // If this register's definition is MUST_REUSE_INPUT, whether a copy must
+ // be introduced before the definition that relaxes the policy.
+ bool mustCopyInput_ = false;
+
+ void operator=(const VirtualRegister&) = delete;
+ VirtualRegister(const VirtualRegister&) = delete;
+
+ public:
+ VirtualRegister() = default;
+
+ void init(LNode* ins, LDefinition* def, bool isTemp) {
+ MOZ_ASSERT(!ins_);
+ ins_ = ins;
+ def_ = def;
+ isTemp_ = isTemp;
+ }
+
+ LNode* ins() const { return ins_; }
+ LDefinition* def() const { return def_; }
+ LDefinition::Type type() const { return def()->type(); }
+ uint32_t vreg() const { return def()->virtualRegister(); }
+ bool isCompatible(const AnyRegister& r) const {
+ return def_->isCompatibleReg(r);
+ }
+ bool isCompatible(const VirtualRegister& vr) const {
+ return def_->isCompatibleDef(*vr.def_);
+ }
+ bool isTemp() const { return isTemp_; }
+
+ void setUsedByPhi() { usedByPhi_ = true; }
+ bool usedByPhi() { return usedByPhi_; }
+
+ void setMustCopyInput() { mustCopyInput_ = true; }
+ bool mustCopyInput() { return mustCopyInput_; }
+
+ LiveRange::RegisterLinkIterator rangesBegin() const {
+ return ranges_.begin();
+ }
+ LiveRange::RegisterLinkIterator rangesBegin(LiveRange* range) const {
+ return ranges_.begin(&range->registerLink);
+ }
+ bool hasRanges() const { return !!rangesBegin(); }
+ LiveRange* firstRange() const { return LiveRange::get(*rangesBegin()); }
+ LiveRange* lastRange() const { return LiveRange::get(ranges_.back()); }
+ LiveRange* rangeFor(CodePosition pos, bool preferRegister = false) const;
+ void removeRange(LiveRange* range);
+ void addRange(LiveRange* range);
+
+ void removeRangeAndIncrement(LiveRange::RegisterLinkIterator& iter) {
+ ranges_.removeAndIncrement(iter);
+ }
+
+ LiveBundle* firstBundle() const { return firstRange()->bundle(); }
+
+ [[nodiscard]] bool addInitialRange(TempAllocator& alloc, CodePosition from,
+ CodePosition to, size_t* numRanges);
+ void addInitialUse(UsePosition* use);
+ void setInitialDefinition(CodePosition from);
+};
+
+// A sequence of code positions, for tellings BacktrackingAllocator::splitAt
+// where to split.
+using SplitPositionVector = js::Vector<CodePosition, 4, SystemAllocPolicy>;
+
+class BacktrackingAllocator : protected RegisterAllocator {
+ friend class JSONSpewer;
+
+ // This flag is set when testing new allocator modifications.
+ bool testbed;
+
+ BitSet* liveIn;
+ FixedList<VirtualRegister> vregs;
+
+ // Allocation state.
+ StackSlotAllocator stackSlotAllocator;
+
+ // Priority queue element: a bundle and the associated priority.
+ struct QueueItem {
+ LiveBundle* bundle;
+
+ QueueItem(LiveBundle* bundle, size_t priority)
+ : bundle(bundle), priority_(priority) {}
+
+ static size_t priority(const QueueItem& v) { return v.priority_; }
+
+ private:
+ size_t priority_;
+ };
+
+ PriorityQueue<QueueItem, QueueItem, 0, SystemAllocPolicy> allocationQueue;
+
+ // This is a set of LiveRanges. They must be non-overlapping. Attempts to
+ // add an overlapping range will cause AvlTree::insert to MOZ_CRASH().
+ using LiveRangeSet = AvlTree<LiveRange*, LiveRange>;
+
+ // Each physical register is associated with the set of ranges over which
+ // that register is currently allocated.
+ struct PhysicalRegister {
+ bool allocatable;
+ AnyRegister reg;
+ LiveRangeSet allocations;
+
+ PhysicalRegister() : allocatable(false) {}
+ };
+ mozilla::Array<PhysicalRegister, AnyRegister::Total> registers;
+
+ // Ranges of code which are considered to be hot, for which good allocation
+ // should be prioritized.
+ LiveRangeSet hotcode;
+
+ struct CallRange : public TempObject, public InlineListNode<CallRange> {
+ LiveRange::Range range;
+
+ CallRange(CodePosition from, CodePosition to) : range(from, to) {}
+
+ // Comparator for use in splay tree.
+ static int compare(CallRange* v0, CallRange* v1) {
+ if (v0->range.to <= v1->range.from) {
+ return -1;
+ }
+ if (v0->range.from >= v1->range.to) {
+ return 1;
+ }
+ return 0;
+ }
+ };
+
+ // Ranges where all registers must be spilled due to call instructions.
+ using CallRangeList = InlineList<CallRange>;
+ CallRangeList callRangesList;
+ AvlTree<CallRange*, CallRange> callRanges;
+
+ // Information about an allocated stack slot.
+ struct SpillSlot : public TempObject,
+ public InlineForwardListNode<SpillSlot> {
+ LStackSlot alloc;
+ LiveRangeSet allocated;
+
+ SpillSlot(uint32_t slot, LifoAlloc* alloc)
+ : alloc(slot), allocated(alloc) {}
+ };
+ using SpillSlotList = InlineForwardList<SpillSlot>;
+
+ // All allocated slots of each width.
+ SpillSlotList normalSlots, doubleSlots, quadSlots;
+
+ Vector<LiveBundle*, 4, SystemAllocPolicy> spilledBundles;
+
+ using LiveBundleVector = Vector<LiveBundle*, 4, SystemAllocPolicy>;
+
+ // Misc accessors
+ bool compilingWasm() { return mir->outerInfo().compilingWasm(); }
+ VirtualRegister& vreg(const LDefinition* def) {
+ return vregs[def->virtualRegister()];
+ }
+ VirtualRegister& vreg(const LAllocation* alloc) {
+ MOZ_ASSERT(alloc->isUse());
+ return vregs[alloc->toUse()->virtualRegister()];
+ }
+
+ // Helpers for creating and adding MoveGroups
+ [[nodiscard]] bool addMove(LMoveGroup* moves, LiveRange* from, LiveRange* to,
+ LDefinition::Type type) {
+ LAllocation fromAlloc = from->bundle()->allocation();
+ LAllocation toAlloc = to->bundle()->allocation();
+ MOZ_ASSERT(fromAlloc != toAlloc);
+ return moves->add(fromAlloc, toAlloc, type);
+ }
+
+ [[nodiscard]] bool moveInput(LInstruction* ins, LiveRange* from,
+ LiveRange* to, LDefinition::Type type) {
+ if (from->bundle()->allocation() == to->bundle()->allocation()) {
+ return true;
+ }
+ LMoveGroup* moves = getInputMoveGroup(ins);
+ return addMove(moves, from, to, type);
+ }
+
+ [[nodiscard]] bool moveAfter(LInstruction* ins, LiveRange* from,
+ LiveRange* to, LDefinition::Type type) {
+ if (from->bundle()->allocation() == to->bundle()->allocation()) {
+ return true;
+ }
+ LMoveGroup* moves = getMoveGroupAfter(ins);
+ return addMove(moves, from, to, type);
+ }
+
+ [[nodiscard]] bool moveAtExit(LBlock* block, LiveRange* from, LiveRange* to,
+ LDefinition::Type type) {
+ if (from->bundle()->allocation() == to->bundle()->allocation()) {
+ return true;
+ }
+ LMoveGroup* moves = block->getExitMoveGroup(alloc());
+ return addMove(moves, from, to, type);
+ }
+
+ [[nodiscard]] bool moveAtEntry(LBlock* block, LiveRange* from, LiveRange* to,
+ LDefinition::Type type) {
+ if (from->bundle()->allocation() == to->bundle()->allocation()) {
+ return true;
+ }
+ LMoveGroup* moves = block->getEntryMoveGroup(alloc());
+ return addMove(moves, from, to, type);
+ }
+
+ // Out-of-line methods, in the same sequence as in BacktrackingAllocator.cpp.
+
+ // Misc helpers: queries about uses
+ bool isReusedInput(LUse* use, LNode* ins, bool considerCopy);
+ bool isRegisterUse(UsePosition* use, LNode* ins, bool considerCopy = false);
+ bool isRegisterDefinition(LiveRange* range);
+
+ // Misc helpers: atomic LIR groups
+ // (these are all in the parent class, RegisterAllocator)
+
+ // Misc helpers: computation of bundle priorities and spill weights
+ size_t computePriority(LiveBundle* bundle);
+ bool minimalDef(LiveRange* range, LNode* ins);
+ bool minimalUse(LiveRange* range, UsePosition* use);
+ bool minimalBundle(LiveBundle* bundle, bool* pfixed = nullptr);
+ size_t computeSpillWeight(LiveBundle* bundle);
+ size_t maximumSpillWeight(const LiveBundleVector& bundles);
+
+ // Initialization of the allocator
+ [[nodiscard]] bool init();
+
+ // Liveness analysis
+ [[nodiscard]] bool addInitialFixedRange(AnyRegister reg, CodePosition from,
+ CodePosition to);
+ [[nodiscard]] bool buildLivenessInfo();
+
+ // Merging and queueing of LiveRange groups
+ [[nodiscard]] bool tryMergeBundles(LiveBundle* bundle0, LiveBundle* bundle1);
+ void allocateStackDefinition(VirtualRegister& reg);
+ [[nodiscard]] bool tryMergeReusedRegister(VirtualRegister& def,
+ VirtualRegister& input);
+ [[nodiscard]] bool mergeAndQueueRegisters();
+
+ // Implementation of splitting decisions, but not the making of those
+ // decisions
+ [[nodiscard]] bool updateVirtualRegisterListsThenRequeueBundles(
+ LiveBundle* bundle, const LiveBundleVector& newBundles);
+
+ // Implementation of splitting decisions, but not the making of those
+ // decisions
+ [[nodiscard]] bool splitAt(LiveBundle* bundle,
+ const SplitPositionVector& splitPositions);
+
+ // Creation of splitting decisions, but not their implementation
+ [[nodiscard]] bool splitAcrossCalls(LiveBundle* bundle);
+ [[nodiscard]] bool trySplitAcrossHotcode(LiveBundle* bundle, bool* success);
+ [[nodiscard]] bool trySplitAfterLastRegisterUse(LiveBundle* bundle,
+ LiveBundle* conflict,
+ bool* success);
+ [[nodiscard]] bool trySplitBeforeFirstRegisterUse(LiveBundle* bundle,
+ LiveBundle* conflict,
+ bool* success);
+
+ // The top level driver for the splitting machinery
+ [[nodiscard]] bool chooseBundleSplit(LiveBundle* bundle, bool fixed,
+ LiveBundle* conflict);
+
+ // Bundle allocation
+ [[nodiscard]] bool computeRequirement(LiveBundle* bundle,
+ Requirement* prequirement,
+ Requirement* phint);
+ [[nodiscard]] bool tryAllocateRegister(PhysicalRegister& r,
+ LiveBundle* bundle, bool* success,
+ bool* pfixed,
+ LiveBundleVector& conflicting);
+ [[nodiscard]] bool tryAllocateAnyRegister(LiveBundle* bundle, bool* success,
+ bool* pfixed,
+ LiveBundleVector& conflicting);
+ [[nodiscard]] bool evictBundle(LiveBundle* bundle);
+ [[nodiscard]] bool tryAllocateFixed(LiveBundle* bundle,
+ Requirement requirement, bool* success,
+ bool* pfixed,
+ LiveBundleVector& conflicting);
+ [[nodiscard]] bool tryAllocateNonFixed(LiveBundle* bundle,
+ Requirement requirement,
+ Requirement hint, bool* success,
+ bool* pfixed,
+ LiveBundleVector& conflicting);
+ [[nodiscard]] bool processBundle(MIRGenerator* mir, LiveBundle* bundle);
+ [[nodiscard]] bool spill(LiveBundle* bundle);
+ [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool
+ tryAllocatingRegistersForSpillBundles();
+
+ // Rewriting of the LIR after bundle processing is done
+ [[nodiscard]] bool insertAllRanges(LiveRangeSet& set, LiveBundle* bundle);
+ [[nodiscard]] bool pickStackSlot(SpillSet* spill);
+ [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool pickStackSlots();
+ [[nodiscard]] bool moveAtEdge(LBlock* predecessor, LBlock* successor,
+ LiveRange* from, LiveRange* to,
+ LDefinition::Type type);
+ [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool deadRange(LiveRange* range);
+ [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool
+ createMoveGroupsFromLiveRangeTransitions();
+ size_t findFirstNonCallSafepoint(CodePosition from);
+ void addLiveRegistersForRange(VirtualRegister& reg, LiveRange* range);
+ [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool installAllocationsInLIR();
+ size_t findFirstSafepoint(CodePosition pos, size_t startFrom);
+ [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool populateSafepoints();
+ [[nodiscard]] AVOID_INLINE_FOR_DEBUGGING bool annotateMoveGroups();
+
+ // Debug-printing support
+#ifdef JS_JITSPEW
+ void dumpLiveRangesByVReg(const char* who);
+ void dumpLiveRangesByBundle(const char* who);
+ void dumpAllocations();
+#endif
+
+ // Top level of the register allocation machinery, and the only externally
+ // visible bit.
+ public:
+ BacktrackingAllocator(MIRGenerator* mir, LIRGenerator* lir, LIRGraph& graph,
+ bool testbed)
+ : RegisterAllocator(mir, lir, graph),
+ testbed(testbed),
+ liveIn(nullptr),
+ callRanges(nullptr) {}
+
+ [[nodiscard]] bool go();
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BacktrackingAllocator_h */
diff --git a/js/src/jit/Bailouts.cpp b/js/src/jit/Bailouts.cpp
new file mode 100644
index 0000000000..fc84969029
--- /dev/null
+++ b/js/src/jit/Bailouts.cpp
@@ -0,0 +1,336 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Bailouts.h"
+
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/ScopeExit.h"
+
+#include "gc/GC.h"
+#include "jit/Assembler.h" // jit::FramePointer
+#include "jit/BaselineJIT.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/JSJitFrameIter.h"
+#include "jit/SafepointIndex.h"
+#include "jit/ScriptFromCalleeToken.h"
+#include "vm/Interpreter.h"
+#include "vm/JSContext.h"
+#include "vm/Stack.h"
+
+#include "vm/JSScript-inl.h"
+#include "vm/Probes-inl.h"
+#include "vm/Stack-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::IsInRange;
+
+#if defined(_WIN32)
+# pragma pack(push, 1)
+#endif
+class js::jit::BailoutStack {
+ RegisterDump::FPUArray fpregs_;
+ RegisterDump::GPRArray regs_;
+ uintptr_t frameSize_;
+ uintptr_t snapshotOffset_;
+
+ public:
+ MachineState machineState() {
+ return MachineState::FromBailout(regs_, fpregs_);
+ }
+ uint32_t snapshotOffset() const { return snapshotOffset_; }
+ uint32_t frameSize() const { return frameSize_; }
+ uint8_t* parentStackPointer() {
+ return (uint8_t*)this + sizeof(BailoutStack);
+ }
+};
+#if defined(_WIN32)
+# pragma pack(pop)
+#endif
+
+// Make sure the compiler doesn't add extra padding on 32-bit platforms.
+static_assert((sizeof(BailoutStack) % 8) == 0,
+ "BailoutStack should be 8-byte aligned.");
+
+BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations,
+ BailoutStack* bailout)
+ : machine_(bailout->machineState()), activation_(nullptr) {
+ uint8_t* sp = bailout->parentStackPointer();
+ framePointer_ = sp + bailout->frameSize();
+ MOZ_RELEASE_ASSERT(uintptr_t(framePointer_) == machine_.read(FramePointer));
+
+ JSScript* script =
+ ScriptFromCalleeToken(((JitFrameLayout*)framePointer_)->calleeToken());
+ topIonScript_ = script->ionScript();
+
+ attachOnJitActivation(activations);
+ snapshotOffset_ = bailout->snapshotOffset();
+}
+
+BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations,
+ InvalidationBailoutStack* bailout)
+ : machine_(bailout->machine()), activation_(nullptr) {
+ framePointer_ = (uint8_t*)bailout->fp();
+ MOZ_RELEASE_ASSERT(uintptr_t(framePointer_) == machine_.read(FramePointer));
+
+ topIonScript_ = bailout->ionScript();
+ attachOnJitActivation(activations);
+
+ uint8_t* returnAddressToFp_ = bailout->osiPointReturnAddress();
+ const OsiIndex* osiIndex = topIonScript_->getOsiIndex(returnAddressToFp_);
+ snapshotOffset_ = osiIndex->snapshotOffset();
+}
+
+BailoutFrameInfo::BailoutFrameInfo(const JitActivationIterator& activations,
+ const JSJitFrameIter& frame)
+ : machine_(frame.machineState()) {
+ framePointer_ = (uint8_t*)frame.fp();
+ topIonScript_ = frame.ionScript();
+ attachOnJitActivation(activations);
+
+ const OsiIndex* osiIndex = frame.osiIndex();
+ snapshotOffset_ = osiIndex->snapshotOffset();
+}
+
+// This address is a magic number made to cause crashes while indicating that we
+// are making an attempt to mark the stack during a bailout.
+static constexpr uint32_t FAKE_EXITFP_FOR_BAILOUT_ADDR = 0xba2;
+static uint8_t* const FAKE_EXITFP_FOR_BAILOUT =
+ reinterpret_cast<uint8_t*>(FAKE_EXITFP_FOR_BAILOUT_ADDR);
+
+static_assert(!(FAKE_EXITFP_FOR_BAILOUT_ADDR & wasm::ExitFPTag),
+ "FAKE_EXITFP_FOR_BAILOUT could be mistaken as a low-bit tagged "
+ "wasm exit fp");
+
+bool jit::Bailout(BailoutStack* sp, BaselineBailoutInfo** bailoutInfo) {
+ JSContext* cx = TlsContext.get();
+ MOZ_ASSERT(bailoutInfo);
+
+ // We don't have an exit frame.
+ MOZ_ASSERT(IsInRange(FAKE_EXITFP_FOR_BAILOUT, 0, 0x1000) &&
+ IsInRange(FAKE_EXITFP_FOR_BAILOUT + sizeof(CommonFrameLayout),
+ 0, 0x1000),
+ "Fake exitfp pointer should be within the first page.");
+
+ cx->activation()->asJit()->setJSExitFP(FAKE_EXITFP_FOR_BAILOUT);
+
+ JitActivationIterator jitActivations(cx);
+ BailoutFrameInfo bailoutData(jitActivations, sp);
+ JSJitFrameIter frame(jitActivations->asJit());
+ MOZ_ASSERT(!frame.ionScript()->invalidated());
+ JitFrameLayout* currentFramePtr = frame.jsFrame();
+
+ JitSpew(JitSpew_IonBailouts, "Took bailout! Snapshot offset: %u",
+ frame.snapshotOffset());
+
+ MOZ_ASSERT(IsBaselineJitEnabled(cx));
+
+ *bailoutInfo = nullptr;
+ bool success =
+ BailoutIonToBaseline(cx, bailoutData.activation(), frame, bailoutInfo,
+ /*exceptionInfo=*/nullptr, BailoutReason::Normal);
+ MOZ_ASSERT_IF(success, *bailoutInfo != nullptr);
+
+ if (!success) {
+ MOZ_ASSERT(cx->isExceptionPending());
+ JSScript* script = frame.script();
+ probes::ExitScript(cx, script, script->function(),
+ /* popProfilerFrame = */ false);
+ }
+
+ // This condition was wrong when we entered this bailout function, but it
+ // might be true now. A GC might have reclaimed all the Jit code and
+ // invalidated all frames which are currently on the stack. As we are
+ // already in a bailout, we could not switch to an invalidation
+ // bailout. When the code of an IonScript which is on the stack is
+ // invalidated (see InvalidateActivation), we remove references to it and
+ // increment the reference counter for each activation that appear on the
+ // stack. As the bailed frame is one of them, we have to decrement it now.
+ if (frame.ionScript()->invalidated()) {
+ frame.ionScript()->decrementInvalidationCount(cx->gcContext());
+ }
+
+ // NB: Commentary on how |lastProfilingFrame| is set from bailouts.
+ //
+ // Once we return to jitcode, any following frames might get clobbered,
+ // but the current frame will not (as it will be clobbered "in-place"
+ // with a baseline frame that will share the same frame prefix).
+ // However, there may be multiple baseline frames unpacked from this
+ // single Ion frame, which means we will need to once again reset
+ // |lastProfilingFrame| to point to the correct unpacked last frame
+ // in |FinishBailoutToBaseline|.
+ //
+ // In the case of error, the jitcode will jump immediately to an
+ // exception handler, which will unwind the frames and properly set
+ // the |lastProfilingFrame| to point to the frame being resumed into
+ // (see |AutoResetLastProfilerFrameOnReturnFromException|).
+ //
+ // In both cases, we want to temporarily set the |lastProfilingFrame|
+ // to the current frame being bailed out, and then fix it up later.
+ if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(
+ cx->runtime())) {
+ cx->jitActivation->setLastProfilingFrame(currentFramePtr);
+ }
+
+ return success;
+}
+
+bool jit::InvalidationBailout(InvalidationBailoutStack* sp,
+ BaselineBailoutInfo** bailoutInfo) {
+ sp->checkInvariants();
+
+ JSContext* cx = TlsContext.get();
+
+ // We don't have an exit frame.
+ cx->activation()->asJit()->setJSExitFP(FAKE_EXITFP_FOR_BAILOUT);
+
+ JitActivationIterator jitActivations(cx);
+ BailoutFrameInfo bailoutData(jitActivations, sp);
+ JSJitFrameIter frame(jitActivations->asJit());
+ JitFrameLayout* currentFramePtr = frame.jsFrame();
+
+ JitSpew(JitSpew_IonBailouts, "Took invalidation bailout! Snapshot offset: %u",
+ frame.snapshotOffset());
+
+ MOZ_ASSERT(IsBaselineJitEnabled(cx));
+
+ *bailoutInfo = nullptr;
+ bool success = BailoutIonToBaseline(cx, bailoutData.activation(), frame,
+ bailoutInfo, /*exceptionInfo=*/nullptr,
+ BailoutReason::Invalidate);
+ MOZ_ASSERT_IF(success, *bailoutInfo != nullptr);
+
+ if (!success) {
+ MOZ_ASSERT(cx->isExceptionPending());
+
+ // If the bailout failed, then bailout trampoline will pop the
+ // current frame and jump straight to exception handling code when
+ // this function returns. Any Gecko Profiler entry pushed for this
+ // frame will be silently forgotten.
+ //
+ // We call ExitScript here to ensure that if the ionScript had Gecko
+ // Profiler instrumentation, then the entry for it is popped.
+ //
+ // However, if the bailout was during argument check, then a
+ // pseudostack frame would not have been pushed in the first
+ // place, so don't pop anything in that case.
+ JSScript* script = frame.script();
+ probes::ExitScript(cx, script, script->function(),
+ /* popProfilerFrame = */ false);
+
+#ifdef JS_JITSPEW
+ JitFrameLayout* layout = frame.jsFrame();
+ JitSpew(JitSpew_IonInvalidate, "Bailout failed (Fatal Error)");
+ JitSpew(JitSpew_IonInvalidate, " calleeToken %p",
+ (void*)layout->calleeToken());
+ JitSpew(JitSpew_IonInvalidate, " callerFramePtr %p",
+ layout->callerFramePtr());
+ JitSpew(JitSpew_IonInvalidate, " ra %p", (void*)layout->returnAddress());
+#endif
+ }
+
+ frame.ionScript()->decrementInvalidationCount(cx->gcContext());
+
+ // Make the frame being bailed out the top profiled frame.
+ if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(
+ cx->runtime())) {
+ cx->jitActivation->setLastProfilingFrame(currentFramePtr);
+ }
+
+ return success;
+}
+
+bool jit::ExceptionHandlerBailout(JSContext* cx,
+ const InlineFrameIterator& frame,
+ ResumeFromException* rfe,
+ const ExceptionBailoutInfo& excInfo) {
+ // If we are resuming in a finally block, the exception has already
+ // been captured.
+ // We can also be propagating debug mode exceptions without there being an
+ // actual exception pending. For instance, when we return false from an
+ // operation callback like a timeout handler.
+ MOZ_ASSERT_IF(
+ !cx->isExceptionPending(),
+ excInfo.isFinally() || excInfo.propagatingIonExceptionForDebugMode());
+
+ JS::AutoSaveExceptionState savedExc(cx);
+
+ JitActivation* act = cx->activation()->asJit();
+ uint8_t* prevExitFP = act->jsExitFP();
+ auto restoreExitFP =
+ mozilla::MakeScopeExit([&]() { act->setJSExitFP(prevExitFP); });
+ act->setJSExitFP(FAKE_EXITFP_FOR_BAILOUT);
+
+ gc::AutoSuppressGC suppress(cx);
+
+ JitActivationIterator jitActivations(cx);
+ BailoutFrameInfo bailoutData(jitActivations, frame.frame());
+ JSJitFrameIter frameView(jitActivations->asJit());
+ JitFrameLayout* currentFramePtr = frameView.jsFrame();
+
+ BaselineBailoutInfo* bailoutInfo = nullptr;
+ bool success = BailoutIonToBaseline(cx, bailoutData.activation(), frameView,
+ &bailoutInfo, &excInfo,
+ BailoutReason::ExceptionHandler);
+ if (success) {
+ MOZ_ASSERT(bailoutInfo);
+
+ // Overwrite the kind so HandleException after the bailout returns
+ // false, jumping directly to the exception tail.
+ if (excInfo.propagatingIonExceptionForDebugMode()) {
+ bailoutInfo->bailoutKind =
+ mozilla::Some(BailoutKind::IonExceptionDebugMode);
+ } else if (excInfo.isFinally()) {
+ bailoutInfo->bailoutKind = mozilla::Some(BailoutKind::Finally);
+ }
+
+ rfe->kind = ExceptionResumeKind::Bailout;
+ rfe->stackPointer = bailoutInfo->incomingStack;
+ rfe->bailoutInfo = bailoutInfo;
+ } else {
+ // Drop the exception that triggered the bailout and instead propagate the
+ // failure caused by processing the bailout (eg. OOM).
+ savedExc.drop();
+ MOZ_ASSERT(!bailoutInfo);
+ MOZ_ASSERT(cx->isExceptionPending());
+ }
+
+ // Make the frame being bailed out the top profiled frame.
+ if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(
+ cx->runtime())) {
+ cx->jitActivation->setLastProfilingFrame(currentFramePtr);
+ }
+
+ return success;
+}
+
+// Initialize the NamedLambdaObject and CallObject of the current frame if
+// needed.
+bool jit::EnsureHasEnvironmentObjects(JSContext* cx, AbstractFramePtr fp) {
+ // Ion does not compile eval scripts.
+ MOZ_ASSERT(!fp.isEvalFrame());
+
+ if (fp.isFunctionFrame() && !fp.hasInitialEnvironment() &&
+ fp.callee()->needsFunctionEnvironmentObjects()) {
+ if (!fp.initFunctionEnvironmentObjects(cx)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void BailoutFrameInfo::attachOnJitActivation(
+ const JitActivationIterator& jitActivations) {
+ MOZ_ASSERT(jitActivations->asJit()->jsExitFP() == FAKE_EXITFP_FOR_BAILOUT);
+ activation_ = jitActivations->asJit();
+ activation_->setBailoutData(this);
+}
+
+BailoutFrameInfo::~BailoutFrameInfo() { activation_->cleanBailoutData(); }
diff --git a/js/src/jit/Bailouts.h b/js/src/jit/Bailouts.h
new file mode 100644
index 0000000000..759f384019
--- /dev/null
+++ b/js/src/jit/Bailouts.h
@@ -0,0 +1,227 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Bailouts_h
+#define jit_Bailouts_h
+
+#include "mozilla/Assertions.h" // MOZ_ASSERT
+
+#include <stddef.h> // size_t
+#include <stdint.h> // uint8_t, uint32_t
+
+#include "jstypes.h"
+
+#include "jit/IonTypes.h" // js::jit::Bailout{Id,Kind}, js::jit::SnapshotOffset
+#include "jit/MachineState.h" // js::jit::MachineState
+#include "js/TypeDecls.h" // jsbytecode
+#include "vm/JSContext.h" // JSContext
+
+namespace js {
+
+class AbstractFramePtr;
+
+namespace jit {
+
+// [SMDOC] IonMonkey Bailouts
+//
+// A "bailout" is the process of recovering a baseline interpreter frame from an
+// IonFrame. Bailouts are implemented in js::jit::BailoutIonToBaseline, which
+// has the following callers:
+//
+// * js::jit::Bailout - This is used when a guard fails in the Ion code
+// itself; for example, an LGuardShape fails or an LAddI overflows. See
+// callers of CodeGenerator::bailoutFrom() for more examples.
+//
+// * js::jit::ExceptionHandlerBailout - Ion doesn't implement `catch` or
+// `finally`. If an exception is thrown and would be caught by an Ion frame,
+// we bail out instead.
+//
+// * js::jit::InvalidationBailout - We returned to Ion code that was
+// invalidated while it was on the stack. See "OSI" below. Ion code can be
+// invalidated for several reasons: when GC evicts Ion code to save memory,
+// for example, or when assumptions baked into the jitted code are
+// invalidated by the VM.
+//
+// (Some stack inspection can be done without bailing out, including GC stack
+// marking, Error object construction, and Gecko profiler sampling.)
+//
+// Consider the first case. When an Ion guard fails, we can't continue in
+// Ion. There's no IC fallback case coming to save us; we've got a broken
+// assumption baked into the code we're running. So we jump to an out-of-line
+// code path that's responsible for abandoning Ion execution and resuming in
+// the baseline interpreter: the bailout path.
+//
+// We were in the midst of optimized Ion code, so bits of program state may be
+// in registers or spilled to the native stack; values may be unboxed; some
+// objects may have been optimized away; thanks to inlining, whole call frames
+// may be missing. The bailout path must put all these pieces back together
+// into the structure the baseline interpreter expects.
+//
+// The data structure that makes this possible is called a *snapshot*.
+// Snapshots are created during Ion codegen and associated with the IonScript;
+// they tell how to recover each value in a BaselineFrame from the current
+// machine state at a given point in the Ion JIT code. This is potentially
+// different at every place in an Ion script where we might bail out. (See
+// Snapshots.h.)
+//
+// The bailout path performs roughly the following steps:
+//
+// 1. Push a snapshot index and the frame size to the native stack.
+// 2. Spill all registers.
+// 3. Call js::jit::Bailout to reconstruct the baseline frame(s).
+// 4. memmove() those to the right place on the native stack.
+// 5. Jump into the baseline interpreter.
+//
+// When C++ code invalidates Ion code, we do on-stack invalidation, or OSI, to
+// arrange for every affected Ion frame on the stack to bail out as soon as
+// control returns to it. OSI patches every instruction in the JIT code that's
+// at a return address currently on the stack. See InvalidateActivation.
+//
+//
+// ## Bailout path implementation details
+//
+// Ion code has a lot of guards, so each bailout path must be small. Steps 2
+// and 3 above are therefore implemented by a shared per-Runtime trampoline,
+// rt->jitRuntime()->getGenericBailoutHandler().
+//
+// We implement step 1 like this:
+//
+// _bailout_ID_1:
+// push 1
+// jmp _deopt
+// _bailout_ID_2:
+// push 2
+// jmp _deopt
+// ...
+// _deopt:
+// push imm(FrameSize)
+// call _global_bailout_handler
+
+// BailoutStack is an architecture specific pointer to the stack, given by the
+// bailout handler.
+class BailoutStack;
+class InvalidationBailoutStack;
+
+class IonScript;
+class InlineFrameIterator;
+class JitActivation;
+class JitActivationIterator;
+class JSJitFrameIter;
+struct ResumeFromException;
+
+// Must be implemented by each architecture.
+
+// This structure is constructed before recovering the baseline frames for a
+// bailout. It records all information extracted from the stack, and which are
+// needed for the JSJitFrameIter.
+class BailoutFrameInfo {
+ MachineState machine_;
+ uint8_t* framePointer_;
+ IonScript* topIonScript_;
+ uint32_t snapshotOffset_;
+ JitActivation* activation_;
+
+ void attachOnJitActivation(const JitActivationIterator& activations);
+
+ public:
+ BailoutFrameInfo(const JitActivationIterator& activations, BailoutStack* sp);
+ BailoutFrameInfo(const JitActivationIterator& activations,
+ InvalidationBailoutStack* sp);
+ BailoutFrameInfo(const JitActivationIterator& activations,
+ const JSJitFrameIter& frame);
+ ~BailoutFrameInfo();
+
+ uint8_t* fp() const { return framePointer_; }
+ SnapshotOffset snapshotOffset() const { return snapshotOffset_; }
+ const MachineState* machineState() const { return &machine_; }
+ IonScript* ionScript() const { return topIonScript_; }
+ JitActivation* activation() const { return activation_; }
+};
+
+[[nodiscard]] bool EnsureHasEnvironmentObjects(JSContext* cx,
+ AbstractFramePtr fp);
+
+struct BaselineBailoutInfo;
+
+// Called from a bailout thunk.
+[[nodiscard]] bool Bailout(BailoutStack* sp, BaselineBailoutInfo** info);
+
+// Called from the invalidation thunk.
+[[nodiscard]] bool InvalidationBailout(InvalidationBailoutStack* sp,
+ BaselineBailoutInfo** info);
+
+class ExceptionBailoutInfo {
+ size_t frameNo_;
+ jsbytecode* resumePC_;
+ size_t numExprSlots_;
+ bool isFinally_ = false;
+ RootedValue finallyException_;
+ bool forcedReturn_;
+
+ public:
+ ExceptionBailoutInfo(JSContext* cx, size_t frameNo, jsbytecode* resumePC,
+ size_t numExprSlots)
+ : frameNo_(frameNo),
+ resumePC_(resumePC),
+ numExprSlots_(numExprSlots),
+ finallyException_(cx),
+ forcedReturn_(cx->isPropagatingForcedReturn()) {}
+
+ explicit ExceptionBailoutInfo(JSContext* cx)
+ : frameNo_(0),
+ resumePC_(nullptr),
+ numExprSlots_(0),
+ finallyException_(cx),
+ forcedReturn_(cx->isPropagatingForcedReturn()) {}
+
+ bool catchingException() const { return !!resumePC_; }
+ bool propagatingIonExceptionForDebugMode() const { return !resumePC_; }
+
+ size_t frameNo() const {
+ MOZ_ASSERT(catchingException());
+ return frameNo_;
+ }
+ jsbytecode* resumePC() const {
+ MOZ_ASSERT(catchingException());
+ return resumePC_;
+ }
+ size_t numExprSlots() const {
+ MOZ_ASSERT(catchingException());
+ return numExprSlots_;
+ }
+
+ bool isFinally() const { return isFinally_; }
+ void setFinallyException(JS::Value& exception) {
+ MOZ_ASSERT(!isFinally());
+ isFinally_ = true;
+ finallyException_ = exception;
+ }
+ HandleValue finallyException() const {
+ MOZ_ASSERT(isFinally());
+ return finallyException_;
+ }
+
+ bool forcedReturn() const { return forcedReturn_; }
+};
+
+// Called from the exception handler to enter a catch or finally block.
+[[nodiscard]] bool ExceptionHandlerBailout(JSContext* cx,
+ const InlineFrameIterator& frame,
+ ResumeFromException* rfe,
+ const ExceptionBailoutInfo& excInfo);
+
+[[nodiscard]] bool FinishBailoutToBaseline(BaselineBailoutInfo* bailoutInfoArg);
+
+#ifdef DEBUG
+[[nodiscard]] bool AssertBailoutStackDepth(JSContext* cx, JSScript* script,
+ jsbytecode* pc, ResumeMode mode,
+ uint32_t exprStackSlots);
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Bailouts_h */
diff --git a/js/src/jit/BaselineBailouts.cpp b/js/src/jit/BaselineBailouts.cpp
new file mode 100644
index 0000000000..795476c4ac
--- /dev/null
+++ b/js/src/jit/BaselineBailouts.cpp
@@ -0,0 +1,2058 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Assertions.h"
+#include "mozilla/ScopeExit.h"
+
+#include "builtin/ModuleObject.h"
+#include "debugger/DebugAPI.h"
+#include "gc/GC.h"
+#include "jit/arm/Simulator-arm.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineIC.h"
+#include "jit/BaselineJIT.h"
+#include "jit/CalleeToken.h"
+#include "jit/Invalidation.h"
+#include "jit/Ion.h"
+#include "jit/IonScript.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/loong64/Simulator-loong64.h"
+#include "jit/mips32/Simulator-mips32.h"
+#include "jit/mips64/Simulator-mips64.h"
+#include "jit/RematerializedFrame.h"
+#include "jit/SharedICRegisters.h"
+#include "js/friend/StackLimits.h" // js::AutoCheckRecursionLimit, js::ReportOverRecursed
+#include "js/Utility.h"
+#include "util/Memory.h"
+#include "vm/ArgumentsObject.h"
+#include "vm/BytecodeUtil.h"
+#include "vm/JitActivation.h"
+
+#include "jit/JitFrames-inl.h"
+#include "vm/JSContext-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+using mozilla::Maybe;
+
+// BaselineStackBuilder may reallocate its buffer if the current one is too
+// small. To avoid dangling pointers, BufferPointer represents a pointer into
+// this buffer as a pointer to the header and a fixed offset.
+template <typename T>
+class BufferPointer {
+ const UniquePtr<BaselineBailoutInfo>& header_;
+ size_t offset_;
+ bool heap_;
+
+ public:
+ BufferPointer(const UniquePtr<BaselineBailoutInfo>& header, size_t offset,
+ bool heap)
+ : header_(header), offset_(offset), heap_(heap) {}
+
+ T* get() const {
+ BaselineBailoutInfo* header = header_.get();
+ if (!heap_) {
+ return (T*)(header->incomingStack + offset_);
+ }
+
+ uint8_t* p = header->copyStackTop - offset_;
+ MOZ_ASSERT(p >= header->copyStackBottom && p < header->copyStackTop);
+ return (T*)p;
+ }
+
+ void set(const T& value) { *get() = value; }
+
+ // Note: we return a copy instead of a reference, to avoid potential memory
+ // safety hazards when the underlying buffer gets resized.
+ const T operator*() const { return *get(); }
+ T* operator->() const { return get(); }
+};
+
+/**
+ * BaselineStackBuilder helps abstract the process of rebuilding the C stack on
+ * the heap. It takes a bailout iterator and keeps track of the point on the C
+ * stack from which the reconstructed frames will be written.
+ *
+ * It exposes methods to write data into the heap memory storing the
+ * reconstructed stack. It also exposes method to easily calculate addresses.
+ * This includes both the virtual address that a particular value will be at
+ * when it's eventually copied onto the stack, as well as the current actual
+ * address of that value (whether on the heap allocated portion being
+ * constructed or the existing stack).
+ *
+ * The abstraction handles transparent re-allocation of the heap memory when it
+ * needs to be enlarged to accommodate new data. Similarly to the C stack, the
+ * data that's written to the reconstructed stack grows from high to low in
+ * memory.
+ *
+ * The lowest region of the allocated memory contains a BaselineBailoutInfo
+ * structure that points to the start and end of the written data.
+ */
+class MOZ_STACK_CLASS BaselineStackBuilder {
+ JSContext* cx_;
+ JitFrameLayout* frame_ = nullptr;
+ SnapshotIterator& iter_;
+ RootedValueVector outermostFrameFormals_;
+
+ size_t bufferTotal_ = 1024;
+ size_t bufferAvail_ = 0;
+ size_t bufferUsed_ = 0;
+ size_t framePushed_ = 0;
+
+ UniquePtr<BaselineBailoutInfo> header_;
+
+ JSScript* script_;
+ JSFunction* fun_;
+ const ExceptionBailoutInfo* excInfo_;
+ ICScript* icScript_;
+
+ jsbytecode* pc_ = nullptr;
+ JSOp op_ = JSOp::Nop;
+ mozilla::Maybe<ResumeMode> resumeMode_;
+ uint32_t exprStackSlots_ = 0;
+ void* prevFramePtr_ = nullptr;
+ Maybe<BufferPointer<BaselineFrame>> blFrame_;
+
+ size_t frameNo_ = 0;
+ JSFunction* nextCallee_ = nullptr;
+
+ BailoutKind bailoutKind_;
+
+ // The baseline frames we will reconstruct on the heap are not
+ // rooted, so GC must be suppressed.
+ gc::AutoSuppressGC suppress_;
+
+ public:
+ BaselineStackBuilder(JSContext* cx, const JSJitFrameIter& frameIter,
+ SnapshotIterator& iter,
+ const ExceptionBailoutInfo* excInfo,
+ BailoutReason reason);
+
+ [[nodiscard]] bool init() {
+ MOZ_ASSERT(!header_);
+ MOZ_ASSERT(bufferUsed_ == 0);
+
+ uint8_t* bufferRaw = cx_->pod_calloc<uint8_t>(bufferTotal_);
+ if (!bufferRaw) {
+ return false;
+ }
+ bufferAvail_ = bufferTotal_ - sizeof(BaselineBailoutInfo);
+
+ header_.reset(new (bufferRaw) BaselineBailoutInfo());
+ header_->incomingStack = reinterpret_cast<uint8_t*>(frame_);
+ header_->copyStackTop = bufferRaw + bufferTotal_;
+ header_->copyStackBottom = header_->copyStackTop;
+ return true;
+ }
+
+ [[nodiscard]] bool buildOneFrame();
+ bool done();
+ void nextFrame();
+
+ JSScript* script() const { return script_; }
+ size_t frameNo() const { return frameNo_; }
+ bool isOutermostFrame() const { return frameNo_ == 0; }
+ MutableHandleValueVector outermostFrameFormals() {
+ return &outermostFrameFormals_;
+ }
+ BailoutKind bailoutKind() const { return bailoutKind_; }
+
+ inline JitFrameLayout* startFrame() { return frame_; }
+
+ BaselineBailoutInfo* info() {
+ MOZ_ASSERT(header_);
+ return header_.get();
+ }
+
+ BaselineBailoutInfo* takeBuffer() {
+ MOZ_ASSERT(header_);
+ return header_.release();
+ }
+
+ private:
+ [[nodiscard]] bool initFrame();
+ [[nodiscard]] bool buildBaselineFrame();
+ [[nodiscard]] bool buildArguments();
+ [[nodiscard]] bool buildFixedSlots();
+ [[nodiscard]] bool fixUpCallerArgs(MutableHandleValueVector savedCallerArgs,
+ bool* fixedUp);
+ [[nodiscard]] bool buildFinallyException();
+ [[nodiscard]] bool buildExpressionStack();
+ [[nodiscard]] bool finishLastFrame();
+
+ [[nodiscard]] bool prepareForNextFrame(HandleValueVector savedCallerArgs);
+ [[nodiscard]] bool finishOuterFrame();
+ [[nodiscard]] bool buildStubFrame(uint32_t frameSize,
+ HandleValueVector savedCallerArgs);
+ [[nodiscard]] bool buildRectifierFrame(uint32_t actualArgc,
+ size_t endOfBaselineStubArgs);
+
+#ifdef DEBUG
+ [[nodiscard]] bool validateFrame();
+#endif
+
+#ifdef DEBUG
+ bool envChainSlotCanBeOptimized();
+#endif
+
+ bool isPrologueBailout();
+ jsbytecode* getResumePC();
+ void* getStubReturnAddress();
+
+ uint32_t exprStackSlots() const { return exprStackSlots_; }
+
+ // Returns true if we're bailing out to a catch or finally block in this frame
+ bool catchingException() const {
+ return excInfo_ && excInfo_->catchingException() &&
+ excInfo_->frameNo() == frameNo_;
+ }
+
+ // Returns true if we're bailing out to a finally block in this frame.
+ bool resumingInFinallyBlock() const {
+ return catchingException() && excInfo_->isFinally();
+ }
+
+ bool forcedReturn() const { return excInfo_ && excInfo_->forcedReturn(); }
+
+ // Returns true if we're bailing out in place for debug mode
+ bool propagatingIonExceptionForDebugMode() const {
+ return excInfo_ && excInfo_->propagatingIonExceptionForDebugMode();
+ }
+
+ void* prevFramePtr() const {
+ MOZ_ASSERT(prevFramePtr_);
+ return prevFramePtr_;
+ }
+ BufferPointer<BaselineFrame>& blFrame() { return blFrame_.ref(); }
+
+ void setNextCallee(JSFunction* nextCallee);
+ JSFunction* nextCallee() const { return nextCallee_; }
+
+ jsbytecode* pc() const { return pc_; }
+ bool resumeAfter() const {
+ return !catchingException() && iter_.resumeAfter();
+ }
+
+ ResumeMode resumeMode() const { return *resumeMode_; }
+
+ bool needToSaveCallerArgs() const {
+ return resumeMode() == ResumeMode::InlinedAccessor;
+ }
+
+ [[nodiscard]] bool enlarge() {
+ MOZ_ASSERT(header_ != nullptr);
+ if (bufferTotal_ & mozilla::tl::MulOverflowMask<2>::value) {
+ ReportOutOfMemory(cx_);
+ return false;
+ }
+
+ size_t newSize = bufferTotal_ * 2;
+ uint8_t* newBufferRaw = cx_->pod_calloc<uint8_t>(newSize);
+ if (!newBufferRaw) {
+ return false;
+ }
+
+ // Initialize the new buffer.
+ //
+ // Before:
+ //
+ // [ Header | .. | Payload ]
+ //
+ // After:
+ //
+ // [ Header | ............... | Payload ]
+ //
+ // Size of Payload is |bufferUsed_|.
+ //
+ // We need to copy from the old buffer and header to the new buffer before
+ // we set header_ (this deletes the old buffer).
+ //
+ // We also need to update |copyStackBottom| and |copyStackTop| because these
+ // fields point to the Payload's start and end, respectively.
+ using BailoutInfoPtr = UniquePtr<BaselineBailoutInfo>;
+ BailoutInfoPtr newHeader(new (newBufferRaw) BaselineBailoutInfo(*header_));
+ newHeader->copyStackTop = newBufferRaw + newSize;
+ newHeader->copyStackBottom = newHeader->copyStackTop - bufferUsed_;
+ memcpy(newHeader->copyStackBottom, header_->copyStackBottom, bufferUsed_);
+ bufferTotal_ = newSize;
+ bufferAvail_ = newSize - (sizeof(BaselineBailoutInfo) + bufferUsed_);
+ header_ = std::move(newHeader);
+ return true;
+ }
+
+ void resetFramePushed() { framePushed_ = 0; }
+
+ size_t framePushed() const { return framePushed_; }
+
+ [[nodiscard]] bool subtract(size_t size, const char* info = nullptr) {
+ // enlarge the buffer if need be.
+ while (size > bufferAvail_) {
+ if (!enlarge()) {
+ return false;
+ }
+ }
+
+ // write out element.
+ header_->copyStackBottom -= size;
+ bufferAvail_ -= size;
+ bufferUsed_ += size;
+ framePushed_ += size;
+ if (info) {
+ JitSpew(JitSpew_BaselineBailouts, " SUB_%03d %p/%p %-15s",
+ (int)size, header_->copyStackBottom,
+ virtualPointerAtStackOffset(0), info);
+ }
+ return true;
+ }
+
+ template <typename T>
+ [[nodiscard]] bool write(const T& t) {
+ MOZ_ASSERT(!(uintptr_t(&t) >= uintptr_t(header_->copyStackBottom) &&
+ uintptr_t(&t) < uintptr_t(header_->copyStackTop)),
+ "Should not reference memory that can be freed");
+ if (!subtract(sizeof(T))) {
+ return false;
+ }
+ memcpy(header_->copyStackBottom, &t, sizeof(T));
+ return true;
+ }
+
+ template <typename T>
+ [[nodiscard]] bool writePtr(T* t, const char* info) {
+ if (!write<T*>(t)) {
+ return false;
+ }
+ if (info) {
+ JitSpew(JitSpew_BaselineBailouts, " WRITE_PTR %p/%p %-15s %p",
+ header_->copyStackBottom, virtualPointerAtStackOffset(0), info,
+ t);
+ }
+ return true;
+ }
+
+ [[nodiscard]] bool writeWord(size_t w, const char* info) {
+ if (!write<size_t>(w)) {
+ return false;
+ }
+ if (info) {
+ if (sizeof(size_t) == 4) {
+ JitSpew(JitSpew_BaselineBailouts, " WRITE_WRD %p/%p %-15s %08zx",
+ header_->copyStackBottom, virtualPointerAtStackOffset(0), info,
+ w);
+ } else {
+ JitSpew(JitSpew_BaselineBailouts, " WRITE_WRD %p/%p %-15s %016zx",
+ header_->copyStackBottom, virtualPointerAtStackOffset(0), info,
+ w);
+ }
+ }
+ return true;
+ }
+
+ [[nodiscard]] bool writeValue(const Value& val, const char* info) {
+ if (!write<Value>(val)) {
+ return false;
+ }
+ if (info) {
+ JitSpew(JitSpew_BaselineBailouts,
+ " WRITE_VAL %p/%p %-15s %016" PRIx64,
+ header_->copyStackBottom, virtualPointerAtStackOffset(0), info,
+ *((uint64_t*)&val));
+ }
+ return true;
+ }
+
+ [[nodiscard]] bool maybeWritePadding(size_t alignment, size_t after,
+ const char* info) {
+ MOZ_ASSERT(framePushed_ % sizeof(Value) == 0);
+ MOZ_ASSERT(after % sizeof(Value) == 0);
+ size_t offset = ComputeByteAlignment(after, alignment);
+ while (framePushed_ % alignment != offset) {
+ if (!writeValue(MagicValue(JS_ARG_POISON), info)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ void setResumeFramePtr(void* resumeFramePtr) {
+ header_->resumeFramePtr = resumeFramePtr;
+ }
+
+ void setResumeAddr(void* resumeAddr) { header_->resumeAddr = resumeAddr; }
+
+ template <typename T>
+ BufferPointer<T> pointerAtStackOffset(size_t offset) {
+ if (offset < bufferUsed_) {
+ // Calculate offset from copyStackTop.
+ offset = header_->copyStackTop - (header_->copyStackBottom + offset);
+ return BufferPointer<T>(header_, offset, /* heap = */ true);
+ }
+
+ return BufferPointer<T>(header_, offset - bufferUsed_, /* heap = */ false);
+ }
+
+ BufferPointer<Value> valuePointerAtStackOffset(size_t offset) {
+ return pointerAtStackOffset<Value>(offset);
+ }
+
+ inline uint8_t* virtualPointerAtStackOffset(size_t offset) {
+ if (offset < bufferUsed_) {
+ return reinterpret_cast<uint8_t*>(frame_) - (bufferUsed_ - offset);
+ }
+ return reinterpret_cast<uint8_t*>(frame_) + (offset - bufferUsed_);
+ }
+};
+
+BaselineStackBuilder::BaselineStackBuilder(JSContext* cx,
+ const JSJitFrameIter& frameIter,
+ SnapshotIterator& iter,
+ const ExceptionBailoutInfo* excInfo,
+ BailoutReason reason)
+ : cx_(cx),
+ frame_(static_cast<JitFrameLayout*>(frameIter.current())),
+ iter_(iter),
+ outermostFrameFormals_(cx),
+ script_(frameIter.script()),
+ fun_(frameIter.maybeCallee()),
+ excInfo_(excInfo),
+ icScript_(script_->jitScript()->icScript()),
+ bailoutKind_(iter.bailoutKind()),
+ suppress_(cx) {
+ MOZ_ASSERT(bufferTotal_ >= sizeof(BaselineBailoutInfo));
+ if (reason == BailoutReason::Invalidate) {
+ bailoutKind_ = BailoutKind::OnStackInvalidation;
+ }
+}
+
+bool BaselineStackBuilder::initFrame() {
+ // Get the pc and ResumeMode. If we are handling an exception, resume at the
+ // pc of the catch or finally block.
+ if (catchingException()) {
+ pc_ = excInfo_->resumePC();
+ resumeMode_ = mozilla::Some(ResumeMode::ResumeAt);
+ } else {
+ pc_ = script_->offsetToPC(iter_.pcOffset());
+ resumeMode_ = mozilla::Some(iter_.resumeMode());
+ }
+ op_ = JSOp(*pc_);
+
+ // If we are catching an exception, we are bailing out to a catch or
+ // finally block and this is the frame where we will resume. Usually the
+ // expression stack should be empty in this case but there can be
+ // iterators on the stack.
+ if (catchingException()) {
+ exprStackSlots_ = excInfo_->numExprSlots();
+ } else {
+ uint32_t totalFrameSlots = iter_.numAllocations();
+ uint32_t fixedSlots = script_->nfixed();
+ uint32_t argSlots = CountArgSlots(script_, fun_);
+ uint32_t intermediates = NumIntermediateValues(resumeMode());
+ exprStackSlots_ = totalFrameSlots - fixedSlots - argSlots - intermediates;
+
+ // Verify that there was no underflow.
+ MOZ_ASSERT(exprStackSlots_ <= totalFrameSlots);
+ }
+
+ JitSpew(JitSpew_BaselineBailouts, " Unpacking %s:%u:%u",
+ script_->filename(), script_->lineno(), script_->column());
+ JitSpew(JitSpew_BaselineBailouts, " [BASELINE-JS FRAME]");
+
+ // Write the previous frame pointer value. For the outermost frame we reuse
+ // the value in the JitFrameLayout already on the stack. Record the virtual
+ // stack offset at this location. Later on, if we end up writing out a
+ // BaselineStub frame for the next callee, we'll need to save the address.
+ if (!isOutermostFrame()) {
+ if (!writePtr(prevFramePtr(), "PrevFramePtr")) {
+ return false;
+ }
+ }
+ prevFramePtr_ = virtualPointerAtStackOffset(0);
+
+ resetFramePushed();
+
+ return true;
+}
+
+void BaselineStackBuilder::setNextCallee(JSFunction* nextCallee) {
+ nextCallee_ = nextCallee;
+
+ // Update icScript_ to point to the icScript of nextCallee
+ const uint32_t pcOff = script_->pcToOffset(pc_);
+ icScript_ = icScript_->findInlinedChild(pcOff);
+}
+
+bool BaselineStackBuilder::done() {
+ if (!iter_.moreFrames()) {
+ MOZ_ASSERT(!nextCallee_);
+ return true;
+ }
+ return catchingException();
+}
+
+void BaselineStackBuilder::nextFrame() {
+ MOZ_ASSERT(nextCallee_);
+ fun_ = nextCallee_;
+ script_ = fun_->nonLazyScript();
+ nextCallee_ = nullptr;
+
+ // Scripts with an IonScript must also have a BaselineScript.
+ MOZ_ASSERT(script_->hasBaselineScript());
+
+ frameNo_++;
+ iter_.nextInstruction();
+}
+
+// Build the BaselineFrame struct
+bool BaselineStackBuilder::buildBaselineFrame() {
+ if (!subtract(BaselineFrame::Size(), "BaselineFrame")) {
+ return false;
+ }
+ blFrame_.reset();
+ blFrame_.emplace(pointerAtStackOffset<BaselineFrame>(0));
+
+ uint32_t flags = BaselineFrame::RUNNING_IN_INTERPRETER;
+
+ // If we are bailing to a script whose execution is observed, mark the
+ // baseline frame as a debuggee frame. This is to cover the case where we
+ // don't rematerialize the Ion frame via the Debugger.
+ if (script_->isDebuggee()) {
+ flags |= BaselineFrame::DEBUGGEE;
+ }
+
+ // Get |envChain|.
+ JSObject* envChain = nullptr;
+ Value envChainSlot = iter_.read();
+ if (envChainSlot.isObject()) {
+ // The env slot has been updated from UndefinedValue. It must be the
+ // complete initial environment.
+ envChain = &envChainSlot.toObject();
+
+ // Set the HAS_INITIAL_ENV flag if needed. See IsFrameInitialEnvironment.
+ MOZ_ASSERT(!script_->isForEval());
+ if (fun_ && fun_->needsFunctionEnvironmentObjects()) {
+ MOZ_ASSERT(fun_->nonLazyScript()->initialEnvironmentShape());
+ flags |= BaselineFrame::HAS_INITIAL_ENV;
+ }
+ } else {
+ MOZ_ASSERT(envChainSlot.isUndefined() ||
+ envChainSlot.isMagic(JS_OPTIMIZED_OUT));
+ MOZ_ASSERT(envChainSlotCanBeOptimized());
+
+ // The env slot has been optimized out.
+ // Get it from the function or script.
+ if (fun_) {
+ envChain = fun_->environment();
+ } else if (script_->isModule()) {
+ envChain = script_->module()->environment();
+ } else {
+ // For global scripts without a non-syntactic env the env
+ // chain is the script's global lexical environment. (We do
+ // not compile scripts with a non-syntactic global scope).
+ // Also note that it's invalid to resume into the prologue in
+ // this case because the prologue expects the env chain in R1
+ // for eval and global scripts.
+ MOZ_ASSERT(!script_->isForEval());
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+ envChain = &(script_->global().lexicalEnvironment());
+ }
+ }
+
+ // Write |envChain|.
+ MOZ_ASSERT(envChain);
+ JitSpew(JitSpew_BaselineBailouts, " EnvChain=%p", envChain);
+ blFrame()->setEnvironmentChain(envChain);
+
+ // Get |returnValue| if present.
+ Value returnValue = UndefinedValue();
+ if (script_->noScriptRval()) {
+ // Don't use the return value (likely a JS_OPTIMIZED_OUT MagicValue) to
+ // not confuse Baseline.
+ iter_.skip();
+ } else {
+ returnValue = iter_.read();
+ flags |= BaselineFrame::HAS_RVAL;
+ }
+
+ // Write |returnValue|.
+ JitSpew(JitSpew_BaselineBailouts, " ReturnValue=%016" PRIx64,
+ *((uint64_t*)&returnValue));
+ blFrame()->setReturnValue(returnValue);
+
+ // Get |argsObj| if present.
+ ArgumentsObject* argsObj = nullptr;
+ if (script_->needsArgsObj()) {
+ Value maybeArgsObj = iter_.read();
+ MOZ_ASSERT(maybeArgsObj.isObject() || maybeArgsObj.isUndefined() ||
+ maybeArgsObj.isMagic(JS_OPTIMIZED_OUT));
+ if (maybeArgsObj.isObject()) {
+ argsObj = &maybeArgsObj.toObject().as<ArgumentsObject>();
+ }
+ }
+
+ // Note: we do not need to initialize the scratchValue field in BaselineFrame.
+
+ // Write |flags|.
+ blFrame()->setFlags(flags);
+
+ // Write |icScript|.
+ JitSpew(JitSpew_BaselineBailouts, " ICScript=%p", icScript_);
+ blFrame()->setICScript(icScript_);
+
+ // initArgsObjUnchecked modifies the frame's flags, so call it after setFlags.
+ if (argsObj) {
+ blFrame()->initArgsObjUnchecked(*argsObj);
+ }
+ return true;
+}
+
+// Overwrite the pushed args present in the calling frame with
+// the unpacked |thisv| and argument values.
+bool BaselineStackBuilder::buildArguments() {
+ Value thisv = iter_.read();
+ JitSpew(JitSpew_BaselineBailouts, " Is function!");
+ JitSpew(JitSpew_BaselineBailouts, " thisv=%016" PRIx64,
+ *((uint64_t*)&thisv));
+
+ size_t thisvOffset = framePushed() + JitFrameLayout::offsetOfThis();
+ valuePointerAtStackOffset(thisvOffset).set(thisv);
+
+ MOZ_ASSERT(iter_.numAllocations() >= CountArgSlots(script_, fun_));
+ JitSpew(JitSpew_BaselineBailouts,
+ " frame slots %u, nargs %zu, nfixed %zu", iter_.numAllocations(),
+ fun_->nargs(), script_->nfixed());
+
+ bool shouldStoreOutermostFormals =
+ isOutermostFrame() && !script_->argsObjAliasesFormals();
+ if (shouldStoreOutermostFormals) {
+ // This is the first (outermost) frame and we don't have an
+ // arguments object aliasing the formals. Due to UCE and phi
+ // elimination, we could store an UndefinedValue() here for
+ // formals we think are unused, but locals may still reference the
+ // original argument slot (MParameter/LArgument) and expect the
+ // original Value. To avoid this problem, store the formals in a
+ // Vector until we are done.
+ MOZ_ASSERT(outermostFrameFormals().empty());
+ if (!outermostFrameFormals().resize(fun_->nargs())) {
+ return false;
+ }
+ }
+
+ for (uint32_t i = 0; i < fun_->nargs(); i++) {
+ Value arg = iter_.read();
+ JitSpew(JitSpew_BaselineBailouts, " arg %d = %016" PRIx64, (int)i,
+ *((uint64_t*)&arg));
+ if (!isOutermostFrame()) {
+ size_t argOffset = framePushed() + JitFrameLayout::offsetOfActualArg(i);
+ valuePointerAtStackOffset(argOffset).set(arg);
+ } else if (shouldStoreOutermostFormals) {
+ outermostFrameFormals()[i].set(arg);
+ } else {
+ // When the arguments object aliases the formal arguments, then
+ // JSOp::SetArg mutates the argument object. In such cases, the
+ // list of arguments reported by the snapshot are only aliases
+ // of argument object slots which are optimized to only store
+ // differences compared to arguments which are on the stack.
+ }
+ }
+ return true;
+}
+
+bool BaselineStackBuilder::buildFixedSlots() {
+ for (uint32_t i = 0; i < script_->nfixed(); i++) {
+ Value slot = iter_.read();
+ if (!writeValue(slot, "FixedValue")) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// The caller side of inlined js::fun_call and accessors must look
+// like the function wasn't inlined.
+bool BaselineStackBuilder::fixUpCallerArgs(
+ MutableHandleValueVector savedCallerArgs, bool* fixedUp) {
+ MOZ_ASSERT(!*fixedUp);
+
+ // Inlining of SpreadCall-like frames not currently supported.
+ MOZ_ASSERT(!IsSpreadOp(op_));
+
+ if (resumeMode() != ResumeMode::InlinedFunCall && !needToSaveCallerArgs()) {
+ return true;
+ }
+
+ // Calculate how many arguments are consumed by the inlined call.
+ // All calls pass |callee| and |this|.
+ uint32_t inlinedArgs = 2;
+ if (resumeMode() == ResumeMode::InlinedFunCall) {
+ // The first argument to an inlined FunCall becomes |this|,
+ // if it exists. The rest are passed normally.
+ MOZ_ASSERT(IsInvokeOp(op_));
+ inlinedArgs += GET_ARGC(pc_) > 0 ? GET_ARGC(pc_) - 1 : 0;
+ } else {
+ MOZ_ASSERT(resumeMode() == ResumeMode::InlinedAccessor);
+ MOZ_ASSERT(IsIonInlinableGetterOrSetterOp(op_));
+ // Setters are passed one argument. Getters are passed none.
+ if (IsSetPropOp(op_)) {
+ inlinedArgs++;
+ }
+ }
+
+ // Calculate how many values are live on the stack across the call,
+ // and push them.
+ MOZ_ASSERT(inlinedArgs <= exprStackSlots());
+ uint32_t liveStackSlots = exprStackSlots() - inlinedArgs;
+
+ JitSpew(JitSpew_BaselineBailouts,
+ " pushing %u expression stack slots before fixup",
+ liveStackSlots);
+ for (uint32_t i = 0; i < liveStackSlots; i++) {
+ Value v = iter_.read();
+ if (!writeValue(v, "StackValue")) {
+ return false;
+ }
+ }
+
+ // When we inline js::fun_call, we bypass the native and inline the
+ // target directly. When rebuilding the stack, we need to fill in
+ // the right number of slots to make it look like the js_native was
+ // actually called.
+ if (resumeMode() == ResumeMode::InlinedFunCall) {
+ // We must transform the stack from |target, this, args| to
+ // |js_fun_call, target, this, args|. The value of |js_fun_call|
+ // will never be observed, so we push |undefined| for it, followed
+ // by the remaining arguments.
+ JitSpew(JitSpew_BaselineBailouts,
+ " pushing undefined to fixup funcall");
+ if (!writeValue(UndefinedValue(), "StackValue")) {
+ return false;
+ }
+ if (GET_ARGC(pc_) > 0) {
+ JitSpew(JitSpew_BaselineBailouts,
+ " pushing %u expression stack slots", inlinedArgs);
+ for (uint32_t i = 0; i < inlinedArgs; i++) {
+ Value arg = iter_.read();
+ if (!writeValue(arg, "StackValue")) {
+ return false;
+ }
+ }
+ } else {
+ // When we inline FunCall with no arguments, we push an extra
+ // |undefined| value for |this|. That value should not appear
+ // in the rebuilt baseline frame.
+ JitSpew(JitSpew_BaselineBailouts, " pushing target of funcall");
+ Value target = iter_.read();
+ if (!writeValue(target, "StackValue")) {
+ return false;
+ }
+ // Skip |this|.
+ iter_.skip();
+ }
+ }
+
+ if (needToSaveCallerArgs()) {
+ // Save the actual arguments. They are needed to rebuild the callee frame.
+ if (!savedCallerArgs.resize(inlinedArgs)) {
+ return false;
+ }
+ for (uint32_t i = 0; i < inlinedArgs; i++) {
+ savedCallerArgs[i].set(iter_.read());
+ }
+
+ if (IsSetPropOp(op_)) {
+ // The RHS argument to SetProp remains on the stack after the
+ // operation and is observable, so we have to fill it in.
+ Value initialArg = savedCallerArgs[inlinedArgs - 1];
+ JitSpew(JitSpew_BaselineBailouts,
+ " pushing setter's initial argument");
+ if (!writeValue(initialArg, "StackValue")) {
+ return false;
+ }
+ }
+ }
+
+ *fixedUp = true;
+ return true;
+}
+
+bool BaselineStackBuilder::buildExpressionStack() {
+ JitSpew(JitSpew_BaselineBailouts, " pushing %u expression stack slots",
+ exprStackSlots());
+ for (uint32_t i = 0; i < exprStackSlots(); i++) {
+ Value v;
+ // If we are in the middle of propagating an exception from Ion by
+ // bailing to baseline due to debug mode, we might not have all
+ // the stack if we are at the newest frame.
+ //
+ // For instance, if calling |f()| pushed an Ion frame which threw,
+ // the snapshot expects the return value to be pushed, but it's
+ // possible nothing was pushed before we threw.
+ //
+ // We therefore use a fallible read here.
+ if (!iter_.tryRead(&v)) {
+ MOZ_ASSERT(propagatingIonExceptionForDebugMode() && !iter_.moreFrames());
+ v = MagicValue(JS_OPTIMIZED_OUT);
+ }
+ if (!writeValue(v, "StackValue")) {
+ return false;
+ }
+ }
+
+ if (resumeMode() == ResumeMode::ResumeAfterCheckIsObject) {
+ JitSpew(JitSpew_BaselineBailouts,
+ " Checking that intermediate value is an object");
+ Value returnVal;
+ if (iter_.tryRead(&returnVal) && !returnVal.isObject()) {
+ MOZ_ASSERT(!returnVal.isMagic());
+ JitSpew(JitSpew_BaselineBailouts,
+ " Not an object! Overwriting bailout kind");
+ bailoutKind_ = BailoutKind::ThrowCheckIsObject;
+ }
+ }
+
+ return true;
+}
+
+bool BaselineStackBuilder::buildFinallyException() {
+ MOZ_ASSERT(resumingInFinallyBlock());
+
+ if (!writeValue(excInfo_->finallyException(), "Exception")) {
+ return false;
+ }
+ if (!writeValue(BooleanValue(true), "throwing")) {
+ return false;
+ }
+
+ return true;
+}
+
+bool BaselineStackBuilder::prepareForNextFrame(
+ HandleValueVector savedCallerArgs) {
+ const uint32_t frameSize = framePushed();
+
+ // Write out descriptor and return address for the baseline frame.
+ // The icEntry in question MUST have an inlinable fallback stub.
+ if (!finishOuterFrame()) {
+ return false;
+ }
+
+ return buildStubFrame(frameSize, savedCallerArgs);
+}
+
+bool BaselineStackBuilder::finishOuterFrame() {
+ // . .
+ // | Descr(BLJS) |
+ // +---------------+
+ // | ReturnAddr |
+ // +===============+
+
+ const BaselineInterpreter& baselineInterp =
+ cx_->runtime()->jitRuntime()->baselineInterpreter();
+
+ blFrame()->setInterpreterFields(script_, pc_);
+
+ // Write out descriptor of BaselineJS frame.
+ size_t baselineFrameDescr = MakeFrameDescriptor(FrameType::BaselineJS);
+ if (!writeWord(baselineFrameDescr, "Descriptor")) {
+ return false;
+ }
+
+ uint8_t* retAddr = baselineInterp.retAddrForIC(op_);
+ return writePtr(retAddr, "ReturnAddr");
+}
+
+bool BaselineStackBuilder::buildStubFrame(uint32_t frameSize,
+ HandleValueVector savedCallerArgs) {
+ // Build baseline stub frame:
+ // +===============+
+ // | FramePtr |
+ // +---------------+
+ // | StubPtr |
+ // +---------------+
+ // | Padding? |
+ // +---------------+
+ // | ArgA |
+ // +---------------+
+ // | ... |
+ // +---------------+
+ // | Arg0 |
+ // +---------------+
+ // | ThisV |
+ // +---------------+
+ // | CalleeToken |
+ // +---------------+
+ // | Descr(BLStub) |
+ // +---------------+
+ // | ReturnAddr |
+ // +===============+
+
+ JitSpew(JitSpew_BaselineBailouts, " [BASELINE-STUB FRAME]");
+
+ // Write previous frame pointer (saved earlier).
+ if (!writePtr(prevFramePtr(), "PrevFramePtr")) {
+ return false;
+ }
+ prevFramePtr_ = virtualPointerAtStackOffset(0);
+
+ // Write stub pointer.
+ uint32_t pcOff = script_->pcToOffset(pc_);
+ JitScript* jitScript = script_->jitScript();
+ const ICEntry& icEntry = jitScript->icEntryFromPCOffset(pcOff);
+ ICFallbackStub* fallback = jitScript->fallbackStubForICEntry(&icEntry);
+ if (!writePtr(fallback, "StubPtr")) {
+ return false;
+ }
+
+ // Write out the arguments, copied from the baseline frame. The order
+ // of the arguments is reversed relative to the baseline frame's stack
+ // values.
+ MOZ_ASSERT(IsIonInlinableOp(op_));
+ bool pushedNewTarget = IsConstructPC(pc_);
+ unsigned actualArgc;
+ Value callee;
+ if (needToSaveCallerArgs()) {
+ // For accessors, the arguments are not on the stack anymore,
+ // but they are copied in a vector and are written here.
+ callee = savedCallerArgs[0];
+ actualArgc = IsSetPropOp(op_) ? 1 : 0;
+
+ // Align the stack based on the number of arguments.
+ size_t afterFrameSize =
+ (actualArgc + 1) * sizeof(Value) + JitFrameLayout::Size();
+ if (!maybeWritePadding(JitStackAlignment, afterFrameSize, "Padding")) {
+ return false;
+ }
+
+ // Push arguments.
+ MOZ_ASSERT(actualArgc + 2 <= exprStackSlots());
+ MOZ_ASSERT(savedCallerArgs.length() == actualArgc + 2);
+ for (unsigned i = 0; i < actualArgc + 1; i++) {
+ size_t arg = savedCallerArgs.length() - (i + 1);
+ if (!writeValue(savedCallerArgs[arg], "ArgVal")) {
+ return false;
+ }
+ }
+ } else if (resumeMode() == ResumeMode::InlinedFunCall && GET_ARGC(pc_) == 0) {
+ // When calling FunCall with 0 arguments, we push |undefined|
+ // for this. See BaselineCacheIRCompiler::pushFunCallArguments.
+ MOZ_ASSERT(!pushedNewTarget);
+ actualArgc = 0;
+ // Align the stack based on pushing |this| and 0 arguments.
+ size_t afterFrameSize = sizeof(Value) + JitFrameLayout::Size();
+ if (!maybeWritePadding(JitStackAlignment, afterFrameSize, "Padding")) {
+ return false;
+ }
+ // Push an undefined value for |this|.
+ if (!writeValue(UndefinedValue(), "ThisValue")) {
+ return false;
+ }
+ size_t calleeSlot = blFrame()->numValueSlots(frameSize) - 1;
+ callee = *blFrame()->valueSlot(calleeSlot);
+
+ } else {
+ MOZ_ASSERT(resumeMode() == ResumeMode::InlinedStandardCall ||
+ resumeMode() == ResumeMode::InlinedFunCall);
+ actualArgc = GET_ARGC(pc_);
+ if (resumeMode() == ResumeMode::InlinedFunCall) {
+ // See BaselineCacheIRCompiler::pushFunCallArguments.
+ MOZ_ASSERT(actualArgc > 0);
+ actualArgc--;
+ }
+
+ // In addition to the formal arguments, we must also push |this|.
+ // When calling a constructor, we must also push |newTarget|.
+ uint32_t numArguments = actualArgc + 1 + pushedNewTarget;
+
+ // Align the stack based on the number of arguments.
+ size_t afterFrameSize =
+ numArguments * sizeof(Value) + JitFrameLayout::Size();
+ if (!maybeWritePadding(JitStackAlignment, afterFrameSize, "Padding")) {
+ return false;
+ }
+
+ // Copy the arguments and |this| from the BaselineFrame, in reverse order.
+ size_t valueSlot = blFrame()->numValueSlots(frameSize) - 1;
+ size_t calleeSlot = valueSlot - numArguments;
+
+ for (size_t i = valueSlot; i > calleeSlot; i--) {
+ Value v = *blFrame()->valueSlot(i);
+ if (!writeValue(v, "ArgVal")) {
+ return false;
+ }
+ }
+
+ callee = *blFrame()->valueSlot(calleeSlot);
+ }
+
+ // In case these arguments need to be copied on the stack again for a
+ // rectifier frame, save the framePushed values here for later use.
+ size_t endOfBaselineStubArgs = framePushed();
+
+ // Push callee token (must be a JS Function)
+ JitSpew(JitSpew_BaselineBailouts, " Callee = %016" PRIx64,
+ callee.asRawBits());
+
+ JSFunction* calleeFun = &callee.toObject().as<JSFunction>();
+ if (!writePtr(CalleeToToken(calleeFun, pushedNewTarget), "CalleeToken")) {
+ return false;
+ }
+ setNextCallee(calleeFun);
+
+ // Push BaselineStub frame descriptor
+ size_t baselineStubFrameDescr =
+ MakeFrameDescriptorForJitCall(FrameType::BaselineStub, actualArgc);
+ if (!writeWord(baselineStubFrameDescr, "Descriptor")) {
+ return false;
+ }
+
+ // Push return address into ICCall_Scripted stub, immediately after the call.
+ void* baselineCallReturnAddr = getStubReturnAddress();
+ MOZ_ASSERT(baselineCallReturnAddr);
+ if (!writePtr(baselineCallReturnAddr, "ReturnAddr")) {
+ return false;
+ }
+
+ // The stack must be aligned after the callee pushes the frame pointer.
+ MOZ_ASSERT((framePushed() + sizeof(void*)) % JitStackAlignment == 0);
+
+ // Build a rectifier frame if necessary
+ if (actualArgc < calleeFun->nargs() &&
+ !buildRectifierFrame(actualArgc, endOfBaselineStubArgs)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool BaselineStackBuilder::buildRectifierFrame(uint32_t actualArgc,
+ size_t endOfBaselineStubArgs) {
+ // Push a reconstructed rectifier frame.
+ // +===============+
+ // | Padding? |
+ // +---------------+
+ // | UndefinedU |
+ // +---------------+
+ // | ... |
+ // +---------------+
+ // | Undefined0 |
+ // +---------------+
+ // | ArgA |
+ // +---------------+
+ // | ... |
+ // +---------------+
+ // | Arg0 |
+ // +---------------+
+ // | ThisV |
+ // +---------------+
+ // | CalleeToken |
+ // +---------------+
+ // | Descr(Rect) |
+ // +---------------+
+ // | ReturnAddr |
+ // +===============+
+
+ JitSpew(JitSpew_BaselineBailouts, " [RECTIFIER FRAME]");
+ bool pushedNewTarget = IsConstructPC(pc_);
+
+ if (!writePtr(prevFramePtr(), "PrevFramePtr")) {
+ return false;
+ }
+ prevFramePtr_ = virtualPointerAtStackOffset(0);
+
+ // Align the stack based on the number of arguments.
+ size_t afterFrameSize =
+ (nextCallee()->nargs() + 1 + pushedNewTarget) * sizeof(Value) +
+ RectifierFrameLayout::Size();
+ if (!maybeWritePadding(JitStackAlignment, afterFrameSize, "Padding")) {
+ return false;
+ }
+
+ // Copy new.target, if necessary.
+ if (pushedNewTarget) {
+ size_t newTargetOffset = (framePushed() - endOfBaselineStubArgs) +
+ (actualArgc + 1) * sizeof(Value);
+ Value newTargetValue = *valuePointerAtStackOffset(newTargetOffset);
+ if (!writeValue(newTargetValue, "CopiedNewTarget")) {
+ return false;
+ }
+ }
+
+ // Push undefined for missing arguments.
+ for (unsigned i = 0; i < (nextCallee()->nargs() - actualArgc); i++) {
+ if (!writeValue(UndefinedValue(), "FillerVal")) {
+ return false;
+ }
+ }
+
+ // Copy arguments + thisv from BaselineStub frame.
+ if (!subtract((actualArgc + 1) * sizeof(Value), "CopiedArgs")) {
+ return false;
+ }
+ BufferPointer<uint8_t> stubArgsEnd =
+ pointerAtStackOffset<uint8_t>(framePushed() - endOfBaselineStubArgs);
+ JitSpew(JitSpew_BaselineBailouts, " MemCpy from %p", stubArgsEnd.get());
+ memcpy(pointerAtStackOffset<uint8_t>(0).get(), stubArgsEnd.get(),
+ (actualArgc + 1) * sizeof(Value));
+
+ // Push calleeToken again.
+ if (!writePtr(CalleeToToken(nextCallee(), pushedNewTarget), "CalleeToken")) {
+ return false;
+ }
+
+ // Push rectifier frame descriptor
+ size_t rectifierFrameDescr =
+ MakeFrameDescriptorForJitCall(FrameType::Rectifier, actualArgc);
+ if (!writeWord(rectifierFrameDescr, "Descriptor")) {
+ return false;
+ }
+
+ // Push return address into the ArgumentsRectifier code, immediately after the
+ // ioncode call.
+ void* rectReturnAddr =
+ cx_->runtime()->jitRuntime()->getArgumentsRectifierReturnAddr().value;
+ MOZ_ASSERT(rectReturnAddr);
+ if (!writePtr(rectReturnAddr, "ReturnAddr")) {
+ return false;
+ }
+
+ // The stack must be aligned after the callee pushes the frame pointer.
+ MOZ_ASSERT((framePushed() + sizeof(void*)) % JitStackAlignment == 0);
+
+ return true;
+}
+
+bool BaselineStackBuilder::finishLastFrame() {
+ const BaselineInterpreter& baselineInterp =
+ cx_->runtime()->jitRuntime()->baselineInterpreter();
+
+ setResumeFramePtr(prevFramePtr());
+
+ // Compute the native address (within the Baseline Interpreter) that we will
+ // resume at and initialize the frame's interpreter fields.
+ uint8_t* resumeAddr;
+ if (isPrologueBailout()) {
+ JitSpew(JitSpew_BaselineBailouts, " Resuming into prologue.");
+ MOZ_ASSERT(pc_ == script_->code());
+ blFrame()->setInterpreterFieldsForPrologue(script_);
+ resumeAddr = baselineInterp.bailoutPrologueEntryAddr();
+ } else if (propagatingIonExceptionForDebugMode()) {
+ // When propagating an exception for debug mode, set the
+ // resume pc to the throwing pc, so that Debugger hooks report
+ // the correct pc offset of the throwing op instead of its
+ // successor.
+ jsbytecode* throwPC = script_->offsetToPC(iter_.pcOffset());
+ blFrame()->setInterpreterFields(script_, throwPC);
+ resumeAddr = baselineInterp.interpretOpAddr().value;
+ } else {
+ jsbytecode* resumePC = getResumePC();
+ blFrame()->setInterpreterFields(script_, resumePC);
+ resumeAddr = baselineInterp.interpretOpAddr().value;
+ }
+ setResumeAddr(resumeAddr);
+ JitSpew(JitSpew_BaselineBailouts, " Set resumeAddr=%p", resumeAddr);
+
+ if (cx_->runtime()->geckoProfiler().enabled()) {
+ // Register bailout with profiler.
+ const char* filename = script_->filename();
+ if (filename == nullptr) {
+ filename = "<unknown>";
+ }
+ unsigned len = strlen(filename) + 200;
+ UniqueChars buf(js_pod_malloc<char>(len));
+ if (buf == nullptr) {
+ ReportOutOfMemory(cx_);
+ return false;
+ }
+ snprintf(buf.get(), len, "%s %s %s on line %u of %s:%u",
+ BailoutKindString(bailoutKind()), resumeAfter() ? "after" : "at",
+ CodeName(op_), PCToLineNumber(script_, pc_), filename,
+ script_->lineno());
+ cx_->runtime()->geckoProfiler().markEvent("Bailout", buf.get());
+ }
+
+ return true;
+}
+
+#ifdef DEBUG
+// The |envChain| slot must not be optimized out if the currently
+// active scope requires any EnvironmentObjects beyond what is
+// available at body scope. This checks that scope chain does not
+// require any such EnvironmentObjects.
+// See also: |CompileInfo::isObservableFrameSlot|
+bool BaselineStackBuilder::envChainSlotCanBeOptimized() {
+ jsbytecode* pc = script_->offsetToPC(iter_.pcOffset());
+ Scope* scopeIter = script_->innermostScope(pc);
+ while (scopeIter != script_->bodyScope()) {
+ if (!scopeIter || scopeIter->hasEnvironment()) {
+ return false;
+ }
+ scopeIter = scopeIter->enclosing();
+ }
+ return true;
+}
+
+bool jit::AssertBailoutStackDepth(JSContext* cx, JSScript* script,
+ jsbytecode* pc, ResumeMode mode,
+ uint32_t exprStackSlots) {
+ if (IsResumeAfter(mode)) {
+ pc = GetNextPc(pc);
+ }
+
+ uint32_t expectedDepth;
+ bool reachablePC;
+ if (!ReconstructStackDepth(cx, script, pc, &expectedDepth, &reachablePC)) {
+ return false;
+ }
+ if (!reachablePC) {
+ return true;
+ }
+
+ JSOp op = JSOp(*pc);
+
+ if (mode == ResumeMode::InlinedFunCall) {
+ // For inlined fun.call(this, ...); the reconstructed stack depth will
+ // include the |this|, but the exprStackSlots won't.
+ // Exception: if there are no arguments, the depths do match.
+ MOZ_ASSERT(IsInvokeOp(op));
+ if (GET_ARGC(pc) > 0) {
+ MOZ_ASSERT(expectedDepth == exprStackSlots + 1);
+ } else {
+ MOZ_ASSERT(expectedDepth == exprStackSlots);
+ }
+ return true;
+ }
+
+ if (mode == ResumeMode::InlinedAccessor) {
+ // Accessors coming out of ion are inlined via a complete lie perpetrated by
+ // the compiler internally. Ion just rearranges the stack, and pretends that
+ // it looked like a call all along.
+ // This means that the depth is actually one *more* than expected by the
+ // interpreter, as there is now a JSFunction, |this| and [arg], rather than
+ // the expected |this| and [arg].
+ // If the inlined accessor is a GetElem operation, the numbers do match, but
+ // that's just because GetElem expects one more item on the stack. Note that
+ // none of that was pushed, but it's still reflected in exprStackSlots.
+ MOZ_ASSERT(IsIonInlinableGetterOrSetterOp(op));
+ if (IsGetElemOp(op)) {
+ MOZ_ASSERT(exprStackSlots == expectedDepth);
+ } else {
+ MOZ_ASSERT(exprStackSlots == expectedDepth + 1);
+ }
+ return true;
+ }
+
+ // In all other cases, the depth must match.
+ MOZ_ASSERT(exprStackSlots == expectedDepth);
+ return true;
+}
+
+bool BaselineStackBuilder::validateFrame() {
+ const uint32_t frameSize = framePushed();
+ blFrame()->setDebugFrameSize(frameSize);
+ JitSpew(JitSpew_BaselineBailouts, " FrameSize=%u", frameSize);
+
+ // debugNumValueSlots() is based on the frame size, do some sanity checks.
+ MOZ_ASSERT(blFrame()->debugNumValueSlots() >= script_->nfixed());
+ MOZ_ASSERT(blFrame()->debugNumValueSlots() <= script_->nslots());
+
+ uint32_t expectedSlots = exprStackSlots();
+ if (resumingInFinallyBlock()) {
+ // If we are resuming in a finally block, we push two extra values on the
+ // stack (the exception, and |throwing|), so the depth at the resume PC
+ // should be the depth at the fault PC plus two.
+ expectedSlots += 2;
+ }
+ return AssertBailoutStackDepth(cx_, script_, pc_, resumeMode(),
+ expectedSlots);
+}
+#endif
+
+void* BaselineStackBuilder::getStubReturnAddress() {
+ const BaselineICFallbackCode& code =
+ cx_->runtime()->jitRuntime()->baselineICFallbackCode();
+
+ if (IsGetPropOp(op_)) {
+ return code.bailoutReturnAddr(BailoutReturnKind::GetProp);
+ }
+ if (IsSetPropOp(op_)) {
+ return code.bailoutReturnAddr(BailoutReturnKind::SetProp);
+ }
+ if (IsGetElemOp(op_)) {
+ return code.bailoutReturnAddr(BailoutReturnKind::GetElem);
+ }
+
+ // This should be a call op of some kind, now.
+ MOZ_ASSERT(IsInvokeOp(op_) && !IsSpreadOp(op_));
+ if (IsConstructOp(op_)) {
+ return code.bailoutReturnAddr(BailoutReturnKind::New);
+ }
+ return code.bailoutReturnAddr(BailoutReturnKind::Call);
+}
+
+static inline jsbytecode* GetNextNonLoopHeadPc(jsbytecode* pc) {
+ JSOp op = JSOp(*pc);
+ switch (op) {
+ case JSOp::Goto:
+ return pc + GET_JUMP_OFFSET(pc);
+
+ case JSOp::LoopHead:
+ case JSOp::Nop:
+ return GetNextPc(pc);
+
+ default:
+ return pc;
+ }
+}
+
+// Returns the pc to resume execution at in Baseline after a bailout.
+jsbytecode* BaselineStackBuilder::getResumePC() {
+ if (resumeAfter()) {
+ return GetNextPc(pc_);
+ }
+
+ // If we are resuming at a LoopHead op, resume at the next op to avoid
+ // a bailout -> enter Ion -> bailout loop with --ion-eager.
+ //
+ // Cycles can cause the loop below to not terminate. Empty loops are one
+ // such example:
+ //
+ // L: loophead
+ // goto L
+ //
+ // We do cycle detection below with the "tortoise and the hare" algorithm.
+ jsbytecode* slowerPc = pc_;
+ jsbytecode* fasterPc = pc_;
+ while (true) {
+ // Advance fasterPc twice as fast as slowerPc.
+ slowerPc = GetNextNonLoopHeadPc(slowerPc);
+ fasterPc = GetNextNonLoopHeadPc(fasterPc);
+ fasterPc = GetNextNonLoopHeadPc(fasterPc);
+
+ // Break on cycles or at the end of goto sequences.
+ if (fasterPc == slowerPc) {
+ break;
+ }
+ }
+
+ return slowerPc;
+}
+
+bool BaselineStackBuilder::isPrologueBailout() {
+ // If we are propagating an exception for debug mode, we will not resume
+ // into baseline code, but instead into HandleExceptionBaseline (i.e.,
+ // never before the prologue).
+ return iter_.pcOffset() == 0 && !iter_.resumeAfter() &&
+ !propagatingIonExceptionForDebugMode();
+}
+
+// Build a baseline stack frame.
+bool BaselineStackBuilder::buildOneFrame() {
+ // Build a baseline frame:
+ // +===============+
+ // | PrevFramePtr | <-- initFrame()
+ // +---------------+
+ // | Baseline | <-- buildBaselineFrame()
+ // | Frame |
+ // +---------------+
+ // | Fixed0 | <-- buildFixedSlots()
+ // +---------------+
+ // | ... |
+ // +---------------+
+ // | FixedF |
+ // +---------------+
+ // | Stack0 | <-- buildExpressionStack() -or- fixupCallerArgs()
+ // +---------------+
+ // | ... |
+ // +---------------+ If we are building the frame in which we will
+ // | StackS | <-- resume, we stop here.
+ // +---------------+ finishLastFrame() sets up the interpreter fields.
+ // . .
+ // . .
+ // . . <-- If there are additional frames inlined into this
+ // | Descr(BLJS) | one, we finish this frame. We generate a stub
+ // +---------------+ frame (and maybe also a rectifier frame) between
+ // | ReturnAddr | this frame and the inlined frame.
+ // +===============+ See: prepareForNextFrame()
+
+ if (!initFrame()) {
+ return false;
+ }
+
+ if (!buildBaselineFrame()) {
+ return false;
+ }
+
+ if (fun_ && !buildArguments()) {
+ return false;
+ }
+
+ if (!buildFixedSlots()) {
+ return false;
+ }
+
+ bool fixedUp = false;
+ RootedValueVector savedCallerArgs(cx_);
+ if (iter_.moreFrames() && !fixUpCallerArgs(&savedCallerArgs, &fixedUp)) {
+ return false;
+ }
+
+ if (!fixedUp) {
+ if (!buildExpressionStack()) {
+ return false;
+ }
+ if (resumingInFinallyBlock() && !buildFinallyException()) {
+ return false;
+ }
+ }
+
+#ifdef DEBUG
+ if (!validateFrame()) {
+ return false;
+ }
+#endif
+
+#ifdef JS_JITSPEW
+ const uint32_t pcOff = script_->pcToOffset(pc());
+ JitSpew(JitSpew_BaselineBailouts,
+ " Resuming %s pc offset %d (op %s) (line %u) of %s:%u:%u",
+ resumeAfter() ? "after" : "at", (int)pcOff, CodeName(op_),
+ PCToLineNumber(script_, pc()), script_->filename(), script_->lineno(),
+ script_->column());
+ JitSpew(JitSpew_BaselineBailouts, " Bailout kind: %s",
+ BailoutKindString(bailoutKind()));
+#endif
+
+ // If this was the last inline frame, or we are bailing out to a catch or
+ // finally block in this frame, then unpacking is almost done.
+ if (done()) {
+ return finishLastFrame();
+ }
+
+ // Otherwise, this is an outer frame for an inlined call or
+ // accessor. We will be building an inner frame. Before that,
+ // we must create a stub frame, and potentially a rectifier frame.
+ return prepareForNextFrame(savedCallerArgs);
+}
+
+bool jit::BailoutIonToBaseline(JSContext* cx, JitActivation* activation,
+ const JSJitFrameIter& iter,
+ BaselineBailoutInfo** bailoutInfo,
+ const ExceptionBailoutInfo* excInfo,
+ BailoutReason reason) {
+ MOZ_ASSERT(bailoutInfo != nullptr);
+ MOZ_ASSERT(*bailoutInfo == nullptr);
+ MOZ_ASSERT(iter.isBailoutJS());
+
+ // Caller should have saved the exception while we perform the bailout.
+ MOZ_ASSERT(!cx->isExceptionPending());
+
+ // Ion bailout can fail due to overrecursion and OOM. In such cases we
+ // cannot honor any further Debugger hooks on the frame, and need to
+ // ensure that its Debugger.Frame entry is cleaned up.
+ auto guardRemoveRematerializedFramesFromDebugger =
+ mozilla::MakeScopeExit([&] {
+ activation->removeRematerializedFramesFromDebugger(cx, iter.fp());
+ });
+
+ // Always remove the RInstructionResults from the JitActivation, even in
+ // case of failures as the stack frame is going away after the bailout.
+ auto removeIonFrameRecovery = mozilla::MakeScopeExit(
+ [&] { activation->removeIonFrameRecovery(iter.jsFrame()); });
+
+ // The caller of the top frame must be one of the following:
+ // IonJS - Ion calling into Ion.
+ // BaselineStub - Baseline calling into Ion.
+ // Entry / WasmToJSJit - Interpreter or other (wasm) calling into Ion.
+ // Rectifier - Arguments rectifier calling into Ion.
+ // BaselineJS - Resume'd Baseline, then likely OSR'd into Ion.
+ MOZ_ASSERT(iter.isBailoutJS());
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ FrameType prevFrameType = iter.prevType();
+ MOZ_ASSERT(JSJitFrameIter::isEntry(prevFrameType) ||
+ prevFrameType == FrameType::IonJS ||
+ prevFrameType == FrameType::BaselineStub ||
+ prevFrameType == FrameType::Rectifier ||
+ prevFrameType == FrameType::IonICCall ||
+ prevFrameType == FrameType::BaselineJS);
+#endif
+
+ // All incoming frames are going to look like this:
+ //
+ // +---------------+
+ // | ... |
+ // +---------------+
+ // | Args |
+ // | ... |
+ // +---------------+
+ // | ThisV |
+ // +---------------+
+ // | ActualArgC |
+ // +---------------+
+ // | CalleeToken |
+ // +---------------+
+ // | Descriptor |
+ // +---------------+
+ // | ReturnAddr |
+ // +---------------+
+ // | ||||| | <---- Overwrite starting here.
+ // | ||||| |
+ // | ||||| |
+ // +---------------+
+
+ JitSpew(JitSpew_BaselineBailouts,
+ "Bailing to baseline %s:%u:%u (IonScript=%p) (FrameType=%d)",
+ iter.script()->filename(), iter.script()->lineno(),
+ iter.script()->column(), (void*)iter.ionScript(), (int)prevFrameType);
+
+ if (excInfo) {
+ if (excInfo->catchingException()) {
+ JitSpew(JitSpew_BaselineBailouts, "Resuming in catch or finally block");
+ }
+ if (excInfo->propagatingIonExceptionForDebugMode()) {
+ JitSpew(JitSpew_BaselineBailouts, "Resuming in-place for debug mode");
+ }
+ }
+
+ JitSpew(JitSpew_BaselineBailouts,
+ " Reading from snapshot offset %u size %zu", iter.snapshotOffset(),
+ iter.ionScript()->snapshotsListSize());
+
+ iter.script()->updateJitCodeRaw(cx->runtime());
+
+ // Under a bailout, there is no need to invalidate the frame after
+ // evaluating the recover instruction, as the invalidation is only needed in
+ // cases where the frame is introspected ahead of the bailout.
+ MaybeReadFallback recoverBailout(cx, activation, &iter,
+ MaybeReadFallback::Fallback_DoNothing);
+
+ // Ensure that all value locations are readable from the SnapshotIterator.
+ // Get the RInstructionResults from the JitActivation if the frame got
+ // recovered ahead of the bailout.
+ SnapshotIterator snapIter(iter, activation->bailoutData()->machineState());
+ if (!snapIter.initInstructionResults(recoverBailout)) {
+ return false;
+ }
+
+#ifdef TRACK_SNAPSHOTS
+ snapIter.spewBailingFrom();
+#endif
+
+ BaselineStackBuilder builder(cx, iter, snapIter, excInfo, reason);
+ if (!builder.init()) {
+ return false;
+ }
+
+ JitSpew(JitSpew_BaselineBailouts, " Incoming frame ptr = %p",
+ builder.startFrame());
+ if (iter.maybeCallee()) {
+ JitSpew(JitSpew_BaselineBailouts, " Callee function (%s:%u:%u)",
+ iter.script()->filename(), iter.script()->lineno(),
+ iter.script()->column());
+ } else {
+ JitSpew(JitSpew_BaselineBailouts, " No callee!");
+ }
+
+ if (iter.isConstructing()) {
+ JitSpew(JitSpew_BaselineBailouts, " Constructing!");
+ } else {
+ JitSpew(JitSpew_BaselineBailouts, " Not constructing!");
+ }
+
+ JitSpew(JitSpew_BaselineBailouts, " Restoring frames:");
+
+ while (true) {
+ // Skip recover instructions as they are already recovered by
+ // |initInstructionResults|.
+ snapIter.settleOnFrame();
+
+ JitSpew(JitSpew_BaselineBailouts, " FrameNo %zu", builder.frameNo());
+
+ if (!builder.buildOneFrame()) {
+ MOZ_ASSERT(cx->isExceptionPending());
+ return false;
+ }
+
+ if (builder.done()) {
+ break;
+ }
+
+ builder.nextFrame();
+ }
+ JitSpew(JitSpew_BaselineBailouts, " Done restoring frames");
+
+ BailoutKind bailoutKind = builder.bailoutKind();
+
+ if (!builder.outermostFrameFormals().empty()) {
+ // Set the first frame's formals, see the comment in InitFromBailout.
+ Value* argv = builder.startFrame()->actualArgs();
+ mozilla::PodCopy(argv, builder.outermostFrameFormals().begin(),
+ builder.outermostFrameFormals().length());
+ }
+
+ // Do stack check.
+ bool overRecursed = false;
+ BaselineBailoutInfo* info = builder.info();
+ size_t numBytesToPush = info->copyStackTop - info->copyStackBottom;
+ MOZ_ASSERT((numBytesToPush % sizeof(uintptr_t)) == 0);
+ uint8_t* newsp = info->incomingStack - numBytesToPush;
+#ifdef JS_SIMULATOR
+ if (Simulator::Current()->overRecursed(uintptr_t(newsp))) {
+ overRecursed = true;
+ }
+#else
+ AutoCheckRecursionLimit recursion(cx);
+ if (!recursion.checkWithStackPointerDontReport(cx, newsp)) {
+ overRecursed = true;
+ }
+#endif
+ if (overRecursed) {
+ JitSpew(JitSpew_BaselineBailouts, " Overrecursion check failed!");
+ ReportOverRecursed(cx);
+ return false;
+ }
+
+ // Take the reconstructed baseline stack so it doesn't get freed when builder
+ // destructs.
+ info = builder.takeBuffer();
+ info->numFrames = builder.frameNo() + 1;
+ info->bailoutKind.emplace(bailoutKind);
+ *bailoutInfo = info;
+ guardRemoveRematerializedFramesFromDebugger.release();
+ return true;
+}
+
+static void InvalidateAfterBailout(JSContext* cx, HandleScript outerScript,
+ const char* reason) {
+ // In some cases, the computation of recover instruction can invalidate the
+ // Ion script before we reach the end of the bailout. Thus, if the outer
+ // script no longer have any Ion script attached, then we just skip the
+ // invalidation.
+ //
+ // For example, such case can happen if the template object for an unboxed
+ // objects no longer match the content of its properties (see Bug 1174547)
+ if (!outerScript->hasIonScript()) {
+ JitSpew(JitSpew_BaselineBailouts, "Ion script is already invalidated");
+ return;
+ }
+
+ MOZ_ASSERT(!outerScript->ionScript()->invalidated());
+
+ JitSpew(JitSpew_BaselineBailouts, "Invalidating due to %s", reason);
+ Invalidate(cx, outerScript);
+}
+
+static void HandleLexicalCheckFailure(JSContext* cx, HandleScript outerScript,
+ HandleScript innerScript) {
+ JitSpew(JitSpew_IonBailouts,
+ "Lexical check failure %s:%u:%u, inlined into %s:%u:%u",
+ innerScript->filename(), innerScript->lineno(), innerScript->column(),
+ outerScript->filename(), outerScript->lineno(),
+ outerScript->column());
+
+ if (!innerScript->failedLexicalCheck()) {
+ innerScript->setFailedLexicalCheck();
+ }
+
+ InvalidateAfterBailout(cx, outerScript, "lexical check failure");
+ if (innerScript->hasIonScript()) {
+ Invalidate(cx, innerScript);
+ }
+}
+
+static bool CopyFromRematerializedFrame(JSContext* cx, JitActivation* act,
+ uint8_t* fp, size_t inlineDepth,
+ BaselineFrame* frame) {
+ RematerializedFrame* rematFrame =
+ act->lookupRematerializedFrame(fp, inlineDepth);
+
+ // We might not have rematerialized a frame if the user never requested a
+ // Debugger.Frame for it.
+ if (!rematFrame) {
+ return true;
+ }
+
+ MOZ_ASSERT(rematFrame->script() == frame->script());
+ MOZ_ASSERT(rematFrame->numActualArgs() == frame->numActualArgs());
+
+ frame->setEnvironmentChain(rematFrame->environmentChain());
+
+ if (frame->isFunctionFrame()) {
+ frame->thisArgument() = rematFrame->thisArgument();
+ }
+
+ for (unsigned i = 0; i < frame->numActualArgs(); i++) {
+ frame->argv()[i] = rematFrame->argv()[i];
+ }
+
+ for (size_t i = 0; i < frame->script()->nfixed(); i++) {
+ *frame->valueSlot(i) = rematFrame->locals()[i];
+ }
+
+ frame->setReturnValue(rematFrame->returnValue());
+
+ // Don't copy over the hasCachedSavedFrame bit. The new BaselineFrame we're
+ // building has a different AbstractFramePtr, so it won't be found in the
+ // LiveSavedFrameCache if we look there.
+
+ JitSpew(JitSpew_BaselineBailouts,
+ " Copied from rematerialized frame at (%p,%zu)", fp, inlineDepth);
+
+ // Propagate the debuggee frame flag. For the case where the Debugger did
+ // not rematerialize an Ion frame, the baseline frame has its debuggee
+ // flag set iff its script is considered a debuggee. See the debuggee case
+ // in InitFromBailout.
+ if (rematFrame->isDebuggee()) {
+ frame->setIsDebuggee();
+ return DebugAPI::handleIonBailout(cx, rematFrame, frame);
+ }
+
+ return true;
+}
+
+enum class BailoutAction {
+ InvalidateImmediately,
+ InvalidateIfFrequent,
+ DisableIfFrequent,
+ NoAction
+};
+
+bool jit::FinishBailoutToBaseline(BaselineBailoutInfo* bailoutInfoArg) {
+ JitSpew(JitSpew_BaselineBailouts, " Done restoring frames");
+
+ // Use UniquePtr to free the bailoutInfo before we return.
+ UniquePtr<BaselineBailoutInfo> bailoutInfo(bailoutInfoArg);
+ bailoutInfoArg = nullptr;
+
+ MOZ_DIAGNOSTIC_ASSERT(*bailoutInfo->bailoutKind != BailoutKind::Unreachable);
+
+ JSContext* cx = TlsContext.get();
+ BaselineFrame* topFrame = GetTopBaselineFrame(cx);
+
+ // We have to get rid of the rematerialized frame, whether it is
+ // restored or unwound.
+ uint8_t* incomingStack = bailoutInfo->incomingStack;
+ auto guardRemoveRematerializedFramesFromDebugger =
+ mozilla::MakeScopeExit([&] {
+ JitActivation* act = cx->activation()->asJit();
+ act->removeRematerializedFramesFromDebugger(cx, incomingStack);
+ });
+
+ // Ensure the frame has a call object if it needs one.
+ if (!EnsureHasEnvironmentObjects(cx, topFrame)) {
+ return false;
+ }
+
+ // Create arguments objects for bailed out frames, to maintain the invariant
+ // that script->needsArgsObj() implies frame->hasArgsObj().
+ RootedScript innerScript(cx, nullptr);
+ RootedScript outerScript(cx, nullptr);
+
+ MOZ_ASSERT(cx->currentlyRunningInJit());
+ JSJitFrameIter iter(cx->activation()->asJit());
+ uint8_t* outerFp = nullptr;
+
+ // Iter currently points at the exit frame. Get the previous frame
+ // (which must be a baseline frame), and set it as the last profiling
+ // frame.
+ if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(
+ cx->runtime())) {
+ MOZ_ASSERT(iter.prevType() == FrameType::BaselineJS);
+ JitFrameLayout* fp = reinterpret_cast<JitFrameLayout*>(iter.prevFp());
+ cx->jitActivation->setLastProfilingFrame(fp);
+ }
+
+ uint32_t numFrames = bailoutInfo->numFrames;
+ MOZ_ASSERT(numFrames > 0);
+
+ uint32_t frameno = 0;
+ while (frameno < numFrames) {
+ MOZ_ASSERT(!iter.isIonJS());
+
+ if (iter.isBaselineJS()) {
+ BaselineFrame* frame = iter.baselineFrame();
+ MOZ_ASSERT(frame->script()->hasBaselineScript());
+
+ // If the frame doesn't even have a env chain set yet, then it's resuming
+ // into the the prologue before the env chain is initialized. Any
+ // necessary args object will also be initialized there.
+ if (frame->environmentChain() && frame->script()->needsArgsObj()) {
+ ArgumentsObject* argsObj;
+ if (frame->hasArgsObj()) {
+ argsObj = &frame->argsObj();
+ } else {
+ argsObj = ArgumentsObject::createExpected(cx, frame);
+ if (!argsObj) {
+ return false;
+ }
+ }
+
+ // The arguments is a local binding and needsArgsObj does not
+ // check if it is clobbered. Ensure that the local binding
+ // restored during bailout before storing the arguments object
+ // to the slot.
+ RootedScript script(cx, frame->script());
+ SetFrameArgumentsObject(cx, frame, script, argsObj);
+ }
+
+ if (frameno == 0) {
+ innerScript = frame->script();
+ }
+
+ if (frameno == numFrames - 1) {
+ outerScript = frame->script();
+ outerFp = iter.fp();
+ MOZ_ASSERT(outerFp == incomingStack);
+ }
+
+ frameno++;
+ }
+
+ ++iter;
+ }
+
+ MOZ_ASSERT(innerScript);
+ MOZ_ASSERT(outerScript);
+ MOZ_ASSERT(outerFp);
+
+ // If we rematerialized Ion frames due to debug mode toggling, copy their
+ // values into the baseline frame. We need to do this even when debug mode
+ // is off, as we should respect the mutations made while debug mode was
+ // on.
+ JitActivation* act = cx->activation()->asJit();
+ if (act->hasRematerializedFrame(outerFp)) {
+ JSJitFrameIter iter(act);
+ size_t inlineDepth = numFrames;
+ bool ok = true;
+ while (inlineDepth > 0) {
+ if (iter.isBaselineJS()) {
+ // We must attempt to copy all rematerialized frames over,
+ // even if earlier ones failed, to invoke the proper frame
+ // cleanup in the Debugger.
+ if (!CopyFromRematerializedFrame(cx, act, outerFp, --inlineDepth,
+ iter.baselineFrame())) {
+ ok = false;
+ }
+ }
+ ++iter;
+ }
+
+ if (!ok) {
+ return false;
+ }
+
+ // After copying from all the rematerialized frames, remove them from
+ // the table to keep the table up to date.
+ guardRemoveRematerializedFramesFromDebugger.release();
+ act->removeRematerializedFrame(outerFp);
+ }
+
+ // If we are unwinding for an exception, we need to unwind scopes.
+ // See |SettleOnTryNote|
+ if (bailoutInfo->faultPC) {
+ EnvironmentIter ei(cx, topFrame, bailoutInfo->faultPC);
+ UnwindEnvironment(cx, ei, bailoutInfo->tryPC);
+ }
+
+ // Check for interrupts now because we might miss an interrupt check in JIT
+ // code when resuming in the prologue, after the stack/interrupt check.
+ if (!cx->isExceptionPending()) {
+ if (!CheckForInterrupt(cx)) {
+ return false;
+ }
+ }
+
+ BailoutKind bailoutKind = *bailoutInfo->bailoutKind;
+ JitSpew(JitSpew_BaselineBailouts,
+ " Restored outerScript=(%s:%u:%u,%u) innerScript=(%s:%u:%u,%u) "
+ "(bailoutKind=%u)",
+ outerScript->filename(), outerScript->lineno(), outerScript->column(),
+ outerScript->getWarmUpCount(), innerScript->filename(),
+ innerScript->lineno(), innerScript->column(),
+ innerScript->getWarmUpCount(), (unsigned)bailoutKind);
+
+ BailoutAction action = BailoutAction::InvalidateImmediately;
+ DebugOnly<bool> saveFailedICHash = false;
+ switch (bailoutKind) {
+ case BailoutKind::TranspiledCacheIR:
+ // A transpiled guard failed. If this happens often enough, we will
+ // invalidate and recompile.
+ action = BailoutAction::InvalidateIfFrequent;
+ saveFailedICHash = true;
+ break;
+
+ case BailoutKind::SpeculativePhi:
+ // A value of an unexpected type flowed into a phi.
+ MOZ_ASSERT(!outerScript->hadSpeculativePhiBailout());
+ outerScript->setHadSpeculativePhiBailout();
+ InvalidateAfterBailout(cx, outerScript, "phi specialization failure");
+ break;
+
+ case BailoutKind::TypePolicy:
+ // A conversion inserted by a type policy failed.
+ // We will invalidate and disable recompilation if this happens too often.
+ action = BailoutAction::DisableIfFrequent;
+ break;
+
+ case BailoutKind::LICM:
+ // LICM may cause spurious bailouts by hoisting unreachable
+ // guards past branches. To prevent bailout loops, when an
+ // instruction hoisted by LICM bails out, we update the
+ // IonScript and resume in baseline. If the guard would have
+ // been executed anyway, then we will hit the baseline fallback,
+ // and call noteBaselineFallback. If that does not happen,
+ // then the next time we reach this point, we will disable LICM
+ // for this script.
+ MOZ_ASSERT(!outerScript->hadLICMInvalidation());
+ if (outerScript->hasIonScript()) {
+ switch (outerScript->ionScript()->licmState()) {
+ case IonScript::LICMState::NeverBailed:
+ outerScript->ionScript()->setHadLICMBailout();
+ action = BailoutAction::NoAction;
+ break;
+ case IonScript::LICMState::Bailed:
+ outerScript->setHadLICMInvalidation();
+ InvalidateAfterBailout(cx, outerScript, "LICM failure");
+ break;
+ case IonScript::LICMState::BailedAndHitFallback:
+ // This bailout is not due to LICM. Treat it like a
+ // regular TranspiledCacheIR bailout.
+ action = BailoutAction::InvalidateIfFrequent;
+ break;
+ }
+ }
+ break;
+
+ case BailoutKind::InstructionReordering:
+ // An instruction moved up by instruction reordering bailed out.
+ outerScript->setHadReorderingBailout();
+ action = BailoutAction::InvalidateIfFrequent;
+ break;
+
+ case BailoutKind::HoistBoundsCheck:
+ // An instruction hoisted or generated by tryHoistBoundsCheck bailed out.
+ MOZ_ASSERT(!outerScript->failedBoundsCheck());
+ outerScript->setFailedBoundsCheck();
+ InvalidateAfterBailout(cx, outerScript, "bounds check failure");
+ break;
+
+ case BailoutKind::EagerTruncation:
+ // An eager truncation generated by range analysis bailed out.
+ // To avoid bailout loops, we set a flag to avoid generating
+ // eager truncations next time we recompile.
+ MOZ_ASSERT(!outerScript->hadEagerTruncationBailout());
+ outerScript->setHadEagerTruncationBailout();
+ InvalidateAfterBailout(cx, outerScript, "eager range analysis failure");
+ break;
+
+ case BailoutKind::UnboxFolding:
+ // An unbox that was hoisted to fold with a load bailed out.
+ // To avoid bailout loops, we set a flag to avoid folding
+ // loads with unboxes next time we recompile.
+ MOZ_ASSERT(!outerScript->hadUnboxFoldingBailout());
+ outerScript->setHadUnboxFoldingBailout();
+ InvalidateAfterBailout(cx, outerScript, "unbox folding failure");
+ break;
+
+ case BailoutKind::TooManyArguments:
+ // A funapply or spread call had more than JIT_ARGS_LENGTH_MAX arguments.
+ // We will invalidate and disable recompilation if this happens too often.
+ action = BailoutAction::DisableIfFrequent;
+ break;
+
+ case BailoutKind::DuringVMCall:
+ if (cx->isExceptionPending()) {
+ // We are bailing out to catch an exception. We will invalidate
+ // and disable recompilation if this happens too often.
+ action = BailoutAction::DisableIfFrequent;
+ }
+ break;
+
+ case BailoutKind::Finally:
+ // We are bailing out for a finally block. We will invalidate
+ // and disable recompilation if this happens too often.
+ action = BailoutAction::DisableIfFrequent;
+ break;
+
+ case BailoutKind::Inevitable:
+ case BailoutKind::Debugger:
+ // Do nothing.
+ action = BailoutAction::NoAction;
+ break;
+
+ case BailoutKind::FirstExecution:
+ // We reached an instruction that had not been executed yet at
+ // the time we compiled. If this happens often enough, we will
+ // invalidate and recompile.
+ action = BailoutAction::InvalidateIfFrequent;
+ saveFailedICHash = true;
+ break;
+
+ case BailoutKind::UninitializedLexical:
+ HandleLexicalCheckFailure(cx, outerScript, innerScript);
+ break;
+
+ case BailoutKind::ThrowCheckIsObject:
+ MOZ_ASSERT(!cx->isExceptionPending());
+ return ThrowCheckIsObject(cx, CheckIsObjectKind::IteratorReturn);
+
+ case BailoutKind::IonExceptionDebugMode:
+ // Return false to resume in HandleException with reconstructed
+ // baseline frame.
+ return false;
+
+ case BailoutKind::OnStackInvalidation:
+ // The script has already been invalidated. There is nothing left to do.
+ action = BailoutAction::NoAction;
+ break;
+
+ default:
+ MOZ_CRASH("Unknown bailout kind!");
+ }
+
+#ifdef DEBUG
+ if (MOZ_UNLIKELY(cx->runtime()->jitRuntime()->ionBailAfterEnabled())) {
+ action = BailoutAction::NoAction;
+ }
+#endif
+
+ if (outerScript->hasIonScript()) {
+ IonScript* ionScript = outerScript->ionScript();
+ switch (action) {
+ case BailoutAction::InvalidateImmediately:
+ // The IonScript should already have been invalidated.
+ MOZ_ASSERT(false);
+ break;
+ case BailoutAction::InvalidateIfFrequent:
+ ionScript->incNumFixableBailouts();
+ if (ionScript->shouldInvalidate()) {
+#ifdef DEBUG
+ if (saveFailedICHash && !JitOptions.disableBailoutLoopCheck) {
+ outerScript->jitScript()->setFailedICHash(ionScript->icHash());
+ }
+#endif
+ InvalidateAfterBailout(cx, outerScript, "fixable bailouts");
+ }
+ break;
+ case BailoutAction::DisableIfFrequent:
+ ionScript->incNumUnfixableBailouts();
+ if (ionScript->shouldInvalidateAndDisable()) {
+ InvalidateAfterBailout(cx, outerScript, "unfixable bailouts");
+ outerScript->disableIon();
+ }
+ break;
+ case BailoutAction::NoAction:
+ break;
+ }
+ }
+
+ return true;
+}
diff --git a/js/src/jit/BaselineCacheIRCompiler.cpp b/js/src/jit/BaselineCacheIRCompiler.cpp
new file mode 100644
index 0000000000..f87189e099
--- /dev/null
+++ b/js/src/jit/BaselineCacheIRCompiler.cpp
@@ -0,0 +1,3475 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BaselineCacheIRCompiler.h"
+
+#include "gc/GC.h"
+#include "jit/CacheIR.h"
+#include "jit/CacheIRCloner.h"
+#include "jit/CacheIRWriter.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitZone.h"
+#include "jit/Linker.h"
+#include "jit/SharedICHelpers.h"
+#include "jit/VMFunctions.h"
+#include "js/experimental/JitInfo.h" // JSJitInfo
+#include "js/friend/DOMProxy.h" // JS::ExpandoAndGeneration
+#include "proxy/DeadObjectProxy.h"
+#include "proxy/Proxy.h"
+#include "util/Unicode.h"
+#include "vm/JSAtom.h"
+#include "vm/StaticStrings.h"
+
+#include "jit/JitScript-inl.h"
+#include "jit/MacroAssembler-inl.h"
+#include "jit/SharedICHelpers-inl.h"
+#include "jit/VMFunctionList-inl.h"
+#include "vm/List-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::Maybe;
+
+using JS::ExpandoAndGeneration;
+
+namespace js {
+namespace jit {
+
+Address CacheRegisterAllocator::addressOf(MacroAssembler& masm,
+ BaselineFrameSlot slot) const {
+ uint32_t offset =
+ stackPushed_ + ICStackValueOffset + slot.slot() * sizeof(JS::Value);
+ return Address(masm.getStackPointer(), offset);
+}
+BaseValueIndex CacheRegisterAllocator::addressOf(MacroAssembler& masm,
+ Register argcReg,
+ BaselineFrameSlot slot) const {
+ uint32_t offset =
+ stackPushed_ + ICStackValueOffset + slot.slot() * sizeof(JS::Value);
+ return BaseValueIndex(masm.getStackPointer(), argcReg, offset);
+}
+
+// BaselineCacheIRCompiler compiles CacheIR to BaselineIC native code.
+BaselineCacheIRCompiler::BaselineCacheIRCompiler(JSContext* cx,
+ TempAllocator& alloc,
+ const CacheIRWriter& writer,
+ uint32_t stubDataOffset)
+ : CacheIRCompiler(cx, alloc, writer, stubDataOffset, Mode::Baseline,
+ StubFieldPolicy::Address),
+ makesGCCalls_(false) {}
+
+// AutoStubFrame methods
+AutoStubFrame::AutoStubFrame(BaselineCacheIRCompiler& compiler)
+ : compiler(compiler)
+#ifdef DEBUG
+ ,
+ framePushedAtEnterStubFrame_(0)
+#endif
+{
+}
+void AutoStubFrame::enter(MacroAssembler& masm, Register scratch,
+ CallCanGC canGC) {
+ MOZ_ASSERT(compiler.allocator.stackPushed() == 0);
+
+ EmitBaselineEnterStubFrame(masm, scratch);
+
+#ifdef DEBUG
+ framePushedAtEnterStubFrame_ = masm.framePushed();
+#endif
+
+ MOZ_ASSERT(!compiler.enteredStubFrame_);
+ compiler.enteredStubFrame_ = true;
+ if (canGC == CallCanGC::CanGC) {
+ compiler.makesGCCalls_ = true;
+ }
+}
+void AutoStubFrame::leave(MacroAssembler& masm) {
+ MOZ_ASSERT(compiler.enteredStubFrame_);
+ compiler.enteredStubFrame_ = false;
+
+#ifdef DEBUG
+ masm.setFramePushed(framePushedAtEnterStubFrame_);
+#endif
+
+ EmitBaselineLeaveStubFrame(masm);
+}
+
+#ifdef DEBUG
+AutoStubFrame::~AutoStubFrame() { MOZ_ASSERT(!compiler.enteredStubFrame_); }
+#endif
+
+} // namespace jit
+} // namespace js
+
+bool BaselineCacheIRCompiler::makesGCCalls() const { return makesGCCalls_; }
+
+Address BaselineCacheIRCompiler::stubAddress(uint32_t offset) const {
+ return Address(ICStubReg, stubDataOffset_ + offset);
+}
+
+template <typename Fn, Fn fn>
+void BaselineCacheIRCompiler::callVM(MacroAssembler& masm) {
+ VMFunctionId id = VMFunctionToId<Fn, fn>::id;
+ callVMInternal(masm, id);
+}
+
+template <typename Fn, Fn fn>
+void BaselineCacheIRCompiler::tailCallVM(MacroAssembler& masm) {
+ TailCallVMFunctionId id = TailCallVMFunctionToId<Fn, fn>::id;
+ tailCallVMInternal(masm, id);
+}
+
+void BaselineCacheIRCompiler::tailCallVMInternal(MacroAssembler& masm,
+ TailCallVMFunctionId id) {
+ MOZ_ASSERT(!enteredStubFrame_);
+
+ TrampolinePtr code = cx_->runtime()->jitRuntime()->getVMWrapper(id);
+ const VMFunctionData& fun = GetVMFunction(id);
+ MOZ_ASSERT(fun.expectTailCall == TailCall);
+ size_t argSize = fun.explicitStackSlots() * sizeof(void*);
+
+ EmitBaselineTailCallVM(code, masm, argSize);
+}
+
+JitCode* BaselineCacheIRCompiler::compile() {
+ AutoCreatedBy acb(masm, "BaselineCacheIRCompiler::compile");
+
+#ifndef JS_USE_LINK_REGISTER
+ // The first value contains the return addres,
+ // which we pull into ICTailCallReg for tail calls.
+ masm.adjustFrame(sizeof(intptr_t));
+#endif
+#ifdef JS_CODEGEN_ARM
+ masm.setSecondScratchReg(BaselineSecondScratchReg);
+#endif
+ // Count stub entries: We count entries rather than successes as it much
+ // easier to ensure ICStubReg is valid at entry than at exit.
+ Address enteredCount(ICStubReg, ICCacheIRStub::offsetOfEnteredCount());
+ masm.add32(Imm32(1), enteredCount);
+
+ CacheIRReader reader(writer_);
+ do {
+ CacheOp op = reader.readOp();
+ perfSpewer_.recordInstruction(masm, op);
+ switch (op) {
+#define DEFINE_OP(op, ...) \
+ case CacheOp::op: \
+ if (!emit##op(reader)) return nullptr; \
+ break;
+ CACHE_IR_OPS(DEFINE_OP)
+#undef DEFINE_OP
+
+ default:
+ MOZ_CRASH("Invalid op");
+ }
+ allocator.nextOp();
+ } while (reader.more());
+
+ MOZ_ASSERT(!enteredStubFrame_);
+ masm.assumeUnreachable("Should have returned from IC");
+
+ // Done emitting the main IC code. Now emit the failure paths.
+ for (size_t i = 0; i < failurePaths.length(); i++) {
+ if (!emitFailurePath(i)) {
+ return nullptr;
+ }
+ EmitStubGuardFailure(masm);
+ }
+
+ Linker linker(masm);
+ Rooted<JitCode*> newStubCode(cx_, linker.newCode(cx_, CodeKind::Baseline));
+ if (!newStubCode) {
+ cx_->recoverFromOutOfMemory();
+ return nullptr;
+ }
+
+ return newStubCode;
+}
+
+bool BaselineCacheIRCompiler::emitGuardShape(ObjOperandId objId,
+ uint32_t shapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch1(allocator, masm);
+
+ bool needSpectreMitigations = objectGuardNeedsSpectreMitigations(objId);
+
+ Maybe<AutoScratchRegister> maybeScratch2;
+ if (needSpectreMitigations) {
+ maybeScratch2.emplace(allocator, masm);
+ }
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address addr(stubAddress(shapeOffset));
+ masm.loadPtr(addr, scratch1);
+ if (needSpectreMitigations) {
+ masm.branchTestObjShape(Assembler::NotEqual, obj, scratch1, *maybeScratch2,
+ obj, failure->label());
+ } else {
+ masm.branchTestObjShapeNoSpectreMitigations(Assembler::NotEqual, obj,
+ scratch1, failure->label());
+ }
+
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardProto(ObjOperandId objId,
+ uint32_t protoOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address addr(stubAddress(protoOffset));
+ masm.loadObjProto(obj, scratch);
+ masm.branchPtr(Assembler::NotEqual, addr, scratch, failure->label());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardCompartment(ObjOperandId objId,
+ uint32_t globalOffset,
+ uint32_t compartmentOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Verify that the global wrapper is still valid, as
+ // it is pre-requisite for doing the compartment check.
+ Address globalWrapper(stubAddress(globalOffset));
+ masm.loadPtr(globalWrapper, scratch);
+ Address handlerAddr(scratch, ProxyObject::offsetOfHandler());
+ masm.branchPtr(Assembler::Equal, handlerAddr,
+ ImmPtr(&DeadObjectProxy::singleton), failure->label());
+
+ Address addr(stubAddress(compartmentOffset));
+ masm.branchTestObjCompartment(Assembler::NotEqual, obj, addr, scratch,
+ failure->label());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardAnyClass(ObjOperandId objId,
+ uint32_t claspOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address testAddr(stubAddress(claspOffset));
+ if (objectGuardNeedsSpectreMitigations(objId)) {
+ masm.branchTestObjClass(Assembler::NotEqual, obj, testAddr, scratch, obj,
+ failure->label());
+ } else {
+ masm.branchTestObjClassNoSpectreMitigations(
+ Assembler::NotEqual, obj, testAddr, scratch, failure->label());
+ }
+
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardHasProxyHandler(ObjOperandId objId,
+ uint32_t handlerOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address testAddr(stubAddress(handlerOffset));
+ masm.loadPtr(testAddr, scratch);
+
+ Address handlerAddr(obj, ProxyObject::offsetOfHandler());
+ masm.branchPtr(Assembler::NotEqual, handlerAddr, scratch, failure->label());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardSpecificObject(ObjOperandId objId,
+ uint32_t expectedOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address addr(stubAddress(expectedOffset));
+ masm.branchPtr(Assembler::NotEqual, addr, obj, failure->label());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardSpecificFunction(
+ ObjOperandId objId, uint32_t expectedOffset, uint32_t nargsAndFlagsOffset) {
+ return emitGuardSpecificObject(objId, expectedOffset);
+}
+
+bool BaselineCacheIRCompiler::emitGuardFunctionScript(
+ ObjOperandId funId, uint32_t expectedOffset, uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register fun = allocator.useRegister(masm, funId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address addr(stubAddress(expectedOffset));
+ masm.loadPrivate(Address(fun, JSFunction::offsetOfJitInfoOrScript()),
+ scratch);
+ masm.branchPtr(Assembler::NotEqual, addr, scratch, failure->label());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardSpecificAtom(StringOperandId strId,
+ uint32_t expectedOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register str = allocator.useRegister(masm, strId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address atomAddr(stubAddress(expectedOffset));
+
+ Label done;
+ masm.branchPtr(Assembler::Equal, atomAddr, str, &done);
+
+ // The pointers are not equal, so if the input string is also an atom it
+ // must be a different string.
+ masm.branchTest32(Assembler::NonZero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::ATOM_BIT), failure->label());
+
+ // Check the length.
+ masm.loadPtr(atomAddr, scratch);
+ masm.loadStringLength(scratch, scratch);
+ masm.branch32(Assembler::NotEqual, Address(str, JSString::offsetOfLength()),
+ scratch, failure->label());
+
+ // We have a non-atomized string with the same length. Call a helper
+ // function to do the comparison.
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSString * str1, JSString * str2);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadPtr(atomAddr, scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(str);
+ masm.callWithABI<Fn, EqualStringsHelperPure>();
+ masm.storeCallPointerResult(scratch);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch);
+ masm.PopRegsInMaskIgnore(volatileRegs, ignore);
+ masm.branchIfFalseBool(scratch, failure->label());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardSpecificSymbol(SymbolOperandId symId,
+ uint32_t expectedOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register sym = allocator.useRegister(masm, symId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address addr(stubAddress(expectedOffset));
+ masm.branchPtr(Assembler::NotEqual, addr, sym, failure->label());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitLoadValueResult(uint32_t valOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ masm.loadValue(stubAddress(valOffset), output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitLoadFixedSlotResult(ObjOperandId objId,
+ uint32_t offsetOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.load32(stubAddress(offsetOffset), scratch);
+ masm.loadValue(BaseIndex(obj, scratch, TimesOne), output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitLoadFixedSlotTypedResult(
+ ObjOperandId objId, uint32_t offsetOffset, ValueType) {
+ // The type is only used by Warp.
+ return emitLoadFixedSlotResult(objId, offsetOffset);
+}
+
+bool BaselineCacheIRCompiler::emitLoadDynamicSlotResult(ObjOperandId objId,
+ uint32_t offsetOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ masm.load32(stubAddress(offsetOffset), scratch);
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch2);
+ masm.loadValue(BaseIndex(scratch2, scratch, TimesOne), output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallScriptedGetterShared(
+ ValOperandId receiverId, uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset, Maybe<uint32_t> icScriptOffset) {
+ ValueOperand receiver = allocator.useValueRegister(masm, receiverId);
+ Address getterAddr(stubAddress(getterOffset));
+
+ AutoScratchRegister code(allocator, masm);
+ AutoScratchRegister callee(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+
+ bool isInlined = icScriptOffset.isSome();
+
+ // First, retrieve raw jitcode for getter.
+ masm.loadPtr(getterAddr, callee);
+ if (isInlined) {
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ masm.loadBaselineJitCodeRaw(callee, code, failure->label());
+ } else {
+ masm.loadJitCodeRaw(callee, code);
+ }
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ if (!sameRealm) {
+ masm.switchToObjectRealm(callee, scratch);
+ }
+
+ // Align the stack such that the JitFrameLayout is aligned on
+ // JitStackAlignment.
+ masm.alignJitStackBasedOnNArgs(0, /*countIncludesThis = */ false);
+
+ // Getter is called with 0 arguments, just |receiver| as thisv.
+ // Note that we use Push, not push, so that callJit will align the stack
+ // properly on ARM.
+ masm.Push(receiver);
+
+ if (isInlined) {
+ // Store icScript in the context.
+ Address icScriptAddr(stubAddress(*icScriptOffset));
+ masm.loadPtr(icScriptAddr, scratch);
+ masm.storeICScriptInJSContext(scratch);
+ }
+
+ masm.Push(callee);
+ masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, /* argc = */ 0);
+
+ // Handle arguments underflow.
+ Label noUnderflow;
+ masm.loadFunctionArgCount(callee, callee);
+ masm.branch32(Assembler::Equal, callee, Imm32(0), &noUnderflow);
+
+ // Call the arguments rectifier.
+ ArgumentsRectifierKind kind = isInlined
+ ? ArgumentsRectifierKind::TrialInlining
+ : ArgumentsRectifierKind::Normal;
+ TrampolinePtr argumentsRectifier =
+ cx_->runtime()->jitRuntime()->getArgumentsRectifier(kind);
+ masm.movePtr(argumentsRectifier, code);
+
+ masm.bind(&noUnderflow);
+ masm.callJit(code);
+
+ stubFrame.leave(masm);
+
+ if (!sameRealm) {
+ masm.switchToBaselineFrameRealm(R1.scratchReg());
+ }
+
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallScriptedGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<uint32_t> icScriptOffset = mozilla::Nothing();
+ return emitCallScriptedGetterShared(receiverId, getterOffset, sameRealm,
+ nargsAndFlagsOffset, icScriptOffset);
+}
+
+bool BaselineCacheIRCompiler::emitCallInlinedGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, uint32_t icScriptOffset,
+ bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitCallScriptedGetterShared(receiverId, getterOffset, sameRealm,
+ nargsAndFlagsOffset,
+ mozilla::Some(icScriptOffset));
+}
+
+bool BaselineCacheIRCompiler::emitCallNativeGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand receiver = allocator.useValueRegister(masm, receiverId);
+ Address getterAddr(stubAddress(getterOffset));
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Load the callee in the scratch register.
+ masm.loadPtr(getterAddr, scratch);
+
+ masm.Push(receiver);
+ masm.Push(scratch);
+
+ using Fn =
+ bool (*)(JSContext*, HandleFunction, HandleValue, MutableHandleValue);
+ callVM<Fn, CallNativeGetter>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallDOMGetterResult(ObjOperandId objId,
+ uint32_t jitInfoOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ Address jitInfoAddr(stubAddress(jitInfoOffset));
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Load the JSJitInfo in the scratch register.
+ masm.loadPtr(jitInfoAddr, scratch);
+
+ masm.Push(obj);
+ masm.Push(scratch);
+
+ using Fn =
+ bool (*)(JSContext*, const JSJitInfo*, HandleObject, MutableHandleValue);
+ callVM<Fn, CallDOMGetter>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitProxyGetResult(ObjOperandId objId,
+ uint32_t idOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Address idAddr(stubAddress(idOffset));
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Load the jsid in the scratch register.
+ masm.loadPtr(idAddr, scratch);
+
+ masm.Push(scratch);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleId, MutableHandleValue);
+ callVM<Fn, ProxyGetProperty>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitFrameIsConstructingResult() {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register outputScratch = output.valueReg().scratchReg();
+
+ // Load the CalleeToken.
+ Address tokenAddr(FramePointer, JitFrameLayout::offsetOfCalleeToken());
+ masm.loadPtr(tokenAddr, outputScratch);
+
+ // The low bit indicates whether this call is constructing, just clear the
+ // other bits.
+ static_assert(CalleeToken_Function == 0x0);
+ static_assert(CalleeToken_FunctionConstructing == 0x1);
+ masm.andPtr(Imm32(0x1), outputScratch);
+
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, outputScratch, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitLoadConstantStringResult(uint32_t strOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.loadPtr(stubAddress(strOffset), scratch);
+ masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCompareStringResult(JSOp op,
+ StringOperandId lhsId,
+ StringOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ Register left = allocator.useRegister(masm, lhsId);
+ Register right = allocator.useRegister(masm, rhsId);
+
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ allocator.discardStack(masm);
+
+ Label slow, done;
+ masm.compareStrings(op, left, right, scratch, &slow);
+ masm.jump(&done);
+ masm.bind(&slow);
+ {
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
+ // - |left <= right| is implemented as |right >= left|.
+ // - |left > right| is implemented as |right < left|.
+ if (op == JSOp::Le || op == JSOp::Gt) {
+ masm.Push(left);
+ masm.Push(right);
+ } else {
+ masm.Push(right);
+ masm.Push(left);
+ }
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ if (op == JSOp::Eq || op == JSOp::StrictEq) {
+ callVM<Fn, jit::StringsEqual<EqualityKind::Equal>>(masm);
+ } else if (op == JSOp::Ne || op == JSOp::StrictNe) {
+ callVM<Fn, jit::StringsEqual<EqualityKind::NotEqual>>(masm);
+ } else if (op == JSOp::Lt || op == JSOp::Gt) {
+ callVM<Fn, jit::StringsCompare<ComparisonKind::LessThan>>(masm);
+ } else {
+ MOZ_ASSERT(op == JSOp::Le || op == JSOp::Ge);
+ callVM<Fn, jit::StringsCompare<ComparisonKind::GreaterThanOrEqual>>(masm);
+ }
+
+ stubFrame.leave(masm);
+ masm.storeCallPointerResult(scratch);
+ }
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitSameValueResult(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegister scratch(allocator, masm);
+ ValueOperand lhs = allocator.useValueRegister(masm, lhsId);
+#ifdef JS_CODEGEN_X86
+ // Use the output to avoid running out of registers.
+ allocator.copyToScratchValueRegister(masm, rhsId, output.valueReg());
+ ValueOperand rhs = output.valueReg();
+#else
+ ValueOperand rhs = allocator.useValueRegister(masm, rhsId);
+#endif
+
+ allocator.discardStack(masm);
+
+ Label done;
+ Label call;
+
+ // Check to see if the values have identical bits.
+ // This is correct for SameValue because SameValue(NaN,NaN) is true,
+ // and SameValue(0,-0) is false.
+ masm.branch64(Assembler::NotEqual, lhs.toRegister64(), rhs.toRegister64(),
+ &call);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ {
+ masm.bind(&call);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.pushValue(lhs);
+ masm.pushValue(rhs);
+
+ using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*);
+ callVM<Fn, SameValue>(masm);
+
+ stubFrame.leave(masm);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, output.valueReg());
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitStoreSlotShared(bool isFixed,
+ ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId) {
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ Maybe<AutoScratchRegister> scratch2;
+ if (!isFixed) {
+ scratch2.emplace(allocator, masm);
+ }
+
+ Address offsetAddr = stubAddress(offsetOffset);
+ masm.load32(offsetAddr, scratch1);
+
+ if (isFixed) {
+ BaseIndex slot(obj, scratch1, TimesOne);
+ EmitPreBarrier(masm, slot, MIRType::Value);
+ masm.storeValue(val, slot);
+ } else {
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch2.ref());
+ BaseIndex slot(scratch2.ref(), scratch1, TimesOne);
+ EmitPreBarrier(masm, slot, MIRType::Value);
+ masm.storeValue(val, slot);
+ }
+
+ emitPostBarrierSlot(obj, val, scratch1);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitStoreFixedSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitStoreSlotShared(true, objId, offsetOffset, rhsId);
+}
+
+bool BaselineCacheIRCompiler::emitStoreDynamicSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitStoreSlotShared(false, objId, offsetOffset, rhsId);
+}
+
+bool BaselineCacheIRCompiler::emitAddAndStoreSlotShared(
+ CacheOp op, ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset, Maybe<uint32_t> numNewSlotsOffset) {
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ Address newShapeAddr = stubAddress(newShapeOffset);
+ Address offsetAddr = stubAddress(offsetOffset);
+
+ if (op == CacheOp::AllocateAndStoreDynamicSlot) {
+ // We have to (re)allocate dynamic slots. Do this first, as it's the
+ // only fallible operation here. Note that growSlotsPure is fallible but
+ // does not GC.
+ Address numNewSlotsAddr = stubAddress(*numNewSlotsOffset);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = bool (*)(JSContext * cx, NativeObject * obj, uint32_t newCount);
+ masm.setupUnalignedABICall(scratch1);
+ masm.loadJSContext(scratch1);
+ masm.passABIArg(scratch1);
+ masm.passABIArg(obj);
+ masm.load32(numNewSlotsAddr, scratch2);
+ masm.passABIArg(scratch2);
+ masm.callWithABI<Fn, NativeObject::growSlotsPure>();
+ masm.storeCallPointerResult(scratch1);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch1);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.branchIfFalseBool(scratch1, failure->label());
+ }
+
+ // Update the object's shape.
+ masm.loadPtr(newShapeAddr, scratch1);
+ masm.storeObjShape(scratch1, obj,
+ [](MacroAssembler& masm, const Address& addr) {
+ EmitPreBarrier(masm, addr, MIRType::Shape);
+ });
+
+ // Perform the store. No pre-barrier required since this is a new
+ // initialization.
+ masm.load32(offsetAddr, scratch1);
+ if (op == CacheOp::AddAndStoreFixedSlot) {
+ BaseIndex slot(obj, scratch1, TimesOne);
+ masm.storeValue(val, slot);
+ } else {
+ MOZ_ASSERT(op == CacheOp::AddAndStoreDynamicSlot ||
+ op == CacheOp::AllocateAndStoreDynamicSlot);
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch2);
+ BaseIndex slot(scratch2, scratch1, TimesOne);
+ masm.storeValue(val, slot);
+ }
+
+ emitPostBarrierSlot(obj, val, scratch1);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitAddAndStoreFixedSlot(
+ ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<uint32_t> numNewSlotsOffset = mozilla::Nothing();
+ return emitAddAndStoreSlotShared(CacheOp::AddAndStoreFixedSlot, objId,
+ offsetOffset, rhsId, newShapeOffset,
+ numNewSlotsOffset);
+}
+
+bool BaselineCacheIRCompiler::emitAddAndStoreDynamicSlot(
+ ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<uint32_t> numNewSlotsOffset = mozilla::Nothing();
+ return emitAddAndStoreSlotShared(CacheOp::AddAndStoreDynamicSlot, objId,
+ offsetOffset, rhsId, newShapeOffset,
+ numNewSlotsOffset);
+}
+
+bool BaselineCacheIRCompiler::emitAllocateAndStoreDynamicSlot(
+ ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset, uint32_t numNewSlotsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitAddAndStoreSlotShared(CacheOp::AllocateAndStoreDynamicSlot, objId,
+ offsetOffset, rhsId, newShapeOffset,
+ mozilla::Some(numNewSlotsOffset));
+}
+
+bool BaselineCacheIRCompiler::emitArrayJoinResult(ObjOperandId objId,
+ StringOperandId sepId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register sep = allocator.useRegister(masm, sepId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ allocator.discardStack(masm);
+
+ // Load obj->elements in scratch.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+ Address lengthAddr(scratch, ObjectElements::offsetOfLength());
+
+ // If array length is 0, return empty string.
+ Label finished;
+
+ {
+ Label arrayNotEmpty;
+ masm.branch32(Assembler::NotEqual, lengthAddr, Imm32(0), &arrayNotEmpty);
+ masm.movePtr(ImmGCPtr(cx_->names().empty), scratch);
+ masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg());
+ masm.jump(&finished);
+ masm.bind(&arrayNotEmpty);
+ }
+
+ Label vmCall;
+
+ // Otherwise, handle array length 1 case.
+ masm.branch32(Assembler::NotEqual, lengthAddr, Imm32(1), &vmCall);
+
+ // But only if initializedLength is also 1.
+ Address initLength(scratch, ObjectElements::offsetOfInitializedLength());
+ masm.branch32(Assembler::NotEqual, initLength, Imm32(1), &vmCall);
+
+ // And only if elem0 is a string.
+ Address elementAddr(scratch, 0);
+ masm.branchTestString(Assembler::NotEqual, elementAddr, &vmCall);
+
+ // Store the value.
+ masm.loadValue(elementAddr, output.valueReg());
+ masm.jump(&finished);
+
+ // Otherwise call into the VM.
+ {
+ masm.bind(&vmCall);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.Push(sep);
+ masm.Push(obj);
+
+ using Fn = JSString* (*)(JSContext*, HandleObject, HandleString);
+ callVM<Fn, jit::ArrayJoin>(masm);
+
+ stubFrame.leave(masm);
+
+ masm.tagValue(JSVAL_TYPE_STRING, ReturnReg, output.valueReg());
+ }
+
+ masm.bind(&finished);
+
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitPackedArraySliceResult(
+ uint32_t templateObjectOffset, ObjOperandId arrayId, Int32OperandId beginId,
+ Int32OperandId endId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output);
+
+ Register array = allocator.useRegister(masm, arrayId);
+ Register begin = allocator.useRegister(masm, beginId);
+ Register end = allocator.useRegister(masm, endId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchArrayIsNotPacked(array, scratch1, scratch2, failure->label());
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch1);
+
+ // Don't attempt to pre-allocate the object, instead always use the slow
+ // path.
+ ImmPtr result(nullptr);
+
+ masm.Push(result);
+ masm.Push(end);
+ masm.Push(begin);
+ masm.Push(array);
+
+ using Fn =
+ JSObject* (*)(JSContext*, HandleObject, int32_t, int32_t, HandleObject);
+ callVM<Fn, ArraySliceDense>(masm);
+
+ stubFrame.leave(masm);
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitArgumentsSliceResult(
+ uint32_t templateObjectOffset, ObjOperandId argsId, Int32OperandId beginId,
+ Int32OperandId endId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register args = allocator.useRegister(masm, argsId);
+ Register begin = allocator.useRegister(masm, beginId);
+ Register end = allocator.useRegister(masm, endId);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Don't attempt to pre-allocate the object, instead always use the slow path.
+ ImmPtr result(nullptr);
+
+ masm.Push(result);
+ masm.Push(end);
+ masm.Push(begin);
+ masm.Push(args);
+
+ using Fn =
+ JSObject* (*)(JSContext*, HandleObject, int32_t, int32_t, HandleObject);
+ callVM<Fn, ArgumentsSliceDense>(masm);
+
+ stubFrame.leave(masm);
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitIsArrayResult(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegisterMaybeOutput scratch2(allocator, masm, output);
+
+ ValueOperand val = allocator.useValueRegister(masm, inputId);
+
+ allocator.discardStack(masm);
+
+ Label isNotArray;
+ // Primitives are never Arrays.
+ masm.fallibleUnboxObject(val, scratch1, &isNotArray);
+
+ Label isArray;
+ masm.branchTestObjClass(Assembler::Equal, scratch1, &ArrayObject::class_,
+ scratch2, scratch1, &isArray);
+
+ // isArray can also return true for Proxy wrapped Arrays.
+ masm.branchTestObjectIsProxy(false, scratch1, scratch2, &isNotArray);
+ Label done;
+ {
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch2);
+
+ masm.Push(scratch1);
+
+ using Fn = bool (*)(JSContext*, HandleObject, bool*);
+ callVM<Fn, js::IsArrayFromJit>(masm);
+
+ stubFrame.leave(masm);
+
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, output.valueReg());
+ masm.jump(&done);
+ }
+
+ masm.bind(&isNotArray);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&isArray);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitIsTypedArrayResult(ObjOperandId objId,
+ bool isPossiblyWrapped) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register obj = allocator.useRegister(masm, objId);
+
+ allocator.discardStack(masm);
+
+ Label notTypedArray, isProxy, done;
+ masm.loadObjClassUnsafe(obj, scratch);
+ masm.branchIfClassIsNotTypedArray(scratch, &notTypedArray);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&notTypedArray);
+ if (isPossiblyWrapped) {
+ masm.branchTestClassIsProxy(true, scratch, &isProxy);
+ }
+ masm.moveValue(BooleanValue(false), output.valueReg());
+
+ if (isPossiblyWrapped) {
+ masm.jump(&done);
+
+ masm.bind(&isProxy);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, JSObject*, bool*);
+ callVM<Fn, jit::IsPossiblyWrappedTypedArray>(masm);
+
+ stubFrame.leave(masm);
+
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, output.valueReg());
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitLoadStringCharResult(StringOperandId strId,
+ Int32OperandId indexId,
+ bool handleOOB) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register str = allocator.useRegister(masm, strId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output);
+ AutoScratchRegister scratch3(allocator, masm);
+
+ // Bounds check, load string char.
+ Label done;
+ Label loadFailed;
+ if (!handleOOB) {
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
+ scratch1, failure->label());
+ masm.loadStringChar(str, index, scratch1, scratch2, scratch3,
+ failure->label());
+
+ allocator.discardStack(masm);
+ } else {
+ // Discard the stack before jumping to |done|.
+ allocator.discardStack(masm);
+
+ // Return the empty string for out-of-bounds access.
+ masm.movePtr(ImmGCPtr(cx_->names().empty), scratch2);
+
+ // This CacheIR op is always preceded by |LinearizeForCharAccess|, so we're
+ // guaranteed to see no nested ropes.
+ masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
+ scratch1, &done);
+ masm.loadStringChar(str, index, scratch1, scratch2, scratch3, &loadFailed);
+ }
+
+ // Load StaticString for this char. For larger code units perform a VM call.
+ Label vmCall;
+ masm.boundsCheck32PowerOfTwo(scratch1, StaticStrings::UNIT_STATIC_LIMIT,
+ &vmCall);
+ masm.movePtr(ImmPtr(&cx_->staticStrings().unitStaticTable), scratch2);
+ masm.loadPtr(BaseIndex(scratch2, scratch1, ScalePointer), scratch2);
+
+ masm.jump(&done);
+
+ if (handleOOB) {
+ masm.bind(&loadFailed);
+ masm.assumeUnreachable("loadStringChar can't fail for linear strings");
+ }
+
+ {
+ masm.bind(&vmCall);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch2);
+
+ masm.Push(scratch1);
+
+ using Fn = JSLinearString* (*)(JSContext*, int32_t);
+ callVM<Fn, jit::StringFromCharCode>(masm);
+
+ stubFrame.leave(masm);
+
+ masm.storeCallPointerResult(scratch2);
+ }
+
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_STRING, scratch2, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitStringFromCodeResult(Int32OperandId codeId,
+ StringCode stringCode) {
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register code = allocator.useRegister(masm, codeId);
+
+ FailurePath* failure = nullptr;
+ if (stringCode == StringCode::CodePoint) {
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ }
+
+ if (stringCode == StringCode::CodePoint) {
+ // Note: This condition must match tryAttachStringFromCodePoint to prevent
+ // failure loops.
+ masm.branch32(Assembler::Above, code, Imm32(unicode::NonBMPMax),
+ failure->label());
+ }
+
+ allocator.discardStack(masm);
+
+ // We pre-allocate atoms for the first UNIT_STATIC_LIMIT characters.
+ // For code units larger than that, we must do a VM call.
+ Label vmCall;
+ masm.boundsCheck32PowerOfTwo(code, StaticStrings::UNIT_STATIC_LIMIT, &vmCall);
+
+ masm.movePtr(ImmPtr(cx_->runtime()->staticStrings->unitStaticTable), scratch);
+ masm.loadPtr(BaseIndex(scratch, code, ScalePointer), scratch);
+ Label done;
+ masm.jump(&done);
+
+ {
+ masm.bind(&vmCall);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.Push(code);
+
+ if (stringCode == StringCode::CodeUnit) {
+ using Fn = JSLinearString* (*)(JSContext*, int32_t);
+ callVM<Fn, jit::StringFromCharCode>(masm);
+ } else {
+ using Fn = JSString* (*)(JSContext*, int32_t);
+ callVM<Fn, jit::StringFromCodePoint>(masm);
+ }
+
+ stubFrame.leave(masm);
+ masm.storeCallPointerResult(scratch);
+ }
+
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitStringFromCharCodeResult(
+ Int32OperandId codeId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ return emitStringFromCodeResult(codeId, StringCode::CodeUnit);
+}
+
+bool BaselineCacheIRCompiler::emitStringFromCodePointResult(
+ Int32OperandId codeId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ return emitStringFromCodeResult(codeId, StringCode::CodePoint);
+}
+
+bool BaselineCacheIRCompiler::emitMathRandomResult(uint32_t rngOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister64 scratch2(allocator, masm);
+ AutoAvailableFloatRegister scratchFloat(*this, FloatReg0);
+
+ Address rngAddr(stubAddress(rngOffset));
+ masm.loadPtr(rngAddr, scratch1);
+
+ masm.randomDouble(scratch1, scratchFloat, scratch2,
+ output.valueReg().toRegister64());
+
+ if (js::SupportDifferentialTesting()) {
+ masm.loadConstantDouble(0.0, scratchFloat);
+ }
+
+ masm.boxDouble(scratchFloat, output.valueReg(), scratchFloat);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitReflectGetPrototypeOfResult(
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register obj = allocator.useRegister(masm, objId);
+
+ allocator.discardStack(masm);
+
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ masm.loadObjProto(obj, scratch);
+
+ Label hasProto;
+ masm.branchPtr(Assembler::Above, scratch, ImmWord(1), &hasProto);
+
+ // Call into the VM for lazy prototypes.
+ Label slow, done;
+ masm.branchPtr(Assembler::Equal, scratch, ImmWord(1), &slow);
+
+ masm.moveValue(NullValue(), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&hasProto);
+ masm.tagValue(JSVAL_TYPE_OBJECT, scratch, output.valueReg());
+ masm.jump(&done);
+
+ {
+ masm.bind(&slow);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, MutableHandleValue);
+ callVM<Fn, jit::GetPrototypeOf>(masm);
+
+ stubFrame.leave(masm);
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitHasClassResult(ObjOperandId objId,
+ uint32_t claspOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Address claspAddr(stubAddress(claspOffset));
+ masm.loadObjClassUnsafe(obj, scratch);
+ masm.cmpPtrSet(Assembler::Equal, claspAddr, scratch.get(), scratch);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ return true;
+}
+
+void BaselineCacheIRCompiler::emitAtomizeString(Register str, Register temp,
+ Label* failure) {
+ Label isAtom;
+ masm.branchTest32(Assembler::NonZero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::ATOM_BIT), &isAtom);
+ {
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = JSAtom* (*)(JSContext * cx, JSString * str);
+ masm.setupUnalignedABICall(temp);
+ masm.loadJSContext(temp);
+ masm.passABIArg(temp);
+ masm.passABIArg(str);
+ masm.callWithABI<Fn, jit::AtomizeStringNoGC>();
+ masm.storeCallPointerResult(temp);
+
+ LiveRegisterSet ignore;
+ ignore.add(temp);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.branchPtr(Assembler::Equal, temp, ImmWord(0), failure);
+ masm.mov(temp, str);
+ }
+ masm.bind(&isAtom);
+}
+
+bool BaselineCacheIRCompiler::emitSetHasStringResult(ObjOperandId setId,
+ StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register set = allocator.useRegister(masm, setId);
+ Register str = allocator.useRegister(masm, strId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ emitAtomizeString(str, scratch1, failure->label());
+ masm.prepareHashString(str, scratch1, scratch2);
+
+ masm.tagValue(JSVAL_TYPE_STRING, str, output.valueReg());
+ masm.setObjectHasNonBigInt(set, output.valueReg(), scratch1, scratch2,
+ scratch3, scratch4);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitMapHasStringResult(ObjOperandId mapId,
+ StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ Register str = allocator.useRegister(masm, strId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ emitAtomizeString(str, scratch1, failure->label());
+ masm.prepareHashString(str, scratch1, scratch2);
+
+ masm.tagValue(JSVAL_TYPE_STRING, str, output.valueReg());
+ masm.mapObjectHasNonBigInt(map, output.valueReg(), scratch1, scratch2,
+ scratch3, scratch4);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitMapGetStringResult(ObjOperandId mapId,
+ StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ Register str = allocator.useRegister(masm, strId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ emitAtomizeString(str, scratch1, failure->label());
+ masm.prepareHashString(str, scratch1, scratch2);
+
+ masm.tagValue(JSVAL_TYPE_STRING, str, output.valueReg());
+ masm.mapObjectGetNonBigInt(map, output.valueReg(), scratch1,
+ output.valueReg(), scratch2, scratch3, scratch4);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallNativeSetter(
+ ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId,
+ bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register receiver = allocator.useRegister(masm, receiverId);
+ Address setterAddr(stubAddress(setterOffset));
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Load the callee in the scratch register.
+ masm.loadPtr(setterAddr, scratch);
+
+ masm.Push(val);
+ masm.Push(receiver);
+ masm.Push(scratch);
+
+ using Fn = bool (*)(JSContext*, HandleFunction, HandleObject, HandleValue);
+ callVM<Fn, CallNativeSetter>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallScriptedSetterShared(
+ ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId,
+ bool sameRealm, uint32_t nargsAndFlagsOffset,
+ Maybe<uint32_t> icScriptOffset) {
+ AutoScratchRegister callee(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+#if defined(JS_CODEGEN_X86)
+ Register code = scratch;
+#else
+ AutoScratchRegister code(allocator, masm);
+#endif
+
+ Register receiver = allocator.useRegister(masm, receiverId);
+ Address setterAddr(stubAddress(setterOffset));
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ bool isInlined = icScriptOffset.isSome();
+
+ // First, load the callee.
+ masm.loadPtr(setterAddr, callee);
+
+ if (isInlined) {
+ // If we are calling a trial-inlined setter, guard that the
+ // target has a BaselineScript.
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ masm.loadBaselineJitCodeRaw(callee, code, failure->label());
+ }
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ if (!sameRealm) {
+ masm.switchToObjectRealm(callee, scratch);
+ }
+
+ // Align the stack such that the JitFrameLayout is aligned on
+ // JitStackAlignment.
+ masm.alignJitStackBasedOnNArgs(1, /*countIncludesThis = */ false);
+
+ // Setter is called with 1 argument, and |receiver| as thisv. Note that we use
+ // Push, not push, so that callJit will align the stack properly on ARM.
+ masm.Push(val);
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(receiver)));
+
+ // Push callee.
+ masm.Push(callee);
+
+ // Push frame descriptor.
+ masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, /* argc = */ 1);
+
+ if (isInlined) {
+ // Store icScript in the context.
+ Address icScriptAddr(stubAddress(*icScriptOffset));
+ masm.loadPtr(icScriptAddr, scratch);
+ masm.storeICScriptInJSContext(scratch);
+ }
+
+ // Load the jitcode pointer.
+ if (isInlined) {
+ // On non-x86 platforms, this pointer is still in a register
+ // after guarding on it above. On x86, we don't have enough
+ // registers and have to reload it here.
+#ifdef JS_CODEGEN_X86
+ masm.loadBaselineJitCodeRaw(callee, code);
+#endif
+ } else {
+ masm.loadJitCodeRaw(callee, code);
+ }
+
+ // Handle arguments underflow. The rhs value is no longer needed and
+ // can be used as scratch.
+ Label noUnderflow;
+ Register scratch2 = val.scratchReg();
+ masm.loadFunctionArgCount(callee, scratch2);
+ masm.branch32(Assembler::BelowOrEqual, scratch2, Imm32(1), &noUnderflow);
+
+ // Call the arguments rectifier.
+ ArgumentsRectifierKind kind = isInlined
+ ? ArgumentsRectifierKind::TrialInlining
+ : ArgumentsRectifierKind::Normal;
+ TrampolinePtr argumentsRectifier =
+ cx_->runtime()->jitRuntime()->getArgumentsRectifier(kind);
+ masm.movePtr(argumentsRectifier, code);
+
+ masm.bind(&noUnderflow);
+ masm.callJit(code);
+
+ stubFrame.leave(masm);
+
+ if (!sameRealm) {
+ masm.switchToBaselineFrameRealm(R1.scratchReg());
+ }
+
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallScriptedSetter(
+ ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId,
+ bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<uint32_t> icScriptOffset = mozilla::Nothing();
+ return emitCallScriptedSetterShared(receiverId, setterOffset, rhsId,
+ sameRealm, nargsAndFlagsOffset,
+ icScriptOffset);
+}
+
+bool BaselineCacheIRCompiler::emitCallInlinedSetter(
+ ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId,
+ uint32_t icScriptOffset, bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitCallScriptedSetterShared(receiverId, setterOffset, rhsId,
+ sameRealm, nargsAndFlagsOffset,
+ mozilla::Some(icScriptOffset));
+}
+
+bool BaselineCacheIRCompiler::emitCallDOMSetter(ObjOperandId objId,
+ uint32_t jitInfoOffset,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+ Address jitInfoAddr(stubAddress(jitInfoOffset));
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Load the JSJitInfo in the scratch register.
+ masm.loadPtr(jitInfoAddr, scratch);
+
+ masm.Push(val);
+ masm.Push(obj);
+ masm.Push(scratch);
+
+ using Fn = bool (*)(JSContext*, const JSJitInfo*, HandleObject, HandleValue);
+ callVM<Fn, CallDOMSetter>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallSetArrayLength(ObjOperandId objId,
+ bool strict,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.Push(Imm32(strict));
+ masm.Push(val);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool);
+ callVM<Fn, jit::SetArrayLength>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitProxySet(ObjOperandId objId,
+ uint32_t idOffset,
+ ValOperandId rhsId, bool strict) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+ Address idAddr(stubAddress(idOffset));
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Load the jsid in the scratch register.
+ masm.loadPtr(idAddr, scratch);
+
+ masm.Push(Imm32(strict));
+ masm.Push(val);
+ masm.Push(scratch);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleId, HandleValue, bool);
+ callVM<Fn, ProxySetProperty>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitProxySetByValue(ObjOperandId objId,
+ ValOperandId idId,
+ ValOperandId rhsId,
+ bool strict) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand idVal = allocator.useValueRegister(masm, idId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+
+ // We need a scratch register but we don't have any registers available on
+ // x86, so temporarily store |obj| in the frame's scratch slot.
+ int scratchOffset = BaselineFrame::reverseOffsetOfScratchValue();
+ masm.storePtr(obj, Address(FramePointer, scratchOffset));
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, obj);
+
+ // Restore |obj|. Because we entered a stub frame we first have to load
+ // the original frame pointer.
+ masm.loadPtr(Address(FramePointer, 0), obj);
+ masm.loadPtr(Address(obj, scratchOffset), obj);
+
+ masm.Push(Imm32(strict));
+ masm.Push(val);
+ masm.Push(idVal);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue, bool);
+ callVM<Fn, ProxySetPropertyByValue>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallAddOrUpdateSparseElementHelper(
+ ObjOperandId objId, Int32OperandId idId, ValOperandId rhsId, bool strict) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register id = allocator.useRegister(masm, idId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.Push(Imm32(strict));
+ masm.Push(val);
+ masm.Push(id);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext * cx, Handle<NativeObject*> obj, int32_t int_id,
+ HandleValue v, bool strict);
+ callVM<Fn, AddOrUpdateSparseElementHelper>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitMegamorphicSetElement(ObjOperandId objId,
+ ValOperandId idId,
+ ValOperandId rhsId,
+ bool strict) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand idVal = allocator.useValueRegister(masm, idId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+
+ // We need a scratch register but we don't have any registers available on
+ // x86, so temporarily store |obj| in the frame's scratch slot.
+ int scratchOffset = BaselineFrame::reverseOffsetOfScratchValue();
+ masm.storePtr(obj, Address(FramePointer, scratchOffset));
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, obj);
+
+ // Restore |obj|. Because we entered a stub frame we first have to load
+ // the original frame pointer.
+ masm.loadPtr(Address(FramePointer, 0), obj);
+ masm.loadPtr(Address(obj, scratchOffset), obj);
+
+ masm.Push(Imm32(strict));
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(obj)));
+ masm.Push(val);
+ masm.Push(idVal);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue,
+ HandleValue, bool);
+ callVM<Fn, SetElementMegamorphic>(masm);
+
+ stubFrame.leave(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitReturnFromIC() {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ allocator.discardStack(masm);
+ EmitReturnFromIC(masm);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitLoadArgumentFixedSlot(ValOperandId resultId,
+ uint8_t slotIndex) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ ValueOperand resultReg = allocator.defineValueRegister(masm, resultId);
+ Address addr = allocator.addressOf(masm, BaselineFrameSlot(slotIndex));
+ masm.loadValue(addr, resultReg);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitLoadArgumentDynamicSlot(ValOperandId resultId,
+ Int32OperandId argcId,
+ uint8_t slotIndex) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ ValueOperand resultReg = allocator.defineValueRegister(masm, resultId);
+ Register argcReg = allocator.useRegister(masm, argcId);
+ BaseValueIndex addr =
+ allocator.addressOf(masm, argcReg, BaselineFrameSlot(slotIndex));
+ masm.loadValue(addr, resultReg);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitGuardDOMExpandoMissingOrGuardShape(
+ ValOperandId expandoId, uint32_t shapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ ValueOperand val = allocator.useValueRegister(masm, expandoId);
+ AutoScratchRegister shapeScratch(allocator, masm);
+ AutoScratchRegister objScratch(allocator, masm);
+ Address shapeAddr(stubAddress(shapeOffset));
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label done;
+ masm.branchTestUndefined(Assembler::Equal, val, &done);
+
+ masm.debugAssertIsObject(val);
+ masm.loadPtr(shapeAddr, shapeScratch);
+ masm.unboxObject(val, objScratch);
+ // The expando object is not used in this case, so we don't need Spectre
+ // mitigations.
+ masm.branchTestObjShapeNoSpectreMitigations(Assembler::NotEqual, objScratch,
+ shapeScratch, failure->label());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitLoadDOMExpandoValueGuardGeneration(
+ ObjOperandId objId, uint32_t expandoAndGenerationOffset,
+ uint32_t generationOffset, ValOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Address expandoAndGenerationAddr(stubAddress(expandoAndGenerationOffset));
+ Address generationAddr(stubAddress(generationOffset));
+
+ AutoScratchRegister scratch(allocator, masm);
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), scratch);
+ Address expandoAddr(scratch,
+ js::detail::ProxyReservedSlots::offsetOfPrivateSlot());
+
+ // Load the ExpandoAndGeneration* in the output scratch register and guard
+ // it matches the proxy's ExpandoAndGeneration.
+ masm.loadPtr(expandoAndGenerationAddr, output.scratchReg());
+ masm.branchPrivatePtr(Assembler::NotEqual, expandoAddr, output.scratchReg(),
+ failure->label());
+
+ // Guard expandoAndGeneration->generation matches the expected generation.
+ masm.branch64(
+ Assembler::NotEqual,
+ Address(output.scratchReg(), ExpandoAndGeneration::offsetOfGeneration()),
+ generationAddr, scratch, failure->label());
+
+ // Load expandoAndGeneration->expando into the output Value register.
+ masm.loadValue(
+ Address(output.scratchReg(), ExpandoAndGeneration::offsetOfExpando()),
+ output);
+ return true;
+}
+
+bool BaselineCacheIRCompiler::init(CacheKind kind) {
+ if (!allocator.init()) {
+ return false;
+ }
+
+ size_t numInputs = writer_.numInputOperands();
+ MOZ_ASSERT(numInputs == NumInputsForCacheKind(kind));
+
+ // Baseline passes the first 2 inputs in R0/R1, other Values are stored on
+ // the stack.
+ size_t numInputsInRegs = std::min(numInputs, size_t(2));
+ AllocatableGeneralRegisterSet available =
+ BaselineICAvailableGeneralRegs(numInputsInRegs);
+
+ switch (kind) {
+ case CacheKind::NewArray:
+ case CacheKind::NewObject:
+ case CacheKind::GetIntrinsic:
+ MOZ_ASSERT(numInputs == 0);
+ outputUnchecked_.emplace(R0);
+ break;
+ case CacheKind::GetProp:
+ case CacheKind::TypeOf:
+ case CacheKind::ToPropertyKey:
+ case CacheKind::GetIterator:
+ case CacheKind::OptimizeSpreadCall:
+ case CacheKind::ToBool:
+ case CacheKind::UnaryArith:
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, R0);
+ outputUnchecked_.emplace(R0);
+ break;
+ case CacheKind::Compare:
+ case CacheKind::GetElem:
+ case CacheKind::GetPropSuper:
+ case CacheKind::In:
+ case CacheKind::HasOwn:
+ case CacheKind::CheckPrivateField:
+ case CacheKind::InstanceOf:
+ case CacheKind::BinaryArith:
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(0, R0);
+ allocator.initInputLocation(1, R1);
+ outputUnchecked_.emplace(R0);
+ break;
+ case CacheKind::SetProp:
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(0, R0);
+ allocator.initInputLocation(1, R1);
+ break;
+ case CacheKind::GetElemSuper:
+ MOZ_ASSERT(numInputs == 3);
+ allocator.initInputLocation(0, BaselineFrameSlot(0));
+ allocator.initInputLocation(1, R1);
+ allocator.initInputLocation(2, R0);
+ outputUnchecked_.emplace(R0);
+ break;
+ case CacheKind::SetElem:
+ MOZ_ASSERT(numInputs == 3);
+ allocator.initInputLocation(0, R0);
+ allocator.initInputLocation(1, R1);
+ allocator.initInputLocation(2, BaselineFrameSlot(0));
+ break;
+ case CacheKind::GetName:
+ case CacheKind::BindName:
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, R0.scratchReg(), JSVAL_TYPE_OBJECT);
+#if defined(JS_NUNBOX32)
+ // availableGeneralRegs can't know that GetName/BindName is only using
+ // the payloadReg and not typeReg on x86.
+ available.add(R0.typeReg());
+#endif
+ outputUnchecked_.emplace(R0);
+ break;
+ case CacheKind::Call:
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, R0.scratchReg(), JSVAL_TYPE_INT32);
+#if defined(JS_NUNBOX32)
+ // availableGeneralRegs can't know that Call is only using
+ // the payloadReg and not typeReg on x86.
+ available.add(R0.typeReg());
+#endif
+ outputUnchecked_.emplace(R0);
+ break;
+ case CacheKind::CloseIter:
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, R0.scratchReg(), JSVAL_TYPE_OBJECT);
+#if defined(JS_NUNBOX32)
+ // availableGeneralRegs can't know that CloseIter is only using
+ // the payloadReg and not typeReg on x86.
+ available.add(R0.typeReg());
+#endif
+ break;
+ }
+
+ // Baseline doesn't allocate float registers so none of them are live.
+ liveFloatRegs_ = LiveFloatRegisterSet(FloatRegisterSet());
+
+ allocator.initAvailableRegs(available);
+ return true;
+}
+
+static void ResetEnteredCounts(const ICEntry* icEntry) {
+ ICStub* stub = icEntry->firstStub();
+ while (true) {
+ stub->resetEnteredCount();
+ if (stub->isFallback()) {
+ return;
+ }
+ stub = stub->toCacheIRStub()->next();
+ }
+}
+
+static ICStubSpace* StubSpaceForStub(bool makesGCCalls, JSScript* script,
+ ICScript* icScript) {
+ if (makesGCCalls) {
+ return icScript->jitScriptStubSpace();
+ }
+ return script->zone()->jitZone()->optimizedStubSpace();
+}
+
+static const uint32_t MaxFoldedShapes = 16;
+
+bool js::jit::TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback,
+ JSScript* script, ICScript* icScript) {
+ ICEntry* icEntry = icScript->icEntryForStub(fallback);
+ ICStub* entryStub = icEntry->firstStub();
+
+ // Don't fold unless there are at least two stubs.
+ if (entryStub == fallback) {
+ return true;
+ }
+ ICCacheIRStub* firstStub = entryStub->toCacheIRStub();
+ if (firstStub->next()->isFallback()) {
+ return true;
+ }
+
+ const uint8_t* firstStubData = firstStub->stubDataStart();
+ const CacheIRStubInfo* stubInfo = firstStub->stubInfo();
+
+ // Check to see if:
+ // a) all of the stubs in this chain have the exact same code.
+ // b) all of the stubs have the same stub field data, except
+ // for a single GuardShape where they differ.
+ // c) at least one stub after the first has a non-zero entry count.
+ //
+ // If all of these conditions hold, then we generate a single stub
+ // that covers all the existing cases by replacing GuardShape with
+ // GuardMultipleShapes.
+
+ uint32_t numActive = 0;
+ Maybe<uint32_t> foldableFieldOffset;
+ RootedValue shape(cx);
+ RootedValueVector shapeList(cx);
+
+ auto addShape = [&shapeList, cx](uintptr_t rawShape) -> bool {
+ Shape* shape = reinterpret_cast<Shape*>(rawShape);
+ if (cx->compartment() != shape->compartment()) {
+ return false;
+ }
+ if (!shapeList.append(PrivateGCThingValue(shape))) {
+ cx->recoverFromOutOfMemory();
+ return false;
+ }
+ return true;
+ };
+
+ for (ICCacheIRStub* other = firstStub->nextCacheIR(); other;
+ other = other->nextCacheIR()) {
+ // Verify that the stubs share the same code.
+ if (other->stubInfo() != stubInfo) {
+ return true;
+ }
+ const uint8_t* otherStubData = other->stubDataStart();
+
+ if (other->enteredCount() > 0) {
+ numActive++;
+ }
+
+ uint32_t fieldIndex = 0;
+ size_t offset = 0;
+ while (stubInfo->fieldType(fieldIndex) != StubField::Type::Limit) {
+ StubField::Type fieldType = stubInfo->fieldType(fieldIndex);
+
+ if (StubField::sizeIsWord(fieldType)) {
+ uintptr_t firstRaw = stubInfo->getStubRawWord(firstStubData, offset);
+ uintptr_t otherRaw = stubInfo->getStubRawWord(otherStubData, offset);
+
+ if (firstRaw != otherRaw) {
+ if (fieldType != StubField::Type::Shape) {
+ // Case 1: a field differs that is not a Shape. We only support
+ // folding GuardShape to GuardMultipleShapes.
+ return true;
+ }
+ if (foldableFieldOffset.isNothing()) {
+ // Case 2: this is the first field where the stub data differs.
+ foldableFieldOffset.emplace(offset);
+ if (!addShape(firstRaw) || !addShape(otherRaw)) {
+ return true;
+ }
+ } else if (*foldableFieldOffset == offset) {
+ // Case 3: this is the corresponding offset in a different stub.
+ if (!addShape(otherRaw)) {
+ return true;
+ }
+ } else {
+ // Case 4: we have found more than one field that differs.
+ return true;
+ }
+ }
+ } else {
+ MOZ_ASSERT(StubField::sizeIsInt64(fieldType));
+
+ // We do not support folding any ops with int64-sized fields.
+ if (stubInfo->getStubRawInt64(firstStubData, offset) !=
+ stubInfo->getStubRawInt64(otherStubData, offset)) {
+ return true;
+ }
+ }
+
+ offset += StubField::sizeInBytes(fieldType);
+ fieldIndex++;
+ }
+
+ // We should never attach two completely identical stubs.
+ MOZ_ASSERT(foldableFieldOffset.isSome());
+ }
+
+ if (numActive == 0) {
+ return true;
+ }
+
+ // Clone the CacheIR, replacing GuardShape with GuardMultipleShapes.
+ CacheIRWriter writer(cx);
+ CacheIRReader reader(stubInfo);
+ CacheIRCloner cloner(firstStub);
+
+ // Initialize the operands.
+ CacheKind cacheKind = stubInfo->kind();
+ for (uint32_t i = 0; i < NumInputsForCacheKind(cacheKind); i++) {
+ writer.setInputOperandId(i);
+ }
+
+ bool success = false;
+ while (reader.more()) {
+ CacheOp op = reader.readOp();
+ switch (op) {
+ case CacheOp::GuardShape: {
+ ObjOperandId objId = reader.objOperandId();
+ uint32_t shapeOffset = reader.stubOffset();
+ if (shapeOffset == *foldableFieldOffset) {
+ // Ensure that the allocation of the ListObject doesn't trigger a GC
+ // and free the stubInfo we're currently reading. Note that
+ // AutoKeepJitScripts isn't sufficient, because optimized stubs can be
+ // discarded even if the JitScript is preserved.
+ gc::AutoSuppressGC suppressGC(cx);
+
+ Rooted<ListObject*> shapeObj(cx, ListObject::create(cx));
+ if (!shapeObj) {
+ return false;
+ }
+ for (uint32_t i = 0; i < shapeList.length(); i++) {
+ if (!shapeObj->append(cx, shapeList[i])) {
+ cx->recoverFromOutOfMemory();
+ return false;
+ }
+ }
+
+ writer.guardMultipleShapes(objId, shapeObj);
+ success = true;
+ } else {
+ Shape* shape = stubInfo->getStubField<Shape*>(firstStub, shapeOffset);
+ writer.guardShape(objId, shape);
+ }
+ break;
+ }
+ default:
+ cloner.cloneOp(op, reader, writer);
+ break;
+ }
+ }
+ if (!success) {
+ // If the shape field that differed was not part of a GuardShape,
+ // we can't fold these stubs together.
+ return true;
+ }
+
+ // Replace the existing stubs with the new folded stub.
+ fallback->discardStubs(cx, icEntry);
+
+ ICAttachResult result = AttachBaselineCacheIRStub(
+ cx, writer, cacheKind, script, icScript, fallback, "StubFold");
+ if (result == ICAttachResult::OOM) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+ MOZ_ASSERT(result == ICAttachResult::Attached);
+
+ fallback->setHasFoldedStub();
+ return true;
+}
+
+static bool AddToFoldedStub(JSContext* cx, const CacheIRWriter& writer,
+ ICScript* icScript, ICFallbackStub* fallback) {
+ ICEntry* icEntry = icScript->icEntryForStub(fallback);
+ ICStub* entryStub = icEntry->firstStub();
+
+ // We only update folded stubs if they're the only stub in the IC.
+ if (entryStub == fallback) {
+ return false;
+ }
+ ICCacheIRStub* stub = entryStub->toCacheIRStub();
+ if (!stub->next()->isFallback()) {
+ return false;
+ }
+
+ const CacheIRStubInfo* stubInfo = stub->stubInfo();
+ const uint8_t* stubData = stub->stubDataStart();
+
+ Maybe<uint32_t> shapeFieldOffset;
+ RootedValue newShape(cx);
+ Rooted<ListObject*> foldedShapes(cx);
+
+ CacheIRReader stubReader(stubInfo);
+ CacheIRReader newReader(writer);
+ while (newReader.more() && stubReader.more()) {
+ CacheOp newOp = newReader.readOp();
+ CacheOp stubOp = stubReader.readOp();
+ switch (stubOp) {
+ case CacheOp::GuardMultipleShapes: {
+ // Check that the new stub has a corresponding GuardShape.
+ if (newOp != CacheOp::GuardShape) {
+ return false;
+ }
+
+ // Check that the object being guarded is the same.
+ if (newReader.objOperandId() != stubReader.objOperandId()) {
+ return false;
+ }
+
+ // Check that the field offset is the same.
+ uint32_t newShapeOffset = newReader.stubOffset();
+ uint32_t stubShapesOffset = stubReader.stubOffset();
+ if (newShapeOffset != stubShapesOffset) {
+ return false;
+ }
+ MOZ_ASSERT(shapeFieldOffset.isNothing());
+ shapeFieldOffset.emplace(newShapeOffset);
+
+ // Get the shape from the new stub
+ StubField shapeField =
+ writer.readStubField(newShapeOffset, StubField::Type::Shape);
+ Shape* shape = reinterpret_cast<Shape*>(shapeField.asWord());
+ newShape = PrivateGCThingValue(shape);
+
+ // Get the shape array from the old stub.
+ JSObject* shapeList =
+ stubInfo->getStubField<JSObject*>(stub, stubShapesOffset);
+ foldedShapes = &shapeList->as<ListObject>();
+ MOZ_ASSERT(foldedShapes->compartment() == shape->compartment());
+ break;
+ }
+ default: {
+ // Check that the op is the same.
+ if (newOp != stubOp) {
+ return false;
+ }
+
+ // Check that the arguments are the same.
+ uint32_t argLength = CacheIROpInfos[size_t(newOp)].argLength;
+ for (uint32_t i = 0; i < argLength; i++) {
+ if (newReader.readByte() != stubReader.readByte()) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+
+ MOZ_ASSERT(shapeFieldOffset.isSome());
+
+ // Check to verify that all the other stub fields are the same.
+ if (!writer.stubDataEqualsIgnoring(stubData, *shapeFieldOffset)) {
+ return false;
+ }
+
+ // Limit the maximum number of shapes we will add before giving up.
+ if (foldedShapes->length() == MaxFoldedShapes) {
+ return false;
+ }
+
+ if (!foldedShapes->append(cx, newShape)) {
+ cx->recoverFromOutOfMemory();
+ return false;
+ }
+
+ return true;
+}
+
+ICAttachResult js::jit::AttachBaselineCacheIRStub(
+ JSContext* cx, const CacheIRWriter& writer, CacheKind kind,
+ JSScript* outerScript, ICScript* icScript, ICFallbackStub* stub,
+ const char* name) {
+ // We shouldn't GC or report OOM (or any other exception) here.
+ AutoAssertNoPendingException aanpe(cx);
+ JS::AutoCheckCannotGC nogc;
+
+ if (writer.tooLarge()) {
+ return ICAttachResult::TooLarge;
+ }
+ if (writer.oom()) {
+ return ICAttachResult::OOM;
+ }
+ MOZ_ASSERT(!writer.failed());
+
+ // Just a sanity check: the caller should ensure we don't attach an
+ // unlimited number of stubs.
+#ifdef DEBUG
+ static const size_t MaxOptimizedCacheIRStubs = 16;
+ MOZ_ASSERT(stub->numOptimizedStubs() < MaxOptimizedCacheIRStubs);
+#endif
+
+ constexpr uint32_t stubDataOffset = sizeof(ICCacheIRStub);
+ static_assert(stubDataOffset % sizeof(uint64_t) == 0,
+ "Stub fields must be aligned");
+
+ JitZone* jitZone = cx->zone()->jitZone();
+
+ // Check if we already have JitCode for this stub.
+ CacheIRStubInfo* stubInfo;
+ CacheIRStubKey::Lookup lookup(kind, ICStubEngine::Baseline,
+ writer.codeStart(), writer.codeLength());
+ JitCode* code = jitZone->getBaselineCacheIRStubCode(lookup, &stubInfo);
+ if (!code) {
+ // We have to generate stub code.
+ TempAllocator temp(&cx->tempLifoAlloc());
+ JitContext jctx(cx);
+ BaselineCacheIRCompiler comp(cx, temp, writer, stubDataOffset);
+ if (!comp.init(kind)) {
+ return ICAttachResult::OOM;
+ }
+
+ code = comp.compile();
+ if (!code) {
+ return ICAttachResult::OOM;
+ }
+
+ comp.perfSpewer().saveProfile(code, name);
+
+ // Allocate the shared CacheIRStubInfo. Note that the
+ // putBaselineCacheIRStubCode call below will transfer ownership
+ // to the stub code HashMap, so we don't have to worry about freeing
+ // it below.
+ MOZ_ASSERT(!stubInfo);
+ stubInfo =
+ CacheIRStubInfo::New(kind, ICStubEngine::Baseline, comp.makesGCCalls(),
+ stubDataOffset, writer);
+ if (!stubInfo) {
+ return ICAttachResult::OOM;
+ }
+
+ CacheIRStubKey key(stubInfo);
+ if (!jitZone->putBaselineCacheIRStubCode(lookup, key, code)) {
+ return ICAttachResult::OOM;
+ }
+ }
+
+ MOZ_ASSERT(code);
+ MOZ_ASSERT(stubInfo);
+ MOZ_ASSERT(stubInfo->stubDataSize() == writer.stubDataSize());
+
+ ICEntry* icEntry = icScript->icEntryForStub(stub);
+
+ // Ensure we don't attach duplicate stubs. This can happen if a stub failed
+ // for some reason and the IR generator doesn't check for exactly the same
+ // conditions.
+ for (ICStub* iter = icEntry->firstStub(); iter != stub;
+ iter = iter->toCacheIRStub()->next()) {
+ auto otherStub = iter->toCacheIRStub();
+ if (otherStub->stubInfo() != stubInfo) {
+ continue;
+ }
+ if (!writer.stubDataEquals(otherStub->stubDataStart())) {
+ continue;
+ }
+
+ // We found a stub that's exactly the same as the stub we're about to
+ // attach. Just return nullptr, the caller should do nothing in this
+ // case.
+ JitSpew(JitSpew_BaselineICFallback,
+ "Tried attaching identical stub for (%s:%u:%u)",
+ outerScript->filename(), outerScript->lineno(),
+ outerScript->column());
+ return ICAttachResult::DuplicateStub;
+ }
+
+ // Try including this case in an existing folded stub.
+ if (stub->hasFoldedStub() && AddToFoldedStub(cx, writer, icScript, stub)) {
+ // Instead of adding a new stub, we have added a new case to an
+ // existing folded stub. We do not have to invalidate Warp,
+ // because the ListObject that stores the cases is shared between
+ // baseline and Warp. Reset the entered count for the fallback
+ // stub so that we can still transpile, and reset the bailout
+ // counter if we have already been transpiled.
+ stub->resetEnteredCount();
+ JSScript* owningScript = icScript->isInlined()
+ ? icScript->inliningRoot()->owningScript()
+ : outerScript;
+ if (stub->usedByTranspiler() && owningScript->hasIonScript()) {
+ owningScript->ionScript()->resetNumFixableBailouts();
+ }
+ return ICAttachResult::Attached;
+ }
+
+ // Time to allocate and attach a new stub.
+
+ size_t bytesNeeded = stubInfo->stubDataOffset() + stubInfo->stubDataSize();
+
+ ICStubSpace* stubSpace =
+ StubSpaceForStub(stubInfo->makesGCCalls(), outerScript, icScript);
+ void* newStubMem = stubSpace->alloc(bytesNeeded);
+ if (!newStubMem) {
+ return ICAttachResult::OOM;
+ }
+
+ // Resetting the entered counts on the IC chain makes subsequent reasoning
+ // about the chain much easier.
+ ResetEnteredCounts(icEntry);
+
+ switch (stub->trialInliningState()) {
+ case TrialInliningState::Initial:
+ case TrialInliningState::Candidate:
+ stub->setTrialInliningState(writer.trialInliningState());
+ break;
+ case TrialInliningState::Inlined:
+ stub->setTrialInliningState(TrialInliningState::Failure);
+ break;
+ case TrialInliningState::Failure:
+ break;
+ }
+
+ auto newStub = new (newStubMem) ICCacheIRStub(code, stubInfo);
+ writer.copyStubData(newStub->stubDataStart());
+ newStub->setTypeData(writer.typeData());
+ stub->addNewStub(icEntry, newStub);
+ return ICAttachResult::Attached;
+}
+
+uint8_t* ICCacheIRStub::stubDataStart() {
+ return reinterpret_cast<uint8_t*>(this) + stubInfo_->stubDataOffset();
+}
+
+bool BaselineCacheIRCompiler::emitCallStringObjectConcatResult(
+ ValOperandId lhsId, ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ ValueOperand lhs = allocator.useValueRegister(masm, lhsId);
+ ValueOperand rhs = allocator.useValueRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+
+ // For the expression decompiler
+ EmitRestoreTailCallReg(masm);
+ masm.pushValue(lhs);
+ masm.pushValue(rhs);
+
+ masm.pushValue(rhs);
+ masm.pushValue(lhs);
+
+ using Fn = bool (*)(JSContext*, HandleValue, HandleValue, MutableHandleValue);
+ tailCallVM<Fn, DoConcatStringObject>(masm);
+
+ return true;
+}
+
+// The value of argc entering the call IC is not always the value of
+// argc entering the callee. (For example, argc for a spread call IC
+// is always 1, but argc for the callee is the length of the array.)
+// In these cases, we update argc as part of the call op itself, to
+// avoid modifying input operands while it is still possible to fail a
+// guard. We also limit callee argc to a reasonable value to avoid
+// blowing the stack limit.
+bool BaselineCacheIRCompiler::updateArgc(CallFlags flags, Register argcReg,
+ Register scratch) {
+ CallFlags::ArgFormat format = flags.getArgFormat();
+ switch (format) {
+ case CallFlags::Standard:
+ // Standard calls have no extra guards, and argc is already correct.
+ return true;
+ case CallFlags::FunCall:
+ // fun_call has no extra guards, and argc will be corrected in
+ // pushFunCallArguments.
+ return true;
+ default:
+ break;
+ }
+
+ // We need to guard the length of the arguments.
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load callee argc into scratch.
+ switch (flags.getArgFormat()) {
+ case CallFlags::Spread:
+ case CallFlags::FunApplyArray: {
+ // Load the length of the elements.
+ BaselineFrameSlot slot(flags.isConstructing());
+ masm.unboxObject(allocator.addressOf(masm, slot), scratch);
+ masm.loadPtr(Address(scratch, NativeObject::offsetOfElements()), scratch);
+ masm.load32(Address(scratch, ObjectElements::offsetOfLength()), scratch);
+ break;
+ }
+ case CallFlags::FunApplyArgsObj: {
+ // Load the arguments object length.
+ BaselineFrameSlot slot(0);
+ masm.unboxObject(allocator.addressOf(masm, slot), scratch);
+ masm.loadArgumentsObjectLength(scratch, scratch, failure->label());
+ break;
+ }
+ default:
+ MOZ_CRASH("Unknown arg format");
+ }
+
+ // Ensure that callee argc does not exceed the limit.
+ masm.branch32(Assembler::Above, scratch, Imm32(JIT_ARGS_LENGTH_MAX),
+ failure->label());
+
+ // We're past the final guard. Update argc with the new value.
+ masm.move32(scratch, argcReg);
+
+ return true;
+}
+
+void BaselineCacheIRCompiler::pushArguments(Register argcReg,
+ Register calleeReg,
+ Register scratch, Register scratch2,
+ CallFlags flags, uint32_t argcFixed,
+ bool isJitCall) {
+ switch (flags.getArgFormat()) {
+ case CallFlags::Standard:
+ pushStandardArguments(argcReg, scratch, scratch2, argcFixed, isJitCall,
+ flags.isConstructing());
+ break;
+ case CallFlags::Spread:
+ pushArrayArguments(argcReg, scratch, scratch2, isJitCall,
+ flags.isConstructing());
+ break;
+ case CallFlags::FunCall:
+ pushFunCallArguments(argcReg, calleeReg, scratch, scratch2, argcFixed,
+ isJitCall);
+ break;
+ case CallFlags::FunApplyArgsObj:
+ pushFunApplyArgsObj(argcReg, calleeReg, scratch, scratch2, isJitCall);
+ break;
+ case CallFlags::FunApplyArray:
+ pushArrayArguments(argcReg, scratch, scratch2, isJitCall,
+ /*isConstructing =*/false);
+ break;
+ default:
+ MOZ_CRASH("Invalid arg format");
+ }
+}
+
+void BaselineCacheIRCompiler::pushStandardArguments(
+ Register argcReg, Register scratch, Register scratch2, uint32_t argcFixed,
+ bool isJitCall, bool isConstructing) {
+ // The arguments to the call IC are pushed on the stack left-to-right.
+ // Our calling conventions want them right-to-left in the callee, so
+ // we duplicate them on the stack in reverse order.
+
+ int additionalArgc = 1 + !isJitCall + isConstructing;
+ if (argcFixed < MaxUnrolledArgCopy) {
+#ifdef DEBUG
+ Label ok;
+ masm.branch32(Assembler::Equal, argcReg, Imm32(argcFixed), &ok);
+ masm.assumeUnreachable("Invalid argcFixed value");
+ masm.bind(&ok);
+#endif
+
+ size_t realArgc = argcFixed + additionalArgc;
+
+ if (isJitCall) {
+ masm.alignJitStackBasedOnNArgs(realArgc, /*countIncludesThis = */ true);
+ }
+
+ for (size_t i = 0; i < realArgc; ++i) {
+ masm.pushValue(Address(
+ FramePointer, BaselineStubFrameLayout::Size() + i * sizeof(Value)));
+ }
+ } else {
+ MOZ_ASSERT(argcFixed == MaxUnrolledArgCopy);
+
+ // argPtr initially points to the last argument. Skip the stub frame.
+ Register argPtr = scratch2;
+ Address argAddress(FramePointer, BaselineStubFrameLayout::Size());
+ masm.computeEffectiveAddress(argAddress, argPtr);
+
+ // countReg contains the total number of arguments to copy.
+ // In addition to the actual arguments, we have to copy hidden arguments.
+ // We always have to copy |this|.
+ // If we are constructing, we have to copy |newTarget|.
+ // If we are not a jit call, we have to copy |callee|.
+ // We use a scratch register to avoid clobbering argc, which is an input
+ // reg.
+ Register countReg = scratch;
+ masm.move32(argcReg, countReg);
+ masm.add32(Imm32(additionalArgc), countReg);
+
+ // Align the stack such that the JitFrameLayout is aligned on the
+ // JitStackAlignment.
+ if (isJitCall) {
+ masm.alignJitStackBasedOnNArgs(countReg, /*countIncludesThis = */ true);
+ }
+
+ // Push all values, starting at the last one.
+ Label loop, done;
+ masm.branchTest32(Assembler::Zero, countReg, countReg, &done);
+ masm.bind(&loop);
+ {
+ masm.pushValue(Address(argPtr, 0));
+ masm.addPtr(Imm32(sizeof(Value)), argPtr);
+
+ masm.branchSub32(Assembler::NonZero, Imm32(1), countReg, &loop);
+ }
+ masm.bind(&done);
+ }
+}
+
+void BaselineCacheIRCompiler::pushArrayArguments(Register argcReg,
+ Register scratch,
+ Register scratch2,
+ bool isJitCall,
+ bool isConstructing) {
+ // Pull the array off the stack before aligning.
+ Register startReg = scratch;
+ size_t arrayOffset =
+ (isConstructing * sizeof(Value)) + BaselineStubFrameLayout::Size();
+ masm.unboxObject(Address(FramePointer, arrayOffset), startReg);
+ masm.loadPtr(Address(startReg, NativeObject::offsetOfElements()), startReg);
+
+ // Align the stack such that the JitFrameLayout is aligned on the
+ // JitStackAlignment.
+ if (isJitCall) {
+ Register alignReg = argcReg;
+ if (isConstructing) {
+ // If we are constructing, we must take newTarget into account.
+ alignReg = scratch2;
+ masm.computeEffectiveAddress(Address(argcReg, 1), alignReg);
+ }
+ masm.alignJitStackBasedOnNArgs(alignReg, /*countIncludesThis =*/false);
+ }
+
+ // Push newTarget, if necessary
+ if (isConstructing) {
+ masm.pushValue(Address(FramePointer, BaselineStubFrameLayout::Size()));
+ }
+
+ // Push arguments: set up endReg to point to &array[argc]
+ Register endReg = scratch2;
+ BaseValueIndex endAddr(startReg, argcReg);
+ masm.computeEffectiveAddress(endAddr, endReg);
+
+ // Copying pre-decrements endReg by 8 until startReg is reached
+ Label copyDone;
+ Label copyStart;
+ masm.bind(&copyStart);
+ masm.branchPtr(Assembler::Equal, endReg, startReg, &copyDone);
+ masm.subPtr(Imm32(sizeof(Value)), endReg);
+ masm.pushValue(Address(endReg, 0));
+ masm.jump(&copyStart);
+ masm.bind(&copyDone);
+
+ // Push |this|.
+ size_t thisvOffset =
+ BaselineStubFrameLayout::Size() + (1 + isConstructing) * sizeof(Value);
+ masm.pushValue(Address(FramePointer, thisvOffset));
+
+ // Push |callee| if needed.
+ if (!isJitCall) {
+ size_t calleeOffset =
+ BaselineStubFrameLayout::Size() + (2 + isConstructing) * sizeof(Value);
+ masm.pushValue(Address(FramePointer, calleeOffset));
+ }
+}
+
+void BaselineCacheIRCompiler::pushFunCallArguments(
+ Register argcReg, Register calleeReg, Register scratch, Register scratch2,
+ uint32_t argcFixed, bool isJitCall) {
+ if (argcFixed == 0) {
+ if (isJitCall) {
+ // Align the stack to 0 args.
+ masm.alignJitStackBasedOnNArgs(0, /*countIncludesThis = */ false);
+ }
+
+ // Store the new |this|.
+ masm.pushValue(UndefinedValue());
+
+ // Store |callee| if needed.
+ if (!isJitCall) {
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(calleeReg)));
+ }
+ } else if (argcFixed < MaxUnrolledArgCopy) {
+ // See below for why we subtract 1 from argcFixed.
+ argcFixed -= 1;
+ masm.sub32(Imm32(1), argcReg);
+ pushStandardArguments(argcReg, scratch, scratch2, argcFixed, isJitCall,
+ /*isConstructing =*/false);
+ } else {
+ Label zeroArgs, done;
+ masm.branchTest32(Assembler::Zero, argcReg, argcReg, &zeroArgs);
+
+ // When we call fun_call, the stack looks like the left column (note
+ // that newTarget will not be present, because fun_call cannot be a
+ // constructor call):
+ //
+ // ***Arguments to fun_call***
+ // callee (fun_call) ***Arguments to target***
+ // this (target function) -----> callee
+ // arg0 (this of target) -----> this
+ // arg1 (arg0 of target) -----> arg0
+ // argN (argN-1 of target) -----> arg1
+ //
+ // As demonstrated in the right column, this is exactly what we need
+ // the stack to look like when calling pushStandardArguments for target,
+ // except with one more argument. If we subtract 1 from argc,
+ // everything works out correctly.
+ masm.sub32(Imm32(1), argcReg);
+
+ pushStandardArguments(argcReg, scratch, scratch2, argcFixed, isJitCall,
+ /*isConstructing =*/false);
+
+ masm.jump(&done);
+ masm.bind(&zeroArgs);
+
+ // The exception is the case where argc == 0:
+ //
+ // ***Arguments to fun_call***
+ // callee (fun_call) ***Arguments to target***
+ // this (target function) -----> callee
+ // <nothing> -----> this
+ //
+ // In this case, we push |undefined| for |this|.
+
+ if (isJitCall) {
+ // Align the stack to 0 args.
+ masm.alignJitStackBasedOnNArgs(0, /*countIncludesThis = */ false);
+ }
+
+ // Store the new |this|.
+ masm.pushValue(UndefinedValue());
+
+ // Store |callee| if needed.
+ if (!isJitCall) {
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(calleeReg)));
+ }
+
+ masm.bind(&done);
+ }
+}
+
+void BaselineCacheIRCompiler::pushFunApplyArgsObj(Register argcReg,
+ Register calleeReg,
+ Register scratch,
+ Register scratch2,
+ bool isJitCall) {
+ // Load the arguments object off the stack before aligning.
+ Register argsReg = scratch;
+ masm.unboxObject(Address(FramePointer, BaselineStubFrameLayout::Size()),
+ argsReg);
+
+ // Align the stack such that the JitFrameLayout is aligned on the
+ // JitStackAlignment.
+ if (isJitCall) {
+ masm.alignJitStackBasedOnNArgs(argcReg, /*countIncludesThis =*/false);
+ }
+
+ // Load ArgumentsData.
+ masm.loadPrivate(Address(argsReg, ArgumentsObject::getDataSlotOffset()),
+ argsReg);
+
+ // We push the arguments onto the stack last-to-first.
+ // Compute the bounds of the arguments array.
+ Register currReg = scratch2;
+ Address argsStartAddr(argsReg, ArgumentsData::offsetOfArgs());
+ masm.computeEffectiveAddress(argsStartAddr, argsReg);
+ BaseValueIndex argsEndAddr(argsReg, argcReg);
+ masm.computeEffectiveAddress(argsEndAddr, currReg);
+
+ // Loop until all arguments have been pushed.
+ Label done, loop;
+ masm.bind(&loop);
+ masm.branchPtr(Assembler::Equal, currReg, argsReg, &done);
+ masm.subPtr(Imm32(sizeof(Value)), currReg);
+
+ Address currArgAddr(currReg, 0);
+#ifdef DEBUG
+ // Arguments are forwarded to the call object if they are closed over.
+ // In this case, OVERRIDDEN_ELEMENTS_BIT should be set.
+ Label notForwarded;
+ masm.branchTestMagic(Assembler::NotEqual, currArgAddr, &notForwarded);
+ masm.assumeUnreachable("Should have checked for overridden elements");
+ masm.bind(&notForwarded);
+#endif
+ masm.pushValue(currArgAddr);
+
+ masm.jump(&loop);
+ masm.bind(&done);
+
+ // Push arg0 as |this| for call
+ masm.pushValue(
+ Address(FramePointer, BaselineStubFrameLayout::Size() + sizeof(Value)));
+
+ // Push |callee| if needed.
+ if (!isJitCall) {
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(calleeReg)));
+ }
+}
+
+bool BaselineCacheIRCompiler::emitCallNativeShared(
+ NativeCallType callType, ObjOperandId calleeId, Int32OperandId argcId,
+ CallFlags flags, uint32_t argcFixed, Maybe<bool> ignoresReturnValue,
+ Maybe<uint32_t> targetOffset) {
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ Register calleeReg = allocator.useRegister(masm, calleeId);
+ Register argcReg = allocator.useRegister(masm, argcId);
+
+ bool isConstructing = flags.isConstructing();
+ bool isSameRealm = flags.isSameRealm();
+
+ if (!updateArgc(flags, argcReg, scratch)) {
+ return false;
+ }
+
+ allocator.discardStack(masm);
+
+ // Push a stub frame so that we can perform a non-tail call.
+ // Note that this leaves the return address in TailCallReg.
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ if (!isSameRealm) {
+ masm.switchToObjectRealm(calleeReg, scratch);
+ }
+
+ pushArguments(argcReg, calleeReg, scratch, scratch2, flags, argcFixed,
+ /*isJitCall =*/false);
+
+ // Native functions have the signature:
+ //
+ // bool (*)(JSContext*, unsigned, Value* vp)
+ //
+ // Where vp[0] is space for callee/return value, vp[1] is |this|, and vp[2]
+ // onward are the function arguments.
+
+ // Initialize vp.
+ masm.moveStackPtrTo(scratch2.get());
+
+ // Construct a native exit frame.
+ masm.push(argcReg);
+
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.push(ICTailCallReg);
+ masm.push(FramePointer);
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrameForNative(scratch, scratch, isConstructing);
+
+ // Execute call.
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(argcReg);
+ masm.passABIArg(scratch2);
+
+ switch (callType) {
+ case NativeCallType::Native: {
+#ifdef JS_SIMULATOR
+ // The simulator requires VM calls to be redirected to a special
+ // swi instruction to handle them, so we store the redirected
+ // pointer in the stub and use that instead of the original one.
+ // (See CacheIRWriter::callNativeFunction.)
+ Address redirectedAddr(stubAddress(*targetOffset));
+ masm.callWithABI(redirectedAddr);
+#else
+ if (*ignoresReturnValue) {
+ masm.loadPrivate(
+ Address(calleeReg, JSFunction::offsetOfJitInfoOrScript()),
+ calleeReg);
+ masm.callWithABI(
+ Address(calleeReg, JSJitInfo::offsetOfIgnoresReturnValueNative()));
+ } else {
+ // This depends on the native function pointer being stored unchanged as
+ // a PrivateValue.
+ masm.callWithABI(Address(calleeReg, JSFunction::offsetOfNativeOrEnv()));
+ }
+#endif
+ } break;
+ case NativeCallType::ClassHook: {
+ Address nativeAddr(stubAddress(*targetOffset));
+ masm.callWithABI(nativeAddr);
+ } break;
+ }
+
+ // Test for failure.
+ masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
+
+ // Load the return value.
+ masm.loadValue(
+ Address(masm.getStackPointer(), NativeExitFrameLayout::offsetOfResult()),
+ output.valueReg());
+
+ stubFrame.leave(masm);
+
+ if (!isSameRealm) {
+ masm.switchToBaselineFrameRealm(scratch2);
+ }
+
+ return true;
+}
+
+#ifdef JS_SIMULATOR
+bool BaselineCacheIRCompiler::emitCallNativeFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed,
+ uint32_t targetOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<bool> ignoresReturnValue;
+ Maybe<uint32_t> targetOffset_ = mozilla::Some(targetOffset);
+ return emitCallNativeShared(NativeCallType::Native, calleeId, argcId, flags,
+ argcFixed, ignoresReturnValue, targetOffset_);
+}
+
+bool BaselineCacheIRCompiler::emitCallDOMFunction(
+ ObjOperandId calleeId, Int32OperandId argcId, ObjOperandId thisObjId,
+ CallFlags flags, uint32_t argcFixed, uint32_t targetOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<bool> ignoresReturnValue;
+ Maybe<uint32_t> targetOffset_ = mozilla::Some(targetOffset);
+ return emitCallNativeShared(NativeCallType::Native, calleeId, argcId, flags,
+ argcFixed, ignoresReturnValue, targetOffset_);
+}
+#else
+bool BaselineCacheIRCompiler::emitCallNativeFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed,
+ bool ignoresReturnValue) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<bool> ignoresReturnValue_ = mozilla::Some(ignoresReturnValue);
+ Maybe<uint32_t> targetOffset;
+ return emitCallNativeShared(NativeCallType::Native, calleeId, argcId, flags,
+ argcFixed, ignoresReturnValue_, targetOffset);
+}
+
+bool BaselineCacheIRCompiler::emitCallDOMFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ ObjOperandId thisObjId,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<bool> ignoresReturnValue = mozilla::Some(false);
+ Maybe<uint32_t> targetOffset;
+ return emitCallNativeShared(NativeCallType::Native, calleeId, argcId, flags,
+ argcFixed, ignoresReturnValue, targetOffset);
+}
+#endif
+
+bool BaselineCacheIRCompiler::emitCallClassHook(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed,
+ uint32_t targetOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<bool> ignoresReturnValue;
+ Maybe<uint32_t> targetOffset_ = mozilla::Some(targetOffset);
+ return emitCallNativeShared(NativeCallType::ClassHook, calleeId, argcId,
+ flags, argcFixed, ignoresReturnValue,
+ targetOffset_);
+}
+
+// Helper function for loading call arguments from the stack. Loads
+// and unboxes an object from a specific slot.
+void BaselineCacheIRCompiler::loadStackObject(ArgumentKind kind,
+ CallFlags flags, Register argcReg,
+ Register dest) {
+ MOZ_ASSERT(enteredStubFrame_);
+
+ bool addArgc = false;
+ int32_t slotIndex = GetIndexOfArgument(kind, flags, &addArgc);
+
+ if (addArgc) {
+ int32_t slotOffset =
+ slotIndex * sizeof(JS::Value) + BaselineStubFrameLayout::Size();
+ BaseValueIndex slotAddr(FramePointer, argcReg, slotOffset);
+ masm.unboxObject(slotAddr, dest);
+ } else {
+ int32_t slotOffset =
+ slotIndex * sizeof(JS::Value) + BaselineStubFrameLayout::Size();
+ Address slotAddr(FramePointer, slotOffset);
+ masm.unboxObject(slotAddr, dest);
+ }
+}
+
+template <typename T>
+void BaselineCacheIRCompiler::storeThis(const T& newThis, Register argcReg,
+ CallFlags flags) {
+ switch (flags.getArgFormat()) {
+ case CallFlags::Standard: {
+ BaseValueIndex thisAddress(
+ FramePointer,
+ argcReg, // Arguments
+ 1 * sizeof(Value) + // NewTarget
+ BaselineStubFrameLayout::Size()); // Stub frame
+ masm.storeValue(newThis, thisAddress);
+ } break;
+ case CallFlags::Spread: {
+ Address thisAddress(FramePointer,
+ 2 * sizeof(Value) + // Arg array, NewTarget
+ BaselineStubFrameLayout::Size()); // Stub frame
+ masm.storeValue(newThis, thisAddress);
+ } break;
+ default:
+ MOZ_CRASH("Invalid arg format for scripted constructor");
+ }
+}
+
+/*
+ * Scripted constructors require a |this| object to be created prior to the
+ * call. When this function is called, the stack looks like (bottom->top):
+ *
+ * [..., Callee, ThisV, Arg0V, ..., ArgNV, NewTarget, StubFrameHeader]
+ *
+ * At this point, |ThisV| is JSWhyMagic::JS_IS_CONSTRUCTING.
+ *
+ * This function calls CreateThis to generate a new |this| object, then
+ * overwrites the magic ThisV on the stack.
+ */
+void BaselineCacheIRCompiler::createThis(Register argcReg, Register calleeReg,
+ Register scratch, CallFlags flags) {
+ MOZ_ASSERT(flags.isConstructing());
+
+ if (flags.needsUninitializedThis()) {
+ storeThis(MagicValue(JS_UNINITIALIZED_LEXICAL), argcReg, flags);
+ return;
+ }
+
+ // Save live registers that don't have to be traced.
+ LiveGeneralRegisterSet liveNonGCRegs;
+ liveNonGCRegs.add(argcReg);
+ liveNonGCRegs.add(ICStubReg);
+ masm.PushRegsInMask(liveNonGCRegs);
+
+ // CreateThis takes two arguments: callee, and newTarget.
+
+ // Push newTarget:
+ loadStackObject(ArgumentKind::NewTarget, flags, argcReg, scratch);
+ masm.push(scratch);
+
+ // Push callee:
+ loadStackObject(ArgumentKind::Callee, flags, argcReg, scratch);
+ masm.push(scratch);
+
+ // Call CreateThisFromIC.
+ using Fn =
+ bool (*)(JSContext*, HandleObject, HandleObject, MutableHandleValue);
+ callVM<Fn, CreateThisFromIC>(masm);
+
+#ifdef DEBUG
+ Label createdThisOK;
+ masm.branchTestObject(Assembler::Equal, JSReturnOperand, &createdThisOK);
+ masm.branchTestMagic(Assembler::Equal, JSReturnOperand, &createdThisOK);
+ masm.assumeUnreachable(
+ "The return of CreateThis must be an object or uninitialized.");
+ masm.bind(&createdThisOK);
+#endif
+
+ // Restore saved registers.
+ masm.PopRegsInMask(liveNonGCRegs);
+
+ // Save |this| value back into pushed arguments on stack.
+ MOZ_ASSERT(!liveNonGCRegs.aliases(JSReturnOperand));
+ storeThis(JSReturnOperand, argcReg, flags);
+
+ // Restore calleeReg. CreateThisFromIC may trigger a GC, so we reload the
+ // callee from the stub frame (which is traced) instead of spilling it to
+ // the stack.
+ loadStackObject(ArgumentKind::Callee, flags, argcReg, calleeReg);
+}
+
+void BaselineCacheIRCompiler::updateReturnValue() {
+ Label skipThisReplace;
+ masm.branchTestObject(Assembler::Equal, JSReturnOperand, &skipThisReplace);
+
+ // If a constructor does not explicitly return an object, the return value
+ // of the constructor is |this|. We load it out of the baseline stub frame.
+
+ // At this point, the stack looks like this:
+ // newTarget
+ // ArgN
+ // ...
+ // Arg0
+ // ThisVal <---- We want this value.
+ // Callee token | Skip two stack slots.
+ // Frame descriptor v
+ // [Top of stack]
+ size_t thisvOffset =
+ JitFrameLayout::offsetOfThis() - JitFrameLayout::bytesPoppedAfterCall();
+ Address thisAddress(masm.getStackPointer(), thisvOffset);
+ masm.loadValue(thisAddress, JSReturnOperand);
+
+#ifdef DEBUG
+ masm.branchTestObject(Assembler::Equal, JSReturnOperand, &skipThisReplace);
+ masm.assumeUnreachable("Return of constructing call should be an object.");
+#endif
+ masm.bind(&skipThisReplace);
+}
+
+bool BaselineCacheIRCompiler::emitCallScriptedFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ Register calleeReg = allocator.useRegister(masm, calleeId);
+ Register argcReg = allocator.useRegister(masm, argcId);
+
+ bool isConstructing = flags.isConstructing();
+ bool isSameRealm = flags.isSameRealm();
+
+ if (!updateArgc(flags, argcReg, scratch)) {
+ return false;
+ }
+
+ allocator.discardStack(masm);
+
+ // Push a stub frame so that we can perform a non-tail call.
+ // Note that this leaves the return address in TailCallReg.
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ if (!isSameRealm) {
+ masm.switchToObjectRealm(calleeReg, scratch);
+ }
+
+ if (isConstructing) {
+ createThis(argcReg, calleeReg, scratch, flags);
+ }
+
+ pushArguments(argcReg, calleeReg, scratch, scratch2, flags, argcFixed,
+ /*isJitCall =*/true);
+
+ // Load the start of the target JitCode.
+ Register code = scratch2;
+ masm.loadJitCodeRaw(calleeReg, code);
+
+ // Note that we use Push, not push, so that callJit will align the stack
+ // properly on ARM.
+ masm.PushCalleeToken(calleeReg, isConstructing);
+ masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, argcReg, scratch);
+
+ // Handle arguments underflow.
+ Label noUnderflow;
+ masm.loadFunctionArgCount(calleeReg, calleeReg);
+ masm.branch32(Assembler::AboveOrEqual, argcReg, calleeReg, &noUnderflow);
+ {
+ // Call the arguments rectifier.
+ TrampolinePtr argumentsRectifier =
+ cx_->runtime()->jitRuntime()->getArgumentsRectifier();
+ masm.movePtr(argumentsRectifier, code);
+ }
+
+ masm.bind(&noUnderflow);
+ masm.callJit(code);
+
+ // If this is a constructing call, and the callee returns a non-object,
+ // replace it with the |this| object passed in.
+ if (isConstructing) {
+ updateReturnValue();
+ }
+
+ stubFrame.leave(masm);
+
+ if (!isSameRealm) {
+ masm.switchToBaselineFrameRealm(scratch2);
+ }
+
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCallWasmFunction(
+ ObjOperandId calleeId, Int32OperandId argcId, CallFlags flags,
+ uint32_t argcFixed, uint32_t funcExportOffset, uint32_t instanceOffset) {
+ return emitCallScriptedFunction(calleeId, argcId, flags, argcFixed);
+}
+
+bool BaselineCacheIRCompiler::emitCallInlinedFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ uint32_t icScriptOffset,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output);
+ AutoScratchRegister codeReg(allocator, masm);
+
+ Register calleeReg = allocator.useRegister(masm, calleeId);
+ Register argcReg = allocator.useRegister(masm, argcId);
+
+ bool isConstructing = flags.isConstructing();
+ bool isSameRealm = flags.isSameRealm();
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadBaselineJitCodeRaw(calleeReg, codeReg, failure->label());
+
+ if (!updateArgc(flags, argcReg, scratch)) {
+ return false;
+ }
+
+ allocator.discardStack(masm);
+
+ // Push a stub frame so that we can perform a non-tail call.
+ // Note that this leaves the return address in TailCallReg.
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ if (!isSameRealm) {
+ masm.switchToObjectRealm(calleeReg, scratch);
+ }
+
+ Label baselineScriptDiscarded;
+ if (isConstructing) {
+ createThis(argcReg, calleeReg, scratch, flags);
+
+ // CreateThisFromIC may trigger a GC and discard the BaselineScript.
+ // We have already called discardStack, so we can't use a FailurePath.
+ // Instead, we skip storing the ICScript in the JSContext and use a
+ // normal non-inlined call.
+ masm.loadBaselineJitCodeRaw(calleeReg, codeReg, &baselineScriptDiscarded);
+ }
+
+ // Store icScript in the context.
+ Address icScriptAddr(stubAddress(icScriptOffset));
+ masm.loadPtr(icScriptAddr, scratch);
+ masm.storeICScriptInJSContext(scratch);
+
+ if (isConstructing) {
+ Label skip;
+ masm.jump(&skip);
+ masm.bind(&baselineScriptDiscarded);
+ masm.loadJitCodeRaw(calleeReg, codeReg);
+ masm.bind(&skip);
+ }
+
+ pushArguments(argcReg, calleeReg, scratch, scratch2, flags, argcFixed,
+ /*isJitCall =*/true);
+
+ // Note that we use Push, not push, so that callJit will align the stack
+ // properly on ARM.
+ masm.PushCalleeToken(calleeReg, isConstructing);
+ masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, argcReg, scratch);
+
+ // Handle arguments underflow.
+ Label noUnderflow;
+ masm.loadFunctionArgCount(calleeReg, calleeReg);
+ masm.branch32(Assembler::AboveOrEqual, argcReg, calleeReg, &noUnderflow);
+
+ // Call the trial-inlining arguments rectifier.
+ ArgumentsRectifierKind kind = ArgumentsRectifierKind::TrialInlining;
+ TrampolinePtr argumentsRectifier =
+ cx_->runtime()->jitRuntime()->getArgumentsRectifier(kind);
+ masm.movePtr(argumentsRectifier, codeReg);
+
+ masm.bind(&noUnderflow);
+ masm.callJit(codeReg);
+
+ // If this is a constructing call, and the callee returns a non-object,
+ // replace it with the |this| object passed in.
+ if (isConstructing) {
+ updateReturnValue();
+ }
+
+ stubFrame.leave(masm);
+
+ if (!isSameRealm) {
+ masm.switchToBaselineFrameRealm(codeReg);
+ }
+
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitNewArrayObjectResult(uint32_t arrayLength,
+ uint32_t shapeOffset,
+ uint32_t siteOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ gc::AllocKind allocKind = GuessArrayGCKind(arrayLength);
+ MOZ_ASSERT(CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_));
+ allocKind = ForegroundToBackgroundAllocKind(allocKind);
+
+ uint32_t slotCount = GetGCKindSlots(allocKind);
+ MOZ_ASSERT(slotCount >= ObjectElements::VALUES_PER_HEADER);
+ uint32_t arrayCapacity = slotCount - ObjectElements::VALUES_PER_HEADER;
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegister result(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegister site(allocator, masm);
+ AutoScratchRegisterMaybeOutput shape(allocator, masm, output);
+
+ Address shapeAddr(stubAddress(shapeOffset));
+ masm.loadPtr(shapeAddr, shape);
+
+ Address siteAddr(stubAddress(siteOffset));
+ masm.loadPtr(siteAddr, site);
+
+ allocator.discardStack(masm);
+
+ Label done;
+ Label fail;
+
+ masm.createArrayWithFixedElements(result, shape, scratch, arrayLength,
+ arrayCapacity, allocKind, gc::DefaultHeap,
+ &fail, AllocSiteInput(site));
+ masm.jump(&done);
+
+ {
+ masm.bind(&fail);
+
+ // We get here if the nursery is full (unlikely) but also for tenured
+ // allocations if the current arena is full and we need to allocate a new
+ // one (fairly common).
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.Push(site);
+ masm.Push(Imm32(int32_t(allocKind)));
+ masm.Push(Imm32(arrayLength));
+
+ using Fn =
+ ArrayObject* (*)(JSContext*, uint32_t, gc::AllocKind, gc::AllocSite*);
+ callVM<Fn, NewArrayObjectBaselineFallback>(masm);
+
+ stubFrame.leave(masm);
+ masm.storeCallPointerResult(result);
+ }
+
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_OBJECT, result, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitNewPlainObjectResult(uint32_t numFixedSlots,
+ uint32_t numDynamicSlots,
+ gc::AllocKind allocKind,
+ uint32_t shapeOffset,
+ uint32_t siteOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegister obj(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegister site(allocator, masm);
+ AutoScratchRegisterMaybeOutput shape(allocator, masm, output);
+
+ Address shapeAddr(stubAddress(shapeOffset));
+ masm.loadPtr(shapeAddr, shape);
+
+ Address siteAddr(stubAddress(siteOffset));
+ masm.loadPtr(siteAddr, site);
+
+ allocator.discardStack(masm);
+
+ Label done;
+ Label fail;
+
+ masm.createPlainGCObject(obj, shape, scratch, shape, numFixedSlots,
+ numDynamicSlots, allocKind, gc::DefaultHeap, &fail,
+ AllocSiteInput(site));
+ masm.jump(&done);
+
+ {
+ masm.bind(&fail);
+
+ // We get here if the nursery is full (unlikely) but also for tenured
+ // allocations if the current arena is full and we need to allocate a new
+ // one (fairly common).
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ masm.Push(site);
+ masm.Push(Imm32(int32_t(allocKind)));
+ masm.loadPtr(shapeAddr, shape); // This might have been overwritten.
+ masm.Push(shape);
+
+ using Fn = JSObject* (*)(JSContext*, Handle<SharedShape*>, gc::AllocKind,
+ gc::AllocSite*);
+ callVM<Fn, NewPlainObjectBaselineFallback>(masm);
+
+ stubFrame.leave(masm);
+ masm.storeCallPointerResult(obj);
+ }
+
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_OBJECT, obj, output.valueReg());
+ return true;
+}
+
+bool BaselineCacheIRCompiler::emitCloseIterScriptedResult(
+ ObjOperandId iterId, ObjOperandId calleeId, CompletionKind kind,
+ uint32_t calleeNargs) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register iter = allocator.useRegister(masm, iterId);
+ Register callee = allocator.useRegister(masm, calleeId);
+
+ AutoScratchRegister code(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+
+ masm.loadJitCodeRaw(callee, code);
+
+ allocator.discardStack(masm);
+
+ AutoStubFrame stubFrame(*this);
+ stubFrame.enter(masm, scratch);
+
+ // Call the return method.
+ masm.alignJitStackBasedOnNArgs(calleeNargs, /*countIncludesThis = */ false);
+ for (uint32_t i = 0; i < calleeNargs; i++) {
+ masm.pushValue(UndefinedValue());
+ }
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(iter)));
+ masm.Push(callee);
+ masm.PushFrameDescriptorForJitCall(FrameType::BaselineStub, /* argc = */ 0);
+
+ masm.callJit(code);
+
+ if (kind != CompletionKind::Throw) {
+ // Verify that the return value is an object.
+ Label success;
+ masm.branchTestObject(Assembler::Equal, JSReturnOperand, &success);
+
+ masm.Push(Imm32(int32_t(CheckIsObjectKind::IteratorReturn)));
+ using Fn = bool (*)(JSContext*, CheckIsObjectKind);
+ callVM<Fn, ThrowCheckIsObject>(masm);
+
+ masm.bind(&success);
+ }
+
+ stubFrame.leave(masm);
+ return true;
+}
diff --git a/js/src/jit/BaselineCacheIRCompiler.h b/js/src/jit/BaselineCacheIRCompiler.h
new file mode 100644
index 0000000000..de3a228815
--- /dev/null
+++ b/js/src/jit/BaselineCacheIRCompiler.h
@@ -0,0 +1,138 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineCacheIRCompiler_h
+#define jit_BaselineCacheIRCompiler_h
+
+#include "mozilla/Attributes.h"
+#include "mozilla/Maybe.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jstypes.h"
+
+#include "jit/CacheIR.h"
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIROpsGenerated.h"
+#include "jit/CacheIRReader.h"
+
+struct JS_PUBLIC_API JSContext;
+
+class JSScript;
+
+namespace js {
+namespace jit {
+
+class CacheIRWriter;
+class ICFallbackStub;
+class ICScript;
+class JitCode;
+class Label;
+class MacroAssembler;
+
+struct Address;
+struct Register;
+
+enum class TailCallVMFunctionId;
+
+enum class ICAttachResult { Attached, DuplicateStub, TooLarge, OOM };
+
+bool TryFoldingStubs(JSContext* cx, ICFallbackStub* fallback, JSScript* script,
+ ICScript* icScript);
+
+ICAttachResult AttachBaselineCacheIRStub(JSContext* cx,
+ const CacheIRWriter& writer,
+ CacheKind kind, JSScript* outerScript,
+ ICScript* icScript,
+ ICFallbackStub* stub,
+ const char* name);
+
+// BaselineCacheIRCompiler compiles CacheIR to BaselineIC native code.
+class MOZ_RAII BaselineCacheIRCompiler : public CacheIRCompiler {
+ bool makesGCCalls_;
+
+ void tailCallVMInternal(MacroAssembler& masm, TailCallVMFunctionId id);
+
+ template <typename Fn, Fn fn>
+ void tailCallVM(MacroAssembler& masm);
+
+ [[nodiscard]] bool emitStoreSlotShared(bool isFixed, ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId);
+ [[nodiscard]] bool emitAddAndStoreSlotShared(
+ CacheOp op, ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset, mozilla::Maybe<uint32_t> numNewSlotsOffset);
+
+ bool updateArgc(CallFlags flags, Register argcReg, Register scratch);
+ void loadStackObject(ArgumentKind kind, CallFlags flags, Register argcReg,
+ Register dest);
+ void pushArguments(Register argcReg, Register calleeReg, Register scratch,
+ Register scratch2, CallFlags flags, uint32_t argcFixed,
+ bool isJitCall);
+ void pushStandardArguments(Register argcReg, Register scratch,
+ Register scratch2, uint32_t argcFixed,
+ bool isJitCall, bool isConstructing);
+ void pushArrayArguments(Register argcReg, Register scratch, Register scratch2,
+ bool isJitCall, bool isConstructing);
+ void pushFunCallArguments(Register argcReg, Register calleeReg,
+ Register scratch, Register scratch2,
+ uint32_t argcFixed, bool isJitCall);
+ void pushFunApplyArgsObj(Register argcReg, Register calleeReg,
+ Register scratch, Register scratch2, bool isJitCall);
+ void createThis(Register argcReg, Register calleeReg, Register scratch,
+ CallFlags flags);
+ template <typename T>
+ void storeThis(const T& newThis, Register argcReg, CallFlags flags);
+ void updateReturnValue();
+
+ enum class NativeCallType { Native, ClassHook };
+ bool emitCallNativeShared(NativeCallType callType, ObjOperandId calleeId,
+ Int32OperandId argcId, CallFlags flags,
+ uint32_t argcFixed,
+ mozilla::Maybe<bool> ignoresReturnValue,
+ mozilla::Maybe<uint32_t> targetOffset);
+
+ enum class StringCode { CodeUnit, CodePoint };
+ bool emitStringFromCodeResult(Int32OperandId codeId, StringCode stringCode);
+
+ void emitAtomizeString(Register str, Register temp, Label* failure);
+
+ bool emitCallScriptedGetterShared(ValOperandId receiverId,
+ uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset,
+ mozilla::Maybe<uint32_t> icScriptOffset);
+ bool emitCallScriptedSetterShared(ObjOperandId receiverId,
+ uint32_t setterOffset, ValOperandId rhsId,
+ bool sameRealm,
+ uint32_t nargsAndFlagsOffset,
+ mozilla::Maybe<uint32_t> icScriptOffset);
+
+ public:
+ friend class AutoStubFrame;
+
+ BaselineCacheIRCompiler(JSContext* cx, TempAllocator& alloc,
+ const CacheIRWriter& writer, uint32_t stubDataOffset);
+
+ [[nodiscard]] bool init(CacheKind kind);
+
+ template <typename Fn, Fn fn>
+ void callVM(MacroAssembler& masm);
+
+ JitCode* compile();
+
+ bool makesGCCalls() const;
+
+ Address stubAddress(uint32_t offset) const;
+
+ private:
+ CACHE_IR_COMPILER_UNSHARED_GENERATED
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BaselineCacheIRCompiler_h */
diff --git a/js/src/jit/BaselineCodeGen.cpp b/js/src/jit/BaselineCodeGen.cpp
new file mode 100644
index 0000000000..1985942ef7
--- /dev/null
+++ b/js/src/jit/BaselineCodeGen.cpp
@@ -0,0 +1,6746 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BaselineCodeGen.h"
+
+#include "mozilla/Casting.h"
+
+#include "gc/GC.h"
+#include "jit/BaselineIC.h"
+#include "jit/BaselineJIT.h"
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIRGenerator.h"
+#include "jit/CalleeToken.h"
+#include "jit/FixedList.h"
+#include "jit/IonOptimizationLevels.h"
+#include "jit/JitcodeMap.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/Linker.h"
+#include "jit/PerfSpewer.h"
+#include "jit/SharedICHelpers.h"
+#include "jit/TrialInlining.h"
+#include "jit/VMFunctions.h"
+#include "js/friend/ErrorMessages.h" // JSMSG_*
+#include "js/UniquePtr.h"
+#include "vm/AsyncFunction.h"
+#include "vm/AsyncIteration.h"
+#include "vm/BuiltinObjectKind.h"
+#include "vm/EnvironmentObject.h"
+#include "vm/FunctionFlags.h" // js::FunctionFlags
+#include "vm/Interpreter.h"
+#include "vm/JSFunction.h"
+#include "vm/Time.h"
+#ifdef MOZ_VTUNE
+# include "vtune/VTuneWrapper.h"
+#endif
+
+#include "debugger/DebugAPI-inl.h"
+#include "jit/BaselineFrameInfo-inl.h"
+#include "jit/JitScript-inl.h"
+#include "jit/MacroAssembler-inl.h"
+#include "jit/SharedICHelpers-inl.h"
+#include "jit/VMFunctionList-inl.h"
+#include "vm/Interpreter-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::TraceKind;
+
+using mozilla::AssertedCast;
+using mozilla::Maybe;
+
+namespace js {
+
+class PlainObject;
+
+namespace jit {
+
+BaselineCompilerHandler::BaselineCompilerHandler(JSContext* cx,
+ MacroAssembler& masm,
+ TempAllocator& alloc,
+ JSScript* script)
+ : frame_(script, masm),
+ alloc_(alloc),
+ analysis_(alloc, script),
+#ifdef DEBUG
+ masm_(masm),
+#endif
+ script_(script),
+ pc_(script->code()),
+ icEntryIndex_(0),
+ compileDebugInstrumentation_(script->isDebuggee()),
+ ionCompileable_(IsIonEnabled(cx) && CanIonCompileScript(cx, script)) {
+}
+
+BaselineInterpreterHandler::BaselineInterpreterHandler(JSContext* cx,
+ MacroAssembler& masm)
+ : frame_(masm) {}
+
+template <typename Handler>
+template <typename... HandlerArgs>
+BaselineCodeGen<Handler>::BaselineCodeGen(JSContext* cx, TempAllocator& alloc,
+ HandlerArgs&&... args)
+ : handler(cx, masm, std::forward<HandlerArgs>(args)...),
+ cx(cx),
+ masm(cx, alloc),
+ frame(handler.frame()) {}
+
+BaselineCompiler::BaselineCompiler(JSContext* cx, TempAllocator& alloc,
+ JSScript* script)
+ : BaselineCodeGen(cx, alloc, /* HandlerArgs = */ alloc, script),
+ profilerPushToggleOffset_() {
+#ifdef JS_CODEGEN_NONE
+ MOZ_CRASH();
+#endif
+}
+
+BaselineInterpreterGenerator::BaselineInterpreterGenerator(JSContext* cx,
+ TempAllocator& alloc)
+ : BaselineCodeGen(cx, alloc /* no handlerArgs */) {}
+
+bool BaselineCompilerHandler::init(JSContext* cx) {
+ if (!analysis_.init(alloc_)) {
+ return false;
+ }
+
+ uint32_t len = script_->length();
+
+ if (!labels_.init(alloc_, len)) {
+ return false;
+ }
+
+ for (size_t i = 0; i < len; i++) {
+ new (&labels_[i]) Label();
+ }
+
+ if (!frame_.init(alloc_)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool BaselineCompiler::init() {
+ if (!handler.init(cx)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool BaselineCompilerHandler::recordCallRetAddr(JSContext* cx,
+ RetAddrEntry::Kind kind,
+ uint32_t retOffset) {
+ uint32_t pcOffset = script_->pcToOffset(pc_);
+
+ // Entries must be sorted by pcOffset for binary search to work.
+ // See BaselineScript::retAddrEntryFromPCOffset.
+ MOZ_ASSERT_IF(!retAddrEntries_.empty(),
+ retAddrEntries_.back().pcOffset() <= pcOffset);
+
+ // Similarly, entries must be sorted by return offset and this offset must be
+ // unique. See BaselineScript::retAddrEntryFromReturnOffset.
+ MOZ_ASSERT_IF(!retAddrEntries_.empty() && !masm_.oom(),
+ retAddrEntries_.back().returnOffset().offset() < retOffset);
+
+ if (!retAddrEntries_.emplaceBack(pcOffset, kind, CodeOffset(retOffset))) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ return true;
+}
+
+bool BaselineInterpreterHandler::recordCallRetAddr(JSContext* cx,
+ RetAddrEntry::Kind kind,
+ uint32_t retOffset) {
+ switch (kind) {
+ case RetAddrEntry::Kind::DebugPrologue:
+ MOZ_ASSERT(callVMOffsets_.debugPrologueOffset == 0,
+ "expected single DebugPrologue call");
+ callVMOffsets_.debugPrologueOffset = retOffset;
+ break;
+ case RetAddrEntry::Kind::DebugEpilogue:
+ MOZ_ASSERT(callVMOffsets_.debugEpilogueOffset == 0,
+ "expected single DebugEpilogue call");
+ callVMOffsets_.debugEpilogueOffset = retOffset;
+ break;
+ case RetAddrEntry::Kind::DebugAfterYield:
+ MOZ_ASSERT(callVMOffsets_.debugAfterYieldOffset == 0,
+ "expected single DebugAfterYield call");
+ callVMOffsets_.debugAfterYieldOffset = retOffset;
+ break;
+ default:
+ break;
+ }
+
+ return true;
+}
+
+bool BaselineInterpreterHandler::addDebugInstrumentationOffset(
+ JSContext* cx, CodeOffset offset) {
+ if (!debugInstrumentationOffsets_.append(offset.offset())) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+ return true;
+}
+
+MethodStatus BaselineCompiler::compile() {
+ AutoCreatedBy acb(masm, "BaselineCompiler::compile");
+
+ JSScript* script = handler.script();
+ JitSpew(JitSpew_BaselineScripts, "Baseline compiling script %s:%u:%u (%p)",
+ script->filename(), script->lineno(), script->column(), script);
+
+ JitSpew(JitSpew_Codegen, "# Emitting baseline code for script %s:%u:%u",
+ script->filename(), script->lineno(), script->column());
+
+ AutoIncrementalTimer timer(cx->realm()->timers.baselineCompileTime);
+
+ AutoKeepJitScripts keepJitScript(cx);
+ if (!script->ensureHasJitScript(cx, keepJitScript)) {
+ return Method_Error;
+ }
+
+ // When code coverage is enabled, we have to create the ScriptCounts if they
+ // do not exist.
+ if (!script->hasScriptCounts() && cx->realm()->collectCoverageForDebug()) {
+ if (!script->initScriptCounts(cx)) {
+ return Method_Error;
+ }
+ }
+
+ // Suppress GC during compilation.
+ gc::AutoSuppressGC suppressGC(cx);
+
+ MOZ_ASSERT(!script->hasBaselineScript());
+
+ if (!emitPrologue()) {
+ return Method_Error;
+ }
+
+ MethodStatus status = emitBody();
+ if (status != Method_Compiled) {
+ return status;
+ }
+
+ if (!emitEpilogue()) {
+ return Method_Error;
+ }
+
+ if (!emitOutOfLinePostBarrierSlot()) {
+ return Method_Error;
+ }
+
+ AutoCreatedBy acb2(masm, "exception_tail");
+ Linker linker(masm);
+ if (masm.oom()) {
+ ReportOutOfMemory(cx);
+ return Method_Error;
+ }
+
+ JitCode* code = linker.newCode(cx, CodeKind::Baseline);
+ if (!code) {
+ return Method_Error;
+ }
+
+ UniquePtr<BaselineScript> baselineScript(
+ BaselineScript::New(
+ cx, warmUpCheckPrologueOffset_.offset(),
+ profilerEnterFrameToggleOffset_.offset(),
+ profilerExitFrameToggleOffset_.offset(),
+ handler.retAddrEntries().length(), handler.osrEntries().length(),
+ debugTrapEntries_.length(), script->resumeOffsets().size()),
+ JS::DeletePolicy<BaselineScript>(cx->runtime()));
+ if (!baselineScript) {
+ return Method_Error;
+ }
+
+ baselineScript->setMethod(code);
+
+ JitSpew(JitSpew_BaselineScripts,
+ "Created BaselineScript %p (raw %p) for %s:%u:%u",
+ (void*)baselineScript.get(), (void*)code->raw(), script->filename(),
+ script->lineno(), script->column());
+
+ baselineScript->copyRetAddrEntries(handler.retAddrEntries().begin());
+ baselineScript->copyOSREntries(handler.osrEntries().begin());
+ baselineScript->copyDebugTrapEntries(debugTrapEntries_.begin());
+
+ // If profiler instrumentation is enabled, toggle instrumentation on.
+ if (cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(
+ cx->runtime())) {
+ baselineScript->toggleProfilerInstrumentation(true);
+ }
+
+ // Compute native resume addresses for the script's resume offsets.
+ baselineScript->computeResumeNativeOffsets(script, resumeOffsetEntries_);
+
+ if (compileDebugInstrumentation()) {
+ baselineScript->setHasDebugInstrumentation();
+ }
+
+ // Always register a native => bytecode mapping entry, since profiler can be
+ // turned on with baseline jitcode on stack, and baseline jitcode cannot be
+ // invalidated.
+ {
+ JitSpew(JitSpew_Profiling,
+ "Added JitcodeGlobalEntry for baseline script %s:%u:%u (%p)",
+ script->filename(), script->lineno(), script->column(),
+ baselineScript.get());
+
+ // Generate profiling string.
+ UniqueChars str = GeckoProfilerRuntime::allocProfileString(cx, script);
+ if (!str) {
+ return Method_Error;
+ }
+
+ auto entry = MakeJitcodeGlobalEntry<BaselineEntry>(
+ cx, code, code->raw(), code->rawEnd(), script, std::move(str));
+ if (!entry) {
+ return Method_Error;
+ }
+
+ JitcodeGlobalTable* globalTable =
+ cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
+ if (!globalTable->addEntry(std::move(entry))) {
+ ReportOutOfMemory(cx);
+ return Method_Error;
+ }
+
+ // Mark the jitcode as having a bytecode map.
+ code->setHasBytecodeMap();
+ }
+
+ script->jitScript()->setBaselineScript(script, baselineScript.release());
+
+ perfSpewer_.saveProfile(cx, script, code);
+
+#ifdef MOZ_VTUNE
+ vtune::MarkScript(code, script, "baseline");
+#endif
+
+ return Method_Compiled;
+}
+
+// On most platforms we use a dedicated bytecode PC register to avoid many
+// dependent loads and stores for sequences of simple bytecode ops. This
+// register must be saved/restored around VM and IC calls.
+//
+// On 32-bit x86 we don't have enough registers for this (because R0-R2 require
+// 6 registers) so there we always store the pc on the frame.
+static constexpr bool HasInterpreterPCReg() {
+ return InterpreterPCReg != InvalidReg;
+}
+
+static Register LoadBytecodePC(MacroAssembler& masm, Register scratch) {
+ if (HasInterpreterPCReg()) {
+ return InterpreterPCReg;
+ }
+
+ Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC());
+ masm.loadPtr(pcAddr, scratch);
+ return scratch;
+}
+
+static void LoadInt8Operand(MacroAssembler& masm, Register dest) {
+ Register pc = LoadBytecodePC(masm, dest);
+ masm.load8SignExtend(Address(pc, sizeof(jsbytecode)), dest);
+}
+
+static void LoadUint8Operand(MacroAssembler& masm, Register dest) {
+ Register pc = LoadBytecodePC(masm, dest);
+ masm.load8ZeroExtend(Address(pc, sizeof(jsbytecode)), dest);
+}
+
+static void LoadUint16Operand(MacroAssembler& masm, Register dest) {
+ Register pc = LoadBytecodePC(masm, dest);
+ masm.load16ZeroExtend(Address(pc, sizeof(jsbytecode)), dest);
+}
+
+static void LoadInt32Operand(MacroAssembler& masm, Register dest) {
+ Register pc = LoadBytecodePC(masm, dest);
+ masm.load32(Address(pc, sizeof(jsbytecode)), dest);
+}
+
+static void LoadInt32OperandSignExtendToPtr(MacroAssembler& masm, Register pc,
+ Register dest) {
+ masm.load32SignExtendToPtr(Address(pc, sizeof(jsbytecode)), dest);
+}
+
+static void LoadUint24Operand(MacroAssembler& masm, size_t offset,
+ Register dest) {
+ // Load the opcode and operand, then left shift to discard the opcode.
+ Register pc = LoadBytecodePC(masm, dest);
+ masm.load32(Address(pc, offset), dest);
+ masm.rshift32(Imm32(8), dest);
+}
+
+static void LoadInlineValueOperand(MacroAssembler& masm, ValueOperand dest) {
+ // Note: the Value might be unaligned but as above we rely on all our
+ // platforms having appropriate support for unaligned accesses (except for
+ // floating point instructions on ARM).
+ Register pc = LoadBytecodePC(masm, dest.scratchReg());
+ masm.loadUnalignedValue(Address(pc, sizeof(jsbytecode)), dest);
+}
+
+template <>
+void BaselineCompilerCodeGen::loadScript(Register dest) {
+ masm.movePtr(ImmGCPtr(handler.script()), dest);
+}
+
+template <>
+void BaselineInterpreterCodeGen::loadScript(Register dest) {
+ masm.loadPtr(frame.addressOfInterpreterScript(), dest);
+}
+
+template <>
+void BaselineCompilerCodeGen::saveInterpreterPCReg() {}
+
+template <>
+void BaselineInterpreterCodeGen::saveInterpreterPCReg() {
+ if (HasInterpreterPCReg()) {
+ masm.storePtr(InterpreterPCReg, frame.addressOfInterpreterPC());
+ }
+}
+
+template <>
+void BaselineCompilerCodeGen::restoreInterpreterPCReg() {}
+
+template <>
+void BaselineInterpreterCodeGen::restoreInterpreterPCReg() {
+ if (HasInterpreterPCReg()) {
+ masm.loadPtr(frame.addressOfInterpreterPC(), InterpreterPCReg);
+ }
+}
+
+template <>
+void BaselineCompilerCodeGen::emitInitializeLocals() {
+ // Initialize all locals to |undefined|. Lexical bindings are temporal
+ // dead zoned in bytecode.
+
+ size_t n = frame.nlocals();
+ if (n == 0) {
+ return;
+ }
+
+ // Use R0 to minimize code size. If the number of locals to push is <
+ // LOOP_UNROLL_FACTOR, then the initialization pushes are emitted directly
+ // and inline. Otherwise, they're emitted in a partially unrolled loop.
+ static const size_t LOOP_UNROLL_FACTOR = 4;
+ size_t toPushExtra = n % LOOP_UNROLL_FACTOR;
+
+ masm.moveValue(UndefinedValue(), R0);
+
+ // Handle any extra pushes left over by the optional unrolled loop below.
+ for (size_t i = 0; i < toPushExtra; i++) {
+ masm.pushValue(R0);
+ }
+
+ // Partially unrolled loop of pushes.
+ if (n >= LOOP_UNROLL_FACTOR) {
+ size_t toPush = n - toPushExtra;
+ MOZ_ASSERT(toPush % LOOP_UNROLL_FACTOR == 0);
+ MOZ_ASSERT(toPush >= LOOP_UNROLL_FACTOR);
+ masm.move32(Imm32(toPush), R1.scratchReg());
+ // Emit unrolled loop with 4 pushes per iteration.
+ Label pushLoop;
+ masm.bind(&pushLoop);
+ for (size_t i = 0; i < LOOP_UNROLL_FACTOR; i++) {
+ masm.pushValue(R0);
+ }
+ masm.branchSub32(Assembler::NonZero, Imm32(LOOP_UNROLL_FACTOR),
+ R1.scratchReg(), &pushLoop);
+ }
+}
+
+template <>
+void BaselineInterpreterCodeGen::emitInitializeLocals() {
+ // Push |undefined| for all locals.
+
+ Register scratch = R0.scratchReg();
+ loadScript(scratch);
+ masm.loadPtr(Address(scratch, JSScript::offsetOfSharedData()), scratch);
+ masm.loadPtr(Address(scratch, SharedImmutableScriptData::offsetOfISD()),
+ scratch);
+ masm.load32(Address(scratch, ImmutableScriptData::offsetOfNfixed()), scratch);
+
+ Label top, done;
+ masm.branchTest32(Assembler::Zero, scratch, scratch, &done);
+ masm.bind(&top);
+ {
+ masm.pushValue(UndefinedValue());
+ masm.branchSub32(Assembler::NonZero, Imm32(1), scratch, &top);
+ }
+ masm.bind(&done);
+}
+
+// On input:
+// R2.scratchReg() contains object being written to.
+// Called with the baseline stack synced, except for R0 which is preserved.
+// All other registers are usable as scratch.
+// This calls:
+// void PostWriteBarrier(JSRuntime* rt, JSObject* obj);
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitOutOfLinePostBarrierSlot() {
+ AutoCreatedBy acb(masm,
+ "BaselineCodeGen<Handler>::emitOutOfLinePostBarrierSlot");
+
+ if (!postBarrierSlot_.used()) {
+ return true;
+ }
+
+ masm.bind(&postBarrierSlot_);
+
+ saveInterpreterPCReg();
+
+ Register objReg = R2.scratchReg();
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.take(R0);
+ regs.take(objReg);
+ Register scratch = regs.takeAny();
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
+ // On ARM, save the link register before calling. It contains the return
+ // address. The |masm.ret()| later will pop this into |pc| to return.
+ masm.push(lr);
+#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ masm.push(ra);
+#elif defined(JS_CODEGEN_LOONG64)
+ masm.push(ra);
+#endif
+ masm.pushValue(R0);
+
+ using Fn = void (*)(JSRuntime * rt, js::gc::Cell * cell);
+ masm.setupUnalignedABICall(scratch);
+ masm.movePtr(ImmPtr(cx->runtime()), scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(objReg);
+ masm.callWithABI<Fn, PostWriteBarrier>();
+
+ restoreInterpreterPCReg();
+
+ masm.popValue(R0);
+ masm.ret();
+ return true;
+}
+
+// Scan the a cache IR stub's fields and create an allocation site for any that
+// refer to the catch-all unknown allocation site. This will be the case for
+// stubs created when running in the interpreter. This happens on transition to
+// baseline.
+static bool CreateAllocSitesForCacheIRStub(JSScript* script,
+ ICCacheIRStub* stub) {
+ const CacheIRStubInfo* stubInfo = stub->stubInfo();
+ uint8_t* stubData = stub->stubDataStart();
+
+ uint32_t field = 0;
+ size_t offset = 0;
+ while (true) {
+ StubField::Type fieldType = stubInfo->fieldType(field);
+ if (fieldType == StubField::Type::Limit) {
+ break;
+ }
+
+ if (fieldType == StubField::Type::AllocSite) {
+ gc::AllocSite* site =
+ stubInfo->getPtrStubField<ICCacheIRStub, gc::AllocSite>(stub, offset);
+ if (site->kind() == gc::AllocSite::Kind::Unknown) {
+ gc::AllocSite* newSite = script->createAllocSite();
+ if (!newSite) {
+ return false;
+ }
+
+ stubInfo->replaceStubRawWord(stubData, offset, uintptr_t(site),
+ uintptr_t(newSite));
+ }
+ }
+
+ field++;
+ offset += StubField::sizeInBytes(fieldType);
+ }
+
+ return true;
+}
+
+static void CreateAllocSitesForICChain(JSScript* script, uint32_t entryIndex) {
+ JitScript* jitScript = script->jitScript();
+ ICStub* stub = jitScript->icEntry(entryIndex).firstStub();
+
+ while (!stub->isFallback()) {
+ if (!CreateAllocSitesForCacheIRStub(script, stub->toCacheIRStub())) {
+ // This is an optimization and safe to skip if we hit OOM or per-zone
+ // limit.
+ return;
+ }
+ stub = stub->toCacheIRStub()->next();
+ }
+}
+
+template <>
+bool BaselineCompilerCodeGen::emitNextIC() {
+ AutoCreatedBy acb(masm, "emitNextIC");
+
+ // Emit a call to an IC stored in JitScript. Calls to this must match the
+ // ICEntry order in JitScript: first the non-op IC entries for |this| and
+ // formal arguments, then the for-op IC entries for JOF_IC ops.
+
+ JSScript* script = handler.script();
+ uint32_t pcOffset = script->pcToOffset(handler.pc());
+
+ // We don't use every ICEntry and we can skip unreachable ops, so we have
+ // to loop until we find an ICEntry for the current pc.
+ const ICFallbackStub* stub;
+ uint32_t entryIndex;
+ do {
+ stub = script->jitScript()->fallbackStub(handler.icEntryIndex());
+ entryIndex = handler.icEntryIndex();
+ handler.moveToNextICEntry();
+ } while (stub->pcOffset() < pcOffset);
+
+ MOZ_ASSERT(stub->pcOffset() == pcOffset);
+ MOZ_ASSERT(BytecodeOpHasIC(JSOp(*handler.pc())));
+
+ if (BytecodeOpCanHaveAllocSite(JSOp(*handler.pc()))) {
+ CreateAllocSitesForICChain(script, entryIndex);
+ }
+
+ // Load stub pointer into ICStubReg.
+ masm.loadPtr(frame.addressOfICScript(), ICStubReg);
+ size_t firstStubOffset = ICScript::offsetOfFirstStub(entryIndex);
+ masm.loadPtr(Address(ICStubReg, firstStubOffset), ICStubReg);
+
+ CodeOffset returnOffset;
+ EmitCallIC(masm, &returnOffset);
+
+ RetAddrEntry::Kind kind = RetAddrEntry::Kind::IC;
+ if (!handler.retAddrEntries().emplaceBack(pcOffset, kind, returnOffset)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emitNextIC() {
+ saveInterpreterPCReg();
+ masm.loadPtr(frame.addressOfInterpreterICEntry(), ICStubReg);
+ masm.loadPtr(Address(ICStubReg, ICEntry::offsetOfFirstStub()), ICStubReg);
+ masm.call(Address(ICStubReg, ICStub::offsetOfStubCode()));
+ uint32_t returnOffset = masm.currentOffset();
+ restoreInterpreterPCReg();
+
+ // If this is an IC for a bytecode op where Ion may inline scripts, we need to
+ // record the return offset for Ion bailouts.
+ if (handler.currentOp()) {
+ JSOp op = *handler.currentOp();
+ MOZ_ASSERT(BytecodeOpHasIC(op));
+ if (IsIonInlinableOp(op)) {
+ if (!handler.icReturnOffsets().emplaceBack(returnOffset, op)) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+template <>
+void BaselineCompilerCodeGen::computeFrameSize(Register dest) {
+ MOZ_ASSERT(!inCall_, "must not be called in the middle of a VM call");
+ masm.move32(Imm32(frame.frameSize()), dest);
+}
+
+template <>
+void BaselineInterpreterCodeGen::computeFrameSize(Register dest) {
+ // dest := FramePointer - StackPointer.
+ MOZ_ASSERT(!inCall_, "must not be called in the middle of a VM call");
+ masm.mov(FramePointer, dest);
+ masm.subStackPtrFrom(dest);
+}
+
+template <typename Handler>
+void BaselineCodeGen<Handler>::prepareVMCall() {
+ pushedBeforeCall_ = masm.framePushed();
+#ifdef DEBUG
+ inCall_ = true;
+#endif
+
+ // Ensure everything is synced.
+ frame.syncStack(0);
+}
+
+template <>
+void BaselineCompilerCodeGen::storeFrameSizeAndPushDescriptor(
+ uint32_t argSize, Register scratch) {
+#ifdef DEBUG
+ masm.store32(Imm32(frame.frameSize()), frame.addressOfDebugFrameSize());
+#endif
+
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+}
+
+template <>
+void BaselineInterpreterCodeGen::storeFrameSizeAndPushDescriptor(
+ uint32_t argSize, Register scratch) {
+#ifdef DEBUG
+ // Store the frame size without VMFunction arguments in debug builds.
+ // scratch := FramePointer - StackPointer - argSize.
+ masm.mov(FramePointer, scratch);
+ masm.subStackPtrFrom(scratch);
+ masm.sub32(Imm32(argSize), scratch);
+ masm.store32(scratch, frame.addressOfDebugFrameSize());
+#endif
+
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+}
+
+static uint32_t GetVMFunctionArgSize(const VMFunctionData& fun) {
+ return fun.explicitStackSlots() * sizeof(void*);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::callVMInternal(VMFunctionId id,
+ RetAddrEntry::Kind kind,
+ CallVMPhase phase) {
+#ifdef DEBUG
+ // Assert prepareVMCall() has been called.
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+
+ TrampolinePtr code = cx->runtime()->jitRuntime()->getVMWrapper(id);
+ const VMFunctionData& fun = GetVMFunction(id);
+
+ uint32_t argSize = GetVMFunctionArgSize(fun);
+
+ // Assert all arguments were pushed.
+ MOZ_ASSERT(masm.framePushed() - pushedBeforeCall_ == argSize);
+
+ saveInterpreterPCReg();
+
+ if (phase == CallVMPhase::AfterPushingLocals) {
+ storeFrameSizeAndPushDescriptor(argSize, R0.scratchReg());
+ } else {
+ MOZ_ASSERT(phase == CallVMPhase::BeforePushingLocals);
+#ifdef DEBUG
+ uint32_t frameBaseSize = BaselineFrame::frameSizeForNumValueSlots(0);
+ masm.store32(Imm32(frameBaseSize), frame.addressOfDebugFrameSize());
+#endif
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ }
+ MOZ_ASSERT(fun.expectTailCall == NonTailCall);
+ // Perform the call.
+ masm.call(code);
+ uint32_t callOffset = masm.currentOffset();
+
+ // Pop arguments from framePushed.
+ masm.implicitPop(argSize);
+
+ restoreInterpreterPCReg();
+
+ return handler.recordCallRetAddr(cx, kind, callOffset);
+}
+
+template <typename Handler>
+template <typename Fn, Fn fn>
+bool BaselineCodeGen<Handler>::callVM(RetAddrEntry::Kind kind,
+ CallVMPhase phase) {
+ VMFunctionId fnId = VMFunctionToId<Fn, fn>::id;
+ return callVMInternal(fnId, kind, phase);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitStackCheck() {
+ Label skipCall;
+ if (handler.mustIncludeSlotsInStackCheck()) {
+ // Subtract the size of script->nslots() first.
+ Register scratch = R1.scratchReg();
+ masm.moveStackPtrTo(scratch);
+ subtractScriptSlotsSize(scratch, R2.scratchReg());
+ masm.branchPtr(Assembler::BelowOrEqual,
+ AbsoluteAddress(cx->addressOfJitStackLimit()), scratch,
+ &skipCall);
+ } else {
+ masm.branchStackPtrRhs(Assembler::BelowOrEqual,
+ AbsoluteAddress(cx->addressOfJitStackLimit()),
+ &skipCall);
+ }
+
+ prepareVMCall();
+ masm.loadBaselineFramePtr(FramePointer, R1.scratchReg());
+ pushArg(R1.scratchReg());
+
+ const CallVMPhase phase = CallVMPhase::BeforePushingLocals;
+ const RetAddrEntry::Kind kind = RetAddrEntry::Kind::StackCheck;
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ if (!callVM<Fn, CheckOverRecursedBaseline>(kind, phase)) {
+ return false;
+ }
+
+ masm.bind(&skipCall);
+ return true;
+}
+
+static void EmitCallFrameIsDebuggeeCheck(MacroAssembler& masm) {
+ using Fn = void (*)(BaselineFrame * frame);
+ masm.setupUnalignedABICall(R0.scratchReg());
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ masm.passABIArg(R0.scratchReg());
+ masm.callWithABI<Fn, FrameIsDebuggeeCheck>();
+}
+
+template <>
+bool BaselineCompilerCodeGen::emitIsDebuggeeCheck() {
+ if (handler.compileDebugInstrumentation()) {
+ EmitCallFrameIsDebuggeeCheck(masm);
+ }
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emitIsDebuggeeCheck() {
+ // Use a toggled jump to call FrameIsDebuggeeCheck only if the debugger is
+ // enabled.
+ //
+ // TODO(bug 1522394): consider having a cx->realm->isDebuggee guard before the
+ // call. Consider moving the callWithABI out-of-line.
+
+ Label skipCheck;
+ CodeOffset toggleOffset = masm.toggledJump(&skipCheck);
+ {
+ saveInterpreterPCReg();
+ EmitCallFrameIsDebuggeeCheck(masm);
+ restoreInterpreterPCReg();
+ }
+ masm.bind(&skipCheck);
+ return handler.addDebugInstrumentationOffset(cx, toggleOffset);
+}
+
+static void MaybeIncrementCodeCoverageCounter(MacroAssembler& masm,
+ JSScript* script,
+ jsbytecode* pc) {
+ if (!script->hasScriptCounts()) {
+ return;
+ }
+ PCCounts* counts = script->maybeGetPCCounts(pc);
+ uint64_t* counterAddr = &counts->numExec();
+ masm.inc64(AbsoluteAddress(counterAddr));
+}
+
+template <>
+bool BaselineCompilerCodeGen::emitHandleCodeCoverageAtPrologue() {
+ // If the main instruction is not a jump target, then we emit the
+ // corresponding code coverage counter.
+ JSScript* script = handler.script();
+ jsbytecode* main = script->main();
+ if (!BytecodeIsJumpTarget(JSOp(*main))) {
+ MaybeIncrementCodeCoverageCounter(masm, script, main);
+ }
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emitHandleCodeCoverageAtPrologue() {
+ Label skipCoverage;
+ CodeOffset toggleOffset = masm.toggledJump(&skipCoverage);
+ masm.call(handler.codeCoverageAtPrologueLabel());
+ masm.bind(&skipCoverage);
+ return handler.codeCoverageOffsets().append(toggleOffset.offset());
+}
+
+template <>
+void BaselineCompilerCodeGen::subtractScriptSlotsSize(Register reg,
+ Register scratch) {
+ uint32_t slotsSize = handler.script()->nslots() * sizeof(Value);
+ masm.subPtr(Imm32(slotsSize), reg);
+}
+
+template <>
+void BaselineInterpreterCodeGen::subtractScriptSlotsSize(Register reg,
+ Register scratch) {
+ // reg = reg - script->nslots() * sizeof(Value)
+ MOZ_ASSERT(reg != scratch);
+ loadScript(scratch);
+ masm.loadPtr(Address(scratch, JSScript::offsetOfSharedData()), scratch);
+ masm.loadPtr(Address(scratch, SharedImmutableScriptData::offsetOfISD()),
+ scratch);
+ masm.load32(Address(scratch, ImmutableScriptData::offsetOfNslots()), scratch);
+ static_assert(sizeof(Value) == 8,
+ "shift by 3 below assumes Value is 8 bytes");
+ masm.lshiftPtr(Imm32(3), scratch);
+ masm.subPtr(scratch, reg);
+}
+
+template <>
+void BaselineCompilerCodeGen::loadGlobalLexicalEnvironment(Register dest) {
+ MOZ_ASSERT(!handler.script()->hasNonSyntacticScope());
+ masm.movePtr(ImmGCPtr(&cx->global()->lexicalEnvironment()), dest);
+}
+
+template <>
+void BaselineInterpreterCodeGen::loadGlobalLexicalEnvironment(Register dest) {
+ masm.loadPtr(AbsoluteAddress(cx->addressOfRealm()), dest);
+ masm.loadPtr(Address(dest, Realm::offsetOfActiveGlobal()), dest);
+ masm.loadPrivate(Address(dest, GlobalObject::offsetOfGlobalDataSlot()), dest);
+ masm.loadPtr(Address(dest, GlobalObjectData::offsetOfLexicalEnvironment()),
+ dest);
+}
+
+template <>
+void BaselineCompilerCodeGen::pushGlobalLexicalEnvironmentValue(
+ ValueOperand scratch) {
+ frame.push(ObjectValue(cx->global()->lexicalEnvironment()));
+}
+
+template <>
+void BaselineInterpreterCodeGen::pushGlobalLexicalEnvironmentValue(
+ ValueOperand scratch) {
+ loadGlobalLexicalEnvironment(scratch.scratchReg());
+ masm.tagValue(JSVAL_TYPE_OBJECT, scratch.scratchReg(), scratch);
+ frame.push(scratch);
+}
+
+template <>
+void BaselineCompilerCodeGen::loadGlobalThisValue(ValueOperand dest) {
+ JSObject* thisObj = cx->global()->lexicalEnvironment().thisObject();
+ masm.moveValue(ObjectValue(*thisObj), dest);
+}
+
+template <>
+void BaselineInterpreterCodeGen::loadGlobalThisValue(ValueOperand dest) {
+ Register scratch = dest.scratchReg();
+ loadGlobalLexicalEnvironment(scratch);
+ static constexpr size_t SlotOffset =
+ GlobalLexicalEnvironmentObject::offsetOfThisValueSlot();
+ masm.loadValue(Address(scratch, SlotOffset), dest);
+}
+
+template <>
+void BaselineCompilerCodeGen::pushScriptArg() {
+ pushArg(ImmGCPtr(handler.script()));
+}
+
+template <>
+void BaselineInterpreterCodeGen::pushScriptArg() {
+ pushArg(frame.addressOfInterpreterScript());
+}
+
+template <>
+void BaselineCompilerCodeGen::pushBytecodePCArg() {
+ pushArg(ImmPtr(handler.pc()));
+}
+
+template <>
+void BaselineInterpreterCodeGen::pushBytecodePCArg() {
+ if (HasInterpreterPCReg()) {
+ pushArg(InterpreterPCReg);
+ } else {
+ pushArg(frame.addressOfInterpreterPC());
+ }
+}
+
+static gc::Cell* GetScriptGCThing(JSScript* script, jsbytecode* pc,
+ ScriptGCThingType type) {
+ switch (type) {
+ case ScriptGCThingType::Atom:
+ return script->getAtom(pc);
+ case ScriptGCThingType::String:
+ return script->getString(pc);
+ case ScriptGCThingType::RegExp:
+ return script->getRegExp(pc);
+ case ScriptGCThingType::Object:
+ return script->getObject(pc);
+ case ScriptGCThingType::Function:
+ return script->getFunction(pc);
+ case ScriptGCThingType::Scope:
+ return script->getScope(pc);
+ case ScriptGCThingType::BigInt:
+ return script->getBigInt(pc);
+ }
+ MOZ_CRASH("Unexpected GCThing type");
+}
+
+template <>
+void BaselineCompilerCodeGen::loadScriptGCThing(ScriptGCThingType type,
+ Register dest,
+ Register scratch) {
+ gc::Cell* thing = GetScriptGCThing(handler.script(), handler.pc(), type);
+ masm.movePtr(ImmGCPtr(thing), dest);
+}
+
+template <>
+void BaselineInterpreterCodeGen::loadScriptGCThing(ScriptGCThingType type,
+ Register dest,
+ Register scratch) {
+ MOZ_ASSERT(dest != scratch);
+
+ // Load the index in |scratch|.
+ LoadInt32Operand(masm, scratch);
+
+ // Load the GCCellPtr.
+ loadScript(dest);
+ masm.loadPtr(Address(dest, JSScript::offsetOfPrivateData()), dest);
+ masm.loadPtr(BaseIndex(dest, scratch, ScalePointer,
+ PrivateScriptData::offsetOfGCThings()),
+ dest);
+
+ // Clear the tag bits.
+ switch (type) {
+ case ScriptGCThingType::Atom:
+ case ScriptGCThingType::String:
+ // Use xorPtr with a 32-bit immediate because it's more efficient than
+ // andPtr on 64-bit.
+ static_assert(uintptr_t(TraceKind::String) == 2,
+ "Unexpected tag bits for string GCCellPtr");
+ masm.xorPtr(Imm32(2), dest);
+ break;
+ case ScriptGCThingType::RegExp:
+ case ScriptGCThingType::Object:
+ case ScriptGCThingType::Function:
+ // No-op because GCCellPtr tag bits are zero for objects.
+ static_assert(uintptr_t(TraceKind::Object) == 0,
+ "Unexpected tag bits for object GCCellPtr");
+ break;
+ case ScriptGCThingType::BigInt:
+ // Use xorPtr with a 32-bit immediate because it's more efficient than
+ // andPtr on 64-bit.
+ static_assert(uintptr_t(TraceKind::BigInt) == 1,
+ "Unexpected tag bits for BigInt GCCellPtr");
+ masm.xorPtr(Imm32(1), dest);
+ break;
+ case ScriptGCThingType::Scope:
+ // Use xorPtr with a 32-bit immediate because it's more efficient than
+ // andPtr on 64-bit.
+ static_assert(uintptr_t(TraceKind::Scope) >= JS::OutOfLineTraceKindMask,
+ "Expected Scopes to have OutOfLineTraceKindMask tag");
+ masm.xorPtr(Imm32(JS::OutOfLineTraceKindMask), dest);
+ break;
+ }
+
+#ifdef DEBUG
+ // Assert low bits are not set.
+ Label ok;
+ masm.branchTestPtr(Assembler::Zero, dest, Imm32(0b111), &ok);
+ masm.assumeUnreachable("GC pointer with tag bits set");
+ masm.bind(&ok);
+#endif
+}
+
+template <>
+void BaselineCompilerCodeGen::pushScriptGCThingArg(ScriptGCThingType type,
+ Register scratch1,
+ Register scratch2) {
+ gc::Cell* thing = GetScriptGCThing(handler.script(), handler.pc(), type);
+ pushArg(ImmGCPtr(thing));
+}
+
+template <>
+void BaselineInterpreterCodeGen::pushScriptGCThingArg(ScriptGCThingType type,
+ Register scratch1,
+ Register scratch2) {
+ loadScriptGCThing(type, scratch1, scratch2);
+ pushArg(scratch1);
+}
+
+template <typename Handler>
+void BaselineCodeGen<Handler>::pushScriptNameArg(Register scratch1,
+ Register scratch2) {
+ pushScriptGCThingArg(ScriptGCThingType::Atom, scratch1, scratch2);
+}
+
+template <>
+void BaselineCompilerCodeGen::pushUint8BytecodeOperandArg(Register) {
+ MOZ_ASSERT(JOF_OPTYPE(JSOp(*handler.pc())) == JOF_UINT8);
+ pushArg(Imm32(GET_UINT8(handler.pc())));
+}
+
+template <>
+void BaselineInterpreterCodeGen::pushUint8BytecodeOperandArg(Register scratch) {
+ LoadUint8Operand(masm, scratch);
+ pushArg(scratch);
+}
+
+template <>
+void BaselineCompilerCodeGen::pushUint16BytecodeOperandArg(Register) {
+ MOZ_ASSERT(JOF_OPTYPE(JSOp(*handler.pc())) == JOF_UINT16);
+ pushArg(Imm32(GET_UINT16(handler.pc())));
+}
+
+template <>
+void BaselineInterpreterCodeGen::pushUint16BytecodeOperandArg(
+ Register scratch) {
+ LoadUint16Operand(masm, scratch);
+ pushArg(scratch);
+}
+
+template <>
+void BaselineCompilerCodeGen::loadInt32LengthBytecodeOperand(Register dest) {
+ uint32_t length = GET_UINT32(handler.pc());
+ MOZ_ASSERT(length <= INT32_MAX,
+ "the bytecode emitter must fail to compile code that would "
+ "produce a length exceeding int32_t range");
+ masm.move32(Imm32(AssertedCast<int32_t>(length)), dest);
+}
+
+template <>
+void BaselineInterpreterCodeGen::loadInt32LengthBytecodeOperand(Register dest) {
+ LoadInt32Operand(masm, dest);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitDebugPrologue() {
+ auto ifDebuggee = [this]() {
+ // Load pointer to BaselineFrame in R0.
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ prepareVMCall();
+ pushArg(R0.scratchReg());
+
+ const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugPrologue;
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ if (!callVM<Fn, jit::DebugPrologue>(kind)) {
+ return false;
+ }
+
+ return true;
+ };
+ return emitDebugInstrumentation(ifDebuggee);
+}
+
+template <>
+void BaselineCompilerCodeGen::emitInitFrameFields(Register nonFunctionEnv) {
+ Register scratch = R0.scratchReg();
+ Register scratch2 = R2.scratchReg();
+ MOZ_ASSERT(nonFunctionEnv != scratch && nonFunctionEnv != scratch2);
+
+ masm.store32(Imm32(0), frame.addressOfFlags());
+ if (handler.function()) {
+ masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), scratch);
+ masm.unboxObject(Address(scratch, JSFunction::offsetOfEnvironment()),
+ scratch);
+ masm.storePtr(scratch, frame.addressOfEnvironmentChain());
+ } else {
+ masm.storePtr(nonFunctionEnv, frame.addressOfEnvironmentChain());
+ }
+
+ // If cx->inlinedICScript contains an inlined ICScript (passed from
+ // the caller), take that ICScript and store it in the frame, then
+ // overwrite cx->inlinedICScript with nullptr.
+ Label notInlined, done;
+ masm.movePtr(ImmPtr(cx->addressOfInlinedICScript()), scratch);
+ Address inlinedAddr(scratch, 0);
+ masm.branchPtr(Assembler::Equal, inlinedAddr, ImmWord(0), &notInlined);
+ masm.loadPtr(inlinedAddr, scratch2);
+ masm.storePtr(scratch2, frame.addressOfICScript());
+ masm.storePtr(ImmPtr(nullptr), inlinedAddr);
+ masm.jump(&done);
+
+ // Otherwise, store this script's default ICSCript in the frame.
+ masm.bind(&notInlined);
+ masm.storePtr(ImmPtr(handler.script()->jitScript()->icScript()),
+ frame.addressOfICScript());
+ masm.bind(&done);
+}
+
+template <>
+void BaselineInterpreterCodeGen::emitInitFrameFields(Register nonFunctionEnv) {
+ MOZ_ASSERT(nonFunctionEnv == R1.scratchReg(),
+ "Don't clobber nonFunctionEnv below");
+
+ // If we have a dedicated PC register we use it as scratch1 to avoid a
+ // register move below.
+ Register scratch1 =
+ HasInterpreterPCReg() ? InterpreterPCReg : R0.scratchReg();
+ Register scratch2 = R2.scratchReg();
+
+ masm.store32(Imm32(BaselineFrame::RUNNING_IN_INTERPRETER),
+ frame.addressOfFlags());
+
+ // Initialize interpreterScript.
+ Label notFunction, done;
+ masm.loadPtr(frame.addressOfCalleeToken(), scratch1);
+ masm.branchTestPtr(Assembler::NonZero, scratch1, Imm32(CalleeTokenScriptBit),
+ &notFunction);
+ {
+ // CalleeToken_Function or CalleeToken_FunctionConstructing.
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1);
+ masm.unboxObject(Address(scratch1, JSFunction::offsetOfEnvironment()),
+ scratch2);
+ masm.storePtr(scratch2, frame.addressOfEnvironmentChain());
+ masm.loadPrivate(Address(scratch1, JSFunction::offsetOfJitInfoOrScript()),
+ scratch1);
+ masm.jump(&done);
+ }
+ masm.bind(&notFunction);
+ {
+ // CalleeToken_Script.
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1);
+ masm.storePtr(nonFunctionEnv, frame.addressOfEnvironmentChain());
+ }
+ masm.bind(&done);
+ masm.storePtr(scratch1, frame.addressOfInterpreterScript());
+
+ // Initialize icScript and interpreterICEntry
+ masm.loadJitScript(scratch1, scratch2);
+ masm.computeEffectiveAddress(Address(scratch2, JitScript::offsetOfICScript()),
+ scratch2);
+ masm.storePtr(scratch2, frame.addressOfICScript());
+ masm.computeEffectiveAddress(Address(scratch2, ICScript::offsetOfICEntries()),
+ scratch2);
+ masm.storePtr(scratch2, frame.addressOfInterpreterICEntry());
+
+ // Initialize interpreter pc.
+ masm.loadPtr(Address(scratch1, JSScript::offsetOfSharedData()), scratch1);
+ masm.loadPtr(Address(scratch1, SharedImmutableScriptData::offsetOfISD()),
+ scratch1);
+ masm.addPtr(Imm32(ImmutableScriptData::offsetOfCode()), scratch1);
+
+ if (HasInterpreterPCReg()) {
+ MOZ_ASSERT(scratch1 == InterpreterPCReg,
+ "pc must be stored in the pc register");
+ } else {
+ masm.storePtr(scratch1, frame.addressOfInterpreterPC());
+ }
+}
+
+template <>
+template <typename F>
+bool BaselineCompilerCodeGen::initEnvironmentChainHelper(
+ const F& initFunctionEnv) {
+ if (handler.function()) {
+ return initFunctionEnv();
+ }
+ return true;
+}
+
+template <>
+template <typename F>
+bool BaselineInterpreterCodeGen::initEnvironmentChainHelper(
+ const F& initFunctionEnv) {
+ // For function scripts use the code emitted by initFunctionEnv. For other
+ // scripts this is a no-op.
+
+ Label done;
+ masm.branchTestPtr(Assembler::NonZero, frame.addressOfCalleeToken(),
+ Imm32(CalleeTokenScriptBit), &done);
+ {
+ if (!initFunctionEnv()) {
+ return false;
+ }
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::initEnvironmentChain() {
+ auto initFunctionEnv = [this]() {
+ auto initEnv = [this]() {
+ // Call into the VM to create the proper environment objects.
+ prepareVMCall();
+
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ pushArg(R0.scratchReg());
+
+ const CallVMPhase phase = CallVMPhase::BeforePushingLocals;
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ return callVMNonOp<Fn, jit::InitFunctionEnvironmentObjects>(phase);
+ };
+ return emitTestScriptFlag(
+ JSScript::ImmutableFlags::NeedsFunctionEnvironmentObjects, true,
+ initEnv, R2.scratchReg());
+ };
+
+ return initEnvironmentChainHelper(initFunctionEnv);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitInterruptCheck() {
+ frame.syncStack(0);
+
+ Label done;
+ masm.branch32(Assembler::Equal, AbsoluteAddress(cx->addressOfInterruptBits()),
+ Imm32(0), &done);
+
+ prepareVMCall();
+
+ // Use a custom RetAddrEntry::Kind so DebugModeOSR can distinguish this call
+ // from other callVMs that might happen at this pc.
+ const RetAddrEntry::Kind kind = RetAddrEntry::Kind::InterruptCheck;
+
+ using Fn = bool (*)(JSContext*);
+ if (!callVM<Fn, InterruptCheck>(kind)) {
+ return false;
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emitWarmUpCounterIncrement() {
+ frame.assertSyncedStack();
+
+ // Record native code offset for OSR from Baseline Interpreter into Baseline
+ // JIT code. This is right before the warm-up check in the Baseline JIT code,
+ // to make sure we can immediately enter Ion if the script is warm enough or
+ // if --ion-eager is used.
+ JSScript* script = handler.script();
+ jsbytecode* pc = handler.pc();
+ if (JSOp(*pc) == JSOp::LoopHead) {
+ uint32_t pcOffset = script->pcToOffset(pc);
+ uint32_t nativeOffset = masm.currentOffset();
+ if (!handler.osrEntries().emplaceBack(pcOffset, nativeOffset)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+ }
+
+ // Emit no warm-up counter increments if Ion is not enabled or if the script
+ // will never be Ion-compileable.
+ if (!handler.maybeIonCompileable()) {
+ return true;
+ }
+
+ Register scriptReg = R2.scratchReg();
+ Register countReg = R0.scratchReg();
+
+ // Load the ICScript* in scriptReg.
+ masm.loadPtr(frame.addressOfICScript(), scriptReg);
+
+ // Bump warm-up counter.
+ Address warmUpCounterAddr(scriptReg, ICScript::offsetOfWarmUpCount());
+ masm.load32(warmUpCounterAddr, countReg);
+ masm.add32(Imm32(1), countReg);
+ masm.store32(countReg, warmUpCounterAddr);
+
+ if (!JitOptions.disableInlining) {
+ // Consider trial inlining.
+ // Note: unlike other warmup thresholds, where we try to enter a
+ // higher tier whenever we are higher than a given warmup count,
+ // trial inlining triggers once when reaching the threshold.
+ Label noTrialInlining;
+ masm.branch32(Assembler::NotEqual, countReg,
+ Imm32(JitOptions.trialInliningWarmUpThreshold),
+ &noTrialInlining);
+ prepareVMCall();
+
+ masm.PushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ if (!callVMNonOp<Fn, DoTrialInlining>()) {
+ return false;
+ }
+ // Reload registers potentially clobbered by the call.
+ masm.loadPtr(frame.addressOfICScript(), scriptReg);
+ masm.load32(warmUpCounterAddr, countReg);
+ masm.bind(&noTrialInlining);
+ }
+
+ if (JSOp(*pc) == JSOp::LoopHead) {
+ // If this is a loop where we can't OSR (for example because it's inside a
+ // catch or finally block), increment the warmup counter but don't attempt
+ // OSR (Ion/Warp only compiles the try block).
+ if (!handler.analysis().info(pc).loopHeadCanOsr) {
+ return true;
+ }
+ }
+
+ Label done;
+
+ const OptimizationInfo* info =
+ IonOptimizations.get(OptimizationLevel::Normal);
+ uint32_t warmUpThreshold = info->compilerWarmUpThreshold(script, pc);
+ masm.branch32(Assembler::LessThan, countReg, Imm32(warmUpThreshold), &done);
+
+ // Don't trigger Warp compilations from trial-inlined scripts.
+ Address depthAddr(scriptReg, ICScript::offsetOfDepth());
+ masm.branch32(Assembler::NotEqual, depthAddr, Imm32(0), &done);
+
+ // Load the IonScript* in scriptReg. We can load this from the ICScript*
+ // because it must be an outer ICScript embedded in the JitScript.
+ constexpr int32_t offset = -int32_t(JitScript::offsetOfICScript()) +
+ int32_t(JitScript::offsetOfIonScript());
+ masm.loadPtr(Address(scriptReg, offset), scriptReg);
+
+ // Do nothing if Ion is already compiling this script off-thread or if Ion has
+ // been disabled for this script.
+ masm.branchPtr(Assembler::Equal, scriptReg, ImmPtr(IonCompilingScriptPtr),
+ &done);
+ masm.branchPtr(Assembler::Equal, scriptReg, ImmPtr(IonDisabledScriptPtr),
+ &done);
+
+ // Try to compile and/or finish a compilation.
+ if (JSOp(*pc) == JSOp::LoopHead) {
+ // Try to OSR into Ion.
+ computeFrameSize(R0.scratchReg());
+
+ prepareVMCall();
+
+ pushBytecodePCArg();
+ pushArg(R0.scratchReg());
+ masm.PushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, uint32_t, jsbytecode*,
+ IonOsrTempData**);
+ if (!callVM<Fn, IonCompileScriptForBaselineOSR>()) {
+ return false;
+ }
+
+ // The return register holds the IonOsrTempData*. Perform OSR if it's not
+ // nullptr.
+ static_assert(ReturnReg != OsrFrameReg,
+ "Code below depends on osrDataReg != OsrFrameReg");
+ Register osrDataReg = ReturnReg;
+ masm.branchTestPtr(Assembler::Zero, osrDataReg, osrDataReg, &done);
+
+ // Success! Switch from Baseline JIT code to Ion JIT code.
+
+ // At this point, stack looks like:
+ //
+ // +-> [...Calling-Frame...]
+ // | [...Actual-Args/ThisV/ArgCount/Callee...]
+ // | [Descriptor]
+ // | [Return-Addr]
+ // +---[Saved-FramePtr]
+ // [...Baseline-Frame...]
+
+#ifdef DEBUG
+ // Get a scratch register that's not osrDataReg or OsrFrameReg.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.take(osrDataReg);
+ regs.take(OsrFrameReg);
+
+ Register scratchReg = regs.takeAny();
+
+ // If profiler instrumentation is on, ensure that lastProfilingFrame is
+ // the frame currently being OSR-ed
+ {
+ Label checkOk;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &checkOk);
+ masm.loadPtr(AbsoluteAddress((void*)&cx->jitActivation), scratchReg);
+ masm.loadPtr(
+ Address(scratchReg, JitActivation::offsetOfLastProfilingFrame()),
+ scratchReg);
+
+ // It may be the case that we entered the baseline frame with
+ // profiling turned off on, then in a call within a loop (i.e. a
+ // callee frame), turn on profiling, then return to this frame,
+ // and then OSR with profiling turned on. In this case, allow for
+ // lastProfilingFrame to be null.
+ masm.branchPtr(Assembler::Equal, scratchReg, ImmWord(0), &checkOk);
+
+ masm.branchPtr(Assembler::Equal, FramePointer, scratchReg, &checkOk);
+ masm.assumeUnreachable("Baseline OSR lastProfilingFrame mismatch.");
+ masm.bind(&checkOk);
+ }
+#endif
+
+ // Restore the stack pointer so that the saved frame pointer is on top of
+ // the stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump into Ion.
+ masm.loadPtr(Address(osrDataReg, IonOsrTempData::offsetOfBaselineFrame()),
+ OsrFrameReg);
+ masm.jump(Address(osrDataReg, IonOsrTempData::offsetOfJitCode()));
+ } else {
+ prepareVMCall();
+
+ masm.PushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ if (!callVMNonOp<Fn, IonCompileScriptForBaselineAtEntry>()) {
+ return false;
+ }
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emitWarmUpCounterIncrement() {
+ Register scriptReg = R2.scratchReg();
+ Register countReg = R0.scratchReg();
+
+ // Load the JitScript* in scriptReg.
+ loadScript(scriptReg);
+ masm.loadJitScript(scriptReg, scriptReg);
+
+ // Bump warm-up counter.
+ Address warmUpCounterAddr(scriptReg, JitScript::offsetOfWarmUpCount());
+ masm.load32(warmUpCounterAddr, countReg);
+ masm.add32(Imm32(1), countReg);
+ masm.store32(countReg, warmUpCounterAddr);
+
+ // If the script is warm enough for Baseline compilation, call into the VM to
+ // compile it.
+ Label done;
+ masm.branch32(Assembler::BelowOrEqual, countReg,
+ Imm32(JitOptions.baselineJitWarmUpThreshold), &done);
+ masm.branchPtr(Assembler::Equal,
+ Address(scriptReg, JitScript::offsetOfBaselineScript()),
+ ImmPtr(BaselineDisabledScriptPtr), &done);
+ {
+ prepareVMCall();
+
+ masm.PushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, uint8_t**);
+ if (!callVM<Fn, BaselineCompileFromBaselineInterpreter>()) {
+ return false;
+ }
+
+ // If the function returned nullptr we either skipped compilation or were
+ // unable to compile the script. Continue running in the interpreter.
+ masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &done);
+
+ // Success! Switch from interpreter to JIT code by jumping to the
+ // corresponding code in the BaselineScript.
+ //
+ // This works because BaselineCompiler uses the same frame layout (stack is
+ // synced at OSR points) and BaselineCompileFromBaselineInterpreter has
+ // already cleared the RUNNING_IN_INTERPRETER flag for us.
+ // See BaselineFrame::prepareForBaselineInterpreterToJitOSR.
+ masm.jump(ReturnReg);
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool BaselineCompiler::emitDebugTrap() {
+ MOZ_ASSERT(compileDebugInstrumentation());
+ MOZ_ASSERT(frame.numUnsyncedSlots() == 0);
+
+ JSScript* script = handler.script();
+ bool enabled = DebugAPI::stepModeEnabled(script) ||
+ DebugAPI::hasBreakpointsAt(script, handler.pc());
+
+ // Emit patchable call to debug trap handler.
+ JitCode* handlerCode = cx->runtime()->jitRuntime()->debugTrapHandler(
+ cx, DebugTrapHandlerKind::Compiler);
+ if (!handlerCode) {
+ return false;
+ }
+
+ CodeOffset nativeOffset = masm.toggledCall(handlerCode, enabled);
+
+ uint32_t pcOffset = script->pcToOffset(handler.pc());
+ if (!debugTrapEntries_.emplaceBack(pcOffset, nativeOffset.offset())) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ // Add a RetAddrEntry for the return offset -> pc mapping.
+ return handler.recordCallRetAddr(cx, RetAddrEntry::Kind::DebugTrap,
+ masm.currentOffset());
+}
+
+template <typename Handler>
+void BaselineCodeGen<Handler>::emitProfilerEnterFrame() {
+ // Store stack position to lastProfilingFrame variable, guarded by a toggled
+ // jump. Starts off initially disabled.
+ Label noInstrument;
+ CodeOffset toggleOffset = masm.toggledJump(&noInstrument);
+ masm.profilerEnterFrame(FramePointer, R0.scratchReg());
+ masm.bind(&noInstrument);
+
+ // Store the start offset in the appropriate location.
+ MOZ_ASSERT(!profilerEnterFrameToggleOffset_.bound());
+ profilerEnterFrameToggleOffset_ = toggleOffset;
+}
+
+template <typename Handler>
+void BaselineCodeGen<Handler>::emitProfilerExitFrame() {
+ // Store previous frame to lastProfilingFrame variable, guarded by a toggled
+ // jump. Starts off initially disabled.
+ Label noInstrument;
+ CodeOffset toggleOffset = masm.toggledJump(&noInstrument);
+ masm.profilerExitFrame();
+ masm.bind(&noInstrument);
+
+ // Store the start offset in the appropriate location.
+ MOZ_ASSERT(!profilerExitFrameToggleOffset_.bound());
+ profilerExitFrameToggleOffset_ = toggleOffset;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Nop() {
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_NopDestructuring() {
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_TryDestructuring() {
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Pop() {
+ frame.pop();
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_PopN() {
+ frame.popn(GET_UINT16(handler.pc()));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_PopN() {
+ LoadUint16Operand(masm, R0.scratchReg());
+ frame.popn(R0.scratchReg());
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_DupAt() {
+ frame.syncStack(0);
+
+ // DupAt takes a value on the stack and re-pushes it on top. It's like
+ // GetLocal but it addresses from the top of the stack instead of from the
+ // stack frame.
+
+ int depth = -(GET_UINT24(handler.pc()) + 1);
+ masm.loadValue(frame.addressOfStackValue(depth), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_DupAt() {
+ LoadUint24Operand(masm, 0, R0.scratchReg());
+ masm.loadValue(frame.addressOfStackValue(R0.scratchReg()), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Dup() {
+ // Keep top stack value in R0, sync the rest so that we can use R1. We use
+ // separate registers because every register can be used by at most one
+ // StackValue.
+ frame.popRegsAndSync(1);
+ masm.moveValue(R0, R1);
+
+ // inc/dec ops use Dup followed by Inc/Dec. Push R0 last to avoid a move.
+ frame.push(R1);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Dup2() {
+ frame.syncStack(0);
+
+ masm.loadValue(frame.addressOfStackValue(-2), R0);
+ masm.loadValue(frame.addressOfStackValue(-1), R1);
+
+ frame.push(R0);
+ frame.push(R1);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Swap() {
+ // Keep top stack values in R0 and R1.
+ frame.popRegsAndSync(2);
+
+ frame.push(R1);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Pick() {
+ frame.syncStack(0);
+
+ // Pick takes a value on the stack and moves it to the top.
+ // For instance, pick 2:
+ // before: A B C D E
+ // after : A B D E C
+
+ // First, move value at -(amount + 1) into R0.
+ int32_t depth = -(GET_INT8(handler.pc()) + 1);
+ masm.loadValue(frame.addressOfStackValue(depth), R0);
+
+ // Move the other values down.
+ depth++;
+ for (; depth < 0; depth++) {
+ Address source = frame.addressOfStackValue(depth);
+ Address dest = frame.addressOfStackValue(depth - 1);
+ masm.loadValue(source, R1);
+ masm.storeValue(R1, dest);
+ }
+
+ // Push R0.
+ frame.pop();
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Pick() {
+ // First, move the value to move up into R0.
+ Register scratch = R2.scratchReg();
+ LoadUint8Operand(masm, scratch);
+ masm.loadValue(frame.addressOfStackValue(scratch), R0);
+
+ // Move the other values down.
+ Label top, done;
+ masm.bind(&top);
+ masm.branchSub32(Assembler::Signed, Imm32(1), scratch, &done);
+ {
+ masm.loadValue(frame.addressOfStackValue(scratch), R1);
+ masm.storeValue(R1, frame.addressOfStackValue(scratch, sizeof(Value)));
+ masm.jump(&top);
+ }
+
+ masm.bind(&done);
+
+ // Replace value on top of the stack with R0.
+ masm.storeValue(R0, frame.addressOfStackValue(-1));
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Unpick() {
+ frame.syncStack(0);
+
+ // Pick takes the top of the stack value and moves it under the nth value.
+ // For instance, unpick 2:
+ // before: A B C D E
+ // after : A B E C D
+
+ // First, move value at -1 into R0.
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ MOZ_ASSERT(GET_INT8(handler.pc()) > 0,
+ "Interpreter code assumes JSOp::Unpick operand > 0");
+
+ // Move the other values up.
+ int32_t depth = -(GET_INT8(handler.pc()) + 1);
+ for (int32_t i = -1; i > depth; i--) {
+ Address source = frame.addressOfStackValue(i - 1);
+ Address dest = frame.addressOfStackValue(i);
+ masm.loadValue(source, R1);
+ masm.storeValue(R1, dest);
+ }
+
+ // Store R0 under the nth value.
+ Address dest = frame.addressOfStackValue(depth);
+ masm.storeValue(R0, dest);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Unpick() {
+ Register scratch = R2.scratchReg();
+ LoadUint8Operand(masm, scratch);
+
+ // Move the top value into R0.
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ // Overwrite the nth stack value with R0 but first save the old value in R1.
+ masm.loadValue(frame.addressOfStackValue(scratch), R1);
+ masm.storeValue(R0, frame.addressOfStackValue(scratch));
+
+ // Now for each slot x in [n-1, 1] do the following:
+ //
+ // * Store the value in slot x in R0.
+ // * Store the value in the previous slot (now in R1) in slot x.
+ // * Move R0 to R1.
+
+#ifdef DEBUG
+ // Assert the operand > 0 so the branchSub32 below doesn't "underflow" to
+ // negative values.
+ {
+ Label ok;
+ masm.branch32(Assembler::GreaterThan, scratch, Imm32(0), &ok);
+ masm.assumeUnreachable("JSOp::Unpick with operand <= 0?");
+ masm.bind(&ok);
+ }
+#endif
+
+ Label top, done;
+ masm.bind(&top);
+ masm.branchSub32(Assembler::Zero, Imm32(1), scratch, &done);
+ {
+ // Overwrite stack slot x with slot x + 1, saving the old value in R1.
+ masm.loadValue(frame.addressOfStackValue(scratch), R0);
+ masm.storeValue(R1, frame.addressOfStackValue(scratch));
+ masm.moveValue(R0, R1);
+ masm.jump(&top);
+ }
+
+ // Finally, replace the value on top of the stack (slot 0) with R1. This is
+ // the value that used to be in slot 1.
+ masm.bind(&done);
+ masm.storeValue(R1, frame.addressOfStackValue(-1));
+ return true;
+}
+
+template <>
+void BaselineCompilerCodeGen::emitJump() {
+ jsbytecode* pc = handler.pc();
+ MOZ_ASSERT(IsJumpOpcode(JSOp(*pc)));
+ frame.assertSyncedStack();
+
+ jsbytecode* target = pc + GET_JUMP_OFFSET(pc);
+ masm.jump(handler.labelOf(target));
+}
+
+template <>
+void BaselineInterpreterCodeGen::emitJump() {
+ // We have to add the current pc's jump offset to the current pc. We can use
+ // R0 and R1 as scratch because we jump to the "next op" label so these
+ // registers aren't in use at this point.
+ Register scratch1 = R0.scratchReg();
+ Register scratch2 = R1.scratchReg();
+ Register pc = LoadBytecodePC(masm, scratch1);
+ LoadInt32OperandSignExtendToPtr(masm, pc, scratch2);
+ if (HasInterpreterPCReg()) {
+ masm.addPtr(scratch2, InterpreterPCReg);
+ } else {
+ masm.addPtr(pc, scratch2);
+ masm.storePtr(scratch2, frame.addressOfInterpreterPC());
+ }
+ masm.jump(handler.interpretOpWithPCRegLabel());
+}
+
+template <>
+void BaselineCompilerCodeGen::emitTestBooleanTruthy(bool branchIfTrue,
+ ValueOperand val) {
+ jsbytecode* pc = handler.pc();
+ MOZ_ASSERT(IsJumpOpcode(JSOp(*pc)));
+ frame.assertSyncedStack();
+
+ jsbytecode* target = pc + GET_JUMP_OFFSET(pc);
+ masm.branchTestBooleanTruthy(branchIfTrue, val, handler.labelOf(target));
+}
+
+template <>
+void BaselineInterpreterCodeGen::emitTestBooleanTruthy(bool branchIfTrue,
+ ValueOperand val) {
+ Label done;
+ masm.branchTestBooleanTruthy(!branchIfTrue, val, &done);
+ emitJump();
+ masm.bind(&done);
+}
+
+template <>
+template <typename F1, typename F2>
+[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag(
+ JSScript::ImmutableFlags flag, const F1& ifSet, const F2& ifNotSet,
+ Register scratch) {
+ if (handler.script()->hasFlag(flag)) {
+ return ifSet();
+ }
+ return ifNotSet();
+}
+
+template <>
+template <typename F1, typename F2>
+[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag(
+ JSScript::ImmutableFlags flag, const F1& ifSet, const F2& ifNotSet,
+ Register scratch) {
+ Label flagNotSet, done;
+ loadScript(scratch);
+ masm.branchTest32(Assembler::Zero,
+ Address(scratch, JSScript::offsetOfImmutableFlags()),
+ Imm32(uint32_t(flag)), &flagNotSet);
+ {
+ if (!ifSet()) {
+ return false;
+ }
+ masm.jump(&done);
+ }
+ masm.bind(&flagNotSet);
+ {
+ if (!ifNotSet()) {
+ return false;
+ }
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <>
+template <typename F>
+[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag(
+ JSScript::ImmutableFlags flag, bool value, const F& emit,
+ Register scratch) {
+ if (handler.script()->hasFlag(flag) == value) {
+ return emit();
+ }
+ return true;
+}
+
+template <>
+template <typename F>
+[[nodiscard]] bool BaselineCompilerCodeGen::emitTestScriptFlag(
+ JSScript::MutableFlags flag, bool value, const F& emit, Register scratch) {
+ if (handler.script()->hasFlag(flag) == value) {
+ return emit();
+ }
+ return true;
+}
+
+template <>
+template <typename F>
+[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag(
+ JSScript::ImmutableFlags flag, bool value, const F& emit,
+ Register scratch) {
+ Label done;
+ loadScript(scratch);
+ masm.branchTest32(value ? Assembler::Zero : Assembler::NonZero,
+ Address(scratch, JSScript::offsetOfImmutableFlags()),
+ Imm32(uint32_t(flag)), &done);
+ {
+ if (!emit()) {
+ return false;
+ }
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <>
+template <typename F>
+[[nodiscard]] bool BaselineInterpreterCodeGen::emitTestScriptFlag(
+ JSScript::MutableFlags flag, bool value, const F& emit, Register scratch) {
+ Label done;
+ loadScript(scratch);
+ masm.branchTest32(value ? Assembler::Zero : Assembler::NonZero,
+ Address(scratch, JSScript::offsetOfMutableFlags()),
+ Imm32(uint32_t(flag)), &done);
+ {
+ if (!emit()) {
+ return false;
+ }
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Goto() {
+ frame.syncStack(0);
+ emitJump();
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitTest(bool branchIfTrue) {
+ bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN);
+
+ // Keep top stack value in R0.
+ frame.popRegsAndSync(1);
+
+ if (!knownBoolean && !emitNextIC()) {
+ return false;
+ }
+
+ // IC will leave a BooleanValue in R0, just need to branch on it.
+ emitTestBooleanTruthy(branchIfTrue, R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_JumpIfFalse() {
+ return emitTest(false);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_JumpIfTrue() {
+ return emitTest(true);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitAndOr(bool branchIfTrue) {
+ bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN);
+
+ // And and Or leave the original value on the stack.
+ frame.syncStack(0);
+
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+ if (!knownBoolean && !emitNextIC()) {
+ return false;
+ }
+
+ emitTestBooleanTruthy(branchIfTrue, R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_And() {
+ return emitAndOr(false);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Or() {
+ return emitAndOr(true);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Coalesce() {
+ // Coalesce leaves the original value on the stack.
+ frame.syncStack(0);
+
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ Label undefinedOrNull;
+
+ masm.branchTestUndefined(Assembler::Equal, R0, &undefinedOrNull);
+ masm.branchTestNull(Assembler::Equal, R0, &undefinedOrNull);
+ emitJump();
+
+ masm.bind(&undefinedOrNull);
+ // fall through
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Not() {
+ bool knownBoolean = frame.stackValueHasKnownType(-1, JSVAL_TYPE_BOOLEAN);
+
+ // Keep top stack value in R0.
+ frame.popRegsAndSync(1);
+
+ if (!knownBoolean && !emitNextIC()) {
+ return false;
+ }
+
+ masm.notBoolean(R0);
+
+ frame.push(R0, JSVAL_TYPE_BOOLEAN);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Pos() {
+ return emitUnaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_ToNumeric() {
+ return emitUnaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_LoopHead() {
+ if (!emit_JumpTarget()) {
+ return false;
+ }
+ if (!emitInterruptCheck()) {
+ return false;
+ }
+ if (!emitWarmUpCounterIncrement()) {
+ return false;
+ }
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Void() {
+ frame.pop();
+ frame.push(UndefinedValue());
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Undefined() {
+ frame.push(UndefinedValue());
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Hole() {
+ frame.push(MagicValue(JS_ELEMENTS_HOLE));
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Null() {
+ frame.push(NullValue());
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckIsObj() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ Label ok;
+ masm.branchTestObject(Assembler::Equal, R0, &ok);
+
+ prepareVMCall();
+
+ pushUint8BytecodeOperandArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, CheckIsObjectKind);
+ if (!callVM<Fn, ThrowCheckIsObject>()) {
+ return false;
+ }
+
+ masm.bind(&ok);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckThis() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ return emitCheckThis(R0);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckThisReinit() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ return emitCheckThis(R0, /* reinit = */ true);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitCheckThis(ValueOperand val, bool reinit) {
+ Label thisOK;
+ if (reinit) {
+ masm.branchTestMagic(Assembler::Equal, val, &thisOK);
+ } else {
+ masm.branchTestMagic(Assembler::NotEqual, val, &thisOK);
+ }
+
+ prepareVMCall();
+
+ if (reinit) {
+ using Fn = bool (*)(JSContext*);
+ if (!callVM<Fn, ThrowInitializedThis>()) {
+ return false;
+ }
+ } else {
+ using Fn = bool (*)(JSContext*);
+ if (!callVM<Fn, ThrowUninitializedThis>()) {
+ return false;
+ }
+ }
+
+ masm.bind(&thisOK);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckReturn() {
+ MOZ_ASSERT_IF(handler.maybeScript(),
+ handler.maybeScript()->isDerivedClassConstructor());
+
+ // Load |this| in R0, return value in R1.
+ frame.popRegsAndSync(1);
+ emitLoadReturnValue(R1);
+
+ Label done, returnBad, checkThis;
+ masm.branchTestObject(Assembler::NotEqual, R1, &checkThis);
+ {
+ masm.moveValue(R1, R0);
+ masm.jump(&done);
+ }
+ masm.bind(&checkThis);
+ masm.branchTestUndefined(Assembler::NotEqual, R1, &returnBad);
+ masm.branchTestMagic(Assembler::NotEqual, R0, &done);
+ masm.bind(&returnBad);
+
+ prepareVMCall();
+ pushArg(R1);
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ if (!callVM<Fn, ThrowBadDerivedReturnOrUninitializedThis>()) {
+ return false;
+ }
+ masm.assumeUnreachable("Should throw on bad derived constructor return");
+
+ masm.bind(&done);
+
+ // Push |rval| or |this| onto the stack.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_FunctionThis() {
+ MOZ_ASSERT_IF(handler.maybeFunction(), !handler.maybeFunction()->isArrow());
+
+ frame.pushThis();
+
+ auto boxThis = [this]() {
+ // Load |thisv| in R0. Skip the call if it's already an object.
+ Label skipCall;
+ frame.popRegsAndSync(1);
+ masm.branchTestObject(Assembler::Equal, R0, &skipCall);
+
+ prepareVMCall();
+ masm.loadBaselineFramePtr(FramePointer, R1.scratchReg());
+
+ pushArg(R1.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, MutableHandleValue);
+ if (!callVM<Fn, BaselineGetFunctionThis>()) {
+ return false;
+ }
+
+ masm.bind(&skipCall);
+ frame.push(R0);
+ return true;
+ };
+
+ // In strict mode code, |this| is left alone.
+ return emitTestScriptFlag(JSScript::ImmutableFlags::Strict, false, boxThis,
+ R2.scratchReg());
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GlobalThis() {
+ frame.syncStack(0);
+
+ loadGlobalThisValue(R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_NonSyntacticGlobalThis() {
+ frame.syncStack(0);
+
+ prepareVMCall();
+
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = void (*)(JSContext*, HandleObject, MutableHandleValue);
+ if (!callVM<Fn, GetNonSyntacticGlobalThis>()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_True() {
+ frame.push(BooleanValue(true));
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_False() {
+ frame.push(BooleanValue(false));
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Zero() {
+ frame.push(Int32Value(0));
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_One() {
+ frame.push(Int32Value(1));
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Int8() {
+ frame.push(Int32Value(GET_INT8(handler.pc())));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Int8() {
+ LoadInt8Operand(masm, R0.scratchReg());
+ masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Int32() {
+ frame.push(Int32Value(GET_INT32(handler.pc())));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Int32() {
+ LoadInt32Operand(masm, R0.scratchReg());
+ masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Uint16() {
+ frame.push(Int32Value(GET_UINT16(handler.pc())));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Uint16() {
+ LoadUint16Operand(masm, R0.scratchReg());
+ masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Uint24() {
+ frame.push(Int32Value(GET_UINT24(handler.pc())));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Uint24() {
+ LoadUint24Operand(masm, 0, R0.scratchReg());
+ masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Double() {
+ frame.push(GET_INLINE_VALUE(handler.pc()));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Double() {
+ LoadInlineValueOperand(masm, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_BigInt() {
+ BigInt* bi = handler.script()->getBigInt(handler.pc());
+ frame.push(BigIntValue(bi));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_BigInt() {
+ Register scratch1 = R0.scratchReg();
+ Register scratch2 = R1.scratchReg();
+ loadScriptGCThing(ScriptGCThingType::BigInt, scratch1, scratch2);
+ masm.tagValue(JSVAL_TYPE_BIGINT, scratch1, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_String() {
+ frame.push(StringValue(handler.script()->getString(handler.pc())));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_String() {
+ Register scratch1 = R0.scratchReg();
+ Register scratch2 = R1.scratchReg();
+ loadScriptGCThing(ScriptGCThingType::String, scratch1, scratch2);
+ masm.tagValue(JSVAL_TYPE_STRING, scratch1, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Symbol() {
+ unsigned which = GET_UINT8(handler.pc());
+ JS::Symbol* sym = cx->runtime()->wellKnownSymbols->get(which);
+ frame.push(SymbolValue(sym));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Symbol() {
+ Register scratch1 = R0.scratchReg();
+ Register scratch2 = R1.scratchReg();
+ LoadUint8Operand(masm, scratch1);
+
+ masm.movePtr(ImmPtr(cx->runtime()->wellKnownSymbols), scratch2);
+ masm.loadPtr(BaseIndex(scratch2, scratch1, ScalePointer), scratch1);
+
+ masm.tagValue(JSVAL_TYPE_SYMBOL, scratch1, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_Object() {
+ frame.push(ObjectValue(*handler.script()->getObject(handler.pc())));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_Object() {
+ Register scratch1 = R0.scratchReg();
+ Register scratch2 = R1.scratchReg();
+ loadScriptGCThing(ScriptGCThingType::Object, scratch1, scratch2);
+ masm.tagValue(JSVAL_TYPE_OBJECT, scratch1, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CallSiteObj() {
+ return emit_Object();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_RegExp() {
+ prepareVMCall();
+ pushScriptGCThingArg(ScriptGCThingType::RegExp, R0.scratchReg(),
+ R1.scratchReg());
+
+ using Fn = JSObject* (*)(JSContext*, Handle<RegExpObject*>);
+ if (!callVM<Fn, CloneRegExpObject>()) {
+ return false;
+ }
+
+ // Box and push return value.
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+#ifdef ENABLE_RECORD_TUPLE
+# define UNSUPPORTED_OPCODE(OP) \
+ template <typename Handler> \
+ bool BaselineCodeGen<Handler>::emit_##OP() { \
+ MOZ_CRASH("Record and Tuple are not supported by jit"); \
+ return false; \
+ }
+
+UNSUPPORTED_OPCODE(InitRecord)
+UNSUPPORTED_OPCODE(AddRecordProperty)
+UNSUPPORTED_OPCODE(AddRecordSpread)
+UNSUPPORTED_OPCODE(FinishRecord)
+UNSUPPORTED_OPCODE(InitTuple)
+UNSUPPORTED_OPCODE(AddTupleElement)
+UNSUPPORTED_OPCODE(FinishTuple)
+
+# undef UNSUPPORTED_OPCODE
+#endif
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Lambda() {
+ prepareVMCall();
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+
+ pushArg(R0.scratchReg());
+ pushScriptGCThingArg(ScriptGCThingType::Function, R0.scratchReg(),
+ R1.scratchReg());
+
+ using Fn = JSObject* (*)(JSContext*, HandleFunction, HandleObject);
+ if (!callVM<Fn, js::Lambda>()) {
+ return false;
+ }
+
+ // Box and push return value.
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetFunName() {
+ frame.popRegsAndSync(2);
+
+ frame.push(R0);
+ frame.syncStack(0);
+
+ masm.unboxObject(R0, R0.scratchReg());
+
+ prepareVMCall();
+
+ pushUint8BytecodeOperandArg(R2.scratchReg());
+ pushArg(R1);
+ pushArg(R0.scratchReg());
+
+ using Fn =
+ bool (*)(JSContext*, HandleFunction, HandleValue, FunctionPrefixKind);
+ return callVM<Fn, SetFunctionName>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_BitOr() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_BitXor() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_BitAnd() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Lsh() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Rsh() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Ursh() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Add() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Sub() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Mul() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Div() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Mod() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Pow() {
+ return emitBinaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitBinaryArith() {
+ // Keep top JSStack value in R0 and R2
+ frame.popRegsAndSync(2);
+
+ // Call IC
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitUnaryArith() {
+ // Keep top stack value in R0.
+ frame.popRegsAndSync(1);
+
+ // Call IC
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_BitNot() {
+ return emitUnaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Neg() {
+ return emitUnaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Inc() {
+ return emitUnaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Dec() {
+ return emitUnaryArith();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Lt() {
+ return emitCompare();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Le() {
+ return emitCompare();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Gt() {
+ return emitCompare();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Ge() {
+ return emitCompare();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Eq() {
+ return emitCompare();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Ne() {
+ return emitCompare();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitCompare() {
+ // CODEGEN
+
+ // Keep top JSStack value in R0 and R1.
+ frame.popRegsAndSync(2);
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0, JSVAL_TYPE_BOOLEAN);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictEq() {
+ return emitCompare();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictNe() {
+ return emitCompare();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Case() {
+ frame.popRegsAndSync(1);
+
+ Label done;
+ masm.branchTestBooleanTruthy(/* branchIfTrue */ false, R0, &done);
+ {
+ // Pop the switch value if the case matches.
+ masm.addToStackPtr(Imm32(sizeof(Value)));
+ emitJump();
+ }
+ masm.bind(&done);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Default() {
+ frame.pop();
+ return emit_Goto();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Lineno() {
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_NewArray() {
+ frame.syncStack(0);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+static void MarkElementsNonPackedIfHoleValue(MacroAssembler& masm,
+ Register elements,
+ ValueOperand val) {
+ Label notHole;
+ masm.branchTestMagic(Assembler::NotEqual, val, &notHole);
+ {
+ Address elementsFlags(elements, ObjectElements::offsetOfFlags());
+ masm.or32(Imm32(ObjectElements::NON_PACKED), elementsFlags);
+ }
+ masm.bind(&notHole);
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_InitElemArray() {
+ // Pop value into R0, keep the object on the stack.
+ frame.popRegsAndSync(1);
+
+ // Load object in R2.
+ Register obj = R2.scratchReg();
+ masm.unboxObject(frame.addressOfStackValue(-1), obj);
+
+ // Load index in R1.
+ Register index = R1.scratchReg();
+ LoadInt32Operand(masm, index);
+
+ // Store the Value. No pre-barrier because this is an initialization.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), obj);
+ masm.storeValue(R0, BaseObjectElementIndex(obj, index));
+
+ // Bump initialized length.
+ Address initLength(obj, ObjectElements::offsetOfInitializedLength());
+ masm.add32(Imm32(1), index);
+ masm.store32(index, initLength);
+
+ // Mark elements as NON_PACKED if we stored the hole value.
+ MarkElementsNonPackedIfHoleValue(masm, obj, R0);
+
+ // Post-barrier.
+ Label skipBarrier;
+ Register scratch = index;
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, scratch, &skipBarrier);
+ {
+ masm.unboxObject(frame.addressOfStackValue(-1), obj);
+ masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch, &skipBarrier);
+ MOZ_ASSERT(obj == R2.scratchReg(), "post barrier expects object in R2");
+ masm.call(&postBarrierSlot_);
+ }
+ masm.bind(&skipBarrier);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_InitElemArray() {
+ // Pop value into R0, keep the object on the stack.
+ Maybe<Value> knownValue = frame.knownStackValue(-1);
+ frame.popRegsAndSync(1);
+
+ // Load object in R2.
+ Register obj = R2.scratchReg();
+ masm.unboxObject(frame.addressOfStackValue(-1), obj);
+
+ uint32_t index = GET_UINT32(handler.pc());
+ MOZ_ASSERT(index <= INT32_MAX,
+ "the bytecode emitter must fail to compile code that would "
+ "produce an index exceeding int32_t range");
+
+ // Store the Value. No pre-barrier because this is an initialization.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), obj);
+ masm.storeValue(R0, Address(obj, index * sizeof(Value)));
+
+ // Bump initialized length.
+ Address initLength(obj, ObjectElements::offsetOfInitializedLength());
+ masm.store32(Imm32(index + 1), initLength);
+
+ // Mark elements as NON_PACKED if we stored the hole value. We know this
+ // statically except when debugger instrumentation is enabled because that
+ // forces a stack-sync (which discards constants and known types) for each op.
+ if (knownValue && knownValue->isMagic(JS_ELEMENTS_HOLE)) {
+ Address elementsFlags(obj, ObjectElements::offsetOfFlags());
+ masm.or32(Imm32(ObjectElements::NON_PACKED), elementsFlags);
+ } else if (handler.compileDebugInstrumentation()) {
+ MarkElementsNonPackedIfHoleValue(masm, obj, R0);
+ } else {
+#ifdef DEBUG
+ Label notHole;
+ masm.branchTestMagic(Assembler::NotEqual, R0, &notHole);
+ masm.assumeUnreachable("Unexpected hole value");
+ masm.bind(&notHole);
+#endif
+ }
+
+ // Post-barrier.
+ if (knownValue) {
+ MOZ_ASSERT(JS::GCPolicy<Value>::isTenured(*knownValue));
+ } else {
+ Label skipBarrier;
+ Register scratch = R1.scratchReg();
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, scratch,
+ &skipBarrier);
+ {
+ masm.unboxObject(frame.addressOfStackValue(-1), obj);
+ masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch,
+ &skipBarrier);
+ MOZ_ASSERT(obj == R2.scratchReg(), "post barrier expects object in R2");
+ masm.call(&postBarrierSlot_);
+ }
+ masm.bind(&skipBarrier);
+ }
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_NewObject() {
+ return emitNewObject();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_NewInit() {
+ return emitNewObject();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitNewObject() {
+ frame.syncStack(0);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitElem() {
+ // Store RHS in the scratch slot.
+ frame.storeStackValue(-1, frame.addressOfScratchValue(), R2);
+ frame.pop();
+
+ // Keep object and index in R0 and R1.
+ frame.popRegsAndSync(2);
+
+ // Push the object to store the result of the IC.
+ frame.push(R0);
+ frame.syncStack(0);
+
+ // Keep RHS on the stack.
+ frame.pushScratchValue();
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Pop the rhs, so that the object is on the top of the stack.
+ frame.pop();
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitHiddenElem() {
+ return emit_InitElem();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitLockedElem() {
+ return emit_InitElem();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_MutateProto() {
+ // Keep values on the stack for the decompiler.
+ frame.syncStack(0);
+
+ masm.unboxObject(frame.addressOfStackValue(-2), R0.scratchReg());
+ masm.loadValue(frame.addressOfStackValue(-1), R1);
+
+ prepareVMCall();
+
+ pushArg(R1);
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, Handle<PlainObject*>, HandleValue);
+ if (!callVM<Fn, MutatePrototype>()) {
+ return false;
+ }
+
+ frame.pop();
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitProp() {
+ // Load lhs in R0, rhs in R1.
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-2), R0);
+ masm.loadValue(frame.addressOfStackValue(-1), R1);
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Leave the object on the stack.
+ frame.pop();
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitLockedProp() {
+ return emit_InitProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitHiddenProp() {
+ return emit_InitProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetElem() {
+ // Keep top two stack values in R0 and R1.
+ frame.popRegsAndSync(2);
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetElemSuper() {
+ // Store obj in the scratch slot.
+ frame.storeStackValue(-1, frame.addressOfScratchValue(), R2);
+ frame.pop();
+
+ // Keep receiver and index in R0 and R1.
+ frame.popRegsAndSync(2);
+
+ // Keep obj on the stack.
+ frame.pushScratchValue();
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.pop();
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetElem() {
+ // Store RHS in the scratch slot.
+ frame.storeStackValue(-1, frame.addressOfScratchValue(), R2);
+ frame.pop();
+
+ // Keep object and index in R0 and R1.
+ frame.popRegsAndSync(2);
+
+ // Keep RHS on the stack.
+ frame.pushScratchValue();
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictSetElem() {
+ return emit_SetElem();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitSetElemSuper(bool strict) {
+ // Incoming stack is |receiver, propval, obj, rval|. We need to shuffle
+ // stack to leave rval when operation is complete.
+
+ // Pop rval into R0, then load receiver into R1 and replace with rval.
+ frame.popRegsAndSync(1);
+ masm.loadValue(frame.addressOfStackValue(-3), R1);
+ masm.storeValue(R0, frame.addressOfStackValue(-3));
+
+ prepareVMCall();
+
+ pushArg(Imm32(strict));
+ pushArg(R0); // rval
+ masm.loadValue(frame.addressOfStackValue(-2), R0);
+ pushArg(R0); // propval
+ pushArg(R1); // receiver
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+ pushArg(R0); // obj
+
+ using Fn = bool (*)(JSContext*, HandleValue, HandleValue, HandleValue,
+ HandleValue, bool);
+ if (!callVM<Fn, js::SetElementSuper>()) {
+ return false;
+ }
+
+ frame.popn(2);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetElemSuper() {
+ return emitSetElemSuper(/* strict = */ false);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictSetElemSuper() {
+ return emitSetElemSuper(/* strict = */ true);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitDelElem(bool strict) {
+ // Keep values on the stack for the decompiler.
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-2), R0);
+ masm.loadValue(frame.addressOfStackValue(-1), R1);
+
+ prepareVMCall();
+
+ pushArg(R1);
+ pushArg(R0);
+
+ using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*);
+ if (strict) {
+ if (!callVM<Fn, DelElemOperation<true>>()) {
+ return false;
+ }
+ } else {
+ if (!callVM<Fn, DelElemOperation<false>>()) {
+ return false;
+ }
+ }
+
+ masm.boxNonDouble(JSVAL_TYPE_BOOLEAN, ReturnReg, R1);
+ frame.popn(2);
+ frame.push(R1);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_DelElem() {
+ return emitDelElem(/* strict = */ false);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictDelElem() {
+ return emitDelElem(/* strict = */ true);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_In() {
+ frame.popRegsAndSync(2);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_HasOwn() {
+ frame.popRegsAndSync(2);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckPrivateField() {
+ // Keep key and val on the stack.
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-2), R0);
+ masm.loadValue(frame.addressOfStackValue(-1), R1);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_NewPrivateName() {
+ prepareVMCall();
+
+ pushScriptNameArg(R0.scratchReg(), R1.scratchReg());
+
+ using Fn = JS::Symbol* (*)(JSContext*, Handle<JSAtom*>);
+ if (!callVM<Fn, NewPrivateName>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_SYMBOL, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::tryOptimizeGetGlobalName() {
+ PropertyName* name = handler.script()->getName(handler.pc());
+
+ // These names are non-configurable on the global and cannot be shadowed.
+ if (name == cx->names().undefined) {
+ frame.push(UndefinedValue());
+ return true;
+ }
+ if (name == cx->names().NaN) {
+ frame.push(JS::NaNValue());
+ return true;
+ }
+ if (name == cx->names().Infinity) {
+ frame.push(JS::InfinityValue());
+ return true;
+ }
+
+ return false;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::tryOptimizeGetGlobalName() {
+ // Interpreter doesn't optimize simple GETGNAMEs.
+ return false;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetGName() {
+ if (tryOptimizeGetGlobalName()) {
+ return true;
+ }
+
+ frame.syncStack(0);
+
+ loadGlobalLexicalEnvironment(R0.scratchReg());
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::tryOptimizeBindGlobalName() {
+ JSScript* script = handler.script();
+ MOZ_ASSERT(!script->hasNonSyntacticScope());
+
+ Rooted<GlobalObject*> global(cx, &script->global());
+ Rooted<PropertyName*> name(cx, script->getName(handler.pc()));
+ if (JSObject* binding = MaybeOptimizeBindGlobalName(cx, global, name)) {
+ frame.push(ObjectValue(*binding));
+ return true;
+ }
+ return false;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::tryOptimizeBindGlobalName() {
+ // Interpreter doesn't optimize simple BindGNames.
+ return false;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_BindGName() {
+ if (tryOptimizeBindGlobalName()) {
+ return true;
+ }
+
+ frame.syncStack(0);
+ loadGlobalLexicalEnvironment(R0.scratchReg());
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_BindVar() {
+ frame.syncStack(0);
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+
+ prepareVMCall();
+ pushArg(R0.scratchReg());
+
+ using Fn = JSObject* (*)(JSContext*, JSObject*);
+ if (!callVM<Fn, BindVarOperation>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetProp() {
+ // Keep lhs in R0, rhs in R1.
+ frame.popRegsAndSync(2);
+
+ // Keep RHS on the stack.
+ frame.push(R1);
+ frame.syncStack(0);
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictSetProp() {
+ return emit_SetProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetName() {
+ return emit_SetProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictSetName() {
+ return emit_SetProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetGName() {
+ return emit_SetProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictSetGName() {
+ return emit_SetProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitSetPropSuper(bool strict) {
+ // Incoming stack is |receiver, obj, rval|. We need to shuffle stack to
+ // leave rval when operation is complete.
+
+ // Pop rval into R0, then load receiver into R1 and replace with rval.
+ frame.popRegsAndSync(1);
+ masm.loadValue(frame.addressOfStackValue(-2), R1);
+ masm.storeValue(R0, frame.addressOfStackValue(-2));
+
+ prepareVMCall();
+
+ pushArg(Imm32(strict));
+ pushArg(R0); // rval
+ pushScriptNameArg(R0.scratchReg(), R2.scratchReg());
+ pushArg(R1); // receiver
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+ pushArg(R0); // obj
+
+ using Fn = bool (*)(JSContext*, HandleValue, HandleValue,
+ Handle<PropertyName*>, HandleValue, bool);
+ if (!callVM<Fn, js::SetPropertySuper>()) {
+ return false;
+ }
+
+ frame.pop();
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetPropSuper() {
+ return emitSetPropSuper(/* strict = */ false);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictSetPropSuper() {
+ return emitSetPropSuper(/* strict = */ true);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetProp() {
+ // Keep object in R0.
+ frame.popRegsAndSync(1);
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetBoundName() {
+ return emit_GetProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetPropSuper() {
+ // Receiver -> R1, ObjectOrNull -> R0
+ frame.popRegsAndSync(1);
+ masm.loadValue(frame.addressOfStackValue(-1), R1);
+ frame.pop();
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitDelProp(bool strict) {
+ // Keep value on the stack for the decompiler.
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ prepareVMCall();
+
+ pushScriptNameArg(R1.scratchReg(), R2.scratchReg());
+ pushArg(R0);
+
+ using Fn = bool (*)(JSContext*, HandleValue, Handle<PropertyName*>, bool*);
+ if (strict) {
+ if (!callVM<Fn, DelPropOperation<true>>()) {
+ return false;
+ }
+ } else {
+ if (!callVM<Fn, DelPropOperation<false>>()) {
+ return false;
+ }
+ }
+
+ masm.boxNonDouble(JSVAL_TYPE_BOOLEAN, ReturnReg, R1);
+ frame.pop();
+ frame.push(R1);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_DelProp() {
+ return emitDelProp(/* strict = */ false);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictDelProp() {
+ return emitDelProp(/* strict = */ true);
+}
+
+template <>
+void BaselineCompilerCodeGen::getEnvironmentCoordinateObject(Register reg) {
+ EnvironmentCoordinate ec(handler.pc());
+
+ masm.loadPtr(frame.addressOfEnvironmentChain(), reg);
+ for (unsigned i = ec.hops(); i; i--) {
+ masm.unboxObject(
+ Address(reg, EnvironmentObject::offsetOfEnclosingEnvironment()), reg);
+ }
+}
+
+template <>
+void BaselineInterpreterCodeGen::getEnvironmentCoordinateObject(Register reg) {
+ MOZ_CRASH("Shouldn't call this for interpreter");
+}
+
+template <>
+Address BaselineCompilerCodeGen::getEnvironmentCoordinateAddressFromObject(
+ Register objReg, Register reg) {
+ EnvironmentCoordinate ec(handler.pc());
+
+ if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
+ return Address(objReg, NativeObject::getFixedSlotOffset(ec.slot()));
+ }
+
+ uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
+ masm.loadPtr(Address(objReg, NativeObject::offsetOfSlots()), reg);
+ return Address(reg, slot * sizeof(Value));
+}
+
+template <>
+Address BaselineInterpreterCodeGen::getEnvironmentCoordinateAddressFromObject(
+ Register objReg, Register reg) {
+ MOZ_CRASH("Shouldn't call this for interpreter");
+}
+
+template <typename Handler>
+Address BaselineCodeGen<Handler>::getEnvironmentCoordinateAddress(
+ Register reg) {
+ getEnvironmentCoordinateObject(reg);
+ return getEnvironmentCoordinateAddressFromObject(reg, reg);
+}
+
+// For a JOF_ENVCOORD op load the number of hops from the bytecode and skip this
+// number of environment objects.
+static void LoadAliasedVarEnv(MacroAssembler& masm, Register env,
+ Register scratch) {
+ static_assert(ENVCOORD_HOPS_LEN == 1,
+ "Code assumes number of hops is stored in uint8 operand");
+ LoadUint8Operand(masm, scratch);
+
+ Label top, done;
+ masm.branchTest32(Assembler::Zero, scratch, scratch, &done);
+ masm.bind(&top);
+ {
+ Address nextEnv(env, EnvironmentObject::offsetOfEnclosingEnvironment());
+ masm.unboxObject(nextEnv, env);
+ masm.branchSub32(Assembler::NonZero, Imm32(1), scratch, &top);
+ }
+ masm.bind(&done);
+}
+
+template <>
+void BaselineCompilerCodeGen::emitGetAliasedVar(ValueOperand dest) {
+ frame.syncStack(0);
+
+ Address address = getEnvironmentCoordinateAddress(R0.scratchReg());
+ masm.loadValue(address, dest);
+}
+
+template <>
+void BaselineInterpreterCodeGen::emitGetAliasedVar(ValueOperand dest) {
+ Register env = R0.scratchReg();
+ Register scratch = R1.scratchReg();
+
+ // Load the right environment object.
+ masm.loadPtr(frame.addressOfEnvironmentChain(), env);
+ LoadAliasedVarEnv(masm, env, scratch);
+
+ // Load the slot index.
+ static_assert(ENVCOORD_SLOT_LEN == 3,
+ "Code assumes slot is stored in uint24 operand");
+ LoadUint24Operand(masm, ENVCOORD_HOPS_LEN, scratch);
+
+ // Load the Value from a fixed or dynamic slot.
+ // See EnvironmentObject::nonExtensibleIsFixedSlot.
+ Label isDynamic, done;
+ masm.branch32(Assembler::AboveOrEqual, scratch,
+ Imm32(NativeObject::MAX_FIXED_SLOTS), &isDynamic);
+ {
+ uint32_t offset = NativeObject::getFixedSlotOffset(0);
+ masm.loadValue(BaseValueIndex(env, scratch, offset), dest);
+ masm.jump(&done);
+ }
+ masm.bind(&isDynamic);
+ {
+ masm.loadPtr(Address(env, NativeObject::offsetOfSlots()), env);
+
+ // Use an offset to subtract the number of fixed slots.
+ int32_t offset = -int32_t(NativeObject::MAX_FIXED_SLOTS * sizeof(Value));
+ masm.loadValue(BaseValueIndex(env, scratch, offset), dest);
+ }
+ masm.bind(&done);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitGetAliasedDebugVar(ValueOperand dest) {
+ frame.syncStack(0);
+ Register env = R0.scratchReg();
+ // Load the right environment object.
+ masm.loadPtr(frame.addressOfEnvironmentChain(), env);
+
+ prepareVMCall();
+ pushBytecodePCArg();
+ pushArg(env);
+
+ using Fn =
+ bool (*)(JSContext*, JSObject * env, jsbytecode*, MutableHandleValue);
+ return callVM<Fn, LoadAliasedDebugVar>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetAliasedDebugVar() {
+ if (!emitGetAliasedDebugVar(R0)) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetAliasedVar() {
+ emitGetAliasedVar(R0);
+
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_SetAliasedVar() {
+ // Keep rvalue in R0.
+ frame.popRegsAndSync(1);
+ Register objReg = R2.scratchReg();
+
+ getEnvironmentCoordinateObject(objReg);
+ Address address =
+ getEnvironmentCoordinateAddressFromObject(objReg, R1.scratchReg());
+ masm.guardedCallPreBarrier(address, MIRType::Value);
+ masm.storeValue(R0, address);
+ frame.push(R0);
+
+ // Only R0 is live at this point.
+ // Scope coordinate object is already in R2.scratchReg().
+ Register temp = R1.scratchReg();
+
+ Label skipBarrier;
+ masm.branchPtrInNurseryChunk(Assembler::Equal, objReg, temp, &skipBarrier);
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
+
+ masm.call(&postBarrierSlot_); // Won't clobber R0
+
+ masm.bind(&skipBarrier);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_SetAliasedVar() {
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.take(R2);
+ if (HasInterpreterPCReg()) {
+ regs.take(InterpreterPCReg);
+ }
+
+ Register env = regs.takeAny();
+ Register scratch1 = regs.takeAny();
+ Register scratch2 = regs.takeAny();
+ Register scratch3 = regs.takeAny();
+
+ // Load the right environment object.
+ masm.loadPtr(frame.addressOfEnvironmentChain(), env);
+ LoadAliasedVarEnv(masm, env, scratch1);
+
+ // Load the slot index.
+ static_assert(ENVCOORD_SLOT_LEN == 3,
+ "Code assumes slot is stored in uint24 operand");
+ LoadUint24Operand(masm, ENVCOORD_HOPS_LEN, scratch1);
+
+ // Store the RHS Value in R2.
+ masm.loadValue(frame.addressOfStackValue(-1), R2);
+
+ // Load a pointer to the fixed or dynamic slot into scratch2. We want to call
+ // guardedCallPreBarrierAnyZone once to avoid code bloat.
+
+ // See EnvironmentObject::nonExtensibleIsFixedSlot.
+ Label isDynamic, done;
+ masm.branch32(Assembler::AboveOrEqual, scratch1,
+ Imm32(NativeObject::MAX_FIXED_SLOTS), &isDynamic);
+ {
+ uint32_t offset = NativeObject::getFixedSlotOffset(0);
+ BaseValueIndex slotAddr(env, scratch1, offset);
+ masm.computeEffectiveAddress(slotAddr, scratch2);
+ masm.jump(&done);
+ }
+ masm.bind(&isDynamic);
+ {
+ masm.loadPtr(Address(env, NativeObject::offsetOfSlots()), scratch2);
+
+ // Use an offset to subtract the number of fixed slots.
+ int32_t offset = -int32_t(NativeObject::MAX_FIXED_SLOTS * sizeof(Value));
+ BaseValueIndex slotAddr(scratch2, scratch1, offset);
+ masm.computeEffectiveAddress(slotAddr, scratch2);
+ }
+ masm.bind(&done);
+
+ // Pre-barrier and store.
+ Address slotAddr(scratch2, 0);
+ masm.guardedCallPreBarrierAnyZone(slotAddr, MIRType::Value, scratch3);
+ masm.storeValue(R2, slotAddr);
+
+ // Post barrier.
+ Label skipBarrier;
+ masm.branchPtrInNurseryChunk(Assembler::Equal, env, scratch1, &skipBarrier);
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, R2, scratch1,
+ &skipBarrier);
+ {
+ // Post barrier code expects the object in R2.
+ masm.movePtr(env, R2.scratchReg());
+ masm.call(&postBarrierSlot_);
+ }
+ masm.bind(&skipBarrier);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetName() {
+ frame.syncStack(0);
+
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_BindName() {
+ frame.syncStack(0);
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_DelName() {
+ frame.syncStack(0);
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+
+ prepareVMCall();
+
+ pushArg(R0.scratchReg());
+ pushScriptNameArg(R1.scratchReg(), R2.scratchReg());
+
+ using Fn = bool (*)(JSContext*, Handle<PropertyName*>, HandleObject,
+ MutableHandleValue);
+ if (!callVM<Fn, js::DeleteNameOperation>()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_GetImport() {
+ JSScript* script = handler.script();
+ ModuleEnvironmentObject* env = GetModuleEnvironmentForScript(script);
+ MOZ_ASSERT(env);
+
+ jsid id = NameToId(script->getName(handler.pc()));
+ ModuleEnvironmentObject* targetEnv;
+ Maybe<PropertyInfo> prop;
+ MOZ_ALWAYS_TRUE(env->lookupImport(id, &targetEnv, &prop));
+
+ frame.syncStack(0);
+
+ uint32_t slot = prop->slot();
+ Register scratch = R0.scratchReg();
+ masm.movePtr(ImmGCPtr(targetEnv), scratch);
+ if (slot < targetEnv->numFixedSlots()) {
+ masm.loadValue(Address(scratch, NativeObject::getFixedSlotOffset(slot)),
+ R0);
+ } else {
+ masm.loadPtr(Address(scratch, NativeObject::offsetOfSlots()), scratch);
+ masm.loadValue(
+ Address(scratch, (slot - targetEnv->numFixedSlots()) * sizeof(Value)),
+ R0);
+ }
+
+ // Imports are initialized by this point except in rare circumstances, so
+ // don't emit a check unless we have to.
+ if (targetEnv->getSlot(slot).isMagic(JS_UNINITIALIZED_LEXICAL)) {
+ if (!emitUninitializedLexicalCheck(R0)) {
+ return false;
+ }
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_GetImport() {
+ frame.syncStack(0);
+
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+
+ prepareVMCall();
+
+ pushBytecodePCArg();
+ pushScriptArg();
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleScript, jsbytecode*,
+ MutableHandleValue);
+ if (!callVM<Fn, GetImportOperation>()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetIntrinsic() {
+ frame.syncStack(0);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetIntrinsic() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ prepareVMCall();
+
+ pushArg(R0);
+ pushBytecodePCArg();
+ pushScriptArg();
+
+ using Fn = bool (*)(JSContext*, JSScript*, jsbytecode*, HandleValue);
+ return callVM<Fn, SetIntrinsicOperation>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GlobalOrEvalDeclInstantiation() {
+ frame.syncStack(0);
+
+ prepareVMCall();
+
+ loadInt32LengthBytecodeOperand(R0.scratchReg());
+ pushArg(R0.scratchReg());
+ pushScriptArg();
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleScript, GCThingIndex);
+ return callVM<Fn, js::GlobalOrEvalDeclInstantiation>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitInitPropGetterSetter() {
+ // Keep values on the stack for the decompiler.
+ frame.syncStack(0);
+
+ prepareVMCall();
+
+ masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg());
+ masm.unboxObject(frame.addressOfStackValue(-2), R1.scratchReg());
+
+ pushArg(R0.scratchReg());
+ pushScriptNameArg(R0.scratchReg(), R2.scratchReg());
+ pushArg(R1.scratchReg());
+ pushBytecodePCArg();
+
+ using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject,
+ Handle<PropertyName*>, HandleObject);
+ if (!callVM<Fn, InitPropGetterSetterOperation>()) {
+ return false;
+ }
+
+ frame.pop();
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitPropGetter() {
+ return emitInitPropGetterSetter();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitHiddenPropGetter() {
+ return emitInitPropGetterSetter();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitPropSetter() {
+ return emitInitPropGetterSetter();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitHiddenPropSetter() {
+ return emitInitPropGetterSetter();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitInitElemGetterSetter() {
+ // Load index and value in R0 and R1, but keep values on the stack for the
+ // decompiler.
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-2), R0);
+ masm.unboxObject(frame.addressOfStackValue(-1), R1.scratchReg());
+
+ prepareVMCall();
+
+ pushArg(R1.scratchReg());
+ pushArg(R0);
+ masm.unboxObject(frame.addressOfStackValue(-3), R0.scratchReg());
+ pushArg(R0.scratchReg());
+ pushBytecodePCArg();
+
+ using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject, HandleValue,
+ HandleObject);
+ if (!callVM<Fn, InitElemGetterSetterOperation>()) {
+ return false;
+ }
+
+ frame.popn(2);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitElemGetter() {
+ return emitInitElemGetterSetter();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitHiddenElemGetter() {
+ return emitInitElemGetterSetter();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitElemSetter() {
+ return emitInitElemGetterSetter();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitHiddenElemSetter() {
+ return emitInitElemGetterSetter();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitElemInc() {
+ // Keep the object and rhs on the stack.
+ frame.syncStack(0);
+
+ // Load object in R0, index in R1.
+ masm.loadValue(frame.addressOfStackValue(-3), R0);
+ masm.loadValue(frame.addressOfStackValue(-2), R1);
+
+ // Call IC.
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Pop the rhs
+ frame.pop();
+
+ // Increment index
+ Address indexAddr = frame.addressOfStackValue(-1);
+#ifdef DEBUG
+ Label isInt32;
+ masm.branchTestInt32(Assembler::Equal, indexAddr, &isInt32);
+ masm.assumeUnreachable("INITELEM_INC index must be Int32");
+ masm.bind(&isInt32);
+#endif
+ masm.incrementInt32Value(indexAddr);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_GetLocal() {
+ frame.pushLocal(GET_LOCALNO(handler.pc()));
+ return true;
+}
+
+static BaseValueIndex ComputeAddressOfLocal(MacroAssembler& masm,
+ Register indexScratch) {
+ // Locals are stored in memory at a negative offset from the frame pointer. We
+ // negate the index first to effectively subtract it.
+ masm.negPtr(indexScratch);
+ return BaseValueIndex(FramePointer, indexScratch,
+ BaselineFrame::reverseOffsetOfLocal(0));
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_GetLocal() {
+ Register scratch = R0.scratchReg();
+ LoadUint24Operand(masm, 0, scratch);
+ BaseValueIndex addr = ComputeAddressOfLocal(masm, scratch);
+ masm.loadValue(addr, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_SetLocal() {
+ // Ensure no other StackValue refers to the old value, for instance i + (i =
+ // 3). This also allows us to use R0 as scratch below.
+ frame.syncStack(1);
+
+ uint32_t local = GET_LOCALNO(handler.pc());
+ frame.storeStackValue(-1, frame.addressOfLocal(local), R0);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_SetLocal() {
+ Register scratch = R0.scratchReg();
+ LoadUint24Operand(masm, 0, scratch);
+ BaseValueIndex addr = ComputeAddressOfLocal(masm, scratch);
+ masm.loadValue(frame.addressOfStackValue(-1), R1);
+ masm.storeValue(R1, addr);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emitFormalArgAccess(JSOp op) {
+ MOZ_ASSERT(op == JSOp::GetArg || op == JSOp::SetArg);
+
+ uint32_t arg = GET_ARGNO(handler.pc());
+
+ // Fast path: the script does not use |arguments| or formals don't
+ // alias the arguments object.
+ if (!handler.script()->argsObjAliasesFormals()) {
+ if (op == JSOp::GetArg) {
+ frame.pushArg(arg);
+ } else {
+ // See the comment in emit_SetLocal.
+ frame.syncStack(1);
+ frame.storeStackValue(-1, frame.addressOfArg(arg), R0);
+ }
+
+ return true;
+ }
+
+ // Sync so that we can use R0.
+ frame.syncStack(0);
+
+ // Load the arguments object data vector.
+ Register reg = R2.scratchReg();
+ masm.loadPtr(frame.addressOfArgsObj(), reg);
+ masm.loadPrivate(Address(reg, ArgumentsObject::getDataSlotOffset()), reg);
+
+ // Load/store the argument.
+ Address argAddr(reg, ArgumentsData::offsetOfArgs() + arg * sizeof(Value));
+ if (op == JSOp::GetArg) {
+ masm.loadValue(argAddr, R0);
+ frame.push(R0);
+ } else {
+ Register temp = R1.scratchReg();
+ masm.guardedCallPreBarrierAnyZone(argAddr, MIRType::Value, temp);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+ masm.storeValue(R0, argAddr);
+
+ MOZ_ASSERT(frame.numUnsyncedSlots() == 0);
+
+ // Reload the arguments object.
+ Register reg = R2.scratchReg();
+ masm.loadPtr(frame.addressOfArgsObj(), reg);
+
+ Label skipBarrier;
+
+ masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &skipBarrier);
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
+
+ masm.call(&postBarrierSlot_);
+
+ masm.bind(&skipBarrier);
+ }
+
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emitFormalArgAccess(JSOp op) {
+ MOZ_ASSERT(op == JSOp::GetArg || op == JSOp::SetArg);
+
+ // Load the index.
+ Register argReg = R1.scratchReg();
+ LoadUint16Operand(masm, argReg);
+
+ // If the frame has no arguments object, this must be an unaliased access.
+ Label isUnaliased, done;
+ masm.branchTest32(Assembler::Zero, frame.addressOfFlags(),
+ Imm32(BaselineFrame::HAS_ARGS_OBJ), &isUnaliased);
+ {
+ Register reg = R2.scratchReg();
+
+ // If it's an unmapped arguments object, this is an unaliased access.
+ loadScript(reg);
+ masm.branchTest32(
+ Assembler::Zero, Address(reg, JSScript::offsetOfImmutableFlags()),
+ Imm32(uint32_t(JSScript::ImmutableFlags::HasMappedArgsObj)),
+ &isUnaliased);
+
+ // Load the arguments object data vector.
+ masm.loadPtr(frame.addressOfArgsObj(), reg);
+ masm.loadPrivate(Address(reg, ArgumentsObject::getDataSlotOffset()), reg);
+
+ // Load/store the argument.
+ BaseValueIndex argAddr(reg, argReg, ArgumentsData::offsetOfArgs());
+ if (op == JSOp::GetArg) {
+ masm.loadValue(argAddr, R0);
+ frame.push(R0);
+ } else {
+ masm.guardedCallPreBarrierAnyZone(argAddr, MIRType::Value,
+ R0.scratchReg());
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+ masm.storeValue(R0, argAddr);
+
+ // Reload the arguments object.
+ masm.loadPtr(frame.addressOfArgsObj(), reg);
+
+ Register temp = R1.scratchReg();
+ masm.branchPtrInNurseryChunk(Assembler::Equal, reg, temp, &done);
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &done);
+
+ masm.call(&postBarrierSlot_);
+ }
+ masm.jump(&done);
+ }
+ masm.bind(&isUnaliased);
+ {
+ BaseValueIndex addr(FramePointer, argReg,
+ JitFrameLayout::offsetOfActualArgs());
+ if (op == JSOp::GetArg) {
+ masm.loadValue(addr, R0);
+ frame.push(R0);
+ } else {
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+ masm.storeValue(R0, addr);
+ }
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetArg() {
+ return emitFormalArgAccess(JSOp::GetArg);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetArg() {
+ return emitFormalArgAccess(JSOp::SetArg);
+}
+
+template <>
+void BaselineCompilerCodeGen::loadNumFormalArguments(Register dest) {
+ masm.move32(Imm32(handler.function()->nargs()), dest);
+}
+
+template <>
+void BaselineInterpreterCodeGen::loadNumFormalArguments(Register dest) {
+ masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(), dest);
+ masm.loadFunctionArgCount(dest, dest);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_NewTarget() {
+ MOZ_ASSERT_IF(handler.maybeFunction(), !handler.maybeFunction()->isArrow());
+
+ frame.syncStack(0);
+
+#ifdef DEBUG
+ Register scratch1 = R0.scratchReg();
+ Register scratch2 = R1.scratchReg();
+
+ Label isFunction;
+ masm.loadPtr(frame.addressOfCalleeToken(), scratch1);
+ masm.branchTestPtr(Assembler::Zero, scratch1, Imm32(CalleeTokenScriptBit),
+ &isFunction);
+ masm.assumeUnreachable("Unexpected non-function script");
+ masm.bind(&isFunction);
+
+ Label notArrow;
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), scratch1);
+ masm.branchFunctionKind(Assembler::NotEqual,
+ FunctionFlags::FunctionKind::Arrow, scratch1,
+ scratch2, &notArrow);
+ masm.assumeUnreachable("Unexpected arrow function");
+ masm.bind(&notArrow);
+#endif
+
+ // if (isConstructing()) push(argv[Max(numActualArgs, numFormalArgs)])
+ Label notConstructing, done;
+ masm.branchTestPtr(Assembler::Zero, frame.addressOfCalleeToken(),
+ Imm32(CalleeToken_FunctionConstructing), &notConstructing);
+ {
+ Register argvLen = R0.scratchReg();
+ Register nformals = R1.scratchReg();
+ masm.loadNumActualArgs(FramePointer, argvLen);
+
+ // If argvLen < nformals, set argvlen := nformals.
+ loadNumFormalArguments(nformals);
+ masm.cmp32Move32(Assembler::Below, argvLen, nformals, nformals, argvLen);
+
+ BaseValueIndex newTarget(FramePointer, argvLen,
+ JitFrameLayout::offsetOfActualArgs());
+ masm.loadValue(newTarget, R0);
+ masm.jump(&done);
+ }
+ // else push(undefined)
+ masm.bind(&notConstructing);
+ masm.moveValue(UndefinedValue(), R0);
+
+ masm.bind(&done);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_ThrowSetConst() {
+ prepareVMCall();
+ pushArg(Imm32(JSMSG_BAD_CONST_ASSIGN));
+
+ using Fn = bool (*)(JSContext*, unsigned);
+ return callVM<Fn, jit::ThrowRuntimeLexicalError>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitUninitializedLexicalCheck(
+ const ValueOperand& val) {
+ Label done;
+ masm.branchTestMagicValue(Assembler::NotEqual, val, JS_UNINITIALIZED_LEXICAL,
+ &done);
+
+ prepareVMCall();
+ pushArg(Imm32(JSMSG_UNINITIALIZED_LEXICAL));
+
+ using Fn = bool (*)(JSContext*, unsigned);
+ if (!callVM<Fn, jit::ThrowRuntimeLexicalError>()) {
+ return false;
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckLexical() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+ return emitUninitializedLexicalCheck(R0);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckAliasedLexical() {
+ return emit_CheckLexical();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitLexical() {
+ return emit_SetLocal();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitGLexical() {
+ frame.popRegsAndSync(1);
+ pushGlobalLexicalEnvironmentValue(R1);
+ frame.push(R0);
+ return emit_SetProp();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitAliasedLexical() {
+ return emit_SetAliasedVar();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Uninitialized() {
+ frame.push(MagicValue(JS_UNINITIALIZED_LEXICAL));
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emitCall(JSOp op) {
+ MOZ_ASSERT(IsInvokeOp(op));
+
+ frame.syncStack(0);
+
+ uint32_t argc = GET_ARGC(handler.pc());
+ masm.move32(Imm32(argc), R0.scratchReg());
+
+ // Call IC
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Update FrameInfo.
+ bool construct = IsConstructOp(op);
+ frame.popn(2 + argc + construct);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emitCall(JSOp op) {
+ MOZ_ASSERT(IsInvokeOp(op));
+
+ // The IC expects argc in R0.
+ LoadUint16Operand(masm, R0.scratchReg());
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Pop the arguments. We have to reload pc/argc because the IC clobbers them.
+ // The return value is in R0 so we can't use that.
+ Register scratch = R1.scratchReg();
+ uint32_t extraValuesToPop = IsConstructOp(op) ? 3 : 2;
+ Register spReg = AsRegister(masm.getStackPointer());
+ LoadUint16Operand(masm, scratch);
+ masm.computeEffectiveAddress(
+ BaseValueIndex(spReg, scratch, extraValuesToPop * sizeof(Value)), spReg);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitSpreadCall(JSOp op) {
+ MOZ_ASSERT(IsInvokeOp(op));
+
+ frame.syncStack(0);
+ masm.move32(Imm32(1), R0.scratchReg());
+
+ // Call IC
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Update FrameInfo.
+ bool construct = op == JSOp::SpreadNew || op == JSOp::SpreadSuperCall;
+ frame.popn(3 + construct);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Call() {
+ return emitCall(JSOp::Call);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CallContent() {
+ return emitCall(JSOp::CallContent);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CallIgnoresRv() {
+ return emitCall(JSOp::CallIgnoresRv);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CallIter() {
+ return emitCall(JSOp::CallIter);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CallContentIter() {
+ return emitCall(JSOp::CallContentIter);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_New() {
+ return emitCall(JSOp::New);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_NewContent() {
+ return emitCall(JSOp::NewContent);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SuperCall() {
+ return emitCall(JSOp::SuperCall);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Eval() {
+ return emitCall(JSOp::Eval);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictEval() {
+ return emitCall(JSOp::StrictEval);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SpreadCall() {
+ return emitSpreadCall(JSOp::SpreadCall);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SpreadNew() {
+ return emitSpreadCall(JSOp::SpreadNew);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SpreadSuperCall() {
+ return emitSpreadCall(JSOp::SpreadSuperCall);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SpreadEval() {
+ return emitSpreadCall(JSOp::SpreadEval);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_StrictSpreadEval() {
+ return emitSpreadCall(JSOp::StrictSpreadEval);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_OptimizeSpreadCall() {
+ frame.popRegsAndSync(1);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_ImplicitThis() {
+ frame.syncStack(0);
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R0.scratchReg());
+
+ prepareVMCall();
+
+ pushScriptNameArg(R1.scratchReg(), R2.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, HandleObject, Handle<PropertyName*>,
+ MutableHandleValue);
+ if (!callVM<Fn, ImplicitThisOperation>()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Instanceof() {
+ frame.popRegsAndSync(2);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Typeof() {
+ frame.popRegsAndSync(1);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_TypeofExpr() {
+ return emit_Typeof();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_ThrowMsg() {
+ prepareVMCall();
+ pushUint8BytecodeOperandArg(R2.scratchReg());
+
+ using Fn = bool (*)(JSContext*, const unsigned);
+ return callVM<Fn, js::ThrowMsgOperation>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Throw() {
+ // Keep value to throw in R0.
+ frame.popRegsAndSync(1);
+
+ prepareVMCall();
+ pushArg(R0);
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ return callVM<Fn, js::ThrowOperation>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Try() {
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Finally() {
+ // To match the interpreter, emit an interrupt check at the start of the
+ // finally block.
+ return emitInterruptCheck();
+}
+
+static void LoadBaselineScriptResumeEntries(MacroAssembler& masm,
+ JSScript* script, Register dest,
+ Register scratch) {
+ MOZ_ASSERT(dest != scratch);
+
+ masm.movePtr(ImmPtr(script->jitScript()), dest);
+ masm.loadPtr(Address(dest, JitScript::offsetOfBaselineScript()), dest);
+ masm.load32(Address(dest, BaselineScript::offsetOfResumeEntriesOffset()),
+ scratch);
+ masm.addPtr(scratch, dest);
+}
+
+template <typename Handler>
+void BaselineCodeGen<Handler>::emitInterpJumpToResumeEntry(Register script,
+ Register resumeIndex,
+ Register scratch) {
+ // Load JSScript::immutableScriptData() into |script|.
+ masm.loadPtr(Address(script, JSScript::offsetOfSharedData()), script);
+ masm.loadPtr(Address(script, SharedImmutableScriptData::offsetOfISD()),
+ script);
+
+ // Load the resume pcOffset in |resumeIndex|.
+ masm.load32(
+ Address(script, ImmutableScriptData::offsetOfResumeOffsetsOffset()),
+ scratch);
+ masm.computeEffectiveAddress(BaseIndex(scratch, resumeIndex, TimesFour),
+ scratch);
+ masm.load32(BaseIndex(script, scratch, TimesOne), resumeIndex);
+
+ // Add resume offset to PC, jump to it.
+ masm.computeEffectiveAddress(BaseIndex(script, resumeIndex, TimesOne,
+ ImmutableScriptData::offsetOfCode()),
+ script);
+ Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC());
+ masm.storePtr(script, pcAddr);
+ emitJumpToInterpretOpLabel();
+}
+
+template <>
+void BaselineCompilerCodeGen::jumpToResumeEntry(Register resumeIndex,
+ Register scratch1,
+ Register scratch2) {
+ LoadBaselineScriptResumeEntries(masm, handler.script(), scratch1, scratch2);
+ masm.loadPtr(
+ BaseIndex(scratch1, resumeIndex, ScaleFromElemWidth(sizeof(uintptr_t))),
+ scratch1);
+ masm.jump(scratch1);
+}
+
+template <>
+void BaselineInterpreterCodeGen::jumpToResumeEntry(Register resumeIndex,
+ Register scratch1,
+ Register scratch2) {
+ loadScript(scratch1);
+ emitInterpJumpToResumeEntry(scratch1, resumeIndex, scratch2);
+}
+
+template <>
+template <typename F1, typename F2>
+[[nodiscard]] bool BaselineCompilerCodeGen::emitDebugInstrumentation(
+ const F1& ifDebuggee, const Maybe<F2>& ifNotDebuggee) {
+ // The JIT calls either ifDebuggee or (if present) ifNotDebuggee, because it
+ // knows statically whether we're compiling with debug instrumentation.
+
+ if (handler.compileDebugInstrumentation()) {
+ return ifDebuggee();
+ }
+
+ if (ifNotDebuggee) {
+ return (*ifNotDebuggee)();
+ }
+
+ return true;
+}
+
+template <>
+template <typename F1, typename F2>
+[[nodiscard]] bool BaselineInterpreterCodeGen::emitDebugInstrumentation(
+ const F1& ifDebuggee, const Maybe<F2>& ifNotDebuggee) {
+ // The interpreter emits both ifDebuggee and (if present) ifNotDebuggee
+ // paths, with a toggled jump followed by a branch on the frame's DEBUGGEE
+ // flag.
+
+ Label isNotDebuggee, done;
+
+ CodeOffset toggleOffset = masm.toggledJump(&isNotDebuggee);
+ if (!handler.addDebugInstrumentationOffset(cx, toggleOffset)) {
+ return false;
+ }
+
+ masm.branchTest32(Assembler::Zero, frame.addressOfFlags(),
+ Imm32(BaselineFrame::DEBUGGEE), &isNotDebuggee);
+
+ if (!ifDebuggee()) {
+ return false;
+ }
+
+ if (ifNotDebuggee) {
+ masm.jump(&done);
+ }
+
+ masm.bind(&isNotDebuggee);
+
+ if (ifNotDebuggee && !(*ifNotDebuggee)()) {
+ return false;
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_PushLexicalEnv() {
+ // Call a stub to push the block on the block chain.
+ prepareVMCall();
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(),
+ R2.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, Handle<LexicalScope*>);
+ return callVM<Fn, jit::PushLexicalEnv>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_PushClassBodyEnv() {
+ prepareVMCall();
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(),
+ R2.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, Handle<ClassBodyScope*>);
+ return callVM<Fn, jit::PushClassBodyEnv>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_PopLexicalEnv() {
+ frame.syncStack(0);
+
+ Register scratch1 = R0.scratchReg();
+
+ auto ifDebuggee = [this, scratch1]() {
+ masm.loadBaselineFramePtr(FramePointer, scratch1);
+
+ prepareVMCall();
+ pushBytecodePCArg();
+ pushArg(scratch1);
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
+ return callVM<Fn, jit::DebugLeaveThenPopLexicalEnv>();
+ };
+ auto ifNotDebuggee = [this, scratch1]() {
+ Register scratch2 = R1.scratchReg();
+ masm.loadPtr(frame.addressOfEnvironmentChain(), scratch1);
+ masm.debugAssertObjectHasClass(scratch1, scratch2,
+ &LexicalEnvironmentObject::class_);
+ Address enclosingAddr(scratch1,
+ EnvironmentObject::offsetOfEnclosingEnvironment());
+ masm.unboxObject(enclosingAddr, scratch1);
+ masm.storePtr(scratch1, frame.addressOfEnvironmentChain());
+ return true;
+ };
+ return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee));
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_FreshenLexicalEnv() {
+ frame.syncStack(0);
+
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ auto ifDebuggee = [this]() {
+ prepareVMCall();
+ pushBytecodePCArg();
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
+ return callVM<Fn, jit::DebugLeaveThenFreshenLexicalEnv>();
+ };
+ auto ifNotDebuggee = [this]() {
+ prepareVMCall();
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ return callVM<Fn, jit::FreshenLexicalEnv>();
+ };
+ return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee));
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_RecreateLexicalEnv() {
+ frame.syncStack(0);
+
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ auto ifDebuggee = [this]() {
+ prepareVMCall();
+ pushBytecodePCArg();
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
+ return callVM<Fn, jit::DebugLeaveThenRecreateLexicalEnv>();
+ };
+ auto ifNotDebuggee = [this]() {
+ prepareVMCall();
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ return callVM<Fn, jit::RecreateLexicalEnv>();
+ };
+ return emitDebugInstrumentation(ifDebuggee, mozilla::Some(ifNotDebuggee));
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_DebugLeaveLexicalEnv() {
+ auto ifDebuggee = [this]() {
+ prepareVMCall();
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ pushBytecodePCArg();
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
+ return callVM<Fn, jit::DebugLeaveLexicalEnv>();
+ };
+ return emitDebugInstrumentation(ifDebuggee);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_PushVarEnv() {
+ prepareVMCall();
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(),
+ R2.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, Handle<Scope*>);
+ return callVM<Fn, jit::PushVarEnv>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_EnterWith() {
+ // Pop "with" object to R0.
+ frame.popRegsAndSync(1);
+
+ // Call a stub to push the object onto the environment chain.
+ prepareVMCall();
+
+ pushScriptGCThingArg(ScriptGCThingType::Scope, R1.scratchReg(),
+ R2.scratchReg());
+ pushArg(R0);
+ masm.loadBaselineFramePtr(FramePointer, R1.scratchReg());
+ pushArg(R1.scratchReg());
+
+ using Fn =
+ bool (*)(JSContext*, BaselineFrame*, HandleValue, Handle<WithScope*>);
+ return callVM<Fn, jit::EnterWith>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_LeaveWith() {
+ // Call a stub to pop the with object from the environment chain.
+ prepareVMCall();
+
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ return callVM<Fn, jit::LeaveWith>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Exception() {
+ prepareVMCall();
+
+ using Fn = bool (*)(JSContext*, MutableHandleValue);
+ if (!callVM<Fn, GetAndClearException>()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Debugger() {
+ prepareVMCall();
+
+ frame.assertSyncedStack();
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ if (!callVM<Fn, jit::OnDebuggerStatement>()) {
+ return false;
+ }
+
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitDebugEpilogue() {
+ auto ifDebuggee = [this]() {
+ // Move return value into the frame's rval slot.
+ masm.storeValue(JSReturnOperand, frame.addressOfReturnValue());
+ masm.or32(Imm32(BaselineFrame::HAS_RVAL), frame.addressOfFlags());
+
+ // Load BaselineFrame pointer in R0.
+ frame.syncStack(0);
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ prepareVMCall();
+ pushBytecodePCArg();
+ pushArg(R0.scratchReg());
+
+ const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugEpilogue;
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, const jsbytecode*);
+ if (!callVM<Fn, jit::DebugEpilogueOnBaselineReturn>(kind)) {
+ return false;
+ }
+
+ masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand);
+ return true;
+ };
+ return emitDebugInstrumentation(ifDebuggee);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitReturn() {
+ if (handler.shouldEmitDebugEpilogueAtReturnOp()) {
+ if (!emitDebugEpilogue()) {
+ return false;
+ }
+ }
+
+ // Only emit the jump if this JSOp::RetRval is not the last instruction.
+ // Not needed for last instruction, because last instruction flows
+ // into return label.
+ if (!handler.isDefinitelyLastOp()) {
+ masm.jump(&return_);
+ }
+
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Return() {
+ frame.assertStackDepth(1);
+
+ frame.popValue(JSReturnOperand);
+ return emitReturn();
+}
+
+template <typename Handler>
+void BaselineCodeGen<Handler>::emitLoadReturnValue(ValueOperand val) {
+ Label done, noRval;
+ masm.branchTest32(Assembler::Zero, frame.addressOfFlags(),
+ Imm32(BaselineFrame::HAS_RVAL), &noRval);
+ masm.loadValue(frame.addressOfReturnValue(), val);
+ masm.jump(&done);
+
+ masm.bind(&noRval);
+ masm.moveValue(UndefinedValue(), val);
+
+ masm.bind(&done);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_RetRval() {
+ frame.assertStackDepth(0);
+
+ masm.moveValue(UndefinedValue(), JSReturnOperand);
+
+ if (!handler.maybeScript() || !handler.maybeScript()->noScriptRval()) {
+ // Return the value in the return value slot, if any.
+ Label done;
+ Address flags = frame.addressOfFlags();
+ masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL),
+ &done);
+ masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand);
+ masm.bind(&done);
+ }
+
+ return emitReturn();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_ToPropertyKey() {
+ frame.popRegsAndSync(1);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_ToAsyncIter() {
+ frame.syncStack(0);
+ masm.unboxObject(frame.addressOfStackValue(-2), R0.scratchReg());
+ masm.loadValue(frame.addressOfStackValue(-1), R1);
+
+ prepareVMCall();
+ pushArg(R1);
+ pushArg(R0.scratchReg());
+
+ using Fn = JSObject* (*)(JSContext*, HandleObject, HandleValue);
+ if (!callVM<Fn, js::CreateAsyncFromSyncIterator>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.popn(2);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CanSkipAwait() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ prepareVMCall();
+ pushArg(R0);
+
+ using Fn = bool (*)(JSContext*, HandleValue, bool* canSkip);
+ if (!callVM<Fn, js::CanSkipAwait>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_MaybeExtractAwaitValue() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-2), R0);
+
+ masm.unboxBoolean(frame.addressOfStackValue(-1), R1.scratchReg());
+
+ Label cantExtract;
+ masm.branchIfFalseBool(R1.scratchReg(), &cantExtract);
+
+ prepareVMCall();
+ pushArg(R0);
+
+ using Fn = bool (*)(JSContext*, HandleValue, MutableHandleValue);
+ if (!callVM<Fn, js::ExtractAwaitValue>()) {
+ return false;
+ }
+
+ masm.storeValue(R0, frame.addressOfStackValue(-2));
+ masm.bind(&cantExtract);
+
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_AsyncAwait() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-2), R1);
+ masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg());
+
+ prepareVMCall();
+ pushArg(R1);
+ pushArg(R0.scratchReg());
+
+ using Fn = JSObject* (*)(JSContext*, Handle<AsyncFunctionGeneratorObject*>,
+ HandleValue);
+ if (!callVM<Fn, js::AsyncFunctionAwait>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.popn(2);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_AsyncResolve() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-2), R1);
+ masm.unboxObject(frame.addressOfStackValue(-1), R0.scratchReg());
+
+ prepareVMCall();
+ pushUint8BytecodeOperandArg(R2.scratchReg());
+ pushArg(R1);
+ pushArg(R0.scratchReg());
+
+ using Fn = JSObject* (*)(JSContext*, Handle<AsyncFunctionGeneratorObject*>,
+ HandleValue, AsyncFunctionResolveKind);
+ if (!callVM<Fn, js::AsyncFunctionResolve>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.popn(2);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckObjCoercible() {
+ frame.syncStack(0);
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ Label fail, done;
+
+ masm.branchTestUndefined(Assembler::Equal, R0, &fail);
+ masm.branchTestNull(Assembler::NotEqual, R0, &done);
+
+ masm.bind(&fail);
+ prepareVMCall();
+
+ pushArg(R0);
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ if (!callVM<Fn, ThrowObjectCoercible>()) {
+ return false;
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_ToString() {
+ // Keep top stack value in R0.
+ frame.popRegsAndSync(1);
+
+ // Inline path for string.
+ Label done;
+ masm.branchTestString(Assembler::Equal, R0, &done);
+
+ prepareVMCall();
+
+ pushArg(R0);
+
+ // Call ToStringSlow which doesn't handle string inputs.
+ using Fn = JSString* (*)(JSContext*, HandleValue);
+ if (!callVM<Fn, ToStringSlow<CanGC>>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_STRING, ReturnReg, R0);
+
+ masm.bind(&done);
+ frame.push(R0);
+ return true;
+}
+
+static constexpr uint32_t TableSwitchOpLowOffset = 1 * JUMP_OFFSET_LEN;
+static constexpr uint32_t TableSwitchOpHighOffset = 2 * JUMP_OFFSET_LEN;
+static constexpr uint32_t TableSwitchOpFirstResumeIndexOffset =
+ 3 * JUMP_OFFSET_LEN;
+
+template <>
+void BaselineCompilerCodeGen::emitGetTableSwitchIndex(ValueOperand val,
+ Register dest,
+ Register scratch1,
+ Register scratch2) {
+ jsbytecode* pc = handler.pc();
+ jsbytecode* defaultpc = pc + GET_JUMP_OFFSET(pc);
+ Label* defaultLabel = handler.labelOf(defaultpc);
+
+ int32_t low = GET_JUMP_OFFSET(pc + TableSwitchOpLowOffset);
+ int32_t high = GET_JUMP_OFFSET(pc + TableSwitchOpHighOffset);
+ int32_t length = high - low + 1;
+
+ // Jump to the 'default' pc if not int32 (tableswitch is only used when
+ // all cases are int32).
+ masm.branchTestInt32(Assembler::NotEqual, val, defaultLabel);
+ masm.unboxInt32(val, dest);
+
+ // Subtract 'low'. Bounds check.
+ if (low != 0) {
+ masm.sub32(Imm32(low), dest);
+ }
+ masm.branch32(Assembler::AboveOrEqual, dest, Imm32(length), defaultLabel);
+}
+
+template <>
+void BaselineInterpreterCodeGen::emitGetTableSwitchIndex(ValueOperand val,
+ Register dest,
+ Register scratch1,
+ Register scratch2) {
+ // Jump to the 'default' pc if not int32 (tableswitch is only used when
+ // all cases are int32).
+ Label done, jumpToDefault;
+ masm.branchTestInt32(Assembler::NotEqual, val, &jumpToDefault);
+ masm.unboxInt32(val, dest);
+
+ Register pcReg = LoadBytecodePC(masm, scratch1);
+ Address lowAddr(pcReg, sizeof(jsbytecode) + TableSwitchOpLowOffset);
+ Address highAddr(pcReg, sizeof(jsbytecode) + TableSwitchOpHighOffset);
+
+ // Jump to default if val > high.
+ masm.branch32(Assembler::LessThan, highAddr, dest, &jumpToDefault);
+
+ // Jump to default if val < low.
+ masm.load32(lowAddr, scratch2);
+ masm.branch32(Assembler::GreaterThan, scratch2, dest, &jumpToDefault);
+
+ // index := val - low.
+ masm.sub32(scratch2, dest);
+ masm.jump(&done);
+
+ masm.bind(&jumpToDefault);
+ emitJump();
+
+ masm.bind(&done);
+}
+
+template <>
+void BaselineCompilerCodeGen::emitTableSwitchJump(Register key,
+ Register scratch1,
+ Register scratch2) {
+ // Jump to resumeEntries[firstResumeIndex + key].
+
+ // Note: BytecodeEmitter::allocateResumeIndex static_asserts
+ // |firstResumeIndex * sizeof(uintptr_t)| fits in int32_t.
+ uint32_t firstResumeIndex =
+ GET_RESUMEINDEX(handler.pc() + TableSwitchOpFirstResumeIndexOffset);
+ LoadBaselineScriptResumeEntries(masm, handler.script(), scratch1, scratch2);
+ masm.loadPtr(BaseIndex(scratch1, key, ScaleFromElemWidth(sizeof(uintptr_t)),
+ firstResumeIndex * sizeof(uintptr_t)),
+ scratch1);
+ masm.jump(scratch1);
+}
+
+template <>
+void BaselineInterpreterCodeGen::emitTableSwitchJump(Register key,
+ Register scratch1,
+ Register scratch2) {
+ // Load the op's firstResumeIndex in scratch1.
+ LoadUint24Operand(masm, TableSwitchOpFirstResumeIndexOffset, scratch1);
+
+ masm.add32(key, scratch1);
+ jumpToResumeEntry(scratch1, key, scratch2);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_TableSwitch() {
+ frame.popRegsAndSync(1);
+
+ Register key = R0.scratchReg();
+ Register scratch1 = R1.scratchReg();
+ Register scratch2 = R2.scratchReg();
+
+ // Call a stub to convert R0 from double to int32 if needed.
+ // Note: this stub may clobber scratch1.
+ masm.call(cx->runtime()->jitRuntime()->getDoubleToInt32ValueStub());
+
+ // Load the index in the jump table in |key|, or branch to default pc if not
+ // int32 or out-of-range.
+ emitGetTableSwitchIndex(R0, key, scratch1, scratch2);
+
+ // Jump to the target pc.
+ emitTableSwitchJump(key, scratch1, scratch2);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Iter() {
+ frame.popRegsAndSync(1);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_MoreIter() {
+ frame.syncStack(0);
+
+ masm.unboxObject(frame.addressOfStackValue(-1), R1.scratchReg());
+
+ masm.iteratorMore(R1.scratchReg(), R0, R2.scratchReg());
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitIsMagicValue() {
+ frame.syncStack(0);
+
+ Label isMagic, done;
+ masm.branchTestMagic(Assembler::Equal, frame.addressOfStackValue(-1),
+ &isMagic);
+ masm.moveValue(BooleanValue(false), R0);
+ masm.jump(&done);
+
+ masm.bind(&isMagic);
+ masm.moveValue(BooleanValue(true), R0);
+
+ masm.bind(&done);
+ frame.push(R0, JSVAL_TYPE_BOOLEAN);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_IsNoIter() {
+ return emitIsMagicValue();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_EndIter() {
+ // Pop iterator value.
+ frame.pop();
+
+ // Pop the iterator object to close in R0.
+ frame.popRegsAndSync(1);
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ if (HasInterpreterPCReg()) {
+ regs.take(InterpreterPCReg);
+ }
+
+ Register obj = R0.scratchReg();
+ regs.take(obj);
+ masm.unboxObject(R0, obj);
+
+ Register temp1 = regs.takeAny();
+ Register temp2 = regs.takeAny();
+ Register temp3 = regs.takeAny();
+ masm.iteratorClose(obj, temp1, temp2, temp3);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CloseIter() {
+ frame.popRegsAndSync(1);
+
+ Register iter = R0.scratchReg();
+ masm.unboxObject(R0, iter);
+
+ return emitNextIC();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_IsGenClosing() {
+ return emitIsMagicValue();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_IsNullOrUndefined() {
+ frame.syncStack(0);
+
+ Label isNullOrUndefined, done;
+ masm.branchTestNull(Assembler::Equal, frame.addressOfStackValue(-1),
+ &isNullOrUndefined);
+ masm.branchTestUndefined(Assembler::Equal, frame.addressOfStackValue(-1),
+ &isNullOrUndefined);
+ masm.moveValue(BooleanValue(false), R0);
+ masm.jump(&done);
+
+ masm.bind(&isNullOrUndefined);
+ masm.moveValue(BooleanValue(true), R0);
+
+ masm.bind(&done);
+ frame.push(R0, JSVAL_TYPE_BOOLEAN);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_GetRval() {
+ frame.syncStack(0);
+
+ emitLoadReturnValue(R0);
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SetRval() {
+ // Store to the frame's return value slot.
+ frame.storeStackValue(-1, frame.addressOfReturnValue(), R2);
+ masm.or32(Imm32(BaselineFrame::HAS_RVAL), frame.addressOfFlags());
+ frame.pop();
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Callee() {
+ MOZ_ASSERT_IF(handler.maybeScript(), handler.maybeScript()->function());
+ frame.syncStack(0);
+ masm.loadFunctionFromCalleeToken(frame.addressOfCalleeToken(),
+ R0.scratchReg());
+ masm.tagValue(JSVAL_TYPE_OBJECT, R0.scratchReg(), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_EnvCallee() {
+ frame.syncStack(0);
+ uint8_t numHops = GET_UINT8(handler.pc());
+ Register scratch = R0.scratchReg();
+
+ masm.loadPtr(frame.addressOfEnvironmentChain(), scratch);
+ for (unsigned i = 0; i < numHops; i++) {
+ Address nextAddr(scratch,
+ EnvironmentObject::offsetOfEnclosingEnvironment());
+ masm.unboxObject(nextAddr, scratch);
+ }
+
+ masm.loadValue(Address(scratch, CallObject::offsetOfCallee()), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_EnvCallee() {
+ Register scratch = R0.scratchReg();
+ Register env = R1.scratchReg();
+
+ static_assert(JSOpLength_EnvCallee - sizeof(jsbytecode) == ENVCOORD_HOPS_LEN,
+ "op must have uint8 operand for LoadAliasedVarEnv");
+
+ // Load the right environment object.
+ masm.loadPtr(frame.addressOfEnvironmentChain(), env);
+ LoadAliasedVarEnv(masm, env, scratch);
+
+ masm.pushValue(Address(env, CallObject::offsetOfCallee()));
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SuperBase() {
+ frame.popRegsAndSync(1);
+
+ Register scratch = R0.scratchReg();
+ Register proto = R1.scratchReg();
+
+ // Unbox callee.
+ masm.unboxObject(R0, scratch);
+
+ // Load [[HomeObject]]
+ Address homeObjAddr(scratch,
+ FunctionExtended::offsetOfMethodHomeObjectSlot());
+
+ masm.assertFunctionIsExtended(scratch);
+#ifdef DEBUG
+ Label isObject;
+ masm.branchTestObject(Assembler::Equal, homeObjAddr, &isObject);
+ masm.assumeUnreachable("[[HomeObject]] must be Object");
+ masm.bind(&isObject);
+#endif
+ masm.unboxObject(homeObjAddr, scratch);
+
+ // Load prototype from [[HomeObject]]
+ masm.loadObjProto(scratch, proto);
+
+#ifdef DEBUG
+ // We won't encounter a lazy proto, because the prototype is guaranteed to
+ // either be a JSFunction or a PlainObject, and only proxy objects can have a
+ // lazy proto.
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ Label proxyCheckDone;
+ masm.branchPtr(Assembler::NotEqual, proto, ImmWord(1), &proxyCheckDone);
+ masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperBase");
+ masm.bind(&proxyCheckDone);
+#endif
+
+ Label nullProto, done;
+ masm.branchPtr(Assembler::Equal, proto, ImmWord(0), &nullProto);
+
+ // Box prototype and return
+ masm.tagValue(JSVAL_TYPE_OBJECT, proto, R1);
+ masm.jump(&done);
+
+ masm.bind(&nullProto);
+ masm.moveValue(NullValue(), R1);
+
+ masm.bind(&done);
+ frame.push(R1);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_SuperFun() {
+ frame.popRegsAndSync(1);
+
+ Register callee = R0.scratchReg();
+ Register proto = R1.scratchReg();
+#ifdef DEBUG
+ Register scratch = R2.scratchReg();
+#endif
+
+ // Unbox callee.
+ masm.unboxObject(R0, callee);
+
+#ifdef DEBUG
+ Label classCheckDone;
+ masm.branchTestObjIsFunction(Assembler::Equal, callee, scratch, callee,
+ &classCheckDone);
+ masm.assumeUnreachable("Unexpected non-JSFunction callee in JSOp::SuperFun");
+ masm.bind(&classCheckDone);
+#endif
+
+ // Load prototype of callee
+ masm.loadObjProto(callee, proto);
+
+#ifdef DEBUG
+ // We won't encounter a lazy proto, because |callee| is guaranteed to be a
+ // JSFunction and only proxy objects can have a lazy proto.
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ Label proxyCheckDone;
+ masm.branchPtr(Assembler::NotEqual, proto, ImmWord(1), &proxyCheckDone);
+ masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperFun");
+ masm.bind(&proxyCheckDone);
+#endif
+
+ Label nullProto, done;
+ masm.branchPtr(Assembler::Equal, proto, ImmWord(0), &nullProto);
+
+ // Box prototype and return
+ masm.tagValue(JSVAL_TYPE_OBJECT, proto, R1);
+ masm.jump(&done);
+
+ masm.bind(&nullProto);
+ masm.moveValue(NullValue(), R1);
+
+ masm.bind(&done);
+ frame.push(R1);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Arguments() {
+ frame.syncStack(0);
+
+ MOZ_ASSERT_IF(handler.maybeScript(), handler.maybeScript()->needsArgsObj());
+
+ prepareVMCall();
+
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, MutableHandleValue);
+ if (!callVM<Fn, jit::NewArgumentsObject>()) {
+ return false;
+ }
+
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Rest() {
+ frame.syncStack(0);
+
+ if (!emitNextIC()) {
+ return false;
+ }
+
+ // Mark R0 as pushed stack value.
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Generator() {
+ frame.assertStackDepth(0);
+
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ prepareVMCall();
+ pushArg(R0.scratchReg());
+
+ using Fn = JSObject* (*)(JSContext*, BaselineFrame*);
+ if (!callVM<Fn, jit::CreateGeneratorFromFrame>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitSuspend(JSOp op) {
+ MOZ_ASSERT(op == JSOp::InitialYield || op == JSOp::Yield ||
+ op == JSOp::Await);
+
+ // Load the generator object in R2, but leave the return value on the
+ // expression stack.
+ Register genObj = R2.scratchReg();
+ if (op == JSOp::InitialYield) {
+ // Generator and return value are one and the same.
+ frame.syncStack(0);
+ frame.assertStackDepth(1);
+ masm.unboxObject(frame.addressOfStackValue(-1), genObj);
+ } else {
+ frame.popRegsAndSync(1);
+ masm.unboxObject(R0, genObj);
+ }
+
+ if (frame.hasKnownStackDepth(1) && !handler.canHaveFixedSlots()) {
+ // If the expression stack is empty, we can inline the Yield. Note that this
+ // branch is never taken for the interpreter because it doesn't know static
+ // stack depths.
+ MOZ_ASSERT_IF(op == JSOp::InitialYield && handler.maybePC(),
+ GET_RESUMEINDEX(handler.maybePC()) == 0);
+ Address resumeIndexSlot(genObj,
+ AbstractGeneratorObject::offsetOfResumeIndexSlot());
+ Register temp = R1.scratchReg();
+ if (op == JSOp::InitialYield) {
+ masm.storeValue(Int32Value(0), resumeIndexSlot);
+ } else {
+ jsbytecode* pc = handler.maybePC();
+ MOZ_ASSERT(pc, "compiler-only code never has a null pc");
+ masm.move32(Imm32(GET_RESUMEINDEX(pc)), temp);
+ masm.storeValue(JSVAL_TYPE_INT32, temp, resumeIndexSlot);
+ }
+
+ Register envObj = R0.scratchReg();
+ Address envChainSlot(
+ genObj, AbstractGeneratorObject::offsetOfEnvironmentChainSlot());
+ masm.loadPtr(frame.addressOfEnvironmentChain(), envObj);
+ masm.guardedCallPreBarrierAnyZone(envChainSlot, MIRType::Value, temp);
+ masm.storeValue(JSVAL_TYPE_OBJECT, envObj, envChainSlot);
+
+ Label skipBarrier;
+ masm.branchPtrInNurseryChunk(Assembler::Equal, genObj, temp, &skipBarrier);
+ masm.branchPtrInNurseryChunk(Assembler::NotEqual, envObj, temp,
+ &skipBarrier);
+ MOZ_ASSERT(genObj == R2.scratchReg());
+ masm.call(&postBarrierSlot_);
+ masm.bind(&skipBarrier);
+ } else {
+ masm.loadBaselineFramePtr(FramePointer, R1.scratchReg());
+ computeFrameSize(R0.scratchReg());
+
+ prepareVMCall();
+ pushBytecodePCArg();
+ pushArg(R0.scratchReg());
+ pushArg(R1.scratchReg());
+ pushArg(genObj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, BaselineFrame*, uint32_t,
+ const jsbytecode*);
+ if (!callVM<Fn, jit::NormalSuspend>()) {
+ return false;
+ }
+ }
+
+ masm.loadValue(frame.addressOfStackValue(-1), JSReturnOperand);
+ if (!emitReturn()) {
+ return false;
+ }
+
+ // Three values are pushed onto the stack when resuming the generator,
+ // replacing the one slot that holds the return value.
+ frame.incStackDepth(2);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitialYield() {
+ return emitSuspend(JSOp::InitialYield);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Yield() {
+ return emitSuspend(JSOp::Yield);
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Await() {
+ return emitSuspend(JSOp::Await);
+}
+
+template <>
+template <typename F>
+bool BaselineCompilerCodeGen::emitAfterYieldDebugInstrumentation(
+ const F& ifDebuggee, Register) {
+ if (handler.compileDebugInstrumentation()) {
+ return ifDebuggee();
+ }
+ return true;
+}
+
+template <>
+template <typename F>
+bool BaselineInterpreterCodeGen::emitAfterYieldDebugInstrumentation(
+ const F& ifDebuggee, Register scratch) {
+ // Note that we can't use emitDebugInstrumentation here because the frame's
+ // DEBUGGEE flag hasn't been initialized yet.
+
+ // If the current Realm is not a debuggee we're done.
+ Label done;
+ CodeOffset toggleOffset = masm.toggledJump(&done);
+ if (!handler.addDebugInstrumentationOffset(cx, toggleOffset)) {
+ return false;
+ }
+ masm.loadPtr(AbsoluteAddress(cx->addressOfRealm()), scratch);
+ masm.branchTest32(Assembler::Zero,
+ Address(scratch, Realm::offsetOfDebugModeBits()),
+ Imm32(Realm::debugModeIsDebuggeeBit()), &done);
+
+ if (!ifDebuggee()) {
+ return false;
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_AfterYield() {
+ if (!emit_JumpTarget()) {
+ return false;
+ }
+
+ auto ifDebuggee = [this]() {
+ frame.assertSyncedStack();
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ prepareVMCall();
+ pushArg(R0.scratchReg());
+
+ const RetAddrEntry::Kind kind = RetAddrEntry::Kind::DebugAfterYield;
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*);
+ if (!callVM<Fn, jit::DebugAfterYield>(kind)) {
+ return false;
+ }
+
+ return true;
+ };
+ return emitAfterYieldDebugInstrumentation(ifDebuggee, R0.scratchReg());
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_FinalYieldRval() {
+ // Store generator in R0.
+ frame.popRegsAndSync(1);
+ masm.unboxObject(R0, R0.scratchReg());
+
+ prepareVMCall();
+ pushBytecodePCArg();
+ pushArg(R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, HandleObject, const jsbytecode*);
+ if (!callVM<Fn, jit::FinalSuspend>()) {
+ return false;
+ }
+
+ masm.loadValue(frame.addressOfReturnValue(), JSReturnOperand);
+ return emitReturn();
+}
+
+template <>
+void BaselineCompilerCodeGen::emitJumpToInterpretOpLabel() {
+ TrampolinePtr code =
+ cx->runtime()->jitRuntime()->baselineInterpreter().interpretOpAddr();
+ masm.jump(code);
+}
+
+template <>
+void BaselineInterpreterCodeGen::emitJumpToInterpretOpLabel() {
+ masm.jump(handler.interpretOpLabel());
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitEnterGeneratorCode(Register script,
+ Register resumeIndex,
+ Register scratch) {
+ // Resume in either the BaselineScript (if present) or Baseline Interpreter.
+
+ static_assert(BaselineDisabledScript == 0x1,
+ "Comparison below requires specific sentinel encoding");
+
+ // Initialize the icScript slot in the baseline frame.
+ masm.loadJitScript(script, scratch);
+ masm.computeEffectiveAddress(Address(scratch, JitScript::offsetOfICScript()),
+ scratch);
+ Address icScriptAddr(FramePointer, BaselineFrame::reverseOffsetOfICScript());
+ masm.storePtr(scratch, icScriptAddr);
+
+ Label noBaselineScript;
+ masm.loadJitScript(script, scratch);
+ masm.loadPtr(Address(scratch, JitScript::offsetOfBaselineScript()), scratch);
+ masm.branchPtr(Assembler::BelowOrEqual, scratch,
+ ImmPtr(BaselineDisabledScriptPtr), &noBaselineScript);
+
+ masm.load32(Address(scratch, BaselineScript::offsetOfResumeEntriesOffset()),
+ script);
+ masm.addPtr(scratch, script);
+ masm.loadPtr(
+ BaseIndex(script, resumeIndex, ScaleFromElemWidth(sizeof(uintptr_t))),
+ scratch);
+ masm.jump(scratch);
+
+ masm.bind(&noBaselineScript);
+
+ // Initialize interpreter frame fields.
+ Address flagsAddr(FramePointer, BaselineFrame::reverseOffsetOfFlags());
+ Address scriptAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfInterpreterScript());
+ masm.or32(Imm32(BaselineFrame::RUNNING_IN_INTERPRETER), flagsAddr);
+ masm.storePtr(script, scriptAddr);
+
+ // Initialize pc and jump to it.
+ emitInterpJumpToResumeEntry(script, resumeIndex, scratch);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_Resume() {
+ frame.syncStack(0);
+ masm.assertStackAlignment(sizeof(Value), 0);
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ if (HasInterpreterPCReg()) {
+ regs.take(InterpreterPCReg);
+ }
+
+ saveInterpreterPCReg();
+
+ // Load generator object.
+ Register genObj = regs.takeAny();
+ masm.unboxObject(frame.addressOfStackValue(-3), genObj);
+
+ // Load callee.
+ Register callee = regs.takeAny();
+ masm.unboxObject(
+ Address(genObj, AbstractGeneratorObject::offsetOfCalleeSlot()), callee);
+
+ // Save a pointer to the JSOp::Resume operand stack Values.
+ Register callerStackPtr = regs.takeAny();
+ masm.computeEffectiveAddress(frame.addressOfStackValue(-1), callerStackPtr);
+
+ // Branch to |interpret| to resume the generator in the C++ interpreter if the
+ // script does not have a JitScript.
+ Label interpret;
+ Register scratch1 = regs.takeAny();
+ masm.loadPrivate(Address(callee, JSFunction::offsetOfJitInfoOrScript()),
+ scratch1);
+ masm.branchIfScriptHasNoJitScript(scratch1, &interpret);
+
+ // Push |undefined| for all formals.
+ Register scratch2 = regs.takeAny();
+ Label loop, loopDone;
+ masm.loadFunctionArgCount(callee, scratch2);
+
+ static_assert(sizeof(Value) == 8);
+ static_assert(JitStackAlignment == 16 || JitStackAlignment == 8);
+ // If JitStackValueAlignment == 1, then we were already correctly aligned on
+ // entry, as guaranteed by the assertStackAlignment at the entry to this
+ // function.
+ if (JitStackValueAlignment > 1) {
+ Register alignment = regs.takeAny();
+ masm.moveStackPtrTo(alignment);
+ masm.alignJitStackBasedOnNArgs(scratch2, false);
+
+ // Compute alignment adjustment.
+ masm.subStackPtrFrom(alignment);
+
+ // Some code, like BaselineFrame::trace, will inspect the whole range of
+ // the stack frame. In order to ensure that garbage data left behind from
+ // previous activations doesn't confuse other machinery, we zero out the
+ // alignment bytes.
+ Label alignmentZero;
+ masm.branchPtr(Assembler::Equal, alignment, ImmWord(0), &alignmentZero);
+
+ // Since we know prior to the stack alignment that the stack was 8 byte
+ // aligned, and JitStackAlignment is 8 or 16 bytes, if we are doing an
+ // alignment then we -must- have aligned by subtracting 8 bytes from
+ // the stack pointer.
+ //
+ // So we can freely store a valid double here.
+ masm.storeValue(DoubleValue(0), Address(masm.getStackPointer(), 0));
+ masm.bind(&alignmentZero);
+ }
+
+ masm.branchTest32(Assembler::Zero, scratch2, scratch2, &loopDone);
+ masm.bind(&loop);
+ {
+ masm.pushValue(UndefinedValue());
+ masm.branchSub32(Assembler::NonZero, Imm32(1), scratch2, &loop);
+ }
+ masm.bind(&loopDone);
+
+ // Push |undefined| for |this|.
+ masm.pushValue(UndefinedValue());
+
+#ifdef DEBUG
+ // Update BaselineFrame debugFrameSize field.
+ masm.mov(FramePointer, scratch2);
+ masm.subStackPtrFrom(scratch2);
+ masm.store32(scratch2, frame.addressOfDebugFrameSize());
+#endif
+
+ masm.PushCalleeToken(callee, /* constructing = */ false);
+ masm.pushFrameDescriptorForJitCall(FrameType::BaselineJS, /* argc = */ 0);
+
+ // PushCalleeToken bumped framePushed. Reset it.
+ MOZ_ASSERT(masm.framePushed() == sizeof(uintptr_t));
+ masm.setFramePushed(0);
+
+ regs.add(callee);
+
+ // Push a fake return address on the stack. We will resume here when the
+ // generator returns.
+ Label genStart, returnTarget;
+#ifdef JS_USE_LINK_REGISTER
+ masm.call(&genStart);
+#else
+ masm.callAndPushReturnAddress(&genStart);
+#endif
+
+ // Record the return address so the return offset -> pc mapping works.
+ if (!handler.recordCallRetAddr(cx, RetAddrEntry::Kind::IC,
+ masm.currentOffset())) {
+ return false;
+ }
+
+ masm.jump(&returnTarget);
+ masm.bind(&genStart);
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+
+ // Construct BaselineFrame.
+ masm.push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ // If profiler instrumentation is on, update lastProfilingFrame on
+ // current JitActivation
+ {
+ Register scratchReg = scratch2;
+ Label skip;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0), &skip);
+ masm.loadJSContext(scratchReg);
+ masm.loadPtr(Address(scratchReg, JSContext::offsetOfProfilingActivation()),
+ scratchReg);
+ masm.storePtr(
+ FramePointer,
+ Address(scratchReg, JitActivation::offsetOfLastProfilingFrame()));
+ masm.bind(&skip);
+ }
+
+ masm.subFromStackPtr(Imm32(BaselineFrame::Size()));
+ masm.assertStackAlignment(sizeof(Value), 0);
+
+ // Store flags and env chain.
+ masm.store32(Imm32(BaselineFrame::HAS_INITIAL_ENV), frame.addressOfFlags());
+ masm.unboxObject(
+ Address(genObj, AbstractGeneratorObject::offsetOfEnvironmentChainSlot()),
+ scratch2);
+ masm.storePtr(scratch2, frame.addressOfEnvironmentChain());
+
+ // Store the arguments object if there is one.
+ Label noArgsObj;
+ Address argsObjSlot(genObj, AbstractGeneratorObject::offsetOfArgsObjSlot());
+ masm.fallibleUnboxObject(argsObjSlot, scratch2, &noArgsObj);
+ {
+ masm.storePtr(scratch2, frame.addressOfArgsObj());
+ masm.or32(Imm32(BaselineFrame::HAS_ARGS_OBJ), frame.addressOfFlags());
+ }
+ masm.bind(&noArgsObj);
+
+ // Push locals and expression slots if needed.
+ Label noStackStorage;
+ Address stackStorageSlot(genObj,
+ AbstractGeneratorObject::offsetOfStackStorageSlot());
+ masm.fallibleUnboxObject(stackStorageSlot, scratch2, &noStackStorage);
+ {
+ Register initLength = regs.takeAny();
+ masm.loadPtr(Address(scratch2, NativeObject::offsetOfElements()), scratch2);
+ masm.load32(Address(scratch2, ObjectElements::offsetOfInitializedLength()),
+ initLength);
+ masm.store32(
+ Imm32(0),
+ Address(scratch2, ObjectElements::offsetOfInitializedLength()));
+
+ Label loop, loopDone;
+ masm.branchTest32(Assembler::Zero, initLength, initLength, &loopDone);
+ masm.bind(&loop);
+ {
+ masm.pushValue(Address(scratch2, 0));
+ masm.guardedCallPreBarrierAnyZone(Address(scratch2, 0), MIRType::Value,
+ scratch1);
+ masm.addPtr(Imm32(sizeof(Value)), scratch2);
+ masm.branchSub32(Assembler::NonZero, Imm32(1), initLength, &loop);
+ }
+ masm.bind(&loopDone);
+ regs.add(initLength);
+ }
+
+ masm.bind(&noStackStorage);
+
+ // Push arg, generator, resumeKind stack Values, in that order.
+ masm.pushValue(Address(callerStackPtr, sizeof(Value)));
+ masm.pushValue(JSVAL_TYPE_OBJECT, genObj);
+ masm.pushValue(Address(callerStackPtr, 0));
+
+ masm.switchToObjectRealm(genObj, scratch2);
+
+ // Load script in scratch1.
+ masm.unboxObject(
+ Address(genObj, AbstractGeneratorObject::offsetOfCalleeSlot()), scratch1);
+ masm.loadPrivate(Address(scratch1, JSFunction::offsetOfJitInfoOrScript()),
+ scratch1);
+
+ // Load resume index in scratch2 and mark generator as running.
+ Address resumeIndexSlot(genObj,
+ AbstractGeneratorObject::offsetOfResumeIndexSlot());
+ masm.unboxInt32(resumeIndexSlot, scratch2);
+ masm.storeValue(Int32Value(AbstractGeneratorObject::RESUME_INDEX_RUNNING),
+ resumeIndexSlot);
+
+ if (!emitEnterGeneratorCode(scratch1, scratch2, regs.getAny())) {
+ return false;
+ }
+
+ // Call into the VM to resume the generator in the C++ interpreter if there's
+ // no JitScript.
+ masm.bind(&interpret);
+
+ prepareVMCall();
+
+ pushArg(callerStackPtr);
+ pushArg(genObj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, Value*, MutableHandleValue);
+ if (!callVM<Fn, jit::InterpretResume>()) {
+ return false;
+ }
+
+ masm.bind(&returnTarget);
+
+ // Restore Stack pointer
+ masm.computeEffectiveAddress(frame.addressOfStackValue(-1),
+ masm.getStackPointer());
+
+ // After the generator returns, we restore the stack pointer, switch back to
+ // the current realm, push the return value, and we're done.
+ if (JSScript* script = handler.maybeScript()) {
+ masm.switchToRealm(script->realm(), R2.scratchReg());
+ } else {
+ masm.switchToBaselineFrameRealm(R2.scratchReg());
+ }
+ restoreInterpreterPCReg();
+ frame.popn(3);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckResumeKind() {
+ // Load resumeKind in R1, generator in R0.
+ frame.popRegsAndSync(2);
+
+#ifdef DEBUG
+ Label ok;
+ masm.branchTestInt32(Assembler::Equal, R1, &ok);
+ masm.assumeUnreachable("Expected int32 resumeKind");
+ masm.bind(&ok);
+#endif
+
+ // If resumeKind is 'next' we don't have to do anything.
+ Label done;
+ masm.unboxInt32(R1, R1.scratchReg());
+ masm.branch32(Assembler::Equal, R1.scratchReg(),
+ Imm32(int32_t(GeneratorResumeKind::Next)), &done);
+
+ prepareVMCall();
+
+ pushArg(R1.scratchReg()); // resumeKind
+
+ masm.loadValue(frame.addressOfStackValue(-1), R2);
+ pushArg(R2); // arg
+
+ masm.unboxObject(R0, R0.scratchReg());
+ pushArg(R0.scratchReg()); // genObj
+
+ masm.loadBaselineFramePtr(FramePointer, R2.scratchReg());
+ pushArg(R2.scratchReg()); // frame
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*,
+ Handle<AbstractGeneratorObject*>, HandleValue, int32_t);
+ if (!callVM<Fn, jit::GeneratorThrowOrReturn>()) {
+ return false;
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_ResumeKind() {
+ GeneratorResumeKind resumeKind = ResumeKindFromPC(handler.pc());
+ frame.push(Int32Value(int32_t(resumeKind)));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_ResumeKind() {
+ LoadUint8Operand(masm, R0.scratchReg());
+ masm.tagValue(JSVAL_TYPE_INT32, R0.scratchReg(), R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_DebugCheckSelfHosted() {
+#ifdef DEBUG
+ frame.syncStack(0);
+
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ prepareVMCall();
+ pushArg(R0);
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ if (!callVM<Fn, js::Debug_CheckSelfHosted>()) {
+ return false;
+ }
+#endif
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_IsConstructing() {
+ frame.push(MagicValue(JS_IS_CONSTRUCTING));
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_JumpTarget() {
+ MaybeIncrementCodeCoverageCounter(masm, handler.script(), handler.pc());
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_JumpTarget() {
+ Register scratch1 = R0.scratchReg();
+ Register scratch2 = R1.scratchReg();
+
+ Label skipCoverage;
+ CodeOffset toggleOffset = masm.toggledJump(&skipCoverage);
+ masm.call(handler.codeCoverageAtPCLabel());
+ masm.bind(&skipCoverage);
+ if (!handler.codeCoverageOffsets().append(toggleOffset.offset())) {
+ return false;
+ }
+
+ // Load icIndex in scratch1.
+ LoadInt32Operand(masm, scratch1);
+
+ // Compute ICEntry* and store to frame->interpreterICEntry.
+ masm.loadPtr(frame.addressOfICScript(), scratch2);
+ static_assert(sizeof(ICEntry) == sizeof(uintptr_t));
+ masm.computeEffectiveAddress(BaseIndex(scratch2, scratch1, ScalePointer,
+ ICScript::offsetOfICEntries()),
+ scratch2);
+ masm.storePtr(scratch2, frame.addressOfInterpreterICEntry());
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_CheckClassHeritage() {
+ frame.syncStack(0);
+
+ // Leave the heritage value on the stack.
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ prepareVMCall();
+ pushArg(R0);
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ return callVM<Fn, js::CheckClassHeritageOperation>();
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_InitHomeObject() {
+ // Load HomeObject in R0.
+ frame.popRegsAndSync(1);
+
+ // Load function off stack
+ Register func = R2.scratchReg();
+ masm.unboxObject(frame.addressOfStackValue(-1), func);
+
+ masm.assertFunctionIsExtended(func);
+
+ // Set HOMEOBJECT_SLOT
+ Register temp = R1.scratchReg();
+ Address addr(func, FunctionExtended::offsetOfMethodHomeObjectSlot());
+ masm.guardedCallPreBarrierAnyZone(addr, MIRType::Value, temp);
+ masm.storeValue(R0, addr);
+
+ Label skipBarrier;
+ masm.branchPtrInNurseryChunk(Assembler::Equal, func, temp, &skipBarrier);
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, R0, temp, &skipBarrier);
+ masm.call(&postBarrierSlot_);
+ masm.bind(&skipBarrier);
+
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_BuiltinObject() {
+ // Built-in objects are constants for a given global.
+ auto kind = BuiltinObjectKind(GET_UINT8(handler.pc()));
+ JSObject* builtin = BuiltinObjectOperation(cx, kind);
+ if (!builtin) {
+ return false;
+ }
+ frame.push(ObjectValue(*builtin));
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_BuiltinObject() {
+ prepareVMCall();
+
+ pushUint8BytecodeOperandArg(R0.scratchReg());
+
+ using Fn = JSObject* (*)(JSContext*, BuiltinObjectKind);
+ if (!callVM<Fn, BuiltinObjectOperation>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_ObjWithProto() {
+ frame.syncStack(0);
+
+ // Leave the proto value on the stack for the decompiler
+ masm.loadValue(frame.addressOfStackValue(-1), R0);
+
+ prepareVMCall();
+ pushArg(R0);
+
+ using Fn = PlainObject* (*)(JSContext*, HandleValue);
+ if (!callVM<Fn, js::ObjectWithProtoOperation>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.pop();
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_FunWithProto() {
+ frame.popRegsAndSync(1);
+
+ masm.unboxObject(R0, R0.scratchReg());
+ masm.loadPtr(frame.addressOfEnvironmentChain(), R1.scratchReg());
+
+ prepareVMCall();
+ pushArg(R0.scratchReg());
+ pushArg(R1.scratchReg());
+ pushScriptGCThingArg(ScriptGCThingType::Function, R0.scratchReg(),
+ R1.scratchReg());
+
+ using Fn =
+ JSObject* (*)(JSContext*, HandleFunction, HandleObject, HandleObject);
+ if (!callVM<Fn, js::FunWithProtoOperation>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_ImportMeta() {
+ // Note: this is like the interpreter implementation, but optimized a bit by
+ // calling GetModuleObjectForScript at compile-time.
+
+ Rooted<ModuleObject*> module(cx, GetModuleObjectForScript(handler.script()));
+ MOZ_ASSERT(module);
+
+ frame.syncStack(0);
+
+ prepareVMCall();
+ pushArg(ImmGCPtr(module));
+
+ using Fn = JSObject* (*)(JSContext*, HandleObject);
+ if (!callVM<Fn, js::GetOrCreateModuleMetaObject>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_ImportMeta() {
+ prepareVMCall();
+
+ pushScriptArg();
+
+ using Fn = JSObject* (*)(JSContext*, HandleScript);
+ if (!callVM<Fn, ImportMetaOperation>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emit_DynamicImport() {
+ // Put specifier into R0 and object value into R1
+ frame.popRegsAndSync(2);
+
+ prepareVMCall();
+ pushArg(R1);
+ pushArg(R0);
+ pushScriptArg();
+
+ using Fn = JSObject* (*)(JSContext*, HandleScript, HandleValue, HandleValue);
+ if (!callVM<Fn, js::StartDynamicModuleImport>()) {
+ return false;
+ }
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, ReturnReg, R0);
+ frame.push(R0);
+ return true;
+}
+
+template <>
+bool BaselineCompilerCodeGen::emit_ForceInterpreter() {
+ // Caller is responsible for checking script->hasForceInterpreterOp().
+ MOZ_CRASH("JSOp::ForceInterpreter in baseline");
+}
+
+template <>
+bool BaselineInterpreterCodeGen::emit_ForceInterpreter() {
+ masm.assumeUnreachable("JSOp::ForceInterpreter");
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitPrologue() {
+ AutoCreatedBy acb(masm, "BaselineCodeGen<Handler>::emitPrologue");
+
+#ifdef JS_USE_LINK_REGISTER
+ // Push link register from generateEnterJIT()'s BLR.
+ masm.pushReturnAddress();
+#endif
+
+ masm.push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ masm.checkStackAlignment();
+
+ emitProfilerEnterFrame();
+
+ masm.subFromStackPtr(Imm32(BaselineFrame::Size()));
+
+ // Initialize BaselineFrame. Also handles env chain pre-initialization (in
+ // case GC gets run during stack check). For global and eval scripts, the env
+ // chain is in R1. For function scripts, the env chain is in the callee.
+ emitInitFrameFields(R1.scratchReg());
+
+ // When compiling with Debugger instrumentation, set the debuggeeness of
+ // the frame before any operation that can call into the VM.
+ if (!emitIsDebuggeeCheck()) {
+ return false;
+ }
+
+ // Initialize the env chain before any operation that may call into the VM and
+ // trigger a GC.
+ if (!initEnvironmentChain()) {
+ return false;
+ }
+
+ // Check for overrecursion before initializing locals.
+ if (!emitStackCheck()) {
+ return false;
+ }
+
+ emitInitializeLocals();
+
+ // Ion prologue bailouts will enter here in the Baseline Interpreter.
+ masm.bind(&bailoutPrologue_);
+
+ frame.assertSyncedStack();
+
+ if (JSScript* script = handler.maybeScript()) {
+ masm.debugAssertContextRealm(script->realm(), R1.scratchReg());
+ }
+
+ if (!emitDebugPrologue()) {
+ return false;
+ }
+
+ if (!emitHandleCodeCoverageAtPrologue()) {
+ return false;
+ }
+
+ if (!emitWarmUpCounterIncrement()) {
+ return false;
+ }
+
+ warmUpCheckPrologueOffset_ = CodeOffset(masm.currentOffset());
+
+ return true;
+}
+
+template <typename Handler>
+bool BaselineCodeGen<Handler>::emitEpilogue() {
+ AutoCreatedBy acb(masm, "BaselineCodeGen<Handler>::emitEpilogue");
+
+ masm.bind(&return_);
+
+ if (!handler.shouldEmitDebugEpilogueAtReturnOp()) {
+ if (!emitDebugEpilogue()) {
+ return false;
+ }
+ }
+
+ emitProfilerExitFrame();
+
+ masm.moveToStackPtr(FramePointer);
+ masm.pop(FramePointer);
+
+ masm.ret();
+ return true;
+}
+
+MethodStatus BaselineCompiler::emitBody() {
+ AutoCreatedBy acb(masm, "BaselineCompiler::emitBody");
+
+ JSScript* script = handler.script();
+ MOZ_ASSERT(handler.pc() == script->code());
+
+ mozilla::DebugOnly<jsbytecode*> prevpc = handler.pc();
+
+ while (true) {
+ JSOp op = JSOp(*handler.pc());
+ JitSpew(JitSpew_BaselineOp, "Compiling op @ %d: %s",
+ int(script->pcToOffset(handler.pc())), CodeName(op));
+
+ BytecodeInfo* info = handler.analysis().maybeInfo(handler.pc());
+
+ // Skip unreachable ops.
+ if (!info) {
+ // Test if last instructions and stop emitting in that case.
+ handler.moveToNextPC();
+ if (handler.pc() >= script->codeEnd()) {
+ break;
+ }
+
+ prevpc = handler.pc();
+ continue;
+ }
+
+ if (info->jumpTarget) {
+ // Fully sync the stack if there are incoming jumps.
+ frame.syncStack(0);
+ frame.setStackDepth(info->stackDepth);
+ masm.bind(handler.labelOf(handler.pc()));
+ } else if (MOZ_UNLIKELY(compileDebugInstrumentation())) {
+ // Also fully sync the stack if the debugger is enabled.
+ frame.syncStack(0);
+ } else {
+ // At the beginning of any op, at most the top 2 stack-values are
+ // unsynced.
+ if (frame.stackDepth() > 2) {
+ frame.syncStack(2);
+ }
+ }
+
+ frame.assertValidState(*info);
+
+ // If the script has a resume offset for this pc we need to keep track of
+ // the native code offset.
+ if (info->hasResumeOffset) {
+ frame.assertSyncedStack();
+ uint32_t pcOffset = script->pcToOffset(handler.pc());
+ uint32_t nativeOffset = masm.currentOffset();
+ if (!resumeOffsetEntries_.emplaceBack(pcOffset, nativeOffset)) {
+ ReportOutOfMemory(cx);
+ return Method_Error;
+ }
+ }
+
+ // Emit traps for breakpoints and step mode.
+ if (MOZ_UNLIKELY(compileDebugInstrumentation()) && !emitDebugTrap()) {
+ return Method_Error;
+ }
+
+ perfSpewer_.recordInstruction(masm, op);
+
+#define EMIT_OP(OP, ...) \
+ case JSOp::OP: { \
+ AutoCreatedBy acb(masm, "op=" #OP); \
+ if (MOZ_UNLIKELY(!this->emit_##OP())) return Method_Error; \
+ } break;
+
+ switch (op) {
+ FOR_EACH_OPCODE(EMIT_OP)
+ default:
+ MOZ_CRASH("Unexpected op");
+ }
+
+#undef EMIT_OP
+
+ MOZ_ASSERT(masm.framePushed() == 0);
+
+ // Test if last instructions and stop emitting in that case.
+ handler.moveToNextPC();
+ if (handler.pc() >= script->codeEnd()) {
+ break;
+ }
+
+#ifdef DEBUG
+ prevpc = handler.pc();
+#endif
+ }
+
+ MOZ_ASSERT(JSOp(*prevpc) == JSOp::RetRval || JSOp(*prevpc) == JSOp::Return);
+ return Method_Compiled;
+}
+
+bool BaselineInterpreterGenerator::emitDebugTrap() {
+ CodeOffset offset = masm.nopPatchableToCall();
+ if (!debugTrapOffsets_.append(offset.offset())) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ return true;
+}
+
+// Register holding the bytecode pc during dispatch. This exists so the debug
+// trap handler can reload the pc into this register when it's done.
+static constexpr Register InterpreterPCRegAtDispatch =
+ HasInterpreterPCReg() ? InterpreterPCReg : R0.scratchReg();
+
+bool BaselineInterpreterGenerator::emitInterpreterLoop() {
+ AutoCreatedBy acb(masm, "BaselineInterpreterGenerator::emitInterpreterLoop");
+
+ Register scratch1 = R0.scratchReg();
+ Register scratch2 = R1.scratchReg();
+
+ // Entry point for interpreting a bytecode op. No registers are live except
+ // for InterpreterPCReg.
+ masm.bind(handler.interpretOpWithPCRegLabel());
+
+ // Emit a patchable call for debugger breakpoints/stepping.
+ if (!emitDebugTrap()) {
+ return false;
+ }
+ Label interpretOpAfterDebugTrap;
+ masm.bind(&interpretOpAfterDebugTrap);
+
+ // Load pc, bytecode op.
+ Register pcReg = LoadBytecodePC(masm, scratch1);
+ masm.load8ZeroExtend(Address(pcReg, 0), scratch1);
+
+ // Jump to table[op].
+ {
+ CodeOffset label = masm.moveNearAddressWithPatch(scratch2);
+ if (!tableLabels_.append(label)) {
+ return false;
+ }
+ BaseIndex pointer(scratch2, scratch1, ScalePointer);
+ masm.branchToComputedAddress(pointer);
+ }
+
+ // At the end of each op, emit code to bump the pc and jump to the
+ // next op (this is also known as a threaded interpreter).
+ auto opEpilogue = [&](JSOp op, size_t opLength) -> bool {
+ MOZ_ASSERT(masm.framePushed() == 0);
+
+ if (!BytecodeFallsThrough(op)) {
+ // Nothing to do.
+ masm.assumeUnreachable("unexpected fall through");
+ return true;
+ }
+
+ // Bump frame->interpreterICEntry if needed.
+ if (BytecodeOpHasIC(op)) {
+ frame.bumpInterpreterICEntry();
+ }
+
+ // Bump bytecode PC.
+ if (HasInterpreterPCReg()) {
+ MOZ_ASSERT(InterpreterPCRegAtDispatch == InterpreterPCReg);
+ masm.addPtr(Imm32(opLength), InterpreterPCReg);
+ } else {
+ MOZ_ASSERT(InterpreterPCRegAtDispatch == scratch1);
+ masm.loadPtr(frame.addressOfInterpreterPC(), InterpreterPCRegAtDispatch);
+ masm.addPtr(Imm32(opLength), InterpreterPCRegAtDispatch);
+ masm.storePtr(InterpreterPCRegAtDispatch, frame.addressOfInterpreterPC());
+ }
+
+ if (!emitDebugTrap()) {
+ return false;
+ }
+
+ // Load the opcode, jump to table[op].
+ masm.load8ZeroExtend(Address(InterpreterPCRegAtDispatch, 0), scratch1);
+ CodeOffset label = masm.moveNearAddressWithPatch(scratch2);
+ if (!tableLabels_.append(label)) {
+ return false;
+ }
+ BaseIndex pointer(scratch2, scratch1, ScalePointer);
+ masm.branchToComputedAddress(pointer);
+ return true;
+ };
+
+ // Emit code for each bytecode op.
+ Label opLabels[JSOP_LIMIT];
+#define EMIT_OP(OP, ...) \
+ { \
+ AutoCreatedBy acb(masm, "op=" #OP); \
+ masm.bind(&opLabels[uint8_t(JSOp::OP)]); \
+ handler.setCurrentOp(JSOp::OP); \
+ if (!this->emit_##OP()) { \
+ return false; \
+ } \
+ if (!opEpilogue(JSOp::OP, JSOpLength_##OP)) { \
+ return false; \
+ } \
+ handler.resetCurrentOp(); \
+ }
+ FOR_EACH_OPCODE(EMIT_OP)
+#undef EMIT_OP
+
+ // External entry point to start interpreting bytecode ops. This is used for
+ // things like exception handling and OSR. DebugModeOSR patches JIT frames to
+ // return here from the DebugTrapHandler.
+ masm.bind(handler.interpretOpLabel());
+ interpretOpOffset_ = masm.currentOffset();
+ restoreInterpreterPCReg();
+ masm.jump(handler.interpretOpWithPCRegLabel());
+
+ // Second external entry point: this skips the debug trap for the first op
+ // and is used by OSR.
+ interpretOpNoDebugTrapOffset_ = masm.currentOffset();
+ restoreInterpreterPCReg();
+ masm.jump(&interpretOpAfterDebugTrap);
+
+ // External entry point for Ion prologue bailouts.
+ bailoutPrologueOffset_ = CodeOffset(masm.currentOffset());
+ restoreInterpreterPCReg();
+ masm.jump(&bailoutPrologue_);
+
+ // Emit debug trap handler code (target of patchable call instructions). This
+ // is just a tail call to the debug trap handler trampoline code.
+ {
+ JitRuntime* jrt = cx->runtime()->jitRuntime();
+ JitCode* handlerCode =
+ jrt->debugTrapHandler(cx, DebugTrapHandlerKind::Interpreter);
+ if (!handlerCode) {
+ return false;
+ }
+
+ debugTrapHandlerOffset_ = masm.currentOffset();
+ masm.jump(handlerCode);
+ }
+
+ // Emit the table.
+ masm.haltingAlign(sizeof(void*));
+
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
+ size_t numInstructions = JSOP_LIMIT * (sizeof(uintptr_t) / sizeof(uint32_t));
+ AutoForbidPoolsAndNops afp(&masm, numInstructions);
+#endif
+
+ tableOffset_ = masm.currentOffset();
+
+ for (size_t i = 0; i < JSOP_LIMIT; i++) {
+ const Label& opLabel = opLabels[i];
+ MOZ_ASSERT(opLabel.bound());
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ cl.target()->bind(opLabel.offset());
+ masm.addCodeLabel(cl);
+ }
+
+ return true;
+}
+
+void BaselineInterpreterGenerator::emitOutOfLineCodeCoverageInstrumentation() {
+ AutoCreatedBy acb(masm,
+ "BaselineInterpreterGenerator::"
+ "emitOutOfLineCodeCoverageInstrumentation");
+
+ masm.bind(handler.codeCoverageAtPrologueLabel());
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+
+ saveInterpreterPCReg();
+
+ using Fn1 = void (*)(BaselineFrame * frame);
+ masm.setupUnalignedABICall(R0.scratchReg());
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ masm.passABIArg(R0.scratchReg());
+ masm.callWithABI<Fn1, HandleCodeCoverageAtPrologue>();
+
+ restoreInterpreterPCReg();
+ masm.ret();
+
+ masm.bind(handler.codeCoverageAtPCLabel());
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+
+ saveInterpreterPCReg();
+
+ using Fn2 = void (*)(BaselineFrame * frame, jsbytecode * pc);
+ masm.setupUnalignedABICall(R0.scratchReg());
+ masm.loadBaselineFramePtr(FramePointer, R0.scratchReg());
+ masm.passABIArg(R0.scratchReg());
+ Register pcReg = LoadBytecodePC(masm, R2.scratchReg());
+ masm.passABIArg(pcReg);
+ masm.callWithABI<Fn2, HandleCodeCoverageAtPC>();
+
+ restoreInterpreterPCReg();
+ masm.ret();
+}
+
+bool BaselineInterpreterGenerator::generate(BaselineInterpreter& interpreter) {
+ AutoCreatedBy acb(masm, "BaselineInterpreterGenerator::generate");
+
+ if (!emitPrologue()) {
+ return false;
+ }
+
+ if (!emitInterpreterLoop()) {
+ return false;
+ }
+
+ if (!emitEpilogue()) {
+ return false;
+ }
+
+ if (!emitOutOfLinePostBarrierSlot()) {
+ return false;
+ }
+
+ emitOutOfLineCodeCoverageInstrumentation();
+
+ {
+ AutoCreatedBy acb(masm, "everything_else");
+ Linker linker(masm);
+ if (masm.oom()) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ JitCode* code = linker.newCode(cx, CodeKind::Other);
+ if (!code) {
+ return false;
+ }
+
+ // Register BaselineInterpreter code with the profiler's JitCode table.
+ {
+ auto entry = MakeJitcodeGlobalEntry<BaselineInterpreterEntry>(
+ cx, code, code->raw(), code->rawEnd());
+ if (!entry) {
+ return false;
+ }
+
+ JitcodeGlobalTable* globalTable =
+ cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
+ if (!globalTable->addEntry(std::move(entry))) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ code->setHasBytecodeMap();
+ }
+
+ // Patch loads now that we know the tableswitch base address.
+ CodeLocationLabel tableLoc(code, CodeOffset(tableOffset_));
+ for (CodeOffset off : tableLabels_) {
+ MacroAssembler::patchNearAddressMove(CodeLocationLabel(code, off),
+ tableLoc);
+ }
+
+ CollectPerfSpewerJitCodeProfile(code, "BaselineInterpreter");
+
+#ifdef MOZ_VTUNE
+ vtune::MarkStub(code, "BaselineInterpreter");
+#endif
+
+ interpreter.init(
+ code, interpretOpOffset_, interpretOpNoDebugTrapOffset_,
+ bailoutPrologueOffset_.offset(),
+ profilerEnterFrameToggleOffset_.offset(),
+ profilerExitFrameToggleOffset_.offset(), debugTrapHandlerOffset_,
+ std::move(handler.debugInstrumentationOffsets()),
+ std::move(debugTrapOffsets_), std::move(handler.codeCoverageOffsets()),
+ std::move(handler.icReturnOffsets()), handler.callVMOffsets());
+ }
+
+ if (cx->runtime()->geckoProfiler().enabled()) {
+ interpreter.toggleProfilerInstrumentation(true);
+ }
+
+ if (coverage::IsLCovEnabled()) {
+ interpreter.toggleCodeCoverageInstrumentationUnchecked(true);
+ }
+
+ return true;
+}
+
+JitCode* JitRuntime::generateDebugTrapHandler(JSContext* cx,
+ DebugTrapHandlerKind kind) {
+ TempAllocator temp(&cx->tempLifoAlloc());
+ StackMacroAssembler masm(cx, temp);
+ AutoCreatedBy acb(masm, "JitRuntime::generateDebugTrapHandler");
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.takeUnchecked(ICStubReg);
+ if (HasInterpreterPCReg()) {
+ regs.takeUnchecked(InterpreterPCReg);
+ }
+#ifdef JS_CODEGEN_ARM
+ regs.takeUnchecked(BaselineSecondScratchReg);
+ masm.setSecondScratchReg(BaselineSecondScratchReg);
+#endif
+ Register scratch1 = regs.takeAny();
+ Register scratch2 = regs.takeAny();
+ Register scratch3 = regs.takeAny();
+
+ if (kind == DebugTrapHandlerKind::Interpreter) {
+ // The interpreter calls this for every script when debugging, so check if
+ // the script has any breakpoints or is in step mode before calling into
+ // C++.
+ Label hasDebugScript;
+ Address scriptAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfInterpreterScript());
+ masm.loadPtr(scriptAddr, scratch1);
+ masm.branchTest32(Assembler::NonZero,
+ Address(scratch1, JSScript::offsetOfMutableFlags()),
+ Imm32(int32_t(JSScript::MutableFlags::HasDebugScript)),
+ &hasDebugScript);
+ masm.abiret();
+ masm.bind(&hasDebugScript);
+
+ if (HasInterpreterPCReg()) {
+ // Update frame's bytecode pc because the debugger depends on it.
+ Address pcAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfInterpreterPC());
+ masm.storePtr(InterpreterPCReg, pcAddr);
+ }
+ }
+
+ // Load the return address in scratch1.
+ masm.loadAbiReturnAddress(scratch1);
+
+ // Load BaselineFrame pointer in scratch2.
+ masm.loadBaselineFramePtr(FramePointer, scratch2);
+
+ // Enter a stub frame and call the HandleDebugTrap VM function. Ensure
+ // the stub frame has a nullptr ICStub pointer, since this pointer is marked
+ // during GC.
+ masm.movePtr(ImmPtr(nullptr), ICStubReg);
+ EmitBaselineEnterStubFrame(masm, scratch3);
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, const uint8_t*);
+ VMFunctionId id = VMFunctionToId<Fn, jit::HandleDebugTrap>::id;
+ TrampolinePtr code = cx->runtime()->jitRuntime()->getVMWrapper(id);
+
+ masm.push(scratch1);
+ masm.push(scratch2);
+ EmitBaselineCallVM(code, masm);
+
+ EmitBaselineLeaveStubFrame(masm);
+
+ if (kind == DebugTrapHandlerKind::Interpreter) {
+ // We have to reload the bytecode pc register.
+ Address pcAddr(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC());
+ masm.loadPtr(pcAddr, InterpreterPCRegAtDispatch);
+ }
+ masm.abiret();
+
+ Linker linker(masm);
+ JitCode* handlerCode = linker.newCode(cx, CodeKind::Other);
+ if (!handlerCode) {
+ return nullptr;
+ }
+
+ CollectPerfSpewerJitCodeProfile(handlerCode, "DebugTrapHandler");
+
+#ifdef MOZ_VTUNE
+ vtune::MarkStub(handlerCode, "DebugTrapHandler");
+#endif
+
+ return handlerCode;
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/BaselineCodeGen.h b/js/src/jit/BaselineCodeGen.h
new file mode 100644
index 0000000000..42118c430a
--- /dev/null
+++ b/js/src/jit/BaselineCodeGen.h
@@ -0,0 +1,522 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineCodeGen_h
+#define jit_BaselineCodeGen_h
+
+#include "jit/BaselineFrameInfo.h"
+#include "jit/BytecodeAnalysis.h"
+#include "jit/FixedList.h"
+#include "jit/MacroAssembler.h"
+#include "jit/PerfSpewer.h"
+
+namespace js {
+
+namespace jit {
+
+enum class ScriptGCThingType {
+ Atom,
+ String,
+ RegExp,
+ Object,
+ Function,
+ Scope,
+ BigInt
+};
+
+// Base class for BaselineCompiler and BaselineInterpreterGenerator. The Handler
+// template is a class storing fields/methods that are interpreter or compiler
+// specific. This can be combined with template specialization of methods in
+// this class to specialize behavior.
+template <typename Handler>
+class BaselineCodeGen {
+ protected:
+ Handler handler;
+
+ JSContext* cx;
+ StackMacroAssembler masm;
+
+ typename Handler::FrameInfoT& frame;
+
+ // Shared epilogue code to return to the caller.
+ NonAssertingLabel return_;
+
+ NonAssertingLabel postBarrierSlot_;
+
+ // Prologue code where we resume for Ion prologue bailouts.
+ NonAssertingLabel bailoutPrologue_;
+
+ CodeOffset profilerEnterFrameToggleOffset_;
+ CodeOffset profilerExitFrameToggleOffset_;
+
+ // Early Ion bailouts will enter at this address. This is after frame
+ // construction and before environment chain is initialized.
+ CodeOffset bailoutPrologueOffset_;
+
+ // Baseline Interpreter can enter Baseline Compiler code at this address. This
+ // is right after the warm-up counter check in the prologue.
+ CodeOffset warmUpCheckPrologueOffset_;
+
+ uint32_t pushedBeforeCall_ = 0;
+#ifdef DEBUG
+ bool inCall_ = false;
+#endif
+
+ template <typename... HandlerArgs>
+ explicit BaselineCodeGen(JSContext* cx, TempAllocator& alloc,
+ HandlerArgs&&... args);
+
+ template <typename T>
+ void pushArg(const T& t) {
+ masm.Push(t);
+ }
+
+ // Pushes the current script as argument for a VM function.
+ void pushScriptArg();
+
+ // Pushes the bytecode pc as argument for a VM function.
+ void pushBytecodePCArg();
+
+ // Pushes a name/object/scope associated with the current bytecode op (and
+ // stored in the script) as argument for a VM function.
+ void loadScriptGCThing(ScriptGCThingType type, Register dest,
+ Register scratch);
+ void pushScriptGCThingArg(ScriptGCThingType type, Register scratch1,
+ Register scratch2);
+ void pushScriptNameArg(Register scratch1, Register scratch2);
+
+ // Pushes a bytecode operand as argument for a VM function.
+ void pushUint8BytecodeOperandArg(Register scratch);
+ void pushUint16BytecodeOperandArg(Register scratch);
+
+ void loadInt32LengthBytecodeOperand(Register dest);
+ void loadNumFormalArguments(Register dest);
+
+ // Loads the current JSScript* in dest.
+ void loadScript(Register dest);
+
+ void saveInterpreterPCReg();
+ void restoreInterpreterPCReg();
+
+ // Subtracts |script->nslots() * sizeof(Value)| from reg.
+ void subtractScriptSlotsSize(Register reg, Register scratch);
+
+ // Jump to the script's resume entry indicated by resumeIndex.
+ void jumpToResumeEntry(Register resumeIndex, Register scratch1,
+ Register scratch2);
+
+ // Load the global's lexical environment.
+ void loadGlobalLexicalEnvironment(Register dest);
+ void pushGlobalLexicalEnvironmentValue(ValueOperand scratch);
+
+ // Load the |this|-value from the global's lexical environment.
+ void loadGlobalThisValue(ValueOperand dest);
+
+ // Computes the frame size. See BaselineFrame::debugFrameSize_.
+ void computeFrameSize(Register dest);
+
+ void prepareVMCall();
+
+ void storeFrameSizeAndPushDescriptor(uint32_t argSize, Register scratch);
+
+ enum class CallVMPhase { BeforePushingLocals, AfterPushingLocals };
+ bool callVMInternal(VMFunctionId id, RetAddrEntry::Kind kind,
+ CallVMPhase phase);
+
+ template <typename Fn, Fn fn>
+ bool callVM(RetAddrEntry::Kind kind = RetAddrEntry::Kind::CallVM,
+ CallVMPhase phase = CallVMPhase::AfterPushingLocals);
+
+ template <typename Fn, Fn fn>
+ bool callVMNonOp(CallVMPhase phase = CallVMPhase::AfterPushingLocals) {
+ return callVM<Fn, fn>(RetAddrEntry::Kind::NonOpCallVM, phase);
+ }
+
+ // ifDebuggee should be a function emitting code for when the script is a
+ // debuggee script. ifNotDebuggee (if present) is called to emit code for
+ // non-debuggee scripts.
+ template <typename F1, typename F2>
+ [[nodiscard]] bool emitDebugInstrumentation(
+ const F1& ifDebuggee, const mozilla::Maybe<F2>& ifNotDebuggee);
+ template <typename F>
+ [[nodiscard]] bool emitDebugInstrumentation(const F& ifDebuggee) {
+ return emitDebugInstrumentation(ifDebuggee, mozilla::Maybe<F>());
+ }
+
+ bool emitSuspend(JSOp op);
+
+ template <typename F>
+ [[nodiscard]] bool emitAfterYieldDebugInstrumentation(const F& ifDebuggee,
+ Register scratch);
+
+ // ifSet should be a function emitting code for when the script has |flag|
+ // set. ifNotSet emits code for when the flag isn't set.
+ template <typename F1, typename F2>
+ [[nodiscard]] bool emitTestScriptFlag(JSScript::ImmutableFlags flag,
+ const F1& ifSet, const F2& ifNotSet,
+ Register scratch);
+
+ // If |script->hasFlag(flag) == value|, execute the code emitted by |emit|.
+ template <typename F>
+ [[nodiscard]] bool emitTestScriptFlag(JSScript::ImmutableFlags flag,
+ bool value, const F& emit,
+ Register scratch);
+ template <typename F>
+ [[nodiscard]] bool emitTestScriptFlag(JSScript::MutableFlags flag, bool value,
+ const F& emit, Register scratch);
+
+ [[nodiscard]] bool emitEnterGeneratorCode(Register script,
+ Register resumeIndex,
+ Register scratch);
+
+ void emitInterpJumpToResumeEntry(Register script, Register resumeIndex,
+ Register scratch);
+ void emitJumpToInterpretOpLabel();
+
+ [[nodiscard]] bool emitCheckThis(ValueOperand val, bool reinit = false);
+ void emitLoadReturnValue(ValueOperand val);
+ void emitGetAliasedVar(ValueOperand dest);
+ [[nodiscard]] bool emitGetAliasedDebugVar(ValueOperand dest);
+
+ [[nodiscard]] bool emitNextIC();
+ [[nodiscard]] bool emitInterruptCheck();
+ [[nodiscard]] bool emitWarmUpCounterIncrement();
+
+#define EMIT_OP(op, ...) bool emit_##op();
+ FOR_EACH_OPCODE(EMIT_OP)
+#undef EMIT_OP
+
+ // JSOp::Pos, JSOp::Neg, JSOp::BitNot, JSOp::Inc, JSOp::Dec, JSOp::ToNumeric.
+ [[nodiscard]] bool emitUnaryArith();
+
+ // JSOp::BitXor, JSOp::Lsh, JSOp::Add etc.
+ [[nodiscard]] bool emitBinaryArith();
+
+ // Handles JSOp::Lt, JSOp::Gt, and friends
+ [[nodiscard]] bool emitCompare();
+
+ // Handles JSOp::NewObject and JSOp::NewInit.
+ [[nodiscard]] bool emitNewObject();
+
+ // For a JOF_JUMP op, jumps to the op's jump target.
+ void emitJump();
+
+ // For a JOF_JUMP op, jumps to the op's jump target depending on the Value
+ // in |val|.
+ void emitTestBooleanTruthy(bool branchIfTrue, ValueOperand val);
+
+ // Converts |val| to an index in the jump table and stores this in |dest|
+ // or branches to the default pc if not int32 or out-of-range.
+ void emitGetTableSwitchIndex(ValueOperand val, Register dest,
+ Register scratch1, Register scratch2);
+
+ // Jumps to the target of a table switch based on |key| and the
+ // firstResumeIndex stored in JSOp::TableSwitch.
+ void emitTableSwitchJump(Register key, Register scratch1, Register scratch2);
+
+ [[nodiscard]] bool emitReturn();
+
+ [[nodiscard]] bool emitTest(bool branchIfTrue);
+ [[nodiscard]] bool emitAndOr(bool branchIfTrue);
+ [[nodiscard]] bool emitCoalesce();
+
+ [[nodiscard]] bool emitCall(JSOp op);
+ [[nodiscard]] bool emitSpreadCall(JSOp op);
+
+ [[nodiscard]] bool emitDelElem(bool strict);
+ [[nodiscard]] bool emitDelProp(bool strict);
+ [[nodiscard]] bool emitSetElemSuper(bool strict);
+ [[nodiscard]] bool emitSetPropSuper(bool strict);
+
+ // Try to bake in the result of GetGName/BindGName instead of using an IC.
+ // Return true if we managed to optimize the op.
+ bool tryOptimizeGetGlobalName();
+ bool tryOptimizeBindGlobalName();
+
+ [[nodiscard]] bool emitInitPropGetterSetter();
+ [[nodiscard]] bool emitInitElemGetterSetter();
+
+ [[nodiscard]] bool emitFormalArgAccess(JSOp op);
+
+ [[nodiscard]] bool emitUninitializedLexicalCheck(const ValueOperand& val);
+
+ [[nodiscard]] bool emitIsMagicValue();
+
+ void getEnvironmentCoordinateObject(Register reg);
+ Address getEnvironmentCoordinateAddressFromObject(Register objReg,
+ Register reg);
+ Address getEnvironmentCoordinateAddress(Register reg);
+
+ [[nodiscard]] bool emitPrologue();
+ [[nodiscard]] bool emitEpilogue();
+ [[nodiscard]] bool emitOutOfLinePostBarrierSlot();
+ [[nodiscard]] bool emitStackCheck();
+ [[nodiscard]] bool emitDebugPrologue();
+ [[nodiscard]] bool emitDebugEpilogue();
+
+ template <typename F>
+ [[nodiscard]] bool initEnvironmentChainHelper(const F& initFunctionEnv);
+ [[nodiscard]] bool initEnvironmentChain();
+
+ [[nodiscard]] bool emitHandleCodeCoverageAtPrologue();
+
+ void emitInitFrameFields(Register nonFunctionEnv);
+ [[nodiscard]] bool emitIsDebuggeeCheck();
+ void emitInitializeLocals();
+
+ void emitProfilerEnterFrame();
+ void emitProfilerExitFrame();
+};
+
+using RetAddrEntryVector = js::Vector<RetAddrEntry, 16, SystemAllocPolicy>;
+
+// Interface used by BaselineCodeGen for BaselineCompiler.
+class BaselineCompilerHandler {
+ CompilerFrameInfo frame_;
+ TempAllocator& alloc_;
+ BytecodeAnalysis analysis_;
+#ifdef DEBUG
+ const MacroAssembler& masm_;
+#endif
+ FixedList<Label> labels_;
+ RetAddrEntryVector retAddrEntries_;
+
+ // Native code offsets for OSR at JSOp::LoopHead ops.
+ using OSREntryVector =
+ Vector<BaselineScript::OSREntry, 16, SystemAllocPolicy>;
+ OSREntryVector osrEntries_;
+
+ JSScript* script_;
+ jsbytecode* pc_;
+
+ // Index of the current ICEntry in the script's JitScript.
+ uint32_t icEntryIndex_;
+
+ bool compileDebugInstrumentation_;
+ bool ionCompileable_;
+
+ public:
+ using FrameInfoT = CompilerFrameInfo;
+
+ BaselineCompilerHandler(JSContext* cx, MacroAssembler& masm,
+ TempAllocator& alloc, JSScript* script);
+
+ [[nodiscard]] bool init(JSContext* cx);
+
+ CompilerFrameInfo& frame() { return frame_; }
+
+ jsbytecode* pc() const { return pc_; }
+ jsbytecode* maybePC() const { return pc_; }
+
+ void moveToNextPC() { pc_ += GetBytecodeLength(pc_); }
+ Label* labelOf(jsbytecode* pc) { return &labels_[script_->pcToOffset(pc)]; }
+
+ bool isDefinitelyLastOp() const { return pc_ == script_->lastPC(); }
+
+ bool shouldEmitDebugEpilogueAtReturnOp() const {
+ // The JIT uses the return address -> pc mapping and bakes in the pc
+ // argument so the DebugEpilogue call needs to be part of the returning
+ // bytecode op for this to work.
+ return true;
+ }
+
+ JSScript* script() const { return script_; }
+ JSScript* maybeScript() const { return script_; }
+
+ JSFunction* function() const { return script_->function(); }
+ JSFunction* maybeFunction() const { return function(); }
+
+ ModuleObject* module() const { return script_->module(); }
+
+ void setCompileDebugInstrumentation() { compileDebugInstrumentation_ = true; }
+ bool compileDebugInstrumentation() const {
+ return compileDebugInstrumentation_;
+ }
+
+ bool maybeIonCompileable() const { return ionCompileable_; }
+
+ uint32_t icEntryIndex() const { return icEntryIndex_; }
+ void moveToNextICEntry() { icEntryIndex_++; }
+
+ BytecodeAnalysis& analysis() { return analysis_; }
+
+ RetAddrEntryVector& retAddrEntries() { return retAddrEntries_; }
+ OSREntryVector& osrEntries() { return osrEntries_; }
+
+ [[nodiscard]] bool recordCallRetAddr(JSContext* cx, RetAddrEntry::Kind kind,
+ uint32_t retOffset);
+
+ // If a script has more |nslots| than this the stack check must account
+ // for these slots explicitly.
+ bool mustIncludeSlotsInStackCheck() const {
+ static constexpr size_t NumSlotsLimit = 128;
+ return script()->nslots() > NumSlotsLimit;
+ }
+
+ bool canHaveFixedSlots() const { return script()->nfixed() != 0; }
+};
+
+using BaselineCompilerCodeGen = BaselineCodeGen<BaselineCompilerHandler>;
+
+class BaselineCompiler final : private BaselineCompilerCodeGen {
+ // Native code offsets for bytecode ops in the script's resume offsets list.
+ ResumeOffsetEntryVector resumeOffsetEntries_;
+
+ // Native code offsets for debug traps if the script is compiled with debug
+ // instrumentation.
+ using DebugTrapEntryVector =
+ Vector<BaselineScript::DebugTrapEntry, 0, SystemAllocPolicy>;
+ DebugTrapEntryVector debugTrapEntries_;
+
+ CodeOffset profilerPushToggleOffset_;
+
+ BaselinePerfSpewer perfSpewer_;
+
+ public:
+ BaselineCompiler(JSContext* cx, TempAllocator& alloc, JSScript* script);
+ [[nodiscard]] bool init();
+
+ MethodStatus compile();
+
+ bool compileDebugInstrumentation() const {
+ return handler.compileDebugInstrumentation();
+ }
+ void setCompileDebugInstrumentation() {
+ handler.setCompileDebugInstrumentation();
+ }
+
+ private:
+ MethodStatus emitBody();
+
+ [[nodiscard]] bool emitDebugTrap();
+};
+
+// Interface used by BaselineCodeGen for BaselineInterpreterGenerator.
+class BaselineInterpreterHandler {
+ InterpreterFrameInfo frame_;
+
+ // Entry point to start interpreting a bytecode op. No registers are live. PC
+ // is loaded from the frame.
+ NonAssertingLabel interpretOp_;
+
+ // Like interpretOp_ but at this point the PC is expected to be in
+ // InterpreterPCReg.
+ NonAssertingLabel interpretOpWithPCReg_;
+
+ // Offsets of toggled jumps for debugger instrumentation.
+ using CodeOffsetVector = Vector<uint32_t, 0, SystemAllocPolicy>;
+ CodeOffsetVector debugInstrumentationOffsets_;
+
+ // Offsets of toggled jumps for code coverage instrumentation.
+ CodeOffsetVector codeCoverageOffsets_;
+ NonAssertingLabel codeCoverageAtPrologueLabel_;
+ NonAssertingLabel codeCoverageAtPCLabel_;
+
+ // Offsets of IC calls for IsIonInlinableOp ops, for Ion bailouts.
+ BaselineInterpreter::ICReturnOffsetVector icReturnOffsets_;
+
+ // Offsets of some callVMs for BaselineDebugModeOSR.
+ BaselineInterpreter::CallVMOffsets callVMOffsets_;
+
+ // The current JSOp we are emitting interpreter code for.
+ mozilla::Maybe<JSOp> currentOp_;
+
+ public:
+ using FrameInfoT = InterpreterFrameInfo;
+
+ explicit BaselineInterpreterHandler(JSContext* cx, MacroAssembler& masm);
+
+ InterpreterFrameInfo& frame() { return frame_; }
+
+ Label* interpretOpLabel() { return &interpretOp_; }
+ Label* interpretOpWithPCRegLabel() { return &interpretOpWithPCReg_; }
+
+ Label* codeCoverageAtPrologueLabel() { return &codeCoverageAtPrologueLabel_; }
+ Label* codeCoverageAtPCLabel() { return &codeCoverageAtPCLabel_; }
+
+ CodeOffsetVector& debugInstrumentationOffsets() {
+ return debugInstrumentationOffsets_;
+ }
+ CodeOffsetVector& codeCoverageOffsets() { return codeCoverageOffsets_; }
+
+ BaselineInterpreter::ICReturnOffsetVector& icReturnOffsets() {
+ return icReturnOffsets_;
+ }
+
+ void setCurrentOp(JSOp op) { currentOp_.emplace(op); }
+ void resetCurrentOp() { currentOp_.reset(); }
+ mozilla::Maybe<JSOp> currentOp() const { return currentOp_; }
+
+ // Interpreter doesn't know the script and pc statically.
+ jsbytecode* maybePC() const { return nullptr; }
+ bool isDefinitelyLastOp() const { return false; }
+ JSScript* maybeScript() const { return nullptr; }
+ JSFunction* maybeFunction() const { return nullptr; }
+
+ bool shouldEmitDebugEpilogueAtReturnOp() const {
+ // The interpreter doesn't use the return address -> pc mapping and doesn't
+ // bake in bytecode PCs so it can emit a shared DebugEpilogue call instead
+ // of duplicating it for every return op.
+ return false;
+ }
+
+ [[nodiscard]] bool addDebugInstrumentationOffset(JSContext* cx,
+ CodeOffset offset);
+
+ const BaselineInterpreter::CallVMOffsets& callVMOffsets() const {
+ return callVMOffsets_;
+ }
+
+ [[nodiscard]] bool recordCallRetAddr(JSContext* cx, RetAddrEntry::Kind kind,
+ uint32_t retOffset);
+
+ bool maybeIonCompileable() const { return true; }
+
+ // The interpreter doesn't know the number of slots statically so we always
+ // include them.
+ bool mustIncludeSlotsInStackCheck() const { return true; }
+
+ bool canHaveFixedSlots() const { return true; }
+};
+
+using BaselineInterpreterCodeGen = BaselineCodeGen<BaselineInterpreterHandler>;
+
+class BaselineInterpreterGenerator final : private BaselineInterpreterCodeGen {
+ // Offsets of patchable call instructions for debugger breakpoints/stepping.
+ Vector<uint32_t, 0, SystemAllocPolicy> debugTrapOffsets_;
+
+ // Offsets of move instructions for tableswitch base address.
+ Vector<CodeOffset, 0, SystemAllocPolicy> tableLabels_;
+
+ // Offset of the first tableswitch entry.
+ uint32_t tableOffset_ = 0;
+
+ // Offset of the code to start interpreting a bytecode op.
+ uint32_t interpretOpOffset_ = 0;
+
+ // Like interpretOpOffset_ but skips the debug trap for the current op.
+ uint32_t interpretOpNoDebugTrapOffset_ = 0;
+
+ // Offset of the jump (tail call) to the debug trap handler trampoline code.
+ // When the debugger is enabled, NOPs are patched to calls to this location.
+ uint32_t debugTrapHandlerOffset_ = 0;
+
+ public:
+ explicit BaselineInterpreterGenerator(JSContext* cx, TempAllocator& alloc);
+
+ [[nodiscard]] bool generate(BaselineInterpreter& interpreter);
+
+ private:
+ [[nodiscard]] bool emitInterpreterLoop();
+ [[nodiscard]] bool emitDebugTrap();
+
+ void emitOutOfLineCodeCoverageInstrumentation();
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BaselineCodeGen_h */
diff --git a/js/src/jit/BaselineDebugModeOSR.cpp b/js/src/jit/BaselineDebugModeOSR.cpp
new file mode 100644
index 0000000000..bbdcccfebc
--- /dev/null
+++ b/js/src/jit/BaselineDebugModeOSR.cpp
@@ -0,0 +1,561 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BaselineDebugModeOSR.h"
+
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineIC.h"
+#include "jit/BaselineJIT.h"
+#include "jit/Invalidation.h"
+#include "jit/IonScript.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JSJitFrameIter.h"
+
+#include "jit/JitScript-inl.h"
+#include "jit/JSJitFrameIter-inl.h"
+#include "vm/JSScript-inl.h"
+#include "vm/Realm-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+struct DebugModeOSREntry {
+ JSScript* script;
+ BaselineScript* oldBaselineScript;
+ uint32_t pcOffset;
+ RetAddrEntry::Kind frameKind;
+
+ explicit DebugModeOSREntry(JSScript* script)
+ : script(script),
+ oldBaselineScript(script->baselineScript()),
+ pcOffset(uint32_t(-1)),
+ frameKind(RetAddrEntry::Kind::Invalid) {}
+
+ DebugModeOSREntry(JSScript* script, const RetAddrEntry& retAddrEntry)
+ : script(script),
+ oldBaselineScript(script->baselineScript()),
+ pcOffset(retAddrEntry.pcOffset()),
+ frameKind(retAddrEntry.kind()) {
+#ifdef DEBUG
+ MOZ_ASSERT(pcOffset == retAddrEntry.pcOffset());
+ MOZ_ASSERT(frameKind == retAddrEntry.kind());
+#endif
+ }
+
+ DebugModeOSREntry(DebugModeOSREntry&& other)
+ : script(other.script),
+ oldBaselineScript(other.oldBaselineScript),
+ pcOffset(other.pcOffset),
+ frameKind(other.frameKind) {}
+
+ bool recompiled() const {
+ return oldBaselineScript != script->baselineScript();
+ }
+};
+
+using DebugModeOSREntryVector = Vector<DebugModeOSREntry>;
+
+class UniqueScriptOSREntryIter {
+ const DebugModeOSREntryVector& entries_;
+ size_t index_;
+
+ public:
+ explicit UniqueScriptOSREntryIter(const DebugModeOSREntryVector& entries)
+ : entries_(entries), index_(0) {}
+
+ bool done() { return index_ == entries_.length(); }
+
+ const DebugModeOSREntry& entry() {
+ MOZ_ASSERT(!done());
+ return entries_[index_];
+ }
+
+ UniqueScriptOSREntryIter& operator++() {
+ MOZ_ASSERT(!done());
+ while (++index_ < entries_.length()) {
+ bool unique = true;
+ for (size_t i = 0; i < index_; i++) {
+ if (entries_[i].script == entries_[index_].script) {
+ unique = false;
+ break;
+ }
+ }
+ if (unique) {
+ break;
+ }
+ }
+ return *this;
+ }
+};
+
+static bool CollectJitStackScripts(JSContext* cx,
+ const DebugAPI::ExecutionObservableSet& obs,
+ const ActivationIterator& activation,
+ DebugModeOSREntryVector& entries) {
+ for (OnlyJSJitFrameIter iter(activation); !iter.done(); ++iter) {
+ const JSJitFrameIter& frame = iter.frame();
+ switch (frame.type()) {
+ case FrameType::BaselineJS: {
+ JSScript* script = frame.script();
+
+ if (!obs.shouldRecompileOrInvalidate(script)) {
+ break;
+ }
+
+ BaselineFrame* baselineFrame = frame.baselineFrame();
+
+ if (baselineFrame->runningInInterpreter()) {
+ // Baseline Interpreter frames for scripts that have a BaselineScript
+ // or IonScript don't need to be patched but they do need to be
+ // invalidated and recompiled. See also CollectInterpreterStackScripts
+ // for C++ interpreter frames.
+ if (!entries.append(DebugModeOSREntry(script))) {
+ return false;
+ }
+ } else {
+ // The frame must be settled on a pc with a RetAddrEntry.
+ uint8_t* retAddr = frame.resumePCinCurrentFrame();
+ const RetAddrEntry& retAddrEntry =
+ script->baselineScript()->retAddrEntryFromReturnAddress(retAddr);
+ if (!entries.append(DebugModeOSREntry(script, retAddrEntry))) {
+ return false;
+ }
+ }
+
+ break;
+ }
+
+ case FrameType::BaselineStub:
+ break;
+
+ case FrameType::IonJS: {
+ InlineFrameIterator inlineIter(cx, &frame);
+ while (true) {
+ if (obs.shouldRecompileOrInvalidate(inlineIter.script())) {
+ if (!entries.append(DebugModeOSREntry(inlineIter.script()))) {
+ return false;
+ }
+ }
+ if (!inlineIter.more()) {
+ break;
+ }
+ ++inlineIter;
+ }
+ break;
+ }
+
+ default:;
+ }
+ }
+
+ return true;
+}
+
+static bool CollectInterpreterStackScripts(
+ JSContext* cx, const DebugAPI::ExecutionObservableSet& obs,
+ const ActivationIterator& activation, DebugModeOSREntryVector& entries) {
+ // Collect interpreter frame stacks with IonScript or BaselineScript as
+ // well. These do not need to be patched, but do need to be invalidated
+ // and recompiled.
+ InterpreterActivation* act = activation.activation()->asInterpreter();
+ for (InterpreterFrameIterator iter(act); !iter.done(); ++iter) {
+ JSScript* script = iter.frame()->script();
+ if (obs.shouldRecompileOrInvalidate(script)) {
+ if (!entries.append(DebugModeOSREntry(iter.frame()->script()))) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+#ifdef JS_JITSPEW
+static const char* RetAddrEntryKindToString(RetAddrEntry::Kind kind) {
+ switch (kind) {
+ case RetAddrEntry::Kind::IC:
+ return "IC";
+ case RetAddrEntry::Kind::CallVM:
+ return "callVM";
+ case RetAddrEntry::Kind::StackCheck:
+ return "stack check";
+ case RetAddrEntry::Kind::InterruptCheck:
+ return "interrupt check";
+ case RetAddrEntry::Kind::DebugTrap:
+ return "debug trap";
+ case RetAddrEntry::Kind::DebugPrologue:
+ return "debug prologue";
+ case RetAddrEntry::Kind::DebugAfterYield:
+ return "debug after yield";
+ case RetAddrEntry::Kind::DebugEpilogue:
+ return "debug epilogue";
+ default:
+ MOZ_CRASH("bad RetAddrEntry kind");
+ }
+}
+#endif // JS_JITSPEW
+
+static void SpewPatchBaselineFrame(const uint8_t* oldReturnAddress,
+ const uint8_t* newReturnAddress,
+ JSScript* script,
+ RetAddrEntry::Kind frameKind,
+ const jsbytecode* pc) {
+ JitSpew(JitSpew_BaselineDebugModeOSR,
+ "Patch return %p -> %p on BaselineJS frame (%s:%u:%u) from %s at %s",
+ oldReturnAddress, newReturnAddress, script->filename(),
+ script->lineno(), script->column(),
+ RetAddrEntryKindToString(frameKind), CodeName(JSOp(*pc)));
+}
+
+static void PatchBaselineFramesForDebugMode(
+ JSContext* cx, const DebugAPI::ExecutionObservableSet& obs,
+ const ActivationIterator& activation, DebugModeOSREntryVector& entries,
+ size_t* start) {
+ //
+ // Recompile Patching Overview
+ //
+ // When toggling debug mode with live baseline scripts on the stack, we
+ // could have entered the VM via the following ways from the baseline
+ // script.
+ //
+ // Off to On:
+ // A. From a non-prologue IC (fallback stub or "can call" stub).
+ // B. From a VM call.
+ // C. From inside the interrupt handler via the prologue stack check.
+ //
+ // On to Off:
+ // - All the ways above.
+ // D. From the debug trap handler.
+ // E. From the debug prologue.
+ // F. From the debug epilogue.
+ // G. From a JSOp::AfterYield instruction.
+ //
+ // In general, we patch the return address from VM calls and ICs to the
+ // corresponding entry in the recompiled BaselineScript. For entries that are
+ // not present in the recompiled script (cases D to G above) we switch the
+ // frame to interpreter mode and resume in the Baseline Interpreter.
+ //
+ // Specifics on what needs to be done are documented below.
+ //
+
+ const BaselineInterpreter& baselineInterp =
+ cx->runtime()->jitRuntime()->baselineInterpreter();
+
+ CommonFrameLayout* prev = nullptr;
+ size_t entryIndex = *start;
+
+ for (OnlyJSJitFrameIter iter(activation); !iter.done(); ++iter) {
+ const JSJitFrameIter& frame = iter.frame();
+ switch (frame.type()) {
+ case FrameType::BaselineJS: {
+ // If the script wasn't recompiled or is not observed, there's
+ // nothing to patch.
+ if (!obs.shouldRecompileOrInvalidate(frame.script())) {
+ break;
+ }
+
+ DebugModeOSREntry& entry = entries[entryIndex];
+
+ if (!entry.recompiled()) {
+ entryIndex++;
+ break;
+ }
+
+ BaselineFrame* baselineFrame = frame.baselineFrame();
+ if (baselineFrame->runningInInterpreter()) {
+ // We recompiled the script's BaselineScript but Baseline Interpreter
+ // frames don't need to be patched.
+ entryIndex++;
+ break;
+ }
+
+ JSScript* script = entry.script;
+ uint32_t pcOffset = entry.pcOffset;
+ jsbytecode* pc = script->offsetToPC(pcOffset);
+
+ MOZ_ASSERT(script == frame.script());
+ MOZ_ASSERT(pcOffset < script->length());
+
+ BaselineScript* bl = script->baselineScript();
+ RetAddrEntry::Kind kind = entry.frameKind;
+ uint8_t* retAddr = nullptr;
+ switch (kind) {
+ case RetAddrEntry::Kind::IC:
+ case RetAddrEntry::Kind::CallVM:
+ case RetAddrEntry::Kind::InterruptCheck:
+ case RetAddrEntry::Kind::StackCheck: {
+ // Cases A, B, C above.
+ //
+ // For the baseline frame here, we resume right after the CallVM or
+ // IC returns.
+ //
+ // For CallVM (case B) the assumption is that all callVMs which can
+ // trigger debug mode OSR are the *only* callVMs generated for their
+ // respective pc locations in the Baseline JIT code.
+ const RetAddrEntry* retAddrEntry = nullptr;
+ switch (kind) {
+ case RetAddrEntry::Kind::IC:
+ case RetAddrEntry::Kind::CallVM:
+ case RetAddrEntry::Kind::InterruptCheck:
+ retAddrEntry = &bl->retAddrEntryFromPCOffset(pcOffset, kind);
+ break;
+ case RetAddrEntry::Kind::StackCheck:
+ retAddrEntry = &bl->prologueRetAddrEntry(kind);
+ break;
+ default:
+ MOZ_CRASH("Unexpected kind");
+ }
+ retAddr = bl->returnAddressForEntry(*retAddrEntry);
+ SpewPatchBaselineFrame(prev->returnAddress(), retAddr, script, kind,
+ pc);
+ break;
+ }
+ case RetAddrEntry::Kind::DebugPrologue:
+ case RetAddrEntry::Kind::DebugEpilogue:
+ case RetAddrEntry::Kind::DebugTrap:
+ case RetAddrEntry::Kind::DebugAfterYield: {
+ // Cases D, E, F, G above.
+ //
+ // Resume in the Baseline Interpreter because these callVMs are not
+ // present in the new BaselineScript if we recompiled without debug
+ // instrumentation.
+ if (kind == RetAddrEntry::Kind::DebugPrologue) {
+ frame.baselineFrame()->switchFromJitToInterpreterAtPrologue(cx);
+ } else {
+ frame.baselineFrame()->switchFromJitToInterpreter(cx, pc);
+ }
+ switch (kind) {
+ case RetAddrEntry::Kind::DebugTrap:
+ // DebugTrap handling is different from the ones below because
+ // it's not a callVM but a trampoline call at the start of the
+ // bytecode op. When we return to the frame we can resume at the
+ // interpretOp label.
+ retAddr = baselineInterp.interpretOpAddr().value;
+ break;
+ case RetAddrEntry::Kind::DebugPrologue:
+ retAddr = baselineInterp.retAddrForDebugPrologueCallVM();
+ break;
+ case RetAddrEntry::Kind::DebugEpilogue:
+ retAddr = baselineInterp.retAddrForDebugEpilogueCallVM();
+ break;
+ case RetAddrEntry::Kind::DebugAfterYield:
+ retAddr = baselineInterp.retAddrForDebugAfterYieldCallVM();
+ break;
+ default:
+ MOZ_CRASH("Unexpected kind");
+ }
+ SpewPatchBaselineFrame(prev->returnAddress(), retAddr, script, kind,
+ pc);
+ break;
+ }
+ case RetAddrEntry::Kind::NonOpCallVM:
+ case RetAddrEntry::Kind::Invalid:
+ // These cannot trigger BaselineDebugModeOSR.
+ MOZ_CRASH("Unexpected RetAddrEntry Kind");
+ }
+
+ prev->setReturnAddress(retAddr);
+ entryIndex++;
+ break;
+ }
+
+ case FrameType::IonJS: {
+ // Nothing to patch.
+ InlineFrameIterator inlineIter(cx, &frame);
+ while (true) {
+ if (obs.shouldRecompileOrInvalidate(inlineIter.script())) {
+ entryIndex++;
+ }
+ if (!inlineIter.more()) {
+ break;
+ }
+ ++inlineIter;
+ }
+ break;
+ }
+
+ default:;
+ }
+
+ prev = frame.current();
+ }
+
+ *start = entryIndex;
+}
+
+static void SkipInterpreterFrameEntries(
+ const DebugAPI::ExecutionObservableSet& obs,
+ const ActivationIterator& activation, size_t* start) {
+ size_t entryIndex = *start;
+
+ // Skip interpreter frames, which do not need patching.
+ InterpreterActivation* act = activation.activation()->asInterpreter();
+ for (InterpreterFrameIterator iter(act); !iter.done(); ++iter) {
+ if (obs.shouldRecompileOrInvalidate(iter.frame()->script())) {
+ entryIndex++;
+ }
+ }
+
+ *start = entryIndex;
+}
+
+static bool RecompileBaselineScriptForDebugMode(
+ JSContext* cx, JSScript* script, DebugAPI::IsObserving observing) {
+ // If a script is on the stack multiple times, it may have already
+ // been recompiled.
+ if (script->baselineScript()->hasDebugInstrumentation() == observing) {
+ return true;
+ }
+
+ JitSpew(JitSpew_BaselineDebugModeOSR, "Recompiling (%s:%u:%u) for %s",
+ script->filename(), script->lineno(), script->column(),
+ observing ? "DEBUGGING" : "NORMAL EXECUTION");
+
+ AutoKeepJitScripts keepJitScripts(cx);
+ BaselineScript* oldBaselineScript =
+ script->jitScript()->clearBaselineScript(cx->gcContext(), script);
+
+ MethodStatus status =
+ BaselineCompile(cx, script, /* forceDebugMode = */ observing);
+ if (status != Method_Compiled) {
+ // We will only fail to recompile for debug mode due to OOM. Restore
+ // the old baseline script in case something doesn't properly
+ // propagate OOM.
+ MOZ_ASSERT(status == Method_Error);
+ script->jitScript()->setBaselineScript(script, oldBaselineScript);
+ return false;
+ }
+
+ // Don't destroy the old baseline script yet, since if we fail any of the
+ // recompiles we need to rollback all the old baseline scripts.
+ MOZ_ASSERT(script->baselineScript()->hasDebugInstrumentation() == observing);
+ return true;
+}
+
+static bool InvalidateScriptsInZone(JSContext* cx, Zone* zone,
+ const Vector<DebugModeOSREntry>& entries) {
+ RecompileInfoVector invalid;
+ for (UniqueScriptOSREntryIter iter(entries); !iter.done(); ++iter) {
+ JSScript* script = iter.entry().script;
+ if (script->zone() != zone) {
+ continue;
+ }
+
+ if (script->hasIonScript()) {
+ if (!invalid.emplaceBack(script, script->ionScript()->compilationId())) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+ }
+
+ // Cancel off-thread Ion compile for anything that has a
+ // BaselineScript. If we relied on the call to Invalidate below to
+ // cancel off-thread Ion compiles, only those with existing IonScripts
+ // would be cancelled.
+ if (script->hasBaselineScript()) {
+ CancelOffThreadIonCompile(script);
+ }
+ }
+
+ // No need to cancel off-thread Ion compiles again, we already did it
+ // above.
+ Invalidate(cx, invalid,
+ /* resetUses = */ true, /* cancelOffThread = */ false);
+ return true;
+}
+
+static void UndoRecompileBaselineScriptsForDebugMode(
+ JSContext* cx, const DebugModeOSREntryVector& entries) {
+ // In case of failure, roll back the entire set of active scripts so that
+ // we don't have to patch return addresses on the stack.
+ for (UniqueScriptOSREntryIter iter(entries); !iter.done(); ++iter) {
+ const DebugModeOSREntry& entry = iter.entry();
+ JSScript* script = entry.script;
+ BaselineScript* baselineScript = script->baselineScript();
+ if (entry.recompiled()) {
+ script->jitScript()->setBaselineScript(script, entry.oldBaselineScript);
+ BaselineScript::Destroy(cx->gcContext(), baselineScript);
+ }
+ }
+}
+
+bool jit::RecompileOnStackBaselineScriptsForDebugMode(
+ JSContext* cx, const DebugAPI::ExecutionObservableSet& obs,
+ DebugAPI::IsObserving observing) {
+ // First recompile the active scripts on the stack and patch the live
+ // frames.
+ Vector<DebugModeOSREntry> entries(cx);
+
+ for (ActivationIterator iter(cx); !iter.done(); ++iter) {
+ if (iter->isJit()) {
+ if (!CollectJitStackScripts(cx, obs, iter, entries)) {
+ return false;
+ }
+ } else if (iter->isInterpreter()) {
+ if (!CollectInterpreterStackScripts(cx, obs, iter, entries)) {
+ return false;
+ }
+ }
+ }
+
+ if (entries.empty()) {
+ return true;
+ }
+
+ // When the profiler is enabled, we need to have suppressed sampling,
+ // since the basline jit scripts are in a state of flux.
+ MOZ_ASSERT(!cx->isProfilerSamplingEnabled());
+
+ // Invalidate all scripts we are recompiling.
+ if (Zone* zone = obs.singleZone()) {
+ if (!InvalidateScriptsInZone(cx, zone, entries)) {
+ return false;
+ }
+ } else {
+ using ZoneRange = DebugAPI::ExecutionObservableSet::ZoneRange;
+ for (ZoneRange r = obs.zones()->all(); !r.empty(); r.popFront()) {
+ if (!InvalidateScriptsInZone(cx, r.front(), entries)) {
+ return false;
+ }
+ }
+ }
+
+ // Try to recompile all the scripts. If we encounter an error, we need to
+ // roll back as if none of the compilations happened, so that we don't
+ // crash.
+ for (size_t i = 0; i < entries.length(); i++) {
+ JSScript* script = entries[i].script;
+ AutoRealm ar(cx, script);
+ if (!RecompileBaselineScriptForDebugMode(cx, script, observing)) {
+ UndoRecompileBaselineScriptsForDebugMode(cx, entries);
+ return false;
+ }
+ }
+
+ // If all recompiles succeeded, destroy the old baseline scripts and patch
+ // the live frames.
+ //
+ // After this point the function must be infallible.
+
+ for (UniqueScriptOSREntryIter iter(entries); !iter.done(); ++iter) {
+ const DebugModeOSREntry& entry = iter.entry();
+ if (entry.recompiled()) {
+ BaselineScript::Destroy(cx->gcContext(), entry.oldBaselineScript);
+ }
+ }
+
+ size_t processed = 0;
+ for (ActivationIterator iter(cx); !iter.done(); ++iter) {
+ if (iter->isJit()) {
+ PatchBaselineFramesForDebugMode(cx, obs, iter, entries, &processed);
+ } else if (iter->isInterpreter()) {
+ SkipInterpreterFrameEntries(obs, iter, &processed);
+ }
+ }
+ MOZ_ASSERT(processed == entries.length());
+
+ return true;
+}
diff --git a/js/src/jit/BaselineDebugModeOSR.h b/js/src/jit/BaselineDebugModeOSR.h
new file mode 100644
index 0000000000..dc7df161e6
--- /dev/null
+++ b/js/src/jit/BaselineDebugModeOSR.h
@@ -0,0 +1,30 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineDebugModeOSR_h
+#define jit_BaselineDebugModeOSR_h
+
+#include "jstypes.h"
+
+#include "debugger/DebugAPI.h"
+
+struct JS_PUBLIC_API JSContext;
+
+namespace js {
+namespace jit {
+
+// Note that this file and the corresponding .cpp implement debug mode
+// on-stack recompilation. This is to be distinguished from ordinary
+// Baseline->Ion OSR, which is used to jump into compiled loops.
+
+[[nodiscard]] bool RecompileOnStackBaselineScriptsForDebugMode(
+ JSContext* cx, const DebugAPI::ExecutionObservableSet& obs,
+ DebugAPI::IsObserving observing);
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_BaselineDebugModeOSR_h
diff --git a/js/src/jit/BaselineFrame-inl.h b/js/src/jit/BaselineFrame-inl.h
new file mode 100644
index 0000000000..73eb9026ac
--- /dev/null
+++ b/js/src/jit/BaselineFrame-inl.h
@@ -0,0 +1,131 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineFrame_inl_h
+#define jit_BaselineFrame_inl_h
+
+#include "jit/BaselineFrame.h"
+
+#include "jit/TrialInlining.h"
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+
+#include "vm/EnvironmentObject-inl.h"
+#include "vm/JSScript-inl.h"
+#include "vm/NativeObject-inl.h" // js::NativeObject::initDenseElementsFromRange
+
+namespace js {
+namespace jit {
+
+template <typename SpecificEnvironment>
+inline void BaselineFrame::pushOnEnvironmentChain(SpecificEnvironment& env) {
+ MOZ_ASSERT(*environmentChain() == env.enclosingEnvironment());
+ envChain_ = &env;
+ if (IsFrameInitialEnvironment(this, env)) {
+ flags_ |= HAS_INITIAL_ENV;
+ }
+}
+
+template <typename SpecificEnvironment>
+inline void BaselineFrame::popOffEnvironmentChain() {
+ MOZ_ASSERT(envChain_->is<SpecificEnvironment>());
+ envChain_ = &envChain_->as<SpecificEnvironment>().enclosingEnvironment();
+}
+
+inline void BaselineFrame::replaceInnermostEnvironment(EnvironmentObject& env) {
+ MOZ_ASSERT(env.enclosingEnvironment() ==
+ envChain_->as<EnvironmentObject>().enclosingEnvironment());
+ envChain_ = &env;
+}
+
+inline bool BaselineFrame::saveGeneratorSlots(JSContext* cx, unsigned nslots,
+ ArrayObject* dest) const {
+ // By convention, generator slots are stored in interpreter order,
+ // which is the reverse of BaselineFrame order.
+
+ MOZ_ASSERT(nslots == numValueSlots(debugFrameSize()) - 1);
+ const Value* end = reinterpret_cast<const Value*>(this);
+ mozilla::Span<const Value> span{end - nslots, end};
+ return dest->initDenseElementsFromRange(cx, span.rbegin(), span.rend());
+}
+
+inline bool BaselineFrame::pushLexicalEnvironment(JSContext* cx,
+ Handle<LexicalScope*> scope) {
+ BlockLexicalEnvironmentObject* env =
+ BlockLexicalEnvironmentObject::createForFrame(cx, scope, this);
+ if (!env) {
+ return false;
+ }
+ pushOnEnvironmentChain(*env);
+
+ return true;
+}
+
+inline bool BaselineFrame::pushClassBodyEnvironment(
+ JSContext* cx, Handle<ClassBodyScope*> scope) {
+ ClassBodyLexicalEnvironmentObject* env =
+ ClassBodyLexicalEnvironmentObject::createForFrame(cx, scope, this);
+ if (!env) {
+ return false;
+ }
+ pushOnEnvironmentChain(*env);
+
+ return true;
+}
+
+inline bool BaselineFrame::freshenLexicalEnvironment(JSContext* cx) {
+ Rooted<BlockLexicalEnvironmentObject*> current(
+ cx, &envChain_->as<BlockLexicalEnvironmentObject>());
+ BlockLexicalEnvironmentObject* clone =
+ BlockLexicalEnvironmentObject::clone(cx, current);
+ if (!clone) {
+ return false;
+ }
+
+ replaceInnermostEnvironment(*clone);
+ return true;
+}
+
+inline bool BaselineFrame::recreateLexicalEnvironment(JSContext* cx) {
+ Rooted<BlockLexicalEnvironmentObject*> current(
+ cx, &envChain_->as<BlockLexicalEnvironmentObject>());
+ BlockLexicalEnvironmentObject* clone =
+ BlockLexicalEnvironmentObject::recreate(cx, current);
+ if (!clone) {
+ return false;
+ }
+
+ replaceInnermostEnvironment(*clone);
+ return true;
+}
+
+inline CallObject& BaselineFrame::callObj() const {
+ MOZ_ASSERT(hasInitialEnvironment());
+ MOZ_ASSERT(callee()->needsCallObject());
+
+ JSObject* obj = environmentChain();
+ while (!obj->is<CallObject>()) {
+ obj = obj->enclosingEnvironment();
+ }
+ return obj->as<CallObject>();
+}
+
+inline JSScript* BaselineFrame::outerScript() const {
+ if (!icScript()->isInlined()) {
+ return script();
+ }
+ return icScript()->inliningRoot()->owningScript();
+}
+
+inline void BaselineFrame::unsetIsDebuggee() {
+ MOZ_ASSERT(!script()->isDebuggee());
+ flags_ &= ~DEBUGGEE;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BaselineFrame_inl_h */
diff --git a/js/src/jit/BaselineFrame.cpp b/js/src/jit/BaselineFrame.cpp
new file mode 100644
index 0000000000..49c70c3735
--- /dev/null
+++ b/js/src/jit/BaselineFrame.cpp
@@ -0,0 +1,180 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BaselineFrame-inl.h"
+
+#include <algorithm>
+
+#include "debugger/DebugAPI.h"
+#include "vm/EnvironmentObject.h"
+#include "vm/JSContext.h"
+
+#include "jit/JSJitFrameIter-inl.h"
+#include "vm/Stack-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+static void TraceLocals(BaselineFrame* frame, JSTracer* trc, unsigned start,
+ unsigned end) {
+ if (start < end) {
+ // Stack grows down.
+ Value* last = frame->valueSlot(end - 1);
+ TraceRootRange(trc, end - start, last, "baseline-stack");
+ }
+}
+
+void BaselineFrame::trace(JSTracer* trc, const JSJitFrameIter& frameIterator) {
+ replaceCalleeToken(TraceCalleeToken(trc, calleeToken()));
+
+ // Trace |this|, actual and formal args.
+ if (isFunctionFrame()) {
+ TraceRoot(trc, &thisArgument(), "baseline-this");
+
+ unsigned numArgs = std::max(numActualArgs(), numFormalArgs());
+ TraceRootRange(trc, numArgs + isConstructing(), argv(), "baseline-args");
+ }
+
+ // Trace environment chain, if it exists.
+ if (envChain_) {
+ TraceRoot(trc, &envChain_, "baseline-envchain");
+ }
+
+ // Trace return value.
+ if (hasReturnValue()) {
+ TraceRoot(trc, returnValue().address(), "baseline-rval");
+ }
+
+ if (hasArgsObj()) {
+ TraceRoot(trc, &argsObj_, "baseline-args-obj");
+ }
+
+ if (runningInInterpreter()) {
+ TraceRoot(trc, &interpreterScript_, "baseline-interpreterScript");
+ }
+
+ // Trace locals and stack values.
+ JSScript* script = this->script();
+ size_t nfixed = script->nfixed();
+ jsbytecode* pc;
+ frameIterator.baselineScriptAndPc(nullptr, &pc);
+ size_t nlivefixed = script->calculateLiveFixed(pc);
+
+ uint32_t numValueSlots = frameIterator.baselineFrameNumValueSlots();
+
+ // NB: It is possible that numValueSlots could be zero, even if nfixed is
+ // nonzero. This is the case when we're initializing the environment chain or
+ // failed the prologue stack check.
+ if (numValueSlots > 0) {
+ MOZ_ASSERT(nfixed <= numValueSlots);
+
+ if (nfixed == nlivefixed) {
+ // All locals are live.
+ TraceLocals(this, trc, 0, numValueSlots);
+ } else {
+ // Trace operand stack.
+ TraceLocals(this, trc, nfixed, numValueSlots);
+
+ // Clear dead block-scoped locals.
+ while (nfixed > nlivefixed) {
+ unaliasedLocal(--nfixed).setUndefined();
+ }
+
+ // Trace live locals.
+ TraceLocals(this, trc, 0, nlivefixed);
+ }
+ }
+
+ if (auto* debugEnvs = script->realm()->debugEnvs()) {
+ debugEnvs->traceLiveFrame(trc, this);
+ }
+}
+
+bool BaselineFrame::uninlineIsProfilerSamplingEnabled(JSContext* cx) {
+ return cx->isProfilerSamplingEnabled();
+}
+
+bool BaselineFrame::initFunctionEnvironmentObjects(JSContext* cx) {
+ return js::InitFunctionEnvironmentObjects(cx, this);
+}
+
+bool BaselineFrame::pushVarEnvironment(JSContext* cx, Handle<Scope*> scope) {
+ return js::PushVarEnvironmentObject(cx, scope, this);
+}
+
+void BaselineFrame::setInterpreterFields(JSScript* script, jsbytecode* pc) {
+ uint32_t pcOffset = script->pcToOffset(pc);
+ interpreterScript_ = script;
+ interpreterPC_ = pc;
+ MOZ_ASSERT(icScript_);
+ interpreterICEntry_ = icScript_->interpreterICEntryFromPCOffset(pcOffset);
+}
+
+void BaselineFrame::setInterpreterFieldsForPrologue(JSScript* script) {
+ interpreterScript_ = script;
+ interpreterPC_ = script->code();
+ if (icScript_->numICEntries() > 0) {
+ interpreterICEntry_ = &icScript_->icEntry(0);
+ } else {
+ // If the script does not have any ICEntries (possible for non-function
+ // scripts) the interpreterICEntry_ field won't be used. Just set it to
+ // nullptr.
+ interpreterICEntry_ = nullptr;
+ }
+}
+
+bool BaselineFrame::initForOsr(InterpreterFrame* fp, uint32_t numStackValues) {
+ mozilla::PodZero(this);
+
+ envChain_ = fp->environmentChain();
+
+ if (fp->hasInitialEnvironmentUnchecked()) {
+ flags_ |= BaselineFrame::HAS_INITIAL_ENV;
+ }
+
+ if (fp->script()->needsArgsObj() && fp->hasArgsObj()) {
+ flags_ |= BaselineFrame::HAS_ARGS_OBJ;
+ argsObj_ = &fp->argsObj();
+ }
+
+ if (fp->hasReturnValue()) {
+ setReturnValue(fp->returnValue());
+ }
+
+ icScript_ = fp->script()->jitScript()->icScript();
+
+ JSContext* cx =
+ fp->script()->runtimeFromMainThread()->mainContextFromOwnThread();
+
+ Activation* interpActivation = cx->activation()->prev();
+ jsbytecode* pc = interpActivation->asInterpreter()->regs().pc;
+ MOZ_ASSERT(fp->script()->containsPC(pc));
+
+ // We are doing OSR into the Baseline Interpreter. We can get the pc from the
+ // C++ interpreter's activation, we just have to skip the JitActivation.
+ flags_ |= BaselineFrame::RUNNING_IN_INTERPRETER;
+ setInterpreterFields(pc);
+
+#ifdef DEBUG
+ debugFrameSize_ = frameSizeForNumValueSlots(numStackValues);
+ MOZ_ASSERT(debugNumValueSlots() == numStackValues);
+#endif
+
+ for (uint32_t i = 0; i < numStackValues; i++) {
+ *valueSlot(i) = fp->slots()[i];
+ }
+
+ if (fp->isDebuggee()) {
+ // For debuggee frames, update any Debugger.Frame objects for the
+ // InterpreterFrame to point to the BaselineFrame.
+ if (!DebugAPI::handleBaselineOsr(cx, fp, this)) {
+ return false;
+ }
+ setIsDebuggee();
+ }
+
+ return true;
+}
diff --git a/js/src/jit/BaselineFrame.h b/js/src/jit/BaselineFrame.h
new file mode 100644
index 0000000000..6614d23e15
--- /dev/null
+++ b/js/src/jit/BaselineFrame.h
@@ -0,0 +1,373 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineFrame_h
+#define jit_BaselineFrame_h
+
+#include <algorithm>
+
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/ScriptFromCalleeToken.h"
+#include "vm/Stack.h"
+
+namespace js {
+namespace jit {
+
+class ICEntry;
+class ICScript;
+class JSJitFrameIter;
+
+// The stack looks like this, fp is the frame pointer:
+//
+// fp+y arguments
+// fp => JitFrameLayout (frame header)
+// fp-x BaselineFrame
+// locals
+// stack values
+
+class BaselineFrame {
+ public:
+ enum Flags : uint32_t {
+ // The frame has a valid return value. See also InterpreterFrame::HAS_RVAL.
+ HAS_RVAL = 1 << 0,
+
+ // The frame is running in the Baseline interpreter instead of JIT.
+ RUNNING_IN_INTERPRETER = 1 << 1,
+
+ // An initial environment has been pushed on the environment chain for
+ // function frames that need a CallObject or eval frames that need a
+ // VarEnvironmentObject.
+ HAS_INITIAL_ENV = 1 << 2,
+
+ // Frame has an arguments object, argsObj_.
+ HAS_ARGS_OBJ = 1 << 4,
+
+ // See InterpreterFrame::PREV_UP_TO_DATE.
+ PREV_UP_TO_DATE = 1 << 5,
+
+ // Frame has execution observed by a Debugger.
+ //
+ // See comment above 'isDebuggee' in vm/Realm.h for explanation
+ // of invariants of debuggee compartments, scripts, and frames.
+ DEBUGGEE = 1 << 6,
+ };
+
+ protected: // Silence Clang warning about unused private fields.
+ // The fields below are only valid if RUNNING_IN_INTERPRETER.
+ JSScript* interpreterScript_;
+ jsbytecode* interpreterPC_;
+ ICEntry* interpreterICEntry_;
+
+ JSObject* envChain_; // Environment chain (always initialized).
+ ICScript* icScript_; // IC script (initialized if Warp is enabled).
+ ArgumentsObject* argsObj_; // If HAS_ARGS_OBJ, the arguments object.
+
+ // We need to split the Value into 2 fields of 32 bits, otherwise the C++
+ // compiler may add some padding between the fields.
+ uint32_t loScratchValue_;
+ uint32_t hiScratchValue_;
+ uint32_t flags_;
+#ifdef DEBUG
+ // Size of the frame. Stored in DEBUG builds when calling into C++. This is
+ // BaselineFrame::Size() + the size of the local and expression stack Values.
+ //
+ // We don't store this in release builds because it's redundant with the frame
+ // size computed from the frame pointers. In debug builds it's still useful
+ // for assertions.
+ uint32_t debugFrameSize_;
+#else
+ uint32_t unused_;
+#endif
+ uint32_t loReturnValue_; // If HAS_RVAL, the frame's return value.
+ uint32_t hiReturnValue_;
+
+ public:
+ [[nodiscard]] bool initForOsr(InterpreterFrame* fp, uint32_t numStackValues);
+
+#ifdef DEBUG
+ uint32_t debugFrameSize() const { return debugFrameSize_; }
+ void setDebugFrameSize(uint32_t frameSize) { debugFrameSize_ = frameSize; }
+#endif
+
+ JSObject* environmentChain() const { return envChain_; }
+ void setEnvironmentChain(JSObject* envChain) { envChain_ = envChain; }
+
+ template <typename SpecificEnvironment>
+ inline void pushOnEnvironmentChain(SpecificEnvironment& env);
+ template <typename SpecificEnvironment>
+ inline void popOffEnvironmentChain();
+ inline void replaceInnermostEnvironment(EnvironmentObject& env);
+
+ CalleeToken calleeToken() const { return framePrefix()->calleeToken(); }
+ void replaceCalleeToken(CalleeToken token) {
+ framePrefix()->replaceCalleeToken(token);
+ }
+ bool isConstructing() const {
+ return CalleeTokenIsConstructing(calleeToken());
+ }
+ JSScript* script() const { return ScriptFromCalleeToken(calleeToken()); }
+ JSFunction* callee() const { return CalleeTokenToFunction(calleeToken()); }
+ Value calleev() const { return ObjectValue(*callee()); }
+
+ size_t numValueSlots(size_t frameSize) const {
+ MOZ_ASSERT(frameSize == debugFrameSize());
+
+ MOZ_ASSERT(frameSize >= BaselineFrame::Size());
+ frameSize -= BaselineFrame::Size();
+
+ MOZ_ASSERT((frameSize % sizeof(Value)) == 0);
+ return frameSize / sizeof(Value);
+ }
+
+#ifdef DEBUG
+ size_t debugNumValueSlots() const { return numValueSlots(debugFrameSize()); }
+#endif
+
+ Value* valueSlot(size_t slot) const {
+ MOZ_ASSERT(slot < debugNumValueSlots());
+ return (Value*)this - (slot + 1);
+ }
+
+ static size_t frameSizeForNumValueSlots(size_t numValueSlots) {
+ return BaselineFrame::Size() + numValueSlots * sizeof(Value);
+ }
+
+ Value& unaliasedFormal(
+ unsigned i, MaybeCheckAliasing checkAliasing = CHECK_ALIASING) const {
+ MOZ_ASSERT(i < numFormalArgs());
+ MOZ_ASSERT_IF(checkAliasing, !script()->argsObjAliasesFormals() &&
+ !script()->formalIsAliased(i));
+ return argv()[i];
+ }
+
+ Value& unaliasedActual(
+ unsigned i, MaybeCheckAliasing checkAliasing = CHECK_ALIASING) const {
+ MOZ_ASSERT(i < numActualArgs());
+ MOZ_ASSERT_IF(checkAliasing, !script()->argsObjAliasesFormals());
+ MOZ_ASSERT_IF(checkAliasing && i < numFormalArgs(),
+ !script()->formalIsAliased(i));
+ return argv()[i];
+ }
+
+ Value& unaliasedLocal(uint32_t i) const {
+ MOZ_ASSERT(i < script()->nfixed());
+ return *valueSlot(i);
+ }
+
+ unsigned numActualArgs() const { return framePrefix()->numActualArgs(); }
+ unsigned numFormalArgs() const { return script()->function()->nargs(); }
+ Value& thisArgument() const {
+ MOZ_ASSERT(isFunctionFrame());
+ return framePrefix()->thisv();
+ }
+ Value* argv() const { return framePrefix()->actualArgs(); }
+
+ [[nodiscard]] bool saveGeneratorSlots(JSContext* cx, unsigned nslots,
+ ArrayObject* dest) const;
+
+ public:
+ void prepareForBaselineInterpreterToJitOSR() {
+ // Clearing the RUNNING_IN_INTERPRETER flag is sufficient, but we also null
+ // out the interpreter fields to ensure we don't use stale values.
+ flags_ &= ~RUNNING_IN_INTERPRETER;
+ interpreterScript_ = nullptr;
+ interpreterPC_ = nullptr;
+ }
+
+ private:
+ bool uninlineIsProfilerSamplingEnabled(JSContext* cx);
+
+ public:
+ // Switch a JIT frame on the stack to Interpreter mode. The caller is
+ // responsible for patching the return address into this frame to a location
+ // in the interpreter code. Also assert profiler sampling has been suppressed
+ // so the sampler thread doesn't see an inconsistent state while we are
+ // patching frames.
+ void switchFromJitToInterpreter(JSContext* cx, jsbytecode* pc) {
+ MOZ_ASSERT(!uninlineIsProfilerSamplingEnabled(cx));
+ MOZ_ASSERT(!runningInInterpreter());
+ flags_ |= RUNNING_IN_INTERPRETER;
+ setInterpreterFields(pc);
+ }
+ void switchFromJitToInterpreterAtPrologue(JSContext* cx) {
+ MOZ_ASSERT(!uninlineIsProfilerSamplingEnabled(cx));
+ MOZ_ASSERT(!runningInInterpreter());
+ flags_ |= RUNNING_IN_INTERPRETER;
+ setInterpreterFieldsForPrologue(script());
+ }
+
+ // Like switchFromJitToInterpreter, but set the interpreterICEntry_ field to
+ // nullptr. Initializing this field requires a binary search on the
+ // JitScript's ICEntry list but the exception handler never returns to this
+ // pc anyway so we can avoid the overhead.
+ void switchFromJitToInterpreterForExceptionHandler(JSContext* cx,
+ jsbytecode* pc) {
+ MOZ_ASSERT(!uninlineIsProfilerSamplingEnabled(cx));
+ MOZ_ASSERT(!runningInInterpreter());
+ flags_ |= RUNNING_IN_INTERPRETER;
+ interpreterScript_ = script();
+ interpreterPC_ = pc;
+ interpreterICEntry_ = nullptr;
+ }
+
+ bool runningInInterpreter() const { return flags_ & RUNNING_IN_INTERPRETER; }
+
+ JSScript* interpreterScript() const {
+ MOZ_ASSERT(runningInInterpreter());
+ return interpreterScript_;
+ }
+
+ jsbytecode* interpreterPC() const {
+ MOZ_ASSERT(runningInInterpreter());
+ return interpreterPC_;
+ }
+
+ void setInterpreterFields(JSScript* script, jsbytecode* pc);
+
+ void setInterpreterFields(jsbytecode* pc) {
+ setInterpreterFields(script(), pc);
+ }
+
+ // Initialize interpreter fields for resuming in the prologue (before the
+ // argument type check ICs).
+ void setInterpreterFieldsForPrologue(JSScript* script);
+
+ ICScript* icScript() const { return icScript_; }
+ void setICScript(ICScript* icScript) { icScript_ = icScript; }
+
+ // The script that owns the current ICScript.
+ JSScript* outerScript() const;
+
+ bool hasReturnValue() const { return flags_ & HAS_RVAL; }
+ MutableHandleValue returnValue() {
+ if (!hasReturnValue()) {
+ addressOfReturnValue()->setUndefined();
+ }
+ return MutableHandleValue::fromMarkedLocation(addressOfReturnValue());
+ }
+ void setReturnValue(const Value& v) {
+ returnValue().set(v);
+ flags_ |= HAS_RVAL;
+ }
+ inline Value* addressOfReturnValue() {
+ return reinterpret_cast<Value*>(&loReturnValue_);
+ }
+
+ bool hasInitialEnvironment() const { return flags_ & HAS_INITIAL_ENV; }
+
+ inline CallObject& callObj() const;
+
+ void setFlags(uint32_t flags) { flags_ = flags; }
+
+ [[nodiscard]] inline bool pushLexicalEnvironment(JSContext* cx,
+ Handle<LexicalScope*> scope);
+ [[nodiscard]] inline bool freshenLexicalEnvironment(JSContext* cx);
+ [[nodiscard]] inline bool recreateLexicalEnvironment(JSContext* cx);
+
+ [[nodiscard]] bool initFunctionEnvironmentObjects(JSContext* cx);
+ [[nodiscard]] bool pushClassBodyEnvironment(JSContext* cx,
+ Handle<ClassBodyScope*> scope);
+ [[nodiscard]] bool pushVarEnvironment(JSContext* cx, Handle<Scope*> scope);
+
+ void initArgsObjUnchecked(ArgumentsObject& argsobj) {
+ flags_ |= HAS_ARGS_OBJ;
+ argsObj_ = &argsobj;
+ }
+ void initArgsObj(ArgumentsObject& argsobj) {
+ MOZ_ASSERT(script()->needsArgsObj());
+ initArgsObjUnchecked(argsobj);
+ }
+ bool hasArgsObj() const { return flags_ & HAS_ARGS_OBJ; }
+ ArgumentsObject& argsObj() const {
+ MOZ_ASSERT(hasArgsObj());
+ MOZ_ASSERT(script()->needsArgsObj());
+ return *argsObj_;
+ }
+
+ bool prevUpToDate() const { return flags_ & PREV_UP_TO_DATE; }
+ void setPrevUpToDate() { flags_ |= PREV_UP_TO_DATE; }
+ void unsetPrevUpToDate() { flags_ &= ~PREV_UP_TO_DATE; }
+
+ bool isDebuggee() const { return flags_ & DEBUGGEE; }
+ void setIsDebuggee() { flags_ |= DEBUGGEE; }
+ inline void unsetIsDebuggee();
+
+ void trace(JSTracer* trc, const JSJitFrameIter& frame);
+
+ bool isGlobalFrame() const { return script()->isGlobalCode(); }
+ bool isModuleFrame() const { return script()->isModule(); }
+ bool isEvalFrame() const { return script()->isForEval(); }
+ bool isFunctionFrame() const {
+ return CalleeTokenIsFunction(calleeToken()) && !isModuleFrame();
+ }
+ bool isDebuggerEvalFrame() const { return false; }
+
+ JitFrameLayout* framePrefix() const {
+ uint8_t* fp = (uint8_t*)this + Size();
+ return (JitFrameLayout*)fp;
+ }
+
+ static size_t Size() { return sizeof(BaselineFrame); }
+
+ // The reverseOffsetOf methods below compute the offset relative to the
+ // frame's base pointer. Since the stack grows down, these offsets are
+ // negative.
+
+#ifdef DEBUG
+ static int reverseOffsetOfDebugFrameSize() {
+ return -int(Size()) + offsetof(BaselineFrame, debugFrameSize_);
+ }
+#endif
+
+ // The scratch value slot can either be used as a Value slot or as two
+ // separate 32-bit integer slots.
+ static int reverseOffsetOfScratchValueLow32() {
+ return -int(Size()) + offsetof(BaselineFrame, loScratchValue_);
+ }
+ static int reverseOffsetOfScratchValueHigh32() {
+ return -int(Size()) + offsetof(BaselineFrame, hiScratchValue_);
+ }
+ static int reverseOffsetOfScratchValue() {
+ return reverseOffsetOfScratchValueLow32();
+ }
+
+ static int reverseOffsetOfEnvironmentChain() {
+ return -int(Size()) + offsetof(BaselineFrame, envChain_);
+ }
+ static int reverseOffsetOfArgsObj() {
+ return -int(Size()) + offsetof(BaselineFrame, argsObj_);
+ }
+ static int reverseOffsetOfFlags() {
+ return -int(Size()) + offsetof(BaselineFrame, flags_);
+ }
+ static int reverseOffsetOfReturnValue() {
+ return -int(Size()) + offsetof(BaselineFrame, loReturnValue_);
+ }
+ static int reverseOffsetOfInterpreterScript() {
+ return -int(Size()) + offsetof(BaselineFrame, interpreterScript_);
+ }
+ static int reverseOffsetOfInterpreterPC() {
+ return -int(Size()) + offsetof(BaselineFrame, interpreterPC_);
+ }
+ static int reverseOffsetOfInterpreterICEntry() {
+ return -int(Size()) + offsetof(BaselineFrame, interpreterICEntry_);
+ }
+ static int reverseOffsetOfICScript() {
+ return -int(Size()) + offsetof(BaselineFrame, icScript_);
+ }
+ static int reverseOffsetOfLocal(size_t index) {
+ return -int(Size()) - (index + 1) * sizeof(Value);
+ }
+};
+
+// Ensure the frame is 8-byte aligned (required on ARM).
+static_assert((sizeof(BaselineFrame) % 8) == 0, "frame must be 8-byte aligned");
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BaselineFrame_h */
diff --git a/js/src/jit/BaselineFrameInfo-inl.h b/js/src/jit/BaselineFrameInfo-inl.h
new file mode 100644
index 0000000000..cd79a8fb64
--- /dev/null
+++ b/js/src/jit/BaselineFrameInfo-inl.h
@@ -0,0 +1,50 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineFrameInfo_inl_h
+#define jit_BaselineFrameInfo_inl_h
+
+#include "jit/BaselineFrameInfo.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+void CompilerFrameInfo::pop(StackAdjustment adjust) {
+ spIndex--;
+ StackValue* popped = &stack[spIndex];
+
+ if (adjust == AdjustStack && popped->kind() == StackValue::Stack) {
+ masm.addToStackPtr(Imm32(sizeof(Value)));
+ }
+ // Assert when anything uses this value.
+ popped->reset();
+}
+
+void CompilerFrameInfo::popn(uint32_t n, StackAdjustment adjust) {
+ uint32_t poppedStack = 0;
+ for (uint32_t i = 0; i < n; i++) {
+ if (peek(-1)->kind() == StackValue::Stack) {
+ poppedStack++;
+ }
+ pop(DontAdjustStack);
+ }
+ if (adjust == AdjustStack && poppedStack > 0) {
+ masm.addToStackPtr(Imm32(sizeof(Value) * poppedStack));
+ }
+}
+
+void InterpreterFrameInfo::pop() { popn(1); }
+
+void InterpreterFrameInfo::popn(uint32_t n) {
+ masm.addToStackPtr(Imm32(n * sizeof(Value)));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BaselineFrameInfo_inl_h */
diff --git a/js/src/jit/BaselineFrameInfo.cpp b/js/src/jit/BaselineFrameInfo.cpp
new file mode 100644
index 0000000000..d641ace2ab
--- /dev/null
+++ b/js/src/jit/BaselineFrameInfo.cpp
@@ -0,0 +1,239 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BaselineFrameInfo.h"
+
+#include <algorithm>
+
+#include "jit/BaselineIC.h"
+#ifdef DEBUG
+# include "jit/BytecodeAnalysis.h"
+#endif
+
+#include "jit/BaselineFrameInfo-inl.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+bool CompilerFrameInfo::init(TempAllocator& alloc) {
+ // An extra slot is needed for global scopes because INITGLEXICAL (stack
+ // depth 1) is compiled as a SETPROP (stack depth 2) on the global lexical
+ // scope.
+ size_t extra = script->isGlobalCode() ? 1 : 0;
+ size_t nstack =
+ std::max(script->nslots() - script->nfixed(), size_t(MinJITStackSize)) +
+ extra;
+ if (!stack.init(alloc, nstack)) {
+ return false;
+ }
+
+ return true;
+}
+
+void CompilerFrameInfo::sync(StackValue* val) {
+ switch (val->kind()) {
+ case StackValue::Stack:
+ break;
+ case StackValue::LocalSlot:
+ masm.pushValue(addressOfLocal(val->localSlot()));
+ break;
+ case StackValue::ArgSlot:
+ masm.pushValue(addressOfArg(val->argSlot()));
+ break;
+ case StackValue::ThisSlot:
+ masm.pushValue(addressOfThis());
+ break;
+ case StackValue::Register:
+ masm.pushValue(val->reg());
+ break;
+ case StackValue::Constant:
+ masm.pushValue(val->constant());
+ break;
+ default:
+ MOZ_CRASH("Invalid kind");
+ }
+
+ val->setStack();
+}
+
+void CompilerFrameInfo::syncStack(uint32_t uses) {
+ MOZ_ASSERT(uses <= stackDepth());
+
+ uint32_t depth = stackDepth() - uses;
+
+ for (uint32_t i = 0; i < depth; i++) {
+ StackValue* current = &stack[i];
+ sync(current);
+ }
+}
+
+uint32_t CompilerFrameInfo::numUnsyncedSlots() {
+ // Start at the bottom, find the first value that's not synced.
+ uint32_t i = 0;
+ for (; i < stackDepth(); i++) {
+ if (peek(-int32_t(i + 1))->kind() == StackValue::Stack) {
+ break;
+ }
+ }
+ return i;
+}
+
+void CompilerFrameInfo::popValue(ValueOperand dest) {
+ StackValue* val = peek(-1);
+
+ switch (val->kind()) {
+ case StackValue::Constant:
+ masm.moveValue(val->constant(), dest);
+ break;
+ case StackValue::LocalSlot:
+ masm.loadValue(addressOfLocal(val->localSlot()), dest);
+ break;
+ case StackValue::ArgSlot:
+ masm.loadValue(addressOfArg(val->argSlot()), dest);
+ break;
+ case StackValue::ThisSlot:
+ masm.loadValue(addressOfThis(), dest);
+ break;
+ case StackValue::Stack:
+ masm.popValue(dest);
+ break;
+ case StackValue::Register:
+ masm.moveValue(val->reg(), dest);
+ break;
+ default:
+ MOZ_CRASH("Invalid kind");
+ }
+
+ // masm.popValue already adjusted the stack pointer, don't do it twice.
+ pop(DontAdjustStack);
+}
+
+void CompilerFrameInfo::popRegsAndSync(uint32_t uses) {
+ // x86 has only 3 Value registers. Only support 2 regs here for now,
+ // so that there's always a scratch Value register for reg -> reg
+ // moves.
+ MOZ_ASSERT(uses > 0);
+ MOZ_ASSERT(uses <= 2);
+ MOZ_ASSERT(uses <= stackDepth());
+
+ syncStack(uses);
+
+ switch (uses) {
+ case 1:
+ popValue(R0);
+ break;
+ case 2: {
+ // If the second value is in R1, move it to R2 so that it's not
+ // clobbered by the first popValue.
+ StackValue* val = peek(-2);
+ if (val->kind() == StackValue::Register && val->reg() == R1) {
+ masm.moveValue(R1, ValueOperand(R2));
+ val->setRegister(R2);
+ }
+ popValue(R1);
+ popValue(R0);
+ break;
+ }
+ default:
+ MOZ_CRASH("Invalid uses");
+ }
+ // On arm64, SP may be < PSP now (that's OK).
+ // eg testcase: tests/bug1580246.js
+}
+
+void InterpreterFrameInfo::popRegsAndSync(uint32_t uses) {
+ switch (uses) {
+ case 1:
+ popValue(R0);
+ break;
+ case 2: {
+ popValue(R1);
+ popValue(R0);
+ break;
+ }
+ default:
+ MOZ_CRASH("Invalid uses");
+ }
+ // On arm64, SP may be < PSP now (that's OK).
+ // eg testcase: tests/backup-point-bug1315634.js
+}
+
+void InterpreterFrameInfo::bumpInterpreterICEntry() {
+ masm.addPtr(Imm32(sizeof(ICEntry)), addressOfInterpreterICEntry());
+}
+
+void CompilerFrameInfo::storeStackValue(int32_t depth, const Address& dest,
+ const ValueOperand& scratch) {
+ const StackValue* source = peek(depth);
+ switch (source->kind()) {
+ case StackValue::Constant:
+ masm.storeValue(source->constant(), dest);
+ break;
+ case StackValue::Register:
+ masm.storeValue(source->reg(), dest);
+ break;
+ case StackValue::LocalSlot:
+ masm.loadValue(addressOfLocal(source->localSlot()), scratch);
+ masm.storeValue(scratch, dest);
+ break;
+ case StackValue::ArgSlot:
+ masm.loadValue(addressOfArg(source->argSlot()), scratch);
+ masm.storeValue(scratch, dest);
+ break;
+ case StackValue::ThisSlot:
+ masm.loadValue(addressOfThis(), scratch);
+ masm.storeValue(scratch, dest);
+ break;
+ case StackValue::Stack:
+ masm.loadValue(addressOfStackValue(depth), scratch);
+ masm.storeValue(scratch, dest);
+ break;
+ default:
+ MOZ_CRASH("Invalid kind");
+ }
+}
+
+#ifdef DEBUG
+void CompilerFrameInfo::assertValidState(const BytecodeInfo& info) {
+ // Check stack depth.
+ MOZ_ASSERT(stackDepth() == info.stackDepth);
+
+ // Start at the bottom, find the first value that's not synced.
+ uint32_t i = 0;
+ for (; i < stackDepth(); i++) {
+ if (stack[i].kind() != StackValue::Stack) {
+ break;
+ }
+ }
+
+ // Assert all values on top of it are also not synced.
+ for (; i < stackDepth(); i++) {
+ MOZ_ASSERT(stack[i].kind() != StackValue::Stack);
+ }
+
+ // Assert every Value register is used by at most one StackValue.
+ // R2 is used as scratch register by the compiler and FrameInfo,
+ // so it shouldn't be used for StackValues.
+ bool usedR0 = false, usedR1 = false;
+
+ for (i = 0; i < stackDepth(); i++) {
+ if (stack[i].kind() == StackValue::Register) {
+ ValueOperand reg = stack[i].reg();
+ if (reg == R0) {
+ MOZ_ASSERT(!usedR0);
+ usedR0 = true;
+ } else if (reg == R1) {
+ MOZ_ASSERT(!usedR1);
+ usedR1 = true;
+ } else {
+ MOZ_CRASH("Invalid register");
+ }
+ }
+ }
+}
+#endif
diff --git a/js/src/jit/BaselineFrameInfo.h b/js/src/jit/BaselineFrameInfo.h
new file mode 100644
index 0000000000..b777aad809
--- /dev/null
+++ b/js/src/jit/BaselineFrameInfo.h
@@ -0,0 +1,434 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineFrameInfo_h
+#define jit_BaselineFrameInfo_h
+
+#include "mozilla/Attributes.h"
+#include "mozilla/Maybe.h"
+
+#include <new>
+
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineJIT.h"
+#include "jit/FixedList.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+struct BytecodeInfo;
+class MacroAssembler;
+
+// [SMDOC] Baseline FrameInfo overview.
+//
+// FrameInfo is used by BaselineCodeGen to track values stored in the frame.
+// There are two implementations:
+//
+// InterpreterFrameInfo
+// --------------------
+// The InterpreterFrameInfo class is used by the interpreter generator and is
+// a very simple interface on top of the MacroAssembler, because the stack is
+// always synced.
+//
+// CompilerFrameInfo
+// -----------------
+// The CompilerFrameInfo class is more complicated because it maintains a
+// virtual stack to optimize some common stack operations. Locals and arguments
+// are always fully synced. Stack values can either be synced, stored as
+// constant, stored in a Value register or refer to a local slot. Syncing a
+// StackValue ensures it's stored on the stack, e.g. kind == Stack.
+//
+// To see how this works, consider the following statement:
+//
+// var y = x + 9;
+//
+// Here two values are pushed: StackValue(LocalSlot(0)) and
+// StackValue(Int32Value(9)). Only when we reach the ADD op, code is generated
+// to load the operands directly into the right operand registers and sync all
+// other stack values.
+//
+// For stack values, the following invariants hold (and are checked between
+// ops):
+//
+// (1) If a value is synced (kind == Stack), all values below it must also be
+// synced. In other words, values with kind other than Stack can only appear
+// on top of the abstract stack.
+//
+// (2) When we call a stub or IC, all values still on the stack must be synced.
+
+// Represents a value pushed on the stack. Note that StackValue is not used for
+// locals or arguments since these are always fully synced.
+class StackValue {
+ public:
+ enum Kind {
+ Constant,
+ Register,
+ Stack,
+ LocalSlot,
+ ArgSlot,
+ ThisSlot,
+#ifdef DEBUG
+ // In debug builds, assert Kind is initialized.
+ Uninitialized,
+#endif
+ };
+
+ private:
+ MOZ_INIT_OUTSIDE_CTOR Kind kind_;
+
+ MOZ_INIT_OUTSIDE_CTOR union Data {
+ JS::Value constant;
+ ValueOperand reg;
+ uint32_t localSlot;
+ uint32_t argSlot;
+
+ // |constant| has a non-trivial constructor and therefore MUST be
+ // placement-new'd into existence.
+ MOZ_PUSH_DISABLE_NONTRIVIAL_UNION_WARNINGS
+ Data() {}
+ MOZ_POP_DISABLE_NONTRIVIAL_UNION_WARNINGS
+ } data;
+
+ MOZ_INIT_OUTSIDE_CTOR JSValueType knownType_;
+
+ public:
+ StackValue() { reset(); }
+
+ Kind kind() const { return kind_; }
+ bool hasKnownType() const { return knownType_ != JSVAL_TYPE_UNKNOWN; }
+ bool hasKnownType(JSValueType type) const {
+ MOZ_ASSERT(type != JSVAL_TYPE_UNKNOWN);
+ return knownType_ == type;
+ }
+ JSValueType knownType() const {
+ MOZ_ASSERT(hasKnownType());
+ return knownType_;
+ }
+ void reset() {
+#ifdef DEBUG
+ kind_ = Uninitialized;
+ knownType_ = JSVAL_TYPE_UNKNOWN;
+#endif
+ }
+ Value constant() const {
+ MOZ_ASSERT(kind_ == Constant);
+ return data.constant;
+ }
+ ValueOperand reg() const {
+ MOZ_ASSERT(kind_ == Register);
+ return data.reg;
+ }
+ uint32_t localSlot() const {
+ MOZ_ASSERT(kind_ == LocalSlot);
+ return data.localSlot;
+ }
+ uint32_t argSlot() const {
+ MOZ_ASSERT(kind_ == ArgSlot);
+ return data.argSlot;
+ }
+
+ void setConstant(const Value& v) {
+ kind_ = Constant;
+ new (&data.constant) Value(v);
+ knownType_ = v.isDouble() ? JSVAL_TYPE_DOUBLE : v.extractNonDoubleType();
+ }
+ void setRegister(const ValueOperand& val,
+ JSValueType knownType = JSVAL_TYPE_UNKNOWN) {
+ kind_ = Register;
+ new (&data.reg) ValueOperand(val);
+ knownType_ = knownType;
+ }
+ void setLocalSlot(uint32_t slot) {
+ kind_ = LocalSlot;
+ new (&data.localSlot) uint32_t(slot);
+ knownType_ = JSVAL_TYPE_UNKNOWN;
+ }
+ void setArgSlot(uint32_t slot) {
+ kind_ = ArgSlot;
+ new (&data.argSlot) uint32_t(slot);
+ knownType_ = JSVAL_TYPE_UNKNOWN;
+ }
+ void setThis() {
+ kind_ = ThisSlot;
+ knownType_ = JSVAL_TYPE_UNKNOWN;
+ }
+ void setStack() {
+ kind_ = Stack;
+ knownType_ = JSVAL_TYPE_UNKNOWN;
+ }
+};
+
+enum StackAdjustment { AdjustStack, DontAdjustStack };
+
+class FrameInfo {
+ protected:
+ MacroAssembler& masm;
+
+ public:
+ explicit FrameInfo(MacroAssembler& masm) : masm(masm) {}
+
+ Address addressOfLocal(size_t local) const {
+ return Address(FramePointer, BaselineFrame::reverseOffsetOfLocal(local));
+ }
+ Address addressOfArg(size_t arg) const {
+ return Address(FramePointer, JitFrameLayout::offsetOfActualArg(arg));
+ }
+ Address addressOfThis() const {
+ return Address(FramePointer, JitFrameLayout::offsetOfThis());
+ }
+ Address addressOfCalleeToken() const {
+ return Address(FramePointer, JitFrameLayout::offsetOfCalleeToken());
+ }
+ Address addressOfEnvironmentChain() const {
+ return Address(FramePointer,
+ BaselineFrame::reverseOffsetOfEnvironmentChain());
+ }
+ Address addressOfICScript() const {
+ return Address(FramePointer, BaselineFrame::reverseOffsetOfICScript());
+ }
+ Address addressOfFlags() const {
+ return Address(FramePointer, BaselineFrame::reverseOffsetOfFlags());
+ }
+ Address addressOfReturnValue() const {
+ return Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue());
+ }
+ Address addressOfArgsObj() const {
+ return Address(FramePointer, BaselineFrame::reverseOffsetOfArgsObj());
+ }
+ Address addressOfScratchValue() const {
+ return Address(FramePointer, BaselineFrame::reverseOffsetOfScratchValue());
+ }
+ Address addressOfScratchValueLow32() const {
+ return Address(FramePointer,
+ BaselineFrame::reverseOffsetOfScratchValueLow32());
+ }
+ Address addressOfScratchValueHigh32() const {
+ return Address(FramePointer,
+ BaselineFrame::reverseOffsetOfScratchValueHigh32());
+ }
+#ifdef DEBUG
+ Address addressOfDebugFrameSize() const {
+ return Address(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ }
+#endif
+};
+
+class CompilerFrameInfo : public FrameInfo {
+ JSScript* script;
+ FixedList<StackValue> stack;
+ size_t spIndex;
+
+ public:
+ CompilerFrameInfo(JSScript* script, MacroAssembler& masm)
+ : FrameInfo(masm), script(script), stack(), spIndex(0) {}
+ [[nodiscard]] bool init(TempAllocator& alloc);
+
+ size_t nlocals() const { return script->nfixed(); }
+ size_t nargs() const { return script->function()->nargs(); }
+
+ private:
+ inline StackValue* rawPush() {
+ StackValue* val = &stack[spIndex++];
+ val->reset();
+ return val;
+ }
+
+ inline StackValue* peek(int32_t index) const {
+ MOZ_ASSERT(index < 0);
+ return const_cast<StackValue*>(&stack[spIndex + index]);
+ }
+
+ public:
+ inline size_t stackDepth() const { return spIndex; }
+ inline void setStackDepth(uint32_t newDepth) {
+ if (newDepth <= stackDepth()) {
+ spIndex = newDepth;
+ } else {
+ uint32_t diff = newDepth - stackDepth();
+ for (uint32_t i = 0; i < diff; i++) {
+ StackValue* val = rawPush();
+ val->setStack();
+ }
+
+ MOZ_ASSERT(spIndex == newDepth);
+ }
+ }
+
+ void assertStackDepth(uint32_t depth) { MOZ_ASSERT(stackDepth() == depth); }
+ void incStackDepth(int32_t diff) { setStackDepth(stackDepth() + diff); }
+ bool hasKnownStackDepth(uint32_t depth) { return stackDepth() == depth; }
+
+ inline void pop(StackAdjustment adjust = AdjustStack);
+ inline void popn(uint32_t n, StackAdjustment adjust = AdjustStack);
+ inline void push(const Value& val) {
+ StackValue* sv = rawPush();
+ sv->setConstant(val);
+ }
+ inline void push(const ValueOperand& val,
+ JSValueType knownType = JSVAL_TYPE_UNKNOWN) {
+ StackValue* sv = rawPush();
+ sv->setRegister(val, knownType);
+ }
+ inline void pushLocal(uint32_t local) {
+ MOZ_ASSERT(local < nlocals());
+ StackValue* sv = rawPush();
+ sv->setLocalSlot(local);
+ }
+ inline void pushArg(uint32_t arg) {
+ StackValue* sv = rawPush();
+ sv->setArgSlot(arg);
+ }
+ inline void pushThis() {
+ StackValue* sv = rawPush();
+ sv->setThis();
+ }
+
+ inline void pushScratchValue() {
+ masm.pushValue(addressOfScratchValue());
+ StackValue* sv = rawPush();
+ sv->setStack();
+ }
+
+ Address addressOfLocal(size_t local) const {
+ MOZ_ASSERT(local < nlocals());
+ return FrameInfo::addressOfLocal(local);
+ }
+ Address addressOfArg(size_t arg) const {
+ MOZ_ASSERT(arg < nargs());
+ return FrameInfo::addressOfArg(arg);
+ }
+
+ Address addressOfStackValue(int32_t depth) const {
+ const StackValue* value = peek(depth);
+ MOZ_ASSERT(value->kind() == StackValue::Stack);
+ size_t slot = value - &stack[0];
+ MOZ_ASSERT(slot < stackDepth());
+ return Address(FramePointer,
+ BaselineFrame::reverseOffsetOfLocal(nlocals() + slot));
+ }
+
+ void popValue(ValueOperand dest);
+
+ void sync(StackValue* val);
+ void syncStack(uint32_t uses);
+ uint32_t numUnsyncedSlots();
+ void popRegsAndSync(uint32_t uses);
+
+ void assertSyncedStack() const {
+ MOZ_ASSERT_IF(stackDepth() > 0, peek(-1)->kind() == StackValue::Stack);
+ }
+
+ bool stackValueHasKnownType(int32_t depth, JSValueType type) const {
+ return peek(depth)->hasKnownType(type);
+ }
+
+ mozilla::Maybe<Value> knownStackValue(int32_t depth) const {
+ StackValue* val = peek(depth);
+ if (val->kind() == StackValue::Constant) {
+ return mozilla::Some(val->constant());
+ }
+ return mozilla::Nothing();
+ }
+
+ void storeStackValue(int32_t depth, const Address& dest,
+ const ValueOperand& scratch);
+
+ uint32_t frameSize() const {
+ return BaselineFrame::frameSizeForNumValueSlots(nlocals() + stackDepth());
+ }
+
+#ifdef DEBUG
+ // Assert the state is valid before excuting "pc".
+ void assertValidState(const BytecodeInfo& info);
+#else
+ inline void assertValidState(const BytecodeInfo& info) {}
+#endif
+};
+
+class InterpreterFrameInfo : public FrameInfo {
+ public:
+ explicit InterpreterFrameInfo(MacroAssembler& masm) : FrameInfo(masm) {}
+
+ // These methods are no-ops in the interpreter, because we don't have a
+ // virtual stack there.
+ void syncStack(uint32_t uses) {}
+ void assertSyncedStack() const {}
+ void assertStackDepth(uint32_t depth) {}
+ void incStackDepth(int32_t diff) {}
+ bool hasKnownStackDepth(uint32_t depth) { return false; }
+ uint32_t numUnsyncedSlots() { return 0; }
+
+ bool stackValueHasKnownType(int32_t depth, JSValueType type) const {
+ return false;
+ }
+
+ mozilla::Maybe<Value> knownStackValue(int32_t depth) const {
+ return mozilla::Nothing();
+ }
+
+ Address addressOfStackValue(int depth) const {
+ MOZ_ASSERT(depth < 0);
+ return Address(masm.getStackPointer(),
+ masm.framePushed() + size_t(-(depth + 1)) * sizeof(Value));
+ }
+
+ BaseIndex addressOfStackValue(Register index, int32_t offset = 0) const {
+ return BaseIndex(masm.getStackPointer(), index, ValueScale, offset);
+ }
+
+ void popRegsAndSync(uint32_t uses);
+
+ inline void pop();
+
+ inline void popn(uint32_t n);
+
+ void popn(Register reg) {
+ // sp := sp + reg * sizeof(Value)
+ Register spReg = AsRegister(masm.getStackPointer());
+ masm.computeEffectiveAddress(BaseValueIndex(spReg, reg), spReg);
+ // On arm64, SP may be < PSP now (that's OK).
+ // eg testcase: tests/arguments/strict-args-generator-flushstack.js
+ }
+
+ void popValue(ValueOperand dest) { masm.popValue(dest); }
+
+ void push(const ValueOperand& val,
+ JSValueType knownType = JSVAL_TYPE_UNKNOWN) {
+ masm.pushValue(val);
+ }
+ void push(const Value& val) { masm.pushValue(val); }
+
+ void pushThis() { masm.pushValue(addressOfThis()); }
+ void pushScratchValue() { masm.pushValue(addressOfScratchValue()); }
+
+ void storeStackValue(int32_t depth, const Address& dest,
+ const ValueOperand& scratch) {
+ masm.loadValue(addressOfStackValue(depth), scratch);
+ masm.storeValue(scratch, dest);
+ }
+
+ void bumpInterpreterICEntry();
+
+ Address addressOfInterpreterScript() const {
+ return Address(FramePointer,
+ BaselineFrame::reverseOffsetOfInterpreterScript());
+ }
+ Address addressOfInterpreterPC() const {
+ return Address(FramePointer, BaselineFrame::reverseOffsetOfInterpreterPC());
+ }
+ Address addressOfInterpreterICEntry() const {
+ return Address(FramePointer,
+ BaselineFrame::reverseOffsetOfInterpreterICEntry());
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BaselineFrameInfo_h */
diff --git a/js/src/jit/BaselineIC.cpp b/js/src/jit/BaselineIC.cpp
new file mode 100644
index 0000000000..c887cec87d
--- /dev/null
+++ b/js/src/jit/BaselineIC.cpp
@@ -0,0 +1,2495 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BaselineIC.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Sprintf.h"
+
+#include "jstypes.h"
+
+#include "builtin/Eval.h"
+#include "jit/BaselineCacheIRCompiler.h"
+#include "jit/CacheIRGenerator.h"
+#include "jit/CacheIRHealth.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/Linker.h"
+#include "jit/PerfSpewer.h"
+#include "jit/SharedICHelpers.h"
+#include "jit/SharedICRegisters.h"
+#include "jit/VMFunctions.h"
+#include "js/Conversions.h"
+#include "js/friend/ErrorMessages.h" // JSMSG_*
+#include "vm/BytecodeIterator.h"
+#include "vm/BytecodeLocation.h"
+#include "vm/BytecodeUtil.h"
+#include "vm/EqualityOperations.h"
+#include "vm/JSFunction.h"
+#include "vm/JSScript.h"
+#include "vm/Opcodes.h"
+#ifdef MOZ_VTUNE
+# include "vtune/VTuneWrapper.h"
+#endif
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/SharedICHelpers-inl.h"
+#include "jit/VMFunctionList-inl.h"
+#include "vm/BytecodeIterator-inl.h"
+#include "vm/BytecodeLocation-inl.h"
+#include "vm/EnvironmentObject-inl.h"
+#include "vm/Interpreter-inl.h"
+#include "vm/JSScript-inl.h"
+
+using mozilla::DebugOnly;
+
+namespace js {
+namespace jit {
+
+// Class used to emit all Baseline IC fallback code when initializing the
+// JitRuntime.
+class MOZ_RAII FallbackICCodeCompiler final {
+ BaselineICFallbackCode& code;
+ MacroAssembler& masm;
+
+ JSContext* cx;
+ bool inStubFrame_ = false;
+
+#ifdef DEBUG
+ bool entersStubFrame_ = false;
+ uint32_t framePushedAtEnterStubFrame_ = 0;
+#endif
+
+ [[nodiscard]] bool emitCall(bool isSpread, bool isConstructing);
+ [[nodiscard]] bool emitGetElem(bool hasReceiver);
+ [[nodiscard]] bool emitGetProp(bool hasReceiver);
+
+ public:
+ FallbackICCodeCompiler(JSContext* cx, BaselineICFallbackCode& code,
+ MacroAssembler& masm)
+ : code(code), masm(masm), cx(cx) {}
+
+#define DEF_METHOD(kind) [[nodiscard]] bool emit_##kind();
+ IC_BASELINE_FALLBACK_CODE_KIND_LIST(DEF_METHOD)
+#undef DEF_METHOD
+
+ void pushCallArguments(MacroAssembler& masm,
+ AllocatableGeneralRegisterSet regs, Register argcReg,
+ bool isConstructing);
+
+ // Push a payload specialized per compiler needed to execute stubs.
+ void PushStubPayload(MacroAssembler& masm, Register scratch);
+ void pushStubPayload(MacroAssembler& masm, Register scratch);
+
+ // Emits a tail call to a VMFunction wrapper.
+ [[nodiscard]] bool tailCallVMInternal(MacroAssembler& masm,
+ TailCallVMFunctionId id);
+
+ template <typename Fn, Fn fn>
+ [[nodiscard]] bool tailCallVM(MacroAssembler& masm);
+
+ // Emits a normal (non-tail) call to a VMFunction wrapper.
+ [[nodiscard]] bool callVMInternal(MacroAssembler& masm, VMFunctionId id);
+
+ template <typename Fn, Fn fn>
+ [[nodiscard]] bool callVM(MacroAssembler& masm);
+
+ // A stub frame is used when a stub wants to call into the VM without
+ // performing a tail call. This is required for the return address
+ // to pc mapping to work.
+ void enterStubFrame(MacroAssembler& masm, Register scratch);
+ void assumeStubFrame();
+ void leaveStubFrame(MacroAssembler& masm);
+};
+
+AllocatableGeneralRegisterSet BaselineICAvailableGeneralRegs(size_t numInputs) {
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+#if defined(JS_CODEGEN_ARM)
+ MOZ_ASSERT(!regs.has(ICTailCallReg));
+ regs.take(BaselineSecondScratchReg);
+#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ MOZ_ASSERT(!regs.has(ICTailCallReg));
+ MOZ_ASSERT(!regs.has(BaselineSecondScratchReg));
+#elif defined(JS_CODEGEN_ARM64)
+ MOZ_ASSERT(!regs.has(PseudoStackPointer));
+ MOZ_ASSERT(!regs.has(RealStackPointer));
+ MOZ_ASSERT(!regs.has(ICTailCallReg));
+#endif
+ regs.take(ICStubReg);
+
+ switch (numInputs) {
+ case 0:
+ break;
+ case 1:
+ regs.take(R0);
+ break;
+ case 2:
+ regs.take(R0);
+ regs.take(R1);
+ break;
+ default:
+ MOZ_CRASH("Invalid numInputs");
+ }
+
+ return regs;
+}
+
+static jsbytecode* StubOffsetToPc(const ICFallbackStub* stub,
+ const JSScript* script) {
+ return script->offsetToPC(stub->pcOffset());
+}
+
+#ifdef JS_JITSPEW
+void FallbackICSpew(JSContext* cx, ICFallbackStub* stub, const char* fmt, ...) {
+ if (JitSpewEnabled(JitSpew_BaselineICFallback)) {
+ RootedScript script(cx, GetTopJitJSScript(cx));
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+
+ char fmtbuf[100];
+ va_list args;
+ va_start(args, fmt);
+ (void)VsprintfLiteral(fmtbuf, fmt, args);
+ va_end(args);
+
+ JitSpew(
+ JitSpew_BaselineICFallback,
+ "Fallback hit for (%s:%u:%u) (pc=%zu,line=%u,uses=%u,stubs=%zu): %s",
+ script->filename(), script->lineno(), script->column(),
+ script->pcToOffset(pc), PCToLineNumber(script, pc),
+ script->getWarmUpCount(), stub->numOptimizedStubs(), fmtbuf);
+ }
+}
+#endif // JS_JITSPEW
+
+void ICEntry::trace(JSTracer* trc) {
+ ICStub* stub = firstStub();
+
+ // Trace CacheIR stubs.
+ while (!stub->isFallback()) {
+ stub->toCacheIRStub()->trace(trc);
+ stub = stub->toCacheIRStub()->next();
+ }
+
+ // Fallback stubs use runtime-wide trampoline code we don't need to trace.
+ MOZ_ASSERT(stub->usesTrampolineCode());
+}
+
+// constexpr table mapping JSOp to BaselineICFallbackKind. Each value in the
+// table is either a fallback kind or a sentinel value (NoICValue) indicating
+// the JSOp is not a JOF_IC op.
+class MOZ_STATIC_CLASS OpToFallbackKindTable {
+ static_assert(sizeof(BaselineICFallbackKind) == sizeof(uint8_t));
+ uint8_t table_[JSOP_LIMIT] = {};
+
+ constexpr void setKind(JSOp op, BaselineICFallbackKind kind) {
+ MOZ_ASSERT(uint8_t(kind) != NoICValue);
+ table_[size_t(op)] = uint8_t(kind);
+ }
+
+ public:
+ static constexpr uint8_t NoICValue = uint8_t(BaselineICFallbackKind::Count);
+
+ uint8_t lookup(JSOp op) const { return table_[size_t(op)]; }
+
+ constexpr OpToFallbackKindTable() {
+ for (size_t i = 0; i < JSOP_LIMIT; i++) {
+ table_[i] = NoICValue;
+ }
+
+ setKind(JSOp::Not, BaselineICFallbackKind::ToBool);
+ setKind(JSOp::And, BaselineICFallbackKind::ToBool);
+ setKind(JSOp::Or, BaselineICFallbackKind::ToBool);
+ setKind(JSOp::JumpIfTrue, BaselineICFallbackKind::ToBool);
+ setKind(JSOp::JumpIfFalse, BaselineICFallbackKind::ToBool);
+
+ setKind(JSOp::BitNot, BaselineICFallbackKind::UnaryArith);
+ setKind(JSOp::Pos, BaselineICFallbackKind::UnaryArith);
+ setKind(JSOp::Neg, BaselineICFallbackKind::UnaryArith);
+ setKind(JSOp::Inc, BaselineICFallbackKind::UnaryArith);
+ setKind(JSOp::Dec, BaselineICFallbackKind::UnaryArith);
+ setKind(JSOp::ToNumeric, BaselineICFallbackKind::UnaryArith);
+
+ setKind(JSOp::BitOr, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::BitXor, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::BitAnd, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Lsh, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Rsh, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Ursh, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Add, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Sub, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Mul, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Div, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Mod, BaselineICFallbackKind::BinaryArith);
+ setKind(JSOp::Pow, BaselineICFallbackKind::BinaryArith);
+
+ setKind(JSOp::Eq, BaselineICFallbackKind::Compare);
+ setKind(JSOp::Ne, BaselineICFallbackKind::Compare);
+ setKind(JSOp::Lt, BaselineICFallbackKind::Compare);
+ setKind(JSOp::Le, BaselineICFallbackKind::Compare);
+ setKind(JSOp::Gt, BaselineICFallbackKind::Compare);
+ setKind(JSOp::Ge, BaselineICFallbackKind::Compare);
+ setKind(JSOp::StrictEq, BaselineICFallbackKind::Compare);
+ setKind(JSOp::StrictNe, BaselineICFallbackKind::Compare);
+
+ setKind(JSOp::NewArray, BaselineICFallbackKind::NewArray);
+
+ setKind(JSOp::NewObject, BaselineICFallbackKind::NewObject);
+ setKind(JSOp::NewInit, BaselineICFallbackKind::NewObject);
+
+ setKind(JSOp::InitElem, BaselineICFallbackKind::SetElem);
+ setKind(JSOp::InitHiddenElem, BaselineICFallbackKind::SetElem);
+ setKind(JSOp::InitLockedElem, BaselineICFallbackKind::SetElem);
+ setKind(JSOp::InitElemInc, BaselineICFallbackKind::SetElem);
+ setKind(JSOp::SetElem, BaselineICFallbackKind::SetElem);
+ setKind(JSOp::StrictSetElem, BaselineICFallbackKind::SetElem);
+
+ setKind(JSOp::InitProp, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::InitLockedProp, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::InitHiddenProp, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::InitGLexical, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::SetProp, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::StrictSetProp, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::SetName, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::StrictSetName, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::SetGName, BaselineICFallbackKind::SetProp);
+ setKind(JSOp::StrictSetGName, BaselineICFallbackKind::SetProp);
+
+ setKind(JSOp::GetProp, BaselineICFallbackKind::GetProp);
+ setKind(JSOp::GetBoundName, BaselineICFallbackKind::GetProp);
+
+ setKind(JSOp::GetPropSuper, BaselineICFallbackKind::GetPropSuper);
+
+ setKind(JSOp::GetElem, BaselineICFallbackKind::GetElem);
+
+ setKind(JSOp::GetElemSuper, BaselineICFallbackKind::GetElemSuper);
+
+ setKind(JSOp::In, BaselineICFallbackKind::In);
+
+ setKind(JSOp::HasOwn, BaselineICFallbackKind::HasOwn);
+
+ setKind(JSOp::CheckPrivateField, BaselineICFallbackKind::CheckPrivateField);
+
+ setKind(JSOp::GetName, BaselineICFallbackKind::GetName);
+ setKind(JSOp::GetGName, BaselineICFallbackKind::GetName);
+
+ setKind(JSOp::BindName, BaselineICFallbackKind::BindName);
+ setKind(JSOp::BindGName, BaselineICFallbackKind::BindName);
+
+ setKind(JSOp::GetIntrinsic, BaselineICFallbackKind::GetIntrinsic);
+
+ setKind(JSOp::Call, BaselineICFallbackKind::Call);
+ setKind(JSOp::CallContent, BaselineICFallbackKind::Call);
+ setKind(JSOp::CallIgnoresRv, BaselineICFallbackKind::Call);
+ setKind(JSOp::CallIter, BaselineICFallbackKind::Call);
+ setKind(JSOp::CallContentIter, BaselineICFallbackKind::Call);
+ setKind(JSOp::Eval, BaselineICFallbackKind::Call);
+ setKind(JSOp::StrictEval, BaselineICFallbackKind::Call);
+
+ setKind(JSOp::SuperCall, BaselineICFallbackKind::CallConstructing);
+ setKind(JSOp::New, BaselineICFallbackKind::CallConstructing);
+ setKind(JSOp::NewContent, BaselineICFallbackKind::CallConstructing);
+
+ setKind(JSOp::SpreadCall, BaselineICFallbackKind::SpreadCall);
+ setKind(JSOp::SpreadEval, BaselineICFallbackKind::SpreadCall);
+ setKind(JSOp::StrictSpreadEval, BaselineICFallbackKind::SpreadCall);
+
+ setKind(JSOp::SpreadSuperCall,
+ BaselineICFallbackKind::SpreadCallConstructing);
+ setKind(JSOp::SpreadNew, BaselineICFallbackKind::SpreadCallConstructing);
+
+ setKind(JSOp::Instanceof, BaselineICFallbackKind::InstanceOf);
+
+ setKind(JSOp::Typeof, BaselineICFallbackKind::TypeOf);
+ setKind(JSOp::TypeofExpr, BaselineICFallbackKind::TypeOf);
+
+ setKind(JSOp::ToPropertyKey, BaselineICFallbackKind::ToPropertyKey);
+
+ setKind(JSOp::Iter, BaselineICFallbackKind::GetIterator);
+
+ setKind(JSOp::OptimizeSpreadCall,
+ BaselineICFallbackKind::OptimizeSpreadCall);
+
+ setKind(JSOp::Rest, BaselineICFallbackKind::Rest);
+
+ setKind(JSOp::CloseIter, BaselineICFallbackKind::CloseIter);
+ }
+};
+
+static constexpr OpToFallbackKindTable FallbackKindTable;
+
+void ICScript::initICEntries(JSContext* cx, JSScript* script) {
+ MOZ_ASSERT(cx->realm()->jitRealm());
+ MOZ_ASSERT(jit::IsBaselineInterpreterEnabled());
+
+ MOZ_ASSERT(numICEntries() == script->numICEntries());
+
+ // Index of the next ICEntry to initialize.
+ uint32_t icEntryIndex = 0;
+
+ const BaselineICFallbackCode& fallbackCode =
+ cx->runtime()->jitRuntime()->baselineICFallbackCode();
+
+ // For JOF_IC ops: initialize ICEntries and fallback stubs.
+ for (BytecodeLocation loc : js::AllBytecodesIterable(script)) {
+ JSOp op = loc.getOp();
+
+ // Assert the frontend stored the correct IC index in jump target ops.
+ MOZ_ASSERT_IF(BytecodeIsJumpTarget(op), loc.icIndex() == icEntryIndex);
+
+ uint8_t tableValue = FallbackKindTable.lookup(op);
+
+ if (tableValue == OpToFallbackKindTable::NoICValue) {
+ MOZ_ASSERT(!BytecodeOpHasIC(op),
+ "Missing entry in OpToFallbackKindTable for JOF_IC op");
+ continue;
+ }
+
+ MOZ_ASSERT(BytecodeOpHasIC(op),
+ "Unexpected fallback kind for non-JOF_IC op");
+
+ BaselineICFallbackKind kind = BaselineICFallbackKind(tableValue);
+ TrampolinePtr stubCode = fallbackCode.addr(kind);
+
+ // Initialize the ICEntry and ICFallbackStub.
+ uint32_t offset = loc.bytecodeToOffset(script);
+ ICEntry& entryRef = this->icEntry(icEntryIndex);
+ ICFallbackStub* stub = fallbackStub(icEntryIndex);
+ icEntryIndex++;
+ new (&entryRef) ICEntry(stub);
+ new (stub) ICFallbackStub(offset, stubCode);
+ }
+
+ // Assert all ICEntries have been initialized.
+ MOZ_ASSERT(icEntryIndex == numICEntries());
+}
+
+bool ICSupportsPolymorphicTypeData(JSOp op) {
+ MOZ_ASSERT(BytecodeOpHasIC(op));
+ BaselineICFallbackKind kind =
+ BaselineICFallbackKind(FallbackKindTable.lookup(op));
+ switch (kind) {
+ case BaselineICFallbackKind::ToBool:
+ case BaselineICFallbackKind::TypeOf:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool ICCacheIRStub::makesGCCalls() const { return stubInfo()->makesGCCalls(); }
+
+void ICFallbackStub::trackNotAttached() { state().trackNotAttached(); }
+
+// When we enter a baseline fallback stub, if a Warp compilation
+// exists that transpiled that IC, we notify that compilation. This
+// helps the bailout code tell whether a bailing instruction hoisted
+// by LICM would have been executed anyway.
+static void MaybeNotifyWarp(JSScript* script, ICFallbackStub* stub) {
+ if (stub->state().usedByTranspiler() && script->hasIonScript()) {
+ script->ionScript()->noteBaselineFallback();
+ }
+}
+
+void ICCacheIRStub::trace(JSTracer* trc) {
+ JitCode* stubJitCode = jitCode();
+ TraceManuallyBarrieredEdge(trc, &stubJitCode, "baseline-ic-stub-code");
+
+ TraceCacheIRStub(trc, this, stubInfo());
+}
+
+static void MaybeTransition(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub) {
+ if (stub->state().shouldTransition()) {
+ if (!TryFoldingStubs(cx, stub, frame->script(), frame->icScript())) {
+ cx->recoverFromOutOfMemory();
+ }
+ if (stub->state().maybeTransition()) {
+ ICEntry* icEntry = frame->icScript()->icEntryForStub(stub);
+#ifdef JS_CACHEIR_SPEW
+ if (cx->spewer().enabled(cx, frame->script(),
+ SpewChannel::CacheIRHealthReport)) {
+ CacheIRHealth cih;
+ RootedScript script(cx, frame->script());
+ cih.healthReportForIC(cx, icEntry, stub, script,
+ SpewContext::Transition);
+ }
+#endif
+ stub->discardStubs(cx, icEntry);
+ }
+ }
+}
+
+// This helper handles ICState updates/transitions while attaching CacheIR
+// stubs.
+template <typename IRGenerator, typename... Args>
+static void TryAttachStub(const char* name, JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, Args&&... args) {
+ MaybeTransition(cx, frame, stub);
+
+ if (stub->state().canAttachStub()) {
+ RootedScript script(cx, frame->script());
+ ICScript* icScript = frame->icScript();
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ bool attached = false;
+ IRGenerator gen(cx, script, pc, stub->state(), std::forward<Args>(args)...);
+ switch (gen.tryAttachStub()) {
+ case AttachDecision::Attach: {
+ ICAttachResult result =
+ AttachBaselineCacheIRStub(cx, gen.writerRef(), gen.cacheKind(),
+ script, icScript, stub, gen.stubName());
+ if (result == ICAttachResult::Attached) {
+ attached = true;
+ JitSpew(JitSpew_BaselineIC, " Attached %s CacheIR stub", name);
+ }
+ } break;
+ case AttachDecision::NoAction:
+ break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ case AttachDecision::Deferred:
+ MOZ_ASSERT_UNREACHABLE("Not expected in generic TryAttachStub");
+ break;
+ }
+ if (!attached) {
+ stub->trackNotAttached();
+ }
+ }
+}
+
+void ICFallbackStub::unlinkStub(Zone* zone, ICEntry* icEntry,
+ ICCacheIRStub* prev, ICCacheIRStub* stub) {
+ if (prev) {
+ MOZ_ASSERT(prev->next() == stub);
+ prev->setNext(stub->next());
+ } else {
+ MOZ_ASSERT(icEntry->firstStub() == stub);
+ icEntry->setFirstStub(stub->next());
+ }
+
+ state_.trackUnlinkedStub();
+
+ // We are removing edges from ICStub to gcthings. Perform a barrier to let the
+ // GC know about those edges.
+ PreWriteBarrier(zone, stub);
+
+#ifdef DEBUG
+ // Poison stub code to ensure we don't call this stub again. However, if
+ // this stub can make calls, a pointer to it may be stored in a stub frame
+ // on the stack, so we can't touch the stubCode_ or GC will crash when
+ // tracing this pointer.
+ if (!stub->makesGCCalls()) {
+ stub->stubCode_ = (uint8_t*)0xbad;
+ }
+#endif
+}
+
+void ICFallbackStub::discardStubs(JSContext* cx, ICEntry* icEntry) {
+ ICStub* stub = icEntry->firstStub();
+ while (stub != this) {
+ unlinkStub(cx->zone(), icEntry, /* prev = */ nullptr,
+ stub->toCacheIRStub());
+ stub = stub->toCacheIRStub()->next();
+ }
+ clearHasFoldedStub();
+}
+
+static void InitMacroAssemblerForICStub(StackMacroAssembler& masm) {
+#ifndef JS_USE_LINK_REGISTER
+ // The first value contains the return addres,
+ // which we pull into ICTailCallReg for tail calls.
+ masm.adjustFrame(sizeof(intptr_t));
+#endif
+#ifdef JS_CODEGEN_ARM
+ masm.setSecondScratchReg(BaselineSecondScratchReg);
+#endif
+}
+
+bool FallbackICCodeCompiler::tailCallVMInternal(MacroAssembler& masm,
+ TailCallVMFunctionId id) {
+ TrampolinePtr code = cx->runtime()->jitRuntime()->getVMWrapper(id);
+ const VMFunctionData& fun = GetVMFunction(id);
+ MOZ_ASSERT(fun.expectTailCall == TailCall);
+ uint32_t argSize = fun.explicitStackSlots() * sizeof(void*);
+ EmitBaselineTailCallVM(code, masm, argSize);
+ return true;
+}
+
+bool FallbackICCodeCompiler::callVMInternal(MacroAssembler& masm,
+ VMFunctionId id) {
+ MOZ_ASSERT(inStubFrame_);
+
+ TrampolinePtr code = cx->runtime()->jitRuntime()->getVMWrapper(id);
+ MOZ_ASSERT(GetVMFunction(id).expectTailCall == NonTailCall);
+
+ EmitBaselineCallVM(code, masm);
+ return true;
+}
+
+template <typename Fn, Fn fn>
+bool FallbackICCodeCompiler::callVM(MacroAssembler& masm) {
+ VMFunctionId id = VMFunctionToId<Fn, fn>::id;
+ return callVMInternal(masm, id);
+}
+
+template <typename Fn, Fn fn>
+bool FallbackICCodeCompiler::tailCallVM(MacroAssembler& masm) {
+ TailCallVMFunctionId id = TailCallVMFunctionToId<Fn, fn>::id;
+ return tailCallVMInternal(masm, id);
+}
+
+void FallbackICCodeCompiler::enterStubFrame(MacroAssembler& masm,
+ Register scratch) {
+ EmitBaselineEnterStubFrame(masm, scratch);
+#ifdef DEBUG
+ framePushedAtEnterStubFrame_ = masm.framePushed();
+#endif
+
+ MOZ_ASSERT(!inStubFrame_);
+ inStubFrame_ = true;
+
+#ifdef DEBUG
+ entersStubFrame_ = true;
+#endif
+}
+
+void FallbackICCodeCompiler::assumeStubFrame() {
+ MOZ_ASSERT(!inStubFrame_);
+ inStubFrame_ = true;
+
+#ifdef DEBUG
+ entersStubFrame_ = true;
+
+ // |framePushed| isn't tracked precisely in ICStubs, so simply assume it to
+ // be the stub frame layout and the pushed ICStub* so that assertions don't
+ // fail in leaveStubFrame
+ framePushedAtEnterStubFrame_ =
+ BaselineStubFrameLayout::Size() + sizeof(ICStub*);
+#endif
+}
+
+void FallbackICCodeCompiler::leaveStubFrame(MacroAssembler& masm) {
+ MOZ_ASSERT(entersStubFrame_ && inStubFrame_);
+ inStubFrame_ = false;
+
+#ifdef DEBUG
+ masm.setFramePushed(framePushedAtEnterStubFrame_);
+#endif
+ EmitBaselineLeaveStubFrame(masm);
+}
+
+void FallbackICCodeCompiler::pushStubPayload(MacroAssembler& masm,
+ Register scratch) {
+ if (inStubFrame_) {
+ masm.loadPtr(Address(FramePointer, 0), scratch);
+ masm.pushBaselineFramePtr(scratch, scratch);
+ } else {
+ masm.pushBaselineFramePtr(FramePointer, scratch);
+ }
+}
+
+void FallbackICCodeCompiler::PushStubPayload(MacroAssembler& masm,
+ Register scratch) {
+ pushStubPayload(masm, scratch);
+ masm.adjustFrame(sizeof(intptr_t));
+}
+
+//
+// ToBool_Fallback
+//
+
+bool DoToBoolFallback(JSContext* cx, BaselineFrame* frame, ICFallbackStub* stub,
+ HandleValue arg, MutableHandleValue ret) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "ToBool");
+
+ TryAttachStub<ToBoolIRGenerator>("ToBool", cx, frame, stub, arg);
+
+ bool cond = ToBoolean(arg);
+ ret.setBoolean(cond);
+
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_ToBool() {
+ static_assert(R0 == JSReturnOperand);
+
+ // Restore the tail call register.
+ EmitRestoreTailCallReg(masm);
+
+ // Push arguments.
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ MutableHandleValue);
+ return tailCallVM<Fn, DoToBoolFallback>(masm);
+}
+
+//
+// GetElem_Fallback
+//
+
+bool DoGetElemFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs, HandleValue rhs,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "GetElem");
+
+#ifdef DEBUG
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+ MOZ_ASSERT(JSOp(*pc) == JSOp::GetElem);
+#endif
+
+ TryAttachStub<GetPropIRGenerator>("GetElem", cx, frame, stub,
+ CacheKind::GetElem, lhs, rhs);
+
+ if (!GetElementOperation(cx, lhs, rhs, res)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool DoGetElemSuperFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs,
+ HandleValue rhs, HandleValue receiver,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+
+ JSOp op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "GetElemSuper(%s)", CodeName(op));
+
+ MOZ_ASSERT(op == JSOp::GetElemSuper);
+
+ // |lhs| is [[HomeObject]].[[Prototype]] which must be an Object or null.
+ MOZ_ASSERT(lhs.isObjectOrNull());
+
+ int lhsIndex = -1;
+ RootedObject lhsObj(
+ cx, ToObjectFromStackForPropertyAccess(cx, lhs, lhsIndex, rhs));
+ if (!lhsObj) {
+ return false;
+ }
+
+ TryAttachStub<GetPropIRGenerator>("GetElemSuper", cx, frame, stub,
+ CacheKind::GetElemSuper, lhs, rhs);
+
+ return GetObjectElementOperation(cx, op, lhsObj, receiver, rhs, res);
+}
+
+bool FallbackICCodeCompiler::emitGetElem(bool hasReceiver) {
+ static_assert(R0 == JSReturnOperand);
+
+ // Restore the tail call register.
+ EmitRestoreTailCallReg(masm);
+
+ // Super property getters use a |this| that differs from base object
+ if (hasReceiver) {
+ // State: receiver in R0, index in R1, obj on the stack
+
+ // Ensure stack is fully synced for the expression decompiler.
+ // We need: receiver, index, obj
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+ masm.pushValue(Address(masm.getStackPointer(), sizeof(Value) * 2));
+
+ // Push arguments.
+ masm.pushValue(R0); // Receiver
+ masm.pushValue(R1); // Index
+ masm.pushValue(Address(masm.getStackPointer(), sizeof(Value) * 5)); // Obj
+ masm.push(ICStubReg);
+ masm.pushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn =
+ bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ HandleValue, HandleValue, MutableHandleValue);
+ if (!tailCallVM<Fn, DoGetElemSuperFallback>(masm)) {
+ return false;
+ }
+ } else {
+ // Ensure stack is fully synced for the expression decompiler.
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+
+ // Push arguments.
+ masm.pushValue(R1);
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ masm.pushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*,
+ HandleValue, HandleValue, MutableHandleValue);
+ if (!tailCallVM<Fn, DoGetElemFallback>(masm)) {
+ return false;
+ }
+ }
+
+ // This is the resume point used when bailout rewrites call stack to undo
+ // Ion inlined frames. The return address pushed onto reconstructed stack
+ // will point here.
+ assumeStubFrame();
+ if (hasReceiver) {
+ code.initBailoutReturnOffset(BailoutReturnKind::GetElemSuper,
+ masm.currentOffset());
+ } else {
+ code.initBailoutReturnOffset(BailoutReturnKind::GetElem,
+ masm.currentOffset());
+ }
+
+ leaveStubFrame(masm);
+
+ EmitReturnFromIC(masm);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_GetElem() {
+ return emitGetElem(/* hasReceiver = */ false);
+}
+
+bool FallbackICCodeCompiler::emit_GetElemSuper() {
+ return emitGetElem(/* hasReceiver = */ true);
+}
+
+bool DoSetElemFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, Value* stack, HandleValue objv,
+ HandleValue index, HandleValue rhs) {
+ using DeferType = SetPropIRGenerator::DeferType;
+
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ RootedScript script(cx, frame->script());
+ RootedScript outerScript(cx, script);
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ JSOp op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "SetElem(%s)", CodeName(JSOp(*pc)));
+
+ MOZ_ASSERT(op == JSOp::SetElem || op == JSOp::StrictSetElem ||
+ op == JSOp::InitElem || op == JSOp::InitHiddenElem ||
+ op == JSOp::InitLockedElem || op == JSOp::InitElemInc);
+
+ int objvIndex = -3;
+ RootedObject obj(
+ cx, ToObjectFromStackForPropertyAccess(cx, objv, objvIndex, index));
+ if (!obj) {
+ return false;
+ }
+
+ Rooted<Shape*> oldShape(cx, obj->shape());
+
+ DeferType deferType = DeferType::None;
+ bool attached = false;
+
+ MaybeTransition(cx, frame, stub);
+
+ if (stub->state().canAttachStub()) {
+ ICScript* icScript = frame->icScript();
+ SetPropIRGenerator gen(cx, script, pc, CacheKind::SetElem, stub->state(),
+ objv, index, rhs);
+ switch (gen.tryAttachStub()) {
+ case AttachDecision::Attach: {
+ ICAttachResult result = AttachBaselineCacheIRStub(
+ cx, gen.writerRef(), gen.cacheKind(), frame->script(), icScript,
+ stub, gen.stubName());
+ if (result == ICAttachResult::Attached) {
+ attached = true;
+ JitSpew(JitSpew_BaselineIC, " Attached SetElem CacheIR stub");
+ }
+ } break;
+ case AttachDecision::NoAction:
+ break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ attached = true;
+ break;
+ case AttachDecision::Deferred:
+ deferType = gen.deferType();
+ MOZ_ASSERT(deferType != DeferType::None);
+ break;
+ }
+ }
+
+ if (op == JSOp::InitElem || op == JSOp::InitHiddenElem ||
+ op == JSOp::InitLockedElem) {
+ if (!InitElemOperation(cx, pc, obj, index, rhs)) {
+ return false;
+ }
+ } else if (op == JSOp::InitElemInc) {
+ if (!InitElemIncOperation(cx, obj.as<ArrayObject>(), index.toInt32(),
+ rhs)) {
+ return false;
+ }
+ } else {
+ if (!SetObjectElementWithReceiver(cx, obj, index, rhs, objv,
+ JSOp(*pc) == JSOp::StrictSetElem)) {
+ return false;
+ }
+ }
+
+ // Overwrite the object on the stack (pushed for the decompiler) with the rhs.
+ MOZ_ASSERT(stack[2] == objv);
+ stack[2] = rhs;
+
+ if (attached) {
+ return true;
+ }
+
+ // The SetObjectElement call might have entered this IC recursively, so try
+ // to transition.
+ MaybeTransition(cx, frame, stub);
+
+ bool canAttachStub = stub->state().canAttachStub();
+
+ if (deferType != DeferType::None && canAttachStub) {
+ SetPropIRGenerator gen(cx, script, pc, CacheKind::SetElem, stub->state(),
+ objv, index, rhs);
+
+ MOZ_ASSERT(deferType == DeferType::AddSlot);
+ AttachDecision decision = gen.tryAttachAddSlotStub(oldShape);
+
+ switch (decision) {
+ case AttachDecision::Attach: {
+ ICScript* icScript = frame->icScript();
+ ICAttachResult result = AttachBaselineCacheIRStub(
+ cx, gen.writerRef(), gen.cacheKind(), frame->script(), icScript,
+ stub, gen.stubName());
+ if (result == ICAttachResult::Attached) {
+ attached = true;
+ JitSpew(JitSpew_BaselineIC, " Attached SetElem CacheIR stub");
+ }
+ } break;
+ case AttachDecision::NoAction:
+ gen.trackAttached(IRGenerator::NotAttached);
+ break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ case AttachDecision::Deferred:
+ MOZ_ASSERT_UNREACHABLE("Invalid attach result");
+ break;
+ }
+ }
+ if (!attached && canAttachStub) {
+ stub->trackNotAttached();
+ }
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_SetElem() {
+ static_assert(R0 == JSReturnOperand);
+
+ EmitRestoreTailCallReg(masm);
+
+ // State: R0: object, R1: index, stack: rhs.
+ // For the decompiler, the stack has to be: object, index, rhs,
+ // so we push the index, then overwrite the rhs Value with R0
+ // and push the rhs value.
+ masm.pushValue(R1);
+ masm.loadValue(Address(masm.getStackPointer(), sizeof(Value)), R1);
+ masm.storeValue(R0, Address(masm.getStackPointer(), sizeof(Value)));
+ masm.pushValue(R1);
+
+ // Push arguments.
+ masm.pushValue(R1); // RHS
+
+ // Push index. On x86 and ARM two push instructions are emitted so use a
+ // separate register to store the old stack pointer.
+ masm.moveStackPtrTo(R1.scratchReg());
+ masm.pushValue(Address(R1.scratchReg(), 2 * sizeof(Value)));
+ masm.pushValue(R0); // Object.
+
+ // Push pointer to stack values, so that the stub can overwrite the object
+ // (pushed for the decompiler) with the rhs.
+ masm.computeEffectiveAddress(
+ Address(masm.getStackPointer(), 3 * sizeof(Value)), R0.scratchReg());
+ masm.push(R0.scratchReg());
+
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, Value*,
+ HandleValue, HandleValue, HandleValue);
+ return tailCallVM<Fn, DoSetElemFallback>(masm);
+}
+
+//
+// In_Fallback
+//
+
+bool DoInFallback(JSContext* cx, BaselineFrame* frame, ICFallbackStub* stub,
+ HandleValue key, HandleValue objValue,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "In");
+
+ if (!objValue.isObject()) {
+ ReportInNotObjectError(cx, key, objValue);
+ return false;
+ }
+
+ TryAttachStub<HasPropIRGenerator>("In", cx, frame, stub, CacheKind::In, key,
+ objValue);
+
+ RootedObject obj(cx, &objValue.toObject());
+ bool cond = false;
+ if (!OperatorIn(cx, key, obj, &cond)) {
+ return false;
+ }
+ res.setBoolean(cond);
+
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_In() {
+ EmitRestoreTailCallReg(masm);
+
+ // Sync for the decompiler.
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+
+ // Push arguments.
+ masm.pushValue(R1);
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ HandleValue, MutableHandleValue);
+ return tailCallVM<Fn, DoInFallback>(masm);
+}
+
+//
+// HasOwn_Fallback
+//
+
+bool DoHasOwnFallback(JSContext* cx, BaselineFrame* frame, ICFallbackStub* stub,
+ HandleValue keyValue, HandleValue objValue,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "HasOwn");
+
+ TryAttachStub<HasPropIRGenerator>("HasOwn", cx, frame, stub,
+ CacheKind::HasOwn, keyValue, objValue);
+
+ bool found;
+ if (!HasOwnProperty(cx, objValue, keyValue, &found)) {
+ return false;
+ }
+
+ res.setBoolean(found);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_HasOwn() {
+ EmitRestoreTailCallReg(masm);
+
+ // Sync for the decompiler.
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+
+ // Push arguments.
+ masm.pushValue(R1);
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ HandleValue, MutableHandleValue);
+ return tailCallVM<Fn, DoHasOwnFallback>(masm);
+}
+
+//
+// CheckPrivate_Fallback
+//
+
+bool DoCheckPrivateFieldFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue objValue,
+ HandleValue keyValue, MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+
+ FallbackICSpew(cx, stub, "CheckPrivateField");
+
+ MOZ_ASSERT(keyValue.isSymbol() && keyValue.toSymbol()->isPrivateName());
+
+ TryAttachStub<CheckPrivateFieldIRGenerator>("CheckPrivate", cx, frame, stub,
+ CacheKind::CheckPrivateField,
+ keyValue, objValue);
+
+ bool result;
+ if (!CheckPrivateFieldOperation(cx, pc, objValue, keyValue, &result)) {
+ return false;
+ }
+
+ res.setBoolean(result);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_CheckPrivateField() {
+ EmitRestoreTailCallReg(masm);
+
+ // Sync for the decompiler.
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+
+ // Push arguments.
+ masm.pushValue(R1);
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ HandleValue, MutableHandleValue);
+ return tailCallVM<Fn, DoCheckPrivateFieldFallback>(masm);
+}
+
+//
+// GetName_Fallback
+//
+
+bool DoGetNameFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleObject envChain,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ mozilla::DebugOnly<JSOp> op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "GetName(%s)", CodeName(JSOp(*pc)));
+
+ MOZ_ASSERT(op == JSOp::GetName || op == JSOp::GetGName);
+
+ Rooted<PropertyName*> name(cx, script->getName(pc));
+
+ TryAttachStub<GetNameIRGenerator>("GetName", cx, frame, stub, envChain, name);
+
+ static_assert(JSOpLength_GetGName == JSOpLength_GetName,
+ "Otherwise our check for JSOp::Typeof isn't ok");
+ if (JSOp(pc[JSOpLength_GetGName]) == JSOp::Typeof) {
+ if (!GetEnvironmentName<GetNameMode::TypeOf>(cx, envChain, name, res)) {
+ return false;
+ }
+ } else {
+ if (!GetEnvironmentName<GetNameMode::Normal>(cx, envChain, name, res)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_GetName() {
+ static_assert(R0 == JSReturnOperand);
+
+ EmitRestoreTailCallReg(masm);
+
+ masm.push(R0.scratchReg());
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleObject,
+ MutableHandleValue);
+ return tailCallVM<Fn, DoGetNameFallback>(masm);
+}
+
+//
+// BindName_Fallback
+//
+
+bool DoBindNameFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleObject envChain,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+ mozilla::DebugOnly<JSOp> op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "BindName(%s)", CodeName(JSOp(*pc)));
+
+ MOZ_ASSERT(op == JSOp::BindName || op == JSOp::BindGName);
+
+ Rooted<PropertyName*> name(cx, frame->script()->getName(pc));
+
+ TryAttachStub<BindNameIRGenerator>("BindName", cx, frame, stub, envChain,
+ name);
+
+ RootedObject scope(cx);
+ if (!LookupNameUnqualified(cx, name, envChain, &scope)) {
+ return false;
+ }
+
+ res.setObject(*scope);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_BindName() {
+ static_assert(R0 == JSReturnOperand);
+
+ EmitRestoreTailCallReg(masm);
+
+ masm.push(R0.scratchReg());
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleObject,
+ MutableHandleValue);
+ return tailCallVM<Fn, DoBindNameFallback>(masm);
+}
+
+//
+// GetIntrinsic_Fallback
+//
+
+bool DoGetIntrinsicFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ mozilla::DebugOnly<JSOp> op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "GetIntrinsic(%s)", CodeName(JSOp(*pc)));
+
+ MOZ_ASSERT(op == JSOp::GetIntrinsic);
+
+ if (!GetIntrinsicOperation(cx, script, pc, res)) {
+ return false;
+ }
+
+ TryAttachStub<GetIntrinsicIRGenerator>("GetIntrinsic", cx, frame, stub, res);
+
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_GetIntrinsic() {
+ EmitRestoreTailCallReg(masm);
+
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn =
+ bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, MutableHandleValue);
+ return tailCallVM<Fn, DoGetIntrinsicFallback>(masm);
+}
+
+//
+// GetProp_Fallback
+//
+
+bool DoGetPropFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, MutableHandleValue val,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ JSOp op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "GetProp(%s)", CodeName(op));
+
+ MOZ_ASSERT(op == JSOp::GetProp || op == JSOp::GetBoundName);
+
+ Rooted<PropertyName*> name(cx, script->getName(pc));
+ RootedValue idVal(cx, StringValue(name));
+
+ TryAttachStub<GetPropIRGenerator>("GetProp", cx, frame, stub,
+ CacheKind::GetProp, val, idVal);
+
+ if (op == JSOp::GetBoundName) {
+ RootedObject env(cx, &val.toObject());
+ RootedId id(cx, NameToId(name));
+ return GetNameBoundInEnvironment(cx, env, id, res);
+ }
+
+ MOZ_ASSERT(op == JSOp::GetProp);
+ if (!GetProperty(cx, val, name, res)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool DoGetPropSuperFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue receiver,
+ MutableHandleValue val, MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ FallbackICSpew(cx, stub, "GetPropSuper(%s)", CodeName(JSOp(*pc)));
+
+ MOZ_ASSERT(JSOp(*pc) == JSOp::GetPropSuper);
+
+ Rooted<PropertyName*> name(cx, script->getName(pc));
+ RootedValue idVal(cx, StringValue(name));
+
+ // |val| is [[HomeObject]].[[Prototype]] which must be an Object or null.
+ MOZ_ASSERT(val.isObjectOrNull());
+
+ int valIndex = -1;
+ RootedObject valObj(
+ cx, ToObjectFromStackForPropertyAccess(cx, val, valIndex, name));
+ if (!valObj) {
+ return false;
+ }
+
+ TryAttachStub<GetPropIRGenerator>("GetPropSuper", cx, frame, stub,
+ CacheKind::GetPropSuper, val, idVal);
+
+ if (!GetProperty(cx, valObj, receiver, name, res)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool FallbackICCodeCompiler::emitGetProp(bool hasReceiver) {
+ static_assert(R0 == JSReturnOperand);
+
+ EmitRestoreTailCallReg(masm);
+
+ // Super property getters use a |this| that differs from base object
+ if (hasReceiver) {
+ // Push arguments.
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+ masm.push(ICStubReg);
+ masm.pushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*,
+ HandleValue, MutableHandleValue, MutableHandleValue);
+ if (!tailCallVM<Fn, DoGetPropSuperFallback>(masm)) {
+ return false;
+ }
+ } else {
+ // Ensure stack is fully synced for the expression decompiler.
+ masm.pushValue(R0);
+
+ // Push arguments.
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ masm.pushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*,
+ MutableHandleValue, MutableHandleValue);
+ if (!tailCallVM<Fn, DoGetPropFallback>(masm)) {
+ return false;
+ }
+ }
+
+ // This is the resume point used when bailout rewrites call stack to undo
+ // Ion inlined frames. The return address pushed onto reconstructed stack
+ // will point here.
+ assumeStubFrame();
+ if (hasReceiver) {
+ code.initBailoutReturnOffset(BailoutReturnKind::GetPropSuper,
+ masm.currentOffset());
+ } else {
+ code.initBailoutReturnOffset(BailoutReturnKind::GetProp,
+ masm.currentOffset());
+ }
+
+ leaveStubFrame(masm);
+
+ EmitReturnFromIC(masm);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_GetProp() {
+ return emitGetProp(/* hasReceiver = */ false);
+}
+
+bool FallbackICCodeCompiler::emit_GetPropSuper() {
+ return emitGetProp(/* hasReceiver = */ true);
+}
+
+//
+// SetProp_Fallback
+//
+
+bool DoSetPropFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, Value* stack, HandleValue lhs,
+ HandleValue rhs) {
+ using DeferType = SetPropIRGenerator::DeferType;
+
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ JSOp op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "SetProp(%s)", CodeName(op));
+
+ MOZ_ASSERT(op == JSOp::SetProp || op == JSOp::StrictSetProp ||
+ op == JSOp::SetName || op == JSOp::StrictSetName ||
+ op == JSOp::SetGName || op == JSOp::StrictSetGName ||
+ op == JSOp::InitProp || op == JSOp::InitLockedProp ||
+ op == JSOp::InitHiddenProp || op == JSOp::InitGLexical);
+
+ Rooted<PropertyName*> name(cx, script->getName(pc));
+ RootedId id(cx, NameToId(name));
+
+ int lhsIndex = -2;
+ RootedObject obj(cx,
+ ToObjectFromStackForPropertyAccess(cx, lhs, lhsIndex, id));
+ if (!obj) {
+ return false;
+ }
+ Rooted<Shape*> oldShape(cx, obj->shape());
+
+ DeferType deferType = DeferType::None;
+ bool attached = false;
+ MaybeTransition(cx, frame, stub);
+
+ if (stub->state().canAttachStub()) {
+ RootedValue idVal(cx, StringValue(name));
+ SetPropIRGenerator gen(cx, script, pc, CacheKind::SetProp, stub->state(),
+ lhs, idVal, rhs);
+ switch (gen.tryAttachStub()) {
+ case AttachDecision::Attach: {
+ ICScript* icScript = frame->icScript();
+ ICAttachResult result = AttachBaselineCacheIRStub(
+ cx, gen.writerRef(), gen.cacheKind(), frame->script(), icScript,
+ stub, gen.stubName());
+ if (result == ICAttachResult::Attached) {
+ attached = true;
+ JitSpew(JitSpew_BaselineIC, " Attached SetProp CacheIR stub");
+ }
+ } break;
+ case AttachDecision::NoAction:
+ break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ attached = true;
+ break;
+ case AttachDecision::Deferred:
+ deferType = gen.deferType();
+ MOZ_ASSERT(deferType != DeferType::None);
+ break;
+ }
+ }
+
+ if (op == JSOp::InitProp || op == JSOp::InitLockedProp ||
+ op == JSOp::InitHiddenProp) {
+ if (!InitPropertyOperation(cx, pc, obj, name, rhs)) {
+ return false;
+ }
+ } else if (op == JSOp::SetName || op == JSOp::StrictSetName ||
+ op == JSOp::SetGName || op == JSOp::StrictSetGName) {
+ if (!SetNameOperation(cx, script, pc, obj, rhs)) {
+ return false;
+ }
+ } else if (op == JSOp::InitGLexical) {
+ ExtensibleLexicalEnvironmentObject* lexicalEnv;
+ if (script->hasNonSyntacticScope()) {
+ lexicalEnv = &NearestEnclosingExtensibleLexicalEnvironment(
+ frame->environmentChain());
+ } else {
+ lexicalEnv = &cx->global()->lexicalEnvironment();
+ }
+ InitGlobalLexicalOperation(cx, lexicalEnv, script, pc, rhs);
+ } else {
+ MOZ_ASSERT(op == JSOp::SetProp || op == JSOp::StrictSetProp);
+
+ ObjectOpResult result;
+ if (!SetProperty(cx, obj, id, rhs, lhs, result) ||
+ !result.checkStrictModeError(cx, obj, id, op == JSOp::StrictSetProp)) {
+ return false;
+ }
+ }
+
+ // Overwrite the LHS on the stack (pushed for the decompiler) with the RHS.
+ MOZ_ASSERT(stack[1] == lhs);
+ stack[1] = rhs;
+
+ if (attached) {
+ return true;
+ }
+
+ // The SetProperty call might have entered this IC recursively, so try
+ // to transition.
+ MaybeTransition(cx, frame, stub);
+
+ bool canAttachStub = stub->state().canAttachStub();
+
+ if (deferType != DeferType::None && canAttachStub) {
+ RootedValue idVal(cx, StringValue(name));
+ SetPropIRGenerator gen(cx, script, pc, CacheKind::SetProp, stub->state(),
+ lhs, idVal, rhs);
+
+ MOZ_ASSERT(deferType == DeferType::AddSlot);
+ AttachDecision decision = gen.tryAttachAddSlotStub(oldShape);
+
+ switch (decision) {
+ case AttachDecision::Attach: {
+ ICScript* icScript = frame->icScript();
+ ICAttachResult result = AttachBaselineCacheIRStub(
+ cx, gen.writerRef(), gen.cacheKind(), frame->script(), icScript,
+ stub, gen.stubName());
+ if (result == ICAttachResult::Attached) {
+ attached = true;
+ JitSpew(JitSpew_BaselineIC, " Attached SetElem CacheIR stub");
+ }
+ } break;
+ case AttachDecision::NoAction:
+ gen.trackAttached(IRGenerator::NotAttached);
+ break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ case AttachDecision::Deferred:
+ MOZ_ASSERT_UNREACHABLE("Invalid attach result");
+ break;
+ }
+ }
+ if (!attached && canAttachStub) {
+ stub->trackNotAttached();
+ }
+
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_SetProp() {
+ static_assert(R0 == JSReturnOperand);
+
+ EmitRestoreTailCallReg(masm);
+
+ // Ensure stack is fully synced for the expression decompiler.
+ // Overwrite the RHS value on top of the stack with the object, then push
+ // the RHS in R1 on top of that.
+ masm.storeValue(R0, Address(masm.getStackPointer(), 0));
+ masm.pushValue(R1);
+
+ // Push arguments.
+ masm.pushValue(R1);
+ masm.pushValue(R0);
+
+ // Push pointer to stack values, so that the stub can overwrite the object
+ // (pushed for the decompiler) with the RHS.
+ masm.computeEffectiveAddress(
+ Address(masm.getStackPointer(), 2 * sizeof(Value)), R0.scratchReg());
+ masm.push(R0.scratchReg());
+
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, Value*,
+ HandleValue, HandleValue);
+ if (!tailCallVM<Fn, DoSetPropFallback>(masm)) {
+ return false;
+ }
+
+ // This is the resume point used when bailout rewrites call stack to undo
+ // Ion inlined frames. The return address pushed onto reconstructed stack
+ // will point here.
+ assumeStubFrame();
+ code.initBailoutReturnOffset(BailoutReturnKind::SetProp,
+ masm.currentOffset());
+
+ leaveStubFrame(masm);
+ EmitReturnFromIC(masm);
+
+ return true;
+}
+
+//
+// Call_Fallback
+//
+
+bool DoCallFallback(JSContext* cx, BaselineFrame* frame, ICFallbackStub* stub,
+ uint32_t argc, Value* vp, MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ JSOp op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "Call(%s)", CodeName(op));
+
+ MOZ_ASSERT(argc == GET_ARGC(pc));
+ bool constructing =
+ (op == JSOp::New || op == JSOp::NewContent || op == JSOp::SuperCall);
+ bool ignoresReturnValue = (op == JSOp::CallIgnoresRv);
+
+ // Ensure vp array is rooted - we may GC in here.
+ size_t numValues = argc + 2 + constructing;
+ RootedExternalValueArray vpRoot(cx, numValues, vp);
+
+ CallArgs callArgs = CallArgsFromSp(argc + constructing, vp + numValues,
+ constructing, ignoresReturnValue);
+ RootedValue callee(cx, vp[0]);
+ RootedValue newTarget(cx, constructing ? callArgs.newTarget() : NullValue());
+
+ // Transition stub state to megamorphic or generic if warranted.
+ MaybeTransition(cx, frame, stub);
+
+ bool canAttachStub = stub->state().canAttachStub();
+ bool handled = false;
+
+ // Only bother to try optimizing JSOp::Call with CacheIR if the chain is still
+ // allowed to attach stubs.
+ if (canAttachStub) {
+ HandleValueArray args = HandleValueArray::fromMarkedLocation(argc, vp + 2);
+ CallIRGenerator gen(cx, script, pc, op, stub->state(), argc, callee,
+ callArgs.thisv(), newTarget, args);
+ switch (gen.tryAttachStub()) {
+ case AttachDecision::NoAction:
+ break;
+ case AttachDecision::Attach: {
+ ICScript* icScript = frame->icScript();
+ ICAttachResult result =
+ AttachBaselineCacheIRStub(cx, gen.writerRef(), gen.cacheKind(),
+ script, icScript, stub, gen.stubName());
+ if (result == ICAttachResult::Attached) {
+ handled = true;
+ JitSpew(JitSpew_BaselineIC, " Attached Call CacheIR stub");
+ }
+ } break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ handled = true;
+ break;
+ case AttachDecision::Deferred:
+ MOZ_CRASH("No deferred Call stubs");
+ }
+ if (!handled) {
+ stub->trackNotAttached();
+ }
+ }
+
+ if (constructing) {
+ if (!ConstructFromStack(cx, callArgs)) {
+ return false;
+ }
+ res.set(callArgs.rval());
+ } else if ((op == JSOp::Eval || op == JSOp::StrictEval) &&
+ cx->global()->valueIsEval(callee)) {
+ if (!DirectEval(cx, callArgs.get(0), res)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(op == JSOp::Call || op == JSOp::CallContent ||
+ op == JSOp::CallIgnoresRv || op == JSOp::CallIter ||
+ op == JSOp::CallContentIter || op == JSOp::Eval ||
+ op == JSOp::StrictEval);
+ if ((op == JSOp::CallIter || op == JSOp::CallContentIter) &&
+ callee.isPrimitive()) {
+ MOZ_ASSERT(argc == 0, "thisv must be on top of the stack");
+ ReportValueError(cx, JSMSG_NOT_ITERABLE, -1, callArgs.thisv(), nullptr);
+ return false;
+ }
+
+ if (!CallFromStack(cx, callArgs)) {
+ return false;
+ }
+
+ res.set(callArgs.rval());
+ }
+
+ return true;
+}
+
+bool DoSpreadCallFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, Value* vp,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+ JSOp op = JSOp(*pc);
+ bool constructing = (op == JSOp::SpreadNew || op == JSOp::SpreadSuperCall);
+ FallbackICSpew(cx, stub, "SpreadCall(%s)", CodeName(op));
+
+ // Ensure vp array is rooted - we may GC in here.
+ RootedExternalValueArray vpRoot(cx, 3 + constructing, vp);
+
+ RootedValue callee(cx, vp[0]);
+ RootedValue thisv(cx, vp[1]);
+ RootedValue arr(cx, vp[2]);
+ RootedValue newTarget(cx, constructing ? vp[3] : NullValue());
+
+ // Transition stub state to megamorphic or generic if warranted.
+ MaybeTransition(cx, frame, stub);
+
+ // Try attaching a call stub.
+ bool handled = false;
+ if (op != JSOp::SpreadEval && op != JSOp::StrictSpreadEval &&
+ stub->state().canAttachStub()) {
+ // Try CacheIR first:
+ Rooted<ArrayObject*> aobj(cx, &arr.toObject().as<ArrayObject>());
+ MOZ_ASSERT(IsPackedArray(aobj));
+
+ HandleValueArray args = HandleValueArray::fromMarkedLocation(
+ aobj->length(), aobj->getDenseElements());
+ CallIRGenerator gen(cx, script, pc, op, stub->state(), 1, callee, thisv,
+ newTarget, args);
+ switch (gen.tryAttachStub()) {
+ case AttachDecision::NoAction:
+ break;
+ case AttachDecision::Attach: {
+ ICScript* icScript = frame->icScript();
+ ICAttachResult result =
+ AttachBaselineCacheIRStub(cx, gen.writerRef(), gen.cacheKind(),
+ script, icScript, stub, gen.stubName());
+
+ if (result == ICAttachResult::Attached) {
+ handled = true;
+ JitSpew(JitSpew_BaselineIC, " Attached Spread Call CacheIR stub");
+ }
+ } break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ handled = true;
+ break;
+ case AttachDecision::Deferred:
+ MOZ_ASSERT_UNREACHABLE("No deferred optimizations for spread calls");
+ break;
+ }
+ if (!handled) {
+ stub->trackNotAttached();
+ }
+ }
+
+ return SpreadCallOperation(cx, script, pc, thisv, callee, arr, newTarget,
+ res);
+}
+
+void FallbackICCodeCompiler::pushCallArguments(
+ MacroAssembler& masm, AllocatableGeneralRegisterSet regs, Register argcReg,
+ bool isConstructing) {
+ MOZ_ASSERT(!regs.has(argcReg));
+
+ // argPtr initially points to the last argument.
+ Register argPtr = regs.takeAny();
+ masm.mov(FramePointer, argPtr);
+
+ // Skip 3 pointers pushed on top of the arguments: the frame descriptor,
+ // return address, and old frame pointer.
+ size_t valueOffset = BaselineStubFrameLayout::Size();
+
+ // We have to push |this|, callee, new.target (if constructing) and argc
+ // arguments. Handle the number of Values we know statically first.
+
+ size_t numNonArgValues = 2 + isConstructing;
+ for (size_t i = 0; i < numNonArgValues; i++) {
+ masm.pushValue(Address(argPtr, valueOffset));
+ valueOffset += sizeof(Value);
+ }
+
+ // If there are no arguments we're done.
+ Label done;
+ masm.branchTest32(Assembler::Zero, argcReg, argcReg, &done);
+
+ // Push argc Values.
+ Label loop;
+ Register count = regs.takeAny();
+ masm.addPtr(Imm32(valueOffset), argPtr);
+ masm.move32(argcReg, count);
+ masm.bind(&loop);
+ {
+ masm.pushValue(Address(argPtr, 0));
+ masm.addPtr(Imm32(sizeof(Value)), argPtr);
+
+ masm.branchSub32(Assembler::NonZero, Imm32(1), count, &loop);
+ }
+ masm.bind(&done);
+}
+
+bool FallbackICCodeCompiler::emitCall(bool isSpread, bool isConstructing) {
+ static_assert(R0 == JSReturnOperand);
+
+ // Values are on the stack left-to-right. Calling convention wants them
+ // right-to-left so duplicate them on the stack in reverse order.
+ // |this| and callee are pushed last.
+
+ AllocatableGeneralRegisterSet regs = BaselineICAvailableGeneralRegs(0);
+
+ if (MOZ_UNLIKELY(isSpread)) {
+ // Push a stub frame so that we can perform a non-tail call.
+ enterStubFrame(masm, R1.scratchReg());
+
+ // Use FramePointer instead of StackPointer because it's not affected by
+ // the stack pushes below.
+
+ // newTarget
+ uint32_t valueOffset = BaselineStubFrameLayout::Size();
+ if (isConstructing) {
+ masm.pushValue(Address(FramePointer, valueOffset));
+ valueOffset += sizeof(Value);
+ }
+
+ // array
+ masm.pushValue(Address(FramePointer, valueOffset));
+ valueOffset += sizeof(Value);
+
+ // this
+ masm.pushValue(Address(FramePointer, valueOffset));
+ valueOffset += sizeof(Value);
+
+ // callee
+ masm.pushValue(Address(FramePointer, valueOffset));
+ valueOffset += sizeof(Value);
+
+ masm.push(masm.getStackPointer());
+ masm.push(ICStubReg);
+
+ PushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, Value*,
+ MutableHandleValue);
+ if (!callVM<Fn, DoSpreadCallFallback>(masm)) {
+ return false;
+ }
+
+ leaveStubFrame(masm);
+ EmitReturnFromIC(masm);
+
+ // SpreadCall is not yet supported in Ion, so do not generate asmcode for
+ // bailout.
+ return true;
+ }
+
+ // Push a stub frame so that we can perform a non-tail call.
+ enterStubFrame(masm, R1.scratchReg());
+
+ regs.take(R0.scratchReg()); // argc.
+
+ pushCallArguments(masm, regs, R0.scratchReg(), isConstructing);
+
+ masm.push(masm.getStackPointer());
+ masm.push(R0.scratchReg());
+ masm.push(ICStubReg);
+
+ PushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, uint32_t,
+ Value*, MutableHandleValue);
+ if (!callVM<Fn, DoCallFallback>(masm)) {
+ return false;
+ }
+
+ leaveStubFrame(masm);
+ EmitReturnFromIC(masm);
+
+ // This is the resume point used when bailout rewrites call stack to undo
+ // Ion inlined frames. The return address pushed onto reconstructed stack
+ // will point here.
+ assumeStubFrame();
+
+ MOZ_ASSERT(!isSpread);
+
+ if (isConstructing) {
+ code.initBailoutReturnOffset(BailoutReturnKind::New, masm.currentOffset());
+ } else {
+ code.initBailoutReturnOffset(BailoutReturnKind::Call, masm.currentOffset());
+ }
+
+ // Load passed-in ThisV into R1 just in case it's needed. Need to do this
+ // before we leave the stub frame since that info will be lost.
+ // Current stack: [...., ThisV, CalleeToken, Descriptor ]
+ size_t thisvOffset =
+ JitFrameLayout::offsetOfThis() - JitFrameLayout::bytesPoppedAfterCall();
+ masm.loadValue(Address(masm.getStackPointer(), thisvOffset), R1);
+
+ leaveStubFrame(masm);
+
+ // If this is a |constructing| call, if the callee returns a non-object, we
+ // replace it with the |this| object passed in.
+ if (isConstructing) {
+ static_assert(JSReturnOperand == R0);
+ Label skipThisReplace;
+
+ masm.branchTestObject(Assembler::Equal, JSReturnOperand, &skipThisReplace);
+ masm.moveValue(R1, R0);
+#ifdef DEBUG
+ masm.branchTestObject(Assembler::Equal, JSReturnOperand, &skipThisReplace);
+ masm.assumeUnreachable("Failed to return object in constructing call.");
+#endif
+ masm.bind(&skipThisReplace);
+ }
+
+ EmitReturnFromIC(masm);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_Call() {
+ return emitCall(/* isSpread = */ false, /* isConstructing = */ false);
+}
+
+bool FallbackICCodeCompiler::emit_CallConstructing() {
+ return emitCall(/* isSpread = */ false, /* isConstructing = */ true);
+}
+
+bool FallbackICCodeCompiler::emit_SpreadCall() {
+ return emitCall(/* isSpread = */ true, /* isConstructing = */ false);
+}
+
+bool FallbackICCodeCompiler::emit_SpreadCallConstructing() {
+ return emitCall(/* isSpread = */ true, /* isConstructing = */ true);
+}
+
+//
+// GetIterator_Fallback
+//
+
+bool DoGetIteratorFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue value,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "GetIterator");
+
+ TryAttachStub<GetIteratorIRGenerator>("GetIterator", cx, frame, stub, value);
+
+ JSObject* iterobj = ValueToIterator(cx, value);
+ if (!iterobj) {
+ return false;
+ }
+
+ res.setObject(*iterobj);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_GetIterator() {
+ EmitRestoreTailCallReg(masm);
+
+ // Sync stack for the decompiler.
+ masm.pushValue(R0);
+
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ MutableHandleValue);
+ return tailCallVM<Fn, DoGetIteratorFallback>(masm);
+}
+
+//
+// OptimizeSpreadCall_Fallback
+//
+
+bool DoOptimizeSpreadCallFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue value,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "OptimizeSpreadCall");
+
+ TryAttachStub<OptimizeSpreadCallIRGenerator>("OptimizeSpreadCall", cx, frame,
+ stub, value);
+
+ return OptimizeSpreadCall(cx, value, res);
+}
+
+bool FallbackICCodeCompiler::emit_OptimizeSpreadCall() {
+ EmitRestoreTailCallReg(masm);
+
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ MutableHandleValue);
+ return tailCallVM<Fn, DoOptimizeSpreadCallFallback>(masm);
+}
+
+//
+// InstanceOf_Fallback
+//
+
+bool DoInstanceOfFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "InstanceOf");
+
+ if (!rhs.isObject()) {
+ ReportValueError(cx, JSMSG_BAD_INSTANCEOF_RHS, -1, rhs, nullptr);
+ return false;
+ }
+
+ RootedObject obj(cx, &rhs.toObject());
+ bool cond = false;
+ if (!InstanceofOperator(cx, obj, lhs, &cond)) {
+ return false;
+ }
+
+ res.setBoolean(cond);
+
+ if (!obj->is<JSFunction>()) {
+ // ensure we've recorded at least one failure, so we can detect there was a
+ // non-optimizable case
+ if (!stub->state().hasFailures()) {
+ stub->trackNotAttached();
+ }
+ return true;
+ }
+
+ TryAttachStub<InstanceOfIRGenerator>("InstanceOf", cx, frame, stub, lhs, obj);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_InstanceOf() {
+ EmitRestoreTailCallReg(masm);
+
+ // Sync stack for the decompiler.
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+
+ masm.pushValue(R1);
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ HandleValue, MutableHandleValue);
+ return tailCallVM<Fn, DoInstanceOfFallback>(masm);
+}
+
+//
+// TypeOf_Fallback
+//
+
+bool DoTypeOfFallback(JSContext* cx, BaselineFrame* frame, ICFallbackStub* stub,
+ HandleValue val, MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "TypeOf");
+
+ TryAttachStub<TypeOfIRGenerator>("TypeOf", cx, frame, stub, val);
+
+ JSType type = js::TypeOfValue(val);
+ RootedString string(cx, TypeName(type, cx->names()));
+ res.setString(string);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_TypeOf() {
+ EmitRestoreTailCallReg(masm);
+
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ MutableHandleValue);
+ return tailCallVM<Fn, DoTypeOfFallback>(masm);
+}
+
+//
+// ToPropertyKey_Fallback
+//
+
+bool DoToPropertyKeyFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue val,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "ToPropertyKey");
+
+ TryAttachStub<ToPropertyKeyIRGenerator>("ToPropertyKey", cx, frame, stub,
+ val);
+
+ return ToPropertyKeyOperation(cx, val, res);
+}
+
+bool FallbackICCodeCompiler::emit_ToPropertyKey() {
+ EmitRestoreTailCallReg(masm);
+
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ MutableHandleValue);
+ return tailCallVM<Fn, DoToPropertyKeyFallback>(masm);
+}
+
+//
+// Rest_Fallback
+//
+
+bool DoRestFallback(JSContext* cx, BaselineFrame* frame, ICFallbackStub* stub,
+ MutableHandleValue res) {
+ unsigned numFormals = frame->numFormalArgs() - 1;
+ unsigned numActuals = frame->numActualArgs();
+ unsigned numRest = numActuals > numFormals ? numActuals - numFormals : 0;
+ Value* rest = frame->argv() + numFormals;
+
+ ArrayObject* obj = NewDenseCopiedArray(cx, numRest, rest);
+ if (!obj) {
+ return false;
+ }
+ res.setObject(*obj);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_Rest() {
+ EmitRestoreTailCallReg(masm);
+
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn =
+ bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, MutableHandleValue);
+ return tailCallVM<Fn, DoRestFallback>(masm);
+}
+
+//
+// UnaryArith_Fallback
+//
+
+bool DoUnaryArithFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue val,
+ MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+ JSOp op = JSOp(*pc);
+ FallbackICSpew(cx, stub, "UnaryArith(%s)", CodeName(op));
+
+ switch (op) {
+ case JSOp::BitNot: {
+ res.set(val);
+ if (!BitNot(cx, res, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Pos: {
+ res.set(val);
+ if (!ToNumber(cx, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Neg: {
+ res.set(val);
+ if (!NegOperation(cx, res, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Inc: {
+ if (!IncOperation(cx, val, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Dec: {
+ if (!DecOperation(cx, val, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::ToNumeric: {
+ res.set(val);
+ if (!ToNumeric(cx, res)) {
+ return false;
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected op");
+ }
+ MOZ_ASSERT(res.isNumeric());
+
+ TryAttachStub<UnaryArithIRGenerator>("UnaryArith", cx, frame, stub, op, val,
+ res);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_UnaryArith() {
+ static_assert(R0 == JSReturnOperand);
+
+ // Restore the tail call register.
+ EmitRestoreTailCallReg(masm);
+
+ // Ensure stack is fully synced for the expression decompiler.
+ masm.pushValue(R0);
+
+ // Push arguments.
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ MutableHandleValue);
+ return tailCallVM<Fn, DoUnaryArithFallback>(masm);
+}
+
+//
+// BinaryArith_Fallback
+//
+
+bool DoBinaryArithFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue ret) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+ JSOp op = JSOp(*pc);
+ FallbackICSpew(
+ cx, stub, "CacheIRBinaryArith(%s,%d,%d)", CodeName(op),
+ int(lhs.isDouble() ? JSVAL_TYPE_DOUBLE : lhs.extractNonDoubleType()),
+ int(rhs.isDouble() ? JSVAL_TYPE_DOUBLE : rhs.extractNonDoubleType()));
+
+ // Don't pass lhs/rhs directly, we need the original values when
+ // generating stubs.
+ RootedValue lhsCopy(cx, lhs);
+ RootedValue rhsCopy(cx, rhs);
+
+ // Perform the arith operation.
+ switch (op) {
+ case JSOp::Add:
+ // Do an add.
+ if (!AddValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Sub:
+ if (!SubValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Mul:
+ if (!MulValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Div:
+ if (!DivValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Mod:
+ if (!ModValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Pow:
+ if (!PowValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::BitOr: {
+ if (!BitOr(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::BitXor: {
+ if (!BitXor(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::BitAnd: {
+ if (!BitAnd(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Lsh: {
+ if (!BitLsh(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Rsh: {
+ if (!BitRsh(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Ursh: {
+ if (!UrshValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Unhandled baseline arith op");
+ }
+
+ TryAttachStub<BinaryArithIRGenerator>("BinaryArith", cx, frame, stub, op, lhs,
+ rhs, ret);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_BinaryArith() {
+ static_assert(R0 == JSReturnOperand);
+
+ // Restore the tail call register.
+ EmitRestoreTailCallReg(masm);
+
+ // Ensure stack is fully synced for the expression decompiler.
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+
+ // Push arguments.
+ masm.pushValue(R1);
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ HandleValue, MutableHandleValue);
+ return tailCallVM<Fn, DoBinaryArithFallback>(masm);
+}
+
+//
+// Compare_Fallback
+//
+bool DoCompareFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs, HandleValue rhs,
+ MutableHandleValue ret) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+ JSOp op = JSOp(*pc);
+
+ FallbackICSpew(cx, stub, "Compare(%s)", CodeName(op));
+
+ // Don't pass lhs/rhs directly, we need the original values when
+ // generating stubs.
+ RootedValue lhsCopy(cx, lhs);
+ RootedValue rhsCopy(cx, rhs);
+
+ // Perform the compare operation.
+ bool out;
+ switch (op) {
+ case JSOp::Lt:
+ if (!LessThan(cx, &lhsCopy, &rhsCopy, &out)) {
+ return false;
+ }
+ break;
+ case JSOp::Le:
+ if (!LessThanOrEqual(cx, &lhsCopy, &rhsCopy, &out)) {
+ return false;
+ }
+ break;
+ case JSOp::Gt:
+ if (!GreaterThan(cx, &lhsCopy, &rhsCopy, &out)) {
+ return false;
+ }
+ break;
+ case JSOp::Ge:
+ if (!GreaterThanOrEqual(cx, &lhsCopy, &rhsCopy, &out)) {
+ return false;
+ }
+ break;
+ case JSOp::Eq:
+ if (!js::LooselyEqual(cx, lhsCopy, rhsCopy, &out)) {
+ return false;
+ }
+ break;
+ case JSOp::Ne:
+ if (!js::LooselyEqual(cx, lhsCopy, rhsCopy, &out)) {
+ return false;
+ }
+ out = !out;
+ break;
+ case JSOp::StrictEq:
+ if (!js::StrictlyEqual(cx, lhsCopy, rhsCopy, &out)) {
+ return false;
+ }
+ break;
+ case JSOp::StrictNe:
+ if (!js::StrictlyEqual(cx, lhsCopy, rhsCopy, &out)) {
+ return false;
+ }
+ out = !out;
+ break;
+ default:
+ MOZ_ASSERT_UNREACHABLE("Unhandled baseline compare op");
+ return false;
+ }
+
+ ret.setBoolean(out);
+
+ TryAttachStub<CompareIRGenerator>("Compare", cx, frame, stub, op, lhs, rhs);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_Compare() {
+ static_assert(R0 == JSReturnOperand);
+
+ // Restore the tail call register.
+ EmitRestoreTailCallReg(masm);
+
+ // Ensure stack is fully synced for the expression decompiler.
+ masm.pushValue(R0);
+ masm.pushValue(R1);
+
+ // Push arguments.
+ masm.pushValue(R1);
+ masm.pushValue(R0);
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn = bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleValue,
+ HandleValue, MutableHandleValue);
+ return tailCallVM<Fn, DoCompareFallback>(masm);
+}
+
+//
+// NewArray_Fallback
+//
+
+bool DoNewArrayFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "NewArray");
+
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+
+ uint32_t length = GET_UINT32(pc);
+ MOZ_ASSERT(length <= INT32_MAX,
+ "the bytecode emitter must fail to compile code that would "
+ "produce a length exceeding int32_t range");
+
+ Rooted<ArrayObject*> array(cx, NewArrayOperation(cx, length));
+ if (!array) {
+ return false;
+ }
+
+ TryAttachStub<NewArrayIRGenerator>("NewArray", cx, frame, stub, JSOp(*pc),
+ array, frame);
+
+ res.setObject(*array);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_NewArray() {
+ EmitRestoreTailCallReg(masm);
+
+ masm.push(ICStubReg); // stub.
+ masm.pushBaselineFramePtr(FramePointer, R0.scratchReg());
+
+ using Fn =
+ bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, MutableHandleValue);
+ return tailCallVM<Fn, DoNewArrayFallback>(masm);
+}
+
+//
+// NewObject_Fallback
+//
+bool DoNewObjectFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, MutableHandleValue res) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "NewObject");
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = StubOffsetToPc(stub, script);
+
+ RootedObject obj(cx, NewObjectOperation(cx, script, pc));
+ if (!obj) {
+ return false;
+ }
+
+ TryAttachStub<NewObjectIRGenerator>("NewObject", cx, frame, stub, JSOp(*pc),
+ obj, frame);
+
+ res.setObject(*obj);
+ return true;
+}
+
+bool FallbackICCodeCompiler::emit_NewObject() {
+ EmitRestoreTailCallReg(masm);
+
+ masm.push(ICStubReg); // stub.
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn =
+ bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, MutableHandleValue);
+ return tailCallVM<Fn, DoNewObjectFallback>(masm);
+}
+
+//
+// CloseIter_Fallback
+//
+
+bool DoCloseIterFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleObject iter) {
+ stub->incrementEnteredCount();
+ MaybeNotifyWarp(frame->outerScript(), stub);
+ FallbackICSpew(cx, stub, "CloseIter");
+
+ jsbytecode* pc = StubOffsetToPc(stub, frame->script());
+ CompletionKind kind = CompletionKind(GET_UINT8(pc));
+
+ TryAttachStub<CloseIterIRGenerator>("CloseIter", cx, frame, stub, iter, kind);
+
+ return CloseIterOperation(cx, iter, kind);
+}
+
+bool FallbackICCodeCompiler::emit_CloseIter() {
+ EmitRestoreTailCallReg(masm);
+
+ masm.push(R0.scratchReg());
+ masm.push(ICStubReg);
+ pushStubPayload(masm, R0.scratchReg());
+
+ using Fn =
+ bool (*)(JSContext*, BaselineFrame*, ICFallbackStub*, HandleObject);
+ return tailCallVM<Fn, DoCloseIterFallback>(masm);
+}
+
+bool JitRuntime::generateBaselineICFallbackCode(JSContext* cx) {
+ TempAllocator temp(&cx->tempLifoAlloc());
+ StackMacroAssembler masm(cx, temp);
+ AutoCreatedBy acb(masm, "JitRuntime::generateBaselineICFallbackCode");
+
+ BaselineICFallbackCode& fallbackCode = baselineICFallbackCode_.ref();
+ FallbackICCodeCompiler compiler(cx, fallbackCode, masm);
+
+ JitSpew(JitSpew_Codegen, "# Emitting Baseline IC fallback code");
+
+#define EMIT_CODE(kind) \
+ { \
+ AutoCreatedBy acb(masm, "kind=" #kind); \
+ uint32_t offset = startTrampolineCode(masm); \
+ InitMacroAssemblerForICStub(masm); \
+ if (!compiler.emit_##kind()) { \
+ return false; \
+ } \
+ fallbackCode.initOffset(BaselineICFallbackKind::kind, offset); \
+ }
+ IC_BASELINE_FALLBACK_CODE_KIND_LIST(EMIT_CODE)
+#undef EMIT_CODE
+
+ Linker linker(masm);
+ JitCode* code = linker.newCode(cx, CodeKind::Other);
+ if (!code) {
+ return false;
+ }
+
+ CollectPerfSpewerJitCodeProfile(code, "BaselineICFallback");
+
+#ifdef MOZ_VTUNE
+ vtune::MarkStub(code, "BaselineICFallback");
+#endif
+
+ fallbackCode.initCode(code);
+ return true;
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/BaselineIC.h b/js/src/jit/BaselineIC.h
new file mode 100644
index 0000000000..9098af4825
--- /dev/null
+++ b/js/src/jit/BaselineIC.h
@@ -0,0 +1,439 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineIC_h
+#define jit_BaselineIC_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/ICState.h"
+#include "jit/JitCode.h"
+#include "jit/shared/Assembler-shared.h"
+#include "jit/TypeData.h"
+#include "js/TypeDecls.h"
+
+class JS_PUBLIC_API JSTracer;
+
+enum class JSOp : uint8_t;
+
+namespace js {
+
+MOZ_COLD void ReportOutOfMemory(JSContext* cx);
+
+namespace jit {
+
+class BaselineFrame;
+class CacheIRStubInfo;
+class ICScript;
+
+enum class TailCallVMFunctionId;
+enum class VMFunctionId;
+
+// [SMDOC] JIT Inline Caches (ICs)
+//
+// Baseline Inline Caches are polymorphic caches that aggressively
+// share their stub code.
+//
+// Every polymorphic site contains a linked list of stubs which are
+// specific to that site. These stubs are composed of a |StubData|
+// structure that stores parametrization information (e.g.
+// the shape pointer for a shape-check-and-property-get stub), any
+// dynamic information (e.g. warm-up counters), a pointer to the stub code,
+// and a pointer to the next stub state in the linked list.
+//
+// Every BaselineScript keeps an table of |CacheDescriptor| data
+// structures, which store the following:
+// A pointer to the first StubData in the cache.
+// The bytecode PC of the relevant IC.
+// The machine-code PC where the call to the stubcode returns.
+//
+// A diagram:
+//
+// Control flow Pointers
+// =======# ----. .---->
+// # | |
+// #======> \-----/
+//
+//
+// .---------------------------------------.
+// | .-------------------------. |
+// | | .----. | |
+// Baseline | | | | | |
+// JIT Code 0 ^ 1 ^ 2 ^ | | |
+// +--------------+ .-->+-----+ +-----+ +-----+ | | |
+// | | #=|==>| |==>| |==>| FB | | | |
+// | | # | +-----+ +-----+ +-----+ | | |
+// | | # | # # # | | |
+// |==============|==# | # # # | | |
+// |=== IC =======| | # # # | | |
+// .->|==============|<===|======#=========#=========# | | |
+// | | | | | | |
+// | | | | | | |
+// | | | | | | |
+// | | | | v | |
+// | | | | +---------+ | |
+// | | | | | Fallback| | |
+// | | | | | Stub | | |
+// | | | | | Code | | |
+// | | | | +---------+ | |
+// | +--------------+ | | |
+// | |_______ | +---------+ | |
+// | | | | Stub |<---/ |
+// | IC | \--. | Code | |
+// | Descriptor | | +---------+ |
+// | Table v | |
+// | +-----------------+ | +---------+ |
+// \--| Ins | PC | Stub |----/ | Stub |<-------/
+// +-----------------+ | Code |
+// | ... | +---------+
+// +-----------------+
+// Shared
+// Stub Code
+//
+
+class ICStub;
+class ICCacheIRStub;
+class ICFallbackStub;
+
+#ifdef JS_JITSPEW
+void FallbackICSpew(JSContext* cx, ICFallbackStub* stub, const char* fmt, ...)
+ MOZ_FORMAT_PRINTF(3, 4);
+#else
+# define FallbackICSpew(...)
+#endif
+
+// An entry in the ICScript IC table. There's one ICEntry per IC.
+class ICEntry {
+ // A pointer to the first IC stub for this instruction.
+ ICStub* firstStub_;
+
+ public:
+ explicit ICEntry(ICStub* firstStub) : firstStub_(firstStub) {}
+
+ ICStub* firstStub() const {
+ MOZ_ASSERT(firstStub_);
+ return firstStub_;
+ }
+
+ void setFirstStub(ICStub* stub) { firstStub_ = stub; }
+
+ static constexpr size_t offsetOfFirstStub() {
+ return offsetof(ICEntry, firstStub_);
+ }
+
+ void trace(JSTracer* trc);
+};
+
+//
+// Base class for all IC stubs.
+//
+class ICStub {
+ friend class ICFallbackStub;
+
+ protected:
+ // The raw jitcode to call for this stub.
+ uint8_t* stubCode_;
+
+ // Counts the number of times the stub was entered
+ //
+ // See Bug 1494473 comment 6 for a mechanism to handle overflow if overflow
+ // becomes a concern.
+ uint32_t enteredCount_ = 0;
+
+ // Tracks input types for some CacheIR stubs, to help optimize
+ // polymorphic cases. Stored in the base class to make use of
+ // padding bytes.
+ TypeData typeData_;
+
+ // Whether this is an ICFallbackStub or an ICCacheIRStub.
+ bool isFallback_;
+
+ ICStub(uint8_t* stubCode, bool isFallback)
+ : stubCode_(stubCode), isFallback_(isFallback) {
+ MOZ_ASSERT(stubCode != nullptr);
+ }
+
+ public:
+ inline bool isFallback() const { return isFallback_; }
+
+ inline ICStub* maybeNext() const;
+
+ inline const ICFallbackStub* toFallbackStub() const {
+ MOZ_ASSERT(isFallback());
+ return reinterpret_cast<const ICFallbackStub*>(this);
+ }
+
+ inline ICFallbackStub* toFallbackStub() {
+ MOZ_ASSERT(isFallback());
+ return reinterpret_cast<ICFallbackStub*>(this);
+ }
+
+ ICCacheIRStub* toCacheIRStub() {
+ MOZ_ASSERT(!isFallback());
+ return reinterpret_cast<ICCacheIRStub*>(this);
+ }
+ const ICCacheIRStub* toCacheIRStub() const {
+ MOZ_ASSERT(!isFallback());
+ return reinterpret_cast<const ICCacheIRStub*>(this);
+ }
+
+ bool usesTrampolineCode() const {
+ // All fallback code is stored in a single JitCode instance, so we can't
+ // call JitCode::FromExecutable on the raw pointer.
+ return isFallback();
+ }
+ JitCode* jitCode() {
+ MOZ_ASSERT(!usesTrampolineCode());
+ return JitCode::FromExecutable(stubCode_);
+ }
+
+ uint32_t enteredCount() const { return enteredCount_; }
+ inline void incrementEnteredCount() { enteredCount_++; }
+ void resetEnteredCount() { enteredCount_ = 0; }
+
+ static constexpr size_t offsetOfStubCode() {
+ return offsetof(ICStub, stubCode_);
+ }
+ static constexpr size_t offsetOfEnteredCount() {
+ return offsetof(ICStub, enteredCount_);
+ }
+};
+
+class ICFallbackStub final : public ICStub {
+ friend class ICStubConstIterator;
+
+ protected:
+ // The PC offset of this IC's bytecode op within the JSScript.
+ uint32_t pcOffset_;
+
+ // The state of this IC.
+ ICState state_{};
+
+ public:
+ explicit ICFallbackStub(uint32_t pcOffset, TrampolinePtr stubCode)
+ : ICStub(stubCode.value, /* isFallback = */ true), pcOffset_(pcOffset) {}
+
+ inline size_t numOptimizedStubs() const { return state_.numOptimizedStubs(); }
+
+ bool newStubIsFirstStub() const { return state_.newStubIsFirstStub(); }
+
+ ICState& state() { return state_; }
+
+ uint32_t pcOffset() const { return pcOffset_; }
+
+ // Add a new stub to the IC chain terminated by this fallback stub.
+ inline void addNewStub(ICEntry* icEntry, ICCacheIRStub* stub);
+
+ void discardStubs(JSContext* cx, ICEntry* icEntry);
+
+ void clearUsedByTranspiler() { state_.clearUsedByTranspiler(); }
+ void setUsedByTranspiler() { state_.setUsedByTranspiler(); }
+ bool usedByTranspiler() const { return state_.usedByTranspiler(); }
+
+ void clearHasFoldedStub() { state_.clearHasFoldedStub(); }
+ void setHasFoldedStub() { state_.setHasFoldedStub(); }
+ bool hasFoldedStub() const { return state_.hasFoldedStub(); }
+
+ TrialInliningState trialInliningState() const {
+ return state_.trialInliningState();
+ }
+ void setTrialInliningState(TrialInliningState state) {
+ state_.setTrialInliningState(state);
+ }
+
+ void trackNotAttached();
+
+ void unlinkStub(Zone* zone, ICEntry* icEntry, ICCacheIRStub* prev,
+ ICCacheIRStub* stub);
+};
+
+class ICCacheIRStub final : public ICStub {
+ // Pointer to next IC stub.
+ ICStub* next_ = nullptr;
+
+ const CacheIRStubInfo* stubInfo_;
+
+#ifndef JS_64BIT
+ // Ensure stub data is 8-byte aligned on 32-bit.
+ uintptr_t padding_ = 0;
+#endif
+
+ public:
+ ICCacheIRStub(JitCode* stubCode, const CacheIRStubInfo* stubInfo)
+ : ICStub(stubCode->raw(), /* isFallback = */ false),
+ stubInfo_(stubInfo) {}
+
+ ICStub* next() const { return next_; }
+ void setNext(ICStub* stub) { next_ = stub; }
+
+ ICCacheIRStub* nextCacheIR() const {
+ return next_->isFallback() ? nullptr : next_->toCacheIRStub();
+ }
+
+ const CacheIRStubInfo* stubInfo() const { return stubInfo_; }
+ uint8_t* stubDataStart();
+
+ void trace(JSTracer* trc);
+
+ // Optimized stubs get purged on GC. But some stubs can be active on the
+ // stack during GC - specifically the ones that can make calls. To ensure
+ // that these do not get purged, all stubs that can make calls are allocated
+ // in the fallback stub space.
+ bool makesGCCalls() const;
+ bool allocatedInFallbackSpace() const { return makesGCCalls(); }
+
+ static constexpr size_t offsetOfNext() {
+ return offsetof(ICCacheIRStub, next_);
+ }
+
+ void setTypeData(TypeData data) { typeData_ = data; }
+ TypeData typeData() const { return typeData_; }
+};
+
+// Assert stub size is what we expect to catch regressions.
+#ifdef JS_64BIT
+static_assert(sizeof(ICFallbackStub) == 3 * sizeof(uintptr_t));
+static_assert(sizeof(ICCacheIRStub) == 4 * sizeof(uintptr_t));
+#else
+static_assert(sizeof(ICFallbackStub) == 5 * sizeof(uintptr_t));
+static_assert(sizeof(ICCacheIRStub) == 6 * sizeof(uintptr_t));
+#endif
+
+inline ICStub* ICStub::maybeNext() const {
+ return isFallback() ? nullptr : toCacheIRStub()->next();
+}
+
+inline void ICFallbackStub::addNewStub(ICEntry* icEntry, ICCacheIRStub* stub) {
+ MOZ_ASSERT(stub->next() == nullptr);
+ stub->setNext(icEntry->firstStub());
+ icEntry->setFirstStub(stub);
+ state_.trackAttached();
+}
+
+AllocatableGeneralRegisterSet BaselineICAvailableGeneralRegs(size_t numInputs);
+
+bool ICSupportsPolymorphicTypeData(JSOp op);
+
+struct IonOsrTempData;
+
+extern bool DoCallFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, uint32_t argc, Value* vp,
+ MutableHandleValue res);
+
+extern bool DoSpreadCallFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, Value* vp,
+ MutableHandleValue res);
+
+extern bool DoToBoolFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue arg,
+ MutableHandleValue ret);
+
+extern bool DoGetElemSuperFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs,
+ HandleValue rhs, HandleValue receiver,
+ MutableHandleValue res);
+
+extern bool DoGetElemFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue res);
+
+extern bool DoSetElemFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, Value* stack,
+ HandleValue objv, HandleValue index,
+ HandleValue rhs);
+
+extern bool DoInFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue key,
+ HandleValue objValue, MutableHandleValue res);
+
+extern bool DoHasOwnFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue keyValue,
+ HandleValue objValue, MutableHandleValue res);
+
+extern bool DoCheckPrivateFieldFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub,
+ HandleValue objValue,
+ HandleValue keyValue,
+ MutableHandleValue res);
+
+extern bool DoGetNameFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleObject envChain,
+ MutableHandleValue res);
+
+extern bool DoBindNameFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleObject envChain,
+ MutableHandleValue res);
+
+extern bool DoGetIntrinsicFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub,
+ MutableHandleValue res);
+
+extern bool DoGetPropFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, MutableHandleValue val,
+ MutableHandleValue res);
+
+extern bool DoGetPropSuperFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue receiver,
+ MutableHandleValue val,
+ MutableHandleValue res);
+
+extern bool DoSetPropFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, Value* stack,
+ HandleValue lhs, HandleValue rhs);
+
+extern bool DoGetIteratorFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue value,
+ MutableHandleValue res);
+
+extern bool DoOptimizeSpreadCallFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub,
+ HandleValue value,
+ MutableHandleValue res);
+
+extern bool DoInstanceOfFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue res);
+
+extern bool DoTypeOfFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue val,
+ MutableHandleValue res);
+
+extern bool DoToPropertyKeyFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue val,
+ MutableHandleValue res);
+
+extern bool DoRestFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, MutableHandleValue res);
+
+extern bool DoUnaryArithFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue val,
+ MutableHandleValue res);
+
+extern bool DoBinaryArithFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue ret);
+
+extern bool DoNewArrayFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, MutableHandleValue res);
+
+extern bool DoNewObjectFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, MutableHandleValue res);
+
+extern bool DoCompareFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue ret);
+
+extern bool DoCloseIterFallback(JSContext* cx, BaselineFrame* frame,
+ ICFallbackStub* stub, HandleObject iter);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BaselineIC_h */
diff --git a/js/src/jit/BaselineICList.h b/js/src/jit/BaselineICList.h
new file mode 100644
index 0000000000..f277b6acdb
--- /dev/null
+++ b/js/src/jit/BaselineICList.h
@@ -0,0 +1,49 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineICList_h
+#define jit_BaselineICList_h
+
+namespace js {
+namespace jit {
+
+// List of trampolines for Baseline IC fallback stubs. Trampoline code is
+// allocated as part of the JitRuntime.
+#define IC_BASELINE_FALLBACK_CODE_KIND_LIST(_) \
+ _(NewArray) \
+ _(NewObject) \
+ _(ToBool) \
+ _(UnaryArith) \
+ _(Call) \
+ _(CallConstructing) \
+ _(SpreadCall) \
+ _(SpreadCallConstructing) \
+ _(GetElem) \
+ _(GetElemSuper) \
+ _(SetElem) \
+ _(In) \
+ _(HasOwn) \
+ _(CheckPrivateField) \
+ _(GetName) \
+ _(BindName) \
+ _(GetIntrinsic) \
+ _(SetProp) \
+ _(GetIterator) \
+ _(OptimizeSpreadCall) \
+ _(InstanceOf) \
+ _(TypeOf) \
+ _(ToPropertyKey) \
+ _(Rest) \
+ _(BinaryArith) \
+ _(Compare) \
+ _(GetProp) \
+ _(GetPropSuper) \
+ _(CloseIter)
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BaselineICList_h */
diff --git a/js/src/jit/BaselineJIT.cpp b/js/src/jit/BaselineJIT.cpp
new file mode 100644
index 0000000000..6259414842
--- /dev/null
+++ b/js/src/jit/BaselineJIT.cpp
@@ -0,0 +1,956 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BaselineJIT.h"
+
+#include "mozilla/BinarySearch.h"
+#include "mozilla/CheckedInt.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MemoryReporting.h"
+
+#include <algorithm>
+
+#include "debugger/DebugAPI.h"
+#include "gc/GCContext.h"
+#include "gc/PublicIterators.h"
+#include "jit/AutoWritableJitCode.h"
+#include "jit/BaselineCodeGen.h"
+#include "jit/BaselineIC.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitCommon.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/MacroAssembler.h"
+#include "js/friend/StackLimits.h" // js::AutoCheckRecursionLimit
+#include "vm/Interpreter.h"
+
+#include "debugger/DebugAPI-inl.h"
+#include "gc/GC-inl.h"
+#include "jit/JitScript-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+#include "vm/JSScript-inl.h"
+#include "vm/Stack-inl.h"
+
+using mozilla::BinarySearchIf;
+using mozilla::CheckedInt;
+using mozilla::DebugOnly;
+
+using namespace js;
+using namespace js::jit;
+
+void ICStubSpace::freeAllAfterMinorGC(Zone* zone) {
+ if (zone->isAtomsZone()) {
+ MOZ_ASSERT(allocator_.isEmpty());
+ } else {
+ JSRuntime* rt = zone->runtimeFromMainThread();
+ rt->gc.queueAllLifoBlocksForFreeAfterMinorGC(&allocator_);
+ }
+}
+
+static bool CheckFrame(InterpreterFrame* fp) {
+ if (fp->isDebuggerEvalFrame()) {
+ // Debugger eval-in-frame. These are likely short-running scripts so
+ // don't bother compiling them for now.
+ JitSpew(JitSpew_BaselineAbort, "debugger frame");
+ return false;
+ }
+
+ if (fp->isFunctionFrame() && fp->numActualArgs() > BASELINE_MAX_ARGS_LENGTH) {
+ // Fall back to the interpreter to avoid running out of stack space.
+ JitSpew(JitSpew_BaselineAbort, "Too many arguments (%u)",
+ fp->numActualArgs());
+ return false;
+ }
+
+ return true;
+}
+
+struct EnterJitData {
+ explicit EnterJitData(JSContext* cx)
+ : jitcode(nullptr),
+ osrFrame(nullptr),
+ calleeToken(nullptr),
+ maxArgv(nullptr),
+ maxArgc(0),
+ numActualArgs(0),
+ osrNumStackValues(0),
+ envChain(cx),
+ result(cx),
+ constructing(false) {}
+
+ uint8_t* jitcode;
+ InterpreterFrame* osrFrame;
+
+ void* calleeToken;
+
+ Value* maxArgv;
+ unsigned maxArgc;
+ unsigned numActualArgs;
+ unsigned osrNumStackValues;
+
+ RootedObject envChain;
+ RootedValue result;
+
+ bool constructing;
+};
+
+static JitExecStatus EnterBaseline(JSContext* cx, EnterJitData& data) {
+ MOZ_ASSERT(data.osrFrame);
+
+ // Check for potential stack overflow before OSR-ing.
+ uint32_t extra =
+ BaselineFrame::Size() + (data.osrNumStackValues * sizeof(Value));
+ AutoCheckRecursionLimit recursion(cx);
+ if (!recursion.checkWithExtra(cx, extra)) {
+ return JitExec_Aborted;
+ }
+
+#ifdef DEBUG
+ // Assert we don't GC before entering JIT code. A GC could discard JIT code
+ // or move the function stored in the CalleeToken (it won't be traced at
+ // this point). We use Maybe<> here so we can call reset() to call the
+ // AutoAssertNoGC destructor before we enter JIT code.
+ mozilla::Maybe<JS::AutoAssertNoGC> nogc;
+ nogc.emplace(cx);
+#endif
+
+ MOZ_ASSERT(IsBaselineInterpreterEnabled());
+ MOZ_ASSERT(CheckFrame(data.osrFrame));
+
+ EnterJitCode enter = cx->runtime()->jitRuntime()->enterJit();
+
+ // Caller must construct |this| before invoking the function.
+ MOZ_ASSERT_IF(data.constructing,
+ data.maxArgv[0].isObject() ||
+ data.maxArgv[0].isMagic(JS_UNINITIALIZED_LEXICAL));
+
+ data.result.setInt32(data.numActualArgs);
+ {
+ AssertRealmUnchanged aru(cx);
+ ActivationEntryMonitor entryMonitor(cx, data.calleeToken);
+ JitActivation activation(cx);
+
+ data.osrFrame->setRunningInJit();
+
+#ifdef DEBUG
+ nogc.reset();
+#endif
+ // Single transition point from Interpreter to Baseline.
+ CALL_GENERATED_CODE(enter, data.jitcode, data.maxArgc, data.maxArgv,
+ data.osrFrame, data.calleeToken, data.envChain.get(),
+ data.osrNumStackValues, data.result.address());
+
+ data.osrFrame->clearRunningInJit();
+ }
+
+ // Jit callers wrap primitive constructor return, except for derived
+ // class constructors, which are forced to do it themselves.
+ if (!data.result.isMagic() && data.constructing &&
+ data.result.isPrimitive()) {
+ MOZ_ASSERT(data.maxArgv[0].isObject());
+ data.result = data.maxArgv[0];
+ }
+
+ // Release temporary buffer used for OSR into Ion.
+ cx->runtime()->jitRuntime()->freeIonOsrTempData();
+
+ MOZ_ASSERT_IF(data.result.isMagic(), data.result.isMagic(JS_ION_ERROR));
+ return data.result.isMagic() ? JitExec_Error : JitExec_Ok;
+}
+
+JitExecStatus jit::EnterBaselineInterpreterAtBranch(JSContext* cx,
+ InterpreterFrame* fp,
+ jsbytecode* pc) {
+ MOZ_ASSERT(JSOp(*pc) == JSOp::LoopHead);
+
+ EnterJitData data(cx);
+
+ // Use the entry point that skips the debug trap because the C++ interpreter
+ // already handled this for the current op.
+ const BaselineInterpreter& interp =
+ cx->runtime()->jitRuntime()->baselineInterpreter();
+ data.jitcode = interp.interpretOpNoDebugTrapAddr().value;
+
+ data.osrFrame = fp;
+ data.osrNumStackValues =
+ fp->script()->nfixed() + cx->interpreterRegs().stackDepth();
+
+ if (fp->isFunctionFrame()) {
+ data.constructing = fp->isConstructing();
+ data.numActualArgs = fp->numActualArgs();
+ data.maxArgc = std::max(fp->numActualArgs(), fp->numFormalArgs()) +
+ 1; // +1 = include |this|
+ data.maxArgv = fp->argv() - 1; // -1 = include |this|
+ data.envChain = nullptr;
+ data.calleeToken = CalleeToToken(&fp->callee(), data.constructing);
+ } else {
+ data.constructing = false;
+ data.numActualArgs = 0;
+ data.maxArgc = 0;
+ data.maxArgv = nullptr;
+ data.envChain = fp->environmentChain();
+ data.calleeToken = CalleeToToken(fp->script());
+ }
+
+ JitExecStatus status = EnterBaseline(cx, data);
+ if (status != JitExec_Ok) {
+ return status;
+ }
+
+ fp->setReturnValue(data.result);
+ return JitExec_Ok;
+}
+
+MethodStatus jit::BaselineCompile(JSContext* cx, JSScript* script,
+ bool forceDebugInstrumentation) {
+ cx->check(script);
+ MOZ_ASSERT(!script->hasBaselineScript());
+ MOZ_ASSERT(script->canBaselineCompile());
+ MOZ_ASSERT(IsBaselineJitEnabled(cx));
+ AutoGeckoProfilerEntry pseudoFrame(
+ cx, "Baseline script compilation",
+ JS::ProfilingCategoryPair::JS_BaselineCompilation);
+
+ TempAllocator temp(&cx->tempLifoAlloc());
+ JitContext jctx(cx);
+
+ BaselineCompiler compiler(cx, temp, script);
+ if (!compiler.init()) {
+ ReportOutOfMemory(cx);
+ return Method_Error;
+ }
+
+ if (forceDebugInstrumentation) {
+ compiler.setCompileDebugInstrumentation();
+ }
+
+ MethodStatus status = compiler.compile();
+
+ MOZ_ASSERT_IF(status == Method_Compiled, script->hasBaselineScript());
+ MOZ_ASSERT_IF(status != Method_Compiled, !script->hasBaselineScript());
+
+ if (status == Method_CantCompile) {
+ script->disableBaselineCompile();
+ }
+
+ return status;
+}
+
+static MethodStatus CanEnterBaselineJIT(JSContext* cx, HandleScript script,
+ AbstractFramePtr osrSourceFrame) {
+ // Skip if the script has been disabled.
+ if (!script->canBaselineCompile()) {
+ return Method_Skipped;
+ }
+
+ if (!IsBaselineJitEnabled(cx)) {
+ script->disableBaselineCompile();
+ return Method_CantCompile;
+ }
+
+ // This check is needed in the following corner case. Consider a function h,
+ //
+ // function h(x) {
+ // if (!x)
+ // return;
+ // h(false);
+ // for (var i = 0; i < N; i++)
+ // /* do stuff */
+ // }
+ //
+ // Suppose h is not yet compiled in baseline and is executing in the
+ // interpreter. Let this interpreter frame be f_older. The debugger marks
+ // f_older as isDebuggee. At the point of the recursive call h(false), h is
+ // compiled in baseline without debug instrumentation, pushing a baseline
+ // frame f_newer. The debugger never flags f_newer as isDebuggee, and never
+ // recompiles h. When the recursive call returns and execution proceeds to
+ // the loop, the interpreter attempts to OSR into baseline. Since h is
+ // already compiled in baseline, execution jumps directly into baseline
+ // code. This is incorrect as h's baseline script does not have debug
+ // instrumentation.
+ if (osrSourceFrame && osrSourceFrame.isDebuggee() &&
+ !DebugAPI::ensureExecutionObservabilityOfOsrFrame(cx, osrSourceFrame)) {
+ return Method_Error;
+ }
+
+ if (script->length() > BaselineMaxScriptLength) {
+ script->disableBaselineCompile();
+ return Method_CantCompile;
+ }
+
+ if (script->nslots() > BaselineMaxScriptSlots) {
+ script->disableBaselineCompile();
+ return Method_CantCompile;
+ }
+
+ if (script->hasBaselineScript()) {
+ return Method_Compiled;
+ }
+
+ // Check script warm-up counter.
+ if (script->getWarmUpCount() <= JitOptions.baselineJitWarmUpThreshold) {
+ return Method_Skipped;
+ }
+
+ // Check this before calling ensureJitRealmExists, so we're less
+ // likely to report OOM in JSRuntime::createJitRuntime.
+ if (!CanLikelyAllocateMoreExecutableMemory()) {
+ return Method_Skipped;
+ }
+
+ if (!cx->realm()->ensureJitRealmExists(cx)) {
+ return Method_Error;
+ }
+
+ if (script->hasForceInterpreterOp()) {
+ script->disableBaselineCompile();
+ return Method_CantCompile;
+ }
+
+ // Frames can be marked as debuggee frames independently of its underlying
+ // script being a debuggee script, e.g., when performing
+ // Debugger.Frame.prototype.eval.
+ bool forceDebugInstrumentation =
+ osrSourceFrame && osrSourceFrame.isDebuggee();
+ return BaselineCompile(cx, script, forceDebugInstrumentation);
+}
+
+bool jit::CanBaselineInterpretScript(JSScript* script) {
+ MOZ_ASSERT(IsBaselineInterpreterEnabled());
+
+ if (script->hasForceInterpreterOp()) {
+ return false;
+ }
+
+ if (script->nslots() > BaselineMaxScriptSlots) {
+ // Avoid overrecursion exceptions when the script has a ton of stack slots
+ // by forcing such scripts to run in the C++ interpreter with heap-allocated
+ // stack frames.
+ return false;
+ }
+
+ return true;
+}
+
+static MethodStatus CanEnterBaselineInterpreter(JSContext* cx,
+ JSScript* script) {
+ MOZ_ASSERT(IsBaselineInterpreterEnabled());
+
+ if (script->hasJitScript()) {
+ return Method_Compiled;
+ }
+
+ if (!CanBaselineInterpretScript(script)) {
+ return Method_CantCompile;
+ }
+
+ // Check script warm-up counter.
+ if (script->getWarmUpCount() <=
+ JitOptions.baselineInterpreterWarmUpThreshold) {
+ return Method_Skipped;
+ }
+
+ if (!cx->realm()->ensureJitRealmExists(cx)) {
+ return Method_Error;
+ }
+
+ AutoKeepJitScripts keepJitScript(cx);
+ if (!script->ensureHasJitScript(cx, keepJitScript)) {
+ return Method_Error;
+ }
+
+ return Method_Compiled;
+}
+
+MethodStatus jit::CanEnterBaselineInterpreterAtBranch(JSContext* cx,
+ InterpreterFrame* fp) {
+ if (!CheckFrame(fp)) {
+ return Method_CantCompile;
+ }
+
+ // JITs do not respect the debugger's OnNativeCall hook, so JIT execution is
+ // disabled if this hook might need to be called.
+ if (cx->insideDebuggerEvaluationWithOnNativeCallHook) {
+ return Method_CantCompile;
+ }
+
+ return CanEnterBaselineInterpreter(cx, fp->script());
+}
+
+template <BaselineTier Tier>
+MethodStatus jit::CanEnterBaselineMethod(JSContext* cx, RunState& state) {
+ if (state.isInvoke()) {
+ InvokeState& invoke = *state.asInvoke();
+ if (invoke.args().length() > BASELINE_MAX_ARGS_LENGTH) {
+ JitSpew(JitSpew_BaselineAbort, "Too many arguments (%u)",
+ invoke.args().length());
+ return Method_CantCompile;
+ }
+ } else {
+ if (state.asExecute()->isDebuggerEval()) {
+ JitSpew(JitSpew_BaselineAbort, "debugger frame");
+ return Method_CantCompile;
+ }
+ }
+
+ RootedScript script(cx, state.script());
+ switch (Tier) {
+ case BaselineTier::Interpreter:
+ return CanEnterBaselineInterpreter(cx, script);
+
+ case BaselineTier::Compiler:
+ return CanEnterBaselineJIT(cx, script,
+ /* osrSourceFrame = */ NullFramePtr());
+ }
+
+ MOZ_CRASH("Unexpected tier");
+}
+
+template MethodStatus jit::CanEnterBaselineMethod<BaselineTier::Interpreter>(
+ JSContext* cx, RunState& state);
+template MethodStatus jit::CanEnterBaselineMethod<BaselineTier::Compiler>(
+ JSContext* cx, RunState& state);
+
+bool jit::BaselineCompileFromBaselineInterpreter(JSContext* cx,
+ BaselineFrame* frame,
+ uint8_t** res) {
+ MOZ_ASSERT(frame->runningInInterpreter());
+
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc = frame->interpreterPC();
+ MOZ_ASSERT(pc == script->code() || JSOp(*pc) == JSOp::LoopHead);
+
+ MethodStatus status = CanEnterBaselineJIT(cx, script,
+ /* osrSourceFrame = */ frame);
+ switch (status) {
+ case Method_Error:
+ return false;
+
+ case Method_CantCompile:
+ case Method_Skipped:
+ *res = nullptr;
+ return true;
+
+ case Method_Compiled: {
+ if (JSOp(*pc) == JSOp::LoopHead) {
+ MOZ_ASSERT(pc > script->code(),
+ "Prologue vs OSR cases must not be ambiguous");
+ BaselineScript* baselineScript = script->baselineScript();
+ uint32_t pcOffset = script->pcToOffset(pc);
+ *res = baselineScript->nativeCodeForOSREntry(pcOffset);
+ } else {
+ *res = script->baselineScript()->warmUpCheckPrologueAddr();
+ }
+ frame->prepareForBaselineInterpreterToJitOSR();
+ return true;
+ }
+ }
+
+ MOZ_CRASH("Unexpected status");
+}
+
+BaselineScript* BaselineScript::New(JSContext* cx,
+ uint32_t warmUpCheckPrologueOffset,
+ uint32_t profilerEnterToggleOffset,
+ uint32_t profilerExitToggleOffset,
+ size_t retAddrEntries, size_t osrEntries,
+ size_t debugTrapEntries,
+ size_t resumeEntries) {
+ // Compute size including trailing arrays.
+ CheckedInt<Offset> size = sizeof(BaselineScript);
+ size += CheckedInt<Offset>(resumeEntries) * sizeof(uintptr_t);
+ size += CheckedInt<Offset>(retAddrEntries) * sizeof(RetAddrEntry);
+ size += CheckedInt<Offset>(osrEntries) * sizeof(OSREntry);
+ size += CheckedInt<Offset>(debugTrapEntries) * sizeof(DebugTrapEntry);
+
+ if (!size.isValid()) {
+ ReportAllocationOverflow(cx);
+ return nullptr;
+ }
+
+ // Allocate contiguous raw buffer.
+ void* raw = cx->pod_malloc<uint8_t>(size.value());
+ MOZ_ASSERT(uintptr_t(raw) % alignof(BaselineScript) == 0);
+ if (!raw) {
+ return nullptr;
+ }
+ BaselineScript* script = new (raw)
+ BaselineScript(warmUpCheckPrologueOffset, profilerEnterToggleOffset,
+ profilerExitToggleOffset);
+
+ Offset cursor = sizeof(BaselineScript);
+
+ MOZ_ASSERT(isAlignedOffset<uintptr_t>(cursor));
+ script->resumeEntriesOffset_ = cursor;
+ cursor += resumeEntries * sizeof(uintptr_t);
+
+ MOZ_ASSERT(isAlignedOffset<RetAddrEntry>(cursor));
+ script->retAddrEntriesOffset_ = cursor;
+ cursor += retAddrEntries * sizeof(RetAddrEntry);
+
+ MOZ_ASSERT(isAlignedOffset<OSREntry>(cursor));
+ script->osrEntriesOffset_ = cursor;
+ cursor += osrEntries * sizeof(OSREntry);
+
+ MOZ_ASSERT(isAlignedOffset<DebugTrapEntry>(cursor));
+ script->debugTrapEntriesOffset_ = cursor;
+ cursor += debugTrapEntries * sizeof(DebugTrapEntry);
+
+ MOZ_ASSERT(isAlignedOffset<uint32_t>(cursor));
+
+ script->allocBytes_ = cursor;
+
+ MOZ_ASSERT(script->endOffset() == size.value());
+
+ return script;
+}
+
+void BaselineScript::trace(JSTracer* trc) {
+ TraceEdge(trc, &method_, "baseline-method");
+}
+
+void BaselineScript::Destroy(JS::GCContext* gcx, BaselineScript* script) {
+ MOZ_ASSERT(!script->hasPendingIonCompileTask());
+
+ // This allocation is tracked by JSScript::setBaselineScriptImpl.
+ gcx->deleteUntracked(script);
+}
+
+void JS::DeletePolicy<js::jit::BaselineScript>::operator()(
+ const js::jit::BaselineScript* script) {
+ BaselineScript::Destroy(rt_->gcContext(),
+ const_cast<BaselineScript*>(script));
+}
+
+const RetAddrEntry& BaselineScript::retAddrEntryFromReturnOffset(
+ CodeOffset returnOffset) {
+ mozilla::Span<RetAddrEntry> entries = retAddrEntries();
+ size_t loc;
+#ifdef DEBUG
+ bool found =
+#endif
+ BinarySearchIf(
+ entries.data(), 0, entries.size(),
+ [&returnOffset](const RetAddrEntry& entry) {
+ size_t roffset = returnOffset.offset();
+ size_t entryRoffset = entry.returnOffset().offset();
+ if (roffset < entryRoffset) {
+ return -1;
+ }
+ if (entryRoffset < roffset) {
+ return 1;
+ }
+ return 0;
+ },
+ &loc);
+
+ MOZ_ASSERT(found);
+ MOZ_ASSERT(entries[loc].returnOffset().offset() == returnOffset.offset());
+ return entries[loc];
+}
+
+template <typename Entry>
+static bool ComputeBinarySearchMid(mozilla::Span<Entry> entries,
+ uint32_t pcOffset, size_t* loc) {
+ return BinarySearchIf(
+ entries.data(), 0, entries.size(),
+ [pcOffset](const Entry& entry) {
+ uint32_t entryOffset = entry.pcOffset();
+ if (pcOffset < entryOffset) {
+ return -1;
+ }
+ if (entryOffset < pcOffset) {
+ return 1;
+ }
+ return 0;
+ },
+ loc);
+}
+
+uint8_t* BaselineScript::returnAddressForEntry(const RetAddrEntry& ent) {
+ return method()->raw() + ent.returnOffset().offset();
+}
+
+const RetAddrEntry& BaselineScript::retAddrEntryFromPCOffset(
+ uint32_t pcOffset, RetAddrEntry::Kind kind) {
+ mozilla::Span<RetAddrEntry> entries = retAddrEntries();
+ size_t mid;
+ MOZ_ALWAYS_TRUE(ComputeBinarySearchMid(entries, pcOffset, &mid));
+ MOZ_ASSERT(mid < entries.size());
+
+ // Search for the first entry for this pc.
+ size_t first = mid;
+ while (first > 0 && entries[first - 1].pcOffset() == pcOffset) {
+ first--;
+ }
+
+ // Search for the last entry for this pc.
+ size_t last = mid;
+ while (last + 1 < entries.size() &&
+ entries[last + 1].pcOffset() == pcOffset) {
+ last++;
+ }
+
+ MOZ_ASSERT(first <= last);
+ MOZ_ASSERT(entries[first].pcOffset() == pcOffset);
+ MOZ_ASSERT(entries[last].pcOffset() == pcOffset);
+
+ for (size_t i = first; i <= last; i++) {
+ const RetAddrEntry& entry = entries[i];
+ if (entry.kind() != kind) {
+ continue;
+ }
+
+#ifdef DEBUG
+ // There must be a unique entry for this pcOffset and Kind to ensure our
+ // return value is well-defined.
+ for (size_t j = i + 1; j <= last; j++) {
+ MOZ_ASSERT(entries[j].kind() != kind);
+ }
+#endif
+
+ return entry;
+ }
+
+ MOZ_CRASH("Didn't find RetAddrEntry.");
+}
+
+const RetAddrEntry& BaselineScript::prologueRetAddrEntry(
+ RetAddrEntry::Kind kind) {
+ MOZ_ASSERT(kind == RetAddrEntry::Kind::StackCheck);
+
+ // The prologue entries will always be at a very low offset, so just do a
+ // linear search from the beginning.
+ for (const RetAddrEntry& entry : retAddrEntries()) {
+ if (entry.pcOffset() != 0) {
+ break;
+ }
+ if (entry.kind() == kind) {
+ return entry;
+ }
+ }
+ MOZ_CRASH("Didn't find prologue RetAddrEntry.");
+}
+
+const RetAddrEntry& BaselineScript::retAddrEntryFromReturnAddress(
+ const uint8_t* returnAddr) {
+ MOZ_ASSERT(returnAddr > method_->raw());
+ MOZ_ASSERT(returnAddr < method_->raw() + method_->instructionsSize());
+ CodeOffset offset(returnAddr - method_->raw());
+ return retAddrEntryFromReturnOffset(offset);
+}
+
+uint8_t* BaselineScript::nativeCodeForOSREntry(uint32_t pcOffset) {
+ mozilla::Span<OSREntry> entries = osrEntries();
+ size_t mid;
+ if (!ComputeBinarySearchMid(entries, pcOffset, &mid)) {
+ return nullptr;
+ }
+
+ uint32_t nativeOffset = entries[mid].nativeOffset();
+ return method_->raw() + nativeOffset;
+}
+
+void BaselineScript::computeResumeNativeOffsets(
+ JSScript* script, const ResumeOffsetEntryVector& entries) {
+ // Translate pcOffset to BaselineScript native address. This may return
+ // nullptr if compiler decided code was unreachable.
+ auto computeNative = [this, &entries](uint32_t pcOffset) -> uint8_t* {
+ mozilla::Span<const ResumeOffsetEntry> entriesSpan =
+ mozilla::Span(entries.begin(), entries.length());
+ size_t mid;
+ if (!ComputeBinarySearchMid(entriesSpan, pcOffset, &mid)) {
+ return nullptr;
+ }
+
+ uint32_t nativeOffset = entries[mid].nativeOffset();
+ return method_->raw() + nativeOffset;
+ };
+
+ mozilla::Span<const uint32_t> pcOffsets = script->resumeOffsets();
+ mozilla::Span<uint8_t*> nativeOffsets = resumeEntryList();
+ std::transform(pcOffsets.begin(), pcOffsets.end(), nativeOffsets.begin(),
+ computeNative);
+}
+
+void BaselineScript::copyRetAddrEntries(const RetAddrEntry* entries) {
+ std::copy_n(entries, retAddrEntries().size(), retAddrEntries().data());
+}
+
+void BaselineScript::copyOSREntries(const OSREntry* entries) {
+ std::copy_n(entries, osrEntries().size(), osrEntries().data());
+}
+
+void BaselineScript::copyDebugTrapEntries(const DebugTrapEntry* entries) {
+ std::copy_n(entries, debugTrapEntries().size(), debugTrapEntries().data());
+}
+
+jsbytecode* BaselineScript::approximatePcForNativeAddress(
+ JSScript* script, uint8_t* nativeAddress) {
+ MOZ_ASSERT(script->baselineScript() == this);
+ MOZ_ASSERT(containsCodeAddress(nativeAddress));
+
+ uint32_t nativeOffset = nativeAddress - method_->raw();
+
+ // Use the RetAddrEntry list (sorted on pc and return address) to look for the
+ // first pc that has a return address >= nativeOffset. This isn't perfect but
+ // it's a reasonable approximation for the profiler because most non-trivial
+ // bytecode ops have a RetAddrEntry.
+
+ for (const RetAddrEntry& entry : retAddrEntries()) {
+ uint32_t retOffset = entry.returnOffset().offset();
+ if (retOffset >= nativeOffset) {
+ return script->offsetToPC(entry.pcOffset());
+ }
+ }
+
+ // Return the last entry's pc. Every BaselineScript has at least one
+ // RetAddrEntry for the prologue stack overflow check.
+ MOZ_ASSERT(!retAddrEntries().empty());
+ const RetAddrEntry& lastEntry = retAddrEntries()[retAddrEntries().size() - 1];
+ return script->offsetToPC(lastEntry.pcOffset());
+}
+
+void BaselineScript::toggleDebugTraps(JSScript* script, jsbytecode* pc) {
+ MOZ_ASSERT(script->baselineScript() == this);
+
+ // Only scripts compiled for debug mode have toggled calls.
+ if (!hasDebugInstrumentation()) {
+ return;
+ }
+
+ AutoWritableJitCode awjc(method());
+
+ for (const DebugTrapEntry& entry : debugTrapEntries()) {
+ jsbytecode* entryPC = script->offsetToPC(entry.pcOffset());
+
+ // If the |pc| argument is non-null we can skip all other bytecode ops.
+ if (pc && pc != entryPC) {
+ continue;
+ }
+
+ bool enabled = DebugAPI::stepModeEnabled(script) ||
+ DebugAPI::hasBreakpointsAt(script, entryPC);
+
+ // Patch the trap.
+ CodeLocationLabel label(method(), CodeOffset(entry.nativeOffset()));
+ Assembler::ToggleCall(label, enabled);
+ }
+}
+
+void BaselineScript::setPendingIonCompileTask(JSRuntime* rt, JSScript* script,
+ IonCompileTask* task) {
+ MOZ_ASSERT(script->baselineScript() == this);
+ MOZ_ASSERT(task);
+ MOZ_ASSERT(!hasPendingIonCompileTask());
+
+ if (script->isIonCompilingOffThread()) {
+ script->jitScript()->clearIsIonCompilingOffThread(script);
+ }
+
+ pendingIonCompileTask_ = task;
+ script->updateJitCodeRaw(rt);
+}
+
+void BaselineScript::removePendingIonCompileTask(JSRuntime* rt,
+ JSScript* script) {
+ MOZ_ASSERT(script->baselineScript() == this);
+ MOZ_ASSERT(hasPendingIonCompileTask());
+
+ pendingIonCompileTask_ = nullptr;
+ script->updateJitCodeRaw(rt);
+}
+
+static void ToggleProfilerInstrumentation(JitCode* code,
+ uint32_t profilerEnterToggleOffset,
+ uint32_t profilerExitToggleOffset,
+ bool enable) {
+ CodeLocationLabel enterToggleLocation(code,
+ CodeOffset(profilerEnterToggleOffset));
+ CodeLocationLabel exitToggleLocation(code,
+ CodeOffset(profilerExitToggleOffset));
+ if (enable) {
+ Assembler::ToggleToCmp(enterToggleLocation);
+ Assembler::ToggleToCmp(exitToggleLocation);
+ } else {
+ Assembler::ToggleToJmp(enterToggleLocation);
+ Assembler::ToggleToJmp(exitToggleLocation);
+ }
+}
+
+void BaselineScript::toggleProfilerInstrumentation(bool enable) {
+ if (enable == isProfilerInstrumentationOn()) {
+ return;
+ }
+
+ JitSpew(JitSpew_BaselineIC, " toggling profiling %s for BaselineScript %p",
+ enable ? "on" : "off", this);
+
+ ToggleProfilerInstrumentation(method_, profilerEnterToggleOffset_,
+ profilerExitToggleOffset_, enable);
+
+ if (enable) {
+ flags_ |= uint32_t(PROFILER_INSTRUMENTATION_ON);
+ } else {
+ flags_ &= ~uint32_t(PROFILER_INSTRUMENTATION_ON);
+ }
+}
+
+void BaselineInterpreter::toggleProfilerInstrumentation(bool enable) {
+ if (!IsBaselineInterpreterEnabled()) {
+ return;
+ }
+
+ AutoWritableJitCode awjc(code_);
+ ToggleProfilerInstrumentation(code_, profilerEnterToggleOffset_,
+ profilerExitToggleOffset_, enable);
+}
+
+void BaselineInterpreter::toggleDebuggerInstrumentation(bool enable) {
+ if (!IsBaselineInterpreterEnabled()) {
+ return;
+ }
+
+ AutoWritableJitCode awjc(code_);
+
+ // Toggle jumps for debugger instrumentation.
+ for (uint32_t offset : debugInstrumentationOffsets_) {
+ CodeLocationLabel label(code_, CodeOffset(offset));
+ if (enable) {
+ Assembler::ToggleToCmp(label);
+ } else {
+ Assembler::ToggleToJmp(label);
+ }
+ }
+
+ // Toggle DebugTrapHandler calls.
+
+ uint8_t* debugTrapHandler = codeAtOffset(debugTrapHandlerOffset_);
+
+ for (uint32_t offset : debugTrapOffsets_) {
+ uint8_t* trap = codeAtOffset(offset);
+ if (enable) {
+ MacroAssembler::patchNopToCall(trap, debugTrapHandler);
+ } else {
+ MacroAssembler::patchCallToNop(trap);
+ }
+ }
+}
+
+void BaselineInterpreter::toggleCodeCoverageInstrumentationUnchecked(
+ bool enable) {
+ if (!IsBaselineInterpreterEnabled()) {
+ return;
+ }
+
+ AutoWritableJitCode awjc(code_);
+
+ for (uint32_t offset : codeCoverageOffsets_) {
+ CodeLocationLabel label(code_, CodeOffset(offset));
+ if (enable) {
+ Assembler::ToggleToCmp(label);
+ } else {
+ Assembler::ToggleToJmp(label);
+ }
+ }
+}
+
+void BaselineInterpreter::toggleCodeCoverageInstrumentation(bool enable) {
+ if (coverage::IsLCovEnabled()) {
+ // Instrumentation is enabled no matter what.
+ return;
+ }
+
+ toggleCodeCoverageInstrumentationUnchecked(enable);
+}
+
+void jit::FinishDiscardBaselineScript(JS::GCContext* gcx, JSScript* script) {
+ MOZ_ASSERT(script->hasBaselineScript());
+ MOZ_ASSERT(!script->jitScript()->active());
+
+ BaselineScript* baseline =
+ script->jitScript()->clearBaselineScript(gcx, script);
+ BaselineScript::Destroy(gcx, baseline);
+}
+
+void jit::AddSizeOfBaselineData(JSScript* script,
+ mozilla::MallocSizeOf mallocSizeOf,
+ size_t* data) {
+ if (script->hasBaselineScript()) {
+ script->baselineScript()->addSizeOfIncludingThis(mallocSizeOf, data);
+ }
+}
+
+void jit::ToggleBaselineProfiling(JSContext* cx, bool enable) {
+ JitRuntime* jrt = cx->runtime()->jitRuntime();
+ if (!jrt) {
+ return;
+ }
+
+ jrt->baselineInterpreter().toggleProfilerInstrumentation(enable);
+
+ for (ZonesIter zone(cx->runtime(), SkipAtoms); !zone.done(); zone.next()) {
+ for (auto base = zone->cellIter<BaseScript>(); !base.done(); base.next()) {
+ if (!base->hasJitScript()) {
+ continue;
+ }
+ JSScript* script = base->asJSScript();
+ if (enable) {
+ script->jitScript()->ensureProfileString(cx, script);
+ }
+ if (!script->hasBaselineScript()) {
+ continue;
+ }
+ AutoWritableJitCode awjc(script->baselineScript()->method());
+ script->baselineScript()->toggleProfilerInstrumentation(enable);
+ }
+ }
+}
+
+void BaselineInterpreter::init(JitCode* code, uint32_t interpretOpOffset,
+ uint32_t interpretOpNoDebugTrapOffset,
+ uint32_t bailoutPrologueOffset,
+ uint32_t profilerEnterToggleOffset,
+ uint32_t profilerExitToggleOffset,
+ uint32_t debugTrapHandlerOffset,
+ CodeOffsetVector&& debugInstrumentationOffsets,
+ CodeOffsetVector&& debugTrapOffsets,
+ CodeOffsetVector&& codeCoverageOffsets,
+ ICReturnOffsetVector&& icReturnOffsets,
+ const CallVMOffsets& callVMOffsets) {
+ code_ = code;
+ interpretOpOffset_ = interpretOpOffset;
+ interpretOpNoDebugTrapOffset_ = interpretOpNoDebugTrapOffset;
+ bailoutPrologueOffset_ = bailoutPrologueOffset;
+ profilerEnterToggleOffset_ = profilerEnterToggleOffset;
+ profilerExitToggleOffset_ = profilerExitToggleOffset;
+ debugTrapHandlerOffset_ = debugTrapHandlerOffset;
+ debugInstrumentationOffsets_ = std::move(debugInstrumentationOffsets);
+ debugTrapOffsets_ = std::move(debugTrapOffsets);
+ codeCoverageOffsets_ = std::move(codeCoverageOffsets);
+ icReturnOffsets_ = std::move(icReturnOffsets);
+ callVMOffsets_ = callVMOffsets;
+}
+
+uint8_t* BaselineInterpreter::retAddrForIC(JSOp op) const {
+ for (const ICReturnOffset& entry : icReturnOffsets_) {
+ if (entry.op == op) {
+ return codeAtOffset(entry.offset);
+ }
+ }
+ MOZ_CRASH("Unexpected op");
+}
+
+bool jit::GenerateBaselineInterpreter(JSContext* cx,
+ BaselineInterpreter& interpreter) {
+ if (IsBaselineInterpreterEnabled()) {
+ TempAllocator temp(&cx->tempLifoAlloc());
+ BaselineInterpreterGenerator generator(cx, temp);
+ return generator.generate(interpreter);
+ }
+
+ return true;
+}
diff --git a/js/src/jit/BaselineJIT.h b/js/src/jit/BaselineJIT.h
new file mode 100644
index 0000000000..7351eeaf7f
--- /dev/null
+++ b/js/src/jit/BaselineJIT.h
@@ -0,0 +1,595 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BaselineJIT_h
+#define jit_BaselineJIT_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Likely.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/Span.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jsfriendapi.h"
+
+#include "jit/IonTypes.h"
+#include "jit/JitCode.h"
+#include "jit/JitContext.h"
+#include "jit/JitOptions.h"
+#include "jit/shared/Assembler-shared.h"
+#include "js/Principals.h"
+#include "js/TypeDecls.h"
+#include "js/Vector.h"
+#include "threading/ProtectedData.h"
+#include "util/TrailingArray.h"
+#include "vm/JSScript.h"
+
+namespace js {
+
+class InterpreterFrame;
+class RunState;
+
+namespace jit {
+
+class BaselineFrame;
+class ExceptionBailoutInfo;
+class IonCompileTask;
+class JitActivation;
+class JSJitFrameIter;
+
+// Base class for entries mapping a pc offset to a native code offset.
+class BasePCToNativeEntry {
+ uint32_t pcOffset_;
+ uint32_t nativeOffset_;
+
+ public:
+ BasePCToNativeEntry(uint32_t pcOffset, uint32_t nativeOffset)
+ : pcOffset_(pcOffset), nativeOffset_(nativeOffset) {}
+ uint32_t pcOffset() const { return pcOffset_; }
+ uint32_t nativeOffset() const { return nativeOffset_; }
+};
+
+// Class used during Baseline compilation to store the native code offset for
+// resume offset ops.
+class ResumeOffsetEntry : public BasePCToNativeEntry {
+ public:
+ using BasePCToNativeEntry::BasePCToNativeEntry;
+};
+
+using ResumeOffsetEntryVector =
+ Vector<ResumeOffsetEntry, 16, SystemAllocPolicy>;
+
+// Largest script that the baseline compiler will attempt to compile.
+#if defined(JS_CODEGEN_ARM)
+// ARM branches can only reach 32MB, and the macroassembler doesn't mitigate
+// that limitation. Use a stricter limit on the acceptable script size to
+// avoid crashing when branches go out of range.
+static constexpr uint32_t BaselineMaxScriptLength = 1000000u;
+#else
+static constexpr uint32_t BaselineMaxScriptLength = 0x0fffffffu;
+#endif
+
+// Limit the locals on a given script so that stack check on baseline frames
+// doesn't overflow a uint32_t value.
+// (BaselineMaxScriptSlots * sizeof(Value)) must fit within a uint32_t.
+//
+// This also applies to the Baseline Interpreter: it ensures we don't run out
+// of stack space (and throw over-recursion exceptions) for scripts with a huge
+// number of locals. The C++ interpreter avoids this by having heap-allocated
+// stack frames.
+static constexpr uint32_t BaselineMaxScriptSlots = 0xffffu;
+
+// An entry in the BaselineScript return address table. These entries are used
+// to determine the bytecode pc for a return address into Baseline code.
+//
+// There must be an entry for each location where we can end up calling into
+// C++ (directly or via script/trampolines) and C++ can request the current
+// bytecode pc (this includes anything that may throw an exception, GC, or walk
+// the stack). We currently add entries for each:
+//
+// * callVM
+// * IC
+// * DebugTrap (trampoline call)
+// * JSOp::Resume (because this is like a scripted call)
+//
+// Note: see also BaselineFrame::HAS_OVERRIDE_PC.
+class RetAddrEntry {
+ // Offset from the start of the JIT code where call instruction is.
+ uint32_t returnOffset_;
+
+ // The offset of this bytecode op within the JSScript.
+ uint32_t pcOffset_ : 28;
+
+ public:
+ enum class Kind : uint32_t {
+ // An IC for a JOF_IC op.
+ IC,
+
+ // A callVM for an op.
+ CallVM,
+
+ // A callVM not for an op (e.g., in the prologue) that can't
+ // trigger debug mode.
+ NonOpCallVM,
+
+ // A callVM for the over-recursion check on function entry.
+ StackCheck,
+
+ // A callVM for an interrupt check.
+ InterruptCheck,
+
+ // DebugTrapHandler (for debugger breakpoints/stepping).
+ DebugTrap,
+
+ // A callVM for Debug{Prologue,AfterYield,Epilogue}.
+ DebugPrologue,
+ DebugAfterYield,
+ DebugEpilogue,
+
+ Invalid
+ };
+
+ private:
+ // What this entry is for.
+ uint32_t kind_ : 4;
+
+ public:
+ RetAddrEntry(uint32_t pcOffset, Kind kind, CodeOffset retOffset)
+ : returnOffset_(uint32_t(retOffset.offset())),
+ pcOffset_(pcOffset),
+ kind_(uint32_t(kind)) {
+ MOZ_ASSERT(returnOffset_ == retOffset.offset(),
+ "retOffset must fit in returnOffset_");
+
+ // The pc offset must fit in at least 28 bits, since we shave off 4 for
+ // the Kind enum.
+ MOZ_ASSERT(pcOffset_ == pcOffset);
+ static_assert(BaselineMaxScriptLength <= (1u << 28) - 1);
+ MOZ_ASSERT(pcOffset <= BaselineMaxScriptLength);
+
+ MOZ_ASSERT(kind < Kind::Invalid);
+ MOZ_ASSERT(this->kind() == kind, "kind must fit in kind_ bit field");
+ }
+
+ CodeOffset returnOffset() const { return CodeOffset(returnOffset_); }
+
+ uint32_t pcOffset() const { return pcOffset_; }
+
+ jsbytecode* pc(JSScript* script) const {
+ return script->offsetToPC(pcOffset_);
+ }
+
+ Kind kind() const {
+ MOZ_ASSERT(kind_ < uint32_t(Kind::Invalid));
+ return Kind(kind_);
+ }
+};
+
+// [SMDOC] BaselineScript
+//
+// This holds the metadata generated by the BaselineCompiler. The machine code
+// associated with this is owned by a JitCode instance. This class instance is
+// followed by several arrays:
+//
+// <BaselineScript itself>
+// --
+// uint8_t*[] resumeEntryList()
+// RetAddrEntry[] retAddrEntries()
+// OSREntry[] osrEntries()
+// DebugTrapEntry[] debugTrapEntries()
+//
+// Note: The arrays are arranged in order of descending alignment requires so
+// that padding is not required.
+class alignas(uintptr_t) BaselineScript final : public TrailingArray {
+ private:
+ // Code pointer containing the actual method.
+ HeapPtr<JitCode*> method_ = nullptr;
+
+ // An ion compilation that is ready, but isn't linked yet.
+ MainThreadData<IonCompileTask*> pendingIonCompileTask_{nullptr};
+
+ // Baseline Interpreter can enter Baseline Compiler code at this address. This
+ // is right after the warm-up counter check in the prologue.
+ uint32_t warmUpCheckPrologueOffset_ = 0;
+
+ // The offsets for the toggledJump instructions for profiler instrumentation.
+ uint32_t profilerEnterToggleOffset_ = 0;
+ uint32_t profilerExitToggleOffset_ = 0;
+
+ private:
+ // Offset (in bytes) from `this` to the start of each trailing array. Each
+ // array ends where following one begins. There is no implicit padding (except
+ // possible at very end).
+ Offset resumeEntriesOffset_ = 0;
+ Offset retAddrEntriesOffset_ = 0;
+ Offset osrEntriesOffset_ = 0;
+ Offset debugTrapEntriesOffset_ = 0;
+ Offset allocBytes_ = 0;
+
+ // See `Flag` type below.
+ uint8_t flags_ = 0;
+
+ // End of fields.
+
+ public:
+ enum Flag {
+ // Flag set when compiled for use with Debugger. Handles various
+ // Debugger hooks and compiles toggled calls for traps.
+ HAS_DEBUG_INSTRUMENTATION = 1 << 0,
+
+ // Flag is set if this script has profiling instrumentation turned on.
+ PROFILER_INSTRUMENTATION_ON = 1 << 1,
+ };
+
+ // Native code offset for OSR from Baseline Interpreter into Baseline JIT at
+ // JSOp::LoopHead ops.
+ class OSREntry : public BasePCToNativeEntry {
+ public:
+ using BasePCToNativeEntry::BasePCToNativeEntry;
+ };
+
+ // Native code offset for a debug trap when the script is compiled with debug
+ // instrumentation.
+ class DebugTrapEntry : public BasePCToNativeEntry {
+ public:
+ using BasePCToNativeEntry::BasePCToNativeEntry;
+ };
+
+ private:
+ // Layout helpers
+ Offset resumeEntriesOffset() const { return resumeEntriesOffset_; }
+ Offset retAddrEntriesOffset() const { return retAddrEntriesOffset_; }
+ Offset osrEntriesOffset() const { return osrEntriesOffset_; }
+ Offset debugTrapEntriesOffset() const { return debugTrapEntriesOffset_; }
+ Offset endOffset() const { return allocBytes_; }
+
+ // Use BaselineScript::New to create new instances. It will properly
+ // allocate trailing objects.
+ BaselineScript(uint32_t warmUpCheckPrologueOffset,
+ uint32_t profilerEnterToggleOffset,
+ uint32_t profilerExitToggleOffset)
+ : warmUpCheckPrologueOffset_(warmUpCheckPrologueOffset),
+ profilerEnterToggleOffset_(profilerEnterToggleOffset),
+ profilerExitToggleOffset_(profilerExitToggleOffset) {}
+
+ template <typename T>
+ mozilla::Span<T> makeSpan(Offset start, Offset end) {
+ return mozilla::Span{offsetToPointer<T>(start), numElements<T>(start, end)};
+ }
+
+ // We store the native code address corresponding to each bytecode offset in
+ // the script's resumeOffsets list.
+ mozilla::Span<uint8_t*> resumeEntryList() {
+ return makeSpan<uint8_t*>(resumeEntriesOffset(), retAddrEntriesOffset());
+ }
+
+ // See each type for documentation of these arrays.
+ mozilla::Span<RetAddrEntry> retAddrEntries() {
+ return makeSpan<RetAddrEntry>(retAddrEntriesOffset(), osrEntriesOffset());
+ }
+ mozilla::Span<OSREntry> osrEntries() {
+ return makeSpan<OSREntry>(osrEntriesOffset(), debugTrapEntriesOffset());
+ }
+ mozilla::Span<DebugTrapEntry> debugTrapEntries() {
+ return makeSpan<DebugTrapEntry>(debugTrapEntriesOffset(), endOffset());
+ }
+
+ public:
+ static BaselineScript* New(JSContext* cx, uint32_t warmUpCheckPrologueOffset,
+ uint32_t profilerEnterToggleOffset,
+ uint32_t profilerExitToggleOffset,
+ size_t retAddrEntries, size_t osrEntries,
+ size_t debugTrapEntries, size_t resumeEntries);
+
+ static void Destroy(JS::GCContext* gcx, BaselineScript* script);
+
+ void trace(JSTracer* trc);
+
+ static inline size_t offsetOfMethod() {
+ return offsetof(BaselineScript, method_);
+ }
+
+ void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf,
+ size_t* data) const {
+ *data += mallocSizeOf(this);
+ }
+
+ void setHasDebugInstrumentation() { flags_ |= HAS_DEBUG_INSTRUMENTATION; }
+ bool hasDebugInstrumentation() const {
+ return flags_ & HAS_DEBUG_INSTRUMENTATION;
+ }
+
+ uint8_t* warmUpCheckPrologueAddr() const {
+ return method_->raw() + warmUpCheckPrologueOffset_;
+ }
+
+ JitCode* method() const { return method_; }
+ void setMethod(JitCode* code) {
+ MOZ_ASSERT(!method_);
+ method_ = code;
+ }
+
+ bool containsCodeAddress(uint8_t* addr) const {
+ return method()->raw() <= addr &&
+ addr <= method()->raw() + method()->instructionsSize();
+ }
+
+ uint8_t* returnAddressForEntry(const RetAddrEntry& ent);
+
+ const RetAddrEntry& retAddrEntryFromPCOffset(uint32_t pcOffset,
+ RetAddrEntry::Kind kind);
+ const RetAddrEntry& prologueRetAddrEntry(RetAddrEntry::Kind kind);
+ const RetAddrEntry& retAddrEntryFromReturnOffset(CodeOffset returnOffset);
+ const RetAddrEntry& retAddrEntryFromReturnAddress(const uint8_t* returnAddr);
+
+ uint8_t* nativeCodeForOSREntry(uint32_t pcOffset);
+
+ void copyRetAddrEntries(const RetAddrEntry* entries);
+ void copyOSREntries(const OSREntry* entries);
+ void copyDebugTrapEntries(const DebugTrapEntry* entries);
+
+ // Copy resumeOffsets list from |script| and convert the pcOffsets
+ // to native addresses in the Baseline code based on |entries|.
+ void computeResumeNativeOffsets(JSScript* script,
+ const ResumeOffsetEntryVector& entries);
+
+ // Return the bytecode offset for a given native code address. Be careful
+ // when using this method: it's an approximation and not guaranteed to be the
+ // correct pc.
+ jsbytecode* approximatePcForNativeAddress(JSScript* script,
+ uint8_t* nativeAddress);
+
+ // Toggle debug traps (used for breakpoints and step mode) in the script.
+ // If |pc| is nullptr, toggle traps for all ops in the script. Else, only
+ // toggle traps at |pc|.
+ void toggleDebugTraps(JSScript* script, jsbytecode* pc);
+
+ void toggleProfilerInstrumentation(bool enable);
+ bool isProfilerInstrumentationOn() const {
+ return flags_ & PROFILER_INSTRUMENTATION_ON;
+ }
+
+ static size_t offsetOfResumeEntriesOffset() {
+ static_assert(sizeof(Offset) == sizeof(uint32_t),
+ "JIT expect Offset to be uint32_t");
+ return offsetof(BaselineScript, resumeEntriesOffset_);
+ }
+
+ bool hasPendingIonCompileTask() const { return !!pendingIonCompileTask_; }
+
+ js::jit::IonCompileTask* pendingIonCompileTask() {
+ MOZ_ASSERT(hasPendingIonCompileTask());
+ return pendingIonCompileTask_;
+ }
+ void setPendingIonCompileTask(JSRuntime* rt, JSScript* script,
+ js::jit::IonCompileTask* task);
+ void removePendingIonCompileTask(JSRuntime* rt, JSScript* script);
+
+ size_t allocBytes() const { return allocBytes_; }
+};
+static_assert(
+ sizeof(BaselineScript) % sizeof(uintptr_t) == 0,
+ "The data attached to the script must be aligned for fast JIT access.");
+
+enum class BaselineTier { Interpreter, Compiler };
+
+template <BaselineTier Tier>
+MethodStatus CanEnterBaselineMethod(JSContext* cx, RunState& state);
+
+MethodStatus CanEnterBaselineInterpreterAtBranch(JSContext* cx,
+ InterpreterFrame* fp);
+
+JitExecStatus EnterBaselineInterpreterAtBranch(JSContext* cx,
+ InterpreterFrame* fp,
+ jsbytecode* pc);
+
+bool CanBaselineInterpretScript(JSScript* script);
+
+// Called by the Baseline Interpreter to compile a script for the Baseline JIT.
+// |res| is set to the native code address in the BaselineScript to jump to, or
+// nullptr if we were unable to compile this script.
+bool BaselineCompileFromBaselineInterpreter(JSContext* cx, BaselineFrame* frame,
+ uint8_t** res);
+
+void FinishDiscardBaselineScript(JS::GCContext* gcx, JSScript* script);
+
+void AddSizeOfBaselineData(JSScript* script, mozilla::MallocSizeOf mallocSizeOf,
+ size_t* data);
+
+void ToggleBaselineProfiling(JSContext* cx, bool enable);
+
+struct alignas(uintptr_t) BaselineBailoutInfo {
+ // Pointer into the current C stack, where overwriting will start.
+ uint8_t* incomingStack = nullptr;
+
+ // The top and bottom heapspace addresses of the reconstructed stack
+ // which will be copied to the bottom.
+ uint8_t* copyStackTop = nullptr;
+ uint8_t* copyStackBottom = nullptr;
+
+ // The value of the frame pointer register on resume.
+ void* resumeFramePtr = nullptr;
+
+ // The native code address to resume into.
+ void* resumeAddr = nullptr;
+
+ // The bytecode pc of try block and fault block.
+ jsbytecode* tryPC = nullptr;
+ jsbytecode* faultPC = nullptr;
+
+ // Number of baseline frames to push on the stack.
+ uint32_t numFrames = 0;
+
+ // The bailout kind.
+ mozilla::Maybe<BailoutKind> bailoutKind = {};
+
+ BaselineBailoutInfo() = default;
+ BaselineBailoutInfo(const BaselineBailoutInfo&) = default;
+
+ void operator=(const BaselineBailoutInfo&) = delete;
+};
+
+enum class BailoutReason {
+ Normal,
+ ExceptionHandler,
+ Invalidate,
+};
+
+[[nodiscard]] bool BailoutIonToBaseline(
+ JSContext* cx, JitActivation* activation, const JSJitFrameIter& iter,
+ BaselineBailoutInfo** bailoutInfo,
+ const ExceptionBailoutInfo* exceptionInfo, BailoutReason reason);
+
+MethodStatus BaselineCompile(JSContext* cx, JSScript* script,
+ bool forceDebugInstrumentation = false);
+
+static const unsigned BASELINE_MAX_ARGS_LENGTH = 20000;
+
+// Class storing the generated Baseline Interpreter code for the runtime.
+class BaselineInterpreter {
+ public:
+ struct CallVMOffsets {
+ uint32_t debugPrologueOffset = 0;
+ uint32_t debugEpilogueOffset = 0;
+ uint32_t debugAfterYieldOffset = 0;
+ };
+ struct ICReturnOffset {
+ uint32_t offset;
+ JSOp op;
+ ICReturnOffset(uint32_t offset, JSOp op) : offset(offset), op(op) {}
+ };
+ using ICReturnOffsetVector = Vector<ICReturnOffset, 0, SystemAllocPolicy>;
+
+ private:
+ // The interpreter code.
+ JitCode* code_ = nullptr;
+
+ // Offset of the code to start interpreting a bytecode op.
+ uint32_t interpretOpOffset_ = 0;
+
+ // Like interpretOpOffset_ but skips the debug trap for the current op.
+ uint32_t interpretOpNoDebugTrapOffset_ = 0;
+
+ // Early Ion bailouts will enter at this address. This is after frame
+ // construction and environment initialization.
+ uint32_t bailoutPrologueOffset_ = 0;
+
+ // The offsets for the toggledJump instructions for profiler instrumentation.
+ uint32_t profilerEnterToggleOffset_ = 0;
+ uint32_t profilerExitToggleOffset_ = 0;
+
+ // Offset of the jump (tail call) to the debug trap handler trampoline code.
+ // When the debugger is enabled, NOPs are patched to calls to this location.
+ uint32_t debugTrapHandlerOffset_ = 0;
+
+ // The offsets of toggled jumps for debugger instrumentation.
+ using CodeOffsetVector = Vector<uint32_t, 0, SystemAllocPolicy>;
+ CodeOffsetVector debugInstrumentationOffsets_;
+
+ // Offsets of toggled calls to the DebugTrapHandler trampoline (for
+ // breakpoints and stepping).
+ CodeOffsetVector debugTrapOffsets_;
+
+ // Offsets of toggled jumps for code coverage.
+ CodeOffsetVector codeCoverageOffsets_;
+
+ // Offsets of IC calls for IsIonInlinableOp ops, for Ion bailouts.
+ ICReturnOffsetVector icReturnOffsets_;
+
+ // Offsets of some callVMs for BaselineDebugModeOSR.
+ CallVMOffsets callVMOffsets_;
+
+ uint8_t* codeAtOffset(uint32_t offset) const {
+ MOZ_ASSERT(offset > 0);
+ MOZ_ASSERT(offset < code_->instructionsSize());
+ return codeRaw() + offset;
+ }
+
+ public:
+ BaselineInterpreter() = default;
+
+ BaselineInterpreter(const BaselineInterpreter&) = delete;
+ void operator=(const BaselineInterpreter&) = delete;
+
+ void init(JitCode* code, uint32_t interpretOpOffset,
+ uint32_t interpretOpNoDebugTrapOffset,
+ uint32_t bailoutPrologueOffset, uint32_t profilerEnterToggleOffset,
+ uint32_t profilerExitToggleOffset, uint32_t debugTrapHandlerOffset,
+ CodeOffsetVector&& debugInstrumentationOffsets,
+ CodeOffsetVector&& debugTrapOffsets,
+ CodeOffsetVector&& codeCoverageOffsets,
+ ICReturnOffsetVector&& icReturnOffsets,
+ const CallVMOffsets& callVMOffsets);
+
+ uint8_t* codeRaw() const { return code_->raw(); }
+
+ uint8_t* retAddrForDebugPrologueCallVM() const {
+ return codeAtOffset(callVMOffsets_.debugPrologueOffset);
+ }
+ uint8_t* retAddrForDebugEpilogueCallVM() const {
+ return codeAtOffset(callVMOffsets_.debugEpilogueOffset);
+ }
+ uint8_t* retAddrForDebugAfterYieldCallVM() const {
+ return codeAtOffset(callVMOffsets_.debugAfterYieldOffset);
+ }
+ uint8_t* bailoutPrologueEntryAddr() const {
+ return codeAtOffset(bailoutPrologueOffset_);
+ }
+
+ uint8_t* retAddrForIC(JSOp op) const;
+
+ TrampolinePtr interpretOpAddr() const {
+ return TrampolinePtr(codeAtOffset(interpretOpOffset_));
+ }
+ TrampolinePtr interpretOpNoDebugTrapAddr() const {
+ return TrampolinePtr(codeAtOffset(interpretOpNoDebugTrapOffset_));
+ }
+
+ void toggleProfilerInstrumentation(bool enable);
+ void toggleDebuggerInstrumentation(bool enable);
+
+ void toggleCodeCoverageInstrumentationUnchecked(bool enable);
+ void toggleCodeCoverageInstrumentation(bool enable);
+};
+
+[[nodiscard]] bool GenerateBaselineInterpreter(
+ JSContext* cx, BaselineInterpreter& interpreter);
+
+inline bool IsBaselineJitEnabled(JSContext* cx) {
+ if (MOZ_UNLIKELY(!IsBaselineInterpreterEnabled())) {
+ return false;
+ }
+ if (MOZ_LIKELY(JitOptions.baselineJit)) {
+ return true;
+ }
+ if (JitOptions.jitForTrustedPrincipals) {
+ JS::Realm* realm = js::GetContextRealm(cx);
+ return realm && JS::GetRealmPrincipals(realm) &&
+ JS::GetRealmPrincipals(realm)->isSystemOrAddonPrincipal();
+ }
+ return false;
+}
+
+} // namespace jit
+} // namespace js
+
+namespace JS {
+
+template <>
+struct DeletePolicy<js::jit::BaselineScript> {
+ explicit DeletePolicy(JSRuntime* rt) : rt_(rt) {}
+ void operator()(const js::jit::BaselineScript* script);
+
+ private:
+ JSRuntime* rt_;
+};
+
+} // namespace JS
+
+#endif /* jit_BaselineJIT_h */
diff --git a/js/src/jit/BitSet.cpp b/js/src/jit/BitSet.cpp
new file mode 100644
index 0000000000..b791ea387c
--- /dev/null
+++ b/js/src/jit/BitSet.cpp
@@ -0,0 +1,111 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BitSet.h"
+
+#include <string.h>
+
+#include "jit/JitAllocPolicy.h"
+
+using namespace js;
+using namespace js::jit;
+
+bool BitSet::init(TempAllocator& alloc) {
+ size_t sizeRequired = numWords() * sizeof(*bits_);
+
+ bits_ = (uint32_t*)alloc.allocate(sizeRequired);
+ if (!bits_) {
+ return false;
+ }
+
+ memset(bits_, 0, sizeRequired);
+
+ return true;
+}
+
+bool BitSet::empty() const {
+ MOZ_ASSERT(bits_);
+ const uint32_t* bits = bits_;
+ for (unsigned int i = 0, e = numWords(); i < e; i++) {
+ if (bits[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void BitSet::insertAll(const BitSet& other) {
+ MOZ_ASSERT(bits_);
+ MOZ_ASSERT(other.numBits_ == numBits_);
+ MOZ_ASSERT(other.bits_);
+
+ uint32_t* bits = bits_;
+ const uint32_t* otherBits = other.bits_;
+ for (unsigned int i = 0, e = numWords(); i < e; i++) {
+ bits[i] |= otherBits[i];
+ }
+}
+
+void BitSet::removeAll(const BitSet& other) {
+ MOZ_ASSERT(bits_);
+ MOZ_ASSERT(other.numBits_ == numBits_);
+ MOZ_ASSERT(other.bits_);
+
+ uint32_t* bits = bits_;
+ const uint32_t* otherBits = other.bits_;
+ for (unsigned int i = 0, e = numWords(); i < e; i++) {
+ bits[i] &= ~otherBits[i];
+ }
+}
+
+void BitSet::intersect(const BitSet& other) {
+ MOZ_ASSERT(bits_);
+ MOZ_ASSERT(other.numBits_ == numBits_);
+ MOZ_ASSERT(other.bits_);
+
+ uint32_t* bits = bits_;
+ const uint32_t* otherBits = other.bits_;
+ for (unsigned int i = 0, e = numWords(); i < e; i++) {
+ bits[i] &= otherBits[i];
+ }
+}
+
+// returns true if the intersection caused the contents of the set to change.
+bool BitSet::fixedPointIntersect(const BitSet& other) {
+ MOZ_ASSERT(bits_);
+ MOZ_ASSERT(other.numBits_ == numBits_);
+ MOZ_ASSERT(other.bits_);
+
+ bool changed = false;
+
+ uint32_t* bits = bits_;
+ const uint32_t* otherBits = other.bits_;
+ for (unsigned int i = 0, e = numWords(); i < e; i++) {
+ uint32_t old = bits[i];
+ bits[i] &= otherBits[i];
+
+ if (!changed && old != bits[i]) {
+ changed = true;
+ }
+ }
+ return changed;
+}
+
+void BitSet::complement() {
+ MOZ_ASSERT(bits_);
+ uint32_t* bits = bits_;
+ for (unsigned int i = 0, e = numWords(); i < e; i++) {
+ bits[i] = ~bits[i];
+ }
+}
+
+void BitSet::clear() {
+ MOZ_ASSERT(bits_);
+ uint32_t* bits = bits_;
+ for (unsigned int i = 0, e = numWords(); i < e; i++) {
+ bits[i] = 0;
+ }
+}
diff --git a/js/src/jit/BitSet.h b/js/src/jit/BitSet.h
new file mode 100644
index 0000000000..4e34b1ecb1
--- /dev/null
+++ b/js/src/jit/BitSet.h
@@ -0,0 +1,167 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BitSet_h
+#define jit_BitSet_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+namespace js {
+namespace jit {
+
+class TempAllocator;
+
+// Provides constant time set insertion and removal, and fast linear
+// set operations such as intersection, difference, and union.
+// N.B. All set operations must be performed on sets with the same number
+// of bits.
+class BitSet {
+ public:
+ static const size_t BitsPerWord = 8 * sizeof(uint32_t);
+
+ static size_t RawLengthForBits(size_t bits) {
+ return (bits + BitsPerWord - 1) / BitsPerWord;
+ }
+
+ private:
+ uint32_t* bits_;
+ const unsigned int numBits_;
+
+ static inline uint32_t bitForValue(unsigned int value) {
+ return 1l << uint32_t(value % BitsPerWord);
+ }
+
+ static inline unsigned int wordForValue(unsigned int value) {
+ return value / BitsPerWord;
+ }
+
+ inline unsigned int numWords() const { return RawLengthForBits(numBits_); }
+
+ BitSet(const BitSet&) = delete;
+ void operator=(const BitSet&) = delete;
+
+ public:
+ class Iterator;
+
+ explicit BitSet(unsigned int numBits) : bits_(nullptr), numBits_(numBits) {}
+
+ [[nodiscard]] bool init(TempAllocator& alloc);
+
+ unsigned int getNumBits() const { return numBits_; }
+
+ // O(1): Check if this set contains the given value.
+ bool contains(unsigned int value) const {
+ MOZ_ASSERT(bits_);
+ MOZ_ASSERT(value < numBits_);
+
+ return !!(bits_[wordForValue(value)] & bitForValue(value));
+ }
+
+ // O(numBits): Check if this set contains any value.
+ bool empty() const;
+
+ // O(1): Insert the given value into this set.
+ void insert(unsigned int value) {
+ MOZ_ASSERT(bits_);
+ MOZ_ASSERT(value < numBits_);
+
+ bits_[wordForValue(value)] |= bitForValue(value);
+ }
+
+ // O(numBits): Insert every element of the given set into this set.
+ void insertAll(const BitSet& other);
+
+ // O(1): Remove the given value from this set.
+ void remove(unsigned int value) {
+ MOZ_ASSERT(bits_);
+ MOZ_ASSERT(value < numBits_);
+
+ bits_[wordForValue(value)] &= ~bitForValue(value);
+ }
+
+ // O(numBits): Remove the every element of the given set from this set.
+ void removeAll(const BitSet& other);
+
+ // O(numBits): Intersect this set with the given set.
+ void intersect(const BitSet& other);
+
+ // O(numBits): Intersect this set with the given set; return whether the
+ // intersection caused the set to change.
+ bool fixedPointIntersect(const BitSet& other);
+
+ // O(numBits): Does inplace complement of the set.
+ void complement();
+
+ // O(numBits): Clear this set.
+ void clear();
+
+ uint32_t* raw() const { return bits_; }
+ size_t rawLength() const { return numWords(); }
+};
+
+class BitSet::Iterator {
+ private:
+ BitSet& set_;
+ unsigned index_;
+ unsigned word_;
+ uint32_t value_;
+
+ void skipEmpty() {
+ // Skip words containing only zeros.
+ unsigned numWords = set_.numWords();
+ const uint32_t* bits = set_.bits_;
+ while (value_ == 0) {
+ word_++;
+ if (word_ == numWords) {
+ return;
+ }
+
+ index_ = word_ * BitSet::BitsPerWord;
+ value_ = bits[word_];
+ }
+
+ // Be careful: the result of CountTrailingZeroes32 is undefined if the
+ // input is 0.
+ int numZeros = mozilla::CountTrailingZeroes32(value_);
+ index_ += numZeros;
+ value_ >>= numZeros;
+
+ MOZ_ASSERT_IF(index_ < set_.numBits_, set_.contains(index_));
+ }
+
+ public:
+ explicit Iterator(BitSet& set)
+ : set_(set), index_(0), word_(0), value_(set.bits_[0]) {
+ skipEmpty();
+ }
+
+ inline bool more() const { return word_ < set_.numWords(); }
+ explicit operator bool() const { return more(); }
+
+ inline void operator++() {
+ MOZ_ASSERT(more());
+ MOZ_ASSERT(index_ < set_.numBits_);
+
+ index_++;
+ value_ >>= 1;
+
+ skipEmpty();
+ }
+
+ unsigned int operator*() {
+ MOZ_ASSERT(index_ < set_.numBits_);
+ return index_;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BitSet_h */
diff --git a/js/src/jit/BytecodeAnalysis.cpp b/js/src/jit/BytecodeAnalysis.cpp
new file mode 100644
index 0000000000..251768d72f
--- /dev/null
+++ b/js/src/jit/BytecodeAnalysis.cpp
@@ -0,0 +1,283 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BytecodeAnalysis.h"
+
+#include "jit/JitSpewer.h"
+#include "jit/WarpBuilder.h"
+#include "vm/BytecodeIterator.h"
+#include "vm/BytecodeLocation.h"
+#include "vm/BytecodeUtil.h"
+
+#include "vm/BytecodeIterator-inl.h"
+#include "vm/BytecodeLocation-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+BytecodeAnalysis::BytecodeAnalysis(TempAllocator& alloc, JSScript* script)
+ : script_(script), infos_(alloc) {}
+
+bool BytecodeAnalysis::init(TempAllocator& alloc) {
+ if (!infos_.growByUninitialized(script_->length())) {
+ return false;
+ }
+
+ // Clear all BytecodeInfo.
+ mozilla::PodZero(infos_.begin(), infos_.length());
+ infos_[0].init(/*stackDepth=*/0);
+
+ // WarpBuilder can compile try blocks, but doesn't support handling
+ // exceptions. If exception unwinding would resume in a catch or finally
+ // block, we instead bail out to the baseline interpreter. Finally blocks can
+ // still be reached by normal means, but the catch block is unreachable and is
+ // not compiled. We therefore need some special machinery to prevent OSR into
+ // Warp code in the following cases:
+ //
+ // (1) Loops in catch blocks:
+ //
+ // try {
+ // ..
+ // } catch (e) {
+ // while (..) {} // Can't OSR here.
+ // }
+ //
+ // (2) Loops only reachable via a catch block:
+ //
+ // for (;;) {
+ // try {
+ // throw 3;
+ // } catch (e) {
+ // break;
+ // }
+ // }
+ // while (..) {} // Loop is only reachable via the catch-block.
+ //
+ // To deal with both of these cases, we track whether the current op is
+ // 'normally reachable' (reachable without exception handling).
+ // Forward jumps propagate this flag to their jump targets (see
+ // BytecodeInfo::jumpTargetNormallyReachable) and when the analysis reaches a
+ // jump target it updates its normallyReachable flag based on the target's
+ // flag.
+ //
+ // Inlining a function without a normally reachable return can cause similar
+ // problems. To avoid this, we mark such functions as uninlineable.
+ bool normallyReachable = true;
+ bool normallyReachableReturn = false;
+
+ for (const BytecodeLocation& it : AllBytecodesIterable(script_)) {
+ JSOp op = it.getOp();
+ uint32_t offset = it.bytecodeToOffset(script_);
+
+ JitSpew(JitSpew_BaselineOp, "Analyzing op @ %u (end=%u): %s",
+ unsigned(offset), unsigned(script_->length()), CodeName(op));
+
+ checkWarpSupport(op);
+
+ // If this bytecode info has not yet been initialized, it's not reachable.
+ if (!infos_[offset].initialized) {
+ continue;
+ }
+
+ uint32_t stackDepth = infos_[offset].stackDepth;
+
+ if (infos_[offset].jumpTarget) {
+ normallyReachable = infos_[offset].jumpTargetNormallyReachable;
+ }
+
+#ifdef DEBUG
+ size_t endOffset = offset + it.length();
+ for (size_t checkOffset = offset + 1; checkOffset < endOffset;
+ checkOffset++) {
+ MOZ_ASSERT(!infos_[checkOffset].initialized);
+ }
+#endif
+ uint32_t nuses = it.useCount();
+ uint32_t ndefs = it.defCount();
+
+ MOZ_ASSERT(stackDepth >= nuses);
+ stackDepth -= nuses;
+ stackDepth += ndefs;
+
+ // If stack depth exceeds max allowed by analysis, fail fast.
+ MOZ_ASSERT(stackDepth <= BytecodeInfo::MAX_STACK_DEPTH);
+
+ switch (op) {
+ case JSOp::TableSwitch: {
+ uint32_t defaultOffset = it.getTableSwitchDefaultOffset(script_);
+ int32_t low = it.getTableSwitchLow();
+ int32_t high = it.getTableSwitchHigh();
+
+ infos_[defaultOffset].init(stackDepth);
+ infos_[defaultOffset].setJumpTarget(normallyReachable);
+
+ uint32_t ncases = high - low + 1;
+
+ for (uint32_t i = 0; i < ncases; i++) {
+ uint32_t targetOffset = it.tableSwitchCaseOffset(script_, i);
+ if (targetOffset != defaultOffset) {
+ infos_[targetOffset].init(stackDepth);
+ infos_[targetOffset].setJumpTarget(normallyReachable);
+ }
+ }
+ break;
+ }
+
+ case JSOp::Try: {
+ for (const TryNote& tn : script_->trynotes()) {
+ if (tn.start == offset + JSOpLength_Try &&
+ (tn.kind() == TryNoteKind::Catch ||
+ tn.kind() == TryNoteKind::Finally)) {
+ uint32_t catchOrFinallyOffset = tn.start + tn.length;
+ uint32_t targetDepth =
+ tn.kind() == TryNoteKind::Finally ? stackDepth + 2 : stackDepth;
+ BytecodeInfo& targetInfo = infos_[catchOrFinallyOffset];
+ targetInfo.init(targetDepth);
+ targetInfo.setJumpTarget(/* normallyReachable = */ false);
+ }
+ }
+ break;
+ }
+
+ case JSOp::LoopHead:
+ infos_[offset].loopHeadCanOsr = normallyReachable;
+ break;
+
+#ifdef DEBUG
+ case JSOp::Exception:
+ // Sanity check: ops only emitted in catch blocks are never
+ // normally reachable.
+ MOZ_ASSERT(!normallyReachable);
+ break;
+#endif
+
+ case JSOp::Return:
+ case JSOp::RetRval:
+ if (normallyReachable) {
+ normallyReachableReturn = true;
+ }
+ break;
+
+ default:
+ break;
+ }
+
+ bool jump = it.isJump();
+ if (jump) {
+ // Case instructions do not push the lvalue back when branching.
+ uint32_t newStackDepth = stackDepth;
+ if (it.is(JSOp::Case)) {
+ newStackDepth--;
+ }
+
+ uint32_t targetOffset = it.getJumpTargetOffset(script_);
+
+#ifdef DEBUG
+ // If this is a backedge, the target JSOp::LoopHead must have been
+ // analyzed already. Furthermore, if the backedge is normally reachable,
+ // the loop head must be normally reachable too (loopHeadCanOsr can be
+ // used to check this since it's equivalent).
+ if (targetOffset < offset) {
+ MOZ_ASSERT(infos_[targetOffset].initialized);
+ MOZ_ASSERT_IF(normallyReachable, infos_[targetOffset].loopHeadCanOsr);
+ }
+#endif
+
+ infos_[targetOffset].init(newStackDepth);
+ infos_[targetOffset].setJumpTarget(normallyReachable);
+ }
+
+ // Handle any fallthrough from this opcode.
+ if (it.fallsThrough()) {
+ BytecodeLocation fallthroughLoc = it.next();
+ MOZ_ASSERT(fallthroughLoc.isInBounds(script_));
+ uint32_t fallthroughOffset = fallthroughLoc.bytecodeToOffset(script_);
+
+ infos_[fallthroughOffset].init(stackDepth);
+
+ // Treat the fallthrough of a branch instruction as a jump target.
+ if (jump) {
+ infos_[fallthroughOffset].setJumpTarget(normallyReachable);
+ }
+ }
+ }
+
+ // Flag (reachable) resume offset instructions.
+ for (uint32_t offset : script_->resumeOffsets()) {
+ BytecodeInfo& info = infos_[offset];
+ if (info.initialized) {
+ info.hasResumeOffset = true;
+ }
+ }
+
+ if (!normallyReachableReturn) {
+ script_->setUninlineable();
+ }
+
+ return true;
+}
+
+void BytecodeAnalysis::checkWarpSupport(JSOp op) {
+ switch (op) {
+#define DEF_CASE(OP) case JSOp::OP:
+ WARP_UNSUPPORTED_OPCODE_LIST(DEF_CASE)
+#undef DEF_CASE
+ if (script_->canIonCompile()) {
+ JitSpew(JitSpew_IonAbort, "Disabling Warp support for %s:%d:%d due to %s",
+ script_->filename(), script_->lineno(), script_->column(),
+ CodeName(op));
+ script_->disableIon();
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+IonBytecodeInfo js::jit::AnalyzeBytecodeForIon(JSContext* cx,
+ JSScript* script) {
+ IonBytecodeInfo result;
+
+ if (script->isModule() || script->initialEnvironmentShape() ||
+ (script->function() &&
+ script->function()->needsSomeEnvironmentObject())) {
+ result.usesEnvironmentChain = true;
+ }
+
+ AllBytecodesIterable iterator(script);
+
+ for (const BytecodeLocation& location : iterator) {
+ switch (location.getOp()) {
+ case JSOp::SetArg:
+ result.modifiesArguments = true;
+ break;
+
+ case JSOp::GetName:
+ case JSOp::BindName:
+ case JSOp::BindVar:
+ case JSOp::SetName:
+ case JSOp::StrictSetName:
+ case JSOp::DelName:
+ case JSOp::GetAliasedVar:
+ case JSOp::SetAliasedVar:
+ case JSOp::Lambda:
+ case JSOp::PushLexicalEnv:
+ case JSOp::PopLexicalEnv:
+ case JSOp::PushVarEnv:
+ case JSOp::ImplicitThis:
+ case JSOp::FunWithProto:
+ case JSOp::GlobalOrEvalDeclInstantiation:
+ result.usesEnvironmentChain = true;
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ return result;
+}
diff --git a/js/src/jit/BytecodeAnalysis.h b/js/src/jit/BytecodeAnalysis.h
new file mode 100644
index 0000000000..524240fd47
--- /dev/null
+++ b/js/src/jit/BytecodeAnalysis.h
@@ -0,0 +1,84 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_BytecodeAnalysis_h
+#define jit_BytecodeAnalysis_h
+
+#include "jit/JitAllocPolicy.h"
+#include "js/Vector.h"
+#include "vm/JSScript.h"
+
+namespace js {
+namespace jit {
+
+// Basic information about bytecodes in the script. Used to help baseline
+// compilation.
+struct BytecodeInfo {
+ static const uint16_t MAX_STACK_DEPTH = 0xffffU;
+ uint16_t stackDepth;
+ bool initialized : 1;
+ bool jumpTarget : 1;
+
+ // If true, this is a JSOp::LoopHead where we can OSR into Ion/Warp code.
+ bool loopHeadCanOsr : 1;
+
+ // See the comment above normallyReachable in BytecodeAnalysis.cpp for how
+ // this works.
+ bool jumpTargetNormallyReachable : 1;
+
+ // True if the script has a resume offset for this bytecode op.
+ bool hasResumeOffset : 1;
+
+ void init(unsigned depth) {
+ MOZ_ASSERT(depth <= MAX_STACK_DEPTH);
+ MOZ_ASSERT_IF(initialized, stackDepth == depth);
+ initialized = true;
+ stackDepth = depth;
+ }
+
+ void setJumpTarget(bool normallyReachable) {
+ jumpTarget = true;
+ if (normallyReachable) {
+ jumpTargetNormallyReachable = true;
+ }
+ }
+};
+
+class BytecodeAnalysis {
+ JSScript* script_;
+ Vector<BytecodeInfo, 0, JitAllocPolicy> infos_;
+
+ public:
+ explicit BytecodeAnalysis(TempAllocator& alloc, JSScript* script);
+
+ [[nodiscard]] bool init(TempAllocator& alloc);
+
+ BytecodeInfo& info(jsbytecode* pc) {
+ uint32_t pcOffset = script_->pcToOffset(pc);
+ MOZ_ASSERT(infos_[pcOffset].initialized);
+ return infos_[pcOffset];
+ }
+
+ BytecodeInfo* maybeInfo(jsbytecode* pc) {
+ uint32_t pcOffset = script_->pcToOffset(pc);
+ if (infos_[pcOffset].initialized) {
+ return &infos_[pcOffset];
+ }
+ return nullptr;
+ }
+
+ void checkWarpSupport(JSOp op);
+};
+
+// Bytecode analysis pass necessary for WarpBuilder. The result is cached in
+// JitScript.
+struct IonBytecodeInfo;
+IonBytecodeInfo AnalyzeBytecodeForIon(JSContext* cx, JSScript* script);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_BytecodeAnalysis_h */
diff --git a/js/src/jit/CacheIR.cpp b/js/src/jit/CacheIR.cpp
new file mode 100644
index 0000000000..9df3105b53
--- /dev/null
+++ b/js/src/jit/CacheIR.cpp
@@ -0,0 +1,12698 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/CacheIR.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/FloatingPoint.h"
+
+#include "jsapi.h"
+#include "jsmath.h"
+
+#include "builtin/DataViewObject.h"
+#include "builtin/MapObject.h"
+#include "builtin/ModuleObject.h"
+#include "builtin/Object.h"
+#include "jit/BaselineIC.h"
+#include "jit/CacheIRCloner.h"
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIRGenerator.h"
+#include "jit/CacheIRSpewer.h"
+#include "jit/CacheIRWriter.h"
+#include "jit/InlinableNatives.h"
+#include "jit/JitContext.h"
+#include "js/experimental/JitInfo.h" // JSJitInfo
+#include "js/friend/DOMProxy.h" // JS::ExpandoAndGeneration
+#include "js/friend/WindowProxy.h" // js::IsWindow, js::IsWindowProxy, js::ToWindowIfWindowProxy
+#include "js/friend/XrayJitInfo.h" // js::jit::GetXrayJitInfo, JS::XrayJitInfo
+#include "js/GCAPI.h" // JS::AutoSuppressGCAnalysis
+#include "js/RegExpFlags.h" // JS::RegExpFlags
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "js/Wrapper.h"
+#include "proxy/DOMProxy.h" // js::GetDOMProxyHandlerFamily
+#include "util/DifferentialTesting.h"
+#include "util/Unicode.h"
+#include "vm/ArrayBufferObject.h"
+#include "vm/BytecodeUtil.h"
+#include "vm/Compartment.h"
+#include "vm/Iteration.h"
+#include "vm/PlainObject.h" // js::PlainObject
+#include "vm/ProxyObject.h"
+#include "vm/RegExpObject.h"
+#include "vm/SelfHosting.h"
+#include "vm/ThrowMsgKind.h" // ThrowCondition
+#include "vm/Watchtower.h"
+#include "wasm/WasmInstance.h"
+
+#include "jit/BaselineFrame-inl.h"
+#include "jit/MacroAssembler-inl.h"
+#include "vm/ArrayBufferObject-inl.h"
+#include "vm/BytecodeUtil-inl.h"
+#include "vm/EnvironmentObject-inl.h"
+#include "vm/JSContext-inl.h"
+#include "vm/JSObject-inl.h"
+#include "vm/JSScript-inl.h"
+#include "vm/NativeObject-inl.h"
+#include "vm/PlainObject-inl.h"
+#include "vm/StringObject-inl.h"
+#include "wasm/WasmInstance-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+using mozilla::Maybe;
+
+using JS::DOMProxyShadowsResult;
+using JS::ExpandoAndGeneration;
+
+const char* const js::jit::CacheKindNames[] = {
+#define DEFINE_KIND(kind) #kind,
+ CACHE_IR_KINDS(DEFINE_KIND)
+#undef DEFINE_KIND
+};
+
+const char* const js::jit::CacheIROpNames[] = {
+#define OPNAME(op, ...) #op,
+ CACHE_IR_OPS(OPNAME)
+#undef OPNAME
+};
+
+const CacheIROpInfo js::jit::CacheIROpInfos[] = {
+#define OPINFO(op, len, transpile, ...) {len, transpile},
+ CACHE_IR_OPS(OPINFO)
+#undef OPINFO
+};
+
+const uint32_t js::jit::CacheIROpHealth[] = {
+#define OPHEALTH(op, len, transpile, health) health,
+ CACHE_IR_OPS(OPHEALTH)
+#undef OPHEALTH
+};
+
+size_t js::jit::NumInputsForCacheKind(CacheKind kind) {
+ switch (kind) {
+ case CacheKind::NewArray:
+ case CacheKind::NewObject:
+ case CacheKind::GetIntrinsic:
+ return 0;
+ case CacheKind::GetProp:
+ case CacheKind::TypeOf:
+ case CacheKind::ToPropertyKey:
+ case CacheKind::GetIterator:
+ case CacheKind::ToBool:
+ case CacheKind::UnaryArith:
+ case CacheKind::GetName:
+ case CacheKind::BindName:
+ case CacheKind::Call:
+ case CacheKind::OptimizeSpreadCall:
+ case CacheKind::CloseIter:
+ return 1;
+ case CacheKind::Compare:
+ case CacheKind::GetElem:
+ case CacheKind::GetPropSuper:
+ case CacheKind::SetProp:
+ case CacheKind::In:
+ case CacheKind::HasOwn:
+ case CacheKind::CheckPrivateField:
+ case CacheKind::InstanceOf:
+ case CacheKind::BinaryArith:
+ return 2;
+ case CacheKind::GetElemSuper:
+ case CacheKind::SetElem:
+ return 3;
+ }
+ MOZ_CRASH("Invalid kind");
+}
+
+#ifdef DEBUG
+void CacheIRWriter::assertSameCompartment(JSObject* obj) {
+ cx_->debugOnlyCheck(obj);
+}
+void CacheIRWriter::assertSameZone(Shape* shape) {
+ MOZ_ASSERT(cx_->zone() == shape->zone());
+}
+#endif
+
+StubField CacheIRWriter::readStubField(uint32_t offset,
+ StubField::Type type) const {
+ size_t index = 0;
+ size_t currentOffset = 0;
+
+ // If we've seen an offset earlier than this before, we know we can start the
+ // search there at least, otherwise, we start the search from the beginning.
+ if (lastOffset_ < offset) {
+ currentOffset = lastOffset_;
+ index = lastIndex_;
+ }
+
+ while (currentOffset != offset) {
+ currentOffset += StubField::sizeInBytes(stubFields_[index].type());
+ index++;
+ MOZ_ASSERT(index < stubFields_.length());
+ }
+
+ MOZ_ASSERT(stubFields_[index].type() == type);
+
+ lastOffset_ = currentOffset;
+ lastIndex_ = index;
+
+ return stubFields_[index];
+}
+
+CacheIRCloner::CacheIRCloner(ICCacheIRStub* stub)
+ : stubInfo_(stub->stubInfo()), stubData_(stub->stubDataStart()) {}
+
+void CacheIRCloner::cloneOp(CacheOp op, CacheIRReader& reader,
+ CacheIRWriter& writer) {
+ switch (op) {
+#define DEFINE_OP(op, ...) \
+ case CacheOp::op: \
+ clone##op(reader, writer); \
+ break;
+ CACHE_IR_OPS(DEFINE_OP)
+#undef DEFINE_OP
+ default:
+ MOZ_CRASH("Invalid op");
+ }
+}
+
+uintptr_t CacheIRCloner::readStubWord(uint32_t offset) {
+ return stubInfo_->getStubRawWord(stubData_, offset);
+}
+int64_t CacheIRCloner::readStubInt64(uint32_t offset) {
+ return stubInfo_->getStubRawInt64(stubData_, offset);
+}
+
+Shape* CacheIRCloner::getShapeField(uint32_t stubOffset) {
+ return reinterpret_cast<Shape*>(readStubWord(stubOffset));
+}
+GetterSetter* CacheIRCloner::getGetterSetterField(uint32_t stubOffset) {
+ return reinterpret_cast<GetterSetter*>(readStubWord(stubOffset));
+}
+JSObject* CacheIRCloner::getObjectField(uint32_t stubOffset) {
+ return reinterpret_cast<JSObject*>(readStubWord(stubOffset));
+}
+JSString* CacheIRCloner::getStringField(uint32_t stubOffset) {
+ return reinterpret_cast<JSString*>(readStubWord(stubOffset));
+}
+JSAtom* CacheIRCloner::getAtomField(uint32_t stubOffset) {
+ return reinterpret_cast<JSAtom*>(readStubWord(stubOffset));
+}
+PropertyName* CacheIRCloner::getPropertyNameField(uint32_t stubOffset) {
+ return reinterpret_cast<PropertyName*>(readStubWord(stubOffset));
+}
+JS::Symbol* CacheIRCloner::getSymbolField(uint32_t stubOffset) {
+ return reinterpret_cast<JS::Symbol*>(readStubWord(stubOffset));
+}
+BaseScript* CacheIRCloner::getBaseScriptField(uint32_t stubOffset) {
+ return reinterpret_cast<BaseScript*>(readStubWord(stubOffset));
+}
+uint32_t CacheIRCloner::getRawInt32Field(uint32_t stubOffset) {
+ return uint32_t(reinterpret_cast<uintptr_t>(readStubWord(stubOffset)));
+}
+const void* CacheIRCloner::getRawPointerField(uint32_t stubOffset) {
+ return reinterpret_cast<const void*>(readStubWord(stubOffset));
+}
+uint64_t CacheIRCloner::getRawInt64Field(uint32_t stubOffset) {
+ return static_cast<uint64_t>(readStubInt64(stubOffset));
+}
+gc::AllocSite* CacheIRCloner::getAllocSiteField(uint32_t stubOffset) {
+ return reinterpret_cast<gc::AllocSite*>(readStubWord(stubOffset));
+}
+
+jsid CacheIRCloner::getIdField(uint32_t stubOffset) {
+ return jsid::fromRawBits(readStubWord(stubOffset));
+}
+const Value CacheIRCloner::getValueField(uint32_t stubOffset) {
+ return Value::fromRawBits(uint64_t(readStubInt64(stubOffset)));
+}
+double CacheIRCloner::getDoubleField(uint32_t stubOffset) {
+ uint64_t bits = uint64_t(readStubInt64(stubOffset));
+ return mozilla::BitwiseCast<double>(bits);
+}
+
+IRGenerator::IRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc,
+ CacheKind cacheKind, ICState state)
+ : writer(cx),
+ cx_(cx),
+ script_(script),
+ pc_(pc),
+ cacheKind_(cacheKind),
+ mode_(state.mode()),
+ isFirstStub_(state.newStubIsFirstStub()) {}
+
+GetPropIRGenerator::GetPropIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ CacheKind cacheKind, HandleValue val,
+ HandleValue idVal)
+ : IRGenerator(cx, script, pc, cacheKind, state), val_(val), idVal_(idVal) {}
+
+static void EmitLoadSlotResult(CacheIRWriter& writer, ObjOperandId holderId,
+ NativeObject* holder, PropertyInfo prop) {
+ if (holder->isFixedSlot(prop.slot())) {
+ writer.loadFixedSlotResult(holderId,
+ NativeObject::getFixedSlotOffset(prop.slot()));
+ } else {
+ size_t dynamicSlotOffset =
+ holder->dynamicSlotIndex(prop.slot()) * sizeof(Value);
+ writer.loadDynamicSlotResult(holderId, dynamicSlotOffset);
+ }
+}
+
+// DOM proxies
+// -----------
+//
+// DOM proxies are proxies that are used to implement various DOM objects like
+// HTMLDocument and NodeList. DOM proxies may have an expando object - a native
+// object that stores extra properties added to the object. The following
+// CacheIR instructions are only used with DOM proxies:
+//
+// * LoadDOMExpandoValue: returns the Value in the proxy's expando slot. This
+// returns either an UndefinedValue (no expando), ObjectValue (the expando
+// object), or PrivateValue(ExpandoAndGeneration*).
+//
+// * LoadDOMExpandoValueGuardGeneration: guards the Value in the proxy's expando
+// slot is the same PrivateValue(ExpandoAndGeneration*), then guards on its
+// generation, then returns expandoAndGeneration->expando. This Value is
+// either an UndefinedValue or ObjectValue.
+//
+// * LoadDOMExpandoValueIgnoreGeneration: assumes the Value in the proxy's
+// expando slot is a PrivateValue(ExpandoAndGeneration*), unboxes it, and
+// returns the expandoAndGeneration->expando Value.
+//
+// * GuardDOMExpandoMissingOrGuardShape: takes an expando Value as input, then
+// guards it's either UndefinedValue or an object with the expected shape.
+
+enum class ProxyStubType {
+ None,
+ DOMExpando,
+ DOMShadowed,
+ DOMUnshadowed,
+ Generic
+};
+
+static bool IsCacheableDOMProxy(ProxyObject* obj) {
+ const BaseProxyHandler* handler = obj->handler();
+ if (handler->family() != GetDOMProxyHandlerFamily()) {
+ return false;
+ }
+
+ // Some DOM proxies have dynamic prototypes. We can't really cache those very
+ // well.
+ return obj->hasStaticPrototype();
+}
+
+static ProxyStubType GetProxyStubType(JSContext* cx, HandleObject obj,
+ HandleId id) {
+ if (!obj->is<ProxyObject>()) {
+ return ProxyStubType::None;
+ }
+ auto proxy = obj.as<ProxyObject>();
+
+ if (!IsCacheableDOMProxy(proxy)) {
+ return ProxyStubType::Generic;
+ }
+
+ DOMProxyShadowsResult shadows = GetDOMProxyShadowsCheck()(cx, proxy, id);
+ if (shadows == DOMProxyShadowsResult::ShadowCheckFailed) {
+ cx->clearPendingException();
+ return ProxyStubType::None;
+ }
+
+ if (DOMProxyIsShadowing(shadows)) {
+ if (shadows == DOMProxyShadowsResult::ShadowsViaDirectExpando ||
+ shadows == DOMProxyShadowsResult::ShadowsViaIndirectExpando) {
+ return ProxyStubType::DOMExpando;
+ }
+ return ProxyStubType::DOMShadowed;
+ }
+
+ MOZ_ASSERT(shadows == DOMProxyShadowsResult::DoesntShadow ||
+ shadows == DOMProxyShadowsResult::DoesntShadowUnique);
+ return ProxyStubType::DOMUnshadowed;
+}
+
+static bool ValueToNameOrSymbolId(JSContext* cx, HandleValue idVal,
+ MutableHandleId id, bool* nameOrSymbol) {
+ *nameOrSymbol = false;
+
+ if (!idVal.isString() && !idVal.isSymbol() && !idVal.isUndefined() &&
+ !idVal.isNull()) {
+ return true;
+ }
+
+ if (!PrimitiveValueToId<CanGC>(cx, idVal, id)) {
+ return false;
+ }
+
+ if (!id.isAtom() && !id.isSymbol()) {
+ id.set(JS::PropertyKey::Void());
+ return true;
+ }
+
+ if (id.isAtom() && id.toAtom()->isIndex()) {
+ id.set(JS::PropertyKey::Void());
+ return true;
+ }
+
+ *nameOrSymbol = true;
+ return true;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ if (cacheKind_ != CacheKind::GetProp) {
+ MOZ_ASSERT_IF(cacheKind_ == CacheKind::GetPropSuper,
+ getSuperReceiverValueId().id() == 1);
+ MOZ_ASSERT_IF(cacheKind_ != CacheKind::GetPropSuper,
+ getElemKeyValueId().id() == 1);
+ writer.setInputOperandId(1);
+ }
+ if (cacheKind_ == CacheKind::GetElemSuper) {
+ MOZ_ASSERT(getSuperReceiverValueId().id() == 2);
+ writer.setInputOperandId(2);
+ }
+
+ RootedId id(cx_);
+ bool nameOrSymbol;
+ if (!ValueToNameOrSymbolId(cx_, idVal_, &id, &nameOrSymbol)) {
+ cx_->clearPendingException();
+ return AttachDecision::NoAction;
+ }
+
+ // |super.prop| getter calls use a |this| value that differs from lookup
+ // object.
+ ValOperandId receiverId = isSuper() ? getSuperReceiverValueId() : valId;
+
+ if (val_.isObject()) {
+ RootedObject obj(cx_, &val_.toObject());
+ ObjOperandId objId = writer.guardToObject(valId);
+ if (nameOrSymbol) {
+ TRY_ATTACH(tryAttachObjectLength(obj, objId, id));
+ TRY_ATTACH(tryAttachTypedArray(obj, objId, id));
+ TRY_ATTACH(tryAttachDataView(obj, objId, id));
+ TRY_ATTACH(tryAttachArrayBufferMaybeShared(obj, objId, id));
+ TRY_ATTACH(tryAttachRegExp(obj, objId, id));
+ TRY_ATTACH(tryAttachMap(obj, objId, id));
+ TRY_ATTACH(tryAttachSet(obj, objId, id));
+ TRY_ATTACH(tryAttachNative(obj, objId, id, receiverId));
+ TRY_ATTACH(tryAttachModuleNamespace(obj, objId, id));
+ TRY_ATTACH(tryAttachWindowProxy(obj, objId, id));
+ TRY_ATTACH(tryAttachCrossCompartmentWrapper(obj, objId, id));
+ TRY_ATTACH(
+ tryAttachXrayCrossCompartmentWrapper(obj, objId, id, receiverId));
+ TRY_ATTACH(tryAttachFunction(obj, objId, id));
+ TRY_ATTACH(tryAttachArgumentsObjectIterator(obj, objId, id));
+ TRY_ATTACH(tryAttachArgumentsObjectCallee(obj, objId, id));
+ TRY_ATTACH(tryAttachProxy(obj, objId, id, receiverId));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetElem ||
+ cacheKind_ == CacheKind::GetElemSuper);
+
+ TRY_ATTACH(tryAttachProxyElement(obj, objId));
+ TRY_ATTACH(tryAttachTypedArrayElement(obj, objId));
+
+ uint32_t index;
+ Int32OperandId indexId;
+ if (maybeGuardInt32Index(idVal_, getElemKeyValueId(), &index, &indexId)) {
+ TRY_ATTACH(tryAttachDenseElement(obj, objId, index, indexId));
+ TRY_ATTACH(tryAttachDenseElementHole(obj, objId, index, indexId));
+ TRY_ATTACH(tryAttachSparseElement(obj, objId, index, indexId));
+ TRY_ATTACH(tryAttachArgumentsObjectArg(obj, objId, index, indexId));
+ TRY_ATTACH(tryAttachArgumentsObjectArgHole(obj, objId, index, indexId));
+ TRY_ATTACH(
+ tryAttachGenericElement(obj, objId, index, indexId, receiverId));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ if (nameOrSymbol) {
+ TRY_ATTACH(tryAttachPrimitive(valId, id));
+ TRY_ATTACH(tryAttachStringLength(valId, id));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ if (idVal_.isInt32()) {
+ ValOperandId indexId = getElemKeyValueId();
+ TRY_ATTACH(tryAttachStringChar(valId, indexId));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+#ifdef DEBUG
+// Any property lookups performed when trying to attach ICs must be pure, i.e.
+// must use LookupPropertyPure() or similar functions. Pure lookups are
+// guaranteed to never modify the prototype chain. This ensures that the holder
+// object can always be found on the prototype chain.
+static bool IsCacheableProtoChain(NativeObject* obj, NativeObject* holder) {
+ while (obj != holder) {
+ JSObject* proto = obj->staticPrototype();
+ if (!proto || !proto->is<NativeObject>()) {
+ return false;
+ }
+ obj = &proto->as<NativeObject>();
+ }
+ return true;
+}
+#endif
+
+static bool IsCacheableGetPropSlot(NativeObject* obj, NativeObject* holder,
+ PropertyInfo prop) {
+ MOZ_ASSERT(IsCacheableProtoChain(obj, holder));
+
+ return prop.isDataProperty();
+}
+
+enum class NativeGetPropKind {
+ None,
+ Missing,
+ Slot,
+ NativeGetter,
+ ScriptedGetter,
+};
+
+static NativeGetPropKind IsCacheableGetPropCall(NativeObject* obj,
+ NativeObject* holder,
+ PropertyInfo prop) {
+ MOZ_ASSERT(IsCacheableProtoChain(obj, holder));
+
+ if (!prop.isAccessorProperty()) {
+ return NativeGetPropKind::None;
+ }
+
+ JSObject* getterObject = holder->getGetter(prop);
+ if (!getterObject || !getterObject->is<JSFunction>()) {
+ return NativeGetPropKind::None;
+ }
+
+ JSFunction& getter = getterObject->as<JSFunction>();
+
+ if (getter.isClassConstructor()) {
+ return NativeGetPropKind::None;
+ }
+
+ // Scripted functions and natives with JIT entry can use the scripted path.
+ if (getter.hasJitEntry()) {
+ return NativeGetPropKind::ScriptedGetter;
+ }
+
+ MOZ_ASSERT(getter.isNativeWithoutJitEntry());
+ return NativeGetPropKind::NativeGetter;
+}
+
+static bool CheckHasNoSuchOwnProperty(JSContext* cx, JSObject* obj, jsid id) {
+ if (!obj->is<NativeObject>()) {
+ return false;
+ }
+ // Don't handle objects with resolve hooks.
+ if (ClassMayResolveId(cx->names(), obj->getClass(), id, obj)) {
+ return false;
+ }
+ if (obj->as<NativeObject>().contains(cx, id)) {
+ return false;
+ }
+ return true;
+}
+
+static bool CheckHasNoSuchProperty(JSContext* cx, JSObject* obj, jsid id) {
+ JSObject* curObj = obj;
+ do {
+ if (!CheckHasNoSuchOwnProperty(cx, curObj, id)) {
+ return false;
+ }
+
+ curObj = curObj->staticPrototype();
+ } while (curObj);
+
+ return true;
+}
+
+static bool IsCacheableNoProperty(JSContext* cx, NativeObject* obj,
+ NativeObject* holder, jsid id,
+ jsbytecode* pc) {
+ MOZ_ASSERT(!holder);
+
+ // If we're doing a name lookup, we have to throw a ReferenceError.
+ if (JSOp(*pc) == JSOp::GetBoundName) {
+ return false;
+ }
+
+ return CheckHasNoSuchProperty(cx, obj, id);
+}
+
+static NativeGetPropKind CanAttachNativeGetProp(JSContext* cx, JSObject* obj,
+ PropertyKey id,
+ NativeObject** holder,
+ Maybe<PropertyInfo>* propInfo,
+ jsbytecode* pc) {
+ MOZ_ASSERT(id.isString() || id.isSymbol());
+ MOZ_ASSERT(!*holder);
+
+ // The lookup needs to be universally pure, otherwise we risk calling hooks
+ // out of turn. We don't mind doing this even when purity isn't required,
+ // because we only miss out on shape hashification, which is only a temporary
+ // perf cost. The limits were arbitrarily set, anyways.
+ NativeObject* baseHolder = nullptr;
+ PropertyResult prop;
+ if (!LookupPropertyPure(cx, obj, id, &baseHolder, &prop)) {
+ return NativeGetPropKind::None;
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ if (prop.isNativeProperty()) {
+ MOZ_ASSERT(baseHolder);
+ *holder = baseHolder;
+ *propInfo = mozilla::Some(prop.propertyInfo());
+
+ if (IsCacheableGetPropSlot(nobj, *holder, propInfo->ref())) {
+ return NativeGetPropKind::Slot;
+ }
+
+ return IsCacheableGetPropCall(nobj, *holder, propInfo->ref());
+ }
+
+ if (!prop.isFound()) {
+ if (IsCacheableNoProperty(cx, nobj, *holder, id, pc)) {
+ return NativeGetPropKind::Missing;
+ }
+ }
+
+ return NativeGetPropKind::None;
+}
+
+static void GuardReceiverProto(CacheIRWriter& writer, NativeObject* obj,
+ ObjOperandId objId) {
+ // Note: we guard on the actual prototype and not on the shape because this is
+ // used for sparse elements where we expect shape changes.
+
+ if (JSObject* proto = obj->staticPrototype()) {
+ writer.guardProto(objId, proto);
+ } else {
+ writer.guardNullProto(objId);
+ }
+}
+
+// Guard that a given object has same class and same OwnProperties (excluding
+// dense elements and dynamic properties).
+static void TestMatchingNativeReceiver(CacheIRWriter& writer, NativeObject* obj,
+ ObjOperandId objId) {
+ writer.guardShapeForOwnProperties(objId, obj->shape());
+}
+
+// Similar to |TestMatchingNativeReceiver|, but specialized for ProxyObject.
+static void TestMatchingProxyReceiver(CacheIRWriter& writer, ProxyObject* obj,
+ ObjOperandId objId) {
+ writer.guardShapeForClass(objId, obj->shape());
+}
+
+static void GeneratePrototypeGuards(CacheIRWriter& writer, JSObject* obj,
+ NativeObject* holder, ObjOperandId objId) {
+ // Assuming target property is on |holder|, generate appropriate guards to
+ // ensure |holder| is still on the prototype chain of |obj| and we haven't
+ // introduced any shadowing definitions.
+ //
+ // For each item in the proto chain before holder, we must ensure that
+ // [[GetPrototypeOf]] still has the expected result, and that
+ // [[GetOwnProperty]] has no definition of the target property.
+ //
+ //
+ // [SMDOC] Shape Teleporting Optimization
+ // --------------------------------------
+ //
+ // Starting with the assumption (and guideline to developers) that mutating
+ // prototypes is an uncommon and fair-to-penalize operation we move cost
+ // from the access side to the mutation side.
+ //
+ // Consider the following proto chain, with B defining a property 'x':
+ //
+ // D -> C -> B{x: 3} -> A -> null
+ //
+ // When accessing |D.x| we refer to D as the "receiver", and B as the
+ // "holder". To optimize this access we need to ensure that neither D nor C
+ // has since defined a shadowing property 'x'. Since C is a prototype that
+ // we assume is rarely mutated we would like to avoid checking each time if
+ // new properties are added. To do this we require that whenever C starts
+ // shadowing a property on its proto chain, we invalidate (and opt out of) the
+ // teleporting optimization by setting the InvalidatedTeleporting flag on the
+ // object we're shadowing, triggering a shape change of that object. As a
+ // result, checking the shape of D and B is sufficient. Note that we do not
+ // care if the shape or properties of A change since the lookup of 'x' will
+ // stop at B.
+ //
+ // The second condition we must verify is that the prototype chain was not
+ // mutated. The same mechanism as above is used. When the prototype link is
+ // changed, we generate a new shape for the object. If the object whose
+ // link we are mutating is itself a prototype, we regenerate shapes down
+ // the chain by setting the InvalidatedTeleporting flag on them. This means
+ // the same two shape checks as above are sufficient.
+ //
+ // Once the InvalidatedTeleporting flag is set, it means the shape will no
+ // longer be changed by ReshapeForProtoMutation and ReshapeForShadowedProp.
+ // In this case we can no longer apply the optimization.
+ //
+ // See:
+ // - ReshapeForProtoMutation
+ // - ReshapeForShadowedProp
+
+ MOZ_ASSERT(holder);
+ MOZ_ASSERT(obj != holder);
+
+ // Receiver guards (see TestMatchingReceiver) ensure the receiver's proto is
+ // unchanged so peel off the receiver.
+ JSObject* pobj = obj->staticPrototype();
+ MOZ_ASSERT(pobj->isUsedAsPrototype());
+
+ // If teleporting is supported for this holder, we are done.
+ if (!holder->hasInvalidatedTeleporting()) {
+ return;
+ }
+
+ // If already at the holder, no further proto checks are needed.
+ if (pobj == holder) {
+ return;
+ }
+
+ // Synchronize pobj and protoId.
+ MOZ_ASSERT(pobj == obj->staticPrototype());
+ ObjOperandId protoId = writer.loadProto(objId);
+
+ // Shape guard each prototype object between receiver and holder. This guards
+ // against both proto changes and shadowing properties.
+ while (pobj != holder) {
+ writer.guardShape(protoId, pobj->shape());
+
+ pobj = pobj->staticPrototype();
+ protoId = writer.loadProto(protoId);
+ }
+}
+
+static void GeneratePrototypeHoleGuards(CacheIRWriter& writer,
+ NativeObject* obj, ObjOperandId objId,
+ bool alwaysGuardFirstProto) {
+ if (alwaysGuardFirstProto) {
+ GuardReceiverProto(writer, obj, objId);
+ }
+
+ JSObject* pobj = obj->staticPrototype();
+ while (pobj) {
+ ObjOperandId protoId = writer.loadObject(pobj);
+
+ // Make sure the shape matches, to ensure the proto is unchanged and to
+ // avoid non-dense elements or anything else that is being checked by
+ // CanAttachDenseElementHole.
+ MOZ_ASSERT(pobj->is<NativeObject>());
+ writer.guardShape(protoId, pobj->shape());
+
+ // Also make sure there are no dense elements.
+ writer.guardNoDenseElements(protoId);
+
+ pobj = pobj->staticPrototype();
+ }
+}
+
+// Similar to |TestMatchingReceiver|, but for the holder object (when it
+// differs from the receiver). The holder may also be the expando of the
+// receiver if it exists.
+static void TestMatchingHolder(CacheIRWriter& writer, NativeObject* obj,
+ ObjOperandId objId) {
+ // The GeneratePrototypeGuards + TestMatchingHolder checks only support
+ // prototype chains composed of NativeObject (excluding the receiver
+ // itself).
+ writer.guardShapeForOwnProperties(objId, obj->shape());
+}
+
+enum class IsCrossCompartment { No, Yes };
+
+// Emit a shape guard for all objects on the proto chain. This does NOT include
+// the receiver; callers must ensure the receiver's proto is the first proto by
+// either emitting a shape guard or a prototype guard for |objId|.
+//
+// Note: this relies on shape implying proto.
+template <IsCrossCompartment MaybeCrossCompartment = IsCrossCompartment::No>
+static void ShapeGuardProtoChain(CacheIRWriter& writer, NativeObject* obj,
+ ObjOperandId objId) {
+ uint32_t depth = 0;
+ static const uint32_t MAX_CACHED_LOADS = 4;
+ ObjOperandId receiverObjId = objId;
+
+ while (true) {
+ JSObject* proto = obj->staticPrototype();
+ if (!proto) {
+ return;
+ }
+
+ obj = &proto->as<NativeObject>();
+
+ // After guarding the shape of an object, we can safely bake that
+ // object's proto into the stub data. Compared to LoadProto, this
+ // takes one load instead of three (object -> shape -> baseshape
+ // -> proto). We cap the depth to avoid bloating the size of the
+ // stub data. To avoid compartment mismatch, we skip this optimization
+ // in the cross-compartment case.
+ if (depth < MAX_CACHED_LOADS &&
+ MaybeCrossCompartment == IsCrossCompartment::No) {
+ objId = writer.loadProtoObject(obj, receiverObjId);
+ } else {
+ objId = writer.loadProto(objId);
+ }
+ depth++;
+
+ writer.guardShape(objId, obj->shape());
+ }
+}
+
+// For cross compartment guards we shape-guard the prototype chain to avoid
+// referencing the holder object.
+//
+// This peels off the first layer because it's guarded against obj == holder.
+//
+// Returns the holder's OperandId.
+static ObjOperandId ShapeGuardProtoChainForCrossCompartmentHolder(
+ CacheIRWriter& writer, NativeObject* obj, ObjOperandId objId,
+ NativeObject* holder) {
+ MOZ_ASSERT(obj != holder);
+ MOZ_ASSERT(holder);
+ while (true) {
+ MOZ_ASSERT(obj->staticPrototype());
+ obj = &obj->staticPrototype()->as<NativeObject>();
+
+ objId = writer.loadProto(objId);
+ if (obj == holder) {
+ TestMatchingHolder(writer, obj, objId);
+ return objId;
+ }
+ writer.guardShapeForOwnProperties(objId, obj->shape());
+ }
+}
+
+// Emit guards for reading a data property on |holder|. Returns the holder's
+// OperandId.
+template <IsCrossCompartment MaybeCrossCompartment = IsCrossCompartment::No>
+static ObjOperandId EmitReadSlotGuard(CacheIRWriter& writer, NativeObject* obj,
+ NativeObject* holder,
+ ObjOperandId objId) {
+ MOZ_ASSERT(holder);
+ TestMatchingNativeReceiver(writer, obj, objId);
+
+ if (obj == holder) {
+ return objId;
+ }
+
+ if (MaybeCrossCompartment == IsCrossCompartment::Yes) {
+ // Guard proto chain integrity.
+ // We use a variant of guards that avoid baking in any cross-compartment
+ // object pointers.
+ return ShapeGuardProtoChainForCrossCompartmentHolder(writer, obj, objId,
+ holder);
+ }
+
+ // Guard proto chain integrity.
+ GeneratePrototypeGuards(writer, obj, holder, objId);
+
+ // Guard on the holder's shape.
+ ObjOperandId holderId = writer.loadObject(holder);
+ TestMatchingHolder(writer, holder, holderId);
+ return holderId;
+}
+
+template <IsCrossCompartment MaybeCrossCompartment = IsCrossCompartment::No>
+static void EmitMissingPropGuard(CacheIRWriter& writer, NativeObject* obj,
+ ObjOperandId objId) {
+ TestMatchingNativeReceiver(writer, obj, objId);
+
+ // The property does not exist. Guard on everything in the prototype
+ // chain. This is guaranteed to see only Native objects because of
+ // CanAttachNativeGetProp().
+ ShapeGuardProtoChain<MaybeCrossCompartment>(writer, obj, objId);
+}
+
+template <IsCrossCompartment MaybeCrossCompartment = IsCrossCompartment::No>
+static void EmitReadSlotResult(CacheIRWriter& writer, NativeObject* obj,
+ NativeObject* holder, PropertyInfo prop,
+ ObjOperandId objId) {
+ MOZ_ASSERT(holder);
+
+ ObjOperandId holderId =
+ EmitReadSlotGuard<MaybeCrossCompartment>(writer, obj, holder, objId);
+
+ MOZ_ASSERT(holderId.valid());
+ EmitLoadSlotResult(writer, holderId, holder, prop);
+}
+
+template <IsCrossCompartment MaybeCrossCompartment = IsCrossCompartment::No>
+static void EmitMissingPropResult(CacheIRWriter& writer, NativeObject* obj,
+ ObjOperandId objId) {
+ EmitMissingPropGuard<MaybeCrossCompartment>(writer, obj, objId);
+ writer.loadUndefinedResult();
+}
+
+static void EmitCallGetterResultNoGuards(JSContext* cx, CacheIRWriter& writer,
+ NativeGetPropKind kind,
+ NativeObject* obj,
+ NativeObject* holder,
+ PropertyInfo prop,
+ ValOperandId receiverId) {
+ MOZ_ASSERT(IsCacheableGetPropCall(obj, holder, prop) == kind);
+
+ JSFunction* target = &holder->getGetter(prop)->as<JSFunction>();
+ bool sameRealm = cx->realm() == target->realm();
+
+ switch (kind) {
+ case NativeGetPropKind::NativeGetter: {
+ writer.callNativeGetterResult(receiverId, target, sameRealm);
+ writer.returnFromIC();
+ break;
+ }
+ case NativeGetPropKind::ScriptedGetter: {
+ writer.callScriptedGetterResult(receiverId, target, sameRealm);
+ writer.returnFromIC();
+ break;
+ }
+ default:
+ // CanAttachNativeGetProp guarantees that the getter is either a native or
+ // a scripted function.
+ MOZ_ASSERT_UNREACHABLE("Can't attach getter");
+ break;
+ }
+}
+
+// See the SMDOC comment in vm/GetterSetter.h for more info on Getter/Setter
+// properties
+static void EmitGuardGetterSetterSlot(CacheIRWriter& writer,
+ NativeObject* holder, PropertyInfo prop,
+ ObjOperandId holderId,
+ bool holderIsConstant = false) {
+ // If the holder is guaranteed to be the same object, and it never had a
+ // slot holding a GetterSetter mutated or deleted, its Shape will change when
+ // that does happen so we don't need to guard on the GetterSetter.
+ if (holderIsConstant && !holder->hadGetterSetterChange()) {
+ return;
+ }
+
+ size_t slot = prop.slot();
+ Value slotVal = holder->getSlot(slot);
+ MOZ_ASSERT(slotVal.isPrivateGCThing());
+
+ if (holder->isFixedSlot(slot)) {
+ size_t offset = NativeObject::getFixedSlotOffset(slot);
+ writer.guardFixedSlotValue(holderId, offset, slotVal);
+ } else {
+ size_t offset = holder->dynamicSlotIndex(slot) * sizeof(Value);
+ writer.guardDynamicSlotValue(holderId, offset, slotVal);
+ }
+}
+
+static void EmitCallGetterResultGuards(CacheIRWriter& writer, NativeObject* obj,
+ NativeObject* holder, HandleId id,
+ PropertyInfo prop, ObjOperandId objId,
+ ICState::Mode mode) {
+ // Use the megamorphic guard if we're in megamorphic mode, except if |obj|
+ // is a Window as GuardHasGetterSetter doesn't support this yet (Window may
+ // require outerizing).
+
+ MOZ_ASSERT(holder->containsPure(id, prop));
+
+ if (mode == ICState::Mode::Specialized || IsWindow(obj)) {
+ TestMatchingNativeReceiver(writer, obj, objId);
+
+ if (obj != holder) {
+ GeneratePrototypeGuards(writer, obj, holder, objId);
+
+ // Guard on the holder's shape.
+ ObjOperandId holderId = writer.loadObject(holder);
+ TestMatchingHolder(writer, holder, holderId);
+
+ EmitGuardGetterSetterSlot(writer, holder, prop, holderId,
+ /* holderIsConstant = */ true);
+ } else {
+ EmitGuardGetterSetterSlot(writer, holder, prop, objId);
+ }
+ } else {
+ GetterSetter* gs = holder->getGetterSetter(prop);
+ writer.guardHasGetterSetter(objId, id, gs);
+ }
+}
+
+static void EmitCallGetterResult(JSContext* cx, CacheIRWriter& writer,
+ NativeGetPropKind kind, NativeObject* obj,
+ NativeObject* holder, HandleId id,
+ PropertyInfo prop, ObjOperandId objId,
+ ValOperandId receiverId, ICState::Mode mode) {
+ EmitCallGetterResultGuards(writer, obj, holder, id, prop, objId, mode);
+ EmitCallGetterResultNoGuards(cx, writer, kind, obj, holder, prop, receiverId);
+}
+
+static bool CanAttachDOMCall(JSContext* cx, JSJitInfo::OpType type,
+ JSObject* obj, JSFunction* fun,
+ ICState::Mode mode) {
+ MOZ_ASSERT(type == JSJitInfo::Getter || type == JSJitInfo::Setter ||
+ type == JSJitInfo::Method);
+
+ if (mode != ICState::Mode::Specialized) {
+ return false;
+ }
+
+ if (!fun->hasJitInfo()) {
+ return false;
+ }
+
+ if (cx->realm() != fun->realm()) {
+ return false;
+ }
+
+ const JSJitInfo* jitInfo = fun->jitInfo();
+ if (jitInfo->type() != type) {
+ return false;
+ }
+
+ MOZ_ASSERT_IF(IsWindow(obj), !jitInfo->needsOuterizedThisObject());
+
+ const JSClass* clasp = obj->getClass();
+ if (!clasp->isDOMClass()) {
+ return false;
+ }
+
+ if (type != JSJitInfo::Method && clasp->isProxyObject()) {
+ return false;
+ }
+
+ // Tell the analysis the |DOMInstanceClassHasProtoAtDepth| hook can't GC.
+ JS::AutoSuppressGCAnalysis nogc;
+
+ DOMInstanceClassHasProtoAtDepth instanceChecker =
+ cx->runtime()->DOMcallbacks->instanceClassMatchesProto;
+ return instanceChecker(clasp, jitInfo->protoID, jitInfo->depth);
+}
+
+static bool CanAttachDOMGetterSetter(JSContext* cx, JSJitInfo::OpType type,
+ NativeObject* obj, NativeObject* holder,
+ PropertyInfo prop, ICState::Mode mode) {
+ MOZ_ASSERT(type == JSJitInfo::Getter || type == JSJitInfo::Setter);
+
+ JSObject* accessor = type == JSJitInfo::Getter ? holder->getGetter(prop)
+ : holder->getSetter(prop);
+ JSFunction* fun = &accessor->as<JSFunction>();
+
+ return CanAttachDOMCall(cx, type, obj, fun, mode);
+}
+
+static void EmitCallDOMGetterResultNoGuards(CacheIRWriter& writer,
+ NativeObject* holder,
+ PropertyInfo prop,
+ ObjOperandId objId) {
+ JSFunction* getter = &holder->getGetter(prop)->as<JSFunction>();
+ writer.callDOMGetterResult(objId, getter->jitInfo());
+ writer.returnFromIC();
+}
+
+static void EmitCallDOMGetterResult(JSContext* cx, CacheIRWriter& writer,
+ NativeObject* obj, NativeObject* holder,
+ HandleId id, PropertyInfo prop,
+ ObjOperandId objId) {
+ // Note: this relies on EmitCallGetterResultGuards emitting a shape guard
+ // for specialized stubs.
+ // The shape guard ensures the receiver's Class is valid for this DOM getter.
+ EmitCallGetterResultGuards(writer, obj, holder, id, prop, objId,
+ ICState::Mode::Specialized);
+ EmitCallDOMGetterResultNoGuards(writer, holder, prop, objId);
+}
+
+void GetPropIRGenerator::attachMegamorphicNativeSlot(ObjOperandId objId,
+ jsid id) {
+ MOZ_ASSERT(mode_ == ICState::Mode::Megamorphic);
+
+ // We don't support GetBoundName because environment objects have
+ // lookupProperty hooks and GetBoundName is usually not megamorphic.
+ MOZ_ASSERT(JSOp(*pc_) != JSOp::GetBoundName);
+
+ if (cacheKind_ == CacheKind::GetProp ||
+ cacheKind_ == CacheKind::GetPropSuper) {
+ writer.megamorphicLoadSlotResult(objId, id.toAtom()->asPropertyName());
+ } else {
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetElem ||
+ cacheKind_ == CacheKind::GetElemSuper);
+ writer.megamorphicLoadSlotByValueResult(objId, getElemKeyValueId());
+ }
+ writer.returnFromIC();
+
+ trackAttached("MegamorphicNativeSlot");
+}
+
+AttachDecision GetPropIRGenerator::tryAttachNative(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id,
+ ValOperandId receiverId) {
+ Maybe<PropertyInfo> prop;
+ NativeObject* holder = nullptr;
+
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, obj, id, &holder, &prop, pc_);
+ switch (kind) {
+ case NativeGetPropKind::None:
+ return AttachDecision::NoAction;
+ case NativeGetPropKind::Missing:
+ case NativeGetPropKind::Slot: {
+ auto* nobj = &obj->as<NativeObject>();
+
+ if (mode_ == ICState::Mode::Megamorphic &&
+ JSOp(*pc_) != JSOp::GetBoundName) {
+ attachMegamorphicNativeSlot(objId, id);
+ return AttachDecision::Attach;
+ }
+
+ maybeEmitIdGuard(id);
+ if (kind == NativeGetPropKind::Slot) {
+ EmitReadSlotResult(writer, nobj, holder, *prop, objId);
+ writer.returnFromIC();
+ trackAttached("NativeSlot");
+ } else {
+ EmitMissingPropResult(writer, nobj, objId);
+ writer.returnFromIC();
+ trackAttached("Missing");
+ }
+ return AttachDecision::Attach;
+ }
+ case NativeGetPropKind::ScriptedGetter:
+ case NativeGetPropKind::NativeGetter: {
+ auto* nobj = &obj->as<NativeObject>();
+
+ maybeEmitIdGuard(id);
+
+ if (!isSuper() && CanAttachDOMGetterSetter(cx_, JSJitInfo::Getter, nobj,
+ holder, *prop, mode_)) {
+ EmitCallDOMGetterResult(cx_, writer, nobj, holder, id, *prop, objId);
+
+ trackAttached("DOMGetter");
+ return AttachDecision::Attach;
+ }
+
+ EmitCallGetterResult(cx_, writer, kind, nobj, holder, id, *prop, objId,
+ receiverId, mode_);
+
+ trackAttached("NativeGetter");
+ return AttachDecision::Attach;
+ }
+ }
+
+ MOZ_CRASH("Bad NativeGetPropKind");
+}
+
+// Returns whether obj is a WindowProxy wrapping the script's global.
+static bool IsWindowProxyForScriptGlobal(JSScript* script, JSObject* obj) {
+ if (!IsWindowProxy(obj)) {
+ return false;
+ }
+
+ MOZ_ASSERT(obj->getClass() ==
+ script->runtimeFromMainThread()->maybeWindowProxyClass());
+
+ JSObject* window = ToWindowIfWindowProxy(obj);
+
+ // Ion relies on the WindowProxy's group changing (and the group getting
+ // marked as having unknown properties) on navigation. If we ever stop
+ // transplanting same-compartment WindowProxies, this assert will fail and we
+ // need to fix that code.
+ MOZ_ASSERT(window == &obj->nonCCWGlobal());
+
+ // This must be a WindowProxy for a global in this compartment. Else it would
+ // be a cross-compartment wrapper and IsWindowProxy returns false for
+ // those.
+ MOZ_ASSERT(script->compartment() == obj->compartment());
+
+ // Only optimize lookups on the WindowProxy for the current global. Other
+ // WindowProxies in the compartment may require security checks (based on
+ // mutable document.domain). See bug 1516775.
+ return window == &script->global();
+}
+
+// Guards objId is a WindowProxy for windowObj. Returns the window's operand id.
+static ObjOperandId GuardAndLoadWindowProxyWindow(CacheIRWriter& writer,
+ ObjOperandId objId,
+ GlobalObject* windowObj) {
+ writer.guardClass(objId, GuardClassKind::WindowProxy);
+ ObjOperandId windowObjId = writer.loadWrapperTarget(objId);
+ writer.guardSpecificObject(windowObjId, windowObj);
+ return windowObjId;
+}
+
+// Whether a getter/setter on the global should have the WindowProxy as |this|
+// value instead of the Window (the global object). This always returns true for
+// scripted functions.
+static bool GetterNeedsWindowProxyThis(NativeObject* holder,
+ PropertyInfo prop) {
+ JSFunction* callee = &holder->getGetter(prop)->as<JSFunction>();
+ return !callee->hasJitInfo() || callee->jitInfo()->needsOuterizedThisObject();
+}
+static bool SetterNeedsWindowProxyThis(NativeObject* holder,
+ PropertyInfo prop) {
+ JSFunction* callee = &holder->getSetter(prop)->as<JSFunction>();
+ return !callee->hasJitInfo() || callee->jitInfo()->needsOuterizedThisObject();
+}
+
+AttachDecision GetPropIRGenerator::tryAttachWindowProxy(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ // Attach a stub when the receiver is a WindowProxy and we can do the lookup
+ // on the Window (the global object).
+
+ if (!IsWindowProxyForScriptGlobal(script_, obj)) {
+ return AttachDecision::NoAction;
+ }
+
+ // If we're megamorphic prefer a generic proxy stub that handles a lot more
+ // cases.
+ if (mode_ == ICState::Mode::Megamorphic) {
+ return AttachDecision::NoAction;
+ }
+
+ // Now try to do the lookup on the Window (the current global).
+ GlobalObject* windowObj = cx_->global();
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, windowObj, id, &holder, &prop, pc_);
+ switch (kind) {
+ case NativeGetPropKind::None:
+ return AttachDecision::NoAction;
+
+ case NativeGetPropKind::Slot: {
+ maybeEmitIdGuard(id);
+ ObjOperandId windowObjId =
+ GuardAndLoadWindowProxyWindow(writer, objId, windowObj);
+ EmitReadSlotResult(writer, windowObj, holder, *prop, windowObjId);
+ writer.returnFromIC();
+
+ trackAttached("WindowProxySlot");
+ return AttachDecision::Attach;
+ }
+
+ case NativeGetPropKind::Missing: {
+ maybeEmitIdGuard(id);
+ ObjOperandId windowObjId =
+ GuardAndLoadWindowProxyWindow(writer, objId, windowObj);
+ EmitMissingPropResult(writer, windowObj, windowObjId);
+ writer.returnFromIC();
+
+ trackAttached("WindowProxyMissing");
+ return AttachDecision::Attach;
+ }
+
+ case NativeGetPropKind::NativeGetter:
+ case NativeGetPropKind::ScriptedGetter: {
+ // If a |super| access, it is not worth the complexity to attach an IC.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ bool needsWindowProxy = GetterNeedsWindowProxyThis(holder, *prop);
+
+ // Guard the incoming object is a WindowProxy and inline a getter call
+ // based on the Window object.
+ maybeEmitIdGuard(id);
+ ObjOperandId windowObjId =
+ GuardAndLoadWindowProxyWindow(writer, objId, windowObj);
+
+ if (CanAttachDOMGetterSetter(cx_, JSJitInfo::Getter, windowObj, holder,
+ *prop, mode_)) {
+ MOZ_ASSERT(!needsWindowProxy);
+ EmitCallDOMGetterResult(cx_, writer, windowObj, holder, id, *prop,
+ windowObjId);
+ trackAttached("WindowProxyDOMGetter");
+ } else {
+ ValOperandId receiverId =
+ writer.boxObject(needsWindowProxy ? objId : windowObjId);
+ EmitCallGetterResult(cx_, writer, kind, windowObj, holder, id, *prop,
+ windowObjId, receiverId, mode_);
+ trackAttached("WindowProxyGetter");
+ }
+
+ return AttachDecision::Attach;
+ }
+ }
+
+ MOZ_CRASH("Unreachable");
+}
+
+AttachDecision GetPropIRGenerator::tryAttachCrossCompartmentWrapper(
+ HandleObject obj, ObjOperandId objId, HandleId id) {
+ // We can only optimize this very wrapper-handler, because others might
+ // have a security policy.
+ if (!IsWrapper(obj) ||
+ Wrapper::wrapperHandler(obj) != &CrossCompartmentWrapper::singleton) {
+ return AttachDecision::NoAction;
+ }
+
+ // If we're megamorphic prefer a generic proxy stub that handles a lot more
+ // cases.
+ if (mode_ == ICState::Mode::Megamorphic) {
+ return AttachDecision::NoAction;
+ }
+
+ RootedObject unwrapped(cx_, Wrapper::wrappedObject(obj));
+ MOZ_ASSERT(unwrapped == UnwrapOneCheckedStatic(obj));
+ MOZ_ASSERT(!IsCrossCompartmentWrapper(unwrapped),
+ "CCWs must not wrap other CCWs");
+
+ // If we allowed different zones we would have to wrap strings.
+ if (unwrapped->compartment()->zone() != cx_->compartment()->zone()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Take the unwrapped object's global, and wrap in a
+ // this-compartment wrapper. This is what will be stored in the IC
+ // keep the compartment alive.
+ RootedObject wrappedTargetGlobal(cx_, &unwrapped->nonCCWGlobal());
+ if (!cx_->compartment()->wrap(cx_, &wrappedTargetGlobal)) {
+ cx_->clearPendingException();
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+
+ // Enter realm of target to prevent failing compartment assertions when doing
+ // the lookup.
+ {
+ AutoRealm ar(cx_, unwrapped);
+
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, unwrapped, id, &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::Slot && kind != NativeGetPropKind::Missing) {
+ return AttachDecision::NoAction;
+ }
+ }
+ auto* unwrappedNative = &unwrapped->as<NativeObject>();
+
+ maybeEmitIdGuard(id);
+ writer.guardIsProxy(objId);
+ writer.guardHasProxyHandler(objId, Wrapper::wrapperHandler(obj));
+
+ // Load the object wrapped by the CCW
+ ObjOperandId wrapperTargetId = writer.loadWrapperTarget(objId);
+
+ // If the compartment of the wrapped object is different we should fail.
+ writer.guardCompartment(wrapperTargetId, wrappedTargetGlobal,
+ unwrappedNative->compartment());
+
+ ObjOperandId unwrappedId = wrapperTargetId;
+ if (holder) {
+ EmitReadSlotResult<IsCrossCompartment::Yes>(writer, unwrappedNative, holder,
+ *prop, unwrappedId);
+ writer.wrapResult();
+ writer.returnFromIC();
+ trackAttached("CCWSlot");
+ } else {
+ EmitMissingPropResult<IsCrossCompartment::Yes>(writer, unwrappedNative,
+ unwrappedId);
+ writer.returnFromIC();
+ trackAttached("CCWMissing");
+ }
+ return AttachDecision::Attach;
+}
+
+static JSObject* NewWrapperWithObjectShape(JSContext* cx,
+ Handle<NativeObject*> obj);
+
+static bool GetXrayExpandoShapeWrapper(JSContext* cx, HandleObject xray,
+ MutableHandleObject wrapper) {
+ Value v = GetProxyReservedSlot(xray, GetXrayJitInfo()->xrayHolderSlot);
+ if (v.isObject()) {
+ NativeObject* holder = &v.toObject().as<NativeObject>();
+ v = holder->getFixedSlot(GetXrayJitInfo()->holderExpandoSlot);
+ if (v.isObject()) {
+ Rooted<NativeObject*> expando(
+ cx, &UncheckedUnwrap(&v.toObject())->as<NativeObject>());
+ wrapper.set(NewWrapperWithObjectShape(cx, expando));
+ return wrapper != nullptr;
+ }
+ }
+ wrapper.set(nullptr);
+ return true;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachXrayCrossCompartmentWrapper(
+ HandleObject obj, ObjOperandId objId, HandleId id,
+ ValOperandId receiverId) {
+ if (!obj->is<ProxyObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ JS::XrayJitInfo* info = GetXrayJitInfo();
+ if (!info || !info->isCrossCompartmentXray(GetProxyHandler(obj))) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!info->compartmentHasExclusiveExpandos(obj)) {
+ return AttachDecision::NoAction;
+ }
+
+ RootedObject target(cx_, UncheckedUnwrap(obj));
+
+ RootedObject expandoShapeWrapper(cx_);
+ if (!GetXrayExpandoShapeWrapper(cx_, obj, &expandoShapeWrapper)) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ // Look for a getter we can call on the xray or its prototype chain.
+ Rooted<Maybe<PropertyDescriptor>> desc(cx_);
+ RootedObject holder(cx_, obj);
+ RootedObjectVector prototypes(cx_);
+ RootedObjectVector prototypeExpandoShapeWrappers(cx_);
+ while (true) {
+ if (!GetOwnPropertyDescriptor(cx_, holder, id, &desc)) {
+ cx_->clearPendingException();
+ return AttachDecision::NoAction;
+ }
+ if (desc.isSome()) {
+ break;
+ }
+ if (!GetPrototype(cx_, holder, &holder)) {
+ cx_->clearPendingException();
+ return AttachDecision::NoAction;
+ }
+ if (!holder || !holder->is<ProxyObject>() ||
+ !info->isCrossCompartmentXray(GetProxyHandler(holder))) {
+ return AttachDecision::NoAction;
+ }
+ RootedObject prototypeExpandoShapeWrapper(cx_);
+ if (!GetXrayExpandoShapeWrapper(cx_, holder,
+ &prototypeExpandoShapeWrapper) ||
+ !prototypes.append(holder) ||
+ !prototypeExpandoShapeWrappers.append(prototypeExpandoShapeWrapper)) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+ }
+ if (!desc->isAccessorDescriptor()) {
+ return AttachDecision::NoAction;
+ }
+
+ RootedObject getter(cx_, desc->getter());
+ if (!getter || !getter->is<JSFunction>() ||
+ !getter->as<JSFunction>().isNativeWithoutJitEntry()) {
+ return AttachDecision::NoAction;
+ }
+
+ maybeEmitIdGuard(id);
+ writer.guardIsProxy(objId);
+ writer.guardHasProxyHandler(objId, GetProxyHandler(obj));
+
+ // Load the object wrapped by the CCW
+ ObjOperandId wrapperTargetId = writer.loadWrapperTarget(objId);
+
+ // Test the wrapped object's class. The properties held by xrays or their
+ // prototypes will be invariant for objects of a given class, except for
+ // changes due to xray expandos or xray prototype mutations.
+ writer.guardAnyClass(wrapperTargetId, target->getClass());
+
+ // Make sure the expandos on the xray and its prototype chain match up with
+ // what we expect. The expando shape needs to be consistent, to ensure it
+ // has not had any shadowing properties added, and the expando cannot have
+ // any custom prototype (xray prototypes are stable otherwise).
+ //
+ // We can only do this for xrays with exclusive access to their expandos
+ // (as we checked earlier), which store a pointer to their expando
+ // directly. Xrays in other compartments may share their expandos with each
+ // other and a VM call is needed just to find the expando.
+ if (expandoShapeWrapper) {
+ writer.guardXrayExpandoShapeAndDefaultProto(objId, expandoShapeWrapper);
+ } else {
+ writer.guardXrayNoExpando(objId);
+ }
+ for (size_t i = 0; i < prototypes.length(); i++) {
+ JSObject* proto = prototypes[i];
+ ObjOperandId protoId = writer.loadObject(proto);
+ if (JSObject* protoShapeWrapper = prototypeExpandoShapeWrappers[i]) {
+ writer.guardXrayExpandoShapeAndDefaultProto(protoId, protoShapeWrapper);
+ } else {
+ writer.guardXrayNoExpando(protoId);
+ }
+ }
+
+ bool sameRealm = cx_->realm() == getter->as<JSFunction>().realm();
+ writer.callNativeGetterResult(receiverId, &getter->as<JSFunction>(),
+ sameRealm);
+ writer.returnFromIC();
+
+ trackAttached("XrayGetter");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachGenericProxy(
+ Handle<ProxyObject*> obj, ObjOperandId objId, HandleId id,
+ bool handleDOMProxies) {
+ writer.guardIsProxy(objId);
+
+ if (!handleDOMProxies) {
+ // Ensure that the incoming object is not a DOM proxy, so that we can get to
+ // the specialized stubs
+ writer.guardIsNotDOMProxy(objId);
+ }
+
+ if (cacheKind_ == CacheKind::GetProp || mode_ == ICState::Mode::Specialized) {
+ MOZ_ASSERT(!isSuper());
+ maybeEmitIdGuard(id);
+ writer.proxyGetResult(objId, id);
+ } else {
+ // Attach a stub that handles every id.
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetElem);
+ MOZ_ASSERT(mode_ == ICState::Mode::Megamorphic);
+ MOZ_ASSERT(!isSuper());
+ writer.proxyGetByValueResult(objId, getElemKeyValueId());
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("GenericProxy");
+ return AttachDecision::Attach;
+}
+
+static bool ValueIsInt64Index(const Value& val, int64_t* index) {
+ // Try to convert the Value to a TypedArray index or DataView offset.
+
+ if (val.isInt32()) {
+ *index = val.toInt32();
+ return true;
+ }
+
+ if (val.isDouble()) {
+ // Use NumberEqualsInt64 because ToPropertyKey(-0) is 0.
+ return mozilla::NumberEqualsInt64(val.toDouble(), index);
+ }
+
+ return false;
+}
+
+IntPtrOperandId IRGenerator::guardToIntPtrIndex(const Value& index,
+ ValOperandId indexId,
+ bool supportOOB) {
+#ifdef DEBUG
+ int64_t indexInt64;
+ MOZ_ASSERT_IF(!supportOOB, ValueIsInt64Index(index, &indexInt64));
+#endif
+
+ if (index.isInt32()) {
+ Int32OperandId int32IndexId = writer.guardToInt32(indexId);
+ return writer.int32ToIntPtr(int32IndexId);
+ }
+
+ MOZ_ASSERT(index.isNumber());
+ NumberOperandId numberIndexId = writer.guardIsNumber(indexId);
+ return writer.guardNumberToIntPtrIndex(numberIndexId, supportOOB);
+}
+
+ObjOperandId IRGenerator::guardDOMProxyExpandoObjectAndShape(
+ ProxyObject* obj, ObjOperandId objId, const Value& expandoVal,
+ NativeObject* expandoObj) {
+ MOZ_ASSERT(IsCacheableDOMProxy(obj));
+
+ TestMatchingProxyReceiver(writer, obj, objId);
+
+ // Shape determines Class, so now it must be a DOM proxy.
+ ValOperandId expandoValId;
+ if (expandoVal.isObject()) {
+ expandoValId = writer.loadDOMExpandoValue(objId);
+ } else {
+ expandoValId = writer.loadDOMExpandoValueIgnoreGeneration(objId);
+ }
+
+ // Guard the expando is an object and shape guard.
+ ObjOperandId expandoObjId = writer.guardToObject(expandoValId);
+ TestMatchingHolder(writer, expandoObj, expandoObjId);
+ return expandoObjId;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachDOMProxyExpando(
+ Handle<ProxyObject*> obj, ObjOperandId objId, HandleId id,
+ ValOperandId receiverId) {
+ MOZ_ASSERT(IsCacheableDOMProxy(obj));
+
+ Value expandoVal = GetProxyPrivate(obj);
+ JSObject* expandoObj;
+ if (expandoVal.isObject()) {
+ expandoObj = &expandoVal.toObject();
+ } else {
+ MOZ_ASSERT(!expandoVal.isUndefined(),
+ "How did a missing expando manage to shadow things?");
+ auto expandoAndGeneration =
+ static_cast<ExpandoAndGeneration*>(expandoVal.toPrivate());
+ MOZ_ASSERT(expandoAndGeneration);
+ expandoObj = &expandoAndGeneration->expando.toObject();
+ }
+
+ // Try to do the lookup on the expando object.
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, expandoObj, id, &holder, &prop, pc_);
+ if (kind == NativeGetPropKind::None) {
+ return AttachDecision::NoAction;
+ }
+ if (!holder) {
+ return AttachDecision::NoAction;
+ }
+ auto* nativeExpandoObj = &expandoObj->as<NativeObject>();
+
+ MOZ_ASSERT(holder == nativeExpandoObj);
+
+ maybeEmitIdGuard(id);
+ ObjOperandId expandoObjId = guardDOMProxyExpandoObjectAndShape(
+ obj, objId, expandoVal, nativeExpandoObj);
+
+ if (kind == NativeGetPropKind::Slot) {
+ // Load from the expando's slots.
+ EmitLoadSlotResult(writer, expandoObjId, nativeExpandoObj, *prop);
+ writer.returnFromIC();
+ } else {
+ // Call the getter. Note that we pass objId, the DOM proxy, as |this|
+ // and not the expando object.
+ MOZ_ASSERT(kind == NativeGetPropKind::NativeGetter ||
+ kind == NativeGetPropKind::ScriptedGetter);
+ EmitGuardGetterSetterSlot(writer, nativeExpandoObj, *prop, expandoObjId);
+ EmitCallGetterResultNoGuards(cx_, writer, kind, nativeExpandoObj,
+ nativeExpandoObj, *prop, receiverId);
+ }
+
+ trackAttached("DOMProxyExpando");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachDOMProxyShadowed(
+ Handle<ProxyObject*> obj, ObjOperandId objId, HandleId id) {
+ MOZ_ASSERT(!isSuper());
+ MOZ_ASSERT(IsCacheableDOMProxy(obj));
+
+ maybeEmitIdGuard(id);
+ TestMatchingProxyReceiver(writer, obj, objId);
+ writer.proxyGetResult(objId, id);
+ writer.returnFromIC();
+
+ trackAttached("DOMProxyShadowed");
+ return AttachDecision::Attach;
+}
+
+// Callers are expected to have already guarded on the shape of the
+// object, which guarantees the object is a DOM proxy.
+static void CheckDOMProxyExpandoDoesNotShadow(CacheIRWriter& writer,
+ ProxyObject* obj, jsid id,
+ ObjOperandId objId) {
+ MOZ_ASSERT(IsCacheableDOMProxy(obj));
+
+ Value expandoVal = GetProxyPrivate(obj);
+
+ ValOperandId expandoId;
+ if (!expandoVal.isObject() && !expandoVal.isUndefined()) {
+ auto expandoAndGeneration =
+ static_cast<ExpandoAndGeneration*>(expandoVal.toPrivate());
+ uint64_t generation = expandoAndGeneration->generation;
+ expandoId = writer.loadDOMExpandoValueGuardGeneration(
+ objId, expandoAndGeneration, generation);
+ expandoVal = expandoAndGeneration->expando;
+ } else {
+ expandoId = writer.loadDOMExpandoValue(objId);
+ }
+
+ if (expandoVal.isUndefined()) {
+ // Guard there's no expando object.
+ writer.guardNonDoubleType(expandoId, ValueType::Undefined);
+ } else if (expandoVal.isObject()) {
+ // Guard the proxy either has no expando object or, if it has one, that
+ // the shape matches the current expando object.
+ NativeObject& expandoObj = expandoVal.toObject().as<NativeObject>();
+ MOZ_ASSERT(!expandoObj.containsPure(id));
+ writer.guardDOMExpandoMissingOrGuardShape(expandoId, expandoObj.shape());
+ } else {
+ MOZ_CRASH("Invalid expando value");
+ }
+}
+
+AttachDecision GetPropIRGenerator::tryAttachDOMProxyUnshadowed(
+ Handle<ProxyObject*> obj, ObjOperandId objId, HandleId id,
+ ValOperandId receiverId) {
+ MOZ_ASSERT(IsCacheableDOMProxy(obj));
+
+ JSObject* checkObj = obj->staticPrototype();
+ if (!checkObj) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, checkObj, id, &holder, &prop, pc_);
+ if (kind == NativeGetPropKind::None) {
+ return AttachDecision::NoAction;
+ }
+ auto* nativeCheckObj = &checkObj->as<NativeObject>();
+
+ maybeEmitIdGuard(id);
+
+ // Guard that our expando object hasn't started shadowing this property.
+ TestMatchingProxyReceiver(writer, obj, objId);
+ CheckDOMProxyExpandoDoesNotShadow(writer, obj, id, objId);
+
+ if (holder) {
+ // Found the property on the prototype chain. Treat it like a native
+ // getprop.
+ GeneratePrototypeGuards(writer, obj, holder, objId);
+
+ // Guard on the holder of the property.
+ ObjOperandId holderId = writer.loadObject(holder);
+ TestMatchingHolder(writer, holder, holderId);
+
+ if (kind == NativeGetPropKind::Slot) {
+ EmitLoadSlotResult(writer, holderId, holder, *prop);
+ writer.returnFromIC();
+ } else {
+ // EmitCallGetterResultNoGuards expects |obj| to be the object the
+ // property is on to do some checks. Since we actually looked at
+ // checkObj, and no extra guards will be generated, we can just
+ // pass that instead.
+ MOZ_ASSERT(kind == NativeGetPropKind::NativeGetter ||
+ kind == NativeGetPropKind::ScriptedGetter);
+ MOZ_ASSERT(!isSuper());
+ EmitGuardGetterSetterSlot(writer, holder, *prop, holderId,
+ /* holderIsConstant = */ true);
+ EmitCallGetterResultNoGuards(cx_, writer, kind, nativeCheckObj, holder,
+ *prop, receiverId);
+ }
+ } else {
+ // Property was not found on the prototype chain. Deoptimize down to
+ // proxy get call.
+ MOZ_ASSERT(kind == NativeGetPropKind::Missing);
+ MOZ_ASSERT(!isSuper());
+ writer.proxyGetResult(objId, id);
+ writer.returnFromIC();
+ }
+
+ trackAttached("DOMProxyUnshadowed");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachProxy(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id,
+ ValOperandId receiverId) {
+ ProxyStubType type = GetProxyStubType(cx_, obj, id);
+ if (type == ProxyStubType::None) {
+ return AttachDecision::NoAction;
+ }
+ auto proxy = obj.as<ProxyObject>();
+
+ // The proxy stubs don't currently support |super| access.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (mode_ == ICState::Mode::Megamorphic) {
+ return tryAttachGenericProxy(proxy, objId, id,
+ /* handleDOMProxies = */ true);
+ }
+
+ switch (type) {
+ case ProxyStubType::None:
+ break;
+ case ProxyStubType::DOMExpando:
+ TRY_ATTACH(tryAttachDOMProxyExpando(proxy, objId, id, receiverId));
+ [[fallthrough]]; // Fall through to the generic shadowed case.
+ case ProxyStubType::DOMShadowed:
+ return tryAttachDOMProxyShadowed(proxy, objId, id);
+ case ProxyStubType::DOMUnshadowed:
+ TRY_ATTACH(tryAttachDOMProxyUnshadowed(proxy, objId, id, receiverId));
+ return tryAttachGenericProxy(proxy, objId, id,
+ /* handleDOMProxies = */ true);
+ case ProxyStubType::Generic:
+ return tryAttachGenericProxy(proxy, objId, id,
+ /* handleDOMProxies = */ false);
+ }
+
+ MOZ_CRASH("Unexpected ProxyStubType");
+}
+
+AttachDecision GetPropIRGenerator::tryAttachObjectLength(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ if (!id.isAtom(cx_->names().length)) {
+ return AttachDecision::NoAction;
+ }
+
+ if (obj->is<ArrayObject>()) {
+ if (obj->as<ArrayObject>().length() > INT32_MAX) {
+ return AttachDecision::NoAction;
+ }
+
+ maybeEmitIdGuard(id);
+ writer.guardClass(objId, GuardClassKind::Array);
+ writer.loadInt32ArrayLengthResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("ArrayLength");
+ return AttachDecision::Attach;
+ }
+
+ if (obj->is<ArgumentsObject>() &&
+ !obj->as<ArgumentsObject>().hasOverriddenLength()) {
+ maybeEmitIdGuard(id);
+ if (obj->is<MappedArgumentsObject>()) {
+ writer.guardClass(objId, GuardClassKind::MappedArguments);
+ } else {
+ MOZ_ASSERT(obj->is<UnmappedArgumentsObject>());
+ writer.guardClass(objId, GuardClassKind::UnmappedArguments);
+ }
+ writer.loadArgumentsObjectLengthResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("ArgumentsObjectLength");
+ return AttachDecision::Attach;
+ }
+
+ return AttachDecision::NoAction;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachTypedArray(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ if (!obj->is<TypedArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (mode_ != ICState::Mode::Specialized) {
+ return AttachDecision::NoAction;
+ }
+
+ // Receiver should be the object.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ bool isLength = id.isAtom(cx_->names().length);
+ bool isByteOffset = id.isAtom(cx_->names().byteOffset);
+ if (!isLength && !isByteOffset && !id.isAtom(cx_->names().byteLength)) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, obj, id, &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::NativeGetter) {
+ return AttachDecision::NoAction;
+ }
+
+ JSFunction& fun = holder->getGetter(*prop)->as<JSFunction>();
+ if (isLength) {
+ if (!TypedArrayObject::isOriginalLengthGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+ } else if (isByteOffset) {
+ if (!TypedArrayObject::isOriginalByteOffsetGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+ } else {
+ if (!TypedArrayObject::isOriginalByteLengthGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+ }
+
+ auto* tarr = &obj->as<TypedArrayObject>();
+
+ maybeEmitIdGuard(id);
+ // Emit all the normal guards for calling this native, but specialize
+ // callNativeGetterResult.
+ EmitCallGetterResultGuards(writer, tarr, holder, id, *prop, objId, mode_);
+ if (isLength) {
+ if (tarr->length() <= INT32_MAX) {
+ writer.loadArrayBufferViewLengthInt32Result(objId);
+ } else {
+ writer.loadArrayBufferViewLengthDoubleResult(objId);
+ }
+ trackAttached("TypedArrayLength");
+ } else if (isByteOffset) {
+ if (tarr->byteOffset() <= INT32_MAX) {
+ writer.arrayBufferViewByteOffsetInt32Result(objId);
+ } else {
+ writer.arrayBufferViewByteOffsetDoubleResult(objId);
+ }
+ trackAttached("TypedArrayByteOffset");
+ } else {
+ if (tarr->byteLength() <= INT32_MAX) {
+ writer.typedArrayByteLengthInt32Result(objId);
+ } else {
+ writer.typedArrayByteLengthDoubleResult(objId);
+ }
+ trackAttached("TypedArrayByteLength");
+ }
+ writer.returnFromIC();
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachDataView(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ if (!obj->is<DataViewObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* dv = &obj->as<DataViewObject>();
+
+ if (mode_ != ICState::Mode::Specialized) {
+ return AttachDecision::NoAction;
+ }
+
+ // Receiver should be the object.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ bool isByteOffset = id.isAtom(cx_->names().byteOffset);
+ if (!isByteOffset && !id.isAtom(cx_->names().byteLength)) {
+ return AttachDecision::NoAction;
+ }
+
+ // byteOffset and byteLength both throw when the ArrayBuffer is detached.
+ if (dv->hasDetachedBuffer()) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, obj, id, &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::NativeGetter) {
+ return AttachDecision::NoAction;
+ }
+
+ auto& fun = holder->getGetter(*prop)->as<JSFunction>();
+ if (isByteOffset) {
+ if (!DataViewObject::isOriginalByteOffsetGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+ } else {
+ if (!DataViewObject::isOriginalByteLengthGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+ }
+
+ maybeEmitIdGuard(id);
+ // Emit all the normal guards for calling this native, but specialize
+ // callNativeGetterResult.
+ EmitCallGetterResultGuards(writer, dv, holder, id, *prop, objId, mode_);
+ writer.guardHasAttachedArrayBuffer(objId);
+ if (isByteOffset) {
+ if (dv->byteOffset() <= INT32_MAX) {
+ writer.arrayBufferViewByteOffsetInt32Result(objId);
+ } else {
+ writer.arrayBufferViewByteOffsetDoubleResult(objId);
+ }
+ trackAttached("DataViewByteOffset");
+ } else {
+ if (dv->byteLength() <= INT32_MAX) {
+ writer.loadArrayBufferViewLengthInt32Result(objId);
+ } else {
+ writer.loadArrayBufferViewLengthDoubleResult(objId);
+ }
+ trackAttached("DataViewByteLength");
+ }
+ writer.returnFromIC();
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachArrayBufferMaybeShared(
+ HandleObject obj, ObjOperandId objId, HandleId id) {
+ if (!obj->is<ArrayBufferObjectMaybeShared>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* buf = &obj->as<ArrayBufferObjectMaybeShared>();
+
+ if (mode_ != ICState::Mode::Specialized) {
+ return AttachDecision::NoAction;
+ }
+
+ // Receiver should be the object.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!id.isAtom(cx_->names().byteLength)) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, obj, id, &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::NativeGetter) {
+ return AttachDecision::NoAction;
+ }
+
+ auto& fun = holder->getGetter(*prop)->as<JSFunction>();
+ if (buf->is<ArrayBufferObject>()) {
+ if (!ArrayBufferObject::isOriginalByteLengthGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+ } else {
+ if (!SharedArrayBufferObject::isOriginalByteLengthGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+ }
+
+ maybeEmitIdGuard(id);
+ // Emit all the normal guards for calling this native, but specialize
+ // callNativeGetterResult.
+ EmitCallGetterResultGuards(writer, buf, holder, id, *prop, objId, mode_);
+ if (buf->byteLength() <= INT32_MAX) {
+ writer.loadArrayBufferByteLengthInt32Result(objId);
+ } else {
+ writer.loadArrayBufferByteLengthDoubleResult(objId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("ArrayBufferMaybeSharedByteLength");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachRegExp(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ if (!obj->is<RegExpObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* regExp = &obj->as<RegExpObject>();
+
+ if (mode_ != ICState::Mode::Specialized) {
+ return AttachDecision::NoAction;
+ }
+
+ // Receiver should be the object.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, obj, id, &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::NativeGetter) {
+ return AttachDecision::NoAction;
+ }
+
+ auto& fun = holder->getGetter(*prop)->as<JSFunction>();
+ JS::RegExpFlags flags = JS::RegExpFlag::NoFlags;
+ if (!RegExpObject::isOriginalFlagGetter(fun.native(), &flags)) {
+ return AttachDecision::NoAction;
+ }
+
+ maybeEmitIdGuard(id);
+ // Emit all the normal guards for calling this native, but specialize
+ // callNativeGetterResult.
+ EmitCallGetterResultGuards(writer, regExp, holder, id, *prop, objId, mode_);
+
+ writer.regExpFlagResult(objId, flags.value());
+ writer.returnFromIC();
+
+ trackAttached("RegExpFlag");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachMap(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ if (!obj->is<MapObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* mapObj = &obj->as<MapObject>();
+
+ if (mode_ != ICState::Mode::Specialized) {
+ return AttachDecision::NoAction;
+ }
+
+ // Receiver should be the object.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!id.isAtom(cx_->names().size)) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, obj, id, &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::NativeGetter) {
+ return AttachDecision::NoAction;
+ }
+
+ auto& fun = holder->getGetter(*prop)->as<JSFunction>();
+ if (!MapObject::isOriginalSizeGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+
+ maybeEmitIdGuard(id);
+
+ // Emit all the normal guards for calling this native, but specialize
+ // callNativeGetterResult.
+ EmitCallGetterResultGuards(writer, mapObj, holder, id, *prop, objId, mode_);
+
+ writer.mapSizeResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("MapSize");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachSet(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ if (!obj->is<SetObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* setObj = &obj->as<SetObject>();
+
+ if (mode_ != ICState::Mode::Specialized) {
+ return AttachDecision::NoAction;
+ }
+
+ // Receiver should be the object.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!id.isAtom(cx_->names().size)) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, obj, id, &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::NativeGetter) {
+ return AttachDecision::NoAction;
+ }
+
+ auto& fun = holder->getGetter(*prop)->as<JSFunction>();
+ if (!SetObject::isOriginalSizeGetter(fun.native())) {
+ return AttachDecision::NoAction;
+ }
+
+ maybeEmitIdGuard(id);
+
+ // Emit all the normal guards for calling this native, but specialize
+ // callNativeGetterResult.
+ EmitCallGetterResultGuards(writer, setObj, holder, id, *prop, objId, mode_);
+
+ writer.setSizeResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("SetSize");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachFunction(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ // Function properties are lazily resolved so they might not be defined yet.
+ // And we might end up in a situation where we always have a fresh function
+ // object during the IC generation.
+ if (!obj->is<JSFunction>()) {
+ return AttachDecision::NoAction;
+ }
+
+ bool isLength = id.isAtom(cx_->names().length);
+ if (!isLength && !id.isAtom(cx_->names().name)) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ PropertyResult prop;
+ // If this property exists already, don't attach the stub.
+ if (LookupPropertyPure(cx_, obj, id, &holder, &prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ JSFunction* fun = &obj->as<JSFunction>();
+
+ if (isLength) {
+ // length was probably deleted from the function.
+ if (fun->hasResolvedLength()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Lazy functions don't store the length.
+ if (!fun->hasBytecode()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Length can be non-int32 for bound functions.
+ if (fun->isBoundFunction()) {
+ constexpr auto lengthSlot = FunctionExtended::BOUND_FUNCTION_LENGTH_SLOT;
+ if (!fun->getExtendedSlot(lengthSlot).isInt32()) {
+ return AttachDecision::NoAction;
+ }
+ }
+ } else {
+ // name was probably deleted from the function.
+ if (fun->hasResolvedName()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Unless the bound function name prefix is present, we need to call into
+ // the VM to compute the full name.
+ if (fun->isBoundFunction() && !fun->hasBoundFunctionNamePrefix()) {
+ return AttachDecision::NoAction;
+ }
+ }
+
+ maybeEmitIdGuard(id);
+ writer.guardClass(objId, GuardClassKind::JSFunction);
+ if (isLength) {
+ writer.loadFunctionLengthResult(objId);
+ writer.returnFromIC();
+ trackAttached("FunctionLength");
+ } else {
+ writer.loadFunctionNameResult(objId);
+ writer.returnFromIC();
+ trackAttached("FunctionName");
+ }
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachArgumentsObjectIterator(
+ HandleObject obj, ObjOperandId objId, HandleId id) {
+ if (!obj->is<ArgumentsObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!id.isWellKnownSymbol(JS::SymbolCode::iterator)) {
+ return AttachDecision::NoAction;
+ }
+
+ Handle<ArgumentsObject*> args = obj.as<ArgumentsObject>();
+ if (args->hasOverriddenIterator()) {
+ return AttachDecision::NoAction;
+ }
+
+ RootedValue iterator(cx_);
+ if (!ArgumentsObject::getArgumentsIterator(cx_, &iterator)) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(iterator.isObject());
+
+ maybeEmitIdGuard(id);
+ if (args->is<MappedArgumentsObject>()) {
+ writer.guardClass(objId, GuardClassKind::MappedArguments);
+ } else {
+ MOZ_ASSERT(args->is<UnmappedArgumentsObject>());
+ writer.guardClass(objId, GuardClassKind::UnmappedArguments);
+ }
+ uint32_t flags = ArgumentsObject::ITERATOR_OVERRIDDEN_BIT;
+ writer.guardArgumentsObjectFlags(objId, flags);
+
+ ObjOperandId iterId = writer.loadObject(&iterator.toObject());
+ writer.loadObjectResult(iterId);
+ writer.returnFromIC();
+
+ trackAttached("ArgumentsObjectIterator");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachModuleNamespace(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id) {
+ if (!obj->is<ModuleNamespaceObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto* ns = &obj->as<ModuleNamespaceObject>();
+ ModuleEnvironmentObject* env = nullptr;
+ Maybe<PropertyInfo> prop;
+ if (!ns->bindings().lookup(id, &env, &prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Don't emit a stub until the target binding has been initialized.
+ if (env->getSlot(prop->slot()).isMagic(JS_UNINITIALIZED_LEXICAL)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check for the specific namespace object.
+ maybeEmitIdGuard(id);
+ writer.guardSpecificObject(objId, ns);
+
+ ObjOperandId envId = writer.loadObject(env);
+ EmitLoadSlotResult(writer, envId, env, *prop);
+ writer.returnFromIC();
+
+ trackAttached("ModuleNamespace");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachPrimitive(ValOperandId valId,
+ HandleId id) {
+ MOZ_ASSERT(!isSuper(), "SuperBase is guaranteed to be an object");
+
+ JSProtoKey protoKey;
+ switch (val_.type()) {
+ case ValueType::String:
+ if (id.isAtom(cx_->names().length)) {
+ // String length is special-cased, see js::GetProperty.
+ return AttachDecision::NoAction;
+ }
+ protoKey = JSProto_String;
+ break;
+ case ValueType::Int32:
+ case ValueType::Double:
+ protoKey = JSProto_Number;
+ break;
+ case ValueType::Boolean:
+ protoKey = JSProto_Boolean;
+ break;
+ case ValueType::Symbol:
+ protoKey = JSProto_Symbol;
+ break;
+ case ValueType::BigInt:
+ protoKey = JSProto_BigInt;
+ break;
+ case ValueType::Null:
+ case ValueType::Undefined:
+ case ValueType::Magic:
+ return AttachDecision::NoAction;
+#ifdef ENABLE_RECORD_TUPLE
+ case ValueType::ExtendedPrimitive:
+#endif
+ case ValueType::Object:
+ case ValueType::PrivateGCThing:
+ MOZ_CRASH("unexpected type");
+ }
+
+ JSObject* proto = cx_->global()->maybeGetPrototype(protoKey);
+ if (!proto) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ NativeGetPropKind kind =
+ CanAttachNativeGetProp(cx_, proto, id, &holder, &prop, pc_);
+ switch (kind) {
+ case NativeGetPropKind::None:
+ return AttachDecision::NoAction;
+ case NativeGetPropKind::Missing:
+ case NativeGetPropKind::Slot: {
+ auto* nproto = &proto->as<NativeObject>();
+
+ if (val_.isNumber()) {
+ writer.guardIsNumber(valId);
+ } else {
+ writer.guardNonDoubleType(valId, val_.type());
+ }
+ maybeEmitIdGuard(id);
+
+ ObjOperandId protoId = writer.loadObject(nproto);
+ if (kind == NativeGetPropKind::Slot) {
+ EmitReadSlotResult(writer, nproto, holder, *prop, protoId);
+ writer.returnFromIC();
+ trackAttached("PrimitiveSlot");
+ } else {
+ EmitMissingPropResult(writer, nproto, protoId);
+ writer.returnFromIC();
+ trackAttached("PrimitiveMissing");
+ }
+ return AttachDecision::Attach;
+ }
+ case NativeGetPropKind::ScriptedGetter:
+ case NativeGetPropKind::NativeGetter: {
+ auto* nproto = &proto->as<NativeObject>();
+
+ if (val_.isNumber()) {
+ writer.guardIsNumber(valId);
+ } else {
+ writer.guardNonDoubleType(valId, val_.type());
+ }
+ maybeEmitIdGuard(id);
+
+ ObjOperandId protoId = writer.loadObject(nproto);
+ EmitCallGetterResult(cx_, writer, kind, nproto, holder, id, *prop,
+ protoId, valId, mode_);
+
+ trackAttached("PrimitiveGetter");
+ return AttachDecision::Attach;
+ }
+ }
+
+ MOZ_CRASH("Bad NativeGetPropKind");
+}
+
+AttachDecision GetPropIRGenerator::tryAttachStringLength(ValOperandId valId,
+ HandleId id) {
+ if (!val_.isString() || !id.isAtom(cx_->names().length)) {
+ return AttachDecision::NoAction;
+ }
+
+ StringOperandId strId = writer.guardToString(valId);
+ maybeEmitIdGuard(id);
+ writer.loadStringLengthResult(strId);
+ writer.returnFromIC();
+
+ trackAttached("StringLength");
+ return AttachDecision::Attach;
+}
+
+enum class AttachStringChar { No, Yes, Linearize, OutOfBounds };
+
+static AttachStringChar CanAttachStringChar(const Value& val,
+ const Value& idVal) {
+ if (!val.isString() || !idVal.isInt32()) {
+ return AttachStringChar::No;
+ }
+
+ int32_t index = idVal.toInt32();
+ if (index < 0) {
+ return AttachStringChar::OutOfBounds;
+ }
+
+ JSString* str = val.toString();
+ if (size_t(index) >= str->length()) {
+ return AttachStringChar::OutOfBounds;
+ }
+
+ // This follows JSString::getChar and MacroAssembler::loadStringChar.
+ if (str->isRope()) {
+ JSRope* rope = &str->asRope();
+ if (size_t(index) < rope->leftChild()->length()) {
+ str = rope->leftChild();
+ } else {
+ str = rope->rightChild();
+ }
+ }
+
+ if (!str->isLinear()) {
+ return AttachStringChar::Linearize;
+ }
+
+ return AttachStringChar::Yes;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachStringChar(ValOperandId valId,
+ ValOperandId indexId) {
+ MOZ_ASSERT(idVal_.isInt32());
+
+ auto attach = CanAttachStringChar(val_, idVal_);
+ if (attach == AttachStringChar::No) {
+ return AttachDecision::NoAction;
+ }
+
+ // Can't attach for out-of-bounds access without guarding that indexed
+ // properties aren't present along the prototype chain of |String.prototype|.
+ if (attach == AttachStringChar::OutOfBounds) {
+ return AttachDecision::NoAction;
+ }
+
+ StringOperandId strId = writer.guardToString(valId);
+ Int32OperandId int32IndexId = writer.guardToInt32Index(indexId);
+ if (attach == AttachStringChar::Linearize) {
+ strId = writer.linearizeForCharAccess(strId, int32IndexId);
+ }
+ writer.loadStringCharResult(strId, int32IndexId, /* handleOOB = */ false);
+ writer.returnFromIC();
+
+ trackAttached("StringChar");
+ return AttachDecision::Attach;
+}
+
+static bool ClassCanHaveExtraProperties(const JSClass* clasp) {
+ return clasp->getResolve() || clasp->getOpsLookupProperty() ||
+ clasp->getOpsGetProperty() || IsTypedArrayClass(clasp);
+}
+
+enum class OwnProperty : bool { No, Yes };
+enum class AllowIndexedReceiver : bool { No, Yes };
+enum class AllowExtraReceiverProperties : bool { No, Yes };
+
+static bool CanAttachDenseElementHole(
+ NativeObject* obj, OwnProperty ownProp,
+ AllowIndexedReceiver allowIndexedReceiver = AllowIndexedReceiver::No,
+ AllowExtraReceiverProperties allowExtraReceiverProperties =
+ AllowExtraReceiverProperties::No) {
+ // Make sure the objects on the prototype don't have any indexed properties
+ // or that such properties can't appear without a shape change.
+ // Otherwise returning undefined for holes would obviously be incorrect,
+ // because we would have to lookup a property on the prototype instead.
+ do {
+ // The first two checks are also relevant to the receiver object.
+ if (allowIndexedReceiver == AllowIndexedReceiver::No && obj->isIndexed()) {
+ return false;
+ }
+ allowIndexedReceiver = AllowIndexedReceiver::No;
+
+ if (allowExtraReceiverProperties == AllowExtraReceiverProperties::No &&
+ ClassCanHaveExtraProperties(obj->getClass())) {
+ return false;
+ }
+ allowExtraReceiverProperties = AllowExtraReceiverProperties::No;
+
+ // Don't need to check prototype for OwnProperty checks
+ if (ownProp == OwnProperty::Yes) {
+ return true;
+ }
+
+ JSObject* proto = obj->staticPrototype();
+ if (!proto) {
+ break;
+ }
+
+ if (!proto->is<NativeObject>()) {
+ return false;
+ }
+
+ // Make sure objects on the prototype don't have dense elements.
+ if (proto->as<NativeObject>().getDenseInitializedLength() != 0) {
+ return false;
+ }
+
+ obj = &proto->as<NativeObject>();
+ } while (true);
+
+ return true;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachArgumentsObjectArg(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId) {
+ if (!obj->is<ArgumentsObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* args = &obj->as<ArgumentsObject>();
+
+ // No elements must have been overridden or deleted.
+ if (args->hasOverriddenElement()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check bounds.
+ if (index >= args->initialLength()) {
+ return AttachDecision::NoAction;
+ }
+
+ // And finally also check that the argument isn't forwarded.
+ if (args->argIsForwarded(index)) {
+ return AttachDecision::NoAction;
+ }
+
+ if (args->is<MappedArgumentsObject>()) {
+ writer.guardClass(objId, GuardClassKind::MappedArguments);
+ } else {
+ MOZ_ASSERT(args->is<UnmappedArgumentsObject>());
+ writer.guardClass(objId, GuardClassKind::UnmappedArguments);
+ }
+
+ writer.loadArgumentsObjectArgResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("ArgumentsObjectArg");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachArgumentsObjectArgHole(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId) {
+ if (!obj->is<ArgumentsObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* args = &obj->as<ArgumentsObject>();
+
+ // No elements must have been overridden or deleted.
+ if (args->hasOverriddenElement()) {
+ return AttachDecision::NoAction;
+ }
+
+ // And also check that the argument isn't forwarded.
+ if (index < args->initialLength() && args->argIsForwarded(index)) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!CanAttachDenseElementHole(args, OwnProperty::No,
+ AllowIndexedReceiver::Yes,
+ AllowExtraReceiverProperties::Yes)) {
+ return AttachDecision::NoAction;
+ }
+
+ // We don't need to guard on the shape, because we check if any element is
+ // overridden. Elements are marked as overridden iff any element is defined,
+ // irrespective of whether the element is in-bounds or out-of-bounds. So when
+ // that flag isn't set, we can guarantee that the arguments object doesn't
+ // have any additional own elements.
+
+ if (args->is<MappedArgumentsObject>()) {
+ writer.guardClass(objId, GuardClassKind::MappedArguments);
+ } else {
+ MOZ_ASSERT(args->is<UnmappedArgumentsObject>());
+ writer.guardClass(objId, GuardClassKind::UnmappedArguments);
+ }
+
+ GeneratePrototypeHoleGuards(writer, args, objId,
+ /* alwaysGuardFirstProto = */ true);
+
+ writer.loadArgumentsObjectArgHoleResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("ArgumentsObjectArgHole");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachArgumentsObjectCallee(
+ HandleObject obj, ObjOperandId objId, HandleId id) {
+ // Only mapped arguments objects have a `callee` property.
+ if (!obj->is<MappedArgumentsObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!id.isAtom(cx_->names().callee)) {
+ return AttachDecision::NoAction;
+ }
+
+ // The callee must not have been overridden or deleted.
+ if (obj->as<MappedArgumentsObject>().hasOverriddenCallee()) {
+ return AttachDecision::NoAction;
+ }
+
+ maybeEmitIdGuard(id);
+ writer.guardClass(objId, GuardClassKind::MappedArguments);
+
+ uint32_t flags = ArgumentsObject::CALLEE_OVERRIDDEN_BIT;
+ writer.guardArgumentsObjectFlags(objId, flags);
+
+ writer.loadFixedSlotResult(objId,
+ MappedArgumentsObject::getCalleeSlotOffset());
+ writer.returnFromIC();
+
+ trackAttached("ArgumentsObjectCallee");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachDenseElement(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId) {
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (!nobj->containsDenseElement(index)) {
+ return AttachDecision::NoAction;
+ }
+
+ if (mode_ == ICState::Mode::Megamorphic) {
+ writer.guardIsNativeObject(objId);
+ } else {
+ TestMatchingNativeReceiver(writer, nobj, objId);
+ }
+ writer.loadDenseElementResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("DenseElement");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachDenseElementHole(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId) {
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (nobj->containsDenseElement(index)) {
+ return AttachDecision::NoAction;
+ }
+ if (!CanAttachDenseElementHole(nobj, OwnProperty::No)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Guard on the shape, to prevent non-dense elements from appearing.
+ TestMatchingNativeReceiver(writer, nobj, objId);
+ GeneratePrototypeHoleGuards(writer, nobj, objId,
+ /* alwaysGuardFirstProto = */ false);
+ writer.loadDenseElementHoleResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("DenseElementHole");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachSparseElement(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId) {
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+ NativeObject* nobj = &obj->as<NativeObject>();
+
+ // Stub doesn't handle negative indices.
+ if (index > INT32_MAX) {
+ return AttachDecision::NoAction;
+ }
+
+ // The object must have sparse elements.
+ if (!nobj->isIndexed()) {
+ return AttachDecision::NoAction;
+ }
+
+ // The index must not be for a dense element.
+ if (nobj->containsDenseElement(index)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Only handle ArrayObject and PlainObject in this stub.
+ if (!nobj->is<ArrayObject>() && !nobj->is<PlainObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // GetSparseElementHelper assumes that the target and the receiver
+ // are the same.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Here, we ensure that the prototype chain does not define any sparse
+ // indexed properties on the shape lineage. This allows us to guard on
+ // the shapes up the prototype chain to ensure that no indexed properties
+ // exist outside of the dense elements.
+ //
+ // The `GeneratePrototypeHoleGuards` call below will guard on the shapes,
+ // as well as ensure that no prototypes contain dense elements, allowing
+ // us to perform a pure shape-search for out-of-bounds integer-indexed
+ // properties on the receiver object.
+ if (PrototypeMayHaveIndexedProperties(nobj)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure that obj is an ArrayObject or PlainObject.
+ if (nobj->is<ArrayObject>()) {
+ writer.guardClass(objId, GuardClassKind::Array);
+ } else {
+ MOZ_ASSERT(nobj->is<PlainObject>());
+ writer.guardClass(objId, GuardClassKind::PlainObject);
+ }
+
+ // The helper we are going to call only applies to non-dense elements.
+ writer.guardIndexIsNotDenseElement(objId, indexId);
+
+ // Ensures we are able to efficiently able to map to an integral jsid.
+ writer.guardInt32IsNonNegative(indexId);
+
+ // Shape guard the prototype chain to avoid shadowing indexes from appearing.
+ // The helper function also ensures that the index does not appear within the
+ // dense element set of the prototypes.
+ GeneratePrototypeHoleGuards(writer, nobj, objId,
+ /* alwaysGuardFirstProto = */ true);
+
+ // At this point, we are guaranteed that the indexed property will not
+ // be found on one of the prototypes. We are assured that we only have
+ // to check that the receiving object has the property.
+
+ writer.callGetSparseElementResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("GetSparseElement");
+ return AttachDecision::Attach;
+}
+
+// For Uint32Array we let the stub return an Int32 if we have not seen a
+// double, to allow better codegen in Warp while avoiding bailout loops.
+static bool ForceDoubleForUint32Array(TypedArrayObject* tarr, uint64_t index) {
+ MOZ_ASSERT(index < tarr->length());
+
+ if (tarr->type() != Scalar::Type::Uint32) {
+ // Return value is only relevant for Uint32Array.
+ return false;
+ }
+
+ Value res;
+ MOZ_ALWAYS_TRUE(tarr->getElementPure(index, &res));
+ MOZ_ASSERT(res.isNumber());
+ return res.isDouble();
+}
+
+AttachDecision GetPropIRGenerator::tryAttachTypedArrayElement(
+ HandleObject obj, ObjOperandId objId) {
+ if (!obj->is<TypedArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!idVal_.isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ TypedArrayObject* tarr = &obj->as<TypedArrayObject>();
+
+ bool handleOOB = false;
+ int64_t indexInt64;
+ if (!ValueIsInt64Index(idVal_, &indexInt64) || indexInt64 < 0 ||
+ uint64_t(indexInt64) >= tarr->length()) {
+ handleOOB = true;
+ }
+
+ // If the number is not representable as an integer the result will be
+ // |undefined| so we leave |forceDoubleForUint32| as false.
+ bool forceDoubleForUint32 = false;
+ if (!handleOOB) {
+ uint64_t index = uint64_t(indexInt64);
+ forceDoubleForUint32 = ForceDoubleForUint32Array(tarr, index);
+ }
+
+ writer.guardShapeForClass(objId, tarr->shape());
+
+ ValOperandId keyId = getElemKeyValueId();
+ IntPtrOperandId intPtrIndexId = guardToIntPtrIndex(idVal_, keyId, handleOOB);
+
+ writer.loadTypedArrayElementResult(objId, intPtrIndexId, tarr->type(),
+ handleOOB, forceDoubleForUint32);
+ writer.returnFromIC();
+
+ trackAttached("TypedElement");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachGenericElement(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId, ValOperandId receiverId) {
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+#ifdef JS_CODEGEN_X86
+ if (isSuper()) {
+ // There aren't enough registers available on x86.
+ return AttachDecision::NoAction;
+ }
+#endif
+
+ // To allow other types to attach in the non-megamorphic case we test the
+ // specific matching native receiver; however, once megamorphic we can attach
+ // for any native
+ if (mode_ == ICState::Mode::Megamorphic) {
+ writer.guardIsNativeObject(objId);
+ } else {
+ NativeObject* nobj = &obj->as<NativeObject>();
+ TestMatchingNativeReceiver(writer, nobj, objId);
+ }
+ writer.guardIndexIsNotDenseElement(objId, indexId);
+ if (isSuper()) {
+ writer.callNativeGetElementSuperResult(objId, indexId, receiverId);
+ } else {
+ writer.callNativeGetElementResult(objId, indexId);
+ }
+ writer.returnFromIC();
+
+ trackAttached(mode_ == ICState::Mode::Megamorphic
+ ? "GenericElementMegamorphic"
+ : "GenericElement");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetPropIRGenerator::tryAttachProxyElement(HandleObject obj,
+ ObjOperandId objId) {
+ if (!obj->is<ProxyObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // The proxy stubs don't currently support |super| access.
+ if (isSuper()) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.guardIsProxy(objId);
+
+ // We are not guarding against DOM proxies here, because there is no other
+ // specialized DOM IC we could attach.
+ // We could call maybeEmitIdGuard here and then emit ProxyGetResult,
+ // but for GetElem we prefer to attach a stub that can handle any Value
+ // so we don't attach a new stub for every id.
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetElem);
+ MOZ_ASSERT(!isSuper());
+ writer.proxyGetByValueResult(objId, getElemKeyValueId());
+ writer.returnFromIC();
+
+ trackAttached("ProxyElement");
+ return AttachDecision::Attach;
+}
+
+void GetPropIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("base", val_);
+ sp.valueProperty("property", idVal_);
+ }
+#endif
+}
+
+void IRGenerator::emitIdGuard(ValOperandId valId, const Value& idVal, jsid id) {
+ if (id.isSymbol()) {
+ MOZ_ASSERT(idVal.toSymbol() == id.toSymbol());
+ SymbolOperandId symId = writer.guardToSymbol(valId);
+ writer.guardSpecificSymbol(symId, id.toSymbol());
+ } else {
+ MOZ_ASSERT(id.isAtom());
+ if (idVal.isUndefined()) {
+ MOZ_ASSERT(id.isAtom(cx_->names().undefined));
+ writer.guardIsUndefined(valId);
+ } else if (idVal.isNull()) {
+ MOZ_ASSERT(id.isAtom(cx_->names().null));
+ writer.guardIsNull(valId);
+ } else {
+ MOZ_ASSERT(idVal.isString());
+ StringOperandId strId = writer.guardToString(valId);
+ writer.guardSpecificAtom(strId, id.toAtom());
+ }
+ }
+}
+
+void GetPropIRGenerator::maybeEmitIdGuard(jsid id) {
+ if (cacheKind_ == CacheKind::GetProp ||
+ cacheKind_ == CacheKind::GetPropSuper) {
+ // Constant PropertyName, no guards necessary.
+ MOZ_ASSERT(&idVal_.toString()->asAtom() == id.toAtom());
+ return;
+ }
+
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetElem ||
+ cacheKind_ == CacheKind::GetElemSuper);
+ emitIdGuard(getElemKeyValueId(), idVal_, id);
+}
+
+void SetPropIRGenerator::maybeEmitIdGuard(jsid id) {
+ if (cacheKind_ == CacheKind::SetProp) {
+ // Constant PropertyName, no guards necessary.
+ MOZ_ASSERT(&idVal_.toString()->asAtom() == id.toAtom());
+ return;
+ }
+
+ MOZ_ASSERT(cacheKind_ == CacheKind::SetElem);
+ emitIdGuard(setElemKeyValueId(), idVal_, id);
+}
+
+GetNameIRGenerator::GetNameIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleObject env,
+ Handle<PropertyName*> name)
+ : IRGenerator(cx, script, pc, CacheKind::GetName, state),
+ env_(env),
+ name_(name) {}
+
+AttachDecision GetNameIRGenerator::tryAttachStub() {
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetName);
+
+ AutoAssertNoPendingException aanpe(cx_);
+
+ ObjOperandId envId(writer.setInputOperandId(0));
+ RootedId id(cx_, NameToId(name_));
+
+ TRY_ATTACH(tryAttachGlobalNameValue(envId, id));
+ TRY_ATTACH(tryAttachGlobalNameGetter(envId, id));
+ TRY_ATTACH(tryAttachEnvironmentName(envId, id));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+static bool CanAttachGlobalName(JSContext* cx,
+ GlobalLexicalEnvironmentObject* globalLexical,
+ PropertyKey id, NativeObject** holder,
+ Maybe<PropertyInfo>* prop) {
+ // The property must be found, and it must be found as a normal data property.
+ NativeObject* current = globalLexical;
+ while (true) {
+ *prop = current->lookup(cx, id);
+ if (prop->isSome()) {
+ break;
+ }
+
+ if (current == globalLexical) {
+ current = &globalLexical->global();
+ } else {
+ // In the browser the global prototype chain should be immutable.
+ if (!current->staticPrototypeIsImmutable()) {
+ return false;
+ }
+
+ JSObject* proto = current->staticPrototype();
+ if (!proto || !proto->is<NativeObject>()) {
+ return false;
+ }
+
+ current = &proto->as<NativeObject>();
+ }
+ }
+
+ *holder = current;
+ return true;
+}
+
+AttachDecision GetNameIRGenerator::tryAttachGlobalNameValue(ObjOperandId objId,
+ HandleId id) {
+ if (!IsGlobalOp(JSOp(*pc_))) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ auto* globalLexical = &env_->as<GlobalLexicalEnvironmentObject>();
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ if (!CanAttachGlobalName(cx_, globalLexical, id, &holder, &prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ // The property must be found, and it must be found as a normal data property.
+ if (!prop->isDataProperty()) {
+ return AttachDecision::NoAction;
+ }
+
+ // This might still be an uninitialized lexical.
+ if (holder->getSlot(prop->slot()).isMagic()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (holder == globalLexical) {
+ // There is no need to guard on the shape. Lexical bindings are
+ // non-configurable, and this stub cannot be shared across globals.
+ size_t dynamicSlotOffset =
+ holder->dynamicSlotIndex(prop->slot()) * sizeof(Value);
+ writer.loadDynamicSlotResult(objId, dynamicSlotOffset);
+ } else {
+ // Check the prototype chain from the global to the holder
+ // prototype. Ignore the global lexical scope as it doesn't figure
+ // into the prototype chain. We guard on the global lexical
+ // scope's shape independently.
+ if (!IsCacheableGetPropSlot(&globalLexical->global(), holder, *prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Shape guard for global lexical.
+ writer.guardShape(objId, globalLexical->shape());
+
+ // Guard on the shape of the GlobalObject.
+ ObjOperandId globalId = writer.loadEnclosingEnvironment(objId);
+ writer.guardShape(globalId, globalLexical->global().shape());
+
+ ObjOperandId holderId = globalId;
+ if (holder != &globalLexical->global()) {
+ // Shape guard holder.
+ holderId = writer.loadObject(holder);
+ writer.guardShape(holderId, holder->shape());
+ }
+
+ EmitLoadSlotResult(writer, holderId, holder, *prop);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("GlobalNameValue");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetNameIRGenerator::tryAttachGlobalNameGetter(ObjOperandId objId,
+ HandleId id) {
+ if (!IsGlobalOp(JSOp(*pc_))) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ Handle<GlobalLexicalEnvironmentObject*> globalLexical =
+ env_.as<GlobalLexicalEnvironmentObject>();
+ MOZ_ASSERT(globalLexical->isGlobal());
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ if (!CanAttachGlobalName(cx_, globalLexical, id, &holder, &prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ if (holder == globalLexical) {
+ return AttachDecision::NoAction;
+ }
+
+ GlobalObject* global = &globalLexical->global();
+
+ NativeGetPropKind kind = IsCacheableGetPropCall(global, holder, *prop);
+ if (kind != NativeGetPropKind::NativeGetter &&
+ kind != NativeGetPropKind::ScriptedGetter) {
+ return AttachDecision::NoAction;
+ }
+
+ bool needsWindowProxy =
+ IsWindow(global) && GetterNeedsWindowProxyThis(holder, *prop);
+
+ // Shape guard for global lexical.
+ writer.guardShape(objId, globalLexical->shape());
+
+ // Guard on the shape of the GlobalObject.
+ ObjOperandId globalId = writer.loadEnclosingEnvironment(objId);
+ writer.guardShape(globalId, global->shape());
+
+ if (holder != global) {
+ // Shape guard holder.
+ ObjOperandId holderId = writer.loadObject(holder);
+ writer.guardShape(holderId, holder->shape());
+ EmitGuardGetterSetterSlot(writer, holder, *prop, holderId,
+ /* holderIsConstant = */ true);
+ } else {
+ // Note: pass true for |holderIsConstant| because the holder must be the
+ // current global object.
+ EmitGuardGetterSetterSlot(writer, holder, *prop, globalId,
+ /* holderIsConstant = */ true);
+ }
+
+ if (CanAttachDOMGetterSetter(cx_, JSJitInfo::Getter, global, holder, *prop,
+ mode_)) {
+ // The global shape guard above ensures the instance JSClass is correct.
+ MOZ_ASSERT(!needsWindowProxy);
+ EmitCallDOMGetterResultNoGuards(writer, holder, *prop, globalId);
+ trackAttached("GlobalNameDOMGetter");
+ } else {
+ ObjOperandId receiverObjId;
+ if (needsWindowProxy) {
+ MOZ_ASSERT(cx_->global()->maybeWindowProxy());
+ receiverObjId = writer.loadObject(cx_->global()->maybeWindowProxy());
+ } else {
+ receiverObjId = globalId;
+ }
+ ValOperandId receiverId = writer.boxObject(receiverObjId);
+ EmitCallGetterResultNoGuards(cx_, writer, kind, global, holder, *prop,
+ receiverId);
+ trackAttached("GlobalNameGetter");
+ }
+
+ return AttachDecision::Attach;
+}
+
+static bool NeedEnvironmentShapeGuard(JSObject* envObj) {
+ if (!envObj->is<CallObject>()) {
+ return true;
+ }
+
+ // We can skip a guard on the call object if the script's bindings are
+ // guaranteed to be immutable (and thus cannot introduce shadowing variables).
+ // If the function is a relazified self-hosted function it has no BaseScript
+ // and we pessimistically create the guard.
+ CallObject* callObj = &envObj->as<CallObject>();
+ JSFunction* fun = &callObj->callee();
+ if (!fun->hasBaseScript() || fun->baseScript()->funHasExtensibleScope()) {
+ return true;
+ }
+
+ return false;
+}
+
+static ValOperandId EmitLoadEnvironmentSlot(CacheIRWriter& writer,
+ NativeObject* holder,
+ ObjOperandId holderId,
+ uint32_t slot) {
+ if (holder->isFixedSlot(slot)) {
+ return writer.loadFixedSlot(holderId,
+ NativeObject::getFixedSlotOffset(slot));
+ }
+ size_t dynamicSlotIndex = holder->dynamicSlotIndex(slot);
+ return writer.loadDynamicSlot(holderId, dynamicSlotIndex);
+}
+
+AttachDecision GetNameIRGenerator::tryAttachEnvironmentName(ObjOperandId objId,
+ HandleId id) {
+ if (IsGlobalOp(JSOp(*pc_)) || script_->hasNonSyntacticScope()) {
+ return AttachDecision::NoAction;
+ }
+
+ JSObject* env = env_;
+ Maybe<PropertyInfo> prop;
+ NativeObject* holder = nullptr;
+
+ while (env) {
+ if (env->is<GlobalObject>()) {
+ prop = env->as<GlobalObject>().lookup(cx_, id);
+ if (prop.isSome()) {
+ break;
+ }
+ return AttachDecision::NoAction;
+ }
+
+ if (!env->is<EnvironmentObject>() || env->is<WithEnvironmentObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check for an 'own' property on the env. There is no need to
+ // check the prototype as non-with scopes do not inherit properties
+ // from any prototype.
+ prop = env->as<NativeObject>().lookup(cx_, id);
+ if (prop.isSome()) {
+ break;
+ }
+
+ env = env->enclosingEnvironment();
+ }
+
+ holder = &env->as<NativeObject>();
+ if (!IsCacheableGetPropSlot(holder, holder, *prop)) {
+ return AttachDecision::NoAction;
+ }
+ if (holder->getSlot(prop->slot()).isMagic()) {
+ MOZ_ASSERT(holder->is<EnvironmentObject>());
+ return AttachDecision::NoAction;
+ }
+
+ ObjOperandId lastObjId = objId;
+ env = env_;
+ while (env) {
+ if (NeedEnvironmentShapeGuard(env)) {
+ writer.guardShape(lastObjId, env->shape());
+ }
+
+ if (env == holder) {
+ break;
+ }
+
+ lastObjId = writer.loadEnclosingEnvironment(lastObjId);
+ env = env->enclosingEnvironment();
+ }
+
+ ValOperandId resId =
+ EmitLoadEnvironmentSlot(writer, holder, lastObjId, prop->slot());
+ if (holder->is<EnvironmentObject>()) {
+ writer.guardIsNotUninitializedLexical(resId);
+ }
+ writer.loadOperandResult(resId);
+ writer.returnFromIC();
+
+ trackAttached("EnvironmentName");
+ return AttachDecision::Attach;
+}
+
+void GetNameIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("base", ObjectValue(*env_));
+ sp.valueProperty("property", StringValue(name_));
+ }
+#endif
+}
+
+BindNameIRGenerator::BindNameIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleObject env,
+ Handle<PropertyName*> name)
+ : IRGenerator(cx, script, pc, CacheKind::BindName, state),
+ env_(env),
+ name_(name) {}
+
+AttachDecision BindNameIRGenerator::tryAttachStub() {
+ MOZ_ASSERT(cacheKind_ == CacheKind::BindName);
+
+ AutoAssertNoPendingException aanpe(cx_);
+
+ ObjOperandId envId(writer.setInputOperandId(0));
+ RootedId id(cx_, NameToId(name_));
+
+ TRY_ATTACH(tryAttachGlobalName(envId, id));
+ TRY_ATTACH(tryAttachEnvironmentName(envId, id));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+AttachDecision BindNameIRGenerator::tryAttachGlobalName(ObjOperandId objId,
+ HandleId id) {
+ if (!IsGlobalOp(JSOp(*pc_))) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ Handle<GlobalLexicalEnvironmentObject*> globalLexical =
+ env_.as<GlobalLexicalEnvironmentObject>();
+ MOZ_ASSERT(globalLexical->isGlobal());
+
+ JSObject* result = nullptr;
+ if (Maybe<PropertyInfo> prop = globalLexical->lookup(cx_, id)) {
+ // If this is an uninitialized lexical or a const, we need to return a
+ // RuntimeLexicalErrorObject.
+ if (globalLexical->getSlot(prop->slot()).isMagic() || !prop->writable()) {
+ return AttachDecision::NoAction;
+ }
+ result = globalLexical;
+ } else {
+ result = &globalLexical->global();
+ }
+
+ if (result == globalLexical) {
+ // Lexical bindings are non-configurable so we can just return the
+ // global lexical.
+ writer.loadObjectResult(objId);
+ } else {
+ // If the property exists on the global and is non-configurable, it cannot
+ // be shadowed by the lexical scope so we can just return the global without
+ // a shape guard.
+ Maybe<PropertyInfo> prop = result->as<GlobalObject>().lookup(cx_, id);
+ if (prop.isNothing() || prop->configurable()) {
+ writer.guardShape(objId, globalLexical->shape());
+ }
+ ObjOperandId globalId = writer.loadEnclosingEnvironment(objId);
+ writer.loadObjectResult(globalId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("GlobalName");
+ return AttachDecision::Attach;
+}
+
+AttachDecision BindNameIRGenerator::tryAttachEnvironmentName(ObjOperandId objId,
+ HandleId id) {
+ if (IsGlobalOp(JSOp(*pc_)) || script_->hasNonSyntacticScope()) {
+ return AttachDecision::NoAction;
+ }
+
+ JSObject* env = env_;
+ Maybe<PropertyInfo> prop;
+ while (true) {
+ if (!env->is<GlobalObject>() && !env->is<EnvironmentObject>()) {
+ return AttachDecision::NoAction;
+ }
+ if (env->is<WithEnvironmentObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // When we reach an unqualified variables object (like the global) we
+ // have to stop looking and return that object.
+ if (env->isUnqualifiedVarObj()) {
+ break;
+ }
+
+ // Check for an 'own' property on the env. There is no need to
+ // check the prototype as non-with scopes do not inherit properties
+ // from any prototype.
+ prop = env->as<NativeObject>().lookup(cx_, id);
+ if (prop.isSome()) {
+ break;
+ }
+
+ env = env->enclosingEnvironment();
+ }
+
+ // If this is an uninitialized lexical or a const, we need to return a
+ // RuntimeLexicalErrorObject.
+ auto* holder = &env->as<NativeObject>();
+ if (prop.isSome() && holder->is<EnvironmentObject>() &&
+ (holder->getSlot(prop->slot()).isMagic() || !prop->writable())) {
+ return AttachDecision::NoAction;
+ }
+
+ ObjOperandId lastObjId = objId;
+ env = env_;
+ while (env) {
+ if (NeedEnvironmentShapeGuard(env) && !env->is<GlobalObject>()) {
+ writer.guardShape(lastObjId, env->shape());
+ }
+
+ if (env == holder) {
+ break;
+ }
+
+ lastObjId = writer.loadEnclosingEnvironment(lastObjId);
+ env = env->enclosingEnvironment();
+ }
+
+ if (prop.isSome() && holder->is<EnvironmentObject>()) {
+ ValOperandId valId =
+ EmitLoadEnvironmentSlot(writer, holder, lastObjId, prop->slot());
+ writer.guardIsNotUninitializedLexical(valId);
+ }
+
+ writer.loadObjectResult(lastObjId);
+ writer.returnFromIC();
+
+ trackAttached("EnvironmentName");
+ return AttachDecision::Attach;
+}
+
+void BindNameIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("base", ObjectValue(*env_));
+ sp.valueProperty("property", StringValue(name_));
+ }
+#endif
+}
+
+HasPropIRGenerator::HasPropIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ CacheKind cacheKind, HandleValue idVal,
+ HandleValue val)
+ : IRGenerator(cx, script, pc, cacheKind, state), val_(val), idVal_(idVal) {}
+
+AttachDecision HasPropIRGenerator::tryAttachDense(HandleObject obj,
+ ObjOperandId objId,
+ uint32_t index,
+ Int32OperandId indexId) {
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (!nobj->containsDenseElement(index)) {
+ return AttachDecision::NoAction;
+ }
+
+ if (mode_ == ICState::Mode::Megamorphic) {
+ writer.guardIsNativeObject(objId);
+ } else {
+ // Guard shape to ensure object class is NativeObject.
+ TestMatchingNativeReceiver(writer, nobj, objId);
+ }
+ writer.loadDenseElementExistsResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("DenseHasProp");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachDenseHole(HandleObject obj,
+ ObjOperandId objId,
+ uint32_t index,
+ Int32OperandId indexId) {
+ bool hasOwn = (cacheKind_ == CacheKind::HasOwn);
+ OwnProperty ownProp = hasOwn ? OwnProperty::Yes : OwnProperty::No;
+
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (nobj->containsDenseElement(index)) {
+ return AttachDecision::NoAction;
+ }
+ if (!CanAttachDenseElementHole(nobj, ownProp)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Guard shape to ensure class is NativeObject and to prevent non-dense
+ // elements being added. Also ensures prototype doesn't change if dynamic
+ // checks aren't emitted.
+ TestMatchingNativeReceiver(writer, nobj, objId);
+
+ // Generate prototype guards if needed. This includes monitoring that
+ // properties were not added in the chain.
+ if (!hasOwn) {
+ GeneratePrototypeHoleGuards(writer, nobj, objId,
+ /* alwaysGuardFirstProto = */ false);
+ }
+
+ writer.loadDenseElementHoleExistsResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("DenseHasPropHole");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachSparse(HandleObject obj,
+ ObjOperandId objId,
+ Int32OperandId indexId) {
+ bool hasOwn = (cacheKind_ == CacheKind::HasOwn);
+ OwnProperty ownProp = hasOwn ? OwnProperty::Yes : OwnProperty::No;
+
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ if (!nobj->isIndexed()) {
+ return AttachDecision::NoAction;
+ }
+ if (!CanAttachDenseElementHole(nobj, ownProp, AllowIndexedReceiver::Yes)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Guard that this is a native object.
+ writer.guardIsNativeObject(objId);
+
+ // Generate prototype guards if needed. This includes monitoring that
+ // properties were not added in the chain.
+ if (!hasOwn) {
+ GeneratePrototypeHoleGuards(writer, nobj, objId,
+ /* alwaysGuardFirstProto = */ true);
+ }
+
+ // Because of the prototype guard we know that the prototype chain
+ // does not include any dense or sparse (i.e indexed) properties.
+ writer.callObjectHasSparseElementResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("Sparse");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachArgumentsObjectArg(
+ HandleObject obj, ObjOperandId objId, Int32OperandId indexId) {
+ bool hasOwn = (cacheKind_ == CacheKind::HasOwn);
+ OwnProperty ownProp = hasOwn ? OwnProperty::Yes : OwnProperty::No;
+
+ if (!obj->is<ArgumentsObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* args = &obj->as<ArgumentsObject>();
+
+ // No elements must have been overridden or deleted.
+ if (args->hasOverriddenElement()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!CanAttachDenseElementHole(args, ownProp, AllowIndexedReceiver::Yes,
+ AllowExtraReceiverProperties::Yes)) {
+ return AttachDecision::NoAction;
+ }
+
+ if (args->is<MappedArgumentsObject>()) {
+ writer.guardClass(objId, GuardClassKind::MappedArguments);
+ } else {
+ MOZ_ASSERT(args->is<UnmappedArgumentsObject>());
+ writer.guardClass(objId, GuardClassKind::UnmappedArguments);
+ }
+
+ if (!hasOwn) {
+ GeneratePrototypeHoleGuards(writer, args, objId,
+ /* alwaysGuardFirstProto = */ true);
+ }
+
+ writer.loadArgumentsObjectArgExistsResult(objId, indexId);
+ writer.returnFromIC();
+
+ trackAttached("ArgumentsObjectArg");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachNamedProp(HandleObject obj,
+ ObjOperandId objId,
+ HandleId key,
+ ValOperandId keyId) {
+ bool hasOwn = (cacheKind_ == CacheKind::HasOwn);
+
+ NativeObject* holder = nullptr;
+ PropertyResult prop;
+
+ if (hasOwn) {
+ if (!LookupOwnPropertyPure(cx_, obj, key, &prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ holder = &obj->as<NativeObject>();
+ } else {
+ if (!LookupPropertyPure(cx_, obj, key, &holder, &prop)) {
+ return AttachDecision::NoAction;
+ }
+ }
+ if (prop.isNotFound()) {
+ return AttachDecision::NoAction;
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ TRY_ATTACH(tryAttachMegamorphic(objId, keyId));
+ TRY_ATTACH(tryAttachNative(nobj, objId, key, keyId, prop, holder));
+
+ return AttachDecision::NoAction;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachMegamorphic(ObjOperandId objId,
+ ValOperandId keyId) {
+ bool hasOwn = (cacheKind_ == CacheKind::HasOwn);
+
+ if (mode_ != ICState::Mode::Megamorphic) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.megamorphicHasPropResult(objId, keyId, hasOwn);
+ writer.returnFromIC();
+ trackAttached("MegamorphicHasProp");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachNative(NativeObject* obj,
+ ObjOperandId objId, jsid key,
+ ValOperandId keyId,
+ PropertyResult prop,
+ NativeObject* holder) {
+ MOZ_ASSERT(IsCacheableProtoChain(obj, holder));
+
+ if (!prop.isNativeProperty()) {
+ return AttachDecision::NoAction;
+ }
+
+ emitIdGuard(keyId, idVal_, key);
+ EmitReadSlotGuard(writer, obj, holder, objId);
+ writer.loadBooleanResult(true);
+ writer.returnFromIC();
+
+ trackAttached("NativeHasProp");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachTypedArray(HandleObject obj,
+ ObjOperandId objId,
+ ValOperandId keyId) {
+ if (!obj->is<TypedArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ int64_t index;
+ if (!ValueIsInt64Index(idVal_, &index)) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.guardIsTypedArray(objId);
+ IntPtrOperandId intPtrIndexId =
+ guardToIntPtrIndex(idVal_, keyId, /* supportOOB = */ true);
+ writer.loadTypedArrayElementExistsResult(objId, intPtrIndexId);
+ writer.returnFromIC();
+
+ trackAttached("TypedArrayObject");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachSlotDoesNotExist(
+ NativeObject* obj, ObjOperandId objId, jsid key, ValOperandId keyId) {
+ bool hasOwn = (cacheKind_ == CacheKind::HasOwn);
+
+ emitIdGuard(keyId, idVal_, key);
+ if (hasOwn) {
+ TestMatchingNativeReceiver(writer, obj, objId);
+ } else {
+ EmitMissingPropGuard(writer, obj, objId);
+ }
+ writer.loadBooleanResult(false);
+ writer.returnFromIC();
+
+ trackAttached("DoesNotExist");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachDoesNotExist(HandleObject obj,
+ ObjOperandId objId,
+ HandleId key,
+ ValOperandId keyId) {
+ bool hasOwn = (cacheKind_ == CacheKind::HasOwn);
+
+ // Check that property doesn't exist on |obj| or it's prototype chain. These
+ // checks allow NativeObjects with a NativeObject prototype chain. They return
+ // NoAction if unknown such as resolve hooks or proxies.
+ if (hasOwn) {
+ if (!CheckHasNoSuchOwnProperty(cx_, obj, key)) {
+ return AttachDecision::NoAction;
+ }
+ } else {
+ if (!CheckHasNoSuchProperty(cx_, obj, key)) {
+ return AttachDecision::NoAction;
+ }
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ TRY_ATTACH(tryAttachMegamorphic(objId, keyId));
+ TRY_ATTACH(tryAttachSlotDoesNotExist(nobj, objId, key, keyId));
+
+ return AttachDecision::NoAction;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachProxyElement(HandleObject obj,
+ ObjOperandId objId,
+ ValOperandId keyId) {
+ bool hasOwn = (cacheKind_ == CacheKind::HasOwn);
+
+ if (!obj->is<ProxyObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.guardIsProxy(objId);
+ writer.proxyHasPropResult(objId, keyId, hasOwn);
+ writer.returnFromIC();
+
+ trackAttached("ProxyHasProp");
+ return AttachDecision::Attach;
+}
+
+AttachDecision HasPropIRGenerator::tryAttachStub() {
+ MOZ_ASSERT(cacheKind_ == CacheKind::In || cacheKind_ == CacheKind::HasOwn);
+
+ AutoAssertNoPendingException aanpe(cx_);
+
+ // NOTE: Argument order is PROPERTY, OBJECT
+ ValOperandId keyId(writer.setInputOperandId(0));
+ ValOperandId valId(writer.setInputOperandId(1));
+
+ if (!val_.isObject()) {
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+ RootedObject obj(cx_, &val_.toObject());
+ ObjOperandId objId = writer.guardToObject(valId);
+
+ // Optimize Proxies
+ TRY_ATTACH(tryAttachProxyElement(obj, objId, keyId));
+
+ RootedId id(cx_);
+ bool nameOrSymbol;
+ if (!ValueToNameOrSymbolId(cx_, idVal_, &id, &nameOrSymbol)) {
+ cx_->clearPendingException();
+ return AttachDecision::NoAction;
+ }
+
+ if (nameOrSymbol) {
+ TRY_ATTACH(tryAttachNamedProp(obj, objId, id, keyId));
+ TRY_ATTACH(tryAttachDoesNotExist(obj, objId, id, keyId));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ TRY_ATTACH(tryAttachTypedArray(obj, objId, keyId));
+
+ uint32_t index;
+ Int32OperandId indexId;
+ if (maybeGuardInt32Index(idVal_, keyId, &index, &indexId)) {
+ TRY_ATTACH(tryAttachDense(obj, objId, index, indexId));
+ TRY_ATTACH(tryAttachDenseHole(obj, objId, index, indexId));
+ TRY_ATTACH(tryAttachSparse(obj, objId, indexId));
+ TRY_ATTACH(tryAttachArgumentsObjectArg(obj, objId, indexId));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+void HasPropIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("base", val_);
+ sp.valueProperty("property", idVal_);
+ }
+#endif
+}
+
+CheckPrivateFieldIRGenerator::CheckPrivateFieldIRGenerator(
+ JSContext* cx, HandleScript script, jsbytecode* pc, ICState state,
+ CacheKind cacheKind, HandleValue idVal, HandleValue val)
+ : IRGenerator(cx, script, pc, cacheKind, state), val_(val), idVal_(idVal) {
+ MOZ_ASSERT(idVal.isSymbol() && idVal.toSymbol()->isPrivateName());
+}
+
+AttachDecision CheckPrivateFieldIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ ValOperandId keyId(writer.setInputOperandId(1));
+
+ if (!val_.isObject()) {
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+ JSObject* obj = &val_.toObject();
+ ObjOperandId objId = writer.guardToObject(valId);
+ PropertyKey key = PropertyKey::Symbol(idVal_.toSymbol());
+
+ ThrowCondition condition;
+ ThrowMsgKind msgKind;
+ GetCheckPrivateFieldOperands(pc_, &condition, &msgKind);
+
+ PropertyResult prop;
+ if (!LookupOwnPropertyPure(cx_, obj, key, &prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ if (CheckPrivateFieldWillThrow(condition, prop.isFound())) {
+ // Don't attach a stub if the operation will throw.
+ return AttachDecision::NoAction;
+ }
+
+ auto* nobj = &obj->as<NativeObject>();
+
+ TRY_ATTACH(tryAttachNative(nobj, objId, key, keyId, prop));
+
+ return AttachDecision::NoAction;
+}
+
+AttachDecision CheckPrivateFieldIRGenerator::tryAttachNative(
+ NativeObject* obj, ObjOperandId objId, jsid key, ValOperandId keyId,
+ PropertyResult prop) {
+ MOZ_ASSERT(prop.isNativeProperty() || prop.isNotFound());
+
+ emitIdGuard(keyId, idVal_, key);
+ TestMatchingNativeReceiver(writer, obj, objId);
+ writer.loadBooleanResult(prop.isFound());
+ writer.returnFromIC();
+
+ trackAttached("NativeCheckPrivateField");
+ return AttachDecision::Attach;
+}
+
+void CheckPrivateFieldIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("base", val_);
+ sp.valueProperty("property", idVal_);
+ }
+#endif
+}
+
+bool IRGenerator::maybeGuardInt32Index(const Value& index, ValOperandId indexId,
+ uint32_t* int32Index,
+ Int32OperandId* int32IndexId) {
+ if (index.isNumber()) {
+ int32_t indexSigned;
+ if (index.isInt32()) {
+ indexSigned = index.toInt32();
+ } else {
+ // We allow negative zero here.
+ if (!mozilla::NumberEqualsInt32(index.toDouble(), &indexSigned)) {
+ return false;
+ }
+ }
+
+ if (indexSigned < 0) {
+ return false;
+ }
+
+ *int32Index = uint32_t(indexSigned);
+ *int32IndexId = writer.guardToInt32Index(indexId);
+ return true;
+ }
+
+ if (index.isString()) {
+ int32_t indexSigned = GetIndexFromString(index.toString());
+ if (indexSigned < 0) {
+ return false;
+ }
+
+ StringOperandId strId = writer.guardToString(indexId);
+ *int32Index = uint32_t(indexSigned);
+ *int32IndexId = writer.guardStringToIndex(strId);
+ return true;
+ }
+
+ return false;
+}
+
+SetPropIRGenerator::SetPropIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, CacheKind cacheKind,
+ ICState state, HandleValue lhsVal,
+ HandleValue idVal, HandleValue rhsVal)
+ : IRGenerator(cx, script, pc, cacheKind, state),
+ lhsVal_(lhsVal),
+ idVal_(idVal),
+ rhsVal_(rhsVal) {}
+
+AttachDecision SetPropIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+
+ ValOperandId objValId(writer.setInputOperandId(0));
+ ValOperandId rhsValId;
+ if (cacheKind_ == CacheKind::SetProp) {
+ rhsValId = ValOperandId(writer.setInputOperandId(1));
+ } else {
+ MOZ_ASSERT(cacheKind_ == CacheKind::SetElem);
+ MOZ_ASSERT(setElemKeyValueId().id() == 1);
+ writer.setInputOperandId(1);
+ rhsValId = ValOperandId(writer.setInputOperandId(2));
+ }
+
+ RootedId id(cx_);
+ bool nameOrSymbol;
+ if (!ValueToNameOrSymbolId(cx_, idVal_, &id, &nameOrSymbol)) {
+ cx_->clearPendingException();
+ return AttachDecision::NoAction;
+ }
+
+ if (lhsVal_.isObject()) {
+ RootedObject obj(cx_, &lhsVal_.toObject());
+
+ ObjOperandId objId = writer.guardToObject(objValId);
+ if (IsPropertySetOp(JSOp(*pc_))) {
+ TRY_ATTACH(tryAttachMegamorphicSetElement(obj, objId, rhsValId));
+ }
+ if (nameOrSymbol) {
+ TRY_ATTACH(tryAttachNativeSetSlot(obj, objId, id, rhsValId));
+ if (IsPropertySetOp(JSOp(*pc_))) {
+ TRY_ATTACH(tryAttachSetArrayLength(obj, objId, id, rhsValId));
+ TRY_ATTACH(tryAttachSetter(obj, objId, id, rhsValId));
+ TRY_ATTACH(tryAttachWindowProxy(obj, objId, id, rhsValId));
+ TRY_ATTACH(tryAttachProxy(obj, objId, id, rhsValId));
+ }
+ if (canAttachAddSlotStub(obj, id)) {
+ deferType_ = DeferType::AddSlot;
+ return AttachDecision::Deferred;
+ }
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(cacheKind_ == CacheKind::SetElem);
+
+ if (IsPropertySetOp(JSOp(*pc_))) {
+ TRY_ATTACH(tryAttachProxyElement(obj, objId, rhsValId));
+ }
+
+ TRY_ATTACH(tryAttachSetTypedArrayElement(obj, objId, rhsValId));
+
+ uint32_t index;
+ Int32OperandId indexId;
+ if (maybeGuardInt32Index(idVal_, setElemKeyValueId(), &index, &indexId)) {
+ TRY_ATTACH(
+ tryAttachSetDenseElement(obj, objId, index, indexId, rhsValId));
+ TRY_ATTACH(
+ tryAttachSetDenseElementHole(obj, objId, index, indexId, rhsValId));
+ TRY_ATTACH(tryAttachAddOrUpdateSparseElement(obj, objId, index, indexId,
+ rhsValId));
+ return AttachDecision::NoAction;
+ }
+ }
+ return AttachDecision::NoAction;
+}
+
+static void EmitStoreSlotAndReturn(CacheIRWriter& writer, ObjOperandId objId,
+ NativeObject* nobj, PropertyInfo prop,
+ ValOperandId rhsId) {
+ if (nobj->isFixedSlot(prop.slot())) {
+ size_t offset = NativeObject::getFixedSlotOffset(prop.slot());
+ writer.storeFixedSlot(objId, offset, rhsId);
+ } else {
+ size_t offset = nobj->dynamicSlotIndex(prop.slot()) * sizeof(Value);
+ writer.storeDynamicSlot(objId, offset, rhsId);
+ }
+ writer.returnFromIC();
+}
+
+static Maybe<PropertyInfo> LookupShapeForSetSlot(JSOp op, NativeObject* obj,
+ jsid id) {
+ Maybe<PropertyInfo> prop = obj->lookupPure(id);
+ if (prop.isNothing() || !prop->isDataProperty() || !prop->writable()) {
+ return mozilla::Nothing();
+ }
+
+ // If this is a property init operation, the property's attributes may have to
+ // be changed too, so make sure the current flags match.
+ if (IsPropertyInitOp(op)) {
+ // Don't support locked init operations.
+ if (IsLockedInitOp(op)) {
+ return mozilla::Nothing();
+ }
+
+ // Can't redefine a non-configurable property.
+ if (!prop->configurable()) {
+ return mozilla::Nothing();
+ }
+
+ // Make sure the enumerable flag matches the init operation.
+ if (IsHiddenInitOp(op) == prop->enumerable()) {
+ return mozilla::Nothing();
+ }
+ }
+
+ return prop;
+}
+
+static bool CanAttachNativeSetSlot(JSOp op, JSObject* obj, PropertyKey id,
+ Maybe<PropertyInfo>* prop) {
+ if (!obj->is<NativeObject>()) {
+ return false;
+ }
+
+ *prop = LookupShapeForSetSlot(op, &obj->as<NativeObject>(), id);
+ return prop->isSome();
+}
+
+// There is no need to guard on the shape. Global lexical bindings are
+// non-configurable and can not be shadowed.
+static bool IsGlobalLexicalSetGName(JSOp op, NativeObject* obj,
+ PropertyInfo prop) {
+ // Ensure that the env can't change.
+ if (op != JSOp::SetGName && op != JSOp::StrictSetGName) {
+ return false;
+ }
+
+ if (!obj->is<GlobalLexicalEnvironmentObject>()) {
+ return false;
+ }
+
+ // Uninitialized let bindings use a RuntimeLexicalErrorObject.
+ MOZ_ASSERT(!obj->getSlot(prop.slot()).isMagic());
+ MOZ_ASSERT(prop.writable());
+ MOZ_ASSERT(!prop.configurable());
+ return true;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachNativeSetSlot(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id,
+ ValOperandId rhsId) {
+ Maybe<PropertyInfo> prop;
+ if (!CanAttachNativeSetSlot(JSOp(*pc_), obj, id, &prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Don't attach a megamorphic store slot stub for ops like JSOp::InitElem.
+ if (mode_ == ICState::Mode::Megamorphic && cacheKind_ == CacheKind::SetProp &&
+ IsPropertySetOp(JSOp(*pc_))) {
+ writer.megamorphicStoreSlot(objId, id.toAtom()->asPropertyName(), rhsId);
+ writer.returnFromIC();
+ trackAttached("MegamorphicNativeSlot");
+ return AttachDecision::Attach;
+ }
+
+ maybeEmitIdGuard(id);
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (!IsGlobalLexicalSetGName(JSOp(*pc_), nobj, *prop)) {
+ TestMatchingNativeReceiver(writer, nobj, objId);
+ }
+
+ EmitStoreSlotAndReturn(writer, objId, nobj, *prop, rhsId);
+
+ trackAttached("NativeSlot");
+ return AttachDecision::Attach;
+}
+
+OperandId IRGenerator::emitNumericGuard(ValOperandId valId, Scalar::Type type) {
+ switch (type) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ return writer.guardToInt32ModUint32(valId);
+
+ case Scalar::Float32:
+ case Scalar::Float64:
+ return writer.guardIsNumber(valId);
+
+ case Scalar::Uint8Clamped:
+ return writer.guardToUint8Clamped(valId);
+
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ return writer.guardToBigInt(valId);
+
+ case Scalar::MaxTypedArrayViewType:
+ case Scalar::Int64:
+ case Scalar::Simd128:
+ break;
+ }
+ MOZ_CRASH("Unsupported TypedArray type");
+}
+
+static bool ValueIsNumeric(Scalar::Type type, const Value& val) {
+ if (Scalar::isBigIntType(type)) {
+ return val.isBigInt();
+ }
+ return val.isNumber();
+}
+
+void SetPropIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.opcodeProperty("op", JSOp(*pc_));
+ sp.valueProperty("base", lhsVal_);
+ sp.valueProperty("property", idVal_);
+ sp.valueProperty("value", rhsVal_);
+ }
+#endif
+}
+
+static bool IsCacheableSetPropCallNative(NativeObject* obj,
+ NativeObject* holder,
+ PropertyInfo prop) {
+ MOZ_ASSERT(IsCacheableProtoChain(obj, holder));
+
+ if (!prop.isAccessorProperty()) {
+ return false;
+ }
+
+ JSObject* setterObject = holder->getSetter(prop);
+ if (!setterObject || !setterObject->is<JSFunction>()) {
+ return false;
+ }
+
+ JSFunction& setter = setterObject->as<JSFunction>();
+ if (!setter.isNativeWithoutJitEntry()) {
+ return false;
+ }
+
+ if (setter.isClassConstructor()) {
+ return false;
+ }
+
+ return true;
+}
+
+static bool IsCacheableSetPropCallScripted(NativeObject* obj,
+ NativeObject* holder,
+ PropertyInfo prop) {
+ MOZ_ASSERT(IsCacheableProtoChain(obj, holder));
+
+ if (!prop.isAccessorProperty()) {
+ return false;
+ }
+
+ JSObject* setterObject = holder->getSetter(prop);
+ if (!setterObject || !setterObject->is<JSFunction>()) {
+ return false;
+ }
+
+ JSFunction& setter = setterObject->as<JSFunction>();
+ if (setter.isClassConstructor()) {
+ return false;
+ }
+
+ // Scripted functions and natives with JIT entry can use the scripted path.
+ return setter.hasJitEntry();
+}
+
+static bool CanAttachSetter(JSContext* cx, jsbytecode* pc, JSObject* obj,
+ PropertyKey id, NativeObject** holder,
+ Maybe<PropertyInfo>* propInfo) {
+ // Don't attach a setter stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc)));
+
+ PropertyResult prop;
+ if (!LookupPropertyPure(cx, obj, id, holder, &prop)) {
+ return false;
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ if (!prop.isNativeProperty()) {
+ return false;
+ }
+
+ if (!IsCacheableSetPropCallScripted(nobj, *holder, prop.propertyInfo()) &&
+ !IsCacheableSetPropCallNative(nobj, *holder, prop.propertyInfo())) {
+ return false;
+ }
+
+ *propInfo = mozilla::Some(prop.propertyInfo());
+ return true;
+}
+
+static void EmitCallSetterNoGuards(JSContext* cx, CacheIRWriter& writer,
+ NativeObject* obj, NativeObject* holder,
+ PropertyInfo prop, ObjOperandId receiverId,
+ ValOperandId rhsId) {
+ JSFunction* target = &holder->getSetter(prop)->as<JSFunction>();
+ bool sameRealm = cx->realm() == target->realm();
+
+ if (target->isNativeWithoutJitEntry()) {
+ MOZ_ASSERT(IsCacheableSetPropCallNative(obj, holder, prop));
+ writer.callNativeSetter(receiverId, target, rhsId, sameRealm);
+ writer.returnFromIC();
+ return;
+ }
+
+ MOZ_ASSERT(IsCacheableSetPropCallScripted(obj, holder, prop));
+ writer.callScriptedSetter(receiverId, target, rhsId, sameRealm);
+ writer.returnFromIC();
+}
+
+static void EmitCallDOMSetterNoGuards(JSContext* cx, CacheIRWriter& writer,
+ NativeObject* holder, PropertyInfo prop,
+ ObjOperandId objId, ValOperandId rhsId) {
+ JSFunction* setter = &holder->getSetter(prop)->as<JSFunction>();
+ MOZ_ASSERT(cx->realm() == setter->realm());
+
+ writer.callDOMSetter(objId, setter->jitInfo(), rhsId);
+ writer.returnFromIC();
+}
+
+AttachDecision SetPropIRGenerator::tryAttachSetter(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id,
+ ValOperandId rhsId) {
+ // Don't attach a setter stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ if (!CanAttachSetter(cx_, pc_, obj, id, &holder, &prop)) {
+ return AttachDecision::NoAction;
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ bool needsWindowProxy =
+ IsWindow(nobj) && SetterNeedsWindowProxyThis(holder, *prop);
+
+ maybeEmitIdGuard(id);
+
+ // Use the megamorphic guard if we're in megamorphic mode, except if |obj|
+ // is a Window as GuardHasGetterSetter doesn't support this yet (Window may
+ // require outerizing).
+ if (mode_ == ICState::Mode::Specialized || IsWindow(nobj)) {
+ TestMatchingNativeReceiver(writer, nobj, objId);
+
+ if (nobj != holder) {
+ GeneratePrototypeGuards(writer, nobj, holder, objId);
+
+ // Guard on the holder's shape.
+ ObjOperandId holderId = writer.loadObject(holder);
+ TestMatchingHolder(writer, holder, holderId);
+
+ EmitGuardGetterSetterSlot(writer, holder, *prop, holderId,
+ /* holderIsConstant = */ true);
+ } else {
+ EmitGuardGetterSetterSlot(writer, holder, *prop, objId);
+ }
+ } else {
+ GetterSetter* gs = holder->getGetterSetter(*prop);
+ writer.guardHasGetterSetter(objId, id, gs);
+ }
+
+ if (CanAttachDOMGetterSetter(cx_, JSJitInfo::Setter, nobj, holder, *prop,
+ mode_)) {
+ MOZ_ASSERT(!needsWindowProxy);
+ EmitCallDOMSetterNoGuards(cx_, writer, holder, *prop, objId, rhsId);
+
+ trackAttached("DOMSetter");
+ return AttachDecision::Attach;
+ }
+
+ ObjOperandId receiverId;
+ if (needsWindowProxy) {
+ MOZ_ASSERT(cx_->global()->maybeWindowProxy());
+ receiverId = writer.loadObject(cx_->global()->maybeWindowProxy());
+ } else {
+ receiverId = objId;
+ }
+ EmitCallSetterNoGuards(cx_, writer, nobj, holder, *prop, receiverId, rhsId);
+
+ trackAttached("Setter");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachSetArrayLength(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id,
+ ValOperandId rhsId) {
+ // Don't attach an array length stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ if (!obj->is<ArrayObject>() || !id.isAtom(cx_->names().length) ||
+ !obj->as<ArrayObject>().lengthIsWritable()) {
+ return AttachDecision::NoAction;
+ }
+
+ maybeEmitIdGuard(id);
+ writer.guardClass(objId, GuardClassKind::Array);
+ writer.callSetArrayLength(objId, IsStrictSetPC(pc_), rhsId);
+ writer.returnFromIC();
+
+ trackAttached("SetArrayLength");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachSetDenseElement(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId, ValOperandId rhsId) {
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (!nobj->containsDenseElement(index) || nobj->denseElementsAreFrozen()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Setting holes requires extra code for marking the elements non-packed.
+ MOZ_ASSERT(!rhsVal_.isMagic(JS_ELEMENTS_HOLE));
+
+ JSOp op = JSOp(*pc_);
+
+ // We don't currently emit locked init for any indexed properties.
+ MOZ_ASSERT(!IsLockedInitOp(op));
+
+ // We don't currently emit hidden init for any existing indexed properties.
+ MOZ_ASSERT(!IsHiddenInitOp(op));
+
+ // Don't optimize InitElem (DefineProperty) on non-extensible objects: when
+ // the elements are sealed, we have to throw an exception. Note that we have
+ // to check !isExtensible instead of denseElementsAreSealed because sealing
+ // a (non-extensible) object does not necessarily trigger a Shape change.
+ if (IsPropertyInitOp(op) && !nobj->isExtensible()) {
+ return AttachDecision::NoAction;
+ }
+
+ TestMatchingNativeReceiver(writer, nobj, objId);
+
+ writer.storeDenseElement(objId, indexId, rhsId);
+ writer.returnFromIC();
+
+ trackAttached("SetDenseElement");
+ return AttachDecision::Attach;
+}
+
+static bool CanAttachAddElement(NativeObject* obj, bool isInit,
+ AllowIndexedReceiver allowIndexedReceiver) {
+ // Make sure the receiver doesn't have any indexed properties and that such
+ // properties can't appear without a shape change.
+ if (allowIndexedReceiver == AllowIndexedReceiver::No && obj->isIndexed()) {
+ return false;
+ }
+
+ do {
+ // This check is also relevant for the receiver object.
+ const JSClass* clasp = obj->getClass();
+ if (clasp != &ArrayObject::class_ &&
+ (clasp->getAddProperty() || clasp->getResolve() ||
+ clasp->getOpsLookupProperty() || clasp->getOpsSetProperty())) {
+ return false;
+ }
+
+ // If we're initializing a property instead of setting one, the objects
+ // on the prototype are not relevant.
+ if (isInit) {
+ break;
+ }
+
+ JSObject* proto = obj->staticPrototype();
+ if (!proto) {
+ break;
+ }
+
+ if (!proto->is<NativeObject>()) {
+ return false;
+ }
+
+ NativeObject* nproto = &proto->as<NativeObject>();
+ if (nproto->isIndexed()) {
+ return false;
+ }
+
+ // We have to make sure the proto has no non-writable (frozen) elements
+ // because we're not allowed to shadow them.
+ if (nproto->denseElementsAreFrozen() &&
+ nproto->getDenseInitializedLength() > 0) {
+ return false;
+ }
+
+ obj = nproto;
+ } while (true);
+
+ return true;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachSetDenseElementHole(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId, ValOperandId rhsId) {
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Setting holes requires extra code for marking the elements non-packed.
+ if (rhsVal_.isMagic(JS_ELEMENTS_HOLE)) {
+ return AttachDecision::NoAction;
+ }
+
+ JSOp op = JSOp(*pc_);
+ MOZ_ASSERT(IsPropertySetOp(op) || IsPropertyInitOp(op));
+
+ // We don't currently emit locked init for any indexed properties.
+ MOZ_ASSERT(!IsLockedInitOp(op));
+
+ // Hidden init can be emitted for absent indexed properties.
+ if (IsHiddenInitOp(op)) {
+ MOZ_ASSERT(op == JSOp::InitHiddenElem);
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (!nobj->isExtensible()) {
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(!nobj->denseElementsAreFrozen(),
+ "Extensible objects should not have frozen elements");
+
+ uint32_t initLength = nobj->getDenseInitializedLength();
+
+ // Optimize if we're adding an element at initLength or writing to a hole.
+ //
+ // In the case where index > initLength, we need noteHasDenseAdd to be called
+ // to ensure Ion is aware that writes have occurred to-out-of-bound indexes
+ // before.
+ //
+ // TODO(post-Warp): noteHasDenseAdd (nee: noteArrayWriteHole) no longer exists
+ bool isAdd = index == initLength;
+ bool isHoleInBounds =
+ index < initLength && !nobj->containsDenseElement(index);
+ if (!isAdd && !isHoleInBounds) {
+ return AttachDecision::NoAction;
+ }
+
+ // Can't add new elements to arrays with non-writable length.
+ if (isAdd && nobj->is<ArrayObject>() &&
+ !nobj->as<ArrayObject>().lengthIsWritable()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Typed arrays don't have dense elements.
+ if (nobj->is<TypedArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check for other indexed properties or class hooks.
+ if (!CanAttachAddElement(nobj, IsPropertyInitOp(op),
+ AllowIndexedReceiver::No)) {
+ return AttachDecision::NoAction;
+ }
+
+ TestMatchingNativeReceiver(writer, nobj, objId);
+
+ // Also shape guard the proto chain, unless this is an InitElem.
+ if (IsPropertySetOp(op)) {
+ ShapeGuardProtoChain(writer, nobj, objId);
+ }
+
+ writer.storeDenseElementHole(objId, indexId, rhsId, isAdd);
+ writer.returnFromIC();
+
+ trackAttached(isAdd ? "AddDenseElement" : "StoreDenseElementHole");
+ return AttachDecision::Attach;
+}
+
+// Add an IC for adding or updating a sparse element.
+AttachDecision SetPropIRGenerator::tryAttachAddOrUpdateSparseElement(
+ HandleObject obj, ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId, ValOperandId rhsId) {
+ JSOp op = JSOp(*pc_);
+ MOZ_ASSERT(IsPropertySetOp(op) || IsPropertyInitOp(op));
+
+ if (op != JSOp::SetElem && op != JSOp::StrictSetElem) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+ NativeObject* nobj = &obj->as<NativeObject>();
+
+ // We cannot attach a stub to a non-extensible object
+ if (!nobj->isExtensible()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Stub doesn't handle negative indices.
+ if (index > INT32_MAX) {
+ return AttachDecision::NoAction;
+ }
+
+ // The index must not be for a dense element.
+ if (nobj->containsDenseElement(index)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Only handle ArrayObject and PlainObject in this stub.
+ if (!nobj->is<ArrayObject>() && !nobj->is<PlainObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Don't attach if we're adding to an array with non-writable length.
+ if (nobj->is<ArrayObject>()) {
+ ArrayObject* aobj = &nobj->as<ArrayObject>();
+ bool isAdd = (index >= aobj->length());
+ if (isAdd && !aobj->lengthIsWritable()) {
+ return AttachDecision::NoAction;
+ }
+ }
+
+ // Check for class hooks or indexed properties on the prototype chain that
+ // we're not allowed to shadow.
+ if (!CanAttachAddElement(nobj, /* isInit = */ false,
+ AllowIndexedReceiver::Yes)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure that obj is an ArrayObject or PlainObject.
+ if (nobj->is<ArrayObject>()) {
+ writer.guardClass(objId, GuardClassKind::Array);
+ } else {
+ MOZ_ASSERT(nobj->is<PlainObject>());
+ writer.guardClass(objId, GuardClassKind::PlainObject);
+ }
+
+ // The helper we are going to call only applies to non-dense elements.
+ writer.guardIndexIsNotDenseElement(objId, indexId);
+
+ // Guard extensible: We may be trying to add a new element, and so we'd best
+ // be able to do so safely.
+ writer.guardIsExtensible(objId);
+
+ // Ensures we are able to efficiently able to map to an integral jsid.
+ writer.guardInt32IsNonNegative(indexId);
+
+ // Shape guard the prototype chain to avoid shadowing indexes from appearing.
+ // Guard the prototype of the receiver explicitly, because the receiver's
+ // shape is not being guarded as a proxy for that.
+ GuardReceiverProto(writer, nobj, objId);
+
+ // Dense elements may appear on the prototype chain (and prototypes may
+ // have a different notion of which elements are dense), but they can
+ // only be data properties, so our specialized Set handler is ok to bind
+ // to them.
+ if (IsPropertySetOp(op)) {
+ ShapeGuardProtoChain(writer, nobj, objId);
+ }
+
+ // Ensure that if we're adding an element to the object, the object's
+ // length is writable.
+ if (nobj->is<ArrayObject>()) {
+ writer.guardIndexIsValidUpdateOrAdd(objId, indexId);
+ }
+
+ writer.callAddOrUpdateSparseElementHelper(
+ objId, indexId, rhsId,
+ /* strict = */ op == JSOp::StrictSetElem);
+ writer.returnFromIC();
+
+ trackAttached("AddOrUpdateSparseElement");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachSetTypedArrayElement(
+ HandleObject obj, ObjOperandId objId, ValOperandId rhsId) {
+ if (!obj->is<TypedArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+ if (!idVal_.isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ TypedArrayObject* tarr = &obj->as<TypedArrayObject>();
+ Scalar::Type elementType = tarr->type();
+
+ // Don't attach if the input type doesn't match the guard added below.
+ if (!ValueIsNumeric(elementType, rhsVal_)) {
+ return AttachDecision::NoAction;
+ }
+
+ bool handleOOB = false;
+ int64_t indexInt64;
+ if (!ValueIsInt64Index(idVal_, &indexInt64) || indexInt64 < 0 ||
+ uint64_t(indexInt64) >= tarr->length()) {
+ handleOOB = true;
+ }
+
+ JSOp op = JSOp(*pc_);
+
+ // The only expected property init operation is InitElem.
+ MOZ_ASSERT_IF(IsPropertyInitOp(op), op == JSOp::InitElem);
+
+ // InitElem (DefineProperty) has to throw an exception on out-of-bounds.
+ if (handleOOB && IsPropertyInitOp(op)) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.guardShapeForClass(objId, tarr->shape());
+
+ OperandId rhsValId = emitNumericGuard(rhsId, elementType);
+
+ ValOperandId keyId = setElemKeyValueId();
+ IntPtrOperandId indexId = guardToIntPtrIndex(idVal_, keyId, handleOOB);
+
+ writer.storeTypedArrayElement(objId, elementType, indexId, rhsValId,
+ handleOOB);
+ writer.returnFromIC();
+
+ trackAttached(handleOOB ? "SetTypedElementOOB" : "SetTypedElement");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachGenericProxy(
+ Handle<ProxyObject*> obj, ObjOperandId objId, HandleId id,
+ ValOperandId rhsId, bool handleDOMProxies) {
+ // Don't attach a proxy stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ writer.guardIsProxy(objId);
+
+ if (!handleDOMProxies) {
+ // Ensure that the incoming object is not a DOM proxy, so that we can
+ // get to the specialized stubs. If handleDOMProxies is true, we were
+ // unable to attach a specialized DOM stub, so we just handle all
+ // proxies here.
+ writer.guardIsNotDOMProxy(objId);
+ }
+
+ if (cacheKind_ == CacheKind::SetProp || mode_ == ICState::Mode::Specialized) {
+ maybeEmitIdGuard(id);
+ writer.proxySet(objId, id, rhsId, IsStrictSetPC(pc_));
+ } else {
+ // Attach a stub that handles every id.
+ MOZ_ASSERT(cacheKind_ == CacheKind::SetElem);
+ MOZ_ASSERT(mode_ == ICState::Mode::Megamorphic);
+ writer.proxySetByValue(objId, setElemKeyValueId(), rhsId,
+ IsStrictSetPC(pc_));
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("GenericProxy");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachDOMProxyShadowed(
+ Handle<ProxyObject*> obj, ObjOperandId objId, HandleId id,
+ ValOperandId rhsId) {
+ // Don't attach a proxy stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ MOZ_ASSERT(IsCacheableDOMProxy(obj));
+
+ maybeEmitIdGuard(id);
+ TestMatchingProxyReceiver(writer, obj, objId);
+ writer.proxySet(objId, id, rhsId, IsStrictSetPC(pc_));
+ writer.returnFromIC();
+
+ trackAttached("DOMProxyShadowed");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachDOMProxyUnshadowed(
+ Handle<ProxyObject*> obj, ObjOperandId objId, HandleId id,
+ ValOperandId rhsId) {
+ // Don't attach a proxy stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ MOZ_ASSERT(IsCacheableDOMProxy(obj));
+
+ JSObject* proto = obj->staticPrototype();
+ if (!proto) {
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* holder = nullptr;
+ Maybe<PropertyInfo> prop;
+ if (!CanAttachSetter(cx_, pc_, proto, id, &holder, &prop)) {
+ return AttachDecision::NoAction;
+ }
+ auto* nproto = &proto->as<NativeObject>();
+
+ maybeEmitIdGuard(id);
+
+ // Guard that our expando object hasn't started shadowing this property.
+ TestMatchingProxyReceiver(writer, obj, objId);
+ CheckDOMProxyExpandoDoesNotShadow(writer, obj, id, objId);
+
+ GeneratePrototypeGuards(writer, obj, holder, objId);
+
+ // Guard on the holder of the property.
+ ObjOperandId holderId = writer.loadObject(holder);
+ TestMatchingHolder(writer, holder, holderId);
+
+ EmitGuardGetterSetterSlot(writer, holder, *prop, holderId,
+ /* holderIsConstant = */ true);
+
+ // EmitCallSetterNoGuards expects |obj| to be the object the property is
+ // on to do some checks. Since we actually looked at proto, and no extra
+ // guards will be generated, we can just pass that instead.
+ EmitCallSetterNoGuards(cx_, writer, nproto, holder, *prop, objId, rhsId);
+
+ trackAttached("DOMProxyUnshadowed");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachDOMProxyExpando(
+ Handle<ProxyObject*> obj, ObjOperandId objId, HandleId id,
+ ValOperandId rhsId) {
+ // Don't attach a proxy stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ MOZ_ASSERT(IsCacheableDOMProxy(obj));
+
+ Value expandoVal = GetProxyPrivate(obj);
+ JSObject* expandoObj;
+ if (expandoVal.isObject()) {
+ expandoObj = &expandoVal.toObject();
+ } else {
+ MOZ_ASSERT(!expandoVal.isUndefined(),
+ "How did a missing expando manage to shadow things?");
+ auto expandoAndGeneration =
+ static_cast<ExpandoAndGeneration*>(expandoVal.toPrivate());
+ MOZ_ASSERT(expandoAndGeneration);
+ expandoObj = &expandoAndGeneration->expando.toObject();
+ }
+
+ Maybe<PropertyInfo> prop;
+ if (CanAttachNativeSetSlot(JSOp(*pc_), expandoObj, id, &prop)) {
+ auto* nativeExpandoObj = &expandoObj->as<NativeObject>();
+
+ maybeEmitIdGuard(id);
+ ObjOperandId expandoObjId = guardDOMProxyExpandoObjectAndShape(
+ obj, objId, expandoVal, nativeExpandoObj);
+
+ EmitStoreSlotAndReturn(writer, expandoObjId, nativeExpandoObj, *prop,
+ rhsId);
+ trackAttached("DOMProxyExpandoSlot");
+ return AttachDecision::Attach;
+ }
+
+ NativeObject* holder = nullptr;
+ if (CanAttachSetter(cx_, pc_, expandoObj, id, &holder, &prop)) {
+ auto* nativeExpandoObj = &expandoObj->as<NativeObject>();
+
+ // Call the setter. Note that we pass objId, the DOM proxy, as |this|
+ // and not the expando object.
+ maybeEmitIdGuard(id);
+ ObjOperandId expandoObjId = guardDOMProxyExpandoObjectAndShape(
+ obj, objId, expandoVal, nativeExpandoObj);
+
+ MOZ_ASSERT(holder == nativeExpandoObj);
+ EmitGuardGetterSetterSlot(writer, nativeExpandoObj, *prop, expandoObjId);
+ EmitCallSetterNoGuards(cx_, writer, nativeExpandoObj, nativeExpandoObj,
+ *prop, objId, rhsId);
+ trackAttached("DOMProxyExpandoSetter");
+ return AttachDecision::Attach;
+ }
+
+ return AttachDecision::NoAction;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachProxy(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id,
+ ValOperandId rhsId) {
+ // Don't attach a proxy stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ ProxyStubType type = GetProxyStubType(cx_, obj, id);
+ if (type == ProxyStubType::None) {
+ return AttachDecision::NoAction;
+ }
+ auto proxy = obj.as<ProxyObject>();
+
+ if (mode_ == ICState::Mode::Megamorphic) {
+ return tryAttachGenericProxy(proxy, objId, id, rhsId,
+ /* handleDOMProxies = */ true);
+ }
+
+ switch (type) {
+ case ProxyStubType::None:
+ break;
+ case ProxyStubType::DOMExpando:
+ TRY_ATTACH(tryAttachDOMProxyExpando(proxy, objId, id, rhsId));
+ [[fallthrough]]; // Fall through to the generic shadowed case.
+ case ProxyStubType::DOMShadowed:
+ return tryAttachDOMProxyShadowed(proxy, objId, id, rhsId);
+ case ProxyStubType::DOMUnshadowed:
+ TRY_ATTACH(tryAttachDOMProxyUnshadowed(proxy, objId, id, rhsId));
+ return tryAttachGenericProxy(proxy, objId, id, rhsId,
+ /* handleDOMProxies = */ true);
+ case ProxyStubType::Generic:
+ return tryAttachGenericProxy(proxy, objId, id, rhsId,
+ /* handleDOMProxies = */ false);
+ }
+
+ MOZ_CRASH("Unexpected ProxyStubType");
+}
+
+AttachDecision SetPropIRGenerator::tryAttachProxyElement(HandleObject obj,
+ ObjOperandId objId,
+ ValOperandId rhsId) {
+ // Don't attach a proxy stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ if (!obj->is<ProxyObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.guardIsProxy(objId);
+
+ // Like GetPropIRGenerator::tryAttachProxyElement, don't check for DOM
+ // proxies here as we don't have specialized DOM stubs for this.
+ MOZ_ASSERT(cacheKind_ == CacheKind::SetElem);
+ writer.proxySetByValue(objId, setElemKeyValueId(), rhsId, IsStrictSetPC(pc_));
+ writer.returnFromIC();
+
+ trackAttached("ProxyElement");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachMegamorphicSetElement(
+ HandleObject obj, ObjOperandId objId, ValOperandId rhsId) {
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ if (mode_ != ICState::Mode::Megamorphic || cacheKind_ != CacheKind::SetElem) {
+ return AttachDecision::NoAction;
+ }
+
+ // The generic proxy stubs are faster.
+ if (obj->is<ProxyObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.megamorphicSetElement(objId, setElemKeyValueId(), rhsId,
+ IsStrictSetPC(pc_));
+ writer.returnFromIC();
+
+ trackAttached("MegamorphicSetElement");
+ return AttachDecision::Attach;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachWindowProxy(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id,
+ ValOperandId rhsId) {
+ // Don't attach a window proxy stub for ops like JSOp::InitElem.
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc_)));
+
+ // Attach a stub when the receiver is a WindowProxy and we can do the set
+ // on the Window (the global object).
+
+ if (!IsWindowProxyForScriptGlobal(script_, obj)) {
+ return AttachDecision::NoAction;
+ }
+
+ // If we're megamorphic prefer a generic proxy stub that handles a lot more
+ // cases.
+ if (mode_ == ICState::Mode::Megamorphic) {
+ return AttachDecision::NoAction;
+ }
+
+ // Now try to do the set on the Window (the current global).
+ GlobalObject* windowObj = cx_->global();
+
+ Maybe<PropertyInfo> prop;
+ if (!CanAttachNativeSetSlot(JSOp(*pc_), windowObj, id, &prop)) {
+ return AttachDecision::NoAction;
+ }
+
+ maybeEmitIdGuard(id);
+
+ ObjOperandId windowObjId =
+ GuardAndLoadWindowProxyWindow(writer, objId, windowObj);
+ writer.guardShape(windowObjId, windowObj->shape());
+
+ EmitStoreSlotAndReturn(writer, windowObjId, windowObj, *prop, rhsId);
+
+ trackAttached("WindowProxySlot");
+ return AttachDecision::Attach;
+}
+
+// Detect if |id| refers to the 'prototype' property of a function object. This
+// property is special-cased in canAttachAddSlotStub().
+static bool IsFunctionPrototype(const JSAtomState& names, JSObject* obj,
+ PropertyKey id) {
+ return obj->is<JSFunction>() && id.isAtom(names.prototype);
+}
+
+bool SetPropIRGenerator::canAttachAddSlotStub(HandleObject obj, HandleId id) {
+ if (!obj->is<NativeObject>()) {
+ return false;
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ // Special-case JSFunction resolve hook to allow redefining the 'prototype'
+ // property without triggering lazy expansion of property and object
+ // allocation.
+ if (IsFunctionPrototype(cx_->names(), nobj, id)) {
+ MOZ_ASSERT(ClassMayResolveId(cx_->names(), nobj->getClass(), id, nobj));
+
+ // We're only interested in functions that have a builtin .prototype
+ // property (needsPrototypeProperty). The stub will guard on this because
+ // the builtin .prototype property is non-configurable/non-enumerable and it
+ // would be wrong to add a property with those attributes to a function that
+ // doesn't have a builtin .prototype.
+ //
+ // Inlining needsPrototypeProperty in JIT code is complicated so we use
+ // isNonBuiltinConstructor as a stronger condition that's easier to check
+ // from JIT code.
+ JSFunction* fun = &nobj->as<JSFunction>();
+ if (!fun->isNonBuiltinConstructor()) {
+ return false;
+ }
+ MOZ_ASSERT(fun->needsPrototypeProperty());
+
+ // If property exists this isn't an "add".
+ if (fun->lookupPure(id)) {
+ return false;
+ }
+ } else {
+ // Normal Case: If property exists this isn't an "add"
+ PropertyResult prop;
+ if (!LookupOwnPropertyPure(cx_, nobj, id, &prop)) {
+ return false;
+ }
+ if (prop.isFound()) {
+ return false;
+ }
+ }
+
+ // For now we don't optimize Watchtower-monitored objects.
+ if (Watchtower::watchesPropertyAdd(nobj)) {
+ return false;
+ }
+
+ // Object must be extensible, or we must be initializing a private
+ // elem.
+ bool canAddNewProperty = nobj->isExtensible() || id.isPrivateName();
+ if (!canAddNewProperty) {
+ return false;
+ }
+
+ JSOp op = JSOp(*pc_);
+ if (IsPropertyInitOp(op)) {
+ return true;
+ }
+
+ MOZ_ASSERT(IsPropertySetOp(op));
+
+ // Walk up the object prototype chain and ensure that all prototypes are
+ // native, and that all prototypes have no setter defined on the property.
+ for (JSObject* proto = nobj->staticPrototype(); proto;
+ proto = proto->staticPrototype()) {
+ if (!proto->is<NativeObject>()) {
+ return false;
+ }
+
+ // If prototype defines this property in a non-plain way, don't optimize.
+ Maybe<PropertyInfo> protoProp = proto->as<NativeObject>().lookup(cx_, id);
+ if (protoProp.isSome() && !protoProp->isDataProperty()) {
+ return false;
+ }
+
+ // Otherwise, if there's no such property, watch out for a resolve hook
+ // that would need to be invoked and thus prevent inlining of property
+ // addition. Allow the JSFunction resolve hook as it only defines plain
+ // data properties and we don't need to invoke it for objects on the
+ // proto chain.
+ if (ClassMayResolveId(cx_->names(), proto->getClass(), id, proto) &&
+ !proto->is<JSFunction>()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static PropertyFlags SetPropertyFlags(JSOp op, bool isFunctionPrototype) {
+ // Locked properties are non-writable, non-enumerable, and non-configurable.
+ if (IsLockedInitOp(op)) {
+ return {};
+ }
+
+ // Hidden properties are writable, non-enumerable, and configurable.
+ if (IsHiddenInitOp(op)) {
+ return {
+ PropertyFlag::Writable,
+ PropertyFlag::Configurable,
+ };
+ }
+
+ // This is a special case to overwrite an unresolved function.prototype
+ // property. The initial property flags of this property are writable,
+ // non-enumerable, and non-configurable. See canAttachAddSlotStub.
+ if (isFunctionPrototype) {
+ return {
+ PropertyFlag::Writable,
+ };
+ }
+
+ // Other properties are writable, enumerable, and configurable.
+ return PropertyFlags::defaultDataPropFlags;
+}
+
+AttachDecision SetPropIRGenerator::tryAttachAddSlotStub(
+ Handle<Shape*> oldShape) {
+ ValOperandId objValId(writer.setInputOperandId(0));
+ ValOperandId rhsValId;
+ if (cacheKind_ == CacheKind::SetProp) {
+ rhsValId = ValOperandId(writer.setInputOperandId(1));
+ } else {
+ MOZ_ASSERT(cacheKind_ == CacheKind::SetElem);
+ MOZ_ASSERT(setElemKeyValueId().id() == 1);
+ writer.setInputOperandId(1);
+ rhsValId = ValOperandId(writer.setInputOperandId(2));
+ }
+
+ RootedId id(cx_);
+ bool nameOrSymbol;
+ if (!ValueToNameOrSymbolId(cx_, idVal_, &id, &nameOrSymbol)) {
+ cx_->clearPendingException();
+ return AttachDecision::NoAction;
+ }
+
+ if (!lhsVal_.isObject() || !nameOrSymbol) {
+ return AttachDecision::NoAction;
+ }
+
+ JSObject* obj = &lhsVal_.toObject();
+
+ PropertyResult prop;
+ if (!LookupOwnPropertyPure(cx_, obj, id, &prop)) {
+ return AttachDecision::NoAction;
+ }
+ if (prop.isNotFound()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!obj->is<NativeObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ PropertyInfo propInfo = prop.propertyInfo();
+ NativeObject* holder = nobj;
+
+ if (holder->inDictionaryMode()) {
+ return AttachDecision::NoAction;
+ }
+
+ SharedShape* oldSharedShape = &oldShape->asShared();
+
+ // The property must be the last added property of the object.
+ SharedShape* newShape = holder->sharedShape();
+ MOZ_RELEASE_ASSERT(newShape->lastProperty() == propInfo);
+
+#ifdef DEBUG
+ // Verify exactly one property was added by comparing the property map
+ // lengths.
+ if (oldSharedShape->propMapLength() == PropMap::Capacity) {
+ MOZ_ASSERT(newShape->propMapLength() == 1);
+ } else {
+ MOZ_ASSERT(newShape->propMapLength() ==
+ oldSharedShape->propMapLength() + 1);
+ }
+#endif
+
+ bool isFunctionPrototype = IsFunctionPrototype(cx_->names(), nobj, id);
+
+ JSOp op = JSOp(*pc_);
+ PropertyFlags flags = SetPropertyFlags(op, isFunctionPrototype);
+
+ // Basic property checks.
+ if (!propInfo.isDataProperty() || propInfo.flags() != flags) {
+ return AttachDecision::NoAction;
+ }
+
+ ObjOperandId objId = writer.guardToObject(objValId);
+ maybeEmitIdGuard(id);
+
+ // Shape guard the object.
+ writer.guardShape(objId, oldShape);
+
+ // If this is the special function.prototype case, we need to guard the
+ // function is a non-builtin constructor. See canAttachAddSlotStub.
+ if (isFunctionPrototype) {
+ MOZ_ASSERT(nobj->as<JSFunction>().isNonBuiltinConstructor());
+ writer.guardFunctionIsNonBuiltinCtor(objId);
+ }
+
+ // Also shape guard the proto chain, unless this is an InitElem.
+ if (IsPropertySetOp(op)) {
+ ShapeGuardProtoChain(writer, nobj, objId);
+ }
+
+ // If the JSClass has an addProperty hook, we need to call a VM function to
+ // invoke this hook. Ignore the Array addProperty hook, because it doesn't do
+ // anything for non-index properties.
+ DebugOnly<uint32_t> index;
+ MOZ_ASSERT_IF(obj->is<ArrayObject>(), !IdIsIndex(id, &index));
+ bool mustCallAddPropertyHook =
+ obj->getClass()->getAddProperty() && !obj->is<ArrayObject>();
+
+ if (mustCallAddPropertyHook) {
+ writer.addSlotAndCallAddPropHook(objId, rhsValId, newShape);
+ trackAttached("AddSlotWithAddPropertyHook");
+ } else if (holder->isFixedSlot(propInfo.slot())) {
+ size_t offset = NativeObject::getFixedSlotOffset(propInfo.slot());
+ writer.addAndStoreFixedSlot(objId, offset, rhsValId, newShape);
+ trackAttached("AddSlot");
+ } else {
+ size_t offset = holder->dynamicSlotIndex(propInfo.slot()) * sizeof(Value);
+ uint32_t numOldSlots = NativeObject::calculateDynamicSlots(oldSharedShape);
+ uint32_t numNewSlots = holder->numDynamicSlots();
+ if (numOldSlots == numNewSlots) {
+ writer.addAndStoreDynamicSlot(objId, offset, rhsValId, newShape);
+ trackAttached("AddSlot");
+ } else {
+ MOZ_ASSERT(numNewSlots > numOldSlots);
+ writer.allocateAndStoreDynamicSlot(objId, offset, rhsValId, newShape,
+ numNewSlots);
+ trackAttached("AllocateSlot");
+ }
+ }
+ writer.returnFromIC();
+
+ return AttachDecision::Attach;
+}
+
+InstanceOfIRGenerator::InstanceOfIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleValue lhs, HandleObject rhs)
+ : IRGenerator(cx, script, pc, CacheKind::InstanceOf, state),
+ lhsVal_(lhs),
+ rhsObj_(rhs) {}
+
+AttachDecision InstanceOfIRGenerator::tryAttachStub() {
+ MOZ_ASSERT(cacheKind_ == CacheKind::InstanceOf);
+ AutoAssertNoPendingException aanpe(cx_);
+
+ // Ensure RHS is a function -- could be a Proxy, which the IC isn't prepared
+ // to handle.
+ if (!rhsObj_->is<JSFunction>()) {
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ HandleFunction fun = rhsObj_.as<JSFunction>();
+
+ if (fun->isBoundFunction()) {
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ // Look up the @@hasInstance property, and check that Function.__proto__ is
+ // the property holder, and that no object further down the prototype chain
+ // (including this function) has shadowed it; together with the fact that
+ // Function.__proto__[@@hasInstance] is immutable, this ensures that the
+ // hasInstance hook will not change without the need to guard on the actual
+ // property value.
+ PropertyResult hasInstanceProp;
+ NativeObject* hasInstanceHolder = nullptr;
+ jsid hasInstanceID = PropertyKey::Symbol(cx_->wellKnownSymbols().hasInstance);
+ if (!LookupPropertyPure(cx_, fun, hasInstanceID, &hasInstanceHolder,
+ &hasInstanceProp) ||
+ !hasInstanceProp.isNativeProperty()) {
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ JSObject& funProto = cx_->global()->getPrototype(JSProto_Function);
+ if (hasInstanceHolder != &funProto) {
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ // If the above succeeded, then these should be true about @@hasInstance,
+ // because the property on Function.__proto__ is an immutable data property:
+ MOZ_ASSERT(hasInstanceProp.propertyInfo().isDataProperty());
+ MOZ_ASSERT(!hasInstanceProp.propertyInfo().configurable());
+ MOZ_ASSERT(!hasInstanceProp.propertyInfo().writable());
+
+ MOZ_ASSERT(IsCacheableProtoChain(fun, hasInstanceHolder));
+
+ // Ensure that the function's prototype slot is the same.
+ Maybe<PropertyInfo> prop = fun->lookupPure(cx_->names().prototype);
+ if (prop.isNothing() || !prop->isDataProperty()) {
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ uint32_t slot = prop->slot();
+ MOZ_ASSERT(slot >= fun->numFixedSlots(), "Stub code relies on this");
+ if (!fun->getSlot(slot).isObject()) {
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+ }
+
+ // Abstract Objects
+ ValOperandId lhs(writer.setInputOperandId(0));
+ ValOperandId rhs(writer.setInputOperandId(1));
+
+ ObjOperandId rhsId = writer.guardToObject(rhs);
+ writer.guardShape(rhsId, fun->shape());
+
+ // Ensure that the shapes up the prototype chain for the RHS remain the same
+ // so that @@hasInstance is not shadowed by some intermediate prototype
+ // object.
+ if (hasInstanceHolder != fun) {
+ GeneratePrototypeGuards(writer, fun, hasInstanceHolder, rhsId);
+ ObjOperandId holderId = writer.loadObject(hasInstanceHolder);
+ TestMatchingHolder(writer, hasInstanceHolder, holderId);
+ }
+
+ // Load the .prototype value and ensure it's an object.
+ ValOperandId protoValId =
+ writer.loadDynamicSlot(rhsId, slot - fun->numFixedSlots());
+ ObjOperandId protoId = writer.guardToObject(protoValId);
+
+ // Needn't guard LHS is object, because the actual stub can handle that
+ // and correctly return false.
+ writer.loadInstanceOfObjectResult(lhs, protoId);
+ writer.returnFromIC();
+ trackAttached("InstanceOf");
+ return AttachDecision::Attach;
+}
+
+void InstanceOfIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("lhs", lhsVal_);
+ sp.valueProperty("rhs", ObjectValue(*rhsObj_));
+ }
+#else
+ // Silence Clang -Wunused-private-field warning.
+ (void)lhsVal_;
+#endif
+}
+
+TypeOfIRGenerator::TypeOfIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleValue value)
+ : IRGenerator(cx, script, pc, CacheKind::TypeOf, state), val_(value) {}
+
+void TypeOfIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("val", val_);
+ }
+#endif
+}
+
+AttachDecision TypeOfIRGenerator::tryAttachStub() {
+ MOZ_ASSERT(cacheKind_ == CacheKind::TypeOf);
+
+ AutoAssertNoPendingException aanpe(cx_);
+
+ ValOperandId valId(writer.setInputOperandId(0));
+
+ TRY_ATTACH(tryAttachPrimitive(valId));
+ TRY_ATTACH(tryAttachObject(valId));
+
+ MOZ_ASSERT_UNREACHABLE("Failed to attach TypeOf");
+ return AttachDecision::NoAction;
+}
+
+AttachDecision TypeOfIRGenerator::tryAttachPrimitive(ValOperandId valId) {
+ if (!val_.isPrimitive()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Note: we don't use GuardIsNumber for int32 values because it's less
+ // efficient in Warp (unboxing to double instead of int32).
+ if (val_.isDouble()) {
+ writer.guardIsNumber(valId);
+ } else {
+ writer.guardNonDoubleType(valId, val_.type());
+ }
+
+ writer.loadConstantStringResult(
+ TypeName(js::TypeOfValue(val_), cx_->names()));
+ writer.returnFromIC();
+ writer.setTypeData(TypeData(JSValueType(val_.type())));
+ trackAttached("Primitive");
+ return AttachDecision::Attach;
+}
+
+AttachDecision TypeOfIRGenerator::tryAttachObject(ValOperandId valId) {
+ if (!val_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ ObjOperandId objId = writer.guardToObject(valId);
+ writer.loadTypeOfObjectResult(objId);
+ writer.returnFromIC();
+ writer.setTypeData(TypeData(JSValueType(val_.type())));
+ trackAttached("Object");
+ return AttachDecision::Attach;
+}
+
+GetIteratorIRGenerator::GetIteratorIRGenerator(JSContext* cx,
+ HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleValue value)
+ : IRGenerator(cx, script, pc, CacheKind::GetIterator, state), val_(value) {}
+
+AttachDecision GetIteratorIRGenerator::tryAttachStub() {
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetIterator);
+
+ AutoAssertNoPendingException aanpe(cx_);
+
+ ValOperandId valId(writer.setInputOperandId(0));
+
+ if (mode_ == ICState::Mode::Megamorphic) {
+ TRY_ATTACH(tryAttachMegamorphic(valId));
+ return AttachDecision::NoAction;
+ }
+
+ TRY_ATTACH(tryAttachNativeIterator(valId));
+ TRY_ATTACH(tryAttachNullOrUndefined(valId));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+AttachDecision GetIteratorIRGenerator::tryAttachNativeIterator(
+ ValOperandId valId) {
+ MOZ_ASSERT(JSOp(*pc_) == JSOp::Iter);
+
+ if (!val_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ RootedObject obj(cx_, &val_.toObject());
+ ObjOperandId objId = writer.guardToObject(valId);
+
+ PropertyIteratorObject* iterobj = LookupInIteratorCache(cx_, obj);
+ if (!iterobj) {
+ return AttachDecision::NoAction;
+ }
+ auto* nobj = &obj->as<NativeObject>();
+
+ // Guard on the receiver's shape.
+ TestMatchingNativeReceiver(writer, nobj, objId);
+
+ // Ensure the receiver has no dense elements.
+ writer.guardNoDenseElements(objId);
+
+ // Do the same for the objects on the proto chain.
+ GeneratePrototypeHoleGuards(writer, nobj, objId,
+ /* alwaysGuardFirstProto = */ false);
+
+ ObjOperandId iterId = writer.guardAndGetIterator(
+ objId, iterobj, cx_->compartment()->enumeratorsAddr());
+ writer.loadObjectResult(iterId);
+ writer.returnFromIC();
+
+ trackAttached("NativeIterator");
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetIteratorIRGenerator::tryAttachMegamorphic(
+ ValOperandId valId) {
+ MOZ_ASSERT(JSOp(*pc_) == JSOp::Iter);
+
+ if (val_.isObject()) {
+ ObjOperandId objId = writer.guardToObject(valId);
+ writer.objectToIteratorResult(objId, cx_->compartment()->enumeratorsAddr());
+ trackAttached("MegamorphicObject");
+ } else {
+ writer.valueToIteratorResult(valId);
+ trackAttached("MegamorphicValue");
+ }
+ writer.returnFromIC();
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision GetIteratorIRGenerator::tryAttachNullOrUndefined(
+ ValOperandId valId) {
+ MOZ_ASSERT(JSOp(*pc_) == JSOp::Iter);
+
+ // For null/undefined we can simply return the empty iterator singleton. This
+ // works because this iterator is unlinked and immutable.
+
+ if (!val_.isNullOrUndefined()) {
+ return AttachDecision::NoAction;
+ }
+
+ PropertyIteratorObject* emptyIter = cx_->global()->maybeEmptyIterator();
+ if (!emptyIter) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.guardIsNullOrUndefined(valId);
+
+ ObjOperandId iterId = writer.loadObject(emptyIter);
+ writer.loadObjectResult(iterId);
+ writer.returnFromIC();
+
+ trackAttached("NullOrUndefined");
+ return AttachDecision::Attach;
+}
+
+void GetIteratorIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("val", val_);
+ }
+#endif
+}
+
+OptimizeSpreadCallIRGenerator::OptimizeSpreadCallIRGenerator(
+ JSContext* cx, HandleScript script, jsbytecode* pc, ICState state,
+ HandleValue value)
+ : IRGenerator(cx, script, pc, CacheKind::OptimizeSpreadCall, state),
+ val_(value) {}
+
+AttachDecision OptimizeSpreadCallIRGenerator::tryAttachStub() {
+ MOZ_ASSERT(cacheKind_ == CacheKind::OptimizeSpreadCall);
+
+ AutoAssertNoPendingException aanpe(cx_);
+
+ TRY_ATTACH(tryAttachArray());
+ TRY_ATTACH(tryAttachArguments());
+ TRY_ATTACH(tryAttachNotOptimizable());
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+static bool IsArrayPrototypeOptimizable(JSContext* cx, ArrayObject* arr,
+ NativeObject** arrProto, uint32_t* slot,
+ JSFunction** iterFun) {
+ // Prototype must be Array.prototype.
+ auto* proto = cx->global()->maybeGetArrayPrototype();
+ if (!proto || arr->staticPrototype() != proto) {
+ return false;
+ }
+ *arrProto = proto;
+
+ // The object must not have an own @@iterator property.
+ PropertyKey iteratorKey =
+ PropertyKey::Symbol(cx->wellKnownSymbols().iterator);
+ if (arr->lookupPure(iteratorKey)) {
+ return false;
+ }
+
+ // Ensure that Array.prototype's @@iterator slot is unchanged.
+ Maybe<PropertyInfo> prop = proto->lookupPure(iteratorKey);
+ if (prop.isNothing() || !prop->isDataProperty()) {
+ return false;
+ }
+
+ *slot = prop->slot();
+ MOZ_ASSERT(proto->numFixedSlots() == 0, "Stub code relies on this");
+
+ const Value& iterVal = proto->getSlot(*slot);
+ if (!iterVal.isObject() || !iterVal.toObject().is<JSFunction>()) {
+ return false;
+ }
+
+ *iterFun = &iterVal.toObject().as<JSFunction>();
+ return IsSelfHostedFunctionWithName(*iterFun, cx->names().ArrayValues);
+}
+
+static bool IsArrayIteratorPrototypeOptimizable(JSContext* cx,
+ NativeObject** arrIterProto,
+ uint32_t* slot,
+ JSFunction** nextFun) {
+ auto* proto = cx->global()->maybeGetArrayIteratorPrototype();
+ if (!proto) {
+ return false;
+ }
+ *arrIterProto = proto;
+
+ // Ensure that %ArrayIteratorPrototype%'s "next" slot is unchanged.
+ Maybe<PropertyInfo> prop = proto->lookupPure(cx->names().next);
+ if (prop.isNothing() || !prop->isDataProperty()) {
+ return false;
+ }
+
+ *slot = prop->slot();
+ MOZ_ASSERT(proto->numFixedSlots() == 0, "Stub code relies on this");
+
+ const Value& nextVal = proto->getSlot(*slot);
+ if (!nextVal.isObject() || !nextVal.toObject().is<JSFunction>()) {
+ return false;
+ }
+
+ *nextFun = &nextVal.toObject().as<JSFunction>();
+ return IsSelfHostedFunctionWithName(*nextFun, cx->names().ArrayIteratorNext);
+}
+
+AttachDecision OptimizeSpreadCallIRGenerator::tryAttachArray() {
+ if (!isFirstStub_) {
+ return AttachDecision::NoAction;
+ }
+
+ // The value must be a packed array.
+ if (!val_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+ JSObject* obj = &val_.toObject();
+ if (!IsPackedArray(obj)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Prototype must be Array.prototype and Array.prototype[@@iterator] must not
+ // be modified.
+ NativeObject* arrProto;
+ uint32_t arrProtoIterSlot;
+ JSFunction* iterFun;
+ if (!IsArrayPrototypeOptimizable(cx_, &obj->as<ArrayObject>(), &arrProto,
+ &arrProtoIterSlot, &iterFun)) {
+ return AttachDecision::NoAction;
+ }
+
+ // %ArrayIteratorPrototype%.next must not be modified.
+ NativeObject* arrayIteratorProto;
+ uint32_t iterNextSlot;
+ JSFunction* nextFun;
+ if (!IsArrayIteratorPrototypeOptimizable(cx_, &arrayIteratorProto,
+ &iterNextSlot, &nextFun)) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ ObjOperandId objId = writer.guardToObject(valId);
+
+ // Guard the object is a packed array with Array.prototype as proto.
+ MOZ_ASSERT(obj->is<ArrayObject>());
+ writer.guardShape(objId, obj->shape());
+ writer.guardArrayIsPacked(objId);
+
+ // Guard on Array.prototype[@@iterator].
+ ObjOperandId arrProtoId = writer.loadObject(arrProto);
+ ObjOperandId iterId = writer.loadObject(iterFun);
+ writer.guardShape(arrProtoId, arrProto->shape());
+ writer.guardDynamicSlotIsSpecificObject(arrProtoId, iterId, arrProtoIterSlot);
+
+ // Guard on %ArrayIteratorPrototype%.next.
+ ObjOperandId iterProtoId = writer.loadObject(arrayIteratorProto);
+ ObjOperandId nextId = writer.loadObject(nextFun);
+ writer.guardShape(iterProtoId, arrayIteratorProto->shape());
+ writer.guardDynamicSlotIsSpecificObject(iterProtoId, nextId, iterNextSlot);
+
+ writer.loadObjectResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("Array");
+ return AttachDecision::Attach;
+}
+
+AttachDecision OptimizeSpreadCallIRGenerator::tryAttachArguments() {
+ // The value must be an arguments object.
+ if (!val_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+ RootedObject obj(cx_, &val_.toObject());
+ if (!obj->is<ArgumentsObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto args = obj.as<ArgumentsObject>();
+
+ // Ensure neither elements, nor the length, nor the iterator has been
+ // overridden. Also ensure no args are forwarded to allow reading them
+ // directly from the frame.
+ if (args->hasOverriddenElement() || args->hasOverriddenLength() ||
+ args->hasOverriddenIterator() || args->anyArgIsForwarded()) {
+ return AttachDecision::NoAction;
+ }
+
+ Rooted<Shape*> shape(cx_, GlobalObject::getArrayShapeWithDefaultProto(cx_));
+ if (!shape) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* arrayIteratorProto;
+ uint32_t slot;
+ JSFunction* nextFun;
+ if (!IsArrayIteratorPrototypeOptimizable(cx_, &arrayIteratorProto, &slot,
+ &nextFun)) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ ObjOperandId objId = writer.guardToObject(valId);
+
+ if (args->is<MappedArgumentsObject>()) {
+ writer.guardClass(objId, GuardClassKind::MappedArguments);
+ } else {
+ MOZ_ASSERT(args->is<UnmappedArgumentsObject>());
+ writer.guardClass(objId, GuardClassKind::UnmappedArguments);
+ }
+ uint8_t flags = ArgumentsObject::ELEMENT_OVERRIDDEN_BIT |
+ ArgumentsObject::LENGTH_OVERRIDDEN_BIT |
+ ArgumentsObject::ITERATOR_OVERRIDDEN_BIT |
+ ArgumentsObject::FORWARDED_ARGUMENTS_BIT;
+ writer.guardArgumentsObjectFlags(objId, flags);
+
+ ObjOperandId protoId = writer.loadObject(arrayIteratorProto);
+ ObjOperandId nextId = writer.loadObject(nextFun);
+
+ writer.guardShape(protoId, arrayIteratorProto->shape());
+
+ // Ensure that proto[slot] == nextFun.
+ writer.guardDynamicSlotIsSpecificObject(protoId, nextId, slot);
+
+ writer.arrayFromArgumentsObjectResult(objId, shape);
+ writer.returnFromIC();
+
+ trackAttached("Arguments");
+ return AttachDecision::Attach;
+}
+
+AttachDecision OptimizeSpreadCallIRGenerator::tryAttachNotOptimizable() {
+ ValOperandId valId(writer.setInputOperandId(0));
+
+ writer.loadUndefinedResult();
+ writer.returnFromIC();
+
+ trackAttached("NotOptimizable");
+ return AttachDecision::Attach;
+}
+
+void OptimizeSpreadCallIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("val", val_);
+ }
+#endif
+}
+
+CallIRGenerator::CallIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, JSOp op, ICState state,
+ uint32_t argc, HandleValue callee,
+ HandleValue thisval, HandleValue newTarget,
+ HandleValueArray args)
+ : IRGenerator(cx, script, pc, CacheKind::Call, state),
+ op_(op),
+ argc_(argc),
+ callee_(callee),
+ thisval_(thisval),
+ newTarget_(newTarget),
+ args_(args) {}
+
+void InlinableNativeIRGenerator::emitNativeCalleeGuard() {
+ // Note: we rely on GuardSpecificFunction to also guard against the same
+ // native from a different realm.
+ MOZ_ASSERT(callee_->isNativeWithoutJitEntry());
+
+ ObjOperandId calleeObjId;
+ if (flags_.getArgFormat() == CallFlags::Standard) {
+ ValOperandId calleeValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Callee, argc_, flags_);
+ calleeObjId = writer.guardToObject(calleeValId);
+ } else if (flags_.getArgFormat() == CallFlags::Spread) {
+ ValOperandId calleeValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Callee, argc_, flags_);
+ calleeObjId = writer.guardToObject(calleeValId);
+ } else if (flags_.getArgFormat() == CallFlags::FunCall) {
+ MOZ_ASSERT(generator_.writer.numOperandIds() > 0, "argcId is initialized");
+
+ Int32OperandId argcId(0);
+ calleeObjId = generator_.emitFunCallOrApplyGuard(argcId);
+ } else {
+ MOZ_ASSERT(flags_.getArgFormat() == CallFlags::FunApplyArray);
+ MOZ_ASSERT(generator_.writer.numOperandIds() > 0, "argcId is initialized");
+
+ Int32OperandId argcId(0);
+ calleeObjId = generator_.emitFunApplyGuard(argcId);
+ }
+
+ writer.guardSpecificFunction(calleeObjId, callee_);
+
+ // If we're constructing we also need to guard newTarget == callee.
+ if (flags_.isConstructing()) {
+ MOZ_ASSERT(flags_.getArgFormat() == CallFlags::Standard);
+ MOZ_ASSERT(&newTarget_.toObject() == callee_);
+
+ ValOperandId newTargetValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::NewTarget, argc_, flags_);
+ ObjOperandId newTargetObjId = writer.guardToObject(newTargetValId);
+ writer.guardSpecificFunction(newTargetObjId, callee_);
+ }
+}
+
+ObjOperandId InlinableNativeIRGenerator::emitLoadArgsArray() {
+ if (flags_.getArgFormat() == CallFlags::Spread) {
+ return writer.loadSpreadArgs();
+ }
+
+ MOZ_ASSERT(flags_.getArgFormat() == CallFlags::FunApplyArray);
+ return generator_.emitFunApplyArgsGuard(flags_.getArgFormat());
+}
+
+void IRGenerator::emitCalleeGuard(ObjOperandId calleeId, JSFunction* callee) {
+ // Guarding on the callee JSFunction* is most efficient, but doesn't work well
+ // for lambda clones (multiple functions with the same BaseScript). We guard
+ // on the function's BaseScript if the callee is scripted and this isn't the
+ // first IC stub.
+ if (isFirstStub_ || !callee->hasBaseScript() ||
+ callee->isSelfHostedBuiltin()) {
+ writer.guardSpecificFunction(calleeId, callee);
+ } else {
+ writer.guardClass(calleeId, GuardClassKind::JSFunction);
+ writer.guardFunctionScript(calleeId, callee->baseScript());
+ }
+}
+
+ObjOperandId CallIRGenerator::emitFunCallOrApplyGuard(Int32OperandId argcId) {
+ JSFunction* callee = &callee_.toObject().as<JSFunction>();
+ MOZ_ASSERT(callee->native() == fun_call || callee->native() == fun_apply);
+
+ // Guard that callee is the |fun_call| or |fun_apply| native function.
+ ValOperandId calleeValId =
+ writer.loadArgumentDynamicSlot(ArgumentKind::Callee, argcId);
+ ObjOperandId calleeObjId = writer.guardToObject(calleeValId);
+ writer.guardSpecificFunction(calleeObjId, callee);
+
+ // Guard that |this| is an object.
+ ValOperandId thisValId =
+ writer.loadArgumentDynamicSlot(ArgumentKind::This, argcId);
+ return writer.guardToObject(thisValId);
+}
+
+ObjOperandId CallIRGenerator::emitFunCallGuard(Int32OperandId argcId) {
+ MOZ_ASSERT(callee_.toObject().as<JSFunction>().native() == fun_call);
+
+ return emitFunCallOrApplyGuard(argcId);
+}
+
+ObjOperandId CallIRGenerator::emitFunApplyGuard(Int32OperandId argcId) {
+ MOZ_ASSERT(callee_.toObject().as<JSFunction>().native() == fun_apply);
+
+ return emitFunCallOrApplyGuard(argcId);
+}
+
+ObjOperandId CallIRGenerator::emitFunApplyArgsGuard(
+ CallFlags::ArgFormat format) {
+ MOZ_ASSERT(argc_ == 2);
+
+ ValOperandId argValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+
+ ObjOperandId argObjId = writer.guardToObject(argValId);
+ if (format == CallFlags::FunApplyArgsObj) {
+ if (args_[1].toObject().is<MappedArgumentsObject>()) {
+ writer.guardClass(argObjId, GuardClassKind::MappedArguments);
+ } else {
+ MOZ_ASSERT(args_[1].toObject().is<UnmappedArgumentsObject>());
+ writer.guardClass(argObjId, GuardClassKind::UnmappedArguments);
+ }
+ uint8_t flags = ArgumentsObject::ELEMENT_OVERRIDDEN_BIT |
+ ArgumentsObject::FORWARDED_ARGUMENTS_BIT;
+ writer.guardArgumentsObjectFlags(argObjId, flags);
+ } else {
+ MOZ_ASSERT(format == CallFlags::FunApplyArray);
+ writer.guardClass(argObjId, GuardClassKind::Array);
+ writer.guardArrayIsPacked(argObjId);
+ }
+
+ return argObjId;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachArrayPush() {
+ // Only optimize on obj.push(val);
+ if (argc_ != 1 || !thisval_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Where |obj| is a native array.
+ JSObject* thisobj = &thisval_.toObject();
+ if (!thisobj->is<ArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto* thisarray = &thisobj->as<ArrayObject>();
+
+ // Check for other indexed properties or class hooks.
+ if (!CanAttachAddElement(thisarray, /* isInit = */ false,
+ AllowIndexedReceiver::No)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Can't add new elements to arrays with non-writable length.
+ if (!thisarray->lengthIsWritable()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check that array is extensible.
+ if (!thisarray->isExtensible()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check that the array is completely initialized (no holes).
+ if (thisarray->getDenseInitializedLength() != thisarray->length()) {
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(!thisarray->denseElementsAreFrozen(),
+ "Extensible arrays should not have frozen elements");
+
+ // After this point, we can generate code fine.
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'push' native function.
+ emitNativeCalleeGuard();
+
+ // Guard this is an array object.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId thisObjId = writer.guardToObject(thisValId);
+
+ // Guard that the shape matches.
+ TestMatchingNativeReceiver(writer, thisarray, thisObjId);
+
+ // Guard proto chain shapes.
+ ShapeGuardProtoChain(writer, thisarray, thisObjId);
+
+ // arr.push(x) is equivalent to arr[arr.length] = x for regular arrays.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ writer.arrayPush(thisObjId, argId);
+
+ writer.returnFromIC();
+
+ trackAttached("ArrayPush");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachArrayPopShift(
+ InlinableNative native) {
+ // Expecting no arguments.
+ if (argc_ != 0) {
+ return AttachDecision::NoAction;
+ }
+
+ // Only optimize if |this| is a packed array.
+ if (!thisval_.isObject() || !IsPackedArray(&thisval_.toObject())) {
+ return AttachDecision::NoAction;
+ }
+
+ // Other conditions:
+ //
+ // * The array length needs to be writable because we're changing it.
+ // * The array must be extensible. Non-extensible arrays require preserving
+ // the |initializedLength == capacity| invariant on ObjectElements.
+ // See NativeObject::shrinkCapacityToInitializedLength.
+ // This also ensures the elements aren't sealed/frozen.
+ // * There must not be a for-in iterator for the elements because the IC stub
+ // does not suppress deleted properties.
+ ArrayObject* arr = &thisval_.toObject().as<ArrayObject>();
+ if (!arr->lengthIsWritable() || !arr->isExtensible() ||
+ arr->denseElementsHaveMaybeInIterationFlag()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'pop' or 'shift' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId objId = writer.guardToObject(thisValId);
+ writer.guardClass(objId, GuardClassKind::Array);
+
+ if (native == InlinableNative::ArrayPop) {
+ writer.packedArrayPopResult(objId);
+ } else {
+ MOZ_ASSERT(native == InlinableNative::ArrayShift);
+ writer.packedArrayShiftResult(objId);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("ArrayPopShift");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachArrayJoin() {
+ // Only handle argc <= 1.
+ if (argc_ > 1) {
+ return AttachDecision::NoAction;
+ }
+
+ // Only optimize if |this| is an array.
+ if (!thisval_.isObject() || !thisval_.toObject().is<ArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // The separator argument must be a string, if present.
+ if (argc_ > 0 && !args_[0].isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // IC stub code can handle non-packed array.
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'join' native function.
+ emitNativeCalleeGuard();
+
+ // Guard this is an array object.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId thisObjId = writer.guardToObject(thisValId);
+ writer.guardClass(thisObjId, GuardClassKind::Array);
+
+ StringOperandId sepId;
+ if (argc_ == 1) {
+ // If argcount is 1, guard that the argument is a string.
+ ValOperandId argValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ sepId = writer.guardToString(argValId);
+ } else {
+ sepId = writer.loadConstantString(cx_->names().comma);
+ }
+
+ // Do the join.
+ writer.arrayJoinResult(thisObjId, sepId);
+
+ writer.returnFromIC();
+
+ trackAttached("ArrayJoin");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachArraySlice() {
+ // Only handle argc <= 2.
+ if (argc_ > 2) {
+ return AttachDecision::NoAction;
+ }
+
+ // Only optimize if |this| is a packed array or an arguments object.
+ if (!thisval_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ bool isPackedArray = IsPackedArray(&thisval_.toObject());
+ if (!isPackedArray) {
+ if (!thisval_.toObject().is<ArgumentsObject>()) {
+ return AttachDecision::NoAction;
+ }
+ auto* args = &thisval_.toObject().as<ArgumentsObject>();
+
+ // No elements must have been overridden or deleted.
+ if (args->hasOverriddenElement()) {
+ return AttachDecision::NoAction;
+ }
+
+ // The length property mustn't be overridden.
+ if (args->hasOverriddenLength()) {
+ return AttachDecision::NoAction;
+ }
+
+ // And finally also check that no argument is forwarded.
+ if (args->anyArgIsForwarded()) {
+ return AttachDecision::NoAction;
+ }
+ }
+
+ // Arguments for the sliced region must be integers.
+ if (argc_ > 0 && !args_[0].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+ if (argc_ > 1 && !args_[1].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ JSObject* templateObj = NewDenseFullyAllocatedArray(cx_, 0, TenuredObject);
+ if (!templateObj) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'slice' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId objId = writer.guardToObject(thisValId);
+
+ if (isPackedArray) {
+ writer.guardClass(objId, GuardClassKind::Array);
+ } else {
+ auto* args = &thisval_.toObject().as<ArgumentsObject>();
+
+ if (args->is<MappedArgumentsObject>()) {
+ writer.guardClass(objId, GuardClassKind::MappedArguments);
+ } else {
+ MOZ_ASSERT(args->is<UnmappedArgumentsObject>());
+ writer.guardClass(objId, GuardClassKind::UnmappedArguments);
+ }
+
+ uint8_t flags = ArgumentsObject::ELEMENT_OVERRIDDEN_BIT |
+ ArgumentsObject::LENGTH_OVERRIDDEN_BIT |
+ ArgumentsObject::FORWARDED_ARGUMENTS_BIT;
+ writer.guardArgumentsObjectFlags(objId, flags);
+ }
+
+ Int32OperandId int32BeginId;
+ if (argc_ > 0) {
+ ValOperandId beginId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ int32BeginId = writer.guardToInt32(beginId);
+ } else {
+ int32BeginId = writer.loadInt32Constant(0);
+ }
+
+ Int32OperandId int32EndId;
+ if (argc_ > 1) {
+ ValOperandId endId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ int32EndId = writer.guardToInt32(endId);
+ } else if (isPackedArray) {
+ int32EndId = writer.loadInt32ArrayLength(objId);
+ } else {
+ int32EndId = writer.loadArgumentsObjectLength(objId);
+ }
+
+ if (isPackedArray) {
+ writer.packedArraySliceResult(templateObj, objId, int32BeginId, int32EndId);
+ } else {
+ writer.argumentsSliceResult(templateObj, objId, int32BeginId, int32EndId);
+ }
+ writer.returnFromIC();
+
+ trackAttached(isPackedArray ? "ArraySlice" : "ArgumentsSlice");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachArrayIsArray() {
+ // Need a single argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'isArray' native function.
+ emitNativeCalleeGuard();
+
+ // Check if the argument is an Array and return result.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ writer.isArrayResult(argId);
+ writer.returnFromIC();
+
+ trackAttached("ArrayIsArray");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachDataViewGet(
+ Scalar::Type type) {
+ // Ensure |this| is a DataViewObject.
+ if (!thisval_.isObject() || !thisval_.toObject().is<DataViewObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Expected arguments: offset (number), optional littleEndian (boolean).
+ if (argc_ < 1 || argc_ > 2) {
+ return AttachDecision::NoAction;
+ }
+ int64_t offsetInt64;
+ if (!ValueIsInt64Index(args_[0], &offsetInt64)) {
+ return AttachDecision::NoAction;
+ }
+ if (argc_ > 1 && !args_[1].isBoolean()) {
+ return AttachDecision::NoAction;
+ }
+
+ DataViewObject* dv = &thisval_.toObject().as<DataViewObject>();
+
+ // Bounds check the offset.
+ if (offsetInt64 < 0 ||
+ !dv->offsetIsInBounds(Scalar::byteSize(type), offsetInt64)) {
+ return AttachDecision::NoAction;
+ }
+
+ // For getUint32 we let the stub return an Int32 if we have not seen a
+ // double, to allow better codegen in Warp while avoiding bailout loops.
+ bool forceDoubleForUint32 = false;
+ if (type == Scalar::Uint32) {
+ bool isLittleEndian = argc_ > 1 && args_[1].toBoolean();
+ uint32_t res = dv->read<uint32_t>(offsetInt64, isLittleEndian);
+ forceDoubleForUint32 = res >= INT32_MAX;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is this DataView native function.
+ emitNativeCalleeGuard();
+
+ // Guard |this| is a DataViewObject.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId objId = writer.guardToObject(thisValId);
+ writer.guardClass(objId, GuardClassKind::DataView);
+
+ // Convert offset to intPtr.
+ ValOperandId offsetId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ IntPtrOperandId intPtrOffsetId =
+ guardToIntPtrIndex(args_[0], offsetId, /* supportOOB = */ false);
+
+ BooleanOperandId boolLittleEndianId;
+ if (argc_ > 1) {
+ ValOperandId littleEndianId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ boolLittleEndianId = writer.guardToBoolean(littleEndianId);
+ } else {
+ boolLittleEndianId = writer.loadBooleanConstant(false);
+ }
+
+ writer.loadDataViewValueResult(objId, intPtrOffsetId, boolLittleEndianId,
+ type, forceDoubleForUint32);
+ writer.returnFromIC();
+
+ trackAttached("DataViewGet");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachDataViewSet(
+ Scalar::Type type) {
+ // Ensure |this| is a DataViewObject.
+ if (!thisval_.isObject() || !thisval_.toObject().is<DataViewObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Expected arguments: offset (number), value, optional littleEndian (boolean)
+ if (argc_ < 2 || argc_ > 3) {
+ return AttachDecision::NoAction;
+ }
+ int64_t offsetInt64;
+ if (!ValueIsInt64Index(args_[0], &offsetInt64)) {
+ return AttachDecision::NoAction;
+ }
+ if (!ValueIsNumeric(type, args_[1])) {
+ return AttachDecision::NoAction;
+ }
+ if (argc_ > 2 && !args_[2].isBoolean()) {
+ return AttachDecision::NoAction;
+ }
+
+ DataViewObject* dv = &thisval_.toObject().as<DataViewObject>();
+
+ // Bounds check the offset.
+ if (offsetInt64 < 0 ||
+ !dv->offsetIsInBounds(Scalar::byteSize(type), offsetInt64)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is this DataView native function.
+ emitNativeCalleeGuard();
+
+ // Guard |this| is a DataViewObject.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId objId = writer.guardToObject(thisValId);
+ writer.guardClass(objId, GuardClassKind::DataView);
+
+ // Convert offset to intPtr.
+ ValOperandId offsetId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ IntPtrOperandId intPtrOffsetId =
+ guardToIntPtrIndex(args_[0], offsetId, /* supportOOB = */ false);
+
+ // Convert value to number or BigInt.
+ ValOperandId valueId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ OperandId numericValueId = emitNumericGuard(valueId, type);
+
+ BooleanOperandId boolLittleEndianId;
+ if (argc_ > 2) {
+ ValOperandId littleEndianId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ boolLittleEndianId = writer.guardToBoolean(littleEndianId);
+ } else {
+ boolLittleEndianId = writer.loadBooleanConstant(false);
+ }
+
+ writer.storeDataViewValueResult(objId, intPtrOffsetId, numericValueId,
+ boolLittleEndianId, type);
+ writer.returnFromIC();
+
+ trackAttached("DataViewSet");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachUnsafeGetReservedSlot(
+ InlinableNative native) {
+ // Self-hosted code calls this with (object, int32) arguments.
+ MOZ_ASSERT(argc_ == 2);
+ MOZ_ASSERT(args_[0].isObject());
+ MOZ_ASSERT(args_[1].isInt32());
+ MOZ_ASSERT(args_[1].toInt32() >= 0);
+
+ uint32_t slot = uint32_t(args_[1].toInt32());
+ if (slot >= NativeObject::MAX_FIXED_SLOTS) {
+ return AttachDecision::NoAction;
+ }
+ size_t offset = NativeObject::getFixedSlotOffset(slot);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Guard that the first argument is an object.
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(arg0Id);
+
+ // BytecodeEmitter::assertSelfHostedUnsafeGetReservedSlot ensures that the
+ // slot argument is constant. (At least for direct calls)
+
+ switch (native) {
+ case InlinableNative::IntrinsicUnsafeGetReservedSlot:
+ writer.loadFixedSlotResult(objId, offset);
+ break;
+ case InlinableNative::IntrinsicUnsafeGetObjectFromReservedSlot:
+ writer.loadFixedSlotTypedResult(objId, offset, ValueType::Object);
+ break;
+ case InlinableNative::IntrinsicUnsafeGetInt32FromReservedSlot:
+ writer.loadFixedSlotTypedResult(objId, offset, ValueType::Int32);
+ break;
+ case InlinableNative::IntrinsicUnsafeGetStringFromReservedSlot:
+ writer.loadFixedSlotTypedResult(objId, offset, ValueType::String);
+ break;
+ default:
+ MOZ_CRASH("unexpected native");
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("UnsafeGetReservedSlot");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachUnsafeSetReservedSlot() {
+ // Self-hosted code calls this with (object, int32, value) arguments.
+ MOZ_ASSERT(argc_ == 3);
+ MOZ_ASSERT(args_[0].isObject());
+ MOZ_ASSERT(args_[1].isInt32());
+ MOZ_ASSERT(args_[1].toInt32() >= 0);
+
+ uint32_t slot = uint32_t(args_[1].toInt32());
+ if (slot >= NativeObject::MAX_FIXED_SLOTS) {
+ return AttachDecision::NoAction;
+ }
+ size_t offset = NativeObject::getFixedSlotOffset(slot);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Guard that the first argument is an object.
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(arg0Id);
+
+ // BytecodeEmitter::assertSelfHostedUnsafeSetReservedSlot ensures that the
+ // slot argument is constant. (At least for direct calls)
+
+ // Get the value to set.
+ ValOperandId valId = writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+
+ // Set the fixed slot and return undefined.
+ writer.storeFixedSlotUndefinedResult(objId, offset, valId);
+
+ // This stub always returns undefined.
+ writer.returnFromIC();
+
+ trackAttached("UnsafeSetReservedSlot");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachIsSuspendedGenerator() {
+ // The IsSuspendedGenerator intrinsic is only called in
+ // self-hosted code, so it's safe to assume we have a single
+ // argument and the callee is our intrinsic.
+
+ MOZ_ASSERT(argc_ == 1);
+
+ initializeInputOperand();
+
+ // Stack layout here is (bottom to top):
+ // 2: Callee
+ // 1: ThisValue
+ // 0: Arg <-- Top of stack.
+ // We only care about the argument.
+ ValOperandId valId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ // Check whether the argument is a suspended generator.
+ // We don't need guards, because IsSuspendedGenerator returns
+ // false for values that are not generator objects.
+ writer.callIsSuspendedGeneratorResult(valId);
+ writer.returnFromIC();
+
+ trackAttached("IsSuspendedGenerator");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachToObject() {
+ // Self-hosted code calls this with a single argument.
+ MOZ_ASSERT(argc_ == 1);
+
+ // Need a single object argument.
+ // TODO(Warp): Support all or more conversions to object.
+ if (!args_[0].isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Guard that the argument is an object.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(argId);
+
+ // Return the object.
+ writer.loadObjectResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("ToObject");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachToInteger() {
+ // Self-hosted code calls this with a single argument.
+ MOZ_ASSERT(argc_ == 1);
+
+ // Need a single int32 argument.
+ // TODO(Warp): Support all or more conversions to integer.
+ // Make sure to update this code correctly if we ever start
+ // returning non-int32 integers.
+ if (!args_[0].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Guard that the argument is an int32.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId int32Id = writer.guardToInt32(argId);
+
+ // Return the int32.
+ writer.loadInt32Result(int32Id);
+ writer.returnFromIC();
+
+ trackAttached("ToInteger");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachToLength() {
+ // Self-hosted code calls this with a single argument.
+ MOZ_ASSERT(argc_ == 1);
+
+ // Need a single int32 argument.
+ if (!args_[0].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // ToLength(int32) is equivalent to max(int32, 0).
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId int32ArgId = writer.guardToInt32(argId);
+ Int32OperandId zeroId = writer.loadInt32Constant(0);
+ bool isMax = true;
+ Int32OperandId maxId = writer.int32MinMax(isMax, int32ArgId, zeroId);
+ writer.loadInt32Result(maxId);
+ writer.returnFromIC();
+
+ trackAttached("ToLength");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachIsObject() {
+ // Self-hosted code calls this with a single argument.
+ MOZ_ASSERT(argc_ == 1);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Type check the argument and return result.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ writer.isObjectResult(argId);
+ writer.returnFromIC();
+
+ trackAttached("IsObject");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachIsPackedArray() {
+ // Self-hosted code calls this with a single object argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Check if the argument is packed and return result.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objArgId = writer.guardToObject(argId);
+ writer.isPackedArrayResult(objArgId);
+ writer.returnFromIC();
+
+ trackAttached("IsPackedArray");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachIsCallable() {
+ // Self-hosted code calls this with a single argument.
+ MOZ_ASSERT(argc_ == 1);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Check if the argument is callable and return result.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ writer.isCallableResult(argId);
+ writer.returnFromIC();
+
+ trackAttached("IsCallable");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachIsConstructor() {
+ // Self-hosted code calls this with a single argument.
+ MOZ_ASSERT(argc_ == 1);
+
+ // Need a single object argument.
+ if (!args_[0].isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Guard that the argument is an object.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(argId);
+
+ // Check if the argument is a constructor and return result.
+ writer.isConstructorResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("IsConstructor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision
+InlinableNativeIRGenerator::tryAttachIsCrossRealmArrayConstructor() {
+ // Self-hosted code calls this with an object argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ if (args_[0].toObject().is<ProxyObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(argId);
+ writer.guardIsNotProxy(objId);
+ writer.isCrossRealmArrayConstructorResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("IsCrossRealmArrayConstructor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachGuardToClass(
+ InlinableNative native) {
+ // Self-hosted code calls this with an object argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ // Class must match.
+ const JSClass* clasp = InlinableNativeGuardToClass(native);
+ if (args_[0].toObject().getClass() != clasp) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Guard that the argument is an object.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(argId);
+
+ // Guard that the object has the correct class.
+ writer.guardAnyClass(objId, clasp);
+
+ // Return the object.
+ writer.loadObjectResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("GuardToClass");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachHasClass(
+ const JSClass* clasp, bool isPossiblyWrapped) {
+ // Self-hosted code calls this with an object argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ // Only optimize when the object isn't a proxy.
+ if (isPossiblyWrapped && args_[0].toObject().is<ProxyObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Perform the Class check.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(argId);
+
+ if (isPossiblyWrapped) {
+ writer.guardIsNotProxy(objId);
+ }
+
+ writer.hasClassResult(objId, clasp);
+ writer.returnFromIC();
+
+ trackAttached("HasClass");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachRegExpMatcherSearcherTester(
+ InlinableNative native) {
+ // Self-hosted code calls this with (object, string, number) arguments.
+ MOZ_ASSERT(argc_ == 3);
+ MOZ_ASSERT(args_[0].isObject());
+ MOZ_ASSERT(args_[1].isString());
+ MOZ_ASSERT(args_[2].isNumber());
+
+ // It's not guaranteed that the JITs have typed |lastIndex| as an Int32.
+ if (!args_[2].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ // Guard argument types.
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId reId = writer.guardToObject(arg0Id);
+
+ ValOperandId arg1Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ StringOperandId inputId = writer.guardToString(arg1Id);
+
+ ValOperandId arg2Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ Int32OperandId lastIndexId = writer.guardToInt32(arg2Id);
+
+ switch (native) {
+ case InlinableNative::RegExpMatcher:
+ writer.callRegExpMatcherResult(reId, inputId, lastIndexId);
+ writer.returnFromIC();
+ trackAttached("RegExpMatcher");
+ break;
+
+ case InlinableNative::RegExpSearcher:
+ writer.callRegExpSearcherResult(reId, inputId, lastIndexId);
+ writer.returnFromIC();
+ trackAttached("RegExpSearcher");
+ break;
+
+ case InlinableNative::RegExpTester:
+ writer.callRegExpTesterResult(reId, inputId, lastIndexId);
+ writer.returnFromIC();
+ trackAttached("RegExpTester");
+ break;
+
+ default:
+ MOZ_CRASH("Unexpected native");
+ }
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision
+InlinableNativeIRGenerator::tryAttachRegExpPrototypeOptimizable() {
+ // Self-hosted code calls this with a single object argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId protoId = writer.guardToObject(arg0Id);
+
+ writer.regExpPrototypeOptimizableResult(protoId);
+ writer.returnFromIC();
+
+ trackAttached("RegExpPrototypeOptimizable");
+ return AttachDecision::Attach;
+}
+
+AttachDecision
+InlinableNativeIRGenerator::tryAttachRegExpInstanceOptimizable() {
+ // Self-hosted code calls this with two object arguments.
+ MOZ_ASSERT(argc_ == 2);
+ MOZ_ASSERT(args_[0].isObject());
+ MOZ_ASSERT(args_[1].isObject());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId regexpId = writer.guardToObject(arg0Id);
+
+ ValOperandId arg1Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ ObjOperandId protoId = writer.guardToObject(arg1Id);
+
+ writer.regExpInstanceOptimizableResult(regexpId, protoId);
+ writer.returnFromIC();
+
+ trackAttached("RegExpInstanceOptimizable");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachGetFirstDollarIndex() {
+ // Self-hosted code calls this with a single string argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isString());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId strId = writer.guardToString(arg0Id);
+
+ writer.getFirstDollarIndexResult(strId);
+ writer.returnFromIC();
+
+ trackAttached("GetFirstDollarIndex");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachSubstringKernel() {
+ // Self-hosted code calls this with (string, int32, int32) arguments.
+ MOZ_ASSERT(argc_ == 3);
+ MOZ_ASSERT(args_[0].isString());
+ MOZ_ASSERT(args_[1].isInt32());
+ MOZ_ASSERT(args_[2].isInt32());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId strId = writer.guardToString(arg0Id);
+
+ ValOperandId arg1Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ Int32OperandId beginId = writer.guardToInt32(arg1Id);
+
+ ValOperandId arg2Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ Int32OperandId lengthId = writer.guardToInt32(arg2Id);
+
+ writer.callSubstringKernelResult(strId, beginId, lengthId);
+ writer.returnFromIC();
+
+ trackAttached("SubstringKernel");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachObjectHasPrototype() {
+ // Self-hosted code calls this with (object, object) arguments.
+ MOZ_ASSERT(argc_ == 2);
+ MOZ_ASSERT(args_[0].isObject());
+ MOZ_ASSERT(args_[1].isObject());
+
+ auto* obj = &args_[0].toObject().as<NativeObject>();
+ auto* proto = &args_[1].toObject().as<NativeObject>();
+
+ // Only attach when obj.__proto__ is proto.
+ if (obj->staticPrototype() != proto) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(arg0Id);
+
+ writer.guardProto(objId, proto);
+ writer.loadBooleanResult(true);
+ writer.returnFromIC();
+
+ trackAttached("ObjectHasPrototype");
+ return AttachDecision::Attach;
+}
+
+static bool CanConvertToString(const Value& v) {
+ return v.isString() || v.isNumber() || v.isBoolean() || v.isNullOrUndefined();
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachString() {
+ // Need a single argument that is or can be converted to a string.
+ if (argc_ != 1 || !CanConvertToString(args_[0])) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'String' function.
+ emitNativeCalleeGuard();
+
+ // Guard that the argument is a string or can be converted to one.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId strId = emitToStringGuard(argId, args_[0]);
+
+ // Return the string.
+ writer.loadStringResult(strId);
+ writer.returnFromIC();
+
+ trackAttached("String");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringConstructor() {
+ // Need a single argument that is or can be converted to a string.
+ if (argc_ != 1 || !CanConvertToString(args_[0])) {
+ return AttachDecision::NoAction;
+ }
+
+ RootedString emptyString(cx_, cx_->runtime()->emptyString);
+ JSObject* templateObj = StringObject::create(
+ cx_, emptyString, /* proto = */ nullptr, TenuredObject);
+ if (!templateObj) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'String' function.
+ emitNativeCalleeGuard();
+
+ // Guard on number and convert to string.
+ ValOperandId argId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_, flags_);
+ StringOperandId strId = emitToStringGuard(argId, args_[0]);
+
+ writer.newStringObjectResult(templateObj, strId);
+ writer.returnFromIC();
+
+ trackAttached("StringConstructor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringToStringValueOf() {
+ // Expecting no arguments.
+ if (argc_ != 0) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure |this| is a primitive string value.
+ if (!thisval_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'toString' OR 'valueOf' native function.
+ emitNativeCalleeGuard();
+
+ // Guard |this| is a string.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ StringOperandId strId = writer.guardToString(thisValId);
+
+ // Return the string
+ writer.loadStringResult(strId);
+ writer.returnFromIC();
+
+ trackAttached("StringToStringValueOf");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringReplaceString() {
+ // Self-hosted code calls this with (string, string, string) arguments.
+ MOZ_ASSERT(argc_ == 3);
+ MOZ_ASSERT(args_[0].isString());
+ MOZ_ASSERT(args_[1].isString());
+ MOZ_ASSERT(args_[2].isString());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId strId = writer.guardToString(arg0Id);
+
+ ValOperandId arg1Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ StringOperandId patternId = writer.guardToString(arg1Id);
+
+ ValOperandId arg2Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ StringOperandId replacementId = writer.guardToString(arg2Id);
+
+ writer.stringReplaceStringResult(strId, patternId, replacementId);
+ writer.returnFromIC();
+
+ trackAttached("StringReplaceString");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringSplitString() {
+ // Self-hosted code calls this with (string, string) arguments.
+ MOZ_ASSERT(argc_ == 2);
+ MOZ_ASSERT(args_[0].isString());
+ MOZ_ASSERT(args_[1].isString());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId strId = writer.guardToString(arg0Id);
+
+ ValOperandId arg1Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ StringOperandId separatorId = writer.guardToString(arg1Id);
+
+ writer.stringSplitStringResult(strId, separatorId);
+ writer.returnFromIC();
+
+ trackAttached("StringSplitString");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringChar(
+ StringChar kind) {
+ // Need one argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ auto attach = CanAttachStringChar(thisval_, args_[0]);
+ if (attach == AttachStringChar::No) {
+ return AttachDecision::NoAction;
+ }
+
+ bool handleOOB = attach == AttachStringChar::OutOfBounds;
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'charCodeAt' or 'charAt' native function.
+ emitNativeCalleeGuard();
+
+ // Guard this is a string.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ StringOperandId strId = writer.guardToString(thisValId);
+
+ // Guard int32 index.
+ ValOperandId indexId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId int32IndexId = writer.guardToInt32Index(indexId);
+
+ // Linearize the string.
+ //
+ // AttachStringChar doesn't have a separate state when OOB access happens on
+ // a string which needs to be linearized, so just linearize unconditionally
+ // for out-of-bounds accesses.
+ if (attach == AttachStringChar::Linearize ||
+ attach == AttachStringChar::OutOfBounds) {
+ strId = writer.linearizeForCharAccess(strId, int32IndexId);
+ }
+
+ // Load string char or code.
+ if (kind == StringChar::CodeAt) {
+ writer.loadStringCharCodeResult(strId, int32IndexId, handleOOB);
+ } else {
+ writer.loadStringCharResult(strId, int32IndexId, handleOOB);
+ }
+
+ writer.returnFromIC();
+
+ if (kind == StringChar::CodeAt) {
+ trackAttached("StringCharCodeAt");
+ } else {
+ trackAttached("StringCharAt");
+ }
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringCharCodeAt() {
+ return tryAttachStringChar(StringChar::CodeAt);
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringCharAt() {
+ return tryAttachStringChar(StringChar::At);
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringFromCharCode() {
+ // Need one number argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'fromCharCode' native function.
+ emitNativeCalleeGuard();
+
+ // Guard int32 argument.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId codeId;
+ if (args_[0].isInt32()) {
+ codeId = writer.guardToInt32(argId);
+ } else {
+ // 'fromCharCode' performs ToUint16 on its input. We can use Uint32
+ // semantics, because ToUint16(ToUint32(v)) == ToUint16(v).
+ codeId = writer.guardToInt32ModUint32(argId);
+ }
+
+ // Return string created from code.
+ writer.stringFromCharCodeResult(codeId);
+ writer.returnFromIC();
+
+ trackAttached("StringFromCharCode");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringFromCodePoint() {
+ // Need one int32 argument.
+ if (argc_ != 1 || !args_[0].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ // String.fromCodePoint throws for invalid code points.
+ int32_t codePoint = args_[0].toInt32();
+ if (codePoint < 0 || codePoint > int32_t(unicode::NonBMPMax)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'fromCodePoint' native function.
+ emitNativeCalleeGuard();
+
+ // Guard int32 argument.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId codeId = writer.guardToInt32(argId);
+
+ // Return string created from code point.
+ writer.stringFromCodePointResult(codeId);
+ writer.returnFromIC();
+
+ trackAttached("StringFromCodePoint");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringIndexOf() {
+ // Need one string argument.
+ if (argc_ != 1 || !args_[0].isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure |this| is a primitive string value.
+ if (!thisval_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'indexOf' native function.
+ emitNativeCalleeGuard();
+
+ // Guard this is a string.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ StringOperandId strId = writer.guardToString(thisValId);
+
+ // Guard string argument.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId searchStrId = writer.guardToString(argId);
+
+ writer.stringIndexOfResult(strId, searchStrId);
+ writer.returnFromIC();
+
+ trackAttached("StringIndexOf");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringStartsWith() {
+ // Need one string argument.
+ if (argc_ != 1 || !args_[0].isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure |this| is a primitive string value.
+ if (!thisval_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'startsWith' native function.
+ emitNativeCalleeGuard();
+
+ // Guard this is a string.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ StringOperandId strId = writer.guardToString(thisValId);
+
+ // Guard string argument.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId searchStrId = writer.guardToString(argId);
+
+ writer.stringStartsWithResult(strId, searchStrId);
+ writer.returnFromIC();
+
+ trackAttached("StringStartsWith");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringEndsWith() {
+ // Need one string argument.
+ if (argc_ != 1 || !args_[0].isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure |this| is a primitive string value.
+ if (!thisval_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'endsWith' native function.
+ emitNativeCalleeGuard();
+
+ // Guard this is a string.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ StringOperandId strId = writer.guardToString(thisValId);
+
+ // Guard string argument.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId searchStrId = writer.guardToString(argId);
+
+ writer.stringEndsWithResult(strId, searchStrId);
+ writer.returnFromIC();
+
+ trackAttached("StringEndsWith");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringToLowerCase() {
+ // Expecting no arguments.
+ if (argc_ != 0) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure |this| is a primitive string value.
+ if (!thisval_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'toLowerCase' native function.
+ emitNativeCalleeGuard();
+
+ // Guard this is a string.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ StringOperandId strId = writer.guardToString(thisValId);
+
+ // Return string converted to lower-case.
+ writer.stringToLowerCaseResult(strId);
+ writer.returnFromIC();
+
+ trackAttached("StringToLowerCase");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStringToUpperCase() {
+ // Expecting no arguments.
+ if (argc_ != 0) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure |this| is a primitive string value.
+ if (!thisval_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'toUpperCase' native function.
+ emitNativeCalleeGuard();
+
+ // Guard this is a string.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ StringOperandId strId = writer.guardToString(thisValId);
+
+ // Return string converted to upper-case.
+ writer.stringToUpperCaseResult(strId);
+ writer.returnFromIC();
+
+ trackAttached("StringToUpperCase");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathRandom() {
+ // Expecting no arguments.
+ if (argc_ != 0) {
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(cx_->realm() == callee_->realm(),
+ "Shouldn't inline cross-realm Math.random because per-realm RNG");
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'random' native function.
+ emitNativeCalleeGuard();
+
+ mozilla::non_crypto::XorShift128PlusRNG* rng =
+ &cx_->realm()->getOrCreateRandomNumberGenerator();
+ writer.mathRandomResult(rng);
+
+ writer.returnFromIC();
+
+ trackAttached("MathRandom");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathAbs() {
+ // Need one argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'abs' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ // abs(INT_MIN) is a double.
+ if (args_[0].isInt32() && args_[0].toInt32() != INT_MIN) {
+ Int32OperandId int32Id = writer.guardToInt32(argumentId);
+ writer.mathAbsInt32Result(int32Id);
+ } else {
+ NumberOperandId numberId = writer.guardIsNumber(argumentId);
+ writer.mathAbsNumberResult(numberId);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("MathAbs");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathClz32() {
+ // Need one (number) argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'clz32' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ Int32OperandId int32Id;
+ if (args_[0].isInt32()) {
+ int32Id = writer.guardToInt32(argId);
+ } else {
+ MOZ_ASSERT(args_[0].isDouble());
+ NumberOperandId numId = writer.guardIsNumber(argId);
+ int32Id = writer.truncateDoubleToUInt32(numId);
+ }
+ writer.mathClz32Result(int32Id);
+ writer.returnFromIC();
+
+ trackAttached("MathClz32");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathSign() {
+ // Need one (number) argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'sign' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ if (args_[0].isInt32()) {
+ Int32OperandId int32Id = writer.guardToInt32(argId);
+ writer.mathSignInt32Result(int32Id);
+ } else {
+ // Math.sign returns a double only if the input is -0 or NaN so try to
+ // optimize the common Number => Int32 case.
+ double d = math_sign_impl(args_[0].toDouble());
+ int32_t unused;
+ bool resultIsInt32 = mozilla::NumberIsInt32(d, &unused);
+
+ NumberOperandId numId = writer.guardIsNumber(argId);
+ if (resultIsInt32) {
+ writer.mathSignNumberToInt32Result(numId);
+ } else {
+ writer.mathSignNumberResult(numId);
+ }
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("MathSign");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathImul() {
+ // Need two (number) arguments.
+ if (argc_ != 2 || !args_[0].isNumber() || !args_[1].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'imul' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ValOperandId arg1Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+
+ Int32OperandId int32Arg0Id, int32Arg1Id;
+ if (args_[0].isInt32() && args_[1].isInt32()) {
+ int32Arg0Id = writer.guardToInt32(arg0Id);
+ int32Arg1Id = writer.guardToInt32(arg1Id);
+ } else {
+ // Treat both arguments as numbers if at least one of them is non-int32.
+ NumberOperandId numArg0Id = writer.guardIsNumber(arg0Id);
+ NumberOperandId numArg1Id = writer.guardIsNumber(arg1Id);
+ int32Arg0Id = writer.truncateDoubleToUInt32(numArg0Id);
+ int32Arg1Id = writer.truncateDoubleToUInt32(numArg1Id);
+ }
+ writer.mathImulResult(int32Arg0Id, int32Arg1Id);
+ writer.returnFromIC();
+
+ trackAttached("MathImul");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathFloor() {
+ // Need one (number) argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check if the result fits in int32.
+ double res = math_floor_impl(args_[0].toNumber());
+ int32_t unused;
+ bool resultIsInt32 = mozilla::NumberIsInt32(res, &unused);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'floor' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ if (args_[0].isInt32()) {
+ MOZ_ASSERT(resultIsInt32);
+
+ // Use an indirect truncation to inform the optimizer it needs to preserve
+ // a bailout when the input can't be represented as an int32, even if the
+ // final result is fully truncated.
+ Int32OperandId intId = writer.guardToInt32(argumentId);
+ writer.indirectTruncateInt32Result(intId);
+ } else {
+ NumberOperandId numberId = writer.guardIsNumber(argumentId);
+
+ if (resultIsInt32) {
+ writer.mathFloorToInt32Result(numberId);
+ } else {
+ writer.mathFloorNumberResult(numberId);
+ }
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("MathFloor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathCeil() {
+ // Need one (number) argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check if the result fits in int32.
+ double res = math_ceil_impl(args_[0].toNumber());
+ int32_t unused;
+ bool resultIsInt32 = mozilla::NumberIsInt32(res, &unused);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'ceil' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ if (args_[0].isInt32()) {
+ MOZ_ASSERT(resultIsInt32);
+
+ // Use an indirect truncation to inform the optimizer it needs to preserve
+ // a bailout when the input can't be represented as an int32, even if the
+ // final result is fully truncated.
+ Int32OperandId intId = writer.guardToInt32(argumentId);
+ writer.indirectTruncateInt32Result(intId);
+ } else {
+ NumberOperandId numberId = writer.guardIsNumber(argumentId);
+
+ if (resultIsInt32) {
+ writer.mathCeilToInt32Result(numberId);
+ } else {
+ writer.mathCeilNumberResult(numberId);
+ }
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("MathCeil");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathTrunc() {
+ // Need one (number) argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check if the result fits in int32.
+ double res = math_trunc_impl(args_[0].toNumber());
+ int32_t unused;
+ bool resultIsInt32 = mozilla::NumberIsInt32(res, &unused);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'trunc' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ if (args_[0].isInt32()) {
+ MOZ_ASSERT(resultIsInt32);
+
+ // We don't need an indirect truncation barrier here, because Math.trunc
+ // always truncates, but never rounds its input away from zero.
+ Int32OperandId intId = writer.guardToInt32(argumentId);
+ writer.loadInt32Result(intId);
+ } else {
+ NumberOperandId numberId = writer.guardIsNumber(argumentId);
+
+ if (resultIsInt32) {
+ writer.mathTruncToInt32Result(numberId);
+ } else {
+ writer.mathTruncNumberResult(numberId);
+ }
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("MathTrunc");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathRound() {
+ // Need one (number) argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check if the result fits in int32.
+ double res = math_round_impl(args_[0].toNumber());
+ int32_t unused;
+ bool resultIsInt32 = mozilla::NumberIsInt32(res, &unused);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'round' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ if (args_[0].isInt32()) {
+ MOZ_ASSERT(resultIsInt32);
+
+ // Use an indirect truncation to inform the optimizer it needs to preserve
+ // a bailout when the input can't be represented as an int32, even if the
+ // final result is fully truncated.
+ Int32OperandId intId = writer.guardToInt32(argumentId);
+ writer.indirectTruncateInt32Result(intId);
+ } else {
+ NumberOperandId numberId = writer.guardIsNumber(argumentId);
+
+ if (resultIsInt32) {
+ writer.mathRoundToInt32Result(numberId);
+ } else {
+ writer.mathFunctionNumberResult(numberId, UnaryMathFunction::Round);
+ }
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("MathRound");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathSqrt() {
+ // Need one (number) argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'sqrt' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ NumberOperandId numberId = writer.guardIsNumber(argumentId);
+ writer.mathSqrtNumberResult(numberId);
+ writer.returnFromIC();
+
+ trackAttached("MathSqrt");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathFRound() {
+ // Need one (number) argument.
+ if (argc_ != 1 || !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'fround' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ NumberOperandId numberId = writer.guardIsNumber(argumentId);
+ writer.mathFRoundNumberResult(numberId);
+ writer.returnFromIC();
+
+ trackAttached("MathFRound");
+ return AttachDecision::Attach;
+}
+
+static bool CanAttachInt32Pow(const Value& baseVal, const Value& powerVal) {
+ auto valToInt32 = [](const Value& v) {
+ if (v.isInt32()) {
+ return v.toInt32();
+ }
+ if (v.isBoolean()) {
+ return int32_t(v.toBoolean());
+ }
+ MOZ_ASSERT(v.isNull());
+ return 0;
+ };
+ int32_t base = valToInt32(baseVal);
+ int32_t power = valToInt32(powerVal);
+
+ // x^y where y < 0 is most of the time not an int32, except when x is 1 or y
+ // gets large enough. It's hard to determine when exactly y is "large enough",
+ // so we don't use Int32PowResult when x != 1 and y < 0.
+ // Note: it's important for this condition to match the code generated by
+ // MacroAssembler::pow32 to prevent failure loops.
+ if (power < 0) {
+ return base == 1;
+ }
+
+ double res = powi(base, power);
+ int32_t unused;
+ return mozilla::NumberIsInt32(res, &unused);
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathPow() {
+ // Need two number arguments.
+ if (argc_ != 2 || !args_[0].isNumber() || !args_[1].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'pow' function.
+ emitNativeCalleeGuard();
+
+ ValOperandId baseId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ValOperandId exponentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+
+ if (args_[0].isInt32() && args_[1].isInt32() &&
+ CanAttachInt32Pow(args_[0], args_[1])) {
+ Int32OperandId baseInt32Id = writer.guardToInt32(baseId);
+ Int32OperandId exponentInt32Id = writer.guardToInt32(exponentId);
+ writer.int32PowResult(baseInt32Id, exponentInt32Id);
+ } else {
+ NumberOperandId baseNumberId = writer.guardIsNumber(baseId);
+ NumberOperandId exponentNumberId = writer.guardIsNumber(exponentId);
+ writer.doublePowResult(baseNumberId, exponentNumberId);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("MathPow");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathHypot() {
+ // Only optimize if there are 2-4 arguments.
+ if (argc_ < 2 || argc_ > 4) {
+ return AttachDecision::NoAction;
+ }
+
+ for (size_t i = 0; i < argc_; i++) {
+ if (!args_[i].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'hypot' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId firstId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ValOperandId secondId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+
+ NumberOperandId firstNumId = writer.guardIsNumber(firstId);
+ NumberOperandId secondNumId = writer.guardIsNumber(secondId);
+
+ ValOperandId thirdId;
+ ValOperandId fourthId;
+ NumberOperandId thirdNumId;
+ NumberOperandId fourthNumId;
+
+ switch (argc_) {
+ case 2:
+ writer.mathHypot2NumberResult(firstNumId, secondNumId);
+ break;
+ case 3:
+ thirdId = writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ thirdNumId = writer.guardIsNumber(thirdId);
+ writer.mathHypot3NumberResult(firstNumId, secondNumId, thirdNumId);
+ break;
+ case 4:
+ thirdId = writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ fourthId = writer.loadArgumentFixedSlot(ArgumentKind::Arg3, argc_);
+ thirdNumId = writer.guardIsNumber(thirdId);
+ fourthNumId = writer.guardIsNumber(fourthId);
+ writer.mathHypot4NumberResult(firstNumId, secondNumId, thirdNumId,
+ fourthNumId);
+ break;
+ default:
+ MOZ_CRASH("Unexpected number of arguments to hypot function.");
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("MathHypot");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathATan2() {
+ // Requires two numbers as arguments.
+ if (argc_ != 2 || !args_[0].isNumber() || !args_[1].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'atan2' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId yId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ValOperandId xId = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+
+ NumberOperandId yNumberId = writer.guardIsNumber(yId);
+ NumberOperandId xNumberId = writer.guardIsNumber(xId);
+
+ writer.mathAtan2NumberResult(yNumberId, xNumberId);
+ writer.returnFromIC();
+
+ trackAttached("MathAtan2");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathMinMax(bool isMax) {
+ // For now only optimize if there are 1-4 arguments.
+ if (argc_ < 1 || argc_ > 4) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure all arguments are numbers.
+ bool allInt32 = true;
+ for (size_t i = 0; i < argc_; i++) {
+ if (!args_[i].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+ if (!args_[i].isInt32()) {
+ allInt32 = false;
+ }
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is this Math function.
+ emitNativeCalleeGuard();
+
+ if (allInt32) {
+ ValOperandId valId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId resId = writer.guardToInt32(valId);
+ for (size_t i = 1; i < argc_; i++) {
+ ValOperandId argId =
+ writer.loadArgumentFixedSlot(ArgumentKindForArgIndex(i), argc_);
+ Int32OperandId argInt32Id = writer.guardToInt32(argId);
+ resId = writer.int32MinMax(isMax, resId, argInt32Id);
+ }
+ writer.loadInt32Result(resId);
+ } else {
+ ValOperandId valId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ NumberOperandId resId = writer.guardIsNumber(valId);
+ for (size_t i = 1; i < argc_; i++) {
+ ValOperandId argId =
+ writer.loadArgumentFixedSlot(ArgumentKindForArgIndex(i), argc_);
+ NumberOperandId argNumId = writer.guardIsNumber(argId);
+ resId = writer.numberMinMax(isMax, resId, argNumId);
+ }
+ writer.loadDoubleResult(resId);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached(isMax ? "MathMax" : "MathMin");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachSpreadMathMinMax(
+ bool isMax) {
+ MOZ_ASSERT(flags_.getArgFormat() == CallFlags::Spread ||
+ flags_.getArgFormat() == CallFlags::FunApplyArray);
+
+ // The result will be an int32 if there is at least one argument,
+ // and all the arguments are int32.
+ bool int32Result = args_.length() > 0;
+ for (size_t i = 0; i < args_.length(); i++) {
+ if (!args_[i].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+ if (!args_[i].isInt32()) {
+ int32Result = false;
+ }
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is this Math function.
+ emitNativeCalleeGuard();
+
+ // Load the argument array.
+ ObjOperandId argsId = emitLoadArgsArray();
+
+ if (int32Result) {
+ writer.int32MinMaxArrayResult(argsId, isMax);
+ } else {
+ writer.numberMinMaxArrayResult(argsId, isMax);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached(isMax ? "MathMax" : "MathMin");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMathFunction(
+ UnaryMathFunction fun) {
+ // Need one argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is this Math function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ NumberOperandId numberId = writer.guardIsNumber(argumentId);
+ writer.mathFunctionNumberResult(numberId, fun);
+ writer.returnFromIC();
+
+ trackAttached("MathFunction");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachNumber() {
+ // Expect a single string argument.
+ if (argc_ != 1 || !args_[0].isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ double num;
+ if (!StringToNumber(cx_, args_[0].toString(), &num)) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the `Number` function.
+ emitNativeCalleeGuard();
+
+ // Guard that the argument is a string.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ StringOperandId strId = writer.guardToString(argId);
+
+ // Return either an Int32 or Double result.
+ int32_t unused;
+ if (mozilla::NumberIsInt32(num, &unused)) {
+ Int32OperandId resultId = writer.guardStringToInt32(strId);
+ writer.loadInt32Result(resultId);
+ } else {
+ NumberOperandId resultId = writer.guardStringToNumber(strId);
+ writer.loadDoubleResult(resultId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("Number");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachNumberParseInt() {
+ // Expected arguments: input (string or number), optional radix (int32).
+ if (argc_ < 1 || argc_ > 2) {
+ return AttachDecision::NoAction;
+ }
+ if (!args_[0].isString() && !args_[0].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+ if (args_[0].isDouble()) {
+ double d = args_[0].toDouble();
+
+ // See num_parseInt for why we have to reject numbers smaller than 1.0e-6.
+ // Negative numbers in the exclusive range (-1, -0) return -0.
+ bool canTruncateToInt32 =
+ (DOUBLE_DECIMAL_IN_SHORTEST_LOW <= d && d <= double(INT32_MAX)) ||
+ (double(INT32_MIN) <= d && d <= -1.0) || (d == 0.0);
+ if (!canTruncateToInt32) {
+ return AttachDecision::NoAction;
+ }
+ }
+ if (argc_ > 1 && !args_[1].isInt32(10)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'parseInt' native function.
+ emitNativeCalleeGuard();
+
+ auto guardRadix = [&]() {
+ ValOperandId radixId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ Int32OperandId intRadixId = writer.guardToInt32(radixId);
+ writer.guardSpecificInt32(intRadixId, 10);
+ return intRadixId;
+ };
+
+ ValOperandId inputId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ if (args_[0].isString()) {
+ StringOperandId strId = writer.guardToString(inputId);
+
+ Int32OperandId intRadixId;
+ if (argc_ > 1) {
+ intRadixId = guardRadix();
+ } else {
+ intRadixId = writer.loadInt32Constant(0);
+ }
+
+ writer.numberParseIntResult(strId, intRadixId);
+ } else if (args_[0].isInt32()) {
+ Int32OperandId intId = writer.guardToInt32(inputId);
+ if (argc_ > 1) {
+ guardRadix();
+ }
+ writer.loadInt32Result(intId);
+ } else {
+ MOZ_ASSERT(args_[0].isDouble());
+
+ NumberOperandId numId = writer.guardIsNumber(inputId);
+ if (argc_ > 1) {
+ guardRadix();
+ }
+ writer.doubleParseIntResult(numId);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("NumberParseInt");
+ return AttachDecision::Attach;
+}
+
+StringOperandId IRGenerator::emitToStringGuard(ValOperandId id,
+ const Value& v) {
+ MOZ_ASSERT(CanConvertToString(v));
+ if (v.isString()) {
+ return writer.guardToString(id);
+ }
+ if (v.isBoolean()) {
+ BooleanOperandId boolId = writer.guardToBoolean(id);
+ return writer.booleanToString(boolId);
+ }
+ if (v.isNull()) {
+ writer.guardIsNull(id);
+ return writer.loadConstantString(cx_->names().null);
+ }
+ if (v.isUndefined()) {
+ writer.guardIsUndefined(id);
+ return writer.loadConstantString(cx_->names().undefined);
+ }
+ if (v.isInt32()) {
+ Int32OperandId intId = writer.guardToInt32(id);
+ return writer.callInt32ToString(intId);
+ }
+ // At this point we are creating an IC that will handle
+ // both Int32 and Double cases.
+ MOZ_ASSERT(v.isNumber());
+ NumberOperandId numId = writer.guardIsNumber(id);
+ return writer.callNumberToString(numId);
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachNumberToString() {
+ // Expecting no arguments or a single int32 argument.
+ if (argc_ > 1) {
+ return AttachDecision::NoAction;
+ }
+ if (argc_ == 1 && !args_[0].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure |this| is a primitive number value.
+ if (!thisval_.isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // No arguments means base 10.
+ int32_t base = 10;
+ if (argc_ > 0) {
+ base = args_[0].toInt32();
+ if (base < 2 || base > 36) {
+ return AttachDecision::NoAction;
+ }
+
+ // Non-decimal bases currently only support int32 inputs.
+ if (base != 10 && !thisval_.isInt32()) {
+ return AttachDecision::NoAction;
+ }
+ }
+ MOZ_ASSERT(2 <= base && base <= 36);
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'toString' native function.
+ emitNativeCalleeGuard();
+
+ // Initialize the |this| operand.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+
+ // Guard on number and convert to string.
+ if (base == 10) {
+ // If an explicit base was passed, guard its value.
+ if (argc_ > 0) {
+ // Guard the `base` argument is an int32.
+ ValOperandId baseId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId intBaseId = writer.guardToInt32(baseId);
+
+ // Guard `base` is 10 for decimal toString representation.
+ writer.guardSpecificInt32(intBaseId, 10);
+ }
+
+ StringOperandId strId = emitToStringGuard(thisValId, thisval_);
+
+ // Return the string.
+ writer.loadStringResult(strId);
+ } else {
+ MOZ_ASSERT(argc_ > 0);
+
+ // Guard the |this| value is an int32.
+ Int32OperandId thisIntId = writer.guardToInt32(thisValId);
+
+ // Guard the `base` argument is an int32.
+ ValOperandId baseId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId intBaseId = writer.guardToInt32(baseId);
+
+ // Return the string.
+ writer.int32ToStringWithBaseResult(thisIntId, intBaseId);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("NumberToString");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachReflectGetPrototypeOf() {
+ // Need one argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!args_[0].isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'getPrototypeOf' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argumentId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(argumentId);
+
+ writer.reflectGetPrototypeOfResult(objId);
+ writer.returnFromIC();
+
+ trackAttached("ReflectGetPrototypeOf");
+ return AttachDecision::Attach;
+}
+
+static bool AtomicsMeetsPreconditions(TypedArrayObject* typedArray,
+ const Value& index) {
+ switch (typedArray->type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ break;
+
+ case Scalar::Float32:
+ case Scalar::Float64:
+ case Scalar::Uint8Clamped:
+ // Exclude floating types and Uint8Clamped.
+ return false;
+
+ case Scalar::MaxTypedArrayViewType:
+ case Scalar::Int64:
+ case Scalar::Simd128:
+ MOZ_CRASH("Unsupported TypedArray type");
+ }
+
+ // Bounds check the index argument.
+ int64_t indexInt64;
+ if (!ValueIsInt64Index(index, &indexInt64)) {
+ return false;
+ }
+ if (indexInt64 < 0 || uint64_t(indexInt64) >= typedArray->length()) {
+ return false;
+ }
+
+ return true;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsCompareExchange() {
+ if (!JitSupportsAtomics()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Need four arguments.
+ if (argc_ != 4) {
+ return AttachDecision::NoAction;
+ }
+
+ // Arguments: typedArray, index (number), expected, replacement.
+ if (!args_[0].isObject() || !args_[0].toObject().is<TypedArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+ if (!args_[1].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ if (!AtomicsMeetsPreconditions(typedArray, args_[1])) {
+ return AttachDecision::NoAction;
+ }
+
+ Scalar::Type elementType = typedArray->type();
+ if (!ValueIsNumeric(elementType, args_[2])) {
+ return AttachDecision::NoAction;
+ }
+ if (!ValueIsNumeric(elementType, args_[3])) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the `compareExchange` native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(arg0Id);
+ writer.guardShapeForClass(objId, typedArray->shape());
+
+ // Convert index to intPtr.
+ ValOperandId indexId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ IntPtrOperandId intPtrIndexId =
+ guardToIntPtrIndex(args_[1], indexId, /* supportOOB = */ false);
+
+ // Convert expected value to int32/BigInt.
+ ValOperandId expectedId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ OperandId numericExpectedId = emitNumericGuard(expectedId, elementType);
+
+ // Convert replacement value to int32/BigInt.
+ ValOperandId replacementId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg3, argc_);
+ OperandId numericReplacementId = emitNumericGuard(replacementId, elementType);
+
+ writer.atomicsCompareExchangeResult(objId, intPtrIndexId, numericExpectedId,
+ numericReplacementId, typedArray->type());
+ writer.returnFromIC();
+
+ trackAttached("AtomicsCompareExchange");
+ return AttachDecision::Attach;
+}
+
+bool InlinableNativeIRGenerator::canAttachAtomicsReadWriteModify() {
+ if (!JitSupportsAtomics()) {
+ return false;
+ }
+
+ // Need three arguments.
+ if (argc_ != 3) {
+ return false;
+ }
+
+ // Arguments: typedArray, index (number), value.
+ if (!args_[0].isObject() || !args_[0].toObject().is<TypedArrayObject>()) {
+ return false;
+ }
+ if (!args_[1].isNumber()) {
+ return false;
+ }
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ if (!AtomicsMeetsPreconditions(typedArray, args_[1])) {
+ return false;
+ }
+ if (!ValueIsNumeric(typedArray->type(), args_[2])) {
+ return false;
+ }
+ return true;
+}
+
+InlinableNativeIRGenerator::AtomicsReadWriteModifyOperands
+InlinableNativeIRGenerator::emitAtomicsReadWriteModifyOperands() {
+ MOZ_ASSERT(canAttachAtomicsReadWriteModify());
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is this Atomics function.
+ emitNativeCalleeGuard();
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(arg0Id);
+ writer.guardShapeForClass(objId, typedArray->shape());
+
+ // Convert index to intPtr.
+ ValOperandId indexId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ IntPtrOperandId intPtrIndexId =
+ guardToIntPtrIndex(args_[1], indexId, /* supportOOB = */ false);
+
+ // Convert value to int32/BigInt.
+ ValOperandId valueId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ OperandId numericValueId = emitNumericGuard(valueId, typedArray->type());
+
+ return {objId, intPtrIndexId, numericValueId};
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsExchange() {
+ if (!canAttachAtomicsReadWriteModify()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto [objId, intPtrIndexId, numericValueId] =
+ emitAtomicsReadWriteModifyOperands();
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+
+ writer.atomicsExchangeResult(objId, intPtrIndexId, numericValueId,
+ typedArray->type());
+ writer.returnFromIC();
+
+ trackAttached("AtomicsExchange");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsAdd() {
+ if (!canAttachAtomicsReadWriteModify()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto [objId, intPtrIndexId, numericValueId] =
+ emitAtomicsReadWriteModifyOperands();
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ bool forEffect = ignoresResult();
+
+ writer.atomicsAddResult(objId, intPtrIndexId, numericValueId,
+ typedArray->type(), forEffect);
+ writer.returnFromIC();
+
+ trackAttached("AtomicsAdd");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsSub() {
+ if (!canAttachAtomicsReadWriteModify()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto [objId, intPtrIndexId, numericValueId] =
+ emitAtomicsReadWriteModifyOperands();
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ bool forEffect = ignoresResult();
+
+ writer.atomicsSubResult(objId, intPtrIndexId, numericValueId,
+ typedArray->type(), forEffect);
+ writer.returnFromIC();
+
+ trackAttached("AtomicsSub");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsAnd() {
+ if (!canAttachAtomicsReadWriteModify()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto [objId, intPtrIndexId, numericValueId] =
+ emitAtomicsReadWriteModifyOperands();
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ bool forEffect = ignoresResult();
+
+ writer.atomicsAndResult(objId, intPtrIndexId, numericValueId,
+ typedArray->type(), forEffect);
+ writer.returnFromIC();
+
+ trackAttached("AtomicsAnd");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsOr() {
+ if (!canAttachAtomicsReadWriteModify()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto [objId, intPtrIndexId, numericValueId] =
+ emitAtomicsReadWriteModifyOperands();
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ bool forEffect = ignoresResult();
+
+ writer.atomicsOrResult(objId, intPtrIndexId, numericValueId,
+ typedArray->type(), forEffect);
+ writer.returnFromIC();
+
+ trackAttached("AtomicsOr");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsXor() {
+ if (!canAttachAtomicsReadWriteModify()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto [objId, intPtrIndexId, numericValueId] =
+ emitAtomicsReadWriteModifyOperands();
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ bool forEffect = ignoresResult();
+
+ writer.atomicsXorResult(objId, intPtrIndexId, numericValueId,
+ typedArray->type(), forEffect);
+ writer.returnFromIC();
+
+ trackAttached("AtomicsXor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsLoad() {
+ if (!JitSupportsAtomics()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Need two arguments.
+ if (argc_ != 2) {
+ return AttachDecision::NoAction;
+ }
+
+ // Arguments: typedArray, index (number).
+ if (!args_[0].isObject() || !args_[0].toObject().is<TypedArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+ if (!args_[1].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ if (!AtomicsMeetsPreconditions(typedArray, args_[1])) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the `load` native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(arg0Id);
+ writer.guardShapeForClass(objId, typedArray->shape());
+
+ // Convert index to intPtr.
+ ValOperandId indexId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ IntPtrOperandId intPtrIndexId =
+ guardToIntPtrIndex(args_[1], indexId, /* supportOOB = */ false);
+
+ writer.atomicsLoadResult(objId, intPtrIndexId, typedArray->type());
+ writer.returnFromIC();
+
+ trackAttached("AtomicsLoad");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsStore() {
+ if (!JitSupportsAtomics()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Need three arguments.
+ if (argc_ != 3) {
+ return AttachDecision::NoAction;
+ }
+
+ // Atomics.store() is annoying because it returns the result of converting the
+ // value by ToInteger(), not the input value, nor the result of converting the
+ // value by ToInt32(). It is especially annoying because almost nobody uses
+ // the result value.
+ //
+ // As an expedient compromise, therefore, we inline only if the result is
+ // obviously unused or if the argument is already Int32 and thus requires no
+ // conversion.
+
+ // Arguments: typedArray, index (number), value.
+ if (!args_[0].isObject() || !args_[0].toObject().is<TypedArrayObject>()) {
+ return AttachDecision::NoAction;
+ }
+ if (!args_[1].isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ auto* typedArray = &args_[0].toObject().as<TypedArrayObject>();
+ if (!AtomicsMeetsPreconditions(typedArray, args_[1])) {
+ return AttachDecision::NoAction;
+ }
+
+ Scalar::Type elementType = typedArray->type();
+ if (!ValueIsNumeric(elementType, args_[2])) {
+ return AttachDecision::NoAction;
+ }
+
+ bool guardIsInt32 = !Scalar::isBigIntType(elementType) && !ignoresResult();
+
+ if (guardIsInt32 && !args_[2].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the `store` native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId arg0Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objId = writer.guardToObject(arg0Id);
+ writer.guardShapeForClass(objId, typedArray->shape());
+
+ // Convert index to intPtr.
+ ValOperandId indexId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ IntPtrOperandId intPtrIndexId =
+ guardToIntPtrIndex(args_[1], indexId, /* supportOOB = */ false);
+
+ // Ensure value is int32 or BigInt.
+ ValOperandId valueId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ OperandId numericValueId;
+ if (guardIsInt32) {
+ numericValueId = writer.guardToInt32(valueId);
+ } else {
+ numericValueId = emitNumericGuard(valueId, elementType);
+ }
+
+ writer.atomicsStoreResult(objId, intPtrIndexId, numericValueId,
+ typedArray->type());
+ writer.returnFromIC();
+
+ trackAttached("AtomicsStore");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAtomicsIsLockFree() {
+ // Need one argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!args_[0].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the `isLockFree` native function.
+ emitNativeCalleeGuard();
+
+ // Ensure value is int32.
+ ValOperandId valueId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId int32ValueId = writer.guardToInt32(valueId);
+
+ writer.atomicsIsLockFreeResult(int32ValueId);
+ writer.returnFromIC();
+
+ trackAttached("AtomicsIsLockFree");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachBoolean() {
+ // Need zero or one argument.
+ if (argc_ > 1) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'Boolean' native function.
+ emitNativeCalleeGuard();
+
+ if (argc_ == 0) {
+ writer.loadBooleanResult(false);
+ } else {
+ ValOperandId valId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ writer.loadValueTruthyResult(valId);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("Boolean");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachBailout() {
+ // Expecting no arguments.
+ if (argc_ != 0) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'bailout' native function.
+ emitNativeCalleeGuard();
+
+ writer.bailout();
+ writer.loadUndefinedResult();
+ writer.returnFromIC();
+
+ trackAttached("Bailout");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAssertFloat32() {
+ // Expecting two arguments.
+ if (argc_ != 2) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'assertFloat32' native function.
+ emitNativeCalleeGuard();
+
+ // TODO: Warp doesn't yet optimize Float32 (bug 1655773).
+
+ // NOP when not in IonMonkey.
+ writer.loadUndefinedResult();
+ writer.returnFromIC();
+
+ trackAttached("AssertFloat32");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachAssertRecoveredOnBailout() {
+ // Expecting two arguments.
+ if (argc_ != 2) {
+ return AttachDecision::NoAction;
+ }
+
+ // (Fuzzing unsafe) testing function which must be called with a constant
+ // boolean as its second argument.
+ bool mustBeRecovered = args_[1].toBoolean();
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'assertRecoveredOnBailout' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId valId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ writer.assertRecoveredOnBailoutResult(valId, mustBeRecovered);
+ writer.returnFromIC();
+
+ trackAttached("AssertRecoveredOnBailout");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachObjectIs() {
+ // Need two arguments.
+ if (argc_ != 2) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the `is` native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId lhsId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ValOperandId rhsId = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+
+ HandleValue lhs = args_[0];
+ HandleValue rhs = args_[1];
+
+ if (!isFirstStub()) {
+ writer.sameValueResult(lhsId, rhsId);
+ } else if (lhs.isNumber() && rhs.isNumber() &&
+ !(lhs.isInt32() && rhs.isInt32())) {
+ NumberOperandId lhsNumId = writer.guardIsNumber(lhsId);
+ NumberOperandId rhsNumId = writer.guardIsNumber(rhsId);
+ writer.compareDoubleSameValueResult(lhsNumId, rhsNumId);
+ } else if (!SameType(lhs, rhs)) {
+ // Compare tags for strictly different types.
+ ValueTagOperandId lhsTypeId = writer.loadValueTag(lhsId);
+ ValueTagOperandId rhsTypeId = writer.loadValueTag(rhsId);
+ writer.guardTagNotEqual(lhsTypeId, rhsTypeId);
+ writer.loadBooleanResult(false);
+ } else {
+ MOZ_ASSERT(lhs.type() == rhs.type());
+ MOZ_ASSERT(lhs.type() != JS::ValueType::Double);
+
+ switch (lhs.type()) {
+ case JS::ValueType::Int32: {
+ Int32OperandId lhsIntId = writer.guardToInt32(lhsId);
+ Int32OperandId rhsIntId = writer.guardToInt32(rhsId);
+ writer.compareInt32Result(JSOp::StrictEq, lhsIntId, rhsIntId);
+ break;
+ }
+ case JS::ValueType::Boolean: {
+ Int32OperandId lhsIntId = writer.guardBooleanToInt32(lhsId);
+ Int32OperandId rhsIntId = writer.guardBooleanToInt32(rhsId);
+ writer.compareInt32Result(JSOp::StrictEq, lhsIntId, rhsIntId);
+ break;
+ }
+ case JS::ValueType::Undefined: {
+ writer.guardIsUndefined(lhsId);
+ writer.guardIsUndefined(rhsId);
+ writer.loadBooleanResult(true);
+ break;
+ }
+ case JS::ValueType::Null: {
+ writer.guardIsNull(lhsId);
+ writer.guardIsNull(rhsId);
+ writer.loadBooleanResult(true);
+ break;
+ }
+ case JS::ValueType::String: {
+ StringOperandId lhsStrId = writer.guardToString(lhsId);
+ StringOperandId rhsStrId = writer.guardToString(rhsId);
+ writer.compareStringResult(JSOp::StrictEq, lhsStrId, rhsStrId);
+ break;
+ }
+ case JS::ValueType::Symbol: {
+ SymbolOperandId lhsSymId = writer.guardToSymbol(lhsId);
+ SymbolOperandId rhsSymId = writer.guardToSymbol(rhsId);
+ writer.compareSymbolResult(JSOp::StrictEq, lhsSymId, rhsSymId);
+ break;
+ }
+ case JS::ValueType::BigInt: {
+ BigIntOperandId lhsBigIntId = writer.guardToBigInt(lhsId);
+ BigIntOperandId rhsBigIntId = writer.guardToBigInt(rhsId);
+ writer.compareBigIntResult(JSOp::StrictEq, lhsBigIntId, rhsBigIntId);
+ break;
+ }
+ case JS::ValueType::Object: {
+ ObjOperandId lhsObjId = writer.guardToObject(lhsId);
+ ObjOperandId rhsObjId = writer.guardToObject(rhsId);
+ writer.compareObjectResult(JSOp::StrictEq, lhsObjId, rhsObjId);
+ break;
+ }
+
+#ifdef ENABLE_RECORD_TUPLE
+ case ValueType::ExtendedPrimitive:
+#endif
+ case JS::ValueType::Double:
+ case JS::ValueType::Magic:
+ case JS::ValueType::PrivateGCThing:
+ MOZ_CRASH("Unexpected type");
+ }
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("ObjectIs");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachObjectIsPrototypeOf() {
+ // Ensure |this| is an object.
+ if (!thisval_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Need a single argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the `isPrototypeOf` native function.
+ emitNativeCalleeGuard();
+
+ // Guard that |this| is an object.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId thisObjId = writer.guardToObject(thisValId);
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ writer.loadInstanceOfObjectResult(argId, thisObjId);
+ writer.returnFromIC();
+
+ trackAttached("ObjectIsPrototypeOf");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachObjectToString() {
+ // Expecting no arguments.
+ if (argc_ != 0) {
+ return AttachDecision::NoAction;
+ }
+
+ // Ensure |this| is an object.
+ if (!thisval_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Don't attach if the object has @@toStringTag or is a proxy.
+ if (!ObjectClassToString(cx_, &thisval_.toObject())) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'toString' native function.
+ emitNativeCalleeGuard();
+
+ // Guard that |this| is an object.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId thisObjId = writer.guardToObject(thisValId);
+
+ writer.objectToStringResult(thisObjId);
+ writer.returnFromIC();
+
+ trackAttached("ObjectToString");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachBigIntAsIntN() {
+ // Need two arguments (Int32, BigInt).
+ if (argc_ != 2 || !args_[0].isInt32() || !args_[1].isBigInt()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Negative bits throws an error.
+ if (args_[0].toInt32() < 0) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'BigInt.asIntN' native function.
+ emitNativeCalleeGuard();
+
+ // Convert bits to int32.
+ ValOperandId bitsId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId int32BitsId = writer.guardToInt32Index(bitsId);
+
+ // Number of bits mustn't be negative.
+ writer.guardInt32IsNonNegative(int32BitsId);
+
+ ValOperandId arg1Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ BigIntOperandId bigIntId = writer.guardToBigInt(arg1Id);
+
+ writer.bigIntAsIntNResult(int32BitsId, bigIntId);
+ writer.returnFromIC();
+
+ trackAttached("BigIntAsIntN");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachBigIntAsUintN() {
+ // Need two arguments (Int32, BigInt).
+ if (argc_ != 2 || !args_[0].isInt32() || !args_[1].isBigInt()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Negative bits throws an error.
+ if (args_[0].toInt32() < 0) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'BigInt.asUintN' native function.
+ emitNativeCalleeGuard();
+
+ // Convert bits to int32.
+ ValOperandId bitsId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ Int32OperandId int32BitsId = writer.guardToInt32Index(bitsId);
+
+ // Number of bits mustn't be negative.
+ writer.guardInt32IsNonNegative(int32BitsId);
+
+ ValOperandId arg1Id = writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ BigIntOperandId bigIntId = writer.guardToBigInt(arg1Id);
+
+ writer.bigIntAsUintNResult(int32BitsId, bigIntId);
+ writer.returnFromIC();
+
+ trackAttached("BigIntAsUintN");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachSetHas() {
+ // Ensure |this| is a SetObject.
+ if (!thisval_.isObject() || !thisval_.toObject().is<SetObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Need a single argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'has' native function.
+ emitNativeCalleeGuard();
+
+ // Guard |this| is a SetObject.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId objId = writer.guardToObject(thisValId);
+ writer.guardClass(objId, GuardClassKind::Set);
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+#ifndef JS_CODEGEN_X86
+ // Assume the hash key will likely always have the same type when attaching
+ // the first stub. If the call is polymorphic on the hash key, attach a stub
+ // which handles any value.
+ if (isFirstStub()) {
+ switch (args_[0].type()) {
+ case ValueType::Double:
+ case ValueType::Int32:
+ case ValueType::Boolean:
+ case ValueType::Undefined:
+ case ValueType::Null: {
+ writer.guardToNonGCThing(argId);
+ writer.setHasNonGCThingResult(objId, argId);
+ break;
+ }
+ case ValueType::String: {
+ StringOperandId strId = writer.guardToString(argId);
+ writer.setHasStringResult(objId, strId);
+ break;
+ }
+ case ValueType::Symbol: {
+ SymbolOperandId symId = writer.guardToSymbol(argId);
+ writer.setHasSymbolResult(objId, symId);
+ break;
+ }
+ case ValueType::BigInt: {
+ BigIntOperandId bigIntId = writer.guardToBigInt(argId);
+ writer.setHasBigIntResult(objId, bigIntId);
+ break;
+ }
+ case ValueType::Object: {
+ // Currently only supported on 64-bit platforms.
+# ifdef JS_PUNBOX64
+ ObjOperandId valId = writer.guardToObject(argId);
+ writer.setHasObjectResult(objId, valId);
+# else
+ writer.setHasResult(objId, argId);
+# endif
+ break;
+ }
+
+# ifdef ENABLE_RECORD_TUPLE
+ case ValueType::ExtendedPrimitive:
+# endif
+ case ValueType::Magic:
+ case ValueType::PrivateGCThing:
+ MOZ_CRASH("Unexpected type");
+ }
+ } else {
+ writer.setHasResult(objId, argId);
+ }
+#else
+ // The optimized versions require too many registers on x86.
+ writer.setHasResult(objId, argId);
+#endif
+
+ writer.returnFromIC();
+
+ trackAttached("SetHas");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMapHas() {
+ // Ensure |this| is a MapObject.
+ if (!thisval_.isObject() || !thisval_.toObject().is<MapObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Need a single argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'has' native function.
+ emitNativeCalleeGuard();
+
+ // Guard |this| is a MapObject.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId objId = writer.guardToObject(thisValId);
+ writer.guardClass(objId, GuardClassKind::Map);
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+#ifndef JS_CODEGEN_X86
+ // Assume the hash key will likely always have the same type when attaching
+ // the first stub. If the call is polymorphic on the hash key, attach a stub
+ // which handles any value.
+ if (isFirstStub()) {
+ switch (args_[0].type()) {
+ case ValueType::Double:
+ case ValueType::Int32:
+ case ValueType::Boolean:
+ case ValueType::Undefined:
+ case ValueType::Null: {
+ writer.guardToNonGCThing(argId);
+ writer.mapHasNonGCThingResult(objId, argId);
+ break;
+ }
+ case ValueType::String: {
+ StringOperandId strId = writer.guardToString(argId);
+ writer.mapHasStringResult(objId, strId);
+ break;
+ }
+ case ValueType::Symbol: {
+ SymbolOperandId symId = writer.guardToSymbol(argId);
+ writer.mapHasSymbolResult(objId, symId);
+ break;
+ }
+ case ValueType::BigInt: {
+ BigIntOperandId bigIntId = writer.guardToBigInt(argId);
+ writer.mapHasBigIntResult(objId, bigIntId);
+ break;
+ }
+ case ValueType::Object: {
+ // Currently only supported on 64-bit platforms.
+# ifdef JS_PUNBOX64
+ ObjOperandId valId = writer.guardToObject(argId);
+ writer.mapHasObjectResult(objId, valId);
+# else
+ writer.mapHasResult(objId, argId);
+# endif
+ break;
+ }
+
+# ifdef ENABLE_RECORD_TUPLE
+ case ValueType::ExtendedPrimitive:
+# endif
+ case ValueType::Magic:
+ case ValueType::PrivateGCThing:
+ MOZ_CRASH("Unexpected type");
+ }
+ } else {
+ writer.mapHasResult(objId, argId);
+ }
+#else
+ // The optimized versions require too many registers on x86.
+ writer.mapHasResult(objId, argId);
+#endif
+
+ writer.returnFromIC();
+
+ trackAttached("MapHas");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachMapGet() {
+ // Ensure |this| is a MapObject.
+ if (!thisval_.isObject() || !thisval_.toObject().is<MapObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Need a single argument.
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'get' native function.
+ emitNativeCalleeGuard();
+
+ // Guard |this| is a MapObject.
+ ValOperandId thisValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::This, argc_);
+ ObjOperandId objId = writer.guardToObject(thisValId);
+ writer.guardClass(objId, GuardClassKind::Map);
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+#ifndef JS_CODEGEN_X86
+ // Assume the hash key will likely always have the same type when attaching
+ // the first stub. If the call is polymorphic on the hash key, attach a stub
+ // which handles any value.
+ if (isFirstStub()) {
+ switch (args_[0].type()) {
+ case ValueType::Double:
+ case ValueType::Int32:
+ case ValueType::Boolean:
+ case ValueType::Undefined:
+ case ValueType::Null: {
+ writer.guardToNonGCThing(argId);
+ writer.mapGetNonGCThingResult(objId, argId);
+ break;
+ }
+ case ValueType::String: {
+ StringOperandId strId = writer.guardToString(argId);
+ writer.mapGetStringResult(objId, strId);
+ break;
+ }
+ case ValueType::Symbol: {
+ SymbolOperandId symId = writer.guardToSymbol(argId);
+ writer.mapGetSymbolResult(objId, symId);
+ break;
+ }
+ case ValueType::BigInt: {
+ BigIntOperandId bigIntId = writer.guardToBigInt(argId);
+ writer.mapGetBigIntResult(objId, bigIntId);
+ break;
+ }
+ case ValueType::Object: {
+ // Currently only supported on 64-bit platforms.
+# ifdef JS_PUNBOX64
+ ObjOperandId valId = writer.guardToObject(argId);
+ writer.mapGetObjectResult(objId, valId);
+# else
+ writer.mapGetResult(objId, argId);
+# endif
+ break;
+ }
+
+# ifdef ENABLE_RECORD_TUPLE
+ case ValueType::ExtendedPrimitive:
+# endif
+ case ValueType::Magic:
+ case ValueType::PrivateGCThing:
+ MOZ_CRASH("Unexpected type");
+ }
+ } else {
+ writer.mapGetResult(objId, argId);
+ }
+#else
+ // The optimized versions require too many registers on x86.
+ writer.mapGetResult(objId, argId);
+#endif
+
+ writer.returnFromIC();
+
+ trackAttached("MapGet");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CallIRGenerator::tryAttachFunCall(HandleFunction callee) {
+ MOZ_ASSERT(callee->isNativeWithoutJitEntry());
+
+ if (callee->native() != fun_call) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!thisval_.isObject() || !thisval_.toObject().is<JSFunction>()) {
+ return AttachDecision::NoAction;
+ }
+ RootedFunction target(cx_, &thisval_.toObject().as<JSFunction>());
+
+ bool isScripted = target->hasJitEntry();
+ MOZ_ASSERT_IF(!isScripted, target->isNativeWithoutJitEntry());
+
+ if (target->isClassConstructor()) {
+ return AttachDecision::NoAction;
+ }
+ Int32OperandId argcId(writer.setInputOperandId(0));
+
+ CallFlags targetFlags(CallFlags::FunCall);
+ if (mode_ == ICState::Mode::Specialized) {
+ if (cx_->realm() == target->realm()) {
+ targetFlags.setIsSameRealm();
+ }
+ }
+
+ if (mode_ == ICState::Mode::Specialized && !isScripted && argc_ > 0) {
+ // The stack layout is already in the correct form for calls with at least
+ // one argument.
+ //
+ // clang-format off
+ //
+ // *** STACK LAYOUT (bottom to top) *** *** INDEX ***
+ // Callee <-- argc+1
+ // ThisValue <-- argc
+ // Args: | Arg0 | <-- argc-1
+ // | Arg1 | <-- argc-2
+ // | ... | <-- ...
+ // | ArgN | <-- 0
+ //
+ // When passing |argc-1| as the number of arguments, we get:
+ //
+ // *** STACK LAYOUT (bottom to top) *** *** INDEX ***
+ // Callee <-- (argc-1)+1 = argc = ThisValue
+ // ThisValue <-- (argc-1) = argc-1 = Arg0
+ // Args: | Arg0 | <-- (argc-1)-1 = argc-2 = Arg1
+ // | Arg1 | <-- (argc-1)-2 = argc-3 = Arg2
+ // | ... | <-- ...
+ //
+ // clang-format on
+ //
+ // This allows to call |loadArgumentFixedSlot(ArgumentKind::Arg0)| and we
+ // still load the correct argument index from |ArgumentKind::Arg1|.
+ //
+ // When no arguments are passed, i.e. |argc==0|, we have to replace
+ // |ArgumentKind::Arg0| with the undefined value. But we don't yet support
+ // this case.
+ HandleValue newTarget = NullHandleValue;
+ HandleValue thisValue = args_[0];
+ HandleValueArray args =
+ HandleValueArray::subarray(args_, 1, args_.length() - 1);
+
+ // Check for specific native-function optimizations.
+ InlinableNativeIRGenerator nativeGen(*this, target, newTarget, thisValue,
+ args, targetFlags);
+ TRY_ATTACH(nativeGen.tryAttachStub());
+ }
+
+ ObjOperandId thisObjId = emitFunCallGuard(argcId);
+
+ if (mode_ == ICState::Mode::Specialized) {
+ // Ensure that |this| is the expected target function.
+ emitCalleeGuard(thisObjId, target);
+
+ if (isScripted) {
+ writer.callScriptedFunction(thisObjId, argcId, targetFlags,
+ ClampFixedArgc(argc_));
+ } else {
+ writer.callNativeFunction(thisObjId, argcId, op_, target, targetFlags,
+ ClampFixedArgc(argc_));
+ }
+ } else {
+ // Guard that |this| is a function.
+ writer.guardClass(thisObjId, GuardClassKind::JSFunction);
+
+ // Guard that function is not a class constructor.
+ writer.guardNotClassConstructor(thisObjId);
+
+ if (isScripted) {
+ writer.guardFunctionHasJitEntry(thisObjId, /*isConstructing =*/false);
+ writer.callScriptedFunction(thisObjId, argcId, targetFlags,
+ ClampFixedArgc(argc_));
+ } else {
+ writer.guardFunctionHasNoJitEntry(thisObjId);
+ writer.callAnyNativeFunction(thisObjId, argcId, targetFlags,
+ ClampFixedArgc(argc_));
+ }
+ }
+
+ writer.returnFromIC();
+
+ if (isScripted) {
+ trackAttached("Scripted fun_call");
+ } else {
+ trackAttached("Native fun_call");
+ }
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachIsTypedArray(
+ bool isPossiblyWrapped) {
+ // Self-hosted code calls this with a single object argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objArgId = writer.guardToObject(argId);
+ writer.isTypedArrayResult(objArgId, isPossiblyWrapped);
+ writer.returnFromIC();
+
+ trackAttached(isPossiblyWrapped ? "IsPossiblyWrappedTypedArray"
+ : "IsTypedArray");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachIsTypedArrayConstructor() {
+ // Self-hosted code calls this with a single object argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objArgId = writer.guardToObject(argId);
+ writer.isTypedArrayConstructorResult(objArgId);
+ writer.returnFromIC();
+
+ trackAttached("IsTypedArrayConstructor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachTypedArrayByteOffset() {
+ // Self-hosted code calls this with a single TypedArrayObject argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+ MOZ_ASSERT(args_[0].toObject().is<TypedArrayObject>());
+
+ auto* tarr = &args_[0].toObject().as<TypedArrayObject>();
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objArgId = writer.guardToObject(argId);
+ if (tarr->byteOffset() <= INT32_MAX) {
+ writer.arrayBufferViewByteOffsetInt32Result(objArgId);
+ } else {
+ writer.arrayBufferViewByteOffsetDoubleResult(objArgId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("TypedArrayByteOffset");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachTypedArrayElementSize() {
+ // Self-hosted code calls this with a single TypedArrayObject argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+ MOZ_ASSERT(args_[0].toObject().is<TypedArrayObject>());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objArgId = writer.guardToObject(argId);
+ writer.typedArrayElementSizeResult(objArgId);
+ writer.returnFromIC();
+
+ trackAttached("TypedArrayElementSize");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachTypedArrayLength(
+ bool isPossiblyWrapped) {
+ // Self-hosted code calls this with a single, possibly wrapped,
+ // TypedArrayObject argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ // Only optimize when the object isn't a wrapper.
+ if (isPossiblyWrapped && IsWrapper(&args_[0].toObject())) {
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(args_[0].toObject().is<TypedArrayObject>());
+
+ auto* tarr = &args_[0].toObject().as<TypedArrayObject>();
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objArgId = writer.guardToObject(argId);
+
+ if (isPossiblyWrapped) {
+ writer.guardIsNotProxy(objArgId);
+ }
+
+ if (tarr->length() <= INT32_MAX) {
+ writer.loadArrayBufferViewLengthInt32Result(objArgId);
+ } else {
+ writer.loadArrayBufferViewLengthDoubleResult(objArgId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("TypedArrayLength");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachArrayBufferByteLength(
+ bool isPossiblyWrapped) {
+ // Self-hosted code calls this with a single, possibly wrapped,
+ // ArrayBufferObject argument.
+ MOZ_ASSERT(argc_ == 1);
+ MOZ_ASSERT(args_[0].isObject());
+
+ // Only optimize when the object isn't a wrapper.
+ if (isPossiblyWrapped && IsWrapper(&args_[0].toObject())) {
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(args_[0].toObject().is<ArrayBufferObject>());
+
+ auto* buffer = &args_[0].toObject().as<ArrayBufferObject>();
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objArgId = writer.guardToObject(argId);
+
+ if (isPossiblyWrapped) {
+ writer.guardIsNotProxy(objArgId);
+ }
+
+ if (buffer->byteLength() <= INT32_MAX) {
+ writer.loadArrayBufferByteLengthInt32Result(objArgId);
+ } else {
+ writer.loadArrayBufferByteLengthDoubleResult(objArgId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("ArrayBufferByteLength");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachIsConstructing() {
+ // Self-hosted code calls this with no arguments in function scripts.
+ MOZ_ASSERT(argc_ == 0);
+ MOZ_ASSERT(script()->isFunction());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ writer.frameIsConstructingResult();
+ writer.returnFromIC();
+
+ trackAttached("IsConstructing");
+ return AttachDecision::Attach;
+}
+
+AttachDecision
+InlinableNativeIRGenerator::tryAttachGetNextMapSetEntryForIterator(bool isMap) {
+ // Self-hosted code calls this with two objects.
+ MOZ_ASSERT(argc_ == 2);
+ if (isMap) {
+ MOZ_ASSERT(args_[0].toObject().is<MapIteratorObject>());
+ } else {
+ MOZ_ASSERT(args_[0].toObject().is<SetIteratorObject>());
+ }
+ MOZ_ASSERT(args_[1].toObject().is<ArrayObject>());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId iterId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objIterId = writer.guardToObject(iterId);
+
+ ValOperandId resultArrId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ ObjOperandId objResultArrId = writer.guardToObject(resultArrId);
+
+ writer.getNextMapSetEntryForIteratorResult(objIterId, objResultArrId, isMap);
+ writer.returnFromIC();
+
+ trackAttached("GetNextMapSetEntryForIterator");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachFinishBoundFunctionInit() {
+ // Self-hosted code calls this with (boundFunction, targetObj, argCount)
+ // arguments.
+ MOZ_ASSERT(argc_ == 3);
+ MOZ_ASSERT(args_[0].toObject().is<JSFunction>());
+ MOZ_ASSERT(args_[1].isObject());
+ MOZ_ASSERT(args_[2].isInt32());
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ValOperandId boundId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ ObjOperandId objBoundId = writer.guardToObject(boundId);
+
+ ValOperandId targetId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_);
+ ObjOperandId objTargetId = writer.guardToObject(targetId);
+
+ ValOperandId argCountId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_);
+ Int32OperandId int32ArgCountId = writer.guardToInt32(argCountId);
+
+ writer.finishBoundFunctionInitResult(objBoundId, objTargetId,
+ int32ArgCountId);
+ writer.returnFromIC();
+
+ trackAttached("FinishBoundFunctionInit");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachNewArrayIterator() {
+ // Self-hosted code calls this without any arguments
+ MOZ_ASSERT(argc_ == 0);
+
+ JSObject* templateObj = NewArrayIteratorTemplate(cx_);
+ if (!templateObj) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ writer.newArrayIteratorResult(templateObj);
+ writer.returnFromIC();
+
+ trackAttached("NewArrayIterator");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachNewStringIterator() {
+ // Self-hosted code calls this without any arguments
+ MOZ_ASSERT(argc_ == 0);
+
+ JSObject* templateObj = NewStringIteratorTemplate(cx_);
+ if (!templateObj) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ writer.newStringIteratorResult(templateObj);
+ writer.returnFromIC();
+
+ trackAttached("NewStringIterator");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachNewRegExpStringIterator() {
+ // Self-hosted code calls this without any arguments
+ MOZ_ASSERT(argc_ == 0);
+
+ JSObject* templateObj = NewRegExpStringIteratorTemplate(cx_);
+ if (!templateObj) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ writer.newRegExpStringIteratorResult(templateObj);
+ writer.returnFromIC();
+
+ trackAttached("NewRegExpStringIterator");
+ return AttachDecision::Attach;
+}
+
+AttachDecision
+InlinableNativeIRGenerator::tryAttachArrayIteratorPrototypeOptimizable() {
+ // Self-hosted code calls this without any arguments
+ MOZ_ASSERT(argc_ == 0);
+
+ if (!isFirstStub()) {
+ // Attach only once to prevent slowdowns for polymorphic calls.
+ return AttachDecision::NoAction;
+ }
+
+ NativeObject* arrayIteratorProto;
+ uint32_t slot;
+ JSFunction* nextFun;
+ if (!IsArrayIteratorPrototypeOptimizable(cx_, &arrayIteratorProto, &slot,
+ &nextFun)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Note: we don't need to call emitNativeCalleeGuard for intrinsics.
+
+ ObjOperandId protoId = writer.loadObject(arrayIteratorProto);
+ ObjOperandId nextId = writer.loadObject(nextFun);
+
+ writer.guardShape(protoId, arrayIteratorProto->shape());
+
+ // Ensure that proto[slot] == nextFun.
+ writer.guardDynamicSlotIsSpecificObject(protoId, nextId, slot);
+ writer.loadBooleanResult(true);
+ writer.returnFromIC();
+
+ trackAttached("ArrayIteratorPrototypeOptimizable");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachObjectCreate() {
+ // Need a single object-or-null argument.
+ if (argc_ != 1 || !args_[0].isObjectOrNull()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!isFirstStub()) {
+ // Attach only once to prevent slowdowns for polymorphic calls.
+ return AttachDecision::NoAction;
+ }
+
+ RootedObject proto(cx_, args_[0].toObjectOrNull());
+ JSObject* templateObj = ObjectCreateImpl(cx_, proto, TenuredObject);
+ if (!templateObj) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'create' native function.
+ emitNativeCalleeGuard();
+
+ // Guard on the proto argument.
+ ValOperandId argId = writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+ if (proto) {
+ ObjOperandId protoId = writer.guardToObject(argId);
+ writer.guardSpecificObject(protoId, proto);
+ } else {
+ writer.guardIsNull(argId);
+ }
+
+ writer.objectCreateResult(templateObj);
+ writer.returnFromIC();
+
+ trackAttached("ObjectCreate");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachObjectConstructor() {
+ // Expecting no arguments or a single object argument.
+ // TODO(Warp): Support all or more conversions to object.
+ if (argc_ > 1) {
+ return AttachDecision::NoAction;
+ }
+ if (argc_ == 1 && !args_[0].isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ PlainObject* templateObj = nullptr;
+ if (argc_ == 0) {
+ // Stub doesn't support metadata builder
+ if (cx_->realm()->hasAllocationMetadataBuilder()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Create a temporary object to act as the template object.
+ templateObj = NewPlainObjectWithAllocKind(cx_, NewObjectGCKind());
+ if (!templateObj) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee and newTarget (if constructing) are this Object constructor
+ // function.
+ emitNativeCalleeGuard();
+
+ if (argc_ == 0) {
+ // TODO: Support pre-tenuring.
+ gc::AllocSite* site = script()->zone()->unknownAllocSite();
+ MOZ_ASSERT(site);
+
+ uint32_t numFixedSlots = templateObj->numUsedFixedSlots();
+ uint32_t numDynamicSlots = templateObj->numDynamicSlots();
+ gc::AllocKind allocKind = templateObj->allocKindForTenure();
+ Shape* shape = templateObj->shape();
+
+ writer.guardNoAllocationMetadataBuilder(
+ cx_->realm()->addressOfMetadataBuilder());
+ writer.newPlainObjectResult(numFixedSlots, numDynamicSlots, allocKind,
+ shape, site);
+ } else {
+ // Use standard call flags when this is an inline Function.prototype.call(),
+ // because GetIndexOfArgument() doesn't yet support |CallFlags::FunCall|.
+ CallFlags flags = flags_;
+ if (flags.getArgFormat() == CallFlags::FunCall) {
+ flags = CallFlags(CallFlags::Standard);
+ }
+
+ // Guard that the argument is an object.
+ ValOperandId argId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_, flags);
+ ObjOperandId objId = writer.guardToObject(argId);
+
+ // Return the object.
+ writer.loadObjectResult(objId);
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("ObjectConstructor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachArrayConstructor() {
+ // Only optimize the |Array()| and |Array(n)| cases (with or without |new|)
+ // for now. Note that self-hosted code calls this without |new| via std_Array.
+ if (argc_ > 1) {
+ return AttachDecision::NoAction;
+ }
+ if (argc_ == 1 && !args_[0].isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ int32_t length = (argc_ == 1) ? args_[0].toInt32() : 0;
+ if (length < 0 || uint32_t(length) > ArrayObject::EagerAllocationMaxLength) {
+ return AttachDecision::NoAction;
+ }
+
+ // We allow inlining this function across realms so make sure the template
+ // object is allocated in that realm. See CanInlineNativeCrossRealm.
+ JSObject* templateObj;
+ {
+ AutoRealm ar(cx_, callee_);
+ templateObj = NewDenseFullyAllocatedArray(cx_, length, TenuredObject);
+ if (!templateObj) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee and newTarget (if constructing) are this Array constructor
+ // function.
+ emitNativeCalleeGuard();
+
+ Int32OperandId lengthId;
+ if (argc_ == 1) {
+ // Use standard call flags when this is an inline Function.prototype.call(),
+ // because GetIndexOfArgument() doesn't yet support |CallFlags::FunCall|.
+ CallFlags flags = flags_;
+ if (flags.getArgFormat() == CallFlags::FunCall) {
+ flags = CallFlags(CallFlags::Standard);
+ }
+
+ ValOperandId arg0Id =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_, flags);
+ lengthId = writer.guardToInt32(arg0Id);
+ } else {
+ MOZ_ASSERT(argc_ == 0);
+ lengthId = writer.loadInt32Constant(0);
+ }
+
+ writer.newArrayFromLengthResult(templateObj, lengthId);
+ writer.returnFromIC();
+
+ trackAttached("ArrayConstructor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision InlinableNativeIRGenerator::tryAttachTypedArrayConstructor() {
+ MOZ_ASSERT(flags_.isConstructing());
+
+ if (argc_ == 0 || argc_ > 3) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!isFirstStub()) {
+ // Attach only once to prevent slowdowns for polymorphic calls.
+ return AttachDecision::NoAction;
+ }
+
+ // The first argument must be int32 or a non-proxy object.
+ if (!args_[0].isInt32() && !args_[0].isObject()) {
+ return AttachDecision::NoAction;
+ }
+ if (args_[0].isObject() && args_[0].toObject().is<ProxyObject>()) {
+ return AttachDecision::NoAction;
+ }
+
+#ifdef JS_CODEGEN_X86
+ // Unfortunately NewTypedArrayFromArrayBufferResult needs more registers than
+ // we can easily support on 32-bit x86 for now.
+ if (args_[0].isObject() &&
+ args_[0].toObject().is<ArrayBufferObjectMaybeShared>()) {
+ return AttachDecision::NoAction;
+ }
+#endif
+
+ RootedObject templateObj(cx_);
+ if (!TypedArrayObject::GetTemplateObjectForNative(cx_, callee_->native(),
+ args_, &templateObj)) {
+ cx_->recoverFromOutOfMemory();
+ return AttachDecision::NoAction;
+ }
+
+ if (!templateObj) {
+ // This can happen for large length values.
+ MOZ_ASSERT(args_[0].isInt32());
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee and newTarget are this TypedArray constructor function.
+ emitNativeCalleeGuard();
+
+ ValOperandId arg0Id =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_, flags_);
+
+ if (args_[0].isInt32()) {
+ // From length.
+ Int32OperandId lengthId = writer.guardToInt32(arg0Id);
+ writer.newTypedArrayFromLengthResult(templateObj, lengthId);
+ } else {
+ JSObject* obj = &args_[0].toObject();
+ ObjOperandId objId = writer.guardToObject(arg0Id);
+
+ if (obj->is<ArrayBufferObjectMaybeShared>()) {
+ // From ArrayBuffer.
+ if (obj->is<ArrayBufferObject>()) {
+ writer.guardClass(objId, GuardClassKind::ArrayBuffer);
+ } else {
+ MOZ_ASSERT(obj->is<SharedArrayBufferObject>());
+ writer.guardClass(objId, GuardClassKind::SharedArrayBuffer);
+ }
+ ValOperandId byteOffsetId;
+ if (argc_ > 1) {
+ byteOffsetId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg1, argc_, flags_);
+ } else {
+ byteOffsetId = writer.loadUndefined();
+ }
+ ValOperandId lengthId;
+ if (argc_ > 2) {
+ lengthId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg2, argc_, flags_);
+ } else {
+ lengthId = writer.loadUndefined();
+ }
+ writer.newTypedArrayFromArrayBufferResult(templateObj, objId,
+ byteOffsetId, lengthId);
+ } else {
+ // From Array-like.
+ writer.guardIsNotArrayBufferMaybeShared(objId);
+ writer.guardIsNotProxy(objId);
+ writer.newTypedArrayFromArrayResult(templateObj, objId);
+ }
+ }
+
+ writer.returnFromIC();
+
+ trackAttached("TypedArrayConstructor");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CallIRGenerator::tryAttachFunApply(HandleFunction calleeFunc) {
+ MOZ_ASSERT(calleeFunc->isNativeWithoutJitEntry());
+
+ if (calleeFunc->native() != fun_apply) {
+ return AttachDecision::NoAction;
+ }
+
+ if (argc_ > 2) {
+ return AttachDecision::NoAction;
+ }
+
+ if (!thisval_.isObject() || !thisval_.toObject().is<JSFunction>()) {
+ return AttachDecision::NoAction;
+ }
+ Rooted<JSFunction*> target(cx_, &thisval_.toObject().as<JSFunction>());
+
+ bool isScripted = target->hasJitEntry();
+ MOZ_ASSERT_IF(!isScripted, target->isNativeWithoutJitEntry());
+
+ if (target->isClassConstructor()) {
+ return AttachDecision::NoAction;
+ }
+
+ CallFlags::ArgFormat format = CallFlags::Standard;
+ if (argc_ < 2) {
+ // |fun.apply()| and |fun.apply(thisValue)| are equivalent to |fun.call()|
+ // resp. |fun.call(thisValue)|.
+ format = CallFlags::FunCall;
+ } else if (args_[1].isObject() && args_[1].toObject().is<ArgumentsObject>()) {
+ auto* argsObj = &args_[1].toObject().as<ArgumentsObject>();
+ if (argsObj->hasOverriddenElement() || argsObj->anyArgIsForwarded() ||
+ argsObj->hasOverriddenLength() ||
+ argsObj->initialLength() > JIT_ARGS_LENGTH_MAX) {
+ return AttachDecision::NoAction;
+ }
+ format = CallFlags::FunApplyArgsObj;
+ } else if (args_[1].isObject() && args_[1].toObject().is<ArrayObject>() &&
+ args_[1].toObject().as<ArrayObject>().length() <=
+ JIT_ARGS_LENGTH_MAX &&
+ IsPackedArray(&args_[1].toObject())) {
+ format = CallFlags::FunApplyArray;
+ } else {
+ return AttachDecision::NoAction;
+ }
+
+ Int32OperandId argcId(writer.setInputOperandId(0));
+
+ CallFlags targetFlags(format);
+ if (mode_ == ICState::Mode::Specialized) {
+ if (cx_->realm() == target->realm()) {
+ targetFlags.setIsSameRealm();
+ }
+ }
+
+ if (mode_ == ICState::Mode::Specialized && !isScripted &&
+ format == CallFlags::FunApplyArray) {
+ HandleValue newTarget = NullHandleValue;
+ HandleValue thisValue = args_[0];
+ Rooted<ArrayObject*> aobj(cx_, &args_[1].toObject().as<ArrayObject>());
+ HandleValueArray args = HandleValueArray::fromMarkedLocation(
+ aobj->length(), aobj->getDenseElements());
+
+ // Check for specific native-function optimizations.
+ InlinableNativeIRGenerator nativeGen(*this, target, newTarget, thisValue,
+ args, targetFlags);
+ TRY_ATTACH(nativeGen.tryAttachStub());
+ }
+
+ // Don't inline when no arguments are passed, cf. |tryAttachFunCall()|.
+ if (mode_ == ICState::Mode::Specialized && !isScripted &&
+ format == CallFlags::FunCall && argc_ > 0) {
+ MOZ_ASSERT(argc_ == 1);
+
+ HandleValue newTarget = NullHandleValue;
+ HandleValue thisValue = args_[0];
+ HandleValueArray args = HandleValueArray::empty();
+
+ // Check for specific native-function optimizations.
+ InlinableNativeIRGenerator nativeGen(*this, target, newTarget, thisValue,
+ args, targetFlags);
+ TRY_ATTACH(nativeGen.tryAttachStub());
+ }
+
+ ObjOperandId thisObjId = emitFunApplyGuard(argcId);
+
+ uint32_t fixedArgc;
+ if (format == CallFlags::FunApplyArray ||
+ format == CallFlags::FunApplyArgsObj) {
+ emitFunApplyArgsGuard(format);
+
+ // We always use MaxUnrolledArgCopy here because the fixed argc is
+ // meaningless in a FunApply case.
+ fixedArgc = MaxUnrolledArgCopy;
+ } else {
+ MOZ_ASSERT(format == CallFlags::FunCall);
+
+ // Whereas for the FunCall case we need to use the actual fixed argc value.
+ fixedArgc = ClampFixedArgc(argc_);
+ }
+
+ if (mode_ == ICState::Mode::Specialized) {
+ // Ensure that |this| is the expected target function.
+ emitCalleeGuard(thisObjId, target);
+
+ if (isScripted) {
+ writer.callScriptedFunction(thisObjId, argcId, targetFlags, fixedArgc);
+ } else {
+ writer.callNativeFunction(thisObjId, argcId, op_, target, targetFlags,
+ fixedArgc);
+ }
+ } else {
+ // Guard that |this| is a function.
+ writer.guardClass(thisObjId, GuardClassKind::JSFunction);
+
+ // Guard that function is not a class constructor.
+ writer.guardNotClassConstructor(thisObjId);
+
+ if (isScripted) {
+ // Guard that function is scripted.
+ writer.guardFunctionHasJitEntry(thisObjId, /*constructing =*/false);
+ writer.callScriptedFunction(thisObjId, argcId, targetFlags, fixedArgc);
+ } else {
+ // Guard that function is native.
+ writer.guardFunctionHasNoJitEntry(thisObjId);
+ writer.callAnyNativeFunction(thisObjId, argcId, targetFlags, fixedArgc);
+ }
+ }
+
+ writer.returnFromIC();
+
+ if (isScripted) {
+ trackAttached("Scripted fun_apply");
+ } else {
+ trackAttached("Native fun_apply");
+ }
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision CallIRGenerator::tryAttachWasmCall(HandleFunction calleeFunc) {
+ // Try to optimize calls into Wasm code by emitting the CallWasmFunction
+ // CacheIR op. Baseline ICs currently treat this as a CallScriptedFunction op
+ // (calling Wasm's JitEntry stub) but Warp transpiles it to a more direct call
+ // into Wasm code.
+ //
+ // Note: some code refers to these optimized Wasm calls as "inlined" calls.
+
+ MOZ_ASSERT(calleeFunc->isWasmWithJitEntry());
+
+ if (!JitOptions.enableWasmIonFastCalls) {
+ return AttachDecision::NoAction;
+ }
+ if (!isFirstStub_) {
+ return AttachDecision::NoAction;
+ }
+ JSOp op = JSOp(*pc_);
+ if (op != JSOp::Call && op != JSOp::CallContent &&
+ op != JSOp::CallIgnoresRv) {
+ return AttachDecision::NoAction;
+ }
+ if (cx_->realm() != calleeFunc->realm()) {
+ return AttachDecision::NoAction;
+ }
+
+ wasm::Instance& inst = wasm::ExportedFunctionToInstance(calleeFunc);
+ uint32_t funcIndex = inst.code().getFuncIndex(calleeFunc);
+
+ auto bestTier = inst.code().bestTier();
+ const wasm::FuncExport& funcExport =
+ inst.metadata(bestTier).lookupFuncExport(funcIndex);
+ const wasm::FuncType& sig = inst.metadata().getFuncExportType(funcExport);
+
+ MOZ_ASSERT(!IsInsideNursery(inst.object()));
+ MOZ_ASSERT(sig.canHaveJitEntry(), "Function should allow a Wasm JitEntry");
+
+ // If there are too many arguments, don't optimize (we won't be able to store
+ // the arguments in the LIR node).
+ static_assert(wasm::MaxArgsForJitInlineCall <= ArgumentKindArgIndexLimit);
+ if (sig.args().length() > wasm::MaxArgsForJitInlineCall ||
+ argc_ > ArgumentKindArgIndexLimit) {
+ return AttachDecision::NoAction;
+ }
+
+ // If there are too many results, don't optimize as Warp currently doesn't
+ // have code to handle this.
+ if (sig.results().length() > wasm::MaxResultsForJitInlineCall) {
+ return AttachDecision::NoAction;
+ }
+
+ // Bug 1631656 - Don't try to optimize with I64 args on 32-bit platforms
+ // because it is more difficult (because it requires multiple LIR arguments
+ // per I64).
+ //
+ // Bug 1631650 - On 64-bit platforms, we also give up optimizing for I64 args
+ // spilled to the stack because it causes problems with register allocation.
+#ifdef JS_64BIT
+ constexpr bool optimizeWithI64 = true;
+#else
+ constexpr bool optimizeWithI64 = false;
+#endif
+ ABIArgGenerator abi;
+ for (const auto& valType : sig.args()) {
+ MIRType mirType = valType.toMIRType();
+ ABIArg abiArg = abi.next(mirType);
+ if (mirType != MIRType::Int64) {
+ continue;
+ }
+ if (!optimizeWithI64 || abiArg.kind() == ABIArg::Stack) {
+ return AttachDecision::NoAction;
+ }
+ }
+
+ // Check that all arguments can be converted to the Wasm type in Warp code
+ // without bailing out.
+ for (size_t i = 0; i < sig.args().length(); i++) {
+ Value argVal = i < argc_ ? args_[i] : UndefinedValue();
+ switch (sig.args()[i].kind()) {
+ case wasm::ValType::I32:
+ case wasm::ValType::F32:
+ case wasm::ValType::F64:
+ if (!argVal.isNumber() && !argVal.isBoolean() &&
+ !argVal.isUndefined()) {
+ return AttachDecision::NoAction;
+ }
+ break;
+ case wasm::ValType::I64:
+ if (!argVal.isBigInt() && !argVal.isBoolean() && !argVal.isString()) {
+ return AttachDecision::NoAction;
+ }
+ break;
+ case wasm::ValType::V128:
+ MOZ_CRASH("Function should not have a Wasm JitEntry");
+ case wasm::ValType::Ref:
+ // canHaveJitEntry restricts args to externref, where all JS values are
+ // valid and can be boxed.
+ MOZ_ASSERT(sig.args()[i].refType().isExtern(),
+ "Unexpected type for Wasm JitEntry");
+ break;
+ }
+ }
+
+ CallFlags flags(/* isConstructing = */ false, /* isSpread = */ false,
+ /* isSameRealm = */ true);
+
+ // Load argc.
+ Int32OperandId argcId(writer.setInputOperandId(0));
+
+ // Load the callee and ensure it is an object
+ ValOperandId calleeValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Callee, argc_, flags);
+ ObjOperandId calleeObjId = writer.guardToObject(calleeValId);
+
+ // Ensure the callee is this Wasm function.
+ emitCalleeGuard(calleeObjId, calleeFunc);
+
+ // Guard the argument types.
+ uint32_t guardedArgs = std::min<uint32_t>(sig.args().length(), argc_);
+ for (uint32_t i = 0; i < guardedArgs; i++) {
+ ArgumentKind argKind = ArgumentKindForArgIndex(i);
+ ValOperandId argId = writer.loadArgumentFixedSlot(argKind, argc_, flags);
+ writer.guardWasmArg(argId, sig.args()[i].kind());
+ }
+
+ writer.callWasmFunction(calleeObjId, argcId, flags, ClampFixedArgc(argc_),
+ &funcExport, inst.object());
+ writer.returnFromIC();
+
+ trackAttached("WasmCall");
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision CallIRGenerator::tryAttachInlinableNative(HandleFunction callee,
+ CallFlags flags) {
+ MOZ_ASSERT(mode_ == ICState::Mode::Specialized);
+ MOZ_ASSERT(callee->isNativeWithoutJitEntry());
+ MOZ_ASSERT(flags.getArgFormat() == CallFlags::Standard ||
+ flags.getArgFormat() == CallFlags::Spread);
+
+ // Special case functions are only optimized for normal calls.
+ if (op_ != JSOp::Call && op_ != JSOp::CallContent && op_ != JSOp::New &&
+ op_ != JSOp::NewContent && op_ != JSOp::CallIgnoresRv &&
+ op_ != JSOp::SpreadCall) {
+ return AttachDecision::NoAction;
+ }
+
+ InlinableNativeIRGenerator nativeGen(*this, callee, newTarget_, thisval_,
+ args_, flags);
+ return nativeGen.tryAttachStub();
+}
+
+#ifdef FUZZING_JS_FUZZILLI
+AttachDecision InlinableNativeIRGenerator::tryAttachFuzzilliHash() {
+ if (argc_ != 1) {
+ return AttachDecision::NoAction;
+ }
+
+ // Initialize the input operand.
+ initializeInputOperand();
+
+ // Guard callee is the 'fuzzilli_hash' native function.
+ emitNativeCalleeGuard();
+
+ ValOperandId argValId =
+ writer.loadArgumentFixedSlot(ArgumentKind::Arg0, argc_);
+
+ writer.fuzzilliHashResult(argValId);
+ writer.returnFromIC();
+
+ trackAttached("FuzzilliHash");
+ return AttachDecision::Attach;
+}
+#endif
+
+AttachDecision InlinableNativeIRGenerator::tryAttachStub() {
+ if (!callee_->hasJitInfo() ||
+ callee_->jitInfo()->type() != JSJitInfo::InlinableNative) {
+ return AttachDecision::NoAction;
+ }
+
+ InlinableNative native = callee_->jitInfo()->inlinableNative;
+
+ // Not all natives can be inlined cross-realm.
+ if (cx_->realm() != callee_->realm() && !CanInlineNativeCrossRealm(native)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check for special-cased native constructors.
+ if (flags_.isConstructing()) {
+ MOZ_ASSERT(flags_.getArgFormat() == CallFlags::Standard);
+
+ // newTarget must match the callee. CacheIR for this is emitted in
+ // emitNativeCalleeGuard.
+ if (ObjectValue(*callee_) != newTarget_) {
+ return AttachDecision::NoAction;
+ }
+ switch (native) {
+ case InlinableNative::Array:
+ return tryAttachArrayConstructor();
+ case InlinableNative::TypedArrayConstructor:
+ return tryAttachTypedArrayConstructor();
+ case InlinableNative::String:
+ return tryAttachStringConstructor();
+ case InlinableNative::Object:
+ return tryAttachObjectConstructor();
+ default:
+ break;
+ }
+ return AttachDecision::NoAction;
+ }
+
+ // Check for special-cased native spread calls.
+ if (flags_.getArgFormat() == CallFlags::Spread ||
+ flags_.getArgFormat() == CallFlags::FunApplyArray) {
+ switch (native) {
+ case InlinableNative::MathMin:
+ return tryAttachSpreadMathMinMax(/*isMax = */ false);
+ case InlinableNative::MathMax:
+ return tryAttachSpreadMathMinMax(/*isMax = */ true);
+ default:
+ break;
+ }
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(flags_.getArgFormat() == CallFlags::Standard ||
+ flags_.getArgFormat() == CallFlags::FunCall);
+
+ // Check for special-cased native functions.
+ switch (native) {
+ // Array natives.
+ case InlinableNative::Array:
+ return tryAttachArrayConstructor();
+ case InlinableNative::ArrayPush:
+ return tryAttachArrayPush();
+ case InlinableNative::ArrayPop:
+ case InlinableNative::ArrayShift:
+ return tryAttachArrayPopShift(native);
+ case InlinableNative::ArrayJoin:
+ return tryAttachArrayJoin();
+ case InlinableNative::ArraySlice:
+ return tryAttachArraySlice();
+ case InlinableNative::ArrayIsArray:
+ return tryAttachArrayIsArray();
+
+ // DataView natives.
+ case InlinableNative::DataViewGetInt8:
+ return tryAttachDataViewGet(Scalar::Int8);
+ case InlinableNative::DataViewGetUint8:
+ return tryAttachDataViewGet(Scalar::Uint8);
+ case InlinableNative::DataViewGetInt16:
+ return tryAttachDataViewGet(Scalar::Int16);
+ case InlinableNative::DataViewGetUint16:
+ return tryAttachDataViewGet(Scalar::Uint16);
+ case InlinableNative::DataViewGetInt32:
+ return tryAttachDataViewGet(Scalar::Int32);
+ case InlinableNative::DataViewGetUint32:
+ return tryAttachDataViewGet(Scalar::Uint32);
+ case InlinableNative::DataViewGetFloat32:
+ return tryAttachDataViewGet(Scalar::Float32);
+ case InlinableNative::DataViewGetFloat64:
+ return tryAttachDataViewGet(Scalar::Float64);
+ case InlinableNative::DataViewGetBigInt64:
+ return tryAttachDataViewGet(Scalar::BigInt64);
+ case InlinableNative::DataViewGetBigUint64:
+ return tryAttachDataViewGet(Scalar::BigUint64);
+ case InlinableNative::DataViewSetInt8:
+ return tryAttachDataViewSet(Scalar::Int8);
+ case InlinableNative::DataViewSetUint8:
+ return tryAttachDataViewSet(Scalar::Uint8);
+ case InlinableNative::DataViewSetInt16:
+ return tryAttachDataViewSet(Scalar::Int16);
+ case InlinableNative::DataViewSetUint16:
+ return tryAttachDataViewSet(Scalar::Uint16);
+ case InlinableNative::DataViewSetInt32:
+ return tryAttachDataViewSet(Scalar::Int32);
+ case InlinableNative::DataViewSetUint32:
+ return tryAttachDataViewSet(Scalar::Uint32);
+ case InlinableNative::DataViewSetFloat32:
+ return tryAttachDataViewSet(Scalar::Float32);
+ case InlinableNative::DataViewSetFloat64:
+ return tryAttachDataViewSet(Scalar::Float64);
+ case InlinableNative::DataViewSetBigInt64:
+ return tryAttachDataViewSet(Scalar::BigInt64);
+ case InlinableNative::DataViewSetBigUint64:
+ return tryAttachDataViewSet(Scalar::BigUint64);
+
+ // Intl natives.
+ case InlinableNative::IntlGuardToCollator:
+ case InlinableNative::IntlGuardToDateTimeFormat:
+ case InlinableNative::IntlGuardToDisplayNames:
+ case InlinableNative::IntlGuardToListFormat:
+ case InlinableNative::IntlGuardToNumberFormat:
+ case InlinableNative::IntlGuardToPluralRules:
+ case InlinableNative::IntlGuardToRelativeTimeFormat:
+ return tryAttachGuardToClass(native);
+
+ // Slot intrinsics.
+ case InlinableNative::IntrinsicUnsafeGetReservedSlot:
+ case InlinableNative::IntrinsicUnsafeGetObjectFromReservedSlot:
+ case InlinableNative::IntrinsicUnsafeGetInt32FromReservedSlot:
+ case InlinableNative::IntrinsicUnsafeGetStringFromReservedSlot:
+ return tryAttachUnsafeGetReservedSlot(native);
+ case InlinableNative::IntrinsicUnsafeSetReservedSlot:
+ return tryAttachUnsafeSetReservedSlot();
+
+ // Intrinsics.
+ case InlinableNative::IntrinsicIsSuspendedGenerator:
+ return tryAttachIsSuspendedGenerator();
+ case InlinableNative::IntrinsicToObject:
+ return tryAttachToObject();
+ case InlinableNative::IntrinsicToInteger:
+ return tryAttachToInteger();
+ case InlinableNative::IntrinsicToLength:
+ return tryAttachToLength();
+ case InlinableNative::IntrinsicIsObject:
+ return tryAttachIsObject();
+ case InlinableNative::IntrinsicIsPackedArray:
+ return tryAttachIsPackedArray();
+ case InlinableNative::IntrinsicIsCallable:
+ return tryAttachIsCallable();
+ case InlinableNative::IntrinsicIsConstructor:
+ return tryAttachIsConstructor();
+ case InlinableNative::IntrinsicIsCrossRealmArrayConstructor:
+ return tryAttachIsCrossRealmArrayConstructor();
+ case InlinableNative::IntrinsicGuardToArrayIterator:
+ case InlinableNative::IntrinsicGuardToMapIterator:
+ case InlinableNative::IntrinsicGuardToSetIterator:
+ case InlinableNative::IntrinsicGuardToStringIterator:
+ case InlinableNative::IntrinsicGuardToRegExpStringIterator:
+ case InlinableNative::IntrinsicGuardToWrapForValidIterator:
+ case InlinableNative::IntrinsicGuardToIteratorHelper:
+ case InlinableNative::IntrinsicGuardToAsyncIteratorHelper:
+ return tryAttachGuardToClass(native);
+ case InlinableNative::IntrinsicSubstringKernel:
+ return tryAttachSubstringKernel();
+ case InlinableNative::IntrinsicIsConstructing:
+ return tryAttachIsConstructing();
+ case InlinableNative::IntrinsicFinishBoundFunctionInit:
+ return tryAttachFinishBoundFunctionInit();
+ case InlinableNative::IntrinsicNewArrayIterator:
+ return tryAttachNewArrayIterator();
+ case InlinableNative::IntrinsicNewStringIterator:
+ return tryAttachNewStringIterator();
+ case InlinableNative::IntrinsicNewRegExpStringIterator:
+ return tryAttachNewRegExpStringIterator();
+ case InlinableNative::IntrinsicArrayIteratorPrototypeOptimizable:
+ return tryAttachArrayIteratorPrototypeOptimizable();
+ case InlinableNative::IntrinsicObjectHasPrototype:
+ return tryAttachObjectHasPrototype();
+
+ // RegExp natives.
+ case InlinableNative::IsRegExpObject:
+ return tryAttachHasClass(&RegExpObject::class_,
+ /* isPossiblyWrapped = */ false);
+ case InlinableNative::IsPossiblyWrappedRegExpObject:
+ return tryAttachHasClass(&RegExpObject::class_,
+ /* isPossiblyWrapped = */ true);
+ case InlinableNative::RegExpMatcher:
+ case InlinableNative::RegExpSearcher:
+ case InlinableNative::RegExpTester:
+ return tryAttachRegExpMatcherSearcherTester(native);
+ case InlinableNative::RegExpPrototypeOptimizable:
+ return tryAttachRegExpPrototypeOptimizable();
+ case InlinableNative::RegExpInstanceOptimizable:
+ return tryAttachRegExpInstanceOptimizable();
+ case InlinableNative::GetFirstDollarIndex:
+ return tryAttachGetFirstDollarIndex();
+
+ // String natives.
+ case InlinableNative::String:
+ return tryAttachString();
+ case InlinableNative::StringToString:
+ case InlinableNative::StringValueOf:
+ return tryAttachStringToStringValueOf();
+ case InlinableNative::StringCharCodeAt:
+ return tryAttachStringCharCodeAt();
+ case InlinableNative::StringCharAt:
+ return tryAttachStringCharAt();
+ case InlinableNative::StringFromCharCode:
+ return tryAttachStringFromCharCode();
+ case InlinableNative::StringFromCodePoint:
+ return tryAttachStringFromCodePoint();
+ case InlinableNative::StringIndexOf:
+ return tryAttachStringIndexOf();
+ case InlinableNative::StringStartsWith:
+ return tryAttachStringStartsWith();
+ case InlinableNative::StringEndsWith:
+ return tryAttachStringEndsWith();
+ case InlinableNative::StringToLowerCase:
+ return tryAttachStringToLowerCase();
+ case InlinableNative::StringToUpperCase:
+ return tryAttachStringToUpperCase();
+ case InlinableNative::IntrinsicStringReplaceString:
+ return tryAttachStringReplaceString();
+ case InlinableNative::IntrinsicStringSplitString:
+ return tryAttachStringSplitString();
+
+ // Math natives.
+ case InlinableNative::MathRandom:
+ return tryAttachMathRandom();
+ case InlinableNative::MathAbs:
+ return tryAttachMathAbs();
+ case InlinableNative::MathClz32:
+ return tryAttachMathClz32();
+ case InlinableNative::MathSign:
+ return tryAttachMathSign();
+ case InlinableNative::MathImul:
+ return tryAttachMathImul();
+ case InlinableNative::MathFloor:
+ return tryAttachMathFloor();
+ case InlinableNative::MathCeil:
+ return tryAttachMathCeil();
+ case InlinableNative::MathTrunc:
+ return tryAttachMathTrunc();
+ case InlinableNative::MathRound:
+ return tryAttachMathRound();
+ case InlinableNative::MathSqrt:
+ return tryAttachMathSqrt();
+ case InlinableNative::MathFRound:
+ return tryAttachMathFRound();
+ case InlinableNative::MathHypot:
+ return tryAttachMathHypot();
+ case InlinableNative::MathATan2:
+ return tryAttachMathATan2();
+ case InlinableNative::MathSin:
+ return tryAttachMathFunction(UnaryMathFunction::Sin);
+ case InlinableNative::MathTan:
+ return tryAttachMathFunction(UnaryMathFunction::Tan);
+ case InlinableNative::MathCos:
+ return tryAttachMathFunction(UnaryMathFunction::Cos);
+ case InlinableNative::MathExp:
+ return tryAttachMathFunction(UnaryMathFunction::Exp);
+ case InlinableNative::MathLog:
+ return tryAttachMathFunction(UnaryMathFunction::Log);
+ case InlinableNative::MathASin:
+ return tryAttachMathFunction(UnaryMathFunction::ASin);
+ case InlinableNative::MathATan:
+ return tryAttachMathFunction(UnaryMathFunction::ATan);
+ case InlinableNative::MathACos:
+ return tryAttachMathFunction(UnaryMathFunction::ACos);
+ case InlinableNative::MathLog10:
+ return tryAttachMathFunction(UnaryMathFunction::Log10);
+ case InlinableNative::MathLog2:
+ return tryAttachMathFunction(UnaryMathFunction::Log2);
+ case InlinableNative::MathLog1P:
+ return tryAttachMathFunction(UnaryMathFunction::Log1P);
+ case InlinableNative::MathExpM1:
+ return tryAttachMathFunction(UnaryMathFunction::ExpM1);
+ case InlinableNative::MathCosH:
+ return tryAttachMathFunction(UnaryMathFunction::CosH);
+ case InlinableNative::MathSinH:
+ return tryAttachMathFunction(UnaryMathFunction::SinH);
+ case InlinableNative::MathTanH:
+ return tryAttachMathFunction(UnaryMathFunction::TanH);
+ case InlinableNative::MathACosH:
+ return tryAttachMathFunction(UnaryMathFunction::ACosH);
+ case InlinableNative::MathASinH:
+ return tryAttachMathFunction(UnaryMathFunction::ASinH);
+ case InlinableNative::MathATanH:
+ return tryAttachMathFunction(UnaryMathFunction::ATanH);
+ case InlinableNative::MathCbrt:
+ return tryAttachMathFunction(UnaryMathFunction::Cbrt);
+ case InlinableNative::MathPow:
+ return tryAttachMathPow();
+ case InlinableNative::MathMin:
+ return tryAttachMathMinMax(/* isMax = */ false);
+ case InlinableNative::MathMax:
+ return tryAttachMathMinMax(/* isMax = */ true);
+
+ // Map intrinsics.
+ case InlinableNative::IntrinsicGuardToMapObject:
+ return tryAttachGuardToClass(native);
+ case InlinableNative::IntrinsicGetNextMapEntryForIterator:
+ return tryAttachGetNextMapSetEntryForIterator(/* isMap = */ true);
+
+ // Number natives.
+ case InlinableNative::Number:
+ return tryAttachNumber();
+ case InlinableNative::NumberParseInt:
+ return tryAttachNumberParseInt();
+ case InlinableNative::NumberToString:
+ return tryAttachNumberToString();
+
+ // Object natives.
+ case InlinableNative::Object:
+ return tryAttachObjectConstructor();
+ case InlinableNative::ObjectCreate:
+ return tryAttachObjectCreate();
+ case InlinableNative::ObjectIs:
+ return tryAttachObjectIs();
+ case InlinableNative::ObjectIsPrototypeOf:
+ return tryAttachObjectIsPrototypeOf();
+ case InlinableNative::ObjectToString:
+ return tryAttachObjectToString();
+
+ // Set intrinsics.
+ case InlinableNative::IntrinsicGuardToSetObject:
+ return tryAttachGuardToClass(native);
+ case InlinableNative::IntrinsicGetNextSetEntryForIterator:
+ return tryAttachGetNextMapSetEntryForIterator(/* isMap = */ false);
+
+ // ArrayBuffer intrinsics.
+ case InlinableNative::IntrinsicGuardToArrayBuffer:
+ return tryAttachGuardToClass(native);
+ case InlinableNative::IntrinsicArrayBufferByteLength:
+ return tryAttachArrayBufferByteLength(/* isPossiblyWrapped = */ false);
+ case InlinableNative::IntrinsicPossiblyWrappedArrayBufferByteLength:
+ return tryAttachArrayBufferByteLength(/* isPossiblyWrapped = */ true);
+
+ // SharedArrayBuffer intrinsics.
+ case InlinableNative::IntrinsicGuardToSharedArrayBuffer:
+ return tryAttachGuardToClass(native);
+
+ // TypedArray intrinsics.
+ case InlinableNative::TypedArrayConstructor:
+ return AttachDecision::NoAction; // Not callable.
+ case InlinableNative::IntrinsicIsTypedArray:
+ return tryAttachIsTypedArray(/* isPossiblyWrapped = */ false);
+ case InlinableNative::IntrinsicIsPossiblyWrappedTypedArray:
+ return tryAttachIsTypedArray(/* isPossiblyWrapped = */ true);
+ case InlinableNative::IntrinsicIsTypedArrayConstructor:
+ return tryAttachIsTypedArrayConstructor();
+ case InlinableNative::IntrinsicTypedArrayByteOffset:
+ return tryAttachTypedArrayByteOffset();
+ case InlinableNative::IntrinsicTypedArrayElementSize:
+ return tryAttachTypedArrayElementSize();
+ case InlinableNative::IntrinsicTypedArrayLength:
+ return tryAttachTypedArrayLength(/* isPossiblyWrapped = */ false);
+ case InlinableNative::IntrinsicPossiblyWrappedTypedArrayLength:
+ return tryAttachTypedArrayLength(/* isPossiblyWrapped = */ true);
+
+ // Reflect natives.
+ case InlinableNative::ReflectGetPrototypeOf:
+ return tryAttachReflectGetPrototypeOf();
+
+ // Atomics intrinsics:
+ case InlinableNative::AtomicsCompareExchange:
+ return tryAttachAtomicsCompareExchange();
+ case InlinableNative::AtomicsExchange:
+ return tryAttachAtomicsExchange();
+ case InlinableNative::AtomicsAdd:
+ return tryAttachAtomicsAdd();
+ case InlinableNative::AtomicsSub:
+ return tryAttachAtomicsSub();
+ case InlinableNative::AtomicsAnd:
+ return tryAttachAtomicsAnd();
+ case InlinableNative::AtomicsOr:
+ return tryAttachAtomicsOr();
+ case InlinableNative::AtomicsXor:
+ return tryAttachAtomicsXor();
+ case InlinableNative::AtomicsLoad:
+ return tryAttachAtomicsLoad();
+ case InlinableNative::AtomicsStore:
+ return tryAttachAtomicsStore();
+ case InlinableNative::AtomicsIsLockFree:
+ return tryAttachAtomicsIsLockFree();
+
+ // BigInt natives.
+ case InlinableNative::BigIntAsIntN:
+ return tryAttachBigIntAsIntN();
+ case InlinableNative::BigIntAsUintN:
+ return tryAttachBigIntAsUintN();
+
+ // Boolean natives.
+ case InlinableNative::Boolean:
+ return tryAttachBoolean();
+
+ // Set natives.
+ case InlinableNative::SetHas:
+ return tryAttachSetHas();
+
+ // Map natives.
+ case InlinableNative::MapHas:
+ return tryAttachMapHas();
+ case InlinableNative::MapGet:
+ return tryAttachMapGet();
+
+ // Testing functions.
+ case InlinableNative::TestBailout:
+ if (js::SupportDifferentialTesting()) {
+ return AttachDecision::NoAction;
+ }
+ return tryAttachBailout();
+ case InlinableNative::TestAssertFloat32:
+ return tryAttachAssertFloat32();
+ case InlinableNative::TestAssertRecoveredOnBailout:
+ if (js::SupportDifferentialTesting()) {
+ return AttachDecision::NoAction;
+ }
+ return tryAttachAssertRecoveredOnBailout();
+
+#ifdef FUZZING_JS_FUZZILLI
+ // Fuzzilli function
+ case InlinableNative::FuzzilliHash:
+ return tryAttachFuzzilliHash();
+#endif
+
+ case InlinableNative::Limit:
+ break;
+ }
+
+ MOZ_CRASH("Shouldn't get here");
+}
+
+// Remember the shape of the this object for any script being called as a
+// constructor, for later use during Ion compilation.
+ScriptedThisResult CallIRGenerator::getThisShapeForScripted(
+ HandleFunction calleeFunc, MutableHandle<Shape*> result) {
+ // Some constructors allocate their own |this| object.
+ if (calleeFunc->constructorNeedsUninitializedThis()) {
+ return ScriptedThisResult::UninitializedThis;
+ }
+
+ // Only attach a stub if the newTarget is a function with a
+ // nonconfigurable prototype.
+ RootedObject newTarget(cx_, &newTarget_.toObject());
+ if (!newTarget->is<JSFunction>() ||
+ !newTarget->as<JSFunction>().hasNonConfigurablePrototypeDataProperty()) {
+ return ScriptedThisResult::NoAction;
+ }
+
+ AutoRealm ar(cx_, calleeFunc);
+ Shape* thisShape = ThisShapeForFunction(cx_, calleeFunc, newTarget);
+ if (!thisShape) {
+ cx_->clearPendingException();
+ return ScriptedThisResult::NoAction;
+ }
+
+ MOZ_ASSERT(thisShape->realm() == calleeFunc->realm());
+ result.set(thisShape);
+ return ScriptedThisResult::PlainObjectShape;
+}
+
+AttachDecision CallIRGenerator::tryAttachCallScripted(
+ HandleFunction calleeFunc) {
+ MOZ_ASSERT(calleeFunc->hasJitEntry());
+
+ if (calleeFunc->isWasmWithJitEntry()) {
+ TRY_ATTACH(tryAttachWasmCall(calleeFunc));
+ }
+
+ bool isSpecialized = mode_ == ICState::Mode::Specialized;
+
+ bool isConstructing = IsConstructPC(pc_);
+ bool isSpread = IsSpreadPC(pc_);
+ bool isSameRealm = isSpecialized && cx_->realm() == calleeFunc->realm();
+ CallFlags flags(isConstructing, isSpread, isSameRealm);
+
+ // If callee is not an interpreted constructor, we have to throw.
+ if (isConstructing && !calleeFunc->isConstructor()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Likewise, if the callee is a class constructor, we have to throw.
+ if (!isConstructing && calleeFunc->isClassConstructor()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (isConstructing && !calleeFunc->hasJitScript()) {
+ // If we're constructing, require the callee to have a JitScript. This isn't
+ // required for correctness but avoids allocating a template object below
+ // for constructors that aren't hot. See bug 1419758.
+ return AttachDecision::TemporarilyUnoptimizable;
+ }
+
+ // Verify that spread calls have a reasonable number of arguments.
+ if (isSpread && args_.length() > JIT_ARGS_LENGTH_MAX) {
+ return AttachDecision::NoAction;
+ }
+
+ Rooted<Shape*> thisShape(cx_);
+ if (isConstructing && isSpecialized) {
+ switch (getThisShapeForScripted(calleeFunc, &thisShape)) {
+ case ScriptedThisResult::PlainObjectShape:
+ break;
+ case ScriptedThisResult::UninitializedThis:
+ flags.setNeedsUninitializedThis();
+ break;
+ case ScriptedThisResult::NoAction:
+ return AttachDecision::NoAction;
+ }
+ }
+
+ // Load argc.
+ Int32OperandId argcId(writer.setInputOperandId(0));
+
+ // Load the callee and ensure it is an object
+ ValOperandId calleeValId =
+ writer.loadArgumentDynamicSlot(ArgumentKind::Callee, argcId, flags);
+ ObjOperandId calleeObjId = writer.guardToObject(calleeValId);
+
+ if (isSpecialized) {
+ MOZ_ASSERT_IF(isConstructing, thisShape || flags.needsUninitializedThis());
+
+ // Ensure callee matches this stub's callee
+ emitCalleeGuard(calleeObjId, calleeFunc);
+ if (thisShape) {
+ // Emit guards to ensure the newTarget's .prototype property is what we
+ // expect. Note that getThisForScripted checked newTarget is a function
+ // with a non-configurable .prototype data property.
+ JSFunction* newTarget = &newTarget_.toObject().as<JSFunction>();
+ Maybe<PropertyInfo> prop = newTarget->lookupPure(cx_->names().prototype);
+ MOZ_ASSERT(prop.isSome());
+ uint32_t slot = prop->slot();
+ MOZ_ASSERT(slot >= newTarget->numFixedSlots(),
+ "Stub code relies on this");
+
+ ValOperandId newTargetValId = writer.loadArgumentDynamicSlot(
+ ArgumentKind::NewTarget, argcId, flags);
+ ObjOperandId newTargetObjId = writer.guardToObject(newTargetValId);
+ writer.guardShape(newTargetObjId, newTarget->shape());
+
+ const Value& value = newTarget->getSlot(slot);
+ if (value.isObject()) {
+ JSObject* prototypeObject = &value.toObject();
+
+ ObjOperandId protoId = writer.loadObject(prototypeObject);
+ writer.guardDynamicSlotIsSpecificObject(
+ newTargetObjId, protoId, slot - newTarget->numFixedSlots());
+ } else {
+ writer.guardDynamicSlotIsNotObject(newTargetObjId,
+ slot - newTarget->numFixedSlots());
+ }
+
+ // Call metaScriptedThisShape before emitting the call, so that Warp can
+ // use the shape to create the |this| object before transpiling the call.
+ writer.metaScriptedThisShape(thisShape);
+ }
+ } else {
+ // Guard that object is a scripted function
+ writer.guardClass(calleeObjId, GuardClassKind::JSFunction);
+ writer.guardFunctionHasJitEntry(calleeObjId, isConstructing);
+
+ if (isConstructing) {
+ // If callee is not a constructor, we have to throw.
+ writer.guardFunctionIsConstructor(calleeObjId);
+ } else {
+ // If callee is a class constructor, we have to throw.
+ writer.guardNotClassConstructor(calleeObjId);
+ }
+ }
+
+ writer.callScriptedFunction(calleeObjId, argcId, flags,
+ ClampFixedArgc(argc_));
+ writer.returnFromIC();
+
+ if (isSpecialized) {
+ trackAttached("Call scripted func");
+ } else {
+ trackAttached("Call any scripted func");
+ }
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision CallIRGenerator::tryAttachCallNative(HandleFunction calleeFunc) {
+ MOZ_ASSERT(calleeFunc->isNativeWithoutJitEntry());
+
+ bool isSpecialized = mode_ == ICState::Mode::Specialized;
+
+ bool isSpread = IsSpreadPC(pc_);
+ bool isSameRealm = isSpecialized && cx_->realm() == calleeFunc->realm();
+ bool isConstructing = IsConstructPC(pc_);
+ CallFlags flags(isConstructing, isSpread, isSameRealm);
+
+ if (isConstructing && !calleeFunc->isConstructor()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Verify that spread calls have a reasonable number of arguments.
+ if (isSpread && args_.length() > JIT_ARGS_LENGTH_MAX) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check for specific native-function optimizations.
+ if (isSpecialized) {
+ TRY_ATTACH(tryAttachInlinableNative(calleeFunc, flags));
+ }
+
+ // Load argc.
+ Int32OperandId argcId(writer.setInputOperandId(0));
+
+ // Load the callee and ensure it is an object
+ ValOperandId calleeValId =
+ writer.loadArgumentDynamicSlot(ArgumentKind::Callee, argcId, flags);
+ ObjOperandId calleeObjId = writer.guardToObject(calleeValId);
+
+ // DOM calls need an additional guard so only try optimizing the first stub.
+ // Can only optimize normal (non-spread) calls.
+ if (isFirstStub_ && !isSpread && thisval_.isObject() &&
+ CanAttachDOMCall(cx_, JSJitInfo::Method, &thisval_.toObject(), calleeFunc,
+ mode_)) {
+ MOZ_ASSERT(!isConstructing, "DOM functions are not constructors");
+
+ // Guard that |this| is an object.
+ ValOperandId thisValId =
+ writer.loadArgumentDynamicSlot(ArgumentKind::This, argcId, flags);
+ ObjOperandId thisObjId = writer.guardToObject(thisValId);
+
+ // Guard on the |this| class to make sure it's the right instance.
+ writer.guardAnyClass(thisObjId, thisval_.toObject().getClass());
+
+ // Ensure callee matches this stub's callee
+ writer.guardSpecificFunction(calleeObjId, calleeFunc);
+ writer.callDOMFunction(calleeObjId, argcId, thisObjId, calleeFunc, flags,
+ ClampFixedArgc(argc_));
+
+ trackAttached("CallDOM");
+ } else if (isSpecialized) {
+ // Ensure callee matches this stub's callee
+ writer.guardSpecificFunction(calleeObjId, calleeFunc);
+ writer.callNativeFunction(calleeObjId, argcId, op_, calleeFunc, flags,
+ ClampFixedArgc(argc_));
+
+ trackAttached("CallNative");
+ } else {
+ // Guard that object is a native function
+ writer.guardClass(calleeObjId, GuardClassKind::JSFunction);
+ writer.guardFunctionHasNoJitEntry(calleeObjId);
+
+ if (isConstructing) {
+ // If callee is not a constructor, we have to throw.
+ writer.guardFunctionIsConstructor(calleeObjId);
+ } else {
+ // If callee is a class constructor, we have to throw.
+ writer.guardNotClassConstructor(calleeObjId);
+ }
+ writer.callAnyNativeFunction(calleeObjId, argcId, flags,
+ ClampFixedArgc(argc_));
+
+ trackAttached("CallAnyNative");
+ }
+
+ writer.returnFromIC();
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision CallIRGenerator::tryAttachCallHook(HandleObject calleeObj) {
+ if (mode_ != ICState::Mode::Specialized) {
+ // We do not have megamorphic call hook stubs.
+ // TODO: Should we attach specialized call hook stubs in
+ // megamorphic mode to avoid going generic?
+ return AttachDecision::NoAction;
+ }
+
+ bool isSpread = IsSpreadPC(pc_);
+ bool isConstructing = IsConstructPC(pc_);
+ CallFlags flags(isConstructing, isSpread);
+ JSNative hook =
+ isConstructing ? calleeObj->constructHook() : calleeObj->callHook();
+ if (!hook) {
+ return AttachDecision::NoAction;
+ }
+
+ // Verify that spread calls have a reasonable number of arguments.
+ if (isSpread && args_.length() > JIT_ARGS_LENGTH_MAX) {
+ return AttachDecision::NoAction;
+ }
+
+ // Load argc.
+ Int32OperandId argcId(writer.setInputOperandId(0));
+
+ // Load the callee and ensure it is an object
+ ValOperandId calleeValId =
+ writer.loadArgumentDynamicSlot(ArgumentKind::Callee, argcId, flags);
+ ObjOperandId calleeObjId = writer.guardToObject(calleeValId);
+
+ // Ensure the callee's class matches the one in this stub.
+ writer.guardAnyClass(calleeObjId, calleeObj->getClass());
+
+ writer.callClassHook(calleeObjId, argcId, hook, flags, ClampFixedArgc(argc_));
+ writer.returnFromIC();
+
+ trackAttached("Call native func");
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision CallIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+
+ // Some opcodes are not yet supported.
+ switch (op_) {
+ case JSOp::Call:
+ case JSOp::CallContent:
+ case JSOp::CallIgnoresRv:
+ case JSOp::CallIter:
+ case JSOp::CallContentIter:
+ case JSOp::SpreadCall:
+ case JSOp::New:
+ case JSOp::NewContent:
+ case JSOp::SpreadNew:
+ case JSOp::SuperCall:
+ case JSOp::SpreadSuperCall:
+ break;
+ default:
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(mode_ != ICState::Mode::Generic);
+
+ // Ensure callee is a function.
+ if (!callee_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ RootedObject calleeObj(cx_, &callee_.toObject());
+ if (!calleeObj->is<JSFunction>()) {
+ return tryAttachCallHook(calleeObj);
+ }
+
+ HandleFunction calleeFunc = calleeObj.as<JSFunction>();
+
+ // Check for scripted optimizations.
+ if (calleeFunc->hasJitEntry()) {
+ return tryAttachCallScripted(calleeFunc);
+ }
+
+ // Check for native-function optimizations.
+ MOZ_ASSERT(calleeFunc->isNativeWithoutJitEntry());
+
+ // Try inlining Function.prototype.{call,apply}. We don't use the
+ // InlinableNative mechanism for this because we want to optimize these more
+ // aggressively than other natives.
+ if (op_ == JSOp::Call || op_ == JSOp::CallContent ||
+ op_ == JSOp::CallIgnoresRv) {
+ TRY_ATTACH(tryAttachFunCall(calleeFunc));
+ TRY_ATTACH(tryAttachFunApply(calleeFunc));
+ }
+
+ return tryAttachCallNative(calleeFunc);
+}
+
+void CallIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("callee", callee_);
+ sp.valueProperty("thisval", thisval_);
+ sp.valueProperty("argc", Int32Value(argc_));
+
+ // Try to log the first two arguments.
+ if (args_.length() >= 1) {
+ sp.valueProperty("arg0", args_[0]);
+ }
+ if (args_.length() >= 2) {
+ sp.valueProperty("arg1", args_[1]);
+ }
+ }
+#endif
+}
+
+// Class which holds a shape pointer for use when caches might reference data in
+// other zones.
+static const JSClass shapeContainerClass = {"ShapeContainer",
+ JSCLASS_HAS_RESERVED_SLOTS(1)};
+
+static const size_t SHAPE_CONTAINER_SLOT = 0;
+
+static JSObject* NewWrapperWithObjectShape(JSContext* cx,
+ Handle<NativeObject*> obj) {
+ MOZ_ASSERT(cx->compartment() != obj->compartment());
+
+ RootedObject wrapper(cx);
+ {
+ AutoRealm ar(cx, obj);
+ wrapper = NewBuiltinClassInstance(cx, &shapeContainerClass);
+ if (!wrapper) {
+ return nullptr;
+ }
+ wrapper->as<NativeObject>().setReservedSlot(
+ SHAPE_CONTAINER_SLOT, PrivateGCThingValue(obj->shape()));
+ }
+ if (!JS_WrapObject(cx, &wrapper)) {
+ return nullptr;
+ }
+ MOZ_ASSERT(IsWrapper(wrapper));
+ return wrapper;
+}
+
+void jit::LoadShapeWrapperContents(MacroAssembler& masm, Register obj,
+ Register dst, Label* failure) {
+ masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), dst);
+ Address privateAddr(dst,
+ js::detail::ProxyReservedSlots::offsetOfPrivateSlot());
+ masm.fallibleUnboxObject(privateAddr, dst, failure);
+ masm.unboxNonDouble(
+ Address(dst, NativeObject::getFixedSlotOffset(SHAPE_CONTAINER_SLOT)), dst,
+ JSVAL_TYPE_PRIVATE_GCTHING);
+}
+
+CompareIRGenerator::CompareIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state, JSOp op,
+ HandleValue lhsVal, HandleValue rhsVal)
+ : IRGenerator(cx, script, pc, CacheKind::Compare, state),
+ op_(op),
+ lhsVal_(lhsVal),
+ rhsVal_(rhsVal) {}
+
+AttachDecision CompareIRGenerator::tryAttachString(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ if (!lhsVal_.isString() || !rhsVal_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ StringOperandId lhsStrId = writer.guardToString(lhsId);
+ StringOperandId rhsStrId = writer.guardToString(rhsId);
+ writer.compareStringResult(op_, lhsStrId, rhsStrId);
+ writer.returnFromIC();
+
+ trackAttached("String");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachObject(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ MOZ_ASSERT(IsEqualityOp(op_));
+
+ if (!lhsVal_.isObject() || !rhsVal_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ ObjOperandId lhsObjId = writer.guardToObject(lhsId);
+ ObjOperandId rhsObjId = writer.guardToObject(rhsId);
+ writer.compareObjectResult(op_, lhsObjId, rhsObjId);
+ writer.returnFromIC();
+
+ trackAttached("Object");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachSymbol(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ MOZ_ASSERT(IsEqualityOp(op_));
+
+ if (!lhsVal_.isSymbol() || !rhsVal_.isSymbol()) {
+ return AttachDecision::NoAction;
+ }
+
+ SymbolOperandId lhsSymId = writer.guardToSymbol(lhsId);
+ SymbolOperandId rhsSymId = writer.guardToSymbol(rhsId);
+ writer.compareSymbolResult(op_, lhsSymId, rhsSymId);
+ writer.returnFromIC();
+
+ trackAttached("Symbol");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachStrictDifferentTypes(
+ ValOperandId lhsId, ValOperandId rhsId) {
+ MOZ_ASSERT(IsEqualityOp(op_));
+
+ if (op_ != JSOp::StrictEq && op_ != JSOp::StrictNe) {
+ return AttachDecision::NoAction;
+ }
+
+ // Probably can't hit some of these.
+ if (SameType(lhsVal_, rhsVal_) ||
+ (lhsVal_.isNumber() && rhsVal_.isNumber())) {
+ return AttachDecision::NoAction;
+ }
+
+ // Compare tags
+ ValueTagOperandId lhsTypeId = writer.loadValueTag(lhsId);
+ ValueTagOperandId rhsTypeId = writer.loadValueTag(rhsId);
+ writer.guardTagNotEqual(lhsTypeId, rhsTypeId);
+
+ // Now that we've passed the guard, we know differing types, so return the
+ // bool result.
+ writer.loadBooleanResult(op_ == JSOp::StrictNe ? true : false);
+ writer.returnFromIC();
+
+ trackAttached("StrictDifferentTypes");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachInt32(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ if ((!lhsVal_.isInt32() && !lhsVal_.isBoolean()) ||
+ (!rhsVal_.isInt32() && !rhsVal_.isBoolean())) {
+ return AttachDecision::NoAction;
+ }
+
+ Int32OperandId lhsIntId = lhsVal_.isBoolean()
+ ? writer.guardBooleanToInt32(lhsId)
+ : writer.guardToInt32(lhsId);
+ Int32OperandId rhsIntId = rhsVal_.isBoolean()
+ ? writer.guardBooleanToInt32(rhsId)
+ : writer.guardToInt32(rhsId);
+
+ // Strictly different types should have been handed by
+ // tryAttachStrictDifferentTypes
+ MOZ_ASSERT_IF(op_ == JSOp::StrictEq || op_ == JSOp::StrictNe,
+ lhsVal_.isInt32() == rhsVal_.isInt32());
+
+ writer.compareInt32Result(op_, lhsIntId, rhsIntId);
+ writer.returnFromIC();
+
+ trackAttached(lhsVal_.isBoolean() ? "Boolean" : "Int32");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachNumber(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ if (!lhsVal_.isNumber() || !rhsVal_.isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ NumberOperandId lhs = writer.guardIsNumber(lhsId);
+ NumberOperandId rhs = writer.guardIsNumber(rhsId);
+ writer.compareDoubleResult(op_, lhs, rhs);
+ writer.returnFromIC();
+
+ trackAttached("Number");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachBigInt(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ if (!lhsVal_.isBigInt() || !rhsVal_.isBigInt()) {
+ return AttachDecision::NoAction;
+ }
+
+ BigIntOperandId lhs = writer.guardToBigInt(lhsId);
+ BigIntOperandId rhs = writer.guardToBigInt(rhsId);
+
+ writer.compareBigIntResult(op_, lhs, rhs);
+ writer.returnFromIC();
+
+ trackAttached("BigInt");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachNumberUndefined(
+ ValOperandId lhsId, ValOperandId rhsId) {
+ if (!(lhsVal_.isUndefined() && rhsVal_.isNumber()) &&
+ !(rhsVal_.isUndefined() && lhsVal_.isNumber())) {
+ return AttachDecision::NoAction;
+ }
+
+ // Should have been handled by tryAttachAnyNullUndefined.
+ MOZ_ASSERT(!IsEqualityOp(op_));
+
+ if (lhsVal_.isNumber()) {
+ writer.guardIsNumber(lhsId);
+ } else {
+ writer.guardIsUndefined(lhsId);
+ }
+
+ if (rhsVal_.isNumber()) {
+ writer.guardIsNumber(rhsId);
+ } else {
+ writer.guardIsUndefined(rhsId);
+ }
+
+ // Relational comparing a number with undefined will always be false.
+ writer.loadBooleanResult(false);
+ writer.returnFromIC();
+
+ trackAttached("NumberUndefined");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachAnyNullUndefined(
+ ValOperandId lhsId, ValOperandId rhsId) {
+ MOZ_ASSERT(IsEqualityOp(op_));
+
+ // Either RHS or LHS needs to be null/undefined.
+ if (!lhsVal_.isNullOrUndefined() && !rhsVal_.isNullOrUndefined()) {
+ return AttachDecision::NoAction;
+ }
+
+ // We assume that the side with null/undefined is usually constant, in
+ // code like `if (x === undefined) { x = {}; }`.
+ // That is why we don't attach when both sides are undefined/null,
+ // because we would basically need to decide by chance which side is
+ // the likely constant.
+ // The actual generated code however handles null/undefined of course.
+ if (lhsVal_.isNullOrUndefined() && rhsVal_.isNullOrUndefined()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (rhsVal_.isNullOrUndefined()) {
+ if (rhsVal_.isNull()) {
+ writer.guardIsNull(rhsId);
+ writer.compareNullUndefinedResult(op_, /* isUndefined */ false, lhsId);
+ trackAttached("AnyNull");
+ } else {
+ writer.guardIsUndefined(rhsId);
+ writer.compareNullUndefinedResult(op_, /* isUndefined */ true, lhsId);
+ trackAttached("AnyUndefined");
+ }
+ } else {
+ if (lhsVal_.isNull()) {
+ writer.guardIsNull(lhsId);
+ writer.compareNullUndefinedResult(op_, /* isUndefined */ false, rhsId);
+ trackAttached("NullAny");
+ } else {
+ writer.guardIsUndefined(lhsId);
+ writer.compareNullUndefinedResult(op_, /* isUndefined */ true, rhsId);
+ trackAttached("UndefinedAny");
+ }
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+// Handle {null/undefined} x {null,undefined} equality comparisons
+AttachDecision CompareIRGenerator::tryAttachNullUndefined(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ if (!lhsVal_.isNullOrUndefined() || !rhsVal_.isNullOrUndefined()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (op_ == JSOp::Eq || op_ == JSOp::Ne) {
+ writer.guardIsNullOrUndefined(lhsId);
+ writer.guardIsNullOrUndefined(rhsId);
+ // Sloppy equality means we actually only care about the op:
+ writer.loadBooleanResult(op_ == JSOp::Eq);
+ trackAttached("SloppyNullUndefined");
+ } else {
+ // Strict equality only hits this branch, and only in the
+ // undef {!,=}== undef and null {!,=}== null cases.
+ // The other cases should have hit tryAttachStrictDifferentTypes.
+ MOZ_ASSERT(lhsVal_.isNull() == rhsVal_.isNull());
+ lhsVal_.isNull() ? writer.guardIsNull(lhsId)
+ : writer.guardIsUndefined(lhsId);
+ rhsVal_.isNull() ? writer.guardIsNull(rhsId)
+ : writer.guardIsUndefined(rhsId);
+ writer.loadBooleanResult(op_ == JSOp::StrictEq);
+ trackAttached("StrictNullUndefinedEquality");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachStringNumber(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ // Ensure String x Number
+ if (!(lhsVal_.isString() && rhsVal_.isNumber()) &&
+ !(rhsVal_.isString() && lhsVal_.isNumber())) {
+ return AttachDecision::NoAction;
+ }
+
+ // Case should have been handled by tryAttachStrictDifferentTypes
+ MOZ_ASSERT(op_ != JSOp::StrictEq && op_ != JSOp::StrictNe);
+
+ auto createGuards = [&](const Value& v, ValOperandId vId) {
+ if (v.isString()) {
+ StringOperandId strId = writer.guardToString(vId);
+ return writer.guardStringToNumber(strId);
+ }
+ MOZ_ASSERT(v.isNumber());
+ NumberOperandId numId = writer.guardIsNumber(vId);
+ return numId;
+ };
+
+ NumberOperandId lhsGuardedId = createGuards(lhsVal_, lhsId);
+ NumberOperandId rhsGuardedId = createGuards(rhsVal_, rhsId);
+ writer.compareDoubleResult(op_, lhsGuardedId, rhsGuardedId);
+ writer.returnFromIC();
+
+ trackAttached("StringNumber");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachPrimitiveSymbol(
+ ValOperandId lhsId, ValOperandId rhsId) {
+ MOZ_ASSERT(IsEqualityOp(op_));
+
+ // The set of primitive cases we want to handle here (excluding null,
+ // undefined, and symbol)
+ auto isPrimitive = [](const Value& x) {
+ return x.isString() || x.isBoolean() || x.isNumber() || x.isBigInt();
+ };
+
+ // Ensure Symbol x {String, Bool, Number, BigInt}.
+ if (!(lhsVal_.isSymbol() && isPrimitive(rhsVal_)) &&
+ !(rhsVal_.isSymbol() && isPrimitive(lhsVal_))) {
+ return AttachDecision::NoAction;
+ }
+
+ auto guardPrimitive = [&](const Value& v, ValOperandId id) {
+ MOZ_ASSERT(isPrimitive(v));
+ if (v.isNumber()) {
+ writer.guardIsNumber(id);
+ return;
+ }
+ switch (v.extractNonDoubleType()) {
+ case JSVAL_TYPE_STRING:
+ writer.guardToString(id);
+ return;
+ case JSVAL_TYPE_BOOLEAN:
+ writer.guardToBoolean(id);
+ return;
+ case JSVAL_TYPE_BIGINT:
+ writer.guardToBigInt(id);
+ return;
+ default:
+ MOZ_CRASH("unexpected type");
+ return;
+ }
+ };
+
+ if (lhsVal_.isSymbol()) {
+ writer.guardToSymbol(lhsId);
+ guardPrimitive(rhsVal_, rhsId);
+ } else {
+ guardPrimitive(lhsVal_, lhsId);
+ writer.guardToSymbol(rhsId);
+ }
+
+ // Comparing a primitive with symbol will always be true for Ne/StrictNe, and
+ // always be false for other compare ops.
+ writer.loadBooleanResult(op_ == JSOp::Ne || op_ == JSOp::StrictNe);
+ writer.returnFromIC();
+
+ trackAttached("PrimitiveSymbol");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachBoolStringOrNumber(
+ ValOperandId lhsId, ValOperandId rhsId) {
+ // Ensure Boolean x {String, Number}.
+ if (!(lhsVal_.isBoolean() && (rhsVal_.isString() || rhsVal_.isNumber())) &&
+ !(rhsVal_.isBoolean() && (lhsVal_.isString() || lhsVal_.isNumber()))) {
+ return AttachDecision::NoAction;
+ }
+
+ // Case should have been handled by tryAttachStrictDifferentTypes
+ MOZ_ASSERT(op_ != JSOp::StrictEq && op_ != JSOp::StrictNe);
+
+ // Case should have been handled by tryAttachInt32
+ MOZ_ASSERT(!lhsVal_.isInt32() && !rhsVal_.isInt32());
+
+ auto createGuards = [&](const Value& v, ValOperandId vId) {
+ if (v.isBoolean()) {
+ BooleanOperandId boolId = writer.guardToBoolean(vId);
+ return writer.booleanToNumber(boolId);
+ }
+ if (v.isString()) {
+ StringOperandId strId = writer.guardToString(vId);
+ return writer.guardStringToNumber(strId);
+ }
+ MOZ_ASSERT(v.isNumber());
+ return writer.guardIsNumber(vId);
+ };
+
+ NumberOperandId lhsGuardedId = createGuards(lhsVal_, lhsId);
+ NumberOperandId rhsGuardedId = createGuards(rhsVal_, rhsId);
+ writer.compareDoubleResult(op_, lhsGuardedId, rhsGuardedId);
+ writer.returnFromIC();
+
+ trackAttached("BoolStringOrNumber");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachBigIntInt32(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ // Ensure BigInt x {Int32, Boolean}.
+ if (!(lhsVal_.isBigInt() && (rhsVal_.isInt32() || rhsVal_.isBoolean())) &&
+ !(rhsVal_.isBigInt() && (lhsVal_.isInt32() || lhsVal_.isBoolean()))) {
+ return AttachDecision::NoAction;
+ }
+
+ // Case should have been handled by tryAttachStrictDifferentTypes
+ MOZ_ASSERT(op_ != JSOp::StrictEq && op_ != JSOp::StrictNe);
+
+ auto createGuards = [&](const Value& v, ValOperandId vId) {
+ if (v.isBoolean()) {
+ return writer.guardBooleanToInt32(vId);
+ }
+ MOZ_ASSERT(v.isInt32());
+ return writer.guardToInt32(vId);
+ };
+
+ if (lhsVal_.isBigInt()) {
+ BigIntOperandId bigIntId = writer.guardToBigInt(lhsId);
+ Int32OperandId intId = createGuards(rhsVal_, rhsId);
+
+ writer.compareBigIntInt32Result(op_, bigIntId, intId);
+ } else {
+ Int32OperandId intId = createGuards(lhsVal_, lhsId);
+ BigIntOperandId bigIntId = writer.guardToBigInt(rhsId);
+
+ writer.compareBigIntInt32Result(ReverseCompareOp(op_), bigIntId, intId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("BigIntInt32");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachBigIntNumber(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ // Ensure BigInt x Number.
+ if (!(lhsVal_.isBigInt() && rhsVal_.isNumber()) &&
+ !(rhsVal_.isBigInt() && lhsVal_.isNumber())) {
+ return AttachDecision::NoAction;
+ }
+
+ // Case should have been handled by tryAttachStrictDifferentTypes
+ MOZ_ASSERT(op_ != JSOp::StrictEq && op_ != JSOp::StrictNe);
+
+ if (lhsVal_.isBigInt()) {
+ BigIntOperandId bigIntId = writer.guardToBigInt(lhsId);
+ NumberOperandId numId = writer.guardIsNumber(rhsId);
+
+ writer.compareBigIntNumberResult(op_, bigIntId, numId);
+ } else {
+ NumberOperandId numId = writer.guardIsNumber(lhsId);
+ BigIntOperandId bigIntId = writer.guardToBigInt(rhsId);
+
+ writer.compareBigIntNumberResult(ReverseCompareOp(op_), bigIntId, numId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("BigIntNumber");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachBigIntString(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ // Ensure BigInt x String.
+ if (!(lhsVal_.isBigInt() && rhsVal_.isString()) &&
+ !(rhsVal_.isBigInt() && lhsVal_.isString())) {
+ return AttachDecision::NoAction;
+ }
+
+ // Case should have been handled by tryAttachStrictDifferentTypes
+ MOZ_ASSERT(op_ != JSOp::StrictEq && op_ != JSOp::StrictNe);
+
+ if (lhsVal_.isBigInt()) {
+ BigIntOperandId bigIntId = writer.guardToBigInt(lhsId);
+ StringOperandId strId = writer.guardToString(rhsId);
+
+ writer.compareBigIntStringResult(op_, bigIntId, strId);
+ } else {
+ StringOperandId strId = writer.guardToString(lhsId);
+ BigIntOperandId bigIntId = writer.guardToBigInt(rhsId);
+
+ writer.compareBigIntStringResult(ReverseCompareOp(op_), bigIntId, strId);
+ }
+ writer.returnFromIC();
+
+ trackAttached("BigIntString");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CompareIRGenerator::tryAttachStub() {
+ MOZ_ASSERT(cacheKind_ == CacheKind::Compare);
+ MOZ_ASSERT(IsEqualityOp(op_) || IsRelationalOp(op_));
+
+ AutoAssertNoPendingException aanpe(cx_);
+
+ constexpr uint8_t lhsIndex = 0;
+ constexpr uint8_t rhsIndex = 1;
+
+ ValOperandId lhsId(writer.setInputOperandId(lhsIndex));
+ ValOperandId rhsId(writer.setInputOperandId(rhsIndex));
+
+ // For sloppy equality ops, there are cases this IC does not handle:
+ // - {Object} x {String, Symbol, Bool, Number, BigInt}.
+ //
+ // (The above lists omits the equivalent case {B} x {A} when {A} x {B} is
+ // already present.)
+
+ if (IsEqualityOp(op_)) {
+ TRY_ATTACH(tryAttachObject(lhsId, rhsId));
+ TRY_ATTACH(tryAttachSymbol(lhsId, rhsId));
+
+ // Handles any (non null or undefined) comparison with null/undefined.
+ TRY_ATTACH(tryAttachAnyNullUndefined(lhsId, rhsId));
+
+ // This covers -strict- equality/inequality using a type tag check, so
+ // catches all different type pairs outside of Numbers, which cannot be
+ // checked on tags alone.
+ TRY_ATTACH(tryAttachStrictDifferentTypes(lhsId, rhsId));
+
+ TRY_ATTACH(tryAttachNullUndefined(lhsId, rhsId));
+
+ TRY_ATTACH(tryAttachPrimitiveSymbol(lhsId, rhsId));
+ }
+
+ // This should preceed the Int32/Number cases to allow
+ // them to not concern themselves with handling undefined
+ // or null.
+ TRY_ATTACH(tryAttachNumberUndefined(lhsId, rhsId));
+
+ // We want these to be last, to allow us to bypass the
+ // strictly-different-types cases in the below attachment code
+ TRY_ATTACH(tryAttachInt32(lhsId, rhsId));
+ TRY_ATTACH(tryAttachNumber(lhsId, rhsId));
+ TRY_ATTACH(tryAttachBigInt(lhsId, rhsId));
+ TRY_ATTACH(tryAttachString(lhsId, rhsId));
+
+ TRY_ATTACH(tryAttachStringNumber(lhsId, rhsId));
+ TRY_ATTACH(tryAttachBoolStringOrNumber(lhsId, rhsId));
+
+ TRY_ATTACH(tryAttachBigIntInt32(lhsId, rhsId));
+ TRY_ATTACH(tryAttachBigIntNumber(lhsId, rhsId));
+ TRY_ATTACH(tryAttachBigIntString(lhsId, rhsId));
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+void CompareIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("lhs", lhsVal_);
+ sp.valueProperty("rhs", rhsVal_);
+ sp.opcodeProperty("op", op_);
+ }
+#endif
+}
+
+ToBoolIRGenerator::ToBoolIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleValue val)
+ : IRGenerator(cx, script, pc, CacheKind::ToBool, state), val_(val) {}
+
+void ToBoolIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("val", val_);
+ }
+#endif
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+ writer.setTypeData(TypeData(JSValueType(val_.type())));
+
+ TRY_ATTACH(tryAttachBool());
+ TRY_ATTACH(tryAttachInt32());
+ TRY_ATTACH(tryAttachNumber());
+ TRY_ATTACH(tryAttachString());
+ TRY_ATTACH(tryAttachNullOrUndefined());
+ TRY_ATTACH(tryAttachObject());
+ TRY_ATTACH(tryAttachSymbol());
+ TRY_ATTACH(tryAttachBigInt());
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachBool() {
+ if (!val_.isBoolean()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ writer.guardNonDoubleType(valId, ValueType::Boolean);
+ writer.loadOperandResult(valId);
+ writer.returnFromIC();
+ trackAttached("ToBoolBool");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachInt32() {
+ if (!val_.isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ writer.guardNonDoubleType(valId, ValueType::Int32);
+ writer.loadInt32TruthyResult(valId);
+ writer.returnFromIC();
+ trackAttached("ToBoolInt32");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachNumber() {
+ if (!val_.isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ NumberOperandId numId = writer.guardIsNumber(valId);
+ writer.loadDoubleTruthyResult(numId);
+ writer.returnFromIC();
+ trackAttached("ToBoolNumber");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachSymbol() {
+ if (!val_.isSymbol()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ writer.guardNonDoubleType(valId, ValueType::Symbol);
+ writer.loadBooleanResult(true);
+ writer.returnFromIC();
+ trackAttached("ToBoolSymbol");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachString() {
+ if (!val_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ StringOperandId strId = writer.guardToString(valId);
+ writer.loadStringTruthyResult(strId);
+ writer.returnFromIC();
+ trackAttached("ToBoolString");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachNullOrUndefined() {
+ if (!val_.isNullOrUndefined()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ writer.guardIsNullOrUndefined(valId);
+ writer.loadBooleanResult(false);
+ writer.returnFromIC();
+ trackAttached("ToBoolNullOrUndefined");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachObject() {
+ if (!val_.isObject()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ ObjOperandId objId = writer.guardToObject(valId);
+ writer.loadObjectTruthyResult(objId);
+ writer.returnFromIC();
+ trackAttached("ToBoolObject");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToBoolIRGenerator::tryAttachBigInt() {
+ if (!val_.isBigInt()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ BigIntOperandId bigIntId = writer.guardToBigInt(valId);
+ writer.loadBigIntTruthyResult(bigIntId);
+ writer.returnFromIC();
+ trackAttached("ToBoolBigInt");
+ return AttachDecision::Attach;
+}
+
+GetIntrinsicIRGenerator::GetIntrinsicIRGenerator(JSContext* cx,
+ HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleValue val)
+ : IRGenerator(cx, script, pc, CacheKind::GetIntrinsic, state), val_(val) {}
+
+void GetIntrinsicIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("val", val_);
+ }
+#endif
+}
+
+AttachDecision GetIntrinsicIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+ writer.loadValueResult(val_);
+ writer.returnFromIC();
+ trackAttached("GetIntrinsic");
+ return AttachDecision::Attach;
+}
+
+UnaryArithIRGenerator::UnaryArithIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ JSOp op, HandleValue val,
+ HandleValue res)
+ : IRGenerator(cx, script, pc, CacheKind::UnaryArith, state),
+ op_(op),
+ val_(val),
+ res_(res) {}
+
+void UnaryArithIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("val", val_);
+ sp.valueProperty("res", res_);
+ }
+#endif
+}
+
+AttachDecision UnaryArithIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+ TRY_ATTACH(tryAttachInt32());
+ TRY_ATTACH(tryAttachNumber());
+ TRY_ATTACH(tryAttachBitwise());
+ TRY_ATTACH(tryAttachBigInt());
+ TRY_ATTACH(tryAttachStringInt32());
+ TRY_ATTACH(tryAttachStringNumber());
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+static bool CanConvertToInt32ForToNumber(const Value& v) {
+ return v.isInt32() || v.isBoolean() || v.isNull();
+}
+
+static Int32OperandId EmitGuardToInt32ForToNumber(CacheIRWriter& writer,
+ ValOperandId id,
+ const Value& v) {
+ if (v.isInt32()) {
+ return writer.guardToInt32(id);
+ }
+ if (v.isNull()) {
+ writer.guardIsNull(id);
+ return writer.loadInt32Constant(0);
+ }
+ MOZ_ASSERT(v.isBoolean());
+ return writer.guardBooleanToInt32(id);
+}
+
+AttachDecision UnaryArithIRGenerator::tryAttachInt32() {
+ if (op_ == JSOp::BitNot) {
+ return AttachDecision::NoAction;
+ }
+ if (!CanConvertToInt32ForToNumber(val_) || !res_.isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+
+ Int32OperandId intId = EmitGuardToInt32ForToNumber(writer, valId, val_);
+ switch (op_) {
+ case JSOp::Pos:
+ writer.loadInt32Result(intId);
+ trackAttached("UnaryArith.Int32Pos");
+ break;
+ case JSOp::Neg:
+ writer.int32NegationResult(intId);
+ trackAttached("UnaryArith.Int32Neg");
+ break;
+ case JSOp::Inc:
+ writer.int32IncResult(intId);
+ trackAttached("UnaryArith.Int32Inc");
+ break;
+ case JSOp::Dec:
+ writer.int32DecResult(intId);
+ trackAttached("UnaryArith.Int32Dec");
+ break;
+ case JSOp::ToNumeric:
+ writer.loadInt32Result(intId);
+ trackAttached("UnaryArith.Int32ToNumeric");
+ break;
+ default:
+ MOZ_CRASH("unexpected OP");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+static bool CanConvertToDoubleForToNumber(const Value& v) {
+ return v.isNumber() || v.isBoolean() || v.isNullOrUndefined();
+}
+
+static NumberOperandId EmitGuardToDoubleForToNumber(CacheIRWriter& writer,
+ ValOperandId id,
+ const Value& v) {
+ if (v.isNumber()) {
+ return writer.guardIsNumber(id);
+ }
+ if (v.isBoolean()) {
+ BooleanOperandId boolId = writer.guardToBoolean(id);
+ return writer.booleanToNumber(boolId);
+ }
+ if (v.isNull()) {
+ writer.guardIsNull(id);
+ return writer.loadDoubleConstant(0.0);
+ }
+ MOZ_ASSERT(v.isUndefined());
+ writer.guardIsUndefined(id);
+ return writer.loadDoubleConstant(JS::GenericNaN());
+}
+
+AttachDecision UnaryArithIRGenerator::tryAttachNumber() {
+ if (op_ == JSOp::BitNot) {
+ return AttachDecision::NoAction;
+ }
+ if (!CanConvertToDoubleForToNumber(val_)) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(res_.isNumber());
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ NumberOperandId numId = EmitGuardToDoubleForToNumber(writer, valId, val_);
+
+ switch (op_) {
+ case JSOp::Pos:
+ writer.loadDoubleResult(numId);
+ trackAttached("UnaryArith.DoublePos");
+ break;
+ case JSOp::Neg:
+ writer.doubleNegationResult(numId);
+ trackAttached("UnaryArith.DoubleNeg");
+ break;
+ case JSOp::Inc:
+ writer.doubleIncResult(numId);
+ trackAttached("UnaryArith.DoubleInc");
+ break;
+ case JSOp::Dec:
+ writer.doubleDecResult(numId);
+ trackAttached("UnaryArith.DoubleDec");
+ break;
+ case JSOp::ToNumeric:
+ writer.loadDoubleResult(numId);
+ trackAttached("UnaryArith.DoubleToNumeric");
+ break;
+ default:
+ MOZ_CRASH("Unexpected OP");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+static bool CanTruncateToInt32(const Value& val) {
+ return val.isNumber() || val.isBoolean() || val.isNullOrUndefined() ||
+ val.isString();
+}
+
+// Convert type into int32 for the bitwise/shift operands.
+static Int32OperandId EmitTruncateToInt32Guard(CacheIRWriter& writer,
+ ValOperandId id,
+ const Value& val) {
+ MOZ_ASSERT(CanTruncateToInt32(val));
+ if (val.isInt32()) {
+ return writer.guardToInt32(id);
+ }
+ if (val.isBoolean()) {
+ return writer.guardBooleanToInt32(id);
+ }
+ if (val.isNullOrUndefined()) {
+ writer.guardIsNullOrUndefined(id);
+ return writer.loadInt32Constant(0);
+ }
+ NumberOperandId numId;
+ if (val.isString()) {
+ StringOperandId strId = writer.guardToString(id);
+ numId = writer.guardStringToNumber(strId);
+ } else {
+ MOZ_ASSERT(val.isDouble());
+ numId = writer.guardIsNumber(id);
+ }
+ return writer.truncateDoubleToUInt32(numId);
+}
+
+AttachDecision UnaryArithIRGenerator::tryAttachBitwise() {
+ // Only bitwise operators.
+ if (op_ != JSOp::BitNot) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check guard conditions
+ if (!CanTruncateToInt32(val_)) {
+ return AttachDecision::NoAction;
+ }
+
+ // Bitwise operators always produce Int32 values.
+ MOZ_ASSERT(res_.isInt32());
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ Int32OperandId intId = EmitTruncateToInt32Guard(writer, valId, val_);
+ writer.int32NotResult(intId);
+ trackAttached("BinaryArith.Bitwise.BitNot");
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+AttachDecision UnaryArithIRGenerator::tryAttachBigInt() {
+ if (!val_.isBigInt()) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(res_.isBigInt());
+
+ MOZ_ASSERT(op_ != JSOp::Pos,
+ "Applying the unary + operator on BigInt values throws an error");
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ BigIntOperandId bigIntId = writer.guardToBigInt(valId);
+ switch (op_) {
+ case JSOp::BitNot:
+ writer.bigIntNotResult(bigIntId);
+ trackAttached("UnaryArith.BigIntNot");
+ break;
+ case JSOp::Neg:
+ writer.bigIntNegationResult(bigIntId);
+ trackAttached("UnaryArith.BigIntNeg");
+ break;
+ case JSOp::Inc:
+ writer.bigIntIncResult(bigIntId);
+ trackAttached("UnaryArith.BigIntInc");
+ break;
+ case JSOp::Dec:
+ writer.bigIntDecResult(bigIntId);
+ trackAttached("UnaryArith.BigIntDec");
+ break;
+ case JSOp::ToNumeric:
+ writer.loadBigIntResult(bigIntId);
+ trackAttached("UnaryArith.BigIntToNumeric");
+ break;
+ default:
+ MOZ_CRASH("Unexpected OP");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+AttachDecision UnaryArithIRGenerator::tryAttachStringInt32() {
+ if (!val_.isString()) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(res_.isNumber());
+
+ // Case should have been handled by tryAttachBitwise.
+ MOZ_ASSERT(op_ != JSOp::BitNot);
+
+ if (!res_.isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ StringOperandId stringId = writer.guardToString(valId);
+ Int32OperandId intId = writer.guardStringToInt32(stringId);
+
+ switch (op_) {
+ case JSOp::Pos:
+ writer.loadInt32Result(intId);
+ trackAttached("UnaryArith.StringInt32Pos");
+ break;
+ case JSOp::Neg:
+ writer.int32NegationResult(intId);
+ trackAttached("UnaryArith.StringInt32Neg");
+ break;
+ case JSOp::Inc:
+ writer.int32IncResult(intId);
+ trackAttached("UnaryArith.StringInt32Inc");
+ break;
+ case JSOp::Dec:
+ writer.int32DecResult(intId);
+ trackAttached("UnaryArith.StringInt32Dec");
+ break;
+ case JSOp::ToNumeric:
+ writer.loadInt32Result(intId);
+ trackAttached("UnaryArith.StringInt32ToNumeric");
+ break;
+ default:
+ MOZ_CRASH("Unexpected OP");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+AttachDecision UnaryArithIRGenerator::tryAttachStringNumber() {
+ if (!val_.isString()) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(res_.isNumber());
+
+ // Case should have been handled by tryAttachBitwise.
+ MOZ_ASSERT(op_ != JSOp::BitNot);
+
+ ValOperandId valId(writer.setInputOperandId(0));
+ StringOperandId stringId = writer.guardToString(valId);
+ NumberOperandId numId = writer.guardStringToNumber(stringId);
+
+ Int32OperandId truncatedId;
+ switch (op_) {
+ case JSOp::Pos:
+ writer.loadDoubleResult(numId);
+ trackAttached("UnaryArith.StringNumberPos");
+ break;
+ case JSOp::Neg:
+ writer.doubleNegationResult(numId);
+ trackAttached("UnaryArith.StringNumberNeg");
+ break;
+ case JSOp::Inc:
+ writer.doubleIncResult(numId);
+ trackAttached("UnaryArith.StringNumberInc");
+ break;
+ case JSOp::Dec:
+ writer.doubleDecResult(numId);
+ trackAttached("UnaryArith.StringNumberDec");
+ break;
+ case JSOp::ToNumeric:
+ writer.loadDoubleResult(numId);
+ trackAttached("UnaryArith.StringNumberToNumeric");
+ break;
+ default:
+ MOZ_CRASH("Unexpected OP");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+ToPropertyKeyIRGenerator::ToPropertyKeyIRGenerator(JSContext* cx,
+ HandleScript script,
+ jsbytecode* pc,
+ ICState state,
+ HandleValue val)
+ : IRGenerator(cx, script, pc, CacheKind::ToPropertyKey, state), val_(val) {}
+
+void ToPropertyKeyIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("val", val_);
+ }
+#endif
+}
+
+AttachDecision ToPropertyKeyIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+ TRY_ATTACH(tryAttachInt32());
+ TRY_ATTACH(tryAttachNumber());
+ TRY_ATTACH(tryAttachString());
+ TRY_ATTACH(tryAttachSymbol());
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+AttachDecision ToPropertyKeyIRGenerator::tryAttachInt32() {
+ if (!val_.isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+
+ Int32OperandId intId = writer.guardToInt32(valId);
+ writer.loadInt32Result(intId);
+ writer.returnFromIC();
+
+ trackAttached("ToPropertyKey.Int32");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToPropertyKeyIRGenerator::tryAttachNumber() {
+ if (!val_.isNumber()) {
+ return AttachDecision::NoAction;
+ }
+
+ // We allow negative zero here because ToPropertyKey(-0.0) is 0.
+ int32_t unused;
+ if (!mozilla::NumberEqualsInt32(val_.toNumber(), &unused)) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+
+ Int32OperandId intId = writer.guardToInt32Index(valId);
+ writer.loadInt32Result(intId);
+ writer.returnFromIC();
+
+ trackAttached("ToPropertyKey.Number");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToPropertyKeyIRGenerator::tryAttachString() {
+ if (!val_.isString()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+
+ StringOperandId strId = writer.guardToString(valId);
+ writer.loadStringResult(strId);
+ writer.returnFromIC();
+
+ trackAttached("ToPropertyKey.String");
+ return AttachDecision::Attach;
+}
+
+AttachDecision ToPropertyKeyIRGenerator::tryAttachSymbol() {
+ if (!val_.isSymbol()) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId valId(writer.setInputOperandId(0));
+
+ SymbolOperandId strId = writer.guardToSymbol(valId);
+ writer.loadSymbolResult(strId);
+ writer.returnFromIC();
+
+ trackAttached("ToPropertyKey.Symbol");
+ return AttachDecision::Attach;
+}
+
+BinaryArithIRGenerator::BinaryArithIRGenerator(JSContext* cx,
+ HandleScript script,
+ jsbytecode* pc, ICState state,
+ JSOp op, HandleValue lhs,
+ HandleValue rhs, HandleValue res)
+ : IRGenerator(cx, script, pc, CacheKind::BinaryArith, state),
+ op_(op),
+ lhs_(lhs),
+ rhs_(rhs),
+ res_(res) {}
+
+void BinaryArithIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.opcodeProperty("op", op_);
+ sp.valueProperty("rhs", rhs_);
+ sp.valueProperty("lhs", lhs_);
+ }
+#endif
+}
+
+AttachDecision BinaryArithIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+ // Arithmetic operations with Int32 operands
+ TRY_ATTACH(tryAttachInt32());
+
+ // Bitwise operations with Int32/Double/Boolean/Null/Undefined/String
+ // operands.
+ TRY_ATTACH(tryAttachBitwise());
+
+ // Arithmetic operations with Double operands. This needs to come after
+ // tryAttachInt32, as the guards overlap, and we'd prefer to attach the
+ // more specialized Int32 IC if it is possible.
+ TRY_ATTACH(tryAttachDouble());
+
+ // String x {String,Number,Boolean,Null,Undefined}
+ TRY_ATTACH(tryAttachStringConcat());
+
+ // String x Object
+ TRY_ATTACH(tryAttachStringObjectConcat());
+
+ // Arithmetic operations or bitwise operations with BigInt operands
+ TRY_ATTACH(tryAttachBigInt());
+
+ // Arithmetic operations (without addition) with String x Int32.
+ TRY_ATTACH(tryAttachStringInt32Arith());
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+AttachDecision BinaryArithIRGenerator::tryAttachBitwise() {
+ // Only bit-wise and shifts.
+ if (op_ != JSOp::BitOr && op_ != JSOp::BitXor && op_ != JSOp::BitAnd &&
+ op_ != JSOp::Lsh && op_ != JSOp::Rsh && op_ != JSOp::Ursh) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check guard conditions
+ if (!CanTruncateToInt32(lhs_) || !CanTruncateToInt32(rhs_)) {
+ return AttachDecision::NoAction;
+ }
+
+ // All ops, with the exception of Ursh, produce Int32 values.
+ MOZ_ASSERT_IF(op_ != JSOp::Ursh, res_.isInt32());
+
+ ValOperandId lhsId(writer.setInputOperandId(0));
+ ValOperandId rhsId(writer.setInputOperandId(1));
+
+ Int32OperandId lhsIntId = EmitTruncateToInt32Guard(writer, lhsId, lhs_);
+ Int32OperandId rhsIntId = EmitTruncateToInt32Guard(writer, rhsId, rhs_);
+
+ switch (op_) {
+ case JSOp::BitOr:
+ writer.int32BitOrResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Bitwise.BitOr");
+ break;
+ case JSOp::BitXor:
+ writer.int32BitXorResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Bitwise.BitXor");
+ break;
+ case JSOp::BitAnd:
+ writer.int32BitAndResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Bitwise.BitAnd");
+ break;
+ case JSOp::Lsh:
+ writer.int32LeftShiftResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Bitwise.LeftShift");
+ break;
+ case JSOp::Rsh:
+ writer.int32RightShiftResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Bitwise.RightShift");
+ break;
+ case JSOp::Ursh:
+ writer.int32URightShiftResult(lhsIntId, rhsIntId, res_.isDouble());
+ trackAttached("BinaryArith.Bitwise.UnsignedRightShift");
+ break;
+ default:
+ MOZ_CRASH("Unhandled op in tryAttachBitwise");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+AttachDecision BinaryArithIRGenerator::tryAttachDouble() {
+ // Check valid opcodes
+ if (op_ != JSOp::Add && op_ != JSOp::Sub && op_ != JSOp::Mul &&
+ op_ != JSOp::Div && op_ != JSOp::Mod && op_ != JSOp::Pow) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check guard conditions.
+ if (!CanConvertToDoubleForToNumber(lhs_) ||
+ !CanConvertToDoubleForToNumber(rhs_)) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId lhsId(writer.setInputOperandId(0));
+ ValOperandId rhsId(writer.setInputOperandId(1));
+
+ NumberOperandId lhs = EmitGuardToDoubleForToNumber(writer, lhsId, lhs_);
+ NumberOperandId rhs = EmitGuardToDoubleForToNumber(writer, rhsId, rhs_);
+
+ switch (op_) {
+ case JSOp::Add:
+ writer.doubleAddResult(lhs, rhs);
+ trackAttached("BinaryArith.Double.Add");
+ break;
+ case JSOp::Sub:
+ writer.doubleSubResult(lhs, rhs);
+ trackAttached("BinaryArith.Double.Sub");
+ break;
+ case JSOp::Mul:
+ writer.doubleMulResult(lhs, rhs);
+ trackAttached("BinaryArith.Double.Mul");
+ break;
+ case JSOp::Div:
+ writer.doubleDivResult(lhs, rhs);
+ trackAttached("BinaryArith.Double.Div");
+ break;
+ case JSOp::Mod:
+ writer.doubleModResult(lhs, rhs);
+ trackAttached("BinaryArith.Double.Mod");
+ break;
+ case JSOp::Pow:
+ writer.doublePowResult(lhs, rhs);
+ trackAttached("BinaryArith.Double.Pow");
+ break;
+ default:
+ MOZ_CRASH("Unhandled Op");
+ }
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+AttachDecision BinaryArithIRGenerator::tryAttachInt32() {
+ // Check guard conditions.
+ if (!CanConvertToInt32ForToNumber(lhs_) ||
+ !CanConvertToInt32ForToNumber(rhs_)) {
+ return AttachDecision::NoAction;
+ }
+
+ // These ICs will failure() if result can't be encoded in an Int32:
+ // If sample result is not Int32, we should avoid IC.
+ if (!res_.isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (op_ != JSOp::Add && op_ != JSOp::Sub && op_ != JSOp::Mul &&
+ op_ != JSOp::Div && op_ != JSOp::Mod && op_ != JSOp::Pow) {
+ return AttachDecision::NoAction;
+ }
+
+ if (op_ == JSOp::Pow && !CanAttachInt32Pow(lhs_, rhs_)) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId lhsId(writer.setInputOperandId(0));
+ ValOperandId rhsId(writer.setInputOperandId(1));
+
+ Int32OperandId lhsIntId = EmitGuardToInt32ForToNumber(writer, lhsId, lhs_);
+ Int32OperandId rhsIntId = EmitGuardToInt32ForToNumber(writer, rhsId, rhs_);
+
+ switch (op_) {
+ case JSOp::Add:
+ writer.int32AddResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Int32.Add");
+ break;
+ case JSOp::Sub:
+ writer.int32SubResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Int32.Sub");
+ break;
+ case JSOp::Mul:
+ writer.int32MulResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Int32.Mul");
+ break;
+ case JSOp::Div:
+ writer.int32DivResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Int32.Div");
+ break;
+ case JSOp::Mod:
+ writer.int32ModResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Int32.Mod");
+ break;
+ case JSOp::Pow:
+ writer.int32PowResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.Int32.Pow");
+ break;
+ default:
+ MOZ_CRASH("Unhandled op in tryAttachInt32");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+AttachDecision BinaryArithIRGenerator::tryAttachStringConcat() {
+ // Only Addition
+ if (op_ != JSOp::Add) {
+ return AttachDecision::NoAction;
+ }
+
+ // One side must be a string, the other side a primitive value we can easily
+ // convert to a string.
+ if (!(lhs_.isString() && CanConvertToString(rhs_)) &&
+ !(CanConvertToString(lhs_) && rhs_.isString())) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId lhsId(writer.setInputOperandId(0));
+ ValOperandId rhsId(writer.setInputOperandId(1));
+
+ StringOperandId lhsStrId = emitToStringGuard(lhsId, lhs_);
+ StringOperandId rhsStrId = emitToStringGuard(rhsId, rhs_);
+
+ writer.callStringConcatResult(lhsStrId, rhsStrId);
+
+ writer.returnFromIC();
+ trackAttached("BinaryArith.StringConcat");
+ return AttachDecision::Attach;
+}
+
+AttachDecision BinaryArithIRGenerator::tryAttachStringObjectConcat() {
+ // Only Addition
+ if (op_ != JSOp::Add) {
+ return AttachDecision::NoAction;
+ }
+
+ // Check Guards
+ if (!(lhs_.isObject() && rhs_.isString()) &&
+ !(lhs_.isString() && rhs_.isObject()))
+ return AttachDecision::NoAction;
+
+ ValOperandId lhsId(writer.setInputOperandId(0));
+ ValOperandId rhsId(writer.setInputOperandId(1));
+
+ // This guard is actually overly tight, as the runtime
+ // helper can handle lhs or rhs being a string, so long
+ // as the other is an object.
+ if (lhs_.isString()) {
+ writer.guardToString(lhsId);
+ writer.guardToObject(rhsId);
+ } else {
+ writer.guardToObject(lhsId);
+ writer.guardToString(rhsId);
+ }
+
+ writer.callStringObjectConcatResult(lhsId, rhsId);
+
+ writer.returnFromIC();
+ trackAttached("BinaryArith.StringObjectConcat");
+ return AttachDecision::Attach;
+}
+
+AttachDecision BinaryArithIRGenerator::tryAttachBigInt() {
+ // Check Guards
+ if (!lhs_.isBigInt() || !rhs_.isBigInt()) {
+ return AttachDecision::NoAction;
+ }
+
+ switch (op_) {
+ case JSOp::Add:
+ case JSOp::Sub:
+ case JSOp::Mul:
+ case JSOp::Div:
+ case JSOp::Mod:
+ case JSOp::Pow:
+ // Arithmetic operations.
+ break;
+
+ case JSOp::BitOr:
+ case JSOp::BitXor:
+ case JSOp::BitAnd:
+ case JSOp::Lsh:
+ case JSOp::Rsh:
+ // Bitwise operations.
+ break;
+
+ default:
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId lhsId(writer.setInputOperandId(0));
+ ValOperandId rhsId(writer.setInputOperandId(1));
+
+ BigIntOperandId lhsBigIntId = writer.guardToBigInt(lhsId);
+ BigIntOperandId rhsBigIntId = writer.guardToBigInt(rhsId);
+
+ switch (op_) {
+ case JSOp::Add:
+ writer.bigIntAddResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.Add");
+ break;
+ case JSOp::Sub:
+ writer.bigIntSubResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.Sub");
+ break;
+ case JSOp::Mul:
+ writer.bigIntMulResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.Mul");
+ break;
+ case JSOp::Div:
+ writer.bigIntDivResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.Div");
+ break;
+ case JSOp::Mod:
+ writer.bigIntModResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.Mod");
+ break;
+ case JSOp::Pow:
+ writer.bigIntPowResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.Pow");
+ break;
+ case JSOp::BitOr:
+ writer.bigIntBitOrResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.BitOr");
+ break;
+ case JSOp::BitXor:
+ writer.bigIntBitXorResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.BitXor");
+ break;
+ case JSOp::BitAnd:
+ writer.bigIntBitAndResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.BitAnd");
+ break;
+ case JSOp::Lsh:
+ writer.bigIntLeftShiftResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.LeftShift");
+ break;
+ case JSOp::Rsh:
+ writer.bigIntRightShiftResult(lhsBigIntId, rhsBigIntId);
+ trackAttached("BinaryArith.BigInt.RightShift");
+ break;
+ default:
+ MOZ_CRASH("Unhandled op in tryAttachBigInt");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+AttachDecision BinaryArithIRGenerator::tryAttachStringInt32Arith() {
+ // Check for either int32 x string or string x int32.
+ if (!(lhs_.isInt32() && rhs_.isString()) &&
+ !(lhs_.isString() && rhs_.isInt32())) {
+ return AttachDecision::NoAction;
+ }
+
+ // The created ICs will fail if the result can't be encoded as as int32.
+ // Thus skip this IC, if the sample result is not an int32.
+ if (!res_.isInt32()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Must _not_ support Add, because it would be string concatenation instead.
+ // For Pow we can't easily determine the CanAttachInt32Pow conditions so we
+ // reject that as well.
+ if (op_ != JSOp::Sub && op_ != JSOp::Mul && op_ != JSOp::Div &&
+ op_ != JSOp::Mod) {
+ return AttachDecision::NoAction;
+ }
+
+ ValOperandId lhsId(writer.setInputOperandId(0));
+ ValOperandId rhsId(writer.setInputOperandId(1));
+
+ auto guardToInt32 = [&](ValOperandId id, const Value& v) {
+ if (v.isInt32()) {
+ return writer.guardToInt32(id);
+ }
+
+ MOZ_ASSERT(v.isString());
+ StringOperandId strId = writer.guardToString(id);
+ return writer.guardStringToInt32(strId);
+ };
+
+ Int32OperandId lhsIntId = guardToInt32(lhsId, lhs_);
+ Int32OperandId rhsIntId = guardToInt32(rhsId, rhs_);
+
+ switch (op_) {
+ case JSOp::Sub:
+ writer.int32SubResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.StringInt32.Sub");
+ break;
+ case JSOp::Mul:
+ writer.int32MulResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.StringInt32.Mul");
+ break;
+ case JSOp::Div:
+ writer.int32DivResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.StringInt32.Div");
+ break;
+ case JSOp::Mod:
+ writer.int32ModResult(lhsIntId, rhsIntId);
+ trackAttached("BinaryArith.StringInt32.Mod");
+ break;
+ default:
+ MOZ_CRASH("Unhandled op in tryAttachStringInt32Arith");
+ }
+
+ writer.returnFromIC();
+ return AttachDecision::Attach;
+}
+
+NewArrayIRGenerator::NewArrayIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state, JSOp op,
+ HandleObject templateObj,
+ BaselineFrame* frame)
+ : IRGenerator(cx, script, pc, CacheKind::NewArray, state),
+#ifdef JS_CACHEIR_SPEW
+ op_(op),
+#endif
+ templateObject_(templateObj),
+ frame_(frame) {
+ MOZ_ASSERT(templateObject_);
+}
+
+void NewArrayIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.opcodeProperty("op", op_);
+ }
+#endif
+}
+
+// Allocation sites are usually created during baseline compilation, but we also
+// need to created them when an IC stub is added to a baseline compiled script
+// and when trial inlining.
+static gc::AllocSite* MaybeCreateAllocSite(jsbytecode* pc,
+ BaselineFrame* frame) {
+ MOZ_ASSERT(BytecodeOpCanHaveAllocSite(JSOp(*pc)));
+
+ JSScript* outerScript = frame->outerScript();
+ bool inInterpreter = frame->runningInInterpreter();
+ bool isInlined = frame->icScript()->isInlined();
+
+ if (inInterpreter && !isInlined) {
+ return outerScript->zone()->unknownAllocSite();
+ }
+
+ return outerScript->createAllocSite();
+}
+
+AttachDecision NewArrayIRGenerator::tryAttachArrayObject() {
+ ArrayObject* arrayObj = &templateObject_->as<ArrayObject>();
+
+ MOZ_ASSERT(arrayObj->numUsedFixedSlots() == 0);
+ MOZ_ASSERT(arrayObj->numDynamicSlots() == 0);
+ MOZ_ASSERT(!arrayObj->isSharedMemory());
+
+ // The macro assembler only supports creating arrays with fixed elements.
+ if (arrayObj->hasDynamicElements()) {
+ return AttachDecision::NoAction;
+ }
+
+ // Stub doesn't support metadata builder
+ if (cx_->realm()->hasAllocationMetadataBuilder()) {
+ return AttachDecision::NoAction;
+ }
+
+ writer.guardNoAllocationMetadataBuilder(
+ cx_->realm()->addressOfMetadataBuilder());
+
+ gc::AllocSite* site = MaybeCreateAllocSite(pc_, frame_);
+ if (!site) {
+ return AttachDecision::NoAction;
+ }
+
+ Shape* shape = arrayObj->shape();
+ uint32_t length = arrayObj->length();
+
+ writer.newArrayObjectResult(length, shape, site);
+
+ writer.returnFromIC();
+
+ trackAttached("NewArrayObject");
+ return AttachDecision::Attach;
+}
+
+AttachDecision NewArrayIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+
+ TRY_ATTACH(tryAttachArrayObject());
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+NewObjectIRGenerator::NewObjectIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ JSOp op, HandleObject templateObj,
+ BaselineFrame* frame)
+ : IRGenerator(cx, script, pc, CacheKind::NewObject, state),
+#ifdef JS_CACHEIR_SPEW
+ op_(op),
+#endif
+ templateObject_(templateObj),
+ frame_(frame) {
+ MOZ_ASSERT(templateObject_);
+}
+
+void NewObjectIRGenerator::trackAttached(const char* name) {
+ stubName_ = name ? name : "NotAttached";
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.opcodeProperty("op", op_);
+ }
+#endif
+}
+
+AttachDecision NewObjectIRGenerator::tryAttachPlainObject() {
+ // Don't optimize allocations with too many dynamic slots. We use an unrolled
+ // loop when initializing slots and this avoids generating too much code.
+ static const uint32_t MaxDynamicSlotsToOptimize = 64;
+
+ NativeObject* nativeObj = &templateObject_->as<NativeObject>();
+ MOZ_ASSERT(nativeObj->is<PlainObject>());
+
+ // Stub doesn't support metadata builder
+ if (cx_->realm()->hasAllocationMetadataBuilder()) {
+ return AttachDecision::NoAction;
+ }
+
+ if (nativeObj->numDynamicSlots() > MaxDynamicSlotsToOptimize) {
+ return AttachDecision::NoAction;
+ }
+
+ MOZ_ASSERT(!nativeObj->hasDynamicElements());
+ MOZ_ASSERT(!nativeObj->isSharedMemory());
+
+ gc::AllocSite* site = MaybeCreateAllocSite(pc_, frame_);
+ if (!site) {
+ return AttachDecision::NoAction;
+ }
+
+ uint32_t numFixedSlots = nativeObj->numUsedFixedSlots();
+ uint32_t numDynamicSlots = nativeObj->numDynamicSlots();
+ gc::AllocKind allocKind = nativeObj->allocKindForTenure();
+ Shape* shape = nativeObj->shape();
+
+ writer.guardNoAllocationMetadataBuilder(
+ cx_->realm()->addressOfMetadataBuilder());
+ writer.newPlainObjectResult(numFixedSlots, numDynamicSlots, allocKind, shape,
+ site);
+
+ writer.returnFromIC();
+
+ trackAttached("NewPlainObject");
+ return AttachDecision::Attach;
+}
+
+AttachDecision NewObjectIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+
+ TRY_ATTACH(tryAttachPlainObject());
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+CloseIterIRGenerator::CloseIterIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleObject iter,
+ CompletionKind kind)
+ : IRGenerator(cx, script, pc, CacheKind::CloseIter, state),
+ iter_(iter),
+ kind_(kind) {}
+
+void CloseIterIRGenerator::trackAttached(const char* name) {
+#ifdef JS_CACHEIR_SPEW
+ if (const CacheIRSpewer::Guard& sp = CacheIRSpewer::Guard(*this, name)) {
+ sp.valueProperty("iter", ObjectValue(*iter_));
+ }
+#endif
+}
+
+AttachDecision CloseIterIRGenerator::tryAttachNoReturnMethod() {
+ Maybe<PropertyInfo> prop;
+ NativeObject* holder = nullptr;
+
+ // If we can guard that the iterator does not have a |return| method,
+ // then this CloseIter is a no-op.
+ NativeGetPropKind kind = CanAttachNativeGetProp(
+ cx_, iter_, NameToId(cx_->names().return_), &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::Missing) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(!holder);
+
+ ObjOperandId objId(writer.setInputOperandId(0));
+
+ EmitMissingPropGuard(writer, &iter_->as<NativeObject>(), objId);
+
+ // There is no return method, so we don't have to do anything.
+ writer.returnFromIC();
+
+ trackAttached("CloseIter.NoReturn");
+ return AttachDecision::Attach;
+}
+
+AttachDecision CloseIterIRGenerator::tryAttachScriptedReturn() {
+ Maybe<PropertyInfo> prop;
+ NativeObject* holder = nullptr;
+
+ NativeGetPropKind kind = CanAttachNativeGetProp(
+ cx_, iter_, NameToId(cx_->names().return_), &holder, &prop, pc_);
+ if (kind != NativeGetPropKind::Slot) {
+ return AttachDecision::NoAction;
+ }
+ MOZ_ASSERT(holder);
+ MOZ_ASSERT(prop->isDataProperty());
+
+ size_t slot = prop->slot();
+ Value calleeVal = holder->getSlot(slot);
+ if (!calleeVal.isObject() || !calleeVal.toObject().is<JSFunction>()) {
+ return AttachDecision::NoAction;
+ }
+
+ JSFunction* callee = &calleeVal.toObject().as<JSFunction>();
+ if (!callee->hasJitEntry()) {
+ return AttachDecision::NoAction;
+ }
+ if (callee->isClassConstructor()) {
+ return AttachDecision::NoAction;
+ }
+
+ // We don't support cross-realm |return|.
+ if (cx_->realm() != callee->realm()) {
+ return AttachDecision::NoAction;
+ }
+
+ ObjOperandId objId(writer.setInputOperandId(0));
+
+ ObjOperandId holderId =
+ EmitReadSlotGuard(writer, &iter_->as<NativeObject>(), holder, objId);
+
+ ValOperandId calleeValId;
+ if (holder->isFixedSlot(slot)) {
+ size_t offset = NativeObject::getFixedSlotOffset(slot);
+ calleeValId = writer.loadFixedSlot(holderId, offset);
+ } else {
+ size_t index = holder->dynamicSlotIndex(slot);
+ calleeValId = writer.loadDynamicSlot(holderId, index);
+ }
+ ObjOperandId calleeId = writer.guardToObject(calleeValId);
+ emitCalleeGuard(calleeId, callee);
+
+ writer.closeIterScriptedResult(objId, calleeId, kind_, callee->nargs());
+
+ writer.returnFromIC();
+ trackAttached("CloseIter.ScriptedReturn");
+
+ return AttachDecision::Attach;
+}
+
+AttachDecision CloseIterIRGenerator::tryAttachStub() {
+ AutoAssertNoPendingException aanpe(cx_);
+
+ TRY_ATTACH(tryAttachNoReturnMethod());
+ TRY_ATTACH(tryAttachScriptedReturn());
+
+ trackAttached(IRGenerator::NotAttached);
+ return AttachDecision::NoAction;
+}
+
+#ifdef JS_SIMULATOR
+bool js::jit::CallAnyNative(JSContext* cx, unsigned argc, Value* vp) {
+ CallArgs args = CallArgsFromVp(argc, vp);
+ JSObject* calleeObj = &args.callee();
+
+ MOZ_ASSERT(calleeObj->is<JSFunction>());
+ auto* calleeFunc = &calleeObj->as<JSFunction>();
+ MOZ_ASSERT(calleeFunc->isNativeWithoutJitEntry());
+
+ JSNative native = calleeFunc->native();
+ return native(cx, args.length(), args.base());
+}
+#endif
diff --git a/js/src/jit/CacheIR.h b/js/src/jit/CacheIR.h
new file mode 100644
index 0000000000..d09072b5f1
--- /dev/null
+++ b/js/src/jit/CacheIR.h
@@ -0,0 +1,523 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CacheIR_h
+#define jit_CacheIR_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jstypes.h"
+
+#include "jit/CacheIROpsGenerated.h"
+#include "js/GCAnnotations.h"
+#include "js/Value.h"
+
+struct JS_PUBLIC_API JSContext;
+
+namespace js {
+namespace jit {
+
+// [SMDOC] CacheIR
+//
+// CacheIR is an (extremely simple) linear IR language for inline caches.
+// From this IR, we can generate machine code for Baseline or Ion IC stubs.
+//
+// IRWriter
+// --------
+// CacheIR bytecode is written using IRWriter. This class also records some
+// metadata that's used by the Baseline and Ion code generators to generate
+// (efficient) machine code.
+//
+// Sharing Baseline stub code
+// --------------------------
+// Baseline stores data (like Shape* and fixed slot offsets) inside the ICStub
+// structure, instead of embedding them directly in the JitCode. This makes
+// Baseline IC code slightly slower, but allows us to share IC code between
+// caches. CacheIR makes it easy to share code between stubs: stubs that have
+// the same CacheIR (and CacheKind), will have the same Baseline stub code.
+//
+// Baseline stubs that share JitCode also share a CacheIRStubInfo structure.
+// This class stores the CacheIR and the location of GC things stored in the
+// stub, for the GC.
+//
+// JitZone has a CacheIRStubInfo* -> JitCode* weak map that's used to share both
+// the IR and JitCode between Baseline CacheIR stubs. This HashMap owns the
+// stubInfo (it uses UniquePtr), so once there are no references left to the
+// shared stub code, we can also free the CacheIRStubInfo.
+//
+// Ion stubs
+// ---------
+// Unlike Baseline stubs, Ion stubs do not share stub code, and data stored in
+// the IonICStub is baked into JIT code. This is one of the reasons Ion stubs
+// are faster than Baseline stubs. Also note that Ion ICs contain more state
+// (see IonGetPropertyIC for example) and use dynamic input/output registers,
+// so sharing stub code for Ion would be much more difficult.
+
+// An OperandId represents either a cache input or a value returned by a
+// CacheIR instruction. Most code should use the ValOperandId and ObjOperandId
+// classes below. The ObjOperandId class represents an operand that's known to
+// be an object, just as StringOperandId represents a known string, etc.
+class OperandId {
+ protected:
+ static const uint16_t InvalidId = UINT16_MAX;
+ uint16_t id_;
+
+ explicit OperandId(uint16_t id) : id_(id) {}
+
+ public:
+ OperandId() : id_(InvalidId) {}
+ uint16_t id() const { return id_; }
+ bool valid() const { return id_ != InvalidId; }
+};
+
+class ValOperandId : public OperandId {
+ public:
+ ValOperandId() = default;
+ explicit ValOperandId(uint16_t id) : OperandId(id) {}
+};
+
+class ValueTagOperandId : public OperandId {
+ public:
+ ValueTagOperandId() = default;
+ explicit ValueTagOperandId(uint16_t id) : OperandId(id) {}
+};
+
+class IntPtrOperandId : public OperandId {
+ public:
+ IntPtrOperandId() = default;
+ explicit IntPtrOperandId(uint16_t id) : OperandId(id) {}
+};
+
+class ObjOperandId : public OperandId {
+ public:
+ ObjOperandId() = default;
+ explicit ObjOperandId(uint16_t id) : OperandId(id) {}
+
+ bool operator==(const ObjOperandId& other) const { return id_ == other.id_; }
+ bool operator!=(const ObjOperandId& other) const { return id_ != other.id_; }
+};
+
+class NumberOperandId : public ValOperandId {
+ public:
+ NumberOperandId() = default;
+ explicit NumberOperandId(uint16_t id) : ValOperandId(id) {}
+};
+
+class StringOperandId : public OperandId {
+ public:
+ StringOperandId() = default;
+ explicit StringOperandId(uint16_t id) : OperandId(id) {}
+};
+
+class SymbolOperandId : public OperandId {
+ public:
+ SymbolOperandId() = default;
+ explicit SymbolOperandId(uint16_t id) : OperandId(id) {}
+};
+
+class BigIntOperandId : public OperandId {
+ public:
+ BigIntOperandId() = default;
+ explicit BigIntOperandId(uint16_t id) : OperandId(id) {}
+};
+
+class BooleanOperandId : public OperandId {
+ public:
+ BooleanOperandId() = default;
+ explicit BooleanOperandId(uint16_t id) : OperandId(id) {}
+};
+
+class Int32OperandId : public OperandId {
+ public:
+ Int32OperandId() = default;
+ explicit Int32OperandId(uint16_t id) : OperandId(id) {}
+};
+
+class TypedOperandId : public OperandId {
+ JSValueType type_;
+
+ public:
+ MOZ_IMPLICIT TypedOperandId(ObjOperandId id)
+ : OperandId(id.id()), type_(JSVAL_TYPE_OBJECT) {}
+ MOZ_IMPLICIT TypedOperandId(StringOperandId id)
+ : OperandId(id.id()), type_(JSVAL_TYPE_STRING) {}
+ MOZ_IMPLICIT TypedOperandId(SymbolOperandId id)
+ : OperandId(id.id()), type_(JSVAL_TYPE_SYMBOL) {}
+ MOZ_IMPLICIT TypedOperandId(BigIntOperandId id)
+ : OperandId(id.id()), type_(JSVAL_TYPE_BIGINT) {}
+ MOZ_IMPLICIT TypedOperandId(BooleanOperandId id)
+ : OperandId(id.id()), type_(JSVAL_TYPE_BOOLEAN) {}
+ MOZ_IMPLICIT TypedOperandId(Int32OperandId id)
+ : OperandId(id.id()), type_(JSVAL_TYPE_INT32) {}
+
+ MOZ_IMPLICIT TypedOperandId(ValueTagOperandId val)
+ : OperandId(val.id()), type_(JSVAL_TYPE_UNKNOWN) {}
+ MOZ_IMPLICIT TypedOperandId(IntPtrOperandId id)
+ : OperandId(id.id()), type_(JSVAL_TYPE_UNKNOWN) {}
+
+ TypedOperandId(ValOperandId val, JSValueType type)
+ : OperandId(val.id()), type_(type) {}
+
+ JSValueType type() const { return type_; }
+};
+
+#define CACHE_IR_KINDS(_) \
+ _(GetProp) \
+ _(GetElem) \
+ _(GetName) \
+ _(GetPropSuper) \
+ _(GetElemSuper) \
+ _(GetIntrinsic) \
+ _(SetProp) \
+ _(SetElem) \
+ _(BindName) \
+ _(In) \
+ _(HasOwn) \
+ _(CheckPrivateField) \
+ _(TypeOf) \
+ _(ToPropertyKey) \
+ _(InstanceOf) \
+ _(GetIterator) \
+ _(CloseIter) \
+ _(OptimizeSpreadCall) \
+ _(Compare) \
+ _(ToBool) \
+ _(Call) \
+ _(UnaryArith) \
+ _(BinaryArith) \
+ _(NewObject) \
+ _(NewArray)
+
+enum class CacheKind : uint8_t {
+#define DEFINE_KIND(kind) kind,
+ CACHE_IR_KINDS(DEFINE_KIND)
+#undef DEFINE_KIND
+};
+
+extern const char* const CacheKindNames[];
+
+extern size_t NumInputsForCacheKind(CacheKind kind);
+
+enum class CacheOp {
+#define DEFINE_OP(op, ...) op,
+ CACHE_IR_OPS(DEFINE_OP)
+#undef DEFINE_OP
+};
+
+// CacheIR opcode info that's read in performance-sensitive code. Stored as a
+// single byte per op for better cache locality.
+struct CacheIROpInfo {
+ uint8_t argLength : 7;
+ bool transpile : 1;
+};
+static_assert(sizeof(CacheIROpInfo) == 1);
+extern const CacheIROpInfo CacheIROpInfos[];
+
+extern const char* const CacheIROpNames[];
+
+inline const char* CacheIRCodeName(CacheOp op) {
+ return CacheIROpNames[static_cast<size_t>(op)];
+}
+
+extern const uint32_t CacheIROpHealth[];
+
+class StubField {
+ public:
+ enum class Type : uint8_t {
+ // These fields take up a single word.
+ RawInt32,
+ RawPointer,
+ Shape,
+ GetterSetter,
+ JSObject,
+ Symbol,
+ String,
+ BaseScript,
+ Id,
+ AllocSite,
+
+ // These fields take up 64 bits on all platforms.
+ RawInt64,
+ First64BitType = RawInt64,
+ Value,
+ Double,
+
+ Limit
+ };
+
+ static bool sizeIsWord(Type type) {
+ MOZ_ASSERT(type != Type::Limit);
+ return type < Type::First64BitType;
+ }
+
+ static bool sizeIsInt64(Type type) {
+ MOZ_ASSERT(type != Type::Limit);
+ return type >= Type::First64BitType;
+ }
+
+ static size_t sizeInBytes(Type type) {
+ if (sizeIsWord(type)) {
+ return sizeof(uintptr_t);
+ }
+ MOZ_ASSERT(sizeIsInt64(type));
+ return sizeof(int64_t);
+ }
+
+ private:
+ uint64_t data_;
+ Type type_;
+
+ public:
+ StubField(uint64_t data, Type type) : data_(data), type_(type) {
+ MOZ_ASSERT_IF(sizeIsWord(), data <= UINTPTR_MAX);
+ }
+
+ Type type() const { return type_; }
+
+ bool sizeIsWord() const { return sizeIsWord(type_); }
+ bool sizeIsInt64() const { return sizeIsInt64(type_); }
+
+ size_t sizeInBytes() const { return sizeInBytes(type_); }
+
+ uintptr_t asWord() const {
+ MOZ_ASSERT(sizeIsWord());
+ return uintptr_t(data_);
+ }
+ uint64_t asInt64() const {
+ MOZ_ASSERT(sizeIsInt64());
+ return data_;
+ }
+} JS_HAZ_GC_POINTER;
+
+// This class is used to wrap up information about a call to make it
+// easier to convey from one function to another. (In particular,
+// CacheIRWriter encodes the CallFlags in CacheIR, and CacheIRReader
+// decodes them and uses them for compilation.)
+class CallFlags {
+ public:
+ enum ArgFormat : uint8_t {
+ Unknown,
+ Standard,
+ Spread,
+ FunCall,
+ FunApplyArgsObj,
+ FunApplyArray,
+ LastArgFormat = FunApplyArray
+ };
+
+ CallFlags() = default;
+ explicit CallFlags(ArgFormat format) : argFormat_(format) {}
+ CallFlags(bool isConstructing, bool isSpread, bool isSameRealm = false,
+ bool needsUninitializedThis = false)
+ : argFormat_(isSpread ? Spread : Standard),
+ isConstructing_(isConstructing),
+ isSameRealm_(isSameRealm),
+ needsUninitializedThis_(needsUninitializedThis) {}
+
+ ArgFormat getArgFormat() const { return argFormat_; }
+ bool isConstructing() const {
+ MOZ_ASSERT_IF(isConstructing_,
+ argFormat_ == Standard || argFormat_ == Spread);
+ return isConstructing_;
+ }
+ bool isSameRealm() const { return isSameRealm_; }
+ void setIsSameRealm() { isSameRealm_ = true; }
+
+ bool needsUninitializedThis() const { return needsUninitializedThis_; }
+ void setNeedsUninitializedThis() { needsUninitializedThis_ = true; }
+
+ uint8_t toByte() const {
+ // See CacheIRReader::callFlags()
+ MOZ_ASSERT(argFormat_ != ArgFormat::Unknown);
+ uint8_t value = getArgFormat();
+ if (isConstructing()) {
+ value |= CallFlags::IsConstructing;
+ }
+ if (isSameRealm()) {
+ value |= CallFlags::IsSameRealm;
+ }
+ if (needsUninitializedThis()) {
+ value |= CallFlags::NeedsUninitializedThis;
+ }
+ return value;
+ }
+
+ private:
+ ArgFormat argFormat_ = ArgFormat::Unknown;
+ bool isConstructing_ = false;
+ bool isSameRealm_ = false;
+ bool needsUninitializedThis_ = false;
+
+ // Used for encoding/decoding
+ static const uint8_t ArgFormatBits = 4;
+ static const uint8_t ArgFormatMask = (1 << ArgFormatBits) - 1;
+ static_assert(LastArgFormat <= ArgFormatMask, "Not enough arg format bits");
+ static const uint8_t IsConstructing = 1 << 5;
+ static const uint8_t IsSameRealm = 1 << 6;
+ static const uint8_t NeedsUninitializedThis = 1 << 7;
+
+ friend class CacheIRReader;
+ friend class CacheIRWriter;
+};
+
+// In baseline, we have to copy args onto the stack. Below this threshold, we
+// will unroll the arg copy loop. We need to clamp this before providing it as
+// an arg to a CacheIR op so that everything 5 or greater can share an IC.
+const uint32_t MaxUnrolledArgCopy = 5;
+inline uint32_t ClampFixedArgc(uint32_t argc) {
+ return std::min(argc, MaxUnrolledArgCopy);
+}
+
+enum class AttachDecision {
+ // We cannot attach a stub.
+ NoAction,
+
+ // We can attach a stub.
+ Attach,
+
+ // We cannot currently attach a stub, but we expect to be able to do so in the
+ // future. In this case, we do not call trackNotAttached().
+ TemporarilyUnoptimizable,
+
+ // We want to attach a stub, but the result of the operation is
+ // needed to generate that stub. For example, AddSlot needs to know
+ // the resulting shape. Note: the attached stub will inspect the
+ // inputs to the operation, so most input checks should be done
+ // before the actual operation, with only minimal checks remaining
+ // for the deferred portion. This prevents arbitrary scripted code
+ // run by the operation from interfering with the conditions being
+ // checked.
+ Deferred
+};
+
+// If the input expression evaluates to an AttachDecision other than NoAction,
+// return that AttachDecision. If it is NoAction, do nothing.
+#define TRY_ATTACH(expr) \
+ do { \
+ AttachDecision tryAttachTempResult_ = expr; \
+ if (tryAttachTempResult_ != AttachDecision::NoAction) { \
+ return tryAttachTempResult_; \
+ } \
+ } while (0)
+
+// Set of arguments supported by GetIndexOfArgument.
+// Support for higher argument indices can be added easily, but is currently
+// unneeded.
+enum class ArgumentKind : uint8_t {
+ Callee,
+ This,
+ NewTarget,
+ Arg0,
+ Arg1,
+ Arg2,
+ Arg3,
+ Arg4,
+ Arg5,
+ Arg6,
+ Arg7,
+ NumKinds
+};
+
+const uint8_t ArgumentKindArgIndexLimit =
+ uint8_t(ArgumentKind::NumKinds) - uint8_t(ArgumentKind::Arg0);
+
+inline ArgumentKind ArgumentKindForArgIndex(uint32_t idx) {
+ MOZ_ASSERT(idx < ArgumentKindArgIndexLimit);
+ return ArgumentKind(uint32_t(ArgumentKind::Arg0) + idx);
+}
+
+// This function calculates the index of an argument based on the call flags.
+// addArgc is an out-parameter, indicating whether the value of argc should
+// be added to the return value to find the actual index.
+inline int32_t GetIndexOfArgument(ArgumentKind kind, CallFlags flags,
+ bool* addArgc) {
+ // *** STACK LAYOUT (bottom to top) *** ******** INDEX ********
+ // Callee <-- argc+1 + isConstructing
+ // ThisValue <-- argc + isConstructing
+ // Args: | Arg0 | | ArgArray | <-- argc-1 + isConstructing
+ // | Arg1 | --or-- | | <-- argc-2 + isConstructing
+ // | ... | | (if spread | <-- ...
+ // | ArgN | | call) | <-- 0 + isConstructing
+ // NewTarget (only if constructing) <-- 0 (if it exists)
+ //
+ // If this is a spread call, then argc is always 1, and we can calculate the
+ // index directly. If this is not a spread call, then the index of any
+ // argument other than NewTarget depends on argc.
+
+ // First we determine whether the caller needs to add argc.
+ switch (flags.getArgFormat()) {
+ case CallFlags::Standard:
+ *addArgc = true;
+ break;
+ case CallFlags::Spread:
+ // Spread calls do not have Arg1 or higher.
+ MOZ_ASSERT(kind <= ArgumentKind::Arg0);
+ *addArgc = false;
+ break;
+ case CallFlags::Unknown:
+ case CallFlags::FunCall:
+ case CallFlags::FunApplyArgsObj:
+ case CallFlags::FunApplyArray:
+ MOZ_CRASH("Currently unreachable");
+ break;
+ }
+
+ // Second, we determine the offset relative to argc.
+ bool hasArgumentArray = !*addArgc;
+ switch (kind) {
+ case ArgumentKind::Callee:
+ return flags.isConstructing() + hasArgumentArray + 1;
+ case ArgumentKind::This:
+ return flags.isConstructing() + hasArgumentArray;
+ case ArgumentKind::Arg0:
+ return flags.isConstructing() + hasArgumentArray - 1;
+ case ArgumentKind::Arg1:
+ return flags.isConstructing() + hasArgumentArray - 2;
+ case ArgumentKind::Arg2:
+ return flags.isConstructing() + hasArgumentArray - 3;
+ case ArgumentKind::Arg3:
+ return flags.isConstructing() + hasArgumentArray - 4;
+ case ArgumentKind::Arg4:
+ return flags.isConstructing() + hasArgumentArray - 5;
+ case ArgumentKind::Arg5:
+ return flags.isConstructing() + hasArgumentArray - 6;
+ case ArgumentKind::Arg6:
+ return flags.isConstructing() + hasArgumentArray - 7;
+ case ArgumentKind::Arg7:
+ return flags.isConstructing() + hasArgumentArray - 8;
+ case ArgumentKind::NewTarget:
+ MOZ_ASSERT(flags.isConstructing());
+ *addArgc = false;
+ return 0;
+ default:
+ MOZ_CRASH("Invalid argument kind");
+ }
+}
+
+// We use this enum as GuardClass operand, instead of storing Class* pointers
+// in the IR, to keep the IR compact and the same size on all platforms.
+enum class GuardClassKind : uint8_t {
+ Array,
+ PlainObject,
+ ArrayBuffer,
+ SharedArrayBuffer,
+ DataView,
+ MappedArguments,
+ UnmappedArguments,
+ WindowProxy,
+ JSFunction,
+ Set,
+ Map,
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_CacheIR_h */
diff --git a/js/src/jit/CacheIRCloner.h b/js/src/jit/CacheIRCloner.h
new file mode 100644
index 0000000000..c9a4224fd2
--- /dev/null
+++ b/js/src/jit/CacheIRCloner.h
@@ -0,0 +1,82 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CacheIRCloner_h
+#define jit_CacheIRCloner_h
+
+#include "mozilla/Attributes.h"
+
+#include <stdint.h>
+
+#include "NamespaceImports.h"
+
+#include "jit/CacheIR.h"
+#include "jit/CacheIROpsGenerated.h"
+#include "jit/CacheIRReader.h"
+#include "jit/CacheIRWriter.h"
+#include "js/Id.h"
+#include "js/Value.h"
+
+class JSAtom;
+class JSObject;
+class JSString;
+
+namespace JS {
+class Symbol;
+}
+
+namespace js {
+
+class BaseScript;
+class GetterSetter;
+class PropertyName;
+class Shape;
+
+namespace gc {
+class AllocSite;
+}
+
+namespace jit {
+
+class CacheIRStubInfo;
+class ICCacheIRStub;
+
+class MOZ_RAII CacheIRCloner {
+ public:
+ explicit CacheIRCloner(ICCacheIRStub* stubInfo);
+
+ void cloneOp(CacheOp op, CacheIRReader& reader, CacheIRWriter& writer);
+
+ CACHE_IR_CLONE_GENERATED
+
+ private:
+ const CacheIRStubInfo* stubInfo_;
+ const uint8_t* stubData_;
+
+ uintptr_t readStubWord(uint32_t offset);
+ int64_t readStubInt64(uint32_t offset);
+
+ Shape* getShapeField(uint32_t stubOffset);
+ GetterSetter* getGetterSetterField(uint32_t stubOffset);
+ JSObject* getObjectField(uint32_t stubOffset);
+ JSString* getStringField(uint32_t stubOffset);
+ JSAtom* getAtomField(uint32_t stubOffset);
+ PropertyName* getPropertyNameField(uint32_t stubOffset);
+ JS::Symbol* getSymbolField(uint32_t stubOffset);
+ BaseScript* getBaseScriptField(uint32_t stubOffset);
+ uint32_t getRawInt32Field(uint32_t stubOffset);
+ const void* getRawPointerField(uint32_t stubOffset);
+ jsid getIdField(uint32_t stubOffset);
+ const Value getValueField(uint32_t stubOffset);
+ uint64_t getRawInt64Field(uint32_t stubOffset);
+ double getDoubleField(uint32_t stubOffset);
+ gc::AllocSite* getAllocSiteField(uint32_t stubOffset);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_CacheIRCloner_h */
diff --git a/js/src/jit/CacheIRCompiler.cpp b/js/src/jit/CacheIRCompiler.cpp
new file mode 100644
index 0000000000..5dc646b375
--- /dev/null
+++ b/js/src/jit/CacheIRCompiler.cpp
@@ -0,0 +1,9573 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/CacheIRCompiler.h"
+
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/FunctionTypeTraits.h"
+#include "mozilla/MaybeOneOf.h"
+#include "mozilla/ScopeExit.h"
+
+#include <type_traits>
+#include <utility>
+
+#include "jslibmath.h"
+#include "jsmath.h"
+
+#include "builtin/DataViewObject.h"
+#include "builtin/MapObject.h"
+#include "builtin/Object.h"
+#include "gc/Allocator.h"
+#include "jit/BaselineCacheIRCompiler.h"
+#include "jit/CacheIRGenerator.h"
+#include "jit/IonCacheIRCompiler.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitZone.h"
+#include "jit/SharedICHelpers.h"
+#include "jit/SharedICRegisters.h"
+#include "jit/VMFunctions.h"
+#include "js/friend/DOMProxy.h" // JS::ExpandoAndGeneration
+#include "js/friend/XrayJitInfo.h" // js::jit::GetXrayJitInfo
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "proxy/DOMProxy.h"
+#include "proxy/Proxy.h"
+#include "vm/ArgumentsObject.h"
+#include "vm/ArrayBufferObject.h"
+#include "vm/ArrayBufferViewObject.h"
+#include "vm/BigIntType.h"
+#include "vm/FunctionFlags.h" // js::FunctionFlags
+#include "vm/GeneratorObject.h"
+#include "vm/GetterSetter.h"
+#include "vm/Interpreter.h"
+#include "vm/Uint8Clamped.h"
+
+#include "builtin/Boolean-inl.h"
+#include "jit/MacroAssembler-inl.h"
+#include "jit/SharedICHelpers-inl.h"
+#include "jit/VMFunctionList-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::BitwiseCast;
+using mozilla::Maybe;
+
+using JS::ExpandoAndGeneration;
+
+ValueOperand CacheRegisterAllocator::useValueRegister(MacroAssembler& masm,
+ ValOperandId op) {
+ OperandLocation& loc = operandLocations_[op.id()];
+
+ switch (loc.kind()) {
+ case OperandLocation::ValueReg:
+ currentOpRegs_.add(loc.valueReg());
+ return loc.valueReg();
+
+ case OperandLocation::ValueStack: {
+ ValueOperand reg = allocateValueRegister(masm);
+ popValue(masm, &loc, reg);
+ return reg;
+ }
+
+ case OperandLocation::BaselineFrame: {
+ ValueOperand reg = allocateValueRegister(masm);
+ Address addr = addressOf(masm, loc.baselineFrameSlot());
+ masm.loadValue(addr, reg);
+ loc.setValueReg(reg);
+ return reg;
+ }
+
+ case OperandLocation::Constant: {
+ ValueOperand reg = allocateValueRegister(masm);
+ masm.moveValue(loc.constant(), reg);
+ loc.setValueReg(reg);
+ return reg;
+ }
+
+ case OperandLocation::PayloadReg: {
+ // Temporarily add the payload register to currentOpRegs_ so
+ // allocateValueRegister will stay away from it.
+ currentOpRegs_.add(loc.payloadReg());
+ ValueOperand reg = allocateValueRegister(masm);
+ masm.tagValue(loc.payloadType(), loc.payloadReg(), reg);
+ currentOpRegs_.take(loc.payloadReg());
+ availableRegs_.add(loc.payloadReg());
+ loc.setValueReg(reg);
+ return reg;
+ }
+
+ case OperandLocation::PayloadStack: {
+ ValueOperand reg = allocateValueRegister(masm);
+ popPayload(masm, &loc, reg.scratchReg());
+ masm.tagValue(loc.payloadType(), reg.scratchReg(), reg);
+ loc.setValueReg(reg);
+ return reg;
+ }
+
+ case OperandLocation::DoubleReg: {
+ ValueOperand reg = allocateValueRegister(masm);
+ {
+ ScratchDoubleScope fpscratch(masm);
+ masm.boxDouble(loc.doubleReg(), reg, fpscratch);
+ }
+ loc.setValueReg(reg);
+ return reg;
+ }
+
+ case OperandLocation::Uninitialized:
+ break;
+ }
+
+ MOZ_CRASH();
+}
+
+// Load a value operand directly into a float register. Caller must have
+// guarded isNumber on the provided val.
+void CacheRegisterAllocator::ensureDoubleRegister(MacroAssembler& masm,
+ NumberOperandId op,
+ FloatRegister dest) const {
+ // If AutoScratchFloatRegister is active, we have to add sizeof(double) to
+ // any stack slot offsets below.
+ int32_t stackOffset = hasAutoScratchFloatRegisterSpill() ? sizeof(double) : 0;
+
+ const OperandLocation& loc = operandLocations_[op.id()];
+
+ Label failure, done;
+ switch (loc.kind()) {
+ case OperandLocation::ValueReg: {
+ masm.ensureDouble(loc.valueReg(), dest, &failure);
+ break;
+ }
+
+ case OperandLocation::ValueStack: {
+ Address addr = valueAddress(masm, &loc);
+ addr.offset += stackOffset;
+ masm.ensureDouble(addr, dest, &failure);
+ break;
+ }
+
+ case OperandLocation::BaselineFrame: {
+ Address addr = addressOf(masm, loc.baselineFrameSlot());
+ addr.offset += stackOffset;
+ masm.ensureDouble(addr, dest, &failure);
+ break;
+ }
+
+ case OperandLocation::DoubleReg: {
+ masm.moveDouble(loc.doubleReg(), dest);
+ return;
+ }
+
+ case OperandLocation::Constant: {
+ MOZ_ASSERT(loc.constant().isNumber(),
+ "Caller must ensure the operand is a number value");
+ masm.loadConstantDouble(loc.constant().toNumber(), dest);
+ return;
+ }
+
+ case OperandLocation::PayloadReg: {
+ // Doubles can't be stored in payload registers, so this must be an int32.
+ MOZ_ASSERT(loc.payloadType() == JSVAL_TYPE_INT32,
+ "Caller must ensure the operand is a number value");
+ masm.convertInt32ToDouble(loc.payloadReg(), dest);
+ return;
+ }
+
+ case OperandLocation::PayloadStack: {
+ // Doubles can't be stored in payload registers, so this must be an int32.
+ MOZ_ASSERT(loc.payloadType() == JSVAL_TYPE_INT32,
+ "Caller must ensure the operand is a number value");
+ MOZ_ASSERT(loc.payloadStack() <= stackPushed_);
+ Address addr = payloadAddress(masm, &loc);
+ addr.offset += stackOffset;
+ masm.convertInt32ToDouble(addr, dest);
+ return;
+ }
+
+ case OperandLocation::Uninitialized:
+ MOZ_CRASH("Unhandled operand type in ensureDoubleRegister");
+ return;
+ }
+ masm.jump(&done);
+ masm.bind(&failure);
+ masm.assumeUnreachable(
+ "Missing guard allowed non-number to hit ensureDoubleRegister");
+ masm.bind(&done);
+}
+
+void CacheRegisterAllocator::copyToScratchRegister(MacroAssembler& masm,
+ TypedOperandId typedId,
+ Register dest) const {
+ // If AutoScratchFloatRegister is active, we have to add sizeof(double) to
+ // any stack slot offsets below.
+ int32_t stackOffset = hasAutoScratchFloatRegisterSpill() ? sizeof(double) : 0;
+
+ const OperandLocation& loc = operandLocations_[typedId.id()];
+
+ switch (loc.kind()) {
+ case OperandLocation::ValueReg: {
+ masm.unboxNonDouble(loc.valueReg(), dest, typedId.type());
+ break;
+ }
+ case OperandLocation::ValueStack: {
+ Address addr = valueAddress(masm, &loc);
+ addr.offset += stackOffset;
+ masm.unboxNonDouble(addr, dest, typedId.type());
+ break;
+ }
+ case OperandLocation::BaselineFrame: {
+ Address addr = addressOf(masm, loc.baselineFrameSlot());
+ addr.offset += stackOffset;
+ masm.unboxNonDouble(addr, dest, typedId.type());
+ break;
+ }
+ case OperandLocation::PayloadReg: {
+ MOZ_ASSERT(loc.payloadType() == typedId.type());
+ masm.mov(loc.payloadReg(), dest);
+ return;
+ }
+ case OperandLocation::PayloadStack: {
+ MOZ_ASSERT(loc.payloadType() == typedId.type());
+ MOZ_ASSERT(loc.payloadStack() <= stackPushed_);
+ Address addr = payloadAddress(masm, &loc);
+ addr.offset += stackOffset;
+ masm.loadPtr(addr, dest);
+ return;
+ }
+ case OperandLocation::DoubleReg:
+ case OperandLocation::Constant:
+ case OperandLocation::Uninitialized:
+ MOZ_CRASH("Unhandled operand location");
+ }
+}
+
+void CacheRegisterAllocator::copyToScratchValueRegister(
+ MacroAssembler& masm, ValOperandId valId, ValueOperand dest) const {
+ MOZ_ASSERT(!addedFailurePath_);
+ MOZ_ASSERT(!hasAutoScratchFloatRegisterSpill());
+
+ const OperandLocation& loc = operandLocations_[valId.id()];
+ switch (loc.kind()) {
+ case OperandLocation::ValueReg:
+ masm.moveValue(loc.valueReg(), dest);
+ break;
+ case OperandLocation::ValueStack: {
+ Address addr = valueAddress(masm, &loc);
+ masm.loadValue(addr, dest);
+ break;
+ }
+ case OperandLocation::BaselineFrame: {
+ Address addr = addressOf(masm, loc.baselineFrameSlot());
+ masm.loadValue(addr, dest);
+ break;
+ }
+ case OperandLocation::Constant:
+ masm.moveValue(loc.constant(), dest);
+ break;
+ case OperandLocation::PayloadReg:
+ masm.tagValue(loc.payloadType(), loc.payloadReg(), dest);
+ break;
+ case OperandLocation::PayloadStack: {
+ Address addr = payloadAddress(masm, &loc);
+ masm.loadPtr(addr, dest.scratchReg());
+ masm.tagValue(loc.payloadType(), dest.scratchReg(), dest);
+ break;
+ }
+ case OperandLocation::DoubleReg: {
+ ScratchDoubleScope fpscratch(masm);
+ masm.boxDouble(loc.doubleReg(), dest, fpscratch);
+ break;
+ }
+ case OperandLocation::Uninitialized:
+ MOZ_CRASH();
+ }
+}
+
+Register CacheRegisterAllocator::useRegister(MacroAssembler& masm,
+ TypedOperandId typedId) {
+ MOZ_ASSERT(!addedFailurePath_);
+ MOZ_ASSERT(!hasAutoScratchFloatRegisterSpill());
+
+ OperandLocation& loc = operandLocations_[typedId.id()];
+ switch (loc.kind()) {
+ case OperandLocation::PayloadReg:
+ currentOpRegs_.add(loc.payloadReg());
+ return loc.payloadReg();
+
+ case OperandLocation::ValueReg: {
+ // It's possible the value is still boxed: as an optimization, we unbox
+ // the first time we use a value as object.
+ ValueOperand val = loc.valueReg();
+ availableRegs_.add(val);
+ Register reg = val.scratchReg();
+ availableRegs_.take(reg);
+ masm.unboxNonDouble(val, reg, typedId.type());
+ loc.setPayloadReg(reg, typedId.type());
+ currentOpRegs_.add(reg);
+ return reg;
+ }
+
+ case OperandLocation::PayloadStack: {
+ Register reg = allocateRegister(masm);
+ popPayload(masm, &loc, reg);
+ return reg;
+ }
+
+ case OperandLocation::ValueStack: {
+ // The value is on the stack, but boxed. If it's on top of the stack we
+ // unbox it and then remove it from the stack, else we just unbox.
+ Register reg = allocateRegister(masm);
+ if (loc.valueStack() == stackPushed_) {
+ masm.unboxNonDouble(Address(masm.getStackPointer(), 0), reg,
+ typedId.type());
+ masm.addToStackPtr(Imm32(sizeof(js::Value)));
+ MOZ_ASSERT(stackPushed_ >= sizeof(js::Value));
+ stackPushed_ -= sizeof(js::Value);
+ } else {
+ MOZ_ASSERT(loc.valueStack() < stackPushed_);
+ masm.unboxNonDouble(
+ Address(masm.getStackPointer(), stackPushed_ - loc.valueStack()),
+ reg, typedId.type());
+ }
+ loc.setPayloadReg(reg, typedId.type());
+ return reg;
+ }
+
+ case OperandLocation::BaselineFrame: {
+ Register reg = allocateRegister(masm);
+ Address addr = addressOf(masm, loc.baselineFrameSlot());
+ masm.unboxNonDouble(addr, reg, typedId.type());
+ loc.setPayloadReg(reg, typedId.type());
+ return reg;
+ };
+
+ case OperandLocation::Constant: {
+ Value v = loc.constant();
+ Register reg = allocateRegister(masm);
+ if (v.isString()) {
+ masm.movePtr(ImmGCPtr(v.toString()), reg);
+ } else if (v.isSymbol()) {
+ masm.movePtr(ImmGCPtr(v.toSymbol()), reg);
+ } else if (v.isBigInt()) {
+ masm.movePtr(ImmGCPtr(v.toBigInt()), reg);
+ } else {
+ MOZ_CRASH("Unexpected Value");
+ }
+ loc.setPayloadReg(reg, v.extractNonDoubleType());
+ return reg;
+ }
+
+ case OperandLocation::DoubleReg:
+ case OperandLocation::Uninitialized:
+ break;
+ }
+
+ MOZ_CRASH();
+}
+
+ConstantOrRegister CacheRegisterAllocator::useConstantOrRegister(
+ MacroAssembler& masm, ValOperandId val) {
+ MOZ_ASSERT(!addedFailurePath_);
+ MOZ_ASSERT(!hasAutoScratchFloatRegisterSpill());
+
+ OperandLocation& loc = operandLocations_[val.id()];
+ switch (loc.kind()) {
+ case OperandLocation::Constant:
+ return loc.constant();
+
+ case OperandLocation::PayloadReg:
+ case OperandLocation::PayloadStack: {
+ JSValueType payloadType = loc.payloadType();
+ Register reg = useRegister(masm, TypedOperandId(val, payloadType));
+ return TypedOrValueRegister(MIRTypeFromValueType(payloadType),
+ AnyRegister(reg));
+ }
+
+ case OperandLocation::ValueReg:
+ case OperandLocation::ValueStack:
+ case OperandLocation::BaselineFrame:
+ return TypedOrValueRegister(useValueRegister(masm, val));
+
+ case OperandLocation::DoubleReg:
+ return TypedOrValueRegister(MIRType::Double,
+ AnyRegister(loc.doubleReg()));
+
+ case OperandLocation::Uninitialized:
+ break;
+ }
+
+ MOZ_CRASH();
+}
+
+Register CacheRegisterAllocator::defineRegister(MacroAssembler& masm,
+ TypedOperandId typedId) {
+ MOZ_ASSERT(!addedFailurePath_);
+ MOZ_ASSERT(!hasAutoScratchFloatRegisterSpill());
+
+ OperandLocation& loc = operandLocations_[typedId.id()];
+ MOZ_ASSERT(loc.kind() == OperandLocation::Uninitialized);
+
+ Register reg = allocateRegister(masm);
+ loc.setPayloadReg(reg, typedId.type());
+ return reg;
+}
+
+ValueOperand CacheRegisterAllocator::defineValueRegister(MacroAssembler& masm,
+ ValOperandId val) {
+ MOZ_ASSERT(!addedFailurePath_);
+ MOZ_ASSERT(!hasAutoScratchFloatRegisterSpill());
+
+ OperandLocation& loc = operandLocations_[val.id()];
+ MOZ_ASSERT(loc.kind() == OperandLocation::Uninitialized);
+
+ ValueOperand reg = allocateValueRegister(masm);
+ loc.setValueReg(reg);
+ return reg;
+}
+
+void CacheRegisterAllocator::freeDeadOperandLocations(MacroAssembler& masm) {
+ // See if any operands are dead so we can reuse their registers. Note that
+ // we skip the input operands, as those are also used by failure paths, and
+ // we currently don't track those uses.
+ for (size_t i = writer_.numInputOperands(); i < operandLocations_.length();
+ i++) {
+ if (!writer_.operandIsDead(i, currentInstruction_)) {
+ continue;
+ }
+
+ OperandLocation& loc = operandLocations_[i];
+ switch (loc.kind()) {
+ case OperandLocation::PayloadReg:
+ availableRegs_.add(loc.payloadReg());
+ break;
+ case OperandLocation::ValueReg:
+ availableRegs_.add(loc.valueReg());
+ break;
+ case OperandLocation::PayloadStack:
+ masm.propagateOOM(freePayloadSlots_.append(loc.payloadStack()));
+ break;
+ case OperandLocation::ValueStack:
+ masm.propagateOOM(freeValueSlots_.append(loc.valueStack()));
+ break;
+ case OperandLocation::Uninitialized:
+ case OperandLocation::BaselineFrame:
+ case OperandLocation::Constant:
+ case OperandLocation::DoubleReg:
+ break;
+ }
+ loc.setUninitialized();
+ }
+}
+
+void CacheRegisterAllocator::discardStack(MacroAssembler& masm) {
+ // This should only be called when we are no longer using the operands,
+ // as we're discarding everything from the native stack. Set all operand
+ // locations to Uninitialized to catch bugs.
+ for (size_t i = 0; i < operandLocations_.length(); i++) {
+ operandLocations_[i].setUninitialized();
+ }
+
+ if (stackPushed_ > 0) {
+ masm.addToStackPtr(Imm32(stackPushed_));
+ stackPushed_ = 0;
+ }
+ freePayloadSlots_.clear();
+ freeValueSlots_.clear();
+}
+
+Register CacheRegisterAllocator::allocateRegister(MacroAssembler& masm) {
+ MOZ_ASSERT(!addedFailurePath_);
+ MOZ_ASSERT(!hasAutoScratchFloatRegisterSpill());
+
+ if (availableRegs_.empty()) {
+ freeDeadOperandLocations(masm);
+ }
+
+ if (availableRegs_.empty()) {
+ // Still no registers available, try to spill unused operands to
+ // the stack.
+ for (size_t i = 0; i < operandLocations_.length(); i++) {
+ OperandLocation& loc = operandLocations_[i];
+ if (loc.kind() == OperandLocation::PayloadReg) {
+ Register reg = loc.payloadReg();
+ if (currentOpRegs_.has(reg)) {
+ continue;
+ }
+
+ spillOperandToStack(masm, &loc);
+ availableRegs_.add(reg);
+ break; // We got a register, so break out of the loop.
+ }
+ if (loc.kind() == OperandLocation::ValueReg) {
+ ValueOperand reg = loc.valueReg();
+ if (currentOpRegs_.aliases(reg)) {
+ continue;
+ }
+
+ spillOperandToStack(masm, &loc);
+ availableRegs_.add(reg);
+ break; // Break out of the loop.
+ }
+ }
+ }
+
+ if (availableRegs_.empty() && !availableRegsAfterSpill_.empty()) {
+ Register reg = availableRegsAfterSpill_.takeAny();
+ masm.push(reg);
+ stackPushed_ += sizeof(uintptr_t);
+
+ masm.propagateOOM(spilledRegs_.append(SpilledRegister(reg, stackPushed_)));
+
+ availableRegs_.add(reg);
+ }
+
+ // At this point, there must be a free register.
+ MOZ_RELEASE_ASSERT(!availableRegs_.empty());
+
+ Register reg = availableRegs_.takeAny();
+ currentOpRegs_.add(reg);
+ return reg;
+}
+
+void CacheRegisterAllocator::allocateFixedRegister(MacroAssembler& masm,
+ Register reg) {
+ MOZ_ASSERT(!addedFailurePath_);
+ MOZ_ASSERT(!hasAutoScratchFloatRegisterSpill());
+
+ // Fixed registers should be allocated first, to ensure they're
+ // still available.
+ MOZ_ASSERT(!currentOpRegs_.has(reg), "Register is in use");
+
+ freeDeadOperandLocations(masm);
+
+ if (availableRegs_.has(reg)) {
+ availableRegs_.take(reg);
+ currentOpRegs_.add(reg);
+ return;
+ }
+
+ // Register may be available only after spilling contents.
+ if (availableRegsAfterSpill_.has(reg)) {
+ availableRegsAfterSpill_.take(reg);
+ masm.push(reg);
+ stackPushed_ += sizeof(uintptr_t);
+
+ masm.propagateOOM(spilledRegs_.append(SpilledRegister(reg, stackPushed_)));
+ currentOpRegs_.add(reg);
+ return;
+ }
+
+ // The register must be used by some operand. Spill it to the stack.
+ for (size_t i = 0; i < operandLocations_.length(); i++) {
+ OperandLocation& loc = operandLocations_[i];
+ if (loc.kind() == OperandLocation::PayloadReg) {
+ if (loc.payloadReg() != reg) {
+ continue;
+ }
+
+ spillOperandToStackOrRegister(masm, &loc);
+ currentOpRegs_.add(reg);
+ return;
+ }
+ if (loc.kind() == OperandLocation::ValueReg) {
+ if (!loc.valueReg().aliases(reg)) {
+ continue;
+ }
+
+ ValueOperand valueReg = loc.valueReg();
+ spillOperandToStackOrRegister(masm, &loc);
+
+ availableRegs_.add(valueReg);
+ availableRegs_.take(reg);
+ currentOpRegs_.add(reg);
+ return;
+ }
+ }
+
+ MOZ_CRASH("Invalid register");
+}
+
+void CacheRegisterAllocator::allocateFixedValueRegister(MacroAssembler& masm,
+ ValueOperand reg) {
+#ifdef JS_NUNBOX32
+ allocateFixedRegister(masm, reg.payloadReg());
+ allocateFixedRegister(masm, reg.typeReg());
+#else
+ allocateFixedRegister(masm, reg.valueReg());
+#endif
+}
+
+#ifdef JS_NUNBOX32
+// Possible miscompilation in clang-12 (bug 1689641)
+MOZ_NEVER_INLINE
+#endif
+ValueOperand CacheRegisterAllocator::allocateValueRegister(
+ MacroAssembler& masm) {
+#ifdef JS_NUNBOX32
+ Register reg1 = allocateRegister(masm);
+ Register reg2 = allocateRegister(masm);
+ return ValueOperand(reg1, reg2);
+#else
+ Register reg = allocateRegister(masm);
+ return ValueOperand(reg);
+#endif
+}
+
+bool CacheRegisterAllocator::init() {
+ if (!origInputLocations_.resize(writer_.numInputOperands())) {
+ return false;
+ }
+ if (!operandLocations_.resize(writer_.numOperandIds())) {
+ return false;
+ }
+ return true;
+}
+
+void CacheRegisterAllocator::initAvailableRegsAfterSpill() {
+ // Registers not in availableRegs_ and not used by input operands are
+ // available after being spilled.
+ availableRegsAfterSpill_.set() = GeneralRegisterSet::Intersect(
+ GeneralRegisterSet::Not(availableRegs_.set()),
+ GeneralRegisterSet::Not(inputRegisterSet()));
+}
+
+void CacheRegisterAllocator::fixupAliasedInputs(MacroAssembler& masm) {
+ // If IC inputs alias each other, make sure they are stored in different
+ // locations so we don't have to deal with this complexity in the rest of
+ // the allocator.
+ //
+ // Note that this can happen in IonMonkey with something like |o.foo = o|
+ // or |o[i] = i|.
+
+ size_t numInputs = writer_.numInputOperands();
+ MOZ_ASSERT(origInputLocations_.length() == numInputs);
+
+ for (size_t i = 1; i < numInputs; i++) {
+ OperandLocation& loc1 = operandLocations_[i];
+ if (!loc1.isInRegister()) {
+ continue;
+ }
+
+ for (size_t j = 0; j < i; j++) {
+ OperandLocation& loc2 = operandLocations_[j];
+ if (!loc1.aliasesReg(loc2)) {
+ continue;
+ }
+
+ // loc1 and loc2 alias so we spill one of them. If one is a
+ // ValueReg and the other is a PayloadReg, we have to spill the
+ // PayloadReg: spilling the ValueReg instead would leave its type
+ // register unallocated on 32-bit platforms.
+ if (loc1.kind() == OperandLocation::ValueReg) {
+ spillOperandToStack(masm, &loc2);
+ } else {
+ MOZ_ASSERT(loc1.kind() == OperandLocation::PayloadReg);
+ spillOperandToStack(masm, &loc1);
+ break; // Spilled loc1, so nothing else will alias it.
+ }
+ }
+ }
+
+#ifdef DEBUG
+ assertValidState();
+#endif
+}
+
+GeneralRegisterSet CacheRegisterAllocator::inputRegisterSet() const {
+ MOZ_ASSERT(origInputLocations_.length() == writer_.numInputOperands());
+
+ AllocatableGeneralRegisterSet result;
+ for (size_t i = 0; i < writer_.numInputOperands(); i++) {
+ const OperandLocation& loc = operandLocations_[i];
+ MOZ_ASSERT(loc == origInputLocations_[i]);
+
+ switch (loc.kind()) {
+ case OperandLocation::PayloadReg:
+ result.addUnchecked(loc.payloadReg());
+ continue;
+ case OperandLocation::ValueReg:
+ result.addUnchecked(loc.valueReg());
+ continue;
+ case OperandLocation::PayloadStack:
+ case OperandLocation::ValueStack:
+ case OperandLocation::BaselineFrame:
+ case OperandLocation::Constant:
+ case OperandLocation::DoubleReg:
+ continue;
+ case OperandLocation::Uninitialized:
+ break;
+ }
+ MOZ_CRASH("Invalid kind");
+ }
+
+ return result.set();
+}
+
+JSValueType CacheRegisterAllocator::knownType(ValOperandId val) const {
+ const OperandLocation& loc = operandLocations_[val.id()];
+
+ switch (loc.kind()) {
+ case OperandLocation::ValueReg:
+ case OperandLocation::ValueStack:
+ case OperandLocation::BaselineFrame:
+ return JSVAL_TYPE_UNKNOWN;
+
+ case OperandLocation::PayloadStack:
+ case OperandLocation::PayloadReg:
+ return loc.payloadType();
+
+ case OperandLocation::Constant:
+ return loc.constant().isDouble() ? JSVAL_TYPE_DOUBLE
+ : loc.constant().extractNonDoubleType();
+
+ case OperandLocation::DoubleReg:
+ return JSVAL_TYPE_DOUBLE;
+
+ case OperandLocation::Uninitialized:
+ break;
+ }
+
+ MOZ_CRASH("Invalid kind");
+}
+
+void CacheRegisterAllocator::initInputLocation(
+ size_t i, const TypedOrValueRegister& reg) {
+ if (reg.hasValue()) {
+ initInputLocation(i, reg.valueReg());
+ } else if (reg.typedReg().isFloat()) {
+ MOZ_ASSERT(reg.type() == MIRType::Double);
+ initInputLocation(i, reg.typedReg().fpu());
+ } else {
+ initInputLocation(i, reg.typedReg().gpr(),
+ ValueTypeFromMIRType(reg.type()));
+ }
+}
+
+void CacheRegisterAllocator::initInputLocation(
+ size_t i, const ConstantOrRegister& value) {
+ if (value.constant()) {
+ initInputLocation(i, value.value());
+ } else {
+ initInputLocation(i, value.reg());
+ }
+}
+
+void CacheRegisterAllocator::spillOperandToStack(MacroAssembler& masm,
+ OperandLocation* loc) {
+ MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end());
+
+ if (loc->kind() == OperandLocation::ValueReg) {
+ if (!freeValueSlots_.empty()) {
+ uint32_t stackPos = freeValueSlots_.popCopy();
+ MOZ_ASSERT(stackPos <= stackPushed_);
+ masm.storeValue(loc->valueReg(),
+ Address(masm.getStackPointer(), stackPushed_ - stackPos));
+ loc->setValueStack(stackPos);
+ return;
+ }
+ stackPushed_ += sizeof(js::Value);
+ masm.pushValue(loc->valueReg());
+ loc->setValueStack(stackPushed_);
+ return;
+ }
+
+ MOZ_ASSERT(loc->kind() == OperandLocation::PayloadReg);
+
+ if (!freePayloadSlots_.empty()) {
+ uint32_t stackPos = freePayloadSlots_.popCopy();
+ MOZ_ASSERT(stackPos <= stackPushed_);
+ masm.storePtr(loc->payloadReg(),
+ Address(masm.getStackPointer(), stackPushed_ - stackPos));
+ loc->setPayloadStack(stackPos, loc->payloadType());
+ return;
+ }
+ stackPushed_ += sizeof(uintptr_t);
+ masm.push(loc->payloadReg());
+ loc->setPayloadStack(stackPushed_, loc->payloadType());
+}
+
+void CacheRegisterAllocator::spillOperandToStackOrRegister(
+ MacroAssembler& masm, OperandLocation* loc) {
+ MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end());
+
+ // If enough registers are available, use them.
+ if (loc->kind() == OperandLocation::ValueReg) {
+ static const size_t BoxPieces = sizeof(Value) / sizeof(uintptr_t);
+ if (availableRegs_.set().size() >= BoxPieces) {
+ ValueOperand reg = availableRegs_.takeAnyValue();
+ masm.moveValue(loc->valueReg(), reg);
+ loc->setValueReg(reg);
+ return;
+ }
+ } else {
+ MOZ_ASSERT(loc->kind() == OperandLocation::PayloadReg);
+ if (!availableRegs_.empty()) {
+ Register reg = availableRegs_.takeAny();
+ masm.movePtr(loc->payloadReg(), reg);
+ loc->setPayloadReg(reg, loc->payloadType());
+ return;
+ }
+ }
+
+ // Not enough registers available, spill to the stack.
+ spillOperandToStack(masm, loc);
+}
+
+void CacheRegisterAllocator::popPayload(MacroAssembler& masm,
+ OperandLocation* loc, Register dest) {
+ MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end());
+ MOZ_ASSERT(stackPushed_ >= sizeof(uintptr_t));
+
+ // The payload is on the stack. If it's on top of the stack we can just
+ // pop it, else we emit a load.
+ if (loc->payloadStack() == stackPushed_) {
+ masm.pop(dest);
+ stackPushed_ -= sizeof(uintptr_t);
+ } else {
+ MOZ_ASSERT(loc->payloadStack() < stackPushed_);
+ masm.loadPtr(payloadAddress(masm, loc), dest);
+ masm.propagateOOM(freePayloadSlots_.append(loc->payloadStack()));
+ }
+
+ loc->setPayloadReg(dest, loc->payloadType());
+}
+
+Address CacheRegisterAllocator::valueAddress(MacroAssembler& masm,
+ const OperandLocation* loc) const {
+ MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end());
+ return Address(masm.getStackPointer(), stackPushed_ - loc->valueStack());
+}
+
+Address CacheRegisterAllocator::payloadAddress(
+ MacroAssembler& masm, const OperandLocation* loc) const {
+ MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end());
+ return Address(masm.getStackPointer(), stackPushed_ - loc->payloadStack());
+}
+
+void CacheRegisterAllocator::popValue(MacroAssembler& masm,
+ OperandLocation* loc, ValueOperand dest) {
+ MOZ_ASSERT(loc >= operandLocations_.begin() && loc < operandLocations_.end());
+ MOZ_ASSERT(stackPushed_ >= sizeof(js::Value));
+
+ // The Value is on the stack. If it's on top of the stack we can just
+ // pop it, else we emit a load.
+ if (loc->valueStack() == stackPushed_) {
+ masm.popValue(dest);
+ stackPushed_ -= sizeof(js::Value);
+ } else {
+ MOZ_ASSERT(loc->valueStack() < stackPushed_);
+ masm.loadValue(
+ Address(masm.getStackPointer(), stackPushed_ - loc->valueStack()),
+ dest);
+ masm.propagateOOM(freeValueSlots_.append(loc->valueStack()));
+ }
+
+ loc->setValueReg(dest);
+}
+
+#ifdef DEBUG
+void CacheRegisterAllocator::assertValidState() const {
+ // Assert different operands don't have aliasing storage. We depend on this
+ // when spilling registers, for instance.
+
+ if (!JitOptions.fullDebugChecks) {
+ return;
+ }
+
+ for (size_t i = 0; i < operandLocations_.length(); i++) {
+ const auto& loc1 = operandLocations_[i];
+ if (loc1.isUninitialized()) {
+ continue;
+ }
+
+ for (size_t j = 0; j < i; j++) {
+ const auto& loc2 = operandLocations_[j];
+ if (loc2.isUninitialized()) {
+ continue;
+ }
+ MOZ_ASSERT(!loc1.aliasesReg(loc2));
+ }
+ }
+}
+#endif
+
+bool OperandLocation::aliasesReg(const OperandLocation& other) const {
+ MOZ_ASSERT(&other != this);
+
+ switch (other.kind_) {
+ case PayloadReg:
+ return aliasesReg(other.payloadReg());
+ case ValueReg:
+ return aliasesReg(other.valueReg());
+ case PayloadStack:
+ case ValueStack:
+ case BaselineFrame:
+ case Constant:
+ case DoubleReg:
+ return false;
+ case Uninitialized:
+ break;
+ }
+
+ MOZ_CRASH("Invalid kind");
+}
+
+void CacheRegisterAllocator::restoreInputState(MacroAssembler& masm,
+ bool shouldDiscardStack) {
+ size_t numInputOperands = origInputLocations_.length();
+ MOZ_ASSERT(writer_.numInputOperands() == numInputOperands);
+
+ for (size_t j = 0; j < numInputOperands; j++) {
+ const OperandLocation& dest = origInputLocations_[j];
+ OperandLocation& cur = operandLocations_[j];
+ if (dest == cur) {
+ continue;
+ }
+
+ auto autoAssign = mozilla::MakeScopeExit([&] { cur = dest; });
+
+ // We have a cycle if a destination register will be used later
+ // as source register. If that happens, just push the current value
+ // on the stack and later get it from there.
+ for (size_t k = j + 1; k < numInputOperands; k++) {
+ OperandLocation& laterSource = operandLocations_[k];
+ if (dest.aliasesReg(laterSource)) {
+ spillOperandToStack(masm, &laterSource);
+ }
+ }
+
+ if (dest.kind() == OperandLocation::ValueReg) {
+ // We have to restore a Value register.
+ switch (cur.kind()) {
+ case OperandLocation::ValueReg:
+ masm.moveValue(cur.valueReg(), dest.valueReg());
+ continue;
+ case OperandLocation::PayloadReg:
+ masm.tagValue(cur.payloadType(), cur.payloadReg(), dest.valueReg());
+ continue;
+ case OperandLocation::PayloadStack: {
+ Register scratch = dest.valueReg().scratchReg();
+ popPayload(masm, &cur, scratch);
+ masm.tagValue(cur.payloadType(), scratch, dest.valueReg());
+ continue;
+ }
+ case OperandLocation::ValueStack:
+ popValue(masm, &cur, dest.valueReg());
+ continue;
+ case OperandLocation::DoubleReg:
+ masm.boxDouble(cur.doubleReg(), dest.valueReg(), cur.doubleReg());
+ continue;
+ case OperandLocation::Constant:
+ case OperandLocation::BaselineFrame:
+ case OperandLocation::Uninitialized:
+ break;
+ }
+ } else if (dest.kind() == OperandLocation::PayloadReg) {
+ // We have to restore a payload register.
+ switch (cur.kind()) {
+ case OperandLocation::ValueReg:
+ MOZ_ASSERT(dest.payloadType() != JSVAL_TYPE_DOUBLE);
+ masm.unboxNonDouble(cur.valueReg(), dest.payloadReg(),
+ dest.payloadType());
+ continue;
+ case OperandLocation::PayloadReg:
+ MOZ_ASSERT(cur.payloadType() == dest.payloadType());
+ masm.mov(cur.payloadReg(), dest.payloadReg());
+ continue;
+ case OperandLocation::PayloadStack: {
+ MOZ_ASSERT(cur.payloadType() == dest.payloadType());
+ popPayload(masm, &cur, dest.payloadReg());
+ continue;
+ }
+ case OperandLocation::ValueStack:
+ MOZ_ASSERT(stackPushed_ >= sizeof(js::Value));
+ MOZ_ASSERT(cur.valueStack() <= stackPushed_);
+ MOZ_ASSERT(dest.payloadType() != JSVAL_TYPE_DOUBLE);
+ masm.unboxNonDouble(
+ Address(masm.getStackPointer(), stackPushed_ - cur.valueStack()),
+ dest.payloadReg(), dest.payloadType());
+ continue;
+ case OperandLocation::Constant:
+ case OperandLocation::BaselineFrame:
+ case OperandLocation::DoubleReg:
+ case OperandLocation::Uninitialized:
+ break;
+ }
+ } else if (dest.kind() == OperandLocation::Constant ||
+ dest.kind() == OperandLocation::BaselineFrame ||
+ dest.kind() == OperandLocation::DoubleReg) {
+ // Nothing to do.
+ continue;
+ }
+
+ MOZ_CRASH("Invalid kind");
+ }
+
+ for (const SpilledRegister& spill : spilledRegs_) {
+ MOZ_ASSERT(stackPushed_ >= sizeof(uintptr_t));
+
+ if (spill.stackPushed == stackPushed_) {
+ masm.pop(spill.reg);
+ stackPushed_ -= sizeof(uintptr_t);
+ } else {
+ MOZ_ASSERT(spill.stackPushed < stackPushed_);
+ masm.loadPtr(
+ Address(masm.getStackPointer(), stackPushed_ - spill.stackPushed),
+ spill.reg);
+ }
+ }
+
+ if (shouldDiscardStack) {
+ discardStack(masm);
+ }
+}
+
+size_t CacheIRStubInfo::stubDataSize() const {
+ size_t field = 0;
+ size_t size = 0;
+ while (true) {
+ StubField::Type type = fieldType(field++);
+ if (type == StubField::Type::Limit) {
+ return size;
+ }
+ size += StubField::sizeInBytes(type);
+ }
+}
+
+template <typename T>
+static GCPtr<T>* AsGCPtr(uintptr_t* ptr) {
+ return reinterpret_cast<GCPtr<T>*>(ptr);
+}
+
+uintptr_t CacheIRStubInfo::getStubRawWord(const uint8_t* stubData,
+ uint32_t offset) const {
+ MOZ_ASSERT(uintptr_t(stubData + offset) % sizeof(uintptr_t) == 0);
+ return *reinterpret_cast<const uintptr_t*>(stubData + offset);
+}
+
+uintptr_t CacheIRStubInfo::getStubRawWord(ICCacheIRStub* stub,
+ uint32_t offset) const {
+ uint8_t* stubData = (uint8_t*)stub + stubDataOffset_;
+ return getStubRawWord(stubData, offset);
+}
+
+int64_t CacheIRStubInfo::getStubRawInt64(const uint8_t* stubData,
+ uint32_t offset) const {
+ MOZ_ASSERT(uintptr_t(stubData + offset) % sizeof(int64_t) == 0);
+ return *reinterpret_cast<const int64_t*>(stubData + offset);
+}
+
+int64_t CacheIRStubInfo::getStubRawInt64(ICCacheIRStub* stub,
+ uint32_t offset) const {
+ uint8_t* stubData = (uint8_t*)stub + stubDataOffset_;
+ return getStubRawInt64(stubData, offset);
+}
+
+void CacheIRStubInfo::replaceStubRawWord(uint8_t* stubData, uint32_t offset,
+ uintptr_t oldWord,
+ uintptr_t newWord) const {
+ MOZ_ASSERT(uintptr_t(stubData + offset) % sizeof(uintptr_t) == 0);
+ uintptr_t* addr = reinterpret_cast<uintptr_t*>(stubData + offset);
+ MOZ_ASSERT(*addr == oldWord);
+ *addr = newWord;
+}
+
+template <class Stub, class T>
+GCPtr<T>& CacheIRStubInfo::getStubField(Stub* stub, uint32_t offset) const {
+ uint8_t* stubData = (uint8_t*)stub + stubDataOffset_;
+ MOZ_ASSERT(uintptr_t(stubData + offset) % sizeof(uintptr_t) == 0);
+
+ return *AsGCPtr<T>((uintptr_t*)(stubData + offset));
+}
+
+template GCPtr<Shape*>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<GetterSetter*>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<JSObject*>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<JSString*>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<JSFunction*>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<JS::Symbol*>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<JS::Value>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<jsid>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<JSClass*>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+template GCPtr<ArrayObject*>& CacheIRStubInfo::getStubField<ICCacheIRStub>(
+ ICCacheIRStub* stub, uint32_t offset) const;
+
+template <class Stub, class T>
+T* CacheIRStubInfo::getPtrStubField(Stub* stub, uint32_t offset) const {
+ uint8_t* stubData = (uint8_t*)stub + stubDataOffset_;
+ MOZ_ASSERT(uintptr_t(stubData + offset) % sizeof(uintptr_t) == 0);
+
+ return *reinterpret_cast<T**>(stubData + offset);
+}
+
+template gc::AllocSite* CacheIRStubInfo::getPtrStubField(ICCacheIRStub* stub,
+ uint32_t offset) const;
+
+template <typename T, typename V>
+static void InitGCPtr(uintptr_t* ptr, V val) {
+ AsGCPtr<T>(ptr)->init(mozilla::BitwiseCast<T>(val));
+}
+
+void CacheIRWriter::copyStubData(uint8_t* dest) const {
+ MOZ_ASSERT(!failed());
+
+ uintptr_t* destWords = reinterpret_cast<uintptr_t*>(dest);
+
+ for (const StubField& field : stubFields_) {
+ MOZ_ASSERT((uintptr_t(destWords) % field.sizeInBytes()) == 0,
+ "Unaligned stub field");
+
+ switch (field.type()) {
+ case StubField::Type::RawInt32:
+ case StubField::Type::RawPointer:
+ case StubField::Type::AllocSite:
+ *destWords = field.asWord();
+ break;
+ case StubField::Type::Shape:
+ InitGCPtr<Shape*>(destWords, field.asWord());
+ break;
+ case StubField::Type::GetterSetter:
+ InitGCPtr<GetterSetter*>(destWords, field.asWord());
+ break;
+ case StubField::Type::JSObject:
+ InitGCPtr<JSObject*>(destWords, field.asWord());
+ break;
+ case StubField::Type::Symbol:
+ InitGCPtr<JS::Symbol*>(destWords, field.asWord());
+ break;
+ case StubField::Type::String:
+ InitGCPtr<JSString*>(destWords, field.asWord());
+ break;
+ case StubField::Type::BaseScript:
+ InitGCPtr<BaseScript*>(destWords, field.asWord());
+ break;
+ case StubField::Type::Id:
+ AsGCPtr<jsid>(destWords)->init(jsid::fromRawBits(field.asWord()));
+ break;
+ case StubField::Type::RawInt64:
+ case StubField::Type::Double:
+ *reinterpret_cast<uint64_t*>(destWords) = field.asInt64();
+ break;
+ case StubField::Type::Value:
+ AsGCPtr<Value>(destWords)->init(
+ Value::fromRawBits(uint64_t(field.asInt64())));
+ break;
+ case StubField::Type::Limit:
+ MOZ_CRASH("Invalid type");
+ }
+ destWords += StubField::sizeInBytes(field.type()) / sizeof(uintptr_t);
+ }
+}
+
+template <typename T>
+void jit::TraceCacheIRStub(JSTracer* trc, T* stub,
+ const CacheIRStubInfo* stubInfo) {
+ uint32_t field = 0;
+ size_t offset = 0;
+ while (true) {
+ StubField::Type fieldType = stubInfo->fieldType(field);
+ switch (fieldType) {
+ case StubField::Type::RawInt32:
+ case StubField::Type::RawPointer:
+ case StubField::Type::RawInt64:
+ case StubField::Type::Double:
+ break;
+ case StubField::Type::Shape: {
+ // For CCW IC stubs, we can store same-zone but cross-compartment
+ // shapes. Use TraceSameZoneCrossCompartmentEdge to not assert in the
+ // GC. Note: CacheIRWriter::writeShapeField asserts we never store
+ // cross-zone shapes.
+ GCPtr<Shape*>& shapeField =
+ stubInfo->getStubField<T, Shape*>(stub, offset);
+ TraceSameZoneCrossCompartmentEdge(trc, &shapeField, "cacheir-shape");
+ break;
+ }
+ case StubField::Type::GetterSetter:
+ TraceEdge(trc, &stubInfo->getStubField<T, GetterSetter*>(stub, offset),
+ "cacheir-getter-setter");
+ break;
+ case StubField::Type::JSObject:
+ TraceEdge(trc, &stubInfo->getStubField<T, JSObject*>(stub, offset),
+ "cacheir-object");
+ break;
+ case StubField::Type::Symbol:
+ TraceEdge(trc, &stubInfo->getStubField<T, JS::Symbol*>(stub, offset),
+ "cacheir-symbol");
+ break;
+ case StubField::Type::String:
+ TraceEdge(trc, &stubInfo->getStubField<T, JSString*>(stub, offset),
+ "cacheir-string");
+ break;
+ case StubField::Type::BaseScript:
+ TraceEdge(trc, &stubInfo->getStubField<T, BaseScript*>(stub, offset),
+ "cacheir-script");
+ break;
+ case StubField::Type::Id:
+ TraceEdge(trc, &stubInfo->getStubField<T, jsid>(stub, offset),
+ "cacheir-id");
+ break;
+ case StubField::Type::Value:
+ TraceEdge(trc, &stubInfo->getStubField<T, JS::Value>(stub, offset),
+ "cacheir-value");
+ break;
+ case StubField::Type::AllocSite: {
+ gc::AllocSite* site =
+ stubInfo->getPtrStubField<T, gc::AllocSite>(stub, offset);
+ site->trace(trc);
+ break;
+ }
+ case StubField::Type::Limit:
+ return; // Done.
+ }
+ field++;
+ offset += StubField::sizeInBytes(fieldType);
+ }
+}
+
+template void jit::TraceCacheIRStub(JSTracer* trc, ICCacheIRStub* stub,
+ const CacheIRStubInfo* stubInfo);
+
+template void jit::TraceCacheIRStub(JSTracer* trc, IonICStub* stub,
+ const CacheIRStubInfo* stubInfo);
+
+bool CacheIRWriter::stubDataEquals(const uint8_t* stubData) const {
+ MOZ_ASSERT(!failed());
+
+ const uintptr_t* stubDataWords = reinterpret_cast<const uintptr_t*>(stubData);
+
+ for (const StubField& field : stubFields_) {
+ if (field.sizeIsWord()) {
+ if (field.asWord() != *stubDataWords) {
+ return false;
+ }
+ stubDataWords++;
+ continue;
+ }
+
+ if (field.asInt64() != *reinterpret_cast<const uint64_t*>(stubDataWords)) {
+ return false;
+ }
+ stubDataWords += sizeof(uint64_t) / sizeof(uintptr_t);
+ }
+
+ return true;
+}
+
+bool CacheIRWriter::stubDataEqualsIgnoring(const uint8_t* stubData,
+ uint32_t ignoreOffset) const {
+ MOZ_ASSERT(!failed());
+
+ uint32_t offset = 0;
+ for (const StubField& field : stubFields_) {
+ if (offset != ignoreOffset) {
+ if (field.sizeIsWord()) {
+ uintptr_t raw = *reinterpret_cast<const uintptr_t*>(stubData + offset);
+ if (field.asWord() != raw) {
+ return false;
+ }
+ } else {
+ uint64_t raw = *reinterpret_cast<const uint64_t*>(stubData + offset);
+ if (field.asInt64() != raw) {
+ return false;
+ }
+ }
+ }
+ offset += StubField::sizeInBytes(field.type());
+ }
+
+ return true;
+}
+
+HashNumber CacheIRStubKey::hash(const CacheIRStubKey::Lookup& l) {
+ HashNumber hash = mozilla::HashBytes(l.code, l.length);
+ hash = mozilla::AddToHash(hash, uint32_t(l.kind));
+ hash = mozilla::AddToHash(hash, uint32_t(l.engine));
+ return hash;
+}
+
+bool CacheIRStubKey::match(const CacheIRStubKey& entry,
+ const CacheIRStubKey::Lookup& l) {
+ if (entry.stubInfo->kind() != l.kind) {
+ return false;
+ }
+
+ if (entry.stubInfo->engine() != l.engine) {
+ return false;
+ }
+
+ if (entry.stubInfo->codeLength() != l.length) {
+ return false;
+ }
+
+ if (!mozilla::ArrayEqual(entry.stubInfo->code(), l.code, l.length)) {
+ return false;
+ }
+
+ return true;
+}
+
+CacheIRReader::CacheIRReader(const CacheIRStubInfo* stubInfo)
+ : CacheIRReader(stubInfo->code(),
+ stubInfo->code() + stubInfo->codeLength()) {}
+
+CacheIRStubInfo* CacheIRStubInfo::New(CacheKind kind, ICStubEngine engine,
+ bool makesGCCalls,
+ uint32_t stubDataOffset,
+ const CacheIRWriter& writer) {
+ size_t numStubFields = writer.numStubFields();
+ size_t bytesNeeded =
+ sizeof(CacheIRStubInfo) + writer.codeLength() +
+ (numStubFields + 1); // +1 for the GCType::Limit terminator.
+ uint8_t* p = js_pod_malloc<uint8_t>(bytesNeeded);
+ if (!p) {
+ return nullptr;
+ }
+
+ // Copy the CacheIR code.
+ uint8_t* codeStart = p + sizeof(CacheIRStubInfo);
+ mozilla::PodCopy(codeStart, writer.codeStart(), writer.codeLength());
+
+ static_assert(sizeof(StubField::Type) == sizeof(uint8_t),
+ "StubField::Type must fit in uint8_t");
+
+ // Copy the stub field types.
+ uint8_t* fieldTypes = codeStart + writer.codeLength();
+ for (size_t i = 0; i < numStubFields; i++) {
+ fieldTypes[i] = uint8_t(writer.stubFieldType(i));
+ }
+ fieldTypes[numStubFields] = uint8_t(StubField::Type::Limit);
+
+ return new (p) CacheIRStubInfo(kind, engine, makesGCCalls, stubDataOffset,
+ codeStart, writer.codeLength(), fieldTypes);
+}
+
+bool OperandLocation::operator==(const OperandLocation& other) const {
+ if (kind_ != other.kind_) {
+ return false;
+ }
+
+ switch (kind()) {
+ case Uninitialized:
+ return true;
+ case PayloadReg:
+ return payloadReg() == other.payloadReg() &&
+ payloadType() == other.payloadType();
+ case ValueReg:
+ return valueReg() == other.valueReg();
+ case PayloadStack:
+ return payloadStack() == other.payloadStack() &&
+ payloadType() == other.payloadType();
+ case ValueStack:
+ return valueStack() == other.valueStack();
+ case BaselineFrame:
+ return baselineFrameSlot() == other.baselineFrameSlot();
+ case Constant:
+ return constant() == other.constant();
+ case DoubleReg:
+ return doubleReg() == other.doubleReg();
+ }
+
+ MOZ_CRASH("Invalid OperandLocation kind");
+}
+
+AutoOutputRegister::AutoOutputRegister(CacheIRCompiler& compiler)
+ : output_(compiler.outputUnchecked_.ref()), alloc_(compiler.allocator) {
+ if (output_.hasValue()) {
+ alloc_.allocateFixedValueRegister(compiler.masm, output_.valueReg());
+ } else if (!output_.typedReg().isFloat()) {
+ alloc_.allocateFixedRegister(compiler.masm, output_.typedReg().gpr());
+ }
+}
+
+AutoOutputRegister::~AutoOutputRegister() {
+ if (output_.hasValue()) {
+ alloc_.releaseValueRegister(output_.valueReg());
+ } else if (!output_.typedReg().isFloat()) {
+ alloc_.releaseRegister(output_.typedReg().gpr());
+ }
+}
+
+bool FailurePath::canShareFailurePath(const FailurePath& other) const {
+ if (stackPushed_ != other.stackPushed_) {
+ return false;
+ }
+
+ if (spilledRegs_.length() != other.spilledRegs_.length()) {
+ return false;
+ }
+
+ for (size_t i = 0; i < spilledRegs_.length(); i++) {
+ if (spilledRegs_[i] != other.spilledRegs_[i]) {
+ return false;
+ }
+ }
+
+ MOZ_ASSERT(inputs_.length() == other.inputs_.length());
+
+ for (size_t i = 0; i < inputs_.length(); i++) {
+ if (inputs_[i] != other.inputs_[i]) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool CacheIRCompiler::addFailurePath(FailurePath** failure) {
+#ifdef DEBUG
+ allocator.setAddedFailurePath();
+#endif
+ MOZ_ASSERT(!allocator.hasAutoScratchFloatRegisterSpill());
+
+ FailurePath newFailure;
+ for (size_t i = 0; i < writer_.numInputOperands(); i++) {
+ if (!newFailure.appendInput(allocator.operandLocation(i))) {
+ return false;
+ }
+ }
+ if (!newFailure.setSpilledRegs(allocator.spilledRegs())) {
+ return false;
+ }
+ newFailure.setStackPushed(allocator.stackPushed());
+
+ // Reuse the previous failure path if the current one is the same, to
+ // avoid emitting duplicate code.
+ if (failurePaths.length() > 0 &&
+ failurePaths.back().canShareFailurePath(newFailure)) {
+ *failure = &failurePaths.back();
+ return true;
+ }
+
+ if (!failurePaths.append(std::move(newFailure))) {
+ return false;
+ }
+
+ *failure = &failurePaths.back();
+ return true;
+}
+
+bool CacheIRCompiler::emitFailurePath(size_t index) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ FailurePath& failure = failurePaths[index];
+
+ allocator.setStackPushed(failure.stackPushed());
+
+ for (size_t i = 0; i < writer_.numInputOperands(); i++) {
+ allocator.setOperandLocation(i, failure.input(i));
+ }
+
+ if (!allocator.setSpilledRegs(failure.spilledRegs())) {
+ return false;
+ }
+
+ masm.bind(failure.label());
+ allocator.restoreInputState(masm);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsNumber(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ JSValueType knownType = allocator.knownType(inputId);
+
+ // Doubles and ints are numbers!
+ if (knownType == JSVAL_TYPE_DOUBLE || knownType == JSVAL_TYPE_INT32) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestNumber(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToObject(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ if (allocator.knownType(inputId) == JSVAL_TYPE_OBJECT) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ masm.branchTestObject(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsNullOrUndefined(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ JSValueType knownType = allocator.knownType(inputId);
+ if (knownType == JSVAL_TYPE_UNDEFINED || knownType == JSVAL_TYPE_NULL) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label success;
+ masm.branchTestNull(Assembler::Equal, input, &success);
+ masm.branchTestUndefined(Assembler::NotEqual, input, failure->label());
+
+ masm.bind(&success);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsNull(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ JSValueType knownType = allocator.knownType(inputId);
+ if (knownType == JSVAL_TYPE_NULL) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestNull(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsUndefined(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ JSValueType knownType = allocator.knownType(inputId);
+ if (knownType == JSVAL_TYPE_UNDEFINED) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestUndefined(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsNotUninitializedLexical(ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestMagicValue(Assembler::Equal, val, JS_UNINITIALIZED_LEXICAL,
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardBooleanToInt32(ValOperandId inputId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register output = allocator.defineRegister(masm, resultId);
+
+ if (allocator.knownType(inputId) == JSVAL_TYPE_BOOLEAN) {
+ Register input =
+ allocator.useRegister(masm, BooleanOperandId(inputId.id()));
+ masm.move32(input, output);
+ return true;
+ }
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.fallibleUnboxBoolean(input, output, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToString(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ if (allocator.knownType(inputId) == JSVAL_TYPE_STRING) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ masm.branchTestString(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToSymbol(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ if (allocator.knownType(inputId) == JSVAL_TYPE_SYMBOL) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ masm.branchTestSymbol(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToBigInt(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ if (allocator.knownType(inputId) == JSVAL_TYPE_BIGINT) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ masm.branchTestBigInt(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToBoolean(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (allocator.knownType(inputId) == JSVAL_TYPE_BOOLEAN) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ masm.branchTestBoolean(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToInt32(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (allocator.knownType(inputId) == JSVAL_TYPE_INT32) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestInt32(Assembler::NotEqual, input, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToNonGCThing(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestGCThing(Assembler::Equal, input, failure->label());
+ return true;
+}
+
+// Infallible |emitDouble| emitters can use this implementation to avoid
+// generating extra clean-up instructions to restore the scratch float register.
+// To select this function simply omit the |Label* fail| parameter for the
+// emitter lambda function.
+template <typename EmitDouble>
+static std::enable_if_t<mozilla::FunctionTypeTraits<EmitDouble>::arity == 1,
+ void>
+EmitGuardDouble(CacheIRCompiler* compiler, MacroAssembler& masm,
+ ValueOperand input, FailurePath* failure,
+ EmitDouble emitDouble) {
+ AutoScratchFloatRegister floatReg(compiler);
+
+ masm.unboxDouble(input, floatReg);
+ emitDouble(floatReg.get());
+}
+
+template <typename EmitDouble>
+static std::enable_if_t<mozilla::FunctionTypeTraits<EmitDouble>::arity == 2,
+ void>
+EmitGuardDouble(CacheIRCompiler* compiler, MacroAssembler& masm,
+ ValueOperand input, FailurePath* failure,
+ EmitDouble emitDouble) {
+ AutoScratchFloatRegister floatReg(compiler, failure);
+
+ masm.unboxDouble(input, floatReg);
+ emitDouble(floatReg.get(), floatReg.failure());
+}
+
+template <typename EmitInt32, typename EmitDouble>
+static void EmitGuardInt32OrDouble(CacheIRCompiler* compiler,
+ MacroAssembler& masm, ValueOperand input,
+ Register output, FailurePath* failure,
+ EmitInt32 emitInt32, EmitDouble emitDouble) {
+ Label done;
+
+ {
+ ScratchTagScope tag(masm, input);
+ masm.splitTagForTest(input, tag);
+
+ Label notInt32;
+ masm.branchTestInt32(Assembler::NotEqual, tag, &notInt32);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ masm.unboxInt32(input, output);
+ emitInt32();
+
+ masm.jump(&done);
+ }
+ masm.bind(&notInt32);
+
+ masm.branchTestDouble(Assembler::NotEqual, tag, failure->label());
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ EmitGuardDouble(compiler, masm, input, failure, emitDouble);
+ }
+ }
+
+ masm.bind(&done);
+}
+
+bool CacheIRCompiler::emitGuardToInt32Index(ValOperandId inputId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register output = allocator.defineRegister(masm, resultId);
+
+ if (allocator.knownType(inputId) == JSVAL_TYPE_INT32) {
+ Register input = allocator.useRegister(masm, Int32OperandId(inputId.id()));
+ masm.move32(input, output);
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ EmitGuardInt32OrDouble(
+ this, masm, input, output, failure,
+ []() {
+ // No-op if the value is already an int32.
+ },
+ [&](FloatRegister floatReg, Label* fail) {
+ // ToPropertyKey(-0.0) is "0", so we can truncate -0.0 to 0 here.
+ masm.convertDoubleToInt32(floatReg, output, fail, false);
+ });
+
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32ToIntPtr(Int32OperandId inputId,
+ IntPtrOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register input = allocator.useRegister(masm, inputId);
+ Register output = allocator.defineRegister(masm, resultId);
+
+ masm.move32SignExtendToPtr(input, output);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardNumberToIntPtrIndex(NumberOperandId inputId,
+ bool supportOOB,
+ IntPtrOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register output = allocator.defineRegister(masm, resultId);
+
+ FailurePath* failure = nullptr;
+ if (!supportOOB) {
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ }
+
+ AutoScratchFloatRegister floatReg(this, failure);
+ allocator.ensureDoubleRegister(masm, inputId, floatReg);
+
+ // ToPropertyKey(-0.0) is "0", so we can truncate -0.0 to 0 here.
+ if (supportOOB) {
+ Label done, fail;
+ masm.convertDoubleToPtr(floatReg, output, &fail, false);
+ masm.jump(&done);
+
+ // Substitute the invalid index with an arbitrary out-of-bounds index.
+ masm.bind(&fail);
+ masm.movePtr(ImmWord(-1), output);
+
+ masm.bind(&done);
+ } else {
+ masm.convertDoubleToPtr(floatReg, output, floatReg.failure(), false);
+ }
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToInt32ModUint32(ValOperandId inputId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register output = allocator.defineRegister(masm, resultId);
+
+ if (allocator.knownType(inputId) == JSVAL_TYPE_INT32) {
+ ConstantOrRegister input = allocator.useConstantOrRegister(masm, inputId);
+ if (input.constant()) {
+ masm.move32(Imm32(input.value().toInt32()), output);
+ } else {
+ MOZ_ASSERT(input.reg().type() == MIRType::Int32);
+ masm.move32(input.reg().typedReg().gpr(), output);
+ }
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ EmitGuardInt32OrDouble(
+ this, masm, input, output, failure,
+ []() {
+ // No-op if the value is already an int32.
+ },
+ [&](FloatRegister floatReg, Label* fail) {
+ masm.branchTruncateDoubleMaybeModUint32(floatReg, output, fail);
+ });
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardToUint8Clamped(ValOperandId inputId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register output = allocator.defineRegister(masm, resultId);
+
+ if (allocator.knownType(inputId) == JSVAL_TYPE_INT32) {
+ ConstantOrRegister input = allocator.useConstantOrRegister(masm, inputId);
+ if (input.constant()) {
+ masm.move32(Imm32(ClampDoubleToUint8(input.value().toInt32())), output);
+ } else {
+ MOZ_ASSERT(input.reg().type() == MIRType::Int32);
+ masm.move32(input.reg().typedReg().gpr(), output);
+ masm.clampIntToUint8(output);
+ }
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ EmitGuardInt32OrDouble(
+ this, masm, input, output, failure,
+ [&]() {
+ // |output| holds the unboxed int32 value.
+ masm.clampIntToUint8(output);
+ },
+ [&](FloatRegister floatReg) {
+ masm.clampDoubleToUint8(floatReg, output);
+ });
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardNonDoubleType(ValOperandId inputId,
+ ValueType type) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (allocator.knownType(inputId) == JSValueType(type)) {
+ return true;
+ }
+
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ switch (type) {
+ case ValueType::String:
+ masm.branchTestString(Assembler::NotEqual, input, failure->label());
+ break;
+ case ValueType::Symbol:
+ masm.branchTestSymbol(Assembler::NotEqual, input, failure->label());
+ break;
+ case ValueType::BigInt:
+ masm.branchTestBigInt(Assembler::NotEqual, input, failure->label());
+ break;
+ case ValueType::Int32:
+ masm.branchTestInt32(Assembler::NotEqual, input, failure->label());
+ break;
+ case ValueType::Boolean:
+ masm.branchTestBoolean(Assembler::NotEqual, input, failure->label());
+ break;
+ case ValueType::Undefined:
+ masm.branchTestUndefined(Assembler::NotEqual, input, failure->label());
+ break;
+ case ValueType::Null:
+ masm.branchTestNull(Assembler::NotEqual, input, failure->label());
+ break;
+ case ValueType::Double:
+ case ValueType::Magic:
+ case ValueType::PrivateGCThing:
+ case ValueType::Object:
+#ifdef ENABLE_RECORD_TUPLE
+ case ValueType::ExtendedPrimitive:
+#endif
+ MOZ_CRASH("unexpected type");
+ }
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardClass(ObjOperandId objId, GuardClassKind kind) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ if (kind == GuardClassKind::JSFunction) {
+ if (objectGuardNeedsSpectreMitigations(objId)) {
+ masm.branchTestObjIsFunction(Assembler::NotEqual, obj, scratch, obj,
+ failure->label());
+ } else {
+ masm.branchTestObjIsFunctionNoSpectreMitigations(
+ Assembler::NotEqual, obj, scratch, failure->label());
+ }
+ return true;
+ }
+
+ const JSClass* clasp = nullptr;
+ switch (kind) {
+ case GuardClassKind::Array:
+ clasp = &ArrayObject::class_;
+ break;
+ case GuardClassKind::PlainObject:
+ clasp = &PlainObject::class_;
+ break;
+ case GuardClassKind::ArrayBuffer:
+ clasp = &ArrayBufferObject::class_;
+ break;
+ case GuardClassKind::SharedArrayBuffer:
+ clasp = &SharedArrayBufferObject::class_;
+ break;
+ case GuardClassKind::DataView:
+ clasp = &DataViewObject::class_;
+ break;
+ case GuardClassKind::MappedArguments:
+ clasp = &MappedArgumentsObject::class_;
+ break;
+ case GuardClassKind::UnmappedArguments:
+ clasp = &UnmappedArgumentsObject::class_;
+ break;
+ case GuardClassKind::WindowProxy:
+ clasp = cx_->runtime()->maybeWindowProxyClass();
+ break;
+ case GuardClassKind::Set:
+ clasp = &SetObject::class_;
+ break;
+ case GuardClassKind::Map:
+ clasp = &MapObject::class_;
+ break;
+ case GuardClassKind::JSFunction:
+ MOZ_CRASH("JSFunction handled before switch");
+ }
+ MOZ_ASSERT(clasp);
+
+ if (objectGuardNeedsSpectreMitigations(objId)) {
+ masm.branchTestObjClass(Assembler::NotEqual, obj, clasp, scratch, obj,
+ failure->label());
+ } else {
+ masm.branchTestObjClassNoSpectreMitigations(Assembler::NotEqual, obj, clasp,
+ scratch, failure->label());
+ }
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardNullProto(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadObjProto(obj, scratch);
+ masm.branchTestPtr(Assembler::NonZero, scratch, scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsExtensible(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchIfObjectNotExtensible(obj, scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardDynamicSlotIsSpecificObject(
+ ObjOperandId objId, ObjOperandId expectedId, uint32_t slotOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register expectedObject = allocator.useRegister(masm, expectedId);
+
+ // Allocate registers before the failure path to make sure they're registered
+ // by addFailurePath.
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Guard on the expected object.
+ StubFieldOffset slot(slotOffset, StubField::Type::RawInt32);
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch1);
+ emitLoadStubField(slot, scratch2);
+ BaseObjectSlotIndex expectedSlot(scratch1, scratch2);
+ masm.fallibleUnboxObject(expectedSlot, scratch1, failure->label());
+ masm.branchPtr(Assembler::NotEqual, expectedObject, scratch1,
+ failure->label());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardDynamicSlotIsNotObject(ObjOperandId objId,
+ uint32_t slotOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Guard that the slot isn't an object.
+ StubFieldOffset slot(slotOffset, StubField::Type::RawInt32);
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch1);
+ emitLoadStubField(slot, scratch2);
+ BaseObjectSlotIndex expectedSlot(scratch1, scratch2);
+ masm.branchTestObject(Assembler::Equal, expectedSlot, failure->label());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardFixedSlotValue(ObjOperandId objId,
+ uint32_t offsetOffset,
+ uint32_t valOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchValueRegister scratchVal(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ StubFieldOffset offset(offsetOffset, StubField::Type::RawInt32);
+ emitLoadStubField(offset, scratch);
+
+ StubFieldOffset val(valOffset, StubField::Type::Value);
+ emitLoadValueStubField(val, scratchVal);
+
+ BaseIndex slotVal(obj, scratch, TimesOne);
+ masm.branchTestValue(Assembler::NotEqual, slotVal, scratchVal,
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardDynamicSlotValue(ObjOperandId objId,
+ uint32_t offsetOffset,
+ uint32_t valOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchValueRegister scratchVal(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch1);
+
+ StubFieldOffset offset(offsetOffset, StubField::Type::RawInt32);
+ emitLoadStubField(offset, scratch2);
+
+ StubFieldOffset val(valOffset, StubField::Type::Value);
+ emitLoadValueStubField(val, scratchVal);
+
+ BaseIndex slotVal(scratch1, scratch2, TimesOne);
+ masm.branchTestValue(Assembler::NotEqual, slotVal, scratchVal,
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadFixedSlot(ValOperandId resultId,
+ ObjOperandId objId,
+ uint32_t offsetOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ StubFieldOffset slotIndex(offsetOffset, StubField::Type::RawInt32);
+ emitLoadStubField(slotIndex, scratch);
+
+ masm.loadValue(BaseIndex(obj, scratch, TimesOne), output);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDynamicSlot(ValOperandId resultId,
+ ObjOperandId objId,
+ uint32_t slotOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch1(allocator, masm);
+ Register scratch2 = output.scratchReg();
+
+ StubFieldOffset slotIndex(slotOffset, StubField::Type::RawInt32);
+ emitLoadStubField(slotIndex, scratch2);
+
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch1);
+ masm.loadValue(BaseObjectSlotIndex(scratch1, scratch2), output);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsNativeObject(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchIfNonNativeObj(obj, scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsProxy(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestObjectIsProxy(false, obj, scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsNotProxy(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestObjectIsProxy(true, obj, scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsNotArrayBufferMaybeShared(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadObjClassUnsafe(obj, scratch);
+ masm.branchPtr(Assembler::Equal, scratch, ImmPtr(&ArrayBufferObject::class_),
+ failure->label());
+ masm.branchPtr(Assembler::Equal, scratch,
+ ImmPtr(&SharedArrayBufferObject::class_), failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsTypedArray(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadObjClassUnsafe(obj, scratch);
+ masm.branchIfClassIsNotTypedArray(scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIsNotDOMProxy(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestProxyHandlerFamily(Assembler::Equal, obj, scratch,
+ GetDOMProxyHandlerFamily(),
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardNoDenseElements(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load obj->elements.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ // Make sure there are no dense elements.
+ Address initLength(scratch, ObjectElements::offsetOfInitializedLength());
+ masm.branch32(Assembler::NotEqual, initLength, Imm32(0), failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardSpecificInt32(Int32OperandId numId,
+ int32_t expected) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register num = allocator.useRegister(masm, numId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branch32(Assembler::NotEqual, num, Imm32(expected), failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardStringToInt32(StringOperandId strId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register str = allocator.useRegister(masm, strId);
+ Register output = allocator.defineRegister(masm, resultId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.guardStringToInt32(str, output, scratch, volatileRegs, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardStringToNumber(StringOperandId strId,
+ NumberOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register str = allocator.useRegister(masm, strId);
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label vmCall, done;
+ // Use indexed value as fast path if possible.
+ masm.loadStringIndexValue(str, scratch, &vmCall);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output);
+ masm.jump(&done);
+ {
+ masm.bind(&vmCall);
+
+ // Reserve stack for holding the result value of the call.
+ masm.reserveStack(sizeof(double));
+ masm.moveStackPtrTo(output.payloadOrValueReg());
+
+ // We cannot use callVM, as callVM expects to be able to clobber all
+ // operands, however, since this op is not the last in the generated IC, we
+ // want to be able to reference other live values.
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSString * str, double* result);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(str);
+ masm.passABIArg(output.payloadOrValueReg());
+ masm.callWithABI<Fn, js::StringToNumberPure>();
+ masm.storeCallPointerResult(scratch);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch);
+ masm.PopRegsInMaskIgnore(volatileRegs, ignore);
+
+ Label ok;
+ masm.branchIfTrueBool(scratch, &ok);
+ {
+ // OOM path, recovered by StringToNumberPure.
+ //
+ // Use addToStackPtr instead of freeStack as freeStack tracks stack height
+ // flow-insensitively, and using it twice would confuse the stack height
+ // tracking.
+ masm.addToStackPtr(Imm32(sizeof(double)));
+ masm.jump(failure->label());
+ }
+ masm.bind(&ok);
+
+ {
+ ScratchDoubleScope fpscratch(masm);
+ masm.loadDouble(Address(output.payloadOrValueReg(), 0), fpscratch);
+ masm.boxDouble(fpscratch, output, fpscratch);
+ }
+ masm.freeStack(sizeof(double));
+ }
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitNumberParseIntResult(StringOperandId strId,
+ Int32OperandId radixId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+ Register radix = allocator.useRegister(masm, radixId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, callvm.output());
+
+#ifdef DEBUG
+ Label ok;
+ masm.branch32(Assembler::Equal, radix, Imm32(0), &ok);
+ masm.branch32(Assembler::Equal, radix, Imm32(10), &ok);
+ masm.assumeUnreachable("radix must be 0 or 10 for indexed value fast path");
+ masm.bind(&ok);
+#endif
+
+ // Discard the stack to ensure it's balanced when we skip the vm-call.
+ allocator.discardStack(masm);
+
+ // Use indexed value as fast path if possible.
+ Label vmCall, done;
+ masm.loadStringIndexValue(str, scratch, &vmCall);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, callvm.outputValueReg());
+ masm.jump(&done);
+ {
+ masm.bind(&vmCall);
+
+ callvm.prepare();
+ masm.Push(radix);
+ masm.Push(str);
+
+ using Fn = bool (*)(JSContext*, HandleString, int32_t, MutableHandleValue);
+ callvm.call<Fn, js::NumberParseInt>();
+ }
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitDoubleParseIntResult(NumberOperandId numId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch2(*this, FloatReg1);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ allocator.ensureDoubleRegister(masm, numId, floatScratch1);
+
+ masm.branchDouble(Assembler::DoubleUnordered, floatScratch1, floatScratch1,
+ failure->label());
+ masm.branchTruncateDoubleToInt32(floatScratch1, scratch, failure->label());
+
+ Label ok;
+ masm.branch32(Assembler::NotEqual, scratch, Imm32(0), &ok);
+ {
+ // Accept both +0 and -0 and return 0.
+ masm.loadConstantDouble(0.0, floatScratch2);
+ masm.branchDouble(Assembler::DoubleEqual, floatScratch1, floatScratch2,
+ &ok);
+
+ // Fail if a non-zero input is in the exclusive range (-1, 1.0e-6).
+ masm.loadConstantDouble(DOUBLE_DECIMAL_IN_SHORTEST_LOW, floatScratch2);
+ masm.branchDouble(Assembler::DoubleLessThan, floatScratch1, floatScratch2,
+ failure->label());
+ }
+ masm.bind(&ok);
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitBooleanToNumber(BooleanOperandId booleanId,
+ NumberOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register boolean = allocator.useRegister(masm, booleanId);
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+ masm.tagValue(JSVAL_TYPE_INT32, boolean, output);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardStringToIndex(StringOperandId strId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register str = allocator.useRegister(masm, strId);
+ Register output = allocator.defineRegister(masm, resultId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label vmCall, done;
+ masm.loadStringIndexValue(str, output, &vmCall);
+ masm.jump(&done);
+
+ {
+ masm.bind(&vmCall);
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = int32_t (*)(JSString * str);
+ masm.setupUnalignedABICall(output);
+ masm.passABIArg(str);
+ masm.callWithABI<Fn, GetIndexFromString>();
+ masm.storeCallInt32Result(output);
+
+ LiveRegisterSet ignore;
+ ignore.add(output);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ // GetIndexFromString returns a negative value on failure.
+ masm.branchTest32(Assembler::Signed, output, output, failure->label());
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadProto(ObjOperandId objId, ObjOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register reg = allocator.defineRegister(masm, resultId);
+ masm.loadObjProto(obj, reg);
+
+#ifdef DEBUG
+ // We shouldn't encounter a null or lazy proto.
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ Label done;
+ masm.branchPtr(Assembler::Above, reg, ImmWord(1), &done);
+ masm.assumeUnreachable("Unexpected null or lazy proto in CacheIR LoadProto");
+ masm.bind(&done);
+#endif
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadEnclosingEnvironment(ObjOperandId objId,
+ ObjOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register reg = allocator.defineRegister(masm, resultId);
+ masm.unboxObject(
+ Address(obj, EnvironmentObject::offsetOfEnclosingEnvironment()), reg);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadWrapperTarget(ObjOperandId objId,
+ ObjOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register reg = allocator.defineRegister(masm, resultId);
+
+ masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), reg);
+ masm.unboxObject(
+ Address(reg, js::detail::ProxyReservedSlots::offsetOfPrivateSlot()), reg);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadValueTag(ValOperandId valId,
+ ValueTagOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+ Register res = allocator.defineRegister(masm, resultId);
+
+ Register tag = masm.extractTag(val, res);
+ if (tag != res) {
+ masm.mov(tag, res);
+ }
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDOMExpandoValue(ObjOperandId objId,
+ ValOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.defineValueRegister(masm, resultId);
+
+ masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()),
+ val.scratchReg());
+ masm.loadValue(Address(val.scratchReg(),
+ js::detail::ProxyReservedSlots::offsetOfPrivateSlot()),
+ val);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDOMExpandoValueIgnoreGeneration(
+ ObjOperandId objId, ValOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+
+ // Determine the expando's Address.
+ Register scratch = output.scratchReg();
+ masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), scratch);
+ Address expandoAddr(scratch,
+ js::detail::ProxyReservedSlots::offsetOfPrivateSlot());
+
+#ifdef DEBUG
+ // Private values are stored as doubles, so assert we have a double.
+ Label ok;
+ masm.branchTestDouble(Assembler::Equal, expandoAddr, &ok);
+ masm.assumeUnreachable("DOM expando is not a PrivateValue!");
+ masm.bind(&ok);
+#endif
+
+ // Load the ExpandoAndGeneration* from the PrivateValue.
+ masm.loadPrivate(expandoAddr, scratch);
+
+ // Load expandoAndGeneration->expando into the output Value register.
+ masm.loadValue(Address(scratch, ExpandoAndGeneration::offsetOfExpando()),
+ output);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadUndefinedResult() {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ masm.moveValue(UndefinedValue(), output.valueReg());
+ return true;
+}
+
+static void EmitStoreBoolean(MacroAssembler& masm, bool b,
+ const AutoOutputRegister& output) {
+ if (output.hasValue()) {
+ Value val = BooleanValue(b);
+ masm.moveValue(val, output.valueReg());
+ } else {
+ MOZ_ASSERT(output.type() == JSVAL_TYPE_BOOLEAN);
+ masm.movePtr(ImmWord(b), output.typedReg().gpr());
+ }
+}
+
+bool CacheIRCompiler::emitLoadBooleanResult(bool val) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ EmitStoreBoolean(masm, val, output);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadOperandResult(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ masm.moveValue(input, output.valueReg());
+ return true;
+}
+
+static void EmitStoreResult(MacroAssembler& masm, Register reg,
+ JSValueType type,
+ const AutoOutputRegister& output) {
+ if (output.hasValue()) {
+ masm.tagValue(type, reg, output.valueReg());
+ return;
+ }
+ if (type == JSVAL_TYPE_INT32 && output.typedReg().isFloat()) {
+ masm.convertInt32ToDouble(reg, output.typedReg().fpu());
+ return;
+ }
+ if (type == output.type()) {
+ masm.mov(reg, output.typedReg().gpr());
+ return;
+ }
+ masm.assumeUnreachable("Should have monitored result");
+}
+
+bool CacheIRCompiler::emitLoadInt32ArrayLengthResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+ masm.load32(Address(scratch, ObjectElements::offsetOfLength()), scratch);
+
+ // Guard length fits in an int32.
+ masm.branchTest32(Assembler::Signed, scratch, scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadInt32ArrayLength(ObjOperandId objId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register res = allocator.defineRegister(masm, resultId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), res);
+ masm.load32(Address(res, ObjectElements::offsetOfLength()), res);
+
+ // Guard length fits in an int32.
+ masm.branchTest32(Assembler::Signed, res, res, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitDoubleAddResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, lhsId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch1);
+
+ masm.addDouble(floatScratch1, floatScratch0);
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+
+ return true;
+}
+bool CacheIRCompiler::emitDoubleSubResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, lhsId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch1);
+
+ masm.subDouble(floatScratch1, floatScratch0);
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+
+ return true;
+}
+bool CacheIRCompiler::emitDoubleMulResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, lhsId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch1);
+
+ masm.mulDouble(floatScratch1, floatScratch0);
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+
+ return true;
+}
+bool CacheIRCompiler::emitDoubleDivResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, lhsId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch1);
+
+ masm.divDouble(floatScratch1, floatScratch0);
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+
+ return true;
+}
+bool CacheIRCompiler::emitDoubleModResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, lhsId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch1);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = double (*)(double a, double b);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch1, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, js::NumberMod>(MoveOp::DOUBLE);
+ masm.storeCallFloatResult(floatScratch0);
+
+ LiveRegisterSet ignore;
+ ignore.add(floatScratch0);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+
+ return true;
+}
+bool CacheIRCompiler::emitDoublePowResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, lhsId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch1);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = double (*)(double x, double y);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch1, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, js::ecmaPow>(MoveOp::DOUBLE);
+ masm.storeCallFloatResult(floatScratch0);
+
+ LiveRegisterSet ignore;
+ ignore.add(floatScratch0);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32AddResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.mov(rhs, scratch);
+ masm.branchAdd32(Assembler::Overflow, lhs, scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+bool CacheIRCompiler::emitInt32SubResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.mov(lhs, scratch);
+ masm.branchSub32(Assembler::Overflow, rhs, scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32MulResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegisterMaybeOutput scratch2(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label maybeNegZero, done;
+ masm.mov(lhs, scratch);
+ masm.branchMul32(Assembler::Overflow, rhs, scratch, failure->label());
+ masm.branchTest32(Assembler::Zero, scratch, scratch, &maybeNegZero);
+ masm.jump(&done);
+
+ masm.bind(&maybeNegZero);
+ masm.mov(lhs, scratch2);
+ // Result is -0 if exactly one of lhs or rhs is negative.
+ masm.or32(rhs, scratch2);
+ masm.branchTest32(Assembler::Signed, scratch2, scratch2, failure->label());
+
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32DivResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+ AutoScratchRegister rem(allocator, masm);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Prevent division by 0.
+ masm.branchTest32(Assembler::Zero, rhs, rhs, failure->label());
+
+ // Prevent -2147483648 / -1.
+ Label notOverflow;
+ masm.branch32(Assembler::NotEqual, lhs, Imm32(INT32_MIN), &notOverflow);
+ masm.branch32(Assembler::Equal, rhs, Imm32(-1), failure->label());
+ masm.bind(&notOverflow);
+
+ // Prevent negative 0.
+ Label notZero;
+ masm.branchTest32(Assembler::NonZero, lhs, lhs, &notZero);
+ masm.branchTest32(Assembler::Signed, rhs, rhs, failure->label());
+ masm.bind(&notZero);
+
+ masm.mov(lhs, scratch);
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.flexibleDivMod32(rhs, scratch, rem, false, volatileRegs);
+
+ // A remainder implies a double result.
+ masm.branchTest32(Assembler::NonZero, rem, rem, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32ModResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // x % 0 results in NaN
+ masm.branchTest32(Assembler::Zero, rhs, rhs, failure->label());
+
+ // Prevent -2147483648 % -1.
+ //
+ // Traps on x86 and has undefined behavior on ARM32 (when __aeabi_idivmod is
+ // called).
+ Label notOverflow;
+ masm.branch32(Assembler::NotEqual, lhs, Imm32(INT32_MIN), &notOverflow);
+ masm.branch32(Assembler::Equal, rhs, Imm32(-1), failure->label());
+ masm.bind(&notOverflow);
+
+ masm.mov(lhs, scratch);
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.flexibleRemainder32(rhs, scratch, false, volatileRegs);
+
+ // Modulo takes the sign of the dividend; we can't return negative zero here.
+ Label notZero;
+ masm.branchTest32(Assembler::NonZero, scratch, scratch, &notZero);
+ masm.branchTest32(Assembler::Signed, lhs, lhs, failure->label());
+ masm.bind(&notZero);
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32PowResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register base = allocator.useRegister(masm, lhsId);
+ Register power = allocator.useRegister(masm, rhsId);
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output);
+ AutoScratchRegister scratch3(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.pow32(base, power, scratch1, scratch2, scratch3, failure->label());
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch1, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32BitOrResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ masm.mov(rhs, scratch);
+ masm.or32(lhs, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+bool CacheIRCompiler::emitInt32BitXorResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ masm.mov(rhs, scratch);
+ masm.xor32(lhs, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+bool CacheIRCompiler::emitInt32BitAndResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ masm.mov(rhs, scratch);
+ masm.and32(lhs, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+bool CacheIRCompiler::emitInt32LeftShiftResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.mov(lhs, scratch);
+ masm.flexibleLshift32(rhs, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32RightShiftResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.mov(lhs, scratch);
+ masm.flexibleRshift32Arithmetic(rhs, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32URightShiftResult(Int32OperandId lhsId,
+ Int32OperandId rhsId,
+ bool forceDouble) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.mov(lhs, scratch);
+ masm.flexibleRshift32(rhs, scratch);
+ if (forceDouble) {
+ ScratchDoubleScope fpscratch(masm);
+ masm.convertUInt32ToDouble(scratch, fpscratch);
+ masm.boxDouble(fpscratch, output.valueReg(), fpscratch);
+ } else {
+ masm.branchTest32(Assembler::Signed, scratch, scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ }
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32NegationResult(Int32OperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register val = allocator.useRegister(masm, inputId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Guard against 0 and MIN_INT by checking if low 31-bits are all zero.
+ // Both of these result in a double.
+ masm.branchTest32(Assembler::Zero, val, Imm32(0x7fffffff), failure->label());
+ masm.mov(val, scratch);
+ masm.neg32(scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32IncResult(Int32OperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register input = allocator.useRegister(masm, inputId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.mov(input, scratch);
+ masm.branchAdd32(Assembler::Overflow, Imm32(1), scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32DecResult(Int32OperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register input = allocator.useRegister(masm, inputId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.mov(input, scratch);
+ masm.branchSub32(Assembler::Overflow, Imm32(1), scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32NotResult(Int32OperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register val = allocator.useRegister(masm, inputId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.mov(val, scratch);
+ masm.not32(scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitDoubleNegationResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ AutoScratchFloatRegister floatReg(this);
+
+ allocator.ensureDoubleRegister(masm, inputId, floatReg);
+
+ masm.negateDouble(floatReg);
+ masm.boxDouble(floatReg, output.valueReg(), floatReg);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitDoubleIncDecResult(bool isInc,
+ NumberOperandId inputId) {
+ AutoOutputRegister output(*this);
+
+ AutoScratchFloatRegister floatReg(this);
+
+ allocator.ensureDoubleRegister(masm, inputId, floatReg);
+
+ {
+ ScratchDoubleScope fpscratch(masm);
+ masm.loadConstantDouble(1.0, fpscratch);
+ if (isInc) {
+ masm.addDouble(fpscratch, floatReg);
+ } else {
+ masm.subDouble(fpscratch, floatReg);
+ }
+ }
+ masm.boxDouble(floatReg, output.valueReg(), floatReg);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitDoubleIncResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitDoubleIncDecResult(true, inputId);
+}
+
+bool CacheIRCompiler::emitDoubleDecResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitDoubleIncDecResult(false, inputId);
+}
+
+template <typename Fn, Fn fn>
+bool CacheIRCompiler::emitBigIntBinaryOperationShared(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ AutoCallVM callvm(masm, this, allocator);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ callvm.prepare();
+
+ masm.Push(rhs);
+ masm.Push(lhs);
+
+ callvm.call<Fn, fn>();
+ return true;
+}
+
+bool CacheIRCompiler::emitBigIntAddResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::add>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntSubResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::sub>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntMulResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::mul>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntDivResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::div>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntModResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::mod>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntPowResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::pow>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntBitAndResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::bitAnd>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntBitOrResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::bitOr>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntBitXorResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::bitXor>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntLeftShiftResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::lsh>(lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitBigIntRightShiftResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ return emitBigIntBinaryOperationShared<Fn, BigInt::rsh>(lhsId, rhsId);
+}
+
+template <typename Fn, Fn fn>
+bool CacheIRCompiler::emitBigIntUnaryOperationShared(BigIntOperandId inputId) {
+ AutoCallVM callvm(masm, this, allocator);
+ Register val = allocator.useRegister(masm, inputId);
+
+ callvm.prepare();
+
+ masm.Push(val);
+
+ callvm.call<Fn, fn>();
+ return true;
+}
+
+bool CacheIRCompiler::emitBigIntNotResult(BigIntOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt);
+ return emitBigIntUnaryOperationShared<Fn, BigInt::bitNot>(inputId);
+}
+
+bool CacheIRCompiler::emitBigIntNegationResult(BigIntOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt);
+ return emitBigIntUnaryOperationShared<Fn, BigInt::neg>(inputId);
+}
+
+bool CacheIRCompiler::emitBigIntIncResult(BigIntOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt);
+ return emitBigIntUnaryOperationShared<Fn, BigInt::inc>(inputId);
+}
+
+bool CacheIRCompiler::emitBigIntDecResult(BigIntOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt);
+ return emitBigIntUnaryOperationShared<Fn, BigInt::dec>(inputId);
+}
+
+bool CacheIRCompiler::emitTruncateDoubleToUInt32(NumberOperandId inputId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register res = allocator.defineRegister(masm, resultId);
+
+ AutoScratchFloatRegister floatReg(this);
+
+ allocator.ensureDoubleRegister(masm, inputId, floatReg);
+
+ Label done, truncateABICall;
+
+ masm.branchTruncateDoubleMaybeModUint32(floatReg, res, &truncateABICall);
+ masm.jump(&done);
+
+ masm.bind(&truncateABICall);
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ save.takeUnchecked(floatReg);
+ // Bug 1451976
+ save.takeUnchecked(floatReg.get().asSingle());
+ masm.PushRegsInMask(save);
+
+ using Fn = int32_t (*)(double);
+ masm.setupUnalignedABICall(res);
+ masm.passABIArg(floatReg, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, JS::ToInt32>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+ masm.storeCallInt32Result(res);
+
+ LiveRegisterSet ignore;
+ ignore.add(res);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArgumentsObjectLengthResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArgumentsObjectLength(obj, scratch, failure->label());
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArgumentsObjectLength(ObjOperandId objId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register res = allocator.defineRegister(masm, resultId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArgumentsObjectLength(obj, res, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArrayBufferByteLengthInt32Result(
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArrayBufferByteLengthIntPtr(obj, scratch);
+ masm.guardNonNegativeIntPtrToInt32(scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArrayBufferByteLengthDoubleResult(
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ ScratchDoubleScope fpscratch(masm);
+ masm.loadArrayBufferByteLengthIntPtr(obj, scratch);
+ masm.convertIntPtrToDouble(scratch, fpscratch);
+ masm.boxDouble(fpscratch, output.valueReg(), fpscratch);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArrayBufferViewLengthInt32Result(
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ masm.guardNonNegativeIntPtrToInt32(scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArrayBufferViewLengthDoubleResult(
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ ScratchDoubleScope fpscratch(masm);
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ masm.convertIntPtrToDouble(scratch, fpscratch);
+ masm.boxDouble(fpscratch, output.valueReg(), fpscratch);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadFunctionLengthResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Get the JSFunction flags and arg count.
+ masm.load32(Address(obj, JSFunction::offsetOfFlagsAndArgCount()), scratch);
+
+ // Functions with a SelfHostedLazyScript must be compiled with the slow-path
+ // before the function length is known. If the length was previously resolved,
+ // the length property may be shadowed.
+ masm.branchTest32(
+ Assembler::NonZero, scratch,
+ Imm32(FunctionFlags::SELFHOSTLAZY | FunctionFlags::RESOLVED_LENGTH),
+ failure->label());
+
+ masm.loadFunctionLength(obj, scratch, scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadFunctionNameResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadFunctionName(obj, scratch, ImmGCPtr(cx_->names().empty),
+ failure->label());
+
+ masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitLinearizeForCharAccess(StringOperandId strId,
+ Int32OperandId indexId,
+ StringOperandId resultId) {
+ Register str = allocator.useRegister(masm, strId);
+ Register index = allocator.useRegister(masm, indexId);
+ Register result = allocator.defineRegister(masm, resultId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label done;
+ masm.movePtr(str, result);
+
+ // We can omit the bounds check, because we only compare the index against the
+ // string length. In the worst case we unnecessarily linearize the string
+ // when the index is out-of-bounds.
+
+ masm.branchIfCanLoadStringChar(str, index, scratch, &done);
+ {
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = JSLinearString* (*)(JSString*);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(str);
+ masm.callWithABI<Fn, js::jit::LinearizeForCharAccessPure>();
+ masm.storeCallPointerResult(result);
+
+ LiveRegisterSet ignore;
+ ignore.add(result);
+ masm.PopRegsInMaskIgnore(volatileRegs, ignore);
+
+ masm.branchTestPtr(Assembler::Zero, result, result, failure->label());
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadStringLengthResult(StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register str = allocator.useRegister(masm, strId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.loadStringLength(str, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadStringCharCodeResult(StringOperandId strId,
+ Int32OperandId indexId,
+ bool handleOOB) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register str = allocator.useRegister(masm, strId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output);
+ AutoScratchRegister scratch3(allocator, masm);
+
+ // Bounds check, load string char.
+ Label done;
+ if (!handleOOB) {
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
+ scratch1, failure->label());
+ masm.loadStringChar(str, index, scratch1, scratch2, scratch3,
+ failure->label());
+ } else {
+ // Return NaN for out-of-bounds access.
+ masm.moveValue(JS::NaNValue(), output.valueReg());
+
+ // The bounds check mustn't use a scratch register which aliases the output.
+ MOZ_ASSERT(!output.valueReg().aliases(scratch3));
+
+ // This CacheIR op is always preceded by |LinearizeForCharAccess|, so we're
+ // guaranteed to see no nested ropes.
+ Label loadFailed;
+ masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
+ scratch3, &done);
+ masm.loadStringChar(str, index, scratch1, scratch2, scratch3, &loadFailed);
+
+ Label loadedChar;
+ masm.jump(&loadedChar);
+ masm.bind(&loadFailed);
+ masm.assumeUnreachable("loadStringChar can't fail for linear strings");
+ masm.bind(&loadedChar);
+ }
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch1, output.valueReg());
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitNewStringObjectResult(uint32_t templateObjectOffset,
+ StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+
+ callvm.prepare();
+ masm.Push(str);
+
+ using Fn = JSObject* (*)(JSContext*, HandleString);
+ callvm.call<Fn, NewStringObject>();
+ return true;
+}
+
+bool CacheIRCompiler::emitStringIndexOfResult(StringOperandId strId,
+ StringOperandId searchStrId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+ Register searchStr = allocator.useRegister(masm, searchStrId);
+
+ callvm.prepare();
+ masm.Push(searchStr);
+ masm.Push(str);
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, int32_t*);
+ callvm.call<Fn, js::StringIndexOf>();
+ return true;
+}
+
+bool CacheIRCompiler::emitStringStartsWithResult(StringOperandId strId,
+ StringOperandId searchStrId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+ Register searchStr = allocator.useRegister(masm, searchStrId);
+
+ callvm.prepare();
+ masm.Push(searchStr);
+ masm.Push(str);
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ callvm.call<Fn, js::StringStartsWith>();
+ return true;
+}
+
+bool CacheIRCompiler::emitStringEndsWithResult(StringOperandId strId,
+ StringOperandId searchStrId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+ Register searchStr = allocator.useRegister(masm, searchStrId);
+
+ callvm.prepare();
+ masm.Push(searchStr);
+ masm.Push(str);
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ callvm.call<Fn, js::StringEndsWith>();
+ return true;
+}
+
+bool CacheIRCompiler::emitStringToLowerCaseResult(StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+
+ callvm.prepare();
+ masm.Push(str);
+
+ using Fn = JSString* (*)(JSContext*, HandleString);
+ callvm.call<Fn, js::StringToLowerCase>();
+ return true;
+}
+
+bool CacheIRCompiler::emitStringToUpperCaseResult(StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+
+ callvm.prepare();
+ masm.Push(str);
+
+ using Fn = JSString* (*)(JSContext*, HandleString);
+ callvm.call<Fn, js::StringToUpperCase>();
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArgumentsObjectArgResult(ObjOperandId objId,
+ Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArgumentsObjectElement(obj, index, output.valueReg(), scratch,
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArgumentsObjectArgHoleResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArgumentsObjectElementHole(obj, index, output.valueReg(), scratch,
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadArgumentsObjectArgExistsResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegisterMaybeOutput scratch2(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArgumentsObjectElementExists(obj, index, scratch2, scratch1,
+ failure->label());
+ EmitStoreResult(masm, scratch2, JSVAL_TYPE_BOOLEAN, output);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDenseElementResult(ObjOperandId objId,
+ Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegisterMaybeOutput scratch2(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load obj->elements.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch1);
+
+ // Bounds check.
+ Address initLength(scratch1, ObjectElements::offsetOfInitializedLength());
+ masm.spectreBoundsCheck32(index, initLength, scratch2, failure->label());
+
+ // Hole check.
+ BaseObjectElementIndex element(scratch1, index);
+ masm.branchTestMagic(Assembler::Equal, element, failure->label());
+ masm.loadTypedOrValue(element, output);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardInt32IsNonNegative(Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register index = allocator.useRegister(masm, indexId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branch32(Assembler::LessThan, index, Imm32(0), failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIndexIsNotDenseElement(ObjOperandId objId,
+ Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoSpectreBoundsScratchRegister spectreScratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load obj->elements.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ // Ensure index >= initLength or the element is a hole.
+ Label notDense;
+ Address capacity(scratch, ObjectElements::offsetOfInitializedLength());
+ masm.spectreBoundsCheck32(index, capacity, spectreScratch, &notDense);
+
+ BaseValueIndex element(scratch, index);
+ masm.branchTestMagic(Assembler::Equal, element, &notDense);
+
+ masm.jump(failure->label());
+
+ masm.bind(&notDense);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardIndexIsValidUpdateOrAdd(ObjOperandId objId,
+ Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoSpectreBoundsScratchRegister spectreScratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load obj->elements.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ Label success;
+
+ // If length is writable, branch to &success. All indices are writable.
+ Address flags(scratch, ObjectElements::offsetOfFlags());
+ masm.branchTest32(Assembler::Zero, flags,
+ Imm32(ObjectElements::Flags::NONWRITABLE_ARRAY_LENGTH),
+ &success);
+
+ // Otherwise, ensure index is in bounds.
+ Address length(scratch, ObjectElements::offsetOfLength());
+ masm.spectreBoundsCheck32(index, length, spectreScratch,
+ /* failure = */ failure->label());
+ masm.bind(&success);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardTagNotEqual(ValueTagOperandId lhsId,
+ ValueTagOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label done;
+ masm.branch32(Assembler::Equal, lhs, rhs, failure->label());
+
+ // If both lhs and rhs are numbers, can't use tag comparison to do inequality
+ // comparison
+ masm.branchTestNumber(Assembler::NotEqual, lhs, &done);
+ masm.branchTestNumber(Assembler::NotEqual, rhs, &done);
+ masm.jump(failure->label());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardXrayExpandoShapeAndDefaultProto(
+ ObjOperandId objId, uint32_t shapeWrapperOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ StubFieldOffset shapeWrapper(shapeWrapperOffset, StubField::Type::JSObject);
+
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), scratch);
+ Address holderAddress(scratch,
+ sizeof(Value) * GetXrayJitInfo()->xrayHolderSlot);
+ Address expandoAddress(scratch, NativeObject::getFixedSlotOffset(
+ GetXrayJitInfo()->holderExpandoSlot));
+
+ masm.fallibleUnboxObject(holderAddress, scratch, failure->label());
+ masm.fallibleUnboxObject(expandoAddress, scratch, failure->label());
+
+ // Unwrap the expando before checking its shape.
+ masm.loadPtr(Address(scratch, ProxyObject::offsetOfReservedSlots()), scratch);
+ masm.unboxObject(
+ Address(scratch, js::detail::ProxyReservedSlots::offsetOfPrivateSlot()),
+ scratch);
+
+ emitLoadStubField(shapeWrapper, scratch2);
+ LoadShapeWrapperContents(masm, scratch2, scratch2, failure->label());
+ masm.branchTestObjShape(Assembler::NotEqual, scratch, scratch2, scratch3,
+ scratch, failure->label());
+
+ // The reserved slots on the expando should all be in fixed slots.
+ Address protoAddress(scratch, NativeObject::getFixedSlotOffset(
+ GetXrayJitInfo()->expandoProtoSlot));
+ masm.branchTestUndefined(Assembler::NotEqual, protoAddress, failure->label());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardXrayNoExpando(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), scratch);
+ Address holderAddress(scratch,
+ sizeof(Value) * GetXrayJitInfo()->xrayHolderSlot);
+ Address expandoAddress(scratch, NativeObject::getFixedSlotOffset(
+ GetXrayJitInfo()->holderExpandoSlot));
+
+ Label done;
+ masm.fallibleUnboxObject(holderAddress, scratch, &done);
+ masm.branchTestObject(Assembler::Equal, expandoAddress, failure->label());
+ masm.bind(&done);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardNoAllocationMetadataBuilder(
+ uint32_t builderAddrOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ StubFieldOffset builderField(builderAddrOffset, StubField::Type::RawPointer);
+ emitLoadStubField(builderField, scratch);
+ masm.branchPtr(Assembler::NotEqual, Address(scratch, 0), ImmWord(0),
+ failure->label());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardFunctionHasJitEntry(ObjOperandId funId,
+ bool constructing) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register fun = allocator.useRegister(masm, funId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchIfFunctionHasNoJitEntry(fun, constructing, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardFunctionHasNoJitEntry(ObjOperandId funId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, funId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchIfFunctionHasJitEntry(obj, /*isConstructing =*/false,
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardFunctionIsNonBuiltinCtor(ObjOperandId funId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register fun = allocator.useRegister(masm, funId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchIfNotFunctionIsNonBuiltinCtor(fun, scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardFunctionIsConstructor(ObjOperandId funId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register funcReg = allocator.useRegister(masm, funId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Ensure obj is a constructor
+ masm.branchTestFunctionFlags(funcReg, FunctionFlags::CONSTRUCTOR,
+ Assembler::Zero, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardNotClassConstructor(ObjOperandId funId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register fun = allocator.useRegister(masm, funId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchFunctionKind(Assembler::Equal, FunctionFlags::ClassConstructor,
+ fun, scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardArrayIsPacked(ObjOperandId arrayId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register array = allocator.useRegister(masm, arrayId);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchArrayIsNotPacked(array, scratch, scratch2, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardArgumentsObjectFlags(ObjOperandId objId,
+ uint8_t flags) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchTestArgumentsObjectFlags(obj, scratch, flags, Assembler::NonZero,
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDenseElementHoleResult(ObjOperandId objId,
+ Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegisterMaybeOutput scratch2(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Make sure the index is nonnegative.
+ masm.branch32(Assembler::LessThan, index, Imm32(0), failure->label());
+
+ // Load obj->elements.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch1);
+
+ // Guard on the initialized length.
+ Label hole;
+ Address initLength(scratch1, ObjectElements::offsetOfInitializedLength());
+ masm.spectreBoundsCheck32(index, initLength, scratch2, &hole);
+
+ // Load the value.
+ Label done;
+ masm.loadValue(BaseObjectElementIndex(scratch1, index), output.valueReg());
+ masm.branchTestMagic(Assembler::NotEqual, output.valueReg(), &done);
+
+ // Load undefined for the hole.
+ masm.bind(&hole);
+ masm.moveValue(UndefinedValue(), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadTypedArrayElementExistsResult(
+ ObjOperandId objId, IntPtrOperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Label outOfBounds, done;
+
+ // Bounds check.
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ masm.branchPtr(Assembler::BelowOrEqual, scratch, index, &outOfBounds);
+ EmitStoreBoolean(masm, true, output);
+ masm.jump(&done);
+
+ masm.bind(&outOfBounds);
+ EmitStoreBoolean(masm, false, output);
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDenseElementExistsResult(ObjOperandId objId,
+ Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load obj->elements.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ // Bounds check. Unsigned compare sends negative indices to next IC.
+ Address initLength(scratch, ObjectElements::offsetOfInitializedLength());
+ masm.branch32(Assembler::BelowOrEqual, initLength, index, failure->label());
+
+ // Hole check.
+ BaseObjectElementIndex element(scratch, index);
+ masm.branchTestMagic(Assembler::Equal, element, failure->label());
+
+ EmitStoreBoolean(masm, true, output);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDenseElementHoleExistsResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Make sure the index is nonnegative.
+ masm.branch32(Assembler::LessThan, index, Imm32(0), failure->label());
+
+ // Load obj->elements.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ // Guard on the initialized length.
+ Label hole;
+ Address initLength(scratch, ObjectElements::offsetOfInitializedLength());
+ masm.branch32(Assembler::BelowOrEqual, initLength, index, &hole);
+
+ // Load value and replace with true.
+ Label done;
+ BaseObjectElementIndex element(scratch, index);
+ masm.branchTestMagic(Assembler::Equal, element, &hole);
+ EmitStoreBoolean(masm, true, output);
+ masm.jump(&done);
+
+ // Load false for the hole.
+ masm.bind(&hole);
+ EmitStoreBoolean(masm, false, output);
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitPackedArrayPopResult(ObjOperandId arrayId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register array = allocator.useRegister(masm, arrayId);
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.packedArrayPop(array, output.valueReg(), scratch1, scratch2,
+ failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitPackedArrayShiftResult(ObjOperandId arrayId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register array = allocator.useRegister(masm, arrayId);
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.packedArrayShift(array, output.valueReg(), scratch1, scratch2,
+ volatileRegs, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitIsObjectResult(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ ValueOperand val = allocator.useValueRegister(masm, inputId);
+
+ masm.testObjectSet(Assembler::Equal, val, scratch);
+
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitIsPackedArrayResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ Register outputScratch = output.valueReg().scratchReg();
+ masm.setIsPackedArray(obj, outputScratch, scratch);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, outputScratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitIsCallableResult(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegisterMaybeOutput scratch2(allocator, masm, output);
+
+ ValueOperand val = allocator.useValueRegister(masm, inputId);
+
+ Label isObject, done;
+ masm.branchTestObject(Assembler::Equal, val, &isObject);
+ // Primitives are never callable.
+ masm.move32(Imm32(0), scratch2);
+ masm.jump(&done);
+
+ masm.bind(&isObject);
+ masm.unboxObject(val, scratch1);
+
+ Label isProxy;
+ masm.isCallable(scratch1, scratch2, &isProxy);
+ masm.jump(&done);
+
+ masm.bind(&isProxy);
+ {
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSObject * obj);
+ masm.setupUnalignedABICall(scratch2);
+ masm.passABIArg(scratch1);
+ masm.callWithABI<Fn, ObjectIsCallable>();
+ masm.storeCallBoolResult(scratch2);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch2);
+ masm.PopRegsInMaskIgnore(volatileRegs, ignore);
+ }
+
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitIsConstructorResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register obj = allocator.useRegister(masm, objId);
+
+ Label isProxy, done;
+ masm.isConstructor(obj, scratch, &isProxy);
+ masm.jump(&done);
+
+ masm.bind(&isProxy);
+ {
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSObject * obj);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, ObjectIsConstructor>();
+ masm.storeCallBoolResult(scratch);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch);
+ masm.PopRegsInMaskIgnore(volatileRegs, ignore);
+ }
+
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitIsCrossRealmArrayConstructorResult(
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register obj = allocator.useRegister(masm, objId);
+
+ masm.setIsCrossRealmArrayConstructor(obj, scratch);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitArrayBufferViewByteOffsetInt32Result(
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register obj = allocator.useRegister(masm, objId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArrayBufferViewByteOffsetIntPtr(obj, scratch);
+ masm.guardNonNegativeIntPtrToInt32(scratch, failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitArrayBufferViewByteOffsetDoubleResult(
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ ScratchDoubleScope fpscratch(masm);
+ masm.loadArrayBufferViewByteOffsetIntPtr(obj, scratch);
+ masm.convertIntPtrToDouble(scratch, fpscratch);
+ masm.boxDouble(fpscratch, output.valueReg(), fpscratch);
+ return true;
+}
+
+bool CacheIRCompiler::emitTypedArrayByteLengthInt32Result(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+ Register obj = allocator.useRegister(masm, objId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch1);
+ masm.guardNonNegativeIntPtrToInt32(scratch1, failure->label());
+ masm.typedArrayElementSize(obj, scratch2);
+
+ masm.branchMul32(Assembler::Overflow, scratch2.get(), scratch1,
+ failure->label());
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch1, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitTypedArrayByteLengthDoubleResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+ Register obj = allocator.useRegister(masm, objId);
+
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch1);
+ masm.typedArrayElementSize(obj, scratch2);
+ masm.mulPtr(scratch2, scratch1);
+
+ ScratchDoubleScope fpscratch(masm);
+ masm.convertIntPtrToDouble(scratch1, fpscratch);
+ masm.boxDouble(fpscratch, output.valueReg(), fpscratch);
+ return true;
+}
+
+bool CacheIRCompiler::emitTypedArrayElementSizeResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register obj = allocator.useRegister(masm, objId);
+
+ masm.typedArrayElementSize(obj, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardHasAttachedArrayBuffer(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoScratchRegister scratch(allocator, masm);
+ Register obj = allocator.useRegister(masm, objId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchIfHasDetachedArrayBuffer(obj, scratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitIsTypedArrayConstructorResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register obj = allocator.useRegister(masm, objId);
+
+ masm.setIsDefinitelyTypedArrayConstructor(obj, scratch);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitGetNextMapSetEntryForIteratorResult(
+ ObjOperandId iterId, ObjOperandId resultArrId, bool isMap) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register iter = allocator.useRegister(masm, iterId);
+ Register resultArr = allocator.useRegister(masm, resultArrId);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ save.takeUnchecked(output.valueReg());
+ save.takeUnchecked(scratch);
+ masm.PushRegsInMask(save);
+
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(iter);
+ masm.passABIArg(resultArr);
+ if (isMap) {
+ using Fn = bool (*)(MapIteratorObject * iter, ArrayObject * resultPairObj);
+ masm.callWithABI<Fn, MapIteratorObject::next>();
+ } else {
+ using Fn = bool (*)(SetIteratorObject * iter, ArrayObject * resultObj);
+ masm.callWithABI<Fn, SetIteratorObject::next>();
+ }
+ masm.storeCallBoolResult(scratch);
+
+ masm.PopRegsInMask(save);
+
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitFinishBoundFunctionInitResult(
+ ObjOperandId boundId, ObjOperandId targetId, Int32OperandId argCountId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register bound = allocator.useRegister(masm, boundId);
+ Register target = allocator.useRegister(masm, targetId);
+ Register argCount = allocator.useRegister(masm, argCountId);
+
+ callvm.prepare();
+
+ masm.Push(argCount);
+ masm.Push(target);
+ masm.Push(bound);
+
+ using Fn = bool (*)(JSContext * cx, HandleFunction bound, HandleObject target,
+ int32_t argCount);
+ callvm.callNoResult<Fn, JSFunction::finishBoundFunctionInit>();
+
+ masm.moveValue(UndefinedValue(), callvm.outputValueReg());
+ return true;
+}
+
+void CacheIRCompiler::emitActivateIterator(Register objBeingIterated,
+ Register iterObject,
+ Register nativeIter,
+ Register scratch, Register scratch2,
+ uint32_t enumeratorsAddrOffset) {
+ // 'objectBeingIterated_' must be nullptr, so we don't need a pre-barrier.
+ Address iterObjAddr(nativeIter,
+ NativeIterator::offsetOfObjectBeingIterated());
+#ifdef DEBUG
+ Label ok;
+ masm.branchPtr(Assembler::Equal, iterObjAddr, ImmPtr(nullptr), &ok);
+ masm.assumeUnreachable("iterator with non-null object");
+ masm.bind(&ok);
+#endif
+
+ // Mark iterator as active.
+ Address iterFlagsAddr(nativeIter, NativeIterator::offsetOfFlagsAndCount());
+ masm.storePtr(objBeingIterated, iterObjAddr);
+ masm.or32(Imm32(NativeIterator::Flags::Active), iterFlagsAddr);
+
+ // Post-write barrier for stores to 'objectBeingIterated_'.
+ emitPostBarrierSlot(
+ iterObject,
+ TypedOrValueRegister(MIRType::Object, AnyRegister(objBeingIterated)),
+ scratch);
+
+ // Chain onto the active iterator stack.
+ StubFieldOffset enumeratorsAddr(enumeratorsAddrOffset,
+ StubField::Type::RawPointer);
+ emitLoadStubField(enumeratorsAddr, scratch);
+ masm.registerIterator(scratch, nativeIter, scratch2);
+}
+
+bool CacheIRCompiler::emitGuardAndGetIterator(ObjOperandId objId,
+ uint32_t iterOffset,
+ uint32_t enumeratorsAddrOffset,
+ ObjOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister niScratch(allocator, masm);
+
+ StubFieldOffset iterField(iterOffset, StubField::Type::JSObject);
+
+ Register output = allocator.defineRegister(masm, resultId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load our PropertyIteratorObject* and its NativeIterator.
+ emitLoadStubField(iterField, output);
+
+ Address slotAddr(output, PropertyIteratorObject::offsetOfIteratorSlot());
+ masm.loadPrivate(slotAddr, niScratch);
+
+ // Ensure the iterator is reusable: see NativeIterator::isReusable.
+ masm.branchIfNativeIteratorNotReusable(niScratch, failure->label());
+
+ emitActivateIterator(obj, output, niScratch, scratch1, scratch2,
+ enumeratorsAddrOffset);
+ return true;
+}
+
+bool CacheIRCompiler::emitObjectToIteratorResult(
+ ObjOperandId objId, uint32_t enumeratorsAddrOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+ Register obj = allocator.useRegister(masm, objId);
+
+ AutoScratchRegister iterObj(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegisterMaybeOutput scratch2(allocator, masm, callvm.output());
+ AutoScratchRegisterMaybeOutputType scratch3(allocator, masm, callvm.output());
+
+ Label callVM, done;
+ masm.maybeLoadIteratorFromShape(obj, iterObj, scratch, scratch2, scratch3,
+ &callVM);
+
+ masm.loadPrivate(
+ Address(iterObj, PropertyIteratorObject::offsetOfIteratorSlot()),
+ scratch);
+
+ emitActivateIterator(obj, iterObj, scratch, scratch2, scratch3,
+ enumeratorsAddrOffset);
+ masm.jump(&done);
+
+ masm.bind(&callVM);
+ callvm.prepare();
+ masm.Push(obj);
+ using Fn = JSObject* (*)(JSContext*, HandleObject);
+ callvm.call<Fn, GetIterator>();
+ masm.storeCallPointerResult(iterObj);
+
+ masm.bind(&done);
+ EmitStoreResult(masm, iterObj, JSVAL_TYPE_OBJECT, callvm.output());
+ return true;
+}
+
+bool CacheIRCompiler::emitValueToIteratorResult(ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+ callvm.prepare();
+
+ masm.Push(val);
+
+ using Fn = JSObject* (*)(JSContext*, HandleValue);
+ callvm.call<Fn, ValueToIterator>();
+ return true;
+}
+
+bool CacheIRCompiler::emitNewArrayIteratorResult(
+ uint32_t templateObjectOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ callvm.prepare();
+
+ using Fn = ArrayIteratorObject* (*)(JSContext*);
+ callvm.call<Fn, NewArrayIterator>();
+ return true;
+}
+
+bool CacheIRCompiler::emitNewStringIteratorResult(
+ uint32_t templateObjectOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ callvm.prepare();
+
+ using Fn = StringIteratorObject* (*)(JSContext*);
+ callvm.call<Fn, NewStringIterator>();
+ return true;
+}
+
+bool CacheIRCompiler::emitNewRegExpStringIteratorResult(
+ uint32_t templateObjectOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ callvm.prepare();
+
+ using Fn = RegExpStringIteratorObject* (*)(JSContext*);
+ callvm.call<Fn, NewRegExpStringIterator>();
+ return true;
+}
+
+bool CacheIRCompiler::emitObjectCreateResult(uint32_t templateObjectOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+ AutoScratchRegister scratch(allocator, masm);
+
+ StubFieldOffset objectField(templateObjectOffset, StubField::Type::JSObject);
+ emitLoadStubField(objectField, scratch);
+
+ callvm.prepare();
+ masm.Push(scratch);
+
+ using Fn = PlainObject* (*)(JSContext*, Handle<PlainObject*>);
+ callvm.call<Fn, ObjectCreateWithTemplate>();
+ return true;
+}
+
+bool CacheIRCompiler::emitNewArrayFromLengthResult(
+ uint32_t templateObjectOffset, Int32OperandId lengthId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+ AutoScratchRegister scratch(allocator, masm);
+ Register length = allocator.useRegister(masm, lengthId);
+
+ StubFieldOffset objectField(templateObjectOffset, StubField::Type::JSObject);
+ emitLoadStubField(objectField, scratch);
+
+ callvm.prepare();
+ masm.Push(length);
+ masm.Push(scratch);
+
+ using Fn = ArrayObject* (*)(JSContext*, Handle<ArrayObject*>, int32_t length);
+ callvm.call<Fn, ArrayConstructorOneArg>();
+ return true;
+}
+
+bool CacheIRCompiler::emitNewTypedArrayFromLengthResult(
+ uint32_t templateObjectOffset, Int32OperandId lengthId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+ AutoScratchRegister scratch(allocator, masm);
+ Register length = allocator.useRegister(masm, lengthId);
+
+ StubFieldOffset objectField(templateObjectOffset, StubField::Type::JSObject);
+ emitLoadStubField(objectField, scratch);
+
+ callvm.prepare();
+ masm.Push(length);
+ masm.Push(scratch);
+
+ using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, int32_t length);
+ callvm.call<Fn, NewTypedArrayWithTemplateAndLength>();
+ return true;
+}
+
+bool CacheIRCompiler::emitNewTypedArrayFromArrayBufferResult(
+ uint32_t templateObjectOffset, ObjOperandId bufferId,
+ ValOperandId byteOffsetId, ValOperandId lengthId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+#ifdef JS_CODEGEN_X86
+ MOZ_CRASH("Instruction not supported on 32-bit x86, not enough registers");
+#endif
+
+ AutoCallVM callvm(masm, this, allocator);
+ AutoScratchRegister scratch(allocator, masm);
+ Register buffer = allocator.useRegister(masm, bufferId);
+ ValueOperand byteOffset = allocator.useValueRegister(masm, byteOffsetId);
+ ValueOperand length = allocator.useValueRegister(masm, lengthId);
+
+ StubFieldOffset objectField(templateObjectOffset, StubField::Type::JSObject);
+ emitLoadStubField(objectField, scratch);
+
+ callvm.prepare();
+ masm.Push(length);
+ masm.Push(byteOffset);
+ masm.Push(buffer);
+ masm.Push(scratch);
+
+ using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, HandleObject,
+ HandleValue, HandleValue);
+ callvm.call<Fn, NewTypedArrayWithTemplateAndBuffer>();
+ return true;
+}
+
+bool CacheIRCompiler::emitNewTypedArrayFromArrayResult(
+ uint32_t templateObjectOffset, ObjOperandId arrayId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+ AutoScratchRegister scratch(allocator, masm);
+ Register array = allocator.useRegister(masm, arrayId);
+
+ StubFieldOffset objectField(templateObjectOffset, StubField::Type::JSObject);
+ emitLoadStubField(objectField, scratch);
+
+ callvm.prepare();
+ masm.Push(array);
+ masm.Push(scratch);
+
+ using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, HandleObject);
+ callvm.call<Fn, NewTypedArrayWithTemplateAndArray>();
+ return true;
+}
+
+bool CacheIRCompiler::emitAddSlotAndCallAddPropHook(ObjOperandId objId,
+ ValOperandId rhsId,
+ uint32_t newShapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ AutoScratchRegister scratch(allocator, masm);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand rhs = allocator.useValueRegister(masm, rhsId);
+
+ StubFieldOffset shapeField(newShapeOffset, StubField::Type::Shape);
+ emitLoadStubField(shapeField, scratch);
+
+ callvm.prepare();
+
+ masm.Push(scratch);
+ masm.Push(rhs);
+ masm.Push(obj);
+
+ using Fn =
+ bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, Handle<Shape*>);
+ callvm.callNoResult<Fn, AddSlotAndCallAddPropHook>();
+ return true;
+}
+
+bool CacheIRCompiler::emitMathAbsInt32Result(Int32OperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Register input = allocator.useRegister(masm, inputId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.mov(input, scratch);
+ // Don't negate already positive values.
+ Label positive;
+ masm.branchTest32(Assembler::NotSigned, scratch, scratch, &positive);
+ // neg32 might overflow for INT_MIN.
+ masm.branchNeg32(Assembler::Overflow, scratch, failure->label());
+ masm.bind(&positive);
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMathAbsNumberResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoAvailableFloatRegister scratch(*this, FloatReg0);
+
+ allocator.ensureDoubleRegister(masm, inputId, scratch);
+
+ masm.absDouble(scratch, scratch);
+ masm.boxDouble(scratch, output.valueReg(), scratch);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathClz32Result(Int32OperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register input = allocator.useRegister(masm, inputId);
+
+ masm.clz32(input, scratch, /* knownNotZero = */ false);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMathSignInt32Result(Int32OperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register input = allocator.useRegister(masm, inputId);
+
+ masm.signInt32(input, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMathSignNumberResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch2(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, inputId, floatScratch1);
+
+ masm.signDouble(floatScratch1, floatScratch2);
+ masm.boxDouble(floatScratch2, output.valueReg(), floatScratch2);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathSignNumberToInt32Result(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch2(*this, FloatReg1);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ allocator.ensureDoubleRegister(masm, inputId, floatScratch1);
+
+ masm.signDoubleToInt32(floatScratch1, scratch, floatScratch2,
+ failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMathImulResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ masm.mov(lhs, scratch);
+ masm.mul32(rhs, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMathSqrtNumberResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoAvailableFloatRegister scratch(*this, FloatReg0);
+
+ allocator.ensureDoubleRegister(masm, inputId, scratch);
+
+ masm.sqrtDouble(scratch, scratch);
+ masm.boxDouble(scratch, output.valueReg(), scratch);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathFloorNumberResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoAvailableFloatRegister scratch(*this, FloatReg0);
+
+ allocator.ensureDoubleRegister(masm, inputId, scratch);
+
+ if (Assembler::HasRoundInstruction(RoundingMode::Down)) {
+ masm.nearbyIntDouble(RoundingMode::Down, scratch, scratch);
+ masm.boxDouble(scratch, output.valueReg(), scratch);
+ return true;
+ }
+
+ return emitMathFunctionNumberResultShared(UnaryMathFunction::Floor, scratch,
+ output.valueReg());
+}
+
+bool CacheIRCompiler::emitMathCeilNumberResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoAvailableFloatRegister scratch(*this, FloatReg0);
+
+ allocator.ensureDoubleRegister(masm, inputId, scratch);
+
+ if (Assembler::HasRoundInstruction(RoundingMode::Up)) {
+ masm.nearbyIntDouble(RoundingMode::Up, scratch, scratch);
+ masm.boxDouble(scratch, output.valueReg(), scratch);
+ return true;
+ }
+
+ return emitMathFunctionNumberResultShared(UnaryMathFunction::Ceil, scratch,
+ output.valueReg());
+}
+
+bool CacheIRCompiler::emitMathTruncNumberResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoAvailableFloatRegister scratch(*this, FloatReg0);
+
+ allocator.ensureDoubleRegister(masm, inputId, scratch);
+
+ if (Assembler::HasRoundInstruction(RoundingMode::TowardsZero)) {
+ masm.nearbyIntDouble(RoundingMode::TowardsZero, scratch, scratch);
+ masm.boxDouble(scratch, output.valueReg(), scratch);
+ return true;
+ }
+
+ return emitMathFunctionNumberResultShared(UnaryMathFunction::Trunc, scratch,
+ output.valueReg());
+}
+
+bool CacheIRCompiler::emitMathFRoundNumberResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoAvailableFloatRegister scratch(*this, FloatReg0);
+ FloatRegister scratchFloat32 = scratch.get().asSingle();
+
+ allocator.ensureDoubleRegister(masm, inputId, scratch);
+
+ masm.convertDoubleToFloat32(scratch, scratchFloat32);
+ masm.convertFloat32ToDouble(scratchFloat32, scratch);
+
+ masm.boxDouble(scratch, output.valueReg(), scratch);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathHypot2NumberResult(NumberOperandId first,
+ NumberOperandId second) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, first, floatScratch0);
+ allocator.ensureDoubleRegister(masm, second, floatScratch1);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = double (*)(double x, double y);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch1, MoveOp::DOUBLE);
+
+ masm.callWithABI<Fn, ecmaHypot>(MoveOp::DOUBLE);
+ masm.storeCallFloatResult(floatScratch0);
+
+ LiveRegisterSet ignore;
+ ignore.add(floatScratch0);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathHypot3NumberResult(NumberOperandId first,
+ NumberOperandId second,
+ NumberOperandId third) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+ AutoAvailableFloatRegister floatScratch2(*this, FloatReg2);
+
+ allocator.ensureDoubleRegister(masm, first, floatScratch0);
+ allocator.ensureDoubleRegister(masm, second, floatScratch1);
+ allocator.ensureDoubleRegister(masm, third, floatScratch2);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = double (*)(double x, double y, double z);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch1, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch2, MoveOp::DOUBLE);
+
+ masm.callWithABI<Fn, hypot3>(MoveOp::DOUBLE);
+ masm.storeCallFloatResult(floatScratch0);
+
+ LiveRegisterSet ignore;
+ ignore.add(floatScratch0);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathHypot4NumberResult(NumberOperandId first,
+ NumberOperandId second,
+ NumberOperandId third,
+ NumberOperandId fourth) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+ AutoAvailableFloatRegister floatScratch2(*this, FloatReg2);
+ AutoAvailableFloatRegister floatScratch3(*this, FloatReg3);
+
+ allocator.ensureDoubleRegister(masm, first, floatScratch0);
+ allocator.ensureDoubleRegister(masm, second, floatScratch1);
+ allocator.ensureDoubleRegister(masm, third, floatScratch2);
+ allocator.ensureDoubleRegister(masm, fourth, floatScratch3);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = double (*)(double x, double y, double z, double w);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch1, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch2, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch3, MoveOp::DOUBLE);
+
+ masm.callWithABI<Fn, hypot4>(MoveOp::DOUBLE);
+ masm.storeCallFloatResult(floatScratch0);
+
+ LiveRegisterSet ignore;
+ ignore.add(floatScratch0);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathAtan2NumberResult(NumberOperandId yId,
+ NumberOperandId xId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, yId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, xId, floatScratch1);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = double (*)(double x, double y);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ masm.passABIArg(floatScratch1, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, js::ecmaAtan2>(MoveOp::DOUBLE);
+ masm.storeCallFloatResult(floatScratch0);
+
+ LiveRegisterSet ignore;
+ ignore.add(floatScratch0);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitMathFloorToInt32Result(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister scratchFloat(*this, FloatReg0);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ allocator.ensureDoubleRegister(masm, inputId, scratchFloat);
+
+ masm.floorDoubleToInt32(scratchFloat, scratch, failure->label());
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMathCeilToInt32Result(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister scratchFloat(*this, FloatReg0);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ allocator.ensureDoubleRegister(masm, inputId, scratchFloat);
+
+ masm.ceilDoubleToInt32(scratchFloat, scratch, failure->label());
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMathTruncToInt32Result(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister scratchFloat(*this, FloatReg0);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ allocator.ensureDoubleRegister(masm, inputId, scratchFloat);
+
+ masm.truncDoubleToInt32(scratchFloat, scratch, failure->label());
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMathRoundToInt32Result(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ AutoAvailableFloatRegister scratchFloat0(*this, FloatReg0);
+ AutoAvailableFloatRegister scratchFloat1(*this, FloatReg1);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ allocator.ensureDoubleRegister(masm, inputId, scratchFloat0);
+
+ masm.roundDoubleToInt32(scratchFloat0, scratch, scratchFloat1,
+ failure->label());
+
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32MinMax(bool isMax, Int32OperandId firstId,
+ Int32OperandId secondId,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register first = allocator.useRegister(masm, firstId);
+ Register second = allocator.useRegister(masm, secondId);
+ Register result = allocator.defineRegister(masm, resultId);
+
+ Assembler::Condition cond =
+ isMax ? Assembler::GreaterThan : Assembler::LessThan;
+ masm.move32(first, result);
+ masm.cmp32Move32(cond, second, first, second, result);
+ return true;
+}
+
+bool CacheIRCompiler::emitNumberMinMax(bool isMax, NumberOperandId firstId,
+ NumberOperandId secondId,
+ NumberOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+
+ AutoAvailableFloatRegister scratch1(*this, FloatReg0);
+ AutoAvailableFloatRegister scratch2(*this, FloatReg1);
+
+ allocator.ensureDoubleRegister(masm, firstId, scratch1);
+ allocator.ensureDoubleRegister(masm, secondId, scratch2);
+
+ if (isMax) {
+ masm.maxDouble(scratch2, scratch1, /* handleNaN = */ true);
+ } else {
+ masm.minDouble(scratch2, scratch1, /* handleNaN = */ true);
+ }
+
+ masm.boxDouble(scratch1, output, scratch1);
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32MinMaxArrayResult(ObjOperandId arrayId,
+ bool isMax) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register array = allocator.useRegister(masm, arrayId);
+
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegisterMaybeOutputType scratch3(allocator, masm, output);
+ AutoScratchRegisterMaybeOutput result(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.minMaxArrayInt32(array, result, scratch, scratch2, scratch3, isMax,
+ failure->label());
+ masm.tagValue(JSVAL_TYPE_INT32, result, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitNumberMinMaxArrayResult(ObjOperandId arrayId,
+ bool isMax) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register array = allocator.useRegister(masm, arrayId);
+
+ AutoAvailableFloatRegister result(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch(*this, FloatReg1);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.minMaxArrayNumber(array, result, floatScratch, scratch1, scratch2, isMax,
+ failure->label());
+ masm.boxDouble(result, output.valueReg(), result);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathFunctionNumberResultShared(
+ UnaryMathFunction fun, FloatRegister inputScratch, ValueOperand output) {
+ UnaryMathFunctionType funPtr = GetUnaryMathFunctionPtr(fun);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ save.takeUnchecked(inputScratch);
+ masm.PushRegsInMask(save);
+
+ masm.setupUnalignedABICall(output.scratchReg());
+ masm.passABIArg(inputScratch, MoveOp::DOUBLE);
+ masm.callWithABI(DynamicFunction<UnaryMathFunctionType>(funPtr),
+ MoveOp::DOUBLE);
+ masm.storeCallFloatResult(inputScratch);
+
+ masm.PopRegsInMask(save);
+
+ masm.boxDouble(inputScratch, output, inputScratch);
+ return true;
+}
+
+bool CacheIRCompiler::emitMathFunctionNumberResult(NumberOperandId inputId,
+ UnaryMathFunction fun) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoAvailableFloatRegister scratch(*this, FloatReg0);
+
+ allocator.ensureDoubleRegister(masm, inputId, scratch);
+
+ return emitMathFunctionNumberResultShared(fun, scratch, output.valueReg());
+}
+
+static void EmitStoreDenseElement(MacroAssembler& masm,
+ const ConstantOrRegister& value,
+ BaseObjectElementIndex target) {
+ if (value.constant()) {
+ Value v = value.value();
+ masm.storeValue(v, target);
+ return;
+ }
+
+ TypedOrValueRegister reg = value.reg();
+ masm.storeTypedOrValue(reg, target);
+}
+
+bool CacheIRCompiler::emitStoreDenseElement(ObjOperandId objId,
+ Int32OperandId indexId,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load obj->elements in scratch.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ // Bounds check. Unfortunately we don't have more registers available on
+ // x86, so use InvalidReg and emit slightly slower code on x86.
+ Register spectreTemp = InvalidReg;
+ Address initLength(scratch, ObjectElements::offsetOfInitializedLength());
+ masm.spectreBoundsCheck32(index, initLength, spectreTemp, failure->label());
+
+ // Hole check.
+ BaseObjectElementIndex element(scratch, index);
+ masm.branchTestMagic(Assembler::Equal, element, failure->label());
+
+ // Perform the store.
+ EmitPreBarrier(masm, element, MIRType::Value);
+ EmitStoreDenseElement(masm, val, element);
+
+ emitPostBarrierElement(obj, val, scratch, index);
+ return true;
+}
+
+static void EmitAssertExtensibleElements(MacroAssembler& masm,
+ Register elementsReg) {
+#ifdef DEBUG
+ // Preceding shape guards ensure the object elements are extensible.
+ Address elementsFlags(elementsReg, ObjectElements::offsetOfFlags());
+ Label ok;
+ masm.branchTest32(Assembler::Zero, elementsFlags,
+ Imm32(ObjectElements::Flags::NOT_EXTENSIBLE), &ok);
+ masm.assumeUnreachable("Unexpected non-extensible elements");
+ masm.bind(&ok);
+#endif
+}
+
+static void EmitAssertWritableArrayLengthElements(MacroAssembler& masm,
+ Register elementsReg) {
+#ifdef DEBUG
+ // Preceding shape guards ensure the array length is writable.
+ Address elementsFlags(elementsReg, ObjectElements::offsetOfFlags());
+ Label ok;
+ masm.branchTest32(Assembler::Zero, elementsFlags,
+ Imm32(ObjectElements::Flags::NONWRITABLE_ARRAY_LENGTH),
+ &ok);
+ masm.assumeUnreachable("Unexpected non-writable array length elements");
+ masm.bind(&ok);
+#endif
+}
+
+bool CacheIRCompiler::emitStoreDenseElementHole(ObjOperandId objId,
+ Int32OperandId indexId,
+ ValOperandId rhsId,
+ bool handleAdd) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load obj->elements in scratch.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ EmitAssertExtensibleElements(masm, scratch);
+ if (handleAdd) {
+ EmitAssertWritableArrayLengthElements(masm, scratch);
+ }
+
+ BaseObjectElementIndex element(scratch, index);
+ Address initLength(scratch, ObjectElements::offsetOfInitializedLength());
+ Address elementsFlags(scratch, ObjectElements::offsetOfFlags());
+
+ // We don't have enough registers on x86 so use InvalidReg. This will emit
+ // slightly less efficient code on x86.
+ Register spectreTemp = InvalidReg;
+
+ Label storeSkipPreBarrier;
+ if (handleAdd) {
+ // Bounds check.
+ Label inBounds, outOfBounds;
+ masm.spectreBoundsCheck32(index, initLength, spectreTemp, &outOfBounds);
+ masm.jump(&inBounds);
+
+ // If we're out-of-bounds, only handle the index == initLength case.
+ masm.bind(&outOfBounds);
+ masm.branch32(Assembler::NotEqual, initLength, index, failure->label());
+
+ // If index < capacity, we can add a dense element inline. If not we
+ // need to allocate more elements.
+ Label allocElement, addNewElement;
+ Address capacity(scratch, ObjectElements::offsetOfCapacity());
+ masm.spectreBoundsCheck32(index, capacity, spectreTemp, &allocElement);
+ masm.jump(&addNewElement);
+
+ masm.bind(&allocElement);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ save.takeUnchecked(scratch);
+ masm.PushRegsInMask(save);
+
+ using Fn = bool (*)(JSContext * cx, NativeObject * obj);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, NativeObject::addDenseElementPure>();
+ masm.storeCallPointerResult(scratch);
+
+ masm.PopRegsInMask(save);
+ masm.branchIfFalseBool(scratch, failure->label());
+
+ // Load the reallocated elements pointer.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ masm.bind(&addNewElement);
+
+ // Increment initLength.
+ masm.add32(Imm32(1), initLength);
+
+ // If length is now <= index, increment length too.
+ Label skipIncrementLength;
+ Address length(scratch, ObjectElements::offsetOfLength());
+ masm.branch32(Assembler::Above, length, index, &skipIncrementLength);
+ masm.add32(Imm32(1), length);
+ masm.bind(&skipIncrementLength);
+
+ // Skip EmitPreBarrier as the memory is uninitialized.
+ masm.jump(&storeSkipPreBarrier);
+
+ masm.bind(&inBounds);
+ } else {
+ // Fail if index >= initLength.
+ masm.spectreBoundsCheck32(index, initLength, spectreTemp, failure->label());
+ }
+
+ EmitPreBarrier(masm, element, MIRType::Value);
+
+ masm.bind(&storeSkipPreBarrier);
+ EmitStoreDenseElement(masm, val, element);
+
+ emitPostBarrierElement(obj, val, scratch, index);
+ return true;
+}
+
+bool CacheIRCompiler::emitArrayPush(ObjOperandId objId, ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ AutoScratchRegisterMaybeOutput scratchLength(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Load obj->elements in scratch.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ EmitAssertExtensibleElements(masm, scratch);
+ EmitAssertWritableArrayLengthElements(masm, scratch);
+
+ Address elementsInitLength(scratch,
+ ObjectElements::offsetOfInitializedLength());
+ Address elementsLength(scratch, ObjectElements::offsetOfLength());
+ Address elementsFlags(scratch, ObjectElements::offsetOfFlags());
+
+ // Fail if length != initLength.
+ masm.load32(elementsInitLength, scratchLength);
+ masm.branch32(Assembler::NotEqual, elementsLength, scratchLength,
+ failure->label());
+
+ // If scratchLength < capacity, we can add a dense element inline. If not we
+ // need to allocate more elements.
+ Label allocElement, addNewElement;
+ Address capacity(scratch, ObjectElements::offsetOfCapacity());
+ masm.spectreBoundsCheck32(scratchLength, capacity, InvalidReg, &allocElement);
+ masm.jump(&addNewElement);
+
+ masm.bind(&allocElement);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ save.takeUnchecked(scratch);
+ masm.PushRegsInMask(save);
+
+ using Fn = bool (*)(JSContext * cx, NativeObject * obj);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, NativeObject::addDenseElementPure>();
+ masm.storeCallPointerResult(scratch);
+
+ masm.PopRegsInMask(save);
+ masm.branchIfFalseBool(scratch, failure->label());
+
+ // Load the reallocated elements pointer.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), scratch);
+
+ masm.bind(&addNewElement);
+
+ // Increment initLength and length.
+ masm.add32(Imm32(1), elementsInitLength);
+ masm.add32(Imm32(1), elementsLength);
+
+ // Store the value.
+ BaseObjectElementIndex element(scratch, scratchLength);
+ masm.storeValue(val, element);
+ emitPostBarrierElement(obj, val, scratch, scratchLength);
+
+ // Return value is new length.
+ masm.add32(Imm32(1), scratchLength);
+ masm.tagValue(JSVAL_TYPE_INT32, scratchLength, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitStoreTypedArrayElement(ObjOperandId objId,
+ Scalar::Type elementType,
+ IntPtrOperandId indexId,
+ uint32_t rhsId,
+ bool handleOOB) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+
+ Maybe<Register> valInt32;
+ Maybe<Register> valBigInt;
+ switch (elementType) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Uint8Clamped:
+ valInt32.emplace(allocator.useRegister(masm, Int32OperandId(rhsId)));
+ break;
+
+ case Scalar::Float32:
+ case Scalar::Float64:
+ allocator.ensureDoubleRegister(masm, NumberOperandId(rhsId),
+ floatScratch0);
+ break;
+
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ valBigInt.emplace(allocator.useRegister(masm, BigIntOperandId(rhsId)));
+ break;
+
+ case Scalar::MaxTypedArrayViewType:
+ case Scalar::Int64:
+ case Scalar::Simd128:
+ MOZ_CRASH("Unsupported TypedArray type");
+ }
+
+ AutoScratchRegister scratch1(allocator, masm);
+ Maybe<AutoScratchRegister> scratch2;
+ Maybe<AutoSpectreBoundsScratchRegister> spectreScratch;
+ if (Scalar::isBigIntType(elementType)) {
+ scratch2.emplace(allocator, masm);
+ } else {
+ spectreScratch.emplace(allocator, masm);
+ }
+
+ FailurePath* failure = nullptr;
+ if (!handleOOB) {
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+ }
+
+ // Bounds check.
+ Label done;
+ Register spectreTemp = scratch2 ? scratch2->get() : spectreScratch->get();
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch1);
+ masm.spectreBoundsCheckPtr(index, scratch1, spectreTemp,
+ handleOOB ? &done : failure->label());
+
+ // Load the elements vector.
+ masm.loadPtr(Address(obj, ArrayBufferViewObject::dataOffset()), scratch1);
+
+ BaseIndex dest(scratch1, index, ScaleFromScalarType(elementType));
+
+ if (Scalar::isBigIntType(elementType)) {
+#ifdef JS_PUNBOX64
+ Register64 temp(scratch2->get());
+#else
+ // We don't have more registers available on x86, so spill |obj|.
+ masm.push(obj);
+ Register64 temp(scratch2->get(), obj);
+#endif
+
+ masm.loadBigInt64(*valBigInt, temp);
+ masm.storeToTypedBigIntArray(elementType, temp, dest);
+
+#ifndef JS_PUNBOX64
+ masm.pop(obj);
+#endif
+ } else if (elementType == Scalar::Float32) {
+ ScratchFloat32Scope fpscratch(masm);
+ masm.convertDoubleToFloat32(floatScratch0, fpscratch);
+ masm.storeToTypedFloatArray(elementType, fpscratch, dest);
+ } else if (elementType == Scalar::Float64) {
+ masm.storeToTypedFloatArray(elementType, floatScratch0, dest);
+ } else {
+ masm.storeToTypedIntArray(elementType, *valInt32, dest);
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+static gc::InitialHeap InitialBigIntHeap(JSContext* cx) {
+ JS::Zone* zone = cx->zone();
+ bool canNurseryAllocate =
+ zone->runtimeFromAnyThread()->gc.nursery().canAllocateBigInts() &&
+ zone->allocNurseryBigInts;
+ return canNurseryAllocate ? gc::DefaultHeap : gc::TenuredHeap;
+}
+
+static void EmitAllocateBigInt(MacroAssembler& masm, Register result,
+ Register temp, const LiveRegisterSet& liveSet,
+ gc::InitialHeap initialHeap, Label* fail) {
+ Label fallback, done;
+ masm.newGCBigInt(result, temp, initialHeap, &fallback);
+ masm.jump(&done);
+ {
+ masm.bind(&fallback);
+
+ // Request a minor collection at a later time if nursery allocation failed.
+ bool requestMinorGC = initialHeap == gc::DefaultHeap;
+
+ masm.PushRegsInMask(liveSet);
+ using Fn = void* (*)(JSContext * cx, bool requestMinorGC);
+ masm.setupUnalignedABICall(temp);
+ masm.loadJSContext(temp);
+ masm.passABIArg(temp);
+ masm.move32(Imm32(requestMinorGC), result);
+ masm.passABIArg(result);
+ masm.callWithABI<Fn, jit::AllocateBigIntNoGC>();
+ masm.storeCallPointerResult(result);
+
+ masm.PopRegsInMask(liveSet);
+ masm.branchPtr(Assembler::Equal, result, ImmWord(0), fail);
+ }
+ masm.bind(&done);
+}
+
+bool CacheIRCompiler::emitLoadTypedArrayElementResult(
+ ObjOperandId objId, IntPtrOperandId indexId, Scalar::Type elementType,
+ bool handleOOB, bool forceDoubleForUint32) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+#ifdef JS_PUNBOX64
+ AutoScratchRegister scratch2(allocator, masm);
+#else
+ // There are too few registers available on x86, so we may need to reuse the
+ // output's scratch register.
+ AutoScratchRegisterMaybeOutput scratch2(allocator, masm, output);
+#endif
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Bounds check.
+ Label outOfBounds;
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch1);
+ masm.spectreBoundsCheckPtr(index, scratch1, scratch2,
+ handleOOB ? &outOfBounds : failure->label());
+
+ // Allocate BigInt if needed. The code after this should be infallible.
+ Maybe<Register> bigInt;
+ if (Scalar::isBigIntType(elementType)) {
+ bigInt.emplace(output.valueReg().scratchReg());
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ save.takeUnchecked(scratch1);
+ save.takeUnchecked(scratch2);
+ save.takeUnchecked(output);
+
+ gc::InitialHeap initialHeap = InitialBigIntHeap(cx_);
+ EmitAllocateBigInt(masm, *bigInt, scratch1, save, initialHeap,
+ failure->label());
+ }
+
+ // Load the elements vector.
+ masm.loadPtr(Address(obj, ArrayBufferViewObject::dataOffset()), scratch1);
+
+ // Load the value.
+ BaseIndex source(scratch1, index, ScaleFromScalarType(elementType));
+
+ if (Scalar::isBigIntType(elementType)) {
+#ifdef JS_PUNBOX64
+ Register64 temp(scratch2);
+#else
+ // We don't have more registers available on x86, so spill |obj| and
+ // additionally use the output's type register.
+ MOZ_ASSERT(output.valueReg().scratchReg() != output.valueReg().typeReg());
+ masm.push(obj);
+ Register64 temp(output.valueReg().typeReg(), obj);
+#endif
+
+ masm.loadFromTypedBigIntArray(elementType, source, *bigInt, temp);
+
+#ifndef JS_PUNBOX64
+ masm.pop(obj);
+#endif
+
+ masm.tagValue(JSVAL_TYPE_BIGINT, *bigInt, output.valueReg());
+ } else {
+ MacroAssembler::Uint32Mode uint32Mode =
+ forceDoubleForUint32 ? MacroAssembler::Uint32Mode::ForceDouble
+ : MacroAssembler::Uint32Mode::FailOnDouble;
+ masm.loadFromTypedArray(elementType, source, output.valueReg(), uint32Mode,
+ scratch1, failure->label());
+ }
+
+ if (handleOOB) {
+ Label done;
+ masm.jump(&done);
+
+ masm.bind(&outOfBounds);
+ masm.moveValue(UndefinedValue(), output.valueReg());
+
+ masm.bind(&done);
+ }
+
+ return true;
+}
+
+static void EmitDataViewBoundsCheck(MacroAssembler& masm, size_t byteSize,
+ Register obj, Register offset,
+ Register scratch, Label* fail) {
+ // Ensure both offset < length and offset + (byteSize - 1) < length.
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ if (byteSize == 1) {
+ masm.spectreBoundsCheckPtr(offset, scratch, InvalidReg, fail);
+ } else {
+ // temp := length - (byteSize - 1)
+ // if temp < 0: fail
+ // if offset >= temp: fail
+ masm.branchSubPtr(Assembler::Signed, Imm32(byteSize - 1), scratch, fail);
+ masm.spectreBoundsCheckPtr(offset, scratch, InvalidReg, fail);
+ }
+}
+
+bool CacheIRCompiler::emitLoadDataViewValueResult(
+ ObjOperandId objId, IntPtrOperandId offsetId,
+ BooleanOperandId littleEndianId, Scalar::Type elementType,
+ bool forceDoubleForUint32) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register offset = allocator.useRegister(masm, offsetId);
+ Register littleEndian = allocator.useRegister(masm, littleEndianId);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+
+ Register64 outputReg64 = output.valueReg().toRegister64();
+ Register outputScratch = outputReg64.scratchReg();
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ const size_t byteSize = Scalar::byteSize(elementType);
+
+ EmitDataViewBoundsCheck(masm, byteSize, obj, offset, outputScratch,
+ failure->label());
+
+ masm.loadPtr(Address(obj, DataViewObject::dataOffset()), outputScratch);
+
+ // Load the value.
+ BaseIndex source(outputScratch, offset, TimesOne);
+ switch (elementType) {
+ case Scalar::Int8:
+ masm.load8SignExtend(source, outputScratch);
+ break;
+ case Scalar::Uint8:
+ masm.load8ZeroExtend(source, outputScratch);
+ break;
+ case Scalar::Int16:
+ masm.load16UnalignedSignExtend(source, outputScratch);
+ break;
+ case Scalar::Uint16:
+ masm.load16UnalignedZeroExtend(source, outputScratch);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ masm.load32Unaligned(source, outputScratch);
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.load64Unaligned(source, outputReg64);
+ break;
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+
+ // Swap the bytes in the loaded value.
+ if (byteSize > 1) {
+ Label skip;
+ masm.branch32(MOZ_LITTLE_ENDIAN() ? Assembler::NotEqual : Assembler::Equal,
+ littleEndian, Imm32(0), &skip);
+
+ switch (elementType) {
+ case Scalar::Int16:
+ masm.byteSwap16SignExtend(outputScratch);
+ break;
+ case Scalar::Uint16:
+ masm.byteSwap16ZeroExtend(outputScratch);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ masm.byteSwap32(outputScratch);
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.byteSwap64(outputReg64);
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid type");
+ }
+
+ masm.bind(&skip);
+ }
+
+ // Move the value into the output register.
+ switch (elementType) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ masm.tagValue(JSVAL_TYPE_INT32, outputScratch, output.valueReg());
+ break;
+ case Scalar::Uint32: {
+ MacroAssembler::Uint32Mode uint32Mode =
+ forceDoubleForUint32 ? MacroAssembler::Uint32Mode::ForceDouble
+ : MacroAssembler::Uint32Mode::FailOnDouble;
+ masm.boxUint32(outputScratch, output.valueReg(), uint32Mode,
+ failure->label());
+ break;
+ }
+ case Scalar::Float32: {
+ FloatRegister scratchFloat32 = floatScratch0.get().asSingle();
+ masm.moveGPRToFloat32(outputScratch, scratchFloat32);
+ masm.canonicalizeFloat(scratchFloat32);
+ masm.convertFloat32ToDouble(scratchFloat32, floatScratch0);
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+ break;
+ }
+ case Scalar::Float64:
+ masm.moveGPR64ToDouble(outputReg64, floatScratch0);
+ masm.canonicalizeDouble(floatScratch0);
+ masm.boxDouble(floatScratch0, output.valueReg(), floatScratch0);
+ break;
+ case Scalar::BigInt64:
+ case Scalar::BigUint64: {
+ // We need two extra registers. Reuse the obj/littleEndian registers.
+ Register bigInt = obj;
+ Register bigIntScratch = littleEndian;
+ masm.push(bigInt);
+ masm.push(bigIntScratch);
+ Label fail, done;
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ save.takeUnchecked(bigInt);
+ save.takeUnchecked(bigIntScratch);
+ gc::InitialHeap initialHeap = InitialBigIntHeap(cx_);
+ EmitAllocateBigInt(masm, bigInt, bigIntScratch, save, initialHeap, &fail);
+ masm.jump(&done);
+
+ masm.bind(&fail);
+ masm.pop(bigIntScratch);
+ masm.pop(bigInt);
+ masm.jump(failure->label());
+
+ masm.bind(&done);
+ masm.initializeBigInt64(elementType, bigInt, outputReg64);
+ masm.tagValue(JSVAL_TYPE_BIGINT, bigInt, output.valueReg());
+ masm.pop(bigIntScratch);
+ masm.pop(bigInt);
+ break;
+ }
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+
+ return true;
+}
+
+bool CacheIRCompiler::emitStoreDataViewValueResult(
+ ObjOperandId objId, IntPtrOperandId offsetId, uint32_t valueId,
+ BooleanOperandId littleEndianId, Scalar::Type elementType) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+#ifdef JS_CODEGEN_X86
+ // Use a scratch register to avoid running out of the registers.
+ Register obj = output.valueReg().typeReg();
+ allocator.copyToScratchRegister(masm, objId, obj);
+#else
+ Register obj = allocator.useRegister(masm, objId);
+#endif
+ Register offset = allocator.useRegister(masm, offsetId);
+ Register littleEndian = allocator.useRegister(masm, littleEndianId);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ Maybe<Register> valInt32;
+ Maybe<Register> valBigInt;
+ switch (elementType) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Uint8Clamped:
+ valInt32.emplace(allocator.useRegister(masm, Int32OperandId(valueId)));
+ break;
+
+ case Scalar::Float32:
+ case Scalar::Float64:
+ allocator.ensureDoubleRegister(masm, NumberOperandId(valueId),
+ floatScratch0);
+ break;
+
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ valBigInt.emplace(allocator.useRegister(masm, BigIntOperandId(valueId)));
+ break;
+
+ case Scalar::MaxTypedArrayViewType:
+ case Scalar::Int64:
+ case Scalar::Simd128:
+ MOZ_CRASH("Unsupported type");
+ }
+
+ Register scratch1 = output.valueReg().scratchReg();
+ MOZ_ASSERT(scratch1 != obj, "scratchReg must not be typeReg");
+
+ // On platforms with enough registers, |scratch2| is an extra scratch register
+ // (pair) used for byte-swapping the value.
+#ifndef JS_CODEGEN_X86
+ mozilla::MaybeOneOf<AutoScratchRegister, AutoScratchRegister64> scratch2;
+ switch (elementType) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ scratch2.construct<AutoScratchRegister>(allocator, masm);
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ scratch2.construct<AutoScratchRegister64>(allocator, masm);
+ break;
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid type");
+ }
+#endif
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ const size_t byteSize = Scalar::byteSize(elementType);
+
+ EmitDataViewBoundsCheck(masm, byteSize, obj, offset, scratch1,
+ failure->label());
+
+ masm.loadPtr(Address(obj, DataViewObject::dataOffset()), scratch1);
+ BaseIndex dest(scratch1, offset, TimesOne);
+
+ if (byteSize == 1) {
+ // Byte swapping has no effect, so just do the byte store.
+ masm.store8(*valInt32, dest);
+ masm.moveValue(UndefinedValue(), output.valueReg());
+ return true;
+ }
+
+ // On 32-bit x86, |obj| is already a scratch register so use that. If we need
+ // a Register64 we also use the littleEndian register and use the stack
+ // location for the check below.
+ bool pushedLittleEndian = false;
+#ifdef JS_CODEGEN_X86
+ if (byteSize == 8) {
+ masm.push(littleEndian);
+ pushedLittleEndian = true;
+ }
+ auto valScratch32 = [&]() -> Register { return obj; };
+ auto valScratch64 = [&]() -> Register64 {
+ return Register64(obj, littleEndian);
+ };
+#else
+ auto valScratch32 = [&]() -> Register {
+ return scratch2.ref<AutoScratchRegister>();
+ };
+ auto valScratch64 = [&]() -> Register64 {
+ return scratch2.ref<AutoScratchRegister64>();
+ };
+#endif
+
+ // Load the value into a gpr register.
+ switch (elementType) {
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ masm.move32(*valInt32, valScratch32());
+ break;
+ case Scalar::Float32: {
+ FloatRegister scratchFloat32 = floatScratch0.get().asSingle();
+ masm.convertDoubleToFloat32(floatScratch0, scratchFloat32);
+ masm.canonicalizeFloatIfDeterministic(scratchFloat32);
+ masm.moveFloat32ToGPR(scratchFloat32, valScratch32());
+ break;
+ }
+ case Scalar::Float64: {
+ masm.canonicalizeDoubleIfDeterministic(floatScratch0);
+ masm.moveDoubleToGPR64(floatScratch0, valScratch64());
+ break;
+ }
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.loadBigInt64(*valBigInt, valScratch64());
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid type");
+ }
+
+ // Swap the bytes in the loaded value.
+ Label skip;
+ if (pushedLittleEndian) {
+ masm.branch32(MOZ_LITTLE_ENDIAN() ? Assembler::NotEqual : Assembler::Equal,
+ Address(masm.getStackPointer(), 0), Imm32(0), &skip);
+ } else {
+ masm.branch32(MOZ_LITTLE_ENDIAN() ? Assembler::NotEqual : Assembler::Equal,
+ littleEndian, Imm32(0), &skip);
+ }
+ switch (elementType) {
+ case Scalar::Int16:
+ masm.byteSwap16SignExtend(valScratch32());
+ break;
+ case Scalar::Uint16:
+ masm.byteSwap16ZeroExtend(valScratch32());
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ masm.byteSwap32(valScratch32());
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.byteSwap64(valScratch64());
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid type");
+ }
+ masm.bind(&skip);
+
+ // Store the value.
+ switch (elementType) {
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ masm.store16Unaligned(valScratch32(), dest);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ masm.store32Unaligned(valScratch32(), dest);
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.store64Unaligned(valScratch64(), dest);
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+
+#ifdef JS_CODEGEN_X86
+ // Restore registers.
+ if (pushedLittleEndian) {
+ masm.pop(littleEndian);
+ }
+#endif
+
+ masm.moveValue(UndefinedValue(), output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitStoreFixedSlotUndefinedResult(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ StubFieldOffset offset(offsetOffset, StubField::Type::RawInt32);
+ emitLoadStubField(offset, scratch);
+
+ BaseIndex slot(obj, scratch, TimesOne);
+ EmitPreBarrier(masm, slot, MIRType::Value);
+ masm.storeValue(val, slot);
+ emitPostBarrierSlot(obj, val, scratch);
+
+ masm.moveValue(UndefinedValue(), output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadObjectResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+
+ EmitStoreResult(masm, obj, JSVAL_TYPE_OBJECT, output);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadStringResult(StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register str = allocator.useRegister(masm, strId);
+
+ masm.tagValue(JSVAL_TYPE_STRING, str, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadSymbolResult(SymbolOperandId symId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register sym = allocator.useRegister(masm, symId);
+
+ masm.tagValue(JSVAL_TYPE_SYMBOL, sym, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadInt32Result(Int32OperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register val = allocator.useRegister(masm, valId);
+
+ masm.tagValue(JSVAL_TYPE_INT32, val, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadBigIntResult(BigIntOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register val = allocator.useRegister(masm, valId);
+
+ masm.tagValue(JSVAL_TYPE_BIGINT, val, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDoubleResult(NumberOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+#ifdef DEBUG
+ Label ok;
+ masm.branchTestDouble(Assembler::Equal, val, &ok);
+ masm.branchTestInt32(Assembler::Equal, val, &ok);
+ masm.assumeUnreachable("input must be double or int32");
+ masm.bind(&ok);
+#endif
+
+ masm.moveValue(val, output.valueReg());
+ masm.convertInt32ValueToDouble(output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadTypeOfObjectResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Label slowCheck, isObject, isCallable, isUndefined, done;
+ masm.typeOfObject(obj, scratch, &slowCheck, &isObject, &isCallable,
+ &isUndefined);
+
+ masm.bind(&isCallable);
+ masm.moveValue(StringValue(cx_->names().function), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&isUndefined);
+ masm.moveValue(StringValue(cx_->names().undefined), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&isObject);
+ masm.moveValue(StringValue(cx_->names().object), output.valueReg());
+ masm.jump(&done);
+
+ {
+ masm.bind(&slowCheck);
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = JSString* (*)(JSObject * obj, JSRuntime * rt);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(obj);
+ masm.movePtr(ImmPtr(cx_->runtime()), scratch);
+ masm.passABIArg(scratch);
+ masm.callWithABI<Fn, TypeOfNameObject>();
+ masm.storeCallPointerResult(scratch);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg());
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadInt32TruthyResult(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ ValueOperand val = allocator.useValueRegister(masm, inputId);
+
+ Label ifFalse, done;
+ masm.branchTestInt32Truthy(false, val, &ifFalse);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&ifFalse);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadStringTruthyResult(StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register str = allocator.useRegister(masm, strId);
+
+ Label ifFalse, done;
+ masm.branch32(Assembler::Equal, Address(str, JSString::offsetOfLength()),
+ Imm32(0), &ifFalse);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&ifFalse);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDoubleTruthyResult(NumberOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ AutoScratchFloatRegister floatReg(this);
+
+ allocator.ensureDoubleRegister(masm, inputId, floatReg);
+
+ Label ifFalse, done;
+
+ masm.branchTestDoubleTruthy(false, floatReg, &ifFalse);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&ifFalse);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadObjectTruthyResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Label emulatesUndefined, slowPath, done;
+ masm.branchIfObjectEmulatesUndefined(obj, scratch, &slowPath,
+ &emulatesUndefined);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&emulatesUndefined);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&slowPath);
+ {
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch);
+ volatileRegs.takeUnchecked(output);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSObject * obj);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, js::EmulatesUndefined>();
+ masm.storeCallBoolResult(scratch);
+ masm.xor32(Imm32(1), scratch);
+
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadBigIntTruthyResult(BigIntOperandId bigIntId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register bigInt = allocator.useRegister(masm, bigIntId);
+
+ Label ifFalse, done;
+ masm.branch32(Assembler::Equal,
+ Address(bigInt, BigInt::offsetOfDigitLength()), Imm32(0),
+ &ifFalse);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&ifFalse);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadValueTruthyResult(ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ ValueOperand value = allocator.useValueRegister(masm, inputId);
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchFloatRegister floatReg(this);
+
+ Label ifFalse, ifTrue, done;
+
+ {
+ ScratchTagScope tag(masm, value);
+ masm.splitTagForTest(value, tag);
+
+ masm.branchTestUndefined(Assembler::Equal, tag, &ifFalse);
+ masm.branchTestNull(Assembler::Equal, tag, &ifFalse);
+
+ Label notBoolean;
+ masm.branchTestBoolean(Assembler::NotEqual, tag, &notBoolean);
+ {
+ ScratchTagScopeRelease _(&tag);
+ masm.branchTestBooleanTruthy(false, value, &ifFalse);
+ masm.jump(&ifTrue);
+ }
+ masm.bind(&notBoolean);
+
+ Label notInt32;
+ masm.branchTestInt32(Assembler::NotEqual, tag, &notInt32);
+ {
+ ScratchTagScopeRelease _(&tag);
+ masm.branchTestInt32Truthy(false, value, &ifFalse);
+ masm.jump(&ifTrue);
+ }
+ masm.bind(&notInt32);
+
+ Label notObject;
+ masm.branchTestObject(Assembler::NotEqual, tag, &notObject);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ Register obj = masm.extractObject(value, scratch1);
+
+ Label slowPath;
+ masm.branchIfObjectEmulatesUndefined(obj, scratch2, &slowPath, &ifFalse);
+ masm.jump(&ifTrue);
+
+ masm.bind(&slowPath);
+ {
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch1);
+ volatileRegs.takeUnchecked(scratch2);
+ volatileRegs.takeUnchecked(output);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSObject * obj);
+ masm.setupUnalignedABICall(scratch2);
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, js::EmulatesUndefined>();
+ masm.storeCallPointerResult(scratch2);
+
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.branchIfTrueBool(scratch2, &ifFalse);
+ masm.jump(&ifTrue);
+ }
+ }
+ masm.bind(&notObject);
+
+ Label notString;
+ masm.branchTestString(Assembler::NotEqual, tag, &notString);
+ {
+ ScratchTagScopeRelease _(&tag);
+ masm.branchTestStringTruthy(false, value, &ifFalse);
+ masm.jump(&ifTrue);
+ }
+ masm.bind(&notString);
+
+ Label notBigInt;
+ masm.branchTestBigInt(Assembler::NotEqual, tag, &notBigInt);
+ {
+ ScratchTagScopeRelease _(&tag);
+ masm.branchTestBigIntTruthy(false, value, &ifFalse);
+ masm.jump(&ifTrue);
+ }
+ masm.bind(&notBigInt);
+
+ masm.branchTestSymbol(Assembler::Equal, tag, &ifTrue);
+
+#ifdef DEBUG
+ Label isDouble;
+ masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
+ masm.assumeUnreachable("Unexpected value type");
+ masm.bind(&isDouble);
+#endif
+
+ {
+ ScratchTagScopeRelease _(&tag);
+ masm.unboxDouble(value, floatReg);
+ masm.branchTestDoubleTruthy(false, floatReg, &ifFalse);
+ }
+
+ // Fall through to true case.
+ }
+
+ masm.bind(&ifTrue);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&ifFalse);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitComparePointerResultShared(JSOp op,
+ TypedOperandId lhsId,
+ TypedOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ Register left = allocator.useRegister(masm, lhsId);
+ Register right = allocator.useRegister(masm, rhsId);
+
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Label ifTrue, done;
+ masm.branchPtr(JSOpToCondition(op, /* signed = */ true), left, right,
+ &ifTrue);
+
+ EmitStoreBoolean(masm, false, output);
+ masm.jump(&done);
+
+ masm.bind(&ifTrue);
+ EmitStoreBoolean(masm, true, output);
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitCompareObjectResult(JSOp op, ObjOperandId lhsId,
+ ObjOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitComparePointerResultShared(op, lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitCompareSymbolResult(JSOp op, SymbolOperandId lhsId,
+ SymbolOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitComparePointerResultShared(op, lhsId, rhsId);
+}
+
+bool CacheIRCompiler::emitCompareInt32Result(JSOp op, Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register left = allocator.useRegister(masm, lhsId);
+ Register right = allocator.useRegister(masm, rhsId);
+
+ Label ifTrue, done;
+ masm.branch32(JSOpToCondition(op, /* signed = */ true), left, right, &ifTrue);
+
+ EmitStoreBoolean(masm, false, output);
+ masm.jump(&done);
+
+ masm.bind(&ifTrue);
+ EmitStoreBoolean(masm, true, output);
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitCompareDoubleResult(JSOp op, NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ allocator.ensureDoubleRegister(masm, lhsId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch1);
+
+ Label done, ifTrue;
+ masm.branchDouble(JSOpToDoubleCondition(op), floatScratch0, floatScratch1,
+ &ifTrue);
+ EmitStoreBoolean(masm, false, output);
+ masm.jump(&done);
+
+ masm.bind(&ifTrue);
+ EmitStoreBoolean(masm, true, output);
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitCompareBigIntResult(JSOp op, BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ masm.setupUnalignedABICall(scratch);
+
+ // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
+ // - |left <= right| is implemented as |right >= left|.
+ // - |left > right| is implemented as |right < left|.
+ if (op == JSOp::Le || op == JSOp::Gt) {
+ masm.passABIArg(rhs);
+ masm.passABIArg(lhs);
+ } else {
+ masm.passABIArg(lhs);
+ masm.passABIArg(rhs);
+ }
+
+ using Fn = bool (*)(BigInt*, BigInt*);
+ Fn fn;
+ if (op == JSOp::Eq || op == JSOp::StrictEq) {
+ fn = jit::BigIntEqual<EqualityKind::Equal>;
+ } else if (op == JSOp::Ne || op == JSOp::StrictNe) {
+ fn = jit::BigIntEqual<EqualityKind::NotEqual>;
+ } else if (op == JSOp::Lt || op == JSOp::Gt) {
+ fn = jit::BigIntCompare<ComparisonKind::LessThan>;
+ } else {
+ MOZ_ASSERT(op == JSOp::Le || op == JSOp::Ge);
+ fn = jit::BigIntCompare<ComparisonKind::GreaterThanOrEqual>;
+ }
+
+ masm.callWithABI(DynamicFunction<Fn>(fn));
+ masm.storeCallBoolResult(scratch);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ EmitStoreResult(masm, scratch, JSVAL_TYPE_BOOLEAN, output);
+ return true;
+}
+
+bool CacheIRCompiler::emitCompareBigIntInt32Result(JSOp op,
+ BigIntOperandId lhsId,
+ Int32OperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register bigInt = allocator.useRegister(masm, lhsId);
+ Register int32 = allocator.useRegister(masm, rhsId);
+
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ Label ifTrue, ifFalse;
+ masm.compareBigIntAndInt32(op, bigInt, int32, scratch1, scratch2, &ifTrue,
+ &ifFalse);
+
+ Label done;
+ masm.bind(&ifFalse);
+ EmitStoreBoolean(masm, false, output);
+ masm.jump(&done);
+
+ masm.bind(&ifTrue);
+ EmitStoreBoolean(masm, true, output);
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitCompareBigIntNumberResult(JSOp op,
+ BigIntOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch0);
+
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ masm.setupUnalignedABICall(scratch);
+
+ // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
+ // - |left <= right| is implemented as |right >= left|.
+ // - |left > right| is implemented as |right < left|.
+ if (op == JSOp::Le || op == JSOp::Gt) {
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ masm.passABIArg(lhs);
+ } else {
+ masm.passABIArg(lhs);
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ }
+
+ using FnBigIntNumber = bool (*)(BigInt*, double);
+ using FnNumberBigInt = bool (*)(double, BigInt*);
+ switch (op) {
+ case JSOp::Eq: {
+ masm.callWithABI<FnBigIntNumber,
+ jit::BigIntNumberEqual<EqualityKind::Equal>>();
+ break;
+ }
+ case JSOp::Ne: {
+ masm.callWithABI<FnBigIntNumber,
+ jit::BigIntNumberEqual<EqualityKind::NotEqual>>();
+ break;
+ }
+ case JSOp::Lt: {
+ masm.callWithABI<FnBigIntNumber,
+ jit::BigIntNumberCompare<ComparisonKind::LessThan>>();
+ break;
+ }
+ case JSOp::Gt: {
+ masm.callWithABI<FnNumberBigInt,
+ jit::NumberBigIntCompare<ComparisonKind::LessThan>>();
+ break;
+ }
+ case JSOp::Le: {
+ masm.callWithABI<
+ FnNumberBigInt,
+ jit::NumberBigIntCompare<ComparisonKind::GreaterThanOrEqual>>();
+ break;
+ }
+ case JSOp::Ge: {
+ masm.callWithABI<
+ FnBigIntNumber,
+ jit::BigIntNumberCompare<ComparisonKind::GreaterThanOrEqual>>();
+ break;
+ }
+ default:
+ MOZ_CRASH("unhandled op");
+ }
+
+ masm.storeCallBoolResult(scratch);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ EmitStoreResult(masm, scratch, JSVAL_TYPE_BOOLEAN, output);
+ return true;
+}
+
+bool CacheIRCompiler::emitCompareBigIntStringResult(JSOp op,
+ BigIntOperandId lhsId,
+ StringOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ callvm.prepare();
+
+ // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
+ // - |left <= right| is implemented as |right >= left|.
+ // - |left > right| is implemented as |right < left|.
+ if (op == JSOp::Le || op == JSOp::Gt) {
+ masm.Push(lhs);
+ masm.Push(rhs);
+ } else {
+ masm.Push(rhs);
+ masm.Push(lhs);
+ }
+
+ using FnBigIntString =
+ bool (*)(JSContext*, HandleBigInt, HandleString, bool*);
+ using FnStringBigInt =
+ bool (*)(JSContext*, HandleString, HandleBigInt, bool*);
+
+ switch (op) {
+ case JSOp::Eq: {
+ constexpr auto Equal = EqualityKind::Equal;
+ callvm.call<FnBigIntString, BigIntStringEqual<Equal>>();
+ break;
+ }
+ case JSOp::Ne: {
+ constexpr auto NotEqual = EqualityKind::NotEqual;
+ callvm.call<FnBigIntString, BigIntStringEqual<NotEqual>>();
+ break;
+ }
+ case JSOp::Lt: {
+ constexpr auto LessThan = ComparisonKind::LessThan;
+ callvm.call<FnBigIntString, BigIntStringCompare<LessThan>>();
+ break;
+ }
+ case JSOp::Gt: {
+ constexpr auto LessThan = ComparisonKind::LessThan;
+ callvm.call<FnStringBigInt, StringBigIntCompare<LessThan>>();
+ break;
+ }
+ case JSOp::Le: {
+ constexpr auto GreaterThanOrEqual = ComparisonKind::GreaterThanOrEqual;
+ callvm.call<FnStringBigInt, StringBigIntCompare<GreaterThanOrEqual>>();
+ break;
+ }
+ case JSOp::Ge: {
+ constexpr auto GreaterThanOrEqual = ComparisonKind::GreaterThanOrEqual;
+ callvm.call<FnBigIntString, BigIntStringCompare<GreaterThanOrEqual>>();
+ break;
+ }
+ default:
+ MOZ_CRASH("unhandled op");
+ }
+ return true;
+}
+
+bool CacheIRCompiler::emitCompareNullUndefinedResult(JSOp op, bool isUndefined,
+ ValOperandId inputId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ ValueOperand input = allocator.useValueRegister(masm, inputId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ if (IsStrictEqualityOp(op)) {
+ if (isUndefined) {
+ masm.testUndefinedSet(JSOpToCondition(op, false), input, scratch);
+ } else {
+ masm.testNullSet(JSOpToCondition(op, false), input, scratch);
+ }
+ EmitStoreResult(masm, scratch, JSVAL_TYPE_BOOLEAN, output);
+ return true;
+ }
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ MOZ_ASSERT(IsLooseEqualityOp(op));
+
+ Label nullOrLikeUndefined, notNullOrLikeUndefined, done;
+ {
+ ScratchTagScope tag(masm, input);
+ masm.splitTagForTest(input, tag);
+
+ if (isUndefined) {
+ masm.branchTestUndefined(Assembler::Equal, tag, &nullOrLikeUndefined);
+ masm.branchTestNull(Assembler::Equal, tag, &nullOrLikeUndefined);
+ } else {
+ masm.branchTestNull(Assembler::Equal, tag, &nullOrLikeUndefined);
+ masm.branchTestUndefined(Assembler::Equal, tag, &nullOrLikeUndefined);
+ }
+ masm.branchTestObject(Assembler::NotEqual, tag, &notNullOrLikeUndefined);
+
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ masm.unboxObject(input, scratch);
+ masm.branchIfObjectEmulatesUndefined(scratch, scratch, failure->label(),
+ &nullOrLikeUndefined);
+ masm.jump(&notNullOrLikeUndefined);
+ }
+ }
+
+ masm.bind(&nullOrLikeUndefined);
+ EmitStoreBoolean(masm, op == JSOp::Eq, output);
+ masm.jump(&done);
+
+ masm.bind(&notNullOrLikeUndefined);
+ EmitStoreBoolean(masm, op == JSOp::Ne, output);
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitCompareDoubleSameValueResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+ AutoAvailableFloatRegister floatScratch1(*this, FloatReg1);
+ AutoAvailableFloatRegister floatScratch2(*this, FloatReg2);
+
+ allocator.ensureDoubleRegister(masm, lhsId, floatScratch0);
+ allocator.ensureDoubleRegister(masm, rhsId, floatScratch1);
+
+ masm.sameValueDouble(floatScratch0, floatScratch1, floatScratch2, scratch);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitIndirectTruncateInt32Result(Int32OperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register val = allocator.useRegister(masm, valId);
+
+ if (output.hasValue()) {
+ masm.tagValue(JSVAL_TYPE_INT32, val, output.valueReg());
+ } else {
+ masm.mov(val, output.typedReg().gpr());
+ }
+ return true;
+}
+
+bool CacheIRCompiler::emitCallPrintString(const char* str) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ masm.printf(str);
+ return true;
+}
+
+bool CacheIRCompiler::emitBreakpoint() {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ masm.breakpoint();
+ return true;
+}
+
+void CacheIRCompiler::emitPostBarrierShared(Register obj,
+ const ConstantOrRegister& val,
+ Register scratch,
+ Register maybeIndex) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (val.constant()) {
+ MOZ_ASSERT_IF(val.value().isGCThing(),
+ !IsInsideNursery(val.value().toGCThing()));
+ return;
+ }
+
+ TypedOrValueRegister reg = val.reg();
+ if (reg.hasTyped() && !NeedsPostBarrier(reg.type())) {
+ return;
+ }
+
+ Label skipBarrier;
+ if (reg.hasValue()) {
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, reg.valueReg(), scratch,
+ &skipBarrier);
+ } else {
+ masm.branchPtrInNurseryChunk(Assembler::NotEqual, reg.typedReg().gpr(),
+ scratch, &skipBarrier);
+ }
+ masm.branchPtrInNurseryChunk(Assembler::Equal, obj, scratch, &skipBarrier);
+
+ // Call one of these, depending on maybeIndex:
+ //
+ // void PostWriteBarrier(JSRuntime* rt, JSObject* obj);
+ // void PostWriteElementBarrier(JSRuntime* rt, JSObject* obj,
+ // int32_t index);
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+ masm.setupUnalignedABICall(scratch);
+ masm.movePtr(ImmPtr(cx_->runtime()), scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(obj);
+ if (maybeIndex != InvalidReg) {
+ masm.passABIArg(maybeIndex);
+ using Fn = void (*)(JSRuntime * rt, JSObject * obj, int32_t index);
+ masm.callWithABI<Fn, PostWriteElementBarrier<IndexInBounds::Yes>>();
+ } else {
+ using Fn = void (*)(JSRuntime * rt, js::gc::Cell * cell);
+ masm.callWithABI<Fn, PostWriteBarrier>();
+ }
+ masm.PopRegsInMask(save);
+
+ masm.bind(&skipBarrier);
+}
+
+bool CacheIRCompiler::emitWrapResult() {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label done;
+ // We only have to wrap objects, because we are in the same zone.
+ masm.branchTestObject(Assembler::NotEqual, output.valueReg(), &done);
+
+ Register obj = output.valueReg().scratchReg();
+ masm.unboxObject(output.valueReg(), obj);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(), liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = JSObject* (*)(JSContext * cx, JSObject * obj);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, WrapObjectPure>();
+ masm.storeCallPointerResult(obj);
+
+ LiveRegisterSet ignore;
+ ignore.add(obj);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ // We could not get a wrapper for this object.
+ masm.branchTestPtr(Assembler::Zero, obj, obj, failure->label());
+
+ // We clobbered the output register, so we have to retag.
+ masm.tagValue(JSVAL_TYPE_OBJECT, obj, output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitMegamorphicLoadSlotByValueResult(ObjOperandId objId,
+ ValOperandId idId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand idVal = allocator.useValueRegister(masm, idId);
+
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // idVal will be in vp[0], result will be stored in vp[1].
+ masm.reserveStack(sizeof(Value));
+ masm.Push(idVal);
+ masm.moveStackPtrTo(idVal.scratchReg());
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch);
+ volatileRegs.takeUnchecked(idVal);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, Value * vp);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(obj);
+ masm.passABIArg(idVal.scratchReg());
+ masm.callWithABI<Fn, GetNativeDataPropertyByValuePure>();
+
+ masm.storeCallPointerResult(scratch);
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.Pop(idVal);
+
+ Label ok;
+ uint32_t framePushed = masm.framePushed();
+ masm.branchIfTrueBool(scratch, &ok);
+ masm.adjustStack(sizeof(Value));
+ masm.jump(failure->label());
+
+ masm.bind(&ok);
+ masm.setFramePushed(framePushed);
+ masm.loadTypedOrValue(Address(masm.getStackPointer(), 0), output);
+ masm.adjustStack(sizeof(Value));
+ return true;
+}
+
+bool CacheIRCompiler::emitMegamorphicHasPropResult(ObjOperandId objId,
+ ValOperandId idId,
+ bool hasOwn) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand idVal = allocator.useValueRegister(masm, idId);
+
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // idVal will be in vp[0], result will be stored in vp[1].
+ masm.reserveStack(sizeof(Value));
+ masm.Push(idVal);
+ masm.moveStackPtrTo(idVal.scratchReg());
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch);
+ volatileRegs.takeUnchecked(idVal);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, Value * vp);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(obj);
+ masm.passABIArg(idVal.scratchReg());
+ if (hasOwn) {
+ masm.callWithABI<Fn, HasNativeDataPropertyPure<true>>();
+ } else {
+ masm.callWithABI<Fn, HasNativeDataPropertyPure<false>>();
+ }
+ masm.storeCallPointerResult(scratch);
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.Pop(idVal);
+
+ Label ok;
+ uint32_t framePushed = masm.framePushed();
+ masm.branchIfTrueBool(scratch, &ok);
+ masm.adjustStack(sizeof(Value));
+ masm.jump(failure->label());
+
+ masm.bind(&ok);
+ masm.setFramePushed(framePushed);
+ masm.loadTypedOrValue(Address(masm.getStackPointer(), 0), output);
+ masm.adjustStack(sizeof(Value));
+ return true;
+}
+
+bool CacheIRCompiler::emitCallObjectHasSparseElementResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.reserveStack(sizeof(Value));
+ masm.moveStackPtrTo(scratch2.get());
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch1);
+ volatileRegs.takeUnchecked(index);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn =
+ bool (*)(JSContext * cx, NativeObject * obj, int32_t index, Value * vp);
+ masm.setupUnalignedABICall(scratch1);
+ masm.loadJSContext(scratch1);
+ masm.passABIArg(scratch1);
+ masm.passABIArg(obj);
+ masm.passABIArg(index);
+ masm.passABIArg(scratch2);
+ masm.callWithABI<Fn, HasNativeElementPure>();
+ masm.storeCallPointerResult(scratch1);
+ masm.PopRegsInMask(volatileRegs);
+
+ Label ok;
+ uint32_t framePushed = masm.framePushed();
+ masm.branchIfTrueBool(scratch1, &ok);
+ masm.adjustStack(sizeof(Value));
+ masm.jump(failure->label());
+
+ masm.bind(&ok);
+ masm.setFramePushed(framePushed);
+ masm.loadTypedOrValue(Address(masm.getStackPointer(), 0), output);
+ masm.adjustStack(sizeof(Value));
+ return true;
+}
+
+/*
+ * Move a constant value into register dest.
+ */
+void CacheIRCompiler::emitLoadStubFieldConstant(StubFieldOffset val,
+ Register dest) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ MOZ_ASSERT(mode_ == Mode::Ion);
+ switch (val.getStubFieldType()) {
+ case StubField::Type::Shape:
+ masm.movePtr(ImmGCPtr(shapeStubField(val.getOffset())), dest);
+ break;
+ case StubField::Type::GetterSetter:
+ masm.movePtr(ImmGCPtr(getterSetterStubField(val.getOffset())), dest);
+ break;
+ case StubField::Type::String:
+ masm.movePtr(ImmGCPtr(stringStubField(val.getOffset())), dest);
+ break;
+ case StubField::Type::JSObject:
+ masm.movePtr(ImmGCPtr(objectStubField(val.getOffset())), dest);
+ break;
+ case StubField::Type::RawPointer:
+ masm.movePtr(ImmPtr(pointerStubField(val.getOffset())), dest);
+ break;
+ case StubField::Type::RawInt32:
+ masm.move32(Imm32(int32StubField(val.getOffset())), dest);
+ break;
+ case StubField::Type::Id:
+ masm.movePropertyKey(idStubField(val.getOffset()), dest);
+ break;
+ default:
+ MOZ_CRASH("Unhandled stub field constant type");
+ }
+}
+
+/*
+ * After this is done executing, dest contains the value; either through a
+ * constant load or through the load from the stub data.
+ *
+ * The current policy is that Baseline will use loads from the stub data (to
+ * allow IC sharing), where as Ion doesn't share ICs, and so we can safely use
+ * constants in the IC.
+ */
+void CacheIRCompiler::emitLoadStubField(StubFieldOffset val, Register dest) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ if (stubFieldPolicy_ == StubFieldPolicy::Constant) {
+ emitLoadStubFieldConstant(val, dest);
+ } else {
+ Address load(ICStubReg, stubDataOffset_ + val.getOffset());
+
+ switch (val.getStubFieldType()) {
+ case StubField::Type::RawPointer:
+ case StubField::Type::Shape:
+ case StubField::Type::GetterSetter:
+ case StubField::Type::JSObject:
+ case StubField::Type::Symbol:
+ case StubField::Type::String:
+ case StubField::Type::Id:
+ masm.loadPtr(load, dest);
+ break;
+ case StubField::Type::RawInt32:
+ masm.load32(load, dest);
+ break;
+ default:
+ MOZ_CRASH("Unhandled stub field constant type");
+ }
+ }
+}
+
+void CacheIRCompiler::emitLoadValueStubField(StubFieldOffset val,
+ ValueOperand dest) {
+ MOZ_ASSERT(val.getStubFieldType() == StubField::Type::Value);
+
+ if (stubFieldPolicy_ == StubFieldPolicy::Constant) {
+ MOZ_ASSERT(mode_ == Mode::Ion);
+ masm.moveValue(valueStubField(val.getOffset()), dest);
+ } else {
+ Address addr(ICStubReg, stubDataOffset_ + val.getOffset());
+ masm.loadValue(addr, dest);
+ }
+}
+
+void CacheIRCompiler::emitLoadDoubleValueStubField(StubFieldOffset val,
+ ValueOperand dest,
+ FloatRegister scratch) {
+ MOZ_ASSERT(val.getStubFieldType() == StubField::Type::Double);
+
+ if (stubFieldPolicy_ == StubFieldPolicy::Constant) {
+ MOZ_ASSERT(mode_ == Mode::Ion);
+ double d = doubleStubField(val.getOffset());
+ masm.moveValue(DoubleValue(d), dest);
+ } else {
+ Address addr(ICStubReg, stubDataOffset_ + val.getOffset());
+ masm.loadDouble(addr, scratch);
+ masm.boxDouble(scratch, dest, scratch);
+ }
+}
+
+bool CacheIRCompiler::emitLoadInstanceOfObjectResult(ValOperandId lhsId,
+ ObjOperandId protoId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ ValueOperand lhs = allocator.useValueRegister(masm, lhsId);
+ Register proto = allocator.useRegister(masm, protoId);
+
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label returnFalse, returnTrue, done;
+ masm.fallibleUnboxObject(lhs, scratch, &returnFalse);
+
+ // LHS is an object. Load its proto.
+ masm.loadObjProto(scratch, scratch);
+ {
+ // Walk the proto chain until we either reach the target object,
+ // nullptr or LazyProto.
+ Label loop;
+ masm.bind(&loop);
+
+ masm.branchPtr(Assembler::Equal, scratch, proto, &returnTrue);
+ masm.branchTestPtr(Assembler::Zero, scratch, scratch, &returnFalse);
+
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+ masm.branchPtr(Assembler::Equal, scratch, ImmWord(1), failure->label());
+
+ masm.loadObjProto(scratch, scratch);
+ masm.jump(&loop);
+ }
+
+ masm.bind(&returnFalse);
+ EmitStoreBoolean(masm, false, output);
+ masm.jump(&done);
+
+ masm.bind(&returnTrue);
+ EmitStoreBoolean(masm, true, output);
+ // fallthrough
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitMegamorphicLoadSlotResult(ObjOperandId objId,
+ uint32_t nameOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ StubFieldOffset name(nameOffset, StubField::Type::String);
+
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.Push(UndefinedValue());
+ masm.moveStackPtrTo(scratch3.get());
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch1);
+ volatileRegs.takeUnchecked(scratch2);
+ volatileRegs.takeUnchecked(scratch3);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn =
+ bool (*)(JSContext * cx, JSObject * obj, PropertyName * name, Value * vp);
+ masm.setupUnalignedABICall(scratch1);
+ masm.loadJSContext(scratch1);
+ masm.passABIArg(scratch1);
+ masm.passABIArg(obj);
+ emitLoadStubField(name, scratch2);
+ masm.passABIArg(scratch2);
+ masm.passABIArg(scratch3);
+ masm.callWithABI<Fn, GetNativeDataPropertyPure>();
+
+ masm.storeCallPointerResult(scratch2);
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.loadTypedOrValue(Address(masm.getStackPointer(), 0), output);
+ masm.adjustStack(sizeof(Value));
+
+ masm.branchIfFalseBool(scratch2, failure->label());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitMegamorphicStoreSlot(ObjOperandId objId,
+ uint32_t nameOffset,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ StubFieldOffset name(nameOffset, StubField::Type::String);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.Push(val);
+ masm.moveStackPtrTo(val.scratchReg());
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch1);
+ volatileRegs.takeUnchecked(scratch2);
+ volatileRegs.takeUnchecked(val);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, PropertyName * name,
+ Value * val);
+ masm.setupUnalignedABICall(scratch1);
+ masm.loadJSContext(scratch1);
+ masm.passABIArg(scratch1);
+ masm.passABIArg(obj);
+ emitLoadStubField(name, scratch2);
+ masm.passABIArg(scratch2);
+ masm.passABIArg(val.scratchReg());
+ masm.callWithABI<Fn, SetNativeDataPropertyPure>();
+
+ masm.storeCallPointerResult(scratch1);
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.loadValue(Address(masm.getStackPointer(), 0), val);
+ masm.adjustStack(sizeof(Value));
+
+ masm.branchIfFalseBool(scratch1, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardHasGetterSetter(ObjOperandId objId,
+ uint32_t idOffset,
+ uint32_t getterSetterOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register obj = allocator.useRegister(masm, objId);
+
+ StubFieldOffset id(idOffset, StubField::Type::Id);
+ StubFieldOffset getterSetter(getterSetterOffset,
+ StubField::Type::GetterSetter);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch1);
+ volatileRegs.takeUnchecked(scratch2);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, jsid id,
+ GetterSetter * getterSetter);
+ masm.setupUnalignedABICall(scratch1);
+ masm.loadJSContext(scratch1);
+ masm.passABIArg(scratch1);
+ masm.passABIArg(obj);
+ emitLoadStubField(id, scratch2);
+ masm.passABIArg(scratch2);
+ emitLoadStubField(getterSetter, scratch3);
+ masm.passABIArg(scratch3);
+ masm.callWithABI<Fn, ObjectHasGetterSetterPure>();
+ masm.storeCallPointerResult(scratch1);
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.branchIfFalseBool(scratch1, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardWasmArg(ValOperandId argId,
+ wasm::ValType::Kind kind) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ // All values can be boxed as AnyRef.
+ if (kind == wasm::ValType::Ref) {
+ return true;
+ }
+ MOZ_ASSERT(kind != wasm::ValType::V128);
+
+ ValueOperand arg = allocator.useValueRegister(masm, argId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Check that the argument can be converted to the Wasm type in Warp code
+ // without bailing out.
+ Label done;
+ switch (kind) {
+ case wasm::ValType::I32:
+ case wasm::ValType::F32:
+ case wasm::ValType::F64: {
+ // Argument must be number, bool, or undefined.
+ masm.branchTestNumber(Assembler::Equal, arg, &done);
+ masm.branchTestBoolean(Assembler::Equal, arg, &done);
+ masm.branchTestUndefined(Assembler::NotEqual, arg, failure->label());
+ break;
+ }
+ case wasm::ValType::I64: {
+ // Argument must be bigint, bool, or string.
+ masm.branchTestBigInt(Assembler::Equal, arg, &done);
+ masm.branchTestBoolean(Assembler::Equal, arg, &done);
+ masm.branchTestString(Assembler::NotEqual, arg, failure->label());
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected kind");
+ }
+ masm.bind(&done);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitGuardMultipleShapes(ObjOperandId objId,
+ uint32_t shapesOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister shapes(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+
+ bool needSpectreMitigations = objectGuardNeedsSpectreMitigations(objId);
+
+ Register spectreScratch = InvalidReg;
+ Maybe<AutoScratchRegister> maybeSpectreScratch;
+ if (needSpectreMitigations) {
+ maybeSpectreScratch.emplace(allocator, masm);
+ spectreScratch = *maybeSpectreScratch;
+ }
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // The stub field contains a ListObject. Load its elements.
+ StubFieldOffset shapeArray(shapesOffset, StubField::Type::JSObject);
+ emitLoadStubField(shapeArray, shapes);
+ masm.loadPtr(Address(shapes, NativeObject::offsetOfElements()), shapes);
+
+ masm.branchTestObjShapeList(Assembler::NotEqual, obj, shapes, scratch,
+ scratch2, spectreScratch, failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadObject(ObjOperandId resultId,
+ uint32_t objOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register reg = allocator.defineRegister(masm, resultId);
+ StubFieldOffset obj(objOffset, StubField::Type::JSObject);
+ emitLoadStubField(obj, reg);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadProtoObject(ObjOperandId resultId,
+ uint32_t objOffset,
+ ObjOperandId receiverObjId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register reg = allocator.defineRegister(masm, resultId);
+ StubFieldOffset obj(objOffset, StubField::Type::JSObject);
+ emitLoadStubField(obj, reg);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadInt32Constant(uint32_t valOffset,
+ Int32OperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register reg = allocator.defineRegister(masm, resultId);
+ StubFieldOffset val(valOffset, StubField::Type::RawInt32);
+ emitLoadStubField(val, reg);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadBooleanConstant(bool val,
+ BooleanOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register reg = allocator.defineRegister(masm, resultId);
+ masm.move32(Imm32(val), reg);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadDoubleConstant(uint32_t valOffset,
+ NumberOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+ StubFieldOffset val(valOffset, StubField::Type::Double);
+
+ AutoScratchFloatRegister floatReg(this);
+
+ emitLoadDoubleValueStubField(val, output, floatReg);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadUndefined(ValOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand reg = allocator.defineValueRegister(masm, resultId);
+ masm.moveValue(UndefinedValue(), reg);
+ return true;
+}
+
+bool CacheIRCompiler::emitLoadConstantString(uint32_t strOffset,
+ StringOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register reg = allocator.defineRegister(masm, resultId);
+ StubFieldOffset str(strOffset, StubField::Type::String);
+ emitLoadStubField(str, reg);
+ return true;
+}
+
+bool CacheIRCompiler::emitCallInt32ToString(Int32OperandId inputId,
+ StringOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register input = allocator.useRegister(masm, inputId);
+ Register result = allocator.defineRegister(masm, resultId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(result);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = JSLinearString* (*)(JSContext * cx, int32_t i);
+ masm.setupUnalignedABICall(result);
+ masm.loadJSContext(result);
+ masm.passABIArg(result);
+ masm.passABIArg(input);
+ masm.callWithABI<Fn, js::Int32ToStringPure>();
+
+ masm.storeCallPointerResult(result);
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.branchPtr(Assembler::Equal, result, ImmPtr(0), failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitCallNumberToString(NumberOperandId inputId,
+ StringOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoAvailableFloatRegister floatScratch0(*this, FloatReg0);
+
+ allocator.ensureDoubleRegister(masm, inputId, floatScratch0);
+ Register result = allocator.defineRegister(masm, resultId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(result);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = JSString* (*)(JSContext * cx, double d);
+ masm.setupUnalignedABICall(result);
+ masm.loadJSContext(result);
+ masm.passABIArg(result);
+ masm.passABIArg(floatScratch0, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, js::NumberToStringPure>();
+
+ masm.storeCallPointerResult(result);
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.branchPtr(Assembler::Equal, result, ImmPtr(0), failure->label());
+ return true;
+}
+
+bool CacheIRCompiler::emitInt32ToStringWithBaseResult(Int32OperandId inputId,
+ Int32OperandId baseId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+ Register input = allocator.useRegister(masm, inputId);
+ Register base = allocator.useRegister(masm, baseId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // AutoCallVM's AutoSaveLiveRegisters aren't accounted for in FailurePath, so
+ // we can't use both at the same time. This isn't an issue here, because Ion
+ // doesn't support CallICs. If that ever changes, this code must be updated.
+ MOZ_ASSERT(isBaseline(), "Can't use FailurePath with AutoCallVM in Ion ICs");
+
+ masm.branch32(Assembler::LessThan, base, Imm32(2), failure->label());
+ masm.branch32(Assembler::GreaterThan, base, Imm32(36), failure->label());
+
+ callvm.prepare();
+
+ masm.Push(base);
+ masm.Push(input);
+
+ using Fn = JSString* (*)(JSContext*, int32_t, int32_t);
+ callvm.call<Fn, js::Int32ToStringWithBase>();
+ return true;
+}
+
+bool CacheIRCompiler::emitBooleanToString(BooleanOperandId inputId,
+ StringOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register boolean = allocator.useRegister(masm, inputId);
+ Register result = allocator.defineRegister(masm, resultId);
+ const JSAtomState& names = cx_->names();
+ Label true_, done;
+
+ masm.branchTest32(Assembler::NonZero, boolean, boolean, &true_);
+
+ // False case
+ masm.movePtr(ImmGCPtr(names.false_), result);
+ masm.jump(&done);
+
+ // True case
+ masm.bind(&true_);
+ masm.movePtr(ImmGCPtr(names.true_), result);
+ masm.bind(&done);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitObjectToStringResult(ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(output.valueReg());
+ volatileRegs.takeUnchecked(scratch);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = JSString* (*)(JSContext*, JSObject*);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, js::ObjectClassToString>();
+ masm.storeCallPointerResult(scratch);
+
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.branchPtr(Assembler::Equal, scratch, ImmPtr(0), failure->label());
+ masm.tagValue(JSVAL_TYPE_STRING, scratch, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitCallStringConcatResult(StringOperandId lhsId,
+ StringOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register lhs = allocator.useRegister(masm, lhsId);
+ Register rhs = allocator.useRegister(masm, rhsId);
+
+ callvm.prepare();
+
+ masm.Push(static_cast<js::jit::Imm32>(js::gc::DefaultHeap));
+ masm.Push(rhs);
+ masm.Push(lhs);
+
+ using Fn = JSString* (*)(JSContext*, HandleString, HandleString,
+ js::gc::InitialHeap);
+ callvm.call<Fn, ConcatStrings<CanGC>>();
+
+ return true;
+}
+
+bool CacheIRCompiler::emitCallIsSuspendedGeneratorResult(ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ AutoScratchRegister scratch2(allocator, masm);
+ ValueOperand input = allocator.useValueRegister(masm, valId);
+
+ // Test if it's an object.
+ Label returnFalse, done;
+ masm.fallibleUnboxObject(input, scratch, &returnFalse);
+
+ // Test if it's a GeneratorObject.
+ masm.branchTestObjClass(Assembler::NotEqual, scratch,
+ &GeneratorObject::class_, scratch2, scratch,
+ &returnFalse);
+
+ // If the resumeIndex slot holds an int32 value < RESUME_INDEX_RUNNING,
+ // the generator is suspended.
+ Address addr(scratch, AbstractGeneratorObject::offsetOfResumeIndexSlot());
+ masm.fallibleUnboxInt32(addr, scratch, &returnFalse);
+ masm.branch32(Assembler::AboveOrEqual, scratch,
+ Imm32(AbstractGeneratorObject::RESUME_INDEX_RUNNING),
+ &returnFalse);
+
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&returnFalse);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+// This op generates no code. It is consumed by the transpiler.
+bool CacheIRCompiler::emitMetaScriptedThisShape(uint32_t) { return true; }
+
+bool CacheIRCompiler::emitCallNativeGetElementResult(ObjOperandId objId,
+ Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+
+ callvm.prepare();
+
+ masm.Push(index);
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(obj)));
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, int32_t,
+ MutableHandleValue);
+ callvm.call<Fn, NativeGetElement>();
+
+ return true;
+}
+
+bool CacheIRCompiler::emitCallNativeGetElementSuperResult(
+ ObjOperandId objId, Int32OperandId indexId, ValOperandId receiverId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ ValueOperand receiver = allocator.useValueRegister(masm, receiverId);
+
+ callvm.prepare();
+
+ masm.Push(index);
+ masm.Push(receiver);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, int32_t,
+ MutableHandleValue);
+ callvm.call<Fn, NativeGetElement>();
+
+ return true;
+}
+
+bool CacheIRCompiler::emitProxyHasPropResult(ObjOperandId objId,
+ ValOperandId idId, bool hasOwn) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand idVal = allocator.useValueRegister(masm, idId);
+
+ callvm.prepare();
+
+ masm.Push(idVal);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
+ if (hasOwn) {
+ callvm.call<Fn, ProxyHasOwn>();
+ } else {
+ callvm.call<Fn, ProxyHas>();
+ }
+ return true;
+}
+
+bool CacheIRCompiler::emitProxyGetByValueResult(ObjOperandId objId,
+ ValOperandId idId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand idVal = allocator.useValueRegister(masm, idId);
+
+ callvm.prepare();
+ masm.Push(idVal);
+ masm.Push(obj);
+
+ using Fn =
+ bool (*)(JSContext*, HandleObject, HandleValue, MutableHandleValue);
+ callvm.call<Fn, ProxyGetPropertyByValue>();
+ return true;
+}
+
+bool CacheIRCompiler::emitCallGetSparseElementResult(ObjOperandId objId,
+ Int32OperandId indexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register obj = allocator.useRegister(masm, objId);
+ Register id = allocator.useRegister(masm, indexId);
+
+ callvm.prepare();
+ masm.Push(id);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext * cx, Handle<NativeObject*> obj, int32_t int_id,
+ MutableHandleValue result);
+ callvm.call<Fn, GetSparseElementHelper>();
+ return true;
+}
+
+bool CacheIRCompiler::emitCallRegExpMatcherResult(ObjOperandId regexpId,
+ StringOperandId inputId,
+ Int32OperandId lastIndexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register regexp = allocator.useRegister(masm, regexpId);
+ Register input = allocator.useRegister(masm, inputId);
+ Register lastIndex = allocator.useRegister(masm, lastIndexId);
+
+ callvm.prepare();
+ masm.Push(ImmWord(0)); // nullptr MatchPairs.
+ masm.Push(lastIndex);
+ masm.Push(input);
+ masm.Push(regexp);
+
+ using Fn = bool (*)(JSContext*, HandleObject regexp, HandleString input,
+ int32_t lastIndex, MatchPairs * pairs,
+ MutableHandleValue output);
+ callvm.call<Fn, RegExpMatcherRaw>();
+ return true;
+}
+
+bool CacheIRCompiler::emitCallRegExpSearcherResult(ObjOperandId regexpId,
+ StringOperandId inputId,
+ Int32OperandId lastIndexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register regexp = allocator.useRegister(masm, regexpId);
+ Register input = allocator.useRegister(masm, inputId);
+ Register lastIndex = allocator.useRegister(masm, lastIndexId);
+
+ callvm.prepare();
+ masm.Push(ImmWord(0)); // nullptr MatchPairs.
+ masm.Push(lastIndex);
+ masm.Push(input);
+ masm.Push(regexp);
+
+ using Fn = bool (*)(JSContext*, HandleObject regexp, HandleString input,
+ int32_t lastIndex, MatchPairs * pairs, int32_t * result);
+ callvm.call<Fn, RegExpSearcherRaw>();
+ return true;
+}
+
+bool CacheIRCompiler::emitCallRegExpTesterResult(ObjOperandId regexpId,
+ StringOperandId inputId,
+ Int32OperandId lastIndexId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register regexp = allocator.useRegister(masm, regexpId);
+ Register input = allocator.useRegister(masm, inputId);
+ Register lastIndex = allocator.useRegister(masm, lastIndexId);
+
+ callvm.prepare();
+ masm.Push(lastIndex);
+ masm.Push(input);
+ masm.Push(regexp);
+
+ using Fn = bool (*)(JSContext*, HandleObject regexp, HandleString input,
+ int32_t lastIndex, int32_t * result);
+ callvm.call<Fn, RegExpTesterRaw>();
+ return true;
+}
+
+bool CacheIRCompiler::emitRegExpFlagResult(ObjOperandId regexpId,
+ int32_t flagsMask) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register regexp = allocator.useRegister(masm, regexpId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Address flagsAddr(
+ regexp, NativeObject::getFixedSlotOffset(RegExpObject::flagsSlot()));
+ masm.unboxInt32(flagsAddr, scratch);
+
+ Label ifFalse, done;
+ masm.branchTest32(Assembler::Zero, scratch, Imm32(flagsMask), &ifFalse);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ masm.bind(&ifFalse);
+ masm.moveValue(BooleanValue(false), output.valueReg());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitCallSubstringKernelResult(StringOperandId strId,
+ Int32OperandId beginId,
+ Int32OperandId lengthId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+ Register begin = allocator.useRegister(masm, beginId);
+ Register length = allocator.useRegister(masm, lengthId);
+
+ callvm.prepare();
+ masm.Push(length);
+ masm.Push(begin);
+ masm.Push(str);
+
+ using Fn = JSString* (*)(JSContext * cx, HandleString str, int32_t begin,
+ int32_t len);
+ callvm.call<Fn, SubstringKernel>();
+ return true;
+}
+
+bool CacheIRCompiler::emitStringReplaceStringResult(
+ StringOperandId strId, StringOperandId patternId,
+ StringOperandId replacementId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+ Register pattern = allocator.useRegister(masm, patternId);
+ Register replacement = allocator.useRegister(masm, replacementId);
+
+ callvm.prepare();
+ masm.Push(replacement);
+ masm.Push(pattern);
+ masm.Push(str);
+
+ using Fn =
+ JSString* (*)(JSContext*, HandleString, HandleString, HandleString);
+ callvm.call<Fn, jit::StringReplace>();
+ return true;
+}
+
+bool CacheIRCompiler::emitStringSplitStringResult(StringOperandId strId,
+ StringOperandId separatorId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+ Register separator = allocator.useRegister(masm, separatorId);
+
+ callvm.prepare();
+ masm.Push(Imm32(INT32_MAX));
+ masm.Push(separator);
+ masm.Push(str);
+
+ using Fn = ArrayObject* (*)(JSContext*, HandleString, HandleString, uint32_t);
+ callvm.call<Fn, js::StringSplitString>();
+ return true;
+}
+
+bool CacheIRCompiler::emitRegExpPrototypeOptimizableResult(
+ ObjOperandId protoId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register proto = allocator.useRegister(masm, protoId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Label slow, done;
+ masm.branchIfNotRegExpPrototypeOptimizable(proto, scratch, &slow);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ {
+ masm.bind(&slow);
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * proto);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(proto);
+ masm.callWithABI<Fn, RegExpPrototypeOptimizableRaw>();
+ masm.storeCallBoolResult(scratch);
+
+ masm.PopRegsInMask(volatileRegs);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitRegExpInstanceOptimizableResult(
+ ObjOperandId regexpId, ObjOperandId protoId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register regexp = allocator.useRegister(masm, regexpId);
+ Register proto = allocator.useRegister(masm, protoId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ Label slow, done;
+ masm.branchIfNotRegExpInstanceOptimizable(regexp, scratch, &slow);
+ masm.moveValue(BooleanValue(true), output.valueReg());
+ masm.jump(&done);
+
+ {
+ masm.bind(&slow);
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, JSObject * proto);
+ masm.setupUnalignedABICall(scratch);
+ masm.loadJSContext(scratch);
+ masm.passABIArg(scratch);
+ masm.passABIArg(regexp);
+ masm.passABIArg(proto);
+ masm.callWithABI<Fn, RegExpInstanceOptimizableRaw>();
+ masm.storeCallBoolResult(scratch);
+
+ masm.PopRegsInMask(volatileRegs);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+ }
+
+ masm.bind(&done);
+ return true;
+}
+
+bool CacheIRCompiler::emitGetFirstDollarIndexResult(StringOperandId strId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register str = allocator.useRegister(masm, strId);
+
+ callvm.prepare();
+ masm.Push(str);
+
+ using Fn = bool (*)(JSContext*, JSString*, int32_t*);
+ callvm.call<Fn, GetFirstDollarIndexRaw>();
+ return true;
+}
+
+bool CacheIRCompiler::emitAtomicsCompareExchangeResult(
+ ObjOperandId objId, IntPtrOperandId indexId, uint32_t expectedId,
+ uint32_t replacementId, Scalar::Type elementType) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Maybe<AutoOutputRegister> output;
+ Maybe<AutoCallVM> callvm;
+ if (!Scalar::isBigIntType(elementType)) {
+ output.emplace(*this);
+ } else {
+ callvm.emplace(masm, this, allocator);
+ }
+#ifdef JS_CODEGEN_X86
+ // Use a scratch register to avoid running out of registers.
+ Register obj = output ? output->valueReg().typeReg()
+ : callvm->outputValueReg().typeReg();
+ allocator.copyToScratchRegister(masm, objId, obj);
+#else
+ Register obj = allocator.useRegister(masm, objId);
+#endif
+ Register index = allocator.useRegister(masm, indexId);
+ Register expected;
+ Register replacement;
+ if (!Scalar::isBigIntType(elementType)) {
+ expected = allocator.useRegister(masm, Int32OperandId(expectedId));
+ replacement = allocator.useRegister(masm, Int32OperandId(replacementId));
+ } else {
+ expected = allocator.useRegister(masm, BigIntOperandId(expectedId));
+ replacement = allocator.useRegister(masm, BigIntOperandId(replacementId));
+ }
+
+ Register scratch = output ? output->valueReg().scratchReg()
+ : callvm->outputValueReg().scratchReg();
+ MOZ_ASSERT(scratch != obj, "scratchReg must not be typeReg");
+
+ // Not enough registers on X86.
+ Register spectreTemp = Register::Invalid();
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // AutoCallVM's AutoSaveLiveRegisters aren't accounted for in FailurePath, so
+ // we can't use both at the same time. This isn't an issue here, because Ion
+ // doesn't support CallICs. If that ever changes, this code must be updated.
+ MOZ_ASSERT(isBaseline(), "Can't use FailurePath with AutoCallVM in Ion ICs");
+
+ // Bounds check.
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ masm.spectreBoundsCheckPtr(index, scratch, spectreTemp, failure->label());
+
+ // Atomic operations are highly platform-dependent, for example x86/x64 has
+ // specific requirements on which registers are used; MIPS needs multiple
+ // additional temporaries. Therefore we're using either an ABI or VM call here
+ // instead of handling each platform separately.
+
+ if (Scalar::isBigIntType(elementType)) {
+ callvm->prepare();
+
+ masm.Push(replacement);
+ masm.Push(expected);
+ masm.Push(index);
+ masm.Push(obj);
+
+ using Fn = BigInt* (*)(JSContext*, TypedArrayObject*, size_t, const BigInt*,
+ const BigInt*);
+ callvm->call<Fn, jit::AtomicsCompareExchange64>();
+ return true;
+ }
+
+ {
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(output->valueReg());
+ volatileRegs.takeUnchecked(scratch);
+ masm.PushRegsInMask(volatileRegs);
+
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(obj);
+ masm.passABIArg(index);
+ masm.passABIArg(expected);
+ masm.passABIArg(replacement);
+ masm.callWithABI(DynamicFunction<AtomicsCompareExchangeFn>(
+ AtomicsCompareExchange(elementType)));
+ masm.storeCallInt32Result(scratch);
+
+ masm.PopRegsInMask(volatileRegs);
+ }
+
+ if (elementType != Scalar::Uint32) {
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output->valueReg());
+ } else {
+ ScratchDoubleScope fpscratch(masm);
+ masm.convertUInt32ToDouble(scratch, fpscratch);
+ masm.boxDouble(fpscratch, output->valueReg(), fpscratch);
+ }
+
+ return true;
+}
+
+bool CacheIRCompiler::emitAtomicsReadModifyWriteResult(
+ ObjOperandId objId, IntPtrOperandId indexId, uint32_t valueId,
+ Scalar::Type elementType, AtomicsReadWriteModifyFn fn) {
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ Register value = allocator.useRegister(masm, Int32OperandId(valueId));
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ // Not enough registers on X86.
+ Register spectreTemp = Register::Invalid();
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Bounds check.
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ masm.spectreBoundsCheckPtr(index, scratch, spectreTemp, failure->label());
+
+ // See comment in emitAtomicsCompareExchange for why we use an ABI call.
+ {
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(output.valueReg());
+ volatileRegs.takeUnchecked(scratch);
+ masm.PushRegsInMask(volatileRegs);
+
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(obj);
+ masm.passABIArg(index);
+ masm.passABIArg(value);
+ masm.callWithABI(DynamicFunction<AtomicsReadWriteModifyFn>(fn));
+ masm.storeCallInt32Result(scratch);
+
+ masm.PopRegsInMask(volatileRegs);
+ }
+
+ if (elementType != Scalar::Uint32) {
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ } else {
+ ScratchDoubleScope fpscratch(masm);
+ masm.convertUInt32ToDouble(scratch, fpscratch);
+ masm.boxDouble(fpscratch, output.valueReg(), fpscratch);
+ }
+
+ return true;
+}
+
+template <CacheIRCompiler::AtomicsReadWriteModify64Fn fn>
+bool CacheIRCompiler::emitAtomicsReadModifyWriteResult64(
+ ObjOperandId objId, IntPtrOperandId indexId, uint32_t valueId) {
+ AutoCallVM callvm(masm, this, allocator);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ Register value = allocator.useRegister(masm, BigIntOperandId(valueId));
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, callvm.output());
+
+ // Not enough registers on X86.
+ Register spectreTemp = Register::Invalid();
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // AutoCallVM's AutoSaveLiveRegisters aren't accounted for in FailurePath, so
+ // we can't use both at the same time. This isn't an issue here, because Ion
+ // doesn't support CallICs. If that ever changes, this code must be updated.
+ MOZ_ASSERT(isBaseline(), "Can't use FailurePath with AutoCallVM in Ion ICs");
+
+ // Bounds check.
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ masm.spectreBoundsCheckPtr(index, scratch, spectreTemp, failure->label());
+
+ // See comment in emitAtomicsCompareExchange for why we use a VM call.
+
+ callvm.prepare();
+
+ masm.Push(value);
+ masm.Push(index);
+ masm.Push(obj);
+
+ callvm.call<AtomicsReadWriteModify64Fn, fn>();
+ return true;
+}
+
+bool CacheIRCompiler::emitAtomicsExchangeResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (Scalar::isBigIntType(elementType)) {
+ return emitAtomicsReadModifyWriteResult64<jit::AtomicsExchange64>(
+ objId, indexId, valueId);
+ }
+ return emitAtomicsReadModifyWriteResult(objId, indexId, valueId, elementType,
+ AtomicsExchange(elementType));
+}
+
+bool CacheIRCompiler::emitAtomicsAddResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (Scalar::isBigIntType(elementType)) {
+ return emitAtomicsReadModifyWriteResult64<jit::AtomicsAdd64>(objId, indexId,
+ valueId);
+ }
+ return emitAtomicsReadModifyWriteResult(objId, indexId, valueId, elementType,
+ AtomicsAdd(elementType));
+}
+
+bool CacheIRCompiler::emitAtomicsSubResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (Scalar::isBigIntType(elementType)) {
+ return emitAtomicsReadModifyWriteResult64<jit::AtomicsSub64>(objId, indexId,
+ valueId);
+ }
+ return emitAtomicsReadModifyWriteResult(objId, indexId, valueId, elementType,
+ AtomicsSub(elementType));
+}
+
+bool CacheIRCompiler::emitAtomicsAndResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (Scalar::isBigIntType(elementType)) {
+ return emitAtomicsReadModifyWriteResult64<jit::AtomicsAnd64>(objId, indexId,
+ valueId);
+ }
+ return emitAtomicsReadModifyWriteResult(objId, indexId, valueId, elementType,
+ AtomicsAnd(elementType));
+}
+
+bool CacheIRCompiler::emitAtomicsOrResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (Scalar::isBigIntType(elementType)) {
+ return emitAtomicsReadModifyWriteResult64<jit::AtomicsOr64>(objId, indexId,
+ valueId);
+ }
+ return emitAtomicsReadModifyWriteResult(objId, indexId, valueId, elementType,
+ AtomicsOr(elementType));
+}
+
+bool CacheIRCompiler::emitAtomicsXorResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ if (Scalar::isBigIntType(elementType)) {
+ return emitAtomicsReadModifyWriteResult64<jit::AtomicsXor64>(objId, indexId,
+ valueId);
+ }
+ return emitAtomicsReadModifyWriteResult(objId, indexId, valueId, elementType,
+ AtomicsXor(elementType));
+}
+
+bool CacheIRCompiler::emitAtomicsLoadResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ Scalar::Type elementType) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Maybe<AutoOutputRegister> output;
+ Maybe<AutoCallVM> callvm;
+ if (!Scalar::isBigIntType(elementType)) {
+ output.emplace(*this);
+ } else {
+ callvm.emplace(masm, this, allocator);
+ }
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm,
+ output ? *output : callvm->output());
+ AutoSpectreBoundsScratchRegister spectreTemp(allocator, masm);
+ AutoAvailableFloatRegister floatReg(*this, FloatReg0);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // AutoCallVM's AutoSaveLiveRegisters aren't accounted for in FailurePath, so
+ // we can't use both at the same time. This isn't an issue here, because Ion
+ // doesn't support CallICs. If that ever changes, this code must be updated.
+ MOZ_ASSERT(isBaseline(), "Can't use FailurePath with AutoCallVM in Ion ICs");
+
+ // Bounds check.
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ masm.spectreBoundsCheckPtr(index, scratch, spectreTemp, failure->label());
+
+ // Atomic operations are highly platform-dependent, for example x86/arm32 has
+ // specific requirements on which registers are used. Therefore we're using a
+ // VM call here instead of handling each platform separately.
+ if (Scalar::isBigIntType(elementType)) {
+ callvm->prepare();
+
+ masm.Push(index);
+ masm.Push(obj);
+
+ using Fn = BigInt* (*)(JSContext*, TypedArrayObject*, size_t);
+ callvm->call<Fn, jit::AtomicsLoad64>();
+ return true;
+ }
+
+ // Load the elements vector.
+ masm.loadPtr(Address(obj, ArrayBufferViewObject::dataOffset()), scratch);
+
+ // Load the value.
+ BaseIndex source(scratch, index, ScaleFromScalarType(elementType));
+
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ auto sync = Synchronization::Load();
+
+ masm.memoryBarrierBefore(sync);
+
+ Label* failUint32 = nullptr;
+ MacroAssembler::Uint32Mode mode = MacroAssembler::Uint32Mode::ForceDouble;
+ masm.loadFromTypedArray(elementType, source, output->valueReg(), mode,
+ scratch, failUint32);
+ masm.memoryBarrierAfter(sync);
+
+ return true;
+}
+
+bool CacheIRCompiler::emitAtomicsStoreResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ Register index = allocator.useRegister(masm, indexId);
+ Maybe<Register> valueInt32;
+ Maybe<Register> valueBigInt;
+ if (!Scalar::isBigIntType(elementType)) {
+ valueInt32.emplace(allocator.useRegister(masm, Int32OperandId(valueId)));
+ } else {
+ valueBigInt.emplace(allocator.useRegister(masm, BigIntOperandId(valueId)));
+ }
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ // Not enough registers on X86.
+ Register spectreTemp = Register::Invalid();
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Bounds check.
+ masm.loadArrayBufferViewLengthIntPtr(obj, scratch);
+ masm.spectreBoundsCheckPtr(index, scratch, spectreTemp, failure->label());
+
+ if (!Scalar::isBigIntType(elementType)) {
+ // Load the elements vector.
+ masm.loadPtr(Address(obj, ArrayBufferViewObject::dataOffset()), scratch);
+
+ // Store the value.
+ BaseIndex dest(scratch, index, ScaleFromScalarType(elementType));
+
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ auto sync = Synchronization::Store();
+
+ masm.memoryBarrierBefore(sync);
+ masm.storeToTypedIntArray(elementType, *valueInt32, dest);
+ masm.memoryBarrierAfter(sync);
+
+ masm.tagValue(JSVAL_TYPE_INT32, *valueInt32, output.valueReg());
+ } else {
+ // See comment in emitAtomicsCompareExchange for why we use an ABI call.
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(output.valueReg());
+ volatileRegs.takeUnchecked(scratch);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = void (*)(TypedArrayObject*, size_t, const BigInt*);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(obj);
+ masm.passABIArg(index);
+ masm.passABIArg(*valueBigInt);
+ masm.callWithABI<Fn, jit::AtomicsStore64>();
+
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.tagValue(JSVAL_TYPE_BIGINT, *valueBigInt, output.valueReg());
+ }
+
+ return true;
+}
+
+bool CacheIRCompiler::emitAtomicsIsLockFreeResult(Int32OperandId valueId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register value = allocator.useRegister(masm, valueId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.atomicIsLockFreeJS(value, scratch);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch, output.valueReg());
+
+ return true;
+}
+
+bool CacheIRCompiler::emitBigIntAsIntNResult(Int32OperandId bitsId,
+ BigIntOperandId bigIntId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register bits = allocator.useRegister(masm, bitsId);
+ Register bigInt = allocator.useRegister(masm, bigIntId);
+
+ callvm.prepare();
+ masm.Push(bits);
+ masm.Push(bigInt);
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, int32_t);
+ callvm.call<Fn, jit::BigIntAsIntN>();
+ return true;
+}
+
+bool CacheIRCompiler::emitBigIntAsUintNResult(Int32OperandId bitsId,
+ BigIntOperandId bigIntId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register bits = allocator.useRegister(masm, bitsId);
+ Register bigInt = allocator.useRegister(masm, bigIntId);
+
+ callvm.prepare();
+ masm.Push(bits);
+ masm.Push(bigInt);
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, int32_t);
+ callvm.call<Fn, jit::BigIntAsUintN>();
+ return true;
+}
+
+bool CacheIRCompiler::emitSetHasResult(ObjOperandId setId, ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register set = allocator.useRegister(masm, setId);
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+ callvm.prepare();
+ masm.Push(val);
+ masm.Push(set);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
+ callvm.call<Fn, jit::SetObjectHas>();
+ return true;
+}
+
+bool CacheIRCompiler::emitSetHasNonGCThingResult(ObjOperandId setId,
+ ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register set = allocator.useRegister(masm, setId);
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoAvailableFloatRegister scratchFloat(*this, FloatReg0);
+
+ masm.toHashableNonGCThing(val, output.valueReg(), scratchFloat);
+ masm.prepareHashNonGCThing(output.valueReg(), scratch1, scratch2);
+
+ masm.setObjectHasNonBigInt(set, output.valueReg(), scratch1, scratch2,
+ scratch3, scratch4);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitSetHasSymbolResult(ObjOperandId setId,
+ SymbolOperandId symId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register set = allocator.useRegister(masm, setId);
+ Register sym = allocator.useRegister(masm, symId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+
+ masm.prepareHashSymbol(sym, scratch1);
+
+ masm.tagValue(JSVAL_TYPE_SYMBOL, sym, output.valueReg());
+ masm.setObjectHasNonBigInt(set, output.valueReg(), scratch1, scratch2,
+ scratch3, scratch4);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitSetHasBigIntResult(ObjOperandId setId,
+ BigIntOperandId bigIntId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register set = allocator.useRegister(masm, setId);
+ Register bigInt = allocator.useRegister(masm, bigIntId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoScratchRegister scratch5(allocator, masm);
+#ifndef JS_CODEGEN_ARM
+ AutoScratchRegister scratch6(allocator, masm);
+#else
+ // We don't have more registers available on ARM32.
+ Register scratch6 = set;
+
+ masm.push(set);
+#endif
+
+ masm.prepareHashBigInt(bigInt, scratch1, scratch2, scratch3, scratch4);
+
+ masm.tagValue(JSVAL_TYPE_BIGINT, bigInt, output.valueReg());
+ masm.setObjectHasBigInt(set, output.valueReg(), scratch1, scratch2, scratch3,
+ scratch4, scratch5, scratch6);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+
+#ifdef JS_CODEGEN_ARM
+ masm.pop(set);
+#endif
+ return true;
+}
+
+bool CacheIRCompiler::emitSetHasObjectResult(ObjOperandId setId,
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register set = allocator.useRegister(masm, setId);
+ Register obj = allocator.useRegister(masm, objId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoScratchRegister scratch5(allocator, masm);
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, obj, output.valueReg());
+ masm.prepareHashObject(set, output.valueReg(), scratch1, scratch2, scratch3,
+ scratch4, scratch5);
+
+ masm.setObjectHasNonBigInt(set, output.valueReg(), scratch1, scratch2,
+ scratch3, scratch4);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitSetSizeResult(ObjOperandId setId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register set = allocator.useRegister(masm, setId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.loadSetObjectSize(set, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMapHasResult(ObjOperandId mapId, ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register map = allocator.useRegister(masm, mapId);
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+ callvm.prepare();
+ masm.Push(val);
+ masm.Push(map);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
+ callvm.call<Fn, jit::MapObjectHas>();
+ return true;
+}
+
+bool CacheIRCompiler::emitMapHasNonGCThingResult(ObjOperandId mapId,
+ ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoAvailableFloatRegister scratchFloat(*this, FloatReg0);
+
+ masm.toHashableNonGCThing(val, output.valueReg(), scratchFloat);
+ masm.prepareHashNonGCThing(output.valueReg(), scratch1, scratch2);
+
+ masm.mapObjectHasNonBigInt(map, output.valueReg(), scratch1, scratch2,
+ scratch3, scratch4);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMapHasSymbolResult(ObjOperandId mapId,
+ SymbolOperandId symId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ Register sym = allocator.useRegister(masm, symId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+
+ masm.prepareHashSymbol(sym, scratch1);
+
+ masm.tagValue(JSVAL_TYPE_SYMBOL, sym, output.valueReg());
+ masm.mapObjectHasNonBigInt(map, output.valueReg(), scratch1, scratch2,
+ scratch3, scratch4);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMapHasBigIntResult(ObjOperandId mapId,
+ BigIntOperandId bigIntId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ Register bigInt = allocator.useRegister(masm, bigIntId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoScratchRegister scratch5(allocator, masm);
+#ifndef JS_CODEGEN_ARM
+ AutoScratchRegister scratch6(allocator, masm);
+#else
+ // We don't have more registers available on ARM32.
+ Register scratch6 = map;
+
+ masm.push(map);
+#endif
+
+ masm.prepareHashBigInt(bigInt, scratch1, scratch2, scratch3, scratch4);
+
+ masm.tagValue(JSVAL_TYPE_BIGINT, bigInt, output.valueReg());
+ masm.mapObjectHasBigInt(map, output.valueReg(), scratch1, scratch2, scratch3,
+ scratch4, scratch5, scratch6);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+
+#ifdef JS_CODEGEN_ARM
+ masm.pop(map);
+#endif
+ return true;
+}
+
+bool CacheIRCompiler::emitMapHasObjectResult(ObjOperandId mapId,
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ Register obj = allocator.useRegister(masm, objId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoScratchRegister scratch5(allocator, masm);
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, obj, output.valueReg());
+ masm.prepareHashObject(map, output.valueReg(), scratch1, scratch2, scratch3,
+ scratch4, scratch5);
+
+ masm.mapObjectHasNonBigInt(map, output.valueReg(), scratch1, scratch2,
+ scratch3, scratch4);
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, scratch2, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitMapGetResult(ObjOperandId mapId, ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register map = allocator.useRegister(masm, mapId);
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+ callvm.prepare();
+ masm.Push(val);
+ masm.Push(map);
+
+ using Fn =
+ bool (*)(JSContext*, HandleObject, HandleValue, MutableHandleValue);
+ callvm.call<Fn, jit::MapObjectGet>();
+ return true;
+}
+
+bool CacheIRCompiler::emitMapGetNonGCThingResult(ObjOperandId mapId,
+ ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ ValueOperand val = allocator.useValueRegister(masm, valId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoAvailableFloatRegister scratchFloat(*this, FloatReg0);
+
+ masm.toHashableNonGCThing(val, output.valueReg(), scratchFloat);
+ masm.prepareHashNonGCThing(output.valueReg(), scratch1, scratch2);
+
+ masm.mapObjectGetNonBigInt(map, output.valueReg(), scratch1,
+ output.valueReg(), scratch2, scratch3, scratch4);
+ return true;
+}
+
+bool CacheIRCompiler::emitMapGetSymbolResult(ObjOperandId mapId,
+ SymbolOperandId symId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ Register sym = allocator.useRegister(masm, symId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+
+ masm.prepareHashSymbol(sym, scratch1);
+
+ masm.tagValue(JSVAL_TYPE_SYMBOL, sym, output.valueReg());
+ masm.mapObjectGetNonBigInt(map, output.valueReg(), scratch1,
+ output.valueReg(), scratch2, scratch3, scratch4);
+ return true;
+}
+
+bool CacheIRCompiler::emitMapGetBigIntResult(ObjOperandId mapId,
+ BigIntOperandId bigIntId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ Register bigInt = allocator.useRegister(masm, bigIntId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoScratchRegister scratch5(allocator, masm);
+#ifndef JS_CODEGEN_ARM
+ AutoScratchRegister scratch6(allocator, masm);
+#else
+ // We don't have more registers available on ARM32.
+ Register scratch6 = map;
+
+ masm.push(map);
+#endif
+
+ masm.prepareHashBigInt(bigInt, scratch1, scratch2, scratch3, scratch4);
+
+ masm.tagValue(JSVAL_TYPE_BIGINT, bigInt, output.valueReg());
+ masm.mapObjectGetBigInt(map, output.valueReg(), scratch1, output.valueReg(),
+ scratch2, scratch3, scratch4, scratch5, scratch6);
+
+#ifdef JS_CODEGEN_ARM
+ masm.pop(map);
+#endif
+ return true;
+}
+
+bool CacheIRCompiler::emitMapGetObjectResult(ObjOperandId mapId,
+ ObjOperandId objId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ Register obj = allocator.useRegister(masm, objId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+ AutoScratchRegister scratch2(allocator, masm);
+ AutoScratchRegister scratch3(allocator, masm);
+ AutoScratchRegister scratch4(allocator, masm);
+ AutoScratchRegister scratch5(allocator, masm);
+
+ masm.tagValue(JSVAL_TYPE_OBJECT, obj, output.valueReg());
+ masm.prepareHashObject(map, output.valueReg(), scratch1, scratch2, scratch3,
+ scratch4, scratch5);
+
+ masm.mapObjectGetNonBigInt(map, output.valueReg(), scratch1,
+ output.valueReg(), scratch2, scratch3, scratch4);
+ return true;
+}
+
+bool CacheIRCompiler::emitMapSizeResult(ObjOperandId mapId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+ Register map = allocator.useRegister(masm, mapId);
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+
+ masm.loadMapObjectSize(map, scratch);
+ masm.tagValue(JSVAL_TYPE_INT32, scratch, output.valueReg());
+ return true;
+}
+
+bool CacheIRCompiler::emitArrayFromArgumentsObjectResult(ObjOperandId objId,
+ uint32_t shapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoCallVM callvm(masm, this, allocator);
+
+ Register obj = allocator.useRegister(masm, objId);
+
+ callvm.prepare();
+ masm.Push(obj);
+
+ using Fn = ArrayObject* (*)(JSContext*, Handle<ArgumentsObject*>);
+ callvm.call<Fn, js::ArrayFromArgumentsObject>();
+ return true;
+}
+
+bool CacheIRCompiler::emitBailout() {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ // Generates no code.
+
+ return true;
+}
+
+bool CacheIRCompiler::emitAssertRecoveredOnBailoutResult(ValOperandId valId,
+ bool mustBeRecovered) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ AutoOutputRegister output(*this);
+
+ // NOP when not in IonMonkey
+ masm.moveValue(UndefinedValue(), output.valueReg());
+
+ return true;
+}
+
+#ifdef FUZZING_JS_FUZZILLI
+bool CacheIRCompiler::emitFuzzilliHashResult(ValOperandId valId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ ValueOperand input = allocator.useValueRegister(masm, valId);
+ AutoScratchRegister scratch(allocator, masm);
+ AutoScratchRegister scratchJSContext(allocator, masm);
+ AutoScratchFloatRegister floatReg(this);
+# ifdef JS_PUNBOX64
+ AutoScratchRegister64 scratch64(allocator, masm);
+# else
+ AutoScratchRegister scratch2(allocator, masm);
+# endif
+
+ Label addFloat, updateHash, done;
+
+ {
+ ScratchTagScope tag(masm, input);
+ masm.splitTagForTest(input, tag);
+
+ Label notInt32;
+ masm.branchTestInt32(Assembler::NotEqual, tag, &notInt32);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ masm.unboxInt32(input, scratch);
+ masm.convertInt32ToDouble(scratch, floatReg);
+ masm.jump(&addFloat);
+ }
+ masm.bind(&notInt32);
+
+ Label notDouble;
+ masm.branchTestDouble(Assembler::NotEqual, tag, &notDouble);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ masm.unboxDouble(input, floatReg);
+ masm.canonicalizeDouble(floatReg);
+ masm.jump(&addFloat);
+ }
+ masm.bind(&notDouble);
+
+ Label notNull;
+ masm.branchTestNull(Assembler::NotEqual, tag, &notNull);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ masm.move32(Imm32(1), scratch);
+ masm.convertInt32ToDouble(scratch, floatReg);
+ masm.jump(&addFloat);
+ }
+ masm.bind(&notNull);
+
+ Label notUndefined;
+ masm.branchTestUndefined(Assembler::NotEqual, tag, &notUndefined);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ masm.move32(Imm32(2), scratch);
+ masm.convertInt32ToDouble(scratch, floatReg);
+ masm.jump(&addFloat);
+ }
+ masm.bind(&notUndefined);
+
+ Label notBoolean;
+ masm.branchTestBoolean(Assembler::NotEqual, tag, &notBoolean);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ masm.unboxBoolean(input, scratch);
+ masm.add32(Imm32(3), scratch);
+ masm.convertInt32ToDouble(scratch, floatReg);
+ masm.jump(&addFloat);
+ }
+ masm.bind(&notBoolean);
+
+ Label notBigInt;
+ masm.branchTestBigInt(Assembler::NotEqual, tag, &notBigInt);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ masm.unboxBigInt(input, scratch);
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(volatileRegs);
+ // TODO: remove floatReg, scratch, scratchJS?
+
+ using Fn = uint32_t (*)(BigInt * bigInt);
+ masm.setupUnalignedABICall(scratchJSContext);
+ masm.loadJSContext(scratchJSContext);
+ masm.passABIArg(scratch);
+ masm.callWithABI<Fn, js::FuzzilliHashBigInt>();
+ masm.storeCallInt32Result(scratch);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch);
+ ignore.add(scratchJSContext);
+ masm.PopRegsInMaskIgnore(volatileRegs, ignore);
+ masm.jump(&updateHash);
+ }
+ masm.bind(&notBigInt);
+
+ Label notObject;
+ masm.branchTestObject(Assembler::NotEqual, tag, &notObject);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ AutoCallVM callvm(masm, this, allocator);
+ Register obj = allocator.allocateRegister(masm);
+ masm.unboxObject(input, obj);
+
+ callvm.prepare();
+ masm.Push(obj);
+
+ using Fn = void (*)(JSContext * cx, JSObject * o);
+ callvm.callNoResult<Fn, js::FuzzilliHashObject>();
+ allocator.releaseRegister(obj);
+
+ masm.jump(&done);
+ }
+ masm.bind(&notObject);
+ {
+ masm.move32(Imm32(0), scratch);
+ masm.jump(&updateHash);
+ }
+ }
+
+ {
+ masm.bind(&addFloat);
+
+ masm.loadJSContext(scratchJSContext);
+ Address addrExecHash(scratchJSContext, offsetof(JSContext, executionHash));
+
+# ifdef JS_PUNBOX64
+ masm.moveDoubleToGPR64(floatReg, scratch64);
+ masm.move32(scratch64.get().reg, scratch);
+ masm.rshift64(Imm32(32), scratch64);
+ masm.add32(scratch64.get().reg, scratch);
+# else
+ Register64 scratch64(scratch, scratch2);
+ masm.moveDoubleToGPR64(floatReg, scratch64);
+ masm.add32(scratch2, scratch);
+# endif
+ }
+
+ {
+ masm.bind(&updateHash);
+
+ masm.loadJSContext(scratchJSContext);
+ Address addrExecHash(scratchJSContext, offsetof(JSContext, executionHash));
+ masm.load32(addrExecHash, scratchJSContext);
+ masm.add32(scratchJSContext, scratch);
+ masm.rotateLeft(Imm32(1), scratch, scratch);
+ masm.loadJSContext(scratchJSContext);
+ masm.store32(scratch, addrExecHash);
+
+ // stats
+ Address addrExecHashInputs(scratchJSContext,
+ offsetof(JSContext, executionHashInputs));
+ masm.load32(addrExecHashInputs, scratch);
+ masm.add32(Imm32(1), scratch);
+ masm.store32(scratch, addrExecHashInputs);
+ }
+
+ masm.bind(&done);
+
+ AutoOutputRegister output(*this);
+ masm.moveValue(UndefinedValue(), output.valueReg());
+ return true;
+}
+#endif
+
+template <typename Fn, Fn fn>
+void CacheIRCompiler::callVM(MacroAssembler& masm) {
+ VMFunctionId id = VMFunctionToId<Fn, fn>::id;
+ callVMInternal(masm, id);
+}
+
+void CacheIRCompiler::callVMInternal(MacroAssembler& masm, VMFunctionId id) {
+ MOZ_ASSERT(enteredStubFrame_);
+ if (mode_ == Mode::Ion) {
+ TrampolinePtr code = cx_->runtime()->jitRuntime()->getVMWrapper(id);
+ const VMFunctionData& fun = GetVMFunction(id);
+ uint32_t frameSize = fun.explicitStackSlots() * sizeof(void*);
+ masm.PushFrameDescriptor(FrameType::IonICCall);
+ masm.callJit(code);
+
+ // Pop rest of the exit frame and the arguments left on the stack.
+ int framePop =
+ sizeof(ExitFrameLayout) - ExitFrameLayout::bytesPoppedAfterCall();
+ masm.implicitPop(frameSize + framePop);
+
+ // Pop IonICCallFrameLayout.
+ masm.Pop(FramePointer);
+ masm.freeStack(IonICCallFrameLayout::Size() - sizeof(void*));
+ return;
+ }
+
+ MOZ_ASSERT(mode_ == Mode::Baseline);
+
+ TrampolinePtr code = cx_->runtime()->jitRuntime()->getVMWrapper(id);
+ MOZ_ASSERT(GetVMFunction(id).expectTailCall == NonTailCall);
+
+ EmitBaselineCallVM(code, masm);
+}
+
+bool CacheIRCompiler::isBaseline() { return mode_ == Mode::Baseline; }
+
+bool CacheIRCompiler::isIon() { return mode_ == Mode::Ion; }
+
+BaselineCacheIRCompiler* CacheIRCompiler::asBaseline() {
+ MOZ_ASSERT(this->isBaseline());
+ return static_cast<BaselineCacheIRCompiler*>(this);
+}
+
+IonCacheIRCompiler* CacheIRCompiler::asIon() {
+ MOZ_ASSERT(this->isIon());
+ return static_cast<IonCacheIRCompiler*>(this);
+}
+
+#ifdef DEBUG
+void CacheIRCompiler::assertFloatRegisterAvailable(FloatRegister reg) {
+ if (isBaseline()) {
+ // Baseline does not have any FloatRegisters live when calling an IC stub.
+ return;
+ }
+
+ asIon()->assertFloatRegisterAvailable(reg);
+}
+#endif
+
+AutoCallVM::AutoCallVM(MacroAssembler& masm, CacheIRCompiler* compiler,
+ CacheRegisterAllocator& allocator)
+ : masm_(masm), compiler_(compiler), allocator_(allocator) {
+ // Ion needs to `enterStubFrame` before it can callVM and it also needs to
+ // initialize AutoSaveLiveRegisters.
+ if (compiler_->mode_ == CacheIRCompiler::Mode::Ion) {
+ // Will need to use a downcast here as well, in order to pass the
+ // stub to AutoSaveLiveRegisters
+ save_.emplace(*compiler_->asIon());
+ }
+
+ if (compiler->outputUnchecked_.isSome()) {
+ output_.emplace(*compiler);
+ }
+
+ if (compiler_->mode_ == CacheIRCompiler::Mode::Baseline) {
+ stubFrame_.emplace(*compiler_->asBaseline());
+ if (output_.isSome()) {
+ scratch_.emplace(allocator_, masm_, output_.ref());
+ } else {
+ scratch_.emplace(allocator_, masm_);
+ }
+ }
+}
+
+void AutoCallVM::prepare() {
+ allocator_.discardStack(masm_);
+ MOZ_ASSERT(compiler_ != nullptr);
+ if (compiler_->mode_ == CacheIRCompiler::Mode::Ion) {
+ compiler_->asIon()->enterStubFrame(masm_, *save_.ptr());
+ return;
+ }
+ MOZ_ASSERT(compiler_->mode_ == CacheIRCompiler::Mode::Baseline);
+ stubFrame_->enter(masm_, scratch_.ref());
+}
+
+void AutoCallVM::storeResult(JSValueType returnType) {
+ MOZ_ASSERT(returnType != JSVAL_TYPE_DOUBLE);
+
+ if (returnType == JSVAL_TYPE_UNKNOWN) {
+ masm_.storeCallResultValue(output_.ref());
+ } else {
+ if (output_->hasValue()) {
+ masm_.tagValue(returnType, ReturnReg, output_->valueReg());
+ } else {
+ masm_.storeCallPointerResult(output_->typedReg().gpr());
+ }
+ }
+}
+
+void AutoCallVM::leaveBaselineStubFrame() {
+ if (compiler_->mode_ == CacheIRCompiler::Mode::Baseline) {
+ stubFrame_->leave(masm_);
+ }
+}
+
+template <typename...>
+struct VMFunctionReturnType;
+
+template <class R, typename... Args>
+struct VMFunctionReturnType<R (*)(JSContext*, Args...)> {
+ using LastArgument = typename LastArg<Args...>::Type;
+
+ // By convention VMFunctions returning `bool` use an output parameter.
+ using ReturnType =
+ std::conditional_t<std::is_same_v<R, bool>, LastArgument, R>;
+};
+
+template <class>
+struct ReturnTypeToJSValueType;
+
+// Definitions for the currently used return types.
+template <>
+struct ReturnTypeToJSValueType<MutableHandleValue> {
+ static constexpr JSValueType result = JSVAL_TYPE_UNKNOWN;
+};
+template <>
+struct ReturnTypeToJSValueType<bool*> {
+ static constexpr JSValueType result = JSVAL_TYPE_BOOLEAN;
+};
+template <>
+struct ReturnTypeToJSValueType<int32_t*> {
+ static constexpr JSValueType result = JSVAL_TYPE_INT32;
+};
+template <>
+struct ReturnTypeToJSValueType<JSString*> {
+ static constexpr JSValueType result = JSVAL_TYPE_STRING;
+};
+template <>
+struct ReturnTypeToJSValueType<BigInt*> {
+ static constexpr JSValueType result = JSVAL_TYPE_BIGINT;
+};
+template <>
+struct ReturnTypeToJSValueType<JSObject*> {
+ static constexpr JSValueType result = JSVAL_TYPE_OBJECT;
+};
+template <>
+struct ReturnTypeToJSValueType<ArrayIteratorObject*> {
+ static constexpr JSValueType result = JSVAL_TYPE_OBJECT;
+};
+template <>
+struct ReturnTypeToJSValueType<StringIteratorObject*> {
+ static constexpr JSValueType result = JSVAL_TYPE_OBJECT;
+};
+template <>
+struct ReturnTypeToJSValueType<RegExpStringIteratorObject*> {
+ static constexpr JSValueType result = JSVAL_TYPE_OBJECT;
+};
+template <>
+struct ReturnTypeToJSValueType<PlainObject*> {
+ static constexpr JSValueType result = JSVAL_TYPE_OBJECT;
+};
+template <>
+struct ReturnTypeToJSValueType<ArrayObject*> {
+ static constexpr JSValueType result = JSVAL_TYPE_OBJECT;
+};
+template <>
+struct ReturnTypeToJSValueType<TypedArrayObject*> {
+ static constexpr JSValueType result = JSVAL_TYPE_OBJECT;
+};
+
+template <typename Fn>
+void AutoCallVM::storeResult() {
+ using ReturnType = typename VMFunctionReturnType<Fn>::ReturnType;
+ storeResult(ReturnTypeToJSValueType<ReturnType>::result);
+}
+
+AutoScratchFloatRegister::AutoScratchFloatRegister(CacheIRCompiler* compiler,
+ FailurePath* failure)
+ : compiler_(compiler), failure_(failure) {
+ // If we're compiling a Baseline IC, FloatReg0 is always available.
+ if (!compiler_->isBaseline()) {
+ MacroAssembler& masm = compiler_->masm;
+ masm.push(FloatReg0);
+ compiler->allocator.setHasAutoScratchFloatRegisterSpill(true);
+ }
+
+ if (failure_) {
+ failure_->setHasAutoScratchFloatRegister();
+ }
+}
+
+AutoScratchFloatRegister::~AutoScratchFloatRegister() {
+ if (failure_) {
+ failure_->clearHasAutoScratchFloatRegister();
+ }
+
+ if (!compiler_->isBaseline()) {
+ MacroAssembler& masm = compiler_->masm;
+ masm.pop(FloatReg0);
+ compiler_->allocator.setHasAutoScratchFloatRegisterSpill(false);
+
+ if (failure_) {
+ Label done;
+ masm.jump(&done);
+ masm.bind(&failurePopReg_);
+ masm.pop(FloatReg0);
+ masm.jump(failure_->label());
+ masm.bind(&done);
+ }
+ }
+}
+
+Label* AutoScratchFloatRegister::failure() {
+ MOZ_ASSERT(failure_);
+
+ if (!compiler_->isBaseline()) {
+ return &failurePopReg_;
+ }
+ return failure_->labelUnchecked();
+}
diff --git a/js/src/jit/CacheIRCompiler.h b/js/src/jit/CacheIRCompiler.h
new file mode 100644
index 0000000000..2bbdd60770
--- /dev/null
+++ b/js/src/jit/CacheIRCompiler.h
@@ -0,0 +1,1310 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CacheIRCompiler_h
+#define jit_CacheIRCompiler_h
+
+#include "mozilla/Casting.h"
+#include "mozilla/Maybe.h"
+
+#include "jit/CacheIR.h"
+#include "jit/CacheIRReader.h"
+#include "jit/CacheIRWriter.h"
+#include "jit/JitOptions.h"
+#include "jit/MacroAssembler.h"
+#include "jit/PerfSpewer.h"
+#include "jit/SharedICRegisters.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+namespace JS {
+class BigInt;
+}
+
+namespace js {
+
+class TypedArrayObject;
+enum class UnaryMathFunction : uint8_t;
+
+namespace jit {
+
+class BaselineCacheIRCompiler;
+class ICCacheIRStub;
+class IonCacheIRCompiler;
+class IonScript;
+
+enum class ICStubEngine : uint8_t;
+
+// [SMDOC] CacheIR Value Representation and Tracking
+//
+// While compiling an IC stub the CacheIR compiler needs to keep track of the
+// physical location for each logical piece of data we care about, as well as
+// ensure that in the case of a stub failing, we are able to restore the input
+// state so that a subsequent stub can attempt to provide a value.
+//
+// OperandIds are created in the CacheIR front-end to keep track of values that
+// are passed between CacheIR ops during the execution of a given CacheIR stub.
+// In the CacheRegisterAllocator these OperandIds are given OperandLocations,
+// that represent the physical location of the OperandId at a given point in
+// time during CacheRegister allocation.
+//
+// In the CacheRegisterAllocator physical locations include the stack, and
+// registers, as well as whether or not the value has been unboxed or not.
+// Constants are also represented separately to provide for on-demand
+// materialization.
+//
+// Intra-op Register allocation:
+//
+// During the emission of a CacheIR op, code can ask the CacheRegisterAllocator
+// for access to a particular OperandId, and the register allocator will
+// generate the required code to fill that request.
+//
+// Input OperandIds should be considered as immutable, and should not be mutated
+// during the execution of a stub.
+//
+// There are also a number of RAII classes that interact with the register
+// allocator, in order to provide access to more registers than just those
+// provided for by the OperandIds.
+//
+// - AutoOutputReg: The register which will hold the output value of the stub.
+// - AutoScratchReg: By default, an arbitrary scratch register, however a
+// specific register can be requested.
+// - AutoScratchRegMaybeOutput: Any arbitrary scratch register, but the output
+// register may be used as well.
+//
+// These RAII classes take ownership of a register for the duration of their
+// lifetime so they can be used for computation or output. The register
+// allocator can spill values with OperandLocations in order to try to ensure
+// that a register is made available for use.
+//
+// If a specific register is required (via AutoScratchRegister), it should be
+// the first register acquired, as the register rallocator will be unable to
+// allocate the fixed register if the current op is using it for something else.
+//
+// If no register can be provided after attempting to spill, a
+// MOZ_RELEASE_ASSERT ensures the browser will crash. The register allocator is
+// not provided enough information in its current design to insert spills and
+// fills at arbitrary locations, and so it can fail to find an allocation
+// solution. However, this will only happen within the implementation of an
+// operand emitter, and because the cache register allocator is mostly
+// determinstic, so long as the operand id emitter is tested, this won't
+// suddenly crop up in an arbitrary webpage. It's worth noting the most
+// difficult platform to support is x86-32, because it has the least number of
+// registers available.
+//
+// FailurePaths checkpoint the state of the register allocator so that the input
+// state can be recomputed from the current state before jumping to the next
+// stub in the IC chain. An important invariant is that the FailurePath must be
+// allocated for each op after all the manipulation of OperandLocations has
+// happened, so that its recording is correct.
+//
+// Inter-op Register Allocation:
+//
+// The RAII register management classes are RAII because all register state
+// outside the OperandLocations is reset before the compilation of each
+// individual CacheIR op. This means that you cannot rely on a value surviving
+// between ops, even if you use the ability of AutoScratchRegister to name a
+// specific register. Values that need to be preserved between ops must be given
+// an OperandId.
+
+// Represents a Value on the Baseline frame's expression stack. Slot 0 is the
+// value on top of the stack (the most recently pushed value), slot 1 is the
+// value pushed before that, etc.
+class BaselineFrameSlot {
+ uint32_t slot_;
+
+ public:
+ explicit BaselineFrameSlot(uint32_t slot) : slot_(slot) {}
+ uint32_t slot() const { return slot_; }
+
+ bool operator==(const BaselineFrameSlot& other) const {
+ return slot_ == other.slot_;
+ }
+ bool operator!=(const BaselineFrameSlot& other) const {
+ return slot_ != other.slot_;
+ }
+};
+
+// OperandLocation represents the location of an OperandId. The operand is
+// either in a register or on the stack, and is either boxed or unboxed.
+class OperandLocation {
+ public:
+ enum Kind {
+ Uninitialized = 0,
+ PayloadReg,
+ DoubleReg,
+ ValueReg,
+ PayloadStack,
+ ValueStack,
+ BaselineFrame,
+ Constant,
+ };
+
+ private:
+ Kind kind_;
+
+ union Data {
+ struct {
+ Register reg;
+ JSValueType type;
+ } payloadReg;
+ FloatRegister doubleReg;
+ ValueOperand valueReg;
+ struct {
+ uint32_t stackPushed;
+ JSValueType type;
+ } payloadStack;
+ uint32_t valueStackPushed;
+ BaselineFrameSlot baselineFrameSlot;
+ Value constant;
+
+ Data() : valueStackPushed(0) {}
+ };
+ Data data_;
+
+ public:
+ OperandLocation() : kind_(Uninitialized) {}
+
+ Kind kind() const { return kind_; }
+
+ void setUninitialized() { kind_ = Uninitialized; }
+
+ ValueOperand valueReg() const {
+ MOZ_ASSERT(kind_ == ValueReg);
+ return data_.valueReg;
+ }
+ Register payloadReg() const {
+ MOZ_ASSERT(kind_ == PayloadReg);
+ return data_.payloadReg.reg;
+ }
+ FloatRegister doubleReg() const {
+ MOZ_ASSERT(kind_ == DoubleReg);
+ return data_.doubleReg;
+ }
+ uint32_t payloadStack() const {
+ MOZ_ASSERT(kind_ == PayloadStack);
+ return data_.payloadStack.stackPushed;
+ }
+ uint32_t valueStack() const {
+ MOZ_ASSERT(kind_ == ValueStack);
+ return data_.valueStackPushed;
+ }
+ JSValueType payloadType() const {
+ if (kind_ == PayloadReg) {
+ return data_.payloadReg.type;
+ }
+ MOZ_ASSERT(kind_ == PayloadStack);
+ return data_.payloadStack.type;
+ }
+ Value constant() const {
+ MOZ_ASSERT(kind_ == Constant);
+ return data_.constant;
+ }
+ BaselineFrameSlot baselineFrameSlot() const {
+ MOZ_ASSERT(kind_ == BaselineFrame);
+ return data_.baselineFrameSlot;
+ }
+
+ void setPayloadReg(Register reg, JSValueType type) {
+ kind_ = PayloadReg;
+ data_.payloadReg.reg = reg;
+ data_.payloadReg.type = type;
+ }
+ void setDoubleReg(FloatRegister reg) {
+ kind_ = DoubleReg;
+ data_.doubleReg = reg;
+ }
+ void setValueReg(ValueOperand reg) {
+ kind_ = ValueReg;
+ data_.valueReg = reg;
+ }
+ void setPayloadStack(uint32_t stackPushed, JSValueType type) {
+ kind_ = PayloadStack;
+ data_.payloadStack.stackPushed = stackPushed;
+ data_.payloadStack.type = type;
+ }
+ void setValueStack(uint32_t stackPushed) {
+ kind_ = ValueStack;
+ data_.valueStackPushed = stackPushed;
+ }
+ void setConstant(const Value& v) {
+ kind_ = Constant;
+ data_.constant = v;
+ }
+ void setBaselineFrame(BaselineFrameSlot slot) {
+ kind_ = BaselineFrame;
+ data_.baselineFrameSlot = slot;
+ }
+
+ bool isUninitialized() const { return kind_ == Uninitialized; }
+ bool isInRegister() const { return kind_ == PayloadReg || kind_ == ValueReg; }
+ bool isOnStack() const {
+ return kind_ == PayloadStack || kind_ == ValueStack;
+ }
+
+ size_t stackPushed() const {
+ if (kind_ == PayloadStack) {
+ return data_.payloadStack.stackPushed;
+ }
+ MOZ_ASSERT(kind_ == ValueStack);
+ return data_.valueStackPushed;
+ }
+ size_t stackSizeInBytes() const {
+ if (kind_ == PayloadStack) {
+ return sizeof(uintptr_t);
+ }
+ MOZ_ASSERT(kind_ == ValueStack);
+ return sizeof(js::Value);
+ }
+ void adjustStackPushed(int32_t diff) {
+ if (kind_ == PayloadStack) {
+ data_.payloadStack.stackPushed += diff;
+ return;
+ }
+ MOZ_ASSERT(kind_ == ValueStack);
+ data_.valueStackPushed += diff;
+ }
+
+ bool aliasesReg(Register reg) const {
+ if (kind_ == PayloadReg) {
+ return payloadReg() == reg;
+ }
+ if (kind_ == ValueReg) {
+ return valueReg().aliases(reg);
+ }
+ return false;
+ }
+ bool aliasesReg(ValueOperand reg) const {
+#if defined(JS_NUNBOX32)
+ return aliasesReg(reg.typeReg()) || aliasesReg(reg.payloadReg());
+#else
+ return aliasesReg(reg.valueReg());
+#endif
+ }
+
+ bool aliasesReg(const OperandLocation& other) const;
+
+ bool operator==(const OperandLocation& other) const;
+ bool operator!=(const OperandLocation& other) const {
+ return !operator==(other);
+ }
+};
+
+struct SpilledRegister {
+ Register reg;
+ uint32_t stackPushed;
+
+ SpilledRegister(Register reg, uint32_t stackPushed)
+ : reg(reg), stackPushed(stackPushed) {}
+ bool operator==(const SpilledRegister& other) const {
+ return reg == other.reg && stackPushed == other.stackPushed;
+ }
+ bool operator!=(const SpilledRegister& other) const {
+ return !(*this == other);
+ }
+};
+
+using SpilledRegisterVector = Vector<SpilledRegister, 2, SystemAllocPolicy>;
+
+// Class to track and allocate registers while emitting IC code.
+class MOZ_RAII CacheRegisterAllocator {
+ // The original location of the inputs to the cache.
+ Vector<OperandLocation, 4, SystemAllocPolicy> origInputLocations_;
+
+ // The current location of each operand.
+ Vector<OperandLocation, 8, SystemAllocPolicy> operandLocations_;
+
+ // Free lists for value- and payload-slots on stack
+ Vector<uint32_t, 2, SystemAllocPolicy> freeValueSlots_;
+ Vector<uint32_t, 2, SystemAllocPolicy> freePayloadSlots_;
+
+ // The registers allocated while emitting the current CacheIR op.
+ // This prevents us from allocating a register and then immediately
+ // clobbering it for something else, while we're still holding on to it.
+ LiveGeneralRegisterSet currentOpRegs_;
+
+ const AllocatableGeneralRegisterSet allocatableRegs_;
+
+ // Registers that are currently unused and available.
+ AllocatableGeneralRegisterSet availableRegs_;
+
+ // Registers that are available, but before use they must be saved and
+ // then restored when returning from the stub.
+ AllocatableGeneralRegisterSet availableRegsAfterSpill_;
+
+ // Registers we took from availableRegsAfterSpill_ and spilled to the stack.
+ SpilledRegisterVector spilledRegs_;
+
+ // The number of bytes pushed on the native stack.
+ uint32_t stackPushed_;
+
+#ifdef DEBUG
+ // Flag used to assert individual CacheIR instructions don't allocate
+ // registers after calling addFailurePath.
+ bool addedFailurePath_;
+#endif
+
+ // The index of the CacheIR instruction we're currently emitting.
+ uint32_t currentInstruction_;
+
+ // Whether the stack contains a double spilled by AutoScratchFloatRegister.
+ bool hasAutoScratchFloatRegisterSpill_ = false;
+
+ const CacheIRWriter& writer_;
+
+ CacheRegisterAllocator(const CacheRegisterAllocator&) = delete;
+ CacheRegisterAllocator& operator=(const CacheRegisterAllocator&) = delete;
+
+ void freeDeadOperandLocations(MacroAssembler& masm);
+
+ void spillOperandToStack(MacroAssembler& masm, OperandLocation* loc);
+ void spillOperandToStackOrRegister(MacroAssembler& masm,
+ OperandLocation* loc);
+
+ void popPayload(MacroAssembler& masm, OperandLocation* loc, Register dest);
+ void popValue(MacroAssembler& masm, OperandLocation* loc, ValueOperand dest);
+ Address payloadAddress(MacroAssembler& masm,
+ const OperandLocation* loc) const;
+ Address valueAddress(MacroAssembler& masm, const OperandLocation* loc) const;
+
+#ifdef DEBUG
+ void assertValidState() const;
+#endif
+
+ public:
+ friend class AutoScratchRegister;
+ friend class AutoScratchRegisterExcluding;
+
+ explicit CacheRegisterAllocator(const CacheIRWriter& writer)
+ : allocatableRegs_(GeneralRegisterSet::All()),
+ stackPushed_(0),
+#ifdef DEBUG
+ addedFailurePath_(false),
+#endif
+ currentInstruction_(0),
+ writer_(writer) {
+ }
+
+ [[nodiscard]] bool init();
+
+ void initAvailableRegs(const AllocatableGeneralRegisterSet& available) {
+ availableRegs_ = available;
+ }
+ void initAvailableRegsAfterSpill();
+
+ void fixupAliasedInputs(MacroAssembler& masm);
+
+ OperandLocation operandLocation(size_t i) const {
+ return operandLocations_[i];
+ }
+ void setOperandLocation(size_t i, const OperandLocation& loc) {
+ operandLocations_[i] = loc;
+ }
+
+ OperandLocation origInputLocation(size_t i) const {
+ return origInputLocations_[i];
+ }
+ void initInputLocation(size_t i, ValueOperand reg) {
+ origInputLocations_[i].setValueReg(reg);
+ operandLocations_[i].setValueReg(reg);
+ }
+ void initInputLocation(size_t i, Register reg, JSValueType type) {
+ origInputLocations_[i].setPayloadReg(reg, type);
+ operandLocations_[i].setPayloadReg(reg, type);
+ }
+ void initInputLocation(size_t i, FloatRegister reg) {
+ origInputLocations_[i].setDoubleReg(reg);
+ operandLocations_[i].setDoubleReg(reg);
+ }
+ void initInputLocation(size_t i, const Value& v) {
+ origInputLocations_[i].setConstant(v);
+ operandLocations_[i].setConstant(v);
+ }
+ void initInputLocation(size_t i, BaselineFrameSlot slot) {
+ origInputLocations_[i].setBaselineFrame(slot);
+ operandLocations_[i].setBaselineFrame(slot);
+ }
+
+ void initInputLocation(size_t i, const TypedOrValueRegister& reg);
+ void initInputLocation(size_t i, const ConstantOrRegister& value);
+
+ const SpilledRegisterVector& spilledRegs() const { return spilledRegs_; }
+
+ [[nodiscard]] bool setSpilledRegs(const SpilledRegisterVector& regs) {
+ spilledRegs_.clear();
+ return spilledRegs_.appendAll(regs);
+ }
+
+ bool hasAutoScratchFloatRegisterSpill() const {
+ return hasAutoScratchFloatRegisterSpill_;
+ }
+ void setHasAutoScratchFloatRegisterSpill(bool b) {
+ MOZ_ASSERT(hasAutoScratchFloatRegisterSpill_ != b);
+ hasAutoScratchFloatRegisterSpill_ = b;
+ }
+
+ void nextOp() {
+#ifdef DEBUG
+ assertValidState();
+ addedFailurePath_ = false;
+#endif
+ currentOpRegs_.clear();
+ currentInstruction_++;
+ }
+
+#ifdef DEBUG
+ void setAddedFailurePath() {
+ MOZ_ASSERT(!addedFailurePath_, "multiple failure paths for instruction");
+ addedFailurePath_ = true;
+ }
+#endif
+
+ bool isDeadAfterInstruction(OperandId opId) const {
+ return writer_.operandIsDead(opId.id(), currentInstruction_ + 1);
+ }
+
+ uint32_t stackPushed() const { return stackPushed_; }
+ void setStackPushed(uint32_t pushed) { stackPushed_ = pushed; }
+
+ bool isAllocatable(Register reg) const { return allocatableRegs_.has(reg); }
+
+ // Allocates a new register.
+ Register allocateRegister(MacroAssembler& masm);
+ ValueOperand allocateValueRegister(MacroAssembler& masm);
+
+ void allocateFixedRegister(MacroAssembler& masm, Register reg);
+ void allocateFixedValueRegister(MacroAssembler& masm, ValueOperand reg);
+
+ // Releases a register so it can be reused later.
+ void releaseRegister(Register reg) {
+ MOZ_ASSERT(currentOpRegs_.has(reg));
+ availableRegs_.add(reg);
+ currentOpRegs_.take(reg);
+ }
+ void releaseValueRegister(ValueOperand reg) {
+#ifdef JS_NUNBOX32
+ releaseRegister(reg.payloadReg());
+ releaseRegister(reg.typeReg());
+#else
+ releaseRegister(reg.valueReg());
+#endif
+ }
+
+ // Removes spilled values from the native stack. This should only be
+ // called after all registers have been allocated.
+ void discardStack(MacroAssembler& masm);
+
+ Address addressOf(MacroAssembler& masm, BaselineFrameSlot slot) const;
+ BaseValueIndex addressOf(MacroAssembler& masm, Register argcReg,
+ BaselineFrameSlot slot) const;
+
+ // Returns the register for the given operand. If the operand is currently
+ // not in a register, it will load it into one.
+ ValueOperand useValueRegister(MacroAssembler& masm, ValOperandId val);
+ Register useRegister(MacroAssembler& masm, TypedOperandId typedId);
+
+ ConstantOrRegister useConstantOrRegister(MacroAssembler& masm,
+ ValOperandId val);
+
+ // Allocates an output register for the given operand.
+ Register defineRegister(MacroAssembler& masm, TypedOperandId typedId);
+ ValueOperand defineValueRegister(MacroAssembler& masm, ValOperandId val);
+
+ // Loads (potentially coercing) and unboxes a value into a float register
+ // This is infallible, as there should have been a previous guard
+ // to ensure the value is already a number.
+ // Does not change the allocator's state.
+ void ensureDoubleRegister(MacroAssembler& masm, NumberOperandId op,
+ FloatRegister dest) const;
+
+ // Loads an unboxed value into a scratch register. This can be useful
+ // especially on 32-bit x86 when there are not enough registers for
+ // useRegister.
+ // Does not change the allocator's state.
+ void copyToScratchRegister(MacroAssembler& masm, TypedOperandId typedId,
+ Register dest) const;
+ void copyToScratchValueRegister(MacroAssembler& masm, ValOperandId valId,
+ ValueOperand dest) const;
+
+ // Returns |val|'s JSValueType or JSVAL_TYPE_UNKNOWN.
+ JSValueType knownType(ValOperandId val) const;
+
+ // Emits code to restore registers and stack to the state at the start of
+ // the stub.
+ void restoreInputState(MacroAssembler& masm, bool discardStack = true);
+
+ // Returns the set of registers storing the IC input operands.
+ GeneralRegisterSet inputRegisterSet() const;
+
+ void saveIonLiveRegisters(MacroAssembler& masm, LiveRegisterSet liveRegs,
+ Register scratch, IonScript* ionScript);
+ void restoreIonLiveRegisters(MacroAssembler& masm, LiveRegisterSet liveRegs);
+};
+
+// RAII class to allocate a scratch register and release it when we're done
+// with it.
+class MOZ_RAII AutoScratchRegister {
+ CacheRegisterAllocator& alloc_;
+ Register reg_;
+
+ AutoScratchRegister(const AutoScratchRegister&) = delete;
+ void operator=(const AutoScratchRegister&) = delete;
+
+ public:
+ AutoScratchRegister(CacheRegisterAllocator& alloc, MacroAssembler& masm,
+ Register reg = InvalidReg)
+ : alloc_(alloc) {
+ if (reg != InvalidReg) {
+ alloc.allocateFixedRegister(masm, reg);
+ reg_ = reg;
+ } else {
+ reg_ = alloc.allocateRegister(masm);
+ }
+ MOZ_ASSERT(alloc_.currentOpRegs_.has(reg_));
+ }
+ ~AutoScratchRegister() { alloc_.releaseRegister(reg_); }
+
+ Register get() const { return reg_; }
+ operator Register() const { return reg_; }
+};
+
+// On x86, spectreBoundsCheck32 can emit better code if it has a scratch
+// register and index masking is enabled.
+class MOZ_RAII AutoSpectreBoundsScratchRegister {
+ mozilla::Maybe<AutoScratchRegister> scratch_;
+ Register reg_ = InvalidReg;
+
+ AutoSpectreBoundsScratchRegister(const AutoSpectreBoundsScratchRegister&) =
+ delete;
+ void operator=(const AutoSpectreBoundsScratchRegister&) = delete;
+
+ public:
+ AutoSpectreBoundsScratchRegister(CacheRegisterAllocator& alloc,
+ MacroAssembler& masm) {
+#ifdef JS_CODEGEN_X86
+ if (JitOptions.spectreIndexMasking) {
+ scratch_.emplace(alloc, masm);
+ reg_ = scratch_->get();
+ }
+#endif
+ }
+
+ Register get() const { return reg_; }
+ operator Register() const { return reg_; }
+};
+
+// Scratch Register64. Implemented with a single AutoScratchRegister on 64-bit
+// platforms and two AutoScratchRegisters on 32-bit platforms.
+class MOZ_RAII AutoScratchRegister64 {
+ AutoScratchRegister reg1_;
+#if JS_BITS_PER_WORD == 32
+ AutoScratchRegister reg2_;
+#endif
+
+ public:
+ AutoScratchRegister64(const AutoScratchRegister64&) = delete;
+ void operator=(const AutoScratchRegister64&) = delete;
+
+#if JS_BITS_PER_WORD == 32
+ AutoScratchRegister64(CacheRegisterAllocator& alloc, MacroAssembler& masm)
+ : reg1_(alloc, masm), reg2_(alloc, masm) {}
+
+ Register64 get() const { return Register64(reg1_, reg2_); }
+#else
+ AutoScratchRegister64(CacheRegisterAllocator& alloc, MacroAssembler& masm)
+ : reg1_(alloc, masm) {}
+
+ Register64 get() const { return Register64(reg1_); }
+#endif
+
+ operator Register64() const { return get(); }
+};
+
+// Scratch ValueOperand. Implemented with a single AutoScratchRegister on 64-bit
+// platforms and two AutoScratchRegisters on 32-bit platforms.
+class MOZ_RAII AutoScratchValueRegister {
+ AutoScratchRegister reg1_;
+#if JS_BITS_PER_WORD == 32
+ AutoScratchRegister reg2_;
+#endif
+
+ public:
+ AutoScratchValueRegister(const AutoScratchValueRegister&) = delete;
+ void operator=(const AutoScratchValueRegister&) = delete;
+
+#if JS_BITS_PER_WORD == 32
+ AutoScratchValueRegister(CacheRegisterAllocator& alloc, MacroAssembler& masm)
+ : reg1_(alloc, masm), reg2_(alloc, masm) {}
+
+ ValueOperand get() const { return ValueOperand(reg1_, reg2_); }
+#else
+ AutoScratchValueRegister(CacheRegisterAllocator& alloc, MacroAssembler& masm)
+ : reg1_(alloc, masm) {}
+
+ ValueOperand get() const { return ValueOperand(reg1_); }
+#endif
+
+ operator ValueOperand() const { return get(); }
+};
+
+// The FailurePath class stores everything we need to generate a failure path
+// at the end of the IC code. The failure path restores the input registers, if
+// needed, and jumps to the next stub.
+class FailurePath {
+ Vector<OperandLocation, 4, SystemAllocPolicy> inputs_;
+ SpilledRegisterVector spilledRegs_;
+ NonAssertingLabel label_;
+ uint32_t stackPushed_;
+#ifdef DEBUG
+ // Flag to ensure FailurePath::label() isn't taken while there's a scratch
+ // float register which still needs to be restored.
+ bool hasAutoScratchFloatRegister_ = false;
+#endif
+
+ public:
+ FailurePath() = default;
+
+ FailurePath(FailurePath&& other)
+ : inputs_(std::move(other.inputs_)),
+ spilledRegs_(std::move(other.spilledRegs_)),
+ label_(other.label_),
+ stackPushed_(other.stackPushed_) {}
+
+ Label* labelUnchecked() { return &label_; }
+ Label* label() {
+ MOZ_ASSERT(!hasAutoScratchFloatRegister_);
+ return labelUnchecked();
+ }
+
+ void setStackPushed(uint32_t i) { stackPushed_ = i; }
+ uint32_t stackPushed() const { return stackPushed_; }
+
+ [[nodiscard]] bool appendInput(const OperandLocation& loc) {
+ return inputs_.append(loc);
+ }
+ OperandLocation input(size_t i) const { return inputs_[i]; }
+
+ const SpilledRegisterVector& spilledRegs() const { return spilledRegs_; }
+
+ [[nodiscard]] bool setSpilledRegs(const SpilledRegisterVector& regs) {
+ MOZ_ASSERT(spilledRegs_.empty());
+ return spilledRegs_.appendAll(regs);
+ }
+
+ // If canShareFailurePath(other) returns true, the same machine code will
+ // be emitted for two failure paths, so we can share them.
+ bool canShareFailurePath(const FailurePath& other) const;
+
+ void setHasAutoScratchFloatRegister() {
+#ifdef DEBUG
+ MOZ_ASSERT(!hasAutoScratchFloatRegister_);
+ hasAutoScratchFloatRegister_ = true;
+#endif
+ }
+
+ void clearHasAutoScratchFloatRegister() {
+#ifdef DEBUG
+ MOZ_ASSERT(hasAutoScratchFloatRegister_);
+ hasAutoScratchFloatRegister_ = false;
+#endif
+ }
+};
+
+/**
+ * Wrap an offset so that a call can decide to embed a constant
+ * or load from the stub data.
+ */
+class StubFieldOffset {
+ private:
+ uint32_t offset_;
+ StubField::Type type_;
+
+ public:
+ StubFieldOffset(uint32_t offset, StubField::Type type)
+ : offset_(offset), type_(type) {}
+
+ uint32_t getOffset() { return offset_; }
+ StubField::Type getStubFieldType() { return type_; }
+};
+
+class AutoOutputRegister;
+
+// Base class for BaselineCacheIRCompiler and IonCacheIRCompiler.
+class MOZ_RAII CacheIRCompiler {
+ protected:
+ friend class AutoOutputRegister;
+ friend class AutoStubFrame;
+ friend class AutoSaveLiveRegisters;
+ friend class AutoCallVM;
+ friend class AutoScratchFloatRegister;
+ friend class AutoAvailableFloatRegister;
+
+ enum class Mode { Baseline, Ion };
+
+ bool enteredStubFrame_;
+
+ bool isBaseline();
+ bool isIon();
+ BaselineCacheIRCompiler* asBaseline();
+ IonCacheIRCompiler* asIon();
+
+ JSContext* cx_;
+ const CacheIRWriter& writer_;
+ StackMacroAssembler masm;
+
+ CacheRegisterAllocator allocator;
+ Vector<FailurePath, 4, SystemAllocPolicy> failurePaths;
+
+ // Float registers that are live. Registers not in this set can be
+ // clobbered and don't need to be saved before performing a VM call.
+ // Doing this for non-float registers is a bit more complicated because
+ // the IC register allocator allocates GPRs.
+ LiveFloatRegisterSet liveFloatRegs_;
+
+ mozilla::Maybe<TypedOrValueRegister> outputUnchecked_;
+ Mode mode_;
+
+ // Distance from the IC to the stub data; mostly will be
+ // sizeof(stubType)
+ uint32_t stubDataOffset_;
+
+ enum class StubFieldPolicy { Address, Constant };
+
+ StubFieldPolicy stubFieldPolicy_;
+
+ CacheIRCompiler(JSContext* cx, TempAllocator& alloc,
+ const CacheIRWriter& writer, uint32_t stubDataOffset,
+ Mode mode, StubFieldPolicy policy)
+ : enteredStubFrame_(false),
+ cx_(cx),
+ writer_(writer),
+ masm(cx, alloc),
+ allocator(writer_),
+ liveFloatRegs_(FloatRegisterSet::All()),
+ mode_(mode),
+ stubDataOffset_(stubDataOffset),
+ stubFieldPolicy_(policy) {
+ MOZ_ASSERT(!writer.failed());
+ }
+
+ [[nodiscard]] bool addFailurePath(FailurePath** failure);
+ [[nodiscard]] bool emitFailurePath(size_t i);
+
+ // Returns the set of volatile float registers that are live. These
+ // registers need to be saved when making non-GC calls with callWithABI.
+ FloatRegisterSet liveVolatileFloatRegs() const {
+ return FloatRegisterSet::Intersect(liveFloatRegs_.set(),
+ FloatRegisterSet::Volatile());
+ }
+
+ bool objectGuardNeedsSpectreMitigations(ObjOperandId objId) const {
+ // Instructions like GuardShape need Spectre mitigations if
+ // (1) mitigations are enabled and (2) the object is used by other
+ // instructions (if the object is *not* used by other instructions,
+ // zeroing its register is pointless).
+ return JitOptions.spectreObjectMitigations &&
+ !allocator.isDeadAfterInstruction(objId);
+ }
+
+ private:
+ void emitPostBarrierShared(Register obj, const ConstantOrRegister& val,
+ Register scratch, Register maybeIndex);
+
+ void emitPostBarrierShared(Register obj, ValueOperand val, Register scratch,
+ Register maybeIndex) {
+ emitPostBarrierShared(obj, ConstantOrRegister(val), scratch, maybeIndex);
+ }
+
+ protected:
+ template <typename T>
+ void emitPostBarrierSlot(Register obj, const T& val, Register scratch) {
+ emitPostBarrierShared(obj, val, scratch, InvalidReg);
+ }
+
+ template <typename T>
+ void emitPostBarrierElement(Register obj, const T& val, Register scratch,
+ Register index) {
+ MOZ_ASSERT(index != InvalidReg);
+ emitPostBarrierShared(obj, val, scratch, index);
+ }
+
+ bool emitComparePointerResultShared(JSOp op, TypedOperandId lhsId,
+ TypedOperandId rhsId);
+
+ [[nodiscard]] bool emitMathFunctionNumberResultShared(
+ UnaryMathFunction fun, FloatRegister inputScratch, ValueOperand output);
+
+ template <typename Fn, Fn fn>
+ [[nodiscard]] bool emitBigIntBinaryOperationShared(BigIntOperandId lhsId,
+ BigIntOperandId rhsId);
+
+ template <typename Fn, Fn fn>
+ [[nodiscard]] bool emitBigIntUnaryOperationShared(BigIntOperandId inputId);
+
+ bool emitDoubleIncDecResult(bool isInc, NumberOperandId inputId);
+
+ using AtomicsReadWriteModifyFn = int32_t (*)(TypedArrayObject*, size_t,
+ int32_t);
+
+ [[nodiscard]] bool emitAtomicsReadModifyWriteResult(
+ ObjOperandId objId, IntPtrOperandId indexId, uint32_t valueId,
+ Scalar::Type elementType, AtomicsReadWriteModifyFn fn);
+
+ using AtomicsReadWriteModify64Fn = JS::BigInt* (*)(JSContext*,
+ TypedArrayObject*, size_t,
+ const JS::BigInt*);
+
+ template <AtomicsReadWriteModify64Fn fn>
+ [[nodiscard]] bool emitAtomicsReadModifyWriteResult64(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId);
+
+ void emitActivateIterator(Register objBeingIterated, Register iterObject,
+ Register nativeIter, Register scratch,
+ Register scratch2, uint32_t enumeratorsAddrOffset);
+
+ CACHE_IR_COMPILER_SHARED_GENERATED
+
+ void emitLoadStubField(StubFieldOffset val, Register dest);
+ void emitLoadStubFieldConstant(StubFieldOffset val, Register dest);
+
+ void emitLoadValueStubField(StubFieldOffset val, ValueOperand dest);
+ void emitLoadDoubleValueStubField(StubFieldOffset val, ValueOperand dest,
+ FloatRegister scratch);
+
+ uintptr_t readStubWord(uint32_t offset, StubField::Type type) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ MOZ_ASSERT((offset % sizeof(uintptr_t)) == 0);
+ return writer_.readStubField(offset, type).asWord();
+ }
+ uint64_t readStubInt64(uint32_t offset, StubField::Type type) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ MOZ_ASSERT((offset % sizeof(uintptr_t)) == 0);
+ return writer_.readStubField(offset, type).asInt64();
+ }
+ int32_t int32StubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return readStubWord(offset, StubField::Type::RawInt32);
+ }
+ uint32_t uint32StubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return readStubWord(offset, StubField::Type::RawInt32);
+ }
+ Shape* shapeStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (Shape*)readStubWord(offset, StubField::Type::Shape);
+ }
+ GetterSetter* getterSetterStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (GetterSetter*)readStubWord(offset, StubField::Type::GetterSetter);
+ }
+ JSObject* objectStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (JSObject*)readStubWord(offset, StubField::Type::JSObject);
+ }
+ Value valueStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ uint64_t raw = readStubInt64(offset, StubField::Type::Value);
+ return Value::fromRawBits(raw);
+ }
+ double doubleStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ uint64_t raw = readStubInt64(offset, StubField::Type::Double);
+ return mozilla::BitwiseCast<double>(raw);
+ }
+ JSString* stringStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (JSString*)readStubWord(offset, StubField::Type::String);
+ }
+ JS::Symbol* symbolStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (JS::Symbol*)readStubWord(offset, StubField::Type::Symbol);
+ }
+ JS::Compartment* compartmentStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (JS::Compartment*)readStubWord(offset, StubField::Type::RawPointer);
+ }
+ BaseScript* baseScriptStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (BaseScript*)readStubWord(offset, StubField::Type::BaseScript);
+ }
+ const JSClass* classStubField(uintptr_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (const JSClass*)readStubWord(offset, StubField::Type::RawPointer);
+ }
+ const void* proxyHandlerStubField(uintptr_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (const void*)readStubWord(offset, StubField::Type::RawPointer);
+ }
+ const void* pointerStubField(uintptr_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return (const void*)readStubWord(offset, StubField::Type::RawPointer);
+ }
+ jsid idStubField(uint32_t offset) {
+ MOZ_ASSERT(stubFieldPolicy_ == StubFieldPolicy::Constant);
+ return jsid::fromRawBits(readStubWord(offset, StubField::Type::Id));
+ }
+
+#ifdef DEBUG
+ void assertFloatRegisterAvailable(FloatRegister reg);
+#endif
+
+ InlineCachePerfSpewer perfSpewer_;
+
+ public:
+ InlineCachePerfSpewer& perfSpewer() { return perfSpewer_; }
+
+ void callVMInternal(MacroAssembler& masm, VMFunctionId id);
+ template <typename Fn, Fn fn>
+ void callVM(MacroAssembler& masm);
+};
+
+// Ensures the IC's output register is available for writing.
+class MOZ_RAII AutoOutputRegister {
+ TypedOrValueRegister output_;
+ CacheRegisterAllocator& alloc_;
+
+ AutoOutputRegister(const AutoOutputRegister&) = delete;
+ void operator=(const AutoOutputRegister&) = delete;
+
+ public:
+ explicit AutoOutputRegister(CacheIRCompiler& compiler);
+ ~AutoOutputRegister();
+
+ Register maybeReg() const {
+ if (output_.hasValue()) {
+ return output_.valueReg().scratchReg();
+ }
+ if (!output_.typedReg().isFloat()) {
+ return output_.typedReg().gpr();
+ }
+ return InvalidReg;
+ }
+
+ bool hasValue() const { return output_.hasValue(); }
+ ValueOperand valueReg() const { return output_.valueReg(); }
+ AnyRegister typedReg() const { return output_.typedReg(); }
+
+ JSValueType type() const {
+ MOZ_ASSERT(!hasValue());
+ return ValueTypeFromMIRType(output_.type());
+ }
+
+ operator TypedOrValueRegister() const { return output_; }
+};
+
+enum class CallCanGC { CanGC, CanNotGC };
+
+// Instructions that have to perform a callVM require a stub frame. Call its
+// enter() and leave() methods to enter/leave the stub frame.
+// Hoisted from jit/BaselineCacheIRCompiler.cpp. See there for method
+// definitions.
+class MOZ_RAII AutoStubFrame {
+ BaselineCacheIRCompiler& compiler;
+#ifdef DEBUG
+ uint32_t framePushedAtEnterStubFrame_;
+#endif
+
+ AutoStubFrame(const AutoStubFrame&) = delete;
+ void operator=(const AutoStubFrame&) = delete;
+
+ public:
+ explicit AutoStubFrame(BaselineCacheIRCompiler& compiler);
+
+ void enter(MacroAssembler& masm, Register scratch,
+ CallCanGC canGC = CallCanGC::CanGC);
+ void leave(MacroAssembler& masm);
+
+#ifdef DEBUG
+ ~AutoStubFrame();
+#endif
+};
+// AutoSaveLiveRegisters must be used when we make a call that can GC. The
+// constructor ensures all live registers are stored on the stack (where the GC
+// expects them) and the destructor restores these registers.
+class MOZ_RAII AutoSaveLiveRegisters {
+ IonCacheIRCompiler& compiler_;
+
+ AutoSaveLiveRegisters(const AutoSaveLiveRegisters&) = delete;
+ void operator=(const AutoSaveLiveRegisters&) = delete;
+
+ public:
+ explicit AutoSaveLiveRegisters(IonCacheIRCompiler& compiler);
+
+ ~AutoSaveLiveRegisters();
+};
+// Like AutoScratchRegister, but reuse a register of |output| if possible.
+class MOZ_RAII AutoScratchRegisterMaybeOutput {
+ mozilla::Maybe<AutoScratchRegister> scratch_;
+ Register scratchReg_;
+
+ AutoScratchRegisterMaybeOutput(const AutoScratchRegisterMaybeOutput&) =
+ delete;
+ void operator=(const AutoScratchRegisterMaybeOutput&) = delete;
+
+ public:
+ AutoScratchRegisterMaybeOutput(CacheRegisterAllocator& alloc,
+ MacroAssembler& masm,
+ const AutoOutputRegister& output) {
+ scratchReg_ = output.maybeReg();
+ if (scratchReg_ == InvalidReg) {
+ scratch_.emplace(alloc, masm);
+ scratchReg_ = scratch_.ref();
+ }
+ }
+ AutoScratchRegisterMaybeOutput(CacheRegisterAllocator& alloc,
+ MacroAssembler& masm) {
+ scratch_.emplace(alloc, masm);
+ scratchReg_ = scratch_.ref();
+ }
+
+ Register get() const { return scratchReg_; }
+ operator Register() const { return scratchReg_; }
+};
+
+// Like AutoScratchRegisterMaybeOutput, but tries to use the ValueOperand's
+// type register for the scratch register on 32-bit.
+//
+// Word of warning: Passing an instance of this class and AutoOutputRegister to
+// functions may not work correctly, because no guarantee is given that the type
+// register is used last when modifying the output's ValueOperand.
+class MOZ_RAII AutoScratchRegisterMaybeOutputType {
+ mozilla::Maybe<AutoScratchRegister> scratch_;
+ Register scratchReg_;
+
+ public:
+ AutoScratchRegisterMaybeOutputType(CacheRegisterAllocator& alloc,
+ MacroAssembler& masm,
+ const AutoOutputRegister& output) {
+#if defined(JS_NUNBOX32)
+ scratchReg_ = output.hasValue() ? output.valueReg().typeReg() : InvalidReg;
+#else
+ scratchReg_ = InvalidReg;
+#endif
+ if (scratchReg_ == InvalidReg) {
+ scratch_.emplace(alloc, masm);
+ scratchReg_ = scratch_.ref();
+ }
+ }
+
+ AutoScratchRegisterMaybeOutputType(
+ const AutoScratchRegisterMaybeOutputType&) = delete;
+
+ void operator=(const AutoScratchRegisterMaybeOutputType&) = delete;
+
+ Register get() const { return scratchReg_; }
+ operator Register() const { return scratchReg_; }
+};
+
+// AutoCallVM is a wrapper class that unifies methods shared by
+// IonCacheIRCompiler and BaselineCacheIRCompiler that perform a callVM, but
+// require stub specific functionality before performing the VM call.
+//
+// Expected Usage:
+//
+// OPs with implementations that may be unified by this class must:
+// - Be listed in the CACHEIR_OPS list but not in the CACHE_IR_SHARED_OPS
+// list
+// - Differ only in their use of `AutoSaveLiveRegisters`,
+// `AutoOutputRegister`, and `AutoScratchRegister`. The Ion
+// implementation will use `AutoSaveLiveRegisters` and
+// `AutoOutputRegister`, while the Baseline implementation will use
+// `AutoScratchRegister`.
+// - Both use the `callVM` method.
+//
+// Using AutoCallVM:
+// - The constructor initializes `AutoOutputRegister` for both compiler
+// types. Additionally it initializes an `AutoSaveLiveRegisters` for
+// CacheIRCompilers with the mode Ion, and initializes
+// `AutoScratchRegisterMaybeOutput` and `AutoStubFrame` variables for
+// compilers with mode Baseline.
+// - The `prepare()` method calls the IonCacheIRCompiler method
+// `prepareVMCall` for IonCacheIRCompilers, calls the `enter()` method of
+// `AutoStubFrame` for BaselineCacheIRCompilers, and calls the
+// `discardStack` method of the `Register` class for both compiler types.
+// - The `call()` method invokes `callVM` on the CacheIRCompiler and stores
+// the call result according to its type. Finally it calls the `leave`
+// method of `AutoStubFrame` for BaselineCacheIRCompilers.
+//
+// Expected Usage Example:
+// See: `CacheIRCompiler::emitCallGetSparseElementResult()`
+//
+// Restrictions:
+// - OPs that do not meet the criteria listed above can not be unified with
+// AutoCallVM
+//
+
+class MOZ_RAII AutoCallVM {
+ MacroAssembler& masm_;
+ CacheIRCompiler* compiler_;
+ CacheRegisterAllocator& allocator_;
+ mozilla::Maybe<AutoOutputRegister> output_;
+
+ // Baseline specific stuff
+ mozilla::Maybe<AutoStubFrame> stubFrame_;
+ mozilla::Maybe<AutoScratchRegisterMaybeOutput> scratch_;
+
+ // Ion specific stuff
+ mozilla::Maybe<AutoSaveLiveRegisters> save_;
+
+ void storeResult(JSValueType returnType);
+
+ template <typename Fn>
+ void storeResult();
+
+ void leaveBaselineStubFrame();
+
+ public:
+ AutoCallVM(MacroAssembler& masm, CacheIRCompiler* compiler,
+ CacheRegisterAllocator& allocator);
+
+ void prepare();
+
+ template <typename Fn, Fn fn>
+ void call() {
+ compiler_->callVM<Fn, fn>(masm_);
+ storeResult<Fn>();
+ leaveBaselineStubFrame();
+ }
+
+ template <typename Fn, Fn fn>
+ void callNoResult() {
+ compiler_->callVM<Fn, fn>(masm_);
+ leaveBaselineStubFrame();
+ }
+
+ const AutoOutputRegister& output() const { return *output_; }
+ ValueOperand outputValueReg() const { return output_->valueReg(); }
+};
+
+// RAII class to allocate FloatReg0 as a scratch register and release it when
+// we're done with it. The previous contents of FloatReg0 may be spilled on the
+// stack and, if necessary, are restored when the destructor runs.
+//
+// When FailurePath is passed to the constructor, FailurePath::label() must not
+// be used during the life time of the AutoScratchFloatRegister. Instead use
+// AutoScratchFloatRegister::failure().
+class MOZ_RAII AutoScratchFloatRegister {
+ Label failurePopReg_{};
+ CacheIRCompiler* compiler_;
+ FailurePath* failure_;
+
+ AutoScratchFloatRegister(const AutoScratchFloatRegister&) = delete;
+ void operator=(const AutoScratchFloatRegister&) = delete;
+
+ public:
+ explicit AutoScratchFloatRegister(CacheIRCompiler* compiler)
+ : AutoScratchFloatRegister(compiler, nullptr) {}
+
+ AutoScratchFloatRegister(CacheIRCompiler* compiler, FailurePath* failure);
+
+ ~AutoScratchFloatRegister();
+
+ Label* failure();
+
+ FloatRegister get() const { return FloatReg0; }
+ operator FloatRegister() const { return FloatReg0; }
+};
+
+// This class can be used to assert a certain FloatRegister is available. In
+// Baseline mode, all float registers are available. In Ion mode, only the
+// registers added as fixed temps in LIRGenerator are available.
+class MOZ_RAII AutoAvailableFloatRegister {
+ FloatRegister reg_;
+
+ AutoAvailableFloatRegister(const AutoAvailableFloatRegister&) = delete;
+ void operator=(const AutoAvailableFloatRegister&) = delete;
+
+ public:
+ explicit AutoAvailableFloatRegister(CacheIRCompiler& compiler,
+ FloatRegister reg)
+ : reg_(reg) {
+#ifdef DEBUG
+ compiler.assertFloatRegisterAvailable(reg);
+#endif
+ }
+
+ FloatRegister get() const { return reg_; }
+ operator FloatRegister() const { return reg_; }
+};
+
+// See the 'Sharing Baseline stub code' comment in CacheIR.h for a description
+// of this class.
+class CacheIRStubInfo {
+ const uint8_t* code_;
+ const uint8_t* fieldTypes_;
+ uint32_t length_;
+
+ CacheKind kind_;
+ ICStubEngine engine_;
+ uint8_t stubDataOffset_;
+ bool makesGCCalls_;
+
+ CacheIRStubInfo(CacheKind kind, ICStubEngine engine, bool makesGCCalls,
+ uint32_t stubDataOffset, const uint8_t* code,
+ uint32_t codeLength, const uint8_t* fieldTypes)
+ : code_(code),
+ fieldTypes_(fieldTypes),
+ length_(codeLength),
+ kind_(kind),
+ engine_(engine),
+ stubDataOffset_(stubDataOffset),
+ makesGCCalls_(makesGCCalls) {
+ MOZ_ASSERT(kind_ == kind, "Kind must fit in bitfield");
+ MOZ_ASSERT(engine_ == engine, "Engine must fit in bitfield");
+ MOZ_ASSERT(stubDataOffset_ == stubDataOffset,
+ "stubDataOffset must fit in uint8_t");
+ }
+
+ CacheIRStubInfo(const CacheIRStubInfo&) = delete;
+ CacheIRStubInfo& operator=(const CacheIRStubInfo&) = delete;
+
+ public:
+ CacheKind kind() const { return kind_; }
+ ICStubEngine engine() const { return engine_; }
+ bool makesGCCalls() const { return makesGCCalls_; }
+
+ const uint8_t* code() const { return code_; }
+ uint32_t codeLength() const { return length_; }
+ uint32_t stubDataOffset() const { return stubDataOffset_; }
+
+ size_t stubDataSize() const;
+
+ StubField::Type fieldType(uint32_t i) const {
+ return (StubField::Type)fieldTypes_[i];
+ }
+
+ static CacheIRStubInfo* New(CacheKind kind, ICStubEngine engine,
+ bool canMakeCalls, uint32_t stubDataOffset,
+ const CacheIRWriter& writer);
+
+ template <class Stub, class T>
+ js::GCPtr<T>& getStubField(Stub* stub, uint32_t offset) const;
+
+ template <class Stub, class T>
+ T* getPtrStubField(Stub* stub, uint32_t offset) const;
+
+ template <class T>
+ js::GCPtr<T>& getStubField(ICCacheIRStub* stub, uint32_t offset) const {
+ return getStubField<ICCacheIRStub, T>(stub, offset);
+ }
+
+ uintptr_t getStubRawWord(const uint8_t* stubData, uint32_t offset) const;
+ uintptr_t getStubRawWord(ICCacheIRStub* stub, uint32_t offset) const;
+
+ int64_t getStubRawInt64(const uint8_t* stubData, uint32_t offset) const;
+ int64_t getStubRawInt64(ICCacheIRStub* stub, uint32_t offset) const;
+
+ void replaceStubRawWord(uint8_t* stubData, uint32_t offset, uintptr_t oldWord,
+ uintptr_t newWord) const;
+};
+
+template <typename T>
+void TraceCacheIRStub(JSTracer* trc, T* stub, const CacheIRStubInfo* stubInfo);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_CacheIRCompiler_h */
diff --git a/js/src/jit/CacheIRGenerator.h b/js/src/jit/CacheIRGenerator.h
new file mode 100644
index 0000000000..52bc7f95d1
--- /dev/null
+++ b/js/src/jit/CacheIRGenerator.h
@@ -0,0 +1,893 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CacheIRGenerator_h
+#define jit_CacheIRGenerator_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+
+#include <stdint.h>
+
+#include "jstypes.h"
+#include "NamespaceImports.h"
+
+#include "jit/CacheIR.h"
+#include "jit/CacheIRWriter.h"
+#include "jit/ICState.h"
+#include "js/Id.h"
+#include "js/RootingAPI.h"
+#include "js/ScalarType.h"
+#include "js/TypeDecls.h"
+#include "js/Value.h"
+#include "js/ValueArray.h"
+#include "vm/Opcodes.h"
+
+class JSFunction;
+
+namespace JS {
+struct XrayJitInfo;
+}
+
+namespace js {
+
+class NativeObject;
+class PropertyResult;
+class ProxyObject;
+enum class UnaryMathFunction : uint8_t;
+
+namespace jit {
+
+class BaselineFrame;
+class Label;
+class MacroAssembler;
+struct Register;
+enum class InlinableNative : uint16_t;
+
+// Some ops refer to shapes that might be in other zones. Instead of putting
+// cross-zone pointers in the caches themselves (which would complicate tracing
+// enormously), these ops instead contain wrappers for objects in the target
+// zone, which refer to the actual shape via a reserved slot.
+void LoadShapeWrapperContents(MacroAssembler& masm, Register obj, Register dst,
+ Label* failure);
+
+class MOZ_RAII IRGenerator {
+ protected:
+ CacheIRWriter writer;
+ JSContext* cx_;
+ HandleScript script_;
+ jsbytecode* pc_;
+ CacheKind cacheKind_;
+ ICState::Mode mode_;
+ bool isFirstStub_;
+
+ // Important: This pointer may be passed to the profiler. If this is non-null,
+ // it must point to a C string literal with static lifetime, not a heap- or
+ // stack- allocated string.
+ const char* stubName_ = nullptr;
+
+ IRGenerator(const IRGenerator&) = delete;
+ IRGenerator& operator=(const IRGenerator&) = delete;
+
+ bool maybeGuardInt32Index(const Value& index, ValOperandId indexId,
+ uint32_t* int32Index, Int32OperandId* int32IndexId);
+
+ IntPtrOperandId guardToIntPtrIndex(const Value& index, ValOperandId indexId,
+ bool supportOOB);
+
+ ObjOperandId guardDOMProxyExpandoObjectAndShape(ProxyObject* obj,
+ ObjOperandId objId,
+ const Value& expandoVal,
+ NativeObject* expandoObj);
+
+ void emitIdGuard(ValOperandId valId, const Value& idVal, jsid id);
+
+ OperandId emitNumericGuard(ValOperandId valId, Scalar::Type type);
+
+ StringOperandId emitToStringGuard(ValOperandId id, const Value& v);
+
+ void emitCalleeGuard(ObjOperandId calleeId, JSFunction* callee);
+
+ friend class CacheIRSpewer;
+
+ public:
+ explicit IRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc,
+ CacheKind cacheKind, ICState state);
+
+ const CacheIRWriter& writerRef() const { return writer; }
+ CacheKind cacheKind() const { return cacheKind_; }
+
+ // See comment on `stubName_` above.
+ const char* stubName() const { return stubName_; }
+
+ static constexpr char* NotAttached = nullptr;
+};
+
+// GetPropIRGenerator generates CacheIR for a GetProp IC.
+class MOZ_RAII GetPropIRGenerator : public IRGenerator {
+ HandleValue val_;
+ HandleValue idVal_;
+
+ AttachDecision tryAttachNative(HandleObject obj, ObjOperandId objId,
+ HandleId id, ValOperandId receiverId);
+ AttachDecision tryAttachObjectLength(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachTypedArray(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachDataView(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachArrayBufferMaybeShared(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachRegExp(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachMap(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachSet(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachModuleNamespace(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachWindowProxy(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachCrossCompartmentWrapper(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachXrayCrossCompartmentWrapper(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id,
+ ValOperandId receiverId);
+ AttachDecision tryAttachFunction(HandleObject obj, ObjOperandId objId,
+ HandleId id);
+ AttachDecision tryAttachArgumentsObjectIterator(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id);
+
+ AttachDecision tryAttachGenericProxy(Handle<ProxyObject*> obj,
+ ObjOperandId objId, HandleId id,
+ bool handleDOMProxies);
+ AttachDecision tryAttachDOMProxyExpando(Handle<ProxyObject*> obj,
+ ObjOperandId objId, HandleId id,
+ ValOperandId receiverId);
+ AttachDecision tryAttachDOMProxyShadowed(Handle<ProxyObject*> obj,
+ ObjOperandId objId, HandleId id);
+ AttachDecision tryAttachDOMProxyUnshadowed(Handle<ProxyObject*> obj,
+ ObjOperandId objId, HandleId id,
+ ValOperandId receiverId);
+ AttachDecision tryAttachProxy(HandleObject obj, ObjOperandId objId,
+ HandleId id, ValOperandId receiverId);
+
+ AttachDecision tryAttachPrimitive(ValOperandId valId, HandleId id);
+ AttachDecision tryAttachStringChar(ValOperandId valId, ValOperandId indexId);
+ AttachDecision tryAttachStringLength(ValOperandId valId, HandleId id);
+
+ AttachDecision tryAttachArgumentsObjectArg(HandleObject obj,
+ ObjOperandId objId, uint32_t index,
+ Int32OperandId indexId);
+ AttachDecision tryAttachArgumentsObjectArgHole(HandleObject obj,
+ ObjOperandId objId,
+ uint32_t index,
+ Int32OperandId indexId);
+ AttachDecision tryAttachArgumentsObjectCallee(HandleObject obj,
+ ObjOperandId objId,
+ HandleId id);
+
+ AttachDecision tryAttachDenseElement(HandleObject obj, ObjOperandId objId,
+ uint32_t index, Int32OperandId indexId);
+ AttachDecision tryAttachDenseElementHole(HandleObject obj, ObjOperandId objId,
+ uint32_t index,
+ Int32OperandId indexId);
+ AttachDecision tryAttachSparseElement(HandleObject obj, ObjOperandId objId,
+ uint32_t index, Int32OperandId indexId);
+ AttachDecision tryAttachTypedArrayElement(HandleObject obj,
+ ObjOperandId objId);
+
+ AttachDecision tryAttachGenericElement(HandleObject obj, ObjOperandId objId,
+ uint32_t index, Int32OperandId indexId,
+ ValOperandId receiverId);
+
+ AttachDecision tryAttachProxyElement(HandleObject obj, ObjOperandId objId);
+
+ void attachMegamorphicNativeSlot(ObjOperandId objId, jsid id);
+
+ ValOperandId getElemKeyValueId() const {
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetElem ||
+ cacheKind_ == CacheKind::GetElemSuper);
+ return ValOperandId(1);
+ }
+
+ ValOperandId getSuperReceiverValueId() const {
+ if (cacheKind_ == CacheKind::GetPropSuper) {
+ return ValOperandId(1);
+ }
+
+ MOZ_ASSERT(cacheKind_ == CacheKind::GetElemSuper);
+ return ValOperandId(2);
+ }
+
+ bool isSuper() const {
+ return (cacheKind_ == CacheKind::GetPropSuper ||
+ cacheKind_ == CacheKind::GetElemSuper);
+ }
+
+ // If this is a GetElem cache, emit instructions to guard the incoming Value
+ // matches |id|.
+ void maybeEmitIdGuard(jsid id);
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ GetPropIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc,
+ ICState state, CacheKind cacheKind, HandleValue val,
+ HandleValue idVal);
+
+ AttachDecision tryAttachStub();
+};
+
+// GetNameIRGenerator generates CacheIR for a GetName IC.
+class MOZ_RAII GetNameIRGenerator : public IRGenerator {
+ HandleObject env_;
+ Handle<PropertyName*> name_;
+
+ AttachDecision tryAttachGlobalNameValue(ObjOperandId objId, HandleId id);
+ AttachDecision tryAttachGlobalNameGetter(ObjOperandId objId, HandleId id);
+ AttachDecision tryAttachEnvironmentName(ObjOperandId objId, HandleId id);
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ GetNameIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc,
+ ICState state, HandleObject env,
+ Handle<PropertyName*> name);
+
+ AttachDecision tryAttachStub();
+};
+
+// BindNameIRGenerator generates CacheIR for a BindName IC.
+class MOZ_RAII BindNameIRGenerator : public IRGenerator {
+ HandleObject env_;
+ Handle<PropertyName*> name_;
+
+ AttachDecision tryAttachGlobalName(ObjOperandId objId, HandleId id);
+ AttachDecision tryAttachEnvironmentName(ObjOperandId objId, HandleId id);
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ BindNameIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc,
+ ICState state, HandleObject env,
+ Handle<PropertyName*> name);
+
+ AttachDecision tryAttachStub();
+};
+
+// SetPropIRGenerator generates CacheIR for a SetProp IC.
+class MOZ_RAII SetPropIRGenerator : public IRGenerator {
+ HandleValue lhsVal_;
+ HandleValue idVal_;
+ HandleValue rhsVal_;
+
+ public:
+ enum class DeferType { None, AddSlot };
+
+ private:
+ DeferType deferType_ = DeferType::None;
+
+ ValOperandId setElemKeyValueId() const {
+ MOZ_ASSERT(cacheKind_ == CacheKind::SetElem);
+ return ValOperandId(1);
+ }
+
+ ValOperandId rhsValueId() const {
+ if (cacheKind_ == CacheKind::SetProp) {
+ return ValOperandId(1);
+ }
+ MOZ_ASSERT(cacheKind_ == CacheKind::SetElem);
+ return ValOperandId(2);
+ }
+
+ // If this is a SetElem cache, emit instructions to guard the incoming Value
+ // matches |id|.
+ void maybeEmitIdGuard(jsid id);
+
+ AttachDecision tryAttachNativeSetSlot(HandleObject obj, ObjOperandId objId,
+ HandleId id, ValOperandId rhsId);
+ AttachDecision tryAttachSetter(HandleObject obj, ObjOperandId objId,
+ HandleId id, ValOperandId rhsId);
+ AttachDecision tryAttachSetArrayLength(HandleObject obj, ObjOperandId objId,
+ HandleId id, ValOperandId rhsId);
+ AttachDecision tryAttachWindowProxy(HandleObject obj, ObjOperandId objId,
+ HandleId id, ValOperandId rhsId);
+
+ AttachDecision tryAttachSetDenseElement(HandleObject obj, ObjOperandId objId,
+ uint32_t index,
+ Int32OperandId indexId,
+ ValOperandId rhsId);
+ AttachDecision tryAttachSetTypedArrayElement(HandleObject obj,
+ ObjOperandId objId,
+ ValOperandId rhsId);
+
+ AttachDecision tryAttachSetDenseElementHole(HandleObject obj,
+ ObjOperandId objId,
+ uint32_t index,
+ Int32OperandId indexId,
+ ValOperandId rhsId);
+
+ AttachDecision tryAttachAddOrUpdateSparseElement(HandleObject obj,
+ ObjOperandId objId,
+ uint32_t index,
+ Int32OperandId indexId,
+ ValOperandId rhsId);
+
+ AttachDecision tryAttachGenericProxy(Handle<ProxyObject*> obj,
+ ObjOperandId objId, HandleId id,
+ ValOperandId rhsId,
+ bool handleDOMProxies);
+ AttachDecision tryAttachDOMProxyShadowed(Handle<ProxyObject*> obj,
+ ObjOperandId objId, HandleId id,
+ ValOperandId rhsId);
+ AttachDecision tryAttachDOMProxyUnshadowed(Handle<ProxyObject*> obj,
+ ObjOperandId objId, HandleId id,
+ ValOperandId rhsId);
+ AttachDecision tryAttachDOMProxyExpando(Handle<ProxyObject*> obj,
+ ObjOperandId objId, HandleId id,
+ ValOperandId rhsId);
+ AttachDecision tryAttachProxy(HandleObject obj, ObjOperandId objId,
+ HandleId id, ValOperandId rhsId);
+ AttachDecision tryAttachProxyElement(HandleObject obj, ObjOperandId objId,
+ ValOperandId rhsId);
+ AttachDecision tryAttachMegamorphicSetElement(HandleObject obj,
+ ObjOperandId objId,
+ ValOperandId rhsId);
+
+ bool canAttachAddSlotStub(HandleObject obj, HandleId id);
+
+ public:
+ SetPropIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc,
+ CacheKind cacheKind, ICState state, HandleValue lhsVal,
+ HandleValue idVal, HandleValue rhsVal);
+
+ AttachDecision tryAttachStub();
+ AttachDecision tryAttachAddSlotStub(Handle<Shape*> oldShape);
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ DeferType deferType() const { return deferType_; }
+};
+
+// HasPropIRGenerator generates CacheIR for a HasProp IC. Used for
+// CacheKind::In / CacheKind::HasOwn.
+class MOZ_RAII HasPropIRGenerator : public IRGenerator {
+ HandleValue val_;
+ HandleValue idVal_;
+
+ AttachDecision tryAttachDense(HandleObject obj, ObjOperandId objId,
+ uint32_t index, Int32OperandId indexId);
+ AttachDecision tryAttachDenseHole(HandleObject obj, ObjOperandId objId,
+ uint32_t index, Int32OperandId indexId);
+ AttachDecision tryAttachTypedArray(HandleObject obj, ObjOperandId objId,
+ ValOperandId keyId);
+ AttachDecision tryAttachSparse(HandleObject obj, ObjOperandId objId,
+ Int32OperandId indexId);
+ AttachDecision tryAttachArgumentsObjectArg(HandleObject obj,
+ ObjOperandId objId,
+ Int32OperandId indexId);
+ AttachDecision tryAttachNamedProp(HandleObject obj, ObjOperandId objId,
+ HandleId key, ValOperandId keyId);
+ AttachDecision tryAttachMegamorphic(ObjOperandId objId, ValOperandId keyId);
+ AttachDecision tryAttachNative(NativeObject* obj, ObjOperandId objId,
+ jsid key, ValOperandId keyId,
+ PropertyResult prop, NativeObject* holder);
+ AttachDecision tryAttachSlotDoesNotExist(NativeObject* obj,
+ ObjOperandId objId, jsid key,
+ ValOperandId keyId);
+ AttachDecision tryAttachDoesNotExist(HandleObject obj, ObjOperandId objId,
+ HandleId key, ValOperandId keyId);
+ AttachDecision tryAttachProxyElement(HandleObject obj, ObjOperandId objId,
+ ValOperandId keyId);
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ // NOTE: Argument order is PROPERTY, OBJECT
+ HasPropIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc,
+ ICState state, CacheKind cacheKind, HandleValue idVal,
+ HandleValue val);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII CheckPrivateFieldIRGenerator : public IRGenerator {
+ HandleValue val_;
+ HandleValue idVal_;
+
+ AttachDecision tryAttachNative(NativeObject* obj, ObjOperandId objId,
+ jsid key, ValOperandId keyId,
+ PropertyResult prop);
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ CheckPrivateFieldIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ CacheKind cacheKind, HandleValue idVal,
+ HandleValue val);
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII InstanceOfIRGenerator : public IRGenerator {
+ HandleValue lhsVal_;
+ HandleObject rhsObj_;
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ InstanceOfIRGenerator(JSContext*, HandleScript, jsbytecode*, ICState,
+ HandleValue, HandleObject);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII TypeOfIRGenerator : public IRGenerator {
+ HandleValue val_;
+
+ AttachDecision tryAttachPrimitive(ValOperandId valId);
+ AttachDecision tryAttachObject(ValOperandId valId);
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ TypeOfIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc, ICState state,
+ HandleValue value);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII GetIteratorIRGenerator : public IRGenerator {
+ HandleValue val_;
+
+ AttachDecision tryAttachNativeIterator(ValOperandId valId);
+ AttachDecision tryAttachNullOrUndefined(ValOperandId valId);
+ AttachDecision tryAttachMegamorphic(ValOperandId valId);
+
+ public:
+ GetIteratorIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc,
+ ICState state, HandleValue value);
+
+ AttachDecision tryAttachStub();
+
+ void trackAttached(const char* name /* must be a C string literal */);
+};
+
+class MOZ_RAII OptimizeSpreadCallIRGenerator : public IRGenerator {
+ HandleValue val_;
+
+ AttachDecision tryAttachArray();
+ AttachDecision tryAttachArguments();
+ AttachDecision tryAttachNotOptimizable();
+
+ public:
+ OptimizeSpreadCallIRGenerator(JSContext* cx, HandleScript script,
+ jsbytecode* pc, ICState state,
+ HandleValue value);
+
+ AttachDecision tryAttachStub();
+
+ void trackAttached(const char* name /* must be a C string literal */);
+};
+
+enum class StringChar { CodeAt, At };
+enum class ScriptedThisResult { NoAction, UninitializedThis, PlainObjectShape };
+
+class MOZ_RAII CallIRGenerator : public IRGenerator {
+ private:
+ JSOp op_;
+ uint32_t argc_;
+ HandleValue callee_;
+ HandleValue thisval_;
+ HandleValue newTarget_;
+ HandleValueArray args_;
+
+ friend class InlinableNativeIRGenerator;
+
+ ScriptedThisResult getThisShapeForScripted(HandleFunction calleeFunc,
+ MutableHandle<Shape*> result);
+
+ ObjOperandId emitFunCallOrApplyGuard(Int32OperandId argcId);
+ ObjOperandId emitFunCallGuard(Int32OperandId argcId);
+ ObjOperandId emitFunApplyGuard(Int32OperandId argcId);
+ ObjOperandId emitFunApplyArgsGuard(CallFlags::ArgFormat format);
+
+ AttachDecision tryAttachFunCall(HandleFunction calleeFunc);
+ AttachDecision tryAttachFunApply(HandleFunction calleeFunc);
+ AttachDecision tryAttachCallScripted(HandleFunction calleeFunc);
+ AttachDecision tryAttachInlinableNative(HandleFunction calleeFunc,
+ CallFlags flags);
+ AttachDecision tryAttachWasmCall(HandleFunction calleeFunc);
+ AttachDecision tryAttachCallNative(HandleFunction calleeFunc);
+ AttachDecision tryAttachCallHook(HandleObject calleeObj);
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ CallIRGenerator(JSContext* cx, HandleScript script, jsbytecode* pc, JSOp op,
+ ICState state, uint32_t argc, HandleValue callee,
+ HandleValue thisval, HandleValue newTarget,
+ HandleValueArray args);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII InlinableNativeIRGenerator {
+ CallIRGenerator& generator_;
+ CacheIRWriter& writer;
+ JSContext* cx_;
+
+ HandleFunction callee_;
+ HandleValue newTarget_;
+ HandleValue thisval_;
+ HandleValueArray args_;
+ uint32_t argc_;
+ CallFlags flags_;
+
+ HandleScript script() const { return generator_.script_; }
+ bool isFirstStub() const { return generator_.isFirstStub_; }
+ bool ignoresResult() const { return generator_.op_ == JSOp::CallIgnoresRv; }
+
+ void emitNativeCalleeGuard();
+
+ ObjOperandId emitLoadArgsArray();
+
+ void initializeInputOperand() {
+ // The input operand is already initialized for FunCall and FunApplyArray.
+ if (flags_.getArgFormat() == CallFlags::FunCall ||
+ flags_.getArgFormat() == CallFlags::FunApplyArray) {
+ return;
+ }
+ (void)writer.setInputOperandId(0);
+ }
+
+ auto emitToStringGuard(ValOperandId id, const Value& v) {
+ return generator_.emitToStringGuard(id, v);
+ }
+
+ auto emitNumericGuard(ValOperandId valId, Scalar::Type type) {
+ return generator_.emitNumericGuard(valId, type);
+ }
+
+ auto guardToIntPtrIndex(const Value& index, ValOperandId indexId,
+ bool supportOOB) {
+ return generator_.guardToIntPtrIndex(index, indexId, supportOOB);
+ }
+
+ bool canAttachAtomicsReadWriteModify();
+
+ struct AtomicsReadWriteModifyOperands {
+ ObjOperandId objId;
+ IntPtrOperandId intPtrIndexId;
+ OperandId numericValueId;
+ };
+
+ AtomicsReadWriteModifyOperands emitAtomicsReadWriteModifyOperands();
+
+ AttachDecision tryAttachArrayPush();
+ AttachDecision tryAttachArrayPopShift(InlinableNative native);
+ AttachDecision tryAttachArrayJoin();
+ AttachDecision tryAttachArraySlice();
+ AttachDecision tryAttachArrayIsArray();
+ AttachDecision tryAttachDataViewGet(Scalar::Type type);
+ AttachDecision tryAttachDataViewSet(Scalar::Type type);
+ AttachDecision tryAttachUnsafeGetReservedSlot(InlinableNative native);
+ AttachDecision tryAttachUnsafeSetReservedSlot();
+ AttachDecision tryAttachIsSuspendedGenerator();
+ AttachDecision tryAttachToObject();
+ AttachDecision tryAttachToInteger();
+ AttachDecision tryAttachToLength();
+ AttachDecision tryAttachIsObject();
+ AttachDecision tryAttachIsPackedArray();
+ AttachDecision tryAttachIsCallable();
+ AttachDecision tryAttachIsConstructor();
+ AttachDecision tryAttachIsCrossRealmArrayConstructor();
+ AttachDecision tryAttachGuardToClass(InlinableNative native);
+ AttachDecision tryAttachHasClass(const JSClass* clasp,
+ bool isPossiblyWrapped);
+ AttachDecision tryAttachRegExpMatcherSearcherTester(InlinableNative native);
+ AttachDecision tryAttachRegExpPrototypeOptimizable();
+ AttachDecision tryAttachRegExpInstanceOptimizable();
+ AttachDecision tryAttachGetFirstDollarIndex();
+ AttachDecision tryAttachSubstringKernel();
+ AttachDecision tryAttachObjectHasPrototype();
+ AttachDecision tryAttachString();
+ AttachDecision tryAttachStringConstructor();
+ AttachDecision tryAttachStringToStringValueOf();
+ AttachDecision tryAttachStringChar(StringChar kind);
+ AttachDecision tryAttachStringCharCodeAt();
+ AttachDecision tryAttachStringCharAt();
+ AttachDecision tryAttachStringFromCharCode();
+ AttachDecision tryAttachStringFromCodePoint();
+ AttachDecision tryAttachStringIndexOf();
+ AttachDecision tryAttachStringStartsWith();
+ AttachDecision tryAttachStringEndsWith();
+ AttachDecision tryAttachStringToLowerCase();
+ AttachDecision tryAttachStringToUpperCase();
+ AttachDecision tryAttachStringReplaceString();
+ AttachDecision tryAttachStringSplitString();
+ AttachDecision tryAttachMathRandom();
+ AttachDecision tryAttachMathAbs();
+ AttachDecision tryAttachMathClz32();
+ AttachDecision tryAttachMathSign();
+ AttachDecision tryAttachMathImul();
+ AttachDecision tryAttachMathFloor();
+ AttachDecision tryAttachMathCeil();
+ AttachDecision tryAttachMathTrunc();
+ AttachDecision tryAttachMathRound();
+ AttachDecision tryAttachMathSqrt();
+ AttachDecision tryAttachMathFRound();
+ AttachDecision tryAttachMathHypot();
+ AttachDecision tryAttachMathATan2();
+ AttachDecision tryAttachMathFunction(UnaryMathFunction fun);
+ AttachDecision tryAttachMathPow();
+ AttachDecision tryAttachMathMinMax(bool isMax);
+ AttachDecision tryAttachSpreadMathMinMax(bool isMax);
+ AttachDecision tryAttachIsTypedArray(bool isPossiblyWrapped);
+ AttachDecision tryAttachIsTypedArrayConstructor();
+ AttachDecision tryAttachTypedArrayByteOffset();
+ AttachDecision tryAttachTypedArrayElementSize();
+ AttachDecision tryAttachTypedArrayLength(bool isPossiblyWrapped);
+ AttachDecision tryAttachArrayBufferByteLength(bool isPossiblyWrapped);
+ AttachDecision tryAttachIsConstructing();
+ AttachDecision tryAttachGetNextMapSetEntryForIterator(bool isMap);
+ AttachDecision tryAttachFinishBoundFunctionInit();
+ AttachDecision tryAttachNewArrayIterator();
+ AttachDecision tryAttachNewStringIterator();
+ AttachDecision tryAttachNewRegExpStringIterator();
+ AttachDecision tryAttachArrayIteratorPrototypeOptimizable();
+ AttachDecision tryAttachObjectCreate();
+ AttachDecision tryAttachObjectConstructor();
+ AttachDecision tryAttachArrayConstructor();
+ AttachDecision tryAttachTypedArrayConstructor();
+ AttachDecision tryAttachNumber();
+ AttachDecision tryAttachNumberParseInt();
+ AttachDecision tryAttachNumberToString();
+ AttachDecision tryAttachReflectGetPrototypeOf();
+ AttachDecision tryAttachAtomicsCompareExchange();
+ AttachDecision tryAttachAtomicsExchange();
+ AttachDecision tryAttachAtomicsAdd();
+ AttachDecision tryAttachAtomicsSub();
+ AttachDecision tryAttachAtomicsAnd();
+ AttachDecision tryAttachAtomicsOr();
+ AttachDecision tryAttachAtomicsXor();
+ AttachDecision tryAttachAtomicsLoad();
+ AttachDecision tryAttachAtomicsStore();
+ AttachDecision tryAttachAtomicsIsLockFree();
+ AttachDecision tryAttachBoolean();
+ AttachDecision tryAttachBailout();
+ AttachDecision tryAttachAssertFloat32();
+ AttachDecision tryAttachAssertRecoveredOnBailout();
+ AttachDecision tryAttachObjectIs();
+ AttachDecision tryAttachObjectIsPrototypeOf();
+ AttachDecision tryAttachObjectToString();
+ AttachDecision tryAttachBigIntAsIntN();
+ AttachDecision tryAttachBigIntAsUintN();
+ AttachDecision tryAttachSetHas();
+ AttachDecision tryAttachMapHas();
+ AttachDecision tryAttachMapGet();
+#ifdef FUZZING_JS_FUZZILLI
+ AttachDecision tryAttachFuzzilliHash();
+#endif
+
+ void trackAttached(const char* name /* must be a C string literal */) {
+ return generator_.trackAttached(name);
+ }
+
+ public:
+ InlinableNativeIRGenerator(CallIRGenerator& generator, HandleFunction callee,
+ HandleValue newTarget, HandleValue thisValue,
+ HandleValueArray args, CallFlags flags)
+ : generator_(generator),
+ writer(generator.writer),
+ cx_(generator.cx_),
+ callee_(callee),
+ newTarget_(newTarget),
+ thisval_(thisValue),
+ args_(args),
+ argc_(args.length()),
+ flags_(flags) {}
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII CompareIRGenerator : public IRGenerator {
+ JSOp op_;
+ HandleValue lhsVal_;
+ HandleValue rhsVal_;
+
+ AttachDecision tryAttachString(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachObject(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachSymbol(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachStrictDifferentTypes(ValOperandId lhsId,
+ ValOperandId rhsId);
+ AttachDecision tryAttachInt32(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachNumber(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachBigInt(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachNumberUndefined(ValOperandId lhsId,
+ ValOperandId rhsId);
+ AttachDecision tryAttachAnyNullUndefined(ValOperandId lhsId,
+ ValOperandId rhsId);
+ AttachDecision tryAttachNullUndefined(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachStringNumber(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachPrimitiveSymbol(ValOperandId lhsId,
+ ValOperandId rhsId);
+ AttachDecision tryAttachBoolStringOrNumber(ValOperandId lhsId,
+ ValOperandId rhsId);
+ AttachDecision tryAttachBigIntInt32(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachBigIntNumber(ValOperandId lhsId, ValOperandId rhsId);
+ AttachDecision tryAttachBigIntString(ValOperandId lhsId, ValOperandId rhsId);
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ CompareIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc, ICState state,
+ JSOp op, HandleValue lhsVal, HandleValue rhsVal);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII ToBoolIRGenerator : public IRGenerator {
+ HandleValue val_;
+
+ AttachDecision tryAttachBool();
+ AttachDecision tryAttachInt32();
+ AttachDecision tryAttachNumber();
+ AttachDecision tryAttachString();
+ AttachDecision tryAttachSymbol();
+ AttachDecision tryAttachNullOrUndefined();
+ AttachDecision tryAttachObject();
+ AttachDecision tryAttachBigInt();
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ ToBoolIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc, ICState state,
+ HandleValue val);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII GetIntrinsicIRGenerator : public IRGenerator {
+ HandleValue val_;
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ GetIntrinsicIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc,
+ ICState state, HandleValue val);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII UnaryArithIRGenerator : public IRGenerator {
+ JSOp op_;
+ HandleValue val_;
+ HandleValue res_;
+
+ AttachDecision tryAttachInt32();
+ AttachDecision tryAttachNumber();
+ AttachDecision tryAttachBitwise();
+ AttachDecision tryAttachBigInt();
+ AttachDecision tryAttachStringInt32();
+ AttachDecision tryAttachStringNumber();
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ UnaryArithIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc,
+ ICState state, JSOp op, HandleValue val,
+ HandleValue res);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII ToPropertyKeyIRGenerator : public IRGenerator {
+ HandleValue val_;
+
+ AttachDecision tryAttachInt32();
+ AttachDecision tryAttachNumber();
+ AttachDecision tryAttachString();
+ AttachDecision tryAttachSymbol();
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ ToPropertyKeyIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc,
+ ICState state, HandleValue val);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII BinaryArithIRGenerator : public IRGenerator {
+ JSOp op_;
+ HandleValue lhs_;
+ HandleValue rhs_;
+ HandleValue res_;
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ AttachDecision tryAttachInt32();
+ AttachDecision tryAttachDouble();
+ AttachDecision tryAttachBitwise();
+ AttachDecision tryAttachStringConcat();
+ AttachDecision tryAttachStringObjectConcat();
+ AttachDecision tryAttachBigInt();
+ AttachDecision tryAttachStringInt32Arith();
+
+ public:
+ BinaryArithIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc,
+ ICState state, JSOp op, HandleValue lhs,
+ HandleValue rhs, HandleValue res);
+
+ AttachDecision tryAttachStub();
+};
+
+class MOZ_RAII NewArrayIRGenerator : public IRGenerator {
+#ifdef JS_CACHEIR_SPEW
+ JSOp op_;
+#endif
+ HandleObject templateObject_;
+ BaselineFrame* frame_;
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ NewArrayIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc,
+ ICState state, JSOp op, HandleObject templateObj,
+ BaselineFrame* frame);
+
+ AttachDecision tryAttachStub();
+ AttachDecision tryAttachArrayObject();
+};
+
+class MOZ_RAII NewObjectIRGenerator : public IRGenerator {
+#ifdef JS_CACHEIR_SPEW
+ JSOp op_;
+#endif
+ HandleObject templateObject_;
+ BaselineFrame* frame_;
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ NewObjectIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc,
+ ICState state, JSOp op, HandleObject templateObj,
+ BaselineFrame* frame);
+
+ AttachDecision tryAttachStub();
+ AttachDecision tryAttachPlainObject();
+};
+
+inline bool BytecodeOpCanHaveAllocSite(JSOp op) {
+ return op == JSOp::NewArray || op == JSOp::NewObject || op == JSOp::NewInit;
+}
+
+class MOZ_RAII CloseIterIRGenerator : public IRGenerator {
+ HandleObject iter_;
+ CompletionKind kind_;
+
+ void trackAttached(const char* name /* must be a C string literal */);
+
+ public:
+ CloseIterIRGenerator(JSContext* cx, HandleScript, jsbytecode* pc,
+ ICState state, HandleObject iter, CompletionKind kind);
+
+ AttachDecision tryAttachStub();
+ AttachDecision tryAttachNoReturnMethod();
+ AttachDecision tryAttachScriptedReturn();
+};
+
+// Retrieve Xray JIT info set by the embedder.
+extern JS::XrayJitInfo* GetXrayJitInfo();
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_CacheIRGenerator_h */
diff --git a/js/src/jit/CacheIRHealth.cpp b/js/src/jit/CacheIRHealth.cpp
new file mode 100644
index 0000000000..dc7119106d
--- /dev/null
+++ b/js/src/jit/CacheIRHealth.cpp
@@ -0,0 +1,416 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#ifdef JS_CACHEIR_SPEW
+
+# include "jit/CacheIRHealth.h"
+
+# include "mozilla/Maybe.h"
+
+# include "gc/Zone.h"
+# include "jit/BaselineIC.h"
+# include "jit/CacheIRCompiler.h"
+# include "jit/JitScript.h"
+# include "vm/JSScript.h"
+
+# include "vm/JSObject-inl.h"
+# include "vm/Realm-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// TODO: Refine how we assign happiness based on total health score.
+CacheIRHealth::Happiness CacheIRHealth::determineStubHappiness(
+ uint32_t stubHealthScore) {
+ if (stubHealthScore >= 30) {
+ return Sad;
+ }
+
+ if (stubHealthScore >= 20) {
+ return MediumSad;
+ }
+
+ if (stubHealthScore >= 10) {
+ return MediumHappy;
+ }
+
+ return Happy;
+}
+
+CacheIRHealth::Happiness CacheIRHealth::spewStubHealth(
+ AutoStructuredSpewer& spew, ICCacheIRStub* stub) {
+ const CacheIRStubInfo* stubInfo = stub->stubInfo();
+ CacheIRReader stubReader(stubInfo);
+ uint32_t totalStubHealth = 0;
+ spew->beginListProperty("cacheIROps");
+ while (stubReader.more()) {
+ CacheOp op = stubReader.readOp();
+ uint32_t opHealth = CacheIROpHealth[size_t(op)];
+ uint32_t argLength = CacheIROpInfos[size_t(op)].argLength;
+ const char* opName = CacheIROpNames[size_t(op)];
+
+ spew->beginObject();
+ if (opHealth == UINT32_MAX) {
+ spew->property("unscoredOp", opName);
+ } else {
+ spew->property("cacheIROp", opName);
+ spew->property("opHealth", opHealth);
+ totalStubHealth += opHealth;
+ }
+ spew->endObject();
+
+ stubReader.skip(argLength);
+ }
+ spew->endList(); // cacheIROps
+
+ spew->property("stubHealth", totalStubHealth);
+
+ Happiness stubHappiness = determineStubHappiness(totalStubHealth);
+ spew->property("stubHappiness", stubHappiness);
+
+ return stubHappiness;
+}
+
+BaseScript* CacheIRHealth::maybeExtractBaseScript(JSContext* cx, Shape* shape) {
+ TaggedProto taggedProto = shape->base()->proto();
+ if (!taggedProto.isObject()) {
+ return nullptr;
+ }
+ Value cval;
+ JSObject* proto = taggedProto.toObject();
+ AutoRealm ar(cx, proto);
+ if (!GetPropertyPure(cx, proto, NameToId(cx->names().constructor), &cval)) {
+ return nullptr;
+ }
+ if (!IsFunctionObject(cval)) {
+ return nullptr;
+ }
+ JSFunction& jsfun = cval.toObject().as<JSFunction>();
+ if (!jsfun.hasBaseScript()) {
+ return nullptr;
+ }
+ return jsfun.baseScript();
+}
+
+void CacheIRHealth::spewShapeInformation(AutoStructuredSpewer& spew,
+ JSContext* cx, ICStub* stub) {
+ bool shapesStarted = false;
+ const CacheIRStubInfo* stubInfo = stub->toCacheIRStub()->stubInfo();
+ size_t offset = 0;
+ uint32_t fieldIndex = 0;
+
+ while (stubInfo->fieldType(fieldIndex) != StubField::Type::Limit) {
+ if (stubInfo->fieldType(fieldIndex) == StubField::Type::Shape) {
+ Shape* shape = reinterpret_cast<Shape*>(
+ stubInfo->getStubRawWord(stub->toCacheIRStub(), offset));
+ if (!shapesStarted) {
+ shapesStarted = true;
+ spew->beginListProperty("shapes");
+ }
+
+ const PropMap* propMap =
+ shape->isNative() ? shape->asNative().propMap() : nullptr;
+ if (propMap) {
+ spew->beginObject();
+ {
+ if (!propMap->isDictionary()) {
+ uint32_t mapLength = shape->asNative().propMapLength();
+ if (mapLength) {
+ PropertyKey lastKey = shape->asNative().lastProperty().key();
+ if (lastKey.isInt()) {
+ spew->property("lastProperty", lastKey.toInt());
+ } else if (lastKey.isString()) {
+ JSString* str = lastKey.toString();
+ if (str && str->isLinear()) {
+ spew->property("lastProperty", &str->asLinear());
+ }
+ } else {
+ MOZ_ASSERT(lastKey.isSymbol());
+ JSString* str = lastKey.toSymbol()->description();
+ if (str && str->isLinear()) {
+ spew->property("lastProperty", &str->asLinear());
+ }
+ }
+ }
+ spew->property("totalKeys", propMap->approximateEntryCount());
+ BaseScript* baseScript = maybeExtractBaseScript(cx, shape);
+ if (baseScript) {
+ spew->beginObjectProperty("shapeAllocSite");
+ {
+ spew->property("filename", baseScript->filename());
+ spew->property("line", baseScript->lineno());
+ spew->property("column", baseScript->column());
+ }
+ spew->endObject();
+ }
+ }
+ }
+ spew->endObject();
+ }
+ }
+ offset += StubField::sizeInBytes(stubInfo->fieldType(fieldIndex));
+ fieldIndex++;
+ }
+
+ if (shapesStarted) {
+ spew->endList();
+ }
+}
+
+bool CacheIRHealth::spewNonFallbackICInformation(AutoStructuredSpewer& spew,
+ JSContext* cx,
+ ICStub* firstStub,
+ Happiness* entryHappiness) {
+ const CacheIRStubInfo* stubInfo = firstStub->toCacheIRStub()->stubInfo();
+ Vector<bool, 8, SystemAllocPolicy> sawDistinctValueAtFieldIndex;
+
+ bool sawNonZeroCount = false;
+ bool sawDifferentCacheIRStubs = false;
+ ICStub* stub = firstStub;
+
+ spew->beginListProperty("stubs");
+ while (stub && !stub->isFallback()) {
+ spew->beginObject();
+ {
+ Happiness stubHappiness = spewStubHealth(spew, stub->toCacheIRStub());
+ if (stubHappiness < *entryHappiness) {
+ *entryHappiness = stubHappiness;
+ }
+
+ spewShapeInformation(spew, cx, stub);
+
+ ICStub* nextStub = stub->toCacheIRStub()->next();
+ if (!nextStub->isFallback()) {
+ if (nextStub->enteredCount() > 0) {
+ // More than one stub has a hit count greater than zero.
+ // This is sad because we do not Warp transpile in this case.
+ *entryHappiness = Sad;
+ sawNonZeroCount = true;
+ }
+
+ if (nextStub->toCacheIRStub()->stubInfo() != stubInfo) {
+ sawDifferentCacheIRStubs = true;
+ }
+
+ // If there are multiple stubs with non zero hit counts and if all
+ // of the stubs have equivalent CacheIR, then keep track of how many
+ // distinct stub field values are seen for each field index.
+ if (sawNonZeroCount && !sawDifferentCacheIRStubs) {
+ uint32_t fieldIndex = 0;
+ size_t offset = 0;
+
+ while (stubInfo->fieldType(fieldIndex) != StubField::Type::Limit) {
+ if (sawDistinctValueAtFieldIndex.length() <= fieldIndex) {
+ if (!sawDistinctValueAtFieldIndex.append(false)) {
+ return false;
+ }
+ }
+
+ if (StubField::sizeIsWord(stubInfo->fieldType(fieldIndex))) {
+ uintptr_t firstRaw =
+ stubInfo->getStubRawWord(firstStub->toCacheIRStub(), offset);
+ uintptr_t nextRaw =
+ stubInfo->getStubRawWord(nextStub->toCacheIRStub(), offset);
+ if (firstRaw != nextRaw) {
+ sawDistinctValueAtFieldIndex[fieldIndex] = true;
+ }
+ } else {
+ MOZ_ASSERT(
+ StubField::sizeIsInt64(stubInfo->fieldType(fieldIndex)));
+ int64_t firstRaw =
+ stubInfo->getStubRawInt64(firstStub->toCacheIRStub(), offset);
+ int64_t nextRaw =
+ stubInfo->getStubRawInt64(nextStub->toCacheIRStub(), offset);
+
+ if (firstRaw != nextRaw) {
+ sawDistinctValueAtFieldIndex[fieldIndex] = true;
+ }
+ }
+
+ offset += StubField::sizeInBytes(stubInfo->fieldType(fieldIndex));
+ fieldIndex++;
+ }
+ }
+ }
+
+ spew->property("hitCount", stub->enteredCount());
+ stub = nextStub;
+ }
+ spew->endObject();
+ }
+ spew->endList(); // stubs
+
+ // If more than one CacheIR stub has an entered count greater than
+ // zero and all the stubs have equivalent CacheIR, then spew
+ // the information collected about the stub fields across the IC.
+ if (sawNonZeroCount && !sawDifferentCacheIRStubs) {
+ spew->beginListProperty("stubFields");
+ for (size_t i = 0; i < sawDistinctValueAtFieldIndex.length(); i++) {
+ spew->beginObject();
+ {
+ spew->property("fieldType", uint8_t(stubInfo->fieldType(i)));
+ spew->property("sawDistinctFieldValues",
+ sawDistinctValueAtFieldIndex[i]);
+ }
+ spew->endObject();
+ }
+ spew->endList();
+ }
+
+ return true;
+}
+
+bool CacheIRHealth::spewICEntryHealth(AutoStructuredSpewer& spew, JSContext* cx,
+ HandleScript script, ICEntry* entry,
+ ICFallbackStub* fallback, jsbytecode* pc,
+ JSOp op, Happiness* entryHappiness) {
+ spew->property("op", CodeName(op));
+
+ // TODO: If a perf issue arises, look into improving the SrcNotes
+ // API call below.
+ unsigned column;
+ spew->property("lineno", PCToLineNumber(script, pc, &column));
+ spew->property("column", column);
+
+ ICStub* firstStub = entry->firstStub();
+ if (!firstStub->isFallback()) {
+ if (!spewNonFallbackICInformation(spew, cx, firstStub, entryHappiness)) {
+ return false;
+ }
+ }
+
+ if (fallback->state().mode() != ICState::Mode::Specialized) {
+ *entryHappiness = Sad;
+ }
+
+ spew->property("entryHappiness", uint8_t(*entryHappiness));
+
+ spew->property("mode", uint8_t(fallback->state().mode()));
+
+ spew->property("fallbackCount", fallback->enteredCount());
+
+ return true;
+}
+
+void CacheIRHealth::spewScriptFinalWarmUpCount(JSContext* cx,
+ const char* filename,
+ JSScript* script,
+ uint32_t warmUpCount) {
+ AutoStructuredSpewer spew(cx, SpewChannel::CacheIRHealthReport, nullptr);
+ if (!spew) {
+ return;
+ }
+
+ spew->property("filename", filename);
+ spew->property("line", script->lineno());
+ spew->property("column", script->column());
+ spew->property("finalWarmUpCount", warmUpCount);
+}
+
+static bool addScriptToFinalWarmUpCountMap(JSContext* cx, HandleScript script) {
+ // Create Zone::scriptFilenameMap if necessary.
+ JS::Zone* zone = script->zone();
+ if (!zone->scriptFinalWarmUpCountMap) {
+ auto map = MakeUnique<ScriptFinalWarmUpCountMap>();
+ if (!map) {
+ return false;
+ }
+
+ zone->scriptFinalWarmUpCountMap = std::move(map);
+ }
+
+ SharedImmutableString sfilename =
+ SharedImmutableStringsCache::getSingleton().getOrCreate(
+ script->filename(), strlen(script->filename()));
+ if (!sfilename) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ if (!zone->scriptFinalWarmUpCountMap->put(
+ script, mozilla::MakeTuple(uint32_t(0), std::move(sfilename)))) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ script->setNeedsFinalWarmUpCount();
+ return true;
+}
+
+void CacheIRHealth::healthReportForIC(JSContext* cx, ICEntry* entry,
+ ICFallbackStub* fallback,
+ HandleScript script,
+ SpewContext context) {
+ AutoStructuredSpewer spew(cx, SpewChannel::CacheIRHealthReport, script);
+ if (!spew) {
+ return;
+ }
+
+ if (!addScriptToFinalWarmUpCountMap(cx, script)) {
+ cx->recoverFromOutOfMemory();
+ return;
+ }
+ spew->property("spewContext", uint8_t(context));
+
+ jsbytecode* op = script->offsetToPC(fallback->pcOffset());
+ JSOp jsOp = JSOp(*op);
+
+ Happiness entryHappiness = Happy;
+ if (!spewICEntryHealth(spew, cx, script, entry, fallback, op, jsOp,
+ &entryHappiness)) {
+ cx->recoverFromOutOfMemory();
+ return;
+ }
+ MOZ_ASSERT(entryHappiness == Sad);
+}
+
+void CacheIRHealth::healthReportForScript(JSContext* cx, HandleScript script,
+ SpewContext context) {
+ jit::JitScript* jitScript = script->maybeJitScript();
+ if (!jitScript) {
+ return;
+ }
+
+ AutoStructuredSpewer spew(cx, SpewChannel::CacheIRHealthReport, script);
+ if (!spew) {
+ return;
+ }
+
+ if (!addScriptToFinalWarmUpCountMap(cx, script)) {
+ cx->recoverFromOutOfMemory();
+ return;
+ }
+
+ spew->property("spewContext", uint8_t(context));
+
+ spew->beginListProperty("entries");
+
+ Happiness scriptHappiness = Happy;
+
+ for (size_t i = 0; i < jitScript->numICEntries(); i++) {
+ ICEntry& entry = jitScript->icEntry(i);
+ ICFallbackStub* fallback = jitScript->fallbackStub(i);
+ jsbytecode* pc = script->offsetToPC(fallback->pcOffset());
+ JSOp op = JSOp(*pc);
+
+ spew->beginObject();
+ Happiness entryHappiness = Happy;
+ if (!spewICEntryHealth(spew, cx, script, &entry, fallback, pc, op,
+ &entryHappiness)) {
+ cx->recoverFromOutOfMemory();
+ return;
+ }
+ if (entryHappiness < scriptHappiness) {
+ scriptHappiness = entryHappiness;
+ }
+ spew->endObject();
+ }
+
+ spew->endList(); // entries
+
+ spew->property("scriptHappiness", uint8_t(scriptHappiness));
+}
+
+#endif /* JS_CACHEIR_SPEW */
diff --git a/js/src/jit/CacheIRHealth.h b/js/src/jit/CacheIRHealth.h
new file mode 100644
index 0000000000..56528aa8d6
--- /dev/null
+++ b/js/src/jit/CacheIRHealth.h
@@ -0,0 +1,109 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CacheIRHealth_h
+#define jit_CacheIRHealth_h
+
+#ifdef JS_CACHEIR_SPEW
+
+# include "mozilla/Sprintf.h"
+
+# include "NamespaceImports.h"
+
+# include "jit/CacheIR.h"
+# include "js/TypeDecls.h"
+
+enum class JSOp : uint8_t;
+
+namespace js {
+
+class AutoStructuredSpewer;
+
+namespace jit {
+
+class ICEntry;
+class ICStub;
+class ICCacheIRStub;
+class ICFallbackStub;
+
+// [SMDOC] CacheIR Health Report
+//
+// The goal of CacheIR health report is to make the costlier
+// CacheIR stubs more apparent and easier to diagnose.
+// This is done by using the scores assigned to different CacheIROps in
+// CacheIROps.yaml (see the description of cost_estimate in the
+// aforementioned file for how these scores are determined), summing
+// the score of each op generated for a particular stub together, and displaying
+// this score for each stub in an inline cache. The higher the total stub score,
+// the more expensive the stub is.
+//
+// There are a few ways to generate a health report for a script:
+// 1. Simply running a JS program with the evironment variable
+// SPEW=CacheIRHealthReport. We generate a health report for a script
+// whenever we reach the trial inlining threshold.
+// ex) SPEW=CacheIRHealthReport dist/bin/js jsprogram.js
+// 2. In the shell you can call cacheIRHealthReport() with no arguments and a
+// report
+// will be generated for all scripts in the current zone.
+// ex) cacheIRHealthReport()
+// 3. You may also call cacheIRHealthReport() on a particular function to see
+// the
+// health report associated with that function's script.
+// ex) cacheIRHealthReport(foo)
+//
+// Once you have generated a health report, you may go to
+// https://mozilla-spidermonkey.github.io/cacheirhealthreport/ to visualize the
+// data and aid in understanding what may be going wrong with the CacheIR for a
+// particular stub. For more information about the tool and why a particular
+// script, inline cache entry, or stub is unhappy go to:
+// https://mozilla-spidermonkey.github.io/cacheirhealthreport/info.html
+//
+enum SpewContext : uint8_t { Shell, Transition, TrialInlining };
+
+class CacheIRHealth {
+ enum Happiness : uint8_t { Sad, MediumSad, MediumHappy, Happy };
+
+ // Get happiness from health score.
+ Happiness determineStubHappiness(uint32_t stubHealthScore);
+ // Health of an individual stub.
+ Happiness spewStubHealth(AutoStructuredSpewer& spew, ICCacheIRStub* stub);
+ // If there is more than just a fallback stub in an IC Entry, then additional
+ // information about the IC entry.
+ bool spewNonFallbackICInformation(AutoStructuredSpewer& spew, JSContext* cx,
+ ICStub* firstStub,
+ Happiness* entryHappiness);
+ // Health of all the stubs in an individual CacheIR Entry.
+ bool spewICEntryHealth(AutoStructuredSpewer& spew, JSContext* cx,
+ HandleScript script, ICEntry* entry,
+ ICFallbackStub* fallback, jsbytecode* pc, JSOp op,
+ Happiness* entryHappiness);
+ // Spews first and last property name for each shape checked by
+ // GuardShape in the stub.
+ void spewShapeInformation(AutoStructuredSpewer& spew, JSContext* cx,
+ ICStub* stub);
+ // Returns the BaseScript of a Shape if available.
+ BaseScript* maybeExtractBaseScript(JSContext* cx, Shape* shape);
+
+ public:
+ // Spews the final hit count for scripts where we care about its final hit
+ // count.
+ void spewScriptFinalWarmUpCount(JSContext* cx, const char* filename,
+ JSScript* script, uint32_t warmUpCount);
+ // Spew the health of a particular ICEntry only.
+ void healthReportForIC(JSContext* cx, ICEntry* entry,
+ ICFallbackStub* fallback, HandleScript script,
+ SpewContext context);
+ // If a JitScript exists, spew the health of all ICEntries that exist
+ // for the specified script.
+ void healthReportForScript(JSContext* cx, HandleScript script,
+ SpewContext context);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* JS_CACHEIR_SPEW */
+#endif /* jit_CacheIRHealth_h */
diff --git a/js/src/jit/CacheIROps.yaml b/js/src/jit/CacheIROps.yaml
new file mode 100644
index 0000000000..3a5d477cf0
--- /dev/null
+++ b/js/src/jit/CacheIROps.yaml
@@ -0,0 +1,3016 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# [SMDOC] CacheIR Opcodes
+# =======================
+# This file defines all CacheIR opcodes and their arguments.
+#
+# Each op has the following attributes:
+#
+# name
+# ====
+# Opcode name. Convention is to use a name ending in *Result for ops that store
+# to the IC's output register.
+#
+# shared
+# ======
+# If true, Baseline and Ion use the same CacheIRCompiler code for this op.
+# If false, the op must be implemented in both BaselineCacheIRCompiler and
+# IonCacheIRCompiler.
+#
+# transpile
+# =========
+# Whether this op can be transpiled to MIR by WarpCacheIRTranspiler.
+#
+# cost_estimate
+# =========
+# Score of an individual CacheIR Opcode's contribution to the overall score for
+# each stub. This score is based off of the cost of the masm calls made by the op's
+# implementation. The higher the score the more costly the op is.
+#
+# How to decide the cost estimate for a CacheIROp:
+# 0 points - Generates no code
+# 1 point - 1-5 simple masm ops, no callVM or callWithABI
+# 2 points - 5-20 masm ops, no callVM or callWithABI
+# 3 points - 20+ masm ops, no callVM or callWithABI
+# 4 points - callWithABI
+# 5 points - callVM
+# 6 points - more than one callWithABI or callVM
+#
+# In the case of the op not being shared, default to counting the Baseline
+# implementation.
+#
+# If the cost estimate is different based off of what branch of a conditional
+# is taken, assign the score of the branch with the highest cost.
+#
+# Note:
+# Currently, the scoring is tentative. It is in place to provide an
+# estimate for the cost of each op. The scoring will be refined.
+#
+# custom_writer (optional)
+# ========================
+# If true, the generated CacheIRWriter method will be private and has a trailing
+# '_'. This is useful for ops that need custom CacheIRWriter logic on top of the
+# generated code.
+#
+# args
+# ====
+# List of arguments encoded in the bytecode stream. There are three argument
+# kinds:
+#
+# - Id (ObjId, ValId, ...): refers to either an IC input or a value defined by
+# a previous CacheIR instruction. This is encoded as integer in the bytecode
+# stream.
+#
+# - Field (ObjectField, StringField, ...): specific value is stored in the stub
+# data and the bytecode stream stores the offset of this field. This means the
+# CacheIR is not specialized for particular values and code can be shared.
+#
+# - Immediate (BoolImm, Int32Imm, JSOpImm, ...): a value baked directly into
+# the bytecode stream. This is useful for bits of state that need to be
+# available to all CacheIR compilers/transpilers.
+#
+# If there's an argument named 'result', the generated CacheIRWriter method will
+# return a new OperandId of this type.
+
+- name: ReturnFromIC
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+
+- name: GuardToObject
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ input: ValId
+
+- name: GuardIsNullOrUndefined
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+
+- name: GuardIsNull
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+
+- name: GuardIsUndefined
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+
+- name: GuardIsNotUninitializedLexical
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: ValId
+
+- name: GuardToBoolean
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ input: ValId
+
+- name: GuardToString
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ input: ValId
+
+- name: GuardToSymbol
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ input: ValId
+
+- name: GuardToBigInt
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ input: ValId
+
+- name: GuardIsNumber
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ input: ValId
+
+- name: GuardToInt32
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ input: ValId
+
+- name: GuardToNonGCThing
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+
+# If the Value is a boolean, convert it to int32.
+- name: GuardBooleanToInt32
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+ result: Int32Id
+
+- name: GuardToInt32Index
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+ result: Int32Id
+
+- name: Int32ToIntPtr
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: Int32Id
+ result: IntPtrId
+
+- name: GuardNumberToIntPtrIndex
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: NumberId
+ supportOOB: BoolImm
+ result: IntPtrId
+
+- name: GuardToInt32ModUint32
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: ValId
+ result: Int32Id
+
+- name: GuardToUint8Clamped
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: ValId
+ result: Int32Id
+
+# Note: this doesn't accept doubles to avoid ambiguity about whether it includes
+# int32 values. Use GuardIsNumber instead.
+- name: GuardNonDoubleType
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+ type: ValueTypeImm
+
+- name: GuardShape
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ shape: ShapeField
+
+- name: GuardMultipleShapes
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ custom_writer: true
+ args:
+ obj: ObjId
+ shapes: ObjectField
+
+- name: GuardProto
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ proto: ObjectField
+
+- name: GuardNullProto
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+# Guard per GuardClassKind.
+- name: GuardClass
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ kind: GuardClassKindImm
+
+# Guard on an arbitrary JSClass.
+- name: GuardAnyClass
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ clasp: RawPointerField
+
+- name: HasClassResult
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ clasp: RawPointerField
+
+- name: CallRegExpMatcherResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ regexp: ObjId
+ input: StringId
+ lastIndex: Int32Id
+
+- name: CallRegExpSearcherResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ regexp: ObjId
+ input: StringId
+ lastIndex: Int32Id
+
+- name: CallRegExpTesterResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ regexp: ObjId
+ input: StringId
+ lastIndex: Int32Id
+
+- name: RegExpFlagResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ regexp: ObjId
+ flagsMask: Int32Imm
+
+- name: CallSubstringKernelResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+ begin: Int32Id
+ length: Int32Id
+
+- name: StringReplaceStringResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+ pattern: StringId
+ replacement: StringId
+
+- name: StringSplitStringResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+ separator: StringId
+
+- name: RegExpPrototypeOptimizableResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ proto: ObjId
+
+- name: RegExpInstanceOptimizableResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ regexp: ObjId
+ proto: ObjId
+
+- name: GetFirstDollarIndexResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+
+# Add a reference to a global in the compartment to keep it alive.
+- name: GuardCompartment
+ shared: false
+ transpile: false
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ global: ObjectField
+ compartment: RawPointerField
+
+- name: GuardIsExtensible
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: GuardIsNativeObject
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: GuardIsProxy
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: GuardIsNotProxy
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: GuardIsNotArrayBufferMaybeShared
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: GuardIsTypedArray
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: GuardHasProxyHandler
+ shared: false
+ transpile: false
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ handler: RawPointerField
+
+- name: GuardIsNotDOMProxy
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: GuardSpecificObject
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ expected: ObjectField
+
+- name: GuardSpecificFunction
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ fun: ObjId
+ expected: ObjectField
+ nargsAndFlags: RawInt32Field
+
+- name: GuardFunctionScript
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ obj: ObjId
+ expected: BaseScriptField
+ nargsAndFlags: RawInt32Field
+
+- name: GuardSpecificAtom
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ args:
+ str: StringId
+ expected: AtomField
+
+- name: GuardSpecificSymbol
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ sym: SymbolId
+ expected: SymbolField
+
+- name: GuardSpecificInt32
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ num: Int32Id
+ expected: Int32Imm
+
+- name: GuardNoDenseElements
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: GuardStringToIndex
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ str: StringId
+ result: Int32Id
+
+- name: GuardStringToInt32
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ str: StringId
+ result: Int32Id
+
+- name: GuardStringToNumber
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ str: StringId
+ result: NumberId
+
+- name: BooleanToNumber
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ boolean: BooleanId
+ result: NumberId
+
+- name: GuardAndGetIterator
+ shared: true
+ transpile: false
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ iter: ObjectField
+ enumeratorsAddr: RawPointerField
+ result: ObjId
+
+- name: GuardHasGetterSetter
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ id: IdField
+ getterSetter: GetterSetterField
+
+- name: GuardInt32IsNonNegative
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ index: Int32Id
+
+- name: GuardIndexIsValidUpdateOrAdd
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: GuardIndexIsNotDenseElement
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: GuardTagNotEqual
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: ValueTagId
+ rhs: ValueTagId
+
+- name: GuardXrayExpandoShapeAndDefaultProto
+ shared: true
+ transpile: false
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ shapeWrapper: ObjectField
+
+- name: GuardXrayNoExpando
+ shared: true
+ transpile: false
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+# Guard obj[slot] == expected.
+- name: GuardDynamicSlotIsSpecificObject
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ expected: ObjId
+ slot: RawInt32Field
+
+# Guard obj[slot] is not an object.
+- name: GuardDynamicSlotIsNotObject
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ slot: RawInt32Field
+
+- name: GuardFixedSlotValue
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ val: ValueField
+
+- name: GuardDynamicSlotValue
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ val: ValueField
+
+- name: LoadFixedSlot
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ result: ValId
+ obj: ObjId
+ offset: RawInt32Field
+
+- name: LoadDynamicSlot
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ result: ValId
+ obj: ObjId
+ slot: RawInt32Field
+
+- name: GuardNoAllocationMetadataBuilder
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ builderAddr: RawPointerField
+
+- name: GuardFunctionHasJitEntry
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ fun: ObjId
+ constructing: BoolImm
+
+- name: GuardFunctionHasNoJitEntry
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ fun: ObjId
+
+- name: GuardFunctionIsNonBuiltinCtor
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ fun: ObjId
+
+- name: GuardFunctionIsConstructor
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ fun: ObjId
+
+- name: GuardNotClassConstructor
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ fun: ObjId
+
+- name: GuardArrayIsPacked
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ array: ObjId
+
+- name: GuardArgumentsObjectFlags
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ flags: ByteImm
+
+- name: LoadObject
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ result: ObjId
+ obj: ObjectField
+
+# This is just LoadObject with extra information for the purpose of optimizing
+# out shape guards if we're just storing to slots of the receiver object.
+- name: LoadProtoObject
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ result: ObjId
+ protoObj: ObjectField
+ receiverObj: ObjId
+
+- name: LoadProto
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ result: ObjId
+
+- name: LoadEnclosingEnvironment
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ result: ObjId
+
+- name: LoadWrapperTarget
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ result: ObjId
+
+- name: LoadValueTag
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: ValId
+ result: ValueTagId
+
+- name: LoadArgumentFixedSlot
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ result: ValId
+ slotIndex: ByteImm
+
+- name: LoadArgumentDynamicSlot
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ custom_writer: true
+ args:
+ result: ValId
+ argc: Int32Id
+ slotIndex: ByteImm
+
+- name: TruncateDoubleToUInt32
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: NumberId
+ result: Int32Id
+
+- name: MegamorphicLoadSlotResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ name: PropertyNameField
+
+- name: MegamorphicLoadSlotByValueResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ id: ValId
+
+- name: MegamorphicStoreSlot
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ name: PropertyNameField
+ rhs: ValId
+
+- name: MegamorphicSetElement
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ id: ValId
+ rhs: ValId
+ strict: BoolImm
+
+- name: MegamorphicHasPropResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ id: ValId
+ hasOwn: BoolImm
+
+- name: ObjectToIteratorResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ enumeratorsAddr: RawPointerField
+
+- name: ValueToIteratorResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ val: ValId
+
+# See CacheIR.cpp 'DOM proxies' comment.
+- name: LoadDOMExpandoValue
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ result: ValId
+
+- name: LoadDOMExpandoValueGuardGeneration
+ shared: false
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ expandoAndGeneration: RawPointerField
+ generation: RawInt64Field
+ result: ValId
+
+- name: LoadDOMExpandoValueIgnoreGeneration
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ result: ValId
+
+- name: GuardDOMExpandoMissingOrGuardShape
+ shared: false
+ transpile: true
+ cost_estimate: 2
+ args:
+ expando: ValId
+ shape: ShapeField
+
+- name: StoreFixedSlot
+ shared: false
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ rhs: ValId
+
+- name: StoreDynamicSlot
+ shared: false
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ rhs: ValId
+
+- name: AddAndStoreFixedSlot
+ shared: false
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ rhs: ValId
+ newShape: ShapeField
+
+- name: AddAndStoreDynamicSlot
+ shared: false
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ rhs: ValId
+ newShape: ShapeField
+
+- name: AllocateAndStoreDynamicSlot
+ shared: false
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ rhs: ValId
+ newShape: ShapeField
+ numNewSlots: RawInt32Field
+
+- name: AddSlotAndCallAddPropHook
+ shared: true
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ rhs: ValId
+ newShape: ShapeField
+
+- name: StoreDenseElement
+ shared: true
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ index: Int32Id
+ rhs: ValId
+
+- name: StoreDenseElementHole
+ shared: true
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ index: Int32Id
+ rhs: ValId
+ handleAdd: BoolImm
+
+- name: ArrayPush
+ shared: true
+ transpile: true
+ cost_estimate: 6
+ args:
+ obj: ObjId
+ rhs: ValId
+
+- name: ArrayJoinResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ sep: StringId
+
+- name: PackedArrayPopResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ array: ObjId
+
+- name: PackedArrayShiftResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ array: ObjId
+
+- name: PackedArraySliceResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+ array: ObjId
+ begin: Int32Id
+ end: Int32Id
+
+- name: ArgumentsSliceResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+ args: ObjId
+ begin: Int32Id
+ end: Int32Id
+
+- name: IsArrayResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ input: ValId
+
+- name: StoreFixedSlotUndefinedResult
+ shared: true
+ transpile: true
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ rhs: ValId
+
+- name: IsObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+
+- name: IsPackedArrayResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: IsCallableResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: ValId
+
+- name: IsConstructorResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+
+- name: IsCrossRealmArrayConstructorResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: IsTypedArrayResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ isPossiblyWrapped: BoolImm
+
+- name: IsTypedArrayConstructorResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: ArrayBufferViewByteOffsetInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: ArrayBufferViewByteOffsetDoubleResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: TypedArrayByteLengthInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: TypedArrayByteLengthDoubleResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: TypedArrayElementSizeResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: GuardHasAttachedArrayBuffer
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: FinishBoundFunctionInitResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ bound: ObjId
+ target: ObjId
+ argCount: Int32Id
+
+- name: NewArrayIteratorResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+
+- name: NewStringIteratorResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+
+- name: NewRegExpStringIteratorResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+
+- name: ObjectCreateResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+
+- name: NewArrayFromLengthResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+ length: Int32Id
+
+- name: NewTypedArrayFromLengthResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+ length: Int32Id
+
+- name: NewTypedArrayFromArrayBufferResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+ buffer: ObjId
+ byteOffset: ValId
+ length: ValId
+
+- name: NewTypedArrayFromArrayResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+ array: ObjId
+
+- name: NewStringObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ templateObject: ObjectField
+ str: StringId
+
+- name: StringFromCharCodeResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ code: Int32Id
+
+- name: StringFromCodePointResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ code: Int32Id
+
+- name: StringIndexOfResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+ searchStr: StringId
+
+- name: StringStartsWithResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+ searchStr: StringId
+
+- name: StringEndsWithResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+ searchStr: StringId
+
+- name: StringToLowerCaseResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+
+- name: StringToUpperCaseResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+
+- name: MathAbsInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: Int32Id
+
+- name: MathAbsNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: NumberId
+
+- name: MathClz32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: Int32Id
+
+- name: MathSignInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: Int32Id
+
+- name: MathSignNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: NumberId
+
+- name: MathSignNumberToInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: NumberId
+
+- name: MathImulResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: MathSqrtNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: NumberId
+
+- name: MathFRoundNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: NumberId
+
+# Because Baseline stub code is shared by all realms in the Zone, this
+# instruction loads a pointer to the RNG from a stub field.
+- name: MathRandomResult
+ shared: false
+ transpile: true
+ cost_estimate: 3
+ args:
+ rng: RawPointerField
+
+- name: MathHypot2NumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ first: NumberId
+ second: NumberId
+
+- name: MathHypot3NumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ first: NumberId
+ second: NumberId
+ third: NumberId
+
+- name: MathHypot4NumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ first: NumberId
+ second: NumberId
+ third: NumberId
+ fourth: NumberId
+
+- name: MathAtan2NumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ lhs: NumberId
+ rhs: NumberId
+
+- name: MathFloorNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: NumberId
+
+- name: MathCeilNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: NumberId
+
+- name: MathTruncNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: NumberId
+
+- name: MathFloorToInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ input: NumberId
+
+- name: MathCeilToInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: NumberId
+
+- name: MathTruncToInt32Result
+ shared: true
+ transpile: true
+ args:
+ input: NumberId
+
+- name: MathRoundToInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: NumberId
+
+- name: Int32MinMax
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ isMax: BoolImm
+ first: Int32Id
+ second: Int32Id
+ result: Int32Id
+
+- name: NumberMinMax
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ isMax: BoolImm
+ first: NumberId
+ second: NumberId
+ result: NumberId
+
+- name: Int32MinMaxArrayResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ array: ObjId
+ isMax: BoolImm
+
+- name: NumberMinMaxArrayResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ array: ObjId
+ isMax: BoolImm
+
+- name: MathFunctionNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: NumberId
+ fun: UnaryMathFunctionImm
+
+- name: NumberParseIntResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+ radix: Int32Id
+
+- name: DoubleParseIntResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ num: NumberId
+
+- name: ObjectToStringResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+
+- name: ReflectGetPrototypeOfResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+
+- name: StoreTypedArrayElement
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ obj: ObjId
+ elementType: ScalarTypeImm
+ index: IntPtrId
+ rhs: RawId
+ handleOOB: BoolImm
+
+- name: AtomicsCompareExchangeResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: IntPtrId
+ expected: RawId
+ replacement: RawId
+ elementType: ScalarTypeImm
+
+- name: AtomicsExchangeResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: IntPtrId
+ value: RawId
+ elementType: ScalarTypeImm
+
+- name: AtomicsAddResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: IntPtrId
+ value: RawId
+ elementType: ScalarTypeImm
+ forEffect: BoolImm
+
+- name: AtomicsSubResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: IntPtrId
+ value: RawId
+ elementType: ScalarTypeImm
+ forEffect: BoolImm
+
+- name: AtomicsAndResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: IntPtrId
+ value: RawId
+ elementType: ScalarTypeImm
+ forEffect: BoolImm
+
+- name: AtomicsOrResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: IntPtrId
+ value: RawId
+ elementType: ScalarTypeImm
+ forEffect: BoolImm
+
+- name: AtomicsXorResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: IntPtrId
+ value: RawId
+ elementType: ScalarTypeImm
+ forEffect: BoolImm
+
+- name: AtomicsLoadResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: IntPtrId
+ elementType: ScalarTypeImm
+
+- name: AtomicsStoreResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: IntPtrId
+ value: RawId
+ elementType: ScalarTypeImm
+
+- name: AtomicsIsLockFreeResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ value: Int32Id
+
+- name: CallNativeSetter
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ custom_writer: true
+ args:
+ receiver: ObjId
+ setter: ObjectField
+ rhs: ValId
+ sameRealm: BoolImm
+ nargsAndFlags: RawInt32Field
+
+- name: CallScriptedSetter
+ shared: false
+ transpile: true
+ cost_estimate: 3
+ custom_writer: true
+ args:
+ receiver: ObjId
+ setter: ObjectField
+ rhs: ValId
+ sameRealm: BoolImm
+ nargsAndFlags: RawInt32Field
+
+- name: CallInlinedSetter
+ shared: false
+ transpile: true
+ cost_estimate: 3
+ custom_writer: true
+ args:
+ receiver: ObjId
+ setter: ObjectField
+ rhs: ValId
+ icScript: RawPointerField
+ sameRealm: BoolImm
+ nargsAndFlags: RawInt32Field
+
+- name: CallDOMSetter
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ jitInfo: RawPointerField
+ rhs: ValId
+
+- name: CallSetArrayLength
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ strict: BoolImm
+ rhs: ValId
+
+- name: ProxySet
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ id: IdField
+ rhs: ValId
+ strict: BoolImm
+
+- name: ProxySetByValue
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ id: ValId
+ rhs: ValId
+ strict: BoolImm
+
+- name: CallAddOrUpdateSparseElementHelper
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ id: Int32Id
+ rhs: ValId
+ strict: BoolImm
+
+- name: CallInt32ToString
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: Int32Id
+ result: StringId
+
+- name: CallNumberToString
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: NumberId
+ result: StringId
+
+- name: Int32ToStringWithBaseResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ input: Int32Id
+ base: Int32Id
+
+- name: BooleanToString
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: BooleanId
+ result: StringId
+
+- name: CallScriptedFunction
+ shared: false
+ transpile: true
+ cost_estimate: 3
+ custom_writer: true
+ args:
+ callee: ObjId
+ argc: Int32Id
+ flags: CallFlagsImm
+ argcFixed: UInt32Imm
+
+- name: CallWasmFunction
+ shared: false
+ transpile: true
+ cost_estimate: 3
+ args:
+ callee: ObjId
+ argc: Int32Id
+ flags: CallFlagsImm
+ argcFixed: UInt32Imm
+ funcExport: RawPointerField
+ instance: ObjectField
+
+- name: GuardWasmArg
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ arg: ValId
+ type: WasmValTypeImm
+
+- name: CallNativeFunction
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ custom_writer: true
+ args:
+ callee: ObjId
+ argc: Int32Id
+ flags: CallFlagsImm
+ argcFixed: UInt32Imm
+#ifdef JS_SIMULATOR
+ target: RawPointerField
+#else
+ ignoresReturnValue: BoolImm
+#endif
+
+- name: CallDOMFunction
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ custom_writer: true
+ args:
+ callee: ObjId
+ argc: Int32Id
+ thisObj: ObjId
+ flags: CallFlagsImm
+ argcFixed: UInt32Imm
+#ifdef JS_SIMULATOR
+ target: RawPointerField
+#endif
+
+- name: CallClassHook
+ shared: false
+ transpile: false
+ cost_estimate: 4
+ custom_writer: true
+ args:
+ callee: ObjId
+ argc: Int32Id
+ flags: CallFlagsImm
+ argcFixed: UInt32Imm
+ target: RawPointerField
+
+- name: CallInlinedFunction
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ custom_writer: true
+ args:
+ callee: ObjId
+ argc: Int32Id
+ icScript: RawPointerField
+ flags: CallFlagsImm
+ argcFixed: UInt32Imm
+
+
+# Meta ops generate no code, but contain data for the Warp Transpiler.
+- name: MetaScriptedThisShape
+ shared: true
+ transpile: true
+ cost_estimate: 0
+ custom_writer: true
+ args:
+ thisShape: ShapeField
+
+- name: LoadFixedSlotResult
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+
+- name: LoadFixedSlotTypedResult
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+ type: ValueTypeImm
+
+- name: LoadDynamicSlotResult
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ offset: RawInt32Field
+
+- name: LoadDenseElementResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: LoadDenseElementHoleResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: CallGetSparseElementResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: LoadDenseElementExistsResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: LoadTypedArrayElementExistsResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: IntPtrId
+
+- name: LoadDenseElementHoleExistsResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: LoadTypedArrayElementResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: IntPtrId
+ elementType: ScalarTypeImm
+ handleOOB: BoolImm
+ forceDoubleForUint32: BoolImm
+
+- name: LoadDataViewValueResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ offset: IntPtrId
+ littleEndian: BooleanId
+ elementType: ScalarTypeImm
+ forceDoubleForUint32: BoolImm
+
+- name: StoreDataViewValueResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ offset: IntPtrId
+ value: RawId
+ littleEndian: BooleanId
+ elementType: ScalarTypeImm
+
+- name: LoadInt32ArrayLengthResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: LoadInt32ArrayLength
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ result: Int32Id
+
+- name: LoadArgumentsObjectArgResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: LoadArgumentsObjectArgHoleResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: LoadArgumentsObjectArgExistsResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: LoadArgumentsObjectLengthResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: LoadArgumentsObjectLength
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+ result: Int32Id
+
+- name: LoadFunctionLengthResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: LoadFunctionNameResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ obj: ObjId
+
+- name: LoadArrayBufferByteLengthInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: LoadArrayBufferByteLengthDoubleResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: LoadArrayBufferViewLengthInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: LoadArrayBufferViewLengthDoubleResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: LinearizeForCharAccess
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ str: StringId
+ index: Int32Id
+ result: StringId
+
+- name: LoadStringCharResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ str: StringId
+ index: Int32Id
+ handleOOB: BoolImm
+
+- name: LoadStringCharCodeResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ str: StringId
+ index: Int32Id
+ handleOOB: BoolImm
+
+- name: LoadStringLengthResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ str: StringId
+
+- name: FrameIsConstructingResult
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+
+- name: LoadObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ obj: ObjId
+
+- name: LoadStringResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ str: StringId
+
+- name: LoadSymbolResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ sym: SymbolId
+
+- name: LoadInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: Int32Id
+
+- name: LoadDoubleResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ val: NumberId
+
+- name: LoadBigIntResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: BigIntId
+
+- name: CallScriptedGetterResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ custom_writer: true
+ args:
+ receiver: ValId
+ getter: ObjectField
+ sameRealm: BoolImm
+ nargsAndFlags: RawInt32Field
+
+- name: CallInlinedGetterResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ custom_writer: true
+ args:
+ receiver: ValId
+ getter: ObjectField
+ icScript: RawPointerField
+ sameRealm: BoolImm
+ nargsAndFlags: RawInt32Field
+
+- name: CallNativeGetterResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ custom_writer: true
+ args:
+ receiver: ValId
+ getter: ObjectField
+ sameRealm: BoolImm
+ nargsAndFlags: RawInt32Field
+
+- name: CallDOMGetterResult
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ jitInfo: RawPointerField
+
+- name: ProxyGetResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ id: IdField
+
+- name: ProxyGetByValueResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ id: ValId
+
+- name: ProxyHasPropResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ id: ValId
+ hasOwn: BoolImm
+
+- name: CallObjectHasSparseElementResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: CallNativeGetElementResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ index: Int32Id
+
+- name: CallNativeGetElementSuperResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ index: Int32Id
+ receiver: ValId
+
+- name: GetNextMapSetEntryForIteratorResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ iter: ObjId
+ resultArr: ObjId
+ isMap: BoolImm
+
+- name: LoadUndefinedResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+
+- name: LoadBooleanResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: BoolImm
+
+- name: LoadInt32Constant
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: RawInt32Field
+ result: Int32Id
+
+- name: LoadDoubleConstant
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: DoubleField
+ result: NumberId
+
+- name: LoadBooleanConstant
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: BoolImm
+ result: BooleanId
+
+- name: LoadUndefined
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ result: ValId
+
+- name: LoadConstantString
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ str: StringField
+ result: StringId
+
+- name: LoadConstantStringResult
+ shared: false
+ transpile: true
+ cost_estimate: 1
+ args:
+ str: StringField
+
+- name: LoadInstanceOfObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ lhs: ValId
+ proto: ObjId
+
+- name: LoadTypeOfObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+
+- name: DoubleAddResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ lhs: NumberId
+ rhs: NumberId
+
+- name: DoubleSubResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ lhs: NumberId
+ rhs: NumberId
+
+- name: DoubleMulResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ lhs: NumberId
+ rhs: NumberId
+
+- name: DoubleDivResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ lhs: NumberId
+ rhs: NumberId
+
+- name: DoubleModResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ lhs: NumberId
+ rhs: NumberId
+
+- name: DoublePowResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ lhs: NumberId
+ rhs: NumberId
+
+- name: Int32AddResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32SubResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32MulResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32DivResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32ModResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32PowResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: BigIntAddResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntSubResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntMulResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntDivResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntModResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntPowResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: Int32BitOrResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32BitXorResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32BitAndResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32LeftShiftResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32RightShiftResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: Int32URightShiftResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ lhs: Int32Id
+ rhs: Int32Id
+ forceDouble: BoolImm
+
+- name: Int32NotResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: Int32Id
+
+- name: BigIntBitOrResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntBitXorResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntBitAndResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntLeftShiftResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntRightShiftResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: BigIntNotResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ input: BigIntId
+
+- name: Int32NegationResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: Int32Id
+
+- name: DoubleNegationResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: NumberId
+
+- name: BigIntNegationResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ input: BigIntId
+
+- name: Int32IncResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: Int32Id
+
+- name: Int32DecResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: Int32Id
+
+- name: DoubleIncResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: NumberId
+
+- name: DoubleDecResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: NumberId
+
+- name: BigIntIncResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ input: BigIntId
+
+- name: BigIntDecResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ input: BigIntId
+
+- name: LoadInt32TruthyResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: ValId
+
+- name: LoadDoubleTruthyResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ input: NumberId
+
+- name: LoadStringTruthyResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ str: StringId
+
+- name: LoadObjectTruthyResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ obj: ObjId
+
+- name: LoadBigIntTruthyResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ bigInt: BigIntId
+
+- name: LoadValueTruthyResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ input: ValId
+
+- name: LoadValueResult
+ shared: false
+ transpile: false
+ cost_estimate: 1
+ args:
+ val: ValueField
+
+- name: LoadOperandResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ input: ValId
+
+- name: NewPlainObjectResult
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ args:
+ numFixedSlots: UInt32Imm
+ numDynamicSlots: UInt32Imm
+ allocKind: AllocKindImm
+ shape: ShapeField
+ site: AllocSiteField
+
+- name: NewArrayObjectResult
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ args:
+ arrayLength: UInt32Imm
+ shape: ShapeField
+ site: AllocSiteField
+
+- name: CallStringConcatResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ lhs: StringId
+ rhs: StringId
+
+- name: CallStringObjectConcatResult
+ shared: false
+ transpile: false
+ cost_estimate: 5
+ args:
+ lhs: ValId
+ rhs: ValId
+
+- name: CallIsSuspendedGeneratorResult
+ shared: true
+ transpile: false
+ cost_estimate: 2
+ args:
+ val: ValId
+
+- name: CompareStringResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ op: JSOpImm
+ lhs: StringId
+ rhs: StringId
+
+- name: CompareObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ op: JSOpImm
+ lhs: ObjId
+ rhs: ObjId
+
+- name: CompareSymbolResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ op: JSOpImm
+ lhs: SymbolId
+ rhs: SymbolId
+
+- name: CompareInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ op: JSOpImm
+ lhs: Int32Id
+ rhs: Int32Id
+
+- name: CompareDoubleResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ op: JSOpImm
+ lhs: NumberId
+ rhs: NumberId
+
+- name: CompareBigIntResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ op: JSOpImm
+ lhs: BigIntId
+ rhs: BigIntId
+
+- name: CompareBigIntInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ op: JSOpImm
+ lhs: BigIntId
+ rhs: Int32Id
+
+- name: CompareBigIntNumberResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ op: JSOpImm
+ lhs: BigIntId
+ rhs: NumberId
+
+- name: CompareBigIntStringResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ op: JSOpImm
+ lhs: BigIntId
+ rhs: StringId
+
+- name: CompareNullUndefinedResult
+ shared: true
+ transpile: true
+ cost_estimate: 2
+ args:
+ op: JSOpImm
+ isUndefined: BoolImm
+ input: ValId
+
+- name: CompareDoubleSameValueResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ lhs: NumberId
+ rhs: NumberId
+
+- name: SameValueResult
+ shared: false
+ transpile: true
+ cost_estimate: 4
+ args:
+ lhs: ValId
+ rhs: ValId
+
+- name: IndirectTruncateInt32Result
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: Int32Id
+
+- name: BigIntAsIntNResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ bits: Int32Id
+ bigInt: BigIntId
+
+- name: BigIntAsUintNResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ bits: Int32Id
+ bigInt: BigIntId
+
+- name: SetHasResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ set: ObjId
+ val: ValId
+
+- name: SetHasNonGCThingResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ set: ObjId
+ val: ValId
+
+- name: SetHasStringResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ set: ObjId
+ str: StringId
+
+- name: SetHasSymbolResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ set: ObjId
+ sym: SymbolId
+
+- name: SetHasBigIntResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ set: ObjId
+ bigInt: BigIntId
+
+- name: SetHasObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ set: ObjId
+ obj: ObjId
+
+- name: SetSizeResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ set: ObjId
+
+- name: MapHasResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ map: ObjId
+ val: ValId
+
+- name: MapHasNonGCThingResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ map: ObjId
+ val: ValId
+
+- name: MapHasStringResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ map: ObjId
+ str: StringId
+
+- name: MapHasSymbolResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ map: ObjId
+ sym: SymbolId
+
+- name: MapHasBigIntResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ map: ObjId
+ bigInt: BigIntId
+
+- name: MapHasObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ map: ObjId
+ obj: ObjId
+
+- name: MapGetResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ map: ObjId
+ val: ValId
+
+- name: MapGetNonGCThingResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ map: ObjId
+ val: ValId
+
+- name: MapGetStringResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ map: ObjId
+ str: StringId
+
+- name: MapGetSymbolResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ map: ObjId
+ sym: SymbolId
+
+- name: MapGetBigIntResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ map: ObjId
+ bigInt: BigIntId
+
+- name: MapGetObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 3
+ args:
+ map: ObjId
+ obj: ObjId
+
+- name: MapSizeResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ map: ObjId
+
+- name: ArrayFromArgumentsObjectResult
+ shared: true
+ transpile: true
+ cost_estimate: 5
+ args:
+ obj: ObjId
+ shape: ShapeField
+
+- name: CloseIterScriptedResult
+ shared: false
+ transpile: true
+ cost_estimate: 5
+ args:
+ iter: ObjId
+ callee: ObjId
+ kind: CompletionKindImm
+ targetNargs: UInt32Imm
+
+- name: CallPrintString
+ shared: true
+ transpile: false
+ cost_estimate: 1
+ args:
+ str: StaticStringImm
+
+- name: Breakpoint
+ shared: true
+ transpile: false
+ cost_estimate: 1
+ args:
+
+- name: WrapResult
+ shared: true
+ transpile: false
+ cost_estimate: 4
+ args:
+
+- name: Bailout
+ shared: true
+ transpile: true
+ cost_estimate: 0
+ args:
+
+- name: AssertRecoveredOnBailoutResult
+ shared: true
+ transpile: true
+ cost_estimate: 1
+ args:
+ val: ValId
+ mustBeRecovered: BoolImm
+
+#ifdef FUZZING_JS_FUZZILLI
+- name: FuzzilliHashResult
+ shared: true
+ transpile: true
+ cost_estimate: 4
+ args:
+ val: ValId
+#endif
diff --git a/js/src/jit/CacheIRReader.h b/js/src/jit/CacheIRReader.h
new file mode 100644
index 0000000000..8326c1d066
--- /dev/null
+++ b/js/src/jit/CacheIRReader.h
@@ -0,0 +1,155 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CacheIRReader_h
+#define jit_CacheIRReader_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+
+#include <stdint.h>
+#include "NamespaceImports.h"
+
+#include "jit/CacheIR.h"
+#include "jit/CacheIRWriter.h"
+#include "jit/CompactBuffer.h"
+#include "js/ScalarType.h"
+#include "js/Value.h"
+#include "wasm/WasmValType.h"
+
+enum class JSOp : uint8_t;
+
+namespace js {
+
+enum class UnaryMathFunction : uint8_t;
+
+namespace gc {
+enum class AllocKind : uint8_t;
+}
+
+namespace jit {
+
+class CacheIRStubInfo;
+
+// Helper class for reading CacheIR bytecode.
+class MOZ_RAII CacheIRReader {
+ CompactBufferReader buffer_;
+
+ CacheIRReader(const CacheIRReader&) = delete;
+ CacheIRReader& operator=(const CacheIRReader&) = delete;
+
+ public:
+ CacheIRReader(const uint8_t* start, const uint8_t* end)
+ : buffer_(start, end) {}
+ explicit CacheIRReader(const CacheIRWriter& writer)
+ : CacheIRReader(writer.codeStart(), writer.codeEnd()) {}
+ explicit CacheIRReader(const CacheIRStubInfo* stubInfo);
+
+ bool more() const { return buffer_.more(); }
+
+ CacheOp readOp() { return CacheOp(buffer_.readUnsigned15Bit()); }
+
+ // Skip data not currently used.
+ void skip() { buffer_.readByte(); }
+ void skip(uint32_t skipLength) {
+ if (skipLength > 0) {
+ buffer_.seek(buffer_.currentPosition(), skipLength);
+ }
+ }
+
+ ValOperandId valOperandId() { return ValOperandId(buffer_.readByte()); }
+ ValueTagOperandId valueTagOperandId() {
+ return ValueTagOperandId(buffer_.readByte());
+ }
+
+ IntPtrOperandId intPtrOperandId() {
+ return IntPtrOperandId(buffer_.readByte());
+ }
+
+ ObjOperandId objOperandId() { return ObjOperandId(buffer_.readByte()); }
+ NumberOperandId numberOperandId() {
+ return NumberOperandId(buffer_.readByte());
+ }
+ StringOperandId stringOperandId() {
+ return StringOperandId(buffer_.readByte());
+ }
+
+ SymbolOperandId symbolOperandId() {
+ return SymbolOperandId(buffer_.readByte());
+ }
+
+ BigIntOperandId bigIntOperandId() {
+ return BigIntOperandId(buffer_.readByte());
+ }
+
+ BooleanOperandId booleanOperandId() {
+ return BooleanOperandId(buffer_.readByte());
+ }
+
+ Int32OperandId int32OperandId() { return Int32OperandId(buffer_.readByte()); }
+
+ uint32_t rawOperandId() { return buffer_.readByte(); }
+
+ uint32_t stubOffset() { return buffer_.readByte() * sizeof(uintptr_t); }
+ GuardClassKind guardClassKind() { return GuardClassKind(buffer_.readByte()); }
+ ValueType valueType() { return ValueType(buffer_.readByte()); }
+ wasm::ValType::Kind wasmValType() {
+ return wasm::ValType::Kind(buffer_.readByte());
+ }
+ gc::AllocKind allocKind() { return gc::AllocKind(buffer_.readByte()); }
+ CompletionKind completionKind() { return CompletionKind(buffer_.readByte()); }
+
+ Scalar::Type scalarType() { return Scalar::Type(buffer_.readByte()); }
+ JSWhyMagic whyMagic() { return JSWhyMagic(buffer_.readByte()); }
+ JSOp jsop() { return JSOp(buffer_.readByte()); }
+ int32_t int32Immediate() { return int32_t(buffer_.readFixedUint32_t()); }
+ uint32_t uint32Immediate() { return buffer_.readFixedUint32_t(); }
+ void* pointer() { return buffer_.readRawPointer(); }
+
+ UnaryMathFunction unaryMathFunction() {
+ return UnaryMathFunction(buffer_.readByte());
+ }
+
+ CallFlags callFlags() {
+ // See CacheIRWriter::writeCallFlagsImm()
+ uint8_t encoded = buffer_.readByte();
+ CallFlags::ArgFormat format =
+ CallFlags::ArgFormat(encoded & CallFlags::ArgFormatMask);
+ bool isConstructing = encoded & CallFlags::IsConstructing;
+ bool isSameRealm = encoded & CallFlags::IsSameRealm;
+ bool needsUninitializedThis = encoded & CallFlags::NeedsUninitializedThis;
+ MOZ_ASSERT_IF(needsUninitializedThis, isConstructing);
+ switch (format) {
+ case CallFlags::Unknown:
+ MOZ_CRASH("Unexpected call flags");
+ case CallFlags::Standard:
+ return CallFlags(isConstructing, /*isSpread =*/false, isSameRealm,
+ needsUninitializedThis);
+ case CallFlags::Spread:
+ return CallFlags(isConstructing, /*isSpread =*/true, isSameRealm,
+ needsUninitializedThis);
+ default:
+ // The existing non-standard argument formats (FunCall and FunApply)
+ // can't be constructors.
+ MOZ_ASSERT(!isConstructing);
+ return CallFlags(format);
+ }
+ }
+
+ uint8_t readByte() { return buffer_.readByte(); }
+ bool readBool() {
+ uint8_t b = buffer_.readByte();
+ MOZ_ASSERT(b <= 1);
+ return bool(b);
+ }
+
+ const uint8_t* currentPosition() const { return buffer_.currentPosition(); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_CacheIRReader_h */
diff --git a/js/src/jit/CacheIRSpewer.cpp b/js/src/jit/CacheIRSpewer.cpp
new file mode 100644
index 0000000000..ef6dbac35b
--- /dev/null
+++ b/js/src/jit/CacheIRSpewer.cpp
@@ -0,0 +1,441 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifdef JS_CACHEIR_SPEW
+
+# include "jit/CacheIRSpewer.h"
+
+# include "mozilla/Sprintf.h"
+
+# include <algorithm>
+# include <stdarg.h>
+
+# include "jsapi.h"
+# include "jsmath.h"
+
+# include "js/ScalarType.h" // js::Scalar::Type
+# include "util/GetPidProvider.h"
+# include "util/Text.h"
+# include "vm/JSFunction.h"
+# include "vm/JSObject.h"
+# include "vm/JSScript.h"
+
+# include "vm/JSObject-inl.h"
+# include "vm/Realm-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// Text spewer for CacheIR ops that can be used with JitSpew.
+// Output looks like this:
+//
+// GuardToInt32 inputId 0, resultId 2
+// GuardToInt32 inputId 1, resultId 3
+// CompareInt32Result op JSOp::Lt, lhsId 2, rhsId 3
+// ReturnFromIC
+class MOZ_RAII CacheIROpsJitSpewer {
+ GenericPrinter& out_;
+
+ // String prepended to each line. Can be used for indentation.
+ const char* prefix_;
+
+ CACHE_IR_SPEWER_GENERATED
+
+ void spewOp(CacheOp op) {
+ const char* opName = CacheIROpNames[size_t(op)];
+ out_.printf("%s%-30s", prefix_, opName);
+ }
+ void spewOpEnd() { out_.printf("\n"); }
+
+ void spewArgSeparator() { out_.printf(", "); }
+
+ void spewOperandId(const char* name, OperandId id) {
+ spewRawOperandId(name, id.id());
+ }
+ void spewRawOperandId(const char* name, uint32_t id) {
+ out_.printf("%s %u", name, id);
+ }
+ void spewField(const char* name, uint32_t offset) {
+ out_.printf("%s %u", name, offset);
+ }
+ void spewBoolImm(const char* name, bool b) {
+ out_.printf("%s %s", name, b ? "true" : "false");
+ }
+ void spewByteImm(const char* name, uint8_t val) {
+ out_.printf("%s %u", name, val);
+ }
+ void spewJSOpImm(const char* name, JSOp op) {
+ out_.printf("%s JSOp::%s", name, CodeName(op));
+ }
+ void spewStaticStringImm(const char* name, const char* str) {
+ out_.printf("%s \"%s\"", name, str);
+ }
+ void spewInt32Imm(const char* name, int32_t val) {
+ out_.printf("%s %d", name, val);
+ }
+ void spewUInt32Imm(const char* name, uint32_t val) {
+ out_.printf("%s %u", name, val);
+ }
+ void spewCallFlagsImm(const char* name, CallFlags flags) {
+ out_.printf(
+ "%s (format %u%s%s%s)", name, flags.getArgFormat(),
+ flags.isConstructing() ? ", isConstructing" : "",
+ flags.isSameRealm() ? ", isSameRealm" : "",
+ flags.needsUninitializedThis() ? ", needsUninitializedThis" : "");
+ }
+ void spewJSWhyMagicImm(const char* name, JSWhyMagic magic) {
+ out_.printf("%s JSWhyMagic(%u)", name, unsigned(magic));
+ }
+ void spewScalarTypeImm(const char* name, Scalar::Type type) {
+ out_.printf("%s Scalar::Type(%u)", name, unsigned(type));
+ }
+ void spewUnaryMathFunctionImm(const char* name, UnaryMathFunction fun) {
+ const char* funName = GetUnaryMathFunctionName(fun);
+ out_.printf("%s UnaryMathFunction::%s", name, funName);
+ }
+ void spewValueTypeImm(const char* name, ValueType type) {
+ out_.printf("%s ValueType(%u)", name, unsigned(type));
+ }
+ void spewJSNativeImm(const char* name, JSNative native) {
+ out_.printf("%s %p", name, native);
+ }
+ void spewGuardClassKindImm(const char* name, GuardClassKind kind) {
+ out_.printf("%s GuardClassKind(%u)", name, unsigned(kind));
+ }
+ void spewWasmValTypeImm(const char* name, wasm::ValType::Kind kind) {
+ out_.printf("%s WasmValTypeKind(%u)", name, unsigned(kind));
+ }
+ void spewAllocKindImm(const char* name, gc::AllocKind kind) {
+ out_.printf("%s AllocKind(%u)", name, unsigned(kind));
+ }
+ void spewCompletionKindImm(const char* name, CompletionKind kind) {
+ out_.printf("%s CompletionKind(%u)", name, unsigned(kind));
+ }
+
+ public:
+ CacheIROpsJitSpewer(GenericPrinter& out, const char* prefix)
+ : out_(out), prefix_(prefix) {}
+
+ void spew(CacheIRReader& reader) {
+ do {
+ switch (reader.readOp()) {
+# define SPEW_OP(op, ...) \
+ case CacheOp::op: \
+ spew##op(reader); \
+ break;
+ CACHE_IR_OPS(SPEW_OP)
+# undef SPEW_OP
+
+ default:
+ MOZ_CRASH("Invalid op");
+ }
+ } while (reader.more());
+ }
+};
+
+void js::jit::SpewCacheIROps(GenericPrinter& out, const char* prefix,
+ const CacheIRStubInfo* info) {
+ CacheIRReader reader(info);
+ CacheIROpsJitSpewer spewer(out, prefix);
+ spewer.spew(reader);
+}
+
+// JSON spewer for CacheIR ops. Output looks like this:
+//
+// ...
+// {
+// "op":"GuardToInt32",
+// "args":[
+// {
+// "name":"inputId",
+// "type":"Id",
+// "value":0
+// },
+// {
+// "name":"resultId",
+// "type":"Id",
+// "value":1
+// }
+// ]
+// },
+// {
+// "op":"Int32IncResult",
+// "args":[
+// {
+// "name":"inputId",
+// "type":"Id",
+// "value":1
+// }
+// ]
+// }
+// ...
+class MOZ_RAII CacheIROpsJSONSpewer {
+ JSONPrinter& j_;
+
+ CACHE_IR_SPEWER_GENERATED
+
+ void spewOp(CacheOp op) {
+ const char* opName = CacheIROpNames[size_t(op)];
+ j_.beginObject();
+ j_.property("op", opName);
+ j_.beginListProperty("args");
+ }
+ void spewOpEnd() {
+ j_.endList();
+ j_.endObject();
+ }
+
+ void spewArgSeparator() {}
+
+ template <typename T>
+ void spewArgImpl(const char* name, const char* type, T value) {
+ j_.beginObject();
+ j_.property("name", name);
+ j_.property("type", type);
+ j_.property("value", value);
+ j_.endObject();
+ }
+
+ void spewOperandId(const char* name, OperandId id) {
+ spewRawOperandId(name, id.id());
+ }
+ void spewRawOperandId(const char* name, uint32_t id) {
+ spewArgImpl(name, "Id", id);
+ }
+ void spewField(const char* name, uint32_t offset) {
+ spewArgImpl(name, "Field", offset);
+ }
+ void spewBoolImm(const char* name, bool b) { spewArgImpl(name, "Imm", b); }
+ void spewByteImm(const char* name, uint8_t val) {
+ spewArgImpl(name, "Imm", val);
+ }
+ void spewJSOpImm(const char* name, JSOp op) {
+ spewArgImpl(name, "JSOp", CodeName(op));
+ }
+ void spewStaticStringImm(const char* name, const char* str) {
+ spewArgImpl(name, "String", str);
+ }
+ void spewInt32Imm(const char* name, int32_t val) {
+ spewArgImpl(name, "Imm", val);
+ }
+ void spewUInt32Imm(const char* name, uint32_t val) {
+ spewArgImpl(name, "Imm", val);
+ }
+ void spewCallFlagsImm(const char* name, CallFlags flags) {
+ spewArgImpl(name, "Imm", flags.toByte());
+ }
+ void spewJSWhyMagicImm(const char* name, JSWhyMagic magic) {
+ spewArgImpl(name, "Imm", unsigned(magic));
+ }
+ void spewScalarTypeImm(const char* name, Scalar::Type type) {
+ spewArgImpl(name, "Imm", unsigned(type));
+ }
+ void spewUnaryMathFunctionImm(const char* name, UnaryMathFunction fun) {
+ const char* funName = GetUnaryMathFunctionName(fun);
+ spewArgImpl(name, "MathFunction", funName);
+ }
+ void spewValueTypeImm(const char* name, ValueType type) {
+ spewArgImpl(name, "Imm", unsigned(type));
+ }
+ void spewJSNativeImm(const char* name, JSNative native) {
+ spewArgImpl(name, "Word", uintptr_t(native));
+ }
+ void spewGuardClassKindImm(const char* name, GuardClassKind kind) {
+ spewArgImpl(name, "Imm", unsigned(kind));
+ }
+ void spewWasmValTypeImm(const char* name, wasm::ValType::Kind kind) {
+ spewArgImpl(name, "Imm", unsigned(kind));
+ }
+ void spewAllocKindImm(const char* name, gc::AllocKind kind) {
+ spewArgImpl(name, "Imm", unsigned(kind));
+ }
+ void spewCompletionKindImm(const char* name, CompletionKind kind) {
+ spewArgImpl(name, "Imm", unsigned(kind));
+ }
+
+ public:
+ explicit CacheIROpsJSONSpewer(JSONPrinter& j) : j_(j) {}
+
+ void spew(CacheIRReader& reader) {
+ do {
+ switch (reader.readOp()) {
+# define SPEW_OP(op, ...) \
+ case CacheOp::op: \
+ spew##op(reader); \
+ break;
+ CACHE_IR_OPS(SPEW_OP)
+# undef SPEW_OP
+
+ default:
+ MOZ_CRASH("Invalid op");
+ }
+ } while (reader.more());
+ }
+};
+
+CacheIRSpewer CacheIRSpewer::cacheIRspewer;
+
+CacheIRSpewer::CacheIRSpewer()
+ : outputLock_(mutexid::CacheIRSpewer), guardCount_(0) {
+ spewInterval_ =
+ getenv("CACHEIR_LOG_FLUSH") ? atoi(getenv("CACHEIR_LOG_FLUSH")) : 10000;
+
+ if (spewInterval_ < 1) {
+ spewInterval_ = 1;
+ }
+}
+
+CacheIRSpewer::~CacheIRSpewer() {
+ if (!enabled()) {
+ return;
+ }
+
+ json_.ref().endList();
+ output_.flush();
+ output_.finish();
+}
+
+# ifndef JIT_SPEW_DIR
+# if defined(_WIN32)
+# define JIT_SPEW_DIR "."
+# elif defined(__ANDROID__)
+# define JIT_SPEW_DIR "/data/local/tmp"
+# else
+# define JIT_SPEW_DIR "/tmp"
+# endif
+# endif
+
+bool CacheIRSpewer::init(const char* filename) {
+ if (enabled()) {
+ return true;
+ }
+
+ char name[256];
+ uint32_t pid = getpid();
+ // Default to JIT_SPEW_DIR/cacheir${pid}.json
+ if (filename[0] == '1') {
+ SprintfLiteral(name, JIT_SPEW_DIR "/cacheir%" PRIu32 ".json", pid);
+ } else {
+ SprintfLiteral(name, "%s%" PRIu32 ".json", filename, pid);
+ }
+
+ if (!output_.init(name)) {
+ return false;
+ }
+
+ json_.emplace(output_);
+ json_->beginList();
+ return true;
+}
+
+void CacheIRSpewer::beginCache(const IRGenerator& gen) {
+ MOZ_ASSERT(enabled());
+ JSONPrinter& j = json_.ref();
+ const char* filename = gen.script_->filename();
+ j.beginObject();
+ j.property("name", CacheKindNames[uint8_t(gen.cacheKind_)]);
+ j.property("file", filename ? filename : "null");
+ j.property("mode", int(gen.mode_));
+ if (jsbytecode* pc = gen.pc_) {
+ unsigned column;
+ j.property("line", PCToLineNumber(gen.script_, pc, &column));
+ j.property("column", column);
+ j.formatProperty("pc", "%p", pc);
+ }
+}
+
+void CacheIRSpewer::valueProperty(const char* name, const Value& v) {
+ MOZ_ASSERT(enabled());
+ JSONPrinter& j = json_.ref();
+
+ j.beginObjectProperty(name);
+
+ const char* type = InformalValueTypeName(v);
+ if (v.isInt32()) {
+ type = "int32";
+ }
+ j.property("type", type);
+
+ if (v.isInt32()) {
+ j.property("value", v.toInt32());
+ } else if (v.isDouble()) {
+ j.floatProperty("value", v.toDouble(), 3);
+ } else if (v.isString() || v.isSymbol()) {
+ JSString* str = v.isString() ? v.toString() : v.toSymbol()->description();
+ if (str && str->isLinear()) {
+ j.property("value", &str->asLinear());
+ }
+ } else if (v.isObject()) {
+ JSObject& object = v.toObject();
+ j.formatProperty("value", "%p (shape: %p)", &object, object.shape());
+
+ if (object.is<JSFunction>()) {
+ if (JSAtom* name = object.as<JSFunction>().displayAtom()) {
+ j.property("funName", name);
+ }
+ }
+
+ if (NativeObject* nobj =
+ object.is<NativeObject>() ? &object.as<NativeObject>() : nullptr) {
+ j.beginListProperty("flags");
+ {
+ if (nobj->isIndexed()) {
+ j.value("indexed");
+ }
+ if (nobj->inDictionaryMode()) {
+ j.value("dictionaryMode");
+ }
+ }
+ j.endList();
+ if (nobj->isIndexed()) {
+ j.beginObjectProperty("indexed");
+ {
+ j.property("denseInitializedLength",
+ nobj->getDenseInitializedLength());
+ j.property("denseCapacity", nobj->getDenseCapacity());
+ j.property("denseElementsAreSealed", nobj->denseElementsAreSealed());
+ j.property("denseElementsAreFrozen", nobj->denseElementsAreFrozen());
+ }
+ j.endObject();
+ }
+ }
+ }
+
+ j.endObject();
+}
+
+void CacheIRSpewer::opcodeProperty(const char* name, const JSOp op) {
+ MOZ_ASSERT(enabled());
+ JSONPrinter& j = json_.ref();
+
+ j.beginStringProperty(name);
+ output_.put(CodeName(op));
+ j.endStringProperty();
+}
+
+void CacheIRSpewer::cacheIRSequence(CacheIRReader& reader) {
+ MOZ_ASSERT(enabled());
+ JSONPrinter& j = json_.ref();
+
+ j.beginListProperty("cacheIR");
+
+ CacheIROpsJSONSpewer spewer(j);
+ spewer.spew(reader);
+
+ j.endList();
+}
+
+void CacheIRSpewer::attached(const char* name) {
+ MOZ_ASSERT(enabled());
+ json_.ref().property("attached", name);
+}
+
+void CacheIRSpewer::endCache() {
+ MOZ_ASSERT(enabled());
+ json_.ref().endObject();
+}
+
+#endif /* JS_CACHEIR_SPEW */
diff --git a/js/src/jit/CacheIRSpewer.h b/js/src/jit/CacheIRSpewer.h
new file mode 100644
index 0000000000..fba33ba990
--- /dev/null
+++ b/js/src/jit/CacheIRSpewer.h
@@ -0,0 +1,116 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CacheIRSpewer_h
+#define jit_CacheIRSpewer_h
+
+#ifdef JS_CACHEIR_SPEW
+
+# include "mozilla/Maybe.h"
+
+# include "jit/CacheIR.h"
+# include "jit/CacheIRGenerator.h"
+# include "jit/CacheIRReader.h"
+# include "jit/CacheIRWriter.h"
+# include "js/TypeDecls.h"
+# include "threading/LockGuard.h"
+# include "vm/JSONPrinter.h"
+# include "vm/MutexIDs.h"
+
+namespace js {
+namespace jit {
+
+class CacheIRSpewer {
+ Mutex outputLock_ MOZ_UNANNOTATED;
+ Fprinter output_;
+ mozilla::Maybe<JSONPrinter> json_;
+ static CacheIRSpewer cacheIRspewer;
+
+ // Counter to record how many times Guard class is called. This is used to
+ // determine when to flush outputs based on the given interval value.
+ // For example, if |spewInterval_ = 2|, outputs will be flushed on
+ // guardCount_ values 0,2,4,6,...
+ uint32_t guardCount_;
+
+ // Interval at which to flush output files. This value can be set with the
+ // environment variable |CACHEIR_LOG_FLUSH|.
+ uint32_t spewInterval_;
+
+ CacheIRSpewer();
+ ~CacheIRSpewer();
+
+ bool enabled() { return json_.isSome(); }
+
+ // These methods can only be called when enabled() is true.
+ Mutex& lock() {
+ MOZ_ASSERT(enabled());
+ return outputLock_;
+ }
+
+ void beginCache(const IRGenerator& generator);
+ void valueProperty(const char* name, const Value& v);
+ void opcodeProperty(const char* name, const JSOp op);
+ void cacheIRSequence(CacheIRReader& reader);
+ void attached(const char* name);
+ void endCache();
+
+ public:
+ static CacheIRSpewer& singleton() { return cacheIRspewer; }
+ bool init(const char* name);
+
+ class MOZ_RAII Guard {
+ CacheIRSpewer& sp_;
+ const IRGenerator& gen_;
+ const char* name_;
+
+ public:
+ Guard(const IRGenerator& gen, const char* name)
+ : sp_(CacheIRSpewer::singleton()), gen_(gen), name_(name) {
+ if (sp_.enabled()) {
+ sp_.lock().lock();
+ sp_.beginCache(gen_);
+ }
+ }
+
+ ~Guard() {
+ if (sp_.enabled()) {
+ const CacheIRWriter& writer = gen_.writerRef();
+ if (!writer.failed() && writer.codeLength() > 0) {
+ CacheIRReader reader(writer);
+ sp_.cacheIRSequence(reader);
+ }
+ if (name_ != nullptr) {
+ sp_.attached(name_);
+ }
+ sp_.endCache();
+ if (sp_.guardCount_++ % sp_.spewInterval_ == 0) {
+ sp_.output_.flush();
+ }
+ sp_.lock().unlock();
+ }
+ }
+
+ void valueProperty(const char* name, const Value& v) const {
+ sp_.valueProperty(name, v);
+ }
+
+ void opcodeProperty(const char* name, const JSOp op) const {
+ sp_.opcodeProperty(name, op);
+ }
+
+ explicit operator bool() const { return sp_.enabled(); }
+ };
+};
+
+extern void SpewCacheIROps(GenericPrinter& out, const char* prefix,
+ const CacheIRStubInfo* info);
+
+} // namespace jit
+} // namespace js
+
+#endif /* JS_CACHEIR_SPEW */
+
+#endif /* jit_CacheIRSpewer_h */
diff --git a/js/src/jit/CacheIRWriter.h b/js/src/jit/CacheIRWriter.h
new file mode 100644
index 0000000000..c286dd30bb
--- /dev/null
+++ b/js/src/jit/CacheIRWriter.h
@@ -0,0 +1,638 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CacheIRWriter_h
+#define jit_CacheIRWriter_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Casting.h"
+#include "mozilla/Maybe.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jstypes.h"
+#include "NamespaceImports.h"
+
+#include "gc/AllocKind.h"
+#include "jit/CacheIR.h"
+#include "jit/CacheIROpsGenerated.h"
+#include "jit/CompactBuffer.h"
+#include "jit/ICState.h"
+#include "jit/Simulator.h"
+#include "jit/TypeData.h"
+#include "js/AllocPolicy.h"
+#include "js/CallArgs.h"
+#include "js/Class.h"
+#include "js/experimental/JitInfo.h"
+#include "js/Id.h"
+#include "js/RootingAPI.h"
+#include "js/ScalarType.h"
+#include "js/Value.h"
+#include "js/Vector.h"
+#include "util/Memory.h"
+#include "vm/JSFunction.h"
+#include "vm/JSScript.h"
+#include "vm/List.h"
+#include "vm/Opcodes.h"
+#include "vm/Shape.h"
+#include "wasm/WasmConstants.h"
+#include "wasm/WasmValType.h"
+
+class JS_PUBLIC_API JSTracer;
+struct JS_PUBLIC_API JSContext;
+
+class JSObject;
+class JSString;
+
+namespace JS {
+class Symbol;
+}
+
+namespace js {
+
+class GetterSetter;
+enum class UnaryMathFunction : uint8_t;
+
+namespace gc {
+class AllocSite;
+}
+
+namespace jit {
+
+class ICScript;
+
+#ifdef JS_SIMULATOR
+bool CallAnyNative(JSContext* cx, unsigned argc, Value* vp);
+#endif
+
+// Class to record CacheIR + some additional metadata for code generation.
+class MOZ_RAII CacheIRWriter : public JS::CustomAutoRooter {
+#ifdef DEBUG
+ JSContext* cx_;
+#endif
+ CompactBufferWriter buffer_;
+
+ uint32_t nextOperandId_;
+ uint32_t nextInstructionId_;
+ uint32_t numInputOperands_;
+
+ TypeData typeData_;
+
+ // The data (shapes, slot offsets, etc.) that will be stored in the ICStub.
+ Vector<StubField, 8, SystemAllocPolicy> stubFields_;
+ size_t stubDataSize_;
+
+ // For each operand id, record which instruction accessed it last. This
+ // information greatly improves register allocation.
+ Vector<uint32_t, 8, SystemAllocPolicy> operandLastUsed_;
+
+ // OperandId and stub offsets are stored in a single byte, so make sure
+ // this doesn't overflow. We use a very conservative limit for now.
+ static const size_t MaxOperandIds = 20;
+ static const size_t MaxStubDataSizeInBytes = 20 * sizeof(uintptr_t);
+ bool tooLarge_;
+
+ // Assume this stub can't be trial inlined until we see a scripted call/inline
+ // instruction.
+ TrialInliningState trialInliningState_ = TrialInliningState::Failure;
+
+ // Basic caching to avoid quadatic lookup behaviour in readStubField.
+ mutable uint32_t lastOffset_;
+ mutable uint32_t lastIndex_;
+
+#ifdef DEBUG
+ // Information for assertLengthMatches.
+ mozilla::Maybe<CacheOp> currentOp_;
+ size_t currentOpArgsStart_ = 0;
+#endif
+
+#ifdef DEBUG
+ void assertSameCompartment(JSObject* obj);
+ void assertSameZone(Shape* shape);
+#else
+ void assertSameCompartment(JSObject* obj) {}
+ void assertSameZone(Shape* shape) {}
+#endif
+
+ void writeOp(CacheOp op) {
+ buffer_.writeUnsigned15Bit(uint32_t(op));
+ nextInstructionId_++;
+#ifdef DEBUG
+ MOZ_ASSERT(currentOp_.isNothing(), "Missing call to assertLengthMatches?");
+ currentOp_.emplace(op);
+ currentOpArgsStart_ = buffer_.length();
+#endif
+ }
+
+ void assertLengthMatches() {
+#ifdef DEBUG
+ // After writing arguments, assert the length matches CacheIROpArgLengths.
+ size_t expectedLen = CacheIROpInfos[size_t(*currentOp_)].argLength;
+ MOZ_ASSERT_IF(!failed(),
+ buffer_.length() - currentOpArgsStart_ == expectedLen);
+ currentOp_.reset();
+#endif
+ }
+
+ void writeOperandId(OperandId opId) {
+ if (opId.id() < MaxOperandIds) {
+ static_assert(MaxOperandIds <= UINT8_MAX,
+ "operand id must fit in a single byte");
+ buffer_.writeByte(opId.id());
+ } else {
+ tooLarge_ = true;
+ return;
+ }
+ if (opId.id() >= operandLastUsed_.length()) {
+ buffer_.propagateOOM(operandLastUsed_.resize(opId.id() + 1));
+ if (buffer_.oom()) {
+ return;
+ }
+ }
+ MOZ_ASSERT(nextInstructionId_ > 0);
+ operandLastUsed_[opId.id()] = nextInstructionId_ - 1;
+ }
+
+ void writeCallFlagsImm(CallFlags flags) { buffer_.writeByte(flags.toByte()); }
+
+ void addStubField(uint64_t value, StubField::Type fieldType) {
+ size_t fieldOffset = stubDataSize_;
+#ifndef JS_64BIT
+ // On 32-bit platforms there are two stub field sizes (4 bytes and 8 bytes).
+ // Ensure 8-byte fields are properly aligned.
+ if (StubField::sizeIsInt64(fieldType)) {
+ fieldOffset = AlignBytes(fieldOffset, sizeof(uint64_t));
+ }
+#endif
+ MOZ_ASSERT((fieldOffset % StubField::sizeInBytes(fieldType)) == 0);
+
+ size_t newStubDataSize = fieldOffset + StubField::sizeInBytes(fieldType);
+ if (newStubDataSize < MaxStubDataSizeInBytes) {
+#ifndef JS_64BIT
+ // Add a RawInt32 stub field for padding if necessary, because when we
+ // iterate over the stub fields we assume there are no 'holes'.
+ if (fieldOffset != stubDataSize_) {
+ MOZ_ASSERT((stubDataSize_ + sizeof(uintptr_t)) == fieldOffset);
+ buffer_.propagateOOM(
+ stubFields_.append(StubField(0, StubField::Type::RawInt32)));
+ }
+#endif
+ buffer_.propagateOOM(stubFields_.append(StubField(value, fieldType)));
+ MOZ_ASSERT((fieldOffset % sizeof(uintptr_t)) == 0);
+ buffer_.writeByte(fieldOffset / sizeof(uintptr_t));
+ stubDataSize_ = newStubDataSize;
+ } else {
+ tooLarge_ = true;
+ }
+ }
+
+ void writeShapeField(Shape* shape) {
+ MOZ_ASSERT(shape);
+ assertSameZone(shape);
+ addStubField(uintptr_t(shape), StubField::Type::Shape);
+ }
+ void writeGetterSetterField(GetterSetter* gs) {
+ MOZ_ASSERT(gs);
+ addStubField(uintptr_t(gs), StubField::Type::GetterSetter);
+ }
+ void writeObjectField(JSObject* obj) {
+ MOZ_ASSERT(obj);
+ assertSameCompartment(obj);
+ addStubField(uintptr_t(obj), StubField::Type::JSObject);
+ }
+ void writeStringField(JSString* str) {
+ MOZ_ASSERT(str);
+ addStubField(uintptr_t(str), StubField::Type::String);
+ }
+ void writeSymbolField(JS::Symbol* sym) {
+ MOZ_ASSERT(sym);
+ addStubField(uintptr_t(sym), StubField::Type::Symbol);
+ }
+ void writeBaseScriptField(BaseScript* script) {
+ MOZ_ASSERT(script);
+ addStubField(uintptr_t(script), StubField::Type::BaseScript);
+ }
+ void writeRawInt32Field(uint32_t val) {
+ addStubField(val, StubField::Type::RawInt32);
+ }
+ void writeRawPointerField(const void* ptr) {
+ addStubField(uintptr_t(ptr), StubField::Type::RawPointer);
+ }
+ void writeIdField(jsid id) {
+ addStubField(id.asRawBits(), StubField::Type::Id);
+ }
+ void writeValueField(const Value& val) {
+ addStubField(val.asRawBits(), StubField::Type::Value);
+ }
+ void writeRawInt64Field(uint64_t val) {
+ addStubField(val, StubField::Type::RawInt64);
+ }
+ void writeDoubleField(double d) {
+ uint64_t bits = mozilla::BitwiseCast<uint64_t>(d);
+ addStubField(bits, StubField::Type::Double);
+ }
+ void writeAllocSiteField(gc::AllocSite* ptr) {
+ addStubField(uintptr_t(ptr), StubField::Type::AllocSite);
+ }
+
+ void writeJSOpImm(JSOp op) {
+ static_assert(sizeof(JSOp) == sizeof(uint8_t), "JSOp must fit in a byte");
+ buffer_.writeByte(uint8_t(op));
+ }
+ void writeGuardClassKindImm(GuardClassKind kind) {
+ static_assert(sizeof(GuardClassKind) == sizeof(uint8_t),
+ "GuardClassKind must fit in a byte");
+ buffer_.writeByte(uint8_t(kind));
+ }
+ void writeValueTypeImm(ValueType type) {
+ static_assert(sizeof(ValueType) == sizeof(uint8_t),
+ "ValueType must fit in uint8_t");
+ buffer_.writeByte(uint8_t(type));
+ }
+ void writeJSWhyMagicImm(JSWhyMagic whyMagic) {
+ static_assert(JS_WHY_MAGIC_COUNT <= UINT8_MAX,
+ "JSWhyMagic must fit in uint8_t");
+ buffer_.writeByte(uint8_t(whyMagic));
+ }
+ void writeScalarTypeImm(Scalar::Type type) {
+ MOZ_ASSERT(size_t(type) <= UINT8_MAX);
+ buffer_.writeByte(uint8_t(type));
+ }
+ void writeUnaryMathFunctionImm(UnaryMathFunction fun) {
+ static_assert(sizeof(UnaryMathFunction) == sizeof(uint8_t),
+ "UnaryMathFunction must fit in a byte");
+ buffer_.writeByte(uint8_t(fun));
+ }
+ void writeCompletionKindImm(CompletionKind kind) {
+ static_assert(sizeof(CompletionKind) == sizeof(uint8_t),
+ "CompletionKind must fit in a byte");
+ buffer_.writeByte(uint8_t(kind));
+ }
+ void writeBoolImm(bool b) { buffer_.writeByte(uint32_t(b)); }
+
+ void writeByteImm(uint32_t b) {
+ MOZ_ASSERT(b <= UINT8_MAX);
+ buffer_.writeByte(b);
+ }
+
+ void writeInt32Imm(int32_t i32) { buffer_.writeFixedUint32_t(i32); }
+ void writeUInt32Imm(uint32_t u32) { buffer_.writeFixedUint32_t(u32); }
+ void writePointer(const void* ptr) { buffer_.writeRawPointer(ptr); }
+
+ void writeJSNativeImm(JSNative native) {
+ writePointer(JS_FUNC_TO_DATA_PTR(void*, native));
+ }
+ void writeStaticStringImm(const char* str) { writePointer(str); }
+
+ void writeWasmValTypeImm(wasm::ValType::Kind kind) {
+ static_assert(unsigned(wasm::TypeCode::Limit) <= UINT8_MAX);
+ buffer_.writeByte(uint8_t(kind));
+ }
+
+ void writeAllocKindImm(gc::AllocKind kind) {
+ static_assert(unsigned(gc::AllocKind::LIMIT) <= UINT8_MAX);
+ buffer_.writeByte(uint8_t(kind));
+ }
+
+ uint32_t newOperandId() { return nextOperandId_++; }
+
+ CacheIRWriter(const CacheIRWriter&) = delete;
+ CacheIRWriter& operator=(const CacheIRWriter&) = delete;
+
+ public:
+ explicit CacheIRWriter(JSContext* cx)
+ : CustomAutoRooter(cx),
+#ifdef DEBUG
+ cx_(cx),
+#endif
+ nextOperandId_(0),
+ nextInstructionId_(0),
+ numInputOperands_(0),
+ stubDataSize_(0),
+ tooLarge_(false),
+ lastOffset_(0),
+ lastIndex_(0) {
+ }
+
+ bool tooLarge() const { return tooLarge_; }
+ bool oom() const { return buffer_.oom(); }
+ bool failed() const { return tooLarge() || oom(); }
+
+ TrialInliningState trialInliningState() const { return trialInliningState_; }
+
+ uint32_t numInputOperands() const { return numInputOperands_; }
+ uint32_t numOperandIds() const { return nextOperandId_; }
+ uint32_t numInstructions() const { return nextInstructionId_; }
+
+ size_t numStubFields() const { return stubFields_.length(); }
+ StubField::Type stubFieldType(uint32_t i) const {
+ return stubFields_[i].type();
+ }
+
+ uint32_t setInputOperandId(uint32_t op) {
+ MOZ_ASSERT(op == nextOperandId_);
+ nextOperandId_++;
+ numInputOperands_++;
+ return op;
+ }
+
+ TypeData typeData() const { return typeData_; }
+ void setTypeData(TypeData data) { typeData_ = data; }
+
+ void trace(JSTracer* trc) override {
+ // For now, assert we only GC before we append stub fields.
+ MOZ_RELEASE_ASSERT(stubFields_.empty());
+ }
+
+ size_t stubDataSize() const { return stubDataSize_; }
+ void copyStubData(uint8_t* dest) const;
+ bool stubDataEquals(const uint8_t* stubData) const;
+ bool stubDataEqualsIgnoring(const uint8_t* stubData,
+ uint32_t ignoreOffset) const;
+
+ bool operandIsDead(uint32_t operandId, uint32_t currentInstruction) const {
+ if (operandId >= operandLastUsed_.length()) {
+ return false;
+ }
+ return currentInstruction > operandLastUsed_[operandId];
+ }
+
+ const uint8_t* codeStart() const {
+ MOZ_ASSERT(!failed());
+ return buffer_.buffer();
+ }
+
+ const uint8_t* codeEnd() const {
+ MOZ_ASSERT(!failed());
+ return buffer_.buffer() + buffer_.length();
+ }
+
+ uint32_t codeLength() const {
+ MOZ_ASSERT(!failed());
+ return buffer_.length();
+ }
+
+ // This should not be used when compiling Baseline code, as Baseline code
+ // shouldn't bake in stub values.
+ StubField readStubField(uint32_t offset, StubField::Type type) const;
+
+ ObjOperandId guardToObject(ValOperandId input) {
+ guardToObject_(input);
+ return ObjOperandId(input.id());
+ }
+
+ StringOperandId guardToString(ValOperandId input) {
+ guardToString_(input);
+ return StringOperandId(input.id());
+ }
+
+ SymbolOperandId guardToSymbol(ValOperandId input) {
+ guardToSymbol_(input);
+ return SymbolOperandId(input.id());
+ }
+
+ BigIntOperandId guardToBigInt(ValOperandId input) {
+ guardToBigInt_(input);
+ return BigIntOperandId(input.id());
+ }
+
+ BooleanOperandId guardToBoolean(ValOperandId input) {
+ guardToBoolean_(input);
+ return BooleanOperandId(input.id());
+ }
+
+ Int32OperandId guardToInt32(ValOperandId input) {
+ guardToInt32_(input);
+ return Int32OperandId(input.id());
+ }
+
+ NumberOperandId guardIsNumber(ValOperandId input) {
+ guardIsNumber_(input);
+ return NumberOperandId(input.id());
+ }
+
+ ValOperandId boxObject(ObjOperandId input) {
+ return ValOperandId(input.id());
+ }
+
+ void guardShapeForClass(ObjOperandId obj, Shape* shape) {
+ // Guard shape to ensure that object class is unchanged. This is true
+ // for all shapes.
+ guardShape(obj, shape);
+ }
+
+ void guardShapeForOwnProperties(ObjOperandId obj, Shape* shape) {
+ // Guard shape to detect changes to (non-dense) own properties. This
+ // also implies |guardShapeForClass|.
+ MOZ_ASSERT(shape->getObjectClass()->isNativeObject());
+ guardShape(obj, shape);
+ }
+
+ public:
+ void guardSpecificFunction(ObjOperandId obj, JSFunction* expected) {
+ // Guard object is a specific function. This implies immutable fields on
+ // the JSFunction struct itself are unchanged.
+ // Bake in the nargs and FunctionFlags so Warp can use them off-main thread,
+ // instead of directly using the JSFunction fields.
+ uint32_t nargsAndFlags = expected->flagsAndArgCountRaw();
+ guardSpecificFunction_(obj, expected, nargsAndFlags);
+ }
+
+ void guardFunctionScript(ObjOperandId fun, BaseScript* expected) {
+ // Guard function has a specific BaseScript. This implies immutable fields
+ // on the JSFunction struct itself are unchanged and are equivalent for
+ // lambda clones.
+ // Bake in the nargs and FunctionFlags so Warp can use them off-main thread,
+ // instead of directly using the JSFunction fields.
+ uint32_t nargsAndFlags = expected->function()->flagsAndArgCountRaw();
+ guardFunctionScript_(fun, expected, nargsAndFlags);
+ }
+
+ ValOperandId loadArgumentFixedSlot(
+ ArgumentKind kind, uint32_t argc,
+ CallFlags flags = CallFlags(CallFlags::Standard)) {
+ bool addArgc;
+ int32_t slotIndex = GetIndexOfArgument(kind, flags, &addArgc);
+ if (addArgc) {
+ slotIndex += argc;
+ }
+ MOZ_ASSERT(slotIndex >= 0);
+ MOZ_ASSERT(slotIndex <= UINT8_MAX);
+ return loadArgumentFixedSlot_(slotIndex);
+ }
+
+ ValOperandId loadArgumentDynamicSlot(
+ ArgumentKind kind, Int32OperandId argcId,
+ CallFlags flags = CallFlags(CallFlags::Standard)) {
+ bool addArgc;
+ int32_t slotIndex = GetIndexOfArgument(kind, flags, &addArgc);
+ if (addArgc) {
+ return loadArgumentDynamicSlot_(argcId, slotIndex);
+ }
+ return loadArgumentFixedSlot_(slotIndex);
+ }
+
+ ObjOperandId loadSpreadArgs() {
+ ArgumentKind kind = ArgumentKind::Arg0;
+ uint32_t argc = 1;
+ CallFlags flags(CallFlags::Spread);
+ return ObjOperandId(loadArgumentFixedSlot(kind, argc, flags).id());
+ }
+
+ void callScriptedFunction(ObjOperandId callee, Int32OperandId argc,
+ CallFlags flags, uint32_t argcFixed) {
+ callScriptedFunction_(callee, argc, flags, argcFixed);
+ trialInliningState_ = TrialInliningState::Candidate;
+ }
+
+ void callInlinedFunction(ObjOperandId callee, Int32OperandId argc,
+ ICScript* icScript, CallFlags flags,
+ uint32_t argcFixed) {
+ callInlinedFunction_(callee, argc, icScript, flags, argcFixed);
+ trialInliningState_ = TrialInliningState::Inlined;
+ }
+
+ void callNativeFunction(ObjOperandId calleeId, Int32OperandId argc, JSOp op,
+ JSFunction* calleeFunc, CallFlags flags,
+ uint32_t argcFixed) {
+ // Some native functions can be implemented faster if we know that
+ // the return value is ignored.
+ bool ignoresReturnValue =
+ op == JSOp::CallIgnoresRv && calleeFunc->hasJitInfo() &&
+ calleeFunc->jitInfo()->type() == JSJitInfo::IgnoresReturnValueNative;
+
+#ifdef JS_SIMULATOR
+ // The simulator requires VM calls to be redirected to a special
+ // swi instruction to handle them, so we store the redirected
+ // pointer in the stub and use that instead of the original one.
+ // If we are calling the ignoresReturnValue version of a native
+ // function, we bake it into the redirected pointer.
+ // (See BaselineCacheIRCompiler::emitCallNativeFunction.)
+ JSNative target = ignoresReturnValue
+ ? calleeFunc->jitInfo()->ignoresReturnValueMethod
+ : calleeFunc->native();
+ void* rawPtr = JS_FUNC_TO_DATA_PTR(void*, target);
+ void* redirected = Simulator::RedirectNativeFunction(rawPtr, Args_General3);
+ callNativeFunction_(calleeId, argc, flags, argcFixed, redirected);
+#else
+ // If we are not running in the simulator, we generate different jitcode
+ // to find the ignoresReturnValue version of a native function.
+ callNativeFunction_(calleeId, argc, flags, argcFixed, ignoresReturnValue);
+#endif
+ }
+
+ void callDOMFunction(ObjOperandId calleeId, Int32OperandId argc,
+ ObjOperandId thisObjId, JSFunction* calleeFunc,
+ CallFlags flags, uint32_t argcFixed) {
+#ifdef JS_SIMULATOR
+ void* rawPtr = JS_FUNC_TO_DATA_PTR(void*, calleeFunc->native());
+ void* redirected = Simulator::RedirectNativeFunction(rawPtr, Args_General3);
+ callDOMFunction_(calleeId, argc, thisObjId, flags, argcFixed, redirected);
+#else
+ callDOMFunction_(calleeId, argc, thisObjId, flags, argcFixed);
+#endif
+ }
+
+ void callAnyNativeFunction(ObjOperandId calleeId, Int32OperandId argc,
+ CallFlags flags, uint32_t argcFixed) {
+ MOZ_ASSERT(!flags.isSameRealm());
+#ifdef JS_SIMULATOR
+ // The simulator requires native calls to be redirected to a
+ // special swi instruction. If we are calling an arbitrary native
+ // function, we can't wrap the real target ahead of time, so we
+ // call a wrapper function (CallAnyNative) that calls the target
+ // itself, and redirect that wrapper.
+ JSNative target = CallAnyNative;
+ void* rawPtr = JS_FUNC_TO_DATA_PTR(void*, target);
+ void* redirected = Simulator::RedirectNativeFunction(rawPtr, Args_General3);
+ callNativeFunction_(calleeId, argc, flags, argcFixed, redirected);
+#else
+ callNativeFunction_(calleeId, argc, flags, argcFixed,
+ /* ignoresReturnValue = */ false);
+#endif
+ }
+
+ void callClassHook(ObjOperandId calleeId, Int32OperandId argc, JSNative hook,
+ CallFlags flags, uint32_t argcFixed) {
+ MOZ_ASSERT(!flags.isSameRealm());
+ void* target = JS_FUNC_TO_DATA_PTR(void*, hook);
+#ifdef JS_SIMULATOR
+ // The simulator requires VM calls to be redirected to a special
+ // swi instruction to handle them, so we store the redirected
+ // pointer in the stub and use that instead of the original one.
+ target = Simulator::RedirectNativeFunction(target, Args_General3);
+#endif
+ callClassHook_(calleeId, argc, flags, argcFixed, target);
+ }
+
+ void callScriptedGetterResult(ValOperandId receiver, JSFunction* getter,
+ bool sameRealm) {
+ MOZ_ASSERT(getter->hasJitEntry());
+ uint32_t nargsAndFlags = getter->flagsAndArgCountRaw();
+ callScriptedGetterResult_(receiver, getter, sameRealm, nargsAndFlags);
+ trialInliningState_ = TrialInliningState::Candidate;
+ }
+
+ void callInlinedGetterResult(ValOperandId receiver, JSFunction* getter,
+ ICScript* icScript, bool sameRealm) {
+ MOZ_ASSERT(getter->hasJitEntry());
+ uint32_t nargsAndFlags = getter->flagsAndArgCountRaw();
+ callInlinedGetterResult_(receiver, getter, icScript, sameRealm,
+ nargsAndFlags);
+ trialInliningState_ = TrialInliningState::Inlined;
+ }
+
+ void callNativeGetterResult(ValOperandId receiver, JSFunction* getter,
+ bool sameRealm) {
+ MOZ_ASSERT(getter->isNativeWithoutJitEntry());
+ uint32_t nargsAndFlags = getter->flagsAndArgCountRaw();
+ callNativeGetterResult_(receiver, getter, sameRealm, nargsAndFlags);
+ }
+
+ void callScriptedSetter(ObjOperandId receiver, JSFunction* setter,
+ ValOperandId rhs, bool sameRealm) {
+ MOZ_ASSERT(setter->hasJitEntry());
+ uint32_t nargsAndFlags = setter->flagsAndArgCountRaw();
+ callScriptedSetter_(receiver, setter, rhs, sameRealm, nargsAndFlags);
+ trialInliningState_ = TrialInliningState::Candidate;
+ }
+
+ void callInlinedSetter(ObjOperandId receiver, JSFunction* setter,
+ ValOperandId rhs, ICScript* icScript, bool sameRealm) {
+ MOZ_ASSERT(setter->hasJitEntry());
+ uint32_t nargsAndFlags = setter->flagsAndArgCountRaw();
+ callInlinedSetter_(receiver, setter, rhs, icScript, sameRealm,
+ nargsAndFlags);
+ trialInliningState_ = TrialInliningState::Inlined;
+ }
+
+ void callNativeSetter(ObjOperandId receiver, JSFunction* setter,
+ ValOperandId rhs, bool sameRealm) {
+ MOZ_ASSERT(setter->isNativeWithoutJitEntry());
+ uint32_t nargsAndFlags = setter->flagsAndArgCountRaw();
+ callNativeSetter_(receiver, setter, rhs, sameRealm, nargsAndFlags);
+ }
+
+ void metaScriptedThisShape(Shape* thisShape) {
+ metaScriptedThisShape_(thisShape);
+ }
+
+ void guardMultipleShapes(ObjOperandId obj, ListObject* shapes) {
+ MOZ_ASSERT(shapes->length() > 0);
+ guardMultipleShapes_(obj, shapes);
+ }
+
+ friend class CacheIRCloner;
+
+ CACHE_IR_WRITER_GENERATED
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_CacheIRWriter_h */
diff --git a/js/src/jit/CalleeToken.h b/js/src/jit/CalleeToken.h
new file mode 100644
index 0000000000..b2944108a5
--- /dev/null
+++ b/js/src/jit/CalleeToken.h
@@ -0,0 +1,66 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CalleeToken_h
+#define jit_CalleeToken_h
+
+#include "mozilla/Assertions.h"
+
+#include <stdint.h>
+
+#include "js/TypeDecls.h"
+
+class JS_PUBLIC_API JSTracer;
+
+namespace js::jit {
+
+using CalleeToken = void*;
+
+enum CalleeTokenTag {
+ CalleeToken_Function = 0x0, // untagged
+ CalleeToken_FunctionConstructing = 0x1,
+ CalleeToken_Script = 0x2
+};
+
+// Any CalleeToken with this bit set must be CalleeToken_Script.
+static const uintptr_t CalleeTokenScriptBit = CalleeToken_Script;
+
+static const uintptr_t CalleeTokenMask = ~uintptr_t(0x3);
+
+static inline CalleeTokenTag GetCalleeTokenTag(CalleeToken token) {
+ CalleeTokenTag tag = CalleeTokenTag(uintptr_t(token) & 0x3);
+ MOZ_ASSERT(tag <= CalleeToken_Script);
+ return tag;
+}
+static inline CalleeToken CalleeToToken(JSFunction* fun, bool constructing) {
+ CalleeTokenTag tag =
+ constructing ? CalleeToken_FunctionConstructing : CalleeToken_Function;
+ return CalleeToken(uintptr_t(fun) | uintptr_t(tag));
+}
+static inline CalleeToken CalleeToToken(JSScript* script) {
+ return CalleeToken(uintptr_t(script) | uintptr_t(CalleeToken_Script));
+}
+static inline bool CalleeTokenIsFunction(CalleeToken token) {
+ CalleeTokenTag tag = GetCalleeTokenTag(token);
+ return tag == CalleeToken_Function || tag == CalleeToken_FunctionConstructing;
+}
+static inline bool CalleeTokenIsConstructing(CalleeToken token) {
+ return GetCalleeTokenTag(token) == CalleeToken_FunctionConstructing;
+}
+static inline JSFunction* CalleeTokenToFunction(CalleeToken token) {
+ MOZ_ASSERT(CalleeTokenIsFunction(token));
+ return (JSFunction*)(uintptr_t(token) & CalleeTokenMask);
+}
+static inline JSScript* CalleeTokenToScript(CalleeToken token) {
+ MOZ_ASSERT(GetCalleeTokenTag(token) == CalleeToken_Script);
+ return (JSScript*)(uintptr_t(token) & CalleeTokenMask);
+}
+
+CalleeToken TraceCalleeToken(JSTracer* trc, CalleeToken token);
+
+} /* namespace js::jit */
+
+#endif /* jit_CalleeToken_h */
diff --git a/js/src/jit/CodeGenerator.cpp b/js/src/jit/CodeGenerator.cpp
new file mode 100644
index 0000000000..5247bd465a
--- /dev/null
+++ b/js/src/jit/CodeGenerator.cpp
@@ -0,0 +1,18002 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/CodeGenerator.h"
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Casting.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/EnumeratedArray.h"
+#include "mozilla/EnumeratedRange.h"
+#include "mozilla/EnumSet.h"
+#include "mozilla/IntegerTypeTraits.h"
+#include "mozilla/Latin1.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/ScopeExit.h"
+#include "mozilla/Tuple.h"
+
+#include <limits>
+#include <type_traits>
+#include <utility>
+
+#include "jslibmath.h"
+#include "jsmath.h"
+#include "jsnum.h"
+
+#include "builtin/MapObject.h"
+#include "builtin/RegExp.h"
+#include "builtin/String.h"
+#include "irregexp/RegExpTypes.h"
+#include "jit/ABIArgGenerator.h"
+#include "jit/CompileInfo.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/Invalidation.h"
+#include "jit/IonIC.h"
+#include "jit/IonScript.h"
+#include "jit/JitcodeMap.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRealm.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/JitZone.h"
+#include "jit/Linker.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MoveEmitter.h"
+#include "jit/RangeAnalysis.h"
+#include "jit/SafepointIndex.h"
+#include "jit/SharedICHelpers.h"
+#include "jit/SharedICRegisters.h"
+#include "jit/VMFunctions.h"
+#include "jit/WarpSnapshot.h"
+#include "js/experimental/JitInfo.h" // JSJit{Getter,Setter}CallArgs, JSJitMethodCallArgsTraits, JSJitInfo
+#include "js/friend/DOMProxy.h" // JS::ExpandoAndGeneration
+#include "js/RegExpFlags.h" // JS::RegExpFlag
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "proxy/DOMProxy.h"
+#include "util/CheckedArithmetic.h"
+#include "util/Unicode.h"
+#include "vm/ArrayBufferViewObject.h"
+#include "vm/AsyncFunction.h"
+#include "vm/AsyncIteration.h"
+#include "vm/BuiltinObjectKind.h"
+#include "vm/FunctionFlags.h" // js::FunctionFlags
+#include "vm/Interpreter.h"
+#include "vm/JSAtom.h"
+#include "vm/MatchPairs.h"
+#include "vm/RegExpObject.h"
+#include "vm/RegExpStatics.h"
+#include "vm/StaticStrings.h"
+#include "vm/StringObject.h"
+#include "vm/StringType.h"
+#include "vm/TypedArrayObject.h"
+#ifdef MOZ_VTUNE
+# include "vtune/VTuneWrapper.h"
+#endif
+#include "wasm/WasmBinary.h"
+#include "wasm/WasmGC.h"
+#include "wasm/WasmGcObject.h"
+#include "wasm/WasmStubs.h"
+
+#include "builtin/Boolean-inl.h"
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+#include "jit/TemplateObject-inl.h"
+#include "jit/VMFunctionList-inl.h"
+#include "vm/JSScript-inl.h"
+#include "wasm/WasmInstance-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::GenericNaN;
+using mozilla::AssertedCast;
+using mozilla::DebugOnly;
+using mozilla::FloatingPoint;
+using mozilla::Maybe;
+using mozilla::NegativeInfinity;
+using mozilla::PositiveInfinity;
+
+using JS::ExpandoAndGeneration;
+
+namespace js {
+namespace jit {
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+template <class Op>
+static void HandleRegisterDump(Op op, MacroAssembler& masm,
+ LiveRegisterSet liveRegs, Register activation,
+ Register scratch) {
+ const size_t baseOffset = JitActivation::offsetOfRegs();
+
+ // Handle live GPRs.
+ for (GeneralRegisterIterator iter(liveRegs.gprs()); iter.more(); ++iter) {
+ Register reg = *iter;
+ Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
+
+ if (reg == activation) {
+ // To use the original value of the activation register (that's
+ // now on top of the stack), we need the scratch register.
+ masm.push(scratch);
+ masm.loadPtr(Address(masm.getStackPointer(), sizeof(uintptr_t)), scratch);
+ op(scratch, dump);
+ masm.pop(scratch);
+ } else {
+ op(reg, dump);
+ }
+ }
+
+ // Handle live FPRs.
+ for (FloatRegisterIterator iter(liveRegs.fpus()); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ Address dump(activation, baseOffset + RegisterDump::offsetOfRegister(reg));
+ op(reg, dump);
+ }
+}
+
+class StoreOp {
+ MacroAssembler& masm;
+
+ public:
+ explicit StoreOp(MacroAssembler& masm) : masm(masm) {}
+
+ void operator()(Register reg, Address dump) { masm.storePtr(reg, dump); }
+ void operator()(FloatRegister reg, Address dump) {
+ if (reg.isDouble()) {
+ masm.storeDouble(reg, dump);
+ } else if (reg.isSingle()) {
+ masm.storeFloat32(reg, dump);
+ } else if (reg.isSimd128()) {
+ MOZ_CRASH("Unexpected case for SIMD");
+ } else {
+ MOZ_CRASH("Unexpected register type.");
+ }
+ }
+};
+
+class VerifyOp {
+ MacroAssembler& masm;
+ Label* failure_;
+
+ public:
+ VerifyOp(MacroAssembler& masm, Label* failure)
+ : masm(masm), failure_(failure) {}
+
+ void operator()(Register reg, Address dump) {
+ masm.branchPtr(Assembler::NotEqual, dump, reg, failure_);
+ }
+ void operator()(FloatRegister reg, Address dump) {
+ if (reg.isDouble()) {
+ ScratchDoubleScope scratch(masm);
+ masm.loadDouble(dump, scratch);
+ masm.branchDouble(Assembler::DoubleNotEqual, scratch, reg, failure_);
+ } else if (reg.isSingle()) {
+ ScratchFloat32Scope scratch(masm);
+ masm.loadFloat32(dump, scratch);
+ masm.branchFloat(Assembler::DoubleNotEqual, scratch, reg, failure_);
+ } else if (reg.isSimd128()) {
+ MOZ_CRASH("Unexpected case for SIMD");
+ } else {
+ MOZ_CRASH("Unexpected register type.");
+ }
+ }
+};
+
+void CodeGenerator::verifyOsiPointRegs(LSafepoint* safepoint) {
+ // Ensure the live registers stored by callVM did not change between
+ // the call and this OsiPoint. Try-catch relies on this invariant.
+
+ // Load pointer to the JitActivation in a scratch register.
+ AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
+ Register scratch = allRegs.takeAny();
+ masm.push(scratch);
+ masm.loadJitActivation(scratch);
+
+ // If we should not check registers (because the instruction did not call
+ // into the VM, or a GC happened), we're done.
+ Label failure, done;
+ Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
+ masm.branch32(Assembler::Equal, checkRegs, Imm32(0), &done);
+
+ // Having more than one VM function call made in one visit function at
+ // runtime is a sec-ciritcal error, because if we conservatively assume that
+ // one of the function call can re-enter Ion, then the invalidation process
+ // will potentially add a call at a random location, by patching the code
+ // before the return address.
+ masm.branch32(Assembler::NotEqual, checkRegs, Imm32(1), &failure);
+
+ // Set checkRegs to 0, so that we don't try to verify registers after we
+ // return from this script to the caller.
+ masm.store32(Imm32(0), checkRegs);
+
+ // Ignore clobbered registers. Some instructions (like LValueToInt32) modify
+ // temps after calling into the VM. This is fine because no other
+ // instructions (including this OsiPoint) will depend on them. Also
+ // backtracking can also use the same register for an input and an output.
+ // These are marked as clobbered and shouldn't get checked.
+ LiveRegisterSet liveRegs;
+ liveRegs.set() = RegisterSet::Intersect(
+ safepoint->liveRegs().set(),
+ RegisterSet::Not(safepoint->clobberedRegs().set()));
+
+ VerifyOp op(masm, &failure);
+ HandleRegisterDump<VerifyOp>(op, masm, liveRegs, scratch, allRegs.getAny());
+
+ masm.jump(&done);
+
+ // Do not profile the callWithABI that occurs below. This is to avoid a
+ // rare corner case that occurs when profiling interacts with itself:
+ //
+ // When slow profiling assertions are turned on, FunctionBoundary ops
+ // (which update the profiler pseudo-stack) may emit a callVM, which
+ // forces them to have an osi point associated with them. The
+ // FunctionBoundary for inline function entry is added to the caller's
+ // graph with a PC from the caller's code, but during codegen it modifies
+ // Gecko Profiler instrumentation to add the callee as the current top-most
+ // script. When codegen gets to the OSIPoint, and the callWithABI below is
+ // emitted, the codegen thinks that the current frame is the callee, but
+ // the PC it's using from the OSIPoint refers to the caller. This causes
+ // the profiler instrumentation of the callWithABI below to ASSERT, since
+ // the script and pc are mismatched. To avoid this, we simply omit
+ // instrumentation for these callWithABIs.
+
+ // Any live register captured by a safepoint (other than temp registers)
+ // must remain unchanged between the call and the OsiPoint instruction.
+ masm.bind(&failure);
+ masm.assumeUnreachable("Modified registers between VM call and OsiPoint");
+
+ masm.bind(&done);
+ masm.pop(scratch);
+}
+
+bool CodeGenerator::shouldVerifyOsiPointRegs(LSafepoint* safepoint) {
+ if (!checkOsiPointRegisters) {
+ return false;
+ }
+
+ if (safepoint->liveRegs().emptyGeneral() &&
+ safepoint->liveRegs().emptyFloat()) {
+ return false; // No registers to check.
+ }
+
+ return true;
+}
+
+void CodeGenerator::resetOsiPointRegs(LSafepoint* safepoint) {
+ if (!shouldVerifyOsiPointRegs(safepoint)) {
+ return;
+ }
+
+ // Set checkRegs to 0. If we perform a VM call, the instruction
+ // will set it to 1.
+ AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
+ Register scratch = allRegs.takeAny();
+ masm.push(scratch);
+ masm.loadJitActivation(scratch);
+ Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
+ masm.store32(Imm32(0), checkRegs);
+ masm.pop(scratch);
+}
+
+static void StoreAllLiveRegs(MacroAssembler& masm, LiveRegisterSet liveRegs) {
+ // Store a copy of all live registers before performing the call.
+ // When we reach the OsiPoint, we can use this to check nothing
+ // modified them in the meantime.
+
+ // Load pointer to the JitActivation in a scratch register.
+ AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
+ Register scratch = allRegs.takeAny();
+ masm.push(scratch);
+ masm.loadJitActivation(scratch);
+
+ Address checkRegs(scratch, JitActivation::offsetOfCheckRegs());
+ masm.add32(Imm32(1), checkRegs);
+
+ StoreOp op(masm);
+ HandleRegisterDump<StoreOp>(op, masm, liveRegs, scratch, allRegs.getAny());
+
+ masm.pop(scratch);
+}
+#endif // CHECK_OSIPOINT_REGISTERS
+
+// Before doing any call to Cpp, you should ensure that volatile
+// registers are evicted by the register allocator.
+void CodeGenerator::callVMInternal(VMFunctionId id, LInstruction* ins) {
+ TrampolinePtr code = gen->jitRuntime()->getVMWrapper(id);
+ const VMFunctionData& fun = GetVMFunction(id);
+
+ // Stack is:
+ // ... frame ...
+ // [args]
+#ifdef DEBUG
+ MOZ_ASSERT(pushedArgs_ == fun.explicitArgs);
+ pushedArgs_ = 0;
+#endif
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ if (shouldVerifyOsiPointRegs(ins->safepoint())) {
+ StoreAllLiveRegs(masm, ins->safepoint()->liveRegs());
+ }
+#endif
+
+#ifdef DEBUG
+ if (ins->mirRaw()) {
+ MOZ_ASSERT(ins->mirRaw()->isInstruction());
+ MInstruction* mir = ins->mirRaw()->toInstruction();
+ MOZ_ASSERT_IF(mir->needsResumePoint(), mir->resumePoint());
+
+ // If this MIR instruction has an overridden AliasSet, set the JitRuntime's
+ // disallowArbitraryCode_ flag so we can assert this VMFunction doesn't call
+ // RunScript. Whitelist MInterruptCheck and MCheckOverRecursed because
+ // interrupt callbacks can call JS (chrome JS or shell testing functions).
+ bool isWhitelisted = mir->isInterruptCheck() || mir->isCheckOverRecursed();
+ if (!mir->hasDefaultAliasSet() && !isWhitelisted) {
+ const void* addr = gen->jitRuntime()->addressOfDisallowArbitraryCode();
+ masm.move32(Imm32(1), ReturnReg);
+ masm.store32(ReturnReg, AbsoluteAddress(addr));
+ }
+ }
+#endif
+
+ // Push an exit frame descriptor.
+ masm.PushFrameDescriptor(FrameType::IonJS);
+
+ // Call the wrapper function. The wrapper is in charge to unwind the stack
+ // when returning from the call. Failures are handled with exceptions based
+ // on the return value of the C functions. To guard the outcome of the
+ // returned value, use another LIR instruction.
+ uint32_t callOffset = masm.callJit(code);
+ markSafepointAt(callOffset, ins);
+
+#ifdef DEBUG
+ // Reset the disallowArbitraryCode flag after the call.
+ {
+ const void* addr = gen->jitRuntime()->addressOfDisallowArbitraryCode();
+ masm.push(ReturnReg);
+ masm.move32(Imm32(0), ReturnReg);
+ masm.store32(ReturnReg, AbsoluteAddress(addr));
+ masm.pop(ReturnReg);
+ }
+#endif
+
+ // Pop rest of the exit frame and the arguments left on the stack.
+ int framePop =
+ sizeof(ExitFrameLayout) - ExitFrameLayout::bytesPoppedAfterCall();
+ masm.implicitPop(fun.explicitStackSlots() * sizeof(void*) + framePop);
+
+ // Stack is:
+ // ... frame ...
+}
+
+template <typename Fn, Fn fn>
+void CodeGenerator::callVM(LInstruction* ins) {
+ VMFunctionId id = VMFunctionToId<Fn, fn>::id;
+ callVMInternal(id, ins);
+}
+
+// ArgSeq store arguments for OutOfLineCallVM.
+//
+// OutOfLineCallVM are created with "oolCallVM" function. The third argument of
+// this function is an instance of a class which provides a "generate" in charge
+// of pushing the argument, with "pushArg", for a VMFunction.
+//
+// Such list of arguments can be created by using the "ArgList" function which
+// creates one instance of "ArgSeq", where the type of the arguments are
+// inferred from the type of the arguments.
+//
+// The list of arguments must be written in the same order as if you were
+// calling the function in C++.
+//
+// Example:
+// ArgList(ToRegister(lir->lhs()), ToRegister(lir->rhs()))
+
+template <typename... ArgTypes>
+class ArgSeq {
+ mozilla::Tuple<std::remove_reference_t<ArgTypes>...> args_;
+
+ template <std::size_t... ISeq>
+ inline void generate(CodeGenerator* codegen,
+ std::index_sequence<ISeq...>) const {
+ // Arguments are pushed in reverse order, from last argument to first
+ // argument.
+ (codegen->pushArg(mozilla::Get<sizeof...(ISeq) - 1 - ISeq>(args_)), ...);
+ }
+
+ public:
+ explicit ArgSeq(ArgTypes&&... args)
+ : args_(std::forward<ArgTypes>(args)...) {}
+
+ inline void generate(CodeGenerator* codegen) const {
+ generate(codegen, std::index_sequence_for<ArgTypes...>{});
+ }
+
+#ifdef DEBUG
+ static constexpr size_t numArgs = sizeof...(ArgTypes);
+#endif
+};
+
+template <typename... ArgTypes>
+inline ArgSeq<ArgTypes...> ArgList(ArgTypes&&... args) {
+ return ArgSeq<ArgTypes...>(std::forward<ArgTypes>(args)...);
+}
+
+// Store wrappers, to generate the right move of data after the VM call.
+
+struct StoreNothing {
+ inline void generate(CodeGenerator* codegen) const {}
+ inline LiveRegisterSet clobbered() const {
+ return LiveRegisterSet(); // No register gets clobbered
+ }
+};
+
+class StoreRegisterTo {
+ private:
+ Register out_;
+
+ public:
+ explicit StoreRegisterTo(Register out) : out_(out) {}
+
+ inline void generate(CodeGenerator* codegen) const {
+ // It's okay to use storePointerResultTo here - the VMFunction wrapper
+ // ensures the upper bytes are zero for bool/int32 return values.
+ codegen->storePointerResultTo(out_);
+ }
+ inline LiveRegisterSet clobbered() const {
+ LiveRegisterSet set;
+ set.add(out_);
+ return set;
+ }
+};
+
+class StoreFloatRegisterTo {
+ private:
+ FloatRegister out_;
+
+ public:
+ explicit StoreFloatRegisterTo(FloatRegister out) : out_(out) {}
+
+ inline void generate(CodeGenerator* codegen) const {
+ codegen->storeFloatResultTo(out_);
+ }
+ inline LiveRegisterSet clobbered() const {
+ LiveRegisterSet set;
+ set.add(out_);
+ return set;
+ }
+};
+
+template <typename Output>
+class StoreValueTo_ {
+ private:
+ Output out_;
+
+ public:
+ explicit StoreValueTo_(const Output& out) : out_(out) {}
+
+ inline void generate(CodeGenerator* codegen) const {
+ codegen->storeResultValueTo(out_);
+ }
+ inline LiveRegisterSet clobbered() const {
+ LiveRegisterSet set;
+ set.add(out_);
+ return set;
+ }
+};
+
+template <typename Output>
+StoreValueTo_<Output> StoreValueTo(const Output& out) {
+ return StoreValueTo_<Output>(out);
+}
+
+template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
+class OutOfLineCallVM : public OutOfLineCodeBase<CodeGenerator> {
+ private:
+ LInstruction* lir_;
+ ArgSeq args_;
+ StoreOutputTo out_;
+
+ public:
+ OutOfLineCallVM(LInstruction* lir, const ArgSeq& args,
+ const StoreOutputTo& out)
+ : lir_(lir), args_(args), out_(out) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineCallVM(this);
+ }
+
+ LInstruction* lir() const { return lir_; }
+ const ArgSeq& args() const { return args_; }
+ const StoreOutputTo& out() const { return out_; }
+};
+
+template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
+OutOfLineCode* CodeGenerator::oolCallVM(LInstruction* lir, const ArgSeq& args,
+ const StoreOutputTo& out) {
+ MOZ_ASSERT(lir->mirRaw());
+ MOZ_ASSERT(lir->mirRaw()->isInstruction());
+
+#ifdef DEBUG
+ VMFunctionId id = VMFunctionToId<Fn, fn>::id;
+ const VMFunctionData& fun = GetVMFunction(id);
+ MOZ_ASSERT(fun.explicitArgs == args.numArgs);
+ MOZ_ASSERT(fun.returnsData() !=
+ (std::is_same_v<StoreOutputTo, StoreNothing>));
+#endif
+
+ OutOfLineCode* ool = new (alloc())
+ OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>(lir, args, out);
+ addOutOfLineCode(ool, lir->mirRaw()->toInstruction());
+ return ool;
+}
+
+template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
+void CodeGenerator::visitOutOfLineCallVM(
+ OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool) {
+ LInstruction* lir = ool->lir();
+
+ saveLive(lir);
+ ool->args().generate(this);
+ callVM<Fn, fn>(lir);
+ ool->out().generate(this);
+ restoreLiveIgnore(lir, ool->out().clobbered());
+ masm.jump(ool->rejoin());
+}
+
+class OutOfLineICFallback : public OutOfLineCodeBase<CodeGenerator> {
+ private:
+ LInstruction* lir_;
+ size_t cacheIndex_;
+ size_t cacheInfoIndex_;
+
+ public:
+ OutOfLineICFallback(LInstruction* lir, size_t cacheIndex,
+ size_t cacheInfoIndex)
+ : lir_(lir), cacheIndex_(cacheIndex), cacheInfoIndex_(cacheInfoIndex) {}
+
+ void bind(MacroAssembler* masm) override {
+ // The binding of the initial jump is done in
+ // CodeGenerator::visitOutOfLineICFallback.
+ }
+
+ size_t cacheIndex() const { return cacheIndex_; }
+ size_t cacheInfoIndex() const { return cacheInfoIndex_; }
+ LInstruction* lir() const { return lir_; }
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineICFallback(this);
+ }
+};
+
+void CodeGeneratorShared::addIC(LInstruction* lir, size_t cacheIndex) {
+ if (cacheIndex == SIZE_MAX) {
+ masm.setOOM();
+ return;
+ }
+
+ DataPtr<IonIC> cache(this, cacheIndex);
+ MInstruction* mir = lir->mirRaw()->toInstruction();
+ cache->setScriptedLocation(mir->block()->info().script(),
+ mir->resumePoint()->pc());
+
+ Register temp = cache->scratchRegisterForEntryJump();
+ icInfo_.back().icOffsetForJump = masm.movWithPatch(ImmWord(-1), temp);
+ masm.jump(Address(temp, 0));
+
+ MOZ_ASSERT(!icInfo_.empty());
+
+ OutOfLineICFallback* ool =
+ new (alloc()) OutOfLineICFallback(lir, cacheIndex, icInfo_.length() - 1);
+ addOutOfLineCode(ool, mir);
+
+ masm.bind(ool->rejoin());
+ cache->setRejoinOffset(CodeOffset(ool->rejoin()->offset()));
+}
+
+void CodeGenerator::visitOutOfLineICFallback(OutOfLineICFallback* ool) {
+ LInstruction* lir = ool->lir();
+ size_t cacheIndex = ool->cacheIndex();
+ size_t cacheInfoIndex = ool->cacheInfoIndex();
+
+ DataPtr<IonIC> ic(this, cacheIndex);
+
+ // Register the location of the OOL path in the IC.
+ ic->setFallbackOffset(CodeOffset(masm.currentOffset()));
+
+ switch (ic->kind()) {
+ case CacheKind::GetProp:
+ case CacheKind::GetElem: {
+ IonGetPropertyIC* getPropIC = ic->asGetPropertyIC();
+
+ saveLive(lir);
+
+ pushArg(getPropIC->id());
+ pushArg(getPropIC->value());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, IonGetPropertyIC*,
+ HandleValue, HandleValue, MutableHandleValue);
+ callVM<Fn, IonGetPropertyIC::update>(lir);
+
+ StoreValueTo(getPropIC->output()).generate(this);
+ restoreLiveIgnore(lir, StoreValueTo(getPropIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::GetPropSuper:
+ case CacheKind::GetElemSuper: {
+ IonGetPropSuperIC* getPropSuperIC = ic->asGetPropSuperIC();
+
+ saveLive(lir);
+
+ pushArg(getPropSuperIC->id());
+ pushArg(getPropSuperIC->receiver());
+ pushArg(getPropSuperIC->object());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn =
+ bool (*)(JSContext*, HandleScript, IonGetPropSuperIC*, HandleObject,
+ HandleValue, HandleValue, MutableHandleValue);
+ callVM<Fn, IonGetPropSuperIC::update>(lir);
+
+ StoreValueTo(getPropSuperIC->output()).generate(this);
+ restoreLiveIgnore(lir,
+ StoreValueTo(getPropSuperIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::SetProp:
+ case CacheKind::SetElem: {
+ IonSetPropertyIC* setPropIC = ic->asSetPropertyIC();
+
+ saveLive(lir);
+
+ pushArg(setPropIC->rhs());
+ pushArg(setPropIC->id());
+ pushArg(setPropIC->object());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, IonSetPropertyIC*,
+ HandleObject, HandleValue, HandleValue);
+ callVM<Fn, IonSetPropertyIC::update>(lir);
+
+ restoreLive(lir);
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::GetName: {
+ IonGetNameIC* getNameIC = ic->asGetNameIC();
+
+ saveLive(lir);
+
+ pushArg(getNameIC->environment());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, IonGetNameIC*, HandleObject,
+ MutableHandleValue);
+ callVM<Fn, IonGetNameIC::update>(lir);
+
+ StoreValueTo(getNameIC->output()).generate(this);
+ restoreLiveIgnore(lir, StoreValueTo(getNameIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::BindName: {
+ IonBindNameIC* bindNameIC = ic->asBindNameIC();
+
+ saveLive(lir);
+
+ pushArg(bindNameIC->environment());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn =
+ JSObject* (*)(JSContext*, HandleScript, IonBindNameIC*, HandleObject);
+ callVM<Fn, IonBindNameIC::update>(lir);
+
+ StoreRegisterTo(bindNameIC->output()).generate(this);
+ restoreLiveIgnore(lir, StoreRegisterTo(bindNameIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::GetIterator: {
+ IonGetIteratorIC* getIteratorIC = ic->asGetIteratorIC();
+
+ saveLive(lir);
+
+ pushArg(getIteratorIC->value());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = JSObject* (*)(JSContext*, HandleScript, IonGetIteratorIC*,
+ HandleValue);
+ callVM<Fn, IonGetIteratorIC::update>(lir);
+
+ StoreRegisterTo(getIteratorIC->output()).generate(this);
+ restoreLiveIgnore(lir,
+ StoreRegisterTo(getIteratorIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::OptimizeSpreadCall: {
+ auto* optimizeSpreadCallIC = ic->asOptimizeSpreadCallIC();
+
+ saveLive(lir);
+
+ pushArg(optimizeSpreadCallIC->value());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, IonOptimizeSpreadCallIC*,
+ HandleValue, MutableHandleValue);
+ callVM<Fn, IonOptimizeSpreadCallIC::update>(lir);
+
+ StoreValueTo(optimizeSpreadCallIC->output()).generate(this);
+ restoreLiveIgnore(
+ lir, StoreValueTo(optimizeSpreadCallIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::In: {
+ IonInIC* inIC = ic->asInIC();
+
+ saveLive(lir);
+
+ pushArg(inIC->object());
+ pushArg(inIC->key());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, IonInIC*, HandleValue,
+ HandleObject, bool*);
+ callVM<Fn, IonInIC::update>(lir);
+
+ StoreRegisterTo(inIC->output()).generate(this);
+ restoreLiveIgnore(lir, StoreRegisterTo(inIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::HasOwn: {
+ IonHasOwnIC* hasOwnIC = ic->asHasOwnIC();
+
+ saveLive(lir);
+
+ pushArg(hasOwnIC->id());
+ pushArg(hasOwnIC->value());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, IonHasOwnIC*, HandleValue,
+ HandleValue, int32_t*);
+ callVM<Fn, IonHasOwnIC::update>(lir);
+
+ StoreRegisterTo(hasOwnIC->output()).generate(this);
+ restoreLiveIgnore(lir, StoreRegisterTo(hasOwnIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::CheckPrivateField: {
+ IonCheckPrivateFieldIC* checkPrivateFieldIC = ic->asCheckPrivateFieldIC();
+
+ saveLive(lir);
+
+ pushArg(checkPrivateFieldIC->id());
+ pushArg(checkPrivateFieldIC->value());
+
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, IonCheckPrivateFieldIC*,
+ HandleValue, HandleValue, bool*);
+ callVM<Fn, IonCheckPrivateFieldIC::update>(lir);
+
+ StoreRegisterTo(checkPrivateFieldIC->output()).generate(this);
+ restoreLiveIgnore(
+ lir, StoreRegisterTo(checkPrivateFieldIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::InstanceOf: {
+ IonInstanceOfIC* hasInstanceOfIC = ic->asInstanceOfIC();
+
+ saveLive(lir);
+
+ pushArg(hasInstanceOfIC->rhs());
+ pushArg(hasInstanceOfIC->lhs());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, IonInstanceOfIC*,
+ HandleValue lhs, HandleObject rhs, bool* res);
+ callVM<Fn, IonInstanceOfIC::update>(lir);
+
+ StoreRegisterTo(hasInstanceOfIC->output()).generate(this);
+ restoreLiveIgnore(lir,
+ StoreRegisterTo(hasInstanceOfIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::UnaryArith: {
+ IonUnaryArithIC* unaryArithIC = ic->asUnaryArithIC();
+
+ saveLive(lir);
+
+ pushArg(unaryArithIC->input());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
+ IonUnaryArithIC * stub, HandleValue val,
+ MutableHandleValue res);
+ callVM<Fn, IonUnaryArithIC::update>(lir);
+
+ StoreValueTo(unaryArithIC->output()).generate(this);
+ restoreLiveIgnore(lir, StoreValueTo(unaryArithIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::ToPropertyKey: {
+ IonToPropertyKeyIC* toPropertyKeyIC = ic->asToPropertyKeyIC();
+
+ saveLive(lir);
+
+ pushArg(toPropertyKeyIC->input());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
+ IonToPropertyKeyIC * ic, HandleValue val,
+ MutableHandleValue res);
+ callVM<Fn, IonToPropertyKeyIC::update>(lir);
+
+ StoreValueTo(toPropertyKeyIC->output()).generate(this);
+ restoreLiveIgnore(lir,
+ StoreValueTo(toPropertyKeyIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::BinaryArith: {
+ IonBinaryArithIC* binaryArithIC = ic->asBinaryArithIC();
+
+ saveLive(lir);
+
+ pushArg(binaryArithIC->rhs());
+ pushArg(binaryArithIC->lhs());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
+ IonBinaryArithIC * stub, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue res);
+ callVM<Fn, IonBinaryArithIC::update>(lir);
+
+ StoreValueTo(binaryArithIC->output()).generate(this);
+ restoreLiveIgnore(lir, StoreValueTo(binaryArithIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::Compare: {
+ IonCompareIC* compareIC = ic->asCompareIC();
+
+ saveLive(lir);
+
+ pushArg(compareIC->rhs());
+ pushArg(compareIC->lhs());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn = bool (*)(JSContext * cx, HandleScript outerScript,
+ IonCompareIC * stub, HandleValue lhs, HandleValue rhs,
+ bool* res);
+ callVM<Fn, IonCompareIC::update>(lir);
+
+ StoreRegisterTo(compareIC->output()).generate(this);
+ restoreLiveIgnore(lir, StoreRegisterTo(compareIC->output()).clobbered());
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::CloseIter: {
+ IonCloseIterIC* closeIterIC = ic->asCloseIterIC();
+
+ saveLive(lir);
+
+ pushArg(closeIterIC->iter());
+ icInfo_[cacheInfoIndex].icOffsetForPush = pushArgWithPatch(ImmWord(-1));
+ pushArg(ImmGCPtr(gen->outerInfo().script()));
+
+ using Fn =
+ bool (*)(JSContext*, HandleScript, IonCloseIterIC*, HandleObject);
+ callVM<Fn, IonCloseIterIC::update>(lir);
+
+ restoreLive(lir);
+
+ masm.jump(ool->rejoin());
+ return;
+ }
+ case CacheKind::Call:
+ case CacheKind::TypeOf:
+ case CacheKind::ToBool:
+ case CacheKind::GetIntrinsic:
+ case CacheKind::NewArray:
+ case CacheKind::NewObject:
+ MOZ_CRASH("Unsupported IC");
+ }
+ MOZ_CRASH();
+}
+
+StringObject* MNewStringObject::templateObj() const {
+ return &templateObj_->as<StringObject>();
+}
+
+CodeGenerator::CodeGenerator(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorSpecific(gen, graph, masm),
+ ionScriptLabels_(gen->alloc()),
+ ionNurseryObjectLabels_(gen->alloc()),
+ scriptCounts_(nullptr),
+ realmStubsToReadBarrier_(0) {}
+
+CodeGenerator::~CodeGenerator() { js_delete(scriptCounts_); }
+
+class OutOfLineZeroIfNaN : public OutOfLineCodeBase<CodeGenerator> {
+ LInstruction* lir_;
+ FloatRegister input_;
+ Register output_;
+
+ public:
+ OutOfLineZeroIfNaN(LInstruction* lir, FloatRegister input, Register output)
+ : lir_(lir), input_(input), output_(output) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineZeroIfNaN(this);
+ }
+ LInstruction* lir() const { return lir_; }
+ FloatRegister input() const { return input_; }
+ Register output() const { return output_; }
+};
+
+void CodeGenerator::visitValueToInt32(LValueToInt32* lir) {
+ ValueOperand operand = ToValue(lir, LValueToInt32::Input);
+ Register output = ToRegister(lir->output());
+ FloatRegister temp = ToFloatRegister(lir->tempFloat());
+
+ Label fails;
+ if (lir->mode() == LValueToInt32::TRUNCATE) {
+ OutOfLineCode* oolDouble = oolTruncateDouble(temp, output, lir->mir());
+
+ // We can only handle strings in truncation contexts, like bitwise
+ // operations.
+ Register stringReg = ToRegister(lir->temp());
+ using Fn = bool (*)(JSContext*, JSString*, double*);
+ auto* oolString = oolCallVM<Fn, StringToNumber>(lir, ArgList(stringReg),
+ StoreFloatRegisterTo(temp));
+ Label* stringEntry = oolString->entry();
+ Label* stringRejoin = oolString->rejoin();
+
+ masm.truncateValueToInt32(operand, stringEntry, stringRejoin,
+ oolDouble->entry(), stringReg, temp, output,
+ &fails);
+ masm.bind(oolDouble->rejoin());
+ } else if (lir->mode() == LValueToInt32::TRUNCATE_NOWRAP) {
+ auto* ool = new (alloc()) OutOfLineZeroIfNaN(lir, temp, output);
+ addOutOfLineCode(ool, lir->mir());
+ masm.truncateNoWrapValueToInt32(operand, temp, output, ool->entry(),
+ &fails);
+ masm.bind(ool->rejoin());
+ } else {
+ masm.convertValueToInt32(operand, temp, output, &fails,
+ lir->mirNormal()->needsNegativeZeroCheck(),
+ lir->mirNormal()->conversion());
+ }
+
+ bailoutFrom(&fails, lir->snapshot());
+}
+
+void CodeGenerator::visitOutOfLineZeroIfNaN(OutOfLineZeroIfNaN* ool) {
+ FloatRegister input = ool->input();
+ Register output = ool->output();
+
+ // NaN triggers the failure path for branchTruncateDoubleToInt32() on x86,
+ // x64, and ARM64, so handle it here. In all other cases bail out.
+
+ Label fails;
+ if (input.isSingle()) {
+ masm.branchFloat(Assembler::DoubleOrdered, input, input, &fails);
+ } else {
+ masm.branchDouble(Assembler::DoubleOrdered, input, input, &fails);
+ }
+
+ // ToInteger(NaN) is 0.
+ masm.move32(Imm32(0), output);
+ masm.jump(ool->rejoin());
+
+ bailoutFrom(&fails, ool->lir()->snapshot());
+}
+
+void CodeGenerator::visitValueToDouble(LValueToDouble* lir) {
+ ValueOperand operand = ToValue(lir, LValueToDouble::InputIndex);
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ // Set if we can handle other primitives beside strings, as long as they're
+ // guaranteed to never throw. This rules out symbols and BigInts, but allows
+ // booleans, undefined, and null.
+ bool hasNonStringPrimitives =
+ lir->mir()->conversion() == MToFPInstruction::NonStringPrimitives;
+
+ Label isDouble, isInt32, isBool, isNull, isUndefined, done;
+
+ {
+ ScratchTagScope tag(masm, operand);
+ masm.splitTagForTest(operand, tag);
+
+ masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
+ masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
+
+ if (hasNonStringPrimitives) {
+ masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
+ masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
+ masm.branchTestNull(Assembler::Equal, tag, &isNull);
+ }
+ }
+
+ bailout(lir->snapshot());
+
+ if (hasNonStringPrimitives) {
+ masm.bind(&isNull);
+ masm.loadConstantDouble(0.0, output);
+ masm.jump(&done);
+ }
+
+ if (hasNonStringPrimitives) {
+ masm.bind(&isUndefined);
+ masm.loadConstantDouble(GenericNaN(), output);
+ masm.jump(&done);
+ }
+
+ if (hasNonStringPrimitives) {
+ masm.bind(&isBool);
+ masm.boolValueToDouble(operand, output);
+ masm.jump(&done);
+ }
+
+ masm.bind(&isInt32);
+ masm.int32ValueToDouble(operand, output);
+ masm.jump(&done);
+
+ masm.bind(&isDouble);
+ masm.unboxDouble(operand, output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitValueToFloat32(LValueToFloat32* lir) {
+ ValueOperand operand = ToValue(lir, LValueToFloat32::InputIndex);
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ // Set if we can handle other primitives beside strings, as long as they're
+ // guaranteed to never throw. This rules out symbols and BigInts, but allows
+ // booleans, undefined, and null.
+ bool hasNonStringPrimitives =
+ lir->mir()->conversion() == MToFPInstruction::NonStringPrimitives;
+
+ Label isDouble, isInt32, isBool, isNull, isUndefined, done;
+
+ {
+ ScratchTagScope tag(masm, operand);
+ masm.splitTagForTest(operand, tag);
+
+ masm.branchTestDouble(Assembler::Equal, tag, &isDouble);
+ masm.branchTestInt32(Assembler::Equal, tag, &isInt32);
+
+ if (hasNonStringPrimitives) {
+ masm.branchTestBoolean(Assembler::Equal, tag, &isBool);
+ masm.branchTestUndefined(Assembler::Equal, tag, &isUndefined);
+ masm.branchTestNull(Assembler::Equal, tag, &isNull);
+ }
+ }
+
+ bailout(lir->snapshot());
+
+ if (hasNonStringPrimitives) {
+ masm.bind(&isNull);
+ masm.loadConstantFloat32(0.0f, output);
+ masm.jump(&done);
+ }
+
+ if (hasNonStringPrimitives) {
+ masm.bind(&isUndefined);
+ masm.loadConstantFloat32(float(GenericNaN()), output);
+ masm.jump(&done);
+ }
+
+ if (hasNonStringPrimitives) {
+ masm.bind(&isBool);
+ masm.boolValueToFloat32(operand, output);
+ masm.jump(&done);
+ }
+
+ masm.bind(&isInt32);
+ masm.int32ValueToFloat32(operand, output);
+ masm.jump(&done);
+
+ masm.bind(&isDouble);
+ // ARM and MIPS may not have a double register available if we've
+ // allocated output as a float32.
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32)
+ ScratchDoubleScope fpscratch(masm);
+ masm.unboxDouble(operand, fpscratch);
+ masm.convertDoubleToFloat32(fpscratch, output);
+#else
+ masm.unboxDouble(operand, output);
+ masm.convertDoubleToFloat32(output, output);
+#endif
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitValueToBigInt(LValueToBigInt* lir) {
+ ValueOperand operand = ToValue(lir, LValueToBigInt::InputIndex);
+ Register output = ToRegister(lir->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleValue);
+ auto* ool =
+ oolCallVM<Fn, ToBigInt>(lir, ArgList(operand), StoreRegisterTo(output));
+
+ Register tag = masm.extractTag(operand, output);
+
+ Label notBigInt, done;
+ masm.branchTestBigInt(Assembler::NotEqual, tag, &notBigInt);
+ masm.unboxBigInt(operand, output);
+ masm.jump(&done);
+ masm.bind(&notBigInt);
+
+ masm.branchTestBoolean(Assembler::Equal, tag, ool->entry());
+ masm.branchTestString(Assembler::Equal, tag, ool->entry());
+
+ // ToBigInt(object) can have side-effects; all other types throw a TypeError.
+ bailout(lir->snapshot());
+
+ masm.bind(ool->rejoin());
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitInt32ToDouble(LInt32ToDouble* lir) {
+ masm.convertInt32ToDouble(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitFloat32ToDouble(LFloat32ToDouble* lir) {
+ masm.convertFloat32ToDouble(ToFloatRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitDoubleToFloat32(LDoubleToFloat32* lir) {
+ masm.convertDoubleToFloat32(ToFloatRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitInt32ToFloat32(LInt32ToFloat32* lir) {
+ masm.convertInt32ToFloat32(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitDoubleToInt32(LDoubleToInt32* lir) {
+ Label fail;
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ masm.convertDoubleToInt32(input, output, &fail,
+ lir->mir()->needsNegativeZeroCheck());
+ bailoutFrom(&fail, lir->snapshot());
+}
+
+void CodeGenerator::visitFloat32ToInt32(LFloat32ToInt32* lir) {
+ Label fail;
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ masm.convertFloat32ToInt32(input, output, &fail,
+ lir->mir()->needsNegativeZeroCheck());
+ bailoutFrom(&fail, lir->snapshot());
+}
+
+void CodeGenerator::visitDoubleToIntegerInt32(LDoubleToIntegerInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ auto* ool = new (alloc()) OutOfLineZeroIfNaN(lir, input, output);
+ addOutOfLineCode(ool, lir->mir());
+
+ masm.branchTruncateDoubleToInt32(input, output, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitFloat32ToIntegerInt32(LFloat32ToIntegerInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ auto* ool = new (alloc()) OutOfLineZeroIfNaN(lir, input, output);
+ addOutOfLineCode(ool, lir->mir());
+
+ masm.branchTruncateFloat32ToInt32(input, output, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitInt32ToIntPtr(LInt32ToIntPtr* lir) {
+#ifdef JS_64BIT
+ Register output = ToRegister(lir->output());
+
+ // This is a no-op if the input can't be negative. In debug builds assert
+ // (1) the upper 32 bits are zero and (2) the value <= INT32_MAX so that sign
+ // extension isn't needed.
+ if (!lir->mir()->canBeNegative()) {
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+# ifdef DEBUG
+ Label ok;
+ masm.branchPtr(Assembler::BelowOrEqual, output, ImmWord(INT32_MAX), &ok);
+ masm.assumeUnreachable("LInt32ToIntPtr: unexpected range for value");
+ masm.bind(&ok);
+# else
+ MOZ_CRASH("Not used in non-debug mode");
+# endif
+ return;
+ }
+
+ const LAllocation* input = lir->input();
+ if (input->isRegister()) {
+ masm.move32SignExtendToPtr(ToRegister(input), output);
+ } else {
+ masm.load32SignExtendToPtr(ToAddress(input), output);
+ }
+#else
+ MOZ_CRASH("Not used on 32-bit platforms");
+#endif
+}
+
+void CodeGenerator::visitNonNegativeIntPtrToInt32(
+ LNonNegativeIntPtrToInt32* lir) {
+#ifdef JS_64BIT
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+
+ Label bail;
+ masm.guardNonNegativeIntPtrToInt32(output, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+#else
+ MOZ_CRASH("Not used on 32-bit platforms");
+#endif
+}
+
+void CodeGenerator::visitIntPtrToDouble(LIntPtrToDouble* lir) {
+ Register input = ToRegister(lir->input());
+ FloatRegister output = ToFloatRegister(lir->output());
+ masm.convertIntPtrToDouble(input, output);
+}
+
+void CodeGenerator::visitAdjustDataViewLength(LAdjustDataViewLength* lir) {
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+
+ uint32_t byteSize = lir->mir()->byteSize();
+
+#ifdef DEBUG
+ Label ok;
+ masm.branchTestPtr(Assembler::NotSigned, output, output, &ok);
+ masm.assumeUnreachable("Unexpected negative value in LAdjustDataViewLength");
+ masm.bind(&ok);
+#endif
+
+ Label bail;
+ masm.branchSubPtr(Assembler::Signed, Imm32(byteSize - 1), output, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::emitOOLTestObject(Register objreg,
+ Label* ifEmulatesUndefined,
+ Label* ifDoesntEmulateUndefined,
+ Register scratch) {
+ saveVolatile(scratch);
+ using Fn = bool (*)(JSObject * obj);
+ masm.setupAlignedABICall();
+ masm.passABIArg(objreg);
+ masm.callWithABI<Fn, js::EmulatesUndefined>();
+ masm.storeCallPointerResult(scratch);
+ restoreVolatile(scratch);
+
+ masm.branchIfTrueBool(scratch, ifEmulatesUndefined);
+ masm.jump(ifDoesntEmulateUndefined);
+}
+
+// Base out-of-line code generator for all tests of the truthiness of an
+// object, where the object might not be truthy. (Recall that per spec all
+// objects are truthy, but we implement the JSCLASS_EMULATES_UNDEFINED class
+// flag to permit objects to look like |undefined| in certain contexts,
+// including in object truthiness testing.) We check truthiness inline except
+// when we're testing it on a proxy, in which case out-of-line code will call
+// EmulatesUndefined for a conclusive answer.
+class OutOfLineTestObject : public OutOfLineCodeBase<CodeGenerator> {
+ Register objreg_;
+ Register scratch_;
+
+ Label* ifEmulatesUndefined_;
+ Label* ifDoesntEmulateUndefined_;
+
+#ifdef DEBUG
+ bool initialized() { return ifEmulatesUndefined_ != nullptr; }
+#endif
+
+ public:
+ OutOfLineTestObject()
+ : ifEmulatesUndefined_(nullptr), ifDoesntEmulateUndefined_(nullptr) {}
+
+ void accept(CodeGenerator* codegen) final {
+ MOZ_ASSERT(initialized());
+ codegen->emitOOLTestObject(objreg_, ifEmulatesUndefined_,
+ ifDoesntEmulateUndefined_, scratch_);
+ }
+
+ // Specify the register where the object to be tested is found, labels to
+ // jump to if the object is truthy or falsy, and a scratch register for
+ // use in the out-of-line path.
+ void setInputAndTargets(Register objreg, Label* ifEmulatesUndefined,
+ Label* ifDoesntEmulateUndefined, Register scratch) {
+ MOZ_ASSERT(!initialized());
+ MOZ_ASSERT(ifEmulatesUndefined);
+ objreg_ = objreg;
+ scratch_ = scratch;
+ ifEmulatesUndefined_ = ifEmulatesUndefined;
+ ifDoesntEmulateUndefined_ = ifDoesntEmulateUndefined;
+ }
+};
+
+// A subclass of OutOfLineTestObject containing two extra labels, for use when
+// the ifTruthy/ifFalsy labels are needed in inline code as well as out-of-line
+// code. The user should bind these labels in inline code, and specify them as
+// targets via setInputAndTargets, as appropriate.
+class OutOfLineTestObjectWithLabels : public OutOfLineTestObject {
+ Label label1_;
+ Label label2_;
+
+ public:
+ OutOfLineTestObjectWithLabels() = default;
+
+ Label* label1() { return &label1_; }
+ Label* label2() { return &label2_; }
+};
+
+void CodeGenerator::testObjectEmulatesUndefinedKernel(
+ Register objreg, Label* ifEmulatesUndefined,
+ Label* ifDoesntEmulateUndefined, Register scratch,
+ OutOfLineTestObject* ool) {
+ ool->setInputAndTargets(objreg, ifEmulatesUndefined, ifDoesntEmulateUndefined,
+ scratch);
+
+ // Perform a fast-path check of the object's class flags if the object's
+ // not a proxy. Let out-of-line code handle the slow cases that require
+ // saving registers, making a function call, and restoring registers.
+ masm.branchIfObjectEmulatesUndefined(objreg, scratch, ool->entry(),
+ ifEmulatesUndefined);
+}
+
+void CodeGenerator::branchTestObjectEmulatesUndefined(
+ Register objreg, Label* ifEmulatesUndefined,
+ Label* ifDoesntEmulateUndefined, Register scratch,
+ OutOfLineTestObject* ool) {
+ MOZ_ASSERT(!ifDoesntEmulateUndefined->bound(),
+ "ifDoesntEmulateUndefined will be bound to the fallthrough path");
+
+ testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
+ ifDoesntEmulateUndefined, scratch, ool);
+ masm.bind(ifDoesntEmulateUndefined);
+}
+
+void CodeGenerator::testObjectEmulatesUndefined(Register objreg,
+ Label* ifEmulatesUndefined,
+ Label* ifDoesntEmulateUndefined,
+ Register scratch,
+ OutOfLineTestObject* ool) {
+ testObjectEmulatesUndefinedKernel(objreg, ifEmulatesUndefined,
+ ifDoesntEmulateUndefined, scratch, ool);
+ masm.jump(ifDoesntEmulateUndefined);
+}
+
+void CodeGenerator::testValueTruthyForType(
+ JSValueType type, ScratchTagScope& tag, const ValueOperand& value,
+ Register tempToUnbox, Register temp, FloatRegister floatTemp,
+ Label* ifTruthy, Label* ifFalsy, OutOfLineTestObject* ool,
+ bool skipTypeTest) {
+#ifdef DEBUG
+ if (skipTypeTest) {
+ Label expected;
+ masm.branchTestType(Assembler::Equal, tag, type, &expected);
+ masm.assumeUnreachable("Unexpected Value type in testValueTruthyForType");
+ masm.bind(&expected);
+ }
+#endif
+
+ // Handle irregular types first.
+ switch (type) {
+ case JSVAL_TYPE_UNDEFINED:
+ case JSVAL_TYPE_NULL:
+ // Undefined and null are falsy.
+ if (!skipTypeTest) {
+ masm.branchTestType(Assembler::Equal, tag, type, ifFalsy);
+ } else {
+ masm.jump(ifFalsy);
+ }
+ return;
+ case JSVAL_TYPE_SYMBOL:
+ // Symbols are truthy.
+ if (!skipTypeTest) {
+ masm.branchTestSymbol(Assembler::Equal, tag, ifTruthy);
+ } else {
+ masm.jump(ifTruthy);
+ }
+ return;
+ case JSVAL_TYPE_OBJECT: {
+ Label notObject;
+ if (!skipTypeTest) {
+ masm.branchTestObject(Assembler::NotEqual, tag, &notObject);
+ }
+ ScratchTagScopeRelease _(&tag);
+ Register objreg = masm.extractObject(value, tempToUnbox);
+ testObjectEmulatesUndefined(objreg, ifFalsy, ifTruthy, temp, ool);
+ masm.bind(&notObject);
+ return;
+ }
+ default:
+ break;
+ }
+
+ // Check the type of the value (unless this is the last possible type).
+ Label differentType;
+ if (!skipTypeTest) {
+ masm.branchTestType(Assembler::NotEqual, tag, type, &differentType);
+ }
+
+ // Branch if the value is falsy.
+ ScratchTagScopeRelease _(&tag);
+ switch (type) {
+ case JSVAL_TYPE_BOOLEAN: {
+ masm.branchTestBooleanTruthy(false, value, ifFalsy);
+ break;
+ }
+ case JSVAL_TYPE_INT32: {
+ masm.branchTestInt32Truthy(false, value, ifFalsy);
+ break;
+ }
+ case JSVAL_TYPE_STRING: {
+ masm.branchTestStringTruthy(false, value, ifFalsy);
+ break;
+ }
+ case JSVAL_TYPE_BIGINT: {
+ masm.branchTestBigIntTruthy(false, value, ifFalsy);
+ break;
+ }
+ case JSVAL_TYPE_DOUBLE: {
+ masm.unboxDouble(value, floatTemp);
+ masm.branchTestDoubleTruthy(false, floatTemp, ifFalsy);
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected value type");
+ }
+
+ // If we reach this point, the value is truthy. We fall through for
+ // truthy on the last test; otherwise, branch.
+ if (!skipTypeTest) {
+ masm.jump(ifTruthy);
+ }
+
+ masm.bind(&differentType);
+}
+
+void CodeGenerator::testValueTruthy(const ValueOperand& value,
+ Register tempToUnbox, Register temp,
+ FloatRegister floatTemp,
+ const TypeDataList& observedTypes,
+ Label* ifTruthy, Label* ifFalsy,
+ OutOfLineTestObject* ool) {
+ ScratchTagScope tag(masm, value);
+ masm.splitTagForTest(value, tag);
+
+ const std::initializer_list<JSValueType> defaultOrder = {
+ JSVAL_TYPE_UNDEFINED, JSVAL_TYPE_NULL, JSVAL_TYPE_BOOLEAN,
+ JSVAL_TYPE_INT32, JSVAL_TYPE_OBJECT, JSVAL_TYPE_STRING,
+ JSVAL_TYPE_DOUBLE, JSVAL_TYPE_SYMBOL, JSVAL_TYPE_BIGINT};
+
+ mozilla::EnumSet<JSValueType, uint32_t> remaining(defaultOrder);
+
+ // Generate tests for previously observed types first.
+ // The TypeDataList is sorted by descending frequency.
+ for (auto& observed : observedTypes) {
+ JSValueType type = observed.type();
+ remaining -= type;
+
+ testValueTruthyForType(type, tag, value, tempToUnbox, temp, floatTemp,
+ ifTruthy, ifFalsy, ool, /*skipTypeTest*/ false);
+ }
+
+ // Generate tests for remaining types.
+ for (auto type : defaultOrder) {
+ if (!remaining.contains(type)) {
+ continue;
+ }
+ remaining -= type;
+
+ // We don't need a type test for the last possible type.
+ bool skipTypeTest = remaining.isEmpty();
+ testValueTruthyForType(type, tag, value, tempToUnbox, temp, floatTemp,
+ ifTruthy, ifFalsy, ool, skipTypeTest);
+ }
+ MOZ_ASSERT(remaining.isEmpty());
+
+ // We fall through if the final test is truthy.
+}
+
+void CodeGenerator::visitTestBIAndBranch(LTestBIAndBranch* lir) {
+ Label* ifTrueLabel = getJumpLabelForBranch(lir->ifTrue());
+ Label* ifFalseLabel = getJumpLabelForBranch(lir->ifFalse());
+ Register input = ToRegister(lir->input());
+
+ if (isNextBlock(lir->ifFalse()->lir())) {
+ masm.branchIfBigIntIsNonZero(input, ifTrueLabel);
+ } else if (isNextBlock(lir->ifTrue()->lir())) {
+ masm.branchIfBigIntIsZero(input, ifFalseLabel);
+ } else {
+ masm.branchIfBigIntIsZero(input, ifFalseLabel);
+ jumpToBlock(lir->ifTrue());
+ }
+}
+
+void CodeGenerator::visitTestOAndBranch(LTestOAndBranch* lir) {
+ Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
+ Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
+ Register input = ToRegister(lir->input());
+
+ auto* ool = new (alloc()) OutOfLineTestObject();
+ addOutOfLineCode(ool, lir->mir());
+
+ testObjectEmulatesUndefined(input, falsy, truthy, ToRegister(lir->temp()),
+ ool);
+}
+
+void CodeGenerator::visitTestVAndBranch(LTestVAndBranch* lir) {
+ auto* ool = new (alloc()) OutOfLineTestObject();
+ addOutOfLineCode(ool, lir->mir());
+
+ Label* truthy = getJumpLabelForBranch(lir->ifTruthy());
+ Label* falsy = getJumpLabelForBranch(lir->ifFalsy());
+
+ ValueOperand input = ToValue(lir, LTestVAndBranch::Input);
+ Register tempToUnbox = ToTempUnboxRegister(lir->temp1());
+ Register temp = ToRegister(lir->temp2());
+ FloatRegister floatTemp = ToFloatRegister(lir->tempFloat());
+ const TypeDataList& observedTypes = lir->mir()->observedTypes();
+
+ testValueTruthy(input, tempToUnbox, temp, floatTemp, observedTypes, truthy,
+ falsy, ool);
+ masm.jump(truthy);
+}
+
+void CodeGenerator::visitBooleanToString(LBooleanToString* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ const JSAtomState& names = gen->runtime->names();
+ Label true_, done;
+
+ masm.branchTest32(Assembler::NonZero, input, input, &true_);
+ masm.movePtr(ImmGCPtr(names.false_), output);
+ masm.jump(&done);
+
+ masm.bind(&true_);
+ masm.movePtr(ImmGCPtr(names.true_), output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::emitIntToString(Register input, Register output,
+ Label* ool) {
+ masm.boundsCheck32PowerOfTwo(input, StaticStrings::INT_STATIC_LIMIT, ool);
+
+ // Fast path for small integers.
+ masm.movePtr(ImmPtr(&gen->runtime->staticStrings().intStaticTable), output);
+ masm.loadPtr(BaseIndex(output, input, ScalePointer), output);
+}
+
+void CodeGenerator::visitIntToString(LIntToString* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ using Fn = JSLinearString* (*)(JSContext*, int);
+ OutOfLineCode* ool = oolCallVM<Fn, Int32ToString<CanGC>>(
+ lir, ArgList(input), StoreRegisterTo(output));
+
+ emitIntToString(input, output, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitDoubleToString(LDoubleToString* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register temp = ToRegister(lir->temp0());
+ Register output = ToRegister(lir->output());
+
+ using Fn = JSString* (*)(JSContext*, double);
+ OutOfLineCode* ool = oolCallVM<Fn, NumberToString<CanGC>>(
+ lir, ArgList(input), StoreRegisterTo(output));
+
+ // Try double to integer conversion and run integer to string code.
+ masm.convertDoubleToInt32(input, temp, ool->entry(), false);
+ emitIntToString(temp, output, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitValueToString(LValueToString* lir) {
+ ValueOperand input = ToValue(lir, LValueToString::InputIndex);
+ Register output = ToRegister(lir->output());
+
+ using Fn = JSString* (*)(JSContext*, HandleValue);
+ OutOfLineCode* ool = oolCallVM<Fn, ToStringSlow<CanGC>>(
+ lir, ArgList(input), StoreRegisterTo(output));
+
+ Label done;
+ Register tag = masm.extractTag(input, output);
+ const JSAtomState& names = gen->runtime->names();
+
+ // String
+ {
+ Label notString;
+ masm.branchTestString(Assembler::NotEqual, tag, &notString);
+ masm.unboxString(input, output);
+ masm.jump(&done);
+ masm.bind(&notString);
+ }
+
+ // Integer
+ {
+ Label notInteger;
+ masm.branchTestInt32(Assembler::NotEqual, tag, &notInteger);
+ Register unboxed = ToTempUnboxRegister(lir->temp0());
+ unboxed = masm.extractInt32(input, unboxed);
+ emitIntToString(unboxed, output, ool->entry());
+ masm.jump(&done);
+ masm.bind(&notInteger);
+ }
+
+ // Double
+ {
+ // Note: no fastpath. Need two extra registers and can only convert doubles
+ // that fit integers and are smaller than StaticStrings::INT_STATIC_LIMIT.
+ masm.branchTestDouble(Assembler::Equal, tag, ool->entry());
+ }
+
+ // Undefined
+ {
+ Label notUndefined;
+ masm.branchTestUndefined(Assembler::NotEqual, tag, &notUndefined);
+ masm.movePtr(ImmGCPtr(names.undefined), output);
+ masm.jump(&done);
+ masm.bind(&notUndefined);
+ }
+
+ // Null
+ {
+ Label notNull;
+ masm.branchTestNull(Assembler::NotEqual, tag, &notNull);
+ masm.movePtr(ImmGCPtr(names.null), output);
+ masm.jump(&done);
+ masm.bind(&notNull);
+ }
+
+ // Boolean
+ {
+ Label notBoolean, true_;
+ masm.branchTestBoolean(Assembler::NotEqual, tag, &notBoolean);
+ masm.branchTestBooleanTruthy(true, input, &true_);
+ masm.movePtr(ImmGCPtr(names.false_), output);
+ masm.jump(&done);
+ masm.bind(&true_);
+ masm.movePtr(ImmGCPtr(names.true_), output);
+ masm.jump(&done);
+ masm.bind(&notBoolean);
+ }
+
+ // Objects/symbols are only possible when |mir->mightHaveSideEffects()|.
+ if (lir->mir()->mightHaveSideEffects()) {
+ // Object
+ if (lir->mir()->supportSideEffects()) {
+ masm.branchTestObject(Assembler::Equal, tag, ool->entry());
+ } else {
+ // Bail.
+ MOZ_ASSERT(lir->mir()->needsSnapshot());
+ Label bail;
+ masm.branchTestObject(Assembler::Equal, tag, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+ }
+
+ // Symbol
+ if (lir->mir()->supportSideEffects()) {
+ masm.branchTestSymbol(Assembler::Equal, tag, ool->entry());
+ } else {
+ // Bail.
+ MOZ_ASSERT(lir->mir()->needsSnapshot());
+ Label bail;
+ masm.branchTestSymbol(Assembler::Equal, tag, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+ }
+ }
+
+ // BigInt
+ {
+ // No fastpath currently implemented.
+ masm.branchTestBigInt(Assembler::Equal, tag, ool->entry());
+ }
+
+ masm.assumeUnreachable("Unexpected type for LValueToString.");
+
+ masm.bind(&done);
+ masm.bind(ool->rejoin());
+}
+
+using StoreBufferMutationFn = void (*)(js::gc::StoreBuffer*, js::gc::Cell**);
+
+static void EmitStoreBufferMutation(MacroAssembler& masm, Register holder,
+ size_t offset, Register buffer,
+ LiveGeneralRegisterSet& liveVolatiles,
+ StoreBufferMutationFn fun) {
+ Label callVM;
+ Label exit;
+
+ // Call into the VM to barrier the write. The only registers that need to
+ // be preserved are those in liveVolatiles, so once they are saved on the
+ // stack all volatile registers are available for use.
+ masm.bind(&callVM);
+ masm.PushRegsInMask(liveVolatiles);
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
+ regs.takeUnchecked(buffer);
+ regs.takeUnchecked(holder);
+ Register addrReg = regs.takeAny();
+
+ masm.computeEffectiveAddress(Address(holder, offset), addrReg);
+
+ bool needExtraReg = !regs.hasAny<GeneralRegisterSet::DefaultType>();
+ if (needExtraReg) {
+ masm.push(holder);
+ masm.setupUnalignedABICall(holder);
+ } else {
+ masm.setupUnalignedABICall(regs.takeAny());
+ }
+ masm.passABIArg(buffer);
+ masm.passABIArg(addrReg);
+ masm.callWithABI(DynamicFunction<StoreBufferMutationFn>(fun), MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ if (needExtraReg) {
+ masm.pop(holder);
+ }
+ masm.PopRegsInMask(liveVolatiles);
+ masm.bind(&exit);
+}
+
+// Warning: this function modifies prev and next.
+static void EmitPostWriteBarrierS(MacroAssembler& masm, Register holder,
+ size_t offset, Register prev, Register next,
+ LiveGeneralRegisterSet& liveVolatiles) {
+ Label exit;
+ Label checkRemove, putCell;
+
+ // if (next && (buffer = next->storeBuffer()))
+ // but we never pass in nullptr for next.
+ Register storebuffer = next;
+ masm.loadStoreBuffer(next, storebuffer);
+ masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &checkRemove);
+
+ // if (prev && prev->storeBuffer())
+ masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &putCell);
+ masm.loadStoreBuffer(prev, prev);
+ masm.branchPtr(Assembler::NotEqual, prev, ImmWord(0), &exit);
+
+ // buffer->putCell(cellp)
+ masm.bind(&putCell);
+ EmitStoreBufferMutation(masm, holder, offset, storebuffer, liveVolatiles,
+ JSString::addCellAddressToStoreBuffer);
+ masm.jump(&exit);
+
+ // if (prev && (buffer = prev->storeBuffer()))
+ masm.bind(&checkRemove);
+ masm.branchPtr(Assembler::Equal, prev, ImmWord(0), &exit);
+ masm.loadStoreBuffer(prev, storebuffer);
+ masm.branchPtr(Assembler::Equal, storebuffer, ImmWord(0), &exit);
+ EmitStoreBufferMutation(masm, holder, offset, storebuffer, liveVolatiles,
+ JSString::removeCellAddressFromStoreBuffer);
+
+ masm.bind(&exit);
+}
+
+void CodeGenerator::visitRegExp(LRegExp* lir) {
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+ JSObject* source = lir->mir()->source();
+
+ using Fn = JSObject* (*)(JSContext*, Handle<RegExpObject*>);
+ OutOfLineCode* ool = oolCallVM<Fn, CloneRegExpObject>(
+ lir, ArgList(ImmGCPtr(source)), StoreRegisterTo(output));
+ if (lir->mir()->hasShared()) {
+ TemplateObject templateObject(source);
+ masm.createGCObject(output, temp, templateObject, gc::DefaultHeap,
+ ool->entry());
+ } else {
+ masm.jump(ool->entry());
+ }
+ masm.bind(ool->rejoin());
+}
+
+static constexpr size_t InputOutputDataSize = sizeof(irregexp::InputOutputData);
+
+// Amount of space to reserve on the stack when executing RegExps inline.
+static constexpr size_t RegExpReservedStack =
+ InputOutputDataSize + sizeof(MatchPairs) +
+ RegExpObject::MaxPairCount * sizeof(MatchPair);
+
+static constexpr int32_t RegExpPairsVectorStartOffset(
+ int32_t inputOutputDataStartOffset) {
+ return inputOutputDataStartOffset + int32_t(InputOutputDataSize) +
+ int32_t(sizeof(MatchPairs));
+}
+
+static Address RegExpPairCountAddress(MacroAssembler& masm,
+ int32_t inputOutputDataStartOffset) {
+ return Address(FramePointer, inputOutputDataStartOffset +
+ int32_t(InputOutputDataSize) +
+ MatchPairs::offsetOfPairCount());
+}
+
+// When the unicode flag is set, if lastIndex points to a trail
+// surrogate, we should step back to the corresponding lead surrogate.
+// See ExecuteRegExp in builtin/RegExp.cpp for more detail.
+static void StepBackToLeadSurrogate(MacroAssembler& masm, Register regexpShared,
+ Register input, Register lastIndex,
+ Register temp1, Register temp2) {
+ Label done;
+
+ // If the unicode flag is not set, there is nothing to do.
+ masm.branchTest32(Assembler::Zero,
+ Address(regexpShared, RegExpShared::offsetOfFlags()),
+ Imm32(int32_t(JS::RegExpFlag::Unicode)), &done);
+
+ // If the input is latin1, there can't be any surrogates.
+ masm.branchLatin1String(input, &done);
+
+ // Check if |lastIndex > 0 && lastIndex < input->length()|.
+ // lastIndex should already have no sign here.
+ masm.branchTest32(Assembler::Zero, lastIndex, lastIndex, &done);
+ masm.loadStringLength(input, temp1);
+ masm.branch32(Assembler::AboveOrEqual, lastIndex, temp1, &done);
+
+ // For TrailSurrogateMin ≤ x ≤ TrailSurrogateMax and
+ // LeadSurrogateMin ≤ x ≤ LeadSurrogateMax, the following
+ // equations hold.
+ //
+ // SurrogateMin ≤ x ≤ SurrogateMax
+ // <> SurrogateMin ≤ x ≤ SurrogateMin + 2^10 - 1
+ // <> ((x - SurrogateMin) >>> 10) = 0 where >>> is an unsigned-shift
+ // See Hacker's Delight, section 4-1 for details.
+ //
+ // ((x - SurrogateMin) >>> 10) = 0
+ // <> floor((x - SurrogateMin) / 1024) = 0
+ // <> floor((x / 1024) - (SurrogateMin / 1024)) = 0
+ // <> floor(x / 1024) = SurrogateMin / 1024
+ // <> floor(x / 1024) * 1024 = SurrogateMin
+ // <> (x >>> 10) << 10 = SurrogateMin
+ // <> x & ~(2^10 - 1) = SurrogateMin
+
+ constexpr char16_t SurrogateMask = 0xFC00;
+
+ Register charsReg = temp1;
+ masm.loadStringChars(input, charsReg, CharEncoding::TwoByte);
+
+ // Check if input[lastIndex] is trail surrogate.
+ masm.loadChar(charsReg, lastIndex, temp2, CharEncoding::TwoByte);
+ masm.and32(Imm32(SurrogateMask), temp2);
+ masm.branch32(Assembler::NotEqual, temp2, Imm32(unicode::TrailSurrogateMin),
+ &done);
+
+ // Check if input[lastIndex-1] is lead surrogate.
+ masm.loadChar(charsReg, lastIndex, temp2, CharEncoding::TwoByte,
+ -int32_t(sizeof(char16_t)));
+ masm.and32(Imm32(SurrogateMask), temp2);
+ masm.branch32(Assembler::NotEqual, temp2, Imm32(unicode::LeadSurrogateMin),
+ &done);
+
+ // Move lastIndex back to lead surrogate.
+ masm.sub32(Imm32(1), lastIndex);
+
+ masm.bind(&done);
+}
+
+static void UpdateRegExpStatics(MacroAssembler& masm, Register regexp,
+ Register input, Register lastIndex,
+ Register staticsReg, Register temp1,
+ Register temp2,
+ gc::InitialHeap initialStringHeap,
+ LiveGeneralRegisterSet& volatileRegs) {
+ Address pendingInputAddress(staticsReg,
+ RegExpStatics::offsetOfPendingInput());
+ Address matchesInputAddress(staticsReg,
+ RegExpStatics::offsetOfMatchesInput());
+ Address lazySourceAddress(staticsReg, RegExpStatics::offsetOfLazySource());
+ Address lazyIndexAddress(staticsReg, RegExpStatics::offsetOfLazyIndex());
+
+ masm.guardedCallPreBarrier(pendingInputAddress, MIRType::String);
+ masm.guardedCallPreBarrier(matchesInputAddress, MIRType::String);
+ masm.guardedCallPreBarrier(lazySourceAddress, MIRType::String);
+
+ if (initialStringHeap == gc::DefaultHeap) {
+ // Writing into RegExpStatics tenured memory; must post-barrier.
+ if (staticsReg.volatile_()) {
+ volatileRegs.add(staticsReg);
+ }
+
+ masm.loadPtr(pendingInputAddress, temp1);
+ masm.storePtr(input, pendingInputAddress);
+ masm.movePtr(input, temp2);
+ EmitPostWriteBarrierS(masm, staticsReg,
+ RegExpStatics::offsetOfPendingInput(),
+ temp1 /* prev */, temp2 /* next */, volatileRegs);
+
+ masm.loadPtr(matchesInputAddress, temp1);
+ masm.storePtr(input, matchesInputAddress);
+ masm.movePtr(input, temp2);
+ EmitPostWriteBarrierS(masm, staticsReg,
+ RegExpStatics::offsetOfMatchesInput(),
+ temp1 /* prev */, temp2 /* next */, volatileRegs);
+ } else {
+ masm.storePtr(input, pendingInputAddress);
+ masm.storePtr(input, matchesInputAddress);
+ }
+
+ masm.storePtr(lastIndex,
+ Address(staticsReg, RegExpStatics::offsetOfLazyIndex()));
+ masm.store32(
+ Imm32(1),
+ Address(staticsReg, RegExpStatics::offsetOfPendingLazyEvaluation()));
+
+ masm.unboxNonDouble(Address(regexp, NativeObject::getFixedSlotOffset(
+ RegExpObject::SHARED_SLOT)),
+ temp1, JSVAL_TYPE_PRIVATE_GCTHING);
+ masm.loadPtr(Address(temp1, RegExpShared::offsetOfSource()), temp2);
+ masm.storePtr(temp2, lazySourceAddress);
+ static_assert(sizeof(JS::RegExpFlags) == 1, "load size must match flag size");
+ masm.load8ZeroExtend(Address(temp1, RegExpShared::offsetOfFlags()), temp2);
+ masm.store8(temp2, Address(staticsReg, RegExpStatics::offsetOfLazyFlags()));
+}
+
+// Prepare an InputOutputData and optional MatchPairs which space has been
+// allocated for on the stack, and try to execute a RegExp on a string input.
+// If the RegExp was successfully executed and matched the input, fallthrough.
+// Otherwise, jump to notFound or failure.
+//
+// inputOutputDataStartOffset is the offset relative to the frame pointer
+// register. This offset is negative for the RegExpTester stub.
+static bool PrepareAndExecuteRegExp(JSContext* cx, MacroAssembler& masm,
+ Register regexp, Register input,
+ Register lastIndex, Register temp1,
+ Register temp2, Register temp3,
+ int32_t inputOutputDataStartOffset,
+ gc::InitialHeap initialStringHeap,
+ Label* notFound, Label* failure) {
+ JitSpew(JitSpew_Codegen, "# Emitting PrepareAndExecuteRegExp");
+
+ using irregexp::InputOutputData;
+
+ /*
+ * [SMDOC] Stack layout for PrepareAndExecuteRegExp
+ *
+ * Before this function is called, the caller is responsible for
+ * allocating enough stack space for the following data:
+ *
+ * inputOutputDataStartOffset +-----> +---------------+
+ * |InputOutputData|
+ * inputStartAddress +----------> inputStart|
+ * inputEndAddress +----------> inputEnd|
+ * startIndexAddress +----------> startIndex|
+ * matchesAddress +----------> matches|-----+
+ * +---------------+ |
+ * matchPairs(Address|Offset) +-----> +---------------+ <--+
+ * | MatchPairs |
+ * pairCountAddress +----------> count |
+ * pairsPointerAddress +----------> pairs |-----+
+ * +---------------+ |
+ * pairsArray(Address|Offset) +-----> +---------------+ <--+
+ * | MatchPair |
+ * firstMatchStartAddress +----------> start | <--+
+ * | limit | |
+ * +---------------+ |
+ * . |
+ * . Reserved space for
+ * . RegExpObject::MaxPairCount
+ * . MatchPair objects
+ * . |
+ * +---------------+ |
+ * | MatchPair | |
+ * | start | |
+ * | limit | <--+
+ * +---------------+
+ */
+
+ int32_t ioOffset = inputOutputDataStartOffset;
+ int32_t matchPairsOffset = ioOffset + int32_t(sizeof(InputOutputData));
+ int32_t pairsArrayOffset = matchPairsOffset + int32_t(sizeof(MatchPairs));
+
+ Address inputStartAddress(FramePointer,
+ ioOffset + InputOutputData::offsetOfInputStart());
+ Address inputEndAddress(FramePointer,
+ ioOffset + InputOutputData::offsetOfInputEnd());
+ Address startIndexAddress(FramePointer,
+ ioOffset + InputOutputData::offsetOfStartIndex());
+ Address matchesAddress(FramePointer,
+ ioOffset + InputOutputData::offsetOfMatches());
+
+ Address matchPairsAddress(FramePointer, matchPairsOffset);
+ Address pairCountAddress(FramePointer,
+ matchPairsOffset + MatchPairs::offsetOfPairCount());
+ Address pairsPointerAddress(FramePointer,
+ matchPairsOffset + MatchPairs::offsetOfPairs());
+
+ Address pairsArrayAddress(FramePointer, pairsArrayOffset);
+ Address firstMatchStartAddress(FramePointer,
+ pairsArrayOffset + MatchPair::offsetOfStart());
+
+ // First, fill in a skeletal MatchPairs instance on the stack. This will be
+ // passed to the OOL stub in the caller if we aren't able to execute the
+ // RegExp inline, and that stub needs to be able to determine whether the
+ // execution finished successfully.
+
+ // Initialize MatchPairs::pairCount to 1. The correct value can only
+ // be determined after loading the RegExpShared. If the RegExpShared
+ // has Kind::Atom, this is the correct pairCount.
+ masm.store32(Imm32(1), pairCountAddress);
+
+ // Initialize MatchPairs::pairs pointer
+ masm.computeEffectiveAddress(pairsArrayAddress, temp1);
+ masm.storePtr(temp1, pairsPointerAddress);
+
+ // Initialize MatchPairs::pairs[0]::start to MatchPair::NoMatch
+ masm.store32(Imm32(MatchPair::NoMatch), firstMatchStartAddress);
+
+ // Check for a linear input string.
+ masm.branchIfRope(input, failure);
+
+ // Load the RegExpShared.
+ Register regexpReg = temp1;
+ Address sharedSlot = Address(
+ regexp, NativeObject::getFixedSlotOffset(RegExpObject::SHARED_SLOT));
+ masm.branchTestUndefined(Assembler::Equal, sharedSlot, failure);
+ masm.unboxNonDouble(sharedSlot, regexpReg, JSVAL_TYPE_PRIVATE_GCTHING);
+
+ // Handle Atom matches
+ Label notAtom, checkSuccess;
+ masm.branchPtr(Assembler::Equal,
+ Address(regexpReg, RegExpShared::offsetOfPatternAtom()),
+ ImmWord(0), &notAtom);
+ {
+ LiveGeneralRegisterSet regsToSave(GeneralRegisterSet::Volatile());
+ regsToSave.takeUnchecked(temp1);
+ regsToSave.takeUnchecked(temp2);
+ regsToSave.takeUnchecked(temp3);
+
+ masm.computeEffectiveAddress(matchPairsAddress, temp3);
+
+ masm.PushRegsInMask(regsToSave);
+ using Fn = RegExpRunStatus (*)(RegExpShared * re, JSLinearString * input,
+ size_t start, MatchPairs * matchPairs);
+ masm.setupUnalignedABICall(temp2);
+ masm.passABIArg(regexpReg);
+ masm.passABIArg(input);
+ masm.passABIArg(lastIndex);
+ masm.passABIArg(temp3);
+ masm.callWithABI<Fn, js::ExecuteRegExpAtomRaw>();
+
+ masm.storeCallInt32Result(temp1);
+ masm.PopRegsInMask(regsToSave);
+
+ masm.jump(&checkSuccess);
+ }
+ masm.bind(&notAtom);
+
+ // Don't handle regexps with too many capture pairs.
+ masm.load32(Address(regexpReg, RegExpShared::offsetOfPairCount()), temp2);
+ masm.branch32(Assembler::Above, temp2, Imm32(RegExpObject::MaxPairCount),
+ failure);
+
+ // Fill in the pair count in the MatchPairs on the stack.
+ masm.store32(temp2, pairCountAddress);
+
+ // Update lastIndex if necessary.
+ StepBackToLeadSurrogate(masm, regexpReg, input, lastIndex, temp2, temp3);
+
+ // Load code pointer and length of input (in bytes).
+ // Store the input start in the InputOutputData.
+ Register codePointer = temp1; // Note: temp1 was previously regexpReg.
+ Register byteLength = temp3;
+ {
+ Label isLatin1, done;
+ masm.loadStringLength(input, byteLength);
+
+ masm.branchLatin1String(input, &isLatin1);
+
+ // Two-byte input
+ masm.loadStringChars(input, temp2, CharEncoding::TwoByte);
+ masm.storePtr(temp2, inputStartAddress);
+ masm.loadPtr(
+ Address(regexpReg, RegExpShared::offsetOfJitCode(/*latin1 =*/false)),
+ codePointer);
+ masm.lshiftPtr(Imm32(1), byteLength);
+ masm.jump(&done);
+
+ // Latin1 input
+ masm.bind(&isLatin1);
+ masm.loadStringChars(input, temp2, CharEncoding::Latin1);
+ masm.storePtr(temp2, inputStartAddress);
+ masm.loadPtr(
+ Address(regexpReg, RegExpShared::offsetOfJitCode(/*latin1 =*/true)),
+ codePointer);
+
+ masm.bind(&done);
+
+ // Store end pointer
+ masm.addPtr(byteLength, temp2);
+ masm.storePtr(temp2, inputEndAddress);
+ }
+
+ // Guard that the RegExpShared has been compiled for this type of input.
+ // If it has not been compiled, we fall back to the OOL case, which will
+ // do a VM call into the interpreter.
+ // TODO: add an interpreter trampoline?
+ masm.branchPtr(Assembler::Equal, codePointer, ImmWord(0), failure);
+ masm.loadPtr(Address(codePointer, JitCode::offsetOfCode()), codePointer);
+
+ // Finish filling in the InputOutputData instance on the stack
+ masm.computeEffectiveAddress(matchPairsAddress, temp2);
+ masm.storePtr(temp2, matchesAddress);
+ masm.storePtr(lastIndex, startIndexAddress);
+
+ // Save any volatile inputs.
+ LiveGeneralRegisterSet volatileRegs;
+ if (lastIndex.volatile_()) {
+ volatileRegs.add(lastIndex);
+ }
+ if (input.volatile_()) {
+ volatileRegs.add(input);
+ }
+ if (regexp.volatile_()) {
+ volatileRegs.add(regexp);
+ }
+
+ // Execute the RegExp.
+ masm.computeEffectiveAddress(
+ Address(FramePointer, inputOutputDataStartOffset), temp2);
+ masm.PushRegsInMask(volatileRegs);
+ masm.setupUnalignedABICall(temp3);
+ masm.passABIArg(temp2);
+ masm.callWithABI(codePointer);
+ masm.storeCallInt32Result(temp1);
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.bind(&checkSuccess);
+ masm.branch32(Assembler::Equal, temp1,
+ Imm32(RegExpRunStatus_Success_NotFound), notFound);
+ masm.branch32(Assembler::Equal, temp1, Imm32(RegExpRunStatus_Error), failure);
+
+ // Lazily update the RegExpStatics.
+ RegExpStatics* res = GlobalObject::getRegExpStatics(cx, cx->global());
+ if (!res) {
+ return false;
+ }
+ masm.movePtr(ImmPtr(res), temp1);
+ UpdateRegExpStatics(masm, regexp, input, lastIndex, temp1, temp2, temp3,
+ initialStringHeap, volatileRegs);
+
+ return true;
+}
+
+static void CopyStringChars(MacroAssembler& masm, Register to, Register from,
+ Register len, Register byteOpScratch,
+ CharEncoding encoding);
+
+class CreateDependentString {
+ CharEncoding encoding_;
+ Register string_;
+ Register temp1_;
+ Register temp2_;
+ Label* failure_;
+
+ enum class FallbackKind : uint8_t {
+ InlineString,
+ FatInlineString,
+ NotInlineString,
+ Count
+ };
+ mozilla::EnumeratedArray<FallbackKind, FallbackKind::Count, Label> fallbacks_,
+ joins_;
+
+ public:
+ CreateDependentString(CharEncoding encoding, Register string, Register temp1,
+ Register temp2, Label* failure)
+ : encoding_(encoding),
+ string_(string),
+ temp1_(temp1),
+ temp2_(temp2),
+ failure_(failure) {}
+
+ Register string() const { return string_; }
+ CharEncoding encoding() const { return encoding_; }
+
+ // Generate code that creates DependentString.
+ // Caller should call generateFallback after masm.ret(), to generate
+ // fallback path.
+ void generate(MacroAssembler& masm, const JSAtomState& names,
+ CompileRuntime* runtime, Register base,
+ BaseIndex startIndexAddress, BaseIndex limitIndexAddress,
+ gc::InitialHeap initialStringHeap);
+
+ // Generate fallback path for creating DependentString.
+ void generateFallback(MacroAssembler& masm);
+};
+
+void CreateDependentString::generate(MacroAssembler& masm,
+ const JSAtomState& names,
+ CompileRuntime* runtime, Register base,
+ BaseIndex startIndexAddress,
+ BaseIndex limitIndexAddress,
+ gc::InitialHeap initialStringHeap) {
+ JitSpew(JitSpew_Codegen, "# Emitting CreateDependentString (encoding=%s)",
+ (encoding_ == CharEncoding::Latin1 ? "Latin-1" : "Two-Byte"));
+
+ auto newGCString = [&](FallbackKind kind) {
+ uint32_t flags = kind == FallbackKind::InlineString
+ ? JSString::INIT_THIN_INLINE_FLAGS
+ : kind == FallbackKind::FatInlineString
+ ? JSString::INIT_FAT_INLINE_FLAGS
+ : JSString::INIT_DEPENDENT_FLAGS;
+ if (encoding_ == CharEncoding::Latin1) {
+ flags |= JSString::LATIN1_CHARS_BIT;
+ }
+
+ if (kind != FallbackKind::FatInlineString) {
+ masm.newGCString(string_, temp2_, initialStringHeap, &fallbacks_[kind]);
+ } else {
+ masm.newGCFatInlineString(string_, temp2_, initialStringHeap,
+ &fallbacks_[kind]);
+ }
+ masm.bind(&joins_[kind]);
+ masm.store32(Imm32(flags), Address(string_, JSString::offsetOfFlags()));
+ };
+
+ // Compute the string length.
+ masm.load32(startIndexAddress, temp2_);
+ masm.load32(limitIndexAddress, temp1_);
+ masm.sub32(temp2_, temp1_);
+
+ Label done, nonEmpty;
+
+ // Zero length matches use the empty string.
+ masm.branchTest32(Assembler::NonZero, temp1_, temp1_, &nonEmpty);
+ masm.movePtr(ImmGCPtr(names.empty), string_);
+ masm.jump(&done);
+
+ masm.bind(&nonEmpty);
+
+ // Complete matches use the base string.
+ Label nonBaseStringMatch;
+ masm.branchTest32(Assembler::NonZero, temp2_, temp2_, &nonBaseStringMatch);
+ masm.branch32(Assembler::NotEqual, Address(base, JSString::offsetOfLength()),
+ temp1_, &nonBaseStringMatch);
+ masm.movePtr(base, string_);
+ masm.jump(&done);
+
+ masm.bind(&nonBaseStringMatch);
+
+ Label notInline;
+
+ int32_t maxInlineLength = encoding_ == CharEncoding::Latin1
+ ? JSFatInlineString::MAX_LENGTH_LATIN1
+ : JSFatInlineString::MAX_LENGTH_TWO_BYTE;
+ masm.branch32(Assembler::Above, temp1_, Imm32(maxInlineLength), &notInline);
+ {
+ // Make a thin or fat inline string.
+ Label stringAllocated, fatInline;
+
+ int32_t maxThinInlineLength = encoding_ == CharEncoding::Latin1
+ ? JSThinInlineString::MAX_LENGTH_LATIN1
+ : JSThinInlineString::MAX_LENGTH_TWO_BYTE;
+ masm.branch32(Assembler::Above, temp1_, Imm32(maxThinInlineLength),
+ &fatInline);
+ if (encoding_ == CharEncoding::Latin1) {
+ // One character Latin-1 strings can be loaded directly from the
+ // static strings table.
+ Label thinInline;
+ masm.branch32(Assembler::Above, temp1_, Imm32(1), &thinInline);
+ {
+ static_assert(
+ StaticStrings::UNIT_STATIC_LIMIT - 1 == JSString::MAX_LATIN1_CHAR,
+ "Latin-1 strings can be loaded from static strings");
+
+ masm.loadStringChars(base, temp1_, encoding_);
+ masm.loadChar(temp1_, temp2_, temp1_, encoding_);
+
+ masm.movePtr(ImmPtr(&runtime->staticStrings().unitStaticTable),
+ string_);
+ masm.loadPtr(BaseIndex(string_, temp1_, ScalePointer), string_);
+
+ masm.jump(&done);
+ }
+ masm.bind(&thinInline);
+ }
+ {
+ newGCString(FallbackKind::InlineString);
+ masm.jump(&stringAllocated);
+ }
+ masm.bind(&fatInline);
+ { newGCString(FallbackKind::FatInlineString); }
+ masm.bind(&stringAllocated);
+
+ masm.store32(temp1_, Address(string_, JSString::offsetOfLength()));
+
+ masm.push(string_);
+ masm.push(base);
+
+ MOZ_ASSERT(startIndexAddress.base == FramePointer,
+ "startIndexAddress is still valid after stack pushes");
+
+ // Load chars pointer for the new string.
+ masm.loadInlineStringCharsForStore(string_, string_);
+
+ // Load the source characters pointer.
+ masm.loadStringChars(base, temp2_, encoding_);
+ masm.load32(startIndexAddress, base);
+ masm.addToCharPtr(temp2_, base, encoding_);
+
+ CopyStringChars(masm, string_, temp2_, temp1_, base, encoding_);
+
+ masm.pop(base);
+ masm.pop(string_);
+
+ masm.jump(&done);
+ }
+
+ masm.bind(&notInline);
+
+ {
+ // Make a dependent string.
+ // Warning: string may be tenured (if the fallback case is hit), so
+ // stores into it must be post barriered.
+ newGCString(FallbackKind::NotInlineString);
+
+ masm.store32(temp1_, Address(string_, JSString::offsetOfLength()));
+
+ masm.loadNonInlineStringChars(base, temp1_, encoding_);
+ masm.load32(startIndexAddress, temp2_);
+ masm.addToCharPtr(temp1_, temp2_, encoding_);
+ masm.storeNonInlineStringChars(temp1_, string_);
+ masm.storeDependentStringBase(base, string_);
+ masm.movePtr(base, temp1_);
+
+ // Follow any base pointer if the input is itself a dependent string.
+ // Watch for undepended strings, which have a base pointer but don't
+ // actually share their characters with it.
+ Label noBase;
+ masm.load32(Address(base, JSString::offsetOfFlags()), temp2_);
+ masm.and32(Imm32(JSString::TYPE_FLAGS_MASK), temp2_);
+ masm.branchTest32(Assembler::Zero, temp2_, Imm32(JSString::DEPENDENT_BIT),
+ &noBase);
+ masm.loadDependentStringBase(base, temp1_);
+ masm.storeDependentStringBase(temp1_, string_);
+ masm.bind(&noBase);
+
+ // Post-barrier the base store, whether it was the direct or indirect
+ // base (both will end up in temp1 here).
+ masm.branchPtrInNurseryChunk(Assembler::Equal, string_, temp2_, &done);
+ masm.branchPtrInNurseryChunk(Assembler::NotEqual, temp1_, temp2_, &done);
+
+ LiveRegisterSet regsToSave(RegisterSet::Volatile());
+ regsToSave.takeUnchecked(temp1_);
+ regsToSave.takeUnchecked(temp2_);
+
+ masm.PushRegsInMask(regsToSave);
+
+ masm.mov(ImmPtr(runtime), temp1_);
+
+ using Fn = void (*)(JSRuntime * rt, js::gc::Cell * cell);
+ masm.setupUnalignedABICall(temp2_);
+ masm.passABIArg(temp1_);
+ masm.passABIArg(string_);
+ masm.callWithABI<Fn, PostWriteBarrier>();
+
+ masm.PopRegsInMask(regsToSave);
+ }
+
+ masm.bind(&done);
+}
+
+void CreateDependentString::generateFallback(MacroAssembler& masm) {
+ JitSpew(JitSpew_Codegen,
+ "# Emitting CreateDependentString fallback (encoding=%s)",
+ (encoding_ == CharEncoding::Latin1 ? "Latin-1" : "Two-Byte"));
+
+ LiveRegisterSet regsToSave(RegisterSet::Volatile());
+ regsToSave.takeUnchecked(string_);
+ regsToSave.takeUnchecked(temp2_);
+
+ for (FallbackKind kind : mozilla::MakeEnumeratedRange(FallbackKind::Count)) {
+ masm.bind(&fallbacks_[kind]);
+
+ masm.PushRegsInMask(regsToSave);
+
+ using Fn = void* (*)(JSContext * cx);
+ masm.setupUnalignedABICall(string_);
+ masm.loadJSContext(string_);
+ masm.passABIArg(string_);
+ if (kind == FallbackKind::FatInlineString) {
+ masm.callWithABI<Fn, AllocateFatInlineString>();
+ } else {
+ masm.callWithABI<Fn, AllocateDependentString>();
+ }
+ masm.storeCallPointerResult(string_);
+
+ masm.PopRegsInMask(regsToSave);
+
+ masm.branchPtr(Assembler::Equal, string_, ImmWord(0), failure_);
+
+ masm.jump(&joins_[kind]);
+ }
+}
+
+static void CreateMatchResultFallback(MacroAssembler& masm, Register object,
+ Register temp1, Register temp2,
+ const TemplateObject& templateObject,
+ Label* fail) {
+ JitSpew(JitSpew_Codegen, "# Emitting CreateMatchResult fallback");
+
+ MOZ_ASSERT(templateObject.isArrayObject());
+
+ LiveRegisterSet regsToSave(RegisterSet::Volatile());
+ regsToSave.takeUnchecked(object);
+ regsToSave.takeUnchecked(temp1);
+ regsToSave.takeUnchecked(temp2);
+
+ masm.PushRegsInMask(regsToSave);
+
+ using Fn =
+ void* (*)(JSContext * cx, gc::AllocKind kind, size_t nDynamicSlots);
+ masm.setupUnalignedABICall(object);
+
+ masm.loadJSContext(object);
+ masm.passABIArg(object);
+ masm.move32(Imm32(int32_t(templateObject.getAllocKind())), temp1);
+ masm.passABIArg(temp1);
+ masm.move32(
+ Imm32(int32_t(templateObject.asTemplateNativeObject().numDynamicSlots())),
+ temp2);
+ masm.passABIArg(temp2);
+ masm.callWithABI<Fn, CreateMatchResultFallbackFunc>();
+ masm.storeCallPointerResult(object);
+
+ masm.PopRegsInMask(regsToSave);
+
+ masm.branchPtr(Assembler::Equal, object, ImmWord(0), fail);
+
+ masm.initGCThing(object, temp1, templateObject);
+}
+
+JitCode* JitRealm::generateRegExpMatcherStub(JSContext* cx) {
+ JitSpew(JitSpew_Codegen, "# Emitting RegExpMatcher stub");
+
+ Register regexp = RegExpMatcherRegExpReg;
+ Register input = RegExpMatcherStringReg;
+ Register lastIndex = RegExpMatcherLastIndexReg;
+ ValueOperand result = JSReturnOperand;
+
+ // We are free to clobber all registers, as LRegExpMatcher is a call
+ // instruction.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(input);
+ regs.take(regexp);
+ regs.take(lastIndex);
+
+ Register temp1 = regs.takeAny();
+ Register temp2 = regs.takeAny();
+ Register temp3 = regs.takeAny();
+ Register maybeTemp4 = InvalidReg;
+ if (!regs.empty()) {
+ // There are not enough registers on x86.
+ maybeTemp4 = regs.takeAny();
+ }
+ Register maybeTemp5 = InvalidReg;
+ if (!regs.empty()) {
+ // There are not enough registers on x86.
+ maybeTemp5 = regs.takeAny();
+ }
+
+ ArrayObject* templateObject =
+ cx->realm()->regExps.getOrCreateMatchResultTemplateObject(cx);
+ if (!templateObject) {
+ return nullptr;
+ }
+ TemplateObject templateObj(templateObject);
+ const TemplateNativeObject& nativeTemplateObj =
+ templateObj.asTemplateNativeObject();
+
+ // The template object should have enough space for the maximum number of
+ // pairs this stub can handle.
+ MOZ_ASSERT(ObjectElements::VALUES_PER_HEADER + RegExpObject::MaxPairCount ==
+ gc::GetGCKindSlots(templateObj.getAllocKind()));
+
+ TempAllocator temp(&cx->tempLifoAlloc());
+ JitContext jcx(cx);
+ StackMacroAssembler masm(cx, temp);
+ AutoCreatedBy acb(masm, "JitRealm::generateRegExpMatcherStub");
+
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+ masm.push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ // The InputOutputData is placed above the frame pointer and return address on
+ // the stack.
+ int32_t inputOutputDataStartOffset = 2 * sizeof(void*);
+
+ Label notFound, oolEntry;
+ if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex, temp1, temp2,
+ temp3, inputOutputDataStartOffset,
+ initialStringHeap, &notFound, &oolEntry)) {
+ return nullptr;
+ }
+
+ // If a regexp has named captures, fall back to the OOL stub, which
+ // will end up calling CreateRegExpMatchResults.
+ Register shared = temp2;
+ masm.unboxNonDouble(Address(regexp, NativeObject::getFixedSlotOffset(
+ RegExpObject::SHARED_SLOT)),
+ shared, JSVAL_TYPE_PRIVATE_GCTHING);
+ masm.branchPtr(Assembler::NotEqual,
+ Address(shared, RegExpShared::offsetOfGroupsTemplate()),
+ ImmWord(0), &oolEntry);
+
+ // Similarly, if the |hasIndices| flag is set, fall back to the OOL stub.
+ masm.branchTest32(Assembler::NonZero,
+ Address(shared, RegExpShared::offsetOfFlags()),
+ Imm32(int32_t(JS::RegExpFlag::HasIndices)), &oolEntry);
+
+ // Construct the result.
+ Register object = temp1;
+ Label matchResultFallback, matchResultJoin;
+ masm.createGCObject(object, temp2, templateObj, gc::DefaultHeap,
+ &matchResultFallback);
+ masm.bind(&matchResultJoin);
+
+ MOZ_ASSERT(nativeTemplateObj.numFixedSlots() == 0);
+ // Dynamic slot count is always one less than a power of 2.
+ MOZ_ASSERT(nativeTemplateObj.numDynamicSlots() == 3);
+ static_assert(RegExpRealm::MatchResultObjectIndexSlot == 0,
+ "First slot holds the 'index' property");
+ static_assert(RegExpRealm::MatchResultObjectInputSlot == 1,
+ "Second slot holds the 'input' property");
+ static_assert(RegExpRealm::MatchResultObjectGroupsSlot == 2,
+ "Third slot holds the 'groups' property");
+
+ // Initialize the slots of the result object with the dummy values
+ // defined in createMatchResultTemplateObject.
+ masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
+ masm.storeValue(
+ nativeTemplateObj.getSlot(RegExpRealm::MatchResultObjectIndexSlot),
+ Address(temp2, RegExpRealm::offsetOfMatchResultObjectIndexSlot()));
+ masm.storeValue(
+ nativeTemplateObj.getSlot(RegExpRealm::MatchResultObjectInputSlot),
+ Address(temp2, RegExpRealm::offsetOfMatchResultObjectInputSlot()));
+ masm.storeValue(
+ nativeTemplateObj.getSlot(RegExpRealm::MatchResultObjectGroupsSlot),
+ Address(temp2, RegExpRealm::offsetOfMatchResultObjectGroupsSlot()));
+
+ // clang-format off
+ /*
+ * [SMDOC] Stack layout for the RegExpMatcher stub
+ *
+ * +---------------+
+ * FramePointer +-----> |Caller-FramePtr|
+ * +---------------+
+ * |Return-Address |
+ * +---------------+
+ * inputOutputDataStartOffset +-----> +---------------+
+ * |InputOutputData|
+ * +---------------+
+ * +---------------+
+ * | MatchPairs |
+ * pairsCountAddress +-----------> count |
+ * | pairs |
+ * | |
+ * +---------------+
+ * pairsVectorStartOffset +-----> +---------------+
+ * | MatchPair |
+ * matchPairStart +------------> start | <-------+
+ * matchPairLimit +------------> limit | | Reserved space for
+ * +---------------+ | `RegExpObject::MaxPairCount`
+ * . | MatchPair objects.
+ * . |
+ * . | `count` objects will be
+ * +---------------+ | initialized and can be
+ * | MatchPair | | accessed below.
+ * | start | <-------+
+ * | limit |
+ * +---------------+
+ */
+ // clang-format on
+
+ static_assert(sizeof(MatchPair) == 2 * sizeof(int32_t),
+ "MatchPair consists of two int32 values representing the start"
+ "and the end offset of the match");
+
+ Address pairCountAddress =
+ RegExpPairCountAddress(masm, inputOutputDataStartOffset);
+
+ int32_t pairsVectorStartOffset =
+ RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
+
+ // Incremented by one below for each match pair.
+ Register matchIndex = temp2;
+ masm.move32(Imm32(0), matchIndex);
+
+ // The element in which to store the result of the current match.
+ size_t elementsOffset = NativeObject::offsetOfFixedElements();
+ BaseObjectElementIndex objectMatchElement(object, matchIndex, elementsOffset);
+
+ // The current match pair's "start" and "limit" member.
+ BaseIndex matchPairStart(FramePointer, matchIndex, TimesEight,
+ pairsVectorStartOffset + MatchPair::offsetOfStart());
+ BaseIndex matchPairLimit(FramePointer, matchIndex, TimesEight,
+ pairsVectorStartOffset + MatchPair::offsetOfLimit());
+
+ Label* depStrFailure = &oolEntry;
+ Label restoreRegExpAndLastIndex;
+
+ Register temp4;
+ if (maybeTemp4 == InvalidReg) {
+ depStrFailure = &restoreRegExpAndLastIndex;
+
+ // We don't have enough registers for a fourth temporary. Reuse |regexp|
+ // as a temporary. We restore its value at |restoreRegExpAndLastIndex|.
+ masm.push(regexp);
+ temp4 = regexp;
+ } else {
+ temp4 = maybeTemp4;
+ }
+
+ Register temp5;
+ if (maybeTemp5 == InvalidReg) {
+ depStrFailure = &restoreRegExpAndLastIndex;
+
+ // We don't have enough registers for a fifth temporary. Reuse |lastIndex|
+ // as a temporary. We restore its value at |restoreRegExpAndLastIndex|.
+ masm.push(lastIndex);
+ temp5 = lastIndex;
+ } else {
+ temp5 = maybeTemp5;
+ }
+
+ auto maybeRestoreRegExpAndLastIndex = [&]() {
+ if (maybeTemp5 == InvalidReg) {
+ masm.pop(lastIndex);
+ }
+ if (maybeTemp4 == InvalidReg) {
+ masm.pop(regexp);
+ }
+ };
+
+ // Loop to construct the match strings. There are two different loops,
+ // depending on whether the input is a Two-Byte or a Latin-1 string.
+ CreateDependentString depStrs[]{
+ {CharEncoding::TwoByte, temp3, temp4, temp5, depStrFailure},
+ {CharEncoding::Latin1, temp3, temp4, temp5, depStrFailure},
+ };
+
+ {
+ Label isLatin1, done;
+ masm.branchLatin1String(input, &isLatin1);
+
+ for (auto& depStr : depStrs) {
+ if (depStr.encoding() == CharEncoding::Latin1) {
+ masm.bind(&isLatin1);
+ }
+
+ Label matchLoop;
+ masm.bind(&matchLoop);
+
+ static_assert(MatchPair::NoMatch == -1,
+ "MatchPair::start is negative if no match was found");
+
+ Label isUndefined, storeDone;
+ masm.branch32(Assembler::LessThan, matchPairStart, Imm32(0),
+ &isUndefined);
+ {
+ depStr.generate(masm, cx->names(), CompileRuntime::get(cx->runtime()),
+ input, matchPairStart, matchPairLimit,
+ initialStringHeap);
+
+ // Storing into nursery-allocated results object's elements; no post
+ // barrier.
+ masm.storeValue(JSVAL_TYPE_STRING, depStr.string(), objectMatchElement);
+ masm.jump(&storeDone);
+ }
+ masm.bind(&isUndefined);
+ { masm.storeValue(UndefinedValue(), objectMatchElement); }
+ masm.bind(&storeDone);
+
+ masm.add32(Imm32(1), matchIndex);
+ masm.branch32(Assembler::LessThanOrEqual, pairCountAddress, matchIndex,
+ &done);
+ masm.jump(&matchLoop);
+ }
+
+#ifdef DEBUG
+ masm.assumeUnreachable("The match string loop doesn't fall through.");
+#endif
+
+ masm.bind(&done);
+ }
+
+ maybeRestoreRegExpAndLastIndex();
+
+ // Fill in the rest of the output object.
+ masm.store32(
+ matchIndex,
+ Address(object,
+ elementsOffset + ObjectElements::offsetOfInitializedLength()));
+ masm.store32(
+ matchIndex,
+ Address(object, elementsOffset + ObjectElements::offsetOfLength()));
+
+ Address firstMatchPairStartAddress(
+ FramePointer, pairsVectorStartOffset + MatchPair::offsetOfStart());
+
+ masm.loadPtr(Address(object, NativeObject::offsetOfSlots()), temp2);
+
+ masm.load32(firstMatchPairStartAddress, temp3);
+ masm.storeValue(JSVAL_TYPE_INT32, temp3, Address(temp2, 0));
+
+ // No post barrier needed (address is within nursery object.)
+ masm.storeValue(JSVAL_TYPE_STRING, input, Address(temp2, sizeof(Value)));
+
+ // All done!
+ masm.tagValue(JSVAL_TYPE_OBJECT, object, result);
+ masm.pop(FramePointer);
+ masm.ret();
+
+ masm.bind(&notFound);
+ masm.moveValue(NullValue(), result);
+ masm.pop(FramePointer);
+ masm.ret();
+
+ // Fallback paths for CreateDependentString.
+ for (auto& depStr : depStrs) {
+ depStr.generateFallback(masm);
+ }
+
+ // Fallback path for createGCObject.
+ masm.bind(&matchResultFallback);
+ CreateMatchResultFallback(masm, object, temp2, temp3, templateObj, &oolEntry);
+ masm.jump(&matchResultJoin);
+
+ // Fall-through to the ool entry after restoring the registers.
+ masm.bind(&restoreRegExpAndLastIndex);
+ maybeRestoreRegExpAndLastIndex();
+
+ // Use an undefined value to signal to the caller that the OOL stub needs to
+ // be called.
+ masm.bind(&oolEntry);
+ masm.moveValue(UndefinedValue(), result);
+ masm.pop(FramePointer);
+ masm.ret();
+
+ Linker linker(masm);
+ JitCode* code = linker.newCode(cx, CodeKind::Other);
+ if (!code) {
+ return nullptr;
+ }
+
+ CollectPerfSpewerJitCodeProfile(code, "RegExpMatcherStub");
+#ifdef MOZ_VTUNE
+ vtune::MarkStub(code, "RegExpMatcherStub");
+#endif
+
+ return code;
+}
+
+class OutOfLineRegExpMatcher : public OutOfLineCodeBase<CodeGenerator> {
+ LRegExpMatcher* lir_;
+
+ public:
+ explicit OutOfLineRegExpMatcher(LRegExpMatcher* lir) : lir_(lir) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineRegExpMatcher(this);
+ }
+
+ LRegExpMatcher* lir() const { return lir_; }
+};
+
+void CodeGenerator::visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher* ool) {
+ LRegExpMatcher* lir = ool->lir();
+ Register lastIndex = ToRegister(lir->lastIndex());
+ Register input = ToRegister(lir->string());
+ Register regexp = ToRegister(lir->regexp());
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(lastIndex);
+ regs.take(input);
+ regs.take(regexp);
+ Register temp = regs.takeAny();
+
+ masm.computeEffectiveAddress(
+ Address(masm.getStackPointer(), InputOutputDataSize), temp);
+
+ pushArg(temp);
+ pushArg(lastIndex);
+ pushArg(input);
+ pushArg(regexp);
+
+ // We are not using oolCallVM because we are in a Call, and that live
+ // registers are already saved by the the register allocator.
+ using Fn = bool (*)(JSContext*, HandleObject regexp, HandleString input,
+ int32_t lastIndex, MatchPairs * pairs,
+ MutableHandleValue output);
+ callVM<Fn, RegExpMatcherRaw>(lir);
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitRegExpMatcher(LRegExpMatcher* lir) {
+ MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpMatcherRegExpReg);
+ MOZ_ASSERT(ToRegister(lir->string()) == RegExpMatcherStringReg);
+ MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpMatcherLastIndexReg);
+ MOZ_ASSERT(ToOutValue(lir) == JSReturnOperand);
+
+#if defined(JS_NUNBOX32)
+ static_assert(RegExpMatcherRegExpReg != JSReturnReg_Type);
+ static_assert(RegExpMatcherRegExpReg != JSReturnReg_Data);
+ static_assert(RegExpMatcherStringReg != JSReturnReg_Type);
+ static_assert(RegExpMatcherStringReg != JSReturnReg_Data);
+ static_assert(RegExpMatcherLastIndexReg != JSReturnReg_Type);
+ static_assert(RegExpMatcherLastIndexReg != JSReturnReg_Data);
+#elif defined(JS_PUNBOX64)
+ static_assert(RegExpMatcherRegExpReg != JSReturnReg);
+ static_assert(RegExpMatcherStringReg != JSReturnReg);
+ static_assert(RegExpMatcherLastIndexReg != JSReturnReg);
+#endif
+
+ masm.reserveStack(RegExpReservedStack);
+
+ OutOfLineRegExpMatcher* ool = new (alloc()) OutOfLineRegExpMatcher(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ const JitRealm* jitRealm = gen->realm->jitRealm();
+ JitCode* regExpMatcherStub =
+ jitRealm->regExpMatcherStubNoBarrier(&realmStubsToReadBarrier_);
+ masm.call(regExpMatcherStub);
+ masm.branchTestUndefined(Assembler::Equal, JSReturnOperand, ool->entry());
+ masm.bind(ool->rejoin());
+
+ masm.freeStack(RegExpReservedStack);
+}
+
+static const int32_t RegExpSearcherResultNotFound = -1;
+static const int32_t RegExpSearcherResultFailed = -2;
+
+JitCode* JitRealm::generateRegExpSearcherStub(JSContext* cx) {
+ JitSpew(JitSpew_Codegen, "# Emitting RegExpSearcher stub");
+
+ Register regexp = RegExpTesterRegExpReg;
+ Register input = RegExpTesterStringReg;
+ Register lastIndex = RegExpTesterLastIndexReg;
+ Register result = ReturnReg;
+
+ // We are free to clobber all registers, as LRegExpSearcher is a call
+ // instruction.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(input);
+ regs.take(regexp);
+ regs.take(lastIndex);
+
+ Register temp1 = regs.takeAny();
+ Register temp2 = regs.takeAny();
+ Register temp3 = regs.takeAny();
+
+ TempAllocator temp(&cx->tempLifoAlloc());
+ JitContext jcx(cx);
+ StackMacroAssembler masm(cx, temp);
+ AutoCreatedBy acb(masm, "JitRealm::generateRegExpSearcherStub");
+
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+ masm.push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ // The InputOutputData is placed above the frame pointer and return address on
+ // the stack.
+ int32_t inputOutputDataStartOffset = 2 * sizeof(void*);
+
+ Label notFound, oolEntry;
+ if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex, temp1, temp2,
+ temp3, inputOutputDataStartOffset,
+ initialStringHeap, &notFound, &oolEntry)) {
+ return nullptr;
+ }
+
+ // clang-format off
+ /*
+ * [SMDOC] Stack layout for the RegExpSearcher stub
+ *
+ * +---------------+
+ * FramePointer +-----> |Caller-FramePtr|
+ * +---------------+
+ * |Return-Address |
+ * +---------------+
+ * inputOutputDataStartOffset +-----> +---------------+
+ * |InputOutputData|
+ * +---------------+
+ * +---------------+
+ * | MatchPairs |
+ * | count |
+ * | pairs |
+ * | |
+ * +---------------+
+ * pairsVectorStartOffset +-----> +---------------+
+ * | MatchPair |
+ * matchPairStart +------------> start | <-------+
+ * matchPairLimit +------------> limit | | Reserved space for
+ * +---------------+ | `RegExpObject::MaxPairCount`
+ * . | MatchPair objects.
+ * . |
+ * . | Only a single object will
+ * +---------------+ | be initialized and can be
+ * | MatchPair | | accessed below.
+ * | start | <-------+
+ * | limit |
+ * +---------------+
+ */
+ // clang-format on
+
+ int32_t pairsVectorStartOffset =
+ RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
+ Address matchPairStart(FramePointer,
+ pairsVectorStartOffset + MatchPair::offsetOfStart());
+ Address matchPairLimit(FramePointer,
+ pairsVectorStartOffset + MatchPair::offsetOfLimit());
+
+ masm.load32(matchPairStart, result);
+ masm.load32(matchPairLimit, input);
+ masm.lshiftPtr(Imm32(15), input);
+ masm.or32(input, result);
+ masm.pop(FramePointer);
+ masm.ret();
+
+ masm.bind(&notFound);
+ masm.move32(Imm32(RegExpSearcherResultNotFound), result);
+ masm.pop(FramePointer);
+ masm.ret();
+
+ masm.bind(&oolEntry);
+ masm.move32(Imm32(RegExpSearcherResultFailed), result);
+ masm.pop(FramePointer);
+ masm.ret();
+
+ Linker linker(masm);
+ JitCode* code = linker.newCode(cx, CodeKind::Other);
+ if (!code) {
+ return nullptr;
+ }
+
+ CollectPerfSpewerJitCodeProfile(code, "RegExpSearcherStub");
+#ifdef MOZ_VTUNE
+ vtune::MarkStub(code, "RegExpSearcherStub");
+#endif
+
+ return code;
+}
+
+class OutOfLineRegExpSearcher : public OutOfLineCodeBase<CodeGenerator> {
+ LRegExpSearcher* lir_;
+
+ public:
+ explicit OutOfLineRegExpSearcher(LRegExpSearcher* lir) : lir_(lir) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineRegExpSearcher(this);
+ }
+
+ LRegExpSearcher* lir() const { return lir_; }
+};
+
+void CodeGenerator::visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher* ool) {
+ LRegExpSearcher* lir = ool->lir();
+ Register lastIndex = ToRegister(lir->lastIndex());
+ Register input = ToRegister(lir->string());
+ Register regexp = ToRegister(lir->regexp());
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(lastIndex);
+ regs.take(input);
+ regs.take(regexp);
+ Register temp = regs.takeAny();
+
+ masm.computeEffectiveAddress(
+ Address(masm.getStackPointer(), InputOutputDataSize), temp);
+
+ pushArg(temp);
+ pushArg(lastIndex);
+ pushArg(input);
+ pushArg(regexp);
+
+ // We are not using oolCallVM because we are in a Call, and that live
+ // registers are already saved by the the register allocator.
+ using Fn = bool (*)(JSContext * cx, HandleObject regexp, HandleString input,
+ int32_t lastIndex, MatchPairs * pairs, int32_t * result);
+ callVM<Fn, RegExpSearcherRaw>(lir);
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitRegExpSearcher(LRegExpSearcher* lir) {
+ MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpTesterRegExpReg);
+ MOZ_ASSERT(ToRegister(lir->string()) == RegExpTesterStringReg);
+ MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpTesterLastIndexReg);
+ MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg);
+
+ static_assert(RegExpTesterRegExpReg != ReturnReg);
+ static_assert(RegExpTesterStringReg != ReturnReg);
+ static_assert(RegExpTesterLastIndexReg != ReturnReg);
+
+ masm.reserveStack(RegExpReservedStack);
+
+ OutOfLineRegExpSearcher* ool = new (alloc()) OutOfLineRegExpSearcher(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ const JitRealm* jitRealm = gen->realm->jitRealm();
+ JitCode* regExpSearcherStub =
+ jitRealm->regExpSearcherStubNoBarrier(&realmStubsToReadBarrier_);
+ masm.call(regExpSearcherStub);
+ masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpSearcherResultFailed),
+ ool->entry());
+ masm.bind(ool->rejoin());
+
+ masm.freeStack(RegExpReservedStack);
+}
+
+static const int32_t RegExpTesterResultNotFound = -1;
+static const int32_t RegExpTesterResultFailed = -2;
+
+JitCode* JitRealm::generateRegExpTesterStub(JSContext* cx) {
+ JitSpew(JitSpew_Codegen, "# Emitting RegExpTester stub");
+
+ Register regexp = RegExpTesterRegExpReg;
+ Register input = RegExpTesterStringReg;
+ Register lastIndex = RegExpTesterLastIndexReg;
+ Register result = ReturnReg;
+
+ TempAllocator temp(&cx->tempLifoAlloc());
+ JitContext jcx(cx);
+ StackMacroAssembler masm(cx, temp);
+ AutoCreatedBy acb(masm, "JitRealm::generateRegExpTesterStub");
+
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+ masm.push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ // We are free to clobber all registers, as LRegExpTester is a call
+ // instruction.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(input);
+ regs.take(regexp);
+ regs.take(lastIndex);
+
+ Register temp1 = regs.takeAny();
+ Register temp2 = regs.takeAny();
+ Register temp3 = regs.takeAny();
+
+ masm.reserveStack(RegExpReservedStack);
+
+ // In visitRegExpMatcher and visitRegExpSearcher, we reserve stack space
+ // before calling the stub. For RegExpTester we call the stub before reserving
+ // stack space, so the offset of the InputOutputData relative to the frame
+ // pointer is negative.
+ constexpr int32_t inputOutputDataStartOffset = -int32_t(RegExpReservedStack);
+
+ // On ARM64, load/store instructions can encode an immediate offset in the
+ // range [-256, 4095]. If we ever fail this assertion, it would be more
+ // efficient to store the data above the frame pointer similar to
+ // RegExpMatcher and RegExpSearcher.
+ static_assert(inputOutputDataStartOffset >= -256);
+
+ Label notFound, oolEntry;
+ if (!PrepareAndExecuteRegExp(cx, masm, regexp, input, lastIndex, temp1, temp2,
+ temp3, inputOutputDataStartOffset,
+ initialStringHeap, &notFound, &oolEntry)) {
+ return nullptr;
+ }
+
+ Label done;
+
+ int32_t pairsVectorStartOffset =
+ RegExpPairsVectorStartOffset(inputOutputDataStartOffset);
+ Address matchPairLimit(FramePointer,
+ pairsVectorStartOffset + MatchPair::offsetOfLimit());
+
+ // RegExpTester returns the end index of the match to update lastIndex.
+ masm.load32(matchPairLimit, result);
+ masm.jump(&done);
+
+ masm.bind(&notFound);
+ masm.move32(Imm32(RegExpTesterResultNotFound), result);
+ masm.jump(&done);
+
+ masm.bind(&oolEntry);
+ masm.move32(Imm32(RegExpTesterResultFailed), result);
+
+ masm.bind(&done);
+ masm.freeStack(RegExpReservedStack);
+ masm.pop(FramePointer);
+ masm.ret();
+
+ Linker linker(masm);
+ JitCode* code = linker.newCode(cx, CodeKind::Other);
+ if (!code) {
+ return nullptr;
+ }
+
+ CollectPerfSpewerJitCodeProfile(code, "RegExpTesterStub");
+#ifdef MOZ_VTUNE
+ vtune::MarkStub(code, "RegExpTesterStub");
+#endif
+
+ return code;
+}
+
+class OutOfLineRegExpTester : public OutOfLineCodeBase<CodeGenerator> {
+ LRegExpTester* lir_;
+
+ public:
+ explicit OutOfLineRegExpTester(LRegExpTester* lir) : lir_(lir) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineRegExpTester(this);
+ }
+
+ LRegExpTester* lir() const { return lir_; }
+};
+
+void CodeGenerator::visitOutOfLineRegExpTester(OutOfLineRegExpTester* ool) {
+ LRegExpTester* lir = ool->lir();
+ Register lastIndex = ToRegister(lir->lastIndex());
+ Register input = ToRegister(lir->string());
+ Register regexp = ToRegister(lir->regexp());
+
+ pushArg(lastIndex);
+ pushArg(input);
+ pushArg(regexp);
+
+ // We are not using oolCallVM because we are in a Call, and that live
+ // registers are already saved by the the register allocator.
+ using Fn = bool (*)(JSContext * cx, HandleObject regexp, HandleString input,
+ int32_t lastIndex, int32_t * result);
+ callVM<Fn, RegExpTesterRaw>(lir);
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitRegExpTester(LRegExpTester* lir) {
+ MOZ_ASSERT(ToRegister(lir->regexp()) == RegExpTesterRegExpReg);
+ MOZ_ASSERT(ToRegister(lir->string()) == RegExpTesterStringReg);
+ MOZ_ASSERT(ToRegister(lir->lastIndex()) == RegExpTesterLastIndexReg);
+ MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg);
+
+ static_assert(RegExpTesterRegExpReg != ReturnReg);
+ static_assert(RegExpTesterStringReg != ReturnReg);
+ static_assert(RegExpTesterLastIndexReg != ReturnReg);
+
+ OutOfLineRegExpTester* ool = new (alloc()) OutOfLineRegExpTester(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ const JitRealm* jitRealm = gen->realm->jitRealm();
+ JitCode* regExpTesterStub =
+ jitRealm->regExpTesterStubNoBarrier(&realmStubsToReadBarrier_);
+ masm.call(regExpTesterStub);
+
+ masm.branch32(Assembler::Equal, ReturnReg, Imm32(RegExpTesterResultFailed),
+ ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+class OutOfLineRegExpPrototypeOptimizable
+ : public OutOfLineCodeBase<CodeGenerator> {
+ LRegExpPrototypeOptimizable* ins_;
+
+ public:
+ explicit OutOfLineRegExpPrototypeOptimizable(LRegExpPrototypeOptimizable* ins)
+ : ins_(ins) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineRegExpPrototypeOptimizable(this);
+ }
+ LRegExpPrototypeOptimizable* ins() const { return ins_; }
+};
+
+void CodeGenerator::visitRegExpPrototypeOptimizable(
+ LRegExpPrototypeOptimizable* ins) {
+ Register object = ToRegister(ins->object());
+ Register output = ToRegister(ins->output());
+ Register temp = ToRegister(ins->temp0());
+
+ OutOfLineRegExpPrototypeOptimizable* ool =
+ new (alloc()) OutOfLineRegExpPrototypeOptimizable(ins);
+ addOutOfLineCode(ool, ins->mir());
+
+ masm.branchIfNotRegExpPrototypeOptimizable(object, temp, ool->entry());
+ masm.move32(Imm32(0x1), output);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineRegExpPrototypeOptimizable(
+ OutOfLineRegExpPrototypeOptimizable* ool) {
+ LRegExpPrototypeOptimizable* ins = ool->ins();
+ Register object = ToRegister(ins->object());
+ Register output = ToRegister(ins->output());
+
+ saveVolatile(output);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * proto);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(output);
+ masm.passABIArg(output);
+ masm.passABIArg(object);
+ masm.callWithABI<Fn, RegExpPrototypeOptimizableRaw>();
+ masm.storeCallBoolResult(output);
+
+ restoreVolatile(output);
+
+ masm.jump(ool->rejoin());
+}
+
+class OutOfLineRegExpInstanceOptimizable
+ : public OutOfLineCodeBase<CodeGenerator> {
+ LRegExpInstanceOptimizable* ins_;
+
+ public:
+ explicit OutOfLineRegExpInstanceOptimizable(LRegExpInstanceOptimizable* ins)
+ : ins_(ins) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineRegExpInstanceOptimizable(this);
+ }
+ LRegExpInstanceOptimizable* ins() const { return ins_; }
+};
+
+void CodeGenerator::visitRegExpInstanceOptimizable(
+ LRegExpInstanceOptimizable* ins) {
+ Register object = ToRegister(ins->object());
+ Register output = ToRegister(ins->output());
+ Register temp = ToRegister(ins->temp0());
+
+ OutOfLineRegExpInstanceOptimizable* ool =
+ new (alloc()) OutOfLineRegExpInstanceOptimizable(ins);
+ addOutOfLineCode(ool, ins->mir());
+
+ masm.branchIfNotRegExpInstanceOptimizable(object, temp, ool->entry());
+ masm.move32(Imm32(0x1), output);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineRegExpInstanceOptimizable(
+ OutOfLineRegExpInstanceOptimizable* ool) {
+ LRegExpInstanceOptimizable* ins = ool->ins();
+ Register object = ToRegister(ins->object());
+ Register proto = ToRegister(ins->proto());
+ Register output = ToRegister(ins->output());
+
+ saveVolatile(output);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, JSObject * proto);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(output);
+ masm.passABIArg(output);
+ masm.passABIArg(object);
+ masm.passABIArg(proto);
+ masm.callWithABI<Fn, RegExpInstanceOptimizableRaw>();
+ masm.storeCallBoolResult(output);
+
+ restoreVolatile(output);
+
+ masm.jump(ool->rejoin());
+}
+
+static void FindFirstDollarIndex(MacroAssembler& masm, Register str,
+ Register len, Register temp0, Register temp1,
+ Register output, CharEncoding encoding) {
+#ifdef DEBUG
+ Label ok;
+ masm.branch32(Assembler::GreaterThan, len, Imm32(0), &ok);
+ masm.assumeUnreachable("Length should be greater than 0.");
+ masm.bind(&ok);
+#endif
+
+ Register chars = temp0;
+ masm.loadStringChars(str, chars, encoding);
+
+ masm.move32(Imm32(0), output);
+
+ Label start, done;
+ masm.bind(&start);
+
+ Register currentChar = temp1;
+ masm.loadChar(chars, output, currentChar, encoding);
+ masm.branch32(Assembler::Equal, currentChar, Imm32('$'), &done);
+
+ masm.add32(Imm32(1), output);
+ masm.branch32(Assembler::NotEqual, output, len, &start);
+
+ masm.move32(Imm32(-1), output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGetFirstDollarIndex(LGetFirstDollarIndex* ins) {
+ Register str = ToRegister(ins->str());
+ Register output = ToRegister(ins->output());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register len = ToRegister(ins->temp2());
+
+ using Fn = bool (*)(JSContext*, JSString*, int32_t*);
+ OutOfLineCode* ool = oolCallVM<Fn, GetFirstDollarIndexRaw>(
+ ins, ArgList(str), StoreRegisterTo(output));
+
+ masm.branchIfRope(str, ool->entry());
+ masm.loadStringLength(str, len);
+
+ Label isLatin1, done;
+ masm.branchLatin1String(str, &isLatin1);
+ {
+ FindFirstDollarIndex(masm, str, len, temp0, temp1, output,
+ CharEncoding::TwoByte);
+ masm.jump(&done);
+ }
+ masm.bind(&isLatin1);
+ {
+ FindFirstDollarIndex(masm, str, len, temp0, temp1, output,
+ CharEncoding::Latin1);
+ }
+ masm.bind(&done);
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitStringReplace(LStringReplace* lir) {
+ if (lir->replacement()->isConstant()) {
+ pushArg(ImmGCPtr(lir->replacement()->toConstant()->toString()));
+ } else {
+ pushArg(ToRegister(lir->replacement()));
+ }
+
+ if (lir->pattern()->isConstant()) {
+ pushArg(ImmGCPtr(lir->pattern()->toConstant()->toString()));
+ } else {
+ pushArg(ToRegister(lir->pattern()));
+ }
+
+ if (lir->string()->isConstant()) {
+ pushArg(ImmGCPtr(lir->string()->toConstant()->toString()));
+ } else {
+ pushArg(ToRegister(lir->string()));
+ }
+
+ using Fn =
+ JSString* (*)(JSContext*, HandleString, HandleString, HandleString);
+ if (lir->mir()->isFlatReplacement()) {
+ callVM<Fn, StringFlatReplaceString>(lir);
+ } else {
+ callVM<Fn, StringReplace>(lir);
+ }
+}
+
+void CodeGenerator::visitBinaryValueCache(LBinaryValueCache* lir) {
+ LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
+ TypedOrValueRegister lhs =
+ TypedOrValueRegister(ToValue(lir, LBinaryValueCache::LhsIndex));
+ TypedOrValueRegister rhs =
+ TypedOrValueRegister(ToValue(lir, LBinaryValueCache::RhsIndex));
+ ValueOperand output = ToOutValue(lir);
+
+ JSOp jsop = JSOp(*lir->mirRaw()->toInstruction()->resumePoint()->pc());
+
+ switch (jsop) {
+ case JSOp::Add:
+ case JSOp::Sub:
+ case JSOp::Mul:
+ case JSOp::Div:
+ case JSOp::Mod:
+ case JSOp::Pow:
+ case JSOp::BitAnd:
+ case JSOp::BitOr:
+ case JSOp::BitXor:
+ case JSOp::Lsh:
+ case JSOp::Rsh:
+ case JSOp::Ursh: {
+ IonBinaryArithIC ic(liveRegs, lhs, rhs, output);
+ addIC(lir, allocateIC(ic));
+ return;
+ }
+ default:
+ MOZ_CRASH("Unsupported jsop in MBinaryValueCache");
+ }
+}
+
+void CodeGenerator::visitBinaryBoolCache(LBinaryBoolCache* lir) {
+ LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
+ TypedOrValueRegister lhs =
+ TypedOrValueRegister(ToValue(lir, LBinaryBoolCache::LhsIndex));
+ TypedOrValueRegister rhs =
+ TypedOrValueRegister(ToValue(lir, LBinaryBoolCache::RhsIndex));
+ Register output = ToRegister(lir->output());
+
+ JSOp jsop = JSOp(*lir->mirRaw()->toInstruction()->resumePoint()->pc());
+
+ switch (jsop) {
+ case JSOp::Lt:
+ case JSOp::Le:
+ case JSOp::Gt:
+ case JSOp::Ge:
+ case JSOp::Eq:
+ case JSOp::Ne:
+ case JSOp::StrictEq:
+ case JSOp::StrictNe: {
+ IonCompareIC ic(liveRegs, lhs, rhs, output);
+ addIC(lir, allocateIC(ic));
+ return;
+ }
+ default:
+ MOZ_CRASH("Unsupported jsop in MBinaryBoolCache");
+ }
+}
+
+void CodeGenerator::visitUnaryCache(LUnaryCache* lir) {
+ LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
+ TypedOrValueRegister input =
+ TypedOrValueRegister(ToValue(lir, LUnaryCache::InputIndex));
+ ValueOperand output = ToOutValue(lir);
+
+ IonUnaryArithIC ic(liveRegs, input, output);
+ addIC(lir, allocateIC(ic));
+}
+
+void CodeGenerator::visitModuleMetadata(LModuleMetadata* lir) {
+ pushArg(ImmPtr(lir->mir()->module()));
+
+ using Fn = JSObject* (*)(JSContext*, HandleObject);
+ callVM<Fn, js::GetOrCreateModuleMetaObject>(lir);
+}
+
+void CodeGenerator::visitDynamicImport(LDynamicImport* lir) {
+ pushArg(ToValue(lir, LDynamicImport::OptionsIndex));
+ pushArg(ToValue(lir, LDynamicImport::SpecifierIndex));
+ pushArg(ImmGCPtr(current->mir()->info().script()));
+
+ using Fn = JSObject* (*)(JSContext*, HandleScript, HandleValue, HandleValue);
+ callVM<Fn, js::StartDynamicModuleImport>(lir);
+}
+
+void CodeGenerator::visitLambda(LLambda* lir) {
+ Register envChain = ToRegister(lir->environmentChain());
+ Register output = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp0());
+
+ JSFunction* fun = lir->mir()->templateFunction();
+
+ using Fn = JSObject* (*)(JSContext*, HandleFunction, HandleObject);
+ OutOfLineCode* ool = oolCallVM<Fn, js::Lambda>(
+ lir, ArgList(ImmGCPtr(fun), envChain), StoreRegisterTo(output));
+
+ TemplateObject templateObject(fun);
+ masm.createGCObject(output, tempReg, templateObject, gc::DefaultHeap,
+ ool->entry());
+
+ masm.storeValue(JSVAL_TYPE_OBJECT, envChain,
+ Address(output, JSFunction::offsetOfEnvironment()));
+ // No post barrier needed because output is guaranteed to be allocated in
+ // the nursery.
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitFunctionWithProto(LFunctionWithProto* lir) {
+ Register envChain = ToRegister(lir->envChain());
+ Register prototype = ToRegister(lir->prototype());
+
+ pushArg(prototype);
+ pushArg(envChain);
+ pushArg(ImmGCPtr(lir->mir()->function()));
+
+ using Fn =
+ JSObject* (*)(JSContext*, HandleFunction, HandleObject, HandleObject);
+ callVM<Fn, js::FunWithProtoOperation>(lir);
+}
+
+void CodeGenerator::visitSetFunName(LSetFunName* lir) {
+ pushArg(Imm32(lir->mir()->prefixKind()));
+ pushArg(ToValue(lir, LSetFunName::NameIndex));
+ pushArg(ToRegister(lir->fun()));
+
+ using Fn =
+ bool (*)(JSContext*, HandleFunction, HandleValue, FunctionPrefixKind);
+ callVM<Fn, js::SetFunctionName>(lir);
+}
+
+void CodeGenerator::visitOsiPoint(LOsiPoint* lir) {
+ // Note: markOsiPoint ensures enough space exists between the last
+ // LOsiPoint and this one to patch adjacent call instructions.
+
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+
+ uint32_t osiCallPointOffset = markOsiPoint(lir);
+
+ LSafepoint* safepoint = lir->associatedSafepoint();
+ MOZ_ASSERT(!safepoint->osiCallPointOffset());
+ safepoint->setOsiCallPointOffset(osiCallPointOffset);
+
+#ifdef DEBUG
+ // There should be no movegroups or other instructions between
+ // an instruction and its OsiPoint. This is necessary because
+ // we use the OsiPoint's snapshot from within VM calls.
+ for (LInstructionReverseIterator iter(current->rbegin(lir));
+ iter != current->rend(); iter++) {
+ if (*iter == lir) {
+ continue;
+ }
+ MOZ_ASSERT(!iter->isMoveGroup());
+ MOZ_ASSERT(iter->safepoint() == safepoint);
+ break;
+ }
+#endif
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ if (shouldVerifyOsiPointRegs(safepoint)) {
+ verifyOsiPointRegs(safepoint);
+ }
+#endif
+}
+
+void CodeGenerator::visitPhi(LPhi* lir) {
+ MOZ_CRASH("Unexpected LPhi in CodeGenerator");
+}
+
+void CodeGenerator::visitGoto(LGoto* lir) { jumpToBlock(lir->target()); }
+
+void CodeGenerator::visitTableSwitch(LTableSwitch* ins) {
+ MTableSwitch* mir = ins->mir();
+ Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
+ const LAllocation* temp;
+
+ if (mir->getOperand(0)->type() != MIRType::Int32) {
+ temp = ins->tempInt()->output();
+
+ // The input is a double, so try and convert it to an integer.
+ // If it does not fit in an integer, take the default case.
+ masm.convertDoubleToInt32(ToFloatRegister(ins->index()), ToRegister(temp),
+ defaultcase, false);
+ } else {
+ temp = ins->index();
+ }
+
+ emitTableSwitchDispatch(mir, ToRegister(temp),
+ ToRegisterOrInvalid(ins->tempPointer()));
+}
+
+void CodeGenerator::visitTableSwitchV(LTableSwitchV* ins) {
+ MTableSwitch* mir = ins->mir();
+ Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
+
+ Register index = ToRegister(ins->tempInt());
+ ValueOperand value = ToValue(ins, LTableSwitchV::InputValue);
+ Register tag = masm.extractTag(value, index);
+ masm.branchTestNumber(Assembler::NotEqual, tag, defaultcase);
+
+ Label unboxInt, isInt;
+ masm.branchTestInt32(Assembler::Equal, tag, &unboxInt);
+ {
+ FloatRegister floatIndex = ToFloatRegister(ins->tempFloat());
+ masm.unboxDouble(value, floatIndex);
+ masm.convertDoubleToInt32(floatIndex, index, defaultcase, false);
+ masm.jump(&isInt);
+ }
+
+ masm.bind(&unboxInt);
+ masm.unboxInt32(value, index);
+
+ masm.bind(&isInt);
+
+ emitTableSwitchDispatch(mir, index, ToRegisterOrInvalid(ins->tempPointer()));
+}
+
+void CodeGenerator::visitParameter(LParameter* lir) {}
+
+void CodeGenerator::visitCallee(LCallee* lir) {
+ Register callee = ToRegister(lir->output());
+ Address ptr(FramePointer, JitFrameLayout::offsetOfCalleeToken());
+
+ masm.loadFunctionFromCalleeToken(ptr, callee);
+}
+
+void CodeGenerator::visitIsConstructing(LIsConstructing* lir) {
+ Register output = ToRegister(lir->output());
+ Address calleeToken(FramePointer, JitFrameLayout::offsetOfCalleeToken());
+ masm.loadPtr(calleeToken, output);
+
+ // We must be inside a function.
+ MOZ_ASSERT(current->mir()->info().script()->function());
+
+ // The low bit indicates whether this call is constructing, just clear the
+ // other bits.
+ static_assert(CalleeToken_Function == 0x0,
+ "CalleeTokenTag value should match");
+ static_assert(CalleeToken_FunctionConstructing == 0x1,
+ "CalleeTokenTag value should match");
+ masm.andPtr(Imm32(0x1), output);
+}
+
+void CodeGenerator::visitReturn(LReturn* lir) {
+#if defined(JS_NUNBOX32)
+ DebugOnly<LAllocation*> type = lir->getOperand(TYPE_INDEX);
+ DebugOnly<LAllocation*> payload = lir->getOperand(PAYLOAD_INDEX);
+ MOZ_ASSERT(ToRegister(type) == JSReturnReg_Type);
+ MOZ_ASSERT(ToRegister(payload) == JSReturnReg_Data);
+#elif defined(JS_PUNBOX64)
+ DebugOnly<LAllocation*> result = lir->getOperand(0);
+ MOZ_ASSERT(ToRegister(result) == JSReturnReg);
+#endif
+ // Don't emit a jump to the return label if this is the last block, as
+ // it'll fall through to the epilogue.
+ //
+ // This is -not- true however for a Generator-return, which may appear in the
+ // middle of the last block, so we should always emit the jump there.
+ if (current->mir() != *gen->graph().poBegin() || lir->isGenerator()) {
+ masm.jump(&returnLabel_);
+ }
+}
+
+void CodeGenerator::visitOsrEntry(LOsrEntry* lir) {
+ Register temp = ToRegister(lir->temp());
+
+ // Remember the OSR entry offset into the code buffer.
+ masm.flushBuffer();
+ setOsrEntryOffset(masm.size());
+
+ // Allocate the full frame for this function
+ // Note we have a new entry here. So we reset MacroAssembler::framePushed()
+ // to 0, before reserving the stack.
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+ masm.setFramePushed(0);
+
+ // The Baseline code ensured both the frame pointer and stack pointer point to
+ // the JitFrameLayout on the stack.
+
+ // If profiling, save the current frame pointer to a per-thread global field.
+ if (isProfilerInstrumentationEnabled()) {
+ masm.profilerEnterFrame(FramePointer, temp);
+ }
+
+ masm.reserveStack(frameSize());
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+
+ // Ensure that the Ion frames is properly aligned.
+ masm.assertStackAlignment(JitStackAlignment, 0);
+}
+
+void CodeGenerator::visitOsrEnvironmentChain(LOsrEnvironmentChain* lir) {
+ const LAllocation* frame = lir->getOperand(0);
+ const LDefinition* object = lir->getDef(0);
+
+ const ptrdiff_t frameOffset =
+ BaselineFrame::reverseOffsetOfEnvironmentChain();
+
+ masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));
+}
+
+void CodeGenerator::visitOsrArgumentsObject(LOsrArgumentsObject* lir) {
+ const LAllocation* frame = lir->getOperand(0);
+ const LDefinition* object = lir->getDef(0);
+
+ const ptrdiff_t frameOffset = BaselineFrame::reverseOffsetOfArgsObj();
+
+ masm.loadPtr(Address(ToRegister(frame), frameOffset), ToRegister(object));
+}
+
+void CodeGenerator::visitOsrValue(LOsrValue* value) {
+ const LAllocation* frame = value->getOperand(0);
+ const ValueOperand out = ToOutValue(value);
+
+ const ptrdiff_t frameOffset = value->mir()->frameOffset();
+
+ masm.loadValue(Address(ToRegister(frame), frameOffset), out);
+}
+
+void CodeGenerator::visitOsrReturnValue(LOsrReturnValue* lir) {
+ const LAllocation* frame = lir->getOperand(0);
+ const ValueOperand out = ToOutValue(lir);
+
+ Address flags =
+ Address(ToRegister(frame), BaselineFrame::reverseOffsetOfFlags());
+ Address retval =
+ Address(ToRegister(frame), BaselineFrame::reverseOffsetOfReturnValue());
+
+ masm.moveValue(UndefinedValue(), out);
+
+ Label done;
+ masm.branchTest32(Assembler::Zero, flags, Imm32(BaselineFrame::HAS_RVAL),
+ &done);
+ masm.loadValue(retval, out);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitStackArgT(LStackArgT* lir) {
+ const LAllocation* arg = lir->arg();
+ MIRType argType = lir->type();
+ uint32_t argslot = lir->argslot();
+ MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount());
+
+ Address dest = AddressOfPassedArg(argslot);
+
+ if (arg->isFloatReg()) {
+ masm.boxDouble(ToFloatRegister(arg), dest);
+ } else if (arg->isRegister()) {
+ masm.storeValue(ValueTypeFromMIRType(argType), ToRegister(arg), dest);
+ } else {
+ masm.storeValue(arg->toConstant()->toJSValue(), dest);
+ }
+}
+
+void CodeGenerator::visitStackArgV(LStackArgV* lir) {
+ ValueOperand val = ToValue(lir, 0);
+ uint32_t argslot = lir->argslot();
+ MOZ_ASSERT(argslot - 1u < graph.argumentSlotCount());
+
+ masm.storeValue(val, AddressOfPassedArg(argslot));
+}
+
+void CodeGenerator::visitMoveGroup(LMoveGroup* group) {
+ if (!group->numMoves()) {
+ return;
+ }
+
+ MoveResolver& resolver = masm.moveResolver();
+
+ for (size_t i = 0; i < group->numMoves(); i++) {
+ const LMove& move = group->getMove(i);
+
+ LAllocation from = move.from();
+ LAllocation to = move.to();
+ LDefinition::Type type = move.type();
+
+ // No bogus moves.
+ MOZ_ASSERT(from != to);
+ MOZ_ASSERT(!from.isConstant());
+ MoveOp::Type moveType;
+ switch (type) {
+ case LDefinition::OBJECT:
+ case LDefinition::SLOTS:
+#ifdef JS_NUNBOX32
+ case LDefinition::TYPE:
+ case LDefinition::PAYLOAD:
+#else
+ case LDefinition::BOX:
+#endif
+ case LDefinition::GENERAL:
+ case LDefinition::STACKRESULTS:
+ moveType = MoveOp::GENERAL;
+ break;
+ case LDefinition::INT32:
+ moveType = MoveOp::INT32;
+ break;
+ case LDefinition::FLOAT32:
+ moveType = MoveOp::FLOAT32;
+ break;
+ case LDefinition::DOUBLE:
+ moveType = MoveOp::DOUBLE;
+ break;
+ case LDefinition::SIMD128:
+ moveType = MoveOp::SIMD128;
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+
+ masm.propagateOOM(
+ resolver.addMove(toMoveOperand(from), toMoveOperand(to), moveType));
+ }
+
+ masm.propagateOOM(resolver.resolve());
+ if (masm.oom()) {
+ return;
+ }
+
+ MoveEmitter emitter(masm);
+
+#ifdef JS_CODEGEN_X86
+ if (group->maybeScratchRegister().isGeneralReg()) {
+ emitter.setScratchRegister(
+ group->maybeScratchRegister().toGeneralReg()->reg());
+ } else {
+ resolver.sortMemoryToMemoryMoves();
+ }
+#endif
+
+ emitter.emit(resolver);
+ emitter.finish();
+}
+
+void CodeGenerator::visitInteger(LInteger* lir) {
+ masm.move32(Imm32(lir->i32()), ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitInteger64(LInteger64* lir) {
+ masm.move64(Imm64(lir->i64()), ToOutRegister64(lir));
+}
+
+void CodeGenerator::visitPointer(LPointer* lir) {
+ masm.movePtr(ImmGCPtr(lir->gcptr()), ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitNurseryObject(LNurseryObject* lir) {
+ Register output = ToRegister(lir->output());
+ uint32_t nurseryIndex = lir->mir()->nurseryIndex();
+
+ // Load a pointer to the entry in IonScript's nursery objects list.
+ CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), output);
+ masm.propagateOOM(ionNurseryObjectLabels_.emplaceBack(label, nurseryIndex));
+
+ // Load the JSObject*.
+ masm.loadPtr(Address(output, 0), output);
+}
+
+void CodeGenerator::visitKeepAliveObject(LKeepAliveObject* lir) {
+ // No-op.
+}
+
+void CodeGenerator::visitSlots(LSlots* lir) {
+ Address slots(ToRegister(lir->object()), NativeObject::offsetOfSlots());
+ masm.loadPtr(slots, ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitLoadDynamicSlotV(LLoadDynamicSlotV* lir) {
+ ValueOperand dest = ToOutValue(lir);
+ Register base = ToRegister(lir->input());
+ int32_t offset = lir->mir()->slot() * sizeof(js::Value);
+
+ masm.loadValue(Address(base, offset), dest);
+}
+
+static ConstantOrRegister ToConstantOrRegister(const LAllocation* value,
+ MIRType valueType) {
+ if (value->isConstant()) {
+ return ConstantOrRegister(value->toConstant()->toJSValue());
+ }
+ return TypedOrValueRegister(valueType, ToAnyRegister(value));
+}
+
+void CodeGenerator::visitStoreDynamicSlotT(LStoreDynamicSlotT* lir) {
+ Register base = ToRegister(lir->slots());
+ int32_t offset = lir->mir()->slot() * sizeof(js::Value);
+ Address dest(base, offset);
+
+ if (lir->mir()->needsBarrier()) {
+ emitPreBarrier(dest);
+ }
+
+ MIRType valueType = lir->mir()->value()->type();
+ ConstantOrRegister value = ToConstantOrRegister(lir->value(), valueType);
+ masm.storeUnboxedValue(value, valueType, dest);
+}
+
+void CodeGenerator::visitStoreDynamicSlotV(LStoreDynamicSlotV* lir) {
+ Register base = ToRegister(lir->slots());
+ int32_t offset = lir->mir()->slot() * sizeof(Value);
+
+ const ValueOperand value = ToValue(lir, LStoreDynamicSlotV::ValueIndex);
+
+ if (lir->mir()->needsBarrier()) {
+ emitPreBarrier(Address(base, offset));
+ }
+
+ masm.storeValue(value, Address(base, offset));
+}
+
+void CodeGenerator::visitElements(LElements* lir) {
+ Address elements(ToRegister(lir->object()), NativeObject::offsetOfElements());
+ masm.loadPtr(elements, ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitFunctionEnvironment(LFunctionEnvironment* lir) {
+ Address environment(ToRegister(lir->function()),
+ JSFunction::offsetOfEnvironment());
+ masm.unboxObject(environment, ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitHomeObject(LHomeObject* lir) {
+ Register func = ToRegister(lir->function());
+ Address homeObject(func, FunctionExtended::offsetOfMethodHomeObjectSlot());
+
+ masm.assertFunctionIsExtended(func);
+#ifdef DEBUG
+ Label isObject;
+ masm.branchTestObject(Assembler::Equal, homeObject, &isObject);
+ masm.assumeUnreachable("[[HomeObject]] must be Object");
+ masm.bind(&isObject);
+#endif
+
+ masm.unboxObject(homeObject, ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitHomeObjectSuperBase(LHomeObjectSuperBase* lir) {
+ Register homeObject = ToRegister(lir->homeObject());
+ ValueOperand output = ToOutValue(lir);
+ Register temp = output.scratchReg();
+
+ masm.loadObjProto(homeObject, temp);
+
+#ifdef DEBUG
+ // We won't encounter a lazy proto, because the prototype is guaranteed to
+ // either be a JSFunction or a PlainObject, and only proxy objects can have a
+ // lazy proto.
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ Label proxyCheckDone;
+ masm.branchPtr(Assembler::NotEqual, temp, ImmWord(1), &proxyCheckDone);
+ masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperBase");
+ masm.bind(&proxyCheckDone);
+#endif
+
+ Label nullProto, done;
+ masm.branchPtr(Assembler::Equal, temp, ImmWord(0), &nullProto);
+
+ // Box prototype and return
+ masm.tagValue(JSVAL_TYPE_OBJECT, temp, output);
+ masm.jump(&done);
+
+ masm.bind(&nullProto);
+ masm.moveValue(NullValue(), output);
+
+ masm.bind(&done);
+}
+
+template <class T>
+static T* ToConstantObject(MDefinition* def) {
+ MOZ_ASSERT(def->isConstant());
+ return &def->toConstant()->toObject().as<T>();
+}
+
+void CodeGenerator::visitNewLexicalEnvironmentObject(
+ LNewLexicalEnvironmentObject* lir) {
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ auto* templateObj = ToConstantObject<BlockLexicalEnvironmentObject>(
+ lir->mir()->templateObj());
+ auto* scope = &templateObj->scope();
+ gc::InitialHeap initialHeap = gc::DefaultHeap;
+
+ using Fn =
+ BlockLexicalEnvironmentObject* (*)(JSContext*, Handle<LexicalScope*>);
+ auto* ool =
+ oolCallVM<Fn, BlockLexicalEnvironmentObject::createWithoutEnclosing>(
+ lir, ArgList(ImmGCPtr(scope)), StoreRegisterTo(output));
+
+ TemplateObject templateObject(templateObj);
+ masm.createGCObject(output, temp, templateObject, initialHeap, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewClassBodyEnvironmentObject(
+ LNewClassBodyEnvironmentObject* lir) {
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ auto* templateObj = ToConstantObject<ClassBodyLexicalEnvironmentObject>(
+ lir->mir()->templateObj());
+ auto* scope = &templateObj->scope();
+ gc::InitialHeap initialHeap = gc::DefaultHeap;
+
+ using Fn = ClassBodyLexicalEnvironmentObject* (*)(JSContext*,
+ Handle<ClassBodyScope*>);
+ auto* ool =
+ oolCallVM<Fn, ClassBodyLexicalEnvironmentObject::createWithoutEnclosing>(
+ lir, ArgList(ImmGCPtr(scope)), StoreRegisterTo(output));
+
+ TemplateObject templateObject(templateObj);
+ masm.createGCObject(output, temp, templateObject, initialHeap, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewVarEnvironmentObject(
+ LNewVarEnvironmentObject* lir) {
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ auto* templateObj =
+ ToConstantObject<VarEnvironmentObject>(lir->mir()->templateObj());
+ auto* scope = &templateObj->scope().as<VarScope>();
+ gc::InitialHeap initialHeap = gc::DefaultHeap;
+
+ using Fn = VarEnvironmentObject* (*)(JSContext*, Handle<VarScope*>);
+ auto* ool = oolCallVM<Fn, VarEnvironmentObject::createWithoutEnclosing>(
+ lir, ArgList(ImmGCPtr(scope)), StoreRegisterTo(output));
+
+ TemplateObject templateObject(templateObj);
+ masm.createGCObject(output, temp, templateObject, initialHeap, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitGuardShape(LGuardShape* guard) {
+ Register obj = ToRegister(guard->input());
+ Register temp = ToTempRegisterOrInvalid(guard->temp0());
+ Label bail;
+ masm.branchTestObjShape(Assembler::NotEqual, obj, guard->mir()->shape(), temp,
+ obj, &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardMultipleShapes(LGuardMultipleShapes* guard) {
+ Register obj = ToRegister(guard->object());
+ Register shapeList = ToRegister(guard->shapeList());
+ Register temp = ToRegister(guard->temp0());
+ Register temp2 = ToRegister(guard->temp1());
+ Register temp3 = ToRegister(guard->temp2());
+ Register spectre = ToTempRegisterOrInvalid(guard->temp3());
+
+ Label bail;
+ masm.loadPtr(Address(shapeList, NativeObject::offsetOfElements()), temp);
+ masm.branchTestObjShapeList(Assembler::NotEqual, obj, temp, temp2, temp3,
+ spectre, &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardProto(LGuardProto* guard) {
+ Register obj = ToRegister(guard->object());
+ Register expected = ToRegister(guard->expected());
+ Register temp = ToRegister(guard->temp0());
+
+ masm.loadObjProto(obj, temp);
+
+ Label bail;
+ masm.branchPtr(Assembler::NotEqual, temp, expected, &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardNullProto(LGuardNullProto* guard) {
+ Register obj = ToRegister(guard->input());
+ Register temp = ToRegister(guard->temp0());
+
+ masm.loadObjProto(obj, temp);
+
+ Label bail;
+ masm.branchTestPtr(Assembler::NonZero, temp, temp, &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardIsNativeObject(LGuardIsNativeObject* guard) {
+ Register obj = ToRegister(guard->input());
+ Register temp = ToRegister(guard->temp0());
+
+ Label bail;
+ masm.branchIfNonNativeObj(obj, temp, &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardIsProxy(LGuardIsProxy* guard) {
+ Register obj = ToRegister(guard->input());
+ Register temp = ToRegister(guard->temp0());
+
+ Label bail;
+ masm.branchTestObjectIsProxy(false, obj, temp, &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardIsNotProxy(LGuardIsNotProxy* guard) {
+ Register obj = ToRegister(guard->input());
+ Register temp = ToRegister(guard->temp0());
+
+ Label bail;
+ masm.branchTestObjectIsProxy(true, obj, temp, &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardIsNotDOMProxy(LGuardIsNotDOMProxy* guard) {
+ Register proxy = ToRegister(guard->proxy());
+ Register temp = ToRegister(guard->temp0());
+
+ Label bail;
+ masm.branchTestProxyHandlerFamily(Assembler::Equal, proxy, temp,
+ GetDOMProxyHandlerFamily(), &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitProxyGet(LProxyGet* lir) {
+ Register proxy = ToRegister(lir->proxy());
+ Register temp = ToRegister(lir->temp0());
+
+ pushArg(lir->mir()->id(), temp);
+ pushArg(proxy);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleId, MutableHandleValue);
+ callVM<Fn, ProxyGetProperty>(lir);
+}
+
+void CodeGenerator::visitProxyGetByValue(LProxyGetByValue* lir) {
+ Register proxy = ToRegister(lir->proxy());
+ ValueOperand idVal = ToValue(lir, LProxyGetByValue::IdIndex);
+
+ pushArg(idVal);
+ pushArg(proxy);
+
+ using Fn =
+ bool (*)(JSContext*, HandleObject, HandleValue, MutableHandleValue);
+ callVM<Fn, ProxyGetPropertyByValue>(lir);
+}
+
+void CodeGenerator::visitProxyHasProp(LProxyHasProp* lir) {
+ Register proxy = ToRegister(lir->proxy());
+ ValueOperand idVal = ToValue(lir, LProxyHasProp::IdIndex);
+
+ pushArg(idVal);
+ pushArg(proxy);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
+ if (lir->mir()->hasOwn()) {
+ callVM<Fn, ProxyHasOwn>(lir);
+ } else {
+ callVM<Fn, ProxyHas>(lir);
+ }
+}
+
+void CodeGenerator::visitProxySet(LProxySet* lir) {
+ Register proxy = ToRegister(lir->proxy());
+ ValueOperand rhs = ToValue(lir, LProxySet::RhsIndex);
+ Register temp = ToRegister(lir->temp0());
+
+ pushArg(Imm32(lir->mir()->strict()));
+ pushArg(rhs);
+ pushArg(lir->mir()->id(), temp);
+ pushArg(proxy);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleId, HandleValue, bool);
+ callVM<Fn, ProxySetProperty>(lir);
+}
+
+void CodeGenerator::visitProxySetByValue(LProxySetByValue* lir) {
+ Register proxy = ToRegister(lir->proxy());
+ ValueOperand idVal = ToValue(lir, LProxySetByValue::IdIndex);
+ ValueOperand rhs = ToValue(lir, LProxySetByValue::RhsIndex);
+
+ pushArg(Imm32(lir->mir()->strict()));
+ pushArg(rhs);
+ pushArg(idVal);
+ pushArg(proxy);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue, bool);
+ callVM<Fn, ProxySetPropertyByValue>(lir);
+}
+
+void CodeGenerator::visitCallSetArrayLength(LCallSetArrayLength* lir) {
+ Register obj = ToRegister(lir->obj());
+ ValueOperand rhs = ToValue(lir, LCallSetArrayLength::RhsIndex);
+
+ pushArg(Imm32(lir->mir()->strict()));
+ pushArg(rhs);
+ pushArg(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool);
+ callVM<Fn, jit::SetArrayLength>(lir);
+}
+
+void CodeGenerator::visitMegamorphicLoadSlot(LMegamorphicLoadSlot* lir) {
+ Register obj = ToRegister(lir->object());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+ ValueOperand output = ToOutValue(lir);
+
+ Label bail, cacheHit;
+ if (JitOptions.enableWatchtowerMegamorphic) {
+ masm.emitMegamorphicCacheLookup(lir->mir()->name(), obj, temp0, temp1,
+ temp2, output, &cacheHit);
+ }
+
+ masm.branchIfNonNativeObj(obj, temp0, &bail);
+
+ masm.Push(UndefinedValue());
+ masm.moveStackPtrTo(temp2);
+
+ using Fn =
+ bool (*)(JSContext * cx, JSObject * obj, PropertyKey id, Value * vp);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp0);
+ masm.passABIArg(temp0);
+ masm.passABIArg(obj);
+ masm.movePropertyKey(lir->mir()->name(), temp1);
+ masm.passABIArg(temp1);
+ masm.passABIArg(temp2);
+
+ masm.callWithABI<Fn, GetNativeDataPropertyPureFallback>();
+
+ MOZ_ASSERT(!output.aliases(ReturnReg));
+ masm.Pop(output);
+
+ masm.branchIfFalseBool(ReturnReg, &bail);
+
+ masm.bind(&cacheHit);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitMegamorphicLoadSlotByValue(
+ LMegamorphicLoadSlotByValue* lir) {
+ Register obj = ToRegister(lir->object());
+ ValueOperand idVal = ToValue(lir, LMegamorphicLoadSlotByValue::IdIndex);
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+ ValueOperand output = ToOutValue(lir);
+
+ Label bail, cacheHit;
+ if (JitOptions.enableWatchtowerMegamorphic) {
+ masm.emitMegamorphicCacheLookupByValue(idVal, obj, temp0, temp1, temp2,
+ output, &cacheHit);
+ }
+
+ masm.branchIfNonNativeObj(obj, temp0, &bail);
+
+ // idVal will be in vp[0], result will be stored in vp[1].
+ masm.reserveStack(sizeof(Value));
+ masm.Push(idVal);
+ masm.moveStackPtrTo(temp0);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, Value * vp);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp1);
+ masm.passABIArg(temp1);
+ masm.passABIArg(obj);
+ masm.passABIArg(temp0);
+ masm.callWithABI<Fn, GetNativeDataPropertyByValuePure>();
+
+ MOZ_ASSERT(!idVal.aliases(temp0));
+ masm.storeCallPointerResult(temp0);
+ masm.Pop(idVal);
+
+ uint32_t framePushed = masm.framePushed();
+ Label ok;
+ masm.branchIfTrueBool(temp0, &ok);
+ masm.freeStack(sizeof(Value)); // Discard result Value.
+ masm.jump(&bail);
+
+ masm.bind(&ok);
+ masm.setFramePushed(framePushed);
+ masm.Pop(output);
+
+ masm.bind(&cacheHit);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitMegamorphicStoreSlot(LMegamorphicStoreSlot* lir) {
+ Register obj = ToRegister(lir->object());
+ ValueOperand rhs = ToValue(lir, LMegamorphicStoreSlot::RhsIndex);
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+
+ masm.Push(rhs);
+ masm.moveStackPtrTo(temp0);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, PropertyName * name,
+ Value * val);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp1);
+ masm.passABIArg(temp1);
+ masm.passABIArg(obj);
+ masm.movePtr(ImmGCPtr(lir->mir()->name()), temp2);
+ masm.passABIArg(temp2);
+ masm.passABIArg(temp0);
+ masm.callWithABI<Fn, SetNativeDataPropertyPure>();
+
+ MOZ_ASSERT(!rhs.aliases(temp0));
+ masm.storeCallPointerResult(temp0);
+ masm.Pop(rhs);
+
+ bailoutIfFalseBool(temp0, lir->snapshot());
+}
+
+void CodeGenerator::visitMegamorphicHasProp(LMegamorphicHasProp* lir) {
+ Register obj = ToRegister(lir->object());
+ ValueOperand idVal = ToValue(lir, LMegamorphicHasProp::IdIndex);
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+ Register output = ToRegister(lir->output());
+
+ Label bail, cacheHit;
+ if (JitOptions.enableWatchtowerMegamorphic) {
+ masm.emitMegamorphicCacheLookupExists(idVal, obj, temp0, temp1, temp2,
+ output, &cacheHit,
+ lir->mir()->hasOwn());
+ }
+
+ masm.branchIfNonNativeObj(obj, temp0, &bail);
+
+ // idVal will be in vp[0], result will be stored in vp[1].
+ masm.reserveStack(sizeof(Value));
+ masm.Push(idVal);
+ masm.moveStackPtrTo(temp0);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, Value * vp);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp1);
+ masm.passABIArg(temp1);
+ masm.passABIArg(obj);
+ masm.passABIArg(temp0);
+ if (lir->mir()->hasOwn()) {
+ masm.callWithABI<Fn, HasNativeDataPropertyPure<true>>();
+ } else {
+ masm.callWithABI<Fn, HasNativeDataPropertyPure<false>>();
+ }
+
+ MOZ_ASSERT(!idVal.aliases(temp0));
+ masm.storeCallPointerResult(temp0);
+ masm.Pop(idVal);
+
+ uint32_t framePushed = masm.framePushed();
+ Label ok;
+ masm.branchIfTrueBool(temp0, &ok);
+ masm.freeStack(sizeof(Value)); // Discard result Value.
+ masm.jump(&bail);
+
+ masm.bind(&ok);
+ masm.setFramePushed(framePushed);
+ masm.unboxBoolean(Address(masm.getStackPointer(), 0), output);
+ masm.freeStack(sizeof(Value));
+ masm.bind(&cacheHit);
+
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardIsNotArrayBufferMaybeShared(
+ LGuardIsNotArrayBufferMaybeShared* guard) {
+ Register obj = ToRegister(guard->input());
+ Register temp = ToRegister(guard->temp0());
+
+ Label bail;
+ masm.loadObjClassUnsafe(obj, temp);
+ masm.branchPtr(Assembler::Equal, temp, ImmPtr(&ArrayBufferObject::class_),
+ &bail);
+ masm.branchPtr(Assembler::Equal, temp,
+ ImmPtr(&SharedArrayBufferObject::class_), &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardIsTypedArray(LGuardIsTypedArray* guard) {
+ Register obj = ToRegister(guard->input());
+ Register temp = ToRegister(guard->temp0());
+
+ Label bail;
+ masm.loadObjClassUnsafe(obj, temp);
+ masm.branchIfClassIsNotTypedArray(temp, &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardObjectIdentity(LGuardObjectIdentity* guard) {
+ Register input = ToRegister(guard->input());
+ Register expected = ToRegister(guard->expected());
+
+ Assembler::Condition cond =
+ guard->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;
+ bailoutCmpPtr(cond, input, expected, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardSpecificFunction(LGuardSpecificFunction* guard) {
+ Register input = ToRegister(guard->input());
+ Register expected = ToRegister(guard->expected());
+
+ bailoutCmpPtr(Assembler::NotEqual, input, expected, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardSpecificAtom(LGuardSpecificAtom* guard) {
+ Register str = ToRegister(guard->str());
+ Register scratch = ToRegister(guard->temp0());
+
+ LiveRegisterSet volatileRegs = liveVolatileRegs(guard);
+ volatileRegs.takeUnchecked(scratch);
+
+ Label bail;
+ masm.guardSpecificAtom(str, guard->mir()->atom(), scratch, volatileRegs,
+ &bail);
+ bailoutFrom(&bail, guard->snapshot());
+}
+
+void CodeGenerator::visitGuardSpecificSymbol(LGuardSpecificSymbol* guard) {
+ Register symbol = ToRegister(guard->symbol());
+
+ bailoutCmpPtr(Assembler::NotEqual, symbol, ImmGCPtr(guard->mir()->expected()),
+ guard->snapshot());
+}
+
+void CodeGenerator::visitGuardSpecificInt32(LGuardSpecificInt32* guard) {
+ Register num = ToRegister(guard->num());
+
+ bailoutCmp32(Assembler::NotEqual, num, Imm32(guard->mir()->expected()),
+ guard->snapshot());
+}
+
+void CodeGenerator::visitGuardStringToIndex(LGuardStringToIndex* lir) {
+ Register str = ToRegister(lir->string());
+ Register output = ToRegister(lir->output());
+
+ Label vmCall, done;
+ masm.loadStringIndexValue(str, output, &vmCall);
+ masm.jump(&done);
+
+ {
+ masm.bind(&vmCall);
+
+ LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
+ volatileRegs.takeUnchecked(output);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = int32_t (*)(JSString * str);
+ masm.setupAlignedABICall();
+ masm.passABIArg(str);
+ masm.callWithABI<Fn, GetIndexFromString>();
+ masm.storeCallInt32Result(output);
+
+ masm.PopRegsInMask(volatileRegs);
+
+ // GetIndexFromString returns a negative value on failure.
+ bailoutTest32(Assembler::Signed, output, output, lir->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGuardStringToInt32(LGuardStringToInt32* lir) {
+ Register str = ToRegister(lir->string());
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
+
+ Label bail;
+ masm.guardStringToInt32(str, output, temp, volatileRegs, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardStringToDouble(LGuardStringToDouble* lir) {
+ Register str = ToRegister(lir->string());
+ FloatRegister output = ToFloatRegister(lir->output());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ Label vmCall, done;
+ // Use indexed value as fast path if possible.
+ masm.loadStringIndexValue(str, temp0, &vmCall);
+ masm.convertInt32ToDouble(temp0, output);
+ masm.jump(&done);
+ {
+ masm.bind(&vmCall);
+
+ // Reserve stack for holding the result value of the call.
+ masm.reserveStack(sizeof(double));
+ masm.moveStackPtrTo(temp0);
+
+ LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
+ volatileRegs.takeUnchecked(temp0);
+ volatileRegs.takeUnchecked(temp1);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSString * str, double* result);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp1);
+ masm.passABIArg(temp1);
+ masm.passABIArg(str);
+ masm.passABIArg(temp0);
+ masm.callWithABI<Fn, StringToNumberPure>();
+ masm.storeCallPointerResult(temp0);
+
+ masm.PopRegsInMask(volatileRegs);
+
+ Label ok;
+ masm.branchIfTrueBool(temp0, &ok);
+ {
+ // OOM path, recovered by StringToNumberPure.
+ //
+ // Use addToStackPtr instead of freeStack as freeStack tracks stack height
+ // flow-insensitively, and using it here would confuse the stack height
+ // tracking.
+ masm.addToStackPtr(Imm32(sizeof(double)));
+ bailout(lir->snapshot());
+ }
+ masm.bind(&ok);
+ masm.Pop(output);
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGuardNoDenseElements(LGuardNoDenseElements* guard) {
+ Register obj = ToRegister(guard->input());
+ Register temp = ToRegister(guard->temp0());
+
+ // Load obj->elements.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), temp);
+
+ // Make sure there are no dense elements.
+ Address initLength(temp, ObjectElements::offsetOfInitializedLength());
+ bailoutCmp32(Assembler::NotEqual, initLength, Imm32(0), guard->snapshot());
+}
+
+void CodeGenerator::visitBooleanToInt64(LBooleanToInt64* lir) {
+ Register input = ToRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ masm.move32To64ZeroExtend(input, output);
+}
+
+void CodeGenerator::emitStringToInt64(LInstruction* lir, Register input,
+ Register64 output) {
+ Register temp = output.scratchReg();
+
+ saveLive(lir);
+
+ masm.reserveStack(sizeof(uint64_t));
+ masm.moveStackPtrTo(temp);
+ pushArg(temp);
+ pushArg(input);
+
+ using Fn = bool (*)(JSContext*, HandleString, uint64_t*);
+ callVM<Fn, DoStringToInt64>(lir);
+
+ masm.load64(Address(masm.getStackPointer(), 0), output);
+ masm.freeStack(sizeof(uint64_t));
+
+ restoreLiveIgnore(lir, StoreValueTo(output).clobbered());
+}
+
+void CodeGenerator::visitStringToInt64(LStringToInt64* lir) {
+ Register input = ToRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ emitStringToInt64(lir, input, output);
+}
+
+void CodeGenerator::visitValueToInt64(LValueToInt64* lir) {
+ ValueOperand input = ToValue(lir, LValueToInt64::InputIndex);
+ Register temp = ToRegister(lir->temp0());
+ Register64 output = ToOutRegister64(lir);
+
+ int checks = 3;
+
+ Label fail, done;
+ // Jump to fail if this is the last check and we fail it,
+ // otherwise to the next test.
+ auto emitTestAndUnbox = [&](auto testAndUnbox) {
+ MOZ_ASSERT(checks > 0);
+
+ checks--;
+ Label notType;
+ Label* target = checks ? &notType : &fail;
+
+ testAndUnbox(target);
+
+ if (checks) {
+ masm.jump(&done);
+ masm.bind(&notType);
+ }
+ };
+
+ Register tag = masm.extractTag(input, temp);
+
+ // BigInt.
+ emitTestAndUnbox([&](Label* target) {
+ masm.branchTestBigInt(Assembler::NotEqual, tag, target);
+ masm.unboxBigInt(input, temp);
+ masm.loadBigInt64(temp, output);
+ });
+
+ // Boolean
+ emitTestAndUnbox([&](Label* target) {
+ masm.branchTestBoolean(Assembler::NotEqual, tag, target);
+ masm.unboxBoolean(input, temp);
+ masm.move32To64ZeroExtend(temp, output);
+ });
+
+ // String
+ emitTestAndUnbox([&](Label* target) {
+ masm.branchTestString(Assembler::NotEqual, tag, target);
+ masm.unboxString(input, temp);
+ emitStringToInt64(lir, temp, output);
+ });
+
+ MOZ_ASSERT(checks == 0);
+
+ bailoutFrom(&fail, lir->snapshot());
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitTruncateBigIntToInt64(LTruncateBigIntToInt64* lir) {
+ Register operand = ToRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ masm.loadBigInt64(operand, output);
+}
+
+OutOfLineCode* CodeGenerator::createBigIntOutOfLine(LInstruction* lir,
+ Scalar::Type type,
+ Register64 input,
+ Register output) {
+#if JS_BITS_PER_WORD == 32
+ using Fn = BigInt* (*)(JSContext*, uint32_t, uint32_t);
+ auto args = ArgList(input.low, input.high);
+#else
+ using Fn = BigInt* (*)(JSContext*, uint64_t);
+ auto args = ArgList(input);
+#endif
+
+ if (type == Scalar::BigInt64) {
+ return oolCallVM<Fn, jit::CreateBigIntFromInt64>(lir, args,
+ StoreRegisterTo(output));
+ }
+ MOZ_ASSERT(type == Scalar::BigUint64);
+ return oolCallVM<Fn, jit::CreateBigIntFromUint64>(lir, args,
+ StoreRegisterTo(output));
+}
+
+void CodeGenerator::emitCreateBigInt(LInstruction* lir, Scalar::Type type,
+ Register64 input, Register output,
+ Register maybeTemp) {
+ OutOfLineCode* ool = createBigIntOutOfLine(lir, type, input, output);
+
+ if (maybeTemp != InvalidReg) {
+ masm.newGCBigInt(output, maybeTemp, initialBigIntHeap(), ool->entry());
+ } else {
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(input);
+ regs.take(output);
+
+ Register temp = regs.takeAny();
+
+ masm.push(temp);
+
+ Label fail, ok;
+ masm.newGCBigInt(output, temp, initialBigIntHeap(), &fail);
+ masm.pop(temp);
+ masm.jump(&ok);
+ masm.bind(&fail);
+ masm.pop(temp);
+ masm.jump(ool->entry());
+ masm.bind(&ok);
+ }
+ masm.initializeBigInt64(type, output, input);
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitInt64ToBigInt(LInt64ToBigInt* lir) {
+ Register64 input = ToRegister64(lir->input());
+ Register temp = ToRegister(lir->temp0());
+ Register output = ToRegister(lir->output());
+
+ emitCreateBigInt(lir, Scalar::BigInt64, input, output, temp);
+}
+
+void CodeGenerator::visitGuardValue(LGuardValue* lir) {
+ ValueOperand input = ToValue(lir, LGuardValue::InputIndex);
+ Value expected = lir->mir()->expected();
+ Label bail;
+ masm.branchTestValue(Assembler::NotEqual, input, expected, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardNullOrUndefined(LGuardNullOrUndefined* lir) {
+ ValueOperand input = ToValue(lir, LGuardNullOrUndefined::InputIndex);
+
+ ScratchTagScope tag(masm, input);
+ masm.splitTagForTest(input, tag);
+
+ Label done;
+ masm.branchTestNull(Assembler::Equal, tag, &done);
+
+ Label bail;
+ masm.branchTestUndefined(Assembler::NotEqual, tag, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGuardIsNotObject(LGuardIsNotObject* lir) {
+ ValueOperand input = ToValue(lir, LGuardIsNotObject::InputIndex);
+
+ Label bail;
+ masm.branchTestObject(Assembler::Equal, input, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardFunctionFlags(LGuardFunctionFlags* lir) {
+ Register function = ToRegister(lir->function());
+
+ Label bail;
+ if (uint16_t flags = lir->mir()->expectedFlags()) {
+ masm.branchTestFunctionFlags(function, flags, Assembler::Zero, &bail);
+ }
+ if (uint16_t flags = lir->mir()->unexpectedFlags()) {
+ masm.branchTestFunctionFlags(function, flags, Assembler::NonZero, &bail);
+ }
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardFunctionIsNonBuiltinCtor(
+ LGuardFunctionIsNonBuiltinCtor* lir) {
+ Register function = ToRegister(lir->function());
+ Register temp = ToRegister(lir->temp0());
+
+ Label bail;
+ masm.branchIfNotFunctionIsNonBuiltinCtor(function, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardFunctionKind(LGuardFunctionKind* lir) {
+ Register function = ToRegister(lir->function());
+ Register temp = ToRegister(lir->temp0());
+
+ Assembler::Condition cond =
+ lir->mir()->bailOnEquality() ? Assembler::Equal : Assembler::NotEqual;
+
+ Label bail;
+ masm.branchFunctionKind(cond, lir->mir()->expected(), function, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardFunctionScript(LGuardFunctionScript* lir) {
+ Register function = ToRegister(lir->function());
+
+ Address scriptAddr(function, JSFunction::offsetOfJitInfoOrScript());
+ bailoutCmpPtr(Assembler::NotEqual, scriptAddr,
+ ImmGCPtr(lir->mir()->expected()), lir->snapshot());
+}
+
+// Out-of-line path to update the store buffer.
+class OutOfLineCallPostWriteBarrier : public OutOfLineCodeBase<CodeGenerator> {
+ LInstruction* lir_;
+ const LAllocation* object_;
+
+ public:
+ OutOfLineCallPostWriteBarrier(LInstruction* lir, const LAllocation* object)
+ : lir_(lir), object_(object) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineCallPostWriteBarrier(this);
+ }
+
+ LInstruction* lir() const { return lir_; }
+ const LAllocation* object() const { return object_; }
+};
+
+static void EmitStoreBufferCheckForConstant(MacroAssembler& masm,
+ const gc::TenuredCell* cell,
+ AllocatableGeneralRegisterSet& regs,
+ Label* exit, Label* callVM) {
+ Register temp = regs.takeAny();
+
+ gc::Arena* arena = cell->arena();
+
+ Register cells = temp;
+ masm.loadPtr(AbsoluteAddress(&arena->bufferedCells()), cells);
+
+ size_t index = gc::ArenaCellSet::getCellIndex(cell);
+ size_t word;
+ uint32_t mask;
+ gc::ArenaCellSet::getWordIndexAndMask(index, &word, &mask);
+ size_t offset = gc::ArenaCellSet::offsetOfBits() + word * sizeof(uint32_t);
+
+ masm.branchTest32(Assembler::NonZero, Address(cells, offset), Imm32(mask),
+ exit);
+
+ // Check whether this is the sentinel set and if so call the VM to allocate
+ // one for this arena.
+ masm.branchPtr(Assembler::Equal,
+ Address(cells, gc::ArenaCellSet::offsetOfArena()),
+ ImmPtr(nullptr), callVM);
+
+ // Add the cell to the set.
+ masm.or32(Imm32(mask), Address(cells, offset));
+ masm.jump(exit);
+
+ regs.add(temp);
+}
+
+static void EmitPostWriteBarrier(MacroAssembler& masm, CompileRuntime* runtime,
+ Register objreg, JSObject* maybeConstant,
+ bool isGlobal,
+ AllocatableGeneralRegisterSet& regs) {
+ MOZ_ASSERT_IF(isGlobal, maybeConstant);
+
+ Label callVM;
+ Label exit;
+
+ // We already have a fast path to check whether a global is in the store
+ // buffer.
+ if (!isGlobal && maybeConstant) {
+ EmitStoreBufferCheckForConstant(masm, &maybeConstant->asTenured(), regs,
+ &exit, &callVM);
+ }
+
+ // Call into the VM to barrier the write.
+ masm.bind(&callVM);
+
+ Register runtimereg = regs.takeAny();
+ masm.mov(ImmPtr(runtime), runtimereg);
+
+ masm.setupAlignedABICall();
+ masm.passABIArg(runtimereg);
+ masm.passABIArg(objreg);
+ if (isGlobal) {
+ using Fn = void (*)(JSRuntime * rt, GlobalObject * obj);
+ masm.callWithABI<Fn, PostGlobalWriteBarrier>();
+ } else {
+ using Fn = void (*)(JSRuntime * rt, js::gc::Cell * obj);
+ masm.callWithABI<Fn, PostWriteBarrier>();
+ }
+
+ masm.bind(&exit);
+}
+
+void CodeGenerator::emitPostWriteBarrier(const LAllocation* obj) {
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
+
+ Register objreg;
+ JSObject* object = nullptr;
+ bool isGlobal = false;
+ if (obj->isConstant()) {
+ object = &obj->toConstant()->toObject();
+ isGlobal = isGlobalObject(object);
+ objreg = regs.takeAny();
+ masm.movePtr(ImmGCPtr(object), objreg);
+ } else {
+ objreg = ToRegister(obj);
+ regs.takeUnchecked(objreg);
+ }
+
+ EmitPostWriteBarrier(masm, gen->runtime, objreg, object, isGlobal, regs);
+}
+
+void CodeGenerator::emitPostWriteBarrier(Register objreg) {
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
+ regs.takeUnchecked(objreg);
+ EmitPostWriteBarrier(masm, gen->runtime, objreg, nullptr, false, regs);
+}
+
+void CodeGenerator::visitOutOfLineCallPostWriteBarrier(
+ OutOfLineCallPostWriteBarrier* ool) {
+ saveLiveVolatile(ool->lir());
+ const LAllocation* obj = ool->object();
+ emitPostWriteBarrier(obj);
+ restoreLiveVolatile(ool->lir());
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal,
+ OutOfLineCode* ool) {
+ // Check whether an object is a global that we have already barriered before
+ // calling into the VM.
+ //
+ // We only check for the script's global, not other globals within the same
+ // compartment, because we bake in a pointer to realm->globalWriteBarriered
+ // and doing that would be invalid for other realms because they could be
+ // collected before the Ion code is discarded.
+
+ if (!maybeGlobal->isConstant()) {
+ return;
+ }
+
+ JSObject* obj = &maybeGlobal->toConstant()->toObject();
+ if (gen->realm->maybeGlobal() != obj) {
+ return;
+ }
+
+ const uint32_t* addr = gen->realm->addressOfGlobalWriteBarriered();
+ masm.branch32(Assembler::NotEqual, AbsoluteAddress(addr), Imm32(0),
+ ool->rejoin());
+}
+
+template <class LPostBarrierType, MIRType nurseryType>
+void CodeGenerator::visitPostWriteBarrierCommon(LPostBarrierType* lir,
+ OutOfLineCode* ool) {
+ static_assert(NeedsPostBarrier(nurseryType));
+
+ addOutOfLineCode(ool, lir->mir());
+
+ Register temp = ToTempRegisterOrInvalid(lir->temp0());
+
+ if (lir->object()->isConstant()) {
+ // Constant nursery objects cannot appear here, see
+ // LIRGenerator::visitPostWriteElementBarrier.
+ MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()));
+ } else {
+ masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()),
+ temp, ool->rejoin());
+ }
+
+ maybeEmitGlobalBarrierCheck(lir->object(), ool);
+
+ Register value = ToRegister(lir->value());
+ if constexpr (nurseryType == MIRType::Object) {
+ MOZ_ASSERT(lir->mir()->value()->type() == MIRType::Object);
+ } else if constexpr (nurseryType == MIRType::String) {
+ MOZ_ASSERT(lir->mir()->value()->type() == MIRType::String);
+ } else {
+ static_assert(nurseryType == MIRType::BigInt);
+ MOZ_ASSERT(lir->mir()->value()->type() == MIRType::BigInt);
+ }
+ masm.branchPtrInNurseryChunk(Assembler::Equal, value, temp, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+template <class LPostBarrierType>
+void CodeGenerator::visitPostWriteBarrierCommonV(LPostBarrierType* lir,
+ OutOfLineCode* ool) {
+ addOutOfLineCode(ool, lir->mir());
+
+ Register temp = ToTempRegisterOrInvalid(lir->temp0());
+
+ if (lir->object()->isConstant()) {
+ // Constant nursery objects cannot appear here, see
+ // LIRGenerator::visitPostWriteElementBarrier.
+ MOZ_ASSERT(!IsInsideNursery(&lir->object()->toConstant()->toObject()));
+ } else {
+ masm.branchPtrInNurseryChunk(Assembler::Equal, ToRegister(lir->object()),
+ temp, ool->rejoin());
+ }
+
+ maybeEmitGlobalBarrierCheck(lir->object(), ool);
+
+ ValueOperand value = ToValue(lir, LPostBarrierType::ValueIndex);
+ masm.branchValueIsNurseryCell(Assembler::Equal, value, temp, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitPostWriteBarrierO(LPostWriteBarrierO* lir) {
+ auto ool = new (alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
+ visitPostWriteBarrierCommon<LPostWriteBarrierO, MIRType::Object>(lir, ool);
+}
+
+void CodeGenerator::visitPostWriteBarrierS(LPostWriteBarrierS* lir) {
+ auto ool = new (alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
+ visitPostWriteBarrierCommon<LPostWriteBarrierS, MIRType::String>(lir, ool);
+}
+
+void CodeGenerator::visitPostWriteBarrierBI(LPostWriteBarrierBI* lir) {
+ auto ool = new (alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
+ visitPostWriteBarrierCommon<LPostWriteBarrierBI, MIRType::BigInt>(lir, ool);
+}
+
+void CodeGenerator::visitPostWriteBarrierV(LPostWriteBarrierV* lir) {
+ auto ool = new (alloc()) OutOfLineCallPostWriteBarrier(lir, lir->object());
+ visitPostWriteBarrierCommonV(lir, ool);
+}
+
+// Out-of-line path to update the store buffer.
+class OutOfLineCallPostWriteElementBarrier
+ : public OutOfLineCodeBase<CodeGenerator> {
+ LInstruction* lir_;
+ const LAllocation* object_;
+ const LAllocation* index_;
+
+ public:
+ OutOfLineCallPostWriteElementBarrier(LInstruction* lir,
+ const LAllocation* object,
+ const LAllocation* index)
+ : lir_(lir), object_(object), index_(index) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineCallPostWriteElementBarrier(this);
+ }
+
+ LInstruction* lir() const { return lir_; }
+
+ const LAllocation* object() const { return object_; }
+
+ const LAllocation* index() const { return index_; }
+};
+
+void CodeGenerator::visitOutOfLineCallPostWriteElementBarrier(
+ OutOfLineCallPostWriteElementBarrier* ool) {
+ saveLiveVolatile(ool->lir());
+
+ const LAllocation* obj = ool->object();
+ const LAllocation* index = ool->index();
+
+ Register objreg = obj->isConstant() ? InvalidReg : ToRegister(obj);
+ Register indexreg = ToRegister(index);
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
+ regs.takeUnchecked(indexreg);
+
+ if (obj->isConstant()) {
+ objreg = regs.takeAny();
+ masm.movePtr(ImmGCPtr(&obj->toConstant()->toObject()), objreg);
+ } else {
+ regs.takeUnchecked(objreg);
+ }
+
+ Register runtimereg = regs.takeAny();
+ using Fn = void (*)(JSRuntime * rt, JSObject * obj, int32_t index);
+ masm.setupAlignedABICall();
+ masm.mov(ImmPtr(gen->runtime), runtimereg);
+ masm.passABIArg(runtimereg);
+ masm.passABIArg(objreg);
+ masm.passABIArg(indexreg);
+ masm.callWithABI<Fn, PostWriteElementBarrier<IndexInBounds::Maybe>>();
+
+ restoreLiveVolatile(ool->lir());
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitPostWriteElementBarrierO(
+ LPostWriteElementBarrierO* lir) {
+ auto ool = new (alloc())
+ OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
+ visitPostWriteBarrierCommon<LPostWriteElementBarrierO, MIRType::Object>(lir,
+ ool);
+}
+
+void CodeGenerator::visitPostWriteElementBarrierS(
+ LPostWriteElementBarrierS* lir) {
+ auto ool = new (alloc())
+ OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
+ visitPostWriteBarrierCommon<LPostWriteElementBarrierS, MIRType::String>(lir,
+ ool);
+}
+
+void CodeGenerator::visitPostWriteElementBarrierBI(
+ LPostWriteElementBarrierBI* lir) {
+ auto ool = new (alloc())
+ OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
+ visitPostWriteBarrierCommon<LPostWriteElementBarrierBI, MIRType::BigInt>(lir,
+ ool);
+}
+
+void CodeGenerator::visitPostWriteElementBarrierV(
+ LPostWriteElementBarrierV* lir) {
+ auto ool = new (alloc())
+ OutOfLineCallPostWriteElementBarrier(lir, lir->object(), lir->index());
+ visitPostWriteBarrierCommonV(lir, ool);
+}
+
+void CodeGenerator::visitAssertCanElidePostWriteBarrier(
+ LAssertCanElidePostWriteBarrier* lir) {
+ Register object = ToRegister(lir->object());
+ ValueOperand value =
+ ToValue(lir, LAssertCanElidePostWriteBarrier::ValueIndex);
+ Register temp = ToRegister(lir->temp0());
+
+ Label ok;
+ masm.branchPtrInNurseryChunk(Assembler::Equal, object, temp, &ok);
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, value, temp, &ok);
+
+ masm.assumeUnreachable("Unexpected missing post write barrier");
+
+ masm.bind(&ok);
+}
+
+void CodeGenerator::visitCallNative(LCallNative* call) {
+ WrappedFunction* target = call->getSingleTarget();
+ MOZ_ASSERT(target);
+ MOZ_ASSERT(target->isNativeWithoutJitEntry());
+
+ int unusedStack = UnusedStackBytesForCall(call->paddedNumStackArgs());
+
+ // Registers used for callWithABI() argument-passing.
+ const Register argContextReg = ToRegister(call->getArgContextReg());
+ const Register argUintNReg = ToRegister(call->getArgUintNReg());
+ const Register argVpReg = ToRegister(call->getArgVpReg());
+
+ // Misc. temporary registers.
+ const Register tempReg = ToRegister(call->getTempReg());
+
+ DebugOnly<uint32_t> initialStack = masm.framePushed();
+
+ masm.checkStackAlignment();
+
+ // Native functions have the signature:
+ // bool (*)(JSContext*, unsigned, Value* vp)
+ // Where vp[0] is space for an outparam, vp[1] is |this|, and vp[2] onward
+ // are the function arguments.
+
+ // Allocate space for the outparam, moving the StackPointer to what will be
+ // &vp[1].
+ masm.adjustStack(unusedStack);
+
+ // Push a Value containing the callee object: natives are allowed to access
+ // their callee before setting the return value. The StackPointer is moved
+ // to &vp[0].
+ masm.Push(ObjectValue(*target->rawNativeJSFunction()));
+
+ // Preload arguments into registers.
+ masm.loadJSContext(argContextReg);
+ masm.move32(Imm32(call->numActualArgs()), argUintNReg);
+ masm.moveStackPtrTo(argVpReg);
+
+ masm.Push(argUintNReg);
+
+ if (call->mir()->maybeCrossRealm()) {
+ masm.movePtr(ImmGCPtr(target->rawNativeJSFunction()), tempReg);
+ masm.switchToObjectRealm(tempReg, tempReg);
+ }
+
+ // Construct native exit frame.
+ uint32_t safepointOffset = masm.buildFakeExitFrame(tempReg);
+ masm.enterFakeExitFrameForNative(argContextReg, tempReg,
+ call->mir()->isConstructing());
+
+ markSafepointAt(safepointOffset, call);
+
+ // Construct and execute call.
+ masm.setupAlignedABICall();
+ masm.passABIArg(argContextReg);
+ masm.passABIArg(argUintNReg);
+ masm.passABIArg(argVpReg);
+ JSNative native = target->native();
+ if (call->ignoresReturnValue() && target->hasJitInfo()) {
+ const JSJitInfo* jitInfo = target->jitInfo();
+ if (jitInfo->type() == JSJitInfo::IgnoresReturnValueNative) {
+ native = jitInfo->ignoresReturnValueMethod;
+ }
+ }
+ masm.callWithABI(DynamicFunction<JSNative>(native), MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ masm.branchIfFalseBool(ReturnReg, masm.failureLabel());
+
+ if (call->mir()->maybeCrossRealm()) {
+ masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
+ }
+
+ // Load the outparam vp[0] into output register(s).
+ masm.loadValue(
+ Address(masm.getStackPointer(), NativeExitFrameLayout::offsetOfResult()),
+ JSReturnOperand);
+
+ // Until C++ code is instrumented against Spectre, prevent speculative
+ // execution from returning any private data.
+ if (JitOptions.spectreJitToCxxCalls && !call->mir()->ignoresReturnValue() &&
+ call->mir()->hasLiveDefUses()) {
+ masm.speculationBarrier();
+ }
+
+ // The next instruction is removing the footer of the exit frame, so there
+ // is no need for leaveFakeExitFrame.
+
+ // Move the StackPointer back to its original location, unwinding the native
+ // exit frame.
+ masm.adjustStack(NativeExitFrameLayout::Size() - unusedStack);
+ MOZ_ASSERT(masm.framePushed() == initialStack);
+}
+
+static void LoadDOMPrivate(MacroAssembler& masm, Register obj, Register priv,
+ DOMObjectKind kind) {
+ // Load the value in DOM_OBJECT_SLOT for a native or proxy DOM object. This
+ // will be in the first slot but may be fixed or non-fixed.
+ MOZ_ASSERT(obj != priv);
+
+ switch (kind) {
+ case DOMObjectKind::Native:
+ // If it's a native object, the value must be in a fixed slot.
+ masm.debugAssertObjHasFixedSlots(obj, priv);
+ masm.loadPrivate(Address(obj, NativeObject::getFixedSlotOffset(0)), priv);
+ break;
+ case DOMObjectKind::Proxy: {
+#ifdef DEBUG
+ // Sanity check: it must be a DOM proxy.
+ Label isDOMProxy;
+ masm.branchTestProxyHandlerFamily(
+ Assembler::Equal, obj, priv, GetDOMProxyHandlerFamily(), &isDOMProxy);
+ masm.assumeUnreachable("Expected a DOM proxy");
+ masm.bind(&isDOMProxy);
+#endif
+ masm.loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()), priv);
+ masm.loadPrivate(
+ Address(priv, js::detail::ProxyReservedSlots::offsetOfSlot(0)), priv);
+ break;
+ }
+ }
+}
+
+void CodeGenerator::visitCallDOMNative(LCallDOMNative* call) {
+ WrappedFunction* target = call->getSingleTarget();
+ MOZ_ASSERT(target);
+ MOZ_ASSERT(target->isNativeWithoutJitEntry());
+ MOZ_ASSERT(target->hasJitInfo());
+ MOZ_ASSERT(call->mir()->isCallDOMNative());
+
+ int unusedStack = UnusedStackBytesForCall(call->paddedNumStackArgs());
+
+ // Registers used for callWithABI() argument-passing.
+ const Register argJSContext = ToRegister(call->getArgJSContext());
+ const Register argObj = ToRegister(call->getArgObj());
+ const Register argPrivate = ToRegister(call->getArgPrivate());
+ const Register argArgs = ToRegister(call->getArgArgs());
+
+ DebugOnly<uint32_t> initialStack = masm.framePushed();
+
+ masm.checkStackAlignment();
+
+ // DOM methods have the signature:
+ // bool (*)(JSContext*, HandleObject, void* private, const
+ // JSJitMethodCallArgs& args)
+ // Where args is initialized from an argc and a vp, vp[0] is space for an
+ // outparam and the callee, vp[1] is |this|, and vp[2] onward are the
+ // function arguments. Note that args stores the argv, not the vp, and
+ // argv == vp + 2.
+
+ // Nestle the stack up against the pushed arguments, leaving StackPointer at
+ // &vp[1]
+ masm.adjustStack(unusedStack);
+ // argObj is filled with the extracted object, then returned.
+ Register obj = masm.extractObject(Address(masm.getStackPointer(), 0), argObj);
+ MOZ_ASSERT(obj == argObj);
+
+ // Push a Value containing the callee object: natives are allowed to access
+ // their callee before setting the return value. After this the StackPointer
+ // points to &vp[0].
+ masm.Push(ObjectValue(*target->rawNativeJSFunction()));
+
+ // Now compute the argv value. Since StackPointer is pointing to &vp[0] and
+ // argv is &vp[2] we just need to add 2*sizeof(Value) to the current
+ // StackPointer.
+ static_assert(JSJitMethodCallArgsTraits::offsetOfArgv == 0);
+ static_assert(JSJitMethodCallArgsTraits::offsetOfArgc ==
+ IonDOMMethodExitFrameLayoutTraits::offsetOfArgcFromArgv);
+ masm.computeEffectiveAddress(
+ Address(masm.getStackPointer(), 2 * sizeof(Value)), argArgs);
+
+ LoadDOMPrivate(masm, obj, argPrivate,
+ static_cast<MCallDOMNative*>(call->mir())->objectKind());
+
+ // Push argc from the call instruction into what will become the IonExitFrame
+ masm.Push(Imm32(call->numActualArgs()));
+
+ // Push our argv onto the stack
+ masm.Push(argArgs);
+ // And store our JSJitMethodCallArgs* in argArgs.
+ masm.moveStackPtrTo(argArgs);
+
+ // Push |this| object for passing HandleObject. We push after argc to
+ // maintain the same sp-relative location of the object pointer with other
+ // DOMExitFrames.
+ masm.Push(argObj);
+ masm.moveStackPtrTo(argObj);
+
+ if (call->mir()->maybeCrossRealm()) {
+ // We use argJSContext as scratch register here.
+ masm.movePtr(ImmGCPtr(target->rawNativeJSFunction()), argJSContext);
+ masm.switchToObjectRealm(argJSContext, argJSContext);
+ }
+
+ // Construct native exit frame.
+ uint32_t safepointOffset = masm.buildFakeExitFrame(argJSContext);
+ masm.loadJSContext(argJSContext);
+ masm.enterFakeExitFrame(argJSContext, argJSContext,
+ ExitFrameType::IonDOMMethod);
+
+ markSafepointAt(safepointOffset, call);
+
+ // Construct and execute call.
+ masm.setupAlignedABICall();
+ masm.loadJSContext(argJSContext);
+ masm.passABIArg(argJSContext);
+ masm.passABIArg(argObj);
+ masm.passABIArg(argPrivate);
+ masm.passABIArg(argArgs);
+ masm.callWithABI(DynamicFunction<JSJitMethodOp>(target->jitInfo()->method),
+ MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ if (target->jitInfo()->isInfallible) {
+ masm.loadValue(Address(masm.getStackPointer(),
+ IonDOMMethodExitFrameLayout::offsetOfResult()),
+ JSReturnOperand);
+ } else {
+ // Test for failure.
+ masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
+
+ // Load the outparam vp[0] into output register(s).
+ masm.loadValue(Address(masm.getStackPointer(),
+ IonDOMMethodExitFrameLayout::offsetOfResult()),
+ JSReturnOperand);
+ }
+
+ // Switch back to the current realm if needed. Note: if the DOM method threw
+ // an exception, the exception handler will do this.
+ if (call->mir()->maybeCrossRealm()) {
+ static_assert(!JSReturnOperand.aliases(ReturnReg),
+ "Clobbering ReturnReg should not affect the return value");
+ masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
+ }
+
+ // Until C++ code is instrumented against Spectre, prevent speculative
+ // execution from returning any private data.
+ if (JitOptions.spectreJitToCxxCalls && call->mir()->hasLiveDefUses()) {
+ masm.speculationBarrier();
+ }
+
+ // The next instruction is removing the footer of the exit frame, so there
+ // is no need for leaveFakeExitFrame.
+
+ // Move the StackPointer back to its original location, unwinding the native
+ // exit frame.
+ masm.adjustStack(IonDOMMethodExitFrameLayout::Size() - unusedStack);
+ MOZ_ASSERT(masm.framePushed() == initialStack);
+}
+
+void CodeGenerator::visitCallGetIntrinsicValue(LCallGetIntrinsicValue* lir) {
+ pushArg(ImmGCPtr(lir->mir()->name()));
+
+ using Fn =
+ bool (*)(JSContext * cx, Handle<PropertyName*>, MutableHandleValue);
+ callVM<Fn, GetIntrinsicValue>(lir);
+}
+
+void CodeGenerator::emitCallInvokeFunction(
+ LInstruction* call, Register calleereg, bool constructing,
+ bool ignoresReturnValue, uint32_t argc, uint32_t unusedStack) {
+ // Nestle %esp up to the argument vector.
+ // Each path must account for framePushed_ separately, for callVM to be valid.
+ masm.freeStack(unusedStack);
+
+ pushArg(masm.getStackPointer()); // argv.
+ pushArg(Imm32(argc)); // argc.
+ pushArg(Imm32(ignoresReturnValue));
+ pushArg(Imm32(constructing)); // constructing.
+ pushArg(calleereg); // JSFunction*.
+
+ using Fn = bool (*)(JSContext*, HandleObject, bool, bool, uint32_t, Value*,
+ MutableHandleValue);
+ callVM<Fn, jit::InvokeFunction>(call);
+
+ // Un-nestle %esp from the argument vector. No prefix was pushed.
+ masm.reserveStack(unusedStack);
+}
+
+void CodeGenerator::visitCallGeneric(LCallGeneric* call) {
+ Register calleereg = ToRegister(call->getFunction());
+ Register objreg = ToRegister(call->getTempObject());
+ Register nargsreg = ToRegister(call->getNargsReg());
+ uint32_t unusedStack = UnusedStackBytesForCall(call->paddedNumStackArgs());
+ Label invoke, thunk, makeCall, end;
+
+ // Known-target case is handled by LCallKnown.
+ MOZ_ASSERT(!call->hasSingleTarget());
+
+ masm.checkStackAlignment();
+
+ // Guard that calleereg is actually a function object.
+ if (call->mir()->needsClassCheck()) {
+ masm.branchTestObjIsFunction(Assembler::NotEqual, calleereg, nargsreg,
+ calleereg, &invoke);
+ }
+
+ // Guard that callee allows the [[Call]] or [[Construct]] operation required.
+ if (call->mir()->isConstructing()) {
+ masm.branchTestFunctionFlags(calleereg, FunctionFlags::CONSTRUCTOR,
+ Assembler::Zero, &invoke);
+ } else {
+ masm.branchFunctionKind(Assembler::Equal, FunctionFlags::ClassConstructor,
+ calleereg, objreg, &invoke);
+ }
+
+ // Use the slow path if CreateThis was unable to create the |this| object.
+ if (call->mir()->needsThisCheck()) {
+ MOZ_ASSERT(call->mir()->isConstructing());
+ Address thisAddr(masm.getStackPointer(), unusedStack);
+ masm.branchTestNull(Assembler::Equal, thisAddr, &invoke);
+ } else {
+#ifdef DEBUG
+ if (call->mir()->isConstructing()) {
+ Address thisAddr(masm.getStackPointer(), unusedStack);
+ Label ok;
+ masm.branchTestNull(Assembler::NotEqual, thisAddr, &ok);
+ masm.assumeUnreachable("Unexpected null this-value");
+ masm.bind(&ok);
+ }
+#endif
+ }
+
+ // Load jitCodeRaw for callee if it exists.
+ masm.branchIfFunctionHasNoJitEntry(calleereg, call->mir()->isConstructing(),
+ &invoke);
+ masm.loadJitCodeRaw(calleereg, objreg);
+
+ // Target may be a different realm even if same compartment.
+ if (call->mir()->maybeCrossRealm()) {
+ masm.switchToObjectRealm(calleereg, nargsreg);
+ }
+
+ // Nestle the StackPointer up to the argument vector.
+ masm.freeStack(unusedStack);
+
+ // Construct the JitFrameLayout.
+ masm.PushCalleeToken(calleereg, call->mir()->isConstructing());
+ masm.PushFrameDescriptorForJitCall(FrameType::IonJS, call->numActualArgs());
+
+ // Check whether the provided arguments satisfy target argc.
+ // We cannot have lowered to LCallGeneric with a known target. Assert that we
+ // didn't add any undefineds in WarpBuilder. NB: MCall::numStackArgs includes
+ // |this|.
+ DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing();
+ MOZ_ASSERT(call->numActualArgs() ==
+ call->mir()->numStackArgs() - numNonArgsOnStack);
+ masm.loadFunctionArgCount(calleereg, nargsreg);
+ masm.branch32(Assembler::Above, nargsreg, Imm32(call->numActualArgs()),
+ &thunk);
+ masm.jump(&makeCall);
+
+ // Argument fixup needed. Load the ArgumentsRectifier.
+ masm.bind(&thunk);
+ {
+ TrampolinePtr argumentsRectifier =
+ gen->jitRuntime()->getArgumentsRectifier();
+ masm.movePtr(argumentsRectifier, objreg);
+ }
+
+ // Finally call the function in objreg.
+ masm.bind(&makeCall);
+ uint32_t callOffset = masm.callJit(objreg);
+ markSafepointAt(callOffset, call);
+
+ if (call->mir()->maybeCrossRealm()) {
+ static_assert(!JSReturnOperand.aliases(ReturnReg),
+ "ReturnReg available as scratch after scripted calls");
+ masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
+ }
+
+ // Restore stack pointer: pop JitFrameLayout fields still left on the stack
+ // and undo the earlier |freeStack(unusedStack)|.
+ int prefixGarbage =
+ sizeof(JitFrameLayout) - JitFrameLayout::bytesPoppedAfterCall();
+ masm.adjustStack(prefixGarbage - unusedStack);
+ masm.jump(&end);
+
+ // Handle uncompiled or native functions.
+ masm.bind(&invoke);
+ emitCallInvokeFunction(call, calleereg, call->isConstructing(),
+ call->ignoresReturnValue(), call->numActualArgs(),
+ unusedStack);
+
+ masm.bind(&end);
+
+ // If the return value of the constructing function is Primitive,
+ // replace the return value with the Object from CreateThis.
+ if (call->mir()->isConstructing()) {
+ Label notPrimitive;
+ masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
+ &notPrimitive);
+ masm.loadValue(Address(masm.getStackPointer(), unusedStack),
+ JSReturnOperand);
+#ifdef DEBUG
+ masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
+ &notPrimitive);
+ masm.assumeUnreachable("CreateThis creates an object");
+#endif
+ masm.bind(&notPrimitive);
+ }
+}
+
+void CodeGenerator::visitCallKnown(LCallKnown* call) {
+ Register calleereg = ToRegister(call->getFunction());
+ Register objreg = ToRegister(call->getTempObject());
+ uint32_t unusedStack = UnusedStackBytesForCall(call->paddedNumStackArgs());
+ WrappedFunction* target = call->getSingleTarget();
+
+ // Native single targets (except wasm) are handled by LCallNative.
+ MOZ_ASSERT(target->hasJitEntry());
+
+ // Missing arguments must have been explicitly appended by WarpBuilder.
+ DebugOnly<unsigned> numNonArgsOnStack = 1 + call->isConstructing();
+ MOZ_ASSERT(target->nargs() <=
+ call->mir()->numStackArgs() - numNonArgsOnStack);
+
+ MOZ_ASSERT_IF(call->isConstructing(), target->isConstructor());
+
+ masm.checkStackAlignment();
+
+ if (target->isClassConstructor() && !call->isConstructing()) {
+ emitCallInvokeFunction(call, calleereg, call->isConstructing(),
+ call->ignoresReturnValue(), call->numActualArgs(),
+ unusedStack);
+ return;
+ }
+
+ MOZ_ASSERT_IF(target->isClassConstructor(), call->isConstructing());
+
+ MOZ_ASSERT(!call->mir()->needsThisCheck());
+
+ if (call->mir()->maybeCrossRealm()) {
+ masm.switchToObjectRealm(calleereg, objreg);
+ }
+
+ masm.loadJitCodeRaw(calleereg, objreg);
+
+ // Nestle the StackPointer up to the argument vector.
+ masm.freeStack(unusedStack);
+
+ // Construct the JitFrameLayout.
+ masm.PushCalleeToken(calleereg, call->mir()->isConstructing());
+ masm.PushFrameDescriptorForJitCall(FrameType::IonJS, call->numActualArgs());
+
+ // Finally call the function in objreg.
+ uint32_t callOffset = masm.callJit(objreg);
+ markSafepointAt(callOffset, call);
+
+ if (call->mir()->maybeCrossRealm()) {
+ static_assert(!JSReturnOperand.aliases(ReturnReg),
+ "ReturnReg available as scratch after scripted calls");
+ masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
+ }
+
+ // Restore stack pointer: pop JitFrameLayout fields still left on the stack
+ // and undo the earlier |freeStack(unusedStack)|.
+ int prefixGarbage =
+ sizeof(JitFrameLayout) - JitFrameLayout::bytesPoppedAfterCall();
+ masm.adjustStack(prefixGarbage - unusedStack);
+
+ // If the return value of the constructing function is Primitive,
+ // replace the return value with the Object from CreateThis.
+ if (call->mir()->isConstructing()) {
+ Label notPrimitive;
+ masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
+ &notPrimitive);
+ masm.loadValue(Address(masm.getStackPointer(), unusedStack),
+ JSReturnOperand);
+#ifdef DEBUG
+ masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
+ &notPrimitive);
+ masm.assumeUnreachable("CreateThis creates an object");
+#endif
+ masm.bind(&notPrimitive);
+ }
+}
+
+template <typename T>
+void CodeGenerator::emitCallInvokeFunction(T* apply) {
+ Register objreg = ToRegister(apply->getTempObject());
+
+ // Push the space used by the arguments.
+ masm.moveStackPtrTo(objreg);
+
+ pushArg(objreg); // argv.
+ pushArg(ToRegister(apply->getArgc())); // argc.
+ pushArg(Imm32(apply->mir()->ignoresReturnValue())); // ignoresReturnValue.
+ pushArg(Imm32(apply->mir()->isConstructing())); // isConstructing.
+ pushArg(ToRegister(apply->getFunction())); // JSFunction*.
+
+ using Fn = bool (*)(JSContext*, HandleObject, bool, bool, uint32_t, Value*,
+ MutableHandleValue);
+ callVM<Fn, jit::InvokeFunction>(apply);
+}
+
+// Do not bailout after the execution of this function since the stack no longer
+// correspond to what is expected by the snapshots.
+void CodeGenerator::emitAllocateSpaceForApply(Register argcreg,
+ Register scratch) {
+ // Use scratch register to calculate stack space (including padding).
+ masm.movePtr(argcreg, scratch);
+
+ // Align the JitFrameLayout on the JitStackAlignment.
+ if (JitStackValueAlignment > 1) {
+ MOZ_ASSERT(frameSize() % JitStackAlignment == 0,
+ "Stack padding assumes that the frameSize is correct");
+ MOZ_ASSERT(JitStackValueAlignment == 2);
+ Label noPaddingNeeded;
+ // if the number of arguments is odd, then we do not need any padding.
+ masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded);
+ masm.addPtr(Imm32(1), scratch);
+ masm.bind(&noPaddingNeeded);
+ }
+
+ // Reserve space for copying the arguments.
+ NativeObject::elementsSizeMustNotOverflow();
+ masm.lshiftPtr(Imm32(ValueShift), scratch);
+ masm.subFromStackPtr(scratch);
+
+#ifdef DEBUG
+ // Put a magic value in the space reserved for padding. Note, this code
+ // cannot be merged with the previous test, as not all architectures can
+ // write below their stack pointers.
+ if (JitStackValueAlignment > 1) {
+ MOZ_ASSERT(JitStackValueAlignment == 2);
+ Label noPaddingNeeded;
+ // if the number of arguments is odd, then we do not need any padding.
+ masm.branchTestPtr(Assembler::NonZero, argcreg, Imm32(1), &noPaddingNeeded);
+ BaseValueIndex dstPtr(masm.getStackPointer(), argcreg);
+ masm.storeValue(MagicValue(JS_ARG_POISON), dstPtr);
+ masm.bind(&noPaddingNeeded);
+ }
+#endif
+}
+
+// Do not bailout after the execution of this function since the stack no longer
+// correspond to what is expected by the snapshots.
+void CodeGenerator::emitAllocateSpaceForConstructAndPushNewTarget(
+ Register argcreg, Register newTargetAndScratch) {
+ // Align the JitFrameLayout on the JitStackAlignment. Contrary to
+ // |emitAllocateSpaceForApply()|, we're always pushing a magic value, because
+ // we can't write to |newTargetAndScratch| before |new.target| has
+ // been pushed onto the stack.
+ if (JitStackValueAlignment > 1) {
+ MOZ_ASSERT(frameSize() % JitStackAlignment == 0,
+ "Stack padding assumes that the frameSize is correct");
+ MOZ_ASSERT(JitStackValueAlignment == 2);
+
+ Label noPaddingNeeded;
+ // If the number of arguments is even, then we do not need any padding.
+ masm.branchTestPtr(Assembler::Zero, argcreg, Imm32(1), &noPaddingNeeded);
+ masm.pushValue(MagicValue(JS_ARG_POISON));
+ masm.bind(&noPaddingNeeded);
+ }
+
+ // Push |new.target| after the padding value, but before any arguments.
+ masm.pushValue(JSVAL_TYPE_OBJECT, newTargetAndScratch);
+
+ // Use newTargetAndScratch to calculate stack space (including padding).
+ masm.movePtr(argcreg, newTargetAndScratch);
+
+ // Reserve space for copying the arguments.
+ NativeObject::elementsSizeMustNotOverflow();
+ masm.lshiftPtr(Imm32(ValueShift), newTargetAndScratch);
+ masm.subFromStackPtr(newTargetAndScratch);
+}
+
+// Destroys argvIndex and copyreg.
+void CodeGenerator::emitCopyValuesForApply(Register argvSrcBase,
+ Register argvIndex, Register copyreg,
+ size_t argvSrcOffset,
+ size_t argvDstOffset) {
+ Label loop;
+ masm.bind(&loop);
+
+ // As argvIndex is off by 1, and we use the decBranchPtr instruction
+ // to loop back, we have to substract the size of the word which are
+ // copied.
+ BaseValueIndex srcPtr(argvSrcBase, argvIndex,
+ int32_t(argvSrcOffset) - sizeof(void*));
+ BaseValueIndex dstPtr(masm.getStackPointer(), argvIndex,
+ int32_t(argvDstOffset) - sizeof(void*));
+ masm.loadPtr(srcPtr, copyreg);
+ masm.storePtr(copyreg, dstPtr);
+
+ // Handle 32 bits architectures.
+ if (sizeof(Value) == 2 * sizeof(void*)) {
+ BaseValueIndex srcPtrLow(argvSrcBase, argvIndex,
+ int32_t(argvSrcOffset) - 2 * sizeof(void*));
+ BaseValueIndex dstPtrLow(masm.getStackPointer(), argvIndex,
+ int32_t(argvDstOffset) - 2 * sizeof(void*));
+ masm.loadPtr(srcPtrLow, copyreg);
+ masm.storePtr(copyreg, dstPtrLow);
+ }
+
+ masm.decBranchPtr(Assembler::NonZero, argvIndex, Imm32(1), &loop);
+}
+
+void CodeGenerator::emitRestoreStackPointerFromFP() {
+ // This is used to restore the stack pointer after a call with a dynamic
+ // number of arguments.
+
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+
+ int32_t offset = -int32_t(frameSize());
+ masm.computeEffectiveAddress(Address(FramePointer, offset),
+ masm.getStackPointer());
+}
+
+void CodeGenerator::emitPushArguments(Register argcreg, Register scratch,
+ Register copyreg, uint32_t extraFormals) {
+ Label end;
+
+ // Skip the copy of arguments if there are none.
+ masm.branchTestPtr(Assembler::Zero, argcreg, argcreg, &end);
+
+ // clang-format off
+ //
+ // We are making a copy of the arguments which are above the JitFrameLayout
+ // of the current Ion frame.
+ //
+ // [arg1] [arg0] <- src [this] [JitFrameLayout] [.. frameSize ..] [pad] [arg1] [arg0] <- dst
+ //
+ // clang-format on
+
+ // Compute the source and destination offsets into the stack.
+ Register argvSrcBase = FramePointer;
+ size_t argvSrcOffset =
+ JitFrameLayout::offsetOfActualArgs() + extraFormals * sizeof(JS::Value);
+ size_t argvDstOffset = 0;
+
+ Register argvIndex = scratch;
+ masm.move32(argcreg, argvIndex);
+
+ // Copy arguments.
+ emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, argvSrcOffset,
+ argvDstOffset);
+
+ // Join with all arguments copied and the extra stack usage computed.
+ masm.bind(&end);
+}
+
+void CodeGenerator::emitPushArguments(LApplyArgsGeneric* apply,
+ Register scratch) {
+ // Holds the function nargs. Initially the number of args to the caller.
+ Register argcreg = ToRegister(apply->getArgc());
+ Register copyreg = ToRegister(apply->getTempObject());
+ uint32_t extraFormals = apply->numExtraFormals();
+
+ emitAllocateSpaceForApply(argcreg, scratch);
+
+ emitPushArguments(argcreg, scratch, copyreg, extraFormals);
+
+ // Push |this|.
+ masm.pushValue(ToValue(apply, LApplyArgsGeneric::ThisIndex));
+}
+
+void CodeGenerator::emitPushArguments(LApplyArgsObj* apply, Register scratch) {
+ // argc and argsObj are mapped to the same calltemp register.
+ MOZ_ASSERT(apply->getArgsObj() == apply->getArgc());
+
+ Register tmpArgc = ToRegister(apply->getTempObject());
+ Register argsObj = ToRegister(apply->getArgsObj());
+
+ // Load argc into tmpArgc.
+ Address lengthAddr(argsObj, ArgumentsObject::getInitialLengthSlotOffset());
+ masm.unboxInt32(lengthAddr, tmpArgc);
+ masm.rshift32(Imm32(ArgumentsObject::PACKED_BITS_COUNT), tmpArgc);
+
+ // Allocate space on the stack for arguments. This modifies scratch.
+ emitAllocateSpaceForApply(tmpArgc, scratch);
+
+ // Load arguments data
+ masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()),
+ argsObj);
+ size_t argsSrcOffset = ArgumentsData::offsetOfArgs();
+
+ // This is the end of the lifetime of argsObj.
+ // After this call, the argsObj register holds the argument count instead.
+ emitPushArrayAsArguments(tmpArgc, argsObj, scratch, argsSrcOffset);
+
+ masm.pushValue(ToValue(apply, LApplyArgsObj::ThisIndex));
+}
+
+void CodeGenerator::emitPushArrayAsArguments(Register tmpArgc,
+ Register srcBaseAndArgc,
+ Register scratch,
+ size_t argvSrcOffset) {
+ // Preconditions:
+ // 1. |tmpArgc| * sizeof(Value) bytes have been allocated at the top of
+ // the stack to hold arguments.
+ // 2. |srcBaseAndArgc| + |srcOffset| points to an array of |tmpArgc| values.
+ //
+ // Postconditions:
+ // 1. The arguments at |srcBaseAndArgc| + |srcOffset| have been copied into
+ // the allocated space.
+ // 2. |srcBaseAndArgc| now contains the original value of |tmpArgc|.
+ //
+ // |scratch| is used as a temp register within this function and clobbered.
+
+ Label noCopy, epilogue;
+
+ // Skip the copy of arguments if there are none.
+ masm.branchTestPtr(Assembler::Zero, tmpArgc, tmpArgc, &noCopy);
+
+ // Copy the values. This code is skipped entirely if there are
+ // no values.
+ size_t argvDstOffset = 0;
+
+ Register argvSrcBase = srcBaseAndArgc;
+ Register copyreg = scratch;
+
+ masm.push(tmpArgc);
+ Register argvIndex = tmpArgc;
+ argvDstOffset += sizeof(void*);
+
+ // Copy
+ emitCopyValuesForApply(argvSrcBase, argvIndex, copyreg, argvSrcOffset,
+ argvDstOffset);
+
+ // Restore.
+ masm.pop(srcBaseAndArgc); // srcBaseAndArgc now contains argc.
+ masm.jump(&epilogue);
+
+ // Clear argc if we skipped the copy step.
+ masm.bind(&noCopy);
+ masm.movePtr(ImmWord(0), srcBaseAndArgc);
+
+ // Join with all arguments copied and the extra stack usage computed.
+ // Note, "srcBase" has become "argc".
+ masm.bind(&epilogue);
+}
+
+void CodeGenerator::emitPushArguments(LApplyArrayGeneric* apply,
+ Register scratch) {
+ Register tmpArgc = ToRegister(apply->getTempObject());
+ Register elementsAndArgc = ToRegister(apply->getElements());
+
+ // Invariants guarded in the caller:
+ // - the array is not too long
+ // - the array length equals its initialized length
+
+ // The array length is our argc for the purposes of allocating space.
+ Address length(ToRegister(apply->getElements()),
+ ObjectElements::offsetOfLength());
+ masm.load32(length, tmpArgc);
+
+ // Allocate space for the values.
+ emitAllocateSpaceForApply(tmpArgc, scratch);
+
+ // After this call "elements" has become "argc".
+ size_t elementsOffset = 0;
+ emitPushArrayAsArguments(tmpArgc, elementsAndArgc, scratch, elementsOffset);
+
+ // Push |this|.
+ masm.pushValue(ToValue(apply, LApplyArrayGeneric::ThisIndex));
+}
+
+void CodeGenerator::emitPushArguments(LConstructArgsGeneric* construct,
+ Register scratch) {
+ MOZ_ASSERT(scratch == ToRegister(construct->getNewTarget()));
+
+ // Holds the function nargs. Initially the number of args to the caller.
+ Register argcreg = ToRegister(construct->getArgc());
+ Register copyreg = ToRegister(construct->getTempObject());
+ uint32_t extraFormals = construct->numExtraFormals();
+
+ // Allocate space for the values.
+ // After this call "newTarget" has become "scratch".
+ emitAllocateSpaceForConstructAndPushNewTarget(argcreg, scratch);
+
+ emitPushArguments(argcreg, scratch, copyreg, extraFormals);
+
+ // Push |this|.
+ masm.pushValue(ToValue(construct, LConstructArgsGeneric::ThisIndex));
+}
+
+void CodeGenerator::emitPushArguments(LConstructArrayGeneric* construct,
+ Register scratch) {
+ MOZ_ASSERT(scratch == ToRegister(construct->getNewTarget()));
+
+ Register tmpArgc = ToRegister(construct->getTempObject());
+ Register elementsAndArgc = ToRegister(construct->getElements());
+
+ // Invariants guarded in the caller:
+ // - the array is not too long
+ // - the array length equals its initialized length
+
+ // The array length is our argc for the purposes of allocating space.
+ Address length(ToRegister(construct->getElements()),
+ ObjectElements::offsetOfLength());
+ masm.load32(length, tmpArgc);
+
+ // Allocate space for the values.
+ emitAllocateSpaceForConstructAndPushNewTarget(tmpArgc, scratch);
+
+ // After this call "elements" has become "argc" and "newTarget" has become
+ // "scratch".
+ size_t elementsOffset = 0;
+ emitPushArrayAsArguments(tmpArgc, elementsAndArgc, scratch, elementsOffset);
+
+ // Push |this|.
+ masm.pushValue(ToValue(construct, LConstructArrayGeneric::ThisIndex));
+}
+
+template <typename T>
+void CodeGenerator::emitApplyGeneric(T* apply) {
+ // Holds the function object.
+ Register calleereg = ToRegister(apply->getFunction());
+
+ // Temporary register for modifying the function object.
+ Register objreg = ToRegister(apply->getTempObject());
+ Register scratch = ToRegister(apply->getTempForArgCopy());
+
+ // Holds the function nargs, computed in the invoker or (for ApplyArray,
+ // ConstructArray, or ApplyArgsObj) in the argument pusher.
+ Register argcreg = ToRegister(apply->getArgc());
+
+ // Copy the arguments of the current function.
+ //
+ // In the case of ApplyArray, ConstructArray, or ApplyArgsObj, also
+ // compute argc. The argc register and the elements/argsObj register
+ // are the same; argc must not be referenced before the call to
+ // emitPushArguments() and elements/argsObj must not be referenced
+ // after it returns.
+ //
+ // In the case of ConstructArray or ConstructArgs, also overwrite newTarget
+ // with scratch; newTarget must not be referenced after this point.
+ //
+ // objreg is dead across this call.
+ emitPushArguments(apply, scratch);
+
+ masm.checkStackAlignment();
+
+ bool constructing = apply->mir()->isConstructing();
+
+ // If the function is native, only emit the call to InvokeFunction.
+ if (apply->hasSingleTarget() &&
+ apply->getSingleTarget()->isNativeWithoutJitEntry()) {
+ emitCallInvokeFunction(apply);
+
+#ifdef DEBUG
+ // Native constructors are guaranteed to return an Object value, so we never
+ // have to replace a primitive result with the previously allocated Object
+ // from CreateThis.
+ if (constructing) {
+ Label notPrimitive;
+ masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
+ &notPrimitive);
+ masm.assumeUnreachable("native constructors don't return primitives");
+ masm.bind(&notPrimitive);
+ }
+#endif
+
+ emitRestoreStackPointerFromFP();
+ return;
+ }
+
+ Label end, invoke;
+
+ // Unless already known, guard that calleereg is actually a function object.
+ if (!apply->hasSingleTarget()) {
+ masm.branchTestObjIsFunction(Assembler::NotEqual, calleereg, objreg,
+ calleereg, &invoke);
+ }
+
+ // Guard that calleereg is an interpreted function with a JSScript.
+ masm.branchIfFunctionHasNoJitEntry(calleereg, constructing, &invoke);
+
+ // Guard that callee allows the [[Call]] or [[Construct]] operation required.
+ if (constructing) {
+ masm.branchTestFunctionFlags(calleereg, FunctionFlags::CONSTRUCTOR,
+ Assembler::Zero, &invoke);
+ } else {
+ masm.branchFunctionKind(Assembler::Equal, FunctionFlags::ClassConstructor,
+ calleereg, objreg, &invoke);
+ }
+
+ // Use the slow path if CreateThis was unable to create the |this| object.
+ if (constructing) {
+ Address thisAddr(masm.getStackPointer(), 0);
+ masm.branchTestNull(Assembler::Equal, thisAddr, &invoke);
+ }
+
+ // Call with an Ion frame or a rectifier frame.
+ {
+ if (apply->mir()->maybeCrossRealm()) {
+ masm.switchToObjectRealm(calleereg, objreg);
+ }
+
+ // Knowing that calleereg is a non-native function, load jitcode.
+ masm.loadJitCodeRaw(calleereg, objreg);
+
+ masm.PushCalleeToken(calleereg, constructing);
+ masm.PushFrameDescriptorForJitCall(FrameType::IonJS, argcreg, scratch);
+
+ Label underflow, rejoin;
+
+ // Check whether the provided arguments satisfy target argc.
+ if (!apply->hasSingleTarget()) {
+ Register nformals = scratch;
+ masm.loadFunctionArgCount(calleereg, nformals);
+ masm.branch32(Assembler::Below, argcreg, nformals, &underflow);
+ } else {
+ masm.branch32(Assembler::Below, argcreg,
+ Imm32(apply->getSingleTarget()->nargs()), &underflow);
+ }
+
+ // Skip the construction of the rectifier frame because we have no
+ // underflow.
+ masm.jump(&rejoin);
+
+ // Argument fixup needed. Get ready to call the argumentsRectifier.
+ {
+ masm.bind(&underflow);
+
+ // Hardcode the address of the argumentsRectifier code.
+ TrampolinePtr argumentsRectifier =
+ gen->jitRuntime()->getArgumentsRectifier();
+ masm.movePtr(argumentsRectifier, objreg);
+ }
+
+ masm.bind(&rejoin);
+
+ // Finally call the function in objreg, as assigned by one of the paths
+ // above.
+ uint32_t callOffset = masm.callJit(objreg);
+ markSafepointAt(callOffset, apply);
+
+ if (apply->mir()->maybeCrossRealm()) {
+ static_assert(!JSReturnOperand.aliases(ReturnReg),
+ "ReturnReg available as scratch after scripted calls");
+ masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
+ }
+
+ // Discard JitFrameLayout fields still left on the stack.
+ masm.freeStack(sizeof(JitFrameLayout) -
+ JitFrameLayout::bytesPoppedAfterCall());
+ masm.jump(&end);
+ }
+
+ // Handle uncompiled or native functions.
+ {
+ masm.bind(&invoke);
+ emitCallInvokeFunction(apply);
+ }
+
+ masm.bind(&end);
+
+ // If the return value of the constructing function is Primitive,
+ // replace the return value with the Object from CreateThis.
+ if (constructing) {
+ Label notPrimitive;
+ masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
+ &notPrimitive);
+ masm.loadValue(Address(masm.getStackPointer(), 0), JSReturnOperand);
+
+#ifdef DEBUG
+ masm.branchTestPrimitive(Assembler::NotEqual, JSReturnOperand,
+ &notPrimitive);
+ masm.assumeUnreachable("CreateThis creates an object");
+#endif
+
+ masm.bind(&notPrimitive);
+ }
+
+ // Pop arguments and continue.
+ emitRestoreStackPointerFromFP();
+}
+
+void CodeGenerator::visitApplyArgsGeneric(LApplyArgsGeneric* apply) {
+ LSnapshot* snapshot = apply->snapshot();
+ Register argcreg = ToRegister(apply->getArgc());
+
+ // Ensure that we have a reasonable number of arguments.
+ bailoutCmp32(Assembler::Above, argcreg, Imm32(JIT_ARGS_LENGTH_MAX), snapshot);
+
+ emitApplyGeneric(apply);
+}
+
+void CodeGenerator::visitApplyArgsObj(LApplyArgsObj* apply) {
+ Register argsObj = ToRegister(apply->getArgsObj());
+ Register temp = ToRegister(apply->getTempObject());
+
+ Label bail;
+ masm.loadArgumentsObjectLength(argsObj, temp, &bail);
+ masm.branch32(Assembler::Above, temp, Imm32(JIT_ARGS_LENGTH_MAX), &bail);
+ bailoutFrom(&bail, apply->snapshot());
+
+ emitApplyGeneric(apply);
+}
+
+void CodeGenerator::visitApplyArrayGeneric(LApplyArrayGeneric* apply) {
+ LSnapshot* snapshot = apply->snapshot();
+ Register tmp = ToRegister(apply->getTempObject());
+
+ Address length(ToRegister(apply->getElements()),
+ ObjectElements::offsetOfLength());
+ masm.load32(length, tmp);
+
+ // Ensure that we have a reasonable number of arguments.
+ bailoutCmp32(Assembler::Above, tmp, Imm32(JIT_ARGS_LENGTH_MAX), snapshot);
+
+ // Ensure that the array does not contain an uninitialized tail.
+
+ Address initializedLength(ToRegister(apply->getElements()),
+ ObjectElements::offsetOfInitializedLength());
+ masm.sub32(initializedLength, tmp);
+ bailoutCmp32(Assembler::NotEqual, tmp, Imm32(0), snapshot);
+
+ emitApplyGeneric(apply);
+}
+
+void CodeGenerator::visitConstructArgsGeneric(LConstructArgsGeneric* lir) {
+ LSnapshot* snapshot = lir->snapshot();
+ Register argcreg = ToRegister(lir->getArgc());
+
+ // Ensure that we have a reasonable number of arguments.
+ bailoutCmp32(Assembler::Above, argcreg, Imm32(JIT_ARGS_LENGTH_MAX), snapshot);
+
+ emitApplyGeneric(lir);
+}
+
+void CodeGenerator::visitConstructArrayGeneric(LConstructArrayGeneric* lir) {
+ LSnapshot* snapshot = lir->snapshot();
+ Register tmp = ToRegister(lir->getTempObject());
+
+ Address length(ToRegister(lir->getElements()),
+ ObjectElements::offsetOfLength());
+ masm.load32(length, tmp);
+
+ // Ensure that we have a reasonable number of arguments.
+ bailoutCmp32(Assembler::Above, tmp, Imm32(JIT_ARGS_LENGTH_MAX), snapshot);
+
+ // Ensure that the array does not contain an uninitialized tail.
+
+ Address initializedLength(ToRegister(lir->getElements()),
+ ObjectElements::offsetOfInitializedLength());
+ masm.sub32(initializedLength, tmp);
+ bailoutCmp32(Assembler::NotEqual, tmp, Imm32(0), snapshot);
+
+ emitApplyGeneric(lir);
+}
+
+void CodeGenerator::visitBail(LBail* lir) { bailout(lir->snapshot()); }
+
+void CodeGenerator::visitUnreachable(LUnreachable* lir) {
+ masm.assumeUnreachable("end-of-block assumed unreachable");
+}
+
+void CodeGenerator::visitEncodeSnapshot(LEncodeSnapshot* lir) {
+ encode(lir->snapshot());
+}
+
+void CodeGenerator::visitUnreachableResultV(LUnreachableResultV* lir) {
+ masm.assumeUnreachable("must be unreachable");
+}
+
+void CodeGenerator::visitUnreachableResultT(LUnreachableResultT* lir) {
+ masm.assumeUnreachable("must be unreachable");
+}
+
+// Out-of-line path to report over-recursed error and fail.
+class CheckOverRecursedFailure : public OutOfLineCodeBase<CodeGenerator> {
+ LInstruction* lir_;
+
+ public:
+ explicit CheckOverRecursedFailure(LInstruction* lir) : lir_(lir) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitCheckOverRecursedFailure(this);
+ }
+
+ LInstruction* lir() const { return lir_; }
+};
+
+void CodeGenerator::visitCheckOverRecursed(LCheckOverRecursed* lir) {
+ // If we don't push anything on the stack, skip the check.
+ if (omitOverRecursedCheck()) {
+ return;
+ }
+
+ // Ensure that this frame will not cross the stack limit.
+ // This is a weak check, justified by Ion using the C stack: we must always
+ // be some distance away from the actual limit, since if the limit is
+ // crossed, an error must be thrown, which requires more frames.
+ //
+ // It must always be possible to trespass past the stack limit.
+ // Ion may legally place frames very close to the limit. Calling additional
+ // C functions may then violate the limit without any checking.
+ //
+ // Since Ion frames exist on the C stack, the stack limit may be
+ // dynamically set by JS_SetThreadStackLimit() and JS_SetNativeStackQuota().
+
+ CheckOverRecursedFailure* ool = new (alloc()) CheckOverRecursedFailure(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ // Conditional forward (unlikely) branch to failure.
+ const void* limitAddr = gen->runtime->addressOfJitStackLimit();
+ masm.branchStackPtrRhs(Assembler::AboveOrEqual, AbsoluteAddress(limitAddr),
+ ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCheckOverRecursedFailure(
+ CheckOverRecursedFailure* ool) {
+ // The OOL path is hit if the recursion depth has been exceeded.
+ // Throw an InternalError for over-recursion.
+
+ // LFunctionEnvironment can appear before LCheckOverRecursed, so we have
+ // to save all live registers to avoid crashes if CheckOverRecursed triggers
+ // a GC.
+ saveLive(ool->lir());
+
+ using Fn = bool (*)(JSContext*);
+ callVM<Fn, CheckOverRecursed>(ool->lir());
+
+ restoreLive(ool->lir());
+ masm.jump(ool->rejoin());
+}
+
+IonScriptCounts* CodeGenerator::maybeCreateScriptCounts() {
+ // If scripts are being profiled, create a new IonScriptCounts for the
+ // profiling data, which will be attached to the associated JSScript or
+ // wasm module after code generation finishes.
+ if (!gen->hasProfilingScripts()) {
+ return nullptr;
+ }
+
+ // This test inhibits IonScriptCount creation for wasm code which is
+ // currently incompatible with wasm codegen for two reasons: (1) wasm code
+ // must be serializable and script count codegen bakes in absolute
+ // addresses, (2) wasm code does not have a JSScript with which to associate
+ // code coverage data.
+ JSScript* script = gen->outerInfo().script();
+ if (!script) {
+ return nullptr;
+ }
+
+ auto counts = MakeUnique<IonScriptCounts>();
+ if (!counts || !counts->init(graph.numBlocks())) {
+ return nullptr;
+ }
+
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ MBasicBlock* block = graph.getBlock(i)->mir();
+
+ uint32_t offset = 0;
+ char* description = nullptr;
+ if (MResumePoint* resume = block->entryResumePoint()) {
+ // Find a PC offset in the outermost script to use. If this
+ // block is from an inlined script, find a location in the
+ // outer script to associate information about the inlining
+ // with.
+ while (resume->caller()) {
+ resume = resume->caller();
+ }
+ offset = script->pcToOffset(resume->pc());
+
+ if (block->entryResumePoint()->caller()) {
+ // Get the filename and line number of the inner script.
+ JSScript* innerScript = block->info().script();
+ description = js_pod_calloc<char>(200);
+ if (description) {
+ snprintf(description, 200, "%s:%u", innerScript->filename(),
+ innerScript->lineno());
+ }
+ }
+ }
+
+ if (!counts->block(i).init(block->id(), offset, description,
+ block->numSuccessors())) {
+ return nullptr;
+ }
+
+ for (size_t j = 0; j < block->numSuccessors(); j++) {
+ counts->block(i).setSuccessor(
+ j, skipTrivialBlocks(block->getSuccessor(j))->id());
+ }
+ }
+
+ scriptCounts_ = counts.release();
+ return scriptCounts_;
+}
+
+// Structure for managing the state tracked for a block by script counters.
+struct ScriptCountBlockState {
+ IonBlockCounts& block;
+ MacroAssembler& masm;
+
+ Sprinter printer;
+
+ public:
+ ScriptCountBlockState(IonBlockCounts* block, MacroAssembler* masm)
+ : block(*block), masm(*masm), printer(GetJitContext()->cx, false) {}
+
+ bool init() {
+ if (!printer.init()) {
+ return false;
+ }
+
+ // Bump the hit count for the block at the start. This code is not
+ // included in either the text for the block or the instruction byte
+ // counts.
+ masm.inc64(AbsoluteAddress(block.addressOfHitCount()));
+
+ // Collect human readable assembly for the code generated in the block.
+ masm.setPrinter(&printer);
+
+ return true;
+ }
+
+ void visitInstruction(LInstruction* ins) {
+#ifdef JS_JITSPEW
+ // Prefix stream of assembly instructions with their LIR instruction
+ // name and any associated high level info.
+ if (const char* extra = ins->getExtraName()) {
+ printer.printf("[%s:%s]\n", ins->opName(), extra);
+ } else {
+ printer.printf("[%s]\n", ins->opName());
+ }
+#endif
+ }
+
+ ~ScriptCountBlockState() {
+ masm.setPrinter(nullptr);
+
+ if (!printer.hadOutOfMemory()) {
+ block.setCode(printer.string());
+ }
+ }
+};
+
+void CodeGenerator::branchIfInvalidated(Register temp, Label* invalidated) {
+ CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), temp);
+ masm.propagateOOM(ionScriptLabels_.append(label));
+
+ // If IonScript::invalidationCount_ != 0, the script has been invalidated.
+ masm.branch32(Assembler::NotEqual,
+ Address(temp, IonScript::offsetOfInvalidationCount()), Imm32(0),
+ invalidated);
+}
+
+#ifdef DEBUG
+void CodeGenerator::emitAssertGCThingResult(Register input,
+ const MDefinition* mir) {
+ MIRType type = mir->type();
+ MOZ_ASSERT(type == MIRType::Object || type == MIRType::String ||
+ type == MIRType::Symbol || type == MIRType::BigInt);
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(input);
+
+ Register temp = regs.takeAny();
+ masm.push(temp);
+
+ // Don't check if the script has been invalidated. In that case invalid
+ // types are expected (until we reach the OsiPoint and bailout).
+ Label done;
+ branchIfInvalidated(temp, &done);
+
+# ifndef JS_SIMULATOR
+ // Check that we have a valid GC pointer.
+ // Disable for wasm because we don't have a context on wasm compilation
+ // threads and this needs a context.
+ // Also disable for simulator builds because the C++ call is a lot slower
+ // there than on actual hardware.
+ if (JitOptions.fullDebugChecks && !IsCompilingWasm()) {
+ saveVolatile();
+ masm.setupUnalignedABICall(temp);
+ masm.loadJSContext(temp);
+ masm.passABIArg(temp);
+ masm.passABIArg(input);
+
+ switch (type) {
+ case MIRType::Object: {
+ using Fn = void (*)(JSContext * cx, JSObject * obj);
+ masm.callWithABI<Fn, AssertValidObjectPtr>();
+ break;
+ }
+ case MIRType::String: {
+ using Fn = void (*)(JSContext * cx, JSString * str);
+ masm.callWithABI<Fn, AssertValidStringPtr>();
+ break;
+ }
+ case MIRType::Symbol: {
+ using Fn = void (*)(JSContext * cx, JS::Symbol * sym);
+ masm.callWithABI<Fn, AssertValidSymbolPtr>();
+ break;
+ }
+ case MIRType::BigInt: {
+ using Fn = void (*)(JSContext * cx, JS::BigInt * bi);
+ masm.callWithABI<Fn, AssertValidBigIntPtr>();
+ break;
+ }
+ default:
+ MOZ_CRASH();
+ }
+
+ restoreVolatile();
+ }
+# endif
+
+ masm.bind(&done);
+ masm.pop(temp);
+}
+
+void CodeGenerator::emitAssertResultV(const ValueOperand input,
+ const MDefinition* mir) {
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(input);
+
+ Register temp1 = regs.takeAny();
+ Register temp2 = regs.takeAny();
+ masm.push(temp1);
+ masm.push(temp2);
+
+ // Don't check if the script has been invalidated. In that case invalid
+ // types are expected (until we reach the OsiPoint and bailout).
+ Label done;
+ branchIfInvalidated(temp1, &done);
+
+ // Check that we have a valid GC pointer.
+ if (JitOptions.fullDebugChecks) {
+ saveVolatile();
+
+ masm.pushValue(input);
+ masm.moveStackPtrTo(temp1);
+
+ using Fn = void (*)(JSContext * cx, Value * v);
+ masm.setupUnalignedABICall(temp2);
+ masm.loadJSContext(temp2);
+ masm.passABIArg(temp2);
+ masm.passABIArg(temp1);
+ masm.callWithABI<Fn, AssertValidValue>();
+ masm.popValue(input);
+ restoreVolatile();
+ }
+
+ masm.bind(&done);
+ masm.pop(temp2);
+ masm.pop(temp1);
+}
+
+void CodeGenerator::emitGCThingResultChecks(LInstruction* lir,
+ MDefinition* mir) {
+ if (lir->numDefs() == 0) {
+ return;
+ }
+
+ MOZ_ASSERT(lir->numDefs() == 1);
+ if (lir->getDef(0)->isBogusTemp()) {
+ return;
+ }
+
+ Register output = ToRegister(lir->getDef(0));
+ emitAssertGCThingResult(output, mir);
+}
+
+void CodeGenerator::emitValueResultChecks(LInstruction* lir, MDefinition* mir) {
+ if (lir->numDefs() == 0) {
+ return;
+ }
+
+ MOZ_ASSERT(lir->numDefs() == BOX_PIECES);
+ if (!lir->getDef(0)->output()->isRegister()) {
+ return;
+ }
+
+ ValueOperand output = ToOutValue(lir);
+
+ emitAssertResultV(output, mir);
+}
+
+void CodeGenerator::emitDebugResultChecks(LInstruction* ins) {
+ // In debug builds, check that LIR instructions return valid values.
+
+ MDefinition* mir = ins->mirRaw();
+ if (!mir) {
+ return;
+ }
+
+ switch (mir->type()) {
+ case MIRType::Object:
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ emitGCThingResultChecks(ins, mir);
+ break;
+ case MIRType::Value:
+ emitValueResultChecks(ins, mir);
+ break;
+ default:
+ break;
+ }
+}
+
+void CodeGenerator::emitDebugForceBailing(LInstruction* lir) {
+ if (MOZ_LIKELY(!gen->options.ionBailAfterEnabled())) {
+ return;
+ }
+ if (!lir->snapshot()) {
+ return;
+ }
+ if (lir->isOsiPoint()) {
+ return;
+ }
+
+ masm.comment("emitDebugForceBailing");
+ const void* bailAfterCounterAddr =
+ gen->runtime->addressOfIonBailAfterCounter();
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+
+ Label done, notBail;
+ masm.branch32(Assembler::Equal, AbsoluteAddress(bailAfterCounterAddr),
+ Imm32(0), &done);
+ {
+ Register temp = regs.takeAny();
+
+ masm.push(temp);
+ masm.load32(AbsoluteAddress(bailAfterCounterAddr), temp);
+ masm.sub32(Imm32(1), temp);
+ masm.store32(temp, AbsoluteAddress(bailAfterCounterAddr));
+
+ masm.branch32(Assembler::NotEqual, temp, Imm32(0), &notBail);
+ {
+ masm.pop(temp);
+ bailout(lir->snapshot());
+ }
+ masm.bind(&notBail);
+ masm.pop(temp);
+ }
+ masm.bind(&done);
+}
+#endif
+
+bool CodeGenerator::generateBody() {
+ JitSpewCont(JitSpew_Codegen, "\n");
+ AutoCreatedBy acb(masm, "CodeGenerator::generateBody");
+
+ JitSpew(JitSpew_Codegen, "==== BEGIN CodeGenerator::generateBody ====");
+ IonScriptCounts* counts = maybeCreateScriptCounts();
+
+ const bool compilingWasm = gen->compilingWasm();
+
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ current = graph.getBlock(i);
+
+ // Don't emit any code for trivial blocks, containing just a goto. Such
+ // blocks are created to split critical edges, and if we didn't end up
+ // putting any instructions in them, we can skip them.
+ if (current->isTrivial()) {
+ continue;
+ }
+
+#ifdef JS_JITSPEW
+ const char* filename = nullptr;
+ size_t lineNumber = 0;
+ unsigned columnNumber = 0;
+ if (current->mir()->info().script()) {
+ filename = current->mir()->info().script()->filename();
+ if (current->mir()->pc()) {
+ lineNumber = PCToLineNumber(current->mir()->info().script(),
+ current->mir()->pc(), &columnNumber);
+ }
+ } else {
+# ifdef DEBUG
+ lineNumber = current->mir()->lineno();
+ columnNumber = current->mir()->columnIndex();
+# endif
+ }
+ JitSpew(JitSpew_Codegen, "--------------------------------");
+ JitSpew(JitSpew_Codegen, "# block%zu %s:%zu:%u%s:", i,
+ filename ? filename : "?", lineNumber, columnNumber,
+ current->mir()->isLoopHeader() ? " (loop header)" : "");
+#endif
+
+ if (current->mir()->isLoopHeader() && compilingWasm) {
+ masm.nopAlign(CodeAlignment);
+ }
+
+ masm.bind(current->label());
+
+ mozilla::Maybe<ScriptCountBlockState> blockCounts;
+ if (counts) {
+ blockCounts.emplace(&counts->block(i), &masm);
+ if (!blockCounts->init()) {
+ return false;
+ }
+ }
+
+ for (LInstructionIterator iter = current->begin(); iter != current->end();
+ iter++) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ perfSpewer_.recordInstruction(masm, iter->op());
+#ifdef JS_JITSPEW
+ JitSpewStart(JitSpew_Codegen, " # LIR=%s",
+ iter->opName());
+ if (const char* extra = iter->getExtraName()) {
+ JitSpewCont(JitSpew_Codegen, ":%s", extra);
+ }
+ JitSpewFin(JitSpew_Codegen);
+#endif
+
+ if (counts) {
+ blockCounts->visitInstruction(*iter);
+ }
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ if (iter->safepoint() && !compilingWasm) {
+ resetOsiPointRegs(iter->safepoint());
+ }
+#endif
+
+ if (!compilingWasm) {
+ if (MDefinition* mir = iter->mirRaw()) {
+ if (!addNativeToBytecodeEntry(mir->trackedSite())) {
+ return false;
+ }
+ }
+ }
+
+ setElement(*iter); // needed to encode correct snapshot location.
+
+#ifdef DEBUG
+ emitDebugForceBailing(*iter);
+#endif
+
+ switch (iter->op()) {
+#ifndef JS_CODEGEN_NONE
+# define LIROP(op) \
+ case LNode::Opcode::op: \
+ visit##op(iter->to##op()); \
+ break;
+ LIR_OPCODE_LIST(LIROP)
+# undef LIROP
+#endif
+ case LNode::Opcode::Invalid:
+ default:
+ MOZ_CRASH("Invalid LIR op");
+ }
+
+#ifdef DEBUG
+ if (!counts) {
+ emitDebugResultChecks(*iter);
+ }
+#endif
+ }
+ if (masm.oom()) {
+ return false;
+ }
+ }
+
+ JitSpew(JitSpew_Codegen, "==== END CodeGenerator::generateBody ====\n");
+ return true;
+}
+
+// Out-of-line object allocation for LNewArray.
+class OutOfLineNewArray : public OutOfLineCodeBase<CodeGenerator> {
+ LNewArray* lir_;
+
+ public:
+ explicit OutOfLineNewArray(LNewArray* lir) : lir_(lir) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineNewArray(this);
+ }
+
+ LNewArray* lir() const { return lir_; }
+};
+
+void CodeGenerator::visitNewArrayCallVM(LNewArray* lir) {
+ Register objReg = ToRegister(lir->output());
+
+ MOZ_ASSERT(!lir->isCall());
+ saveLive(lir);
+
+ JSObject* templateObject = lir->mir()->templateObject();
+
+ if (templateObject) {
+ pushArg(ImmGCPtr(templateObject->shape()));
+ pushArg(Imm32(lir->mir()->length()));
+
+ using Fn = ArrayObject* (*)(JSContext*, uint32_t, Handle<Shape*>);
+ callVM<Fn, NewArrayWithShape>(lir);
+ } else {
+ pushArg(Imm32(GenericObject));
+ pushArg(Imm32(lir->mir()->length()));
+
+ using Fn = ArrayObject* (*)(JSContext*, uint32_t, NewObjectKind);
+ callVM<Fn, NewArrayOperation>(lir);
+ }
+
+ masm.storeCallPointerResult(objReg);
+
+ MOZ_ASSERT(!lir->safepoint()->liveRegs().has(objReg));
+ restoreLive(lir);
+}
+
+void CodeGenerator::visitAtan2D(LAtan2D* lir) {
+ FloatRegister y = ToFloatRegister(lir->y());
+ FloatRegister x = ToFloatRegister(lir->x());
+
+ using Fn = double (*)(double x, double y);
+ masm.setupAlignedABICall();
+ masm.passABIArg(y, MoveOp::DOUBLE);
+ masm.passABIArg(x, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, ecmaAtan2>(MoveOp::DOUBLE);
+
+ MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg);
+}
+
+void CodeGenerator::visitHypot(LHypot* lir) {
+ uint32_t numArgs = lir->numArgs();
+ masm.setupAlignedABICall();
+
+ for (uint32_t i = 0; i < numArgs; ++i) {
+ masm.passABIArg(ToFloatRegister(lir->getOperand(i)), MoveOp::DOUBLE);
+ }
+
+ switch (numArgs) {
+ case 2: {
+ using Fn = double (*)(double x, double y);
+ masm.callWithABI<Fn, ecmaHypot>(MoveOp::DOUBLE);
+ break;
+ }
+ case 3: {
+ using Fn = double (*)(double x, double y, double z);
+ masm.callWithABI<Fn, hypot3>(MoveOp::DOUBLE);
+ break;
+ }
+ case 4: {
+ using Fn = double (*)(double x, double y, double z, double w);
+ masm.callWithABI<Fn, hypot4>(MoveOp::DOUBLE);
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected number of arguments to hypot function.");
+ }
+ MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg);
+}
+
+void CodeGenerator::visitNewArray(LNewArray* lir) {
+ Register objReg = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp());
+ DebugOnly<uint32_t> length = lir->mir()->length();
+
+ MOZ_ASSERT(length <= NativeObject::MAX_DENSE_ELEMENTS_COUNT);
+
+ if (lir->mir()->isVMCall()) {
+ visitNewArrayCallVM(lir);
+ return;
+ }
+
+ OutOfLineNewArray* ool = new (alloc()) OutOfLineNewArray(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ TemplateObject templateObject(lir->mir()->templateObject());
+#ifdef DEBUG
+ size_t numInlineElements = gc::GetGCKindSlots(templateObject.getAllocKind()) -
+ ObjectElements::VALUES_PER_HEADER;
+ MOZ_ASSERT(length <= numInlineElements,
+ "Inline allocation only supports inline elements");
+#endif
+ masm.createGCObject(objReg, tempReg, templateObject,
+ lir->mir()->initialHeap(), ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineNewArray(OutOfLineNewArray* ool) {
+ visitNewArrayCallVM(ool->lir());
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitNewArrayDynamicLength(LNewArrayDynamicLength* lir) {
+ Register lengthReg = ToRegister(lir->length());
+ Register objReg = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp0());
+
+ JSObject* templateObject = lir->mir()->templateObject();
+ gc::InitialHeap initialHeap = lir->mir()->initialHeap();
+
+ using Fn = ArrayObject* (*)(JSContext*, Handle<ArrayObject*>, int32_t length);
+ OutOfLineCode* ool = oolCallVM<Fn, ArrayConstructorOneArg>(
+ lir, ArgList(ImmGCPtr(templateObject), lengthReg),
+ StoreRegisterTo(objReg));
+
+ bool canInline = true;
+ size_t inlineLength = 0;
+ if (templateObject->as<ArrayObject>().hasFixedElements()) {
+ size_t numSlots =
+ gc::GetGCKindSlots(templateObject->asTenured().getAllocKind());
+ inlineLength = numSlots - ObjectElements::VALUES_PER_HEADER;
+ } else {
+ canInline = false;
+ }
+
+ if (canInline) {
+ // Try to do the allocation inline if the template object is big enough
+ // for the length in lengthReg. If the length is bigger we could still
+ // use the template object and not allocate the elements, but it's more
+ // efficient to do a single big allocation than (repeatedly) reallocating
+ // the array later on when filling it.
+ masm.branch32(Assembler::Above, lengthReg, Imm32(inlineLength),
+ ool->entry());
+
+ TemplateObject templateObj(templateObject);
+ masm.createGCObject(objReg, tempReg, templateObj, initialHeap,
+ ool->entry());
+
+ size_t lengthOffset = NativeObject::offsetOfFixedElements() +
+ ObjectElements::offsetOfLength();
+ masm.store32(lengthReg, Address(objReg, lengthOffset));
+ } else {
+ masm.jump(ool->entry());
+ }
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewIterator(LNewIterator* lir) {
+ Register objReg = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp0());
+
+ OutOfLineCode* ool;
+ switch (lir->mir()->type()) {
+ case MNewIterator::ArrayIterator: {
+ using Fn = ArrayIteratorObject* (*)(JSContext*);
+ ool = oolCallVM<Fn, NewArrayIterator>(lir, ArgList(),
+ StoreRegisterTo(objReg));
+ break;
+ }
+ case MNewIterator::StringIterator: {
+ using Fn = StringIteratorObject* (*)(JSContext*);
+ ool = oolCallVM<Fn, NewStringIterator>(lir, ArgList(),
+ StoreRegisterTo(objReg));
+ break;
+ }
+ case MNewIterator::RegExpStringIterator: {
+ using Fn = RegExpStringIteratorObject* (*)(JSContext*);
+ ool = oolCallVM<Fn, NewRegExpStringIterator>(lir, ArgList(),
+ StoreRegisterTo(objReg));
+ break;
+ }
+ default:
+ MOZ_CRASH("unexpected iterator type");
+ }
+
+ TemplateObject templateObject(lir->mir()->templateObject());
+ masm.createGCObject(objReg, tempReg, templateObject, gc::DefaultHeap,
+ ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewTypedArray(LNewTypedArray* lir) {
+ Register objReg = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp0());
+ Register lengthReg = ToRegister(lir->temp1());
+ LiveRegisterSet liveRegs = liveVolatileRegs(lir);
+
+ JSObject* templateObject = lir->mir()->templateObject();
+ gc::InitialHeap initialHeap = lir->mir()->initialHeap();
+
+ TypedArrayObject* ttemplate = &templateObject->as<TypedArrayObject>();
+
+ size_t n = ttemplate->length();
+ MOZ_ASSERT(n <= INT32_MAX,
+ "Template objects are only created for int32 lengths");
+
+ using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, int32_t length);
+ OutOfLineCode* ool = oolCallVM<Fn, NewTypedArrayWithTemplateAndLength>(
+ lir, ArgList(ImmGCPtr(templateObject), Imm32(n)),
+ StoreRegisterTo(objReg));
+
+ TemplateObject templateObj(templateObject);
+ masm.createGCObject(objReg, tempReg, templateObj, initialHeap, ool->entry());
+
+ masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(),
+ ttemplate, MacroAssembler::TypedArrayLength::Fixed);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewTypedArrayDynamicLength(
+ LNewTypedArrayDynamicLength* lir) {
+ Register lengthReg = ToRegister(lir->length());
+ Register objReg = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp0());
+ LiveRegisterSet liveRegs = liveVolatileRegs(lir);
+
+ JSObject* templateObject = lir->mir()->templateObject();
+ gc::InitialHeap initialHeap = lir->mir()->initialHeap();
+
+ TypedArrayObject* ttemplate = &templateObject->as<TypedArrayObject>();
+
+ using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, int32_t length);
+ OutOfLineCode* ool = oolCallVM<Fn, NewTypedArrayWithTemplateAndLength>(
+ lir, ArgList(ImmGCPtr(templateObject), lengthReg),
+ StoreRegisterTo(objReg));
+
+ // Volatile |lengthReg| is saved across the ABI call in |initTypedArraySlots|.
+ MOZ_ASSERT_IF(lengthReg.volatile_(), liveRegs.has(lengthReg));
+
+ TemplateObject templateObj(templateObject);
+ masm.createGCObject(objReg, tempReg, templateObj, initialHeap, ool->entry());
+
+ masm.initTypedArraySlots(objReg, tempReg, lengthReg, liveRegs, ool->entry(),
+ ttemplate,
+ MacroAssembler::TypedArrayLength::Dynamic);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewTypedArrayFromArray(LNewTypedArrayFromArray* lir) {
+ pushArg(ToRegister(lir->array()));
+ pushArg(ImmGCPtr(lir->mir()->templateObject()));
+
+ using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, HandleObject);
+ callVM<Fn, js::NewTypedArrayWithTemplateAndArray>(lir);
+}
+
+void CodeGenerator::visitNewTypedArrayFromArrayBuffer(
+ LNewTypedArrayFromArrayBuffer* lir) {
+ pushArg(ToValue(lir, LNewTypedArrayFromArrayBuffer::LengthIndex));
+ pushArg(ToValue(lir, LNewTypedArrayFromArrayBuffer::ByteOffsetIndex));
+ pushArg(ToRegister(lir->arrayBuffer()));
+ pushArg(ImmGCPtr(lir->mir()->templateObject()));
+
+ using Fn = TypedArrayObject* (*)(JSContext*, HandleObject, HandleObject,
+ HandleValue, HandleValue);
+ callVM<Fn, js::NewTypedArrayWithTemplateAndBuffer>(lir);
+}
+
+// Out-of-line object allocation for JSOp::NewObject.
+class OutOfLineNewObject : public OutOfLineCodeBase<CodeGenerator> {
+ LNewObject* lir_;
+
+ public:
+ explicit OutOfLineNewObject(LNewObject* lir) : lir_(lir) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineNewObject(this);
+ }
+
+ LNewObject* lir() const { return lir_; }
+};
+
+void CodeGenerator::visitNewObjectVMCall(LNewObject* lir) {
+ Register objReg = ToRegister(lir->output());
+
+ MOZ_ASSERT(!lir->isCall());
+ saveLive(lir);
+
+ JSObject* templateObject = lir->mir()->templateObject();
+
+ // If we're making a new object with a class prototype (that is, an object
+ // that derives its class from its prototype instead of being
+ // PlainObject::class_'d) from self-hosted code, we need a different init
+ // function.
+ switch (lir->mir()->mode()) {
+ case MNewObject::ObjectLiteral: {
+ MOZ_ASSERT(!templateObject);
+ pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
+ pushArg(ImmGCPtr(lir->mir()->block()->info().script()));
+
+ using Fn = JSObject* (*)(JSContext*, HandleScript, const jsbytecode* pc);
+ callVM<Fn, NewObjectOperation>(lir);
+ break;
+ }
+ case MNewObject::ObjectCreate: {
+ pushArg(ImmGCPtr(templateObject));
+
+ using Fn = PlainObject* (*)(JSContext*, Handle<PlainObject*>);
+ callVM<Fn, ObjectCreateWithTemplate>(lir);
+ break;
+ }
+ }
+
+ masm.storeCallPointerResult(objReg);
+
+ MOZ_ASSERT(!lir->safepoint()->liveRegs().has(objReg));
+ restoreLive(lir);
+}
+
+static bool ShouldInitFixedSlots(LNewPlainObject* lir, const Shape* shape,
+ uint32_t nfixed) {
+ // Look for StoreFixedSlot instructions following an object allocation
+ // that write to this object before a GC is triggered or this object is
+ // passed to a VM call. If all fixed slots will be initialized, the
+ // allocation code doesn't need to set the slots to |undefined|.
+
+ if (nfixed == 0) {
+ return false;
+ }
+
+ // Keep track of the fixed slots that are initialized. initializedSlots is
+ // a bit mask with a bit for each slot.
+ MOZ_ASSERT(nfixed <= NativeObject::MAX_FIXED_SLOTS);
+ static_assert(NativeObject::MAX_FIXED_SLOTS <= 32,
+ "Slot bits must fit in 32 bits");
+ uint32_t initializedSlots = 0;
+ uint32_t numInitialized = 0;
+
+ MInstruction* allocMir = lir->mir();
+ MBasicBlock* block = allocMir->block();
+
+ // Skip the allocation instruction.
+ MInstructionIterator iter = block->begin(allocMir);
+ MOZ_ASSERT(*iter == allocMir);
+ iter++;
+
+ // Handle the leading shape guard, if present.
+ for (; iter != block->end(); iter++) {
+ if (iter->isConstant()) {
+ // This instruction won't trigger a GC or read object slots.
+ continue;
+ }
+ if (iter->isGuardShape()) {
+ auto* guard = iter->toGuardShape();
+ if (guard->object() != allocMir || guard->shape() != shape) {
+ return true;
+ }
+ allocMir = guard;
+ iter++;
+ }
+ break;
+ }
+
+ for (; iter != block->end(); iter++) {
+ if (iter->isConstant() || iter->isPostWriteBarrier()) {
+ // These instructions won't trigger a GC or read object slots.
+ continue;
+ }
+
+ if (iter->isStoreFixedSlot()) {
+ MStoreFixedSlot* store = iter->toStoreFixedSlot();
+ if (store->object() != allocMir) {
+ return true;
+ }
+
+ // We may not initialize this object slot on allocation, so the
+ // pre-barrier could read uninitialized memory. Simply disable
+ // the barrier for this store: the object was just initialized
+ // so the barrier is not necessary.
+ store->setNeedsBarrier(false);
+
+ uint32_t slot = store->slot();
+ MOZ_ASSERT(slot < nfixed);
+ if ((initializedSlots & (1 << slot)) == 0) {
+ numInitialized++;
+ initializedSlots |= (1 << slot);
+
+ if (numInitialized == nfixed) {
+ // All fixed slots will be initialized.
+ MOZ_ASSERT(mozilla::CountPopulation32(initializedSlots) == nfixed);
+ return false;
+ }
+ }
+ continue;
+ }
+
+ // Unhandled instruction, assume it bails or reads object slots.
+ return true;
+ }
+
+ MOZ_CRASH("Shouldn't get here");
+}
+
+void CodeGenerator::visitNewObject(LNewObject* lir) {
+ Register objReg = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp());
+
+ if (lir->mir()->isVMCall()) {
+ visitNewObjectVMCall(lir);
+ return;
+ }
+
+ OutOfLineNewObject* ool = new (alloc()) OutOfLineNewObject(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ TemplateObject templateObject(lir->mir()->templateObject());
+
+ masm.createGCObject(objReg, tempReg, templateObject,
+ lir->mir()->initialHeap(), ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineNewObject(OutOfLineNewObject* ool) {
+ visitNewObjectVMCall(ool->lir());
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitNewPlainObject(LNewPlainObject* lir) {
+ Register objReg = ToRegister(lir->output());
+ Register temp0Reg = ToRegister(lir->temp0());
+ Register temp1Reg = ToRegister(lir->temp1());
+ Register shapeReg = ToRegister(lir->temp2());
+
+ auto* mir = lir->mir();
+ const Shape* shape = mir->shape();
+ gc::InitialHeap initialHeap = mir->initialHeap();
+ gc::AllocKind allocKind = mir->allocKind();
+
+ using Fn = JSObject* (*)(JSContext*, Handle<SharedShape*>, gc::AllocKind,
+ gc::InitialHeap);
+ OutOfLineCode* ool = oolCallVM<Fn, NewPlainObjectOptimizedFallback>(
+ lir,
+ ArgList(ImmGCPtr(shape), Imm32(int32_t(allocKind)), Imm32(initialHeap)),
+ StoreRegisterTo(objReg));
+
+ bool initContents = ShouldInitFixedSlots(lir, shape, mir->numFixedSlots());
+
+ masm.movePtr(ImmGCPtr(shape), shapeReg);
+ masm.createPlainGCObject(
+ objReg, shapeReg, temp0Reg, temp1Reg, mir->numFixedSlots(),
+ mir->numDynamicSlots(), allocKind, initialHeap, ool->entry(),
+ AllocSiteInput(gc::CatchAllAllocSite::Optimized), initContents);
+
+#ifdef DEBUG
+ // ShouldInitFixedSlots expects that the leading GuardShape will never fail,
+ // so ensure the newly created object has the correct shape. Should the guard
+ // ever fail, we may end up with uninitialized fixed slots, which can confuse
+ // the GC.
+ Label ok;
+ masm.branchTestObjShape(Assembler::Equal, objReg, shape, temp0Reg, objReg,
+ &ok);
+ masm.assumeUnreachable("Newly created object has the correct shape");
+ masm.bind(&ok);
+#endif
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewArrayObject(LNewArrayObject* lir) {
+ Register objReg = ToRegister(lir->output());
+ Register temp0Reg = ToRegister(lir->temp0());
+ Register shapeReg = ToRegister(lir->temp1());
+
+ auto* mir = lir->mir();
+ uint32_t arrayLength = mir->length();
+
+ gc::AllocKind allocKind = GuessArrayGCKind(arrayLength);
+ MOZ_ASSERT(CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_));
+ allocKind = ForegroundToBackgroundAllocKind(allocKind);
+
+ uint32_t slotCount = GetGCKindSlots(allocKind);
+ MOZ_ASSERT(slotCount >= ObjectElements::VALUES_PER_HEADER);
+ uint32_t arrayCapacity = slotCount - ObjectElements::VALUES_PER_HEADER;
+
+ const Shape* shape = mir->shape();
+
+ NewObjectKind objectKind =
+ mir->initialHeap() == gc::TenuredHeap ? TenuredObject : GenericObject;
+
+ using Fn =
+ ArrayObject* (*)(JSContext*, uint32_t, gc::AllocKind, NewObjectKind);
+ OutOfLineCode* ool = oolCallVM<Fn, NewArrayObjectOptimizedFallback>(
+ lir,
+ ArgList(Imm32(arrayLength), Imm32(int32_t(allocKind)), Imm32(objectKind)),
+ StoreRegisterTo(objReg));
+
+ masm.movePtr(ImmPtr(shape), shapeReg);
+ masm.createArrayWithFixedElements(
+ objReg, shapeReg, temp0Reg, arrayLength, arrayCapacity, allocKind,
+ mir->initialHeap(), ool->entry(),
+ AllocSiteInput(gc::CatchAllAllocSite::Optimized));
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewNamedLambdaObject(LNewNamedLambdaObject* lir) {
+ Register objReg = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp0());
+ const CompileInfo& info = lir->mir()->block()->info();
+
+ using Fn = js::NamedLambdaObject* (*)(JSContext*, HandleFunction);
+ OutOfLineCode* ool = oolCallVM<Fn, NamedLambdaObject::createWithoutEnclosing>(
+ lir, ArgList(info.funMaybeLazy()), StoreRegisterTo(objReg));
+
+ TemplateObject templateObject(lir->mir()->templateObj());
+
+ masm.createGCObject(objReg, tempReg, templateObject, gc::DefaultHeap,
+ ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewCallObject(LNewCallObject* lir) {
+ Register objReg = ToRegister(lir->output());
+ Register tempReg = ToRegister(lir->temp0());
+
+ CallObject* templateObj = lir->mir()->templateObject();
+
+ using Fn = CallObject* (*)(JSContext*, Handle<SharedShape*>);
+ OutOfLineCode* ool = oolCallVM<Fn, CallObject::createWithShape>(
+ lir, ArgList(ImmGCPtr(templateObj->sharedShape())),
+ StoreRegisterTo(objReg));
+
+ // Inline call object creation, using the OOL path only for tricky cases.
+ TemplateObject templateObject(templateObj);
+ masm.createGCObject(objReg, tempReg, templateObject, gc::DefaultHeap,
+ ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitNewStringObject(LNewStringObject* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ StringObject* templateObj = lir->mir()->templateObj();
+
+ using Fn = JSObject* (*)(JSContext*, HandleString);
+ OutOfLineCode* ool = oolCallVM<Fn, NewStringObject>(lir, ArgList(input),
+ StoreRegisterTo(output));
+
+ TemplateObject templateObject(templateObj);
+ masm.createGCObject(output, temp, templateObject, gc::DefaultHeap,
+ ool->entry());
+
+ masm.loadStringLength(input, temp);
+
+ masm.storeValue(JSVAL_TYPE_STRING, input,
+ Address(output, StringObject::offsetOfPrimitiveValue()));
+ masm.storeValue(JSVAL_TYPE_INT32, temp,
+ Address(output, StringObject::offsetOfLength()));
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitInitElemGetterSetter(LInitElemGetterSetter* lir) {
+ Register obj = ToRegister(lir->object());
+ Register value = ToRegister(lir->value());
+
+ pushArg(value);
+ pushArg(ToValue(lir, LInitElemGetterSetter::IdIndex));
+ pushArg(obj);
+ pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
+
+ using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject, HandleValue,
+ HandleObject);
+ callVM<Fn, InitElemGetterSetterOperation>(lir);
+}
+
+void CodeGenerator::visitMutateProto(LMutateProto* lir) {
+ Register objReg = ToRegister(lir->object());
+
+ pushArg(ToValue(lir, LMutateProto::ValueIndex));
+ pushArg(objReg);
+
+ using Fn =
+ bool (*)(JSContext * cx, Handle<PlainObject*> obj, HandleValue value);
+ callVM<Fn, MutatePrototype>(lir);
+}
+
+void CodeGenerator::visitInitPropGetterSetter(LInitPropGetterSetter* lir) {
+ Register obj = ToRegister(lir->object());
+ Register value = ToRegister(lir->value());
+
+ pushArg(value);
+ pushArg(ImmGCPtr(lir->mir()->name()));
+ pushArg(obj);
+ pushArg(ImmPtr(lir->mir()->resumePoint()->pc()));
+
+ using Fn = bool (*)(JSContext*, jsbytecode*, HandleObject,
+ Handle<PropertyName*>, HandleObject);
+ callVM<Fn, InitPropGetterSetterOperation>(lir);
+}
+
+void CodeGenerator::visitCreateThis(LCreateThis* lir) {
+ const LAllocation* callee = lir->callee();
+ const LAllocation* newTarget = lir->newTarget();
+
+ if (newTarget->isConstant()) {
+ pushArg(ImmGCPtr(&newTarget->toConstant()->toObject()));
+ } else {
+ pushArg(ToRegister(newTarget));
+ }
+
+ if (callee->isConstant()) {
+ pushArg(ImmGCPtr(&callee->toConstant()->toObject()));
+ } else {
+ pushArg(ToRegister(callee));
+ }
+
+ using Fn = bool (*)(JSContext * cx, HandleObject callee,
+ HandleObject newTarget, MutableHandleValue rval);
+ callVM<Fn, jit::CreateThisFromIon>(lir);
+}
+
+void CodeGenerator::visitCreateArgumentsObject(LCreateArgumentsObject* lir) {
+ // This should be getting constructed in the first block only, and not any OSR
+ // entry blocks.
+ MOZ_ASSERT(lir->mir()->block()->id() == 0);
+
+ Register callObj = ToRegister(lir->callObject());
+ Register temp0 = ToRegister(lir->temp0());
+ Label done;
+
+ if (ArgumentsObject* templateObj = lir->mir()->templateObject()) {
+ Register objTemp = ToRegister(lir->temp1());
+ Register cxTemp = ToRegister(lir->temp2());
+
+ masm.Push(callObj);
+
+ // Try to allocate an arguments object. This will leave the reserved
+ // slots uninitialized, so it's important we don't GC until we
+ // initialize these slots in ArgumentsObject::finishForIonPure.
+ Label failure;
+ TemplateObject templateObject(templateObj);
+ masm.createGCObject(objTemp, temp0, templateObject, gc::DefaultHeap,
+ &failure,
+ /* initContents = */ false);
+
+ masm.moveStackPtrTo(temp0);
+ masm.addPtr(Imm32(masm.framePushed()), temp0);
+
+ using Fn =
+ ArgumentsObject* (*)(JSContext * cx, jit::JitFrameLayout * frame,
+ JSObject * scopeChain, ArgumentsObject * obj);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(cxTemp);
+ masm.passABIArg(cxTemp);
+ masm.passABIArg(temp0);
+ masm.passABIArg(callObj);
+ masm.passABIArg(objTemp);
+
+ masm.callWithABI<Fn, ArgumentsObject::finishForIonPure>();
+ masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &failure);
+
+ // Discard saved callObj on the stack.
+ masm.addToStackPtr(Imm32(sizeof(uintptr_t)));
+ masm.jump(&done);
+
+ masm.bind(&failure);
+ masm.Pop(callObj);
+ }
+
+ masm.moveStackPtrTo(temp0);
+ masm.addPtr(Imm32(frameSize()), temp0);
+
+ pushArg(callObj);
+ pushArg(temp0);
+
+ using Fn = ArgumentsObject* (*)(JSContext*, JitFrameLayout*, HandleObject);
+ callVM<Fn, ArgumentsObject::createForIon>(lir);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitCreateInlinedArgumentsObject(
+ LCreateInlinedArgumentsObject* lir) {
+ Register callObj = ToRegister(lir->getCallObject());
+ Register callee = ToRegister(lir->getCallee());
+ Register argsAddress = ToRegister(lir->temp1());
+ Register argsObj = ToRegister(lir->temp2());
+
+ // TODO: Do we have to worry about alignment here?
+
+ // Create a contiguous array of values for ArgumentsObject::create
+ // by pushing the arguments onto the stack in reverse order.
+ uint32_t argc = lir->mir()->numActuals();
+ for (uint32_t i = 0; i < argc; i++) {
+ uint32_t argNum = argc - i - 1;
+ uint32_t index = LCreateInlinedArgumentsObject::ArgIndex(argNum);
+ ConstantOrRegister arg =
+ toConstantOrRegister(lir, index, lir->mir()->getArg(argNum)->type());
+ masm.Push(arg);
+ }
+ masm.moveStackPtrTo(argsAddress);
+
+ Label done;
+ if (ArgumentsObject* templateObj = lir->mir()->templateObject()) {
+ LiveRegisterSet liveRegs;
+ liveRegs.add(callObj);
+ liveRegs.add(callee);
+
+ masm.PushRegsInMask(liveRegs);
+
+ // We are free to clobber all registers, as LCreateInlinedArgumentsObject is
+ // a call instruction.
+ AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
+ allRegs.take(callObj);
+ allRegs.take(callee);
+ allRegs.take(argsObj);
+ allRegs.take(argsAddress);
+
+ Register temp3 = allRegs.takeAny();
+ Register temp4 = allRegs.takeAny();
+
+ // Try to allocate an arguments object. This will leave the reserved slots
+ // uninitialized, so it's important we don't GC until we initialize these
+ // slots in ArgumentsObject::finishForIonPure.
+ Label failure;
+ TemplateObject templateObject(templateObj);
+ masm.createGCObject(argsObj, temp3, templateObject, gc::DefaultHeap,
+ &failure,
+ /* initContents = */ false);
+
+ Register numActuals = temp3;
+ masm.move32(Imm32(argc), numActuals);
+
+ using Fn = ArgumentsObject* (*)(JSContext*, JSObject*, JSFunction*, Value*,
+ uint32_t, ArgumentsObject*);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp4);
+ masm.passABIArg(temp4);
+ masm.passABIArg(callObj);
+ masm.passABIArg(callee);
+ masm.passABIArg(argsAddress);
+ masm.passABIArg(numActuals);
+ masm.passABIArg(argsObj);
+
+ masm.callWithABI<Fn, ArgumentsObject::finishInlineForIonPure>();
+ masm.branchTestPtr(Assembler::Zero, ReturnReg, ReturnReg, &failure);
+
+ // Discard saved callObj, callee, and values array on the stack.
+ masm.addToStackPtr(
+ Imm32(masm.PushRegsInMaskSizeInBytes(liveRegs) + argc * sizeof(Value)));
+ masm.jump(&done);
+
+ masm.bind(&failure);
+ masm.PopRegsInMask(liveRegs);
+
+ // Reload argsAddress because it may have been overridden.
+ masm.moveStackPtrTo(argsAddress);
+ }
+
+ pushArg(Imm32(argc));
+ pushArg(callObj);
+ pushArg(callee);
+ pushArg(argsAddress);
+
+ using Fn = ArgumentsObject* (*)(JSContext*, Value*, HandleFunction,
+ HandleObject, uint32_t);
+ callVM<Fn, ArgumentsObject::createForInlinedIon>(lir);
+
+ // Discard the array of values.
+ masm.freeStack(argc * sizeof(Value));
+
+ masm.bind(&done);
+}
+
+template <class GetInlinedArgument>
+void CodeGenerator::emitGetInlinedArgument(GetInlinedArgument* lir,
+ Register index,
+ ValueOperand output) {
+ uint32_t numActuals = lir->mir()->numActuals();
+ MOZ_ASSERT(numActuals > 0 && numActuals <= ArgumentsObject::MaxInlinedArgs);
+
+ // Check the first n-1 possible indices.
+ Label done;
+ for (uint32_t i = 0; i < numActuals - 1; i++) {
+ Label skip;
+ ConstantOrRegister arg = toConstantOrRegister(
+ lir, GetInlinedArgument::ArgIndex(i), lir->mir()->getArg(i)->type());
+ masm.branch32(Assembler::NotEqual, index, Imm32(i), &skip);
+ masm.moveValue(arg, output);
+
+ masm.jump(&done);
+ masm.bind(&skip);
+ }
+
+#ifdef DEBUG
+ Label skip;
+ masm.branch32(Assembler::Equal, index, Imm32(numActuals - 1), &skip);
+ masm.assumeUnreachable("LGetInlinedArgument: invalid index");
+ masm.bind(&skip);
+#endif
+
+ // The index has already been bounds-checked, so load the last argument.
+ uint32_t lastIdx = numActuals - 1;
+ ConstantOrRegister arg =
+ toConstantOrRegister(lir, GetInlinedArgument::ArgIndex(lastIdx),
+ lir->mir()->getArg(lastIdx)->type());
+ masm.moveValue(arg, output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGetInlinedArgument(LGetInlinedArgument* lir) {
+ Register index = ToRegister(lir->getIndex());
+ ValueOperand output = ToOutValue(lir);
+
+ emitGetInlinedArgument(lir, index, output);
+}
+
+void CodeGenerator::visitGetInlinedArgumentHole(LGetInlinedArgumentHole* lir) {
+ Register index = ToRegister(lir->getIndex());
+ ValueOperand output = ToOutValue(lir);
+
+ uint32_t numActuals = lir->mir()->numActuals();
+
+ if (numActuals == 0) {
+ bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
+ masm.moveValue(UndefinedValue(), output);
+ return;
+ }
+
+ Label outOfBounds, done;
+ masm.branch32(Assembler::AboveOrEqual, index, Imm32(numActuals),
+ &outOfBounds);
+
+ emitGetInlinedArgument(lir, index, output);
+ masm.jump(&done);
+
+ masm.bind(&outOfBounds);
+ bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
+ masm.moveValue(UndefinedValue(), output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGetArgumentsObjectArg(LGetArgumentsObjectArg* lir) {
+ Register temp = ToRegister(lir->temp0());
+ Register argsObj = ToRegister(lir->argsObject());
+ ValueOperand out = ToOutValue(lir);
+
+ masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()),
+ temp);
+ Address argAddr(temp, ArgumentsData::offsetOfArgs() +
+ lir->mir()->argno() * sizeof(Value));
+ masm.loadValue(argAddr, out);
+#ifdef DEBUG
+ Label success;
+ masm.branchTestMagic(Assembler::NotEqual, out, &success);
+ masm.assumeUnreachable(
+ "Result from ArgumentObject shouldn't be JSVAL_TYPE_MAGIC.");
+ masm.bind(&success);
+#endif
+}
+
+void CodeGenerator::visitSetArgumentsObjectArg(LSetArgumentsObjectArg* lir) {
+ Register temp = ToRegister(lir->getTemp(0));
+ Register argsObj = ToRegister(lir->argsObject());
+ ValueOperand value = ToValue(lir, LSetArgumentsObjectArg::ValueIndex);
+
+ masm.loadPrivate(Address(argsObj, ArgumentsObject::getDataSlotOffset()),
+ temp);
+ Address argAddr(temp, ArgumentsData::offsetOfArgs() +
+ lir->mir()->argno() * sizeof(Value));
+ emitPreBarrier(argAddr);
+#ifdef DEBUG
+ Label success;
+ masm.branchTestMagic(Assembler::NotEqual, argAddr, &success);
+ masm.assumeUnreachable(
+ "Result in ArgumentObject shouldn't be JSVAL_TYPE_MAGIC.");
+ masm.bind(&success);
+#endif
+ masm.storeValue(value, argAddr);
+}
+
+void CodeGenerator::visitLoadArgumentsObjectArg(LLoadArgumentsObjectArg* lir) {
+ Register temp = ToRegister(lir->temp0());
+ Register argsObj = ToRegister(lir->argsObject());
+ Register index = ToRegister(lir->index());
+ ValueOperand out = ToOutValue(lir);
+
+ Label bail;
+ masm.loadArgumentsObjectElement(argsObj, index, out, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitLoadArgumentsObjectArgHole(
+ LLoadArgumentsObjectArgHole* lir) {
+ Register temp = ToRegister(lir->temp0());
+ Register argsObj = ToRegister(lir->argsObject());
+ Register index = ToRegister(lir->index());
+ ValueOperand out = ToOutValue(lir);
+
+ Label bail;
+ masm.loadArgumentsObjectElementHole(argsObj, index, out, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitInArgumentsObjectArg(LInArgumentsObjectArg* lir) {
+ Register temp = ToRegister(lir->temp0());
+ Register argsObj = ToRegister(lir->argsObject());
+ Register index = ToRegister(lir->index());
+ Register out = ToRegister(lir->output());
+
+ Label bail;
+ masm.loadArgumentsObjectElementExists(argsObj, index, out, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitArgumentsObjectLength(LArgumentsObjectLength* lir) {
+ Register argsObj = ToRegister(lir->argsObject());
+ Register out = ToRegister(lir->output());
+
+ Label bail;
+ masm.loadArgumentsObjectLength(argsObj, out, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitArrayFromArgumentsObject(
+ LArrayFromArgumentsObject* lir) {
+ pushArg(ToRegister(lir->argsObject()));
+
+ using Fn = ArrayObject* (*)(JSContext*, Handle<ArgumentsObject*>);
+ callVM<Fn, js::ArrayFromArgumentsObject>(lir);
+}
+
+void CodeGenerator::visitGuardArgumentsObjectFlags(
+ LGuardArgumentsObjectFlags* lir) {
+ Register argsObj = ToRegister(lir->argsObject());
+ Register temp = ToRegister(lir->temp0());
+
+ Label bail;
+ masm.branchTestArgumentsObjectFlags(argsObj, temp, lir->mir()->flags(),
+ Assembler::NonZero, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitReturnFromCtor(LReturnFromCtor* lir) {
+ ValueOperand value = ToValue(lir, LReturnFromCtor::ValueIndex);
+ Register obj = ToRegister(lir->object());
+ Register output = ToRegister(lir->output());
+
+ Label valueIsObject, end;
+
+ masm.branchTestObject(Assembler::Equal, value, &valueIsObject);
+
+ // Value is not an object. Return that other object.
+ masm.movePtr(obj, output);
+ masm.jump(&end);
+
+ // Value is an object. Return unbox(Value).
+ masm.bind(&valueIsObject);
+ Register payload = masm.extractObject(value, output);
+ if (payload != output) {
+ masm.movePtr(payload, output);
+ }
+
+ masm.bind(&end);
+}
+
+class OutOfLineBoxNonStrictThis : public OutOfLineCodeBase<CodeGenerator> {
+ LBoxNonStrictThis* ins_;
+
+ public:
+ explicit OutOfLineBoxNonStrictThis(LBoxNonStrictThis* ins) : ins_(ins) {}
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineBoxNonStrictThis(this);
+ }
+ LBoxNonStrictThis* ins() const { return ins_; }
+};
+
+void CodeGenerator::visitBoxNonStrictThis(LBoxNonStrictThis* lir) {
+ ValueOperand value = ToValue(lir, LBoxNonStrictThis::ValueIndex);
+ Register output = ToRegister(lir->output());
+
+ auto* ool = new (alloc()) OutOfLineBoxNonStrictThis(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ masm.fallibleUnboxObject(value, output, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineBoxNonStrictThis(
+ OutOfLineBoxNonStrictThis* ool) {
+ LBoxNonStrictThis* lir = ool->ins();
+
+ ValueOperand value = ToValue(lir, LBoxNonStrictThis::ValueIndex);
+ Register output = ToRegister(lir->output());
+
+ Label notNullOrUndefined;
+ {
+ Label isNullOrUndefined;
+ ScratchTagScope tag(masm, value);
+ masm.splitTagForTest(value, tag);
+ masm.branchTestUndefined(Assembler::Equal, tag, &isNullOrUndefined);
+ masm.branchTestNull(Assembler::NotEqual, tag, &notNullOrUndefined);
+ masm.bind(&isNullOrUndefined);
+ masm.movePtr(ImmGCPtr(lir->mir()->globalThis()), output);
+ masm.jump(ool->rejoin());
+ }
+
+ masm.bind(&notNullOrUndefined);
+
+ saveLive(lir);
+
+ pushArg(value);
+ using Fn = JSObject* (*)(JSContext*, HandleValue);
+ callVM<Fn, BoxNonStrictThis>(lir);
+
+ StoreRegisterTo(output).generate(this);
+ restoreLiveIgnore(lir, StoreRegisterTo(output).clobbered());
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitImplicitThis(LImplicitThis* lir) {
+ pushArg(ImmGCPtr(lir->mir()->name()));
+ pushArg(ToRegister(lir->env()));
+
+ using Fn = bool (*)(JSContext*, HandleObject, Handle<PropertyName*>,
+ MutableHandleValue);
+ callVM<Fn, ImplicitThisOperation>(lir);
+}
+
+void CodeGenerator::visitArrayLength(LArrayLength* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register output = ToRegister(lir->output());
+
+ Address length(elements, ObjectElements::offsetOfLength());
+ masm.load32(length, output);
+
+ // Bail out if the length doesn't fit in int32.
+ bailoutTest32(Assembler::Signed, output, output, lir->snapshot());
+}
+
+static void SetLengthFromIndex(MacroAssembler& masm, const LAllocation* index,
+ const Address& length) {
+ if (index->isConstant()) {
+ masm.store32(Imm32(ToInt32(index) + 1), length);
+ } else {
+ Register newLength = ToRegister(index);
+ masm.add32(Imm32(1), newLength);
+ masm.store32(newLength, length);
+ masm.sub32(Imm32(1), newLength);
+ }
+}
+
+void CodeGenerator::visitSetArrayLength(LSetArrayLength* lir) {
+ Address length(ToRegister(lir->elements()), ObjectElements::offsetOfLength());
+ SetLengthFromIndex(masm, lir->index(), length);
+}
+
+void CodeGenerator::visitFunctionLength(LFunctionLength* lir) {
+ Register function = ToRegister(lir->function());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+
+ // Get the JSFunction flags.
+ masm.load32(Address(function, JSFunction::offsetOfFlagsAndArgCount()),
+ output);
+
+ // Functions with a SelfHostedLazyScript must be compiled with the slow-path
+ // before the function length is known. If the length was previously resolved,
+ // the length property may be shadowed.
+ masm.branchTest32(
+ Assembler::NonZero, output,
+ Imm32(FunctionFlags::SELFHOSTLAZY | FunctionFlags::RESOLVED_LENGTH),
+ &bail);
+
+ masm.loadFunctionLength(function, output, output, &bail);
+
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitFunctionName(LFunctionName* lir) {
+ Register function = ToRegister(lir->function());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+
+ const JSAtomState& names = gen->runtime->names();
+ masm.loadFunctionName(function, output, ImmGCPtr(names.empty), &bail);
+
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+template <class OrderedHashTable>
+static void RangeFront(MacroAssembler&, Register, Register, Register);
+
+template <>
+void RangeFront<ValueMap>(MacroAssembler& masm, Register range, Register i,
+ Register front) {
+ masm.loadPtr(Address(range, ValueMap::Range::offsetOfHashTable()), front);
+ masm.loadPtr(Address(front, ValueMap::offsetOfImplData()), front);
+
+ MOZ_ASSERT(ValueMap::offsetOfImplDataElement() == 0,
+ "offsetof(Data, element) is 0");
+ static_assert(ValueMap::sizeofImplData() == 24, "sizeof(Data) is 24");
+ masm.mulBy3(i, i);
+ masm.lshiftPtr(Imm32(3), i);
+ masm.addPtr(i, front);
+}
+
+template <>
+void RangeFront<ValueSet>(MacroAssembler& masm, Register range, Register i,
+ Register front) {
+ masm.loadPtr(Address(range, ValueSet::Range::offsetOfHashTable()), front);
+ masm.loadPtr(Address(front, ValueSet::offsetOfImplData()), front);
+
+ MOZ_ASSERT(ValueSet::offsetOfImplDataElement() == 0,
+ "offsetof(Data, element) is 0");
+ static_assert(ValueSet::sizeofImplData() == 16, "sizeof(Data) is 16");
+ masm.lshiftPtr(Imm32(4), i);
+ masm.addPtr(i, front);
+}
+
+template <class OrderedHashTable>
+static void RangePopFront(MacroAssembler& masm, Register range, Register front,
+ Register dataLength, Register temp) {
+ Register i = temp;
+
+ masm.add32(Imm32(1),
+ Address(range, OrderedHashTable::Range::offsetOfCount()));
+
+ masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), i);
+
+ Label done, seek;
+ masm.bind(&seek);
+ masm.add32(Imm32(1), i);
+ masm.branch32(Assembler::AboveOrEqual, i, dataLength, &done);
+
+ // We can add sizeof(Data) to |front| to select the next element, because
+ // |front| and |range.ht.data[i]| point to the same location.
+ MOZ_ASSERT(OrderedHashTable::offsetOfImplDataElement() == 0,
+ "offsetof(Data, element) is 0");
+ masm.addPtr(Imm32(OrderedHashTable::sizeofImplData()), front);
+
+ masm.branchTestMagic(Assembler::Equal,
+ Address(front, OrderedHashTable::offsetOfEntryKey()),
+ JS_HASH_KEY_EMPTY, &seek);
+
+ masm.bind(&done);
+ masm.store32(i, Address(range, OrderedHashTable::Range::offsetOfI()));
+}
+
+template <class OrderedHashTable>
+static inline void RangeDestruct(MacroAssembler& masm, Register iter,
+ Register range, Register temp0,
+ Register temp1) {
+ Register next = temp0;
+ Register prevp = temp1;
+
+ masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfNext()), next);
+ masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfPrevP()), prevp);
+ masm.storePtr(next, Address(prevp, 0));
+
+ Label hasNoNext;
+ masm.branchTestPtr(Assembler::Zero, next, next, &hasNoNext);
+
+ masm.storePtr(prevp, Address(next, OrderedHashTable::Range::offsetOfPrevP()));
+
+ masm.bind(&hasNoNext);
+
+ Label nurseryAllocated;
+ masm.branchPtrInNurseryChunk(Assembler::Equal, iter, temp0,
+ &nurseryAllocated);
+
+ masm.callFreeStub(range);
+
+ masm.bind(&nurseryAllocated);
+}
+
+template <>
+void CodeGenerator::emitLoadIteratorValues<ValueMap>(Register result,
+ Register temp,
+ Register front) {
+ size_t elementsOffset = NativeObject::offsetOfFixedElements();
+
+ Address keyAddress(front, ValueMap::Entry::offsetOfKey());
+ Address valueAddress(front, ValueMap::Entry::offsetOfValue());
+ Address keyElemAddress(result, elementsOffset);
+ Address valueElemAddress(result, elementsOffset + sizeof(Value));
+ masm.guardedCallPreBarrier(keyElemAddress, MIRType::Value);
+ masm.guardedCallPreBarrier(valueElemAddress, MIRType::Value);
+ masm.storeValue(keyAddress, keyElemAddress, temp);
+ masm.storeValue(valueAddress, valueElemAddress, temp);
+
+ Label emitBarrier, skipBarrier;
+ masm.branchValueIsNurseryCell(Assembler::Equal, keyAddress, temp,
+ &emitBarrier);
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, valueAddress, temp,
+ &skipBarrier);
+ {
+ masm.bind(&emitBarrier);
+ saveVolatile(temp);
+ emitPostWriteBarrier(result);
+ restoreVolatile(temp);
+ }
+ masm.bind(&skipBarrier);
+}
+
+template <>
+void CodeGenerator::emitLoadIteratorValues<ValueSet>(Register result,
+ Register temp,
+ Register front) {
+ size_t elementsOffset = NativeObject::offsetOfFixedElements();
+
+ Address keyAddress(front, ValueSet::offsetOfEntryKey());
+ Address keyElemAddress(result, elementsOffset);
+ masm.guardedCallPreBarrier(keyElemAddress, MIRType::Value);
+ masm.storeValue(keyAddress, keyElemAddress, temp);
+
+ Label skipBarrier;
+ masm.branchValueIsNurseryCell(Assembler::NotEqual, keyAddress, temp,
+ &skipBarrier);
+ {
+ saveVolatile(temp);
+ emitPostWriteBarrier(result);
+ restoreVolatile(temp);
+ }
+ masm.bind(&skipBarrier);
+}
+
+template <class IteratorObject, class OrderedHashTable>
+void CodeGenerator::emitGetNextEntryForIterator(LGetNextEntryForIterator* lir) {
+ Register iter = ToRegister(lir->iter());
+ Register result = ToRegister(lir->result());
+ Register temp = ToRegister(lir->temp0());
+ Register dataLength = ToRegister(lir->temp1());
+ Register range = ToRegister(lir->temp2());
+ Register output = ToRegister(lir->output());
+
+#ifdef DEBUG
+ // Self-hosted code is responsible for ensuring GetNextEntryForIterator is
+ // only called with the correct iterator class. Assert here all self-
+ // hosted callers of GetNextEntryForIterator perform this class check.
+ // No Spectre mitigations are needed because this is DEBUG-only code.
+ Label success;
+ masm.branchTestObjClassNoSpectreMitigations(
+ Assembler::Equal, iter, &IteratorObject::class_, temp, &success);
+ masm.assumeUnreachable("Iterator object should have the correct class.");
+ masm.bind(&success);
+#endif
+
+ masm.loadPrivate(Address(iter, NativeObject::getFixedSlotOffset(
+ IteratorObject::RangeSlot)),
+ range);
+
+ Label iterAlreadyDone, iterDone, done;
+ masm.branchTestPtr(Assembler::Zero, range, range, &iterAlreadyDone);
+
+ masm.load32(Address(range, OrderedHashTable::Range::offsetOfI()), temp);
+ masm.loadPtr(Address(range, OrderedHashTable::Range::offsetOfHashTable()),
+ dataLength);
+ masm.load32(Address(dataLength, OrderedHashTable::offsetOfImplDataLength()),
+ dataLength);
+ masm.branch32(Assembler::AboveOrEqual, temp, dataLength, &iterDone);
+ {
+ masm.Push(iter);
+
+ Register front = iter;
+ RangeFront<OrderedHashTable>(masm, range, temp, front);
+
+ emitLoadIteratorValues<OrderedHashTable>(result, temp, front);
+
+ RangePopFront<OrderedHashTable>(masm, range, front, dataLength, temp);
+
+ masm.Pop(iter);
+ masm.move32(Imm32(0), output);
+ }
+ masm.jump(&done);
+ {
+ masm.bind(&iterDone);
+
+ RangeDestruct<OrderedHashTable>(masm, iter, range, temp, dataLength);
+
+ masm.storeValue(PrivateValue(nullptr),
+ Address(iter, NativeObject::getFixedSlotOffset(
+ IteratorObject::RangeSlot)));
+
+ masm.bind(&iterAlreadyDone);
+
+ masm.move32(Imm32(1), output);
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGetNextEntryForIterator(
+ LGetNextEntryForIterator* lir) {
+ if (lir->mir()->mode() == MGetNextEntryForIterator::Map) {
+ emitGetNextEntryForIterator<MapIteratorObject, ValueMap>(lir);
+ } else {
+ MOZ_ASSERT(lir->mir()->mode() == MGetNextEntryForIterator::Set);
+ emitGetNextEntryForIterator<SetIteratorObject, ValueSet>(lir);
+ }
+}
+
+// The point of these is to inform Ion of where these values already are; they
+// don't normally generate (much) code.
+void CodeGenerator::visitWasmRegisterPairResult(LWasmRegisterPairResult* lir) {}
+void CodeGenerator::visitWasmStackResult(LWasmStackResult* lir) {}
+void CodeGenerator::visitWasmStackResult64(LWasmStackResult64* lir) {}
+
+void CodeGenerator::visitWasmStackResultArea(LWasmStackResultArea* lir) {
+ LAllocation* output = lir->getDef(0)->output();
+ MOZ_ASSERT(output->isStackArea());
+ bool tempInit = false;
+ for (auto iter = output->toStackArea()->results(); iter; iter.next()) {
+ // Zero out ref stack results.
+ if (iter.isGcPointer()) {
+ Register temp = ToRegister(lir->temp0());
+ if (!tempInit) {
+ masm.xorPtr(temp, temp);
+ tempInit = true;
+ }
+ masm.storePtr(temp, ToAddress(iter.alloc()));
+ }
+ }
+}
+
+void CodeGenerator::visitWasmRegisterResult(LWasmRegisterResult* lir) {
+#ifdef JS_64BIT
+ if (MWasmRegisterResult* mir = lir->mir()) {
+ if (mir->type() == MIRType::Int32) {
+ masm.widenInt32(ToRegister(lir->output()));
+ }
+ }
+#endif
+}
+
+void CodeGenerator::visitWasmCall(LWasmCall* lir) {
+ const MWasmCallBase* callBase = lir->callBase();
+
+ // If this call is in Wasm try code block, initialise a wasm::TryNote for this
+ // call.
+ bool inTry = callBase->inTry();
+ if (inTry) {
+ size_t tryNoteIndex = callBase->tryNoteIndex();
+ wasm::TryNoteVector& tryNotes = masm.tryNotes();
+ wasm::TryNote& tryNote = tryNotes[tryNoteIndex];
+ tryNote.setTryBodyBegin(masm.currentOffset());
+ }
+
+ MOZ_ASSERT((sizeof(wasm::Frame) + masm.framePushed()) % WasmStackAlignment ==
+ 0);
+ static_assert(
+ WasmStackAlignment >= ABIStackAlignment &&
+ WasmStackAlignment % ABIStackAlignment == 0,
+ "The wasm stack alignment should subsume the ABI-required alignment");
+
+#ifdef DEBUG
+ Label ok;
+ masm.branchTestStackPtr(Assembler::Zero, Imm32(WasmStackAlignment - 1), &ok);
+ masm.breakpoint();
+ masm.bind(&ok);
+#endif
+
+ // LWasmCallBase::isCallPreserved() assumes that all MWasmCalls preserve the
+ // instance and pinned regs. The only case where where we don't have to
+ // reload the instance and pinned regs is when the callee preserves them.
+ bool reloadRegs = true;
+ bool switchRealm = true;
+
+ const wasm::CallSiteDesc& desc = callBase->desc();
+ const wasm::CalleeDesc& callee = callBase->callee();
+ CodeOffset retOffset;
+ CodeOffset secondRetOffset;
+ switch (callee.which()) {
+ case wasm::CalleeDesc::Func:
+ retOffset = masm.call(desc, callee.funcIndex());
+ reloadRegs = false;
+ switchRealm = false;
+ break;
+ case wasm::CalleeDesc::Import:
+ retOffset = masm.wasmCallImport(desc, callee);
+ break;
+ case wasm::CalleeDesc::AsmJSTable:
+ retOffset = masm.asmCallIndirect(desc, callee);
+ break;
+ case wasm::CalleeDesc::WasmTable: {
+ Label* boundsCheckFailed = nullptr;
+ if (lir->needsBoundsCheck()) {
+ OutOfLineAbortingWasmTrap* ool =
+ new (alloc()) OutOfLineAbortingWasmTrap(
+ wasm::BytecodeOffset(desc.lineOrBytecode()),
+ wasm::Trap::OutOfBounds);
+ if (lir->isCatchable()) {
+ addOutOfLineCode(ool, lir->mirCatchable());
+ } else {
+ addOutOfLineCode(ool, lir->mirUncatchable());
+ }
+ boundsCheckFailed = ool->entry();
+ }
+ Label* nullCheckFailed = nullptr;
+#ifndef WASM_HAS_HEAPREG
+ {
+ OutOfLineAbortingWasmTrap* ool =
+ new (alloc()) OutOfLineAbortingWasmTrap(
+ wasm::BytecodeOffset(desc.lineOrBytecode()),
+ wasm::Trap::IndirectCallToNull);
+ if (lir->isCatchable()) {
+ addOutOfLineCode(ool, lir->mirCatchable());
+ } else {
+ addOutOfLineCode(ool, lir->mirUncatchable());
+ }
+ nullCheckFailed = ool->entry();
+ }
+#endif
+ masm.wasmCallIndirect(desc, callee, boundsCheckFailed, nullCheckFailed,
+ lir->tableSize(), &retOffset, &secondRetOffset);
+ // Register reloading and realm switching are handled dynamically inside
+ // wasmCallIndirect. There are two return offsets, one for each call
+ // instruction (fast path and slow path).
+ reloadRegs = false;
+ switchRealm = false;
+ break;
+ }
+ case wasm::CalleeDesc::Builtin:
+ retOffset = masm.call(desc, callee.builtin());
+ reloadRegs = false;
+ switchRealm = false;
+ break;
+ case wasm::CalleeDesc::BuiltinInstanceMethod:
+ retOffset = masm.wasmCallBuiltinInstanceMethod(
+ desc, callBase->instanceArg(), callee.builtin(),
+ callBase->builtinMethodFailureMode());
+ switchRealm = false;
+ break;
+ case wasm::CalleeDesc::FuncRef:
+ // Register reloading and realm switching are handled dynamically inside
+ // wasmCallRef. There are two return offsets, one for each call
+ // instruction (fast path and slow path).
+ masm.wasmCallRef(desc, callee, &retOffset, &secondRetOffset);
+ reloadRegs = false;
+ switchRealm = false;
+ break;
+ }
+
+ // Note the assembler offset for the associated LSafePoint.
+ markSafepointAt(retOffset.offset(), lir);
+
+ // Now that all the outbound in-memory args are on the stack, note the
+ // required lower boundary point of the associated StackMap.
+ uint32_t framePushedAtStackMapBase =
+ masm.framePushed() - callBase->stackArgAreaSizeUnaligned();
+ lir->safepoint()->setFramePushedAtStackMapBase(framePushedAtStackMapBase);
+ MOZ_ASSERT(!lir->safepoint()->isWasmTrap());
+
+ // Note the assembler offset and framePushed for use by the adjunct
+ // LSafePoint, see visitor for LWasmCallIndirectAdjunctSafepoint below.
+ if (callee.which() == wasm::CalleeDesc::WasmTable) {
+ lir->adjunctSafepoint()->recordSafepointInfo(secondRetOffset,
+ framePushedAtStackMapBase);
+ }
+
+ if (reloadRegs) {
+ masm.loadPtr(
+ Address(masm.getStackPointer(), WasmCallerInstanceOffsetBeforeCall),
+ InstanceReg);
+ masm.loadWasmPinnedRegsFromInstance();
+ if (switchRealm) {
+ masm.switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1);
+ }
+ } else {
+ MOZ_ASSERT(!switchRealm);
+ }
+
+ if (inTry) {
+ // Set the end of the try note range
+ size_t tryNoteIndex = callBase->tryNoteIndex();
+ wasm::TryNoteVector& tryNotes = masm.tryNotes();
+ wasm::TryNote& tryNote = tryNotes[tryNoteIndex];
+
+ // Don't set the end of the try note if we've OOM'ed, as the above
+ // instructions may not have been emitted, which will trigger an assert
+ // about zero-length try-notes. This is okay as this compilation will be
+ // thrown away.
+ if (!masm.oom()) {
+ tryNote.setTryBodyEnd(masm.currentOffset());
+ }
+
+ // This instruction or the adjunct safepoint must be the last instruction
+ // in the block. No other instructions may be inserted.
+ LBlock* block = lir->block();
+ MOZ_RELEASE_ASSERT(*block->rbegin() == lir ||
+ (block->rbegin()->isWasmCallIndirectAdjunctSafepoint() &&
+ *(++block->rbegin()) == lir));
+
+ // Jump to the fallthrough block
+ jumpToBlock(lir->mirCatchable()->getSuccessor(
+ MWasmCallCatchable::FallthroughBranchIndex));
+ }
+}
+
+void CodeGenerator::visitWasmCallLandingPrePad(LWasmCallLandingPrePad* lir) {
+ LBlock* block = lir->block();
+ MWasmCallLandingPrePad* mir = lir->mir();
+ MBasicBlock* mirBlock = mir->block();
+ MBasicBlock* callMirBlock = mir->callBlock();
+
+ // This block must be the pre-pad successor of the call block. No blocks may
+ // be inserted between us, such as for critical edge splitting.
+ MOZ_RELEASE_ASSERT(mirBlock == callMirBlock->getSuccessor(
+ MWasmCallCatchable::PrePadBranchIndex));
+
+ // This instruction or a move group must be the first instruction in the
+ // block. No other instructions may be inserted.
+ MOZ_RELEASE_ASSERT(*block->begin() == lir || (block->begin()->isMoveGroup() &&
+ *(++block->begin()) == lir));
+
+ wasm::TryNoteVector& tryNotes = masm.tryNotes();
+ wasm::TryNote& tryNote = tryNotes[mir->tryNoteIndex()];
+ // Set the entry point for the call try note to be the beginning of this
+ // block. The above assertions (and assertions in visitWasmCall) guarantee
+ // that we are not skipping over instructions that should be executed.
+ tryNote.setLandingPad(block->label()->offset(), masm.framePushed());
+}
+
+void CodeGenerator::visitWasmCallIndirectAdjunctSafepoint(
+ LWasmCallIndirectAdjunctSafepoint* lir) {
+ markSafepointAt(lir->safepointLocation().offset(), lir);
+ lir->safepoint()->setFramePushedAtStackMapBase(
+ lir->framePushedAtStackMapBase());
+}
+
+template <typename InstructionWithMaybeTrapSite>
+void EmitSignalNullCheckTrapSite(MacroAssembler& masm,
+ InstructionWithMaybeTrapSite* ins) {
+ if (!ins->maybeTrap()) {
+ return;
+ }
+ wasm::BytecodeOffset trapOffset(ins->maybeTrap()->offset);
+ masm.append(wasm::Trap::NullPointerDereference,
+ wasm::TrapSite(masm.currentOffset(), trapOffset));
+}
+
+void CodeGenerator::visitWasmLoadSlot(LWasmLoadSlot* ins) {
+ MIRType type = ins->type();
+ MWideningOp wideningOp = ins->wideningOp();
+ Register container = ToRegister(ins->containerRef());
+ Address addr(container, ins->offset());
+ AnyRegister dst = ToAnyRegister(ins->output());
+
+ EmitSignalNullCheckTrapSite(masm, ins);
+ switch (type) {
+ case MIRType::Int32:
+ switch (wideningOp) {
+ case MWideningOp::None:
+ masm.load32(addr, dst.gpr());
+ break;
+ case MWideningOp::FromU16:
+ masm.load16ZeroExtend(addr, dst.gpr());
+ break;
+ case MWideningOp::FromS16:
+ masm.load16SignExtend(addr, dst.gpr());
+ break;
+ case MWideningOp::FromU8:
+ masm.load8ZeroExtend(addr, dst.gpr());
+ break;
+ case MWideningOp::FromS8:
+ masm.load8SignExtend(addr, dst.gpr());
+ break;
+ default:
+ MOZ_CRASH("unexpected widening op in ::visitWasmLoadSlot");
+ }
+ break;
+ case MIRType::Float32:
+ MOZ_ASSERT(wideningOp == MWideningOp::None);
+ masm.loadFloat32(addr, dst.fpu());
+ break;
+ case MIRType::Double:
+ MOZ_ASSERT(wideningOp == MWideningOp::None);
+ masm.loadDouble(addr, dst.fpu());
+ break;
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ MOZ_ASSERT(wideningOp == MWideningOp::None);
+ masm.loadPtr(addr, dst.gpr());
+ break;
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ MOZ_ASSERT(wideningOp == MWideningOp::None);
+ masm.loadUnalignedSimd128(addr, dst.fpu());
+ break;
+#endif
+ default:
+ MOZ_CRASH("unexpected type in ::visitWasmLoadSlot");
+ }
+}
+
+void CodeGenerator::visitWasmStoreSlot(LWasmStoreSlot* ins) {
+ MIRType type = ins->type();
+ MNarrowingOp narrowingOp = ins->narrowingOp();
+ Register container = ToRegister(ins->containerRef());
+ Address addr(container, ins->offset());
+ AnyRegister src = ToAnyRegister(ins->value());
+ if (type != MIRType::Int32) {
+ MOZ_RELEASE_ASSERT(narrowingOp == MNarrowingOp::None);
+ }
+
+ EmitSignalNullCheckTrapSite(masm, ins);
+ switch (type) {
+ case MIRType::Int32:
+ switch (narrowingOp) {
+ case MNarrowingOp::None:
+ masm.store32(src.gpr(), addr);
+ break;
+ case MNarrowingOp::To16:
+ masm.store16(src.gpr(), addr);
+ break;
+ case MNarrowingOp::To8:
+ masm.store8(src.gpr(), addr);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case MIRType::Float32:
+ masm.storeFloat32(src.fpu(), addr);
+ break;
+ case MIRType::Double:
+ masm.storeDouble(src.fpu(), addr);
+ break;
+ case MIRType::Pointer:
+ // This could be correct, but it would be a new usage, so check carefully.
+ MOZ_CRASH("Unexpected type in visitWasmStoreSlot.");
+ case MIRType::RefOrNull:
+ MOZ_CRASH("Bad type in visitWasmStoreSlot. Use LWasmStoreRef.");
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ masm.storeUnalignedSimd128(src.fpu(), addr);
+ break;
+#endif
+ default:
+ MOZ_CRASH("unexpected type in StorePrimitiveValue");
+ }
+}
+
+void CodeGenerator::visitWasmLoadTableElement(LWasmLoadTableElement* ins) {
+ Register elements = ToRegister(ins->elements());
+ Register index = ToRegister(ins->index());
+ Register output = ToRegister(ins->output());
+ masm.loadPtr(BaseIndex(elements, index, ScalePointer), output);
+}
+
+void CodeGenerator::visitWasmDerivedPointer(LWasmDerivedPointer* ins) {
+ masm.movePtr(ToRegister(ins->base()), ToRegister(ins->output()));
+ masm.addPtr(Imm32(int32_t(ins->offset())), ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitWasmDerivedIndexPointer(
+ LWasmDerivedIndexPointer* ins) {
+ Register base = ToRegister(ins->base());
+ Register index = ToRegister(ins->index());
+ Register output = ToRegister(ins->output());
+ masm.computeEffectiveAddress(BaseIndex(base, index, ins->scale()), output);
+}
+
+void CodeGenerator::visitWasmStoreRef(LWasmStoreRef* ins) {
+ Register instance = ToRegister(ins->instance());
+ Register valueAddr = ToRegister(ins->valueAddr());
+ Register value = ToRegister(ins->value());
+ Register temp = ToRegister(ins->temp0());
+
+ Label skipPreBarrier;
+ wasm::EmitWasmPreBarrierGuard(masm, instance, temp, valueAddr,
+ &skipPreBarrier);
+ wasm::EmitWasmPreBarrierCall(masm, instance, temp, valueAddr);
+ masm.bind(&skipPreBarrier);
+
+ masm.storePtr(value, Address(valueAddr, 0));
+ // The postbarrier is handled separately.
+}
+
+void CodeGenerator::visitWasmLoadSlotI64(LWasmLoadSlotI64* ins) {
+ Register container = ToRegister(ins->containerRef());
+ Address addr(container, ins->offset());
+ Register64 output = ToOutRegister64(ins);
+ EmitSignalNullCheckTrapSite(masm, ins);
+ masm.load64(addr, output);
+}
+
+void CodeGenerator::visitWasmStoreSlotI64(LWasmStoreSlotI64* ins) {
+ Register container = ToRegister(ins->containerRef());
+ Address addr(container, ins->offset());
+ Register64 value = ToRegister64(ins->value());
+ EmitSignalNullCheckTrapSite(masm, ins);
+ masm.store64(value, addr);
+}
+
+void CodeGenerator::visitArrayBufferByteLength(LArrayBufferByteLength* lir) {
+ Register obj = ToRegister(lir->object());
+ Register out = ToRegister(lir->output());
+ masm.loadArrayBufferByteLengthIntPtr(obj, out);
+}
+
+void CodeGenerator::visitArrayBufferViewLength(LArrayBufferViewLength* lir) {
+ Register obj = ToRegister(lir->object());
+ Register out = ToRegister(lir->output());
+ masm.loadArrayBufferViewLengthIntPtr(obj, out);
+}
+
+void CodeGenerator::visitArrayBufferViewByteOffset(
+ LArrayBufferViewByteOffset* lir) {
+ Register obj = ToRegister(lir->object());
+ Register out = ToRegister(lir->output());
+ masm.loadArrayBufferViewByteOffsetIntPtr(obj, out);
+}
+
+void CodeGenerator::visitArrayBufferViewElements(
+ LArrayBufferViewElements* lir) {
+ Register obj = ToRegister(lir->object());
+ Register out = ToRegister(lir->output());
+ masm.loadPtr(Address(obj, ArrayBufferViewObject::dataOffset()), out);
+}
+
+void CodeGenerator::visitTypedArrayElementSize(LTypedArrayElementSize* lir) {
+ Register obj = ToRegister(lir->object());
+ Register out = ToRegister(lir->output());
+
+ masm.typedArrayElementSize(obj, out);
+}
+
+void CodeGenerator::visitGuardHasAttachedArrayBuffer(
+ LGuardHasAttachedArrayBuffer* lir) {
+ Register obj = ToRegister(lir->object());
+ Register temp = ToRegister(lir->temp0());
+
+ Label bail;
+ masm.branchIfHasDetachedArrayBuffer(obj, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+class OutOfLineGuardNumberToIntPtrIndex
+ : public OutOfLineCodeBase<CodeGenerator> {
+ LGuardNumberToIntPtrIndex* lir_;
+
+ public:
+ explicit OutOfLineGuardNumberToIntPtrIndex(LGuardNumberToIntPtrIndex* lir)
+ : lir_(lir) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineGuardNumberToIntPtrIndex(this);
+ }
+ LGuardNumberToIntPtrIndex* lir() const { return lir_; }
+};
+
+void CodeGenerator::visitGuardNumberToIntPtrIndex(
+ LGuardNumberToIntPtrIndex* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ if (!lir->mir()->supportOOB()) {
+ Label bail;
+ masm.convertDoubleToPtr(input, output, &bail, false);
+ bailoutFrom(&bail, lir->snapshot());
+ return;
+ }
+
+ auto* ool = new (alloc()) OutOfLineGuardNumberToIntPtrIndex(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ masm.convertDoubleToPtr(input, output, ool->entry(), false);
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineGuardNumberToIntPtrIndex(
+ OutOfLineGuardNumberToIntPtrIndex* ool) {
+ // Substitute the invalid index with an arbitrary out-of-bounds index.
+ masm.movePtr(ImmWord(-1), ToRegister(ool->lir()->output()));
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitStringLength(LStringLength* lir) {
+ Register input = ToRegister(lir->string());
+ Register output = ToRegister(lir->output());
+
+ masm.loadStringLength(input, output);
+}
+
+void CodeGenerator::visitMinMaxI(LMinMaxI* ins) {
+ Register first = ToRegister(ins->first());
+ Register output = ToRegister(ins->output());
+
+ MOZ_ASSERT(first == output);
+
+ Assembler::Condition cond =
+ ins->mir()->isMax() ? Assembler::GreaterThan : Assembler::LessThan;
+
+ if (ins->second()->isConstant()) {
+ Label done;
+ masm.branch32(cond, first, Imm32(ToInt32(ins->second())), &done);
+ masm.move32(Imm32(ToInt32(ins->second())), output);
+ masm.bind(&done);
+ } else {
+ Register second = ToRegister(ins->second());
+ masm.cmp32Move32(cond, second, first, second, output);
+ }
+}
+
+void CodeGenerator::visitMinMaxArrayI(LMinMaxArrayI* ins) {
+ Register array = ToRegister(ins->array());
+ Register output = ToRegister(ins->output());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ bool isMax = ins->isMax();
+
+ Label bail;
+ masm.minMaxArrayInt32(array, output, temp1, temp2, temp3, isMax, &bail);
+ bailoutFrom(&bail, ins->snapshot());
+}
+
+void CodeGenerator::visitMinMaxArrayD(LMinMaxArrayD* ins) {
+ Register array = ToRegister(ins->array());
+ FloatRegister output = ToFloatRegister(ins->output());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ FloatRegister floatTemp = ToFloatRegister(ins->floatTemp());
+ bool isMax = ins->isMax();
+
+ Label bail;
+ masm.minMaxArrayNumber(array, output, floatTemp, temp1, temp2, isMax, &bail);
+ bailoutFrom(&bail, ins->snapshot());
+}
+
+// For Abs*, lowering will have tied input to output on platforms where that is
+// sensible, and otherwise left them untied.
+
+void CodeGenerator::visitAbsI(LAbsI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ if (ins->mir()->fallible()) {
+ Label positive;
+ if (input != output) {
+ masm.move32(input, output);
+ }
+ masm.branchTest32(Assembler::NotSigned, output, output, &positive);
+ Label bail;
+ masm.branchNeg32(Assembler::Overflow, output, &bail);
+ bailoutFrom(&bail, ins->snapshot());
+ masm.bind(&positive);
+ } else {
+ masm.abs32(input, output);
+ }
+}
+
+void CodeGenerator::visitAbsD(LAbsD* ins) {
+ masm.absDouble(ToFloatRegister(ins->input()), ToFloatRegister(ins->output()));
+}
+
+void CodeGenerator::visitAbsF(LAbsF* ins) {
+ masm.absFloat32(ToFloatRegister(ins->input()),
+ ToFloatRegister(ins->output()));
+}
+
+void CodeGenerator::visitPowII(LPowII* ins) {
+ Register value = ToRegister(ins->value());
+ Register power = ToRegister(ins->power());
+ Register output = ToRegister(ins->output());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+
+ Label bailout;
+ masm.pow32(value, power, output, temp0, temp1, &bailout);
+ bailoutFrom(&bailout, ins->snapshot());
+}
+
+void CodeGenerator::visitPowI(LPowI* ins) {
+ FloatRegister value = ToFloatRegister(ins->value());
+ Register power = ToRegister(ins->power());
+
+ using Fn = double (*)(double x, int32_t y);
+ masm.setupAlignedABICall();
+ masm.passABIArg(value, MoveOp::DOUBLE);
+ masm.passABIArg(power);
+
+ masm.callWithABI<Fn, js::powi>(MoveOp::DOUBLE);
+ MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
+}
+
+void CodeGenerator::visitPowD(LPowD* ins) {
+ FloatRegister value = ToFloatRegister(ins->value());
+ FloatRegister power = ToFloatRegister(ins->power());
+
+ using Fn = double (*)(double x, double y);
+ masm.setupAlignedABICall();
+ masm.passABIArg(value, MoveOp::DOUBLE);
+ masm.passABIArg(power, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, ecmaPow>(MoveOp::DOUBLE);
+
+ MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
+}
+
+void CodeGenerator::visitPowOfTwoI(LPowOfTwoI* ins) {
+ Register power = ToRegister(ins->power());
+ Register output = ToRegister(ins->output());
+
+ uint32_t base = ins->base();
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(base));
+
+ uint32_t n = mozilla::FloorLog2(base);
+ MOZ_ASSERT(n != 0);
+
+ // Hacker's Delight, 2nd edition, theorem D2.
+ auto ceilingDiv = [](uint32_t x, uint32_t y) { return (x + y - 1) / y; };
+
+ // Take bailout if |power| is greater-or-equals |log_y(2^31)| or is negative.
+ // |2^(n*y) < 2^31| must hold, hence |n*y < 31| resp. |y < 31/n|.
+ //
+ // Note: it's important for this condition to match the code in CacheIR.cpp
+ // (CanAttachInt32Pow) to prevent failure loops.
+ bailoutCmp32(Assembler::AboveOrEqual, power, Imm32(ceilingDiv(31, n)),
+ ins->snapshot());
+
+ // Compute (2^n)^y as 2^(n*y) using repeated shifts. We could directly scale
+ // |power| and perform a single shift, but due to the lack of necessary
+ // MacroAssembler functionality, like multiplying a register with an
+ // immediate, we restrict the number of generated shift instructions when
+ // lowering this operation.
+ masm.move32(Imm32(1), output);
+ do {
+ masm.lshift32(power, output);
+ n--;
+ } while (n > 0);
+}
+
+void CodeGenerator::visitSqrtD(LSqrtD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+ masm.sqrtDouble(input, output);
+}
+
+void CodeGenerator::visitSqrtF(LSqrtF* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+ masm.sqrtFloat32(input, output);
+}
+
+void CodeGenerator::visitSignI(LSignI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+ masm.signInt32(input, output);
+}
+
+void CodeGenerator::visitSignD(LSignD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+ masm.signDouble(input, output);
+}
+
+void CodeGenerator::visitSignDI(LSignDI* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister temp = ToFloatRegister(ins->temp0());
+ Register output = ToRegister(ins->output());
+
+ Label bail;
+ masm.signDoubleToInt32(input, output, temp, &bail);
+ bailoutFrom(&bail, ins->snapshot());
+}
+
+void CodeGenerator::visitMathFunctionD(LMathFunctionD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
+
+ UnaryMathFunction fun = ins->mir()->function();
+ UnaryMathFunctionType funPtr = GetUnaryMathFunctionPtr(fun);
+
+ masm.setupAlignedABICall();
+
+ masm.passABIArg(input, MoveOp::DOUBLE);
+ masm.callWithABI(DynamicFunction<UnaryMathFunctionType>(funPtr),
+ MoveOp::DOUBLE);
+}
+
+void CodeGenerator::visitMathFunctionF(LMathFunctionF* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnFloat32Reg);
+
+ masm.setupAlignedABICall();
+ masm.passABIArg(input, MoveOp::FLOAT32);
+
+ using Fn = float (*)(float x);
+ Fn funptr = nullptr;
+ CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check;
+ switch (ins->mir()->function()) {
+ case UnaryMathFunction::Floor:
+ funptr = floorf;
+ check = CheckUnsafeCallWithABI::DontCheckOther;
+ break;
+ case UnaryMathFunction::Round:
+ funptr = math_roundf_impl;
+ break;
+ case UnaryMathFunction::Trunc:
+ funptr = math_truncf_impl;
+ break;
+ case UnaryMathFunction::Ceil:
+ funptr = ceilf;
+ check = CheckUnsafeCallWithABI::DontCheckOther;
+ break;
+ default:
+ MOZ_CRASH("Unknown or unsupported float32 math function");
+ }
+
+ masm.callWithABI(DynamicFunction<Fn>(funptr), MoveOp::FLOAT32, check);
+}
+
+void CodeGenerator::visitModD(LModD* ins) {
+ MOZ_ASSERT(!gen->compilingWasm());
+
+ FloatRegister lhs = ToFloatRegister(ins->lhs());
+ FloatRegister rhs = ToFloatRegister(ins->rhs());
+
+ MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
+
+ using Fn = double (*)(double a, double b);
+ masm.setupAlignedABICall();
+ masm.passABIArg(lhs, MoveOp::DOUBLE);
+ masm.passABIArg(rhs, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, NumberMod>(MoveOp::DOUBLE);
+}
+
+void CodeGenerator::visitModPowTwoD(LModPowTwoD* ins) {
+ FloatRegister lhs = ToFloatRegister(ins->lhs());
+ uint32_t divisor = ins->divisor();
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(divisor));
+
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ // Compute |n % d| using |copysign(n - (d * trunc(n / d)), n)|.
+ //
+ // This doesn't work if |d| isn't a power of two, because we may lose too much
+ // precision. For example |Number.MAX_VALUE % 3 == 2|, but
+ // |3 * trunc(Number.MAX_VALUE / 3) == Infinity|.
+
+ Label done;
+ {
+ ScratchDoubleScope scratch(masm);
+
+ // Subnormals can lead to performance degradation, which can make calling
+ // |fmod| faster than this inline implementation. Work around this issue by
+ // directly returning the input for any value in the interval ]-1, +1[.
+ Label notSubnormal;
+ masm.loadConstantDouble(1.0, scratch);
+ masm.loadConstantDouble(-1.0, output);
+ masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, lhs, scratch,
+ &notSubnormal);
+ masm.branchDouble(Assembler::DoubleLessThanOrEqual, lhs, output,
+ &notSubnormal);
+
+ masm.moveDouble(lhs, output);
+ masm.jump(&done);
+
+ masm.bind(&notSubnormal);
+
+ if (divisor == 1) {
+ // The pattern |n % 1 == 0| is used to detect integer numbers. We can skip
+ // the multiplication by one in this case.
+ masm.moveDouble(lhs, output);
+ masm.nearbyIntDouble(RoundingMode::TowardsZero, output, scratch);
+ masm.subDouble(scratch, output);
+ } else {
+ masm.loadConstantDouble(1.0 / double(divisor), scratch);
+ masm.loadConstantDouble(double(divisor), output);
+
+ masm.mulDouble(lhs, scratch);
+ masm.nearbyIntDouble(RoundingMode::TowardsZero, scratch, scratch);
+ masm.mulDouble(output, scratch);
+
+ masm.moveDouble(lhs, output);
+ masm.subDouble(scratch, output);
+ }
+ }
+
+ masm.copySignDouble(output, lhs, output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitWasmBuiltinModD(LWasmBuiltinModD* ins) {
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ FloatRegister lhs = ToFloatRegister(ins->lhs());
+ FloatRegister rhs = ToFloatRegister(ins->rhs());
+
+ MOZ_ASSERT(ToFloatRegister(ins->output()) == ReturnDoubleReg);
+
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs, MoveOp::DOUBLE);
+ masm.passABIArg(rhs, MoveOp::DOUBLE);
+
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ masm.callWithABI(ins->mir()->bytecodeOffset(), wasm::SymbolicAddress::ModD,
+ mozilla::Some(instanceOffset), MoveOp::DOUBLE);
+
+ masm.Pop(InstanceReg);
+}
+
+void CodeGenerator::visitBigIntAdd(LBigIntAdd* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::add>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // 0n + x == x
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(rhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // x + 0n == x
+ Label rhsNonZero;
+ masm.branchIfBigIntIsNonZero(rhs, &rhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&rhsNonZero);
+
+ // Call into the VM when either operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntNonZero(lhs, temp1, ool->entry());
+ masm.loadBigIntNonZero(rhs, temp2, ool->entry());
+
+ masm.branchAddPtr(Assembler::Overflow, temp2, temp1, ool->entry());
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntSub(LBigIntSub* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::sub>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // x - 0n == x
+ Label rhsNonZero;
+ masm.branchIfBigIntIsNonZero(rhs, &rhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&rhsNonZero);
+
+ // Call into the VM when either operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigInt(lhs, temp1, ool->entry());
+ masm.loadBigIntNonZero(rhs, temp2, ool->entry());
+
+ masm.branchSubPtr(Assembler::Overflow, temp2, temp1, ool->entry());
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntMul(LBigIntMul* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::mul>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // 0n * x == 0n
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // x * 0n == 0n
+ Label rhsNonZero;
+ masm.branchIfBigIntIsNonZero(rhs, &rhsNonZero);
+ masm.movePtr(rhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&rhsNonZero);
+
+ // Call into the VM when either operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntNonZero(lhs, temp1, ool->entry());
+ masm.loadBigIntNonZero(rhs, temp2, ool->entry());
+
+ masm.branchMulPtr(Assembler::Overflow, temp2, temp1, ool->entry());
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntDiv(LBigIntDiv* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::div>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // x / 0 throws an error.
+ if (ins->mir()->canBeDivideByZero()) {
+ masm.branchIfBigIntIsZero(rhs, ool->entry());
+ }
+
+ // 0n / x == 0n
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // Call into the VM when either operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntNonZero(lhs, temp1, ool->entry());
+ masm.loadBigIntNonZero(rhs, temp2, ool->entry());
+
+ // |BigInt::div()| returns |lhs| for |lhs / 1n|, which means there's no
+ // allocation which might trigger a minor GC to free up nursery space. This
+ // requires us to apply the same optimization here, otherwise we'd end up with
+ // always entering the OOL call, because the nursery is never evicted.
+ Label notOne;
+ masm.branchPtr(Assembler::NotEqual, temp2, ImmWord(1), &notOne);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&notOne);
+
+ static constexpr auto DigitMin = std::numeric_limits<
+ mozilla::SignedStdintTypeForSize<sizeof(BigInt::Digit)>::Type>::min();
+
+ // Handle an integer overflow from INT{32,64}_MIN / -1.
+ Label notOverflow;
+ masm.branchPtr(Assembler::NotEqual, temp1, ImmWord(DigitMin), &notOverflow);
+ masm.branchPtr(Assembler::Equal, temp2, ImmWord(-1), ool->entry());
+ masm.bind(&notOverflow);
+
+ emitBigIntDiv(ins, temp1, temp2, output, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntMod(LBigIntMod* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::mod>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // x % 0 throws an error.
+ if (ins->mir()->canBeDivideByZero()) {
+ masm.branchIfBigIntIsZero(rhs, ool->entry());
+ }
+
+ // 0n % x == 0n
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // Call into the VM when either operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntAbsolute(lhs, temp1, ool->entry());
+ masm.loadBigIntAbsolute(rhs, temp2, ool->entry());
+
+ // Similar to the case for BigInt division, we must apply the same allocation
+ // optimizations as performed in |BigInt::mod()|.
+ Label notBelow;
+ masm.branchPtr(Assembler::AboveOrEqual, temp1, temp2, &notBelow);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&notBelow);
+
+ // Convert both digits to signed pointer-sized values.
+ masm.bigIntDigitToSignedPtr(lhs, temp1, ool->entry());
+ masm.bigIntDigitToSignedPtr(rhs, temp2, ool->entry());
+
+ static constexpr auto DigitMin = std::numeric_limits<
+ mozilla::SignedStdintTypeForSize<sizeof(BigInt::Digit)>::Type>::min();
+
+ // Handle an integer overflow from INT{32,64}_MIN / -1.
+ Label notOverflow;
+ masm.branchPtr(Assembler::NotEqual, temp1, ImmWord(DigitMin), &notOverflow);
+ masm.branchPtr(Assembler::NotEqual, temp2, ImmWord(-1), &notOverflow);
+ masm.movePtr(ImmWord(0), temp1);
+ masm.bind(&notOverflow);
+
+ emitBigIntMod(ins, temp1, temp2, output, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntPow(LBigIntPow* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::pow>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // x ** -y throws an error.
+ if (ins->mir()->canBeNegativeExponent()) {
+ masm.branchIfBigIntIsNegative(rhs, ool->entry());
+ }
+
+ Register dest = temp1;
+ Register base = temp2;
+ Register exponent = output;
+
+ Label done;
+ masm.movePtr(ImmWord(1), dest); // p = 1
+
+ // 1n ** y == 1n
+ // -1n ** y == 1n when y is even
+ // -1n ** y == -1n when y is odd
+ Label lhsNotOne;
+ masm.branch32(Assembler::Above, Address(lhs, BigInt::offsetOfLength()),
+ Imm32(1), &lhsNotOne);
+ masm.loadFirstBigIntDigitOrZero(lhs, base);
+ masm.branchPtr(Assembler::NotEqual, base, Imm32(1), &lhsNotOne);
+ {
+ masm.loadFirstBigIntDigitOrZero(rhs, exponent);
+
+ Label lhsNonNegative;
+ masm.branchIfBigIntIsNonNegative(lhs, &lhsNonNegative);
+ masm.branchTestPtr(Assembler::Zero, exponent, Imm32(1), &done);
+ masm.bind(&lhsNonNegative);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ }
+ masm.bind(&lhsNotOne);
+
+ // x ** 0n == 1n
+ masm.branchIfBigIntIsZero(rhs, &done);
+
+ // 0n ** y == 0n with y != 0n
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ {
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ }
+ masm.bind(&lhsNonZero);
+
+ // Call into the VM when the exponent can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntAbsolute(rhs, exponent, ool->entry());
+
+ // x ** y with x > 1 and y >= DigitBits can't be pointer-sized.
+ masm.branchPtr(Assembler::AboveOrEqual, exponent, Imm32(BigInt::DigitBits),
+ ool->entry());
+
+ // x ** 1n == x
+ Label rhsNotOne;
+ masm.branch32(Assembler::NotEqual, exponent, Imm32(1), &rhsNotOne);
+ {
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ }
+ masm.bind(&rhsNotOne);
+
+ // Call into the VM when the base operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntNonZero(lhs, base, ool->entry());
+
+ // MacroAssembler::pow32() adjusted to work on pointer-sized registers.
+ {
+ // m = base
+ // n = exponent
+
+ Label start, loop;
+ masm.jump(&start);
+ masm.bind(&loop);
+
+ // m *= m
+ masm.branchMulPtr(Assembler::Overflow, base, base, ool->entry());
+
+ masm.bind(&start);
+
+ // if ((n & 1) != 0) p *= m
+ Label even;
+ masm.branchTest32(Assembler::Zero, exponent, Imm32(1), &even);
+ masm.branchMulPtr(Assembler::Overflow, base, dest, ool->entry());
+ masm.bind(&even);
+
+ // n >>= 1
+ // if (n == 0) return p
+ masm.branchRshift32(Assembler::NonZero, Imm32(1), exponent, &loop);
+ }
+
+ MOZ_ASSERT(temp1 == dest);
+
+ // Create and return the result.
+ masm.bind(&done);
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntBitAnd(LBigIntBitAnd* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::bitAnd>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // 0n & x == 0n
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // x & 0n == 0n
+ Label rhsNonZero;
+ masm.branchIfBigIntIsNonZero(rhs, &rhsNonZero);
+ masm.movePtr(rhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&rhsNonZero);
+
+ // Call into the VM when either operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntNonZero(lhs, temp1, ool->entry());
+ masm.loadBigIntNonZero(rhs, temp2, ool->entry());
+
+ masm.andPtr(temp2, temp1);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntBitOr(LBigIntBitOr* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::bitOr>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // 0n | x == x
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(rhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // x | 0n == x
+ Label rhsNonZero;
+ masm.branchIfBigIntIsNonZero(rhs, &rhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&rhsNonZero);
+
+ // Call into the VM when either operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntNonZero(lhs, temp1, ool->entry());
+ masm.loadBigIntNonZero(rhs, temp2, ool->entry());
+
+ masm.orPtr(temp2, temp1);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntBitXor(LBigIntBitXor* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::bitXor>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // 0n ^ x == x
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(rhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // x ^ 0n == x
+ Label rhsNonZero;
+ masm.branchIfBigIntIsNonZero(rhs, &rhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&rhsNonZero);
+
+ // Call into the VM when either operand can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigIntNonZero(lhs, temp1, ool->entry());
+ masm.loadBigIntNonZero(rhs, temp2, ool->entry());
+
+ masm.xorPtr(temp2, temp1);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntLsh(LBigIntLsh* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::lsh>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // 0n << x == 0n
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // x << 0n == x
+ Label rhsNonZero;
+ masm.branchIfBigIntIsNonZero(rhs, &rhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&rhsNonZero);
+
+ // Inline |BigInt::lsh| for the case when |lhs| contains a single digit.
+
+ Label rhsTooLarge;
+ masm.loadBigIntAbsolute(rhs, temp2, &rhsTooLarge);
+
+ // Call into the VM when the left-hand side operand can't be loaded into a
+ // pointer-sized register.
+ masm.loadBigIntAbsolute(lhs, temp1, ool->entry());
+
+ // Handle shifts exceeding |BigInt::DigitBits| first.
+ Label shift, create;
+ masm.branchPtr(Assembler::Below, temp2, Imm32(BigInt::DigitBits), &shift);
+ {
+ masm.bind(&rhsTooLarge);
+
+ // x << DigitBits with x != 0n always exceeds pointer-sized storage.
+ masm.branchIfBigIntIsNonNegative(rhs, ool->entry());
+
+ // x << -DigitBits == x >> DigitBits, which is either 0n or -1n.
+ masm.move32(Imm32(0), temp1);
+ masm.branchIfBigIntIsNonNegative(lhs, &create);
+ masm.move32(Imm32(1), temp1);
+ masm.jump(&create);
+ }
+ masm.bind(&shift);
+
+ Label nonNegative;
+ masm.branchIfBigIntIsNonNegative(rhs, &nonNegative);
+ {
+ masm.movePtr(temp1, temp3);
+
+ // |x << -y| is computed as |x >> y|.
+ masm.rshiftPtr(temp2, temp1);
+
+ // For negative numbers, round down if any bit was shifted out.
+ masm.branchIfBigIntIsNonNegative(lhs, &create);
+
+ // Compute |mask = (static_cast<Digit>(1) << shift) - 1|.
+ masm.movePtr(ImmWord(-1), output);
+ masm.lshiftPtr(temp2, output);
+ masm.notPtr(output);
+
+ // Add plus one when |(lhs.digit(0) & mask) != 0|.
+ masm.branchTestPtr(Assembler::Zero, output, temp3, &create);
+ masm.addPtr(ImmWord(1), temp1);
+ masm.jump(&create);
+ }
+ masm.bind(&nonNegative);
+ {
+ masm.movePtr(temp2, temp3);
+
+ // Compute |grow = lhs.digit(0) >> (DigitBits - shift)|.
+ masm.negPtr(temp2);
+ masm.addPtr(Imm32(BigInt::DigitBits), temp2);
+ masm.movePtr(temp1, output);
+ masm.rshiftPtr(temp2, output);
+
+ // Call into the VM when any bit will be shifted out.
+ masm.branchTestPtr(Assembler::NonZero, output, output, ool->entry());
+
+ masm.movePtr(temp3, temp2);
+ masm.lshiftPtr(temp2, temp1);
+ }
+ masm.bind(&create);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigIntAbsolute(output, temp1);
+
+ // Set the sign bit when the left-hand side is negative.
+ masm.branchIfBigIntIsNonNegative(lhs, ool->rejoin());
+ masm.or32(Imm32(BigInt::signBitMask()),
+ Address(output, BigInt::offsetOfFlags()));
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntRsh(LBigIntRsh* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::rsh>(ins, ArgList(lhs, rhs),
+ StoreRegisterTo(output));
+
+ // 0n >> x == 0n
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(lhs, &lhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // x >> 0n == x
+ Label rhsNonZero;
+ masm.branchIfBigIntIsNonZero(rhs, &rhsNonZero);
+ masm.movePtr(lhs, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&rhsNonZero);
+
+ // Inline |BigInt::rsh| for the case when |lhs| contains a single digit.
+
+ Label rhsTooLarge;
+ masm.loadBigIntAbsolute(rhs, temp2, &rhsTooLarge);
+
+ // Call into the VM when the left-hand side operand can't be loaded into a
+ // pointer-sized register.
+ masm.loadBigIntAbsolute(lhs, temp1, ool->entry());
+
+ // Handle shifts exceeding |BigInt::DigitBits| first.
+ Label shift, create;
+ masm.branchPtr(Assembler::Below, temp2, Imm32(BigInt::DigitBits), &shift);
+ {
+ masm.bind(&rhsTooLarge);
+
+ // x >> -DigitBits == x << DigitBits, which exceeds pointer-sized storage.
+ masm.branchIfBigIntIsNegative(rhs, ool->entry());
+
+ // x >> DigitBits is either 0n or -1n.
+ masm.move32(Imm32(0), temp1);
+ masm.branchIfBigIntIsNonNegative(lhs, &create);
+ masm.move32(Imm32(1), temp1);
+ masm.jump(&create);
+ }
+ masm.bind(&shift);
+
+ Label nonNegative;
+ masm.branchIfBigIntIsNonNegative(rhs, &nonNegative);
+ {
+ masm.movePtr(temp2, temp3);
+
+ // Compute |grow = lhs.digit(0) >> (DigitBits - shift)|.
+ masm.negPtr(temp2);
+ masm.addPtr(Imm32(BigInt::DigitBits), temp2);
+ masm.movePtr(temp1, output);
+ masm.rshiftPtr(temp2, output);
+
+ // Call into the VM when any bit will be shifted out.
+ masm.branchTestPtr(Assembler::NonZero, output, output, ool->entry());
+
+ // |x >> -y| is computed as |x << y|.
+ masm.movePtr(temp3, temp2);
+ masm.lshiftPtr(temp2, temp1);
+ masm.jump(&create);
+ }
+ masm.bind(&nonNegative);
+ {
+ masm.movePtr(temp1, temp3);
+
+ masm.rshiftPtr(temp2, temp1);
+
+ // For negative numbers, round down if any bit was shifted out.
+ masm.branchIfBigIntIsNonNegative(lhs, &create);
+
+ // Compute |mask = (static_cast<Digit>(1) << shift) - 1|.
+ masm.movePtr(ImmWord(-1), output);
+ masm.lshiftPtr(temp2, output);
+ masm.notPtr(output);
+
+ // Add plus one when |(lhs.digit(0) & mask) != 0|.
+ masm.branchTestPtr(Assembler::Zero, output, temp3, &create);
+ masm.addPtr(ImmWord(1), temp1);
+ }
+ masm.bind(&create);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigIntAbsolute(output, temp1);
+
+ // Set the sign bit when the left-hand side is negative.
+ masm.branchIfBigIntIsNonNegative(lhs, ool->rejoin());
+ masm.or32(Imm32(BigInt::signBitMask()),
+ Address(output, BigInt::offsetOfFlags()));
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntIncrement(LBigIntIncrement* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt);
+ auto* ool =
+ oolCallVM<Fn, BigInt::inc>(ins, ArgList(input), StoreRegisterTo(output));
+
+ // Call into the VM when the input can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigInt(input, temp1, ool->entry());
+ masm.movePtr(ImmWord(1), temp2);
+
+ masm.branchAddPtr(Assembler::Overflow, temp2, temp1, ool->entry());
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntDecrement(LBigIntDecrement* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt);
+ auto* ool =
+ oolCallVM<Fn, BigInt::dec>(ins, ArgList(input), StoreRegisterTo(output));
+
+ // Call into the VM when the input can't be loaded into a pointer-sized
+ // register.
+ masm.loadBigInt(input, temp1, ool->entry());
+ masm.movePtr(ImmWord(1), temp2);
+
+ masm.branchSubPtr(Assembler::Overflow, temp2, temp1, ool->entry());
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigInt(output, temp1);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntNegate(LBigIntNegate* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp = ToRegister(ins->temp());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt);
+ auto* ool =
+ oolCallVM<Fn, BigInt::neg>(ins, ArgList(input), StoreRegisterTo(output));
+
+ // -0n == 0n
+ Label lhsNonZero;
+ masm.branchIfBigIntIsNonZero(input, &lhsNonZero);
+ masm.movePtr(input, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&lhsNonZero);
+
+ // Call into the VM when the input uses heap digits.
+ masm.copyBigIntWithInlineDigits(input, output, temp, initialBigIntHeap(),
+ ool->entry());
+
+ // Flip the sign bit.
+ masm.xor32(Imm32(BigInt::signBitMask()),
+ Address(output, BigInt::offsetOfFlags()));
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitBigIntBitNot(LBigIntBitNot* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt);
+ auto* ool = oolCallVM<Fn, BigInt::bitNot>(ins, ArgList(input),
+ StoreRegisterTo(output));
+
+ masm.loadBigIntAbsolute(input, temp1, ool->entry());
+
+ // This follows the C++ implementation because it let's us support the full
+ // range [-2^64, 2^64 - 1] on 64-bit resp. [-2^32, 2^32 - 1] on 32-bit.
+ Label nonNegative, done;
+ masm.branchIfBigIntIsNonNegative(input, &nonNegative);
+ {
+ // ~(-x) == ~(~(x-1)) == x-1
+ masm.subPtr(Imm32(1), temp1);
+ masm.jump(&done);
+ }
+ masm.bind(&nonNegative);
+ {
+ // ~x == -x-1 == -(x+1)
+ masm.movePtr(ImmWord(1), temp2);
+ masm.branchAddPtr(Assembler::CarrySet, temp2, temp1, ool->entry());
+ }
+ masm.bind(&done);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, temp2, initialBigIntHeap(), ool->entry());
+ masm.initializeBigIntAbsolute(output, temp1);
+
+ // Set the sign bit when the input is positive.
+ masm.branchIfBigIntIsNegative(input, ool->rejoin());
+ masm.or32(Imm32(BigInt::signBitMask()),
+ Address(output, BigInt::offsetOfFlags()));
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitInt32ToStringWithBase(LInt32ToStringWithBase* lir) {
+ Register input = ToRegister(lir->input());
+ RegisterOrInt32 base = ToRegisterOrInt32(lir->base());
+ Register output = ToRegister(lir->output());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ using Fn = JSString* (*)(JSContext*, int32_t, int32_t);
+ if (base.is<Register>()) {
+ auto* ool = oolCallVM<Fn, js::Int32ToStringWithBase>(
+ lir, ArgList(input, base.as<Register>()), StoreRegisterTo(output));
+
+ LiveRegisterSet liveRegs = liveVolatileRegs(lir);
+ masm.loadInt32ToStringWithBase(input, base.as<Register>(), output, temp0,
+ temp1, gen->runtime->staticStrings(),
+ liveRegs, ool->entry());
+ masm.bind(ool->rejoin());
+ } else {
+ auto* ool = oolCallVM<Fn, js::Int32ToStringWithBase>(
+ lir, ArgList(input, Imm32(base.as<int32_t>())),
+ StoreRegisterTo(output));
+
+ masm.loadInt32ToStringWithBase(input, base.as<int32_t>(), output, temp0,
+ temp1, gen->runtime->staticStrings(),
+ ool->entry());
+ masm.bind(ool->rejoin());
+ }
+}
+
+void CodeGenerator::visitNumberParseInt(LNumberParseInt* lir) {
+ Register string = ToRegister(lir->string());
+ Register radix = ToRegister(lir->radix());
+ ValueOperand output = ToOutValue(lir);
+ Register temp = ToRegister(lir->temp0());
+
+#ifdef DEBUG
+ Label ok;
+ masm.branch32(Assembler::Equal, radix, Imm32(0), &ok);
+ masm.branch32(Assembler::Equal, radix, Imm32(10), &ok);
+ masm.assumeUnreachable("radix must be 0 or 10 for indexed value fast path");
+ masm.bind(&ok);
+#endif
+
+ // Use indexed value as fast path if possible.
+ Label vmCall, done;
+ masm.loadStringIndexValue(string, temp, &vmCall);
+ masm.tagValue(JSVAL_TYPE_INT32, temp, output);
+ masm.jump(&done);
+ {
+ masm.bind(&vmCall);
+
+ pushArg(radix);
+ pushArg(string);
+
+ using Fn = bool (*)(JSContext*, HandleString, int32_t, MutableHandleValue);
+ callVM<Fn, js::NumberParseInt>(lir);
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitDoubleParseInt(LDoubleParseInt* lir) {
+ FloatRegister number = ToFloatRegister(lir->number());
+ Register output = ToRegister(lir->output());
+ FloatRegister temp = ToFloatRegister(lir->temp0());
+
+ Label bail;
+ masm.branchDouble(Assembler::DoubleUnordered, number, number, &bail);
+ masm.branchTruncateDoubleToInt32(number, output, &bail);
+
+ Label ok;
+ masm.branch32(Assembler::NotEqual, output, Imm32(0), &ok);
+ {
+ // Accept both +0 and -0 and return 0.
+ masm.loadConstantDouble(0.0, temp);
+ masm.branchDouble(Assembler::DoubleEqual, number, temp, &ok);
+
+ // Fail if a non-zero input is in the exclusive range (-1, 1.0e-6).
+ masm.loadConstantDouble(DOUBLE_DECIMAL_IN_SHORTEST_LOW, temp);
+ masm.branchDouble(Assembler::DoubleLessThan, number, temp, &bail);
+ }
+ masm.bind(&ok);
+
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitFloor(LFloor* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+ masm.floorDoubleToInt32(input, output, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitFloorF(LFloorF* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+ masm.floorFloat32ToInt32(input, output, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitCeil(LCeil* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+ masm.ceilDoubleToInt32(input, output, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitCeilF(LCeilF* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+ masm.ceilFloat32ToInt32(input, output, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitRound(LRound* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister temp = ToFloatRegister(lir->temp0());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+ masm.roundDoubleToInt32(input, output, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitRoundF(LRoundF* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister temp = ToFloatRegister(lir->temp0());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+ masm.roundFloat32ToInt32(input, output, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitTrunc(LTrunc* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+ masm.truncDoubleToInt32(input, output, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitTruncF(LTruncF* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ Label bail;
+ masm.truncFloat32ToInt32(input, output, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitCompareS(LCompareS* lir) {
+ JSOp op = lir->mir()->jsop();
+ Register left = ToRegister(lir->left());
+ Register right = ToRegister(lir->right());
+ Register output = ToRegister(lir->output());
+
+ OutOfLineCode* ool = nullptr;
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ if (op == JSOp::Eq || op == JSOp::StrictEq) {
+ ool = oolCallVM<Fn, jit::StringsEqual<EqualityKind::Equal>>(
+ lir, ArgList(left, right), StoreRegisterTo(output));
+ } else if (op == JSOp::Ne || op == JSOp::StrictNe) {
+ ool = oolCallVM<Fn, jit::StringsEqual<EqualityKind::NotEqual>>(
+ lir, ArgList(left, right), StoreRegisterTo(output));
+ } else if (op == JSOp::Lt) {
+ ool = oolCallVM<Fn, jit::StringsCompare<ComparisonKind::LessThan>>(
+ lir, ArgList(left, right), StoreRegisterTo(output));
+ } else if (op == JSOp::Le) {
+ // Push the operands in reverse order for JSOp::Le:
+ // - |left <= right| is implemented as |right >= left|.
+ ool =
+ oolCallVM<Fn, jit::StringsCompare<ComparisonKind::GreaterThanOrEqual>>(
+ lir, ArgList(right, left), StoreRegisterTo(output));
+ } else if (op == JSOp::Gt) {
+ // Push the operands in reverse order for JSOp::Gt:
+ // - |left > right| is implemented as |right < left|.
+ ool = oolCallVM<Fn, jit::StringsCompare<ComparisonKind::LessThan>>(
+ lir, ArgList(right, left), StoreRegisterTo(output));
+ } else {
+ MOZ_ASSERT(op == JSOp::Ge);
+ ool =
+ oolCallVM<Fn, jit::StringsCompare<ComparisonKind::GreaterThanOrEqual>>(
+ lir, ArgList(left, right), StoreRegisterTo(output));
+ }
+
+ masm.compareStrings(op, left, right, output, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+template <typename T, typename CharT>
+static inline T CopyCharacters(const CharT* chars) {
+ T value = 0;
+ std::memcpy(&value, chars, sizeof(T));
+ return value;
+}
+
+template <typename T>
+static inline T CopyCharacters(const JSLinearString* str, size_t index) {
+ JS::AutoCheckCannotGC nogc;
+
+ if (str->hasLatin1Chars()) {
+ MOZ_ASSERT(index + sizeof(T) / sizeof(JS::Latin1Char) <= str->length());
+ return CopyCharacters<T>(str->latin1Chars(nogc) + index);
+ }
+
+ MOZ_ASSERT(sizeof(T) >= sizeof(char16_t));
+ MOZ_ASSERT(index + sizeof(T) / sizeof(char16_t) <= str->length());
+ return CopyCharacters<T>(str->twoByteChars(nogc) + index);
+}
+
+enum class CompareDirection { Forward, Backward };
+
+// NOTE: Clobbers the input when CompareDirection is backward.
+static void CompareCharacters(MacroAssembler& masm, Register input,
+ const JSLinearString* str, Register output,
+ JSOp op, CompareDirection direction, Label* done,
+ Label* oolEntry) {
+ MOZ_ASSERT(input != output);
+
+ size_t length = str->length();
+ MOZ_ASSERT(length > 0);
+
+ CharEncoding encoding =
+ str->hasLatin1Chars() ? CharEncoding::Latin1 : CharEncoding::TwoByte;
+ size_t encodingSize = encoding == CharEncoding::Latin1
+ ? sizeof(JS::Latin1Char)
+ : sizeof(char16_t);
+ size_t byteLength = length * encodingSize;
+
+ // Take the OOL path when the string is a rope or has a different character
+ // representation.
+ masm.branchIfRope(input, oolEntry);
+ if (encoding == CharEncoding::Latin1) {
+ masm.branchTwoByteString(input, oolEntry);
+ } else {
+ JS::AutoCheckCannotGC nogc;
+ if (mozilla::IsUtf16Latin1(str->twoByteRange(nogc))) {
+ masm.branchLatin1String(input, oolEntry);
+ } else {
+ // This case was already handled in the caller.
+#ifdef DEBUG
+ Label ok;
+ masm.branchTwoByteString(input, &ok);
+ masm.assumeUnreachable("Unexpected Latin-1 string");
+ masm.bind(&ok);
+#endif
+ }
+ }
+
+#ifdef DEBUG
+ {
+ Label ok;
+ masm.branch32(Assembler::AboveOrEqual,
+ Address(input, JSString::offsetOfLength()), Imm32(length),
+ &ok);
+ masm.assumeUnreachable("Input mustn't be smaller than search string");
+ masm.bind(&ok);
+ }
+#endif
+
+ // Load the input string's characters.
+ Register stringChars = output;
+ masm.loadStringChars(input, stringChars, encoding);
+
+ if (direction == CompareDirection::Backward) {
+ masm.loadStringLength(input, input);
+ masm.sub32(Imm32(length), input);
+
+ masm.addToCharPtr(stringChars, input, encoding);
+ }
+
+ // Prefer a single compare-and-set instruction if possible.
+ if (byteLength == 1 || byteLength == 2 || byteLength == 4 ||
+ byteLength == 8) {
+ auto cond = JSOpToCondition(op, /* isSigned = */ false);
+
+ Address addr(stringChars, 0);
+ switch (byteLength) {
+ case 8: {
+ auto x = CopyCharacters<uint64_t>(str, 0);
+ masm.cmp64Set(cond, addr, Imm64(x), output);
+ break;
+ }
+ case 4: {
+ auto x = CopyCharacters<uint32_t>(str, 0);
+ masm.cmp32Set(cond, addr, Imm32(x), output);
+ break;
+ }
+ case 2: {
+ auto x = CopyCharacters<uint16_t>(str, 0);
+ masm.cmp16Set(cond, addr, Imm32(x), output);
+ break;
+ }
+ case 1: {
+ auto x = CopyCharacters<uint8_t>(str, 0);
+ masm.cmp8Set(cond, addr, Imm32(x), output);
+ break;
+ }
+ }
+ } else {
+ Label setNotEqualResult;
+
+ size_t pos = 0;
+ for (size_t stride : {8, 4, 2, 1}) {
+ while (byteLength >= stride) {
+ Address addr(stringChars, pos * encodingSize);
+ switch (stride) {
+ case 8: {
+ auto x = CopyCharacters<uint64_t>(str, pos);
+ masm.branch64(Assembler::NotEqual, addr, Imm64(x),
+ &setNotEqualResult);
+ break;
+ }
+ case 4: {
+ auto x = CopyCharacters<uint32_t>(str, pos);
+ masm.branch32(Assembler::NotEqual, addr, Imm32(x),
+ &setNotEqualResult);
+ break;
+ }
+ case 2: {
+ auto x = CopyCharacters<uint16_t>(str, pos);
+ masm.branch16(Assembler::NotEqual, addr, Imm32(x),
+ &setNotEqualResult);
+ break;
+ }
+ case 1: {
+ auto x = CopyCharacters<uint8_t>(str, pos);
+ masm.branch8(Assembler::NotEqual, addr, Imm32(x),
+ &setNotEqualResult);
+ break;
+ }
+ }
+
+ byteLength -= stride;
+ pos += stride / encodingSize;
+ }
+
+ // Prefer a single comparison for trailing bytes instead of doing
+ // multiple consecutive comparisons.
+ //
+ // For example when comparing against the string "example", emit two
+ // four-byte comparisons against "exam" and "mple" instead of doing
+ // three comparisons against "exam", "pl", and finally "e".
+ if (pos > 0 && byteLength > stride / 2) {
+ MOZ_ASSERT(stride == 8 || stride == 4);
+
+ size_t prev = pos - (stride - byteLength) / encodingSize;
+ Address addr(stringChars, prev * encodingSize);
+ switch (stride) {
+ case 8: {
+ auto x = CopyCharacters<uint64_t>(str, prev);
+ masm.branch64(Assembler::NotEqual, addr, Imm64(x),
+ &setNotEqualResult);
+ break;
+ }
+ case 4: {
+ auto x = CopyCharacters<uint32_t>(str, prev);
+ masm.branch32(Assembler::NotEqual, addr, Imm32(x),
+ &setNotEqualResult);
+ break;
+ }
+ }
+
+ // Break from the loop, because we've finished the complete string.
+ break;
+ }
+ }
+
+ // Falls through if both strings are equal.
+
+ masm.move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq), output);
+ masm.jump(done);
+
+ masm.bind(&setNotEqualResult);
+ masm.move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), output);
+ }
+}
+
+void CodeGenerator::visitCompareSInline(LCompareSInline* lir) {
+ JSOp op = lir->mir()->jsop();
+ MOZ_ASSERT(IsEqualityOp(op));
+
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ const JSLinearString* str = lir->constant();
+ MOZ_ASSERT(str->length() > 0);
+
+ OutOfLineCode* ool = nullptr;
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ if (op == JSOp::Eq || op == JSOp::StrictEq) {
+ ool = oolCallVM<Fn, jit::StringsEqual<EqualityKind::Equal>>(
+ lir, ArgList(ImmGCPtr(str), input), StoreRegisterTo(output));
+ } else {
+ MOZ_ASSERT(op == JSOp::Ne || op == JSOp::StrictNe);
+ ool = oolCallVM<Fn, jit::StringsEqual<EqualityKind::NotEqual>>(
+ lir, ArgList(ImmGCPtr(str), input), StoreRegisterTo(output));
+ }
+
+ Label compareChars;
+ {
+ Label notPointerEqual;
+
+ // If operands point to the same instance, the strings are trivially equal.
+ masm.branchPtr(Assembler::NotEqual, input, ImmGCPtr(str), &notPointerEqual);
+ masm.move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq), output);
+ masm.jump(ool->rejoin());
+
+ masm.bind(&notPointerEqual);
+
+ Label setNotEqualResult;
+ if (str->isAtom()) {
+ // Atoms cannot be equal to each other if they point to different strings.
+ Imm32 atomBit(JSString::ATOM_BIT);
+ masm.branchTest32(Assembler::NonZero,
+ Address(input, JSString::offsetOfFlags()), atomBit,
+ &setNotEqualResult);
+ }
+
+ if (str->hasTwoByteChars()) {
+ // Pure two-byte strings can't be equal to Latin-1 strings.
+ JS::AutoCheckCannotGC nogc;
+ if (!mozilla::IsUtf16Latin1(str->twoByteRange(nogc))) {
+ masm.branchLatin1String(input, &setNotEqualResult);
+ }
+ }
+
+ // Strings of different length can never be equal.
+ masm.branch32(Assembler::Equal, Address(input, JSString::offsetOfLength()),
+ Imm32(str->length()), &compareChars);
+
+ masm.bind(&setNotEqualResult);
+ masm.move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), output);
+ masm.jump(ool->rejoin());
+ }
+
+ masm.bind(&compareChars);
+
+ CompareCharacters(masm, input, str, output, op, CompareDirection::Forward,
+ ool->rejoin(), ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCompareBigInt(LCompareBigInt* lir) {
+ JSOp op = lir->mir()->jsop();
+ Register left = ToRegister(lir->left());
+ Register right = ToRegister(lir->right());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+ Register output = ToRegister(lir->output());
+
+ Label notSame;
+ Label compareSign;
+ Label compareLength;
+ Label compareDigit;
+
+ Label* notSameSign;
+ Label* notSameLength;
+ Label* notSameDigit;
+ if (IsEqualityOp(op)) {
+ notSameSign = &notSame;
+ notSameLength = &notSame;
+ notSameDigit = &notSame;
+ } else {
+ notSameSign = &compareSign;
+ notSameLength = &compareLength;
+ notSameDigit = &compareDigit;
+ }
+
+ masm.equalBigInts(left, right, temp0, temp1, temp2, output, notSameSign,
+ notSameLength, notSameDigit);
+
+ Label done;
+ masm.move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq || op == JSOp::Le ||
+ op == JSOp::Ge),
+ output);
+ masm.jump(&done);
+
+ if (IsEqualityOp(op)) {
+ masm.bind(&notSame);
+ masm.move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), output);
+ } else {
+ Label invertWhenNegative;
+
+ // There are two cases when sign(left) != sign(right):
+ // 1. sign(left) = positive and sign(right) = negative,
+ // 2. or the dual case with reversed signs.
+ //
+ // For case 1, |left| <cmp> |right| is true for cmp=Gt or cmp=Ge and false
+ // for cmp=Lt or cmp=Le. Initialize the result for case 1 and handle case 2
+ // with |invertWhenNegative|.
+ masm.bind(&compareSign);
+ masm.move32(Imm32(op == JSOp::Gt || op == JSOp::Ge), output);
+ masm.jump(&invertWhenNegative);
+
+ // For sign(left) = sign(right) and len(digits(left)) != len(digits(right)),
+ // we have to consider the two cases:
+ // 1. len(digits(left)) < len(digits(right))
+ // 2. len(digits(left)) > len(digits(right))
+ //
+ // For |left| <cmp> |right| with cmp=Lt:
+ // Assume both BigInts are positive, then |left < right| is true for case 1
+ // and false for case 2. When both are negative, the result is reversed.
+ //
+ // The other comparison operators can be handled similarly.
+ //
+ // |temp0| holds the digits length of the right-hand side operand.
+ masm.bind(&compareLength);
+ masm.cmp32Set(JSOpToCondition(op, /* isSigned = */ false),
+ Address(left, BigInt::offsetOfLength()), temp0, output);
+ masm.jump(&invertWhenNegative);
+
+ // Similar to the case above, compare the current digit to determine the
+ // overall comparison result.
+ //
+ // |temp1| points to the current digit of the left-hand side operand.
+ // |output| holds the current digit of the right-hand side operand.
+ masm.bind(&compareDigit);
+ masm.cmpPtrSet(JSOpToCondition(op, /* isSigned = */ false),
+ Address(temp1, 0), output, output);
+
+ Label nonNegative;
+ masm.bind(&invertWhenNegative);
+ masm.branchIfBigIntIsNonNegative(left, &nonNegative);
+ masm.xor32(Imm32(1), output);
+ masm.bind(&nonNegative);
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitCompareBigIntInt32(LCompareBigIntInt32* lir) {
+ JSOp op = lir->mir()->jsop();
+ Register left = ToRegister(lir->left());
+ Register right = ToRegister(lir->right());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register output = ToRegister(lir->output());
+
+ Label ifTrue, ifFalse;
+ masm.compareBigIntAndInt32(op, left, right, temp0, temp1, &ifTrue, &ifFalse);
+
+ Label done;
+ masm.bind(&ifFalse);
+ masm.move32(Imm32(0), output);
+ masm.jump(&done);
+ masm.bind(&ifTrue);
+ masm.move32(Imm32(1), output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitCompareBigIntDouble(LCompareBigIntDouble* lir) {
+ JSOp op = lir->mir()->jsop();
+ Register left = ToRegister(lir->left());
+ FloatRegister right = ToFloatRegister(lir->right());
+ Register output = ToRegister(lir->output());
+
+ masm.setupAlignedABICall();
+
+ // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
+ // - |left <= right| is implemented as |right >= left|.
+ // - |left > right| is implemented as |right < left|.
+ if (op == JSOp::Le || op == JSOp::Gt) {
+ masm.passABIArg(right, MoveOp::DOUBLE);
+ masm.passABIArg(left);
+ } else {
+ masm.passABIArg(left);
+ masm.passABIArg(right, MoveOp::DOUBLE);
+ }
+
+ using FnBigIntNumber = bool (*)(BigInt*, double);
+ using FnNumberBigInt = bool (*)(double, BigInt*);
+ switch (op) {
+ case JSOp::Eq: {
+ masm.callWithABI<FnBigIntNumber,
+ jit::BigIntNumberEqual<EqualityKind::Equal>>();
+ break;
+ }
+ case JSOp::Ne: {
+ masm.callWithABI<FnBigIntNumber,
+ jit::BigIntNumberEqual<EqualityKind::NotEqual>>();
+ break;
+ }
+ case JSOp::Lt: {
+ masm.callWithABI<FnBigIntNumber,
+ jit::BigIntNumberCompare<ComparisonKind::LessThan>>();
+ break;
+ }
+ case JSOp::Gt: {
+ masm.callWithABI<FnNumberBigInt,
+ jit::NumberBigIntCompare<ComparisonKind::LessThan>>();
+ break;
+ }
+ case JSOp::Le: {
+ masm.callWithABI<
+ FnNumberBigInt,
+ jit::NumberBigIntCompare<ComparisonKind::GreaterThanOrEqual>>();
+ break;
+ }
+ case JSOp::Ge: {
+ masm.callWithABI<
+ FnBigIntNumber,
+ jit::BigIntNumberCompare<ComparisonKind::GreaterThanOrEqual>>();
+ break;
+ }
+ default:
+ MOZ_CRASH("unhandled op");
+ }
+
+ masm.storeCallBoolResult(output);
+}
+
+void CodeGenerator::visitCompareBigIntString(LCompareBigIntString* lir) {
+ JSOp op = lir->mir()->jsop();
+ Register left = ToRegister(lir->left());
+ Register right = ToRegister(lir->right());
+
+ // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
+ // - |left <= right| is implemented as |right >= left|.
+ // - |left > right| is implemented as |right < left|.
+ if (op == JSOp::Le || op == JSOp::Gt) {
+ pushArg(left);
+ pushArg(right);
+ } else {
+ pushArg(right);
+ pushArg(left);
+ }
+
+ using FnBigIntString =
+ bool (*)(JSContext*, HandleBigInt, HandleString, bool*);
+ using FnStringBigInt =
+ bool (*)(JSContext*, HandleString, HandleBigInt, bool*);
+
+ switch (op) {
+ case JSOp::Eq: {
+ constexpr auto Equal = EqualityKind::Equal;
+ callVM<FnBigIntString, BigIntStringEqual<Equal>>(lir);
+ break;
+ }
+ case JSOp::Ne: {
+ constexpr auto NotEqual = EqualityKind::NotEqual;
+ callVM<FnBigIntString, BigIntStringEqual<NotEqual>>(lir);
+ break;
+ }
+ case JSOp::Lt: {
+ constexpr auto LessThan = ComparisonKind::LessThan;
+ callVM<FnBigIntString, BigIntStringCompare<LessThan>>(lir);
+ break;
+ }
+ case JSOp::Gt: {
+ constexpr auto LessThan = ComparisonKind::LessThan;
+ callVM<FnStringBigInt, StringBigIntCompare<LessThan>>(lir);
+ break;
+ }
+ case JSOp::Le: {
+ constexpr auto GreaterThanOrEqual = ComparisonKind::GreaterThanOrEqual;
+ callVM<FnStringBigInt, StringBigIntCompare<GreaterThanOrEqual>>(lir);
+ break;
+ }
+ case JSOp::Ge: {
+ constexpr auto GreaterThanOrEqual = ComparisonKind::GreaterThanOrEqual;
+ callVM<FnBigIntString, BigIntStringCompare<GreaterThanOrEqual>>(lir);
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected compare op");
+ }
+}
+
+void CodeGenerator::visitIsNullOrLikeUndefinedV(LIsNullOrLikeUndefinedV* lir) {
+ MOZ_ASSERT(lir->mir()->compareType() == MCompare::Compare_Undefined ||
+ lir->mir()->compareType() == MCompare::Compare_Null);
+
+ JSOp op = lir->mir()->jsop();
+ MOZ_ASSERT(IsLooseEqualityOp(op));
+
+ const ValueOperand value = ToValue(lir, LIsNullOrLikeUndefinedV::ValueIndex);
+ Register output = ToRegister(lir->output());
+
+ auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
+ addOutOfLineCode(ool, lir->mir());
+
+ Label* nullOrLikeUndefined = ool->label1();
+ Label* notNullOrLikeUndefined = ool->label2();
+
+ {
+ ScratchTagScope tag(masm, value);
+ masm.splitTagForTest(value, tag);
+
+ masm.branchTestNull(Assembler::Equal, tag, nullOrLikeUndefined);
+ masm.branchTestUndefined(Assembler::Equal, tag, nullOrLikeUndefined);
+
+ // Check whether it's a truthy object or a falsy object that emulates
+ // undefined.
+ masm.branchTestObject(Assembler::NotEqual, tag, notNullOrLikeUndefined);
+ }
+
+ Register objreg =
+ masm.extractObject(value, ToTempUnboxRegister(lir->temp0()));
+ branchTestObjectEmulatesUndefined(objreg, nullOrLikeUndefined,
+ notNullOrLikeUndefined, output, ool);
+ // fall through
+
+ Label done;
+
+ // It's not null or undefined, and if it's an object it doesn't
+ // emulate undefined, so it's not like undefined.
+ masm.move32(Imm32(op == JSOp::Ne), output);
+ masm.jump(&done);
+
+ masm.bind(nullOrLikeUndefined);
+ masm.move32(Imm32(op == JSOp::Eq), output);
+
+ // Both branches meet here.
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitIsNullOrLikeUndefinedAndBranchV(
+ LIsNullOrLikeUndefinedAndBranchV* lir) {
+ MOZ_ASSERT(lir->cmpMir()->compareType() == MCompare::Compare_Undefined ||
+ lir->cmpMir()->compareType() == MCompare::Compare_Null);
+
+ JSOp op = lir->cmpMir()->jsop();
+ MOZ_ASSERT(IsLooseEqualityOp(op));
+
+ const ValueOperand value =
+ ToValue(lir, LIsNullOrLikeUndefinedAndBranchV::Value);
+
+ MBasicBlock* ifTrue = lir->ifTrue();
+ MBasicBlock* ifFalse = lir->ifFalse();
+
+ if (op == JSOp::Ne) {
+ // Swap branches.
+ std::swap(ifTrue, ifFalse);
+ }
+
+ auto* ool = new (alloc()) OutOfLineTestObject();
+ addOutOfLineCode(ool, lir->cmpMir());
+
+ Label* ifTrueLabel = getJumpLabelForBranch(ifTrue);
+ Label* ifFalseLabel = getJumpLabelForBranch(ifFalse);
+
+ {
+ ScratchTagScope tag(masm, value);
+ masm.splitTagForTest(value, tag);
+
+ masm.branchTestNull(Assembler::Equal, tag, ifTrueLabel);
+ masm.branchTestUndefined(Assembler::Equal, tag, ifTrueLabel);
+
+ masm.branchTestObject(Assembler::NotEqual, tag, ifFalseLabel);
+ }
+
+ // Objects that emulate undefined are loosely equal to null/undefined.
+ Register objreg =
+ masm.extractObject(value, ToTempUnboxRegister(lir->tempToUnbox()));
+ Register scratch = ToRegister(lir->temp());
+ testObjectEmulatesUndefined(objreg, ifTrueLabel, ifFalseLabel, scratch, ool);
+}
+
+void CodeGenerator::visitIsNullOrLikeUndefinedT(LIsNullOrLikeUndefinedT* lir) {
+ MOZ_ASSERT(lir->mir()->compareType() == MCompare::Compare_Undefined ||
+ lir->mir()->compareType() == MCompare::Compare_Null);
+ MOZ_ASSERT(lir->mir()->lhs()->type() == MIRType::Object);
+
+ JSOp op = lir->mir()->jsop();
+ MOZ_ASSERT(IsLooseEqualityOp(op), "Strict equality should have been folded");
+
+ Register objreg = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
+ addOutOfLineCode(ool, lir->mir());
+
+ Label* emulatesUndefined = ool->label1();
+ Label* doesntEmulateUndefined = ool->label2();
+
+ branchTestObjectEmulatesUndefined(objreg, emulatesUndefined,
+ doesntEmulateUndefined, output, ool);
+
+ Label done;
+
+ masm.move32(Imm32(op == JSOp::Ne), output);
+ masm.jump(&done);
+
+ masm.bind(emulatesUndefined);
+ masm.move32(Imm32(op == JSOp::Eq), output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitIsNullOrLikeUndefinedAndBranchT(
+ LIsNullOrLikeUndefinedAndBranchT* lir) {
+ MOZ_ASSERT(lir->cmpMir()->compareType() == MCompare::Compare_Undefined ||
+ lir->cmpMir()->compareType() == MCompare::Compare_Null);
+ MOZ_ASSERT(lir->cmpMir()->lhs()->type() == MIRType::Object);
+
+ JSOp op = lir->cmpMir()->jsop();
+ MOZ_ASSERT(IsLooseEqualityOp(op), "Strict equality should have been folded");
+
+ MBasicBlock* ifTrue = lir->ifTrue();
+ MBasicBlock* ifFalse = lir->ifFalse();
+
+ if (op == JSOp::Ne) {
+ // Swap branches.
+ std::swap(ifTrue, ifFalse);
+ }
+
+ Register input = ToRegister(lir->getOperand(0));
+
+ auto* ool = new (alloc()) OutOfLineTestObject();
+ addOutOfLineCode(ool, lir->cmpMir());
+
+ Label* ifTrueLabel = getJumpLabelForBranch(ifTrue);
+ Label* ifFalseLabel = getJumpLabelForBranch(ifFalse);
+
+ // Objects that emulate undefined are loosely equal to null/undefined.
+ Register scratch = ToRegister(lir->temp());
+ testObjectEmulatesUndefined(input, ifTrueLabel, ifFalseLabel, scratch, ool);
+}
+
+void CodeGenerator::visitIsNull(LIsNull* lir) {
+ MCompare::CompareType compareType = lir->mir()->compareType();
+ MOZ_ASSERT(compareType == MCompare::Compare_Null);
+
+ JSOp op = lir->mir()->jsop();
+ MOZ_ASSERT(IsStrictEqualityOp(op));
+
+ const ValueOperand value = ToValue(lir, LIsNull::ValueIndex);
+ Register output = ToRegister(lir->output());
+
+ Assembler::Condition cond = JSOpToCondition(compareType, op);
+ masm.testNullSet(cond, value, output);
+}
+
+void CodeGenerator::visitIsUndefined(LIsUndefined* lir) {
+ MCompare::CompareType compareType = lir->mir()->compareType();
+ MOZ_ASSERT(compareType == MCompare::Compare_Undefined);
+
+ JSOp op = lir->mir()->jsop();
+ MOZ_ASSERT(IsStrictEqualityOp(op));
+
+ const ValueOperand value = ToValue(lir, LIsUndefined::ValueIndex);
+ Register output = ToRegister(lir->output());
+
+ Assembler::Condition cond = JSOpToCondition(compareType, op);
+ masm.testUndefinedSet(cond, value, output);
+}
+
+void CodeGenerator::visitIsNullAndBranch(LIsNullAndBranch* lir) {
+ MCompare::CompareType compareType = lir->cmpMir()->compareType();
+ MOZ_ASSERT(compareType == MCompare::Compare_Null);
+
+ JSOp op = lir->cmpMir()->jsop();
+ MOZ_ASSERT(IsStrictEqualityOp(op));
+
+ const ValueOperand value = ToValue(lir, LIsNullAndBranch::Value);
+
+ Assembler::Condition cond = JSOpToCondition(compareType, op);
+ testNullEmitBranch(cond, value, lir->ifTrue(), lir->ifFalse());
+}
+
+void CodeGenerator::visitIsUndefinedAndBranch(LIsUndefinedAndBranch* lir) {
+ MCompare::CompareType compareType = lir->cmpMir()->compareType();
+ MOZ_ASSERT(compareType == MCompare::Compare_Undefined);
+
+ JSOp op = lir->cmpMir()->jsop();
+ MOZ_ASSERT(IsStrictEqualityOp(op));
+
+ const ValueOperand value = ToValue(lir, LIsUndefinedAndBranch::Value);
+
+ Assembler::Condition cond = JSOpToCondition(compareType, op);
+ testUndefinedEmitBranch(cond, value, lir->ifTrue(), lir->ifFalse());
+}
+
+void CodeGenerator::visitSameValueDouble(LSameValueDouble* lir) {
+ FloatRegister left = ToFloatRegister(lir->left());
+ FloatRegister right = ToFloatRegister(lir->right());
+ FloatRegister temp = ToFloatRegister(lir->temp0());
+ Register output = ToRegister(lir->output());
+
+ masm.sameValueDouble(left, right, temp, output);
+}
+
+void CodeGenerator::visitSameValue(LSameValue* lir) {
+ ValueOperand lhs = ToValue(lir, LSameValue::LhsIndex);
+ ValueOperand rhs = ToValue(lir, LSameValue::RhsIndex);
+ Register output = ToRegister(lir->output());
+
+ using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*);
+ OutOfLineCode* ool =
+ oolCallVM<Fn, SameValue>(lir, ArgList(lhs, rhs), StoreRegisterTo(output));
+
+ // First check to see if the values have identical bits.
+ // This is correct for SameValue because SameValue(NaN,NaN) is true,
+ // and SameValue(0,-0) is false.
+ masm.branch64(Assembler::NotEqual, lhs.toRegister64(), rhs.toRegister64(),
+ ool->entry());
+ masm.move32(Imm32(1), output);
+
+ // If this fails, call SameValue.
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::emitConcat(LInstruction* lir, Register lhs, Register rhs,
+ Register output) {
+ using Fn = JSString* (*)(JSContext*, HandleString, HandleString,
+ js::gc::InitialHeap);
+ OutOfLineCode* ool = oolCallVM<Fn, ConcatStrings<CanGC>>(
+ lir, ArgList(lhs, rhs, static_cast<Imm32>(gc::DefaultHeap)),
+ StoreRegisterTo(output));
+
+ const JitRealm* jitRealm = gen->realm->jitRealm();
+ JitCode* stringConcatStub =
+ jitRealm->stringConcatStubNoBarrier(&realmStubsToReadBarrier_);
+ masm.call(stringConcatStub);
+ masm.branchTestPtr(Assembler::Zero, output, output, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitConcat(LConcat* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+
+ Register output = ToRegister(lir->output());
+
+ MOZ_ASSERT(lhs == CallTempReg0);
+ MOZ_ASSERT(rhs == CallTempReg1);
+ MOZ_ASSERT(ToRegister(lir->temp0()) == CallTempReg0);
+ MOZ_ASSERT(ToRegister(lir->temp1()) == CallTempReg1);
+ MOZ_ASSERT(ToRegister(lir->temp2()) == CallTempReg2);
+ MOZ_ASSERT(ToRegister(lir->temp3()) == CallTempReg3);
+ MOZ_ASSERT(ToRegister(lir->temp4()) == CallTempReg4);
+ MOZ_ASSERT(output == CallTempReg5);
+
+ emitConcat(lir, lhs, rhs, output);
+}
+
+static void CopyStringChars(MacroAssembler& masm, Register to, Register from,
+ Register len, Register byteOpScratch,
+ CharEncoding fromEncoding,
+ CharEncoding toEncoding) {
+ // Copy |len| char16_t code units from |from| to |to|. Assumes len > 0
+ // (checked below in debug builds), and when done |to| must point to the
+ // next available char.
+
+#ifdef DEBUG
+ Label ok;
+ masm.branch32(Assembler::GreaterThan, len, Imm32(0), &ok);
+ masm.assumeUnreachable("Length should be greater than 0.");
+ masm.bind(&ok);
+#endif
+
+ MOZ_ASSERT_IF(toEncoding == CharEncoding::Latin1,
+ fromEncoding == CharEncoding::Latin1);
+
+ size_t fromWidth =
+ fromEncoding == CharEncoding::Latin1 ? sizeof(char) : sizeof(char16_t);
+ size_t toWidth =
+ toEncoding == CharEncoding::Latin1 ? sizeof(char) : sizeof(char16_t);
+
+ Label start;
+ masm.bind(&start);
+ masm.loadChar(Address(from, 0), byteOpScratch, fromEncoding);
+ masm.storeChar(byteOpScratch, Address(to, 0), toEncoding);
+ masm.addPtr(Imm32(fromWidth), from);
+ masm.addPtr(Imm32(toWidth), to);
+ masm.branchSub32(Assembler::NonZero, Imm32(1), len, &start);
+}
+
+static void CopyStringChars(MacroAssembler& masm, Register to, Register from,
+ Register len, Register byteOpScratch,
+ CharEncoding encoding) {
+ CopyStringChars(masm, to, from, len, byteOpScratch, encoding, encoding);
+}
+
+static void CopyStringCharsMaybeInflate(MacroAssembler& masm, Register input,
+ Register destChars, Register temp1,
+ Register temp2) {
+ // destChars is TwoByte and input is a Latin1 or TwoByte string, so we may
+ // have to inflate.
+
+ Label isLatin1, done;
+ masm.loadStringLength(input, temp1);
+ masm.branchLatin1String(input, &isLatin1);
+ {
+ masm.loadStringChars(input, temp2, CharEncoding::TwoByte);
+ masm.movePtr(temp2, input);
+ CopyStringChars(masm, destChars, input, temp1, temp2,
+ CharEncoding::TwoByte);
+ masm.jump(&done);
+ }
+ masm.bind(&isLatin1);
+ {
+ masm.loadStringChars(input, temp2, CharEncoding::Latin1);
+ masm.movePtr(temp2, input);
+ CopyStringChars(masm, destChars, input, temp1, temp2, CharEncoding::Latin1,
+ CharEncoding::TwoByte);
+ }
+ masm.bind(&done);
+}
+
+static void AllocateThinOrFatInlineString(MacroAssembler& masm, Register output,
+ Register length, Register temp,
+ gc::InitialHeap initialStringHeap,
+ Label* failure,
+ CharEncoding encoding) {
+#ifdef DEBUG
+ size_t maxInlineLength;
+ if (encoding == CharEncoding::Latin1) {
+ maxInlineLength = JSFatInlineString::MAX_LENGTH_LATIN1;
+ } else {
+ maxInlineLength = JSFatInlineString::MAX_LENGTH_TWO_BYTE;
+ }
+
+ Label ok;
+ masm.branch32(Assembler::BelowOrEqual, length, Imm32(maxInlineLength), &ok);
+ masm.assumeUnreachable("string length too large to be allocated as inline");
+ masm.bind(&ok);
+#endif
+
+ size_t maxThinInlineLength;
+ if (encoding == CharEncoding::Latin1) {
+ maxThinInlineLength = JSThinInlineString::MAX_LENGTH_LATIN1;
+ } else {
+ maxThinInlineLength = JSThinInlineString::MAX_LENGTH_TWO_BYTE;
+ }
+
+ Label isFat, allocDone;
+ masm.branch32(Assembler::Above, length, Imm32(maxThinInlineLength), &isFat);
+ {
+ uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
+ if (encoding == CharEncoding::Latin1) {
+ flags |= JSString::LATIN1_CHARS_BIT;
+ }
+ masm.newGCString(output, temp, initialStringHeap, failure);
+ masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
+ masm.jump(&allocDone);
+ }
+ masm.bind(&isFat);
+ {
+ uint32_t flags = JSString::INIT_FAT_INLINE_FLAGS;
+ if (encoding == CharEncoding::Latin1) {
+ flags |= JSString::LATIN1_CHARS_BIT;
+ }
+ masm.newGCFatInlineString(output, temp, initialStringHeap, failure);
+ masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
+ }
+ masm.bind(&allocDone);
+
+ // Store length.
+ masm.store32(length, Address(output, JSString::offsetOfLength()));
+}
+
+static void ConcatInlineString(MacroAssembler& masm, Register lhs, Register rhs,
+ Register output, Register temp1, Register temp2,
+ Register temp3,
+ gc::InitialHeap initialStringHeap,
+ Label* failure, CharEncoding encoding) {
+ JitSpew(JitSpew_Codegen, "# Emitting ConcatInlineString (encoding=%s)",
+ (encoding == CharEncoding::Latin1 ? "Latin-1" : "Two-Byte"));
+
+ // State: result length in temp2.
+
+ // Ensure both strings are linear.
+ masm.branchIfRope(lhs, failure);
+ masm.branchIfRope(rhs, failure);
+
+ // Allocate a JSThinInlineString or JSFatInlineString.
+ AllocateThinOrFatInlineString(masm, output, temp2, temp1, initialStringHeap,
+ failure, encoding);
+
+ // Load chars pointer in temp2.
+ masm.loadInlineStringCharsForStore(output, temp2);
+
+ auto copyChars = [&](Register src) {
+ if (encoding == CharEncoding::TwoByte) {
+ CopyStringCharsMaybeInflate(masm, src, temp2, temp1, temp3);
+ } else {
+ masm.loadStringLength(src, temp3);
+ masm.loadStringChars(src, temp1, CharEncoding::Latin1);
+ masm.movePtr(temp1, src);
+ CopyStringChars(masm, temp2, src, temp3, temp1, CharEncoding::Latin1);
+ }
+ };
+
+ // Copy lhs chars. Note that this advances temp2 to point to the next
+ // char. This also clobbers the lhs register.
+ copyChars(lhs);
+
+ // Copy rhs chars. Clobbers the rhs register.
+ copyChars(rhs);
+
+ masm.ret();
+}
+
+void CodeGenerator::visitSubstr(LSubstr* lir) {
+ Register string = ToRegister(lir->string());
+ Register begin = ToRegister(lir->begin());
+ Register length = ToRegister(lir->length());
+ Register output = ToRegister(lir->output());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp2 = ToRegister(lir->temp2());
+
+ // On x86 there are not enough registers. In that case reuse the string
+ // register as temporary.
+ Register temp1 =
+ lir->temp1()->isBogusTemp() ? string : ToRegister(lir->temp1());
+
+ Label isLatin1, notInline, nonZero, nonInput, isInlinedLatin1;
+
+ // For every edge case use the C++ variant.
+ // Note: we also use this upon allocation failure in newGCString and
+ // newGCFatInlineString. To squeeze out even more performance those failures
+ // can be handled by allocate in ool code and returning to jit code to fill
+ // in all data.
+ using Fn = JSString* (*)(JSContext * cx, HandleString str, int32_t begin,
+ int32_t len);
+ OutOfLineCode* ool = oolCallVM<Fn, SubstringKernel>(
+ lir, ArgList(string, begin, length), StoreRegisterTo(output));
+ Label* slowPath = ool->entry();
+ Label* done = ool->rejoin();
+
+ // Zero length, return emptystring.
+ masm.branchTest32(Assembler::NonZero, length, length, &nonZero);
+ const JSAtomState& names = gen->runtime->names();
+ masm.movePtr(ImmGCPtr(names.empty), output);
+ masm.jump(done);
+
+ // Substring from 0..|str.length|, return str.
+ masm.bind(&nonZero);
+ masm.branch32(Assembler::NotEqual,
+ Address(string, JSString::offsetOfLength()), length, &nonInput);
+#ifdef DEBUG
+ {
+ Label ok;
+ masm.branchTest32(Assembler::Zero, begin, begin, &ok);
+ masm.assumeUnreachable("length == str.length implies begin == 0");
+ masm.bind(&ok);
+ }
+#endif
+ masm.movePtr(string, output);
+ masm.jump(done);
+
+ // Use slow path for ropes.
+ masm.bind(&nonInput);
+ masm.branchIfRope(string, slowPath);
+
+ // Allocate either a JSThinInlineString or JSFatInlineString, or jump to
+ // notInline if we need a dependent string.
+ {
+ static_assert(JSThinInlineString::MAX_LENGTH_LATIN1 <
+ JSFatInlineString::MAX_LENGTH_LATIN1);
+ static_assert(JSThinInlineString::MAX_LENGTH_TWO_BYTE <
+ JSFatInlineString::MAX_LENGTH_TWO_BYTE);
+
+ // Use temp2 to store the JS(Thin|Fat)InlineString flags. This avoids having
+ // duplicate newGCString/newGCFatInlineString codegen for Latin1 vs TwoByte
+ // strings.
+
+ Label isLatin1, allocFat, allocThin, allocDone;
+ masm.branchLatin1String(string, &isLatin1);
+ {
+ masm.branch32(Assembler::Above, length,
+ Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE), &notInline);
+ masm.move32(Imm32(0), temp2);
+ masm.branch32(Assembler::Above, length,
+ Imm32(JSThinInlineString::MAX_LENGTH_TWO_BYTE), &allocFat);
+ masm.jump(&allocThin);
+ }
+
+ masm.bind(&isLatin1);
+ {
+ masm.branch32(Assembler::Above, length,
+ Imm32(JSFatInlineString::MAX_LENGTH_LATIN1), &notInline);
+ masm.move32(Imm32(JSString::LATIN1_CHARS_BIT), temp2);
+ masm.branch32(Assembler::Above, length,
+ Imm32(JSThinInlineString::MAX_LENGTH_LATIN1), &allocFat);
+ }
+
+ masm.bind(&allocThin);
+ {
+ masm.newGCString(output, temp0, initialStringHeap(), slowPath);
+ masm.or32(Imm32(JSString::INIT_THIN_INLINE_FLAGS), temp2);
+ masm.jump(&allocDone);
+ }
+ masm.bind(&allocFat);
+ {
+ masm.newGCFatInlineString(output, temp0, initialStringHeap(), slowPath);
+ masm.or32(Imm32(JSString::INIT_FAT_INLINE_FLAGS), temp2);
+ }
+
+ masm.bind(&allocDone);
+ masm.store32(temp2, Address(output, JSString::offsetOfFlags()));
+ masm.store32(length, Address(output, JSString::offsetOfLength()));
+ }
+
+ auto initializeInlineString = [&](CharEncoding encoding) {
+ masm.loadStringChars(string, temp0, encoding);
+ masm.addToCharPtr(temp0, begin, encoding);
+ if (temp1 == string) {
+ masm.push(string);
+ }
+ masm.loadInlineStringCharsForStore(output, temp1);
+ CopyStringChars(masm, temp1, temp0, length, temp2, encoding);
+ masm.loadStringLength(output, length);
+ if (temp1 == string) {
+ masm.pop(string);
+ }
+ masm.jump(done);
+ };
+
+ masm.branchLatin1String(string, &isInlinedLatin1);
+ initializeInlineString(CharEncoding::TwoByte);
+
+ masm.bind(&isInlinedLatin1);
+ initializeInlineString(CharEncoding::Latin1);
+
+ // Handle other cases with a DependentString.
+ masm.bind(&notInline);
+ masm.newGCString(output, temp0, gen->initialStringHeap(), slowPath);
+ masm.store32(length, Address(output, JSString::offsetOfLength()));
+ masm.storeDependentStringBase(string, output);
+
+ auto initializeDependentString = [&](CharEncoding encoding) {
+ uint32_t flags = JSString::INIT_DEPENDENT_FLAGS;
+ if (encoding == CharEncoding::Latin1) {
+ flags |= JSString::LATIN1_CHARS_BIT;
+ }
+
+ masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
+ masm.loadNonInlineStringChars(string, temp0, encoding);
+ masm.addToCharPtr(temp0, begin, encoding);
+ masm.storeNonInlineStringChars(temp0, output);
+ masm.jump(done);
+ };
+
+ masm.branchLatin1String(string, &isLatin1);
+ initializeDependentString(CharEncoding::TwoByte);
+
+ masm.bind(&isLatin1);
+ initializeDependentString(CharEncoding::Latin1);
+
+ masm.bind(done);
+}
+
+JitCode* JitRealm::generateStringConcatStub(JSContext* cx) {
+ JitSpew(JitSpew_Codegen, "# Emitting StringConcat stub");
+
+ TempAllocator temp(&cx->tempLifoAlloc());
+ JitContext jcx(cx);
+ StackMacroAssembler masm(cx, temp);
+ AutoCreatedBy acb(masm, "JitRealm::generateStringConcatStub");
+
+ Register lhs = CallTempReg0;
+ Register rhs = CallTempReg1;
+ Register temp1 = CallTempReg2;
+ Register temp2 = CallTempReg3;
+ Register temp3 = CallTempReg4;
+ Register output = CallTempReg5;
+
+ Label failure;
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+ // If lhs is empty, return rhs.
+ Label leftEmpty;
+ masm.loadStringLength(lhs, temp1);
+ masm.branchTest32(Assembler::Zero, temp1, temp1, &leftEmpty);
+
+ // If rhs is empty, return lhs.
+ Label rightEmpty;
+ masm.loadStringLength(rhs, temp2);
+ masm.branchTest32(Assembler::Zero, temp2, temp2, &rightEmpty);
+
+ masm.add32(temp1, temp2);
+
+ // Check if we can use a JSInlineString. The result is a Latin1 string if
+ // lhs and rhs are both Latin1, so we AND the flags.
+ Label isInlineTwoByte, isInlineLatin1;
+ masm.load32(Address(lhs, JSString::offsetOfFlags()), temp1);
+ masm.and32(Address(rhs, JSString::offsetOfFlags()), temp1);
+
+ Label isLatin1, notInline;
+ masm.branchTest32(Assembler::NonZero, temp1,
+ Imm32(JSString::LATIN1_CHARS_BIT), &isLatin1);
+ {
+ masm.branch32(Assembler::BelowOrEqual, temp2,
+ Imm32(JSFatInlineString::MAX_LENGTH_TWO_BYTE),
+ &isInlineTwoByte);
+ masm.jump(&notInline);
+ }
+ masm.bind(&isLatin1);
+ {
+ masm.branch32(Assembler::BelowOrEqual, temp2,
+ Imm32(JSFatInlineString::MAX_LENGTH_LATIN1), &isInlineLatin1);
+ }
+ masm.bind(&notInline);
+
+ // Keep AND'ed flags in temp1.
+
+ // Ensure result length <= JSString::MAX_LENGTH.
+ masm.branch32(Assembler::Above, temp2, Imm32(JSString::MAX_LENGTH), &failure);
+
+ // Allocate a new rope, guaranteed to be in the nursery if initialStringHeap
+ // == gc::DefaultHeap. (As a result, no post barriers are needed below.)
+ masm.newGCString(output, temp3, initialStringHeap, &failure);
+
+ // Store rope length and flags. temp1 still holds the result of AND'ing the
+ // lhs and rhs flags, so we just have to clear the other flags to get our rope
+ // flags (Latin1 if both lhs and rhs are Latin1).
+ static_assert(JSString::INIT_ROPE_FLAGS == 0,
+ "Rope type flags must have no bits set");
+ masm.and32(Imm32(JSString::LATIN1_CHARS_BIT), temp1);
+ masm.store32(temp1, Address(output, JSString::offsetOfFlags()));
+ masm.store32(temp2, Address(output, JSString::offsetOfLength()));
+
+ // Store left and right nodes.
+ masm.storeRopeChildren(lhs, rhs, output);
+ masm.ret();
+
+ masm.bind(&leftEmpty);
+ masm.mov(rhs, output);
+ masm.ret();
+
+ masm.bind(&rightEmpty);
+ masm.mov(lhs, output);
+ masm.ret();
+
+ masm.bind(&isInlineTwoByte);
+ ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
+ initialStringHeap, &failure, CharEncoding::TwoByte);
+
+ masm.bind(&isInlineLatin1);
+ ConcatInlineString(masm, lhs, rhs, output, temp1, temp2, temp3,
+ initialStringHeap, &failure, CharEncoding::Latin1);
+
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ masm.bind(&failure);
+ masm.movePtr(ImmPtr(nullptr), output);
+ masm.ret();
+
+ Linker linker(masm);
+ JitCode* code = linker.newCode(cx, CodeKind::Other);
+
+ CollectPerfSpewerJitCodeProfile(code, "StringConcatStub");
+#ifdef MOZ_VTUNE
+ vtune::MarkStub(code, "StringConcatStub");
+#endif
+
+ return code;
+}
+
+void JitRuntime::generateFreeStub(MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateFreeStub");
+
+ const Register regSlots = CallTempReg0;
+
+ freeStubOffset_ = startTrampolineCode(masm);
+
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+ AllocatableRegisterSet regs(RegisterSet::Volatile());
+ regs.takeUnchecked(regSlots);
+ LiveRegisterSet save(regs.asLiveSet());
+ masm.PushRegsInMask(save);
+
+ const Register regTemp = regs.takeAnyGeneral();
+ MOZ_ASSERT(regTemp != regSlots);
+
+ using Fn = void (*)(void* p);
+ masm.setupUnalignedABICall(regTemp);
+ masm.passABIArg(regSlots);
+ masm.callWithABI<Fn, js_free>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.PopRegsInMask(save);
+
+ masm.ret();
+}
+
+void JitRuntime::generateLazyLinkStub(MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateLazyLinkStub");
+
+ lazyLinkStubOffset_ = startTrampolineCode(masm);
+
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+ masm.Push(FramePointer);
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
+ Register temp0 = regs.takeAny();
+ Register temp1 = regs.takeAny();
+ Register temp2 = regs.takeAny();
+
+ masm.loadJSContext(temp0);
+ masm.enterFakeExitFrame(temp0, temp2, ExitFrameType::LazyLink);
+ masm.moveStackPtrTo(temp1);
+
+ using Fn = uint8_t* (*)(JSContext * cx, LazyLinkExitFrameLayout * frame);
+ masm.setupUnalignedABICall(temp2);
+ masm.passABIArg(temp0);
+ masm.passABIArg(temp1);
+ masm.callWithABI<Fn, LazyLinkTopActivation>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Discard exit frame and frame pointer.
+ masm.leaveExitFrame(sizeof(void*));
+
+#ifdef JS_USE_LINK_REGISTER
+ // Restore the return address such that the emitPrologue function of the
+ // CodeGenerator can push it back on the stack with pushReturnAddress.
+ masm.popReturnAddress();
+#endif
+ masm.jump(ReturnReg);
+}
+
+void JitRuntime::generateInterpreterStub(MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateInterpreterStub");
+
+ interpreterStubOffset_ = startTrampolineCode(masm);
+
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+ masm.Push(FramePointer);
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::Volatile());
+ Register temp0 = regs.takeAny();
+ Register temp1 = regs.takeAny();
+ Register temp2 = regs.takeAny();
+
+ masm.loadJSContext(temp0);
+ masm.enterFakeExitFrame(temp0, temp2, ExitFrameType::InterpreterStub);
+ masm.moveStackPtrTo(temp1);
+
+ using Fn = bool (*)(JSContext * cx, InterpreterStubExitFrameLayout * frame);
+ masm.setupUnalignedABICall(temp2);
+ masm.passABIArg(temp0);
+ masm.passABIArg(temp1);
+ masm.callWithABI<Fn, InvokeFromInterpreterStub>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ masm.branchIfFalseBool(ReturnReg, masm.failureLabel());
+
+ // Discard exit frame and frame pointer.
+ masm.leaveExitFrame(sizeof(void*));
+
+ // InvokeFromInterpreterStub stores the return value in argv[0], where the
+ // caller stored |this|. Subtract |sizeof(void*)| for the frame pointer we
+ // just popped.
+ masm.loadValue(Address(masm.getStackPointer(),
+ JitFrameLayout::offsetOfThis() - sizeof(void*)),
+ JSReturnOperand);
+ masm.ret();
+}
+
+void JitRuntime::generateDoubleToInt32ValueStub(MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateDoubleToInt32ValueStub");
+ doubleToInt32ValueStubOffset_ = startTrampolineCode(masm);
+
+ Label done;
+ masm.branchTestDouble(Assembler::NotEqual, R0, &done);
+
+ masm.unboxDouble(R0, FloatReg0);
+ masm.convertDoubleToInt32(FloatReg0, R1.scratchReg(), &done,
+ /* negativeZeroCheck = */ false);
+ masm.tagValue(JSVAL_TYPE_INT32, R1.scratchReg(), R0);
+
+ masm.bind(&done);
+ masm.abiret();
+}
+
+void CodeGenerator::visitLinearizeForCharAccess(LLinearizeForCharAccess* lir) {
+ Register str = ToRegister(lir->str());
+ Register index = ToRegister(lir->index());
+ Register output = ToRegister(lir->output());
+
+ using Fn = JSLinearString* (*)(JSContext*, JSString*);
+ auto* ool = oolCallVM<Fn, jit::LinearizeForCharAccess>(
+ lir, ArgList(str), StoreRegisterTo(output));
+
+ masm.branchIfNotCanLoadStringChar(str, index, output, ool->entry());
+
+ masm.movePtr(str, output);
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCharCodeAt(LCharCodeAt* lir) {
+ Register str = ToRegister(lir->str());
+ Register index = ToRegister(lir->index());
+ Register output = ToRegister(lir->output());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ using Fn = bool (*)(JSContext*, HandleString, int32_t, uint32_t*);
+ OutOfLineCode* ool = oolCallVM<Fn, jit::CharCodeAt>(lir, ArgList(str, index),
+ StoreRegisterTo(output));
+ masm.loadStringChar(str, index, output, temp0, temp1, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCharCodeAtMaybeOutOfBounds(
+ LCharCodeAtMaybeOutOfBounds* lir) {
+ Register str = ToRegister(lir->str());
+ Register index = ToRegister(lir->index());
+ ValueOperand output = ToOutValue(lir);
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ using Fn = bool (*)(JSContext*, HandleString, int32_t, uint32_t*);
+ auto* ool = oolCallVM<Fn, jit::CharCodeAt>(
+ lir, ArgList(str, index), StoreRegisterTo(output.scratchReg()));
+
+ // Return NaN for out-of-bounds access.
+ Label done;
+ masm.moveValue(JS::NaNValue(), output);
+
+ masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
+ temp0, &done);
+
+ masm.loadStringChar(str, index, output.scratchReg(), temp0, temp1,
+ ool->entry());
+ masm.bind(ool->rejoin());
+
+ masm.tagValue(JSVAL_TYPE_INT32, output.scratchReg(), output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitCharAtMaybeOutOfBounds(LCharAtMaybeOutOfBounds* lir) {
+ Register str = ToRegister(lir->str());
+ Register index = ToRegister(lir->index());
+ Register output = ToRegister(lir->output());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ using Fn1 = bool (*)(JSContext*, HandleString, int32_t, uint32_t*);
+ auto* oolLoadChar = oolCallVM<Fn1, jit::CharCodeAt>(lir, ArgList(str, index),
+ StoreRegisterTo(output));
+
+ using Fn2 = JSLinearString* (*)(JSContext*, int32_t);
+ auto* oolFromCharCode = oolCallVM<Fn2, jit::StringFromCharCode>(
+ lir, ArgList(output), StoreRegisterTo(output));
+
+ // Return the empty string for out-of-bounds access.
+ const JSAtomState& names = gen->runtime->names();
+ masm.movePtr(ImmGCPtr(names.empty), output);
+
+ masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
+ temp0, oolFromCharCode->rejoin());
+
+ masm.loadStringChar(str, index, output, temp0, temp1, oolLoadChar->entry());
+ masm.bind(oolLoadChar->rejoin());
+
+ // OOL path if code >= UNIT_STATIC_LIMIT.
+ masm.boundsCheck32PowerOfTwo(output, StaticStrings::UNIT_STATIC_LIMIT,
+ oolFromCharCode->entry());
+
+ masm.movePtr(ImmPtr(&gen->runtime->staticStrings().unitStaticTable), temp0);
+ masm.loadPtr(BaseIndex(temp0, output, ScalePointer), output);
+
+ masm.bind(oolFromCharCode->rejoin());
+}
+
+void CodeGenerator::visitFromCharCode(LFromCharCode* lir) {
+ Register code = ToRegister(lir->code());
+ Register output = ToRegister(lir->output());
+
+ using Fn = JSLinearString* (*)(JSContext*, int32_t);
+ OutOfLineCode* ool = oolCallVM<Fn, jit::StringFromCharCode>(
+ lir, ArgList(code), StoreRegisterTo(output));
+
+ // OOL path if code >= UNIT_STATIC_LIMIT.
+ masm.boundsCheck32PowerOfTwo(code, StaticStrings::UNIT_STATIC_LIMIT,
+ ool->entry());
+
+ masm.movePtr(ImmPtr(&gen->runtime->staticStrings().unitStaticTable), output);
+ masm.loadPtr(BaseIndex(output, code, ScalePointer), output);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitFromCodePoint(LFromCodePoint* lir) {
+ Register codePoint = ToRegister(lir->codePoint());
+ Register output = ToRegister(lir->output());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ LSnapshot* snapshot = lir->snapshot();
+
+ // The OOL path is only taken when we can't allocate the inline string.
+ using Fn = JSString* (*)(JSContext*, int32_t);
+ OutOfLineCode* ool = oolCallVM<Fn, jit::StringFromCodePoint>(
+ lir, ArgList(codePoint), StoreRegisterTo(output));
+
+ Label isTwoByte;
+ Label* done = ool->rejoin();
+
+ static_assert(
+ StaticStrings::UNIT_STATIC_LIMIT - 1 == JSString::MAX_LATIN1_CHAR,
+ "Latin-1 strings can be loaded from static strings");
+ masm.boundsCheck32PowerOfTwo(codePoint, StaticStrings::UNIT_STATIC_LIMIT,
+ &isTwoByte);
+ {
+ masm.movePtr(ImmPtr(&gen->runtime->staticStrings().unitStaticTable),
+ output);
+ masm.loadPtr(BaseIndex(output, codePoint, ScalePointer), output);
+ masm.jump(done);
+ }
+ masm.bind(&isTwoByte);
+ {
+ // Use a bailout if the input is not a valid code point, because
+ // MFromCodePoint is movable and it'd be observable when a moved
+ // fromCodePoint throws an exception before its actual call site.
+ bailoutCmp32(Assembler::Above, codePoint, Imm32(unicode::NonBMPMax),
+ snapshot);
+
+ // Allocate a JSThinInlineString.
+ {
+ static_assert(JSThinInlineString::MAX_LENGTH_TWO_BYTE >= 2,
+ "JSThinInlineString can hold a supplementary code point");
+
+ uint32_t flags = JSString::INIT_THIN_INLINE_FLAGS;
+ masm.newGCString(output, temp0, gen->initialStringHeap(), ool->entry());
+ masm.store32(Imm32(flags), Address(output, JSString::offsetOfFlags()));
+ }
+
+ Label isSupplementary;
+ masm.branch32(Assembler::AboveOrEqual, codePoint, Imm32(unicode::NonBMPMin),
+ &isSupplementary);
+ {
+ // Store length.
+ masm.store32(Imm32(1), Address(output, JSString::offsetOfLength()));
+
+ // Load chars pointer in temp0.
+ masm.loadInlineStringCharsForStore(output, temp0);
+
+ masm.store16(codePoint, Address(temp0, 0));
+
+ masm.jump(done);
+ }
+ masm.bind(&isSupplementary);
+ {
+ // Store length.
+ masm.store32(Imm32(2), Address(output, JSString::offsetOfLength()));
+
+ // Load chars pointer in temp0.
+ masm.loadInlineStringCharsForStore(output, temp0);
+
+ // Inlined unicode::LeadSurrogate(uint32_t).
+ masm.move32(codePoint, temp1);
+ masm.rshift32(Imm32(10), temp1);
+ masm.add32(Imm32(unicode::LeadSurrogateMin - (unicode::NonBMPMin >> 10)),
+ temp1);
+
+ masm.store16(temp1, Address(temp0, 0));
+
+ // Inlined unicode::TrailSurrogate(uint32_t).
+ masm.move32(codePoint, temp1);
+ masm.and32(Imm32(0x3FF), temp1);
+ masm.or32(Imm32(unicode::TrailSurrogateMin), temp1);
+
+ masm.store16(temp1, Address(temp0, sizeof(char16_t)));
+ }
+ }
+
+ masm.bind(done);
+}
+
+void CodeGenerator::visitStringIndexOf(LStringIndexOf* lir) {
+ pushArg(ToRegister(lir->searchString()));
+ pushArg(ToRegister(lir->string()));
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, int32_t*);
+ callVM<Fn, js::StringIndexOf>(lir);
+}
+
+void CodeGenerator::visitStringStartsWith(LStringStartsWith* lir) {
+ pushArg(ToRegister(lir->searchString()));
+ pushArg(ToRegister(lir->string()));
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ callVM<Fn, js::StringStartsWith>(lir);
+}
+
+void CodeGenerator::visitStringStartsWithInline(LStringStartsWithInline* lir) {
+ Register string = ToRegister(lir->string());
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ const JSLinearString* searchString = lir->searchString();
+
+ size_t length = searchString->length();
+ MOZ_ASSERT(length > 0);
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ auto* ool = oolCallVM<Fn, js::StringStartsWith>(
+ lir, ArgList(string, ImmGCPtr(searchString)), StoreRegisterTo(output));
+
+ masm.move32(Imm32(0), output);
+
+ // Can't be a prefix when the string is smaller than the search string.
+ masm.branch32(Assembler::Below, Address(string, JSString::offsetOfLength()),
+ Imm32(length), ool->rejoin());
+
+ // Unwind ropes at the start if possible.
+ Label compare;
+ masm.movePtr(string, temp);
+ masm.branchIfNotRope(temp, &compare);
+
+ Label unwindRope;
+ masm.bind(&unwindRope);
+ masm.loadRopeLeftChild(temp, output);
+ masm.movePtr(output, temp);
+
+ // If the left child is smaller than the search string, jump into the VM to
+ // linearize the string.
+ masm.branch32(Assembler::Below, Address(temp, JSString::offsetOfLength()),
+ Imm32(length), ool->entry());
+
+ // Otherwise keep unwinding ropes.
+ masm.branchIfRope(temp, &unwindRope);
+
+ masm.bind(&compare);
+
+ // If operands point to the same instance, it's trivially a prefix.
+ Label notPointerEqual;
+ masm.branchPtr(Assembler::NotEqual, temp, ImmGCPtr(searchString),
+ &notPointerEqual);
+ masm.move32(Imm32(1), output);
+ masm.jump(ool->rejoin());
+ masm.bind(&notPointerEqual);
+
+ if (searchString->hasTwoByteChars()) {
+ // Pure two-byte strings can't be a prefix of Latin-1 strings.
+ JS::AutoCheckCannotGC nogc;
+ if (!mozilla::IsUtf16Latin1(searchString->twoByteRange(nogc))) {
+ Label compareChars;
+ masm.branchTwoByteString(temp, &compareChars);
+ masm.move32(Imm32(0), output);
+ masm.jump(ool->rejoin());
+ masm.bind(&compareChars);
+ }
+ }
+
+ // Otherwise start comparing character by character.
+ CompareCharacters(masm, temp, searchString, output, JSOp::Eq,
+ CompareDirection::Forward, ool->rejoin(), ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitStringEndsWith(LStringEndsWith* lir) {
+ pushArg(ToRegister(lir->searchString()));
+ pushArg(ToRegister(lir->string()));
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ callVM<Fn, js::StringEndsWith>(lir);
+}
+
+void CodeGenerator::visitStringEndsWithInline(LStringEndsWithInline* lir) {
+ Register string = ToRegister(lir->string());
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ const JSLinearString* searchString = lir->searchString();
+
+ size_t length = searchString->length();
+ MOZ_ASSERT(length > 0);
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ auto* ool = oolCallVM<Fn, js::StringEndsWith>(
+ lir, ArgList(string, ImmGCPtr(searchString)), StoreRegisterTo(output));
+
+ masm.move32(Imm32(0), output);
+
+ // Can't be a suffix when the string is smaller than the search string.
+ masm.branch32(Assembler::Below, Address(string, JSString::offsetOfLength()),
+ Imm32(length), ool->rejoin());
+
+ // Unwind ropes at the end if possible.
+ Label compare;
+ masm.movePtr(string, temp);
+ masm.branchIfNotRope(temp, &compare);
+
+ Label unwindRope;
+ masm.bind(&unwindRope);
+ masm.loadRopeRightChild(temp, output);
+ masm.movePtr(output, temp);
+
+ // If the right child is smaller than the search string, jump into the VM to
+ // linearize the string.
+ masm.branch32(Assembler::Below, Address(temp, JSString::offsetOfLength()),
+ Imm32(length), ool->entry());
+
+ // Otherwise keep unwinding ropes.
+ masm.branchIfRope(temp, &unwindRope);
+
+ masm.bind(&compare);
+
+ // If operands point to the same instance, it's trivially a suffix.
+ Label notPointerEqual;
+ masm.branchPtr(Assembler::NotEqual, temp, ImmGCPtr(searchString),
+ &notPointerEqual);
+ masm.move32(Imm32(1), output);
+ masm.jump(ool->rejoin());
+ masm.bind(&notPointerEqual);
+
+ if (searchString->hasTwoByteChars()) {
+ // Pure two-byte strings can't be a suffix of Latin-1 strings.
+ JS::AutoCheckCannotGC nogc;
+ if (!mozilla::IsUtf16Latin1(searchString->twoByteRange(nogc))) {
+ Label compareChars;
+ masm.branchTwoByteString(temp, &compareChars);
+ masm.move32(Imm32(0), output);
+ masm.jump(ool->rejoin());
+ masm.bind(&compareChars);
+ }
+ }
+
+ // Otherwise start comparing character by character.
+ CompareCharacters(masm, temp, searchString, output, JSOp::Eq,
+ CompareDirection::Backward, ool->rejoin(), ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitStringToLowerCase(LStringToLowerCase* lir) {
+ Register string = ToRegister(lir->string());
+ Register output = ToRegister(lir->output());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+
+ // On x86 there are not enough registers. In that case reuse the string
+ // register as a temporary.
+ Register temp3 =
+ lir->temp3()->isBogusTemp() ? string : ToRegister(lir->temp3());
+ Register temp4 = ToRegister(lir->temp4());
+
+ using Fn = JSString* (*)(JSContext*, HandleString);
+ OutOfLineCode* ool = oolCallVM<Fn, js::StringToLowerCase>(
+ lir, ArgList(string), StoreRegisterTo(output));
+
+ // Take the slow path if the string isn't a linear Latin-1 string.
+ Imm32 linearLatin1Bits(JSString::LINEAR_BIT | JSString::LATIN1_CHARS_BIT);
+ Register flags = temp0;
+ masm.load32(Address(string, JSString::offsetOfFlags()), flags);
+ masm.and32(linearLatin1Bits, flags);
+ masm.branch32(Assembler::NotEqual, flags, linearLatin1Bits, ool->entry());
+
+ Register length = temp0;
+ masm.loadStringLength(string, length);
+
+ // Return the input if it's the empty string.
+ Label notEmptyString;
+ masm.branch32(Assembler::NotEqual, length, Imm32(0), &notEmptyString);
+ {
+ masm.movePtr(string, output);
+ masm.jump(ool->rejoin());
+ }
+ masm.bind(&notEmptyString);
+
+ Register inputChars = temp1;
+ masm.loadStringChars(string, inputChars, CharEncoding::Latin1);
+
+ Register toLowerCaseTable = temp2;
+ masm.movePtr(ImmPtr(unicode::latin1ToLowerCaseTable), toLowerCaseTable);
+
+ // Single element strings can be directly retrieved from static strings cache.
+ Label notSingleElementString;
+ masm.branch32(Assembler::NotEqual, length, Imm32(1), &notSingleElementString);
+ {
+ Register current = temp4;
+
+ masm.loadChar(Address(inputChars, 0), current, CharEncoding::Latin1);
+ masm.load8ZeroExtend(BaseIndex(toLowerCaseTable, current, TimesOne),
+ current);
+ masm.movePtr(ImmPtr(&gen->runtime->staticStrings().unitStaticTable),
+ output);
+ masm.loadPtr(BaseIndex(output, current, ScalePointer), output);
+
+ masm.jump(ool->rejoin());
+ }
+ masm.bind(&notSingleElementString);
+
+ // Use the OOL-path when the string is too long. This prevents scanning long
+ // strings which have upper case characters only near the end a second time in
+ // the VM.
+ constexpr int32_t MaxInlineLength = 64;
+ masm.branch32(Assembler::Above, length, Imm32(MaxInlineLength), ool->entry());
+
+ {
+ // Check if there are any characters which need to be converted.
+ //
+ // This extra loop gives a small performance improvement for strings which
+ // are already lower cased and lets us avoid calling into the runtime for
+ // non-inline, all lower case strings. But more importantly it avoids
+ // repeated inline allocation failures:
+ // |AllocateThinOrFatInlineString| below takes the OOL-path and calls the
+ // |js::StringToLowerCase| runtime function when the result string can't be
+ // allocated inline. And |js::StringToLowerCase| directly returns the input
+ // string when no characters need to be converted. That means it won't
+ // trigger GC to clear up the free nursery space, so the next toLowerCase()
+ // call will again fail to inline allocate the result string.
+ Label hasUpper;
+ {
+ Register checkInputChars = output;
+ masm.movePtr(inputChars, checkInputChars);
+
+ Register current = temp4;
+
+ Label start;
+ masm.bind(&start);
+ masm.loadChar(Address(checkInputChars, 0), current, CharEncoding::Latin1);
+ masm.branch8(Assembler::NotEqual,
+ BaseIndex(toLowerCaseTable, current, TimesOne), current,
+ &hasUpper);
+ masm.addPtr(Imm32(sizeof(Latin1Char)), checkInputChars);
+ masm.branchSub32(Assembler::NonZero, Imm32(1), length, &start);
+
+ // Input is already in lower case.
+ masm.movePtr(string, output);
+ masm.jump(ool->rejoin());
+ }
+ masm.bind(&hasUpper);
+
+ // |length| was clobbered above, reload.
+ masm.loadStringLength(string, length);
+
+ // Call into the runtime when we can't create an inline string.
+ masm.branch32(Assembler::Above, length,
+ Imm32(JSFatInlineString::MAX_LENGTH_LATIN1), ool->entry());
+
+ AllocateThinOrFatInlineString(masm, output, length, temp4,
+ initialStringHeap(), ool->entry(),
+ CharEncoding::Latin1);
+
+ if (temp3 == string) {
+ masm.push(string);
+ }
+
+ Register outputChars = temp3;
+ masm.loadInlineStringCharsForStore(output, outputChars);
+
+ {
+ Register current = temp4;
+
+ Label start;
+ masm.bind(&start);
+ masm.loadChar(Address(inputChars, 0), current, CharEncoding::Latin1);
+ masm.load8ZeroExtend(BaseIndex(toLowerCaseTable, current, TimesOne),
+ current);
+ masm.storeChar(current, Address(outputChars, 0), CharEncoding::Latin1);
+ masm.addPtr(Imm32(sizeof(Latin1Char)), inputChars);
+ masm.addPtr(Imm32(sizeof(Latin1Char)), outputChars);
+ masm.branchSub32(Assembler::NonZero, Imm32(1), length, &start);
+ }
+
+ if (temp3 == string) {
+ masm.pop(string);
+ }
+ }
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitStringToUpperCase(LStringToUpperCase* lir) {
+ pushArg(ToRegister(lir->string()));
+
+ using Fn = JSString* (*)(JSContext*, HandleString);
+ callVM<Fn, js::StringToUpperCase>(lir);
+}
+
+void CodeGenerator::visitStringSplit(LStringSplit* lir) {
+ pushArg(Imm32(INT32_MAX));
+ pushArg(ToRegister(lir->separator()));
+ pushArg(ToRegister(lir->string()));
+
+ using Fn = ArrayObject* (*)(JSContext*, HandleString, HandleString, uint32_t);
+ callVM<Fn, js::StringSplitString>(lir);
+}
+
+void CodeGenerator::visitInitializedLength(LInitializedLength* lir) {
+ Address initLength(ToRegister(lir->elements()),
+ ObjectElements::offsetOfInitializedLength());
+ masm.load32(initLength, ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitSetInitializedLength(LSetInitializedLength* lir) {
+ Address initLength(ToRegister(lir->elements()),
+ ObjectElements::offsetOfInitializedLength());
+ SetLengthFromIndex(masm, lir->index(), initLength);
+}
+
+void CodeGenerator::visitNotBI(LNotBI* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ masm.cmp32Set(Assembler::Equal, Address(input, BigInt::offsetOfLength()),
+ Imm32(0), output);
+}
+
+void CodeGenerator::visitNotO(LNotO* lir) {
+ auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
+ addOutOfLineCode(ool, lir->mir());
+
+ Label* ifEmulatesUndefined = ool->label1();
+ Label* ifDoesntEmulateUndefined = ool->label2();
+
+ Register objreg = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ branchTestObjectEmulatesUndefined(objreg, ifEmulatesUndefined,
+ ifDoesntEmulateUndefined, output, ool);
+ // fall through
+
+ Label join;
+
+ masm.move32(Imm32(0), output);
+ masm.jump(&join);
+
+ masm.bind(ifEmulatesUndefined);
+ masm.move32(Imm32(1), output);
+
+ masm.bind(&join);
+}
+
+void CodeGenerator::visitNotV(LNotV* lir) {
+ auto* ool = new (alloc()) OutOfLineTestObjectWithLabels();
+ addOutOfLineCode(ool, lir->mir());
+
+ Label* ifTruthy = ool->label1();
+ Label* ifFalsy = ool->label2();
+
+ ValueOperand input = ToValue(lir, LNotV::InputIndex);
+ Register tempToUnbox = ToTempUnboxRegister(lir->temp1());
+ FloatRegister floatTemp = ToFloatRegister(lir->temp0());
+ Register output = ToRegister(lir->output());
+ const TypeDataList& observedTypes = lir->mir()->observedTypes();
+
+ testValueTruthy(input, tempToUnbox, output, floatTemp, observedTypes,
+ ifTruthy, ifFalsy, ool);
+
+ Label join;
+
+ // Note that the testValueTruthy call above may choose to fall through
+ // to ifTruthy instead of branching there.
+ masm.bind(ifTruthy);
+ masm.move32(Imm32(0), output);
+ masm.jump(&join);
+
+ masm.bind(ifFalsy);
+ masm.move32(Imm32(1), output);
+
+ // both branches meet here.
+ masm.bind(&join);
+}
+
+void CodeGenerator::visitBoundsCheck(LBoundsCheck* lir) {
+ const LAllocation* index = lir->index();
+ const LAllocation* length = lir->length();
+ LSnapshot* snapshot = lir->snapshot();
+
+ MIRType type = lir->mir()->type();
+
+ auto bailoutCmp = [&](Assembler::Condition cond, auto lhs, auto rhs) {
+ if (type == MIRType::Int32) {
+ bailoutCmp32(cond, lhs, rhs, snapshot);
+ } else {
+ MOZ_ASSERT(type == MIRType::IntPtr);
+ bailoutCmpPtr(cond, lhs, rhs, snapshot);
+ }
+ };
+
+ auto bailoutCmpConstant = [&](Assembler::Condition cond, auto lhs,
+ int32_t rhs) {
+ if (type == MIRType::Int32) {
+ bailoutCmp32(cond, lhs, Imm32(rhs), snapshot);
+ } else {
+ MOZ_ASSERT(type == MIRType::IntPtr);
+ bailoutCmpPtr(cond, lhs, ImmWord(rhs), snapshot);
+ }
+ };
+
+ if (index->isConstant()) {
+ // Use uint32 so that the comparison is unsigned.
+ uint32_t idx = ToInt32(index);
+ if (length->isConstant()) {
+ uint32_t len = ToInt32(lir->length());
+ if (idx < len) {
+ return;
+ }
+ bailout(snapshot);
+ return;
+ }
+
+ if (length->isRegister()) {
+ bailoutCmpConstant(Assembler::BelowOrEqual, ToRegister(length), idx);
+ } else {
+ bailoutCmpConstant(Assembler::BelowOrEqual, ToAddress(length), idx);
+ }
+ return;
+ }
+
+ Register indexReg = ToRegister(index);
+ if (length->isConstant()) {
+ bailoutCmpConstant(Assembler::AboveOrEqual, indexReg, ToInt32(length));
+ } else if (length->isRegister()) {
+ bailoutCmp(Assembler::BelowOrEqual, ToRegister(length), indexReg);
+ } else {
+ bailoutCmp(Assembler::BelowOrEqual, ToAddress(length), indexReg);
+ }
+}
+
+void CodeGenerator::visitBoundsCheckRange(LBoundsCheckRange* lir) {
+ int32_t min = lir->mir()->minimum();
+ int32_t max = lir->mir()->maximum();
+ MOZ_ASSERT(max >= min);
+
+ LSnapshot* snapshot = lir->snapshot();
+ MIRType type = lir->mir()->type();
+
+ const LAllocation* length = lir->length();
+ Register temp = ToRegister(lir->getTemp(0));
+
+ auto bailoutCmp = [&](Assembler::Condition cond, auto lhs, auto rhs) {
+ if (type == MIRType::Int32) {
+ bailoutCmp32(cond, lhs, rhs, snapshot);
+ } else {
+ MOZ_ASSERT(type == MIRType::IntPtr);
+ bailoutCmpPtr(cond, lhs, rhs, snapshot);
+ }
+ };
+
+ auto bailoutCmpConstant = [&](Assembler::Condition cond, auto lhs,
+ int32_t rhs) {
+ if (type == MIRType::Int32) {
+ bailoutCmp32(cond, lhs, Imm32(rhs), snapshot);
+ } else {
+ MOZ_ASSERT(type == MIRType::IntPtr);
+ bailoutCmpPtr(cond, lhs, ImmWord(rhs), snapshot);
+ }
+ };
+
+ if (lir->index()->isConstant()) {
+ int32_t nmin, nmax;
+ int32_t index = ToInt32(lir->index());
+ if (SafeAdd(index, min, &nmin) && SafeAdd(index, max, &nmax) && nmin >= 0) {
+ if (length->isRegister()) {
+ bailoutCmpConstant(Assembler::BelowOrEqual, ToRegister(length), nmax);
+ } else {
+ bailoutCmpConstant(Assembler::BelowOrEqual, ToAddress(length), nmax);
+ }
+ return;
+ }
+ masm.mov(ImmWord(index), temp);
+ } else {
+ masm.mov(ToRegister(lir->index()), temp);
+ }
+
+ // If the minimum and maximum differ then do an underflow check first.
+ // If the two are the same then doing an unsigned comparison on the
+ // length will also catch a negative index.
+ if (min != max) {
+ if (min != 0) {
+ Label bail;
+ if (type == MIRType::Int32) {
+ masm.branchAdd32(Assembler::Overflow, Imm32(min), temp, &bail);
+ } else {
+ masm.branchAddPtr(Assembler::Overflow, Imm32(min), temp, &bail);
+ }
+ bailoutFrom(&bail, snapshot);
+ }
+
+ bailoutCmpConstant(Assembler::LessThan, temp, 0);
+
+ if (min != 0) {
+ int32_t diff;
+ if (SafeSub(max, min, &diff)) {
+ max = diff;
+ } else {
+ if (type == MIRType::Int32) {
+ masm.sub32(Imm32(min), temp);
+ } else {
+ masm.subPtr(Imm32(min), temp);
+ }
+ }
+ }
+ }
+
+ // Compute the maximum possible index. No overflow check is needed when
+ // max > 0. We can only wraparound to a negative number, which will test as
+ // larger than all nonnegative numbers in the unsigned comparison, and the
+ // length is required to be nonnegative (else testing a negative length
+ // would succeed on any nonnegative index).
+ if (max != 0) {
+ if (max < 0) {
+ Label bail;
+ if (type == MIRType::Int32) {
+ masm.branchAdd32(Assembler::Overflow, Imm32(max), temp, &bail);
+ } else {
+ masm.branchAddPtr(Assembler::Overflow, Imm32(max), temp, &bail);
+ }
+ bailoutFrom(&bail, snapshot);
+ } else {
+ if (type == MIRType::Int32) {
+ masm.add32(Imm32(max), temp);
+ } else {
+ masm.addPtr(Imm32(max), temp);
+ }
+ }
+ }
+
+ if (length->isRegister()) {
+ bailoutCmp(Assembler::BelowOrEqual, ToRegister(length), temp);
+ } else {
+ bailoutCmp(Assembler::BelowOrEqual, ToAddress(length), temp);
+ }
+}
+
+void CodeGenerator::visitBoundsCheckLower(LBoundsCheckLower* lir) {
+ int32_t min = lir->mir()->minimum();
+ bailoutCmp32(Assembler::LessThan, ToRegister(lir->index()), Imm32(min),
+ lir->snapshot());
+}
+
+void CodeGenerator::visitSpectreMaskIndex(LSpectreMaskIndex* lir) {
+ MOZ_ASSERT(JitOptions.spectreIndexMasking);
+
+ const LAllocation* length = lir->length();
+ Register index = ToRegister(lir->index());
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->type() == MIRType::Int32) {
+ if (length->isRegister()) {
+ masm.spectreMaskIndex32(index, ToRegister(length), output);
+ } else {
+ masm.spectreMaskIndex32(index, ToAddress(length), output);
+ }
+ } else {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::IntPtr);
+ if (length->isRegister()) {
+ masm.spectreMaskIndexPtr(index, ToRegister(length), output);
+ } else {
+ masm.spectreMaskIndexPtr(index, ToAddress(length), output);
+ }
+ }
+}
+
+class OutOfLineStoreElementHole : public OutOfLineCodeBase<CodeGenerator> {
+ LInstruction* ins_;
+
+ public:
+ explicit OutOfLineStoreElementHole(LInstruction* ins) : ins_(ins) {
+ MOZ_ASSERT(ins->isStoreElementHoleV() || ins->isStoreElementHoleT());
+ }
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineStoreElementHole(this);
+ }
+
+ MStoreElementHole* mir() const {
+ return ins_->isStoreElementHoleV() ? ins_->toStoreElementHoleV()->mir()
+ : ins_->toStoreElementHoleT()->mir();
+ }
+ LInstruction* ins() const { return ins_; }
+};
+
+void CodeGenerator::emitStoreHoleCheck(Register elements,
+ const LAllocation* index,
+ LSnapshot* snapshot) {
+ Label bail;
+ if (index->isConstant()) {
+ Address dest(elements, ToInt32(index) * sizeof(js::Value));
+ masm.branchTestMagic(Assembler::Equal, dest, &bail);
+ } else {
+ BaseObjectElementIndex dest(elements, ToRegister(index));
+ masm.branchTestMagic(Assembler::Equal, dest, &bail);
+ }
+ bailoutFrom(&bail, snapshot);
+}
+
+void CodeGenerator::emitStoreElementTyped(const LAllocation* value,
+ MIRType valueType, Register elements,
+ const LAllocation* index) {
+ MOZ_ASSERT(valueType != MIRType::MagicHole);
+ ConstantOrRegister v = ToConstantOrRegister(value, valueType);
+ if (index->isConstant()) {
+ Address dest(elements, ToInt32(index) * sizeof(js::Value));
+ masm.storeUnboxedValue(v, valueType, dest);
+ } else {
+ BaseObjectElementIndex dest(elements, ToRegister(index));
+ masm.storeUnboxedValue(v, valueType, dest);
+ }
+}
+
+void CodeGenerator::visitStoreElementT(LStoreElementT* store) {
+ Register elements = ToRegister(store->elements());
+ const LAllocation* index = store->index();
+
+ if (store->mir()->needsBarrier()) {
+ emitPreBarrier(elements, index);
+ }
+
+ if (store->mir()->needsHoleCheck()) {
+ emitStoreHoleCheck(elements, index, store->snapshot());
+ }
+
+ emitStoreElementTyped(store->value(), store->mir()->value()->type(), elements,
+ index);
+}
+
+void CodeGenerator::visitStoreElementV(LStoreElementV* lir) {
+ const ValueOperand value = ToValue(lir, LStoreElementV::Value);
+ Register elements = ToRegister(lir->elements());
+ const LAllocation* index = lir->index();
+
+ if (lir->mir()->needsBarrier()) {
+ emitPreBarrier(elements, index);
+ }
+
+ if (lir->mir()->needsHoleCheck()) {
+ emitStoreHoleCheck(elements, index, lir->snapshot());
+ }
+
+ if (lir->index()->isConstant()) {
+ Address dest(elements, ToInt32(lir->index()) * sizeof(js::Value));
+ masm.storeValue(value, dest);
+ } else {
+ BaseObjectElementIndex dest(elements, ToRegister(lir->index()));
+ masm.storeValue(value, dest);
+ }
+}
+
+void CodeGenerator::visitStoreHoleValueElement(LStoreHoleValueElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register index = ToRegister(lir->index());
+
+ Address elementsFlags(elements, ObjectElements::offsetOfFlags());
+ masm.or32(Imm32(ObjectElements::NON_PACKED), elementsFlags);
+
+ BaseObjectElementIndex element(elements, index);
+ masm.storeValue(MagicValue(JS_ELEMENTS_HOLE), element);
+}
+
+void CodeGenerator::visitStoreElementHoleT(LStoreElementHoleT* lir) {
+ auto* ool = new (alloc()) OutOfLineStoreElementHole(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ Register elements = ToRegister(lir->elements());
+ Register index = ToRegister(lir->index());
+ Register temp = ToRegister(lir->temp0());
+
+ Address initLength(elements, ObjectElements::offsetOfInitializedLength());
+ masm.spectreBoundsCheck32(index, initLength, temp, ool->entry());
+
+ emitPreBarrier(elements, lir->index());
+
+ masm.bind(ool->rejoin());
+ emitStoreElementTyped(lir->value(), lir->mir()->value()->type(), elements,
+ lir->index());
+}
+
+void CodeGenerator::visitStoreElementHoleV(LStoreElementHoleV* lir) {
+ auto* ool = new (alloc()) OutOfLineStoreElementHole(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ Register elements = ToRegister(lir->elements());
+ Register index = ToRegister(lir->index());
+ const ValueOperand value = ToValue(lir, LStoreElementHoleV::ValueIndex);
+ Register temp = ToRegister(lir->temp0());
+
+ Address initLength(elements, ObjectElements::offsetOfInitializedLength());
+ masm.spectreBoundsCheck32(index, initLength, temp, ool->entry());
+
+ emitPreBarrier(elements, lir->index());
+
+ masm.bind(ool->rejoin());
+ masm.storeValue(value, BaseObjectElementIndex(elements, index));
+}
+
+void CodeGenerator::visitOutOfLineStoreElementHole(
+ OutOfLineStoreElementHole* ool) {
+ Register object, elements, index;
+ LInstruction* ins = ool->ins();
+ mozilla::Maybe<ConstantOrRegister> value;
+ Register temp;
+
+ if (ins->isStoreElementHoleV()) {
+ LStoreElementHoleV* store = ins->toStoreElementHoleV();
+ object = ToRegister(store->object());
+ elements = ToRegister(store->elements());
+ index = ToRegister(store->index());
+ value.emplace(
+ TypedOrValueRegister(ToValue(store, LStoreElementHoleV::ValueIndex)));
+ temp = ToRegister(store->temp0());
+ } else {
+ LStoreElementHoleT* store = ins->toStoreElementHoleT();
+ object = ToRegister(store->object());
+ elements = ToRegister(store->elements());
+ index = ToRegister(store->index());
+ if (store->value()->isConstant()) {
+ value.emplace(
+ ConstantOrRegister(store->value()->toConstant()->toJSValue()));
+ } else {
+ MIRType valueType = store->mir()->value()->type();
+ value.emplace(
+ TypedOrValueRegister(valueType, ToAnyRegister(store->value())));
+ }
+ temp = ToRegister(store->temp0());
+ }
+
+ Address initLength(elements, ObjectElements::offsetOfInitializedLength());
+
+ // We're out-of-bounds. We only handle the index == initlength case.
+ // If index > initializedLength, bail out. Note that this relies on the
+ // condition flags sticking from the incoming branch.
+ // Also note: this branch does not need Spectre mitigations, doing that for
+ // the capacity check below is sufficient.
+ Label allocElement, addNewElement;
+#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_LOONG64)
+ // Had to reimplement for MIPS because there are no flags.
+ bailoutCmp32(Assembler::NotEqual, initLength, index, ins->snapshot());
+#else
+ bailoutIf(Assembler::NotEqual, ins->snapshot());
+#endif
+
+ // If index < capacity, we can add a dense element inline. If not, we need
+ // to allocate more elements first.
+ masm.spectreBoundsCheck32(
+ index, Address(elements, ObjectElements::offsetOfCapacity()), temp,
+ &allocElement);
+ masm.jump(&addNewElement);
+
+ masm.bind(&allocElement);
+
+ // Save all live volatile registers, except |temp|.
+ LiveRegisterSet liveRegs = liveVolatileRegs(ins);
+ liveRegs.takeUnchecked(temp);
+ masm.PushRegsInMask(liveRegs);
+
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp);
+ masm.passABIArg(temp);
+ masm.passABIArg(object);
+
+ using Fn = bool (*)(JSContext*, NativeObject*);
+ masm.callWithABI<Fn, NativeObject::addDenseElementPure>();
+ masm.storeCallPointerResult(temp);
+
+ masm.PopRegsInMask(liveRegs);
+ bailoutIfFalseBool(temp, ins->snapshot());
+
+ // Load the reallocated elements pointer.
+ masm.loadPtr(Address(object, NativeObject::offsetOfElements()), elements);
+
+ masm.bind(&addNewElement);
+
+ // Increment initLength
+ masm.add32(Imm32(1), initLength);
+
+ // If length is now <= index, increment length too.
+ Label skipIncrementLength;
+ Address length(elements, ObjectElements::offsetOfLength());
+ masm.branch32(Assembler::Above, length, index, &skipIncrementLength);
+ masm.add32(Imm32(1), length);
+ masm.bind(&skipIncrementLength);
+
+ // Jump to the inline path where we will store the value.
+ // We rejoin after the prebarrier, because the memory is uninitialized.
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitArrayPopShift(LArrayPopShift* lir) {
+ Register obj = ToRegister(lir->object());
+ Register temp1 = ToRegister(lir->temp0());
+ Register temp2 = ToRegister(lir->temp1());
+ ValueOperand out = ToOutValue(lir);
+
+ Label bail;
+ if (lir->mir()->mode() == MArrayPopShift::Pop) {
+ masm.packedArrayPop(obj, out, temp1, temp2, &bail);
+ } else {
+ MOZ_ASSERT(lir->mir()->mode() == MArrayPopShift::Shift);
+ LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
+ masm.packedArrayShift(obj, out, temp1, temp2, volatileRegs, &bail);
+ }
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+class OutOfLineArrayPush : public OutOfLineCodeBase<CodeGenerator> {
+ LArrayPush* ins_;
+
+ public:
+ explicit OutOfLineArrayPush(LArrayPush* ins) : ins_(ins) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineArrayPush(this);
+ }
+
+ LArrayPush* ins() const { return ins_; }
+};
+
+void CodeGenerator::visitArrayPush(LArrayPush* lir) {
+ Register obj = ToRegister(lir->object());
+ Register elementsTemp = ToRegister(lir->temp0());
+ Register length = ToRegister(lir->output());
+ ValueOperand value = ToValue(lir, LArrayPush::ValueIndex);
+ Register spectreTemp = ToTempRegisterOrInvalid(lir->temp1());
+
+ auto* ool = new (alloc()) OutOfLineArrayPush(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ // Load elements and length.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfElements()), elementsTemp);
+ masm.load32(Address(elementsTemp, ObjectElements::offsetOfLength()), length);
+
+ // TODO(post-Warp): reuse/share the CacheIR implementation when IonBuilder and
+ // TI are gone (bug 1654180).
+
+ // Bailout if the incremented length does not fit in int32.
+ bailoutCmp32(Assembler::AboveOrEqual, length, Imm32(INT32_MAX),
+ lir->snapshot());
+
+ // Guard length == initializedLength.
+ Address initLength(elementsTemp, ObjectElements::offsetOfInitializedLength());
+ masm.branch32(Assembler::NotEqual, initLength, length, ool->entry());
+
+ // Guard length < capacity.
+ Address capacity(elementsTemp, ObjectElements::offsetOfCapacity());
+ masm.spectreBoundsCheck32(length, capacity, spectreTemp, ool->entry());
+
+ // Do the store.
+ masm.storeValue(value, BaseObjectElementIndex(elementsTemp, length));
+
+ masm.add32(Imm32(1), length);
+
+ // Update length and initialized length.
+ masm.store32(length, Address(elementsTemp, ObjectElements::offsetOfLength()));
+ masm.store32(length, Address(elementsTemp,
+ ObjectElements::offsetOfInitializedLength()));
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineArrayPush(OutOfLineArrayPush* ool) {
+ LArrayPush* ins = ool->ins();
+
+ Register object = ToRegister(ins->object());
+ Register temp = ToRegister(ins->temp0());
+ Register output = ToRegister(ins->output());
+ ValueOperand value = ToValue(ins, LArrayPush::ValueIndex);
+
+ // Save all live volatile registers, except |temp| and |output|, because both
+ // are overwritten anyway.
+ LiveRegisterSet liveRegs = liveVolatileRegs(ins);
+ liveRegs.takeUnchecked(temp);
+ liveRegs.takeUnchecked(output);
+
+ masm.PushRegsInMask(liveRegs);
+
+ masm.Push(value);
+ masm.moveStackPtrTo(output);
+
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp);
+ masm.passABIArg(temp);
+ masm.passABIArg(object);
+ masm.passABIArg(output);
+
+ using Fn = bool (*)(JSContext*, ArrayObject*, Value*);
+ masm.callWithABI<Fn, jit::ArrayPushDensePure>();
+ masm.storeCallPointerResult(temp);
+
+ masm.freeStack(sizeof(Value)); // Discard pushed Value.
+
+ MOZ_ASSERT(!liveRegs.has(temp));
+ masm.PopRegsInMask(liveRegs);
+
+ bailoutIfFalseBool(temp, ins->snapshot());
+
+ // Load the new length into the output register.
+ masm.loadPtr(Address(object, NativeObject::offsetOfElements()), output);
+ masm.load32(Address(output, ObjectElements::offsetOfLength()), output);
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitArraySlice(LArraySlice* lir) {
+ Register object = ToRegister(lir->object());
+ Register begin = ToRegister(lir->begin());
+ Register end = ToRegister(lir->end());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ Label call, fail;
+
+ Label bail;
+ masm.branchArrayIsNotPacked(object, temp0, temp1, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+
+ // Try to allocate an object.
+ TemplateObject templateObject(lir->mir()->templateObj());
+ masm.createGCObject(temp0, temp1, templateObject, lir->mir()->initialHeap(),
+ &fail);
+
+ masm.jump(&call);
+ {
+ masm.bind(&fail);
+ masm.movePtr(ImmPtr(nullptr), temp0);
+ }
+ masm.bind(&call);
+
+ pushArg(temp0);
+ pushArg(end);
+ pushArg(begin);
+ pushArg(object);
+
+ using Fn =
+ JSObject* (*)(JSContext*, HandleObject, int32_t, int32_t, HandleObject);
+ callVM<Fn, ArraySliceDense>(lir);
+}
+
+void CodeGenerator::visitArgumentsSlice(LArgumentsSlice* lir) {
+ Register object = ToRegister(lir->object());
+ Register begin = ToRegister(lir->begin());
+ Register end = ToRegister(lir->end());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ Label call, fail;
+
+ // Try to allocate an object.
+ TemplateObject templateObject(lir->mir()->templateObj());
+ masm.createGCObject(temp0, temp1, templateObject, lir->mir()->initialHeap(),
+ &fail);
+
+ masm.jump(&call);
+ {
+ masm.bind(&fail);
+ masm.movePtr(ImmPtr(nullptr), temp0);
+ }
+ masm.bind(&call);
+
+ pushArg(temp0);
+ pushArg(end);
+ pushArg(begin);
+ pushArg(object);
+
+ using Fn =
+ JSObject* (*)(JSContext*, HandleObject, int32_t, int32_t, HandleObject);
+ callVM<Fn, ArgumentsSliceDense>(lir);
+}
+
+#ifdef DEBUG
+void CodeGenerator::emitAssertArgumentsSliceBounds(const RegisterOrInt32& begin,
+ const RegisterOrInt32& count,
+ Register numActualArgs) {
+ // |begin| must be positive or zero.
+ if (begin.is<Register>()) {
+ Label beginOk;
+ masm.branch32(Assembler::GreaterThanOrEqual, begin.as<Register>(), Imm32(0),
+ &beginOk);
+ masm.assumeUnreachable("begin < 0");
+ masm.bind(&beginOk);
+ } else {
+ MOZ_ASSERT(begin.as<int32_t>() >= 0);
+ }
+
+ // |count| must be positive or zero.
+ if (count.is<Register>()) {
+ Label countOk;
+ masm.branch32(Assembler::GreaterThanOrEqual, count.as<Register>(), Imm32(0),
+ &countOk);
+ masm.assumeUnreachable("count < 0");
+ masm.bind(&countOk);
+ } else {
+ MOZ_ASSERT(count.as<int32_t>() >= 0);
+ }
+
+ // |begin| must be less-or-equal to |numActualArgs|.
+ Label argsBeginOk;
+ if (begin.is<Register>()) {
+ masm.branchPtr(Assembler::AboveOrEqual, numActualArgs, begin.as<Register>(),
+ &argsBeginOk);
+ } else {
+ masm.branchPtr(Assembler::AboveOrEqual, numActualArgs,
+ Imm32(begin.as<int32_t>()), &argsBeginOk);
+ }
+ masm.assumeUnreachable("begin <= numActualArgs");
+ masm.bind(&argsBeginOk);
+
+ // |count| must be less-or-equal to |numActualArgs|.
+ Label argsCountOk;
+ if (count.is<Register>()) {
+ masm.branchPtr(Assembler::AboveOrEqual, numActualArgs, count.as<Register>(),
+ &argsCountOk);
+ } else {
+ masm.branchPtr(Assembler::AboveOrEqual, numActualArgs,
+ Imm32(count.as<int32_t>()), &argsCountOk);
+ }
+ masm.assumeUnreachable("count <= numActualArgs");
+ masm.bind(&argsCountOk);
+
+ // |begin| and |count| must be preserved, but |numActualArgs| can be changed.
+ //
+ // Pre-condition: |count| <= |numActualArgs|
+ // Condition to test: |begin + count| <= |numActualArgs|
+ // Transform to: |begin| <= |numActualArgs - count|
+ if (count.is<Register>()) {
+ masm.subPtr(count.as<Register>(), numActualArgs);
+ } else {
+ masm.subPtr(Imm32(count.as<int32_t>()), numActualArgs);
+ }
+
+ // |begin + count| must be less-or-equal to |numActualArgs|.
+ Label argsBeginCountOk;
+ if (begin.is<Register>()) {
+ masm.branchPtr(Assembler::AboveOrEqual, numActualArgs, begin.as<Register>(),
+ &argsBeginCountOk);
+ } else {
+ masm.branchPtr(Assembler::AboveOrEqual, numActualArgs,
+ Imm32(begin.as<int32_t>()), &argsBeginCountOk);
+ }
+ masm.assumeUnreachable("begin + count <= numActualArgs");
+ masm.bind(&argsBeginCountOk);
+}
+#endif
+
+template <class ArgumentsSlice>
+void CodeGenerator::emitNewArray(ArgumentsSlice* lir,
+ const RegisterOrInt32& count, Register output,
+ Register temp) {
+ using Fn = ArrayObject* (*)(JSContext*, int32_t);
+ auto* ool = count.match(
+ [&](Register count) {
+ return oolCallVM<Fn, NewArrayObjectEnsureDenseInitLength>(
+ lir, ArgList(count), StoreRegisterTo(output));
+ },
+ [&](int32_t count) {
+ return oolCallVM<Fn, NewArrayObjectEnsureDenseInitLength>(
+ lir, ArgList(Imm32(count)), StoreRegisterTo(output));
+ });
+
+ TemplateObject templateObject(lir->mir()->templateObj());
+ MOZ_ASSERT(templateObject.isArrayObject());
+
+ auto templateNativeObj = templateObject.asTemplateNativeObject();
+ MOZ_ASSERT(templateNativeObj.getArrayLength() == 0);
+ MOZ_ASSERT(templateNativeObj.getDenseInitializedLength() == 0);
+ MOZ_ASSERT(!templateNativeObj.hasDynamicElements());
+
+ // Check array capacity. Call into the VM if the template object's capacity
+ // is too small.
+ bool tryAllocate = count.match(
+ [&](Register count) {
+ masm.branch32(Assembler::Above, count,
+ Imm32(templateNativeObj.getDenseCapacity()),
+ ool->entry());
+ return true;
+ },
+ [&](int32_t count) {
+ MOZ_ASSERT(count >= 0);
+ if (uint32_t(count) > templateNativeObj.getDenseCapacity()) {
+ masm.jump(ool->entry());
+ return false;
+ }
+ return true;
+ });
+
+ if (tryAllocate) {
+ // Try to allocate an object.
+ masm.createGCObject(output, temp, templateObject, lir->mir()->initialHeap(),
+ ool->entry());
+
+ auto setInitializedLengthAndLength = [&](auto count) {
+ const int elementsOffset = NativeObject::offsetOfFixedElements();
+
+ // Update initialized length.
+ Address initLength(
+ output, elementsOffset + ObjectElements::offsetOfInitializedLength());
+ masm.store32(count, initLength);
+
+ // Update length.
+ Address length(output, elementsOffset + ObjectElements::offsetOfLength());
+ masm.store32(count, length);
+ };
+
+ // The array object was successfully created. Set the length and initialized
+ // length and then proceed to fill the elements.
+ count.match([&](Register count) { setInitializedLengthAndLength(count); },
+ [&](int32_t count) {
+ if (count > 0) {
+ setInitializedLengthAndLength(Imm32(count));
+ }
+ });
+ }
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitFrameArgumentsSlice(LFrameArgumentsSlice* lir) {
+ Register begin = ToRegister(lir->begin());
+ Register count = ToRegister(lir->count());
+ Register temp = ToRegister(lir->temp0());
+ Register output = ToRegister(lir->output());
+
+#ifdef DEBUG
+ masm.loadNumActualArgs(FramePointer, temp);
+ emitAssertArgumentsSliceBounds(RegisterOrInt32(begin), RegisterOrInt32(count),
+ temp);
+#endif
+
+ emitNewArray(lir, RegisterOrInt32(count), output, temp);
+
+ Label done;
+ masm.branch32(Assembler::Equal, count, Imm32(0), &done);
+ {
+ AllocatableGeneralRegisterSet allRegs(GeneralRegisterSet::All());
+ allRegs.take(begin);
+ allRegs.take(count);
+ allRegs.take(temp);
+ allRegs.take(output);
+
+ ValueOperand value = allRegs.takeAnyValue();
+
+ LiveRegisterSet liveRegs;
+ liveRegs.add(output);
+ liveRegs.add(begin);
+ liveRegs.add(value);
+
+ masm.PushRegsInMask(liveRegs);
+
+ // Initialize all elements.
+
+ Register elements = output;
+ masm.loadPtr(Address(output, NativeObject::offsetOfElements()), elements);
+
+ Register argIndex = begin;
+
+ Register index = temp;
+ masm.move32(Imm32(0), index);
+
+ size_t argvOffset = JitFrameLayout::offsetOfActualArgs();
+ BaseValueIndex argPtr(FramePointer, argIndex, argvOffset);
+
+ Label loop;
+ masm.bind(&loop);
+
+ masm.loadValue(argPtr, value);
+
+ // We don't need a pre-barrier, because the element at |index| is guaranteed
+ // to be a non-GC thing (either uninitialized memory or the magic hole
+ // value).
+ masm.storeValue(value, BaseObjectElementIndex(elements, index));
+
+ masm.add32(Imm32(1), index);
+ masm.add32(Imm32(1), argIndex);
+
+ masm.branch32(Assembler::LessThan, index, count, &loop);
+
+ masm.PopRegsInMask(liveRegs);
+
+ // Emit a post-write barrier if |output| is tenured.
+ //
+ // We expect that |output| is nursery allocated, so it isn't worth the
+ // trouble to check if no frame argument is a nursery thing, which would
+ // allow to omit the post-write barrier.
+ masm.branchPtrInNurseryChunk(Assembler::Equal, output, temp, &done);
+
+ LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
+ volatileRegs.takeUnchecked(temp);
+ if (output.volatile_()) {
+ volatileRegs.addUnchecked(output);
+ }
+
+ masm.PushRegsInMask(volatileRegs);
+ emitPostWriteBarrier(output);
+ masm.PopRegsInMask(volatileRegs);
+ }
+ masm.bind(&done);
+}
+
+CodeGenerator::RegisterOrInt32 CodeGenerator::ToRegisterOrInt32(
+ const LAllocation* allocation) {
+ if (allocation->isConstant()) {
+ return RegisterOrInt32(allocation->toConstant()->toInt32());
+ }
+ return RegisterOrInt32(ToRegister(allocation));
+}
+
+void CodeGenerator::visitInlineArgumentsSlice(LInlineArgumentsSlice* lir) {
+ RegisterOrInt32 begin = ToRegisterOrInt32(lir->begin());
+ RegisterOrInt32 count = ToRegisterOrInt32(lir->count());
+ Register temp = ToRegister(lir->temp());
+ Register output = ToRegister(lir->output());
+
+ uint32_t numActuals = lir->mir()->numActuals();
+
+#ifdef DEBUG
+ masm.move32(Imm32(numActuals), temp);
+
+ emitAssertArgumentsSliceBounds(begin, count, temp);
+#endif
+
+ emitNewArray(lir, count, output, temp);
+
+ // We're done if there are no actual arguments.
+ if (numActuals == 0) {
+ return;
+ }
+
+ // Check if any arguments have to be copied.
+ Label done;
+ if (count.is<Register>()) {
+ masm.branch32(Assembler::Equal, count.as<Register>(), Imm32(0), &done);
+ } else if (count.as<int32_t>() == 0) {
+ return;
+ }
+
+ auto getArg = [&](uint32_t i) {
+ return toConstantOrRegister(lir, LInlineArgumentsSlice::ArgIndex(i),
+ lir->mir()->getArg(i)->type());
+ };
+
+ auto storeArg = [&](uint32_t i, auto dest) {
+ // We don't need a pre-barrier because the element at |index| is guaranteed
+ // to be a non-GC thing (either uninitialized memory or the magic hole
+ // value).
+ masm.storeConstantOrRegister(getArg(i), dest);
+ };
+
+ // Initialize all elements.
+ if (numActuals == 1) {
+ // There's exactly one argument. We've checked that |count| is non-zero,
+ // which implies that |begin| must be zero.
+ MOZ_ASSERT_IF(begin.is<int32_t>(), begin.as<int32_t>() == 0);
+
+ Register elements = temp;
+ masm.loadPtr(Address(output, NativeObject::offsetOfElements()), elements);
+
+ storeArg(0, Address(elements, 0));
+ } else if (begin.is<Register>()) {
+ // There is more than one argument and |begin| isn't a compile-time
+ // constant. Iterate through 0..numActuals to search for |begin| and then
+ // start copying |count| arguments from that index.
+
+ LiveGeneralRegisterSet liveRegs;
+ liveRegs.add(output);
+ liveRegs.add(begin.as<Register>());
+
+ masm.PushRegsInMask(liveRegs);
+
+ Register elements = output;
+ masm.loadPtr(Address(output, NativeObject::offsetOfElements()), elements);
+
+ Register argIndex = begin.as<Register>();
+
+ Register index = temp;
+ masm.move32(Imm32(0), index);
+
+ Label doneLoop;
+ for (uint32_t i = 0; i < numActuals; ++i) {
+ Label next;
+ masm.branch32(Assembler::NotEqual, argIndex, Imm32(i), &next);
+
+ storeArg(i, BaseObjectElementIndex(elements, index));
+
+ masm.add32(Imm32(1), index);
+ masm.add32(Imm32(1), argIndex);
+
+ if (count.is<Register>()) {
+ masm.branch32(Assembler::GreaterThanOrEqual, index,
+ count.as<Register>(), &doneLoop);
+ } else {
+ masm.branch32(Assembler::GreaterThanOrEqual, index,
+ Imm32(count.as<int32_t>()), &doneLoop);
+ }
+
+ masm.bind(&next);
+ }
+ masm.bind(&doneLoop);
+
+ masm.PopRegsInMask(liveRegs);
+ } else {
+ // There is more than one argument and |begin| is a compile-time constant.
+
+ Register elements = temp;
+ masm.loadPtr(Address(output, NativeObject::offsetOfElements()), elements);
+
+ int32_t argIndex = begin.as<int32_t>();
+
+ int32_t index = 0;
+
+ Label doneLoop;
+ for (uint32_t i = argIndex; i < numActuals; ++i) {
+ storeArg(i, Address(elements, index * sizeof(Value)));
+
+ index += 1;
+
+ if (count.is<Register>()) {
+ masm.branch32(Assembler::LessThanOrEqual, count.as<Register>(),
+ Imm32(index), &doneLoop);
+ } else {
+ if (index >= count.as<int32_t>()) {
+ break;
+ }
+ }
+ }
+ masm.bind(&doneLoop);
+ }
+
+ // Determine if we have to emit post-write barrier.
+ //
+ // If either |begin| or |count| is a constant, use their value directly.
+ // Otherwise assume we copy all inline arguments from 0..numActuals.
+ bool postWriteBarrier = false;
+ uint32_t actualBegin = begin.match([](Register) { return 0; },
+ [](int32_t value) { return value; });
+ uint32_t actualCount =
+ count.match([=](Register) { return numActuals; },
+ [](int32_t value) -> uint32_t { return value; });
+ for (uint32_t i = 0; i < actualCount; ++i) {
+ ConstantOrRegister arg = getArg(actualBegin + i);
+ if (arg.constant()) {
+ Value v = arg.value();
+ if (v.isGCThing() && IsInsideNursery(v.toGCThing())) {
+ postWriteBarrier = true;
+ }
+ } else {
+ MIRType type = arg.reg().type();
+ if (type == MIRType::Value || NeedsPostBarrier(type)) {
+ postWriteBarrier = true;
+ }
+ }
+ }
+
+ // Emit a post-write barrier if |output| is tenured and we couldn't
+ // determine at compile-time that no barrier is needed.
+ if (postWriteBarrier) {
+ masm.branchPtrInNurseryChunk(Assembler::Equal, output, temp, &done);
+
+ LiveRegisterSet volatileRegs = liveVolatileRegs(lir);
+ volatileRegs.takeUnchecked(temp);
+ if (output.volatile_()) {
+ volatileRegs.addUnchecked(output);
+ }
+
+ masm.PushRegsInMask(volatileRegs);
+ emitPostWriteBarrier(output);
+ masm.PopRegsInMask(volatileRegs);
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitNormalizeSliceTerm(LNormalizeSliceTerm* lir) {
+ Register value = ToRegister(lir->value());
+ Register length = ToRegister(lir->length());
+ Register output = ToRegister(lir->output());
+
+ masm.move32(value, output);
+
+ Label positive;
+ masm.branch32(Assembler::GreaterThanOrEqual, value, Imm32(0), &positive);
+
+ Label done;
+ masm.add32(length, output);
+ masm.branch32(Assembler::GreaterThanOrEqual, output, Imm32(0), &done);
+ masm.move32(Imm32(0), output);
+ masm.jump(&done);
+
+ masm.bind(&positive);
+ masm.cmp32Move32(Assembler::LessThan, length, value, length, output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitArrayJoin(LArrayJoin* lir) {
+ Label skipCall;
+
+ Register output = ToRegister(lir->output());
+ Register sep = ToRegister(lir->separator());
+ Register array = ToRegister(lir->array());
+ Register temp = ToRegister(lir->temp0());
+
+ // Fast path for simple length <= 1 cases.
+ {
+ masm.loadPtr(Address(array, NativeObject::offsetOfElements()), temp);
+ Address length(temp, ObjectElements::offsetOfLength());
+ Address initLength(temp, ObjectElements::offsetOfInitializedLength());
+
+ // Check for length == 0
+ Label notEmpty;
+ masm.branch32(Assembler::NotEqual, length, Imm32(0), &notEmpty);
+ const JSAtomState& names = gen->runtime->names();
+ masm.movePtr(ImmGCPtr(names.empty), output);
+ masm.jump(&skipCall);
+
+ masm.bind(&notEmpty);
+ Label notSingleString;
+ // Check for length == 1, initializedLength >= 1, arr[0].isString()
+ masm.branch32(Assembler::NotEqual, length, Imm32(1), &notSingleString);
+ masm.branch32(Assembler::LessThan, initLength, Imm32(1), &notSingleString);
+
+ Address elem0(temp, 0);
+ masm.branchTestString(Assembler::NotEqual, elem0, &notSingleString);
+
+ // At this point, 'output' can be used as a scratch register, since we're
+ // guaranteed to succeed.
+ masm.unboxString(elem0, output);
+ masm.jump(&skipCall);
+ masm.bind(&notSingleString);
+ }
+
+ pushArg(sep);
+ pushArg(array);
+
+ using Fn = JSString* (*)(JSContext*, HandleObject, HandleString);
+ callVM<Fn, jit::ArrayJoin>(lir);
+ masm.bind(&skipCall);
+}
+
+void CodeGenerator::visitGetIteratorCache(LGetIteratorCache* lir) {
+ LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
+ TypedOrValueRegister val =
+ toConstantOrRegister(lir, LGetIteratorCache::ValueIndex,
+ lir->mir()->value()->type())
+ .reg();
+ Register output = ToRegister(lir->output());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ IonGetIteratorIC ic(liveRegs, val, output, temp0, temp1);
+ addIC(lir, allocateIC(ic));
+}
+
+void CodeGenerator::visitOptimizeSpreadCallCache(
+ LOptimizeSpreadCallCache* lir) {
+ LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
+ ValueOperand val = ToValue(lir, LOptimizeSpreadCallCache::ValueIndex);
+ ValueOperand output = ToOutValue(lir);
+ Register temp = ToRegister(lir->temp0());
+
+ IonOptimizeSpreadCallIC ic(liveRegs, val, output, temp);
+ addIC(lir, allocateIC(ic));
+}
+
+void CodeGenerator::visitCloseIterCache(LCloseIterCache* lir) {
+ LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
+ Register iter = ToRegister(lir->iter());
+ Register temp = ToRegister(lir->temp0());
+ CompletionKind kind = CompletionKind(lir->mir()->completionKind());
+
+ IonCloseIterIC ic(liveRegs, iter, temp, kind);
+ addIC(lir, allocateIC(ic));
+}
+
+void CodeGenerator::visitIteratorMore(LIteratorMore* lir) {
+ const Register obj = ToRegister(lir->iterator());
+ const ValueOperand output = ToOutValue(lir);
+ const Register temp = ToRegister(lir->temp0());
+
+ masm.iteratorMore(obj, output, temp);
+}
+
+void CodeGenerator::visitIsNoIterAndBranch(LIsNoIterAndBranch* lir) {
+ ValueOperand input = ToValue(lir, LIsNoIterAndBranch::Input);
+ Label* ifTrue = getJumpLabelForBranch(lir->ifTrue());
+ Label* ifFalse = getJumpLabelForBranch(lir->ifFalse());
+
+ masm.branchTestMagic(Assembler::Equal, input, ifTrue);
+
+ if (!isNextBlock(lir->ifFalse()->lir())) {
+ masm.jump(ifFalse);
+ }
+}
+
+void CodeGenerator::visitIteratorEnd(LIteratorEnd* lir) {
+ const Register obj = ToRegister(lir->object());
+ const Register temp0 = ToRegister(lir->temp0());
+ const Register temp1 = ToRegister(lir->temp1());
+ const Register temp2 = ToRegister(lir->temp2());
+
+ masm.iteratorClose(obj, temp0, temp1, temp2);
+}
+
+void CodeGenerator::visitArgumentsLength(LArgumentsLength* lir) {
+ // read number of actual arguments from the JS frame.
+ Register argc = ToRegister(lir->output());
+ masm.loadNumActualArgs(FramePointer, argc);
+}
+
+void CodeGenerator::visitGetFrameArgument(LGetFrameArgument* lir) {
+ ValueOperand result = ToOutValue(lir);
+ const LAllocation* index = lir->index();
+ size_t argvOffset = JitFrameLayout::offsetOfActualArgs();
+
+ if (index->isConstant()) {
+ int32_t i = index->toConstant()->toInt32();
+ Address argPtr(FramePointer, sizeof(Value) * i + argvOffset);
+ masm.loadValue(argPtr, result);
+ } else {
+ Register i = ToRegister(index);
+ BaseValueIndex argPtr(FramePointer, i, argvOffset);
+ masm.loadValue(argPtr, result);
+ }
+}
+
+void CodeGenerator::visitGetFrameArgumentHole(LGetFrameArgumentHole* lir) {
+ ValueOperand result = ToOutValue(lir);
+ Register index = ToRegister(lir->index());
+ Register length = ToRegister(lir->length());
+ Register spectreTemp = ToTempRegisterOrInvalid(lir->temp0());
+ size_t argvOffset = JitFrameLayout::offsetOfActualArgs();
+
+ Label outOfBounds, done;
+ masm.spectreBoundsCheck32(index, length, spectreTemp, &outOfBounds);
+
+ BaseValueIndex argPtr(FramePointer, index, argvOffset);
+ masm.loadValue(argPtr, result);
+ masm.jump(&done);
+
+ masm.bind(&outOfBounds);
+ bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
+ masm.moveValue(UndefinedValue(), result);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitRest(LRest* lir) {
+ Register numActuals = ToRegister(lir->numActuals());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+ unsigned numFormals = lir->mir()->numFormals();
+
+ if (Shape* shape = lir->mir()->shape()) {
+ uint32_t arrayLength = 0;
+ uint32_t arrayCapacity = 2;
+ gc::AllocKind allocKind = GuessArrayGCKind(arrayCapacity);
+ MOZ_ASSERT(CanChangeToBackgroundAllocKind(allocKind, &ArrayObject::class_));
+ allocKind = ForegroundToBackgroundAllocKind(allocKind);
+ MOZ_ASSERT(GetGCKindSlots(allocKind) ==
+ arrayCapacity + ObjectElements::VALUES_PER_HEADER);
+
+ Label joinAlloc, failAlloc;
+ masm.movePtr(ImmGCPtr(shape), temp0);
+ masm.createArrayWithFixedElements(temp2, temp0, temp1, arrayLength,
+ arrayCapacity, allocKind, gc::DefaultHeap,
+ &failAlloc);
+ masm.jump(&joinAlloc);
+ {
+ masm.bind(&failAlloc);
+ masm.movePtr(ImmPtr(nullptr), temp2);
+ }
+ masm.bind(&joinAlloc);
+ } else {
+ masm.movePtr(ImmPtr(nullptr), temp2);
+ }
+
+ // Set temp1 to the address of the first actual argument.
+ size_t actualsOffset = JitFrameLayout::offsetOfActualArgs();
+ masm.computeEffectiveAddress(Address(FramePointer, actualsOffset), temp1);
+
+ // Compute array length: max(numActuals - numFormals, 0).
+ Register lengthReg;
+ if (numFormals) {
+ lengthReg = temp0;
+ Label emptyLength, joinLength;
+ masm.branch32(Assembler::LessThanOrEqual, numActuals, Imm32(numFormals),
+ &emptyLength);
+ {
+ masm.move32(numActuals, lengthReg);
+ masm.sub32(Imm32(numFormals), lengthReg);
+
+ // Skip formal arguments.
+ masm.addPtr(Imm32(sizeof(Value) * numFormals), temp1);
+
+ masm.jump(&joinLength);
+ }
+ masm.bind(&emptyLength);
+ {
+ masm.move32(Imm32(0), lengthReg);
+
+ // Leave temp1 pointed to the start of actuals() when the rest-array
+ // length is zero. We don't use |actuals() + numFormals| because
+ // |numFormals| can be any non-negative int32 value when this MRest was
+ // created from scalar replacement optimizations. And it seems
+ // questionable to compute a Value* pointer which points to who knows
+ // where.
+ }
+ masm.bind(&joinLength);
+ } else {
+ // Use numActuals directly when there are no formals.
+ lengthReg = numActuals;
+ }
+
+ pushArg(temp2);
+ pushArg(temp1);
+ pushArg(lengthReg);
+
+ using Fn = JSObject* (*)(JSContext*, uint32_t, Value*, HandleObject);
+ callVM<Fn, InitRestParameter>(lir);
+}
+
+// Create a stackmap from the given safepoint, with the structure:
+//
+// <reg dump area, if trap>
+// | ++ <body (general spill)>
+// | ++ <space for Frame>
+// | ++ <inbound args>
+// | |
+// Lowest Addr Highest Addr
+//
+// The caller owns the resulting stackmap. This assumes a grow-down stack.
+//
+// For non-debug builds, if the stackmap would contain no pointers, no
+// stackmap is created, and nullptr is returned. For a debug build, a
+// stackmap is always created and returned.
+static bool CreateStackMapFromLSafepoint(LSafepoint& safepoint,
+ const RegisterOffsets& trapExitLayout,
+ size_t trapExitLayoutNumWords,
+ size_t nInboundStackArgBytes,
+ wasm::StackMap** result) {
+ // Ensure this is defined on all return paths.
+ *result = nullptr;
+
+ // The size of the wasm::Frame itself.
+ const size_t nFrameBytes = sizeof(wasm::Frame);
+
+ // This is the number of bytes in the general spill area, below the Frame.
+ const size_t nBodyBytes = safepoint.framePushedAtStackMapBase();
+
+ // This is the number of bytes in the general spill area, the Frame, and the
+ // incoming args, but not including any trap (register dump) area.
+ const size_t nNonTrapBytes = nBodyBytes + nFrameBytes + nInboundStackArgBytes;
+ MOZ_ASSERT(nNonTrapBytes % sizeof(void*) == 0);
+
+ // This is the total number of bytes covered by the map.
+ const DebugOnly<size_t> nTotalBytes =
+ nNonTrapBytes +
+ (safepoint.isWasmTrap() ? (trapExitLayoutNumWords * sizeof(void*)) : 0);
+
+ // Create the stackmap initially in this vector. Since most frames will
+ // contain 128 or fewer words, heap allocation is avoided in the majority of
+ // cases. vec[0] is for the lowest address in the map, vec[N-1] is for the
+ // highest address in the map.
+ wasm::StackMapBoolVector vec;
+
+ // Keep track of whether we've actually seen any refs.
+ bool hasRefs = false;
+
+ // REG DUMP AREA, if any.
+ const LiveGeneralRegisterSet gcRegs = safepoint.gcRegs();
+ GeneralRegisterForwardIterator gcRegsIter(gcRegs);
+ if (safepoint.isWasmTrap()) {
+ // Deal with roots in registers. This can only happen for safepoints
+ // associated with a trap. For safepoints associated with a call, we
+ // don't expect to have any live values in registers, hence no roots in
+ // registers.
+ if (!vec.appendN(false, trapExitLayoutNumWords)) {
+ return false;
+ }
+ for (; gcRegsIter.more(); ++gcRegsIter) {
+ Register reg = *gcRegsIter;
+ size_t offsetFromTop = trapExitLayout.getOffset(reg);
+
+ // If this doesn't hold, the associated register wasn't saved by
+ // the trap exit stub. Better to crash now than much later, in
+ // some obscure place, and possibly with security consequences.
+ MOZ_RELEASE_ASSERT(offsetFromTop < trapExitLayoutNumWords);
+
+ // offsetFromTop is an offset in words down from the highest
+ // address in the exit stub save area. Switch it around to be an
+ // offset up from the bottom of the (integer register) save area.
+ size_t offsetFromBottom = trapExitLayoutNumWords - 1 - offsetFromTop;
+
+ vec[offsetFromBottom] = true;
+ hasRefs = true;
+ }
+ } else {
+ // This map is associated with a call instruction. We expect there to be
+ // no live ref-carrying registers, and if there are we're in deep trouble.
+ MOZ_RELEASE_ASSERT(!gcRegsIter.more());
+ }
+
+ // BODY (GENERAL SPILL) AREA and FRAME and INCOMING ARGS
+ // Deal with roots on the stack.
+ size_t wordsSoFar = vec.length();
+ if (!vec.appendN(false, nNonTrapBytes / sizeof(void*))) {
+ return false;
+ }
+ const LSafepoint::SlotList& gcSlots = safepoint.gcSlots();
+ for (SafepointSlotEntry gcSlot : gcSlots) {
+ // The following needs to correspond with JitFrameLayout::slotRef
+ // gcSlot.stack == 0 means the slot is in the args area
+ if (gcSlot.stack) {
+ // It's a slot in the body allocation, so .slot is interpreted
+ // as an index downwards from the Frame*
+ MOZ_ASSERT(gcSlot.slot <= nBodyBytes);
+ uint32_t offsetInBytes = nBodyBytes - gcSlot.slot;
+ MOZ_ASSERT(offsetInBytes % sizeof(void*) == 0);
+ vec[wordsSoFar + offsetInBytes / sizeof(void*)] = true;
+ } else {
+ // It's an argument slot
+ MOZ_ASSERT(gcSlot.slot < nInboundStackArgBytes);
+ uint32_t offsetInBytes = nBodyBytes + nFrameBytes + gcSlot.slot;
+ MOZ_ASSERT(offsetInBytes % sizeof(void*) == 0);
+ vec[wordsSoFar + offsetInBytes / sizeof(void*)] = true;
+ }
+ hasRefs = true;
+ }
+
+#ifndef DEBUG
+ // We saw no references, and this is a non-debug build, so don't bother
+ // building the stackmap.
+ if (!hasRefs) {
+ return true;
+ }
+#endif
+
+ // Convert vec into a wasm::StackMap.
+ MOZ_ASSERT(vec.length() * sizeof(void*) == nTotalBytes);
+ wasm::StackMap* stackMap =
+ wasm::ConvertStackMapBoolVectorToStackMap(vec, hasRefs);
+ if (!stackMap) {
+ return false;
+ }
+ if (safepoint.isWasmTrap()) {
+ stackMap->setExitStubWords(trapExitLayoutNumWords);
+ }
+
+ // Record in the map, how far down from the highest address the Frame* is.
+ // Take the opportunity to check that we haven't marked any part of the
+ // Frame itself as a pointer.
+ stackMap->setFrameOffsetFromTop((nInboundStackArgBytes + nFrameBytes) /
+ sizeof(void*));
+#ifdef DEBUG
+ for (uint32_t i = 0; i < nFrameBytes / sizeof(void*); i++) {
+ MOZ_ASSERT(stackMap->getBit(stackMap->header.numMappedWords -
+ stackMap->header.frameOffsetFromTop + i) == 0);
+ }
+#endif
+
+ *result = stackMap;
+ return true;
+}
+
+bool CodeGenerator::generateWasm(
+ wasm::CallIndirectId callIndirectId, wasm::BytecodeOffset trapOffset,
+ const wasm::ArgTypeVector& argTypes, const RegisterOffsets& trapExitLayout,
+ size_t trapExitLayoutNumWords, wasm::FuncOffsets* offsets,
+ wasm::StackMaps* stackMaps, wasm::Decoder* decoder) {
+ AutoCreatedBy acb(masm, "CodeGenerator::generateWasm");
+
+ JitSpew(JitSpew_Codegen, "# Emitting wasm code");
+
+ size_t nInboundStackArgBytes = StackArgAreaSizeUnaligned(argTypes);
+
+ wasm::GenerateFunctionPrologue(masm, callIndirectId, mozilla::Nothing(),
+ offsets);
+
+ MOZ_ASSERT(masm.framePushed() == 0);
+
+ // Very large frames are implausible, probably an attack.
+ if (frameSize() > wasm::MaxFrameSize) {
+ return decoder->fail(decoder->beginOffset(), "stack frame is too large");
+ }
+
+ if (omitOverRecursedCheck()) {
+ masm.reserveStack(frameSize());
+ } else {
+ std::pair<CodeOffset, uint32_t> pair =
+ masm.wasmReserveStackChecked(frameSize(), trapOffset);
+ CodeOffset trapInsnOffset = pair.first;
+ size_t nBytesReservedBeforeTrap = pair.second;
+
+ wasm::StackMap* functionEntryStackMap = nullptr;
+ if (!CreateStackMapForFunctionEntryTrap(
+ argTypes, trapExitLayout, trapExitLayoutNumWords,
+ nBytesReservedBeforeTrap, nInboundStackArgBytes,
+ &functionEntryStackMap)) {
+ return false;
+ }
+
+ // In debug builds, we'll always have a stack map, even if there are no
+ // refs to track.
+ MOZ_ASSERT(functionEntryStackMap);
+
+ if (functionEntryStackMap &&
+ !stackMaps->add((uint8_t*)(uintptr_t)trapInsnOffset.offset(),
+ functionEntryStackMap)) {
+ functionEntryStackMap->destroy();
+ return false;
+ }
+ }
+
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+
+ if (!generateBody()) {
+ return false;
+ }
+
+ masm.bind(&returnLabel_);
+ wasm::GenerateFunctionEpilogue(masm, frameSize(), offsets);
+
+ if (!generateOutOfLineCode()) {
+ return false;
+ }
+
+ masm.flush();
+ if (masm.oom()) {
+ return false;
+ }
+
+ offsets->end = masm.currentOffset();
+
+ MOZ_ASSERT(!masm.failureLabel()->used());
+ MOZ_ASSERT(snapshots_.listSize() == 0);
+ MOZ_ASSERT(snapshots_.RVATableSize() == 0);
+ MOZ_ASSERT(recovers_.size() == 0);
+ MOZ_ASSERT(graph.numConstants() == 0);
+ MOZ_ASSERT(osiIndices_.empty());
+ MOZ_ASSERT(icList_.empty());
+ MOZ_ASSERT(safepoints_.size() == 0);
+ MOZ_ASSERT(!scriptCounts_);
+
+ // Convert the safepoints to stackmaps and add them to our running
+ // collection thereof.
+ for (CodegenSafepointIndex& index : safepointIndices_) {
+ wasm::StackMap* stackMap = nullptr;
+ if (!CreateStackMapFromLSafepoint(*index.safepoint(), trapExitLayout,
+ trapExitLayoutNumWords,
+ nInboundStackArgBytes, &stackMap)) {
+ return false;
+ }
+
+ // In debug builds, we'll always have a stack map.
+ MOZ_ASSERT(stackMap);
+ if (!stackMap) {
+ continue;
+ }
+
+ if (!stackMaps->add((uint8_t*)(uintptr_t)index.displacement(), stackMap)) {
+ stackMap->destroy();
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool CodeGenerator::generate() {
+ AutoCreatedBy acb(masm, "CodeGenerator::generate");
+
+ JitSpew(JitSpew_Codegen, "# Emitting code for script %s:%u:%u",
+ gen->outerInfo().script()->filename(),
+ gen->outerInfo().script()->lineno(),
+ gen->outerInfo().script()->column());
+
+ // Initialize native code table with an entry to the start of
+ // top-level script.
+ InlineScriptTree* tree = gen->outerInfo().inlineScriptTree();
+ jsbytecode* startPC = tree->script()->code();
+ BytecodeSite* startSite = new (gen->alloc()) BytecodeSite(tree, startPC);
+ if (!addNativeToBytecodeEntry(startSite)) {
+ return false;
+ }
+
+ if (!safepoints_.init(gen->alloc())) {
+ return false;
+ }
+
+ if (!generatePrologue()) {
+ return false;
+ }
+
+ // Reset native => bytecode map table with top-level script and startPc.
+ if (!addNativeToBytecodeEntry(startSite)) {
+ return false;
+ }
+
+ if (!generateBody()) {
+ return false;
+ }
+
+ // Reset native => bytecode map table with top-level script and startPc.
+ if (!addNativeToBytecodeEntry(startSite)) {
+ return false;
+ }
+
+ if (!generateEpilogue()) {
+ return false;
+ }
+
+ // Reset native => bytecode map table with top-level script and startPc.
+ if (!addNativeToBytecodeEntry(startSite)) {
+ return false;
+ }
+
+ generateInvalidateEpilogue();
+
+ // native => bytecode entries for OOL code will be added
+ // by CodeGeneratorShared::generateOutOfLineCode
+ if (!generateOutOfLineCode()) {
+ return false;
+ }
+
+ // Add terminal entry.
+ if (!addNativeToBytecodeEntry(startSite)) {
+ return false;
+ }
+
+ // Dump Native to bytecode entries to spew.
+ dumpNativeToBytecodeEntries();
+
+ return !masm.oom();
+}
+
+static bool AddInlinedCompilations(JSContext* cx, HandleScript script,
+ IonCompilationId compilationId,
+ const WarpSnapshot* snapshot,
+ bool* isValid) {
+ MOZ_ASSERT(!*isValid);
+ RecompileInfo recompileInfo(script, compilationId);
+
+ JitZone* jitZone = cx->zone()->jitZone();
+
+ for (const auto* scriptSnapshot : snapshot->scripts()) {
+ JSScript* inlinedScript = scriptSnapshot->script();
+ if (inlinedScript == script) {
+ continue;
+ }
+
+ // TODO(post-Warp): This matches FinishCompilation and is necessary to
+ // ensure in-progress compilations are canceled when an inlined functon
+ // becomes a debuggee. See the breakpoint-14.js jit-test.
+ // When TI is gone, try to clean this up by moving AddInlinedCompilations to
+ // WarpOracle so that we can handle this as part of addPendingRecompile
+ // instead of requiring this separate check.
+ if (inlinedScript->isDebuggee()) {
+ *isValid = false;
+ return true;
+ }
+
+ if (!jitZone->addInlinedCompilation(recompileInfo, inlinedScript)) {
+ return false;
+ }
+ }
+
+ *isValid = true;
+ return true;
+}
+
+bool CodeGenerator::link(JSContext* cx, const WarpSnapshot* snapshot) {
+ AutoCreatedBy acb(masm, "CodeGenerator::link");
+
+ // We cancel off-thread Ion compilations in a few places during GC, but if
+ // this compilation was performed off-thread it will already have been
+ // removed from the relevant lists by this point. Don't allow GC here.
+ JS::AutoAssertNoGC nogc(cx);
+
+ RootedScript script(cx, gen->outerInfo().script());
+ MOZ_ASSERT(!script->hasIonScript());
+
+ // Perform any read barriers which were skipped while compiling the
+ // script, which may have happened off-thread.
+ const JitRealm* jr = gen->realm->jitRealm();
+ jr->performStubReadBarriers(realmStubsToReadBarrier_);
+
+ if (scriptCounts_ && !script->hasScriptCounts() &&
+ !script->initScriptCounts(cx)) {
+ return false;
+ }
+
+ // Check to make sure we didn't have a mid-build invalidation. If so, we
+ // will trickle to jit::Compile() and return Method_Skipped.
+ uint32_t warmUpCount = script->getWarmUpCount();
+
+ IonCompilationId compilationId =
+ cx->runtime()->jitRuntime()->nextCompilationId();
+ JitZone* jitZone = cx->zone()->jitZone();
+ jitZone->currentCompilationIdRef().emplace(compilationId);
+ auto resetCurrentId = mozilla::MakeScopeExit(
+ [jitZone] { jitZone->currentCompilationIdRef().reset(); });
+
+ // Record constraints. If an error occured, returns false and potentially
+ // prevent future compilations. Otherwise, if an invalidation occured, then
+ // skip the current compilation.
+ bool isValid = false;
+
+ // If an inlined script is invalidated (for example, by attaching
+ // a debugger), we must also invalidate the parent IonScript.
+ if (!AddInlinedCompilations(cx, script, compilationId, snapshot, &isValid)) {
+ return false;
+ }
+ if (!isValid) {
+ return true;
+ }
+
+ // IonMonkey could have inferred better type information during
+ // compilation. Since adding the new information to the actual type
+ // information can reset the usecount, increase it back to what it was
+ // before.
+ if (warmUpCount > script->getWarmUpCount()) {
+ script->incWarmUpCounter(warmUpCount - script->getWarmUpCount());
+ }
+
+ uint32_t argumentSlots = (gen->outerInfo().nargs() + 1) * sizeof(Value);
+
+ // We encode safepoints after the OSI-point offsets have been determined.
+ if (!encodeSafepoints()) {
+ return false;
+ }
+
+ size_t numNurseryObjects = snapshot->nurseryObjects().length();
+
+ IonScript* ionScript = IonScript::New(
+ cx, compilationId, graph.localSlotsSize(), argumentSlots, frameDepth_,
+ snapshots_.listSize(), snapshots_.RVATableSize(), recovers_.size(),
+ graph.numConstants(), numNurseryObjects, safepointIndices_.length(),
+ osiIndices_.length(), icList_.length(), runtimeData_.length(),
+ safepoints_.size());
+ if (!ionScript) {
+ return false;
+ }
+#ifdef DEBUG
+ ionScript->setICHash(snapshot->icHash());
+#endif
+
+ auto freeIonScript = mozilla::MakeScopeExit([&ionScript] {
+ // Use js_free instead of IonScript::Destroy: the cache list is still
+ // uninitialized.
+ js_free(ionScript);
+ });
+
+ Linker linker(masm);
+ JitCode* code = linker.newCode(cx, CodeKind::Ion);
+ if (!code) {
+ return false;
+ }
+
+ // Encode native to bytecode map if profiling is enabled.
+ if (isProfilerInstrumentationEnabled()) {
+ // Generate native-to-bytecode main table.
+ IonEntry::ScriptList scriptList;
+ if (!generateCompactNativeToBytecodeMap(cx, code, scriptList)) {
+ return false;
+ }
+
+ uint8_t* ionTableAddr =
+ ((uint8_t*)nativeToBytecodeMap_.get()) + nativeToBytecodeTableOffset_;
+ JitcodeIonTable* ionTable = (JitcodeIonTable*)ionTableAddr;
+
+ // Construct the IonEntry that will go into the global table.
+ auto entry = MakeJitcodeGlobalEntry<IonEntry>(
+ cx, code, code->raw(), code->rawEnd(), std::move(scriptList), ionTable);
+ if (!entry) {
+ return false;
+ }
+ (void)nativeToBytecodeMap_.release(); // Table is now owned by |entry|.
+
+ // Add entry to the global table.
+ JitcodeGlobalTable* globalTable =
+ cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
+ if (!globalTable->addEntry(std::move(entry))) {
+ return false;
+ }
+
+ // Mark the jitcode as having a bytecode map.
+ code->setHasBytecodeMap();
+ } else {
+ // Add a dumy jitcodeGlobalTable entry.
+ auto entry = MakeJitcodeGlobalEntry<DummyEntry>(cx, code, code->raw(),
+ code->rawEnd());
+ if (!entry) {
+ return false;
+ }
+
+ // Add entry to the global table.
+ JitcodeGlobalTable* globalTable =
+ cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
+ if (!globalTable->addEntry(std::move(entry))) {
+ return false;
+ }
+
+ // Mark the jitcode as having a bytecode map.
+ code->setHasBytecodeMap();
+ }
+
+ ionScript->setMethod(code);
+
+ // If the Gecko Profiler is enabled, mark IonScript as having been
+ // instrumented accordingly.
+ if (isProfilerInstrumentationEnabled()) {
+ ionScript->setHasProfilingInstrumentation();
+ }
+
+ Assembler::PatchDataWithValueCheck(
+ CodeLocationLabel(code, invalidateEpilogueData_), ImmPtr(ionScript),
+ ImmPtr((void*)-1));
+
+ for (CodeOffset offset : ionScriptLabels_) {
+ Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, offset),
+ ImmPtr(ionScript), ImmPtr((void*)-1));
+ }
+
+ for (NurseryObjectLabel label : ionNurseryObjectLabels_) {
+ void* entry = ionScript->addressOfNurseryObject(label.nurseryIndex);
+ Assembler::PatchDataWithValueCheck(CodeLocationLabel(code, label.offset),
+ ImmPtr(entry), ImmPtr((void*)-1));
+ }
+
+ // for generating inline caches during the execution.
+ if (runtimeData_.length()) {
+ ionScript->copyRuntimeData(&runtimeData_[0]);
+ }
+ if (icList_.length()) {
+ ionScript->copyICEntries(&icList_[0]);
+ }
+
+ for (size_t i = 0; i < icInfo_.length(); i++) {
+ IonIC& ic = ionScript->getICFromIndex(i);
+ Assembler::PatchDataWithValueCheck(
+ CodeLocationLabel(code, icInfo_[i].icOffsetForJump),
+ ImmPtr(ic.codeRawPtr()), ImmPtr((void*)-1));
+ Assembler::PatchDataWithValueCheck(
+ CodeLocationLabel(code, icInfo_[i].icOffsetForPush), ImmPtr(&ic),
+ ImmPtr((void*)-1));
+ }
+
+ JitSpew(JitSpew_Codegen, "Created IonScript %p (raw %p)", (void*)ionScript,
+ (void*)code->raw());
+
+ ionScript->setInvalidationEpilogueDataOffset(
+ invalidateEpilogueData_.offset());
+ if (jsbytecode* osrPc = gen->outerInfo().osrPc()) {
+ ionScript->setOsrPc(osrPc);
+ ionScript->setOsrEntryOffset(getOsrEntryOffset());
+ }
+ ionScript->setInvalidationEpilogueOffset(invalidate_.offset());
+
+ if (PerfEnabled()) {
+ perfSpewer_.saveProfile(cx, script, code);
+ }
+
+#ifdef MOZ_VTUNE
+ vtune::MarkScript(code, script, "ion");
+#endif
+
+ // for marking during GC.
+ if (safepointIndices_.length()) {
+ ionScript->copySafepointIndices(&safepointIndices_[0]);
+ }
+ if (safepoints_.size()) {
+ ionScript->copySafepoints(&safepoints_);
+ }
+
+ // for recovering from an Ion Frame.
+ if (osiIndices_.length()) {
+ ionScript->copyOsiIndices(&osiIndices_[0]);
+ }
+ if (snapshots_.listSize()) {
+ ionScript->copySnapshots(&snapshots_);
+ }
+ MOZ_ASSERT_IF(snapshots_.listSize(), recovers_.size());
+ if (recovers_.size()) {
+ ionScript->copyRecovers(&recovers_);
+ }
+ if (graph.numConstants()) {
+ const Value* vp = graph.constantPool();
+ ionScript->copyConstants(vp);
+ for (size_t i = 0; i < graph.numConstants(); i++) {
+ const Value& v = vp[i];
+ if (v.isGCThing()) {
+ if (gc::StoreBuffer* sb = v.toGCThing()->storeBuffer()) {
+ sb->putWholeCell(script);
+ break;
+ }
+ }
+ }
+ }
+
+ // Attach any generated script counts to the script.
+ if (IonScriptCounts* counts = extractScriptCounts()) {
+ script->addIonCounts(counts);
+ }
+
+ // WARNING: Code after this point must be infallible!
+
+ // Copy the list of nursery objects. Note that the store buffer can add
+ // HeapPtr edges that must be cleared in IonScript::Destroy. See the
+ // infallibility warning above.
+ const auto& nurseryObjects = snapshot->nurseryObjects();
+ for (size_t i = 0; i < nurseryObjects.length(); i++) {
+ ionScript->nurseryObjects()[i].init(nurseryObjects[i]);
+ }
+
+ // Transfer ownership of the IonScript to the JitScript. At this point enough
+ // of the IonScript must be initialized for IonScript::Destroy to work.
+ freeIonScript.release();
+ script->jitScript()->setIonScript(script, ionScript);
+
+ return true;
+}
+
+// An out-of-line path to convert a boxed int32 to either a float or double.
+class OutOfLineUnboxFloatingPoint : public OutOfLineCodeBase<CodeGenerator> {
+ LUnboxFloatingPoint* unboxFloatingPoint_;
+
+ public:
+ explicit OutOfLineUnboxFloatingPoint(LUnboxFloatingPoint* unboxFloatingPoint)
+ : unboxFloatingPoint_(unboxFloatingPoint) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineUnboxFloatingPoint(this);
+ }
+
+ LUnboxFloatingPoint* unboxFloatingPoint() const {
+ return unboxFloatingPoint_;
+ }
+};
+
+void CodeGenerator::visitUnboxFloatingPoint(LUnboxFloatingPoint* lir) {
+ const ValueOperand box = ToValue(lir, LUnboxFloatingPoint::Input);
+ const LDefinition* result = lir->output();
+
+ // Out-of-line path to convert int32 to double or bailout
+ // if this instruction is fallible.
+ OutOfLineUnboxFloatingPoint* ool =
+ new (alloc()) OutOfLineUnboxFloatingPoint(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ FloatRegister resultReg = ToFloatRegister(result);
+ masm.branchTestDouble(Assembler::NotEqual, box, ool->entry());
+ masm.unboxDouble(box, resultReg);
+ if (lir->type() == MIRType::Float32) {
+ masm.convertDoubleToFloat32(resultReg, resultReg);
+ }
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineUnboxFloatingPoint(
+ OutOfLineUnboxFloatingPoint* ool) {
+ LUnboxFloatingPoint* ins = ool->unboxFloatingPoint();
+ const ValueOperand value = ToValue(ins, LUnboxFloatingPoint::Input);
+
+ if (ins->mir()->fallible()) {
+ Label bail;
+ masm.branchTestInt32(Assembler::NotEqual, value, &bail);
+ bailoutFrom(&bail, ins->snapshot());
+ }
+ masm.int32ValueToFloatingPoint(value, ToFloatRegister(ins->output()),
+ ins->type());
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitCallBindVar(LCallBindVar* lir) {
+ pushArg(ToRegister(lir->environmentChain()));
+
+ using Fn = JSObject* (*)(JSContext*, JSObject*);
+ callVM<Fn, BindVarOperation>(lir);
+}
+
+void CodeGenerator::visitMegamorphicSetElement(LMegamorphicSetElement* lir) {
+ Register obj = ToRegister(lir->getOperand(0));
+ pushArg(Imm32(lir->mir()->strict()));
+ pushArg(TypedOrValueRegister(MIRType::Object, AnyRegister(obj)));
+ pushArg(ToValue(lir, LMegamorphicSetElement::ValueIndex));
+ pushArg(ToValue(lir, LMegamorphicSetElement::IndexIndex));
+ pushArg(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue,
+ HandleValue, bool);
+ callVM<Fn, js::jit::SetElementMegamorphic>(lir);
+}
+
+void CodeGenerator::visitLoadFixedSlotV(LLoadFixedSlotV* ins) {
+ const Register obj = ToRegister(ins->getOperand(0));
+ size_t slot = ins->mir()->slot();
+ ValueOperand result = ToOutValue(ins);
+
+ masm.loadValue(Address(obj, NativeObject::getFixedSlotOffset(slot)), result);
+}
+
+void CodeGenerator::visitLoadFixedSlotT(LLoadFixedSlotT* ins) {
+ const Register obj = ToRegister(ins->getOperand(0));
+ size_t slot = ins->mir()->slot();
+ AnyRegister result = ToAnyRegister(ins->getDef(0));
+ MIRType type = ins->mir()->type();
+
+ masm.loadUnboxedValue(Address(obj, NativeObject::getFixedSlotOffset(slot)),
+ type, result);
+}
+
+template <typename T>
+static void EmitLoadAndUnbox(MacroAssembler& masm, const T& src, MIRType type,
+ bool fallible, AnyRegister dest, Label* fail) {
+ if (type == MIRType::Double) {
+ MOZ_ASSERT(dest.isFloat());
+ masm.ensureDouble(src, dest.fpu(), fail);
+ return;
+ }
+ if (fallible) {
+ switch (type) {
+ case MIRType::Int32:
+ masm.fallibleUnboxInt32(src, dest.gpr(), fail);
+ break;
+ case MIRType::Boolean:
+ masm.fallibleUnboxBoolean(src, dest.gpr(), fail);
+ break;
+ case MIRType::Object:
+ masm.fallibleUnboxObject(src, dest.gpr(), fail);
+ break;
+ case MIRType::String:
+ masm.fallibleUnboxString(src, dest.gpr(), fail);
+ break;
+ case MIRType::Symbol:
+ masm.fallibleUnboxSymbol(src, dest.gpr(), fail);
+ break;
+ case MIRType::BigInt:
+ masm.fallibleUnboxBigInt(src, dest.gpr(), fail);
+ break;
+ default:
+ MOZ_CRASH("Unexpected MIRType");
+ }
+ return;
+ }
+ masm.loadUnboxedValue(src, type, dest);
+}
+
+void CodeGenerator::visitLoadFixedSlotAndUnbox(LLoadFixedSlotAndUnbox* ins) {
+ const MLoadFixedSlotAndUnbox* mir = ins->mir();
+ MIRType type = mir->type();
+ Register input = ToRegister(ins->object());
+ AnyRegister result = ToAnyRegister(ins->output());
+ size_t slot = mir->slot();
+
+ Address address(input, NativeObject::getFixedSlotOffset(slot));
+
+ Label bail;
+ EmitLoadAndUnbox(masm, address, type, mir->fallible(), result, &bail);
+ if (mir->fallible()) {
+ bailoutFrom(&bail, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitLoadDynamicSlotAndUnbox(
+ LLoadDynamicSlotAndUnbox* ins) {
+ const MLoadDynamicSlotAndUnbox* mir = ins->mir();
+ MIRType type = mir->type();
+ Register input = ToRegister(ins->slots());
+ AnyRegister result = ToAnyRegister(ins->output());
+ size_t slot = mir->slot();
+
+ Address address(input, slot * sizeof(JS::Value));
+
+ Label bail;
+ EmitLoadAndUnbox(masm, address, type, mir->fallible(), result, &bail);
+ if (mir->fallible()) {
+ bailoutFrom(&bail, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitLoadElementAndUnbox(LLoadElementAndUnbox* ins) {
+ const MLoadElementAndUnbox* mir = ins->mir();
+ MIRType type = mir->type();
+ Register elements = ToRegister(ins->elements());
+ AnyRegister result = ToAnyRegister(ins->output());
+
+ Label bail;
+ if (ins->index()->isConstant()) {
+ NativeObject::elementsSizeMustNotOverflow();
+ int32_t offset = ToInt32(ins->index()) * sizeof(Value);
+ Address address(elements, offset);
+ EmitLoadAndUnbox(masm, address, type, mir->fallible(), result, &bail);
+ } else {
+ BaseObjectElementIndex address(elements, ToRegister(ins->index()));
+ EmitLoadAndUnbox(masm, address, type, mir->fallible(), result, &bail);
+ }
+
+ if (mir->fallible()) {
+ bailoutFrom(&bail, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitAddAndStoreSlot(LAddAndStoreSlot* ins) {
+ const Register obj = ToRegister(ins->getOperand(0));
+ const ValueOperand value = ToValue(ins, LAddAndStoreSlot::ValueIndex);
+ const Register maybeTemp = ToTempRegisterOrInvalid(ins->temp0());
+
+ Shape* shape = ins->mir()->shape();
+ masm.storeObjShape(shape, obj, [](MacroAssembler& masm, const Address& addr) {
+ EmitPreBarrier(masm, addr, MIRType::Shape);
+ });
+
+ // Perform the store. No pre-barrier required since this is a new
+ // initialization.
+
+ uint32_t offset = ins->mir()->slotOffset();
+ if (ins->mir()->kind() == MAddAndStoreSlot::Kind::FixedSlot) {
+ Address slot(obj, offset);
+ masm.storeValue(value, slot);
+ } else {
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), maybeTemp);
+ Address slot(maybeTemp, offset);
+ masm.storeValue(value, slot);
+ }
+}
+
+void CodeGenerator::visitAllocateAndStoreSlot(LAllocateAndStoreSlot* ins) {
+ const Register obj = ToRegister(ins->getOperand(0));
+ const ValueOperand value = ToValue(ins, LAllocateAndStoreSlot::ValueIndex);
+ const Register temp0 = ToRegister(ins->temp0());
+ const Register temp1 = ToRegister(ins->temp1());
+
+ masm.Push(obj);
+ masm.Push(value);
+
+ using Fn = bool (*)(JSContext * cx, NativeObject * obj, uint32_t newCount);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp0);
+ masm.passABIArg(temp0);
+ masm.passABIArg(obj);
+ masm.move32(Imm32(ins->mir()->numNewSlots()), temp1);
+ masm.passABIArg(temp1);
+ masm.callWithABI<Fn, NativeObject::growSlotsPure>();
+ masm.storeCallPointerResult(temp0);
+
+ masm.Pop(value);
+ masm.Pop(obj);
+
+ bailoutIfFalseBool(temp0, ins->snapshot());
+
+ masm.storeObjShape(ins->mir()->shape(), obj,
+ [](MacroAssembler& masm, const Address& addr) {
+ EmitPreBarrier(masm, addr, MIRType::Shape);
+ });
+
+ // Perform the store. No pre-barrier required since this is a new
+ // initialization.
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), temp0);
+ Address slot(temp0, ins->mir()->slotOffset());
+ masm.storeValue(value, slot);
+}
+
+void CodeGenerator::visitAddSlotAndCallAddPropHook(
+ LAddSlotAndCallAddPropHook* ins) {
+ const Register obj = ToRegister(ins->object());
+ const ValueOperand value =
+ ToValue(ins, LAddSlotAndCallAddPropHook::ValueIndex);
+
+ pushArg(ImmGCPtr(ins->mir()->shape()));
+ pushArg(value);
+ pushArg(obj);
+
+ using Fn =
+ bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, Handle<Shape*>);
+ callVM<Fn, AddSlotAndCallAddPropHook>(ins);
+}
+
+void CodeGenerator::visitStoreFixedSlotV(LStoreFixedSlotV* ins) {
+ const Register obj = ToRegister(ins->getOperand(0));
+ size_t slot = ins->mir()->slot();
+
+ const ValueOperand value = ToValue(ins, LStoreFixedSlotV::ValueIndex);
+
+ Address address(obj, NativeObject::getFixedSlotOffset(slot));
+ if (ins->mir()->needsBarrier()) {
+ emitPreBarrier(address);
+ }
+
+ masm.storeValue(value, address);
+}
+
+void CodeGenerator::visitStoreFixedSlotT(LStoreFixedSlotT* ins) {
+ const Register obj = ToRegister(ins->getOperand(0));
+ size_t slot = ins->mir()->slot();
+
+ const LAllocation* value = ins->value();
+ MIRType valueType = ins->mir()->value()->type();
+
+ Address address(obj, NativeObject::getFixedSlotOffset(slot));
+ if (ins->mir()->needsBarrier()) {
+ emitPreBarrier(address);
+ }
+
+ ConstantOrRegister nvalue =
+ value->isConstant()
+ ? ConstantOrRegister(value->toConstant()->toJSValue())
+ : TypedOrValueRegister(valueType, ToAnyRegister(value));
+ masm.storeConstantOrRegister(nvalue, address);
+}
+
+void CodeGenerator::visitGetNameCache(LGetNameCache* ins) {
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+ Register envChain = ToRegister(ins->envObj());
+ ValueOperand output = ToOutValue(ins);
+ Register temp = ToRegister(ins->temp0());
+
+ IonGetNameIC ic(liveRegs, envChain, output, temp);
+ addIC(ins, allocateIC(ic));
+}
+
+void CodeGenerator::addGetPropertyCache(LInstruction* ins,
+ LiveRegisterSet liveRegs,
+ TypedOrValueRegister value,
+ const ConstantOrRegister& id,
+ ValueOperand output) {
+ CacheKind kind = CacheKind::GetElem;
+ if (id.constant() && id.value().isString()) {
+ JSString* idString = id.value().toString();
+ if (idString->isAtom() && !idString->asAtom().isIndex()) {
+ kind = CacheKind::GetProp;
+ }
+ }
+ IonGetPropertyIC cache(kind, liveRegs, value, id, output);
+ addIC(ins, allocateIC(cache));
+}
+
+void CodeGenerator::addSetPropertyCache(LInstruction* ins,
+ LiveRegisterSet liveRegs,
+ Register objReg, Register temp,
+ const ConstantOrRegister& id,
+ const ConstantOrRegister& value,
+ bool strict) {
+ CacheKind kind = CacheKind::SetElem;
+ if (id.constant() && id.value().isString()) {
+ JSString* idString = id.value().toString();
+ if (idString->isAtom() && !idString->asAtom().isIndex()) {
+ kind = CacheKind::SetProp;
+ }
+ }
+ IonSetPropertyIC cache(kind, liveRegs, objReg, temp, id, value, strict);
+ addIC(ins, allocateIC(cache));
+}
+
+ConstantOrRegister CodeGenerator::toConstantOrRegister(LInstruction* lir,
+ size_t n, MIRType type) {
+ if (type == MIRType::Value) {
+ return TypedOrValueRegister(ToValue(lir, n));
+ }
+
+ const LAllocation* value = lir->getOperand(n);
+ if (value->isConstant()) {
+ return ConstantOrRegister(value->toConstant()->toJSValue());
+ }
+
+ return TypedOrValueRegister(type, ToAnyRegister(value));
+}
+
+void CodeGenerator::visitGetPropertyCache(LGetPropertyCache* ins) {
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+ TypedOrValueRegister value =
+ toConstantOrRegister(ins, LGetPropertyCache::ValueIndex,
+ ins->mir()->value()->type())
+ .reg();
+ ConstantOrRegister id = toConstantOrRegister(ins, LGetPropertyCache::IdIndex,
+ ins->mir()->idval()->type());
+ ValueOperand output = ToOutValue(ins);
+ addGetPropertyCache(ins, liveRegs, value, id, output);
+}
+
+void CodeGenerator::visitGetPropSuperCache(LGetPropSuperCache* ins) {
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+ Register obj = ToRegister(ins->obj());
+ TypedOrValueRegister receiver =
+ toConstantOrRegister(ins, LGetPropSuperCache::ReceiverIndex,
+ ins->mir()->receiver()->type())
+ .reg();
+ ConstantOrRegister id = toConstantOrRegister(ins, LGetPropSuperCache::IdIndex,
+ ins->mir()->idval()->type());
+ ValueOperand output = ToOutValue(ins);
+
+ CacheKind kind = CacheKind::GetElemSuper;
+ if (id.constant() && id.value().isString()) {
+ JSString* idString = id.value().toString();
+ if (idString->isAtom() && !idString->asAtom().isIndex()) {
+ kind = CacheKind::GetPropSuper;
+ }
+ }
+
+ IonGetPropSuperIC cache(kind, liveRegs, obj, receiver, id, output);
+ addIC(ins, allocateIC(cache));
+}
+
+void CodeGenerator::visitBindNameCache(LBindNameCache* ins) {
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+ Register envChain = ToRegister(ins->environmentChain());
+ Register output = ToRegister(ins->output());
+ Register temp = ToRegister(ins->temp0());
+
+ IonBindNameIC ic(liveRegs, envChain, output, temp);
+ addIC(ins, allocateIC(ic));
+}
+
+void CodeGenerator::visitHasOwnCache(LHasOwnCache* ins) {
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+ TypedOrValueRegister value =
+ toConstantOrRegister(ins, LHasOwnCache::ValueIndex,
+ ins->mir()->value()->type())
+ .reg();
+ TypedOrValueRegister id = toConstantOrRegister(ins, LHasOwnCache::IdIndex,
+ ins->mir()->idval()->type())
+ .reg();
+ Register output = ToRegister(ins->output());
+
+ IonHasOwnIC cache(liveRegs, value, id, output);
+ addIC(ins, allocateIC(cache));
+}
+
+void CodeGenerator::visitCheckPrivateFieldCache(LCheckPrivateFieldCache* ins) {
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+ TypedOrValueRegister value =
+ toConstantOrRegister(ins, LCheckPrivateFieldCache::ValueIndex,
+ ins->mir()->value()->type())
+ .reg();
+ TypedOrValueRegister id =
+ toConstantOrRegister(ins, LCheckPrivateFieldCache::IdIndex,
+ ins->mir()->idval()->type())
+ .reg();
+ Register output = ToRegister(ins->output());
+
+ IonCheckPrivateFieldIC cache(liveRegs, value, id, output);
+ addIC(ins, allocateIC(cache));
+}
+
+void CodeGenerator::visitNewPrivateName(LNewPrivateName* ins) {
+ pushArg(ImmGCPtr(ins->mir()->name()));
+
+ using Fn = JS::Symbol* (*)(JSContext*, Handle<JSAtom*>);
+ callVM<Fn, NewPrivateName>(ins);
+}
+
+void CodeGenerator::visitCallDeleteProperty(LCallDeleteProperty* lir) {
+ pushArg(ImmGCPtr(lir->mir()->name()));
+ pushArg(ToValue(lir, LCallDeleteProperty::ValueIndex));
+
+ using Fn = bool (*)(JSContext*, HandleValue, Handle<PropertyName*>, bool*);
+ if (lir->mir()->strict()) {
+ callVM<Fn, DelPropOperation<true>>(lir);
+ } else {
+ callVM<Fn, DelPropOperation<false>>(lir);
+ }
+}
+
+void CodeGenerator::visitCallDeleteElement(LCallDeleteElement* lir) {
+ pushArg(ToValue(lir, LCallDeleteElement::IndexIndex));
+ pushArg(ToValue(lir, LCallDeleteElement::ValueIndex));
+
+ using Fn = bool (*)(JSContext*, HandleValue, HandleValue, bool*);
+ if (lir->mir()->strict()) {
+ callVM<Fn, DelElemOperation<true>>(lir);
+ } else {
+ callVM<Fn, DelElemOperation<false>>(lir);
+ }
+}
+
+void CodeGenerator::visitObjectToIterator(LObjectToIterator* lir) {
+ Register obj = ToRegister(lir->object());
+ Register iterObj = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+ Register temp2 = ToRegister(lir->temp1());
+ Register temp3 = ToRegister(lir->temp2());
+
+ using Fn = JSObject* (*)(JSContext*, HandleObject);
+ OutOfLineCode* ool =
+ oolCallVM<Fn, GetIterator>(lir, ArgList(obj), StoreRegisterTo(iterObj));
+
+ masm.maybeLoadIteratorFromShape(obj, iterObj, temp, temp2, temp3,
+ ool->entry());
+
+ Register nativeIter = temp;
+ masm.loadPrivate(
+ Address(iterObj, PropertyIteratorObject::offsetOfIteratorSlot()),
+ nativeIter);
+
+ Address iterFlagsAddr(nativeIter, NativeIterator::offsetOfFlagsAndCount());
+ masm.storePtr(
+ obj, Address(nativeIter, NativeIterator::offsetOfObjectBeingIterated()));
+ masm.or32(Imm32(NativeIterator::Flags::Active), iterFlagsAddr);
+ // The postbarrier for |objectBeingIterated| is generated separately by the
+ // transpiler.
+
+ Register enumeratorsAddr = temp2;
+ masm.movePtr(ImmPtr(lir->mir()->enumeratorsAddr()), enumeratorsAddr);
+ masm.registerIterator(enumeratorsAddr, nativeIter, temp3);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitValueToIterator(LValueToIterator* lir) {
+ pushArg(ToValue(lir, LValueToIterator::ValueIndex));
+
+ using Fn = JSObject* (*)(JSContext*, HandleValue);
+ callVM<Fn, ValueToIterator>(lir);
+}
+
+void CodeGenerator::visitSetPropertyCache(LSetPropertyCache* ins) {
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+ Register objReg = ToRegister(ins->object());
+ Register temp = ToRegister(ins->temp0());
+
+ ConstantOrRegister id = toConstantOrRegister(ins, LSetPropertyCache::IdIndex,
+ ins->mir()->idval()->type());
+ ConstantOrRegister value = toConstantOrRegister(
+ ins, LSetPropertyCache::ValueIndex, ins->mir()->value()->type());
+
+ addSetPropertyCache(ins, liveRegs, objReg, temp, id, value,
+ ins->mir()->strict());
+}
+
+void CodeGenerator::visitThrow(LThrow* lir) {
+ pushArg(ToValue(lir, LThrow::ValueIndex));
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ callVM<Fn, js::ThrowOperation>(lir);
+}
+
+class OutOfLineTypeOfV : public OutOfLineCodeBase<CodeGenerator> {
+ LTypeOfV* ins_;
+
+ public:
+ explicit OutOfLineTypeOfV(LTypeOfV* ins) : ins_(ins) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineTypeOfV(this);
+ }
+ LTypeOfV* ins() const { return ins_; }
+};
+
+void CodeGenerator::emitTypeOfJSType(JSValueType type, Register output) {
+ switch (type) {
+ case JSVAL_TYPE_OBJECT:
+ masm.move32(Imm32(JSTYPE_OBJECT), output);
+ break;
+ case JSVAL_TYPE_DOUBLE:
+ case JSVAL_TYPE_INT32:
+ masm.move32(Imm32(JSTYPE_NUMBER), output);
+ break;
+ case JSVAL_TYPE_BOOLEAN:
+ masm.move32(Imm32(JSTYPE_BOOLEAN), output);
+ break;
+ case JSVAL_TYPE_UNDEFINED:
+ masm.move32(Imm32(JSTYPE_UNDEFINED), output);
+ break;
+ case JSVAL_TYPE_NULL:
+ masm.move32(Imm32(JSTYPE_OBJECT), output);
+ break;
+ case JSVAL_TYPE_STRING:
+ masm.move32(Imm32(JSTYPE_STRING), output);
+ break;
+ case JSVAL_TYPE_SYMBOL:
+ masm.move32(Imm32(JSTYPE_SYMBOL), output);
+ break;
+ case JSVAL_TYPE_BIGINT:
+ masm.move32(Imm32(JSTYPE_BIGINT), output);
+ break;
+ default:
+ MOZ_CRASH("Unsupported JSValueType");
+ }
+}
+
+void CodeGenerator::emitTypeOfCheck(JSValueType type, Register tag,
+ Register output, Label* done,
+ Label* oolObject) {
+ Label notMatch;
+ switch (type) {
+ case JSVAL_TYPE_OBJECT:
+ // The input may be a callable object (result is "function") or
+ // may emulate undefined (result is "undefined"). Use an OOL path.
+ masm.branchTestObject(Assembler::Equal, tag, oolObject);
+ return;
+ case JSVAL_TYPE_DOUBLE:
+ case JSVAL_TYPE_INT32:
+ masm.branchTestNumber(Assembler::NotEqual, tag, &notMatch);
+ break;
+ default:
+ masm.branchTestType(Assembler::NotEqual, tag, type, &notMatch);
+ break;
+ }
+
+ emitTypeOfJSType(type, output);
+ masm.jump(done);
+ masm.bind(&notMatch);
+}
+
+void CodeGenerator::visitTypeOfV(LTypeOfV* lir) {
+ const ValueOperand value = ToValue(lir, LTypeOfV::InputIndex);
+ Register output = ToRegister(lir->output());
+ Register tag = masm.extractTag(value, output);
+
+ Label done;
+
+ auto* ool = new (alloc()) OutOfLineTypeOfV(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ const std::initializer_list<JSValueType> defaultOrder = {
+ JSVAL_TYPE_OBJECT, JSVAL_TYPE_DOUBLE, JSVAL_TYPE_UNDEFINED,
+ JSVAL_TYPE_NULL, JSVAL_TYPE_BOOLEAN, JSVAL_TYPE_STRING,
+ JSVAL_TYPE_SYMBOL, JSVAL_TYPE_BIGINT};
+
+ mozilla::EnumSet<JSValueType, uint32_t> remaining(defaultOrder);
+
+ // Generate checks for previously observed types first.
+ // The TypeDataList is sorted by descending frequency.
+ for (auto& observed : lir->mir()->observedTypes()) {
+ JSValueType type = observed.type();
+
+ // Unify number types.
+ if (type == JSVAL_TYPE_INT32) {
+ type = JSVAL_TYPE_DOUBLE;
+ }
+
+ remaining -= type;
+
+ emitTypeOfCheck(type, tag, output, &done, ool->entry());
+ }
+
+ // Generate checks for remaining types.
+ for (auto type : defaultOrder) {
+ if (!remaining.contains(type)) {
+ continue;
+ }
+ remaining -= type;
+
+ if (remaining.isEmpty() && type != JSVAL_TYPE_OBJECT) {
+ // We can skip the check for the last remaining type, unless the type is
+ // JSVAL_TYPE_OBJECT, which may have to go through the OOL path.
+#ifdef DEBUG
+ emitTypeOfCheck(type, tag, output, &done, ool->entry());
+ masm.assumeUnreachable("Unexpected Value type in visitTypeOfV");
+#else
+ emitTypeOfJSType(type, output);
+#endif
+ } else {
+ emitTypeOfCheck(type, tag, output, &done, ool->entry());
+ }
+ }
+ MOZ_ASSERT(remaining.isEmpty());
+
+ masm.bind(&done);
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::emitTypeOfObject(Register obj, Register output,
+ Label* done) {
+ Label slowCheck, isObject, isCallable, isUndefined;
+ masm.typeOfObject(obj, output, &slowCheck, &isObject, &isCallable,
+ &isUndefined);
+
+ masm.bind(&isCallable);
+ masm.move32(Imm32(JSTYPE_FUNCTION), output);
+ masm.jump(done);
+
+ masm.bind(&isUndefined);
+ masm.move32(Imm32(JSTYPE_UNDEFINED), output);
+ masm.jump(done);
+
+ masm.bind(&isObject);
+ masm.move32(Imm32(JSTYPE_OBJECT), output);
+ masm.jump(done);
+
+ masm.bind(&slowCheck);
+
+ saveVolatile(output);
+ using Fn = JSType (*)(JSObject*);
+ masm.setupAlignedABICall();
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, js::TypeOfObject>();
+ masm.storeCallInt32Result(output);
+ restoreVolatile(output);
+}
+
+void CodeGenerator::visitOutOfLineTypeOfV(OutOfLineTypeOfV* ool) {
+ LTypeOfV* ins = ool->ins();
+
+ ValueOperand input = ToValue(ins, LTypeOfV::InputIndex);
+ Register temp = ToTempUnboxRegister(ins->temp0());
+ Register output = ToRegister(ins->output());
+
+ Register obj = masm.extractObject(input, temp);
+ emitTypeOfObject(obj, output, ool->rejoin());
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitTypeOfO(LTypeOfO* lir) {
+ Register obj = ToRegister(lir->object());
+ Register output = ToRegister(lir->output());
+
+ Label done;
+ emitTypeOfObject(obj, output, &done);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitTypeOfName(LTypeOfName* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+#ifdef DEBUG
+ Label ok;
+ masm.branch32(Assembler::Below, input, Imm32(JSTYPE_LIMIT), &ok);
+ masm.assumeUnreachable("bad JSType");
+ masm.bind(&ok);
+#endif
+
+ static_assert(JSTYPE_UNDEFINED == 0);
+
+ masm.movePtr(ImmPtr(&gen->runtime->names().undefined), output);
+ masm.loadPtr(BaseIndex(output, input, ScalePointer), output);
+}
+
+class OutOfLineTypeOfIsNonPrimitiveV : public OutOfLineCodeBase<CodeGenerator> {
+ LTypeOfIsNonPrimitiveV* ins_;
+
+ public:
+ explicit OutOfLineTypeOfIsNonPrimitiveV(LTypeOfIsNonPrimitiveV* ins)
+ : ins_(ins) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineTypeOfIsNonPrimitiveV(this);
+ }
+ auto* ins() const { return ins_; }
+};
+
+class OutOfLineTypeOfIsNonPrimitiveO : public OutOfLineCodeBase<CodeGenerator> {
+ LTypeOfIsNonPrimitiveO* ins_;
+
+ public:
+ explicit OutOfLineTypeOfIsNonPrimitiveO(LTypeOfIsNonPrimitiveO* ins)
+ : ins_(ins) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineTypeOfIsNonPrimitiveO(this);
+ }
+ auto* ins() const { return ins_; }
+};
+
+void CodeGenerator::emitTypeOfIsObjectOOL(MTypeOfIs* mir, Register obj,
+ Register output) {
+ saveVolatile(output);
+ using Fn = JSType (*)(JSObject*);
+ masm.setupAlignedABICall();
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, js::TypeOfObject>();
+ masm.storeCallInt32Result(output);
+ restoreVolatile(output);
+
+ auto cond = JSOpToCondition(mir->jsop(), /* isSigned = */ false);
+ masm.cmp32Set(cond, output, Imm32(mir->jstype()), output);
+}
+
+void CodeGenerator::visitOutOfLineTypeOfIsNonPrimitiveV(
+ OutOfLineTypeOfIsNonPrimitiveV* ool) {
+ auto* ins = ool->ins();
+ ValueOperand input = ToValue(ins, LTypeOfIsNonPrimitiveV::InputIndex);
+ Register output = ToRegister(ins->output());
+ Register temp = ToTempUnboxRegister(ins->temp0());
+
+ Register obj = masm.extractObject(input, temp);
+
+ emitTypeOfIsObjectOOL(ins->mir(), obj, output);
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineTypeOfIsNonPrimitiveO(
+ OutOfLineTypeOfIsNonPrimitiveO* ool) {
+ auto* ins = ool->ins();
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ emitTypeOfIsObjectOOL(ins->mir(), input, output);
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::emitTypeOfIsObject(MTypeOfIs* mir, Register obj,
+ Register output, Label* success,
+ Label* fail, Label* slowCheck) {
+ Label* isObject = fail;
+ Label* isFunction = fail;
+ Label* isUndefined = fail;
+
+ switch (mir->jstype()) {
+ case JSTYPE_UNDEFINED:
+ isUndefined = success;
+ break;
+
+ case JSTYPE_OBJECT:
+ isObject = success;
+ break;
+
+ case JSTYPE_FUNCTION:
+ isFunction = success;
+ break;
+
+ case JSTYPE_STRING:
+ case JSTYPE_NUMBER:
+ case JSTYPE_BOOLEAN:
+ case JSTYPE_SYMBOL:
+ case JSTYPE_BIGINT:
+#ifdef ENABLE_RECORD_TUPLE
+ case JSTYPE_RECORD:
+ case JSTYPE_TUPLE:
+#endif
+ case JSTYPE_LIMIT:
+ MOZ_CRASH("Primitive type");
+ }
+
+ masm.typeOfObject(obj, output, slowCheck, isObject, isFunction, isUndefined);
+
+ auto op = mir->jsop();
+
+ Label done;
+ masm.bind(fail);
+ masm.move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), output);
+ masm.jump(&done);
+ masm.bind(success);
+ masm.move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq), output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitTypeOfIsNonPrimitiveV(LTypeOfIsNonPrimitiveV* lir) {
+ ValueOperand input = ToValue(lir, LTypeOfIsNonPrimitiveV::InputIndex);
+ Register output = ToRegister(lir->output());
+ Register temp = ToTempUnboxRegister(lir->temp0());
+
+ auto* mir = lir->mir();
+
+ auto* ool = new (alloc()) OutOfLineTypeOfIsNonPrimitiveV(lir);
+ addOutOfLineCode(ool, mir);
+
+ Label success, fail;
+
+ switch (mir->jstype()) {
+ case JSTYPE_UNDEFINED: {
+ ScratchTagScope tag(masm, input);
+ masm.splitTagForTest(input, tag);
+
+ masm.branchTestUndefined(Assembler::Equal, tag, &success);
+ masm.branchTestObject(Assembler::NotEqual, tag, &fail);
+ break;
+ }
+
+ case JSTYPE_OBJECT: {
+ ScratchTagScope tag(masm, input);
+ masm.splitTagForTest(input, tag);
+
+ masm.branchTestNull(Assembler::Equal, tag, &success);
+ masm.branchTestObject(Assembler::NotEqual, tag, &fail);
+ break;
+ }
+
+ case JSTYPE_FUNCTION: {
+ masm.branchTestObject(Assembler::NotEqual, input, &fail);
+ break;
+ }
+
+ case JSTYPE_STRING:
+ case JSTYPE_NUMBER:
+ case JSTYPE_BOOLEAN:
+ case JSTYPE_SYMBOL:
+ case JSTYPE_BIGINT:
+#ifdef ENABLE_RECORD_TUPLE
+ case JSTYPE_RECORD:
+ case JSTYPE_TUPLE:
+#endif
+ case JSTYPE_LIMIT:
+ MOZ_CRASH("Primitive type");
+ }
+
+ Register obj = masm.extractObject(input, temp);
+
+ emitTypeOfIsObject(mir, obj, output, &success, &fail, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitTypeOfIsNonPrimitiveO(LTypeOfIsNonPrimitiveO* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ auto* mir = lir->mir();
+
+ auto* ool = new (alloc()) OutOfLineTypeOfIsNonPrimitiveO(lir);
+ addOutOfLineCode(ool, mir);
+
+ Label success, fail;
+ emitTypeOfIsObject(mir, input, output, &success, &fail, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitTypeOfIsPrimitive(LTypeOfIsPrimitive* lir) {
+ ValueOperand input = ToValue(lir, LTypeOfIsPrimitive::InputIndex);
+ Register output = ToRegister(lir->output());
+
+ auto* mir = lir->mir();
+ auto cond = JSOpToCondition(mir->jsop(), /* isSigned = */ false);
+
+ switch (mir->jstype()) {
+ case JSTYPE_STRING:
+ masm.testStringSet(cond, input, output);
+ break;
+ case JSTYPE_NUMBER:
+ masm.testNumberSet(cond, input, output);
+ break;
+ case JSTYPE_BOOLEAN:
+ masm.testBooleanSet(cond, input, output);
+ break;
+ case JSTYPE_SYMBOL:
+ masm.testSymbolSet(cond, input, output);
+ break;
+ case JSTYPE_BIGINT:
+ masm.testBigIntSet(cond, input, output);
+ break;
+
+ case JSTYPE_UNDEFINED:
+ case JSTYPE_OBJECT:
+ case JSTYPE_FUNCTION:
+#ifdef ENABLE_RECORD_TUPLE
+ case JSTYPE_RECORD:
+ case JSTYPE_TUPLE:
+#endif
+ case JSTYPE_LIMIT:
+ MOZ_CRASH("Non-primitive type");
+ }
+}
+
+void CodeGenerator::visitToAsyncIter(LToAsyncIter* lir) {
+ pushArg(ToValue(lir, LToAsyncIter::NextMethodIndex));
+ pushArg(ToRegister(lir->iterator()));
+
+ using Fn = JSObject* (*)(JSContext*, HandleObject, HandleValue);
+ callVM<Fn, js::CreateAsyncFromSyncIterator>(lir);
+}
+
+void CodeGenerator::visitToPropertyKeyCache(LToPropertyKeyCache* lir) {
+ LiveRegisterSet liveRegs = lir->safepoint()->liveRegs();
+ ValueOperand input = ToValue(lir, LToPropertyKeyCache::InputIndex);
+ ValueOperand output = ToOutValue(lir);
+
+ IonToPropertyKeyIC ic(liveRegs, input, output);
+ addIC(lir, allocateIC(ic));
+}
+
+void CodeGenerator::visitLoadElementV(LLoadElementV* load) {
+ Register elements = ToRegister(load->elements());
+ const ValueOperand out = ToOutValue(load);
+
+ if (load->index()->isConstant()) {
+ NativeObject::elementsSizeMustNotOverflow();
+ int32_t offset = ToInt32(load->index()) * sizeof(Value);
+ masm.loadValue(Address(elements, offset), out);
+ } else {
+ masm.loadValue(BaseObjectElementIndex(elements, ToRegister(load->index())),
+ out);
+ }
+
+ Label testMagic;
+ masm.branchTestMagic(Assembler::Equal, out, &testMagic);
+ bailoutFrom(&testMagic, load->snapshot());
+}
+
+void CodeGenerator::visitLoadElementHole(LLoadElementHole* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register index = ToRegister(lir->index());
+ Register initLength = ToRegister(lir->initLength());
+ const ValueOperand out = ToOutValue(lir);
+
+ const MLoadElementHole* mir = lir->mir();
+
+ // If the index is out of bounds, load |undefined|. Otherwise, load the
+ // value.
+ Label outOfBounds, done;
+ masm.spectreBoundsCheck32(index, initLength, out.scratchReg(), &outOfBounds);
+
+ masm.loadValue(BaseObjectElementIndex(elements, index), out);
+
+ // If the value wasn't a hole, we're done. Otherwise, we'll load undefined.
+ masm.branchTestMagic(Assembler::NotEqual, out, &done);
+
+ if (mir->needsNegativeIntCheck()) {
+ Label loadUndefined;
+ masm.jump(&loadUndefined);
+
+ masm.bind(&outOfBounds);
+
+ bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
+
+ masm.bind(&loadUndefined);
+ } else {
+ masm.bind(&outOfBounds);
+ }
+ masm.moveValue(UndefinedValue(), out);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitLoadUnboxedScalar(LLoadUnboxedScalar* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToTempRegisterOrInvalid(lir->temp0());
+ AnyRegister out = ToAnyRegister(lir->output());
+
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ Label fail;
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.loadFromTypedArray(storageType, source, out, temp, &fail);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.loadFromTypedArray(storageType, source, out, temp, &fail);
+ }
+
+ if (fail.used()) {
+ bailoutFrom(&fail, lir->snapshot());
+ }
+}
+
+void CodeGenerator::visitLoadUnboxedBigInt(LLoadUnboxedBigInt* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ }
+
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitLoadDataViewElement(LLoadDataViewElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ const LAllocation* littleEndian = lir->littleEndian();
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ Register64 temp64 = ToTempRegister64OrInvalid(lir->temp64());
+ AnyRegister out = ToAnyRegister(lir->output());
+
+ const MLoadDataViewElement* mir = lir->mir();
+ Scalar::Type storageType = mir->storageType();
+
+ BaseIndex source(elements, ToRegister(lir->index()), TimesOne);
+
+ bool noSwap = littleEndian->isConstant() &&
+ ToBoolean(littleEndian) == MOZ_LITTLE_ENDIAN();
+
+ // Directly load if no byte swap is needed and the platform supports unaligned
+ // accesses for the access. (Such support is assumed for integer types.)
+ if (noSwap && (!Scalar::isFloatingType(storageType) ||
+ MacroAssembler::SupportsFastUnalignedFPAccesses())) {
+ if (!Scalar::isBigIntType(storageType)) {
+ Label fail;
+ masm.loadFromTypedArray(storageType, source, out, temp, &fail);
+
+ if (fail.used()) {
+ bailoutFrom(&fail, lir->snapshot());
+ }
+ } else {
+ masm.load64(source, temp64);
+
+ emitCreateBigInt(lir, storageType, temp64, out.gpr(), temp);
+ }
+ return;
+ }
+
+ // Load the value into a gpr register.
+ switch (storageType) {
+ case Scalar::Int16:
+ masm.load16UnalignedSignExtend(source, out.gpr());
+ break;
+ case Scalar::Uint16:
+ masm.load16UnalignedZeroExtend(source, out.gpr());
+ break;
+ case Scalar::Int32:
+ masm.load32Unaligned(source, out.gpr());
+ break;
+ case Scalar::Uint32:
+ masm.load32Unaligned(source, out.isFloat() ? temp : out.gpr());
+ break;
+ case Scalar::Float32:
+ masm.load32Unaligned(source, temp);
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.load64Unaligned(source, temp64);
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+
+ if (!noSwap) {
+ // Swap the bytes in the loaded value.
+ Label skip;
+ if (!littleEndian->isConstant()) {
+ masm.branch32(
+ MOZ_LITTLE_ENDIAN() ? Assembler::NotEqual : Assembler::Equal,
+ ToRegister(littleEndian), Imm32(0), &skip);
+ }
+
+ switch (storageType) {
+ case Scalar::Int16:
+ masm.byteSwap16SignExtend(out.gpr());
+ break;
+ case Scalar::Uint16:
+ masm.byteSwap16ZeroExtend(out.gpr());
+ break;
+ case Scalar::Int32:
+ masm.byteSwap32(out.gpr());
+ break;
+ case Scalar::Uint32:
+ masm.byteSwap32(out.isFloat() ? temp : out.gpr());
+ break;
+ case Scalar::Float32:
+ masm.byteSwap32(temp);
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.byteSwap64(temp64);
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+
+ if (skip.used()) {
+ masm.bind(&skip);
+ }
+ }
+
+ // Move the value into the output register.
+ switch (storageType) {
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ break;
+ case Scalar::Uint32:
+ if (out.isFloat()) {
+ masm.convertUInt32ToDouble(temp, out.fpu());
+ } else {
+ // Bail out if the value doesn't fit into a signed int32 value. This
+ // is what allows MLoadDataViewElement to have a type() of
+ // MIRType::Int32 for UInt32 array loads.
+ bailoutTest32(Assembler::Signed, out.gpr(), out.gpr(), lir->snapshot());
+ }
+ break;
+ case Scalar::Float32:
+ masm.moveGPRToFloat32(temp, out.fpu());
+ masm.canonicalizeFloat(out.fpu());
+ break;
+ case Scalar::Float64:
+ masm.moveGPR64ToDouble(temp64, out.fpu());
+ masm.canonicalizeDouble(out.fpu());
+ break;
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ emitCreateBigInt(lir, storageType, temp64, out.gpr(), temp);
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+}
+
+void CodeGenerator::visitLoadTypedArrayElementHole(
+ LLoadTypedArrayElementHole* lir) {
+ Register object = ToRegister(lir->object());
+ const ValueOperand out = ToOutValue(lir);
+
+ // Load the length.
+ Register scratch = out.scratchReg();
+ Register scratch2 = ToRegister(lir->temp0());
+ Register index = ToRegister(lir->index());
+ masm.loadArrayBufferViewLengthIntPtr(object, scratch);
+
+ // Load undefined if index >= length.
+ Label outOfBounds, done;
+ masm.spectreBoundsCheckPtr(index, scratch, scratch2, &outOfBounds);
+
+ // Load the elements vector.
+ masm.loadPtr(Address(object, ArrayBufferViewObject::dataOffset()), scratch);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ Label fail;
+ BaseIndex source(scratch, index, ScaleFromScalarType(arrayType));
+ MacroAssembler::Uint32Mode uint32Mode =
+ lir->mir()->forceDouble() ? MacroAssembler::Uint32Mode::ForceDouble
+ : MacroAssembler::Uint32Mode::FailOnDouble;
+ masm.loadFromTypedArray(arrayType, source, out, uint32Mode, out.scratchReg(),
+ &fail);
+ masm.jump(&done);
+
+ masm.bind(&outOfBounds);
+ masm.moveValue(UndefinedValue(), out);
+
+ if (fail.used()) {
+ bailoutFrom(&fail, lir->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitLoadTypedArrayElementHoleBigInt(
+ LLoadTypedArrayElementHoleBigInt* lir) {
+ Register object = ToRegister(lir->object());
+ const ValueOperand out = ToOutValue(lir);
+
+ // On x86 there are not enough registers. In that case reuse the output's
+ // type register as temporary.
+#ifdef JS_CODEGEN_X86
+ MOZ_ASSERT(lir->temp()->isBogusTemp());
+ Register temp = out.typeReg();
+#else
+ Register temp = ToRegister(lir->temp());
+#endif
+ Register64 temp64 = ToRegister64(lir->temp64());
+
+ // Load the length.
+ Register scratch = out.scratchReg();
+ Register index = ToRegister(lir->index());
+ masm.loadArrayBufferViewLengthIntPtr(object, scratch);
+
+ // Load undefined if index >= length.
+ Label outOfBounds, done;
+ masm.spectreBoundsCheckPtr(index, scratch, temp, &outOfBounds);
+
+ // Load the elements vector.
+ masm.loadPtr(Address(object, ArrayBufferViewObject::dataOffset()), scratch);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ BaseIndex source(scratch, index, ScaleFromScalarType(arrayType));
+ masm.load64(source, temp64);
+
+ Register bigInt = out.scratchReg();
+ emitCreateBigInt(lir, arrayType, temp64, bigInt, temp);
+
+ masm.tagValue(JSVAL_TYPE_BIGINT, bigInt, out);
+ masm.jump(&done);
+
+ masm.bind(&outOfBounds);
+ masm.moveValue(UndefinedValue(), out);
+
+ masm.bind(&done);
+}
+
+template <SwitchTableType tableType>
+class OutOfLineSwitch : public OutOfLineCodeBase<CodeGenerator> {
+ using LabelsVector = Vector<Label, 0, JitAllocPolicy>;
+ using CodeLabelsVector = Vector<CodeLabel, 0, JitAllocPolicy>;
+ LabelsVector labels_;
+ CodeLabelsVector codeLabels_;
+ CodeLabel start_;
+ bool isOutOfLine_;
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineSwitch(this);
+ }
+
+ public:
+ explicit OutOfLineSwitch(TempAllocator& alloc)
+ : labels_(alloc), codeLabels_(alloc), isOutOfLine_(false) {}
+
+ CodeLabel* start() { return &start_; }
+
+ CodeLabelsVector& codeLabels() { return codeLabels_; }
+ LabelsVector& labels() { return labels_; }
+
+ void jumpToCodeEntries(MacroAssembler& masm, Register index, Register temp) {
+ Register base;
+ if (tableType == SwitchTableType::Inline) {
+#if defined(JS_CODEGEN_ARM)
+ base = ::js::jit::pc;
+#else
+ MOZ_CRASH("NYI: SwitchTableType::Inline");
+#endif
+ } else {
+#if defined(JS_CODEGEN_ARM)
+ MOZ_CRASH("NYI: SwitchTableType::OutOfLine");
+#else
+ masm.mov(start(), temp);
+ base = temp;
+#endif
+ }
+ BaseIndex jumpTarget(base, index, ScalePointer);
+ masm.branchToComputedAddress(jumpTarget);
+ }
+
+ // Register an entry in the switch table.
+ void addTableEntry(MacroAssembler& masm) {
+ if ((!isOutOfLine_ && tableType == SwitchTableType::Inline) ||
+ (isOutOfLine_ && tableType == SwitchTableType::OutOfLine)) {
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ masm.propagateOOM(codeLabels_.append(std::move(cl)));
+ }
+ }
+ // Register the code, to which the table will jump to.
+ void addCodeEntry(MacroAssembler& masm) {
+ Label entry;
+ masm.bind(&entry);
+ masm.propagateOOM(labels_.append(std::move(entry)));
+ }
+
+ void setOutOfLine() { isOutOfLine_ = true; }
+};
+
+template <SwitchTableType tableType>
+void CodeGenerator::visitOutOfLineSwitch(
+ OutOfLineSwitch<tableType>* jumpTable) {
+ jumpTable->setOutOfLine();
+ auto& labels = jumpTable->labels();
+
+ if (tableType == SwitchTableType::OutOfLine) {
+#if defined(JS_CODEGEN_ARM)
+ MOZ_CRASH("NYI: SwitchTableType::OutOfLine");
+#elif defined(JS_CODEGEN_NONE)
+ MOZ_CRASH();
+#else
+
+# if defined(JS_CODEGEN_ARM64)
+ AutoForbidPoolsAndNops afp(
+ &masm,
+ (labels.length() + 1) * (sizeof(void*) / vixl::kInstructionSize));
+# endif
+
+ masm.haltingAlign(sizeof(void*));
+
+ // Bind the address of the jump table and reserve the space for code
+ // pointers to jump in the newly generated code.
+ masm.bind(jumpTable->start());
+ masm.addCodeLabel(*jumpTable->start());
+ for (size_t i = 0, e = labels.length(); i < e; i++) {
+ jumpTable->addTableEntry(masm);
+ }
+#endif
+ }
+
+ // Register all reserved pointers of the jump table to target labels. The
+ // entries of the jump table need to be absolute addresses and thus must be
+ // patched after codegen is finished.
+ auto& codeLabels = jumpTable->codeLabels();
+ for (size_t i = 0, e = codeLabels.length(); i < e; i++) {
+ auto& cl = codeLabels[i];
+ cl.target()->bind(labels[i].offset());
+ masm.addCodeLabel(cl);
+ }
+}
+
+template void CodeGenerator::visitOutOfLineSwitch(
+ OutOfLineSwitch<SwitchTableType::Inline>* jumpTable);
+template void CodeGenerator::visitOutOfLineSwitch(
+ OutOfLineSwitch<SwitchTableType::OutOfLine>* jumpTable);
+
+template <typename T>
+static inline void StoreToTypedArray(MacroAssembler& masm,
+ Scalar::Type writeType,
+ const LAllocation* value, const T& dest) {
+ if (writeType == Scalar::Float32 || writeType == Scalar::Float64) {
+ masm.storeToTypedFloatArray(writeType, ToFloatRegister(value), dest);
+ } else {
+ if (value->isConstant()) {
+ masm.storeToTypedIntArray(writeType, Imm32(ToInt32(value)), dest);
+ } else {
+ masm.storeToTypedIntArray(writeType, ToRegister(value), dest);
+ }
+ }
+}
+
+void CodeGenerator::visitStoreUnboxedScalar(LStoreUnboxedScalar* lir) {
+ Register elements = ToRegister(lir->elements());
+ const LAllocation* value = lir->value();
+
+ const MStoreUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type writeType = mir->writeType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ StoreToTypedArray(masm, writeType, value, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ StoreToTypedArray(masm, writeType, value, dest);
+ }
+}
+
+void CodeGenerator::visitStoreUnboxedBigInt(LStoreUnboxedBigInt* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp = ToRegister64(lir->temp());
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.storeToTypedBigIntArray(writeType, temp, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.storeToTypedBigIntArray(writeType, temp, dest);
+ }
+}
+
+void CodeGenerator::visitStoreDataViewElement(LStoreDataViewElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ const LAllocation* value = lir->value();
+ const LAllocation* littleEndian = lir->littleEndian();
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ Register64 temp64 = ToTempRegister64OrInvalid(lir->temp64());
+
+ const MStoreDataViewElement* mir = lir->mir();
+ Scalar::Type writeType = mir->writeType();
+
+ BaseIndex dest(elements, ToRegister(lir->index()), TimesOne);
+
+ bool noSwap = littleEndian->isConstant() &&
+ ToBoolean(littleEndian) == MOZ_LITTLE_ENDIAN();
+
+ // Directly store if no byte swap is needed and the platform supports
+ // unaligned accesses for the access. (Such support is assumed for integer
+ // types.)
+ if (noSwap && (!Scalar::isFloatingType(writeType) ||
+ MacroAssembler::SupportsFastUnalignedFPAccesses())) {
+ if (!Scalar::isBigIntType(writeType)) {
+ StoreToTypedArray(masm, writeType, value, dest);
+ } else {
+ masm.loadBigInt64(ToRegister(value), temp64);
+ masm.storeToTypedBigIntArray(writeType, temp64, dest);
+ }
+ return;
+ }
+
+ // Load the value into a gpr register.
+ switch (writeType) {
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ if (value->isConstant()) {
+ masm.move32(Imm32(ToInt32(value)), temp);
+ } else {
+ masm.move32(ToRegister(value), temp);
+ }
+ break;
+ case Scalar::Float32: {
+ FloatRegister fvalue = ToFloatRegister(value);
+ masm.canonicalizeFloatIfDeterministic(fvalue);
+ masm.moveFloat32ToGPR(fvalue, temp);
+ break;
+ }
+ case Scalar::Float64: {
+ FloatRegister fvalue = ToFloatRegister(value);
+ masm.canonicalizeDoubleIfDeterministic(fvalue);
+ masm.moveDoubleToGPR64(fvalue, temp64);
+ break;
+ }
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.loadBigInt64(ToRegister(value), temp64);
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+
+ if (!noSwap) {
+ // Swap the bytes in the loaded value.
+ Label skip;
+ if (!littleEndian->isConstant()) {
+ masm.branch32(
+ MOZ_LITTLE_ENDIAN() ? Assembler::NotEqual : Assembler::Equal,
+ ToRegister(littleEndian), Imm32(0), &skip);
+ }
+
+ switch (writeType) {
+ case Scalar::Int16:
+ masm.byteSwap16SignExtend(temp);
+ break;
+ case Scalar::Uint16:
+ masm.byteSwap16ZeroExtend(temp);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ masm.byteSwap32(temp);
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.byteSwap64(temp64);
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+
+ if (skip.used()) {
+ masm.bind(&skip);
+ }
+ }
+
+ // Store the value into the destination.
+ switch (writeType) {
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ masm.store16Unaligned(temp, dest);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ masm.store32Unaligned(temp, dest);
+ break;
+ case Scalar::Float64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ masm.store64Unaligned(temp64, dest);
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+}
+
+void CodeGenerator::visitStoreTypedArrayElementHole(
+ LStoreTypedArrayElementHole* lir) {
+ Register elements = ToRegister(lir->elements());
+ const LAllocation* value = lir->value();
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ Register index = ToRegister(lir->index());
+ const LAllocation* length = lir->length();
+ Register spectreTemp = ToTempRegisterOrInvalid(lir->temp0());
+
+ Label skip;
+ if (length->isRegister()) {
+ masm.spectreBoundsCheckPtr(index, ToRegister(length), spectreTemp, &skip);
+ } else {
+ masm.spectreBoundsCheckPtr(index, ToAddress(length), spectreTemp, &skip);
+ }
+
+ BaseIndex dest(elements, index, ScaleFromScalarType(arrayType));
+ StoreToTypedArray(masm, arrayType, value, dest);
+
+ masm.bind(&skip);
+}
+
+void CodeGenerator::visitStoreTypedArrayElementHoleBigInt(
+ LStoreTypedArrayElementHoleBigInt* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp = ToRegister64(lir->temp());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ Register index = ToRegister(lir->index());
+ const LAllocation* length = lir->length();
+ Register spectreTemp = temp.scratchReg();
+
+ Label skip;
+ if (length->isRegister()) {
+ masm.spectreBoundsCheckPtr(index, ToRegister(length), spectreTemp, &skip);
+ } else {
+ masm.spectreBoundsCheckPtr(index, ToAddress(length), spectreTemp, &skip);
+ }
+
+ masm.loadBigInt64(value, temp);
+
+ BaseIndex dest(elements, index, ScaleFromScalarType(arrayType));
+ masm.storeToTypedBigIntArray(arrayType, temp, dest);
+
+ masm.bind(&skip);
+}
+
+void CodeGenerator::visitAtomicIsLockFree(LAtomicIsLockFree* lir) {
+ Register value = ToRegister(lir->value());
+ Register output = ToRegister(lir->output());
+
+ masm.atomicIsLockFreeJS(value, output);
+}
+
+void CodeGenerator::visitClampIToUint8(LClampIToUint8* lir) {
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(output == ToRegister(lir->input()));
+ masm.clampIntToUint8(output);
+}
+
+void CodeGenerator::visitClampDToUint8(LClampDToUint8* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ masm.clampDoubleToUint8(input, output);
+}
+
+void CodeGenerator::visitClampVToUint8(LClampVToUint8* lir) {
+ ValueOperand operand = ToValue(lir, LClampVToUint8::InputIndex);
+ FloatRegister tempFloat = ToFloatRegister(lir->temp0());
+ Register output = ToRegister(lir->output());
+
+ using Fn = bool (*)(JSContext*, JSString*, double*);
+ OutOfLineCode* oolString = oolCallVM<Fn, StringToNumber>(
+ lir, ArgList(output), StoreFloatRegisterTo(tempFloat));
+ Label* stringEntry = oolString->entry();
+ Label* stringRejoin = oolString->rejoin();
+
+ Label fails;
+ masm.clampValueToUint8(operand, stringEntry, stringRejoin, output, tempFloat,
+ output, &fails);
+
+ bailoutFrom(&fails, lir->snapshot());
+}
+
+void CodeGenerator::visitInCache(LInCache* ins) {
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+
+ ConstantOrRegister key =
+ toConstantOrRegister(ins, LInCache::LhsIndex, ins->mir()->key()->type());
+ Register object = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+ Register temp = ToRegister(ins->temp0());
+
+ IonInIC cache(liveRegs, key, object, output, temp);
+ addIC(ins, allocateIC(cache));
+}
+
+void CodeGenerator::visitInArray(LInArray* lir) {
+ const MInArray* mir = lir->mir();
+ Register elements = ToRegister(lir->elements());
+ Register initLength = ToRegister(lir->initLength());
+ Register output = ToRegister(lir->output());
+
+ Label falseBranch, done, trueBranch;
+
+ if (lir->index()->isConstant()) {
+ int32_t index = ToInt32(lir->index());
+
+ if (index < 0) {
+ MOZ_ASSERT(mir->needsNegativeIntCheck());
+ bailout(lir->snapshot());
+ return;
+ }
+
+ masm.branch32(Assembler::BelowOrEqual, initLength, Imm32(index),
+ &falseBranch);
+
+ NativeObject::elementsSizeMustNotOverflow();
+ Address address = Address(elements, index * sizeof(Value));
+ masm.branchTestMagic(Assembler::Equal, address, &falseBranch);
+ } else {
+ Register index = ToRegister(lir->index());
+
+ Label negativeIntCheck;
+ Label* failedInitLength = &falseBranch;
+ if (mir->needsNegativeIntCheck()) {
+ failedInitLength = &negativeIntCheck;
+ }
+
+ masm.branch32(Assembler::BelowOrEqual, initLength, index, failedInitLength);
+
+ BaseObjectElementIndex address(elements, index);
+ masm.branchTestMagic(Assembler::Equal, address, &falseBranch);
+
+ if (mir->needsNegativeIntCheck()) {
+ masm.jump(&trueBranch);
+ masm.bind(&negativeIntCheck);
+
+ bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
+
+ masm.jump(&falseBranch);
+ }
+ }
+
+ masm.bind(&trueBranch);
+ masm.move32(Imm32(1), output);
+ masm.jump(&done);
+
+ masm.bind(&falseBranch);
+ masm.move32(Imm32(0), output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGuardElementNotHole(LGuardElementNotHole* lir) {
+ Register elements = ToRegister(lir->elements());
+ const LAllocation* index = lir->index();
+
+ Label testMagic;
+ if (index->isConstant()) {
+ Address address(elements, ToInt32(index) * sizeof(js::Value));
+ masm.branchTestMagic(Assembler::Equal, address, &testMagic);
+ } else {
+ BaseObjectElementIndex address(elements, ToRegister(index));
+ masm.branchTestMagic(Assembler::Equal, address, &testMagic);
+ }
+ bailoutFrom(&testMagic, lir->snapshot());
+}
+
+void CodeGenerator::visitInstanceOfO(LInstanceOfO* ins) {
+ Register protoReg = ToRegister(ins->rhs());
+ emitInstanceOf(ins, protoReg);
+}
+
+void CodeGenerator::visitInstanceOfV(LInstanceOfV* ins) {
+ Register protoReg = ToRegister(ins->rhs());
+ emitInstanceOf(ins, protoReg);
+}
+
+void CodeGenerator::emitInstanceOf(LInstruction* ins, Register protoReg) {
+ // This path implements fun_hasInstance when the function's prototype is
+ // known to be the object in protoReg
+
+ Label done;
+ Register output = ToRegister(ins->getDef(0));
+
+ // If the lhs is a primitive, the result is false.
+ Register objReg;
+ if (ins->isInstanceOfV()) {
+ Label isObject;
+ ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LhsIndex);
+ masm.branchTestObject(Assembler::Equal, lhsValue, &isObject);
+ masm.mov(ImmWord(0), output);
+ masm.jump(&done);
+ masm.bind(&isObject);
+ objReg = masm.extractObject(lhsValue, output);
+ } else {
+ objReg = ToRegister(ins->toInstanceOfO()->lhs());
+ }
+
+ // Crawl the lhs's prototype chain in a loop to search for prototypeObject.
+ // This follows the main loop of js::IsPrototypeOf, though additionally breaks
+ // out of the loop on Proxy::LazyProto.
+
+ // Load the lhs's prototype.
+ masm.loadObjProto(objReg, output);
+
+ Label testLazy;
+ {
+ Label loopPrototypeChain;
+ masm.bind(&loopPrototypeChain);
+
+ // Test for the target prototype object.
+ Label notPrototypeObject;
+ masm.branchPtr(Assembler::NotEqual, output, protoReg, &notPrototypeObject);
+ masm.mov(ImmWord(1), output);
+ masm.jump(&done);
+ masm.bind(&notPrototypeObject);
+
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ // Test for nullptr or Proxy::LazyProto
+ masm.branchPtr(Assembler::BelowOrEqual, output, ImmWord(1), &testLazy);
+
+ // Load the current object's prototype.
+ masm.loadObjProto(output, output);
+
+ masm.jump(&loopPrototypeChain);
+ }
+
+ // Make a VM call if an object with a lazy proto was found on the prototype
+ // chain. This currently occurs only for cross compartment wrappers, which
+ // we do not expect to be compared with non-wrapper functions from this
+ // compartment. Otherwise, we stopped on a nullptr prototype and the output
+ // register is already correct.
+
+ using Fn = bool (*)(JSContext*, HandleObject, JSObject*, bool*);
+ auto* ool = oolCallVM<Fn, IsPrototypeOf>(ins, ArgList(protoReg, objReg),
+ StoreRegisterTo(output));
+
+ // Regenerate the original lhs object for the VM call.
+ Label regenerate, *lazyEntry;
+ if (objReg != output) {
+ lazyEntry = ool->entry();
+ } else {
+ masm.bind(&regenerate);
+ lazyEntry = &regenerate;
+ if (ins->isInstanceOfV()) {
+ ValueOperand lhsValue = ToValue(ins, LInstanceOfV::LhsIndex);
+ objReg = masm.extractObject(lhsValue, output);
+ } else {
+ objReg = ToRegister(ins->toInstanceOfO()->lhs());
+ }
+ MOZ_ASSERT(objReg == output);
+ masm.jump(ool->entry());
+ }
+
+ masm.bind(&testLazy);
+ masm.branchPtr(Assembler::Equal, output, ImmWord(1), lazyEntry);
+
+ masm.bind(&done);
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitInstanceOfCache(LInstanceOfCache* ins) {
+ // The Lowering ensures that RHS is an object, and that LHS is a value.
+ LiveRegisterSet liveRegs = ins->safepoint()->liveRegs();
+ TypedOrValueRegister lhs =
+ TypedOrValueRegister(ToValue(ins, LInstanceOfCache::LHS));
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+
+ IonInstanceOfIC ic(liveRegs, lhs, rhs, output);
+ addIC(ins, allocateIC(ic));
+}
+
+void CodeGenerator::visitGetDOMProperty(LGetDOMProperty* ins) {
+ const Register JSContextReg = ToRegister(ins->getJSContextReg());
+ const Register ObjectReg = ToRegister(ins->getObjectReg());
+ const Register PrivateReg = ToRegister(ins->getPrivReg());
+ const Register ValueReg = ToRegister(ins->getValueReg());
+
+ Label haveValue;
+ if (ins->mir()->valueMayBeInSlot()) {
+ size_t slot = ins->mir()->domMemberSlotIndex();
+ // It's a bit annoying to redo these slot calculations, which duplcate
+ // LSlots and a few other things like that, but I'm not sure there's a
+ // way to reuse those here.
+ //
+ // If this ever gets fixed to work with proxies (by not assuming that
+ // reserved slot indices, which is what domMemberSlotIndex() returns,
+ // match fixed slot indices), we can reenable MGetDOMProperty for
+ // proxies in IonBuilder.
+ if (slot < NativeObject::MAX_FIXED_SLOTS) {
+ masm.loadValue(Address(ObjectReg, NativeObject::getFixedSlotOffset(slot)),
+ JSReturnOperand);
+ } else {
+ // It's a dynamic slot.
+ slot -= NativeObject::MAX_FIXED_SLOTS;
+ // Use PrivateReg as a scratch register for the slots pointer.
+ masm.loadPtr(Address(ObjectReg, NativeObject::offsetOfSlots()),
+ PrivateReg);
+ masm.loadValue(Address(PrivateReg, slot * sizeof(js::Value)),
+ JSReturnOperand);
+ }
+ masm.branchTestUndefined(Assembler::NotEqual, JSReturnOperand, &haveValue);
+ }
+
+ DebugOnly<uint32_t> initialStack = masm.framePushed();
+
+ masm.checkStackAlignment();
+
+ // Make space for the outparam. Pre-initialize it to UndefinedValue so we
+ // can trace it at GC time.
+ masm.Push(UndefinedValue());
+ // We pass the pointer to our out param as an instance of
+ // JSJitGetterCallArgs, since on the binary level it's the same thing.
+ static_assert(sizeof(JSJitGetterCallArgs) == sizeof(Value*));
+ masm.moveStackPtrTo(ValueReg);
+
+ masm.Push(ObjectReg);
+
+ LoadDOMPrivate(masm, ObjectReg, PrivateReg, ins->mir()->objectKind());
+
+ // Rooting will happen at GC time.
+ masm.moveStackPtrTo(ObjectReg);
+
+ Realm* getterRealm = ins->mir()->getterRealm();
+ if (gen->realm->realmPtr() != getterRealm) {
+ // We use JSContextReg as scratch register here.
+ masm.switchToRealm(getterRealm, JSContextReg);
+ }
+
+ uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg);
+ masm.loadJSContext(JSContextReg);
+ masm.enterFakeExitFrame(JSContextReg, JSContextReg,
+ ExitFrameType::IonDOMGetter);
+
+ markSafepointAt(safepointOffset, ins);
+
+ masm.setupAlignedABICall();
+ masm.loadJSContext(JSContextReg);
+ masm.passABIArg(JSContextReg);
+ masm.passABIArg(ObjectReg);
+ masm.passABIArg(PrivateReg);
+ masm.passABIArg(ValueReg);
+ masm.callWithABI(DynamicFunction<JSJitGetterOp>(ins->mir()->fun()),
+ MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ if (ins->mir()->isInfallible()) {
+ masm.loadValue(Address(masm.getStackPointer(),
+ IonDOMExitFrameLayout::offsetOfResult()),
+ JSReturnOperand);
+ } else {
+ masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
+
+ masm.loadValue(Address(masm.getStackPointer(),
+ IonDOMExitFrameLayout::offsetOfResult()),
+ JSReturnOperand);
+ }
+
+ // Switch back to the current realm if needed. Note: if the getter threw an
+ // exception, the exception handler will do this.
+ if (gen->realm->realmPtr() != getterRealm) {
+ static_assert(!JSReturnOperand.aliases(ReturnReg),
+ "Clobbering ReturnReg should not affect the return value");
+ masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
+ }
+
+ // Until C++ code is instrumented against Spectre, prevent speculative
+ // execution from returning any private data.
+ if (JitOptions.spectreJitToCxxCalls && ins->mir()->hasLiveDefUses()) {
+ masm.speculationBarrier();
+ }
+
+ masm.adjustStack(IonDOMExitFrameLayout::Size());
+
+ masm.bind(&haveValue);
+
+ MOZ_ASSERT(masm.framePushed() == initialStack);
+}
+
+void CodeGenerator::visitGetDOMMemberV(LGetDOMMemberV* ins) {
+ // It's simpler to duplicate visitLoadFixedSlotV here than it is to try to
+ // use an LLoadFixedSlotV or some subclass of it for this case: that would
+ // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then
+ // we'd have to duplicate a bunch of stuff we now get for free from
+ // MGetDOMProperty.
+ //
+ // If this ever gets fixed to work with proxies (by not assuming that
+ // reserved slot indices, which is what domMemberSlotIndex() returns,
+ // match fixed slot indices), we can reenable MGetDOMMember for
+ // proxies in IonBuilder.
+ Register object = ToRegister(ins->object());
+ size_t slot = ins->mir()->domMemberSlotIndex();
+ ValueOperand result = ToOutValue(ins);
+
+ masm.loadValue(Address(object, NativeObject::getFixedSlotOffset(slot)),
+ result);
+}
+
+void CodeGenerator::visitGetDOMMemberT(LGetDOMMemberT* ins) {
+ // It's simpler to duplicate visitLoadFixedSlotT here than it is to try to
+ // use an LLoadFixedSlotT or some subclass of it for this case: that would
+ // require us to have MGetDOMMember inherit from MLoadFixedSlot, and then
+ // we'd have to duplicate a bunch of stuff we now get for free from
+ // MGetDOMProperty.
+ //
+ // If this ever gets fixed to work with proxies (by not assuming that
+ // reserved slot indices, which is what domMemberSlotIndex() returns,
+ // match fixed slot indices), we can reenable MGetDOMMember for
+ // proxies in IonBuilder.
+ Register object = ToRegister(ins->object());
+ size_t slot = ins->mir()->domMemberSlotIndex();
+ AnyRegister result = ToAnyRegister(ins->getDef(0));
+ MIRType type = ins->mir()->type();
+
+ masm.loadUnboxedValue(Address(object, NativeObject::getFixedSlotOffset(slot)),
+ type, result);
+}
+
+void CodeGenerator::visitSetDOMProperty(LSetDOMProperty* ins) {
+ const Register JSContextReg = ToRegister(ins->getJSContextReg());
+ const Register ObjectReg = ToRegister(ins->getObjectReg());
+ const Register PrivateReg = ToRegister(ins->getPrivReg());
+ const Register ValueReg = ToRegister(ins->getValueReg());
+
+ DebugOnly<uint32_t> initialStack = masm.framePushed();
+
+ masm.checkStackAlignment();
+
+ // Push the argument. Rooting will happen at GC time.
+ ValueOperand argVal = ToValue(ins, LSetDOMProperty::Value);
+ masm.Push(argVal);
+ // We pass the pointer to our out param as an instance of
+ // JSJitGetterCallArgs, since on the binary level it's the same thing.
+ static_assert(sizeof(JSJitSetterCallArgs) == sizeof(Value*));
+ masm.moveStackPtrTo(ValueReg);
+
+ masm.Push(ObjectReg);
+
+ LoadDOMPrivate(masm, ObjectReg, PrivateReg, ins->mir()->objectKind());
+
+ // Rooting will happen at GC time.
+ masm.moveStackPtrTo(ObjectReg);
+
+ Realm* setterRealm = ins->mir()->setterRealm();
+ if (gen->realm->realmPtr() != setterRealm) {
+ // We use JSContextReg as scratch register here.
+ masm.switchToRealm(setterRealm, JSContextReg);
+ }
+
+ uint32_t safepointOffset = masm.buildFakeExitFrame(JSContextReg);
+ masm.loadJSContext(JSContextReg);
+ masm.enterFakeExitFrame(JSContextReg, JSContextReg,
+ ExitFrameType::IonDOMSetter);
+
+ markSafepointAt(safepointOffset, ins);
+
+ masm.setupAlignedABICall();
+ masm.loadJSContext(JSContextReg);
+ masm.passABIArg(JSContextReg);
+ masm.passABIArg(ObjectReg);
+ masm.passABIArg(PrivateReg);
+ masm.passABIArg(ValueReg);
+ masm.callWithABI(DynamicFunction<JSJitSetterOp>(ins->mir()->fun()),
+ MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
+
+ // Switch back to the current realm if needed. Note: if the setter threw an
+ // exception, the exception handler will do this.
+ if (gen->realm->realmPtr() != setterRealm) {
+ masm.switchToRealm(gen->realm->realmPtr(), ReturnReg);
+ }
+
+ masm.adjustStack(IonDOMExitFrameLayout::Size());
+
+ MOZ_ASSERT(masm.framePushed() == initialStack);
+}
+
+void CodeGenerator::visitLoadDOMExpandoValue(LLoadDOMExpandoValue* ins) {
+ Register proxy = ToRegister(ins->proxy());
+ ValueOperand out = ToOutValue(ins);
+
+ masm.loadPtr(Address(proxy, ProxyObject::offsetOfReservedSlots()),
+ out.scratchReg());
+ masm.loadValue(Address(out.scratchReg(),
+ js::detail::ProxyReservedSlots::offsetOfPrivateSlot()),
+ out);
+}
+
+void CodeGenerator::visitLoadDOMExpandoValueGuardGeneration(
+ LLoadDOMExpandoValueGuardGeneration* ins) {
+ Register proxy = ToRegister(ins->proxy());
+ ValueOperand out = ToOutValue(ins);
+
+ Label bail;
+ masm.loadDOMExpandoValueGuardGeneration(proxy, out,
+ ins->mir()->expandoAndGeneration(),
+ ins->mir()->generation(), &bail);
+ bailoutFrom(&bail, ins->snapshot());
+}
+
+void CodeGenerator::visitLoadDOMExpandoValueIgnoreGeneration(
+ LLoadDOMExpandoValueIgnoreGeneration* ins) {
+ Register proxy = ToRegister(ins->proxy());
+ ValueOperand out = ToOutValue(ins);
+
+ masm.loadPtr(Address(proxy, ProxyObject::offsetOfReservedSlots()),
+ out.scratchReg());
+
+ // Load the ExpandoAndGeneration* from the PrivateValue.
+ masm.loadPrivate(
+ Address(out.scratchReg(),
+ js::detail::ProxyReservedSlots::offsetOfPrivateSlot()),
+ out.scratchReg());
+
+ // Load expandoAndGeneration->expando into the output Value register.
+ masm.loadValue(
+ Address(out.scratchReg(), ExpandoAndGeneration::offsetOfExpando()), out);
+}
+
+void CodeGenerator::visitGuardDOMExpandoMissingOrGuardShape(
+ LGuardDOMExpandoMissingOrGuardShape* ins) {
+ Register temp = ToRegister(ins->temp0());
+ ValueOperand input =
+ ToValue(ins, LGuardDOMExpandoMissingOrGuardShape::InputIndex);
+
+ Label done;
+ masm.branchTestUndefined(Assembler::Equal, input, &done);
+
+ masm.debugAssertIsObject(input);
+ masm.unboxObject(input, temp);
+ // The expando object is not used in this case, so we don't need Spectre
+ // mitigations.
+ Label bail;
+ masm.branchTestObjShapeNoSpectreMitigations(Assembler::NotEqual, temp,
+ ins->mir()->shape(), &bail);
+ bailoutFrom(&bail, ins->snapshot());
+
+ masm.bind(&done);
+}
+
+class OutOfLineIsCallable : public OutOfLineCodeBase<CodeGenerator> {
+ Register object_;
+ Register output_;
+
+ public:
+ OutOfLineIsCallable(Register object, Register output)
+ : object_(object), output_(output) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineIsCallable(this);
+ }
+ Register object() const { return object_; }
+ Register output() const { return output_; }
+};
+
+void CodeGenerator::visitIsCallableO(LIsCallableO* ins) {
+ Register object = ToRegister(ins->object());
+ Register output = ToRegister(ins->output());
+
+ OutOfLineIsCallable* ool = new (alloc()) OutOfLineIsCallable(object, output);
+ addOutOfLineCode(ool, ins->mir());
+
+ masm.isCallable(object, output, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitIsCallableV(LIsCallableV* ins) {
+ ValueOperand val = ToValue(ins, LIsCallableV::ObjectIndex);
+ Register output = ToRegister(ins->output());
+ Register temp = ToRegister(ins->temp0());
+
+ Label notObject;
+ masm.fallibleUnboxObject(val, temp, &notObject);
+
+ OutOfLineIsCallable* ool = new (alloc()) OutOfLineIsCallable(temp, output);
+ addOutOfLineCode(ool, ins->mir());
+
+ masm.isCallable(temp, output, ool->entry());
+ masm.jump(ool->rejoin());
+
+ masm.bind(&notObject);
+ masm.move32(Imm32(0), output);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineIsCallable(OutOfLineIsCallable* ool) {
+ Register object = ool->object();
+ Register output = ool->output();
+
+ saveVolatile(output);
+ using Fn = bool (*)(JSObject * obj);
+ masm.setupAlignedABICall();
+ masm.passABIArg(object);
+ masm.callWithABI<Fn, ObjectIsCallable>();
+ masm.storeCallBoolResult(output);
+ restoreVolatile(output);
+ masm.jump(ool->rejoin());
+}
+
+class OutOfLineIsConstructor : public OutOfLineCodeBase<CodeGenerator> {
+ LIsConstructor* ins_;
+
+ public:
+ explicit OutOfLineIsConstructor(LIsConstructor* ins) : ins_(ins) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineIsConstructor(this);
+ }
+ LIsConstructor* ins() const { return ins_; }
+};
+
+void CodeGenerator::visitIsConstructor(LIsConstructor* ins) {
+ Register object = ToRegister(ins->object());
+ Register output = ToRegister(ins->output());
+
+ OutOfLineIsConstructor* ool = new (alloc()) OutOfLineIsConstructor(ins);
+ addOutOfLineCode(ool, ins->mir());
+
+ masm.isConstructor(object, output, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineIsConstructor(OutOfLineIsConstructor* ool) {
+ LIsConstructor* ins = ool->ins();
+ Register object = ToRegister(ins->object());
+ Register output = ToRegister(ins->output());
+
+ saveVolatile(output);
+ using Fn = bool (*)(JSObject * obj);
+ masm.setupAlignedABICall();
+ masm.passABIArg(object);
+ masm.callWithABI<Fn, ObjectIsConstructor>();
+ masm.storeCallBoolResult(output);
+ restoreVolatile(output);
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitIsCrossRealmArrayConstructor(
+ LIsCrossRealmArrayConstructor* ins) {
+ Register object = ToRegister(ins->object());
+ Register output = ToRegister(ins->output());
+
+ masm.setIsCrossRealmArrayConstructor(object, output);
+}
+
+static void EmitObjectIsArray(MacroAssembler& masm, OutOfLineCode* ool,
+ Register obj, Register output,
+ Label* notArray = nullptr) {
+ masm.loadObjClassUnsafe(obj, output);
+
+ Label isArray;
+ masm.branchPtr(Assembler::Equal, output, ImmPtr(&ArrayObject::class_),
+ &isArray);
+
+ // Branch to OOL path if it's a proxy.
+ masm.branchTestClassIsProxy(true, output, ool->entry());
+
+ if (notArray) {
+ masm.bind(notArray);
+ }
+ masm.move32(Imm32(0), output);
+ masm.jump(ool->rejoin());
+
+ masm.bind(&isArray);
+ masm.move32(Imm32(1), output);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitIsArrayO(LIsArrayO* lir) {
+ Register object = ToRegister(lir->object());
+ Register output = ToRegister(lir->output());
+
+ using Fn = bool (*)(JSContext*, HandleObject, bool*);
+ OutOfLineCode* ool = oolCallVM<Fn, js::IsArrayFromJit>(
+ lir, ArgList(object), StoreRegisterTo(output));
+ EmitObjectIsArray(masm, ool, object, output);
+}
+
+void CodeGenerator::visitIsArrayV(LIsArrayV* lir) {
+ ValueOperand val = ToValue(lir, LIsArrayV::ValueIndex);
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ Label notArray;
+ masm.fallibleUnboxObject(val, temp, &notArray);
+
+ using Fn = bool (*)(JSContext*, HandleObject, bool*);
+ OutOfLineCode* ool = oolCallVM<Fn, js::IsArrayFromJit>(
+ lir, ArgList(temp), StoreRegisterTo(output));
+ EmitObjectIsArray(masm, ool, temp, output, &notArray);
+}
+
+void CodeGenerator::visitIsTypedArray(LIsTypedArray* lir) {
+ Register object = ToRegister(lir->object());
+ Register output = ToRegister(lir->output());
+
+ OutOfLineCode* ool = nullptr;
+ if (lir->mir()->isPossiblyWrapped()) {
+ using Fn = bool (*)(JSContext*, JSObject*, bool*);
+ ool = oolCallVM<Fn, jit::IsPossiblyWrappedTypedArray>(
+ lir, ArgList(object), StoreRegisterTo(output));
+ }
+
+ Label notTypedArray;
+ Label done;
+
+ masm.loadObjClassUnsafe(object, output);
+ masm.branchIfClassIsNotTypedArray(output, &notTypedArray);
+
+ masm.move32(Imm32(1), output);
+ masm.jump(&done);
+ masm.bind(&notTypedArray);
+ if (ool) {
+ masm.branchTestClassIsProxy(true, output, ool->entry());
+ }
+ masm.move32(Imm32(0), output);
+ masm.bind(&done);
+ if (ool) {
+ masm.bind(ool->rejoin());
+ }
+}
+
+void CodeGenerator::visitIsObject(LIsObject* ins) {
+ Register output = ToRegister(ins->output());
+ ValueOperand value = ToValue(ins, LIsObject::ObjectIndex);
+ masm.testObjectSet(Assembler::Equal, value, output);
+}
+
+void CodeGenerator::visitIsObjectAndBranch(LIsObjectAndBranch* ins) {
+ ValueOperand value = ToValue(ins, LIsObjectAndBranch::Input);
+ testObjectEmitBranch(Assembler::Equal, value, ins->ifTrue(), ins->ifFalse());
+}
+
+void CodeGenerator::visitIsNullOrUndefined(LIsNullOrUndefined* ins) {
+ Register output = ToRegister(ins->output());
+ ValueOperand value = ToValue(ins, LIsNullOrUndefined::InputIndex);
+
+ Label isNotNull, done;
+ masm.branchTestNull(Assembler::NotEqual, value, &isNotNull);
+
+ masm.move32(Imm32(1), output);
+ masm.jump(&done);
+
+ masm.bind(&isNotNull);
+ masm.testUndefinedSet(Assembler::Equal, value, output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitIsNullOrUndefinedAndBranch(
+ LIsNullOrUndefinedAndBranch* ins) {
+ Label* ifTrue = getJumpLabelForBranch(ins->ifTrue());
+ Label* ifFalse = getJumpLabelForBranch(ins->ifFalse());
+ ValueOperand value = ToValue(ins, LIsNullOrUndefinedAndBranch::Input);
+
+ ScratchTagScope tag(masm, value);
+ masm.splitTagForTest(value, tag);
+
+ masm.branchTestNull(Assembler::Equal, tag, ifTrue);
+ masm.branchTestUndefined(Assembler::Equal, tag, ifTrue);
+
+ if (!isNextBlock(ins->ifFalse()->lir())) {
+ masm.jump(ifFalse);
+ }
+}
+
+void CodeGenerator::loadOutermostJSScript(Register reg) {
+ // The "outermost" JSScript means the script that we are compiling
+ // basically; this is not always the script associated with the
+ // current basic block, which might be an inlined script.
+
+ MIRGraph& graph = current->mir()->graph();
+ MBasicBlock* entryBlock = graph.entryBlock();
+ masm.movePtr(ImmGCPtr(entryBlock->info().script()), reg);
+}
+
+void CodeGenerator::loadJSScriptForBlock(MBasicBlock* block, Register reg) {
+ // The current JSScript means the script for the current
+ // basic block. This may be an inlined script.
+
+ JSScript* script = block->info().script();
+ masm.movePtr(ImmGCPtr(script), reg);
+}
+
+void CodeGenerator::visitHasClass(LHasClass* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register output = ToRegister(ins->output());
+
+ masm.loadObjClassUnsafe(lhs, output);
+ masm.cmpPtrSet(Assembler::Equal, output, ImmPtr(ins->mir()->getClass()),
+ output);
+}
+
+void CodeGenerator::visitGuardToClass(LGuardToClass* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register temp = ToRegister(ins->temp0());
+
+ // branchTestObjClass may zero the object register on speculative paths
+ // (we should have a defineReuseInput allocation in this case).
+ Register spectreRegToZero = lhs;
+
+ Label notEqual;
+
+ masm.branchTestObjClass(Assembler::NotEqual, lhs, ins->mir()->getClass(),
+ temp, spectreRegToZero, &notEqual);
+
+ // Can't return null-return here, so bail.
+ bailoutFrom(&notEqual, ins->snapshot());
+}
+
+void CodeGenerator::visitGuardToFunction(LGuardToFunction* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register temp = ToRegister(ins->temp0());
+
+ // branchTestObjClass may zero the object register on speculative paths
+ // (we should have a defineReuseInput allocation in this case).
+ Register spectreRegToZero = lhs;
+
+ Label notEqual;
+
+ masm.branchTestObjIsFunction(Assembler::NotEqual, lhs, temp, spectreRegToZero,
+ &notEqual);
+
+ // Can't return null-return here, so bail.
+ bailoutFrom(&notEqual, ins->snapshot());
+}
+
+void CodeGenerator::visitObjectClassToString(LObjectClassToString* lir) {
+ Register obj = ToRegister(lir->lhs());
+ Register temp = ToRegister(lir->temp0());
+
+ using Fn = JSString* (*)(JSContext*, JSObject*);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp);
+ masm.passABIArg(temp);
+ masm.passABIArg(obj);
+ masm.callWithABI<Fn, js::ObjectClassToString>();
+
+ bailoutCmpPtr(Assembler::Equal, ReturnReg, ImmWord(0), lir->snapshot());
+}
+
+void CodeGenerator::visitWasmParameter(LWasmParameter* lir) {}
+
+void CodeGenerator::visitWasmParameterI64(LWasmParameterI64* lir) {}
+
+void CodeGenerator::visitWasmReturn(LWasmReturn* lir) {
+ // Don't emit a jump to the return label if this is the last block.
+ if (current->mir() != *gen->graph().poBegin()) {
+ masm.jump(&returnLabel_);
+ }
+}
+
+void CodeGenerator::visitWasmReturnI64(LWasmReturnI64* lir) {
+ // Don't emit a jump to the return label if this is the last block.
+ if (current->mir() != *gen->graph().poBegin()) {
+ masm.jump(&returnLabel_);
+ }
+}
+
+void CodeGenerator::visitWasmReturnVoid(LWasmReturnVoid* lir) {
+ // Don't emit a jump to the return label if this is the last block.
+ if (current->mir() != *gen->graph().poBegin()) {
+ masm.jump(&returnLabel_);
+ }
+}
+
+void CodeGenerator::emitAssertRangeI(MIRType type, const Range* r,
+ Register input) {
+ // Check the lower bound.
+ if (r->hasInt32LowerBound() && r->lower() > INT32_MIN) {
+ Label success;
+ if (type == MIRType::Int32 || type == MIRType::Boolean) {
+ masm.branch32(Assembler::GreaterThanOrEqual, input, Imm32(r->lower()),
+ &success);
+ } else {
+ MOZ_ASSERT(type == MIRType::IntPtr);
+ masm.branchPtr(Assembler::GreaterThanOrEqual, input, Imm32(r->lower()),
+ &success);
+ }
+ masm.assumeUnreachable(
+ "Integer input should be equal or higher than Lowerbound.");
+ masm.bind(&success);
+ }
+
+ // Check the upper bound.
+ if (r->hasInt32UpperBound() && r->upper() < INT32_MAX) {
+ Label success;
+ if (type == MIRType::Int32 || type == MIRType::Boolean) {
+ masm.branch32(Assembler::LessThanOrEqual, input, Imm32(r->upper()),
+ &success);
+ } else {
+ MOZ_ASSERT(type == MIRType::IntPtr);
+ masm.branchPtr(Assembler::LessThanOrEqual, input, Imm32(r->upper()),
+ &success);
+ }
+ masm.assumeUnreachable(
+ "Integer input should be lower or equal than Upperbound.");
+ masm.bind(&success);
+ }
+
+ // For r->canHaveFractionalPart(), r->canBeNegativeZero(), and
+ // r->exponent(), there's nothing to check, because if we ended up in the
+ // integer range checking code, the value is already in an integer register
+ // in the integer range.
+}
+
+void CodeGenerator::emitAssertRangeD(const Range* r, FloatRegister input,
+ FloatRegister temp) {
+ // Check the lower bound.
+ if (r->hasInt32LowerBound()) {
+ Label success;
+ masm.loadConstantDouble(r->lower(), temp);
+ if (r->canBeNaN()) {
+ masm.branchDouble(Assembler::DoubleUnordered, input, input, &success);
+ }
+ masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp,
+ &success);
+ masm.assumeUnreachable(
+ "Double input should be equal or higher than Lowerbound.");
+ masm.bind(&success);
+ }
+ // Check the upper bound.
+ if (r->hasInt32UpperBound()) {
+ Label success;
+ masm.loadConstantDouble(r->upper(), temp);
+ if (r->canBeNaN()) {
+ masm.branchDouble(Assembler::DoubleUnordered, input, input, &success);
+ }
+ masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp, &success);
+ masm.assumeUnreachable(
+ "Double input should be lower or equal than Upperbound.");
+ masm.bind(&success);
+ }
+
+ // This code does not yet check r->canHaveFractionalPart(). This would require
+ // new assembler interfaces to make rounding instructions available.
+
+ if (!r->canBeNegativeZero()) {
+ Label success;
+
+ // First, test for being equal to 0.0, which also includes -0.0.
+ masm.loadConstantDouble(0.0, temp);
+ masm.branchDouble(Assembler::DoubleNotEqualOrUnordered, input, temp,
+ &success);
+
+ // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0 is
+ // -Infinity instead of Infinity.
+ masm.loadConstantDouble(1.0, temp);
+ masm.divDouble(input, temp);
+ masm.branchDouble(Assembler::DoubleGreaterThan, temp, input, &success);
+
+ masm.assumeUnreachable("Input shouldn't be negative zero.");
+
+ masm.bind(&success);
+ }
+
+ if (!r->hasInt32Bounds() && !r->canBeInfiniteOrNaN() &&
+ r->exponent() < FloatingPoint<double>::kExponentBias) {
+ // Check the bounds implied by the maximum exponent.
+ Label exponentLoOk;
+ masm.loadConstantDouble(pow(2.0, r->exponent() + 1), temp);
+ masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentLoOk);
+ masm.branchDouble(Assembler::DoubleLessThanOrEqual, input, temp,
+ &exponentLoOk);
+ masm.assumeUnreachable("Check for exponent failed.");
+ masm.bind(&exponentLoOk);
+
+ Label exponentHiOk;
+ masm.loadConstantDouble(-pow(2.0, r->exponent() + 1), temp);
+ masm.branchDouble(Assembler::DoubleUnordered, input, input, &exponentHiOk);
+ masm.branchDouble(Assembler::DoubleGreaterThanOrEqual, input, temp,
+ &exponentHiOk);
+ masm.assumeUnreachable("Check for exponent failed.");
+ masm.bind(&exponentHiOk);
+ } else if (!r->hasInt32Bounds() && !r->canBeNaN()) {
+ // If we think the value can't be NaN, check that it isn't.
+ Label notnan;
+ masm.branchDouble(Assembler::DoubleOrdered, input, input, &notnan);
+ masm.assumeUnreachable("Input shouldn't be NaN.");
+ masm.bind(&notnan);
+
+ // If we think the value also can't be an infinity, check that it isn't.
+ if (!r->canBeInfiniteOrNaN()) {
+ Label notposinf;
+ masm.loadConstantDouble(PositiveInfinity<double>(), temp);
+ masm.branchDouble(Assembler::DoubleLessThan, input, temp, &notposinf);
+ masm.assumeUnreachable("Input shouldn't be +Inf.");
+ masm.bind(&notposinf);
+
+ Label notneginf;
+ masm.loadConstantDouble(NegativeInfinity<double>(), temp);
+ masm.branchDouble(Assembler::DoubleGreaterThan, input, temp, &notneginf);
+ masm.assumeUnreachable("Input shouldn't be -Inf.");
+ masm.bind(&notneginf);
+ }
+ }
+}
+
+void CodeGenerator::visitAssertClass(LAssertClass* ins) {
+ Register obj = ToRegister(ins->input());
+ Register temp = ToRegister(ins->getTemp(0));
+
+ Label success;
+ if (ins->mir()->getClass() == &FunctionClass) {
+ // Allow both possible function classes here.
+ masm.branchTestObjIsFunctionNoSpectreMitigations(Assembler::Equal, obj,
+ temp, &success);
+ } else {
+ masm.branchTestObjClassNoSpectreMitigations(
+ Assembler::Equal, obj, ins->mir()->getClass(), temp, &success);
+ }
+ masm.assumeUnreachable("Wrong KnownClass during run-time");
+ masm.bind(&success);
+}
+
+void CodeGenerator::visitAssertShape(LAssertShape* ins) {
+ Register obj = ToRegister(ins->input());
+
+ Label success;
+ masm.branchTestObjShapeNoSpectreMitigations(Assembler::Equal, obj,
+ ins->mir()->shape(), &success);
+ masm.assumeUnreachable("Wrong Shape during run-time");
+ masm.bind(&success);
+}
+
+void CodeGenerator::visitAssertRangeI(LAssertRangeI* ins) {
+ Register input = ToRegister(ins->input());
+ const Range* r = ins->range();
+
+ emitAssertRangeI(ins->mir()->input()->type(), r, input);
+}
+
+void CodeGenerator::visitAssertRangeD(LAssertRangeD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister temp = ToFloatRegister(ins->temp());
+ const Range* r = ins->range();
+
+ emitAssertRangeD(r, input, temp);
+}
+
+void CodeGenerator::visitAssertRangeF(LAssertRangeF* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister temp = ToFloatRegister(ins->temp());
+ FloatRegister temp2 = ToFloatRegister(ins->temp2());
+
+ const Range* r = ins->range();
+
+ masm.convertFloat32ToDouble(input, temp);
+ emitAssertRangeD(r, temp, temp2);
+}
+
+void CodeGenerator::visitAssertRangeV(LAssertRangeV* ins) {
+ const Range* r = ins->range();
+ const ValueOperand value = ToValue(ins, LAssertRangeV::Input);
+ Label done;
+
+ {
+ ScratchTagScope tag(masm, value);
+ masm.splitTagForTest(value, tag);
+
+ {
+ Label isNotInt32;
+ masm.branchTestInt32(Assembler::NotEqual, tag, &isNotInt32);
+ {
+ ScratchTagScopeRelease _(&tag);
+ Register unboxInt32 = ToTempUnboxRegister(ins->temp());
+ Register input = masm.extractInt32(value, unboxInt32);
+ emitAssertRangeI(MIRType::Int32, r, input);
+ masm.jump(&done);
+ }
+ masm.bind(&isNotInt32);
+ }
+
+ {
+ Label isNotDouble;
+ masm.branchTestDouble(Assembler::NotEqual, tag, &isNotDouble);
+ {
+ ScratchTagScopeRelease _(&tag);
+ FloatRegister input = ToFloatRegister(ins->floatTemp1());
+ FloatRegister temp = ToFloatRegister(ins->floatTemp2());
+ masm.unboxDouble(value, input);
+ emitAssertRangeD(r, input, temp);
+ masm.jump(&done);
+ }
+ masm.bind(&isNotDouble);
+ }
+ }
+
+ masm.assumeUnreachable("Incorrect range for Value.");
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitInterruptCheck(LInterruptCheck* lir) {
+ using Fn = bool (*)(JSContext*);
+ OutOfLineCode* ool =
+ oolCallVM<Fn, InterruptCheck>(lir, ArgList(), StoreNothing());
+
+ const void* interruptAddr = gen->runtime->addressOfInterruptBits();
+ masm.branch32(Assembler::NotEqual, AbsoluteAddress(interruptAddr), Imm32(0),
+ ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineResumableWasmTrap(
+ OutOfLineResumableWasmTrap* ool) {
+ LInstruction* lir = ool->lir();
+ masm.wasmTrap(ool->trap(), ool->bytecodeOffset());
+
+ markSafepointAt(masm.currentOffset(), lir);
+
+ // Note that masm.framePushed() doesn't include the register dump area.
+ // That will be taken into account when the StackMap is created from the
+ // LSafepoint.
+ lir->safepoint()->setFramePushedAtStackMapBase(ool->framePushed());
+ lir->safepoint()->setIsWasmTrap();
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitOutOfLineAbortingWasmTrap(
+ OutOfLineAbortingWasmTrap* ool) {
+ masm.wasmTrap(ool->trap(), ool->bytecodeOffset());
+}
+
+void CodeGenerator::visitWasmInterruptCheck(LWasmInterruptCheck* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+
+ OutOfLineResumableWasmTrap* ool = new (alloc()) OutOfLineResumableWasmTrap(
+ lir, masm.framePushed(), lir->mir()->bytecodeOffset(),
+ wasm::Trap::CheckInterrupt);
+ addOutOfLineCode(ool, lir->mir());
+ masm.branch32(
+ Assembler::NotEqual,
+ Address(ToRegister(lir->instance()), wasm::Instance::offsetOfInterrupt()),
+ Imm32(0), ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitWasmTrap(LWasmTrap* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ const MWasmTrap* mir = lir->mir();
+
+ masm.wasmTrap(mir->trap(), mir->bytecodeOffset());
+}
+
+void CodeGenerator::visitWasmGcObjectIsSubtypeOf(
+ LWasmGcObjectIsSubtypeOf* ins) {
+ MOZ_ASSERT(gen->compilingWasm());
+ const MWasmGcObjectIsSubtypeOf* mir = ins->mir();
+ Register object = ToRegister(ins->object());
+ Register superTypeDef = ToRegister(ins->superTypeDef());
+ Register subTypeDef = ToRegister(ins->temp0());
+ Register scratch = ins->temp1()->isBogusTemp() ? Register::Invalid()
+ : ToRegister(ins->temp1());
+ Register result = ToRegister(ins->output());
+ Label failed;
+ Label success;
+ Label join;
+ masm.branchTestPtr(Assembler::Zero, object, object, &failed);
+ masm.loadPtr(Address(object, WasmGcObject::offsetOfTypeDef()), subTypeDef);
+ masm.branchWasmTypeDefIsSubtype(subTypeDef, superTypeDef, scratch,
+ mir->subTypingDepth(), &success, true);
+ masm.bind(&failed);
+ masm.xor32(result, result);
+ masm.jump(&join);
+ masm.bind(&success);
+ masm.move32(Imm32(1), result);
+ masm.bind(&join);
+}
+
+void CodeGenerator::visitWasmGcObjectIsSubtypeOfAndBranch(
+ LWasmGcObjectIsSubtypeOfAndBranch* ins) {
+ MOZ_ASSERT(gen->compilingWasm());
+ Register object = ToRegister(ins->object());
+ Register superTypeDef = ToRegister(ins->superTypeDef());
+ Register subTypeDef = ToRegister(ins->temp0());
+ Register scratch = ins->temp1()->isBogusTemp() ? Register::Invalid()
+ : ToRegister(ins->temp1());
+ masm.branchTestPtr(Assembler::Zero, object, object,
+ ins->ifFalse()->lir()->label());
+ masm.loadPtr(Address(object, WasmGcObject::offsetOfTypeDef()), subTypeDef);
+ masm.branchWasmTypeDefIsSubtype(subTypeDef, superTypeDef, scratch,
+ ins->subTypingDepth(),
+ ins->ifTrue()->lir()->label(), true);
+ jumpToBlock(ins->ifFalse());
+}
+
+void CodeGenerator::visitWasmBoundsCheck(LWasmBoundsCheck* ins) {
+ const MWasmBoundsCheck* mir = ins->mir();
+ Register ptr = ToRegister(ins->ptr());
+ Register boundsCheckLimit = ToRegister(ins->boundsCheckLimit());
+ // When there are no spectre mitigations in place, branching out-of-line to
+ // the trap is a big performance win, but with mitigations it's trickier. See
+ // bug 1680243.
+ if (JitOptions.spectreIndexMasking) {
+ Label ok;
+ masm.wasmBoundsCheck32(Assembler::Below, ptr, boundsCheckLimit, &ok);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
+ masm.bind(&ok);
+ } else {
+ OutOfLineAbortingWasmTrap* ool = new (alloc()) OutOfLineAbortingWasmTrap(
+ mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
+ addOutOfLineCode(ool, mir);
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptr, boundsCheckLimit,
+ ool->entry());
+ }
+}
+
+void CodeGenerator::visitWasmBoundsCheck64(LWasmBoundsCheck64* ins) {
+ const MWasmBoundsCheck* mir = ins->mir();
+ Register64 ptr = ToRegister64(ins->ptr());
+ Register64 boundsCheckLimit = ToRegister64(ins->boundsCheckLimit());
+ // See above.
+ if (JitOptions.spectreIndexMasking) {
+ Label ok;
+ masm.wasmBoundsCheck64(Assembler::Below, ptr, boundsCheckLimit, &ok);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
+ masm.bind(&ok);
+ } else {
+ OutOfLineAbortingWasmTrap* ool = new (alloc()) OutOfLineAbortingWasmTrap(
+ mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
+ addOutOfLineCode(ool, mir);
+ masm.wasmBoundsCheck64(Assembler::AboveOrEqual, ptr, boundsCheckLimit,
+ ool->entry());
+ }
+}
+
+void CodeGenerator::visitWasmAlignmentCheck(LWasmAlignmentCheck* ins) {
+ const MWasmAlignmentCheck* mir = ins->mir();
+ Register ptr = ToRegister(ins->ptr());
+ OutOfLineAbortingWasmTrap* ool = new (alloc()) OutOfLineAbortingWasmTrap(
+ mir->bytecodeOffset(), wasm::Trap::UnalignedAccess);
+ addOutOfLineCode(ool, mir);
+ masm.branchTest32(Assembler::NonZero, ptr, Imm32(mir->byteSize() - 1),
+ ool->entry());
+}
+
+void CodeGenerator::visitWasmAlignmentCheck64(LWasmAlignmentCheck64* ins) {
+ const MWasmAlignmentCheck* mir = ins->mir();
+ Register64 ptr = ToRegister64(ins->ptr());
+#ifdef JS_64BIT
+ Register r = ptr.reg;
+#else
+ Register r = ptr.low;
+#endif
+ OutOfLineAbortingWasmTrap* ool = new (alloc()) OutOfLineAbortingWasmTrap(
+ mir->bytecodeOffset(), wasm::Trap::UnalignedAccess);
+ addOutOfLineCode(ool, mir);
+ masm.branchTestPtr(Assembler::NonZero, r, Imm32(mir->byteSize() - 1),
+ ool->entry());
+}
+
+void CodeGenerator::visitWasmLoadInstance(LWasmLoadInstance* ins) {
+ switch (ins->mir()->type()) {
+ case MIRType::RefOrNull:
+ case MIRType::Pointer:
+ masm.loadPtr(Address(ToRegister(ins->instance()), ins->mir()->offset()),
+ ToRegister(ins->output()));
+ break;
+ case MIRType::Int32:
+ masm.load32(Address(ToRegister(ins->instance()), ins->mir()->offset()),
+ ToRegister(ins->output()));
+ break;
+ default:
+ MOZ_CRASH("MIRType not supported in WasmLoadInstance");
+ }
+}
+
+void CodeGenerator::visitWasmLoadInstance64(LWasmLoadInstance64* ins) {
+ MOZ_ASSERT(ins->mir()->type() == MIRType::Int64);
+ masm.load64(Address(ToRegister(ins->instance()), ins->mir()->offset()),
+ ToOutRegister64(ins));
+}
+
+void CodeGenerator::incrementWarmUpCounter(AbsoluteAddress warmUpCount,
+ JSScript* script, Register tmp) {
+ // The code depends on the JitScript* not being discarded without also
+ // invalidating Ion code. Assert this.
+#ifdef DEBUG
+ Label ok;
+ masm.movePtr(ImmGCPtr(script), tmp);
+ masm.loadJitScript(tmp, tmp);
+ masm.branchPtr(Assembler::Equal, tmp, ImmPtr(script->jitScript()), &ok);
+ masm.assumeUnreachable("Didn't find JitScript?");
+ masm.bind(&ok);
+#endif
+
+ masm.load32(warmUpCount, tmp);
+ masm.add32(Imm32(1), tmp);
+ masm.store32(tmp, warmUpCount);
+}
+
+void CodeGenerator::visitIncrementWarmUpCounter(LIncrementWarmUpCounter* ins) {
+ Register tmp = ToRegister(ins->temp0());
+
+ AbsoluteAddress warmUpCount =
+ AbsoluteAddress(ins->mir()->script()->jitScript())
+ .offset(JitScript::offsetOfWarmUpCount());
+ incrementWarmUpCounter(warmUpCount, ins->mir()->script(), tmp);
+}
+
+void CodeGenerator::visitLexicalCheck(LLexicalCheck* ins) {
+ ValueOperand inputValue = ToValue(ins, LLexicalCheck::InputIndex);
+ Label bail;
+ masm.branchTestMagicValue(Assembler::Equal, inputValue,
+ JS_UNINITIALIZED_LEXICAL, &bail);
+ bailoutFrom(&bail, ins->snapshot());
+}
+
+void CodeGenerator::visitThrowRuntimeLexicalError(
+ LThrowRuntimeLexicalError* ins) {
+ pushArg(Imm32(ins->mir()->errorNumber()));
+
+ using Fn = bool (*)(JSContext*, unsigned);
+ callVM<Fn, jit::ThrowRuntimeLexicalError>(ins);
+}
+
+void CodeGenerator::visitThrowMsg(LThrowMsg* ins) {
+ pushArg(Imm32(static_cast<int32_t>(ins->mir()->throwMsgKind())));
+
+ using Fn = bool (*)(JSContext*, unsigned);
+ callVM<Fn, js::ThrowMsgOperation>(ins);
+}
+
+void CodeGenerator::visitGlobalDeclInstantiation(
+ LGlobalDeclInstantiation* ins) {
+ pushArg(ImmPtr(ins->mir()->resumePoint()->pc()));
+ pushArg(ImmGCPtr(ins->mir()->block()->info().script()));
+
+ using Fn = bool (*)(JSContext*, HandleScript, const jsbytecode*);
+ callVM<Fn, GlobalDeclInstantiationFromIon>(ins);
+}
+
+void CodeGenerator::visitDebugger(LDebugger* ins) {
+ Register cx = ToRegister(ins->temp0());
+
+ masm.loadJSContext(cx);
+ using Fn = bool (*)(JSContext * cx);
+ masm.setupAlignedABICall();
+ masm.passABIArg(cx);
+ masm.callWithABI<Fn, GlobalHasLiveOnDebuggerStatement>();
+
+ Label bail;
+ masm.branchIfTrueBool(ReturnReg, &bail);
+ bailoutFrom(&bail, ins->snapshot());
+}
+
+void CodeGenerator::visitNewTarget(LNewTarget* ins) {
+ ValueOperand output = ToOutValue(ins);
+
+ // if (isConstructing) output = argv[Max(numActualArgs, numFormalArgs)]
+ Label notConstructing, done;
+ Address calleeToken(FramePointer, JitFrameLayout::offsetOfCalleeToken());
+ masm.branchTestPtr(Assembler::Zero, calleeToken,
+ Imm32(CalleeToken_FunctionConstructing), &notConstructing);
+
+ Register argvLen = output.scratchReg();
+ masm.loadNumActualArgs(FramePointer, argvLen);
+
+ Label useNFormals;
+
+ size_t numFormalArgs = ins->mirRaw()->block()->info().nargs();
+ masm.branchPtr(Assembler::Below, argvLen, Imm32(numFormalArgs), &useNFormals);
+
+ size_t argsOffset = JitFrameLayout::offsetOfActualArgs();
+ {
+ BaseValueIndex newTarget(FramePointer, argvLen, argsOffset);
+ masm.loadValue(newTarget, output);
+ masm.jump(&done);
+ }
+
+ masm.bind(&useNFormals);
+
+ {
+ Address newTarget(FramePointer,
+ argsOffset + (numFormalArgs * sizeof(Value)));
+ masm.loadValue(newTarget, output);
+ masm.jump(&done);
+ }
+
+ // else output = undefined
+ masm.bind(&notConstructing);
+ masm.moveValue(UndefinedValue(), output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitCheckReturn(LCheckReturn* ins) {
+ ValueOperand returnValue = ToValue(ins, LCheckReturn::ReturnValueIndex);
+ ValueOperand thisValue = ToValue(ins, LCheckReturn::ThisValueIndex);
+ ValueOperand output = ToOutValue(ins);
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ OutOfLineCode* ool = oolCallVM<Fn, ThrowBadDerivedReturnOrUninitializedThis>(
+ ins, ArgList(returnValue), StoreNothing());
+
+ Label noChecks;
+ masm.branchTestObject(Assembler::Equal, returnValue, &noChecks);
+ masm.branchTestUndefined(Assembler::NotEqual, returnValue, ool->entry());
+ masm.branchTestMagic(Assembler::Equal, thisValue, ool->entry());
+ masm.moveValue(thisValue, output);
+ masm.jump(ool->rejoin());
+ masm.bind(&noChecks);
+ masm.moveValue(returnValue, output);
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCheckIsObj(LCheckIsObj* ins) {
+ ValueOperand value = ToValue(ins, LCheckIsObj::ValueIndex);
+ Register output = ToRegister(ins->output());
+
+ using Fn = bool (*)(JSContext*, CheckIsObjectKind);
+ OutOfLineCode* ool = oolCallVM<Fn, ThrowCheckIsObject>(
+ ins, ArgList(Imm32(ins->mir()->checkKind())), StoreNothing());
+
+ masm.fallibleUnboxObject(value, output, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCheckObjCoercible(LCheckObjCoercible* ins) {
+ ValueOperand checkValue = ToValue(ins, LCheckObjCoercible::ValueIndex);
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ OutOfLineCode* ool = oolCallVM<Fn, ThrowObjectCoercible>(
+ ins, ArgList(checkValue), StoreNothing());
+ masm.branchTestNull(Assembler::Equal, checkValue, ool->entry());
+ masm.branchTestUndefined(Assembler::Equal, checkValue, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCheckClassHeritage(LCheckClassHeritage* ins) {
+ ValueOperand heritage = ToValue(ins, LCheckClassHeritage::HeritageIndex);
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+
+ using Fn = bool (*)(JSContext*, HandleValue);
+ OutOfLineCode* ool = oolCallVM<Fn, CheckClassHeritageOperation>(
+ ins, ArgList(heritage), StoreNothing());
+
+ masm.branchTestNull(Assembler::Equal, heritage, ool->rejoin());
+ masm.fallibleUnboxObject(heritage, temp0, ool->entry());
+
+ masm.isConstructor(temp0, temp1, ool->entry());
+ masm.branchTest32(Assembler::Zero, temp1, temp1, ool->entry());
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCheckThis(LCheckThis* ins) {
+ ValueOperand thisValue = ToValue(ins, LCheckThis::ValueIndex);
+
+ using Fn = bool (*)(JSContext*);
+ OutOfLineCode* ool =
+ oolCallVM<Fn, ThrowUninitializedThis>(ins, ArgList(), StoreNothing());
+ masm.branchTestMagic(Assembler::Equal, thisValue, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCheckThisReinit(LCheckThisReinit* ins) {
+ ValueOperand thisValue = ToValue(ins, LCheckThisReinit::ThisValueIndex);
+
+ using Fn = bool (*)(JSContext*);
+ OutOfLineCode* ool =
+ oolCallVM<Fn, ThrowInitializedThis>(ins, ArgList(), StoreNothing());
+ masm.branchTestMagic(Assembler::NotEqual, thisValue, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitGenerator(LGenerator* lir) {
+ Register callee = ToRegister(lir->callee());
+ Register environmentChain = ToRegister(lir->environmentChain());
+ Register argsObject = ToRegister(lir->argsObject());
+
+ pushArg(argsObject);
+ pushArg(environmentChain);
+ pushArg(ImmGCPtr(current->mir()->info().script()));
+ pushArg(callee);
+
+ using Fn = JSObject* (*)(JSContext * cx, HandleFunction, HandleScript,
+ HandleObject, HandleObject);
+ callVM<Fn, CreateGenerator>(lir);
+}
+
+void CodeGenerator::visitAsyncResolve(LAsyncResolve* lir) {
+ Register generator = ToRegister(lir->generator());
+ ValueOperand valueOrReason = ToValue(lir, LAsyncResolve::ValueOrReasonIndex);
+ AsyncFunctionResolveKind resolveKind = lir->mir()->resolveKind();
+
+ pushArg(Imm32(static_cast<int32_t>(resolveKind)));
+ pushArg(valueOrReason);
+ pushArg(generator);
+
+ using Fn = JSObject* (*)(JSContext*, Handle<AsyncFunctionGeneratorObject*>,
+ HandleValue, AsyncFunctionResolveKind);
+ callVM<Fn, js::AsyncFunctionResolve>(lir);
+}
+
+void CodeGenerator::visitAsyncAwait(LAsyncAwait* lir) {
+ ValueOperand value = ToValue(lir, LAsyncAwait::ValueIndex);
+ Register generator = ToRegister(lir->generator());
+
+ pushArg(value);
+ pushArg(generator);
+
+ using Fn = JSObject* (*)(JSContext * cx,
+ Handle<AsyncFunctionGeneratorObject*> genObj,
+ HandleValue value);
+ callVM<Fn, js::AsyncFunctionAwait>(lir);
+}
+
+void CodeGenerator::visitCanSkipAwait(LCanSkipAwait* lir) {
+ ValueOperand value = ToValue(lir, LCanSkipAwait::ValueIndex);
+
+ pushArg(value);
+
+ using Fn = bool (*)(JSContext*, HandleValue, bool* canSkip);
+ callVM<Fn, js::CanSkipAwait>(lir);
+}
+
+void CodeGenerator::visitMaybeExtractAwaitValue(LMaybeExtractAwaitValue* lir) {
+ ValueOperand value = ToValue(lir, LMaybeExtractAwaitValue::ValueIndex);
+ ValueOperand output = ToOutValue(lir);
+ Register canSkip = ToRegister(lir->canSkip());
+
+ Label cantExtract, finished;
+ masm.branchIfFalseBool(canSkip, &cantExtract);
+
+ pushArg(value);
+
+ using Fn = bool (*)(JSContext*, HandleValue, MutableHandleValue);
+ callVM<Fn, js::ExtractAwaitValue>(lir);
+ masm.jump(&finished);
+ masm.bind(&cantExtract);
+
+ masm.moveValue(value, output);
+
+ masm.bind(&finished);
+}
+
+void CodeGenerator::visitDebugCheckSelfHosted(LDebugCheckSelfHosted* ins) {
+ ValueOperand checkValue = ToValue(ins, LDebugCheckSelfHosted::ValueIndex);
+ pushArg(checkValue);
+ using Fn = bool (*)(JSContext*, HandleValue);
+ callVM<Fn, js::Debug_CheckSelfHosted>(ins);
+}
+
+void CodeGenerator::visitRandom(LRandom* ins) {
+ using mozilla::non_crypto::XorShift128PlusRNG;
+
+ FloatRegister output = ToFloatRegister(ins->output());
+ Register rngReg = ToRegister(ins->temp0());
+
+ Register64 temp1 = ToRegister64(ins->temp1());
+ Register64 temp2 = ToRegister64(ins->temp2());
+
+ const XorShift128PlusRNG* rng = gen->realm->addressOfRandomNumberGenerator();
+ masm.movePtr(ImmPtr(rng), rngReg);
+
+ masm.randomDouble(rngReg, output, temp1, temp2);
+ if (js::SupportDifferentialTesting()) {
+ masm.loadConstantDouble(0.0, output);
+ }
+}
+
+void CodeGenerator::visitSignExtendInt32(LSignExtendInt32* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ switch (ins->mode()) {
+ case MSignExtendInt32::Byte:
+ masm.move8SignExtend(input, output);
+ break;
+ case MSignExtendInt32::Half:
+ masm.move16SignExtend(input, output);
+ break;
+ }
+}
+
+void CodeGenerator::visitRotate(LRotate* ins) {
+ MRotate* mir = ins->mir();
+ Register input = ToRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+
+ const LAllocation* count = ins->count();
+ if (count->isConstant()) {
+ int32_t c = ToInt32(count) & 0x1F;
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft(Imm32(c), input, dest);
+ } else {
+ masm.rotateRight(Imm32(c), input, dest);
+ }
+ } else {
+ Register creg = ToRegister(count);
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft(creg, input, dest);
+ } else {
+ masm.rotateRight(creg, input, dest);
+ }
+ }
+}
+
+class OutOfLineNaNToZero : public OutOfLineCodeBase<CodeGenerator> {
+ LNaNToZero* lir_;
+
+ public:
+ explicit OutOfLineNaNToZero(LNaNToZero* lir) : lir_(lir) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineNaNToZero(this);
+ }
+ LNaNToZero* lir() const { return lir_; }
+};
+
+void CodeGenerator::visitOutOfLineNaNToZero(OutOfLineNaNToZero* ool) {
+ FloatRegister output = ToFloatRegister(ool->lir()->output());
+ masm.loadConstantDouble(0.0, output);
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitNaNToZero(LNaNToZero* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+
+ OutOfLineNaNToZero* ool = new (alloc()) OutOfLineNaNToZero(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ if (lir->mir()->operandIsNeverNegativeZero()) {
+ masm.branchDouble(Assembler::DoubleUnordered, input, input, ool->entry());
+ } else {
+ FloatRegister scratch = ToFloatRegister(lir->temp0());
+ masm.loadConstantDouble(0.0, scratch);
+ masm.branchDouble(Assembler::DoubleEqualOrUnordered, input, scratch,
+ ool->entry());
+ }
+ masm.bind(ool->rejoin());
+}
+
+static void BoundFunctionLength(MacroAssembler& masm, Register target,
+ Register targetFlagsAndArgCount,
+ Register argCount, Register output,
+ Label* slowPath) {
+ masm.loadFunctionLength(target, targetFlagsAndArgCount, output, slowPath);
+
+ // Compute the bound function length: Max(0, target.length - argCount).
+ Label nonNegative;
+ masm.sub32(argCount, output);
+ masm.branch32(Assembler::GreaterThanOrEqual, output, Imm32(0), &nonNegative);
+ masm.move32(Imm32(0), output);
+ masm.bind(&nonNegative);
+}
+
+static void BoundFunctionName(MacroAssembler& masm, Register target,
+ Register targetFlags, Register output,
+ const JSAtomState& names, Label* slowPath) {
+ Label notBoundTarget, loadName;
+ masm.branchTest32(Assembler::Zero, targetFlags,
+ Imm32(FunctionFlags::BOUND_FUN), &notBoundTarget);
+ {
+ // Call into the VM if the target's name atom contains the bound
+ // function prefix.
+ masm.branchTest32(Assembler::NonZero, targetFlags,
+ Imm32(FunctionFlags::HAS_BOUND_FUNCTION_NAME_PREFIX),
+ slowPath);
+
+ // Bound functions reuse HAS_GUESSED_ATOM for
+ // HAS_BOUND_FUNCTION_NAME_PREFIX, so skip the guessed atom check below.
+ static_assert(
+ FunctionFlags::HAS_BOUND_FUNCTION_NAME_PREFIX ==
+ FunctionFlags::HAS_GUESSED_ATOM,
+ "HAS_BOUND_FUNCTION_NAME_PREFIX is shared with HAS_GUESSED_ATOM");
+ masm.jump(&loadName);
+ }
+ masm.bind(&notBoundTarget);
+
+ Label guessed, hasName;
+ masm.branchTest32(Assembler::NonZero, targetFlags,
+ Imm32(FunctionFlags::HAS_GUESSED_ATOM), &guessed);
+
+ masm.bind(&loadName);
+ Address atom(Address(target, JSFunction::offsetOfAtom()));
+ masm.branchTestUndefined(Assembler::Equal, atom, &guessed);
+ masm.unboxString(atom, output);
+ masm.jump(&hasName);
+
+ {
+ masm.bind(&guessed);
+
+ // An absent name property defaults to the empty string.
+ masm.movePtr(ImmGCPtr(names.empty), output);
+ }
+ masm.bind(&hasName);
+}
+
+static void BoundFunctionFlagsAndArgCount(MacroAssembler& masm,
+ Register targetFlags, Register bound,
+ Register output) {
+ // Set the BOUND_FN flag and, if the target is a constructor, the
+ // CONSTRUCTOR flag.
+ Label isConstructor, boundFlagsComputed;
+ masm.load32(Address(bound, JSFunction::offsetOfFlagsAndArgCount()), output);
+ masm.branchTest32(Assembler::NonZero, targetFlags,
+ Imm32(FunctionFlags::CONSTRUCTOR), &isConstructor);
+ {
+ masm.or32(Imm32(FunctionFlags::BOUND_FUN), output);
+ masm.jump(&boundFlagsComputed);
+ }
+ masm.bind(&isConstructor);
+ {
+ masm.or32(Imm32(FunctionFlags::BOUND_FUN | FunctionFlags::CONSTRUCTOR),
+ output);
+ }
+ masm.bind(&boundFlagsComputed);
+}
+
+void CodeGenerator::visitFinishBoundFunctionInit(
+ LFinishBoundFunctionInit* lir) {
+ Register bound = ToRegister(lir->bound());
+ Register target = ToRegister(lir->target());
+ Register argCount = ToRegister(lir->argCount());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ using Fn = bool (*)(JSContext * cx, HandleFunction bound, HandleObject target,
+ int32_t argCount);
+ OutOfLineCode* ool = oolCallVM<Fn, JSFunction::finishBoundFunctionInit>(
+ lir, ArgList(bound, target, argCount), StoreNothing());
+ Label* slowPath = ool->entry();
+
+ const size_t boundLengthOffset =
+ FunctionExtended::offsetOfBoundFunctionLengthSlot();
+
+ // Take the slow path if the target is not a JSFunction.
+ masm.branchTestObjIsFunction(Assembler::NotEqual, target, temp0, target,
+ slowPath);
+
+ // Take the slow path if we'd need to adjust the [[Prototype]].
+ masm.loadObjProto(bound, temp0);
+ masm.loadObjProto(target, temp1);
+ masm.branchPtr(Assembler::NotEqual, temp0, temp1, slowPath);
+
+ // Get the function flags and arg count.
+ masm.load32(Address(target, JSFunction::offsetOfFlagsAndArgCount()), temp0);
+
+ // Functions with a SelfHostedLazyScript must be compiled with the slow-path
+ // before the function length is known. If the length or name property is
+ // resolved, it might be shadowed.
+ masm.branchTest32(
+ Assembler::NonZero, temp0,
+ Imm32(FunctionFlags::SELFHOSTLAZY | FunctionFlags::RESOLVED_NAME |
+ FunctionFlags::RESOLVED_LENGTH),
+ slowPath);
+
+ // Store the bound function's length into the extended slot.
+ BoundFunctionLength(masm, target, temp0, argCount, temp1, slowPath);
+ masm.storeValue(JSVAL_TYPE_INT32, temp1, Address(bound, boundLengthOffset));
+
+ // Store the target's name atom in the bound function as is.
+ BoundFunctionName(masm, target, temp0, temp1, gen->runtime->names(),
+ slowPath);
+ masm.storeValue(JSVAL_TYPE_STRING, temp1,
+ Address(bound, JSFunction::offsetOfAtom()));
+
+ // Update the bound function's flags.
+ BoundFunctionFlagsAndArgCount(masm, temp0, bound, temp1);
+ masm.store32(temp1, Address(bound, JSFunction::offsetOfFlagsAndArgCount()));
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitIsPackedArray(LIsPackedArray* lir) {
+ Register obj = ToRegister(lir->object());
+ Register output = ToRegister(lir->output());
+ Register temp = ToRegister(lir->temp0());
+
+ masm.setIsPackedArray(obj, output, temp);
+}
+
+void CodeGenerator::visitGuardArrayIsPacked(LGuardArrayIsPacked* lir) {
+ Register array = ToRegister(lir->array());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+
+ Label bail;
+ masm.branchArrayIsNotPacked(array, temp0, temp1, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGetPrototypeOf(LGetPrototypeOf* lir) {
+ Register target = ToRegister(lir->target());
+ ValueOperand out = ToOutValue(lir);
+ Register scratch = out.scratchReg();
+
+ using Fn = bool (*)(JSContext*, HandleObject, MutableHandleValue);
+ OutOfLineCode* ool = oolCallVM<Fn, jit::GetPrototypeOf>(lir, ArgList(target),
+ StoreValueTo(out));
+
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ masm.loadObjProto(target, scratch);
+
+ Label hasProto;
+ masm.branchPtr(Assembler::Above, scratch, ImmWord(1), &hasProto);
+
+ // Call into the VM for lazy prototypes.
+ masm.branchPtr(Assembler::Equal, scratch, ImmWord(1), ool->entry());
+
+ masm.moveValue(NullValue(), out);
+ masm.jump(ool->rejoin());
+
+ masm.bind(&hasProto);
+ masm.tagValue(JSVAL_TYPE_OBJECT, scratch, out);
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitObjectWithProto(LObjectWithProto* lir) {
+ pushArg(ToValue(lir, LObjectWithProto::PrototypeIndex));
+
+ using Fn = PlainObject* (*)(JSContext*, HandleValue);
+ callVM<Fn, js::ObjectWithProtoOperation>(lir);
+}
+
+void CodeGenerator::visitObjectStaticProto(LObjectStaticProto* lir) {
+ Register obj = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ masm.loadObjProto(obj, output);
+
+#ifdef DEBUG
+ // We shouldn't encounter a null or lazy proto.
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ Label done;
+ masm.branchPtr(Assembler::Above, output, ImmWord(1), &done);
+ masm.assumeUnreachable("Unexpected null or lazy proto in MObjectStaticProto");
+ masm.bind(&done);
+#endif
+}
+
+void CodeGenerator::visitBuiltinObject(LBuiltinObject* lir) {
+ pushArg(Imm32(static_cast<int32_t>(lir->mir()->builtinObjectKind())));
+
+ using Fn = JSObject* (*)(JSContext*, BuiltinObjectKind);
+ callVM<Fn, js::BuiltinObjectOperation>(lir);
+}
+
+void CodeGenerator::visitSuperFunction(LSuperFunction* lir) {
+ Register callee = ToRegister(lir->callee());
+ ValueOperand out = ToOutValue(lir);
+ Register temp = ToRegister(lir->temp0());
+
+#ifdef DEBUG
+ Label classCheckDone;
+ masm.branchTestObjIsFunction(Assembler::Equal, callee, temp, callee,
+ &classCheckDone);
+ masm.assumeUnreachable("Unexpected non-JSFunction callee in JSOp::SuperFun");
+ masm.bind(&classCheckDone);
+#endif
+
+ // Load prototype of callee
+ masm.loadObjProto(callee, temp);
+
+#ifdef DEBUG
+ // We won't encounter a lazy proto, because |callee| is guaranteed to be a
+ // JSFunction and only proxy objects can have a lazy proto.
+ MOZ_ASSERT(uintptr_t(TaggedProto::LazyProto) == 1);
+
+ Label proxyCheckDone;
+ masm.branchPtr(Assembler::NotEqual, temp, ImmWord(1), &proxyCheckDone);
+ masm.assumeUnreachable("Unexpected lazy proto in JSOp::SuperFun");
+ masm.bind(&proxyCheckDone);
+#endif
+
+ Label nullProto, done;
+ masm.branchPtr(Assembler::Equal, temp, ImmWord(0), &nullProto);
+
+ // Box prototype and return
+ masm.tagValue(JSVAL_TYPE_OBJECT, temp, out);
+ masm.jump(&done);
+
+ masm.bind(&nullProto);
+ masm.moveValue(NullValue(), out);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitInitHomeObject(LInitHomeObject* lir) {
+ Register func = ToRegister(lir->function());
+ ValueOperand homeObject = ToValue(lir, LInitHomeObject::HomeObjectIndex);
+
+ masm.assertFunctionIsExtended(func);
+
+ Address addr(func, FunctionExtended::offsetOfMethodHomeObjectSlot());
+
+ emitPreBarrier(addr);
+ masm.storeValue(homeObject, addr);
+}
+
+void CodeGenerator::visitIsTypedArrayConstructor(
+ LIsTypedArrayConstructor* lir) {
+ Register object = ToRegister(lir->object());
+ Register output = ToRegister(lir->output());
+
+ masm.setIsDefinitelyTypedArrayConstructor(object, output);
+}
+
+void CodeGenerator::visitLoadValueTag(LLoadValueTag* lir) {
+ ValueOperand value = ToValue(lir, LLoadValueTag::ValueIndex);
+ Register output = ToRegister(lir->output());
+
+ Register tag = masm.extractTag(value, output);
+ if (tag != output) {
+ masm.mov(tag, output);
+ }
+}
+
+void CodeGenerator::visitGuardTagNotEqual(LGuardTagNotEqual* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+
+ bailoutCmp32(Assembler::Equal, lhs, rhs, lir->snapshot());
+
+ // If both lhs and rhs are numbers, can't use tag comparison to do inequality
+ // comparison
+ Label done;
+ masm.branchTestNumber(Assembler::NotEqual, lhs, &done);
+ masm.branchTestNumber(Assembler::NotEqual, rhs, &done);
+ bailout(lir->snapshot());
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitLoadWrapperTarget(LLoadWrapperTarget* lir) {
+ Register object = ToRegister(lir->object());
+ Register output = ToRegister(lir->output());
+
+ masm.loadPtr(Address(object, ProxyObject::offsetOfReservedSlots()), output);
+ masm.unboxObject(
+ Address(output, js::detail::ProxyReservedSlots::offsetOfPrivateSlot()),
+ output);
+}
+
+void CodeGenerator::visitGuardHasGetterSetter(LGuardHasGetterSetter* lir) {
+ Register object = ToRegister(lir->object());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+
+ masm.movePropertyKey(lir->mir()->propId(), temp1);
+ masm.movePtr(ImmGCPtr(lir->mir()->getterSetter()), temp2);
+
+ using Fn = bool (*)(JSContext * cx, JSObject * obj, jsid id,
+ GetterSetter * getterSetter);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp0);
+ masm.passABIArg(temp0);
+ masm.passABIArg(object);
+ masm.passABIArg(temp1);
+ masm.passABIArg(temp2);
+ masm.callWithABI<Fn, ObjectHasGetterSetterPure>();
+
+ bailoutIfFalseBool(ReturnReg, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardIsExtensible(LGuardIsExtensible* lir) {
+ Register object = ToRegister(lir->object());
+ Register temp = ToRegister(lir->temp0());
+
+ Label bail;
+ masm.branchIfObjectNotExtensible(object, temp, &bail);
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitGuardInt32IsNonNegative(
+ LGuardInt32IsNonNegative* lir) {
+ Register index = ToRegister(lir->index());
+
+ bailoutCmp32(Assembler::LessThan, index, Imm32(0), lir->snapshot());
+}
+
+void CodeGenerator::visitGuardInt32Range(LGuardInt32Range* lir) {
+ Register input = ToRegister(lir->input());
+
+ bailoutCmp32(Assembler::LessThan, input, Imm32(lir->mir()->minimum()),
+ lir->snapshot());
+ bailoutCmp32(Assembler::GreaterThan, input, Imm32(lir->mir()->maximum()),
+ lir->snapshot());
+}
+
+void CodeGenerator::visitGuardIndexIsNotDenseElement(
+ LGuardIndexIsNotDenseElement* lir) {
+ Register object = ToRegister(lir->object());
+ Register index = ToRegister(lir->index());
+ Register temp = ToRegister(lir->temp0());
+ Register spectreTemp = ToTempRegisterOrInvalid(lir->temp1());
+
+ // Load obj->elements.
+ masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp);
+
+ // Ensure index >= initLength or the element is a hole.
+ Label notDense;
+ Address capacity(temp, ObjectElements::offsetOfInitializedLength());
+ masm.spectreBoundsCheck32(index, capacity, spectreTemp, &notDense);
+
+ BaseValueIndex element(temp, index);
+ masm.branchTestMagic(Assembler::Equal, element, &notDense);
+
+ bailout(lir->snapshot());
+
+ masm.bind(&notDense);
+}
+
+void CodeGenerator::visitGuardIndexIsValidUpdateOrAdd(
+ LGuardIndexIsValidUpdateOrAdd* lir) {
+ Register object = ToRegister(lir->object());
+ Register index = ToRegister(lir->index());
+ Register temp = ToRegister(lir->temp0());
+ Register spectreTemp = ToTempRegisterOrInvalid(lir->temp1());
+
+ // Load obj->elements.
+ masm.loadPtr(Address(object, NativeObject::offsetOfElements()), temp);
+
+ Label success;
+
+ // If length is writable, branch to &success. All indices are writable.
+ Address flags(temp, ObjectElements::offsetOfFlags());
+ masm.branchTest32(Assembler::Zero, flags,
+ Imm32(ObjectElements::Flags::NONWRITABLE_ARRAY_LENGTH),
+ &success);
+
+ // Otherwise, ensure index is in bounds.
+ Label bail;
+ Address length(temp, ObjectElements::offsetOfLength());
+ masm.spectreBoundsCheck32(index, length, spectreTemp, &bail);
+ masm.bind(&success);
+
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitCallAddOrUpdateSparseElement(
+ LCallAddOrUpdateSparseElement* lir) {
+ Register object = ToRegister(lir->object());
+ Register index = ToRegister(lir->index());
+ ValueOperand value = ToValue(lir, LCallAddOrUpdateSparseElement::ValueIndex);
+
+ pushArg(Imm32(lir->mir()->strict()));
+ pushArg(value);
+ pushArg(index);
+ pushArg(object);
+
+ using Fn =
+ bool (*)(JSContext*, Handle<NativeObject*>, int32_t, HandleValue, bool);
+ callVM<Fn, js::AddOrUpdateSparseElementHelper>(lir);
+}
+
+void CodeGenerator::visitCallGetSparseElement(LCallGetSparseElement* lir) {
+ Register object = ToRegister(lir->object());
+ Register index = ToRegister(lir->index());
+
+ pushArg(index);
+ pushArg(object);
+
+ using Fn =
+ bool (*)(JSContext*, Handle<NativeObject*>, int32_t, MutableHandleValue);
+ callVM<Fn, js::GetSparseElementHelper>(lir);
+}
+
+void CodeGenerator::visitCallNativeGetElement(LCallNativeGetElement* lir) {
+ Register object = ToRegister(lir->object());
+ Register index = ToRegister(lir->index());
+
+ pushArg(index);
+ pushArg(TypedOrValueRegister(MIRType::Object, AnyRegister(object)));
+ pushArg(object);
+
+ using Fn = bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, int32_t,
+ MutableHandleValue);
+ callVM<Fn, js::NativeGetElement>(lir);
+}
+
+void CodeGenerator::visitCallNativeGetElementSuper(
+ LCallNativeGetElementSuper* lir) {
+ Register object = ToRegister(lir->object());
+ Register index = ToRegister(lir->index());
+ ValueOperand receiver =
+ ToValue(lir, LCallNativeGetElementSuper::ReceiverIndex);
+
+ pushArg(index);
+ pushArg(receiver);
+ pushArg(object);
+
+ using Fn = bool (*)(JSContext*, Handle<NativeObject*>, HandleValue, int32_t,
+ MutableHandleValue);
+ callVM<Fn, js::NativeGetElement>(lir);
+}
+
+void CodeGenerator::visitCallObjectHasSparseElement(
+ LCallObjectHasSparseElement* lir) {
+ Register object = ToRegister(lir->object());
+ Register index = ToRegister(lir->index());
+ Register temp0 = ToRegister(lir->temp0());
+ Register temp1 = ToRegister(lir->temp1());
+ Register output = ToRegister(lir->output());
+
+ masm.reserveStack(sizeof(Value));
+ masm.moveStackPtrTo(temp1);
+
+ using Fn = bool (*)(JSContext*, NativeObject*, int32_t, Value*);
+ masm.setupAlignedABICall();
+ masm.loadJSContext(temp0);
+ masm.passABIArg(temp0);
+ masm.passABIArg(object);
+ masm.passABIArg(index);
+ masm.passABIArg(temp1);
+ masm.callWithABI<Fn, HasNativeElementPure>();
+ masm.storeCallPointerResult(temp0);
+
+ Label bail, ok;
+ uint32_t framePushed = masm.framePushed();
+ masm.branchIfTrueBool(temp0, &ok);
+ masm.adjustStack(sizeof(Value));
+ masm.jump(&bail);
+
+ masm.bind(&ok);
+ masm.setFramePushed(framePushed);
+ masm.unboxBoolean(Address(masm.getStackPointer(), 0), output);
+ masm.adjustStack(sizeof(Value));
+
+ bailoutFrom(&bail, lir->snapshot());
+}
+
+void CodeGenerator::visitBigIntAsIntN(LBigIntAsIntN* ins) {
+ Register bits = ToRegister(ins->bits());
+ Register input = ToRegister(ins->input());
+
+ pushArg(bits);
+ pushArg(input);
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, int32_t);
+ callVM<Fn, jit::BigIntAsIntN>(ins);
+}
+
+void CodeGenerator::visitBigIntAsIntN64(LBigIntAsIntN64* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp = ToRegister(ins->temp());
+ Register64 temp64 = ToRegister64(ins->temp64());
+ Register output = ToRegister(ins->output());
+
+ Label done, create;
+
+ masm.movePtr(input, output);
+
+ // Load the BigInt value as an int64.
+ masm.loadBigInt64(input, temp64);
+
+ // Create a new BigInt when the input exceeds the int64 range.
+ masm.branch32(Assembler::Above, Address(input, BigInt::offsetOfLength()),
+ Imm32(64 / BigInt::DigitBits), &create);
+
+ // And create a new BigInt when the value and the BigInt have different signs.
+ Label nonNegative;
+ masm.branchIfBigIntIsNonNegative(input, &nonNegative);
+ masm.branchTest64(Assembler::NotSigned, temp64, temp64, temp, &create);
+ masm.jump(&done);
+
+ masm.bind(&nonNegative);
+ masm.branchTest64(Assembler::NotSigned, temp64, temp64, temp, &done);
+
+ masm.bind(&create);
+ emitCreateBigInt(ins, Scalar::BigInt64, temp64, output, temp);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitBigIntAsIntN32(LBigIntAsIntN32* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp = ToRegister(ins->temp());
+ Register64 temp64 = ToRegister64(ins->temp64());
+ Register output = ToRegister(ins->output());
+
+ Label done, create;
+
+ masm.movePtr(input, output);
+
+ // Load the absolute value of the first digit.
+ masm.loadFirstBigIntDigitOrZero(input, temp);
+
+ // If the absolute value exceeds the int32 range, create a new BigInt.
+ masm.branchPtr(Assembler::Above, temp, Imm32(INT32_MAX), &create);
+
+ // Also create a new BigInt if we have more than one digit.
+ masm.branch32(Assembler::BelowOrEqual,
+ Address(input, BigInt::offsetOfLength()), Imm32(1), &done);
+
+ masm.bind(&create);
+
+ // |temp| stores the absolute value, negate it when the sign flag is set.
+ Label nonNegative;
+ masm.branchIfBigIntIsNonNegative(input, &nonNegative);
+ masm.negPtr(temp);
+ masm.bind(&nonNegative);
+
+ masm.move32To64SignExtend(temp, temp64);
+ emitCreateBigInt(ins, Scalar::BigInt64, temp64, output, temp);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitBigIntAsUintN(LBigIntAsUintN* ins) {
+ Register bits = ToRegister(ins->bits());
+ Register input = ToRegister(ins->input());
+
+ pushArg(bits);
+ pushArg(input);
+
+ using Fn = BigInt* (*)(JSContext*, HandleBigInt, int32_t);
+ callVM<Fn, jit::BigIntAsUintN>(ins);
+}
+
+void CodeGenerator::visitBigIntAsUintN64(LBigIntAsUintN64* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp = ToRegister(ins->temp());
+ Register64 temp64 = ToRegister64(ins->temp64());
+ Register output = ToRegister(ins->output());
+
+ Label done, create;
+
+ masm.movePtr(input, output);
+
+ // Load the BigInt value as an uint64.
+ masm.loadBigInt64(input, temp64);
+
+ // Create a new BigInt when the input exceeds the uint64 range.
+ masm.branch32(Assembler::Above, Address(input, BigInt::offsetOfLength()),
+ Imm32(64 / BigInt::DigitBits), &create);
+
+ // And create a new BigInt when the input has the sign flag set.
+ masm.branchIfBigIntIsNonNegative(input, &done);
+
+ masm.bind(&create);
+ emitCreateBigInt(ins, Scalar::BigUint64, temp64, output, temp);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitBigIntAsUintN32(LBigIntAsUintN32* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp = ToRegister(ins->temp());
+ Register64 temp64 = ToRegister64(ins->temp64());
+ Register output = ToRegister(ins->output());
+
+ Label done, create;
+
+ masm.movePtr(input, output);
+
+ // Load the absolute value of the first digit.
+ masm.loadFirstBigIntDigitOrZero(input, temp);
+
+ // If the absolute value exceeds the uint32 range, create a new BigInt.
+#if JS_PUNBOX64
+ masm.branchPtr(Assembler::Above, temp, ImmWord(UINT32_MAX), &create);
+#endif
+
+ // Also create a new BigInt if we have more than one digit.
+ masm.branch32(Assembler::Above, Address(input, BigInt::offsetOfLength()),
+ Imm32(1), &create);
+
+ // And create a new BigInt when the input has the sign flag set.
+ masm.branchIfBigIntIsNonNegative(input, &done);
+
+ masm.bind(&create);
+
+ // |temp| stores the absolute value, negate it when the sign flag is set.
+ Label nonNegative;
+ masm.branchIfBigIntIsNonNegative(input, &nonNegative);
+ masm.negPtr(temp);
+ masm.bind(&nonNegative);
+
+ masm.move32To64ZeroExtend(temp, temp64);
+ emitCreateBigInt(ins, Scalar::BigUint64, temp64, output, temp);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitGuardNonGCThing(LGuardNonGCThing* ins) {
+ ValueOperand input = ToValue(ins, LGuardNonGCThing::InputIndex);
+
+ Label bail;
+ masm.branchTestGCThing(Assembler::Equal, input, &bail);
+ bailoutFrom(&bail, ins->snapshot());
+}
+
+void CodeGenerator::visitToHashableNonGCThing(LToHashableNonGCThing* ins) {
+ ValueOperand input = ToValue(ins, LToHashableNonGCThing::InputIndex);
+ FloatRegister tempFloat = ToFloatRegister(ins->temp0());
+ ValueOperand output = ToOutValue(ins);
+
+ masm.toHashableNonGCThing(input, output, tempFloat);
+}
+
+void CodeGenerator::visitToHashableString(LToHashableString* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ using Fn = JSAtom* (*)(JSContext*, JSString*);
+ auto* ool = oolCallVM<Fn, js::AtomizeString>(ins, ArgList(input),
+ StoreRegisterTo(output));
+
+ masm.branchTest32(Assembler::Zero, Address(input, JSString::offsetOfFlags()),
+ Imm32(JSString::ATOM_BIT), ool->entry());
+ masm.movePtr(input, output);
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitToHashableValue(LToHashableValue* ins) {
+ ValueOperand input = ToValue(ins, LToHashableValue::InputIndex);
+ FloatRegister tempFloat = ToFloatRegister(ins->temp0());
+ ValueOperand output = ToOutValue(ins);
+
+ Register str = output.scratchReg();
+
+ using Fn = JSAtom* (*)(JSContext*, JSString*);
+ auto* ool =
+ oolCallVM<Fn, js::AtomizeString>(ins, ArgList(str), StoreRegisterTo(str));
+
+ masm.toHashableValue(input, output, tempFloat, ool->entry(), ool->rejoin());
+}
+
+void CodeGenerator::visitHashNonGCThing(LHashNonGCThing* ins) {
+ ValueOperand input = ToValue(ins, LHashNonGCThing::InputIndex);
+ Register temp = ToRegister(ins->temp0());
+ Register output = ToRegister(ins->output());
+
+ masm.prepareHashNonGCThing(input, output, temp);
+}
+
+void CodeGenerator::visitHashString(LHashString* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp = ToRegister(ins->temp0());
+ Register output = ToRegister(ins->output());
+
+ masm.prepareHashString(input, output, temp);
+}
+
+void CodeGenerator::visitHashSymbol(LHashSymbol* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.prepareHashSymbol(input, output);
+}
+
+void CodeGenerator::visitHashBigInt(LHashBigInt* ins) {
+ Register input = ToRegister(ins->input());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register output = ToRegister(ins->output());
+
+ masm.prepareHashBigInt(input, output, temp0, temp1, temp2);
+}
+
+void CodeGenerator::visitHashObject(LHashObject* ins) {
+ Register setObj = ToRegister(ins->setObject());
+ ValueOperand input = ToValue(ins, LHashObject::InputIndex);
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ Register output = ToRegister(ins->output());
+
+ masm.prepareHashObject(setObj, input, output, temp0, temp1, temp2, temp3);
+}
+
+void CodeGenerator::visitHashValue(LHashValue* ins) {
+ Register setObj = ToRegister(ins->setObject());
+ ValueOperand input = ToValue(ins, LHashValue::InputIndex);
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ Register output = ToRegister(ins->output());
+
+ masm.prepareHashValue(setObj, input, output, temp0, temp1, temp2, temp3);
+}
+
+void CodeGenerator::visitSetObjectHasNonBigInt(LSetObjectHasNonBigInt* ins) {
+ Register setObj = ToRegister(ins->setObject());
+ ValueOperand input = ToValue(ins, LSetObjectHasNonBigInt::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register output = ToRegister(ins->output());
+
+ masm.setObjectHasNonBigInt(setObj, input, hash, output, temp0, temp1);
+}
+
+void CodeGenerator::visitSetObjectHasBigInt(LSetObjectHasBigInt* ins) {
+ Register setObj = ToRegister(ins->setObject());
+ ValueOperand input = ToValue(ins, LSetObjectHasBigInt::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ Register output = ToRegister(ins->output());
+
+ masm.setObjectHasBigInt(setObj, input, hash, output, temp0, temp1, temp2,
+ temp3);
+}
+
+void CodeGenerator::visitSetObjectHasValue(LSetObjectHasValue* ins) {
+ Register setObj = ToRegister(ins->setObject());
+ ValueOperand input = ToValue(ins, LSetObjectHasValue::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ Register output = ToRegister(ins->output());
+
+ masm.setObjectHasValue(setObj, input, hash, output, temp0, temp1, temp2,
+ temp3);
+}
+
+void CodeGenerator::visitSetObjectHasValueVMCall(
+ LSetObjectHasValueVMCall* ins) {
+ pushArg(ToValue(ins, LSetObjectHasValueVMCall::InputIndex));
+ pushArg(ToRegister(ins->setObject()));
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
+ callVM<Fn, jit::SetObjectHas>(ins);
+}
+
+void CodeGenerator::visitSetObjectSize(LSetObjectSize* ins) {
+ Register setObj = ToRegister(ins->setObject());
+ Register output = ToRegister(ins->output());
+
+ masm.loadSetObjectSize(setObj, output);
+}
+
+void CodeGenerator::visitMapObjectHasNonBigInt(LMapObjectHasNonBigInt* ins) {
+ Register mapObj = ToRegister(ins->mapObject());
+ ValueOperand input = ToValue(ins, LMapObjectHasNonBigInt::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register output = ToRegister(ins->output());
+
+ masm.mapObjectHasNonBigInt(mapObj, input, hash, output, temp0, temp1);
+}
+
+void CodeGenerator::visitMapObjectHasBigInt(LMapObjectHasBigInt* ins) {
+ Register mapObj = ToRegister(ins->mapObject());
+ ValueOperand input = ToValue(ins, LMapObjectHasBigInt::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ Register output = ToRegister(ins->output());
+
+ masm.mapObjectHasBigInt(mapObj, input, hash, output, temp0, temp1, temp2,
+ temp3);
+}
+
+void CodeGenerator::visitMapObjectHasValue(LMapObjectHasValue* ins) {
+ Register mapObj = ToRegister(ins->mapObject());
+ ValueOperand input = ToValue(ins, LMapObjectHasValue::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ Register output = ToRegister(ins->output());
+
+ masm.mapObjectHasValue(mapObj, input, hash, output, temp0, temp1, temp2,
+ temp3);
+}
+
+void CodeGenerator::visitMapObjectHasValueVMCall(
+ LMapObjectHasValueVMCall* ins) {
+ pushArg(ToValue(ins, LMapObjectHasValueVMCall::InputIndex));
+ pushArg(ToRegister(ins->mapObject()));
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool*);
+ callVM<Fn, jit::MapObjectHas>(ins);
+}
+
+void CodeGenerator::visitMapObjectGetNonBigInt(LMapObjectGetNonBigInt* ins) {
+ Register mapObj = ToRegister(ins->mapObject());
+ ValueOperand input = ToValue(ins, LMapObjectGetNonBigInt::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ ValueOperand output = ToOutValue(ins);
+
+ masm.mapObjectGetNonBigInt(mapObj, input, hash, output, temp0, temp1,
+ output.scratchReg());
+}
+
+void CodeGenerator::visitMapObjectGetBigInt(LMapObjectGetBigInt* ins) {
+ Register mapObj = ToRegister(ins->mapObject());
+ ValueOperand input = ToValue(ins, LMapObjectGetBigInt::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ ValueOperand output = ToOutValue(ins);
+
+ masm.mapObjectGetBigInt(mapObj, input, hash, output, temp0, temp1, temp2,
+ temp3, output.scratchReg());
+}
+
+void CodeGenerator::visitMapObjectGetValue(LMapObjectGetValue* ins) {
+ Register mapObj = ToRegister(ins->mapObject());
+ ValueOperand input = ToValue(ins, LMapObjectGetValue::InputIndex);
+ Register hash = ToRegister(ins->hash());
+ Register temp0 = ToRegister(ins->temp0());
+ Register temp1 = ToRegister(ins->temp1());
+ Register temp2 = ToRegister(ins->temp2());
+ Register temp3 = ToRegister(ins->temp3());
+ ValueOperand output = ToOutValue(ins);
+
+ masm.mapObjectGetValue(mapObj, input, hash, output, temp0, temp1, temp2,
+ temp3, output.scratchReg());
+}
+
+void CodeGenerator::visitMapObjectGetValueVMCall(
+ LMapObjectGetValueVMCall* ins) {
+ pushArg(ToValue(ins, LMapObjectGetValueVMCall::InputIndex));
+ pushArg(ToRegister(ins->mapObject()));
+
+ using Fn =
+ bool (*)(JSContext*, HandleObject, HandleValue, MutableHandleValue);
+ callVM<Fn, jit::MapObjectGet>(ins);
+}
+
+void CodeGenerator::visitMapObjectSize(LMapObjectSize* ins) {
+ Register mapObj = ToRegister(ins->mapObject());
+ Register output = ToRegister(ins->output());
+
+ masm.loadMapObjectSize(mapObj, output);
+}
+
+template <size_t NumDefs>
+void CodeGenerator::emitIonToWasmCallBase(LIonToWasmCallBase<NumDefs>* lir) {
+ wasm::JitCallStackArgVector stackArgs;
+ masm.propagateOOM(stackArgs.reserve(lir->numOperands()));
+ if (masm.oom()) {
+ return;
+ }
+
+ MIonToWasmCall* mir = lir->mir();
+ const wasm::FuncExport& funcExport = mir->funcExport();
+ const wasm::FuncType& sig =
+ mir->instance()->metadata().getFuncExportType(funcExport);
+
+ WasmABIArgGenerator abi;
+ for (size_t i = 0; i < lir->numOperands(); i++) {
+ MIRType argMir;
+ switch (sig.args()[i].kind()) {
+ case wasm::ValType::I32:
+ case wasm::ValType::I64:
+ case wasm::ValType::F32:
+ case wasm::ValType::F64:
+ argMir = sig.args()[i].toMIRType();
+ break;
+ case wasm::ValType::V128:
+ MOZ_CRASH("unexpected argument type when calling from ion to wasm");
+ case wasm::ValType::Ref:
+ // temporarilyUnsupportedReftypeForEntry() restricts args to externref
+ MOZ_RELEASE_ASSERT(sig.args()[i].refType().isExtern());
+ // Argument is boxed on the JS side to an anyref, so passed as a
+ // pointer here.
+ argMir = sig.args()[i].toMIRType();
+ break;
+ }
+
+ ABIArg arg = abi.next(argMir);
+ switch (arg.kind()) {
+ case ABIArg::GPR:
+ case ABIArg::FPU: {
+ MOZ_ASSERT(ToAnyRegister(lir->getOperand(i)) == arg.reg());
+ stackArgs.infallibleEmplaceBack(wasm::JitCallStackArg());
+ break;
+ }
+ case ABIArg::Stack: {
+ const LAllocation* larg = lir->getOperand(i);
+ if (larg->isConstant()) {
+ stackArgs.infallibleEmplaceBack(ToInt32(larg));
+ } else if (larg->isGeneralReg()) {
+ stackArgs.infallibleEmplaceBack(ToRegister(larg));
+ } else if (larg->isFloatReg()) {
+ stackArgs.infallibleEmplaceBack(ToFloatRegister(larg));
+ } else {
+ // Always use the stack pointer here because GenerateDirectCallFromJit
+ // depends on this.
+ Address addr = ToAddress<BaseRegForAddress::SP>(larg);
+ stackArgs.infallibleEmplaceBack(addr);
+ }
+ break;
+ }
+#ifdef JS_CODEGEN_REGISTER_PAIR
+ case ABIArg::GPR_PAIR: {
+ MOZ_CRASH(
+ "no way to pass i64, and wasm uses hardfp for function calls");
+ }
+#endif
+ case ABIArg::Uninitialized: {
+ MOZ_CRASH("Uninitialized ABIArg kind");
+ }
+ }
+ }
+
+ const wasm::ValTypeVector& results = sig.results();
+ if (results.length() == 0) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Value);
+ } else {
+ MOZ_ASSERT(results.length() == 1, "multi-value return unimplemented");
+ switch (results[0].kind()) {
+ case wasm::ValType::I32:
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int32);
+ MOZ_ASSERT(ToRegister(lir->output()) == ReturnReg);
+ break;
+ case wasm::ValType::I64:
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(ToOutRegister64(lir) == ReturnReg64);
+ break;
+ case wasm::ValType::F32:
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Float32);
+ MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnFloat32Reg);
+ break;
+ case wasm::ValType::F64:
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(ToFloatRegister(lir->output()) == ReturnDoubleReg);
+ break;
+ case wasm::ValType::V128:
+ MOZ_CRASH("unexpected return type when calling from ion to wasm");
+ case wasm::ValType::Ref:
+ // The wasm stubs layer unboxes anything that needs to be unboxed
+ // and leaves it in a Value. A FuncRef/EqRef we could in principle
+ // leave it as a raw object pointer but for now it complicates the
+ // API to do so.
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Value);
+ break;
+ }
+ }
+
+ WasmInstanceObject* instObj = lir->mir()->instanceObject();
+
+ Register scratch = ToRegister(lir->temp());
+
+ uint32_t callOffset;
+ GenerateDirectCallFromJit(masm, funcExport, instObj->instance(), stackArgs,
+ scratch, &callOffset);
+
+ // Add the instance object to the constant pool, so it is transferred to
+ // the owning IonScript and so that it gets traced as long as the IonScript
+ // lives.
+
+ uint32_t unused;
+ masm.propagateOOM(graph.addConstantToPool(ObjectValue(*instObj), &unused));
+
+ markSafepointAt(callOffset, lir);
+}
+
+void CodeGenerator::visitIonToWasmCall(LIonToWasmCall* lir) {
+ emitIonToWasmCallBase(lir);
+}
+void CodeGenerator::visitIonToWasmCallV(LIonToWasmCallV* lir) {
+ emitIonToWasmCallBase(lir);
+}
+void CodeGenerator::visitIonToWasmCallI64(LIonToWasmCallI64* lir) {
+ emitIonToWasmCallBase(lir);
+}
+
+void CodeGenerator::visitWasmNullConstant(LWasmNullConstant* lir) {
+ masm.xorPtr(ToRegister(lir->output()), ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmFence(LWasmFence* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ masm.memoryBarrier(MembarFull);
+}
+
+void CodeGenerator::visitWasmBoxValue(LWasmBoxValue* lir) {
+ ValueOperand input = ToValue(lir, LWasmBoxValue::InputIndex);
+ Register output = ToRegister(lir->output());
+
+ Label nullValue, objectValue, done;
+ {
+ ScratchTagScope tag(masm, input);
+ masm.splitTagForTest(input, tag);
+ masm.branchTestObject(Assembler::Equal, tag, &objectValue);
+ masm.branchTestNull(Assembler::Equal, tag, &nullValue);
+ }
+
+ using Fn = JSObject* (*)(JSContext*, HandleValue);
+ OutOfLineCode* oolBoxValue = oolCallVM<Fn, wasm::BoxBoxableValue>(
+ lir, ArgList(input), StoreRegisterTo(output));
+
+ masm.jump(oolBoxValue->entry());
+
+ masm.bind(&nullValue);
+ // See the definition of AnyRef for a discussion of pointer representation.
+ masm.xorPtr(output, output);
+ masm.jump(&done);
+
+ masm.bind(&objectValue);
+ // See the definition of AnyRef for a discussion of pointer representation.
+ masm.unboxObject(input, output);
+
+ masm.bind(&done);
+ masm.bind(oolBoxValue->rejoin());
+}
+
+void CodeGenerator::visitWasmAnyRefFromJSObject(LWasmAnyRefFromJSObject* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ // See the definition of AnyRef for a discussion of pointer representation.
+ if (input != output) {
+ masm.movePtr(input, output);
+ }
+}
+
+#ifdef FUZZING_JS_FUZZILLI
+void CodeGenerator::emitFuzzilliHashDouble(FloatRegister floatDouble,
+ Register scratch, Register output) {
+# ifdef JS_PUNBOX64
+ Register64 reg64_1(scratch);
+ Register64 reg64_2(output);
+ masm.moveDoubleToGPR64(floatDouble, reg64_1);
+ masm.move64(reg64_1, reg64_2);
+ masm.rshift64(Imm32(32), reg64_2);
+ masm.add32(scratch, output);
+# else
+ Register64 reg64(scratch, output);
+ masm.moveDoubleToGPR64(floatDouble, reg64);
+ masm.add32(scratch, output);
+# endif
+}
+
+void CodeGenerator::emitFuzzilliHashObject(LInstruction* lir, Register obj,
+ Register output) {
+ using Fn = void (*)(JSContext * cx, JSObject * obj, uint32_t * out);
+ OutOfLineCode* ool = oolCallVM<Fn, FuzzilliHashObjectInl>(
+ lir, ArgList(obj), StoreRegisterTo(output));
+
+ masm.jump(ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::emitFuzzilliHashBigInt(Register bigInt, Register output) {
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::All(),
+ FloatRegisterSet::All());
+ volatileRegs.takeUnchecked(output);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = uint32_t (*)(BigInt * bigInt);
+ masm.setupUnalignedABICall(output);
+ masm.passABIArg(bigInt);
+ masm.callWithABI<Fn, js::FuzzilliHashBigInt>();
+ masm.storeCallInt32Result(output);
+
+ masm.PopRegsInMask(volatileRegs);
+}
+
+void CodeGenerator::visitFuzzilliHashV(LFuzzilliHashV* ins) {
+ MOZ_ASSERT(ins->mir()->getOperand(0)->type() == MIRType::Value);
+
+ ValueOperand value = ToValue(ins, 0);
+
+ Label isDouble, isObject, isBigInt, done;
+
+ FloatRegister scratchFloat = ToFloatRegister(ins->getTemp(1));
+ Register scratch = ToRegister(ins->getTemp(0));
+ Register output = ToRegister(ins->output());
+ MOZ_ASSERT(scratch != output);
+
+# ifdef JS_PUNBOX64
+ Register tagReg = ToRegister(ins->getTemp(0));
+ masm.splitTag(value, tagReg);
+# else
+ Register tagReg = value.typeReg();
+# endif
+
+ Label noBigInt;
+ masm.branchTestBigInt(Assembler::NotEqual, tagReg, &noBigInt);
+ masm.unboxBigInt(value, scratch);
+ masm.jump(&isBigInt);
+ masm.bind(&noBigInt);
+
+ Label noObject;
+ masm.branchTestObject(Assembler::NotEqual, tagReg, &noObject);
+ masm.unboxObject(value, scratch);
+ masm.jump(&isObject);
+ masm.bind(&noObject);
+
+ Label noInt32;
+ masm.branchTestInt32(Assembler::NotEqual, tagReg, &noInt32);
+ masm.unboxInt32(value, scratch);
+ masm.convertInt32ToDouble(scratch, scratchFloat);
+ masm.jump(&isDouble);
+ masm.bind(&noInt32);
+
+ Label noNull;
+ masm.branchTestNull(Assembler::NotEqual, tagReg, &noNull);
+ masm.move32(Imm32(1), scratch);
+ masm.convertInt32ToDouble(scratch, scratchFloat);
+ masm.jump(&isDouble);
+ masm.bind(&noNull);
+
+ Label noUndefined;
+ masm.branchTestUndefined(Assembler::NotEqual, tagReg, &noUndefined);
+ masm.move32(Imm32(2), scratch);
+ masm.convertInt32ToDouble(scratch, scratchFloat);
+ masm.jump(&isDouble);
+ masm.bind(&noUndefined);
+
+ Label noBoolean;
+ masm.branchTestBoolean(Assembler::NotEqual, tagReg, &noBoolean);
+ masm.unboxBoolean(value, scratch);
+ masm.add32(Imm32(3), scratch);
+ masm.convertInt32ToDouble(scratch, scratchFloat);
+ masm.jump(&isDouble);
+ masm.bind(&noBoolean);
+
+ Label noDouble;
+ masm.branchTestDouble(Assembler::NotEqual, tagReg, &noDouble);
+ masm.unboxDouble(value, scratchFloat);
+ masm.canonicalizeDoubleIfDeterministic(scratchFloat);
+
+ masm.jump(&isDouble);
+ masm.bind(&noDouble);
+ masm.move32(Imm32(0), output);
+ masm.jump(&done);
+
+ masm.bind(&isBigInt);
+ emitFuzzilliHashBigInt(scratch, output);
+ masm.jump(&done);
+
+ masm.bind(&isObject);
+ emitFuzzilliHashObject(ins, scratch, output);
+ masm.jump(&done);
+
+ masm.bind(&isDouble);
+ emitFuzzilliHashDouble(scratchFloat, scratch, output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitFuzzilliHashT(LFuzzilliHashT* ins) {
+ const LAllocation* value = ins->value();
+ MIRType mirType = ins->mir()->getOperand(0)->type();
+
+ FloatRegister scratchFloat = ToFloatRegister(ins->getTemp(1));
+ Register scratch = ToRegister(ins->getTemp(0));
+ Register output = ToRegister(ins->output());
+ MOZ_ASSERT(scratch != output);
+
+ if (mirType == MIRType::Object) {
+ MOZ_ASSERT(value->isGeneralReg());
+ masm.mov(value->toGeneralReg()->reg(), scratch);
+ emitFuzzilliHashObject(ins, scratch, output);
+ } else if (mirType == MIRType::BigInt) {
+ MOZ_ASSERT(value->isGeneralReg());
+ masm.mov(value->toGeneralReg()->reg(), scratch);
+ emitFuzzilliHashBigInt(scratch, output);
+ } else if (mirType == MIRType::Double) {
+ MOZ_ASSERT(value->isFloatReg());
+ masm.moveDouble(value->toFloatReg()->reg(), scratchFloat);
+ masm.canonicalizeDoubleIfDeterministic(scratchFloat);
+ emitFuzzilliHashDouble(scratchFloat, scratch, output);
+ } else if (mirType == MIRType::Float32) {
+ MOZ_ASSERT(value->isFloatReg());
+ masm.convertFloat32ToDouble(value->toFloatReg()->reg(), scratchFloat);
+ masm.canonicalizeDoubleIfDeterministic(scratchFloat);
+ emitFuzzilliHashDouble(scratchFloat, scratch, output);
+ } else if (mirType == MIRType::Int32) {
+ MOZ_ASSERT(value->isGeneralReg());
+ masm.mov(value->toGeneralReg()->reg(), scratch);
+ masm.convertInt32ToDouble(scratch, scratchFloat);
+ emitFuzzilliHashDouble(scratchFloat, scratch, output);
+ } else if (mirType == MIRType::Null) {
+ MOZ_ASSERT(value->isBogus());
+ masm.move32(Imm32(1), scratch);
+ masm.convertInt32ToDouble(scratch, scratchFloat);
+ emitFuzzilliHashDouble(scratchFloat, scratch, output);
+ } else if (mirType == MIRType::Undefined) {
+ MOZ_ASSERT(value->isBogus());
+ masm.move32(Imm32(2), scratch);
+ masm.convertInt32ToDouble(scratch, scratchFloat);
+ emitFuzzilliHashDouble(scratchFloat, scratch, output);
+ } else if (mirType == MIRType::Boolean) {
+ MOZ_ASSERT(value->isGeneralReg());
+ masm.mov(value->toGeneralReg()->reg(), scratch);
+ masm.add32(Imm32(3), scratch);
+ masm.convertInt32ToDouble(scratch, scratchFloat);
+ emitFuzzilliHashDouble(scratchFloat, scratch, output);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void CodeGenerator::visitFuzzilliHashStore(LFuzzilliHashStore* ins) {
+ const LAllocation* value = ins->value();
+ MOZ_ASSERT(ins->mir()->getOperand(0)->type() == MIRType::Int32);
+ MOZ_ASSERT(value->isGeneralReg());
+
+ Register scratchJSContext = ToRegister(ins->getTemp(0));
+ Register scratch = ToRegister(ins->getTemp(1));
+
+ masm.loadJSContext(scratchJSContext);
+
+ // stats
+ Address addrExecHashInputs(scratchJSContext,
+ offsetof(JSContext, executionHashInputs));
+ masm.load32(addrExecHashInputs, scratch);
+ masm.add32(Imm32(1), scratch);
+ masm.store32(scratch, addrExecHashInputs);
+
+ Address addrExecHash(scratchJSContext, offsetof(JSContext, executionHash));
+ masm.load32(addrExecHash, scratch);
+ masm.add32(value->toGeneralReg()->reg(), scratch);
+ masm.rotateLeft(Imm32(1), scratch, scratch);
+ masm.store32(scratch, addrExecHash);
+}
+#endif
+
+static_assert(!std::is_polymorphic_v<CodeGenerator>,
+ "CodeGenerator should not have any virtual methods");
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/CodeGenerator.h b/js/src/jit/CodeGenerator.h
new file mode 100644
index 0000000000..a56dbd8b22
--- /dev/null
+++ b/js/src/jit/CodeGenerator.h
@@ -0,0 +1,427 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CodeGenerator_h
+#define jit_CodeGenerator_h
+
+#include "jit/PerfSpewer.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/CodeGenerator-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/CodeGenerator-x64.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/CodeGenerator-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/CodeGenerator-arm64.h"
+#elif defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/CodeGenerator-mips32.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/CodeGenerator-mips64.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/CodeGenerator-loong64.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/CodeGenerator-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/CodeGenerator-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+namespace js {
+
+namespace wasm {
+class Decoder;
+class StackMaps;
+} // namespace wasm
+
+namespace jit {
+
+class WarpSnapshot;
+
+template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
+class OutOfLineCallVM;
+
+enum class SwitchTableType { Inline, OutOfLine };
+
+template <SwitchTableType tableType>
+class OutOfLineSwitch;
+class OutOfLineTestObject;
+class OutOfLineNewArray;
+class OutOfLineNewObject;
+class CheckOverRecursedFailure;
+class OutOfLineUnboxFloatingPoint;
+class OutOfLineStoreElementHole;
+class OutOfLineTypeOfV;
+class OutOfLineTypeOfIsNonPrimitiveV;
+class OutOfLineTypeOfIsNonPrimitiveO;
+class OutOfLineUpdateCache;
+class OutOfLineICFallback;
+class OutOfLineCallPostWriteBarrier;
+class OutOfLineCallPostWriteElementBarrier;
+class OutOfLineIsCallable;
+class OutOfLineIsConstructor;
+class OutOfLineRegExpMatcher;
+class OutOfLineRegExpSearcher;
+class OutOfLineRegExpTester;
+class OutOfLineRegExpPrototypeOptimizable;
+class OutOfLineRegExpInstanceOptimizable;
+class OutOfLineNaNToZero;
+class OutOfLineResumableWasmTrap;
+class OutOfLineAbortingWasmTrap;
+class OutOfLineZeroIfNaN;
+class OutOfLineGuardNumberToIntPtrIndex;
+class OutOfLineBoxNonStrictThis;
+class OutOfLineArrayPush;
+
+class CodeGenerator final : public CodeGeneratorSpecific {
+ [[nodiscard]] bool generateBody();
+
+ ConstantOrRegister toConstantOrRegister(LInstruction* lir, size_t n,
+ MIRType type);
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ void resetOsiPointRegs(LSafepoint* safepoint);
+ bool shouldVerifyOsiPointRegs(LSafepoint* safepoint);
+ void verifyOsiPointRegs(LSafepoint* safepoint);
+#endif
+
+ void callVMInternal(VMFunctionId id, LInstruction* ins);
+
+ template <typename Fn, Fn fn>
+ void callVM(LInstruction* ins);
+
+ template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
+ inline OutOfLineCode* oolCallVM(LInstruction* ins, const ArgSeq& args,
+ const StoreOutputTo& out);
+
+ public:
+ CodeGenerator(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm = nullptr);
+ ~CodeGenerator();
+
+ [[nodiscard]] bool generate();
+ [[nodiscard]] bool generateWasm(
+ wasm::CallIndirectId callIndirectId, wasm::BytecodeOffset trapOffset,
+ const wasm::ArgTypeVector& argTys, const RegisterOffsets& trapExitLayout,
+ size_t trapExitLayoutNumWords, wasm::FuncOffsets* offsets,
+ wasm::StackMaps* stackMaps, wasm::Decoder* decoder);
+
+ [[nodiscard]] bool link(JSContext* cx, const WarpSnapshot* snapshot);
+
+ void emitOOLTestObject(Register objreg, Label* ifTruthy, Label* ifFalsy,
+ Register scratch);
+ void emitIntToString(Register input, Register output, Label* ool);
+
+ void emitTypeOfCheck(JSValueType type, Register tag, Register output,
+ Label* done, Label* oolObject);
+ void emitTypeOfJSType(JSValueType type, Register output);
+ void emitTypeOfObject(Register obj, Register output, Label* done);
+ void emitTypeOfIsObject(MTypeOfIs* mir, Register obj, Register output,
+ Label* success, Label* fail, Label* slowCheck);
+ void emitTypeOfIsObjectOOL(MTypeOfIs* mir, Register obj, Register output);
+
+ template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
+ void visitOutOfLineCallVM(
+ OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool);
+
+ void visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher* ool);
+ void visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher* ool);
+ void visitOutOfLineRegExpTester(OutOfLineRegExpTester* ool);
+ void visitOutOfLineRegExpPrototypeOptimizable(
+ OutOfLineRegExpPrototypeOptimizable* ool);
+ void visitOutOfLineRegExpInstanceOptimizable(
+ OutOfLineRegExpInstanceOptimizable* ool);
+
+ void visitOutOfLineTypeOfV(OutOfLineTypeOfV* ool);
+ void visitOutOfLineTypeOfIsNonPrimitiveV(OutOfLineTypeOfIsNonPrimitiveV* ool);
+ void visitOutOfLineTypeOfIsNonPrimitiveO(OutOfLineTypeOfIsNonPrimitiveO* ool);
+
+ template <SwitchTableType tableType>
+ void visitOutOfLineSwitch(OutOfLineSwitch<tableType>* ool);
+
+ void visitOutOfLineIsCallable(OutOfLineIsCallable* ool);
+ void visitOutOfLineIsConstructor(OutOfLineIsConstructor* ool);
+
+ void visitOutOfLineNaNToZero(OutOfLineNaNToZero* ool);
+ void visitOutOfLineZeroIfNaN(OutOfLineZeroIfNaN* ool);
+
+ void visitOutOfLineResumableWasmTrap(OutOfLineResumableWasmTrap* ool);
+ void visitOutOfLineAbortingWasmTrap(OutOfLineAbortingWasmTrap* ool);
+ void visitCheckOverRecursedFailure(CheckOverRecursedFailure* ool);
+
+ void visitOutOfLineUnboxFloatingPoint(OutOfLineUnboxFloatingPoint* ool);
+ void visitOutOfLineStoreElementHole(OutOfLineStoreElementHole* ool);
+
+ void visitOutOfLineBoxNonStrictThis(OutOfLineBoxNonStrictThis* ool);
+
+ void visitOutOfLineICFallback(OutOfLineICFallback* ool);
+
+ void visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool);
+ void visitOutOfLineCallPostWriteElementBarrier(
+ OutOfLineCallPostWriteElementBarrier* ool);
+
+ void visitOutOfLineNewArray(OutOfLineNewArray* ool);
+ void visitOutOfLineNewObject(OutOfLineNewObject* ool);
+
+ void visitOutOfLineGuardNumberToIntPtrIndex(
+ OutOfLineGuardNumberToIntPtrIndex* ool);
+
+ void visitOutOfLineArrayPush(OutOfLineArrayPush* ool);
+
+ private:
+ void emitPostWriteBarrier(const LAllocation* obj);
+ void emitPostWriteBarrier(Register objreg);
+ void emitPostWriteBarrierS(Address address, Register prev, Register next);
+
+ template <class LPostBarrierType, MIRType nurseryType>
+ void visitPostWriteBarrierCommon(LPostBarrierType* lir, OutOfLineCode* ool);
+ template <class LPostBarrierType>
+ void visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode* ool);
+
+ void emitCallInvokeFunction(LInstruction* call, Register callereg,
+ bool isConstructing, bool ignoresReturnValue,
+ uint32_t argc, uint32_t unusedStack);
+ template <typename T>
+ void emitApplyGeneric(T* apply);
+ template <typename T>
+ void emitCallInvokeFunction(T* apply);
+ void emitAllocateSpaceForApply(Register argcreg, Register scratch);
+ void emitAllocateSpaceForConstructAndPushNewTarget(
+ Register argcreg, Register newTargetAndScratch);
+ void emitCopyValuesForApply(Register argvSrcBase, Register argvIndex,
+ Register copyreg, size_t argvSrcOffset,
+ size_t argvDstOffset);
+ void emitRestoreStackPointerFromFP();
+ void emitPushArguments(Register argcreg, Register scratch, Register copyreg,
+ uint32_t extraFormals);
+ void emitPushArrayAsArguments(Register tmpArgc, Register srcBaseAndArgc,
+ Register scratch, size_t argvSrcOffset);
+ void emitPushArguments(LApplyArgsGeneric* apply, Register scratch);
+ void emitPushArguments(LApplyArgsObj* apply, Register scratch);
+ void emitPushArguments(LApplyArrayGeneric* apply, Register scratch);
+ void emitPushArguments(LConstructArgsGeneric* construct, Register scratch);
+ void emitPushArguments(LConstructArrayGeneric* construct, Register scratch);
+
+ template <class GetInlinedArgument>
+ void emitGetInlinedArgument(GetInlinedArgument* lir, Register index,
+ ValueOperand output);
+
+ using RegisterOrInt32 = mozilla::Variant<Register, int32_t>;
+
+ static RegisterOrInt32 ToRegisterOrInt32(const LAllocation* allocation);
+
+#ifdef DEBUG
+ void emitAssertArgumentsSliceBounds(const RegisterOrInt32& begin,
+ const RegisterOrInt32& count,
+ Register numActualArgs);
+#endif
+
+ template <class ArgumentsSlice>
+ void emitNewArray(ArgumentsSlice* lir, const RegisterOrInt32& count,
+ Register output, Register temp);
+
+ void visitNewArrayCallVM(LNewArray* lir);
+ void visitNewObjectVMCall(LNewObject* lir);
+
+ void emitConcat(LInstruction* lir, Register lhs, Register rhs,
+ Register output);
+
+ void emitInstanceOf(LInstruction* ins, Register protoReg);
+
+ void loadJSScriptForBlock(MBasicBlock* block, Register reg);
+ void loadOutermostJSScript(Register reg);
+
+#ifdef DEBUG
+ void emitAssertResultV(const ValueOperand output, const MDefinition* mir);
+ void emitAssertGCThingResult(Register input, const MDefinition* mir);
+#endif
+
+#ifdef DEBUG
+ void emitDebugForceBailing(LInstruction* lir);
+#endif
+
+ IonScriptCounts* extractScriptCounts() {
+ IonScriptCounts* counts = scriptCounts_;
+ scriptCounts_ = nullptr; // prevent delete in dtor
+ return counts;
+ }
+
+ void addGetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs,
+ TypedOrValueRegister value,
+ const ConstantOrRegister& id, ValueOperand output);
+ void addSetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs,
+ Register objReg, Register temp,
+ const ConstantOrRegister& id,
+ const ConstantOrRegister& value, bool strict);
+
+ template <class IteratorObject, class OrderedHashTable>
+ void emitGetNextEntryForIterator(LGetNextEntryForIterator* lir);
+
+ template <class OrderedHashTable>
+ void emitLoadIteratorValues(Register result, Register temp, Register front);
+
+ void emitStringToInt64(LInstruction* lir, Register input, Register64 output);
+
+ OutOfLineCode* createBigIntOutOfLine(LInstruction* lir, Scalar::Type type,
+ Register64 input, Register output);
+
+ void emitCreateBigInt(LInstruction* lir, Scalar::Type type, Register64 input,
+ Register output, Register maybeTemp);
+
+ template <size_t NumDefs>
+ void emitIonToWasmCallBase(LIonToWasmCallBase<NumDefs>* lir);
+
+ IonScriptCounts* maybeCreateScriptCounts();
+
+ void emitWasmCompareAndSelect(LWasmCompareAndSelect* ins);
+
+ void testValueTruthyForType(JSValueType type, ScratchTagScope& tag,
+ const ValueOperand& value, Register tempToUnbox,
+ Register temp, FloatRegister floatTemp,
+ Label* ifTruthy, Label* ifFalsy,
+ OutOfLineTestObject* ool, bool skipTypeTest);
+
+ // Test whether value is truthy or not and jump to the corresponding label.
+ // The control flow falls through when the object is truthy, as an
+ // optimization.
+ void testValueTruthy(const ValueOperand& value, Register tempToUnbox,
+ Register temp, FloatRegister floatTemp,
+ const TypeDataList& observedTypes, Label* ifTruthy,
+ Label* ifFalsy, OutOfLineTestObject* ool);
+
+ // This function behaves like testObjectEmulatesUndefined with the exception
+ // that it can choose to let control flow fall through when the object
+ // doesn't emulate undefined, as an optimization. Use the regular
+ // testObjectEmulatesUndefined when it's required to branch to one of the
+ // two labels.
+ void testObjectEmulatesUndefinedKernel(Register objreg,
+ Label* ifEmulatesUndefined,
+ Label* ifDoesntEmulateUndefined,
+ Register scratch,
+ OutOfLineTestObject* ool);
+
+ // Test whether an object emulates |undefined|. If it does, jump to
+ // |ifEmulatesUndefined|; the caller is responsible for binding this label.
+ // If it doesn't, fall through; the label |ifDoesntEmulateUndefined| (which
+ // must be initially unbound) will be bound at this point.
+ void branchTestObjectEmulatesUndefined(Register objreg,
+ Label* ifEmulatesUndefined,
+ Label* ifDoesntEmulateUndefined,
+ Register scratch,
+ OutOfLineTestObject* ool);
+
+ // Test whether an object emulates |undefined|, and jump to the
+ // corresponding label.
+ //
+ // This method should be used when subsequent code can't be laid out in a
+ // straight line; if it can, branchTest* should be used instead.
+ void testObjectEmulatesUndefined(Register objreg, Label* ifEmulatesUndefined,
+ Label* ifDoesntEmulateUndefined,
+ Register scratch, OutOfLineTestObject* ool);
+
+ void emitStoreElementTyped(const LAllocation* value, MIRType valueType,
+ Register elements, const LAllocation* index);
+
+ // Bailout if an element about to be written to is a hole.
+ void emitStoreHoleCheck(Register elements, const LAllocation* index,
+ LSnapshot* snapshot);
+
+ void emitAssertRangeI(MIRType type, const Range* r, Register input);
+ void emitAssertRangeD(const Range* r, FloatRegister input,
+ FloatRegister temp);
+
+ void maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal,
+ OutOfLineCode* ool);
+
+ void incrementWarmUpCounter(AbsoluteAddress warmUpCount, JSScript* script,
+ Register tmp);
+
+ Vector<CodeOffset, 0, JitAllocPolicy> ionScriptLabels_;
+
+ // Used to bake in a pointer into the IonScript's list of nursery objects, for
+ // MNurseryObject codegen.
+ struct NurseryObjectLabel {
+ CodeOffset offset;
+ uint32_t nurseryIndex;
+ NurseryObjectLabel(CodeOffset offset, uint32_t nurseryIndex)
+ : offset(offset), nurseryIndex(nurseryIndex) {}
+ };
+ Vector<NurseryObjectLabel, 0, JitAllocPolicy> ionNurseryObjectLabels_;
+
+ void branchIfInvalidated(Register temp, Label* invalidated);
+
+#ifdef DEBUG
+ void emitDebugResultChecks(LInstruction* ins);
+ void emitGCThingResultChecks(LInstruction* lir, MDefinition* mir);
+ void emitValueResultChecks(LInstruction* lir, MDefinition* mir);
+#endif
+
+ // Script counts created during code generation.
+ IonScriptCounts* scriptCounts_;
+
+ IonPerfSpewer perfSpewer_;
+
+ // Bit mask of JitRealm stubs that are to be read-barriered.
+ uint32_t realmStubsToReadBarrier_;
+
+#ifdef FUZZING_JS_FUZZILLI
+ void emitFuzzilliHashDouble(FloatRegister floatDouble, Register scratch,
+ Register output);
+ void emitFuzzilliHashObject(LInstruction* lir, Register obj, Register output);
+ void emitFuzzilliHashBigInt(Register bigInt, Register output);
+#endif
+
+#define LIR_OP(op) void visit##op(L##op* ins);
+ LIR_OPCODE_LIST(LIR_OP)
+#undef LIR_OP
+};
+
+class OutOfLineResumableWasmTrap : public OutOfLineCodeBase<CodeGenerator> {
+ LInstruction* lir_;
+ size_t framePushed_;
+ wasm::BytecodeOffset bytecodeOffset_;
+ wasm::Trap trap_;
+
+ public:
+ OutOfLineResumableWasmTrap(LInstruction* lir, size_t framePushed,
+ wasm::BytecodeOffset bytecodeOffset,
+ wasm::Trap trap)
+ : lir_(lir),
+ framePushed_(framePushed),
+ bytecodeOffset_(bytecodeOffset),
+ trap_(trap) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineResumableWasmTrap(this);
+ }
+ LInstruction* lir() const { return lir_; }
+ size_t framePushed() const { return framePushed_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+ wasm::Trap trap() const { return trap_; }
+};
+
+class OutOfLineAbortingWasmTrap : public OutOfLineCodeBase<CodeGenerator> {
+ wasm::BytecodeOffset bytecodeOffset_;
+ wasm::Trap trap_;
+
+ public:
+ OutOfLineAbortingWasmTrap(wasm::BytecodeOffset bytecodeOffset,
+ wasm::Trap trap)
+ : bytecodeOffset_(bytecodeOffset), trap_(trap) {}
+
+ void accept(CodeGenerator* codegen) override {
+ codegen->visitOutOfLineAbortingWasmTrap(this);
+ }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+ wasm::Trap trap() const { return trap_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_CodeGenerator_h */
diff --git a/js/src/jit/CompactBuffer.h b/js/src/jit/CompactBuffer.h
new file mode 100644
index 0000000000..2c7fefcb9a
--- /dev/null
+++ b/js/src/jit/CompactBuffer.h
@@ -0,0 +1,254 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Compactbuffer_h
+#define jit_Compactbuffer_h
+
+#include "mozilla/Assertions.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "js/AllocPolicy.h"
+#include "js/Vector.h"
+
+namespace js {
+namespace jit {
+
+class CompactBufferWriter;
+
+// CompactBuffers are byte streams designed for compressable integers. It has
+// helper functions for writing bytes, fixed-size integers, and variable-sized
+// integers. Variable sized integers are encoded in 1-5 bytes, each byte
+// containing 7 bits of the integer and a bit which specifies whether the next
+// byte is also part of the integer.
+//
+// Fixed-width integers are also available, in case the actual value will not
+// be known until later.
+
+class CompactBufferReader {
+ const uint8_t* buffer_;
+ const uint8_t* end_;
+
+ uint32_t readVariableLength() {
+ uint32_t val = 0;
+ uint32_t shift = 0;
+ uint8_t byte;
+ while (true) {
+ MOZ_ASSERT(shift < 32);
+ byte = readByte();
+ val |= (uint32_t(byte) >> 1) << shift;
+ shift += 7;
+ if (!(byte & 1)) {
+ return val;
+ }
+ }
+ }
+
+ uint64_t readVariableLength64() {
+ uint64_t val = 0;
+ uint32_t shift = 0;
+ uint8_t byte;
+ while (true) {
+ MOZ_ASSERT(shift < 64);
+ byte = readByte();
+ val |= (uint64_t(byte) >> 1) << shift;
+ shift += 7;
+ if (!(byte & 1)) {
+ return val;
+ }
+ }
+ }
+
+ public:
+ CompactBufferReader(const uint8_t* start, const uint8_t* end)
+ : buffer_(start), end_(end) {}
+ inline explicit CompactBufferReader(const CompactBufferWriter& writer);
+ uint8_t readByte() {
+ MOZ_ASSERT(buffer_ < end_);
+ return *buffer_++;
+ }
+ uint32_t readFixedUint32_t() {
+ uint32_t b0 = readByte();
+ uint32_t b1 = readByte();
+ uint32_t b2 = readByte();
+ uint32_t b3 = readByte();
+ return b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
+ }
+ uint16_t readFixedUint16_t() {
+ uint32_t b0 = readByte();
+ uint32_t b1 = readByte();
+ return b0 | (b1 << 8);
+ }
+ uint32_t readNativeEndianUint32_t() {
+ // Must be at 4-byte boundary
+ MOZ_ASSERT(uintptr_t(buffer_) % sizeof(uint32_t) == 0);
+ return *reinterpret_cast<const uint32_t*>(buffer_);
+ }
+ uint32_t readUnsigned() { return readVariableLength(); }
+ uint64_t readUnsigned64() { return readVariableLength64(); }
+ int32_t readSigned() {
+ uint8_t b = readByte();
+ bool isNegative = !!(b & (1 << 0));
+ bool more = !!(b & (1 << 1));
+ int32_t result = b >> 2;
+ if (more) {
+ result |= readUnsigned() << 6;
+ }
+ if (isNegative) {
+ return -result;
+ }
+ return result;
+ }
+ // Reads a value written by writeUnsigned15Bit.
+ uint32_t readUnsigned15Bit() {
+ uint8_t byte = readByte();
+ uint32_t val = byte >> 1;
+ if (byte & 1) {
+ val |= uint32_t(readByte()) << 7;
+ }
+ return val;
+ }
+ void* readRawPointer() {
+ uintptr_t ptrWord = 0;
+ for (unsigned i = 0; i < sizeof(uintptr_t); i++) {
+ ptrWord |= static_cast<uintptr_t>(readByte()) << (i * 8);
+ }
+ return reinterpret_cast<void*>(ptrWord);
+ }
+
+ bool more() const {
+ MOZ_ASSERT(buffer_ <= end_);
+ return buffer_ < end_;
+ }
+
+ void seek(const uint8_t* start, uint32_t offset) {
+ buffer_ = start + offset;
+ MOZ_ASSERT(start < end_);
+ MOZ_ASSERT(buffer_ <= end_);
+ }
+
+ const uint8_t* currentPosition() const { return buffer_; }
+};
+
+class CompactBufferWriter {
+ js::Vector<uint8_t, 32, SystemAllocPolicy> buffer_;
+ bool enoughMemory_;
+
+ public:
+ CompactBufferWriter() : enoughMemory_(true) {}
+
+ void setOOM() { enoughMemory_ = false; }
+
+ // Note: writeByte() takes uint32 to catch implicit casts with a runtime
+ // assert.
+ void writeByte(uint32_t byte) {
+ MOZ_ASSERT(byte <= 0xFF);
+ if (!buffer_.append(byte)) {
+ enoughMemory_ = false;
+ }
+ }
+ void writeByteAt(uint32_t pos, uint32_t byte) {
+ MOZ_ASSERT(byte <= 0xFF);
+ if (!oom()) {
+ buffer_[pos] = byte;
+ }
+ }
+ // Writes a variable-length value similar to writeUnsigned, but optimized for
+ // small 15-bit values that fit in one or two variable-length-encoded bytes.
+ // Must be read using readUnsigned15Bit.
+ void writeUnsigned15Bit(uint32_t value) {
+ uint8_t byte1 = ((value & 0x7F) << 1) | (value > 0x7F);
+ writeByte(byte1);
+ value >>= 7;
+ if (value) {
+ MOZ_ASSERT(value <= 0xFF);
+ writeByte(value);
+ }
+ }
+ void writeUnsigned(uint32_t value) {
+ do {
+ uint8_t byte = ((value & 0x7F) << 1) | (value > 0x7F);
+ writeByte(byte);
+ value >>= 7;
+ } while (value);
+ }
+ void writeUnsignedAt(uint32_t pos, uint32_t value, uint32_t original) {
+ MOZ_ASSERT(value <= original);
+ do {
+ uint8_t byte = ((value & 0x7F) << 1) | (original > 0x7F);
+ writeByteAt(pos++, byte);
+ value >>= 7;
+ original >>= 7;
+ } while (original);
+ }
+ void writeUnsigned64(uint64_t value) {
+ do {
+ uint8_t byte = ((value & 0x7F) << 1) | (value > 0x7F);
+ writeByte(byte);
+ value >>= 7;
+ } while (value);
+ }
+ void writeSigned(int32_t v) {
+ bool isNegative = v < 0;
+ uint32_t value = isNegative ? -v : v;
+ uint8_t byte =
+ ((value & 0x3F) << 2) | ((value > 0x3F) << 1) | uint32_t(isNegative);
+ writeByte(byte);
+
+ // Write out the rest of the bytes, if needed.
+ value >>= 6;
+ if (value == 0) {
+ return;
+ }
+ writeUnsigned(value);
+ }
+ void writeFixedUint32_t(uint32_t value) {
+ writeByte(value & 0xFF);
+ writeByte((value >> 8) & 0xFF);
+ writeByte((value >> 16) & 0xFF);
+ writeByte((value >> 24) & 0xFF);
+ }
+ void writeFixedUint16_t(uint16_t value) {
+ writeByte(value & 0xFF);
+ writeByte(value >> 8);
+ }
+ void writeNativeEndianUint32_t(uint32_t value) {
+ // Must be at 4-byte boundary
+ MOZ_ASSERT_IF(!oom(), length() % sizeof(uint32_t) == 0);
+ writeFixedUint32_t(0);
+ if (oom()) {
+ return;
+ }
+ uint8_t* endPtr = buffer() + length();
+ reinterpret_cast<uint32_t*>(endPtr)[-1] = value;
+ }
+ void writeRawPointer(const void* ptr) {
+ uintptr_t ptrWord = reinterpret_cast<uintptr_t>(ptr);
+ for (unsigned i = 0; i < sizeof(uintptr_t); i++) {
+ writeByte((ptrWord >> (i * 8)) & 0xFF);
+ }
+ }
+ size_t length() const { return buffer_.length(); }
+ uint8_t* buffer() {
+ MOZ_ASSERT(!oom());
+ return &buffer_[0];
+ }
+ const uint8_t* buffer() const {
+ MOZ_ASSERT(!oom());
+ return &buffer_[0];
+ }
+ bool oom() const { return !enoughMemory_; }
+ void propagateOOM(bool success) { enoughMemory_ &= success; }
+};
+
+CompactBufferReader::CompactBufferReader(const CompactBufferWriter& writer)
+ : buffer_(writer.buffer()), end_(writer.buffer() + writer.length()) {}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Compactbuffer_h */
diff --git a/js/src/jit/CompileInfo.h b/js/src/jit/CompileInfo.h
new file mode 100644
index 0000000000..311c1d30b7
--- /dev/null
+++ b/js/src/jit/CompileInfo.h
@@ -0,0 +1,382 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CompileInfo_h
+#define jit_CompileInfo_h
+
+#include "mozilla/Assertions.h" // MOZ_ASSERT
+#include "mozilla/Maybe.h" // mozilla::Maybe, mozilla::Some
+
+#include <algorithm> // std::max
+#include <stdint.h> // uint32_t
+
+#include "jit/CompileWrappers.h" // CompileRuntime
+#include "jit/JitFrames.h" // MinJITStackSize
+#include "jit/shared/Assembler-shared.h"
+#include "js/TypeDecls.h" // jsbytecode
+#include "vm/BindingKind.h" // BindingLocation
+#include "vm/JSAtomState.h" // JSAtomState
+#include "vm/JSFunction.h" // JSFunction
+#include "vm/JSScript.h" // JSScript
+#include "vm/Opcodes.h" // JSOp
+#include "vm/Scope.h" // BindingIter
+
+namespace js {
+
+class ModuleObject;
+
+namespace jit {
+
+class InlineScriptTree;
+
+inline unsigned StartArgSlot(JSScript* script) {
+ // Reserved slots:
+ // Slot 0: Environment chain.
+ // Slot 1: Return value.
+
+ // When needed:
+ // Slot 2: Argumentsobject.
+
+ // Note: when updating this, please also update the assert in
+ // SnapshotWriter::startFrame
+ return 2 + (script->needsArgsObj() ? 1 : 0);
+}
+
+inline unsigned CountArgSlots(JSScript* script, JSFunction* fun) {
+ // Slot x + 0: This value.
+ // Slot x + 1: Argument 1.
+ // ...
+ // Slot x + n: Argument n.
+
+ // Note: when updating this, please also update the assert in
+ // SnapshotWriter::startFrame
+ return StartArgSlot(script) + (fun ? fun->nargs() + 1 : 0);
+}
+
+inline unsigned CountArgSlots(JSScript* script, bool hasFun,
+ uint32_t funArgCount) {
+ // Same as the previous function, for use when the JSFunction is not
+ // available.
+ return StartArgSlot(script) + (hasFun ? funArgCount + 1 : 0);
+}
+
+// Contains information about the compilation source for IR being generated.
+class CompileInfo {
+ public:
+ CompileInfo(CompileRuntime* runtime, JSScript* script, JSFunction* fun,
+ jsbytecode* osrPc, bool scriptNeedsArgsObj,
+ InlineScriptTree* inlineScriptTree)
+ : script_(script),
+ fun_(fun),
+ osrPc_(osrPc),
+ scriptNeedsArgsObj_(scriptNeedsArgsObj),
+ hadEagerTruncationBailout_(script->hadEagerTruncationBailout()),
+ hadSpeculativePhiBailout_(script->hadSpeculativePhiBailout()),
+ hadLICMInvalidation_(script->hadLICMInvalidation()),
+ hadReorderingBailout_(script->hadReorderingBailout()),
+ hadBoundsCheckBailout_(script->failedBoundsCheck()),
+ hadUnboxFoldingBailout_(script->hadUnboxFoldingBailout()),
+ mayReadFrameArgsDirectly_(script->mayReadFrameArgsDirectly()),
+ anyFormalIsForwarded_(script->anyFormalIsForwarded()),
+ isDerivedClassConstructor_(script->isDerivedClassConstructor()),
+ inlineScriptTree_(inlineScriptTree) {
+ MOZ_ASSERT_IF(osrPc, JSOp(*osrPc) == JSOp::LoopHead);
+
+ // The function here can flow in from anywhere so look up the canonical
+ // function to ensure that we do not try to embed a nursery pointer in
+ // jit-code. Precisely because it can flow in from anywhere, it's not
+ // guaranteed to be non-lazy. Hence, don't access its script!
+ if (fun_) {
+ fun_ = fun_->baseScript()->function();
+ MOZ_ASSERT(fun_->isTenured());
+ }
+
+ nimplicit_ = StartArgSlot(script) /* env chain and argument obj */
+ + (fun ? 1 : 0); /* this */
+ nargs_ = fun ? fun->nargs() : 0;
+ nlocals_ = script->nfixed();
+
+ // An extra slot is needed for global scopes because InitGLexical (stack
+ // depth 1) is compiled as a SetProp (stack depth 2) on the global lexical
+ // scope.
+ uint32_t extra = script->isGlobalCode() ? 1 : 0;
+ nstack_ = std::max<unsigned>(script->nslots() - script->nfixed(),
+ MinJITStackSize) +
+ extra;
+ nslots_ = nimplicit_ + nargs_ + nlocals_ + nstack_;
+
+ // For derived class constructors, find and cache the frame slot for
+ // the .this binding. This slot is assumed to be always
+ // observable. See isObservableFrameSlot.
+ if (script->isDerivedClassConstructor()) {
+ MOZ_ASSERT(script->functionHasThisBinding());
+ for (BindingIter bi(script); bi; bi++) {
+ if (bi.name() != runtime->names().dotThis) {
+ continue;
+ }
+ BindingLocation loc = bi.location();
+ if (loc.kind() == BindingLocation::Kind::Frame) {
+ thisSlotForDerivedClassConstructor_ =
+ mozilla::Some(localSlot(loc.slot()));
+ break;
+ }
+ }
+ }
+
+ // If the script uses an environment in body, the environment chain
+ // will need to be observable.
+ needsBodyEnvironmentObject_ = script->needsBodyEnvironment();
+ funNeedsSomeEnvironmentObject_ =
+ fun ? fun->needsSomeEnvironmentObject() : false;
+ }
+
+ explicit CompileInfo(unsigned nlocals)
+ : script_(nullptr),
+ fun_(nullptr),
+ osrPc_(nullptr),
+ scriptNeedsArgsObj_(false),
+ hadEagerTruncationBailout_(false),
+ hadSpeculativePhiBailout_(false),
+ hadLICMInvalidation_(false),
+ hadReorderingBailout_(false),
+ hadBoundsCheckBailout_(false),
+ hadUnboxFoldingBailout_(false),
+ mayReadFrameArgsDirectly_(false),
+ anyFormalIsForwarded_(false),
+ inlineScriptTree_(nullptr),
+ needsBodyEnvironmentObject_(false),
+ funNeedsSomeEnvironmentObject_(false) {
+ nimplicit_ = 0;
+ nargs_ = 0;
+ nlocals_ = nlocals;
+ nstack_ = 1; /* For FunctionCompiler::pushPhiInput/popPhiOutput */
+ nslots_ = nlocals_ + nstack_;
+ }
+
+ JSScript* script() const { return script_; }
+ bool compilingWasm() const { return script() == nullptr; }
+ ModuleObject* module() const { return script_->module(); }
+ jsbytecode* osrPc() const { return osrPc_; }
+ InlineScriptTree* inlineScriptTree() const { return inlineScriptTree_; }
+
+ // It's not safe to access the JSFunction off main thread.
+ bool hasFunMaybeLazy() const { return fun_; }
+ ImmGCPtr funMaybeLazy() const { return ImmGCPtr(fun_); }
+
+ const char* filename() const { return script_->filename(); }
+
+ unsigned lineno() const { return script_->lineno(); }
+
+ // Total number of slots: args, locals, and stack.
+ unsigned nslots() const { return nslots_; }
+
+ // Number of slots needed for env chain, return value,
+ // maybe argumentsobject and this value.
+ unsigned nimplicit() const { return nimplicit_; }
+ // Number of arguments (without counting this value).
+ unsigned nargs() const { return nargs_; }
+ // Number of slots needed for all local variables. This includes "fixed
+ // vars" (see above) and also block-scoped locals.
+ unsigned nlocals() const { return nlocals_; }
+ unsigned ninvoke() const { return nslots_ - nstack_; }
+
+ uint32_t environmentChainSlot() const {
+ MOZ_ASSERT(script());
+ return 0;
+ }
+ uint32_t returnValueSlot() const {
+ MOZ_ASSERT(script());
+ return 1;
+ }
+ uint32_t argsObjSlot() const {
+ MOZ_ASSERT(needsArgsObj());
+ return 2;
+ }
+ uint32_t thisSlot() const {
+ MOZ_ASSERT(hasFunMaybeLazy());
+ MOZ_ASSERT(nimplicit_ > 0);
+ return nimplicit_ - 1;
+ }
+ uint32_t firstArgSlot() const { return nimplicit_; }
+ uint32_t argSlotUnchecked(uint32_t i) const {
+ // During initialization, some routines need to get at arg
+ // slots regardless of how regular argument access is done.
+ MOZ_ASSERT(i < nargs_);
+ return nimplicit_ + i;
+ }
+ uint32_t argSlot(uint32_t i) const {
+ // This should only be accessed when compiling functions for
+ // which argument accesses don't need to go through the
+ // argument object.
+ MOZ_ASSERT(!argsObjAliasesFormals());
+ return argSlotUnchecked(i);
+ }
+ uint32_t firstLocalSlot() const { return nimplicit_ + nargs_; }
+ uint32_t localSlot(uint32_t i) const { return firstLocalSlot() + i; }
+ uint32_t firstStackSlot() const { return firstLocalSlot() + nlocals(); }
+ uint32_t stackSlot(uint32_t i) const { return firstStackSlot() + i; }
+
+ uint32_t totalSlots() const {
+ MOZ_ASSERT(script() && hasFunMaybeLazy());
+ return nimplicit() + nargs() + nlocals();
+ }
+
+ bool hasMappedArgsObj() const { return script()->hasMappedArgsObj(); }
+ bool needsArgsObj() const { return scriptNeedsArgsObj_; }
+ bool argsObjAliasesFormals() const {
+ return scriptNeedsArgsObj_ && script()->hasMappedArgsObj();
+ }
+
+ bool needsBodyEnvironmentObject() const {
+ return needsBodyEnvironmentObject_;
+ }
+
+ enum class SlotObservableKind {
+ // This slot must be preserved because it's observable outside SSA uses.
+ // It can't be recovered before or during bailout.
+ ObservableNotRecoverable,
+
+ // This slot must be preserved because it's observable, but it can be
+ // recovered.
+ ObservableRecoverable,
+
+ // This slot is not observable outside SSA uses.
+ NotObservable,
+ };
+
+ inline SlotObservableKind getSlotObservableKind(uint32_t slot) const {
+ // Locals and expression stack slots.
+ if (slot >= firstLocalSlot()) {
+ // The |this| slot for a derived class constructor is a local slot.
+ // It should never be optimized out, as a Debugger might need to perform
+ // TDZ checks on it via, e.g., an exceptionUnwind handler. The TDZ check
+ // is required for correctness if the handler decides to continue
+ // execution.
+ if (thisSlotForDerivedClassConstructor_ &&
+ *thisSlotForDerivedClassConstructor_ == slot) {
+ return SlotObservableKind::ObservableNotRecoverable;
+ }
+ return SlotObservableKind::NotObservable;
+ }
+
+ // Formal argument slots.
+ if (slot >= firstArgSlot()) {
+ MOZ_ASSERT(hasFunMaybeLazy());
+ MOZ_ASSERT(slot - firstArgSlot() < nargs());
+
+ // Preserve formal arguments if they might be read when creating a rest or
+ // arguments object. In non-strict scripts, Function.arguments can create
+ // an arguments object dynamically so we always preserve the arguments.
+ if (mayReadFrameArgsDirectly_ || !script()->strict()) {
+ return SlotObservableKind::ObservableRecoverable;
+ }
+ return SlotObservableKind::NotObservable;
+ }
+
+ // |this| slot is observable but it can be recovered.
+ if (hasFunMaybeLazy() && slot == thisSlot()) {
+ return SlotObservableKind::ObservableRecoverable;
+ }
+
+ // Environment chain slot.
+ if (slot == environmentChainSlot()) {
+ // If environments can be added in the body (after the prologue) we need
+ // to preserve the environment chain slot. It can't be recovered.
+ if (needsBodyEnvironmentObject()) {
+ return SlotObservableKind::ObservableNotRecoverable;
+ }
+ // If the function may need an arguments object, also preserve the
+ // environment chain because it may be needed to reconstruct the arguments
+ // object during bailout.
+ if (funNeedsSomeEnvironmentObject_ || needsArgsObj()) {
+ return SlotObservableKind::ObservableRecoverable;
+ }
+ return SlotObservableKind::NotObservable;
+ }
+
+ // The arguments object is observable. If it does not escape, it can
+ // be recovered.
+ if (needsArgsObj() && slot == argsObjSlot()) {
+ MOZ_ASSERT(hasFunMaybeLazy());
+ return SlotObservableKind::ObservableRecoverable;
+ }
+
+ MOZ_ASSERT(slot == returnValueSlot());
+ return SlotObservableKind::NotObservable;
+ }
+
+ // Returns true if a slot can be observed out-side the current frame while
+ // the frame is active on the stack. This implies that these definitions
+ // would have to be executed and that they cannot be removed even if they
+ // are unused.
+ inline bool isObservableSlot(uint32_t slot) const {
+ SlotObservableKind kind = getSlotObservableKind(slot);
+ return (kind == SlotObservableKind::ObservableNotRecoverable ||
+ kind == SlotObservableKind::ObservableRecoverable);
+ }
+
+ // Returns true if a slot can be recovered before or during a bailout. A
+ // definition which can be observed and recovered, implies that this
+ // definition can be optimized away as long as we can compute its values.
+ bool isRecoverableOperand(uint32_t slot) const {
+ SlotObservableKind kind = getSlotObservableKind(slot);
+ return (kind == SlotObservableKind::ObservableRecoverable ||
+ kind == SlotObservableKind::NotObservable);
+ }
+
+ // Check previous bailout states to prevent doing the same bailout in the
+ // next compilation.
+ bool hadEagerTruncationBailout() const { return hadEagerTruncationBailout_; }
+ bool hadSpeculativePhiBailout() const { return hadSpeculativePhiBailout_; }
+ bool hadLICMInvalidation() const { return hadLICMInvalidation_; }
+ bool hadReorderingBailout() const { return hadReorderingBailout_; }
+ bool hadBoundsCheckBailout() const { return hadBoundsCheckBailout_; }
+ bool hadUnboxFoldingBailout() const { return hadUnboxFoldingBailout_; }
+
+ bool mayReadFrameArgsDirectly() const { return mayReadFrameArgsDirectly_; }
+ bool anyFormalIsForwarded() const { return anyFormalIsForwarded_; }
+
+ bool isDerivedClassConstructor() const { return isDerivedClassConstructor_; }
+
+ private:
+ unsigned nimplicit_;
+ unsigned nargs_;
+ unsigned nlocals_;
+ unsigned nstack_;
+ unsigned nslots_;
+ mozilla::Maybe<unsigned> thisSlotForDerivedClassConstructor_;
+ JSScript* script_;
+ JSFunction* fun_;
+ jsbytecode* osrPc_;
+
+ bool scriptNeedsArgsObj_;
+
+ // Record the state of previous bailouts in order to prevent compiling the
+ // same function identically the next time.
+ bool hadEagerTruncationBailout_;
+ bool hadSpeculativePhiBailout_;
+ bool hadLICMInvalidation_;
+ bool hadReorderingBailout_;
+ bool hadBoundsCheckBailout_;
+ bool hadUnboxFoldingBailout_;
+
+ bool mayReadFrameArgsDirectly_;
+ bool anyFormalIsForwarded_;
+
+ bool isDerivedClassConstructor_;
+
+ InlineScriptTree* inlineScriptTree_;
+
+ // Whether a script needs environments within its body. This informs us
+ // that the environment chain is not easy to reconstruct.
+ bool needsBodyEnvironmentObject_;
+ bool funNeedsSomeEnvironmentObject_;
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_CompileInfo_h */
diff --git a/js/src/jit/CompileWrappers.cpp b/js/src/jit/CompileWrappers.cpp
new file mode 100644
index 0000000000..d7cc54db30
--- /dev/null
+++ b/js/src/jit/CompileWrappers.cpp
@@ -0,0 +1,235 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/CompileWrappers.h"
+
+#include "gc/Heap.h"
+#include "gc/Zone.h"
+#include "jit/Ion.h"
+#include "jit/JitRuntime.h"
+#include "vm/Realm.h"
+
+using namespace js;
+using namespace js::jit;
+
+JSRuntime* CompileRuntime::runtime() {
+ return reinterpret_cast<JSRuntime*>(this);
+}
+
+/* static */
+CompileRuntime* CompileRuntime::get(JSRuntime* rt) {
+ return reinterpret_cast<CompileRuntime*>(rt);
+}
+
+#ifdef JS_GC_ZEAL
+const uint32_t* CompileRuntime::addressOfGCZealModeBits() {
+ return runtime()->gc.addressOfZealModeBits();
+}
+#endif
+
+const JitRuntime* CompileRuntime::jitRuntime() {
+ return runtime()->jitRuntime();
+}
+
+GeckoProfilerRuntime& CompileRuntime::geckoProfiler() {
+ return runtime()->geckoProfiler();
+}
+
+bool CompileRuntime::hadOutOfMemory() { return runtime()->hadOutOfMemory; }
+
+bool CompileRuntime::profilingScripts() { return runtime()->profilingScripts; }
+
+const JSAtomState& CompileRuntime::names() { return *runtime()->commonNames; }
+
+const PropertyName* CompileRuntime::emptyString() {
+ return runtime()->emptyString;
+}
+
+const StaticStrings& CompileRuntime::staticStrings() {
+ return *runtime()->staticStrings;
+}
+
+const WellKnownSymbols& CompileRuntime::wellKnownSymbols() {
+ return *runtime()->wellKnownSymbols;
+}
+
+const JSClass* CompileRuntime::maybeWindowProxyClass() {
+ return runtime()->maybeWindowProxyClass();
+}
+
+const void* CompileRuntime::mainContextPtr() {
+ return runtime()->mainContextFromAnyThread();
+}
+
+const void* CompileRuntime::addressOfJitStackLimit() {
+ return runtime()->mainContextFromAnyThread()->addressOfJitStackLimit();
+}
+
+const void* CompileRuntime::addressOfInterruptBits() {
+ return runtime()->mainContextFromAnyThread()->addressOfInterruptBits();
+}
+
+const void* CompileRuntime::addressOfZone() {
+ return runtime()->mainContextFromAnyThread()->addressOfZone();
+}
+
+const void* CompileRuntime::addressOfMegamorphicCache() {
+ return &runtime()->caches().megamorphicCache;
+}
+
+const void* CompileRuntime::addressOfStringToAtomCache() {
+ return &runtime()->caches().stringToAtomCache;
+}
+
+const DOMCallbacks* CompileRuntime::DOMcallbacks() {
+ return runtime()->DOMcallbacks;
+}
+
+bool CompileRuntime::runtimeMatches(JSRuntime* rt) { return rt == runtime(); }
+
+Zone* CompileZone::zone() { return reinterpret_cast<Zone*>(this); }
+
+/* static */
+CompileZone* CompileZone::get(Zone* zone) {
+ return reinterpret_cast<CompileZone*>(zone);
+}
+
+CompileRuntime* CompileZone::runtime() {
+ return CompileRuntime::get(zone()->runtimeFromAnyThread());
+}
+
+bool CompileZone::isAtomsZone() { return zone()->isAtomsZone(); }
+
+#ifdef DEBUG
+const void* CompileRuntime::addressOfIonBailAfterCounter() {
+ return runtime()->jitRuntime()->addressOfIonBailAfterCounter();
+}
+#endif
+
+const uint32_t* CompileZone::addressOfNeedsIncrementalBarrier() {
+ // Cast away relaxed atomic wrapper for JIT access to barrier state.
+ const mozilla::Atomic<uint32_t, mozilla::Relaxed>* ptr =
+ zone()->addressOfNeedsIncrementalBarrier();
+ return reinterpret_cast<const uint32_t*>(ptr);
+}
+
+uint32_t* CompileZone::addressOfTenuredAllocCount() {
+ return zone()->addressOfTenuredAllocCount();
+}
+
+gc::FreeSpan** CompileZone::addressOfFreeList(gc::AllocKind allocKind) {
+ return zone()->arenas.addressOfFreeList(allocKind);
+}
+
+void* CompileZone::addressOfNurseryPosition() {
+ return zone()->runtimeFromAnyThread()->gc.addressOfNurseryPosition();
+}
+
+void* CompileZone::addressOfStringNurseryPosition() {
+ // Objects and strings share a nursery, for now at least.
+ return zone()->runtimeFromAnyThread()->gc.addressOfNurseryPosition();
+}
+
+void* CompileZone::addressOfBigIntNurseryPosition() {
+ // Objects and BigInts share a nursery, for now at least.
+ return zone()->runtimeFromAnyThread()->gc.addressOfNurseryPosition();
+}
+
+const void* CompileZone::addressOfNurseryCurrentEnd() {
+ return zone()->runtimeFromAnyThread()->gc.addressOfNurseryCurrentEnd();
+}
+
+const void* CompileZone::addressOfStringNurseryCurrentEnd() {
+ // Although objects and strings share a nursery (and this may change)
+ // there is still a separate string end address. The only time it
+ // is different from the regular end address, is when nursery strings are
+ // disabled (it will be NULL).
+ //
+ // This function returns _a pointer to_ that end address.
+ return zone()->runtimeFromAnyThread()->gc.addressOfStringNurseryCurrentEnd();
+}
+
+const void* CompileZone::addressOfBigIntNurseryCurrentEnd() {
+ // Similar to Strings, BigInts also share the nursery with other nursery
+ // allocatable things.
+ return zone()->runtimeFromAnyThread()->gc.addressOfBigIntNurseryCurrentEnd();
+}
+
+uint32_t* CompileZone::addressOfNurseryAllocCount() {
+ return zone()->runtimeFromAnyThread()->gc.addressOfNurseryAllocCount();
+}
+
+void* CompileZone::addressOfNurseryAllocatedSites() {
+ JSRuntime* rt = zone()->runtimeFromAnyThread();
+ return rt->gc.nursery().addressOfNurseryAllocatedSites();
+}
+
+bool CompileZone::canNurseryAllocateStrings() {
+ return zone()->runtimeFromAnyThread()->gc.nursery().canAllocateStrings() &&
+ zone()->allocNurseryStrings;
+}
+
+bool CompileZone::canNurseryAllocateBigInts() {
+ return zone()->runtimeFromAnyThread()->gc.nursery().canAllocateBigInts() &&
+ zone()->allocNurseryBigInts;
+}
+
+uintptr_t CompileZone::nurseryCellHeader(JS::TraceKind traceKind,
+ gc::CatchAllAllocSite siteKind) {
+ gc::AllocSite* site = siteKind == gc::CatchAllAllocSite::Optimized
+ ? zone()->optimizedAllocSite()
+ : zone()->unknownAllocSite();
+ return gc::NurseryCellHeader::MakeValue(site, traceKind);
+}
+
+JS::Realm* CompileRealm::realm() { return reinterpret_cast<JS::Realm*>(this); }
+
+/* static */
+CompileRealm* CompileRealm::get(JS::Realm* realm) {
+ return reinterpret_cast<CompileRealm*>(realm);
+}
+
+CompileZone* CompileRealm::zone() { return CompileZone::get(realm()->zone()); }
+
+CompileRuntime* CompileRealm::runtime() {
+ return CompileRuntime::get(realm()->runtimeFromAnyThread());
+}
+
+const mozilla::non_crypto::XorShift128PlusRNG*
+CompileRealm::addressOfRandomNumberGenerator() {
+ return realm()->addressOfRandomNumberGenerator();
+}
+
+const JitRealm* CompileRealm::jitRealm() { return realm()->jitRealm(); }
+
+const GlobalObject* CompileRealm::maybeGlobal() {
+ // This uses unsafeUnbarrieredMaybeGlobal() so as not to trigger the read
+ // barrier on the global from off thread. This is safe because we
+ // abort Ion compilation when we GC.
+ return realm()->unsafeUnbarrieredMaybeGlobal();
+}
+
+const uint32_t* CompileRealm::addressOfGlobalWriteBarriered() {
+ return &realm()->globalWriteBarriered;
+}
+
+bool CompileRealm::hasAllocationMetadataBuilder() {
+ return realm()->hasAllocationMetadataBuilder();
+}
+
+JitCompileOptions::JitCompileOptions()
+ : profilerSlowAssertionsEnabled_(false),
+ offThreadCompilationAvailable_(false) {}
+
+JitCompileOptions::JitCompileOptions(JSContext* cx) {
+ profilerSlowAssertionsEnabled_ =
+ cx->runtime()->geckoProfiler().enabled() &&
+ cx->runtime()->geckoProfiler().slowAssertionsEnabled();
+ offThreadCompilationAvailable_ = OffThreadCompilationAvailable(cx);
+#ifdef DEBUG
+ ionBailAfterEnabled_ = cx->runtime()->jitRuntime()->ionBailAfterEnabled();
+#endif
+}
diff --git a/js/src/jit/CompileWrappers.h b/js/src/jit/CompileWrappers.h
new file mode 100644
index 0000000000..9af994b690
--- /dev/null
+++ b/js/src/jit/CompileWrappers.h
@@ -0,0 +1,178 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_CompileWrappers_h
+#define jit_CompileWrappers_h
+
+#include <stdint.h>
+
+#include "gc/Pretenuring.h"
+#include "js/TypeDecls.h"
+
+struct JSAtomState;
+
+namespace mozilla::non_crypto {
+class XorShift128PlusRNG;
+}
+
+namespace JS {
+enum class TraceKind;
+}
+
+namespace js {
+
+class GeckoProfilerRuntime;
+class GlobalObject;
+struct JSDOMCallbacks;
+class PropertyName;
+class StaticStrings;
+struct WellKnownSymbols;
+
+using DOMCallbacks = struct JSDOMCallbacks;
+
+namespace gc {
+
+enum class AllocKind : uint8_t;
+
+class FreeSpan;
+
+} // namespace gc
+
+namespace jit {
+
+class JitRuntime;
+
+// During Ion compilation we need access to various bits of the current
+// compartment, runtime and so forth. However, since compilation can run off
+// thread while the main thread is mutating the VM, this access needs
+// to be restricted. The classes below give the compiler an interface to access
+// all necessary information in a threadsafe fashion.
+
+class CompileRuntime {
+ JSRuntime* runtime();
+
+ public:
+ static CompileRuntime* get(JSRuntime* rt);
+
+#ifdef JS_GC_ZEAL
+ const uint32_t* addressOfGCZealModeBits();
+#endif
+
+ const JitRuntime* jitRuntime();
+
+ // Compilation does not occur off thread when the Gecko Profiler is enabled.
+ GeckoProfilerRuntime& geckoProfiler();
+
+ bool hadOutOfMemory();
+ bool profilingScripts();
+
+ const JSAtomState& names();
+ const PropertyName* emptyString();
+ const StaticStrings& staticStrings();
+ const WellKnownSymbols& wellKnownSymbols();
+ const JSClass* maybeWindowProxyClass();
+
+ const void* mainContextPtr();
+ const void* addressOfJitStackLimit();
+ const void* addressOfInterruptBits();
+ const void* addressOfZone();
+ const void* addressOfMegamorphicCache();
+ const void* addressOfStringToAtomCache();
+
+#ifdef DEBUG
+ const void* addressOfIonBailAfterCounter();
+#endif
+
+ // DOM callbacks must be threadsafe (and will hopefully be removed soon).
+ const DOMCallbacks* DOMcallbacks();
+
+ bool runtimeMatches(JSRuntime* rt);
+};
+
+class CompileZone {
+ friend class MacroAssembler;
+ JS::Zone* zone();
+
+ public:
+ static CompileZone* get(JS::Zone* zone);
+
+ CompileRuntime* runtime();
+ bool isAtomsZone();
+
+ const uint32_t* addressOfNeedsIncrementalBarrier();
+ uint32_t* addressOfTenuredAllocCount();
+ gc::FreeSpan** addressOfFreeList(gc::AllocKind allocKind);
+ void* addressOfNurseryPosition();
+ void* addressOfStringNurseryPosition();
+ void* addressOfBigIntNurseryPosition();
+ const void* addressOfNurseryCurrentEnd();
+ const void* addressOfStringNurseryCurrentEnd();
+ const void* addressOfBigIntNurseryCurrentEnd();
+
+ uint32_t* addressOfNurseryAllocCount();
+
+ void* addressOfNurseryAllocatedSites();
+
+ bool canNurseryAllocateStrings();
+ bool canNurseryAllocateBigInts();
+
+ uintptr_t nurseryCellHeader(JS::TraceKind traceKind,
+ gc::CatchAllAllocSite siteKind);
+};
+
+class JitRealm;
+
+class CompileRealm {
+ JS::Realm* realm();
+
+ public:
+ static CompileRealm* get(JS::Realm* realm);
+
+ CompileZone* zone();
+ CompileRuntime* runtime();
+
+ const void* realmPtr() { return realm(); }
+
+ const mozilla::non_crypto::XorShift128PlusRNG*
+ addressOfRandomNumberGenerator();
+
+ const JitRealm* jitRealm();
+
+ const GlobalObject* maybeGlobal();
+ const uint32_t* addressOfGlobalWriteBarriered();
+
+ bool hasAllocationMetadataBuilder();
+};
+
+class JitCompileOptions {
+ public:
+ JitCompileOptions();
+ explicit JitCompileOptions(JSContext* cx);
+
+ bool profilerSlowAssertionsEnabled() const {
+ return profilerSlowAssertionsEnabled_;
+ }
+
+ bool offThreadCompilationAvailable() const {
+ return offThreadCompilationAvailable_;
+ }
+
+#ifdef DEBUG
+ bool ionBailAfterEnabled() const { return ionBailAfterEnabled_; }
+#endif
+
+ private:
+ bool profilerSlowAssertionsEnabled_;
+ bool offThreadCompilationAvailable_;
+#ifdef DEBUG
+ bool ionBailAfterEnabled_ = false;
+#endif
+};
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_CompileWrappers_h
diff --git a/js/src/jit/Disassemble.cpp b/js/src/jit/Disassemble.cpp
new file mode 100644
index 0000000000..6026df27d6
--- /dev/null
+++ b/js/src/jit/Disassemble.cpp
@@ -0,0 +1,109 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Disassemble.h"
+
+#include <stddef.h> // size_t
+#include <stdint.h> // uint8_t, int32_t
+
+#include "js/Printf.h" // JS_smprintf
+#include "js/Utility.h" // JS::UniqueChars
+
+#if defined(JS_JITSPEW)
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+# include "zydis/ZydisAPI.h" // zydisDisassemble
+# elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm/disasm/Disasm-arm.h" // js::jit::disasm::*
+# include "jit/arm64/vixl/Decoder-vixl.h" // vixl::Decoder
+# include "jit/arm64/vixl/Disasm-vixl.h" // vixl::Disassembler
+# include "jit/arm64/vixl/Instructions-vixl.h" // vixl::Instruction
+# elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/disasm/Disasm-arm.h" // js::jit::disasm::*
+# endif
+#endif
+
+namespace js {
+namespace jit {
+
+#if defined(JS_JITSPEW) && (defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64))
+
+bool HasDisassembler() { return true; }
+
+void Disassemble(uint8_t* code, size_t length, InstrCallback callback) {
+ zydisDisassemble(code, length, callback);
+}
+
+#elif defined(JS_JITSPEW) && defined(JS_CODEGEN_ARM64)
+
+class ARM64Disassembler : public vixl::Disassembler {
+ public:
+ explicit ARM64Disassembler(InstrCallback callback) : callback_(callback) {}
+
+ protected:
+ void ProcessOutput(const vixl::Instruction* instr) override {
+ JS::UniqueChars formatted = JS_smprintf(
+ "0x%p %08x %s", instr, instr->InstructionBits(), GetOutput());
+ callback_(formatted.get());
+ }
+
+ private:
+ InstrCallback callback_;
+};
+
+bool HasDisassembler() { return true; }
+
+void Disassemble(uint8_t* code, size_t length, InstrCallback callback) {
+ ARM64Disassembler dis(callback);
+ vixl::Decoder decoder;
+ decoder.AppendVisitor(&dis);
+
+ uint8_t* instr = code;
+ uint8_t* end = code + length;
+
+ while (instr < end) {
+ decoder.Decode(reinterpret_cast<vixl::Instruction*>(instr));
+
+ instr += sizeof(vixl::Instr);
+ }
+}
+
+#elif defined(JS_JITSPEW) && defined(JS_CODEGEN_ARM)
+
+bool HasDisassembler() { return true; }
+
+void Disassemble(uint8_t* code, size_t length, InstrCallback callback) {
+ disasm::NameConverter converter;
+ disasm::Disassembler d(converter);
+
+ uint8_t* instr = code;
+ uint8_t* end = code + length;
+
+ while (instr < end) {
+ disasm::EmbeddedVector<char, disasm::ReasonableBufferSize> buffer;
+ buffer[0] = '\0';
+ uint8_t* next_instr = instr + d.InstructionDecode(buffer, instr);
+
+ JS::UniqueChars formatted =
+ JS_smprintf("0x%p %08x %s", instr, *reinterpret_cast<int32_t*>(instr),
+ buffer.start());
+ callback(formatted.get());
+
+ instr = next_instr;
+ }
+}
+
+#else
+
+bool HasDisassembler() { return false; }
+
+void Disassemble(uint8_t* code, size_t length, InstrCallback callback) {
+ callback("*** No disassembly available ***\n");
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/Disassemble.h b/js/src/jit/Disassemble.h
new file mode 100644
index 0000000000..ad132a8554
--- /dev/null
+++ b/js/src/jit/Disassemble.h
@@ -0,0 +1,24 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Disassemble_h
+#define jit_Disassemble_h
+
+#include <stddef.h>
+#include <stdint.h>
+
+namespace js {
+namespace jit {
+
+using InstrCallback = void (*)(const char* text);
+
+extern bool HasDisassembler();
+extern void Disassemble(uint8_t* code, size_t length, InstrCallback callback);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Disassemble_h */
diff --git a/js/src/jit/EdgeCaseAnalysis.cpp b/js/src/jit/EdgeCaseAnalysis.cpp
new file mode 100644
index 0000000000..a92ca93ddd
--- /dev/null
+++ b/js/src/jit/EdgeCaseAnalysis.cpp
@@ -0,0 +1,50 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/EdgeCaseAnalysis.h"
+
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+using namespace js;
+using namespace js::jit;
+
+EdgeCaseAnalysis::EdgeCaseAnalysis(MIRGenerator* mir, MIRGraph& graph)
+ : mir(mir), graph(graph) {}
+
+bool EdgeCaseAnalysis::analyzeLate() {
+ // Renumber definitions for NeedNegativeZeroCheck under
+ // analyzeEdgeCasesBackward.
+ uint32_t nextId = 0;
+
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); block++) {
+ for (MDefinitionIterator iter(*block); iter; iter++) {
+ if (mir->shouldCancel("Analyze Late (first loop)")) {
+ return false;
+ }
+
+ iter->setId(nextId++);
+ iter->analyzeEdgeCasesForward();
+ }
+ block->lastIns()->setId(nextId++);
+ }
+
+ for (PostorderIterator block(graph.poBegin()); block != graph.poEnd();
+ block++) {
+ for (MInstructionReverseIterator riter(block->rbegin());
+ riter != block->rend(); riter++) {
+ if (mir->shouldCancel("Analyze Late (second loop)")) {
+ return false;
+ }
+
+ riter->analyzeEdgeCasesBackward();
+ }
+ }
+
+ return true;
+}
diff --git a/js/src/jit/EdgeCaseAnalysis.h b/js/src/jit/EdgeCaseAnalysis.h
new file mode 100644
index 0000000000..d3e2a58d31
--- /dev/null
+++ b/js/src/jit/EdgeCaseAnalysis.h
@@ -0,0 +1,28 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_EdgeCaseAnalysis_h
+#define jit_EdgeCaseAnalysis_h
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+class MIRGraph;
+
+class EdgeCaseAnalysis {
+ MIRGenerator* mir;
+ MIRGraph& graph;
+
+ public:
+ EdgeCaseAnalysis(MIRGenerator* mir, MIRGraph& graph);
+ [[nodiscard]] bool analyzeLate();
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_EdgeCaseAnalysis_h */
diff --git a/js/src/jit/EffectiveAddressAnalysis.cpp b/js/src/jit/EffectiveAddressAnalysis.cpp
new file mode 100644
index 0000000000..7d78549387
--- /dev/null
+++ b/js/src/jit/EffectiveAddressAnalysis.cpp
@@ -0,0 +1,256 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/EffectiveAddressAnalysis.h"
+
+#include "jit/IonAnalysis.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "util/CheckedArithmetic.h"
+
+using namespace js;
+using namespace jit;
+
+static void AnalyzeLsh(TempAllocator& alloc, MLsh* lsh) {
+ if (lsh->type() != MIRType::Int32) {
+ return;
+ }
+
+ if (lsh->isRecoveredOnBailout()) {
+ return;
+ }
+
+ MDefinition* index = lsh->lhs();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ MConstant* shiftValue = lsh->rhs()->maybeConstantValue();
+ if (!shiftValue) {
+ return;
+ }
+
+ if (shiftValue->type() != MIRType::Int32 ||
+ !IsShiftInScaleRange(shiftValue->toInt32())) {
+ return;
+ }
+
+ Scale scale = ShiftToScale(shiftValue->toInt32());
+
+ int32_t displacement = 0;
+ MInstruction* last = lsh;
+ MDefinition* base = nullptr;
+ while (true) {
+ if (!last->hasOneUse()) {
+ break;
+ }
+
+ MUseIterator use = last->usesBegin();
+ if (!use->consumer()->isDefinition() ||
+ !use->consumer()->toDefinition()->isAdd()) {
+ break;
+ }
+
+ MAdd* add = use->consumer()->toDefinition()->toAdd();
+ if (add->type() != MIRType::Int32 || !add->isTruncated()) {
+ break;
+ }
+
+ MDefinition* other = add->getOperand(1 - add->indexOf(*use));
+
+ if (MConstant* otherConst = other->maybeConstantValue()) {
+ displacement += otherConst->toInt32();
+ } else {
+ if (base) {
+ break;
+ }
+ base = other;
+ }
+
+ last = add;
+ if (last->isRecoveredOnBailout()) {
+ return;
+ }
+ }
+
+ if (!base) {
+ uint32_t elemSize = 1 << ScaleToShift(scale);
+ if (displacement % elemSize != 0) {
+ return;
+ }
+
+ if (!last->hasOneUse()) {
+ return;
+ }
+
+ MUseIterator use = last->usesBegin();
+ if (!use->consumer()->isDefinition() ||
+ !use->consumer()->toDefinition()->isBitAnd()) {
+ return;
+ }
+
+ MBitAnd* bitAnd = use->consumer()->toDefinition()->toBitAnd();
+ if (bitAnd->isRecoveredOnBailout()) {
+ return;
+ }
+
+ MDefinition* other = bitAnd->getOperand(1 - bitAnd->indexOf(*use));
+ MConstant* otherConst = other->maybeConstantValue();
+ if (!otherConst || otherConst->type() != MIRType::Int32) {
+ return;
+ }
+
+ uint32_t bitsClearedByShift = elemSize - 1;
+ uint32_t bitsClearedByMask = ~uint32_t(otherConst->toInt32());
+ if ((bitsClearedByShift & bitsClearedByMask) != bitsClearedByMask) {
+ return;
+ }
+
+ bitAnd->replaceAllUsesWith(last);
+ return;
+ }
+
+ if (base->isRecoveredOnBailout()) {
+ return;
+ }
+
+ MEffectiveAddress* eaddr =
+ MEffectiveAddress::New(alloc, base, index, scale, displacement);
+ last->replaceAllUsesWith(eaddr);
+ last->block()->insertAfter(last, eaddr);
+}
+
+// Transform:
+//
+// [AddI]
+// addl $9, %esi
+// [LoadUnboxedScalar]
+// movsd 0x0(%rbx,%rsi,8), %xmm4
+//
+// into:
+//
+// [LoadUnboxedScalar]
+// movsd 0x48(%rbx,%rsi,8), %xmm4
+//
+// This is possible when the AddI is only used by the LoadUnboxedScalar opcode.
+static void AnalyzeLoadUnboxedScalar(MLoadUnboxedScalar* load) {
+ if (load->isRecoveredOnBailout()) {
+ return;
+ }
+
+ if (!load->getOperand(1)->isAdd()) {
+ return;
+ }
+
+ JitSpew(JitSpew_EAA, "analyze: %s%u", load->opName(), load->id());
+
+ MAdd* add = load->getOperand(1)->toAdd();
+
+ if (add->type() != MIRType::Int32 || !add->hasUses() ||
+ add->truncateKind() != TruncateKind::Truncate) {
+ return;
+ }
+
+ MDefinition* lhs = add->lhs();
+ MDefinition* rhs = add->rhs();
+ MDefinition* constant = nullptr;
+ MDefinition* node = nullptr;
+
+ if (lhs->isConstant()) {
+ constant = lhs;
+ node = rhs;
+ } else if (rhs->isConstant()) {
+ constant = rhs;
+ node = lhs;
+ } else
+ return;
+
+ MOZ_ASSERT(constant->type() == MIRType::Int32);
+
+ size_t storageSize = Scalar::byteSize(load->storageType());
+ int32_t c1 = load->offsetAdjustment();
+ int32_t c2 = 0;
+ if (!SafeMul(constant->maybeConstantValue()->toInt32(), storageSize, &c2)) {
+ return;
+ }
+
+ int32_t offset = 0;
+ if (!SafeAdd(c1, c2, &offset)) {
+ return;
+ }
+
+ JitSpew(JitSpew_EAA, "set offset: %d + %d = %d on: %s%u", c1, c2, offset,
+ load->opName(), load->id());
+ load->setOffsetAdjustment(offset);
+ load->replaceOperand(1, node);
+
+ if (!add->hasLiveDefUses() && DeadIfUnused(add) &&
+ add->canRecoverOnBailout()) {
+ JitSpew(JitSpew_EAA, "mark as recovered on bailout: %s%u", add->opName(),
+ add->id());
+ add->setRecoveredOnBailoutUnchecked();
+ }
+}
+
+template <typename AsmJSMemoryAccess>
+void EffectiveAddressAnalysis::analyzeAsmJSHeapAccess(AsmJSMemoryAccess* ins) {
+ MDefinition* base = ins->base();
+
+ if (base->isConstant()) {
+ // If the index is within the minimum heap length, we can optimize away the
+ // bounds check. Asm.js accesses always have an int32 base, the memory is
+ // always a memory32.
+ int32_t imm = base->toConstant()->toInt32();
+ if (imm >= 0) {
+ int32_t end = (uint32_t)imm + ins->byteSize();
+ if (end >= imm && (uint32_t)end <= mir_->minWasmHeapLength()) {
+ ins->removeBoundsCheck();
+ }
+ }
+ }
+}
+
+// This analysis converts patterns of the form:
+// truncate(x + (y << {0,1,2,3}))
+// truncate(x + (y << {0,1,2,3}) + imm32)
+// into a single lea instruction, and patterns of the form:
+// asmload(x + imm32)
+// asmload(x << {0,1,2,3})
+// asmload((x << {0,1,2,3}) + imm32)
+// asmload((x << {0,1,2,3}) & mask) (where mask is redundant
+// with shift)
+// asmload(((x << {0,1,2,3}) + imm32) & mask) (where mask is redundant
+// with shift + imm32)
+// into a single asmload instruction (and for asmstore too).
+//
+// Additionally, we should consider the general forms:
+// truncate(x + y + imm32)
+// truncate((y << {0,1,2,3}) + imm32)
+bool EffectiveAddressAnalysis::analyze() {
+ JitSpew(JitSpew_EAA, "Begin");
+ for (ReversePostorderIterator block(graph_.rpoBegin());
+ block != graph_.rpoEnd(); block++) {
+ for (MInstructionIterator i = block->begin(); i != block->end(); i++) {
+ if (!graph_.alloc().ensureBallast()) {
+ return false;
+ }
+
+ // Note that we don't check for MWasmCompareExchangeHeap
+ // or MWasmAtomicBinopHeap, because the backend and the OOB
+ // mechanism don't support non-zero offsets for them yet
+ // (TODO bug 1254935).
+ if (i->isLsh()) {
+ AnalyzeLsh(graph_.alloc(), i->toLsh());
+ } else if (i->isLoadUnboxedScalar()) {
+ AnalyzeLoadUnboxedScalar(i->toLoadUnboxedScalar());
+ } else if (i->isAsmJSLoadHeap()) {
+ analyzeAsmJSHeapAccess(i->toAsmJSLoadHeap());
+ } else if (i->isAsmJSStoreHeap()) {
+ analyzeAsmJSHeapAccess(i->toAsmJSStoreHeap());
+ }
+ }
+ }
+ return true;
+}
diff --git a/js/src/jit/EffectiveAddressAnalysis.h b/js/src/jit/EffectiveAddressAnalysis.h
new file mode 100644
index 0000000000..f66ba758e5
--- /dev/null
+++ b/js/src/jit/EffectiveAddressAnalysis.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_EffectiveAddressAnalysis_h
+#define jit_EffectiveAddressAnalysis_h
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+class MIRGraph;
+
+class EffectiveAddressAnalysis {
+ MIRGenerator* mir_;
+ MIRGraph& graph_;
+
+ template <typename AsmJSMemoryAccess>
+ void analyzeAsmJSHeapAccess(AsmJSMemoryAccess* ins);
+
+ public:
+ EffectiveAddressAnalysis(MIRGenerator* mir, MIRGraph& graph)
+ : mir_(mir), graph_(graph) {}
+
+ [[nodiscard]] bool analyze();
+};
+
+} /* namespace jit */
+} /* namespace js */
+
+#endif /* jit_EffectiveAddressAnalysis_h */
diff --git a/js/src/jit/ExecutableAllocator.cpp b/js/src/jit/ExecutableAllocator.cpp
new file mode 100644
index 0000000000..cbae513455
--- /dev/null
+++ b/js/src/jit/ExecutableAllocator.cpp
@@ -0,0 +1,329 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ *
+ * Copyright (C) 2008 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "jit/ExecutableAllocator.h"
+
+#include "js/MemoryMetrics.h"
+#include "util/Poison.h"
+
+using namespace js::jit;
+
+ExecutablePool::~ExecutablePool() {
+#ifdef DEBUG
+ for (size_t bytes : m_codeBytes) {
+ MOZ_ASSERT(bytes == 0);
+ }
+#endif
+
+ MOZ_ASSERT(!isMarked());
+
+ m_allocator->releasePoolPages(this);
+}
+
+void ExecutablePool::release(bool willDestroy) {
+ MOZ_ASSERT(m_refCount != 0);
+ MOZ_ASSERT_IF(willDestroy, m_refCount == 1);
+ if (--m_refCount == 0) {
+ js_delete(this);
+ }
+}
+
+void ExecutablePool::release(size_t n, CodeKind kind) {
+ m_codeBytes[kind] -= n;
+ MOZ_ASSERT(m_codeBytes[kind] < m_allocation.size); // Shouldn't underflow.
+
+ release();
+}
+
+void ExecutablePool::addRef() {
+ // It should be impossible for us to roll over, because only small
+ // pools have multiple holders, and they have one holder per chunk
+ // of generated code, and they only hold 16KB or so of code.
+ MOZ_ASSERT(m_refCount);
+ ++m_refCount;
+ MOZ_ASSERT(m_refCount, "refcount overflow");
+}
+
+void* ExecutablePool::alloc(size_t n, CodeKind kind) {
+ MOZ_ASSERT(n <= available());
+ void* result = m_freePtr;
+ m_freePtr += n;
+
+ m_codeBytes[kind] += n;
+
+ MOZ_MAKE_MEM_UNDEFINED(result, n);
+ return result;
+}
+
+size_t ExecutablePool::available() const {
+ MOZ_ASSERT(m_end >= m_freePtr);
+ return m_end - m_freePtr;
+}
+
+ExecutableAllocator::~ExecutableAllocator() {
+ for (size_t i = 0; i < m_smallPools.length(); i++) {
+ m_smallPools[i]->release(/* willDestroy = */ true);
+ }
+
+ // If this asserts we have a pool leak.
+ MOZ_ASSERT(m_pools.empty());
+}
+
+ExecutablePool* ExecutableAllocator::poolForSize(size_t n) {
+ // Try to fit in an existing small allocator. Use the pool with the
+ // least available space that is big enough (best-fit). This is the
+ // best strategy because (a) it maximizes the chance of the next
+ // allocation fitting in a small pool, and (b) it minimizes the
+ // potential waste when a small pool is next abandoned.
+ ExecutablePool* minPool = nullptr;
+ for (size_t i = 0; i < m_smallPools.length(); i++) {
+ ExecutablePool* pool = m_smallPools[i];
+ if (n <= pool->available() &&
+ (!minPool || pool->available() < minPool->available())) {
+ minPool = pool;
+ }
+ }
+ if (minPool) {
+ minPool->addRef();
+ return minPool;
+ }
+
+ // If the request is large, we just provide a unshared allocator
+ if (n > ExecutableCodePageSize) {
+ return createPool(n);
+ }
+
+ // Create a new allocator
+ ExecutablePool* pool = createPool(ExecutableCodePageSize);
+ if (!pool) {
+ return nullptr;
+ }
+ // At this point, local |pool| is the owner.
+
+ if (m_smallPools.length() < maxSmallPools) {
+ // We haven't hit the maximum number of live pools; add the new pool.
+ // If append() OOMs, we just return an unshared allocator.
+ if (m_smallPools.append(pool)) {
+ pool->addRef();
+ }
+ } else {
+ // Find the pool with the least space.
+ int iMin = 0;
+ for (size_t i = 1; i < m_smallPools.length(); i++) {
+ if (m_smallPools[i]->available() < m_smallPools[iMin]->available()) {
+ iMin = i;
+ }
+ }
+
+ // If the new allocator will result in more free space than the small
+ // pool with the least space, then we will use it instead
+ ExecutablePool* minPool = m_smallPools[iMin];
+ if ((pool->available() - n) > minPool->available()) {
+ minPool->release();
+ m_smallPools[iMin] = pool;
+ pool->addRef();
+ }
+ }
+
+ // Pass ownership to the caller.
+ return pool;
+}
+
+/* static */
+size_t ExecutableAllocator::roundUpAllocationSize(size_t request,
+ size_t granularity) {
+ if ((std::numeric_limits<size_t>::max() - granularity) <= request) {
+ return OVERSIZE_ALLOCATION;
+ }
+
+ // Round up to next page boundary
+ size_t size = request + (granularity - 1);
+ size = size & ~(granularity - 1);
+ MOZ_ASSERT(size >= request);
+ return size;
+}
+
+ExecutablePool* ExecutableAllocator::createPool(size_t n) {
+ size_t allocSize = roundUpAllocationSize(n, ExecutableCodePageSize);
+ if (allocSize == OVERSIZE_ALLOCATION) {
+ return nullptr;
+ }
+
+ ExecutablePool::Allocation a = systemAlloc(allocSize);
+ if (!a.pages) {
+ return nullptr;
+ }
+
+ ExecutablePool* pool = js_new<ExecutablePool>(this, a);
+ if (!pool) {
+ systemRelease(a);
+ return nullptr;
+ }
+
+ if (!m_pools.put(pool)) {
+ // Note: this will call |systemRelease(a)|.
+ js_delete(pool);
+ return nullptr;
+ }
+
+ return pool;
+}
+
+void* ExecutableAllocator::alloc(JSContext* cx, size_t n,
+ ExecutablePool** poolp, CodeKind type) {
+ // Caller must ensure 'n' is word-size aligned. If all allocations are
+ // of word sized quantities, then all subsequent allocations will be
+ // aligned.
+ MOZ_ASSERT(roundUpAllocationSize(n, sizeof(void*)) == n);
+
+ if (n == OVERSIZE_ALLOCATION) {
+ *poolp = nullptr;
+ return nullptr;
+ }
+
+ *poolp = poolForSize(n);
+ if (!*poolp) {
+ return nullptr;
+ }
+
+ // This alloc is infallible because poolForSize() just obtained
+ // (found, or created if necessary) a pool that had enough space.
+ void* result = (*poolp)->alloc(n, type);
+ MOZ_ASSERT(result);
+
+ return result;
+}
+
+void ExecutableAllocator::releasePoolPages(ExecutablePool* pool) {
+ MOZ_ASSERT(pool->m_allocation.pages);
+ systemRelease(pool->m_allocation);
+
+ // Pool may not be present in m_pools if we hit OOM during creation.
+ if (auto ptr = m_pools.lookup(pool)) {
+ m_pools.remove(ptr);
+ }
+}
+
+void ExecutableAllocator::purge() {
+ for (size_t i = 0; i < m_smallPools.length();) {
+ ExecutablePool* pool = m_smallPools[i];
+ if (pool->m_refCount > 1) {
+ // Releasing this pool is not going to deallocate it, so we might as
+ // well hold on to it and reuse it for future allocations.
+ i++;
+ continue;
+ }
+
+ MOZ_ASSERT(pool->m_refCount == 1);
+ pool->release();
+ m_smallPools.erase(&m_smallPools[i]);
+ }
+}
+
+void ExecutableAllocator::addSizeOfCode(JS::CodeSizes* sizes) const {
+ for (ExecPoolHashSet::Range r = m_pools.all(); !r.empty(); r.popFront()) {
+ ExecutablePool* pool = r.front();
+ sizes->ion += pool->m_codeBytes[CodeKind::Ion];
+ sizes->baseline += pool->m_codeBytes[CodeKind::Baseline];
+ sizes->regexp += pool->m_codeBytes[CodeKind::RegExp];
+ sizes->other += pool->m_codeBytes[CodeKind::Other];
+ sizes->unused += pool->m_allocation.size - pool->usedCodeBytes();
+ }
+}
+
+/* static */
+void ExecutableAllocator::reprotectPool(JSRuntime* rt, ExecutablePool* pool,
+ ProtectionSetting protection,
+ MustFlushICache flushICache) {
+ char* start = pool->m_allocation.pages;
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!ReprotectRegion(start, pool->m_freePtr - start, protection,
+ flushICache)) {
+ oomUnsafe.crash("ExecutableAllocator::reprotectPool");
+ }
+}
+
+/* static */
+void ExecutableAllocator::poisonCode(JSRuntime* rt,
+ JitPoisonRangeVector& ranges) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+#ifdef DEBUG
+ // Make sure no pools have the mark bit set.
+ for (size_t i = 0; i < ranges.length(); i++) {
+ MOZ_ASSERT(!ranges[i].pool->isMarked());
+ }
+#endif
+
+ for (size_t i = 0; i < ranges.length(); i++) {
+ ExecutablePool* pool = ranges[i].pool;
+ if (pool->m_refCount == 1) {
+ // This is the last reference so the release() call below will
+ // unmap the memory. Don't bother poisoning it.
+ continue;
+ }
+
+ MOZ_ASSERT(pool->m_refCount > 1);
+
+ // Use the pool's mark bit to indicate we made the pool writable.
+ // This avoids reprotecting a pool multiple times.
+ if (!pool->isMarked()) {
+ reprotectPool(rt, pool, ProtectionSetting::Writable, MustFlushICache::No);
+ pool->mark();
+ }
+
+ // Note: we use memset instead of js::Poison because we want to poison
+ // JIT code in release builds too. Furthermore, we don't want the
+ // invalid-ObjectValue poisoning js::Poison does in debug builds.
+ memset(ranges[i].start, JS_SWEPT_CODE_PATTERN, ranges[i].size);
+ MOZ_MAKE_MEM_NOACCESS(ranges[i].start, ranges[i].size);
+ }
+
+ // Make the pools executable again and drop references. We don't flush the
+ // ICache here to not add extra overhead.
+ for (size_t i = 0; i < ranges.length(); i++) {
+ ExecutablePool* pool = ranges[i].pool;
+ if (pool->isMarked()) {
+ reprotectPool(rt, pool, ProtectionSetting::Executable,
+ MustFlushICache::No);
+ pool->unmark();
+ }
+ pool->release();
+ }
+}
+
+ExecutablePool::Allocation ExecutableAllocator::systemAlloc(size_t n) {
+ void* allocation = AllocateExecutableMemory(n, ProtectionSetting::Executable,
+ MemCheckKind::MakeNoAccess);
+ ExecutablePool::Allocation alloc = {reinterpret_cast<char*>(allocation), n};
+ return alloc;
+}
+
+void ExecutableAllocator::systemRelease(
+ const ExecutablePool::Allocation& alloc) {
+ DeallocateExecutableMemory(alloc.pages, alloc.size);
+}
diff --git a/js/src/jit/ExecutableAllocator.h b/js/src/jit/ExecutableAllocator.h
new file mode 100644
index 0000000000..85c01562c3
--- /dev/null
+++ b/js/src/jit/ExecutableAllocator.h
@@ -0,0 +1,205 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ *
+ * Copyright (C) 2008 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef jit_ExecutableAllocator_h
+#define jit_ExecutableAllocator_h
+
+#include "mozilla/EnumeratedArray.h"
+
+#include <limits>
+#include <stddef.h> // for ptrdiff_t
+
+#include "jit/ProcessExecutableMemory.h"
+#include "js/AllocPolicy.h"
+#include "js/HashTable.h"
+#include "js/TypeDecls.h"
+#include "js/Vector.h"
+
+namespace JS {
+struct CodeSizes;
+} // namespace JS
+
+namespace js {
+namespace jit {
+
+enum class CodeKind : uint8_t { Ion, Baseline, RegExp, Other, Count };
+
+class ExecutableAllocator;
+
+// These are reference-counted. A new one starts with a count of 1.
+class ExecutablePool {
+ friend class ExecutableAllocator;
+
+ private:
+ struct Allocation {
+ char* pages;
+ size_t size;
+ };
+
+ ExecutableAllocator* m_allocator;
+ char* m_freePtr;
+ char* m_end;
+ Allocation m_allocation;
+
+ // Reference count for automatic reclamation.
+ unsigned m_refCount : 31;
+
+ // Flag that can be used by algorithms operating on pools.
+ bool m_mark : 1;
+
+ // Number of bytes currently allocated for each CodeKind.
+ mozilla::EnumeratedArray<CodeKind, CodeKind::Count, size_t> m_codeBytes;
+
+ public:
+ void release(bool willDestroy = false);
+ void release(size_t n, CodeKind kind);
+
+ void addRef();
+
+ ExecutablePool(ExecutableAllocator* allocator, Allocation a)
+ : m_allocator(allocator),
+ m_freePtr(a.pages),
+ m_end(m_freePtr + a.size),
+ m_allocation(a),
+ m_refCount(1),
+ m_mark(false) {
+ for (size_t& count : m_codeBytes) {
+ count = 0;
+ }
+ }
+
+ ~ExecutablePool();
+
+ void mark() {
+ MOZ_ASSERT(!m_mark);
+ m_mark = true;
+ }
+ void unmark() {
+ MOZ_ASSERT(m_mark);
+ m_mark = false;
+ }
+ bool isMarked() const { return m_mark; }
+
+ private:
+ ExecutablePool(const ExecutablePool&) = delete;
+ void operator=(const ExecutablePool&) = delete;
+
+ void* alloc(size_t n, CodeKind kind);
+
+ size_t available() const;
+
+ // Returns the number of bytes that are currently in use (referenced by
+ // live JitCode objects).
+ size_t usedCodeBytes() const {
+ size_t res = 0;
+ for (size_t count : m_codeBytes) {
+ res += count;
+ }
+ return res;
+ }
+};
+
+struct JitPoisonRange {
+ jit::ExecutablePool* pool;
+ void* start;
+ size_t size;
+
+ JitPoisonRange(jit::ExecutablePool* pool, void* start, size_t size)
+ : pool(pool), start(start), size(size) {}
+};
+
+typedef Vector<JitPoisonRange, 0, SystemAllocPolicy> JitPoisonRangeVector;
+
+class ExecutableAllocator {
+ public:
+ ExecutableAllocator() = default;
+ ~ExecutableAllocator();
+
+ void purge();
+
+ // alloc() returns a pointer to some memory, and also (by reference) a
+ // pointer to reference-counted pool. The caller owns a reference to the
+ // pool; i.e. alloc() increments the count before returning the object.
+ void* alloc(JSContext* cx, size_t n, ExecutablePool** poolp, CodeKind type);
+
+ void releasePoolPages(ExecutablePool* pool);
+
+ void addSizeOfCode(JS::CodeSizes* sizes) const;
+
+ private:
+ static const size_t OVERSIZE_ALLOCATION = size_t(-1);
+
+ static size_t roundUpAllocationSize(size_t request, size_t granularity);
+
+ // On OOM, this will return an Allocation where pages is nullptr.
+ ExecutablePool::Allocation systemAlloc(size_t n);
+ static void systemRelease(const ExecutablePool::Allocation& alloc);
+
+ ExecutablePool* createPool(size_t n);
+ ExecutablePool* poolForSize(size_t n);
+
+ static void reprotectPool(JSRuntime* rt, ExecutablePool* pool,
+ ProtectionSetting protection,
+ MustFlushICache flushICache);
+
+ public:
+ [[nodiscard]] static bool makeWritable(void* start, size_t size) {
+ return ReprotectRegion(start, size, ProtectionSetting::Writable,
+ MustFlushICache::No);
+ }
+
+ [[nodiscard]] static bool makeExecutableAndFlushICache(void* start,
+ size_t size) {
+ return ReprotectRegion(start, size, ProtectionSetting::Executable,
+ MustFlushICache::Yes);
+ }
+
+ static void poisonCode(JSRuntime* rt, JitPoisonRangeVector& ranges);
+
+ private:
+ ExecutableAllocator(const ExecutableAllocator&) = delete;
+ void operator=(const ExecutableAllocator&) = delete;
+
+ // These are strong references; they keep pools alive.
+ static const size_t maxSmallPools = 4;
+ typedef js::Vector<ExecutablePool*, maxSmallPools, js::SystemAllocPolicy>
+ SmallExecPoolVector;
+ SmallExecPoolVector m_smallPools;
+
+ // All live pools are recorded here, just for stats purposes. These are
+ // weak references; they don't keep pools alive. When a pool is destroyed
+ // its reference is removed from m_pools.
+ typedef js::HashSet<ExecutablePool*, js::DefaultHasher<ExecutablePool*>,
+ js::SystemAllocPolicy>
+ ExecPoolHashSet;
+ ExecPoolHashSet m_pools; // All pools, just for stats purposes.
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ExecutableAllocator_h */
diff --git a/js/src/jit/FixedList.h b/js/src/jit/FixedList.h
new file mode 100644
index 0000000000..e8422e24d3
--- /dev/null
+++ b/js/src/jit/FixedList.h
@@ -0,0 +1,99 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_FixedList_h
+#define jit_FixedList_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Likely.h"
+
+#include <stddef.h>
+
+#include "jit/JitAllocPolicy.h"
+#include "js/Utility.h"
+
+namespace js {
+namespace jit {
+
+// List of a fixed length, but the length is unknown until runtime.
+template <typename T>
+class FixedList {
+ T* list_;
+ size_t length_;
+
+ private:
+ FixedList(const FixedList&); // no copy definition.
+ void operator=(const FixedList*); // no assignment definition.
+
+ public:
+ FixedList() : list_(nullptr), length_(0) {}
+
+ // Dynamic memory allocation requires the ability to report failure.
+ [[nodiscard]] bool init(TempAllocator& alloc, size_t length) {
+ if (length == 0) {
+ return true;
+ }
+
+ list_ = alloc.allocateArray<T>(length);
+ if (!list_) {
+ return false;
+ }
+
+ length_ = length;
+ return true;
+ }
+
+ size_t empty() const { return length_ == 0; }
+
+ size_t length() const { return length_; }
+
+ void shrink(size_t num) {
+ MOZ_ASSERT(num < length_);
+ length_ -= num;
+ }
+
+ [[nodiscard]] bool growBy(TempAllocator& alloc, size_t num) {
+ size_t newlength = length_ + num;
+ if (newlength < length_) {
+ return false;
+ }
+ size_t bytes;
+ if (MOZ_UNLIKELY(!CalculateAllocSize<T>(newlength, &bytes))) {
+ return false;
+ }
+ T* list = (T*)alloc.allocate(bytes);
+ if (MOZ_UNLIKELY(!list)) {
+ return false;
+ }
+
+ for (size_t i = 0; i < length_; i++) {
+ list[i] = list_[i];
+ }
+
+ length_ += num;
+ list_ = list;
+ return true;
+ }
+
+ T& operator[](size_t index) {
+ MOZ_ASSERT(index < length_);
+ return list_[index];
+ }
+ const T& operator[](size_t index) const {
+ MOZ_ASSERT(index < length_);
+ return list_[index];
+ }
+
+ T* data() { return list_; }
+
+ T* begin() { return list_; }
+ T* end() { return list_ + length_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_FixedList_h */
diff --git a/js/src/jit/FlushICache.cpp b/js/src/jit/FlushICache.cpp
new file mode 100644
index 0000000000..1e2ec69272
--- /dev/null
+++ b/js/src/jit/FlushICache.cpp
@@ -0,0 +1,132 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/FlushICache.h"
+
+#ifdef JS_CODEGEN_ARM64
+# include "jit/arm64/vixl/MozCachingDecoder.h"
+# include "jit/arm64/vixl/Simulator-vixl.h"
+#endif
+
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
+
+# ifdef __linux__
+# include <linux/version.h>
+# define LINUX_HAS_MEMBARRIER (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0))
+# else
+# define LINUX_HAS_MEMBARRIER 0
+# endif
+
+# if LINUX_HAS_MEMBARRIER || defined(__android__)
+# include <string.h>
+
+# if LINUX_HAS_MEMBARRIER
+# include <linux/membarrier.h>
+# include <sys/syscall.h>
+# include <sys/utsname.h>
+# include <unistd.h>
+# elif defined(__android__)
+# include <sys/syscall.h>
+# include <unistd.h>
+# else
+# error "Missing platform-specific declarations for membarrier syscall!"
+# endif // __linux__ / ANDROID
+
+static int membarrier(int cmd, int flags) {
+ return syscall(__NR_membarrier, cmd, flags);
+}
+
+// These definitions come from the Linux kernel source, for kernels before 4.16
+// which didn't have access to these membarrier commands.
+# ifndef MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE
+# define MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE (1 << 5)
+# endif
+
+# ifndef MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE
+# define MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE (1 << 6)
+# endif
+# endif // LINUX_HAS_MEMBARRIER || defined(__android__)
+
+using namespace js;
+using namespace js::jit;
+
+namespace js {
+namespace jit {
+
+bool CanFlushExecutionContextForAllThreads() {
+# if (LINUX_HAS_MEMBARRIER || defined(__android__))
+ // On linux, check the kernel supports membarrier(2), that is, it's a kernel
+ // above Linux 4.16 included.
+ //
+ // Note: this code has been extracted (August 2020) from
+ // https://android.googlesource.com/platform/art/+/58520dfba31d6eeef75f5babff15e09aa28e5db8/libartbase/base/membarrier.cc#50
+ static constexpr int kRequiredMajor = 4;
+ static constexpr int kRequiredMinor = 16;
+
+ static bool computed = false;
+ static bool kernelHasMembarrier = false;
+
+ if (computed) {
+ return kernelHasMembarrier;
+ }
+
+ struct utsname uts;
+ int major, minor;
+ kernelHasMembarrier = uname(&uts) == 0 && strcmp(uts.sysname, "Linux") == 0 &&
+ sscanf(uts.release, "%d.%d", &major, &minor) == 2 &&
+ major >= kRequiredMajor &&
+ (major != kRequiredMajor || minor >= kRequiredMinor);
+
+ // As a test bed, try to run the syscall with the command registering the
+ // intent to use the actual membarrier we'll want to carry out later.
+ //
+ // IMPORTANT: This is required or else running the membarrier later won't
+ // actually interrupt the threads in this process.
+ if (kernelHasMembarrier &&
+ membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED_SYNC_CORE, 0) != 0) {
+ kernelHasMembarrier = false;
+ }
+
+ computed = true;
+ return kernelHasMembarrier;
+# else
+ // On other platforms, we assume that the syscall for flushing the icache
+ // will flush the execution context for other cores.
+ return true;
+# endif
+}
+
+void FlushExecutionContextForAllThreads() {
+ // Callers must check that this operation is available.
+ MOZ_RELEASE_ASSERT(CanFlushExecutionContextForAllThreads());
+
+# if defined(JS_SIMULATOR_ARM64) && defined(JS_CACHE_SIMULATOR_ARM64)
+ // Emulate what the real hardware would do by emitting a membarrier that'll
+ // interrupt and flush the execution context of all threads.
+ using js::jit::SimulatorProcess;
+ js::jit::AutoLockSimulatorCache alsc;
+ SimulatorProcess::membarrier();
+# elif (LINUX_HAS_MEMBARRIER || defined(__android__))
+ // The caller has checked this can be performed, which will have registered
+ // this process to receive the membarrier. See above.
+ //
+ // membarrier will trigger an inter-processor-interrupt on any active threads
+ // of this process. This is an execution context synchronization event
+ // equivalent to running an `isb` instruction.
+ if (membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED_SYNC_CORE, 0) != 0) {
+ // Better safe than sorry.
+ MOZ_CRASH("membarrier can't be executed");
+ }
+# else
+ // On other platforms, we assume that the syscall for flushing the icache
+ // will flush the execution context for other cores.
+# endif
+}
+
+} // namespace jit
+} // namespace js
+
+#endif
diff --git a/js/src/jit/FlushICache.h b/js/src/jit/FlushICache.h
new file mode 100644
index 0000000000..7fcb9f7d02
--- /dev/null
+++ b/js/src/jit/FlushICache.h
@@ -0,0 +1,92 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* Flush the instruction cache of instructions in an address range. */
+
+#ifndef jit_FlushICache_h
+#define jit_FlushICache_h
+
+#include "mozilla/Assertions.h" // MOZ_CRASH
+
+#include <stddef.h> // size_t
+
+namespace js {
+namespace jit {
+
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+
+inline void FlushICache(void* code, size_t size) {
+ // No-op. Code and data caches are coherent on x86 and x64.
+}
+
+#elif (defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)) || \
+ (defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)) || \
+ defined(JS_CODEGEN_LOONG64)
+
+// Invalidate the given code range from the icache. This will also flush the
+// execution context for this core. If this code is to be executed on another
+// thread, that thread must perform an execution context flush first using
+// `FlushExecutionContext` below.
+extern void FlushICache(void* code, size_t size);
+
+#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32)
+
+inline void FlushICache(void* code, size_t size) { MOZ_CRASH(); }
+
+#else
+# error "Unknown architecture!"
+#endif
+
+#if (defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)) || \
+ (defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)) || \
+ defined(JS_CODEGEN_LOONG64)
+
+inline void FlushExecutionContext() {
+ // No-op. Execution context is coherent with instruction cache.
+}
+inline bool CanFlushExecutionContextForAllThreads() { return true; }
+inline void FlushExecutionContextForAllThreads() {
+ // No-op. Execution context is coherent with instruction cache.
+}
+
+#elif defined(JS_CODEGEN_NONE) || defined(JS_CODEGEN_WASM32)
+
+inline void FlushExecutionContext() { MOZ_CRASH(); }
+inline bool CanFlushExecutionContextForAllThreads() { MOZ_CRASH(); }
+inline void FlushExecutionContextForAllThreads() { MOZ_CRASH(); }
+
+#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64)
+
+// ARM and ARM64 must flush the instruction pipeline of the current core
+// before executing newly JIT'ed code. This will remove any stale data from
+// the pipeline that may have referenced invalidated instructions.
+//
+// `FlushICache` will perform this for the thread that compiles the code, but
+// other threads that may execute the code are responsible to call
+// this method.
+extern void FlushExecutionContext();
+
+// Some platforms can flush the excecution context for other threads using a
+// syscall. This is required when JIT'ed code will be published to multiple
+// threads without a synchronization point where a `FlushExecutionContext`
+// could be inserted.
+extern bool CanFlushExecutionContextForAllThreads();
+
+// Flushes the execution context of all threads in this process, equivalent to
+// running `FlushExecutionContext` on every thread.
+//
+// Callers must ensure `CanFlushExecutionContextForAllThreads` is true, or
+// else this will crash.
+extern void FlushExecutionContextForAllThreads();
+
+#else
+# error "Unknown architecture!"
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_FlushICache_h
diff --git a/js/src/jit/FoldLinearArithConstants.cpp b/js/src/jit/FoldLinearArithConstants.cpp
new file mode 100644
index 0000000000..5cb5ef62f6
--- /dev/null
+++ b/js/src/jit/FoldLinearArithConstants.cpp
@@ -0,0 +1,112 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/FoldLinearArithConstants.h"
+
+#include "jit/IonAnalysis.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+using namespace js;
+using namespace jit;
+
+namespace js {
+namespace jit {
+
+// Mark this node and its children as RecoveredOnBailout when they are not used.
+// The marked nodes will be removed during DCE. Marking as RecoveredOnBailout is
+// necessary because the Sink pass is ran before this pass.
+static void markNodesAsRecoveredOnBailout(MDefinition* def) {
+ if (def->hasLiveDefUses() || !DeadIfUnused(def) ||
+ !def->canRecoverOnBailout()) {
+ return;
+ }
+
+ JitSpew(JitSpew_FLAC, "mark as recovered on bailout: %s%u", def->opName(),
+ def->id());
+ def->setRecoveredOnBailoutUnchecked();
+
+ // Recursively mark nodes that do not have multiple uses. This loop is
+ // necessary because a node could be an unused right shift zero or an
+ // unused add, and both need to be marked as RecoveredOnBailout.
+ for (size_t i = 0; i < def->numOperands(); i++) {
+ markNodesAsRecoveredOnBailout(def->getOperand(i));
+ }
+}
+
+// Fold AddIs with one variable and two or more constants into one AddI.
+static void AnalyzeAdd(TempAllocator& alloc, MAdd* add) {
+ if (add->type() != MIRType::Int32 || add->isRecoveredOnBailout()) {
+ return;
+ }
+
+ if (!add->hasUses()) {
+ return;
+ }
+
+ JitSpew(JitSpew_FLAC, "analyze add: %s%u", add->opName(), add->id());
+
+ SimpleLinearSum sum = ExtractLinearSum(add);
+ if (sum.constant == 0 || !sum.term) {
+ return;
+ }
+
+ // Determine which operand is the constant.
+ int idx = add->getOperand(0)->isConstant() ? 0 : 1;
+ if (add->getOperand(idx)->isConstant()) {
+ // Do not replace an add where the outcome is the same add instruction.
+ MOZ_ASSERT(add->getOperand(idx)->toConstant()->type() == MIRType::Int32);
+ if (sum.term == add->getOperand(1 - idx) ||
+ sum.constant == add->getOperand(idx)->toConstant()->toInt32()) {
+ return;
+ }
+ }
+
+ MInstruction* rhs = MConstant::New(alloc, Int32Value(sum.constant));
+ add->block()->insertBefore(add, rhs);
+
+ MAdd* addNew = MAdd::New(alloc, sum.term, rhs, add->truncateKind());
+ addNew->setBailoutKind(add->bailoutKind());
+
+ add->replaceAllLiveUsesWith(addNew);
+ add->block()->insertBefore(add, addNew);
+ JitSpew(JitSpew_FLAC, "replaced with: %s%u", addNew->opName(), addNew->id());
+ JitSpew(JitSpew_FLAC, "and constant: %s%u (%d)", rhs->opName(), rhs->id(),
+ sum.constant);
+
+ // Mark the stale nodes as RecoveredOnBailout since the Sink pass has
+ // been run before this pass. DCE will then remove the unused nodes.
+ markNodesAsRecoveredOnBailout(add);
+}
+
+bool FoldLinearArithConstants(MIRGenerator* mir, MIRGraph& graph) {
+ JitSpew(JitSpew_FLAC, "Begin");
+ for (PostorderIterator block(graph.poBegin()); block != graph.poEnd();
+ block++) {
+ if (mir->shouldCancel("Fold Linear Arithmetic Constants (main loop)")) {
+ return false;
+ }
+
+ for (MInstructionIterator i = block->begin(); i != block->end(); i++) {
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+
+ if (mir->shouldCancel("Fold Linear Arithmetic Constants (inner loop)")) {
+ return false;
+ }
+
+ if (i->isAdd()) {
+ AnalyzeAdd(graph.alloc(), i->toAdd());
+ }
+ }
+ }
+ return true;
+}
+
+} /* namespace jit */
+} /* namespace js */
diff --git a/js/src/jit/FoldLinearArithConstants.h b/js/src/jit/FoldLinearArithConstants.h
new file mode 100644
index 0000000000..78f86f02e5
--- /dev/null
+++ b/js/src/jit/FoldLinearArithConstants.h
@@ -0,0 +1,21 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_FoldLinearArithConstants_h
+#define jit_FoldLinearArithConstants_h
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+class MIRGraph;
+
+[[nodiscard]] bool FoldLinearArithConstants(MIRGenerator* mir, MIRGraph& graph);
+
+} /* namespace jit */
+} /* namespace js */
+
+#endif /* jit_FoldLinearArithConstants_h */
diff --git a/js/src/jit/GenerateAtomicOperations.py b/js/src/jit/GenerateAtomicOperations.py
new file mode 100644
index 0000000000..24b5a191cf
--- /dev/null
+++ b/js/src/jit/GenerateAtomicOperations.py
@@ -0,0 +1,873 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script generates jit/AtomicOperationsGenerated.h
+#
+# See the big comment in jit/AtomicOperations.h for an explanation.
+
+import buildconfig
+
+is_64bit = "JS_64BIT" in buildconfig.defines
+cpu_arch = buildconfig.substs["CPU_ARCH"]
+is_gcc = buildconfig.substs["CC_TYPE"] == "gcc"
+
+
+def fmt_insn(s):
+ return '"' + s + '\\n\\t"\n'
+
+
+def gen_seqcst(fun_name):
+ if cpu_arch in ("x86", "x86_64"):
+ return r"""
+ INLINE_ATTR void %(fun_name)s() {
+ asm volatile ("mfence\n\t" ::: "memory");
+ }""" % {
+ "fun_name": fun_name,
+ }
+ if cpu_arch == "aarch64":
+ return r"""
+ INLINE_ATTR void %(fun_name)s() {
+ asm volatile ("dmb ish\n\t" ::: "memory");
+ }""" % {
+ "fun_name": fun_name,
+ }
+ if cpu_arch == "arm":
+ return r"""
+ INLINE_ATTR void %(fun_name)s() {
+ asm volatile ("dmb sy\n\t" ::: "memory");
+ }""" % {
+ "fun_name": fun_name,
+ }
+ raise Exception("Unexpected arch")
+
+
+def gen_load(fun_name, cpp_type, size, barrier):
+ # NOTE: the assembly code must match the generated code in:
+ # - CacheIRCompiler::emitAtomicsLoadResult
+ # - LIRGenerator::visitLoadUnboxedScalar
+ # - CodeGenerator::visitAtomicLoad64 (on 64-bit platforms)
+ # - MacroAssembler::wasmLoad
+ if cpu_arch in ("x86", "x86_64"):
+ insns = ""
+ if barrier:
+ insns += fmt_insn("mfence")
+ if size == 8:
+ insns += fmt_insn("movb (%[arg]), %[res]")
+ elif size == 16:
+ insns += fmt_insn("movw (%[arg]), %[res]")
+ elif size == 32:
+ insns += fmt_insn("movl (%[arg]), %[res]")
+ else:
+ assert size == 64
+ insns += fmt_insn("movq (%[arg]), %[res]")
+ if barrier:
+ insns += fmt_insn("mfence")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(const %(cpp_type)s* arg) {
+ %(cpp_type)s res;
+ asm volatile (%(insns)s
+ : [res] "=r" (res)
+ : [arg] "r" (arg)
+ : "memory");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "aarch64":
+ insns = ""
+ if barrier:
+ insns += fmt_insn("dmb ish")
+ if size == 8:
+ insns += fmt_insn("ldrb %w[res], [%x[arg]]")
+ elif size == 16:
+ insns += fmt_insn("ldrh %w[res], [%x[arg]]")
+ elif size == 32:
+ insns += fmt_insn("ldr %w[res], [%x[arg]]")
+ else:
+ assert size == 64
+ insns += fmt_insn("ldr %x[res], [%x[arg]]")
+ if barrier:
+ insns += fmt_insn("dmb ish")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(const %(cpp_type)s* arg) {
+ %(cpp_type)s res;
+ asm volatile (%(insns)s
+ : [res] "=r" (res)
+ : [arg] "r" (arg)
+ : "memory");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "arm":
+ insns = ""
+ if barrier:
+ insns += fmt_insn("dmb sy")
+ if size == 8:
+ insns += fmt_insn("ldrb %[res], [%[arg]]")
+ elif size == 16:
+ insns += fmt_insn("ldrh %[res], [%[arg]]")
+ else:
+ assert size == 32
+ insns += fmt_insn("ldr %[res], [%[arg]]")
+ if barrier:
+ insns += fmt_insn("dmb sy")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(const %(cpp_type)s* arg) {
+ %(cpp_type)s res;
+ asm volatile (%(insns)s
+ : [res] "=r" (res)
+ : [arg] "r" (arg)
+ : "memory");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ raise Exception("Unexpected arch")
+
+
+def gen_store(fun_name, cpp_type, size, barrier):
+ # NOTE: the assembly code must match the generated code in:
+ # - CacheIRCompiler::emitAtomicsStoreResult
+ # - LIRGenerator::visitStoreUnboxedScalar
+ # - CodeGenerator::visitAtomicStore64 (on 64-bit platforms)
+ # - MacroAssembler::wasmStore
+ if cpu_arch in ("x86", "x86_64"):
+ insns = ""
+ if barrier:
+ insns += fmt_insn("mfence")
+ if size == 8:
+ insns += fmt_insn("movb %[val], (%[addr])")
+ elif size == 16:
+ insns += fmt_insn("movw %[val], (%[addr])")
+ elif size == 32:
+ insns += fmt_insn("movl %[val], (%[addr])")
+ else:
+ assert size == 64
+ insns += fmt_insn("movq %[val], (%[addr])")
+ if barrier:
+ insns += fmt_insn("mfence")
+ return """
+ INLINE_ATTR void %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ asm volatile (%(insns)s
+ :
+ : [addr] "r" (addr), [val] "r"(val)
+ : "memory");
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "aarch64":
+ insns = ""
+ if barrier:
+ insns += fmt_insn("dmb ish")
+ if size == 8:
+ insns += fmt_insn("strb %w[val], [%x[addr]]")
+ elif size == 16:
+ insns += fmt_insn("strh %w[val], [%x[addr]]")
+ elif size == 32:
+ insns += fmt_insn("str %w[val], [%x[addr]]")
+ else:
+ assert size == 64
+ insns += fmt_insn("str %x[val], [%x[addr]]")
+ if barrier:
+ insns += fmt_insn("dmb ish")
+ return """
+ INLINE_ATTR void %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ asm volatile (%(insns)s
+ :
+ : [addr] "r" (addr), [val] "r"(val)
+ : "memory");
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "arm":
+ insns = ""
+ if barrier:
+ insns += fmt_insn("dmb sy")
+ if size == 8:
+ insns += fmt_insn("strb %[val], [%[addr]]")
+ elif size == 16:
+ insns += fmt_insn("strh %[val], [%[addr]]")
+ else:
+ assert size == 32
+ insns += fmt_insn("str %[val], [%[addr]]")
+ if barrier:
+ insns += fmt_insn("dmb sy")
+ return """
+ INLINE_ATTR void %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ asm volatile (%(insns)s
+ :
+ : [addr] "r" (addr), [val] "r"(val)
+ : "memory");
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ raise Exception("Unexpected arch")
+
+
+def gen_exchange(fun_name, cpp_type, size):
+ # NOTE: the assembly code must match the generated code in:
+ # - MacroAssembler::atomicExchange
+ # - MacroAssembler::atomicExchange64 (on 64-bit platforms)
+ if cpu_arch in ("x86", "x86_64"):
+ # Request an input/output register for `val` so that we can simply XCHG it
+ # with *addr.
+ insns = ""
+ if size == 8:
+ insns += fmt_insn("xchgb %[val], (%[addr])")
+ elif size == 16:
+ insns += fmt_insn("xchgw %[val], (%[addr])")
+ elif size == 32:
+ insns += fmt_insn("xchgl %[val], (%[addr])")
+ else:
+ assert size == 64
+ insns += fmt_insn("xchgq %[val], (%[addr])")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ asm volatile (%(insns)s
+ : [val] "+r" (val)
+ : [addr] "r" (addr)
+ : "memory");
+ return val;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "aarch64":
+ insns = ""
+ insns += fmt_insn("dmb ish")
+ insns += fmt_insn("0:")
+ if size == 8:
+ insns += fmt_insn("ldxrb %w[res], [%x[addr]]")
+ insns += fmt_insn("stxrb %w[scratch], %w[val], [%x[addr]]")
+ elif size == 16:
+ insns += fmt_insn("ldxrh %w[res], [%x[addr]]")
+ insns += fmt_insn("stxrh %w[scratch], %w[val], [%x[addr]]")
+ elif size == 32:
+ insns += fmt_insn("ldxr %w[res], [%x[addr]]")
+ insns += fmt_insn("stxr %w[scratch], %w[val], [%x[addr]]")
+ else:
+ assert size == 64
+ insns += fmt_insn("ldxr %x[res], [%x[addr]]")
+ insns += fmt_insn("stxr %w[scratch], %x[val], [%x[addr]]")
+ insns += fmt_insn("cbnz %w[scratch], 0b")
+ insns += fmt_insn("dmb ish")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ %(cpp_type)s res;
+ uint32_t scratch;
+ asm volatile (%(insns)s
+ : [res] "=&r"(res), [scratch] "=&r"(scratch)
+ : [addr] "r" (addr), [val] "r"(val)
+ : "memory", "cc");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "arm":
+ insns = ""
+ insns += fmt_insn("dmb sy")
+ insns += fmt_insn("0:")
+ if size == 8:
+ insns += fmt_insn("ldrexb %[res], [%[addr]]")
+ insns += fmt_insn("strexb %[scratch], %[val], [%[addr]]")
+ elif size == 16:
+ insns += fmt_insn("ldrexh %[res], [%[addr]]")
+ insns += fmt_insn("strexh %[scratch], %[val], [%[addr]]")
+ else:
+ assert size == 32
+ insns += fmt_insn("ldrex %[res], [%[addr]]")
+ insns += fmt_insn("strex %[scratch], %[val], [%[addr]]")
+ insns += fmt_insn("cmp %[scratch], #1")
+ insns += fmt_insn("beq 0b")
+ insns += fmt_insn("dmb sy")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ %(cpp_type)s res;
+ uint32_t scratch;
+ asm volatile (%(insns)s
+ : [res] "=&r"(res), [scratch] "=&r"(scratch)
+ : [addr] "r" (addr), [val] "r"(val)
+ : "memory", "cc");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ raise Exception("Unexpected arch")
+
+
+def gen_cmpxchg(fun_name, cpp_type, size):
+ # NOTE: the assembly code must match the generated code in:
+ # - MacroAssembler::compareExchange
+ # - MacroAssembler::compareExchange64
+ if cpu_arch == "x86" and size == 64:
+ # Use a +A constraint to load `oldval` into EDX:EAX as input/output.
+ # `newval` is loaded into ECX:EBX.
+ return r"""
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr,
+ %(cpp_type)s oldval,
+ %(cpp_type)s newval) {
+ asm volatile ("lock; cmpxchg8b (%%[addr])\n\t"
+ : "+A" (oldval)
+ : [addr] "r" (addr),
+ "b" (uint32_t(newval & 0xffff'ffff)),
+ "c" (uint32_t(newval >> 32))
+ : "memory", "cc");
+ return oldval;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ }
+ if cpu_arch == "arm" and size == 64:
+ return r"""
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr,
+ %(cpp_type)s oldval,
+ %(cpp_type)s newval) {
+ uint32_t oldval0 = oldval & 0xffff'ffff;
+ uint32_t oldval1 = oldval >> 32;
+ uint32_t newval0 = newval & 0xffff'ffff;
+ uint32_t newval1 = newval >> 32;
+ asm volatile (
+ "dmb sy\n\t"
+ "0: ldrexd r0, r1, [%%[addr]]\n\t"
+ "cmp r0, %%[oldval0]\n\t"
+ "bne 1f\n\t"
+ "cmp r1, %%[oldval1]\n\t"
+ "bne 1f\n\t"
+ "mov r2, %%[newval0]\n\t"
+ "mov r3, %%[newval1]\n\t"
+ "strexd r4, r2, r3, [%%[addr]]\n\t"
+ "cmp r4, #1\n\t"
+ "beq 0b\n\t"
+ "1: dmb sy\n\t"
+ "mov %%[oldval0], r0\n\t"
+ "mov %%[oldval1], r1\n\t"
+ : [oldval0] "+&r" (oldval0), [oldval1] "+&r"(oldval1)
+ : [addr] "r" (addr), [newval0] "r" (newval0), [newval1] "r" (newval1)
+ : "memory", "cc", "r0", "r1", "r2", "r3", "r4");
+ return uint64_t(oldval0) | (uint64_t(oldval1) << 32);
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ }
+ if cpu_arch in ("x86", "x86_64"):
+ # Use a +a constraint to load `oldval` into RAX as input/output register.
+ insns = ""
+ if size == 8:
+ insns += fmt_insn("lock; cmpxchgb %[newval], (%[addr])")
+ elif size == 16:
+ insns += fmt_insn("lock; cmpxchgw %[newval], (%[addr])")
+ elif size == 32:
+ insns += fmt_insn("lock; cmpxchgl %[newval], (%[addr])")
+ else:
+ assert size == 64
+ insns += fmt_insn("lock; cmpxchgq %[newval], (%[addr])")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr,
+ %(cpp_type)s oldval,
+ %(cpp_type)s newval) {
+ asm volatile (%(insns)s
+ : [oldval] "+a" (oldval)
+ : [addr] "r" (addr), [newval] "r" (newval)
+ : "memory", "cc");
+ return oldval;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "aarch64":
+ insns = ""
+ insns += fmt_insn("dmb ish")
+ insns += fmt_insn("0:")
+ if size == 8:
+ insns += fmt_insn("uxtb %w[scratch], %w[oldval]")
+ insns += fmt_insn("ldxrb %w[res], [%x[addr]]")
+ insns += fmt_insn("cmp %w[res], %w[scratch]")
+ insns += fmt_insn("b.ne 1f")
+ insns += fmt_insn("stxrb %w[scratch], %w[newval], [%x[addr]]")
+ elif size == 16:
+ insns += fmt_insn("uxth %w[scratch], %w[oldval]")
+ insns += fmt_insn("ldxrh %w[res], [%x[addr]]")
+ insns += fmt_insn("cmp %w[res], %w[scratch]")
+ insns += fmt_insn("b.ne 1f")
+ insns += fmt_insn("stxrh %w[scratch], %w[newval], [%x[addr]]")
+ elif size == 32:
+ insns += fmt_insn("mov %w[scratch], %w[oldval]")
+ insns += fmt_insn("ldxr %w[res], [%x[addr]]")
+ insns += fmt_insn("cmp %w[res], %w[scratch]")
+ insns += fmt_insn("b.ne 1f")
+ insns += fmt_insn("stxr %w[scratch], %w[newval], [%x[addr]]")
+ else:
+ assert size == 64
+ insns += fmt_insn("mov %x[scratch], %x[oldval]")
+ insns += fmt_insn("ldxr %x[res], [%x[addr]]")
+ insns += fmt_insn("cmp %x[res], %x[scratch]")
+ insns += fmt_insn("b.ne 1f")
+ insns += fmt_insn("stxr %w[scratch], %x[newval], [%x[addr]]")
+ insns += fmt_insn("cbnz %w[scratch], 0b")
+ insns += fmt_insn("1: dmb ish")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr,
+ %(cpp_type)s oldval,
+ %(cpp_type)s newval) {
+ %(cpp_type)s res, scratch;
+ asm volatile (%(insns)s
+ : [res] "=&r" (res), [scratch] "=&r" (scratch)
+ : [addr] "r" (addr), [oldval] "r"(oldval), [newval] "r" (newval)
+ : "memory", "cc");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "arm":
+ insns = ""
+ insns += fmt_insn("dmb sy")
+ insns += fmt_insn("0:")
+ if size == 8:
+ insns += fmt_insn("uxtb %[scratch], %[oldval]")
+ insns += fmt_insn("ldrexb %[res], [%[addr]]")
+ insns += fmt_insn("cmp %[res], %[scratch]")
+ insns += fmt_insn("bne 1f")
+ insns += fmt_insn("strexb %[scratch], %[newval], [%[addr]]")
+ elif size == 16:
+ insns += fmt_insn("uxth %[scratch], %[oldval]")
+ insns += fmt_insn("ldrexh %[res], [%[addr]]")
+ insns += fmt_insn("cmp %[res], %[scratch]")
+ insns += fmt_insn("bne 1f")
+ insns += fmt_insn("strexh %[scratch], %[newval], [%[addr]]")
+ else:
+ assert size == 32
+ insns += fmt_insn("mov %[scratch], %[oldval]")
+ insns += fmt_insn("ldrex %[res], [%[addr]]")
+ insns += fmt_insn("cmp %[res], %[scratch]")
+ insns += fmt_insn("bne 1f")
+ insns += fmt_insn("strex %[scratch], %[newval], [%[addr]]")
+ insns += fmt_insn("cmp %[scratch], #1")
+ insns += fmt_insn("beq 0b")
+ insns += fmt_insn("1: dmb sy")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr,
+ %(cpp_type)s oldval,
+ %(cpp_type)s newval) {
+ %(cpp_type)s res, scratch;
+ asm volatile (%(insns)s
+ : [res] "=&r" (res), [scratch] "=&r" (scratch)
+ : [addr] "r" (addr), [oldval] "r"(oldval), [newval] "r" (newval)
+ : "memory", "cc");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ raise Exception("Unexpected arch")
+
+
+def gen_fetchop(fun_name, cpp_type, size, op):
+ # NOTE: the assembly code must match the generated code in:
+ # - MacroAssembler::atomicFetchOp
+ # - MacroAssembler::atomicFetchOp64 (on 64-bit platforms)
+ if cpu_arch in ("x86", "x86_64"):
+ # The `add` operation can be optimized with XADD.
+ if op == "add":
+ insns = ""
+ if size == 8:
+ insns += fmt_insn("lock; xaddb %[val], (%[addr])")
+ elif size == 16:
+ insns += fmt_insn("lock; xaddw %[val], (%[addr])")
+ elif size == 32:
+ insns += fmt_insn("lock; xaddl %[val], (%[addr])")
+ else:
+ assert size == 64
+ insns += fmt_insn("lock; xaddq %[val], (%[addr])")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ asm volatile (%(insns)s
+ : [val] "+&r" (val)
+ : [addr] "r" (addr)
+ : "memory", "cc");
+ return val;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ # Use a +a constraint to ensure `res` is stored in RAX. This is required
+ # for the CMPXCHG instruction.
+ insns = ""
+ if size == 8:
+ insns += fmt_insn("movb (%[addr]), %[res]")
+ insns += fmt_insn("0: movb %[res], %[scratch]")
+ insns += fmt_insn("OPb %[val], %[scratch]")
+ insns += fmt_insn("lock; cmpxchgb %[scratch], (%[addr])")
+ elif size == 16:
+ insns += fmt_insn("movw (%[addr]), %[res]")
+ insns += fmt_insn("0: movw %[res], %[scratch]")
+ insns += fmt_insn("OPw %[val], %[scratch]")
+ insns += fmt_insn("lock; cmpxchgw %[scratch], (%[addr])")
+ elif size == 32:
+ insns += fmt_insn("movl (%[addr]), %[res]")
+ insns += fmt_insn("0: movl %[res], %[scratch]")
+ insns += fmt_insn("OPl %[val], %[scratch]")
+ insns += fmt_insn("lock; cmpxchgl %[scratch], (%[addr])")
+ else:
+ assert size == 64
+ insns += fmt_insn("movq (%[addr]), %[res]")
+ insns += fmt_insn("0: movq %[res], %[scratch]")
+ insns += fmt_insn("OPq %[val], %[scratch]")
+ insns += fmt_insn("lock; cmpxchgq %[scratch], (%[addr])")
+ insns = insns.replace("OP", op)
+ insns += fmt_insn("jnz 0b")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ %(cpp_type)s res, scratch;
+ asm volatile (%(insns)s
+ : [res] "=&a" (res), [scratch] "=&r" (scratch)
+ : [addr] "r" (addr), [val] "r"(val)
+ : "memory", "cc");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "aarch64":
+ insns = ""
+ insns += fmt_insn("dmb ish")
+ insns += fmt_insn("0:")
+ if size == 8:
+ insns += fmt_insn("ldxrb %w[res], [%x[addr]]")
+ insns += fmt_insn("OP %x[scratch1], %x[res], %x[val]")
+ insns += fmt_insn("stxrb %w[scratch2], %w[scratch1], [%x[addr]]")
+ elif size == 16:
+ insns += fmt_insn("ldxrh %w[res], [%x[addr]]")
+ insns += fmt_insn("OP %x[scratch1], %x[res], %x[val]")
+ insns += fmt_insn("stxrh %w[scratch2], %w[scratch1], [%x[addr]]")
+ elif size == 32:
+ insns += fmt_insn("ldxr %w[res], [%x[addr]]")
+ insns += fmt_insn("OP %x[scratch1], %x[res], %x[val]")
+ insns += fmt_insn("stxr %w[scratch2], %w[scratch1], [%x[addr]]")
+ else:
+ assert size == 64
+ insns += fmt_insn("ldxr %x[res], [%x[addr]]")
+ insns += fmt_insn("OP %x[scratch1], %x[res], %x[val]")
+ insns += fmt_insn("stxr %w[scratch2], %x[scratch1], [%x[addr]]")
+ cpu_op = op
+ if cpu_op == "or":
+ cpu_op = "orr"
+ if cpu_op == "xor":
+ cpu_op = "eor"
+ insns = insns.replace("OP", cpu_op)
+ insns += fmt_insn("cbnz %w[scratch2], 0b")
+ insns += fmt_insn("dmb ish")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ %(cpp_type)s res;
+ uintptr_t scratch1, scratch2;
+ asm volatile (%(insns)s
+ : [res] "=&r" (res), [scratch1] "=&r" (scratch1), [scratch2] "=&r"(scratch2)
+ : [addr] "r" (addr), [val] "r"(val)
+ : "memory", "cc");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ if cpu_arch == "arm":
+ insns = ""
+ insns += fmt_insn("dmb sy")
+ insns += fmt_insn("0:")
+ if size == 8:
+ insns += fmt_insn("ldrexb %[res], [%[addr]]")
+ insns += fmt_insn("OP %[scratch1], %[res], %[val]")
+ insns += fmt_insn("strexb %[scratch2], %[scratch1], [%[addr]]")
+ elif size == 16:
+ insns += fmt_insn("ldrexh %[res], [%[addr]]")
+ insns += fmt_insn("OP %[scratch1], %[res], %[val]")
+ insns += fmt_insn("strexh %[scratch2], %[scratch1], [%[addr]]")
+ else:
+ assert size == 32
+ insns += fmt_insn("ldrex %[res], [%[addr]]")
+ insns += fmt_insn("OP %[scratch1], %[res], %[val]")
+ insns += fmt_insn("strex %[scratch2], %[scratch1], [%[addr]]")
+ cpu_op = op
+ if cpu_op == "or":
+ cpu_op = "orr"
+ if cpu_op == "xor":
+ cpu_op = "eor"
+ insns = insns.replace("OP", cpu_op)
+ insns += fmt_insn("cmp %[scratch2], #1")
+ insns += fmt_insn("beq 0b")
+ insns += fmt_insn("dmb sy")
+ return """
+ INLINE_ATTR %(cpp_type)s %(fun_name)s(%(cpp_type)s* addr, %(cpp_type)s val) {
+ %(cpp_type)s res;
+ uintptr_t scratch1, scratch2;
+ asm volatile (%(insns)s
+ : [res] "=&r" (res), [scratch1] "=&r" (scratch1), [scratch2] "=&r"(scratch2)
+ : [addr] "r" (addr), [val] "r"(val)
+ : "memory", "cc");
+ return res;
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+ raise Exception("Unexpected arch")
+
+
+def gen_copy(fun_name, cpp_type, size, unroll, direction):
+ assert direction in ("down", "up")
+ offset = 0
+ if direction == "up":
+ offset = unroll - 1
+ insns = ""
+ for i in range(unroll):
+ if cpu_arch in ("x86", "x86_64"):
+ if size == 1:
+ insns += fmt_insn("movb OFFSET(%[src]), %[scratch]")
+ insns += fmt_insn("movb %[scratch], OFFSET(%[dst])")
+ elif size == 4:
+ insns += fmt_insn("movl OFFSET(%[src]), %[scratch]")
+ insns += fmt_insn("movl %[scratch], OFFSET(%[dst])")
+ else:
+ assert size == 8
+ insns += fmt_insn("movq OFFSET(%[src]), %[scratch]")
+ insns += fmt_insn("movq %[scratch], OFFSET(%[dst])")
+ elif cpu_arch == "aarch64":
+ if size == 1:
+ insns += fmt_insn("ldrb %w[scratch], [%x[src], OFFSET]")
+ insns += fmt_insn("strb %w[scratch], [%x[dst], OFFSET]")
+ else:
+ assert size == 8
+ insns += fmt_insn("ldr %x[scratch], [%x[src], OFFSET]")
+ insns += fmt_insn("str %x[scratch], [%x[dst], OFFSET]")
+ elif cpu_arch == "arm":
+ if size == 1:
+ insns += fmt_insn("ldrb %[scratch], [%[src], #OFFSET]")
+ insns += fmt_insn("strb %[scratch], [%[dst], #OFFSET]")
+ else:
+ assert size == 4
+ insns += fmt_insn("ldr %[scratch], [%[src], #OFFSET]")
+ insns += fmt_insn("str %[scratch], [%[dst], #OFFSET]")
+ else:
+ raise Exception("Unexpected arch")
+ insns = insns.replace("OFFSET", str(offset * size))
+
+ if direction == "down":
+ offset += 1
+ else:
+ offset -= 1
+
+ return """
+ INLINE_ATTR void %(fun_name)s(uint8_t* dst, const uint8_t* src) {
+ %(cpp_type)s* dst_ = reinterpret_cast<%(cpp_type)s*>(dst);
+ const %(cpp_type)s* src_ = reinterpret_cast<const %(cpp_type)s*>(src);
+ %(cpp_type)s scratch;
+ asm volatile (%(insns)s
+ : [scratch] "=&r" (scratch)
+ : [dst] "r" (dst_), [src] "r"(src_)
+ : "memory");
+ }""" % {
+ "cpp_type": cpp_type,
+ "fun_name": fun_name,
+ "insns": insns,
+ }
+
+
+HEADER_TEMPLATE = """\
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_AtomicOperationsGenerated_h
+#define jit_AtomicOperationsGenerated_h
+
+/* This file is generated by jit/GenerateAtomicOperations.py. Do not edit! */
+
+#include "mozilla/Attributes.h"
+
+namespace js {
+namespace jit {
+
+%(contents)s
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_AtomicOperationsGenerated_h
+"""
+
+
+def generate_atomics_header(c_out):
+ contents = ""
+ if cpu_arch in ("x86", "x86_64", "aarch64") or (
+ cpu_arch == "arm" and int(buildconfig.substs["ARM_ARCH"]) >= 7
+ ):
+ contents += "#define JS_HAVE_GENERATED_ATOMIC_OPS 1"
+
+ # `fence` performs a full memory barrier.
+ contents += gen_seqcst("AtomicFenceSeqCst")
+
+ contents += gen_load("AtomicLoad8SeqCst", "uint8_t", 8, True)
+ contents += gen_load("AtomicLoad16SeqCst", "uint16_t", 16, True)
+ contents += gen_load("AtomicLoad32SeqCst", "uint32_t", 32, True)
+ if is_64bit:
+ contents += gen_load("AtomicLoad64SeqCst", "uint64_t", 64, True)
+
+ # These are access-atomic up to sizeof(uintptr_t).
+ contents += gen_load("AtomicLoad8Unsynchronized", "uint8_t", 8, False)
+ contents += gen_load("AtomicLoad16Unsynchronized", "uint16_t", 16, False)
+ contents += gen_load("AtomicLoad32Unsynchronized", "uint32_t", 32, False)
+ if is_64bit:
+ contents += gen_load("AtomicLoad64Unsynchronized", "uint64_t", 64, False)
+
+ contents += gen_store("AtomicStore8SeqCst", "uint8_t", 8, True)
+ contents += gen_store("AtomicStore16SeqCst", "uint16_t", 16, True)
+ contents += gen_store("AtomicStore32SeqCst", "uint32_t", 32, True)
+ if is_64bit:
+ contents += gen_store("AtomicStore64SeqCst", "uint64_t", 64, True)
+
+ # These are access-atomic up to sizeof(uintptr_t).
+ contents += gen_store("AtomicStore8Unsynchronized", "uint8_t", 8, False)
+ contents += gen_store("AtomicStore16Unsynchronized", "uint16_t", 16, False)
+ contents += gen_store("AtomicStore32Unsynchronized", "uint32_t", 32, False)
+ if is_64bit:
+ contents += gen_store("AtomicStore64Unsynchronized", "uint64_t", 64, False)
+
+ # `exchange` takes a cell address and a value. It stores it in the cell and
+ # returns the value previously in the cell.
+ contents += gen_exchange("AtomicExchange8SeqCst", "uint8_t", 8)
+ contents += gen_exchange("AtomicExchange16SeqCst", "uint16_t", 16)
+ contents += gen_exchange("AtomicExchange32SeqCst", "uint32_t", 32)
+ if is_64bit:
+ contents += gen_exchange("AtomicExchange64SeqCst", "uint64_t", 64)
+
+ # `cmpxchg` takes a cell address, an expected value and a replacement value.
+ # If the value in the cell equals the expected value then the replacement value
+ # is stored in the cell. It always returns the value previously in the cell.
+ contents += gen_cmpxchg("AtomicCmpXchg8SeqCst", "uint8_t", 8)
+ contents += gen_cmpxchg("AtomicCmpXchg16SeqCst", "uint16_t", 16)
+ contents += gen_cmpxchg("AtomicCmpXchg32SeqCst", "uint32_t", 32)
+ contents += gen_cmpxchg("AtomicCmpXchg64SeqCst", "uint64_t", 64)
+
+ # `add` adds a value atomically to the cell and returns the old value in the
+ # cell. (There is no `sub`; just add the negated value.)
+ contents += gen_fetchop("AtomicAdd8SeqCst", "uint8_t", 8, "add")
+ contents += gen_fetchop("AtomicAdd16SeqCst", "uint16_t", 16, "add")
+ contents += gen_fetchop("AtomicAdd32SeqCst", "uint32_t", 32, "add")
+ if is_64bit:
+ contents += gen_fetchop("AtomicAdd64SeqCst", "uint64_t", 64, "add")
+
+ # `and` bitwise-ands a value atomically into the cell and returns the old value
+ # in the cell.
+ contents += gen_fetchop("AtomicAnd8SeqCst", "uint8_t", 8, "and")
+ contents += gen_fetchop("AtomicAnd16SeqCst", "uint16_t", 16, "and")
+ contents += gen_fetchop("AtomicAnd32SeqCst", "uint32_t", 32, "and")
+ if is_64bit:
+ contents += gen_fetchop("AtomicAnd64SeqCst", "uint64_t", 64, "and")
+
+ # `or` bitwise-ors a value atomically into the cell and returns the old value
+ # in the cell.
+ contents += gen_fetchop("AtomicOr8SeqCst", "uint8_t", 8, "or")
+ contents += gen_fetchop("AtomicOr16SeqCst", "uint16_t", 16, "or")
+ contents += gen_fetchop("AtomicOr32SeqCst", "uint32_t", 32, "or")
+ if is_64bit:
+ contents += gen_fetchop("AtomicOr64SeqCst", "uint64_t", 64, "or")
+
+ # `xor` bitwise-xors a value atomically into the cell and returns the old value
+ # in the cell.
+ contents += gen_fetchop("AtomicXor8SeqCst", "uint8_t", 8, "xor")
+ contents += gen_fetchop("AtomicXor16SeqCst", "uint16_t", 16, "xor")
+ contents += gen_fetchop("AtomicXor32SeqCst", "uint32_t", 32, "xor")
+ if is_64bit:
+ contents += gen_fetchop("AtomicXor64SeqCst", "uint64_t", 64, "xor")
+
+ # See comment in jit/AtomicOperations-shared-jit.cpp for an explanation.
+ wordsize = 8 if is_64bit else 4
+ words_in_block = 8
+ blocksize = words_in_block * wordsize
+
+ contents += gen_copy(
+ "AtomicCopyUnalignedBlockDownUnsynchronized",
+ "uint8_t",
+ 1,
+ blocksize,
+ "down",
+ )
+ contents += gen_copy(
+ "AtomicCopyUnalignedBlockUpUnsynchronized", "uint8_t", 1, blocksize, "up"
+ )
+
+ contents += gen_copy(
+ "AtomicCopyUnalignedWordDownUnsynchronized", "uint8_t", 1, wordsize, "down"
+ )
+ contents += gen_copy(
+ "AtomicCopyUnalignedWordUpUnsynchronized", "uint8_t", 1, wordsize, "up"
+ )
+
+ contents += gen_copy(
+ "AtomicCopyBlockDownUnsynchronized",
+ "uintptr_t",
+ wordsize,
+ words_in_block,
+ "down",
+ )
+ contents += gen_copy(
+ "AtomicCopyBlockUpUnsynchronized",
+ "uintptr_t",
+ wordsize,
+ words_in_block,
+ "up",
+ )
+
+ contents += gen_copy(
+ "AtomicCopyWordUnsynchronized", "uintptr_t", wordsize, 1, "down"
+ )
+ contents += gen_copy("AtomicCopyByteUnsynchronized", "uint8_t", 1, 1, "down")
+
+ contents += "\n"
+ contents += (
+ "constexpr size_t JS_GENERATED_ATOMICS_BLOCKSIZE = "
+ + str(blocksize)
+ + ";\n"
+ )
+ contents += (
+ "constexpr size_t JS_GENERATED_ATOMICS_WORDSIZE = " + str(wordsize) + ";\n"
+ )
+
+ # Work around a GCC issue on 32-bit x86 by adding MOZ_NEVER_INLINE.
+ # See bug 1756347.
+ if is_gcc and cpu_arch == "x86":
+ contents = contents.replace("INLINE_ATTR", "MOZ_NEVER_INLINE inline")
+ else:
+ contents = contents.replace("INLINE_ATTR", "inline")
+
+ c_out.write(
+ HEADER_TEMPLATE
+ % {
+ "contents": contents,
+ }
+ )
diff --git a/js/src/jit/GenerateCacheIRFiles.py b/js/src/jit/GenerateCacheIRFiles.py
new file mode 100644
index 0000000000..7d0b6fb685
--- /dev/null
+++ b/js/src/jit/GenerateCacheIRFiles.py
@@ -0,0 +1,539 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script generates jit/CacheIROpsGenerated.h from CacheIROps.yaml
+
+from collections import OrderedDict
+
+import buildconfig
+import six
+import yaml
+from mozbuild.preprocessor import Preprocessor
+
+HEADER_TEMPLATE = """\
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef %(includeguard)s
+#define %(includeguard)s
+
+/* This file is generated by jit/GenerateCacheIRFiles.py. Do not edit! */
+
+%(contents)s
+
+#endif // %(includeguard)s
+"""
+
+
+def generate_header(c_out, includeguard, contents):
+ c_out.write(
+ HEADER_TEMPLATE
+ % {
+ "includeguard": includeguard,
+ "contents": contents,
+ }
+ )
+
+
+def load_yaml(yaml_path):
+ # First invoke preprocessor.py so that we can use #ifdef JS_SIMULATOR in
+ # the YAML file.
+ pp = Preprocessor()
+ pp.context.update(buildconfig.defines["ALLDEFINES"])
+ pp.out = six.StringIO()
+ pp.do_filter("substitution")
+ pp.do_include(yaml_path)
+ contents = pp.out.getvalue()
+
+ # Load into an OrderedDict to ensure order is preserved. Note: Python 3.7+
+ # also preserves ordering for normal dictionaries.
+ # Code based on https://stackoverflow.com/a/21912744.
+ class OrderedLoader(yaml.Loader):
+ pass
+
+ def construct_mapping(loader, node):
+ loader.flatten_mapping(node)
+ return OrderedDict(loader.construct_pairs(node))
+
+ tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
+ OrderedLoader.add_constructor(tag, construct_mapping)
+ return yaml.load(contents, OrderedLoader)
+
+
+# Information for generating CacheIRWriter code for a single argument. Tuple
+# stores the C++ argument type and the CacheIRWriter method to call.
+arg_writer_info = {
+ "ValId": ("ValOperandId", "writeOperandId"),
+ "ObjId": ("ObjOperandId", "writeOperandId"),
+ "StringId": ("StringOperandId", "writeOperandId"),
+ "SymbolId": ("SymbolOperandId", "writeOperandId"),
+ "BooleanId": ("BooleanOperandId", "writeOperandId"),
+ "Int32Id": ("Int32OperandId", "writeOperandId"),
+ "NumberId": ("NumberOperandId", "writeOperandId"),
+ "BigIntId": ("BigIntOperandId", "writeOperandId"),
+ "ValueTagId": ("ValueTagOperandId", "writeOperandId"),
+ "IntPtrId": ("IntPtrOperandId", "writeOperandId"),
+ "RawId": ("OperandId", "writeOperandId"),
+ "ShapeField": ("Shape*", "writeShapeField"),
+ "GetterSetterField": ("GetterSetter*", "writeGetterSetterField"),
+ "ObjectField": ("JSObject*", "writeObjectField"),
+ "StringField": ("JSString*", "writeStringField"),
+ "AtomField": ("JSAtom*", "writeStringField"),
+ "PropertyNameField": ("PropertyName*", "writeStringField"),
+ "SymbolField": ("JS::Symbol*", "writeSymbolField"),
+ "BaseScriptField": ("BaseScript*", "writeBaseScriptField"),
+ "RawInt32Field": ("uint32_t", "writeRawInt32Field"),
+ "RawPointerField": ("const void*", "writeRawPointerField"),
+ "IdField": ("jsid", "writeIdField"),
+ "ValueField": ("const Value&", "writeValueField"),
+ "RawInt64Field": ("uint64_t", "writeRawInt64Field"),
+ "DoubleField": ("double", "writeDoubleField"),
+ "AllocSiteField": ("gc::AllocSite*", "writeAllocSiteField"),
+ "JSOpImm": ("JSOp", "writeJSOpImm"),
+ "BoolImm": ("bool", "writeBoolImm"),
+ "ByteImm": ("uint32_t", "writeByteImm"), # uint32_t to enable fits-in-byte asserts.
+ "GuardClassKindImm": ("GuardClassKind", "writeGuardClassKindImm"),
+ "ValueTypeImm": ("ValueType", "writeValueTypeImm"),
+ "JSWhyMagicImm": ("JSWhyMagic", "writeJSWhyMagicImm"),
+ "CallFlagsImm": ("CallFlags", "writeCallFlagsImm"),
+ "ScalarTypeImm": ("Scalar::Type", "writeScalarTypeImm"),
+ "UnaryMathFunctionImm": ("UnaryMathFunction", "writeUnaryMathFunctionImm"),
+ "WasmValTypeImm": ("wasm::ValType::Kind", "writeWasmValTypeImm"),
+ "Int32Imm": ("int32_t", "writeInt32Imm"),
+ "UInt32Imm": ("uint32_t", "writeUInt32Imm"),
+ "JSNativeImm": ("JSNative", "writeJSNativeImm"),
+ "StaticStringImm": ("const char*", "writeStaticStringImm"),
+ "AllocKindImm": ("gc::AllocKind", "writeAllocKindImm"),
+ "CompletionKindImm": ("CompletionKind", "writeCompletionKindImm"),
+}
+
+
+def gen_writer_method(name, args, custom_writer):
+ """Generates a CacheIRWRiter method for a single opcode."""
+
+ # Generate a single method that writes the opcode and each argument.
+ # For example:
+ #
+ # void guardShape(ObjOperandId obj, Shape* shape) {
+ # writeOp(CacheOp::GuardShape);
+ # writeOperandId(obj);
+ # writeShapeField(shape);
+ # assertLengthMatches();
+ # }
+ #
+ # The assertLengthMatches() call is to assert the information in the
+ # arg_length dictionary below matches what's written.
+
+ # Method names start with a lowercase letter.
+ method_name = name[0].lower() + name[1:]
+ if custom_writer:
+ method_name += "_"
+
+ method_args = []
+ ret_type = "void"
+ args_code = ""
+ if args:
+ for arg_name, arg_type in six.iteritems(args):
+ cpp_type, write_method = arg_writer_info[arg_type]
+ if arg_name == "result":
+ ret_type = cpp_type
+ args_code += " {} result(newOperandId());\\\n".format(cpp_type)
+ args_code += " writeOperandId(result);\\\n"
+ else:
+ method_args.append("{} {}".format(cpp_type, arg_name))
+ args_code += " {}({});\\\n".format(write_method, arg_name)
+
+ code = ""
+ if custom_writer:
+ code += "private:\\\n"
+ code += "{} {}({}) {{\\\n".format(ret_type, method_name, ", ".join(method_args))
+ code += " writeOp(CacheOp::{});\\\n".format(name)
+ code += args_code
+ code += " assertLengthMatches();\\\n"
+ if ret_type != "void":
+ code += " return result;\\\n"
+ code += "}"
+ if custom_writer:
+ code += "\\\npublic:"
+ return code
+
+
+# Information for generating code using CacheIRReader for a single argument.
+# Tuple stores the C++ type, the suffix used for arguments/variables of this
+# type, and the expression to read this type from CacheIRReader.
+arg_reader_info = {
+ "ValId": ("ValOperandId", "Id", "reader.valOperandId()"),
+ "ObjId": ("ObjOperandId", "Id", "reader.objOperandId()"),
+ "StringId": ("StringOperandId", "Id", "reader.stringOperandId()"),
+ "SymbolId": ("SymbolOperandId", "Id", "reader.symbolOperandId()"),
+ "BooleanId": ("BooleanOperandId", "Id", "reader.booleanOperandId()"),
+ "Int32Id": ("Int32OperandId", "Id", "reader.int32OperandId()"),
+ "NumberId": ("NumberOperandId", "Id", "reader.numberOperandId()"),
+ "BigIntId": ("BigIntOperandId", "Id", "reader.bigIntOperandId()"),
+ "ValueTagId": ("ValueTagOperandId", "Id", "reader.valueTagOperandId()"),
+ "IntPtrId": ("IntPtrOperandId", "Id", "reader.intPtrOperandId()"),
+ "RawId": ("uint32_t", "Id", "reader.rawOperandId()"),
+ "ShapeField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "GetterSetterField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "ObjectField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "StringField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "AtomField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "PropertyNameField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "SymbolField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "BaseScriptField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "RawInt32Field": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "RawPointerField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "IdField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "ValueField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "RawInt64Field": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "DoubleField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "AllocSiteField": ("uint32_t", "Offset", "reader.stubOffset()"),
+ "JSOpImm": ("JSOp", "", "reader.jsop()"),
+ "BoolImm": ("bool", "", "reader.readBool()"),
+ "ByteImm": ("uint8_t", "", "reader.readByte()"),
+ "GuardClassKindImm": ("GuardClassKind", "", "reader.guardClassKind()"),
+ "ValueTypeImm": ("ValueType", "", "reader.valueType()"),
+ "JSWhyMagicImm": ("JSWhyMagic", "", "reader.whyMagic()"),
+ "CallFlagsImm": ("CallFlags", "", "reader.callFlags()"),
+ "ScalarTypeImm": ("Scalar::Type", "", "reader.scalarType()"),
+ "UnaryMathFunctionImm": ("UnaryMathFunction", "", "reader.unaryMathFunction()"),
+ "WasmValTypeImm": ("wasm::ValType::Kind", "", "reader.wasmValType()"),
+ "Int32Imm": ("int32_t", "", "reader.int32Immediate()"),
+ "UInt32Imm": ("uint32_t", "", "reader.uint32Immediate()"),
+ "JSNativeImm": ("JSNative", "", "reinterpret_cast<JSNative>(reader.pointer())"),
+ "StaticStringImm": ("const char*", "", "reinterpret_cast<char*>(reader.pointer())"),
+ "AllocKindImm": ("gc::AllocKind", "", "reader.allocKind()"),
+ "CompletionKindImm": ("CompletionKind", "", "reader.completionKind()"),
+}
+
+
+def gen_compiler_method(name, args):
+ """Generates CacheIRCompiler or WarpCacheIRTranspiler header code for a
+ single opcode."""
+
+ method_name = "emit" + name
+
+ # We generate the signature of the method that needs to be implemented and a
+ # separate function forwarding to it. For example:
+ #
+ # [[nodiscard]] bool emitGuardShape(ObjOperandId objId, uint32_t shapeOffset);
+ # [[nodiscard]] bool emitGuardShape(CacheIRReader& reader) {
+ # ObjOperandId objId = reader.objOperandId();
+ # uint32_t shapeOffset = reader.stubOffset();
+ # return emitGuardShape(objId, shapeOffset);
+ # }
+ cpp_args = []
+ method_args = []
+ args_code = ""
+ if args:
+ for arg_name, arg_type in six.iteritems(args):
+ cpp_type, suffix, readexpr = arg_reader_info[arg_type]
+ cpp_name = arg_name + suffix
+ cpp_args.append(cpp_name)
+ method_args.append("{} {}".format(cpp_type, cpp_name))
+ args_code += " {} {} = {};\\\n".format(cpp_type, cpp_name, readexpr)
+
+ # Generate signature.
+ code = "[[nodiscard]] bool {}({});\\\n".format(method_name, ", ".join(method_args))
+
+ # Generate the method forwarding to it.
+ code += "[[nodiscard]] bool {}(CacheIRReader& reader) {{\\\n".format(method_name)
+ code += args_code
+ code += " return {}({});\\\n".format(method_name, ", ".join(cpp_args))
+ code += "}\\\n"
+
+ return code
+
+
+# For each argument type, the method name for printing it.
+arg_spewer_method = {
+ "ValId": "spewOperandId",
+ "ObjId": "spewOperandId",
+ "StringId": "spewOperandId",
+ "SymbolId": "spewOperandId",
+ "BooleanId": "spewOperandId",
+ "Int32Id": "spewOperandId",
+ "NumberId": "spewOperandId",
+ "BigIntId": "spewOperandId",
+ "ValueTagId": "spewOperandId",
+ "IntPtrId": "spewOperandId",
+ "RawId": "spewRawOperandId",
+ "ShapeField": "spewField",
+ "GetterSetterField": "spewField",
+ "ObjectField": "spewField",
+ "StringField": "spewField",
+ "AtomField": "spewField",
+ "PropertyNameField": "spewField",
+ "SymbolField": "spewField",
+ "BaseScriptField": "spewField",
+ "RawInt32Field": "spewField",
+ "RawPointerField": "spewField",
+ "IdField": "spewField",
+ "ValueField": "spewField",
+ "RawInt64Field": "spewField",
+ "DoubleField": "spewField",
+ "AllocSiteField": "spewField",
+ "JSOpImm": "spewJSOpImm",
+ "BoolImm": "spewBoolImm",
+ "ByteImm": "spewByteImm",
+ "GuardClassKindImm": "spewGuardClassKindImm",
+ "ValueTypeImm": "spewValueTypeImm",
+ "JSWhyMagicImm": "spewJSWhyMagicImm",
+ "CallFlagsImm": "spewCallFlagsImm",
+ "ScalarTypeImm": "spewScalarTypeImm",
+ "UnaryMathFunctionImm": "spewUnaryMathFunctionImm",
+ "WasmValTypeImm": "spewWasmValTypeImm",
+ "Int32Imm": "spewInt32Imm",
+ "UInt32Imm": "spewUInt32Imm",
+ "JSNativeImm": "spewJSNativeImm",
+ "StaticStringImm": "spewStaticStringImm",
+ "AllocKindImm": "spewAllocKindImm",
+ "CompletionKindImm": "spewCompletionKindImm",
+}
+
+
+def gen_spewer_method(name, args):
+ """Generates spewer code for a single opcode."""
+
+ method_name = "spew" + name
+
+ # Generate code like this:
+ #
+ # void spewGuardShape(CacheIRReader& reader) {
+ # spewOp(CacheOp::GuardShape);
+ # spewOperandId("objId", reader.objOperandId());
+ # spewOperandSeparator();
+ # spewField("shapeOffset", reader.stubOffset());
+ # spewOpEnd();
+ # }
+ args_code = ""
+ if args:
+ is_first = True
+ for arg_name, arg_type in six.iteritems(args):
+ _, suffix, readexpr = arg_reader_info[arg_type]
+ arg_name += suffix
+ spew_method = arg_spewer_method[arg_type]
+ if not is_first:
+ args_code += " spewArgSeparator();\\\n"
+ args_code += ' {}("{}", {});\\\n'.format(spew_method, arg_name, readexpr)
+ is_first = False
+
+ code = "void {}(CacheIRReader& reader) {{\\\n".format(method_name)
+ code += " spewOp(CacheOp::{});\\\n".format(name)
+ code += args_code
+ code += " spewOpEnd();\\\n"
+ code += "}\\\n"
+
+ return code
+
+
+def gen_clone_method(name, args):
+ """Generates code for cloning a single opcode."""
+
+ method_name = "clone" + name
+
+ # Generate code like this:
+ #
+ # void cloneGuardShape(CacheIRReader& reader, CacheIRWriter& writer) {
+ # writer.writeOp(CacheOp::GuardShape);
+ # ObjOperandId objId = reader.objOperandId();
+ # writer.writeOperandId(objId);
+ # uint32_t shapeOffset = reader.stubOffset();
+ # Shape* shape = getShapeField(shapeOffset);
+ # writer.writeShapeField(shape);
+ # writer.assertLengthMatches();
+ # }
+
+ args_code = ""
+ if args:
+ for arg_name, arg_type in six.iteritems(args):
+ if arg_type == "RawId":
+ arg_type = "ValId"
+
+ read_type, suffix, readexpr = arg_reader_info[arg_type]
+ read_name = arg_name + suffix
+ value_name = read_name
+ args_code += " {} {} = {};\\\n".format(read_type, read_name, readexpr)
+
+ write_type, write_method = arg_writer_info[arg_type]
+ if arg_name == "result":
+ args_code += " writer.newOperandId();\\\n"
+ if suffix == "Offset":
+ # If the write function takes T&, the intermediate variable
+ # should be of type T.
+ if write_type.endswith("&"):
+ write_type = write_type[:-1]
+ value_name = arg_name
+ args_code += " {} {} = get{}({});\\\n".format(
+ write_type, value_name, arg_type, read_name
+ )
+ args_code += " writer.{}({});\\\n".format(write_method, value_name)
+
+ code = "void {}".format(method_name)
+ code += "(CacheIRReader& reader, CacheIRWriter& writer) {{\\\n"
+ code += " writer.writeOp(CacheOp::{});\\\n".format(name)
+ code += args_code
+ code += " writer.assertLengthMatches();\\\n"
+ code += "}}\\\n"
+
+ return code
+
+
+# Length in bytes for each argument type, either an integer or a C++ expression.
+# This is used to generate the CacheIROpArgLengths array. CacheIRWriter asserts
+# the number of bytes written matches the value in that array.
+arg_length = {
+ "ValId": 1,
+ "ObjId": 1,
+ "StringId": 1,
+ "SymbolId": 1,
+ "BooleanId": 1,
+ "Int32Id": 1,
+ "NumberId": 1,
+ "BigIntId": 1,
+ "ValueTagId": 1,
+ "IntPtrId": 1,
+ "RawId": 1,
+ "ShapeField": 1,
+ "GetterSetterField": 1,
+ "ObjectField": 1,
+ "StringField": 1,
+ "AtomField": 1,
+ "PropertyNameField": 1,
+ "SymbolField": 1,
+ "BaseScriptField": 1,
+ "RawInt32Field": 1,
+ "RawPointerField": 1,
+ "RawInt64Field": 1,
+ "DoubleField": 1,
+ "IdField": 1,
+ "ValueField": 1,
+ "AllocSiteField": 1,
+ "ByteImm": 1,
+ "BoolImm": 1,
+ "CallFlagsImm": 1,
+ "ScalarTypeImm": 1,
+ "UnaryMathFunctionImm": 1,
+ "JSOpImm": 1,
+ "ValueTypeImm": 1,
+ "GuardClassKindImm": 1,
+ "JSWhyMagicImm": 1,
+ "WasmValTypeImm": 1,
+ "Int32Imm": 4,
+ "UInt32Imm": 4,
+ "JSNativeImm": "sizeof(uintptr_t)",
+ "StaticStringImm": "sizeof(uintptr_t)",
+ "AllocKindImm": 1,
+ "CompletionKindImm": 1,
+}
+
+
+def generate_cacheirops_header(c_out, yaml_path):
+ """Generate CacheIROpsGenerated.h from CacheIROps.yaml. The generated file
+ contains a list of all CacheIR ops and generated source code for
+ CacheIRWriter and CacheIRCompiler."""
+
+ data = load_yaml(yaml_path)
+
+ # CACHE_IR_OPS items. Each item stores an opcode name and arguments length
+ # expression. For example: _(GuardShape, 1 + 1)
+ ops_items = []
+
+ # Generated CacheIRWriter methods.
+ writer_methods = []
+
+ # Generated CacheIRCompiler methods.
+ compiler_shared_methods = []
+ compiler_unshared_methods = []
+
+ # Generated WarpCacheIRTranspiler methods.
+ transpiler_methods = []
+
+ # List of ops supported by WarpCacheIRTranspiler.
+ transpiler_ops = []
+
+ # Generated methods for spewers.
+ spewer_methods = []
+
+ # Generated methods for cloning IC stubs
+ clone_methods = []
+
+ for op in data:
+ name = op["name"]
+
+ args = op["args"]
+ assert args is None or isinstance(args, OrderedDict)
+
+ shared = op["shared"]
+ assert isinstance(shared, bool)
+
+ transpile = op["transpile"]
+ assert isinstance(transpile, bool)
+
+ # Unscored Ops default to UINT32_MAX
+ cost_estimate = op.get("cost_estimate", int(0xFFFFFFFF))
+ assert isinstance(cost_estimate, int)
+
+ custom_writer = op.get("custom_writer", False)
+ assert isinstance(custom_writer, bool)
+
+ if args:
+ args_length = " + ".join([str(arg_length[v]) for v in args.values()])
+ else:
+ args_length = "0"
+
+ transpile_str = "true" if transpile else "false"
+ ops_items.append(
+ "_({}, {}, {}, {})".format(name, args_length, transpile_str, cost_estimate)
+ )
+
+ writer_methods.append(gen_writer_method(name, args, custom_writer))
+
+ if shared:
+ compiler_shared_methods.append(gen_compiler_method(name, args))
+ else:
+ compiler_unshared_methods.append(gen_compiler_method(name, args))
+
+ if transpile:
+ transpiler_methods.append(gen_compiler_method(name, args))
+ transpiler_ops.append("_({})".format(name))
+
+ spewer_methods.append(gen_spewer_method(name, args))
+
+ clone_methods.append(gen_clone_method(name, args))
+
+ contents = "#define CACHE_IR_OPS(_)\\\n"
+ contents += "\\\n".join(ops_items)
+ contents += "\n\n"
+
+ contents += "#define CACHE_IR_WRITER_GENERATED \\\n"
+ contents += "\\\n".join(writer_methods)
+ contents += "\n\n"
+
+ contents += "#define CACHE_IR_COMPILER_SHARED_GENERATED \\\n"
+ contents += "\\\n".join(compiler_shared_methods)
+ contents += "\n\n"
+
+ contents += "#define CACHE_IR_COMPILER_UNSHARED_GENERATED \\\n"
+ contents += "\\\n".join(compiler_unshared_methods)
+ contents += "\n\n"
+
+ contents += "#define CACHE_IR_TRANSPILER_GENERATED \\\n"
+ contents += "\\\n".join(transpiler_methods)
+ contents += "\n\n"
+
+ contents += "#define CACHE_IR_TRANSPILER_OPS(_)\\\n"
+ contents += "\\\n".join(transpiler_ops)
+ contents += "\n\n"
+
+ contents += "#define CACHE_IR_SPEWER_GENERATED \\\n"
+ contents += "\\\n".join(spewer_methods)
+ contents += "\n\n"
+
+ contents += "#define CACHE_IR_CLONE_GENERATED \\\n"
+ contents += "\\\n".join(clone_methods)
+ contents += "\n\n"
+
+ generate_header(c_out, "jit_CacheIROpsGenerated_h", contents)
diff --git a/js/src/jit/GenerateLIRFiles.py b/js/src/jit/GenerateLIRFiles.py
new file mode 100644
index 0000000000..86882966fd
--- /dev/null
+++ b/js/src/jit/GenerateLIRFiles.py
@@ -0,0 +1,298 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script generates jit/LIROpsGenerated.h (list of LIR instructions)
+# from LIROps.yaml.
+
+from collections import OrderedDict
+
+import buildconfig
+import six
+import yaml
+from mozbuild.preprocessor import Preprocessor
+
+HEADER_TEMPLATE = """\
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef %(includeguard)s
+#define %(includeguard)s
+
+/* This file is generated by jit/GenerateLIRFiles.py. Do not edit! */
+
+%(contents)s
+
+#endif // %(includeguard)s
+"""
+
+
+def load_yaml(yaml_path):
+ # First invoke preprocessor.py so that we can use #ifdef JS_SIMULATOR in
+ # the YAML file.
+ pp = Preprocessor()
+ pp.context.update(buildconfig.defines["ALLDEFINES"])
+ pp.out = six.StringIO()
+ pp.do_filter("substitution")
+ pp.do_include(yaml_path)
+ contents = pp.out.getvalue()
+
+ # Load into an OrderedDict to ensure order is preserved. Note: Python 3.7
+ # also preserves ordering for normal dictionaries.
+ # Code based on https://stackoverflow.com/a/21912744.
+ class OrderedLoader(yaml.Loader):
+ pass
+
+ def construct_mapping(loader, node):
+ loader.flatten_mapping(node)
+ return OrderedDict(loader.construct_pairs(node))
+
+ tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
+ OrderedLoader.add_constructor(tag, construct_mapping)
+ return yaml.load(contents, OrderedLoader)
+
+
+def generate_header(c_out, includeguard, contents):
+ c_out.write(
+ HEADER_TEMPLATE
+ % {
+ "includeguard": includeguard,
+ "contents": contents,
+ }
+ )
+
+
+operand_types = {
+ "WordSized": "LAllocation",
+ "BoxedValue": "LBoxAllocation",
+ "Int64": "LInt64Allocation",
+}
+
+
+result_types = {
+ "WordSized": "1",
+ "BoxedValue": "BOX_PIECES",
+ "Int64": "INT64_PIECES",
+}
+
+
+def gen_helper_template_value(num_regular_allocs, num_value_allocs, num_int64_allocs):
+ template_str = ""
+ if num_value_allocs:
+ template_str += str(num_value_allocs) + " * BOX_PIECES + "
+ if num_int64_allocs:
+ template_str += str(num_int64_allocs) + " * INT64_PIECES + "
+ template_str += str(num_regular_allocs)
+ return template_str
+
+
+def build_index_def(num_specials_operands, index_value, num_reg_operands, piece):
+ if num_specials_operands:
+ return " static const size_t {} = {} + {} * {};\\\n".format(
+ index_value, num_reg_operands, piece, num_specials_operands
+ )
+ else:
+ return " static const size_t {} = {};\\\n".format(
+ index_value, num_reg_operands
+ )
+
+
+def gen_lir_class(
+ name, result_type, operands, arguments, num_temps, call_instruction, mir_op
+):
+ """Generates class definition for a single LIR opcode."""
+ class_name = "L" + name
+
+ getters = []
+ setters = []
+ # Operand index definitions.
+ oper_indices = []
+ # Parameters for the class constructor.
+ constructor_params = []
+
+ num_reg_operands = 0
+ num_value_operands = 0
+ num_int64_operands = 0
+ if operands:
+ # Get number of LAllocations to use for defining indices.
+ for operand in operands:
+ if operands[operand] == "WordSized":
+ num_reg_operands += 1
+
+ current_reg_oper = 0
+ for operand in operands:
+ op_type = operands[operand]
+ op_alloc_type = operand_types[op_type]
+ constructor_params.append("const " + op_alloc_type + "& " + operand)
+ if op_type == "WordSized":
+ index_value = str(current_reg_oper)
+ current_reg_oper += 1
+ getters.append(
+ " const "
+ + op_alloc_type
+ + "* "
+ + operand
+ + "() { return getOperand("
+ + index_value
+ + "); }"
+ )
+ setters.append(" setOperand(" + index_value + ", " + operand + ");")
+ elif op_type == "BoxedValue":
+ index_value = operand[0].upper() + operand[1:] + "Index"
+ oper_indices.append(
+ build_index_def(
+ num_value_operands, index_value, num_reg_operands, "BOX_PIECES"
+ )
+ )
+ num_value_operands += 1
+ # No getters generated for BoxedValue operands.
+ setters.append(
+ " setBoxOperand(" + index_value + ", " + operand + ");"
+ )
+ elif op_type == "Int64":
+ index_value = operand[0].upper() + operand[1:] + "Index"
+ oper_indices.append(
+ build_index_def(
+ num_int64_operands,
+ index_value,
+ num_reg_operands,
+ "INT64_PIECES",
+ )
+ )
+ num_int64_operands += 1
+ getters.append(
+ " const "
+ + op_alloc_type
+ + " "
+ + operand
+ + "() { return getInt64Operand("
+ + index_value
+ + "); }"
+ )
+ setters.append(
+ " setInt64Operand(" + index_value + ", " + operand + ");"
+ )
+ else:
+ raise Exception("Invalid operand type: " + op_type)
+ if num_temps:
+ for temp in range(num_temps):
+ constructor_params.append("const LDefinition& temp" + str(temp))
+ setters.append(" setTemp(" + str(temp) + ", temp" + str(temp) + ");")
+ getters.append(
+ " const LDefinition* temp"
+ + str(temp)
+ + "() { return getTemp("
+ + str(temp)
+ + "); }"
+ )
+ code = "class {} : public LInstructionHelper<".format(class_name)
+ if result_type:
+ code += result_types[result_type] + ", "
+ else:
+ code += "0, "
+ code += gen_helper_template_value(
+ num_reg_operands, num_value_operands, num_int64_operands
+ )
+ code += ", {}> {{\\\n".format(num_temps)
+ if arguments:
+ for arg_name in arguments:
+ arg_type_sig = arguments[arg_name]
+ constructor_params.append(arg_type_sig + " " + arg_name)
+ code += " " + arg_type_sig + " " + arg_name + "_;\\\n"
+ code += " public:\\\n LIR_HEADER({})\\\n".format(name)
+ code += " explicit {}(".format(class_name)
+ code += ", ".join(constructor_params)
+ code += ") : LInstructionHelper(classOpcode)"
+ if arguments:
+ for arg_name in arguments:
+ code += ", " + arg_name + "_(" + arg_name + ")"
+ code += " {"
+ if call_instruction:
+ code += "\\\n this->setIsCall();"
+ code += "\\\n"
+ code += "\\\n".join(setters)
+ code += "\\\n }\\\n"
+ code += "\\\n".join(getters)
+ if arguments:
+ for arg_name in arguments:
+ code += " " + arguments[arg_name] + " " + arg_name + "() const { "
+ code += "return " + arg_name + "_; }\\\n"
+ code += "\\\n"
+ if operands:
+ code += "\\\n".join(oper_indices)
+ if mir_op:
+ if mir_op is True:
+ code += " M{}* mir() const {{ return mir_->to{}(); }};\\\n".format(
+ name, name
+ )
+ else:
+ code += " M{}* mir() const {{ return mir_->to{}(); }};\\\n".format(
+ mir_op, mir_op
+ )
+ code += "};\\\n"
+ return code
+
+
+def generate_lir_header(c_out, yaml_path):
+ data = load_yaml(yaml_path)
+
+ # LIR_OPCODE_LIST opcode.
+ ops = []
+
+ # Generated LIR op class definitions.
+ lir_op_classes = []
+
+ for op in data:
+ name = op["name"]
+
+ gen_boilerplate = op.get("gen_boilerplate", True)
+ assert isinstance(gen_boilerplate, bool)
+
+ if gen_boilerplate:
+ result_type = op.get("result_type", None)
+ assert result_type is None or str
+ if result_type:
+ assert result_types[result_type]
+
+ operands = op.get("operands", None)
+ assert operands is None or OrderedDict
+
+ arguments = op.get("arguments", None)
+ assert arguments is None or isinstance(arguments, OrderedDict)
+
+ num_temps = op.get("num_temps", 0)
+ assert num_temps is None or int
+
+ gen_boilerplate = op.get("gen_boilerplate", True)
+ assert isinstance(gen_boilerplate, bool)
+
+ call_instruction = op.get("call_instruction", None)
+ assert call_instruction is None or True
+
+ mir_op = op.get("mir_op", None)
+ assert mir_op is None or True or str
+
+ lir_op_classes.append(
+ gen_lir_class(
+ name,
+ result_type,
+ operands,
+ arguments,
+ num_temps,
+ call_instruction,
+ mir_op,
+ )
+ )
+
+ ops.append("_({})".format(name))
+
+ contents = "#define LIR_OPCODE_LIST(_)\\\n"
+ contents += "\\\n".join(ops)
+ contents += "\n\n"
+
+ contents += "#define LIR_OPCODE_CLASS_GENERATED \\\n"
+ contents += "\\\n".join(lir_op_classes)
+ contents += "\n\n"
+
+ generate_header(c_out, "jit_LIROpsGenerated_h", contents)
diff --git a/js/src/jit/GenerateMIRFiles.py b/js/src/jit/GenerateMIRFiles.py
new file mode 100644
index 0000000000..faaa82cdcf
--- /dev/null
+++ b/js/src/jit/GenerateMIRFiles.py
@@ -0,0 +1,403 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script generates jit/MIROpsGenerated.h (list of MIR instructions)
+# from MIROps.yaml, as well as MIR op definitions.
+
+from collections import OrderedDict
+
+import buildconfig
+import six
+import yaml
+from mozbuild.preprocessor import Preprocessor
+
+HEADER_TEMPLATE = """\
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef %(includeguard)s
+#define %(includeguard)s
+
+/* This file is generated by jit/GenerateMIRFiles.py. Do not edit! */
+
+%(contents)s
+
+#endif // %(includeguard)s
+"""
+
+
+def generate_header(c_out, includeguard, contents):
+ c_out.write(
+ HEADER_TEMPLATE
+ % {
+ "includeguard": includeguard,
+ "contents": contents,
+ }
+ )
+
+
+def load_yaml(yaml_path):
+ # First invoke preprocessor.py so that we can use #ifdef JS_SIMULATOR in
+ # the YAML file.
+ pp = Preprocessor()
+ pp.context.update(buildconfig.defines["ALLDEFINES"])
+ pp.out = six.StringIO()
+ pp.do_filter("substitution")
+ pp.do_include(yaml_path)
+ contents = pp.out.getvalue()
+
+ # Load into an OrderedDict to ensure order is preserved. Note: Python 3.7+
+ # also preserves ordering for normal dictionaries.
+ # Code based on https://stackoverflow.com/a/21912744.
+ class OrderedLoader(yaml.Loader):
+ pass
+
+ def construct_mapping(loader, node):
+ loader.flatten_mapping(node)
+ return OrderedDict(loader.construct_pairs(node))
+
+ tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
+ OrderedLoader.add_constructor(tag, construct_mapping)
+ return yaml.load(contents, OrderedLoader)
+
+
+type_policies = {
+ "Object": "ObjectPolicy",
+ "Value": "BoxPolicy",
+ "Int32": "UnboxedInt32Policy",
+ "BigInt": "BigIntPolicy",
+ "Double": "DoublePolicy",
+ "String": "StringPolicy",
+ "Symbol": "SymbolPolicy",
+}
+
+
+def decide_type_policy(types, no_type_policy):
+ if no_type_policy:
+ return "public NoTypePolicy::Data"
+
+ if len(types) == 1:
+ return "public {}<0>::Data".format(type_policies[types[0]])
+
+ type_num = 0
+ mixed_type_policies = []
+ for mir_type in types:
+ policy = type_policies[mir_type]
+ mixed_type_policies.append("{}<{}>".format(policy, type_num))
+ type_num += 1
+
+ return "public MixPolicy<{}>::Data".format(", ".join(mixed_type_policies))
+
+
+mir_base_class = [
+ "MNullaryInstruction",
+ "MUnaryInstruction",
+ "MBinaryInstruction",
+ "MTernaryInstruction",
+ "MQuaternaryInstruction",
+]
+
+
+gc_pointer_types = [
+ "JSObject*",
+ "NativeObject*",
+ "JSFunction*",
+ "BaseScript*",
+ "PropertyName*",
+ "Shape*",
+ "GetterSetter*",
+ "JSAtom*",
+ "ClassBodyScope*",
+ "VarScope*",
+ "NamedLambdaObject*",
+ "RegExpObject*",
+ "JSScript*",
+ "LexicalScope*",
+]
+
+
+def gen_mir_class(
+ name,
+ operands,
+ arguments,
+ no_type_policy,
+ result,
+ guard,
+ movable,
+ folds_to,
+ congruent_to,
+ alias_set,
+ might_alias,
+ possibly_calls,
+ compute_range,
+ can_recover,
+ clone,
+):
+ """Generates class definition for a single MIR opcode."""
+
+ # Generate a MIR opcode class definition.
+ # For example:
+ # class MGuardIndexIsValidUpdateOrAdd
+ # : public MBinaryInstruction,
+ # public MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<1>>::Data {
+ # explicit MGuardIndexIsValidUpdateOrAdd(MDefinition* object,
+ # MDefinition* index)
+ # : MBinaryInstruction(classOpcode, object, index) {
+ # setGuard();
+ # setMovable();
+ # setResultType(MIRType::Int32);
+ # }
+ # public:
+ # INSTRUCTION_HEADER(GetFrameArgument)
+ # TRIVIAL_NEW_WRAPPERS
+ # NAMED_OPERANDS((0, object), (1, index))
+ # AliasSet getAliasSet() const override { return AliasSet::None(); }
+ # bool congruentTo(const MDefinition* ins) const override {
+ # return congruentIfOperandsEqual(ins); }
+ # };
+ #
+
+ type_policy = ""
+ # MIR op constructor operands.
+ mir_operands = []
+ # MIR op base class constructor operands.
+ mir_base_class_operands = []
+ # Types of each constructor operand.
+ mir_types = []
+ # Items for NAMED_OPERANDS.
+ named_operands = []
+ if operands:
+ current_oper_num = 0
+ for oper_name in operands:
+ oper = "MDefinition* " + oper_name
+ mir_operands.append(oper)
+ mir_base_class_operands.append(", " + oper_name)
+ # Collect all the MIR argument types to use for determining the
+ # ops type policy.
+ mir_types.append(operands[oper_name])
+ # Collecting named operands for defining accessors.
+ named_operands.append("({}, {})".format(current_oper_num, oper_name))
+ current_oper_num += 1
+ type_policy = decide_type_policy(mir_types, no_type_policy)
+
+ class_name = "M" + name
+
+ assert len(mir_operands) < 5
+ base_class = mir_base_class[len(mir_operands)]
+ assert base_class
+ if base_class != "MNullaryInstruction":
+ assert type_policy
+ type_policy = ", " + type_policy
+ code = "class {} : public {}{} {{\\\n".format(class_name, base_class, type_policy)
+
+ # Arguments to class constructor that require accessors.
+ mir_args = []
+ if arguments:
+ for arg_name in arguments:
+ arg_type_sig = arguments[arg_name]
+ mir_args.append(arg_type_sig + " " + arg_name)
+ if arg_type_sig in gc_pointer_types:
+ code += " CompilerGCPointer<" + arg_type_sig + ">"
+ else:
+ code += " " + arg_type_sig
+ code += " " + arg_name + "_;\\\n"
+
+ code += " explicit {}({}) : {}(classOpcode{})".format(
+ class_name,
+ ", ".join(mir_operands + mir_args),
+ base_class,
+ "".join(mir_base_class_operands),
+ )
+ if arguments:
+ for arg_name in arguments:
+ code += ", " + arg_name + "_(" + arg_name + ")"
+ code += " {\\\n"
+ if guard:
+ code += " setGuard();\\\n"
+ if movable:
+ code += " setMovable();\\\n"
+ if result:
+ code += " setResultType(MIRType::{});\\\n".format(result)
+ code += " }\\\n public:\\\n"
+ if arguments:
+ for arg_name in arguments:
+ code += " " + arguments[arg_name] + " " + arg_name + "() const { "
+ code += "return " + arg_name + "_; }\\\n"
+ code += " INSTRUCTION_HEADER({})\\\n".format(name)
+ code += " TRIVIAL_NEW_WRAPPERS\\\n"
+ if named_operands:
+ code += " NAMED_OPERANDS({})\\\n".format(", ".join(named_operands))
+ if alias_set:
+ if alias_set == "custom":
+ code += " AliasSet getAliasSet() const override;\\\n"
+ else:
+ assert alias_set == "none"
+ code += (
+ " AliasSet getAliasSet() const override { "
+ "return AliasSet::None(); }\\\n"
+ )
+ if might_alias:
+ code += " AliasType mightAlias(const MDefinition* store) const override;\\\n"
+ if folds_to:
+ code += " MDefinition* foldsTo(TempAllocator& alloc) override;\\\n"
+ if congruent_to:
+ if congruent_to == "custom":
+ code += " bool congruentTo(const MDefinition* ins) const override;\\\n"
+ else:
+ assert congruent_to == "if_operands_equal"
+ code += (
+ " bool congruentTo(const MDefinition* ins) const override { "
+ "return congruentIfOperandsEqual(ins); }\\\n"
+ )
+ if possibly_calls:
+ if possibly_calls == "custom":
+ code += " bool possiblyCalls() const override;\\\n"
+ else:
+ code += " bool possiblyCalls() const override { return true; }\\\n"
+ if compute_range:
+ code += " void computeRange(TempAllocator& alloc) override;\\\n"
+ if can_recover:
+ code += " [[nodiscard]] bool writeRecoverData(\\\n"
+ code += " CompactBufferWriter& writer) const override;\\\n"
+ if can_recover == "custom":
+ code += " bool canRecoverOnBailout() const override;\\\n"
+ else:
+ code += " bool canRecoverOnBailout() const override { return true; }\\\n"
+ if clone:
+ code += " ALLOW_CLONE(" + class_name + ")\\\n"
+ code += "};\\\n"
+ return code
+
+
+def gen_non_gc_pointer_type_assertions(seen_types):
+ """Generates a list of static assertions used to ensure that all argument
+ types seen are not derived from gc::Cell, ensuring that gc pointer arguments
+ are added to the gc_pointer_types list.
+ """
+ assertions = []
+
+ for seen_type in sorted(seen_types):
+ assertions.append(
+ "static_assert(!std::is_base_of_v<gc::Cell, " + seen_type.strip("*") + ">, "
+ '"Ensure that '
+ + seen_type.strip("*")
+ + ' is added to the gc_pointer_types list in GenerateMIRFiles.py."'
+ ");"
+ )
+
+ return assertions
+
+
+def generate_mir_header(c_out, yaml_path):
+ """Generate MIROpsGenerated.h from MIROps.yaml. The generated file
+ has a list of MIR ops and boilerplate for MIR op definitions.
+ """
+
+ data = load_yaml(yaml_path)
+
+ # MIR_OPCODE_LIST items. Stores the name of each MIR op.
+ ops_items = []
+
+ # Generated MIR op class definitions.
+ mir_op_classes = []
+
+ # Unique and non gc pointer types seen for arguments to the MIR constructor.
+ seen_non_gc_pointer_argument_types = set()
+
+ for op in data:
+ name = op["name"]
+
+ ops_items.append("_({})".format(name))
+
+ gen_boilerplate = op.get("gen_boilerplate", True)
+ assert isinstance(gen_boilerplate, bool)
+
+ if gen_boilerplate:
+ operands = op.get("operands", None)
+ assert operands is None or isinstance(operands, OrderedDict)
+
+ arguments = op.get("arguments", None)
+ assert arguments is None or isinstance(arguments, OrderedDict)
+
+ no_type_policy = op.get("type_policy", None)
+ assert no_type_policy is None or no_type_policy == "none"
+
+ result = op.get("result_type", None)
+ assert result is None or isinstance(result, str)
+
+ guard = op.get("guard", None)
+ assert guard is None or True
+
+ movable = op.get("movable", None)
+ assert movable is None or True
+
+ folds_to = op.get("folds_to", None)
+ assert folds_to is None or folds_to == "custom"
+
+ congruent_to = op.get("congruent_to", None)
+ assert (
+ congruent_to is None
+ or congruent_to == "if_operands_equal"
+ or congruent_to == "custom"
+ )
+
+ alias_set = op.get("alias_set", None)
+ assert alias_set is None or True or isinstance(alias_set, str)
+
+ might_alias = op.get("might_alias", None)
+ assert might_alias is None or might_alias == "custom"
+
+ possibly_calls = op.get("possibly_calls", None)
+ assert possibly_calls is None or True or possibly_calls == "custom"
+
+ compute_range = op.get("compute_range", None)
+ assert compute_range is None or compute_range == "custom"
+
+ can_recover = op.get("can_recover", None)
+ assert can_recover is None or True or can_recover == "custom"
+
+ clone = op.get("clone", None)
+ assert clone is None or True
+
+ code = gen_mir_class(
+ name,
+ operands,
+ arguments,
+ no_type_policy,
+ result,
+ guard,
+ movable,
+ folds_to,
+ congruent_to,
+ alias_set,
+ might_alias,
+ possibly_calls,
+ compute_range,
+ can_recover,
+ clone,
+ )
+ mir_op_classes.append(code)
+
+ if arguments:
+ for argument in arguments:
+ arg_type = arguments[argument]
+ if arg_type not in gc_pointer_types:
+ seen_non_gc_pointer_argument_types.add(arg_type)
+
+ contents = "#define MIR_OPCODE_LIST(_)\\\n"
+ contents += "\\\n".join(ops_items)
+ contents += "\n\n"
+
+ contents += "#define MIR_OPCODE_CLASS_GENERATED \\\n"
+ contents += "\\\n".join(mir_op_classes)
+ contents += "\n\n"
+
+ contents += "#define NON_GC_POINTER_TYPE_ASSERTIONS_GENERATED \\\n"
+ contents += "\\\n".join(
+ gen_non_gc_pointer_type_assertions(seen_non_gc_pointer_argument_types)
+ )
+ contents += "\n\n"
+
+ generate_header(c_out, "jit_MIROpsGenerated_h", contents)
diff --git a/js/src/jit/ICState.h b/js/src/jit/ICState.h
new file mode 100644
index 0000000000..83b7865955
--- /dev/null
+++ b/js/src/jit/ICState.h
@@ -0,0 +1,212 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ICState_h
+#define jit_ICState_h
+
+#include "jit/JitOptions.h"
+
+namespace js {
+namespace jit {
+
+// Used to track trial inlining status for a Baseline IC.
+// See also setTrialInliningState below.
+enum class TrialInliningState : uint8_t {
+ Initial = 0,
+ Candidate,
+ Inlined,
+ Failure,
+};
+
+// ICState stores information about a Baseline or Ion IC.
+class ICState {
+ public:
+ // When we attach the maximum number of stubs, we discard all stubs and
+ // transition the IC to Megamorphic to attach stubs that are more generic
+ // (handle more cases). If we again attach the maximum number of stubs, we
+ // transition to Generic and (depending on the IC) will either attach a
+ // single stub that handles everything or stop attaching new stubs.
+ //
+ // We also transition to Generic when we repeatedly fail to attach a stub,
+ // to avoid wasting time trying.
+ enum class Mode : uint8_t { Specialized = 0, Megamorphic, Generic };
+
+ private:
+ uint8_t mode_ : 2;
+
+ // The TrialInliningState for a Baseline IC.
+ uint8_t trialInliningState_ : 2;
+
+ // Whether WarpOracle created a snapshot based on stubs attached to this
+ // Baseline IC.
+ bool usedByTranspiler_ : 1;
+
+ // Whether stubs attached to this IC have been folded together into
+ // a single stub. Used as a hint when attaching additional stubs to
+ // try folding them too.
+ bool hasFoldedStub_ : 1;
+
+ // Number of optimized stubs currently attached to this IC.
+ uint8_t numOptimizedStubs_;
+
+ // Number of times we failed to attach a stub.
+ uint8_t numFailures_;
+
+ static const size_t MaxOptimizedStubs = 6;
+
+ void setMode(Mode mode) {
+ mode_ = uint32_t(mode);
+ MOZ_ASSERT(Mode(mode_) == mode, "mode must fit in bitfield");
+ }
+
+ void transition(Mode mode) {
+ MOZ_ASSERT(mode > this->mode());
+ setMode(mode);
+ numFailures_ = 0;
+ }
+
+ MOZ_ALWAYS_INLINE size_t maxFailures() const {
+ // Allow more failures if we attached stubs.
+ static_assert(MaxOptimizedStubs == 6,
+ "numFailures_/maxFailures should fit in uint8_t");
+ size_t res = 5 + size_t(40) * numOptimizedStubs_;
+ MOZ_ASSERT(res <= UINT8_MAX, "numFailures_ should not overflow");
+ return res;
+ }
+
+ public:
+ ICState() { reset(); }
+
+ Mode mode() const { return Mode(mode_); }
+ size_t numOptimizedStubs() const { return numOptimizedStubs_; }
+ bool hasFailures() const { return (numFailures_ != 0); }
+ bool newStubIsFirstStub() const {
+ return (mode() == Mode::Specialized && numOptimizedStubs() == 0);
+ }
+
+ MOZ_ALWAYS_INLINE bool canAttachStub() const {
+ // Note: we cannot assert that numOptimizedStubs_ <= MaxOptimizedStubs
+ // because old-style baseline ICs may attach more stubs than
+ // MaxOptimizedStubs allows.
+ if (mode() == Mode::Generic || JitOptions.disableCacheIR) {
+ return false;
+ }
+ return true;
+ }
+
+ [[nodiscard]] MOZ_ALWAYS_INLINE bool shouldTransition() {
+ // Note: we cannot assert that numOptimizedStubs_ <= MaxOptimizedStubs
+ // because old-style baseline ICs may attach more stubs than
+ // MaxOptimizedStubs allows.
+ if (mode() == Mode::Generic) {
+ return false;
+ }
+ if (numOptimizedStubs_ < MaxOptimizedStubs &&
+ numFailures_ < maxFailures()) {
+ return false;
+ }
+ return true;
+ }
+
+ // If this returns true, we transitioned to a new mode and the caller
+ // should discard all stubs.
+ [[nodiscard]] MOZ_ALWAYS_INLINE bool maybeTransition() {
+ if (!shouldTransition()) {
+ return false;
+ }
+ if (numFailures_ == maxFailures() || mode() == Mode::Megamorphic) {
+ transition(Mode::Generic);
+ return true;
+ }
+ MOZ_ASSERT(mode() == Mode::Specialized);
+ transition(Mode::Megamorphic);
+ return true;
+ }
+
+ void reset() {
+ setMode(Mode::Specialized);
+#ifdef DEBUG
+ if (JitOptions.forceMegamorphicICs) {
+ setMode(Mode::Megamorphic);
+ }
+#endif
+ trialInliningState_ = uint32_t(TrialInliningState::Initial);
+ usedByTranspiler_ = false;
+ hasFoldedStub_ = false;
+ numOptimizedStubs_ = 0;
+ numFailures_ = 0;
+ }
+ void trackAttached() {
+ // We'd like to assert numOptimizedStubs_ < MaxOptimizedStubs, but
+ // since this code is also used for non-CacheIR Baseline stubs, assert
+ // < 16 for now. Note that we do have the stronger assert in other
+ // methods, because they are only used by CacheIR ICs.
+ MOZ_ASSERT(numOptimizedStubs_ < 16);
+ numOptimizedStubs_++;
+ // As a heuristic, reduce the failure count after each successful attach
+ // to delay hitting Generic mode. Reset to 1 instead of 0 so that
+ // code which inspects state can distinguish no-failures from rare-failures.
+ numFailures_ = std::min(numFailures_, static_cast<uint8_t>(1));
+ }
+ void trackNotAttached() {
+ // Note: we can't assert numFailures_ < maxFailures() because
+ // maxFailures() depends on numOptimizedStubs_ and it's possible a
+ // GC discarded stubs before we got here.
+ numFailures_++;
+ MOZ_ASSERT(numFailures_ > 0, "numFailures_ should not overflow");
+ }
+ void trackUnlinkedStub() {
+ MOZ_ASSERT(numOptimizedStubs_ > 0);
+ numOptimizedStubs_--;
+ }
+ void trackUnlinkedAllStubs() { numOptimizedStubs_ = 0; }
+
+ void clearUsedByTranspiler() { usedByTranspiler_ = false; }
+ void setUsedByTranspiler() { usedByTranspiler_ = true; }
+ bool usedByTranspiler() const { return usedByTranspiler_; }
+
+ void clearHasFoldedStub() { hasFoldedStub_ = false; }
+ void setHasFoldedStub() { hasFoldedStub_ = true; }
+ bool hasFoldedStub() const { return hasFoldedStub_; }
+
+ TrialInliningState trialInliningState() const {
+ return TrialInliningState(trialInliningState_);
+ }
+ void setTrialInliningState(TrialInliningState state) {
+#ifdef DEBUG
+ // Moving to the Failure state is always valid. The other states should
+ // happen in this order:
+ //
+ // Initial -> Candidate -> Inlined
+ //
+ // This ensures we perform trial inlining at most once per IC site.
+ if (state != TrialInliningState::Failure) {
+ switch (trialInliningState()) {
+ case TrialInliningState::Initial:
+ MOZ_ASSERT(state == TrialInliningState::Candidate);
+ break;
+ case TrialInliningState::Candidate:
+ MOZ_ASSERT(state == TrialInliningState::Candidate ||
+ state == TrialInliningState::Inlined);
+ break;
+ case TrialInliningState::Inlined:
+ case TrialInliningState::Failure:
+ MOZ_CRASH("Inlined and Failure can only change to Failure");
+ break;
+ }
+ }
+#endif
+
+ trialInliningState_ = uint32_t(state);
+ MOZ_ASSERT(trialInliningState() == state,
+ "TrialInliningState must fit in bitfield");
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ICState_h */
diff --git a/js/src/jit/ICStubSpace.h b/js/src/jit/ICStubSpace.h
new file mode 100644
index 0000000000..cf922231ca
--- /dev/null
+++ b/js/src/jit/ICStubSpace.h
@@ -0,0 +1,66 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ICStubSpace_h
+#define jit_ICStubSpace_h
+
+#include "ds/LifoAlloc.h"
+
+namespace js {
+namespace jit {
+
+// ICStubSpace is an abstraction for allocation policy and storage for CacheIR
+// stub data. There are two kinds of Baseline CacheIR stubs:
+//
+// (1) CacheIR stubs that can make non-tail calls that can GC. These are
+// allocated in a LifoAlloc stored in JitScript.
+// See JitScriptICStubSpace.
+//
+// (2) Other CacheIR stubs (aka optimized IC stubs). Allocated in a per-Zone
+// LifoAlloc and purged when JIT-code is discarded.
+// See OptimizedICStubSpace.
+class ICStubSpace {
+ protected:
+ LifoAlloc allocator_;
+
+ explicit ICStubSpace(size_t chunkSize) : allocator_(chunkSize) {}
+
+ public:
+ inline void* alloc(size_t size) { return allocator_.alloc(size); }
+
+ JS_DECLARE_NEW_METHODS(allocate, alloc, inline)
+
+ void freeAllAfterMinorGC(JS::Zone* zone);
+
+#ifdef DEBUG
+ bool isEmpty() const { return allocator_.isEmpty(); }
+#endif
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ return allocator_.sizeOfExcludingThis(mallocSizeOf);
+ }
+};
+
+// Space for optimized stubs. Every JitZone has a single OptimizedICStubSpace.
+struct OptimizedICStubSpace : public ICStubSpace {
+ static const size_t STUB_DEFAULT_CHUNK_SIZE = 4096;
+
+ public:
+ OptimizedICStubSpace() : ICStubSpace(STUB_DEFAULT_CHUNK_SIZE) {}
+};
+
+// Space for Can-GC stubs. Every JitScript has a JitScriptICStubSpace.
+struct JitScriptICStubSpace : public ICStubSpace {
+ static const size_t STUB_DEFAULT_CHUNK_SIZE = 4096;
+
+ public:
+ JitScriptICStubSpace() : ICStubSpace(STUB_DEFAULT_CHUNK_SIZE) {}
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ICStubSpace_h */
diff --git a/js/src/jit/InlinableNatives.cpp b/js/src/jit/InlinableNatives.cpp
new file mode 100644
index 0000000000..255265dd6f
--- /dev/null
+++ b/js/src/jit/InlinableNatives.cpp
@@ -0,0 +1,297 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/InlinableNatives.h"
+
+#ifdef JS_HAS_INTL_API
+# include "builtin/intl/Collator.h"
+# include "builtin/intl/DateTimeFormat.h"
+# include "builtin/intl/DisplayNames.h"
+# include "builtin/intl/ListFormat.h"
+# include "builtin/intl/NumberFormat.h"
+# include "builtin/intl/PluralRules.h"
+# include "builtin/intl/RelativeTimeFormat.h"
+#endif
+#include "builtin/MapObject.h"
+#include "js/experimental/JitInfo.h"
+#include "vm/ArrayBufferObject.h"
+#include "vm/AsyncIteration.h"
+#include "vm/Iteration.h"
+#include "vm/SharedArrayObject.h"
+
+using namespace js;
+using namespace js::jit;
+
+#define ADD_NATIVE(native) \
+ const JSJitInfo js::jit::JitInfo_##native{ \
+ {nullptr}, \
+ {uint16_t(InlinableNative::native)}, \
+ {0}, \
+ JSJitInfo::InlinableNative};
+INLINABLE_NATIVE_LIST(ADD_NATIVE)
+#undef ADD_NATIVE
+
+const JSClass* js::jit::InlinableNativeGuardToClass(InlinableNative native) {
+ switch (native) {
+#ifdef JS_HAS_INTL_API
+ // Intl natives.
+ case InlinableNative::IntlGuardToCollator:
+ return &CollatorObject::class_;
+ case InlinableNative::IntlGuardToDateTimeFormat:
+ return &DateTimeFormatObject::class_;
+ case InlinableNative::IntlGuardToDisplayNames:
+ return &DisplayNamesObject::class_;
+ case InlinableNative::IntlGuardToListFormat:
+ return &ListFormatObject::class_;
+ case InlinableNative::IntlGuardToNumberFormat:
+ return &NumberFormatObject::class_;
+ case InlinableNative::IntlGuardToPluralRules:
+ return &PluralRulesObject::class_;
+ case InlinableNative::IntlGuardToRelativeTimeFormat:
+ return &RelativeTimeFormatObject::class_;
+#else
+ case InlinableNative::IntlGuardToCollator:
+ case InlinableNative::IntlGuardToDateTimeFormat:
+ case InlinableNative::IntlGuardToDisplayNames:
+ case InlinableNative::IntlGuardToListFormat:
+ case InlinableNative::IntlGuardToNumberFormat:
+ case InlinableNative::IntlGuardToPluralRules:
+ case InlinableNative::IntlGuardToRelativeTimeFormat:
+ MOZ_CRASH("Intl API disabled");
+#endif
+
+ // Utility intrinsics.
+ case InlinableNative::IntrinsicGuardToArrayIterator:
+ return &ArrayIteratorObject::class_;
+ case InlinableNative::IntrinsicGuardToMapIterator:
+ return &MapIteratorObject::class_;
+ case InlinableNative::IntrinsicGuardToSetIterator:
+ return &SetIteratorObject::class_;
+ case InlinableNative::IntrinsicGuardToStringIterator:
+ return &StringIteratorObject::class_;
+ case InlinableNative::IntrinsicGuardToRegExpStringIterator:
+ return &RegExpStringIteratorObject::class_;
+ case InlinableNative::IntrinsicGuardToWrapForValidIterator:
+ return &WrapForValidIteratorObject::class_;
+ case InlinableNative::IntrinsicGuardToIteratorHelper:
+ return &IteratorHelperObject::class_;
+ case InlinableNative::IntrinsicGuardToAsyncIteratorHelper:
+ return &AsyncIteratorHelperObject::class_;
+
+ case InlinableNative::IntrinsicGuardToMapObject:
+ return &MapObject::class_;
+ case InlinableNative::IntrinsicGuardToSetObject:
+ return &SetObject::class_;
+ case InlinableNative::IntrinsicGuardToArrayBuffer:
+ return &ArrayBufferObject::class_;
+ case InlinableNative::IntrinsicGuardToSharedArrayBuffer:
+ return &SharedArrayBufferObject::class_;
+
+ default:
+ MOZ_CRASH("Not a GuardTo instruction");
+ }
+}
+
+// Returns true if |native| can be inlined cross-realm. Especially inlined
+// natives that can allocate objects or throw exceptions shouldn't be inlined
+// cross-realm without a careful analysis because we might use the wrong realm!
+//
+// Note that self-hosting intrinsics are never called cross-realm. See the
+// MOZ_CRASH below.
+//
+// If you are adding a new inlinable native, the safe thing is to |return false|
+// here.
+bool js::jit::CanInlineNativeCrossRealm(InlinableNative native) {
+ switch (native) {
+ case InlinableNative::MathAbs:
+ case InlinableNative::MathFloor:
+ case InlinableNative::MathCeil:
+ case InlinableNative::MathRound:
+ case InlinableNative::MathClz32:
+ case InlinableNative::MathSqrt:
+ case InlinableNative::MathATan2:
+ case InlinableNative::MathHypot:
+ case InlinableNative::MathMax:
+ case InlinableNative::MathMin:
+ case InlinableNative::MathPow:
+ case InlinableNative::MathImul:
+ case InlinableNative::MathFRound:
+ case InlinableNative::MathTrunc:
+ case InlinableNative::MathSign:
+ case InlinableNative::MathSin:
+ case InlinableNative::MathTan:
+ case InlinableNative::MathCos:
+ case InlinableNative::MathExp:
+ case InlinableNative::MathLog:
+ case InlinableNative::MathASin:
+ case InlinableNative::MathATan:
+ case InlinableNative::MathACos:
+ case InlinableNative::MathLog10:
+ case InlinableNative::MathLog2:
+ case InlinableNative::MathLog1P:
+ case InlinableNative::MathExpM1:
+ case InlinableNative::MathCosH:
+ case InlinableNative::MathSinH:
+ case InlinableNative::MathTanH:
+ case InlinableNative::MathACosH:
+ case InlinableNative::MathASinH:
+ case InlinableNative::MathATanH:
+ case InlinableNative::MathCbrt:
+ case InlinableNative::Boolean:
+ return true;
+
+ case InlinableNative::Array:
+ // Cross-realm case handled by inlineArray.
+ return true;
+
+ case InlinableNative::MathRandom:
+ // RNG state is per-realm.
+ return false;
+
+ case InlinableNative::IntlGuardToCollator:
+ case InlinableNative::IntlGuardToDateTimeFormat:
+ case InlinableNative::IntlGuardToDisplayNames:
+ case InlinableNative::IntlGuardToListFormat:
+ case InlinableNative::IntlGuardToNumberFormat:
+ case InlinableNative::IntlGuardToPluralRules:
+ case InlinableNative::IntlGuardToRelativeTimeFormat:
+ case InlinableNative::IsRegExpObject:
+ case InlinableNative::IsPossiblyWrappedRegExpObject:
+ case InlinableNative::RegExpMatcher:
+ case InlinableNative::RegExpSearcher:
+ case InlinableNative::RegExpTester:
+ case InlinableNative::RegExpPrototypeOptimizable:
+ case InlinableNative::RegExpInstanceOptimizable:
+ case InlinableNative::GetFirstDollarIndex:
+ case InlinableNative::IntrinsicNewArrayIterator:
+ case InlinableNative::IntrinsicNewStringIterator:
+ case InlinableNative::IntrinsicNewRegExpStringIterator:
+ case InlinableNative::IntrinsicStringReplaceString:
+ case InlinableNative::IntrinsicStringSplitString:
+ case InlinableNative::IntrinsicUnsafeSetReservedSlot:
+ case InlinableNative::IntrinsicUnsafeGetReservedSlot:
+ case InlinableNative::IntrinsicUnsafeGetObjectFromReservedSlot:
+ case InlinableNative::IntrinsicUnsafeGetInt32FromReservedSlot:
+ case InlinableNative::IntrinsicUnsafeGetStringFromReservedSlot:
+ case InlinableNative::IntrinsicIsCallable:
+ case InlinableNative::IntrinsicIsConstructor:
+ case InlinableNative::IntrinsicToObject:
+ case InlinableNative::IntrinsicIsObject:
+ case InlinableNative::IntrinsicIsCrossRealmArrayConstructor:
+ case InlinableNative::IntrinsicToInteger:
+ case InlinableNative::IntrinsicToLength:
+ case InlinableNative::IntrinsicIsConstructing:
+ case InlinableNative::IntrinsicIsSuspendedGenerator:
+ case InlinableNative::IntrinsicSubstringKernel:
+ case InlinableNative::IntrinsicGuardToArrayIterator:
+ case InlinableNative::IntrinsicGuardToMapIterator:
+ case InlinableNative::IntrinsicGuardToSetIterator:
+ case InlinableNative::IntrinsicGuardToStringIterator:
+ case InlinableNative::IntrinsicGuardToRegExpStringIterator:
+ case InlinableNative::IntrinsicGuardToWrapForValidIterator:
+ case InlinableNative::IntrinsicGuardToIteratorHelper:
+ case InlinableNative::IntrinsicGuardToAsyncIteratorHelper:
+ case InlinableNative::IntrinsicObjectHasPrototype:
+ case InlinableNative::IntrinsicFinishBoundFunctionInit:
+ case InlinableNative::IntrinsicIsPackedArray:
+ case InlinableNative::IntrinsicGuardToMapObject:
+ case InlinableNative::IntrinsicGetNextMapEntryForIterator:
+ case InlinableNative::IntrinsicGuardToSetObject:
+ case InlinableNative::IntrinsicGetNextSetEntryForIterator:
+ case InlinableNative::IntrinsicGuardToArrayBuffer:
+ case InlinableNative::IntrinsicArrayBufferByteLength:
+ case InlinableNative::IntrinsicPossiblyWrappedArrayBufferByteLength:
+ case InlinableNative::IntrinsicGuardToSharedArrayBuffer:
+ case InlinableNative::IntrinsicIsTypedArrayConstructor:
+ case InlinableNative::IntrinsicIsTypedArray:
+ case InlinableNative::IntrinsicIsPossiblyWrappedTypedArray:
+ case InlinableNative::IntrinsicPossiblyWrappedTypedArrayLength:
+ case InlinableNative::IntrinsicTypedArrayLength:
+ case InlinableNative::IntrinsicTypedArrayByteOffset:
+ case InlinableNative::IntrinsicTypedArrayElementSize:
+ case InlinableNative::IntrinsicArrayIteratorPrototypeOptimizable:
+ MOZ_CRASH("Unexpected cross-realm intrinsic call");
+
+ case InlinableNative::TestBailout:
+ case InlinableNative::TestAssertFloat32:
+ case InlinableNative::TestAssertRecoveredOnBailout:
+ // Testing functions, not worth inlining cross-realm.
+ return false;
+
+ case InlinableNative::ArrayIsArray:
+ case InlinableNative::ArrayJoin:
+ case InlinableNative::ArrayPop:
+ case InlinableNative::ArrayShift:
+ case InlinableNative::ArrayPush:
+ case InlinableNative::ArraySlice:
+ case InlinableNative::AtomicsCompareExchange:
+ case InlinableNative::AtomicsExchange:
+ case InlinableNative::AtomicsLoad:
+ case InlinableNative::AtomicsStore:
+ case InlinableNative::AtomicsAdd:
+ case InlinableNative::AtomicsSub:
+ case InlinableNative::AtomicsAnd:
+ case InlinableNative::AtomicsOr:
+ case InlinableNative::AtomicsXor:
+ case InlinableNative::AtomicsIsLockFree:
+ case InlinableNative::BigIntAsIntN:
+ case InlinableNative::BigIntAsUintN:
+ case InlinableNative::DataViewGetInt8:
+ case InlinableNative::DataViewGetUint8:
+ case InlinableNative::DataViewGetInt16:
+ case InlinableNative::DataViewGetUint16:
+ case InlinableNative::DataViewGetInt32:
+ case InlinableNative::DataViewGetUint32:
+ case InlinableNative::DataViewGetFloat32:
+ case InlinableNative::DataViewGetFloat64:
+ case InlinableNative::DataViewGetBigInt64:
+ case InlinableNative::DataViewGetBigUint64:
+ case InlinableNative::DataViewSetInt8:
+ case InlinableNative::DataViewSetUint8:
+ case InlinableNative::DataViewSetInt16:
+ case InlinableNative::DataViewSetUint16:
+ case InlinableNative::DataViewSetInt32:
+ case InlinableNative::DataViewSetUint32:
+ case InlinableNative::DataViewSetFloat32:
+ case InlinableNative::DataViewSetFloat64:
+ case InlinableNative::DataViewSetBigInt64:
+ case InlinableNative::DataViewSetBigUint64:
+ case InlinableNative::MapGet:
+ case InlinableNative::MapHas:
+ case InlinableNative::Number:
+ case InlinableNative::NumberParseInt:
+ case InlinableNative::NumberToString:
+ case InlinableNative::ReflectGetPrototypeOf:
+ case InlinableNative::SetHas:
+ case InlinableNative::String:
+ case InlinableNative::StringToString:
+ case InlinableNative::StringValueOf:
+ case InlinableNative::StringCharCodeAt:
+ case InlinableNative::StringFromCharCode:
+ case InlinableNative::StringFromCodePoint:
+ case InlinableNative::StringCharAt:
+ case InlinableNative::StringIndexOf:
+ case InlinableNative::StringStartsWith:
+ case InlinableNative::StringEndsWith:
+ case InlinableNative::StringToLowerCase:
+ case InlinableNative::StringToUpperCase:
+ case InlinableNative::Object:
+ case InlinableNative::ObjectCreate:
+ case InlinableNative::ObjectIs:
+ case InlinableNative::ObjectIsPrototypeOf:
+ case InlinableNative::ObjectToString:
+ case InlinableNative::TypedArrayConstructor:
+#ifdef FUZZING_JS_FUZZILLI
+ case InlinableNative::FuzzilliHash:
+#endif
+ // Default to false for most natives.
+ return false;
+
+ case InlinableNative::Limit:
+ break;
+ }
+ MOZ_CRASH("Unknown native");
+}
diff --git a/js/src/jit/InlinableNatives.h b/js/src/jit/InlinableNatives.h
new file mode 100644
index 0000000000..7aa176a675
--- /dev/null
+++ b/js/src/jit/InlinableNatives.h
@@ -0,0 +1,236 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_InlinableNatives_h
+#define jit_InlinableNatives_h
+
+#include <stdint.h> // For uint16_t
+
+#ifdef FUZZING_JS_FUZZILLI
+# define INLINABLE_NATIVE_FUZZILLI_LIST(_) _(FuzzilliHash)
+#else
+# define INLINABLE_NATIVE_FUZZILLI_LIST(_)
+#endif
+
+#define INLINABLE_NATIVE_LIST(_) \
+ _(Array) \
+ _(ArrayIsArray) \
+ _(ArrayJoin) \
+ _(ArrayPop) \
+ _(ArrayShift) \
+ _(ArrayPush) \
+ _(ArraySlice) \
+ \
+ _(AtomicsCompareExchange) \
+ _(AtomicsExchange) \
+ _(AtomicsLoad) \
+ _(AtomicsStore) \
+ _(AtomicsAdd) \
+ _(AtomicsSub) \
+ _(AtomicsAnd) \
+ _(AtomicsOr) \
+ _(AtomicsXor) \
+ _(AtomicsIsLockFree) \
+ \
+ _(BigIntAsIntN) \
+ _(BigIntAsUintN) \
+ \
+ _(Boolean) \
+ \
+ _(DataViewGetInt8) \
+ _(DataViewGetUint8) \
+ _(DataViewGetInt16) \
+ _(DataViewGetUint16) \
+ _(DataViewGetInt32) \
+ _(DataViewGetUint32) \
+ _(DataViewGetFloat32) \
+ _(DataViewGetFloat64) \
+ _(DataViewGetBigInt64) \
+ _(DataViewGetBigUint64) \
+ _(DataViewSetInt8) \
+ _(DataViewSetUint8) \
+ _(DataViewSetInt16) \
+ _(DataViewSetUint16) \
+ _(DataViewSetInt32) \
+ _(DataViewSetUint32) \
+ _(DataViewSetFloat32) \
+ _(DataViewSetFloat64) \
+ _(DataViewSetBigInt64) \
+ _(DataViewSetBigUint64) \
+ \
+ _(IntlGuardToCollator) \
+ _(IntlGuardToDateTimeFormat) \
+ _(IntlGuardToDisplayNames) \
+ _(IntlGuardToListFormat) \
+ _(IntlGuardToNumberFormat) \
+ _(IntlGuardToPluralRules) \
+ _(IntlGuardToRelativeTimeFormat) \
+ \
+ _(MapGet) \
+ _(MapHas) \
+ \
+ _(MathAbs) \
+ _(MathFloor) \
+ _(MathCeil) \
+ _(MathRound) \
+ _(MathClz32) \
+ _(MathSqrt) \
+ _(MathATan2) \
+ _(MathHypot) \
+ _(MathMax) \
+ _(MathMin) \
+ _(MathPow) \
+ _(MathRandom) \
+ _(MathImul) \
+ _(MathFRound) \
+ _(MathSin) \
+ _(MathTan) \
+ _(MathCos) \
+ _(MathExp) \
+ _(MathLog) \
+ _(MathASin) \
+ _(MathATan) \
+ _(MathACos) \
+ _(MathLog10) \
+ _(MathLog2) \
+ _(MathLog1P) \
+ _(MathExpM1) \
+ _(MathSinH) \
+ _(MathTanH) \
+ _(MathCosH) \
+ _(MathASinH) \
+ _(MathATanH) \
+ _(MathACosH) \
+ _(MathSign) \
+ _(MathTrunc) \
+ _(MathCbrt) \
+ \
+ _(Number) \
+ _(NumberParseInt) \
+ _(NumberToString) \
+ \
+ _(ReflectGetPrototypeOf) \
+ \
+ _(RegExpMatcher) \
+ _(RegExpSearcher) \
+ _(RegExpTester) \
+ _(IsRegExpObject) \
+ _(IsPossiblyWrappedRegExpObject) \
+ _(RegExpPrototypeOptimizable) \
+ _(RegExpInstanceOptimizable) \
+ _(GetFirstDollarIndex) \
+ \
+ _(SetHas) \
+ \
+ _(String) \
+ _(StringToString) \
+ _(StringValueOf) \
+ _(StringCharCodeAt) \
+ _(StringFromCharCode) \
+ _(StringFromCodePoint) \
+ _(StringCharAt) \
+ _(StringIndexOf) \
+ _(StringStartsWith) \
+ _(StringEndsWith) \
+ _(StringToLowerCase) \
+ _(StringToUpperCase) \
+ \
+ _(IntrinsicStringReplaceString) \
+ _(IntrinsicStringSplitString) \
+ \
+ _(Object) \
+ _(ObjectCreate) \
+ _(ObjectIs) \
+ _(ObjectIsPrototypeOf) \
+ _(ObjectToString) \
+ \
+ _(TestBailout) \
+ _(TestAssertFloat32) \
+ _(TestAssertRecoveredOnBailout) \
+ \
+ _(IntrinsicUnsafeSetReservedSlot) \
+ _(IntrinsicUnsafeGetReservedSlot) \
+ _(IntrinsicUnsafeGetObjectFromReservedSlot) \
+ _(IntrinsicUnsafeGetInt32FromReservedSlot) \
+ _(IntrinsicUnsafeGetStringFromReservedSlot) \
+ \
+ _(IntrinsicIsCallable) \
+ _(IntrinsicIsConstructor) \
+ _(IntrinsicToObject) \
+ _(IntrinsicIsObject) \
+ _(IntrinsicIsCrossRealmArrayConstructor) \
+ _(IntrinsicToInteger) \
+ _(IntrinsicToLength) \
+ _(IntrinsicIsConstructing) \
+ _(IntrinsicSubstringKernel) \
+ _(IntrinsicObjectHasPrototype) \
+ _(IntrinsicFinishBoundFunctionInit) \
+ _(IntrinsicIsPackedArray) \
+ \
+ _(IntrinsicIsSuspendedGenerator) \
+ \
+ _(IntrinsicGuardToArrayIterator) \
+ _(IntrinsicGuardToMapIterator) \
+ _(IntrinsicGuardToSetIterator) \
+ _(IntrinsicGuardToStringIterator) \
+ _(IntrinsicGuardToRegExpStringIterator) \
+ _(IntrinsicGuardToWrapForValidIterator) \
+ _(IntrinsicGuardToIteratorHelper) \
+ _(IntrinsicGuardToAsyncIteratorHelper) \
+ \
+ _(IntrinsicGuardToMapObject) \
+ _(IntrinsicGetNextMapEntryForIterator) \
+ \
+ _(IntrinsicGuardToSetObject) \
+ _(IntrinsicGetNextSetEntryForIterator) \
+ \
+ _(IntrinsicNewArrayIterator) \
+ _(IntrinsicNewStringIterator) \
+ _(IntrinsicNewRegExpStringIterator) \
+ _(IntrinsicArrayIteratorPrototypeOptimizable) \
+ \
+ _(IntrinsicGuardToArrayBuffer) \
+ _(IntrinsicArrayBufferByteLength) \
+ _(IntrinsicPossiblyWrappedArrayBufferByteLength) \
+ \
+ _(IntrinsicGuardToSharedArrayBuffer) \
+ \
+ _(TypedArrayConstructor) \
+ _(IntrinsicIsTypedArrayConstructor) \
+ _(IntrinsicIsTypedArray) \
+ _(IntrinsicIsPossiblyWrappedTypedArray) \
+ _(IntrinsicTypedArrayLength) \
+ _(IntrinsicPossiblyWrappedTypedArrayLength) \
+ _(IntrinsicTypedArrayByteOffset) \
+ _(IntrinsicTypedArrayElementSize) \
+ \
+ INLINABLE_NATIVE_FUZZILLI_LIST(_)
+
+struct JSClass;
+class JSJitInfo;
+
+namespace js {
+namespace jit {
+
+enum class InlinableNative : uint16_t {
+#define ADD_NATIVE(native) native,
+ INLINABLE_NATIVE_LIST(ADD_NATIVE)
+#undef ADD_NATIVE
+ Limit
+};
+
+#define ADD_NATIVE(native) extern const JSJitInfo JitInfo_##native;
+INLINABLE_NATIVE_LIST(ADD_NATIVE)
+#undef ADD_NATIVE
+
+const JSClass* InlinableNativeGuardToClass(InlinableNative native);
+
+bool CanInlineNativeCrossRealm(InlinableNative native);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_InlinableNatives_h */
diff --git a/js/src/jit/InlineList.h b/js/src/jit/InlineList.h
new file mode 100644
index 0000000000..44e445872b
--- /dev/null
+++ b/js/src/jit/InlineList.h
@@ -0,0 +1,590 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_InlineList_h
+#define jit_InlineList_h
+
+#include "mozilla/Assertions.h"
+
+namespace js {
+
+template <typename T>
+class InlineForwardList;
+template <typename T>
+class InlineForwardListIterator;
+
+template <typename T>
+class InlineForwardListNode {
+ public:
+ InlineForwardListNode() : next(nullptr) {}
+ explicit InlineForwardListNode(InlineForwardListNode<T>* n) : next(n) {}
+
+ InlineForwardListNode(const InlineForwardListNode<T>&) = delete;
+
+ protected:
+ friend class InlineForwardList<T>;
+ friend class InlineForwardListIterator<T>;
+
+ InlineForwardListNode<T>* next;
+};
+
+template <typename T>
+class InlineForwardList : protected InlineForwardListNode<T> {
+ friend class InlineForwardListIterator<T>;
+
+ using Node = InlineForwardListNode<T>;
+
+ Node* tail_;
+#ifdef DEBUG
+ int modifyCount_;
+#endif
+
+ InlineForwardList<T>* thisFromConstructor() { return this; }
+
+ public:
+ InlineForwardList() : tail_(thisFromConstructor()) {
+#ifdef DEBUG
+ modifyCount_ = 0;
+#endif
+ }
+
+ public:
+ using iterator = InlineForwardListIterator<T>;
+
+ public:
+ iterator begin() const { return iterator(this); }
+ iterator begin(Node* item) const { return iterator(this, item); }
+ iterator end() const { return iterator(nullptr); }
+ void removeAt(iterator where) { removeAfter(where.prev, where.iter); }
+ void pushFront(Node* t) { insertAfter(this, t); }
+ void pushBack(Node* t) {
+ MOZ_ASSERT(t->next == nullptr);
+#ifdef DEBUG
+ modifyCount_++;
+#endif
+ tail_->next = t;
+ tail_ = t;
+ }
+ T* popFront() {
+ MOZ_ASSERT(!empty());
+ T* result = static_cast<T*>(this->next);
+ removeAfter(this, result);
+ return result;
+ }
+ T* back() const {
+ MOZ_ASSERT(!empty());
+ return static_cast<T*>(tail_);
+ }
+ void insertAfter(Node* at, Node* item) {
+ MOZ_ASSERT(item->next == nullptr);
+#ifdef DEBUG
+ modifyCount_++;
+#endif
+ if (at == tail_) {
+ tail_ = item;
+ }
+ item->next = at->next;
+ at->next = item;
+ }
+ void removeAfter(Node* at, Node* item) {
+#ifdef DEBUG
+ modifyCount_++;
+#endif
+ if (item == tail_) {
+ tail_ = at;
+ }
+ MOZ_ASSERT(at->next == item);
+ at->next = item->next;
+ item->next = nullptr;
+ }
+ void removeAndIncrement(iterator& where) {
+ // Do not change modifyCount_ here. The iterator can still be used
+ // after calling this method, unlike the other methods that modify
+ // the list.
+ Node* item = where.iter;
+ where.iter = item->next;
+ if (item == tail_) {
+ tail_ = where.prev;
+ }
+ MOZ_ASSERT(where.prev->next == item);
+ where.prev->next = where.iter;
+ item->next = nullptr;
+ }
+ void splitAfter(Node* at, InlineForwardList<T>* to) {
+ MOZ_ASSERT(to->empty());
+ if (!at) {
+ at = this;
+ }
+ if (at == tail_) {
+ return;
+ }
+#ifdef DEBUG
+ modifyCount_++;
+#endif
+ to->next = at->next;
+ to->tail_ = tail_;
+ tail_ = at;
+ at->next = nullptr;
+ }
+ bool empty() const { return tail_ == this; }
+ void clear() {
+ this->next = nullptr;
+ tail_ = this;
+#ifdef DEBUG
+ modifyCount_ = 0;
+#endif
+ }
+};
+
+template <typename T>
+class InlineForwardListIterator {
+ private:
+ friend class InlineForwardList<T>;
+
+ using Node = InlineForwardListNode<T>;
+
+ explicit InlineForwardListIterator(const InlineForwardList<T>* owner)
+ : prev(const_cast<Node*>(static_cast<const Node*>(owner))),
+ iter(owner ? owner->next : nullptr)
+#ifdef DEBUG
+ ,
+ owner_(owner),
+ modifyCount_(owner ? owner->modifyCount_ : 0)
+#endif
+ {
+ }
+
+ InlineForwardListIterator(const InlineForwardList<T>* owner, Node* node)
+ : prev(nullptr),
+ iter(node)
+#ifdef DEBUG
+ ,
+ owner_(owner),
+ modifyCount_(owner ? owner->modifyCount_ : 0)
+#endif
+ {
+ }
+
+ public:
+ InlineForwardListIterator<T>& operator++() {
+ MOZ_ASSERT(modifyCount_ == owner_->modifyCount_);
+ prev = iter;
+ iter = iter->next;
+ return *this;
+ }
+ InlineForwardListIterator<T> operator++(int) {
+ InlineForwardListIterator<T> old(*this);
+ operator++();
+ return old;
+ }
+ T* operator*() const {
+ MOZ_ASSERT(modifyCount_ == owner_->modifyCount_);
+ return static_cast<T*>(iter);
+ }
+ T* operator->() const {
+ MOZ_ASSERT(modifyCount_ == owner_->modifyCount_);
+ return static_cast<T*>(iter);
+ }
+ bool operator!=(const InlineForwardListIterator<T>& where) const {
+ return iter != where.iter;
+ }
+ bool operator==(const InlineForwardListIterator<T>& where) const {
+ return iter == where.iter;
+ }
+ explicit operator bool() const { return iter != nullptr; }
+
+ private:
+ Node* prev;
+ Node* iter;
+
+#ifdef DEBUG
+ const InlineForwardList<T>* owner_;
+ int modifyCount_;
+#endif
+};
+
+template <typename T>
+class InlineList;
+template <typename T>
+class InlineListIterator;
+template <typename T>
+class InlineListReverseIterator;
+
+template <typename T>
+class InlineListNode : public InlineForwardListNode<T> {
+ public:
+ InlineListNode() : InlineForwardListNode<T>(nullptr), prev(nullptr) {}
+ InlineListNode(InlineListNode<T>* n, InlineListNode<T>* p)
+ : InlineForwardListNode<T>(n), prev(p) {}
+
+ // Move constructor. Nodes may be moved without being removed from their
+ // containing lists. For example, this allows list nodes to be safely
+ // stored in a resizable Vector -- when the Vector resizes, the new storage
+ // is initialized by this move constructor. |other| is a reference to the
+ // old node which the |this| node here is replacing.
+ InlineListNode(InlineListNode<T>&& other)
+ : InlineForwardListNode<T>(other.next) {
+ InlineListNode<T>* newNext = static_cast<InlineListNode<T>*>(other.next);
+ InlineListNode<T>* newPrev = other.prev;
+ prev = newPrev;
+
+ // Update the pointers in the adjacent nodes to point to this node's new
+ // location.
+ newNext->prev = this;
+ newPrev->next = this;
+ }
+
+ InlineListNode(const InlineListNode<T>&) = delete;
+ void operator=(const InlineListNode<T>&) = delete;
+
+ bool isInList() { return prev != nullptr && this->next != nullptr; }
+
+ protected:
+ friend class InlineList<T>;
+ friend class InlineListIterator<T>;
+ friend class InlineListReverseIterator<T>;
+
+ InlineListNode<T>* prev;
+};
+
+template <typename T>
+class InlineList : protected InlineListNode<T> {
+ using Node = InlineListNode<T>;
+
+ public:
+ InlineList() : InlineListNode<T>(this, this) {}
+
+ public:
+ using iterator = InlineListIterator<T>;
+ using reverse_iterator = InlineListReverseIterator<T>;
+
+ public:
+ iterator begin() const { return iterator(static_cast<Node*>(this->next)); }
+ iterator begin(Node* t) const { return iterator(t); }
+ iterator end() const { return iterator(this); }
+ reverse_iterator rbegin() const { return reverse_iterator(this->prev); }
+ reverse_iterator rbegin(Node* t) const { return reverse_iterator(t); }
+ reverse_iterator rend() const { return reverse_iterator(this); }
+ void pushFront(Node* t) { insertAfter(this, t); }
+ void pushFrontUnchecked(Node* t) { insertAfterUnchecked(this, t); }
+ void pushBack(Node* t) { insertBefore(this, t); }
+ void pushBackUnchecked(Node* t) { insertBeforeUnchecked(this, t); }
+ T* popFront() {
+ MOZ_ASSERT(!empty());
+ T* t = static_cast<T*>(this->next);
+ remove(t);
+ return t;
+ }
+ T* popBack() {
+ MOZ_ASSERT(!empty());
+ T* t = static_cast<T*>(this->prev);
+ remove(t);
+ return t;
+ }
+ T* peekBack() const {
+ iterator iter = end();
+ iter--;
+ return *iter;
+ }
+ void insertBefore(Node* at, Node* item) {
+ MOZ_ASSERT(item->prev == nullptr);
+ MOZ_ASSERT(item->next == nullptr);
+ insertBeforeUnchecked(at, item);
+ }
+ void insertBeforeUnchecked(Node* at, Node* item) {
+ Node* atPrev = at->prev;
+ item->next = at;
+ item->prev = atPrev;
+ atPrev->next = item;
+ at->prev = item;
+ }
+ void insertAfter(Node* at, Node* item) {
+ MOZ_ASSERT(item->prev == nullptr);
+ MOZ_ASSERT(item->next == nullptr);
+ insertAfterUnchecked(at, item);
+ }
+ void insertAfterUnchecked(Node* at, Node* item) {
+ Node* atNext = static_cast<Node*>(at->next);
+ item->next = atNext;
+ item->prev = at;
+ atNext->prev = item;
+ at->next = item;
+ }
+ void remove(Node* t) {
+ Node* tNext = static_cast<Node*>(t->next);
+ Node* tPrev = t->prev;
+ tPrev->next = tNext;
+ tNext->prev = tPrev;
+ t->next = nullptr;
+ t->prev = nullptr;
+ }
+ // Remove |old| from the list and insert |now| in its place.
+ void replace(Node* old, Node* now) {
+ MOZ_ASSERT(now->next == nullptr && now->prev == nullptr);
+ Node* listNext = static_cast<Node*>(old->next);
+ Node* listPrev = old->prev;
+ listPrev->next = now;
+ listNext->prev = now;
+ now->next = listNext;
+ now->prev = listPrev;
+ old->next = nullptr;
+ old->prev = nullptr;
+ }
+ void clear() { this->next = this->prev = this; }
+ bool empty() const { return begin() == end(); }
+ void takeElements(InlineList& l) {
+ MOZ_ASSERT(&l != this, "cannot takeElements from this");
+ Node* lprev = l.prev;
+ static_cast<Node*>(l.next)->prev = this;
+ lprev->next = this->next;
+ static_cast<Node*>(this->next)->prev = l.prev;
+ this->next = l.next;
+ l.clear();
+ }
+};
+
+template <typename T>
+class InlineListIterator {
+ private:
+ friend class InlineList<T>;
+
+ using Node = InlineListNode<T>;
+
+ explicit InlineListIterator(const Node* iter)
+ : iter(const_cast<Node*>(iter)) {}
+
+ public:
+ InlineListIterator<T>& operator++() {
+ iter = static_cast<Node*>(iter->next);
+ return *this;
+ }
+ InlineListIterator<T> operator++(int) {
+ InlineListIterator<T> old(*this);
+ operator++();
+ return old;
+ }
+ InlineListIterator<T>& operator--() {
+ iter = iter->prev;
+ return *this;
+ }
+ InlineListIterator<T> operator--(int) {
+ InlineListIterator<T> old(*this);
+ operator--();
+ return old;
+ }
+ T* operator*() const { return static_cast<T*>(iter); }
+ T* operator->() const { return static_cast<T*>(iter); }
+ bool operator!=(const InlineListIterator<T>& where) const {
+ return iter != where.iter;
+ }
+ bool operator==(const InlineListIterator<T>& where) const {
+ return iter == where.iter;
+ }
+
+ private:
+ Node* iter;
+};
+
+template <typename T>
+class InlineListReverseIterator {
+ private:
+ friend class InlineList<T>;
+
+ using Node = InlineListNode<T>;
+
+ explicit InlineListReverseIterator(const Node* iter)
+ : iter(const_cast<Node*>(iter)) {}
+
+ public:
+ InlineListReverseIterator<T>& operator++() {
+ iter = iter->prev;
+ return *this;
+ }
+ InlineListReverseIterator<T> operator++(int) {
+ InlineListReverseIterator<T> old(*this);
+ operator++();
+ return old;
+ }
+ InlineListReverseIterator<T>& operator--() {
+ iter = static_cast<Node*>(iter->next);
+ return *this;
+ }
+ InlineListReverseIterator<T> operator--(int) {
+ InlineListReverseIterator<T> old(*this);
+ operator--();
+ return old;
+ }
+ T* operator*() { return static_cast<T*>(iter); }
+ T* operator->() { return static_cast<T*>(iter); }
+ bool operator!=(const InlineListReverseIterator<T>& where) const {
+ return iter != where.iter;
+ }
+ bool operator==(const InlineListReverseIterator<T>& where) const {
+ return iter == where.iter;
+ }
+
+ private:
+ Node* iter;
+};
+
+// This list type is more or less exactly an InlineForwardList without a
+// sentinel node. It is useful in cases where you are doing algorithms that deal
+// with many merging singleton lists, rather than often empty ones.
+template <typename T>
+class InlineConcatListIterator;
+template <typename T>
+class InlineConcatList {
+ private:
+ using Node = InlineConcatList<T>;
+
+ InlineConcatList<T>* thisFromConstructor() { return this; }
+
+ public:
+ InlineConcatList() : next(nullptr), tail(thisFromConstructor()) {}
+
+ using iterator = InlineConcatListIterator<T>;
+
+ iterator begin() const { return iterator(this); }
+
+ iterator end() const { return iterator(nullptr); }
+
+ void append(InlineConcatList<T>* adding) {
+ MOZ_ASSERT(tail);
+ MOZ_ASSERT(!tail->next);
+ MOZ_ASSERT(adding->tail);
+ MOZ_ASSERT(!adding->tail->next);
+
+ tail->next = adding;
+ tail = adding->tail;
+ adding->tail = nullptr;
+ }
+
+ protected:
+ friend class InlineConcatListIterator<T>;
+ Node* next;
+ Node* tail;
+};
+
+template <typename T>
+class InlineConcatListIterator {
+ private:
+ friend class InlineConcatList<T>;
+
+ using Node = InlineConcatList<T>;
+
+ explicit InlineConcatListIterator(const Node* iter)
+ : iter(const_cast<Node*>(iter)) {}
+
+ public:
+ InlineConcatListIterator<T>& operator++() {
+ iter = static_cast<Node*>(iter->next);
+ return *this;
+ }
+ InlineConcatListIterator<T> operator++(int) {
+ InlineConcatListIterator<T> old(*this);
+ operator++();
+ return old;
+ }
+ T* operator*() const { return static_cast<T*>(iter); }
+ T* operator->() const { return static_cast<T*>(iter); }
+ bool operator!=(const InlineConcatListIterator<T>& where) const {
+ return iter != where.iter;
+ }
+ bool operator==(const InlineConcatListIterator<T>& where) const {
+ return iter == where.iter;
+ }
+
+ private:
+ Node* iter;
+};
+
+template <typename T>
+class InlineSpaghettiStack;
+template <typename T>
+class InlineSpaghettiStackNode;
+template <typename T>
+class InlineSpaghettiStackIterator;
+
+template <typename T>
+class InlineSpaghettiStackNode : public InlineForwardListNode<T> {
+ using Parent = InlineForwardListNode<T>;
+
+ public:
+ InlineSpaghettiStackNode() : Parent() {}
+
+ explicit InlineSpaghettiStackNode(InlineSpaghettiStackNode<T>* n)
+ : Parent(n) {}
+
+ InlineSpaghettiStackNode(const InlineSpaghettiStackNode<T>&) = delete;
+
+ protected:
+ friend class InlineSpaghettiStack<T>;
+ friend class InlineSpaghettiStackIterator<T>;
+};
+
+template <typename T>
+class InlineSpaghettiStack : protected InlineSpaghettiStackNode<T> {
+ friend class InlineSpaghettiStackIterator<T>;
+
+ using Node = InlineSpaghettiStackNode<T>;
+
+ public:
+ InlineSpaghettiStack() = default;
+
+ public:
+ using iterator = InlineSpaghettiStackIterator<T>;
+
+ public:
+ iterator begin() const { return iterator(this); }
+ iterator end() const { return iterator(nullptr); }
+
+ void push(Node* t) {
+ MOZ_ASSERT(t->next == nullptr);
+ t->next = this->next;
+ this->next = t;
+ }
+
+ void copy(const InlineSpaghettiStack<T>& stack) { this->next = stack.next; }
+
+ bool empty() const { return this->next == nullptr; }
+};
+
+template <typename T>
+class InlineSpaghettiStackIterator {
+ private:
+ friend class InlineSpaghettiStack<T>;
+
+ using Node = InlineSpaghettiStackNode<T>;
+
+ explicit InlineSpaghettiStackIterator(const InlineSpaghettiStack<T>* owner)
+ : iter(owner ? static_cast<Node*>(owner->next) : nullptr) {}
+
+ public:
+ InlineSpaghettiStackIterator<T>& operator++() {
+ iter = static_cast<Node*>(iter->next);
+ return *this;
+ }
+ InlineSpaghettiStackIterator<T> operator++(int) {
+ InlineSpaghettiStackIterator<T> old(*this);
+ operator++();
+ return old;
+ }
+ T* operator*() const { return static_cast<T*>(iter); }
+ T* operator->() const { return static_cast<T*>(iter); }
+ bool operator!=(const InlineSpaghettiStackIterator<T>& where) const {
+ return iter != where.iter;
+ }
+ bool operator==(const InlineSpaghettiStackIterator<T>& where) const {
+ return iter == where.iter;
+ }
+
+ private:
+ Node* iter;
+};
+
+} // namespace js
+
+#endif /* jit_InlineList_h */
diff --git a/js/src/jit/InlineScriptTree-inl.h b/js/src/jit/InlineScriptTree-inl.h
new file mode 100644
index 0000000000..21d55bb039
--- /dev/null
+++ b/js/src/jit/InlineScriptTree-inl.h
@@ -0,0 +1,67 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_InlineScriptTree_inl_h
+#define jit_InlineScriptTree_inl_h
+
+#include "jit/InlineScriptTree.h"
+
+#include "mozilla/Assertions.h"
+
+#include "jit/JitAllocPolicy.h"
+#include "js/TypeDecls.h"
+#include "vm/JSScript.h"
+
+namespace js {
+namespace jit {
+
+InlineScriptTree* InlineScriptTree::New(TempAllocator* allocator,
+ InlineScriptTree* callerTree,
+ jsbytecode* callerPc,
+ JSScript* script) {
+ MOZ_ASSERT_IF(!callerTree, !callerPc);
+ MOZ_ASSERT_IF(callerTree, callerTree->script()->containsPC(callerPc));
+
+ // Allocate a new InlineScriptTree
+ void* treeMem = allocator->allocate(sizeof(InlineScriptTree));
+ if (!treeMem) {
+ return nullptr;
+ }
+
+ // Initialize it.
+ return new (treeMem) InlineScriptTree(callerTree, callerPc, script);
+}
+
+InlineScriptTree* InlineScriptTree::addCallee(TempAllocator* allocator,
+ jsbytecode* callerPc,
+ JSScript* calleeScript) {
+ MOZ_ASSERT(script_ && script_->containsPC(callerPc));
+ InlineScriptTree* calleeTree = New(allocator, this, callerPc, calleeScript);
+ if (!calleeTree) {
+ return nullptr;
+ }
+
+ calleeTree->nextCallee_ = children_;
+ children_ = calleeTree;
+ return calleeTree;
+}
+
+void InlineScriptTree::removeCallee(InlineScriptTree* callee) {
+ InlineScriptTree** prevPtr = &children_;
+ for (InlineScriptTree* child = children_; child; child = child->nextCallee_) {
+ if (child == callee) {
+ *prevPtr = child->nextCallee_;
+ return;
+ }
+ prevPtr = &child->nextCallee_;
+ }
+ MOZ_CRASH("Callee not found");
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_InlineScriptTree_inl_h */
diff --git a/js/src/jit/InlineScriptTree.h b/js/src/jit/InlineScriptTree.h
new file mode 100644
index 0000000000..0e5e2767ff
--- /dev/null
+++ b/js/src/jit/InlineScriptTree.h
@@ -0,0 +1,112 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_InlineScriptTree_h
+#define jit_InlineScriptTree_h
+
+#include "mozilla/Assertions.h"
+
+#include "jit/JitAllocPolicy.h"
+#include "js/TypeDecls.h"
+
+namespace js {
+namespace jit {
+
+// The compiler at various points needs to be able to store references to the
+// current inline path (the sequence of scripts and call-pcs that lead to the
+// current function being inlined).
+//
+// To support this, use a tree that records the inlinings done during
+// compilation.
+class InlineScriptTree {
+ // InlineScriptTree for the caller
+ InlineScriptTree* caller_;
+
+ // PC in the caller corresponding to this script.
+ jsbytecode* callerPc_;
+
+ // Script for this entry.
+ JSScript* script_;
+
+ // Child entries (linked together by nextCallee pointer)
+ InlineScriptTree* children_;
+ InlineScriptTree* nextCallee_;
+
+ public:
+ InlineScriptTree(InlineScriptTree* caller, jsbytecode* callerPc,
+ JSScript* script)
+ : caller_(caller),
+ callerPc_(callerPc),
+ script_(script),
+ children_(nullptr),
+ nextCallee_(nullptr) {}
+
+ static inline InlineScriptTree* New(TempAllocator* allocator,
+ InlineScriptTree* caller,
+ jsbytecode* callerPc, JSScript* script);
+
+ inline InlineScriptTree* addCallee(TempAllocator* allocator,
+ jsbytecode* callerPc,
+ JSScript* calleeScript);
+ inline void removeCallee(InlineScriptTree* callee);
+
+ InlineScriptTree* caller() const { return caller_; }
+
+ bool isOutermostCaller() const { return caller_ == nullptr; }
+ bool hasCaller() const { return caller_ != nullptr; }
+
+ jsbytecode* callerPc() const { return callerPc_; }
+
+ JSScript* script() const { return script_; }
+
+ bool hasChildren() const { return children_ != nullptr; }
+ InlineScriptTree* firstChild() const {
+ MOZ_ASSERT(hasChildren());
+ return children_;
+ }
+
+ bool hasNextCallee() const { return nextCallee_ != nullptr; }
+ InlineScriptTree* nextCallee() const {
+ MOZ_ASSERT(hasNextCallee());
+ return nextCallee_;
+ }
+
+ unsigned depth() const {
+ if (isOutermostCaller()) {
+ return 1;
+ }
+ return 1 + caller_->depth();
+ }
+};
+
+class BytecodeSite : public TempObject {
+ // InlineScriptTree identifying innermost active function at site.
+ InlineScriptTree* tree_;
+
+ // Bytecode address within innermost active function.
+ jsbytecode* pc_;
+
+ public:
+ // Wasm compilation leaves both fields null.
+ BytecodeSite() : tree_(nullptr), pc_(nullptr) {}
+
+ // Warp compilation sets both fields to non-null values.
+ BytecodeSite(InlineScriptTree* tree, jsbytecode* pc) : tree_(tree), pc_(pc) {
+ MOZ_ASSERT(tree_ != nullptr);
+ MOZ_ASSERT(pc_ != nullptr);
+ }
+
+ InlineScriptTree* tree() const { return tree_; }
+
+ jsbytecode* pc() const { return pc_; }
+
+ JSScript* script() const { return tree_ ? tree_->script() : nullptr; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_InlineScriptTree_h */
diff --git a/js/src/jit/InstructionReordering.cpp b/js/src/jit/InstructionReordering.cpp
new file mode 100644
index 0000000000..b399e0bb8c
--- /dev/null
+++ b/js/src/jit/InstructionReordering.cpp
@@ -0,0 +1,248 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/InstructionReordering.h"
+#include "jit/MIRGraph.h"
+
+using namespace js;
+using namespace js::jit;
+
+static void MoveBefore(MBasicBlock* block, MInstruction* at,
+ MInstruction* ins) {
+ if (at == ins) {
+ return;
+ }
+
+ // Update instruction numbers.
+ for (MInstructionIterator iter(block->begin(at)); *iter != ins; iter++) {
+ MOZ_ASSERT(iter->id() < ins->id());
+ iter->setId(iter->id() + 1);
+ }
+ ins->setId(at->id() - 1);
+ block->moveBefore(at, ins);
+}
+
+static bool IsLastUse(MDefinition* ins, MDefinition* input,
+ MBasicBlock* loopHeader) {
+ // If we are in a loop, this cannot be the last use of any definitions from
+ // outside the loop, as those definitions can be used in future iterations.
+ if (loopHeader && input->block()->id() < loopHeader->id()) {
+ return false;
+ }
+ for (MUseDefIterator iter(input); iter; iter++) {
+ // Watch for uses defined in blocks which ReorderInstructions hasn't
+ // processed yet. These nodes have not had their ids set yet.
+ if (iter.def()->block()->id() > ins->block()->id()) {
+ return false;
+ }
+ if (iter.def()->id() > ins->id()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static void MoveConstantsToStart(MBasicBlock* block,
+ MInstruction* insertionPoint) {
+ // Move constants with a single use in the current block to the start of the
+ // block. Constants won't be reordered by ReorderInstructions, as they have no
+ // inputs. Moving them up as high as possible can allow their use to be moved
+ // up further, though, and has no cost if the constant is emitted at its use.
+
+ MInstructionIterator iter(block->begin(insertionPoint));
+ while (iter != block->end()) {
+ MInstruction* ins = *iter;
+ iter++;
+
+ if (!ins->isConstant() || !ins->hasOneUse() ||
+ ins->usesBegin()->consumer()->block() != block ||
+ IsFloatingPointType(ins->type())) {
+ continue;
+ }
+
+ MOZ_ASSERT(ins->isMovable());
+ MOZ_ASSERT(insertionPoint != ins);
+
+ // Note: we don't need to use MoveBefore here because MoveConstantsToStart
+ // is called right before we renumber all instructions in this block.
+ block->moveBefore(insertionPoint, ins);
+ }
+}
+
+bool jit::ReorderInstructions(MIRGraph& graph) {
+ // Renumber all instructions in the graph as we go.
+ size_t nextId = 0;
+
+ // List of the headers of any loops we are in.
+ Vector<MBasicBlock*, 4, SystemAllocPolicy> loopHeaders;
+
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); block++) {
+ // Don't reorder instructions within entry blocks, which have special
+ // requirements.
+ bool isEntryBlock =
+ *block == graph.entryBlock() || *block == graph.osrBlock();
+
+ MInstruction* insertionPoint = nullptr;
+ if (!isEntryBlock) {
+ // Move constants to the start of the block before renumbering all
+ // instructions.
+ insertionPoint = block->safeInsertTop();
+ MoveConstantsToStart(*block, insertionPoint);
+ }
+
+ // Renumber all definitions inside the basic blocks.
+ for (MPhiIterator iter(block->phisBegin()); iter != block->phisEnd();
+ iter++) {
+ iter->setId(nextId++);
+ }
+
+ for (MInstructionIterator iter(block->begin()); iter != block->end();
+ iter++) {
+ iter->setId(nextId++);
+ }
+
+ if (isEntryBlock) {
+ continue;
+ }
+
+ if (block->isLoopHeader()) {
+ if (!loopHeaders.append(*block)) {
+ return false;
+ }
+ }
+
+ MBasicBlock* innerLoop = loopHeaders.empty() ? nullptr : loopHeaders.back();
+
+ MInstructionReverseIterator rtop = ++block->rbegin(insertionPoint);
+ for (MInstructionIterator iter(block->begin(insertionPoint));
+ iter != block->end();) {
+ MInstruction* ins = *iter;
+
+ // Filter out some instructions which are never reordered.
+ if (ins->isEffectful() || !ins->isMovable() || ins->resumePoint() ||
+ ins == block->lastIns()) {
+ iter++;
+ continue;
+ }
+
+ // Look for inputs where this instruction is the last use of that
+ // input. If we move this instruction up, the input's lifetime will
+ // be shortened, modulo resume point uses (which don't need to be
+ // stored in a register, and can be handled by the register
+ // allocator by just spilling at some point with no reload).
+ Vector<MDefinition*, 4, SystemAllocPolicy> lastUsedInputs;
+ for (size_t i = 0; i < ins->numOperands(); i++) {
+ MDefinition* input = ins->getOperand(i);
+ if (!input->isConstant() && IsLastUse(ins, input, innerLoop)) {
+ if (!lastUsedInputs.append(input)) {
+ return false;
+ }
+ }
+ }
+
+ // Don't try to move instructions which aren't the last use of any
+ // of their inputs (we really ought to move these down instead).
+ if (lastUsedInputs.length() < 2) {
+ iter++;
+ continue;
+ }
+
+ MInstruction* target = ins;
+ MInstruction* postCallTarget = nullptr;
+ for (MInstructionReverseIterator riter = ++block->rbegin(ins);
+ riter != rtop; riter++) {
+ MInstruction* prev = *riter;
+ if (prev->isInterruptCheck()) {
+ break;
+ }
+ if (prev->isSetInitializedLength()) {
+ break;
+ }
+
+ // The instruction can't be moved before any of its uses.
+ bool isUse = false;
+ for (size_t i = 0; i < ins->numOperands(); i++) {
+ if (ins->getOperand(i) == prev) {
+ isUse = true;
+ break;
+ }
+ }
+ if (isUse) {
+ break;
+ }
+
+ // The instruction can't be moved before an instruction that
+ // stores to a location read by the instruction.
+ if (prev->isEffectful() &&
+ (ins->getAliasSet().flags() & prev->getAliasSet().flags()) &&
+ ins->mightAlias(prev) != MDefinition::AliasType::NoAlias) {
+ break;
+ }
+
+ // Make sure the instruction will still be the last use of one
+ // of its inputs when moved up this far.
+ for (size_t i = 0; i < lastUsedInputs.length();) {
+ bool found = false;
+ for (size_t j = 0; j < prev->numOperands(); j++) {
+ if (prev->getOperand(j) == lastUsedInputs[i]) {
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ lastUsedInputs[i] = lastUsedInputs.back();
+ lastUsedInputs.popBack();
+ } else {
+ i++;
+ }
+ }
+ if (lastUsedInputs.length() < 2) {
+ break;
+ }
+
+ // If we see a captured call result, either move the instruction before
+ // the corresponding call or don't move it at all.
+ if (prev->isCallResultCapture()) {
+ if (!postCallTarget) {
+ postCallTarget = target;
+ }
+ } else if (postCallTarget) {
+ MOZ_ASSERT(MWasmCallBase::IsWasmCall(prev) ||
+ prev->isIonToWasmCall());
+ postCallTarget = nullptr;
+ }
+
+ // We can move the instruction before this one.
+ target = prev;
+ }
+
+ if (postCallTarget) {
+ // We would have plonked this instruction between a call and its
+ // captured return value. Instead put it after the last corresponding
+ // return value.
+ target = postCallTarget;
+ }
+
+ iter++;
+ MoveBefore(*block, target, ins);
+
+ // Instruction reordering can move a bailing instruction up past a call
+ // that throws an exception, causing spurious bailouts. This should rarely
+ // be an issue in practice, so we only update the bailout kind if we don't
+ // have anything more specific.
+ if (ins->bailoutKind() == BailoutKind::TranspiledCacheIR) {
+ ins->setBailoutKind(BailoutKind::InstructionReordering);
+ }
+ }
+
+ if (block->isLoopBackedge()) {
+ loopHeaders.popBack();
+ }
+ }
+
+ return true;
+}
diff --git a/js/src/jit/InstructionReordering.h b/js/src/jit/InstructionReordering.h
new file mode 100644
index 0000000000..055e6da5b3
--- /dev/null
+++ b/js/src/jit/InstructionReordering.h
@@ -0,0 +1,20 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_InstructionReordering_h
+#define jit_InstructionReordering_h
+
+#include "jit/IonAnalysis.h"
+
+namespace js {
+namespace jit {
+
+[[nodiscard]] bool ReorderInstructions(MIRGraph& graph);
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_InstructionReordering_h
diff --git a/js/src/jit/Invalidation.h b/js/src/jit/Invalidation.h
new file mode 100644
index 0000000000..f0a5410967
--- /dev/null
+++ b/js/src/jit/Invalidation.h
@@ -0,0 +1,59 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Invalidation_h
+#define jit_Invalidation_h
+
+#include "jit/IonTypes.h"
+#include "js/AllocPolicy.h"
+#include "js/GCVector.h"
+
+namespace js {
+namespace jit {
+
+class IonScript;
+
+class RecompileInfo {
+ JSScript* script_;
+ IonCompilationId id_;
+
+ public:
+ RecompileInfo(JSScript* script, IonCompilationId id)
+ : script_(script), id_(id) {}
+
+ JSScript* script() const { return script_; }
+
+ IonScript* maybeIonScriptToInvalidate() const;
+
+ bool traceWeak(JSTracer* trc);
+
+ bool operator==(const RecompileInfo& other) const {
+ return script_ == other.script_ && id_ == other.id_;
+ }
+};
+
+// The RecompileInfoVector has a MinInlineCapacity of one so that invalidating a
+// single IonScript doesn't require an allocation.
+using RecompileInfoVector = JS::GCVector<RecompileInfo, 1, SystemAllocPolicy>;
+
+// Called from Zone::discardJitCode().
+void InvalidateAll(JS::GCContext* gcx, JS::Zone* zone);
+void FinishInvalidation(JS::GCContext* gcx, JSScript* script);
+
+// Add compilations involving |script| (outer script or inlined) to the vector.
+void AddPendingInvalidation(jit::RecompileInfoVector& invalid,
+ JSScript* script);
+
+// Walk the stack and invalidate active Ion frames for the invalid scripts.
+void Invalidate(JSContext* cx, const RecompileInfoVector& invalid,
+ bool resetUses = true, bool cancelOffThread = true);
+void Invalidate(JSContext* cx, JSScript* script, bool resetUses = true,
+ bool cancelOffThread = true);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Invalidation_h */
diff --git a/js/src/jit/Ion.cpp b/js/src/jit/Ion.cpp
new file mode 100644
index 0000000000..e345d9857c
--- /dev/null
+++ b/js/src/jit/Ion.cpp
@@ -0,0 +1,2560 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Ion.h"
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/IntegerPrintfMacros.h"
+#include "mozilla/MemoryReporting.h"
+#include "mozilla/ThreadLocal.h"
+
+#include "gc/GCContext.h"
+#include "gc/PublicIterators.h"
+#include "jit/AliasAnalysis.h"
+#include "jit/AlignmentMaskAnalysis.h"
+#include "jit/AutoWritableJitCode.h"
+#include "jit/BacktrackingAllocator.h"
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineJIT.h"
+#include "jit/CodeGenerator.h"
+#include "jit/CompileInfo.h"
+#include "jit/EdgeCaseAnalysis.h"
+#include "jit/EffectiveAddressAnalysis.h"
+#include "jit/ExecutableAllocator.h"
+#include "jit/FoldLinearArithConstants.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/InstructionReordering.h"
+#include "jit/Invalidation.h"
+#include "jit/IonAnalysis.h"
+#include "jit/IonCompileTask.h"
+#include "jit/IonIC.h"
+#include "jit/IonOptimizationLevels.h"
+#include "jit/IonScript.h"
+#include "jit/JitcodeMap.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRealm.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/JitZone.h"
+#include "jit/LICM.h"
+#include "jit/Linker.h"
+#include "jit/LIR.h"
+#include "jit/Lowering.h"
+#include "jit/PerfSpewer.h"
+#include "jit/RangeAnalysis.h"
+#include "jit/ScalarReplacement.h"
+#include "jit/ScriptFromCalleeToken.h"
+#include "jit/Sink.h"
+#include "jit/ValueNumbering.h"
+#include "jit/WarpBuilder.h"
+#include "jit/WarpOracle.h"
+#include "jit/WasmBCE.h"
+#include "js/Printf.h"
+#include "js/UniquePtr.h"
+#include "util/Memory.h"
+#include "util/WindowsWrapper.h"
+#include "vm/HelperThreads.h"
+#include "vm/Realm.h"
+#ifdef MOZ_VTUNE
+# include "vtune/VTuneWrapper.h"
+#endif
+
+#include "gc/GC-inl.h"
+#include "jit/InlineScriptTree-inl.h"
+#include "jit/MacroAssembler-inl.h"
+#include "jit/SafepointIndex-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+#include "vm/JSScript-inl.h"
+#include "vm/Realm-inl.h"
+
+#if defined(ANDROID)
+# include <sys/system_properties.h>
+#endif
+
+using mozilla::CheckedInt;
+using mozilla::DebugOnly;
+
+using namespace js;
+using namespace js::jit;
+
+JitRuntime::~JitRuntime() {
+ MOZ_ASSERT(numFinishedOffThreadTasks_ == 0);
+ MOZ_ASSERT(ionLazyLinkListSize_ == 0);
+ MOZ_ASSERT(ionLazyLinkList_.ref().isEmpty());
+
+ // By this point, the jitcode global table should be empty.
+ MOZ_ASSERT_IF(jitcodeGlobalTable_, jitcodeGlobalTable_->empty());
+ js_delete(jitcodeGlobalTable_.ref());
+}
+
+uint32_t JitRuntime::startTrampolineCode(MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "startTrampolineCode");
+
+ masm.assumeUnreachable("Shouldn't get here");
+ masm.flushBuffer();
+ masm.haltingAlign(CodeAlignment);
+ masm.setFramePushed(0);
+ return masm.currentOffset();
+}
+
+bool JitRuntime::initialize(JSContext* cx) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cx->runtime()));
+
+ AutoAllocInAtomsZone az(cx);
+ JitContext jctx(cx);
+
+ if (!generateTrampolines(cx)) {
+ return false;
+ }
+
+ if (!generateBaselineICFallbackCode(cx)) {
+ return false;
+ }
+
+ jitcodeGlobalTable_ = cx->new_<JitcodeGlobalTable>();
+ if (!jitcodeGlobalTable_) {
+ return false;
+ }
+
+ if (!GenerateBaselineInterpreter(cx, baselineInterpreter_)) {
+ return false;
+ }
+
+ // Initialize the jitCodeRaw of the Runtime's canonical SelfHostedLazyScript
+ // to point to the interpreter trampoline.
+ cx->runtime()->selfHostedLazyScript.ref().jitCodeRaw_ =
+ interpreterStub().value;
+
+ return true;
+}
+
+bool JitRuntime::generateTrampolines(JSContext* cx) {
+ TempAllocator temp(&cx->tempLifoAlloc());
+ StackMacroAssembler masm(cx, temp);
+
+ Label bailoutTail;
+ JitSpew(JitSpew_Codegen, "# Emitting bailout tail stub");
+ generateBailoutTailStub(masm, &bailoutTail);
+
+ JitSpew(JitSpew_Codegen, "# Emitting bailout handler");
+ generateBailoutHandler(masm, &bailoutTail);
+
+ JitSpew(JitSpew_Codegen, "# Emitting invalidator");
+ generateInvalidator(masm, &bailoutTail);
+
+ // The arguments rectifier has to use the same frame layout as the function
+ // frames it rectifies.
+ static_assert(std::is_base_of_v<JitFrameLayout, RectifierFrameLayout>,
+ "a rectifier frame can be used with jit frame");
+ static_assert(std::is_base_of_v<JitFrameLayout, WasmToJSJitFrameLayout>,
+ "wasm frames simply are jit frames");
+ static_assert(sizeof(JitFrameLayout) == sizeof(WasmToJSJitFrameLayout),
+ "thus a rectifier frame can be used with a wasm frame");
+
+ JitSpew(JitSpew_Codegen, "# Emitting arguments rectifier");
+ generateArgumentsRectifier(masm, ArgumentsRectifierKind::Normal);
+
+ JitSpew(JitSpew_Codegen, "# Emitting trial inlining arguments rectifier");
+ generateArgumentsRectifier(masm, ArgumentsRectifierKind::TrialInlining);
+
+ JitSpew(JitSpew_Codegen, "# Emitting EnterJIT sequence");
+ generateEnterJIT(cx, masm);
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Value");
+ valuePreBarrierOffset_ = generatePreBarrier(cx, masm, MIRType::Value);
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for String");
+ stringPreBarrierOffset_ = generatePreBarrier(cx, masm, MIRType::String);
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Object");
+ objectPreBarrierOffset_ = generatePreBarrier(cx, masm, MIRType::Object);
+
+ JitSpew(JitSpew_Codegen, "# Emitting Pre Barrier for Shape");
+ shapePreBarrierOffset_ = generatePreBarrier(cx, masm, MIRType::Shape);
+
+ JitSpew(JitSpew_Codegen, "# Emitting free stub");
+ generateFreeStub(masm);
+
+ JitSpew(JitSpew_Codegen, "# Emitting lazy link stub");
+ generateLazyLinkStub(masm);
+
+ JitSpew(JitSpew_Codegen, "# Emitting interpreter stub");
+ generateInterpreterStub(masm);
+
+ JitSpew(JitSpew_Codegen, "# Emitting double-to-int32-value stub");
+ generateDoubleToInt32ValueStub(masm);
+
+ JitSpew(JitSpew_Codegen, "# Emitting VM function wrappers");
+ if (!generateVMWrappers(cx, masm)) {
+ return false;
+ }
+
+ JitSpew(JitSpew_Codegen, "# Emitting profiler exit frame tail stub");
+ Label profilerExitTail;
+ generateProfilerExitFrameTailStub(masm, &profilerExitTail);
+
+ JitSpew(JitSpew_Codegen, "# Emitting exception tail stub");
+ generateExceptionTailStub(masm, &profilerExitTail, &bailoutTail);
+
+ Linker linker(masm);
+ trampolineCode_ = linker.newCode(cx, CodeKind::Other);
+ if (!trampolineCode_) {
+ return false;
+ }
+
+ CollectPerfSpewerJitCodeProfile(trampolineCode_, "Trampolines");
+#ifdef MOZ_VTUNE
+ vtune::MarkStub(trampolineCode_, "Trampolines");
+#endif
+
+ return true;
+}
+
+JitCode* JitRuntime::debugTrapHandler(JSContext* cx,
+ DebugTrapHandlerKind kind) {
+ if (!debugTrapHandlers_[kind]) {
+ // JitRuntime code stubs are shared across compartments and have to
+ // be allocated in the atoms zone.
+ mozilla::Maybe<AutoAllocInAtomsZone> az;
+ if (!cx->zone()->isAtomsZone()) {
+ az.emplace(cx);
+ }
+ debugTrapHandlers_[kind] = generateDebugTrapHandler(cx, kind);
+ }
+ return debugTrapHandlers_[kind];
+}
+
+JitRuntime::IonCompileTaskList& JitRuntime::ionLazyLinkList(JSRuntime* rt) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt),
+ "Should only be mutated by the main thread.");
+ return ionLazyLinkList_.ref();
+}
+
+void JitRuntime::ionLazyLinkListRemove(JSRuntime* rt,
+ jit::IonCompileTask* task) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt),
+ "Should only be mutated by the main thread.");
+ MOZ_ASSERT(rt == task->script()->runtimeFromMainThread());
+ MOZ_ASSERT(ionLazyLinkListSize_ > 0);
+
+ task->removeFrom(ionLazyLinkList(rt));
+ ionLazyLinkListSize_--;
+
+ MOZ_ASSERT(ionLazyLinkList(rt).isEmpty() == (ionLazyLinkListSize_ == 0));
+}
+
+void JitRuntime::ionLazyLinkListAdd(JSRuntime* rt, jit::IonCompileTask* task) {
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt),
+ "Should only be mutated by the main thread.");
+ MOZ_ASSERT(rt == task->script()->runtimeFromMainThread());
+ ionLazyLinkList(rt).insertFront(task);
+ ionLazyLinkListSize_++;
+}
+
+uint8_t* JitRuntime::allocateIonOsrTempData(size_t size) {
+ // Free the old buffer (if needed) before allocating a new one. Note that we
+ // could use realloc here but it's likely not worth the complexity.
+ freeIonOsrTempData();
+ ionOsrTempData_.ref().reset(static_cast<uint8_t*>(js_malloc(size)));
+ return ionOsrTempData_.ref().get();
+}
+
+void JitRuntime::freeIonOsrTempData() { ionOsrTempData_.ref().reset(); }
+
+JitRealm::JitRealm() : initialStringHeap(gc::TenuredHeap) {}
+
+void JitRealm::initialize(bool zoneHasNurseryStrings) {
+ setStringsCanBeInNursery(zoneHasNurseryStrings);
+}
+
+template <typename T>
+static T PopNextBitmaskValue(uint32_t* bitmask) {
+ MOZ_ASSERT(*bitmask);
+ uint32_t index = mozilla::CountTrailingZeroes32(*bitmask);
+ *bitmask ^= 1 << index;
+
+ MOZ_ASSERT(index < uint32_t(T::Count));
+ return T(index);
+}
+
+void JitRealm::performStubReadBarriers(uint32_t stubsToBarrier) const {
+ while (stubsToBarrier) {
+ auto stub = PopNextBitmaskValue<StubIndex>(&stubsToBarrier);
+ const WeakHeapPtr<JitCode*>& jitCode = stubs_[stub];
+ MOZ_ASSERT(jitCode);
+ jitCode.get();
+ }
+}
+
+static bool LinkCodeGen(JSContext* cx, CodeGenerator* codegen,
+ HandleScript script, const WarpSnapshot* snapshot) {
+ if (!codegen->link(cx, snapshot)) {
+ return false;
+ }
+
+ return true;
+}
+
+static bool LinkBackgroundCodeGen(JSContext* cx, IonCompileTask* task) {
+ CodeGenerator* codegen = task->backgroundCodegen();
+ if (!codegen) {
+ return false;
+ }
+
+ JitContext jctx(cx);
+ RootedScript script(cx, task->script());
+ return LinkCodeGen(cx, codegen, script, task->snapshot());
+}
+
+void jit::LinkIonScript(JSContext* cx, HandleScript calleeScript) {
+ // Get the pending IonCompileTask from the script.
+ MOZ_ASSERT(calleeScript->hasBaselineScript());
+ IonCompileTask* task =
+ calleeScript->baselineScript()->pendingIonCompileTask();
+ calleeScript->baselineScript()->removePendingIonCompileTask(cx->runtime(),
+ calleeScript);
+
+ // Remove from pending.
+ cx->runtime()->jitRuntime()->ionLazyLinkListRemove(cx->runtime(), task);
+
+ {
+ gc::AutoSuppressGC suppressGC(cx);
+ if (!LinkBackgroundCodeGen(cx, task)) {
+ // Silently ignore OOM during code generation. The assembly code
+ // doesn't have code to handle it after linking happened. So it's
+ // not OK to throw a catchable exception from there.
+ cx->clearPendingException();
+ }
+ }
+
+ {
+ AutoLockHelperThreadState lock;
+ FinishOffThreadTask(cx->runtime(), task, lock);
+ }
+}
+
+uint8_t* jit::LazyLinkTopActivation(JSContext* cx,
+ LazyLinkExitFrameLayout* frame) {
+ RootedScript calleeScript(
+ cx, ScriptFromCalleeToken(frame->jsFrame()->calleeToken()));
+
+ LinkIonScript(cx, calleeScript);
+
+ MOZ_ASSERT(calleeScript->hasBaselineScript());
+ MOZ_ASSERT(calleeScript->jitCodeRaw());
+
+ return calleeScript->jitCodeRaw();
+}
+
+/* static */
+void JitRuntime::TraceAtomZoneRoots(JSTracer* trc) {
+ MOZ_ASSERT(!JS::RuntimeHeapIsMinorCollecting());
+
+ // Shared stubs are allocated in the atoms zone, so do not iterate
+ // them after the atoms heap after it has been "finished."
+ if (trc->runtime()->atomsAreFinished()) {
+ return;
+ }
+
+ Zone* zone = trc->runtime()->atomsZone();
+ for (auto i = zone->cellIterUnsafe<JitCode>(); !i.done(); i.next()) {
+ JitCode* code = i;
+ TraceRoot(trc, &code, "wrapper");
+ }
+}
+
+/* static */
+bool JitRuntime::MarkJitcodeGlobalTableIteratively(GCMarker* marker) {
+ if (marker->runtime()->hasJitRuntime() &&
+ marker->runtime()->jitRuntime()->hasJitcodeGlobalTable()) {
+ return marker->runtime()
+ ->jitRuntime()
+ ->getJitcodeGlobalTable()
+ ->markIteratively(marker);
+ }
+ return false;
+}
+
+/* static */
+void JitRuntime::TraceWeakJitcodeGlobalTable(JSRuntime* rt, JSTracer* trc) {
+ if (rt->hasJitRuntime() && rt->jitRuntime()->hasJitcodeGlobalTable()) {
+ rt->jitRuntime()->getJitcodeGlobalTable()->traceWeak(rt, trc);
+ }
+}
+
+void JitRealm::traceWeak(JSTracer* trc, JS::Realm* realm) {
+ // Any outstanding compilations should have been cancelled by the GC.
+ MOZ_ASSERT(!HasOffThreadIonCompile(realm));
+
+ for (WeakHeapPtr<JitCode*>& stub : stubs_) {
+ TraceWeakEdge(trc, &stub, "JitRealm::stubs_");
+ }
+}
+
+bool JitZone::addInlinedCompilation(const RecompileInfo& info,
+ JSScript* inlined) {
+ MOZ_ASSERT(inlined != info.script());
+
+ auto p = inlinedCompilations_.lookupForAdd(inlined);
+ if (p) {
+ auto& compilations = p->value();
+ if (!compilations.empty() && compilations.back() == info) {
+ return true;
+ }
+ return compilations.append(info);
+ }
+
+ RecompileInfoVector compilations;
+ if (!compilations.append(info)) {
+ return false;
+ }
+ return inlinedCompilations_.add(p, inlined, std::move(compilations));
+}
+
+void jit::AddPendingInvalidation(RecompileInfoVector& invalid,
+ JSScript* script) {
+ MOZ_ASSERT(script);
+
+ CancelOffThreadIonCompile(script);
+
+ // Let the script warm up again before attempting another compile.
+ script->resetWarmUpCounterToDelayIonCompilation();
+
+ JitScript* jitScript = script->maybeJitScript();
+ if (!jitScript) {
+ return;
+ }
+
+ auto addPendingInvalidation = [&invalid](const RecompileInfo& info) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!invalid.append(info)) {
+ // BUG 1536159: For diagnostics, compute the size of the failed
+ // allocation. This presumes the vector growth strategy is to double. This
+ // is only used for crash reporting so not a problem if we get it wrong.
+ size_t allocSize = 2 * sizeof(RecompileInfo) * invalid.capacity();
+ oomUnsafe.crash(allocSize, "Could not update RecompileInfoVector");
+ }
+ };
+
+ // Trigger invalidation of the IonScript.
+ if (jitScript->hasIonScript()) {
+ RecompileInfo info(script, jitScript->ionScript()->compilationId());
+ addPendingInvalidation(info);
+ }
+
+ // Trigger invalidation of any callers inlining this script.
+ auto* inlinedCompilations =
+ script->zone()->jitZone()->maybeInlinedCompilations(script);
+ if (inlinedCompilations) {
+ for (const RecompileInfo& info : *inlinedCompilations) {
+ addPendingInvalidation(info);
+ }
+ script->zone()->jitZone()->removeInlinedCompilations(script);
+ }
+}
+
+IonScript* RecompileInfo::maybeIonScriptToInvalidate() const {
+ // Make sure this is not called under CodeGenerator::link (before the
+ // IonScript is created).
+ MOZ_ASSERT_IF(
+ script_->zone()->jitZone()->currentCompilationId(),
+ script_->zone()->jitZone()->currentCompilationId().ref() != id_);
+
+ if (!script_->hasIonScript() ||
+ script_->ionScript()->compilationId() != id_) {
+ return nullptr;
+ }
+
+ return script_->ionScript();
+}
+
+bool RecompileInfo::traceWeak(JSTracer* trc) {
+ // Sweep the RecompileInfo if either the script is dead or the IonScript has
+ // been invalidated.
+
+ if (!TraceManuallyBarrieredWeakEdge(trc, &script_, "RecompileInfo::script")) {
+ return false;
+ }
+
+ return maybeIonScriptToInvalidate() != nullptr;
+}
+
+void JitZone::traceWeak(JSTracer* trc) {
+ baselineCacheIRStubCodes_.traceWeak(trc);
+ inlinedCompilations_.traceWeak(trc);
+}
+
+size_t JitRealm::sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ return mallocSizeOf(this);
+}
+
+void JitZone::addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf,
+ JS::CodeSizes* code, size_t* jitZone,
+ size_t* baselineStubsOptimized) const {
+ *jitZone += mallocSizeOf(this);
+ *jitZone +=
+ baselineCacheIRStubCodes_.shallowSizeOfExcludingThis(mallocSizeOf);
+ *jitZone += ionCacheIRStubInfoSet_.shallowSizeOfExcludingThis(mallocSizeOf);
+
+ execAlloc().addSizeOfCode(code);
+
+ *baselineStubsOptimized +=
+ optimizedStubSpace_.sizeOfExcludingThis(mallocSizeOf);
+}
+
+void JitCodeHeader::init(JitCode* jitCode) {
+ // As long as JitCode isn't moveable, we can avoid tracing this and
+ // mutating executable data.
+ MOZ_ASSERT(!gc::IsMovableKind(gc::AllocKind::JITCODE));
+ jitCode_ = jitCode;
+}
+
+template <AllowGC allowGC>
+JitCode* JitCode::New(JSContext* cx, uint8_t* code, uint32_t totalSize,
+ uint32_t headerSize, ExecutablePool* pool,
+ CodeKind kind) {
+ uint32_t bufferSize = totalSize - headerSize;
+ JitCode* codeObj =
+ cx->newCell<JitCode, allowGC>(code, bufferSize, headerSize, pool, kind);
+ if (!codeObj) {
+ // The caller already allocated `totalSize` bytes of executable memory.
+ pool->release(totalSize, kind);
+ return nullptr;
+ }
+
+ cx->zone()->incJitMemory(totalSize);
+
+ return codeObj;
+}
+
+template JitCode* JitCode::New<CanGC>(JSContext* cx, uint8_t* code,
+ uint32_t bufferSize, uint32_t headerSize,
+ ExecutablePool* pool, CodeKind kind);
+
+template JitCode* JitCode::New<NoGC>(JSContext* cx, uint8_t* code,
+ uint32_t bufferSize, uint32_t headerSize,
+ ExecutablePool* pool, CodeKind kind);
+
+void JitCode::copyFrom(MacroAssembler& masm) {
+ // Store the JitCode pointer in the JitCodeHeader so we can recover the
+ // gcthing from relocation tables.
+ JitCodeHeader::FromExecutable(raw())->init(this);
+
+ insnSize_ = masm.instructionsSize();
+ masm.executableCopy(raw());
+
+ jumpRelocTableBytes_ = masm.jumpRelocationTableBytes();
+ masm.copyJumpRelocationTable(raw() + jumpRelocTableOffset());
+
+ dataRelocTableBytes_ = masm.dataRelocationTableBytes();
+ masm.copyDataRelocationTable(raw() + dataRelocTableOffset());
+
+ masm.processCodeLabels(raw());
+}
+
+void JitCode::traceChildren(JSTracer* trc) {
+ // Note that we cannot mark invalidated scripts, since we've basically
+ // corrupted the code stream by injecting bailouts.
+ if (invalidated()) {
+ return;
+ }
+
+ if (jumpRelocTableBytes_) {
+ uint8_t* start = raw() + jumpRelocTableOffset();
+ CompactBufferReader reader(start, start + jumpRelocTableBytes_);
+ MacroAssembler::TraceJumpRelocations(trc, this, reader);
+ }
+ if (dataRelocTableBytes_) {
+ uint8_t* start = raw() + dataRelocTableOffset();
+ CompactBufferReader reader(start, start + dataRelocTableBytes_);
+ MacroAssembler::TraceDataRelocations(trc, this, reader);
+ }
+}
+
+void JitCode::finalize(JS::GCContext* gcx) {
+ // If this jitcode had a bytecode map, it must have already been removed.
+#ifdef DEBUG
+ JSRuntime* rt = gcx->runtime();
+ if (hasBytecodeMap_) {
+ MOZ_ASSERT(rt->jitRuntime()->hasJitcodeGlobalTable());
+ MOZ_ASSERT(!rt->jitRuntime()->getJitcodeGlobalTable()->lookup(raw()));
+ }
+#endif
+
+#ifdef MOZ_VTUNE
+ vtune::UnmarkCode(this);
+#endif
+
+ MOZ_ASSERT(pool_);
+
+ // With W^X JIT code, reprotecting memory for each JitCode instance is
+ // slow, so we record the ranges and poison them later all at once. It's
+ // safe to ignore OOM here, it just means we won't poison the code.
+ if (gcx->appendJitPoisonRange(JitPoisonRange(pool_, raw() - headerSize_,
+ headerSize_ + bufferSize_))) {
+ pool_->addRef();
+ }
+ setHeaderPtr(nullptr);
+
+#ifdef JS_ION_PERF
+ // Code buffers are stored inside ExecutablePools. Pools are refcounted.
+ // Releasing the pool may free it. Horrible hack: if we are using perf
+ // integration, we don't want to reuse code addresses, so we just leak the
+ // memory instead.
+ if (!PerfEnabled()) {
+ pool_->release(headerSize_ + bufferSize_, CodeKind(kind_));
+ }
+#else
+ pool_->release(headerSize_ + bufferSize_, CodeKind(kind_));
+#endif
+
+ zone()->decJitMemory(headerSize_ + bufferSize_);
+
+ pool_ = nullptr;
+}
+
+IonScript::IonScript(IonCompilationId compilationId, uint32_t localSlotsSize,
+ uint32_t argumentSlotsSize, uint32_t frameSize)
+ : localSlotsSize_(localSlotsSize),
+ argumentSlotsSize_(argumentSlotsSize),
+ frameSize_(frameSize),
+ compilationId_(compilationId) {}
+
+IonScript* IonScript::New(JSContext* cx, IonCompilationId compilationId,
+ uint32_t localSlotsSize, uint32_t argumentSlotsSize,
+ uint32_t frameSize, size_t snapshotsListSize,
+ size_t snapshotsRVATableSize, size_t recoversSize,
+ size_t constants, size_t nurseryObjects,
+ size_t safepointIndices, size_t osiIndices,
+ size_t icEntries, size_t runtimeSize,
+ size_t safepointsSize) {
+ if (snapshotsListSize >= MAX_BUFFER_SIZE) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ // Verify the hardcoded sizes in header are accurate.
+ static_assert(SizeOf_OsiIndex == sizeof(OsiIndex),
+ "IonScript has wrong size for OsiIndex");
+ static_assert(SizeOf_SafepointIndex == sizeof(SafepointIndex),
+ "IonScript has wrong size for SafepointIndex");
+
+ CheckedInt<Offset> allocSize = sizeof(IonScript);
+ allocSize += CheckedInt<Offset>(constants) * sizeof(Value);
+ allocSize += CheckedInt<Offset>(runtimeSize);
+ allocSize += CheckedInt<Offset>(nurseryObjects) * sizeof(HeapPtr<JSObject*>);
+ allocSize += CheckedInt<Offset>(osiIndices) * sizeof(OsiIndex);
+ allocSize += CheckedInt<Offset>(safepointIndices) * sizeof(SafepointIndex);
+ allocSize += CheckedInt<Offset>(icEntries) * sizeof(uint32_t);
+ allocSize += CheckedInt<Offset>(safepointsSize);
+ allocSize += CheckedInt<Offset>(snapshotsListSize);
+ allocSize += CheckedInt<Offset>(snapshotsRVATableSize);
+ allocSize += CheckedInt<Offset>(recoversSize);
+
+ if (!allocSize.isValid()) {
+ ReportAllocationOverflow(cx);
+ return nullptr;
+ }
+
+ void* raw = cx->pod_malloc<uint8_t>(allocSize.value());
+ MOZ_ASSERT(uintptr_t(raw) % alignof(IonScript) == 0);
+ if (!raw) {
+ return nullptr;
+ }
+ IonScript* script = new (raw)
+ IonScript(compilationId, localSlotsSize, argumentSlotsSize, frameSize);
+
+ Offset offsetCursor = sizeof(IonScript);
+
+ MOZ_ASSERT(offsetCursor % alignof(Value) == 0);
+ script->constantTableOffset_ = offsetCursor;
+ offsetCursor += constants * sizeof(Value);
+
+ MOZ_ASSERT(offsetCursor % alignof(uint64_t) == 0);
+ script->runtimeDataOffset_ = offsetCursor;
+ offsetCursor += runtimeSize;
+
+ MOZ_ASSERT(offsetCursor % alignof(HeapPtr<JSObject*>) == 0);
+ script->initElements<HeapPtr<JSObject*>>(offsetCursor, nurseryObjects);
+ script->nurseryObjectsOffset_ = offsetCursor;
+ offsetCursor += nurseryObjects * sizeof(HeapPtr<JSObject*>);
+
+ MOZ_ASSERT(offsetCursor % alignof(OsiIndex) == 0);
+ script->osiIndexOffset_ = offsetCursor;
+ offsetCursor += osiIndices * sizeof(OsiIndex);
+
+ MOZ_ASSERT(offsetCursor % alignof(SafepointIndex) == 0);
+ script->safepointIndexOffset_ = offsetCursor;
+ offsetCursor += safepointIndices * sizeof(SafepointIndex);
+
+ MOZ_ASSERT(offsetCursor % alignof(uint32_t) == 0);
+ script->icIndexOffset_ = offsetCursor;
+ offsetCursor += icEntries * sizeof(uint32_t);
+
+ script->safepointsOffset_ = offsetCursor;
+ offsetCursor += safepointsSize;
+
+ script->snapshotsOffset_ = offsetCursor;
+ offsetCursor += snapshotsListSize;
+
+ script->rvaTableOffset_ = offsetCursor;
+ offsetCursor += snapshotsRVATableSize;
+
+ script->recoversOffset_ = offsetCursor;
+ offsetCursor += recoversSize;
+
+ script->allocBytes_ = offsetCursor;
+
+ MOZ_ASSERT(script->numConstants() == constants);
+ MOZ_ASSERT(script->runtimeSize() == runtimeSize);
+ MOZ_ASSERT(script->numNurseryObjects() == nurseryObjects);
+ MOZ_ASSERT(script->numOsiIndices() == osiIndices);
+ MOZ_ASSERT(script->numSafepointIndices() == safepointIndices);
+ MOZ_ASSERT(script->numICs() == icEntries);
+ MOZ_ASSERT(script->safepointsSize() == safepointsSize);
+ MOZ_ASSERT(script->snapshotsListSize() == snapshotsListSize);
+ MOZ_ASSERT(script->snapshotsRVATableSize() == snapshotsRVATableSize);
+ MOZ_ASSERT(script->recoversSize() == recoversSize);
+ MOZ_ASSERT(script->endOffset() == offsetCursor);
+
+ return script;
+}
+
+void IonScript::trace(JSTracer* trc) {
+ if (method_) {
+ TraceEdge(trc, &method_, "method");
+ }
+
+ for (size_t i = 0; i < numConstants(); i++) {
+ TraceEdge(trc, &getConstant(i), "constant");
+ }
+
+ for (size_t i = 0; i < numNurseryObjects(); i++) {
+ TraceEdge(trc, &nurseryObjects()[i], "nursery-object");
+ }
+
+ // Trace caches so that the JSScript pointer can be updated if moved.
+ for (size_t i = 0; i < numICs(); i++) {
+ getICFromIndex(i).trace(trc, this);
+ }
+}
+
+/* static */
+void IonScript::preWriteBarrier(Zone* zone, IonScript* ionScript) {
+ PreWriteBarrier(zone, ionScript);
+}
+
+void IonScript::copySnapshots(const SnapshotWriter* writer) {
+ MOZ_ASSERT(writer->listSize() == snapshotsListSize());
+ memcpy(offsetToPointer<uint8_t>(snapshotsOffset()), writer->listBuffer(),
+ snapshotsListSize());
+
+ MOZ_ASSERT(snapshotsRVATableSize());
+ MOZ_ASSERT(writer->RVATableSize() == snapshotsRVATableSize());
+ memcpy(offsetToPointer<uint8_t>(rvaTableOffset()), writer->RVATableBuffer(),
+ snapshotsRVATableSize());
+}
+
+void IonScript::copyRecovers(const RecoverWriter* writer) {
+ MOZ_ASSERT(writer->size() == recoversSize());
+ memcpy(offsetToPointer<uint8_t>(recoversOffset()), writer->buffer(),
+ recoversSize());
+}
+
+void IonScript::copySafepoints(const SafepointWriter* writer) {
+ MOZ_ASSERT(writer->size() == safepointsSize());
+ memcpy(offsetToPointer<uint8_t>(safepointsOffset()), writer->buffer(),
+ safepointsSize());
+}
+
+void IonScript::copyConstants(const Value* vp) {
+ for (size_t i = 0; i < numConstants(); i++) {
+ constants()[i].init(vp[i]);
+ }
+}
+
+void IonScript::copySafepointIndices(const CodegenSafepointIndex* si) {
+ // Convert CodegenSafepointIndex to more compact form.
+ SafepointIndex* table = safepointIndices();
+ for (size_t i = 0; i < numSafepointIndices(); ++i) {
+ table[i] = SafepointIndex(si[i]);
+ }
+}
+
+void IonScript::copyOsiIndices(const OsiIndex* oi) {
+ memcpy(osiIndices(), oi, numOsiIndices() * sizeof(OsiIndex));
+}
+
+void IonScript::copyRuntimeData(const uint8_t* data) {
+ memcpy(runtimeData(), data, runtimeSize());
+}
+
+void IonScript::copyICEntries(const uint32_t* icEntries) {
+ memcpy(icIndex(), icEntries, numICs() * sizeof(uint32_t));
+
+ // Update the codeRaw_ field in the ICs now that we know the code address.
+ for (size_t i = 0; i < numICs(); i++) {
+ getICFromIndex(i).resetCodeRaw(this);
+ }
+}
+
+const SafepointIndex* IonScript::getSafepointIndex(uint32_t disp) const {
+ MOZ_ASSERT(numSafepointIndices() > 0);
+
+ const SafepointIndex* table = safepointIndices();
+ if (numSafepointIndices() == 1) {
+ MOZ_ASSERT(disp == table[0].displacement());
+ return &table[0];
+ }
+
+ size_t minEntry = 0;
+ size_t maxEntry = numSafepointIndices() - 1;
+ uint32_t min = table[minEntry].displacement();
+ uint32_t max = table[maxEntry].displacement();
+
+ // Raise if the element is not in the list.
+ MOZ_ASSERT(min <= disp && disp <= max);
+
+ // Approximate the location of the FrameInfo.
+ size_t guess = (disp - min) * (maxEntry - minEntry) / (max - min) + minEntry;
+ uint32_t guessDisp = table[guess].displacement();
+
+ if (table[guess].displacement() == disp) {
+ return &table[guess];
+ }
+
+ // Doing a linear scan from the guess should be more efficient in case of
+ // small group which are equally distributed on the code.
+ //
+ // such as: <... ... ... ... . ... ...>
+ if (guessDisp > disp) {
+ while (--guess >= minEntry) {
+ guessDisp = table[guess].displacement();
+ MOZ_ASSERT(guessDisp >= disp);
+ if (guessDisp == disp) {
+ return &table[guess];
+ }
+ }
+ } else {
+ while (++guess <= maxEntry) {
+ guessDisp = table[guess].displacement();
+ MOZ_ASSERT(guessDisp <= disp);
+ if (guessDisp == disp) {
+ return &table[guess];
+ }
+ }
+ }
+
+ MOZ_CRASH("displacement not found.");
+}
+
+const OsiIndex* IonScript::getOsiIndex(uint32_t disp) const {
+ const OsiIndex* end = osiIndices() + numOsiIndices();
+ for (const OsiIndex* it = osiIndices(); it != end; ++it) {
+ if (it->returnPointDisplacement() == disp) {
+ return it;
+ }
+ }
+
+ MOZ_CRASH("Failed to find OSI point return address");
+}
+
+const OsiIndex* IonScript::getOsiIndex(uint8_t* retAddr) const {
+ JitSpew(JitSpew_IonInvalidate, "IonScript %p has method %p raw %p",
+ (void*)this, (void*)method(), method()->raw());
+
+ MOZ_ASSERT(containsCodeAddress(retAddr));
+ uint32_t disp = retAddr - method()->raw();
+ return getOsiIndex(disp);
+}
+
+void IonScript::Destroy(JS::GCContext* gcx, IonScript* script) {
+ // Make sure there are no pointers into the IonScript's nursery objects list
+ // in the store buffer. Because this can be called during sweeping when
+ // discarding JIT code, we have to lock the store buffer when we find an
+ // object that's (still) in the nursery.
+ mozilla::Maybe<gc::AutoLockStoreBuffer> lock;
+ for (size_t i = 0, len = script->numNurseryObjects(); i < len; i++) {
+ JSObject* obj = script->nurseryObjects()[i];
+ if (!IsInsideNursery(obj)) {
+ continue;
+ }
+ if (lock.isNothing()) {
+ lock.emplace(&gcx->runtime()->gc.storeBuffer());
+ }
+ script->nurseryObjects()[i] = HeapPtr<JSObject*>();
+ }
+
+ // This allocation is tracked by JSScript::setIonScriptImpl.
+ gcx->deleteUntracked(script);
+}
+
+void JS::DeletePolicy<js::jit::IonScript>::operator()(
+ const js::jit::IonScript* script) {
+ IonScript::Destroy(rt_->gcContext(), const_cast<IonScript*>(script));
+}
+
+void IonScript::purgeICs(Zone* zone) {
+ for (size_t i = 0; i < numICs(); i++) {
+ getICFromIndex(i).reset(zone, this);
+ }
+}
+
+namespace js {
+namespace jit {
+
+bool OptimizeMIR(MIRGenerator* mir) {
+ MIRGraph& graph = mir->graph();
+ GraphSpewer& gs = mir->graphSpewer();
+
+ if (mir->shouldCancel("Start")) {
+ return false;
+ }
+
+ gs.spewPass("BuildSSA");
+ AssertBasicGraphCoherency(graph);
+
+ if (JitSpewEnabled(JitSpew_MIRExpressions)) {
+ JitSpewCont(JitSpew_MIRExpressions, "\n");
+ DumpMIRExpressions(JitSpewPrinter(), graph, mir->outerInfo(),
+ "BuildSSA (== input to OptimizeMIR)");
+ }
+
+ if (!JitOptions.disablePruning && !mir->compilingWasm()) {
+ JitSpewCont(JitSpew_Prune, "\n");
+ if (!PruneUnusedBranches(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("Prune Unused Branches");
+ AssertBasicGraphCoherency(graph);
+
+ if (mir->shouldCancel("Prune Unused Branches")) {
+ return false;
+ }
+ }
+
+ {
+ if (!FoldEmptyBlocks(graph)) {
+ return false;
+ }
+ gs.spewPass("Fold Empty Blocks");
+ AssertBasicGraphCoherency(graph);
+
+ if (mir->shouldCancel("Fold Empty Blocks")) {
+ return false;
+ }
+ }
+
+ // Remove trivially dead resume point operands before folding tests, so the
+ // latter pass can optimize more aggressively.
+ if (!mir->compilingWasm()) {
+ if (!EliminateTriviallyDeadResumePointOperands(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("Eliminate trivially dead resume point operands");
+ AssertBasicGraphCoherency(graph);
+
+ if (mir->shouldCancel("Eliminate trivially dead resume point operands")) {
+ return false;
+ }
+ }
+
+ {
+ if (!FoldTests(graph)) {
+ return false;
+ }
+ gs.spewPass("Fold Tests");
+ AssertBasicGraphCoherency(graph);
+
+ if (mir->shouldCancel("Fold Tests")) {
+ return false;
+ }
+ }
+
+ {
+ if (!SplitCriticalEdges(graph)) {
+ return false;
+ }
+ gs.spewPass("Split Critical Edges");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Split Critical Edges")) {
+ return false;
+ }
+ }
+
+ {
+ RenumberBlocks(graph);
+ gs.spewPass("Renumber Blocks");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Renumber Blocks")) {
+ return false;
+ }
+ }
+
+ {
+ if (!BuildDominatorTree(graph)) {
+ return false;
+ }
+ // No spew: graph not changed.
+
+ if (mir->shouldCancel("Dominator Tree")) {
+ return false;
+ }
+ }
+
+ {
+ // Aggressive phi elimination must occur before any code elimination. If the
+ // script contains a try-statement, we only compiled the try block and not
+ // the catch or finally blocks, so in this case it's also invalid to use
+ // aggressive phi elimination.
+ Observability observability = graph.hasTryBlock()
+ ? ConservativeObservability
+ : AggressiveObservability;
+ if (!EliminatePhis(mir, graph, observability)) {
+ return false;
+ }
+ gs.spewPass("Eliminate phis");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Eliminate phis")) {
+ return false;
+ }
+
+ if (!BuildPhiReverseMapping(graph)) {
+ return false;
+ }
+ AssertExtendedGraphCoherency(graph);
+ // No spew: graph not changed.
+
+ if (mir->shouldCancel("Phi reverse mapping")) {
+ return false;
+ }
+ }
+
+ if (!JitOptions.disableRecoverIns &&
+ mir->optimizationInfo().scalarReplacementEnabled()) {
+ JitSpewCont(JitSpew_Escape, "\n");
+ if (!ScalarReplacement(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("Scalar Replacement");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Scalar Replacement")) {
+ return false;
+ }
+ }
+
+ if (!mir->compilingWasm()) {
+ if (!ApplyTypeInformation(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("Apply types");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Apply types")) {
+ return false;
+ }
+ }
+
+ if (mir->optimizationInfo().amaEnabled()) {
+ AlignmentMaskAnalysis ama(graph);
+ if (!ama.analyze()) {
+ return false;
+ }
+ gs.spewPass("Alignment Mask Analysis");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Alignment Mask Analysis")) {
+ return false;
+ }
+ }
+
+ ValueNumberer gvn(mir, graph);
+
+ // Alias analysis is required for LICM and GVN so that we don't move
+ // loads across stores. We also use alias information when removing
+ // redundant shapeguards.
+ if (mir->optimizationInfo().licmEnabled() ||
+ mir->optimizationInfo().gvnEnabled() ||
+ mir->optimizationInfo().eliminateRedundantShapeGuardsEnabled()) {
+ {
+ AliasAnalysis analysis(mir, graph);
+ JitSpewCont(JitSpew_Alias, "\n");
+ if (!analysis.analyze()) {
+ return false;
+ }
+
+ gs.spewPass("Alias analysis");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Alias analysis")) {
+ return false;
+ }
+ }
+
+ if (!mir->compilingWasm()) {
+ // Eliminating dead resume point operands requires basic block
+ // instructions to be numbered. Reuse the numbering computed during
+ // alias analysis.
+ if (!EliminateDeadResumePointOperands(mir, graph)) {
+ return false;
+ }
+
+ gs.spewPass("Eliminate dead resume point operands");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Eliminate dead resume point operands")) {
+ return false;
+ }
+ }
+ }
+
+ if (mir->optimizationInfo().gvnEnabled()) {
+ JitSpewCont(JitSpew_GVN, "\n");
+ if (!gvn.run(ValueNumberer::UpdateAliasAnalysis)) {
+ return false;
+ }
+ gs.spewPass("GVN");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("GVN")) {
+ return false;
+ }
+ }
+
+ // LICM can hoist instructions from conditional branches and
+ // trigger bailouts. Disable it if bailing out of a hoisted
+ // instruction has previously invalidated this script.
+ if (mir->optimizationInfo().licmEnabled() &&
+ !mir->outerInfo().hadLICMInvalidation()) {
+ JitSpewCont(JitSpew_LICM, "\n");
+ if (!LICM(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("LICM");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("LICM")) {
+ return false;
+ }
+ }
+
+ RangeAnalysis r(mir, graph);
+ if (mir->optimizationInfo().rangeAnalysisEnabled()) {
+ JitSpewCont(JitSpew_Range, "\n");
+ if (!r.addBetaNodes()) {
+ return false;
+ }
+ gs.spewPass("Beta");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("RA Beta")) {
+ return false;
+ }
+
+ if (!r.analyze() || !r.addRangeAssertions()) {
+ return false;
+ }
+ gs.spewPass("Range Analysis");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Range Analysis")) {
+ return false;
+ }
+
+ if (!r.removeBetaNodes()) {
+ return false;
+ }
+ gs.spewPass("De-Beta");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("RA De-Beta")) {
+ return false;
+ }
+
+ if (mir->optimizationInfo().gvnEnabled()) {
+ bool shouldRunUCE = false;
+ if (!r.prepareForUCE(&shouldRunUCE)) {
+ return false;
+ }
+ gs.spewPass("RA check UCE");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("RA check UCE")) {
+ return false;
+ }
+
+ if (shouldRunUCE) {
+ if (!gvn.run(ValueNumberer::DontUpdateAliasAnalysis)) {
+ return false;
+ }
+ gs.spewPass("UCE After RA");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("UCE After RA")) {
+ return false;
+ }
+ }
+ }
+
+ if (mir->optimizationInfo().autoTruncateEnabled()) {
+ if (!r.truncate()) {
+ return false;
+ }
+ gs.spewPass("Truncate Doubles");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Truncate Doubles")) {
+ return false;
+ }
+ }
+ }
+
+ if (!JitOptions.disableRecoverIns) {
+ JitSpewCont(JitSpew_Sink, "\n");
+ if (!Sink(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("Sink");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Sink")) {
+ return false;
+ }
+ }
+
+ if (!JitOptions.disableRecoverIns &&
+ mir->optimizationInfo().rangeAnalysisEnabled()) {
+ JitSpewCont(JitSpew_Range, "\n");
+ if (!r.removeUnnecessaryBitops()) {
+ return false;
+ }
+ gs.spewPass("Remove Unnecessary Bitops");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Remove Unnecessary Bitops")) {
+ return false;
+ }
+ }
+
+ {
+ JitSpewCont(JitSpew_FLAC, "\n");
+ if (!FoldLinearArithConstants(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("Fold Linear Arithmetic Constants");
+ AssertBasicGraphCoherency(graph);
+
+ if (mir->shouldCancel("Fold Linear Arithmetic Constants")) {
+ return false;
+ }
+ }
+
+ if (mir->optimizationInfo().eaaEnabled()) {
+ EffectiveAddressAnalysis eaa(mir, graph);
+ JitSpewCont(JitSpew_EAA, "\n");
+ if (!eaa.analyze()) {
+ return false;
+ }
+ gs.spewPass("Effective Address Analysis");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Effective Address Analysis")) {
+ return false;
+ }
+ }
+
+ // BCE marks bounds checks as dead, so do BCE before DCE.
+ if (mir->compilingWasm()) {
+ JitSpewCont(JitSpew_WasmBCE, "\n");
+ if (!EliminateBoundsChecks(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("Redundant Bounds Check Elimination");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("BCE")) {
+ return false;
+ }
+ }
+
+ {
+ if (!EliminateDeadCode(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("DCE");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("DCE")) {
+ return false;
+ }
+ }
+
+ if (mir->optimizationInfo().instructionReorderingEnabled() &&
+ !mir->outerInfo().hadReorderingBailout()) {
+ if (!ReorderInstructions(graph)) {
+ return false;
+ }
+ gs.spewPass("Reordering");
+
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Reordering")) {
+ return false;
+ }
+ }
+
+ // Make loops contiguous. We do this after GVN/UCE and range analysis,
+ // which can remove CFG edges, exposing more blocks that can be moved.
+ {
+ if (!MakeLoopsContiguous(graph)) {
+ return false;
+ }
+ gs.spewPass("Make loops contiguous");
+ AssertExtendedGraphCoherency(graph);
+
+ if (mir->shouldCancel("Make loops contiguous")) {
+ return false;
+ }
+ }
+ AssertExtendedGraphCoherency(graph, /* underValueNumberer = */ false,
+ /* force = */ true);
+
+ // Remove unreachable blocks created by MBasicBlock::NewFakeLoopPredecessor
+ // to ensure every loop header has two predecessors. (This only happens due
+ // to OSR.) After this point, it is no longer possible to build the
+ // dominator tree.
+ if (!mir->compilingWasm() && graph.osrBlock()) {
+ graph.removeFakeLoopPredecessors();
+ gs.spewPass("Remove fake loop predecessors");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Remove fake loop predecessors")) {
+ return false;
+ }
+ }
+
+ // Passes after this point must not move instructions; these analyses
+ // depend on knowing the final order in which instructions will execute.
+
+ if (mir->optimizationInfo().edgeCaseAnalysisEnabled()) {
+ EdgeCaseAnalysis edgeCaseAnalysis(mir, graph);
+ if (!edgeCaseAnalysis.analyzeLate()) {
+ return false;
+ }
+ gs.spewPass("Edge Case Analysis (Late)");
+ AssertGraphCoherency(graph);
+
+ if (mir->shouldCancel("Edge Case Analysis (Late)")) {
+ return false;
+ }
+ }
+
+ if (mir->optimizationInfo().eliminateRedundantChecksEnabled()) {
+ // Note: check elimination has to run after all other passes that move
+ // instructions. Since check uses are replaced with the actual index,
+ // code motion after this pass could incorrectly move a load or store
+ // before its bounds check.
+ if (!EliminateRedundantChecks(graph)) {
+ return false;
+ }
+ gs.spewPass("Bounds Check Elimination");
+ AssertGraphCoherency(graph);
+ }
+
+ if (mir->optimizationInfo().eliminateRedundantShapeGuardsEnabled()) {
+ if (!EliminateRedundantShapeGuards(graph)) {
+ return false;
+ }
+ gs.spewPass("Shape Guard Elimination");
+ AssertGraphCoherency(graph);
+ }
+
+ if (!mir->compilingWasm() && !mir->outerInfo().hadUnboxFoldingBailout()) {
+ if (!FoldLoadsWithUnbox(mir, graph)) {
+ return false;
+ }
+ gs.spewPass("FoldLoadsWithUnbox");
+ AssertGraphCoherency(graph);
+ }
+
+ if (!mir->compilingWasm()) {
+ if (!AddKeepAliveInstructions(graph)) {
+ return false;
+ }
+ gs.spewPass("Add KeepAlive Instructions");
+ AssertGraphCoherency(graph);
+ }
+
+ AssertGraphCoherency(graph, /* force = */ true);
+
+ if (JitSpewEnabled(JitSpew_MIRExpressions)) {
+ JitSpewCont(JitSpew_MIRExpressions, "\n");
+ DumpMIRExpressions(JitSpewPrinter(), graph, mir->outerInfo(),
+ "BeforeLIR (== result of OptimizeMIR)");
+ }
+
+ return true;
+}
+
+LIRGraph* GenerateLIR(MIRGenerator* mir) {
+ MIRGraph& graph = mir->graph();
+ GraphSpewer& gs = mir->graphSpewer();
+
+ LIRGraph* lir = mir->alloc().lifoAlloc()->new_<LIRGraph>(&graph);
+ if (!lir || !lir->init()) {
+ return nullptr;
+ }
+
+ LIRGenerator lirgen(mir, graph, *lir);
+ {
+ if (!lirgen.generate()) {
+ return nullptr;
+ }
+ gs.spewPass("Generate LIR");
+
+ if (mir->shouldCancel("Generate LIR")) {
+ return nullptr;
+ }
+ }
+
+#ifdef DEBUG
+ AllocationIntegrityState integrity(*lir);
+#endif
+
+ {
+ IonRegisterAllocator allocator =
+ mir->optimizationInfo().registerAllocator();
+
+ switch (allocator) {
+ case RegisterAllocator_Backtracking:
+ case RegisterAllocator_Testbed: {
+#ifdef DEBUG
+ if (JitOptions.fullDebugChecks) {
+ if (!integrity.record()) {
+ return nullptr;
+ }
+ }
+#endif
+
+ BacktrackingAllocator regalloc(mir, &lirgen, *lir,
+ allocator == RegisterAllocator_Testbed);
+ if (!regalloc.go()) {
+ return nullptr;
+ }
+
+#ifdef DEBUG
+ if (JitOptions.fullDebugChecks) {
+ if (!integrity.check()) {
+ return nullptr;
+ }
+ }
+#endif
+
+ gs.spewPass("Allocate Registers [Backtracking]");
+ break;
+ }
+
+ default:
+ MOZ_CRASH("Bad regalloc");
+ }
+
+ if (mir->shouldCancel("Allocate Registers")) {
+ return nullptr;
+ }
+ }
+
+ return lir;
+}
+
+CodeGenerator* GenerateCode(MIRGenerator* mir, LIRGraph* lir) {
+ auto codegen = MakeUnique<CodeGenerator>(mir, lir);
+ if (!codegen) {
+ return nullptr;
+ }
+
+ if (!codegen->generate()) {
+ return nullptr;
+ }
+
+ return codegen.release();
+}
+
+CodeGenerator* CompileBackEnd(MIRGenerator* mir, WarpSnapshot* snapshot) {
+ // Everything in CompileBackEnd can potentially run on a helper thread.
+ AutoEnterIonBackend enter;
+ AutoSpewEndFunction spewEndFunction(mir);
+
+ {
+ WarpCompilation comp(mir->alloc());
+ WarpBuilder builder(*snapshot, *mir, &comp);
+ if (!builder.build()) {
+ return nullptr;
+ }
+ }
+
+ if (!OptimizeMIR(mir)) {
+ return nullptr;
+ }
+
+ LIRGraph* lir = GenerateLIR(mir);
+ if (!lir) {
+ return nullptr;
+ }
+
+ return GenerateCode(mir, lir);
+}
+
+static AbortReasonOr<WarpSnapshot*> CreateWarpSnapshot(JSContext* cx,
+ MIRGenerator* mirGen,
+ HandleScript script) {
+ // Suppress GC during compilation.
+ gc::AutoSuppressGC suppressGC(cx);
+
+ SpewBeginFunction(mirGen, script);
+
+ WarpOracle oracle(cx, *mirGen, script);
+
+ AbortReasonOr<WarpSnapshot*> result = oracle.createSnapshot();
+
+ MOZ_ASSERT_IF(result.isErr(), result.unwrapErr() == AbortReason::Alloc ||
+ result.unwrapErr() == AbortReason::Error ||
+ result.unwrapErr() == AbortReason::Disable);
+ MOZ_ASSERT_IF(!result.isErr(), result.unwrap());
+
+ return result;
+}
+
+static AbortReason IonCompile(JSContext* cx, HandleScript script,
+ jsbytecode* osrPc) {
+ cx->check(script);
+
+ auto alloc =
+ cx->make_unique<LifoAlloc>(TempAllocator::PreferredLifoChunkSize);
+ if (!alloc) {
+ return AbortReason::Error;
+ }
+
+ if (!cx->realm()->ensureJitRealmExists(cx)) {
+ return AbortReason::Error;
+ }
+
+ if (!cx->realm()->jitRealm()->ensureIonStubsExist(cx)) {
+ return AbortReason::Error;
+ }
+
+ TempAllocator* temp = alloc->new_<TempAllocator>(alloc.get());
+ if (!temp) {
+ return AbortReason::Alloc;
+ }
+
+ MIRGraph* graph = alloc->new_<MIRGraph>(temp);
+ if (!graph) {
+ return AbortReason::Alloc;
+ }
+
+ InlineScriptTree* inlineScriptTree =
+ InlineScriptTree::New(temp, nullptr, nullptr, script);
+ if (!inlineScriptTree) {
+ return AbortReason::Alloc;
+ }
+
+ CompileInfo* info = alloc->new_<CompileInfo>(
+ CompileRuntime::get(cx->runtime()), script, script->function(), osrPc,
+ script->needsArgsObj(), inlineScriptTree);
+ if (!info) {
+ return AbortReason::Alloc;
+ }
+
+ const OptimizationInfo* optimizationInfo =
+ IonOptimizations.get(OptimizationLevel::Normal);
+ const JitCompileOptions options(cx);
+
+ MIRGenerator* mirGen =
+ alloc->new_<MIRGenerator>(CompileRealm::get(cx->realm()), options, temp,
+ graph, info, optimizationInfo);
+ if (!mirGen) {
+ return AbortReason::Alloc;
+ }
+
+ MOZ_ASSERT(!script->baselineScript()->hasPendingIonCompileTask());
+ MOZ_ASSERT(!script->hasIonScript());
+ MOZ_ASSERT(script->canIonCompile());
+
+ if (osrPc) {
+ script->jitScript()->setHadIonOSR();
+ }
+
+ AbortReasonOr<WarpSnapshot*> result = CreateWarpSnapshot(cx, mirGen, script);
+ if (result.isErr()) {
+ return result.unwrapErr();
+ }
+ WarpSnapshot* snapshot = result.unwrap();
+
+ // If possible, compile the script off thread.
+ if (options.offThreadCompilationAvailable()) {
+ JitSpew(JitSpew_IonSyncLogs,
+ "Can't log script %s:%u:%u"
+ ". (Compiled on background thread.)",
+ script->filename(), script->lineno(), script->column());
+
+ IonCompileTask* task = alloc->new_<IonCompileTask>(cx, *mirGen, snapshot);
+ if (!task) {
+ return AbortReason::Alloc;
+ }
+
+ AutoLockHelperThreadState lock;
+ if (!StartOffThreadIonCompile(task, lock)) {
+ JitSpew(JitSpew_IonAbort, "Unable to start off-thread ion compilation.");
+ mirGen->graphSpewer().endFunction();
+ return AbortReason::Alloc;
+ }
+
+ script->jitScript()->setIsIonCompilingOffThread(script);
+
+ // The allocator and associated data will be destroyed after being
+ // processed in the finishedOffThreadCompilations list.
+ (void)alloc.release();
+
+ return AbortReason::NoAbort;
+ }
+
+ bool succeeded = false;
+ {
+ gc::AutoSuppressGC suppressGC(cx);
+ JitContext jctx(cx);
+ UniquePtr<CodeGenerator> codegen(CompileBackEnd(mirGen, snapshot));
+ if (!codegen) {
+ JitSpew(JitSpew_IonAbort, "Failed during back-end compilation.");
+ if (cx->isExceptionPending()) {
+ return AbortReason::Error;
+ }
+ return AbortReason::Disable;
+ }
+
+ succeeded = LinkCodeGen(cx, codegen.get(), script, snapshot);
+ }
+
+ if (succeeded) {
+ return AbortReason::NoAbort;
+ }
+ if (cx->isExceptionPending()) {
+ return AbortReason::Error;
+ }
+ return AbortReason::Disable;
+}
+
+static bool CheckFrame(JSContext* cx, BaselineFrame* frame) {
+ MOZ_ASSERT(!frame->isDebuggerEvalFrame());
+ MOZ_ASSERT(!frame->isEvalFrame());
+
+ // This check is to not overrun the stack.
+ if (frame->isFunctionFrame()) {
+ if (TooManyActualArguments(frame->numActualArgs())) {
+ JitSpew(JitSpew_IonAbort, "too many actual arguments");
+ return false;
+ }
+
+ if (TooManyFormalArguments(frame->numFormalArgs())) {
+ JitSpew(JitSpew_IonAbort, "too many arguments");
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool CanIonCompileOrInlineScript(JSScript* script, const char** reason) {
+ if (script->isForEval()) {
+ // Eval frames are not yet supported. Supporting this will require new
+ // logic in pushBailoutFrame to deal with linking prev.
+ // Additionally, JSOp::GlobalOrEvalDeclInstantiation support will require
+ // baking in isEvalFrame().
+ *reason = "eval script";
+ return false;
+ }
+
+ if (script->isAsync()) {
+ if (script->isModule()) {
+ *reason = "async module";
+ return false;
+ }
+ }
+
+ if (script->hasNonSyntacticScope() && !script->function()) {
+ // Support functions with a non-syntactic global scope but not other
+ // scripts. For global scripts, WarpBuilder currently uses the global
+ // object as scope chain, this is not valid when the script has a
+ // non-syntactic global scope.
+ *reason = "has non-syntactic global scope";
+ return false;
+ }
+
+ return true;
+} // namespace jit
+
+static bool ScriptIsTooLarge(JSContext* cx, JSScript* script) {
+ if (!JitOptions.limitScriptSize) {
+ return false;
+ }
+
+ size_t numLocalsAndArgs = NumLocalsAndArgs(script);
+
+ bool canCompileOffThread = OffThreadCompilationAvailable(cx);
+ size_t maxScriptSize = canCompileOffThread
+ ? JitOptions.ionMaxScriptSize
+ : JitOptions.ionMaxScriptSizeMainThread;
+ size_t maxLocalsAndArgs = canCompileOffThread
+ ? JitOptions.ionMaxLocalsAndArgs
+ : JitOptions.ionMaxLocalsAndArgsMainThread;
+
+ if (script->length() > maxScriptSize || numLocalsAndArgs > maxLocalsAndArgs) {
+ JitSpew(JitSpew_IonAbort,
+ "Script too large (%zu bytes) (%zu locals/args) @ %s:%u:%u",
+ script->length(), numLocalsAndArgs, script->filename(),
+ script->lineno(), script->column());
+ return true;
+ }
+
+ return false;
+}
+
+bool CanIonCompileScript(JSContext* cx, JSScript* script) {
+ if (!script->canIonCompile()) {
+ return false;
+ }
+
+ const char* reason = nullptr;
+ if (!CanIonCompileOrInlineScript(script, &reason)) {
+ JitSpew(JitSpew_IonAbort, "%s", reason);
+ return false;
+ }
+
+ if (ScriptIsTooLarge(cx, script)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool CanIonInlineScript(JSScript* script) {
+ if (!script->canIonCompile()) {
+ return false;
+ }
+
+ const char* reason = nullptr;
+ if (!CanIonCompileOrInlineScript(script, &reason)) {
+ JitSpew(JitSpew_Inlining, "Cannot Ion compile script (%s)", reason);
+ return false;
+ }
+
+ return true;
+}
+
+static MethodStatus Compile(JSContext* cx, HandleScript script,
+ BaselineFrame* osrFrame, jsbytecode* osrPc) {
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+ MOZ_ASSERT(jit::IsBaselineJitEnabled(cx));
+
+ MOZ_ASSERT(script->hasBaselineScript());
+ MOZ_ASSERT(!script->baselineScript()->hasPendingIonCompileTask());
+ MOZ_ASSERT(!script->hasIonScript());
+
+ AutoGeckoProfilerEntry pseudoFrame(
+ cx, "Ion script compilation",
+ JS::ProfilingCategoryPair::JS_IonCompilation);
+
+ if (script->isDebuggee() || (osrFrame && osrFrame->isDebuggee())) {
+ JitSpew(JitSpew_IonAbort, "debugging");
+ return Method_Skipped;
+ }
+
+ if (!CanIonCompileScript(cx, script)) {
+ JitSpew(JitSpew_IonAbort, "Aborted compilation of %s:%u:%u",
+ script->filename(), script->lineno(), script->column());
+ return Method_CantCompile;
+ }
+
+ OptimizationLevel optimizationLevel =
+ IonOptimizations.levelForScript(script, osrPc);
+ if (optimizationLevel == OptimizationLevel::DontCompile) {
+ return Method_Skipped;
+ }
+
+ MOZ_ASSERT(optimizationLevel == OptimizationLevel::Normal);
+
+ if (!CanLikelyAllocateMoreExecutableMemory()) {
+ script->resetWarmUpCounterToDelayIonCompilation();
+ return Method_Skipped;
+ }
+
+ MOZ_ASSERT(!script->hasIonScript());
+
+ AbortReason reason = IonCompile(cx, script, osrPc);
+ if (reason == AbortReason::Error) {
+ MOZ_ASSERT(cx->isExceptionPending());
+ return Method_Error;
+ }
+
+ if (reason == AbortReason::Disable) {
+ return Method_CantCompile;
+ }
+
+ if (reason == AbortReason::Alloc) {
+ ReportOutOfMemory(cx);
+ return Method_Error;
+ }
+
+ // Compilation succeeded or we invalidated right away or an inlining/alloc
+ // abort
+ if (script->hasIonScript()) {
+ return Method_Compiled;
+ }
+ return Method_Skipped;
+}
+
+} // namespace jit
+} // namespace js
+
+bool jit::OffThreadCompilationAvailable(JSContext* cx) {
+ // Even if off thread compilation is enabled, compilation must still occur
+ // on the main thread in some cases.
+ //
+ // Require cpuCount > 1 so that Ion compilation jobs and active-thread
+ // execution are not competing for the same resources.
+ return cx->runtime()->canUseOffthreadIonCompilation() &&
+ GetHelperThreadCPUCount() > 1 && CanUseExtraThreads();
+}
+
+MethodStatus jit::CanEnterIon(JSContext* cx, RunState& state) {
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+
+ HandleScript script = state.script();
+ MOZ_ASSERT(!script->hasIonScript());
+
+ // Skip if the script has been disabled.
+ if (!script->canIonCompile()) {
+ return Method_Skipped;
+ }
+
+ // Skip if the script is being compiled off thread.
+ if (script->isIonCompilingOffThread()) {
+ return Method_Skipped;
+ }
+
+ if (state.isInvoke()) {
+ InvokeState& invoke = *state.asInvoke();
+
+ if (TooManyActualArguments(invoke.args().length())) {
+ JitSpew(JitSpew_IonAbort, "too many actual args");
+ ForbidCompilation(cx, script);
+ return Method_CantCompile;
+ }
+
+ if (TooManyFormalArguments(
+ invoke.args().callee().as<JSFunction>().nargs())) {
+ JitSpew(JitSpew_IonAbort, "too many args");
+ ForbidCompilation(cx, script);
+ return Method_CantCompile;
+ }
+ }
+
+ // If --ion-eager is used, compile with Baseline first, so that we
+ // can directly enter IonMonkey.
+ if (JitOptions.eagerIonCompilation() && !script->hasBaselineScript()) {
+ MethodStatus status =
+ CanEnterBaselineMethod<BaselineTier::Compiler>(cx, state);
+ if (status != Method_Compiled) {
+ return status;
+ }
+ // Bytecode analysis may forbid compilation for a script.
+ if (!script->canIonCompile()) {
+ return Method_CantCompile;
+ }
+ }
+
+ if (!script->hasBaselineScript()) {
+ return Method_Skipped;
+ }
+
+ MOZ_ASSERT(!script->isIonCompilingOffThread());
+ MOZ_ASSERT(script->canIonCompile());
+
+ // Attempt compilation. Returns Method_Compiled if already compiled.
+ MethodStatus status = Compile(cx, script, /* osrFrame = */ nullptr,
+ /* osrPc = */ nullptr);
+ if (status != Method_Compiled) {
+ if (status == Method_CantCompile) {
+ ForbidCompilation(cx, script);
+ }
+ return status;
+ }
+
+ if (state.script()->baselineScript()->hasPendingIonCompileTask()) {
+ LinkIonScript(cx, state.script());
+ if (!state.script()->hasIonScript()) {
+ return jit::Method_Skipped;
+ }
+ }
+
+ return Method_Compiled;
+}
+
+static MethodStatus BaselineCanEnterAtEntry(JSContext* cx, HandleScript script,
+ BaselineFrame* frame) {
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+ MOZ_ASSERT(script->canIonCompile());
+ MOZ_ASSERT(!script->isIonCompilingOffThread());
+ MOZ_ASSERT(!script->hasIonScript());
+ MOZ_ASSERT(frame->isFunctionFrame());
+
+ // Mark as forbidden if frame can't be handled.
+ if (!CheckFrame(cx, frame)) {
+ ForbidCompilation(cx, script);
+ return Method_CantCompile;
+ }
+
+ if (script->baselineScript()->hasPendingIonCompileTask()) {
+ LinkIonScript(cx, script);
+ if (script->hasIonScript()) {
+ return Method_Compiled;
+ }
+ }
+
+ // Attempt compilation. Returns Method_Compiled if already compiled.
+ MethodStatus status = Compile(cx, script, frame, nullptr);
+ if (status != Method_Compiled) {
+ if (status == Method_CantCompile) {
+ ForbidCompilation(cx, script);
+ }
+ return status;
+ }
+
+ return Method_Compiled;
+}
+
+// Decide if a transition from baseline execution to Ion code should occur.
+// May compile or recompile the target JSScript.
+static MethodStatus BaselineCanEnterAtBranch(JSContext* cx, HandleScript script,
+ BaselineFrame* osrFrame,
+ jsbytecode* pc) {
+ MOZ_ASSERT(jit::IsIonEnabled(cx));
+ MOZ_ASSERT((JSOp)*pc == JSOp::LoopHead);
+
+ // Skip if the script has been disabled.
+ if (!script->canIonCompile()) {
+ return Method_Skipped;
+ }
+
+ // Skip if the script is being compiled off thread.
+ if (script->isIonCompilingOffThread()) {
+ return Method_Skipped;
+ }
+
+ // Optionally ignore on user request.
+ if (!JitOptions.osr) {
+ return Method_Skipped;
+ }
+
+ // Mark as forbidden if frame can't be handled.
+ if (!CheckFrame(cx, osrFrame)) {
+ ForbidCompilation(cx, script);
+ return Method_CantCompile;
+ }
+
+ // Check if the jitcode still needs to get linked and do this
+ // to have a valid IonScript.
+ if (script->baselineScript()->hasPendingIonCompileTask()) {
+ LinkIonScript(cx, script);
+ }
+
+ // By default a recompilation doesn't happen on osr mismatch.
+ // Decide if we want to force a recompilation if this happens too much.
+ if (script->hasIonScript()) {
+ if (pc == script->ionScript()->osrPc()) {
+ return Method_Compiled;
+ }
+
+ uint32_t count = script->ionScript()->incrOsrPcMismatchCounter();
+ if (count <= JitOptions.osrPcMismatchesBeforeRecompile &&
+ !JitOptions.eagerIonCompilation()) {
+ return Method_Skipped;
+ }
+
+ JitSpew(JitSpew_IonScripts, "Forcing OSR Mismatch Compilation");
+ Invalidate(cx, script);
+ }
+
+ // Attempt compilation.
+ // - Returns Method_Compiled if the right ionscript is present
+ // (Meaning it was present or a sequantial compile finished)
+ // - Returns Method_Skipped if pc doesn't match
+ // (This means a background thread compilation with that pc could have
+ // started or not.)
+ MethodStatus status = Compile(cx, script, osrFrame, pc);
+ if (status != Method_Compiled) {
+ if (status == Method_CantCompile) {
+ ForbidCompilation(cx, script);
+ }
+ return status;
+ }
+
+ // Return the compilation was skipped when the osr pc wasn't adjusted.
+ // This can happen when there was still an IonScript available and a
+ // background compilation started, but hasn't finished yet.
+ // Or when we didn't force a recompile.
+ if (script->hasIonScript() && pc != script->ionScript()->osrPc()) {
+ return Method_Skipped;
+ }
+
+ return Method_Compiled;
+}
+
+static bool IonCompileScriptForBaseline(JSContext* cx, BaselineFrame* frame,
+ jsbytecode* pc) {
+ MOZ_ASSERT(IsIonEnabled(cx));
+
+ RootedScript script(cx, frame->script());
+ bool isLoopHead = JSOp(*pc) == JSOp::LoopHead;
+
+ // The Baseline JIT code checks for Ion disabled or compiling off-thread.
+ MOZ_ASSERT(script->canIonCompile());
+ MOZ_ASSERT(!script->isIonCompilingOffThread());
+
+ // If Ion script exists, but PC is not at a loop entry, then Ion will be
+ // entered for this script at an appropriate LOOPENTRY or the next time this
+ // function is called.
+ if (script->hasIonScript() && !isLoopHead) {
+ JitSpew(JitSpew_BaselineOSR, "IonScript exists, but not at loop entry!");
+ // TODO: ASSERT that a ion-script-already-exists checker stub doesn't exist.
+ // TODO: Clear all optimized stubs.
+ // TODO: Add a ion-script-already-exists checker stub.
+ return true;
+ }
+
+ // Ensure that Ion-compiled code is available.
+ JitSpew(JitSpew_BaselineOSR,
+ "WarmUpCounter for %s:%u:%u reached %d at pc %p, trying to switch to "
+ "Ion!",
+ script->filename(), script->lineno(), script->column(),
+ (int)script->getWarmUpCount(), (void*)pc);
+
+ MethodStatus stat;
+ if (isLoopHead) {
+ JitSpew(JitSpew_BaselineOSR, " Compile at loop head!");
+ stat = BaselineCanEnterAtBranch(cx, script, frame, pc);
+ } else if (frame->isFunctionFrame()) {
+ JitSpew(JitSpew_BaselineOSR,
+ " Compile function from top for later entry!");
+ stat = BaselineCanEnterAtEntry(cx, script, frame);
+ } else {
+ return true;
+ }
+
+ if (stat == Method_Error) {
+ JitSpew(JitSpew_BaselineOSR, " Compile with Ion errored!");
+ return false;
+ }
+
+ if (stat == Method_CantCompile) {
+ MOZ_ASSERT(!script->canIonCompile());
+ JitSpew(JitSpew_BaselineOSR, " Can't compile with Ion!");
+ } else if (stat == Method_Skipped) {
+ JitSpew(JitSpew_BaselineOSR, " Skipped compile with Ion!");
+ } else if (stat == Method_Compiled) {
+ JitSpew(JitSpew_BaselineOSR, " Compiled with Ion!");
+ } else {
+ MOZ_CRASH("Invalid MethodStatus!");
+ }
+
+ return true;
+}
+
+bool jit::IonCompileScriptForBaselineAtEntry(JSContext* cx,
+ BaselineFrame* frame) {
+ JSScript* script = frame->script();
+ return IonCompileScriptForBaseline(cx, frame, script->code());
+}
+
+/* clang-format off */
+// The following data is kept in a temporary heap-allocated buffer, stored in
+// JitRuntime (high memory addresses at top, low at bottom):
+//
+// +----->+=================================+ -- <---- High Address
+// | | | |
+// | | ...BaselineFrame... | |-- Copy of BaselineFrame + stack values
+// | | | |
+// | +---------------------------------+ |
+// | | | |
+// | | ...Locals/Stack... | |
+// | | | |
+// | +=================================+ --
+// | | Padding(Maybe Empty) |
+// | +=================================+ --
+// +------|-- baselineFrame | |-- IonOsrTempData
+// | jitcode | |
+// +=================================+ -- <---- Low Address
+//
+// A pointer to the IonOsrTempData is returned.
+/* clang-format on */
+
+static IonOsrTempData* PrepareOsrTempData(JSContext* cx, BaselineFrame* frame,
+ uint32_t frameSize, void* jitcode) {
+ uint32_t numValueSlots = frame->numValueSlots(frameSize);
+
+ // Calculate the amount of space to allocate:
+ // BaselineFrame space:
+ // (sizeof(Value) * numValueSlots)
+ // + sizeof(BaselineFrame)
+ //
+ // IonOsrTempData space:
+ // sizeof(IonOsrTempData)
+
+ size_t frameSpace = sizeof(BaselineFrame) + sizeof(Value) * numValueSlots;
+ size_t ionOsrTempDataSpace = sizeof(IonOsrTempData);
+
+ size_t totalSpace = AlignBytes(frameSpace, sizeof(Value)) +
+ AlignBytes(ionOsrTempDataSpace, sizeof(Value));
+
+ JitRuntime* jrt = cx->runtime()->jitRuntime();
+ uint8_t* buf = jrt->allocateIonOsrTempData(totalSpace);
+ if (!buf) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ IonOsrTempData* info = new (buf) IonOsrTempData();
+ info->jitcode = jitcode;
+
+ // Copy the BaselineFrame + local/stack Values to the buffer. Arguments and
+ // |this| are not copied but left on the stack: the Baseline and Ion frame
+ // share the same frame prefix and Ion won't clobber these values. Note
+ // that info->baselineFrame will point to the *end* of the frame data, like
+ // the frame pointer register in baseline frames.
+ uint8_t* frameStart =
+ (uint8_t*)info + AlignBytes(ionOsrTempDataSpace, sizeof(Value));
+ info->baselineFrame = frameStart + frameSpace;
+
+ memcpy(frameStart, (uint8_t*)frame - numValueSlots * sizeof(Value),
+ frameSpace);
+
+ JitSpew(JitSpew_BaselineOSR, "Allocated IonOsrTempData at %p", info);
+ JitSpew(JitSpew_BaselineOSR, "Jitcode is %p", info->jitcode);
+
+ // All done.
+ return info;
+}
+
+bool jit::IonCompileScriptForBaselineOSR(JSContext* cx, BaselineFrame* frame,
+ uint32_t frameSize, jsbytecode* pc,
+ IonOsrTempData** infoPtr) {
+ MOZ_ASSERT(infoPtr);
+ *infoPtr = nullptr;
+
+ MOZ_ASSERT(frame->debugFrameSize() == frameSize);
+ MOZ_ASSERT(JSOp(*pc) == JSOp::LoopHead);
+
+ if (!IonCompileScriptForBaseline(cx, frame, pc)) {
+ return false;
+ }
+
+ RootedScript script(cx, frame->script());
+ if (!script->hasIonScript() || script->ionScript()->osrPc() != pc ||
+ frame->isDebuggee()) {
+ return true;
+ }
+
+ IonScript* ion = script->ionScript();
+ MOZ_ASSERT(cx->runtime()->geckoProfiler().enabled() ==
+ ion->hasProfilingInstrumentation());
+ MOZ_ASSERT(ion->osrPc() == pc);
+
+ ion->resetOsrPcMismatchCounter();
+
+ JitSpew(JitSpew_BaselineOSR, " OSR possible!");
+ void* jitcode = ion->method()->raw() + ion->osrEntryOffset();
+
+ // Prepare the temporary heap copy of the fake InterpreterFrame and actual
+ // args list.
+ JitSpew(JitSpew_BaselineOSR, "Got jitcode. Preparing for OSR into ion.");
+ IonOsrTempData* info = PrepareOsrTempData(cx, frame, frameSize, jitcode);
+ if (!info) {
+ return false;
+ }
+
+ *infoPtr = info;
+ return true;
+}
+
+static void InvalidateActivation(JS::GCContext* gcx,
+ const JitActivationIterator& activations,
+ bool invalidateAll) {
+ JitSpew(JitSpew_IonInvalidate, "BEGIN invalidating activation");
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ if (JitOptions.checkOsiPointRegisters) {
+ activations->asJit()->setCheckRegs(false);
+ }
+#endif
+
+ size_t frameno = 1;
+
+ for (OnlyJSJitFrameIter iter(activations); !iter.done(); ++iter, ++frameno) {
+ const JSJitFrameIter& frame = iter.frame();
+ MOZ_ASSERT_IF(frameno == 1, frame.isExitFrame() ||
+ frame.type() == FrameType::Bailout ||
+ frame.type() == FrameType::JSJitToWasm);
+
+#ifdef JS_JITSPEW
+ switch (frame.type()) {
+ case FrameType::Exit:
+ JitSpew(JitSpew_IonInvalidate, "#%zu exit frame @ %p", frameno,
+ frame.fp());
+ break;
+ case FrameType::JSJitToWasm:
+ JitSpew(JitSpew_IonInvalidate, "#%zu wasm exit frame @ %p", frameno,
+ frame.fp());
+ break;
+ case FrameType::BaselineJS:
+ case FrameType::IonJS:
+ case FrameType::Bailout: {
+ MOZ_ASSERT(frame.isScripted());
+ const char* type = "Unknown";
+ if (frame.isIonJS()) {
+ type = "Optimized";
+ } else if (frame.isBaselineJS()) {
+ type = "Baseline";
+ } else if (frame.isBailoutJS()) {
+ type = "Bailing";
+ }
+ JSScript* script = frame.maybeForwardedScript();
+ JitSpew(JitSpew_IonInvalidate,
+ "#%zu %s JS frame @ %p, %s:%u:%u (fun: %p, script: %p, pc %p)",
+ frameno, type, frame.fp(), script->maybeForwardedFilename(),
+ script->lineno(), script->column(), frame.maybeCallee(), script,
+ frame.resumePCinCurrentFrame());
+ break;
+ }
+ case FrameType::BaselineStub:
+ JitSpew(JitSpew_IonInvalidate, "#%zu baseline stub frame @ %p", frameno,
+ frame.fp());
+ break;
+ case FrameType::Rectifier:
+ JitSpew(JitSpew_IonInvalidate, "#%zu rectifier frame @ %p", frameno,
+ frame.fp());
+ break;
+ case FrameType::IonICCall:
+ JitSpew(JitSpew_IonInvalidate, "#%zu ion IC call frame @ %p", frameno,
+ frame.fp());
+ break;
+ case FrameType::CppToJSJit:
+ JitSpew(JitSpew_IonInvalidate, "#%zu entry frame @ %p", frameno,
+ frame.fp());
+ break;
+ case FrameType::WasmToJSJit:
+ JitSpew(JitSpew_IonInvalidate, "#%zu wasm frames @ %p", frameno,
+ frame.fp());
+ break;
+ }
+#endif // JS_JITSPEW
+
+ if (!frame.isIonScripted()) {
+ continue;
+ }
+
+ // See if the frame has already been invalidated.
+ if (frame.checkInvalidation()) {
+ continue;
+ }
+
+ JSScript* script = frame.maybeForwardedScript();
+ if (!script->hasIonScript()) {
+ continue;
+ }
+
+ if (!invalidateAll && !script->ionScript()->invalidated()) {
+ continue;
+ }
+
+ IonScript* ionScript = script->ionScript();
+
+ // Purge ICs before we mark this script as invalidated. This will
+ // prevent lastJump_ from appearing to be a bogus pointer, just
+ // in case anyone tries to read it.
+ ionScript->purgeICs(script->zone());
+
+ // This frame needs to be invalidated. We do the following:
+ //
+ // 1. Increment the reference counter to keep the ionScript alive
+ // for the invalidation bailout or for the exception handler.
+ // 2. Determine safepoint that corresponds to the current call.
+ // 3. From safepoint, get distance to the OSI-patchable offset.
+ // 4. From the IonScript, determine the distance between the
+ // call-patchable offset and the invalidation epilogue.
+ // 5. Patch the OSI point with a call-relative to the
+ // invalidation epilogue.
+ //
+ // The code generator ensures that there's enough space for us
+ // to patch in a call-relative operation at each invalidation
+ // point.
+ //
+ // Note: you can't simplify this mechanism to "just patch the
+ // instruction immediately after the call" because things may
+ // need to move into a well-defined register state (using move
+ // instructions after the call) in to capture an appropriate
+ // snapshot after the call occurs.
+
+ ionScript->incrementInvalidationCount();
+
+ JitCode* ionCode = ionScript->method();
+
+ // We're about to remove edges from the JSScript to GC things embedded in
+ // the JitCode. Perform a barrier to let the GC know about those edges.
+ PreWriteBarrier(script->zone(), ionCode, [](JSTracer* trc, JitCode* code) {
+ code->traceChildren(trc);
+ });
+
+ ionCode->setInvalidated();
+
+ // Don't adjust OSI points in a bailout path.
+ if (frame.isBailoutJS()) {
+ continue;
+ }
+
+ // Write the delta (from the return address offset to the
+ // IonScript pointer embedded into the invalidation epilogue)
+ // where the safepointed call instruction used to be. We rely on
+ // the call sequence causing the safepoint being >= the size of
+ // a uint32, which is checked during safepoint index
+ // construction.
+ AutoWritableJitCode awjc(ionCode);
+ const SafepointIndex* si =
+ ionScript->getSafepointIndex(frame.resumePCinCurrentFrame());
+ CodeLocationLabel dataLabelToMunge(frame.resumePCinCurrentFrame());
+ ptrdiff_t delta = ionScript->invalidateEpilogueDataOffset() -
+ (frame.resumePCinCurrentFrame() - ionCode->raw());
+ Assembler::PatchWrite_Imm32(dataLabelToMunge, Imm32(delta));
+
+ CodeLocationLabel osiPatchPoint =
+ SafepointReader::InvalidationPatchPoint(ionScript, si);
+ CodeLocationLabel invalidateEpilogue(
+ ionCode, CodeOffset(ionScript->invalidateEpilogueOffset()));
+
+ JitSpew(
+ JitSpew_IonInvalidate,
+ " ! Invalidate ionScript %p (inv count %zu) -> patching osipoint %p",
+ ionScript, ionScript->invalidationCount(), (void*)osiPatchPoint.raw());
+ Assembler::PatchWrite_NearCall(osiPatchPoint, invalidateEpilogue);
+ }
+
+ JitSpew(JitSpew_IonInvalidate, "END invalidating activation");
+}
+
+void jit::InvalidateAll(JS::GCContext* gcx, Zone* zone) {
+ // The caller should previously have cancelled off thread compilation.
+#ifdef DEBUG
+ for (RealmsInZoneIter realm(zone); !realm.done(); realm.next()) {
+ MOZ_ASSERT(!HasOffThreadIonCompile(realm));
+ }
+#endif
+ if (zone->isAtomsZone()) {
+ return;
+ }
+ JSContext* cx = TlsContext.get();
+ for (JitActivationIterator iter(cx); !iter.done(); ++iter) {
+ if (iter->compartment()->zone() == zone) {
+ JitSpew(JitSpew_IonInvalidate, "Invalidating all frames for GC");
+ InvalidateActivation(gcx, iter, true);
+ }
+ }
+}
+
+static void ClearIonScriptAfterInvalidation(JSContext* cx, JSScript* script,
+ IonScript* ionScript,
+ bool resetUses) {
+ // Null out the JitScript's IonScript pointer. The caller is responsible for
+ // destroying the IonScript using the invalidation count mechanism.
+ DebugOnly<IonScript*> clearedIonScript =
+ script->jitScript()->clearIonScript(cx->gcContext(), script);
+ MOZ_ASSERT(clearedIonScript == ionScript);
+
+ // Wait for the scripts to get warm again before doing another
+ // compile, unless we are recompiling *because* a script got hot
+ // (resetUses is false).
+ if (resetUses) {
+ script->resetWarmUpCounterToDelayIonCompilation();
+ }
+}
+
+void jit::Invalidate(JSContext* cx, const RecompileInfoVector& invalid,
+ bool resetUses, bool cancelOffThread) {
+ JitSpew(JitSpew_IonInvalidate, "Start invalidation.");
+
+ // Add an invalidation reference to all invalidated IonScripts to indicate
+ // to the traversal which frames have been invalidated.
+ size_t numInvalidations = 0;
+ for (const RecompileInfo& info : invalid) {
+ if (cancelOffThread) {
+ CancelOffThreadIonCompile(info.script());
+ }
+
+ IonScript* ionScript = info.maybeIonScriptToInvalidate();
+ if (!ionScript) {
+ continue;
+ }
+
+ JitSpew(JitSpew_IonInvalidate, " Invalidate %s:%u:%u, IonScript %p",
+ info.script()->filename(), info.script()->lineno(),
+ info.script()->column(), ionScript);
+
+ // Keep the ion script alive during the invalidation and flag this
+ // ionScript as being invalidated. This increment is removed by the
+ // loop after the calls to InvalidateActivation.
+ ionScript->incrementInvalidationCount();
+ numInvalidations++;
+ }
+
+ if (!numInvalidations) {
+ JitSpew(JitSpew_IonInvalidate, " No IonScript invalidation.");
+ return;
+ }
+
+ JS::GCContext* gcx = cx->gcContext();
+ for (JitActivationIterator iter(cx); !iter.done(); ++iter) {
+ InvalidateActivation(gcx, iter, false);
+ }
+
+ // Drop the references added above. If a script was never active, its
+ // IonScript will be immediately destroyed. Otherwise, it will be held live
+ // until its last invalidated frame is destroyed.
+ for (const RecompileInfo& info : invalid) {
+ IonScript* ionScript = info.maybeIonScriptToInvalidate();
+ if (!ionScript) {
+ continue;
+ }
+
+ if (ionScript->invalidationCount() == 1) {
+ // decrementInvalidationCount will destroy the IonScript so null out
+ // jitScript->ionScript_ now. We don't want to do this unconditionally
+ // because maybeIonScriptToInvalidate depends on script->ionScript() (we
+ // would leak the IonScript if |invalid| contains duplicates).
+ ClearIonScriptAfterInvalidation(cx, info.script(), ionScript, resetUses);
+ }
+
+ ionScript->decrementInvalidationCount(gcx);
+ numInvalidations--;
+ }
+
+ // Make sure we didn't leak references by invalidating the same IonScript
+ // multiple times in the above loop.
+ MOZ_ASSERT(!numInvalidations);
+
+ // Finally, null out jitScript->ionScript_ for IonScripts that are still on
+ // the stack.
+ for (const RecompileInfo& info : invalid) {
+ if (IonScript* ionScript = info.maybeIonScriptToInvalidate()) {
+ ClearIonScriptAfterInvalidation(cx, info.script(), ionScript, resetUses);
+ }
+ }
+}
+
+void jit::IonScript::invalidate(JSContext* cx, JSScript* script, bool resetUses,
+ const char* reason) {
+ // Note: we could short circuit here if we already invalidated this
+ // IonScript, but jit::Invalidate also cancels off-thread compilations of
+ // |script|.
+ MOZ_RELEASE_ASSERT(invalidated() || script->ionScript() == this);
+
+ JitSpew(JitSpew_IonInvalidate, " Invalidate IonScript %p: %s", this, reason);
+
+ // RecompileInfoVector has inline space for at least one element.
+ RecompileInfoVector list;
+ MOZ_RELEASE_ASSERT(list.reserve(1));
+ list.infallibleEmplaceBack(script, compilationId());
+
+ Invalidate(cx, list, resetUses, true);
+}
+
+void jit::Invalidate(JSContext* cx, JSScript* script, bool resetUses,
+ bool cancelOffThread) {
+ MOZ_ASSERT(script->hasIonScript());
+
+ if (cx->runtime()->geckoProfiler().enabled()) {
+ // Register invalidation with profiler.
+ // Format of event payload string:
+ // "<filename>:<lineno>"
+
+ // Get the script filename, if any, and its length.
+ const char* filename = script->filename();
+ if (filename == nullptr) {
+ filename = "<unknown>";
+ }
+
+ // Construct the descriptive string.
+ UniqueChars buf =
+ JS_smprintf("%s:%u:%u", filename, script->lineno(), script->column());
+
+ // Ignore the event on allocation failure.
+ if (buf) {
+ cx->runtime()->geckoProfiler().markEvent("Invalidate", buf.get());
+ }
+ }
+
+ // RecompileInfoVector has inline space for at least one element.
+ RecompileInfoVector scripts;
+ MOZ_ASSERT(script->hasIonScript());
+ MOZ_RELEASE_ASSERT(scripts.reserve(1));
+ scripts.infallibleEmplaceBack(script, script->ionScript()->compilationId());
+
+ Invalidate(cx, scripts, resetUses, cancelOffThread);
+}
+
+void jit::FinishInvalidation(JS::GCContext* gcx, JSScript* script) {
+ if (!script->hasIonScript()) {
+ return;
+ }
+
+ // In all cases, null out jitScript->ionScript_ to avoid re-entry.
+ IonScript* ion = script->jitScript()->clearIonScript(gcx, script);
+
+ // If this script has Ion code on the stack, invalidated() will return
+ // true. In this case we have to wait until destroying it.
+ if (!ion->invalidated()) {
+ jit::IonScript::Destroy(gcx, ion);
+ }
+}
+
+void jit::ForbidCompilation(JSContext* cx, JSScript* script) {
+ JitSpew(JitSpew_IonAbort, "Disabling Ion compilation of script %s:%u:%u",
+ script->filename(), script->lineno(), script->column());
+
+ CancelOffThreadIonCompile(script);
+
+ if (script->hasIonScript()) {
+ Invalidate(cx, script, false);
+ }
+
+ script->disableIon();
+}
+
+size_t jit::SizeOfIonData(JSScript* script,
+ mozilla::MallocSizeOf mallocSizeOf) {
+ size_t result = 0;
+
+ if (script->hasIonScript()) {
+ result += script->ionScript()->sizeOfIncludingThis(mallocSizeOf);
+ }
+
+ return result;
+}
+
+// If you change these, please also change the comment in TempAllocator.
+/* static */ const size_t TempAllocator::BallastSize = 16 * 1024;
+/* static */ const size_t TempAllocator::PreferredLifoChunkSize = 32 * 1024;
diff --git a/js/src/jit/Ion.h b/js/src/jit/Ion.h
new file mode 100644
index 0000000000..babbdb84e6
--- /dev/null
+++ b/js/src/jit/Ion.h
@@ -0,0 +1,158 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Ion_h
+#define jit_Ion_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Likely.h"
+#include "mozilla/MemoryReporting.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jsfriendapi.h"
+#include "jspubtd.h"
+
+#include "jit/BaselineJIT.h"
+#include "jit/IonTypes.h"
+#include "jit/JitContext.h"
+#include "jit/JitOptions.h"
+#include "js/Principals.h"
+#include "js/TypeDecls.h"
+#include "vm/BytecodeUtil.h"
+#include "vm/JSContext.h"
+#include "vm/JSFunction.h"
+#include "vm/JSScript.h"
+
+namespace js {
+
+class RunState;
+
+namespace jit {
+
+class BaselineFrame;
+
+bool CanIonCompileScript(JSContext* cx, JSScript* script);
+bool CanIonInlineScript(JSScript* script);
+
+[[nodiscard]] bool IonCompileScriptForBaselineAtEntry(JSContext* cx,
+ BaselineFrame* frame);
+
+struct IonOsrTempData {
+ void* jitcode;
+ uint8_t* baselineFrame;
+
+ static constexpr size_t offsetOfJitCode() {
+ return offsetof(IonOsrTempData, jitcode);
+ }
+ static constexpr size_t offsetOfBaselineFrame() {
+ return offsetof(IonOsrTempData, baselineFrame);
+ }
+};
+
+[[nodiscard]] bool IonCompileScriptForBaselineOSR(JSContext* cx,
+ BaselineFrame* frame,
+ uint32_t frameSize,
+ jsbytecode* pc,
+ IonOsrTempData** infoPtr);
+
+MethodStatus CanEnterIon(JSContext* cx, RunState& state);
+
+class MIRGenerator;
+class LIRGraph;
+class CodeGenerator;
+class LazyLinkExitFrameLayout;
+class WarpSnapshot;
+
+[[nodiscard]] bool OptimizeMIR(MIRGenerator* mir);
+LIRGraph* GenerateLIR(MIRGenerator* mir);
+CodeGenerator* GenerateCode(MIRGenerator* mir, LIRGraph* lir);
+CodeGenerator* CompileBackEnd(MIRGenerator* mir, WarpSnapshot* snapshot);
+
+void LinkIonScript(JSContext* cx, HandleScript calleescript);
+uint8_t* LazyLinkTopActivation(JSContext* cx, LazyLinkExitFrameLayout* frame);
+
+inline bool IsIonInlinableGetterOrSetterOp(JSOp op) {
+ // JSOp::GetProp, JSOp::CallProp, JSOp::Length, JSOp::GetElem,
+ // and JSOp::CallElem. (Inlined Getters)
+ // JSOp::SetProp, JSOp::SetName, JSOp::SetGName (Inlined Setters)
+ return IsGetPropOp(op) || IsGetElemOp(op) || IsSetPropOp(op);
+}
+
+inline bool IsIonInlinableOp(JSOp op) {
+ // JSOp::Call, JSOp::FunCall, JSOp::Eval, JSOp::New (Normal Callsites) or an
+ // inlinable getter or setter.
+ return (IsInvokeOp(op) && !IsSpreadOp(op)) ||
+ IsIonInlinableGetterOrSetterOp(op);
+}
+
+inline bool TooManyActualArguments(unsigned nargs) {
+ return nargs > JitOptions.maxStackArgs;
+}
+
+inline bool TooManyFormalArguments(unsigned nargs) {
+ return nargs >= SNAPSHOT_MAX_NARGS || TooManyActualArguments(nargs);
+}
+
+inline size_t NumLocalsAndArgs(JSScript* script) {
+ size_t num = 1 /* this */ + script->nfixed();
+ if (JSFunction* fun = script->function()) {
+ num += fun->nargs();
+ }
+ return num;
+}
+
+// Debugging RAII class which marks the current thread as performing an Ion
+// backend compilation.
+class MOZ_RAII AutoEnterIonBackend {
+ public:
+ AutoEnterIonBackend() {
+#ifdef DEBUG
+ JitContext* jcx = GetJitContext();
+ jcx->enterIonBackend();
+#endif
+ }
+
+#ifdef DEBUG
+ ~AutoEnterIonBackend() {
+ JitContext* jcx = GetJitContext();
+ jcx->leaveIonBackend();
+ }
+#endif
+};
+
+bool OffThreadCompilationAvailable(JSContext* cx);
+
+void ForbidCompilation(JSContext* cx, JSScript* script);
+
+size_t SizeOfIonData(JSScript* script, mozilla::MallocSizeOf mallocSizeOf);
+
+inline bool IsIonEnabled(JSContext* cx) {
+ if (MOZ_UNLIKELY(!IsBaselineJitEnabled(cx) || cx->options().disableIon())) {
+ return false;
+ }
+
+ if (MOZ_LIKELY(JitOptions.ion)) {
+ return true;
+ }
+ if (JitOptions.jitForTrustedPrincipals) {
+ JS::Realm* realm = js::GetContextRealm(cx);
+ return realm && JS::GetRealmPrincipals(realm) &&
+ JS::GetRealmPrincipals(realm)->isSystemOrAddonPrincipal();
+ }
+ return false;
+}
+
+// Implemented per-platform. Returns true if the flags will not require
+// further (lazy) computation.
+bool CPUFlagsHaveBeenComputed();
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Ion_h */
diff --git a/js/src/jit/IonAnalysis.cpp b/js/src/jit/IonAnalysis.cpp
new file mode 100644
index 0000000000..3ce0f28332
--- /dev/null
+++ b/js/src/jit/IonAnalysis.cpp
@@ -0,0 +1,4666 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/IonAnalysis.h"
+
+#include <algorithm>
+#include <utility> // for ::std::pair
+
+#include "jit/AliasAnalysis.h"
+#include "jit/CompileInfo.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "util/CheckedArithmetic.h"
+
+#include "vm/BytecodeUtil-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+
+// Stack used by FlagPhiInputsAsImplicitlyUsed. It stores the Phi instruction
+// pointer and the MUseIterator which should be visited next.
+using MPhiUseIteratorStack =
+ Vector<std::pair<MPhi*, MUseIterator>, 16, SystemAllocPolicy>;
+
+// Look for Phi uses with a depth-first search. If any uses are found the stack
+// of MPhi instructions is returned in the |worklist| argument.
+static bool DepthFirstSearchUse(MIRGenerator* mir,
+ MPhiUseIteratorStack& worklist, MPhi* phi) {
+ // Push a Phi and the next use to iterate over in the worklist.
+ auto push = [&worklist](MPhi* phi, MUseIterator use) -> bool {
+ phi->setInWorklist();
+ return worklist.append(std::make_pair(phi, use));
+ };
+
+#ifdef DEBUG
+ // Used to assert that when we have no uses, we at least visited all the
+ // transitive uses.
+ size_t refUseCount = phi->useCount();
+ size_t useCount = 0;
+#endif
+ MOZ_ASSERT(worklist.empty());
+ if (!push(phi, phi->usesBegin())) {
+ return false;
+ }
+
+ while (!worklist.empty()) {
+ // Resume iterating over the last phi-use pair added by the next loop.
+ auto pair = worklist.popCopy();
+ MPhi* producer = pair.first;
+ MUseIterator use = pair.second;
+ MUseIterator end(producer->usesEnd());
+ producer->setNotInWorklist();
+
+ // Keep going down the tree of uses, skipping (continue)
+ // non-observable/unused cases and Phi which are already listed in the
+ // worklist. Stop (return) as soon as one use is found.
+ while (use != end) {
+ MNode* consumer = (*use)->consumer();
+ MUseIterator it = use;
+ use++;
+#ifdef DEBUG
+ useCount++;
+#endif
+ if (mir->shouldCancel("FlagPhiInputsAsImplicitlyUsed inner loop")) {
+ return false;
+ }
+
+ if (consumer->isResumePoint()) {
+ MResumePoint* rp = consumer->toResumePoint();
+ // Observable operands are similar to potential uses.
+ if (rp->isObservableOperand(*it)) {
+ return push(producer, use);
+ }
+ continue;
+ }
+
+ MDefinition* cdef = consumer->toDefinition();
+ if (!cdef->isPhi()) {
+ // The producer is explicitly used by a definition.
+ return push(producer, use);
+ }
+
+ MPhi* cphi = cdef->toPhi();
+ if (cphi->getUsageAnalysis() == PhiUsage::Used ||
+ cphi->isImplicitlyUsed()) {
+ // The information got cached on the Phi the last time it
+ // got visited, or when flagging operands of implicitly used
+ // instructions.
+ return push(producer, use);
+ }
+
+ if (cphi->isInWorklist() || cphi == producer) {
+ // We are already iterating over the uses of this Phi instruction which
+ // are part of a loop, instead of trying to handle loops, conservatively
+ // mark them as used.
+ return push(producer, use);
+ }
+
+ if (cphi->getUsageAnalysis() == PhiUsage::Unused) {
+ // The instruction already got visited and is known to have
+ // no uses. Skip it.
+ continue;
+ }
+
+ // We found another Phi instruction, move the use iterator to
+ // the next use push it to the worklist stack. Then, continue
+ // with a depth search.
+ if (!push(producer, use)) {
+ return false;
+ }
+ producer = cphi;
+ use = producer->usesBegin();
+ end = producer->usesEnd();
+#ifdef DEBUG
+ refUseCount += producer->useCount();
+#endif
+ }
+
+ // When unused, we cannot bubble up this information without iterating
+ // over the rest of the previous Phi instruction consumers.
+ MOZ_ASSERT(use == end);
+ producer->setUsageAnalysis(PhiUsage::Unused);
+ }
+
+ MOZ_ASSERT(useCount == refUseCount);
+ return true;
+}
+
+static bool FlagPhiInputsAsImplicitlyUsed(MIRGenerator* mir, MBasicBlock* block,
+ MBasicBlock* succ,
+ MPhiUseIteratorStack& worklist) {
+ // When removing an edge between 2 blocks, we might remove the ability of
+ // later phases to figure out that the uses of a Phi should be considered as
+ // a use of all its inputs. Thus we need to mark the Phi inputs as being
+ // implicitly used iff the phi has any uses.
+ //
+ //
+ // +--------------------+ +---------------------+
+ // |12 MFoo 6 | |32 MBar 5 |
+ // | | | |
+ // | ... | | ... |
+ // | | | |
+ // |25 MGoto Block 4 | |43 MGoto Block 4 |
+ // +--------------------+ +---------------------+
+ // | |
+ // | | |
+ // | | |
+ // | +-----X------------------------+
+ // | Edge |
+ // | Removed |
+ // | |
+ // | +------------v-----------+
+ // | |50 MPhi 12 32 |
+ // | | |
+ // | | ... |
+ // | | |
+ // | |70 MReturn 50 |
+ // | +------------------------+
+ // |
+ // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ // |
+ // v
+ //
+ // ^ +--------------------+ +---------------------+
+ // /!\ |12 MConst opt-out | |32 MBar 5 |
+ // '---' | | | |
+ // | ... | | ... |
+ // |78 MBail | | |
+ // |80 MUnreachable | |43 MGoto Block 4 |
+ // +--------------------+ +---------------------+
+ // |
+ // |
+ // |
+ // +---------------+
+ // |
+ // |
+ // |
+ // +------------v-----------+
+ // |50 MPhi 32 |
+ // | |
+ // | ... |
+ // | |
+ // |70 MReturn 50 |
+ // +------------------------+
+ //
+ //
+ // If the inputs of the Phi are not flagged as implicitly used, then
+ // later compilation phase might optimize them out. The problem is that a
+ // bailout will use this value and give it back to baseline, which will then
+ // use the OptimizedOut magic value in a computation.
+ //
+ // Unfortunately, we cannot be too conservative about flagging Phi inputs as
+ // having implicit uses, as this would prevent many optimizations from being
+ // used. Thus, the following code is in charge of flagging Phi instructions
+ // as Unused or Used, and setting ImplicitlyUsed accordingly.
+ size_t predIndex = succ->getPredecessorIndex(block);
+ MPhiIterator end = succ->phisEnd();
+ MPhiIterator it = succ->phisBegin();
+ for (; it != end; it++) {
+ MPhi* phi = *it;
+
+ if (mir->shouldCancel("FlagPhiInputsAsImplicitlyUsed outer loop")) {
+ return false;
+ }
+
+ // We are looking to mark the Phi inputs which are used across the edge
+ // between the |block| and its successor |succ|.
+ MDefinition* def = phi->getOperand(predIndex);
+ if (def->isImplicitlyUsed()) {
+ continue;
+ }
+
+ // If the Phi is either Used or Unused, set the ImplicitlyUsed flag
+ // accordingly.
+ if (phi->getUsageAnalysis() == PhiUsage::Used || phi->isImplicitlyUsed()) {
+ def->setImplicitlyUsedUnchecked();
+ continue;
+ } else if (phi->getUsageAnalysis() == PhiUsage::Unused) {
+ continue;
+ }
+
+ // We do not know if the Phi was Used or Unused, iterate over all uses
+ // with a depth-search of uses. Returns the matching stack in the
+ // worklist as soon as one use is found.
+ MOZ_ASSERT(worklist.empty());
+ if (!DepthFirstSearchUse(mir, worklist, phi)) {
+ return false;
+ }
+
+ MOZ_ASSERT_IF(worklist.empty(),
+ phi->getUsageAnalysis() == PhiUsage::Unused);
+ if (!worklist.empty()) {
+ // One of the Phis is used, set Used flags on all the Phis which are
+ // in the use chain.
+ def->setImplicitlyUsedUnchecked();
+ do {
+ auto pair = worklist.popCopy();
+ MPhi* producer = pair.first;
+ producer->setUsageAnalysis(PhiUsage::Used);
+ producer->setNotInWorklist();
+ } while (!worklist.empty());
+ }
+ MOZ_ASSERT(phi->getUsageAnalysis() != PhiUsage::Unknown);
+ }
+
+ return true;
+}
+
+static MInstructionIterator FindFirstInstructionAfterBail(MBasicBlock* block) {
+ MOZ_ASSERT(block->alwaysBails());
+ for (MInstructionIterator it = block->begin(); it != block->end(); it++) {
+ MInstruction* ins = *it;
+ if (ins->isBail()) {
+ it++;
+ return it;
+ }
+ }
+ MOZ_CRASH("Expected MBail in alwaysBails block");
+}
+
+// Given an iterator pointing to the first removed instruction, mark
+// the operands of each removed instruction as having implicit uses.
+static bool FlagOperandsAsImplicitlyUsedAfter(
+ MIRGenerator* mir, MBasicBlock* block, MInstructionIterator firstRemoved) {
+ MOZ_ASSERT(firstRemoved->block() == block);
+
+ const CompileInfo& info = block->info();
+
+ // Flag operands of removed instructions as having implicit uses.
+ MInstructionIterator end = block->end();
+ for (MInstructionIterator it = firstRemoved; it != end; it++) {
+ if (mir->shouldCancel("FlagOperandsAsImplicitlyUsedAfter (loop 1)")) {
+ return false;
+ }
+
+ MInstruction* ins = *it;
+ for (size_t i = 0, e = ins->numOperands(); i < e; i++) {
+ ins->getOperand(i)->setImplicitlyUsedUnchecked();
+ }
+
+ // Flag observable resume point operands as having implicit uses.
+ if (MResumePoint* rp = ins->resumePoint()) {
+ // Note: no need to iterate over the caller's of the resume point as
+ // this is the same as the entry resume point.
+ MOZ_ASSERT(&rp->block()->info() == &info);
+ for (size_t i = 0, e = rp->numOperands(); i < e; i++) {
+ if (info.isObservableSlot(i)) {
+ rp->getOperand(i)->setImplicitlyUsedUnchecked();
+ }
+ }
+ }
+ }
+
+ // Flag Phi inputs of the successors as having implicit uses.
+ MPhiUseIteratorStack worklist;
+ for (size_t i = 0, e = block->numSuccessors(); i < e; i++) {
+ if (mir->shouldCancel("FlagOperandsAsImplicitlyUsedAfter (loop 2)")) {
+ return false;
+ }
+
+ if (!FlagPhiInputsAsImplicitlyUsed(mir, block, block->getSuccessor(i),
+ worklist)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool FlagEntryResumePointOperands(MIRGenerator* mir,
+ MBasicBlock* block) {
+ // Flag observable operands of the entry resume point as having implicit uses.
+ MResumePoint* rp = block->entryResumePoint();
+ while (rp) {
+ if (mir->shouldCancel("FlagEntryResumePointOperands")) {
+ return false;
+ }
+
+ const CompileInfo& info = rp->block()->info();
+ for (size_t i = 0, e = rp->numOperands(); i < e; i++) {
+ if (info.isObservableSlot(i)) {
+ rp->getOperand(i)->setImplicitlyUsedUnchecked();
+ }
+ }
+
+ rp = rp->caller();
+ }
+
+ return true;
+}
+
+static bool FlagAllOperandsAsImplicitlyUsed(MIRGenerator* mir,
+ MBasicBlock* block) {
+ return FlagEntryResumePointOperands(mir, block) &&
+ FlagOperandsAsImplicitlyUsedAfter(mir, block, block->begin());
+}
+
+// WarpBuilder sets the alwaysBails flag on blocks that contain an
+// unconditional bailout. We trim any instructions in those blocks
+// after the first unconditional bailout, and remove any blocks that
+// are only reachable through bailing blocks.
+bool jit::PruneUnusedBranches(MIRGenerator* mir, MIRGraph& graph) {
+ JitSpew(JitSpew_Prune, "Begin");
+
+ // Pruning is guided by unconditional bailouts. Wasm does not have bailouts.
+ MOZ_ASSERT(!mir->compilingWasm());
+
+ Vector<MBasicBlock*, 16, SystemAllocPolicy> worklist;
+ uint32_t numMarked = 0;
+ bool needsTrim = false;
+
+ auto markReachable = [&](MBasicBlock* block) -> bool {
+ block->mark();
+ numMarked++;
+ if (block->alwaysBails()) {
+ needsTrim = true;
+ }
+ return worklist.append(block);
+ };
+
+ // The entry block is always reachable.
+ if (!markReachable(graph.entryBlock())) {
+ return false;
+ }
+
+ // The OSR entry block is always reachable if it exists.
+ if (graph.osrBlock() && !markReachable(graph.osrBlock())) {
+ return false;
+ }
+
+ // Iteratively mark all reachable blocks.
+ while (!worklist.empty()) {
+ if (mir->shouldCancel("Prune unused branches (marking reachable)")) {
+ return false;
+ }
+ MBasicBlock* block = worklist.popCopy();
+
+ JitSpew(JitSpew_Prune, "Visit block %u:", block->id());
+ JitSpewIndent indent(JitSpew_Prune);
+
+ // If this block always bails, then it does not reach its successors.
+ if (block->alwaysBails()) {
+ continue;
+ }
+
+ for (size_t i = 0; i < block->numSuccessors(); i++) {
+ MBasicBlock* succ = block->getSuccessor(i);
+ if (succ->isMarked()) {
+ continue;
+ }
+ JitSpew(JitSpew_Prune, "Reaches block %u", succ->id());
+ if (!markReachable(succ)) {
+ return false;
+ }
+ }
+ }
+
+ if (!needsTrim && numMarked == graph.numBlocks()) {
+ // There is nothing to prune.
+ graph.unmarkBlocks();
+ return true;
+ }
+
+ JitSpew(JitSpew_Prune, "Remove unreachable instructions and blocks:");
+ JitSpewIndent indent(JitSpew_Prune);
+
+ // The operands of removed instructions may be needed in baseline
+ // after bailing out.
+ for (PostorderIterator it(graph.poBegin()); it != graph.poEnd();) {
+ if (mir->shouldCancel("Prune unused branches (marking operands)")) {
+ return false;
+ }
+
+ MBasicBlock* block = *it++;
+ if (!block->isMarked()) {
+ // If we are removing the block entirely, mark the operands of every
+ // instruction as being implicitly used.
+ FlagAllOperandsAsImplicitlyUsed(mir, block);
+ } else if (block->alwaysBails()) {
+ // If we are only trimming instructions after a bail, only mark operands
+ // of removed instructions.
+ MInstructionIterator firstRemoved = FindFirstInstructionAfterBail(block);
+ FlagOperandsAsImplicitlyUsedAfter(mir, block, firstRemoved);
+ }
+ }
+
+ // Remove the blocks in post-order such that consumers are visited before
+ // the predecessors, the only exception being the Phi nodes of loop headers.
+ for (PostorderIterator it(graph.poBegin()); it != graph.poEnd();) {
+ if (mir->shouldCancel("Prune unused branches (removal loop)")) {
+ return false;
+ }
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+
+ MBasicBlock* block = *it++;
+ if (block->isMarked() && !block->alwaysBails()) {
+ continue;
+ }
+
+ // As we are going to replace/remove the last instruction, we first have
+ // to remove this block from the predecessor list of its successors.
+ size_t numSucc = block->numSuccessors();
+ for (uint32_t i = 0; i < numSucc; i++) {
+ MBasicBlock* succ = block->getSuccessor(i);
+ if (succ->isDead()) {
+ continue;
+ }
+
+ // Our dominators code expects all loop headers to have two predecessors.
+ // If we are removing the normal entry to a loop, but can still reach
+ // the loop header via OSR, we create a fake unreachable predecessor.
+ if (succ->isLoopHeader() && block != succ->backedge()) {
+ MOZ_ASSERT(graph.osrBlock());
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+
+ MBasicBlock* fake = MBasicBlock::NewFakeLoopPredecessor(graph, succ);
+ if (!fake) {
+ return false;
+ }
+ // Mark the block to avoid removing it as unreachable.
+ fake->mark();
+
+ JitSpew(JitSpew_Prune,
+ "Header %u only reachable by OSR. Add fake predecessor %u",
+ succ->id(), fake->id());
+ }
+
+ JitSpew(JitSpew_Prune, "Remove block edge %u -> %u.", block->id(),
+ succ->id());
+ succ->removePredecessor(block);
+ }
+
+ if (!block->isMarked()) {
+ // Remove unreachable blocks from the CFG.
+ JitSpew(JitSpew_Prune, "Remove block %u.", block->id());
+ graph.removeBlock(block);
+ } else {
+ // Remove unreachable instructions after unconditional bailouts.
+ JitSpew(JitSpew_Prune, "Trim block %u.", block->id());
+
+ // Discard all instructions after the first MBail.
+ MInstructionIterator firstRemoved = FindFirstInstructionAfterBail(block);
+ block->discardAllInstructionsStartingAt(firstRemoved);
+
+ if (block->outerResumePoint()) {
+ block->clearOuterResumePoint();
+ }
+
+ block->end(MUnreachable::New(graph.alloc()));
+ }
+ }
+ graph.unmarkBlocks();
+
+ return true;
+}
+
+static bool SplitCriticalEdgesForBlock(MIRGraph& graph, MBasicBlock* block) {
+ if (block->numSuccessors() < 2) {
+ return true;
+ }
+ for (size_t i = 0; i < block->numSuccessors(); i++) {
+ MBasicBlock* target = block->getSuccessor(i);
+ if (target->numPredecessors() < 2) {
+ continue;
+ }
+
+ // Create a simple new block which contains a goto and which split the
+ // edge between block and target.
+ MBasicBlock* split = MBasicBlock::NewSplitEdge(graph, block, i, target);
+ if (!split) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// A critical edge is an edge which is neither its successor's only predecessor
+// nor its predecessor's only successor. Critical edges must be split to
+// prevent copy-insertion and code motion from affecting other edges.
+bool jit::SplitCriticalEdges(MIRGraph& graph) {
+ for (MBasicBlockIterator iter(graph.begin()); iter != graph.end(); iter++) {
+ MBasicBlock* block = *iter;
+ if (!SplitCriticalEdgesForBlock(graph, block)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool jit::IsUint32Type(const MDefinition* def) {
+ if (def->isBeta()) {
+ def = def->getOperand(0);
+ }
+
+ if (def->type() != MIRType::Int32) {
+ return false;
+ }
+
+ return def->isUrsh() && def->getOperand(1)->isConstant() &&
+ def->getOperand(1)->toConstant()->type() == MIRType::Int32 &&
+ def->getOperand(1)->toConstant()->toInt32() == 0;
+}
+
+// Determine whether phiBlock/testBlock simply compute a phi and perform a
+// test on it.
+static bool BlockIsSingleTest(MBasicBlock* phiBlock, MBasicBlock* testBlock,
+ MPhi** pphi, MTest** ptest) {
+ *pphi = nullptr;
+ *ptest = nullptr;
+
+ if (phiBlock != testBlock) {
+ MOZ_ASSERT(phiBlock->numSuccessors() == 1 &&
+ phiBlock->getSuccessor(0) == testBlock);
+ if (!phiBlock->begin()->isGoto()) {
+ return false;
+ }
+ }
+
+ auto iter = testBlock->rbegin();
+ if (!iter->isTest()) {
+ return false;
+ }
+ MTest* test = iter->toTest();
+
+ // Unwrap boolean conversion performed through the '!!' idiom.
+ MInstruction* testOrNot = test;
+ bool hasOddNumberOfNots = false;
+ while (++iter != testBlock->rend()) {
+ if (iter->isNot()) {
+ // The MNot must only be used by |testOrNot|.
+ auto* notIns = iter->toNot();
+ if (testOrNot->getOperand(0) != notIns) {
+ return false;
+ }
+ if (!notIns->hasOneUse()) {
+ return false;
+ }
+
+ testOrNot = notIns;
+ hasOddNumberOfNots = !hasOddNumberOfNots;
+ } else {
+ // Fail if there are any other instructions than MNot.
+ return false;
+ }
+ }
+
+ // There's an odd number of MNot, so this can't be the '!!' idiom.
+ if (hasOddNumberOfNots) {
+ return false;
+ }
+
+ MOZ_ASSERT(testOrNot->isTest() || testOrNot->isNot());
+
+ MDefinition* testInput = testOrNot->getOperand(0);
+ if (!testInput->isPhi()) {
+ return false;
+ }
+ MPhi* phi = testInput->toPhi();
+ if (phi->block() != phiBlock) {
+ return false;
+ }
+
+ for (MUseIterator iter = phi->usesBegin(); iter != phi->usesEnd(); ++iter) {
+ MUse* use = *iter;
+ if (use->consumer() == testOrNot) {
+ continue;
+ }
+ if (use->consumer()->isResumePoint()) {
+ MBasicBlock* useBlock = use->consumer()->block();
+ if (useBlock == phiBlock || useBlock == testBlock) {
+ continue;
+ }
+ }
+ return false;
+ }
+
+ for (MPhiIterator iter = phiBlock->phisBegin(); iter != phiBlock->phisEnd();
+ ++iter) {
+ if (*iter != phi) {
+ return false;
+ }
+ }
+
+ if (phiBlock != testBlock && !testBlock->phisEmpty()) {
+ return false;
+ }
+
+ *pphi = phi;
+ *ptest = test;
+
+ return true;
+}
+
+// Determine if value is directly or indirectly the test input.
+static bool IsTestInputMaybeToBool(MTest* test, MDefinition* value) {
+ auto* input = test->input();
+ bool hasEvenNumberOfNots = true;
+ while (true) {
+ // Only accept if there's an even number of MNot.
+ if (input == value && hasEvenNumberOfNots) {
+ return true;
+ }
+
+ // Unwrap boolean conversion performed through the '!!' idiom.
+ if (input->isNot()) {
+ input = input->toNot()->input();
+ hasEvenNumberOfNots = !hasEvenNumberOfNots;
+ continue;
+ }
+
+ return false;
+ }
+}
+
+// Change block so that it ends in a goto to the specific target block.
+// existingPred is an existing predecessor of the block.
+[[nodiscard]] static bool UpdateGotoSuccessor(TempAllocator& alloc,
+ MBasicBlock* block,
+ MBasicBlock* target,
+ MBasicBlock* existingPred) {
+ MInstruction* ins = block->lastIns();
+ MOZ_ASSERT(ins->isGoto());
+ ins->toGoto()->target()->removePredecessor(block);
+ block->discardLastIns();
+
+ MGoto* newGoto = MGoto::New(alloc, target);
+ block->end(newGoto);
+
+ return target->addPredecessorSameInputsAs(block, existingPred);
+}
+
+// Change block so that it ends in a test of the specified value, going to
+// either ifTrue or ifFalse. existingPred is an existing predecessor of ifTrue
+// or ifFalse with the same values incoming to ifTrue/ifFalse as block.
+// existingPred is not required to be a predecessor of ifTrue/ifFalse if block
+// already ends in a test going to that block on a true/false result.
+[[nodiscard]] static bool UpdateTestSuccessors(
+ TempAllocator& alloc, MBasicBlock* block, MDefinition* value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse, MBasicBlock* existingPred) {
+ MInstruction* ins = block->lastIns();
+ if (ins->isTest()) {
+ MTest* test = ins->toTest();
+ MOZ_ASSERT(test->input() == value);
+
+ if (ifTrue != test->ifTrue()) {
+ test->ifTrue()->removePredecessor(block);
+ if (!ifTrue->addPredecessorSameInputsAs(block, existingPred)) {
+ return false;
+ }
+ MOZ_ASSERT(test->ifTrue() == test->getSuccessor(0));
+ test->replaceSuccessor(0, ifTrue);
+ }
+
+ if (ifFalse != test->ifFalse()) {
+ test->ifFalse()->removePredecessor(block);
+ if (!ifFalse->addPredecessorSameInputsAs(block, existingPred)) {
+ return false;
+ }
+ MOZ_ASSERT(test->ifFalse() == test->getSuccessor(1));
+ test->replaceSuccessor(1, ifFalse);
+ }
+
+ return true;
+ }
+
+ MOZ_ASSERT(ins->isGoto());
+ ins->toGoto()->target()->removePredecessor(block);
+ block->discardLastIns();
+
+ MTest* test = MTest::New(alloc, value, ifTrue, ifFalse);
+ block->end(test);
+
+ if (!ifTrue->addPredecessorSameInputsAs(block, existingPred)) {
+ return false;
+ }
+ if (!ifFalse->addPredecessorSameInputsAs(block, existingPred)) {
+ return false;
+ }
+ return true;
+}
+
+/*
+ * Look for a diamond pattern:
+ *
+ * initialBlock
+ * / \
+ * trueBranch falseBranch
+ * \ /
+ * phiBlock
+ * |
+ * testBlock
+ */
+static bool IsDiamondPattern(MBasicBlock* initialBlock) {
+ MInstruction* ins = initialBlock->lastIns();
+ if (!ins->isTest()) {
+ return false;
+ }
+ MTest* initialTest = ins->toTest();
+
+ MBasicBlock* trueBranch = initialTest->ifTrue();
+ if (trueBranch->numPredecessors() != 1 || trueBranch->numSuccessors() != 1) {
+ return false;
+ }
+
+ MBasicBlock* falseBranch = initialTest->ifFalse();
+ if (falseBranch->numPredecessors() != 1 ||
+ falseBranch->numSuccessors() != 1) {
+ return false;
+ }
+
+ MBasicBlock* phiBlock = trueBranch->getSuccessor(0);
+ if (phiBlock != falseBranch->getSuccessor(0)) {
+ return false;
+ }
+ if (phiBlock->numPredecessors() != 2) {
+ return false;
+ }
+ return true;
+}
+
+static bool MaybeFoldDiamondConditionBlock(MIRGraph& graph,
+ MBasicBlock* initialBlock) {
+ MOZ_ASSERT(IsDiamondPattern(initialBlock));
+
+ // Optimize the MIR graph to improve the code generated for conditional
+ // operations. A test like 'if (a ? b : c)' normally requires four blocks,
+ // with a phi for the intermediate value. This can be improved to use three
+ // blocks with no phi value.
+
+ /*
+ * Look for a diamond pattern:
+ *
+ * initialBlock
+ * / \
+ * trueBranch falseBranch
+ * \ /
+ * phiBlock
+ * |
+ * testBlock
+ *
+ * Where phiBlock contains a single phi combining values pushed onto the
+ * stack by trueBranch and falseBranch, and testBlock contains a test on
+ * that phi. phiBlock and testBlock may be the same block; generated code
+ * will use different blocks if the (?:) op is in an inlined function.
+ */
+
+ MTest* initialTest = initialBlock->lastIns()->toTest();
+
+ MBasicBlock* trueBranch = initialTest->ifTrue();
+ MBasicBlock* falseBranch = initialTest->ifFalse();
+ if (initialBlock->isLoopBackedge() || trueBranch->isLoopBackedge() ||
+ falseBranch->isLoopBackedge()) {
+ return true;
+ }
+
+ MBasicBlock* phiBlock = trueBranch->getSuccessor(0);
+ MBasicBlock* testBlock = phiBlock;
+ if (testBlock->numSuccessors() == 1) {
+ if (testBlock->isLoopBackedge()) {
+ return true;
+ }
+ testBlock = testBlock->getSuccessor(0);
+ if (testBlock->numPredecessors() != 1) {
+ return true;
+ }
+ }
+
+ MPhi* phi;
+ MTest* finalTest;
+ if (!BlockIsSingleTest(phiBlock, testBlock, &phi, &finalTest)) {
+ return true;
+ }
+
+ MOZ_ASSERT(phi->numOperands() == 2);
+
+ // Make sure the test block does not have any outgoing loop backedges.
+ if (!SplitCriticalEdgesForBlock(graph, testBlock)) {
+ return false;
+ }
+
+ MDefinition* trueResult =
+ phi->getOperand(phiBlock->indexForPredecessor(trueBranch));
+ MDefinition* falseResult =
+ phi->getOperand(phiBlock->indexForPredecessor(falseBranch));
+
+ // OK, we found the desired pattern, now transform the graph.
+
+ // Remove the phi from phiBlock.
+ phiBlock->discardPhi(*phiBlock->phisBegin());
+
+ // Change the end of the block to a test that jumps directly to successors of
+ // testBlock, rather than to testBlock itself.
+
+ if (IsTestInputMaybeToBool(initialTest, trueResult)) {
+ if (!UpdateGotoSuccessor(graph.alloc(), trueBranch, finalTest->ifTrue(),
+ testBlock)) {
+ return false;
+ }
+ } else {
+ if (!UpdateTestSuccessors(graph.alloc(), trueBranch, trueResult,
+ finalTest->ifTrue(), finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ }
+
+ if (IsTestInputMaybeToBool(initialTest, falseResult)) {
+ if (!UpdateGotoSuccessor(graph.alloc(), falseBranch, finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ } else {
+ if (!UpdateTestSuccessors(graph.alloc(), falseBranch, falseResult,
+ finalTest->ifTrue(), finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ }
+
+ // Remove phiBlock, if different from testBlock.
+ if (phiBlock != testBlock) {
+ testBlock->removePredecessor(phiBlock);
+ graph.removeBlock(phiBlock);
+ }
+
+ // Remove testBlock itself.
+ finalTest->ifTrue()->removePredecessor(testBlock);
+ finalTest->ifFalse()->removePredecessor(testBlock);
+ graph.removeBlock(testBlock);
+
+ return true;
+}
+
+/*
+ * Look for a triangle pattern:
+ *
+ * initialBlock
+ * / \
+ * trueBranch |
+ * \ /
+ * phiBlock+falseBranch
+ * |
+ * testBlock
+ *
+ * Or:
+ *
+ * initialBlock
+ * / \
+ * | falseBranch
+ * \ /
+ * phiBlock+trueBranch
+ * |
+ * testBlock
+ */
+static bool IsTrianglePattern(MBasicBlock* initialBlock) {
+ MInstruction* ins = initialBlock->lastIns();
+ if (!ins->isTest()) {
+ return false;
+ }
+ MTest* initialTest = ins->toTest();
+
+ MBasicBlock* trueBranch = initialTest->ifTrue();
+ MBasicBlock* falseBranch = initialTest->ifFalse();
+
+ if (trueBranch->numSuccessors() == 1 &&
+ trueBranch->getSuccessor(0) == falseBranch) {
+ if (trueBranch->numPredecessors() != 1) {
+ return false;
+ }
+ if (falseBranch->numPredecessors() != 2) {
+ return false;
+ }
+ return true;
+ }
+
+ if (falseBranch->numSuccessors() == 1 &&
+ falseBranch->getSuccessor(0) == trueBranch) {
+ if (trueBranch->numPredecessors() != 2) {
+ return false;
+ }
+ if (falseBranch->numPredecessors() != 1) {
+ return false;
+ }
+ return true;
+ }
+
+ return false;
+}
+
+static bool MaybeFoldTriangleConditionBlock(MIRGraph& graph,
+ MBasicBlock* initialBlock) {
+ MOZ_ASSERT(IsTrianglePattern(initialBlock));
+
+ // Optimize the MIR graph to improve the code generated for boolean
+ // operations. A test like 'if (a && b)' normally requires three blocks, with
+ // a phi for the intermediate value. This can be improved to use no phi value.
+
+ /*
+ * Look for a triangle pattern:
+ *
+ * initialBlock
+ * / \
+ * trueBranch |
+ * \ /
+ * phiBlock+falseBranch
+ * |
+ * testBlock
+ *
+ * Or:
+ *
+ * initialBlock
+ * / \
+ * | falseBranch
+ * \ /
+ * phiBlock+trueBranch
+ * |
+ * testBlock
+ *
+ * Where phiBlock contains a single phi combining values pushed onto the stack
+ * by trueBranch and falseBranch, and testBlock contains a test on that phi.
+ * phiBlock and testBlock may be the same block; generated code will use
+ * different blocks if the (&&) op is in an inlined function.
+ */
+
+ MTest* initialTest = initialBlock->lastIns()->toTest();
+
+ MBasicBlock* trueBranch = initialTest->ifTrue();
+ MBasicBlock* falseBranch = initialTest->ifFalse();
+ if (initialBlock->isLoopBackedge() || trueBranch->isLoopBackedge() ||
+ falseBranch->isLoopBackedge()) {
+ return true;
+ }
+
+ MBasicBlock* phiBlock;
+ if (trueBranch->numSuccessors() == 1 &&
+ trueBranch->getSuccessor(0) == falseBranch) {
+ phiBlock = falseBranch;
+ } else {
+ MOZ_ASSERT(falseBranch->getSuccessor(0) == trueBranch);
+ phiBlock = trueBranch;
+ }
+
+ MBasicBlock* testBlock = phiBlock;
+ if (testBlock->numSuccessors() == 1) {
+ MOZ_ASSERT(!testBlock->isLoopBackedge());
+
+ testBlock = testBlock->getSuccessor(0);
+ if (testBlock->numPredecessors() != 1) {
+ return true;
+ }
+ }
+
+ MPhi* phi;
+ MTest* finalTest;
+ if (!BlockIsSingleTest(phiBlock, testBlock, &phi, &finalTest)) {
+ return true;
+ }
+
+ MOZ_ASSERT(phi->numOperands() == 2);
+
+ // If the phi-operand doesn't match the initial input, we can't fold the test.
+ auto* phiInputForInitialBlock =
+ phi->getOperand(phiBlock->indexForPredecessor(initialBlock));
+ if (!IsTestInputMaybeToBool(initialTest, phiInputForInitialBlock)) {
+ return true;
+ }
+
+ // Make sure the test block does not have any outgoing loop backedges.
+ if (!SplitCriticalEdgesForBlock(graph, testBlock)) {
+ return false;
+ }
+
+ MDefinition* trueResult;
+ MDefinition* falseResult;
+ if (phiBlock == trueBranch) {
+ trueResult = phi->getOperand(phiBlock->indexForPredecessor(initialBlock));
+ falseResult = phi->getOperand(phiBlock->indexForPredecessor(falseBranch));
+ } else {
+ trueResult = phi->getOperand(phiBlock->indexForPredecessor(trueBranch));
+ falseResult = phi->getOperand(phiBlock->indexForPredecessor(initialBlock));
+ }
+
+ // OK, we found the desired pattern, now transform the graph.
+
+ // Remove the phi from phiBlock.
+ phiBlock->discardPhi(*phiBlock->phisBegin());
+
+ // Change the end of the block to a test that jumps directly to successors of
+ // testBlock, rather than to testBlock itself.
+
+ if (phiBlock == trueBranch) {
+ if (!UpdateTestSuccessors(graph.alloc(), initialBlock, initialTest->input(),
+ finalTest->ifTrue(), initialTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ } else if (IsTestInputMaybeToBool(initialTest, trueResult)) {
+ if (!UpdateGotoSuccessor(graph.alloc(), trueBranch, finalTest->ifTrue(),
+ testBlock)) {
+ return false;
+ }
+ } else {
+ if (!UpdateTestSuccessors(graph.alloc(), trueBranch, trueResult,
+ finalTest->ifTrue(), finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ }
+
+ if (phiBlock == falseBranch) {
+ if (!UpdateTestSuccessors(graph.alloc(), initialBlock, initialTest->input(),
+ initialTest->ifTrue(), finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ } else if (IsTestInputMaybeToBool(initialTest, falseResult)) {
+ if (!UpdateGotoSuccessor(graph.alloc(), falseBranch, finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ } else {
+ if (!UpdateTestSuccessors(graph.alloc(), falseBranch, falseResult,
+ finalTest->ifTrue(), finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ }
+
+ // Remove phiBlock, if different from testBlock.
+ if (phiBlock != testBlock) {
+ testBlock->removePredecessor(phiBlock);
+ graph.removeBlock(phiBlock);
+ }
+
+ // Remove testBlock itself.
+ finalTest->ifTrue()->removePredecessor(testBlock);
+ finalTest->ifFalse()->removePredecessor(testBlock);
+ graph.removeBlock(testBlock);
+
+ return true;
+}
+
+static bool MaybeFoldConditionBlock(MIRGraph& graph,
+ MBasicBlock* initialBlock) {
+ if (IsDiamondPattern(initialBlock)) {
+ return MaybeFoldDiamondConditionBlock(graph, initialBlock);
+ }
+ if (IsTrianglePattern(initialBlock)) {
+ return MaybeFoldTriangleConditionBlock(graph, initialBlock);
+ }
+ return true;
+}
+
+static bool MaybeFoldTestBlock(MIRGraph& graph, MBasicBlock* initialBlock) {
+ // Handle test expressions on more than two inputs. For example
+ // |if ((x > 10) && (y > 20) && (z > 30)) { ... }|, which results in the below
+ // pattern.
+ //
+ // Look for the pattern:
+ // ┌─────────────────┐
+ // 1 │ 1 compare │
+ // ┌─────┤ 2 test compare1 │
+ // │ └──────┬──────────┘
+ // │ │0
+ // ┌───────▼─────────┐ │
+ // │ 3 compare │ │
+ // │ 4 test compare3 │ └──────────┐
+ // └──┬──────────┬───┘ │
+ // 1│ │0 │
+ // ┌──────────▼──────┐ │ │
+ // │ 5 compare │ └─────────┐ │
+ // │ 6 goto │ │ │
+ // └───────┬─────────┘ │ │
+ // │ │ │
+ // │ ┌──────────────────▼───────▼───────┐
+ // └───►│ 9 phi compare1 compare3 compare5 │
+ // │10 goto │
+ // └────────────────┬─────────────────┘
+ // │
+ // ┌────────▼────────┐
+ // │11 test phi9 │
+ // └─────┬─────┬─────┘
+ // 1│ │0
+ // ┌────────────┐ │ │ ┌─────────────┐
+ // │ TrueBranch │◄────┘ └─────►│ FalseBranch │
+ // └────────────┘ └─────────────┘
+ //
+ // And transform it to:
+ //
+ // ┌─────────────────┐
+ // 1 │ 1 compare │
+ // ┌───┤ 2 test compare1 │
+ // │ └──────────┬──────┘
+ // │ │0
+ // ┌───────▼─────────┐ │
+ // │ 3 compare │ │
+ // │ 4 test compare3 │ │
+ // └──┬─────────┬────┘ │
+ // 1│ │0 │
+ // ┌──────────▼──────┐ │ │
+ // │ 5 compare │ └──────┐ │
+ // │ 6 test compare5 │ │ │
+ // └────┬────────┬───┘ │ │
+ // 1│ │0 │ │
+ // ┌─────▼──────┐ │ ┌───▼──▼──────┐
+ // │ TrueBranch │ └─────────► FalseBranch │
+ // └────────────┘ └─────────────┘
+
+ auto* ins = initialBlock->lastIns();
+ if (!ins->isTest()) {
+ return true;
+ }
+ auto* initialTest = ins->toTest();
+
+ MBasicBlock* trueBranch = initialTest->ifTrue();
+ MBasicBlock* falseBranch = initialTest->ifFalse();
+
+ // MaybeFoldConditionBlock handles the case for two operands.
+ MBasicBlock* phiBlock;
+ if (trueBranch->numPredecessors() > 2) {
+ phiBlock = trueBranch;
+ } else if (falseBranch->numPredecessors() > 2) {
+ phiBlock = falseBranch;
+ } else {
+ return true;
+ }
+
+ MBasicBlock* testBlock = phiBlock;
+ if (testBlock->numSuccessors() == 1) {
+ if (testBlock->isLoopBackedge()) {
+ return true;
+ }
+ testBlock = testBlock->getSuccessor(0);
+ if (testBlock->numPredecessors() != 1) {
+ return true;
+ }
+ }
+
+ MOZ_ASSERT(!phiBlock->isLoopBackedge());
+
+ MPhi* phi = nullptr;
+ MTest* finalTest = nullptr;
+ if (!BlockIsSingleTest(phiBlock, testBlock, &phi, &finalTest)) {
+ return true;
+ }
+
+ MOZ_ASSERT(phiBlock->numPredecessors() == phi->numOperands());
+
+ // If the phi-operand doesn't match the initial input, we can't fold the test.
+ auto* phiInputForInitialBlock =
+ phi->getOperand(phiBlock->indexForPredecessor(initialBlock));
+ if (!IsTestInputMaybeToBool(initialTest, phiInputForInitialBlock)) {
+ return true;
+ }
+
+ MBasicBlock* newTestBlock = nullptr;
+ MDefinition* newTestInput = nullptr;
+
+ // The block of each phi operand must either end with a test instruction on
+ // that phi operand or it's the sole block which ends with a goto instruction.
+ for (size_t i = 0; i < phiBlock->numPredecessors(); i++) {
+ auto* pred = phiBlock->getPredecessor(i);
+ auto* operand = phi->getOperand(i);
+
+ // Each predecessor must end with either a test or goto instruction.
+ auto* lastIns = pred->lastIns();
+ if (lastIns->isGoto() && !newTestBlock) {
+ newTestBlock = pred;
+ newTestInput = operand;
+ } else if (lastIns->isTest()) {
+ if (!IsTestInputMaybeToBool(lastIns->toTest(), operand)) {
+ return true;
+ }
+ } else {
+ return true;
+ }
+
+ MOZ_ASSERT(!pred->isLoopBackedge());
+ }
+
+ // Ensure we found the single goto block.
+ if (!newTestBlock) {
+ return true;
+ }
+
+ // Make sure the test block does not have any outgoing loop backedges.
+ if (!SplitCriticalEdgesForBlock(graph, testBlock)) {
+ return false;
+ }
+
+ // OK, we found the desired pattern, now transform the graph.
+
+ // Remove the phi from phiBlock.
+ phiBlock->discardPhi(*phiBlock->phisBegin());
+
+ // Create the new test instruction.
+ if (!UpdateTestSuccessors(graph.alloc(), newTestBlock, newTestInput,
+ finalTest->ifTrue(), finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+
+ // Update all test instructions to point to the final target.
+ while (phiBlock->numPredecessors()) {
+ mozilla::DebugOnly<size_t> oldNumPred = phiBlock->numPredecessors();
+
+ auto* pred = phiBlock->getPredecessor(0);
+ auto* test = pred->lastIns()->toTest();
+ if (test->ifTrue() == phiBlock) {
+ if (!UpdateTestSuccessors(graph.alloc(), pred, test->input(),
+ finalTest->ifTrue(), test->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(test->ifFalse() == phiBlock);
+ if (!UpdateTestSuccessors(graph.alloc(), pred, test->input(),
+ test->ifTrue(), finalTest->ifFalse(),
+ testBlock)) {
+ return false;
+ }
+ }
+
+ // Ensure we've made progress.
+ MOZ_ASSERT(phiBlock->numPredecessors() + 1 == oldNumPred);
+ }
+
+ // Remove phiBlock, if different from testBlock.
+ if (phiBlock != testBlock) {
+ testBlock->removePredecessor(phiBlock);
+ graph.removeBlock(phiBlock);
+ }
+
+ // Remove testBlock itself.
+ finalTest->ifTrue()->removePredecessor(testBlock);
+ finalTest->ifFalse()->removePredecessor(testBlock);
+ graph.removeBlock(testBlock);
+
+ return true;
+}
+
+bool jit::FoldTests(MIRGraph& graph) {
+ for (PostorderIterator block(graph.poBegin()); block != graph.poEnd();
+ block++) {
+ if (!MaybeFoldConditionBlock(graph, *block)) {
+ return false;
+ }
+ if (!MaybeFoldTestBlock(graph, *block)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+bool jit::FoldEmptyBlocks(MIRGraph& graph) {
+ for (MBasicBlockIterator iter(graph.begin()); iter != graph.end();) {
+ MBasicBlock* block = *iter;
+ iter++;
+
+ if (block->numPredecessors() != 1 || block->numSuccessors() != 1) {
+ continue;
+ }
+
+ if (!block->phisEmpty()) {
+ continue;
+ }
+
+ if (block->outerResumePoint()) {
+ continue;
+ }
+
+ if (*block->begin() != *block->rbegin()) {
+ continue;
+ }
+
+ MBasicBlock* succ = block->getSuccessor(0);
+ MBasicBlock* pred = block->getPredecessor(0);
+
+ if (succ->numPredecessors() != 1) {
+ continue;
+ }
+
+ size_t pos = pred->getSuccessorIndex(block);
+ pred->lastIns()->replaceSuccessor(pos, succ);
+
+ graph.removeBlock(block);
+
+ if (!succ->addPredecessorSameInputsAs(pred, block)) {
+ return false;
+ }
+ succ->removePredecessor(block);
+ }
+ return true;
+}
+
+static void EliminateTriviallyDeadResumePointOperands(MIRGraph& graph,
+ MResumePoint* rp) {
+ // If we will pop the top of the stack immediately after resuming,
+ // then don't preserve the top value in the resume point.
+ if (rp->mode() != ResumeMode::ResumeAt) {
+ return;
+ }
+
+ jsbytecode* pc = rp->pc();
+ if (JSOp(*pc) == JSOp::JumpTarget) {
+ pc += JSOpLength_JumpTarget;
+ }
+ if (JSOp(*pc) != JSOp::Pop) {
+ return;
+ }
+
+ size_t top = rp->stackDepth() - 1;
+ MOZ_ASSERT(!rp->isObservableOperand(top));
+
+ MDefinition* def = rp->getOperand(top);
+ if (def->isConstant()) {
+ return;
+ }
+
+ MConstant* constant = rp->block()->optimizedOutConstant(graph.alloc());
+ rp->replaceOperand(top, constant);
+}
+
+// Operands to a resume point which are dead at the point of the resume can be
+// replaced with a magic value. This pass only replaces resume points which are
+// trivially dead.
+//
+// This is intended to ensure that extra resume points within a basic block
+// will not artificially extend the lifetimes of any SSA values. This could
+// otherwise occur if the new resume point captured a value which is created
+// between the old and new resume point and is dead at the new resume point.
+bool jit::EliminateTriviallyDeadResumePointOperands(MIRGenerator* mir,
+ MIRGraph& graph) {
+ for (auto* block : graph) {
+ if (MResumePoint* rp = block->entryResumePoint()) {
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+ ::EliminateTriviallyDeadResumePointOperands(graph, rp);
+ }
+ }
+ return true;
+}
+
+// Operands to a resume point which are dead at the point of the resume can be
+// replaced with a magic value. This analysis supports limited detection of
+// dead operands, pruning those which are defined in the resume point's basic
+// block and have no uses outside the block or at points later than the resume
+// point.
+//
+// This is intended to ensure that extra resume points within a basic block
+// will not artificially extend the lifetimes of any SSA values. This could
+// otherwise occur if the new resume point captured a value which is created
+// between the old and new resume point and is dead at the new resume point.
+bool jit::EliminateDeadResumePointOperands(MIRGenerator* mir, MIRGraph& graph) {
+ // If we are compiling try blocks, locals and arguments may be observable
+ // from catch or finally blocks (which Ion does not compile). For now just
+ // disable the pass in this case.
+ if (graph.hasTryBlock()) {
+ return true;
+ }
+
+ for (PostorderIterator block = graph.poBegin(); block != graph.poEnd();
+ block++) {
+ if (mir->shouldCancel("Eliminate Dead Resume Point Operands (main loop)")) {
+ return false;
+ }
+
+ if (MResumePoint* rp = block->entryResumePoint()) {
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+ ::EliminateTriviallyDeadResumePointOperands(graph, rp);
+ }
+
+ // The logic below can get confused on infinite loops.
+ if (block->isLoopHeader() && block->backedge() == *block) {
+ continue;
+ }
+
+ for (MInstructionIterator ins = block->begin(); ins != block->end();
+ ins++) {
+ if (MResumePoint* rp = ins->resumePoint()) {
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+ ::EliminateTriviallyDeadResumePointOperands(graph, rp);
+ }
+
+ // No benefit to replacing constant operands with other constants.
+ if (ins->isConstant()) {
+ continue;
+ }
+
+ // Scanning uses does not give us sufficient information to tell
+ // where instructions that are involved in box/unbox operations or
+ // parameter passing might be live. Rewriting uses of these terms
+ // in resume points may affect the interpreter's behavior. Rather
+ // than doing a more sophisticated analysis, just ignore these.
+ if (ins->isUnbox() || ins->isParameter() || ins->isBoxNonStrictThis()) {
+ continue;
+ }
+
+ // Early intermediate values captured by resume points, such as
+ // ArrayState and its allocation, may be legitimately dead in Ion code,
+ // but are still needed if we bail out. They can recover on bailout.
+ if (ins->isRecoveredOnBailout()) {
+ MOZ_ASSERT(ins->canRecoverOnBailout());
+ continue;
+ }
+
+ // If the instruction's behavior has been constant folded into a
+ // separate instruction, we can't determine precisely where the
+ // instruction becomes dead and can't eliminate its uses.
+ if (ins->isImplicitlyUsed()) {
+ continue;
+ }
+
+ // Check if this instruction's result is only used within the
+ // current block, and keep track of its last use in a definition
+ // (not resume point). This requires the instructions in the block
+ // to be numbered, ensured by running this immediately after alias
+ // analysis.
+ uint32_t maxDefinition = 0;
+ for (MUseIterator uses(ins->usesBegin()); uses != ins->usesEnd();
+ uses++) {
+ MNode* consumer = uses->consumer();
+ if (consumer->isResumePoint()) {
+ // If the instruction's is captured by one of the resume point, then
+ // it might be observed indirectly while the frame is live on the
+ // stack, so it has to be computed.
+ MResumePoint* resume = consumer->toResumePoint();
+ if (resume->isObservableOperand(*uses)) {
+ maxDefinition = UINT32_MAX;
+ break;
+ }
+ continue;
+ }
+
+ MDefinition* def = consumer->toDefinition();
+ if (def->block() != *block || def->isBox() || def->isPhi()) {
+ maxDefinition = UINT32_MAX;
+ break;
+ }
+ maxDefinition = std::max(maxDefinition, def->id());
+ }
+ if (maxDefinition == UINT32_MAX) {
+ continue;
+ }
+
+ // Walk the uses a second time, removing any in resume points after
+ // the last use in a definition.
+ for (MUseIterator uses(ins->usesBegin()); uses != ins->usesEnd();) {
+ MUse* use = *uses++;
+ if (use->consumer()->isDefinition()) {
+ continue;
+ }
+ MResumePoint* mrp = use->consumer()->toResumePoint();
+ if (mrp->block() != *block || !mrp->instruction() ||
+ mrp->instruction() == *ins ||
+ mrp->instruction()->id() <= maxDefinition) {
+ continue;
+ }
+
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+
+ // Store an optimized out magic value in place of all dead
+ // resume point operands. Making any such substitution can in
+ // general alter the interpreter's behavior, even though the
+ // code is dead, as the interpreter will still execute opcodes
+ // whose effects cannot be observed. If the magic value value
+ // were to flow to, say, a dead property access the
+ // interpreter could throw an exception; we avoid this problem
+ // by removing dead operands before removing dead code.
+ MConstant* constant =
+ MConstant::New(graph.alloc(), MagicValue(JS_OPTIMIZED_OUT));
+ block->insertBefore(*(block->begin()), constant);
+ use->replaceProducer(constant);
+ }
+ }
+ }
+
+ return true;
+}
+
+// Test whether |def| would be needed if it had no uses.
+bool js::jit::DeadIfUnused(const MDefinition* def) {
+ // Effectful instructions of course cannot be removed.
+ if (def->isEffectful()) {
+ return false;
+ }
+
+ // Never eliminate guard instructions.
+ if (def->isGuard()) {
+ return false;
+ }
+
+ // Required to be preserved, as the type guard related to this instruction
+ // is part of the semantics of a transformation.
+ if (def->isGuardRangeBailouts()) {
+ return false;
+ }
+
+ // Control instructions have no uses, but also shouldn't be optimized out
+ if (def->isControlInstruction()) {
+ return false;
+ }
+
+ // Used when lowering to generate the corresponding snapshots and aggregate
+ // the list of recover instructions to be repeated.
+ if (def->isInstruction() && def->toInstruction()->resumePoint()) {
+ return false;
+ }
+
+ return true;
+}
+
+// Similar to DeadIfUnused(), but additionally allows effectful instructions.
+bool js::jit::DeadIfUnusedAllowEffectful(const MDefinition* def) {
+ // Never eliminate guard instructions.
+ if (def->isGuard()) {
+ return false;
+ }
+
+ // Required to be preserved, as the type guard related to this instruction
+ // is part of the semantics of a transformation.
+ if (def->isGuardRangeBailouts()) {
+ return false;
+ }
+
+ // Control instructions have no uses, but also shouldn't be optimized out
+ if (def->isControlInstruction()) {
+ return false;
+ }
+
+ // Used when lowering to generate the corresponding snapshots and aggregate
+ // the list of recover instructions to be repeated.
+ if (def->isInstruction() && def->toInstruction()->resumePoint()) {
+ // All effectful instructions must have a resume point attached. We're
+ // allowing effectful instructions here, so we have to ignore any resume
+ // points if we want to consider effectful instructions as dead.
+ if (!def->isEffectful()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// Test whether |def| may be safely discarded, due to being dead or due to being
+// located in a basic block which has itself been marked for discarding.
+bool js::jit::IsDiscardable(const MDefinition* def) {
+ return !def->hasUses() && (DeadIfUnused(def) || def->block()->isMarked());
+}
+
+// Similar to IsDiscardable(), but additionally allows effectful instructions.
+bool js::jit::IsDiscardableAllowEffectful(const MDefinition* def) {
+ return !def->hasUses() &&
+ (DeadIfUnusedAllowEffectful(def) || def->block()->isMarked());
+}
+
+// Instructions are useless if they are unused and have no side effects.
+// This pass eliminates useless instructions.
+// The graph itself is unchanged.
+bool jit::EliminateDeadCode(MIRGenerator* mir, MIRGraph& graph) {
+ // Traverse in postorder so that we hit uses before definitions.
+ // Traverse instruction list backwards for the same reason.
+ for (PostorderIterator block = graph.poBegin(); block != graph.poEnd();
+ block++) {
+ if (mir->shouldCancel("Eliminate Dead Code (main loop)")) {
+ return false;
+ }
+
+ // Remove unused instructions.
+ for (MInstructionReverseIterator iter = block->rbegin();
+ iter != block->rend();) {
+ MInstruction* inst = *iter++;
+ if (js::jit::IsDiscardable(inst)) {
+ block->discard(inst);
+ }
+ }
+ }
+
+ return true;
+}
+
+static inline bool IsPhiObservable(MPhi* phi, Observability observe) {
+ // If the phi has uses which are not reflected in SSA, then behavior in the
+ // interpreter may be affected by removing the phi.
+ if (phi->isImplicitlyUsed()) {
+ return true;
+ }
+
+ // Check for uses of this phi node outside of other phi nodes.
+ // Note that, initially, we skip reading resume points, which we
+ // don't count as actual uses. If the only uses are resume points,
+ // then the SSA name is never consumed by the program. However,
+ // after optimizations have been performed, it's possible that the
+ // actual uses in the program have been (incorrectly) optimized
+ // away, so we must be more conservative and consider resume
+ // points as well.
+ for (MUseIterator iter(phi->usesBegin()); iter != phi->usesEnd(); iter++) {
+ MNode* consumer = iter->consumer();
+ if (consumer->isResumePoint()) {
+ MResumePoint* resume = consumer->toResumePoint();
+ if (observe == ConservativeObservability) {
+ return true;
+ }
+ if (resume->isObservableOperand(*iter)) {
+ return true;
+ }
+ } else {
+ MDefinition* def = consumer->toDefinition();
+ if (!def->isPhi()) {
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+// Handles cases like:
+// x is phi(a, x) --> a
+// x is phi(a, a) --> a
+static inline MDefinition* IsPhiRedundant(MPhi* phi) {
+ MDefinition* first = phi->operandIfRedundant();
+ if (first == nullptr) {
+ return nullptr;
+ }
+
+ // Propagate the ImplicitlyUsed flag if |phi| is replaced with another phi.
+ if (phi->isImplicitlyUsed()) {
+ first->setImplicitlyUsedUnchecked();
+ }
+
+ return first;
+}
+
+bool jit::EliminatePhis(MIRGenerator* mir, MIRGraph& graph,
+ Observability observe) {
+ // Eliminates redundant or unobservable phis from the graph. A
+ // redundant phi is something like b = phi(a, a) or b = phi(a, b),
+ // both of which can be replaced with a. An unobservable phi is
+ // one that whose value is never used in the program.
+ //
+ // Note that we must be careful not to eliminate phis representing
+ // values that the interpreter will require later. When the graph
+ // is first constructed, we can be more aggressive, because there
+ // is a greater correspondence between the CFG and the bytecode.
+ // After optimizations such as GVN have been performed, however,
+ // the bytecode and CFG may not correspond as closely to one
+ // another. In that case, we must be more conservative. The flag
+ // |conservativeObservability| is used to indicate that eliminate
+ // phis is being run after some optimizations have been performed,
+ // and thus we should use more conservative rules about
+ // observability. The particular danger is that we can optimize
+ // away uses of a phi because we think they are not executable,
+ // but the foundation for that assumption is false TI information
+ // that will eventually be invalidated. Therefore, if
+ // |conservativeObservability| is set, we will consider any use
+ // from a resume point to be observable. Otherwise, we demand a
+ // use from an actual instruction.
+
+ Vector<MPhi*, 16, SystemAllocPolicy> worklist;
+
+ // Add all observable phis to a worklist. We use the "in worklist" bit to
+ // mean "this phi is live".
+ for (PostorderIterator block = graph.poBegin(); block != graph.poEnd();
+ block++) {
+ MPhiIterator iter = block->phisBegin();
+ while (iter != block->phisEnd()) {
+ MPhi* phi = *iter++;
+
+ if (mir->shouldCancel("Eliminate Phis (populate loop)")) {
+ return false;
+ }
+
+ // Flag all as unused, only observable phis would be marked as used
+ // when processed by the work list.
+ phi->setUnused();
+
+ // If the phi is redundant, remove it here.
+ if (MDefinition* redundant = IsPhiRedundant(phi)) {
+ phi->justReplaceAllUsesWith(redundant);
+ block->discardPhi(phi);
+ continue;
+ }
+
+ // Enqueue observable Phis.
+ if (IsPhiObservable(phi, observe)) {
+ phi->setInWorklist();
+ if (!worklist.append(phi)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ // Iteratively mark all phis reachable from live phis.
+ while (!worklist.empty()) {
+ if (mir->shouldCancel("Eliminate Phis (worklist)")) {
+ return false;
+ }
+
+ MPhi* phi = worklist.popCopy();
+ MOZ_ASSERT(phi->isUnused());
+ phi->setNotInWorklist();
+
+ // The removal of Phis can produce newly redundant phis.
+ if (MDefinition* redundant = IsPhiRedundant(phi)) {
+ // Add to the worklist the used phis which are impacted.
+ for (MUseDefIterator it(phi); it; it++) {
+ if (it.def()->isPhi()) {
+ MPhi* use = it.def()->toPhi();
+ if (!use->isUnused()) {
+ use->setUnusedUnchecked();
+ use->setInWorklist();
+ if (!worklist.append(use)) {
+ return false;
+ }
+ }
+ }
+ }
+ phi->justReplaceAllUsesWith(redundant);
+ } else {
+ // Otherwise flag them as used.
+ phi->setNotUnused();
+ }
+
+ // The current phi is/was used, so all its operands are used.
+ for (size_t i = 0, e = phi->numOperands(); i < e; i++) {
+ MDefinition* in = phi->getOperand(i);
+ if (!in->isPhi() || !in->isUnused() || in->isInWorklist()) {
+ continue;
+ }
+ in->setInWorklist();
+ if (!worklist.append(in->toPhi())) {
+ return false;
+ }
+ }
+ }
+
+ // Sweep dead phis.
+ for (PostorderIterator block = graph.poBegin(); block != graph.poEnd();
+ block++) {
+ MPhiIterator iter = block->phisBegin();
+ while (iter != block->phisEnd()) {
+ MPhi* phi = *iter++;
+ if (phi->isUnused()) {
+ if (!phi->optimizeOutAllUses(graph.alloc())) {
+ return false;
+ }
+ block->discardPhi(phi);
+ }
+ }
+ }
+
+ return true;
+}
+
+namespace {
+
+// The type analysis algorithm inserts conversions and box/unbox instructions
+// to make the IR graph well-typed for future passes.
+//
+// Phi adjustment: If a phi's inputs are all the same type, the phi is
+// specialized to return that type.
+//
+// Input adjustment: Each input is asked to apply conversion operations to its
+// inputs. This may include Box, Unbox, or other instruction-specific type
+// conversion operations.
+//
+class TypeAnalyzer {
+ MIRGenerator* mir;
+ MIRGraph& graph;
+ Vector<MPhi*, 0, SystemAllocPolicy> phiWorklist_;
+
+ TempAllocator& alloc() const { return graph.alloc(); }
+
+ bool addPhiToWorklist(MPhi* phi) {
+ if (phi->isInWorklist()) {
+ return true;
+ }
+ if (!phiWorklist_.append(phi)) {
+ return false;
+ }
+ phi->setInWorklist();
+ return true;
+ }
+ MPhi* popPhi() {
+ MPhi* phi = phiWorklist_.popCopy();
+ phi->setNotInWorklist();
+ return phi;
+ }
+
+ [[nodiscard]] bool propagateAllPhiSpecializations();
+
+ bool respecialize(MPhi* phi, MIRType type);
+ bool propagateSpecialization(MPhi* phi);
+ bool specializePhis();
+ bool specializeOsrOnlyPhis();
+ void replaceRedundantPhi(MPhi* phi);
+ bool adjustPhiInputs(MPhi* phi);
+ bool adjustInputs(MDefinition* def);
+ bool insertConversions();
+
+ bool checkFloatCoherency();
+ bool graphContainsFloat32();
+ bool markPhiConsumers();
+ bool markPhiProducers();
+ bool specializeValidFloatOps();
+ bool tryEmitFloatOperations();
+
+ bool shouldSpecializeOsrPhis() const;
+ MIRType guessPhiType(MPhi* phi) const;
+
+ public:
+ TypeAnalyzer(MIRGenerator* mir, MIRGraph& graph) : mir(mir), graph(graph) {}
+
+ bool analyze();
+};
+
+} /* anonymous namespace */
+
+bool TypeAnalyzer::shouldSpecializeOsrPhis() const {
+ // [SMDOC] OSR Phi Type Specialization
+ //
+ // Without special handling for OSR phis, we end up with unspecialized phis
+ // (MIRType::Value) in the loop (pre)header and other blocks, resulting in
+ // unnecessary boxing and unboxing in the loop body.
+ //
+ // To fix this, phi type specialization needs special code to deal with the
+ // OSR entry block. Recall that OSR results in the following basic block
+ // structure:
+ //
+ // +------------------+ +-----------------+
+ // | Code before loop | | OSR entry block |
+ // +------------------+ +-----------------+
+ // | |
+ // | |
+ // | +---------------+ |
+ // +---------> | OSR preheader | <---------+
+ // +---------------+
+ // |
+ // V
+ // +---------------+
+ // | Loop header |<-----+
+ // +---------------+ |
+ // | |
+ // ... |
+ // | |
+ // +---------------+ |
+ // | Loop backedge |------+
+ // +---------------+
+ //
+ // OSR phi specialization happens in three steps:
+ //
+ // (1) Specialize phis but ignore MOsrValue phi inputs. In other words,
+ // pretend the OSR entry block doesn't exist. See guessPhiType.
+ //
+ // (2) Once phi specialization is done, look at the types of loop header phis
+ // and add these types to the corresponding preheader phis. This way, the
+ // types of the preheader phis are based on the code before the loop and
+ // the code in the loop body. These are exactly the types we expect for
+ // the OSR Values. See the last part of TypeAnalyzer::specializePhis.
+ //
+ // (3) For type-specialized preheader phis, add guard/unbox instructions to
+ // the OSR entry block to guard the incoming Value indeed has this type.
+ // This happens in:
+ //
+ // * TypeAnalyzer::adjustPhiInputs: adds a fallible unbox for values that
+ // can be unboxed.
+ //
+ // * TypeAnalyzer::replaceRedundantPhi: adds a type guard for values that
+ // can't be unboxed (null/undefined/magic Values).
+ if (!mir->graph().osrBlock()) {
+ return false;
+ }
+
+ return !mir->outerInfo().hadSpeculativePhiBailout();
+}
+
+// Try to specialize this phi based on its non-cyclic inputs.
+MIRType TypeAnalyzer::guessPhiType(MPhi* phi) const {
+#ifdef DEBUG
+ // Check that different magic constants aren't flowing together. Ignore
+ // JS_OPTIMIZED_OUT, since an operand could be legitimately optimized
+ // away.
+ MIRType magicType = MIRType::None;
+ for (size_t i = 0; i < phi->numOperands(); i++) {
+ MDefinition* in = phi->getOperand(i);
+ if (in->type() == MIRType::MagicHole ||
+ in->type() == MIRType::MagicIsConstructing) {
+ if (magicType == MIRType::None) {
+ magicType = in->type();
+ }
+ MOZ_ASSERT(magicType == in->type());
+ }
+ }
+#endif
+
+ MIRType type = MIRType::None;
+ bool convertibleToFloat32 = false;
+ bool hasOSRValueInput = false;
+ DebugOnly<bool> hasSpecializableInput = false;
+ for (size_t i = 0, e = phi->numOperands(); i < e; i++) {
+ MDefinition* in = phi->getOperand(i);
+ if (in->isPhi()) {
+ hasSpecializableInput = true;
+ if (!in->toPhi()->triedToSpecialize()) {
+ continue;
+ }
+ if (in->type() == MIRType::None) {
+ // The operand is a phi we tried to specialize, but we were
+ // unable to guess its type. propagateSpecialization will
+ // propagate the type to this phi when it becomes known.
+ continue;
+ }
+ }
+
+ // See shouldSpecializeOsrPhis comment. This is the first step mentioned
+ // there.
+ if (shouldSpecializeOsrPhis() && in->isOsrValue()) {
+ hasOSRValueInput = true;
+ hasSpecializableInput = true;
+ continue;
+ }
+
+ if (type == MIRType::None) {
+ type = in->type();
+ if (in->canProduceFloat32() &&
+ !mir->outerInfo().hadSpeculativePhiBailout()) {
+ convertibleToFloat32 = true;
+ }
+ continue;
+ }
+
+ if (type == in->type()) {
+ convertibleToFloat32 = convertibleToFloat32 && in->canProduceFloat32();
+ } else {
+ if (convertibleToFloat32 && in->type() == MIRType::Float32) {
+ // If we only saw definitions that can be converted into Float32 before
+ // and encounter a Float32 value, promote previous values to Float32
+ type = MIRType::Float32;
+ } else if (IsTypeRepresentableAsDouble(type) &&
+ IsTypeRepresentableAsDouble(in->type())) {
+ // Specialize phis with int32 and double operands as double.
+ type = MIRType::Double;
+ convertibleToFloat32 = convertibleToFloat32 && in->canProduceFloat32();
+ } else {
+ return MIRType::Value;
+ }
+ }
+ }
+
+ if (hasOSRValueInput && type == MIRType::Float32) {
+ // TODO(post-Warp): simplify float32 handling in this function or (better)
+ // make the float32 analysis a stand-alone optimization pass instead of
+ // complicating type analysis. See bug 1655773.
+ type = MIRType::Double;
+ }
+
+ MOZ_ASSERT_IF(type == MIRType::None, hasSpecializableInput);
+ return type;
+}
+
+bool TypeAnalyzer::respecialize(MPhi* phi, MIRType type) {
+ if (phi->type() == type) {
+ return true;
+ }
+ phi->specialize(type);
+ return addPhiToWorklist(phi);
+}
+
+bool TypeAnalyzer::propagateSpecialization(MPhi* phi) {
+ MOZ_ASSERT(phi->type() != MIRType::None);
+
+ // Verify that this specialization matches any phis depending on it.
+ for (MUseDefIterator iter(phi); iter; iter++) {
+ if (!iter.def()->isPhi()) {
+ continue;
+ }
+ MPhi* use = iter.def()->toPhi();
+ if (!use->triedToSpecialize()) {
+ continue;
+ }
+ if (use->type() == MIRType::None) {
+ // We tried to specialize this phi, but were unable to guess its
+ // type. Now that we know the type of one of its operands, we can
+ // specialize it. If it can't be specialized as float32, specialize
+ // as double.
+ MIRType type = phi->type();
+ if (type == MIRType::Float32 && !use->canProduceFloat32()) {
+ type = MIRType::Double;
+ }
+ if (!respecialize(use, type)) {
+ return false;
+ }
+ continue;
+ }
+ if (use->type() != phi->type()) {
+ // Specialize phis with int32 that can be converted to float and float
+ // operands as floats.
+ if ((use->type() == MIRType::Int32 && use->canProduceFloat32() &&
+ phi->type() == MIRType::Float32) ||
+ (phi->type() == MIRType::Int32 && phi->canProduceFloat32() &&
+ use->type() == MIRType::Float32)) {
+ if (!respecialize(use, MIRType::Float32)) {
+ return false;
+ }
+ continue;
+ }
+
+ // Specialize phis with int32 and double operands as double.
+ if (IsTypeRepresentableAsDouble(use->type()) &&
+ IsTypeRepresentableAsDouble(phi->type())) {
+ if (!respecialize(use, MIRType::Double)) {
+ return false;
+ }
+ continue;
+ }
+
+ // This phi in our use chain can now no longer be specialized.
+ if (!respecialize(use, MIRType::Value)) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+bool TypeAnalyzer::propagateAllPhiSpecializations() {
+ while (!phiWorklist_.empty()) {
+ if (mir->shouldCancel("Specialize Phis (worklist)")) {
+ return false;
+ }
+
+ MPhi* phi = popPhi();
+ if (!propagateSpecialization(phi)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// If branch pruning removes the path from the entry block to the OSR
+// preheader, we may have phis (or chains of phis) with no operands
+// other than OsrValues. These phis will still have MIRType::None.
+// Since we don't have any information about them, we specialize them
+// as MIRType::Value.
+bool TypeAnalyzer::specializeOsrOnlyPhis() {
+ MOZ_ASSERT(graph.osrBlock());
+ MOZ_ASSERT(graph.osrPreHeaderBlock()->numPredecessors() == 1);
+
+ for (PostorderIterator block(graph.poBegin()); block != graph.poEnd();
+ block++) {
+ if (mir->shouldCancel("Specialize osr-only phis (main loop)")) {
+ return false;
+ }
+
+ for (MPhiIterator phi(block->phisBegin()); phi != block->phisEnd(); phi++) {
+ if (mir->shouldCancel("Specialize osr-only phis (inner loop)")) {
+ return false;
+ }
+
+ if (phi->type() == MIRType::None) {
+ phi->specialize(MIRType::Value);
+ }
+ }
+ }
+ return true;
+}
+
+bool TypeAnalyzer::specializePhis() {
+ for (PostorderIterator block(graph.poBegin()); block != graph.poEnd();
+ block++) {
+ if (mir->shouldCancel("Specialize Phis (main loop)")) {
+ return false;
+ }
+
+ for (MPhiIterator phi(block->phisBegin()); phi != block->phisEnd(); phi++) {
+ if (mir->shouldCancel("Specialize Phis (inner loop)")) {
+ return false;
+ }
+
+ MIRType type = guessPhiType(*phi);
+ phi->specialize(type);
+ if (type == MIRType::None) {
+ // We tried to guess the type but failed because all operands are
+ // phis we still have to visit. Set the triedToSpecialize flag but
+ // don't propagate the type to other phis, propagateSpecialization
+ // will do that once we know the type of one of the operands.
+ continue;
+ }
+ if (!propagateSpecialization(*phi)) {
+ return false;
+ }
+ }
+ }
+
+ if (!propagateAllPhiSpecializations()) {
+ return false;
+ }
+
+ if (shouldSpecializeOsrPhis()) {
+ // See shouldSpecializeOsrPhis comment. This is the second step, propagating
+ // loop header phi types to preheader phis.
+ MBasicBlock* preHeader = graph.osrPreHeaderBlock();
+ MBasicBlock* header = preHeader->getSingleSuccessor();
+
+ if (preHeader->numPredecessors() == 1) {
+ MOZ_ASSERT(preHeader->getPredecessor(0) == graph.osrBlock());
+ // Branch pruning has removed the path from the entry block
+ // to the preheader. Specialize any phis with no non-osr inputs.
+ if (!specializeOsrOnlyPhis()) {
+ return false;
+ }
+ } else if (header->isLoopHeader()) {
+ for (MPhiIterator phi(header->phisBegin()); phi != header->phisEnd();
+ phi++) {
+ MPhi* preHeaderPhi = phi->getOperand(0)->toPhi();
+ MOZ_ASSERT(preHeaderPhi->block() == preHeader);
+
+ if (preHeaderPhi->type() == MIRType::Value) {
+ // Already includes everything.
+ continue;
+ }
+
+ MIRType loopType = phi->type();
+ if (!respecialize(preHeaderPhi, loopType)) {
+ return false;
+ }
+ }
+ if (!propagateAllPhiSpecializations()) {
+ return false;
+ }
+ } else {
+ // Edge case: there is no backedge in this loop. This can happen
+ // if the header is a 'pending' loop header when control flow in
+ // the loop body is terminated unconditionally, or if a block
+ // that dominates the backedge unconditionally bails out. In
+ // this case the header only has the preheader as predecessor
+ // and we don't need to do anything.
+ MOZ_ASSERT(header->numPredecessors() == 1);
+ }
+ }
+
+ MOZ_ASSERT(phiWorklist_.empty());
+ return true;
+}
+
+bool TypeAnalyzer::adjustPhiInputs(MPhi* phi) {
+ MIRType phiType = phi->type();
+ MOZ_ASSERT(phiType != MIRType::None);
+
+ // If we specialized a type that's not Value, there are 3 cases:
+ // 1. Every input is of that type.
+ // 2. Every observed input is of that type (i.e., some inputs haven't been
+ // executed yet).
+ // 3. Inputs were doubles and int32s, and was specialized to double.
+ if (phiType != MIRType::Value) {
+ for (size_t i = 0, e = phi->numOperands(); i < e; i++) {
+ MDefinition* in = phi->getOperand(i);
+ if (in->type() == phiType) {
+ continue;
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ if (in->isBox() && in->toBox()->input()->type() == phiType) {
+ phi->replaceOperand(i, in->toBox()->input());
+ } else {
+ MInstruction* replacement;
+
+ if (phiType == MIRType::Double && IsFloatType(in->type())) {
+ // Convert int32 operands to double.
+ replacement = MToDouble::New(alloc(), in);
+ } else if (phiType == MIRType::Float32) {
+ if (in->type() == MIRType::Int32 || in->type() == MIRType::Double) {
+ replacement = MToFloat32::New(alloc(), in);
+ } else {
+ // See comment below
+ if (in->type() != MIRType::Value) {
+ MBox* box = MBox::New(alloc(), in);
+ in->block()->insertBefore(in->block()->lastIns(), box);
+ in = box;
+ }
+
+ MUnbox* unbox =
+ MUnbox::New(alloc(), in, MIRType::Double, MUnbox::Fallible);
+ unbox->setBailoutKind(BailoutKind::SpeculativePhi);
+ in->block()->insertBefore(in->block()->lastIns(), unbox);
+ replacement = MToFloat32::New(alloc(), in);
+ }
+ } else {
+ // If we know this branch will fail to convert to phiType,
+ // insert a box that'll immediately fail in the fallible unbox
+ // below.
+ if (in->type() != MIRType::Value) {
+ MBox* box = MBox::New(alloc(), in);
+ in->block()->insertBefore(in->block()->lastIns(), box);
+ in = box;
+ }
+
+ // Be optimistic and insert unboxes when the operand is a
+ // value.
+ replacement = MUnbox::New(alloc(), in, phiType, MUnbox::Fallible);
+ }
+
+ replacement->setBailoutKind(BailoutKind::SpeculativePhi);
+ in->block()->insertBefore(in->block()->lastIns(), replacement);
+ phi->replaceOperand(i, replacement);
+ }
+ }
+
+ return true;
+ }
+
+ // Box every typed input.
+ for (size_t i = 0, e = phi->numOperands(); i < e; i++) {
+ MDefinition* in = phi->getOperand(i);
+ if (in->type() == MIRType::Value) {
+ continue;
+ }
+
+ // The input is being explicitly unboxed, so sneak past and grab the
+ // original box. Don't bother optimizing if magic values are involved.
+ if (in->isUnbox()) {
+ MDefinition* unboxInput = in->toUnbox()->input();
+ if (!IsMagicType(unboxInput->type()) && phi->typeIncludes(unboxInput)) {
+ in = in->toUnbox()->input();
+ }
+ }
+
+ if (in->type() != MIRType::Value) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ MBasicBlock* pred = phi->block()->getPredecessor(i);
+ in = AlwaysBoxAt(alloc(), pred->lastIns(), in);
+ }
+
+ phi->replaceOperand(i, in);
+ }
+
+ return true;
+}
+
+bool TypeAnalyzer::adjustInputs(MDefinition* def) {
+ // Definitions such as MPhi have no type policy.
+ if (!def->isInstruction()) {
+ return true;
+ }
+
+ MInstruction* ins = def->toInstruction();
+ const TypePolicy* policy = ins->typePolicy();
+ if (policy && !policy->adjustInputs(alloc(), ins)) {
+ return false;
+ }
+ return true;
+}
+
+void TypeAnalyzer::replaceRedundantPhi(MPhi* phi) {
+ MBasicBlock* block = phi->block();
+ js::Value v;
+ switch (phi->type()) {
+ case MIRType::Undefined:
+ v = UndefinedValue();
+ break;
+ case MIRType::Null:
+ v = NullValue();
+ break;
+ case MIRType::MagicOptimizedOut:
+ v = MagicValue(JS_OPTIMIZED_OUT);
+ break;
+ case MIRType::MagicUninitializedLexical:
+ v = MagicValue(JS_UNINITIALIZED_LEXICAL);
+ break;
+ case MIRType::MagicIsConstructing:
+ v = MagicValue(JS_IS_CONSTRUCTING);
+ break;
+ case MIRType::MagicHole:
+ default:
+ MOZ_CRASH("unexpected type");
+ }
+ MConstant* c = MConstant::New(alloc(), v);
+ // The instruction pass will insert the box
+ block->insertBefore(*(block->begin()), c);
+ phi->justReplaceAllUsesWith(c);
+
+ if (shouldSpecializeOsrPhis()) {
+ // See shouldSpecializeOsrPhis comment. This is part of the third step,
+ // guard the incoming MOsrValue is of this type.
+ for (uint32_t i = 0; i < phi->numOperands(); i++) {
+ MDefinition* def = phi->getOperand(i);
+ if (def->type() != phi->type()) {
+ MOZ_ASSERT(def->isOsrValue() || def->isPhi());
+ MOZ_ASSERT(def->type() == MIRType::Value);
+ MGuardValue* guard = MGuardValue::New(alloc(), def, v);
+ guard->setBailoutKind(BailoutKind::SpeculativePhi);
+ def->block()->insertBefore(def->block()->lastIns(), guard);
+ }
+ }
+ }
+}
+
+bool TypeAnalyzer::insertConversions() {
+ // Instructions are processed in reverse postorder: all uses are defs are
+ // seen before uses. This ensures that output adjustment (which may rewrite
+ // inputs of uses) does not conflict with input adjustment.
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); block++) {
+ if (mir->shouldCancel("Insert Conversions")) {
+ return false;
+ }
+
+ for (MPhiIterator iter(block->phisBegin()), end(block->phisEnd());
+ iter != end;) {
+ MPhi* phi = *iter++;
+ if (IsNullOrUndefined(phi->type()) || IsMagicType(phi->type())) {
+ // We can replace this phi with a constant.
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ replaceRedundantPhi(phi);
+ block->discardPhi(phi);
+ } else {
+ if (!adjustPhiInputs(phi)) {
+ return false;
+ }
+ }
+ }
+
+ // AdjustInputs can add/remove/mutate instructions before and after the
+ // current instruction. Only increment the iterator after it is finished.
+ for (MInstructionIterator iter(block->begin()); iter != block->end();
+ iter++) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ if (!adjustInputs(*iter)) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+/* clang-format off */
+//
+// This function tries to emit Float32 specialized operations whenever it's possible.
+// MIR nodes are flagged as:
+// - Producers, when they can create Float32 that might need to be coerced into a Double.
+// Loads in Float32 arrays and conversions to Float32 are producers.
+// - Consumers, when they can have Float32 as inputs and validate a legal use of a Float32.
+// Stores in Float32 arrays and conversions to Float32 are consumers.
+// - Float32 commutative, when using the Float32 instruction instead of the Double instruction
+// does not result in a compound loss of precision. This is the case for +, -, /, * with 2
+// operands, for instance. However, an addition with 3 operands is not commutative anymore,
+// so an intermediate coercion is needed.
+// Except for phis, all these flags are known after Ion building, so they cannot change during
+// the process.
+//
+// The idea behind the algorithm is easy: whenever we can prove that a commutative operation
+// has only producers as inputs and consumers as uses, we can specialize the operation as a
+// float32 operation. Otherwise, we have to convert all float32 inputs to doubles. Even
+// if a lot of conversions are produced, GVN will take care of eliminating the redundant ones.
+//
+// Phis have a special status. Phis need to be flagged as producers or consumers as they can
+// be inputs or outputs of commutative instructions. Fortunately, producers and consumers
+// properties are such that we can deduce the property using all non phis inputs first (which form
+// an initial phi graph) and then propagate all properties from one phi to another using a
+// fixed point algorithm. The algorithm is ensured to terminate as each iteration has less or as
+// many flagged phis as the previous iteration (so the worst steady state case is all phis being
+// flagged as false).
+//
+// In a nutshell, the algorithm applies three passes:
+// 1 - Determine which phis are consumers. Each phi gets an initial value by making a global AND on
+// all its non-phi inputs. Then each phi propagates its value to other phis. If after propagation,
+// the flag value changed, we have to reapply the algorithm on all phi operands, as a phi is a
+// consumer if all of its uses are consumers.
+// 2 - Determine which phis are producers. It's the same algorithm, except that we have to reapply
+// the algorithm on all phi uses, as a phi is a producer if all of its operands are producers.
+// 3 - Go through all commutative operations and ensure their inputs are all producers and their
+// uses are all consumers.
+//
+/* clang-format on */
+bool TypeAnalyzer::markPhiConsumers() {
+ MOZ_ASSERT(phiWorklist_.empty());
+
+ // Iterate in postorder so worklist is initialized to RPO.
+ for (PostorderIterator block(graph.poBegin()); block != graph.poEnd();
+ ++block) {
+ if (mir->shouldCancel(
+ "Ensure Float32 commutativity - Consumer Phis - Initial state")) {
+ return false;
+ }
+
+ for (MPhiIterator phi(block->phisBegin()); phi != block->phisEnd(); ++phi) {
+ MOZ_ASSERT(!phi->isInWorklist());
+ bool canConsumeFloat32 = !phi->isImplicitlyUsed();
+ for (MUseDefIterator use(*phi); canConsumeFloat32 && use; use++) {
+ MDefinition* usedef = use.def();
+ canConsumeFloat32 &=
+ usedef->isPhi() || usedef->canConsumeFloat32(use.use());
+ }
+ phi->setCanConsumeFloat32(canConsumeFloat32);
+ if (canConsumeFloat32 && !addPhiToWorklist(*phi)) {
+ return false;
+ }
+ }
+ }
+
+ while (!phiWorklist_.empty()) {
+ if (mir->shouldCancel(
+ "Ensure Float32 commutativity - Consumer Phis - Fixed point")) {
+ return false;
+ }
+
+ MPhi* phi = popPhi();
+ MOZ_ASSERT(phi->canConsumeFloat32(nullptr /* unused */));
+
+ bool validConsumer = true;
+ for (MUseDefIterator use(phi); use; use++) {
+ MDefinition* def = use.def();
+ if (def->isPhi() && !def->canConsumeFloat32(use.use())) {
+ validConsumer = false;
+ break;
+ }
+ }
+
+ if (validConsumer) {
+ continue;
+ }
+
+ // Propagate invalidated phis
+ phi->setCanConsumeFloat32(false);
+ for (size_t i = 0, e = phi->numOperands(); i < e; ++i) {
+ MDefinition* input = phi->getOperand(i);
+ if (input->isPhi() && !input->isInWorklist() &&
+ input->canConsumeFloat32(nullptr /* unused */)) {
+ if (!addPhiToWorklist(input->toPhi())) {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+}
+
+bool TypeAnalyzer::markPhiProducers() {
+ MOZ_ASSERT(phiWorklist_.empty());
+
+ // Iterate in reverse postorder so worklist is initialized to PO.
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); ++block) {
+ if (mir->shouldCancel(
+ "Ensure Float32 commutativity - Producer Phis - initial state")) {
+ return false;
+ }
+
+ for (MPhiIterator phi(block->phisBegin()); phi != block->phisEnd(); ++phi) {
+ MOZ_ASSERT(!phi->isInWorklist());
+ bool canProduceFloat32 = true;
+ for (size_t i = 0, e = phi->numOperands(); canProduceFloat32 && i < e;
+ ++i) {
+ MDefinition* input = phi->getOperand(i);
+ canProduceFloat32 &= input->isPhi() || input->canProduceFloat32();
+ }
+ phi->setCanProduceFloat32(canProduceFloat32);
+ if (canProduceFloat32 && !addPhiToWorklist(*phi)) {
+ return false;
+ }
+ }
+ }
+
+ while (!phiWorklist_.empty()) {
+ if (mir->shouldCancel(
+ "Ensure Float32 commutativity - Producer Phis - Fixed point")) {
+ return false;
+ }
+
+ MPhi* phi = popPhi();
+ MOZ_ASSERT(phi->canProduceFloat32());
+
+ bool validProducer = true;
+ for (size_t i = 0, e = phi->numOperands(); i < e; ++i) {
+ MDefinition* input = phi->getOperand(i);
+ if (input->isPhi() && !input->canProduceFloat32()) {
+ validProducer = false;
+ break;
+ }
+ }
+
+ if (validProducer) {
+ continue;
+ }
+
+ // Propagate invalidated phis
+ phi->setCanProduceFloat32(false);
+ for (MUseDefIterator use(phi); use; use++) {
+ MDefinition* def = use.def();
+ if (def->isPhi() && !def->isInWorklist() && def->canProduceFloat32()) {
+ if (!addPhiToWorklist(def->toPhi())) {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+}
+
+bool TypeAnalyzer::specializeValidFloatOps() {
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); ++block) {
+ if (mir->shouldCancel("Ensure Float32 commutativity - Instructions")) {
+ return false;
+ }
+
+ for (MInstructionIterator ins(block->begin()); ins != block->end(); ++ins) {
+ if (!ins->isFloat32Commutative()) {
+ continue;
+ }
+
+ if (ins->type() == MIRType::Float32) {
+ continue;
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ // This call will try to specialize the instruction iff all uses are
+ // consumers and all inputs are producers.
+ ins->trySpecializeFloat32(alloc());
+ }
+ }
+ return true;
+}
+
+bool TypeAnalyzer::graphContainsFloat32() {
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); ++block) {
+ for (MDefinitionIterator def(*block); def; def++) {
+ if (mir->shouldCancel(
+ "Ensure Float32 commutativity - Graph contains Float32")) {
+ return false;
+ }
+
+ if (def->type() == MIRType::Float32) {
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+bool TypeAnalyzer::tryEmitFloatOperations() {
+ // Asm.js uses the ahead of time type checks to specialize operations, no need
+ // to check them again at this point.
+ if (mir->compilingWasm()) {
+ return true;
+ }
+
+ // Check ahead of time that there is at least one definition typed as Float32,
+ // otherwise we don't need this pass.
+ if (!graphContainsFloat32()) {
+ return true;
+ }
+
+ // WarpBuilder skips over code that can't be reached except through
+ // a catch block. Locals and arguments may be observable in such
+ // code after bailing out, so we can't rely on seeing all uses.
+ if (graph.hasTryBlock()) {
+ return true;
+ }
+
+ if (!markPhiConsumers()) {
+ return false;
+ }
+ if (!markPhiProducers()) {
+ return false;
+ }
+ if (!specializeValidFloatOps()) {
+ return false;
+ }
+ return true;
+}
+
+bool TypeAnalyzer::checkFloatCoherency() {
+#ifdef DEBUG
+ // Asserts that all Float32 instructions are flowing into Float32 consumers or
+ // specialized operations
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); ++block) {
+ if (mir->shouldCancel("Check Float32 coherency")) {
+ return false;
+ }
+
+ for (MDefinitionIterator def(*block); def; def++) {
+ if (def->type() != MIRType::Float32) {
+ continue;
+ }
+
+ for (MUseDefIterator use(*def); use; use++) {
+ MDefinition* consumer = use.def();
+ MOZ_ASSERT(consumer->isConsistentFloat32Use(use.use()));
+ }
+ }
+ }
+#endif
+ return true;
+}
+
+bool TypeAnalyzer::analyze() {
+ if (!tryEmitFloatOperations()) {
+ return false;
+ }
+ if (!specializePhis()) {
+ return false;
+ }
+ if (!insertConversions()) {
+ return false;
+ }
+ if (!checkFloatCoherency()) {
+ return false;
+ }
+ return true;
+}
+
+bool jit::ApplyTypeInformation(MIRGenerator* mir, MIRGraph& graph) {
+ TypeAnalyzer analyzer(mir, graph);
+
+ if (!analyzer.analyze()) {
+ return false;
+ }
+
+ return true;
+}
+
+void jit::RenumberBlocks(MIRGraph& graph) {
+ size_t id = 0;
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); block++) {
+ block->setId(id++);
+ }
+}
+
+// A utility for code which deletes blocks. Renumber the remaining blocks,
+// recompute dominators, and optionally recompute AliasAnalysis dependencies.
+bool jit::AccountForCFGChanges(MIRGenerator* mir, MIRGraph& graph,
+ bool updateAliasAnalysis,
+ bool underValueNumberer) {
+ // Renumber the blocks and clear out the old dominator info.
+ size_t id = 0;
+ for (ReversePostorderIterator i(graph.rpoBegin()), e(graph.rpoEnd()); i != e;
+ ++i) {
+ i->clearDominatorInfo();
+ i->setId(id++);
+ }
+
+ // Recompute dominator info.
+ if (!BuildDominatorTree(graph)) {
+ return false;
+ }
+
+ // If needed, update alias analysis dependencies.
+ if (updateAliasAnalysis) {
+ if (!AliasAnalysis(mir, graph).analyze()) {
+ return false;
+ }
+ }
+
+ AssertExtendedGraphCoherency(graph, underValueNumberer);
+ return true;
+}
+
+// Remove all blocks not marked with isMarked(). Unmark all remaining blocks.
+// Alias analysis dependencies may be invalid after calling this function.
+bool jit::RemoveUnmarkedBlocks(MIRGenerator* mir, MIRGraph& graph,
+ uint32_t numMarkedBlocks) {
+ if (numMarkedBlocks == graph.numBlocks()) {
+ // If all blocks are marked, no blocks need removal. Just clear the
+ // marks. We'll still need to update the dominator tree below though,
+ // since we may have removed edges even if we didn't remove any blocks.
+ graph.unmarkBlocks();
+ } else {
+ // As we are going to remove edges and basic blocks, we have to mark
+ // instructions which would be needed by baseline if we were to
+ // bailout.
+ for (PostorderIterator it(graph.poBegin()); it != graph.poEnd();) {
+ MBasicBlock* block = *it++;
+ if (block->isMarked()) {
+ continue;
+ }
+
+ FlagAllOperandsAsImplicitlyUsed(mir, block);
+ }
+
+ // Find unmarked blocks and remove them.
+ for (ReversePostorderIterator iter(graph.rpoBegin());
+ iter != graph.rpoEnd();) {
+ MBasicBlock* block = *iter++;
+
+ if (block->isMarked()) {
+ block->unmark();
+ continue;
+ }
+
+ // The block is unreachable. Clear out the loop header flag, as
+ // we're doing the sweep of a mark-and-sweep here, so we no longer
+ // need to worry about whether an unmarked block is a loop or not.
+ if (block->isLoopHeader()) {
+ block->clearLoopHeader();
+ }
+
+ for (size_t i = 0, e = block->numSuccessors(); i != e; ++i) {
+ block->getSuccessor(i)->removePredecessor(block);
+ }
+ graph.removeBlock(block);
+ }
+ }
+
+ // Renumber the blocks and update the dominator tree.
+ return AccountForCFGChanges(mir, graph, /*updateAliasAnalysis=*/false);
+}
+
+// A Simple, Fast Dominance Algorithm by Cooper et al.
+// Modified to support empty intersections for OSR, and in RPO.
+static MBasicBlock* IntersectDominators(MBasicBlock* block1,
+ MBasicBlock* block2) {
+ MBasicBlock* finger1 = block1;
+ MBasicBlock* finger2 = block2;
+
+ MOZ_ASSERT(finger1);
+ MOZ_ASSERT(finger2);
+
+ // In the original paper, the block ID comparisons are on the postorder index.
+ // This implementation iterates in RPO, so the comparisons are reversed.
+
+ // For this function to be called, the block must have multiple predecessors.
+ // If a finger is then found to be self-dominating, it must therefore be
+ // reachable from multiple roots through non-intersecting control flow.
+ // nullptr is returned in this case, to denote an empty intersection.
+
+ while (finger1->id() != finger2->id()) {
+ while (finger1->id() > finger2->id()) {
+ MBasicBlock* idom = finger1->immediateDominator();
+ if (idom == finger1) {
+ return nullptr; // Empty intersection.
+ }
+ finger1 = idom;
+ }
+
+ while (finger2->id() > finger1->id()) {
+ MBasicBlock* idom = finger2->immediateDominator();
+ if (idom == finger2) {
+ return nullptr; // Empty intersection.
+ }
+ finger2 = idom;
+ }
+ }
+ return finger1;
+}
+
+void jit::ClearDominatorTree(MIRGraph& graph) {
+ for (MBasicBlockIterator iter = graph.begin(); iter != graph.end(); iter++) {
+ iter->clearDominatorInfo();
+ }
+}
+
+static void ComputeImmediateDominators(MIRGraph& graph) {
+ // The default start block is a root and therefore only self-dominates.
+ MBasicBlock* startBlock = graph.entryBlock();
+ startBlock->setImmediateDominator(startBlock);
+
+ // Any OSR block is a root and therefore only self-dominates.
+ MBasicBlock* osrBlock = graph.osrBlock();
+ if (osrBlock) {
+ osrBlock->setImmediateDominator(osrBlock);
+ }
+
+ bool changed = true;
+
+ while (changed) {
+ changed = false;
+
+ ReversePostorderIterator block = graph.rpoBegin();
+
+ // For each block in RPO, intersect all dominators.
+ for (; block != graph.rpoEnd(); block++) {
+ // If a node has once been found to have no exclusive dominator,
+ // it will never have an exclusive dominator, so it may be skipped.
+ if (block->immediateDominator() == *block) {
+ continue;
+ }
+
+ // A block with no predecessors is not reachable from any entry, so
+ // it self-dominates.
+ if (MOZ_UNLIKELY(block->numPredecessors() == 0)) {
+ block->setImmediateDominator(*block);
+ continue;
+ }
+
+ MBasicBlock* newIdom = block->getPredecessor(0);
+
+ // Find the first common dominator.
+ for (size_t i = 1; i < block->numPredecessors(); i++) {
+ MBasicBlock* pred = block->getPredecessor(i);
+ if (pred->immediateDominator() == nullptr) {
+ continue;
+ }
+
+ newIdom = IntersectDominators(pred, newIdom);
+
+ // If there is no common dominator, the block self-dominates.
+ if (newIdom == nullptr) {
+ block->setImmediateDominator(*block);
+ changed = true;
+ break;
+ }
+ }
+
+ if (newIdom && block->immediateDominator() != newIdom) {
+ block->setImmediateDominator(newIdom);
+ changed = true;
+ }
+ }
+ }
+
+#ifdef DEBUG
+ // Assert that all blocks have dominator information.
+ for (MBasicBlockIterator block(graph.begin()); block != graph.end();
+ block++) {
+ MOZ_ASSERT(block->immediateDominator() != nullptr);
+ }
+#endif
+}
+
+bool jit::BuildDominatorTree(MIRGraph& graph) {
+ MOZ_ASSERT(graph.canBuildDominators());
+
+ ComputeImmediateDominators(graph);
+
+ Vector<MBasicBlock*, 4, JitAllocPolicy> worklist(graph.alloc());
+
+ // Traversing through the graph in post-order means that every non-phi use
+ // of a definition is visited before the def itself. Since a def
+ // dominates its uses, by the time we reach a particular
+ // block, we have processed all of its dominated children, so
+ // block->numDominated() is accurate.
+ for (PostorderIterator i(graph.poBegin()); i != graph.poEnd(); i++) {
+ MBasicBlock* child = *i;
+ MBasicBlock* parent = child->immediateDominator();
+
+ // Dominance is defined such that blocks always dominate themselves.
+ child->addNumDominated(1);
+
+ // If the block only self-dominates, it has no definite parent.
+ // Add it to the worklist as a root for pre-order traversal.
+ // This includes all roots. Order does not matter.
+ if (child == parent) {
+ if (!worklist.append(child)) {
+ return false;
+ }
+ continue;
+ }
+
+ if (!parent->addImmediatelyDominatedBlock(child)) {
+ return false;
+ }
+
+ parent->addNumDominated(child->numDominated());
+ }
+
+#ifdef DEBUG
+ // If compiling with OSR, many blocks will self-dominate.
+ // Without OSR, there is only one root block which dominates all.
+ if (!graph.osrBlock()) {
+ MOZ_ASSERT(graph.entryBlock()->numDominated() == graph.numBlocks());
+ }
+#endif
+ // Now, iterate through the dominator tree in pre-order and annotate every
+ // block with its index in the traversal.
+ size_t index = 0;
+ while (!worklist.empty()) {
+ MBasicBlock* block = worklist.popCopy();
+ block->setDomIndex(index);
+
+ if (!worklist.append(block->immediatelyDominatedBlocksBegin(),
+ block->immediatelyDominatedBlocksEnd())) {
+ return false;
+ }
+ index++;
+ }
+
+ return true;
+}
+
+bool jit::BuildPhiReverseMapping(MIRGraph& graph) {
+ // Build a mapping such that given a basic block, whose successor has one or
+ // more phis, we can find our specific input to that phi. To make this fast
+ // mapping work we rely on a specific property of our structured control
+ // flow graph: For a block with phis, its predecessors each have only one
+ // successor with phis. Consider each case:
+ // * Blocks with less than two predecessors cannot have phis.
+ // * Breaks. A break always has exactly one successor, and the break
+ // catch block has exactly one predecessor for each break, as
+ // well as a final predecessor for the actual loop exit.
+ // * Continues. A continue always has exactly one successor, and the
+ // continue catch block has exactly one predecessor for each
+ // continue, as well as a final predecessor for the actual
+ // loop continuation. The continue itself has exactly one
+ // successor.
+ // * An if. Each branch as exactly one predecessor.
+ // * A switch. Each branch has exactly one predecessor.
+ // * Loop tail. A new block is always created for the exit, and if a
+ // break statement is present, the exit block will forward
+ // directly to the break block.
+ for (MBasicBlockIterator block(graph.begin()); block != graph.end();
+ block++) {
+ if (block->phisEmpty()) {
+ continue;
+ }
+
+ // Assert on the above.
+ for (size_t j = 0; j < block->numPredecessors(); j++) {
+ MBasicBlock* pred = block->getPredecessor(j);
+
+#ifdef DEBUG
+ size_t numSuccessorsWithPhis = 0;
+ for (size_t k = 0; k < pred->numSuccessors(); k++) {
+ MBasicBlock* successor = pred->getSuccessor(k);
+ if (!successor->phisEmpty()) {
+ numSuccessorsWithPhis++;
+ }
+ }
+ MOZ_ASSERT(numSuccessorsWithPhis <= 1);
+#endif
+
+ pred->setSuccessorWithPhis(*block, j);
+ }
+ }
+
+ return true;
+}
+
+#ifdef DEBUG
+static bool CheckSuccessorImpliesPredecessor(MBasicBlock* A, MBasicBlock* B) {
+ // Assuming B = succ(A), verify A = pred(B).
+ for (size_t i = 0; i < B->numPredecessors(); i++) {
+ if (A == B->getPredecessor(i)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+static bool CheckPredecessorImpliesSuccessor(MBasicBlock* A, MBasicBlock* B) {
+ // Assuming B = pred(A), verify A = succ(B).
+ for (size_t i = 0; i < B->numSuccessors(); i++) {
+ if (A == B->getSuccessor(i)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// If you have issues with the usesBalance assertions, then define the macro
+// _DEBUG_CHECK_OPERANDS_USES_BALANCE to spew information on the error output.
+// This output can then be processed with the following awk script to filter and
+// highlight which checks are missing or if there is an unexpected operand /
+// use.
+//
+// define _DEBUG_CHECK_OPERANDS_USES_BALANCE 1
+/*
+
+$ ./js 2>stderr.log
+$ gawk '
+ /^==Check/ { context = ""; state = $2; }
+ /^[a-z]/ { context = context "\n\t" $0; }
+ /^==End/ {
+ if (state == "Operand") {
+ list[context] = list[context] - 1;
+ } else if (state == "Use") {
+ list[context] = list[context] + 1;
+ }
+ }
+ END {
+ for (ctx in list) {
+ if (list[ctx] > 0) {
+ print "Missing operand check", ctx, "\n"
+ }
+ if (list[ctx] < 0) {
+ print "Missing use check", ctx, "\n"
+ }
+ };
+ }' < stderr.log
+
+*/
+
+static void CheckOperand(const MNode* consumer, const MUse* use,
+ int32_t* usesBalance) {
+ MOZ_ASSERT(use->hasProducer());
+ MDefinition* producer = use->producer();
+ MOZ_ASSERT(!producer->isDiscarded());
+ MOZ_ASSERT(producer->block() != nullptr);
+ MOZ_ASSERT(use->consumer() == consumer);
+# ifdef _DEBUG_CHECK_OPERANDS_USES_BALANCE
+ Fprinter print(stderr);
+ print.printf("==Check Operand\n");
+ use->producer()->dump(print);
+ print.printf(" index: %zu\n", use->consumer()->indexOf(use));
+ use->consumer()->dump(print);
+ print.printf("==End\n");
+# endif
+ --*usesBalance;
+}
+
+static void CheckUse(const MDefinition* producer, const MUse* use,
+ int32_t* usesBalance) {
+ MOZ_ASSERT(!use->consumer()->block()->isDead());
+ MOZ_ASSERT_IF(use->consumer()->isDefinition(),
+ !use->consumer()->toDefinition()->isDiscarded());
+ MOZ_ASSERT(use->consumer()->block() != nullptr);
+ MOZ_ASSERT(use->consumer()->getOperand(use->index()) == producer);
+# ifdef _DEBUG_CHECK_OPERANDS_USES_BALANCE
+ Fprinter print(stderr);
+ print.printf("==Check Use\n");
+ use->producer()->dump(print);
+ print.printf(" index: %zu\n", use->consumer()->indexOf(use));
+ use->consumer()->dump(print);
+ print.printf("==End\n");
+# endif
+ ++*usesBalance;
+}
+
+// To properly encode entry resume points, we have to ensure that all the
+// operands of the entry resume point are located before the safeInsertTop
+// location.
+static void AssertOperandsBeforeSafeInsertTop(MResumePoint* resume) {
+ MBasicBlock* block = resume->block();
+ if (block == block->graph().osrBlock()) {
+ return;
+ }
+ MInstruction* stop = block->safeInsertTop();
+ for (size_t i = 0, e = resume->numOperands(); i < e; ++i) {
+ MDefinition* def = resume->getOperand(i);
+ if (def->block() != block) {
+ continue;
+ }
+ if (def->isPhi()) {
+ continue;
+ }
+
+ for (MInstructionIterator ins = block->begin(); true; ins++) {
+ if (*ins == def) {
+ break;
+ }
+ MOZ_ASSERT(
+ *ins != stop,
+ "Resume point operand located after the safeInsertTop location");
+ }
+ }
+}
+#endif // DEBUG
+
+void jit::AssertBasicGraphCoherency(MIRGraph& graph, bool force) {
+#ifdef DEBUG
+ if (!JitOptions.fullDebugChecks && !force) {
+ return;
+ }
+
+ MOZ_ASSERT(graph.entryBlock()->numPredecessors() == 0);
+ MOZ_ASSERT(graph.entryBlock()->phisEmpty());
+ MOZ_ASSERT(!graph.entryBlock()->unreachable());
+
+ if (MBasicBlock* osrBlock = graph.osrBlock()) {
+ MOZ_ASSERT(osrBlock->numPredecessors() == 0);
+ MOZ_ASSERT(osrBlock->phisEmpty());
+ MOZ_ASSERT(osrBlock != graph.entryBlock());
+ MOZ_ASSERT(!osrBlock->unreachable());
+ }
+
+ if (MResumePoint* resumePoint = graph.entryResumePoint()) {
+ MOZ_ASSERT(resumePoint->block() == graph.entryBlock());
+ }
+
+ // Assert successor and predecessor list coherency.
+ uint32_t count = 0;
+ int32_t usesBalance = 0;
+ for (MBasicBlockIterator block(graph.begin()); block != graph.end();
+ block++) {
+ count++;
+
+ MOZ_ASSERT(&block->graph() == &graph);
+ MOZ_ASSERT(!block->isDead());
+ MOZ_ASSERT_IF(block->outerResumePoint() != nullptr,
+ block->entryResumePoint() != nullptr);
+
+ for (size_t i = 0; i < block->numSuccessors(); i++) {
+ MOZ_ASSERT(
+ CheckSuccessorImpliesPredecessor(*block, block->getSuccessor(i)));
+ }
+
+ for (size_t i = 0; i < block->numPredecessors(); i++) {
+ MOZ_ASSERT(
+ CheckPredecessorImpliesSuccessor(*block, block->getPredecessor(i)));
+ }
+
+ if (MResumePoint* resume = block->entryResumePoint()) {
+ MOZ_ASSERT(!resume->instruction());
+ MOZ_ASSERT(resume->block() == *block);
+ AssertOperandsBeforeSafeInsertTop(resume);
+ }
+ if (MResumePoint* resume = block->outerResumePoint()) {
+ MOZ_ASSERT(!resume->instruction());
+ MOZ_ASSERT(resume->block() == *block);
+ }
+ for (MResumePointIterator iter(block->resumePointsBegin());
+ iter != block->resumePointsEnd(); iter++) {
+ // We cannot yet assert that is there is no instruction then this is
+ // the entry resume point because we are still storing resume points
+ // in the InlinePropertyTable.
+ MOZ_ASSERT_IF(iter->instruction(),
+ iter->instruction()->block() == *block);
+ for (uint32_t i = 0, e = iter->numOperands(); i < e; i++) {
+ CheckOperand(*iter, iter->getUseFor(i), &usesBalance);
+ }
+ }
+ for (MPhiIterator phi(block->phisBegin()); phi != block->phisEnd(); phi++) {
+ MOZ_ASSERT(phi->numOperands() == block->numPredecessors());
+ MOZ_ASSERT(!phi->isRecoveredOnBailout());
+ MOZ_ASSERT(phi->type() != MIRType::None);
+ MOZ_ASSERT(phi->dependency() == nullptr);
+ }
+ for (MDefinitionIterator iter(*block); iter; iter++) {
+ MOZ_ASSERT(iter->block() == *block);
+ MOZ_ASSERT_IF(iter->hasUses(), iter->type() != MIRType::None);
+ MOZ_ASSERT(!iter->isDiscarded());
+ MOZ_ASSERT_IF(iter->isStart(),
+ *block == graph.entryBlock() || *block == graph.osrBlock());
+ MOZ_ASSERT_IF(iter->isParameter(),
+ *block == graph.entryBlock() || *block == graph.osrBlock());
+ MOZ_ASSERT_IF(iter->isOsrEntry(), *block == graph.osrBlock());
+ MOZ_ASSERT_IF(iter->isOsrValue(), *block == graph.osrBlock());
+
+ // Assert that use chains are valid for this instruction.
+ for (uint32_t i = 0, end = iter->numOperands(); i < end; i++) {
+ CheckOperand(*iter, iter->getUseFor(i), &usesBalance);
+ }
+ for (MUseIterator use(iter->usesBegin()); use != iter->usesEnd(); use++) {
+ CheckUse(*iter, *use, &usesBalance);
+ }
+
+ if (iter->isInstruction()) {
+ if (MResumePoint* resume = iter->toInstruction()->resumePoint()) {
+ MOZ_ASSERT(resume->instruction() == *iter);
+ MOZ_ASSERT(resume->block() == *block);
+ MOZ_ASSERT(resume->block()->entryResumePoint() != nullptr);
+ }
+ }
+
+ if (iter->isRecoveredOnBailout()) {
+ MOZ_ASSERT(!iter->hasLiveDefUses());
+ }
+ }
+
+ // The control instruction is not visited by the MDefinitionIterator.
+ MControlInstruction* control = block->lastIns();
+ MOZ_ASSERT(control->block() == *block);
+ MOZ_ASSERT(!control->hasUses());
+ MOZ_ASSERT(control->type() == MIRType::None);
+ MOZ_ASSERT(!control->isDiscarded());
+ MOZ_ASSERT(!control->isRecoveredOnBailout());
+ MOZ_ASSERT(control->resumePoint() == nullptr);
+ for (uint32_t i = 0, end = control->numOperands(); i < end; i++) {
+ CheckOperand(control, control->getUseFor(i), &usesBalance);
+ }
+ for (size_t i = 0; i < control->numSuccessors(); i++) {
+ MOZ_ASSERT(control->getSuccessor(i));
+ }
+ }
+
+ // In case issues, see the _DEBUG_CHECK_OPERANDS_USES_BALANCE macro above.
+ MOZ_ASSERT(usesBalance <= 0, "More use checks than operand checks");
+ MOZ_ASSERT(usesBalance >= 0, "More operand checks than use checks");
+ MOZ_ASSERT(graph.numBlocks() == count);
+#endif
+}
+
+#ifdef DEBUG
+static void AssertReversePostorder(MIRGraph& graph) {
+ // Check that every block is visited after all its predecessors (except
+ // backedges).
+ for (ReversePostorderIterator iter(graph.rpoBegin()); iter != graph.rpoEnd();
+ ++iter) {
+ MBasicBlock* block = *iter;
+ MOZ_ASSERT(!block->isMarked());
+
+ for (size_t i = 0; i < block->numPredecessors(); i++) {
+ MBasicBlock* pred = block->getPredecessor(i);
+ if (!pred->isMarked()) {
+ MOZ_ASSERT(pred->isLoopBackedge());
+ MOZ_ASSERT(block->backedge() == pred);
+ }
+ }
+
+ block->mark();
+ }
+
+ graph.unmarkBlocks();
+}
+#endif
+
+#ifdef DEBUG
+static void AssertDominatorTree(MIRGraph& graph) {
+ // Check dominators.
+
+ MOZ_ASSERT(graph.entryBlock()->immediateDominator() == graph.entryBlock());
+ if (MBasicBlock* osrBlock = graph.osrBlock()) {
+ MOZ_ASSERT(osrBlock->immediateDominator() == osrBlock);
+ } else {
+ MOZ_ASSERT(graph.entryBlock()->numDominated() == graph.numBlocks());
+ }
+
+ size_t i = graph.numBlocks();
+ size_t totalNumDominated = 0;
+ for (MBasicBlockIterator block(graph.begin()); block != graph.end();
+ block++) {
+ MOZ_ASSERT(block->dominates(*block));
+
+ MBasicBlock* idom = block->immediateDominator();
+ MOZ_ASSERT(idom->dominates(*block));
+ MOZ_ASSERT(idom == *block || idom->id() < block->id());
+
+ if (idom == *block) {
+ totalNumDominated += block->numDominated();
+ } else {
+ bool foundInParent = false;
+ for (size_t j = 0; j < idom->numImmediatelyDominatedBlocks(); j++) {
+ if (idom->getImmediatelyDominatedBlock(j) == *block) {
+ foundInParent = true;
+ break;
+ }
+ }
+ MOZ_ASSERT(foundInParent);
+ }
+
+ size_t numDominated = 1;
+ for (size_t j = 0; j < block->numImmediatelyDominatedBlocks(); j++) {
+ MBasicBlock* dom = block->getImmediatelyDominatedBlock(j);
+ MOZ_ASSERT(block->dominates(dom));
+ MOZ_ASSERT(dom->id() > block->id());
+ MOZ_ASSERT(dom->immediateDominator() == *block);
+
+ numDominated += dom->numDominated();
+ }
+ MOZ_ASSERT(block->numDominated() == numDominated);
+ MOZ_ASSERT(block->numDominated() <= i);
+ MOZ_ASSERT(block->numSuccessors() != 0 || block->numDominated() == 1);
+ i--;
+ }
+ MOZ_ASSERT(i == 0);
+ MOZ_ASSERT(totalNumDominated == graph.numBlocks());
+}
+#endif
+
+void jit::AssertGraphCoherency(MIRGraph& graph, bool force) {
+#ifdef DEBUG
+ if (!JitOptions.checkGraphConsistency) {
+ return;
+ }
+ if (!JitOptions.fullDebugChecks && !force) {
+ return;
+ }
+ AssertBasicGraphCoherency(graph, force);
+ AssertReversePostorder(graph);
+#endif
+}
+
+#ifdef DEBUG
+static bool IsResumableMIRType(MIRType type) {
+ // see CodeGeneratorShared::encodeAllocation
+ switch (type) {
+ case MIRType::Undefined:
+ case MIRType::Null:
+ case MIRType::Boolean:
+ case MIRType::Int32:
+ case MIRType::Double:
+ case MIRType::Float32:
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ case MIRType::Object:
+ case MIRType::Shape:
+ case MIRType::MagicOptimizedOut:
+ case MIRType::MagicUninitializedLexical:
+ case MIRType::MagicIsConstructing:
+ case MIRType::Value:
+ case MIRType::Simd128:
+ return true;
+
+ case MIRType::MagicHole:
+ case MIRType::None:
+ case MIRType::Slots:
+ case MIRType::Elements:
+ case MIRType::Pointer:
+ case MIRType::Int64:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ case MIRType::IntPtr:
+ return false;
+ }
+ MOZ_CRASH("Unknown MIRType.");
+}
+
+static void AssertResumableOperands(MNode* node) {
+ for (size_t i = 0, e = node->numOperands(); i < e; ++i) {
+ MDefinition* op = node->getOperand(i);
+ if (op->isRecoveredOnBailout()) {
+ continue;
+ }
+ MOZ_ASSERT(IsResumableMIRType(op->type()),
+ "Resume point cannot encode its operands");
+ }
+}
+
+static void AssertIfResumableInstruction(MDefinition* def) {
+ if (!def->isRecoveredOnBailout()) {
+ return;
+ }
+ AssertResumableOperands(def);
+}
+
+static void AssertResumePointDominatedByOperands(MResumePoint* resume) {
+ for (size_t i = 0, e = resume->numOperands(); i < e; ++i) {
+ MDefinition* op = resume->getOperand(i);
+ MOZ_ASSERT(op->block()->dominates(resume->block()),
+ "Resume point is not dominated by its operands");
+ }
+}
+#endif // DEBUG
+
+void jit::AssertExtendedGraphCoherency(MIRGraph& graph, bool underValueNumberer,
+ bool force) {
+ // Checks the basic GraphCoherency but also other conditions that
+ // do not hold immediately (such as the fact that critical edges
+ // are split)
+
+#ifdef DEBUG
+ if (!JitOptions.checkGraphConsistency) {
+ return;
+ }
+ if (!JitOptions.fullDebugChecks && !force) {
+ return;
+ }
+
+ AssertGraphCoherency(graph, force);
+
+ AssertDominatorTree(graph);
+
+ DebugOnly<uint32_t> idx = 0;
+ for (MBasicBlockIterator block(graph.begin()); block != graph.end();
+ block++) {
+ MOZ_ASSERT(block->id() == idx);
+ ++idx;
+
+ // No critical edges:
+ if (block->numSuccessors() > 1) {
+ for (size_t i = 0; i < block->numSuccessors(); i++) {
+ MOZ_ASSERT(block->getSuccessor(i)->numPredecessors() == 1);
+ }
+ }
+
+ if (block->isLoopHeader()) {
+ if (underValueNumberer && block->numPredecessors() == 3) {
+ // Fixup block.
+ MOZ_ASSERT(block->getPredecessor(1)->numPredecessors() == 0);
+ MOZ_ASSERT(graph.osrBlock(),
+ "Fixup blocks should only exists if we have an osr block.");
+ } else {
+ MOZ_ASSERT(block->numPredecessors() == 2);
+ }
+ MBasicBlock* backedge = block->backedge();
+ MOZ_ASSERT(backedge->id() >= block->id());
+ MOZ_ASSERT(backedge->numSuccessors() == 1);
+ MOZ_ASSERT(backedge->getSuccessor(0) == *block);
+ }
+
+ if (!block->phisEmpty()) {
+ for (size_t i = 0; i < block->numPredecessors(); i++) {
+ MBasicBlock* pred = block->getPredecessor(i);
+ MOZ_ASSERT(pred->successorWithPhis() == *block);
+ MOZ_ASSERT(pred->positionInPhiSuccessor() == i);
+ }
+ }
+
+ uint32_t successorWithPhis = 0;
+ for (size_t i = 0; i < block->numSuccessors(); i++) {
+ if (!block->getSuccessor(i)->phisEmpty()) {
+ successorWithPhis++;
+ }
+ }
+
+ MOZ_ASSERT(successorWithPhis <= 1);
+ MOZ_ASSERT((successorWithPhis != 0) ==
+ (block->successorWithPhis() != nullptr));
+
+ // Verify that phi operands dominate the corresponding CFG predecessor
+ // edges.
+ for (MPhiIterator iter(block->phisBegin()), end(block->phisEnd());
+ iter != end; ++iter) {
+ MPhi* phi = *iter;
+ for (size_t i = 0, e = phi->numOperands(); i < e; ++i) {
+ MOZ_ASSERT(
+ phi->getOperand(i)->block()->dominates(block->getPredecessor(i)),
+ "Phi input is not dominated by its operand");
+ }
+ }
+
+ // Verify that instructions are dominated by their operands.
+ for (MInstructionIterator iter(block->begin()), end(block->end());
+ iter != end; ++iter) {
+ MInstruction* ins = *iter;
+ for (size_t i = 0, e = ins->numOperands(); i < e; ++i) {
+ MDefinition* op = ins->getOperand(i);
+ MBasicBlock* opBlock = op->block();
+ MOZ_ASSERT(opBlock->dominates(*block),
+ "Instruction is not dominated by its operands");
+
+ // If the operand is an instruction in the same block, check
+ // that it comes first.
+ if (opBlock == *block && !op->isPhi()) {
+ MInstructionIterator opIter = block->begin(op->toInstruction());
+ do {
+ ++opIter;
+ MOZ_ASSERT(opIter != block->end(),
+ "Operand in same block as instruction does not precede");
+ } while (*opIter != ins);
+ }
+ }
+ AssertIfResumableInstruction(ins);
+ if (MResumePoint* resume = ins->resumePoint()) {
+ AssertResumePointDominatedByOperands(resume);
+ AssertResumableOperands(resume);
+ }
+ }
+
+ // Verify that the block resume points are dominated by their operands.
+ if (MResumePoint* resume = block->entryResumePoint()) {
+ AssertResumePointDominatedByOperands(resume);
+ AssertResumableOperands(resume);
+ }
+ if (MResumePoint* resume = block->outerResumePoint()) {
+ AssertResumePointDominatedByOperands(resume);
+ AssertResumableOperands(resume);
+ }
+ }
+#endif
+}
+
+struct BoundsCheckInfo {
+ MBoundsCheck* check;
+ uint32_t validEnd;
+};
+
+typedef HashMap<uint32_t, BoundsCheckInfo, DefaultHasher<uint32_t>,
+ JitAllocPolicy>
+ BoundsCheckMap;
+
+// Compute a hash for bounds checks which ignores constant offsets in the index.
+static HashNumber BoundsCheckHashIgnoreOffset(MBoundsCheck* check) {
+ SimpleLinearSum indexSum = ExtractLinearSum(check->index());
+ uintptr_t index = indexSum.term ? uintptr_t(indexSum.term) : 0;
+ uintptr_t length = uintptr_t(check->length());
+ return index ^ length;
+}
+
+static MBoundsCheck* FindDominatingBoundsCheck(BoundsCheckMap& checks,
+ MBoundsCheck* check,
+ size_t index) {
+ // Since we are traversing the dominator tree in pre-order, when we
+ // are looking at the |index|-th block, the next numDominated() blocks
+ // we traverse are precisely the set of blocks that are dominated.
+ //
+ // So, this value is visible in all blocks if:
+ // index <= index + ins->block->numDominated()
+ // and becomes invalid after that.
+ HashNumber hash = BoundsCheckHashIgnoreOffset(check);
+ BoundsCheckMap::Ptr p = checks.lookup(hash);
+ if (!p || index >= p->value().validEnd) {
+ // We didn't find a dominating bounds check.
+ BoundsCheckInfo info;
+ info.check = check;
+ info.validEnd = index + check->block()->numDominated();
+
+ if (!checks.put(hash, info)) return nullptr;
+
+ return check;
+ }
+
+ return p->value().check;
+}
+
+static MathSpace ExtractMathSpace(MDefinition* ins) {
+ MOZ_ASSERT(ins->isAdd() || ins->isSub());
+ MBinaryArithInstruction* arith = nullptr;
+ if (ins->isAdd()) {
+ arith = ins->toAdd();
+ } else {
+ arith = ins->toSub();
+ }
+ switch (arith->truncateKind()) {
+ case TruncateKind::NoTruncate:
+ case TruncateKind::TruncateAfterBailouts:
+ // TruncateAfterBailouts is considered as infinite space because the
+ // LinearSum will effectively remove the bailout check.
+ return MathSpace::Infinite;
+ case TruncateKind::IndirectTruncate:
+ case TruncateKind::Truncate:
+ return MathSpace::Modulo;
+ }
+ MOZ_CRASH("Unknown TruncateKind");
+}
+
+static bool MonotoneAdd(int32_t lhs, int32_t rhs) {
+ return (lhs >= 0 && rhs >= 0) || (lhs <= 0 && rhs <= 0);
+}
+
+static bool MonotoneSub(int32_t lhs, int32_t rhs) {
+ return (lhs >= 0 && rhs <= 0) || (lhs <= 0 && rhs >= 0);
+}
+
+// Extract a linear sum from ins, if possible (otherwise giving the
+// sum 'ins + 0').
+SimpleLinearSum jit::ExtractLinearSum(MDefinition* ins, MathSpace space,
+ int32_t recursionDepth) {
+ const int32_t SAFE_RECURSION_LIMIT = 100;
+ if (recursionDepth > SAFE_RECURSION_LIMIT) {
+ return SimpleLinearSum(ins, 0);
+ }
+
+ // Unwrap Int32ToIntPtr. This instruction only changes the representation
+ // (int32_t to intptr_t) without affecting the value.
+ if (ins->isInt32ToIntPtr()) {
+ ins = ins->toInt32ToIntPtr()->input();
+ }
+
+ if (ins->isBeta()) {
+ ins = ins->getOperand(0);
+ }
+
+ MOZ_ASSERT(!ins->isInt32ToIntPtr());
+
+ if (ins->type() != MIRType::Int32) {
+ return SimpleLinearSum(ins, 0);
+ }
+
+ if (ins->isConstant()) {
+ return SimpleLinearSum(nullptr, ins->toConstant()->toInt32());
+ }
+
+ if (!ins->isAdd() && !ins->isSub()) {
+ return SimpleLinearSum(ins, 0);
+ }
+
+ // Only allow math which are in the same space.
+ MathSpace insSpace = ExtractMathSpace(ins);
+ if (space == MathSpace::Unknown) {
+ space = insSpace;
+ } else if (space != insSpace) {
+ return SimpleLinearSum(ins, 0);
+ }
+ MOZ_ASSERT(space == MathSpace::Modulo || space == MathSpace::Infinite);
+
+ MDefinition* lhs = ins->getOperand(0);
+ MDefinition* rhs = ins->getOperand(1);
+ if (lhs->type() != MIRType::Int32 || rhs->type() != MIRType::Int32) {
+ return SimpleLinearSum(ins, 0);
+ }
+
+ // Extract linear sums of each operand.
+ SimpleLinearSum lsum = ExtractLinearSum(lhs, space, recursionDepth + 1);
+ SimpleLinearSum rsum = ExtractLinearSum(rhs, space, recursionDepth + 1);
+
+ // LinearSum only considers a single term operand, if both sides have
+ // terms, then ignore extracted linear sums.
+ if (lsum.term && rsum.term) {
+ return SimpleLinearSum(ins, 0);
+ }
+
+ // Check if this is of the form <SUM> + n or n + <SUM>.
+ if (ins->isAdd()) {
+ int32_t constant;
+ if (space == MathSpace::Modulo) {
+ constant = uint32_t(lsum.constant) + uint32_t(rsum.constant);
+ } else if (!SafeAdd(lsum.constant, rsum.constant, &constant) ||
+ !MonotoneAdd(lsum.constant, rsum.constant)) {
+ return SimpleLinearSum(ins, 0);
+ }
+ return SimpleLinearSum(lsum.term ? lsum.term : rsum.term, constant);
+ }
+
+ MOZ_ASSERT(ins->isSub());
+ // Check if this is of the form <SUM> - n.
+ if (lsum.term) {
+ int32_t constant;
+ if (space == MathSpace::Modulo) {
+ constant = uint32_t(lsum.constant) - uint32_t(rsum.constant);
+ } else if (!SafeSub(lsum.constant, rsum.constant, &constant) ||
+ !MonotoneSub(lsum.constant, rsum.constant)) {
+ return SimpleLinearSum(ins, 0);
+ }
+ return SimpleLinearSum(lsum.term, constant);
+ }
+
+ // Ignore any of the form n - <SUM>.
+ return SimpleLinearSum(ins, 0);
+}
+
+// Extract a linear inequality holding when a boolean test goes in the
+// specified direction, of the form 'lhs + lhsN <= rhs' (or >=).
+bool jit::ExtractLinearInequality(MTest* test, BranchDirection direction,
+ SimpleLinearSum* plhs, MDefinition** prhs,
+ bool* plessEqual) {
+ if (!test->getOperand(0)->isCompare()) {
+ return false;
+ }
+
+ MCompare* compare = test->getOperand(0)->toCompare();
+
+ MDefinition* lhs = compare->getOperand(0);
+ MDefinition* rhs = compare->getOperand(1);
+
+ // TODO: optimize Compare_UInt32
+ if (!compare->isInt32Comparison()) {
+ return false;
+ }
+
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+
+ JSOp jsop = compare->jsop();
+ if (direction == FALSE_BRANCH) {
+ jsop = NegateCompareOp(jsop);
+ }
+
+ SimpleLinearSum lsum = ExtractLinearSum(lhs);
+ SimpleLinearSum rsum = ExtractLinearSum(rhs);
+
+ if (!SafeSub(lsum.constant, rsum.constant, &lsum.constant)) {
+ return false;
+ }
+
+ // Normalize operations to use <= or >=.
+ switch (jsop) {
+ case JSOp::Le:
+ *plessEqual = true;
+ break;
+ case JSOp::Lt:
+ /* x < y ==> x + 1 <= y */
+ if (!SafeAdd(lsum.constant, 1, &lsum.constant)) {
+ return false;
+ }
+ *plessEqual = true;
+ break;
+ case JSOp::Ge:
+ *plessEqual = false;
+ break;
+ case JSOp::Gt:
+ /* x > y ==> x - 1 >= y */
+ if (!SafeSub(lsum.constant, 1, &lsum.constant)) {
+ return false;
+ }
+ *plessEqual = false;
+ break;
+ default:
+ return false;
+ }
+
+ *plhs = lsum;
+ *prhs = rsum.term;
+
+ return true;
+}
+
+static bool TryEliminateBoundsCheck(BoundsCheckMap& checks, size_t blockIndex,
+ MBoundsCheck* dominated, bool* eliminated) {
+ MOZ_ASSERT(!*eliminated);
+
+ // Replace all uses of the bounds check with the actual index.
+ // This is (a) necessary, because we can coalesce two different
+ // bounds checks and would otherwise use the wrong index and
+ // (b) helps register allocation. Note that this is safe since
+ // no other pass after bounds check elimination moves instructions.
+ dominated->replaceAllUsesWith(dominated->index());
+
+ if (!dominated->isMovable()) {
+ return true;
+ }
+
+ if (!dominated->fallible()) {
+ return true;
+ }
+
+ MBoundsCheck* dominating =
+ FindDominatingBoundsCheck(checks, dominated, blockIndex);
+ if (!dominating) {
+ return false;
+ }
+
+ if (dominating == dominated) {
+ // We didn't find a dominating bounds check.
+ return true;
+ }
+
+ // We found two bounds checks with the same hash number, but we still have
+ // to make sure the lengths and index terms are equal.
+ if (dominating->length() != dominated->length()) {
+ return true;
+ }
+
+ SimpleLinearSum sumA = ExtractLinearSum(dominating->index());
+ SimpleLinearSum sumB = ExtractLinearSum(dominated->index());
+
+ // Both terms should be nullptr or the same definition.
+ if (sumA.term != sumB.term) {
+ return true;
+ }
+
+ // This bounds check is redundant.
+ *eliminated = true;
+
+ // Normalize the ranges according to the constant offsets in the two indexes.
+ int32_t minimumA, maximumA, minimumB, maximumB;
+ if (!SafeAdd(sumA.constant, dominating->minimum(), &minimumA) ||
+ !SafeAdd(sumA.constant, dominating->maximum(), &maximumA) ||
+ !SafeAdd(sumB.constant, dominated->minimum(), &minimumB) ||
+ !SafeAdd(sumB.constant, dominated->maximum(), &maximumB)) {
+ return false;
+ }
+
+ // Update the dominating check to cover both ranges, denormalizing the
+ // result per the constant offset in the index.
+ int32_t newMinimum, newMaximum;
+ if (!SafeSub(std::min(minimumA, minimumB), sumA.constant, &newMinimum) ||
+ !SafeSub(std::max(maximumA, maximumB), sumA.constant, &newMaximum)) {
+ return false;
+ }
+
+ dominating->setMinimum(newMinimum);
+ dominating->setMaximum(newMaximum);
+ dominating->setBailoutKind(BailoutKind::HoistBoundsCheck);
+
+ return true;
+}
+
+// Eliminate checks which are redundant given each other or other instructions.
+//
+// A bounds check is considered redundant if it's dominated by another bounds
+// check with the same length and the indexes differ by only a constant amount.
+// In this case we eliminate the redundant bounds check and update the other one
+// to cover the ranges of both checks.
+//
+// Bounds checks are added to a hash map and since the hash function ignores
+// differences in constant offset, this offers a fast way to find redundant
+// checks.
+bool jit::EliminateRedundantChecks(MIRGraph& graph) {
+ BoundsCheckMap checks(graph.alloc());
+
+ // Stack for pre-order CFG traversal.
+ Vector<MBasicBlock*, 1, JitAllocPolicy> worklist(graph.alloc());
+
+ // The index of the current block in the CFG traversal.
+ size_t index = 0;
+
+ // Add all self-dominating blocks to the worklist.
+ // This includes all roots. Order does not matter.
+ for (MBasicBlockIterator i(graph.begin()); i != graph.end(); i++) {
+ MBasicBlock* block = *i;
+ if (block->immediateDominator() == block) {
+ if (!worklist.append(block)) {
+ return false;
+ }
+ }
+ }
+
+ // Starting from each self-dominating block, traverse the CFG in pre-order.
+ while (!worklist.empty()) {
+ MBasicBlock* block = worklist.popCopy();
+
+ // Add all immediate dominators to the front of the worklist.
+ if (!worklist.append(block->immediatelyDominatedBlocksBegin(),
+ block->immediatelyDominatedBlocksEnd())) {
+ return false;
+ }
+
+ for (MDefinitionIterator iter(block); iter;) {
+ MDefinition* def = *iter++;
+
+ if (!def->isBoundsCheck()) {
+ continue;
+ }
+ auto* boundsCheck = def->toBoundsCheck();
+
+ bool eliminated = false;
+ if (!TryEliminateBoundsCheck(checks, index, boundsCheck, &eliminated)) {
+ return false;
+ }
+
+ if (eliminated) {
+ block->discard(boundsCheck);
+ }
+ }
+ index++;
+ }
+
+ MOZ_ASSERT(index == graph.numBlocks());
+
+ return true;
+}
+
+static bool ShapeGuardIsRedundant(MGuardShape* guard,
+ const MDefinition* storeObject,
+ const Shape* storeShape) {
+ const MDefinition* guardObject = guard->object()->skipObjectGuards();
+ if (guardObject != storeObject) {
+ JitSpew(JitSpew_RedundantShapeGuards, "SKIP: different objects (%d vs %d)",
+ guardObject->id(), storeObject->id());
+ return false;
+ }
+
+ const Shape* guardShape = guard->shape();
+ if (guardShape != storeShape) {
+ JitSpew(JitSpew_RedundantShapeGuards, "SKIP: different shapes");
+ return false;
+ }
+
+ return true;
+}
+
+// Eliminate shape guards which are redundant given other instructions.
+//
+// A shape guard is redundant if we can prove that the object being
+// guarded already has the correct shape. The conditions for doing so
+// are as follows:
+//
+// 1. We can see the most recent change to the shape of this object.
+// (This can be an AddAndStoreSlot, an AllocateAndStoreSlot, or the
+// creation of the object itself.
+// 2. That mutation dominates the shape guard.
+// 3. The shape that was assigned at that point matches the shape
+// we expect.
+//
+// If all of these conditions hold, then we can remove the shape guard.
+// In debug, we replace it with an AssertShape to help verify correctness.
+bool jit::EliminateRedundantShapeGuards(MIRGraph& graph) {
+ JitSpew(JitSpew_RedundantShapeGuards, "Begin");
+
+ for (ReversePostorderIterator block = graph.rpoBegin();
+ block != graph.rpoEnd(); block++) {
+ for (MInstructionIterator insIter(block->begin());
+ insIter != block->end();) {
+ MInstruction* ins = *insIter;
+ insIter++;
+
+ // Skip instructions that aren't shape guards.
+ if (!ins->isGuardShape()) {
+ continue;
+ }
+ MGuardShape* guard = ins->toGuardShape();
+ MDefinition* lastStore = guard->dependency();
+
+ JitSpew(JitSpew_RedundantShapeGuards, "Visit shape guard %d",
+ guard->id());
+ JitSpewIndent spewIndent(JitSpew_RedundantShapeGuards);
+
+ if (lastStore->isDiscarded() || lastStore->block()->isDead() ||
+ !lastStore->block()->dominates(guard->block())) {
+ JitSpew(JitSpew_RedundantShapeGuards,
+ "SKIP: ins %d does not dominate block %d", lastStore->id(),
+ guard->block()->id());
+ continue;
+ }
+
+ if (lastStore->isAddAndStoreSlot()) {
+ auto* add = lastStore->toAddAndStoreSlot();
+ auto* addObject = add->object()->skipObjectGuards();
+ if (!ShapeGuardIsRedundant(guard, addObject, add->shape())) {
+ continue;
+ }
+ } else if (lastStore->isAllocateAndStoreSlot()) {
+ auto* allocate = lastStore->toAllocateAndStoreSlot();
+ auto* allocateObject = allocate->object()->skipObjectGuards();
+ if (!ShapeGuardIsRedundant(guard, allocateObject, allocate->shape())) {
+ continue;
+ }
+ } else if (lastStore->isStart()) {
+ // The guard doesn't depend on any other instruction that is modifying
+ // the object operand, so we check the object operand directly.
+ auto* obj = guard->object()->skipObjectGuards();
+
+ const Shape* initialShape = nullptr;
+ if (obj->isNewObject()) {
+ auto* templateObject = obj->toNewObject()->templateObject();
+ if (!templateObject) {
+ JitSpew(JitSpew_RedundantShapeGuards, "SKIP: no template");
+ continue;
+ }
+ initialShape = templateObject->shape();
+ } else if (obj->isNewPlainObject()) {
+ initialShape = obj->toNewPlainObject()->shape();
+ } else {
+ JitSpew(JitSpew_RedundantShapeGuards,
+ "SKIP: not NewObject or NewPlainObject (%d)", obj->id());
+ continue;
+ }
+ if (initialShape != guard->shape()) {
+ JitSpew(JitSpew_RedundantShapeGuards, "SKIP: shapes don't match");
+ continue;
+ }
+ } else {
+ JitSpew(JitSpew_RedundantShapeGuards,
+ "SKIP: Last store not supported (%d)", lastStore->id());
+ continue;
+ }
+
+#ifdef DEBUG
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+ auto* assert = MAssertShape::New(graph.alloc(), guard->object(),
+ const_cast<Shape*>(guard->shape()));
+ guard->block()->insertBefore(guard, assert);
+#endif
+
+ JitSpew(JitSpew_RedundantShapeGuards, "SUCCESS: Removing shape guard %d",
+ guard->id());
+ guard->replaceAllUsesWith(guard->input());
+ guard->block()->discard(guard);
+ }
+ }
+
+ return true;
+}
+
+static bool NeedsKeepAlive(MInstruction* slotsOrElements, MInstruction* use) {
+ MOZ_ASSERT(slotsOrElements->type() == MIRType::Elements ||
+ slotsOrElements->type() == MIRType::Slots);
+
+ if (slotsOrElements->block() != use->block()) {
+ return true;
+ }
+
+ if (use->type() == MIRType::BigInt) {
+ return true;
+ }
+
+ MBasicBlock* block = use->block();
+ MInstructionIterator iter(block->begin(slotsOrElements));
+ MOZ_ASSERT(*iter == slotsOrElements);
+ ++iter;
+
+ while (true) {
+ if (*iter == use) {
+ return false;
+ }
+
+ switch (iter->op()) {
+ case MDefinition::Opcode::Nop:
+ case MDefinition::Opcode::Constant:
+ case MDefinition::Opcode::KeepAliveObject:
+ case MDefinition::Opcode::Unbox:
+ case MDefinition::Opcode::LoadDynamicSlot:
+ case MDefinition::Opcode::StoreDynamicSlot:
+ case MDefinition::Opcode::LoadFixedSlot:
+ case MDefinition::Opcode::StoreFixedSlot:
+ case MDefinition::Opcode::LoadElement:
+ case MDefinition::Opcode::LoadElementAndUnbox:
+ case MDefinition::Opcode::StoreElement:
+ case MDefinition::Opcode::StoreHoleValueElement:
+ case MDefinition::Opcode::InitializedLength:
+ case MDefinition::Opcode::ArrayLength:
+ case MDefinition::Opcode::BoundsCheck:
+ case MDefinition::Opcode::GuardElementNotHole:
+ case MDefinition::Opcode::SpectreMaskIndex:
+ iter++;
+ break;
+ default:
+ return true;
+ }
+ }
+
+ MOZ_CRASH("Unreachable");
+}
+
+bool jit::AddKeepAliveInstructions(MIRGraph& graph) {
+ for (MBasicBlockIterator i(graph.begin()); i != graph.end(); i++) {
+ MBasicBlock* block = *i;
+
+ for (MInstructionIterator insIter(block->begin()); insIter != block->end();
+ insIter++) {
+ MInstruction* ins = *insIter;
+ if (ins->type() != MIRType::Elements && ins->type() != MIRType::Slots) {
+ continue;
+ }
+
+ MDefinition* ownerObject;
+ switch (ins->op()) {
+ case MDefinition::Opcode::Elements:
+ case MDefinition::Opcode::ArrayBufferViewElements:
+ MOZ_ASSERT(ins->numOperands() == 1);
+ ownerObject = ins->getOperand(0);
+ break;
+ case MDefinition::Opcode::Slots:
+ ownerObject = ins->toSlots()->object();
+ break;
+ default:
+ MOZ_CRASH("Unexpected op");
+ }
+
+ MOZ_ASSERT(ownerObject->type() == MIRType::Object);
+
+ if (ownerObject->isConstant()) {
+ // Constants are kept alive by other pointers, for instance
+ // ImmGCPtr in JIT code.
+ continue;
+ }
+
+ for (MUseDefIterator uses(ins); uses; uses++) {
+ MInstruction* use = uses.def()->toInstruction();
+
+ if (use->isStoreElementHole()) {
+ // StoreElementHole has an explicit object operand. If GVN
+ // is disabled, we can get different unbox instructions with
+ // the same object as input, so we check for that case.
+ MOZ_ASSERT_IF(!use->toStoreElementHole()->object()->isUnbox() &&
+ !ownerObject->isUnbox(),
+ use->toStoreElementHole()->object() == ownerObject);
+ continue;
+ }
+
+ if (use->isInArray()) {
+ // See StoreElementHole case above.
+ MOZ_ASSERT_IF(
+ !use->toInArray()->object()->isUnbox() && !ownerObject->isUnbox(),
+ use->toInArray()->object() == ownerObject);
+ continue;
+ }
+
+ if (!NeedsKeepAlive(ins, use)) {
+ continue;
+ }
+
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+ MKeepAliveObject* keepAlive =
+ MKeepAliveObject::New(graph.alloc(), ownerObject);
+ use->block()->insertAfter(use, keepAlive);
+ }
+ }
+ }
+
+ return true;
+}
+
+bool LinearSum::multiply(int32_t scale) {
+ for (size_t i = 0; i < terms_.length(); i++) {
+ if (!SafeMul(scale, terms_[i].scale, &terms_[i].scale)) {
+ return false;
+ }
+ }
+ return SafeMul(scale, constant_, &constant_);
+}
+
+bool LinearSum::divide(uint32_t scale) {
+ MOZ_ASSERT(scale > 0);
+
+ for (size_t i = 0; i < terms_.length(); i++) {
+ if (terms_[i].scale % scale != 0) {
+ return false;
+ }
+ }
+ if (constant_ % scale != 0) {
+ return false;
+ }
+
+ for (size_t i = 0; i < terms_.length(); i++) {
+ terms_[i].scale /= scale;
+ }
+ constant_ /= scale;
+
+ return true;
+}
+
+bool LinearSum::add(const LinearSum& other, int32_t scale /* = 1 */) {
+ for (size_t i = 0; i < other.terms_.length(); i++) {
+ int32_t newScale = scale;
+ if (!SafeMul(scale, other.terms_[i].scale, &newScale)) {
+ return false;
+ }
+ if (!add(other.terms_[i].term, newScale)) {
+ return false;
+ }
+ }
+ int32_t newConstant = scale;
+ if (!SafeMul(scale, other.constant_, &newConstant)) {
+ return false;
+ }
+ return add(newConstant);
+}
+
+bool LinearSum::add(SimpleLinearSum other, int32_t scale) {
+ if (other.term && !add(other.term, scale)) {
+ return false;
+ }
+
+ int32_t constant;
+ if (!SafeMul(other.constant, scale, &constant)) {
+ return false;
+ }
+
+ return add(constant);
+}
+
+bool LinearSum::add(MDefinition* term, int32_t scale) {
+ MOZ_ASSERT(term);
+
+ if (scale == 0) {
+ return true;
+ }
+
+ if (MConstant* termConst = term->maybeConstantValue()) {
+ int32_t constant = termConst->toInt32();
+ if (!SafeMul(constant, scale, &constant)) {
+ return false;
+ }
+ return add(constant);
+ }
+
+ for (size_t i = 0; i < terms_.length(); i++) {
+ if (term == terms_[i].term) {
+ if (!SafeAdd(scale, terms_[i].scale, &terms_[i].scale)) {
+ return false;
+ }
+ if (terms_[i].scale == 0) {
+ terms_[i] = terms_.back();
+ terms_.popBack();
+ }
+ return true;
+ }
+ }
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!terms_.append(LinearTerm(term, scale))) {
+ oomUnsafe.crash("LinearSum::add");
+ }
+
+ return true;
+}
+
+bool LinearSum::add(int32_t constant) {
+ return SafeAdd(constant, constant_, &constant_);
+}
+
+void LinearSum::dump(GenericPrinter& out) const {
+ for (size_t i = 0; i < terms_.length(); i++) {
+ int32_t scale = terms_[i].scale;
+ int32_t id = terms_[i].term->id();
+ MOZ_ASSERT(scale);
+ if (scale > 0) {
+ if (i) {
+ out.printf("+");
+ }
+ if (scale == 1) {
+ out.printf("#%d", id);
+ } else {
+ out.printf("%d*#%d", scale, id);
+ }
+ } else if (scale == -1) {
+ out.printf("-#%d", id);
+ } else {
+ out.printf("%d*#%d", scale, id);
+ }
+ }
+ if (constant_ > 0) {
+ out.printf("+%d", constant_);
+ } else if (constant_ < 0) {
+ out.printf("%d", constant_);
+ }
+}
+
+void LinearSum::dump() const {
+ Fprinter out(stderr);
+ dump(out);
+ out.finish();
+}
+
+MDefinition* jit::ConvertLinearSum(TempAllocator& alloc, MBasicBlock* block,
+ const LinearSum& sum,
+ BailoutKind bailoutKind) {
+ MDefinition* def = nullptr;
+
+ for (size_t i = 0; i < sum.numTerms(); i++) {
+ LinearTerm term = sum.term(i);
+ MOZ_ASSERT(!term.term->isConstant());
+ if (term.scale == 1) {
+ if (def) {
+ def = MAdd::New(alloc, def, term.term, MIRType::Int32);
+ def->setBailoutKind(bailoutKind);
+ block->insertAtEnd(def->toInstruction());
+ def->computeRange(alloc);
+ } else {
+ def = term.term;
+ }
+ } else if (term.scale == -1) {
+ if (!def) {
+ def = MConstant::New(alloc, Int32Value(0));
+ block->insertAtEnd(def->toInstruction());
+ def->computeRange(alloc);
+ }
+ def = MSub::New(alloc, def, term.term, MIRType::Int32);
+ def->setBailoutKind(bailoutKind);
+ block->insertAtEnd(def->toInstruction());
+ def->computeRange(alloc);
+ } else {
+ MOZ_ASSERT(term.scale != 0);
+ MConstant* factor = MConstant::New(alloc, Int32Value(term.scale));
+ block->insertAtEnd(factor);
+ MMul* mul = MMul::New(alloc, term.term, factor, MIRType::Int32);
+ mul->setBailoutKind(bailoutKind);
+ block->insertAtEnd(mul);
+ mul->computeRange(alloc);
+ if (def) {
+ def = MAdd::New(alloc, def, mul, MIRType::Int32);
+ def->setBailoutKind(bailoutKind);
+ block->insertAtEnd(def->toInstruction());
+ def->computeRange(alloc);
+ } else {
+ def = mul;
+ }
+ }
+ }
+
+ if (!def) {
+ def = MConstant::New(alloc, Int32Value(0));
+ block->insertAtEnd(def->toInstruction());
+ def->computeRange(alloc);
+ }
+
+ return def;
+}
+
+// Mark all the blocks that are in the loop with the given header.
+// Returns the number of blocks marked. Set *canOsr to true if the loop is
+// reachable from both the normal entry and the OSR entry.
+size_t jit::MarkLoopBlocks(MIRGraph& graph, MBasicBlock* header, bool* canOsr) {
+#ifdef DEBUG
+ for (ReversePostorderIterator i = graph.rpoBegin(), e = graph.rpoEnd();
+ i != e; ++i) {
+ MOZ_ASSERT(!i->isMarked(), "Some blocks already marked");
+ }
+#endif
+
+ MBasicBlock* osrBlock = graph.osrBlock();
+ *canOsr = false;
+
+ // The blocks are in RPO; start at the loop backedge, which marks the bottom
+ // of the loop, and walk up until we get to the header. Loops may be
+ // discontiguous, so we trace predecessors to determine which blocks are
+ // actually part of the loop. The backedge is always part of the loop, and
+ // so are its predecessors, transitively, up to the loop header or an OSR
+ // entry.
+ MBasicBlock* backedge = header->backedge();
+ backedge->mark();
+ size_t numMarked = 1;
+ for (PostorderIterator i = graph.poBegin(backedge);; ++i) {
+ MOZ_ASSERT(
+ i != graph.poEnd(),
+ "Reached the end of the graph while searching for the loop header");
+ MBasicBlock* block = *i;
+ // If we've reached the loop header, we're done.
+ if (block == header) {
+ break;
+ }
+ // A block not marked by the time we reach it is not in the loop.
+ if (!block->isMarked()) {
+ continue;
+ }
+
+ // This block is in the loop; trace to its predecessors.
+ for (size_t p = 0, e = block->numPredecessors(); p != e; ++p) {
+ MBasicBlock* pred = block->getPredecessor(p);
+ if (pred->isMarked()) {
+ continue;
+ }
+
+ // Blocks dominated by the OSR entry are not part of the loop
+ // (unless they aren't reachable from the normal entry).
+ if (osrBlock && pred != header && osrBlock->dominates(pred) &&
+ !osrBlock->dominates(header)) {
+ *canOsr = true;
+ continue;
+ }
+
+ MOZ_ASSERT(pred->id() >= header->id() && pred->id() <= backedge->id(),
+ "Loop block not between loop header and loop backedge");
+
+ pred->mark();
+ ++numMarked;
+
+ // A nested loop may not exit back to the enclosing loop at its
+ // bottom. If we just marked its header, then the whole nested loop
+ // is part of the enclosing loop.
+ if (pred->isLoopHeader()) {
+ MBasicBlock* innerBackedge = pred->backedge();
+ if (!innerBackedge->isMarked()) {
+ // Mark its backedge so that we add all of its blocks to the
+ // outer loop as we walk upwards.
+ innerBackedge->mark();
+ ++numMarked;
+
+ // If the nested loop is not contiguous, we may have already
+ // passed its backedge. If this happens, back up.
+ if (innerBackedge->id() > block->id()) {
+ i = graph.poBegin(innerBackedge);
+ --i;
+ }
+ }
+ }
+ }
+ }
+
+ // If there's no path connecting the header to the backedge, then this isn't
+ // actually a loop. This can happen when the code starts with a loop but GVN
+ // folds some branches away.
+ if (!header->isMarked()) {
+ jit::UnmarkLoopBlocks(graph, header);
+ return 0;
+ }
+
+ return numMarked;
+}
+
+// Unmark all the blocks that are in the loop with the given header.
+void jit::UnmarkLoopBlocks(MIRGraph& graph, MBasicBlock* header) {
+ MBasicBlock* backedge = header->backedge();
+ for (ReversePostorderIterator i = graph.rpoBegin(header);; ++i) {
+ MOZ_ASSERT(i != graph.rpoEnd(),
+ "Reached the end of the graph while searching for the backedge");
+ MBasicBlock* block = *i;
+ if (block->isMarked()) {
+ block->unmark();
+ if (block == backedge) {
+ break;
+ }
+ }
+ }
+
+#ifdef DEBUG
+ for (ReversePostorderIterator i = graph.rpoBegin(), e = graph.rpoEnd();
+ i != e; ++i) {
+ MOZ_ASSERT(!i->isMarked(), "Not all blocks got unmarked");
+ }
+#endif
+}
+
+bool jit::FoldLoadsWithUnbox(MIRGenerator* mir, MIRGraph& graph) {
+ // This pass folds MLoadFixedSlot, MLoadDynamicSlot, MLoadElement instructions
+ // followed by MUnbox into a single instruction. For LoadElement this allows
+ // us to fuse the hole check with the type check for the unbox.
+
+ for (MBasicBlockIterator block(graph.begin()); block != graph.end();
+ block++) {
+ if (mir->shouldCancel("FoldLoadsWithUnbox")) {
+ return false;
+ }
+
+ for (MInstructionIterator insIter(block->begin());
+ insIter != block->end();) {
+ MInstruction* ins = *insIter;
+ insIter++;
+
+ // We're only interested in loads producing a Value.
+ if (!ins->isLoadFixedSlot() && !ins->isLoadDynamicSlot() &&
+ !ins->isLoadElement()) {
+ continue;
+ }
+ if (ins->type() != MIRType::Value) {
+ continue;
+ }
+
+ MInstruction* load = ins;
+
+ // Ensure there's a single def-use (ignoring resume points) and it's an
+ // unbox.
+ MDefinition* defUse = load->maybeSingleDefUse();
+ if (!defUse) {
+ continue;
+ }
+ if (!defUse->isUnbox()) {
+ continue;
+ }
+
+ // For now require the load and unbox to be in the same block. This isn't
+ // strictly necessary but it's the common case and could prevent bailouts
+ // when moving the unbox before a loop.
+ MUnbox* unbox = defUse->toUnbox();
+ if (unbox->block() != *block) {
+ continue;
+ }
+
+ MOZ_ASSERT(!IsMagicType(unbox->type()));
+
+ // If this is a LoadElement, we only support folding it with a fallible
+ // unbox so that we can eliminate the hole check.
+ if (load->isLoadElement() && !unbox->fallible()) {
+ continue;
+ }
+
+ // Combine the load and unbox into a single MIR instruction.
+ if (!graph.alloc().ensureBallast()) {
+ return false;
+ }
+
+ MIRType type = unbox->type();
+ MUnbox::Mode mode = unbox->mode();
+
+ MInstruction* replacement;
+ switch (load->op()) {
+ case MDefinition::Opcode::LoadFixedSlot: {
+ auto* loadIns = load->toLoadFixedSlot();
+ replacement = MLoadFixedSlotAndUnbox::New(
+ graph.alloc(), loadIns->object(), loadIns->slot(), mode, type);
+ break;
+ }
+ case MDefinition::Opcode::LoadDynamicSlot: {
+ auto* loadIns = load->toLoadDynamicSlot();
+ replacement = MLoadDynamicSlotAndUnbox::New(
+ graph.alloc(), loadIns->slots(), loadIns->slot(), mode, type);
+ break;
+ }
+ case MDefinition::Opcode::LoadElement: {
+ auto* loadIns = load->toLoadElement();
+ MOZ_ASSERT(unbox->fallible());
+ replacement = MLoadElementAndUnbox::New(
+ graph.alloc(), loadIns->elements(), loadIns->index(), mode, type);
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected instruction");
+ }
+ replacement->setBailoutKind(BailoutKind::UnboxFolding);
+
+ block->insertBefore(load, replacement);
+ unbox->replaceAllUsesWith(replacement);
+ load->replaceAllUsesWith(replacement);
+
+ if (*insIter == unbox) {
+ insIter++;
+ }
+ block->discard(unbox);
+ block->discard(load);
+ }
+ }
+
+ return true;
+}
+
+// Reorder the blocks in the loop starting at the given header to be contiguous.
+static void MakeLoopContiguous(MIRGraph& graph, MBasicBlock* header,
+ size_t numMarked) {
+ MBasicBlock* backedge = header->backedge();
+
+ MOZ_ASSERT(header->isMarked(), "Loop header is not part of loop");
+ MOZ_ASSERT(backedge->isMarked(), "Loop backedge is not part of loop");
+
+ // If there are any blocks between the loop header and the loop backedge
+ // that are not part of the loop, prepare to move them to the end. We keep
+ // them in order, which preserves RPO.
+ ReversePostorderIterator insertIter = graph.rpoBegin(backedge);
+ insertIter++;
+ MBasicBlock* insertPt = *insertIter;
+
+ // Visit all the blocks from the loop header to the loop backedge.
+ size_t headerId = header->id();
+ size_t inLoopId = headerId;
+ size_t notInLoopId = inLoopId + numMarked;
+ ReversePostorderIterator i = graph.rpoBegin(header);
+ for (;;) {
+ MBasicBlock* block = *i++;
+ MOZ_ASSERT(block->id() >= header->id() && block->id() <= backedge->id(),
+ "Loop backedge should be last block in loop");
+
+ if (block->isMarked()) {
+ // This block is in the loop.
+ block->unmark();
+ block->setId(inLoopId++);
+ // If we've reached the loop backedge, we're done!
+ if (block == backedge) {
+ break;
+ }
+ } else {
+ // This block is not in the loop. Move it to the end.
+ graph.moveBlockBefore(insertPt, block);
+ block->setId(notInLoopId++);
+ }
+ }
+ MOZ_ASSERT(header->id() == headerId, "Loop header id changed");
+ MOZ_ASSERT(inLoopId == headerId + numMarked,
+ "Wrong number of blocks kept in loop");
+ MOZ_ASSERT(notInLoopId == (insertIter != graph.rpoEnd() ? insertPt->id()
+ : graph.numBlocks()),
+ "Wrong number of blocks moved out of loop");
+}
+
+// Reorder the blocks in the graph so that loops are contiguous.
+bool jit::MakeLoopsContiguous(MIRGraph& graph) {
+ // Visit all loop headers (in any order).
+ for (MBasicBlockIterator i(graph.begin()); i != graph.end(); i++) {
+ MBasicBlock* header = *i;
+ if (!header->isLoopHeader()) {
+ continue;
+ }
+
+ // Mark all blocks that are actually part of the loop.
+ bool canOsr;
+ size_t numMarked = MarkLoopBlocks(graph, header, &canOsr);
+
+ // If the loop isn't a loop, don't try to optimize it.
+ if (numMarked == 0) {
+ continue;
+ }
+
+ // If there's an OSR block entering the loop in the middle, it's tricky,
+ // so don't try to handle it, for now.
+ if (canOsr) {
+ UnmarkLoopBlocks(graph, header);
+ continue;
+ }
+
+ // Move all blocks between header and backedge that aren't marked to
+ // the end of the loop, making the loop itself contiguous.
+ MakeLoopContiguous(graph, header, numMarked);
+ }
+
+ return true;
+}
+
+void jit::DumpMIRDefinition(GenericPrinter& out, MDefinition* def) {
+#ifdef JS_JITSPEW
+ out.printf("%u = %s.", def->id(), StringFromMIRType(def->type()));
+ if (def->isConstant()) {
+ def->printOpcode(out);
+ } else {
+ MDefinition::PrintOpcodeName(out, def->op());
+ }
+
+ // Get any extra bits of text that the MIR node wants to show us. Both the
+ // vector and the strings added to it belong to this function, so both will
+ // be automatically freed at exit.
+ ExtrasCollector extras;
+ def->getExtras(&extras);
+ for (size_t i = 0; i < extras.count(); i++) {
+ out.printf(" %s", extras.get(i).get());
+ }
+
+ for (size_t i = 0; i < def->numOperands(); i++) {
+ out.printf(" %u", def->getOperand(i)->id());
+ }
+#endif
+}
+
+void jit::DumpMIRExpressions(GenericPrinter& out, MIRGraph& graph,
+ const CompileInfo& info, const char* phase) {
+#ifdef JS_JITSPEW
+ if (!JitSpewEnabled(JitSpew_MIRExpressions)) {
+ return;
+ }
+
+ out.printf("===== %s =====\n", phase);
+
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); block++) {
+ out.printf(" Block%u:\n", block->id());
+ for (MPhiIterator iter(block->phisBegin()), end(block->phisEnd());
+ iter != end; iter++) {
+ out.printf(" ");
+ jit::DumpMIRDefinition(out, *iter);
+ out.printf("\n");
+ }
+ for (MInstructionIterator iter(block->begin()), end(block->end());
+ iter != end; iter++) {
+ out.printf(" ");
+ DumpMIRDefinition(out, *iter);
+ out.printf("\n");
+ }
+ }
+
+ if (info.compilingWasm()) {
+ out.printf("===== end wasm MIR dump =====\n");
+ } else {
+ out.printf("===== %s:%u =====\n", info.filename(), info.lineno());
+ }
+#endif
+}
diff --git a/js/src/jit/IonAnalysis.h b/js/src/jit/IonAnalysis.h
new file mode 100644
index 0000000000..1f84887dcd
--- /dev/null
+++ b/js/src/jit/IonAnalysis.h
@@ -0,0 +1,189 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_IonAnalysis_h
+#define jit_IonAnalysis_h
+
+// This file declares various analysis passes that operate on MIR.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/IonTypes.h"
+#include "jit/JitAllocPolicy.h"
+#include "js/TypeDecls.h"
+#include "js/Utility.h"
+#include "js/Vector.h"
+
+namespace js {
+
+class GenericPrinter;
+class PlainObject;
+
+namespace jit {
+
+class MBasicBlock;
+class MCompare;
+class MDefinition;
+class MIRGenerator;
+class MIRGraph;
+class MTest;
+
+[[nodiscard]] bool PruneUnusedBranches(MIRGenerator* mir, MIRGraph& graph);
+
+[[nodiscard]] bool FoldTests(MIRGraph& graph);
+
+[[nodiscard]] bool FoldEmptyBlocks(MIRGraph& graph);
+
+[[nodiscard]] bool SplitCriticalEdges(MIRGraph& graph);
+
+bool IsUint32Type(const MDefinition* def);
+
+enum Observability { ConservativeObservability, AggressiveObservability };
+
+[[nodiscard]] bool EliminatePhis(MIRGenerator* mir, MIRGraph& graph,
+ Observability observe);
+
+size_t MarkLoopBlocks(MIRGraph& graph, MBasicBlock* header, bool* canOsr);
+
+void UnmarkLoopBlocks(MIRGraph& graph, MBasicBlock* header);
+
+[[nodiscard]] bool MakeLoopsContiguous(MIRGraph& graph);
+
+[[nodiscard]] bool EliminateTriviallyDeadResumePointOperands(MIRGenerator* mir,
+ MIRGraph& graph);
+
+[[nodiscard]] bool EliminateDeadResumePointOperands(MIRGenerator* mir,
+ MIRGraph& graph);
+
+[[nodiscard]] bool EliminateDeadCode(MIRGenerator* mir, MIRGraph& graph);
+
+[[nodiscard]] bool FoldLoadsWithUnbox(MIRGenerator* mir, MIRGraph& graph);
+
+[[nodiscard]] bool ApplyTypeInformation(MIRGenerator* mir, MIRGraph& graph);
+
+void RenumberBlocks(MIRGraph& graph);
+
+[[nodiscard]] bool AccountForCFGChanges(MIRGenerator* mir, MIRGraph& graph,
+ bool updateAliasAnalysis,
+ bool underValueNumberer = false);
+
+[[nodiscard]] bool RemoveUnmarkedBlocks(MIRGenerator* mir, MIRGraph& graph,
+ uint32_t numMarkedBlocks);
+
+void ClearDominatorTree(MIRGraph& graph);
+
+[[nodiscard]] bool BuildDominatorTree(MIRGraph& graph);
+
+[[nodiscard]] bool BuildPhiReverseMapping(MIRGraph& graph);
+
+void AssertBasicGraphCoherency(MIRGraph& graph, bool force = false);
+
+void AssertGraphCoherency(MIRGraph& graph, bool force = false);
+
+void AssertExtendedGraphCoherency(MIRGraph& graph,
+ bool underValueNumberer = false,
+ bool force = false);
+
+[[nodiscard]] bool EliminateRedundantChecks(MIRGraph& graph);
+
+[[nodiscard]] bool EliminateRedundantShapeGuards(MIRGraph& graph);
+
+[[nodiscard]] bool AddKeepAliveInstructions(MIRGraph& graph);
+
+// Simple linear sum of the form 'n' or 'x + n'.
+struct SimpleLinearSum {
+ MDefinition* term;
+ int32_t constant;
+
+ SimpleLinearSum(MDefinition* term, int32_t constant)
+ : term(term), constant(constant) {}
+};
+
+// Math done in a Linear sum can either be in a modulo space, in which case
+// overflow are wrapped around, or they can be computed in the integer-space in
+// which case we have to check that no overflow can happen when summing
+// constants.
+//
+// When the caller ignores which space it is, the definition would be used to
+// deduce it.
+enum class MathSpace { Modulo, Infinite, Unknown };
+
+SimpleLinearSum ExtractLinearSum(MDefinition* ins,
+ MathSpace space = MathSpace::Unknown,
+ int32_t recursionDepth = 0);
+
+[[nodiscard]] bool ExtractLinearInequality(MTest* test,
+ BranchDirection direction,
+ SimpleLinearSum* plhs,
+ MDefinition** prhs,
+ bool* plessEqual);
+
+struct LinearTerm {
+ MDefinition* term;
+ int32_t scale;
+
+ LinearTerm(MDefinition* term, int32_t scale) : term(term), scale(scale) {}
+};
+
+// General linear sum of the form 'x1*n1 + x2*n2 + ... + n'
+class LinearSum {
+ public:
+ explicit LinearSum(TempAllocator& alloc) : terms_(alloc), constant_(0) {}
+
+ LinearSum(const LinearSum& other)
+ : terms_(other.terms_.allocPolicy()), constant_(other.constant_) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!terms_.appendAll(other.terms_)) {
+ oomUnsafe.crash("LinearSum::LinearSum");
+ }
+ }
+
+ // These return false on an integer overflow, and afterwards the sum must
+ // not be used.
+ [[nodiscard]] bool multiply(int32_t scale);
+ [[nodiscard]] bool add(const LinearSum& other, int32_t scale = 1);
+ [[nodiscard]] bool add(SimpleLinearSum other, int32_t scale = 1);
+ [[nodiscard]] bool add(MDefinition* term, int32_t scale);
+ [[nodiscard]] bool add(int32_t constant);
+
+ // Unlike the above function, on failure this leaves the sum unchanged and
+ // it can still be used.
+ [[nodiscard]] bool divide(uint32_t scale);
+
+ int32_t constant() const { return constant_; }
+ size_t numTerms() const { return terms_.length(); }
+ LinearTerm term(size_t i) const { return terms_[i]; }
+ void replaceTerm(size_t i, MDefinition* def) { terms_[i].term = def; }
+
+ void dump(GenericPrinter& out) const;
+ void dump() const;
+
+ private:
+ Vector<LinearTerm, 2, JitAllocPolicy> terms_;
+ int32_t constant_;
+};
+
+// Convert all components of a linear sum (except the constant)
+// and add any new instructions to the end of block.
+MDefinition* ConvertLinearSum(TempAllocator& alloc, MBasicBlock* block,
+ const LinearSum& sum, BailoutKind bailoutKind);
+
+bool DeadIfUnused(const MDefinition* def);
+bool DeadIfUnusedAllowEffectful(const MDefinition* def);
+
+bool IsDiscardable(const MDefinition* def);
+bool IsDiscardableAllowEffectful(const MDefinition* def);
+
+class CompileInfo;
+void DumpMIRExpressions(GenericPrinter& out, MIRGraph& graph,
+ const CompileInfo& info, const char* phase);
+void DumpMIRDefinition(GenericPrinter& out, MDefinition* def);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_IonAnalysis_h */
diff --git a/js/src/jit/IonCacheIRCompiler.cpp b/js/src/jit/IonCacheIRCompiler.cpp
new file mode 100644
index 0000000000..dc7918f886
--- /dev/null
+++ b/js/src/jit/IonCacheIRCompiler.cpp
@@ -0,0 +1,2093 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/IonCacheIRCompiler.h"
+#include "mozilla/Maybe.h"
+
+#include <algorithm>
+
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIRWriter.h"
+#include "jit/IonIC.h"
+#include "jit/JitcodeMap.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitZone.h"
+#include "jit/JSJitFrameIter.h"
+#include "jit/Linker.h"
+#include "jit/SharedICHelpers.h"
+#include "jit/VMFunctions.h"
+#include "proxy/DeadObjectProxy.h"
+#include "proxy/Proxy.h"
+#include "util/Memory.h"
+#include "vm/StaticStrings.h"
+
+#include "jit/JSJitFrameIter-inl.h"
+#include "jit/MacroAssembler-inl.h"
+#include "jit/VMFunctionList-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::Maybe;
+
+namespace JS {
+struct ExpandoAndGeneration;
+}
+
+using JS::ExpandoAndGeneration;
+
+namespace js {
+namespace jit {
+
+// IonCacheIRCompiler compiles CacheIR to IonIC native code.
+IonCacheIRCompiler::IonCacheIRCompiler(JSContext* cx, TempAllocator& alloc,
+ const CacheIRWriter& writer, IonIC* ic,
+ IonScript* ionScript,
+ uint32_t stubDataOffset)
+ : CacheIRCompiler(cx, alloc, writer, stubDataOffset, Mode::Ion,
+ StubFieldPolicy::Constant),
+ writer_(writer),
+ ic_(ic),
+ ionScript_(ionScript),
+ savedLiveRegs_(false) {
+ MOZ_ASSERT(ic_);
+ MOZ_ASSERT(ionScript_);
+}
+
+template <typename T>
+T IonCacheIRCompiler::rawPointerStubField(uint32_t offset) {
+ static_assert(sizeof(T) == sizeof(uintptr_t), "T must have pointer size");
+ return (T)readStubWord(offset, StubField::Type::RawPointer);
+}
+
+template <typename T>
+T IonCacheIRCompiler::rawInt64StubField(uint32_t offset) {
+ static_assert(sizeof(T) == sizeof(int64_t), "T musthave int64 size");
+ return (T)readStubInt64(offset, StubField::Type::RawInt64);
+}
+
+template <typename Fn, Fn fn>
+void IonCacheIRCompiler::callVM(MacroAssembler& masm) {
+ VMFunctionId id = VMFunctionToId<Fn, fn>::id;
+ callVMInternal(masm, id);
+}
+
+void IonCacheIRCompiler::pushStubCodePointer() {
+ stubJitCodeOffset_.emplace(masm.PushWithPatch(ImmPtr((void*)-1)));
+}
+
+// AutoSaveLiveRegisters must be used when we make a call that can GC. The
+// constructor ensures all live registers are stored on the stack (where the GC
+// expects them) and the destructor restores these registers.
+AutoSaveLiveRegisters::AutoSaveLiveRegisters(IonCacheIRCompiler& compiler)
+ : compiler_(compiler) {
+ MOZ_ASSERT(compiler_.liveRegs_.isSome());
+ MOZ_ASSERT(compiler_.ic_);
+ compiler_.allocator.saveIonLiveRegisters(
+ compiler_.masm, compiler_.liveRegs_.ref(),
+ compiler_.ic_->scratchRegisterForEntryJump(), compiler_.ionScript_);
+ compiler_.savedLiveRegs_ = true;
+}
+AutoSaveLiveRegisters::~AutoSaveLiveRegisters() {
+ MOZ_ASSERT(compiler_.stubJitCodeOffset_.isSome(),
+ "Must have pushed JitCode* pointer");
+ compiler_.allocator.restoreIonLiveRegisters(compiler_.masm,
+ compiler_.liveRegs_.ref());
+ MOZ_ASSERT(compiler_.masm.framePushed() == compiler_.ionScript_->frameSize());
+}
+
+} // namespace jit
+} // namespace js
+
+void CacheRegisterAllocator::saveIonLiveRegisters(MacroAssembler& masm,
+ LiveRegisterSet liveRegs,
+ Register scratch,
+ IonScript* ionScript) {
+ // We have to push all registers in liveRegs on the stack. It's possible we
+ // stored other values in our live registers and stored operands on the
+ // stack (where our live registers should go), so this requires some careful
+ // work. Try to keep it simple by taking one small step at a time.
+
+ // Step 1. Discard any dead operands so we can reuse their registers.
+ freeDeadOperandLocations(masm);
+
+ // Step 2. Figure out the size of our live regs. This is consistent with
+ // the fact that we're using storeRegsInMask to generate the save code and
+ // PopRegsInMask to generate the restore code.
+ size_t sizeOfLiveRegsInBytes = masm.PushRegsInMaskSizeInBytes(liveRegs);
+
+ MOZ_ASSERT(sizeOfLiveRegsInBytes > 0);
+
+ // Step 3. Ensure all non-input operands are on the stack.
+ size_t numInputs = writer_.numInputOperands();
+ for (size_t i = numInputs; i < operandLocations_.length(); i++) {
+ OperandLocation& loc = operandLocations_[i];
+ if (loc.isInRegister()) {
+ spillOperandToStack(masm, &loc);
+ }
+ }
+
+ // Step 4. Restore the register state, but don't discard the stack as
+ // non-input operands are stored there.
+ restoreInputState(masm, /* shouldDiscardStack = */ false);
+
+ // We just restored the input state, so no input operands should be stored
+ // on the stack.
+#ifdef DEBUG
+ for (size_t i = 0; i < numInputs; i++) {
+ const OperandLocation& loc = operandLocations_[i];
+ MOZ_ASSERT(!loc.isOnStack());
+ }
+#endif
+
+ // Step 5. At this point our register state is correct. Stack values,
+ // however, may cover the space where we have to store the live registers.
+ // Move them out of the way.
+
+ bool hasOperandOnStack = false;
+ for (size_t i = numInputs; i < operandLocations_.length(); i++) {
+ OperandLocation& loc = operandLocations_[i];
+ if (!loc.isOnStack()) {
+ continue;
+ }
+
+ hasOperandOnStack = true;
+
+ size_t operandSize = loc.stackSizeInBytes();
+ size_t operandStackPushed = loc.stackPushed();
+ MOZ_ASSERT(operandSize > 0);
+ MOZ_ASSERT(stackPushed_ >= operandStackPushed);
+ MOZ_ASSERT(operandStackPushed >= operandSize);
+
+ // If this operand doesn't cover the live register space, there's
+ // nothing to do.
+ if (operandStackPushed - operandSize >= sizeOfLiveRegsInBytes) {
+ MOZ_ASSERT(stackPushed_ > sizeOfLiveRegsInBytes);
+ continue;
+ }
+
+ // Reserve stack space for the live registers if needed.
+ if (sizeOfLiveRegsInBytes > stackPushed_) {
+ size_t extraBytes = sizeOfLiveRegsInBytes - stackPushed_;
+ MOZ_ASSERT((extraBytes % sizeof(uintptr_t)) == 0);
+ masm.subFromStackPtr(Imm32(extraBytes));
+ stackPushed_ += extraBytes;
+ }
+
+ // Push the operand below the live register space.
+ if (loc.kind() == OperandLocation::PayloadStack) {
+ masm.push(
+ Address(masm.getStackPointer(), stackPushed_ - operandStackPushed));
+ stackPushed_ += operandSize;
+ loc.setPayloadStack(stackPushed_, loc.payloadType());
+ continue;
+ }
+ MOZ_ASSERT(loc.kind() == OperandLocation::ValueStack);
+ masm.pushValue(
+ Address(masm.getStackPointer(), stackPushed_ - operandStackPushed));
+ stackPushed_ += operandSize;
+ loc.setValueStack(stackPushed_);
+ }
+
+ // Step 6. If we have any operands on the stack, adjust their stackPushed
+ // values to not include sizeOfLiveRegsInBytes (this simplifies code down
+ // the line). Then push/store the live registers.
+ if (hasOperandOnStack) {
+ MOZ_ASSERT(stackPushed_ > sizeOfLiveRegsInBytes);
+ stackPushed_ -= sizeOfLiveRegsInBytes;
+
+ for (size_t i = numInputs; i < operandLocations_.length(); i++) {
+ OperandLocation& loc = operandLocations_[i];
+ if (loc.isOnStack()) {
+ loc.adjustStackPushed(-int32_t(sizeOfLiveRegsInBytes));
+ }
+ }
+
+ size_t stackBottom = stackPushed_ + sizeOfLiveRegsInBytes;
+ masm.storeRegsInMask(liveRegs, Address(masm.getStackPointer(), stackBottom),
+ scratch);
+ masm.setFramePushed(masm.framePushed() + sizeOfLiveRegsInBytes);
+ } else {
+ // If no operands are on the stack, discard the unused stack space.
+ if (stackPushed_ > 0) {
+ masm.addToStackPtr(Imm32(stackPushed_));
+ stackPushed_ = 0;
+ }
+ masm.PushRegsInMask(liveRegs);
+ }
+ freePayloadSlots_.clear();
+ freeValueSlots_.clear();
+
+ MOZ_ASSERT(masm.framePushed() ==
+ ionScript->frameSize() + sizeOfLiveRegsInBytes);
+
+ // Step 7. All live registers and non-input operands are stored on the stack
+ // now, so at this point all registers except for the input registers are
+ // available.
+ availableRegs_.set() = GeneralRegisterSet::Not(inputRegisterSet());
+ availableRegsAfterSpill_.set() = GeneralRegisterSet();
+
+ // Step 8. We restored our input state, so we have to fix up aliased input
+ // registers again.
+ fixupAliasedInputs(masm);
+}
+
+void CacheRegisterAllocator::restoreIonLiveRegisters(MacroAssembler& masm,
+ LiveRegisterSet liveRegs) {
+ masm.PopRegsInMask(liveRegs);
+
+ availableRegs_.set() = GeneralRegisterSet();
+ availableRegsAfterSpill_.set() = GeneralRegisterSet::All();
+}
+
+static void* GetReturnAddressToIonCode(JSContext* cx) {
+ JSJitFrameIter frame(cx->activation()->asJit());
+ MOZ_ASSERT(frame.type() == FrameType::Exit,
+ "An exit frame is expected as update functions are called with a "
+ "VMFunction.");
+
+ void* returnAddr = frame.returnAddress();
+#ifdef DEBUG
+ ++frame;
+ MOZ_ASSERT(frame.isIonJS());
+#endif
+ return returnAddr;
+}
+
+// The AutoSaveLiveRegisters parameter is used to ensure registers were saved
+void IonCacheIRCompiler::enterStubFrame(MacroAssembler& masm,
+ const AutoSaveLiveRegisters&) {
+ MOZ_ASSERT(!enteredStubFrame_);
+ pushStubCodePointer();
+ masm.PushFrameDescriptor(FrameType::IonJS);
+ masm.Push(ImmPtr(GetReturnAddressToIonCode(cx_)));
+
+ masm.Push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ enteredStubFrame_ = true;
+}
+
+bool IonCacheIRCompiler::init() {
+ if (!allocator.init()) {
+ return false;
+ }
+
+ size_t numInputs = writer_.numInputOperands();
+ MOZ_ASSERT(numInputs == NumInputsForCacheKind(ic_->kind()));
+
+ AllocatableGeneralRegisterSet available;
+
+ switch (ic_->kind()) {
+ case CacheKind::GetProp:
+ case CacheKind::GetElem: {
+ IonGetPropertyIC* ic = ic_->asGetPropertyIC();
+ ValueOperand output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(output);
+
+ MOZ_ASSERT(numInputs == 1 || numInputs == 2);
+
+ allocator.initInputLocation(0, ic->value());
+ if (numInputs > 1) {
+ allocator.initInputLocation(1, ic->id());
+ }
+ break;
+ }
+ case CacheKind::GetPropSuper:
+ case CacheKind::GetElemSuper: {
+ IonGetPropSuperIC* ic = ic_->asGetPropSuperIC();
+ ValueOperand output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(output);
+
+ MOZ_ASSERT(numInputs == 2 || numInputs == 3);
+
+ allocator.initInputLocation(0, ic->object(), JSVAL_TYPE_OBJECT);
+
+ if (ic->kind() == CacheKind::GetPropSuper) {
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(1, ic->receiver());
+ } else {
+ MOZ_ASSERT(numInputs == 3);
+ allocator.initInputLocation(1, ic->id());
+ allocator.initInputLocation(2, ic->receiver());
+ }
+ break;
+ }
+ case CacheKind::SetProp:
+ case CacheKind::SetElem: {
+ IonSetPropertyIC* ic = ic_->asSetPropertyIC();
+
+ available.add(ic->temp());
+
+ liveRegs_.emplace(ic->liveRegs());
+
+ allocator.initInputLocation(0, ic->object(), JSVAL_TYPE_OBJECT);
+
+ if (ic->kind() == CacheKind::SetProp) {
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(1, ic->rhs());
+ } else {
+ MOZ_ASSERT(numInputs == 3);
+ allocator.initInputLocation(1, ic->id());
+ allocator.initInputLocation(2, ic->rhs());
+ }
+ break;
+ }
+ case CacheKind::GetName: {
+ IonGetNameIC* ic = ic_->asGetNameIC();
+ ValueOperand output = ic->output();
+
+ available.add(output);
+ available.add(ic->temp());
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(output);
+
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, ic->environment(), JSVAL_TYPE_OBJECT);
+ break;
+ }
+ case CacheKind::BindName: {
+ IonBindNameIC* ic = ic_->asBindNameIC();
+ Register output = ic->output();
+
+ available.add(output);
+ available.add(ic->temp());
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(
+ TypedOrValueRegister(MIRType::Object, AnyRegister(output)));
+
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, ic->environment(), JSVAL_TYPE_OBJECT);
+ break;
+ }
+ case CacheKind::GetIterator: {
+ IonGetIteratorIC* ic = ic_->asGetIteratorIC();
+ Register output = ic->output();
+
+ available.add(output);
+ available.add(ic->temp1());
+ available.add(ic->temp2());
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(
+ TypedOrValueRegister(MIRType::Object, AnyRegister(output)));
+
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, ic->value());
+ break;
+ }
+ case CacheKind::OptimizeSpreadCall: {
+ auto* ic = ic_->asOptimizeSpreadCallIC();
+ ValueOperand output = ic->output();
+
+ available.add(output);
+ available.add(ic->temp());
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(output);
+
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, ic->value());
+ break;
+ }
+ case CacheKind::In: {
+ IonInIC* ic = ic_->asInIC();
+ Register output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(
+ TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
+
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(0, ic->key());
+ allocator.initInputLocation(
+ 1, TypedOrValueRegister(MIRType::Object, AnyRegister(ic->object())));
+ break;
+ }
+ case CacheKind::HasOwn: {
+ IonHasOwnIC* ic = ic_->asHasOwnIC();
+ Register output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(
+ TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
+
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(0, ic->id());
+ allocator.initInputLocation(1, ic->value());
+ break;
+ }
+ case CacheKind::CheckPrivateField: {
+ IonCheckPrivateFieldIC* ic = ic_->asCheckPrivateFieldIC();
+ Register output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(
+ TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
+
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(0, ic->value());
+ allocator.initInputLocation(1, ic->id());
+ break;
+ }
+ case CacheKind::InstanceOf: {
+ IonInstanceOfIC* ic = ic_->asInstanceOfIC();
+ Register output = ic->output();
+ available.add(output);
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(
+ TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
+
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(0, ic->lhs());
+ allocator.initInputLocation(
+ 1, TypedOrValueRegister(MIRType::Object, AnyRegister(ic->rhs())));
+ break;
+ }
+ case CacheKind::ToPropertyKey: {
+ IonToPropertyKeyIC* ic = ic_->asToPropertyKeyIC();
+ ValueOperand output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(TypedOrValueRegister(output));
+
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, ic->input());
+ break;
+ }
+ case CacheKind::UnaryArith: {
+ IonUnaryArithIC* ic = ic_->asUnaryArithIC();
+ ValueOperand output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(TypedOrValueRegister(output));
+
+ MOZ_ASSERT(numInputs == 1);
+ allocator.initInputLocation(0, ic->input());
+ break;
+ }
+ case CacheKind::BinaryArith: {
+ IonBinaryArithIC* ic = ic_->asBinaryArithIC();
+ ValueOperand output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(TypedOrValueRegister(output));
+
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(0, ic->lhs());
+ allocator.initInputLocation(1, ic->rhs());
+ break;
+ }
+ case CacheKind::Compare: {
+ IonCompareIC* ic = ic_->asCompareIC();
+ Register output = ic->output();
+
+ available.add(output);
+
+ liveRegs_.emplace(ic->liveRegs());
+ outputUnchecked_.emplace(
+ TypedOrValueRegister(MIRType::Boolean, AnyRegister(output)));
+
+ MOZ_ASSERT(numInputs == 2);
+ allocator.initInputLocation(0, ic->lhs());
+ allocator.initInputLocation(1, ic->rhs());
+ break;
+ }
+ case CacheKind::CloseIter: {
+ IonCloseIterIC* ic = ic_->asCloseIterIC();
+
+ available.add(ic->temp());
+
+ liveRegs_.emplace(ic->liveRegs());
+ allocator.initInputLocation(0, ic->iter(), JSVAL_TYPE_OBJECT);
+ break;
+ }
+ case CacheKind::Call:
+ case CacheKind::TypeOf:
+ case CacheKind::ToBool:
+ case CacheKind::GetIntrinsic:
+ case CacheKind::NewArray:
+ case CacheKind::NewObject:
+ MOZ_CRASH("Unsupported IC");
+ }
+
+ liveFloatRegs_ = LiveFloatRegisterSet(liveRegs_->fpus());
+
+ allocator.initAvailableRegs(available);
+ allocator.initAvailableRegsAfterSpill();
+ return true;
+}
+
+JitCode* IonCacheIRCompiler::compile(IonICStub* stub) {
+ AutoCreatedBy acb(masm, "IonCacheIRCompiler::compile");
+
+ masm.setFramePushed(ionScript_->frameSize());
+ if (cx_->runtime()->geckoProfiler().enabled()) {
+ masm.enableProfilingInstrumentation();
+ }
+
+ allocator.fixupAliasedInputs(masm);
+
+ CacheIRReader reader(writer_);
+ do {
+ switch (reader.readOp()) {
+#define DEFINE_OP(op, ...) \
+ case CacheOp::op: \
+ if (!emit##op(reader)) return nullptr; \
+ break;
+ CACHE_IR_OPS(DEFINE_OP)
+#undef DEFINE_OP
+
+ default:
+ MOZ_CRASH("Invalid op");
+ }
+ allocator.nextOp();
+ } while (reader.more());
+
+ masm.assumeUnreachable("Should have returned from IC");
+
+ // Done emitting the main IC code. Now emit the failure paths.
+ for (size_t i = 0; i < failurePaths.length(); i++) {
+ if (!emitFailurePath(i)) {
+ return nullptr;
+ }
+ Register scratch = ic_->scratchRegisterForEntryJump();
+ CodeOffset offset = masm.movWithPatch(ImmWord(-1), scratch);
+ masm.jump(Address(scratch, 0));
+ if (!nextCodeOffsets_.append(offset)) {
+ return nullptr;
+ }
+ }
+
+ Linker linker(masm);
+ Rooted<JitCode*> newStubCode(cx_, linker.newCode(cx_, CodeKind::Ion));
+ if (!newStubCode) {
+ cx_->recoverFromOutOfMemory();
+ return nullptr;
+ }
+
+ for (CodeOffset offset : nextCodeOffsets_) {
+ Assembler::PatchDataWithValueCheck(CodeLocationLabel(newStubCode, offset),
+ ImmPtr(stub->nextCodeRawPtr()),
+ ImmPtr((void*)-1));
+ }
+ if (stubJitCodeOffset_) {
+ Assembler::PatchDataWithValueCheck(
+ CodeLocationLabel(newStubCode, *stubJitCodeOffset_),
+ ImmPtr(newStubCode.get()), ImmPtr((void*)-1));
+ }
+
+ return newStubCode;
+}
+
+#ifdef DEBUG
+void IonCacheIRCompiler::assertFloatRegisterAvailable(FloatRegister reg) {
+ switch (ic_->kind()) {
+ case CacheKind::GetProp:
+ case CacheKind::GetElem:
+ case CacheKind::GetPropSuper:
+ case CacheKind::GetElemSuper:
+ case CacheKind::GetName:
+ case CacheKind::BindName:
+ case CacheKind::GetIterator:
+ case CacheKind::In:
+ case CacheKind::HasOwn:
+ case CacheKind::CheckPrivateField:
+ case CacheKind::InstanceOf:
+ case CacheKind::UnaryArith:
+ case CacheKind::ToPropertyKey:
+ case CacheKind::OptimizeSpreadCall:
+ case CacheKind::CloseIter:
+ MOZ_CRASH("No float registers available");
+ case CacheKind::SetProp:
+ case CacheKind::SetElem:
+ // FloatReg0 is available per LIRGenerator::visitSetPropertyCache.
+ MOZ_ASSERT(reg == FloatReg0);
+ break;
+ case CacheKind::BinaryArith:
+ case CacheKind::Compare:
+ // FloatReg0 and FloatReg1 are available per
+ // LIRGenerator::visitBinaryCache.
+ MOZ_ASSERT(reg == FloatReg0 || reg == FloatReg1);
+ break;
+ case CacheKind::Call:
+ case CacheKind::TypeOf:
+ case CacheKind::ToBool:
+ case CacheKind::GetIntrinsic:
+ case CacheKind::NewArray:
+ case CacheKind::NewObject:
+ MOZ_CRASH("Unsupported IC");
+ }
+}
+#endif
+
+bool IonCacheIRCompiler::emitGuardShape(ObjOperandId objId,
+ uint32_t shapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ Shape* shape = shapeStubField(shapeOffset);
+
+ bool needSpectreMitigations = objectGuardNeedsSpectreMitigations(objId);
+
+ Maybe<AutoScratchRegister> maybeScratch;
+ if (needSpectreMitigations) {
+ maybeScratch.emplace(allocator, masm);
+ }
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ if (needSpectreMitigations) {
+ masm.branchTestObjShape(Assembler::NotEqual, obj, shape, *maybeScratch, obj,
+ failure->label());
+ } else {
+ masm.branchTestObjShapeNoSpectreMitigations(Assembler::NotEqual, obj, shape,
+ failure->label());
+ }
+
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardProto(ObjOperandId objId,
+ uint32_t protoOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ JSObject* proto = objectStubField(protoOffset);
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadObjProto(obj, scratch);
+ masm.branchPtr(Assembler::NotEqual, scratch, ImmGCPtr(proto),
+ failure->label());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardCompartment(ObjOperandId objId,
+ uint32_t globalOffset,
+ uint32_t compartmentOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ JSObject* globalWrapper = objectStubField(globalOffset);
+ JS::Compartment* compartment = compartmentStubField(compartmentOffset);
+ AutoScratchRegister scratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Verify that the global wrapper is still valid, as
+ // it is pre-requisite for doing the compartment check.
+ masm.movePtr(ImmGCPtr(globalWrapper), scratch);
+ Address handlerAddr(scratch, ProxyObject::offsetOfHandler());
+ masm.branchPtr(Assembler::Equal, handlerAddr,
+ ImmPtr(&DeadObjectProxy::singleton), failure->label());
+
+ masm.branchTestObjCompartment(Assembler::NotEqual, obj, compartment, scratch,
+ failure->label());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardAnyClass(ObjOperandId objId,
+ uint32_t claspOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ const JSClass* clasp = classStubField(claspOffset);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ if (objectGuardNeedsSpectreMitigations(objId)) {
+ masm.branchTestObjClass(Assembler::NotEqual, obj, clasp, scratch, obj,
+ failure->label());
+ } else {
+ masm.branchTestObjClassNoSpectreMitigations(Assembler::NotEqual, obj, clasp,
+ scratch, failure->label());
+ }
+
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardHasProxyHandler(ObjOperandId objId,
+ uint32_t handlerOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ const void* handler = proxyHandlerStubField(handlerOffset);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Address handlerAddr(obj, ProxyObject::offsetOfHandler());
+ masm.branchPtr(Assembler::NotEqual, handlerAddr, ImmPtr(handler),
+ failure->label());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardSpecificObject(ObjOperandId objId,
+ uint32_t expectedOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ JSObject* expected = objectStubField(expectedOffset);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchPtr(Assembler::NotEqual, obj, ImmGCPtr(expected),
+ failure->label());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardSpecificFunction(
+ ObjOperandId objId, uint32_t expectedOffset, uint32_t nargsAndFlagsOffset) {
+ return emitGuardSpecificObject(objId, expectedOffset);
+}
+
+bool IonCacheIRCompiler::emitGuardSpecificAtom(StringOperandId strId,
+ uint32_t expectedOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register str = allocator.useRegister(masm, strId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ JSAtom* atom = &stringStubField(expectedOffset)->asAtom();
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch);
+
+ masm.guardSpecificAtom(str, atom, scratch, volatileRegs, failure->label());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardSpecificSymbol(SymbolOperandId symId,
+ uint32_t expectedOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register sym = allocator.useRegister(masm, symId);
+ JS::Symbol* expected = symbolStubField(expectedOffset);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.branchPtr(Assembler::NotEqual, sym, ImmGCPtr(expected),
+ failure->label());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitLoadValueResult(uint32_t valOffset) {
+ MOZ_CRASH("Baseline-specific op");
+}
+
+bool IonCacheIRCompiler::emitLoadFixedSlotResult(ObjOperandId objId,
+ uint32_t offsetOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ int32_t offset = int32StubField(offsetOffset);
+ masm.loadTypedOrValue(Address(obj, offset), output);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitLoadFixedSlotTypedResult(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValueType) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitLoadDynamicSlotResult(ObjOperandId objId,
+ uint32_t offsetOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register obj = allocator.useRegister(masm, objId);
+ int32_t offset = int32StubField(offsetOffset);
+
+ AutoScratchRegisterMaybeOutput scratch(allocator, masm, output);
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch);
+ masm.loadTypedOrValue(Address(scratch, offset), output);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallScriptedGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+ AutoOutputRegister output(*this);
+
+ ValueOperand receiver = allocator.useValueRegister(masm, receiverId);
+
+ JSFunction* target = &objectStubField(getterOffset)->as<JSFunction>();
+ AutoScratchRegister scratch(allocator, masm);
+
+ MOZ_ASSERT(sameRealm == (cx_->realm() == target->realm()));
+
+ allocator.discardStack(masm);
+
+ uint32_t framePushedBefore = masm.framePushed();
+
+ enterStubFrame(masm, save);
+
+ // The JitFrameLayout pushed below will be aligned to JitStackAlignment,
+ // so we just have to make sure the stack is aligned after we push the
+ // |this| + argument Values.
+ uint32_t argSize = (target->nargs() + 1) * sizeof(Value);
+ uint32_t padding =
+ ComputeByteAlignment(masm.framePushed() + argSize, JitStackAlignment);
+ MOZ_ASSERT(padding % sizeof(uintptr_t) == 0);
+ MOZ_ASSERT(padding < JitStackAlignment);
+ masm.reserveStack(padding);
+
+ for (size_t i = 0; i < target->nargs(); i++) {
+ masm.Push(UndefinedValue());
+ }
+ masm.Push(receiver);
+
+ if (!sameRealm) {
+ masm.switchToRealm(target->realm(), scratch);
+ }
+
+ masm.movePtr(ImmGCPtr(target), scratch);
+
+ masm.Push(scratch);
+ masm.PushFrameDescriptorForJitCall(FrameType::IonICCall, /* argc = */ 0);
+
+ // Check stack alignment. Add 2 * sizeof(uintptr_t) for the return address and
+ // frame pointer pushed by the call/callee.
+ MOZ_ASSERT(
+ ((masm.framePushed() + 2 * sizeof(uintptr_t)) % JitStackAlignment) == 0);
+
+ MOZ_ASSERT(target->hasJitEntry());
+ masm.loadJitCodeRaw(scratch, scratch);
+ masm.callJit(scratch);
+
+ if (!sameRealm) {
+ static_assert(!JSReturnOperand.aliases(ReturnReg),
+ "ReturnReg available as scratch after scripted calls");
+ masm.switchToRealm(cx_->realm(), ReturnReg);
+ }
+
+ masm.storeCallResultValue(output);
+
+ // Restore the frame pointer and stack pointer.
+ masm.loadPtr(Address(FramePointer, 0), FramePointer);
+ masm.freeStack(masm.framePushed() - framePushedBefore);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallInlinedGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, uint32_t icScriptOffset,
+ bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ MOZ_CRASH("Trial inlining not supported in Ion");
+}
+
+bool IonCacheIRCompiler::emitCallNativeGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+ AutoOutputRegister output(*this);
+
+ ValueOperand receiver = allocator.useValueRegister(masm, receiverId);
+
+ JSFunction* target = &objectStubField(getterOffset)->as<JSFunction>();
+ MOZ_ASSERT(target->isNativeFun());
+
+ AutoScratchRegisterMaybeOutput argJSContext(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType argUintN(allocator, masm, output);
+ AutoScratchRegister argVp(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ // Native functions have the signature:
+ // bool (*)(JSContext*, unsigned, Value* vp)
+ // Where vp[0] is space for an outparam, vp[1] is |this|, and vp[2] onward
+ // are the function arguments.
+
+ // Construct vp array:
+ // Push receiver value for |this|
+ masm.Push(receiver);
+ // Push callee/outparam.
+ masm.Push(ObjectValue(*target));
+
+ // Preload arguments into registers.
+ masm.loadJSContext(argJSContext);
+ masm.move32(Imm32(0), argUintN);
+ masm.moveStackPtrTo(argVp.get());
+
+ // Push marking data for later use.
+ masm.Push(argUintN);
+ pushStubCodePointer();
+
+ if (!masm.icBuildOOLFakeExitFrame(GetReturnAddressToIonCode(cx_), save)) {
+ return false;
+ }
+ masm.enterFakeExitFrame(argJSContext, scratch, ExitFrameType::IonOOLNative);
+
+ if (!sameRealm) {
+ masm.switchToRealm(target->realm(), scratch);
+ }
+
+ // Construct and execute call.
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(argJSContext);
+ masm.passABIArg(argUintN);
+ masm.passABIArg(argVp);
+ masm.callWithABI(DynamicFunction<JSNative>(target->native()), MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
+
+ if (!sameRealm) {
+ masm.switchToRealm(cx_->realm(), ReturnReg);
+ }
+
+ // Load the outparam vp[0] into output register(s).
+ Address outparam(masm.getStackPointer(),
+ IonOOLNativeExitFrameLayout::offsetOfResult());
+ masm.loadValue(outparam, output.valueReg());
+
+ if (JitOptions.spectreJitToCxxCalls) {
+ masm.speculationBarrier();
+ }
+
+ masm.adjustStack(IonOOLNativeExitFrameLayout::Size(0));
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallDOMGetterResult(ObjOperandId objId,
+ uint32_t jitInfoOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+ AutoOutputRegister output(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+
+ const JSJitInfo* info = rawPointerStubField<const JSJitInfo*>(jitInfoOffset);
+
+ allocator.discardStack(masm);
+ enterStubFrame(masm, save);
+
+ masm.Push(obj);
+ masm.Push(ImmPtr(info));
+
+ using Fn =
+ bool (*)(JSContext*, const JSJitInfo*, HandleObject, MutableHandleValue);
+ callVM<Fn, jit::CallDOMGetter>(masm);
+
+ masm.storeCallResultValue(output);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallDOMSetter(ObjOperandId objId,
+ uint32_t jitInfoOffset,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ const JSJitInfo* info = rawPointerStubField<const JSJitInfo*>(jitInfoOffset);
+
+ allocator.discardStack(masm);
+ enterStubFrame(masm, save);
+
+ masm.Push(val);
+ masm.Push(obj);
+ masm.Push(ImmPtr(info));
+
+ using Fn = bool (*)(JSContext*, const JSJitInfo*, HandleObject, HandleValue);
+ callVM<Fn, jit::CallDOMSetter>(masm);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitProxyGetResult(ObjOperandId objId,
+ uint32_t idOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+ AutoOutputRegister output(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ jsid id = idStubField(idOffset);
+
+ // ProxyGetProperty(JSContext* cx, HandleObject proxy, HandleId id,
+ // MutableHandleValue vp)
+ AutoScratchRegisterMaybeOutput argJSContext(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType argProxy(allocator, masm, output);
+ AutoScratchRegister argId(allocator, masm);
+ AutoScratchRegister argVp(allocator, masm);
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ // Push stubCode for marking.
+ pushStubCodePointer();
+
+ // Push args on stack first so we can take pointers to make handles.
+ masm.Push(UndefinedValue());
+ masm.moveStackPtrTo(argVp.get());
+
+ masm.Push(id, scratch);
+ masm.moveStackPtrTo(argId.get());
+
+ // Push the proxy. Also used as receiver.
+ masm.Push(obj);
+ masm.moveStackPtrTo(argProxy.get());
+
+ masm.loadJSContext(argJSContext);
+
+ if (!masm.icBuildOOLFakeExitFrame(GetReturnAddressToIonCode(cx_), save)) {
+ return false;
+ }
+ masm.enterFakeExitFrame(argJSContext, scratch, ExitFrameType::IonOOLProxy);
+
+ // Make the call.
+ using Fn = bool (*)(JSContext * cx, HandleObject proxy, HandleId id,
+ MutableHandleValue vp);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(argJSContext);
+ masm.passABIArg(argProxy);
+ masm.passABIArg(argId);
+ masm.passABIArg(argVp);
+ masm.callWithABI<Fn, ProxyGetProperty>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
+
+ // Load the outparam vp[0] into output register(s).
+ Address outparam(masm.getStackPointer(),
+ IonOOLProxyExitFrameLayout::offsetOfResult());
+ masm.loadValue(outparam, output.valueReg());
+
+ // Spectre mitigation in case of speculative execution within C++ code.
+ if (JitOptions.spectreJitToCxxCalls) {
+ masm.speculationBarrier();
+ }
+
+ // masm.leaveExitFrame & pop locals
+ masm.adjustStack(IonOOLProxyExitFrameLayout::Size());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitFrameIsConstructingResult() {
+ MOZ_CRASH("Baseline-specific op");
+}
+
+bool IonCacheIRCompiler::emitLoadConstantStringResult(uint32_t strOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ MOZ_CRASH("not used in ion");
+}
+
+bool IonCacheIRCompiler::emitCompareStringResult(JSOp op, StringOperandId lhsId,
+ StringOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+ AutoOutputRegister output(*this);
+
+ Register left = allocator.useRegister(masm, lhsId);
+ Register right = allocator.useRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+
+ Label slow, done;
+ MOZ_ASSERT(!output.hasValue());
+ masm.compareStrings(op, left, right, output.typedReg().gpr(), &slow);
+
+ masm.jump(&done);
+ masm.bind(&slow);
+
+ enterStubFrame(masm, save);
+
+ // Push the operands in reverse order for JSOp::Le and JSOp::Gt:
+ // - |left <= right| is implemented as |right >= left|.
+ // - |left > right| is implemented as |right < left|.
+ if (op == JSOp::Le || op == JSOp::Gt) {
+ masm.Push(left);
+ masm.Push(right);
+ } else {
+ masm.Push(right);
+ masm.Push(left);
+ }
+
+ using Fn = bool (*)(JSContext*, HandleString, HandleString, bool*);
+ if (op == JSOp::Eq || op == JSOp::StrictEq) {
+ callVM<Fn, jit::StringsEqual<EqualityKind::Equal>>(masm);
+ } else if (op == JSOp::Ne || op == JSOp::StrictNe) {
+ callVM<Fn, jit::StringsEqual<EqualityKind::NotEqual>>(masm);
+ } else if (op == JSOp::Lt || op == JSOp::Gt) {
+ callVM<Fn, jit::StringsCompare<ComparisonKind::LessThan>>(masm);
+ } else {
+ MOZ_ASSERT(op == JSOp::Le || op == JSOp::Ge);
+ callVM<Fn, jit::StringsCompare<ComparisonKind::GreaterThanOrEqual>>(masm);
+ }
+
+ masm.storeCallBoolResult(output.typedReg().gpr());
+ masm.bind(&done);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitStoreFixedSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ int32_t offset = int32StubField(offsetOffset);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ Address slot(obj, offset);
+ EmitPreBarrier(masm, slot, MIRType::Value);
+ masm.storeConstantOrRegister(val, slot);
+ emitPostBarrierSlot(obj, val, scratch);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitStoreDynamicSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ int32_t offset = int32StubField(offsetOffset);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+ AutoScratchRegister scratch(allocator, masm);
+
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch);
+ Address slot(scratch, offset);
+ EmitPreBarrier(masm, slot, MIRType::Value);
+ masm.storeConstantOrRegister(val, slot);
+ emitPostBarrierSlot(obj, val, scratch);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitAddAndStoreSlotShared(
+ CacheOp op, ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset, Maybe<uint32_t> numNewSlotsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ int32_t offset = int32StubField(offsetOffset);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+
+ AutoScratchRegister scratch1(allocator, masm);
+
+ Maybe<AutoScratchRegister> scratch2;
+ if (op == CacheOp::AllocateAndStoreDynamicSlot) {
+ scratch2.emplace(allocator, masm);
+ }
+
+ Shape* newShape = shapeStubField(newShapeOffset);
+
+ if (op == CacheOp::AllocateAndStoreDynamicSlot) {
+ // We have to (re)allocate dynamic slots. Do this first, as it's the
+ // only fallible operation here. Note that growSlotsPure is
+ // fallible but does not GC.
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ int32_t numNewSlots = int32StubField(*numNewSlotsOffset);
+ MOZ_ASSERT(numNewSlots > 0);
+
+ LiveRegisterSet save(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ masm.PushRegsInMask(save);
+
+ using Fn = bool (*)(JSContext * cx, NativeObject * obj, uint32_t newCount);
+ masm.setupUnalignedABICall(scratch1);
+ masm.loadJSContext(scratch1);
+ masm.passABIArg(scratch1);
+ masm.passABIArg(obj);
+ masm.move32(Imm32(numNewSlots), scratch2.ref());
+ masm.passABIArg(scratch2.ref());
+ masm.callWithABI<Fn, NativeObject::growSlotsPure>();
+ masm.storeCallPointerResult(scratch1);
+
+ LiveRegisterSet ignore;
+ ignore.add(scratch1);
+ masm.PopRegsInMaskIgnore(save, ignore);
+
+ masm.branchIfFalseBool(scratch1, failure->label());
+ }
+
+ // Update the object's shape.
+ masm.storeObjShape(newShape, obj,
+ [](MacroAssembler& masm, const Address& addr) {
+ EmitPreBarrier(masm, addr, MIRType::Shape);
+ });
+
+ // Perform the store. No pre-barrier required since this is a new
+ // initialization.
+ if (op == CacheOp::AddAndStoreFixedSlot) {
+ Address slot(obj, offset);
+ masm.storeConstantOrRegister(val, slot);
+ } else {
+ MOZ_ASSERT(op == CacheOp::AddAndStoreDynamicSlot ||
+ op == CacheOp::AllocateAndStoreDynamicSlot);
+ masm.loadPtr(Address(obj, NativeObject::offsetOfSlots()), scratch1);
+ Address slot(scratch1, offset);
+ masm.storeConstantOrRegister(val, slot);
+ }
+
+ emitPostBarrierSlot(obj, val, scratch1);
+
+ return true;
+}
+
+bool IonCacheIRCompiler::emitAddAndStoreFixedSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId,
+ uint32_t newShapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<uint32_t> numNewSlotsOffset = mozilla::Nothing();
+ return emitAddAndStoreSlotShared(CacheOp::AddAndStoreFixedSlot, objId,
+ offsetOffset, rhsId, newShapeOffset,
+ numNewSlotsOffset);
+}
+
+bool IonCacheIRCompiler::emitAddAndStoreDynamicSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId,
+ uint32_t newShapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Maybe<uint32_t> numNewSlotsOffset = mozilla::Nothing();
+ return emitAddAndStoreSlotShared(CacheOp::AddAndStoreDynamicSlot, objId,
+ offsetOffset, rhsId, newShapeOffset,
+ numNewSlotsOffset);
+}
+
+bool IonCacheIRCompiler::emitAllocateAndStoreDynamicSlot(
+ ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset, uint32_t numNewSlotsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ return emitAddAndStoreSlotShared(CacheOp::AllocateAndStoreDynamicSlot, objId,
+ offsetOffset, rhsId, newShapeOffset,
+ mozilla::Some(numNewSlotsOffset));
+}
+
+bool IonCacheIRCompiler::emitLoadStringCharResult(StringOperandId strId,
+ Int32OperandId indexId,
+ bool handleOOB) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoOutputRegister output(*this);
+ Register str = allocator.useRegister(masm, strId);
+ Register index = allocator.useRegister(masm, indexId);
+ AutoScratchRegisterMaybeOutput scratch1(allocator, masm, output);
+ AutoScratchRegisterMaybeOutputType scratch2(allocator, masm, output);
+ AutoScratchRegister scratch3(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ // Bounds check, load string char.
+ Label done;
+ Label loadFailed;
+ if (!handleOOB) {
+ masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
+ scratch1, failure->label());
+ masm.loadStringChar(str, index, scratch1, scratch2, scratch3,
+ failure->label());
+ } else {
+ // Return the empty string for out-of-bounds access.
+ masm.movePtr(ImmGCPtr(cx_->runtime()->emptyString), scratch2);
+
+ // This CacheIR op is always preceded by |LinearizeForCharAccess|, so we're
+ // guaranteed to see no nested ropes.
+ masm.spectreBoundsCheck32(index, Address(str, JSString::offsetOfLength()),
+ scratch1, &done);
+ masm.loadStringChar(str, index, scratch1, scratch2, scratch3, &loadFailed);
+ }
+
+ // Load StaticString for this char. For larger code units perform a VM call.
+ Label vmCall;
+ masm.boundsCheck32PowerOfTwo(scratch1, StaticStrings::UNIT_STATIC_LIMIT,
+ &vmCall);
+ masm.movePtr(ImmPtr(&cx_->staticStrings().unitStaticTable), scratch2);
+ masm.loadPtr(BaseIndex(scratch2, scratch1, ScalePointer), scratch2);
+
+ masm.jump(&done);
+
+ if (handleOOB) {
+ masm.bind(&loadFailed);
+ masm.assumeUnreachable("loadStringChar can't fail for linear strings");
+ }
+
+ {
+ masm.bind(&vmCall);
+
+ // FailurePath and AutoSaveLiveRegisters don't get along very well. Both are
+ // modifying the stack and expect that no other stack manipulations are
+ // made. Therefore we need to use an ABI call instead of a VM call here.
+
+ LiveRegisterSet volatileRegs(GeneralRegisterSet::Volatile(),
+ liveVolatileFloatRegs());
+ volatileRegs.takeUnchecked(scratch1);
+ volatileRegs.takeUnchecked(scratch2);
+ volatileRegs.takeUnchecked(scratch3);
+ volatileRegs.takeUnchecked(output);
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = JSLinearString* (*)(JSContext * cx, int32_t code);
+ masm.setupUnalignedABICall(scratch2);
+ masm.loadJSContext(scratch2);
+ masm.passABIArg(scratch2);
+ masm.passABIArg(scratch1);
+ masm.callWithABI<Fn, jit::StringFromCharCodeNoGC>();
+ masm.storeCallPointerResult(scratch2);
+
+ masm.PopRegsInMask(volatileRegs);
+
+ masm.branchPtr(Assembler::Equal, scratch2, ImmWord(0), failure->label());
+ }
+
+ masm.bind(&done);
+ masm.tagValue(JSVAL_TYPE_STRING, scratch2, output.valueReg());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallNativeSetter(ObjOperandId receiverId,
+ uint32_t setterOffset,
+ ValOperandId rhsId,
+ bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register receiver = allocator.useRegister(masm, receiverId);
+ JSFunction* target = &objectStubField(setterOffset)->as<JSFunction>();
+ MOZ_ASSERT(target->isNativeFun());
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+
+ AutoScratchRegister argJSContext(allocator, masm);
+ AutoScratchRegister argVp(allocator, masm);
+ AutoScratchRegister argUintN(allocator, masm);
+#ifndef JS_CODEGEN_X86
+ AutoScratchRegister scratch(allocator, masm);
+#else
+ // Not enough registers on x86.
+ Register scratch = argUintN;
+#endif
+
+ allocator.discardStack(masm);
+
+ // Set up the call:
+ // bool (*)(JSContext*, unsigned, Value* vp)
+ // vp[0] is callee/outparam
+ // vp[1] is |this|
+ // vp[2] is the value
+
+ // Build vp and move the base into argVpReg.
+ masm.Push(val);
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(receiver)));
+ masm.Push(ObjectValue(*target));
+ masm.moveStackPtrTo(argVp.get());
+
+ // Preload other regs.
+ masm.loadJSContext(argJSContext);
+ masm.move32(Imm32(1), argUintN);
+
+ // Push marking data for later use.
+ masm.Push(argUintN);
+ pushStubCodePointer();
+
+ if (!masm.icBuildOOLFakeExitFrame(GetReturnAddressToIonCode(cx_), save)) {
+ return false;
+ }
+ masm.enterFakeExitFrame(argJSContext, scratch, ExitFrameType::IonOOLNative);
+
+ if (!sameRealm) {
+ masm.switchToRealm(target->realm(), scratch);
+ }
+
+ // Make the call.
+ masm.setupUnalignedABICall(scratch);
+#ifdef JS_CODEGEN_X86
+ // Reload argUintN because it was clobbered.
+ masm.move32(Imm32(1), argUintN);
+#endif
+ masm.passABIArg(argJSContext);
+ masm.passABIArg(argUintN);
+ masm.passABIArg(argVp);
+ masm.callWithABI(DynamicFunction<JSNative>(target->native()), MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ masm.branchIfFalseBool(ReturnReg, masm.exceptionLabel());
+
+ if (!sameRealm) {
+ masm.switchToRealm(cx_->realm(), ReturnReg);
+ }
+
+ masm.adjustStack(IonOOLNativeExitFrameLayout::Size(1));
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallScriptedSetter(ObjOperandId receiverId,
+ uint32_t setterOffset,
+ ValOperandId rhsId,
+ bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register receiver = allocator.useRegister(masm, receiverId);
+ JSFunction* target = &objectStubField(setterOffset)->as<JSFunction>();
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+
+ MOZ_ASSERT(sameRealm == (cx_->realm() == target->realm()));
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+
+ uint32_t framePushedBefore = masm.framePushed();
+
+ enterStubFrame(masm, save);
+
+ // The JitFrameLayout pushed below will be aligned to JitStackAlignment,
+ // so we just have to make sure the stack is aligned after we push the
+ // |this| + argument Values.
+ size_t numArgs = std::max<size_t>(1, target->nargs());
+ uint32_t argSize = (numArgs + 1) * sizeof(Value);
+ uint32_t padding =
+ ComputeByteAlignment(masm.framePushed() + argSize, JitStackAlignment);
+ MOZ_ASSERT(padding % sizeof(uintptr_t) == 0);
+ MOZ_ASSERT(padding < JitStackAlignment);
+ masm.reserveStack(padding);
+
+ for (size_t i = 1; i < target->nargs(); i++) {
+ masm.Push(UndefinedValue());
+ }
+ masm.Push(val);
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(receiver)));
+
+ if (!sameRealm) {
+ masm.switchToRealm(target->realm(), scratch);
+ }
+
+ masm.movePtr(ImmGCPtr(target), scratch);
+
+ masm.Push(scratch);
+ masm.PushFrameDescriptorForJitCall(FrameType::IonICCall, /* argc = */ 1);
+
+ // Check stack alignment. Add 2 * sizeof(uintptr_t) for the return address and
+ // frame pointer pushed by the call/callee.
+ MOZ_ASSERT(
+ ((masm.framePushed() + 2 * sizeof(uintptr_t)) % JitStackAlignment) == 0);
+
+ MOZ_ASSERT(target->hasJitEntry());
+ masm.loadJitCodeRaw(scratch, scratch);
+ masm.callJit(scratch);
+
+ if (!sameRealm) {
+ masm.switchToRealm(cx_->realm(), ReturnReg);
+ }
+
+ // Restore the frame pointer and stack pointer.
+ masm.loadPtr(Address(FramePointer, 0), FramePointer);
+ masm.freeStack(masm.framePushed() - framePushedBefore);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallInlinedSetter(
+ ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId,
+ uint32_t icScriptOffset, bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ MOZ_CRASH("Trial inlining not supported in Ion");
+}
+
+bool IonCacheIRCompiler::emitCallSetArrayLength(ObjOperandId objId, bool strict,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+ enterStubFrame(masm, save);
+
+ masm.Push(Imm32(strict));
+ masm.Push(val);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, bool);
+ callVM<Fn, jit::SetArrayLength>(masm);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitProxySet(ObjOperandId objId, uint32_t idOffset,
+ ValOperandId rhsId, bool strict) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+ jsid id = idStubField(idOffset);
+
+ AutoScratchRegister scratch(allocator, masm);
+
+ allocator.discardStack(masm);
+ enterStubFrame(masm, save);
+
+ masm.Push(Imm32(strict));
+ masm.Push(val);
+ masm.Push(id, scratch);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleId, HandleValue, bool);
+ callVM<Fn, ProxySetProperty>(masm);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitProxySetByValue(ObjOperandId objId,
+ ValOperandId idId,
+ ValOperandId rhsId, bool strict) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ConstantOrRegister idVal = allocator.useConstantOrRegister(masm, idId);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+ enterStubFrame(masm, save);
+
+ masm.Push(Imm32(strict));
+ masm.Push(val);
+ masm.Push(idVal);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue, bool);
+ callVM<Fn, ProxySetPropertyByValue>(masm);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallAddOrUpdateSparseElementHelper(
+ ObjOperandId objId, Int32OperandId idId, ValOperandId rhsId, bool strict) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ Register id = allocator.useRegister(masm, idId);
+ ValueOperand val = allocator.useValueRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+ enterStubFrame(masm, save);
+
+ masm.Push(Imm32(strict));
+ masm.Push(val);
+ masm.Push(id);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext * cx, Handle<NativeObject*> obj, int32_t int_id,
+ HandleValue v, bool strict);
+ callVM<Fn, AddOrUpdateSparseElementHelper>(masm);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitMegamorphicSetElement(ObjOperandId objId,
+ ValOperandId idId,
+ ValOperandId rhsId,
+ bool strict) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register obj = allocator.useRegister(masm, objId);
+ ConstantOrRegister idVal = allocator.useConstantOrRegister(masm, idId);
+ ConstantOrRegister val = allocator.useConstantOrRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+ enterStubFrame(masm, save);
+
+ masm.Push(Imm32(strict));
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(obj)));
+ masm.Push(val);
+ masm.Push(idVal);
+ masm.Push(obj);
+
+ using Fn = bool (*)(JSContext*, HandleObject, HandleValue, HandleValue,
+ HandleValue, bool);
+ callVM<Fn, SetElementMegamorphic>(masm);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitReturnFromIC() {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ if (!savedLiveRegs_) {
+ allocator.restoreInputState(masm);
+ }
+
+ uint8_t* rejoinAddr = ic_->rejoinAddr(ionScript_);
+ masm.jump(ImmPtr(rejoinAddr));
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardDOMExpandoMissingOrGuardShape(
+ ValOperandId expandoId, uint32_t shapeOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ ValueOperand val = allocator.useValueRegister(masm, expandoId);
+ Shape* shape = shapeStubField(shapeOffset);
+
+ AutoScratchRegister objScratch(allocator, masm);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ Label done;
+ masm.branchTestUndefined(Assembler::Equal, val, &done);
+
+ masm.debugAssertIsObject(val);
+ masm.unboxObject(val, objScratch);
+ // The expando object is not used in this case, so we don't need Spectre
+ // mitigations.
+ masm.branchTestObjShapeNoSpectreMitigations(Assembler::NotEqual, objScratch,
+ shape, failure->label());
+
+ masm.bind(&done);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitLoadDOMExpandoValueGuardGeneration(
+ ObjOperandId objId, uint32_t expandoAndGenerationOffset,
+ uint32_t generationOffset, ValOperandId resultId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ Register obj = allocator.useRegister(masm, objId);
+ ExpandoAndGeneration* expandoAndGeneration =
+ rawPointerStubField<ExpandoAndGeneration*>(expandoAndGenerationOffset);
+ uint64_t generation = rawInt64StubField<uint64_t>(generationOffset);
+
+ ValueOperand output = allocator.defineValueRegister(masm, resultId);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadDOMExpandoValueGuardGeneration(obj, output, expandoAndGeneration,
+ generation, failure->label());
+ return true;
+}
+
+void IonIC::attachCacheIRStub(JSContext* cx, const CacheIRWriter& writer,
+ CacheKind kind, IonScript* ionScript,
+ bool* attached) {
+ // We shouldn't GC or report OOM (or any other exception) here.
+ AutoAssertNoPendingException aanpe(cx);
+ JS::AutoCheckCannotGC nogc;
+
+ MOZ_ASSERT(!*attached);
+
+ // Do nothing if the IR generator failed or triggered a GC that invalidated
+ // the script.
+ if (writer.failed() || ionScript->invalidated()) {
+ return;
+ }
+
+ JitZone* jitZone = cx->zone()->jitZone();
+
+ constexpr uint32_t stubDataOffset = sizeof(IonICStub);
+ static_assert(stubDataOffset % sizeof(uint64_t) == 0,
+ "Stub fields must be aligned");
+
+ // Try to reuse a previously-allocated CacheIRStubInfo.
+ CacheIRStubKey::Lookup lookup(kind, ICStubEngine::IonIC, writer.codeStart(),
+ writer.codeLength());
+ CacheIRStubInfo* stubInfo = jitZone->getIonCacheIRStubInfo(lookup);
+ if (!stubInfo) {
+ // Allocate the shared CacheIRStubInfo. Note that the
+ // putIonCacheIRStubInfo call below will transfer ownership to
+ // the stub info HashSet, so we don't have to worry about freeing
+ // it below.
+
+ // For Ion ICs, we don't track/use the makesGCCalls flag, so just pass true.
+ bool makesGCCalls = true;
+ stubInfo = CacheIRStubInfo::New(kind, ICStubEngine::IonIC, makesGCCalls,
+ stubDataOffset, writer);
+ if (!stubInfo) {
+ return;
+ }
+
+ CacheIRStubKey key(stubInfo);
+ if (!jitZone->putIonCacheIRStubInfo(lookup, key)) {
+ return;
+ }
+ }
+
+ MOZ_ASSERT(stubInfo);
+
+ // Ensure we don't attach duplicate stubs. This can happen if a stub failed
+ // for some reason and the IR generator doesn't check for exactly the same
+ // conditions.
+ for (IonICStub* stub = firstStub_; stub; stub = stub->next()) {
+ if (stub->stubInfo() != stubInfo) {
+ continue;
+ }
+ if (!writer.stubDataEquals(stub->stubDataStart())) {
+ continue;
+ }
+ return;
+ }
+
+ size_t bytesNeeded = stubInfo->stubDataOffset() + stubInfo->stubDataSize();
+
+ // Allocate the IonICStub in the optimized stub space. Ion stubs and
+ // CacheIRStubInfo instances for Ion stubs can be purged on GC. That's okay
+ // because the stub code is rooted separately when we make a VM call, and
+ // stub code should never access the IonICStub after making a VM call. The
+ // IonICStub::poison method poisons the stub to catch bugs in this area.
+ ICStubSpace* stubSpace = cx->zone()->jitZone()->optimizedStubSpace();
+ void* newStubMem = stubSpace->alloc(bytesNeeded);
+ if (!newStubMem) {
+ return;
+ }
+
+ IonICStub* newStub =
+ new (newStubMem) IonICStub(fallbackAddr(ionScript), stubInfo);
+ writer.copyStubData(newStub->stubDataStart());
+
+ TempAllocator temp(&cx->tempLifoAlloc());
+ JitContext jctx(cx);
+ IonCacheIRCompiler compiler(cx, temp, writer, this, ionScript,
+ stubDataOffset);
+ if (!compiler.init()) {
+ return;
+ }
+
+ JitCode* code = compiler.compile(newStub);
+ if (!code) {
+ return;
+ }
+
+ // Add an entry to the profiler's code table, so that the profiler can
+ // identify this as Ion code.
+ if (ionScript->hasProfilingInstrumentation()) {
+ uint8_t* addr = rejoinAddr(ionScript);
+ auto entry = MakeJitcodeGlobalEntry<IonICEntry>(cx, code, code->raw(),
+ code->rawEnd(), addr);
+ if (!entry) {
+ cx->recoverFromOutOfMemory();
+ return;
+ }
+
+ auto* globalTable = cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
+ if (!globalTable->addEntry(std::move(entry))) {
+ return;
+ }
+ }
+
+ attachStub(newStub, code);
+ *attached = true;
+}
+
+bool IonCacheIRCompiler::emitCallStringObjectConcatResult(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+ AutoOutputRegister output(*this);
+
+ ValueOperand lhs = allocator.useValueRegister(masm, lhsId);
+ ValueOperand rhs = allocator.useValueRegister(masm, rhsId);
+
+ allocator.discardStack(masm);
+
+ enterStubFrame(masm, save);
+ masm.Push(rhs);
+ masm.Push(lhs);
+
+ using Fn = bool (*)(JSContext*, HandleValue, HandleValue, MutableHandleValue);
+ callVM<Fn, DoConcatStringObject>(masm);
+
+ masm.storeCallResultValue(output);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCloseIterScriptedResult(ObjOperandId iterId,
+ ObjOperandId calleeId,
+ CompletionKind kind,
+ uint32_t calleeNargs) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+ AutoSaveLiveRegisters save(*this);
+
+ Register iter = allocator.useRegister(masm, iterId);
+ Register callee = allocator.useRegister(masm, calleeId);
+
+ allocator.discardStack(masm);
+
+ uint32_t framePushedBefore = masm.framePushed();
+
+ // Construct IonICCallFrameLayout.
+ enterStubFrame(masm, save);
+
+ uint32_t stubFramePushed = masm.framePushed();
+
+ // The JitFrameLayout pushed below will be aligned to JitStackAlignment,
+ // so we just have to make sure the stack is aligned after we push |this|
+ // and |calleeNargs| undefined arguments.
+ uint32_t argSize = (calleeNargs + 1) * sizeof(Value);
+ uint32_t padding =
+ ComputeByteAlignment(masm.framePushed() + argSize, JitStackAlignment);
+ MOZ_ASSERT(padding % sizeof(uintptr_t) == 0);
+ MOZ_ASSERT(padding < JitStackAlignment);
+ masm.reserveStack(padding);
+
+ for (uint32_t i = 0; i < calleeNargs; i++) {
+ masm.Push(UndefinedValue());
+ }
+ masm.Push(TypedOrValueRegister(MIRType::Object, AnyRegister(iter)));
+
+ masm.Push(callee);
+ masm.PushFrameDescriptorForJitCall(FrameType::IonICCall, /* argc = */ 0);
+
+ masm.loadJitCodeRaw(callee, callee);
+ masm.callJit(callee);
+
+ if (kind != CompletionKind::Throw) {
+ // Verify that the return value is an object.
+ Label success;
+ masm.branchTestObject(Assembler::Equal, JSReturnOperand, &success);
+
+ // We can reuse the same stub frame, but we first have to pop the arguments
+ // from the previous call.
+ uint32_t framePushedAfterCall = masm.framePushed();
+ masm.freeStack(masm.framePushed() - stubFramePushed);
+
+ masm.push(Imm32(int32_t(CheckIsObjectKind::IteratorReturn)));
+ using Fn = bool (*)(JSContext*, CheckIsObjectKind);
+ callVM<Fn, ThrowCheckIsObject>(masm);
+
+ masm.bind(&success);
+ masm.setFramePushed(framePushedAfterCall);
+ }
+
+ // Restore the frame pointer and stack pointer.
+ masm.loadPtr(Address(FramePointer, 0), FramePointer);
+ masm.freeStack(masm.framePushed() - framePushedBefore);
+ return true;
+}
+
+bool IonCacheIRCompiler::emitGuardFunctionScript(ObjOperandId funId,
+ uint32_t expectedOffset,
+ uint32_t nargsAndFlagsOffset) {
+ JitSpew(JitSpew_Codegen, "%s", __FUNCTION__);
+
+ Register fun = allocator.useRegister(masm, funId);
+ AutoScratchRegister scratch(allocator, masm);
+ BaseScript* expected = baseScriptStubField(expectedOffset);
+
+ FailurePath* failure;
+ if (!addFailurePath(&failure)) {
+ return false;
+ }
+
+ masm.loadPrivate(Address(fun, JSFunction::offsetOfJitInfoOrScript()),
+ scratch);
+ masm.branchPtr(Assembler::NotEqual, scratch, ImmGCPtr(expected),
+ failure->label());
+ return true;
+}
+
+bool IonCacheIRCompiler::emitCallScriptedFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitCallWasmFunction(
+ ObjOperandId calleeId, Int32OperandId argcId, CallFlags flags,
+ uint32_t argcFixed, uint32_t funcExportOffset, uint32_t instanceOffset) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+#ifdef JS_SIMULATOR
+bool IonCacheIRCompiler::emitCallNativeFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed,
+ uint32_t targetOffset) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitCallDOMFunction(
+ ObjOperandId calleeId, Int32OperandId argcId, ObjOperandId thisObjId,
+ CallFlags flags, uint32_t argcFixed, uint32_t targetOffset) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+#else
+bool IonCacheIRCompiler::emitCallNativeFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed,
+ bool ignoresReturnValue) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitCallDOMFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ ObjOperandId thisObjId,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+#endif
+
+bool IonCacheIRCompiler::emitCallClassHook(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags, uint32_t argcFixed,
+ uint32_t targetOffset) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitCallInlinedFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ uint32_t icScriptOffset,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitLoadArgumentFixedSlot(ValOperandId resultId,
+ uint8_t slotIndex) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitLoadArgumentDynamicSlot(ValOperandId resultId,
+ Int32OperandId argcId,
+ uint8_t slotIndex) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitArrayJoinResult(ObjOperandId objId,
+ StringOperandId sepId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitPackedArraySliceResult(
+ uint32_t templateObjectOffset, ObjOperandId arrayId, Int32OperandId beginId,
+ Int32OperandId endId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitArgumentsSliceResult(uint32_t templateObjectOffset,
+ ObjOperandId argsId,
+ Int32OperandId beginId,
+ Int32OperandId endId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitIsArrayResult(ValOperandId inputId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitIsTypedArrayResult(ObjOperandId objId,
+ bool isPossiblyWrapped) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitStringFromCharCodeResult(Int32OperandId codeId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitStringFromCodePointResult(Int32OperandId codeId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitMathRandomResult(uint32_t rngOffset) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitReflectGetPrototypeOfResult(ObjOperandId objId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitHasClassResult(ObjOperandId objId,
+ uint32_t claspOffset) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitSameValueResult(ValOperandId lhs,
+ ValOperandId rhs) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitSetHasStringResult(ObjOperandId setId,
+ StringOperandId strId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitMapHasStringResult(ObjOperandId mapId,
+ StringOperandId strId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitMapGetStringResult(ObjOperandId mapId,
+ StringOperandId strId) {
+ MOZ_CRASH("Call ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitNewArrayObjectResult(uint32_t arrayLength,
+ uint32_t shapeOffset,
+ uint32_t siteOffset) {
+ MOZ_CRASH("NewArray ICs not used in ion");
+}
+
+bool IonCacheIRCompiler::emitNewPlainObjectResult(uint32_t numFixedSlots,
+ uint32_t numDynamicSlots,
+ gc::AllocKind allocKind,
+ uint32_t shapeOffset,
+ uint32_t siteOffset) {
+ MOZ_CRASH("NewObject ICs not used in ion");
+}
diff --git a/js/src/jit/IonCacheIRCompiler.h b/js/src/jit/IonCacheIRCompiler.h
new file mode 100644
index 0000000000..890e54947d
--- /dev/null
+++ b/js/src/jit/IonCacheIRCompiler.h
@@ -0,0 +1,89 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_IonCacheIRCompiler_h
+#define jit_IonCacheIRCompiler_h
+
+#include "mozilla/Attributes.h"
+#include "mozilla/Maybe.h"
+
+#include <stdint.h>
+
+#include "jstypes.h"
+
+#include "jit/CacheIR.h"
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIROpsGenerated.h"
+#include "jit/CacheIRReader.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "js/Vector.h"
+
+struct JS_PUBLIC_API JSContext;
+
+namespace js {
+namespace jit {
+
+class CacheIRWriter;
+class CodeOffset;
+class IonIC;
+class IonICStub;
+class IonScript;
+class JitCode;
+class MacroAssembler;
+
+// IonCacheIRCompiler compiles CacheIR to IonIC native code.
+class MOZ_RAII IonCacheIRCompiler : public CacheIRCompiler {
+ public:
+ friend class AutoSaveLiveRegisters;
+ friend class AutoCallVM;
+
+ IonCacheIRCompiler(JSContext* cx, TempAllocator& alloc,
+ const CacheIRWriter& writer, IonIC* ic,
+ IonScript* ionScript, uint32_t stubDataOffset);
+
+ [[nodiscard]] bool init();
+ JitCode* compile(IonICStub* stub);
+
+#ifdef DEBUG
+ void assertFloatRegisterAvailable(FloatRegister reg);
+#endif
+
+ private:
+ const CacheIRWriter& writer_;
+ IonIC* ic_;
+ IonScript* ionScript_;
+
+ Vector<CodeOffset, 4, SystemAllocPolicy> nextCodeOffsets_;
+ mozilla::Maybe<LiveRegisterSet> liveRegs_;
+ mozilla::Maybe<CodeOffset> stubJitCodeOffset_;
+
+ bool savedLiveRegs_;
+
+ template <typename T>
+ T rawPointerStubField(uint32_t offset);
+
+ template <typename T>
+ T rawInt64StubField(uint32_t offset);
+
+ void enterStubFrame(MacroAssembler& masm, const AutoSaveLiveRegisters&);
+
+ template <typename Fn, Fn fn>
+ void callVM(MacroAssembler& masm);
+
+ [[nodiscard]] bool emitAddAndStoreSlotShared(
+ CacheOp op, ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset, mozilla::Maybe<uint32_t> numNewSlotsOffset);
+
+ void pushStubCodePointer();
+
+ CACHE_IR_COMPILER_UNSHARED_GENERATED
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_IonCacheIRCompiler_h */
diff --git a/js/src/jit/IonCompileTask.cpp b/js/src/jit/IonCompileTask.cpp
new file mode 100644
index 0000000000..0c01112908
--- /dev/null
+++ b/js/src/jit/IonCompileTask.cpp
@@ -0,0 +1,203 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/IonCompileTask.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/Ion.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitScript.h"
+#include "jit/WarpSnapshot.h"
+#include "vm/HelperThreadState.h"
+#include "vm/JSScript.h"
+
+#include "vm/JSScript-inl.h"
+#include "vm/Realm-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void IonCompileTask::runHelperThreadTask(AutoLockHelperThreadState& locked) {
+ // The build is taken by this thread. Unfreeze the LifoAlloc to allow
+ // mutations.
+ alloc().lifoAlloc()->setReadWrite();
+
+ {
+ AutoUnlockHelperThreadState unlock(locked);
+ runTask();
+ }
+
+ FinishOffThreadIonCompile(this, locked);
+
+ JSRuntime* rt = script()->runtimeFromAnyThread();
+
+ // Ping the main thread so that the compiled code can be incorporated at the
+ // next interrupt callback.
+ //
+ // This must happen before the current task is reset. DestroyContext
+ // cancels in progress Ion compilations before destroying its target
+ // context, and after we reset the current task we are no longer considered
+ // to be Ion compiling.
+ rt->mainContextFromAnyThread()->requestInterrupt(
+ InterruptReason::AttachIonCompilations);
+}
+
+void IonCompileTask::runTask() {
+ // This is the entry point when ion compiles are run offthread.
+
+ jit::JitContext jctx(mirGen_.realm->runtime());
+ setBackgroundCodegen(jit::CompileBackEnd(&mirGen_, snapshot_));
+}
+
+void IonCompileTask::trace(JSTracer* trc) {
+ if (!mirGen_.runtime->runtimeMatches(trc->runtime())) {
+ return;
+ }
+
+ snapshot_->trace(trc);
+}
+
+IonCompileTask::IonCompileTask(JSContext* cx, MIRGenerator& mirGen,
+ WarpSnapshot* snapshot)
+ : mirGen_(mirGen),
+ snapshot_(snapshot),
+ isExecuting_(cx->isExecutingRef()) {}
+
+size_t IonCompileTask::sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) {
+ // See js::jit::FreeIonCompileTask.
+ // The IonCompileTask and most of its contents live in the LifoAlloc we point
+ // to.
+
+ size_t result = alloc().lifoAlloc()->sizeOfIncludingThis(mallocSizeOf);
+
+ if (backgroundCodegen_) {
+ result += mallocSizeOf(backgroundCodegen_);
+ }
+
+ return result;
+}
+
+static inline bool TooManyUnlinkedTasks(JSRuntime* rt) {
+ static const size_t MaxUnlinkedTasks = 100;
+ return rt->jitRuntime()->ionLazyLinkListSize() > MaxUnlinkedTasks;
+}
+
+static void MoveFinishedTasksToLazyLinkList(
+ JSRuntime* rt, const AutoLockHelperThreadState& lock) {
+ // Incorporate any off thread compilations for the runtime which have
+ // finished, failed or have been cancelled.
+
+ GlobalHelperThreadState::IonCompileTaskVector& finished =
+ HelperThreadState().ionFinishedList(lock);
+
+ for (size_t i = 0; i < finished.length(); i++) {
+ // Find a finished task for the runtime.
+ IonCompileTask* task = finished[i];
+ if (task->script()->runtimeFromAnyThread() != rt) {
+ continue;
+ }
+
+ HelperThreadState().remove(finished, &i);
+ rt->jitRuntime()->numFinishedOffThreadTasksRef(lock)--;
+
+ JSScript* script = task->script();
+ MOZ_ASSERT(script->hasBaselineScript());
+ script->baselineScript()->setPendingIonCompileTask(rt, script, task);
+ rt->jitRuntime()->ionLazyLinkListAdd(rt, task);
+ }
+}
+
+static void EagerlyLinkExcessTasks(JSContext* cx,
+ AutoLockHelperThreadState& lock) {
+ JSRuntime* rt = cx->runtime();
+ MOZ_ASSERT(TooManyUnlinkedTasks(rt));
+
+ do {
+ jit::IonCompileTask* task = rt->jitRuntime()->ionLazyLinkList(rt).getLast();
+ RootedScript script(cx, task->script());
+
+ AutoUnlockHelperThreadState unlock(lock);
+ AutoRealm ar(cx, script);
+ jit::LinkIonScript(cx, script);
+ } while (TooManyUnlinkedTasks(rt));
+}
+
+void jit::AttachFinishedCompilations(JSContext* cx) {
+ JSRuntime* rt = cx->runtime();
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(rt));
+
+ if (!rt->jitRuntime() || !rt->jitRuntime()->numFinishedOffThreadTasks()) {
+ return;
+ }
+
+ AutoLockHelperThreadState lock;
+
+ while (true) {
+ MoveFinishedTasksToLazyLinkList(rt, lock);
+
+ if (!TooManyUnlinkedTasks(rt)) {
+ break;
+ }
+
+ EagerlyLinkExcessTasks(cx, lock);
+
+ // Linking releases the lock so we must now check the finished list
+ // again.
+ }
+
+ MOZ_ASSERT(!rt->jitRuntime()->numFinishedOffThreadTasks());
+}
+
+void jit::FreeIonCompileTask(IonCompileTask* task) {
+ // The task is allocated into its LifoAlloc, so destroying that will
+ // destroy the task and all other data accumulated during compilation,
+ // except any final codegen (which includes an assembler and needs to be
+ // explicitly destroyed).
+ js_delete(task->backgroundCodegen());
+ js_delete(task->alloc().lifoAlloc());
+}
+
+void IonFreeTask::runHelperThreadTask(AutoLockHelperThreadState& locked) {
+ {
+ AutoUnlockHelperThreadState unlock(locked);
+ jit::FreeIonCompileTask(task_);
+ }
+
+ js_delete(this);
+}
+
+void jit::FinishOffThreadTask(JSRuntime* runtime, IonCompileTask* task,
+ const AutoLockHelperThreadState& locked) {
+ MOZ_ASSERT(runtime);
+
+ JSScript* script = task->script();
+
+ // Clean the references to the pending IonCompileTask, if we just finished it.
+ if (script->baselineScript()->hasPendingIonCompileTask() &&
+ script->baselineScript()->pendingIonCompileTask() == task) {
+ script->baselineScript()->removePendingIonCompileTask(runtime, script);
+ }
+
+ // If the task is still in one of the helper thread lists, then remove it.
+ if (task->isInList()) {
+ runtime->jitRuntime()->ionLazyLinkListRemove(runtime, task);
+ }
+
+ // Clean up if compilation did not succeed.
+ if (script->isIonCompilingOffThread()) {
+ script->jitScript()->clearIsIonCompilingOffThread(script);
+
+ const AbortReasonOr<Ok>& status = task->mirGen().getOffThreadStatus();
+ if (status.isErr() && status.inspectErr() == AbortReason::Disable) {
+ script->disableIon();
+ }
+ }
+
+ // Free Ion LifoAlloc off-thread. Free on the main thread if this OOMs.
+ if (!StartOffThreadIonFree(task, locked)) {
+ FreeIonCompileTask(task);
+ }
+}
diff --git a/js/src/jit/IonCompileTask.h b/js/src/jit/IonCompileTask.h
new file mode 100644
index 0000000000..c997c13bc6
--- /dev/null
+++ b/js/src/jit/IonCompileTask.h
@@ -0,0 +1,89 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_IonCompileTask_h
+#define jit_IonCompileTask_h
+
+#include "mozilla/LinkedList.h"
+
+#include "jit/MIRGenerator.h"
+
+#include "js/Utility.h"
+#include "vm/HelperThreadTask.h"
+
+struct JS_PUBLIC_API JSContext;
+
+namespace js {
+namespace jit {
+
+class CodeGenerator;
+class WarpSnapshot;
+
+// IonCompileTask represents a single off-thread Ion compilation task.
+class IonCompileTask final : public HelperThreadTask,
+ public mozilla::LinkedListElement<IonCompileTask> {
+ MIRGenerator& mirGen_;
+
+ // If off thread compilation is successful, the final code generator is
+ // attached here. Code has been generated, but not linked (there is not yet
+ // an IonScript). This is heap allocated, and must be explicitly destroyed,
+ // performed by FinishOffThreadTask().
+ CodeGenerator* backgroundCodegen_ = nullptr;
+
+ WarpSnapshot* snapshot_ = nullptr;
+
+ // Alias of the JSContext field of this task, to determine the priority of
+ // compiling this script. Contexts are destroyed after the pending tasks are
+ // removed from the helper threads. Thus this should be safe.
+ const mozilla::Atomic<bool, mozilla::ReleaseAcquire>& isExecuting_;
+
+ public:
+ explicit IonCompileTask(JSContext* cx, MIRGenerator& mirGen,
+ WarpSnapshot* snapshot);
+
+ JSScript* script() { return mirGen_.outerInfo().script(); }
+ MIRGenerator& mirGen() { return mirGen_; }
+ TempAllocator& alloc() { return mirGen_.alloc(); }
+ WarpSnapshot* snapshot() { return snapshot_; }
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf);
+ void trace(JSTracer* trc);
+
+ CodeGenerator* backgroundCodegen() const { return backgroundCodegen_; }
+ void setBackgroundCodegen(CodeGenerator* codegen) {
+ backgroundCodegen_ = codegen;
+ }
+
+ // Return whether the main thread which scheduled this task is currently
+ // executing JS code. This changes the way we prioritize tasks.
+ bool isMainThreadRunningJS() const { return isExecuting_; }
+
+ ThreadType threadType() override { return THREAD_TYPE_ION; }
+ void runTask();
+ void runHelperThreadTask(AutoLockHelperThreadState& locked) override;
+};
+
+class IonFreeTask : public HelperThreadTask {
+ public:
+ explicit IonFreeTask(IonCompileTask* task) : task_(task) {}
+ IonCompileTask* compileTask() { return task_; }
+
+ ThreadType threadType() override { return THREAD_TYPE_ION_FREE; }
+ void runHelperThreadTask(AutoLockHelperThreadState& locked) override;
+
+ private:
+ IonCompileTask* task_;
+};
+
+void AttachFinishedCompilations(JSContext* cx);
+void FinishOffThreadTask(JSRuntime* runtime, IonCompileTask* task,
+ const AutoLockHelperThreadState& lock);
+void FreeIonCompileTask(IonCompileTask* task);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_IonCompileTask_h */
diff --git a/js/src/jit/IonIC.cpp b/js/src/jit/IonIC.cpp
new file mode 100644
index 0000000000..9a8332055b
--- /dev/null
+++ b/js/src/jit/IonIC.cpp
@@ -0,0 +1,722 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/IonIC.h"
+
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIRGenerator.h"
+#include "jit/IonScript.h"
+#include "jit/VMFunctions.h"
+#include "util/DiagnosticAssertions.h"
+#include "vm/EqualityOperations.h"
+#include "vm/Iteration.h"
+
+#include "vm/Interpreter-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void IonIC::resetCodeRaw(IonScript* ionScript) {
+ codeRaw_ = fallbackAddr(ionScript);
+}
+
+uint8_t* IonIC::fallbackAddr(IonScript* ionScript) const {
+ return ionScript->method()->raw() + fallbackOffset_;
+}
+
+uint8_t* IonIC::rejoinAddr(IonScript* ionScript) const {
+ return ionScript->method()->raw() + rejoinOffset_;
+}
+
+Register IonIC::scratchRegisterForEntryJump() {
+ switch (kind_) {
+ case CacheKind::GetProp:
+ case CacheKind::GetElem:
+ return asGetPropertyIC()->output().scratchReg();
+ case CacheKind::GetPropSuper:
+ case CacheKind::GetElemSuper:
+ return asGetPropSuperIC()->output().scratchReg();
+ case CacheKind::SetProp:
+ case CacheKind::SetElem:
+ return asSetPropertyIC()->temp();
+ case CacheKind::GetName:
+ return asGetNameIC()->temp();
+ case CacheKind::BindName:
+ return asBindNameIC()->temp();
+ case CacheKind::In:
+ return asInIC()->temp();
+ case CacheKind::HasOwn:
+ return asHasOwnIC()->output();
+ case CacheKind::CheckPrivateField:
+ return asCheckPrivateFieldIC()->output();
+ case CacheKind::GetIterator:
+ return asGetIteratorIC()->temp1();
+ case CacheKind::OptimizeSpreadCall:
+ return asOptimizeSpreadCallIC()->temp();
+ case CacheKind::InstanceOf:
+ return asInstanceOfIC()->output();
+ case CacheKind::UnaryArith:
+ return asUnaryArithIC()->output().scratchReg();
+ case CacheKind::ToPropertyKey:
+ return asToPropertyKeyIC()->output().scratchReg();
+ case CacheKind::BinaryArith:
+ return asBinaryArithIC()->output().scratchReg();
+ case CacheKind::Compare:
+ return asCompareIC()->output();
+ case CacheKind::CloseIter:
+ return asCloseIterIC()->temp();
+ case CacheKind::Call:
+ case CacheKind::TypeOf:
+ case CacheKind::ToBool:
+ case CacheKind::GetIntrinsic:
+ case CacheKind::NewArray:
+ case CacheKind::NewObject:
+ MOZ_CRASH("Unsupported IC");
+ }
+
+ MOZ_CRASH("Invalid kind");
+}
+
+void IonIC::discardStubs(Zone* zone, IonScript* ionScript) {
+ if (firstStub_) {
+ // We are removing edges from IonIC to gcthings. Perform a write barrier to
+ // let the GC know about those edges.
+ PreWriteBarrier(zone, ionScript);
+ }
+
+#ifdef JS_CRASH_DIAGNOSTICS
+ IonICStub* stub = firstStub_;
+ while (stub) {
+ IonICStub* next = stub->next();
+ stub->poison();
+ stub = next;
+ }
+#endif
+
+ firstStub_ = nullptr;
+ resetCodeRaw(ionScript);
+ state_.trackUnlinkedAllStubs();
+}
+
+void IonIC::reset(Zone* zone, IonScript* ionScript) {
+ discardStubs(zone, ionScript);
+ state_.reset();
+}
+
+void IonIC::trace(JSTracer* trc, IonScript* ionScript) {
+ if (script_) {
+ TraceManuallyBarrieredEdge(trc, &script_, "IonIC::script_");
+ }
+
+ uint8_t* nextCodeRaw = codeRaw_;
+ for (IonICStub* stub = firstStub_; stub; stub = stub->next()) {
+ JitCode* code = JitCode::FromExecutable(nextCodeRaw);
+ TraceManuallyBarrieredEdge(trc, &code, "ion-ic-code");
+
+ TraceCacheIRStub(trc, stub, stub->stubInfo());
+
+ nextCodeRaw = stub->nextCodeRaw();
+ }
+
+ MOZ_ASSERT(nextCodeRaw == fallbackAddr(ionScript));
+}
+
+// This helper handles ICState updates/transitions while attaching CacheIR
+// stubs.
+template <typename IRGenerator, typename... Args>
+static void TryAttachIonStub(JSContext* cx, IonIC* ic, IonScript* ionScript,
+ Args&&... args) {
+ if (ic->state().maybeTransition()) {
+ ic->discardStubs(cx->zone(), ionScript);
+ }
+
+ if (ic->state().canAttachStub()) {
+ RootedScript script(cx, ic->script());
+ bool attached = false;
+ IRGenerator gen(cx, script, ic->pc(), ic->state(),
+ std::forward<Args>(args)...);
+ switch (gen.tryAttachStub()) {
+ case AttachDecision::Attach:
+ ic->attachCacheIRStub(cx, gen.writerRef(), gen.cacheKind(), ionScript,
+ &attached);
+ break;
+ case AttachDecision::NoAction:
+ break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ attached = true;
+ break;
+ case AttachDecision::Deferred:
+ MOZ_ASSERT_UNREACHABLE("Not expected in generic TryAttachIonStub");
+ break;
+ }
+ if (!attached) {
+ ic->state().trackNotAttached();
+ }
+ }
+}
+
+/* static */
+bool IonGetPropertyIC::update(JSContext* cx, HandleScript outerScript,
+ IonGetPropertyIC* ic, HandleValue val,
+ HandleValue idVal, MutableHandleValue res) {
+ IonScript* ionScript = outerScript->ionScript();
+
+ // Optimized-arguments and other magic values must not escape to Ion ICs.
+ MOZ_ASSERT(!val.isMagic());
+
+ TryAttachIonStub<GetPropIRGenerator>(cx, ic, ionScript, ic->kind(), val,
+ idVal);
+
+ if (ic->kind() == CacheKind::GetProp) {
+ Rooted<PropertyName*> name(cx, idVal.toString()->asAtom().asPropertyName());
+ if (!GetProperty(cx, val, name, res)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(ic->kind() == CacheKind::GetElem);
+ if (!GetElementOperation(cx, val, idVal, res)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+/* static */
+bool IonGetPropSuperIC::update(JSContext* cx, HandleScript outerScript,
+ IonGetPropSuperIC* ic, HandleObject obj,
+ HandleValue receiver, HandleValue idVal,
+ MutableHandleValue res) {
+ IonScript* ionScript = outerScript->ionScript();
+
+ if (ic->state().maybeTransition()) {
+ ic->discardStubs(cx->zone(), ionScript);
+ }
+
+ RootedValue val(cx, ObjectValue(*obj));
+
+ TryAttachIonStub<GetPropIRGenerator>(cx, ic, ionScript, ic->kind(), val,
+ idVal);
+
+ if (ic->kind() == CacheKind::GetPropSuper) {
+ Rooted<PropertyName*> name(cx, idVal.toString()->asAtom().asPropertyName());
+ if (!GetProperty(cx, obj, receiver, name, res)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(ic->kind() == CacheKind::GetElemSuper);
+
+ JSOp op = JSOp(*ic->pc());
+ MOZ_ASSERT(op == JSOp::GetElemSuper);
+
+ if (!GetObjectElementOperation(cx, op, obj, receiver, idVal, res)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+/* static */
+bool IonSetPropertyIC::update(JSContext* cx, HandleScript outerScript,
+ IonSetPropertyIC* ic, HandleObject obj,
+ HandleValue idVal, HandleValue rhs) {
+ using DeferType = SetPropIRGenerator::DeferType;
+
+ Rooted<Shape*> oldShape(cx);
+ IonScript* ionScript = outerScript->ionScript();
+
+ bool attached = false;
+ DeferType deferType = DeferType::None;
+
+ if (ic->state().maybeTransition()) {
+ ic->discardStubs(cx->zone(), ionScript);
+ }
+
+ if (ic->state().canAttachStub()) {
+ oldShape = obj->shape();
+
+ RootedValue objv(cx, ObjectValue(*obj));
+ RootedScript script(cx, ic->script());
+ jsbytecode* pc = ic->pc();
+
+ SetPropIRGenerator gen(cx, script, pc, ic->kind(), ic->state(), objv, idVal,
+ rhs);
+ switch (gen.tryAttachStub()) {
+ case AttachDecision::Attach:
+ ic->attachCacheIRStub(cx, gen.writerRef(), gen.cacheKind(), ionScript,
+ &attached);
+ break;
+ case AttachDecision::NoAction:
+ break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ attached = true;
+ break;
+ case AttachDecision::Deferred:
+ deferType = gen.deferType();
+ MOZ_ASSERT(deferType != DeferType::None);
+ break;
+ }
+ }
+
+ jsbytecode* pc = ic->pc();
+ if (ic->kind() == CacheKind::SetElem) {
+ if (JSOp(*pc) == JSOp::InitElemInc) {
+ if (!InitElemIncOperation(cx, obj.as<ArrayObject>(), idVal.toInt32(),
+ rhs)) {
+ return false;
+ }
+ } else if (IsPropertyInitOp(JSOp(*pc))) {
+ if (!InitElemOperation(cx, pc, obj, idVal, rhs)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc)));
+ if (!SetObjectElement(cx, obj, idVal, rhs, ic->strict())) {
+ return false;
+ }
+ }
+ } else {
+ MOZ_ASSERT(ic->kind() == CacheKind::SetProp);
+
+ if (JSOp(*pc) == JSOp::InitGLexical) {
+ RootedScript script(cx, ic->script());
+ MOZ_ASSERT(!script->hasNonSyntacticScope());
+ InitGlobalLexicalOperation(cx, &cx->global()->lexicalEnvironment(),
+ script, pc, rhs);
+ } else if (IsPropertyInitOp(JSOp(*pc))) {
+ Rooted<PropertyName*> name(cx,
+ idVal.toString()->asAtom().asPropertyName());
+ if (!InitPropertyOperation(cx, pc, obj, name, rhs)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(IsPropertySetOp(JSOp(*pc)));
+ Rooted<PropertyName*> name(cx,
+ idVal.toString()->asAtom().asPropertyName());
+ if (!SetProperty(cx, obj, name, rhs, ic->strict(), pc)) {
+ return false;
+ }
+ }
+ }
+
+ if (attached) {
+ return true;
+ }
+
+ // The SetProperty call might have entered this IC recursively, so try
+ // to transition.
+ if (ic->state().maybeTransition()) {
+ ic->discardStubs(cx->zone(), ionScript);
+ }
+
+ bool canAttachStub = ic->state().canAttachStub();
+ if (deferType != DeferType::None && canAttachStub) {
+ RootedValue objv(cx, ObjectValue(*obj));
+ RootedScript script(cx, ic->script());
+ jsbytecode* pc = ic->pc();
+ SetPropIRGenerator gen(cx, script, pc, ic->kind(), ic->state(), objv, idVal,
+ rhs);
+ MOZ_ASSERT(deferType == DeferType::AddSlot);
+ AttachDecision decision = gen.tryAttachAddSlotStub(oldShape);
+
+ switch (decision) {
+ case AttachDecision::Attach:
+ ic->attachCacheIRStub(cx, gen.writerRef(), gen.cacheKind(), ionScript,
+ &attached);
+ break;
+ case AttachDecision::NoAction:
+ gen.trackAttached(IRGenerator::NotAttached);
+ break;
+ case AttachDecision::TemporarilyUnoptimizable:
+ case AttachDecision::Deferred:
+ MOZ_ASSERT_UNREACHABLE("Invalid attach result");
+ break;
+ }
+ }
+ if (!attached && canAttachStub) {
+ ic->state().trackNotAttached();
+ }
+
+ return true;
+}
+
+/* static */
+bool IonGetNameIC::update(JSContext* cx, HandleScript outerScript,
+ IonGetNameIC* ic, HandleObject envChain,
+ MutableHandleValue res) {
+ IonScript* ionScript = outerScript->ionScript();
+ jsbytecode* pc = ic->pc();
+ Rooted<PropertyName*> name(cx, ic->script()->getName(pc));
+
+ TryAttachIonStub<GetNameIRGenerator>(cx, ic, ionScript, envChain, name);
+
+ RootedObject obj(cx);
+ RootedObject holder(cx);
+ PropertyResult prop;
+ if (!LookupName(cx, name, envChain, &obj, &holder, &prop)) {
+ return false;
+ }
+
+ if (JSOp(*GetNextPc(pc)) == JSOp::Typeof) {
+ return FetchName<GetNameMode::TypeOf>(cx, obj, holder, name, prop, res);
+ }
+
+ return FetchName<GetNameMode::Normal>(cx, obj, holder, name, prop, res);
+}
+
+/* static */
+JSObject* IonBindNameIC::update(JSContext* cx, HandleScript outerScript,
+ IonBindNameIC* ic, HandleObject envChain) {
+ IonScript* ionScript = outerScript->ionScript();
+ jsbytecode* pc = ic->pc();
+ Rooted<PropertyName*> name(cx, ic->script()->getName(pc));
+
+ TryAttachIonStub<BindNameIRGenerator>(cx, ic, ionScript, envChain, name);
+
+ RootedObject holder(cx);
+ if (!LookupNameUnqualified(cx, name, envChain, &holder)) {
+ return nullptr;
+ }
+
+ return holder;
+}
+
+/* static */
+JSObject* IonGetIteratorIC::update(JSContext* cx, HandleScript outerScript,
+ IonGetIteratorIC* ic, HandleValue value) {
+ IonScript* ionScript = outerScript->ionScript();
+
+ TryAttachIonStub<GetIteratorIRGenerator>(cx, ic, ionScript, value);
+
+ return ValueToIterator(cx, value);
+}
+
+/* static */
+bool IonOptimizeSpreadCallIC::update(JSContext* cx, HandleScript outerScript,
+ IonOptimizeSpreadCallIC* ic,
+ HandleValue value,
+ MutableHandleValue result) {
+ IonScript* ionScript = outerScript->ionScript();
+
+ TryAttachIonStub<OptimizeSpreadCallIRGenerator>(cx, ic, ionScript, value);
+
+ return OptimizeSpreadCall(cx, value, result);
+}
+
+/* static */
+bool IonHasOwnIC::update(JSContext* cx, HandleScript outerScript,
+ IonHasOwnIC* ic, HandleValue val, HandleValue idVal,
+ int32_t* res) {
+ IonScript* ionScript = outerScript->ionScript();
+
+ TryAttachIonStub<HasPropIRGenerator>(cx, ic, ionScript, CacheKind::HasOwn,
+ idVal, val);
+
+ bool found;
+ if (!HasOwnProperty(cx, val, idVal, &found)) {
+ return false;
+ }
+
+ *res = found;
+ return true;
+}
+
+/* static */
+bool IonCheckPrivateFieldIC::update(JSContext* cx, HandleScript outerScript,
+ IonCheckPrivateFieldIC* ic, HandleValue val,
+ HandleValue idVal, bool* res) {
+ IonScript* ionScript = outerScript->ionScript();
+ jsbytecode* pc = ic->pc();
+
+ TryAttachIonStub<CheckPrivateFieldIRGenerator>(
+ cx, ic, ionScript, CacheKind::CheckPrivateField, idVal, val);
+
+ return CheckPrivateFieldOperation(cx, pc, val, idVal, res);
+}
+
+/* static */
+bool IonInIC::update(JSContext* cx, HandleScript outerScript, IonInIC* ic,
+ HandleValue key, HandleObject obj, bool* res) {
+ IonScript* ionScript = outerScript->ionScript();
+ RootedValue objV(cx, ObjectValue(*obj));
+
+ TryAttachIonStub<HasPropIRGenerator>(cx, ic, ionScript, CacheKind::In, key,
+ objV);
+
+ return OperatorIn(cx, key, obj, res);
+}
+/* static */
+bool IonInstanceOfIC::update(JSContext* cx, HandleScript outerScript,
+ IonInstanceOfIC* ic, HandleValue lhs,
+ HandleObject rhs, bool* res) {
+ IonScript* ionScript = outerScript->ionScript();
+
+ TryAttachIonStub<InstanceOfIRGenerator>(cx, ic, ionScript, lhs, rhs);
+
+ return InstanceofOperator(cx, rhs, lhs, res);
+}
+
+/* static */
+bool IonToPropertyKeyIC::update(JSContext* cx, HandleScript outerScript,
+ IonToPropertyKeyIC* ic, HandleValue val,
+ MutableHandleValue res) {
+ IonScript* ionScript = outerScript->ionScript();
+
+ TryAttachIonStub<ToPropertyKeyIRGenerator>(cx, ic, ionScript, val);
+
+ return ToPropertyKeyOperation(cx, val, res);
+}
+
+/* static */
+bool IonCloseIterIC::update(JSContext* cx, HandleScript outerScript,
+ IonCloseIterIC* ic, HandleObject iter) {
+ IonScript* ionScript = outerScript->ionScript();
+ CompletionKind kind = ic->completionKind();
+
+ TryAttachIonStub<CloseIterIRGenerator>(cx, ic, ionScript, iter, kind);
+
+ return CloseIterOperation(cx, iter, kind);
+}
+
+/* static */
+bool IonUnaryArithIC::update(JSContext* cx, HandleScript outerScript,
+ IonUnaryArithIC* ic, HandleValue val,
+ MutableHandleValue res) {
+ IonScript* ionScript = outerScript->ionScript();
+ RootedScript script(cx, ic->script());
+ jsbytecode* pc = ic->pc();
+ JSOp op = JSOp(*pc);
+
+ switch (op) {
+ case JSOp::BitNot: {
+ res.set(val);
+ if (!BitNot(cx, res, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Pos: {
+ res.set(val);
+ if (!ToNumber(cx, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Neg: {
+ res.set(val);
+ if (!NegOperation(cx, res, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Inc: {
+ if (!IncOperation(cx, val, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Dec: {
+ if (!DecOperation(cx, val, res)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::ToNumeric: {
+ res.set(val);
+ if (!ToNumeric(cx, res)) {
+ return false;
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected op");
+ }
+ MOZ_ASSERT(res.isNumeric());
+
+ TryAttachIonStub<UnaryArithIRGenerator>(cx, ic, ionScript, op, val, res);
+
+ return true;
+}
+
+/* static */
+bool IonBinaryArithIC::update(JSContext* cx, HandleScript outerScript,
+ IonBinaryArithIC* ic, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue ret) {
+ IonScript* ionScript = outerScript->ionScript();
+ RootedScript script(cx, ic->script());
+ jsbytecode* pc = ic->pc();
+ JSOp op = JSOp(*pc);
+
+ // Don't pass lhs/rhs directly, we need the original values when
+ // generating stubs.
+ RootedValue lhsCopy(cx, lhs);
+ RootedValue rhsCopy(cx, rhs);
+
+ // Perform the compare operation.
+ switch (op) {
+ case JSOp::Add:
+ // Do an add.
+ if (!AddValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Sub:
+ if (!SubValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Mul:
+ if (!MulValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Div:
+ if (!DivValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Mod:
+ if (!ModValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::Pow:
+ if (!PowValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ case JSOp::BitOr: {
+ if (!BitOr(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::BitXor: {
+ if (!BitXor(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::BitAnd: {
+ if (!BitAnd(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Lsh: {
+ if (!BitLsh(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Rsh: {
+ if (!BitRsh(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ case JSOp::Ursh: {
+ if (!UrshValues(cx, &lhsCopy, &rhsCopy, ret)) {
+ return false;
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Unhandled binary arith op");
+ }
+
+ TryAttachIonStub<BinaryArithIRGenerator>(cx, ic, ionScript, op, lhs, rhs,
+ ret);
+
+ return true;
+}
+
+/* static */
+bool IonCompareIC::update(JSContext* cx, HandleScript outerScript,
+ IonCompareIC* ic, HandleValue lhs, HandleValue rhs,
+ bool* res) {
+ IonScript* ionScript = outerScript->ionScript();
+ RootedScript script(cx, ic->script());
+ jsbytecode* pc = ic->pc();
+ JSOp op = JSOp(*pc);
+
+ // Don't pass lhs/rhs directly, we need the original values when
+ // generating stubs.
+ RootedValue lhsCopy(cx, lhs);
+ RootedValue rhsCopy(cx, rhs);
+
+ // Perform the compare operation.
+ switch (op) {
+ case JSOp::Lt:
+ if (!LessThan(cx, &lhsCopy, &rhsCopy, res)) {
+ return false;
+ }
+ break;
+ case JSOp::Le:
+ if (!LessThanOrEqual(cx, &lhsCopy, &rhsCopy, res)) {
+ return false;
+ }
+ break;
+ case JSOp::Gt:
+ if (!GreaterThan(cx, &lhsCopy, &rhsCopy, res)) {
+ return false;
+ }
+ break;
+ case JSOp::Ge:
+ if (!GreaterThanOrEqual(cx, &lhsCopy, &rhsCopy, res)) {
+ return false;
+ }
+ break;
+ case JSOp::Eq:
+ if (!js::LooselyEqual(cx, lhsCopy, rhsCopy, res)) {
+ return false;
+ }
+ break;
+ case JSOp::Ne:
+ if (!js::LooselyEqual(cx, lhsCopy, rhsCopy, res)) {
+ return false;
+ }
+ *res = !*res;
+ break;
+ case JSOp::StrictEq:
+ if (!js::StrictlyEqual(cx, lhsCopy, rhsCopy, res)) {
+ return false;
+ }
+ break;
+ case JSOp::StrictNe:
+ if (!js::StrictlyEqual(cx, lhsCopy, rhsCopy, res)) {
+ return false;
+ }
+ *res = !*res;
+ break;
+ default:
+ MOZ_ASSERT_UNREACHABLE("Unhandled ion compare op");
+ return false;
+ }
+
+ TryAttachIonStub<CompareIRGenerator>(cx, ic, ionScript, op, lhs, rhs);
+
+ return true;
+}
+
+uint8_t* IonICStub::stubDataStart() {
+ return reinterpret_cast<uint8_t*>(this) + stubInfo_->stubDataOffset();
+}
+
+void IonIC::attachStub(IonICStub* newStub, JitCode* code) {
+ MOZ_ASSERT(newStub);
+ MOZ_ASSERT(code);
+
+ if (firstStub_) {
+ newStub->setNext(firstStub_, codeRaw_);
+ }
+ firstStub_ = newStub;
+ codeRaw_ = code->raw();
+
+ state_.trackAttached();
+}
diff --git a/js/src/jit/IonIC.h b/js/src/jit/IonIC.h
new file mode 100644
index 0000000000..2f5c61bcc6
--- /dev/null
+++ b/js/src/jit/IonIC.h
@@ -0,0 +1,664 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_IonIC_h
+#define jit_IonIC_h
+
+#include "jit/CacheIR.h"
+#include "jit/ICState.h"
+#include "jit/shared/Assembler-shared.h"
+
+namespace js {
+namespace jit {
+
+class CacheIRStubInfo;
+class CacheIRWriter;
+class IonScript;
+
+// An optimized stub attached to an IonIC.
+class IonICStub {
+ // Code to jump to when this stub fails. This is either the next optimized
+ // stub or the OOL fallback path.
+ uint8_t* nextCodeRaw_;
+
+ // The next optimized stub in this chain, or nullptr if this is the last
+ // one.
+ IonICStub* next_;
+
+ // Info about this stub.
+ CacheIRStubInfo* stubInfo_;
+
+#ifndef JS_64BIT
+ protected: // Silence Clang warning about unused private fields.
+ // Ensure stub data is 8-byte aligned on 32-bit.
+ uintptr_t padding_ = 0;
+#endif
+
+ public:
+ IonICStub(uint8_t* fallbackCode, CacheIRStubInfo* stubInfo)
+ : nextCodeRaw_(fallbackCode), next_(nullptr), stubInfo_(stubInfo) {}
+
+ uint8_t* nextCodeRaw() const { return nextCodeRaw_; }
+ uint8_t** nextCodeRawPtr() { return &nextCodeRaw_; }
+ CacheIRStubInfo* stubInfo() const { return stubInfo_; }
+ IonICStub* next() const { return next_; }
+
+ uint8_t* stubDataStart();
+
+ void setNext(IonICStub* next, uint8_t* nextCodeRaw) {
+ MOZ_ASSERT(!next_);
+ MOZ_ASSERT(next && nextCodeRaw);
+ next_ = next;
+ nextCodeRaw_ = nextCodeRaw;
+ }
+
+ // Null out pointers when we unlink stubs, to ensure we never use
+ // discarded stubs.
+ void poison() {
+ nextCodeRaw_ = nullptr;
+ next_ = nullptr;
+ stubInfo_ = nullptr;
+ }
+};
+
+class IonGetPropertyIC;
+class IonSetPropertyIC;
+class IonGetPropSuperIC;
+class IonGetNameIC;
+class IonBindNameIC;
+class IonGetIteratorIC;
+class IonHasOwnIC;
+class IonCheckPrivateFieldIC;
+class IonInIC;
+class IonInstanceOfIC;
+class IonCompareIC;
+class IonUnaryArithIC;
+class IonBinaryArithIC;
+class IonToPropertyKeyIC;
+class IonOptimizeSpreadCallIC;
+class IonCloseIterIC;
+
+class IonIC {
+ // This either points at the OOL path for the fallback path, or the code for
+ // the first stub.
+ uint8_t* codeRaw_;
+
+ // The first optimized stub, or nullptr.
+ IonICStub* firstStub_;
+
+ // Location of this IC.
+ JSScript* script_;
+ jsbytecode* pc_;
+
+ // The offset of the rejoin location in the IonScript's code (stubs jump to
+ // this location).
+ uint32_t rejoinOffset_;
+
+ // The offset of the OOL path in the IonScript's code that calls the IC's
+ // update function.
+ uint32_t fallbackOffset_;
+
+ CacheKind kind_;
+ ICState state_;
+
+ protected:
+ explicit IonIC(CacheKind kind)
+ : codeRaw_(nullptr),
+ firstStub_(nullptr),
+ script_(nullptr),
+ pc_(nullptr),
+ rejoinOffset_(0),
+ fallbackOffset_(0),
+ kind_(kind),
+ state_() {}
+
+ void attachStub(IonICStub* newStub, JitCode* code);
+
+ public:
+ void setScriptedLocation(JSScript* script, jsbytecode* pc) {
+ MOZ_ASSERT(!script_ && !pc_);
+ MOZ_ASSERT(script && pc);
+ script_ = script;
+ pc_ = pc;
+ }
+
+ JSScript* script() const {
+ MOZ_ASSERT(script_);
+ return script_;
+ }
+ jsbytecode* pc() const {
+ MOZ_ASSERT(pc_);
+ return pc_;
+ }
+
+ // Discard all stubs.
+ void discardStubs(Zone* zone, IonScript* ionScript);
+
+ // Discard all stubs and reset the ICState.
+ void reset(Zone* zone, IonScript* ionScript);
+
+ ICState& state() { return state_; }
+
+ CacheKind kind() const { return kind_; }
+ uint8_t** codeRawPtr() { return &codeRaw_; }
+
+ void setFallbackOffset(CodeOffset offset) {
+ fallbackOffset_ = offset.offset();
+ }
+ void setRejoinOffset(CodeOffset offset) { rejoinOffset_ = offset.offset(); }
+
+ void resetCodeRaw(IonScript* ionScript);
+
+ uint8_t* fallbackAddr(IonScript* ionScript) const;
+ uint8_t* rejoinAddr(IonScript* ionScript) const;
+
+ IonGetPropertyIC* asGetPropertyIC() {
+ MOZ_ASSERT(kind_ == CacheKind::GetProp || kind_ == CacheKind::GetElem);
+ return (IonGetPropertyIC*)this;
+ }
+ IonSetPropertyIC* asSetPropertyIC() {
+ MOZ_ASSERT(kind_ == CacheKind::SetProp || kind_ == CacheKind::SetElem);
+ return (IonSetPropertyIC*)this;
+ }
+ IonGetPropSuperIC* asGetPropSuperIC() {
+ MOZ_ASSERT(kind_ == CacheKind::GetPropSuper ||
+ kind_ == CacheKind::GetElemSuper);
+ return (IonGetPropSuperIC*)this;
+ }
+ IonGetNameIC* asGetNameIC() {
+ MOZ_ASSERT(kind_ == CacheKind::GetName);
+ return (IonGetNameIC*)this;
+ }
+ IonBindNameIC* asBindNameIC() {
+ MOZ_ASSERT(kind_ == CacheKind::BindName);
+ return (IonBindNameIC*)this;
+ }
+ IonGetIteratorIC* asGetIteratorIC() {
+ MOZ_ASSERT(kind_ == CacheKind::GetIterator);
+ return (IonGetIteratorIC*)this;
+ }
+ IonOptimizeSpreadCallIC* asOptimizeSpreadCallIC() {
+ MOZ_ASSERT(kind_ == CacheKind::OptimizeSpreadCall);
+ return (IonOptimizeSpreadCallIC*)this;
+ }
+ IonHasOwnIC* asHasOwnIC() {
+ MOZ_ASSERT(kind_ == CacheKind::HasOwn);
+ return (IonHasOwnIC*)this;
+ }
+ IonCheckPrivateFieldIC* asCheckPrivateFieldIC() {
+ MOZ_ASSERT(kind_ == CacheKind::CheckPrivateField);
+ return (IonCheckPrivateFieldIC*)this;
+ }
+ IonInIC* asInIC() {
+ MOZ_ASSERT(kind_ == CacheKind::In);
+ return (IonInIC*)this;
+ }
+ IonInstanceOfIC* asInstanceOfIC() {
+ MOZ_ASSERT(kind_ == CacheKind::InstanceOf);
+ return (IonInstanceOfIC*)this;
+ }
+ IonCompareIC* asCompareIC() {
+ MOZ_ASSERT(kind_ == CacheKind::Compare);
+ return (IonCompareIC*)this;
+ }
+ IonUnaryArithIC* asUnaryArithIC() {
+ MOZ_ASSERT(kind_ == CacheKind::UnaryArith);
+ return (IonUnaryArithIC*)this;
+ }
+ IonBinaryArithIC* asBinaryArithIC() {
+ MOZ_ASSERT(kind_ == CacheKind::BinaryArith);
+ return (IonBinaryArithIC*)this;
+ }
+ IonToPropertyKeyIC* asToPropertyKeyIC() {
+ MOZ_ASSERT(kind_ == CacheKind::ToPropertyKey);
+ return (IonToPropertyKeyIC*)this;
+ }
+ IonCloseIterIC* asCloseIterIC() {
+ MOZ_ASSERT(kind_ == CacheKind::CloseIter);
+ return (IonCloseIterIC*)this;
+ }
+
+ // Returns the Register to use as scratch when entering IC stubs. This
+ // should either be an output register or a temp.
+ Register scratchRegisterForEntryJump();
+
+ void trace(JSTracer* trc, IonScript* ionScript);
+
+ void attachCacheIRStub(JSContext* cx, const CacheIRWriter& writer,
+ CacheKind kind, IonScript* ionScript, bool* attached);
+};
+
+class IonGetPropertyIC : public IonIC {
+ private:
+ LiveRegisterSet liveRegs_;
+
+ TypedOrValueRegister value_;
+ ConstantOrRegister id_;
+ ValueOperand output_;
+
+ public:
+ IonGetPropertyIC(CacheKind kind, LiveRegisterSet liveRegs,
+ TypedOrValueRegister value, const ConstantOrRegister& id,
+ ValueOperand output)
+ : IonIC(kind),
+ liveRegs_(liveRegs),
+ value_(value),
+ id_(id),
+ output_(output) {}
+
+ TypedOrValueRegister value() const { return value_; }
+ ConstantOrRegister id() const { return id_; }
+ ValueOperand output() const { return output_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonGetPropertyIC* ic, HandleValue val,
+ HandleValue idVal, MutableHandleValue res);
+};
+
+class IonGetPropSuperIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ Register object_;
+ TypedOrValueRegister receiver_;
+ ConstantOrRegister id_;
+ ValueOperand output_;
+
+ public:
+ IonGetPropSuperIC(CacheKind kind, LiveRegisterSet liveRegs, Register object,
+ TypedOrValueRegister receiver, const ConstantOrRegister& id,
+ ValueOperand output)
+ : IonIC(kind),
+ liveRegs_(liveRegs),
+ object_(object),
+ receiver_(receiver),
+ id_(id),
+ output_(output) {}
+
+ Register object() const { return object_; }
+ TypedOrValueRegister receiver() const { return receiver_; }
+ ConstantOrRegister id() const { return id_; }
+ ValueOperand output() const { return output_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonGetPropSuperIC* ic, HandleObject obj,
+ HandleValue receiver, HandleValue idVal,
+ MutableHandleValue res);
+};
+
+class IonSetPropertyIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ Register object_;
+ Register temp_;
+ ConstantOrRegister id_;
+ ConstantOrRegister rhs_;
+ bool strict_ : 1;
+
+ public:
+ IonSetPropertyIC(CacheKind kind, LiveRegisterSet liveRegs, Register object,
+ Register temp, const ConstantOrRegister& id,
+ const ConstantOrRegister& rhs, bool strict)
+ : IonIC(kind),
+ liveRegs_(liveRegs),
+ object_(object),
+ temp_(temp),
+ id_(id),
+ rhs_(rhs),
+ strict_(strict) {}
+
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+ Register object() const { return object_; }
+ ConstantOrRegister id() const { return id_; }
+ ConstantOrRegister rhs() const { return rhs_; }
+
+ Register temp() const { return temp_; }
+
+ bool strict() const { return strict_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonSetPropertyIC* ic, HandleObject obj,
+ HandleValue idVal, HandleValue rhs);
+};
+
+class IonGetNameIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ Register environment_;
+ ValueOperand output_;
+ Register temp_;
+
+ public:
+ IonGetNameIC(LiveRegisterSet liveRegs, Register environment,
+ ValueOperand output, Register temp)
+ : IonIC(CacheKind::GetName),
+ liveRegs_(liveRegs),
+ environment_(environment),
+ output_(output),
+ temp_(temp) {}
+
+ Register environment() const { return environment_; }
+ ValueOperand output() const { return output_; }
+ Register temp() const { return temp_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonGetNameIC* ic, HandleObject envChain,
+ MutableHandleValue res);
+};
+
+class IonBindNameIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ Register environment_;
+ Register output_;
+ Register temp_;
+
+ public:
+ IonBindNameIC(LiveRegisterSet liveRegs, Register environment, Register output,
+ Register temp)
+ : IonIC(CacheKind::BindName),
+ liveRegs_(liveRegs),
+ environment_(environment),
+ output_(output),
+ temp_(temp) {}
+
+ Register environment() const { return environment_; }
+ Register output() const { return output_; }
+ Register temp() const { return temp_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ static JSObject* update(JSContext* cx, HandleScript outerScript,
+ IonBindNameIC* ic, HandleObject envChain);
+};
+
+class IonGetIteratorIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+ TypedOrValueRegister value_;
+ Register output_;
+ Register temp1_;
+ Register temp2_;
+
+ public:
+ IonGetIteratorIC(LiveRegisterSet liveRegs, TypedOrValueRegister value,
+ Register output, Register temp1, Register temp2)
+ : IonIC(CacheKind::GetIterator),
+ liveRegs_(liveRegs),
+ value_(value),
+ output_(output),
+ temp1_(temp1),
+ temp2_(temp2) {}
+
+ TypedOrValueRegister value() const { return value_; }
+ Register output() const { return output_; }
+ Register temp1() const { return temp1_; }
+ Register temp2() const { return temp2_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ static JSObject* update(JSContext* cx, HandleScript outerScript,
+ IonGetIteratorIC* ic, HandleValue value);
+};
+
+class IonOptimizeSpreadCallIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+ ValueOperand value_;
+ ValueOperand output_;
+ Register temp_;
+
+ public:
+ IonOptimizeSpreadCallIC(LiveRegisterSet liveRegs, ValueOperand value,
+ ValueOperand output, Register temp)
+ : IonIC(CacheKind::OptimizeSpreadCall),
+ liveRegs_(liveRegs),
+ value_(value),
+ output_(output),
+ temp_(temp) {}
+
+ ValueOperand value() const { return value_; }
+ ValueOperand output() const { return output_; }
+ Register temp() const { return temp_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ static bool update(JSContext* cx, HandleScript outerScript,
+ IonOptimizeSpreadCallIC* ic, HandleValue value,
+ MutableHandleValue result);
+};
+
+class IonHasOwnIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ TypedOrValueRegister value_;
+ TypedOrValueRegister id_;
+ Register output_;
+
+ public:
+ IonHasOwnIC(LiveRegisterSet liveRegs, TypedOrValueRegister value,
+ TypedOrValueRegister id, Register output)
+ : IonIC(CacheKind::HasOwn),
+ liveRegs_(liveRegs),
+ value_(value),
+ id_(id),
+ output_(output) {}
+
+ TypedOrValueRegister value() const { return value_; }
+ TypedOrValueRegister id() const { return id_; }
+ Register output() const { return output_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonHasOwnIC* ic, HandleValue val,
+ HandleValue idVal, int32_t* res);
+};
+
+class IonCheckPrivateFieldIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ TypedOrValueRegister value_;
+ TypedOrValueRegister id_;
+ Register output_;
+
+ public:
+ IonCheckPrivateFieldIC(LiveRegisterSet liveRegs, TypedOrValueRegister value,
+ TypedOrValueRegister id, Register output)
+ : IonIC(CacheKind::CheckPrivateField),
+ liveRegs_(liveRegs),
+ value_(value),
+ id_(id),
+ output_(output) {}
+
+ TypedOrValueRegister value() const { return value_; }
+ TypedOrValueRegister id() const { return id_; }
+ Register output() const { return output_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonCheckPrivateFieldIC* ic, HandleValue val,
+ HandleValue idVal, bool* res);
+};
+
+class IonInIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ ConstantOrRegister key_;
+ Register object_;
+ Register output_;
+ Register temp_;
+
+ public:
+ IonInIC(LiveRegisterSet liveRegs, const ConstantOrRegister& key,
+ Register object, Register output, Register temp)
+ : IonIC(CacheKind::In),
+ liveRegs_(liveRegs),
+ key_(key),
+ object_(object),
+ output_(output),
+ temp_(temp) {}
+
+ ConstantOrRegister key() const { return key_; }
+ Register object() const { return object_; }
+ Register output() const { return output_; }
+ Register temp() const { return temp_; }
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonInIC* ic, HandleValue key,
+ HandleObject obj, bool* res);
+};
+
+class IonInstanceOfIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ TypedOrValueRegister lhs_;
+ Register rhs_;
+ Register output_;
+
+ public:
+ IonInstanceOfIC(LiveRegisterSet liveRegs, TypedOrValueRegister lhs,
+ Register rhs, Register output)
+ : IonIC(CacheKind::InstanceOf),
+ liveRegs_(liveRegs),
+ lhs_(lhs),
+ rhs_(rhs),
+ output_(output) {}
+
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+ TypedOrValueRegister lhs() const { return lhs_; }
+ Register rhs() const { return rhs_; }
+ Register output() const { return output_; }
+
+ // This signature mimics that of TryAttachInstanceOfStub in baseline
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonInstanceOfIC* ic, HandleValue lhs,
+ HandleObject rhs, bool* attached);
+};
+
+class IonCompareIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ TypedOrValueRegister lhs_;
+ TypedOrValueRegister rhs_;
+ Register output_;
+
+ public:
+ IonCompareIC(LiveRegisterSet liveRegs, TypedOrValueRegister lhs,
+ TypedOrValueRegister rhs, Register output)
+ : IonIC(CacheKind::Compare),
+ liveRegs_(liveRegs),
+ lhs_(lhs),
+ rhs_(rhs),
+ output_(output) {}
+
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+ TypedOrValueRegister lhs() const { return lhs_; }
+ TypedOrValueRegister rhs() const { return rhs_; }
+ Register output() const { return output_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonCompareIC* stub, HandleValue lhs,
+ HandleValue rhs, bool* res);
+};
+
+class IonUnaryArithIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ TypedOrValueRegister input_;
+ ValueOperand output_;
+
+ public:
+ IonUnaryArithIC(LiveRegisterSet liveRegs, TypedOrValueRegister input,
+ ValueOperand output)
+ : IonIC(CacheKind::UnaryArith),
+ liveRegs_(liveRegs),
+ input_(input),
+ output_(output) {}
+
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+ TypedOrValueRegister input() const { return input_; }
+ ValueOperand output() const { return output_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonUnaryArithIC* stub, HandleValue val,
+ MutableHandleValue res);
+};
+
+class IonToPropertyKeyIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+ ValueOperand input_;
+ ValueOperand output_;
+
+ public:
+ IonToPropertyKeyIC(LiveRegisterSet liveRegs, ValueOperand input,
+ ValueOperand output)
+ : IonIC(CacheKind::ToPropertyKey),
+ liveRegs_(liveRegs),
+ input_(input),
+ output_(output) {}
+
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+ ValueOperand input() const { return input_; }
+ ValueOperand output() const { return output_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonToPropertyKeyIC* ic, HandleValue val,
+ MutableHandleValue res);
+};
+
+class IonBinaryArithIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ TypedOrValueRegister lhs_;
+ TypedOrValueRegister rhs_;
+ ValueOperand output_;
+
+ public:
+ IonBinaryArithIC(LiveRegisterSet liveRegs, TypedOrValueRegister lhs,
+ TypedOrValueRegister rhs, ValueOperand output)
+ : IonIC(CacheKind::BinaryArith),
+ liveRegs_(liveRegs),
+ lhs_(lhs),
+ rhs_(rhs),
+ output_(output) {}
+
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+ TypedOrValueRegister lhs() const { return lhs_; }
+ TypedOrValueRegister rhs() const { return rhs_; }
+ ValueOperand output() const { return output_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonBinaryArithIC* stub, HandleValue lhs,
+ HandleValue rhs, MutableHandleValue res);
+};
+
+class IonCloseIterIC : public IonIC {
+ LiveRegisterSet liveRegs_;
+
+ Register iter_;
+ Register temp_;
+ CompletionKind completionKind_;
+
+ public:
+ IonCloseIterIC(LiveRegisterSet liveRegs, Register iter, Register temp,
+ CompletionKind completionKind)
+ : IonIC(CacheKind::CloseIter),
+ liveRegs_(liveRegs),
+ iter_(iter),
+ temp_(temp),
+ completionKind_(completionKind) {}
+
+ LiveRegisterSet liveRegs() const { return liveRegs_; }
+ Register temp() const { return temp_; }
+ Register iter() const { return iter_; }
+ CompletionKind completionKind() const { return completionKind_; }
+
+ [[nodiscard]] static bool update(JSContext* cx, HandleScript outerScript,
+ IonCloseIterIC* ic, HandleObject iter);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_IonIC_h */
diff --git a/js/src/jit/IonOptimizationLevels.cpp b/js/src/jit/IonOptimizationLevels.cpp
new file mode 100644
index 0000000000..92e3da1bc3
--- /dev/null
+++ b/js/src/jit/IonOptimizationLevels.cpp
@@ -0,0 +1,139 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/IonOptimizationLevels.h"
+
+#include "jit/Ion.h"
+#include "vm/JSScript.h"
+
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+namespace js {
+namespace jit {
+
+const OptimizationLevelInfo IonOptimizations;
+
+void OptimizationInfo::initNormalOptimizationInfo() {
+ level_ = OptimizationLevel::Normal;
+
+ autoTruncate_ = true;
+ eaa_ = true;
+ edgeCaseAnalysis_ = true;
+ eliminateRedundantChecks_ = true;
+ eliminateRedundantShapeGuards_ = true;
+ inlineInterpreted_ = true;
+ inlineNative_ = true;
+ licm_ = true;
+ gvn_ = true;
+ rangeAnalysis_ = true;
+ reordering_ = true;
+ scalarReplacement_ = true;
+ sink_ = true;
+
+ registerAllocator_ = RegisterAllocator_Backtracking;
+}
+
+void OptimizationInfo::initWasmOptimizationInfo() {
+ // The Wasm optimization level
+ // Disables some passes that don't work well with wasm.
+
+ // Take normal option values for not specified values.
+ initNormalOptimizationInfo();
+
+ level_ = OptimizationLevel::Wasm;
+
+ ama_ = true;
+ autoTruncate_ = false;
+ edgeCaseAnalysis_ = false;
+ eliminateRedundantChecks_ = false;
+ eliminateRedundantShapeGuards_ = false;
+ scalarReplacement_ = false; // wasm has no objects.
+ sink_ = false;
+}
+
+uint32_t OptimizationInfo::compilerWarmUpThreshold(JSScript* script,
+ jsbytecode* pc) const {
+ MOZ_ASSERT(pc == nullptr || pc == script->code() ||
+ JSOp(*pc) == JSOp::LoopHead);
+
+ // The script must not start with a LoopHead op or the code below would be
+ // wrong. See bug 1602681.
+ MOZ_ASSERT_IF(pc && JSOp(*pc) == JSOp::LoopHead, pc > script->code());
+
+ if (pc == script->code()) {
+ pc = nullptr;
+ }
+
+ uint32_t warmUpThreshold = baseCompilerWarmUpThreshold();
+
+ // If the script is too large to compile on the main thread, we can still
+ // compile it off thread. In these cases, increase the warm-up counter
+ // threshold to improve the compilation's type information and hopefully
+ // avoid later recompilation.
+
+ if (script->length() > JitOptions.ionMaxScriptSizeMainThread) {
+ warmUpThreshold *=
+ (script->length() / double(JitOptions.ionMaxScriptSizeMainThread));
+ }
+
+ uint32_t numLocalsAndArgs = NumLocalsAndArgs(script);
+ if (numLocalsAndArgs > JitOptions.ionMaxLocalsAndArgsMainThread) {
+ warmUpThreshold *=
+ (numLocalsAndArgs / double(JitOptions.ionMaxLocalsAndArgsMainThread));
+ }
+
+ if (!pc || JitOptions.eagerIonCompilation()) {
+ return warmUpThreshold;
+ }
+
+ // It's more efficient to enter outer loops, rather than inner loops, via OSR.
+ // To accomplish this, we use a slightly higher threshold for inner loops.
+ // Note that the loop depth is always > 0 so we will prefer non-OSR over OSR.
+ uint32_t loopDepth = LoopHeadDepthHint(pc);
+ MOZ_ASSERT(loopDepth > 0);
+ return warmUpThreshold + loopDepth * (baseCompilerWarmUpThreshold() / 10);
+}
+
+uint32_t OptimizationInfo::recompileWarmUpThreshold(JSScript* script,
+ jsbytecode* pc) const {
+ MOZ_ASSERT(pc == script->code() || JSOp(*pc) == JSOp::LoopHead);
+
+ uint32_t threshold = compilerWarmUpThreshold(script, pc);
+ if (JSOp(*pc) != JSOp::LoopHead || JitOptions.eagerIonCompilation()) {
+ return threshold;
+ }
+
+ // If we're stuck in a long-running loop at a low optimization level, we have
+ // to invalidate to be able to tier up. This is worse than recompiling at
+ // function entry (because in that case we can use the lazy link mechanism and
+ // avoid invalidation completely). Use a very high recompilation threshold for
+ // loop edges so that this only affects very long-running loops.
+
+ uint32_t loopDepth = LoopHeadDepthHint(pc);
+ MOZ_ASSERT(loopDepth > 0);
+ return threshold + loopDepth * (baseCompilerWarmUpThreshold() / 10);
+}
+
+OptimizationLevelInfo::OptimizationLevelInfo() {
+ infos_[OptimizationLevel::Normal].initNormalOptimizationInfo();
+ infos_[OptimizationLevel::Wasm].initWasmOptimizationInfo();
+}
+
+OptimizationLevel OptimizationLevelInfo::levelForScript(JSScript* script,
+ jsbytecode* pc) const {
+ const OptimizationInfo* info = get(OptimizationLevel::Normal);
+ if (script->getWarmUpCount() < info->compilerWarmUpThreshold(script, pc)) {
+ return OptimizationLevel::DontCompile;
+ }
+
+ return OptimizationLevel::Normal;
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/IonOptimizationLevels.h b/js/src/jit/IonOptimizationLevels.h
new file mode 100644
index 0000000000..e39e5d5026
--- /dev/null
+++ b/js/src/jit/IonOptimizationLevels.h
@@ -0,0 +1,194 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_IonOptimizationLevels_h
+#define jit_IonOptimizationLevels_h
+
+#include "mozilla/EnumeratedArray.h"
+
+#include "jstypes.h"
+
+#include "jit/JitOptions.h"
+#include "js/TypeDecls.h"
+
+namespace js {
+namespace jit {
+
+enum class OptimizationLevel : uint8_t { Normal, Wasm, Count, DontCompile };
+
+#ifdef JS_JITSPEW
+inline const char* OptimizationLevelString(OptimizationLevel level) {
+ switch (level) {
+ case OptimizationLevel::DontCompile:
+ return "Optimization_DontCompile";
+ case OptimizationLevel::Normal:
+ return "Optimization_Normal";
+ case OptimizationLevel::Wasm:
+ return "Optimization_Wasm";
+ case OptimizationLevel::Count:;
+ }
+ MOZ_CRASH("Invalid OptimizationLevel");
+}
+#endif
+
+// Class representing the Ion optimization settings for an OptimizationLevel.
+class OptimizationInfo {
+ OptimizationLevel level_;
+
+ // Toggles whether Effective Address Analysis is performed.
+ bool eaa_;
+
+ // Toggles whether Alignment Mask Analysis is performed.
+ bool ama_;
+
+ // Toggles whether Edge Case Analysis is used.
+ bool edgeCaseAnalysis_;
+
+ // Toggles whether redundant checks get removed.
+ bool eliminateRedundantChecks_;
+
+ // Toggles whether redundant shape guards get removed.
+ bool eliminateRedundantShapeGuards_;
+
+ // Toggles whether interpreted scripts get inlined.
+ bool inlineInterpreted_;
+
+ // Toggles whether native scripts get inlined.
+ bool inlineNative_;
+
+ // Toggles whether global value numbering is used.
+ bool gvn_;
+
+ // Toggles whether loop invariant code motion is performed.
+ bool licm_;
+
+ // Toggles whether Range Analysis is used.
+ bool rangeAnalysis_;
+
+ // Toggles whether instruction reordering is performed.
+ bool reordering_;
+
+ // Toggles whether Truncation based on Range Analysis is used.
+ bool autoTruncate_;
+
+ // Toggles whether sink is used.
+ bool sink_;
+
+ // Toggles whether scalar replacement is used.
+ bool scalarReplacement_;
+
+ // Describes which register allocator to use.
+ IonRegisterAllocator registerAllocator_;
+
+ uint32_t baseCompilerWarmUpThreshold() const {
+ MOZ_ASSERT(level_ == OptimizationLevel::Normal);
+ return JitOptions.normalIonWarmUpThreshold;
+ }
+
+ public:
+ constexpr OptimizationInfo()
+ : level_(OptimizationLevel::Normal),
+ eaa_(false),
+ ama_(false),
+ edgeCaseAnalysis_(false),
+ eliminateRedundantChecks_(false),
+ eliminateRedundantShapeGuards_(false),
+ inlineInterpreted_(false),
+ inlineNative_(false),
+ gvn_(false),
+ licm_(false),
+ rangeAnalysis_(false),
+ reordering_(false),
+ autoTruncate_(false),
+ sink_(false),
+ scalarReplacement_(false),
+ registerAllocator_(RegisterAllocator_Backtracking) {}
+
+ void initNormalOptimizationInfo();
+ void initWasmOptimizationInfo();
+
+ OptimizationLevel level() const { return level_; }
+
+ bool inlineInterpreted() const {
+ return inlineInterpreted_ && !JitOptions.disableInlining;
+ }
+
+ bool inlineNative() const {
+ return inlineNative_ && !JitOptions.disableInlining;
+ }
+
+ uint32_t compilerWarmUpThreshold(JSScript* script,
+ jsbytecode* pc = nullptr) const;
+
+ uint32_t recompileWarmUpThreshold(JSScript* script, jsbytecode* pc) const;
+
+ bool gvnEnabled() const { return gvn_ && !JitOptions.disableGvn; }
+
+ bool licmEnabled() const { return licm_ && !JitOptions.disableLicm; }
+
+ bool rangeAnalysisEnabled() const {
+ return rangeAnalysis_ && !JitOptions.disableRangeAnalysis;
+ }
+
+ bool instructionReorderingEnabled() const {
+ return reordering_ && !JitOptions.disableInstructionReordering;
+ }
+
+ bool autoTruncateEnabled() const {
+ return autoTruncate_ && rangeAnalysisEnabled();
+ }
+
+ bool sinkEnabled() const { return sink_ && !JitOptions.disableSink; }
+
+ bool eaaEnabled() const { return eaa_ && !JitOptions.disableEaa; }
+
+ bool amaEnabled() const { return ama_ && !JitOptions.disableAma; }
+
+ bool edgeCaseAnalysisEnabled() const {
+ return edgeCaseAnalysis_ && !JitOptions.disableEdgeCaseAnalysis;
+ }
+
+ bool eliminateRedundantChecksEnabled() const {
+ return eliminateRedundantChecks_;
+ }
+
+ bool eliminateRedundantShapeGuardsEnabled() const {
+ return eliminateRedundantShapeGuards_ &&
+ !JitOptions.disableRedundantShapeGuards;
+ }
+
+ IonRegisterAllocator registerAllocator() const {
+ return JitOptions.forcedRegisterAllocator.valueOr(registerAllocator_);
+ }
+
+ bool scalarReplacementEnabled() const {
+ return scalarReplacement_ && !JitOptions.disableScalarReplacement;
+ }
+};
+
+class OptimizationLevelInfo {
+ private:
+ mozilla::EnumeratedArray<OptimizationLevel, OptimizationLevel::Count,
+ OptimizationInfo>
+ infos_;
+
+ public:
+ OptimizationLevelInfo();
+
+ const OptimizationInfo* get(OptimizationLevel level) const {
+ return &infos_[level];
+ }
+
+ OptimizationLevel levelForScript(JSScript* script,
+ jsbytecode* pc = nullptr) const;
+};
+
+extern const OptimizationLevelInfo IonOptimizations;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_IonOptimizationLevels_h */
diff --git a/js/src/jit/IonScript.h b/js/src/jit/IonScript.h
new file mode 100644
index 0000000000..c1f7a4810a
--- /dev/null
+++ b/js/src/jit/IonScript.h
@@ -0,0 +1,590 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_IonScript_h
+#define jit_IonScript_h
+
+#include "mozilla/MemoryReporting.h" // MallocSizeOf
+
+#include <stddef.h> // size_t
+#include <stdint.h> // uint8_t, uint32_t
+
+#include "jstypes.h"
+
+#include "gc/Barrier.h" // HeapPtr{JitCode,Object}
+#include "jit/IonTypes.h" // IonCompilationId
+#include "jit/JitCode.h" // JitCode
+#include "jit/JitOptions.h" // JitOptions
+#include "js/TypeDecls.h" // jsbytecode
+#include "util/TrailingArray.h" // TrailingArray
+
+namespace js {
+namespace jit {
+
+class SnapshotWriter;
+class RecoverWriter;
+class SafepointWriter;
+class CodegenSafepointIndex;
+class SafepointIndex;
+class OsiIndex;
+class IonIC;
+
+// An IonScript attaches Ion-generated information to a JSScript. The header
+// structure is followed by several arrays of data. These trailing arrays have a
+// layout based on offsets (bytes from 'this') stored in the IonScript header.
+//
+// <IonScript itself>
+// --
+// PreBarriered<Value>[] constantTable()
+// uint8_t[] runtimeData()
+// OsiIndex[] osiIndex()
+// SafepointIndex[] safepointIndex()
+// uint32_t[] icIndex()
+// --
+// uint8_t[] safepoints()
+// uint8_t[] snapshots()
+// uint8_t[] snapshotsRVATable()
+// uint8_t[] recovers()
+//
+// Note: These are arranged in order of descending alignment requirements to
+// avoid the need for padding. The `runtimeData` uses uint64_t alignement due to
+// its use of mozilla::AlignedStorage2.
+class alignas(8) IonScript final : public TrailingArray {
+ private:
+ // Offset (in bytes) from `this` to the start of each trailing array. Each
+ // array ends where following one begins. There is no implicit padding (except
+ // possible at very end).
+ Offset constantTableOffset_ = 0; // JS::Value aligned
+ Offset runtimeDataOffset_ = 0; // uint64_t aligned
+ Offset nurseryObjectsOffset_ = 0; // pointer aligned
+ Offset osiIndexOffset_ = 0;
+ Offset safepointIndexOffset_ = 0;
+ Offset icIndexOffset_ = 0;
+ Offset safepointsOffset_ = 0;
+ Offset snapshotsOffset_ = 0;
+ Offset rvaTableOffset_ = 0;
+ Offset recoversOffset_ = 0;
+ Offset allocBytes_ = 0;
+
+ // Code pointer containing the actual method.
+ HeapPtr<JitCode*> method_ = nullptr;
+
+ // Entrypoint for OSR, or nullptr.
+ jsbytecode* osrPc_ = nullptr;
+
+ // Offset to OSR entrypoint from method_->raw(), or 0.
+ uint32_t osrEntryOffset_ = 0;
+
+ // Offset of the invalidation epilogue (which pushes this IonScript
+ // and calls the invalidation thunk).
+ uint32_t invalidateEpilogueOffset_ = 0;
+
+ // The offset immediately after the IonScript immediate.
+ // NOTE: technically a constant delta from
+ // |invalidateEpilogueOffset_|, so we could hard-code this
+ // per-platform if we want.
+ uint32_t invalidateEpilogueDataOffset_ = 0;
+
+ // Number of bailouts that have occurred for reasons that could be
+ // fixed if we invalidated and recompiled.
+ uint16_t numFixableBailouts_ = 0;
+
+ // Number of bailouts that have occurred for reasons that can't be
+ // fixed by recompiling: for example, bailing out to catch an exception.
+ uint16_t numUnfixableBailouts_ = 0;
+
+ public:
+ enum class LICMState : uint8_t { NeverBailed, Bailed, BailedAndHitFallback };
+
+ private:
+ // Tracks the state of LICM bailouts.
+ LICMState licmState_ = LICMState::NeverBailed;
+
+ // Flag set if IonScript was compiled with profiling enabled.
+ bool hasProfilingInstrumentation_ = false;
+
+ // Number of bytes this function reserves on the stack for slots spilled by
+ // the register allocator.
+ uint32_t localSlotsSize_ = 0;
+
+ // Number of bytes used passed in as formal arguments or |this|.
+ uint32_t argumentSlotsSize_ = 0;
+
+ // Frame size is the value that can be added to the StackPointer along
+ // with the frame prefix to get a valid JitFrameLayout.
+ uint32_t frameSize_ = 0;
+
+ // Number of references from invalidation records.
+ uint32_t invalidationCount_ = 0;
+
+ // Identifier of the compilation which produced this code.
+ IonCompilationId compilationId_;
+
+ // Number of times we tried to enter this script via OSR but failed due to
+ // a LOOPENTRY pc other than osrPc_.
+ uint32_t osrPcMismatchCounter_ = 0;
+
+#ifdef DEBUG
+ // A hash of the ICScripts used in this compilation.
+ mozilla::HashNumber icHash_ = 0;
+#endif
+
+ // End of fields.
+
+ private:
+ // Layout helpers
+ Offset constantTableOffset() const { return constantTableOffset_; }
+ Offset runtimeDataOffset() const { return runtimeDataOffset_; }
+ Offset nurseryObjectsOffset() const { return nurseryObjectsOffset_; }
+ Offset osiIndexOffset() const { return osiIndexOffset_; }
+ Offset safepointIndexOffset() const { return safepointIndexOffset_; }
+ Offset icIndexOffset() const { return icIndexOffset_; }
+ Offset safepointsOffset() const { return safepointsOffset_; }
+ Offset snapshotsOffset() const { return snapshotsOffset_; }
+ Offset rvaTableOffset() const { return rvaTableOffset_; }
+ Offset recoversOffset() const { return recoversOffset_; }
+ Offset endOffset() const { return allocBytes_; }
+
+ // Hardcode size of incomplete types. These are verified in Ion.cpp.
+ static constexpr size_t SizeOf_OsiIndex = 2 * sizeof(uint32_t);
+ static constexpr size_t SizeOf_SafepointIndex = 2 * sizeof(uint32_t);
+
+ public:
+ //
+ // Table of constants referenced in snapshots. (JS::Value alignment)
+ //
+ PreBarriered<Value>* constants() {
+ // Nursery constants are manually barriered in CodeGenerator::link() so a
+ // post barrier is not required..
+ return offsetToPointer<PreBarriered<Value>>(constantTableOffset());
+ }
+ size_t numConstants() const {
+ return numElements<PreBarriered<Value>>(constantTableOffset(),
+ runtimeDataOffset());
+ }
+
+ //
+ // IonIC data structures. (uint64_t alignment)
+ //
+ uint8_t* runtimeData() {
+ return offsetToPointer<uint8_t>(runtimeDataOffset());
+ }
+ size_t runtimeSize() const {
+ return numElements<uint8_t>(runtimeDataOffset(), nurseryObjectsOffset());
+ }
+
+ //
+ // List of (originally) nursery-allocated objects referenced from JIT code.
+ // (JSObject* alignment)
+ //
+ HeapPtr<JSObject*>* nurseryObjects() {
+ return offsetToPointer<HeapPtr<JSObject*>>(nurseryObjectsOffset());
+ }
+ size_t numNurseryObjects() const {
+ return numElements<HeapPtr<JSObject*>>(nurseryObjectsOffset(),
+ osiIndexOffset());
+ }
+ void* addressOfNurseryObject(uint32_t index) {
+ MOZ_ASSERT(index < numNurseryObjects());
+ return &nurseryObjects()[index];
+ }
+
+ //
+ // Map OSI-point displacement to snapshot.
+ //
+ OsiIndex* osiIndices() { return offsetToPointer<OsiIndex>(osiIndexOffset()); }
+ const OsiIndex* osiIndices() const {
+ return offsetToPointer<OsiIndex>(osiIndexOffset());
+ }
+ size_t numOsiIndices() const {
+ return numElements<SizeOf_OsiIndex>(osiIndexOffset(),
+ safepointIndexOffset());
+ }
+
+ //
+ // Map code displacement to safepoint / OSI-patch-delta.
+ //
+ SafepointIndex* safepointIndices() {
+ return offsetToPointer<SafepointIndex>(safepointIndexOffset());
+ }
+ const SafepointIndex* safepointIndices() const {
+ return offsetToPointer<SafepointIndex>(safepointIndexOffset());
+ }
+ size_t numSafepointIndices() const {
+ return numElements<SizeOf_SafepointIndex>(safepointIndexOffset(),
+ icIndexOffset());
+ }
+
+ //
+ // Offset into `runtimeData` for each (variable-length) IonIC.
+ //
+ uint32_t* icIndex() { return offsetToPointer<uint32_t>(icIndexOffset()); }
+ size_t numICs() const {
+ return numElements<uint32_t>(icIndexOffset(), safepointsOffset());
+ }
+
+ //
+ // Safepoint table as a CompactBuffer.
+ //
+ const uint8_t* safepoints() const {
+ return offsetToPointer<uint8_t>(safepointsOffset());
+ }
+ size_t safepointsSize() const {
+ return numElements<uint8_t>(safepointsOffset(), snapshotsOffset());
+ }
+
+ //
+ // Snapshot and RValueAllocation tables as CompactBuffers.
+ //
+ const uint8_t* snapshots() const {
+ return offsetToPointer<uint8_t>(snapshotsOffset());
+ }
+ size_t snapshotsListSize() const {
+ return numElements<uint8_t>(snapshotsOffset(), rvaTableOffset());
+ }
+ size_t snapshotsRVATableSize() const {
+ return numElements<uint8_t>(rvaTableOffset(), recoversOffset());
+ }
+
+ //
+ // Recover instruction table as a CompactBuffer.
+ //
+ const uint8_t* recovers() const {
+ return offsetToPointer<uint8_t>(recoversOffset());
+ }
+ size_t recoversSize() const {
+ return numElements<uint8_t>(recoversOffset(), endOffset());
+ }
+
+ private:
+ IonScript(IonCompilationId compilationId, uint32_t localSlotsSize,
+ uint32_t argumentSlotsSize, uint32_t frameSize);
+
+ public:
+ static IonScript* New(JSContext* cx, IonCompilationId compilationId,
+ uint32_t localSlotsSize, uint32_t argumentSlotsSize,
+ uint32_t frameSize, size_t snapshotsListSize,
+ size_t snapshotsRVATableSize, size_t recoversSize,
+ size_t constants, size_t nurseryObjects,
+ size_t safepointIndices, size_t osiIndices,
+ size_t icEntries, size_t runtimeSize,
+ size_t safepointsSize);
+
+ static void Destroy(JS::GCContext* gcx, IonScript* script);
+
+ void trace(JSTracer* trc);
+
+ static inline size_t offsetOfInvalidationCount() {
+ return offsetof(IonScript, invalidationCount_);
+ }
+
+ public:
+ JitCode* method() const { return method_; }
+ void setMethod(JitCode* code) {
+ MOZ_ASSERT(!invalidated());
+ method_ = code;
+ }
+ void setOsrPc(jsbytecode* osrPc) { osrPc_ = osrPc; }
+ jsbytecode* osrPc() const { return osrPc_; }
+ void setOsrEntryOffset(uint32_t offset) {
+ MOZ_ASSERT(!osrEntryOffset_);
+ osrEntryOffset_ = offset;
+ }
+ uint32_t osrEntryOffset() const { return osrEntryOffset_; }
+ bool containsCodeAddress(uint8_t* addr) const {
+ return method()->raw() <= addr &&
+ addr <= method()->raw() + method()->instructionsSize();
+ }
+ bool containsReturnAddress(uint8_t* addr) const {
+ // This accounts for an off by one error caused by the return address of a
+ // bailout sitting outside the range of the containing function.
+ return method()->raw() <= addr &&
+ addr <= method()->raw() + method()->instructionsSize();
+ }
+ void setInvalidationEpilogueOffset(uint32_t offset) {
+ MOZ_ASSERT(!invalidateEpilogueOffset_);
+ invalidateEpilogueOffset_ = offset;
+ }
+ uint32_t invalidateEpilogueOffset() const {
+ MOZ_ASSERT(invalidateEpilogueOffset_);
+ return invalidateEpilogueOffset_;
+ }
+ void setInvalidationEpilogueDataOffset(uint32_t offset) {
+ MOZ_ASSERT(!invalidateEpilogueDataOffset_);
+ invalidateEpilogueDataOffset_ = offset;
+ }
+ uint32_t invalidateEpilogueDataOffset() const {
+ MOZ_ASSERT(invalidateEpilogueDataOffset_);
+ return invalidateEpilogueDataOffset_;
+ }
+
+ void incNumFixableBailouts() { numFixableBailouts_++; }
+ void resetNumFixableBailouts() { numFixableBailouts_ = 0; }
+ void incNumUnfixableBailouts() { numUnfixableBailouts_++; }
+
+ bool shouldInvalidate() const {
+ return numFixableBailouts_ >= JitOptions.frequentBailoutThreshold;
+ }
+ bool shouldInvalidateAndDisable() const {
+ return numUnfixableBailouts_ >= JitOptions.frequentBailoutThreshold * 5;
+ }
+
+ LICMState licmState() const { return licmState_; }
+ void setHadLICMBailout() {
+ if (licmState_ == LICMState::NeverBailed) {
+ licmState_ = LICMState::Bailed;
+ }
+ }
+ void noteBaselineFallback() {
+ if (licmState_ == LICMState::Bailed) {
+ licmState_ = LICMState::BailedAndHitFallback;
+ }
+ }
+
+ void setHasProfilingInstrumentation() { hasProfilingInstrumentation_ = true; }
+ void clearHasProfilingInstrumentation() {
+ hasProfilingInstrumentation_ = false;
+ }
+ bool hasProfilingInstrumentation() const {
+ return hasProfilingInstrumentation_;
+ }
+
+ size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ return mallocSizeOf(this);
+ }
+ PreBarriered<Value>& getConstant(size_t index) {
+ MOZ_ASSERT(index < numConstants());
+ return constants()[index];
+ }
+ uint32_t localSlotsSize() const { return localSlotsSize_; }
+ uint32_t argumentSlotsSize() const { return argumentSlotsSize_; }
+ uint32_t frameSize() const { return frameSize_; }
+ const SafepointIndex* getSafepointIndex(uint32_t disp) const;
+ const SafepointIndex* getSafepointIndex(uint8_t* retAddr) const {
+ MOZ_ASSERT(containsCodeAddress(retAddr));
+ return getSafepointIndex(retAddr - method()->raw());
+ }
+ const OsiIndex* getOsiIndex(uint32_t disp) const;
+ const OsiIndex* getOsiIndex(uint8_t* retAddr) const;
+
+ IonIC& getICFromIndex(uint32_t index) {
+ MOZ_ASSERT(index < numICs());
+ uint32_t offset = icIndex()[index];
+ return getIC(offset);
+ }
+ inline IonIC& getIC(uint32_t offset) {
+ MOZ_ASSERT(offset < runtimeSize());
+ return *reinterpret_cast<IonIC*>(runtimeData() + offset);
+ }
+ void purgeICs(Zone* zone);
+ void copySnapshots(const SnapshotWriter* writer);
+ void copyRecovers(const RecoverWriter* writer);
+ void copyConstants(const Value* vp);
+ void copySafepointIndices(const CodegenSafepointIndex* si);
+ void copyOsiIndices(const OsiIndex* oi);
+ void copyRuntimeData(const uint8_t* data);
+ void copyICEntries(const uint32_t* icEntries);
+ void copySafepoints(const SafepointWriter* writer);
+
+ bool invalidated() const { return invalidationCount_ != 0; }
+
+ // Invalidate the current compilation.
+ void invalidate(JSContext* cx, JSScript* script, bool resetUses,
+ const char* reason);
+
+ size_t invalidationCount() const { return invalidationCount_; }
+ void incrementInvalidationCount() { invalidationCount_++; }
+ void decrementInvalidationCount(JS::GCContext* gcx) {
+ MOZ_ASSERT(invalidationCount_);
+ invalidationCount_--;
+ if (!invalidationCount_) {
+ Destroy(gcx, this);
+ }
+ }
+ IonCompilationId compilationId() const { return compilationId_; }
+ uint32_t incrOsrPcMismatchCounter() { return ++osrPcMismatchCounter_; }
+ void resetOsrPcMismatchCounter() { osrPcMismatchCounter_ = 0; }
+
+ size_t allocBytes() const { return allocBytes_; }
+
+ static void preWriteBarrier(Zone* zone, IonScript* ionScript);
+
+#ifdef DEBUG
+ mozilla::HashNumber icHash() const { return icHash_; }
+ void setICHash(mozilla::HashNumber hash) { icHash_ = hash; }
+#endif
+};
+
+// Execution information for a basic block which may persist after the
+// accompanying IonScript is destroyed, for use during profiling.
+struct IonBlockCounts {
+ private:
+ uint32_t id_;
+
+ // Approximate bytecode in the outer (not inlined) script this block
+ // was generated from.
+ uint32_t offset_;
+
+ // File and line of the inner script this block was generated from.
+ char* description_;
+
+ // ids for successors of this block.
+ uint32_t numSuccessors_;
+ uint32_t* successors_;
+
+ // Hit count for this block.
+ uint64_t hitCount_;
+
+ // Text information about the code generated for this block.
+ char* code_;
+
+ public:
+ [[nodiscard]] bool init(uint32_t id, uint32_t offset, char* description,
+ uint32_t numSuccessors) {
+ id_ = id;
+ offset_ = offset;
+ description_ = description;
+ numSuccessors_ = numSuccessors;
+ if (numSuccessors) {
+ successors_ = js_pod_calloc<uint32_t>(numSuccessors);
+ if (!successors_) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ void destroy() {
+ js_free(description_);
+ js_free(successors_);
+ js_free(code_);
+ }
+
+ uint32_t id() const { return id_; }
+
+ uint32_t offset() const { return offset_; }
+
+ const char* description() const { return description_; }
+
+ size_t numSuccessors() const { return numSuccessors_; }
+
+ void setSuccessor(size_t i, uint32_t id) {
+ MOZ_ASSERT(i < numSuccessors_);
+ successors_[i] = id;
+ }
+
+ uint32_t successor(size_t i) const {
+ MOZ_ASSERT(i < numSuccessors_);
+ return successors_[i];
+ }
+
+ uint64_t* addressOfHitCount() { return &hitCount_; }
+
+ uint64_t hitCount() const { return hitCount_; }
+
+ void setCode(const char* code) {
+ char* ncode = js_pod_malloc<char>(strlen(code) + 1);
+ if (ncode) {
+ strcpy(ncode, code);
+ code_ = ncode;
+ }
+ }
+
+ const char* code() const { return code_; }
+
+ size_t sizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ return mallocSizeOf(description_) + mallocSizeOf(successors_) +
+ mallocSizeOf(code_);
+ }
+};
+
+// Execution information for a compiled script which may persist after the
+// IonScript is destroyed, for use during profiling.
+struct IonScriptCounts {
+ private:
+ // Any previous invalidated compilation(s) for the script.
+ IonScriptCounts* previous_ = nullptr;
+
+ // Information about basic blocks in this script.
+ size_t numBlocks_ = 0;
+ IonBlockCounts* blocks_ = nullptr;
+
+ public:
+ IonScriptCounts() = default;
+
+ ~IonScriptCounts() {
+ for (size_t i = 0; i < numBlocks_; i++) {
+ blocks_[i].destroy();
+ }
+ js_free(blocks_);
+ // The list can be long in some corner cases (bug 1140084), so
+ // unroll the recursion.
+ IonScriptCounts* victims = previous_;
+ while (victims) {
+ IonScriptCounts* victim = victims;
+ victims = victim->previous_;
+ victim->previous_ = nullptr;
+ js_delete(victim);
+ }
+ }
+
+ [[nodiscard]] bool init(size_t numBlocks) {
+ blocks_ = js_pod_calloc<IonBlockCounts>(numBlocks);
+ if (!blocks_) {
+ return false;
+ }
+
+ numBlocks_ = numBlocks;
+ return true;
+ }
+
+ size_t numBlocks() const { return numBlocks_; }
+
+ IonBlockCounts& block(size_t i) {
+ MOZ_ASSERT(i < numBlocks_);
+ return blocks_[i];
+ }
+
+ void setPrevious(IonScriptCounts* previous) { previous_ = previous; }
+
+ IonScriptCounts* previous() const { return previous_; }
+
+ size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ size_t size = 0;
+ auto currCounts = this;
+ do {
+ size += currCounts->sizeOfOneIncludingThis(mallocSizeOf);
+ currCounts = currCounts->previous_;
+ } while (currCounts);
+ return size;
+ }
+
+ size_t sizeOfOneIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const {
+ size_t size = mallocSizeOf(this) + mallocSizeOf(blocks_);
+ for (size_t i = 0; i < numBlocks_; i++) {
+ blocks_[i].sizeOfExcludingThis(mallocSizeOf);
+ }
+ return size;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+namespace JS {
+
+template <>
+struct DeletePolicy<js::jit::IonScript> {
+ explicit DeletePolicy(JSRuntime* rt) : rt_(rt) {}
+ void operator()(const js::jit::IonScript* script);
+
+ private:
+ JSRuntime* rt_;
+};
+
+} // namespace JS
+
+#endif /* jit_IonScript_h */
diff --git a/js/src/jit/IonTypes.h b/js/src/jit/IonTypes.h
new file mode 100644
index 0000000000..ee80054f5a
--- /dev/null
+++ b/js/src/jit/IonTypes.h
@@ -0,0 +1,1090 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_IonTypes_h
+#define jit_IonTypes_h
+
+#include "mozilla/HashFunctions.h"
+
+#include <algorithm>
+#include <initializer_list>
+#include <stdint.h>
+
+#include "jstypes.h"
+#include "NamespaceImports.h"
+
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "js/Value.h"
+
+namespace js {
+
+// Each IonScript has a unique compilation id. This is used to sweep/ignore
+// constraints for IonScripts that have been invalidated/destroyed.
+class IonCompilationId {
+ // Use two 32-bit integers instead of uint64_t to avoid 8-byte alignment on
+ // some 32-bit platforms.
+ uint32_t idLo_;
+ uint32_t idHi_;
+
+ public:
+ explicit IonCompilationId(uint64_t id)
+ : idLo_(id & UINT32_MAX), idHi_(id >> 32) {}
+ bool operator==(const IonCompilationId& other) const {
+ return idLo_ == other.idLo_ && idHi_ == other.idHi_;
+ }
+ bool operator!=(const IonCompilationId& other) const {
+ return !operator==(other);
+ }
+};
+
+namespace jit {
+
+using RecoverOffset = uint32_t;
+using SnapshotOffset = uint32_t;
+
+// The maximum size of any buffer associated with an assembler or code object.
+// This is chosen to not overflow a signed integer, leaving room for an extra
+// bit on offsets.
+static const uint32_t MAX_BUFFER_SIZE = (1 << 30) - 1;
+
+// Maximum number of scripted arg slots.
+static const uint32_t SNAPSHOT_MAX_NARGS = 127;
+
+static const SnapshotOffset INVALID_RECOVER_OFFSET = uint32_t(-1);
+static const SnapshotOffset INVALID_SNAPSHOT_OFFSET = uint32_t(-1);
+
+/*
+ * [SMDOC] Avoiding repeated bailouts / invalidations
+ *
+ * To avoid getting trapped in a "compilation -> bailout -> invalidation ->
+ * recompilation -> bailout -> invalidation -> ..." loop, every snapshot in
+ * Warp code is assigned a BailoutKind. If we bail out at that snapshot,
+ * FinishBailoutToBaseline will examine the BailoutKind and take appropriate
+ * action. In general:
+ *
+ * 1. If the bailing instruction comes from transpiled CacheIR, then when we
+ * bail out and continue execution in the baseline interpreter, the
+ * corresponding stub should fail a guard. As a result, we will either
+ * increment the enteredCount for a subsequent stub or attach a new stub,
+ * either of which will prevent WarpOracle from transpiling the failing stub
+ * when we recompile.
+ *
+ * Note: this means that every CacheIR op that can bail out in Warp must
+ * have an equivalent guard in the baseline CacheIR implementation.
+ *
+ * FirstExecution works according to the same principles: we have never hit
+ * this IC before, but after we bail to baseline we will attach a stub and
+ * recompile with better CacheIR information.
+ *
+ * 2. If the bailout occurs because an assumption we made in WarpBuilder was
+ * invalidated, then FinishBailoutToBaseline will set a flag on the script
+ * to avoid that assumption in the future: for example, UninitializedLexical.
+ *
+ * 3. Similarly, if the bailing instruction is generated or modified by a MIR
+ * optimization, then FinishBailoutToBaseline will set a flag on the script
+ * to make that optimization more conservative in the future. Examples
+ * include LICM, EagerTruncation, and HoistBoundsCheck.
+ *
+ * 4. Some bailouts can't be handled in Warp, even after a recompile. For
+ * example, Warp does not support catching exceptions. If this happens
+ * too often, then the cost of bailing out repeatedly outweighs the
+ * benefit of Warp compilation, so we invalidate the script and disable
+ * Warp compilation.
+ *
+ * 5. Some bailouts don't happen in performance-sensitive code: for example,
+ * the |debugger| statement. We just ignore those.
+ */
+enum class BailoutKind : uint8_t {
+ Unknown,
+
+ // An instruction generated by the transpiler. If this instruction bails out,
+ // attaching a new stub in baseline will invalidate the current Warp script
+ // and avoid a bailout loop.
+ TranspiledCacheIR,
+
+ // An optimistic unbox on the cold path for a non-Value phi failed. If this
+ // instruction bails out, we will invalidate the script and mark the
+ // HadSpeculativePhiBailout flag on the script.
+ SpeculativePhi,
+
+ // A conversion inserted by a type policy. If this instruction bails out,
+ // we expect to throw an error. If this happens too frequently, we will
+ // invalidate the current Warp script and disable recompilation.
+ TypePolicy,
+
+ // An instruction hoisted by LICM. If this instruction bails out, we will
+ // bail out to baseline to see if we attach a new stub. If we do, then the
+ // more than once, we will invalidate the current Warp script and
+ // mark the hadLICMInvalidation flag on the script.
+ LICM,
+
+ // An instruction moved up by InstructionReordering. If this
+ // instruction bails out, we will mark the ReorderingBailout flag on
+ // the script. If this happens too frequently, we will invalidate
+ // the script.
+ InstructionReordering,
+
+ // An instruction created or hoisted by tryHoistBoundsCheck.
+ // If this instruction bails out, we will invalidate the current Warp script
+ // and mark the HoistBoundsCheckBailout flag on the script.
+ HoistBoundsCheck,
+
+ // An eager truncation generated by range analysis.
+ // If this instruction bails out, we will invalidate the current Warp script
+ // and mark the EagerTruncationBailout flag on the script.
+ EagerTruncation,
+
+ // A folded unbox instruction generated by FoldLoadsWithUnbox.
+ // If this instruction bails out, we will invalidate the current Warp script
+ // and mark the UnboxFoldingBailout flag on the script.
+ UnboxFolding,
+
+ // An inevitable bailout (MBail instruction or type barrier that always bails)
+ Inevitable,
+
+ // Bailing out during a VM call. Many possible causes that are hard
+ // to distinguish statically at snapshot construction time.
+ // We just lump them together.
+ DuringVMCall,
+
+ // A spread call or funapply had more than JIT_ARGS_LENGTH_MAX arguments.
+ // We bail out to handle this in the VM. If this happens too frequently,
+ // we will invalidate the current Warp script and disable recompilation.
+ TooManyArguments,
+
+ // We hit an active |debugger;| statement.
+ Debugger,
+
+ // We hit this code for the first time.
+ FirstExecution,
+
+ // A lexical check failed. We will set lexical checks as unmovable.
+ UninitializedLexical,
+
+ // A bailout to baseline from Ion on exception to handle Debugger hooks.
+ IonExceptionDebugMode,
+
+ // A bailout to baseline from Ion on exception to handle a finally block.
+ Finally,
+
+ // We returned to a stack frame after invalidating its IonScript.
+ OnStackInvalidation,
+
+ // We returned to a stack frame while calling the |return| method of an
+ // iterator, and we have to throw an exception because the return value
+ // was not an object.
+ ThrowCheckIsObject,
+
+ // We have executed code that should be unreachable, and need to assert.
+ Unreachable,
+
+ Limit
+};
+
+inline const char* BailoutKindString(BailoutKind kind) {
+ switch (kind) {
+ case BailoutKind::Unknown:
+ return "Unknown";
+ case BailoutKind::TranspiledCacheIR:
+ return "TranspiledCacheIR";
+ case BailoutKind::SpeculativePhi:
+ return "SpeculativePhi";
+ case BailoutKind::TypePolicy:
+ return "TypePolicy";
+ case BailoutKind::LICM:
+ return "LICM";
+ case BailoutKind::InstructionReordering:
+ return "InstructionReordering";
+ case BailoutKind::HoistBoundsCheck:
+ return "HoistBoundsCheck";
+ case BailoutKind::EagerTruncation:
+ return "EagerTruncation";
+ case BailoutKind::UnboxFolding:
+ return "UnboxFolding";
+ case BailoutKind::Inevitable:
+ return "Inevitable";
+ case BailoutKind::DuringVMCall:
+ return "DuringVMCall";
+ case BailoutKind::TooManyArguments:
+ return "TooManyArguments";
+ case BailoutKind::Debugger:
+ return "Debugger";
+ case BailoutKind::FirstExecution:
+ return "FirstExecution";
+ case BailoutKind::UninitializedLexical:
+ return "UninitializedLexical";
+ case BailoutKind::IonExceptionDebugMode:
+ return "IonExceptionDebugMode";
+ case BailoutKind::Finally:
+ return "Finally";
+ case BailoutKind::OnStackInvalidation:
+ return "OnStackInvalidation";
+ case BailoutKind::ThrowCheckIsObject:
+ return "ThrowCheckIsObject";
+ case BailoutKind::Unreachable:
+ return "Unreachable";
+
+ case BailoutKind::Limit:
+ break;
+ }
+
+ MOZ_CRASH("Invalid BailoutKind");
+}
+
+static const uint32_t ELEMENT_TYPE_BITS = 5;
+static const uint32_t ELEMENT_TYPE_SHIFT = 0;
+static const uint32_t ELEMENT_TYPE_MASK = (1 << ELEMENT_TYPE_BITS) - 1;
+static const uint32_t VECTOR_TYPE_BITS = 1;
+static const uint32_t VECTOR_TYPE_SHIFT =
+ ELEMENT_TYPE_BITS + ELEMENT_TYPE_SHIFT;
+static const uint32_t VECTOR_TYPE_MASK = (1 << VECTOR_TYPE_BITS) - 1;
+
+// The integer SIMD types have a lot of operations that do the exact same thing
+// for signed and unsigned integer types. Sometimes it is simpler to treat
+// signed and unsigned integer SIMD types as the same type, using a SimdSign to
+// distinguish the few cases where there is a difference.
+enum class SimdSign {
+ // Signedness is not applicable to this type. (i.e., Float or Bool).
+ NotApplicable,
+ // Treat as an unsigned integer with a range 0 .. 2^N-1.
+ Unsigned,
+ // Treat as a signed integer in two's complement encoding.
+ Signed,
+};
+
+class SimdConstant {
+ public:
+ enum Type {
+ Int8x16,
+ Int16x8,
+ Int32x4,
+ Int64x2,
+ Float32x4,
+ Float64x2,
+ Undefined = -1
+ };
+
+ typedef int8_t I8x16[16];
+ typedef int16_t I16x8[8];
+ typedef int32_t I32x4[4];
+ typedef int64_t I64x2[2];
+ typedef float F32x4[4];
+ typedef double F64x2[2];
+
+ private:
+ Type type_;
+ union {
+ I8x16 i8x16;
+ I16x8 i16x8;
+ I32x4 i32x4;
+ I64x2 i64x2;
+ F32x4 f32x4;
+ F64x2 f64x2;
+ } u;
+
+ bool defined() const { return type_ != Undefined; }
+
+ public:
+ // Doesn't have a default constructor, as it would prevent it from being
+ // included in unions.
+
+ static SimdConstant CreateX16(const int8_t* array) {
+ SimdConstant cst;
+ cst.type_ = Int8x16;
+ memcpy(cst.u.i8x16, array, sizeof(cst.u));
+ return cst;
+ }
+ static SimdConstant SplatX16(int8_t v) {
+ SimdConstant cst;
+ cst.type_ = Int8x16;
+ std::fill_n(cst.u.i8x16, 16, v);
+ return cst;
+ }
+ static SimdConstant CreateX8(const int16_t* array) {
+ SimdConstant cst;
+ cst.type_ = Int16x8;
+ memcpy(cst.u.i16x8, array, sizeof(cst.u));
+ return cst;
+ }
+ static SimdConstant SplatX8(int16_t v) {
+ SimdConstant cst;
+ cst.type_ = Int16x8;
+ std::fill_n(cst.u.i16x8, 8, v);
+ return cst;
+ }
+ static SimdConstant CreateX4(const int32_t* array) {
+ SimdConstant cst;
+ cst.type_ = Int32x4;
+ memcpy(cst.u.i32x4, array, sizeof(cst.u));
+ return cst;
+ }
+ static SimdConstant SplatX4(int32_t v) {
+ SimdConstant cst;
+ cst.type_ = Int32x4;
+ std::fill_n(cst.u.i32x4, 4, v);
+ return cst;
+ }
+ static SimdConstant CreateX2(const int64_t* array) {
+ SimdConstant cst;
+ cst.type_ = Int64x2;
+ memcpy(cst.u.i64x2, array, sizeof(cst.u));
+ return cst;
+ }
+ static SimdConstant SplatX2(int64_t v) {
+ SimdConstant cst;
+ cst.type_ = Int64x2;
+ std::fill_n(cst.u.i64x2, 2, v);
+ return cst;
+ }
+ static SimdConstant CreateX4(const float* array) {
+ SimdConstant cst;
+ cst.type_ = Float32x4;
+ memcpy(cst.u.f32x4, array, sizeof(cst.u));
+ return cst;
+ }
+ static SimdConstant SplatX4(float v) {
+ SimdConstant cst;
+ cst.type_ = Float32x4;
+ std::fill_n(cst.u.f32x4, 4, v);
+ return cst;
+ }
+ static SimdConstant CreateX2(const double* array) {
+ SimdConstant cst;
+ cst.type_ = Float64x2;
+ memcpy(cst.u.f64x2, array, sizeof(cst.u));
+ return cst;
+ }
+ static SimdConstant SplatX2(double v) {
+ SimdConstant cst;
+ cst.type_ = Float64x2;
+ std::fill_n(cst.u.f64x2, 2, v);
+ return cst;
+ }
+
+ // Overloads for use by templates.
+ static SimdConstant CreateSimd128(const int8_t* array) {
+ return CreateX16(array);
+ }
+ static SimdConstant CreateSimd128(const int16_t* array) {
+ return CreateX8(array);
+ }
+ static SimdConstant CreateSimd128(const int32_t* array) {
+ return CreateX4(array);
+ }
+ static SimdConstant CreateSimd128(const int64_t* array) {
+ return CreateX2(array);
+ }
+ static SimdConstant CreateSimd128(const float* array) {
+ return CreateX4(array);
+ }
+ static SimdConstant CreateSimd128(const double* array) {
+ return CreateX2(array);
+ }
+
+ Type type() const {
+ MOZ_ASSERT(defined());
+ return type_;
+ }
+
+ bool isFloatingType() const {
+ MOZ_ASSERT(defined());
+ return type_ >= Float32x4;
+ }
+
+ bool isIntegerType() const {
+ MOZ_ASSERT(defined());
+ return type_ <= Int64x2;
+ }
+
+ // Get the raw bytes of the constant.
+ const void* bytes() const { return u.i8x16; }
+
+ const I8x16& asInt8x16() const {
+ MOZ_ASSERT(defined() && type_ == Int8x16);
+ return u.i8x16;
+ }
+
+ const I16x8& asInt16x8() const {
+ MOZ_ASSERT(defined() && type_ == Int16x8);
+ return u.i16x8;
+ }
+
+ const I32x4& asInt32x4() const {
+ MOZ_ASSERT(defined() && type_ == Int32x4);
+ return u.i32x4;
+ }
+
+ const I64x2& asInt64x2() const {
+ MOZ_ASSERT(defined() && type_ == Int64x2);
+ return u.i64x2;
+ }
+
+ const F32x4& asFloat32x4() const {
+ MOZ_ASSERT(defined() && type_ == Float32x4);
+ return u.f32x4;
+ }
+
+ const F64x2& asFloat64x2() const {
+ MOZ_ASSERT(defined() && type_ == Float64x2);
+ return u.f64x2;
+ }
+
+ bool bitwiseEqual(const SimdConstant& rhs) const {
+ MOZ_ASSERT(defined() && rhs.defined());
+ return memcmp(&u, &rhs.u, sizeof(u)) == 0;
+ }
+
+ bool isZeroBits() const {
+ MOZ_ASSERT(defined());
+ return u.i64x2[0] == 0 && u.i64x2[1] == 0;
+ }
+
+ bool isOneBits() const {
+ MOZ_ASSERT(defined());
+ return ~u.i64x2[0] == 0 && ~u.i64x2[1] == 0;
+ }
+
+ // SimdConstant is a HashPolicy. Currently we discriminate by type, but it
+ // may be that we should only be discriminating by int vs float.
+ using Lookup = SimdConstant;
+
+ static HashNumber hash(const SimdConstant& val) {
+ uint32_t hash = mozilla::HashBytes(&val.u, sizeof(val.u));
+ return mozilla::AddToHash(hash, val.type_);
+ }
+
+ static bool match(const SimdConstant& lhs, const SimdConstant& rhs) {
+ return lhs.type() == rhs.type() && lhs.bitwiseEqual(rhs);
+ }
+};
+
+enum class IntConversionBehavior {
+ // These two try to convert the input to an int32 using ToNumber and
+ // will fail if the resulting int32 isn't strictly equal to the input.
+ Normal, // Succeeds on -0: converts to 0.
+ NegativeZeroCheck, // Fails on -0.
+ // These three will convert the input to an int32 with loss of precision.
+ Truncate,
+ TruncateNoWrap,
+ ClampToUint8,
+};
+
+enum class IntConversionInputKind { NumbersOnly, NumbersOrBoolsOnly, Any };
+
+// The ordering of this enumeration is important: Anything < Value is a
+// specialized type. Furthermore, anything < String has trivial conversion to
+// a number.
+enum class MIRType : uint8_t {
+ Undefined,
+ Null,
+ Boolean,
+ Int32,
+ Int64,
+ IntPtr,
+ Double,
+ Float32,
+ // Types above have trivial conversion to a number.
+ String,
+ Symbol,
+ BigInt,
+ Simd128,
+ // Types above are primitive (including undefined and null).
+ Object,
+ MagicOptimizedOut, // JS_OPTIMIZED_OUT magic value.
+ MagicHole, // JS_ELEMENTS_HOLE magic value.
+ MagicIsConstructing, // JS_IS_CONSTRUCTING magic value.
+ MagicUninitializedLexical, // JS_UNINITIALIZED_LEXICAL magic value.
+ // Types above are specialized.
+ Value,
+ None, // Invalid, used as a placeholder.
+ Slots, // A slots vector
+ Elements, // An elements vector
+ Pointer, // An opaque pointer that receives no special treatment
+ RefOrNull, // Wasm Ref/AnyRef/NullRef: a raw JSObject* or a raw (void*)0
+ StackResults, // Wasm multi-value stack result area, which may contain refs
+ Shape, // A Shape pointer.
+ Last = Shape
+};
+
+static inline MIRType TargetWordMIRType() {
+#ifdef JS_64BIT
+ return MIRType::Int64;
+#else
+ return MIRType::Int32;
+#endif
+}
+
+static inline MIRType MIRTypeFromValueType(JSValueType type) {
+ // This function does not deal with magic types. Magic constants should be
+ // filtered out in MIRTypeFromValue.
+ switch (type) {
+ case JSVAL_TYPE_DOUBLE:
+ return MIRType::Double;
+ case JSVAL_TYPE_INT32:
+ return MIRType::Int32;
+ case JSVAL_TYPE_UNDEFINED:
+ return MIRType::Undefined;
+ case JSVAL_TYPE_STRING:
+ return MIRType::String;
+ case JSVAL_TYPE_SYMBOL:
+ return MIRType::Symbol;
+ case JSVAL_TYPE_BIGINT:
+ return MIRType::BigInt;
+ case JSVAL_TYPE_BOOLEAN:
+ return MIRType::Boolean;
+ case JSVAL_TYPE_NULL:
+ return MIRType::Null;
+ case JSVAL_TYPE_OBJECT:
+ return MIRType::Object;
+ case JSVAL_TYPE_UNKNOWN:
+ return MIRType::Value;
+ default:
+ MOZ_CRASH("unexpected jsval type");
+ }
+}
+
+static inline JSValueType ValueTypeFromMIRType(MIRType type) {
+ switch (type) {
+ case MIRType::Undefined:
+ return JSVAL_TYPE_UNDEFINED;
+ case MIRType::Null:
+ return JSVAL_TYPE_NULL;
+ case MIRType::Boolean:
+ return JSVAL_TYPE_BOOLEAN;
+ case MIRType::Int32:
+ return JSVAL_TYPE_INT32;
+ case MIRType::Float32: // Fall through, there's no JSVAL for Float32
+ case MIRType::Double:
+ return JSVAL_TYPE_DOUBLE;
+ case MIRType::String:
+ return JSVAL_TYPE_STRING;
+ case MIRType::Symbol:
+ return JSVAL_TYPE_SYMBOL;
+ case MIRType::BigInt:
+ return JSVAL_TYPE_BIGINT;
+ case MIRType::MagicOptimizedOut:
+ case MIRType::MagicHole:
+ case MIRType::MagicIsConstructing:
+ case MIRType::MagicUninitializedLexical:
+ return JSVAL_TYPE_MAGIC;
+ default:
+ MOZ_ASSERT(type == MIRType::Object);
+ return JSVAL_TYPE_OBJECT;
+ }
+}
+
+static inline JSValueTag MIRTypeToTag(MIRType type) {
+ return JSVAL_TYPE_TO_TAG(ValueTypeFromMIRType(type));
+}
+
+static inline size_t MIRTypeToSize(MIRType type) {
+ switch (type) {
+ case MIRType::Int32:
+ return 4;
+ case MIRType::Int64:
+ return 8;
+ case MIRType::Float32:
+ return 4;
+ case MIRType::Double:
+ return 8;
+ case MIRType::Simd128:
+ return 16;
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ return sizeof(uintptr_t);
+ default:
+ MOZ_CRASH("MIRTypeToSize - unhandled case");
+ }
+}
+
+static inline const char* StringFromMIRType(MIRType type) {
+ switch (type) {
+ case MIRType::Undefined:
+ return "Undefined";
+ case MIRType::Null:
+ return "Null";
+ case MIRType::Boolean:
+ return "Bool";
+ case MIRType::Int32:
+ return "Int32";
+ case MIRType::Int64:
+ return "Int64";
+ case MIRType::IntPtr:
+ return "IntPtr";
+ case MIRType::Double:
+ return "Double";
+ case MIRType::Float32:
+ return "Float32";
+ case MIRType::String:
+ return "String";
+ case MIRType::Symbol:
+ return "Symbol";
+ case MIRType::BigInt:
+ return "BigInt";
+ case MIRType::Object:
+ return "Object";
+ case MIRType::MagicOptimizedOut:
+ return "MagicOptimizedOut";
+ case MIRType::MagicHole:
+ return "MagicHole";
+ case MIRType::MagicIsConstructing:
+ return "MagicIsConstructing";
+ case MIRType::MagicUninitializedLexical:
+ return "MagicUninitializedLexical";
+ case MIRType::Value:
+ return "Value";
+ case MIRType::None:
+ return "None";
+ case MIRType::Slots:
+ return "Slots";
+ case MIRType::Elements:
+ return "Elements";
+ case MIRType::Pointer:
+ return "Pointer";
+ case MIRType::RefOrNull:
+ return "RefOrNull";
+ case MIRType::StackResults:
+ return "StackResults";
+ case MIRType::Shape:
+ return "Shape";
+ case MIRType::Simd128:
+ return "Simd128";
+ }
+ MOZ_CRASH("Unknown MIRType.");
+}
+
+static inline bool IsIntType(MIRType type) {
+ return type == MIRType::Int32 || type == MIRType::Int64;
+}
+
+static inline bool IsNumberType(MIRType type) {
+ return type == MIRType::Int32 || type == MIRType::Double ||
+ type == MIRType::Float32 || type == MIRType::Int64;
+}
+
+static inline bool IsNumericType(MIRType type) {
+ return IsNumberType(type) || type == MIRType::BigInt;
+}
+
+static inline bool IsTypeRepresentableAsDouble(MIRType type) {
+ return type == MIRType::Int32 || type == MIRType::Double ||
+ type == MIRType::Float32;
+}
+
+static inline bool IsFloatType(MIRType type) {
+ return type == MIRType::Int32 || type == MIRType::Float32;
+}
+
+static inline bool IsFloatingPointType(MIRType type) {
+ return type == MIRType::Double || type == MIRType::Float32;
+}
+
+static inline bool IsNullOrUndefined(MIRType type) {
+ return type == MIRType::Null || type == MIRType::Undefined;
+}
+
+static inline bool IsMagicType(MIRType type) {
+ return type == MIRType::MagicHole || type == MIRType::MagicOptimizedOut ||
+ type == MIRType::MagicIsConstructing ||
+ type == MIRType::MagicUninitializedLexical;
+}
+
+static inline bool IsNonGCThing(MIRType type) {
+ return type == MIRType::Undefined || type == MIRType::Null ||
+ type == MIRType::Boolean || IsNumberType(type);
+}
+
+static inline MIRType ScalarTypeToMIRType(Scalar::Type type) {
+ switch (type) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Uint8Clamped:
+ return MIRType::Int32;
+ case Scalar::Int64:
+ return MIRType::Int64;
+ case Scalar::Float32:
+ return MIRType::Float32;
+ case Scalar::Float64:
+ return MIRType::Double;
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ MOZ_CRASH("NYI");
+ case Scalar::Simd128:
+ return MIRType::Simd128;
+ case Scalar::MaxTypedArrayViewType:
+ break;
+ }
+ MOZ_CRASH("unexpected kind");
+}
+
+static constexpr bool NeedsPostBarrier(MIRType type) {
+ MOZ_ASSERT(type != MIRType::Value);
+ return type == MIRType::Object || type == MIRType::String ||
+ type == MIRType::BigInt;
+}
+
+#ifdef DEBUG
+
+// Track the pipeline of opcodes which has produced a snapshot.
+# define TRACK_SNAPSHOTS 1
+
+// Make sure registers are not modified between an instruction and
+// its OsiPoint.
+# define CHECK_OSIPOINT_REGISTERS 1
+
+#endif // DEBUG
+
+enum ABIArgType {
+ // A pointer sized integer
+ ArgType_General = 0x1,
+ // A 32-bit integer
+ ArgType_Int32 = 0x2,
+ // A 64-bit integer
+ ArgType_Int64 = 0x3,
+ // A 32-bit floating point number
+ ArgType_Float32 = 0x4,
+ // A 64-bit floating point number
+ ArgType_Float64 = 0x5,
+
+ RetType_Shift = 0x0,
+ ArgType_Shift = 0x3,
+ ArgType_Mask = 0x7
+};
+
+namespace detail {
+
+static constexpr uint64_t MakeABIFunctionType(
+ ABIArgType ret, std::initializer_list<ABIArgType> args) {
+ uint64_t abiType = (uint64_t)ret << RetType_Shift;
+ int i = 1;
+ for (auto arg : args) {
+ abiType |= ((uint64_t)arg << (ArgType_Shift * i));
+ i++;
+ }
+ return abiType;
+}
+
+} // namespace detail
+
+enum ABIFunctionType : uint64_t {
+ // The enum must be explicitly typed to avoid UB: some validly constructed
+ // members are larger than any explicitly declared members.
+
+ // VM functions that take 0-9 non-double arguments
+ // and return a non-double value.
+ Args_General0 = ArgType_General << RetType_Shift,
+ Args_General1 = Args_General0 | (ArgType_General << (ArgType_Shift * 1)),
+ Args_General2 = Args_General1 | (ArgType_General << (ArgType_Shift * 2)),
+ Args_General3 = Args_General2 | (ArgType_General << (ArgType_Shift * 3)),
+ Args_General4 = Args_General3 | (ArgType_General << (ArgType_Shift * 4)),
+ Args_General5 = Args_General4 | (ArgType_General << (ArgType_Shift * 5)),
+ Args_General6 = Args_General5 | (ArgType_General << (ArgType_Shift * 6)),
+ Args_General7 = Args_General6 | (ArgType_General << (ArgType_Shift * 7)),
+ Args_General8 = Args_General7 | (ArgType_General << (ArgType_Shift * 8)),
+
+ // int64 f(double)
+ Args_Int64_Double =
+ (ArgType_Int64 << RetType_Shift) | (ArgType_Float64 << ArgType_Shift),
+
+ // double f()
+ Args_Double_None = ArgType_Float64 << RetType_Shift,
+
+ // int f(double)
+ Args_Int_Double = Args_General0 | (ArgType_Float64 << ArgType_Shift),
+
+ // int f(float32)
+ Args_Int_Float32 = Args_General0 | (ArgType_Float32 << ArgType_Shift),
+
+ // float f(float)
+ Args_Float32_Float32 =
+ (ArgType_Float32 << RetType_Shift) | (ArgType_Float32 << ArgType_Shift),
+
+ // float f(int, int)
+ Args_Float32_IntInt = (ArgType_Float32 << RetType_Shift) |
+ (ArgType_General << (ArgType_Shift * 1)) |
+ (ArgType_General << (ArgType_Shift * 2)),
+
+ // double f(double)
+ Args_Double_Double = Args_Double_None | (ArgType_Float64 << ArgType_Shift),
+
+ // double f(int)
+ Args_Double_Int = Args_Double_None | (ArgType_General << ArgType_Shift),
+
+ // double f(int, int)
+ Args_Double_IntInt =
+ Args_Double_Int | (ArgType_General << (ArgType_Shift * 2)),
+
+ // double f(double, int)
+ Args_Double_DoubleInt = Args_Double_None |
+ (ArgType_General << (ArgType_Shift * 1)) |
+ (ArgType_Float64 << (ArgType_Shift * 2)),
+
+ // double f(double, double)
+ Args_Double_DoubleDouble =
+ Args_Double_Double | (ArgType_Float64 << (ArgType_Shift * 2)),
+
+ // float f(float, float)
+ Args_Float32_Float32Float32 =
+ Args_Float32_Float32 | (ArgType_Float32 << (ArgType_Shift * 2)),
+
+ // double f(int, double)
+ Args_Double_IntDouble = Args_Double_None |
+ (ArgType_Float64 << (ArgType_Shift * 1)) |
+ (ArgType_General << (ArgType_Shift * 2)),
+
+ // int f(int, double)
+ Args_Int_IntDouble = Args_General0 |
+ (ArgType_Float64 << (ArgType_Shift * 1)) |
+ (ArgType_General << (ArgType_Shift * 2)),
+
+ // int f(double, int)
+ Args_Int_DoubleInt = Args_General0 |
+ (ArgType_General << (ArgType_Shift * 1)) |
+ (ArgType_Float64 << (ArgType_Shift * 2)),
+
+ // double f(double, double, double)
+ Args_Double_DoubleDoubleDouble =
+ Args_Double_DoubleDouble | (ArgType_Float64 << (ArgType_Shift * 3)),
+
+ // double f(double, double, double, double)
+ Args_Double_DoubleDoubleDoubleDouble =
+ Args_Double_DoubleDoubleDouble | (ArgType_Float64 << (ArgType_Shift * 4)),
+
+ // int f(double, int, int)
+ Args_Int_DoubleIntInt = Args_General0 |
+ (ArgType_General << (ArgType_Shift * 1)) |
+ (ArgType_General << (ArgType_Shift * 2)) |
+ (ArgType_Float64 << (ArgType_Shift * 3)),
+
+ // int f(int, double, int, int)
+ Args_Int_IntDoubleIntInt = Args_General0 |
+ (ArgType_General << (ArgType_Shift * 1)) |
+ (ArgType_General << (ArgType_Shift * 2)) |
+ (ArgType_Float64 << (ArgType_Shift * 3)) |
+ (ArgType_General << (ArgType_Shift * 4)),
+
+ Args_Int_GeneralGeneralGeneralInt64 =
+ Args_General0 | (ArgType_General << (ArgType_Shift * 1)) |
+ (ArgType_General << (ArgType_Shift * 2)) |
+ (ArgType_General << (ArgType_Shift * 3)) |
+ (ArgType_Int64 << (ArgType_Shift * 4)),
+
+ Args_Int_GeneralGeneralInt64Int64 = Args_General0 |
+ (ArgType_General << (ArgType_Shift * 1)) |
+ (ArgType_General << (ArgType_Shift * 2)) |
+ (ArgType_Int64 << (ArgType_Shift * 3)) |
+ (ArgType_Int64 << (ArgType_Shift * 4)),
+
+ // int32_t f(...) variants
+ Args_Int32_General =
+ detail::MakeABIFunctionType(ArgType_Int32, {ArgType_General}),
+ Args_Int32_GeneralInt32 = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int32}),
+ Args_Int32_GeneralInt32Int32 = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32}),
+ Args_Int32_GeneralInt32Int32Int32Int32 = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32,
+ ArgType_Int32, ArgType_Int32}),
+ Args_Int32_GeneralInt32Int32Int32Int32Int32 = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32,
+ ArgType_Int32, ArgType_Int32, ArgType_Int32}),
+ Args_Int32_GeneralInt32Int32Int32Int32General = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32,
+ ArgType_Int32, ArgType_Int32, ArgType_General}),
+ Args_Int32_GeneralGeneralInt32GeneralInt32Int32Int32 =
+ detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_General, ArgType_Int32, ArgType_General,
+ ArgType_Int32, ArgType_Int32, ArgType_Int32}),
+ Args_Int32_GeneralInt32Int32Int32Int32Int32Int32General =
+ detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int32,
+ ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_General}),
+ Args_Int32_GeneralInt32Float32Float32Int32Int32Int32General =
+ detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int32, ArgType_Float32, ArgType_Float32,
+ ArgType_Int32, ArgType_Int32, ArgType_Int32, ArgType_General}),
+ Args_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General =
+ detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int32, ArgType_Float32, ArgType_Float32,
+ ArgType_Float32, ArgType_Float32, ArgType_Int32, ArgType_Int32,
+ ArgType_Int32, ArgType_Int32, ArgType_General}),
+ Args_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General =
+ detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int32, ArgType_Float32, ArgType_Float32,
+ ArgType_Int32, ArgType_Float32, ArgType_Float32, ArgType_Int32,
+ ArgType_Float32, ArgType_Int32, ArgType_Int32, ArgType_Int32,
+ ArgType_Int32, ArgType_General}),
+ Args_Int32_GeneralInt32Int32Int32General = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_Int32,
+ ArgType_Int32, ArgType_General}),
+ Args_Int32_GeneralInt32Int32Int64 = detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_Int64}),
+ Args_Int32_GeneralInt32Int32General = detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int32, ArgType_Int32, ArgType_General}),
+ Args_Int32_GeneralInt32Int64Int64 = detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int32, ArgType_Int64, ArgType_Int64}),
+ Args_Int32_GeneralInt32GeneralInt32 = detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int32, ArgType_General, ArgType_Int32}),
+ Args_Int32_GeneralInt32GeneralInt32Int32 = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int32, ArgType_General,
+ ArgType_Int32, ArgType_Int32}),
+ Args_Int32_GeneralGeneral = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_General}),
+ Args_Int32_GeneralGeneralGeneral = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_General, ArgType_General}),
+ Args_Int32_GeneralGeneralInt32Int32 = detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_General, ArgType_Int32, ArgType_Int32}),
+
+ // general f(...) variants
+ Args_General_GeneralInt32 = detail::MakeABIFunctionType(
+ ArgType_General, {ArgType_General, ArgType_Int32}),
+ Args_General_GeneralInt32Int32 = detail::MakeABIFunctionType(
+ ArgType_General, {ArgType_General, ArgType_Int32, ArgType_Int32}),
+ Args_General_GeneralInt32General = detail::MakeABIFunctionType(
+ ArgType_General, {ArgType_General, ArgType_Int32, ArgType_General}),
+ Args_General_GeneralInt32Int32GeneralInt32 = detail::MakeABIFunctionType(
+ ArgType_General, {ArgType_General, ArgType_Int32, ArgType_Int32,
+ ArgType_General, ArgType_Int32}),
+ Args_Int32_GeneralInt64Int32Int32Int32 = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int64, ArgType_Int32,
+ ArgType_Int32, ArgType_Int32}),
+ Args_Int32_GeneralInt64Int32 = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int64, ArgType_Int32}),
+ Args_Int32_GeneralInt64Int32Int64 = detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int64, ArgType_Int32, ArgType_Int64}),
+ Args_Int32_GeneralInt64Int32Int64General = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int64, ArgType_Int32,
+ ArgType_Int64, ArgType_General}),
+ Args_Int32_GeneralInt64Int64Int64 = detail::MakeABIFunctionType(
+ ArgType_Int32,
+ {ArgType_General, ArgType_Int64, ArgType_Int64, ArgType_Int64}),
+ Args_Int32_GeneralInt64Int64Int64General = detail::MakeABIFunctionType(
+ ArgType_Int32, {ArgType_General, ArgType_Int64, ArgType_Int64,
+ ArgType_Int64, ArgType_General}),
+
+ // Functions that return Int64 are tricky because SpiderMonkey's ReturnRegI64
+ // does not match the ABI int64 return register on x86. Wasm only!
+ Args_Int64_General =
+ detail::MakeABIFunctionType(ArgType_Int64, {ArgType_General}),
+ Args_Int64_GeneralInt64 = detail::MakeABIFunctionType(
+ ArgType_Int64, {ArgType_General, ArgType_Int64}),
+
+};
+
+static constexpr ABIFunctionType MakeABIFunctionType(
+ ABIArgType ret, std::initializer_list<ABIArgType> args) {
+ return ABIFunctionType(detail::MakeABIFunctionType(ret, args));
+}
+
+// Rounding modes for round instructions.
+enum class RoundingMode { Down, Up, NearestTiesToEven, TowardsZero };
+
+// If a function contains no calls, we can assume the caller has checked the
+// stack limit up to this maximum frame size. This works because the jit stack
+// limit has a generous buffer before the real end of the native stack.
+static const uint32_t MAX_UNCHECKED_LEAF_FRAME_SIZE = 64;
+
+// Truncating conversion modifiers.
+using TruncFlags = uint32_t;
+static const TruncFlags TRUNC_UNSIGNED = TruncFlags(1) << 0;
+static const TruncFlags TRUNC_SATURATING = TruncFlags(1) << 1;
+
+enum BranchDirection { FALSE_BRANCH, TRUE_BRANCH };
+
+template <typename T>
+constexpr T SplatByteToUInt(uint8_t val, uint8_t x) {
+ T splatted = val;
+ for (; x > 1; x--) {
+ splatted |= splatted << 8;
+ }
+ return splatted;
+}
+
+// Resume information for a frame, stored in a resume point.
+enum class ResumeMode : uint8_t {
+ // Innermost frame. Resume at the next bytecode op when bailing out.
+ ResumeAfter,
+
+ // Innermost frame. This resume point captures an additional value
+ // that is not on the expression stack. Resume at the next bytecode
+ // op when bailing out, but first check that the intermediate value
+ // is an object. This is used if calling the |return| method for a
+ // CloseIter causes an invalidation bailout.
+ ResumeAfterCheckIsObject,
+
+ // Innermost frame. Resume at the current bytecode op when bailing out.
+ ResumeAt,
+
+ // Outer frame for an inlined "standard" call at an IsInvokeOp bytecode op.
+ InlinedStandardCall,
+
+ // Outer frame for an inlined js::fun_call at an IsInvokeOp bytecode op.
+ InlinedFunCall,
+
+ // Outer frame for an inlined getter/setter at a Get*/Set* bytecode op.
+ InlinedAccessor,
+
+ Last = InlinedAccessor
+};
+
+inline const char* ResumeModeToString(ResumeMode mode) {
+ switch (mode) {
+ case ResumeMode::ResumeAfter:
+ return "ResumeAfter";
+ case ResumeMode::ResumeAt:
+ return "ResumeAt";
+ case ResumeMode::InlinedStandardCall:
+ return "InlinedStandardCall";
+ case ResumeMode::InlinedFunCall:
+ return "InlinedFunCall";
+ case ResumeMode::InlinedAccessor:
+ return "InlinedAccessor";
+ case ResumeMode::ResumeAfterCheckIsObject:
+ return "ResumeAfterCheckIsObject";
+ }
+ MOZ_CRASH("Invalid mode");
+}
+
+inline bool IsResumeAfter(ResumeMode mode) {
+ switch (mode) {
+ case ResumeMode::ResumeAfter:
+ case ResumeMode::ResumeAfterCheckIsObject:
+ return true;
+ default:
+ return false;
+ }
+}
+
+// The number of intermediate values captured by this resume point
+// that aren't on the expression stack, but are needed during bailouts.
+inline uint32_t NumIntermediateValues(ResumeMode mode) {
+ switch (mode) {
+ case ResumeMode::ResumeAfterCheckIsObject:
+ return 1;
+ default:
+ return 0;
+ }
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_IonTypes_h */
diff --git a/js/src/jit/JSJitFrameIter-inl.h b/js/src/jit/JSJitFrameIter-inl.h
new file mode 100644
index 0000000000..58c56e4ae4
--- /dev/null
+++ b/js/src/jit/JSJitFrameIter-inl.h
@@ -0,0 +1,65 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JSJitFrameIter_inl_h
+#define jit_JSJitFrameIter_inl_h
+
+#include "jit/JSJitFrameIter.h"
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/ScriptFromCalleeToken.h"
+
+namespace js {
+namespace jit {
+
+inline uint8_t* JSJitFrameIter::returnAddress() const {
+ CommonFrameLayout* current = (CommonFrameLayout*)current_;
+ return current->returnAddress();
+}
+
+inline FrameType JSJitFrameIter::prevType() const {
+ CommonFrameLayout* current = (CommonFrameLayout*)current_;
+ return current->prevType();
+}
+
+inline ExitFrameLayout* JSJitFrameIter::exitFrame() const {
+ MOZ_ASSERT(isExitFrame());
+ return (ExitFrameLayout*)fp();
+}
+
+inline JitFrameLayout* JSJitProfilingFrameIterator::framePtr() const {
+ MOZ_ASSERT(!done());
+ return (JitFrameLayout*)fp_;
+}
+
+inline JSScript* JSJitProfilingFrameIterator::frameScript() const {
+ return ScriptFromCalleeToken(framePtr()->calleeToken());
+}
+
+inline BaselineFrame* JSJitFrameIter::baselineFrame() const {
+ MOZ_ASSERT(isBaselineJS());
+ return (BaselineFrame*)(fp() - BaselineFrame::Size());
+}
+
+inline uint32_t JSJitFrameIter::baselineFrameNumValueSlots() const {
+ MOZ_ASSERT(isBaselineJS());
+ return baselineFrame()->numValueSlots(*baselineFrameSize_);
+}
+
+template <typename T>
+bool JSJitFrameIter::isExitFrameLayout() const {
+ if (!isExitFrame()) {
+ return false;
+ }
+ return exitFrame()->is<T>();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JSJitFrameIter_inl_h */
diff --git a/js/src/jit/JSJitFrameIter.cpp b/js/src/jit/JSJitFrameIter.cpp
new file mode 100644
index 0000000000..f87b7eb89c
--- /dev/null
+++ b/js/src/jit/JSJitFrameIter.cpp
@@ -0,0 +1,775 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/JSJitFrameIter-inl.h"
+
+#include "jit/CalleeToken.h"
+#include "jit/IonScript.h"
+#include "jit/JitcodeMap.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitScript.h"
+#include "jit/MacroAssembler.h" // js::jit::Assembler::GetPointer
+#include "jit/SafepointIndex.h"
+#include "jit/Safepoints.h"
+#include "jit/ScriptFromCalleeToken.h"
+#include "jit/VMFunctions.h"
+#include "js/friend/DumpFunctions.h" // js::DumpObject, js::DumpValue
+#include "vm/JitActivation.h"
+
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+JSJitFrameIter::JSJitFrameIter(const JitActivation* activation)
+ : JSJitFrameIter(activation, FrameType::Exit, activation->jsExitFP()) {}
+
+JSJitFrameIter::JSJitFrameIter(const JitActivation* activation,
+ FrameType frameType, uint8_t* fp)
+ : current_(fp),
+ type_(frameType),
+ resumePCinCurrentFrame_(nullptr),
+ cachedSafepointIndex_(nullptr),
+ activation_(activation) {
+ MOZ_ASSERT(type_ == FrameType::JSJitToWasm || type_ == FrameType::Exit);
+ if (activation_->bailoutData()) {
+ current_ = activation_->bailoutData()->fp();
+ type_ = FrameType::Bailout;
+ } else {
+ MOZ_ASSERT(!TlsContext.get()->inUnsafeCallWithABI);
+ }
+}
+
+bool JSJitFrameIter::checkInvalidation() const {
+ IonScript* dummy;
+ return checkInvalidation(&dummy);
+}
+
+bool JSJitFrameIter::checkInvalidation(IonScript** ionScriptOut) const {
+ JSScript* script = this->script();
+ if (isBailoutJS()) {
+ *ionScriptOut = activation_->bailoutData()->ionScript();
+ return !script->hasIonScript() || script->ionScript() != *ionScriptOut;
+ }
+
+ uint8_t* returnAddr = resumePCinCurrentFrame();
+ // N.B. the current IonScript is not the same as the frame's
+ // IonScript if the frame has since been invalidated.
+ bool invalidated = !script->hasIonScript() ||
+ !script->ionScript()->containsReturnAddress(returnAddr);
+ if (!invalidated) {
+ return false;
+ }
+
+ int32_t invalidationDataOffset = ((int32_t*)returnAddr)[-1];
+ uint8_t* ionScriptDataOffset = returnAddr + invalidationDataOffset;
+ IonScript* ionScript = (IonScript*)Assembler::GetPointer(ionScriptDataOffset);
+ MOZ_ASSERT(ionScript->containsReturnAddress(returnAddr));
+ *ionScriptOut = ionScript;
+ return true;
+}
+
+CalleeToken JSJitFrameIter::calleeToken() const {
+ return ((JitFrameLayout*)current_)->calleeToken();
+}
+
+JSFunction* JSJitFrameIter::callee() const {
+ MOZ_ASSERT(isScripted());
+ MOZ_ASSERT(isFunctionFrame());
+ return CalleeTokenToFunction(calleeToken());
+}
+
+JSFunction* JSJitFrameIter::maybeCallee() const {
+ if (isScripted() && isFunctionFrame()) {
+ return callee();
+ }
+ return nullptr;
+}
+
+bool JSJitFrameIter::isBareExit() const {
+ if (type_ != FrameType::Exit) {
+ return false;
+ }
+ return exitFrame()->isBareExit();
+}
+
+bool JSJitFrameIter::isUnwoundJitExit() const {
+ if (type_ != FrameType::Exit) {
+ return false;
+ }
+ return exitFrame()->isUnwoundJitExit();
+}
+
+bool JSJitFrameIter::isFunctionFrame() const {
+ return CalleeTokenIsFunction(calleeToken());
+}
+
+JSScript* JSJitFrameIter::script() const {
+ MOZ_ASSERT(isScripted());
+ JSScript* script = ScriptFromCalleeToken(calleeToken());
+ MOZ_ASSERT(script);
+ return script;
+}
+
+JSScript* JSJitFrameIter::maybeForwardedScript() const {
+ MOZ_ASSERT(isScripted());
+ if (isBaselineJS()) {
+ return MaybeForwardedScriptFromCalleeToken(baselineFrame()->calleeToken());
+ }
+ JSScript* script = MaybeForwardedScriptFromCalleeToken(calleeToken());
+ MOZ_ASSERT(script);
+ return script;
+}
+
+void JSJitFrameIter::baselineScriptAndPc(JSScript** scriptRes,
+ jsbytecode** pcRes) const {
+ MOZ_ASSERT(isBaselineJS());
+ JSScript* script = this->script();
+ if (scriptRes) {
+ *scriptRes = script;
+ }
+
+ MOZ_ASSERT(pcRes);
+
+ // The Baseline Interpreter stores the bytecode pc in the frame.
+ if (baselineFrame()->runningInInterpreter()) {
+ MOZ_ASSERT(baselineFrame()->interpreterScript() == script);
+ *pcRes = baselineFrame()->interpreterPC();
+ return;
+ }
+
+ // There must be a BaselineScript with a RetAddrEntry for the current return
+ // address.
+ uint8_t* retAddr = resumePCinCurrentFrame();
+ const RetAddrEntry& entry =
+ script->baselineScript()->retAddrEntryFromReturnAddress(retAddr);
+ *pcRes = entry.pc(script);
+}
+
+Value* JSJitFrameIter::actualArgs() const { return jsFrame()->actualArgs(); }
+
+uint8_t* JSJitFrameIter::prevFp() const { return current()->callerFramePtr(); }
+
+// Compute the size of a Baseline frame excluding pushed VMFunction arguments or
+// callee frame headers. This is used to calculate the number of Value slots in
+// the frame. The caller asserts this matches BaselineFrame::debugFrameSize.
+static uint32_t ComputeBaselineFrameSize(const JSJitFrameIter& frame) {
+ MOZ_ASSERT(frame.prevType() == FrameType::BaselineJS);
+
+ uint32_t frameSize = frame.current()->callerFramePtr() - frame.fp();
+
+ if (frame.isBaselineStub()) {
+ return frameSize - BaselineStubFrameLayout::Size();
+ }
+
+ // Note: an UnwoundJit exit frame is a JitFrameLayout that was turned into an
+ // ExitFrameLayout by EnsureUnwoundJitExitFrame. We have to use the original
+ // header size here because that's what we have on the stack.
+ if (frame.isScripted() || frame.isUnwoundJitExit()) {
+ return frameSize - JitFrameLayout::Size();
+ }
+
+ if (frame.isExitFrame()) {
+ frameSize -= ExitFrameLayout::Size();
+ if (frame.exitFrame()->isWrapperExit()) {
+ const VMFunctionData* data = frame.exitFrame()->footer()->function();
+ frameSize -= data->explicitStackSlots() * sizeof(void*);
+ }
+ return frameSize;
+ }
+
+ MOZ_CRASH("Unexpected frame");
+}
+
+void JSJitFrameIter::operator++() {
+ MOZ_ASSERT(!isEntry());
+
+ // Compute BaselineFrame size. In debug builds this is equivalent to
+ // BaselineFrame::debugFrameSize_. This is asserted at the end of this method.
+ if (current()->prevType() == FrameType::BaselineJS) {
+ uint32_t frameSize = ComputeBaselineFrameSize(*this);
+ baselineFrameSize_ = mozilla::Some(frameSize);
+ } else {
+ baselineFrameSize_ = mozilla::Nothing();
+ }
+
+ cachedSafepointIndex_ = nullptr;
+
+ // If the next frame is the entry frame, just exit. Don't update current_,
+ // since the entry and first frames overlap.
+ if (isEntry(current()->prevType())) {
+ type_ = current()->prevType();
+ return;
+ }
+
+ type_ = current()->prevType();
+ resumePCinCurrentFrame_ = current()->returnAddress();
+ current_ = prevFp();
+
+ MOZ_ASSERT_IF(isBaselineJS(),
+ baselineFrame()->debugFrameSize() == *baselineFrameSize_);
+}
+
+uintptr_t* JSJitFrameIter::spillBase() const {
+ MOZ_ASSERT(isIonJS());
+
+ // Get the base address to where safepoint registers are spilled.
+ // Out-of-line calls do not unwind the extra padding space used to
+ // aggregate bailout tables, so we use frameSize instead of frameLocals,
+ // which would only account for local stack slots.
+ return reinterpret_cast<uintptr_t*>(fp() - ionScript()->frameSize());
+}
+
+MachineState JSJitFrameIter::machineState() const {
+ MOZ_ASSERT(isIonScripted());
+
+ // The MachineState is used by GCs for tracing call-sites.
+ if (MOZ_UNLIKELY(isBailoutJS())) {
+ return *activation_->bailoutData()->machineState();
+ }
+
+ SafepointReader reader(ionScript(), safepoint());
+
+ FloatRegisterSet fregs = reader.allFloatSpills().set().reduceSetForPush();
+ GeneralRegisterSet regs = reader.allGprSpills().set();
+
+ uintptr_t* spill = spillBase();
+ uint8_t* spillAlign =
+ alignDoubleSpill(reinterpret_cast<uint8_t*>(spill - regs.size()));
+ char* floatSpill = reinterpret_cast<char*>(spillAlign);
+
+ return MachineState::FromSafepoint(fregs, regs, floatSpill, spill);
+}
+
+JitFrameLayout* JSJitFrameIter::jsFrame() const {
+ MOZ_ASSERT(isScripted());
+ if (isBailoutJS()) {
+ return (JitFrameLayout*)activation_->bailoutData()->fp();
+ }
+
+ return (JitFrameLayout*)fp();
+}
+
+IonScript* JSJitFrameIter::ionScript() const {
+ MOZ_ASSERT(isIonScripted());
+ if (isBailoutJS()) {
+ return activation_->bailoutData()->ionScript();
+ }
+
+ IonScript* ionScript = nullptr;
+ if (checkInvalidation(&ionScript)) {
+ return ionScript;
+ }
+ return ionScriptFromCalleeToken();
+}
+
+IonScript* JSJitFrameIter::ionScriptFromCalleeToken() const {
+ MOZ_ASSERT(isIonJS());
+ MOZ_ASSERT(!checkInvalidation());
+ return script()->ionScript();
+}
+
+const SafepointIndex* JSJitFrameIter::safepoint() const {
+ MOZ_ASSERT(isIonJS());
+ if (!cachedSafepointIndex_) {
+ cachedSafepointIndex_ =
+ ionScript()->getSafepointIndex(resumePCinCurrentFrame());
+ }
+ return cachedSafepointIndex_;
+}
+
+SnapshotOffset JSJitFrameIter::snapshotOffset() const {
+ MOZ_ASSERT(isIonScripted());
+ if (isBailoutJS()) {
+ return activation_->bailoutData()->snapshotOffset();
+ }
+ return osiIndex()->snapshotOffset();
+}
+
+const OsiIndex* JSJitFrameIter::osiIndex() const {
+ MOZ_ASSERT(isIonJS());
+ SafepointReader reader(ionScript(), safepoint());
+ return ionScript()->getOsiIndex(reader.osiReturnPointOffset());
+}
+
+bool JSJitFrameIter::isConstructing() const {
+ return CalleeTokenIsConstructing(calleeToken());
+}
+
+unsigned JSJitFrameIter::numActualArgs() const {
+ if (isScripted()) {
+ return jsFrame()->numActualArgs();
+ }
+
+ MOZ_ASSERT(isExitFrameLayout<NativeExitFrameLayout>());
+ return exitFrame()->as<NativeExitFrameLayout>()->argc();
+}
+
+void JSJitFrameIter::dumpBaseline() const {
+ MOZ_ASSERT(isBaselineJS());
+
+ fprintf(stderr, " JS Baseline frame\n");
+ if (isFunctionFrame()) {
+ fprintf(stderr, " callee fun: ");
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ DumpObject(callee());
+#else
+ fprintf(stderr, "?\n");
+#endif
+ } else {
+ fprintf(stderr, " global frame, no callee\n");
+ }
+
+ fprintf(stderr, " file %s line %u\n", script()->filename(),
+ script()->lineno());
+
+ JSContext* cx = TlsContext.get();
+ RootedScript script(cx);
+ jsbytecode* pc;
+ baselineScriptAndPc(script.address(), &pc);
+
+ fprintf(stderr, " script = %p, pc = %p (offset %u)\n", (void*)script, pc,
+ uint32_t(script->pcToOffset(pc)));
+ fprintf(stderr, " current op: %s\n", CodeName(JSOp(*pc)));
+
+ fprintf(stderr, " actual args: %u\n", numActualArgs());
+
+ for (unsigned i = 0; i < baselineFrameNumValueSlots(); i++) {
+ fprintf(stderr, " slot %u: ", i);
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ Value* v = baselineFrame()->valueSlot(i);
+ DumpValue(*v);
+#else
+ fprintf(stderr, "?\n");
+#endif
+ }
+}
+
+void JSJitFrameIter::dump() const {
+ switch (type_) {
+ case FrameType::CppToJSJit:
+ fprintf(stderr, " Entry frame\n");
+ fprintf(stderr, " Caller frame ptr: %p\n", current()->callerFramePtr());
+ break;
+ case FrameType::BaselineJS:
+ dumpBaseline();
+ break;
+ case FrameType::BaselineStub:
+ fprintf(stderr, " Baseline stub frame\n");
+ fprintf(stderr, " Caller frame ptr: %p\n", current()->callerFramePtr());
+ break;
+ case FrameType::Bailout:
+ case FrameType::IonJS: {
+ InlineFrameIterator frames(TlsContext.get(), this);
+ for (;;) {
+ frames.dump();
+ if (!frames.more()) {
+ break;
+ }
+ ++frames;
+ }
+ break;
+ }
+ case FrameType::Rectifier:
+ fprintf(stderr, " Rectifier frame\n");
+ fprintf(stderr, " Caller frame ptr: %p\n", current()->callerFramePtr());
+ break;
+ case FrameType::IonICCall:
+ fprintf(stderr, " Ion IC call\n");
+ fprintf(stderr, " Caller frame ptr: %p\n", current()->callerFramePtr());
+ break;
+ case FrameType::WasmToJSJit:
+ fprintf(stderr, " Fast wasm-to-JS entry frame\n");
+ fprintf(stderr, " Caller frame ptr: %p\n", current()->callerFramePtr());
+ break;
+ case FrameType::Exit:
+ fprintf(stderr, " Exit frame\n");
+ break;
+ case FrameType::JSJitToWasm:
+ fprintf(stderr, " Wasm exit frame\n");
+ break;
+ };
+ fputc('\n', stderr);
+}
+
+#ifdef DEBUG
+bool JSJitFrameIter::verifyReturnAddressUsingNativeToBytecodeMap() {
+ MOZ_ASSERT(resumePCinCurrentFrame_ != nullptr);
+
+ // Only handle Ion frames for now.
+ if (type_ != FrameType::IonJS && type_ != FrameType::BaselineJS) {
+ return true;
+ }
+
+ JSRuntime* rt = TlsContext.get()->runtime();
+
+ // Don't verify while off thread.
+ if (!CurrentThreadCanAccessRuntime(rt)) {
+ return true;
+ }
+
+ // Don't verify if sampling is being suppressed.
+ if (!TlsContext.get()->isProfilerSamplingEnabled()) {
+ return true;
+ }
+
+ if (JS::RuntimeHeapIsMinorCollecting()) {
+ return true;
+ }
+
+ JitRuntime* jitrt = rt->jitRuntime();
+
+ // Look up and print bytecode info for the native address.
+ const JitcodeGlobalEntry* entry =
+ jitrt->getJitcodeGlobalTable()->lookup(resumePCinCurrentFrame_);
+ if (!entry) {
+ return true;
+ }
+
+ JitSpew(JitSpew_Profiling, "Found nativeToBytecode entry for %p: %p - %p",
+ resumePCinCurrentFrame_, entry->nativeStartAddr(),
+ entry->nativeEndAddr());
+
+ BytecodeLocationVector location;
+ uint32_t depth = UINT32_MAX;
+ if (!entry->callStackAtAddr(rt, resumePCinCurrentFrame_, location, &depth)) {
+ return false;
+ }
+ MOZ_ASSERT(depth > 0 && depth != UINT32_MAX);
+ MOZ_ASSERT(location.length() == depth);
+
+ JitSpew(JitSpew_Profiling, "Found bytecode location of depth %u:", depth);
+ for (size_t i = 0; i < location.length(); i++) {
+ JitSpew(JitSpew_Profiling, " %s:%u - %zu",
+ location[i].getDebugOnlyScript()->filename(),
+ location[i].getDebugOnlyScript()->lineno(),
+ size_t(location[i].toRawBytecode() -
+ location[i].getDebugOnlyScript()->code()));
+ }
+
+ if (type_ == FrameType::IonJS) {
+ // Create an InlineFrameIterator here and verify the mapped info against the
+ // iterator info.
+ InlineFrameIterator inlineFrames(TlsContext.get(), this);
+ for (size_t idx = 0; idx < location.length(); idx++) {
+ MOZ_ASSERT(idx < location.length());
+ MOZ_ASSERT_IF(idx < location.length() - 1, inlineFrames.more());
+
+ JitSpew(JitSpew_Profiling, "Match %d: ION %s:%u(%zu) vs N2B %s:%u(%zu)",
+ (int)idx, inlineFrames.script()->filename(),
+ inlineFrames.script()->lineno(),
+ size_t(inlineFrames.pc() - inlineFrames.script()->code()),
+ location[idx].getDebugOnlyScript()->filename(),
+ location[idx].getDebugOnlyScript()->lineno(),
+ size_t(location[idx].toRawBytecode() -
+ location[idx].getDebugOnlyScript()->code()));
+
+ MOZ_ASSERT(inlineFrames.script() == location[idx].getDebugOnlyScript());
+
+ if (inlineFrames.more()) {
+ ++inlineFrames;
+ }
+ }
+ }
+
+ return true;
+}
+#endif // DEBUG
+
+JSJitProfilingFrameIterator::JSJitProfilingFrameIterator(JSContext* cx,
+ void* pc, void* sp) {
+ // If no profilingActivation is live, initialize directly to
+ // end-of-iteration state.
+ if (!cx->profilingActivation()) {
+ type_ = FrameType::CppToJSJit;
+ fp_ = nullptr;
+ resumePCinCurrentFrame_ = nullptr;
+ return;
+ }
+
+ MOZ_ASSERT(cx->profilingActivation()->isJit());
+
+ JitActivation* act = cx->profilingActivation()->asJit();
+
+ // If the top JitActivation has a null lastProfilingFrame, assume that
+ // it's a trivially empty activation, and initialize directly
+ // to end-of-iteration state.
+ if (!act->lastProfilingFrame()) {
+ type_ = FrameType::CppToJSJit;
+ fp_ = nullptr;
+ resumePCinCurrentFrame_ = nullptr;
+ return;
+ }
+
+ // Get the fp from the current profilingActivation
+ fp_ = (uint8_t*)act->lastProfilingFrame();
+
+ // Use fp_ as endStackAddress_. For cases below where we know we're currently
+ // executing JIT code, we use the current stack pointer instead.
+ endStackAddress_ = fp_;
+
+ // Profiler sampling must NOT be suppressed if we are here.
+ MOZ_ASSERT(cx->isProfilerSamplingEnabled());
+
+ // Try initializing with sampler pc
+ if (tryInitWithPC(pc)) {
+ endStackAddress_ = sp;
+ return;
+ }
+
+ // Try initializing with sampler pc using native=>bytecode table.
+ JitcodeGlobalTable* table =
+ cx->runtime()->jitRuntime()->getJitcodeGlobalTable();
+ if (tryInitWithTable(table, pc, /* forLastCallSite = */ false)) {
+ endStackAddress_ = sp;
+ return;
+ }
+
+ // Try initializing with lastProfilingCallSite pc
+ void* lastCallSite = act->lastProfilingCallSite();
+ if (lastCallSite) {
+ if (tryInitWithPC(lastCallSite)) {
+ return;
+ }
+
+ // Try initializing with lastProfilingCallSite pc using native=>bytecode
+ // table.
+ if (tryInitWithTable(table, lastCallSite, /* forLastCallSite = */ true)) {
+ return;
+ }
+ }
+
+ // If nothing matches, for now just assume we are at the start of the last
+ // frame's baseline jit code or interpreter code.
+ type_ = FrameType::BaselineJS;
+ if (frameScript()->hasBaselineScript()) {
+ resumePCinCurrentFrame_ = frameScript()->baselineScript()->method()->raw();
+ } else {
+ MOZ_ASSERT(IsBaselineInterpreterEnabled());
+ resumePCinCurrentFrame_ =
+ cx->runtime()->jitRuntime()->baselineInterpreter().codeRaw();
+ }
+}
+
+template <typename ReturnType = CommonFrameLayout*>
+static inline ReturnType GetPreviousRawFrame(CommonFrameLayout* frame) {
+ return ReturnType(frame->callerFramePtr());
+}
+
+JSJitProfilingFrameIterator::JSJitProfilingFrameIterator(
+ CommonFrameLayout* fp) {
+ endStackAddress_ = fp;
+ moveToNextFrame(fp);
+}
+
+bool JSJitProfilingFrameIterator::tryInitWithPC(void* pc) {
+ JSScript* callee = frameScript();
+
+ // Check for Ion first, since it's more likely for hot code.
+ if (callee->hasIonScript() &&
+ callee->ionScript()->method()->containsNativePC(pc)) {
+ type_ = FrameType::IonJS;
+ resumePCinCurrentFrame_ = pc;
+ return true;
+ }
+
+ // Check for containment in Baseline jitcode second.
+ if (callee->hasBaselineScript() &&
+ callee->baselineScript()->method()->containsNativePC(pc)) {
+ type_ = FrameType::BaselineJS;
+ resumePCinCurrentFrame_ = pc;
+ return true;
+ }
+
+ return false;
+}
+
+bool JSJitProfilingFrameIterator::tryInitWithTable(JitcodeGlobalTable* table,
+ void* pc,
+ bool forLastCallSite) {
+ if (!pc) {
+ return false;
+ }
+
+ const JitcodeGlobalEntry* entry = table->lookup(pc);
+ if (!entry) {
+ return false;
+ }
+
+ JSScript* callee = frameScript();
+
+ MOZ_ASSERT(entry->isIon() || entry->isIonIC() || entry->isBaseline() ||
+ entry->isBaselineInterpreter() || entry->isDummy());
+
+ // Treat dummy lookups as an empty frame sequence.
+ if (entry->isDummy()) {
+ type_ = FrameType::CppToJSJit;
+ fp_ = nullptr;
+ resumePCinCurrentFrame_ = nullptr;
+ return true;
+ }
+
+ // For IonICEntry, use the corresponding IonEntry.
+ if (entry->isIonIC()) {
+ entry = table->lookup(entry->asIonIC().rejoinAddr());
+ MOZ_ASSERT(entry);
+ MOZ_RELEASE_ASSERT(entry->isIon());
+ }
+
+ if (entry->isIon()) {
+ // If looked-up callee doesn't match frame callee, don't accept
+ // lastProfilingCallSite
+ if (entry->asIon().getScript(0) != callee) {
+ return false;
+ }
+
+ type_ = FrameType::IonJS;
+ resumePCinCurrentFrame_ = pc;
+ return true;
+ }
+
+ if (entry->isBaseline()) {
+ // If looked-up callee doesn't match frame callee, don't accept
+ // lastProfilingCallSite
+ if (forLastCallSite && entry->asBaseline().script() != callee) {
+ return false;
+ }
+
+ type_ = FrameType::BaselineJS;
+ resumePCinCurrentFrame_ = pc;
+ return true;
+ }
+
+ if (entry->isBaselineInterpreter()) {
+ type_ = FrameType::BaselineJS;
+ resumePCinCurrentFrame_ = pc;
+ return true;
+ }
+
+ return false;
+}
+
+const char* JSJitProfilingFrameIterator::baselineInterpreterLabel() const {
+ MOZ_ASSERT(type_ == FrameType::BaselineJS);
+ return frameScript()->jitScript()->profileString();
+}
+
+void JSJitProfilingFrameIterator::baselineInterpreterScriptPC(
+ JSScript** script, jsbytecode** pc, uint64_t* realmID) const {
+ MOZ_ASSERT(type_ == FrameType::BaselineJS);
+ BaselineFrame* blFrame = (BaselineFrame*)(fp_ - BaselineFrame::Size());
+ *script = frameScript();
+ *pc = (*script)->code();
+
+ if (blFrame->runningInInterpreter() &&
+ blFrame->interpreterScript() == *script) {
+ jsbytecode* interpPC = blFrame->interpreterPC();
+ if ((*script)->containsPC(interpPC)) {
+ *pc = interpPC;
+ }
+
+ *realmID = (*script)->realm()->creationOptions().profilerRealmID();
+ }
+}
+
+void JSJitProfilingFrameIterator::operator++() {
+ JitFrameLayout* frame = framePtr();
+ moveToNextFrame(frame);
+}
+
+void JSJitProfilingFrameIterator::moveToNextFrame(CommonFrameLayout* frame) {
+ /*
+ * fp_ points to a Baseline or Ion frame. The possible call-stacks
+ * patterns occurring between this frame and a previous Ion, Baseline or Entry
+ * frame are as follows:
+ *
+ * <Baseline-Or-Ion>
+ * ^
+ * |
+ * ^--- Ion (or Baseline JSOp::Resume)
+ * |
+ * ^--- Baseline Stub <---- Baseline
+ * |
+ * ^--- IonICCall <---- Ion
+ * |
+ * ^--- WasmToJSJit <---- (other wasm frames, not handled by this iterator)
+ * |
+ * ^--- Arguments Rectifier
+ * | ^
+ * | |
+ * | ^--- Ion
+ * | |
+ * | ^--- Baseline Stub <---- Baseline
+ * | |
+ * | ^--- WasmToJSJit <--- (other wasm frames)
+ * | |
+ * | ^--- Entry Frame (CppToJSJit)
+ * |
+ * ^--- Entry Frame (CppToJSJit)
+ *
+ * NOTE: Keep this in sync with JitRuntime::generateProfilerExitFrameTailStub!
+ */
+
+ // Unwrap rectifier frames.
+ if (frame->prevType() == FrameType::Rectifier) {
+ frame = GetPreviousRawFrame<RectifierFrameLayout*>(frame);
+ MOZ_ASSERT(frame->prevType() == FrameType::IonJS ||
+ frame->prevType() == FrameType::BaselineStub ||
+ frame->prevType() == FrameType::WasmToJSJit ||
+ frame->prevType() == FrameType::CppToJSJit);
+ }
+
+ FrameType prevType = frame->prevType();
+ switch (prevType) {
+ case FrameType::IonJS:
+ case FrameType::BaselineJS:
+ resumePCinCurrentFrame_ = frame->returnAddress();
+ fp_ = GetPreviousRawFrame<uint8_t*>(frame);
+ type_ = prevType;
+ return;
+
+ case FrameType::BaselineStub:
+ case FrameType::IonICCall: {
+ FrameType stubPrevType = (prevType == FrameType::BaselineStub)
+ ? FrameType::BaselineJS
+ : FrameType::IonJS;
+ auto* stubFrame = GetPreviousRawFrame<CommonFrameLayout*>(frame);
+ MOZ_ASSERT(stubFrame->prevType() == stubPrevType);
+ resumePCinCurrentFrame_ = stubFrame->returnAddress();
+ fp_ = GetPreviousRawFrame<uint8_t*>(stubFrame);
+ type_ = stubPrevType;
+ return;
+ }
+
+ case FrameType::WasmToJSJit:
+ // No previous js jit frame, this is a transition frame, used to
+ // pass a wasm iterator the correct value of FP.
+ resumePCinCurrentFrame_ = nullptr;
+ fp_ = GetPreviousRawFrame<uint8_t*>(frame);
+ type_ = FrameType::WasmToJSJit;
+ MOZ_ASSERT(!done());
+ return;
+
+ case FrameType::CppToJSJit:
+ // No previous frame, set to nullptr to indicate that
+ // JSJitProfilingFrameIterator is done().
+ resumePCinCurrentFrame_ = nullptr;
+ fp_ = nullptr;
+ type_ = FrameType::CppToJSJit;
+ return;
+
+ case FrameType::Rectifier:
+ case FrameType::Exit:
+ case FrameType::Bailout:
+ case FrameType::JSJitToWasm:
+ // Rectifier frames are handled before this switch. The other frame types
+ // can't call JS functions directly.
+ break;
+ }
+
+ MOZ_CRASH("Bad frame type.");
+}
diff --git a/js/src/jit/JSJitFrameIter.h b/js/src/jit/JSJitFrameIter.h
new file mode 100644
index 0000000000..ff8e5ad623
--- /dev/null
+++ b/js/src/jit/JSJitFrameIter.h
@@ -0,0 +1,803 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JSJitFrameIter_h
+#define jit_JSJitFrameIter_h
+
+#include "mozilla/Maybe.h"
+
+#include "jstypes.h"
+
+#include "jit/JitCode.h"
+#include "jit/MachineState.h"
+#include "jit/Snapshots.h"
+#include "js/ProfilingFrameIterator.h"
+#include "vm/JSFunction.h"
+#include "vm/JSScript.h"
+
+namespace js {
+
+class ArgumentsObject;
+
+namespace jit {
+
+enum class FrameType {
+ // A JS frame is analogous to a js::InterpreterFrame, representing one
+ // scripted function activation. IonJS frames are used by the optimizing
+ // compiler.
+ IonJS,
+
+ // JS frame used by the Baseline Interpreter and Baseline JIT.
+ BaselineJS,
+
+ // Frame pushed by Baseline stubs that make non-tail calls, so that the
+ // return address -> ICEntry mapping works.
+ BaselineStub,
+
+ // The entry frame is the initial prologue block transitioning from the VM
+ // into the Ion world.
+ CppToJSJit,
+
+ // A rectifier frame sits in between two JS frames, adapting argc != nargs
+ // mismatches in calls.
+ Rectifier,
+
+ // Ion IC calling a scripted getter/setter or a VMFunction.
+ IonICCall,
+
+ // An exit frame is necessary for transitioning from a JS frame into C++.
+ // From within C++, an exit frame is always the last frame in any
+ // JitActivation.
+ Exit,
+
+ // A bailout frame is a special IonJS jit frame after a bailout, and before
+ // the reconstruction of the BaselineJS frame. From within C++, a bailout
+ // frame is always the last frame in a JitActivation iff the bailout frame
+ // information is recorded on the JitActivation.
+ Bailout,
+
+ // A wasm to JS frame is constructed during fast calls from wasm to the JS
+ // jits, used as a marker to interleave JS jit and wasm frames. From the
+ // point of view of JS JITs, this is just another kind of entry frame.
+ WasmToJSJit,
+
+ // A JS to wasm frame is constructed during fast calls from any JS jits to
+ // wasm, and is a special kind of exit frame that doesn't have the exit
+ // footer. From the point of view of the jit, it can be skipped as an exit.
+ JSJitToWasm,
+};
+
+enum ReadFrameArgsBehavior {
+ // Only read formals (i.e. [0 ... callee()->nargs]
+ ReadFrame_Formals,
+
+ // Only read overflown args (i.e. [callee()->nargs ... numActuals()]
+ ReadFrame_Overflown,
+
+ // Read all args (i.e. [0 ... numActuals()])
+ ReadFrame_Actuals
+};
+
+class CommonFrameLayout;
+class JitFrameLayout;
+class ExitFrameLayout;
+
+class BaselineFrame;
+class JitActivation;
+class SafepointIndex;
+class OsiIndex;
+
+// Iterate over the JIT stack to assert that all invariants are respected.
+// - Check that all entry frames are aligned on JitStackAlignment.
+// - Check that all rectifier frames keep the JitStackAlignment.
+
+void AssertJitStackInvariants(JSContext* cx);
+
+// A JSJitFrameIter can iterate over a linear frame group of JS jit frames
+// only. It will stop at the first frame that is not of the same kind, or at
+// the end of an activation.
+//
+// If you want to handle every kind of frames (including wasm frames), use
+// JitFrameIter. If you want to skip interleaved frames of other kinds, use
+// OnlyJSJitFrameIter.
+
+class JSJitFrameIter {
+ protected:
+ uint8_t* current_;
+ FrameType type_;
+ uint8_t* resumePCinCurrentFrame_;
+
+ // Size of the current Baseline frame. Equivalent to
+ // BaselineFrame::debugFrameSize_ in debug builds.
+ mozilla::Maybe<uint32_t> baselineFrameSize_;
+
+ private:
+ mutable const SafepointIndex* cachedSafepointIndex_;
+ const JitActivation* activation_;
+
+ void dumpBaseline() const;
+
+ public:
+ // See comment above the class.
+ explicit JSJitFrameIter(const JitActivation* activation);
+
+ // A constructor specialized for jit->wasm frames, which starts at a
+ // specific FP.
+ JSJitFrameIter(const JitActivation* activation, FrameType frameType,
+ uint8_t* fp);
+
+ void setResumePCInCurrentFrame(uint8_t* newAddr) {
+ resumePCinCurrentFrame_ = newAddr;
+ }
+
+ // Current frame information.
+ FrameType type() const { return type_; }
+ uint8_t* fp() const { return current_; }
+ const JitActivation* activation() const { return activation_; }
+
+ CommonFrameLayout* current() const { return (CommonFrameLayout*)current_; }
+
+ inline uint8_t* returnAddress() const;
+
+ // Return the pointer of the JitFrame, the iterator is assumed to be settled
+ // on a scripted frame.
+ JitFrameLayout* jsFrame() const;
+
+ inline ExitFrameLayout* exitFrame() const;
+
+ // Returns whether the JS frame has been invalidated and, if so,
+ // places the invalidated Ion script in |ionScript|.
+ bool checkInvalidation(IonScript** ionScript) const;
+ bool checkInvalidation() const;
+
+ bool isExitFrame() const { return type_ == FrameType::Exit; }
+ bool isScripted() const {
+ return type_ == FrameType::BaselineJS || type_ == FrameType::IonJS ||
+ type_ == FrameType::Bailout;
+ }
+ bool isBaselineJS() const { return type_ == FrameType::BaselineJS; }
+ bool isIonScripted() const {
+ return type_ == FrameType::IonJS || type_ == FrameType::Bailout;
+ }
+ bool isIonJS() const { return type_ == FrameType::IonJS; }
+ bool isIonICCall() const { return type_ == FrameType::IonICCall; }
+ bool isBailoutJS() const { return type_ == FrameType::Bailout; }
+ bool isBaselineStub() const { return type_ == FrameType::BaselineStub; }
+ bool isRectifier() const { return type_ == FrameType::Rectifier; }
+ bool isBareExit() const;
+ bool isUnwoundJitExit() const;
+ template <typename T>
+ bool isExitFrameLayout() const;
+
+ static bool isEntry(FrameType type) {
+ return type == FrameType::CppToJSJit || type == FrameType::WasmToJSJit;
+ }
+ bool isEntry() const { return isEntry(type_); }
+
+ bool isFunctionFrame() const;
+
+ bool isConstructing() const;
+
+ void* calleeToken() const;
+ JSFunction* callee() const;
+ JSFunction* maybeCallee() const;
+ unsigned numActualArgs() const;
+ JSScript* script() const;
+ JSScript* maybeForwardedScript() const;
+ void baselineScriptAndPc(JSScript** scriptRes, jsbytecode** pcRes) const;
+ Value* actualArgs() const;
+
+ // Returns the address of the next instruction that will execute in this
+ // frame, once control returns to this frame.
+ uint8_t* resumePCinCurrentFrame() const { return resumePCinCurrentFrame_; }
+
+ // Previous frame information extracted from the current frame.
+ inline FrameType prevType() const;
+ uint8_t* prevFp() const;
+
+ // Functions used to iterate on frames. When prevType is an entry,
+ // the current frame is the last JS Jit frame.
+ bool done() const { return isEntry(); }
+ void operator++();
+
+ // Returns the IonScript associated with this JS frame.
+ IonScript* ionScript() const;
+
+ // Returns the IonScript associated with this JS frame; the frame must
+ // not be invalidated.
+ IonScript* ionScriptFromCalleeToken() const;
+
+ // Returns the Safepoint associated with this JS frame. Incurs a lookup
+ // overhead.
+ const SafepointIndex* safepoint() const;
+
+ // Returns the OSI index associated with this JS frame. Incurs a lookup
+ // overhead.
+ const OsiIndex* osiIndex() const;
+
+ // Returns the Snapshot offset associated with this JS frame. Incurs a
+ // lookup overhead.
+ SnapshotOffset snapshotOffset() const;
+
+ uintptr_t* spillBase() const;
+ MachineState machineState() const;
+
+ template <class Op>
+ void unaliasedForEachActual(Op op, ReadFrameArgsBehavior behavior) const {
+ MOZ_ASSERT(isBaselineJS());
+
+ unsigned nactual = numActualArgs();
+ unsigned start, end;
+ switch (behavior) {
+ case ReadFrame_Formals:
+ start = 0;
+ end = callee()->nargs();
+ break;
+ case ReadFrame_Overflown:
+ start = callee()->nargs();
+ end = nactual;
+ break;
+ case ReadFrame_Actuals:
+ start = 0;
+ end = nactual;
+ }
+
+ Value* argv = actualArgs();
+ for (unsigned i = start; i < end; i++) {
+ op(argv[i]);
+ }
+ }
+
+ void dump() const;
+
+ inline BaselineFrame* baselineFrame() const;
+
+ // Returns the number of local and expression stack Values for the current
+ // Baseline frame.
+ inline uint32_t baselineFrameNumValueSlots() const;
+
+ // This function isn't used, but we keep it here (debug-only) because it is
+ // helpful when chasing issues with the jitcode map.
+#ifdef DEBUG
+ bool verifyReturnAddressUsingNativeToBytecodeMap();
+#else
+ bool verifyReturnAddressUsingNativeToBytecodeMap() { return true; }
+#endif
+};
+
+class JitcodeGlobalTable;
+
+class JSJitProfilingFrameIterator {
+ uint8_t* fp_;
+ // See JS::ProfilingFrameIterator::endStackAddress_ comment.
+ void* endStackAddress_ = nullptr;
+ FrameType type_;
+ void* resumePCinCurrentFrame_;
+
+ inline JSScript* frameScript() const;
+ [[nodiscard]] bool tryInitWithPC(void* pc);
+ [[nodiscard]] bool tryInitWithTable(JitcodeGlobalTable* table, void* pc,
+ bool forLastCallSite);
+
+ void moveToNextFrame(CommonFrameLayout* frame);
+
+ public:
+ JSJitProfilingFrameIterator(JSContext* cx, void* pc, void* sp);
+ explicit JSJitProfilingFrameIterator(CommonFrameLayout* exitFP);
+
+ void operator++();
+ bool done() const { return fp_ == nullptr; }
+
+ const char* baselineInterpreterLabel() const;
+ void baselineInterpreterScriptPC(JSScript** script, jsbytecode** pc,
+ uint64_t* realmID) const;
+
+ void* fp() const {
+ MOZ_ASSERT(!done());
+ return fp_;
+ }
+ inline JitFrameLayout* framePtr() const;
+ void* stackAddress() const { return fp(); }
+ FrameType frameType() const {
+ MOZ_ASSERT(!done());
+ return type_;
+ }
+ void* resumePCinCurrentFrame() const {
+ MOZ_ASSERT(!done());
+ return resumePCinCurrentFrame_;
+ }
+
+ void* endStackAddress() const { return endStackAddress_; }
+};
+
+class RInstructionResults {
+ // Vector of results of recover instructions.
+ typedef mozilla::Vector<HeapPtr<Value>, 1, SystemAllocPolicy> Values;
+ UniquePtr<Values> results_;
+
+ // The frame pointer is used as a key to check if the current frame already
+ // bailed out.
+ JitFrameLayout* fp_;
+
+ // Record if we tried and succeed at allocating and filling the vector of
+ // recover instruction results, if needed. This flag is needed in order to
+ // avoid evaluating the recover instruction twice.
+ bool initialized_;
+
+ public:
+ explicit RInstructionResults(JitFrameLayout* fp);
+ RInstructionResults(RInstructionResults&& src);
+
+ RInstructionResults& operator=(RInstructionResults&& rhs);
+
+ ~RInstructionResults();
+
+ [[nodiscard]] bool init(JSContext* cx, uint32_t numResults);
+ bool isInitialized() const;
+ size_t length() const;
+
+ JitFrameLayout* frame() const;
+
+ HeapPtr<Value>& operator[](size_t index);
+
+ void trace(JSTracer* trc);
+};
+
+struct MaybeReadFallback {
+ enum FallbackConsequence { Fallback_Invalidate, Fallback_DoNothing };
+
+ JSContext* maybeCx = nullptr;
+ JitActivation* activation = nullptr;
+ const JSJitFrameIter* frame = nullptr;
+ const FallbackConsequence consequence = Fallback_Invalidate;
+
+ MaybeReadFallback() = default;
+
+ MaybeReadFallback(JSContext* cx, JitActivation* activation,
+ const JSJitFrameIter* frame,
+ FallbackConsequence consequence = Fallback_Invalidate)
+ : maybeCx(cx),
+ activation(activation),
+ frame(frame),
+ consequence(consequence) {}
+
+ bool canRecoverResults() { return maybeCx; }
+};
+
+class RResumePoint;
+
+// Reads frame information in snapshot-encoding order (that is, outermost frame
+// to innermost frame).
+class SnapshotIterator {
+ protected:
+ SnapshotReader snapshot_;
+ RecoverReader recover_;
+ JitFrameLayout* fp_;
+ const MachineState* machine_;
+ IonScript* ionScript_;
+ RInstructionResults* instructionResults_;
+
+ enum class ReadMethod : bool {
+ // Read the normal value.
+ Normal,
+
+ // Read the default value, or the normal value if there is no default.
+ AlwaysDefault,
+ };
+
+ private:
+ // Read a spilled register from the machine state.
+ bool hasRegister(Register reg) const { return machine_->has(reg); }
+ uintptr_t fromRegister(Register reg) const { return machine_->read(reg); }
+
+ bool hasRegister(FloatRegister reg) const { return machine_->has(reg); }
+ template <typename T>
+ T fromRegister(FloatRegister reg) const {
+ return machine_->read<T>(reg);
+ }
+
+ // Read an uintptr_t from the stack.
+ bool hasStack(int32_t offset) const { return true; }
+ uintptr_t fromStack(int32_t offset) const;
+
+ bool hasInstructionResult(uint32_t index) const {
+ return instructionResults_;
+ }
+ bool hasInstructionResults() const { return instructionResults_; }
+ Value fromInstructionResult(uint32_t index) const;
+
+ Value allocationValue(const RValueAllocation& a,
+ ReadMethod rm = ReadMethod::Normal);
+ [[nodiscard]] bool allocationReadable(const RValueAllocation& a,
+ ReadMethod rm = ReadMethod::Normal);
+ void writeAllocationValuePayload(const RValueAllocation& a, const Value& v);
+ void warnUnreadableAllocation();
+
+ public:
+ // Handle iterating over RValueAllocations of the snapshots.
+ inline RValueAllocation readAllocation() {
+ MOZ_ASSERT(moreAllocations());
+ return snapshot_.readAllocation();
+ }
+ void skip() { snapshot_.skipAllocation(); }
+
+ const RResumePoint* resumePoint() const;
+ const RInstruction* instruction() const { return recover_.instruction(); }
+
+ uint32_t numAllocations() const;
+ inline bool moreAllocations() const {
+ return snapshot_.numAllocationsRead() < numAllocations();
+ }
+
+ JitFrameLayout* frame() { return fp_; };
+
+ // Used by recover instruction to store the value back into the instruction
+ // results array.
+ void storeInstructionResult(const Value& v);
+
+ public:
+ // Exhibits frame properties contained in the snapshot.
+ uint32_t pcOffset() const;
+ ResumeMode resumeMode() const;
+
+ bool resumeAfter() const {
+ // Calls in outer frames are never considered resume-after.
+ MOZ_ASSERT_IF(moreFrames(), !IsResumeAfter(resumeMode()));
+ return IsResumeAfter(resumeMode());
+ }
+ inline BailoutKind bailoutKind() const { return snapshot_.bailoutKind(); }
+
+ public:
+ // Read the next instruction available and get ready to either skip it or
+ // evaluate it.
+ inline void nextInstruction() {
+ MOZ_ASSERT(snapshot_.numAllocationsRead() == numAllocations());
+ recover_.nextInstruction();
+ snapshot_.resetNumAllocationsRead();
+ }
+
+ // Skip an Instruction by walking to the next instruction and by skipping
+ // all the allocations corresponding to this instruction.
+ void skipInstruction();
+
+ inline bool moreInstructions() const { return recover_.moreInstructions(); }
+
+ // Register a vector used for storing the results of the evaluation of
+ // recover instructions. This vector should be registered before the
+ // beginning of the iteration. This function is in charge of allocating
+ // enough space for all instructions results, and return false iff it fails.
+ [[nodiscard]] bool initInstructionResults(MaybeReadFallback& fallback);
+
+ protected:
+ // This function is used internally for computing the result of the recover
+ // instructions.
+ [[nodiscard]] bool computeInstructionResults(
+ JSContext* cx, RInstructionResults* results) const;
+
+ public:
+ // Handle iterating over frames of the snapshots.
+ void nextFrame();
+ void settleOnFrame();
+
+ inline bool moreFrames() const {
+ // The last instruction is recovering the innermost frame, so as long as
+ // there is more instruction there is necesseray more frames.
+ return moreInstructions();
+ }
+
+ public:
+ // Connect all informations about the current script in order to recover the
+ // content of baseline frames.
+
+ SnapshotIterator(const JSJitFrameIter& iter,
+ const MachineState* machineState);
+ SnapshotIterator();
+
+ Value read() { return allocationValue(readAllocation()); }
+
+ // Read the |Normal| value unless it is not available and that the snapshot
+ // provides a |Default| value. This is useful to avoid invalidations of the
+ // frame while we are only interested in a few properties which are provided
+ // by the |Default| value.
+ Value readWithDefault(RValueAllocation* alloc) {
+ *alloc = RValueAllocation();
+ RValueAllocation a = readAllocation();
+ if (allocationReadable(a)) {
+ return allocationValue(a);
+ }
+
+ *alloc = a;
+ return allocationValue(a, ReadMethod::AlwaysDefault);
+ }
+
+ Value maybeRead(const RValueAllocation& a, MaybeReadFallback& fallback);
+ Value maybeRead(MaybeReadFallback& fallback) {
+ RValueAllocation a = readAllocation();
+ return maybeRead(a, fallback);
+ }
+
+ bool tryRead(Value* result);
+
+ void traceAllocation(JSTracer* trc);
+
+ template <class Op>
+ void readFunctionFrameArgs(Op& op, ArgumentsObject** argsObj, Value* thisv,
+ unsigned start, unsigned end, JSScript* script,
+ MaybeReadFallback& fallback) {
+ // Assumes that the common frame arguments have already been read.
+ if (script->needsArgsObj()) {
+ if (argsObj) {
+ Value v = maybeRead(fallback);
+ if (v.isObject()) {
+ *argsObj = &v.toObject().as<ArgumentsObject>();
+ }
+ } else {
+ skip();
+ }
+ }
+
+ if (thisv) {
+ *thisv = maybeRead(fallback);
+ } else {
+ skip();
+ }
+
+ unsigned i = 0;
+ if (end < start) {
+ i = start;
+ }
+
+ for (; i < start; i++) {
+ skip();
+ }
+ for (; i < end; i++) {
+ // We are not always able to read values from the snapshots, some values
+ // such as non-gc things may still be live in registers and cause an
+ // error while reading the machine state.
+ Value v = maybeRead(fallback);
+ op(v);
+ }
+ }
+
+ // Iterate over all the allocations and return only the value of the
+ // allocation located at one index.
+ Value maybeReadAllocByIndex(size_t index);
+
+#ifdef TRACK_SNAPSHOTS
+ void spewBailingFrom() const { snapshot_.spewBailingFrom(); }
+#endif
+};
+
+// Reads frame information in callstack order (that is, innermost frame to
+// outermost frame).
+class InlineFrameIterator {
+ const JSJitFrameIter* frame_;
+ SnapshotIterator start_;
+ SnapshotIterator si_;
+ uint32_t framesRead_;
+
+ // When the inline-frame-iterator is created, this variable is defined to
+ // UINT32_MAX. Then the first iteration of findNextFrame, which settle on
+ // the innermost frame, is used to update this counter to the number of
+ // frames contained in the recover buffer.
+ uint32_t frameCount_;
+
+ // The |calleeTemplate_| fields contains either the JSFunction or the
+ // template from which it is supposed to be cloned. The |calleeRVA_| is an
+ // Invalid value allocation, if the |calleeTemplate_| field is the effective
+ // JSFunction, and not its template. On the other hand, any other value
+ // allocation implies that the |calleeTemplate_| is the template JSFunction
+ // from which the effective one would be derived and cached by the Recover
+ // instruction result.
+ RootedFunction calleeTemplate_;
+ RValueAllocation calleeRVA_;
+
+ RootedScript script_;
+ jsbytecode* pc_;
+ uint32_t numActualArgs_;
+
+ // Register state, used by all snapshot iterators.
+ MachineState machine_;
+
+ struct Nop {
+ void operator()(const Value& v) {}
+ };
+
+ private:
+ void findNextFrame();
+ JSObject* computeEnvironmentChain(const Value& envChainValue,
+ MaybeReadFallback& fallback,
+ bool* hasInitialEnv = nullptr) const;
+
+ public:
+ InlineFrameIterator(JSContext* cx, const JSJitFrameIter* iter);
+ InlineFrameIterator(JSContext* cx, const InlineFrameIterator* iter);
+
+ bool more() const { return frame_ && framesRead_ < frameCount_; }
+
+ // Due to optimizations, we are not always capable of reading the callee of
+ // inlined frames without invalidating the IonCode. This function might
+ // return either the effective callee of the JSFunction which might be used
+ // to create it.
+ //
+ // As such, the |calleeTemplate()| can be used to read most of the metadata
+ // which are conserved across clones.
+ JSFunction* calleeTemplate() const {
+ MOZ_ASSERT(isFunctionFrame());
+ return calleeTemplate_;
+ }
+ JSFunction* maybeCalleeTemplate() const { return calleeTemplate_; }
+
+ JSFunction* callee(MaybeReadFallback& fallback) const;
+
+ unsigned numActualArgs() const {
+ // The number of actual arguments for inline frames is determined by this
+ // iterator based on the caller's bytecode instruction (Call, FunCall,
+ // GetProp/SetProp, etc). For the outer function it's stored in the stack
+ // frame.
+ if (more()) {
+ return numActualArgs_;
+ }
+
+ return frame_->numActualArgs();
+ }
+
+ template <class ArgOp, class LocalOp>
+ void readFrameArgsAndLocals(JSContext* cx, ArgOp& argOp, LocalOp& localOp,
+ JSObject** envChain, bool* hasInitialEnv,
+ Value* rval, ArgumentsObject** argsObj,
+ Value* thisv, ReadFrameArgsBehavior behavior,
+ MaybeReadFallback& fallback) const {
+ SnapshotIterator s(si_);
+
+ // Read the env chain.
+ if (envChain) {
+ Value envChainValue = s.maybeRead(fallback);
+ *envChain =
+ computeEnvironmentChain(envChainValue, fallback, hasInitialEnv);
+ } else {
+ s.skip();
+ }
+
+ // Read return value.
+ if (rval) {
+ *rval = s.maybeRead(fallback);
+ } else {
+ s.skip();
+ }
+
+ // Read arguments, which only function frames have.
+ if (isFunctionFrame()) {
+ unsigned nactual = numActualArgs();
+ unsigned nformal = calleeTemplate()->nargs();
+
+ // Get the non overflown arguments, which are taken from the inlined
+ // frame, because it will have the updated value when JSOp::SetArg is
+ // done.
+ if (behavior != ReadFrame_Overflown) {
+ s.readFunctionFrameArgs(argOp, argsObj, thisv, 0, nformal, script(),
+ fallback);
+ }
+
+ if (behavior != ReadFrame_Formals) {
+ if (more()) {
+ // There is still a parent frame of this inlined frame. All
+ // arguments (also the overflown) are the last pushed values
+ // in the parent frame. To get the overflown arguments, we
+ // need to take them from there.
+
+ // The overflown arguments are not available in current frame.
+ // They are the last pushed arguments in the parent frame of
+ // this inlined frame.
+ InlineFrameIterator it(cx, this);
+ ++it;
+ unsigned argsObjAdj = it.script()->needsArgsObj() ? 1 : 0;
+ bool hasNewTarget = isConstructing();
+ SnapshotIterator parent_s(it.snapshotIterator());
+
+ // Skip over all slots until we get to the last slots
+ // (= arguments slots of callee) the +3 is for [this], [returnvalue],
+ // [envchain], and maybe +1 for [argsObj]
+ MOZ_ASSERT(parent_s.numAllocations() >=
+ nactual + 3 + argsObjAdj + hasNewTarget);
+ unsigned skip = parent_s.numAllocations() - nactual - 3 - argsObjAdj -
+ hasNewTarget;
+ for (unsigned j = 0; j < skip; j++) {
+ parent_s.skip();
+ }
+
+ // Get the overflown arguments
+ parent_s.skip(); // env chain
+ parent_s.skip(); // return value
+ parent_s.readFunctionFrameArgs(argOp, nullptr, nullptr, nformal,
+ nactual, it.script(), fallback);
+ } else {
+ // There is no parent frame to this inlined frame, we can read
+ // from the frame's Value vector directly.
+ Value* argv = frame_->actualArgs();
+ for (unsigned i = nformal; i < nactual; i++) {
+ argOp(argv[i]);
+ }
+ }
+ }
+ }
+
+ // At this point we've read all the formals in s, and can read the
+ // locals.
+ for (unsigned i = 0; i < script()->nfixed(); i++) {
+ localOp(s.maybeRead(fallback));
+ }
+ }
+
+ template <class Op>
+ void unaliasedForEachActual(JSContext* cx, Op op,
+ ReadFrameArgsBehavior behavior,
+ MaybeReadFallback& fallback) const {
+ Nop nop;
+ readFrameArgsAndLocals(cx, op, nop, nullptr, nullptr, nullptr, nullptr,
+ nullptr, behavior, fallback);
+ }
+
+ JSScript* script() const { return script_; }
+ jsbytecode* pc() const { return pc_; }
+ SnapshotIterator snapshotIterator() const { return si_; }
+ bool isFunctionFrame() const;
+ bool isModuleFrame() const;
+ bool isConstructing() const;
+
+ JSObject* environmentChain(MaybeReadFallback& fallback,
+ bool* hasInitialEnvironment = nullptr) const {
+ SnapshotIterator s(si_);
+
+ // envChain
+ Value v = s.maybeRead(fallback);
+ return computeEnvironmentChain(v, fallback, hasInitialEnvironment);
+ }
+
+ Value thisArgument(MaybeReadFallback& fallback) const {
+ SnapshotIterator s(si_);
+
+ // envChain
+ s.skip();
+
+ // return value
+ s.skip();
+
+ // Arguments object.
+ if (script()->needsArgsObj()) {
+ s.skip();
+ }
+
+ return s.maybeRead(fallback);
+ }
+
+ InlineFrameIterator& operator++() {
+ findNextFrame();
+ return *this;
+ }
+
+ void dump() const;
+
+ void resetOn(const JSJitFrameIter* iter);
+
+ const JSJitFrameIter& frame() const { return *frame_; }
+
+ // Inline frame number, 0 for the outermost (non-inlined) frame.
+ size_t frameNo() const { return frameCount() - framesRead_; }
+ size_t frameCount() const {
+ MOZ_ASSERT(frameCount_ != UINT32_MAX);
+ return frameCount_;
+ }
+
+ private:
+ InlineFrameIterator() = delete;
+ InlineFrameIterator(const InlineFrameIterator& iter) = delete;
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JSJitFrameIter_h */
diff --git a/js/src/jit/JSONSpewer.cpp b/js/src/jit/JSONSpewer.cpp
new file mode 100644
index 0000000000..81ce1a2859
--- /dev/null
+++ b/js/src/jit/JSONSpewer.cpp
@@ -0,0 +1,287 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifdef JS_JITSPEW
+
+# include "jit/JSONSpewer.h"
+
+# include "jit/BacktrackingAllocator.h"
+# include "jit/LIR.h"
+# include "jit/MIR.h"
+# include "jit/MIRGraph.h"
+# include "jit/RangeAnalysis.h"
+
+using namespace js;
+using namespace js::jit;
+
+void JSONSpewer::beginFunction(JSScript* script) {
+ beginObject();
+ formatProperty("name", "%s:%u", script->filename(), script->lineno());
+ beginListProperty("passes");
+}
+
+void JSONSpewer::beginWasmFunction(unsigned funcIndex) {
+ beginObject();
+ formatProperty("name", "wasm-func%u", funcIndex);
+ beginListProperty("passes");
+}
+
+void JSONSpewer::beginPass(const char* pass) {
+ beginObject();
+ property("name", pass);
+}
+
+void JSONSpewer::spewMResumePoint(MResumePoint* rp) {
+ if (!rp) {
+ return;
+ }
+
+ beginObjectProperty("resumePoint");
+
+ if (rp->caller()) {
+ property("caller", rp->caller()->block()->id());
+ }
+
+ property("mode", ResumeModeToString(rp->mode()));
+
+ beginListProperty("operands");
+ for (MResumePoint* iter = rp; iter; iter = iter->caller()) {
+ for (int i = iter->numOperands() - 1; i >= 0; i--) {
+ value(iter->getOperand(i)->id());
+ }
+ if (iter->caller()) {
+ value("|");
+ }
+ }
+ endList();
+
+ endObject();
+}
+
+void JSONSpewer::spewMDef(MDefinition* def) {
+ beginObject();
+
+ property("id", def->id());
+
+ propertyName("opcode");
+ out_.printf("\"");
+ def->printOpcode(out_);
+ out_.printf("\"");
+
+ beginListProperty("attributes");
+# define OUTPUT_ATTRIBUTE(X) \
+ do { \
+ if (def->is##X()) value(#X); \
+ } while (0);
+ MIR_FLAG_LIST(OUTPUT_ATTRIBUTE);
+# undef OUTPUT_ATTRIBUTE
+ endList();
+
+ beginListProperty("inputs");
+ for (size_t i = 0, e = def->numOperands(); i < e; i++) {
+ value(def->getOperand(i)->id());
+ }
+ endList();
+
+ beginListProperty("uses");
+ for (MUseDefIterator use(def); use; use++) {
+ value(use.def()->id());
+ }
+ endList();
+
+ if (!def->isLowered()) {
+ beginListProperty("memInputs");
+ if (def->dependency()) {
+ value(def->dependency()->id());
+ }
+ endList();
+ }
+
+ bool isTruncated = false;
+ if (def->isAdd() || def->isSub() || def->isMod() || def->isMul() ||
+ def->isDiv()) {
+ isTruncated = static_cast<MBinaryArithInstruction*>(def)->isTruncated();
+ }
+
+ if (def->type() != MIRType::None && def->range()) {
+ beginStringProperty("type");
+ def->range()->dump(out_);
+ out_.printf(" : %s%s", StringFromMIRType(def->type()),
+ (isTruncated ? " (t)" : ""));
+ endStringProperty();
+ } else {
+ formatProperty("type", "%s%s", StringFromMIRType(def->type()),
+ (isTruncated ? " (t)" : ""));
+ }
+
+ if (def->isInstruction()) {
+ if (MResumePoint* rp = def->toInstruction()->resumePoint()) {
+ spewMResumePoint(rp);
+ }
+ }
+
+ endObject();
+}
+
+void JSONSpewer::spewMIR(MIRGraph* mir) {
+ beginObjectProperty("mir");
+ beginListProperty("blocks");
+
+ for (MBasicBlockIterator block(mir->begin()); block != mir->end(); block++) {
+ beginObject();
+
+ property("number", block->id());
+
+ beginListProperty("attributes");
+ if (block->hasLastIns()) {
+ if (block->isLoopBackedge()) {
+ value("backedge");
+ }
+ if (block->isLoopHeader()) {
+ value("loopheader");
+ }
+ if (block->isSplitEdge()) {
+ value("splitedge");
+ }
+ }
+ endList();
+
+ beginListProperty("predecessors");
+ for (size_t i = 0; i < block->numPredecessors(); i++) {
+ value(block->getPredecessor(i)->id());
+ }
+ endList();
+
+ beginListProperty("successors");
+ if (block->hasLastIns()) {
+ for (size_t i = 0; i < block->numSuccessors(); i++) {
+ value(block->getSuccessor(i)->id());
+ }
+ }
+ endList();
+
+ beginListProperty("instructions");
+ for (MPhiIterator phi(block->phisBegin()); phi != block->phisEnd(); phi++) {
+ spewMDef(*phi);
+ }
+ for (MInstructionIterator i(block->begin()); i != block->end(); i++) {
+ spewMDef(*i);
+ }
+ endList();
+
+ spewMResumePoint(block->entryResumePoint());
+
+ endObject();
+ }
+
+ endList();
+ endObject();
+}
+
+void JSONSpewer::spewLIns(LNode* ins) {
+ beginObject();
+
+ property("id", ins->id());
+
+ propertyName("opcode");
+ out_.printf("\"");
+ ins->dump(out_);
+ out_.printf("\"");
+
+ beginListProperty("defs");
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ if (ins->isPhi()) {
+ value(ins->toPhi()->getDef(i)->virtualRegister());
+ } else {
+ value(ins->toInstruction()->getDef(i)->virtualRegister());
+ }
+ }
+ endList();
+
+ endObject();
+}
+
+void JSONSpewer::spewLIR(MIRGraph* mir) {
+ beginObjectProperty("lir");
+ beginListProperty("blocks");
+
+ for (MBasicBlockIterator i(mir->begin()); i != mir->end(); i++) {
+ LBlock* block = i->lir();
+ if (!block) {
+ continue;
+ }
+
+ beginObject();
+ property("number", i->id());
+
+ beginListProperty("instructions");
+ for (size_t p = 0; p < block->numPhis(); p++) {
+ spewLIns(block->getPhi(p));
+ }
+ for (LInstructionIterator ins(block->begin()); ins != block->end(); ins++) {
+ spewLIns(*ins);
+ }
+ endList();
+
+ endObject();
+ }
+
+ endList();
+ endObject();
+}
+
+void JSONSpewer::spewRanges(BacktrackingAllocator* regalloc) {
+ beginObjectProperty("ranges");
+ beginListProperty("blocks");
+
+ for (size_t bno = 0; bno < regalloc->graph.numBlocks(); bno++) {
+ beginObject();
+ property("number", bno);
+ beginListProperty("vregs");
+
+ LBlock* lir = regalloc->graph.getBlock(bno);
+ for (LInstructionIterator ins = lir->begin(); ins != lir->end(); ins++) {
+ for (size_t k = 0; k < ins->numDefs(); k++) {
+ uint32_t id = ins->getDef(k)->virtualRegister();
+ VirtualRegister* vreg = &regalloc->vregs[id];
+
+ beginObject();
+ property("vreg", id);
+ beginListProperty("ranges");
+
+ for (LiveRange::RegisterLinkIterator iter = vreg->rangesBegin(); iter;
+ iter++) {
+ LiveRange* range = LiveRange::get(*iter);
+
+ beginObject();
+ property("allocation",
+ range->bundle()->allocation().toString().get());
+ property("start", range->from().bits());
+ property("end", range->to().bits());
+ endObject();
+ }
+
+ endList();
+ endObject();
+ }
+ }
+
+ endList();
+ endObject();
+ }
+
+ endList();
+ endObject();
+}
+
+void JSONSpewer::endPass() { endObject(); }
+
+void JSONSpewer::endFunction() {
+ endList();
+ endObject();
+}
+
+#endif /* JS_JITSPEW */
diff --git a/js/src/jit/JSONSpewer.h b/js/src/jit/JSONSpewer.h
new file mode 100644
index 0000000000..6c663c041a
--- /dev/null
+++ b/js/src/jit/JSONSpewer.h
@@ -0,0 +1,48 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JSONSpewer_h
+#define jit_JSONSpewer_h
+
+#ifdef JS_JITSPEW
+
+# include <stdio.h>
+
+# include "js/TypeDecls.h"
+# include "vm/JSONPrinter.h"
+
+namespace js {
+namespace jit {
+
+class BacktrackingAllocator;
+class MDefinition;
+class MIRGraph;
+class MResumePoint;
+class LNode;
+
+class JSONSpewer : JSONPrinter {
+ public:
+ explicit JSONSpewer(GenericPrinter& out) : JSONPrinter(out) {}
+
+ void beginFunction(JSScript* script);
+ void beginWasmFunction(unsigned funcIndex);
+ void beginPass(const char* pass);
+ void spewMDef(MDefinition* def);
+ void spewMResumePoint(MResumePoint* rp);
+ void spewMIR(MIRGraph* mir);
+ void spewLIns(LNode* ins);
+ void spewLIR(MIRGraph* mir);
+ void spewRanges(BacktrackingAllocator* regalloc);
+ void endPass();
+ void endFunction();
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* JS_JITSPEW */
+
+#endif /* jit_JSONSpewer_h */
diff --git a/js/src/jit/Jit.cpp b/js/src/jit/Jit.cpp
new file mode 100644
index 0000000000..cd7cba5b1c
--- /dev/null
+++ b/js/src/jit/Jit.cpp
@@ -0,0 +1,206 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Jit.h"
+
+#include "jit/BaselineJIT.h"
+#include "jit/CalleeToken.h"
+#include "jit/Ion.h"
+#include "jit/JitCommon.h"
+#include "jit/JitRuntime.h"
+#include "js/friend/StackLimits.h" // js::AutoCheckRecursionLimit
+#include "vm/Interpreter.h"
+#include "vm/JitActivation.h"
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+
+#include "vm/Activation-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+static EnterJitStatus JS_HAZ_JSNATIVE_CALLER EnterJit(JSContext* cx,
+ RunState& state,
+ uint8_t* code) {
+ // We don't want to call the interpreter stub here (because
+ // C++ -> interpreterStub -> C++ is slower than staying in C++).
+ MOZ_ASSERT(code);
+ MOZ_ASSERT(code != cx->runtime()->jitRuntime()->interpreterStub().value);
+ MOZ_ASSERT(IsBaselineInterpreterEnabled());
+
+ AutoCheckRecursionLimit recursion(cx);
+ if (!recursion.check(cx)) {
+ return EnterJitStatus::Error;
+ }
+
+#ifdef DEBUG
+ // Assert we don't GC before entering JIT code. A GC could discard JIT code
+ // or move the function stored in the CalleeToken (it won't be traced at
+ // this point). We use Maybe<> here so we can call reset() to call the
+ // AutoAssertNoGC destructor before we enter JIT code.
+ mozilla::Maybe<JS::AutoAssertNoGC> nogc;
+ nogc.emplace(cx);
+#endif
+
+ JSScript* script = state.script();
+ size_t numActualArgs;
+ bool constructing;
+ size_t maxArgc;
+ Value* maxArgv;
+ JSObject* envChain;
+ CalleeToken calleeToken;
+
+ if (state.isInvoke()) {
+ const CallArgs& args = state.asInvoke()->args();
+ numActualArgs = args.length();
+
+ if (TooManyActualArguments(numActualArgs)) {
+ // Too many arguments for Ion. Baseline supports more actual
+ // arguments, so in that case force Baseline code.
+ if (numActualArgs > BASELINE_MAX_ARGS_LENGTH) {
+ return EnterJitStatus::NotEntered;
+ }
+ if (script->hasBaselineScript()) {
+ code = script->baselineScript()->method()->raw();
+ } else {
+ code = cx->runtime()->jitRuntime()->baselineInterpreter().codeRaw();
+ }
+ }
+
+ constructing = state.asInvoke()->constructing();
+ maxArgc = args.length() + 1;
+ maxArgv = args.array() - 1; // -1 to include |this|
+ envChain = nullptr;
+ calleeToken = CalleeToToken(&args.callee().as<JSFunction>(), constructing);
+
+ unsigned numFormals = script->function()->nargs();
+ if (numFormals > numActualArgs) {
+ code = cx->runtime()->jitRuntime()->getArgumentsRectifier().value;
+ }
+ } else {
+ numActualArgs = 0;
+ constructing = false;
+ maxArgc = 0;
+ maxArgv = nullptr;
+ envChain = state.asExecute()->environmentChain();
+ calleeToken = CalleeToToken(state.script());
+ }
+
+ // Caller must construct |this| before invoking the function.
+ MOZ_ASSERT_IF(constructing, maxArgv[0].isObject() ||
+ maxArgv[0].isMagic(JS_UNINITIALIZED_LEXICAL));
+
+ RootedValue result(cx, Int32Value(numActualArgs));
+ {
+ AssertRealmUnchanged aru(cx);
+ ActivationEntryMonitor entryMonitor(cx, calleeToken);
+ JitActivation activation(cx);
+ EnterJitCode enter = cx->runtime()->jitRuntime()->enterJit();
+
+#ifdef DEBUG
+ nogc.reset();
+#endif
+ CALL_GENERATED_CODE(enter, code, maxArgc, maxArgv, /* osrFrame = */ nullptr,
+ calleeToken, envChain, /* osrNumStackValues = */ 0,
+ result.address());
+ }
+
+ // Release temporary buffer used for OSR into Ion.
+ cx->runtime()->jitRuntime()->freeIonOsrTempData();
+
+ if (result.isMagic()) {
+ MOZ_ASSERT(result.isMagic(JS_ION_ERROR));
+ return EnterJitStatus::Error;
+ }
+
+ // Jit callers wrap primitive constructor return, except for derived
+ // class constructors, which are forced to do it themselves.
+ if (constructing && result.isPrimitive()) {
+ MOZ_ASSERT(maxArgv[0].isObject());
+ result = maxArgv[0];
+ }
+
+ state.setReturnValue(result);
+ return EnterJitStatus::Ok;
+}
+
+EnterJitStatus js::jit::MaybeEnterJit(JSContext* cx, RunState& state) {
+ if (!IsBaselineInterpreterEnabled()) {
+ // All JITs are disabled.
+ return EnterJitStatus::NotEntered;
+ }
+
+ // JITs do not respect the debugger's OnNativeCall hook, so JIT execution is
+ // disabled if this hook might need to be called.
+ if (cx->insideDebuggerEvaluationWithOnNativeCallHook) {
+ return EnterJitStatus::NotEntered;
+ }
+
+ JSScript* script = state.script();
+
+ uint8_t* code = script->jitCodeRaw();
+
+#ifdef JS_CACHEIR_SPEW
+ cx->spewer().enableSpewing();
+#endif
+
+ do {
+ // Make sure we can enter Baseline Interpreter code. Note that the prologue
+ // has warm-up checks to tier up if needed.
+ if (script->hasJitScript()) {
+ break;
+ }
+
+ script->incWarmUpCounter();
+
+ // Try to Ion-compile.
+ if (jit::IsIonEnabled(cx)) {
+ jit::MethodStatus status = jit::CanEnterIon(cx, state);
+ if (status == jit::Method_Error) {
+ return EnterJitStatus::Error;
+ }
+ if (status == jit::Method_Compiled) {
+ code = script->jitCodeRaw();
+ break;
+ }
+ }
+
+ // Try to Baseline-compile.
+ if (jit::IsBaselineJitEnabled(cx)) {
+ jit::MethodStatus status =
+ jit::CanEnterBaselineMethod<BaselineTier::Compiler>(cx, state);
+ if (status == jit::Method_Error) {
+ return EnterJitStatus::Error;
+ }
+ if (status == jit::Method_Compiled) {
+ code = script->jitCodeRaw();
+ break;
+ }
+ }
+
+ // Try to enter the Baseline Interpreter.
+ if (IsBaselineInterpreterEnabled()) {
+ jit::MethodStatus status =
+ jit::CanEnterBaselineMethod<BaselineTier::Interpreter>(cx, state);
+ if (status == jit::Method_Error) {
+ return EnterJitStatus::Error;
+ }
+ if (status == jit::Method_Compiled) {
+ code = script->jitCodeRaw();
+ break;
+ }
+ }
+
+ return EnterJitStatus::NotEntered;
+ } while (false);
+
+#ifdef JS_CACHEIR_SPEW
+ cx->spewer().disableSpewing();
+#endif
+
+ return EnterJit(cx, state, code);
+}
diff --git a/js/src/jit/Jit.h b/js/src/jit/Jit.h
new file mode 100644
index 0000000000..bd2b9b726a
--- /dev/null
+++ b/js/src/jit/Jit.h
@@ -0,0 +1,39 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Jit_h
+#define jit_Jit_h
+
+#include "jstypes.h"
+
+struct JS_PUBLIC_API JSContext;
+
+namespace js {
+
+class RunState;
+
+namespace jit {
+
+enum class EnterJitStatus {
+ // An error occurred, either before we entered JIT code or the script threw
+ // an exception. Usually the context will have a pending exception, except
+ // for uncatchable exceptions (interrupts).
+ Error,
+
+ // Entered and returned from JIT code.
+ Ok,
+
+ // We didn't enter JIT code, for instance because the script still has to
+ // warm up or cannot be compiled.
+ NotEntered,
+};
+
+extern EnterJitStatus MaybeEnterJit(JSContext* cx, RunState& state);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Jit_h */
diff --git a/js/src/jit/JitAllocPolicy.h b/js/src/jit/JitAllocPolicy.h
new file mode 100644
index 0000000000..3980f3f2fb
--- /dev/null
+++ b/js/src/jit/JitAllocPolicy.h
@@ -0,0 +1,179 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitAllocPolicy_h
+#define jit_JitAllocPolicy_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Likely.h"
+#include "mozilla/OperatorNewExtensions.h"
+#include "mozilla/TemplateLib.h"
+
+#include <algorithm>
+#include <stddef.h>
+#include <string.h>
+#include <type_traits>
+#include <utility>
+
+#include "ds/LifoAlloc.h"
+#include "jit/InlineList.h"
+#include "js/Utility.h"
+
+namespace js {
+namespace jit {
+
+class TempAllocator {
+ LifoAllocScope lifoScope_;
+
+ public:
+ // Most infallible JIT allocations are small, so we use a ballast of 16
+ // KiB. And with a ballast of 16 KiB, a chunk size of 32 KiB works well,
+ // because TempAllocators with a peak allocation size of less than 16 KiB
+ // (which is most of them) only have to allocate a single chunk.
+ static const size_t BallastSize; // 16 KiB
+ static const size_t PreferredLifoChunkSize; // 32 KiB
+
+ explicit TempAllocator(LifoAlloc* lifoAlloc) : lifoScope_(lifoAlloc) {
+ lifoAlloc->setAsInfallibleByDefault();
+ }
+
+ void* allocateInfallible(size_t bytes) {
+ return lifoScope_.alloc().allocInfallible(bytes);
+ }
+
+ [[nodiscard]] void* allocate(size_t bytes) {
+ LifoAlloc::AutoFallibleScope fallibleAllocator(lifoAlloc());
+ return lifoScope_.alloc().allocEnsureUnused(bytes, BallastSize);
+ }
+
+ template <typename T>
+ [[nodiscard]] T* allocateArray(size_t n) {
+ LifoAlloc::AutoFallibleScope fallibleAllocator(lifoAlloc());
+ size_t bytes;
+ if (MOZ_UNLIKELY(!CalculateAllocSize<T>(n, &bytes))) {
+ return nullptr;
+ }
+ return static_cast<T*>(
+ lifoScope_.alloc().allocEnsureUnused(bytes, BallastSize));
+ }
+
+ // View this allocator as a fallible allocator.
+ struct Fallible {
+ TempAllocator& alloc;
+ };
+ Fallible fallible() { return {*this}; }
+
+ LifoAlloc* lifoAlloc() { return &lifoScope_.alloc(); }
+
+ [[nodiscard]] bool ensureBallast() {
+ JS_OOM_POSSIBLY_FAIL_BOOL();
+ return lifoScope_.alloc().ensureUnusedApproximate(BallastSize);
+ }
+};
+
+class JitAllocPolicy {
+ TempAllocator& alloc_;
+
+ public:
+ MOZ_IMPLICIT JitAllocPolicy(TempAllocator& alloc) : alloc_(alloc) {}
+ template <typename T>
+ T* maybe_pod_malloc(size_t numElems) {
+ size_t bytes;
+ if (MOZ_UNLIKELY(!CalculateAllocSize<T>(numElems, &bytes))) {
+ return nullptr;
+ }
+ return static_cast<T*>(alloc_.allocate(bytes));
+ }
+ template <typename T>
+ T* maybe_pod_calloc(size_t numElems) {
+ T* p = maybe_pod_malloc<T>(numElems);
+ if (MOZ_LIKELY(p)) {
+ memset(p, 0, numElems * sizeof(T));
+ }
+ return p;
+ }
+ template <typename T>
+ T* maybe_pod_realloc(T* p, size_t oldSize, size_t newSize) {
+ T* n = pod_malloc<T>(newSize);
+ if (MOZ_UNLIKELY(!n)) {
+ return n;
+ }
+ MOZ_ASSERT(!(oldSize & mozilla::tl::MulOverflowMask<sizeof(T)>::value));
+ memcpy(n, p, std::min(oldSize * sizeof(T), newSize * sizeof(T)));
+ return n;
+ }
+ template <typename T>
+ T* pod_malloc(size_t numElems) {
+ return maybe_pod_malloc<T>(numElems);
+ }
+ template <typename T>
+ T* pod_calloc(size_t numElems) {
+ return maybe_pod_calloc<T>(numElems);
+ }
+ template <typename T>
+ T* pod_realloc(T* ptr, size_t oldSize, size_t newSize) {
+ return maybe_pod_realloc<T>(ptr, oldSize, newSize);
+ }
+ template <typename T>
+ void free_(T* p, size_t numElems = 0) {}
+ void reportAllocOverflow() const {}
+ [[nodiscard]] bool checkSimulatedOOM() const {
+ return !js::oom::ShouldFailWithOOM();
+ }
+};
+
+struct TempObject {
+ inline void* operator new(size_t nbytes,
+ TempAllocator::Fallible view) noexcept(true) {
+ return view.alloc.allocate(nbytes);
+ }
+ inline void* operator new(size_t nbytes, TempAllocator& alloc) {
+ return alloc.allocateInfallible(nbytes);
+ }
+ template <class T>
+ inline void* operator new(size_t nbytes, T* pos) {
+ static_assert(std::is_convertible_v<T*, TempObject*>,
+ "Placement new argument type must inherit from TempObject");
+ return pos;
+ }
+ template <class T>
+ inline void* operator new(size_t nbytes, mozilla::NotNullTag, T* pos) {
+ static_assert(std::is_convertible_v<T*, TempObject*>,
+ "Placement new argument type must inherit from TempObject");
+ MOZ_ASSERT(pos);
+ return pos;
+ }
+};
+
+template <typename T>
+class TempObjectPool {
+ TempAllocator* alloc_;
+ InlineForwardList<T> freed_;
+
+ public:
+ TempObjectPool() : alloc_(nullptr) {}
+ void setAllocator(TempAllocator& alloc) {
+ MOZ_ASSERT(freed_.empty());
+ alloc_ = &alloc;
+ }
+ template <typename... Args>
+ T* allocate(Args&&... args) {
+ MOZ_ASSERT(alloc_);
+ if (freed_.empty()) {
+ return new (alloc_->fallible()) T(std::forward<Args>(args)...);
+ }
+ T* res = freed_.popFront();
+ return new (res) T(std::forward<Args>(args)...);
+ }
+ void free(T* obj) { freed_.pushFront(obj); }
+ void clear() { freed_.clear(); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitAllocPolicy_h */
diff --git a/js/src/jit/JitCode.h b/js/src/jit/JitCode.h
new file mode 100644
index 0000000000..5a96445c12
--- /dev/null
+++ b/js/src/jit/JitCode.h
@@ -0,0 +1,171 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitCode_h
+#define jit_JitCode_h
+
+#include "mozilla/MemoryReporting.h" // MallocSizeOf
+
+#include <stddef.h> // size_t
+#include <stdint.h> // uint8_t, uint32_t
+
+#include "jstypes.h"
+
+#include "gc/Allocator.h" // AllowGC
+#include "gc/Cell.h" // gc::TenuredCellWithNonGCPointer
+#include "js/TraceKind.h" // JS::TraceKind
+#include "js/UbiNode.h" // ubi::{TracerConcrete, Size, CourseType}
+
+namespace js {
+namespace jit {
+
+class ExecutablePool;
+class JitCode;
+class MacroAssembler;
+
+enum class CodeKind : uint8_t;
+
+// Header at start of raw code buffer
+struct JitCodeHeader {
+ // Link back to corresponding gcthing
+ JitCode* jitCode_;
+
+ void init(JitCode* jitCode);
+
+ static JitCodeHeader* FromExecutable(uint8_t* buffer) {
+ return (JitCodeHeader*)(buffer - sizeof(JitCodeHeader));
+ }
+};
+
+class JitCode : public gc::TenuredCellWithNonGCPointer<uint8_t> {
+ friend class gc::CellAllocator;
+
+ public:
+ // Raw code pointer, stored in the cell header.
+ uint8_t* raw() const { return headerPtr(); }
+
+ protected:
+ ExecutablePool* pool_;
+ uint32_t bufferSize_; // Total buffer size. Does not include headerSize_.
+ uint32_t insnSize_; // Instruction stream size.
+ uint32_t dataSize_; // Size of the read-only data area.
+ uint32_t jumpRelocTableBytes_; // Size of the jump relocation table.
+ uint32_t dataRelocTableBytes_; // Size of the data relocation table.
+ uint8_t headerSize_ : 5; // Number of bytes allocated before codeStart.
+ uint8_t kind_ : 3; // jit::CodeKind, for the memory reporters.
+ bool invalidated_ : 1; // Whether the code object has been invalidated.
+ // This is necessary to prevent GC tracing.
+ bool hasBytecodeMap_ : 1; // Whether the code object has been registered with
+ // native=>bytecode mapping tables.
+
+ JitCode() = delete;
+ JitCode(uint8_t* code, uint32_t bufferSize, uint32_t headerSize,
+ ExecutablePool* pool, CodeKind kind)
+ : TenuredCellWithNonGCPointer(code),
+ pool_(pool),
+ bufferSize_(bufferSize),
+ insnSize_(0),
+ dataSize_(0),
+ jumpRelocTableBytes_(0),
+ dataRelocTableBytes_(0),
+ headerSize_(headerSize),
+ kind_(uint8_t(kind)),
+ invalidated_(false),
+ hasBytecodeMap_(false) {
+ MOZ_ASSERT(CodeKind(kind_) == kind);
+ MOZ_ASSERT(headerSize_ == headerSize);
+ }
+
+ uint32_t dataOffset() const { return insnSize_; }
+ uint32_t jumpRelocTableOffset() const { return dataOffset() + dataSize_; }
+ uint32_t dataRelocTableOffset() const {
+ return jumpRelocTableOffset() + jumpRelocTableBytes_;
+ }
+
+ public:
+ uint8_t* rawEnd() const { return raw() + insnSize_; }
+ bool containsNativePC(const void* addr) const {
+ const uint8_t* addr_u8 = (const uint8_t*)addr;
+ return raw() <= addr_u8 && addr_u8 < rawEnd();
+ }
+ size_t instructionsSize() const { return insnSize_; }
+ size_t bufferSize() const { return bufferSize_; }
+ size_t headerSize() const { return headerSize_; }
+
+ void traceChildren(JSTracer* trc);
+ void finalize(JS::GCContext* gcx);
+ void setInvalidated() { invalidated_ = true; }
+
+ void setHasBytecodeMap() { hasBytecodeMap_ = true; }
+
+ // If this JitCode object has been, effectively, corrupted due to
+ // invalidation patching, then we have to remember this so we don't try and
+ // trace relocation entries that may now be corrupt.
+ bool invalidated() const { return !!invalidated_; }
+
+ template <typename T>
+ T as() const {
+ return JS_DATA_TO_FUNC_PTR(T, raw());
+ }
+
+ void copyFrom(MacroAssembler& masm);
+
+ static JitCode* FromExecutable(uint8_t* buffer) {
+ JitCode* code = JitCodeHeader::FromExecutable(buffer)->jitCode_;
+ MOZ_ASSERT(code->raw() == buffer);
+ return code;
+ }
+
+ static size_t offsetOfCode() { return offsetOfHeaderPtr(); }
+
+ uint8_t* jumpRelocTable() { return raw() + jumpRelocTableOffset(); }
+
+ // Allocates a new JitCode object which will be managed by the GC. If no
+ // object can be allocated, nullptr is returned. On failure, |pool| is
+ // automatically released, so the code may be freed.
+ template <AllowGC allowGC>
+ static JitCode* New(JSContext* cx, uint8_t* code, uint32_t totalSize,
+ uint32_t headerSize, ExecutablePool* pool, CodeKind kind);
+
+ public:
+ static const JS::TraceKind TraceKind = JS::TraceKind::JitCode;
+};
+
+} // namespace jit
+} // namespace js
+
+// JS::ubi::Nodes can point to js::jit::JitCode instances; they're js::gc::Cell
+// instances with no associated compartment.
+namespace JS {
+namespace ubi {
+template <>
+class Concrete<js::jit::JitCode> : TracerConcrete<js::jit::JitCode> {
+ protected:
+ explicit Concrete(js::jit::JitCode* ptr)
+ : TracerConcrete<js::jit::JitCode>(ptr) {}
+
+ public:
+ static void construct(void* storage, js::jit::JitCode* ptr) {
+ new (storage) Concrete(ptr);
+ }
+
+ CoarseType coarseType() const final { return CoarseType::Script; }
+
+ Size size(mozilla::MallocSizeOf mallocSizeOf) const override {
+ Size size = js::gc::Arena::thingSize(get().asTenured().getAllocKind());
+ size += get().bufferSize();
+ size += get().headerSize();
+ return size;
+ }
+
+ const char16_t* typeName() const override { return concreteTypeName; }
+ static const char16_t concreteTypeName[];
+};
+
+} // namespace ubi
+} // namespace JS
+
+#endif /* jit_JitCode_h */
diff --git a/js/src/jit/JitCommon.h b/js/src/jit/JitCommon.h
new file mode 100644
index 0000000000..906aa55934
--- /dev/null
+++ b/js/src/jit/JitCommon.h
@@ -0,0 +1,53 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitCommon_h
+#define jit_JitCommon_h
+
+// Various macros used by all JITs.
+
+#include "jit/Simulator.h"
+
+#ifdef JS_SIMULATOR
+// Call into cross-jitted code by following the ABI of the simulated
+// architecture.
+# define CALL_GENERATED_CODE(entry, p0, p1, p2, p3, p4, p5, p6, p7) \
+ (js::jit::Simulator::Current()->call( \
+ JS_FUNC_TO_DATA_PTR(uint8_t*, entry), 8, intptr_t(p0), intptr_t(p1), \
+ intptr_t(p2), intptr_t(p3), intptr_t(p4), intptr_t(p5), intptr_t(p6), \
+ intptr_t(p7)))
+
+# define CALL_GENERATED_0(entry) \
+ (js::jit::Simulator::Current()->call(JS_FUNC_TO_DATA_PTR(uint8_t*, entry), \
+ 0))
+
+# define CALL_GENERATED_1(entry, p0) \
+ (js::jit::Simulator::Current()->call(JS_FUNC_TO_DATA_PTR(uint8_t*, entry), \
+ 1, intptr_t(p0)))
+
+# define CALL_GENERATED_2(entry, p0, p1) \
+ (js::jit::Simulator::Current()->call(JS_FUNC_TO_DATA_PTR(uint8_t*, entry), \
+ 2, intptr_t(p0), intptr_t(p1)))
+
+# define CALL_GENERATED_3(entry, p0, p1, p2) \
+ (js::jit::Simulator::Current()->call(JS_FUNC_TO_DATA_PTR(uint8_t*, entry), \
+ 3, intptr_t(p0), intptr_t(p1), \
+ intptr_t(p2)))
+
+#else
+
+// Call into jitted code by following the ABI of the native architecture.
+# define CALL_GENERATED_CODE(entry, p0, p1, p2, p3, p4, p5, p6, p7) \
+ entry(p0, p1, p2, p3, p4, p5, p6, p7)
+
+# define CALL_GENERATED_0(entry) entry()
+# define CALL_GENERATED_1(entry, p0) entry(p0)
+# define CALL_GENERATED_2(entry, p0, p1) entry(p0, p1)
+# define CALL_GENERATED_3(entry, p0, p1, p2) entry(p0, p1, p2)
+
+#endif
+
+#endif // jit_JitCommon_h
diff --git a/js/src/jit/JitContext.cpp b/js/src/jit/JitContext.cpp
new file mode 100644
index 0000000000..4a31062949
--- /dev/null
+++ b/js/src/jit/JitContext.cpp
@@ -0,0 +1,161 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/JitContext.h"
+
+#include "mozilla/Assertions.h"
+#include "mozilla/ThreadLocal.h"
+
+#include <stdlib.h>
+
+#include "jit/CacheIRSpewer.h"
+#include "jit/CompileWrappers.h"
+#include "jit/Ion.h"
+#include "jit/JitCode.h"
+#include "jit/JitOptions.h"
+#include "jit/JitSpewer.h"
+#include "jit/MacroAssembler.h"
+#include "jit/PerfSpewer.h"
+#include "js/HeapAPI.h"
+#include "vm/JSContext.h"
+
+#ifdef JS_CODEGEN_ARM64
+# include "jit/arm64/vixl/Cpu-vixl.h"
+#endif
+
+#if defined(ANDROID)
+# include <sys/system_properties.h>
+#endif
+
+using namespace js;
+using namespace js::jit;
+
+namespace js::jit {
+class TempAllocator;
+}
+
+// Assert that JitCode is gc::Cell aligned.
+static_assert(sizeof(JitCode) % gc::CellAlignBytes == 0);
+
+static MOZ_THREAD_LOCAL(JitContext*) TlsJitContext;
+
+static JitContext* CurrentJitContext() {
+ if (!TlsJitContext.init()) {
+ return nullptr;
+ }
+ return TlsJitContext.get();
+}
+
+void jit::SetJitContext(JitContext* ctx) {
+ MOZ_ASSERT(!TlsJitContext.get());
+ TlsJitContext.set(ctx);
+}
+
+JitContext* jit::GetJitContext() {
+ MOZ_ASSERT(CurrentJitContext());
+ return CurrentJitContext();
+}
+
+JitContext* jit::MaybeGetJitContext() { return CurrentJitContext(); }
+
+JitContext::JitContext(CompileRuntime* rt) : runtime(rt) {
+ MOZ_ASSERT(rt);
+ SetJitContext(this);
+}
+
+JitContext::JitContext(JSContext* cx)
+ : cx(cx), runtime(CompileRuntime::get(cx->runtime())) {
+ SetJitContext(this);
+}
+
+JitContext::JitContext() {
+#ifdef DEBUG
+ isCompilingWasm_ = true;
+#endif
+ SetJitContext(this);
+}
+
+JitContext::~JitContext() {
+ MOZ_ASSERT(TlsJitContext.get() == this);
+ TlsJitContext.set(nullptr);
+}
+
+bool jit::InitializeJit() {
+ if (!TlsJitContext.init()) {
+ return false;
+ }
+
+ CheckLogging();
+
+#ifdef JS_CACHEIR_SPEW
+ const char* env = getenv("CACHEIR_LOGS");
+ if (env && env[0] && env[0] != '0') {
+ CacheIRSpewer::singleton().init(env);
+ }
+#endif
+
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ // Compute flags.
+ js::jit::CPUInfo::ComputeFlags();
+#endif
+
+#if defined(JS_CODEGEN_ARM)
+ InitARMFlags();
+#endif
+
+#ifdef JS_CODEGEN_ARM64
+ // Initialize instruction cache flushing.
+ vixl::CPU::SetUp();
+#endif
+
+#ifndef JS_CODEGEN_NONE
+ MOZ_ASSERT(js::jit::CPUFlagsHaveBeenComputed());
+#endif
+
+ // Note: jit flags need to be initialized after the InitARMFlags call above.
+ // This is the final point where we can set disableJitBackend = true, before
+ // we use this flag below with the HasJitBackend call.
+ if (!MacroAssembler::SupportsFloatingPoint()) {
+ JitOptions.disableJitBackend = true;
+ }
+ JitOptions.supportsUnalignedAccesses =
+ MacroAssembler::SupportsUnalignedAccesses();
+
+ if (HasJitBackend()) {
+ if (!InitProcessExecutableMemory()) {
+ return false;
+ }
+ }
+
+ CheckPerf();
+ return true;
+}
+
+void jit::ShutdownJit() {
+ if (HasJitBackend() && !JSRuntime::hasLiveRuntimes()) {
+ ReleaseProcessExecutableMemory();
+ }
+}
+
+bool jit::JitSupportsWasmSimd() {
+#if defined(ENABLE_WASM_SIMD)
+ return js::jit::MacroAssembler::SupportsWasmSimd();
+#else
+ return false;
+#endif
+}
+
+bool jit::JitSupportsAtomics() {
+#if defined(JS_CODEGEN_ARM)
+ // Bug 1146902, bug 1077318: Enable Ion inlining of Atomics
+ // operations on ARM only when the CPU has byte, halfword, and
+ // doubleword load-exclusive and store-exclusive instructions,
+ // until we can add support for systems that don't have those.
+ return js::jit::HasLDSTREXBHD();
+#else
+ return true;
+#endif
+}
diff --git a/js/src/jit/JitContext.h b/js/src/jit/JitContext.h
new file mode 100644
index 0000000000..b19f2dd371
--- /dev/null
+++ b/js/src/jit/JitContext.h
@@ -0,0 +1,168 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitContext_h
+#define jit_JitContext_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Result.h"
+
+#include <stdint.h>
+
+#include "jstypes.h"
+
+struct JS_PUBLIC_API JSContext;
+
+namespace js {
+namespace jit {
+
+class CompileRealm;
+class CompileRuntime;
+class TempAllocator;
+
+enum MethodStatus {
+ Method_Error,
+ Method_CantCompile,
+ Method_Skipped,
+ Method_Compiled
+};
+
+// Use only even, non-zero values for errors, to allow using the UnusedZero and
+// HasFreeLSB optimizations for mozilla::Result (see specializations of
+// UnusedZero/HasFreeLSB below).
+enum class AbortReason : uint8_t {
+ NoAbort,
+ Alloc = 2,
+ Disable = 4,
+ Error = 6,
+};
+} // namespace jit
+} // namespace js
+
+namespace mozilla::detail {
+
+template <>
+struct UnusedZero<js::jit::AbortReason> : UnusedZeroEnum<js::jit::AbortReason> {
+};
+
+template <>
+struct HasFreeLSB<js::jit::AbortReason> {
+ static const bool value = true;
+};
+
+} // namespace mozilla::detail
+
+namespace js {
+namespace jit {
+
+template <typename V>
+using AbortReasonOr = mozilla::Result<V, AbortReason>;
+using mozilla::Err;
+using mozilla::Ok;
+
+static_assert(sizeof(AbortReasonOr<Ok>) <= sizeof(uintptr_t),
+ "Unexpected size of AbortReasonOr<Ok>");
+static_assert(mozilla::detail::SelectResultImpl<bool, AbortReason>::value ==
+ mozilla::detail::PackingStrategy::NullIsOk);
+static_assert(sizeof(AbortReasonOr<bool>) <= sizeof(uintptr_t),
+ "Unexpected size of AbortReasonOr<bool>");
+static_assert(sizeof(AbortReasonOr<uint16_t*>) == sizeof(uintptr_t),
+ "Unexpected size of AbortReasonOr<uint16_t*>");
+
+// A JIT context is needed for parts of the compiler backend such as the
+// MacroAssembler. It points to the JSContext (nullptr for off-thread
+// compilations or Wasm compilations) and also stores some extra information,
+// most of it only used in debug builds.
+//
+// JIT contexts must not be nested.
+
+class MOZ_RAII JitContext {
+#ifdef DEBUG
+ // Whether this thread is actively Ion compiling (does not include Wasm or
+ // WarpOracle).
+ bool inIonBackend_ = false;
+
+ bool isCompilingWasm_ = false;
+ bool oom_ = false;
+#endif
+
+ public:
+ // Running context when compiling on the main thread. Not available during
+ // off-thread compilation.
+ JSContext* cx = nullptr;
+
+ // Wrapper with information about the current runtime. nullptr for Wasm
+ // compilations.
+ CompileRuntime* runtime = nullptr;
+
+ // Constructor for compilations happening on the main thread.
+ explicit JitContext(JSContext* cx);
+
+ // Constructor for off-thread Ion compilations.
+ explicit JitContext(CompileRuntime* rt);
+
+ // Constructor for Wasm compilation.
+ JitContext();
+
+ ~JitContext();
+
+#ifdef DEBUG
+ bool isCompilingWasm() { return isCompilingWasm_; }
+ bool setIsCompilingWasm(bool flag) {
+ bool oldFlag = isCompilingWasm_;
+ isCompilingWasm_ = flag;
+ return oldFlag;
+ }
+ bool hasOOM() { return oom_; }
+ void setOOM() { oom_ = true; }
+
+ bool inIonBackend() const { return inIonBackend_; }
+
+ void enterIonBackend() {
+ MOZ_ASSERT(!inIonBackend_);
+ inIonBackend_ = true;
+ }
+ void leaveIonBackend() {
+ MOZ_ASSERT(inIonBackend_);
+ inIonBackend_ = false;
+ }
+#endif
+};
+
+// Process-wide initialization and shutdown of JIT data structures.
+[[nodiscard]] bool InitializeJit();
+void ShutdownJit();
+
+// Get and set the current JIT context.
+JitContext* GetJitContext();
+JitContext* MaybeGetJitContext();
+
+void SetJitContext(JitContext* ctx);
+
+enum JitExecStatus {
+ // The method call had to be aborted due to a stack limit check. This
+ // error indicates that Ion never attempted to clean up frames.
+ JitExec_Aborted,
+
+ // The method call resulted in an error, and IonMonkey has cleaned up
+ // frames.
+ JitExec_Error,
+
+ // The method call succeeded and returned a value.
+ JitExec_Ok
+};
+
+static inline bool IsErrorStatus(JitExecStatus status) {
+ return status == JitExec_Error || status == JitExec_Aborted;
+}
+
+bool JitSupportsWasmSimd();
+bool JitSupportsAtomics();
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitContext_h */
diff --git a/js/src/jit/JitFrames-inl.h b/js/src/jit/JitFrames-inl.h
new file mode 100644
index 0000000000..ddbc76e103
--- /dev/null
+++ b/js/src/jit/JitFrames-inl.h
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitFrames_inl_h
+#define jit_JitFrames_inl_h
+
+#include "jit/JitFrames.h"
+#include "vm/JSContext.h"
+
+#include "jit/JSJitFrameIter-inl.h"
+
+namespace js {
+namespace jit {
+
+inline BaselineFrame* GetTopBaselineFrame(JSContext* cx) {
+ JSJitFrameIter frame(cx->activation()->asJit());
+ MOZ_ASSERT(frame.type() == FrameType::Exit);
+ ++frame;
+ if (frame.isBaselineStub()) {
+ ++frame;
+ }
+ MOZ_ASSERT(frame.isBaselineJS());
+ return frame.baselineFrame();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitFrames_inl_h */
diff --git a/js/src/jit/JitFrames.cpp b/js/src/jit/JitFrames.cpp
new file mode 100644
index 0000000000..90ede62a20
--- /dev/null
+++ b/js/src/jit/JitFrames.cpp
@@ -0,0 +1,2528 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/JitFrames-inl.h"
+
+#include "mozilla/ScopeExit.h"
+
+#include <algorithm>
+
+#include "builtin/ModuleObject.h"
+#include "gc/GC.h"
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineIC.h"
+#include "jit/BaselineJIT.h"
+#include "jit/Ion.h"
+#include "jit/IonScript.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/LIR.h"
+#include "jit/PcScriptCache.h"
+#include "jit/Recover.h"
+#include "jit/Safepoints.h"
+#include "jit/ScriptFromCalleeToken.h"
+#include "jit/Snapshots.h"
+#include "jit/VMFunctions.h"
+#include "js/Exception.h"
+#include "js/friend/DumpFunctions.h" // js::DumpObject, js::DumpValue
+#include "vm/Interpreter.h"
+#include "vm/JSContext.h"
+#include "vm/JSFunction.h"
+#include "vm/JSObject.h"
+#include "vm/JSScript.h"
+#include "wasm/WasmBuiltins.h"
+#include "wasm/WasmInstance.h"
+
+#include "debugger/DebugAPI-inl.h"
+#include "jit/JSJitFrameIter-inl.h"
+#include "vm/GeckoProfiler-inl.h"
+#include "vm/JSScript-inl.h"
+#include "vm/Probes-inl.h"
+
+namespace js {
+namespace jit {
+
+// Given a slot index, returns the offset, in bytes, of that slot from an
+// JitFrameLayout. Slot distances are uniform across architectures, however,
+// the distance does depend on the size of the frame header.
+static inline int32_t OffsetOfFrameSlot(int32_t slot) { return -slot; }
+
+static inline uint8_t* AddressOfFrameSlot(JitFrameLayout* fp, int32_t slot) {
+ return (uint8_t*)fp + OffsetOfFrameSlot(slot);
+}
+
+static inline uintptr_t ReadFrameSlot(JitFrameLayout* fp, int32_t slot) {
+ return *(uintptr_t*)AddressOfFrameSlot(fp, slot);
+}
+
+static inline void WriteFrameSlot(JitFrameLayout* fp, int32_t slot,
+ uintptr_t value) {
+ *(uintptr_t*)AddressOfFrameSlot(fp, slot) = value;
+}
+
+static inline double ReadFrameDoubleSlot(JitFrameLayout* fp, int32_t slot) {
+ return *(double*)AddressOfFrameSlot(fp, slot);
+}
+
+static inline float ReadFrameFloat32Slot(JitFrameLayout* fp, int32_t slot) {
+ return *(float*)AddressOfFrameSlot(fp, slot);
+}
+
+static inline int32_t ReadFrameInt32Slot(JitFrameLayout* fp, int32_t slot) {
+ return *(int32_t*)AddressOfFrameSlot(fp, slot);
+}
+
+static inline bool ReadFrameBooleanSlot(JitFrameLayout* fp, int32_t slot) {
+ return *(bool*)AddressOfFrameSlot(fp, slot);
+}
+
+static uint32_t NumArgAndLocalSlots(const InlineFrameIterator& frame) {
+ JSScript* script = frame.script();
+ return CountArgSlots(script, frame.maybeCalleeTemplate()) + script->nfixed();
+}
+
+static void CloseLiveIteratorIon(JSContext* cx,
+ const InlineFrameIterator& frame,
+ const TryNote* tn) {
+ MOZ_ASSERT(tn->kind() == TryNoteKind::ForIn ||
+ tn->kind() == TryNoteKind::Destructuring);
+
+ bool isDestructuring = tn->kind() == TryNoteKind::Destructuring;
+ MOZ_ASSERT_IF(!isDestructuring, tn->stackDepth > 0);
+ MOZ_ASSERT_IF(isDestructuring, tn->stackDepth > 1);
+
+ // Save any pending exception, because some recover operations call into
+ // AutoUnsafeCallWithABI functions, which don't allow pending exceptions.
+ JS::AutoSaveExceptionState savedExc(cx);
+
+ SnapshotIterator si = frame.snapshotIterator();
+
+ // Skip stack slots until we reach the iterator object on the stack. For
+ // the destructuring case, we also need to get the "done" value.
+ uint32_t stackSlot = tn->stackDepth;
+ uint32_t adjust = isDestructuring ? 2 : 1;
+ uint32_t skipSlots = NumArgAndLocalSlots(frame) + stackSlot - adjust;
+
+ for (unsigned i = 0; i < skipSlots; i++) {
+ si.skip();
+ }
+
+ MaybeReadFallback recover(cx, cx->activation()->asJit(), &frame.frame(),
+ MaybeReadFallback::Fallback_DoNothing);
+ Value v = si.maybeRead(recover);
+ MOZ_RELEASE_ASSERT(v.isObject());
+ RootedObject iterObject(cx, &v.toObject());
+
+ if (isDestructuring) {
+ RootedValue doneValue(cx, si.read());
+ MOZ_RELEASE_ASSERT(!doneValue.isMagic());
+ bool done = ToBoolean(doneValue);
+ // Do not call IteratorClose if the destructuring iterator is already
+ // done.
+ if (done) {
+ return;
+ }
+ }
+
+ // Restore any pending exception before the closing the iterator.
+ savedExc.restore();
+
+ if (cx->isExceptionPending()) {
+ if (tn->kind() == TryNoteKind::ForIn) {
+ CloseIterator(iterObject);
+ } else {
+ IteratorCloseForException(cx, iterObject);
+ }
+ } else {
+ UnwindIteratorForUncatchableException(iterObject);
+ }
+}
+
+class IonTryNoteFilter {
+ uint32_t depth_;
+
+ public:
+ explicit IonTryNoteFilter(const InlineFrameIterator& frame) {
+ uint32_t base = NumArgAndLocalSlots(frame);
+ SnapshotIterator si = frame.snapshotIterator();
+ MOZ_ASSERT(si.numAllocations() >= base);
+ depth_ = si.numAllocations() - base;
+ }
+
+ bool operator()(const TryNote* note) { return note->stackDepth <= depth_; }
+};
+
+class TryNoteIterIon : public TryNoteIter<IonTryNoteFilter> {
+ public:
+ TryNoteIterIon(JSContext* cx, const InlineFrameIterator& frame)
+ : TryNoteIter(cx, frame.script(), frame.pc(), IonTryNoteFilter(frame)) {}
+};
+
+static bool ShouldBailoutForDebugger(JSContext* cx,
+ const InlineFrameIterator& frame,
+ bool hitBailoutException) {
+ if (hitBailoutException) {
+ MOZ_ASSERT(!cx->isPropagatingForcedReturn());
+ return false;
+ }
+
+ // Bail out if we're propagating a forced return from an inlined frame,
+ // even if the realm is no longer a debuggee.
+ if (cx->isPropagatingForcedReturn() && frame.more()) {
+ return true;
+ }
+
+ if (!cx->realm()->isDebuggee()) {
+ return false;
+ }
+
+ // Bail out if there's a catchable exception and we are the debuggee of a
+ // Debugger with a live onExceptionUnwind hook.
+ if (cx->isExceptionPending() &&
+ DebugAPI::hasExceptionUnwindHook(cx->global())) {
+ return true;
+ }
+
+ // Bail out if a Debugger has observed this frame (e.g., for onPop).
+ JitActivation* act = cx->activation()->asJit();
+ RematerializedFrame* rematFrame =
+ act->lookupRematerializedFrame(frame.frame().fp(), frame.frameNo());
+ return rematFrame && rematFrame->isDebuggee();
+}
+
+static void OnLeaveIonFrame(JSContext* cx, const InlineFrameIterator& frame,
+ ResumeFromException* rfe) {
+ bool returnFromThisFrame =
+ cx->isPropagatingForcedReturn() || cx->isClosingGenerator();
+ if (!returnFromThisFrame) {
+ return;
+ }
+
+ JitActivation* act = cx->activation()->asJit();
+ RematerializedFrame* rematFrame = nullptr;
+ {
+ JS::AutoSaveExceptionState savedExc(cx);
+ rematFrame =
+ act->getRematerializedFrame(cx, frame.frame(), frame.frameNo());
+ if (!rematFrame) {
+ return;
+ }
+ }
+
+ MOZ_ASSERT(!frame.more());
+
+ if (cx->isClosingGenerator()) {
+ HandleClosingGeneratorReturn(cx, rematFrame, /*frameOk=*/true);
+ } else {
+ cx->clearPropagatingForcedReturn();
+ }
+
+ Value& rval = rematFrame->returnValue();
+ MOZ_RELEASE_ASSERT(!rval.isMagic());
+
+ // Set both framePointer and stackPointer to the address of the
+ // JitFrameLayout.
+ rfe->kind = ExceptionResumeKind::ForcedReturnIon;
+ rfe->framePointer = frame.frame().fp();
+ rfe->stackPointer = frame.frame().fp();
+ rfe->exception = rval;
+
+ act->removeIonFrameRecovery(frame.frame().jsFrame());
+ act->removeRematerializedFrame(frame.frame().fp());
+}
+
+static void HandleExceptionIon(JSContext* cx, const InlineFrameIterator& frame,
+ ResumeFromException* rfe,
+ bool* hitBailoutException) {
+ if (ShouldBailoutForDebugger(cx, frame, *hitBailoutException)) {
+ // We do the following:
+ //
+ // 1. Bailout to baseline to reconstruct a baseline frame.
+ // 2. Resume immediately into the exception tail afterwards, and
+ // handle the exception again with the top frame now a baseline
+ // frame.
+ //
+ // An empty exception info denotes that we're propagating an Ion
+ // exception due to debug mode, which BailoutIonToBaseline needs to
+ // know. This is because we might not be able to fully reconstruct up
+ // to the stack depth at the snapshot, as we could've thrown in the
+ // middle of a call.
+ ExceptionBailoutInfo propagateInfo(cx);
+ if (ExceptionHandlerBailout(cx, frame, rfe, propagateInfo)) {
+ return;
+ }
+ *hitBailoutException = true;
+ }
+
+ RootedScript script(cx, frame.script());
+
+ for (TryNoteIterIon tni(cx, frame); !tni.done(); ++tni) {
+ const TryNote* tn = *tni;
+ switch (tn->kind()) {
+ case TryNoteKind::ForIn:
+ case TryNoteKind::Destructuring:
+ CloseLiveIteratorIon(cx, frame, tn);
+ break;
+
+ case TryNoteKind::Catch:
+ // If we're closing a generator, we have to skip catch blocks.
+ if (cx->isClosingGenerator()) {
+ break;
+ }
+
+ if (cx->isExceptionPending()) {
+ // Ion can compile try-catch, but bailing out to catch
+ // exceptions is slow. Reset the warm-up counter so that if we
+ // catch many exceptions we won't Ion-compile the script.
+ script->resetWarmUpCounterToDelayIonCompilation();
+
+ if (*hitBailoutException) {
+ break;
+ }
+
+ // Bailout at the start of the catch block.
+ jsbytecode* catchPC = script->offsetToPC(tn->start + tn->length);
+ ExceptionBailoutInfo excInfo(cx, frame.frameNo(), catchPC,
+ tn->stackDepth);
+ if (ExceptionHandlerBailout(cx, frame, rfe, excInfo)) {
+ // Record exception locations to allow scope unwinding in
+ // |FinishBailoutToBaseline|
+ MOZ_ASSERT(cx->isExceptionPending());
+ rfe->bailoutInfo->tryPC =
+ UnwindEnvironmentToTryPc(frame.script(), tn);
+ rfe->bailoutInfo->faultPC = frame.pc();
+ return;
+ }
+
+ *hitBailoutException = true;
+ MOZ_ASSERT(cx->isExceptionPending());
+ }
+ break;
+
+ case TryNoteKind::Finally: {
+ if (!cx->isExceptionPending()) {
+ // We don't catch uncatchable exceptions.
+ break;
+ }
+
+ script->resetWarmUpCounterToDelayIonCompilation();
+
+ if (*hitBailoutException) {
+ break;
+ }
+
+ // Bailout at the start of the finally block.
+ jsbytecode* finallyPC = script->offsetToPC(tn->start + tn->length);
+ ExceptionBailoutInfo excInfo(cx, frame.frameNo(), finallyPC,
+ tn->stackDepth);
+
+ RootedValue exception(cx);
+ if (!cx->getPendingException(&exception)) {
+ exception = UndefinedValue();
+ }
+ excInfo.setFinallyException(exception.get());
+ cx->clearPendingException();
+
+ if (ExceptionHandlerBailout(cx, frame, rfe, excInfo)) {
+ // Record exception locations to allow scope unwinding in
+ // |FinishBailoutToBaseline|
+ rfe->bailoutInfo->tryPC =
+ UnwindEnvironmentToTryPc(frame.script(), tn);
+ rfe->bailoutInfo->faultPC = frame.pc();
+ return;
+ }
+
+ *hitBailoutException = true;
+ MOZ_ASSERT(cx->isExceptionPending());
+ break;
+ }
+
+ case TryNoteKind::ForOf:
+ case TryNoteKind::Loop:
+ break;
+
+ // TryNoteKind::ForOfIterclose is handled internally by the try note
+ // iterator.
+ default:
+ MOZ_CRASH("Unexpected try note");
+ }
+ }
+
+ OnLeaveIonFrame(cx, frame, rfe);
+}
+
+static void OnLeaveBaselineFrame(JSContext* cx, const JSJitFrameIter& frame,
+ jsbytecode* pc, ResumeFromException* rfe,
+ bool frameOk) {
+ BaselineFrame* baselineFrame = frame.baselineFrame();
+ bool returnFromThisFrame = jit::DebugEpilogue(cx, baselineFrame, pc, frameOk);
+ if (returnFromThisFrame) {
+ rfe->kind = ExceptionResumeKind::ForcedReturnBaseline;
+ rfe->framePointer = frame.fp();
+ rfe->stackPointer = reinterpret_cast<uint8_t*>(baselineFrame);
+ }
+}
+
+static inline void BaselineFrameAndStackPointersFromTryNote(
+ const TryNote* tn, const JSJitFrameIter& frame, uint8_t** framePointer,
+ uint8_t** stackPointer) {
+ JSScript* script = frame.baselineFrame()->script();
+ *framePointer = frame.fp();
+ *stackPointer = *framePointer - BaselineFrame::Size() -
+ (script->nfixed() + tn->stackDepth) * sizeof(Value);
+}
+
+static void SettleOnTryNote(JSContext* cx, const TryNote* tn,
+ const JSJitFrameIter& frame, EnvironmentIter& ei,
+ ResumeFromException* rfe, jsbytecode** pc) {
+ RootedScript script(cx, frame.baselineFrame()->script());
+
+ // Unwind environment chain (pop block objects).
+ if (cx->isExceptionPending()) {
+ UnwindEnvironment(cx, ei, UnwindEnvironmentToTryPc(script, tn));
+ }
+
+ // Compute base pointer and stack pointer.
+ BaselineFrameAndStackPointersFromTryNote(tn, frame, &rfe->framePointer,
+ &rfe->stackPointer);
+
+ // Compute the pc.
+ *pc = script->offsetToPC(tn->start + tn->length);
+}
+
+class BaselineTryNoteFilter {
+ const JSJitFrameIter& frame_;
+
+ public:
+ explicit BaselineTryNoteFilter(const JSJitFrameIter& frame) : frame_(frame) {}
+ bool operator()(const TryNote* note) {
+ BaselineFrame* frame = frame_.baselineFrame();
+
+ uint32_t numValueSlots = frame_.baselineFrameNumValueSlots();
+ MOZ_RELEASE_ASSERT(numValueSlots >= frame->script()->nfixed());
+
+ uint32_t currDepth = numValueSlots - frame->script()->nfixed();
+ return note->stackDepth <= currDepth;
+ }
+};
+
+class TryNoteIterBaseline : public TryNoteIter<BaselineTryNoteFilter> {
+ public:
+ TryNoteIterBaseline(JSContext* cx, const JSJitFrameIter& frame,
+ jsbytecode* pc)
+ : TryNoteIter(cx, frame.script(), pc, BaselineTryNoteFilter(frame)) {}
+};
+
+// Close all live iterators on a BaselineFrame due to exception unwinding. The
+// pc parameter is updated to where the envs have been unwound to.
+static void CloseLiveIteratorsBaselineForUncatchableException(
+ JSContext* cx, const JSJitFrameIter& frame, jsbytecode* pc) {
+ for (TryNoteIterBaseline tni(cx, frame, pc); !tni.done(); ++tni) {
+ const TryNote* tn = *tni;
+ switch (tn->kind()) {
+ case TryNoteKind::ForIn: {
+ uint8_t* framePointer;
+ uint8_t* stackPointer;
+ BaselineFrameAndStackPointersFromTryNote(tn, frame, &framePointer,
+ &stackPointer);
+ Value iterValue(*(Value*)stackPointer);
+ RootedObject iterObject(cx, &iterValue.toObject());
+ UnwindIteratorForUncatchableException(iterObject);
+ break;
+ }
+
+ default:
+ break;
+ }
+ }
+}
+
+static bool ProcessTryNotesBaseline(JSContext* cx, const JSJitFrameIter& frame,
+ EnvironmentIter& ei,
+ ResumeFromException* rfe, jsbytecode** pc) {
+ MOZ_ASSERT(frame.baselineFrame()->runningInInterpreter(),
+ "Caller must ensure frame is an interpreter frame");
+
+ RootedScript script(cx, frame.baselineFrame()->script());
+
+ for (TryNoteIterBaseline tni(cx, frame, *pc); !tni.done(); ++tni) {
+ const TryNote* tn = *tni;
+
+ MOZ_ASSERT(cx->isExceptionPending());
+ switch (tn->kind()) {
+ case TryNoteKind::Catch: {
+ // If we're closing a generator, we have to skip catch blocks.
+ if (cx->isClosingGenerator()) {
+ break;
+ }
+
+ SettleOnTryNote(cx, tn, frame, ei, rfe, pc);
+
+ // Ion can compile try-catch, but bailing out to catch
+ // exceptions is slow. Reset the warm-up counter so that if we
+ // catch many exceptions we won't Ion-compile the script.
+ script->resetWarmUpCounterToDelayIonCompilation();
+
+ // Resume at the start of the catch block.
+ const BaselineInterpreter& interp =
+ cx->runtime()->jitRuntime()->baselineInterpreter();
+ frame.baselineFrame()->setInterpreterFields(*pc);
+ rfe->kind = ExceptionResumeKind::Catch;
+ rfe->target = interp.interpretOpAddr().value;
+ return true;
+ }
+
+ case TryNoteKind::Finally: {
+ SettleOnTryNote(cx, tn, frame, ei, rfe, pc);
+
+ const BaselineInterpreter& interp =
+ cx->runtime()->jitRuntime()->baselineInterpreter();
+ frame.baselineFrame()->setInterpreterFields(*pc);
+ rfe->kind = ExceptionResumeKind::Finally;
+ rfe->target = interp.interpretOpAddr().value;
+
+ // Drop the exception instead of leaking cross compartment data.
+ if (!cx->getPendingException(
+ MutableHandleValue::fromMarkedLocation(&rfe->exception))) {
+ rfe->exception = UndefinedValue();
+ }
+ cx->clearPendingException();
+ return true;
+ }
+
+ case TryNoteKind::ForIn: {
+ uint8_t* framePointer;
+ uint8_t* stackPointer;
+ BaselineFrameAndStackPointersFromTryNote(tn, frame, &framePointer,
+ &stackPointer);
+ Value iterValue(*reinterpret_cast<Value*>(stackPointer));
+ JSObject* iterObject = &iterValue.toObject();
+ CloseIterator(iterObject);
+ break;
+ }
+
+ case TryNoteKind::Destructuring: {
+ uint8_t* framePointer;
+ uint8_t* stackPointer;
+ BaselineFrameAndStackPointersFromTryNote(tn, frame, &framePointer,
+ &stackPointer);
+ // Note: if this ever changes, also update the
+ // TryNoteKind::Destructuring code in WarpBuilder.cpp!
+ RootedValue doneValue(cx, *(reinterpret_cast<Value*>(stackPointer)));
+ MOZ_RELEASE_ASSERT(!doneValue.isMagic());
+ bool done = ToBoolean(doneValue);
+ if (!done) {
+ Value iterValue(*(reinterpret_cast<Value*>(stackPointer) + 1));
+ RootedObject iterObject(cx, &iterValue.toObject());
+ if (!IteratorCloseForException(cx, iterObject)) {
+ SettleOnTryNote(cx, tn, frame, ei, rfe, pc);
+ return false;
+ }
+ }
+ break;
+ }
+
+ case TryNoteKind::ForOf:
+ case TryNoteKind::Loop:
+ break;
+
+ // TryNoteKind::ForOfIterClose is handled internally by the try note
+ // iterator.
+ default:
+ MOZ_CRASH("Invalid try note");
+ }
+ }
+ return true;
+}
+
+static void HandleExceptionBaseline(JSContext* cx, JSJitFrameIter& frame,
+ CommonFrameLayout* prevFrame,
+ ResumeFromException* rfe) {
+ MOZ_ASSERT(frame.isBaselineJS());
+ MOZ_ASSERT(prevFrame);
+
+ jsbytecode* pc;
+ frame.baselineScriptAndPc(nullptr, &pc);
+
+ // Ensure the BaselineFrame is an interpreter frame. This is easy to do and
+ // simplifies the code below and interaction with DebugModeOSR.
+ //
+ // Note that we never return to this frame via the previous frame's return
+ // address. We could set the return address to nullptr to ensure it's never
+ // used, but the profiler expects a non-null return value for its JitCode map
+ // lookup so we have to use an address in the interpreter code instead.
+ if (!frame.baselineFrame()->runningInInterpreter()) {
+ const BaselineInterpreter& interp =
+ cx->runtime()->jitRuntime()->baselineInterpreter();
+ uint8_t* retAddr = interp.codeRaw();
+ BaselineFrame* baselineFrame = frame.baselineFrame();
+
+ // Suppress profiler sampling while we fix up the frame to ensure the
+ // sampler thread doesn't see an inconsistent state.
+ AutoSuppressProfilerSampling suppressProfilerSampling(cx);
+ baselineFrame->switchFromJitToInterpreterForExceptionHandler(cx, pc);
+ prevFrame->setReturnAddress(retAddr);
+
+ // Ensure the current iterator's resumePCInCurrentFrame_ isn't used
+ // anywhere.
+ frame.setResumePCInCurrentFrame(nullptr);
+ }
+
+ bool frameOk = false;
+ RootedScript script(cx, frame.baselineFrame()->script());
+
+ if (script->hasScriptCounts()) {
+ PCCounts* counts = script->getThrowCounts(pc);
+ // If we failed to allocate, then skip the increment and continue to
+ // handle the exception.
+ if (counts) {
+ counts->numExec()++;
+ }
+ }
+
+ bool hasTryNotes = !script->trynotes().empty();
+
+again:
+ if (cx->isExceptionPending()) {
+ if (!cx->isClosingGenerator()) {
+ if (!DebugAPI::onExceptionUnwind(cx, frame.baselineFrame())) {
+ if (!cx->isExceptionPending()) {
+ goto again;
+ }
+ }
+ // Ensure that the debugger hasn't returned 'true' while clearing the
+ // exception state.
+ MOZ_ASSERT(cx->isExceptionPending());
+ }
+
+ if (hasTryNotes) {
+ EnvironmentIter ei(cx, frame.baselineFrame(), pc);
+ if (!ProcessTryNotesBaseline(cx, frame, ei, rfe, &pc)) {
+ goto again;
+ }
+ if (rfe->kind != ExceptionResumeKind::EntryFrame) {
+ // No need to increment the PCCounts number of execution here,
+ // as the interpreter increments any PCCounts if present.
+ MOZ_ASSERT_IF(script->hasScriptCounts(), script->maybeGetPCCounts(pc));
+ return;
+ }
+ }
+
+ frameOk = HandleClosingGeneratorReturn(cx, frame.baselineFrame(), frameOk);
+ } else {
+ if (hasTryNotes) {
+ CloseLiveIteratorsBaselineForUncatchableException(cx, frame, pc);
+ }
+
+ // We may be propagating a forced return from a debugger hook function.
+ if (MOZ_UNLIKELY(cx->isPropagatingForcedReturn())) {
+ cx->clearPropagatingForcedReturn();
+ frameOk = true;
+ }
+ }
+
+ OnLeaveBaselineFrame(cx, frame, pc, rfe, frameOk);
+}
+
+static JitFrameLayout* GetLastProfilingFrame(ResumeFromException* rfe) {
+ switch (rfe->kind) {
+ case ExceptionResumeKind::EntryFrame:
+ case ExceptionResumeKind::Wasm:
+ case ExceptionResumeKind::WasmCatch:
+ return nullptr;
+
+ // The following all return into Baseline or Ion frames.
+ case ExceptionResumeKind::Catch:
+ case ExceptionResumeKind::Finally:
+ case ExceptionResumeKind::ForcedReturnBaseline:
+ case ExceptionResumeKind::ForcedReturnIon:
+ return reinterpret_cast<JitFrameLayout*>(rfe->framePointer);
+
+ // When resuming into a bailed-out ion frame, use the bailout info to
+ // find the frame we are resuming into.
+ case ExceptionResumeKind::Bailout:
+ return reinterpret_cast<JitFrameLayout*>(rfe->bailoutInfo->incomingStack);
+ }
+
+ MOZ_CRASH("Invalid ResumeFromException type!");
+ return nullptr;
+}
+
+void HandleExceptionWasm(JSContext* cx, wasm::WasmFrameIter* iter,
+ ResumeFromException* rfe) {
+ MOZ_ASSERT(cx->activation()->asJit()->hasWasmExitFP());
+ wasm::HandleThrow(cx, *iter, rfe);
+ MOZ_ASSERT(iter->done());
+}
+
+void HandleException(ResumeFromException* rfe) {
+ JSContext* cx = TlsContext.get();
+
+#ifdef DEBUG
+ cx->runtime()->jitRuntime()->clearDisallowArbitraryCode();
+#endif
+
+ auto resetProfilerFrame = mozilla::MakeScopeExit([=] {
+ if (!cx->runtime()->jitRuntime()->isProfilerInstrumentationEnabled(
+ cx->runtime())) {
+ return;
+ }
+
+ MOZ_ASSERT(cx->jitActivation == cx->profilingActivation());
+
+ auto* lastProfilingFrame = GetLastProfilingFrame(rfe);
+ cx->jitActivation->setLastProfilingFrame(lastProfilingFrame);
+ });
+
+ rfe->kind = ExceptionResumeKind::EntryFrame;
+
+ JitSpew(JitSpew_IonInvalidate, "handling exception");
+
+ JitActivation* activation = cx->activation()->asJit();
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ if (JitOptions.checkOsiPointRegisters) {
+ activation->setCheckRegs(false);
+ }
+#endif
+
+ JitFrameIter iter(cx->activation()->asJit(),
+ /* mustUnwindActivation = */ true);
+ CommonFrameLayout* prevJitFrame = nullptr;
+ while (!iter.done()) {
+ if (iter.isWasm()) {
+ prevJitFrame = nullptr;
+ HandleExceptionWasm(cx, &iter.asWasm(), rfe);
+ // If a wasm try-catch handler is found, we can immediately jump to it
+ // and quit iterating through the stack.
+ if (rfe->kind == ExceptionResumeKind::WasmCatch) {
+ return;
+ }
+ if (!iter.done()) {
+ ++iter;
+ }
+ continue;
+ }
+
+ JSJitFrameIter& frame = iter.asJSJit();
+
+ // JIT code can enter same-compartment realms, so reset cx->realm to
+ // this frame's realm.
+ if (frame.isScripted()) {
+ cx->setRealmForJitExceptionHandler(iter.realm());
+ }
+
+ if (frame.isIonJS()) {
+ // Search each inlined frame for live iterator objects, and close
+ // them.
+ InlineFrameIterator frames(cx, &frame);
+
+ // Invalidation state will be the same for all inlined scripts in the
+ // frame.
+ IonScript* ionScript = nullptr;
+ bool invalidated = frame.checkInvalidation(&ionScript);
+
+ // If we hit OOM or overrecursion while bailing out, we don't
+ // attempt to bail out a second time for this Ion frame. Just unwind
+ // and continue at the next frame.
+ bool hitBailoutException = false;
+ for (;;) {
+ HandleExceptionIon(cx, frames, rfe, &hitBailoutException);
+
+ if (rfe->kind == ExceptionResumeKind::Bailout ||
+ rfe->kind == ExceptionResumeKind::ForcedReturnIon) {
+ if (invalidated) {
+ ionScript->decrementInvalidationCount(cx->gcContext());
+ }
+ return;
+ }
+
+ MOZ_ASSERT(rfe->kind == ExceptionResumeKind::EntryFrame);
+
+ // When profiling, each frame popped needs a notification that
+ // the function has exited, so invoke the probe that a function
+ // is exiting.
+
+ JSScript* script = frames.script();
+ probes::ExitScript(cx, script, script->function(),
+ /* popProfilerFrame = */ false);
+ if (!frames.more()) {
+ break;
+ }
+ ++frames;
+ }
+
+ // Remove left-over state which might have been needed for bailout.
+ activation->removeIonFrameRecovery(frame.jsFrame());
+ activation->removeRematerializedFrame(frame.fp());
+
+ // If invalidated, decrement the number of frames remaining on the
+ // stack for the given IonScript.
+ if (invalidated) {
+ ionScript->decrementInvalidationCount(cx->gcContext());
+ }
+
+ } else if (frame.isBaselineJS()) {
+ HandleExceptionBaseline(cx, frame, prevJitFrame, rfe);
+
+ if (rfe->kind != ExceptionResumeKind::EntryFrame &&
+ rfe->kind != ExceptionResumeKind::ForcedReturnBaseline) {
+ return;
+ }
+
+ // Unwind profiler pseudo-stack
+ JSScript* script = frame.script();
+ probes::ExitScript(cx, script, script->function(),
+ /* popProfilerFrame = */ false);
+
+ if (rfe->kind == ExceptionResumeKind::ForcedReturnBaseline) {
+ return;
+ }
+ }
+
+ prevJitFrame = frame.current();
+ ++iter;
+ }
+
+ // Wasm sets its own value of SP in HandleExceptionWasm.
+ if (iter.isJSJit()) {
+ MOZ_ASSERT(rfe->kind == ExceptionResumeKind::EntryFrame);
+ rfe->framePointer = iter.asJSJit().current()->callerFramePtr();
+ rfe->stackPointer =
+ iter.asJSJit().fp() + CommonFrameLayout::offsetOfReturnAddress();
+ }
+}
+
+// Turns a JitFrameLayout into an UnwoundJit ExitFrameLayout.
+void EnsureUnwoundJitExitFrame(JitActivation* act, JitFrameLayout* frame) {
+ ExitFrameLayout* exitFrame = reinterpret_cast<ExitFrameLayout*>(frame);
+
+ if (act->jsExitFP() == (uint8_t*)frame) {
+ // If we already called this function for the current frame, do
+ // nothing.
+ MOZ_ASSERT(exitFrame->isUnwoundJitExit());
+ return;
+ }
+
+#ifdef DEBUG
+ JSJitFrameIter iter(act);
+ while (!iter.isScripted()) {
+ ++iter;
+ }
+ MOZ_ASSERT(iter.current() == frame, "|frame| must be the top JS frame");
+
+ MOZ_ASSERT(!!act->jsExitFP());
+ MOZ_ASSERT((uint8_t*)exitFrame->footer() >= act->jsExitFP(),
+ "Must have space for ExitFooterFrame before jsExitFP");
+#endif
+
+ act->setJSExitFP((uint8_t*)frame);
+ exitFrame->footer()->setUnwoundJitExitFrame();
+ MOZ_ASSERT(exitFrame->isUnwoundJitExit());
+}
+
+JSScript* MaybeForwardedScriptFromCalleeToken(CalleeToken token) {
+ switch (GetCalleeTokenTag(token)) {
+ case CalleeToken_Script:
+ return MaybeForwarded(CalleeTokenToScript(token));
+ case CalleeToken_Function:
+ case CalleeToken_FunctionConstructing: {
+ JSFunction* fun = MaybeForwarded(CalleeTokenToFunction(token));
+ return MaybeForwarded(fun)->nonLazyScript();
+ }
+ }
+ MOZ_CRASH("invalid callee token tag");
+}
+
+CalleeToken TraceCalleeToken(JSTracer* trc, CalleeToken token) {
+ switch (CalleeTokenTag tag = GetCalleeTokenTag(token)) {
+ case CalleeToken_Function:
+ case CalleeToken_FunctionConstructing: {
+ JSFunction* fun = CalleeTokenToFunction(token);
+ TraceRoot(trc, &fun, "jit-callee");
+ return CalleeToToken(fun, tag == CalleeToken_FunctionConstructing);
+ }
+ case CalleeToken_Script: {
+ JSScript* script = CalleeTokenToScript(token);
+ TraceRoot(trc, &script, "jit-script");
+ return CalleeToToken(script);
+ }
+ default:
+ MOZ_CRASH("unknown callee token type");
+ }
+}
+
+uintptr_t* JitFrameLayout::slotRef(SafepointSlotEntry where) {
+ if (where.stack) {
+ return (uintptr_t*)((uint8_t*)this - where.slot);
+ }
+ return (uintptr_t*)((uint8_t*)thisAndActualArgs() + where.slot);
+}
+
+#ifdef JS_NUNBOX32
+static inline uintptr_t ReadAllocation(const JSJitFrameIter& frame,
+ const LAllocation* a) {
+ if (a->isGeneralReg()) {
+ Register reg = a->toGeneralReg()->reg();
+ return frame.machineState().read(reg);
+ }
+ return *frame.jsFrame()->slotRef(SafepointSlotEntry(a));
+}
+#endif
+
+static void TraceThisAndArguments(JSTracer* trc, const JSJitFrameIter& frame,
+ JitFrameLayout* layout) {
+ // Trace |this| and any extra actual arguments for an Ion frame. Tracing
+ // of formal arguments is taken care of by the frame's safepoint/snapshot,
+ // except when the script might have lazy arguments or rest, in which case
+ // we trace them as well. We also have to trace formals if we have a
+ // LazyLink frame or an InterpreterStub frame or a special JSJit to wasm
+ // frame (since wasm doesn't use snapshots).
+
+ if (!CalleeTokenIsFunction(layout->calleeToken())) {
+ return;
+ }
+
+ size_t nargs = layout->numActualArgs();
+ size_t nformals = 0;
+
+ JSFunction* fun = CalleeTokenToFunction(layout->calleeToken());
+ if (frame.type() != FrameType::JSJitToWasm &&
+ !frame.isExitFrameLayout<CalledFromJitExitFrameLayout>() &&
+ !fun->nonLazyScript()->mayReadFrameArgsDirectly()) {
+ nformals = fun->nargs();
+ }
+
+ size_t newTargetOffset = std::max(nargs, fun->nargs());
+
+ Value* argv = layout->thisAndActualArgs();
+
+ // Trace |this|.
+ TraceRoot(trc, argv, "ion-thisv");
+
+ // Trace actual arguments beyond the formals. Note + 1 for thisv.
+ for (size_t i = nformals + 1; i < nargs + 1; i++) {
+ TraceRoot(trc, &argv[i], "ion-argv");
+ }
+
+ // Always trace the new.target from the frame. It's not in the snapshots.
+ // +1 to pass |this|
+ if (CalleeTokenIsConstructing(layout->calleeToken())) {
+ TraceRoot(trc, &argv[1 + newTargetOffset], "ion-newTarget");
+ }
+}
+
+#ifdef JS_NUNBOX32
+static inline void WriteAllocation(const JSJitFrameIter& frame,
+ const LAllocation* a, uintptr_t value) {
+ if (a->isGeneralReg()) {
+ Register reg = a->toGeneralReg()->reg();
+ frame.machineState().write(reg, value);
+ } else {
+ *frame.jsFrame()->slotRef(SafepointSlotEntry(a)) = value;
+ }
+}
+#endif
+
+static void TraceIonJSFrame(JSTracer* trc, const JSJitFrameIter& frame) {
+ JitFrameLayout* layout = (JitFrameLayout*)frame.fp();
+
+ layout->replaceCalleeToken(TraceCalleeToken(trc, layout->calleeToken()));
+
+ IonScript* ionScript = nullptr;
+ if (frame.checkInvalidation(&ionScript)) {
+ // This frame has been invalidated, meaning that its IonScript is no
+ // longer reachable through the callee token (JSFunction/JSScript->ion
+ // is now nullptr or recompiled). Manually trace it here.
+ ionScript->trace(trc);
+ } else {
+ ionScript = frame.ionScriptFromCalleeToken();
+ }
+
+ TraceThisAndArguments(trc, frame, frame.jsFrame());
+
+ const SafepointIndex* si =
+ ionScript->getSafepointIndex(frame.resumePCinCurrentFrame());
+
+ SafepointReader safepoint(ionScript, si);
+
+ // Scan through slots which contain pointers (or on punboxing systems,
+ // actual values).
+ SafepointSlotEntry entry;
+
+ while (safepoint.getGcSlot(&entry)) {
+ uintptr_t* ref = layout->slotRef(entry);
+ TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(ref),
+ "ion-gc-slot");
+ }
+
+ uintptr_t* spill = frame.spillBase();
+ LiveGeneralRegisterSet gcRegs = safepoint.gcSpills();
+ LiveGeneralRegisterSet valueRegs = safepoint.valueSpills();
+ for (GeneralRegisterBackwardIterator iter(safepoint.allGprSpills());
+ iter.more(); ++iter) {
+ --spill;
+ if (gcRegs.has(*iter)) {
+ TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(spill),
+ "ion-gc-spill");
+ } else if (valueRegs.has(*iter)) {
+ TraceRoot(trc, reinterpret_cast<Value*>(spill), "ion-value-spill");
+ }
+ }
+
+#ifdef JS_PUNBOX64
+ while (safepoint.getValueSlot(&entry)) {
+ Value* v = (Value*)layout->slotRef(entry);
+ TraceRoot(trc, v, "ion-gc-slot");
+ }
+#else
+ LAllocation type, payload;
+ while (safepoint.getNunboxSlot(&type, &payload)) {
+ JSValueTag tag = JSValueTag(ReadAllocation(frame, &type));
+ uintptr_t rawPayload = ReadAllocation(frame, &payload);
+
+ Value v = Value::fromTagAndPayload(tag, rawPayload);
+ TraceRoot(trc, &v, "ion-torn-value");
+
+ if (v != Value::fromTagAndPayload(tag, rawPayload)) {
+ // GC moved the value, replace the stored payload.
+ rawPayload = v.toNunboxPayload();
+ WriteAllocation(frame, &payload, rawPayload);
+ }
+ }
+#endif
+}
+
+static void TraceBailoutFrame(JSTracer* trc, const JSJitFrameIter& frame) {
+ JitFrameLayout* layout = (JitFrameLayout*)frame.fp();
+
+ layout->replaceCalleeToken(TraceCalleeToken(trc, layout->calleeToken()));
+
+ // We have to trace the list of actual arguments, as only formal arguments
+ // are represented in the Snapshot.
+ TraceThisAndArguments(trc, frame, frame.jsFrame());
+
+ // Under a bailout, do not have a Safepoint to only iterate over GC-things.
+ // Thus we use a SnapshotIterator to trace all the locations which would be
+ // used to reconstruct the Baseline frame.
+ //
+ // Note that at the time where this function is called, we have not yet
+ // started to reconstruct baseline frames.
+
+ // The vector of recover instructions is already traced as part of the
+ // JitActivation.
+ SnapshotIterator snapIter(frame,
+ frame.activation()->bailoutData()->machineState());
+
+ // For each instruction, we read the allocations without evaluating the
+ // recover instruction, nor reconstructing the frame. We are only looking at
+ // tracing readable allocations.
+ while (true) {
+ while (snapIter.moreAllocations()) {
+ snapIter.traceAllocation(trc);
+ }
+
+ if (!snapIter.moreInstructions()) {
+ break;
+ }
+ snapIter.nextInstruction();
+ }
+}
+
+static void UpdateIonJSFrameForMinorGC(JSRuntime* rt,
+ const JSJitFrameIter& frame) {
+ // Minor GCs may move slots/elements allocated in the nursery. Update
+ // any slots/elements pointers stored in this frame.
+
+ JitFrameLayout* layout = (JitFrameLayout*)frame.fp();
+
+ IonScript* ionScript = nullptr;
+ if (frame.checkInvalidation(&ionScript)) {
+ // This frame has been invalidated, meaning that its IonScript is no
+ // longer reachable through the callee token (JSFunction/JSScript->ion
+ // is now nullptr or recompiled).
+ } else {
+ ionScript = frame.ionScriptFromCalleeToken();
+ }
+
+ Nursery& nursery = rt->gc.nursery();
+
+ const SafepointIndex* si =
+ ionScript->getSafepointIndex(frame.resumePCinCurrentFrame());
+ SafepointReader safepoint(ionScript, si);
+
+ LiveGeneralRegisterSet slotsRegs = safepoint.slotsOrElementsSpills();
+ uintptr_t* spill = frame.spillBase();
+ for (GeneralRegisterBackwardIterator iter(safepoint.allGprSpills());
+ iter.more(); ++iter) {
+ --spill;
+ if (slotsRegs.has(*iter)) {
+ nursery.forwardBufferPointer(spill);
+ }
+ }
+
+ // Skip to the right place in the safepoint
+ SafepointSlotEntry entry;
+ while (safepoint.getGcSlot(&entry)) {
+ }
+
+#ifdef JS_PUNBOX64
+ while (safepoint.getValueSlot(&entry)) {
+ }
+#else
+ LAllocation type, payload;
+ while (safepoint.getNunboxSlot(&type, &payload)) {
+ }
+#endif
+
+ while (safepoint.getSlotsOrElementsSlot(&entry)) {
+ nursery.forwardBufferPointer(layout->slotRef(entry));
+ }
+}
+
+static void TraceBaselineStubFrame(JSTracer* trc, const JSJitFrameIter& frame) {
+ // Trace the ICStub pointer stored in the stub frame. This is necessary
+ // so that we don't destroy the stub code after unlinking the stub.
+
+ MOZ_ASSERT(frame.type() == FrameType::BaselineStub);
+ BaselineStubFrameLayout* layout = (BaselineStubFrameLayout*)frame.fp();
+
+ if (ICStub* stub = layout->maybeStubPtr()) {
+ if (stub->isFallback()) {
+ // Fallback stubs use runtime-wide trampoline code we don't need to trace.
+ MOZ_ASSERT(stub->usesTrampolineCode());
+ } else {
+ MOZ_ASSERT(stub->toCacheIRStub()->makesGCCalls());
+ stub->toCacheIRStub()->trace(trc);
+ }
+ }
+}
+
+static void TraceIonICCallFrame(JSTracer* trc, const JSJitFrameIter& frame) {
+ MOZ_ASSERT(frame.type() == FrameType::IonICCall);
+ IonICCallFrameLayout* layout = (IonICCallFrameLayout*)frame.fp();
+ TraceRoot(trc, layout->stubCode(), "ion-ic-call-code");
+}
+
+#if defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32)
+uint8_t* alignDoubleSpill(uint8_t* pointer) {
+ uintptr_t address = reinterpret_cast<uintptr_t>(pointer);
+ address &= ~(uintptr_t(ABIStackAlignment) - 1);
+ return reinterpret_cast<uint8_t*>(address);
+}
+#endif
+
+#ifdef JS_CODEGEN_MIPS32
+static void TraceJitExitFrameCopiedArguments(JSTracer* trc,
+ const VMFunctionData* f,
+ ExitFooterFrame* footer) {
+ uint8_t* doubleArgs = footer->alignedForABI();
+ if (f->outParam == Type_Handle) {
+ doubleArgs -= sizeof(Value);
+ }
+ doubleArgs -= f->doubleByRefArgs() * sizeof(double);
+
+ for (uint32_t explicitArg = 0; explicitArg < f->explicitArgs; explicitArg++) {
+ if (f->argProperties(explicitArg) == VMFunctionData::DoubleByRef) {
+ // Arguments with double size can only have RootValue type.
+ if (f->argRootType(explicitArg) == VMFunctionData::RootValue) {
+ TraceRoot(trc, reinterpret_cast<Value*>(doubleArgs), "ion-vm-args");
+ } else {
+ MOZ_ASSERT(f->argRootType(explicitArg) == VMFunctionData::RootNone);
+ }
+ doubleArgs += sizeof(double);
+ }
+ }
+}
+#else
+static void TraceJitExitFrameCopiedArguments(JSTracer* trc,
+ const VMFunctionData* f,
+ ExitFooterFrame* footer) {
+ // This is NO-OP on other platforms.
+}
+#endif
+
+static void TraceJitExitFrame(JSTracer* trc, const JSJitFrameIter& frame) {
+ ExitFooterFrame* footer = frame.exitFrame()->footer();
+
+ // This corresponds to the case where we have build a fake exit frame which
+ // handles the case of a native function call. We need to trace the argument
+ // vector of the function call, and also new.target if it was a constructing
+ // call.
+ if (frame.isExitFrameLayout<NativeExitFrameLayout>()) {
+ NativeExitFrameLayout* native =
+ frame.exitFrame()->as<NativeExitFrameLayout>();
+ size_t len = native->argc() + 2;
+ Value* vp = native->vp();
+ TraceRootRange(trc, len, vp, "ion-native-args");
+ if (frame.isExitFrameLayout<ConstructNativeExitFrameLayout>()) {
+ TraceRoot(trc, vp + len, "ion-native-new-target");
+ }
+ return;
+ }
+
+ if (frame.isExitFrameLayout<IonOOLNativeExitFrameLayout>()) {
+ IonOOLNativeExitFrameLayout* oolnative =
+ frame.exitFrame()->as<IonOOLNativeExitFrameLayout>();
+ TraceRoot(trc, oolnative->stubCode(), "ion-ool-native-code");
+ TraceRoot(trc, oolnative->vp(), "iol-ool-native-vp");
+ size_t len = oolnative->argc() + 1;
+ TraceRootRange(trc, len, oolnative->thisp(), "ion-ool-native-thisargs");
+ return;
+ }
+
+ if (frame.isExitFrameLayout<IonOOLProxyExitFrameLayout>()) {
+ IonOOLProxyExitFrameLayout* oolproxy =
+ frame.exitFrame()->as<IonOOLProxyExitFrameLayout>();
+ TraceRoot(trc, oolproxy->stubCode(), "ion-ool-proxy-code");
+ TraceRoot(trc, oolproxy->vp(), "ion-ool-proxy-vp");
+ TraceRoot(trc, oolproxy->id(), "ion-ool-proxy-id");
+ TraceRoot(trc, oolproxy->proxy(), "ion-ool-proxy-proxy");
+ return;
+ }
+
+ if (frame.isExitFrameLayout<IonDOMExitFrameLayout>()) {
+ IonDOMExitFrameLayout* dom = frame.exitFrame()->as<IonDOMExitFrameLayout>();
+ TraceRoot(trc, dom->thisObjAddress(), "ion-dom-args");
+ if (dom->isMethodFrame()) {
+ IonDOMMethodExitFrameLayout* method =
+ reinterpret_cast<IonDOMMethodExitFrameLayout*>(dom);
+ size_t len = method->argc() + 2;
+ Value* vp = method->vp();
+ TraceRootRange(trc, len, vp, "ion-dom-args");
+ } else {
+ TraceRoot(trc, dom->vp(), "ion-dom-args");
+ }
+ return;
+ }
+
+ if (frame.isExitFrameLayout<CalledFromJitExitFrameLayout>()) {
+ auto* layout = frame.exitFrame()->as<CalledFromJitExitFrameLayout>();
+ JitFrameLayout* jsLayout = layout->jsFrame();
+ jsLayout->replaceCalleeToken(
+ TraceCalleeToken(trc, jsLayout->calleeToken()));
+ TraceThisAndArguments(trc, frame, jsLayout);
+ return;
+ }
+
+ if (frame.isExitFrameLayout<DirectWasmJitCallFrameLayout>()) {
+ // Nothing needs to be traced here at the moment -- the arguments to the
+ // callee are traced by the callee, and the inlined caller does not push
+ // anything else.
+ return;
+ }
+
+ if (frame.isBareExit() || frame.isUnwoundJitExit()) {
+ // Nothing to trace. Fake exit frame pushed for VM functions with
+ // nothing to trace on the stack or unwound JitFrameLayout.
+ return;
+ }
+
+ MOZ_ASSERT(frame.exitFrame()->isWrapperExit());
+
+ const VMFunctionData* f = footer->function();
+ MOZ_ASSERT(f);
+
+ // Trace arguments of the VM wrapper.
+ uint8_t* argBase = frame.exitFrame()->argBase();
+ for (uint32_t explicitArg = 0; explicitArg < f->explicitArgs; explicitArg++) {
+ switch (f->argRootType(explicitArg)) {
+ case VMFunctionData::RootNone:
+ break;
+ case VMFunctionData::RootObject: {
+ // Sometimes we can bake in HandleObjects to nullptr.
+ JSObject** pobj = reinterpret_cast<JSObject**>(argBase);
+ if (*pobj) {
+ TraceRoot(trc, pobj, "ion-vm-args");
+ }
+ break;
+ }
+ case VMFunctionData::RootString:
+ TraceRoot(trc, reinterpret_cast<JSString**>(argBase), "ion-vm-args");
+ break;
+ case VMFunctionData::RootValue:
+ TraceRoot(trc, reinterpret_cast<Value*>(argBase), "ion-vm-args");
+ break;
+ case VMFunctionData::RootId:
+ TraceRoot(trc, reinterpret_cast<jsid*>(argBase), "ion-vm-args");
+ break;
+ case VMFunctionData::RootCell:
+ TraceGenericPointerRoot(trc, reinterpret_cast<gc::Cell**>(argBase),
+ "ion-vm-args");
+ break;
+ case VMFunctionData::RootBigInt:
+ TraceRoot(trc, reinterpret_cast<JS::BigInt**>(argBase), "ion-vm-args");
+ break;
+ }
+
+ switch (f->argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ case VMFunctionData::WordByRef:
+ argBase += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByValue:
+ case VMFunctionData::DoubleByRef:
+ argBase += 2 * sizeof(void*);
+ break;
+ }
+ }
+
+ if (f->outParam == Type_Handle) {
+ switch (f->outParamRootType) {
+ case VMFunctionData::RootNone:
+ MOZ_CRASH("Handle outparam must have root type");
+ case VMFunctionData::RootObject:
+ TraceRoot(trc, footer->outParam<JSObject*>(), "ion-vm-out");
+ break;
+ case VMFunctionData::RootString:
+ TraceRoot(trc, footer->outParam<JSString*>(), "ion-vm-out");
+ break;
+ case VMFunctionData::RootValue:
+ TraceRoot(trc, footer->outParam<Value>(), "ion-vm-outvp");
+ break;
+ case VMFunctionData::RootId:
+ TraceRoot(trc, footer->outParam<jsid>(), "ion-vm-outvp");
+ break;
+ case VMFunctionData::RootCell:
+ TraceGenericPointerRoot(trc, footer->outParam<gc::Cell*>(),
+ "ion-vm-out");
+ break;
+ case VMFunctionData::RootBigInt:
+ TraceRoot(trc, footer->outParam<JS::BigInt*>(), "ion-vm-out");
+ break;
+ }
+ }
+
+ TraceJitExitFrameCopiedArguments(trc, f, footer);
+}
+
+static void TraceRectifierFrame(JSTracer* trc, const JSJitFrameIter& frame) {
+ // Trace thisv.
+ //
+ // Baseline JIT code generated as part of the ICCall_Fallback stub may use
+ // it if we're calling a constructor that returns a primitive value.
+ RectifierFrameLayout* layout = (RectifierFrameLayout*)frame.fp();
+ TraceRoot(trc, &layout->thisv(), "rectifier-thisv");
+}
+
+static void TraceJSJitToWasmFrame(JSTracer* trc, const JSJitFrameIter& frame) {
+ // This is doing a subset of TraceIonJSFrame, since the callee doesn't
+ // have a script.
+ JitFrameLayout* layout = (JitFrameLayout*)frame.fp();
+ layout->replaceCalleeToken(TraceCalleeToken(trc, layout->calleeToken()));
+ TraceThisAndArguments(trc, frame, layout);
+}
+
+static void TraceJitActivation(JSTracer* trc, JitActivation* activation) {
+#ifdef CHECK_OSIPOINT_REGISTERS
+ if (JitOptions.checkOsiPointRegisters) {
+ // GC can modify spilled registers, breaking our register checks.
+ // To handle this, we disable these checks for the current VM call
+ // when a GC happens.
+ activation->setCheckRegs(false);
+ }
+#endif
+
+ activation->traceRematerializedFrames(trc);
+ activation->traceIonRecovery(trc);
+
+ // This is used for sanity checking continuity of the sequence of wasm stack
+ // maps as we unwind. It has no functional purpose.
+ uintptr_t highestByteVisitedInPrevWasmFrame = 0;
+
+ for (JitFrameIter frames(activation); !frames.done(); ++frames) {
+ if (frames.isJSJit()) {
+ const JSJitFrameIter& jitFrame = frames.asJSJit();
+ switch (jitFrame.type()) {
+ case FrameType::Exit:
+ TraceJitExitFrame(trc, jitFrame);
+ break;
+ case FrameType::BaselineJS:
+ jitFrame.baselineFrame()->trace(trc, jitFrame);
+ break;
+ case FrameType::IonJS:
+ TraceIonJSFrame(trc, jitFrame);
+ break;
+ case FrameType::BaselineStub:
+ TraceBaselineStubFrame(trc, jitFrame);
+ break;
+ case FrameType::Bailout:
+ TraceBailoutFrame(trc, jitFrame);
+ break;
+ case FrameType::Rectifier:
+ TraceRectifierFrame(trc, jitFrame);
+ break;
+ case FrameType::IonICCall:
+ TraceIonICCallFrame(trc, jitFrame);
+ break;
+ case FrameType::WasmToJSJit:
+ // Ignore: this is a special marker used to let the
+ // JitFrameIter know the frame above is a wasm frame, handled
+ // in the next iteration.
+ break;
+ case FrameType::JSJitToWasm:
+ TraceJSJitToWasmFrame(trc, jitFrame);
+ break;
+ default:
+ MOZ_CRASH("unexpected frame type");
+ }
+ highestByteVisitedInPrevWasmFrame = 0; /* "unknown" */
+ } else {
+ MOZ_ASSERT(frames.isWasm());
+ uint8_t* nextPC = frames.resumePCinCurrentFrame();
+ MOZ_ASSERT(nextPC != 0);
+ wasm::WasmFrameIter& wasmFrameIter = frames.asWasm();
+ wasm::Instance* instance = wasmFrameIter.instance();
+ wasm::TraceInstanceEdge(trc, instance, "WasmFrameIter instance");
+ highestByteVisitedInPrevWasmFrame = instance->traceFrame(
+ trc, wasmFrameIter, nextPC, highestByteVisitedInPrevWasmFrame);
+ }
+ }
+}
+
+void TraceJitActivations(JSContext* cx, JSTracer* trc) {
+ for (JitActivationIterator activations(cx); !activations.done();
+ ++activations) {
+ TraceJitActivation(trc, activations->asJit());
+ }
+}
+
+void UpdateJitActivationsForMinorGC(JSRuntime* rt) {
+ MOZ_ASSERT(JS::RuntimeHeapIsMinorCollecting());
+ JSContext* cx = rt->mainContextFromOwnThread();
+ for (JitActivationIterator activations(cx); !activations.done();
+ ++activations) {
+ for (OnlyJSJitFrameIter iter(activations); !iter.done(); ++iter) {
+ if (iter.frame().type() == FrameType::IonJS) {
+ UpdateIonJSFrameForMinorGC(rt, iter.frame());
+ }
+ }
+ }
+}
+
+JSScript* GetTopJitJSScript(JSContext* cx) {
+ JSJitFrameIter frame(cx->activation()->asJit());
+ MOZ_ASSERT(frame.type() == FrameType::Exit);
+ ++frame;
+
+ if (frame.isBaselineStub()) {
+ ++frame;
+ MOZ_ASSERT(frame.isBaselineJS());
+ }
+
+ MOZ_ASSERT(frame.isScripted());
+ return frame.script();
+}
+
+void GetPcScript(JSContext* cx, JSScript** scriptRes, jsbytecode** pcRes) {
+ JitSpew(JitSpew_IonSnapshots, "Recover PC & Script from the last frame.");
+
+ // Recover the return address so that we can look it up in the
+ // PcScriptCache, as script/pc computation is expensive.
+ JitActivationIterator actIter(cx);
+ OnlyJSJitFrameIter it(actIter);
+ uint8_t* retAddr;
+ if (it.frame().isExitFrame()) {
+ ++it;
+
+ // Skip rectifier frames.
+ if (it.frame().isRectifier()) {
+ ++it;
+ MOZ_ASSERT(it.frame().isBaselineStub() || it.frame().isBaselineJS() ||
+ it.frame().isIonJS());
+ }
+
+ // Skip Baseline/Ion stub and IC call frames.
+ if (it.frame().isBaselineStub()) {
+ ++it;
+ MOZ_ASSERT(it.frame().isBaselineJS());
+ } else if (it.frame().isIonICCall()) {
+ ++it;
+ MOZ_ASSERT(it.frame().isIonJS());
+ }
+
+ MOZ_ASSERT(it.frame().isBaselineJS() || it.frame().isIonJS());
+
+ // Don't use the return address and the cache if the BaselineFrame is
+ // running in the Baseline Interpreter. In this case the bytecode pc is
+ // cheap to get, so we won't benefit from the cache, and the return address
+ // does not map to a single bytecode pc.
+ if (it.frame().isBaselineJS() &&
+ it.frame().baselineFrame()->runningInInterpreter()) {
+ it.frame().baselineScriptAndPc(scriptRes, pcRes);
+ return;
+ }
+
+ retAddr = it.frame().resumePCinCurrentFrame();
+ } else {
+ MOZ_ASSERT(it.frame().isBailoutJS());
+ retAddr = it.frame().returnAddress();
+ }
+
+ MOZ_ASSERT(retAddr);
+
+ uint32_t hash = PcScriptCache::Hash(retAddr);
+
+ // Lazily initialize the cache. The allocation may safely fail and will not
+ // GC.
+ if (MOZ_UNLIKELY(cx->ionPcScriptCache == nullptr)) {
+ cx->ionPcScriptCache =
+ MakeUnique<PcScriptCache>(cx->runtime()->gc.gcNumber());
+ }
+
+ if (cx->ionPcScriptCache.ref() &&
+ cx->ionPcScriptCache->get(cx->runtime(), hash, retAddr, scriptRes,
+ pcRes)) {
+ return;
+ }
+
+ // Lookup failed: undertake expensive process to determine script and pc.
+ if (it.frame().isIonJS() || it.frame().isBailoutJS()) {
+ InlineFrameIterator ifi(cx, &it.frame());
+ *scriptRes = ifi.script();
+ *pcRes = ifi.pc();
+ } else {
+ MOZ_ASSERT(it.frame().isBaselineJS());
+ it.frame().baselineScriptAndPc(scriptRes, pcRes);
+ }
+
+ // Add entry to cache.
+ if (cx->ionPcScriptCache.ref()) {
+ cx->ionPcScriptCache->add(hash, retAddr, *pcRes, *scriptRes);
+ }
+}
+
+RInstructionResults::RInstructionResults(JitFrameLayout* fp)
+ : results_(nullptr), fp_(fp), initialized_(false) {}
+
+RInstructionResults::RInstructionResults(RInstructionResults&& src)
+ : results_(std::move(src.results_)),
+ fp_(src.fp_),
+ initialized_(src.initialized_) {
+ src.initialized_ = false;
+}
+
+RInstructionResults& RInstructionResults::operator=(RInstructionResults&& rhs) {
+ MOZ_ASSERT(&rhs != this, "self-moves are prohibited");
+ this->~RInstructionResults();
+ new (this) RInstructionResults(std::move(rhs));
+ return *this;
+}
+
+RInstructionResults::~RInstructionResults() {
+ // results_ is freed by the UniquePtr.
+}
+
+bool RInstructionResults::init(JSContext* cx, uint32_t numResults) {
+ if (numResults) {
+ results_ = cx->make_unique<Values>();
+ if (!results_) {
+ return false;
+ }
+ if (!results_->growBy(numResults)) {
+ ReportOutOfMemory(cx);
+ return false;
+ }
+
+ Value guard = MagicValue(JS_ION_BAILOUT);
+ for (size_t i = 0; i < numResults; i++) {
+ (*results_)[i].init(guard);
+ }
+ }
+
+ initialized_ = true;
+ return true;
+}
+
+bool RInstructionResults::isInitialized() const { return initialized_; }
+
+size_t RInstructionResults::length() const { return results_->length(); }
+
+JitFrameLayout* RInstructionResults::frame() const {
+ MOZ_ASSERT(fp_);
+ return fp_;
+}
+
+HeapPtr<Value>& RInstructionResults::operator[](size_t index) {
+ return (*results_)[index];
+}
+
+void RInstructionResults::trace(JSTracer* trc) {
+ // Note: The vector necessary exists, otherwise this object would not have
+ // been stored on the activation from where the trace function is called.
+ TraceRange(trc, results_->length(), results_->begin(), "ion-recover-results");
+}
+
+SnapshotIterator::SnapshotIterator(const JSJitFrameIter& iter,
+ const MachineState* machineState)
+ : snapshot_(iter.ionScript()->snapshots(), iter.snapshotOffset(),
+ iter.ionScript()->snapshotsRVATableSize(),
+ iter.ionScript()->snapshotsListSize()),
+ recover_(snapshot_, iter.ionScript()->recovers(),
+ iter.ionScript()->recoversSize()),
+ fp_(iter.jsFrame()),
+ machine_(machineState),
+ ionScript_(iter.ionScript()),
+ instructionResults_(nullptr) {}
+
+SnapshotIterator::SnapshotIterator()
+ : snapshot_(nullptr, 0, 0, 0),
+ recover_(snapshot_, nullptr, 0),
+ fp_(nullptr),
+ machine_(nullptr),
+ ionScript_(nullptr),
+ instructionResults_(nullptr) {}
+
+uintptr_t SnapshotIterator::fromStack(int32_t offset) const {
+ return ReadFrameSlot(fp_, offset);
+}
+
+static Value FromObjectPayload(uintptr_t payload) {
+ MOZ_ASSERT(payload != 0);
+ return ObjectValue(*reinterpret_cast<JSObject*>(payload));
+}
+
+static Value FromStringPayload(uintptr_t payload) {
+ return StringValue(reinterpret_cast<JSString*>(payload));
+}
+
+static Value FromSymbolPayload(uintptr_t payload) {
+ return SymbolValue(reinterpret_cast<JS::Symbol*>(payload));
+}
+
+static Value FromBigIntPayload(uintptr_t payload) {
+ return BigIntValue(reinterpret_cast<JS::BigInt*>(payload));
+}
+
+static Value FromTypedPayload(JSValueType type, uintptr_t payload) {
+ switch (type) {
+ case JSVAL_TYPE_INT32:
+ return Int32Value(payload);
+ case JSVAL_TYPE_BOOLEAN:
+ return BooleanValue(!!payload);
+ case JSVAL_TYPE_STRING:
+ return FromStringPayload(payload);
+ case JSVAL_TYPE_SYMBOL:
+ return FromSymbolPayload(payload);
+ case JSVAL_TYPE_BIGINT:
+ return FromBigIntPayload(payload);
+ case JSVAL_TYPE_OBJECT:
+ return FromObjectPayload(payload);
+ default:
+ MOZ_CRASH("unexpected type - needs payload");
+ }
+}
+
+bool SnapshotIterator::allocationReadable(const RValueAllocation& alloc,
+ ReadMethod rm) {
+ // If we have to recover stores, and if we are not interested in the
+ // default value of the instruction, then we have to check if the recover
+ // instruction results are available.
+ if (alloc.needSideEffect() && rm != ReadMethod::AlwaysDefault) {
+ if (!hasInstructionResults()) {
+ return false;
+ }
+ }
+
+ switch (alloc.mode()) {
+ case RValueAllocation::DOUBLE_REG:
+ return hasRegister(alloc.fpuReg());
+
+ case RValueAllocation::TYPED_REG:
+ return hasRegister(alloc.reg2());
+
+#if defined(JS_NUNBOX32)
+ case RValueAllocation::UNTYPED_REG_REG:
+ return hasRegister(alloc.reg()) && hasRegister(alloc.reg2());
+ case RValueAllocation::UNTYPED_REG_STACK:
+ return hasRegister(alloc.reg()) && hasStack(alloc.stackOffset2());
+ case RValueAllocation::UNTYPED_STACK_REG:
+ return hasStack(alloc.stackOffset()) && hasRegister(alloc.reg2());
+ case RValueAllocation::UNTYPED_STACK_STACK:
+ return hasStack(alloc.stackOffset()) && hasStack(alloc.stackOffset2());
+#elif defined(JS_PUNBOX64)
+ case RValueAllocation::UNTYPED_REG:
+ return hasRegister(alloc.reg());
+ case RValueAllocation::UNTYPED_STACK:
+ return hasStack(alloc.stackOffset());
+#endif
+
+ case RValueAllocation::RECOVER_INSTRUCTION:
+ return hasInstructionResult(alloc.index());
+ case RValueAllocation::RI_WITH_DEFAULT_CST:
+ return rm == ReadMethod::AlwaysDefault ||
+ hasInstructionResult(alloc.index());
+
+ default:
+ return true;
+ }
+}
+
+Value SnapshotIterator::allocationValue(const RValueAllocation& alloc,
+ ReadMethod rm) {
+ switch (alloc.mode()) {
+ case RValueAllocation::CONSTANT:
+ return ionScript_->getConstant(alloc.index());
+
+ case RValueAllocation::CST_UNDEFINED:
+ return UndefinedValue();
+
+ case RValueAllocation::CST_NULL:
+ return NullValue();
+
+ case RValueAllocation::DOUBLE_REG:
+ return DoubleValue(fromRegister<double>(alloc.fpuReg()));
+
+ case RValueAllocation::ANY_FLOAT_REG:
+ return Float32Value(fromRegister<float>(alloc.fpuReg()));
+
+ case RValueAllocation::ANY_FLOAT_STACK:
+ return Float32Value(ReadFrameFloat32Slot(fp_, alloc.stackOffset()));
+
+ case RValueAllocation::TYPED_REG:
+ return FromTypedPayload(alloc.knownType(), fromRegister(alloc.reg2()));
+
+ case RValueAllocation::TYPED_STACK: {
+ switch (alloc.knownType()) {
+ case JSVAL_TYPE_DOUBLE:
+ return DoubleValue(ReadFrameDoubleSlot(fp_, alloc.stackOffset2()));
+ case JSVAL_TYPE_INT32:
+ return Int32Value(ReadFrameInt32Slot(fp_, alloc.stackOffset2()));
+ case JSVAL_TYPE_BOOLEAN:
+ return BooleanValue(ReadFrameBooleanSlot(fp_, alloc.stackOffset2()));
+ case JSVAL_TYPE_STRING:
+ return FromStringPayload(fromStack(alloc.stackOffset2()));
+ case JSVAL_TYPE_SYMBOL:
+ return FromSymbolPayload(fromStack(alloc.stackOffset2()));
+ case JSVAL_TYPE_BIGINT:
+ return FromBigIntPayload(fromStack(alloc.stackOffset2()));
+ case JSVAL_TYPE_OBJECT:
+ return FromObjectPayload(fromStack(alloc.stackOffset2()));
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+ }
+
+#if defined(JS_NUNBOX32)
+ case RValueAllocation::UNTYPED_REG_REG: {
+ return Value::fromTagAndPayload(JSValueTag(fromRegister(alloc.reg())),
+ fromRegister(alloc.reg2()));
+ }
+
+ case RValueAllocation::UNTYPED_REG_STACK: {
+ return Value::fromTagAndPayload(JSValueTag(fromRegister(alloc.reg())),
+ fromStack(alloc.stackOffset2()));
+ }
+
+ case RValueAllocation::UNTYPED_STACK_REG: {
+ return Value::fromTagAndPayload(
+ JSValueTag(fromStack(alloc.stackOffset())),
+ fromRegister(alloc.reg2()));
+ }
+
+ case RValueAllocation::UNTYPED_STACK_STACK: {
+ return Value::fromTagAndPayload(
+ JSValueTag(fromStack(alloc.stackOffset())),
+ fromStack(alloc.stackOffset2()));
+ }
+#elif defined(JS_PUNBOX64)
+ case RValueAllocation::UNTYPED_REG: {
+ return Value::fromRawBits(fromRegister(alloc.reg()));
+ }
+
+ case RValueAllocation::UNTYPED_STACK: {
+ return Value::fromRawBits(fromStack(alloc.stackOffset()));
+ }
+#endif
+
+ case RValueAllocation::RECOVER_INSTRUCTION:
+ return fromInstructionResult(alloc.index());
+
+ case RValueAllocation::RI_WITH_DEFAULT_CST:
+ if (rm == ReadMethod::Normal && hasInstructionResult(alloc.index())) {
+ return fromInstructionResult(alloc.index());
+ }
+ MOZ_ASSERT(rm == ReadMethod::AlwaysDefault);
+ return ionScript_->getConstant(alloc.index2());
+
+ default:
+ MOZ_CRASH("huh?");
+ }
+}
+
+Value SnapshotIterator::maybeRead(const RValueAllocation& a,
+ MaybeReadFallback& fallback) {
+ if (allocationReadable(a)) {
+ return allocationValue(a);
+ }
+
+ if (fallback.canRecoverResults()) {
+ // Code paths which are calling maybeRead are not always capable of
+ // returning an error code, as these code paths used to be infallible.
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!initInstructionResults(fallback)) {
+ oomUnsafe.crash("js::jit::SnapshotIterator::maybeRead");
+ }
+
+ if (allocationReadable(a)) {
+ return allocationValue(a);
+ }
+
+ MOZ_ASSERT_UNREACHABLE("All allocations should be readable.");
+ }
+
+ return UndefinedValue();
+}
+
+bool SnapshotIterator::tryRead(Value* result) {
+ RValueAllocation a = readAllocation();
+ if (allocationReadable(a)) {
+ *result = allocationValue(a);
+ return true;
+ }
+ return false;
+}
+
+void SnapshotIterator::writeAllocationValuePayload(
+ const RValueAllocation& alloc, const Value& v) {
+ MOZ_ASSERT(v.isGCThing());
+
+ switch (alloc.mode()) {
+ case RValueAllocation::CONSTANT:
+ ionScript_->getConstant(alloc.index()) = v;
+ break;
+
+ case RValueAllocation::CST_UNDEFINED:
+ case RValueAllocation::CST_NULL:
+ case RValueAllocation::DOUBLE_REG:
+ case RValueAllocation::ANY_FLOAT_REG:
+ case RValueAllocation::ANY_FLOAT_STACK:
+ MOZ_CRASH("Not a GC thing: Unexpected write");
+ break;
+
+ case RValueAllocation::TYPED_REG:
+ machine_->write(alloc.reg2(), uintptr_t(v.toGCThing()));
+ break;
+
+ case RValueAllocation::TYPED_STACK:
+ switch (alloc.knownType()) {
+ default:
+ MOZ_CRASH("Not a GC thing: Unexpected write");
+ break;
+ case JSVAL_TYPE_STRING:
+ case JSVAL_TYPE_SYMBOL:
+ case JSVAL_TYPE_BIGINT:
+ case JSVAL_TYPE_OBJECT:
+ WriteFrameSlot(fp_, alloc.stackOffset2(), uintptr_t(v.toGCThing()));
+ break;
+ }
+ break;
+
+#if defined(JS_NUNBOX32)
+ case RValueAllocation::UNTYPED_REG_REG:
+ case RValueAllocation::UNTYPED_STACK_REG:
+ machine_->write(alloc.reg2(), uintptr_t(v.toGCThing()));
+ break;
+
+ case RValueAllocation::UNTYPED_REG_STACK:
+ case RValueAllocation::UNTYPED_STACK_STACK:
+ WriteFrameSlot(fp_, alloc.stackOffset2(), uintptr_t(v.toGCThing()));
+ break;
+#elif defined(JS_PUNBOX64)
+ case RValueAllocation::UNTYPED_REG:
+ machine_->write(alloc.reg(), v.asRawBits());
+ break;
+
+ case RValueAllocation::UNTYPED_STACK:
+ WriteFrameSlot(fp_, alloc.stackOffset(), v.asRawBits());
+ break;
+#endif
+
+ case RValueAllocation::RECOVER_INSTRUCTION:
+ MOZ_CRASH("Recover instructions are handled by the JitActivation.");
+ break;
+
+ case RValueAllocation::RI_WITH_DEFAULT_CST:
+ // Assume that we are always going to be writing on the default value
+ // while tracing.
+ ionScript_->getConstant(alloc.index2()) = v;
+ break;
+
+ default:
+ MOZ_CRASH("huh?");
+ }
+}
+
+void SnapshotIterator::traceAllocation(JSTracer* trc) {
+ RValueAllocation alloc = readAllocation();
+ if (!allocationReadable(alloc, ReadMethod::AlwaysDefault)) {
+ return;
+ }
+
+ Value v = allocationValue(alloc, ReadMethod::AlwaysDefault);
+ if (!v.isGCThing()) {
+ return;
+ }
+
+ Value copy = v;
+ TraceRoot(trc, &v, "ion-typed-reg");
+ if (v != copy) {
+ MOZ_ASSERT(SameType(v, copy));
+ writeAllocationValuePayload(alloc, v);
+ }
+}
+
+const RResumePoint* SnapshotIterator::resumePoint() const {
+ return instruction()->toResumePoint();
+}
+
+uint32_t SnapshotIterator::numAllocations() const {
+ return instruction()->numOperands();
+}
+
+uint32_t SnapshotIterator::pcOffset() const {
+ return resumePoint()->pcOffset();
+}
+
+ResumeMode SnapshotIterator::resumeMode() const {
+ return resumePoint()->mode();
+}
+
+void SnapshotIterator::skipInstruction() {
+ MOZ_ASSERT(snapshot_.numAllocationsRead() == 0);
+ size_t numOperands = instruction()->numOperands();
+ for (size_t i = 0; i < numOperands; i++) {
+ skip();
+ }
+ nextInstruction();
+}
+
+bool SnapshotIterator::initInstructionResults(MaybeReadFallback& fallback) {
+ MOZ_ASSERT(fallback.canRecoverResults());
+ JSContext* cx = fallback.maybeCx;
+
+ // If there is only one resume point in the list of instructions, then there
+ // is no instruction to recover, and thus no need to register any results.
+ if (recover_.numInstructions() == 1) {
+ return true;
+ }
+
+ JitFrameLayout* fp = fallback.frame->jsFrame();
+ RInstructionResults* results = fallback.activation->maybeIonFrameRecovery(fp);
+ if (!results) {
+ AutoRealm ar(cx, fallback.frame->script());
+
+ // We do not have the result yet, which means that an observable stack
+ // slot is requested. As we do not want to bailout every time for the
+ // same reason, we need to recompile without optimizing away the
+ // observable stack slots. The script would later be recompiled to have
+ // support for Argument objects.
+ if (fallback.consequence == MaybeReadFallback::Fallback_Invalidate) {
+ ionScript_->invalidate(cx, fallback.frame->script(),
+ /* resetUses = */ false,
+ "Observe recovered instruction.");
+ }
+
+ // Register the list of result on the activation. We need to do that
+ // before we initialize the list such as if any recover instruction
+ // cause a GC, we can ensure that the results are properly traced by the
+ // activation.
+ RInstructionResults tmp(fallback.frame->jsFrame());
+ if (!fallback.activation->registerIonFrameRecovery(std::move(tmp))) {
+ return false;
+ }
+
+ results = fallback.activation->maybeIonFrameRecovery(fp);
+
+ // Start a new snapshot at the beginning of the JSJitFrameIter. This
+ // SnapshotIterator is used for evaluating the content of all recover
+ // instructions. The result is then saved on the JitActivation.
+ MachineState machine = fallback.frame->machineState();
+ SnapshotIterator s(*fallback.frame, &machine);
+ if (!s.computeInstructionResults(cx, results)) {
+ // If the evaluation failed because of OOMs, then we discard the
+ // current set of result that we collected so far.
+ fallback.activation->removeIonFrameRecovery(fp);
+ return false;
+ }
+ }
+
+ MOZ_ASSERT(results->isInitialized());
+ MOZ_RELEASE_ASSERT(results->length() == recover_.numInstructions() - 1);
+ instructionResults_ = results;
+ return true;
+}
+
+bool SnapshotIterator::computeInstructionResults(
+ JSContext* cx, RInstructionResults* results) const {
+ MOZ_ASSERT(!results->isInitialized());
+ MOZ_ASSERT(recover_.numInstructionsRead() == 1);
+
+ // The last instruction will always be a resume point.
+ size_t numResults = recover_.numInstructions() - 1;
+ if (!results->isInitialized()) {
+ if (!results->init(cx, numResults)) {
+ return false;
+ }
+
+ // No need to iterate over the only resume point.
+ if (!numResults) {
+ MOZ_ASSERT(results->isInitialized());
+ return true;
+ }
+
+ // Avoid invoking the object metadata callback, which could try to walk the
+ // stack while bailing out.
+ gc::AutoSuppressGC suppressGC(cx);
+ js::AutoSuppressAllocationMetadataBuilder suppressMetadata(cx);
+
+ // Fill with the results of recover instructions.
+ SnapshotIterator s(*this);
+ s.instructionResults_ = results;
+ while (s.moreInstructions()) {
+ // Skip resume point and only interpret recover instructions.
+ if (s.instruction()->isResumePoint()) {
+ s.skipInstruction();
+ continue;
+ }
+
+ if (!s.instruction()->recover(cx, s)) {
+ return false;
+ }
+ s.nextInstruction();
+ }
+ }
+
+ MOZ_ASSERT(results->isInitialized());
+ return true;
+}
+
+void SnapshotIterator::storeInstructionResult(const Value& v) {
+ uint32_t currIns = recover_.numInstructionsRead() - 1;
+ MOZ_ASSERT((*instructionResults_)[currIns].isMagic(JS_ION_BAILOUT));
+ (*instructionResults_)[currIns] = v;
+}
+
+Value SnapshotIterator::fromInstructionResult(uint32_t index) const {
+ MOZ_ASSERT(!(*instructionResults_)[index].isMagic(JS_ION_BAILOUT));
+ return (*instructionResults_)[index];
+}
+
+void SnapshotIterator::settleOnFrame() {
+ // Check that the current instruction can still be use.
+ MOZ_ASSERT(snapshot_.numAllocationsRead() == 0);
+ while (!instruction()->isResumePoint()) {
+ skipInstruction();
+ }
+}
+
+void SnapshotIterator::nextFrame() {
+ nextInstruction();
+ settleOnFrame();
+}
+
+Value SnapshotIterator::maybeReadAllocByIndex(size_t index) {
+ while (index--) {
+ MOZ_ASSERT(moreAllocations());
+ skip();
+ }
+
+ Value s;
+ {
+ // This MaybeReadFallback method cannot GC.
+ JS::AutoSuppressGCAnalysis nogc;
+ MaybeReadFallback fallback;
+ s = maybeRead(fallback);
+ }
+
+ while (moreAllocations()) {
+ skip();
+ }
+
+ return s;
+}
+
+InlineFrameIterator::InlineFrameIterator(JSContext* cx,
+ const JSJitFrameIter* iter)
+ : calleeTemplate_(cx),
+ calleeRVA_(),
+ script_(cx),
+ pc_(nullptr),
+ numActualArgs_(0) {
+ resetOn(iter);
+}
+
+InlineFrameIterator::InlineFrameIterator(JSContext* cx,
+ const InlineFrameIterator* iter)
+ : frame_(iter ? iter->frame_ : nullptr),
+ framesRead_(0),
+ frameCount_(iter ? iter->frameCount_ : UINT32_MAX),
+ calleeTemplate_(cx),
+ calleeRVA_(),
+ script_(cx),
+ pc_(nullptr),
+ numActualArgs_(0) {
+ if (frame_) {
+ machine_ = iter->machine_;
+ start_ = SnapshotIterator(*frame_, &machine_);
+
+ // findNextFrame will iterate to the next frame and init. everything.
+ // Therefore to settle on the same frame, we report one frame less readed.
+ framesRead_ = iter->framesRead_ - 1;
+ findNextFrame();
+ }
+}
+
+void InlineFrameIterator::resetOn(const JSJitFrameIter* iter) {
+ frame_ = iter;
+ framesRead_ = 0;
+ frameCount_ = UINT32_MAX;
+
+ if (iter) {
+ machine_ = iter->machineState();
+ start_ = SnapshotIterator(*iter, &machine_);
+ findNextFrame();
+ }
+}
+
+void InlineFrameIterator::findNextFrame() {
+ MOZ_ASSERT(more());
+
+ si_ = start_;
+
+ // Read the initial frame out of the C stack.
+ calleeTemplate_ = frame_->maybeCallee();
+ calleeRVA_ = RValueAllocation();
+ script_ = frame_->script();
+ MOZ_ASSERT(script_->hasBaselineScript());
+
+ // Settle on the outermost frame without evaluating any instructions before
+ // looking for a pc.
+ si_.settleOnFrame();
+
+ pc_ = script_->offsetToPC(si_.pcOffset());
+ numActualArgs_ = 0xbadbad;
+
+ // This unfortunately is O(n*m), because we must skip over outer frames
+ // before reading inner ones.
+
+ // The first time (frameCount_ == UINT32_MAX) we do not know the number of
+ // frames that we are going to inspect. So we are iterating until there is
+ // no more frames, to settle on the inner most frame and to count the number
+ // of frames.
+ size_t remaining = (frameCount_ != UINT32_MAX) ? frameNo() - 1 : SIZE_MAX;
+
+ size_t i = 1;
+ for (; i <= remaining && si_.moreFrames(); i++) {
+ ResumeMode mode = si_.resumeMode();
+ MOZ_ASSERT(IsIonInlinableOp(JSOp(*pc_)));
+
+ // Recover the number of actual arguments from the script.
+ if (IsInvokeOp(JSOp(*pc_))) {
+ MOZ_ASSERT(mode == ResumeMode::InlinedStandardCall ||
+ mode == ResumeMode::InlinedFunCall);
+ numActualArgs_ = GET_ARGC(pc_);
+ if (mode == ResumeMode::InlinedFunCall && numActualArgs_ > 0) {
+ numActualArgs_--;
+ }
+ } else if (IsGetPropPC(pc_) || IsGetElemPC(pc_)) {
+ MOZ_ASSERT(mode == ResumeMode::InlinedAccessor);
+ numActualArgs_ = 0;
+ } else {
+ MOZ_RELEASE_ASSERT(IsSetPropPC(pc_));
+ MOZ_ASSERT(mode == ResumeMode::InlinedAccessor);
+ numActualArgs_ = 1;
+ }
+
+ // Skip over non-argument slots, as well as |this|.
+ bool skipNewTarget = IsConstructPC(pc_);
+ unsigned skipCount =
+ (si_.numAllocations() - 1) - numActualArgs_ - 1 - skipNewTarget;
+ for (unsigned j = 0; j < skipCount; j++) {
+ si_.skip();
+ }
+
+ // This value should correspond to the function which is being inlined.
+ // The value must be readable to iterate over the inline frame. Most of
+ // the time, these functions are stored as JSFunction constants,
+ // register which are holding the JSFunction pointer, or recover
+ // instruction with Default value.
+ Value funval = si_.readWithDefault(&calleeRVA_);
+
+ // Skip extra value allocations.
+ while (si_.moreAllocations()) {
+ si_.skip();
+ }
+
+ si_.nextFrame();
+
+ calleeTemplate_ = &funval.toObject().as<JSFunction>();
+ script_ = calleeTemplate_->nonLazyScript();
+ MOZ_ASSERT(script_->hasBaselineScript());
+
+ pc_ = script_->offsetToPC(si_.pcOffset());
+ }
+
+ // The first time we do not know the number of frames, we only settle on the
+ // last frame, and update the number of frames based on the number of
+ // iteration that we have done.
+ if (frameCount_ == UINT32_MAX) {
+ MOZ_ASSERT(!si_.moreFrames());
+ frameCount_ = i;
+ }
+
+ framesRead_++;
+}
+
+JSFunction* InlineFrameIterator::callee(MaybeReadFallback& fallback) const {
+ MOZ_ASSERT(isFunctionFrame());
+ if (calleeRVA_.mode() == RValueAllocation::INVALID ||
+ !fallback.canRecoverResults()) {
+ return calleeTemplate_;
+ }
+
+ SnapshotIterator s(si_);
+ // :TODO: Handle allocation failures from recover instruction.
+ Value funval = s.maybeRead(calleeRVA_, fallback);
+ return &funval.toObject().as<JSFunction>();
+}
+
+JSObject* InlineFrameIterator::computeEnvironmentChain(
+ const Value& envChainValue, MaybeReadFallback& fallback,
+ bool* hasInitialEnv) const {
+ if (envChainValue.isObject()) {
+ if (hasInitialEnv) {
+ if (fallback.canRecoverResults()) {
+ RootedObject obj(fallback.maybeCx, &envChainValue.toObject());
+ *hasInitialEnv = isFunctionFrame() &&
+ callee(fallback)->needsFunctionEnvironmentObjects();
+ return obj;
+ }
+ JS::AutoSuppressGCAnalysis
+ nogc; // If we cannot recover then we cannot GC.
+ *hasInitialEnv = isFunctionFrame() &&
+ callee(fallback)->needsFunctionEnvironmentObjects();
+ }
+
+ return &envChainValue.toObject();
+ }
+
+ // Note we can hit this case even for functions with a CallObject, in case
+ // we are walking the frame during the function prologue, before the env
+ // chain has been initialized.
+ if (isFunctionFrame()) {
+ return callee(fallback)->environment();
+ }
+
+ if (isModuleFrame()) {
+ return script()->module()->environment();
+ }
+
+ // Ion does not handle non-function scripts that have anything other than
+ // the global on their env chain.
+ MOZ_ASSERT(!script()->isForEval());
+ MOZ_ASSERT(!script()->hasNonSyntacticScope());
+ return &script()->global().lexicalEnvironment();
+}
+
+bool InlineFrameIterator::isFunctionFrame() const { return !!calleeTemplate_; }
+
+bool InlineFrameIterator::isModuleFrame() const { return script()->isModule(); }
+
+uintptr_t* MachineState::SafepointState::addressOfRegister(Register reg) const {
+ size_t offset = regs.offsetOfPushedRegister(reg);
+
+ MOZ_ASSERT((offset % sizeof(uintptr_t)) == 0);
+ uint32_t index = offset / sizeof(uintptr_t);
+
+#ifdef DEBUG
+ // Assert correctness with a slower algorithm in debug builds.
+ uint32_t expectedIndex = 0;
+ bool found = false;
+ for (GeneralRegisterBackwardIterator iter(regs); iter.more(); ++iter) {
+ expectedIndex++;
+ if (*iter == reg) {
+ found = true;
+ break;
+ }
+ }
+ MOZ_ASSERT(found);
+ MOZ_ASSERT(expectedIndex == index);
+#endif
+
+ return spillBase - index;
+}
+
+char* MachineState::SafepointState::addressOfRegister(FloatRegister reg) const {
+ // Note: this could be optimized similar to the GPR case above by implementing
+ // offsetOfPushedRegister for FloatRegisterSet. Float register sets are
+ // complicated though and this case is very uncommon: it's only reachable for
+ // exception bailouts with live float registers.
+ MOZ_ASSERT(!reg.isSimd128());
+ char* ptr = floatSpillBase;
+ for (FloatRegisterBackwardIterator iter(floatRegs); iter.more(); ++iter) {
+ ptr -= (*iter).size();
+ for (uint32_t a = 0; a < (*iter).numAlignedAliased(); a++) {
+ // Only say that registers that actually start here start here.
+ // e.g. d0 should not start at s1, only at s0.
+ FloatRegister ftmp = (*iter).alignedAliased(a);
+ if (ftmp == reg) {
+ return ptr;
+ }
+ }
+ }
+ MOZ_CRASH("Invalid register");
+}
+
+uintptr_t MachineState::read(Register reg) const {
+ if (state_.is<BailoutState>()) {
+ return state_.as<BailoutState>().regs[reg.code()].r;
+ }
+ if (state_.is<SafepointState>()) {
+ uintptr_t* addr = state_.as<SafepointState>().addressOfRegister(reg);
+ return *addr;
+ }
+ MOZ_CRASH("Invalid state");
+}
+
+template <typename T>
+T MachineState::read(FloatRegister reg) const {
+ MOZ_ASSERT(reg.size() == sizeof(T));
+
+#if !defined(JS_CODEGEN_NONE) && !defined(JS_CODEGEN_WASM32)
+ if (state_.is<BailoutState>()) {
+ uint32_t offset = reg.getRegisterDumpOffsetInBytes();
+ MOZ_ASSERT((offset % sizeof(T)) == 0);
+ MOZ_ASSERT((offset + sizeof(T)) <= sizeof(RegisterDump::FPUArray));
+
+ const BailoutState& state = state_.as<BailoutState>();
+ char* addr = reinterpret_cast<char*>(state.floatRegs.begin()) + offset;
+ return *reinterpret_cast<T*>(addr);
+ }
+ if (state_.is<SafepointState>()) {
+ char* addr = state_.as<SafepointState>().addressOfRegister(reg);
+ return *reinterpret_cast<T*>(addr);
+ }
+#endif
+ MOZ_CRASH("Invalid state");
+}
+
+void MachineState::write(Register reg, uintptr_t value) const {
+ if (state_.is<SafepointState>()) {
+ uintptr_t* addr = state_.as<SafepointState>().addressOfRegister(reg);
+ *addr = value;
+ return;
+ }
+ MOZ_CRASH("Invalid state");
+}
+
+bool InlineFrameIterator::isConstructing() const {
+ // Skip the current frame and look at the caller's.
+ if (more()) {
+ InlineFrameIterator parent(TlsContext.get(), this);
+ ++parent;
+
+ // In the case of a JS frame, look up the pc from the snapshot.
+ JSOp parentOp = JSOp(*parent.pc());
+
+ // Inlined Getters and Setters are never constructing.
+ if (IsIonInlinableGetterOrSetterOp(parentOp)) {
+ return false;
+ }
+
+ MOZ_ASSERT(IsInvokeOp(parentOp) && !IsSpreadOp(parentOp));
+
+ return IsConstructOp(parentOp);
+ }
+
+ return frame_->isConstructing();
+}
+
+void SnapshotIterator::warnUnreadableAllocation() {
+ fprintf(stderr,
+ "Warning! Tried to access unreadable value allocation (possible "
+ "f.arguments).\n");
+}
+
+struct DumpOp {
+ explicit DumpOp(unsigned int i) : i_(i) {}
+
+ unsigned int i_;
+ void operator()(const Value& v) {
+ fprintf(stderr, " actual (arg %u): ", i_);
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ DumpValue(v);
+#else
+ fprintf(stderr, "?\n");
+#endif
+ i_++;
+ }
+};
+
+void InlineFrameIterator::dump() const {
+ MaybeReadFallback fallback;
+
+ if (more()) {
+ fprintf(stderr, " JS frame (inlined)\n");
+ } else {
+ fprintf(stderr, " JS frame\n");
+ }
+
+ bool isFunction = false;
+ if (isFunctionFrame()) {
+ isFunction = true;
+ fprintf(stderr, " callee fun: ");
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ DumpObject(callee(fallback));
+#else
+ fprintf(stderr, "?\n");
+#endif
+ } else {
+ fprintf(stderr, " global frame, no callee\n");
+ }
+
+ fprintf(stderr, " file %s line %u\n", script()->filename(),
+ script()->lineno());
+
+ fprintf(stderr, " script = %p, pc = %p\n", (void*)script(), pc());
+ fprintf(stderr, " current op: %s\n", CodeName(JSOp(*pc())));
+
+ if (!more()) {
+ numActualArgs();
+ }
+
+ SnapshotIterator si = snapshotIterator();
+ fprintf(stderr, " slots: %u\n", si.numAllocations() - 1);
+ for (unsigned i = 0; i < si.numAllocations() - 1; i++) {
+ if (isFunction) {
+ if (i == 0) {
+ fprintf(stderr, " env chain: ");
+ } else if (i == 1) {
+ fprintf(stderr, " this: ");
+ } else if (i - 2 < calleeTemplate()->nargs()) {
+ fprintf(stderr, " formal (arg %u): ", i - 2);
+ } else {
+ if (i - 2 == calleeTemplate()->nargs() &&
+ numActualArgs() > calleeTemplate()->nargs()) {
+ DumpOp d(calleeTemplate()->nargs());
+ unaliasedForEachActual(TlsContext.get(), d, ReadFrame_Overflown,
+ fallback);
+ }
+
+ fprintf(stderr, " slot %d: ", int(i - 2 - calleeTemplate()->nargs()));
+ }
+ } else
+ fprintf(stderr, " slot %u: ", i);
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ DumpValue(si.maybeRead(fallback));
+#else
+ fprintf(stderr, "?\n");
+#endif
+ }
+
+ fputc('\n', stderr);
+}
+
+JitFrameLayout* InvalidationBailoutStack::fp() const {
+ return (JitFrameLayout*)(sp() + ionScript_->frameSize());
+}
+
+void InvalidationBailoutStack::checkInvariants() const {
+#ifdef DEBUG
+ JitFrameLayout* frame = fp();
+ CalleeToken token = frame->calleeToken();
+ MOZ_ASSERT(token);
+
+ uint8_t* rawBase = ionScript()->method()->raw();
+ uint8_t* rawLimit = rawBase + ionScript()->method()->instructionsSize();
+ uint8_t* osiPoint = osiPointReturnAddress();
+ MOZ_ASSERT(rawBase <= osiPoint && osiPoint <= rawLimit);
+#endif
+}
+
+void AssertJitStackInvariants(JSContext* cx) {
+ for (JitActivationIterator activations(cx); !activations.done();
+ ++activations) {
+ JitFrameIter iter(activations->asJit());
+ if (iter.isJSJit()) {
+ JSJitFrameIter& frames = iter.asJSJit();
+ size_t prevFrameSize = 0;
+ size_t frameSize = 0;
+ bool isScriptedCallee = false;
+ for (; !frames.done(); ++frames) {
+ size_t calleeFp = reinterpret_cast<size_t>(frames.fp());
+ size_t callerFp = reinterpret_cast<size_t>(frames.prevFp());
+ MOZ_ASSERT(callerFp >= calleeFp);
+ prevFrameSize = frameSize;
+ frameSize = callerFp - calleeFp;
+
+ if (frames.isScripted() && frames.prevType() == FrameType::Rectifier) {
+ MOZ_RELEASE_ASSERT(frameSize % JitStackAlignment == 0,
+ "The rectifier frame should keep the alignment");
+
+ size_t expectedFrameSize =
+ sizeof(Value) *
+ (frames.callee()->nargs() + 1 /* |this| argument */ +
+ frames.isConstructing() /* new.target */) +
+ sizeof(JitFrameLayout);
+ MOZ_RELEASE_ASSERT(frameSize >= expectedFrameSize,
+ "The frame is large enough to hold all arguments");
+ MOZ_RELEASE_ASSERT(expectedFrameSize + JitStackAlignment > frameSize,
+ "The frame size is optimal");
+ }
+
+ if (frames.isExitFrame()) {
+ // For the moment, we do not keep the JitStackAlignment
+ // alignment for exit frames.
+ frameSize -= ExitFrameLayout::Size();
+ }
+
+ if (frames.isIonJS()) {
+ // Ideally, we should not have such requirement, but keep the
+ // alignment-delta as part of the Safepoint such that we can pad
+ // accordingly when making out-of-line calls. In the mean time,
+ // let us have check-points where we can garantee that
+ // everything can properly be aligned before adding complexity.
+ MOZ_RELEASE_ASSERT(
+ frames.ionScript()->frameSize() % JitStackAlignment == 0,
+ "Ensure that if the Ion frame is aligned, then the spill base is "
+ "also aligned");
+
+ if (isScriptedCallee) {
+ MOZ_RELEASE_ASSERT(prevFrameSize % JitStackAlignment == 0,
+ "The ion frame should keep the alignment");
+ }
+ }
+
+ // The stack is dynamically aligned by baseline stubs before calling
+ // any jitted code.
+ if (frames.prevType() == FrameType::BaselineStub && isScriptedCallee) {
+ MOZ_RELEASE_ASSERT(calleeFp % JitStackAlignment == 0,
+ "The baseline stub restores the stack alignment");
+ }
+
+ isScriptedCallee =
+ frames.isScripted() || frames.type() == FrameType::Rectifier;
+ }
+
+ MOZ_RELEASE_ASSERT(
+ JSJitFrameIter::isEntry(frames.type()),
+ "The first frame of a Jit activation should be an entry frame");
+ MOZ_RELEASE_ASSERT(
+ reinterpret_cast<size_t>(frames.fp()) % JitStackAlignment == 0,
+ "The entry frame should be properly aligned");
+ } else {
+ MOZ_ASSERT(iter.isWasm());
+ wasm::WasmFrameIter& frames = iter.asWasm();
+ while (!frames.done()) {
+ ++frames;
+ }
+ }
+ }
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/JitFrames.h b/js/src/jit/JitFrames.h
new file mode 100644
index 0000000000..4df99cf407
--- /dev/null
+++ b/js/src/jit/JitFrames.h
@@ -0,0 +1,741 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitFrames_h
+#define jit_JitFrames_h
+
+#include "mozilla/Assertions.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/CalleeToken.h"
+#include "jit/MachineState.h"
+#include "jit/Registers.h"
+#include "js/Id.h"
+#include "js/TypeDecls.h"
+#include "js/Value.h"
+
+namespace js {
+
+namespace wasm {
+class Instance;
+}
+
+namespace jit {
+
+enum class FrameType;
+class IonScript;
+class JitActivation;
+class JitFrameLayout;
+struct SafepointSlotEntry;
+struct VMFunctionData;
+
+// [SMDOC] JIT Frame Layout
+//
+// Frame Headers:
+//
+// In between every two frames lies a small header describing both frames. This
+// header, minimally, contains a returnAddress word and a descriptor word (See
+// CommonFrameLayout). The descriptor describes the type of the older (caller)
+// frame, whereas the returnAddress describes the address the newer (callee)
+// frame will return to. For JitFrameLayout, the descriptor also stores the
+// number of arguments passed from the caller to the callee frame.
+//
+// Special Frames:
+//
+// Two special frame types exist:
+// - Entry frames begin a JitActivation, and therefore there is exactly one
+// per activation of EnterJit or EnterBaseline. These reuse JitFrameLayout.
+// - Exit frames are necessary to leave JIT code and enter C++, and thus,
+// C++ code will always begin iterating from the topmost exit frame.
+//
+// Approximate Layout:
+//
+// The layout of an Ion frame on the C stack is roughly:
+// argN _
+// ... \ - These are jsvals
+// arg0 /
+// -3 this _/
+// -2 callee
+// -1 descriptor
+// 0 returnAddress
+// .. locals ..
+
+// [SMDOC] Frame Descriptor Layout
+//
+// A frame descriptor word has the following data:
+//
+// high bits: [ numActualArgs |
+// has-cached-saved-frame bit |
+// low bits: frame type ]
+//
+// * numActualArgs: for JitFrameLayout, the number of arguments passed by the
+// caller.
+// * Has-cache-saved-frame bit: Used to power the LiveSavedFrameCache
+// optimization. See the comment in Activation.h
+// * Frame Type: BaselineJS, Exit, etc. (jit::FrameType)
+//
+
+static const uintptr_t FRAMETYPE_BITS = 4;
+static const uintptr_t FRAMETYPE_MASK = (1 << FRAMETYPE_BITS) - 1;
+static const uintptr_t HASCACHEDSAVEDFRAME_BIT = 1 << FRAMETYPE_BITS;
+static const uintptr_t NUMACTUALARGS_SHIFT =
+ FRAMETYPE_BITS + 1 /* HASCACHEDSAVEDFRAME_BIT */;
+
+struct BaselineBailoutInfo;
+
+enum class ExceptionResumeKind : int32_t {
+ // There is no exception handler in this activation.
+ // Return from the entry frame.
+ EntryFrame,
+
+ // The exception was caught in baseline.
+ // Restore state and jump to the catch block.
+ Catch,
+
+ // A finally block must be executed in baseline.
+ // Stash the exception on the stack and jump to the finally block.
+ Finally,
+
+ // We are forcing an early return with a specific return value.
+ // This is used by the debugger and when closing generators.
+ // Immediately return from the current frame with the given value.
+ ForcedReturnBaseline,
+ ForcedReturnIon,
+
+ // This frame is currently executing in Ion, but we must bail out
+ // to baseline before handling the exception.
+ // Jump to the bailout tail stub.
+ Bailout,
+
+ // The innermost frame was a wasm frame.
+ // Return to the wasm entry frame.
+ Wasm,
+
+ // The exception was caught by a wasm catch handler.
+ // Restore state and jump to it.
+ WasmCatch
+};
+
+// Data needed to recover from an exception.
+struct ResumeFromException {
+ uint8_t* framePointer;
+ uint8_t* stackPointer;
+ uint8_t* target;
+ ExceptionResumeKind kind;
+ wasm::Instance* instance;
+
+ // Value to push when resuming into a |finally| block.
+ // Also used by Wasm to send the exception object to the throw stub.
+ JS::Value exception;
+
+ BaselineBailoutInfo* bailoutInfo;
+
+#if defined(JS_CODEGEN_ARM64)
+ uint64_t padding_;
+#endif
+
+ static size_t offsetOfFramePointer() {
+ return offsetof(ResumeFromException, framePointer);
+ }
+ static size_t offsetOfStackPointer() {
+ return offsetof(ResumeFromException, stackPointer);
+ }
+ static size_t offsetOfTarget() {
+ return offsetof(ResumeFromException, target);
+ }
+ static size_t offsetOfKind() { return offsetof(ResumeFromException, kind); }
+ static size_t offsetOfInstance() {
+ return offsetof(ResumeFromException, instance);
+ }
+ static size_t offsetOfException() {
+ return offsetof(ResumeFromException, exception);
+ }
+ static size_t offsetOfBailoutInfo() {
+ return offsetof(ResumeFromException, bailoutInfo);
+ }
+};
+
+#if defined(JS_CODEGEN_ARM64)
+static_assert(sizeof(ResumeFromException) % 16 == 0,
+ "ResumeFromException should be aligned");
+#endif
+
+void HandleException(ResumeFromException* rfe);
+
+void EnsureUnwoundJitExitFrame(JitActivation* act, JitFrameLayout* frame);
+
+void TraceJitActivations(JSContext* cx, JSTracer* trc);
+
+void UpdateJitActivationsForMinorGC(JSRuntime* rt);
+
+static inline uint32_t MakeFrameDescriptor(FrameType type) {
+ return uint32_t(type);
+}
+
+// For JitFrameLayout, the descriptor also stores the number of arguments passed
+// by the caller. Note that |type| is the type of the *older* frame and |argc|
+// is the number of arguments passed to the *newer* frame.
+static inline uint32_t MakeFrameDescriptorForJitCall(FrameType type,
+ uint32_t argc) {
+ uint32_t descriptor = (argc << NUMACTUALARGS_SHIFT) | uint32_t(type);
+ MOZ_ASSERT((descriptor >> NUMACTUALARGS_SHIFT) == argc,
+ "argc must fit in descriptor");
+ return descriptor;
+}
+
+// Returns the JSScript associated with the topmost JIT frame.
+JSScript* GetTopJitJSScript(JSContext* cx);
+
+#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_ARM64)
+uint8_t* alignDoubleSpill(uint8_t* pointer);
+#else
+inline uint8_t* alignDoubleSpill(uint8_t* pointer) {
+ // This is a no-op on most platforms.
+ return pointer;
+}
+#endif
+
+// Layout of the frame prefix. This assumes the stack architecture grows down.
+// If this is ever not the case, we'll have to refactor.
+class CommonFrameLayout {
+ uint8_t* callerFramePtr_;
+ uint8_t* returnAddress_;
+ uintptr_t descriptor_;
+
+ public:
+ static constexpr size_t offsetOfDescriptor() {
+ return offsetof(CommonFrameLayout, descriptor_);
+ }
+ uintptr_t descriptor() const { return descriptor_; }
+ static constexpr size_t offsetOfReturnAddress() {
+ return offsetof(CommonFrameLayout, returnAddress_);
+ }
+ FrameType prevType() const { return FrameType(descriptor_ & FRAMETYPE_MASK); }
+ void changePrevType(FrameType type) {
+ descriptor_ &= ~FRAMETYPE_MASK;
+ descriptor_ |= uintptr_t(type);
+ }
+ bool hasCachedSavedFrame() const {
+ return descriptor_ & HASCACHEDSAVEDFRAME_BIT;
+ }
+ void setHasCachedSavedFrame() { descriptor_ |= HASCACHEDSAVEDFRAME_BIT; }
+ void clearHasCachedSavedFrame() { descriptor_ &= ~HASCACHEDSAVEDFRAME_BIT; }
+ uint8_t* returnAddress() const { return returnAddress_; }
+ void setReturnAddress(uint8_t* addr) { returnAddress_ = addr; }
+
+ uint8_t* callerFramePtr() const { return callerFramePtr_; }
+ static constexpr size_t offsetOfCallerFramePtr() {
+ return offsetof(CommonFrameLayout, callerFramePtr_);
+ }
+ static constexpr size_t bytesPoppedAfterCall() {
+ // The return address and frame pointer are popped by the callee/call.
+ return 2 * sizeof(void*);
+ }
+};
+
+class JitFrameLayout : public CommonFrameLayout {
+ CalleeToken calleeToken_;
+
+ public:
+ CalleeToken calleeToken() const { return calleeToken_; }
+ void replaceCalleeToken(CalleeToken calleeToken) {
+ calleeToken_ = calleeToken;
+ }
+
+ static constexpr size_t offsetOfCalleeToken() {
+ return offsetof(JitFrameLayout, calleeToken_);
+ }
+ static constexpr size_t offsetOfThis() { return sizeof(JitFrameLayout); }
+ static constexpr size_t offsetOfActualArgs() {
+ return offsetOfThis() + sizeof(JS::Value);
+ }
+ static constexpr size_t offsetOfActualArg(size_t arg) {
+ return offsetOfActualArgs() + arg * sizeof(JS::Value);
+ }
+
+ JS::Value& thisv() {
+ MOZ_ASSERT(CalleeTokenIsFunction(calleeToken()));
+ return thisAndActualArgs()[0];
+ }
+ JS::Value* thisAndActualArgs() {
+ MOZ_ASSERT(CalleeTokenIsFunction(calleeToken()));
+ return (JS::Value*)(this + 1);
+ }
+ JS::Value* actualArgs() { return thisAndActualArgs() + 1; }
+ uintptr_t numActualArgs() const {
+ return descriptor() >> NUMACTUALARGS_SHIFT;
+ }
+
+ // Computes a reference to a stack or argument slot, where a slot is a
+ // distance from the base frame pointer, as would be used for LStackSlot
+ // or LArgument.
+ uintptr_t* slotRef(SafepointSlotEntry where);
+
+ static inline size_t Size() { return sizeof(JitFrameLayout); }
+};
+
+class RectifierFrameLayout : public JitFrameLayout {
+ public:
+ static inline size_t Size() { return sizeof(RectifierFrameLayout); }
+};
+
+class WasmToJSJitFrameLayout : public JitFrameLayout {
+ public:
+ static inline size_t Size() { return sizeof(WasmToJSJitFrameLayout); }
+};
+
+class IonICCallFrameLayout : public CommonFrameLayout {
+ protected:
+ // Pointer to root the stub's JitCode.
+ JitCode* stubCode_;
+
+ public:
+ JitCode** stubCode() { return &stubCode_; }
+ static size_t Size() { return sizeof(IonICCallFrameLayout); }
+};
+
+enum class ExitFrameType : uint8_t {
+ CallNative = 0x0,
+ ConstructNative = 0x1,
+ IonDOMGetter = 0x2,
+ IonDOMSetter = 0x3,
+ IonDOMMethod = 0x4,
+ IonOOLNative = 0x5,
+ IonOOLProxy = 0x6,
+ WasmGenericJitEntry = 0x7,
+ DirectWasmJitCall = 0x8,
+ UnwoundJit = 0xFB,
+ InterpreterStub = 0xFC,
+ VMFunction = 0xFD,
+ LazyLink = 0xFE,
+ Bare = 0xFF,
+};
+
+// GC related data used to keep alive data surrounding the Exit frame.
+class ExitFooterFrame {
+ // Stores the ExitFrameType or, for ExitFrameType::VMFunction, the
+ // VMFunctionData*.
+ uintptr_t data_;
+
+ public:
+ static constexpr size_t Size() { return sizeof(ExitFooterFrame); }
+ void setUnwoundJitExitFrame() {
+ data_ = uintptr_t(ExitFrameType::UnwoundJit);
+ }
+ ExitFrameType type() const {
+ static_assert(sizeof(ExitFrameType) == sizeof(uint8_t),
+ "Code assumes ExitFrameType fits in a byte");
+ if (data_ > UINT8_MAX) {
+ return ExitFrameType::VMFunction;
+ }
+ MOZ_ASSERT(ExitFrameType(data_) != ExitFrameType::VMFunction);
+ return ExitFrameType(data_);
+ }
+ inline const VMFunctionData* function() const {
+ MOZ_ASSERT(type() == ExitFrameType::VMFunction);
+ return reinterpret_cast<const VMFunctionData*>(data_);
+ }
+
+#ifdef JS_CODEGEN_MIPS32
+ uint8_t* alignedForABI() {
+ // See: MacroAssemblerMIPSCompat::alignStackPointer()
+ uint8_t* address = reinterpret_cast<uint8_t*>(this);
+ address -= sizeof(intptr_t);
+ return alignDoubleSpill(address);
+ }
+#else
+ uint8_t* alignedForABI() {
+ // This is NO-OP on non-MIPS platforms.
+ return reinterpret_cast<uint8_t*>(this);
+ }
+#endif
+
+ // This should only be called for function()->outParam == Type_Handle
+ template <typename T>
+ T* outParam() {
+ uint8_t* address = alignedForABI();
+ return reinterpret_cast<T*>(address - sizeof(T));
+ }
+};
+
+class NativeExitFrameLayout;
+class IonOOLNativeExitFrameLayout;
+class IonOOLProxyExitFrameLayout;
+class IonDOMExitFrameLayout;
+
+// this is the frame layout when we are exiting ion code, and about to enter
+// platform ABI code
+class ExitFrameLayout : public CommonFrameLayout {
+ inline uint8_t* top() { return reinterpret_cast<uint8_t*>(this + 1); }
+
+ public:
+ static inline size_t Size() { return sizeof(ExitFrameLayout); }
+ static inline size_t SizeWithFooter() {
+ return Size() + ExitFooterFrame::Size();
+ }
+
+ inline ExitFooterFrame* footer() {
+ uint8_t* sp = reinterpret_cast<uint8_t*>(this);
+ return reinterpret_cast<ExitFooterFrame*>(sp - ExitFooterFrame::Size());
+ }
+
+ // argBase targets the point which precedes the exit frame. Arguments of VM
+ // each wrapper are pushed before the exit frame. This correspond exactly
+ // to the value of the argBase register of the generateVMWrapper function.
+ inline uint8_t* argBase() {
+ MOZ_ASSERT(isWrapperExit());
+ return top();
+ }
+
+ inline bool isWrapperExit() {
+ return footer()->type() == ExitFrameType::VMFunction;
+ }
+ inline bool isBareExit() { return footer()->type() == ExitFrameType::Bare; }
+ inline bool isUnwoundJitExit() {
+ return footer()->type() == ExitFrameType::UnwoundJit;
+ }
+
+ // See the various exit frame layouts below.
+ template <typename T>
+ inline bool is() {
+ return footer()->type() == T::Type();
+ }
+ template <typename T>
+ inline T* as() {
+ MOZ_ASSERT(this->is<T>());
+ return reinterpret_cast<T*>(footer());
+ }
+};
+
+// Cannot inherit implementation since we need to extend the top of
+// ExitFrameLayout.
+class NativeExitFrameLayout {
+ protected: // only to silence a clang warning about unused private fields
+ ExitFooterFrame footer_;
+ ExitFrameLayout exit_;
+ uintptr_t argc_;
+
+ // We need to split the Value into 2 fields of 32 bits, otherwise the C++
+ // compiler may add some padding between the fields.
+ uint32_t loCalleeResult_;
+ uint32_t hiCalleeResult_;
+
+ public:
+ static inline size_t Size() { return sizeof(NativeExitFrameLayout); }
+
+ static size_t offsetOfResult() {
+ return offsetof(NativeExitFrameLayout, loCalleeResult_);
+ }
+ inline JS::Value* vp() {
+ return reinterpret_cast<JS::Value*>(&loCalleeResult_);
+ }
+ inline uintptr_t argc() const { return argc_; }
+};
+
+class CallNativeExitFrameLayout : public NativeExitFrameLayout {
+ public:
+ static ExitFrameType Type() { return ExitFrameType::CallNative; }
+};
+
+class ConstructNativeExitFrameLayout : public NativeExitFrameLayout {
+ public:
+ static ExitFrameType Type() { return ExitFrameType::ConstructNative; }
+};
+
+template <>
+inline bool ExitFrameLayout::is<NativeExitFrameLayout>() {
+ return is<CallNativeExitFrameLayout>() ||
+ is<ConstructNativeExitFrameLayout>();
+}
+
+class IonOOLNativeExitFrameLayout {
+ protected: // only to silence a clang warning about unused private fields
+ ExitFooterFrame footer_;
+ ExitFrameLayout exit_;
+
+ // pointer to root the stub's JitCode
+ JitCode* stubCode_;
+
+ uintptr_t argc_;
+
+ // We need to split the Value into 2 fields of 32 bits, otherwise the C++
+ // compiler may add some padding between the fields.
+ uint32_t loCalleeResult_;
+ uint32_t hiCalleeResult_;
+
+ // Split Value for |this| and args above.
+ uint32_t loThis_;
+ uint32_t hiThis_;
+
+ public:
+ static ExitFrameType Type() { return ExitFrameType::IonOOLNative; }
+
+ static inline size_t Size(size_t argc) {
+ // The frame accounts for the callee/result and |this|, so we only need
+ // args.
+ return sizeof(IonOOLNativeExitFrameLayout) + (argc * sizeof(JS::Value));
+ }
+
+ static size_t offsetOfResult() {
+ return offsetof(IonOOLNativeExitFrameLayout, loCalleeResult_);
+ }
+
+ inline JitCode** stubCode() { return &stubCode_; }
+ inline JS::Value* vp() {
+ return reinterpret_cast<JS::Value*>(&loCalleeResult_);
+ }
+ inline JS::Value* thisp() { return reinterpret_cast<JS::Value*>(&loThis_); }
+ inline uintptr_t argc() const { return argc_; }
+};
+
+// ProxyGetProperty(JSContext* cx, HandleObject proxy, HandleId id,
+// MutableHandleValue vp)
+// ProxyCallProperty(JSContext* cx, HandleObject proxy, HandleId id,
+// MutableHandleValue vp)
+// ProxySetProperty(JSContext* cx, HandleObject proxy, HandleId id,
+// MutableHandleValue vp, bool strict)
+class IonOOLProxyExitFrameLayout {
+ protected: // only to silence a clang warning about unused private fields
+ ExitFooterFrame footer_;
+ ExitFrameLayout exit_;
+
+ // The proxy object.
+ JSObject* proxy_;
+
+ // id for HandleId
+ jsid id_;
+
+ // space for MutableHandleValue result
+ // use two uint32_t so compiler doesn't align.
+ uint32_t vp0_;
+ uint32_t vp1_;
+
+ // pointer to root the stub's JitCode
+ JitCode* stubCode_;
+
+ public:
+ static ExitFrameType Type() { return ExitFrameType::IonOOLProxy; }
+
+ static inline size_t Size() { return sizeof(IonOOLProxyExitFrameLayout); }
+
+ static size_t offsetOfResult() {
+ return offsetof(IonOOLProxyExitFrameLayout, vp0_);
+ }
+
+ inline JitCode** stubCode() { return &stubCode_; }
+ inline JS::Value* vp() { return reinterpret_cast<JS::Value*>(&vp0_); }
+ inline jsid* id() { return &id_; }
+ inline JSObject** proxy() { return &proxy_; }
+};
+
+class IonDOMExitFrameLayout {
+ protected: // only to silence a clang warning about unused private fields
+ ExitFooterFrame footer_;
+ ExitFrameLayout exit_;
+ JSObject* thisObj;
+
+ // We need to split the Value into 2 fields of 32 bits, otherwise the C++
+ // compiler may add some padding between the fields.
+ uint32_t loCalleeResult_;
+ uint32_t hiCalleeResult_;
+
+ public:
+ static ExitFrameType GetterType() { return ExitFrameType::IonDOMGetter; }
+ static ExitFrameType SetterType() { return ExitFrameType::IonDOMSetter; }
+
+ static inline size_t Size() { return sizeof(IonDOMExitFrameLayout); }
+
+ static size_t offsetOfResult() {
+ return offsetof(IonDOMExitFrameLayout, loCalleeResult_);
+ }
+ inline JS::Value* vp() {
+ return reinterpret_cast<JS::Value*>(&loCalleeResult_);
+ }
+ inline JSObject** thisObjAddress() { return &thisObj; }
+ inline bool isMethodFrame();
+};
+
+struct IonDOMMethodExitFrameLayoutTraits;
+
+class IonDOMMethodExitFrameLayout {
+ protected: // only to silence a clang warning about unused private fields
+ ExitFooterFrame footer_;
+ ExitFrameLayout exit_;
+ // This must be the last thing pushed, so as to stay common with
+ // IonDOMExitFrameLayout.
+ JSObject* thisObj_;
+ JS::Value* argv_;
+ uintptr_t argc_;
+
+ // We need to split the Value into 2 fields of 32 bits, otherwise the C++
+ // compiler may add some padding between the fields.
+ uint32_t loCalleeResult_;
+ uint32_t hiCalleeResult_;
+
+ friend struct IonDOMMethodExitFrameLayoutTraits;
+
+ public:
+ static ExitFrameType Type() { return ExitFrameType::IonDOMMethod; }
+
+ static inline size_t Size() { return sizeof(IonDOMMethodExitFrameLayout); }
+
+ static size_t offsetOfResult() {
+ return offsetof(IonDOMMethodExitFrameLayout, loCalleeResult_);
+ }
+
+ inline JS::Value* vp() {
+ // The code in visitCallDOMNative depends on this static assert holding
+ static_assert(
+ offsetof(IonDOMMethodExitFrameLayout, loCalleeResult_) ==
+ (offsetof(IonDOMMethodExitFrameLayout, argc_) + sizeof(uintptr_t)));
+ return reinterpret_cast<JS::Value*>(&loCalleeResult_);
+ }
+ inline JSObject** thisObjAddress() { return &thisObj_; }
+ inline uintptr_t argc() { return argc_; }
+};
+
+inline bool IonDOMExitFrameLayout::isMethodFrame() {
+ return footer_.type() == IonDOMMethodExitFrameLayout::Type();
+}
+
+template <>
+inline bool ExitFrameLayout::is<IonDOMExitFrameLayout>() {
+ ExitFrameType type = footer()->type();
+ return type == IonDOMExitFrameLayout::GetterType() ||
+ type == IonDOMExitFrameLayout::SetterType() ||
+ type == IonDOMMethodExitFrameLayout::Type();
+}
+
+template <>
+inline IonDOMExitFrameLayout* ExitFrameLayout::as<IonDOMExitFrameLayout>() {
+ MOZ_ASSERT(is<IonDOMExitFrameLayout>());
+ return reinterpret_cast<IonDOMExitFrameLayout*>(footer());
+}
+
+struct IonDOMMethodExitFrameLayoutTraits {
+ static const size_t offsetOfArgcFromArgv =
+ offsetof(IonDOMMethodExitFrameLayout, argc_) -
+ offsetof(IonDOMMethodExitFrameLayout, argv_);
+};
+
+// Cannot inherit implementation since we need to extend the top of
+// ExitFrameLayout.
+class CalledFromJitExitFrameLayout {
+ protected: // silence clang warning about unused private fields
+ ExitFooterFrame footer_;
+ JitFrameLayout exit_;
+
+ public:
+ static inline size_t Size() { return sizeof(CalledFromJitExitFrameLayout); }
+ inline JitFrameLayout* jsFrame() { return &exit_; }
+ static size_t offsetOfExitFrame() {
+ return offsetof(CalledFromJitExitFrameLayout, exit_);
+ }
+};
+
+class LazyLinkExitFrameLayout : public CalledFromJitExitFrameLayout {
+ public:
+ static ExitFrameType Type() { return ExitFrameType::LazyLink; }
+};
+
+class InterpreterStubExitFrameLayout : public CalledFromJitExitFrameLayout {
+ public:
+ static ExitFrameType Type() { return ExitFrameType::InterpreterStub; }
+};
+
+class WasmGenericJitEntryFrameLayout : CalledFromJitExitFrameLayout {
+ public:
+ static ExitFrameType Type() { return ExitFrameType::WasmGenericJitEntry; }
+};
+
+template <>
+inline bool ExitFrameLayout::is<CalledFromJitExitFrameLayout>() {
+ return is<InterpreterStubExitFrameLayout>() ||
+ is<LazyLinkExitFrameLayout>() || is<WasmGenericJitEntryFrameLayout>();
+}
+
+template <>
+inline CalledFromJitExitFrameLayout*
+ExitFrameLayout::as<CalledFromJitExitFrameLayout>() {
+ MOZ_ASSERT(is<CalledFromJitExitFrameLayout>());
+ uint8_t* sp = reinterpret_cast<uint8_t*>(this);
+ sp -= CalledFromJitExitFrameLayout::offsetOfExitFrame();
+ return reinterpret_cast<CalledFromJitExitFrameLayout*>(sp);
+}
+
+class DirectWasmJitCallFrameLayout {
+ protected: // silence clang warning about unused private fields
+ ExitFooterFrame footer_;
+ ExitFrameLayout exit_;
+
+ public:
+ static ExitFrameType Type() { return ExitFrameType::DirectWasmJitCall; }
+};
+
+class ICStub;
+
+class BaselineStubFrameLayout : public CommonFrameLayout {
+ // Info on the stack
+ //
+ // +-----------------------+
+ // |BaselineStubFrameLayout|
+ // +-----------------------+
+ // | - Descriptor | => Marks end of FrameType::BaselineJS
+ // | - Return address |
+ // | - CallerFramePtr |
+ // +-----------------------+
+ // | - StubPtr | Technically this last field is not part
+ // +-----------------------+ of the frame layout.
+
+ public:
+ static constexpr size_t ICStubOffset = sizeof(void*);
+ static constexpr int ICStubOffsetFromFP = -int(ICStubOffset);
+
+ static inline size_t Size() { return sizeof(BaselineStubFrameLayout); }
+
+ inline ICStub* maybeStubPtr() {
+ uint8_t* fp = reinterpret_cast<uint8_t*>(this);
+ return *reinterpret_cast<ICStub**>(fp - ICStubOffset);
+ }
+};
+
+// An invalidation bailout stack is at the stack pointer for the callee frame.
+class InvalidationBailoutStack {
+ RegisterDump::FPUArray fpregs_;
+ RegisterDump::GPRArray regs_;
+ IonScript* ionScript_;
+ uint8_t* osiPointReturnAddress_;
+
+ public:
+ uint8_t* sp() const {
+ return (uint8_t*)this + sizeof(InvalidationBailoutStack);
+ }
+ JitFrameLayout* fp() const;
+ MachineState machine() { return MachineState::FromBailout(regs_, fpregs_); }
+
+ IonScript* ionScript() const { return ionScript_; }
+ uint8_t* osiPointReturnAddress() const { return osiPointReturnAddress_; }
+ static size_t offsetOfFpRegs() {
+ return offsetof(InvalidationBailoutStack, fpregs_);
+ }
+ static size_t offsetOfRegs() {
+ return offsetof(InvalidationBailoutStack, regs_);
+ }
+
+ void checkInvariants() const;
+};
+
+void GetPcScript(JSContext* cx, JSScript** scriptRes, jsbytecode** pcRes);
+
+// Baseline requires one slot for this/argument type checks.
+static const uint32_t MinJITStackSize = 1;
+
+} /* namespace jit */
+} /* namespace js */
+
+#endif /* jit_JitFrames_h */
diff --git a/js/src/jit/JitOptions.cpp b/js/src/jit/JitOptions.cpp
new file mode 100644
index 0000000000..01be0a5b11
--- /dev/null
+++ b/js/src/jit/JitOptions.cpp
@@ -0,0 +1,393 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/JitOptions.h"
+
+#include <cstdlib>
+#include <type_traits>
+
+#include "vm/JSScript.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::Maybe;
+
+namespace js {
+namespace jit {
+
+DefaultJitOptions JitOptions;
+
+static void Warn(const char* env, const char* value) {
+ fprintf(stderr, "Warning: I didn't understand %s=\"%s\"\n", env, value);
+}
+
+static Maybe<int> ParseInt(const char* str) {
+ char* endp;
+ int retval = strtol(str, &endp, 0);
+ if (*endp == '\0') {
+ return mozilla::Some(retval);
+ }
+ return mozilla::Nothing();
+}
+
+template <typename T>
+T overrideDefault(const char* param, T dflt) {
+ char* str = getenv(param);
+ if (!str) {
+ return dflt;
+ }
+ if constexpr (std::is_same_v<T, bool>) {
+ if (strcmp(str, "true") == 0 || strcmp(str, "yes") == 0) {
+ return true;
+ }
+ if (strcmp(str, "false") == 0 || strcmp(str, "no") == 0) {
+ return false;
+ }
+ Warn(param, str);
+ } else {
+ Maybe<int> value = ParseInt(str);
+ if (value.isSome()) {
+ return value.ref();
+ }
+ Warn(param, str);
+ }
+ return dflt;
+}
+
+#define SET_DEFAULT(var, dflt) var = overrideDefault("JIT_OPTION_" #var, dflt)
+DefaultJitOptions::DefaultJitOptions() {
+ // Whether to perform expensive graph-consistency DEBUG-only assertions.
+ // It can be useful to disable this to reduce DEBUG-compile time of large
+ // wasm programs.
+ SET_DEFAULT(checkGraphConsistency, true);
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ // Emit extra code to verify live regs at the start of a VM call
+ // are not modified before its OsiPoint.
+ SET_DEFAULT(checkOsiPointRegisters, false);
+#endif
+
+ // Whether to enable extra code to perform dynamic validation of
+ // RangeAnalysis results.
+ SET_DEFAULT(checkRangeAnalysis, false);
+
+ // Toggles whether Alignment Mask Analysis is globally disabled.
+ SET_DEFAULT(disableAma, false);
+
+ // Toggles whether Effective Address Analysis is globally disabled.
+ SET_DEFAULT(disableEaa, false);
+
+ // Toggles whether Edge Case Analysis is gobally disabled.
+ SET_DEFAULT(disableEdgeCaseAnalysis, false);
+
+ // Toggle whether global value numbering is globally disabled.
+ SET_DEFAULT(disableGvn, false);
+
+ // Toggles whether inlining is globally disabled.
+ SET_DEFAULT(disableInlining, false);
+
+ // Toggles whether loop invariant code motion is globally disabled.
+ SET_DEFAULT(disableLicm, false);
+
+ // Toggle whether branch pruning is globally disabled.
+ SET_DEFAULT(disablePruning, false);
+
+ // Toggles whether instruction reordering is globally disabled.
+ SET_DEFAULT(disableInstructionReordering, false);
+
+ // Toggles whether Range Analysis is globally disabled.
+ SET_DEFAULT(disableRangeAnalysis, false);
+
+ // Toggles wheter Recover instructions is globally disabled.
+ SET_DEFAULT(disableRecoverIns, false);
+
+ // Toggle whether eager scalar replacement is globally disabled.
+ SET_DEFAULT(disableScalarReplacement, false);
+
+ // Toggles whether CacheIR stubs are used.
+ SET_DEFAULT(disableCacheIR, false);
+
+ // Toggles whether sink code motion is globally disabled.
+ SET_DEFAULT(disableSink, true);
+
+ // Toggles whether redundant shape guard elimination is globally disabled.
+ SET_DEFAULT(disableRedundantShapeGuards, false);
+
+ // Toggles whether we verify that we don't recompile with the same CacheIR.
+ SET_DEFAULT(disableBailoutLoopCheck, false);
+
+ // Whether the Baseline Interpreter is enabled.
+ SET_DEFAULT(baselineInterpreter, true);
+
+ // Whether the Baseline JIT is enabled.
+ SET_DEFAULT(baselineJit, true);
+
+ // Whether the IonMonkey JIT is enabled.
+ SET_DEFAULT(ion, true);
+
+ // Whether the IonMonkey and Baseline JITs are enabled for Trusted Principals.
+ // (Ignored if ion or baselineJit is set to true.)
+ SET_DEFAULT(jitForTrustedPrincipals, false);
+
+ // Whether the RegExp JIT is enabled.
+ SET_DEFAULT(nativeRegExp, true);
+
+ // Whether Warp should use ICs instead of transpiling Baseline CacheIR.
+ SET_DEFAULT(forceInlineCaches, false);
+
+ // Whether all ICs should be initialized as megamorphic ICs.
+ SET_DEFAULT(forceMegamorphicICs, false);
+
+ // Toggles whether large scripts are rejected.
+ SET_DEFAULT(limitScriptSize, true);
+
+ // Toggles whether functions may be entered at loop headers.
+ SET_DEFAULT(osr, true);
+
+ // Whether the JIT backend (used by JITs, Wasm, Baseline Interpreter) has been
+ // disabled for this process. See JS::DisableJitBackend.
+ SET_DEFAULT(disableJitBackend, false);
+
+ // Whether to enable extra code to perform dynamic validations.
+ SET_DEFAULT(runExtraChecks, false);
+
+ // How many invocations or loop iterations are needed before functions
+ // enter the Baseline Interpreter.
+ SET_DEFAULT(baselineInterpreterWarmUpThreshold, 10);
+
+ // How many invocations or loop iterations are needed before functions
+ // are compiled with the baseline compiler.
+ // Duplicated in all.js - ensure both match.
+ SET_DEFAULT(baselineJitWarmUpThreshold, 100);
+
+ // How many invocations or loop iterations are needed before functions
+ // are considered for trial inlining.
+ SET_DEFAULT(trialInliningWarmUpThreshold, 500);
+
+ // The initial warm-up count for ICScripts created by trial inlining.
+ //
+ // Note: the difference between trialInliningInitialWarmUpCount and
+ // trialInliningWarmUpThreshold must be:
+ //
+ // * Small enough to allow inlining multiple levels deep before the outer
+ // script reaches its normalIonWarmUpThreshold.
+ //
+ // * Greater than inliningEntryThreshold or no scripts can be inlined.
+ SET_DEFAULT(trialInliningInitialWarmUpCount, 250);
+
+ // How many invocations or loop iterations are needed before functions
+ // are compiled with the Ion compiler at OptimizationLevel::Normal.
+ // Duplicated in all.js - ensure both match.
+ SET_DEFAULT(normalIonWarmUpThreshold, 1500);
+
+ // How many invocations are needed before regexps are compiled to
+ // native code.
+ SET_DEFAULT(regexpWarmUpThreshold, 10);
+
+ // Number of exception bailouts (resuming into catch/finally block) before
+ // we invalidate and forbid Ion compilation.
+ SET_DEFAULT(exceptionBailoutThreshold, 10);
+
+ // Number of bailouts without invalidation before we set
+ // JSScript::hadFrequentBailouts and invalidate.
+ // Duplicated in all.js - ensure both match.
+ SET_DEFAULT(frequentBailoutThreshold, 10);
+
+ // Whether to run all debug checks in debug builds.
+ // Disabling might make it more enjoyable to run JS in debug builds.
+ SET_DEFAULT(fullDebugChecks, true);
+
+ // How many actual arguments are accepted on the C stack.
+ SET_DEFAULT(maxStackArgs, 4096);
+
+ // How many times we will try to enter a script via OSR before
+ // invalidating the script.
+ SET_DEFAULT(osrPcMismatchesBeforeRecompile, 6000);
+
+ // The bytecode length limit for small function.
+ SET_DEFAULT(smallFunctionMaxBytecodeLength, 130);
+
+ // The minimum entry count for an IC stub before it can be trial-inlined.
+ SET_DEFAULT(inliningEntryThreshold, 100);
+
+ // An artificial testing limit for the maximum supported offset of
+ // pc-relative jump and call instructions.
+ SET_DEFAULT(jumpThreshold, UINT32_MAX);
+
+ // Branch pruning heuristic is based on a scoring system, which is look at
+ // different metrics and provide a score. The score is computed as a
+ // projection where each factor defines the weight of each metric. Then this
+ // score is compared against a threshold to prevent a branch from being
+ // removed.
+ SET_DEFAULT(branchPruningHitCountFactor, 1);
+ SET_DEFAULT(branchPruningInstFactor, 10);
+ SET_DEFAULT(branchPruningBlockSpanFactor, 100);
+ SET_DEFAULT(branchPruningEffectfulInstFactor, 3500);
+ SET_DEFAULT(branchPruningThreshold, 4000);
+
+ // Limits on bytecode length and number of locals/arguments for Ion
+ // compilation. There are different (lower) limits for when off-thread Ion
+ // compilation isn't available.
+ SET_DEFAULT(ionMaxScriptSize, 100 * 1000);
+ SET_DEFAULT(ionMaxScriptSizeMainThread, 2 * 1000);
+ SET_DEFAULT(ionMaxLocalsAndArgs, 10 * 1000);
+ SET_DEFAULT(ionMaxLocalsAndArgsMainThread, 256);
+
+ // Force the used register allocator instead of letting the optimization
+ // pass decide.
+ const char* forcedRegisterAllocatorEnv = "JIT_OPTION_forcedRegisterAllocator";
+ if (const char* env = getenv(forcedRegisterAllocatorEnv)) {
+ forcedRegisterAllocator = LookupRegisterAllocator(env);
+ if (!forcedRegisterAllocator.isSome()) {
+ Warn(forcedRegisterAllocatorEnv, env);
+ }
+ }
+
+#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_LOONG64)
+ SET_DEFAULT(spectreIndexMasking, false);
+ SET_DEFAULT(spectreObjectMitigations, false);
+ SET_DEFAULT(spectreStringMitigations, false);
+ SET_DEFAULT(spectreValueMasking, false);
+ SET_DEFAULT(spectreJitToCxxCalls, false);
+#else
+ SET_DEFAULT(spectreIndexMasking, true);
+ SET_DEFAULT(spectreObjectMitigations, true);
+ SET_DEFAULT(spectreStringMitigations, true);
+ SET_DEFAULT(spectreValueMasking, true);
+ SET_DEFAULT(spectreJitToCxxCalls, true);
+#endif
+
+ // This is set to its actual value in InitializeJit.
+ SET_DEFAULT(supportsUnalignedAccesses, false);
+
+ // To access local (non-argument) slots, it's more efficient to use the frame
+ // pointer (FP) instead of the stack pointer (SP) as base register on x86 and
+ // x64 (because instructions are one byte shorter, for example).
+ //
+ // However, because this requires a negative offset from FP, on ARM64 it can
+ // be more efficient to use SP-relative addresses for larger stack frames
+ // because the range for load/store immediate offsets is [-256, 4095] and
+ // offsets outside this range will require an extra instruction.
+ //
+ // We default to FP-relative addresses on x86/x64 and SP-relative on other
+ // platforms, but to improve fuzzing we allow changing this in the shell:
+ //
+ // setJitCompilerOption("base-reg-for-locals", N); // 0 for SP, 1 for FP
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ baseRegForLocals = BaseRegForAddress::FP;
+#else
+ baseRegForLocals = BaseRegForAddress::SP;
+#endif
+
+ // Toggles the optimization whereby offsets are folded into loads and not
+ // included in the bounds check.
+ SET_DEFAULT(wasmFoldOffsets, true);
+
+ // Controls whether two-tiered compilation should be requested when
+ // compiling a new wasm module, independently of other heuristics, and
+ // should be delayed to test both baseline and ion paths in compiled code,
+ // as well as the transition from one tier to the other.
+ SET_DEFAULT(wasmDelayTier2, false);
+
+ // Until which wasm bytecode size should we accumulate functions, in order
+ // to compile efficiently on helper threads. Baseline code compiles much
+ // faster than Ion code so use scaled thresholds (see also bug 1320374).
+ SET_DEFAULT(wasmBatchBaselineThreshold, 10000);
+ SET_DEFAULT(wasmBatchIonThreshold, 1100);
+
+ // Controls how much assertion checking code is emitted
+ SET_DEFAULT(lessDebugCode, false);
+
+ // Whether the MegamorphicCache is enabled.
+ SET_DEFAULT(enableWatchtowerMegamorphic, true);
+
+ SET_DEFAULT(enableWasmJitExit, true);
+ SET_DEFAULT(enableWasmJitEntry, true);
+ SET_DEFAULT(enableWasmIonFastCalls, true);
+#ifdef WASM_CODEGEN_DEBUG
+ SET_DEFAULT(enableWasmImportCallSpew, false);
+ SET_DEFAULT(enableWasmFuncCallSpew, false);
+#endif
+
+ // This is used to control whether regexps tier up from interpreted to
+ // compiled. We control this with --no-native-regexp and
+ // --regexp-warmup-threshold.
+ SET_DEFAULT(regexp_tier_up, true);
+
+ // Dumps a representation of parsed regexps to stderr
+ SET_DEFAULT(trace_regexp_parser, false);
+ // Dumps the calls made to the regexp assembler to stderr
+ SET_DEFAULT(trace_regexp_assembler, false);
+ // Dumps the bytecodes interpreted by the regexp engine to stderr
+ SET_DEFAULT(trace_regexp_bytecodes, false);
+ // Dumps the changes made by the regexp peephole optimizer to stderr
+ SET_DEFAULT(trace_regexp_peephole_optimization, false);
+
+ // ***** Irregexp shim flags *****
+
+ // V8 uses this for differential fuzzing to handle stack overflows.
+ // We address the same problem in StackLimitCheck::HasOverflowed.
+ SET_DEFAULT(correctness_fuzzer_suppressions, false);
+ // Instead of using a flag for this, we provide an implementation of
+ // CanReadUnaligned in SMRegExpMacroAssembler.
+ SET_DEFAULT(enable_regexp_unaligned_accesses, false);
+ // This is used to guard an old prototype implementation of possessive
+ // quantifiers, which never got past the point of adding parser support.
+ SET_DEFAULT(regexp_possessive_quantifier, false);
+ // These affect the default level of optimization. We can still turn
+ // optimization off on a case-by-case basis in CompilePattern - for
+ // example, if a regexp is too long - so we might as well turn these
+ // flags on unconditionally.
+ SET_DEFAULT(regexp_optimization, true);
+#if MOZ_BIG_ENDIAN()
+ // peephole optimization not supported on big endian
+ SET_DEFAULT(regexp_peephole_optimization, false);
+#else
+ SET_DEFAULT(regexp_peephole_optimization, true);
+#endif
+}
+
+bool DefaultJitOptions::isSmallFunction(JSScript* script) const {
+ return script->length() <= smallFunctionMaxBytecodeLength;
+}
+
+void DefaultJitOptions::enableGvn(bool enable) { disableGvn = !enable; }
+
+void DefaultJitOptions::setEagerBaselineCompilation() {
+ baselineInterpreterWarmUpThreshold = 0;
+ baselineJitWarmUpThreshold = 0;
+ regexpWarmUpThreshold = 0;
+}
+
+void DefaultJitOptions::setEagerIonCompilation() {
+ setEagerBaselineCompilation();
+ normalIonWarmUpThreshold = 0;
+}
+
+void DefaultJitOptions::setFastWarmUp() {
+ baselineInterpreterWarmUpThreshold = 4;
+ baselineJitWarmUpThreshold = 10;
+ trialInliningWarmUpThreshold = 14;
+ trialInliningInitialWarmUpCount = 12;
+ normalIonWarmUpThreshold = 30;
+
+ inliningEntryThreshold = 2;
+ smallFunctionMaxBytecodeLength = 2000;
+}
+
+void DefaultJitOptions::setNormalIonWarmUpThreshold(uint32_t warmUpThreshold) {
+ normalIonWarmUpThreshold = warmUpThreshold;
+}
+
+void DefaultJitOptions::resetNormalIonWarmUpThreshold() {
+ jit::DefaultJitOptions defaultValues;
+ setNormalIonWarmUpThreshold(defaultValues.normalIonWarmUpThreshold);
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/JitOptions.h b/js/src/jit/JitOptions.h
new file mode 100644
index 0000000000..802290162d
--- /dev/null
+++ b/js/src/jit/JitOptions.h
@@ -0,0 +1,174 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitOptions_h
+#define jit_JitOptions_h
+
+#include "mozilla/Maybe.h"
+
+#include "jit/IonTypes.h"
+#include "js/TypeDecls.h"
+
+namespace js {
+namespace jit {
+
+// Possible register allocators which may be used.
+enum IonRegisterAllocator {
+ RegisterAllocator_Backtracking,
+ RegisterAllocator_Testbed,
+};
+
+// Which register to use as base register to access stack slots: frame pointer,
+// stack pointer, or whichever is the default for this platform. See comment
+// for baseRegForLocals in JitOptions.cpp for more information.
+enum class BaseRegForAddress { Default, FP, SP };
+
+static inline mozilla::Maybe<IonRegisterAllocator> LookupRegisterAllocator(
+ const char* name) {
+ if (!strcmp(name, "backtracking")) {
+ return mozilla::Some(RegisterAllocator_Backtracking);
+ }
+ if (!strcmp(name, "testbed")) {
+ return mozilla::Some(RegisterAllocator_Testbed);
+ }
+ return mozilla::Nothing();
+}
+
+struct DefaultJitOptions {
+ bool checkGraphConsistency;
+#ifdef CHECK_OSIPOINT_REGISTERS
+ bool checkOsiPointRegisters;
+#endif
+ bool checkRangeAnalysis;
+ bool runExtraChecks;
+ bool disableJitBackend;
+ bool disableAma;
+ bool disableEaa;
+ bool disableEdgeCaseAnalysis;
+ bool disableGvn;
+ bool disableInlining;
+ bool disableLicm;
+ bool disablePruning;
+ bool disableInstructionReordering;
+ bool disableRangeAnalysis;
+ bool disableRecoverIns;
+ bool disableScalarReplacement;
+ bool disableCacheIR;
+ bool disableSink;
+ bool disableRedundantShapeGuards;
+ bool disableBailoutLoopCheck;
+ bool baselineInterpreter;
+ bool baselineJit;
+ bool ion;
+ bool jitForTrustedPrincipals;
+ bool nativeRegExp;
+ bool forceInlineCaches;
+ bool forceMegamorphicICs;
+ bool fullDebugChecks;
+ bool limitScriptSize;
+ bool osr;
+ bool wasmFoldOffsets;
+ bool wasmDelayTier2;
+ bool lessDebugCode;
+ bool enableWatchtowerMegamorphic;
+ bool enableWasmJitExit;
+ bool enableWasmJitEntry;
+ bool enableWasmIonFastCalls;
+#ifdef WASM_CODEGEN_DEBUG
+ bool enableWasmImportCallSpew;
+ bool enableWasmFuncCallSpew;
+#endif
+ uint32_t baselineInterpreterWarmUpThreshold;
+ uint32_t baselineJitWarmUpThreshold;
+ uint32_t trialInliningWarmUpThreshold;
+ uint32_t trialInliningInitialWarmUpCount;
+ uint32_t normalIonWarmUpThreshold;
+ uint32_t regexpWarmUpThreshold;
+ uint32_t exceptionBailoutThreshold;
+ uint32_t frequentBailoutThreshold;
+ uint32_t maxStackArgs;
+ uint32_t osrPcMismatchesBeforeRecompile;
+ uint32_t smallFunctionMaxBytecodeLength;
+ uint32_t inliningEntryThreshold;
+ uint32_t jumpThreshold;
+ uint32_t branchPruningHitCountFactor;
+ uint32_t branchPruningInstFactor;
+ uint32_t branchPruningBlockSpanFactor;
+ uint32_t branchPruningEffectfulInstFactor;
+ uint32_t branchPruningThreshold;
+ uint32_t ionMaxScriptSize;
+ uint32_t ionMaxScriptSizeMainThread;
+ uint32_t ionMaxLocalsAndArgs;
+ uint32_t ionMaxLocalsAndArgsMainThread;
+ uint32_t wasmBatchBaselineThreshold;
+ uint32_t wasmBatchIonThreshold;
+ mozilla::Maybe<IonRegisterAllocator> forcedRegisterAllocator;
+
+ // Spectre mitigation flags. Each mitigation has its own flag in order to
+ // measure the effectiveness of each mitigation with various proof of
+ // concept.
+ bool spectreIndexMasking;
+ bool spectreObjectMitigations;
+ bool spectreStringMitigations;
+ bool spectreValueMasking;
+ bool spectreJitToCxxCalls;
+
+ bool supportsUnalignedAccesses;
+ BaseRegForAddress baseRegForLocals;
+
+ // Irregexp shim flags
+ bool correctness_fuzzer_suppressions;
+ bool enable_regexp_unaligned_accesses;
+ bool regexp_possessive_quantifier;
+ bool regexp_optimization;
+ bool regexp_peephole_optimization;
+ bool regexp_tier_up;
+ bool trace_regexp_assembler;
+ bool trace_regexp_bytecodes;
+ bool trace_regexp_parser;
+ bool trace_regexp_peephole_optimization;
+
+ DefaultJitOptions();
+ bool isSmallFunction(JSScript* script) const;
+ void setEagerBaselineCompilation();
+ void setEagerIonCompilation();
+ void setNormalIonWarmUpThreshold(uint32_t warmUpThreshold);
+ void resetNormalIonWarmUpThreshold();
+ void enableGvn(bool val);
+ void setFastWarmUp();
+
+ bool eagerIonCompilation() const { return normalIonWarmUpThreshold == 0; }
+};
+
+extern DefaultJitOptions JitOptions;
+
+inline bool HasJitBackend() {
+#if defined(JS_CODEGEN_NONE)
+ return false;
+#else
+ return !JitOptions.disableJitBackend;
+#endif
+}
+
+inline bool IsBaselineInterpreterEnabled() {
+ return HasJitBackend() && JitOptions.baselineInterpreter;
+}
+
+} // namespace jit
+
+extern mozilla::Atomic<bool> fuzzingSafe;
+
+static inline bool IsFuzzing() {
+#ifdef FUZZING
+ return true;
+#else
+ return fuzzingSafe;
+#endif
+}
+
+} // namespace js
+
+#endif /* jit_JitOptions_h */
diff --git a/js/src/jit/JitRealm.h b/js/src/jit/JitRealm.h
new file mode 100644
index 0000000000..e5b1bc172a
--- /dev/null
+++ b/js/src/jit/JitRealm.h
@@ -0,0 +1,161 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitRealm_h
+#define jit_JitRealm_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/EnumeratedArray.h"
+#include "mozilla/MemoryReporting.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "gc/Barrier.h"
+#include "gc/ZoneAllocator.h"
+#include "js/GCHashTable.h"
+#include "js/RootingAPI.h"
+#include "js/TracingAPI.h"
+#include "js/TypeDecls.h"
+
+namespace js {
+
+MOZ_COLD void ReportOutOfMemory(JSContext* cx);
+
+namespace jit {
+
+class JitCode;
+
+class JitRealm {
+ friend class JitActivation;
+
+ // The JitRealm stores stubs to concatenate strings inline and perform RegExp
+ // calls inline. These bake in zone and realm specific pointers and can't be
+ // stored in JitRuntime. They also are dependent on the value of
+ // 'initialStringHeap' and must be flushed when its value changes.
+ //
+ // These are weak pointers, but they can by accessed during off-thread Ion
+ // compilation and therefore can't use the usual read barrier. Instead, we
+ // record which stubs have been read and perform the appropriate barriers in
+ // CodeGenerator::link().
+
+ enum StubIndex : uint32_t {
+ StringConcat = 0,
+ RegExpMatcher,
+ RegExpSearcher,
+ RegExpTester,
+ Count
+ };
+
+ mozilla::EnumeratedArray<StubIndex, StubIndex::Count, WeakHeapPtr<JitCode*>>
+ stubs_;
+
+ gc::InitialHeap initialStringHeap;
+
+ JitCode* generateStringConcatStub(JSContext* cx);
+ JitCode* generateRegExpMatcherStub(JSContext* cx);
+ JitCode* generateRegExpSearcherStub(JSContext* cx);
+ JitCode* generateRegExpTesterStub(JSContext* cx);
+
+ JitCode* getStubNoBarrier(StubIndex stub,
+ uint32_t* requiredBarriersOut) const {
+ MOZ_ASSERT(CurrentThreadIsIonCompiling());
+ *requiredBarriersOut |= 1 << uint32_t(stub);
+ return stubs_[stub].unbarrieredGet();
+ }
+
+ public:
+ JitRealm();
+
+ void initialize(bool zoneHasNurseryStrings);
+
+ // Initialize code stubs only used by Ion, not Baseline.
+ [[nodiscard]] bool ensureIonStubsExist(JSContext* cx) {
+ if (stubs_[StringConcat]) {
+ return true;
+ }
+ stubs_[StringConcat] = generateStringConcatStub(cx);
+ return stubs_[StringConcat];
+ }
+
+ void traceWeak(JSTracer* trc, JS::Realm* realm);
+
+ void discardStubs() {
+ for (WeakHeapPtr<JitCode*>& stubRef : stubs_) {
+ stubRef = nullptr;
+ }
+ }
+
+ bool hasStubs() const {
+ for (const WeakHeapPtr<JitCode*>& stubRef : stubs_) {
+ if (stubRef) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ void setStringsCanBeInNursery(bool allow) {
+ MOZ_ASSERT(!hasStubs());
+ initialStringHeap = allow ? gc::DefaultHeap : gc::TenuredHeap;
+ }
+
+ JitCode* stringConcatStubNoBarrier(uint32_t* requiredBarriersOut) const {
+ return getStubNoBarrier(StringConcat, requiredBarriersOut);
+ }
+
+ JitCode* regExpMatcherStubNoBarrier(uint32_t* requiredBarriersOut) const {
+ return getStubNoBarrier(RegExpMatcher, requiredBarriersOut);
+ }
+
+ [[nodiscard]] bool ensureRegExpMatcherStubExists(JSContext* cx) {
+ if (stubs_[RegExpMatcher]) {
+ return true;
+ }
+ stubs_[RegExpMatcher] = generateRegExpMatcherStub(cx);
+ return stubs_[RegExpMatcher];
+ }
+
+ JitCode* regExpSearcherStubNoBarrier(uint32_t* requiredBarriersOut) const {
+ return getStubNoBarrier(RegExpSearcher, requiredBarriersOut);
+ }
+
+ [[nodiscard]] bool ensureRegExpSearcherStubExists(JSContext* cx) {
+ if (stubs_[RegExpSearcher]) {
+ return true;
+ }
+ stubs_[RegExpSearcher] = generateRegExpSearcherStub(cx);
+ return stubs_[RegExpSearcher];
+ }
+
+ JitCode* regExpTesterStubNoBarrier(uint32_t* requiredBarriersOut) const {
+ return getStubNoBarrier(RegExpTester, requiredBarriersOut);
+ }
+
+ [[nodiscard]] bool ensureRegExpTesterStubExists(JSContext* cx) {
+ if (stubs_[RegExpTester]) {
+ return true;
+ }
+ stubs_[RegExpTester] = generateRegExpTesterStub(cx);
+ return stubs_[RegExpTester];
+ }
+
+ // Perform the necessary read barriers on stubs described by the bitmasks
+ // passed in. This function can only be called from the main thread.
+ //
+ // The stub pointers must still be valid by the time these methods are
+ // called. This is arranged by cancelling off-thread Ion compilation at the
+ // start of GC and at the start of sweeping.
+ void performStubReadBarriers(uint32_t stubsToBarrier) const;
+
+ size_t sizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const;
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitRealm_h */
diff --git a/js/src/jit/JitRuntime.h b/js/src/jit/JitRuntime.h
new file mode 100644
index 0000000000..1e1018e7f7
--- /dev/null
+++ b/js/src/jit/JitRuntime.h
@@ -0,0 +1,415 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitRuntime_h
+#define jit_JitRuntime_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/EnumeratedArray.h"
+#include "mozilla/LinkedList.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jstypes.h"
+
+#include "jit/ABIFunctions.h"
+#include "jit/BaselineICList.h"
+#include "jit/BaselineJIT.h"
+#include "jit/CalleeToken.h"
+#include "jit/IonCompileTask.h"
+#include "jit/IonTypes.h"
+#include "jit/JitCode.h"
+#include "jit/shared/Assembler-shared.h"
+#include "js/AllocPolicy.h"
+#include "js/ProfilingFrameIterator.h"
+#include "js/TypeDecls.h"
+#include "js/UniquePtr.h"
+#include "js/Vector.h"
+#include "threading/ProtectedData.h"
+#include "vm/GeckoProfiler.h"
+#include "vm/Runtime.h"
+
+class JS_PUBLIC_API JSTracer;
+
+namespace js {
+
+class AutoLockHelperThreadState;
+class GCMarker;
+
+namespace jit {
+
+class FrameSizeClass;
+class JitRealm;
+class Label;
+class MacroAssembler;
+struct VMFunctionData;
+
+enum class TailCallVMFunctionId;
+enum class VMFunctionId;
+
+enum class BaselineICFallbackKind : uint8_t {
+#define DEF_ENUM_KIND(kind) kind,
+ IC_BASELINE_FALLBACK_CODE_KIND_LIST(DEF_ENUM_KIND)
+#undef DEF_ENUM_KIND
+ Count
+};
+
+enum class BailoutReturnKind {
+ GetProp,
+ GetPropSuper,
+ SetProp,
+ GetElem,
+ GetElemSuper,
+ Call,
+ New,
+ Count
+};
+
+// Class storing code and offsets for all Baseline IC fallback trampolines. This
+// is stored in JitRuntime and generated when creating the JitRuntime.
+class BaselineICFallbackCode {
+ JitCode* code_ = nullptr;
+ using OffsetArray =
+ mozilla::EnumeratedArray<BaselineICFallbackKind,
+ BaselineICFallbackKind::Count, uint32_t>;
+ OffsetArray offsets_ = {};
+
+ // Keep track of offset into various baseline stubs' code at return
+ // point from called script.
+ using BailoutReturnArray =
+ mozilla::EnumeratedArray<BailoutReturnKind, BailoutReturnKind::Count,
+ uint32_t>;
+ BailoutReturnArray bailoutReturnOffsets_ = {};
+
+ public:
+ BaselineICFallbackCode() = default;
+ BaselineICFallbackCode(const BaselineICFallbackCode&) = delete;
+ void operator=(const BaselineICFallbackCode&) = delete;
+
+ void initOffset(BaselineICFallbackKind kind, uint32_t offset) {
+ offsets_[kind] = offset;
+ }
+ void initCode(JitCode* code) { code_ = code; }
+ void initBailoutReturnOffset(BailoutReturnKind kind, uint32_t offset) {
+ bailoutReturnOffsets_[kind] = offset;
+ }
+ TrampolinePtr addr(BaselineICFallbackKind kind) const {
+ return TrampolinePtr(code_->raw() + offsets_[kind]);
+ }
+ uint8_t* bailoutReturnAddr(BailoutReturnKind kind) const {
+ return code_->raw() + bailoutReturnOffsets_[kind];
+ }
+};
+
+enum class ArgumentsRectifierKind { Normal, TrialInlining };
+
+enum class DebugTrapHandlerKind { Interpreter, Compiler, Count };
+
+using EnterJitCode = void (*)(void*, unsigned int, Value*, InterpreterFrame*,
+ CalleeToken, JSObject*, size_t, Value*);
+
+class JitcodeGlobalTable;
+
+class JitRuntime {
+ private:
+ friend class JitRealm;
+
+ MainThreadData<uint64_t> nextCompilationId_{0};
+
+ // Buffer for OSR from baseline to Ion. To avoid holding on to this for too
+ // long it's also freed in EnterBaseline and EnterJit (after returning from
+ // JIT code).
+ MainThreadData<js::UniquePtr<uint8_t>> ionOsrTempData_{nullptr};
+
+ // Shared exception-handler tail.
+ WriteOnceData<uint32_t> exceptionTailOffset_{0};
+
+ // Shared profiler exit frame tail.
+ WriteOnceData<uint32_t> profilerExitFrameTailOffset_{0};
+
+ // Trampoline for entering JIT code.
+ WriteOnceData<uint32_t> enterJITOffset_{0};
+
+ // Generic bailout table; used if the bailout table overflows.
+ WriteOnceData<uint32_t> bailoutHandlerOffset_{0};
+
+ // Argument-rectifying thunks, in the case of insufficient arguments passed
+ // to a function call site. The return offset is used to rebuild stack frames
+ // when bailing out.
+ WriteOnceData<uint32_t> argumentsRectifierOffset_{0};
+ WriteOnceData<uint32_t> trialInliningArgumentsRectifierOffset_{0};
+ WriteOnceData<uint32_t> argumentsRectifierReturnOffset_{0};
+
+ // Thunk that invalides an (Ion compiled) caller on the Ion stack.
+ WriteOnceData<uint32_t> invalidatorOffset_{0};
+
+ // Thunk that calls the GC pre barrier.
+ WriteOnceData<uint32_t> valuePreBarrierOffset_{0};
+ WriteOnceData<uint32_t> stringPreBarrierOffset_{0};
+ WriteOnceData<uint32_t> objectPreBarrierOffset_{0};
+ WriteOnceData<uint32_t> shapePreBarrierOffset_{0};
+
+ // Thunk to call malloc/free.
+ WriteOnceData<uint32_t> freeStubOffset_{0};
+
+ // Thunk called to finish compilation of an IonScript.
+ WriteOnceData<uint32_t> lazyLinkStubOffset_{0};
+
+ // Thunk to enter the interpreter from JIT code.
+ WriteOnceData<uint32_t> interpreterStubOffset_{0};
+
+ // Thunk to convert the value in R0 to int32 if it's a double.
+ // Note: this stub treats -0 as +0 and may clobber R1.scratchReg().
+ WriteOnceData<uint32_t> doubleToInt32ValueStubOffset_{0};
+
+ // Thunk used by the debugger for breakpoint and step mode.
+ mozilla::EnumeratedArray<DebugTrapHandlerKind, DebugTrapHandlerKind::Count,
+ WriteOnceData<JitCode*>>
+ debugTrapHandlers_;
+
+ // BaselineInterpreter state.
+ BaselineInterpreter baselineInterpreter_;
+
+ // Code for trampolines and VMFunction wrappers.
+ WriteOnceData<JitCode*> trampolineCode_{nullptr};
+
+ // Maps VMFunctionId to the offset of the wrapper code in trampolineCode_.
+ using VMWrapperOffsets = Vector<uint32_t, 0, SystemAllocPolicy>;
+ VMWrapperOffsets functionWrapperOffsets_;
+
+ // Maps TailCallVMFunctionId to the offset of the wrapper code in
+ // trampolineCode_.
+ VMWrapperOffsets tailCallFunctionWrapperOffsets_;
+
+ MainThreadData<BaselineICFallbackCode> baselineICFallbackCode_;
+
+ // Global table of jitcode native address => bytecode address mappings.
+ UnprotectedData<JitcodeGlobalTable*> jitcodeGlobalTable_{nullptr};
+
+#ifdef DEBUG
+ // The number of possible bailing places encountered before forcefully bailing
+ // in that place if the counter reaches zero. Note that zero also means
+ // inactive.
+ MainThreadData<uint32_t> ionBailAfterCounter_{0};
+
+ // Whether the bailAfter mechanism is enabled. Used to avoid generating the
+ // Ion code instrumentation for ionBailAfterCounter_ if the testing function
+ // isn't used.
+ MainThreadData<bool> ionBailAfterEnabled_{false};
+#endif
+
+ // Number of Ion compilations which were finished off thread and are
+ // waiting to be lazily linked. This is only set while holding the helper
+ // thread state lock, but may be read from at other times.
+ typedef mozilla::Atomic<size_t, mozilla::SequentiallyConsistent>
+ NumFinishedOffThreadTasksType;
+ NumFinishedOffThreadTasksType numFinishedOffThreadTasks_{0};
+
+ // List of Ion compilation waiting to get linked.
+ using IonCompileTaskList = mozilla::LinkedList<js::jit::IonCompileTask>;
+ MainThreadData<IonCompileTaskList> ionLazyLinkList_;
+ MainThreadData<size_t> ionLazyLinkListSize_{0};
+
+#ifdef DEBUG
+ // Flag that can be set from JIT code to indicate it's invalid to call
+ // arbitrary JS code in a particular region. This is checked in RunScript.
+ MainThreadData<uint32_t> disallowArbitraryCode_{false};
+#endif
+
+ bool generateTrampolines(JSContext* cx);
+ bool generateBaselineICFallbackCode(JSContext* cx);
+
+ void generateLazyLinkStub(MacroAssembler& masm);
+ void generateInterpreterStub(MacroAssembler& masm);
+ void generateDoubleToInt32ValueStub(MacroAssembler& masm);
+ void generateProfilerExitFrameTailStub(MacroAssembler& masm,
+ Label* profilerExitTail);
+ void generateExceptionTailStub(MacroAssembler& masm, Label* profilerExitTail,
+ Label* bailoutTail);
+ void generateBailoutTailStub(MacroAssembler& masm, Label* bailoutTail);
+ void generateEnterJIT(JSContext* cx, MacroAssembler& masm);
+ void generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind);
+ void generateBailoutHandler(MacroAssembler& masm, Label* bailoutTail);
+ void generateInvalidator(MacroAssembler& masm, Label* bailoutTail);
+ uint32_t generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type);
+ void generateFreeStub(MacroAssembler& masm);
+ JitCode* generateDebugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind);
+
+ bool generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset);
+
+ template <typename IdT>
+ bool generateVMWrappers(JSContext* cx, MacroAssembler& masm,
+ VMWrapperOffsets& offsets);
+ bool generateVMWrappers(JSContext* cx, MacroAssembler& masm);
+
+ uint32_t startTrampolineCode(MacroAssembler& masm);
+
+ TrampolinePtr trampolineCode(uint32_t offset) const {
+ MOZ_ASSERT(offset > 0);
+ MOZ_ASSERT(offset < trampolineCode_->instructionsSize());
+ return TrampolinePtr(trampolineCode_->raw() + offset);
+ }
+
+ public:
+ JitRuntime() = default;
+ ~JitRuntime();
+ [[nodiscard]] bool initialize(JSContext* cx);
+
+ static void TraceAtomZoneRoots(JSTracer* trc);
+ [[nodiscard]] static bool MarkJitcodeGlobalTableIteratively(GCMarker* marker);
+ static void TraceWeakJitcodeGlobalTable(JSRuntime* rt, JSTracer* trc);
+
+ const BaselineICFallbackCode& baselineICFallbackCode() const {
+ return baselineICFallbackCode_.ref();
+ }
+
+ IonCompilationId nextCompilationId() {
+ return IonCompilationId(nextCompilationId_++);
+ }
+
+#ifdef DEBUG
+ bool disallowArbitraryCode() const { return disallowArbitraryCode_; }
+ void clearDisallowArbitraryCode() { disallowArbitraryCode_ = false; }
+ const void* addressOfDisallowArbitraryCode() const {
+ return &disallowArbitraryCode_.refNoCheck();
+ }
+#endif
+
+ uint8_t* allocateIonOsrTempData(size_t size);
+ void freeIonOsrTempData();
+
+ TrampolinePtr getVMWrapper(VMFunctionId funId) const {
+ MOZ_ASSERT(trampolineCode_);
+ return trampolineCode(functionWrapperOffsets_[size_t(funId)]);
+ }
+ TrampolinePtr getVMWrapper(TailCallVMFunctionId funId) const {
+ MOZ_ASSERT(trampolineCode_);
+ return trampolineCode(tailCallFunctionWrapperOffsets_[size_t(funId)]);
+ }
+
+ JitCode* debugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind);
+
+ BaselineInterpreter& baselineInterpreter() { return baselineInterpreter_; }
+
+ TrampolinePtr getGenericBailoutHandler() const {
+ return trampolineCode(bailoutHandlerOffset_);
+ }
+
+ TrampolinePtr getExceptionTail() const {
+ return trampolineCode(exceptionTailOffset_);
+ }
+
+ TrampolinePtr getProfilerExitFrameTail() const {
+ return trampolineCode(profilerExitFrameTailOffset_);
+ }
+
+ TrampolinePtr getArgumentsRectifier(
+ ArgumentsRectifierKind kind = ArgumentsRectifierKind::Normal) const {
+ if (kind == ArgumentsRectifierKind::TrialInlining) {
+ return trampolineCode(trialInliningArgumentsRectifierOffset_);
+ }
+ return trampolineCode(argumentsRectifierOffset_);
+ }
+
+ TrampolinePtr getArgumentsRectifierReturnAddr() const {
+ return trampolineCode(argumentsRectifierReturnOffset_);
+ }
+
+ TrampolinePtr getInvalidationThunk() const {
+ return trampolineCode(invalidatorOffset_);
+ }
+
+ EnterJitCode enterJit() const {
+ return JS_DATA_TO_FUNC_PTR(EnterJitCode,
+ trampolineCode(enterJITOffset_).value);
+ }
+
+ // Return the registers from the native caller frame of the given JIT frame.
+ // Nothing{} if frameStackAddress is NOT pointing at a native-to-JIT entry
+ // frame, or if the information is not accessible/implemented on this
+ // platform.
+ static mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+ getCppEntryRegisters(JitFrameLayout* frameStackAddress);
+
+ TrampolinePtr preBarrier(MIRType type) const {
+ switch (type) {
+ case MIRType::Value:
+ return trampolineCode(valuePreBarrierOffset_);
+ case MIRType::String:
+ return trampolineCode(stringPreBarrierOffset_);
+ case MIRType::Object:
+ return trampolineCode(objectPreBarrierOffset_);
+ case MIRType::Shape:
+ return trampolineCode(shapePreBarrierOffset_);
+ default:
+ MOZ_CRASH();
+ }
+ }
+
+ TrampolinePtr freeStub() const { return trampolineCode(freeStubOffset_); }
+
+ TrampolinePtr lazyLinkStub() const {
+ return trampolineCode(lazyLinkStubOffset_);
+ }
+ TrampolinePtr interpreterStub() const {
+ return trampolineCode(interpreterStubOffset_);
+ }
+
+ TrampolinePtr getDoubleToInt32ValueStub() const {
+ return trampolineCode(doubleToInt32ValueStubOffset_);
+ }
+
+ bool hasJitcodeGlobalTable() const { return jitcodeGlobalTable_ != nullptr; }
+
+ JitcodeGlobalTable* getJitcodeGlobalTable() {
+ MOZ_ASSERT(hasJitcodeGlobalTable());
+ return jitcodeGlobalTable_;
+ }
+
+ bool isProfilerInstrumentationEnabled(JSRuntime* rt) {
+ return rt->geckoProfiler().enabled();
+ }
+
+ bool isOptimizationTrackingEnabled(JSRuntime* rt) {
+ return isProfilerInstrumentationEnabled(rt);
+ }
+
+#ifdef DEBUG
+ void* addressOfIonBailAfterCounter() { return &ionBailAfterCounter_; }
+
+ // Set after how many bailing places we should forcefully bail.
+ // Zero disables this feature.
+ void setIonBailAfterCounter(uint32_t after) { ionBailAfterCounter_ = after; }
+ bool ionBailAfterEnabled() const { return ionBailAfterEnabled_; }
+ void setIonBailAfterEnabled(bool enabled) { ionBailAfterEnabled_ = enabled; }
+#endif
+
+ size_t numFinishedOffThreadTasks() const {
+ return numFinishedOffThreadTasks_;
+ }
+ NumFinishedOffThreadTasksType& numFinishedOffThreadTasksRef(
+ const AutoLockHelperThreadState& locked) {
+ return numFinishedOffThreadTasks_;
+ }
+
+ IonCompileTaskList& ionLazyLinkList(JSRuntime* rt);
+
+ size_t ionLazyLinkListSize() const { return ionLazyLinkListSize_; }
+
+ void ionLazyLinkListRemove(JSRuntime* rt, js::jit::IonCompileTask* task);
+ void ionLazyLinkListAdd(JSRuntime* rt, js::jit::IonCompileTask* task);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitRuntime_h */
diff --git a/js/src/jit/JitScript-inl.h b/js/src/jit/JitScript-inl.h
new file mode 100644
index 0000000000..e27745ffab
--- /dev/null
+++ b/js/src/jit/JitScript-inl.h
@@ -0,0 +1,43 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitScript_inl_h
+#define jit_JitScript_inl_h
+
+#include "jit/JitScript.h"
+
+#include "mozilla/Assertions.h"
+
+#include "gc/Zone.h"
+#include "jit/JitZone.h"
+#include "vm/JSContext.h"
+#include "vm/JSScript.h"
+
+namespace js {
+namespace jit {
+
+inline AutoKeepJitScripts::AutoKeepJitScripts(JSContext* cx)
+ : zone_(cx->zone()->jitZone()), prev_(zone_->keepJitScripts()) {
+ zone_->setKeepJitScripts(true);
+}
+
+inline AutoKeepJitScripts::~AutoKeepJitScripts() {
+ MOZ_ASSERT(zone_->keepJitScripts());
+ zone_->setKeepJitScripts(prev_);
+}
+
+} // namespace jit
+} // namespace js
+
+inline bool JSScript::ensureHasJitScript(JSContext* cx,
+ js::jit::AutoKeepJitScripts&) {
+ if (hasJitScript()) {
+ return true;
+ }
+ return createJitScript(cx);
+}
+
+#endif /* jit_JitScript_inl_h */
diff --git a/js/src/jit/JitScript.cpp b/js/src/jit/JitScript.cpp
new file mode 100644
index 0000000000..21fbbd1b9e
--- /dev/null
+++ b/js/src/jit/JitScript.cpp
@@ -0,0 +1,735 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/JitScript-inl.h"
+
+#include "mozilla/BinarySearch.h"
+#include "mozilla/CheckedInt.h"
+
+#include <utility>
+
+#include "jit/BaselineIC.h"
+#include "jit/BaselineJIT.h"
+#include "jit/BytecodeAnalysis.h"
+#include "jit/IonScript.h"
+#include "jit/JitFrames.h"
+#include "jit/JitSpewer.h"
+#include "jit/ScriptFromCalleeToken.h"
+#include "jit/TrialInlining.h"
+#include "vm/BytecodeUtil.h"
+#include "vm/Compartment.h"
+#include "vm/FrameIter.h" // js::OnlyJSJitFrameIter
+#include "vm/JitActivation.h"
+#include "vm/JSScript.h"
+
+#include "gc/GCContext-inl.h"
+#include "jit/JSJitFrameIter-inl.h"
+#include "vm/JSContext-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::CheckedInt;
+
+JitScript::JitScript(JSScript* script, Offset fallbackStubsOffset,
+ Offset endOffset, const char* profileString)
+ : profileString_(profileString),
+ endOffset_(endOffset),
+ icScript_(script->getWarmUpCount(),
+ fallbackStubsOffset - offsetOfICScript(),
+ endOffset - offsetOfICScript(),
+ /*depth=*/0) {
+ // Ensure the baselineScript_ and ionScript_ fields match the BaselineDisabled
+ // and IonDisabled script flags.
+ if (!script->canBaselineCompile()) {
+ setBaselineScriptImpl(script, BaselineDisabledScriptPtr);
+ }
+ if (!script->canIonCompile()) {
+ setIonScriptImpl(script, IonDisabledScriptPtr);
+ }
+}
+
+#ifdef DEBUG
+JitScript::~JitScript() {
+ // The contents of the stub space are removed and freed separately after the
+ // next minor GC. See prepareForDestruction.
+ MOZ_ASSERT(jitScriptStubSpace_.isEmpty());
+
+ // BaselineScript and IonScript must have been destroyed at this point.
+ MOZ_ASSERT(!hasBaselineScript());
+ MOZ_ASSERT(!hasIonScript());
+}
+#else
+JitScript::~JitScript() = default;
+#endif
+
+bool JSScript::createJitScript(JSContext* cx) {
+ MOZ_ASSERT(!hasJitScript());
+ cx->check(this);
+
+ // Scripts with a JitScript can run in the Baseline Interpreter. Make sure
+ // we don't create a JitScript for scripts we shouldn't Baseline interpret.
+ MOZ_ASSERT_IF(IsBaselineInterpreterEnabled(),
+ CanBaselineInterpretScript(this));
+
+ // Store the profile string in the JitScript if the profiler is enabled.
+ const char* profileString = nullptr;
+ if (cx->runtime()->geckoProfiler().enabled()) {
+ profileString = cx->runtime()->geckoProfiler().profileString(cx, this);
+ if (!profileString) {
+ return false;
+ }
+ }
+
+ static_assert(sizeof(JitScript) % sizeof(uintptr_t) == 0,
+ "Trailing arrays must be aligned properly");
+ static_assert(sizeof(ICEntry) % sizeof(uintptr_t) == 0,
+ "Trailing arrays must be aligned properly");
+
+ static_assert(
+ sizeof(JitScript) == offsetof(JitScript, icScript_) + sizeof(ICScript),
+ "icScript_ must be the last field");
+
+ // Calculate allocation size.
+ CheckedInt<uint32_t> allocSize = sizeof(JitScript);
+ allocSize += CheckedInt<uint32_t>(numICEntries()) * sizeof(ICEntry);
+ allocSize += CheckedInt<uint32_t>(numICEntries()) * sizeof(ICFallbackStub);
+ if (!allocSize.isValid()) {
+ ReportAllocationOverflow(cx);
+ return false;
+ }
+
+ void* raw = cx->pod_malloc<uint8_t>(allocSize.value());
+ MOZ_ASSERT(uintptr_t(raw) % alignof(JitScript) == 0);
+ if (!raw) {
+ return false;
+ }
+
+ size_t fallbackStubsOffset =
+ sizeof(JitScript) + numICEntries() * sizeof(ICEntry);
+
+ UniquePtr<JitScript> jitScript(new (raw) JitScript(
+ this, fallbackStubsOffset, allocSize.value(), profileString));
+
+ // Sanity check the length computation.
+ MOZ_ASSERT(jitScript->numICEntries() == numICEntries());
+
+ jitScript->icScript()->initICEntries(cx, this);
+
+ warmUpData_.initJitScript(jitScript.release());
+ AddCellMemory(this, allocSize.value(), MemoryUse::JitScript);
+
+ // We have a JitScript so we can set the script's jitCodeRaw pointer to the
+ // Baseline Interpreter code.
+ updateJitCodeRaw(cx->runtime());
+
+ return true;
+}
+
+void JSScript::maybeReleaseJitScript(JS::GCContext* gcx) {
+ MOZ_ASSERT(hasJitScript());
+
+ if (zone()->jitZone()->keepJitScripts() || jitScript()->hasBaselineScript() ||
+ jitScript()->active()) {
+ return;
+ }
+
+ releaseJitScript(gcx);
+}
+
+void JSScript::releaseJitScript(JS::GCContext* gcx) {
+ MOZ_ASSERT(hasJitScript());
+ MOZ_ASSERT(!hasBaselineScript());
+ MOZ_ASSERT(!hasIonScript());
+
+ gcx->removeCellMemory(this, jitScript()->allocBytes(), MemoryUse::JitScript);
+
+ JitScript::Destroy(zone(), jitScript());
+ warmUpData_.clearJitScript();
+ updateJitCodeRaw(gcx->runtime());
+}
+
+void JSScript::releaseJitScriptOnFinalize(JS::GCContext* gcx) {
+ MOZ_ASSERT(hasJitScript());
+
+ if (hasIonScript()) {
+ IonScript* ion = jitScript()->clearIonScript(gcx, this);
+ jit::IonScript::Destroy(gcx, ion);
+ }
+
+ if (hasBaselineScript()) {
+ BaselineScript* baseline = jitScript()->clearBaselineScript(gcx, this);
+ jit::BaselineScript::Destroy(gcx, baseline);
+ }
+
+ releaseJitScript(gcx);
+}
+
+void JitScript::CachedIonData::trace(JSTracer* trc) {
+ TraceNullableEdge(trc, &templateEnv, "jitscript-iondata-template-env");
+}
+
+void JitScript::trace(JSTracer* trc) {
+ icScript_.trace(trc);
+
+ if (hasBaselineScript()) {
+ baselineScript()->trace(trc);
+ }
+
+ if (hasIonScript()) {
+ ionScript()->trace(trc);
+ }
+
+ if (hasCachedIonData()) {
+ cachedIonData().trace(trc);
+ }
+
+ if (hasInliningRoot()) {
+ inliningRoot()->trace(trc);
+ }
+}
+
+void ICScript::trace(JSTracer* trc) {
+ // Mark all IC stub codes hanging off the IC stub entries.
+ for (size_t i = 0; i < numICEntries(); i++) {
+ ICEntry& ent = icEntry(i);
+ ent.trace(trc);
+ }
+}
+
+bool ICScript::addInlinedChild(JSContext* cx, UniquePtr<ICScript> child,
+ uint32_t pcOffset) {
+ MOZ_ASSERT(!hasInlinedChild(pcOffset));
+
+ if (!inlinedChildren_) {
+ inlinedChildren_ = cx->make_unique<Vector<CallSite>>(cx);
+ if (!inlinedChildren_) {
+ return false;
+ }
+ }
+
+ // First reserve space in inlinedChildren_ to ensure that if the ICScript is
+ // added to the inlining root, it can also be added to inlinedChildren_.
+ CallSite callsite(child.get(), pcOffset);
+ if (!inlinedChildren_->reserve(inlinedChildren_->length() + 1)) {
+ return false;
+ }
+ if (!inliningRoot()->addInlinedScript(std::move(child))) {
+ return false;
+ }
+ inlinedChildren_->infallibleAppend(callsite);
+ return true;
+}
+
+ICScript* ICScript::findInlinedChild(uint32_t pcOffset) {
+ for (auto& callsite : *inlinedChildren_) {
+ if (callsite.pcOffset_ == pcOffset) {
+ return callsite.callee_;
+ }
+ }
+ MOZ_CRASH("Inlined child expected at pcOffset");
+}
+
+void ICScript::removeInlinedChild(uint32_t pcOffset) {
+ MOZ_ASSERT(inliningRoot());
+ inlinedChildren_->eraseIf([pcOffset](const CallSite& callsite) -> bool {
+ return callsite.pcOffset_ == pcOffset;
+ });
+}
+
+bool ICScript::hasInlinedChild(uint32_t pcOffset) {
+ if (!inlinedChildren_) {
+ return false;
+ }
+ for (auto& callsite : *inlinedChildren_) {
+ if (callsite.pcOffset_ == pcOffset) {
+ return true;
+ }
+ }
+ return false;
+}
+
+void JitScript::resetWarmUpCount(uint32_t count) {
+ icScript_.resetWarmUpCount(count);
+ if (hasInliningRoot()) {
+ inliningRoot()->resetWarmUpCounts(count);
+ }
+}
+
+void JitScript::ensureProfileString(JSContext* cx, JSScript* script) {
+ MOZ_ASSERT(cx->runtime()->geckoProfiler().enabled());
+
+ if (profileString_) {
+ return;
+ }
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ profileString_ = cx->runtime()->geckoProfiler().profileString(cx, script);
+ if (!profileString_) {
+ oomUnsafe.crash("Failed to allocate profile string");
+ }
+}
+
+/* static */
+void JitScript::Destroy(Zone* zone, JitScript* script) {
+ script->prepareForDestruction(zone);
+
+ js_delete(script);
+}
+
+void JitScript::prepareForDestruction(Zone* zone) {
+ // When the script contains pointers to nursery things, the store buffer can
+ // contain entries that point into the fallback stub space. Since we can
+ // destroy scripts outside the context of a GC, this situation could result
+ // in us trying to mark invalid store buffer entries.
+ //
+ // Defer freeing any allocated blocks until after the next minor GC.
+ jitScriptStubSpace_.freeAllAfterMinorGC(zone);
+
+ // Trigger write barriers.
+ baselineScript_.set(zone, nullptr);
+ ionScript_.set(zone, nullptr);
+}
+
+struct FallbackStubs {
+ ICScript* const icScript_;
+
+ explicit FallbackStubs(ICScript* icScript) : icScript_(icScript) {}
+
+ size_t numEntries() const { return icScript_->numICEntries(); }
+ ICFallbackStub* operator[](size_t index) const {
+ return icScript_->fallbackStub(index);
+ }
+};
+
+static bool ComputeBinarySearchMid(FallbackStubs stubs, uint32_t pcOffset,
+ size_t* loc) {
+ return mozilla::BinarySearchIf(
+ stubs, 0, stubs.numEntries(),
+ [pcOffset](const ICFallbackStub* stub) {
+ if (pcOffset < stub->pcOffset()) {
+ return -1;
+ }
+ if (stub->pcOffset() < pcOffset) {
+ return 1;
+ }
+ return 0;
+ },
+ loc);
+}
+
+ICEntry& ICScript::icEntryFromPCOffset(uint32_t pcOffset) {
+ size_t mid;
+ MOZ_ALWAYS_TRUE(ComputeBinarySearchMid(FallbackStubs(this), pcOffset, &mid));
+
+ MOZ_ASSERT(mid < numICEntries());
+
+ ICEntry& entry = icEntry(mid);
+ MOZ_ASSERT(fallbackStubForICEntry(&entry)->pcOffset() == pcOffset);
+ return entry;
+}
+
+ICEntry* ICScript::interpreterICEntryFromPCOffset(uint32_t pcOffset) {
+ // We have to return the entry to store in BaselineFrame::interpreterICEntry
+ // when resuming in the Baseline Interpreter at pcOffset. The bytecode op at
+ // pcOffset does not necessarily have an ICEntry, so we want to return the
+ // first ICEntry for which the following is true:
+ //
+ // entry.pcOffset() >= pcOffset
+ //
+ // Fortunately, ComputeBinarySearchMid returns exactly this entry.
+
+ size_t mid;
+ ComputeBinarySearchMid(FallbackStubs(this), pcOffset, &mid);
+
+ if (mid < numICEntries()) {
+ ICEntry& entry = icEntry(mid);
+ MOZ_ASSERT(fallbackStubForICEntry(&entry)->pcOffset() >= pcOffset);
+ return &entry;
+ }
+
+ // Resuming at a pc after the last ICEntry. Just return nullptr:
+ // BaselineFrame::interpreterICEntry will never be used in this case.
+ return nullptr;
+}
+
+void JitScript::purgeOptimizedStubs(JSScript* script) {
+ MOZ_ASSERT(script->jitScript() == this);
+
+ Zone* zone = script->zone();
+ if (IsAboutToBeFinalizedUnbarriered(script)) {
+ // We're sweeping and the script is dead. Don't purge optimized stubs
+ // because (1) accessing CacheIRStubInfo pointers in ICStubs is invalid
+ // because we may have swept them already when we started (incremental)
+ // sweeping and (2) it's unnecessary because this script will be finalized
+ // soon anyway.
+ return;
+ }
+
+ JitSpew(JitSpew_BaselineIC, "Purging optimized stubs");
+
+ icScript()->purgeOptimizedStubs(zone);
+ if (hasInliningRoot()) {
+ inliningRoot()->purgeOptimizedStubs(zone);
+ }
+#ifdef DEBUG
+ failedICHash_.reset();
+ hasPurgedStubs_ = true;
+#endif
+}
+
+void ICScript::purgeOptimizedStubs(Zone* zone) {
+ for (size_t i = 0; i < numICEntries(); i++) {
+ ICEntry& entry = icEntry(i);
+ ICStub* lastStub = entry.firstStub();
+ while (!lastStub->isFallback()) {
+ lastStub = lastStub->toCacheIRStub()->next();
+ }
+
+ // Unlink all stubs allocated in the optimized space.
+ ICStub* stub = entry.firstStub();
+ ICCacheIRStub* prev = nullptr;
+
+ while (stub != lastStub) {
+ if (!stub->toCacheIRStub()->allocatedInFallbackSpace()) {
+ lastStub->toFallbackStub()->unlinkStub(zone, &entry, prev,
+ stub->toCacheIRStub());
+ stub = stub->toCacheIRStub()->next();
+ continue;
+ }
+
+ prev = stub->toCacheIRStub();
+ stub = stub->toCacheIRStub()->next();
+ }
+
+ lastStub->toFallbackStub()->clearHasFoldedStub();
+ }
+
+#ifdef DEBUG
+ // All remaining stubs must be allocated in the fallback space.
+ for (size_t i = 0; i < numICEntries(); i++) {
+ ICEntry& entry = icEntry(i);
+ ICStub* stub = entry.firstStub();
+ while (!stub->isFallback()) {
+ MOZ_ASSERT(stub->toCacheIRStub()->allocatedInFallbackSpace());
+ stub = stub->toCacheIRStub()->next();
+ }
+ }
+#endif
+}
+
+JitScript::CachedIonData::CachedIonData(EnvironmentObject* templateEnv,
+ IonBytecodeInfo bytecodeInfo)
+ : templateEnv(templateEnv), bytecodeInfo(bytecodeInfo) {}
+
+bool JitScript::ensureHasCachedIonData(JSContext* cx, HandleScript script) {
+ MOZ_ASSERT(script->jitScript() == this);
+
+ if (hasCachedIonData()) {
+ return true;
+ }
+
+ Rooted<EnvironmentObject*> templateEnv(cx);
+ if (script->function()) {
+ RootedFunction fun(cx, script->function());
+
+ if (fun->needsNamedLambdaEnvironment()) {
+ templateEnv = NamedLambdaObject::createTemplateObject(cx, fun);
+ if (!templateEnv) {
+ return false;
+ }
+ }
+
+ if (fun->needsCallObject()) {
+ templateEnv = CallObject::createTemplateObject(cx, script, templateEnv);
+ if (!templateEnv) {
+ return false;
+ }
+ }
+ }
+
+ IonBytecodeInfo bytecodeInfo = AnalyzeBytecodeForIon(cx, script);
+
+ UniquePtr<CachedIonData> data =
+ cx->make_unique<CachedIonData>(templateEnv, bytecodeInfo);
+ if (!data) {
+ return false;
+ }
+
+ cachedIonData_ = std::move(data);
+ return true;
+}
+
+void JitScript::setBaselineScriptImpl(JSScript* script,
+ BaselineScript* baselineScript) {
+ JSRuntime* rt = script->runtimeFromMainThread();
+ setBaselineScriptImpl(rt->gcContext(), script, baselineScript);
+}
+
+void JitScript::setBaselineScriptImpl(JS::GCContext* gcx, JSScript* script,
+ BaselineScript* baselineScript) {
+ if (hasBaselineScript()) {
+ gcx->removeCellMemory(script, baselineScript_->allocBytes(),
+ MemoryUse::BaselineScript);
+ baselineScript_.set(script->zone(), nullptr);
+ }
+
+ MOZ_ASSERT(ionScript_ == nullptr || ionScript_ == IonDisabledScriptPtr);
+
+ baselineScript_.set(script->zone(), baselineScript);
+ if (hasBaselineScript()) {
+ AddCellMemory(script, baselineScript_->allocBytes(),
+ MemoryUse::BaselineScript);
+ }
+
+ script->resetWarmUpResetCounter();
+ script->updateJitCodeRaw(gcx->runtime());
+}
+
+void JitScript::setIonScriptImpl(JSScript* script, IonScript* ionScript) {
+ JSRuntime* rt = script->runtimeFromMainThread();
+ setIonScriptImpl(rt->gcContext(), script, ionScript);
+}
+
+void JitScript::setIonScriptImpl(JS::GCContext* gcx, JSScript* script,
+ IonScript* ionScript) {
+ MOZ_ASSERT_IF(ionScript != IonDisabledScriptPtr,
+ !baselineScript()->hasPendingIonCompileTask());
+
+ JS::Zone* zone = script->zone();
+ if (hasIonScript()) {
+ gcx->removeCellMemory(script, ionScript_->allocBytes(),
+ MemoryUse::IonScript);
+ ionScript_.set(zone, nullptr);
+ }
+
+ ionScript_.set(zone, ionScript);
+ MOZ_ASSERT_IF(hasIonScript(), hasBaselineScript());
+ if (hasIonScript()) {
+ AddCellMemory(script, ionScript_->allocBytes(), MemoryUse::IonScript);
+ }
+
+ script->updateJitCodeRaw(gcx->runtime());
+}
+
+#ifdef JS_STRUCTURED_SPEW
+static bool HasEnteredCounters(ICEntry& entry) {
+ ICStub* stub = entry.firstStub();
+ if (stub && !stub->isFallback()) {
+ return true;
+ }
+ return false;
+}
+
+void jit::JitSpewBaselineICStats(JSScript* script, const char* dumpReason) {
+ MOZ_ASSERT(script->hasJitScript());
+ JSContext* cx = TlsContext.get();
+ AutoStructuredSpewer spew(cx, SpewChannel::BaselineICStats, script);
+ if (!spew) {
+ return;
+ }
+
+ JitScript* jitScript = script->jitScript();
+ spew->property("reason", dumpReason);
+ spew->beginListProperty("entries");
+ for (size_t i = 0; i < jitScript->numICEntries(); i++) {
+ ICEntry& entry = jitScript->icEntry(i);
+ ICFallbackStub* fallback = jitScript->fallbackStub(i);
+ if (!HasEnteredCounters(entry)) {
+ continue;
+ }
+
+ uint32_t pcOffset = fallback->pcOffset();
+ jsbytecode* pc = script->offsetToPC(pcOffset);
+
+ unsigned column;
+ unsigned int line = PCToLineNumber(script, pc, &column);
+
+ spew->beginObject();
+ spew->property("op", CodeName(JSOp(*pc)));
+ spew->property("pc", pcOffset);
+ spew->property("line", line);
+ spew->property("column", column);
+
+ spew->beginListProperty("counts");
+ ICStub* stub = entry.firstStub();
+ while (stub && !stub->isFallback()) {
+ uint32_t count = stub->enteredCount();
+ spew->value(count);
+ stub = stub->toCacheIRStub()->next();
+ }
+ spew->endList();
+ spew->property("fallback_count", fallback->enteredCount());
+ spew->endObject();
+ }
+ spew->endList();
+}
+#endif
+
+static void MarkActiveJitScripts(JSContext* cx,
+ const JitActivationIterator& activation) {
+ for (OnlyJSJitFrameIter iter(activation); !iter.done(); ++iter) {
+ const JSJitFrameIter& frame = iter.frame();
+ switch (frame.type()) {
+ case FrameType::BaselineJS:
+ frame.script()->jitScript()->setActive();
+ break;
+ case FrameType::Exit:
+ if (frame.exitFrame()->is<LazyLinkExitFrameLayout>()) {
+ LazyLinkExitFrameLayout* ll =
+ frame.exitFrame()->as<LazyLinkExitFrameLayout>();
+ JSScript* script =
+ ScriptFromCalleeToken(ll->jsFrame()->calleeToken());
+ script->jitScript()->setActive();
+ }
+ break;
+ case FrameType::Bailout:
+ case FrameType::IonJS: {
+ // Keep the JitScript and BaselineScript around, since bailouts from
+ // the ion jitcode need to re-enter into the Baseline code.
+ frame.script()->jitScript()->setActive();
+ for (InlineFrameIterator inlineIter(cx, &frame); inlineIter.more();
+ ++inlineIter) {
+ inlineIter.script()->jitScript()->setActive();
+ }
+ break;
+ }
+ default:;
+ }
+ }
+}
+
+void jit::MarkActiveJitScripts(Zone* zone) {
+ if (zone->isAtomsZone()) {
+ return;
+ }
+ JSContext* cx = TlsContext.get();
+ for (JitActivationIterator iter(cx); !iter.done(); ++iter) {
+ if (iter->compartment()->zone() == zone) {
+ MarkActiveJitScripts(cx, iter);
+ }
+ }
+}
+
+InliningRoot* JitScript::getOrCreateInliningRoot(JSContext* cx,
+ JSScript* script) {
+ if (!inliningRoot_) {
+ inliningRoot_ = js::MakeUnique<InliningRoot>(cx, script);
+ if (!inliningRoot_) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+ icScript_.inliningRoot_ = inliningRoot_.get();
+ }
+ return inliningRoot_.get();
+}
+
+gc::AllocSite* JitScript::createAllocSite(JSScript* script) {
+ MOZ_ASSERT(script->jitScript() == this);
+
+ Nursery& nursery = script->runtimeFromMainThread()->gc.nursery();
+ if (!nursery.canCreateAllocSite()) {
+ // Don't block attaching an optimized stub, but don't process allocations
+ // for this site.
+ return script->zone()->unknownAllocSite();
+ }
+
+ if (!allocSites_.reserve(allocSites_.length() + 1)) {
+ return nullptr;
+ }
+
+ ICStubSpace* stubSpace = jitScriptStubSpace();
+ auto* site =
+ static_cast<gc::AllocSite*>(stubSpace->alloc(sizeof(gc::AllocSite)));
+ if (!site) {
+ return nullptr;
+ }
+
+ new (site) gc::AllocSite(script->zone(), script);
+
+ allocSites_.infallibleAppend(site);
+
+ nursery.noteAllocSiteCreated();
+
+ return site;
+}
+
+bool JitScript::resetAllocSites(bool resetNurserySites,
+ bool resetPretenuredSites) {
+ MOZ_ASSERT(resetNurserySites || resetPretenuredSites);
+
+ bool anyReset = false;
+
+ for (gc::AllocSite* site : allocSites_) {
+ if ((resetNurserySites && site->initialHeap() == gc::DefaultHeap) ||
+ (resetPretenuredSites && site->initialHeap() == gc::TenuredHeap)) {
+ if (site->maybeResetState()) {
+ anyReset = true;
+ }
+ }
+ }
+
+ return anyReset;
+}
+
+JitScriptICStubSpace* ICScript::jitScriptStubSpace() {
+ if (isInlined()) {
+ return inliningRoot_->jitScriptStubSpace();
+ }
+ return outerJitScript()->jitScriptStubSpace();
+}
+
+JitScript* ICScript::outerJitScript() {
+ MOZ_ASSERT(!isInlined());
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(this);
+ return reinterpret_cast<JitScript*>(ptr - JitScript::offsetOfICScript());
+}
+
+#ifdef DEBUG
+// This hash is used to verify that we do not recompile after a
+// TranspiledCacheIR invalidation with the exact same ICs.
+//
+// It should change iff an ICEntry in this ICScript (or an ICScript
+// inlined into this ICScript) is modified such that we will make a
+// different decision in WarpScriptOracle::maybeInlineIC. This means:
+//
+// 1. The hash will change if we attach a new stub.
+// 2. The hash will change if the entered count of any CacheIR stub
+// other than the first changes from 0.
+// 3. The hash will change if the entered count of the fallback stub
+// changes from 0.
+//
+HashNumber ICScript::hash() {
+ HashNumber h = 0;
+ for (size_t i = 0; i < numICEntries(); i++) {
+ ICStub* stub = icEntry(i).firstStub();
+
+ // Hash the address of the first stub.
+ h = mozilla::AddToHash(h, stub);
+
+ // Hash whether subsequent stubs have entry count 0.
+ if (!stub->isFallback()) {
+ stub = stub->toCacheIRStub()->next();
+ while (!stub->isFallback()) {
+ h = mozilla::AddToHash(h, stub->enteredCount() == 0);
+ stub = stub->toCacheIRStub()->next();
+ }
+ }
+
+ // Hash whether the fallback has entry count 0.
+ MOZ_ASSERT(stub->isFallback());
+ h = mozilla::AddToHash(h, stub->enteredCount() == 0);
+ }
+
+ if (inlinedChildren_) {
+ for (auto& callsite : *inlinedChildren_) {
+ h = mozilla::AddToHash(h, callsite.callee_->hash());
+ }
+ }
+ return h;
+}
+#endif
diff --git a/js/src/jit/JitScript.h b/js/src/jit/JitScript.h
new file mode 100644
index 0000000000..ecb2c78751
--- /dev/null
+++ b/js/src/jit/JitScript.h
@@ -0,0 +1,576 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitScript_h
+#define jit_JitScript_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/HashFunctions.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/MemoryReporting.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jstypes.h"
+#include "NamespaceImports.h"
+
+#include "gc/Barrier.h"
+#include "jit/BaselineIC.h"
+#include "jit/ICStubSpace.h"
+#include "js/TypeDecls.h"
+#include "js/UniquePtr.h"
+#include "js/Vector.h"
+#include "util/TrailingArray.h"
+#include "vm/EnvironmentObject.h"
+
+class JS_PUBLIC_API JSScript;
+class JS_PUBLIC_API JSTracer;
+struct JS_PUBLIC_API JSContext;
+
+namespace JS {
+class Zone;
+}
+
+namespace js {
+
+class SystemAllocPolicy;
+
+namespace gc {
+class AllocSite;
+}
+
+namespace jit {
+
+class BaselineScript;
+class InliningRoot;
+class IonScript;
+class JitScript;
+class JitZone;
+
+// Information about a script's bytecode, used by WarpBuilder. This is cached
+// in JitScript.
+struct IonBytecodeInfo {
+ bool usesEnvironmentChain = false;
+ bool modifiesArguments = false;
+};
+
+// Magic BaselineScript value indicating Baseline compilation has been disabled.
+static constexpr uintptr_t BaselineDisabledScript = 0x1;
+
+static BaselineScript* const BaselineDisabledScriptPtr =
+ reinterpret_cast<BaselineScript*>(BaselineDisabledScript);
+
+// Magic IonScript values indicating Ion compilation has been disabled or the
+// script is being Ion-compiled off-thread.
+static constexpr uintptr_t IonDisabledScript = 0x1;
+static constexpr uintptr_t IonCompilingScript = 0x2;
+
+static IonScript* const IonDisabledScriptPtr =
+ reinterpret_cast<IonScript*>(IonDisabledScript);
+static IonScript* const IonCompilingScriptPtr =
+ reinterpret_cast<IonScript*>(IonCompilingScript);
+
+/* [SMDOC] ICScript Lifetimes
+ *
+ * An ICScript owns an array of ICEntries, each of which owns a linked
+ * list of ICStubs.
+ *
+ * A JitScript contains an embedded ICScript. If it has done any trial
+ * inlining, it also owns an InliningRoot. The InliningRoot owns all
+ * of the ICScripts that have been created for inlining into the
+ * corresponding JitScript. This ties the lifetime of the inlined
+ * ICScripts to the lifetime of the JitScript itself.
+ *
+ * We store pointers to ICScripts in two other places: on the stack in
+ * BaselineFrame, and in IC stubs for CallInlinedFunction.
+ *
+ * The ICScript pointer in a BaselineFrame either points to the
+ * ICScript embedded in the JitScript for that frame, or to an inlined
+ * ICScript owned by a caller. In each case, there must be a frame on
+ * the stack corresponding to the JitScript that owns the current
+ * ICScript, which will keep the ICScript alive.
+ *
+ * Each ICStub is owned by an ICScript and, indirectly, a
+ * JitScript. An ICStub that uses CallInlinedFunction contains an
+ * ICScript for use by the callee. The ICStub and the callee ICScript
+ * are always owned by the same JitScript, so the callee ICScript will
+ * not be freed while the ICStub is alive.
+ *
+ * The lifetime of an ICScript is independent of the lifetimes of the
+ * BaselineScript and IonScript/WarpScript to which it
+ * corresponds. They can be destroyed and recreated, and the ICScript
+ * will remain valid.
+ */
+
+class alignas(uintptr_t) ICScript final : public TrailingArray {
+ public:
+ ICScript(uint32_t warmUpCount, Offset fallbackStubsOffset, Offset endOffset,
+ uint32_t depth, InliningRoot* inliningRoot = nullptr)
+ : inliningRoot_(inliningRoot),
+ warmUpCount_(warmUpCount),
+ fallbackStubsOffset_(fallbackStubsOffset),
+ endOffset_(endOffset),
+ depth_(depth) {}
+
+ bool isInlined() const { return depth_ > 0; }
+
+ void initICEntries(JSContext* cx, JSScript* script);
+
+ ICEntry& icEntry(size_t index) {
+ MOZ_ASSERT(index < numICEntries());
+ return icEntries()[index];
+ }
+
+ ICFallbackStub* fallbackStub(size_t index) {
+ MOZ_ASSERT(index < numICEntries());
+ return fallbackStubs() + index;
+ }
+
+ ICEntry* icEntryForStub(const ICFallbackStub* stub) {
+ size_t index = stub - fallbackStubs();
+ MOZ_ASSERT(index < numICEntries());
+ return &icEntry(index);
+ }
+ ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
+ size_t index = entry - icEntries();
+ MOZ_ASSERT(index < numICEntries());
+ return fallbackStub(index);
+ }
+
+ InliningRoot* inliningRoot() const { return inliningRoot_; }
+ uint32_t depth() const { return depth_; }
+
+ void resetWarmUpCount(uint32_t count) { warmUpCount_ = count; }
+
+ static constexpr size_t offsetOfFirstStub(uint32_t entryIndex) {
+ return sizeof(ICScript) + entryIndex * sizeof(ICEntry) +
+ ICEntry::offsetOfFirstStub();
+ }
+
+ static constexpr Offset offsetOfWarmUpCount() {
+ return offsetof(ICScript, warmUpCount_);
+ }
+ static constexpr Offset offsetOfDepth() { return offsetof(ICScript, depth_); }
+
+ static constexpr Offset offsetOfICEntries() { return sizeof(ICScript); }
+ uint32_t numICEntries() const {
+ return numElements<ICEntry>(icEntriesOffset(), fallbackStubsOffset());
+ }
+
+ ICEntry* interpreterICEntryFromPCOffset(uint32_t pcOffset);
+
+ ICEntry& icEntryFromPCOffset(uint32_t pcOffset);
+
+ [[nodiscard]] bool addInlinedChild(JSContext* cx,
+ js::UniquePtr<ICScript> child,
+ uint32_t pcOffset);
+ ICScript* findInlinedChild(uint32_t pcOffset);
+ void removeInlinedChild(uint32_t pcOffset);
+ bool hasInlinedChild(uint32_t pcOffset);
+
+ JitScriptICStubSpace* jitScriptStubSpace();
+ void purgeOptimizedStubs(Zone* zone);
+
+ void trace(JSTracer* trc);
+
+#ifdef DEBUG
+ mozilla::HashNumber hash();
+#endif
+
+ private:
+ class CallSite {
+ public:
+ CallSite(ICScript* callee, uint32_t pcOffset)
+ : callee_(callee), pcOffset_(pcOffset) {}
+ ICScript* callee_;
+ uint32_t pcOffset_;
+ };
+
+ // If this ICScript was created for trial inlining or has another
+ // ICScript inlined into it, a pointer to the root of the inlining
+ // tree. Otherwise, nullptr.
+ InliningRoot* inliningRoot_ = nullptr;
+
+ // ICScripts that have been inlined into this ICScript.
+ js::UniquePtr<Vector<CallSite>> inlinedChildren_;
+
+ // Number of times this copy of the script has been called or has had
+ // backedges taken. Reset if the script's JIT code is forcibly discarded.
+ // See also the ScriptWarmUpData class.
+ mozilla::Atomic<uint32_t, mozilla::Relaxed> warmUpCount_ = {};
+
+ // The offset of the ICFallbackStub array.
+ Offset fallbackStubsOffset_;
+
+ // The size of this allocation.
+ Offset endOffset_;
+
+ // The inlining depth of this ICScript. 0 for the inlining root.
+ uint32_t depth_;
+
+ Offset icEntriesOffset() const { return offsetOfICEntries(); }
+ Offset fallbackStubsOffset() const { return fallbackStubsOffset_; }
+ Offset endOffset() const { return endOffset_; }
+
+ ICEntry* icEntries() { return offsetToPointer<ICEntry>(icEntriesOffset()); }
+
+ ICFallbackStub* fallbackStubs() {
+ return offsetToPointer<ICFallbackStub>(fallbackStubsOffset());
+ }
+
+ JitScript* outerJitScript();
+
+ friend class JitScript;
+};
+
+// [SMDOC] JitScript
+//
+// JitScript stores type inference data, Baseline ICs and other JIT-related data
+// for a script. Scripts with a JitScript can run in the Baseline Interpreter.
+//
+// IC Data
+// =======
+// All IC data for Baseline (Interpreter and JIT) is stored in an ICScript. Each
+// JitScript contains an ICScript as the last field. Additional free-standing
+// ICScripts may be created during trial inlining. Ion has its own IC chains
+// stored in IonScript.
+//
+// For each IC we store an ICEntry, which points to the first ICStub in the
+// chain, and an ICFallbackStub. Note that multiple stubs in the same zone can
+// share Baseline IC code. This works because the stub data is stored in the
+// ICStub instead of baked in in the stub code.
+//
+// Storing this separate from BaselineScript allows us to use the same ICs in
+// the Baseline Interpreter and Baseline JIT. It also simplifies debug mode OSR
+// because the JitScript can be reused when we have to recompile the
+// BaselineScript.
+//
+// The JitScript contains a stub space. This stores the "can GC" CacheIR stubs.
+// These stubs are never purged before destroying the JitScript. Other stubs are
+// stored in the optimized stub space stored in JitZone and can be purged more
+// eagerly. See JitScript::purgeOptimizedStubs.
+//
+// An ICScript contains a list of IC entries and a list of fallback stubs.
+// There's one ICEntry and ICFallbackStub for each JOF_IC bytecode op.
+//
+// The ICScript also contains the warmUpCount for the script.
+//
+// Inlining Data
+// =============
+// JitScript also contains a list of Warp compilations inlining this script, for
+// invalidation.
+//
+// Memory Layout
+// =============
+// JitScript contains an ICScript as the last field. ICScript has trailing
+// (variable length) arrays for ICEntry and ICFallbackStub. The memory layout is
+// as follows:
+//
+// Item | Offset
+// ------------------------+------------------------
+// JitScript | 0
+// -->ICScript (field) |
+// ICEntry[] | icEntriesOffset()
+// ICFallbackStub[] | fallbackStubsOffset()
+//
+// These offsets are also used to compute numICEntries.
+class alignas(uintptr_t) JitScript final : public TrailingArray {
+ friend class ::JSScript;
+
+ // Allocated space for Can-GC CacheIR stubs.
+ JitScriptICStubSpace jitScriptStubSpace_ = {};
+
+ // Profile string used by the profiler for Baseline Interpreter frames.
+ const char* profileString_ = nullptr;
+
+ // Data allocated lazily the first time this script is compiled, inlined, or
+ // analyzed by WarpBuilder. This is done lazily to improve performance and
+ // memory usage as most scripts are never Warp-compiled.
+ struct CachedIonData {
+ // For functions with a call object, template objects to use for the call
+ // object and decl env object (linked via the call object's enclosing
+ // scope).
+ const HeapPtr<EnvironmentObject*> templateEnv = nullptr;
+
+ // Analysis information based on the script and its bytecode.
+ IonBytecodeInfo bytecodeInfo = {};
+
+ CachedIonData(EnvironmentObject* templateEnv, IonBytecodeInfo bytecodeInfo);
+
+ CachedIonData(const CachedIonData&) = delete;
+ void operator=(const CachedIonData&) = delete;
+
+ void trace(JSTracer* trc);
+ };
+ js::UniquePtr<CachedIonData> cachedIonData_;
+
+ // Baseline code for the script. Either nullptr, BaselineDisabledScriptPtr or
+ // a valid BaselineScript*.
+ GCStructPtr<BaselineScript*> baselineScript_;
+
+ // Ion code for this script. Either nullptr, IonDisabledScriptPtr,
+ // IonCompilingScriptPtr or a valid IonScript*.
+ GCStructPtr<IonScript*> ionScript_;
+
+ // The size of this allocation.
+ Offset endOffset_ = 0;
+
+ struct Flags {
+ // Flag set when discarding JIT code to indicate this script is on the stack
+ // and type information and JIT code should not be discarded.
+ bool active : 1;
+
+ // True if this script entered Ion via OSR at a loop header.
+ bool hadIonOSR : 1;
+ };
+ Flags flags_ = {}; // Zero-initialize flags.
+
+ js::UniquePtr<InliningRoot> inliningRoot_;
+
+#ifdef DEBUG
+ // If the last warp compilation invalidated because of TranspiledCacheIR
+ // bailouts, this is a hash of the ICScripts used in that compilation.
+ // When recompiling, we assert that the hash has changed.
+ mozilla::Maybe<mozilla::HashNumber> failedICHash_;
+
+ // To avoid pathological cases, we skip the check if we have purged
+ // stubs due to GC pressure.
+ bool hasPurgedStubs_ = false;
+#endif
+
+ // List of allocation sites referred to by ICs in this script.
+ Vector<gc::AllocSite*, 0, SystemAllocPolicy> allocSites_;
+
+ ICScript icScript_;
+ // End of fields.
+
+ Offset endOffset() const { return endOffset_; }
+
+ bool hasCachedIonData() const { return !!cachedIonData_; }
+
+ CachedIonData& cachedIonData() {
+ MOZ_ASSERT(hasCachedIonData());
+ return *cachedIonData_.get();
+ }
+ const CachedIonData& cachedIonData() const {
+ MOZ_ASSERT(hasCachedIonData());
+ return *cachedIonData_.get();
+ }
+
+ public:
+ JitScript(JSScript* script, Offset fallbackStubsOffset, Offset endOffset,
+ const char* profileString);
+
+ ~JitScript();
+
+ [[nodiscard]] bool ensureHasCachedIonData(JSContext* cx, HandleScript script);
+
+ void setHadIonOSR() { flags_.hadIonOSR = true; }
+ bool hadIonOSR() const { return flags_.hadIonOSR; }
+
+ uint32_t numICEntries() const { return icScript_.numICEntries(); }
+
+ bool active() const { return flags_.active; }
+ void setActive() { flags_.active = true; }
+ void resetActive() { flags_.active = false; }
+
+ void ensureProfileString(JSContext* cx, JSScript* script);
+
+ const char* profileString() const {
+ MOZ_ASSERT(profileString_);
+ return profileString_;
+ }
+
+ static void Destroy(Zone* zone, JitScript* script);
+
+ static constexpr Offset offsetOfICEntries() { return sizeof(JitScript); }
+
+ static constexpr size_t offsetOfBaselineScript() {
+ return offsetof(JitScript, baselineScript_);
+ }
+ static constexpr size_t offsetOfIonScript() {
+ return offsetof(JitScript, ionScript_);
+ }
+ static constexpr size_t offsetOfICScript() {
+ return offsetof(JitScript, icScript_);
+ }
+ static constexpr size_t offsetOfWarmUpCount() {
+ return offsetOfICScript() + ICScript::offsetOfWarmUpCount();
+ }
+
+ uint32_t warmUpCount() const { return icScript_.warmUpCount_; }
+ void incWarmUpCount(uint32_t amount) { icScript_.warmUpCount_ += amount; }
+ void resetWarmUpCount(uint32_t count);
+
+ void prepareForDestruction(Zone* zone);
+
+ JitScriptICStubSpace* jitScriptStubSpace() { return &jitScriptStubSpace_; }
+
+ void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, size_t* data,
+ size_t* fallbackStubs) const {
+ *data += mallocSizeOf(this);
+
+ // |data| already includes the ICStubSpace itself, so use
+ // sizeOfExcludingThis.
+ *fallbackStubs += jitScriptStubSpace_.sizeOfExcludingThis(mallocSizeOf);
+ }
+
+ ICEntry& icEntry(size_t index) { return icScript_.icEntry(index); }
+
+ ICFallbackStub* fallbackStub(size_t index) {
+ return icScript_.fallbackStub(index);
+ }
+
+ ICEntry* icEntryForStub(const ICFallbackStub* stub) {
+ return icScript_.icEntryForStub(stub);
+ }
+ ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
+ return icScript_.fallbackStubForICEntry(entry);
+ }
+
+ void trace(JSTracer* trc);
+ void purgeOptimizedStubs(JSScript* script);
+
+ ICEntry& icEntryFromPCOffset(uint32_t pcOffset) {
+ return icScript_.icEntryFromPCOffset(pcOffset);
+ };
+
+ size_t allocBytes() const { return endOffset(); }
+
+ EnvironmentObject* templateEnvironment() const {
+ return cachedIonData().templateEnv;
+ }
+
+ bool modifiesArguments() const {
+ return cachedIonData().bytecodeInfo.modifiesArguments;
+ }
+ bool usesEnvironmentChain() const {
+ return cachedIonData().bytecodeInfo.usesEnvironmentChain;
+ }
+
+ gc::AllocSite* createAllocSite(JSScript* script);
+
+ bool resetAllocSites(bool resetNurserySites, bool resetPretenuredSites);
+
+ private:
+ // Methods to set baselineScript_ to a BaselineScript*, nullptr, or
+ // BaselineDisabledScriptPtr.
+ void setBaselineScriptImpl(JSScript* script, BaselineScript* baselineScript);
+ void setBaselineScriptImpl(JS::GCContext* gcx, JSScript* script,
+ BaselineScript* baselineScript);
+
+ public:
+ // Methods for getting/setting/clearing a BaselineScript*.
+ bool hasBaselineScript() const {
+ bool res = baselineScript_ && baselineScript_ != BaselineDisabledScriptPtr;
+ MOZ_ASSERT_IF(!res, !hasIonScript());
+ return res;
+ }
+ BaselineScript* baselineScript() const {
+ MOZ_ASSERT(hasBaselineScript());
+ return baselineScript_;
+ }
+ void setBaselineScript(JSScript* script, BaselineScript* baselineScript) {
+ MOZ_ASSERT(!hasBaselineScript());
+ setBaselineScriptImpl(script, baselineScript);
+ MOZ_ASSERT(hasBaselineScript());
+ }
+ [[nodiscard]] BaselineScript* clearBaselineScript(JS::GCContext* gcx,
+ JSScript* script) {
+ BaselineScript* baseline = baselineScript();
+ setBaselineScriptImpl(gcx, script, nullptr);
+ return baseline;
+ }
+
+ private:
+ // Methods to set ionScript_ to an IonScript*, nullptr, or one of the special
+ // Ion{Disabled,Compiling}ScriptPtr values.
+ void setIonScriptImpl(JS::GCContext* gcx, JSScript* script,
+ IonScript* ionScript);
+ void setIonScriptImpl(JSScript* script, IonScript* ionScript);
+
+ public:
+ // Methods for getting/setting/clearing an IonScript*.
+ bool hasIonScript() const {
+ bool res = ionScript_ && ionScript_ != IonDisabledScriptPtr &&
+ ionScript_ != IonCompilingScriptPtr;
+ MOZ_ASSERT_IF(res, baselineScript_);
+ return res;
+ }
+ IonScript* ionScript() const {
+ MOZ_ASSERT(hasIonScript());
+ return ionScript_;
+ }
+ void setIonScript(JSScript* script, IonScript* ionScript) {
+ MOZ_ASSERT(!hasIonScript());
+ setIonScriptImpl(script, ionScript);
+ MOZ_ASSERT(hasIonScript());
+ }
+ [[nodiscard]] IonScript* clearIonScript(JS::GCContext* gcx,
+ JSScript* script) {
+ IonScript* ion = ionScript();
+ setIonScriptImpl(gcx, script, nullptr);
+ return ion;
+ }
+
+ // Methods for off-thread compilation.
+ bool isIonCompilingOffThread() const {
+ return ionScript_ == IonCompilingScriptPtr;
+ }
+ void setIsIonCompilingOffThread(JSScript* script) {
+ MOZ_ASSERT(ionScript_ == nullptr);
+ setIonScriptImpl(script, IonCompilingScriptPtr);
+ }
+ void clearIsIonCompilingOffThread(JSScript* script) {
+ MOZ_ASSERT(isIonCompilingOffThread());
+ setIonScriptImpl(script, nullptr);
+ }
+ ICScript* icScript() { return &icScript_; }
+
+ bool hasInliningRoot() const { return !!inliningRoot_; }
+ InliningRoot* inliningRoot() const { return inliningRoot_.get(); }
+ InliningRoot* getOrCreateInliningRoot(JSContext* cx, JSScript* script);
+
+#ifdef DEBUG
+ bool hasFailedICHash() const { return failedICHash_.isSome(); }
+ mozilla::HashNumber getFailedICHash() { return failedICHash_.extract(); }
+ void setFailedICHash(mozilla::HashNumber hash) {
+ MOZ_ASSERT(failedICHash_.isNothing());
+ if (!hasPurgedStubs_) {
+ failedICHash_.emplace(hash);
+ }
+ }
+#endif
+};
+
+// Ensures no JitScripts are purged in the current zone.
+class MOZ_RAII AutoKeepJitScripts {
+ jit::JitZone* zone_;
+ bool prev_;
+
+ AutoKeepJitScripts(const AutoKeepJitScripts&) = delete;
+ void operator=(const AutoKeepJitScripts&) = delete;
+
+ public:
+ explicit inline AutoKeepJitScripts(JSContext* cx);
+ inline ~AutoKeepJitScripts();
+};
+
+// Mark JitScripts on the stack as active, so that they are not discarded
+// during GC.
+void MarkActiveJitScripts(Zone* zone);
+
+#ifdef JS_STRUCTURED_SPEW
+void JitSpewBaselineICStats(JSScript* script, const char* dumpReason);
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitScript_h */
diff --git a/js/src/jit/JitSpewer.cpp b/js/src/jit/JitSpewer.cpp
new file mode 100644
index 0000000000..d85fbcf3d3
--- /dev/null
+++ b/js/src/jit/JitSpewer.cpp
@@ -0,0 +1,657 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifdef JS_JITSPEW
+
+# include "jit/JitSpewer.h"
+
+# include "mozilla/Atomics.h"
+# include "mozilla/Sprintf.h"
+
+# include "jit/MIR.h"
+# include "jit/MIRGenerator.h"
+# include "jit/MIRGraph.h"
+# include "threading/LockGuard.h"
+# include "util/GetPidProvider.h" // getpid()
+# include "vm/MutexIDs.h"
+
+# ifndef JIT_SPEW_DIR
+# if defined(_WIN32)
+# define JIT_SPEW_DIR "."
+# elif defined(__ANDROID__)
+# define JIT_SPEW_DIR "/data/local/tmp"
+# else
+# define JIT_SPEW_DIR "/tmp"
+# endif
+# endif
+
+using namespace js;
+using namespace js::jit;
+
+class IonSpewer {
+ private:
+ Mutex outputLock_ MOZ_UNANNOTATED;
+ Fprinter jsonOutput_;
+ bool firstFunction_;
+ bool asyncLogging_;
+ bool inited_;
+
+ void release();
+
+ public:
+ IonSpewer()
+ : outputLock_(mutexid::IonSpewer),
+ firstFunction_(false),
+ asyncLogging_(false),
+ inited_(false) {}
+
+ // File output is terminated safely upon destruction.
+ ~IonSpewer();
+
+ bool init();
+ bool isEnabled() { return inited_; }
+ void setAsyncLogging(bool incremental) { asyncLogging_ = incremental; }
+ bool getAsyncLogging() { return asyncLogging_; }
+
+ void beginFunction();
+ void spewPass(GraphSpewer* gs);
+ void endFunction(GraphSpewer* gs);
+};
+
+// IonSpewer singleton.
+static IonSpewer ionspewer;
+
+static bool LoggingChecked = false;
+static_assert(JitSpew_Terminator <= 64,
+ "Increase the size of the LoggingBits global.");
+static uint64_t LoggingBits = 0;
+static mozilla::Atomic<uint32_t, mozilla::Relaxed> filteredOutCompilations(0);
+
+static const char* const ChannelNames[] = {
+# define JITSPEW_CHANNEL(name) # name,
+ JITSPEW_CHANNEL_LIST(JITSPEW_CHANNEL)
+# undef JITSPEW_CHANNEL
+};
+
+static size_t ChannelIndentLevel[] = {
+# define JITSPEW_CHANNEL(name) 0,
+ JITSPEW_CHANNEL_LIST(JITSPEW_CHANNEL)
+# undef JITSPEW_CHANNEL
+};
+
+// The IONFILTER environment variable specifies an expression to select only
+// certain functions for spewing to reduce amount of log data generated.
+static const char* gSpewFilter = nullptr;
+
+static bool FilterContainsLocation(JSScript* function) {
+ // If there is no filter we accept all outputs.
+ if (!gSpewFilter || !gSpewFilter[0]) {
+ return true;
+ }
+
+ // Disable wasm output when filter is set.
+ if (!function) {
+ return false;
+ }
+
+ const char* filename = function->filename();
+ const size_t line = function->lineno();
+ const size_t filelen = strlen(filename);
+ const char* index = strstr(gSpewFilter, filename);
+ while (index) {
+ if (index == gSpewFilter || index[-1] == ',') {
+ if (index[filelen] == 0 || index[filelen] == ',') {
+ return true;
+ }
+ if (index[filelen] == ':' && line != size_t(-1)) {
+ size_t read_line = strtoul(&index[filelen + 1], nullptr, 10);
+ if (read_line == line) {
+ return true;
+ }
+ }
+ }
+ index = strstr(index + filelen, filename);
+ }
+ return false;
+}
+
+void jit::EnableIonDebugSyncLogging() {
+ ionspewer.init();
+ ionspewer.setAsyncLogging(false);
+ EnableChannel(JitSpew_IonSyncLogs);
+}
+
+void jit::EnableIonDebugAsyncLogging() {
+ ionspewer.init();
+ ionspewer.setAsyncLogging(true);
+}
+
+void IonSpewer::release() {
+ if (jsonOutput_.isInitialized()) {
+ jsonOutput_.finish();
+ }
+ inited_ = false;
+}
+
+bool IonSpewer::init() {
+ if (inited_) {
+ return true;
+ }
+
+ // Filter expression for spewing
+ gSpewFilter = getenv("IONFILTER");
+
+ const size_t bufferLength = 256;
+ char jsonBuffer[bufferLength];
+ const char* jsonFilename = JIT_SPEW_DIR "/ion.json";
+
+ const char* usePid = getenv("ION_SPEW_BY_PID");
+ if (usePid && *usePid != 0) {
+ uint32_t pid = getpid();
+ size_t len;
+ len = SprintfLiteral(jsonBuffer, JIT_SPEW_DIR "/ion%" PRIu32 ".json", pid);
+ if (bufferLength <= len) {
+ fprintf(stderr, "Warning: IonSpewer::init: Cannot serialize file name.");
+ return false;
+ }
+ jsonFilename = jsonBuffer;
+ }
+
+ if (!jsonOutput_.init(jsonFilename)) {
+ release();
+ return false;
+ }
+
+ jsonOutput_.printf("{\n \"functions\": [\n");
+ firstFunction_ = true;
+
+ inited_ = true;
+ return true;
+}
+
+void IonSpewer::beginFunction() {
+ // If we are doing a synchronous logging then we spew everything as we go,
+ // as this is useful in case of failure during the compilation. On the other
+ // hand, it is recommended to disable off thread compilation.
+ if (!getAsyncLogging() && !firstFunction_) {
+ LockGuard<Mutex> guard(outputLock_);
+ jsonOutput_.put(","); // separate functions
+ }
+}
+
+void IonSpewer::spewPass(GraphSpewer* gs) {
+ if (!getAsyncLogging()) {
+ LockGuard<Mutex> guard(outputLock_);
+ gs->dump(jsonOutput_);
+ }
+}
+
+void IonSpewer::endFunction(GraphSpewer* gs) {
+ LockGuard<Mutex> guard(outputLock_);
+ if (getAsyncLogging() && !firstFunction_) {
+ jsonOutput_.put(","); // separate functions
+ }
+
+ gs->dump(jsonOutput_);
+ firstFunction_ = false;
+}
+
+IonSpewer::~IonSpewer() {
+ if (!inited_) {
+ return;
+ }
+
+ jsonOutput_.printf("\n]}\n");
+ release();
+}
+
+GraphSpewer::GraphSpewer(TempAllocator* alloc)
+ : graph_(nullptr),
+ jsonPrinter_(alloc->lifoAlloc()),
+ jsonSpewer_(jsonPrinter_) {}
+
+void GraphSpewer::init(MIRGraph* graph, JSScript* function) {
+ MOZ_ASSERT(!isSpewing());
+ if (!ionspewer.isEnabled()) {
+ return;
+ }
+
+ if (!FilterContainsLocation(function)) {
+ // filter out logs during the compilation.
+ filteredOutCompilations++;
+ MOZ_ASSERT(!isSpewing());
+ return;
+ }
+
+ graph_ = graph;
+ MOZ_ASSERT(isSpewing());
+}
+
+void GraphSpewer::beginFunction(JSScript* function) {
+ if (!isSpewing()) {
+ return;
+ }
+ jsonSpewer_.beginFunction(function);
+ ionspewer.beginFunction();
+}
+
+void GraphSpewer::beginWasmFunction(unsigned funcIndex) {
+ if (!isSpewing()) {
+ return;
+ }
+ jsonSpewer_.beginWasmFunction(funcIndex);
+ ionspewer.beginFunction();
+}
+
+void GraphSpewer::spewPass(const char* pass) {
+ if (!isSpewing()) {
+ return;
+ }
+
+ jsonSpewer_.beginPass(pass);
+ jsonSpewer_.spewMIR(graph_);
+ jsonSpewer_.spewLIR(graph_);
+ jsonSpewer_.endPass();
+
+ ionspewer.spewPass(this);
+
+ // As this function is used for debugging, we ignore any of the previous
+ // failures and ensure there is enough ballast space, such that we do not
+ // exhaust the ballast space before running the next phase.
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!graph_->alloc().ensureBallast()) {
+ oomUnsafe.crash(
+ "Could not ensure enough ballast space after spewing graph "
+ "information.");
+ }
+}
+
+void GraphSpewer::spewPass(const char* pass, BacktrackingAllocator* ra) {
+ if (!isSpewing()) {
+ return;
+ }
+
+ jsonSpewer_.beginPass(pass);
+ jsonSpewer_.spewMIR(graph_);
+ jsonSpewer_.spewLIR(graph_);
+ jsonSpewer_.spewRanges(ra);
+ jsonSpewer_.endPass();
+
+ ionspewer.spewPass(this);
+}
+
+void GraphSpewer::endFunction() {
+ if (!ionspewer.isEnabled()) {
+ return;
+ }
+
+ if (!isSpewing()) {
+ MOZ_ASSERT(filteredOutCompilations != 0);
+ filteredOutCompilations--;
+ return;
+ }
+
+ jsonSpewer_.endFunction();
+
+ ionspewer.endFunction(this);
+ graph_ = nullptr;
+}
+
+void GraphSpewer::dump(Fprinter& jsonOut) {
+ if (!jsonPrinter_.hadOutOfMemory()) {
+ jsonPrinter_.exportInto(jsonOut);
+ } else {
+ jsonOut.put("{}");
+ }
+ jsonOut.flush();
+ jsonPrinter_.clear();
+}
+
+void jit::SpewBeginFunction(MIRGenerator* mir, JSScript* function) {
+ MIRGraph* graph = &mir->graph();
+ mir->graphSpewer().init(graph, function);
+ mir->graphSpewer().beginFunction(function);
+}
+
+void jit::SpewBeginWasmFunction(MIRGenerator* mir, unsigned funcIndex) {
+ MIRGraph* graph = &mir->graph();
+ mir->graphSpewer().init(graph, nullptr);
+ mir->graphSpewer().beginWasmFunction(funcIndex);
+}
+
+AutoSpewEndFunction::~AutoSpewEndFunction() {
+ mir_->graphSpewer().endFunction();
+}
+
+Fprinter& jit::JitSpewPrinter() {
+ static Fprinter out;
+ return out;
+}
+
+static void PrintHelpAndExit(int status = 0) {
+ fflush(nullptr);
+ printf(
+ "\n"
+ "usage: IONFLAGS=option,option,option,... where options can be:\n"
+ "\n"
+ " aborts Compilation abort messages\n"
+ " scripts Compiled scripts\n"
+ " mir MIR information\n"
+ " prune Prune unused branches\n"
+ " escape Escape analysis\n"
+ " alias Alias analysis\n"
+ " alias-sum Alias analysis: shows summaries for every block\n"
+ " gvn Global Value Numbering\n"
+ " licm Loop invariant code motion\n"
+ " flac Fold linear arithmetic constants\n"
+ " eaa Effective address analysis\n"
+ " sink Sink transformation\n"
+ " regalloc Register allocation\n"
+ " inline Inlining\n"
+ " snapshots Snapshot information\n"
+ " codegen Native code generation\n"
+ " bailouts Bailouts\n"
+ " caches Inline caches\n"
+ " osi Invalidation\n"
+ " safepoints Safepoints\n"
+ " pools Literal Pools (ARM only for now)\n"
+ " cacheflush Instruction Cache flushes (ARM only for now)\n"
+ " range Range Analysis\n"
+ " wasmbce Wasm Bounds Check Elimination\n"
+ " shapeguards Redundant shape guard elimination\n"
+ " logs JSON visualization logging\n"
+ " logs-sync Same as logs, but flushes between each pass (sync. "
+ "compiled functions only).\n"
+ " profiling Profiling-related information\n"
+ " dump-mir-expr Dump the MIR expressions\n"
+ " scriptstats Tracelogger summary stats\n"
+ " warp-snapshots WarpSnapshots created by WarpOracle\n"
+ " warp-transpiler Warp CacheIR transpiler\n"
+ " warp-trial-inlining Trial inlining for Warp\n"
+ " all Everything\n"
+ "\n"
+ " bl-aborts Baseline compiler abort messages\n"
+ " bl-scripts Baseline script-compilation\n"
+ " bl-op Baseline compiler detailed op-specific messages\n"
+ " bl-ic Baseline inline-cache messages\n"
+ " bl-ic-fb Baseline IC fallback stub messages\n"
+ " bl-osr Baseline IC OSR messages\n"
+ " bl-bails Baseline bailouts\n"
+ " bl-dbg-osr Baseline debug mode on stack recompile messages\n"
+ " bl-all All baseline spew\n"
+ "\n"
+ "See also SPEW=help for information on the Structured Spewer."
+ "\n");
+ exit(status);
+}
+
+static bool IsFlag(const char* found, const char* flag) {
+ return strlen(found) == strlen(flag) && strcmp(found, flag) == 0;
+}
+
+void jit::CheckLogging() {
+ if (LoggingChecked) {
+ return;
+ }
+
+ LoggingChecked = true;
+
+ char* env = getenv("IONFLAGS");
+ if (!env) {
+ return;
+ }
+
+ const char* found = strtok(env, ",");
+ while (found) {
+ fprintf(stderr, "found tag: %s\n", found);
+ // We're at the end of a flag; check if the previous substring was a
+ // known flag (i-1 is the last character of the flag we just read).
+ if (IsFlag(found, "help")) {
+ PrintHelpAndExit();
+ } else if (IsFlag(found, "aborts")) {
+ EnableChannel(JitSpew_IonAbort);
+ } else if (IsFlag(found, "prune")) {
+ EnableChannel(JitSpew_Prune);
+ } else if (IsFlag(found, "escape")) {
+ EnableChannel(JitSpew_Escape);
+ } else if (IsFlag(found, "alias")) {
+ EnableChannel(JitSpew_Alias);
+ } else if (IsFlag(found, "alias-sum")) {
+ EnableChannel(JitSpew_AliasSummaries);
+ } else if (IsFlag(found, "scripts")) {
+ EnableChannel(JitSpew_IonScripts);
+ } else if (IsFlag(found, "mir")) {
+ EnableChannel(JitSpew_IonMIR);
+ } else if (IsFlag(found, "gvn")) {
+ EnableChannel(JitSpew_GVN);
+ } else if (IsFlag(found, "range")) {
+ EnableChannel(JitSpew_Range);
+ } else if (IsFlag(found, "wasmbce")) {
+ EnableChannel(JitSpew_WasmBCE);
+ } else if (IsFlag(found, "licm")) {
+ EnableChannel(JitSpew_LICM);
+ } else if (IsFlag(found, "flac")) {
+ EnableChannel(JitSpew_FLAC);
+ } else if (IsFlag(found, "eaa")) {
+ EnableChannel(JitSpew_EAA);
+ } else if (IsFlag(found, "sink")) {
+ EnableChannel(JitSpew_Sink);
+ } else if (IsFlag(found, "regalloc")) {
+ EnableChannel(JitSpew_RegAlloc);
+ } else if (IsFlag(found, "inline")) {
+ EnableChannel(JitSpew_Inlining);
+ } else if (IsFlag(found, "snapshots")) {
+ EnableChannel(JitSpew_IonSnapshots);
+ } else if (IsFlag(found, "codegen")) {
+ EnableChannel(JitSpew_Codegen);
+ } else if (IsFlag(found, "bailouts")) {
+ EnableChannel(JitSpew_IonBailouts);
+ } else if (IsFlag(found, "osi")) {
+ EnableChannel(JitSpew_IonInvalidate);
+ } else if (IsFlag(found, "caches")) {
+ EnableChannel(JitSpew_IonIC);
+ } else if (IsFlag(found, "safepoints")) {
+ EnableChannel(JitSpew_Safepoints);
+ } else if (IsFlag(found, "pools")) {
+ EnableChannel(JitSpew_Pools);
+ } else if (IsFlag(found, "cacheflush")) {
+ EnableChannel(JitSpew_CacheFlush);
+ } else if (IsFlag(found, "shapeguards")) {
+ EnableChannel(JitSpew_RedundantShapeGuards);
+ } else if (IsFlag(found, "logs")) {
+ EnableIonDebugAsyncLogging();
+ } else if (IsFlag(found, "logs-sync")) {
+ EnableIonDebugSyncLogging();
+ } else if (IsFlag(found, "profiling")) {
+ EnableChannel(JitSpew_Profiling);
+ } else if (IsFlag(found, "dump-mir-expr")) {
+ EnableChannel(JitSpew_MIRExpressions);
+ } else if (IsFlag(found, "scriptstats")) {
+ EnableChannel(JitSpew_ScriptStats);
+ } else if (IsFlag(found, "warp-snapshots")) {
+ EnableChannel(JitSpew_WarpSnapshots);
+ } else if (IsFlag(found, "warp-transpiler")) {
+ EnableChannel(JitSpew_WarpTranspiler);
+ } else if (IsFlag(found, "warp-trial-inlining")) {
+ EnableChannel(JitSpew_WarpTrialInlining);
+ } else if (IsFlag(found, "all")) {
+ LoggingBits = uint64_t(-1);
+ } else if (IsFlag(found, "bl-aborts")) {
+ EnableChannel(JitSpew_BaselineAbort);
+ } else if (IsFlag(found, "bl-scripts")) {
+ EnableChannel(JitSpew_BaselineScripts);
+ } else if (IsFlag(found, "bl-op")) {
+ EnableChannel(JitSpew_BaselineOp);
+ } else if (IsFlag(found, "bl-ic")) {
+ EnableChannel(JitSpew_BaselineIC);
+ } else if (IsFlag(found, "bl-ic-fb")) {
+ EnableChannel(JitSpew_BaselineICFallback);
+ } else if (IsFlag(found, "bl-osr")) {
+ EnableChannel(JitSpew_BaselineOSR);
+ } else if (IsFlag(found, "bl-bails")) {
+ EnableChannel(JitSpew_BaselineBailouts);
+ } else if (IsFlag(found, "bl-dbg-osr")) {
+ EnableChannel(JitSpew_BaselineDebugModeOSR);
+ } else if (IsFlag(found, "bl-all")) {
+ EnableChannel(JitSpew_BaselineAbort);
+ EnableChannel(JitSpew_BaselineScripts);
+ EnableChannel(JitSpew_BaselineOp);
+ EnableChannel(JitSpew_BaselineIC);
+ EnableChannel(JitSpew_BaselineICFallback);
+ EnableChannel(JitSpew_BaselineOSR);
+ EnableChannel(JitSpew_BaselineBailouts);
+ EnableChannel(JitSpew_BaselineDebugModeOSR);
+ } else {
+ fprintf(stderr, "Unknown flag.\n");
+ PrintHelpAndExit(64);
+ }
+ found = strtok(nullptr, ",");
+ }
+
+ FILE* spewfh = stderr;
+ const char* filename = getenv("ION_SPEW_FILENAME");
+ if (filename && *filename) {
+ char actual_filename[2048] = {0};
+ SprintfLiteral(actual_filename, "%s.%d", filename, getpid());
+ spewfh = fopen(actual_filename, "w");
+ MOZ_RELEASE_ASSERT(spewfh);
+ setbuf(spewfh, nullptr); // Make unbuffered
+ }
+ JitSpewPrinter().init(spewfh);
+}
+
+JitSpewIndent::JitSpewIndent(JitSpewChannel channel) : channel_(channel) {
+ ChannelIndentLevel[channel]++;
+}
+
+JitSpewIndent::~JitSpewIndent() { ChannelIndentLevel[channel_]--; }
+
+void jit::JitSpewStartVA(JitSpewChannel channel, const char* fmt, va_list ap) {
+ if (!JitSpewEnabled(channel)) {
+ return;
+ }
+
+ JitSpewHeader(channel);
+ Fprinter& out = JitSpewPrinter();
+ out.vprintf(fmt, ap);
+}
+
+void jit::JitSpewContVA(JitSpewChannel channel, const char* fmt, va_list ap) {
+ if (!JitSpewEnabled(channel)) {
+ return;
+ }
+
+ Fprinter& out = JitSpewPrinter();
+ out.vprintf(fmt, ap);
+}
+
+void jit::JitSpewFin(JitSpewChannel channel) {
+ if (!JitSpewEnabled(channel)) {
+ return;
+ }
+
+ Fprinter& out = JitSpewPrinter();
+ out.put("\n");
+}
+
+void jit::JitSpewVA(JitSpewChannel channel, const char* fmt, va_list ap) {
+ JitSpewStartVA(channel, fmt, ap);
+ JitSpewFin(channel);
+}
+
+void jit::JitSpew(JitSpewChannel channel, const char* fmt, ...) {
+ va_list ap;
+ va_start(ap, fmt);
+ JitSpewVA(channel, fmt, ap);
+ va_end(ap);
+}
+
+void jit::JitSpewDef(JitSpewChannel channel, const char* str,
+ MDefinition* def) {
+ if (!JitSpewEnabled(channel)) {
+ return;
+ }
+
+ JitSpewHeader(channel);
+ Fprinter& out = JitSpewPrinter();
+ out.put(str);
+ def->dump(out);
+ def->dumpLocation(out);
+}
+
+void jit::JitSpewStart(JitSpewChannel channel, const char* fmt, ...) {
+ va_list ap;
+ va_start(ap, fmt);
+ JitSpewStartVA(channel, fmt, ap);
+ va_end(ap);
+}
+void jit::JitSpewCont(JitSpewChannel channel, const char* fmt, ...) {
+ va_list ap;
+ va_start(ap, fmt);
+ JitSpewContVA(channel, fmt, ap);
+ va_end(ap);
+}
+
+void jit::JitSpewHeader(JitSpewChannel channel) {
+ if (!JitSpewEnabled(channel)) {
+ return;
+ }
+
+ Fprinter& out = JitSpewPrinter();
+ out.printf("[%s] ", ChannelNames[channel]);
+ for (size_t i = ChannelIndentLevel[channel]; i != 0; i--) {
+ out.put(" ");
+ }
+}
+
+bool jit::JitSpewEnabled(JitSpewChannel channel) {
+ MOZ_ASSERT(LoggingChecked);
+ return (LoggingBits & (uint64_t(1) << uint32_t(channel))) &&
+ !filteredOutCompilations;
+}
+
+void jit::EnableChannel(JitSpewChannel channel) {
+ MOZ_ASSERT(LoggingChecked);
+ LoggingBits |= uint64_t(1) << uint32_t(channel);
+}
+
+void jit::DisableChannel(JitSpewChannel channel) {
+ MOZ_ASSERT(LoggingChecked);
+ LoggingBits &= ~(uint64_t(1) << uint32_t(channel));
+}
+
+const char* js::jit::ValTypeToString(JSValueType type) {
+ switch (type) {
+ case JSVAL_TYPE_DOUBLE:
+ return "Double";
+ case JSVAL_TYPE_INT32:
+ return "Int32";
+ case JSVAL_TYPE_BOOLEAN:
+ return "Boolean";
+ case JSVAL_TYPE_UNDEFINED:
+ return "Undefined";
+ case JSVAL_TYPE_NULL:
+ return "Null";
+ case JSVAL_TYPE_MAGIC:
+ return "Magic";
+ case JSVAL_TYPE_STRING:
+ return "String";
+ case JSVAL_TYPE_SYMBOL:
+ return "Symbol";
+ case JSVAL_TYPE_PRIVATE_GCTHING:
+ return "PrivateGCThing";
+ case JSVAL_TYPE_BIGINT:
+ return "BigInt";
+ case JSVAL_TYPE_OBJECT:
+ return "Object";
+ case JSVAL_TYPE_UNKNOWN:
+ return "None";
+ default:
+ MOZ_CRASH("Unknown JSValueType");
+ }
+}
+
+#endif /* JS_JITSPEW */
diff --git a/js/src/jit/JitSpewer.h b/js/src/jit/JitSpewer.h
new file mode 100644
index 0000000000..0eda228695
--- /dev/null
+++ b/js/src/jit/JitSpewer.h
@@ -0,0 +1,284 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitSpewer_h
+#define jit_JitSpewer_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/IntegerPrintfMacros.h"
+
+#include <stdarg.h>
+
+#include "jit/JSONSpewer.h"
+#include "js/TypeDecls.h"
+#include "vm/Printer.h"
+
+enum JSValueType : uint8_t;
+
+namespace js {
+namespace jit {
+
+// New channels may be added below.
+#define JITSPEW_CHANNEL_LIST(_) \
+ /* Information during sinking */ \
+ _(Prune) \
+ /* Information during escape analysis */ \
+ _(Escape) \
+ /* Information during alias analysis */ \
+ _(Alias) \
+ /* Information during alias analysis */ \
+ _(AliasSummaries) \
+ /* Information during GVN */ \
+ _(GVN) \
+ /* Information during sinking */ \
+ _(Sink) \
+ /* Information during Range analysis */ \
+ _(Range) \
+ /* Information during LICM */ \
+ _(LICM) \
+ /* Info about fold linear constants */ \
+ _(FLAC) \
+ /* Effective address analysis info */ \
+ _(EAA) \
+ /* Wasm Bounds Check Elimination */ \
+ _(WasmBCE) \
+ /* Information during regalloc */ \
+ _(RegAlloc) \
+ /* Information during inlining */ \
+ _(Inlining) \
+ /* Information during codegen */ \
+ _(Codegen) \
+ /* Debug info about safepoints */ \
+ _(Safepoints) \
+ /* Debug info about Pools*/ \
+ _(Pools) \
+ /* Profiling-related information */ \
+ _(Profiling) \
+ /* Debug info about the I$ */ \
+ _(CacheFlush) \
+ /* Info about redundant shape guards */ \
+ _(RedundantShapeGuards) \
+ /* Output a list of MIR expressions */ \
+ _(MIRExpressions) \
+ /* Spew Tracelogger summary stats */ \
+ _(ScriptStats) \
+ \
+ /* BASELINE COMPILER SPEW */ \
+ \
+ /* Aborting Script Compilation. */ \
+ _(BaselineAbort) \
+ /* Script Compilation. */ \
+ _(BaselineScripts) \
+ /* Detailed op-specific spew. */ \
+ _(BaselineOp) \
+ /* Inline caches. */ \
+ _(BaselineIC) \
+ /* Inline cache fallbacks. */ \
+ _(BaselineICFallback) \
+ /* OSR from Baseline => Ion. */ \
+ _(BaselineOSR) \
+ /* Bailouts. */ \
+ _(BaselineBailouts) \
+ /* Debug Mode On Stack Recompile . */ \
+ _(BaselineDebugModeOSR) \
+ \
+ /* ION COMPILER SPEW */ \
+ \
+ /* Used to abort SSA construction */ \
+ _(IonAbort) \
+ /* Information about compiled scripts */ \
+ _(IonScripts) \
+ /* Info about failing to log script */ \
+ _(IonSyncLogs) \
+ /* Information during MIR building */ \
+ _(IonMIR) \
+ /* Information during bailouts */ \
+ _(IonBailouts) \
+ /* Information during OSI */ \
+ _(IonInvalidate) \
+ /* Debug info about snapshots */ \
+ _(IonSnapshots) \
+ /* Generated inline cache stubs */ \
+ _(IonIC) \
+ \
+ /* WARP SPEW */ \
+ \
+ /* Generated WarpSnapshots */ \
+ _(WarpSnapshots) \
+ /* CacheIR transpiler logging */ \
+ _(WarpTranspiler) \
+ /* Trial inlining for Warp */ \
+ _(WarpTrialInlining)
+
+enum JitSpewChannel {
+#define JITSPEW_CHANNEL(name) JitSpew_##name,
+ JITSPEW_CHANNEL_LIST(JITSPEW_CHANNEL)
+#undef JITSPEW_CHANNEL
+ JitSpew_Terminator
+};
+
+class BacktrackingAllocator;
+class MDefinition;
+class MIRGenerator;
+class MIRGraph;
+class TempAllocator;
+
+// The JitSpewer is only available on debug builds.
+// None of the global functions have effect on non-debug builds.
+#ifdef JS_JITSPEW
+
+// Class made to hold the MIR and LIR graphs of an Wasm / Ion compilation.
+class GraphSpewer {
+ private:
+ MIRGraph* graph_;
+ LSprinter jsonPrinter_;
+ JSONSpewer jsonSpewer_;
+
+ public:
+ explicit GraphSpewer(TempAllocator* alloc);
+
+ bool isSpewing() const { return graph_; }
+ void init(MIRGraph* graph, JSScript* function);
+ void beginFunction(JSScript* function);
+ void beginWasmFunction(unsigned funcIndex);
+ void spewPass(const char* pass);
+ void spewPass(const char* pass, BacktrackingAllocator* ra);
+ void endFunction();
+
+ void dump(Fprinter& json);
+};
+
+void SpewBeginFunction(MIRGenerator* mir, JSScript* function);
+void SpewBeginWasmFunction(MIRGenerator* mir, unsigned funcIndex);
+
+class AutoSpewEndFunction {
+ private:
+ MIRGenerator* mir_;
+
+ public:
+ explicit AutoSpewEndFunction(MIRGenerator* mir) : mir_(mir) {}
+ ~AutoSpewEndFunction();
+};
+
+void CheckLogging();
+Fprinter& JitSpewPrinter();
+
+class JitSpewIndent {
+ JitSpewChannel channel_;
+
+ public:
+ explicit JitSpewIndent(JitSpewChannel channel);
+ ~JitSpewIndent();
+};
+
+void JitSpew(JitSpewChannel channel, const char* fmt, ...)
+ MOZ_FORMAT_PRINTF(2, 3);
+void JitSpewStart(JitSpewChannel channel, const char* fmt, ...)
+ MOZ_FORMAT_PRINTF(2, 3);
+void JitSpewCont(JitSpewChannel channel, const char* fmt, ...)
+ MOZ_FORMAT_PRINTF(2, 3);
+void JitSpewFin(JitSpewChannel channel);
+void JitSpewHeader(JitSpewChannel channel);
+bool JitSpewEnabled(JitSpewChannel channel);
+void JitSpewVA(JitSpewChannel channel, const char* fmt, va_list ap)
+ MOZ_FORMAT_PRINTF(2, 0);
+void JitSpewStartVA(JitSpewChannel channel, const char* fmt, va_list ap)
+ MOZ_FORMAT_PRINTF(2, 0);
+void JitSpewContVA(JitSpewChannel channel, const char* fmt, va_list ap)
+ MOZ_FORMAT_PRINTF(2, 0);
+void JitSpewDef(JitSpewChannel channel, const char* str, MDefinition* def);
+
+void EnableChannel(JitSpewChannel channel);
+void DisableChannel(JitSpewChannel channel);
+void EnableIonDebugSyncLogging();
+void EnableIonDebugAsyncLogging();
+
+const char* ValTypeToString(JSValueType type);
+
+# define JitSpewIfEnabled(channel, fmt, ...) \
+ do { \
+ if (JitSpewEnabled(channel)) { \
+ JitSpew(channel, fmt, __VA_ARGS__); \
+ } \
+ } while (false);
+
+#else
+
+class GraphSpewer {
+ public:
+ explicit GraphSpewer(TempAllocator* alloc) {}
+
+ bool isSpewing() { return false; }
+ void init(MIRGraph* graph, JSScript* function) {}
+ void beginFunction(JSScript* function) {}
+ void spewPass(const char* pass) {}
+ void spewPass(const char* pass, BacktrackingAllocator* ra) {}
+ void endFunction() {}
+
+ void dump(Fprinter& c1, Fprinter& json) {}
+};
+
+static inline void SpewBeginFunction(MIRGenerator* mir, JSScript* function) {}
+static inline void SpewBeginWasmFunction(MIRGenerator* mir,
+ unsigned funcIndex) {}
+
+class AutoSpewEndFunction {
+ public:
+ explicit AutoSpewEndFunction(MIRGenerator* mir) {}
+ ~AutoSpewEndFunction() {}
+};
+
+static inline void CheckLogging() {}
+static inline Fprinter& JitSpewPrinter() {
+ MOZ_CRASH("No empty backend for JitSpewPrinter");
+}
+
+class JitSpewIndent {
+ public:
+ explicit JitSpewIndent(JitSpewChannel channel) {}
+ ~JitSpewIndent() {}
+};
+
+// The computation of some of the argument of the spewing functions might be
+// costly, thus we use variaidic macros to ignore any argument of these
+// functions.
+static inline void JitSpewCheckArguments(JitSpewChannel channel,
+ const char* fmt) {}
+
+# define JitSpewCheckExpandedArgs(channel, fmt, ...) \
+ JitSpewCheckArguments(channel, fmt)
+# define JitSpewCheckExpandedArgs_(ArgList) \
+ JitSpewCheckExpandedArgs ArgList /* Fix MSVC issue */
+# define JitSpew(...) JitSpewCheckExpandedArgs_((__VA_ARGS__))
+# define JitSpewStart(...) JitSpewCheckExpandedArgs_((__VA_ARGS__))
+# define JitSpewCont(...) JitSpewCheckExpandedArgs_((__VA_ARGS__))
+
+# define JitSpewIfEnabled(channel, fmt, ...) \
+ JitSpewCheckArguments(channel, fmt)
+
+static inline void JitSpewFin(JitSpewChannel channel) {}
+
+static inline void JitSpewHeader(JitSpewChannel channel) {}
+static inline bool JitSpewEnabled(JitSpewChannel channel) { return false; }
+static inline MOZ_FORMAT_PRINTF(2, 0) void JitSpewVA(JitSpewChannel channel,
+ const char* fmt,
+ va_list ap) {}
+static inline void JitSpewDef(JitSpewChannel channel, const char* str,
+ MDefinition* def) {}
+
+static inline void EnableChannel(JitSpewChannel) {}
+static inline void DisableChannel(JitSpewChannel) {}
+static inline void EnableIonDebugSyncLogging() {}
+static inline void EnableIonDebugAsyncLogging() {}
+
+#endif /* JS_JITSPEW */
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitSpewer_h */
diff --git a/js/src/jit/JitZone.h b/js/src/jit/JitZone.h
new file mode 100644
index 0000000000..8ed0e2d110
--- /dev/null
+++ b/js/src/jit/JitZone.h
@@ -0,0 +1,182 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitZone_h
+#define jit_JitZone_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/HashFunctions.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/MemoryReporting.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <utility>
+
+#include "gc/Barrier.h"
+#include "jit/ExecutableAllocator.h"
+#include "jit/ICStubSpace.h"
+#include "jit/Invalidation.h"
+#include "js/AllocPolicy.h"
+#include "js/GCHashTable.h"
+#include "js/HashTable.h"
+#include "js/TracingAPI.h"
+#include "js/TypeDecls.h"
+#include "js/UniquePtr.h"
+#include "js/Utility.h"
+#include "threading/ProtectedData.h"
+
+namespace JS {
+struct CodeSizes;
+}
+
+namespace js {
+namespace jit {
+
+enum class CacheKind : uint8_t;
+class CacheIRStubInfo;
+class JitCode;
+
+enum class ICStubEngine : uint8_t {
+ // Baseline IC, see BaselineIC.h.
+ Baseline = 0,
+
+ // Ion IC, see IonIC.h.
+ IonIC
+};
+
+struct CacheIRStubKey : public DefaultHasher<CacheIRStubKey> {
+ struct Lookup {
+ CacheKind kind;
+ ICStubEngine engine;
+ const uint8_t* code;
+ uint32_t length;
+
+ Lookup(CacheKind kind, ICStubEngine engine, const uint8_t* code,
+ uint32_t length)
+ : kind(kind), engine(engine), code(code), length(length) {}
+ };
+
+ static HashNumber hash(const Lookup& l);
+ static bool match(const CacheIRStubKey& entry, const Lookup& l);
+
+ UniquePtr<CacheIRStubInfo, JS::FreePolicy> stubInfo;
+
+ explicit CacheIRStubKey(CacheIRStubInfo* info) : stubInfo(info) {}
+ CacheIRStubKey(CacheIRStubKey&& other)
+ : stubInfo(std::move(other.stubInfo)) {}
+
+ void operator=(CacheIRStubKey&& other) {
+ stubInfo = std::move(other.stubInfo);
+ }
+};
+
+struct BaselineCacheIRStubCodeMapGCPolicy {
+ static bool traceWeak(JSTracer* trc, CacheIRStubKey*,
+ WeakHeapPtr<JitCode*>* value) {
+ return TraceWeakEdge(trc, value, "traceWeak");
+ }
+};
+
+class JitZone {
+ // Allocated space for optimized baseline stubs.
+ OptimizedICStubSpace optimizedStubSpace_;
+
+ // Set of CacheIRStubInfo instances used by Ion stubs in this Zone.
+ using IonCacheIRStubInfoSet =
+ HashSet<CacheIRStubKey, CacheIRStubKey, SystemAllocPolicy>;
+ IonCacheIRStubInfoSet ionCacheIRStubInfoSet_;
+
+ // Map CacheIRStubKey to shared JitCode objects.
+ using BaselineCacheIRStubCodeMap =
+ GCHashMap<CacheIRStubKey, WeakHeapPtr<JitCode*>, CacheIRStubKey,
+ SystemAllocPolicy, BaselineCacheIRStubCodeMapGCPolicy>;
+ BaselineCacheIRStubCodeMap baselineCacheIRStubCodes_;
+
+ // Executable allocator for all code except wasm code.
+ MainThreadData<ExecutableAllocator> execAlloc_;
+
+ // HashMap that maps scripts to compilations inlining those scripts.
+ using InlinedScriptMap =
+ GCHashMap<WeakHeapPtr<BaseScript*>, RecompileInfoVector,
+ MovableCellHasher<WeakHeapPtr<BaseScript*>>, SystemAllocPolicy>;
+ InlinedScriptMap inlinedCompilations_;
+
+ mozilla::Maybe<IonCompilationId> currentCompilationId_;
+ bool keepJitScripts_ = false;
+
+ public:
+ ~JitZone() { MOZ_ASSERT(!keepJitScripts_); }
+
+ void traceWeak(JSTracer* trc);
+
+ void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf,
+ JS::CodeSizes* code, size_t* jitZone,
+ size_t* baselineStubsOptimized) const;
+
+ OptimizedICStubSpace* optimizedStubSpace() { return &optimizedStubSpace_; }
+
+ JitCode* getBaselineCacheIRStubCode(const CacheIRStubKey::Lookup& key,
+ CacheIRStubInfo** stubInfo) {
+ auto p = baselineCacheIRStubCodes_.lookup(key);
+ if (p) {
+ *stubInfo = p->key().stubInfo.get();
+ return p->value();
+ }
+ *stubInfo = nullptr;
+ return nullptr;
+ }
+ [[nodiscard]] bool putBaselineCacheIRStubCode(
+ const CacheIRStubKey::Lookup& lookup, CacheIRStubKey& key,
+ JitCode* stubCode) {
+ auto p = baselineCacheIRStubCodes_.lookupForAdd(lookup);
+ MOZ_ASSERT(!p);
+ return baselineCacheIRStubCodes_.add(p, std::move(key), stubCode);
+ }
+
+ CacheIRStubInfo* getIonCacheIRStubInfo(const CacheIRStubKey::Lookup& key) {
+ IonCacheIRStubInfoSet::Ptr p = ionCacheIRStubInfoSet_.lookup(key);
+ return p ? p->stubInfo.get() : nullptr;
+ }
+ [[nodiscard]] bool putIonCacheIRStubInfo(const CacheIRStubKey::Lookup& lookup,
+ CacheIRStubKey& key) {
+ IonCacheIRStubInfoSet::AddPtr p =
+ ionCacheIRStubInfoSet_.lookupForAdd(lookup);
+ MOZ_ASSERT(!p);
+ return ionCacheIRStubInfoSet_.add(p, std::move(key));
+ }
+ void purgeIonCacheIRStubInfo() { ionCacheIRStubInfoSet_.clearAndCompact(); }
+
+ ExecutableAllocator& execAlloc() { return execAlloc_.ref(); }
+ const ExecutableAllocator& execAlloc() const { return execAlloc_.ref(); }
+
+ [[nodiscard]] bool addInlinedCompilation(const RecompileInfo& info,
+ JSScript* inlined);
+
+ RecompileInfoVector* maybeInlinedCompilations(JSScript* inlined) {
+ auto p = inlinedCompilations_.lookup(inlined);
+ return p ? &p->value() : nullptr;
+ }
+
+ void removeInlinedCompilations(JSScript* inlined) {
+ inlinedCompilations_.remove(inlined);
+ }
+
+ bool keepJitScripts() const { return keepJitScripts_; }
+ void setKeepJitScripts(bool keep) { keepJitScripts_ = keep; }
+
+ mozilla::Maybe<IonCompilationId> currentCompilationId() const {
+ return currentCompilationId_;
+ }
+ mozilla::Maybe<IonCompilationId>& currentCompilationIdRef() {
+ return currentCompilationId_;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitZone_h */
diff --git a/js/src/jit/JitcodeMap.cpp b/js/src/jit/JitcodeMap.cpp
new file mode 100644
index 0000000000..1c1e21c72e
--- /dev/null
+++ b/js/src/jit/JitcodeMap.cpp
@@ -0,0 +1,1145 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/JitcodeMap.h"
+
+#include "mozilla/ArrayUtils.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/ScopeExit.h"
+
+#include "gc/Marking.h"
+#include "jit/BaselineJIT.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "js/Vector.h"
+#include "vm/BytecodeLocation.h" // for BytecodeLocation
+#include "vm/GeckoProfiler.h"
+
+#include "vm/GeckoProfiler-inl.h"
+#include "vm/JSScript-inl.h"
+
+using mozilla::Maybe;
+
+namespace js {
+namespace jit {
+
+static inline JitcodeRegionEntry RegionAtAddr(const IonEntry& entry, void* ptr,
+ uint32_t* ptrOffset) {
+ MOZ_ASSERT(entry.containsPointer(ptr));
+ *ptrOffset = reinterpret_cast<uint8_t*>(ptr) -
+ reinterpret_cast<uint8_t*>(entry.nativeStartAddr());
+
+ uint32_t regionIdx = entry.regionTable()->findRegionEntry(*ptrOffset);
+ MOZ_ASSERT(regionIdx < entry.regionTable()->numRegions());
+
+ return entry.regionTable()->regionEntry(regionIdx);
+}
+
+void* IonEntry::canonicalNativeAddrFor(void* ptr) const {
+ uint32_t ptrOffset;
+ JitcodeRegionEntry region = RegionAtAddr(*this, ptr, &ptrOffset);
+ return (void*)(((uint8_t*)nativeStartAddr()) + region.nativeOffset());
+}
+
+bool IonEntry::callStackAtAddr(void* ptr, BytecodeLocationVector& results,
+ uint32_t* depth) const {
+ uint32_t ptrOffset;
+ JitcodeRegionEntry region = RegionAtAddr(*this, ptr, &ptrOffset);
+ *depth = region.scriptDepth();
+
+ JitcodeRegionEntry::ScriptPcIterator locationIter = region.scriptPcIterator();
+ MOZ_ASSERT(locationIter.hasMore());
+ bool first = true;
+ while (locationIter.hasMore()) {
+ uint32_t scriptIdx, pcOffset;
+ locationIter.readNext(&scriptIdx, &pcOffset);
+ // For the first entry pushed (innermost frame), the pcOffset is obtained
+ // from the delta-run encodings.
+ if (first) {
+ pcOffset = region.findPcOffset(ptrOffset, pcOffset);
+ first = false;
+ }
+ JSScript* script = getScript(scriptIdx);
+ jsbytecode* pc = script->offsetToPC(pcOffset);
+ if (!results.append(BytecodeLocation(script, pc))) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+uint32_t IonEntry::callStackAtAddr(void* ptr, const char** results,
+ uint32_t maxResults) const {
+ MOZ_ASSERT(maxResults >= 1);
+
+ uint32_t ptrOffset;
+ JitcodeRegionEntry region = RegionAtAddr(*this, ptr, &ptrOffset);
+
+ JitcodeRegionEntry::ScriptPcIterator locationIter = region.scriptPcIterator();
+ MOZ_ASSERT(locationIter.hasMore());
+ uint32_t count = 0;
+ while (locationIter.hasMore()) {
+ uint32_t scriptIdx, pcOffset;
+
+ locationIter.readNext(&scriptIdx, &pcOffset);
+ MOZ_ASSERT(getStr(scriptIdx));
+
+ results[count++] = getStr(scriptIdx);
+ if (count >= maxResults) {
+ break;
+ }
+ }
+
+ return count;
+}
+
+uint64_t IonEntry::lookupRealmID(void* ptr) const {
+ uint32_t ptrOffset;
+ JitcodeRegionEntry region = RegionAtAddr(*this, ptr, &ptrOffset);
+ JitcodeRegionEntry::ScriptPcIterator locationIter = region.scriptPcIterator();
+ MOZ_ASSERT(locationIter.hasMore());
+ uint32_t scriptIdx, pcOffset;
+ locationIter.readNext(&scriptIdx, &pcOffset);
+
+ JSScript* script = getScript(scriptIdx);
+ return script->realm()->creationOptions().profilerRealmID();
+}
+
+IonEntry::~IonEntry() {
+ // The region table is stored at the tail of the compacted data,
+ // which means the start of the region table is a pointer to
+ // the _middle_ of the memory space allocated for it.
+ //
+ // When freeing it, obtain the payload start pointer first.
+ MOZ_ASSERT(regionTable_);
+ js_free((void*)(regionTable_->payloadStart()));
+ regionTable_ = nullptr;
+}
+
+static IonEntry& IonEntryForIonIC(JSRuntime* rt, const IonICEntry* icEntry) {
+ // The table must have an IonEntry for the IC's rejoin address.
+ auto* table = rt->jitRuntime()->getJitcodeGlobalTable();
+ auto* entry = table->lookup(icEntry->rejoinAddr());
+ MOZ_ASSERT(entry);
+ MOZ_RELEASE_ASSERT(entry->isIon());
+ return entry->asIon();
+}
+
+void* IonICEntry::canonicalNativeAddrFor(void* ptr) const { return ptr; }
+
+bool IonICEntry::callStackAtAddr(JSRuntime* rt, void* ptr,
+ BytecodeLocationVector& results,
+ uint32_t* depth) const {
+ const IonEntry& entry = IonEntryForIonIC(rt, this);
+ return entry.callStackAtAddr(rejoinAddr(), results, depth);
+}
+
+uint32_t IonICEntry::callStackAtAddr(JSRuntime* rt, void* ptr,
+ const char** results,
+ uint32_t maxResults) const {
+ const IonEntry& entry = IonEntryForIonIC(rt, this);
+ return entry.callStackAtAddr(rejoinAddr(), results, maxResults);
+}
+
+uint64_t IonICEntry::lookupRealmID(JSRuntime* rt, void* ptr) const {
+ const IonEntry& entry = IonEntryForIonIC(rt, this);
+ return entry.lookupRealmID(rejoinAddr());
+}
+
+void* BaselineEntry::canonicalNativeAddrFor(void* ptr) const {
+ // TODO: We can't yet normalize Baseline addresses until we unify
+ // BaselineScript's PCMappingEntries with JitcodeGlobalTable.
+ return ptr;
+}
+
+bool BaselineEntry::callStackAtAddr(void* ptr, BytecodeLocationVector& results,
+ uint32_t* depth) const {
+ MOZ_ASSERT(containsPointer(ptr));
+ MOZ_ASSERT(script_->hasBaselineScript());
+
+ uint8_t* addr = reinterpret_cast<uint8_t*>(ptr);
+ jsbytecode* pc =
+ script_->baselineScript()->approximatePcForNativeAddress(script_, addr);
+ if (!results.append(BytecodeLocation(script_, pc))) {
+ return false;
+ }
+
+ *depth = 1;
+
+ return true;
+}
+
+uint32_t BaselineEntry::callStackAtAddr(void* ptr, const char** results,
+ uint32_t maxResults) const {
+ MOZ_ASSERT(containsPointer(ptr));
+ MOZ_ASSERT(maxResults >= 1);
+
+ results[0] = str();
+ return 1;
+}
+
+uint64_t BaselineEntry::lookupRealmID() const {
+ return script_->realm()->creationOptions().profilerRealmID();
+}
+
+void* BaselineInterpreterEntry::canonicalNativeAddrFor(void* ptr) const {
+ return ptr;
+}
+
+bool BaselineInterpreterEntry::callStackAtAddr(void* ptr,
+ BytecodeLocationVector& results,
+ uint32_t* depth) const {
+ MOZ_CRASH("shouldn't be called for BaselineInterpreter entries");
+}
+
+uint32_t BaselineInterpreterEntry::callStackAtAddr(void* ptr,
+ const char** results,
+ uint32_t maxResults) const {
+ MOZ_CRASH("shouldn't be called for BaselineInterpreter entries");
+}
+
+uint64_t BaselineInterpreterEntry::lookupRealmID() const {
+ MOZ_CRASH("shouldn't be called for BaselineInterpreter entries");
+}
+
+const JitcodeGlobalEntry* JitcodeGlobalTable::lookupForSampler(
+ void* ptr, JSRuntime* rt, uint64_t samplePosInBuffer) {
+ JitcodeGlobalEntry* entry = lookupInternal(ptr);
+ if (!entry) {
+ return nullptr;
+ }
+
+ entry->setSamplePositionInBuffer(samplePosInBuffer);
+
+ // IonIC entries must keep their corresponding Ion entries alive.
+ if (entry->isIonIC()) {
+ IonEntry& ionEntry = IonEntryForIonIC(rt, &entry->asIonIC());
+ ionEntry.setSamplePositionInBuffer(samplePosInBuffer);
+ }
+
+ // JitcodeGlobalEntries are marked at the end of the mark phase. A read
+ // barrier is not needed. Any JS frames sampled during the sweep phase of
+ // the GC must be on stack, and on-stack frames must already be marked at
+ // the beginning of the sweep phase. It's not possible to assert this here
+ // as we may be off main thread when called from the gecko profiler.
+
+ return entry;
+}
+
+JitcodeGlobalEntry* JitcodeGlobalTable::lookupInternal(void* ptr) {
+ // Search for an entry containing the one-byte range starting at |ptr|.
+ JitCodeRange range(ptr, static_cast<uint8_t*>(ptr) + 1);
+
+ if (JitCodeRange** entry = tree_.maybeLookup(&range)) {
+ MOZ_ASSERT((*entry)->containsPointer(ptr));
+ return static_cast<JitcodeGlobalEntry*>(*entry);
+ }
+
+ return nullptr;
+}
+
+bool JitcodeGlobalTable::addEntry(UniqueJitcodeGlobalEntry entry) {
+ MOZ_ASSERT(entry->isIon() || entry->isIonIC() || entry->isBaseline() ||
+ entry->isBaselineInterpreter() || entry->isDummy());
+
+ // Assert the new entry does not have a code range that's equal to (or
+ // contained in) one of the existing entries, because that would confuse the
+ // AVL tree.
+ MOZ_ASSERT(!tree_.maybeLookup(entry.get()));
+
+ // Suppress profiler sampling while data structures are being mutated.
+ AutoSuppressProfilerSampling suppressSampling(TlsContext.get());
+
+ if (!entries_.append(std::move(entry))) {
+ return false;
+ }
+ if (!tree_.insert(entries_.back().get())) {
+ entries_.popBack();
+ return false;
+ }
+
+ return true;
+}
+
+void JitcodeGlobalTable::setAllEntriesAsExpired() {
+ AutoSuppressProfilerSampling suppressSampling(TlsContext.get());
+ for (EntryVector::Range r(entries_.all()); !r.empty(); r.popFront()) {
+ auto& entry = r.front();
+ entry->setAsExpired();
+ }
+}
+
+bool JitcodeGlobalTable::markIteratively(GCMarker* marker) {
+ // JitcodeGlobalTable must keep entries that are in the sampler buffer
+ // alive. This conditionality is akin to holding the entries weakly.
+ //
+ // If this table were marked at the beginning of the mark phase, then
+ // sampling would require a read barrier for sampling in between
+ // incremental GC slices. However, invoking read barriers from the sampler
+ // is wildly unsafe. The sampler may run at any time, including during GC
+ // itself.
+ //
+ // Instead, JitcodeGlobalTable is marked at the beginning of the sweep
+ // phase, along with weak references. The key assumption is the
+ // following. At the beginning of the sweep phase, any JS frames that the
+ // sampler may put in its buffer that are not already there at the
+ // beginning of the mark phase must have already been marked, as either 1)
+ // the frame was on-stack at the beginning of the sweep phase, or 2) the
+ // frame was pushed between incremental sweep slices. Frames of case 1)
+ // are already marked. Frames of case 2) must have been reachable to have
+ // been newly pushed, and thus are already marked.
+ //
+ // The approach above obviates the need for read barriers. The assumption
+ // above is checked in JitcodeGlobalTable::lookupForSampler.
+
+ MOZ_ASSERT(!JS::RuntimeHeapIsMinorCollecting());
+
+ AutoSuppressProfilerSampling suppressSampling(TlsContext.get());
+
+ // If the profiler is off, rangeStart will be Nothing() and all entries are
+ // considered to be expired.
+ Maybe<uint64_t> rangeStart =
+ marker->runtime()->profilerSampleBufferRangeStart();
+
+ bool markedAny = false;
+ for (EntryVector::Range r(entries_.all()); !r.empty(); r.popFront()) {
+ auto& entry = r.front();
+
+ // If an entry is not sampled, reset its buffer position to the invalid
+ // position, and conditionally mark the rest of the entry if its
+ // JitCode is not already marked. This conditional marking ensures
+ // that so long as the JitCode *may* be sampled, we keep any
+ // information that may be handed out to the sampler, like tracked
+ // types used by optimizations and scripts used for pc to line number
+ // mapping, alive as well.
+ if (!rangeStart || !entry->isSampled(*rangeStart)) {
+ entry->setAsExpired();
+ if (!entry->isJitcodeMarkedFromAnyThread(marker->runtime())) {
+ continue;
+ }
+ }
+
+ // The table is runtime-wide. Not all zones may be participating in
+ // the GC.
+ if (!entry->zone()->isCollecting() || entry->zone()->isGCFinished()) {
+ continue;
+ }
+
+ markedAny |= entry->trace(marker->tracer());
+ }
+
+ return markedAny;
+}
+
+void JitcodeGlobalTable::traceWeak(JSRuntime* rt, JSTracer* trc) {
+ AutoSuppressProfilerSampling suppressSampling(rt->mainContextFromOwnThread());
+
+ entries_.eraseIf([&](auto& entry) {
+ if (!entry->zone()->isCollecting() || entry->zone()->isGCFinished()) {
+ return false;
+ }
+
+ if (TraceManuallyBarrieredWeakEdge(
+ trc, entry->jitcodePtr(),
+ "JitcodeGlobalTable::JitcodeGlobalEntry::jitcode_")) {
+ entry->traceWeak(trc);
+ return false;
+ }
+
+ // We have to remove the entry.
+#ifdef DEBUG
+ Maybe<uint64_t> rangeStart = rt->profilerSampleBufferRangeStart();
+ MOZ_ASSERT_IF(rangeStart, !entry->isSampled(*rangeStart));
+#endif
+ tree_.remove(entry.get());
+ return true;
+ });
+
+ MOZ_ASSERT(tree_.empty() == entries_.empty());
+}
+
+bool JitcodeGlobalEntry::traceJitcode(JSTracer* trc) {
+ if (!IsMarkedUnbarriered(trc->runtime(), jitcode_)) {
+ TraceManuallyBarrieredEdge(trc, &jitcode_,
+ "jitcodglobaltable-baseentry-jitcode");
+ return true;
+ }
+ return false;
+}
+
+bool JitcodeGlobalEntry::isJitcodeMarkedFromAnyThread(JSRuntime* rt) {
+ return IsMarkedUnbarriered(rt, jitcode_);
+}
+
+bool BaselineEntry::trace(JSTracer* trc) {
+ if (!IsMarkedUnbarriered(trc->runtime(), script_)) {
+ TraceManuallyBarrieredEdge(trc, &script_,
+ "jitcodeglobaltable-baselineentry-script");
+ return true;
+ }
+ return false;
+}
+
+void BaselineEntry::traceWeak(JSTracer* trc) {
+ MOZ_ALWAYS_TRUE(
+ TraceManuallyBarrieredWeakEdge(trc, &script_, "BaselineEntry::script_"));
+}
+
+bool IonEntry::trace(JSTracer* trc) {
+ bool tracedAny = false;
+
+ JSRuntime* rt = trc->runtime();
+ for (auto& pair : scriptList_) {
+ if (!IsMarkedUnbarriered(rt, pair.script)) {
+ TraceManuallyBarrieredEdge(trc, &pair.script,
+ "jitcodeglobaltable-ionentry-script");
+ tracedAny = true;
+ }
+ }
+
+ return tracedAny;
+}
+
+void IonEntry::traceWeak(JSTracer* trc) {
+ for (auto& pair : scriptList_) {
+ JSScript** scriptp = &pair.script;
+ MOZ_ALWAYS_TRUE(
+ TraceManuallyBarrieredWeakEdge(trc, scriptp, "IonEntry script"));
+ }
+}
+
+bool IonICEntry::trace(JSTracer* trc) {
+ IonEntry& entry = IonEntryForIonIC(trc->runtime(), this);
+ return entry.trace(trc);
+}
+
+void IonICEntry::traceWeak(JSTracer* trc) {
+ IonEntry& entry = IonEntryForIonIC(trc->runtime(), this);
+ entry.traceWeak(trc);
+}
+
+[[nodiscard]] bool JitcodeGlobalEntry::callStackAtAddr(
+ JSRuntime* rt, void* ptr, BytecodeLocationVector& results,
+ uint32_t* depth) const {
+ switch (kind()) {
+ case Kind::Ion:
+ return asIon().callStackAtAddr(ptr, results, depth);
+ case Kind::IonIC:
+ return asIonIC().callStackAtAddr(rt, ptr, results, depth);
+ case Kind::Baseline:
+ return asBaseline().callStackAtAddr(ptr, results, depth);
+ case Kind::BaselineInterpreter:
+ return asBaselineInterpreter().callStackAtAddr(ptr, results, depth);
+ case Kind::Dummy:
+ return asDummy().callStackAtAddr(rt, ptr, results, depth);
+ }
+ MOZ_CRASH("Invalid kind");
+}
+
+uint32_t JitcodeGlobalEntry::callStackAtAddr(JSRuntime* rt, void* ptr,
+ const char** results,
+ uint32_t maxResults) const {
+ switch (kind()) {
+ case Kind::Ion:
+ return asIon().callStackAtAddr(ptr, results, maxResults);
+ case Kind::IonIC:
+ return asIonIC().callStackAtAddr(rt, ptr, results, maxResults);
+ case Kind::Baseline:
+ return asBaseline().callStackAtAddr(ptr, results, maxResults);
+ case Kind::BaselineInterpreter:
+ return asBaselineInterpreter().callStackAtAddr(ptr, results, maxResults);
+ case Kind::Dummy:
+ return asDummy().callStackAtAddr(rt, ptr, results, maxResults);
+ }
+ MOZ_CRASH("Invalid kind");
+}
+
+uint64_t JitcodeGlobalEntry::lookupRealmID(JSRuntime* rt, void* ptr) const {
+ switch (kind()) {
+ case Kind::Ion:
+ return asIon().lookupRealmID(ptr);
+ case Kind::IonIC:
+ return asIonIC().lookupRealmID(rt, ptr);
+ case Kind::Baseline:
+ return asBaseline().lookupRealmID();
+ case Kind::Dummy:
+ return asDummy().lookupRealmID();
+ case Kind::BaselineInterpreter:
+ break;
+ }
+ MOZ_CRASH("Invalid kind");
+}
+
+bool JitcodeGlobalEntry::trace(JSTracer* trc) {
+ bool tracedAny = traceJitcode(trc);
+ switch (kind()) {
+ case Kind::Ion:
+ tracedAny |= asIon().trace(trc);
+ break;
+ case Kind::IonIC:
+ tracedAny |= asIonIC().trace(trc);
+ break;
+ case Kind::Baseline:
+ tracedAny |= asBaseline().trace(trc);
+ break;
+ case Kind::BaselineInterpreter:
+ case Kind::Dummy:
+ break;
+ }
+ return tracedAny;
+}
+
+void JitcodeGlobalEntry::traceWeak(JSTracer* trc) {
+ switch (kind()) {
+ case Kind::Ion:
+ asIon().traceWeak(trc);
+ break;
+ case Kind::IonIC:
+ asIonIC().traceWeak(trc);
+ break;
+ case Kind::Baseline:
+ asBaseline().traceWeak(trc);
+ break;
+ case Kind::BaselineInterpreter:
+ case Kind::Dummy:
+ break;
+ }
+}
+
+void* JitcodeGlobalEntry::canonicalNativeAddrFor(JSRuntime* rt,
+ void* ptr) const {
+ switch (kind()) {
+ case Kind::Ion:
+ return asIon().canonicalNativeAddrFor(ptr);
+ case Kind::IonIC:
+ return asIonIC().canonicalNativeAddrFor(ptr);
+ case Kind::Baseline:
+ return asBaseline().canonicalNativeAddrFor(ptr);
+ case Kind::Dummy:
+ return asDummy().canonicalNativeAddrFor(rt, ptr);
+ case Kind::BaselineInterpreter:
+ break;
+ }
+ MOZ_CRASH("Invalid kind");
+}
+
+// static
+void JitcodeGlobalEntry::DestroyPolicy::operator()(JitcodeGlobalEntry* entry) {
+ switch (entry->kind()) {
+ case JitcodeGlobalEntry::Kind::Ion:
+ js_delete(&entry->asIon());
+ break;
+ case JitcodeGlobalEntry::Kind::IonIC:
+ js_delete(&entry->asIonIC());
+ break;
+ case JitcodeGlobalEntry::Kind::Baseline:
+ js_delete(&entry->asBaseline());
+ break;
+ case JitcodeGlobalEntry::Kind::BaselineInterpreter:
+ js_delete(&entry->asBaselineInterpreter());
+ break;
+ case JitcodeGlobalEntry::Kind::Dummy:
+ js_delete(&entry->asDummy());
+ break;
+ }
+}
+
+/* static */
+void JitcodeRegionEntry::WriteHead(CompactBufferWriter& writer,
+ uint32_t nativeOffset, uint8_t scriptDepth) {
+ writer.writeUnsigned(nativeOffset);
+ writer.writeByte(scriptDepth);
+}
+
+/* static */
+void JitcodeRegionEntry::ReadHead(CompactBufferReader& reader,
+ uint32_t* nativeOffset,
+ uint8_t* scriptDepth) {
+ *nativeOffset = reader.readUnsigned();
+ *scriptDepth = reader.readByte();
+}
+
+/* static */
+void JitcodeRegionEntry::WriteScriptPc(CompactBufferWriter& writer,
+ uint32_t scriptIdx, uint32_t pcOffset) {
+ writer.writeUnsigned(scriptIdx);
+ writer.writeUnsigned(pcOffset);
+}
+
+/* static */
+void JitcodeRegionEntry::ReadScriptPc(CompactBufferReader& reader,
+ uint32_t* scriptIdx, uint32_t* pcOffset) {
+ *scriptIdx = reader.readUnsigned();
+ *pcOffset = reader.readUnsigned();
+}
+
+/* static */
+void JitcodeRegionEntry::WriteDelta(CompactBufferWriter& writer,
+ uint32_t nativeDelta, int32_t pcDelta) {
+ if (pcDelta >= 0) {
+ // 1 and 2-byte formats possible.
+
+ // NNNN-BBB0
+ if (pcDelta <= ENC1_PC_DELTA_MAX && nativeDelta <= ENC1_NATIVE_DELTA_MAX) {
+ uint8_t encVal = ENC1_MASK_VAL | (pcDelta << ENC1_PC_DELTA_SHIFT) |
+ (nativeDelta << ENC1_NATIVE_DELTA_SHIFT);
+ writer.writeByte(encVal);
+ return;
+ }
+
+ // NNNN-NNNN BBBB-BB01
+ if (pcDelta <= ENC2_PC_DELTA_MAX && nativeDelta <= ENC2_NATIVE_DELTA_MAX) {
+ uint16_t encVal = ENC2_MASK_VAL | (pcDelta << ENC2_PC_DELTA_SHIFT) |
+ (nativeDelta << ENC2_NATIVE_DELTA_SHIFT);
+ writer.writeByte(encVal & 0xff);
+ writer.writeByte((encVal >> 8) & 0xff);
+ return;
+ }
+ }
+
+ // NNNN-NNNN NNNB-BBBB BBBB-B011
+ if (pcDelta >= ENC3_PC_DELTA_MIN && pcDelta <= ENC3_PC_DELTA_MAX &&
+ nativeDelta <= ENC3_NATIVE_DELTA_MAX) {
+ uint32_t encVal =
+ ENC3_MASK_VAL |
+ ((uint32_t(pcDelta) << ENC3_PC_DELTA_SHIFT) & ENC3_PC_DELTA_MASK) |
+ (nativeDelta << ENC3_NATIVE_DELTA_SHIFT);
+ writer.writeByte(encVal & 0xff);
+ writer.writeByte((encVal >> 8) & 0xff);
+ writer.writeByte((encVal >> 16) & 0xff);
+ return;
+ }
+
+ // NNNN-NNNN NNNN-NNNN BBBB-BBBB BBBB-B111
+ if (pcDelta >= ENC4_PC_DELTA_MIN && pcDelta <= ENC4_PC_DELTA_MAX &&
+ nativeDelta <= ENC4_NATIVE_DELTA_MAX) {
+ uint32_t encVal =
+ ENC4_MASK_VAL |
+ ((uint32_t(pcDelta) << ENC4_PC_DELTA_SHIFT) & ENC4_PC_DELTA_MASK) |
+ (nativeDelta << ENC4_NATIVE_DELTA_SHIFT);
+ writer.writeByte(encVal & 0xff);
+ writer.writeByte((encVal >> 8) & 0xff);
+ writer.writeByte((encVal >> 16) & 0xff);
+ writer.writeByte((encVal >> 24) & 0xff);
+ return;
+ }
+
+ // Should never get here.
+ MOZ_CRASH("pcDelta/nativeDelta values are too large to encode.");
+}
+
+/* static */
+void JitcodeRegionEntry::ReadDelta(CompactBufferReader& reader,
+ uint32_t* nativeDelta, int32_t* pcDelta) {
+ // NB:
+ // It's possible to get nativeDeltas with value 0 in two cases:
+ //
+ // 1. The last region's run. This is because the region table's start
+ // must be 4-byte aligned, and we must insert padding bytes to align the
+ // payload section before emitting the table.
+ //
+ // 2. A zero-offset nativeDelta with a negative pcDelta.
+ //
+ // So if nativeDelta is zero, then pcDelta must be <= 0.
+
+ // NNNN-BBB0
+ const uint32_t firstByte = reader.readByte();
+ if ((firstByte & ENC1_MASK) == ENC1_MASK_VAL) {
+ uint32_t encVal = firstByte;
+ *nativeDelta = encVal >> ENC1_NATIVE_DELTA_SHIFT;
+ *pcDelta = (encVal & ENC1_PC_DELTA_MASK) >> ENC1_PC_DELTA_SHIFT;
+ MOZ_ASSERT_IF(*nativeDelta == 0, *pcDelta <= 0);
+ return;
+ }
+
+ // NNNN-NNNN BBBB-BB01
+ const uint32_t secondByte = reader.readByte();
+ if ((firstByte & ENC2_MASK) == ENC2_MASK_VAL) {
+ uint32_t encVal = firstByte | secondByte << 8;
+ *nativeDelta = encVal >> ENC2_NATIVE_DELTA_SHIFT;
+ *pcDelta = (encVal & ENC2_PC_DELTA_MASK) >> ENC2_PC_DELTA_SHIFT;
+ MOZ_ASSERT(*pcDelta != 0);
+ MOZ_ASSERT_IF(*nativeDelta == 0, *pcDelta <= 0);
+ return;
+ }
+
+ // NNNN-NNNN NNNB-BBBB BBBB-B011
+ const uint32_t thirdByte = reader.readByte();
+ if ((firstByte & ENC3_MASK) == ENC3_MASK_VAL) {
+ uint32_t encVal = firstByte | secondByte << 8 | thirdByte << 16;
+ *nativeDelta = encVal >> ENC3_NATIVE_DELTA_SHIFT;
+
+ uint32_t pcDeltaU = (encVal & ENC3_PC_DELTA_MASK) >> ENC3_PC_DELTA_SHIFT;
+ // Fix sign if necessary.
+ if (pcDeltaU > static_cast<uint32_t>(ENC3_PC_DELTA_MAX)) {
+ pcDeltaU |= ~ENC3_PC_DELTA_MAX;
+ }
+ *pcDelta = pcDeltaU;
+ MOZ_ASSERT(*pcDelta != 0);
+ MOZ_ASSERT_IF(*nativeDelta == 0, *pcDelta <= 0);
+ return;
+ }
+
+ // NNNN-NNNN NNNN-NNNN BBBB-BBBB BBBB-B111
+ MOZ_ASSERT((firstByte & ENC4_MASK) == ENC4_MASK_VAL);
+ const uint32_t fourthByte = reader.readByte();
+ uint32_t encVal =
+ firstByte | secondByte << 8 | thirdByte << 16 | fourthByte << 24;
+ *nativeDelta = encVal >> ENC4_NATIVE_DELTA_SHIFT;
+
+ uint32_t pcDeltaU = (encVal & ENC4_PC_DELTA_MASK) >> ENC4_PC_DELTA_SHIFT;
+ // fix sign if necessary
+ if (pcDeltaU > static_cast<uint32_t>(ENC4_PC_DELTA_MAX)) {
+ pcDeltaU |= ~ENC4_PC_DELTA_MAX;
+ }
+ *pcDelta = pcDeltaU;
+
+ MOZ_ASSERT(*pcDelta != 0);
+ MOZ_ASSERT_IF(*nativeDelta == 0, *pcDelta <= 0);
+}
+
+/* static */
+uint32_t JitcodeRegionEntry::ExpectedRunLength(const NativeToBytecode* entry,
+ const NativeToBytecode* end) {
+ MOZ_ASSERT(entry < end);
+
+ // We always use the first entry, so runLength starts at 1
+ uint32_t runLength = 1;
+
+ uint32_t curNativeOffset = entry->nativeOffset.offset();
+ uint32_t curBytecodeOffset = entry->tree->script()->pcToOffset(entry->pc);
+
+ for (auto nextEntry = entry + 1; nextEntry != end; nextEntry += 1) {
+ // If the next run moves to a different inline site, stop the run.
+ if (nextEntry->tree != entry->tree) {
+ break;
+ }
+
+ uint32_t nextNativeOffset = nextEntry->nativeOffset.offset();
+ uint32_t nextBytecodeOffset =
+ nextEntry->tree->script()->pcToOffset(nextEntry->pc);
+ MOZ_ASSERT(nextNativeOffset >= curNativeOffset);
+
+ uint32_t nativeDelta = nextNativeOffset - curNativeOffset;
+ int32_t bytecodeDelta =
+ int32_t(nextBytecodeOffset) - int32_t(curBytecodeOffset);
+
+ // If deltas are too large (very unlikely), stop the run.
+ if (!IsDeltaEncodeable(nativeDelta, bytecodeDelta)) {
+ break;
+ }
+
+ runLength++;
+
+ // If the run has grown to its maximum length, stop the run.
+ if (runLength == MAX_RUN_LENGTH) {
+ break;
+ }
+
+ curNativeOffset = nextNativeOffset;
+ curBytecodeOffset = nextBytecodeOffset;
+ }
+
+ return runLength;
+}
+
+struct JitcodeMapBufferWriteSpewer {
+#ifdef JS_JITSPEW
+ CompactBufferWriter* writer;
+ uint32_t startPos;
+
+ static const uint32_t DumpMaxBytes = 50;
+
+ explicit JitcodeMapBufferWriteSpewer(CompactBufferWriter& w)
+ : writer(&w), startPos(writer->length()) {}
+
+ void spewAndAdvance(const char* name) {
+ if (writer->oom()) {
+ return;
+ }
+
+ uint32_t curPos = writer->length();
+ const uint8_t* start = writer->buffer() + startPos;
+ const uint8_t* end = writer->buffer() + curPos;
+ const char* MAP = "0123456789ABCDEF";
+ uint32_t bytes = end - start;
+
+ char buffer[DumpMaxBytes * 3];
+ for (uint32_t i = 0; i < bytes; i++) {
+ buffer[i * 3] = MAP[(start[i] >> 4) & 0xf];
+ buffer[i * 3 + 1] = MAP[(start[i] >> 0) & 0xf];
+ buffer[i * 3 + 2] = ' ';
+ }
+ if (bytes >= DumpMaxBytes) {
+ buffer[DumpMaxBytes * 3 - 1] = '\0';
+ } else {
+ buffer[bytes * 3 - 1] = '\0';
+ }
+
+ JitSpew(JitSpew_Profiling, "%s@%d[%d bytes] - %s", name, int(startPos),
+ int(bytes), buffer);
+
+ // Move to the end of the current buffer.
+ startPos = writer->length();
+ }
+#else // !JS_JITSPEW
+ explicit JitcodeMapBufferWriteSpewer(CompactBufferWriter& w) {}
+ void spewAndAdvance(const char* name) {}
+#endif // JS_JITSPEW
+};
+
+// Write a run, starting at the given NativeToBytecode entry, into the given
+// buffer writer.
+/* static */
+bool JitcodeRegionEntry::WriteRun(CompactBufferWriter& writer,
+ const IonEntry::ScriptList& scriptList,
+ uint32_t runLength,
+ const NativeToBytecode* entry) {
+ MOZ_ASSERT(runLength > 0);
+ MOZ_ASSERT(runLength <= MAX_RUN_LENGTH);
+
+ // Calculate script depth.
+ MOZ_ASSERT(entry->tree->depth() <= 0xff);
+ uint8_t scriptDepth = entry->tree->depth();
+ uint32_t regionNativeOffset = entry->nativeOffset.offset();
+
+ JitcodeMapBufferWriteSpewer spewer(writer);
+
+ // Write the head info.
+ JitSpew(JitSpew_Profiling, " Head Info: nativeOffset=%d scriptDepth=%d",
+ int(regionNativeOffset), int(scriptDepth));
+ WriteHead(writer, regionNativeOffset, scriptDepth);
+ spewer.spewAndAdvance(" ");
+
+ // Write each script/pc pair.
+ {
+ InlineScriptTree* curTree = entry->tree;
+ jsbytecode* curPc = entry->pc;
+ for (uint8_t i = 0; i < scriptDepth; i++) {
+ // Find the index of the script within the list.
+ // NB: scriptList is guaranteed to contain curTree->script()
+ uint32_t scriptIdx = 0;
+ for (; scriptIdx < scriptList.length(); scriptIdx++) {
+ if (scriptList[scriptIdx].script == curTree->script()) {
+ break;
+ }
+ }
+ MOZ_ASSERT(scriptIdx < scriptList.length());
+
+ uint32_t pcOffset = curTree->script()->pcToOffset(curPc);
+
+ JitSpew(JitSpew_Profiling, " Script/PC %d: scriptIdx=%d pcOffset=%d",
+ int(i), int(scriptIdx), int(pcOffset));
+ WriteScriptPc(writer, scriptIdx, pcOffset);
+ spewer.spewAndAdvance(" ");
+
+ MOZ_ASSERT_IF(i < scriptDepth - 1, curTree->hasCaller());
+ curPc = curTree->callerPc();
+ curTree = curTree->caller();
+ }
+ }
+
+ // Start writing runs.
+ uint32_t curNativeOffset = entry->nativeOffset.offset();
+ uint32_t curBytecodeOffset = entry->tree->script()->pcToOffset(entry->pc);
+
+ JitSpew(JitSpew_Profiling,
+ " Writing Delta Run from nativeOffset=%d bytecodeOffset=%d",
+ int(curNativeOffset), int(curBytecodeOffset));
+
+ // Skip first entry because it is implicit in the header. Start at subsequent
+ // entry.
+ for (uint32_t i = 1; i < runLength; i++) {
+ MOZ_ASSERT(entry[i].tree == entry->tree);
+
+ uint32_t nextNativeOffset = entry[i].nativeOffset.offset();
+ uint32_t nextBytecodeOffset =
+ entry[i].tree->script()->pcToOffset(entry[i].pc);
+ MOZ_ASSERT(nextNativeOffset >= curNativeOffset);
+
+ uint32_t nativeDelta = nextNativeOffset - curNativeOffset;
+ int32_t bytecodeDelta =
+ int32_t(nextBytecodeOffset) - int32_t(curBytecodeOffset);
+ MOZ_ASSERT(IsDeltaEncodeable(nativeDelta, bytecodeDelta));
+
+ JitSpew(JitSpew_Profiling,
+ " RunEntry native: %d-%d [%d] bytecode: %d-%d [%d]",
+ int(curNativeOffset), int(nextNativeOffset), int(nativeDelta),
+ int(curBytecodeOffset), int(nextBytecodeOffset),
+ int(bytecodeDelta));
+ WriteDelta(writer, nativeDelta, bytecodeDelta);
+
+ // Spew the bytecode in these ranges.
+ if (curBytecodeOffset < nextBytecodeOffset) {
+ JitSpewStart(JitSpew_Profiling, " OPS: ");
+ uint32_t curBc = curBytecodeOffset;
+ while (curBc < nextBytecodeOffset) {
+ jsbytecode* pc = entry[i].tree->script()->offsetToPC(curBc);
+#ifdef JS_JITSPEW
+ JSOp op = JSOp(*pc);
+ JitSpewCont(JitSpew_Profiling, "%s ", CodeName(op));
+#endif
+ curBc += GetBytecodeLength(pc);
+ }
+ JitSpewFin(JitSpew_Profiling);
+ }
+ spewer.spewAndAdvance(" ");
+
+ curNativeOffset = nextNativeOffset;
+ curBytecodeOffset = nextBytecodeOffset;
+ }
+
+ if (writer.oom()) {
+ return false;
+ }
+
+ return true;
+}
+
+void JitcodeRegionEntry::unpack() {
+ CompactBufferReader reader(data_, end_);
+ ReadHead(reader, &nativeOffset_, &scriptDepth_);
+ MOZ_ASSERT(scriptDepth_ > 0);
+
+ scriptPcStack_ = reader.currentPosition();
+ // Skip past script/pc stack
+ for (unsigned i = 0; i < scriptDepth_; i++) {
+ uint32_t scriptIdx, pcOffset;
+ ReadScriptPc(reader, &scriptIdx, &pcOffset);
+ }
+
+ deltaRun_ = reader.currentPosition();
+}
+
+uint32_t JitcodeRegionEntry::findPcOffset(uint32_t queryNativeOffset,
+ uint32_t startPcOffset) const {
+ DeltaIterator iter = deltaIterator();
+ uint32_t curNativeOffset = nativeOffset();
+ uint32_t curPcOffset = startPcOffset;
+ while (iter.hasMore()) {
+ uint32_t nativeDelta;
+ int32_t pcDelta;
+ iter.readNext(&nativeDelta, &pcDelta);
+
+ // The start address of the next delta-run entry is counted towards
+ // the current delta-run entry, because return addresses should
+ // associate with the bytecode op prior (the call) not the op after.
+ if (queryNativeOffset <= curNativeOffset + nativeDelta) {
+ break;
+ }
+ curNativeOffset += nativeDelta;
+ curPcOffset += pcDelta;
+ }
+ return curPcOffset;
+}
+
+uint32_t JitcodeIonTable::findRegionEntry(uint32_t nativeOffset) const {
+ static const uint32_t LINEAR_SEARCH_THRESHOLD = 8;
+ uint32_t regions = numRegions();
+ MOZ_ASSERT(regions > 0);
+
+ // For small region lists, just search linearly.
+ if (regions <= LINEAR_SEARCH_THRESHOLD) {
+ JitcodeRegionEntry previousEntry = regionEntry(0);
+ for (uint32_t i = 1; i < regions; i++) {
+ JitcodeRegionEntry nextEntry = regionEntry(i);
+ MOZ_ASSERT(nextEntry.nativeOffset() >= previousEntry.nativeOffset());
+
+ // See note in binary-search code below about why we use '<=' here
+ // instead of '<'. Short explanation: regions are closed at their
+ // ending addresses, and open at their starting addresses.
+ if (nativeOffset <= nextEntry.nativeOffset()) {
+ return i - 1;
+ }
+
+ previousEntry = nextEntry;
+ }
+ // If nothing found, assume it falls within last region.
+ return regions - 1;
+ }
+
+ // For larger ones, binary search the region table.
+ uint32_t idx = 0;
+ uint32_t count = regions;
+ while (count > 1) {
+ uint32_t step = count / 2;
+ uint32_t mid = idx + step;
+ JitcodeRegionEntry midEntry = regionEntry(mid);
+
+ // A region memory range is closed at its ending address, not starting
+ // address. This is because the return address for calls must associate
+ // with the call's bytecode PC, not the PC of the bytecode operator after
+ // the call.
+ //
+ // So a query is < an entry if the query nativeOffset is <= the start
+ // address of the entry, and a query is >= an entry if the query
+ // nativeOffset is > the start address of an entry.
+ if (nativeOffset <= midEntry.nativeOffset()) {
+ // Target entry is below midEntry.
+ count = step;
+ } else { // if (nativeOffset > midEntry.nativeOffset())
+ // Target entry is at midEntry or above.
+ idx = mid;
+ count -= step;
+ }
+ }
+ return idx;
+}
+
+/* static */
+bool JitcodeIonTable::WriteIonTable(CompactBufferWriter& writer,
+ const IonEntry::ScriptList& scriptList,
+ const NativeToBytecode* start,
+ const NativeToBytecode* end,
+ uint32_t* tableOffsetOut,
+ uint32_t* numRegionsOut) {
+ MOZ_ASSERT(tableOffsetOut != nullptr);
+ MOZ_ASSERT(numRegionsOut != nullptr);
+ MOZ_ASSERT(writer.length() == 0);
+ MOZ_ASSERT(scriptList.length() > 0);
+
+ JitSpew(JitSpew_Profiling,
+ "Writing native to bytecode map for %s:%u:%u (%zu entries)",
+ scriptList[0].script->filename(), scriptList[0].script->lineno(),
+ scriptList[0].script->column(),
+ mozilla::PointerRangeSize(start, end));
+
+ JitSpew(JitSpew_Profiling, " ScriptList of size %u",
+ unsigned(scriptList.length()));
+ for (uint32_t i = 0; i < scriptList.length(); i++) {
+ JitSpew(JitSpew_Profiling, " Script %u - %s:%u:%u", i,
+ scriptList[i].script->filename(), scriptList[i].script->lineno(),
+ scriptList[i].script->column());
+ }
+
+ // Write out runs first. Keep a vector tracking the positive offsets from
+ // payload start to the run.
+ const NativeToBytecode* curEntry = start;
+ js::Vector<uint32_t, 32, SystemAllocPolicy> runOffsets;
+
+ while (curEntry != end) {
+ // Calculate the length of the next run.
+ uint32_t runLength = JitcodeRegionEntry::ExpectedRunLength(curEntry, end);
+ MOZ_ASSERT(runLength > 0);
+ MOZ_ASSERT(runLength <= uintptr_t(end - curEntry));
+ JitSpew(JitSpew_Profiling, " Run at entry %d, length %d, buffer offset %d",
+ int(curEntry - start), int(runLength), int(writer.length()));
+
+ // Store the offset of the run.
+ if (!runOffsets.append(writer.length())) {
+ return false;
+ }
+
+ // Encode the run.
+ if (!JitcodeRegionEntry::WriteRun(writer, scriptList, runLength,
+ curEntry)) {
+ return false;
+ }
+
+ curEntry += runLength;
+ }
+
+ // Done encoding regions. About to start table. Ensure we are aligned to 4
+ // bytes since table is composed of uint32_t values.
+ uint32_t padding = sizeof(uint32_t) - (writer.length() % sizeof(uint32_t));
+ if (padding == sizeof(uint32_t)) {
+ padding = 0;
+ }
+ JitSpew(JitSpew_Profiling, " Padding %d bytes after run @%d", int(padding),
+ int(writer.length()));
+ for (uint32_t i = 0; i < padding; i++) {
+ writer.writeByte(0);
+ }
+
+ // Now at start of table.
+ uint32_t tableOffset = writer.length();
+
+ // The table being written at this point will be accessed directly via
+ // uint32_t pointers, so all writes below use native endianness.
+
+ // Write out numRegions
+ JitSpew(JitSpew_Profiling, " Writing numRuns=%d", int(runOffsets.length()));
+ writer.writeNativeEndianUint32_t(runOffsets.length());
+
+ // Write out region offset table. The offsets in |runOffsets| are currently
+ // forward offsets from the beginning of the buffer. We convert them to
+ // backwards offsets from the start of the table before writing them into
+ // their table entries.
+ for (uint32_t i = 0; i < runOffsets.length(); i++) {
+ JitSpew(JitSpew_Profiling, " Run %d offset=%d backOffset=%d @%d", int(i),
+ int(runOffsets[i]), int(tableOffset - runOffsets[i]),
+ int(writer.length()));
+ writer.writeNativeEndianUint32_t(tableOffset - runOffsets[i]);
+ }
+
+ if (writer.oom()) {
+ return false;
+ }
+
+ *tableOffsetOut = tableOffset;
+ *numRegionsOut = runOffsets.length();
+ return true;
+}
+
+} // namespace jit
+} // namespace js
+
+JS::ProfiledFrameHandle::ProfiledFrameHandle(JSRuntime* rt,
+ js::jit::JitcodeGlobalEntry& entry,
+ void* addr, const char* label,
+ uint32_t depth)
+ : rt_(rt),
+ entry_(entry),
+ addr_(addr),
+ canonicalAddr_(nullptr),
+ label_(label),
+ depth_(depth) {
+ if (!canonicalAddr_) {
+ canonicalAddr_ = entry_.canonicalNativeAddrFor(rt_, addr_);
+ }
+}
+
+JS_PUBLIC_API JS::ProfilingFrameIterator::FrameKind
+JS::ProfiledFrameHandle::frameKind() const {
+ if (entry_.isBaselineInterpreter()) {
+ return JS::ProfilingFrameIterator::Frame_BaselineInterpreter;
+ }
+ if (entry_.isBaseline()) {
+ return JS::ProfilingFrameIterator::Frame_Baseline;
+ }
+ return JS::ProfilingFrameIterator::Frame_Ion;
+}
+
+JS_PUBLIC_API uint64_t JS::ProfiledFrameHandle::realmID() const {
+ return entry_.lookupRealmID(rt_, addr_);
+}
+
+JS_PUBLIC_API JS::ProfiledFrameRange JS::GetProfiledFrames(JSContext* cx,
+ void* addr) {
+ JSRuntime* rt = cx->runtime();
+ js::jit::JitcodeGlobalTable* table =
+ rt->jitRuntime()->getJitcodeGlobalTable();
+ js::jit::JitcodeGlobalEntry* entry = table->lookup(addr);
+
+ ProfiledFrameRange result(rt, addr, entry);
+
+ if (entry) {
+ result.depth_ = entry->callStackAtAddr(rt, addr, result.labels_,
+ MOZ_ARRAY_LENGTH(result.labels_));
+ }
+ return result;
+}
+
+JS::ProfiledFrameHandle JS::ProfiledFrameRange::Iter::operator*() const {
+ // The iterator iterates in high depth to low depth order. index_ goes up,
+ // and the depth we need to pass to ProfiledFrameHandle goes down.
+ uint32_t depth = range_.depth_ - 1 - index_;
+ return ProfiledFrameHandle(range_.rt_, *range_.entry_, range_.addr_,
+ range_.labels_[depth], depth);
+}
diff --git a/js/src/jit/JitcodeMap.h b/js/src/jit/JitcodeMap.h
new file mode 100644
index 0000000000..b4ed8ae7ff
--- /dev/null
+++ b/js/src/jit/JitcodeMap.h
@@ -0,0 +1,808 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitcodeMap_h
+#define jit_JitcodeMap_h
+
+#include "mozilla/Assertions.h" // MOZ_ASSERT, MOZ_ASSERT_IF, MOZ_CRASH
+
+#include <stddef.h> // size_t
+#include <stdint.h> // uint8_t, uint32_t, uint64_t
+
+#include "ds/AvlTree.h" // AvlTree
+#include "jit/CompactBuffer.h" // CompactBufferReader, CompactBufferWriter
+#include "jit/shared/Assembler-shared.h" // CodeOffset
+#include "js/AllocPolicy.h" // SystemAllocPolicy
+#include "js/TypeDecls.h" // jsbytecode
+#include "js/Vector.h" // Vector
+#include "vm/BytecodeLocation.h" // BytecodeLocation
+
+class JSScript;
+class JSTracer;
+struct JSRuntime;
+
+namespace JS {
+class Zone;
+} // namespace JS
+
+namespace js {
+
+class GCMarker;
+
+namespace jit {
+
+class InlineScriptTree;
+
+/*
+ * The jitcode map implements tables to allow mapping from addresses in jitcode
+ * to the list of (JSScript*, jsbytecode*) pairs that are implicitly active in
+ * the frame at that point in the native code.
+ *
+ * To represent this information efficiently, a multi-level table is used.
+ *
+ * At the top level, a global AVL-tree of JitcodeGlobalEntry describing the
+ * mapping for each individual JitCode generated by compiles. The entries are
+ * ordered by their nativeStartAddr.
+ *
+ * Every entry in the table is of fixed size, but there are different entry
+ * types, distinguished by the kind field.
+ */
+
+class JitcodeGlobalTable;
+class JitcodeIonTable;
+class JitcodeRegionEntry;
+
+struct NativeToBytecode {
+ CodeOffset nativeOffset;
+ InlineScriptTree* tree;
+ jsbytecode* pc;
+};
+
+// Describes range [start, end) of JIT-generated code.
+class JitCodeRange {
+ protected:
+ void* const nativeStartAddr_;
+ void* const nativeEndAddr_;
+
+ public:
+ JitCodeRange(void* start, void* end)
+ : nativeStartAddr_(start), nativeEndAddr_(end) {
+ MOZ_ASSERT(start < end);
+ }
+
+ // Comparator used by the AvlTree.
+ static int compare(const JitCodeRange* r1, const JitCodeRange* r2) {
+ // JitCodeRange includes 'start' but excludes 'end'.
+ if (r1->nativeEndAddr_ <= r2->nativeStartAddr_) {
+ return -1;
+ }
+ if (r1->nativeStartAddr_ >= r2->nativeEndAddr_) {
+ return 1;
+ }
+ return 0;
+ }
+
+ void* nativeStartAddr() const { return nativeStartAddr_; }
+ void* nativeEndAddr() const { return nativeEndAddr_; }
+
+ bool containsPointer(void* ptr) const {
+ return nativeStartAddr() <= ptr && ptr < nativeEndAddr();
+ }
+};
+
+typedef Vector<BytecodeLocation, 0, SystemAllocPolicy> BytecodeLocationVector;
+
+class IonEntry;
+class IonICEntry;
+class BaselineEntry;
+class BaselineInterpreterEntry;
+class DummyEntry;
+
+// Base class for all entries.
+class JitcodeGlobalEntry : public JitCodeRange {
+ protected:
+ JitCode* jitcode_;
+ // If this entry is referenced from the profiler buffer, this is the
+ // position where the most recent sample that references it starts.
+ // Otherwise set to kNoSampleInBuffer.
+ static const uint64_t kNoSampleInBuffer = UINT64_MAX;
+ uint64_t samplePositionInBuffer_ = kNoSampleInBuffer;
+
+ public:
+ enum class Kind : uint8_t {
+ Ion,
+ IonIC,
+ Baseline,
+ BaselineInterpreter,
+ Dummy
+ };
+
+ protected:
+ Kind kind_;
+
+ JitcodeGlobalEntry(Kind kind, JitCode* code, void* nativeStartAddr,
+ void* nativeEndAddr)
+ : JitCodeRange(nativeStartAddr, nativeEndAddr),
+ jitcode_(code),
+ kind_(kind) {
+ MOZ_ASSERT(code);
+ MOZ_ASSERT(nativeStartAddr);
+ MOZ_ASSERT(nativeEndAddr);
+ }
+
+ // Protected destructor to ensure this is called through DestroyPolicy.
+ ~JitcodeGlobalEntry() = default;
+
+ JitcodeGlobalEntry(const JitcodeGlobalEntry& other) = delete;
+ void operator=(const JitcodeGlobalEntry& other) = delete;
+
+ public:
+ struct DestroyPolicy {
+ void operator()(JitcodeGlobalEntry* entry);
+ };
+
+ void setSamplePositionInBuffer(uint64_t bufferWritePos) {
+ samplePositionInBuffer_ = bufferWritePos;
+ }
+ void setAsExpired() { samplePositionInBuffer_ = kNoSampleInBuffer; }
+ bool isSampled(uint64_t bufferRangeStart) {
+ if (samplePositionInBuffer_ == kNoSampleInBuffer) {
+ return false;
+ }
+ return bufferRangeStart <= samplePositionInBuffer_;
+ }
+
+ Kind kind() const { return kind_; }
+ bool isIon() const { return kind() == Kind::Ion; }
+ bool isIonIC() const { return kind() == Kind::IonIC; }
+ bool isBaseline() const { return kind() == Kind::Baseline; }
+ bool isBaselineInterpreter() const {
+ return kind() == Kind::BaselineInterpreter;
+ }
+ bool isDummy() const { return kind() == Kind::Dummy; }
+
+ inline const IonEntry& asIon() const;
+ inline const IonICEntry& asIonIC() const;
+ inline const BaselineEntry& asBaseline() const;
+ inline const BaselineInterpreterEntry& asBaselineInterpreter() const;
+ inline const DummyEntry& asDummy() const;
+
+ inline IonEntry& asIon();
+ inline IonICEntry& asIonIC();
+ inline BaselineEntry& asBaseline();
+ inline BaselineInterpreterEntry& asBaselineInterpreter();
+ inline DummyEntry& asDummy();
+
+ JitCode* jitcode() const { return jitcode_; }
+ JitCode** jitcodePtr() { return &jitcode_; }
+ Zone* zone() const { return jitcode()->zone(); }
+
+ bool traceJitcode(JSTracer* trc);
+ bool isJitcodeMarkedFromAnyThread(JSRuntime* rt);
+
+ bool trace(JSTracer* trc);
+ void traceWeak(JSTracer* trc);
+ uint64_t lookupRealmID(JSRuntime* rt, void* ptr) const;
+ void* canonicalNativeAddrFor(JSRuntime* rt, void* ptr) const;
+
+ // Read the inline call stack at a given point in the native code and append
+ // into the given vector. Innermost (script,pc) pair will be appended first,
+ // and outermost appended last.
+ //
+ // Returns false on memory failure.
+ [[nodiscard]] bool callStackAtAddr(JSRuntime* rt, void* ptr,
+ BytecodeLocationVector& results,
+ uint32_t* depth) const;
+ uint32_t callStackAtAddr(JSRuntime* rt, void* ptr, const char** results,
+ uint32_t maxResults) const;
+};
+
+using UniqueJitcodeGlobalEntry =
+ UniquePtr<JitcodeGlobalEntry, JitcodeGlobalEntry::DestroyPolicy>;
+
+template <typename T, typename... Args>
+inline UniqueJitcodeGlobalEntry MakeJitcodeGlobalEntry(JSContext* cx,
+ Args&&... args) {
+ UniqueJitcodeGlobalEntry res(js_new<T>(std::forward<Args>(args)...));
+ if (!res) {
+ ReportOutOfMemory(cx);
+ }
+ return res;
+}
+
+class IonEntry : public JitcodeGlobalEntry {
+ public:
+ struct ScriptNamePair {
+ JSScript* script;
+ UniqueChars str;
+ ScriptNamePair(JSScript* script, UniqueChars str)
+ : script(script), str(std::move(str)) {}
+ };
+ using ScriptList = Vector<ScriptNamePair, 2, SystemAllocPolicy>;
+
+ private:
+ ScriptList scriptList_;
+
+ // regionTable_ points to the start of the region table within the
+ // packed map for compile represented by this entry. Since the
+ // region table occurs at the tail of the memory region, this pointer
+ // points somewhere inside the region memory space, and not to the start
+ // of the memory space.
+ const JitcodeIonTable* regionTable_;
+
+ public:
+ IonEntry(JitCode* code, void* nativeStartAddr, void* nativeEndAddr,
+ ScriptList&& scriptList, JitcodeIonTable* regionTable)
+ : JitcodeGlobalEntry(Kind::Ion, code, nativeStartAddr, nativeEndAddr),
+ scriptList_(std::move(scriptList)),
+ regionTable_(regionTable) {
+ MOZ_ASSERT(regionTable);
+ }
+
+ ~IonEntry();
+
+ ScriptList& scriptList() { return scriptList_; }
+
+ size_t numScripts() const { return scriptList_.length(); }
+
+ JSScript* getScript(unsigned idx) const {
+ MOZ_ASSERT(idx < numScripts());
+ return scriptList_[idx].script;
+ }
+
+ const char* getStr(unsigned idx) const {
+ MOZ_ASSERT(idx < numScripts());
+ return scriptList_[idx].str.get();
+ }
+
+ const JitcodeIonTable* regionTable() const { return regionTable_; }
+
+ void* canonicalNativeAddrFor(void* ptr) const;
+
+ [[nodiscard]] bool callStackAtAddr(void* ptr, BytecodeLocationVector& results,
+ uint32_t* depth) const;
+
+ uint32_t callStackAtAddr(void* ptr, const char** results,
+ uint32_t maxResults) const;
+
+ uint64_t lookupRealmID(void* ptr) const;
+
+ bool trace(JSTracer* trc);
+ void traceWeak(JSTracer* trc);
+};
+
+class IonICEntry : public JitcodeGlobalEntry {
+ // Address for this IC in the IonScript code. Most operations on IonICEntry
+ // use this to forward to the IonEntry.
+ void* rejoinAddr_;
+
+ public:
+ IonICEntry(JitCode* code, void* nativeStartAddr, void* nativeEndAddr,
+ void* rejoinAddr)
+ : JitcodeGlobalEntry(Kind::IonIC, code, nativeStartAddr, nativeEndAddr),
+ rejoinAddr_(rejoinAddr) {
+ MOZ_ASSERT(rejoinAddr_);
+ }
+
+ void* rejoinAddr() const { return rejoinAddr_; }
+
+ void* canonicalNativeAddrFor(void* ptr) const;
+
+ [[nodiscard]] bool callStackAtAddr(JSRuntime* rt, void* ptr,
+ BytecodeLocationVector& results,
+ uint32_t* depth) const;
+
+ uint32_t callStackAtAddr(JSRuntime* rt, void* ptr, const char** results,
+ uint32_t maxResults) const;
+
+ uint64_t lookupRealmID(JSRuntime* rt, void* ptr) const;
+
+ bool trace(JSTracer* trc);
+ void traceWeak(JSTracer* trc);
+};
+
+class BaselineEntry : public JitcodeGlobalEntry {
+ JSScript* script_;
+ UniqueChars str_;
+
+ public:
+ BaselineEntry(JitCode* code, void* nativeStartAddr, void* nativeEndAddr,
+ JSScript* script, UniqueChars str)
+ : JitcodeGlobalEntry(Kind::Baseline, code, nativeStartAddr,
+ nativeEndAddr),
+ script_(script),
+ str_(std::move(str)) {
+ MOZ_ASSERT(script_);
+ MOZ_ASSERT(str_);
+ }
+
+ JSScript* script() const { return script_; }
+
+ const char* str() const { return str_.get(); }
+
+ void* canonicalNativeAddrFor(void* ptr) const;
+
+ [[nodiscard]] bool callStackAtAddr(void* ptr, BytecodeLocationVector& results,
+ uint32_t* depth) const;
+
+ uint32_t callStackAtAddr(void* ptr, const char** results,
+ uint32_t maxResults) const;
+
+ uint64_t lookupRealmID() const;
+
+ bool trace(JSTracer* trc);
+ void traceWeak(JSTracer* trc);
+};
+
+class BaselineInterpreterEntry : public JitcodeGlobalEntry {
+ public:
+ BaselineInterpreterEntry(JitCode* code, void* nativeStartAddr,
+ void* nativeEndAddr)
+ : JitcodeGlobalEntry(Kind::BaselineInterpreter, code, nativeStartAddr,
+ nativeEndAddr) {}
+
+ void* canonicalNativeAddrFor(void* ptr) const;
+
+ [[nodiscard]] bool callStackAtAddr(void* ptr, BytecodeLocationVector& results,
+ uint32_t* depth) const;
+
+ uint32_t callStackAtAddr(void* ptr, const char** results,
+ uint32_t maxResults) const;
+
+ uint64_t lookupRealmID() const;
+};
+
+// Dummy entries are created for jitcode generated when profiling is not
+// turned on, so that they have representation in the global table if they are
+// on the stack when profiling is enabled.
+class DummyEntry : public JitcodeGlobalEntry {
+ public:
+ DummyEntry(JitCode* code, void* nativeStartAddr, void* nativeEndAddr)
+ : JitcodeGlobalEntry(Kind::Dummy, code, nativeStartAddr, nativeEndAddr) {}
+
+ void* canonicalNativeAddrFor(JSRuntime* rt, void* ptr) const {
+ return nullptr;
+ }
+
+ [[nodiscard]] bool callStackAtAddr(JSRuntime* rt, void* ptr,
+ BytecodeLocationVector& results,
+ uint32_t* depth) const {
+ return true;
+ }
+
+ uint32_t callStackAtAddr(JSRuntime* rt, void* ptr, const char** results,
+ uint32_t maxResults) const {
+ return 0;
+ }
+
+ uint64_t lookupRealmID() const { return 0; }
+};
+
+inline const IonEntry& JitcodeGlobalEntry::asIon() const {
+ MOZ_ASSERT(isIon());
+ return *static_cast<const IonEntry*>(this);
+}
+
+inline const IonICEntry& JitcodeGlobalEntry::asIonIC() const {
+ MOZ_ASSERT(isIonIC());
+ return *static_cast<const IonICEntry*>(this);
+}
+
+inline const BaselineEntry& JitcodeGlobalEntry::asBaseline() const {
+ MOZ_ASSERT(isBaseline());
+ return *static_cast<const BaselineEntry*>(this);
+}
+
+inline const BaselineInterpreterEntry&
+JitcodeGlobalEntry::asBaselineInterpreter() const {
+ MOZ_ASSERT(isBaselineInterpreter());
+ return *static_cast<const BaselineInterpreterEntry*>(this);
+}
+
+inline const DummyEntry& JitcodeGlobalEntry::asDummy() const {
+ MOZ_ASSERT(isDummy());
+ return *static_cast<const DummyEntry*>(this);
+}
+
+inline IonEntry& JitcodeGlobalEntry::asIon() {
+ MOZ_ASSERT(isIon());
+ return *static_cast<IonEntry*>(this);
+}
+
+inline IonICEntry& JitcodeGlobalEntry::asIonIC() {
+ MOZ_ASSERT(isIonIC());
+ return *static_cast<IonICEntry*>(this);
+}
+
+inline BaselineEntry& JitcodeGlobalEntry::asBaseline() {
+ MOZ_ASSERT(isBaseline());
+ return *static_cast<BaselineEntry*>(this);
+}
+
+inline BaselineInterpreterEntry& JitcodeGlobalEntry::asBaselineInterpreter() {
+ MOZ_ASSERT(isBaselineInterpreter());
+ return *static_cast<BaselineInterpreterEntry*>(this);
+}
+
+inline DummyEntry& JitcodeGlobalEntry::asDummy() {
+ MOZ_ASSERT(isDummy());
+ return *static_cast<DummyEntry*>(this);
+}
+
+// Global table of JitcodeGlobalEntry entries.
+class JitcodeGlobalTable {
+ private:
+ // Vector containing (and owning) all entries. This is unsorted and used for
+ // iterating over all entries, because the AvlTree currently doesn't support
+ // modifications while iterating.
+ using EntryVector = Vector<UniqueJitcodeGlobalEntry, 0, SystemAllocPolicy>;
+ EntryVector entries_;
+
+ // AVL tree containing all entries in the Vector above. This is used to
+ // efficiently look up the entry corresponding to a native code address.
+ using EntryTree = AvlTree<JitCodeRange*, JitCodeRange>;
+ static const size_t LIFO_CHUNK_SIZE = 16 * 1024;
+ LifoAlloc alloc_;
+ EntryTree tree_;
+
+ public:
+ JitcodeGlobalTable() : alloc_(LIFO_CHUNK_SIZE), tree_(&alloc_) {}
+
+ bool empty() const {
+ MOZ_ASSERT(entries_.empty() == tree_.empty());
+ return entries_.empty();
+ }
+
+ JitcodeGlobalEntry* lookup(void* ptr) { return lookupInternal(ptr); }
+
+ const JitcodeGlobalEntry* lookupForSampler(void* ptr, JSRuntime* rt,
+ uint64_t samplePosInBuffer);
+
+ [[nodiscard]] bool addEntry(UniqueJitcodeGlobalEntry entry);
+
+ void setAllEntriesAsExpired();
+ [[nodiscard]] bool markIteratively(GCMarker* marker);
+ void traceWeak(JSRuntime* rt, JSTracer* trc);
+
+ private:
+ JitcodeGlobalEntry* lookupInternal(void* ptr);
+};
+
+// clang-format off
+/*
+ * Container class for main jitcode table.
+ * The Region table's memory is structured as follows:
+ *
+ * +------------------------------------------------+ |
+ * | Region 1 Run | |
+ * |------------------------------------------------| |
+ * | Region 2 Run | |
+ * | | |
+ * | | |
+ * |------------------------------------------------| |
+ * | Region 3 Run | |
+ * | | |
+ * |------------------------------------------------| |-- Payload
+ * | | |
+ * | ... | |
+ * | | |
+ * |------------------------------------------------| |
+ * | Region M Run | |
+ * | | |
+ * +================================================+ <- RegionTable pointer points here
+ * | uint23_t numRegions = M | |
+ * +------------------------------------------------+ |
+ * | Region 1 | |
+ * | uint32_t entryOffset = size(Payload) | |
+ * +------------------------------------------------+ |
+ * | | |-- Table
+ * | ... | |
+ * | | |
+ * +------------------------------------------------+ |
+ * | Region M | |
+ * | uint32_t entryOffset | |
+ * +------------------------------------------------+ |
+ *
+ * The region table is composed of two sections: a tail section that contains a table of
+ * fixed-size entries containing offsets into the the head section, and a head section that
+ * holds a sequence of variable-sized runs. The table in the tail section serves to
+ * locate the variable-length encoded structures in the head section.
+ *
+ * The entryOffsets in the table indicate the bytes offset to subtract from the regionTable
+ * pointer to arrive at the encoded region in the payload.
+ *
+ *
+ * Variable-length entries in payload
+ * ----------------------------------
+ * The entryOffsets in the region table's fixed-sized entries refer to a location within the
+ * variable-length payload section. This location contains a compactly encoded "run" of
+ * mappings.
+ *
+ * Each run starts by describing the offset within the native code it starts at, and the
+ * sequence of (JSScript*, jsbytecode*) pairs active at that site. Following that, there
+ * are a number of variable-length entries encoding (nativeOffsetDelta, bytecodeOffsetDelta)
+ * pairs for the run.
+ *
+ * VarUint32 nativeOffset;
+ * - The offset from nativeStartAddr in the global table entry at which
+ * the jitcode for this region starts.
+ *
+ * Uint8_t scriptDepth;
+ * - The depth of inlined scripts for this region.
+ *
+ * List<VarUint32> inlineScriptPcStack;
+ * - We encode (2 * scriptDepth) VarUint32s here. Each pair of uint32s are taken
+ * as an index into the scriptList in the global table entry, and a pcOffset
+ * respectively.
+ *
+ * List<NativeAndBytecodeDelta> deltaRun;
+ * - The rest of the entry is a deltaRun that stores a series of variable-length
+ * encoded NativeAndBytecodeDelta datums.
+ */
+// clang-format on
+class JitcodeRegionEntry {
+ private:
+ static const unsigned MAX_RUN_LENGTH = 100;
+
+ public:
+ static void WriteHead(CompactBufferWriter& writer, uint32_t nativeOffset,
+ uint8_t scriptDepth);
+ static void ReadHead(CompactBufferReader& reader, uint32_t* nativeOffset,
+ uint8_t* scriptDepth);
+
+ static void WriteScriptPc(CompactBufferWriter& writer, uint32_t scriptIdx,
+ uint32_t pcOffset);
+ static void ReadScriptPc(CompactBufferReader& reader, uint32_t* scriptIdx,
+ uint32_t* pcOffset);
+
+ static void WriteDelta(CompactBufferWriter& writer, uint32_t nativeDelta,
+ int32_t pcDelta);
+ static void ReadDelta(CompactBufferReader& reader, uint32_t* nativeDelta,
+ int32_t* pcDelta);
+
+ // Given a pointer into an array of NativeToBytecode (and a pointer to the end
+ // of the array), compute the number of entries that would be consume by
+ // outputting a run starting at this one.
+ static uint32_t ExpectedRunLength(const NativeToBytecode* entry,
+ const NativeToBytecode* end);
+
+ // Write a run, starting at the given NativeToBytecode entry, into the given
+ // buffer writer.
+ [[nodiscard]] static bool WriteRun(CompactBufferWriter& writer,
+ const IonEntry::ScriptList& scriptList,
+ uint32_t runLength,
+ const NativeToBytecode* entry);
+
+ // Delta Run entry formats are encoded little-endian:
+ //
+ // byte 0
+ // NNNN-BBB0
+ // Single byte format. nativeDelta in [0, 15], pcDelta in [0, 7]
+ //
+ static const uint32_t ENC1_MASK = 0x1;
+ static const uint32_t ENC1_MASK_VAL = 0x0;
+
+ static const uint32_t ENC1_NATIVE_DELTA_MAX = 0xf;
+ static const unsigned ENC1_NATIVE_DELTA_SHIFT = 4;
+
+ static const uint32_t ENC1_PC_DELTA_MASK = 0x0e;
+ static const int32_t ENC1_PC_DELTA_MAX = 0x7;
+ static const unsigned ENC1_PC_DELTA_SHIFT = 1;
+
+ // byte 1 byte 0
+ // NNNN-NNNN BBBB-BB01
+ // Two-byte format. nativeDelta in [0, 255], pcDelta in [0, 63]
+ //
+ static const uint32_t ENC2_MASK = 0x3;
+ static const uint32_t ENC2_MASK_VAL = 0x1;
+
+ static const uint32_t ENC2_NATIVE_DELTA_MAX = 0xff;
+ static const unsigned ENC2_NATIVE_DELTA_SHIFT = 8;
+
+ static const uint32_t ENC2_PC_DELTA_MASK = 0x00fc;
+ static const int32_t ENC2_PC_DELTA_MAX = 0x3f;
+ static const unsigned ENC2_PC_DELTA_SHIFT = 2;
+
+ // byte 2 byte 1 byte 0
+ // NNNN-NNNN NNNB-BBBB BBBB-B011
+ // Three-byte format. nativeDelta in [0, 2047], pcDelta in [-512, 511]
+ //
+ static const uint32_t ENC3_MASK = 0x7;
+ static const uint32_t ENC3_MASK_VAL = 0x3;
+
+ static const uint32_t ENC3_NATIVE_DELTA_MAX = 0x7ff;
+ static const unsigned ENC3_NATIVE_DELTA_SHIFT = 13;
+
+ static const uint32_t ENC3_PC_DELTA_MASK = 0x001ff8;
+ static const int32_t ENC3_PC_DELTA_MAX = 0x1ff;
+ static const int32_t ENC3_PC_DELTA_MIN = -ENC3_PC_DELTA_MAX - 1;
+ static const unsigned ENC3_PC_DELTA_SHIFT = 3;
+
+ // byte 3 byte 2 byte 1 byte 0
+ // NNNN-NNNN NNNN-NNNN BBBB-BBBB BBBB-B111
+ // Three-byte format. nativeDelta in [0, 65535],
+ // pcDelta in [-4096, 4095]
+ static const uint32_t ENC4_MASK = 0x7;
+ static const uint32_t ENC4_MASK_VAL = 0x7;
+
+ static const uint32_t ENC4_NATIVE_DELTA_MAX = 0xffff;
+ static const unsigned ENC4_NATIVE_DELTA_SHIFT = 16;
+
+ static const uint32_t ENC4_PC_DELTA_MASK = 0x0000fff8;
+ static const int32_t ENC4_PC_DELTA_MAX = 0xfff;
+ static const int32_t ENC4_PC_DELTA_MIN = -ENC4_PC_DELTA_MAX - 1;
+ static const unsigned ENC4_PC_DELTA_SHIFT = 3;
+
+ static bool IsDeltaEncodeable(uint32_t nativeDelta, int32_t pcDelta) {
+ return (nativeDelta <= ENC4_NATIVE_DELTA_MAX) &&
+ (pcDelta >= ENC4_PC_DELTA_MIN) && (pcDelta <= ENC4_PC_DELTA_MAX);
+ }
+
+ private:
+ const uint8_t* data_;
+ const uint8_t* end_;
+
+ // Unpacked state from jitcode entry.
+ uint32_t nativeOffset_;
+ uint8_t scriptDepth_;
+ const uint8_t* scriptPcStack_;
+ const uint8_t* deltaRun_;
+
+ void unpack();
+
+ public:
+ JitcodeRegionEntry(const uint8_t* data, const uint8_t* end)
+ : data_(data),
+ end_(end),
+ nativeOffset_(0),
+ scriptDepth_(0),
+ scriptPcStack_(nullptr),
+ deltaRun_(nullptr) {
+ MOZ_ASSERT(data_ < end_);
+ unpack();
+ MOZ_ASSERT(scriptPcStack_ < end_);
+ MOZ_ASSERT(deltaRun_ <= end_);
+ }
+
+ uint32_t nativeOffset() const { return nativeOffset_; }
+ uint32_t scriptDepth() const { return scriptDepth_; }
+
+ class ScriptPcIterator {
+ private:
+ const uint8_t* start_;
+ const uint8_t* end_;
+#ifdef DEBUG
+ uint32_t count_;
+#endif
+ uint32_t idx_;
+ const uint8_t* cur_;
+
+ public:
+ ScriptPcIterator(const uint8_t* start, const uint8_t* end, uint32_t count)
+ : start_(start),
+ end_(end),
+#ifdef DEBUG
+ count_(count),
+#endif
+ idx_(0),
+ cur_(start_) {
+ }
+
+ bool hasMore() const {
+ MOZ_ASSERT((idx_ == count_) == (cur_ == end_));
+ MOZ_ASSERT((idx_ < count_) == (cur_ < end_));
+ return cur_ < end_;
+ }
+
+ void readNext(uint32_t* scriptIdxOut, uint32_t* pcOffsetOut) {
+ MOZ_ASSERT(scriptIdxOut);
+ MOZ_ASSERT(pcOffsetOut);
+ MOZ_ASSERT(hasMore());
+
+ CompactBufferReader reader(cur_, end_);
+ ReadScriptPc(reader, scriptIdxOut, pcOffsetOut);
+
+ cur_ = reader.currentPosition();
+ MOZ_ASSERT(cur_ <= end_);
+
+ idx_++;
+ MOZ_ASSERT_IF(idx_ == count_, cur_ == end_);
+ }
+
+ void reset() {
+ idx_ = 0;
+ cur_ = start_;
+ }
+ };
+
+ ScriptPcIterator scriptPcIterator() const {
+ // End of script+pc sequence is the start of the delta run.
+ return ScriptPcIterator(scriptPcStack_, deltaRun_, scriptDepth_);
+ }
+
+ class DeltaIterator {
+ private:
+ const uint8_t* start_;
+ const uint8_t* end_;
+ const uint8_t* cur_;
+
+ public:
+ DeltaIterator(const uint8_t* start, const uint8_t* end)
+ : start_(start), end_(end), cur_(start) {}
+
+ bool hasMore() const {
+ MOZ_ASSERT(cur_ <= end_);
+ return cur_ < end_;
+ }
+
+ void readNext(uint32_t* nativeDeltaOut, int32_t* pcDeltaOut) {
+ MOZ_ASSERT(nativeDeltaOut != nullptr);
+ MOZ_ASSERT(pcDeltaOut != nullptr);
+
+ MOZ_ASSERT(hasMore());
+
+ CompactBufferReader reader(cur_, end_);
+ ReadDelta(reader, nativeDeltaOut, pcDeltaOut);
+
+ cur_ = reader.currentPosition();
+ MOZ_ASSERT(cur_ <= end_);
+ }
+
+ void reset() { cur_ = start_; }
+ };
+ DeltaIterator deltaIterator() const { return DeltaIterator(deltaRun_, end_); }
+
+ uint32_t findPcOffset(uint32_t queryNativeOffset,
+ uint32_t startPcOffset) const;
+};
+
+class JitcodeIonTable {
+ private:
+ /* Variable length payload section "below" here. */
+ uint32_t numRegions_;
+ uint32_t regionOffsets_[1];
+
+ const uint8_t* payloadEnd() const {
+ return reinterpret_cast<const uint8_t*>(this);
+ }
+
+ public:
+ JitcodeIonTable() = delete;
+
+ uint32_t numRegions() const { return numRegions_; }
+
+ uint32_t regionOffset(uint32_t regionIndex) const {
+ MOZ_ASSERT(regionIndex < numRegions());
+ return regionOffsets_[regionIndex];
+ }
+
+ JitcodeRegionEntry regionEntry(uint32_t regionIndex) const {
+ const uint8_t* regionStart = payloadEnd() - regionOffset(regionIndex);
+ const uint8_t* regionEnd = payloadEnd();
+ if (regionIndex < numRegions_ - 1) {
+ regionEnd -= regionOffset(regionIndex + 1);
+ }
+ return JitcodeRegionEntry(regionStart, regionEnd);
+ }
+
+ uint32_t findRegionEntry(uint32_t offset) const;
+
+ const uint8_t* payloadStart() const {
+ // The beginning of the payload the beginning of the first region are the
+ // same.
+ return payloadEnd() - regionOffset(0);
+ }
+
+ [[nodiscard]] static bool WriteIonTable(
+ CompactBufferWriter& writer, const IonEntry::ScriptList& scriptList,
+ const NativeToBytecode* start, const NativeToBytecode* end,
+ uint32_t* tableOffsetOut, uint32_t* numRegionsOut);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitcodeMap_h */
diff --git a/js/src/jit/Jitdump.h b/js/src/jit/Jitdump.h
new file mode 100644
index 0000000000..a1d61bd69e
--- /dev/null
+++ b/js/src/jit/Jitdump.h
@@ -0,0 +1,78 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_JitDump_h
+#define jit_JitDump_h
+
+/*
+ This file provides the necessary data structures to meet the JitDump
+ specification as of
+ https://github.com/torvalds/linux/blob/f2906aa863381afb0015a9eb7fefad885d4e5a56/tools/perf/Documentation/jitdump-specification.txt
+*/
+
+namespace js {
+namespace jit {
+
+// JitDump record types
+enum {
+ JIT_CODE_LOAD = 0,
+ JIT_CODE_MOVE,
+ JIT_CODE_DEBUG_INFO,
+ JIT_CODE_CLOSE,
+ JIT_CODE_UNWINDING_INFO
+};
+
+// File header
+struct JitDumpHeader {
+ uint32_t magic;
+ uint32_t version;
+ uint32_t total_size;
+ uint32_t elf_mach;
+ uint32_t pad1;
+ uint32_t pid;
+ uint64_t timestamp;
+ uint64_t flags;
+};
+
+// Header for each record
+struct JitDumpRecordHeader {
+ uint32_t id;
+ uint32_t total_size;
+ uint64_t timestamp;
+};
+
+// Load record
+struct JitDumpLoadRecord {
+ JitDumpRecordHeader header;
+
+ // Payload
+ uint32_t pid;
+ uint32_t tid;
+ uint64_t vma;
+ uint64_t code_addr;
+ uint64_t code_size;
+ uint64_t code_index;
+};
+
+// Debug record
+struct JitDumpDebugRecord {
+ JitDumpRecordHeader header;
+
+ // Debug header
+ uint64_t code_addr;
+ uint64_t nr_entry;
+};
+
+struct JitDumpDebugEntry {
+ uint64_t code_addr;
+ uint32_t line;
+ uint32_t discrim;
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_JitDump_h */
diff --git a/js/src/jit/KnownClass.cpp b/js/src/jit/KnownClass.cpp
new file mode 100644
index 0000000000..fcb7715f5f
--- /dev/null
+++ b/js/src/jit/KnownClass.cpp
@@ -0,0 +1,109 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/KnownClass.h"
+
+#include "jit/MIR.h"
+#include "vm/ArrayObject.h"
+#include "vm/Iteration.h"
+#include "vm/JSFunction.h"
+#include "vm/PlainObject.h" // js::PlainObject
+#include "vm/RegExpObject.h"
+
+using namespace js;
+using namespace js::jit;
+
+KnownClass jit::GetObjectKnownClass(const MDefinition* def) {
+ MOZ_ASSERT(def->type() == MIRType::Object);
+
+ switch (def->op()) {
+ case MDefinition::Opcode::NewArray:
+ case MDefinition::Opcode::NewArrayDynamicLength:
+ case MDefinition::Opcode::NewArrayObject:
+ case MDefinition::Opcode::Rest:
+ case MDefinition::Opcode::ArgumentsSlice:
+ case MDefinition::Opcode::FrameArgumentsSlice:
+ case MDefinition::Opcode::InlineArgumentsSlice:
+ return KnownClass::Array;
+
+ case MDefinition::Opcode::NewObject:
+ case MDefinition::Opcode::NewPlainObject:
+ case MDefinition::Opcode::CreateThis:
+ return KnownClass::PlainObject;
+
+ case MDefinition::Opcode::Lambda:
+ case MDefinition::Opcode::FunctionWithProto:
+ return KnownClass::Function;
+
+ case MDefinition::Opcode::RegExp:
+ return KnownClass::RegExp;
+
+ case MDefinition::Opcode::NewIterator:
+ switch (def->toNewIterator()->type()) {
+ case MNewIterator::ArrayIterator:
+ return KnownClass::ArrayIterator;
+ case MNewIterator::StringIterator:
+ return KnownClass::StringIterator;
+ case MNewIterator::RegExpStringIterator:
+ return KnownClass::RegExpStringIterator;
+ }
+ MOZ_CRASH("unreachable");
+
+ case MDefinition::Opcode::Phi: {
+ if (def->numOperands() == 0) {
+ return KnownClass::None;
+ }
+
+ MDefinition* op = def->getOperand(0);
+ // Check for Phis to avoid recursion for now.
+ if (op->isPhi()) {
+ return KnownClass::None;
+ }
+
+ KnownClass known = GetObjectKnownClass(op);
+ if (known == KnownClass::None) {
+ return KnownClass::None;
+ }
+
+ for (size_t i = 1; i < def->numOperands(); i++) {
+ op = def->getOperand(i);
+ if (op->isPhi() || GetObjectKnownClass(op) != known) {
+ return KnownClass::None;
+ }
+ }
+
+ return known;
+ }
+
+ default:
+ break;
+ }
+
+ return KnownClass::None;
+}
+
+const JSClass* jit::GetObjectKnownJSClass(const MDefinition* def) {
+ switch (GetObjectKnownClass(def)) {
+ case KnownClass::PlainObject:
+ return &PlainObject::class_;
+ case KnownClass::Array:
+ return &ArrayObject::class_;
+ case KnownClass::Function:
+ return &FunctionClass;
+ case KnownClass::RegExp:
+ return &RegExpObject::class_;
+ case KnownClass::ArrayIterator:
+ return &ArrayIteratorObject::class_;
+ case KnownClass::StringIterator:
+ return &StringIteratorObject::class_;
+ case KnownClass::RegExpStringIterator:
+ return &RegExpStringIteratorObject::class_;
+ case KnownClass::None:
+ break;
+ }
+
+ return nullptr;
+}
diff --git a/js/src/jit/KnownClass.h b/js/src/jit/KnownClass.h
new file mode 100644
index 0000000000..862c0f1cb8
--- /dev/null
+++ b/js/src/jit/KnownClass.h
@@ -0,0 +1,36 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_KnownClass_h
+#define jit_KnownClass_h
+
+#include "jspubtd.h"
+
+namespace js {
+namespace jit {
+
+class MDefinition;
+
+// Users of this enum often can't handle Proxy and Wrapper classes,
+// as well as non-Function callables.
+enum class KnownClass {
+ PlainObject,
+ Array,
+ Function,
+ RegExp,
+ ArrayIterator,
+ StringIterator,
+ RegExpStringIterator,
+ None
+};
+
+KnownClass GetObjectKnownClass(const MDefinition* def);
+const JSClass* GetObjectKnownJSClass(const MDefinition* def);
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_KnownClass_h
diff --git a/js/src/jit/LICM.cpp b/js/src/jit/LICM.cpp
new file mode 100644
index 0000000000..ba00199fc7
--- /dev/null
+++ b/js/src/jit/LICM.cpp
@@ -0,0 +1,367 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/LICM.h"
+
+#include "jit/IonAnalysis.h"
+#include "jit/JitSpewer.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+using namespace js;
+using namespace js::jit;
+
+// There are two constants which control whether a loop is LICM'd or is left
+// unchanged. For rationale see comment in jit::LICM() below.
+//
+// A bit of quick profiling with the wasm Embenchen suite on x64 shows that
+// the threshold pair (100,25) has either no effect or gives a small net
+// reduction in memory traffic, compared to unconstrained LICMing. Halving
+// them to (50,12) gives a small overall increase in memory traffic,
+// suggesting it excludes too many loops from LICM. Doubling them to (200,50)
+// gives a win that is even smaller than (100,25), hence (100,25) seems the
+// best choice.
+//
+// If a loop has more than this number of basic blocks in its body, it won't
+// be LICM'd.
+static constexpr size_t LargestAllowedLoop = 100;
+
+// If a loop contains an MTableSwitch instruction that has more than this many
+// successors, it won't be LICM'd.
+static constexpr size_t LargestAllowedTableSwitch = 25;
+
+// Test whether any instruction in the loop possiblyCalls().
+static bool LoopContainsPossibleCall(MIRGraph& graph, MBasicBlock* header,
+ MBasicBlock* backedge) {
+ for (auto i(graph.rpoBegin(header));; ++i) {
+ MOZ_ASSERT(i != graph.rpoEnd(),
+ "Reached end of graph searching for blocks in loop");
+ MBasicBlock* block = *i;
+ if (!block->isMarked()) {
+ continue;
+ }
+
+ for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd;
+ ++insIter) {
+ MInstruction* ins = *insIter;
+ if (ins->possiblyCalls()) {
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_LICM, " Possible call found at %s%u", ins->opName(),
+ ins->id());
+#endif
+ return true;
+ }
+ }
+
+ if (block == backedge) {
+ break;
+ }
+ }
+ return false;
+}
+
+// Tests whether any instruction in the loop is a table-switch with more than
+// `LargestAllowedTableSwitch` successors. If it returns true, it also
+// returns the actual number of successors of the instruction in question,
+// although that is used only for statistics/debug printing.
+static bool LoopContainsBigTableSwitch(MIRGraph& graph, MBasicBlock* header,
+ /*OUT*/ size_t* numSuccessors) {
+ MBasicBlock* backedge = header->backedge();
+
+ for (auto i(graph.rpoBegin(header));; ++i) {
+ MOZ_ASSERT(i != graph.rpoEnd(),
+ "Reached end of graph searching for blocks in loop");
+ MBasicBlock* block = *i;
+ if (!block->isMarked()) {
+ continue;
+ }
+
+ for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd;
+ ++insIter) {
+ MInstruction* ins = *insIter;
+ if (ins->isTableSwitch() &&
+ ins->toTableSwitch()->numSuccessors() > LargestAllowedTableSwitch) {
+ *numSuccessors = ins->toTableSwitch()->numSuccessors();
+ return true;
+ }
+ }
+
+ if (block == backedge) {
+ break;
+ }
+ }
+ return false;
+}
+
+// When a nested loop has no exits back into what would be its parent loop,
+// MarkLoopBlocks on the parent loop doesn't mark the blocks of the nested
+// loop, since they technically aren't part of the loop. However, AliasAnalysis
+// currently does consider such nested loops to be part of their parent
+// loops. Consequently, we can't use IsInLoop on dependency() values; we must
+// test whether a dependency() is *before* the loop, even if it is not
+// technically in the loop.
+static bool IsBeforeLoop(MDefinition* ins, MBasicBlock* header) {
+ return ins->block()->id() < header->id();
+}
+
+// Test whether the given instruction is inside the loop (and thus not
+// loop-invariant).
+static bool IsInLoop(MDefinition* ins) { return ins->block()->isMarked(); }
+
+// Test whether the given instruction is cheap and not worth hoisting unless
+// one of its users will be hoisted as well.
+static bool RequiresHoistedUse(const MDefinition* ins, bool hasCalls) {
+ if (ins->isBox()) {
+ MOZ_ASSERT(!ins->toBox()->input()->isBox(),
+ "Box of a box could lead to unbounded recursion");
+ return true;
+ }
+
+ // Integer constants are usually cheap and aren't worth hoisting on their
+ // own, in general. Floating-point constants typically are worth hoisting,
+ // unless they'll end up being spilled (eg. due to a call).
+ if (ins->isConstant() && (!IsFloatingPointType(ins->type()) || hasCalls)) {
+ return true;
+ }
+
+ return false;
+}
+
+// Test whether the given instruction has any operands defined within the loop.
+static bool HasOperandInLoop(MInstruction* ins, bool hasCalls) {
+ // An instruction is only loop invariant if it and all of its operands can
+ // be safely hoisted into the loop preheader.
+ for (size_t i = 0, e = ins->numOperands(); i != e; ++i) {
+ MDefinition* op = ins->getOperand(i);
+
+ if (!IsInLoop(op)) {
+ continue;
+ }
+
+ if (RequiresHoistedUse(op, hasCalls)) {
+ // Recursively test for loop invariance. Note that the recursion is
+ // bounded because we require RequiresHoistedUse to be set at each
+ // level.
+ if (!HasOperandInLoop(op->toInstruction(), hasCalls)) {
+ continue;
+ }
+ }
+
+ return true;
+ }
+ return false;
+}
+
+// Test whether the given instruction is hoistable, ignoring memory
+// dependencies.
+static bool IsHoistableIgnoringDependency(MInstruction* ins, bool hasCalls) {
+ return ins->isMovable() && !ins->isEffectful() &&
+ !HasOperandInLoop(ins, hasCalls);
+}
+
+// Test whether the given instruction has a memory dependency inside the loop.
+static bool HasDependencyInLoop(MInstruction* ins, MBasicBlock* header) {
+ // Don't hoist if this instruction depends on a store inside the loop.
+ if (MDefinition* dep = ins->dependency()) {
+ return !IsBeforeLoop(dep, header);
+ }
+ return false;
+}
+
+// Test whether the given instruction is hoistable.
+static bool IsHoistable(MInstruction* ins, MBasicBlock* header, bool hasCalls) {
+ return IsHoistableIgnoringDependency(ins, hasCalls) &&
+ !HasDependencyInLoop(ins, header);
+}
+
+// In preparation for hoisting an instruction, hoist any of its operands which
+// were too cheap to hoist on their own.
+static void MoveDeferredOperands(MInstruction* ins, MInstruction* hoistPoint,
+ bool hasCalls) {
+ // If any of our operands were waiting for a user to be hoisted, make a note
+ // to hoist them.
+ for (size_t i = 0, e = ins->numOperands(); i != e; ++i) {
+ MDefinition* op = ins->getOperand(i);
+ if (!IsInLoop(op)) {
+ continue;
+ }
+ MOZ_ASSERT(RequiresHoistedUse(op, hasCalls),
+ "Deferred loop-invariant operand is not cheap");
+ MInstruction* opIns = op->toInstruction();
+
+ // Recursively move the operands. Note that the recursion is bounded
+ // because we require RequiresHoistedUse to be set at each level.
+ MoveDeferredOperands(opIns, hoistPoint, hasCalls);
+
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_LICM,
+ " Hoisting %s%u (now that a user will be hoisted)",
+ opIns->opName(), opIns->id());
+#endif
+
+ opIns->block()->moveBefore(hoistPoint, opIns);
+ opIns->setBailoutKind(BailoutKind::LICM);
+ }
+}
+
+static void VisitLoopBlock(MBasicBlock* block, MBasicBlock* header,
+ MInstruction* hoistPoint, bool hasCalls) {
+ for (auto insIter(block->begin()), insEnd(block->end()); insIter != insEnd;) {
+ MInstruction* ins = *insIter++;
+
+ if (!IsHoistable(ins, header, hasCalls)) {
+#ifdef JS_JITSPEW
+ if (IsHoistableIgnoringDependency(ins, hasCalls)) {
+ JitSpew(JitSpew_LICM,
+ " %s%u isn't hoistable due to dependency on %s%u",
+ ins->opName(), ins->id(), ins->dependency()->opName(),
+ ins->dependency()->id());
+ }
+#endif
+ continue;
+ }
+
+ // Don't hoist a cheap constant if it doesn't enable us to hoist one of
+ // its uses. We want those instructions as close as possible to their
+ // use, to minimize register pressure.
+ if (RequiresHoistedUse(ins, hasCalls)) {
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_LICM, " %s%u will be hoisted only if its users are",
+ ins->opName(), ins->id());
+#endif
+ continue;
+ }
+
+ // Hoist operands which were too cheap to hoist on their own.
+ MoveDeferredOperands(ins, hoistPoint, hasCalls);
+
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_LICM, " Hoisting %s%u", ins->opName(), ins->id());
+#endif
+
+ // Move the instruction to the hoistPoint.
+ block->moveBefore(hoistPoint, ins);
+ ins->setBailoutKind(BailoutKind::LICM);
+ }
+}
+
+static void VisitLoop(MIRGraph& graph, MBasicBlock* header) {
+ MInstruction* hoistPoint = header->loopPredecessor()->lastIns();
+
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_LICM, " Visiting loop with header block%u, hoisting to %s%u",
+ header->id(), hoistPoint->opName(), hoistPoint->id());
+#endif
+
+ MBasicBlock* backedge = header->backedge();
+
+ // This indicates whether the loop contains calls or other things which
+ // clobber most or all floating-point registers. In such loops,
+ // floating-point constants should not be hoisted unless it enables further
+ // hoisting.
+ bool hasCalls = LoopContainsPossibleCall(graph, header, backedge);
+
+ for (auto i(graph.rpoBegin(header));; ++i) {
+ MOZ_ASSERT(i != graph.rpoEnd(),
+ "Reached end of graph searching for blocks in loop");
+ MBasicBlock* block = *i;
+ if (!block->isMarked()) {
+ continue;
+ }
+
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_LICM, " Visiting block%u", block->id());
+#endif
+
+ VisitLoopBlock(block, header, hoistPoint, hasCalls);
+
+ if (block == backedge) {
+ break;
+ }
+ }
+}
+
+bool jit::LICM(MIRGenerator* mir, MIRGraph& graph) {
+ JitSpew(JitSpew_LICM, "Beginning LICM pass");
+
+ // Iterate in RPO to visit outer loops before inner loops. We'd hoist the
+ // same things either way, but outer first means we do a little less work.
+ for (auto i(graph.rpoBegin()), e(graph.rpoEnd()); i != e; ++i) {
+ MBasicBlock* header = *i;
+ if (!header->isLoopHeader()) {
+ continue;
+ }
+
+ bool canOsr;
+ size_t numBlocks = MarkLoopBlocks(graph, header, &canOsr);
+
+ if (numBlocks == 0) {
+ JitSpew(JitSpew_LICM,
+ " Skipping loop with header block%u -- contains zero blocks",
+ header->id());
+ continue;
+ }
+
+ // There are various reasons why we might choose not to LICM a given loop:
+ //
+ // (a) Hoisting out of a loop that has an entry from the OSR block in
+ // addition to its normal entry is tricky. In theory we could clone
+ // the instruction and insert phis. In practice we don't bother.
+ //
+ // (b) If the loop contains a large number of blocks, we play safe and
+ // punt, in order to reduce the risk of creating excessive register
+ // pressure by hoisting lots of values out of the loop. In a larger
+ // loop there's more likely to be duplication of invariant expressions
+ // within the loop body, and that duplication will be GVN'd but only
+ // within the scope of the loop body, so there's less loss from not
+ // lifting them out of the loop entirely.
+ //
+ // (c) If the loop contains a multiway switch with many successors, there
+ // could be paths with low probabilities, from which LICMing will be a
+ // net loss, especially if a large number of values are hoisted out.
+ // See bug 1708381 for a spectacular example and bug 1712078 for
+ // further discussion.
+ //
+ // It's preferable to perform test (c) only if (a) and (b) pass since (c)
+ // is more expensive to determine -- requiring a visit to all the MIR
+ // nodes -- than (a) or (b), which only involve visiting all blocks.
+
+ bool doVisit = true;
+ if (canOsr) {
+ JitSpew(JitSpew_LICM, " Skipping loop with header block%u due to OSR",
+ header->id());
+ doVisit = false;
+ } else if (numBlocks > LargestAllowedLoop) {
+ JitSpew(JitSpew_LICM,
+ " Skipping loop with header block%u "
+ "due to too many blocks (%u > thresh %u)",
+ header->id(), (uint32_t)numBlocks, (uint32_t)LargestAllowedLoop);
+ doVisit = false;
+ } else {
+ size_t switchSize = 0;
+ if (LoopContainsBigTableSwitch(graph, header, &switchSize)) {
+ JitSpew(JitSpew_LICM,
+ " Skipping loop with header block%u "
+ "due to oversize tableswitch (%u > thresh %u)",
+ header->id(), (uint32_t)switchSize,
+ (uint32_t)LargestAllowedTableSwitch);
+ doVisit = false;
+ }
+ }
+
+ if (doVisit) {
+ VisitLoop(graph, header);
+ }
+
+ UnmarkLoopBlocks(graph, header);
+
+ if (mir->shouldCancel("LICM (main loop)")) {
+ return false;
+ }
+ }
+
+ return true;
+}
diff --git a/js/src/jit/LICM.h b/js/src/jit/LICM.h
new file mode 100644
index 0000000000..38012615f5
--- /dev/null
+++ b/js/src/jit/LICM.h
@@ -0,0 +1,23 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_LICM_h
+#define jit_LICM_h
+
+// This file represents the Loop Invariant Code Motion optimization pass
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+class MIRGraph;
+
+[[nodiscard]] bool LICM(MIRGenerator* mir, MIRGraph& graph);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_LICM_h */
diff --git a/js/src/jit/LIR.cpp b/js/src/jit/LIR.cpp
new file mode 100644
index 0000000000..a083f347ef
--- /dev/null
+++ b/js/src/jit/LIR.cpp
@@ -0,0 +1,732 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/LIR.h"
+
+#include "mozilla/ScopeExit.h"
+
+#include <type_traits>
+
+#include "jit/JitSpewer.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "js/Printf.h"
+#include "util/Unicode.h"
+
+using namespace js;
+using namespace js::jit;
+
+const char* const js::jit::LIROpNames[] = {
+#define OPNAME(op, ...) #op,
+ LIR_OPCODE_LIST(OPNAME)
+#undef OPNAME
+};
+
+LIRGraph::LIRGraph(MIRGraph* mir)
+ : blocks_(),
+ constantPool_(mir->alloc()),
+ constantPoolMap_(mir->alloc()),
+ safepoints_(mir->alloc()),
+ nonCallSafepoints_(mir->alloc()),
+ numVirtualRegisters_(0),
+ numInstructions_(1), // First id is 1.
+ localSlotsSize_(0),
+ argumentSlotCount_(0),
+ mir_(*mir) {}
+
+bool LIRGraph::addConstantToPool(const Value& v, uint32_t* index) {
+ ConstantPoolMap::AddPtr p = constantPoolMap_.lookupForAdd(v);
+ if (p) {
+ *index = p->value();
+ return true;
+ }
+ *index = constantPool_.length();
+ return constantPool_.append(v) && constantPoolMap_.add(p, v, *index);
+}
+
+bool LIRGraph::noteNeedsSafepoint(LInstruction* ins) {
+ // Instructions with safepoints must be in linear order.
+ MOZ_ASSERT_IF(!safepoints_.empty(), safepoints_.back()->id() < ins->id());
+ if (!ins->isCall() && !nonCallSafepoints_.append(ins)) {
+ return false;
+ }
+ return safepoints_.append(ins);
+}
+
+#ifdef JS_JITSPEW
+void LIRGraph::dump(GenericPrinter& out) {
+ for (size_t i = 0; i < numBlocks(); i++) {
+ getBlock(i)->dump(out);
+ out.printf("\n");
+ }
+}
+
+void LIRGraph::dump() {
+ Fprinter out(stderr);
+ dump(out);
+ out.finish();
+}
+#endif
+
+LBlock::LBlock(MBasicBlock* from)
+ : block_(from), phis_(), entryMoveGroup_(nullptr), exitMoveGroup_(nullptr) {
+ from->assignLir(this);
+}
+
+bool LBlock::init(TempAllocator& alloc) {
+ // Count the number of LPhis we'll need.
+ size_t numLPhis = 0;
+ for (MPhiIterator i(block_->phisBegin()), e(block_->phisEnd()); i != e; ++i) {
+ MPhi* phi = *i;
+ switch (phi->type()) {
+ case MIRType::Value:
+ numLPhis += BOX_PIECES;
+ break;
+ case MIRType::Int64:
+ numLPhis += INT64_PIECES;
+ break;
+ default:
+ numLPhis += 1;
+ break;
+ }
+ }
+
+ // Allocate space for the LPhis.
+ if (!phis_.init(alloc, numLPhis)) {
+ return false;
+ }
+
+ // For each MIR phi, set up LIR phis as appropriate. We'll fill in their
+ // operands on each incoming edge, and set their definitions at the start of
+ // their defining block.
+ size_t phiIndex = 0;
+ size_t numPreds = block_->numPredecessors();
+ for (MPhiIterator i(block_->phisBegin()), e(block_->phisEnd()); i != e; ++i) {
+ MPhi* phi = *i;
+ MOZ_ASSERT(phi->numOperands() == numPreds);
+
+ int numPhis;
+ switch (phi->type()) {
+ case MIRType::Value:
+ numPhis = BOX_PIECES;
+ break;
+ case MIRType::Int64:
+ numPhis = INT64_PIECES;
+ break;
+ default:
+ numPhis = 1;
+ break;
+ }
+ for (int i = 0; i < numPhis; i++) {
+ LAllocation* inputs = alloc.allocateArray<LAllocation>(numPreds);
+ if (!inputs) {
+ return false;
+ }
+
+ void* addr = &phis_[phiIndex++];
+ LPhi* lphi = new (addr) LPhi(phi, inputs);
+ lphi->setBlock(this);
+ }
+ }
+ return true;
+}
+
+const LInstruction* LBlock::firstInstructionWithId() const {
+ for (LInstructionIterator i(instructions_.begin()); i != instructions_.end();
+ ++i) {
+ if (i->id()) {
+ return *i;
+ }
+ }
+ return 0;
+}
+
+LMoveGroup* LBlock::getEntryMoveGroup(TempAllocator& alloc) {
+ if (entryMoveGroup_) {
+ return entryMoveGroup_;
+ }
+ entryMoveGroup_ = LMoveGroup::New(alloc);
+ insertBefore(*begin(), entryMoveGroup_);
+ return entryMoveGroup_;
+}
+
+LMoveGroup* LBlock::getExitMoveGroup(TempAllocator& alloc) {
+ if (exitMoveGroup_) {
+ return exitMoveGroup_;
+ }
+ exitMoveGroup_ = LMoveGroup::New(alloc);
+ insertBefore(*rbegin(), exitMoveGroup_);
+ return exitMoveGroup_;
+}
+
+#ifdef JS_JITSPEW
+void LBlock::dump(GenericPrinter& out) {
+ out.printf("block%u:\n", mir()->id());
+ for (size_t i = 0; i < numPhis(); ++i) {
+ getPhi(i)->dump(out);
+ out.printf("\n");
+ }
+ for (LInstructionIterator iter = begin(); iter != end(); iter++) {
+ iter->dump(out);
+ if (iter->safepoint()) {
+ out.printf(" SAFEPOINT(0x%p) ", iter->safepoint());
+ }
+ out.printf("\n");
+ }
+}
+
+void LBlock::dump() {
+ Fprinter out(stderr);
+ dump(out);
+ out.finish();
+}
+#endif
+
+static size_t TotalOperandCount(LRecoverInfo* recoverInfo) {
+ size_t accum = 0;
+ for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) {
+ if (!it->isRecoveredOnBailout()) {
+ accum++;
+ }
+ }
+ return accum;
+}
+
+LRecoverInfo::LRecoverInfo(TempAllocator& alloc)
+ : instructions_(alloc), recoverOffset_(INVALID_RECOVER_OFFSET) {}
+
+LRecoverInfo* LRecoverInfo::New(MIRGenerator* gen, MResumePoint* mir) {
+ LRecoverInfo* recoverInfo = new (gen->alloc()) LRecoverInfo(gen->alloc());
+ if (!recoverInfo || !recoverInfo->init(mir)) {
+ return nullptr;
+ }
+
+ JitSpew(JitSpew_IonSnapshots, "Generating LIR recover info %p from MIR (%p)",
+ (void*)recoverInfo, (void*)mir);
+
+ return recoverInfo;
+}
+
+// de-virtualise MResumePoint::getOperand calls.
+template <typename Node>
+bool LRecoverInfo::appendOperands(Node* ins) {
+ for (size_t i = 0, end = ins->numOperands(); i < end; i++) {
+ MDefinition* def = ins->getOperand(i);
+
+ // As there is no cycle in the data-flow (without MPhi), checking for
+ // isInWorkList implies that the definition is already in the
+ // instruction vector, and not processed by a caller of the current
+ // function.
+ if (def->isRecoveredOnBailout() && !def->isInWorklist()) {
+ if (!appendDefinition(def)) {
+ return false;
+ }
+ }
+ }
+
+ return true;
+}
+
+bool LRecoverInfo::appendDefinition(MDefinition* def) {
+ MOZ_ASSERT(def->isRecoveredOnBailout());
+ def->setInWorklist();
+ auto clearWorklistFlagOnFailure =
+ mozilla::MakeScopeExit([&] { def->setNotInWorklist(); });
+
+ if (!appendOperands(def)) {
+ return false;
+ }
+
+ if (!instructions_.append(def)) {
+ return false;
+ }
+
+ clearWorklistFlagOnFailure.release();
+ return true;
+}
+
+bool LRecoverInfo::appendResumePoint(MResumePoint* rp) {
+ // Stores should be recovered first.
+ for (auto iter(rp->storesBegin()), end(rp->storesEnd()); iter != end;
+ ++iter) {
+ if (!appendDefinition(iter->operand)) {
+ return false;
+ }
+ }
+
+ if (rp->caller() && !appendResumePoint(rp->caller())) {
+ return false;
+ }
+
+ if (!appendOperands(rp)) {
+ return false;
+ }
+
+ return instructions_.append(rp);
+}
+
+bool LRecoverInfo::init(MResumePoint* rp) {
+ // Before exiting this function, remove temporary flags from all definitions
+ // added in the vector.
+ auto clearWorklistFlags = mozilla::MakeScopeExit([&] {
+ for (MNode** it = begin(); it != end(); it++) {
+ if (!(*it)->isDefinition()) {
+ continue;
+ }
+ (*it)->toDefinition()->setNotInWorklist();
+ }
+ });
+
+ // Sort operations in the order in which we need to restore the stack. This
+ // implies that outer frames, as well as operations needed to recover the
+ // current frame, are located before the current frame. The inner-most
+ // resume point should be the last element in the list.
+ if (!appendResumePoint(rp)) {
+ return false;
+ }
+
+ MOZ_ASSERT(mir() == rp);
+ return true;
+}
+
+LSnapshot::LSnapshot(LRecoverInfo* recoverInfo, BailoutKind kind)
+ : slots_(nullptr),
+ recoverInfo_(recoverInfo),
+ snapshotOffset_(INVALID_SNAPSHOT_OFFSET),
+ numSlots_(TotalOperandCount(recoverInfo) * BOX_PIECES),
+ bailoutKind_(kind) {}
+
+bool LSnapshot::init(MIRGenerator* gen) {
+ slots_ = gen->allocate<LAllocation>(numSlots_);
+ return !!slots_;
+}
+
+LSnapshot* LSnapshot::New(MIRGenerator* gen, LRecoverInfo* recover,
+ BailoutKind kind) {
+ LSnapshot* snapshot = new (gen->alloc()) LSnapshot(recover, kind);
+ if (!snapshot || !snapshot->init(gen)) {
+ return nullptr;
+ }
+
+ JitSpew(JitSpew_IonSnapshots, "Generating LIR snapshot %p from recover (%p)",
+ (void*)snapshot, (void*)recover);
+
+ return snapshot;
+}
+
+void LSnapshot::rewriteRecoveredInput(LUse input) {
+ // Mark any operands to this snapshot with the same value as input as being
+ // equal to the instruction's result.
+ for (size_t i = 0; i < numEntries(); i++) {
+ if (getEntry(i)->isUse() &&
+ getEntry(i)->toUse()->virtualRegister() == input.virtualRegister()) {
+ setEntry(i, LUse(input.virtualRegister(), LUse::RECOVERED_INPUT));
+ }
+ }
+}
+
+#ifdef JS_JITSPEW
+void LNode::printName(GenericPrinter& out, Opcode op) {
+ static const char* const names[] = {
+# define LIROP(x) #x,
+ LIR_OPCODE_LIST(LIROP)
+# undef LIROP
+ };
+ const char* name = names[uint32_t(op)];
+ size_t len = strlen(name);
+ for (size_t i = 0; i < len; i++) {
+ out.printf("%c", unicode::ToLowerCase(name[i]));
+ }
+}
+
+void LNode::printName(GenericPrinter& out) { printName(out, op()); }
+#endif
+
+bool LAllocation::aliases(const LAllocation& other) const {
+ if (isFloatReg() && other.isFloatReg()) {
+ return toFloatReg()->reg().aliases(other.toFloatReg()->reg());
+ }
+ return *this == other;
+}
+
+#ifdef JS_JITSPEW
+static const char* DefTypeName(LDefinition::Type type) {
+ switch (type) {
+ case LDefinition::GENERAL:
+ return "g";
+ case LDefinition::INT32:
+ return "i";
+ case LDefinition::OBJECT:
+ return "o";
+ case LDefinition::SLOTS:
+ return "s";
+ case LDefinition::FLOAT32:
+ return "f";
+ case LDefinition::DOUBLE:
+ return "d";
+ case LDefinition::SIMD128:
+ return "simd128";
+ case LDefinition::STACKRESULTS:
+ return "stackresults";
+# ifdef JS_NUNBOX32
+ case LDefinition::TYPE:
+ return "t";
+ case LDefinition::PAYLOAD:
+ return "p";
+# else
+ case LDefinition::BOX:
+ return "x";
+# endif
+ }
+ MOZ_CRASH("Invalid type");
+}
+
+UniqueChars LDefinition::toString() const {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+
+ UniqueChars buf;
+ if (isBogusTemp()) {
+ buf = JS_smprintf("bogus");
+ } else {
+ buf = JS_smprintf("v%u<%s>", virtualRegister(), DefTypeName(type()));
+ if (buf) {
+ if (policy() == LDefinition::FIXED) {
+ buf = JS_sprintf_append(std::move(buf), ":%s",
+ output()->toString().get());
+ } else if (policy() == LDefinition::MUST_REUSE_INPUT) {
+ buf = JS_sprintf_append(std::move(buf), ":tied(%u)", getReusedInput());
+ }
+ }
+ }
+
+ if (!buf) {
+ oomUnsafe.crash("LDefinition::toString()");
+ }
+
+ return buf;
+}
+
+static UniqueChars PrintUse(const LUse* use) {
+ switch (use->policy()) {
+ case LUse::REGISTER:
+ return JS_smprintf("v%u:R", use->virtualRegister());
+ case LUse::FIXED:
+ return JS_smprintf("v%u:F:%s", use->virtualRegister(),
+ AnyRegister::FromCode(use->registerCode()).name());
+ case LUse::ANY:
+ return JS_smprintf("v%u:A", use->virtualRegister());
+ case LUse::KEEPALIVE:
+ return JS_smprintf("v%u:KA", use->virtualRegister());
+ case LUse::STACK:
+ return JS_smprintf("v%u:S", use->virtualRegister());
+ case LUse::RECOVERED_INPUT:
+ return JS_smprintf("v%u:RI", use->virtualRegister());
+ default:
+ MOZ_CRASH("invalid use policy");
+ }
+}
+
+UniqueChars LAllocation::toString() const {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+
+ UniqueChars buf;
+ if (isBogus()) {
+ buf = JS_smprintf("bogus");
+ } else {
+ switch (kind()) {
+ case LAllocation::CONSTANT_VALUE:
+ case LAllocation::CONSTANT_INDEX:
+ buf = JS_smprintf("c");
+ break;
+ case LAllocation::GPR:
+ buf = JS_smprintf("%s", toGeneralReg()->reg().name());
+ break;
+ case LAllocation::FPU:
+ buf = JS_smprintf("%s", toFloatReg()->reg().name());
+ break;
+ case LAllocation::STACK_SLOT:
+ buf = JS_smprintf("stack:%u", toStackSlot()->slot());
+ break;
+ case LAllocation::ARGUMENT_SLOT:
+ buf = JS_smprintf("arg:%u", toArgument()->index());
+ break;
+ case LAllocation::STACK_AREA:
+ buf = JS_smprintf("stackarea:%u+%u", toStackArea()->base(),
+ toStackArea()->size());
+ break;
+ case LAllocation::USE:
+ buf = PrintUse(toUse());
+ break;
+ default:
+ MOZ_CRASH("what?");
+ }
+ }
+
+ if (!buf) {
+ oomUnsafe.crash("LAllocation::toString()");
+ }
+
+ return buf;
+}
+
+void LAllocation::dump() const { fprintf(stderr, "%s\n", toString().get()); }
+
+void LDefinition::dump() const { fprintf(stderr, "%s\n", toString().get()); }
+
+template <typename T>
+static void PrintOperands(GenericPrinter& out, T* node) {
+ size_t numOperands = node->numOperands();
+
+ for (size_t i = 0; i < numOperands; i++) {
+ out.printf(" (%s)", node->getOperand(i)->toString().get());
+ if (i != numOperands - 1) {
+ out.printf(",");
+ }
+ }
+}
+
+void LNode::printOperands(GenericPrinter& out) {
+ if (isMoveGroup()) {
+ toMoveGroup()->printOperands(out);
+ return;
+ }
+
+ if (isPhi()) {
+ PrintOperands(out, toPhi());
+ } else {
+ PrintOperands(out, toInstruction());
+ }
+}
+#endif
+
+void LInstruction::assignSnapshot(LSnapshot* snapshot) {
+ MOZ_ASSERT(!snapshot_);
+ snapshot_ = snapshot;
+
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_IonSnapshots)) {
+ JitSpewHeader(JitSpew_IonSnapshots);
+ Fprinter& out = JitSpewPrinter();
+ out.printf("Assigning snapshot %p to instruction %p (", (void*)snapshot,
+ (void*)this);
+ printName(out);
+ out.printf(")\n");
+ }
+#endif
+}
+
+#ifdef JS_JITSPEW
+static size_t NumSuccessorsHelper(const LNode* ins) { return 0; }
+
+template <size_t Succs, size_t Operands, size_t Temps>
+static size_t NumSuccessorsHelper(
+ const LControlInstructionHelper<Succs, Operands, Temps>* ins) {
+ return Succs;
+}
+
+static size_t NumSuccessors(const LInstruction* ins) {
+ switch (ins->op()) {
+ default:
+ MOZ_CRASH("Unexpected LIR op");
+# define LIROP(x) \
+ case LNode::Opcode::x: \
+ return NumSuccessorsHelper(ins->to##x());
+ LIR_OPCODE_LIST(LIROP)
+# undef LIROP
+ }
+}
+
+static MBasicBlock* GetSuccessorHelper(const LNode* ins, size_t i) {
+ MOZ_CRASH("Unexpected instruction with successors");
+}
+
+template <size_t Succs, size_t Operands, size_t Temps>
+static MBasicBlock* GetSuccessorHelper(
+ const LControlInstructionHelper<Succs, Operands, Temps>* ins, size_t i) {
+ return ins->getSuccessor(i);
+}
+
+static MBasicBlock* GetSuccessor(const LInstruction* ins, size_t i) {
+ MOZ_ASSERT(i < NumSuccessors(ins));
+
+ switch (ins->op()) {
+ default:
+ MOZ_CRASH("Unexpected LIR op");
+# define LIROP(x) \
+ case LNode::Opcode::x: \
+ return GetSuccessorHelper(ins->to##x(), i);
+ LIR_OPCODE_LIST(LIROP)
+# undef LIROP
+ }
+}
+#endif
+
+#ifdef JS_JITSPEW
+void LNode::dump(GenericPrinter& out) {
+ if (numDefs() != 0) {
+ out.printf("{");
+ for (size_t i = 0; i < numDefs(); i++) {
+ const LDefinition* def =
+ isPhi() ? toPhi()->getDef(i) : toInstruction()->getDef(i);
+ out.printf("%s", def->toString().get());
+ if (i != numDefs() - 1) {
+ out.printf(", ");
+ }
+ }
+ out.printf("} <- ");
+ }
+
+ printName(out);
+ printOperands(out);
+
+ if (isInstruction()) {
+ LInstruction* ins = toInstruction();
+ size_t numTemps = ins->numTemps();
+ if (numTemps > 0) {
+ out.printf(" t=(");
+ for (size_t i = 0; i < numTemps; i++) {
+ out.printf("%s", ins->getTemp(i)->toString().get());
+ if (i != numTemps - 1) {
+ out.printf(", ");
+ }
+ }
+ out.printf(")");
+ }
+
+ size_t numSuccessors = NumSuccessors(ins);
+ if (numSuccessors > 0) {
+ out.printf(" s=(");
+ for (size_t i = 0; i < numSuccessors; i++) {
+ MBasicBlock* succ = GetSuccessor(ins, i);
+ out.printf("block%u", succ->id());
+ if (i != numSuccessors - 1) {
+ out.printf(", ");
+ }
+ }
+ out.printf(")");
+ }
+ }
+}
+
+void LNode::dump() {
+ Fprinter out(stderr);
+ dump(out);
+ out.printf("\n");
+ out.finish();
+}
+
+const char* LNode::getExtraName() const {
+ switch (op()) {
+ default:
+ MOZ_CRASH("Unexpected LIR op");
+# define LIROP(x) \
+ case LNode::Opcode::x: \
+ return to##x()->extraName();
+ LIR_OPCODE_LIST(LIROP)
+# undef LIROP
+ }
+}
+#endif
+
+void LInstruction::initSafepoint(TempAllocator& alloc) {
+ MOZ_ASSERT(!safepoint_);
+ safepoint_ = new (alloc) LSafepoint(alloc);
+ MOZ_ASSERT(safepoint_);
+}
+
+bool LMoveGroup::add(LAllocation from, LAllocation to, LDefinition::Type type) {
+#ifdef DEBUG
+ MOZ_ASSERT(from != to);
+ for (size_t i = 0; i < moves_.length(); i++) {
+ MOZ_ASSERT(to != moves_[i].to());
+ }
+
+ // Check that SIMD moves are aligned according to ABI requirements.
+ // clang-format off
+# ifdef ENABLE_WASM_SIMD
+ // Alignment is not currently required for SIMD on x86/x64/arm64. See also
+ // CodeGeneratorShared::CodeGeneratorShared and in general everywhere
+ // SimdMemoryAignment is used. Likely, alignment requirements will return.
+# if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) || \
+ defined(JS_CODEGEN_ARM64)
+ // No need for any check on x86/x64/arm64.
+# else
+# error "Need to consider SIMD alignment on this target."
+ // The following code may be of use if we need alignment checks on
+ // some future target.
+ //if (LDefinition(type).type() == LDefinition::SIMD128) {
+ // MOZ_ASSERT(from.isMemory() || from.isFloatReg());
+ // if (from.isMemory()) {
+ // if (from.isArgument()) {
+ // MOZ_ASSERT(from.toArgument()->index() % SimdMemoryAlignment == 0);
+ // } else {
+ // MOZ_ASSERT(from.toStackSlot()->slot() % SimdMemoryAlignment == 0);
+ // }
+ // }
+ // MOZ_ASSERT(to.isMemory() || to.isFloatReg());
+ // if (to.isMemory()) {
+ // if (to.isArgument()) {
+ // MOZ_ASSERT(to.toArgument()->index() % SimdMemoryAlignment == 0);
+ // } else {
+ // MOZ_ASSERT(to.toStackSlot()->slot() % SimdMemoryAlignment == 0);
+ // }
+ // }
+ //}
+# endif
+# endif
+ // clang-format on
+
+#endif
+ return moves_.append(LMove(from, to, type));
+}
+
+bool LMoveGroup::addAfter(LAllocation from, LAllocation to,
+ LDefinition::Type type) {
+ // Transform the operands to this move so that performing the result
+ // simultaneously with existing moves in the group will have the same
+ // effect as if the original move took place after the existing moves.
+
+ for (size_t i = 0; i < moves_.length(); i++) {
+ if (moves_[i].to() == from) {
+ from = moves_[i].from();
+ break;
+ }
+ }
+
+ if (from == to) {
+ return true;
+ }
+
+ for (size_t i = 0; i < moves_.length(); i++) {
+ if (to == moves_[i].to()) {
+ moves_[i] = LMove(from, to, type);
+ return true;
+ }
+ }
+
+ return add(from, to, type);
+}
+
+#ifdef JS_JITSPEW
+void LMoveGroup::printOperands(GenericPrinter& out) {
+ for (size_t i = 0; i < numMoves(); i++) {
+ const LMove& move = getMove(i);
+ out.printf(" [%s -> %s", move.from().toString().get(),
+ move.to().toString().get());
+ out.printf(", %s", DefTypeName(move.type()));
+ out.printf("]");
+ if (i != numMoves() - 1) {
+ out.printf(",");
+ }
+ }
+}
+#endif
+
+#define LIROP(x) \
+ static_assert(!std::is_polymorphic_v<L##x>, \
+ "LIR instructions should not have virtual methods");
+LIR_OPCODE_LIST(LIROP)
+#undef LIROP
diff --git a/js/src/jit/LIR.h b/js/src/jit/LIR.h
new file mode 100644
index 0000000000..740246b66b
--- /dev/null
+++ b/js/src/jit/LIR.h
@@ -0,0 +1,1992 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_LIR_h
+#define jit_LIR_h
+
+// This file declares the core data structures for LIR: storage allocations for
+// inputs and outputs, as well as the interface instructions must conform to.
+
+#include "mozilla/Array.h"
+#include "mozilla/Casting.h"
+
+#include "jit/Bailouts.h"
+#include "jit/FixedList.h"
+#include "jit/InlineList.h"
+#include "jit/JitAllocPolicy.h"
+#include "jit/LIROpsGenerated.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "jit/Registers.h"
+#include "jit/Safepoints.h"
+#include "util/Memory.h"
+
+namespace js {
+namespace jit {
+
+class LUse;
+class LGeneralReg;
+class LFloatReg;
+class LStackSlot;
+class LStackArea;
+class LArgument;
+class LConstantIndex;
+class LInstruction;
+class LDefinition;
+class MBasicBlock;
+class MIRGenerator;
+
+static const uint32_t VREG_INCREMENT = 1;
+
+static const uint32_t THIS_FRAME_ARGSLOT = 0;
+
+#if defined(JS_NUNBOX32)
+# define BOX_PIECES 2
+static const uint32_t VREG_TYPE_OFFSET = 0;
+static const uint32_t VREG_DATA_OFFSET = 1;
+static const uint32_t TYPE_INDEX = 0;
+static const uint32_t PAYLOAD_INDEX = 1;
+static const uint32_t INT64LOW_INDEX = 0;
+static const uint32_t INT64HIGH_INDEX = 1;
+#elif defined(JS_PUNBOX64)
+# define BOX_PIECES 1
+#else
+# error "Unknown!"
+#endif
+
+static const uint32_t INT64_PIECES = sizeof(int64_t) / sizeof(uintptr_t);
+
+// Represents storage for an operand. For constants, the pointer is tagged
+// with a single bit, and the untagged pointer is a pointer to a Value.
+class LAllocation : public TempObject {
+ uintptr_t bits_;
+
+ // 3 bits gives us enough for an interesting set of Kinds and also fits
+ // within the alignment bits of pointers to Value, which are always
+ // 8-byte aligned.
+ static const uintptr_t KIND_BITS = 3;
+ static const uintptr_t KIND_SHIFT = 0;
+ static const uintptr_t KIND_MASK = (1 << KIND_BITS) - 1;
+
+ protected:
+#ifdef JS_64BIT
+ static const uintptr_t DATA_BITS = sizeof(uint32_t) * 8;
+#else
+ static const uintptr_t DATA_BITS = (sizeof(uint32_t) * 8) - KIND_BITS;
+#endif
+ static const uintptr_t DATA_SHIFT = KIND_SHIFT + KIND_BITS;
+
+ public:
+ enum Kind {
+ CONSTANT_VALUE, // MConstant*.
+ CONSTANT_INDEX, // Constant arbitrary index.
+ USE, // Use of a virtual register, with physical allocation policy.
+ GPR, // General purpose register.
+ FPU, // Floating-point register.
+ STACK_SLOT, // Stack slot.
+ STACK_AREA, // Stack area.
+ ARGUMENT_SLOT // Argument slot.
+ };
+
+ static const uintptr_t DATA_MASK = (uintptr_t(1) << DATA_BITS) - 1;
+
+ protected:
+ uint32_t data() const {
+ MOZ_ASSERT(!hasIns());
+ return mozilla::AssertedCast<uint32_t>(bits_ >> DATA_SHIFT);
+ }
+ void setData(uintptr_t data) {
+ MOZ_ASSERT(!hasIns());
+ MOZ_ASSERT(data <= DATA_MASK);
+ bits_ &= ~(DATA_MASK << DATA_SHIFT);
+ bits_ |= (data << DATA_SHIFT);
+ }
+ void setKindAndData(Kind kind, uintptr_t data) {
+ MOZ_ASSERT(data <= DATA_MASK);
+ bits_ = (uintptr_t(kind) << KIND_SHIFT) | data << DATA_SHIFT;
+ MOZ_ASSERT(!hasIns());
+ }
+
+ bool hasIns() const { return isStackArea(); }
+ const LInstruction* ins() const {
+ MOZ_ASSERT(hasIns());
+ return reinterpret_cast<const LInstruction*>(bits_ &
+ ~(KIND_MASK << KIND_SHIFT));
+ }
+ LInstruction* ins() {
+ MOZ_ASSERT(hasIns());
+ return reinterpret_cast<LInstruction*>(bits_ & ~(KIND_MASK << KIND_SHIFT));
+ }
+ void setKindAndIns(Kind kind, LInstruction* ins) {
+ uintptr_t data = reinterpret_cast<uintptr_t>(ins);
+ MOZ_ASSERT((data & (KIND_MASK << KIND_SHIFT)) == 0);
+ bits_ = data | (uintptr_t(kind) << KIND_SHIFT);
+ MOZ_ASSERT(hasIns());
+ }
+
+ LAllocation(Kind kind, uintptr_t data) { setKindAndData(kind, data); }
+ LAllocation(Kind kind, LInstruction* ins) { setKindAndIns(kind, ins); }
+ explicit LAllocation(Kind kind) { setKindAndData(kind, 0); }
+
+ public:
+ LAllocation() : bits_(0) { MOZ_ASSERT(isBogus()); }
+
+ // The MConstant pointer must have its low bits cleared.
+ explicit LAllocation(const MConstant* c) {
+ MOZ_ASSERT(c);
+ bits_ = uintptr_t(c);
+ MOZ_ASSERT((bits_ & (KIND_MASK << KIND_SHIFT)) == 0);
+ bits_ |= CONSTANT_VALUE << KIND_SHIFT;
+ }
+ inline explicit LAllocation(AnyRegister reg);
+
+ Kind kind() const { return (Kind)((bits_ >> KIND_SHIFT) & KIND_MASK); }
+
+ bool isBogus() const { return bits_ == 0; }
+ bool isUse() const { return kind() == USE; }
+ bool isConstant() const { return isConstantValue() || isConstantIndex(); }
+ bool isConstantValue() const { return kind() == CONSTANT_VALUE; }
+ bool isConstantIndex() const { return kind() == CONSTANT_INDEX; }
+ bool isGeneralReg() const { return kind() == GPR; }
+ bool isFloatReg() const { return kind() == FPU; }
+ bool isStackSlot() const { return kind() == STACK_SLOT; }
+ bool isStackArea() const { return kind() == STACK_AREA; }
+ bool isArgument() const { return kind() == ARGUMENT_SLOT; }
+ bool isRegister() const { return isGeneralReg() || isFloatReg(); }
+ bool isRegister(bool needFloat) const {
+ return needFloat ? isFloatReg() : isGeneralReg();
+ }
+ bool isMemory() const { return isStackSlot() || isArgument(); }
+ inline uint32_t memorySlot() const;
+ inline LUse* toUse();
+ inline const LUse* toUse() const;
+ inline const LGeneralReg* toGeneralReg() const;
+ inline const LFloatReg* toFloatReg() const;
+ inline const LStackSlot* toStackSlot() const;
+ inline LStackArea* toStackArea();
+ inline const LStackArea* toStackArea() const;
+ inline const LArgument* toArgument() const;
+ inline const LConstantIndex* toConstantIndex() const;
+ inline AnyRegister toRegister() const;
+
+ const MConstant* toConstant() const {
+ MOZ_ASSERT(isConstantValue());
+ return reinterpret_cast<const MConstant*>(bits_ &
+ ~(KIND_MASK << KIND_SHIFT));
+ }
+
+ bool operator==(const LAllocation& other) const {
+ return bits_ == other.bits_;
+ }
+
+ bool operator!=(const LAllocation& other) const {
+ return bits_ != other.bits_;
+ }
+
+ HashNumber hash() const { return bits_; }
+
+ bool aliases(const LAllocation& other) const;
+
+#ifdef JS_JITSPEW
+ UniqueChars toString() const;
+ void dump() const;
+#endif
+};
+
+class LUse : public LAllocation {
+ static const uint32_t POLICY_BITS = 3;
+ static const uint32_t POLICY_SHIFT = 0;
+ static const uint32_t POLICY_MASK = (1 << POLICY_BITS) - 1;
+#ifdef JS_CODEGEN_ARM64
+ static const uint32_t REG_BITS = 7;
+#else
+ static const uint32_t REG_BITS = 6;
+#endif
+ static const uint32_t REG_SHIFT = POLICY_SHIFT + POLICY_BITS;
+ static const uint32_t REG_MASK = (1 << REG_BITS) - 1;
+
+ // Whether the physical register for this operand may be reused for a def.
+ static const uint32_t USED_AT_START_BITS = 1;
+ static const uint32_t USED_AT_START_SHIFT = REG_SHIFT + REG_BITS;
+ static const uint32_t USED_AT_START_MASK = (1 << USED_AT_START_BITS) - 1;
+
+ // The REG field will hold the register code for any Register or
+ // FloatRegister, though not for an AnyRegister.
+ static_assert(std::max(Registers::Total, FloatRegisters::Total) <=
+ REG_MASK + 1,
+ "The field must be able to represent any register code");
+
+ public:
+ // Virtual registers get the remaining bits.
+ static const uint32_t VREG_BITS =
+ DATA_BITS - (USED_AT_START_SHIFT + USED_AT_START_BITS);
+ static const uint32_t VREG_SHIFT = USED_AT_START_SHIFT + USED_AT_START_BITS;
+ static const uint32_t VREG_MASK = (1 << VREG_BITS) - 1;
+
+ enum Policy {
+ // Input should be in a read-only register or stack slot.
+ ANY,
+
+ // Input must be in a read-only register.
+ REGISTER,
+
+ // Input must be in a specific, read-only register.
+ FIXED,
+
+ // Keep the used virtual register alive, and use whatever allocation is
+ // available. This is similar to ANY but hints to the register allocator
+ // that it is never useful to optimize this site.
+ KEEPALIVE,
+
+ // Input must be allocated on the stack. Only used when extracting stack
+ // results from stack result areas.
+ STACK,
+
+ // For snapshot inputs, indicates that the associated instruction will
+ // write this input to its output register before bailing out.
+ // The register allocator may thus allocate that output register, and
+ // does not need to keep the virtual register alive (alternatively,
+ // this may be treated as KEEPALIVE).
+ RECOVERED_INPUT
+ };
+
+ void set(Policy policy, uint32_t reg, bool usedAtStart) {
+ MOZ_ASSERT(reg <= REG_MASK, "Register code must fit in field");
+ setKindAndData(USE, (policy << POLICY_SHIFT) | (reg << REG_SHIFT) |
+ ((usedAtStart ? 1 : 0) << USED_AT_START_SHIFT));
+ }
+
+ public:
+ LUse(uint32_t vreg, Policy policy, bool usedAtStart = false) {
+ set(policy, 0, usedAtStart);
+ setVirtualRegister(vreg);
+ }
+ explicit LUse(Policy policy, bool usedAtStart = false) {
+ set(policy, 0, usedAtStart);
+ }
+ explicit LUse(Register reg, bool usedAtStart = false) {
+ set(FIXED, reg.code(), usedAtStart);
+ }
+ explicit LUse(FloatRegister reg, bool usedAtStart = false) {
+ set(FIXED, reg.code(), usedAtStart);
+ }
+ LUse(Register reg, uint32_t virtualRegister, bool usedAtStart = false) {
+ set(FIXED, reg.code(), usedAtStart);
+ setVirtualRegister(virtualRegister);
+ }
+ LUse(FloatRegister reg, uint32_t virtualRegister, bool usedAtStart = false) {
+ set(FIXED, reg.code(), usedAtStart);
+ setVirtualRegister(virtualRegister);
+ }
+
+ void setVirtualRegister(uint32_t index) {
+ MOZ_ASSERT(index < VREG_MASK);
+
+ uint32_t old = data() & ~(VREG_MASK << VREG_SHIFT);
+ setData(old | (index << VREG_SHIFT));
+ }
+
+ Policy policy() const {
+ Policy policy = (Policy)((data() >> POLICY_SHIFT) & POLICY_MASK);
+ return policy;
+ }
+ uint32_t virtualRegister() const {
+ uint32_t index = (data() >> VREG_SHIFT) & VREG_MASK;
+ MOZ_ASSERT(index != 0);
+ return index;
+ }
+ uint32_t registerCode() const {
+ MOZ_ASSERT(policy() == FIXED);
+ return (data() >> REG_SHIFT) & REG_MASK;
+ }
+ bool isFixedRegister() const { return policy() == FIXED; }
+ bool usedAtStart() const {
+ return !!((data() >> USED_AT_START_SHIFT) & USED_AT_START_MASK);
+ }
+};
+
+static const uint32_t MAX_VIRTUAL_REGISTERS = LUse::VREG_MASK;
+
+class LBoxAllocation {
+#ifdef JS_NUNBOX32
+ LAllocation type_;
+ LAllocation payload_;
+#else
+ LAllocation value_;
+#endif
+
+ public:
+#ifdef JS_NUNBOX32
+ LBoxAllocation(LAllocation type, LAllocation payload)
+ : type_(type), payload_(payload) {}
+
+ LAllocation type() const { return type_; }
+ LAllocation payload() const { return payload_; }
+#else
+ explicit LBoxAllocation(LAllocation value) : value_(value) {}
+
+ LAllocation value() const { return value_; }
+#endif
+};
+
+template <class ValT>
+class LInt64Value {
+#if JS_BITS_PER_WORD == 32
+ ValT high_;
+ ValT low_;
+#else
+ ValT value_;
+#endif
+
+ public:
+ LInt64Value() = default;
+
+#if JS_BITS_PER_WORD == 32
+ LInt64Value(ValT high, ValT low) : high_(high), low_(low) {}
+
+ ValT high() const { return high_; }
+ ValT low() const { return low_; }
+
+ const ValT* pointerHigh() const { return &high_; }
+ const ValT* pointerLow() const { return &low_; }
+#else
+ explicit LInt64Value(ValT value) : value_(value) {}
+
+ ValT value() const { return value_; }
+ const ValT* pointer() const { return &value_; }
+#endif
+};
+
+using LInt64Allocation = LInt64Value<LAllocation>;
+
+class LGeneralReg : public LAllocation {
+ public:
+ explicit LGeneralReg(Register reg) : LAllocation(GPR, reg.code()) {}
+
+ Register reg() const { return Register::FromCode(data()); }
+};
+
+class LFloatReg : public LAllocation {
+ public:
+ explicit LFloatReg(FloatRegister reg) : LAllocation(FPU, reg.code()) {}
+
+ FloatRegister reg() const { return FloatRegister::FromCode(data()); }
+};
+
+// Arbitrary constant index.
+class LConstantIndex : public LAllocation {
+ explicit LConstantIndex(uint32_t index)
+ : LAllocation(CONSTANT_INDEX, index) {}
+
+ public:
+ static LConstantIndex FromIndex(uint32_t index) {
+ return LConstantIndex(index);
+ }
+
+ uint32_t index() const { return data(); }
+};
+
+// Stack slots are indices into the stack. The indices are byte indices.
+class LStackSlot : public LAllocation {
+ public:
+ explicit LStackSlot(uint32_t slot) : LAllocation(STACK_SLOT, slot) {}
+
+ uint32_t slot() const { return data(); }
+};
+
+// Stack area indicates a contiguous stack allocation meant to receive call
+// results that don't fit in registers.
+class LStackArea : public LAllocation {
+ public:
+ explicit LStackArea(LInstruction* stackArea)
+ : LAllocation(STACK_AREA, stackArea) {}
+
+ // Byte index of base of stack area, in the same coordinate space as
+ // LStackSlot::slot().
+ inline uint32_t base() const;
+ inline void setBase(uint32_t base);
+
+ // Size in bytes of the stack area.
+ inline uint32_t size() const;
+ inline uint32_t alignment() const { return 8; }
+
+ class ResultIterator {
+ const LStackArea& alloc_;
+ uint32_t idx_;
+
+ public:
+ explicit ResultIterator(const LStackArea& alloc) : alloc_(alloc), idx_(0) {}
+
+ inline bool done() const;
+ inline void next();
+ inline LAllocation alloc() const;
+ inline bool isGcPointer() const;
+
+ explicit operator bool() const { return !done(); }
+ };
+
+ ResultIterator results() const { return ResultIterator(*this); }
+
+ inline LStackSlot resultAlloc(LInstruction* lir, LDefinition* def) const;
+};
+
+// Arguments are reverse indices into the stack. The indices are byte indices.
+class LArgument : public LAllocation {
+ public:
+ explicit LArgument(uint32_t index) : LAllocation(ARGUMENT_SLOT, index) {}
+
+ uint32_t index() const { return data(); }
+};
+
+inline uint32_t LAllocation::memorySlot() const {
+ MOZ_ASSERT(isMemory());
+ return isStackSlot() ? toStackSlot()->slot() : toArgument()->index();
+}
+
+// Represents storage for a definition.
+class LDefinition {
+ // Bits containing policy, type, and virtual register.
+ uint32_t bits_;
+
+ // Before register allocation, this optionally contains a fixed policy.
+ // Register allocation assigns this field to a physical policy if none is
+ // fixed.
+ //
+ // Right now, pre-allocated outputs are limited to the following:
+ // * Physical argument stack slots.
+ // * Physical registers.
+ LAllocation output_;
+
+ static const uint32_t TYPE_BITS = 4;
+ static const uint32_t TYPE_SHIFT = 0;
+ static const uint32_t TYPE_MASK = (1 << TYPE_BITS) - 1;
+ static const uint32_t POLICY_BITS = 2;
+ static const uint32_t POLICY_SHIFT = TYPE_SHIFT + TYPE_BITS;
+ static const uint32_t POLICY_MASK = (1 << POLICY_BITS) - 1;
+
+ static const uint32_t VREG_BITS =
+ (sizeof(uint32_t) * 8) - (POLICY_BITS + TYPE_BITS);
+ static const uint32_t VREG_SHIFT = POLICY_SHIFT + POLICY_BITS;
+ static const uint32_t VREG_MASK = (1 << VREG_BITS) - 1;
+
+ public:
+ // Note that definitions, by default, are always allocated a register,
+ // unless the policy specifies that an input can be re-used and that input
+ // is a stack slot.
+ enum Policy {
+ // The policy is predetermined by the LAllocation attached to this
+ // definition. The allocation may be:
+ // * A register, which may not appear as any fixed temporary.
+ // * A stack slot or argument.
+ //
+ // Register allocation will not modify a fixed allocation.
+ FIXED,
+
+ // A random register of an appropriate class will be assigned.
+ REGISTER,
+
+ // An area on the stack must be assigned. Used when defining stack results
+ // and stack result areas.
+ STACK,
+
+ // One definition per instruction must re-use the first input
+ // allocation, which (for now) must be a register.
+ MUST_REUSE_INPUT
+ };
+
+ enum Type {
+ GENERAL, // Generic, integer or pointer-width data (GPR).
+ INT32, // int32 data (GPR).
+ OBJECT, // Pointer that may be collected as garbage (GPR).
+ SLOTS, // Slots/elements pointer that may be moved by minor GCs (GPR).
+ FLOAT32, // 32-bit floating-point value (FPU).
+ DOUBLE, // 64-bit floating-point value (FPU).
+ SIMD128, // 128-bit SIMD vector (FPU).
+ STACKRESULTS, // A variable-size stack allocation that may contain objects.
+#ifdef JS_NUNBOX32
+ // A type virtual register must be followed by a payload virtual
+ // register, as both will be tracked as a single gcthing.
+ TYPE,
+ PAYLOAD
+#else
+ BOX // Joined box, for punbox systems. (GPR, gcthing)
+#endif
+ };
+
+ void set(uint32_t index, Type type, Policy policy) {
+ static_assert(MAX_VIRTUAL_REGISTERS <= VREG_MASK);
+ bits_ =
+ (index << VREG_SHIFT) | (policy << POLICY_SHIFT) | (type << TYPE_SHIFT);
+#ifndef ENABLE_WASM_SIMD
+ MOZ_ASSERT(this->type() != SIMD128);
+#endif
+ }
+
+ public:
+ LDefinition(uint32_t index, Type type, Policy policy = REGISTER) {
+ set(index, type, policy);
+ }
+
+ explicit LDefinition(Type type, Policy policy = REGISTER) {
+ set(0, type, policy);
+ }
+
+ LDefinition(Type type, const LAllocation& a) : output_(a) {
+ set(0, type, FIXED);
+ }
+
+ LDefinition(uint32_t index, Type type, const LAllocation& a) : output_(a) {
+ set(index, type, FIXED);
+ }
+
+ LDefinition() : bits_(0) { MOZ_ASSERT(isBogusTemp()); }
+
+ static LDefinition BogusTemp() { return LDefinition(); }
+
+ Policy policy() const {
+ return (Policy)((bits_ >> POLICY_SHIFT) & POLICY_MASK);
+ }
+ Type type() const { return (Type)((bits_ >> TYPE_SHIFT) & TYPE_MASK); }
+
+ static bool isFloatRegCompatible(Type type, FloatRegister reg) {
+ if (type == FLOAT32) {
+ return reg.isSingle();
+ }
+ if (type == DOUBLE) {
+ return reg.isDouble();
+ }
+ MOZ_ASSERT(type == SIMD128);
+ return reg.isSimd128();
+ }
+
+ bool isCompatibleReg(const AnyRegister& r) const {
+ if (isFloatReg() && r.isFloat()) {
+ return isFloatRegCompatible(type(), r.fpu());
+ }
+ return !isFloatReg() && !r.isFloat();
+ }
+ bool isCompatibleDef(const LDefinition& other) const {
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32)
+ if (isFloatReg() && other.isFloatReg()) {
+ return type() == other.type();
+ }
+ return !isFloatReg() && !other.isFloatReg();
+#else
+ return isFloatReg() == other.isFloatReg();
+#endif
+ }
+
+ static bool isFloatReg(Type type) {
+ return type == FLOAT32 || type == DOUBLE || type == SIMD128;
+ }
+ bool isFloatReg() const { return isFloatReg(type()); }
+
+ uint32_t virtualRegister() const {
+ uint32_t index = (bits_ >> VREG_SHIFT) & VREG_MASK;
+ // MOZ_ASSERT(index != 0);
+ return index;
+ }
+ LAllocation* output() { return &output_; }
+ const LAllocation* output() const { return &output_; }
+ bool isFixed() const { return policy() == FIXED; }
+ bool isBogusTemp() const { return isFixed() && output()->isBogus(); }
+ void setVirtualRegister(uint32_t index) {
+ MOZ_ASSERT(index < VREG_MASK);
+ bits_ &= ~(VREG_MASK << VREG_SHIFT);
+ bits_ |= index << VREG_SHIFT;
+ }
+ void setOutput(const LAllocation& a) {
+ output_ = a;
+ if (!a.isUse()) {
+ bits_ &= ~(POLICY_MASK << POLICY_SHIFT);
+ bits_ |= FIXED << POLICY_SHIFT;
+ }
+ }
+ void setReusedInput(uint32_t operand) {
+ output_ = LConstantIndex::FromIndex(operand);
+ }
+ uint32_t getReusedInput() const {
+ MOZ_ASSERT(policy() == LDefinition::MUST_REUSE_INPUT);
+ return output_.toConstantIndex()->index();
+ }
+
+ static inline Type TypeFrom(MIRType type) {
+ switch (type) {
+ case MIRType::Boolean:
+ case MIRType::Int32:
+ // The stack slot allocator doesn't currently support allocating
+ // 1-byte slots, so for now we lower MIRType::Boolean into INT32.
+ static_assert(sizeof(bool) <= sizeof(int32_t),
+ "bool doesn't fit in an int32 slot");
+ return LDefinition::INT32;
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ case MIRType::Object:
+ case MIRType::RefOrNull:
+ return LDefinition::OBJECT;
+ case MIRType::Double:
+ return LDefinition::DOUBLE;
+ case MIRType::Float32:
+ return LDefinition::FLOAT32;
+#if defined(JS_PUNBOX64)
+ case MIRType::Value:
+ return LDefinition::BOX;
+#endif
+ case MIRType::Slots:
+ case MIRType::Elements:
+ return LDefinition::SLOTS;
+ case MIRType::Pointer:
+ case MIRType::IntPtr:
+ return LDefinition::GENERAL;
+#if defined(JS_PUNBOX64)
+ case MIRType::Int64:
+ return LDefinition::GENERAL;
+#endif
+ case MIRType::StackResults:
+ return LDefinition::STACKRESULTS;
+ case MIRType::Simd128:
+ return LDefinition::SIMD128;
+ default:
+ MOZ_CRASH("unexpected type");
+ }
+ }
+
+ UniqueChars toString() const;
+
+#ifdef JS_JITSPEW
+ void dump() const;
+#endif
+};
+
+class LInt64Definition : public LInt64Value<LDefinition> {
+ public:
+ using LInt64Value<LDefinition>::LInt64Value;
+
+ static LInt64Definition BogusTemp() { return LInt64Definition(); }
+
+ bool isBogusTemp() const {
+#if JS_BITS_PER_WORD == 32
+ MOZ_ASSERT(high().isBogusTemp() == low().isBogusTemp());
+ return high().isBogusTemp();
+#else
+ return value().isBogusTemp();
+#endif
+ }
+};
+
+// Forward declarations of LIR types.
+#define LIROP(op) class L##op;
+LIR_OPCODE_LIST(LIROP)
+#undef LIROP
+
+class LSnapshot;
+class LSafepoint;
+class LElementVisitor;
+
+constexpr size_t MaxNumLInstructionOperands = 63;
+
+// The common base class for LPhi and LInstruction.
+class LNode {
+ protected:
+ MDefinition* mir_;
+
+ private:
+ LBlock* block_;
+ uint32_t id_;
+
+ protected:
+ // Bitfields below are all uint32_t to make sure MSVC packs them correctly.
+ uint32_t op_ : 10;
+ uint32_t isCall_ : 1;
+
+ // LPhi::numOperands() may not fit in this bitfield, so we only use this
+ // field for LInstruction.
+ uint32_t nonPhiNumOperands_ : 6;
+ static_assert((1 << 6) - 1 == MaxNumLInstructionOperands,
+ "packing constraints");
+
+ // For LInstruction, the first operand is stored at offset
+ // sizeof(LInstruction) + nonPhiOperandsOffset_ * sizeof(uintptr_t).
+ uint32_t nonPhiOperandsOffset_ : 5;
+ uint32_t numDefs_ : 4;
+ uint32_t numTemps_ : 4;
+
+ public:
+ enum class Opcode {
+#define LIROP(name) name,
+ LIR_OPCODE_LIST(LIROP)
+#undef LIROP
+ Invalid
+ };
+
+ LNode(Opcode op, uint32_t nonPhiNumOperands, uint32_t numDefs,
+ uint32_t numTemps)
+ : mir_(nullptr),
+ block_(nullptr),
+ id_(0),
+ op_(uint32_t(op)),
+ isCall_(false),
+ nonPhiNumOperands_(nonPhiNumOperands),
+ nonPhiOperandsOffset_(0),
+ numDefs_(numDefs),
+ numTemps_(numTemps) {
+ MOZ_ASSERT(op < Opcode::Invalid);
+ MOZ_ASSERT(op_ == uint32_t(op), "opcode must fit in bitfield");
+ MOZ_ASSERT(nonPhiNumOperands_ == nonPhiNumOperands,
+ "nonPhiNumOperands must fit in bitfield");
+ MOZ_ASSERT(numDefs_ == numDefs, "numDefs must fit in bitfield");
+ MOZ_ASSERT(numTemps_ == numTemps, "numTemps must fit in bitfield");
+ }
+
+ const char* opName() {
+ switch (op()) {
+#define LIR_NAME_INS(name) \
+ case Opcode::name: \
+ return #name;
+ LIR_OPCODE_LIST(LIR_NAME_INS)
+#undef LIR_NAME_INS
+ default:
+ MOZ_CRASH("Invalid op");
+ }
+ }
+
+ // Hook for opcodes to add extra high level detail about what code will be
+ // emitted for the op.
+ private:
+ const char* extraName() const { return nullptr; }
+
+ public:
+#ifdef JS_JITSPEW
+ const char* getExtraName() const;
+#endif
+
+ Opcode op() const { return Opcode(op_); }
+
+ bool isInstruction() const { return op() != Opcode::Phi; }
+ inline LInstruction* toInstruction();
+ inline const LInstruction* toInstruction() const;
+
+ // Returns the number of outputs of this instruction. If an output is
+ // unallocated, it is an LDefinition, defining a virtual register.
+ size_t numDefs() const { return numDefs_; }
+
+ bool isCall() const { return isCall_; }
+
+ // Does this call preserve the given register?
+ // By default, it is assumed that all registers are clobbered by a call.
+ inline bool isCallPreserved(AnyRegister reg) const;
+
+ uint32_t id() const { return id_; }
+ void setId(uint32_t id) {
+ MOZ_ASSERT(!id_);
+ MOZ_ASSERT(id);
+ id_ = id;
+ }
+ void setMir(MDefinition* mir) { mir_ = mir; }
+ MDefinition* mirRaw() const {
+ /* Untyped MIR for this op. Prefer mir() methods in subclasses. */
+ return mir_;
+ }
+ LBlock* block() const { return block_; }
+ void setBlock(LBlock* block) { block_ = block; }
+
+ // For an instruction which has a MUST_REUSE_INPUT output, whether that
+ // output register will be restored to its original value when bailing out.
+ inline bool recoversInput() const;
+
+#ifdef JS_JITSPEW
+ void dump(GenericPrinter& out);
+ void dump();
+ static void printName(GenericPrinter& out, Opcode op);
+ void printName(GenericPrinter& out);
+ void printOperands(GenericPrinter& out);
+#endif
+
+ public:
+ // Opcode testing and casts.
+#define LIROP(name) \
+ bool is##name() const { return op() == Opcode::name; } \
+ inline L##name* to##name(); \
+ inline const L##name* to##name() const;
+ LIR_OPCODE_LIST(LIROP)
+#undef LIROP
+
+// Note: GenerateOpcodeFiles.py generates LIROpsGenerated.h based on this
+// macro.
+#define LIR_HEADER(opcode) \
+ static constexpr LNode::Opcode classOpcode = LNode::Opcode::opcode;
+};
+
+extern const char* const LIROpNames[];
+inline const char* LIRCodeName(LNode::Opcode op) {
+ return LIROpNames[static_cast<size_t>(op)];
+}
+
+class LInstruction : public LNode,
+ public TempObject,
+ public InlineListNode<LInstruction> {
+ // This snapshot could be set after a ResumePoint. It is used to restart
+ // from the resume point pc.
+ LSnapshot* snapshot_;
+
+ // Structure capturing the set of stack slots and registers which are known
+ // to hold either gcthings or Values.
+ LSafepoint* safepoint_;
+
+ LMoveGroup* inputMoves_;
+ LMoveGroup* fixReuseMoves_;
+ LMoveGroup* movesAfter_;
+
+ protected:
+ LInstruction(Opcode opcode, uint32_t numOperands, uint32_t numDefs,
+ uint32_t numTemps)
+ : LNode(opcode, numOperands, numDefs, numTemps),
+ snapshot_(nullptr),
+ safepoint_(nullptr),
+ inputMoves_(nullptr),
+ fixReuseMoves_(nullptr),
+ movesAfter_(nullptr) {}
+
+ void setIsCall() { isCall_ = true; }
+
+ public:
+ inline LDefinition* getDef(size_t index);
+
+ void setDef(size_t index, const LDefinition& def) { *getDef(index) = def; }
+
+ LAllocation* getOperand(size_t index) const {
+ MOZ_ASSERT(index < numOperands());
+ MOZ_ASSERT(nonPhiOperandsOffset_ > 0);
+ uintptr_t p = reinterpret_cast<uintptr_t>(this + 1) +
+ nonPhiOperandsOffset_ * sizeof(uintptr_t);
+ return reinterpret_cast<LAllocation*>(p) + index;
+ }
+ void setOperand(size_t index, const LAllocation& a) {
+ *getOperand(index) = a;
+ }
+
+ void initOperandsOffset(size_t offset) {
+ MOZ_ASSERT(nonPhiOperandsOffset_ == 0);
+ MOZ_ASSERT(offset >= sizeof(LInstruction));
+ MOZ_ASSERT(((offset - sizeof(LInstruction)) % sizeof(uintptr_t)) == 0);
+ offset = (offset - sizeof(LInstruction)) / sizeof(uintptr_t);
+ nonPhiOperandsOffset_ = offset;
+ MOZ_ASSERT(nonPhiOperandsOffset_ == offset, "offset must fit in bitfield");
+ }
+
+ // Returns information about temporary registers needed. Each temporary
+ // register is an LDefinition with a fixed or virtual register and
+ // either GENERAL, FLOAT32, or DOUBLE type.
+ size_t numTemps() const { return numTemps_; }
+ inline LDefinition* getTemp(size_t index);
+
+ LSnapshot* snapshot() const { return snapshot_; }
+ LSafepoint* safepoint() const { return safepoint_; }
+ LMoveGroup* inputMoves() const { return inputMoves_; }
+ void setInputMoves(LMoveGroup* moves) { inputMoves_ = moves; }
+ LMoveGroup* fixReuseMoves() const { return fixReuseMoves_; }
+ void setFixReuseMoves(LMoveGroup* moves) { fixReuseMoves_ = moves; }
+ LMoveGroup* movesAfter() const { return movesAfter_; }
+ void setMovesAfter(LMoveGroup* moves) { movesAfter_ = moves; }
+ uint32_t numOperands() const { return nonPhiNumOperands_; }
+ void assignSnapshot(LSnapshot* snapshot);
+ void initSafepoint(TempAllocator& alloc);
+
+ class InputIterator;
+};
+
+LInstruction* LNode::toInstruction() {
+ MOZ_ASSERT(isInstruction());
+ return static_cast<LInstruction*>(this);
+}
+
+const LInstruction* LNode::toInstruction() const {
+ MOZ_ASSERT(isInstruction());
+ return static_cast<const LInstruction*>(this);
+}
+
+class LElementVisitor {
+#ifdef TRACK_SNAPSHOTS
+ LInstruction* ins_ = nullptr;
+#endif
+
+ protected:
+#ifdef TRACK_SNAPSHOTS
+ LInstruction* instruction() { return ins_; }
+
+ void setElement(LInstruction* ins) { ins_ = ins; }
+#else
+ void setElement(LInstruction* ins) {}
+#endif
+};
+
+using LInstructionIterator = InlineList<LInstruction>::iterator;
+using LInstructionReverseIterator = InlineList<LInstruction>::reverse_iterator;
+
+class MPhi;
+
+// Phi is a pseudo-instruction that emits no code, and is an annotation for the
+// register allocator. Like its equivalent in MIR, phis are collected at the
+// top of blocks and are meant to be executed in parallel, choosing the input
+// corresponding to the predecessor taken in the control flow graph.
+class LPhi final : public LNode {
+ LAllocation* const inputs_;
+ LDefinition def_;
+
+ public:
+ LIR_HEADER(Phi)
+
+ LPhi(MPhi* ins, LAllocation* inputs)
+ : LNode(classOpcode,
+ /* nonPhiNumOperands = */ 0,
+ /* numDefs = */ 1,
+ /* numTemps = */ 0),
+ inputs_(inputs) {
+ setMir(ins);
+ }
+
+ LDefinition* getDef(size_t index) {
+ MOZ_ASSERT(index == 0);
+ return &def_;
+ }
+ void setDef(size_t index, const LDefinition& def) {
+ MOZ_ASSERT(index == 0);
+ def_ = def;
+ }
+ size_t numOperands() const { return mir_->toPhi()->numOperands(); }
+ LAllocation* getOperand(size_t index) {
+ MOZ_ASSERT(index < numOperands());
+ return &inputs_[index];
+ }
+ void setOperand(size_t index, const LAllocation& a) {
+ MOZ_ASSERT(index < numOperands());
+ inputs_[index] = a;
+ }
+
+ // Phis don't have temps, so calling numTemps/getTemp is pointless.
+ size_t numTemps() const = delete;
+ LDefinition* getTemp(size_t index) = delete;
+};
+
+class LMoveGroup;
+class LBlock {
+ MBasicBlock* block_;
+ FixedList<LPhi> phis_;
+ InlineList<LInstruction> instructions_;
+ LMoveGroup* entryMoveGroup_;
+ LMoveGroup* exitMoveGroup_;
+ Label label_;
+
+ public:
+ explicit LBlock(MBasicBlock* block);
+ [[nodiscard]] bool init(TempAllocator& alloc);
+
+ void add(LInstruction* ins) {
+ ins->setBlock(this);
+ instructions_.pushBack(ins);
+ }
+ size_t numPhis() const { return phis_.length(); }
+ LPhi* getPhi(size_t index) { return &phis_[index]; }
+ const LPhi* getPhi(size_t index) const { return &phis_[index]; }
+ MBasicBlock* mir() const { return block_; }
+ LInstructionIterator begin() { return instructions_.begin(); }
+ LInstructionIterator begin(LInstruction* at) {
+ return instructions_.begin(at);
+ }
+ LInstructionIterator end() { return instructions_.end(); }
+ LInstructionReverseIterator rbegin() { return instructions_.rbegin(); }
+ LInstructionReverseIterator rbegin(LInstruction* at) {
+ return instructions_.rbegin(at);
+ }
+ LInstructionReverseIterator rend() { return instructions_.rend(); }
+ InlineList<LInstruction>& instructions() { return instructions_; }
+ void insertAfter(LInstruction* at, LInstruction* ins) {
+ instructions_.insertAfter(at, ins);
+ }
+ void insertBefore(LInstruction* at, LInstruction* ins) {
+ instructions_.insertBefore(at, ins);
+ }
+ const LNode* firstElementWithId() const {
+ return !phis_.empty() ? static_cast<const LNode*>(getPhi(0))
+ : firstInstructionWithId();
+ }
+ uint32_t firstId() const { return firstElementWithId()->id(); }
+ uint32_t lastId() const { return lastInstructionWithId()->id(); }
+ const LInstruction* firstInstructionWithId() const;
+ const LInstruction* lastInstructionWithId() const {
+ const LInstruction* last = *instructions_.rbegin();
+ MOZ_ASSERT(last->id());
+ // The last instruction is a control flow instruction which does not have
+ // any output.
+ MOZ_ASSERT(last->numDefs() == 0);
+ return last;
+ }
+
+ // Return the label to branch to when branching to this block.
+ Label* label() {
+ MOZ_ASSERT(!isTrivial());
+ return &label_;
+ }
+
+ LMoveGroup* getEntryMoveGroup(TempAllocator& alloc);
+ LMoveGroup* getExitMoveGroup(TempAllocator& alloc);
+
+ // Test whether this basic block is empty except for a simple goto, and
+ // which is not forming a loop. No code will be emitted for such blocks.
+ bool isTrivial() { return begin()->isGoto() && !mir()->isLoopHeader(); }
+
+#ifdef JS_JITSPEW
+ void dump(GenericPrinter& out);
+ void dump();
+#endif
+};
+
+namespace details {
+template <size_t Defs, size_t Temps>
+class LInstructionFixedDefsTempsHelper : public LInstruction {
+ mozilla::Array<LDefinition, Defs + Temps> defsAndTemps_;
+
+ protected:
+ LInstructionFixedDefsTempsHelper(Opcode opcode, uint32_t numOperands)
+ : LInstruction(opcode, numOperands, Defs, Temps) {}
+
+ public:
+ // Override the methods in LInstruction with more optimized versions
+ // for when we know the exact instruction type.
+ LDefinition* getDef(size_t index) {
+ MOZ_ASSERT(index < Defs);
+ return &defsAndTemps_[index];
+ }
+ LDefinition* getTemp(size_t index) {
+ MOZ_ASSERT(index < Temps);
+ return &defsAndTemps_[Defs + index];
+ }
+ LInt64Definition getInt64Temp(size_t index) {
+ MOZ_ASSERT(index + INT64_PIECES <= Temps);
+#if JS_BITS_PER_WORD == 32
+ return LInt64Definition(defsAndTemps_[Defs + index + INT64HIGH_INDEX],
+ defsAndTemps_[Defs + index + INT64LOW_INDEX]);
+#else
+ return LInt64Definition(defsAndTemps_[Defs + index]);
+#endif
+ }
+
+ void setDef(size_t index, const LDefinition& def) {
+ MOZ_ASSERT(index < Defs);
+ defsAndTemps_[index] = def;
+ }
+ void setTemp(size_t index, const LDefinition& a) {
+ MOZ_ASSERT(index < Temps);
+ defsAndTemps_[Defs + index] = a;
+ }
+ void setInt64Temp(size_t index, const LInt64Definition& a) {
+#if JS_BITS_PER_WORD == 32
+ setTemp(index, a.low());
+ setTemp(index + 1, a.high());
+#else
+ setTemp(index, a.value());
+#endif
+ }
+
+ // Default accessors, assuming a single input and output, respectively.
+ const LAllocation* input() {
+ MOZ_ASSERT(numOperands() == 1);
+ return getOperand(0);
+ }
+ const LDefinition* output() {
+ MOZ_ASSERT(numDefs() == 1);
+ return getDef(0);
+ }
+ static size_t offsetOfDef(size_t index) {
+ using T = LInstructionFixedDefsTempsHelper<0, 0>;
+ return offsetof(T, defsAndTemps_) + index * sizeof(LDefinition);
+ }
+ static size_t offsetOfTemp(uint32_t numDefs, uint32_t index) {
+ using T = LInstructionFixedDefsTempsHelper<0, 0>;
+ return offsetof(T, defsAndTemps_) + (numDefs + index) * sizeof(LDefinition);
+ }
+};
+} // namespace details
+
+inline LDefinition* LInstruction::getDef(size_t index) {
+ MOZ_ASSERT(index < numDefs());
+ using T = details::LInstructionFixedDefsTempsHelper<0, 0>;
+ uint8_t* p = reinterpret_cast<uint8_t*>(this) + T::offsetOfDef(index);
+ return reinterpret_cast<LDefinition*>(p);
+}
+
+inline LDefinition* LInstruction::getTemp(size_t index) {
+ MOZ_ASSERT(index < numTemps());
+ using T = details::LInstructionFixedDefsTempsHelper<0, 0>;
+ uint8_t* p =
+ reinterpret_cast<uint8_t*>(this) + T::offsetOfTemp(numDefs(), index);
+ return reinterpret_cast<LDefinition*>(p);
+}
+
+template <size_t Defs, size_t Operands, size_t Temps>
+class LInstructionHelper
+ : public details::LInstructionFixedDefsTempsHelper<Defs, Temps> {
+ mozilla::Array<LAllocation, Operands> operands_;
+
+ protected:
+ explicit LInstructionHelper(LNode::Opcode opcode)
+ : details::LInstructionFixedDefsTempsHelper<Defs, Temps>(opcode,
+ Operands) {
+ static_assert(
+ Operands == 0 || sizeof(operands_) == Operands * sizeof(LAllocation),
+ "mozilla::Array should not contain other fields");
+ if (Operands > 0) {
+ using T = LInstructionHelper<Defs, Operands, Temps>;
+ this->initOperandsOffset(offsetof(T, operands_));
+ }
+ }
+
+ public:
+ // Override the methods in LInstruction with more optimized versions
+ // for when we know the exact instruction type.
+ LAllocation* getOperand(size_t index) { return &operands_[index]; }
+ void setOperand(size_t index, const LAllocation& a) { operands_[index] = a; }
+ void setBoxOperand(size_t index, const LBoxAllocation& alloc) {
+#ifdef JS_NUNBOX32
+ operands_[index + TYPE_INDEX] = alloc.type();
+ operands_[index + PAYLOAD_INDEX] = alloc.payload();
+#else
+ operands_[index] = alloc.value();
+#endif
+ }
+ void setInt64Operand(size_t index, const LInt64Allocation& alloc) {
+#if JS_BITS_PER_WORD == 32
+ operands_[index + INT64LOW_INDEX] = alloc.low();
+ operands_[index + INT64HIGH_INDEX] = alloc.high();
+#else
+ operands_[index] = alloc.value();
+#endif
+ }
+ const LInt64Allocation getInt64Operand(size_t offset) {
+#if JS_BITS_PER_WORD == 32
+ return LInt64Allocation(operands_[offset + INT64HIGH_INDEX],
+ operands_[offset + INT64LOW_INDEX]);
+#else
+ return LInt64Allocation(operands_[offset]);
+#endif
+ }
+};
+
+template <size_t Defs, size_t Temps>
+class LVariadicInstruction
+ : public details::LInstructionFixedDefsTempsHelper<Defs, Temps> {
+ protected:
+ LVariadicInstruction(LNode::Opcode opcode, size_t numOperands)
+ : details::LInstructionFixedDefsTempsHelper<Defs, Temps>(opcode,
+ numOperands) {}
+
+ public:
+ void setBoxOperand(size_t index, const LBoxAllocation& a) {
+#ifdef JS_NUNBOX32
+ this->setOperand(index + TYPE_INDEX, a.type());
+ this->setOperand(index + PAYLOAD_INDEX, a.payload());
+#else
+ this->setOperand(index, a.value());
+#endif
+ }
+};
+
+template <size_t Defs, size_t Operands, size_t Temps>
+class LCallInstructionHelper
+ : public LInstructionHelper<Defs, Operands, Temps> {
+ protected:
+ explicit LCallInstructionHelper(LNode::Opcode opcode)
+ : LInstructionHelper<Defs, Operands, Temps>(opcode) {
+ this->setIsCall();
+ }
+};
+
+template <size_t Defs, size_t Temps>
+class LBinaryCallInstructionHelper
+ : public LCallInstructionHelper<Defs, 2, Temps> {
+ protected:
+ explicit LBinaryCallInstructionHelper(LNode::Opcode opcode)
+ : LCallInstructionHelper<Defs, 2, Temps>(opcode) {}
+
+ public:
+ const LAllocation* lhs() { return this->getOperand(0); }
+ const LAllocation* rhs() { return this->getOperand(1); }
+};
+
+class LRecoverInfo : public TempObject {
+ public:
+ typedef Vector<MNode*, 2, JitAllocPolicy> Instructions;
+
+ private:
+ // List of instructions needed to recover the stack frames.
+ // Outer frames are stored before inner frames.
+ Instructions instructions_;
+
+ // Cached offset where this resume point is encoded.
+ RecoverOffset recoverOffset_;
+
+ explicit LRecoverInfo(TempAllocator& alloc);
+ [[nodiscard]] bool init(MResumePoint* mir);
+
+ // Fill the instruction vector such as all instructions needed for the
+ // recovery are pushed before the current instruction.
+ template <typename Node>
+ [[nodiscard]] bool appendOperands(Node* ins);
+ [[nodiscard]] bool appendDefinition(MDefinition* def);
+ [[nodiscard]] bool appendResumePoint(MResumePoint* rp);
+
+ public:
+ static LRecoverInfo* New(MIRGenerator* gen, MResumePoint* mir);
+
+ // Resume point of the inner most function.
+ MResumePoint* mir() const { return instructions_.back()->toResumePoint(); }
+ RecoverOffset recoverOffset() const { return recoverOffset_; }
+ void setRecoverOffset(RecoverOffset offset) {
+ MOZ_ASSERT(recoverOffset_ == INVALID_RECOVER_OFFSET);
+ recoverOffset_ = offset;
+ }
+
+ MNode** begin() { return instructions_.begin(); }
+ MNode** end() { return instructions_.end(); }
+ size_t numInstructions() const { return instructions_.length(); }
+
+ class OperandIter {
+ private:
+ MNode** it_;
+ MNode** end_;
+ size_t op_;
+ size_t opEnd_;
+ MResumePoint* rp_;
+ MNode* node_;
+
+ public:
+ explicit OperandIter(LRecoverInfo* recoverInfo)
+ : it_(recoverInfo->begin()),
+ end_(recoverInfo->end()),
+ op_(0),
+ opEnd_(0),
+ rp_(nullptr),
+ node_(nullptr) {
+ settle();
+ }
+
+ void settle() {
+ opEnd_ = (*it_)->numOperands();
+ while (opEnd_ == 0) {
+ ++it_;
+ op_ = 0;
+ opEnd_ = (*it_)->numOperands();
+ }
+ node_ = *it_;
+ if (node_->isResumePoint()) {
+ rp_ = node_->toResumePoint();
+ }
+ }
+
+ MDefinition* operator*() {
+ if (rp_) { // de-virtualize MResumePoint::getOperand calls.
+ return rp_->getOperand(op_);
+ }
+ return node_->getOperand(op_);
+ }
+ MDefinition* operator->() {
+ if (rp_) { // de-virtualize MResumePoint::getOperand calls.
+ return rp_->getOperand(op_);
+ }
+ return node_->getOperand(op_);
+ }
+
+ OperandIter& operator++() {
+ ++op_;
+ if (op_ != opEnd_) {
+ return *this;
+ }
+ op_ = 0;
+ ++it_;
+ node_ = rp_ = nullptr;
+ if (!*this) {
+ settle();
+ }
+ return *this;
+ }
+
+ explicit operator bool() const { return it_ == end_; }
+
+#ifdef DEBUG
+ bool canOptimizeOutIfUnused();
+#endif
+ };
+};
+
+// An LSnapshot is the reflection of an MResumePoint in LIR. Unlike
+// MResumePoints, they cannot be shared, as they are filled in by the register
+// allocator in order to capture the precise low-level stack state in between an
+// instruction's input and output. During code generation, LSnapshots are
+// compressed and saved in the compiled script.
+class LSnapshot : public TempObject {
+ private:
+ LAllocation* slots_;
+ LRecoverInfo* recoverInfo_;
+ SnapshotOffset snapshotOffset_;
+ uint32_t numSlots_;
+ BailoutKind bailoutKind_;
+
+ LSnapshot(LRecoverInfo* recover, BailoutKind kind);
+ [[nodiscard]] bool init(MIRGenerator* gen);
+
+ public:
+ static LSnapshot* New(MIRGenerator* gen, LRecoverInfo* recover,
+ BailoutKind kind);
+
+ size_t numEntries() const { return numSlots_; }
+ size_t numSlots() const { return numSlots_ / BOX_PIECES; }
+ LAllocation* payloadOfSlot(size_t i) {
+ MOZ_ASSERT(i < numSlots());
+ size_t entryIndex = (i * BOX_PIECES) + (BOX_PIECES - 1);
+ return getEntry(entryIndex);
+ }
+#ifdef JS_NUNBOX32
+ LAllocation* typeOfSlot(size_t i) {
+ MOZ_ASSERT(i < numSlots());
+ size_t entryIndex = (i * BOX_PIECES) + (BOX_PIECES - 2);
+ return getEntry(entryIndex);
+ }
+#endif
+ LAllocation* getEntry(size_t i) {
+ MOZ_ASSERT(i < numSlots_);
+ return &slots_[i];
+ }
+ void setEntry(size_t i, const LAllocation& alloc) {
+ MOZ_ASSERT(i < numSlots_);
+ slots_[i] = alloc;
+ }
+ LRecoverInfo* recoverInfo() const { return recoverInfo_; }
+ MResumePoint* mir() const { return recoverInfo()->mir(); }
+ SnapshotOffset snapshotOffset() const { return snapshotOffset_; }
+ void setSnapshotOffset(SnapshotOffset offset) {
+ MOZ_ASSERT(snapshotOffset_ == INVALID_SNAPSHOT_OFFSET);
+ snapshotOffset_ = offset;
+ }
+ BailoutKind bailoutKind() const { return bailoutKind_; }
+ void rewriteRecoveredInput(LUse input);
+};
+
+struct SafepointSlotEntry {
+ // Flag indicating whether this is a slot in the stack or argument space.
+ uint32_t stack : 1;
+
+ // Byte offset of the slot, as in LStackSlot or LArgument.
+ uint32_t slot : 31;
+
+ SafepointSlotEntry() : stack(0), slot(0) {}
+ SafepointSlotEntry(bool stack, uint32_t slot) : stack(stack), slot(slot) {}
+ explicit SafepointSlotEntry(const LAllocation* a)
+ : stack(a->isStackSlot()), slot(a->memorySlot()) {}
+};
+
+struct SafepointNunboxEntry {
+ uint32_t typeVreg;
+ LAllocation type;
+ LAllocation payload;
+
+ SafepointNunboxEntry() : typeVreg(0) {}
+ SafepointNunboxEntry(uint32_t typeVreg, LAllocation type, LAllocation payload)
+ : typeVreg(typeVreg), type(type), payload(payload) {}
+};
+
+class LSafepoint : public TempObject {
+ using SlotEntry = SafepointSlotEntry;
+ using NunboxEntry = SafepointNunboxEntry;
+
+ public:
+ typedef Vector<SlotEntry, 0, JitAllocPolicy> SlotList;
+ typedef Vector<NunboxEntry, 0, JitAllocPolicy> NunboxList;
+
+ private:
+ // The information in a safepoint describes the registers and gc related
+ // values that are live at the start of the associated instruction.
+
+ // The set of registers which are live at an OOL call made within the
+ // instruction. This includes any registers for inputs which are not
+ // use-at-start, any registers for temps, and any registers live after the
+ // call except outputs of the instruction.
+ //
+ // For call instructions, the live regs are empty. Call instructions may
+ // have register inputs or temporaries, which will *not* be in the live
+ // registers: if passed to the call, the values passed will be marked via
+ // TraceJitExitFrame, and no registers can be live after the instruction
+ // except its outputs.
+ LiveRegisterSet liveRegs_;
+
+ // The subset of liveRegs which contains gcthing pointers.
+ LiveGeneralRegisterSet gcRegs_;
+
+#ifdef CHECK_OSIPOINT_REGISTERS
+ // Clobbered regs of the current instruction. This set is never written to
+ // the safepoint; it's only used by assertions during compilation.
+ LiveRegisterSet clobberedRegs_;
+#endif
+
+ // Offset to a position in the safepoint stream, or
+ // INVALID_SAFEPOINT_OFFSET.
+ uint32_t safepointOffset_;
+
+ // Assembler buffer displacement to OSI point's call location.
+ uint32_t osiCallPointOffset_;
+
+ // List of slots which have gcthing pointers.
+ SlotList gcSlots_;
+
+#ifdef JS_NUNBOX32
+ // List of registers (in liveRegs) and slots which contain pieces of Values.
+ NunboxList nunboxParts_;
+#elif JS_PUNBOX64
+ // List of slots which have Values.
+ SlotList valueSlots_;
+
+ // The subset of liveRegs which have Values.
+ LiveGeneralRegisterSet valueRegs_;
+#endif
+
+ // The subset of liveRegs which contains pointers to slots/elements.
+ LiveGeneralRegisterSet slotsOrElementsRegs_;
+
+ // List of slots which have slots/elements pointers.
+ SlotList slotsOrElementsSlots_;
+
+ // Wasm only: with what kind of instruction is this LSafepoint associated?
+ // true => wasm trap, false => wasm call.
+ bool isWasmTrap_;
+
+ // Wasm only: what is the value of masm.framePushed() that corresponds to
+ // the lowest-addressed word covered by the StackMap that we will generate
+ // from this LSafepoint? This depends on the instruction:
+ //
+ // if isWasmTrap_ == true:
+ // masm.framePushed() unmodified. Note that when constructing the
+ // StackMap we will add entries below this point to take account of
+ // registers dumped on the stack as a result of the trap.
+ //
+ // if isWasmTrap_ == false:
+ // masm.framePushed() - StackArgAreaSizeUnaligned(arg types for the call),
+ // because the map does not include the outgoing args themselves, but
+ // it does cover any and all alignment space above them.
+ uint32_t framePushedAtStackMapBase_;
+
+ public:
+ void assertInvariants() {
+ // Every register in valueRegs and gcRegs should also be in liveRegs.
+#ifndef JS_NUNBOX32
+ MOZ_ASSERT((valueRegs().bits() & ~liveRegs().gprs().bits()) == 0);
+#endif
+ MOZ_ASSERT((gcRegs().bits() & ~liveRegs().gprs().bits()) == 0);
+ }
+
+ explicit LSafepoint(TempAllocator& alloc)
+ : safepointOffset_(INVALID_SAFEPOINT_OFFSET),
+ osiCallPointOffset_(0),
+ gcSlots_(alloc),
+#ifdef JS_NUNBOX32
+ nunboxParts_(alloc),
+#else
+ valueSlots_(alloc),
+#endif
+ slotsOrElementsSlots_(alloc),
+ isWasmTrap_(false),
+ framePushedAtStackMapBase_(0) {
+ assertInvariants();
+ }
+ void addLiveRegister(AnyRegister reg) {
+ liveRegs_.addUnchecked(reg);
+ assertInvariants();
+ }
+ const LiveRegisterSet& liveRegs() const { return liveRegs_; }
+#ifdef CHECK_OSIPOINT_REGISTERS
+ void addClobberedRegister(AnyRegister reg) {
+ clobberedRegs_.addUnchecked(reg);
+ assertInvariants();
+ }
+ const LiveRegisterSet& clobberedRegs() const { return clobberedRegs_; }
+#endif
+ void addGcRegister(Register reg) {
+ gcRegs_.addUnchecked(reg);
+ assertInvariants();
+ }
+ LiveGeneralRegisterSet gcRegs() const { return gcRegs_; }
+ [[nodiscard]] bool addGcSlot(bool stack, uint32_t slot) {
+ bool result = gcSlots_.append(SlotEntry(stack, slot));
+ if (result) {
+ assertInvariants();
+ }
+ return result;
+ }
+ SlotList& gcSlots() { return gcSlots_; }
+
+ SlotList& slotsOrElementsSlots() { return slotsOrElementsSlots_; }
+ LiveGeneralRegisterSet slotsOrElementsRegs() const {
+ return slotsOrElementsRegs_;
+ }
+ void addSlotsOrElementsRegister(Register reg) {
+ slotsOrElementsRegs_.addUnchecked(reg);
+ assertInvariants();
+ }
+ [[nodiscard]] bool addSlotsOrElementsSlot(bool stack, uint32_t slot) {
+ bool result = slotsOrElementsSlots_.append(SlotEntry(stack, slot));
+ if (result) {
+ assertInvariants();
+ }
+ return result;
+ }
+ [[nodiscard]] bool addSlotsOrElementsPointer(LAllocation alloc) {
+ if (alloc.isMemory()) {
+ return addSlotsOrElementsSlot(alloc.isStackSlot(), alloc.memorySlot());
+ }
+ MOZ_ASSERT(alloc.isRegister());
+ addSlotsOrElementsRegister(alloc.toRegister().gpr());
+ assertInvariants();
+ return true;
+ }
+ bool hasSlotsOrElementsPointer(LAllocation alloc) const {
+ if (alloc.isRegister()) {
+ return slotsOrElementsRegs().has(alloc.toRegister().gpr());
+ }
+ for (size_t i = 0; i < slotsOrElementsSlots_.length(); i++) {
+ const SlotEntry& entry = slotsOrElementsSlots_[i];
+ if (entry.stack == alloc.isStackSlot() &&
+ entry.slot == alloc.memorySlot()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ [[nodiscard]] bool addGcPointer(LAllocation alloc) {
+ if (alloc.isMemory()) {
+ return addGcSlot(alloc.isStackSlot(), alloc.memorySlot());
+ }
+ if (alloc.isRegister()) {
+ addGcRegister(alloc.toRegister().gpr());
+ }
+ assertInvariants();
+ return true;
+ }
+
+ bool hasGcPointer(LAllocation alloc) const {
+ if (alloc.isRegister()) {
+ return gcRegs().has(alloc.toRegister().gpr());
+ }
+ MOZ_ASSERT(alloc.isMemory());
+ for (size_t i = 0; i < gcSlots_.length(); i++) {
+ if (gcSlots_[i].stack == alloc.isStackSlot() &&
+ gcSlots_[i].slot == alloc.memorySlot()) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Return true if all GC-managed pointers from `alloc` are recorded in this
+ // safepoint.
+ bool hasAllGcPointersFromStackArea(LAllocation alloc) const {
+ for (LStackArea::ResultIterator iter = alloc.toStackArea()->results(); iter;
+ iter.next()) {
+ if (iter.isGcPointer() && !hasGcPointer(iter.alloc())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+#ifdef JS_NUNBOX32
+ [[nodiscard]] bool addNunboxParts(uint32_t typeVreg, LAllocation type,
+ LAllocation payload) {
+ bool result = nunboxParts_.append(NunboxEntry(typeVreg, type, payload));
+ if (result) {
+ assertInvariants();
+ }
+ return result;
+ }
+
+ [[nodiscard]] bool addNunboxType(uint32_t typeVreg, LAllocation type) {
+ for (size_t i = 0; i < nunboxParts_.length(); i++) {
+ if (nunboxParts_[i].type == type) {
+ return true;
+ }
+ if (nunboxParts_[i].type == LUse(typeVreg, LUse::ANY)) {
+ nunboxParts_[i].type = type;
+ return true;
+ }
+ }
+
+ // vregs for nunbox pairs are adjacent, with the type coming first.
+ uint32_t payloadVreg = typeVreg + 1;
+ bool result = nunboxParts_.append(
+ NunboxEntry(typeVreg, type, LUse(payloadVreg, LUse::ANY)));
+ if (result) {
+ assertInvariants();
+ }
+ return result;
+ }
+
+ [[nodiscard]] bool addNunboxPayload(uint32_t payloadVreg,
+ LAllocation payload) {
+ for (size_t i = 0; i < nunboxParts_.length(); i++) {
+ if (nunboxParts_[i].payload == payload) {
+ return true;
+ }
+ if (nunboxParts_[i].payload == LUse(payloadVreg, LUse::ANY)) {
+ nunboxParts_[i].payload = payload;
+ return true;
+ }
+ }
+
+ // vregs for nunbox pairs are adjacent, with the type coming first.
+ uint32_t typeVreg = payloadVreg - 1;
+ bool result = nunboxParts_.append(
+ NunboxEntry(typeVreg, LUse(typeVreg, LUse::ANY), payload));
+ if (result) {
+ assertInvariants();
+ }
+ return result;
+ }
+
+ LAllocation findTypeAllocation(uint32_t typeVreg) {
+ // Look for some allocation for the specified type vreg, to go with a
+ // partial nunbox entry for the payload. Note that we don't need to
+ // look at the value slots in the safepoint, as these aren't used by
+ // register allocators which add partial nunbox entries.
+ for (size_t i = 0; i < nunboxParts_.length(); i++) {
+ if (nunboxParts_[i].typeVreg == typeVreg &&
+ !nunboxParts_[i].type.isUse()) {
+ return nunboxParts_[i].type;
+ }
+ }
+ return LUse(typeVreg, LUse::ANY);
+ }
+
+# ifdef DEBUG
+ bool hasNunboxPayload(LAllocation payload) const {
+ for (size_t i = 0; i < nunboxParts_.length(); i++) {
+ if (nunboxParts_[i].payload == payload) {
+ return true;
+ }
+ }
+ return false;
+ }
+# endif
+
+ NunboxList& nunboxParts() { return nunboxParts_; }
+
+#elif JS_PUNBOX64
+ [[nodiscard]] bool addValueSlot(bool stack, uint32_t slot) {
+ bool result = valueSlots_.append(SlotEntry(stack, slot));
+ if (result) {
+ assertInvariants();
+ }
+ return result;
+ }
+ SlotList& valueSlots() { return valueSlots_; }
+
+ bool hasValueSlot(bool stack, uint32_t slot) const {
+ for (size_t i = 0; i < valueSlots_.length(); i++) {
+ if (valueSlots_[i].stack == stack && valueSlots_[i].slot == slot) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ void addValueRegister(Register reg) {
+ valueRegs_.add(reg);
+ assertInvariants();
+ }
+ LiveGeneralRegisterSet valueRegs() const { return valueRegs_; }
+
+ [[nodiscard]] bool addBoxedValue(LAllocation alloc) {
+ if (alloc.isRegister()) {
+ Register reg = alloc.toRegister().gpr();
+ if (!valueRegs().has(reg)) {
+ addValueRegister(reg);
+ }
+ return true;
+ }
+ if (hasValueSlot(alloc.isStackSlot(), alloc.memorySlot())) {
+ return true;
+ }
+ return addValueSlot(alloc.isStackSlot(), alloc.memorySlot());
+ }
+
+ bool hasBoxedValue(LAllocation alloc) const {
+ if (alloc.isRegister()) {
+ return valueRegs().has(alloc.toRegister().gpr());
+ }
+ return hasValueSlot(alloc.isStackSlot(), alloc.memorySlot());
+ }
+
+#endif // JS_PUNBOX64
+
+ bool encoded() const { return safepointOffset_ != INVALID_SAFEPOINT_OFFSET; }
+ uint32_t offset() const {
+ MOZ_ASSERT(encoded());
+ return safepointOffset_;
+ }
+ void setOffset(uint32_t offset) { safepointOffset_ = offset; }
+ uint32_t osiReturnPointOffset() const {
+ // In general, pointer arithmetic on code is bad, but in this case,
+ // getting the return address from a call instruction, stepping over pools
+ // would be wrong.
+ return osiCallPointOffset_ + Assembler::PatchWrite_NearCallSize();
+ }
+ uint32_t osiCallPointOffset() const { return osiCallPointOffset_; }
+ void setOsiCallPointOffset(uint32_t osiCallPointOffset) {
+ MOZ_ASSERT(!osiCallPointOffset_);
+ osiCallPointOffset_ = osiCallPointOffset;
+ }
+
+ bool isWasmTrap() const { return isWasmTrap_; }
+ void setIsWasmTrap() { isWasmTrap_ = true; }
+
+ uint32_t framePushedAtStackMapBase() const {
+ return framePushedAtStackMapBase_;
+ }
+ void setFramePushedAtStackMapBase(uint32_t n) {
+ MOZ_ASSERT(framePushedAtStackMapBase_ == 0);
+ framePushedAtStackMapBase_ = n;
+ }
+};
+
+class LInstruction::InputIterator {
+ private:
+ LInstruction& ins_;
+ size_t idx_;
+ bool snapshot_;
+
+ void handleOperandsEnd() {
+ // Iterate on the snapshot when iteration over all operands is done.
+ if (!snapshot_ && idx_ == ins_.numOperands() && ins_.snapshot()) {
+ idx_ = 0;
+ snapshot_ = true;
+ }
+ }
+
+ public:
+ explicit InputIterator(LInstruction& ins)
+ : ins_(ins), idx_(0), snapshot_(false) {
+ handleOperandsEnd();
+ }
+
+ bool more() const {
+ if (snapshot_) {
+ return idx_ < ins_.snapshot()->numEntries();
+ }
+ if (idx_ < ins_.numOperands()) {
+ return true;
+ }
+ if (ins_.snapshot() && ins_.snapshot()->numEntries()) {
+ return true;
+ }
+ return false;
+ }
+
+ bool isSnapshotInput() const { return snapshot_; }
+
+ void next() {
+ MOZ_ASSERT(more());
+ idx_++;
+ handleOperandsEnd();
+ }
+
+ void replace(const LAllocation& alloc) {
+ if (snapshot_) {
+ ins_.snapshot()->setEntry(idx_, alloc);
+ } else {
+ ins_.setOperand(idx_, alloc);
+ }
+ }
+
+ LAllocation* operator*() const {
+ if (snapshot_) {
+ return ins_.snapshot()->getEntry(idx_);
+ }
+ return ins_.getOperand(idx_);
+ }
+
+ LAllocation* operator->() const { return **this; }
+};
+
+class LIRGraph {
+ struct ValueHasher {
+ using Lookup = Value;
+ static HashNumber hash(const Value& v) { return HashNumber(v.asRawBits()); }
+ static bool match(const Value& lhs, const Value& rhs) { return lhs == rhs; }
+ };
+
+ FixedList<LBlock> blocks_;
+
+ // constantPool_ is a mozilla::Vector, not a js::Vector, because
+ // js::Vector<Value> is prohibited as unsafe. This particular Vector of
+ // Values is safe because it is only used within the scope of an
+ // AutoSuppressGC (in IonCompile), which inhibits GC.
+ mozilla::Vector<Value, 0, JitAllocPolicy> constantPool_;
+ typedef HashMap<Value, uint32_t, ValueHasher, JitAllocPolicy> ConstantPoolMap;
+ ConstantPoolMap constantPoolMap_;
+ Vector<LInstruction*, 0, JitAllocPolicy> safepoints_;
+ Vector<LInstruction*, 0, JitAllocPolicy> nonCallSafepoints_;
+ uint32_t numVirtualRegisters_;
+ uint32_t numInstructions_;
+
+ // Size of stack slots needed for local spills.
+ uint32_t localSlotsSize_;
+ // Number of JS::Value stack slots needed for argument construction for calls.
+ uint32_t argumentSlotCount_;
+
+ MIRGraph& mir_;
+
+ public:
+ explicit LIRGraph(MIRGraph* mir);
+
+ [[nodiscard]] bool init() {
+ return blocks_.init(mir_.alloc(), mir_.numBlocks());
+ }
+ MIRGraph& mir() const { return mir_; }
+ size_t numBlocks() const { return blocks_.length(); }
+ LBlock* getBlock(size_t i) { return &blocks_[i]; }
+ uint32_t numBlockIds() const { return mir_.numBlockIds(); }
+ [[nodiscard]] bool initBlock(MBasicBlock* mir) {
+ auto* block = &blocks_[mir->id()];
+ auto* lir = new (block) LBlock(mir);
+ return lir->init(mir_.alloc());
+ }
+ uint32_t getVirtualRegister() {
+ numVirtualRegisters_ += VREG_INCREMENT;
+ return numVirtualRegisters_;
+ }
+ uint32_t numVirtualRegisters() const {
+ // Virtual registers are 1-based, not 0-based, so add one as a
+ // convenience for 0-based arrays.
+ return numVirtualRegisters_ + 1;
+ }
+ uint32_t getInstructionId() { return numInstructions_++; }
+ uint32_t numInstructions() const { return numInstructions_; }
+ void setLocalSlotsSize(uint32_t localSlotsSize) {
+ localSlotsSize_ = localSlotsSize;
+ }
+ uint32_t localSlotsSize() const { return localSlotsSize_; }
+ void setArgumentSlotCount(uint32_t argumentSlotCount) {
+ argumentSlotCount_ = argumentSlotCount;
+ }
+ uint32_t argumentSlotCount() const { return argumentSlotCount_; }
+ [[nodiscard]] bool addConstantToPool(const Value& v, uint32_t* index);
+ size_t numConstants() const { return constantPool_.length(); }
+ Value* constantPool() { return &constantPool_[0]; }
+
+ bool noteNeedsSafepoint(LInstruction* ins);
+ size_t numNonCallSafepoints() const { return nonCallSafepoints_.length(); }
+ LInstruction* getNonCallSafepoint(size_t i) const {
+ return nonCallSafepoints_[i];
+ }
+ size_t numSafepoints() const { return safepoints_.length(); }
+ LInstruction* getSafepoint(size_t i) const { return safepoints_[i]; }
+
+#ifdef JS_JITSPEW
+ void dump(GenericPrinter& out);
+ void dump();
+#endif
+};
+
+LAllocation::LAllocation(AnyRegister reg) {
+ if (reg.isFloat()) {
+ *this = LFloatReg(reg.fpu());
+ } else {
+ *this = LGeneralReg(reg.gpr());
+ }
+}
+
+AnyRegister LAllocation::toRegister() const {
+ MOZ_ASSERT(isRegister());
+ if (isFloatReg()) {
+ return AnyRegister(toFloatReg()->reg());
+ }
+ return AnyRegister(toGeneralReg()->reg());
+}
+
+} // namespace jit
+} // namespace js
+
+#include "jit/shared/LIR-shared.h"
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+# if defined(JS_CODEGEN_X86)
+# include "jit/x86/LIR-x86.h"
+# elif defined(JS_CODEGEN_X64)
+# include "jit/x64/LIR-x64.h"
+# endif
+# include "jit/x86-shared/LIR-x86-shared.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/LIR-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/LIR-arm64.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/LIR-loong64.h"
+#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+# if defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/LIR-mips32.h"
+# elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/LIR-mips64.h"
+# endif
+# include "jit/mips-shared/LIR-mips-shared.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/LIR-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/LIR-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+#undef LIR_HEADER
+
+namespace js {
+namespace jit {
+
+#define LIROP(name) \
+ L##name* LNode::to##name() { \
+ MOZ_ASSERT(is##name()); \
+ return static_cast<L##name*>(this); \
+ } \
+ const L##name* LNode::to##name() const { \
+ MOZ_ASSERT(is##name()); \
+ return static_cast<const L##name*>(this); \
+ }
+LIR_OPCODE_LIST(LIROP)
+#undef LIROP
+
+#define LALLOC_CAST(type) \
+ L##type* LAllocation::to##type() { \
+ MOZ_ASSERT(is##type()); \
+ return static_cast<L##type*>(this); \
+ }
+#define LALLOC_CONST_CAST(type) \
+ const L##type* LAllocation::to##type() const { \
+ MOZ_ASSERT(is##type()); \
+ return static_cast<const L##type*>(this); \
+ }
+
+LALLOC_CAST(Use)
+LALLOC_CONST_CAST(Use)
+LALLOC_CONST_CAST(GeneralReg)
+LALLOC_CONST_CAST(FloatReg)
+LALLOC_CONST_CAST(StackSlot)
+LALLOC_CAST(StackArea)
+LALLOC_CONST_CAST(StackArea)
+LALLOC_CONST_CAST(Argument)
+LALLOC_CONST_CAST(ConstantIndex)
+
+#undef LALLOC_CAST
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_LIR_h */
diff --git a/js/src/jit/LIROps.yaml b/js/src/jit/LIROps.yaml
new file mode 100644
index 0000000000..2842532056
--- /dev/null
+++ b/js/src/jit/LIROps.yaml
@@ -0,0 +1,3866 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# [SMDOC] LIR Opcodes
+# =======================
+# This file defines all LIR opcodes as well as LIR opcode class
+# definitions. It is parsed by GenerateLIRFiles.py at build time to
+# create LIROpsGenerated.h. Each opcode consists of a
+# name and a set of attributes that are described below. Unless
+# marked as required, attributes are optional.
+#
+# name [required]
+# ====
+# Opcode name.
+# Possible values:
+# - opcode string: used as the name for LIR opcode.
+#
+# gen_boilerplate
+# ===============
+# Used to decide to generate LIR boilerplate.
+# - true (default): auto generate boilerplate for this LIR opcode
+# - false: do not generate boilerplate for this LIR opcode
+#
+# result_type
+# ===========
+# Specifies the result type that is produced by this LIR instruction.
+# The result type can be any of the following: WordSized, BoxedValue,
+# or Int64.
+# - attribute not specified (default): there is no result produced
+# by this LIR instruction
+# - result type: sets result type for this LIR instruction
+#
+# operands
+# ========
+# A list of operands to the LIR node. Each operand will be
+# passed into and set in the instruction's constructor. A simple getter
+# will also be auto generated for the operand. Each operand in the
+# following list is defined by its name and an type.
+# The type can be WordSized, BoxedValue, or Int64.
+#
+# For example:
+# operands:
+# lhs: BoxedValue
+# rhs: WordSized
+#
+# Will result in:
+# explicit LInstanceOfV(const LBoxAllocation& lhs, const LAllocation& rhs)
+# : LInstructionHelper(classOpcode) {
+# setBoxOperand(lhsIndex, lhs);
+# setOperand(rhsIndex, rhs);
+# }
+# const LAllocation* rhs() { return getOperand(0); }
+#
+# static const size_t lhsIndex = 0;
+# static const size_t rhsIndex = BOX_PIECES;
+#
+# - attribute not specified (default): no code generated
+# - list of operand names with their types: operand getters and setters
+# are generated and passed into the constructor
+#
+# arguments
+# =========
+# A list of non-LIR node arguments to the LIR op class constructor
+# that are passed along with the operands. The arguments require
+# both a name and a full type signature for each item in the list.
+#
+# For example:
+# offset: size_t
+# type: MIRType
+#
+# For each argument a private variable declaration will be autogenerated
+# in the LIR op class, as well as simple accessor for that variable. The
+# above arguments list will result in the following declarations and
+# accessors:
+#
+# size_t offset_;
+# MIRType type_;
+#
+# size_t offset() const { return offset_; }
+# MIRType type() const { return type_; }
+#
+# - attribute not specified (default): no code generated
+# - argument list: argument names and their full type signature
+#
+# num_temps
+# ========
+# Specifies the number of temporary virtual registers, LDefinitions, used by
+# this LIR op.
+# - attribute not specified (default): number of temps is set to 0
+# - number of LDefinition temps: sets number of temps max 15
+#
+# call_instruction
+# ================
+# Used to define call instructions.
+# - attribute not specified (default): no code generated
+# - true: generates a call to setIsCall in the op's constructor
+#
+# mir_op
+# ======
+# If a LIR instruction corresponds one-to-one with a particular MIR
+# instruction, this will generate a method that returns that MIR
+# instruction.
+# - attribute not specified (default): no code generated
+# - true: generates a method to return MIR instruction
+# - mir string: returns this specified MIR instruction
+#
+
+- name: Phi
+ gen_boilerplate: false
+
+- name: Box
+ gen_boilerplate: false
+
+- name: OsiPoint
+ gen_boilerplate: false
+
+- name: MoveGroup
+ gen_boilerplate: false
+
+# Constant 32-bit integer.
+- name: Integer
+ result_type: WordSized
+ arguments:
+ i32: int32_t
+
+# Constant 64-bit integer.
+- name: Integer64
+ result_type: Int64
+ arguments:
+ i64: int64_t
+
+# Constant pointer.
+- name: Pointer
+ result_type: WordSized
+ arguments:
+ gcptr: gc::Cell*
+
+# Constant double.
+- name: Double
+ result_type: WordSized
+ arguments:
+ value: double
+
+# Constant float32.
+- name: Float32
+ result_type: WordSized
+ arguments:
+ value: float
+
+- name: Value
+ gen_boilerplate: false
+
+- name: NurseryObject
+ result_type: WordSized
+ mir_op: true
+
+# Formal argument for a function, returning a box. Formal arguments are
+# initially read from the stack.
+- name: Parameter
+ result_type: BoxedValue
+
+# Stack offset for a word-sized immutable input value to a frame.
+- name: Callee
+ result_type: WordSized
+
+- name: IsConstructing
+ result_type: WordSized
+
+- name: Goto
+ gen_boilerplate: false
+
+- name: NewArray
+ gen_boilerplate: false
+
+- name: NewArrayDynamicLength
+ result_type: WordSized
+ operands:
+ length: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: NewIterator
+ result_type: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: NewTypedArray
+ result_type: WordSized
+ num_temps: 2
+ mir_op: true
+
+- name: NewTypedArrayDynamicLength
+ result_type: WordSized
+ operands:
+ length: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: NewTypedArrayFromArray
+ result_type: WordSized
+ operands:
+ array: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: NewTypedArrayFromArrayBuffer
+ result_type: WordSized
+ operands:
+ arrayBuffer: WordSized
+ byteOffset: BoxedValue
+ length: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: NewObject
+ gen_boilerplate: false
+
+- name: NewPlainObject
+ result_type: WordSized
+ num_temps: 3
+ mir_op: true
+
+- name: NewArrayObject
+ result_type: WordSized
+ num_temps: 2
+ mir_op: true
+
+# Allocates a new NamedLambdaObject.
+#
+# This instruction generates two possible instruction sets:
+# (1) An inline allocation of the call object is attempted.
+# (2) Otherwise, a callVM create a new object.
+#
+- name: NewNamedLambdaObject
+ result_type: WordSized
+ num_temps: 1
+ mir_op: true
+
+# Allocates a new CallObject.
+#
+# This instruction generates two possible instruction sets:
+# (1) If the call object is extensible, this is a callVM to create the
+# call object.
+# (2) Otherwise, an inline allocation of the call object is attempted.
+#
+- name: NewCallObject
+ result_type: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: NewStringObject
+ result_type: WordSized
+ operands:
+ input: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: InitElemGetterSetter
+ operands:
+ object: WordSized
+ id: BoxedValue
+ value: WordSized
+ call_instruction: true
+ mir_op: true
+
+# Takes in an Object and a Value.
+- name: MutateProto
+ operands:
+ object: WordSized
+ value: BoxedValue
+ call_instruction: true
+
+- name: InitPropGetterSetter
+ operands:
+ object: WordSized
+ value: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: CheckOverRecursed
+ mir_op: true
+
+- name: WasmTrap
+ mir_op: true
+
+- name: WasmGcObjectIsSubtypeOf
+ mir_op: true
+ operands:
+ object: WordSized
+ superTypeDef: WordSized
+ result_type: WordSized
+ num_temps: 2
+
+- name: WasmGcObjectIsSubtypeOfAndBranch
+ gen_boilerplate: false
+
+- name: WasmReinterpret
+ gen_boilerplate: false
+
+- name: WasmReinterpretFromI64
+ gen_boilerplate: false
+
+- name: WasmReinterpretToI64
+ gen_boilerplate: false
+
+- name: Rotate
+ gen_boilerplate: false
+
+- name: RotateI64
+ gen_boilerplate: false
+
+- name: InterruptCheck
+ mir_op: true
+
+- name: WasmInterruptCheck
+ operands:
+ instance: WordSized
+ mir_op: true
+
+- name: TypeOfV
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ num_temps: 1
+ mir_op: TypeOf
+
+- name: TypeOfO
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: TypeOf
+
+- name: TypeOfName
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: true
+
+- name: TypeOfIsNonPrimitiveV
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ num_temps: 1
+ mir_op: TypeOfIs
+
+- name: TypeOfIsNonPrimitiveO
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: TypeOfIs
+
+- name: TypeOfIsPrimitive
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ mir_op: TypeOfIs
+
+- name: ToAsyncIter
+ result_type: WordSized
+ operands:
+ iterator: WordSized
+ nextMethod: BoxedValue
+ call_instruction: true
+
+- name: ToPropertyKeyCache
+ result_type: BoxedValue
+ operands:
+ input: BoxedValue
+ mir_op: true
+
+# Allocate an object for |new| on the caller-side,
+# when there is no templateObject or prototype known
+- name: CreateThis
+ result_type: BoxedValue
+ operands:
+ callee: WordSized
+ newTarget: WordSized
+ call_instruction: true
+ mir_op: true
+
+# Allocate a new arguments object for the frame.
+- name: CreateArgumentsObject
+ result_type: WordSized
+ operands:
+ callObject: WordSized
+ num_temps: 3
+ call_instruction: true
+ mir_op: true
+
+- name: CreateInlinedArgumentsObject
+ gen_boilerplate: false
+
+- name: GetInlinedArgument
+ gen_boilerplate: false
+
+- name: GetInlinedArgumentHole
+ gen_boilerplate: false
+
+# Get argument from arguments object.
+- name: GetArgumentsObjectArg
+ result_type: BoxedValue
+ operands:
+ argsObject: WordSized
+ num_temps: 1
+ mir_op: true
+
+# Set argument on arguments object.
+- name: SetArgumentsObjectArg
+ operands:
+ argsObject: WordSized
+ value: BoxedValue
+ num_temps: 1
+ mir_op: true
+
+# Load an element from an arguments object.
+- name: LoadArgumentsObjectArg
+ result_type: BoxedValue
+ operands:
+ argsObject: WordSized
+ index: WordSized
+ num_temps: 1
+
+# Load an element from an arguments object. Handles out-of-bounds accesses.
+- name: LoadArgumentsObjectArgHole
+ result_type: BoxedValue
+ operands:
+ argsObject: WordSized
+ index: WordSized
+ num_temps: 1
+
+# Return true if the element exists in the arguments object.
+- name: InArgumentsObjectArg
+ result_type: WordSized
+ operands:
+ argsObject: WordSized
+ index: WordSized
+ num_temps: 1
+
+# Return |arguments.length| unless it has been overridden.
+- name: ArgumentsObjectLength
+ result_type: WordSized
+ operands:
+ argsObject: WordSized
+
+# Create an array from an arguments object.
+- name: ArrayFromArgumentsObject
+ result_type: WordSized
+ operands:
+ argsObject: WordSized
+ call_instruction: true
+ mir_op: true
+
+# Guard that the given flags are not set on the arguments object.
+- name: GuardArgumentsObjectFlags
+ operands:
+ argsObject: WordSized
+ num_temps: 1
+ mir_op: true
+
+# If the Value is an Object, return unbox(Value).
+# Otherwise, return the other Object.
+- name: ReturnFromCtor
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ object: WordSized
+
+- name: BoxNonStrictThis
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ mir_op: true
+
+- name: ImplicitThis
+ result_type: BoxedValue
+ operands:
+ env: WordSized
+ call_instruction: true
+ mir_op: true
+
+# Writes a typed argument for a function call to the frame's argument vector.
+- name: StackArgT
+ operands:
+ arg: WordSized
+ arguments:
+ # Index into frame-scope argument vector.
+ argslot: uint32_t
+ type: MIRType
+
+# Writes a typed argument for a function call to the frame's argument vector.
+- name: StackArgV
+ operands:
+ value: BoxedValue
+ arguments:
+ # Index into frame-scope argument vector.
+ argslot: uint32_t
+
+- name: CallGeneric
+ gen_boilerplate: false
+
+- name: CallKnown
+ gen_boilerplate: false
+
+- name: CallNative
+ gen_boilerplate: false
+
+- name: CallDOMNative
+ gen_boilerplate: false
+
+- name: Bail
+
+- name: Unreachable
+ gen_boilerplate: false
+
+- name: EncodeSnapshot
+
+- name: UnreachableResultV
+ gen_boilerplate: false
+
+- name: UnreachableResultT
+ result_type: WordSized
+
+- name: GetDOMProperty
+ gen_boilerplate: false
+
+- name: GetDOMMemberV
+ gen_boilerplate: false
+
+- name: GetDOMMemberT
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: GetDOMMember
+
+- name: SetDOMProperty
+ gen_boilerplate: false
+
+- name: LoadDOMExpandoValue
+ result_type: BoxedValue
+ operands:
+ proxy: WordSized
+ mir_op: true
+
+- name: LoadDOMExpandoValueGuardGeneration
+ result_type: BoxedValue
+ operands:
+ proxy: WordSized
+ mir_op: true
+
+- name: LoadDOMExpandoValueIgnoreGeneration
+ result_type: BoxedValue
+ operands:
+ proxy: WordSized
+ mir_op: true
+
+- name: GuardDOMExpandoMissingOrGuardShape
+ operands:
+ input: BoxedValue
+ num_temps: 1
+ mir_op: true
+
+- name: ApplyArgsGeneric
+ gen_boilerplate: false
+
+- name: ApplyArgsObj
+ gen_boilerplate: false
+
+- name: ApplyArrayGeneric
+ gen_boilerplate: false
+
+- name: ConstructArgsGeneric
+ gen_boilerplate: false
+
+- name: ConstructArrayGeneric
+ gen_boilerplate: false
+
+- name: TestIAndBranch
+ gen_boilerplate: false
+
+- name: TestI64AndBranch
+ gen_boilerplate: false
+
+- name: TestDAndBranch
+ gen_boilerplate: false
+
+- name: TestFAndBranch
+ gen_boilerplate: false
+
+- name: TestBIAndBranch
+ gen_boilerplate: false
+
+- name: TestOAndBranch
+ gen_boilerplate: false
+
+- name: TestVAndBranch
+ gen_boilerplate: false
+
+- name: Compare
+ gen_boilerplate: false
+
+- name: CompareI64
+ gen_boilerplate: false
+
+- name: CompareI64AndBranch
+ gen_boilerplate: false
+
+- name: CompareAndBranch
+ gen_boilerplate: false
+
+- name: CompareD
+ result_type: WordSized
+ operands:
+ left: WordSized
+ right: WordSized
+ mir_op: Compare
+
+- name: CompareF
+ result_type: WordSized
+ operands:
+ left: WordSized
+ right: WordSized
+ mir_op: Compare
+
+- name: CompareDAndBranch
+ gen_boilerplate: false
+
+- name: CompareFAndBranch
+ gen_boilerplate: false
+
+- name: CompareS
+ result_type: WordSized
+ operands:
+ left: WordSized
+ right: WordSized
+ mir_op: Compare
+
+- name: CompareSInline
+ result_type: WordSized
+ operands:
+ input: WordSized
+ arguments:
+ constant: JSLinearString*
+ mir_op: Compare
+
+- name: CompareBigInt
+ result_type: WordSized
+ operands:
+ left: WordSized
+ right: WordSized
+ num_temps: 3
+ mir_op: Compare
+
+- name: CompareBigIntInt32
+ result_type: WordSized
+ operands:
+ left: WordSized
+ right: WordSized
+ num_temps: 2
+ mir_op: Compare
+
+- name: CompareBigIntDouble
+ result_type: WordSized
+ operands:
+ left: WordSized
+ right: WordSized
+ call_instruction: true
+ mir_op: Compare
+
+- name: CompareBigIntString
+ result_type: WordSized
+ operands:
+ left: WordSized
+ right: WordSized
+ call_instruction: true
+ mir_op: Compare
+
+- name: BitAndAndBranch
+ gen_boilerplate: false
+
+# Takes a value and tests whether it is null, undefined, or is an object that
+# emulates |undefined|, as determined by the JSCLASS_EMULATES_UNDEFINED class
+# flag on unwrapped objects. See also js::EmulatesUndefined.
+- name: IsNullOrLikeUndefinedV
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ num_temps: 1
+ mir_op: Compare
+
+# Takes an object pointer and tests whether it is an object that emulates
+# |undefined|, as above.
+- name: IsNullOrLikeUndefinedT
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: Compare
+
+# Takes a value and tests whether it is null.
+- name: IsNull
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ mir_op: Compare
+
+# Takes a value and tests whether it is undefined.
+- name: IsUndefined
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ mir_op: Compare
+
+- name: IsNullOrLikeUndefinedAndBranchV
+ gen_boilerplate: false
+
+- name: IsNullOrLikeUndefinedAndBranchT
+ gen_boilerplate: false
+
+- name: IsNullAndBranch
+ gen_boilerplate: false
+
+- name: IsUndefinedAndBranch
+ gen_boilerplate: false
+
+- name: SameValueDouble
+ result_type: WordSized
+ operands:
+ left: WordSized
+ right: WordSized
+ num_temps: 1
+
+- name: SameValue
+ result_type: WordSized
+ operands:
+ lhs: BoxedValue
+ rhs: BoxedValue
+
+# Not operation on an integer.
+- name: NotI
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+# Not operation on an int64.
+- name: NotI64
+ result_type: WordSized
+ operands:
+ inputI64: Int64
+
+# Not operation on a double.
+- name: NotD
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: Not
+
+# Not operation on a float32.
+- name: NotF
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: Not
+
+# Not operation on a BigInt.
+- name: NotBI
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: Not
+
+# Boolean complement operation on an object.
+- name: NotO
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: Not
+
+# Boolean complement operation on a value.
+- name: NotV
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ num_temps: 2
+ mir_op: Not
+
+- name: BitNotI
+ gen_boilerplate: false
+
+- name: BitNotI64
+ gen_boilerplate: false
+
+- name: BitOpI
+ gen_boilerplate: false
+
+- name: BitOpI64
+ gen_boilerplate: false
+
+- name: ShiftI
+ gen_boilerplate: false
+
+- name: ShiftI64
+ gen_boilerplate: false
+
+- name: SignExtendInt32
+ result_type: WordSized
+ operands:
+ num: WordSized
+ arguments:
+ mode: MSignExtendInt32::Mode
+
+- name: SignExtendInt64
+ gen_boilerplate: false
+
+- name: UrshD
+ gen_boilerplate: false
+
+- name: Return
+ gen_boilerplate: false
+
+- name: Throw
+ operands:
+ value: BoxedValue
+ call_instruction: true
+
+- name: MinMaxI
+ gen_boilerplate: false
+
+- name: MinMaxD
+ gen_boilerplate: false
+
+- name: MinMaxF
+ gen_boilerplate: false
+
+- name: MinMaxArrayI
+ gen_boilerplate: false
+
+- name: MinMaxArrayD
+ gen_boilerplate: false
+
+# Negative of integer
+- name: NegI
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Negative of an int64
+- name: NegI64
+ result_type: Int64
+ operands:
+ num: Int64
+
+# Negative of double
+- name: NegD
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Negative of float32
+- name: NegF
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Absolute value of an integer.
+- name: AbsI
+ result_type: WordSized
+ operands:
+ num: WordSized
+ mir_op: Abs
+
+# Absolute value of a double.
+- name: AbsD
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Absolute value of a float32.
+- name: AbsF
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+- name: CopySignD
+ gen_boilerplate: false
+
+- name: CopySignF
+ gen_boilerplate: false
+
+# Count leading zeroes on an int32.
+- name: ClzI
+ result_type: WordSized
+ operands:
+ num: WordSized
+ mir_op: Clz
+
+# Count leading zeroes on an int64.
+- name: ClzI64
+ result_type: Int64
+ operands:
+ num: Int64
+ mir_op: Clz
+
+# Count trailing zeroes on an int32.
+- name: CtzI
+ result_type: WordSized
+ operands:
+ num: WordSized
+ mir_op: Ctz
+
+# Count trailing zeroes on an int64.
+- name: CtzI64
+ result_type: Int64
+ operands:
+ num: Int64
+ mir_op: Ctz
+
+# Count population on an int32.
+- name: PopcntI
+ result_type: WordSized
+ operands:
+ num: WordSized
+ num_temps: 1
+ mir_op: Popcnt
+
+# Count population on an int64.
+- name: PopcntI64
+ result_type: Int64
+ operands:
+ num: Int64
+ num_temps: 1
+ mir_op: Popcnt
+
+- name: SqrtD
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+- name: SqrtF
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+- name: Atan2D
+ gen_boilerplate: false
+
+- name: Hypot
+ gen_boilerplate: false
+
+# Double raised to an integer power.
+- name: PowI
+ result_type: WordSized
+ operands:
+ value: WordSized
+ power: WordSized
+ call_instruction: true
+
+# Integer raised to an integer power.
+- name: PowII
+ result_type: WordSized
+ operands:
+ value: WordSized
+ power: WordSized
+ num_temps: 2
+ mir_op: Pow
+
+# Double raised to a double power.
+- name: PowD
+ result_type: WordSized
+ operands:
+ value: WordSized
+ power: WordSized
+ call_instruction: true
+
+# Constant of a power of two raised to an integer power.
+- name: PowOfTwoI
+ result_type: WordSized
+ operands:
+ power: WordSized
+ arguments:
+ base: uint32_t
+
+# Sign value of an integer.
+- name: SignI
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Sign value of an integer.
+- name: SignD
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Sign value of a double with expected int32 result.
+- name: SignDI
+ result_type: WordSized
+ operands:
+ input: WordSized
+ num_temps: 1
+
+- name: MathFunctionD
+ gen_boilerplate: false
+
+- name: MathFunctionF
+ gen_boilerplate: false
+
+- name: AddI
+ gen_boilerplate: false
+
+- name: AddI64
+ gen_boilerplate: false
+
+- name: SubI
+ gen_boilerplate: false
+
+- name: SubI64
+ gen_boilerplate: false
+
+- name: MulI64
+ gen_boilerplate: false
+
+- name: MathD
+ gen_boilerplate: false
+
+- name: MathF
+ gen_boilerplate: false
+
+- name: ModD
+ gen_boilerplate: false
+
+- name: ModPowTwoD
+ gen_boilerplate: false
+
+- name: WasmBuiltinModD
+ result_type: WordSized
+ operands:
+ lhs: WordSized
+ rhs: WordSized
+ instance: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: BigIntAdd
+ gen_boilerplate: false
+
+- name: BigIntSub
+ gen_boilerplate: false
+
+- name: BigIntMul
+ gen_boilerplate: false
+
+- name: BigIntDiv
+ gen_boilerplate: false
+
+- name: BigIntMod
+ gen_boilerplate: false
+
+- name: BigIntPow
+ gen_boilerplate: false
+
+- name: BigIntBitAnd
+ gen_boilerplate: false
+
+- name: BigIntBitOr
+ gen_boilerplate: false
+
+- name: BigIntBitXor
+ gen_boilerplate: false
+
+- name: BigIntLsh
+ gen_boilerplate: false
+
+- name: BigIntRsh
+ gen_boilerplate: false
+
+- name: BigIntIncrement
+ gen_boilerplate: false
+
+- name: BigIntDecrement
+ gen_boilerplate: false
+
+- name: BigIntNegate
+ gen_boilerplate: false
+
+- name: BigIntBitNot
+ gen_boilerplate: false
+
+- name: Int32ToStringWithBase
+ result_type: WordSized
+ operands:
+ input: WordSized
+ base: WordSized
+ num_temps: 2
+ mir_op: true
+
+- name: NumberParseInt
+ result_type: BoxedValue
+ operands:
+ string: WordSized
+ radix: WordSized
+ num_temps: 1
+ call_instruction: true
+ mir_op: true
+
+- name: DoubleParseInt
+ result_type: WordSized
+ operands:
+ number: WordSized
+ num_temps: 1
+ mir_op: true
+
+# Adds two string, returning a string.
+- name: Concat
+ result_type: WordSized
+ operands:
+ lhs: WordSized
+ rhs: WordSized
+ num_temps: 5
+
+# Linearize a string before a character access.
+- name: LinearizeForCharAccess
+ result_type: WordSized
+ operands:
+ str: WordSized
+ index: WordSized
+ mir_op: true
+
+# Get uint16 character code from a string.
+- name: CharCodeAt
+ result_type: WordSized
+ operands:
+ str: WordSized
+ index: WordSized
+ num_temps: 2
+
+# Get uint16 character code from a string. Return NaN on out-of-bounds.
+- name: CharCodeAtMaybeOutOfBounds
+ result_type: BoxedValue
+ operands:
+ str: WordSized
+ index: WordSized
+ num_temps: 2
+
+# Get uint16 character code from a string and convert it to a string. Return
+# the empty string on out-of-bounds.
+- name: CharAtMaybeOutOfBounds
+ result_type: WordSized
+ operands:
+ str: WordSized
+ index: WordSized
+ num_temps: 2
+
+# Convert uint16 character code to a string.
+- name: FromCharCode
+ result_type: WordSized
+ operands:
+ code: WordSized
+
+# Convert uint32 code point to a string.
+- name: FromCodePoint
+ result_type: WordSized
+ operands:
+ codePoint: WordSized
+ num_temps: 2
+
+# Search for the first index of the search string.
+- name: StringIndexOf
+ result_type: WordSized
+ operands:
+ string: WordSized
+ searchString: WordSized
+ call_instruction: true
+
+# Test if a string starts with the search string
+- name: StringStartsWith
+ result_type: WordSized
+ operands:
+ string: WordSized
+ searchString: WordSized
+ call_instruction: true
+
+# Test if a string starts with the constant search string
+- name: StringStartsWithInline
+ result_type: WordSized
+ operands:
+ string: WordSized
+ arguments:
+ searchString: JSLinearString*
+ num_temps: 1
+
+# Test if a string ends with the search string
+- name: StringEndsWith
+ result_type: WordSized
+ operands:
+ string: WordSized
+ searchString: WordSized
+ call_instruction: true
+
+# Test if a string ends with the constant search string
+- name: StringEndsWithInline
+ result_type: WordSized
+ operands:
+ string: WordSized
+ arguments:
+ searchString: JSLinearString*
+ num_temps: 1
+
+# Calls the ToLowerCase case conversion function. Inlines the case conversion
+# when possible.
+- name: StringToLowerCase
+ result_type: WordSized
+ operands:
+ string: WordSized
+ num_temps: 5
+ mir_op: StringConvertCase
+
+# Calls the ToUpperCase case conversion function.
+- name: StringToUpperCase
+ result_type: WordSized
+ operands:
+ string: WordSized
+ call_instruction: true
+ mir_op: StringConvertCase
+
+- name: StringSplit
+ result_type: WordSized
+ operands:
+ string: WordSized
+ separator: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: Substr
+ result_type: WordSized
+ operands:
+ string: WordSized
+ begin: WordSized
+ length: WordSized
+ num_temps: 3
+ mir_op: StringSplit
+
+- name: Int32ToDouble
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+- name: Float32ToDouble
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+- name: DoubleToFloat32
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+- name: Int32ToFloat32
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+# Convert a value to a double.
+- name: ValueToDouble
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ mir_op: ToDouble
+
+# Convert a value to a float32.
+- name: ValueToFloat32
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ mir_op: ToFloat32
+
+- name: ValueToInt32
+ gen_boilerplate: false
+
+# Convert a value to a BigInt.
+- name: ValueToBigInt
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ mir_op: ToBigInt
+
+# Convert a double to an int32.
+# Input: floating-point Register
+# Output: 32-bit integer
+# Bailout: if the double cannot be converted to an integer.
+- name: DoubleToInt32
+ result_type: WordSized
+ operands:
+ in: WordSized
+ mir_op: ToNumberInt32
+
+# Convert a float32 to an int32.
+# Input: floating-point Register
+# Output: 32-bit integer
+# Bailout: if the float32 cannot be converted to an integer.
+- name: Float32ToInt32
+ result_type: WordSized
+ operands:
+ in: WordSized
+ mir_op: ToNumberInt32
+
+# Truncates a double to an int32.
+# Input: floating-point Register
+# Output: 32-bit integer
+# Bailout: if the double when converted to an integer exceeds the int32
+# bounds. No bailout for NaN or negative zero.
+- name: DoubleToIntegerInt32
+ result_type: WordSized
+ operands:
+ in: WordSized
+ mir_op: ToIntegerInt32
+
+# Truncates a float to an int32.
+# Input: floating-point Register
+# Output: 32-bit integer
+# Bailout: if the double when converted to an integer exceeds the int32
+# bounds. No bailout for NaN or negative zero.
+- name: Float32ToIntegerInt32
+ result_type: WordSized
+ operands:
+ in: WordSized
+ mir_op: ToIntegerInt32
+
+# Convert a double to a truncated int32.
+# Input: floating-point Register
+# Output: 32-bit integer
+- name: TruncateDToInt32
+ result_type: WordSized
+ operands:
+ in: WordSized
+ num_temps: 1
+ mir_op: TruncateToInt32
+
+# Convert a double to a truncated int32 with instance offset because we need it
+# for the slow ool path.
+- name: WasmBuiltinTruncateDToInt32
+ result_type: WordSized
+ operands:
+ in: WordSized
+ instance: WordSized
+ num_temps: 1
+ mir_op: WasmBuiltinTruncateToInt32
+
+# Convert a float32 to a truncated int32.
+# Input: floating-point Register
+# Output: 32-bit integer
+- name: TruncateFToInt32
+ result_type: WordSized
+ operands:
+ in: WordSized
+ num_temps: 1
+ mir_op: TruncateToInt32
+
+# Convert a float32 to a truncated int32 with instance offset because we need
+# it for the slow ool path.
+- name: WasmBuiltinTruncateFToInt32
+ result_type: WordSized
+ operands:
+ in: WordSized
+ instance: WordSized
+ num_temps: 1
+ mir_op: WasmBuiltinTruncateToInt32
+
+- name: WasmTruncateToInt32
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: true
+
+- name: WrapInt64ToInt32
+ result_type: WordSized
+ operands:
+ input: Int64
+ mir_op: true
+
+- name: ExtendInt32ToInt64
+ result_type: Int64
+ operands:
+ input: WordSized
+ mir_op: true
+
+# Convert a boolean value to a string.
+- name: BooleanToString
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: ToString
+
+# Convert an integer hosted on one definition to a string with a function call.
+- name: IntToString
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: ToString
+
+# Convert a double hosted on one definition to a string with a function call.
+- name: DoubleToString
+ result_type: WordSized
+ operands:
+ input: WordSized
+ num_temps: 1
+ mir_op: ToString
+
+# Convert a primitive to a string with a function call.
+- name: ValueToString
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ num_temps: 1
+ mir_op: ToString
+
+- name: PowHalfD
+ gen_boilerplate: false
+
+- name: NaNToZero
+ result_type: WordSized
+ operands:
+ input: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: OsrEntry
+ gen_boilerplate: false
+
+# Materialize a Value stored in an interpreter frame for OSR.
+- name: OsrValue
+ result_type: BoxedValue
+ operands:
+ entry: WordSized
+ mir_op: true
+
+# Materialize a JSObject env chain stored in an interpreter frame for OSR.
+- name: OsrEnvironmentChain
+ result_type: WordSized
+ operands:
+ entry: WordSized
+ mir_op: true
+
+# Materialize a JSObject env chain stored in an interpreter frame for OSR.
+- name: OsrReturnValue
+ result_type: BoxedValue
+ operands:
+ entry: WordSized
+ mir_op: true
+
+- name: OsrArgumentsObject
+ result_type: WordSized
+ operands:
+ entry: WordSized
+ mir_op: true
+
+- name: RegExp
+ result_type: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: RegExpMatcher
+ result_type: BoxedValue
+ operands:
+ regexp: WordSized
+ string: WordSized
+ lastIndex: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: RegExpSearcher
+ result_type: WordSized
+ operands:
+ regexp: WordSized
+ string: WordSized
+ lastIndex: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: RegExpTester
+ result_type: WordSized
+ operands:
+ regexp: WordSized
+ string: WordSized
+ lastIndex: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: RegExpPrototypeOptimizable
+ result_type: WordSized
+ operands:
+ object: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: RegExpInstanceOptimizable
+ result_type: WordSized
+ operands:
+ object: WordSized
+ proto: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: GetFirstDollarIndex
+ result_type: WordSized
+ operands:
+ str: WordSized
+ num_temps: 3
+
+- name: StringReplace
+ result_type: WordSized
+ operands:
+ string: WordSized
+ pattern: WordSized
+ replacement: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: BinaryValueCache
+ result_type: BoxedValue
+ operands:
+ lhs: BoxedValue
+ rhs: BoxedValue
+ # Takes two temps: these are intended to be FloatReg0 and FloatReg1
+ # to allow the actual cache code to safely clobber those values without
+ # save and restore.
+ num_temps: 2
+ mir_op: BinaryCache
+
+- name: BinaryBoolCache
+ result_type: WordSized
+ operands:
+ lhs: BoxedValue
+ rhs: BoxedValue
+ # Takes two temps: these are intendend to be FloatReg0 and FloatReg1
+ # To allow the actual cache code to safely clobber those values without
+ # save and restore.
+ num_temps: 2
+ mir_op: BinaryCache
+
+- name: UnaryCache
+ result_type: BoxedValue
+ operands:
+ input: BoxedValue
+ mir_op_cast: true
+
+- name: ModuleMetadata
+ result_type: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: DynamicImport
+ result_type: WordSized
+ operands:
+ specifier: BoxedValue
+ options: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: Lambda
+ result_type: WordSized
+ operands:
+ environmentChain: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: FunctionWithProto
+ result_type: WordSized
+ operands:
+ envChain: WordSized
+ prototype: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: SetFunName
+ result_type: WordSized
+ operands:
+ fun: WordSized
+ name: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: KeepAliveObject
+ operands:
+ object: WordSized
+
+# Load the "slots" member out of a JSObject.
+# Input: JSObject pointer
+# Output: slots pointer
+- name: Slots
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+# Load the "elements" member out of a JSObject.
+# Input: JSObject pointer
+# Output: elements pointer
+- name: Elements
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: true
+
+# Load the initialized length from an elements header.
+- name: InitializedLength
+ result_type: WordSized
+ operands:
+ elements: WordSized
+
+# Store to the initialized length in an elements header. Note the input is an
+# *index*, one less than the desired initialized length.
+- name: SetInitializedLength
+ operands:
+ elements: WordSized
+ index: WordSized
+
+# Load the length from an elements header.
+- name: ArrayLength
+ result_type: WordSized
+ operands:
+ elements: WordSized
+
+# Store to the length in an elements header. Note the input is an *index*,
+# one less than the desired length.
+- name: SetArrayLength
+ operands:
+ elements: WordSized
+ index: WordSized
+
+# Load the "length" property of a function.
+- name: FunctionLength
+ result_type: WordSized
+ operands:
+ function: WordSized
+
+# Load the "name" property of a function.
+- name: FunctionName
+ result_type: WordSized
+ operands:
+ function: WordSized
+
+- name: GetNextEntryForIterator
+ result_type: WordSized
+ operands:
+ iter: WordSized
+ result: WordSized
+ num_temps: 3
+ mir_op: true
+
+- name: ArrayBufferByteLength
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+- name: ArrayBufferViewLength
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: true
+
+# Read the byteOffset of an array buffer view.
+- name: ArrayBufferViewByteOffset
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+# Load an array buffer view's elements vector.
+- name: ArrayBufferViewElements
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+# Return the element size of a typed array.
+- name: TypedArrayElementSize
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+- name: GuardHasAttachedArrayBuffer
+ operands:
+ object: WordSized
+ num_temps: 1
+
+# Double to IntPtr, eligible for accessing into a TypedArray or DataView. If
+# the index isn't exactly representable as an IntPtr, depending on the
+# supportOOB flag on the MIR instruction, either bail out or produce an IntPtr
+# which is equivalent to an OOB access.
+- name: GuardNumberToIntPtrIndex
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: true
+
+# Bailout if index >= length.
+- name: BoundsCheck
+ operands:
+ index: WordSized
+ length: WordSized
+ mir_op: true
+
+- name: BoundsCheckRange
+ gen_boilerplate: false
+
+# Bailout if index < minimum.
+- name: BoundsCheckLower
+ operands:
+ index: WordSized
+ mir_op: true
+
+- name: SpectreMaskIndex
+ result_type: WordSized
+ operands:
+ index: WordSized
+ length: WordSized
+ mir_op: true
+
+- name: LoadElementV
+ gen_boilerplate: false
+
+- name: InArray
+ result_type: WordSized
+ operands:
+ elements: WordSized
+ index: WordSized
+ initLength: WordSized
+ mir_op: true
+
+- name: GuardElementNotHole
+ operands:
+ elements: WordSized
+ index: WordSized
+
+- name: LoadElementHole
+ gen_boilerplate: false
+
+- name: StoreElementV
+ gen_boilerplate: false
+
+- name: StoreElementT
+ gen_boilerplate: false
+
+- name: StoreHoleValueElement
+ operands:
+ elements: WordSized
+ index: WordSized
+
+# Like LStoreElementV, but supports indexes >= initialized length.
+- name: StoreElementHoleV
+ operands:
+ object: WordSized
+ elements: WordSized
+ index: WordSized
+ value: BoxedValue
+ num_temps: 1
+ mir_op: StoreElementHole
+
+# Like LStoreElementT, but supports indexes >= initialized length.
+- name: StoreElementHoleT
+ operands:
+ object: WordSized
+ elements: WordSized
+ index: WordSized
+ value: WordSized
+ num_temps: 1
+ mir_op: StoreElementHole
+
+- name: ArrayPopShift
+ gen_boilerplate: false
+
+- name: ArrayPush
+ result_type: WordSized
+ operands:
+ object: WordSized
+ value: BoxedValue
+ num_temps: 2
+ mir_op: true
+
+- name: ArraySlice
+ result_type: WordSized
+ operands:
+ object: WordSized
+ begin: WordSized
+ end: WordSized
+ num_temps: 2
+ call_instruction: true
+ mir_op: true
+
+- name: ArgumentsSlice
+ result_type: WordSized
+ operands:
+ object: WordSized
+ begin: WordSized
+ end: WordSized
+ num_temps: 2
+ call_instruction: true
+ mir_op: true
+
+- name: FrameArgumentsSlice
+ result_type: WordSized
+ operands:
+ begin: WordSized
+ count: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: InlineArgumentsSlice
+ gen_boilerplate: false
+
+- name: NormalizeSliceTerm
+ result_type: WordSized
+ operands:
+ value: WordSized
+ length: WordSized
+ mir_op: true
+
+- name: ArrayJoin
+ result_type: WordSized
+ operands:
+ array: WordSized
+ separator: WordSized
+ num_temps: 1
+ call_instruction: true
+ mir_op: true
+
+- name: LoadUnboxedScalar
+ result_type: WordSized
+ operands:
+ elements: WordSized
+ index: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: LoadUnboxedBigInt
+ gen_boilerplate: false
+
+- name: LoadDataViewElement
+ gen_boilerplate: false
+
+- name: LoadTypedArrayElementHole
+ result_type: BoxedValue
+ operands:
+ object: WordSized
+ index: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: LoadTypedArrayElementHoleBigInt
+ gen_boilerplate: false
+
+- name: StoreUnboxedScalar
+ operands:
+ elements: WordSized
+ index: WordSized
+ value: WordSized
+ mir_op: true
+
+- name: StoreUnboxedBigInt
+ gen_boilerplate: false
+
+- name: StoreDataViewElement
+ gen_boilerplate: false
+
+- name: StoreTypedArrayElementHole
+ operands:
+ elements: WordSized
+ length: WordSized
+ index: WordSized
+ value: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: StoreTypedArrayElementHoleBigInt
+ gen_boilerplate: false
+
+- name: AtomicIsLockFree
+ result_type: WordSized
+ operands:
+ value: WordSized
+
+- name: CompareExchangeTypedArrayElement
+ gen_boilerplate: false
+
+- name: AtomicExchangeTypedArrayElement
+ gen_boilerplate: false
+
+- name: AtomicTypedArrayElementBinop
+ gen_boilerplate: false
+
+- name: AtomicTypedArrayElementBinopForEffect
+ gen_boilerplate: false
+
+- name: AtomicLoad64
+ gen_boilerplate: false
+
+- name: AtomicStore64
+ gen_boilerplate: false
+
+- name: CompareExchangeTypedArrayElement64
+ gen_boilerplate: false
+
+- name: AtomicExchangeTypedArrayElement64
+ gen_boilerplate: false
+
+- name: AtomicTypedArrayElementBinop64
+ gen_boilerplate: false
+
+- name: AtomicTypedArrayElementBinopForEffect64
+ gen_boilerplate: false
+
+- name: EffectiveAddress
+ result_type: WordSized
+ operands:
+ base: WordSized
+ index: WordSized
+ mir_op: true
+
+- name: ClampIToUint8
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+- name: ClampDToUint8
+ result_type: WordSized
+ operands:
+ in: WordSized
+ num_temps: 1
+
+- name: ClampVToUint8
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ num_temps: 1
+ mir_op: ClampToUint8
+
+# Load a boxed value from an object's fixed slot.
+- name: LoadFixedSlotV
+ result_type: BoxedValue
+ operands:
+ object: WordSized
+ mir_op: LoadFixedSlot
+
+# Load a typed value from an object's fixed slot.
+- name: LoadFixedSlotT
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: LoadFixedSlot
+
+- name: LoadFixedSlotAndUnbox
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: true
+
+- name: LoadDynamicSlotAndUnbox
+ result_type: WordSized
+ operands:
+ slots: WordSized
+ mir_op: true
+
+- name: LoadElementAndUnbox
+ result_type: WordSized
+ operands:
+ elements: WordSized
+ index: WordSized
+ mir_op: true
+
+- name: AddAndStoreSlot
+ operands:
+ object: WordSized
+ value: BoxedValue
+ num_temps: 1
+ mir_op: true
+
+- name: AllocateAndStoreSlot
+ operands:
+ object: WordSized
+ value: BoxedValue
+ num_temps: 2
+ call_instruction: true
+ mir_op: true
+
+- name: AddSlotAndCallAddPropHook
+ operands:
+ object: WordSized
+ value: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+# Store a boxed value to an object's fixed slot.
+- name: StoreFixedSlotV
+ operands:
+ obj: WordSized
+ value: BoxedValue
+ mir_op: StoreFixedSlot
+
+# Store a typed value to an object's fixed slot.
+- name: StoreFixedSlotT
+ operands:
+ obj: WordSized
+ value: WordSized
+ mir_op: StoreFixedSlot
+
+# Note, Name ICs always return a Value. There are no V/T variants.
+- name: GetNameCache
+ result_type: BoxedValue
+ operands:
+ envObj: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: CallGetIntrinsicValue
+ result_type: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: GetPropSuperCache
+ result_type: BoxedValue
+ operands:
+ obj: WordSized
+ receiver: BoxedValue
+ id: BoxedValue
+ mir_op: true
+
+# Patchable jump to stubs generated for a GetProperty cache, which loads a
+# boxed value.
+- name: GetPropertyCache
+ result_type: BoxedValue
+ operands:
+ value: BoxedValue
+ id: BoxedValue
+ mir_op: true
+
+- name: BindNameCache
+ result_type: WordSized
+ operands:
+ environmentChain: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: CallBindVar
+ result_type: WordSized
+ operands:
+ environmentChain: WordSized
+ mir_op: true
+
+# Load a value from an object's dslots or a slots vector.
+- name: LoadDynamicSlotV
+ result_type: BoxedValue
+ operands:
+ in: WordSized
+ mir_op: LoadDynamicSlot
+
+# Store a value to an object's dslots or a slots vector.
+- name: StoreDynamicSlotV
+ operands:
+ slots: WordSized
+ value: BoxedValue
+ mir_op: StoreDynamicSlot
+
+# Store a typed value to an object's dslots or a slots vector. This has a
+# few advantages over LStoreDynamicSlotV:
+# 1) We can bypass storing the type tag if the slot has the same type as
+# the value.
+# 2) Better Register allocation: we can store constants and FP regs directly
+# without requiring a second Register for the value.
+- name: StoreDynamicSlotT
+ operands:
+ slots: WordSized
+ value: WordSized
+ mir_op: StoreDynamicSlot
+
+# Read length field of a JSString*.
+- name: StringLength
+ result_type: WordSized
+ operands:
+ string: WordSized
+
+# Take the floor of a double precision number and converts it to an int32.
+# Implements Math.floor().
+- name: Floor
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Take the floor of a single precision number and converts it to an int32.
+# Implements Math.floor().
+- name: FloorF
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Take the ceiling of a double precision number and converts it to an int32.
+# Implements Math.ceil().
+- name: Ceil
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Take the ceiling of a single precision number and converts it to an int32.
+# Implements Math.ceil().
+- name: CeilF
+ result_type: WordSized
+ operands:
+ string: WordSized
+
+# Round a double precision number and converts it to an int32.
+# Implements Math.round().
+- name: Round
+ result_type: WordSized
+ operands:
+ num: WordSized
+ num_temps: 1
+ mir_op: true
+
+# Round a single precision number and converts it to an int32.
+# Implements Math.round().
+- name: RoundF
+ result_type: WordSized
+ operands:
+ num: WordSized
+ num_temps: 1
+ mir_op: Round
+
+# Truncates a double precision number and converts it to an int32.
+# Implements Math.trunc().
+- name: Trunc
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Truncates a single precision number and converts it to an int32.
+# Implements Math.trunc().
+- name: TruncF
+ result_type: WordSized
+ operands:
+ num: WordSized
+
+# Rounds a double precision number accordingly to mir()->roundingMode(),
+# and keeps a double output.
+- name: NearbyInt
+ result_type: WordSized
+ operands:
+ num: WordSized
+ mir_op: true
+
+# Rounds a single precision number accordingly to mir()->roundingMode(),
+# and keeps a single output.
+- name: NearbyIntF
+ result_type: WordSized
+ operands:
+ num: WordSized
+ mir_op: NearbyInt
+
+# Load a function's call environment.
+- name: FunctionEnvironment
+ result_type: WordSized
+ operands:
+ function: WordSized
+
+- name: HomeObject
+ result_type: WordSized
+ operands:
+ function: WordSized
+
+- name: HomeObjectSuperBase
+ result_type: BoxedValue
+ operands:
+ homeObject: WordSized
+
+- name: NewLexicalEnvironmentObject
+ result_type: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: NewClassBodyEnvironmentObject
+ result_type: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: NewVarEnvironmentObject
+ result_type: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: MegamorphicSetElement
+ operands:
+ object: WordSized
+ index: BoxedValue
+ value: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: CallDeleteProperty
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ call_instruction: true
+ mir_op: DeleteProperty
+
+- name: CallDeleteElement
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ index: BoxedValue
+ call_instruction: true
+ mir_op: DeleteElement
+
+- name: ObjectToIterator
+ result_type: WordSized
+ operands:
+ object: WordSized
+ num_temps: 3
+ mir_op: true
+
+- name: ValueToIterator
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ call_instruction: true
+ mir_op: ValueToIterator
+
+# Patchable jump to stubs generated for a SetProperty cache.
+- name: SetPropertyCache
+ operands:
+ object: WordSized
+ id: BoxedValue
+ value: BoxedValue
+ # Takes an additional temp: this is intendend to be FloatReg0 to allow the
+ # actual cache code to safely clobber that value without save and restore.
+ num_temps: 2
+ mir_op: true
+
+- name: GetIteratorCache
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ num_temps: 2
+ mir_op: true
+
+- name: OptimizeSpreadCallCache
+ result_type: BoxedValue
+ operands:
+ value: BoxedValue
+ num_temps: 1
+ mir_op: true
+
+- name: IteratorMore
+ result_type: BoxedValue
+ operands:
+ iterator: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: IsNoIterAndBranch
+ gen_boilerplate: false
+
+- name: IteratorEnd
+ operands:
+ object: WordSized
+ num_temps: 3
+ mir_op: true
+
+- name: CloseIterCache
+ operands:
+ iter: WordSized
+ num_temps: 1
+ mir_op: true
+
+# Read the number of actual arguments.
+- name: ArgumentsLength
+ result_type: WordSized
+
+# Load a value from the actual arguments.
+- name: GetFrameArgument
+ result_type: BoxedValue
+ operands:
+ index: WordSized
+
+# Load a value from the actual arguments.
+# Returns undefined if |index| is larger-or-equals to |length|. Bails out if
+# |index| is negative.
+- name: GetFrameArgumentHole
+ result_type: BoxedValue
+ operands:
+ index: WordSized
+ length: WordSized
+ num_temps: 1
+
+# Create the rest parameter.
+- name: Rest
+ result_type: WordSized
+ operands:
+ numActuals: WordSized
+ num_temps: 3
+ call_instruction: true
+ mir_op: true
+
+- name: Int32ToIntPtr
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: true
+
+- name: NonNegativeIntPtrToInt32
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+- name: IntPtrToDouble
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+- name: AdjustDataViewLength
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: true
+
+# Convert a Boolean to an Int64, following ToBigInt.
+- name: BooleanToInt64
+ result_type: Int64
+ operands:
+ input: WordSized
+ mir_op: ToInt64
+
+# Convert a String to an Int64, following ToBigInt.
+- name: StringToInt64
+ result_type: Int64
+ operands:
+ input: WordSized
+ mir_op: ToInt64
+
+# Simulate ToBigInt on a Value and produce a matching Int64.
+- name: ValueToInt64
+ result_type: Int64
+ operands:
+ input: BoxedValue
+ num_temps: 1
+ mir_op: ToInt64
+
+# Truncate a BigInt to an unboxed int64.
+- name: TruncateBigIntToInt64
+ result_type: Int64
+ operands:
+ input: WordSized
+ mir_op: true
+
+# Create a new BigInt* from an unboxed int64.
+- name: Int64ToBigInt
+ result_type: WordSized
+ operands:
+ input: Int64
+ num_temps: 1
+ mir_op: true
+
+# Generational write barrier used when writing an object to another object.
+- name: PostWriteBarrierO
+ operands:
+ object: WordSized
+ value: WordSized
+ num_temps: 1
+ mir_op: PostWriteBarrier
+
+# Generational write barrier used when writing a string to an object.
+- name: PostWriteBarrierS
+ operands:
+ object: WordSized
+ value: WordSized
+ num_temps: 1
+ mir_op: PostWriteBarrier
+
+# Generational write barrier used when writing a BigInt to an object.
+- name: PostWriteBarrierBI
+ operands:
+ object: WordSized
+ value: WordSized
+ num_temps: 1
+ mir_op: PostWriteBarrier
+
+# Generational write barrier used when writing a value to another object.
+- name: PostWriteBarrierV
+ operands:
+ object: WordSized
+ value: BoxedValue
+ num_temps: 1
+ mir_op: PostWriteBarrier
+
+# Generational write barrier used when writing an object to another object's
+# elements.
+- name: PostWriteElementBarrierO
+ operands:
+ object: WordSized
+ value: WordSized
+ index: WordSized
+ num_temps: 1
+ mir_op: PostWriteElementBarrier
+
+# Generational write barrier used when writing a string to an object's
+# elements.
+- name: PostWriteElementBarrierS
+ operands:
+ object: WordSized
+ value: WordSized
+ index: WordSized
+ num_temps: 1
+ mir_op: PostWriteElementBarrier
+
+# Generational write barrier used when writing a BigInt to an object's
+# elements.
+- name: PostWriteElementBarrierBI
+ operands:
+ object: WordSized
+ value: WordSized
+ index: WordSized
+ num_temps: 1
+ mir_op: PostWriteElementBarrier
+
+# Generational write barrier used when writing a value to another object's
+# elements.
+- name: PostWriteElementBarrierV
+ operands:
+ object: WordSized
+ index: WordSized
+ value: BoxedValue
+ num_temps: 1
+ mir_op: PostWriteElementBarrier
+
+# Assert in debug mode that a post write barrier can be elided.
+- name: AssertCanElidePostWriteBarrier
+ operands:
+ object: WordSized
+ value: BoxedValue
+ num_temps: 1
+
+# Guard against an object's identity.
+- name: GuardObjectIdentity
+ operands:
+ input: WordSized
+ expected: WordSized
+ mir_op: true
+
+# Guard against an function's identity.
+- name: GuardSpecificFunction
+ operands:
+ input: WordSized
+ expected: WordSized
+
+- name: GuardSpecificAtom
+ operands:
+ str: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: GuardSpecificSymbol
+ operands:
+ symbol: WordSized
+ mir_op: true
+
+- name: GuardSpecificInt32
+ operands:
+ num: WordSized
+ mir_op: true
+
+- name: GuardStringToIndex
+ result_type: WordSized
+ operands:
+ string: WordSized
+
+- name: GuardStringToInt32
+ result_type: WordSized
+ operands:
+ string: WordSized
+ num_temps: 1
+
+- name: GuardStringToDouble
+ result_type: WordSized
+ operands:
+ string: WordSized
+ num_temps: 2
+
+- name: GuardShape
+ result_type: WordSized
+ operands:
+ in: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: GuardMultipleShapes
+ result_type: WordSized
+ operands:
+ object: WordSized
+ shapeList: WordSized
+ num_temps: 4
+ mir_op: true
+
+- name: GuardProto
+ operands:
+ object: WordSized
+ expected: WordSized
+ num_temps: 1
+
+- name: GuardNullProto
+ operands:
+ object: WordSized
+ num_temps: 1
+
+- name: GuardIsNativeObject
+ operands:
+ object: WordSized
+ num_temps: 1
+
+- name: GuardIsProxy
+ operands:
+ object: WordSized
+ num_temps: 1
+
+- name: GuardIsNotProxy
+ operands:
+ object: WordSized
+ num_temps: 1
+
+- name: GuardIsNotDOMProxy
+ operands:
+ proxy: WordSized
+ num_temps: 1
+
+- name: ProxyGet
+ result_type: BoxedValue
+ operands:
+ proxy: WordSized
+ num_temps: 1
+ call_instruction: true
+ mir_op: true
+
+- name: ProxyGetByValue
+ result_type: BoxedValue
+ operands:
+ proxy: WordSized
+ id: BoxedValue
+ call_instruction: true
+
+- name: ProxyHasProp
+ result_type: BoxedValue
+ operands:
+ proxy: WordSized
+ id: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: ProxySet
+ operands:
+ proxy: WordSized
+ rhs: BoxedValue
+ num_temps: 1
+ call_instruction: true
+ mir_op: true
+
+- name: ProxySetByValue
+ operands:
+ proxy: WordSized
+ id: BoxedValue
+ rhs: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: CallSetArrayLength
+ operands:
+ obj: WordSized
+ rhs: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: MegamorphicLoadSlot
+ result_type: BoxedValue
+ operands:
+ object: WordSized
+ num_temps: 3
+ call_instruction: true
+ mir_op: true
+
+- name: MegamorphicLoadSlotByValue
+ result_type: BoxedValue
+ operands:
+ object: WordSized
+ id: BoxedValue
+ num_temps: 3
+ call_instruction: true
+ mir_op: true
+
+- name: MegamorphicStoreSlot
+ result_type: BoxedValue
+ operands:
+ object: WordSized
+ rhs: BoxedValue
+ num_temps: 3
+ call_instruction: true
+ mir_op: true
+
+- name: MegamorphicHasProp
+ result_type: WordSized
+ operands:
+ object: WordSized
+ id: BoxedValue
+ num_temps: 3
+ call_instruction: true
+ mir_op: true
+
+- name: GuardIsNotArrayBufferMaybeShared
+ operands:
+ object: WordSized
+ num_temps: 1
+
+- name: GuardIsTypedArray
+ operands:
+ object: WordSized
+ num_temps: 1
+
+- name: GuardNoDenseElements
+ operands:
+ in: WordSized
+ num_temps: 1
+
+- name: InCache
+ result_type: WordSized
+ operands:
+ lhs: BoxedValue
+ rhs: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: HasOwnCache
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ id: BoxedValue
+ mir_op: true
+
+- name: CheckPrivateFieldCache
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ id: BoxedValue
+ mir_op: true
+
+- name: NewPrivateName
+ result_type: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: InstanceOfO
+ result_type: WordSized
+ operands:
+ lhs: WordSized
+ rhs: WordSized
+ mir_op: InstanceOf
+
+- name: InstanceOfV
+ result_type: WordSized
+ operands:
+ lhs: BoxedValue
+ rhs: WordSized
+ mir_op: InstanceOf
+
+- name: InstanceOfCache
+ gen_boilerplate: false
+
+- name: IsCallableO
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: IsCallable
+
+- name: IsCallableV
+ result_type: WordSized
+ operands:
+ object: BoxedValue
+ num_temps: 1
+ mir_op: IsCallable
+
+- name: IsConstructor
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: true
+
+- name: IsCrossRealmArrayConstructor
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+- name: IsArrayO
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: IsArray
+
+- name: IsArrayV
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ num_temps: 1
+ mir_op: IsArray
+
+- name: IsTypedArray
+ result_type: WordSized
+ operands:
+ object: WordSized
+ mir_op: true
+
+- name: IsObject
+ result_type: WordSized
+ operands:
+ object: BoxedValue
+ mir_op: true
+
+- name: IsObjectAndBranch
+ gen_boilerplate: false
+
+- name: IsNullOrUndefined
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ mir_op: true
+
+- name: IsNullOrUndefinedAndBranch
+ gen_boilerplate: false
+
+- name: HasClass
+ result_type: WordSized
+ operands:
+ lhs: WordSized
+ mir_op: true
+
+- name: GuardToClass
+ result_type: WordSized
+ operands:
+ lhs: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: GuardToFunction
+ result_type: WordSized
+ operands:
+ lhs: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: ObjectClassToString
+ result_type: WordSized
+ operands:
+ lhs: WordSized
+ num_temps: 1
+ call_instruction: true
+ mir_op: true
+
+- name: WasmSelect
+ gen_boilerplate: false
+
+- name: WasmSelectI64
+ gen_boilerplate: false
+
+- name: WasmCompareAndSelect
+ gen_boilerplate: false
+
+- name: WasmAddOffset
+ result_type: WordSized
+ operands:
+ base: WordSized
+ mir_op: true
+
+- name: WasmAddOffset64
+ result_type: Int64
+ operands:
+ base: Int64
+ mir_op: WasmAddOffset
+
+- name: WasmBoundsCheck
+ result_type: WordSized
+ operands:
+ ptr: WordSized
+ boundsCheckLimit: WordSized
+ mir_op: true
+
+- name: WasmBoundsCheck64
+ gen_boilerplate: false
+
+- name: WasmExtendU32Index
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: true
+
+- name: WasmWrapU32Index
+ result_type: WordSized
+ operands:
+ input: WordSized
+ mir_op: true
+
+- name: WasmAlignmentCheck
+ operands:
+ ptr: WordSized
+ mir_op: true
+
+- name: WasmAlignmentCheck64
+ operands:
+ ptr: Int64
+ mir_op: WasmAlignmentCheck
+
+- name: WasmLoadInstance
+ result_type: WordSized
+ operands:
+ instance: WordSized
+ mir_op: true
+
+- name: WasmLoadInstance64
+ result_type: Int64
+ operands:
+ instance: WordSized
+ mir_op: WasmLoadInstance
+
+- name: WasmHeapBase
+ result_type: WordSized
+ operands:
+ instance: WordSized
+ mir_op: true
+
+- name: WasmLoad
+ gen_boilerplate: false
+
+- name: WasmLoadI64
+ gen_boilerplate: false
+
+- name: WasmStore
+ gen_boilerplate: false
+
+- name: WasmStoreI64
+ gen_boilerplate: false
+
+- name: AsmJSLoadHeap
+ result_type: WordSized
+ operands:
+ ptr: WordSized
+ boundsCheckLimit: WordSized
+ memoryBase: WordSized
+ mir_op: true
+
+- name: AsmJSStoreHeap
+ result_type: WordSized
+ operands:
+ ptr: WordSized
+ value: WordSized
+ boundsCheckLimit: WordSized
+ memoryBase: WordSized
+ mir_op: true
+
+- name: WasmCompareExchangeHeap
+ gen_boilerplate: false
+
+- name: WasmFence
+
+- name: WasmAtomicExchangeHeap
+ gen_boilerplate: false
+
+- name: WasmAtomicBinopHeap
+ gen_boilerplate: false
+
+- name: WasmAtomicBinopHeapForEffect
+ gen_boilerplate: false
+
+- name: WasmLoadSlot
+ result_type: WordSized
+ operands:
+ containerRef: WordSized
+ arguments:
+ offset: size_t
+ type: MIRType
+ wideningOp: MWideningOp
+ maybeTrap: MaybeTrapSiteInfo
+
+- name: WasmLoadSlotI64
+ result_type: Int64
+ operands:
+ containerRef: WordSized
+ arguments:
+ offset: size_t
+ maybeTrap: MaybeTrapSiteInfo
+
+- name: WasmStoreSlot
+ operands:
+ value: WordSized
+ containerRef: WordSized
+ arguments:
+ offset: size_t
+ type: MIRType
+ narrowingOp: MNarrowingOp
+ maybeTrap: MaybeTrapSiteInfo
+
+- name: WasmStoreSlotI64
+ operands:
+ value: Int64
+ containerRef: WordSized
+ arguments:
+ offset: size_t
+ maybeTrap: MaybeTrapSiteInfo
+
+- name: WasmLoadTableElement
+ result_type: WordSized
+ operands:
+ elements: WordSized
+ index: WordSized
+
+- name: WasmDerivedPointer
+ gen_boilerplate: false
+
+- name: WasmDerivedIndexPointer
+ gen_boilerplate: false
+
+- name: WasmStoreRef
+ operands:
+ instance: WordSized
+ valueAddr: WordSized
+ value: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: WasmParameter
+ result_type: WordSized
+
+- name: WasmParameterI64
+ gen_boilerplate: false
+
+- name: WasmReturn
+ operands:
+ rval: WordSized
+ instance: WordSized
+
+- name: WasmReturnI64
+ operands:
+ rval: Int64
+ instance: WordSized
+
+- name: WasmReturnVoid
+ operands:
+ rval: WordSized
+
+- name: WasmStackArg
+ operands:
+ arg: WordSized
+ mir_op: true
+
+- name: WasmStackArgI64
+ operands:
+ arg: Int64
+ mir_op: WasmStackArg
+
+- name: WasmNullConstant
+ result_type: WordSized
+
+- name: WasmCallIndirectAdjunctSafepoint
+ gen_boilerplate: false
+
+- name: WasmCall
+ gen_boilerplate: false
+
+- name: WasmCallLandingPrePad
+ mir_op: true
+
+- name: WasmRegisterResult
+ gen_boilerplate: false
+
+- name: WasmRegisterPairResult
+ gen_boilerplate: false
+
+- name: WasmStackResultArea
+ result_type: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: WasmStackResult
+ gen_boilerplate: false
+
+- name: WasmStackResult64
+ gen_boilerplate: false
+
+- name: AssertRangeI
+ gen_boilerplate: false
+
+- name: AssertRangeD
+ gen_boilerplate: false
+
+- name: AssertRangeF
+ gen_boilerplate: false
+
+- name: AssertRangeV
+ gen_boilerplate: false
+
+- name: AssertClass
+ operands:
+ input: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: AssertShape
+ operands:
+ input: WordSized
+ mir_op: true
+
+- name: GuardValue
+ operands:
+ input: BoxedValue
+ mir_op: true
+
+- name: GuardNullOrUndefined
+ operands:
+ input: BoxedValue
+ mir_op: true
+
+- name: GuardIsNotObject
+ operands:
+ input: BoxedValue
+ mir_op: true
+
+- name: GuardFunctionFlags
+ operands:
+ function: WordSized
+ mir_op: true
+
+- name: GuardFunctionIsNonBuiltinCtor
+ operands:
+ function: WordSized
+ num_temps: 1
+
+- name: GuardFunctionKind
+ operands:
+ function: WordSized
+ num_temps: 1
+ mir_op: true
+
+- name: GuardFunctionScript
+ operands:
+ function: WordSized
+ mir_op: true
+
+- name: IncrementWarmUpCounter
+ num_temps: 1
+ mir_op: true
+
+- name: LexicalCheck
+ operands:
+ input: BoxedValue
+ mir_op: true
+
+- name: ThrowRuntimeLexicalError
+ call_instruction: true
+ mir_op: true
+
+- name: ThrowMsg
+ call_instruction: true
+ mir_op: true
+
+- name: GlobalDeclInstantiation
+ mir_op: true
+
+- name: MemoryBarrier
+ gen_boilerplate: false
+
+- name: Debugger
+ num_temps: 1
+ call_instruction: true
+
+- name: NewTarget
+ result_type: BoxedValue
+
+- name: Random
+ gen_boilerplate: false
+
+- name: CheckReturn
+ result_type: BoxedValue
+ operands:
+ returnValue: BoxedValue
+ thisValue: BoxedValue
+
+- name: CheckIsObj
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ mir_op: true
+
+- name: CheckObjCoercible
+ operands:
+ value: BoxedValue
+
+- name: CheckClassHeritage
+ operands:
+ heritage: BoxedValue
+ num_temps: 2
+
+- name: CheckThis
+ operands:
+ value: BoxedValue
+
+- name: CheckThisReinit
+ operands:
+ thisValue: BoxedValue
+
+- name: Generator
+ result_type: WordSized
+ operands:
+ callee: WordSized
+ environmentChain: WordSized
+ argsObject: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: AsyncResolve
+ result_type: WordSized
+ operands:
+ generator: WordSized
+ valueOrReason: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: AsyncAwait
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ generator: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: CanSkipAwait
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: MaybeExtractAwaitValue
+ result_type: BoxedValue
+ operands:
+ value: BoxedValue
+ canSkip: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: DebugCheckSelfHosted
+ operands:
+ value: BoxedValue
+ call_instruction: true
+
+- name: FinishBoundFunctionInit
+ operands:
+ bound: WordSized
+ target: WordSized
+ argCount: WordSized
+ num_temps: 2
+
+- name: IsPackedArray
+ result_type: WordSized
+ operands:
+ object: WordSized
+ num_temps: 1
+
+- name: GuardArrayIsPacked
+ operands:
+ array: WordSized
+ num_temps: 2
+ mir_op: true
+
+- name: GetPrototypeOf
+ result_type: BoxedValue
+ operands:
+ target: WordSized
+
+- name: ObjectWithProto
+ result_type: WordSized
+ operands:
+ prototype: BoxedValue
+ call_instruction: true
+
+- name: ObjectStaticProto
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+- name: BuiltinObject
+ result_type: WordSized
+ call_instruction: true
+ mir_op: true
+
+- name: SuperFunction
+ result_type: BoxedValue
+ operands:
+ callee: WordSized
+ num_temps: 1
+
+- name: InitHomeObject
+ result_type: WordSized
+ operands:
+ function: WordSized
+ homeObject: BoxedValue
+
+- name: IsTypedArrayConstructor
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+- name: LoadValueTag
+ result_type: WordSized
+ operands:
+ value: BoxedValue
+
+- name: GuardTagNotEqual
+ operands:
+ lhs: WordSized
+ rhs: WordSized
+
+- name: LoadWrapperTarget
+ result_type: WordSized
+ operands:
+ object: WordSized
+
+- name: GuardHasGetterSetter
+ operands:
+ object: WordSized
+ num_temps: 3
+ call_instruction: true
+ mir_op: true
+
+- name: GuardIsExtensible
+ operands:
+ object: WordSized
+ num_temps: 1
+
+- name: GuardInt32IsNonNegative
+ operands:
+ index: WordSized
+
+- name: GuardInt32Range
+ operands:
+ input: WordSized
+ mir_op: true
+
+- name: GuardIndexIsNotDenseElement
+ operands:
+ object: WordSized
+ index: WordSized
+ num_temps: 2
+
+- name: GuardIndexIsValidUpdateOrAdd
+ operands:
+ object: WordSized
+ index: WordSized
+ num_temps: 2
+
+- name: CallAddOrUpdateSparseElement
+ operands:
+ object: WordSized
+ index: WordSized
+ value: BoxedValue
+ call_instruction: true
+ mir_op: true
+
+- name: CallGetSparseElement
+ result_type: BoxedValue
+ operands:
+ object: WordSized
+ index: WordSized
+ call_instruction: true
+
+- name: CallNativeGetElement
+ result_type: BoxedValue
+ operands:
+ object: WordSized
+ index: WordSized
+ call_instruction: true
+
+- name: CallNativeGetElementSuper
+ result_type: BoxedValue
+ operands:
+ object: WordSized
+ index: WordSized
+ receiver: BoxedValue
+ call_instruction: true
+
+- name: CallObjectHasSparseElement
+ result_type: WordSized
+ operands:
+ object: WordSized
+ index: WordSized
+ num_temps: 2
+ call_instruction: true
+
+- name: BigIntAsIntN
+ result_type: WordSized
+ operands:
+ bits: WordSized
+ input: WordSized
+ call_instruction: true
+
+- name: BigIntAsIntN64
+ gen_boilerplate: false
+
+- name: BigIntAsIntN32
+ gen_boilerplate: false
+
+- name: GuardNonGCThing
+ operands:
+ input: BoxedValue
+
+- name: ToHashableNonGCThing
+ result_type: BoxedValue
+ operands:
+ input: BoxedValue
+ num_temps: 1
+
+- name: ToHashableString
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+- name: ToHashableValue
+ result_type: BoxedValue
+ operands:
+ input: BoxedValue
+ num_temps: 1
+
+- name: HashNonGCThing
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+ num_temps: 1
+
+- name: HashString
+ result_type: WordSized
+ operands:
+ input: WordSized
+ num_temps: 1
+
+- name: HashSymbol
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+- name: HashBigInt
+ result_type: WordSized
+ operands:
+ input: WordSized
+ num_temps: 3
+
+- name: HashObject
+ result_type: WordSized
+ operands:
+ setObject: WordSized
+ input: BoxedValue
+ num_temps: 4
+
+- name: HashValue
+ result_type: WordSized
+ operands:
+ setObject: WordSized
+ input: BoxedValue
+ num_temps: 4
+
+- name: SetObjectHasNonBigInt
+ result_type: WordSized
+ operands:
+ setObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 2
+
+- name: SetObjectHasBigInt
+ result_type: WordSized
+ operands:
+ setObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 4
+
+- name: SetObjectHasValue
+ result_type: WordSized
+ operands:
+ setObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 4
+
+- name: SetObjectHasValueVMCall
+ result_type: WordSized
+ call_instruction: true
+ operands:
+ setObject: WordSized
+ input: BoxedValue
+
+- name: SetObjectSize
+ result_type: WordSized
+ operands:
+ setObject: WordSized
+
+- name: MapObjectHasNonBigInt
+ result_type: WordSized
+ operands:
+ mapObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 2
+
+- name: MapObjectHasBigInt
+ result_type: WordSized
+ operands:
+ mapObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 4
+
+- name: MapObjectHasValue
+ result_type: WordSized
+ operands:
+ mapObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 4
+
+- name: MapObjectHasValueVMCall
+ result_type: WordSized
+ call_instruction: true
+ operands:
+ mapObject: WordSized
+ input: BoxedValue
+
+- name: MapObjectGetNonBigInt
+ result_type: BoxedValue
+ operands:
+ mapObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 2
+
+- name: MapObjectGetBigInt
+ result_type: BoxedValue
+ operands:
+ mapObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 4
+
+- name: MapObjectGetValue
+ result_type: BoxedValue
+ operands:
+ mapObject: WordSized
+ input: BoxedValue
+ hash: WordSized
+ num_temps: 4
+
+- name: MapObjectGetValueVMCall
+ result_type: BoxedValue
+ call_instruction: true
+ operands:
+ mapObject: WordSized
+ input: BoxedValue
+
+- name: MapObjectSize
+ result_type: WordSized
+ operands:
+ mapObject: WordSized
+
+- name: BigIntAsUintN
+ result_type: WordSized
+ operands:
+ bits: WordSized
+ input: WordSized
+ call_instruction: true
+
+- name: BigIntAsUintN64
+ gen_boilerplate: false
+
+- name: BigIntAsUintN32
+ gen_boilerplate: false
+
+- name: IonToWasmCall
+ gen_boilerplate: false
+
+- name: IonToWasmCallV
+ gen_boilerplate: false
+
+- name: IonToWasmCallI64
+ gen_boilerplate: false
+
+- name: WasmBoxValue
+ result_type: WordSized
+ operands:
+ input: BoxedValue
+
+- name: WasmAnyRefFromJSObject
+ result_type: WordSized
+ operands:
+ input: WordSized
+
+# Constant Simd128
+- name: Simd128
+ result_type: WordSized
+ arguments:
+ simd128: SimdConstant
+
+- name: WasmTernarySimd128
+ gen_boilerplate: false
+
+- name: WasmBinarySimd128
+ gen_boilerplate: false
+
+- name: WasmBinarySimd128WithConstant
+ gen_boilerplate: false
+
+- name: WasmVariableShiftSimd128
+ gen_boilerplate: false
+
+- name: WasmConstantShiftSimd128
+ gen_boilerplate: false
+
+- name: WasmSignReplicationSimd128
+ gen_boilerplate: false
+
+- name: WasmShuffleSimd128
+ gen_boilerplate: false
+
+- name: WasmPermuteSimd128
+ gen_boilerplate: false
+
+- name: WasmReplaceLaneSimd128
+ gen_boilerplate: false
+
+- name: WasmReplaceInt64LaneSimd128
+ gen_boilerplate: false
+
+- name: WasmScalarToSimd128
+ gen_boilerplate: false
+
+- name: WasmInt64ToSimd128
+ gen_boilerplate: false
+
+- name: WasmUnarySimd128
+ gen_boilerplate: false
+
+- name: WasmReduceSimd128
+ gen_boilerplate: false
+
+- name: WasmReduceAndBranchSimd128
+ gen_boilerplate: false
+
+- name: WasmReduceSimd128ToInt64
+ gen_boilerplate: false
+
+- name: WasmLoadLaneSimd128
+ gen_boilerplate: false
+
+- name: WasmStoreLaneSimd128
+ gen_boilerplate: false
+
+- name: Unbox
+ gen_boilerplate: false
+
+- name: UnboxFloatingPoint
+ gen_boilerplate: false
+
+- name: WasmUint32ToDouble
+ gen_boilerplate: false
+
+- name: WasmUint32ToFloat32
+ gen_boilerplate: false
+
+- name: DivI
+ gen_boilerplate: false
+
+- name: ModI
+ gen_boilerplate: false
+
+- name: DivPowTwoI
+ gen_boilerplate: false
+
+- name: ModPowTwoI
+ gen_boilerplate: false
+
+- name: TableSwitch
+ gen_boilerplate: false
+
+- name: TableSwitchV
+ gen_boilerplate: false
+
+- name: MulI
+ gen_boilerplate: false
+
+#ifdef JS_CODEGEN_X86
+- name: BoxFloatingPoint
+ gen_boilerplate: false
+
+- name: DivOrModI64
+ gen_boilerplate: false
+
+- name: UDivOrModI64
+ gen_boilerplate: false
+
+- name: DivOrModConstantI
+ gen_boilerplate: false
+
+- name: UDivOrMod
+ gen_boilerplate: false
+
+- name: UDivOrModConstant
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt64
+ gen_boilerplate: false
+
+- name: Int64ToFloatingPoint
+ gen_boilerplate: false
+
+- name: WasmAtomicLoadI64
+ gen_boilerplate: false
+
+- name: WasmAtomicStoreI64
+ gen_boilerplate: false
+
+- name: WasmCompareExchangeI64
+ gen_boilerplate: false
+
+- name: WasmAtomicBinopI64
+ gen_boilerplate: false
+
+- name: WasmAtomicExchangeI64
+ gen_boilerplate: false
+#endif
+
+#ifdef JS_CODEGEN_X64
+- name: DivOrModI64
+ gen_boilerplate: false
+
+- name: UDivOrModI64
+ gen_boilerplate: false
+
+- name: DivOrModConstantI
+ gen_boilerplate: false
+
+- name: UDivOrMod
+ gen_boilerplate: false
+
+- name: UDivOrModConstant
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt64
+ gen_boilerplate: false
+
+- name: Int64ToFloatingPoint
+ gen_boilerplate: false
+#endif
+
+#ifdef JS_CODEGEN_ARM
+- name: BoxFloatingPoint
+ gen_boilerplate: false
+
+- name: DivOrModI64
+ gen_boilerplate: false
+
+- name: UDivOrModI64
+ gen_boilerplate: false
+
+- name: SoftDivI
+ gen_boilerplate: false
+
+- name: SoftModI
+ gen_boilerplate: false
+
+- name: ModMaskI
+ gen_boilerplate: false
+
+- name: UDiv
+ gen_boilerplate: false
+
+- name: UMod
+ gen_boilerplate: false
+
+- name: SoftUDivOrMod
+ gen_boilerplate: false
+
+- name: Int64ToFloatingPointCall
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt64
+ gen_boilerplate: false
+
+- name: WasmAtomicLoadI64
+ gen_boilerplate: false
+
+- name: WasmAtomicStoreI64
+ gen_boilerplate: false
+
+- name: WasmCompareExchangeI64
+ gen_boilerplate: false
+
+- name: WasmAtomicBinopI64
+ gen_boilerplate: false
+
+- name: WasmAtomicExchangeI64
+ gen_boilerplate: false
+#endif
+
+#ifdef JS_CODEGEN_ARM64
+- name: DivOrModI64
+ gen_boilerplate: false
+
+- name: UDivOrModI64
+ gen_boilerplate: false
+
+- name: DivConstantI
+ gen_boilerplate: false
+
+- name: UDivConstantI
+ gen_boilerplate: false
+
+- name: ModMaskI
+ gen_boilerplate: false
+
+- name: UDiv
+ gen_boilerplate: false
+
+- name: UMod
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt64
+ gen_boilerplate: false
+
+- name: Int64ToFloatingPoint
+ gen_boilerplate: false
+#endif
+
+#ifdef JS_CODEGEN_MIPS32
+- name: BoxFloatingPoint
+ gen_boilerplate: false
+
+- name: DivOrModI64
+ gen_boilerplate: false
+
+- name: UDivOrModI64
+ gen_boilerplate: false
+
+- name: UDivOrMod
+ gen_boilerplate: false
+
+- name: ModMaskI
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt64
+ gen_boilerplate: false
+
+- name: Int64ToFloatingPoint
+ gen_boilerplate: false
+
+- name: WasmUnalignedLoad
+ gen_boilerplate: false
+
+- name: WasmUnalignedLoadI64
+ gen_boilerplate: false
+
+- name: WasmUnalignedStore
+ gen_boilerplate: false
+
+- name: WasmUnalignedStoreI64
+ gen_boilerplate: false
+
+- name: WasmAtomicLoadI64
+ gen_boilerplate: false
+
+- name: WasmAtomicStoreI64
+ gen_boilerplate: false
+
+- name: WasmCompareExchangeI64
+ gen_boilerplate: false
+
+- name: WasmAtomicBinopI64
+ gen_boilerplate: false
+
+- name: WasmAtomicExchangeI64
+ gen_boilerplate: false
+#endif
+
+#ifdef JS_CODEGEN_MIPS64
+- name: DivOrModI64
+ gen_boilerplate: false
+
+- name: UDivOrMod
+ gen_boilerplate: false
+
+- name: UDivOrModI64
+ gen_boilerplate: false
+
+- name: ModMaskI
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt64
+ gen_boilerplate: false
+
+- name: Int64ToFloatingPoint
+ gen_boilerplate: false
+
+- name: WasmUnalignedLoad
+ gen_boilerplate: false
+
+- name: WasmUnalignedLoadI64
+ gen_boilerplate: false
+
+- name: WasmUnalignedStore
+ gen_boilerplate: false
+
+- name: WasmUnalignedStoreI64
+ gen_boilerplate: false
+
+- name: WasmCompareExchangeI64
+ gen_boilerplate: false
+
+- name: WasmAtomicBinopI64
+ gen_boilerplate: false
+
+- name: WasmAtomicExchangeI64
+ gen_boilerplate: false
+#endif
+
+#ifdef JS_CODEGEN_LOONG64
+- name: DivOrModI64
+ gen_boilerplate: false
+
+- name: UDivOrMod
+ gen_boilerplate: false
+
+- name: UDivOrModI64
+ gen_boilerplate: false
+
+- name: ModMaskI
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt64
+ gen_boilerplate: false
+
+- name: Int64ToFloatingPoint
+ gen_boilerplate: false
+
+- name: WasmCompareExchangeI64
+ gen_boilerplate: false
+
+- name: WasmAtomicBinopI64
+ gen_boilerplate: false
+
+- name: WasmAtomicExchangeI64
+ gen_boilerplate: false
+#endif
+
+#ifdef FUZZING_JS_FUZZILLI
+- name: FuzzilliHashT
+ gen_boilerplate: false
+
+- name: FuzzilliHashV
+ gen_boilerplate: false
+
+- name: FuzzilliHashStore
+ gen_boilerplate: false
+#endif
diff --git a/js/src/jit/Label.cpp b/js/src/jit/Label.cpp
new file mode 100644
index 0000000000..e1c9db76c4
--- /dev/null
+++ b/js/src/jit/Label.cpp
@@ -0,0 +1,29 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Label.h"
+
+#include "mozilla/Assertions.h"
+
+#include "jit/CompileWrappers.h"
+#include "jit/JitContext.h"
+#include "js/Utility.h"
+
+namespace js::jit {
+
+#ifdef DEBUG
+Label::~Label() {
+ // The assertion below doesn't hold if an error occurred.
+ JitContext* context = MaybeGetJitContext();
+ bool hadError =
+ js::oom::HadSimulatedOOM() ||
+ (context && context->runtime && context->runtime->hadOutOfMemory()) ||
+ (context && !context->runtime && context->hasOOM());
+ MOZ_ASSERT_IF(!hadError, !used());
+}
+#endif
+
+} // namespace js::jit
diff --git a/js/src/jit/Label.h b/js/src/jit/Label.h
new file mode 100644
index 0000000000..c1ed1c6b25
--- /dev/null
+++ b/js/src/jit/Label.h
@@ -0,0 +1,106 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Label_h
+#define jit_Label_h
+
+#include "mozilla/Assertions.h"
+
+#include <stdint.h>
+
+namespace js {
+namespace jit {
+
+struct LabelBase {
+ private:
+ // We use uint32_t instead of bool to ensure MSVC packs these fields
+ // correctly.
+ uint32_t bound_ : 1;
+
+ // offset_ < INVALID_OFFSET means that the label is either bound or has
+ // incoming uses and needs to be bound.
+ uint32_t offset_ : 31;
+
+ void operator=(const LabelBase& label) = delete;
+
+#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_LOONG64)
+ public:
+#endif
+ static const uint32_t INVALID_OFFSET = 0x7fffffff; // UINT31_MAX.
+
+ public:
+ LabelBase() : bound_(false), offset_(INVALID_OFFSET) {}
+
+ // If the label is bound, all incoming edges have been patched and any
+ // future incoming edges will be immediately patched.
+ bool bound() const { return bound_; }
+ int32_t offset() const {
+ MOZ_ASSERT(bound() || used());
+ return offset_;
+ }
+ // Returns whether the label is not bound, but has incoming uses.
+ bool used() const { return !bound() && offset_ < INVALID_OFFSET; }
+ // Binds the label, fixing its final position in the code stream.
+ void bind(int32_t offset) {
+ MOZ_ASSERT(!bound());
+ MOZ_ASSERT(offset >= 0);
+ MOZ_ASSERT(uint32_t(offset) < INVALID_OFFSET);
+ offset_ = offset;
+ bound_ = true;
+ MOZ_ASSERT(offset_ == offset, "offset fits in 31 bits");
+ }
+ // Marks the label as neither bound nor used.
+ void reset() {
+ offset_ = INVALID_OFFSET;
+ bound_ = false;
+ }
+ // Sets the label's latest used position.
+ void use(int32_t offset) {
+ MOZ_ASSERT(!bound());
+ MOZ_ASSERT(offset >= 0);
+ MOZ_ASSERT(uint32_t(offset) < INVALID_OFFSET);
+ offset_ = offset;
+ MOZ_ASSERT(offset_ == offset, "offset fits in 31 bits");
+ }
+};
+
+// A label represents a position in an assembly buffer that may or may not have
+// already been generated. Labels can either be "bound" or "unbound", the
+// former meaning that its position is known and the latter that its position
+// is not yet known.
+//
+// A jump to an unbound label adds that jump to the label's incoming queue. A
+// jump to a bound label automatically computes the jump distance. The process
+// of binding a label automatically corrects all incoming jumps.
+class Label : public LabelBase {
+ public:
+#ifdef DEBUG
+ ~Label();
+#endif
+};
+
+static_assert(sizeof(Label) == sizeof(uint32_t),
+ "Label should have same size as uint32_t");
+
+// Label's destructor asserts that if it has been used it has also been bound.
+// In the case long-lived labels, however, failed compilation (e.g. OOM) will
+// trigger this failure innocuously. This Label silences the assertion.
+class NonAssertingLabel : public Label {
+ public:
+#ifdef DEBUG
+ ~NonAssertingLabel() {
+ if (used()) {
+ bind(0);
+ }
+ }
+#endif
+};
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_Label_h
diff --git a/js/src/jit/Linker.cpp b/js/src/jit/Linker.cpp
new file mode 100644
index 0000000000..aedebcecbe
--- /dev/null
+++ b/js/src/jit/Linker.cpp
@@ -0,0 +1,79 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Linker.h"
+
+#include "jit/JitZone.h"
+#include "util/Memory.h"
+
+#include "gc/StoreBuffer-inl.h"
+
+namespace js {
+namespace jit {
+
+JitCode* Linker::newCode(JSContext* cx, CodeKind kind) {
+ JS::AutoAssertNoGC nogc(cx);
+ if (masm.oom()) {
+ return fail(cx);
+ }
+
+ static const size_t ExecutableAllocatorAlignment = sizeof(void*);
+ static_assert(CodeAlignment >= ExecutableAllocatorAlignment,
+ "Unexpected alignment requirements");
+
+ // We require enough bytes for the code, header, and worst-case alignment
+ // padding.
+ size_t bytesNeeded = masm.bytesNeeded() + sizeof(JitCodeHeader) +
+ (CodeAlignment - ExecutableAllocatorAlignment);
+ if (bytesNeeded >= MAX_BUFFER_SIZE) {
+ return fail(cx);
+ }
+
+ // ExecutableAllocator requires bytesNeeded to be aligned.
+ bytesNeeded = AlignBytes(bytesNeeded, ExecutableAllocatorAlignment);
+
+ JitZone* jitZone = cx->zone()->getJitZone(cx);
+ if (!jitZone) {
+ // Note: don't call fail(cx) here, getJitZone reports OOM.
+ return nullptr;
+ }
+
+ ExecutablePool* pool;
+ uint8_t* result =
+ (uint8_t*)jitZone->execAlloc().alloc(cx, bytesNeeded, &pool, kind);
+ if (!result) {
+ return fail(cx);
+ }
+
+ // The JitCodeHeader will be stored right before the code buffer.
+ uint8_t* codeStart = result + sizeof(JitCodeHeader);
+
+ // Bump the code up to a nice alignment.
+ codeStart = (uint8_t*)AlignBytes((uintptr_t)codeStart, CodeAlignment);
+ MOZ_ASSERT(codeStart + masm.bytesNeeded() <= result + bytesNeeded);
+ uint32_t headerSize = codeStart - result;
+ JitCode* code =
+ JitCode::New<NoGC>(cx, codeStart, bytesNeeded, headerSize, pool, kind);
+ if (!code) {
+ return fail(cx);
+ }
+ if (masm.oom()) {
+ return fail(cx);
+ }
+ awjcf.emplace(result, bytesNeeded);
+ if (!awjcf->makeWritable()) {
+ return fail(cx);
+ }
+ code->copyFrom(masm);
+ masm.link(code);
+ if (masm.embedsNurseryPointers()) {
+ cx->runtime()->gc.storeBuffer().putWholeCell(code);
+ }
+ return code;
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/Linker.h b/js/src/jit/Linker.h
new file mode 100644
index 0000000000..01df1c819a
--- /dev/null
+++ b/js/src/jit/Linker.h
@@ -0,0 +1,52 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Linker_h
+#define jit_Linker_h
+
+#include "mozilla/Maybe.h"
+
+#include <stdint.h>
+
+#include "jstypes.h"
+
+#include "jit/AutoWritableJitCode.h"
+#include "jit/MacroAssembler.h"
+#include "vm/Runtime.h"
+
+struct JS_PUBLIC_API JSContext;
+
+namespace js {
+namespace jit {
+
+class JitCode;
+
+enum class CodeKind : uint8_t;
+
+class Linker {
+ MacroAssembler& masm;
+ mozilla::Maybe<AutoWritableJitCodeFallible> awjcf;
+
+ JitCode* fail(JSContext* cx) {
+ ReportOutOfMemory(cx);
+ return nullptr;
+ }
+
+ public:
+ // Construct a linker with a rooted macro assembler.
+ explicit Linker(MacroAssembler& masm) : masm(masm) { masm.finish(); }
+
+ // Create a new JitCode object and populate it with the contents of the
+ // macro assember buffer.
+ //
+ // This method cannot GC. Errors are reported to the context.
+ JitCode* newCode(JSContext* cx, CodeKind kind);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Linker_h */
diff --git a/js/src/jit/Lowering.cpp b/js/src/jit/Lowering.cpp
new file mode 100644
index 0000000000..d209333511
--- /dev/null
+++ b/js/src/jit/Lowering.cpp
@@ -0,0 +1,7004 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Lowering.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <type_traits>
+
+#include "jit/ABIArgGenerator.h"
+#include "jit/IonOptimizationLevels.h"
+#include "jit/JitSpewer.h"
+#include "jit/LIR.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "jit/SharedICRegisters.h"
+#include "js/experimental/JitInfo.h" // JSJitInfo
+#include "util/Memory.h"
+#include "wasm/WasmCodegenTypes.h"
+#include "wasm/WasmInstanceData.h"
+#include "wasm/WasmJS.h" // for wasm::ReportSimdAnalysis
+
+#include "jit/shared/Lowering-shared-inl.h"
+#include "vm/BytecodeUtil-inl.h"
+
+using namespace js;
+using namespace jit;
+
+using JS::GenericNaN;
+using mozilla::DebugOnly;
+
+LBoxAllocation LIRGenerator::useBoxFixedAtStart(MDefinition* mir,
+ ValueOperand op) {
+#if defined(JS_NUNBOX32)
+ return useBoxFixed(mir, op.typeReg(), op.payloadReg(), true);
+#elif defined(JS_PUNBOX64)
+ return useBoxFixed(mir, op.valueReg(), op.scratchReg(), true);
+#endif
+}
+
+LBoxAllocation LIRGenerator::useBoxAtStart(MDefinition* mir,
+ LUse::Policy policy) {
+ return useBox(mir, policy, /* useAtStart = */ true);
+}
+
+void LIRGenerator::visitParameter(MParameter* param) {
+ ptrdiff_t offset;
+ if (param->index() == MParameter::THIS_SLOT) {
+ offset = THIS_FRAME_ARGSLOT;
+ } else {
+ offset = 1 + param->index();
+ }
+
+ LParameter* ins = new (alloc()) LParameter;
+ defineBox(ins, param, LDefinition::FIXED);
+
+ offset *= sizeof(Value);
+#if defined(JS_NUNBOX32)
+# if MOZ_BIG_ENDIAN()
+ ins->getDef(0)->setOutput(LArgument(offset));
+ ins->getDef(1)->setOutput(LArgument(offset + 4));
+# else
+ ins->getDef(0)->setOutput(LArgument(offset + 4));
+ ins->getDef(1)->setOutput(LArgument(offset));
+# endif
+#elif defined(JS_PUNBOX64)
+ ins->getDef(0)->setOutput(LArgument(offset));
+#endif
+}
+
+void LIRGenerator::visitCallee(MCallee* ins) {
+ define(new (alloc()) LCallee(), ins);
+}
+
+void LIRGenerator::visitIsConstructing(MIsConstructing* ins) {
+ define(new (alloc()) LIsConstructing(), ins);
+}
+
+void LIRGenerator::visitGoto(MGoto* ins) {
+ add(new (alloc()) LGoto(ins->target()));
+}
+
+void LIRGenerator::visitTableSwitch(MTableSwitch* tableswitch) {
+ MDefinition* opd = tableswitch->getOperand(0);
+
+ // There should be at least 1 successor. The default case!
+ MOZ_ASSERT(tableswitch->numSuccessors() > 0);
+
+ // If there are no cases, the default case is always taken.
+ if (tableswitch->numSuccessors() == 1) {
+ add(new (alloc()) LGoto(tableswitch->getDefault()));
+ return;
+ }
+
+ // If we don't know the type.
+ if (opd->type() == MIRType::Value) {
+ LTableSwitchV* lir = newLTableSwitchV(tableswitch);
+ add(lir);
+ return;
+ }
+
+ // Case indices are numeric, so other types will always go to the default
+ // case.
+ if (opd->type() != MIRType::Int32 && opd->type() != MIRType::Double) {
+ add(new (alloc()) LGoto(tableswitch->getDefault()));
+ return;
+ }
+
+ // Return an LTableSwitch, capable of handling either an integer or
+ // floating-point index.
+ LAllocation index;
+ LDefinition tempInt;
+ if (opd->type() == MIRType::Int32) {
+ index = useRegisterAtStart(opd);
+ tempInt = tempCopy(opd, 0);
+ } else {
+ index = useRegister(opd);
+ tempInt = temp(LDefinition::GENERAL);
+ }
+ add(newLTableSwitch(index, tempInt, tableswitch));
+}
+
+void LIRGenerator::visitCheckOverRecursed(MCheckOverRecursed* ins) {
+ LCheckOverRecursed* lir = new (alloc()) LCheckOverRecursed();
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewArray(MNewArray* ins) {
+ LNewArray* lir = new (alloc()) LNewArray(temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewArrayDynamicLength(MNewArrayDynamicLength* ins) {
+ MDefinition* length = ins->length();
+ MOZ_ASSERT(length->type() == MIRType::Int32);
+
+ LNewArrayDynamicLength* lir =
+ new (alloc()) LNewArrayDynamicLength(useRegister(length), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewIterator(MNewIterator* ins) {
+ LNewIterator* lir = new (alloc()) LNewIterator(temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewTypedArray(MNewTypedArray* ins) {
+ LNewTypedArray* lir = new (alloc()) LNewTypedArray(temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewTypedArrayDynamicLength(
+ MNewTypedArrayDynamicLength* ins) {
+ MDefinition* length = ins->length();
+ MOZ_ASSERT(length->type() == MIRType::Int32);
+
+ LNewTypedArrayDynamicLength* lir =
+ new (alloc()) LNewTypedArrayDynamicLength(useRegister(length), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewTypedArrayFromArray(MNewTypedArrayFromArray* ins) {
+ MDefinition* array = ins->array();
+ MOZ_ASSERT(array->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LNewTypedArrayFromArray(useRegisterAtStart(array));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewTypedArrayFromArrayBuffer(
+ MNewTypedArrayFromArrayBuffer* ins) {
+ MDefinition* arrayBuffer = ins->arrayBuffer();
+ MDefinition* byteOffset = ins->byteOffset();
+ MDefinition* length = ins->length();
+ MOZ_ASSERT(arrayBuffer->type() == MIRType::Object);
+ MOZ_ASSERT(byteOffset->type() == MIRType::Value);
+ MOZ_ASSERT(length->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LNewTypedArrayFromArrayBuffer(
+ useRegisterAtStart(arrayBuffer), useBoxAtStart(byteOffset),
+ useBoxAtStart(length));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewObject(MNewObject* ins) {
+ LNewObject* lir = new (alloc()) LNewObject(temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewPlainObject(MNewPlainObject* ins) {
+ LNewPlainObject* lir = new (alloc()) LNewPlainObject(temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewArrayObject(MNewArrayObject* ins) {
+ LNewArrayObject* lir = new (alloc()) LNewArrayObject(temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewNamedLambdaObject(MNewNamedLambdaObject* ins) {
+ LNewNamedLambdaObject* lir = new (alloc()) LNewNamedLambdaObject(temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewCallObject(MNewCallObject* ins) {
+ LNewCallObject* lir = new (alloc()) LNewCallObject(temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewStringObject(MNewStringObject* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::String);
+
+ LNewStringObject* lir =
+ new (alloc()) LNewStringObject(useRegister(ins->input()), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitInitElemGetterSetter(MInitElemGetterSetter* ins) {
+ LInitElemGetterSetter* lir = new (alloc()) LInitElemGetterSetter(
+ useRegisterAtStart(ins->object()), useBoxAtStart(ins->id()),
+ useRegisterAtStart(ins->value()));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitMutateProto(MMutateProto* ins) {
+ LMutateProto* lir = new (alloc()) LMutateProto(
+ useRegisterAtStart(ins->object()), useBoxAtStart(ins->value()));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitInitPropGetterSetter(MInitPropGetterSetter* ins) {
+ LInitPropGetterSetter* lir = new (alloc()) LInitPropGetterSetter(
+ useRegisterAtStart(ins->object()), useRegisterAtStart(ins->value()));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCreateThis(MCreateThis* ins) {
+ LCreateThis* lir =
+ new (alloc()) LCreateThis(useRegisterOrConstantAtStart(ins->callee()),
+ useRegisterOrConstantAtStart(ins->newTarget()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCreateArgumentsObject(MCreateArgumentsObject* ins) {
+ LAllocation callObj = useRegisterAtStart(ins->getCallObject());
+ LCreateArgumentsObject* lir = new (alloc())
+ LCreateArgumentsObject(callObj, tempFixed(CallTempReg0),
+ tempFixed(CallTempReg1), tempFixed(CallTempReg2));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCreateInlinedArgumentsObject(
+ MCreateInlinedArgumentsObject* ins) {
+ LAllocation callObj = useRegisterAtStart(ins->getCallObject());
+ LAllocation callee = useRegisterAtStart(ins->getCallee());
+ uint32_t numActuals = ins->numActuals();
+ uint32_t numOperands = numActuals * BOX_PIECES +
+ LCreateInlinedArgumentsObject::NumNonArgumentOperands;
+
+ auto* lir = allocateVariadic<LCreateInlinedArgumentsObject>(
+ numOperands, tempFixed(CallTempReg0), tempFixed(CallTempReg1));
+ if (!lir) {
+ abort(AbortReason::Alloc,
+ "OOM: LIRGenerator::visitCreateInlinedArgumentsObject");
+ return;
+ }
+
+ lir->setOperand(LCreateInlinedArgumentsObject::CallObj, callObj);
+ lir->setOperand(LCreateInlinedArgumentsObject::Callee, callee);
+ for (uint32_t i = 0; i < numActuals; i++) {
+ MDefinition* arg = ins->getArg(i);
+ uint32_t index = LCreateInlinedArgumentsObject::ArgIndex(i);
+ lir->setBoxOperand(index, useBoxOrTypedOrConstant(arg,
+ /*useConstant = */ true,
+ /*useAtStart = */ true));
+ }
+
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGetInlinedArgument(MGetInlinedArgument* ins) {
+#if defined(JS_PUNBOX64)
+ // On 64-bit architectures, we don't support boxing a typed register
+ // in-place without using a scratch register, so the result register
+ // can't be the same as any of the inputs. Fortunately, those
+ // architectures have registers to spare.
+ const bool useAtStart = false;
+#else
+ const bool useAtStart = true;
+#endif
+
+ LAllocation index =
+ useAtStart ? useRegisterAtStart(ins->index()) : useRegister(ins->index());
+ uint32_t numActuals = ins->numActuals();
+ uint32_t numOperands =
+ numActuals * BOX_PIECES + LGetInlinedArgument::NumNonArgumentOperands;
+
+ auto* lir = allocateVariadic<LGetInlinedArgument>(numOperands);
+ if (!lir) {
+ abort(AbortReason::Alloc, "OOM: LIRGenerator::visitGetInlinedArgument");
+ return;
+ }
+
+ lir->setOperand(LGetInlinedArgument::Index, index);
+ for (uint32_t i = 0; i < numActuals; i++) {
+ MDefinition* arg = ins->getArg(i);
+ uint32_t index = LGetInlinedArgument::ArgIndex(i);
+ lir->setBoxOperand(
+ index, useBoxOrTypedOrConstant(arg,
+ /*useConstant = */ true, useAtStart));
+ }
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitGetInlinedArgumentHole(MGetInlinedArgumentHole* ins) {
+#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_MIPS64)
+ // On some 64-bit architectures, we don't support boxing a typed
+ // register in-place without using a scratch register, so the result
+ // register can't be the same as any of the inputs. Fortunately,
+ // those architectures have registers to spare.
+ const bool useAtStart = false;
+#else
+ const bool useAtStart = true;
+#endif
+
+ LAllocation index =
+ useAtStart ? useRegisterAtStart(ins->index()) : useRegister(ins->index());
+ uint32_t numActuals = ins->numActuals();
+ uint32_t numOperands =
+ numActuals * BOX_PIECES + LGetInlinedArgumentHole::NumNonArgumentOperands;
+
+ auto* lir = allocateVariadic<LGetInlinedArgumentHole>(numOperands);
+ if (!lir) {
+ abort(AbortReason::Alloc, "OOM: LIRGenerator::visitGetInlinedArgumentHole");
+ return;
+ }
+
+ lir->setOperand(LGetInlinedArgumentHole::Index, index);
+ for (uint32_t i = 0; i < numActuals; i++) {
+ MDefinition* arg = ins->getArg(i);
+ uint32_t index = LGetInlinedArgumentHole::ArgIndex(i);
+ lir->setBoxOperand(
+ index, useBoxOrTypedOrConstant(arg,
+ /*useConstant = */ true, useAtStart));
+ }
+ assignSnapshot(lir, ins->bailoutKind());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitGetArgumentsObjectArg(MGetArgumentsObjectArg* ins) {
+ LAllocation argsObj = useRegister(ins->argsObject());
+ LGetArgumentsObjectArg* lir =
+ new (alloc()) LGetArgumentsObjectArg(argsObj, temp());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitSetArgumentsObjectArg(MSetArgumentsObjectArg* ins) {
+ LAllocation argsObj = useRegister(ins->argsObject());
+ LSetArgumentsObjectArg* lir = new (alloc())
+ LSetArgumentsObjectArg(argsObj, useBox(ins->value()), temp());
+ add(lir, ins);
+}
+
+void LIRGenerator::visitLoadArgumentsObjectArg(MLoadArgumentsObjectArg* ins) {
+ MDefinition* argsObj = ins->argsObject();
+ MOZ_ASSERT(argsObj->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ auto* lir = new (alloc())
+ LLoadArgumentsObjectArg(useRegister(argsObj), useRegister(index), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitLoadArgumentsObjectArgHole(
+ MLoadArgumentsObjectArgHole* ins) {
+ MDefinition* argsObj = ins->argsObject();
+ MOZ_ASSERT(argsObj->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LLoadArgumentsObjectArgHole(
+ useRegister(argsObj), useRegister(index), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitInArgumentsObjectArg(MInArgumentsObjectArg* ins) {
+ MDefinition* argsObj = ins->argsObject();
+ MOZ_ASSERT(argsObj->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ auto* lir = new (alloc())
+ LInArgumentsObjectArg(useRegister(argsObj), useRegister(index), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitArgumentsObjectLength(MArgumentsObjectLength* ins) {
+ MDefinition* argsObj = ins->argsObject();
+ MOZ_ASSERT(argsObj->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LArgumentsObjectLength(useRegister(argsObj));
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitArrayFromArgumentsObject(
+ MArrayFromArgumentsObject* ins) {
+ MDefinition* argsObj = ins->argsObject();
+ MOZ_ASSERT(argsObj->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LArrayFromArgumentsObject(useRegisterAtStart(argsObj));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGuardArgumentsObjectFlags(
+ MGuardArgumentsObjectFlags* ins) {
+ MDefinition* argsObj = ins->argsObject();
+ MOZ_ASSERT(argsObj->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LGuardArgumentsObjectFlags(useRegister(argsObj), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, argsObj);
+}
+
+void LIRGenerator::visitReturnFromCtor(MReturnFromCtor* ins) {
+ LReturnFromCtor* lir = new (alloc())
+ LReturnFromCtor(useBox(ins->value()), useRegister(ins->object()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitBoxNonStrictThis(MBoxNonStrictThis* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+ MOZ_ASSERT(ins->input()->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LBoxNonStrictThis(useBox(ins->input()));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitImplicitThis(MImplicitThis* ins) {
+ MDefinition* env = ins->envChain();
+ MOZ_ASSERT(env->type() == MIRType::Object);
+
+ LImplicitThis* lir = new (alloc()) LImplicitThis(useRegisterAtStart(env));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+bool LIRGenerator::lowerCallArguments(MCall* call) {
+ uint32_t argc = call->numStackArgs();
+
+ // Align the arguments of a call such that the callee would keep the same
+ // alignment as the caller.
+ uint32_t baseSlot = 0;
+ if (JitStackValueAlignment > 1) {
+ baseSlot = AlignBytes(argc, JitStackValueAlignment);
+ } else {
+ baseSlot = argc;
+ }
+
+ // Save the maximum number of argument, such that we can have one unique
+ // frame size.
+ if (baseSlot > maxargslots_) {
+ maxargslots_ = baseSlot;
+ }
+
+ for (size_t i = 0; i < argc; i++) {
+ MDefinition* arg = call->getArg(i);
+ uint32_t argslot = baseSlot - i;
+
+ // Values take a slow path.
+ if (arg->type() == MIRType::Value) {
+ LStackArgV* stack = new (alloc()) LStackArgV(useBox(arg), argslot);
+ add(stack);
+ } else {
+ // Known types can move constant types and/or payloads.
+ LStackArgT* stack = new (alloc())
+ LStackArgT(useRegisterOrConstant(arg), argslot, arg->type());
+ add(stack);
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void LIRGenerator::visitCall(MCall* call) {
+ MOZ_ASSERT(call->getCallee()->type() == MIRType::Object);
+
+ // In case of oom, skip the rest of the allocations.
+ if (!lowerCallArguments(call)) {
+ abort(AbortReason::Alloc, "OOM: LIRGenerator::visitCall");
+ return;
+ }
+
+ WrappedFunction* target = call->getSingleTarget();
+
+ LInstruction* lir;
+
+ if (call->isCallDOMNative()) {
+ // Call DOM functions.
+ MOZ_ASSERT(target && target->isNativeWithoutJitEntry());
+ Register cxReg, objReg, privReg, argsReg;
+ GetTempRegForIntArg(0, 0, &cxReg);
+ GetTempRegForIntArg(1, 0, &objReg);
+ GetTempRegForIntArg(2, 0, &privReg);
+ mozilla::DebugOnly<bool> ok = GetTempRegForIntArg(3, 0, &argsReg);
+ MOZ_ASSERT(ok, "How can we not have four temp registers?");
+ lir = new (alloc()) LCallDOMNative(tempFixed(cxReg), tempFixed(objReg),
+ tempFixed(privReg), tempFixed(argsReg));
+ } else if (target) {
+ // Call known functions.
+ if (target->isNativeWithoutJitEntry()) {
+ Register cxReg, numReg, vpReg, tmpReg;
+ GetTempRegForIntArg(0, 0, &cxReg);
+ GetTempRegForIntArg(1, 0, &numReg);
+ GetTempRegForIntArg(2, 0, &vpReg);
+
+ // Even though this is just a temp reg, use the same API to avoid
+ // register collisions.
+ mozilla::DebugOnly<bool> ok = GetTempRegForIntArg(3, 0, &tmpReg);
+ MOZ_ASSERT(ok, "How can we not have four temp registers?");
+
+ lir = new (alloc()) LCallNative(tempFixed(cxReg), tempFixed(numReg),
+ tempFixed(vpReg), tempFixed(tmpReg));
+ } else {
+ lir = new (alloc()) LCallKnown(useRegisterAtStart(call->getCallee()),
+ tempFixed(CallTempReg0));
+ }
+ } else {
+ // Call anything, using the most generic code.
+ lir = new (alloc())
+ LCallGeneric(useRegisterAtStart(call->getCallee()),
+ tempFixed(CallTempReg0), tempFixed(CallTempReg1));
+ }
+ defineReturn(lir, call);
+ assignSafepoint(lir, call);
+}
+
+void LIRGenerator::visitApplyArgs(MApplyArgs* apply) {
+ MOZ_ASSERT(apply->getFunction()->type() == MIRType::Object);
+
+ // Assert if the return value is already erased.
+ static_assert(CallTempReg2 != JSReturnReg_Type);
+ static_assert(CallTempReg2 != JSReturnReg_Data);
+
+ LApplyArgsGeneric* lir = new (alloc()) LApplyArgsGeneric(
+ useFixedAtStart(apply->getFunction(), CallTempReg3),
+ useFixedAtStart(apply->getArgc(), CallTempReg0),
+ useBoxFixedAtStart(apply->getThis(), CallTempReg4, CallTempReg5),
+ tempFixed(CallTempReg1), // object register
+ tempFixed(CallTempReg2)); // stack counter register
+
+ // Bailout is needed in the case of too many values in the arguments array.
+ assignSnapshot(lir, apply->bailoutKind());
+
+ defineReturn(lir, apply);
+ assignSafepoint(lir, apply);
+}
+
+void LIRGenerator::visitApplyArgsObj(MApplyArgsObj* apply) {
+ MOZ_ASSERT(apply->getFunction()->type() == MIRType::Object);
+
+ // Assert if the return value is already erased.
+ static_assert(CallTempReg2 != JSReturnReg_Type);
+ static_assert(CallTempReg2 != JSReturnReg_Data);
+
+ LApplyArgsObj* lir = new (alloc()) LApplyArgsObj(
+ useFixedAtStart(apply->getFunction(), CallTempReg3),
+ useFixedAtStart(apply->getArgsObj(), CallTempReg0),
+ useBoxFixedAtStart(apply->getThis(), CallTempReg4, CallTempReg5),
+ tempFixed(CallTempReg1), // object register
+ tempFixed(CallTempReg2)); // stack counter register
+
+ // Bailout is needed in the case of too many values in the arguments array.
+ assignSnapshot(lir, apply->bailoutKind());
+
+ defineReturn(lir, apply);
+ assignSafepoint(lir, apply);
+}
+
+void LIRGenerator::visitApplyArray(MApplyArray* apply) {
+ MOZ_ASSERT(apply->getFunction()->type() == MIRType::Object);
+
+ // Assert if the return value is already erased.
+ static_assert(CallTempReg2 != JSReturnReg_Type);
+ static_assert(CallTempReg2 != JSReturnReg_Data);
+
+ LApplyArrayGeneric* lir = new (alloc()) LApplyArrayGeneric(
+ useFixedAtStart(apply->getFunction(), CallTempReg3),
+ useFixedAtStart(apply->getElements(), CallTempReg0),
+ useBoxFixedAtStart(apply->getThis(), CallTempReg4, CallTempReg5),
+ tempFixed(CallTempReg1), // object register
+ tempFixed(CallTempReg2)); // stack counter register
+
+ // Bailout is needed in the case of too many values in the array, or empty
+ // space at the end of the array.
+ assignSnapshot(lir, apply->bailoutKind());
+
+ defineReturn(lir, apply);
+ assignSafepoint(lir, apply);
+}
+
+void LIRGenerator::visitConstructArgs(MConstructArgs* mir) {
+ MOZ_ASSERT(mir->getFunction()->type() == MIRType::Object);
+ MOZ_ASSERT(mir->getArgc()->type() == MIRType::Int32);
+ MOZ_ASSERT(mir->getNewTarget()->type() == MIRType::Object);
+ MOZ_ASSERT(mir->getThis()->type() == MIRType::Value);
+
+ // Assert if the return value is already erased.
+ static_assert(CallTempReg2 != JSReturnReg_Type);
+ static_assert(CallTempReg2 != JSReturnReg_Data);
+
+ auto* lir = new (alloc()) LConstructArgsGeneric(
+ useFixedAtStart(mir->getFunction(), CallTempReg3),
+ useFixedAtStart(mir->getArgc(), CallTempReg0),
+ useFixedAtStart(mir->getNewTarget(), CallTempReg1),
+ useBoxFixedAtStart(mir->getThis(), CallTempReg4, CallTempReg5),
+ tempFixed(CallTempReg2));
+
+ // Bailout is needed in the case of too many values in the arguments array.
+ assignSnapshot(lir, mir->bailoutKind());
+
+ defineReturn(lir, mir);
+ assignSafepoint(lir, mir);
+}
+
+void LIRGenerator::visitConstructArray(MConstructArray* mir) {
+ MOZ_ASSERT(mir->getFunction()->type() == MIRType::Object);
+ MOZ_ASSERT(mir->getElements()->type() == MIRType::Elements);
+ MOZ_ASSERT(mir->getNewTarget()->type() == MIRType::Object);
+ MOZ_ASSERT(mir->getThis()->type() == MIRType::Value);
+
+ // Assert if the return value is already erased.
+ static_assert(CallTempReg2 != JSReturnReg_Type);
+ static_assert(CallTempReg2 != JSReturnReg_Data);
+
+ auto* lir = new (alloc()) LConstructArrayGeneric(
+ useFixedAtStart(mir->getFunction(), CallTempReg3),
+ useFixedAtStart(mir->getElements(), CallTempReg0),
+ useFixedAtStart(mir->getNewTarget(), CallTempReg1),
+ useBoxFixedAtStart(mir->getThis(), CallTempReg4, CallTempReg5),
+ tempFixed(CallTempReg2));
+
+ // Bailout is needed in the case of too many values in the array, or empty
+ // space at the end of the array.
+ assignSnapshot(lir, mir->bailoutKind());
+
+ defineReturn(lir, mir);
+ assignSafepoint(lir, mir);
+}
+
+void LIRGenerator::visitBail(MBail* bail) {
+ LBail* lir = new (alloc()) LBail();
+ assignSnapshot(lir, bail->bailoutKind());
+ add(lir, bail);
+}
+
+void LIRGenerator::visitUnreachable(MUnreachable* unreachable) {
+ LUnreachable* lir = new (alloc()) LUnreachable();
+ add(lir, unreachable);
+}
+
+void LIRGenerator::visitEncodeSnapshot(MEncodeSnapshot* mir) {
+ LEncodeSnapshot* lir = new (alloc()) LEncodeSnapshot();
+ assignSnapshot(lir, mir->bailoutKind());
+ add(lir, mir);
+}
+
+void LIRGenerator::visitUnreachableResult(MUnreachableResult* mir) {
+ if (mir->type() == MIRType::Value) {
+ auto* lir = new (alloc()) LUnreachableResultV();
+ defineBox(lir, mir);
+ } else {
+ auto* lir = new (alloc()) LUnreachableResultT();
+ define(lir, mir);
+ }
+}
+
+void LIRGenerator::visitAssertFloat32(MAssertFloat32* assertion) {
+ MIRType type = assertion->input()->type();
+ DebugOnly<bool> checkIsFloat32 = assertion->mustBeFloat32();
+
+ if (type != MIRType::Value && !JitOptions.eagerIonCompilation()) {
+ MOZ_ASSERT_IF(checkIsFloat32, type == MIRType::Float32);
+ MOZ_ASSERT_IF(!checkIsFloat32, type != MIRType::Float32);
+ }
+}
+
+void LIRGenerator::visitAssertRecoveredOnBailout(
+ MAssertRecoveredOnBailout* assertion) {
+ MOZ_CRASH("AssertRecoveredOnBailout nodes are always recovered on bailouts.");
+}
+
+[[nodiscard]] static JSOp ReorderComparison(JSOp op, MDefinition** lhsp,
+ MDefinition** rhsp) {
+ MDefinition* lhs = *lhsp;
+ MDefinition* rhs = *rhsp;
+
+ if (lhs->maybeConstantValue()) {
+ *rhsp = lhs;
+ *lhsp = rhs;
+ return ReverseCompareOp(op);
+ }
+ return op;
+}
+
+void LIRGenerator::visitTest(MTest* test) {
+ MDefinition* opd = test->getOperand(0);
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ // String is converted to length of string in the type analysis phase (see
+ // TestPolicy).
+ MOZ_ASSERT(opd->type() != MIRType::String);
+
+ // Testing a constant.
+ if (MConstant* constant = opd->maybeConstantValue()) {
+ bool b;
+ if (constant->valueToBoolean(&b)) {
+ add(new (alloc()) LGoto(b ? ifTrue : ifFalse));
+ return;
+ }
+ }
+
+ if (opd->type() == MIRType::Value) {
+ auto* lir = new (alloc()) LTestVAndBranch(
+ ifTrue, ifFalse, useBox(opd), tempDouble(), tempToUnbox(), temp());
+ add(lir, test);
+ return;
+ }
+
+ // Objects are truthy, except if it might emulate undefined.
+ if (opd->type() == MIRType::Object) {
+ add(new (alloc())
+ LTestOAndBranch(useRegister(opd), ifTrue, ifFalse, temp()),
+ test);
+ return;
+ }
+
+ // These must be explicitly sniffed out since they are constants and have
+ // no payload.
+ if (opd->type() == MIRType::Undefined || opd->type() == MIRType::Null) {
+ add(new (alloc()) LGoto(ifFalse));
+ return;
+ }
+
+ // All symbols are truthy.
+ if (opd->type() == MIRType::Symbol) {
+ add(new (alloc()) LGoto(ifTrue));
+ return;
+ }
+
+ // Try to match the pattern
+ // test=MTest(
+ // comp=MCompare(
+ // {EQ,NE} for {Int,UInt}{32,64},
+ // bitAnd={MBitAnd,MWasmBinaryBitwise(And{32,64})}(x, y),
+ // MConstant(0)
+ // )
+ // )
+ // and produce a single LBitAndAndBranch node. This requires both `comp`
+ // and `bitAnd` to be marked emit-at-uses. Since we can't use
+ // LBitAndAndBranch to represent a 64-bit AND on a 32-bit target, the 64-bit
+ // case is restricted to 64-bit targets.
+ if (opd->isCompare() && opd->isEmittedAtUses()) {
+#ifdef JS_64BIT
+ constexpr bool targetIs64 = true;
+#else
+ constexpr bool targetIs64 = false;
+#endif
+ MCompare* comp = opd->toCompare();
+ Assembler::Condition compCond =
+ JSOpToCondition(comp->compareType(), comp->jsop());
+ MDefinition* compL = comp->getOperand(0);
+ MDefinition* compR = comp->getOperand(1);
+ if ((comp->compareType() == MCompare::Compare_Int32 ||
+ comp->compareType() == MCompare::Compare_UInt32 ||
+ (targetIs64 && comp->compareType() == MCompare::Compare_Int64) ||
+ (targetIs64 && comp->compareType() == MCompare::Compare_UInt64)) &&
+ (compCond == Assembler::Equal || compCond == Assembler::NotEqual) &&
+ compR->isConstant() &&
+ (compR->toConstant()->isInt32(0) ||
+ (targetIs64 && compR->toConstant()->isInt64(0))) &&
+ (compL->isBitAnd() || (compL->isWasmBinaryBitwise() &&
+ compL->toWasmBinaryBitwise()->subOpcode() ==
+ MWasmBinaryBitwise::SubOpcode::And))) {
+ // The MCompare is OK; now check its first operand (the and-ish node).
+ MDefinition* bitAnd = compL;
+ MDefinition* bitAndL = bitAnd->getOperand(0);
+ MDefinition* bitAndR = bitAnd->getOperand(1);
+ MIRType bitAndLTy = bitAndL->type();
+ MIRType bitAndRTy = bitAndR->type();
+ if (bitAnd->isEmittedAtUses() && bitAndLTy == bitAndRTy &&
+ (bitAndLTy == MIRType::Int32 ||
+ (targetIs64 && bitAndLTy == MIRType::Int64))) {
+ // Pattern match succeeded.
+ ReorderCommutative(&bitAndL, &bitAndR, test);
+ if (compCond == Assembler::Equal) {
+ compCond = Assembler::Zero;
+ } else if (compCond == Assembler::NotEqual) {
+ compCond = Assembler::NonZero;
+ } else {
+ MOZ_ASSERT_UNREACHABLE("inequality operators cannot be folded");
+ }
+ MOZ_ASSERT_IF(!targetIs64, bitAndLTy == MIRType::Int32);
+ lowerForBitAndAndBranch(
+ new (alloc()) LBitAndAndBranch(
+ ifTrue, ifFalse, bitAndLTy == MIRType::Int64, compCond),
+ test, bitAndL, bitAndR);
+ return;
+ }
+ }
+ }
+
+ // Check if the operand for this test is a compare operation. If it is, we
+ // want to emit an LCompare*AndBranch rather than an LTest*AndBranch, to fuse
+ // the compare and jump instructions.
+ if (opd->isCompare() && opd->isEmittedAtUses()) {
+ MCompare* comp = opd->toCompare();
+ MDefinition* left = comp->lhs();
+ MDefinition* right = comp->rhs();
+
+ // Try to fold the comparison so that we don't have to handle all cases.
+ bool result;
+ if (comp->tryFold(&result)) {
+ add(new (alloc()) LGoto(result ? ifTrue : ifFalse));
+ return;
+ }
+
+ // Emit LCompare*AndBranch.
+
+ // Compare and branch null/undefined.
+ // The second operand has known null/undefined type,
+ // so just test the first operand.
+ if (comp->compareType() == MCompare::Compare_Null ||
+ comp->compareType() == MCompare::Compare_Undefined) {
+ if (left->type() == MIRType::Object) {
+ auto* lir = new (alloc()) LIsNullOrLikeUndefinedAndBranchT(
+ comp, useRegister(left), ifTrue, ifFalse, temp());
+ add(lir, test);
+ return;
+ }
+
+ if (IsLooseEqualityOp(comp->jsop())) {
+ auto* lir = new (alloc()) LIsNullOrLikeUndefinedAndBranchV(
+ comp, ifTrue, ifFalse, useBox(left), temp(), tempToUnbox());
+ add(lir, test);
+ return;
+ }
+
+ if (comp->compareType() == MCompare::Compare_Null) {
+ auto* lir =
+ new (alloc()) LIsNullAndBranch(comp, ifTrue, ifFalse, useBox(left));
+ add(lir, test);
+ return;
+ }
+
+ auto* lir = new (alloc())
+ LIsUndefinedAndBranch(comp, ifTrue, ifFalse, useBox(left));
+ add(lir, test);
+ return;
+ }
+
+ // Compare and branch Int32, Symbol, Object, or RefOrNull pointers.
+ if (comp->isInt32Comparison() ||
+ comp->compareType() == MCompare::Compare_UInt32 ||
+ comp->compareType() == MCompare::Compare_UIntPtr ||
+ comp->compareType() == MCompare::Compare_Object ||
+ comp->compareType() == MCompare::Compare_Symbol ||
+ comp->compareType() == MCompare::Compare_RefOrNull) {
+ JSOp op = ReorderComparison(comp->jsop(), &left, &right);
+ LAllocation lhs = useRegister(left);
+ LAllocation rhs;
+ if (comp->isInt32Comparison() ||
+ comp->compareType() == MCompare::Compare_UInt32 ||
+ comp->compareType() == MCompare::Compare_UIntPtr) {
+ rhs = useAnyOrInt32Constant(right);
+ } else {
+ rhs = useAny(right);
+ }
+ LCompareAndBranch* lir =
+ new (alloc()) LCompareAndBranch(comp, op, lhs, rhs, ifTrue, ifFalse);
+ add(lir, test);
+ return;
+ }
+
+ // Compare and branch Int64.
+ if (comp->compareType() == MCompare::Compare_Int64 ||
+ comp->compareType() == MCompare::Compare_UInt64) {
+ JSOp op = ReorderComparison(comp->jsop(), &left, &right);
+ lowerForCompareI64AndBranch(test, comp, op, left, right, ifTrue, ifFalse);
+ return;
+ }
+
+ // Compare and branch doubles.
+ if (comp->isDoubleComparison()) {
+ LAllocation lhs = useRegister(left);
+ LAllocation rhs = useRegister(right);
+ LCompareDAndBranch* lir =
+ new (alloc()) LCompareDAndBranch(comp, lhs, rhs, ifTrue, ifFalse);
+ add(lir, test);
+ return;
+ }
+
+ // Compare and branch floats.
+ if (comp->isFloat32Comparison()) {
+ LAllocation lhs = useRegister(left);
+ LAllocation rhs = useRegister(right);
+ LCompareFAndBranch* lir =
+ new (alloc()) LCompareFAndBranch(comp, lhs, rhs, ifTrue, ifFalse);
+ add(lir, test);
+ return;
+ }
+ }
+
+ // Check if the operand for this test is a bitand operation. If it is, we want
+ // to emit an LBitAndAndBranch rather than an LTest*AndBranch.
+ if (opd->isBitAnd() && opd->isEmittedAtUses()) {
+ MDefinition* lhs = opd->getOperand(0);
+ MDefinition* rhs = opd->getOperand(1);
+ if (lhs->type() == MIRType::Int32 && rhs->type() == MIRType::Int32) {
+ ReorderCommutative(&lhs, &rhs, test);
+ lowerForBitAndAndBranch(new (alloc()) LBitAndAndBranch(ifTrue, ifFalse,
+ /*is64=*/false),
+ test, lhs, rhs);
+ return;
+ }
+ }
+
+#if defined(ENABLE_WASM_SIMD) && \
+ (defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64) || \
+ defined(JS_CODEGEN_ARM64))
+ // Check if the operand for this test is an any_true/all_true SIMD operation.
+ // If it is, we want to emit an LWasmReduceAndBranchSimd128 node to avoid
+ // generating an intermediate boolean result.
+ if (opd->isWasmReduceSimd128() && opd->isEmittedAtUses()) {
+ MWasmReduceSimd128* node = opd->toWasmReduceSimd128();
+ if (canFoldReduceSimd128AndBranch(node->simdOp())) {
+# ifdef DEBUG
+ js::wasm::ReportSimdAnalysis("simd128-to-scalar-and-branch -> folded");
+# endif
+ auto* lir = new (alloc()) LWasmReduceAndBranchSimd128(
+ useRegister(node->input()), node->simdOp(), ifTrue, ifFalse);
+ add(lir, test);
+ return;
+ }
+ }
+#endif
+
+ if (opd->isIsObject() && opd->isEmittedAtUses()) {
+ MDefinition* input = opd->toIsObject()->input();
+ MOZ_ASSERT(input->type() == MIRType::Value);
+
+ LIsObjectAndBranch* lir =
+ new (alloc()) LIsObjectAndBranch(ifTrue, ifFalse, useBoxAtStart(input));
+ add(lir, test);
+ return;
+ }
+
+ if (opd->isWasmGcObjectIsSubtypeOf() && opd->isEmittedAtUses()) {
+ MWasmGcObjectIsSubtypeOf* isSubTypeOf = opd->toWasmGcObjectIsSubtypeOf();
+ LAllocation object = useRegister(isSubTypeOf->object());
+ LAllocation superTypeDef = useRegister(isSubTypeOf->superTypeDef());
+ uint32_t subTypingDepth = isSubTypeOf->subTypingDepth();
+ LDefinition subTypeDepth = temp();
+ LDefinition scratch = subTypingDepth >= wasm::MinSuperTypeVectorLength
+ ? temp()
+ : LDefinition();
+ add(new (alloc()) LWasmGcObjectIsSubtypeOfAndBranch(
+ ifTrue, ifFalse, object, superTypeDef, subTypingDepth, subTypeDepth,
+ scratch),
+ test);
+ return;
+ }
+
+ if (opd->isIsNullOrUndefined() && opd->isEmittedAtUses()) {
+ MIsNullOrUndefined* isNullOrUndefined = opd->toIsNullOrUndefined();
+ MDefinition* input = isNullOrUndefined->value();
+ MOZ_ASSERT(input->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LIsNullOrUndefinedAndBranch(
+ isNullOrUndefined, ifTrue, ifFalse, useBoxAtStart(input));
+ add(lir, test);
+ return;
+ }
+
+ if (opd->isIsNoIter()) {
+ MOZ_ASSERT(opd->isEmittedAtUses());
+
+ MDefinition* input = opd->toIsNoIter()->input();
+ MOZ_ASSERT(input->type() == MIRType::Value);
+
+ LIsNoIterAndBranch* lir =
+ new (alloc()) LIsNoIterAndBranch(ifTrue, ifFalse, useBox(input));
+ add(lir, test);
+ return;
+ }
+
+ switch (opd->type()) {
+ case MIRType::Double:
+ add(new (alloc()) LTestDAndBranch(useRegister(opd), ifTrue, ifFalse));
+ break;
+ case MIRType::Float32:
+ add(new (alloc()) LTestFAndBranch(useRegister(opd), ifTrue, ifFalse));
+ break;
+ case MIRType::Int32:
+ case MIRType::Boolean:
+ add(new (alloc()) LTestIAndBranch(useRegister(opd), ifTrue, ifFalse));
+ break;
+ case MIRType::Int64:
+ add(new (alloc())
+ LTestI64AndBranch(useInt64Register(opd), ifTrue, ifFalse));
+ break;
+ case MIRType::BigInt:
+ add(new (alloc()) LTestBIAndBranch(useRegister(opd), ifTrue, ifFalse));
+ break;
+ default:
+ MOZ_CRASH("Bad type");
+ }
+}
+
+static inline bool CanEmitCompareAtUses(MInstruction* ins) {
+ if (!ins->canEmitAtUses()) {
+ return false;
+ }
+
+ // If the result is never used, we can usefully defer emission to the use
+ // point, since that will never happen.
+ MUseIterator iter(ins->usesBegin());
+ if (iter == ins->usesEnd()) {
+ return true;
+ }
+
+ // If the first use isn't of the expected form, the answer is No.
+ MNode* node = iter->consumer();
+ if (!node->isDefinition()) {
+ return false;
+ }
+
+ MDefinition* use = node->toDefinition();
+ if (!use->isTest() && !use->isWasmSelect()) {
+ return false;
+ }
+
+ // Emission can be deferred to the first use point, but only if there are no
+ // other use points.
+ iter++;
+ return iter == ins->usesEnd();
+}
+
+static bool CanCompareCharactersInline(const JSLinearString* linear) {
+ size_t length = linear->length();
+
+ // Limit the number of inline instructions used for character comparisons. Use
+ // the same instruction limit for both encodings, i.e. two-byte uses half the
+ // limit of Latin-1 strings.
+ constexpr size_t Latin1StringCompareCutoff = 32;
+ constexpr size_t TwoByteStringCompareCutoff = 16;
+
+ return length > 0 &&
+ (linear->hasLatin1Chars() ? length <= Latin1StringCompareCutoff
+ : length <= TwoByteStringCompareCutoff);
+}
+
+void LIRGenerator::visitCompare(MCompare* comp) {
+ MDefinition* left = comp->lhs();
+ MDefinition* right = comp->rhs();
+
+ // Try to fold the comparison so that we don't have to handle all cases.
+ bool result;
+ if (comp->tryFold(&result)) {
+ define(new (alloc()) LInteger(result), comp);
+ return;
+ }
+
+ // Move below the emitAtUses call if we ever implement
+ // LCompareSAndBranch. Doing this now wouldn't be wrong, but doesn't
+ // make sense and avoids confusion.
+ if (comp->compareType() == MCompare::Compare_String) {
+ if (IsEqualityOp(comp->jsop())) {
+ MConstant* constant = nullptr;
+ if (left->isConstant()) {
+ constant = left->toConstant();
+ } else if (right->isConstant()) {
+ constant = right->toConstant();
+ }
+
+ if (constant) {
+ JSLinearString* linear = &constant->toString()->asLinear();
+
+ if (CanCompareCharactersInline(linear)) {
+ MDefinition* input = left->isConstant() ? right : left;
+
+ auto* lir = new (alloc()) LCompareSInline(useRegister(input), linear);
+ define(lir, comp);
+ assignSafepoint(lir, comp);
+ return;
+ }
+ }
+ }
+
+ LCompareS* lir =
+ new (alloc()) LCompareS(useRegister(left), useRegister(right));
+ define(lir, comp);
+ assignSafepoint(lir, comp);
+ return;
+ }
+
+ // Compare two BigInts.
+ if (comp->compareType() == MCompare::Compare_BigInt) {
+ auto* lir = new (alloc()) LCompareBigInt(
+ useRegister(left), useRegister(right), temp(), temp(), temp());
+ define(lir, comp);
+ return;
+ }
+
+ // Compare BigInt with Int32.
+ if (comp->compareType() == MCompare::Compare_BigInt_Int32) {
+ auto* lir = new (alloc()) LCompareBigIntInt32(
+ useRegister(left), useRegister(right), temp(), temp());
+ define(lir, comp);
+ return;
+ }
+
+ // Compare BigInt with Double.
+ if (comp->compareType() == MCompare::Compare_BigInt_Double) {
+ auto* lir = new (alloc()) LCompareBigIntDouble(useRegisterAtStart(left),
+ useRegisterAtStart(right));
+ defineReturn(lir, comp);
+ return;
+ }
+
+ // Compare BigInt with String.
+ if (comp->compareType() == MCompare::Compare_BigInt_String) {
+ auto* lir = new (alloc()) LCompareBigIntString(useRegisterAtStart(left),
+ useRegisterAtStart(right));
+ defineReturn(lir, comp);
+ assignSafepoint(lir, comp);
+ return;
+ }
+
+ // Sniff out if the output of this compare is used only for a branching.
+ // If it is, then we will emit an LCompare*AndBranch instruction in place
+ // of this compare and any test that uses this compare. Thus, we can
+ // ignore this Compare.
+ if (CanEmitCompareAtUses(comp)) {
+ emitAtUses(comp);
+ return;
+ }
+
+ // Compare Null and Undefined.
+ if (comp->compareType() == MCompare::Compare_Null ||
+ comp->compareType() == MCompare::Compare_Undefined) {
+ if (left->type() == MIRType::Object) {
+ define(new (alloc()) LIsNullOrLikeUndefinedT(useRegister(left)), comp);
+ return;
+ }
+
+ if (IsLooseEqualityOp(comp->jsop())) {
+ auto* lir =
+ new (alloc()) LIsNullOrLikeUndefinedV(useBox(left), tempToUnbox());
+ define(lir, comp);
+ return;
+ }
+
+ if (comp->compareType() == MCompare::Compare_Null) {
+ auto* lir = new (alloc()) LIsNull(useBox(left));
+ define(lir, comp);
+ return;
+ }
+
+ auto* lir = new (alloc()) LIsUndefined(useBox(left));
+ define(lir, comp);
+ return;
+ }
+
+ // Compare Int32, Symbol, Object or Wasm pointers.
+ if (comp->isInt32Comparison() ||
+ comp->compareType() == MCompare::Compare_UInt32 ||
+ comp->compareType() == MCompare::Compare_UIntPtr ||
+ comp->compareType() == MCompare::Compare_Object ||
+ comp->compareType() == MCompare::Compare_Symbol ||
+ comp->compareType() == MCompare::Compare_RefOrNull) {
+ JSOp op = ReorderComparison(comp->jsop(), &left, &right);
+ LAllocation lhs = useRegister(left);
+ LAllocation rhs;
+ if (comp->isInt32Comparison() ||
+ comp->compareType() == MCompare::Compare_UInt32 ||
+ comp->compareType() == MCompare::Compare_UIntPtr) {
+ rhs = useAnyOrInt32Constant(right);
+ } else {
+ rhs = useAny(right);
+ }
+ define(new (alloc()) LCompare(op, lhs, rhs), comp);
+ return;
+ }
+
+ // Compare Int64.
+ if (comp->compareType() == MCompare::Compare_Int64 ||
+ comp->compareType() == MCompare::Compare_UInt64) {
+ JSOp op = ReorderComparison(comp->jsop(), &left, &right);
+ define(new (alloc()) LCompareI64(op, useInt64Register(left),
+ useInt64OrConstant(right)),
+ comp);
+ return;
+ }
+
+ // Compare doubles.
+ if (comp->isDoubleComparison()) {
+ define(new (alloc()) LCompareD(useRegister(left), useRegister(right)),
+ comp);
+ return;
+ }
+
+ // Compare float32.
+ if (comp->isFloat32Comparison()) {
+ define(new (alloc()) LCompareF(useRegister(left), useRegister(right)),
+ comp);
+ return;
+ }
+
+ MOZ_CRASH("Unrecognized compare type.");
+}
+
+void LIRGenerator::visitSameValueDouble(MSameValueDouble* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Double);
+ MOZ_ASSERT(rhs->type() == MIRType::Double);
+
+ auto* lir = new (alloc())
+ LSameValueDouble(useRegister(lhs), useRegister(rhs), tempDouble());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitSameValue(MSameValue* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Value);
+ MOZ_ASSERT(rhs->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LSameValue(useBox(lhs), useBox(rhs));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::lowerBitOp(JSOp op, MBinaryInstruction* ins) {
+ MDefinition* lhs = ins->getOperand(0);
+ MDefinition* rhs = ins->getOperand(1);
+ MOZ_ASSERT(IsIntType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+ ReorderCommutative(&lhs, &rhs, ins);
+ lowerForALU(new (alloc()) LBitOpI(op), ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int64);
+ MOZ_ASSERT(rhs->type() == MIRType::Int64);
+ ReorderCommutative(&lhs, &rhs, ins);
+ lowerForALUInt64(new (alloc()) LBitOpI64(op), ins, lhs, rhs);
+ return;
+ }
+
+ MOZ_CRASH("Unhandled integer specialization");
+}
+
+void LIRGenerator::visitTypeOf(MTypeOf* ins) {
+ MDefinition* opd = ins->input();
+
+ if (opd->type() == MIRType::Object) {
+ auto* lir = new (alloc()) LTypeOfO(useRegister(opd));
+ define(lir, ins);
+ return;
+ }
+
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LTypeOfV* lir = new (alloc()) LTypeOfV(useBox(opd), tempToUnbox());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitTypeOfName(MTypeOfName* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LTypeOfName(useRegister(input));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitTypeOfIs(MTypeOfIs* ins) {
+ MDefinition* input = ins->input();
+
+ MOZ_ASSERT(input->type() == MIRType::Object ||
+ input->type() == MIRType::Value);
+
+ switch (ins->jstype()) {
+ case JSTYPE_UNDEFINED:
+ case JSTYPE_OBJECT:
+ case JSTYPE_FUNCTION: {
+ if (input->type() == MIRType::Object) {
+ auto* lir = new (alloc()) LTypeOfIsNonPrimitiveO(useRegister(input));
+ define(lir, ins);
+ } else {
+ auto* lir =
+ new (alloc()) LTypeOfIsNonPrimitiveV(useBox(input), tempToUnbox());
+ define(lir, ins);
+ }
+ return;
+ }
+
+ case JSTYPE_STRING:
+ case JSTYPE_NUMBER:
+ case JSTYPE_BOOLEAN:
+ case JSTYPE_SYMBOL:
+ case JSTYPE_BIGINT: {
+ MOZ_ASSERT(input->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LTypeOfIsPrimitive(useBoxAtStart(input));
+ define(lir, ins);
+ return;
+ }
+
+#ifdef ENABLE_RECORD_TUPLE
+ case JSTYPE_RECORD:
+ case JSTYPE_TUPLE:
+#endif
+ case JSTYPE_LIMIT:
+ break;
+ }
+ MOZ_CRASH("Unhandled JSType");
+}
+
+void LIRGenerator::visitToAsyncIter(MToAsyncIter* ins) {
+ LToAsyncIter* lir = new (alloc()) LToAsyncIter(
+ useRegisterAtStart(ins->iterator()), useBoxAtStart(ins->nextMethod()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitToPropertyKeyCache(MToPropertyKeyCache* ins) {
+ MDefinition* input = ins->getOperand(0);
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LToPropertyKeyCache(useBox(input));
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBitNot(MBitNot* ins) {
+ MDefinition* input = ins->getOperand(0);
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(input->type() == MIRType::Int32);
+ lowerForALU(new (alloc()) LBitNotI(), ins, input);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(input->type() == MIRType::Int64);
+ lowerForALUInt64(new (alloc()) LBitNotI64(), ins, input);
+ return;
+ }
+
+ MOZ_CRASH("Unhandled integer specialization");
+}
+
+static bool CanEmitBitAndAtUses(MInstruction* ins) {
+ if (!ins->canEmitAtUses()) {
+ return false;
+ }
+
+ MIRType tyL = ins->getOperand(0)->type();
+ MIRType tyR = ins->getOperand(1)->type();
+ if (tyL != tyR || (tyL != MIRType::Int32 && tyL != MIRType::Int64)) {
+ return false;
+ }
+
+ MUseIterator iter(ins->usesBegin());
+ if (iter == ins->usesEnd()) {
+ return false;
+ }
+
+ MNode* node = iter->consumer();
+ if (!node->isDefinition() || !node->toDefinition()->isInstruction()) {
+ return false;
+ }
+
+ MInstruction* use = node->toDefinition()->toInstruction();
+ if (!use->isTest() && !(use->isCompare() && CanEmitCompareAtUses(use))) {
+ return false;
+ }
+
+ iter++;
+ return iter == ins->usesEnd();
+}
+
+void LIRGenerator::visitBitAnd(MBitAnd* ins) {
+ // Sniff out if the output of this bitand is used only for a branching.
+ // If it is, then we will emit an LBitAndAndBranch instruction in place
+ // of this bitand and any test that uses this bitand. Thus, we can
+ // ignore this BitAnd.
+ if (CanEmitBitAndAtUses(ins)) {
+ emitAtUses(ins);
+ } else {
+ lowerBitOp(JSOp::BitAnd, ins);
+ }
+}
+
+void LIRGenerator::visitBitOr(MBitOr* ins) { lowerBitOp(JSOp::BitOr, ins); }
+
+void LIRGenerator::visitBitXor(MBitXor* ins) { lowerBitOp(JSOp::BitXor, ins); }
+
+void LIRGenerator::visitWasmBinaryBitwise(MWasmBinaryBitwise* ins) {
+ switch (ins->subOpcode()) {
+ case MWasmBinaryBitwise::SubOpcode::And:
+ if (CanEmitBitAndAtUses(ins)) {
+ emitAtUses(ins);
+ } else {
+ lowerBitOp(JSOp::BitAnd, ins);
+ }
+ break;
+ case MWasmBinaryBitwise::SubOpcode::Or:
+ lowerBitOp(JSOp::BitOr, ins);
+ break;
+ case MWasmBinaryBitwise::SubOpcode::Xor:
+ lowerBitOp(JSOp::BitXor, ins);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+void LIRGenerator::lowerShiftOp(JSOp op, MShiftInstruction* ins) {
+ MDefinition* lhs = ins->getOperand(0);
+ MDefinition* rhs = ins->getOperand(1);
+
+ if (op == JSOp::Ursh && ins->type() == MIRType::Double) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+ lowerUrshD(ins->toUrsh());
+ return;
+ }
+
+ MOZ_ASSERT(IsIntType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+
+ LShiftI* lir = new (alloc()) LShiftI(op);
+ if (op == JSOp::Ursh) {
+ if (ins->toUrsh()->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ }
+ lowerForShift(lir, ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int64);
+ MOZ_ASSERT(rhs->type() == MIRType::Int64);
+ lowerForShiftInt64(new (alloc()) LShiftI64(op), ins, lhs, rhs);
+ return;
+ }
+
+ MOZ_CRASH("Unhandled integer specialization");
+}
+
+void LIRGenerator::visitLsh(MLsh* ins) { lowerShiftOp(JSOp::Lsh, ins); }
+
+void LIRGenerator::visitRsh(MRsh* ins) { lowerShiftOp(JSOp::Rsh, ins); }
+
+void LIRGenerator::visitUrsh(MUrsh* ins) { lowerShiftOp(JSOp::Ursh, ins); }
+
+void LIRGenerator::visitSignExtendInt32(MSignExtendInt32* ins) {
+ LInstructionHelper<1, 1, 0>* lir;
+
+ if (ins->mode() == MSignExtendInt32::Byte) {
+ lir = new (alloc())
+ LSignExtendInt32(useByteOpRegisterAtStart(ins->input()), ins->mode());
+ } else {
+ lir = new (alloc())
+ LSignExtendInt32(useRegisterAtStart(ins->input()), ins->mode());
+ }
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitRotate(MRotate* ins) {
+ MDefinition* input = ins->input();
+ MDefinition* count = ins->count();
+
+ if (ins->type() == MIRType::Int32) {
+ auto* lir = new (alloc()) LRotate();
+ lowerForShift(lir, ins, input, count);
+ } else if (ins->type() == MIRType::Int64) {
+ auto* lir = new (alloc()) LRotateI64();
+ lowerForShiftInt64(lir, ins, input, count);
+ } else {
+ MOZ_CRASH("unexpected type in visitRotate");
+ }
+}
+
+void LIRGenerator::visitFloor(MFloor* ins) {
+ MIRType type = ins->input()->type();
+ MOZ_ASSERT(IsFloatingPointType(type));
+
+ LInstructionHelper<1, 1, 0>* lir;
+ if (type == MIRType::Double) {
+ lir = new (alloc()) LFloor(useRegister(ins->input()));
+ } else {
+ lir = new (alloc()) LFloorF(useRegister(ins->input()));
+ }
+
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitCeil(MCeil* ins) {
+ MIRType type = ins->input()->type();
+ MOZ_ASSERT(IsFloatingPointType(type));
+
+ LInstructionHelper<1, 1, 0>* lir;
+ if (type == MIRType::Double) {
+ lir = new (alloc()) LCeil(useRegister(ins->input()));
+ } else {
+ lir = new (alloc()) LCeilF(useRegister(ins->input()));
+ }
+
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitRound(MRound* ins) {
+ MIRType type = ins->input()->type();
+ MOZ_ASSERT(IsFloatingPointType(type));
+
+ LInstructionHelper<1, 1, 1>* lir;
+ if (type == MIRType::Double) {
+ lir = new (alloc()) LRound(useRegister(ins->input()), tempDouble());
+ } else {
+ lir = new (alloc()) LRoundF(useRegister(ins->input()), tempFloat32());
+ }
+
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitTrunc(MTrunc* ins) {
+ MIRType type = ins->input()->type();
+ MOZ_ASSERT(IsFloatingPointType(type));
+
+ LInstructionHelper<1, 1, 0>* lir;
+ if (type == MIRType::Double) {
+ lir = new (alloc()) LTrunc(useRegister(ins->input()));
+ } else {
+ lir = new (alloc()) LTruncF(useRegister(ins->input()));
+ }
+
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitNearbyInt(MNearbyInt* ins) {
+ MIRType inputType = ins->input()->type();
+ MOZ_ASSERT(IsFloatingPointType(inputType));
+ MOZ_ASSERT(ins->type() == inputType);
+
+ LInstructionHelper<1, 1, 0>* lir;
+ if (inputType == MIRType::Double) {
+ lir = new (alloc()) LNearbyInt(useRegisterAtStart(ins->input()));
+ } else {
+ lir = new (alloc()) LNearbyIntF(useRegisterAtStart(ins->input()));
+ }
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitMinMax(MMinMax* ins) {
+ MDefinition* first = ins->getOperand(0);
+ MDefinition* second = ins->getOperand(1);
+
+ ReorderCommutative(&first, &second, ins);
+
+ LMinMaxBase* lir;
+ switch (ins->type()) {
+ case MIRType::Int32:
+ lir = new (alloc())
+ LMinMaxI(useRegisterAtStart(first), useRegisterOrConstant(second));
+ break;
+ case MIRType::Float32:
+ lir = new (alloc())
+ LMinMaxF(useRegisterAtStart(first), useRegister(second));
+ break;
+ case MIRType::Double:
+ lir = new (alloc())
+ LMinMaxD(useRegisterAtStart(first), useRegister(second));
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ // Input reuse is OK (for now) even on ARM64: floating min/max are fairly
+ // expensive due to SNaN -> QNaN conversion, and int min/max is for asm.js.
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitMinMaxArray(MMinMaxArray* ins) {
+ LInstructionHelper<1, 1, 3>* lir;
+ if (ins->type() == MIRType::Int32) {
+ lir = new (alloc())
+ LMinMaxArrayI(useRegisterAtStart(ins->array()), temp(), temp(), temp());
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ lir = new (alloc()) LMinMaxArrayD(useRegisterAtStart(ins->array()),
+ tempDouble(), temp(), temp());
+ }
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+LInstructionHelper<1, 1, 0>* LIRGenerator::allocateAbs(MAbs* ins,
+ LAllocation input) {
+ MDefinition* num = ins->input();
+ MOZ_ASSERT(IsNumberType(num->type()));
+
+ LInstructionHelper<1, 1, 0>* lir;
+ switch (num->type()) {
+ case MIRType::Int32:
+ lir = new (alloc()) LAbsI(input);
+ // needed to handle abs(INT32_MIN)
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ break;
+ case MIRType::Float32:
+ lir = new (alloc()) LAbsF(input);
+ break;
+ case MIRType::Double:
+ lir = new (alloc()) LAbsD(input);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ return lir;
+}
+
+void LIRGenerator::visitClz(MClz* ins) {
+ MDefinition* num = ins->num();
+
+ MOZ_ASSERT(IsIntType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ LClzI* lir = new (alloc()) LClzI(useRegisterAtStart(num));
+ define(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LClzI64(useInt64RegisterAtStart(num));
+ defineInt64(lir, ins);
+}
+
+void LIRGenerator::visitCtz(MCtz* ins) {
+ MDefinition* num = ins->num();
+
+ MOZ_ASSERT(IsIntType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ LCtzI* lir = new (alloc()) LCtzI(useRegisterAtStart(num));
+ define(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LCtzI64(useInt64RegisterAtStart(num));
+ defineInt64(lir, ins);
+}
+
+void LIRGenerator::visitPopcnt(MPopcnt* ins) {
+ MDefinition* num = ins->num();
+
+ MOZ_ASSERT(IsIntType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ LPopcntI* lir = new (alloc()) LPopcntI(useRegisterAtStart(num), temp());
+ define(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LPopcntI64(useInt64RegisterAtStart(num), temp());
+ defineInt64(lir, ins);
+}
+
+void LIRGenerator::visitSqrt(MSqrt* ins) {
+ MDefinition* num = ins->input();
+ MOZ_ASSERT(IsFloatingPointType(num->type()));
+
+ LInstructionHelper<1, 1, 0>* lir;
+ if (num->type() == MIRType::Double) {
+ lir = new (alloc()) LSqrtD(useRegisterAtStart(num));
+ } else {
+ lir = new (alloc()) LSqrtF(useRegisterAtStart(num));
+ }
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtan2(MAtan2* ins) {
+ MDefinition* y = ins->y();
+ MOZ_ASSERT(y->type() == MIRType::Double);
+
+ MDefinition* x = ins->x();
+ MOZ_ASSERT(x->type() == MIRType::Double);
+
+ LAtan2D* lir =
+ new (alloc()) LAtan2D(useRegisterAtStart(y), useRegisterAtStart(x));
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitHypot(MHypot* ins) {
+ LHypot* lir = nullptr;
+ uint32_t length = ins->numOperands();
+ for (uint32_t i = 0; i < length; ++i) {
+ MOZ_ASSERT(ins->getOperand(i)->type() == MIRType::Double);
+ }
+
+ switch (length) {
+ case 2:
+ lir = new (alloc()) LHypot(useRegisterAtStart(ins->getOperand(0)),
+ useRegisterAtStart(ins->getOperand(1)));
+ break;
+ case 3:
+ lir = new (alloc()) LHypot(useRegisterAtStart(ins->getOperand(0)),
+ useRegisterAtStart(ins->getOperand(1)),
+ useRegisterAtStart(ins->getOperand(2)));
+ break;
+ case 4:
+ lir = new (alloc()) LHypot(useRegisterAtStart(ins->getOperand(0)),
+ useRegisterAtStart(ins->getOperand(1)),
+ useRegisterAtStart(ins->getOperand(2)),
+ useRegisterAtStart(ins->getOperand(3)));
+ break;
+ default:
+ MOZ_CRASH("Unexpected number of arguments to LHypot.");
+ }
+
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitPow(MPow* ins) {
+ MDefinition* input = ins->input();
+ MDefinition* power = ins->power();
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(input->type() == MIRType::Int32);
+ MOZ_ASSERT(power->type() == MIRType::Int32);
+
+ if (input->isConstant()) {
+ // Restrict this optimization to |base <= 256| to avoid generating too
+ // many consecutive shift instructions.
+ int32_t base = input->toConstant()->toInt32();
+ if (2 <= base && base <= 256 && mozilla::IsPowerOfTwo(uint32_t(base))) {
+ lowerPowOfTwoI(ins);
+ return;
+ }
+ }
+
+ auto* lir = new (alloc())
+ LPowII(useRegister(input), useRegister(power), temp(), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+ return;
+ }
+
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ MOZ_ASSERT(input->type() == MIRType::Double);
+ MOZ_ASSERT(power->type() == MIRType::Int32 ||
+ power->type() == MIRType::Double);
+
+ LInstruction* lir;
+ if (power->type() == MIRType::Int32) {
+ lir = new (alloc())
+ LPowI(useRegisterAtStart(input), useRegisterAtStart(power));
+ } else {
+ lir = new (alloc())
+ LPowD(useRegisterAtStart(input), useRegisterAtStart(power));
+ }
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitSign(MSign* ins) {
+ if (ins->type() == ins->input()->type()) {
+ LInstructionHelper<1, 1, 0>* lir;
+ if (ins->type() == MIRType::Int32) {
+ lir = new (alloc()) LSignI(useRegister(ins->input()));
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ lir = new (alloc()) LSignD(useRegister(ins->input()));
+ }
+ define(lir, ins);
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->input()->type() == MIRType::Double);
+
+ auto* lir = new (alloc()) LSignDI(useRegister(ins->input()), tempDouble());
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitMathFunction(MMathFunction* ins) {
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+ MOZ_ASSERT(ins->type() == ins->input()->type());
+
+ LInstruction* lir;
+ if (ins->type() == MIRType::Double) {
+ lir = new (alloc()) LMathFunctionD(useRegisterAtStart(ins->input()));
+ } else {
+ lir = new (alloc()) LMathFunctionF(useRegisterAtStart(ins->input()));
+ }
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitRandom(MRandom* ins) {
+ auto* lir = new (alloc()) LRandom(temp(), tempInt64(), tempInt64());
+ define(lir, ins);
+}
+
+// Try to mark an add or sub instruction as able to recover its input when
+// bailing out.
+template <typename S, typename T>
+static void MaybeSetRecoversInput(S* mir, T* lir) {
+ MOZ_ASSERT(lir->mirRaw() == mir);
+ if (!mir->fallible() || !lir->snapshot()) {
+ return;
+ }
+
+ if (lir->output()->policy() != LDefinition::MUST_REUSE_INPUT) {
+ return;
+ }
+
+ // The original operands to an add or sub can't be recovered if they both
+ // use the same register.
+ if (lir->lhs()->isUse() && lir->rhs()->isUse() &&
+ lir->lhs()->toUse()->virtualRegister() ==
+ lir->rhs()->toUse()->virtualRegister()) {
+ return;
+ }
+
+ // Add instructions that are on two different values can recover
+ // the input they clobbered via MUST_REUSE_INPUT. Thus, a copy
+ // of that input does not need to be kept alive in the snapshot
+ // for the instruction.
+
+ lir->setRecoversInput();
+
+ const LUse* input = lir->getOperand(lir->output()->getReusedInput())->toUse();
+ lir->snapshot()->rewriteRecoveredInput(*input);
+}
+
+void LIRGenerator::visitAdd(MAdd* ins) {
+ MDefinition* lhs = ins->getOperand(0);
+ MDefinition* rhs = ins->getOperand(1);
+
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(IsNumberType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ ReorderCommutative(&lhs, &rhs, ins);
+ LAddI* lir = new (alloc()) LAddI;
+
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+
+ lowerForALU(lir, ins, lhs, rhs);
+ MaybeSetRecoversInput(ins, lir);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int64);
+ ReorderCommutative(&lhs, &rhs, ins);
+ LAddI64* lir = new (alloc()) LAddI64;
+ lowerForALUInt64(lir, ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Double) {
+ MOZ_ASSERT(lhs->type() == MIRType::Double);
+ ReorderCommutative(&lhs, &rhs, ins);
+ lowerForFPU(new (alloc()) LMathD(JSOp::Add), ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Float32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Float32);
+ ReorderCommutative(&lhs, &rhs, ins);
+ lowerForFPU(new (alloc()) LMathF(JSOp::Add), ins, lhs, rhs);
+ return;
+ }
+
+ MOZ_CRASH("Unhandled number specialization");
+}
+
+void LIRGenerator::visitSub(MSub* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(IsNumberType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+
+ LSubI* lir = new (alloc()) LSubI;
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+
+ // If our LHS is a constant 0 and we don't have to worry about results that
+ // can't be represented as an int32, we can optimize to an LNegI.
+ if (!ins->fallible() && lhs->isConstant() &&
+ lhs->toConstant()->toInt32() == 0) {
+ lowerNegI(ins, rhs);
+ return;
+ }
+
+ lowerForALU(lir, ins, lhs, rhs);
+ MaybeSetRecoversInput(ins, lir);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int64);
+
+ // If our LHS is a constant 0, we can optimize to an LNegI64.
+ if (lhs->isConstant() && lhs->toConstant()->toInt64() == 0) {
+ lowerNegI64(ins, rhs);
+ return;
+ }
+
+ LSubI64* lir = new (alloc()) LSubI64;
+ lowerForALUInt64(lir, ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Double) {
+ MOZ_ASSERT(lhs->type() == MIRType::Double);
+ lowerForFPU(new (alloc()) LMathD(JSOp::Sub), ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Float32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Float32);
+ lowerForFPU(new (alloc()) LMathF(JSOp::Sub), ins, lhs, rhs);
+ return;
+ }
+
+ MOZ_CRASH("Unhandled number specialization");
+}
+
+void LIRGenerator::visitMul(MMul* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(IsNumberType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ ReorderCommutative(&lhs, &rhs, ins);
+
+ // If our RHS is a constant -1 and we don't have to worry about results that
+ // can't be represented as an int32, we can optimize to an LNegI.
+ if (!ins->fallible() && rhs->isConstant() &&
+ rhs->toConstant()->toInt32() == -1) {
+ lowerNegI(ins, lhs);
+ return;
+ }
+
+ lowerMulI(ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int64);
+ ReorderCommutative(&lhs, &rhs, ins);
+
+ // If our RHS is a constant -1, we can optimize to an LNegI64.
+ if (rhs->isConstant() && rhs->toConstant()->toInt64() == -1) {
+ lowerNegI64(ins, lhs);
+ return;
+ }
+
+ LMulI64* lir = new (alloc()) LMulI64;
+ lowerForMulInt64(lir, ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Double) {
+ MOZ_ASSERT(lhs->type() == MIRType::Double);
+ ReorderCommutative(&lhs, &rhs, ins);
+
+ // If our RHS is a constant -1.0, we can optimize to an LNegD.
+ if (!ins->mustPreserveNaN() && rhs->isConstant() &&
+ rhs->toConstant()->toDouble() == -1.0) {
+ defineReuseInput(new (alloc()) LNegD(useRegisterAtStart(lhs)), ins, 0);
+ return;
+ }
+
+ lowerForFPU(new (alloc()) LMathD(JSOp::Mul), ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Float32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Float32);
+ ReorderCommutative(&lhs, &rhs, ins);
+
+ // We apply the same optimizations as for doubles
+ if (!ins->mustPreserveNaN() && rhs->isConstant() &&
+ rhs->toConstant()->toFloat32() == -1.0f) {
+ defineReuseInput(new (alloc()) LNegF(useRegisterAtStart(lhs)), ins, 0);
+ return;
+ }
+
+ lowerForFPU(new (alloc()) LMathF(JSOp::Mul), ins, lhs, rhs);
+ return;
+ }
+
+ MOZ_CRASH("Unhandled number specialization");
+}
+
+void LIRGenerator::visitDiv(MDiv* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(IsNumberType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ lowerDivI(ins);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(lhs->type() == MIRType::Int64);
+ lowerDivI64(ins);
+ return;
+ }
+
+ if (ins->type() == MIRType::Double) {
+ MOZ_ASSERT(lhs->type() == MIRType::Double);
+ lowerForFPU(new (alloc()) LMathD(JSOp::Div), ins, lhs, rhs);
+ return;
+ }
+
+ if (ins->type() == MIRType::Float32) {
+ MOZ_ASSERT(lhs->type() == MIRType::Float32);
+ lowerForFPU(new (alloc()) LMathF(JSOp::Div), ins, lhs, rhs);
+ return;
+ }
+
+ MOZ_CRASH("Unhandled number specialization");
+}
+
+void LIRGenerator::visitWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ lowerWasmBuiltinDivI64(div);
+}
+
+void LIRGenerator::visitWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ lowerWasmBuiltinModI64(mod);
+}
+
+void LIRGenerator::visitBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ lowerBuiltinInt64ToFloatingPoint(ins);
+}
+
+void LIRGenerator::visitWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ lowerWasmBuiltinTruncateToInt64(ins);
+}
+
+void LIRGenerator::visitWasmBuiltinModD(MWasmBuiltinModD* ins) {
+ MOZ_ASSERT(gen->compilingWasm());
+ LWasmBuiltinModD* lir = new (alloc()) LWasmBuiltinModD(
+ useRegisterAtStart(ins->lhs()), useRegisterAtStart(ins->rhs()),
+ useFixedAtStart(ins->instance(), InstanceReg));
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitMod(MMod* ins) {
+ MOZ_ASSERT(ins->lhs()->type() == ins->rhs()->type());
+ MOZ_ASSERT(IsNumberType(ins->type()));
+
+ if (ins->type() == MIRType::Int32) {
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->lhs()->type() == MIRType::Int32);
+ lowerModI(ins);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(ins->type() == MIRType::Int64);
+ MOZ_ASSERT(ins->lhs()->type() == MIRType::Int64);
+ lowerModI64(ins);
+ return;
+ }
+
+ if (ins->type() == MIRType::Double) {
+ MOZ_ASSERT(ins->lhs()->type() == MIRType::Double);
+ MOZ_ASSERT(ins->rhs()->type() == MIRType::Double);
+
+ MOZ_ASSERT(!gen->compilingWasm());
+
+ if (Assembler::HasRoundInstruction(RoundingMode::TowardsZero)) {
+ if (ins->rhs()->isConstant()) {
+ double d = ins->rhs()->toConstant()->toDouble();
+ int32_t div;
+ if (mozilla::NumberIsInt32(d, &div) && div > 0 &&
+ mozilla::IsPowerOfTwo(uint32_t(div))) {
+ auto* lir = new (alloc()) LModPowTwoD(useRegister(ins->lhs()), div);
+ define(lir, ins);
+ return;
+ }
+ }
+ }
+
+ LModD* lir = new (alloc())
+ LModD(useRegisterAtStart(ins->lhs()), useRegisterAtStart(ins->rhs()));
+ defineReturn(lir, ins);
+ return;
+ }
+
+ MOZ_CRASH("Unhandled number specialization");
+}
+
+void LIRGenerator::visitBigIntAdd(MBigIntAdd* ins) {
+ auto* lir = new (alloc()) LBigIntAdd(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntSub(MBigIntSub* ins) {
+ auto* lir = new (alloc()) LBigIntSub(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntMul(MBigIntMul* ins) {
+ auto* lir = new (alloc()) LBigIntMul(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntDiv(MBigIntDiv* ins) { lowerBigIntDiv(ins); }
+
+void LIRGenerator::visitBigIntMod(MBigIntMod* ins) { lowerBigIntMod(ins); }
+
+void LIRGenerator::visitBigIntPow(MBigIntPow* ins) {
+ auto* lir = new (alloc()) LBigIntPow(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntBitAnd(MBigIntBitAnd* ins) {
+ auto* lir = new (alloc()) LBigIntBitAnd(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntBitOr(MBigIntBitOr* ins) {
+ auto* lir = new (alloc()) LBigIntBitOr(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntBitXor(MBigIntBitXor* ins) {
+ auto* lir = new (alloc()) LBigIntBitXor(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntLsh(MBigIntLsh* ins) { lowerBigIntLsh(ins); }
+
+void LIRGenerator::visitBigIntRsh(MBigIntRsh* ins) { lowerBigIntRsh(ins); }
+
+void LIRGenerator::visitBigIntIncrement(MBigIntIncrement* ins) {
+ auto* lir =
+ new (alloc()) LBigIntIncrement(useRegister(ins->input()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntDecrement(MBigIntDecrement* ins) {
+ auto* lir =
+ new (alloc()) LBigIntDecrement(useRegister(ins->input()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntNegate(MBigIntNegate* ins) {
+ auto* lir = new (alloc()) LBigIntNegate(useRegister(ins->input()), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntBitNot(MBigIntBitNot* ins) {
+ auto* lir =
+ new (alloc()) LBigIntBitNot(useRegister(ins->input()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitInt32ToStringWithBase(MInt32ToStringWithBase* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->base()->type() == MIRType::Int32);
+
+ int32_t baseInt =
+ ins->base()->isConstant() ? ins->base()->toConstant()->toInt32() : 0;
+
+ LAllocation base;
+ if (2 <= baseInt && baseInt <= 36) {
+ base = useRegisterOrConstant(ins->base());
+ } else {
+ base = useRegister(ins->base());
+ }
+
+ auto* lir = new (alloc())
+ LInt32ToStringWithBase(useRegister(ins->input()), base, temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNumberParseInt(MNumberParseInt* ins) {
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ MOZ_ASSERT(ins->radix()->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LNumberParseInt(useRegisterAtStart(ins->string()),
+ useRegisterAtStart(ins->radix()),
+ tempFixed(CallTempReg0));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitDoubleParseInt(MDoubleParseInt* ins) {
+ MOZ_ASSERT(ins->number()->type() == MIRType::Double);
+
+ auto* lir =
+ new (alloc()) LDoubleParseInt(useRegister(ins->number()), tempDouble());
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitConcat(MConcat* ins) {
+ MDefinition* lhs = ins->getOperand(0);
+ MDefinition* rhs = ins->getOperand(1);
+
+ MOZ_ASSERT(lhs->type() == MIRType::String);
+ MOZ_ASSERT(rhs->type() == MIRType::String);
+ MOZ_ASSERT(ins->type() == MIRType::String);
+
+ LConcat* lir = new (alloc()) LConcat(
+ useFixedAtStart(lhs, CallTempReg0), useFixedAtStart(rhs, CallTempReg1),
+ tempFixed(CallTempReg0), tempFixed(CallTempReg1), tempFixed(CallTempReg2),
+ tempFixed(CallTempReg3), tempFixed(CallTempReg4));
+ defineFixed(lir, ins, LAllocation(AnyRegister(CallTempReg5)));
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitLinearizeForCharAccess(MLinearizeForCharAccess* ins) {
+ MDefinition* str = ins->string();
+ MDefinition* idx = ins->index();
+
+ MOZ_ASSERT(str->type() == MIRType::String);
+ MOZ_ASSERT(idx->type() == MIRType::Int32);
+
+ auto* lir =
+ new (alloc()) LLinearizeForCharAccess(useRegister(str), useRegister(idx));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCharCodeAt(MCharCodeAt* ins) {
+ MDefinition* str = ins->string();
+ MDefinition* idx = ins->index();
+
+ MOZ_ASSERT(str->type() == MIRType::String);
+ MOZ_ASSERT(idx->type() == MIRType::Int32);
+
+ LCharCodeAt* lir = new (alloc())
+ LCharCodeAt(useRegister(str), useRegister(idx), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCharCodeAtMaybeOutOfBounds(
+ MCharCodeAtMaybeOutOfBounds* ins) {
+ MDefinition* str = ins->string();
+ MDefinition* idx = ins->index();
+
+ MOZ_ASSERT(str->type() == MIRType::String);
+ MOZ_ASSERT(idx->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LCharCodeAtMaybeOutOfBounds(
+ useRegister(str), useRegister(idx), temp(), temp());
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCharAtMaybeOutOfBounds(MCharAtMaybeOutOfBounds* ins) {
+ MDefinition* str = ins->string();
+ MDefinition* idx = ins->index();
+
+ MOZ_ASSERT(str->type() == MIRType::String);
+ MOZ_ASSERT(idx->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LCharAtMaybeOutOfBounds(
+ useRegister(str), useRegister(idx), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitFromCharCode(MFromCharCode* ins) {
+ MDefinition* code = ins->getOperand(0);
+
+ MOZ_ASSERT(code->type() == MIRType::Int32);
+
+ LFromCharCode* lir = new (alloc()) LFromCharCode(useRegister(code));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitFromCodePoint(MFromCodePoint* ins) {
+ MDefinition* codePoint = ins->getOperand(0);
+
+ MOZ_ASSERT(codePoint->type() == MIRType::Int32);
+
+ LFromCodePoint* lir =
+ new (alloc()) LFromCodePoint(useRegister(codePoint), temp(), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitStringIndexOf(MStringIndexOf* ins) {
+ auto* string = ins->string();
+ MOZ_ASSERT(string->type() == MIRType::String);
+
+ auto* searchStr = ins->searchString();
+ MOZ_ASSERT(searchStr->type() == MIRType::String);
+
+ auto* lir = new (alloc())
+ LStringIndexOf(useRegisterAtStart(string), useRegisterAtStart(searchStr));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitStringStartsWith(MStringStartsWith* ins) {
+ auto* string = ins->string();
+ MOZ_ASSERT(string->type() == MIRType::String);
+
+ auto* searchStr = ins->searchString();
+ MOZ_ASSERT(searchStr->type() == MIRType::String);
+
+ if (searchStr->isConstant()) {
+ JSLinearString* linear = &searchStr->toConstant()->toString()->asLinear();
+
+ if (CanCompareCharactersInline(linear)) {
+ auto* lir = new (alloc())
+ LStringStartsWithInline(useRegister(string), temp(), linear);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+ }
+
+ auto* lir = new (alloc()) LStringStartsWith(useRegisterAtStart(string),
+ useRegisterAtStart(searchStr));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitStringEndsWith(MStringEndsWith* ins) {
+ auto* string = ins->string();
+ MOZ_ASSERT(string->type() == MIRType::String);
+
+ auto* searchStr = ins->searchString();
+ MOZ_ASSERT(searchStr->type() == MIRType::String);
+
+ if (searchStr->isConstant()) {
+ JSLinearString* linear = &searchStr->toConstant()->toString()->asLinear();
+
+ if (CanCompareCharactersInline(linear)) {
+ auto* lir = new (alloc())
+ LStringEndsWithInline(useRegister(string), temp(), linear);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+ }
+
+ auto* lir = new (alloc()) LStringEndsWith(useRegisterAtStart(string),
+ useRegisterAtStart(searchStr));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitStringConvertCase(MStringConvertCase* ins) {
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+
+ if (ins->mode() == MStringConvertCase::LowerCase) {
+#ifdef JS_CODEGEN_X86
+ // Due to lack of registers on x86, we reuse the string register as
+ // temporary. As a result we only need four temporary registers and take a
+ // bogus temporary as the fifth argument.
+ LDefinition temp4 = LDefinition::BogusTemp();
+#else
+ LDefinition temp4 = temp();
+#endif
+ auto* lir = new (alloc())
+ LStringToLowerCase(useRegister(ins->string()), temp(), temp(), temp(),
+ temp4, tempByteOpRegister());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ } else {
+ auto* lir =
+ new (alloc()) LStringToUpperCase(useRegisterAtStart(ins->string()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+ }
+}
+
+void LIRGenerator::visitStart(MStart* start) {}
+
+void LIRGenerator::visitNop(MNop* nop) {}
+
+void LIRGenerator::visitLimitedTruncate(MLimitedTruncate* nop) {
+ redefine(nop, nop->input());
+}
+
+void LIRGenerator::visitOsrEntry(MOsrEntry* entry) {
+ LOsrEntry* lir = new (alloc()) LOsrEntry(temp());
+ defineFixed(lir, entry, LAllocation(AnyRegister(OsrFrameReg)));
+}
+
+void LIRGenerator::visitOsrValue(MOsrValue* value) {
+ LOsrValue* lir = new (alloc()) LOsrValue(useRegister(value->entry()));
+ defineBox(lir, value);
+}
+
+void LIRGenerator::visitOsrReturnValue(MOsrReturnValue* value) {
+ LOsrReturnValue* lir =
+ new (alloc()) LOsrReturnValue(useRegister(value->entry()));
+ defineBox(lir, value);
+}
+
+void LIRGenerator::visitOsrEnvironmentChain(MOsrEnvironmentChain* object) {
+ LOsrEnvironmentChain* lir =
+ new (alloc()) LOsrEnvironmentChain(useRegister(object->entry()));
+ define(lir, object);
+}
+
+void LIRGenerator::visitOsrArgumentsObject(MOsrArgumentsObject* object) {
+ LOsrArgumentsObject* lir =
+ new (alloc()) LOsrArgumentsObject(useRegister(object->entry()));
+ define(lir, object);
+}
+
+void LIRGenerator::visitToDouble(MToDouble* convert) {
+ MDefinition* opd = convert->input();
+ mozilla::DebugOnly<MToFPInstruction::ConversionKind> conversion =
+ convert->conversion();
+
+ switch (opd->type()) {
+ case MIRType::Value: {
+ LValueToDouble* lir = new (alloc()) LValueToDouble(useBox(opd));
+ assignSnapshot(lir, convert->bailoutKind());
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::Null:
+ MOZ_ASSERT(conversion == MToFPInstruction::NonStringPrimitives);
+ lowerConstantDouble(0, convert);
+ break;
+
+ case MIRType::Undefined:
+ MOZ_ASSERT(conversion == MToFPInstruction::NonStringPrimitives);
+ lowerConstantDouble(GenericNaN(), convert);
+ break;
+
+ case MIRType::Boolean:
+ MOZ_ASSERT(conversion == MToFPInstruction::NonStringPrimitives);
+ [[fallthrough]];
+
+ case MIRType::Int32: {
+ LInt32ToDouble* lir =
+ new (alloc()) LInt32ToDouble(useRegisterAtStart(opd));
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::Float32: {
+ LFloat32ToDouble* lir =
+ new (alloc()) LFloat32ToDouble(useRegisterAtStart(opd));
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::Double:
+ redefine(convert, opd);
+ break;
+
+ default:
+ // Objects might be effectful. Symbols will throw.
+ // Strings are complicated - we don't handle them yet.
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitToFloat32(MToFloat32* convert) {
+ MDefinition* opd = convert->input();
+ mozilla::DebugOnly<MToFloat32::ConversionKind> conversion =
+ convert->conversion();
+
+ switch (opd->type()) {
+ case MIRType::Value: {
+ LValueToFloat32* lir = new (alloc()) LValueToFloat32(useBox(opd));
+ assignSnapshot(lir, convert->bailoutKind());
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::Null:
+ MOZ_ASSERT(conversion == MToFPInstruction::NonStringPrimitives);
+ lowerConstantFloat32(0, convert);
+ break;
+
+ case MIRType::Undefined:
+ MOZ_ASSERT(conversion == MToFPInstruction::NonStringPrimitives);
+ lowerConstantFloat32(GenericNaN(), convert);
+ break;
+
+ case MIRType::Boolean:
+ MOZ_ASSERT(conversion == MToFPInstruction::NonStringPrimitives);
+ [[fallthrough]];
+
+ case MIRType::Int32: {
+ LInt32ToFloat32* lir =
+ new (alloc()) LInt32ToFloat32(useRegisterAtStart(opd));
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::Double: {
+ LDoubleToFloat32* lir =
+ new (alloc()) LDoubleToFloat32(useRegisterAtStart(opd));
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::Float32:
+ redefine(convert, opd);
+ break;
+
+ default:
+ // Objects might be effectful. Symbols will throw.
+ // Strings are complicated - we don't handle them yet.
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitToNumberInt32(MToNumberInt32* convert) {
+ MDefinition* opd = convert->input();
+
+ switch (opd->type()) {
+ case MIRType::Value: {
+ auto* lir = new (alloc()) LValueToInt32(useBox(opd), tempDouble(), temp(),
+ LValueToInt32::NORMAL);
+ assignSnapshot(lir, convert->bailoutKind());
+ define(lir, convert);
+ if (lir->mode() == LValueToInt32::TRUNCATE ||
+ lir->mode() == LValueToInt32::TRUNCATE_NOWRAP) {
+ assignSafepoint(lir, convert);
+ }
+ break;
+ }
+
+ case MIRType::Null:
+ MOZ_ASSERT(convert->conversion() == IntConversionInputKind::Any);
+ define(new (alloc()) LInteger(0), convert);
+ break;
+
+ case MIRType::Boolean:
+ MOZ_ASSERT(convert->conversion() == IntConversionInputKind::Any ||
+ convert->conversion() ==
+ IntConversionInputKind::NumbersOrBoolsOnly);
+ redefine(convert, opd);
+ break;
+
+ case MIRType::Int32:
+ redefine(convert, opd);
+ break;
+
+ case MIRType::Float32: {
+ LFloat32ToInt32* lir = new (alloc()) LFloat32ToInt32(useRegister(opd));
+ assignSnapshot(lir, convert->bailoutKind());
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::Double: {
+ LDoubleToInt32* lir = new (alloc()) LDoubleToInt32(useRegister(opd));
+ assignSnapshot(lir, convert->bailoutKind());
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ case MIRType::Object:
+ case MIRType::Undefined:
+ // Objects might be effectful. Symbols and BigInts throw. Undefined
+ // coerces to NaN, not int32.
+ MOZ_CRASH("ToInt32 invalid input type");
+
+ default:
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitToIntegerInt32(MToIntegerInt32* convert) {
+ MDefinition* opd = convert->input();
+
+ switch (opd->type()) {
+ case MIRType::Value: {
+ auto* lir = new (alloc()) LValueToInt32(useBox(opd), tempDouble(), temp(),
+ LValueToInt32::TRUNCATE_NOWRAP);
+ assignSnapshot(lir, convert->bailoutKind());
+ define(lir, convert);
+ assignSafepoint(lir, convert);
+ break;
+ }
+
+ case MIRType::Undefined:
+ case MIRType::Null:
+ define(new (alloc()) LInteger(0), convert);
+ break;
+
+ case MIRType::Boolean:
+ case MIRType::Int32:
+ redefine(convert, opd);
+ break;
+
+ case MIRType::Float32: {
+ auto* lir = new (alloc()) LFloat32ToIntegerInt32(useRegister(opd));
+ assignSnapshot(lir, convert->bailoutKind());
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::Double: {
+ auto* lir = new (alloc()) LDoubleToIntegerInt32(useRegister(opd));
+ assignSnapshot(lir, convert->bailoutKind());
+ define(lir, convert);
+ break;
+ }
+
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ case MIRType::Object:
+ // Objects might be effectful. Symbols and BigInts throw.
+ // Strings are complicated - we don't handle them yet.
+ MOZ_CRASH("ToIntegerInt32 invalid input type");
+
+ default:
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitTruncateToInt32(MTruncateToInt32* truncate) {
+ MDefinition* opd = truncate->input();
+
+ switch (opd->type()) {
+ case MIRType::Value: {
+ LValueToInt32* lir = new (alloc()) LValueToInt32(
+ useBox(opd), tempDouble(), temp(), LValueToInt32::TRUNCATE);
+ assignSnapshot(lir, truncate->bailoutKind());
+ define(lir, truncate);
+ assignSafepoint(lir, truncate);
+ break;
+ }
+
+ case MIRType::Null:
+ case MIRType::Undefined:
+ define(new (alloc()) LInteger(0), truncate);
+ break;
+
+ case MIRType::Int32:
+ case MIRType::Boolean:
+ redefine(truncate, opd);
+ break;
+
+ case MIRType::Double:
+ // May call into JS::ToInt32() on the slow OOL path.
+ gen->setNeedsStaticStackAlignment();
+ lowerTruncateDToInt32(truncate);
+ break;
+
+ case MIRType::Float32:
+ // May call into JS::ToInt32() on the slow OOL path.
+ gen->setNeedsStaticStackAlignment();
+ lowerTruncateFToInt32(truncate);
+ break;
+
+ default:
+ // Objects might be effectful. Symbols throw.
+ // Strings are complicated - we don't handle them yet.
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitInt32ToIntPtr(MInt32ToIntPtr* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->type() == MIRType::IntPtr);
+
+#ifdef JS_64BIT
+ // If the result is only used by instructions that expect a bounds-checked
+ // index, we must have eliminated or hoisted a bounds check and we can assume
+ // the index is non-negative. This lets us generate more efficient code.
+ // In debug builds we verify this non-negative assumption at runtime.
+ if (ins->canBeNegative()) {
+ bool canBeNegative = false;
+ for (MUseDefIterator iter(ins); iter; iter++) {
+ if (!iter.def()->isSpectreMaskIndex() &&
+ !iter.def()->isLoadUnboxedScalar() &&
+ !iter.def()->isStoreUnboxedScalar() &&
+ !iter.def()->isLoadDataViewElement() &&
+ !iter.def()->isStoreDataViewElement()) {
+ canBeNegative = true;
+ break;
+ }
+ }
+ if (!canBeNegative) {
+ ins->setCanNotBeNegative();
+ }
+ }
+
+ if (ins->canBeNegative()) {
+ auto* lir = new (alloc()) LInt32ToIntPtr(useAnyAtStart(input));
+ define(lir, ins);
+ } else {
+# ifdef DEBUG
+ auto* lir = new (alloc()) LInt32ToIntPtr(useRegisterAtStart(input));
+ defineReuseInput(lir, ins, 0);
+# else
+ // In non-debug mode this is a no-op.
+ redefine(ins, input);
+# endif
+ }
+#else
+ // On 32-bit platforms this is a no-op.
+ redefine(ins, input);
+#endif
+}
+
+void LIRGenerator::visitNonNegativeIntPtrToInt32(
+ MNonNegativeIntPtrToInt32* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::IntPtr);
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+
+#ifdef JS_64BIT
+ auto* lir =
+ new (alloc()) LNonNegativeIntPtrToInt32(useRegisterAtStart(input));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReuseInput(lir, ins, 0);
+#else
+ // On 32-bit platforms this is a no-op.
+ redefine(ins, input);
+#endif
+}
+
+void LIRGenerator::visitWasmExtendU32Index(MWasmExtendU32Index* ins) {
+#ifdef JS_64BIT
+ // Technically this produces an Int64 register and I guess we could clean that
+ // up, but it's a 64-bit only operation, so it doesn't actually matter.
+
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->type() == MIRType::Int64);
+
+ // Input reuse is OK even on ARM64 because this node *must* reuse its input in
+ // order not to generate any code at all, as is the intent.
+ auto* lir = new (alloc()) LWasmExtendU32Index(useRegisterAtStart(input));
+ defineReuseInput(lir, ins, 0);
+#else
+ MOZ_CRASH("64-bit only");
+#endif
+}
+
+void LIRGenerator::visitWasmWrapU32Index(MWasmWrapU32Index* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Int64);
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+
+ // Tricky: On 64-bit, this just returns its input (except on MIPS64 there may
+ // be a sign/zero extension). On 32-bit, it returns the low register of the
+ // input, and should generate no code.
+
+ // If this assertion does not hold then using "input" unadorned as an alias
+ // for the low register will not work.
+#if defined(JS_NUNBOX32)
+ static_assert(INT64LOW_INDEX == 0);
+#endif
+
+ auto* lir = new (alloc()) LWasmWrapU32Index(useRegisterAtStart(input));
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitIntPtrToDouble(MIntPtrToDouble* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::IntPtr);
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+
+ auto* lir = new (alloc()) LIntPtrToDouble(useRegister(input));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAdjustDataViewLength(MAdjustDataViewLength* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::IntPtr);
+
+ auto* lir = new (alloc()) LAdjustDataViewLength(useRegisterAtStart(input));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitToBigInt(MToBigInt* ins) {
+ MDefinition* opd = ins->input();
+
+ switch (opd->type()) {
+ case MIRType::Value: {
+ auto* lir = new (alloc()) LValueToBigInt(useBox(opd));
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+
+ case MIRType::BigInt:
+ redefine(ins, opd);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitToInt64(MToInt64* ins) {
+ MDefinition* opd = ins->input();
+
+ switch (opd->type()) {
+ case MIRType::Value: {
+ auto* lir = new (alloc()) LValueToInt64(useBox(opd), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ defineInt64(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+
+ case MIRType::Boolean: {
+ auto* lir = new (alloc()) LBooleanToInt64(useRegisterAtStart(opd));
+ defineInt64(lir, ins);
+ break;
+ }
+
+ case MIRType::String: {
+ auto* lir = new (alloc()) LStringToInt64(useRegister(opd));
+ defineInt64(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+
+ // An Int64 may be passed here from a BigInt to Int64 conversion.
+ case MIRType::Int64: {
+ redefine(ins, opd);
+ break;
+ }
+
+ default:
+ // Undefined, Null, Number, and Symbol throw.
+ // Objects may be effectful.
+ // BigInt operands are eliminated by the type policy.
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitTruncateBigIntToInt64(MTruncateBigIntToInt64* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::BigInt);
+ auto* lir = new (alloc()) LTruncateBigIntToInt64(useRegister(ins->input()));
+ defineInt64(lir, ins);
+}
+
+void LIRGenerator::visitInt64ToBigInt(MInt64ToBigInt* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int64);
+ auto* lir =
+ new (alloc()) LInt64ToBigInt(useInt64Register(ins->input()), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt32(MWasmTruncateToInt32* ins) {
+ MDefinition* input = ins->input();
+ switch (input->type()) {
+ case MIRType::Double:
+ case MIRType::Float32: {
+ auto* lir = new (alloc()) LWasmTruncateToInt32(useRegisterAtStart(input));
+ define(lir, ins);
+ break;
+ }
+ default:
+ MOZ_CRASH("unexpected type in WasmTruncateToInt32");
+ }
+}
+
+void LIRGenerator::visitWasmBuiltinTruncateToInt32(
+ MWasmBuiltinTruncateToInt32* truncate) {
+ mozilla::DebugOnly<MDefinition*> opd = truncate->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ // May call into JS::ToInt32() on the slow OOL path.
+ gen->setNeedsStaticStackAlignment();
+ lowerWasmBuiltinTruncateToInt32(truncate);
+}
+
+void LIRGenerator::visitWasmBoxValue(MWasmBoxValue* ins) {
+ LWasmBoxValue* lir = new (alloc()) LWasmBoxValue(useBox(ins->input()));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmAnyRefFromJSObject(MWasmAnyRefFromJSObject* ins) {
+ LWasmAnyRefFromJSObject* lir =
+ new (alloc()) LWasmAnyRefFromJSObject(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWrapInt64ToInt32(MWrapInt64ToInt32* ins) {
+ define(new (alloc()) LWrapInt64ToInt32(useInt64AtStart(ins->input())), ins);
+}
+
+void LIRGenerator::visitToString(MToString* ins) {
+ MDefinition* opd = ins->input();
+
+ switch (opd->type()) {
+ case MIRType::Null: {
+ const JSAtomState& names = gen->runtime->names();
+ LPointer* lir = new (alloc()) LPointer(names.null);
+ define(lir, ins);
+ break;
+ }
+
+ case MIRType::Undefined: {
+ const JSAtomState& names = gen->runtime->names();
+ LPointer* lir = new (alloc()) LPointer(names.undefined);
+ define(lir, ins);
+ break;
+ }
+
+ case MIRType::Boolean: {
+ LBooleanToString* lir = new (alloc()) LBooleanToString(useRegister(opd));
+ define(lir, ins);
+ break;
+ }
+
+ case MIRType::Double: {
+ LDoubleToString* lir =
+ new (alloc()) LDoubleToString(useRegister(opd), temp());
+
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+
+ case MIRType::Int32: {
+ LIntToString* lir = new (alloc()) LIntToString(useRegister(opd));
+
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+
+ case MIRType::String:
+ redefine(ins, ins->input());
+ break;
+
+ case MIRType::Value: {
+ LValueToString* lir =
+ new (alloc()) LValueToString(useBox(opd), tempToUnbox());
+ if (ins->needsSnapshot()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+
+ default:
+ // Float32, symbols, bigint, and objects are not supported.
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitRegExp(MRegExp* ins) {
+ LRegExp* lir = new (alloc()) LRegExp(temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitRegExpMatcher(MRegExpMatcher* ins) {
+ MOZ_ASSERT(ins->regexp()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ MOZ_ASSERT(ins->lastIndex()->type() == MIRType::Int32);
+
+ LRegExpMatcher* lir = new (alloc()) LRegExpMatcher(
+ useFixedAtStart(ins->regexp(), RegExpMatcherRegExpReg),
+ useFixedAtStart(ins->string(), RegExpMatcherStringReg),
+ useFixedAtStart(ins->lastIndex(), RegExpMatcherLastIndexReg));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitRegExpSearcher(MRegExpSearcher* ins) {
+ MOZ_ASSERT(ins->regexp()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ MOZ_ASSERT(ins->lastIndex()->type() == MIRType::Int32);
+
+ LRegExpSearcher* lir = new (alloc()) LRegExpSearcher(
+ useFixedAtStart(ins->regexp(), RegExpTesterRegExpReg),
+ useFixedAtStart(ins->string(), RegExpTesterStringReg),
+ useFixedAtStart(ins->lastIndex(), RegExpTesterLastIndexReg));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitRegExpTester(MRegExpTester* ins) {
+ MOZ_ASSERT(ins->regexp()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ MOZ_ASSERT(ins->lastIndex()->type() == MIRType::Int32);
+
+ LRegExpTester* lir = new (alloc()) LRegExpTester(
+ useFixedAtStart(ins->regexp(), RegExpTesterRegExpReg),
+ useFixedAtStart(ins->string(), RegExpTesterStringReg),
+ useFixedAtStart(ins->lastIndex(), RegExpTesterLastIndexReg));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitRegExpPrototypeOptimizable(
+ MRegExpPrototypeOptimizable* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+ LRegExpPrototypeOptimizable* lir = new (alloc())
+ LRegExpPrototypeOptimizable(useRegister(ins->object()), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitRegExpInstanceOptimizable(
+ MRegExpInstanceOptimizable* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->proto()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+ LRegExpInstanceOptimizable* lir = new (alloc()) LRegExpInstanceOptimizable(
+ useRegister(ins->object()), useRegister(ins->proto()), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitGetFirstDollarIndex(MGetFirstDollarIndex* ins) {
+ MOZ_ASSERT(ins->str()->type() == MIRType::String);
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+ LGetFirstDollarIndex* lir = new (alloc())
+ LGetFirstDollarIndex(useRegister(ins->str()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitStringReplace(MStringReplace* ins) {
+ MOZ_ASSERT(ins->pattern()->type() == MIRType::String);
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ MOZ_ASSERT(ins->replacement()->type() == MIRType::String);
+
+ LStringReplace* lir = new (alloc())
+ LStringReplace(useRegisterOrConstantAtStart(ins->string()),
+ useRegisterAtStart(ins->pattern()),
+ useRegisterOrConstantAtStart(ins->replacement()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBinaryCache(MBinaryCache* ins) {
+ MDefinition* lhs = ins->getOperand(0);
+ MDefinition* rhs = ins->getOperand(1);
+
+ MOZ_ASSERT(ins->type() == MIRType::Value || ins->type() == MIRType::Boolean);
+ LInstruction* lir;
+ if (ins->type() == MIRType::Value) {
+ LBinaryValueCache* valueLir = new (alloc()) LBinaryValueCache(
+ useBox(lhs), useBox(rhs), tempFixed(FloatReg0), tempFixed(FloatReg1));
+ defineBox(valueLir, ins);
+ lir = valueLir;
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+ LBinaryBoolCache* boolLir = new (alloc()) LBinaryBoolCache(
+ useBox(lhs), useBox(rhs), tempFixed(FloatReg0), tempFixed(FloatReg1));
+ define(boolLir, ins);
+ lir = boolLir;
+ }
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitUnaryCache(MUnaryCache* ins) {
+ MDefinition* input = ins->getOperand(0);
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ LUnaryCache* lir = new (alloc()) LUnaryCache(useBox(input));
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitModuleMetadata(MModuleMetadata* ins) {
+ LModuleMetadata* lir = new (alloc()) LModuleMetadata();
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitDynamicImport(MDynamicImport* ins) {
+ LDynamicImport* lir = new (alloc()) LDynamicImport(
+ useBoxAtStart(ins->specifier()), useBoxAtStart(ins->options()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitLambda(MLambda* ins) {
+ MOZ_ASSERT(ins->environmentChain()->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LLambda(useRegister(ins->environmentChain()), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitFunctionWithProto(MFunctionWithProto* ins) {
+ MOZ_ASSERT(ins->environmentChain()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->prototype()->type() == MIRType::Object);
+
+ auto* lir = new (alloc())
+ LFunctionWithProto(useRegisterAtStart(ins->environmentChain()),
+ useRegisterAtStart(ins->prototype()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitSetFunName(MSetFunName* ins) {
+ MOZ_ASSERT(ins->fun()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->name()->type() == MIRType::Value);
+
+ LSetFunName* lir = new (alloc())
+ LSetFunName(useRegisterAtStart(ins->fun()), useBoxAtStart(ins->name()));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewLexicalEnvironmentObject(
+ MNewLexicalEnvironmentObject* ins) {
+ auto* lir = new (alloc()) LNewLexicalEnvironmentObject(temp());
+
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewClassBodyEnvironmentObject(
+ MNewClassBodyEnvironmentObject* ins) {
+ auto* lir = new (alloc()) LNewClassBodyEnvironmentObject(temp());
+
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewVarEnvironmentObject(MNewVarEnvironmentObject* ins) {
+ auto* lir = new (alloc()) LNewVarEnvironmentObject(temp());
+
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitKeepAliveObject(MKeepAliveObject* ins) {
+ MDefinition* obj = ins->object();
+ MOZ_ASSERT(obj->type() == MIRType::Object);
+
+ add(new (alloc()) LKeepAliveObject(useKeepalive(obj)), ins);
+}
+
+void LIRGenerator::visitSlots(MSlots* ins) {
+ define(new (alloc()) LSlots(useRegisterAtStart(ins->object())), ins);
+}
+
+void LIRGenerator::visitElements(MElements* ins) {
+ define(new (alloc()) LElements(useRegisterAtStart(ins->object())), ins);
+}
+
+void LIRGenerator::visitLoadDynamicSlot(MLoadDynamicSlot* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+ defineBox(new (alloc()) LLoadDynamicSlotV(useRegisterAtStart(ins->slots())),
+ ins);
+}
+
+void LIRGenerator::visitFunctionEnvironment(MFunctionEnvironment* ins) {
+ define(new (alloc())
+ LFunctionEnvironment(useRegisterAtStart(ins->function())),
+ ins);
+}
+
+void LIRGenerator::visitHomeObject(MHomeObject* ins) {
+ define(new (alloc()) LHomeObject(useRegisterAtStart(ins->function())), ins);
+}
+
+void LIRGenerator::visitHomeObjectSuperBase(MHomeObjectSuperBase* ins) {
+ MOZ_ASSERT(ins->homeObject()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ auto lir =
+ new (alloc()) LHomeObjectSuperBase(useRegisterAtStart(ins->homeObject()));
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitInterruptCheck(MInterruptCheck* ins) {
+ LInstruction* lir = new (alloc()) LInterruptCheck();
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmInterruptCheck(MWasmInterruptCheck* ins) {
+ auto* lir =
+ new (alloc()) LWasmInterruptCheck(useRegisterAtStart(ins->instance()));
+ add(lir, ins);
+ assignWasmSafepoint(lir);
+}
+
+void LIRGenerator::visitWasmTrap(MWasmTrap* ins) {
+ add(new (alloc()) LWasmTrap, ins);
+}
+
+void LIRGenerator::visitWasmReinterpret(MWasmReinterpret* ins) {
+ if (ins->type() == MIRType::Int64) {
+ defineInt64(new (alloc())
+ LWasmReinterpretToI64(useRegisterAtStart(ins->input())),
+ ins);
+ } else if (ins->input()->type() == MIRType::Int64) {
+ define(new (alloc())
+ LWasmReinterpretFromI64(useInt64RegisterAtStart(ins->input())),
+ ins);
+ } else {
+ define(new (alloc()) LWasmReinterpret(useRegisterAtStart(ins->input())),
+ ins);
+ }
+}
+
+void LIRGenerator::visitStoreDynamicSlot(MStoreDynamicSlot* ins) {
+ LInstruction* lir;
+
+ switch (ins->value()->type()) {
+ case MIRType::Value:
+ lir = new (alloc())
+ LStoreDynamicSlotV(useRegister(ins->slots()), useBox(ins->value()));
+ add(lir, ins);
+ break;
+
+ case MIRType::Double:
+ add(new (alloc()) LStoreDynamicSlotT(useRegister(ins->slots()),
+ useRegister(ins->value())),
+ ins);
+ break;
+
+ case MIRType::Float32:
+ MOZ_CRASH("Float32 shouldn't be stored in a slot.");
+
+ default:
+ add(new (alloc()) LStoreDynamicSlotT(useRegister(ins->slots()),
+ useRegisterOrConstant(ins->value())),
+ ins);
+ break;
+ }
+}
+
+// Returns true iff |def| is a constant that's either not a GC thing or is not
+// allocated in the nursery.
+static bool IsNonNurseryConstant(MDefinition* def) {
+ if (!def->isConstant()) {
+ return false;
+ }
+ Value v = def->toConstant()->toJSValue();
+ return !v.isGCThing() || !IsInsideNursery(v.toGCThing());
+}
+
+void LIRGenerator::visitPostWriteBarrier(MPostWriteBarrier* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ // LPostWriteBarrier assumes that if it has a constant object then that
+ // object is tenured, and does not need to be tested for being in the
+ // nursery. Ensure that assumption holds by lowering constant nursery
+ // objects to a register.
+ bool useConstantObject = IsNonNurseryConstant(ins->object());
+
+ switch (ins->value()->type()) {
+ case MIRType::Object: {
+ LDefinition tmp =
+ needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
+ LPostWriteBarrierO* lir = new (alloc())
+ LPostWriteBarrierO(useConstantObject ? useOrConstant(ins->object())
+ : useRegister(ins->object()),
+ useRegister(ins->value()), tmp);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+ case MIRType::String: {
+ LDefinition tmp =
+ needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
+ LPostWriteBarrierS* lir = new (alloc())
+ LPostWriteBarrierS(useConstantObject ? useOrConstant(ins->object())
+ : useRegister(ins->object()),
+ useRegister(ins->value()), tmp);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+ case MIRType::BigInt: {
+ LDefinition tmp =
+ needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
+ auto* lir = new (alloc())
+ LPostWriteBarrierBI(useConstantObject ? useOrConstant(ins->object())
+ : useRegister(ins->object()),
+ useRegister(ins->value()), tmp);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+ case MIRType::Value: {
+ LDefinition tmp =
+ needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
+ LPostWriteBarrierV* lir = new (alloc())
+ LPostWriteBarrierV(useConstantObject ? useOrConstant(ins->object())
+ : useRegister(ins->object()),
+ useBox(ins->value()), tmp);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+ default:
+ // Currently, only objects and strings can be in the nursery. Other
+ // instruction types cannot hold nursery pointers.
+ break;
+ }
+}
+
+void LIRGenerator::visitPostWriteElementBarrier(MPostWriteElementBarrier* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+
+ // LPostWriteElementBarrier assumes that if it has a constant object then that
+ // object is tenured, and does not need to be tested for being in the
+ // nursery. Ensure that assumption holds by lowering constant nursery
+ // objects to a register.
+ bool useConstantObject =
+ ins->object()->isConstant() &&
+ !IsInsideNursery(&ins->object()->toConstant()->toObject());
+
+ switch (ins->value()->type()) {
+ case MIRType::Object: {
+ LDefinition tmp =
+ needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
+ LPostWriteElementBarrierO* lir = new (alloc()) LPostWriteElementBarrierO(
+ useConstantObject ? useOrConstant(ins->object())
+ : useRegister(ins->object()),
+ useRegister(ins->value()), useRegister(ins->index()), tmp);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+ case MIRType::String: {
+ LDefinition tmp =
+ needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
+ LPostWriteElementBarrierS* lir = new (alloc()) LPostWriteElementBarrierS(
+ useConstantObject ? useOrConstant(ins->object())
+ : useRegister(ins->object()),
+ useRegister(ins->value()), useRegister(ins->index()), tmp);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+ case MIRType::BigInt: {
+ LDefinition tmp =
+ needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
+ auto* lir = new (alloc()) LPostWriteElementBarrierBI(
+ useConstantObject ? useOrConstant(ins->object())
+ : useRegister(ins->object()),
+ useRegister(ins->value()), useRegister(ins->index()), tmp);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+ case MIRType::Value: {
+ LDefinition tmp =
+ needTempForPostBarrier() ? temp() : LDefinition::BogusTemp();
+ LPostWriteElementBarrierV* lir = new (alloc()) LPostWriteElementBarrierV(
+ useConstantObject ? useOrConstant(ins->object())
+ : useRegister(ins->object()),
+ useRegister(ins->index()), useBox(ins->value()), tmp);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+ default:
+ // Currently, only objects, strings, and bigints can be in the nursery.
+ // Other instruction types cannot hold nursery pointers.
+ break;
+ }
+}
+
+void LIRGenerator::visitAssertCanElidePostWriteBarrier(
+ MAssertCanElidePostWriteBarrier* ins) {
+ auto* lir = new (alloc()) LAssertCanElidePostWriteBarrier(
+ useRegister(ins->object()), useBox(ins->value()), temp());
+ add(lir, ins);
+}
+
+void LIRGenerator::visitArrayLength(MArrayLength* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ auto* lir = new (alloc()) LArrayLength(useRegisterAtStart(ins->elements()));
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitSetArrayLength(MSetArrayLength* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+
+ MOZ_ASSERT(ins->index()->isConstant());
+ add(new (alloc()) LSetArrayLength(useRegister(ins->elements()),
+ useRegisterOrConstant(ins->index())),
+ ins);
+}
+
+void LIRGenerator::visitFunctionLength(MFunctionLength* ins) {
+ MOZ_ASSERT(ins->function()->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LFunctionLength(useRegister(ins->function()));
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitFunctionName(MFunctionName* ins) {
+ MOZ_ASSERT(ins->function()->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LFunctionName(useRegister(ins->function()));
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitGetNextEntryForIterator(MGetNextEntryForIterator* ins) {
+ MOZ_ASSERT(ins->iter()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->result()->type() == MIRType::Object);
+ auto lir = new (alloc()) LGetNextEntryForIterator(useRegister(ins->iter()),
+ useRegister(ins->result()),
+ temp(), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitArrayBufferByteLength(MArrayBufferByteLength* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::IntPtr);
+
+ auto* lir =
+ new (alloc()) LArrayBufferByteLength(useRegisterAtStart(ins->object()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitArrayBufferViewLength(MArrayBufferViewLength* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::IntPtr);
+
+ auto* lir =
+ new (alloc()) LArrayBufferViewLength(useRegisterAtStart(ins->object()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitArrayBufferViewByteOffset(
+ MArrayBufferViewByteOffset* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::IntPtr);
+
+ auto* lir = new (alloc())
+ LArrayBufferViewByteOffset(useRegisterAtStart(ins->object()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitArrayBufferViewElements(MArrayBufferViewElements* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Elements);
+ define(new (alloc())
+ LArrayBufferViewElements(useRegisterAtStart(ins->object())),
+ ins);
+}
+
+void LIRGenerator::visitTypedArrayElementSize(MTypedArrayElementSize* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ define(new (alloc())
+ LTypedArrayElementSize(useRegisterAtStart(ins->object())),
+ ins);
+}
+
+void LIRGenerator::visitGuardHasAttachedArrayBuffer(
+ MGuardHasAttachedArrayBuffer* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ auto* lir = new (alloc())
+ LGuardHasAttachedArrayBuffer(useRegister(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardNumberToIntPtrIndex(
+ MGuardNumberToIntPtrIndex* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Double);
+
+ auto* lir = new (alloc()) LGuardNumberToIntPtrIndex(useRegister(input));
+ if (!ins->supportOOB()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ define(lir, ins);
+}
+
+void LIRGenerator::visitInitializedLength(MInitializedLength* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ define(new (alloc()) LInitializedLength(useRegisterAtStart(ins->elements())),
+ ins);
+}
+
+void LIRGenerator::visitSetInitializedLength(MSetInitializedLength* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+
+ MOZ_ASSERT(ins->index()->isConstant());
+ add(new (alloc()) LSetInitializedLength(useRegister(ins->elements()),
+ useRegisterOrConstant(ins->index())),
+ ins);
+}
+
+void LIRGenerator::visitNot(MNot* ins) {
+ MDefinition* op = ins->input();
+
+ // String is converted to length of string in the type analysis phase (see
+ // TestPolicy).
+ MOZ_ASSERT(op->type() != MIRType::String);
+
+ // - boolean: x xor 1
+ // - int32: LCompare(x, 0)
+ // - double: LCompare(x, 0)
+ // - null or undefined: true
+ // - symbol: false
+ // - bigint: LNotBI(x)
+ // - object: false if it never emulates undefined, else LNotO(x)
+ switch (op->type()) {
+ case MIRType::Boolean: {
+ MConstant* cons = MConstant::New(alloc(), Int32Value(1));
+ ins->block()->insertBefore(ins, cons);
+ lowerForALU(new (alloc()) LBitOpI(JSOp::BitXor), ins, op, cons);
+ break;
+ }
+ case MIRType::Int32:
+ define(new (alloc()) LNotI(useRegisterAtStart(op)), ins);
+ break;
+ case MIRType::Int64:
+ define(new (alloc()) LNotI64(useInt64RegisterAtStart(op)), ins);
+ break;
+ case MIRType::Double:
+ define(new (alloc()) LNotD(useRegister(op)), ins);
+ break;
+ case MIRType::Float32:
+ define(new (alloc()) LNotF(useRegister(op)), ins);
+ break;
+ case MIRType::Undefined:
+ case MIRType::Null:
+ define(new (alloc()) LInteger(1), ins);
+ break;
+ case MIRType::Symbol:
+ define(new (alloc()) LInteger(0), ins);
+ break;
+ case MIRType::BigInt:
+ define(new (alloc()) LNotBI(useRegisterAtStart(op)), ins);
+ break;
+ case MIRType::Object:
+ define(new (alloc()) LNotO(useRegister(op)), ins);
+ break;
+ case MIRType::Value: {
+ auto* lir = new (alloc()) LNotV(useBox(op), tempDouble(), tempToUnbox());
+ define(lir, ins);
+ break;
+ }
+
+ default:
+ MOZ_CRASH("Unexpected MIRType.");
+ }
+}
+
+void LIRGenerator::visitBoundsCheck(MBoundsCheck* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Int32 || ins->type() == MIRType::IntPtr);
+ MOZ_ASSERT(ins->index()->type() == ins->type());
+ MOZ_ASSERT(ins->length()->type() == ins->type());
+
+ if (!ins->fallible()) {
+ return;
+ }
+
+ LInstruction* check;
+ if (ins->minimum() || ins->maximum()) {
+ check = new (alloc())
+ LBoundsCheckRange(useRegisterOrInt32Constant(ins->index()),
+ useAny(ins->length()), temp());
+ } else {
+ check = new (alloc()) LBoundsCheck(useRegisterOrInt32Constant(ins->index()),
+ useAnyOrInt32Constant(ins->length()));
+ }
+ assignSnapshot(check, ins->bailoutKind());
+ add(check, ins);
+}
+
+void LIRGenerator::visitSpectreMaskIndex(MSpectreMaskIndex* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Int32 || ins->type() == MIRType::IntPtr);
+ MOZ_ASSERT(ins->index()->type() == ins->type());
+ MOZ_ASSERT(ins->length()->type() == ins->type());
+
+ auto* lir = new (alloc())
+ LSpectreMaskIndex(useRegister(ins->index()), useAny(ins->length()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitBoundsCheckLower(MBoundsCheckLower* ins) {
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+
+ if (!ins->fallible()) {
+ return;
+ }
+
+ LInstruction* check =
+ new (alloc()) LBoundsCheckLower(useRegister(ins->index()));
+ assignSnapshot(check, ins->bailoutKind());
+ add(check, ins);
+}
+
+void LIRGenerator::visitInArray(MInArray* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->initLength()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+
+ auto* lir = new (alloc()) LInArray(useRegister(ins->elements()),
+ useRegisterOrConstant(ins->index()),
+ useRegister(ins->initLength()));
+ if (ins->needsNegativeIntCheck()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ define(lir, ins);
+}
+
+void LIRGenerator::visitGuardElementNotHole(MGuardElementNotHole* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+
+ auto* guard = new (alloc())
+ LGuardElementNotHole(useRegisterAtStart(ins->elements()),
+ useRegisterOrConstantAtStart(ins->index()));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+}
+
+void LIRGenerator::visitLoadElement(MLoadElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LLoadElementV(useRegister(ins->elements()),
+ useRegisterOrConstant(ins->index()));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitLoadElementHole(MLoadElementHole* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->initLength()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ LLoadElementHole* lir = new (alloc())
+ LLoadElementHole(useRegister(ins->elements()), useRegister(ins->index()),
+ useRegister(ins->initLength()));
+ if (ins->needsNegativeIntCheck()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitStoreElement(MStoreElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index = useRegisterOrConstant(ins->index());
+
+ switch (ins->value()->type()) {
+ case MIRType::Value: {
+ LInstruction* lir =
+ new (alloc()) LStoreElementV(elements, index, useBox(ins->value()));
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ add(lir, ins);
+ break;
+ }
+
+ default: {
+ const LAllocation value = useRegisterOrNonDoubleConstant(ins->value());
+ LInstruction* lir = new (alloc()) LStoreElementT(elements, index, value);
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ add(lir, ins);
+ break;
+ }
+ }
+}
+
+void LIRGenerator::visitStoreHoleValueElement(MStoreHoleValueElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LStoreHoleValueElement(useRegister(ins->elements()),
+ useRegister(ins->index()));
+ add(lir, ins);
+}
+
+static bool BoundsCheckNeedsSpectreTemp() {
+ // On x86, spectreBoundsCheck32 can emit better code if it has a scratch
+ // register and index masking is enabled.
+#ifdef JS_CODEGEN_X86
+ return JitOptions.spectreIndexMasking;
+#else
+ return false;
+#endif
+}
+
+void LIRGenerator::visitStoreElementHole(MStoreElementHole* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Int32);
+
+ const LUse object = useRegister(ins->object());
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index = useRegister(ins->index());
+
+ LInstruction* lir;
+ switch (ins->value()->type()) {
+ case MIRType::Value:
+ lir = new (alloc()) LStoreElementHoleV(object, elements, index,
+ useBox(ins->value()), temp());
+ break;
+
+ default: {
+ const LAllocation value = useRegisterOrNonDoubleConstant(ins->value());
+ lir = new (alloc())
+ LStoreElementHoleT(object, elements, index, value, temp());
+ break;
+ }
+ }
+
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitEffectiveAddress(MEffectiveAddress* ins) {
+ define(new (alloc()) LEffectiveAddress(useRegister(ins->base()),
+ useRegister(ins->index())),
+ ins);
+}
+
+void LIRGenerator::visitArrayPopShift(MArrayPopShift* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ auto* lir =
+ new (alloc()) LArrayPopShift(useRegister(ins->object()), temp(), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ defineBox(lir, ins);
+
+ if (ins->mode() == MArrayPopShift::Shift) {
+ assignSafepoint(lir, ins);
+ }
+}
+
+void LIRGenerator::visitArrayPush(MArrayPush* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->value()->type() == MIRType::Value);
+
+ LUse object = useRegister(ins->object());
+
+ LDefinition spectreTemp =
+ BoundsCheckNeedsSpectreTemp() ? temp() : LDefinition::BogusTemp();
+
+ auto* lir = new (alloc())
+ LArrayPush(object, useBox(ins->value()), temp(), spectreTemp);
+ // We will bailout before pushing if the length would overflow INT32_MAX.
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitArraySlice(MArraySlice* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->begin()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->end()->type() == MIRType::Int32);
+
+ LArraySlice* lir = new (alloc()) LArraySlice(
+ useRegisterAtStart(ins->object()), useRegisterAtStart(ins->begin()),
+ useRegisterAtStart(ins->end()), tempFixed(CallTempReg0),
+ tempFixed(CallTempReg1));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitArgumentsSlice(MArgumentsSlice* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->begin()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->end()->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LArgumentsSlice(
+ useRegisterAtStart(ins->object()), useRegisterAtStart(ins->begin()),
+ useRegisterAtStart(ins->end()), tempFixed(CallTempReg0),
+ tempFixed(CallTempReg1));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitFrameArgumentsSlice(MFrameArgumentsSlice* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+ MOZ_ASSERT(ins->begin()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->count()->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LFrameArgumentsSlice(
+ useRegister(ins->begin()), useRegister(ins->count()), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitInlineArgumentsSlice(MInlineArgumentsSlice* ins) {
+ LAllocation begin = useRegisterOrConstant(ins->begin());
+ LAllocation count = useRegisterOrConstant(ins->count());
+ uint32_t numActuals = ins->numActuals();
+ uint32_t numOperands =
+ numActuals * BOX_PIECES + LInlineArgumentsSlice::NumNonArgumentOperands;
+
+ auto* lir = allocateVariadic<LInlineArgumentsSlice>(numOperands, temp());
+ if (!lir) {
+ abort(AbortReason::Alloc, "OOM: LIRGenerator::visitInlineArgumentsSlice");
+ return;
+ }
+
+ lir->setOperand(LInlineArgumentsSlice::Begin, begin);
+ lir->setOperand(LInlineArgumentsSlice::Count, count);
+ for (uint32_t i = 0; i < numActuals; i++) {
+ MDefinition* arg = ins->getArg(i);
+ uint32_t index = LInlineArgumentsSlice::ArgIndex(i);
+ lir->setBoxOperand(index,
+ useBoxOrTypedOrConstant(arg, /*useConstant = */ true));
+ }
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNormalizeSliceTerm(MNormalizeSliceTerm* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->value()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->length()->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LNormalizeSliceTerm(useRegister(ins->value()),
+ useRegister(ins->length()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitArrayJoin(MArrayJoin* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::String);
+ MOZ_ASSERT(ins->array()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->sep()->type() == MIRType::String);
+
+ auto* lir = new (alloc())
+ LArrayJoin(useRegisterAtStart(ins->array()),
+ useRegisterAtStart(ins->sep()), tempFixed(CallTempReg0));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitStringSplit(MStringSplit* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ MOZ_ASSERT(ins->separator()->type() == MIRType::String);
+
+ LStringSplit* lir = new (alloc()) LStringSplit(
+ useRegisterAtStart(ins->string()), useRegisterAtStart(ins->separator()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitLoadUnboxedScalar(MLoadUnboxedScalar* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+ MOZ_ASSERT(IsNumericType(ins->type()) || ins->type() == MIRType::Boolean);
+
+ if (Scalar::isBigIntType(ins->storageType()) &&
+ ins->requiresMemoryBarrier()) {
+ lowerAtomicLoad64(ins);
+ return;
+ }
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index = useRegisterOrIndexConstant(
+ ins->index(), ins->storageType(), ins->offsetAdjustment());
+
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ Synchronization sync = Synchronization::Load();
+ if (ins->requiresMemoryBarrier()) {
+ LMemoryBarrier* fence = new (alloc()) LMemoryBarrier(sync.barrierBefore);
+ add(fence, ins);
+ }
+
+ if (!Scalar::isBigIntType(ins->storageType())) {
+ // We need a temp register for Uint32Array with known double result.
+ LDefinition tempDef = LDefinition::BogusTemp();
+ if (ins->storageType() == Scalar::Uint32 &&
+ IsFloatingPointType(ins->type())) {
+ tempDef = temp();
+ }
+
+ auto* lir = new (alloc()) LLoadUnboxedScalar(elements, index, tempDef);
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ define(lir, ins);
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::BigInt);
+
+ auto* lir =
+ new (alloc()) LLoadUnboxedBigInt(elements, index, temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ }
+
+ if (ins->requiresMemoryBarrier()) {
+ LMemoryBarrier* fence = new (alloc()) LMemoryBarrier(sync.barrierAfter);
+ add(fence, ins);
+ }
+}
+
+void LIRGenerator::visitLoadDataViewElement(MLoadDataViewElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ MOZ_ASSERT(IsNumericType(ins->type()));
+
+ const LUse elements = useRegister(ins->elements());
+ const LUse index = useRegister(ins->index());
+ const LAllocation littleEndian = useRegisterOrConstant(ins->littleEndian());
+
+ // We need a temp register for:
+ // - Uint32Array with known double result,
+ // - Float32Array,
+ // - and BigInt64Array and BigUint64Array.
+ LDefinition tempDef = LDefinition::BogusTemp();
+ if ((ins->storageType() == Scalar::Uint32 &&
+ IsFloatingPointType(ins->type())) ||
+ ins->storageType() == Scalar::Float32) {
+ tempDef = temp();
+ }
+ if (Scalar::isBigIntType(ins->storageType())) {
+#ifdef JS_CODEGEN_X86
+ // There are not enough registers on x86.
+ if (littleEndian.isConstant()) {
+ tempDef = temp();
+ }
+#else
+ tempDef = temp();
+#endif
+ }
+
+ // We also need a separate 64-bit temp register for:
+ // - Float64Array
+ // - and BigInt64Array and BigUint64Array.
+ LInt64Definition temp64Def = LInt64Definition::BogusTemp();
+ if (Scalar::byteSize(ins->storageType()) == 8) {
+ temp64Def = tempInt64();
+ }
+
+ auto* lir = new (alloc())
+ LLoadDataViewElement(elements, index, littleEndian, tempDef, temp64Def);
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ define(lir, ins);
+ if (Scalar::isBigIntType(ins->storageType())) {
+ assignSafepoint(lir, ins);
+ }
+}
+
+void LIRGenerator::visitClampToUint8(MClampToUint8* ins) {
+ MDefinition* in = ins->input();
+
+ switch (in->type()) {
+ case MIRType::Boolean:
+ redefine(ins, in);
+ break;
+
+ case MIRType::Int32:
+ defineReuseInput(new (alloc()) LClampIToUint8(useRegisterAtStart(in)),
+ ins, 0);
+ break;
+
+ case MIRType::Double:
+ // LClampDToUint8 clobbers its input register. Making it available as
+ // a temp copy describes this behavior to the register allocator.
+ define(new (alloc())
+ LClampDToUint8(useRegisterAtStart(in), tempCopy(in, 0)),
+ ins);
+ break;
+
+ case MIRType::Value: {
+ LClampVToUint8* lir =
+ new (alloc()) LClampVToUint8(useBox(in), tempDouble());
+ assignSnapshot(lir, ins->bailoutKind());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ break;
+ }
+
+ default:
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGenerator::visitLoadTypedArrayElementHole(
+ MLoadTypedArrayElementHole* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ const LUse object = useRegister(ins->object());
+ const LAllocation index = useRegister(ins->index());
+
+ if (!Scalar::isBigIntType(ins->arrayType())) {
+ auto* lir = new (alloc()) LLoadTypedArrayElementHole(object, index, temp());
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ defineBox(lir, ins);
+ } else {
+#ifdef JS_CODEGEN_X86
+ LDefinition tmp = LDefinition::BogusTemp();
+#else
+ LDefinition tmp = temp();
+#endif
+
+ auto* lir = new (alloc())
+ LLoadTypedArrayElementHoleBigInt(object, index, tmp, tempInt64());
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+ }
+}
+
+void LIRGenerator::visitStoreUnboxedScalar(MStoreUnboxedScalar* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (ins->isFloatWrite()) {
+ MOZ_ASSERT_IF(ins->writeType() == Scalar::Float32,
+ ins->value()->type() == MIRType::Float32);
+ MOZ_ASSERT_IF(ins->writeType() == Scalar::Float64,
+ ins->value()->type() == MIRType::Double);
+ } else if (ins->isBigIntWrite()) {
+ MOZ_ASSERT(ins->value()->type() == MIRType::BigInt);
+ } else {
+ MOZ_ASSERT(ins->value()->type() == MIRType::Int32);
+ }
+
+ if (ins->isBigIntWrite() && ins->requiresMemoryBarrier()) {
+ lowerAtomicStore64(ins);
+ return;
+ }
+
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value;
+
+ // For byte arrays, the value has to be in a byte register on x86.
+ if (ins->isByteWrite()) {
+ value = useByteOpRegisterOrNonDoubleConstant(ins->value());
+ } else if (ins->isBigIntWrite()) {
+ value = useRegister(ins->value());
+ } else {
+ value = useRegisterOrNonDoubleConstant(ins->value());
+ }
+
+ // Optimization opportunity for atomics: on some platforms there
+ // is a store instruction that incorporates the necessary
+ // barriers, and we could use that instead of separate barrier and
+ // store instructions. See bug #1077027.
+ //
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ Synchronization sync = Synchronization::Store();
+ if (ins->requiresMemoryBarrier()) {
+ LMemoryBarrier* fence = new (alloc()) LMemoryBarrier(sync.barrierBefore);
+ add(fence, ins);
+ }
+ if (!ins->isBigIntWrite()) {
+ add(new (alloc()) LStoreUnboxedScalar(elements, index, value), ins);
+ } else {
+ add(new (alloc()) LStoreUnboxedBigInt(elements, index, value, tempInt64()),
+ ins);
+ }
+ if (ins->requiresMemoryBarrier()) {
+ LMemoryBarrier* fence = new (alloc()) LMemoryBarrier(sync.barrierAfter);
+ add(fence, ins);
+ }
+}
+
+void LIRGenerator::visitStoreDataViewElement(MStoreDataViewElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+ MOZ_ASSERT(ins->littleEndian()->type() == MIRType::Boolean);
+
+ if (ins->isFloatWrite()) {
+ MOZ_ASSERT_IF(ins->writeType() == Scalar::Float32,
+ ins->value()->type() == MIRType::Float32);
+ MOZ_ASSERT_IF(ins->writeType() == Scalar::Float64,
+ ins->value()->type() == MIRType::Double);
+ } else if (ins->isBigIntWrite()) {
+ MOZ_ASSERT(ins->value()->type() == MIRType::BigInt);
+ } else {
+ MOZ_ASSERT(ins->value()->type() == MIRType::Int32);
+ }
+
+ LUse elements = useRegister(ins->elements());
+ LUse index = useRegister(ins->index());
+ LAllocation value;
+ if (ins->isBigIntWrite()) {
+ value = useRegister(ins->value());
+ } else {
+ value = useRegisterOrNonDoubleConstant(ins->value());
+ }
+ LAllocation littleEndian = useRegisterOrConstant(ins->littleEndian());
+
+ LDefinition tempDef = LDefinition::BogusTemp();
+ LInt64Definition temp64Def = LInt64Definition::BogusTemp();
+ if (Scalar::byteSize(ins->writeType()) < 8) {
+ tempDef = temp();
+ } else {
+ temp64Def = tempInt64();
+ }
+
+ add(new (alloc()) LStoreDataViewElement(elements, index, value, littleEndian,
+ tempDef, temp64Def),
+ ins);
+}
+
+void LIRGenerator::visitStoreTypedArrayElementHole(
+ MStoreTypedArrayElementHole* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+ MOZ_ASSERT(ins->length()->type() == MIRType::IntPtr);
+
+ if (ins->isFloatWrite()) {
+ MOZ_ASSERT_IF(ins->arrayType() == Scalar::Float32,
+ ins->value()->type() == MIRType::Float32);
+ MOZ_ASSERT_IF(ins->arrayType() == Scalar::Float64,
+ ins->value()->type() == MIRType::Double);
+ } else if (ins->isBigIntWrite()) {
+ MOZ_ASSERT(ins->value()->type() == MIRType::BigInt);
+ } else {
+ MOZ_ASSERT(ins->value()->type() == MIRType::Int32);
+ }
+
+ LUse elements = useRegister(ins->elements());
+ LAllocation length = useAny(ins->length());
+ LAllocation index = useRegister(ins->index());
+
+ // For byte arrays, the value has to be in a byte register on x86.
+ LAllocation value;
+ if (ins->isByteWrite()) {
+ value = useByteOpRegisterOrNonDoubleConstant(ins->value());
+ } else if (ins->isBigIntWrite()) {
+ value = useRegister(ins->value());
+ } else {
+ value = useRegisterOrNonDoubleConstant(ins->value());
+ }
+
+ if (!ins->isBigIntWrite()) {
+ LDefinition spectreTemp =
+ BoundsCheckNeedsSpectreTemp() ? temp() : LDefinition::BogusTemp();
+ auto* lir = new (alloc()) LStoreTypedArrayElementHole(
+ elements, length, index, value, spectreTemp);
+ add(lir, ins);
+ } else {
+ auto* lir = new (alloc()) LStoreTypedArrayElementHoleBigInt(
+ elements, length, index, value, tempInt64());
+ add(lir, ins);
+ }
+}
+
+void LIRGenerator::visitLoadFixedSlot(MLoadFixedSlot* ins) {
+ MDefinition* obj = ins->object();
+ MOZ_ASSERT(obj->type() == MIRType::Object);
+
+ MIRType type = ins->type();
+
+ if (type == MIRType::Value) {
+ LLoadFixedSlotV* lir =
+ new (alloc()) LLoadFixedSlotV(useRegisterAtStart(obj));
+ defineBox(lir, ins);
+ } else {
+ LLoadFixedSlotT* lir =
+ new (alloc()) LLoadFixedSlotT(useRegisterForTypedLoad(obj, type));
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitLoadFixedSlotAndUnbox(MLoadFixedSlotAndUnbox* ins) {
+ MDefinition* obj = ins->object();
+ MOZ_ASSERT(obj->type() == MIRType::Object);
+
+ LLoadFixedSlotAndUnbox* lir =
+ new (alloc()) LLoadFixedSlotAndUnbox(useRegisterAtStart(obj));
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ define(lir, ins);
+}
+
+void LIRGenerator::visitLoadDynamicSlotAndUnbox(MLoadDynamicSlotAndUnbox* ins) {
+ MDefinition* slots = ins->slots();
+ MOZ_ASSERT(slots->type() == MIRType::Slots);
+
+ auto* lir = new (alloc()) LLoadDynamicSlotAndUnbox(useRegisterAtStart(slots));
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ define(lir, ins);
+}
+
+void LIRGenerator::visitLoadElementAndUnbox(MLoadElementAndUnbox* ins) {
+ MDefinition* elements = ins->elements();
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ auto* lir = new (alloc())
+ LLoadElementAndUnbox(useRegister(elements), useRegisterOrConstant(index));
+ if (ins->fallible()) {
+ assignSnapshot(lir, ins->bailoutKind());
+ }
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAddAndStoreSlot(MAddAndStoreSlot* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ LDefinition maybeTemp = LDefinition::BogusTemp();
+ if (ins->kind() != MAddAndStoreSlot::Kind::FixedSlot) {
+ maybeTemp = temp();
+ }
+
+ auto* lir = new (alloc()) LAddAndStoreSlot(useRegister(ins->object()),
+ useBox(ins->value()), maybeTemp);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitAllocateAndStoreSlot(MAllocateAndStoreSlot* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LAllocateAndStoreSlot(
+ useRegisterAtStart(ins->object()), useBoxAtStart(ins->value()),
+ tempFixed(CallTempReg0), tempFixed(CallTempReg1));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+}
+
+void LIRGenerator::visitAddSlotAndCallAddPropHook(
+ MAddSlotAndCallAddPropHook* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->value()->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LAddSlotAndCallAddPropHook(
+ useRegisterAtStart(ins->object()), useBoxAtStart(ins->value()));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitStoreFixedSlot(MStoreFixedSlot* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ if (ins->value()->type() == MIRType::Value) {
+ LStoreFixedSlotV* lir = new (alloc())
+ LStoreFixedSlotV(useRegister(ins->object()), useBox(ins->value()));
+ add(lir, ins);
+ } else {
+ LStoreFixedSlotT* lir = new (alloc()) LStoreFixedSlotT(
+ useRegister(ins->object()), useRegisterOrConstant(ins->value()));
+ add(lir, ins);
+ }
+}
+
+void LIRGenerator::visitGetNameCache(MGetNameCache* ins) {
+ MOZ_ASSERT(ins->envObj()->type() == MIRType::Object);
+
+ // Emit an overrecursed check: this is necessary because the cache can
+ // attach a scripted getter stub that calls this script recursively.
+ gen->setNeedsOverrecursedCheck();
+
+ LGetNameCache* lir =
+ new (alloc()) LGetNameCache(useRegister(ins->envObj()), temp());
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCallGetIntrinsicValue(MCallGetIntrinsicValue* ins) {
+ LCallGetIntrinsicValue* lir = new (alloc()) LCallGetIntrinsicValue();
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGetPropSuperCache(MGetPropSuperCache* ins) {
+ MDefinition* obj = ins->object();
+ MDefinition* receiver = ins->receiver();
+ MDefinition* id = ins->idval();
+
+ gen->setNeedsOverrecursedCheck();
+
+ bool useConstId =
+ id->type() == MIRType::String || id->type() == MIRType::Symbol;
+
+ auto* lir = new (alloc())
+ LGetPropSuperCache(useRegister(obj), useBoxOrTyped(receiver),
+ useBoxOrTypedOrConstant(id, useConstId));
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGetPropertyCache(MGetPropertyCache* ins) {
+ MDefinition* value = ins->value();
+ MOZ_ASSERT(value->type() == MIRType::Object ||
+ value->type() == MIRType::Value);
+
+ MDefinition* id = ins->idval();
+ MOZ_ASSERT(id->type() == MIRType::String || id->type() == MIRType::Symbol ||
+ id->type() == MIRType::Int32 || id->type() == MIRType::Value);
+
+ // Emit an overrecursed check: this is necessary because the cache can
+ // attach a scripted getter stub that calls this script recursively.
+ gen->setNeedsOverrecursedCheck();
+
+ // If this is a GetProp, the id is a constant string. Allow passing it as a
+ // constant to reduce register allocation pressure.
+ bool useConstId =
+ id->type() == MIRType::String || id->type() == MIRType::Symbol;
+
+ auto* lir = new (alloc()) LGetPropertyCache(
+ useBoxOrTyped(value), useBoxOrTypedOrConstant(id, useConstId));
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBindNameCache(MBindNameCache* ins) {
+ MOZ_ASSERT(ins->envChain()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ LBindNameCache* lir =
+ new (alloc()) LBindNameCache(useRegister(ins->envChain()), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCallBindVar(MCallBindVar* ins) {
+ MOZ_ASSERT(ins->environmentChain()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ LCallBindVar* lir =
+ new (alloc()) LCallBindVar(useRegister(ins->environmentChain()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitGuardObjectIdentity(MGuardObjectIdentity* ins) {
+ LGuardObjectIdentity* guard = new (alloc()) LGuardObjectIdentity(
+ useRegister(ins->object()), useRegister(ins->expected()));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardSpecificFunction(MGuardSpecificFunction* ins) {
+ auto* guard = new (alloc()) LGuardSpecificFunction(
+ useRegister(ins->function()), useRegister(ins->expected()));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, ins->function());
+}
+
+void LIRGenerator::visitGuardSpecificAtom(MGuardSpecificAtom* ins) {
+ auto* guard =
+ new (alloc()) LGuardSpecificAtom(useRegister(ins->str()), temp());
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, ins->str());
+ assignSafepoint(guard, ins);
+}
+
+void LIRGenerator::visitGuardSpecificSymbol(MGuardSpecificSymbol* ins) {
+ auto* guard = new (alloc()) LGuardSpecificSymbol(useRegister(ins->symbol()));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, ins->symbol());
+}
+
+void LIRGenerator::visitGuardSpecificInt32(MGuardSpecificInt32* ins) {
+ auto* guard = new (alloc()) LGuardSpecificInt32(useRegister(ins->num()));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, ins->num());
+}
+
+void LIRGenerator::visitGuardStringToIndex(MGuardStringToIndex* ins) {
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ auto* guard = new (alloc()) LGuardStringToIndex(useRegister(ins->string()));
+ assignSnapshot(guard, ins->bailoutKind());
+ define(guard, ins);
+ assignSafepoint(guard, ins);
+}
+
+void LIRGenerator::visitGuardStringToInt32(MGuardStringToInt32* ins) {
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ auto* guard =
+ new (alloc()) LGuardStringToInt32(useRegister(ins->string()), temp());
+ assignSnapshot(guard, ins->bailoutKind());
+ define(guard, ins);
+ assignSafepoint(guard, ins);
+}
+
+void LIRGenerator::visitGuardStringToDouble(MGuardStringToDouble* ins) {
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ auto* guard = new (alloc())
+ LGuardStringToDouble(useRegister(ins->string()), temp(), temp());
+ assignSnapshot(guard, ins->bailoutKind());
+ define(guard, ins);
+ assignSafepoint(guard, ins);
+}
+
+void LIRGenerator::visitGuardNoDenseElements(MGuardNoDenseElements* ins) {
+ auto* guard =
+ new (alloc()) LGuardNoDenseElements(useRegister(ins->object()), temp());
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardShape(MGuardShape* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ if (JitOptions.spectreObjectMitigations) {
+ auto* lir =
+ new (alloc()) LGuardShape(useRegisterAtStart(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReuseInput(lir, ins, 0);
+ } else {
+ auto* lir = new (alloc())
+ LGuardShape(useRegister(ins->object()), LDefinition::BogusTemp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+ }
+}
+
+void LIRGenerator::visitGuardMultipleShapes(MGuardMultipleShapes* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ if (JitOptions.spectreObjectMitigations) {
+ auto* lir = new (alloc()) LGuardMultipleShapes(
+ useRegisterAtStart(ins->object()), useRegister(ins->shapeList()),
+ temp(), temp(), temp(), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReuseInput(lir, ins, 0);
+ } else {
+ auto* lir = new (alloc()) LGuardMultipleShapes(
+ useRegister(ins->object()), useRegister(ins->shapeList()), temp(),
+ temp(), temp(), LDefinition::BogusTemp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+ }
+}
+
+void LIRGenerator::visitGuardProto(MGuardProto* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->expected()->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LGuardProto(useRegister(ins->object()),
+ useRegister(ins->expected()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardNullProto(MGuardNullProto* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LGuardNullProto(useRegister(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardIsNativeObject(MGuardIsNativeObject* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LGuardIsNativeObject(useRegister(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardIsProxy(MGuardIsProxy* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LGuardIsProxy(useRegister(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardIsNotProxy(MGuardIsNotProxy* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LGuardIsNotProxy(useRegister(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardIsNotDOMProxy(MGuardIsNotDOMProxy* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LGuardIsNotDOMProxy(useRegister(ins->proxy()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->proxy());
+}
+
+void LIRGenerator::visitProxyGet(MProxyGet* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+ auto* lir = new (alloc())
+ LProxyGet(useRegisterAtStart(ins->proxy()), tempFixed(CallTempReg0));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitProxyGetByValue(MProxyGetByValue* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->idVal()->type() == MIRType::Value);
+ auto* lir = new (alloc()) LProxyGetByValue(useRegisterAtStart(ins->proxy()),
+ useBoxAtStart(ins->idVal()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitProxyHasProp(MProxyHasProp* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->idVal()->type() == MIRType::Value);
+ auto* lir = new (alloc()) LProxyHasProp(useRegisterAtStart(ins->proxy()),
+ useBoxAtStart(ins->idVal()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitProxySet(MProxySet* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->rhs()->type() == MIRType::Value);
+ auto* lir = new (alloc())
+ LProxySet(useRegisterAtStart(ins->proxy()), useBoxAtStart(ins->rhs()),
+ tempFixed(CallTempReg0));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitProxySetByValue(MProxySetByValue* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->idVal()->type() == MIRType::Value);
+ MOZ_ASSERT(ins->rhs()->type() == MIRType::Value);
+ auto* lir = new (alloc())
+ LProxySetByValue(useRegisterAtStart(ins->proxy()),
+ useBoxAtStart(ins->idVal()), useBoxAtStart(ins->rhs()));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCallSetArrayLength(MCallSetArrayLength* ins) {
+ MOZ_ASSERT(ins->obj()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->rhs()->type() == MIRType::Value);
+ auto* lir = new (alloc()) LCallSetArrayLength(useRegisterAtStart(ins->obj()),
+ useBoxAtStart(ins->rhs()));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitMegamorphicLoadSlot(MMegamorphicLoadSlot* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ auto* lir = new (alloc()) LMegamorphicLoadSlot(
+ useRegisterAtStart(ins->object()), tempFixed(CallTempReg0),
+ tempFixed(CallTempReg1), tempFixed(CallTempReg2));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitMegamorphicLoadSlotByValue(
+ MMegamorphicLoadSlotByValue* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->idVal()->type() == MIRType::Value);
+ auto* lir = new (alloc()) LMegamorphicLoadSlotByValue(
+ useRegisterAtStart(ins->object()), useBoxAtStart(ins->idVal()),
+ tempFixed(CallTempReg0), tempFixed(CallTempReg1),
+ tempFixed(CallTempReg2));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitMegamorphicStoreSlot(MMegamorphicStoreSlot* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->rhs()->type() == MIRType::Value);
+ auto* lir = new (alloc())
+ LMegamorphicStoreSlot(useRegisterAtStart(ins->object()),
+ useBoxAtStart(ins->rhs()), tempFixed(CallTempReg0),
+ tempFixed(CallTempReg1), tempFixed(CallTempReg2));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+}
+
+void LIRGenerator::visitMegamorphicHasProp(MMegamorphicHasProp* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->idVal()->type() == MIRType::Value);
+ auto* lir = new (alloc())
+ LMegamorphicHasProp(useRegisterAtStart(ins->object()),
+ useBoxAtStart(ins->idVal()), tempFixed(CallTempReg0),
+ tempFixed(CallTempReg1), tempFixed(CallTempReg2));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitGuardIsNotArrayBufferMaybeShared(
+ MGuardIsNotArrayBufferMaybeShared* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ auto* lir = new (alloc())
+ LGuardIsNotArrayBufferMaybeShared(useRegister(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitGuardIsTypedArray(MGuardIsTypedArray* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LGuardIsTypedArray(useRegister(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->object());
+}
+
+void LIRGenerator::visitNurseryObject(MNurseryObject* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LNurseryObject();
+ define(lir, ins);
+}
+
+void LIRGenerator::visitGuardValue(MGuardValue* ins) {
+ MOZ_ASSERT(ins->value()->type() == MIRType::Value);
+ auto* lir = new (alloc()) LGuardValue(useBox(ins->value()));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->value());
+}
+
+void LIRGenerator::visitGuardNullOrUndefined(MGuardNullOrUndefined* ins) {
+ MOZ_ASSERT(ins->value()->type() == MIRType::Value);
+ auto* lir = new (alloc()) LGuardNullOrUndefined(useBox(ins->value()));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->value());
+}
+
+void LIRGenerator::visitGuardIsNotObject(MGuardIsNotObject* ins) {
+ MOZ_ASSERT(ins->value()->type() == MIRType::Value);
+ auto* lir = new (alloc()) LGuardIsNotObject(useBox(ins->value()));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->value());
+}
+
+void LIRGenerator::visitGuardFunctionFlags(MGuardFunctionFlags* ins) {
+ MOZ_ASSERT(ins->function()->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LGuardFunctionFlags(useRegister(ins->function()));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->function());
+}
+
+void LIRGenerator::visitGuardFunctionIsNonBuiltinCtor(
+ MGuardFunctionIsNonBuiltinCtor* ins) {
+ MOZ_ASSERT(ins->function()->type() == MIRType::Object);
+
+ auto* lir = new (alloc())
+ LGuardFunctionIsNonBuiltinCtor(useRegister(ins->function()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->function());
+}
+
+void LIRGenerator::visitGuardFunctionKind(MGuardFunctionKind* ins) {
+ MOZ_ASSERT(ins->function()->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LGuardFunctionKind(useRegister(ins->function()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->function());
+}
+
+void LIRGenerator::visitGuardFunctionScript(MGuardFunctionScript* ins) {
+ MOZ_ASSERT(ins->function()->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LGuardFunctionScript(useRegister(ins->function()));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->function());
+}
+
+void LIRGenerator::visitAssertRange(MAssertRange* ins) {
+ MDefinition* input = ins->input();
+ LInstruction* lir = nullptr;
+
+ switch (input->type()) {
+ case MIRType::Boolean:
+ case MIRType::Int32:
+ case MIRType::IntPtr:
+ lir = new (alloc()) LAssertRangeI(useRegisterAtStart(input));
+ break;
+
+ case MIRType::Double:
+ lir = new (alloc()) LAssertRangeD(useRegister(input), tempDouble());
+ break;
+
+ case MIRType::Float32:
+ lir = new (alloc())
+ LAssertRangeF(useRegister(input), tempDouble(), tempDouble());
+ break;
+
+ case MIRType::Value:
+ lir = new (alloc()) LAssertRangeV(useBox(input), tempToUnbox(),
+ tempDouble(), tempDouble());
+ break;
+
+ default:
+ MOZ_CRASH("Unexpected Range for MIRType");
+ break;
+ }
+
+ lir->setMir(ins);
+ add(lir);
+}
+
+void LIRGenerator::visitAssertClass(MAssertClass* ins) {
+ auto* lir =
+ new (alloc()) LAssertClass(useRegisterAtStart(ins->input()), temp());
+ add(lir, ins);
+}
+
+void LIRGenerator::visitAssertShape(MAssertShape* ins) {
+ auto* lir = new (alloc()) LAssertShape(useRegisterAtStart(ins->input()));
+ add(lir, ins);
+}
+
+void LIRGenerator::visitDeleteProperty(MDeleteProperty* ins) {
+ LCallDeleteProperty* lir =
+ new (alloc()) LCallDeleteProperty(useBoxAtStart(ins->value()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitDeleteElement(MDeleteElement* ins) {
+ LCallDeleteElement* lir = new (alloc()) LCallDeleteElement(
+ useBoxAtStart(ins->value()), useBoxAtStart(ins->index()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitObjectToIterator(MObjectToIterator* ins) {
+ auto* lir = new (alloc())
+ LObjectToIterator(useRegister(ins->object()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitValueToIterator(MValueToIterator* ins) {
+ auto* lir = new (alloc()) LValueToIterator(useBoxAtStart(ins->value()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitSetPropertyCache(MSetPropertyCache* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+
+ MDefinition* id = ins->idval();
+ MOZ_ASSERT(id->type() == MIRType::String || id->type() == MIRType::Symbol ||
+ id->type() == MIRType::Int32 || id->type() == MIRType::Value);
+
+ // If this is a SetProp, the id is a constant string. Allow passing it as a
+ // constant to reduce register allocation pressure.
+ bool useConstId =
+ id->type() == MIRType::String || id->type() == MIRType::Symbol;
+ bool useConstValue = IsNonNurseryConstant(ins->value());
+
+ // Emit an overrecursed check: this is necessary because the cache can
+ // attach a scripted setter stub that calls this script recursively.
+ gen->setNeedsOverrecursedCheck();
+
+ // We need a double temp register for TypedArray or TypedObject stubs.
+ LDefinition tempD = tempFixed(FloatReg0);
+
+ LInstruction* lir = new (alloc()) LSetPropertyCache(
+ useRegister(ins->object()), useBoxOrTypedOrConstant(id, useConstId),
+ useBoxOrTypedOrConstant(ins->value(), useConstValue), temp(), tempD);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitMegamorphicSetElement(MMegamorphicSetElement* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->index()->type() == MIRType::Value);
+ MOZ_ASSERT(ins->value()->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LMegamorphicSetElement(
+ useRegisterAtStart(ins->object()), useBoxAtStart(ins->index()),
+ useBoxAtStart(ins->value()));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGetIteratorCache(MGetIteratorCache* ins) {
+ MDefinition* value = ins->value();
+ MOZ_ASSERT(value->type() == MIRType::Object ||
+ value->type() == MIRType::Value);
+
+ LGetIteratorCache* lir =
+ new (alloc()) LGetIteratorCache(useBoxOrTyped(value), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitOptimizeSpreadCallCache(MOptimizeSpreadCallCache* ins) {
+ MDefinition* value = ins->value();
+ MOZ_ASSERT(value->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LOptimizeSpreadCallCache(useBox(value), temp());
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitIteratorMore(MIteratorMore* ins) {
+ LIteratorMore* lir =
+ new (alloc()) LIteratorMore(useRegister(ins->iterator()), temp());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitIsNoIter(MIsNoIter* ins) {
+ MOZ_ASSERT(ins->hasOneUse());
+ emitAtUses(ins);
+}
+
+void LIRGenerator::visitIteratorEnd(MIteratorEnd* ins) {
+ LIteratorEnd* lir = new (alloc())
+ LIteratorEnd(useRegister(ins->iterator()), temp(), temp(), temp());
+ add(lir, ins);
+}
+
+void LIRGenerator::visitCloseIterCache(MCloseIterCache* ins) {
+ LCloseIterCache* lir =
+ new (alloc()) LCloseIterCache(useRegister(ins->iter()), temp());
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitStringLength(MStringLength* ins) {
+ MOZ_ASSERT(ins->string()->type() == MIRType::String);
+ define(new (alloc()) LStringLength(useRegisterAtStart(ins->string())), ins);
+}
+
+void LIRGenerator::visitArgumentsLength(MArgumentsLength* ins) {
+ define(new (alloc()) LArgumentsLength(), ins);
+}
+
+void LIRGenerator::visitGetFrameArgument(MGetFrameArgument* ins) {
+ LGetFrameArgument* lir =
+ new (alloc()) LGetFrameArgument(useRegisterOrConstant(ins->index()));
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitGetFrameArgumentHole(MGetFrameArgumentHole* ins) {
+ LDefinition spectreTemp =
+ BoundsCheckNeedsSpectreTemp() ? temp() : LDefinition::BogusTemp();
+
+ auto* lir = new (alloc()) LGetFrameArgumentHole(
+ useRegister(ins->index()), useRegister(ins->length()), spectreTemp);
+ assignSnapshot(lir, ins->bailoutKind());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitNewTarget(MNewTarget* ins) {
+ LNewTarget* lir = new (alloc()) LNewTarget();
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitRest(MRest* ins) {
+ MOZ_ASSERT(ins->numActuals()->type() == MIRType::Int32);
+
+ LRest* lir = new (alloc())
+ LRest(useRegisterAtStart(ins->numActuals()), tempFixed(CallTempReg0),
+ tempFixed(CallTempReg1), tempFixed(CallTempReg2));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitThrow(MThrow* ins) {
+ MDefinition* value = ins->getOperand(0);
+ MOZ_ASSERT(value->type() == MIRType::Value);
+
+ LThrow* lir = new (alloc()) LThrow(useBoxAtStart(value));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitInCache(MInCache* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::String || lhs->type() == MIRType::Symbol ||
+ lhs->type() == MIRType::Int32 || lhs->type() == MIRType::Value);
+ MOZ_ASSERT(rhs->type() == MIRType::Object);
+
+ LInCache* lir =
+ new (alloc()) LInCache(useBoxOrTyped(lhs), useRegister(rhs), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitHasOwnCache(MHasOwnCache* ins) {
+ MDefinition* value = ins->value();
+ MOZ_ASSERT(value->type() == MIRType::Object ||
+ value->type() == MIRType::Value);
+
+ MDefinition* id = ins->idval();
+ MOZ_ASSERT(id->type() == MIRType::String || id->type() == MIRType::Symbol ||
+ id->type() == MIRType::Int32 || id->type() == MIRType::Value);
+
+ // Emit an overrecursed check: this is necessary because the cache can
+ // attach a scripted getter stub that calls this script recursively.
+ gen->setNeedsOverrecursedCheck();
+
+ LHasOwnCache* lir =
+ new (alloc()) LHasOwnCache(useBoxOrTyped(value), useBoxOrTyped(id));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCheckPrivateFieldCache(MCheckPrivateFieldCache* ins) {
+ MDefinition* value = ins->value();
+ MOZ_ASSERT(value->type() == MIRType::Object ||
+ value->type() == MIRType::Value);
+
+ MDefinition* id = ins->idval();
+ MOZ_ASSERT(id->type() == MIRType::String || id->type() == MIRType::Symbol ||
+ id->type() == MIRType::Int32 || id->type() == MIRType::Value);
+
+ LCheckPrivateFieldCache* lir = new (alloc())
+ LCheckPrivateFieldCache(useBoxOrTyped(value), useBoxOrTyped(id));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitNewPrivateName(MNewPrivateName* ins) {
+ auto* lir = new (alloc()) LNewPrivateName();
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitInstanceOf(MInstanceOf* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Value || lhs->type() == MIRType::Object);
+ MOZ_ASSERT(rhs->type() == MIRType::Object);
+
+ if (lhs->type() == MIRType::Object) {
+ auto* lir = new (alloc()) LInstanceOfO(useRegister(lhs), useRegister(rhs));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ } else {
+ auto* lir = new (alloc()) LInstanceOfV(useBox(lhs), useRegister(rhs));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ }
+}
+
+void LIRGenerator::visitInstanceOfCache(MInstanceOfCache* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Value);
+ MOZ_ASSERT(rhs->type() == MIRType::Object);
+
+ LInstanceOfCache* lir =
+ new (alloc()) LInstanceOfCache(useBox(lhs), useRegister(rhs));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitIsArray(MIsArray* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+
+ if (ins->value()->type() == MIRType::Object) {
+ LIsArrayO* lir = new (alloc()) LIsArrayO(useRegister(ins->value()));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ } else {
+ MOZ_ASSERT(ins->value()->type() == MIRType::Value);
+ LIsArrayV* lir = new (alloc()) LIsArrayV(useBox(ins->value()), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ }
+}
+
+void LIRGenerator::visitIsTypedArray(MIsTypedArray* ins) {
+ MOZ_ASSERT(ins->value()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+
+ auto* lir = new (alloc()) LIsTypedArray(useRegister(ins->value()));
+ define(lir, ins);
+
+ if (ins->isPossiblyWrapped()) {
+ assignSafepoint(lir, ins);
+ }
+}
+
+void LIRGenerator::visitIsCallable(MIsCallable* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+
+ if (ins->object()->type() == MIRType::Object) {
+ define(new (alloc()) LIsCallableO(useRegister(ins->object())), ins);
+ } else {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Value);
+ define(new (alloc()) LIsCallableV(useBox(ins->object()), temp()), ins);
+ }
+}
+
+void LIRGenerator::visitIsConstructor(MIsConstructor* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+ define(new (alloc()) LIsConstructor(useRegister(ins->object())), ins);
+}
+
+void LIRGenerator::visitIsCrossRealmArrayConstructor(
+ MIsCrossRealmArrayConstructor* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+ define(new (alloc())
+ LIsCrossRealmArrayConstructor(useRegister(ins->object())),
+ ins);
+}
+
+static bool CanEmitAtUseForSingleTest(MInstruction* ins) {
+ if (!ins->canEmitAtUses()) {
+ return false;
+ }
+
+ MUseIterator iter(ins->usesBegin());
+ if (iter == ins->usesEnd()) {
+ return false;
+ }
+
+ MNode* node = iter->consumer();
+ if (!node->isDefinition()) {
+ return false;
+ }
+
+ if (!node->toDefinition()->isTest()) {
+ return false;
+ }
+
+ iter++;
+ return iter == ins->usesEnd();
+}
+
+void LIRGenerator::visitIsObject(MIsObject* ins) {
+ if (CanEmitAtUseForSingleTest(ins)) {
+ emitAtUses(ins);
+ return;
+ }
+
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+ LIsObject* lir = new (alloc()) LIsObject(useBoxAtStart(opd));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitIsNullOrUndefined(MIsNullOrUndefined* ins) {
+ if (CanEmitAtUseForSingleTest(ins)) {
+ emitAtUses(ins);
+ return;
+ }
+
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+ LIsNullOrUndefined* lir =
+ new (alloc()) LIsNullOrUndefined(useBoxAtStart(opd));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitHasClass(MHasClass* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+ define(new (alloc()) LHasClass(useRegister(ins->object())), ins);
+}
+
+void LIRGenerator::visitGuardToClass(MGuardToClass* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+ LGuardToClass* lir =
+ new (alloc()) LGuardToClass(useRegisterAtStart(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitGuardToFunction(MGuardToFunction* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+ LGuardToFunction* lir =
+ new (alloc()) LGuardToFunction(useRegisterAtStart(ins->object()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitObjectClassToString(MObjectClassToString* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::String);
+ auto* lir = new (alloc()) LObjectClassToString(
+ useRegisterAtStart(ins->object()), tempFixed(CallTempReg0));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitWasmAddOffset(MWasmAddOffset* ins) {
+ MOZ_ASSERT(ins->offset());
+ if (ins->base()->type() == MIRType::Int32) {
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->offset() <= UINT32_MAX); // Because memory32
+ define(new (alloc()) LWasmAddOffset(useRegisterAtStart(ins->base())), ins);
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Int64);
+#ifdef JS_64BIT
+ defineInt64(new (alloc())
+ LWasmAddOffset64(useInt64RegisterAtStart(ins->base())),
+ ins);
+#else
+ // Avoid situation where the input is (a,b) and the output is (b,a).
+ defineInt64ReuseInput(
+ new (alloc()) LWasmAddOffset64(useInt64RegisterAtStart(ins->base())),
+ ins, 0);
+#endif
+ }
+}
+
+void LIRGenerator::visitWasmLoadInstance(MWasmLoadInstance* ins) {
+ if (ins->type() == MIRType::Int64) {
+#ifdef JS_PUNBOX64
+ LAllocation instance = useRegisterAtStart(ins->instance());
+#else
+ // Avoid reusing instance for a 64-bit output pair as the load clobbers the
+ // first half of that pair before loading the second half.
+ LAllocation instance = useRegister(ins->instance());
+#endif
+ auto* lir = new (alloc()) LWasmLoadInstance64(instance);
+ defineInt64(lir, ins);
+ } else {
+ auto* lir =
+ new (alloc()) LWasmLoadInstance(useRegisterAtStart(ins->instance()));
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitWasmStoreInstance(MWasmStoreInstance* ins) {
+ MDefinition* value = ins->value();
+ if (value->type() == MIRType::Int64) {
+#ifdef JS_PUNBOX64
+ LAllocation instance = useRegisterAtStart(ins->instance());
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+#else
+ LAllocation instance = useRegister(ins->instance());
+ LInt64Allocation valueAlloc = useInt64Register(value);
+#endif
+ add(new (alloc()) LWasmStoreSlotI64(valueAlloc, instance, ins->offset(),
+ mozilla::Nothing()),
+ ins);
+ } else {
+ MOZ_ASSERT(value->type() != MIRType::RefOrNull);
+ LAllocation instance = useRegisterAtStart(ins->instance());
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ add(new (alloc())
+ LWasmStoreSlot(valueAlloc, instance, ins->offset(), value->type(),
+ MNarrowingOp::None, mozilla::Nothing()),
+ ins);
+ }
+}
+
+void LIRGenerator::visitWasmBoundsCheck(MWasmBoundsCheck* ins) {
+ MOZ_ASSERT(!ins->isRedundant());
+
+ MDefinition* index = ins->index();
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+
+ MOZ_ASSERT(boundsCheckLimit->type() == index->type());
+
+ if (index->type() == MIRType::Int64) {
+ if (JitOptions.spectreIndexMasking) {
+ auto* lir = new (alloc()) LWasmBoundsCheck64(
+ useInt64RegisterAtStart(index), useInt64Register(boundsCheckLimit));
+ defineInt64ReuseInput(lir, ins, 0);
+ } else {
+ auto* lir = new (alloc())
+ LWasmBoundsCheck64(useInt64RegisterAtStart(index),
+ useInt64RegisterAtStart(boundsCheckLimit));
+ add(lir, ins);
+ }
+ } else {
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ if (JitOptions.spectreIndexMasking) {
+ auto* lir = new (alloc()) LWasmBoundsCheck(useRegisterAtStart(index),
+ useRegister(boundsCheckLimit));
+ defineReuseInput(lir, ins, 0);
+ } else {
+ auto* lir = new (alloc()) LWasmBoundsCheck(
+ useRegisterAtStart(index), useRegisterAtStart(boundsCheckLimit));
+ add(lir, ins);
+ }
+ }
+}
+
+void LIRGenerator::visitWasmAlignmentCheck(MWasmAlignmentCheck* ins) {
+ MDefinition* index = ins->index();
+ if (index->type() == MIRType::Int64) {
+ auto* lir =
+ new (alloc()) LWasmAlignmentCheck64(useInt64RegisterAtStart(index));
+ add(lir, ins);
+ } else {
+ auto* lir = new (alloc()) LWasmAlignmentCheck(useRegisterAtStart(index));
+ add(lir, ins);
+ }
+}
+
+void LIRGenerator::visitWasmLoadGlobalVar(MWasmLoadGlobalVar* ins) {
+ size_t offs = wasm::Instance::offsetOfGlobalArea() + ins->globalDataOffset();
+ if (ins->type() == MIRType::Int64) {
+#ifdef JS_PUNBOX64
+ LAllocation instance = useRegisterAtStart(ins->instance());
+#else
+ // Avoid reusing instance for the output pair as the load clobbers the first
+ // half of that pair before loading the second half.
+ LAllocation instance = useRegister(ins->instance());
+#endif
+ defineInt64(new (alloc())
+ LWasmLoadSlotI64(instance, offs, mozilla::Nothing()),
+ ins);
+ } else {
+ LAllocation instance = useRegisterAtStart(ins->instance());
+ define(new (alloc()) LWasmLoadSlot(instance, offs, ins->type(),
+ MWideningOp::None, mozilla::Nothing()),
+ ins);
+ }
+}
+
+void LIRGenerator::visitWasmLoadGlobalCell(MWasmLoadGlobalCell* ins) {
+ if (ins->type() == MIRType::Int64) {
+#ifdef JS_PUNBOX64
+ LAllocation cellPtr = useRegisterAtStart(ins->cellPtr());
+#else
+ // Avoid reusing cellPtr for the output pair as the load clobbers the first
+ // half of that pair before loading the second half.
+ LAllocation cellPtr = useRegister(ins->cellPtr());
+#endif
+ defineInt64(new (alloc())
+ LWasmLoadSlotI64(cellPtr, /*offset=*/0, mozilla::Nothing()),
+ ins);
+ } else {
+ LAllocation cellPtr = useRegisterAtStart(ins->cellPtr());
+ define(new (alloc()) LWasmLoadSlot(cellPtr, /*offset=*/0, ins->type(),
+ MWideningOp::None, mozilla::Nothing()),
+ ins);
+ }
+}
+
+void LIRGenerator::visitWasmLoadTableElement(MWasmLoadTableElement* ins) {
+ LAllocation elements = useRegisterAtStart(ins->elements());
+ LAllocation index = useRegisterAtStart(ins->index());
+ define(new (alloc()) LWasmLoadTableElement(elements, index), ins);
+}
+
+void LIRGenerator::visitWasmStoreGlobalVar(MWasmStoreGlobalVar* ins) {
+ MDefinition* value = ins->value();
+ size_t offs = wasm::Instance::offsetOfGlobalArea() + ins->globalDataOffset();
+ if (value->type() == MIRType::Int64) {
+#ifdef JS_PUNBOX64
+ LAllocation instance = useRegisterAtStart(ins->instance());
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+#else
+ LAllocation instance = useRegister(ins->instance());
+ LInt64Allocation valueAlloc = useInt64Register(value);
+#endif
+ add(new (alloc())
+ LWasmStoreSlotI64(valueAlloc, instance, offs, mozilla::Nothing()),
+ ins);
+ } else {
+ MOZ_ASSERT(value->type() != MIRType::RefOrNull);
+ LAllocation instance = useRegisterAtStart(ins->instance());
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ add(new (alloc()) LWasmStoreSlot(valueAlloc, instance, offs, value->type(),
+ MNarrowingOp::None, mozilla::Nothing()),
+ ins);
+ }
+}
+
+void LIRGenerator::visitWasmStoreGlobalCell(MWasmStoreGlobalCell* ins) {
+ MDefinition* value = ins->value();
+ size_t offs = 0;
+ if (value->type() == MIRType::Int64) {
+#ifdef JS_PUNBOX64
+ LAllocation cellPtr = useRegisterAtStart(ins->cellPtr());
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+#else
+ LAllocation cellPtr = useRegister(ins->cellPtr());
+ LInt64Allocation valueAlloc = useInt64Register(value);
+#endif
+ add(new (alloc())
+ LWasmStoreSlotI64(valueAlloc, cellPtr, offs, mozilla::Nothing()));
+ } else {
+ MOZ_ASSERT(value->type() != MIRType::RefOrNull);
+ LAllocation cellPtr = useRegisterAtStart(ins->cellPtr());
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ add(new (alloc()) LWasmStoreSlot(valueAlloc, cellPtr, offs, value->type(),
+ MNarrowingOp::None, mozilla::Nothing()));
+ }
+}
+
+void LIRGenerator::visitWasmStoreStackResult(MWasmStoreStackResult* ins) {
+ MDefinition* stackResultArea = ins->stackResultArea();
+ MDefinition* value = ins->value();
+ size_t offs = ins->offset();
+ LInstruction* lir;
+ if (value->type() == MIRType::Int64) {
+ lir = new (alloc())
+ LWasmStoreSlotI64(useInt64Register(value), useRegister(stackResultArea),
+ offs, mozilla::Nothing());
+ } else {
+ MOZ_ASSERT(value->type() != MIRType::RefOrNull);
+ lir = new (alloc())
+ LWasmStoreSlot(useRegister(value), useRegister(stackResultArea), offs,
+ value->type(), MNarrowingOp::None, mozilla::Nothing());
+ }
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmDerivedPointer(MWasmDerivedPointer* ins) {
+ LAllocation base = useRegisterAtStart(ins->base());
+ define(new (alloc()) LWasmDerivedPointer(base), ins);
+}
+
+void LIRGenerator::visitWasmDerivedIndexPointer(MWasmDerivedIndexPointer* ins) {
+ LAllocation base = useRegisterAtStart(ins->base());
+ LAllocation index = useRegisterAtStart(ins->index());
+ define(new (alloc()) LWasmDerivedIndexPointer(base, index), ins);
+}
+
+void LIRGenerator::visitWasmStoreRef(MWasmStoreRef* ins) {
+ LAllocation instance = useRegister(ins->instance());
+ LAllocation valueAddr = useFixed(ins->valueAddr(), PreBarrierReg);
+ LAllocation value = useRegister(ins->value());
+ add(new (alloc()) LWasmStoreRef(instance, valueAddr, value, temp()), ins);
+}
+
+void LIRGenerator::visitWasmParameter(MWasmParameter* ins) {
+ ABIArg abi = ins->abi();
+ if (ins->type() == MIRType::StackResults) {
+ // Functions that return stack results receive an extra incoming parameter
+ // with type MIRType::StackResults. This value is a pointer to fresh
+ // memory. Here we treat it as if it were in fact MIRType::Pointer.
+ auto* lir = new (alloc()) LWasmParameter;
+ LDefinition def(LDefinition::TypeFrom(MIRType::Pointer),
+ LDefinition::FIXED);
+ def.setOutput(abi.argInRegister() ? LAllocation(abi.reg())
+ : LArgument(abi.offsetFromArgBase()));
+ define(lir, ins, def);
+ return;
+ }
+ if (abi.argInRegister()) {
+#if defined(JS_NUNBOX32)
+ if (abi.isGeneralRegPair()) {
+ defineInt64Fixed(
+ new (alloc()) LWasmParameterI64, ins,
+ LInt64Allocation(LAllocation(AnyRegister(abi.gpr64().high)),
+ LAllocation(AnyRegister(abi.gpr64().low))));
+ return;
+ }
+#endif
+ defineFixed(new (alloc()) LWasmParameter, ins, LAllocation(abi.reg()));
+ return;
+ }
+ if (ins->type() == MIRType::Int64) {
+ MOZ_ASSERT(!abi.argInRegister());
+ defineInt64Fixed(
+ new (alloc()) LWasmParameterI64, ins,
+#if defined(JS_NUNBOX32)
+ LInt64Allocation(LArgument(abi.offsetFromArgBase() + INT64HIGH_OFFSET),
+ LArgument(abi.offsetFromArgBase() + INT64LOW_OFFSET))
+#else
+ LInt64Allocation(LArgument(abi.offsetFromArgBase()))
+#endif
+ );
+ } else {
+ MOZ_ASSERT(IsNumberType(ins->type()) || ins->type() == MIRType::RefOrNull
+#ifdef ENABLE_WASM_SIMD
+ || ins->type() == MIRType::Simd128
+#endif
+ );
+ defineFixed(new (alloc()) LWasmParameter, ins,
+ LArgument(abi.offsetFromArgBase()));
+ }
+}
+
+void LIRGenerator::visitWasmReturn(MWasmReturn* ins) {
+ MDefinition* rval = ins->getOperand(0);
+ MDefinition* instance = ins->getOperand(1);
+
+ if (rval->type() == MIRType::Int64) {
+ add(new (alloc()) LWasmReturnI64(useInt64Fixed(rval, ReturnReg64),
+ useFixed(instance, InstanceReg)));
+ return;
+ }
+
+ LAllocation returnReg;
+ if (rval->type() == MIRType::Float32) {
+ returnReg = useFixed(rval, ReturnFloat32Reg);
+ } else if (rval->type() == MIRType::Double) {
+ returnReg = useFixed(rval, ReturnDoubleReg);
+#ifdef ENABLE_WASM_SIMD
+ } else if (rval->type() == MIRType::Simd128) {
+ returnReg = useFixed(rval, ReturnSimd128Reg);
+#endif
+ } else if (rval->type() == MIRType::Int32 ||
+ rval->type() == MIRType::RefOrNull) {
+ returnReg = useFixed(rval, ReturnReg);
+ } else {
+ MOZ_CRASH("Unexpected wasm return type");
+ }
+
+ LWasmReturn* lir =
+ new (alloc()) LWasmReturn(useFixed(instance, InstanceReg), returnReg);
+ add(lir);
+}
+
+void LIRGenerator::visitWasmReturnVoid(MWasmReturnVoid* ins) {
+ MDefinition* instance = ins->getOperand(0);
+ LWasmReturnVoid* lir =
+ new (alloc()) LWasmReturnVoid(useFixed(instance, InstanceReg));
+ add(lir);
+}
+
+void LIRGenerator::visitWasmStackArg(MWasmStackArg* ins) {
+ if (ins->arg()->type() == MIRType::Int64) {
+ add(new (alloc())
+ LWasmStackArgI64(useInt64RegisterOrConstantAtStart(ins->arg())),
+ ins);
+ } else if (IsFloatingPointType(ins->arg()->type())) {
+ MOZ_ASSERT(!ins->arg()->isEmittedAtUses());
+ add(new (alloc()) LWasmStackArg(useRegisterAtStart(ins->arg())), ins);
+ } else {
+ add(new (alloc()) LWasmStackArg(useRegisterOrConstantAtStart(ins->arg())),
+ ins);
+ }
+}
+
+void LIRGenerator::visitWasmRegisterResult(MWasmRegisterResult* ins) {
+ auto* lir = new (alloc()) LWasmRegisterResult();
+ uint32_t vreg = getVirtualRegister();
+ MOZ_ASSERT(ins->type() != MIRType::Int64);
+ auto type = LDefinition::TypeFrom(ins->type());
+ lir->setDef(0, LDefinition(vreg, type, LGeneralReg(ins->loc())));
+ ins->setVirtualRegister(vreg);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmFloatRegisterResult(MWasmFloatRegisterResult* ins) {
+ auto* lir = new (alloc()) LWasmRegisterResult();
+ uint32_t vreg = getVirtualRegister();
+ auto type = LDefinition::TypeFrom(ins->type());
+ lir->setDef(0, LDefinition(vreg, type, LFloatReg(ins->loc())));
+ ins->setVirtualRegister(vreg);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmRegister64Result(MWasmRegister64Result* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Int64);
+ uint32_t vreg = getVirtualRegister();
+
+#if defined(JS_NUNBOX32)
+ auto* lir = new (alloc()) LWasmRegisterPairResult();
+ lir->setDef(INT64LOW_INDEX,
+ LDefinition(vreg + INT64LOW_INDEX, LDefinition::GENERAL,
+ LGeneralReg(ins->loc().low)));
+ lir->setDef(INT64HIGH_INDEX,
+ LDefinition(vreg + INT64HIGH_INDEX, LDefinition::GENERAL,
+ LGeneralReg(ins->loc().high)));
+ getVirtualRegister();
+#elif defined(JS_PUNBOX64)
+ auto* lir = new (alloc()) LWasmRegisterResult();
+ lir->setDef(
+ 0, LDefinition(vreg, LDefinition::GENERAL, LGeneralReg(ins->loc().reg)));
+#else
+# error expected either JS_NUNBOX32 or JS_PUNBOX64
+#endif
+
+ ins->setVirtualRegister(vreg);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmStackResultArea(MWasmStackResultArea* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::StackResults);
+ auto* lir = new (alloc()) LWasmStackResultArea(temp());
+ uint32_t vreg = getVirtualRegister();
+ lir->setDef(0,
+ LDefinition(vreg, LDefinition::STACKRESULTS, LDefinition::STACK));
+ ins->setVirtualRegister(vreg);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmStackResult(MWasmStackResult* ins) {
+ MWasmStackResultArea* area = ins->resultArea()->toWasmStackResultArea();
+ LDefinition::Policy pol = LDefinition::STACK;
+
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new (alloc()) LWasmStackResult64;
+ lir->setOperand(0, use(area, LUse(LUse::STACK, /* usedAtStart = */ true)));
+ uint32_t vreg = getVirtualRegister();
+ LDefinition::Type typ = LDefinition::GENERAL;
+#if defined(JS_NUNBOX32)
+ getVirtualRegister();
+ lir->setDef(INT64LOW_INDEX, LDefinition(vreg + INT64LOW_INDEX, typ, pol));
+ lir->setDef(INT64HIGH_INDEX, LDefinition(vreg + INT64HIGH_INDEX, typ, pol));
+#else
+ lir->setDef(0, LDefinition(vreg, typ, pol));
+#endif
+ ins->setVirtualRegister(vreg);
+ add(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LWasmStackResult;
+ lir->setOperand(0, use(area, LUse(LUse::STACK, /* usedAtStart = */ true)));
+ uint32_t vreg = getVirtualRegister();
+ LDefinition::Type typ = LDefinition::TypeFrom(ins->type());
+ lir->setDef(0, LDefinition(vreg, typ, pol));
+ ins->setVirtualRegister(vreg);
+ add(lir, ins);
+}
+
+template <class MWasmCallT>
+void LIRGenerator::visitWasmCall(MWasmCallT ins) {
+ bool needsBoundsCheck = true;
+ mozilla::Maybe<uint32_t> tableSize;
+
+ if (ins->callee().isTable()) {
+ MDefinition* index = ins->getOperand(ins->numArgs());
+
+ if (ins->callee().which() == wasm::CalleeDesc::WasmTable) {
+ uint32_t minLength = ins->callee().wasmTableMinLength();
+ mozilla::Maybe<uint32_t> maxLength = ins->callee().wasmTableMaxLength();
+ if (index->isConstant() &&
+ uint32_t(index->toConstant()->toInt32()) < minLength) {
+ needsBoundsCheck = false;
+ }
+ if (maxLength.isSome() && *maxLength == minLength) {
+ tableSize = maxLength;
+ }
+ }
+ }
+
+ auto* lir = allocateVariadic<LWasmCall>(ins->numOperands(), needsBoundsCheck,
+ tableSize);
+ if (!lir) {
+ abort(AbortReason::Alloc, "OOM: LIRGenerator::lowerWasmCall");
+ return;
+ }
+
+ for (unsigned i = 0; i < ins->numArgs(); i++) {
+ lir->setOperand(
+ i, useFixedAtStart(ins->getOperand(i), ins->registerForArg(i)));
+ }
+
+ if (ins->callee().isTable()) {
+ MDefinition* index = ins->getOperand(ins->numArgs());
+ lir->setOperand(ins->numArgs(),
+ useFixedAtStart(index, WasmTableCallIndexReg));
+ }
+ if (ins->callee().isFuncRef()) {
+ MDefinition* ref = ins->getOperand(ins->numArgs());
+ lir->setOperand(ins->numArgs(), useFixedAtStart(ref, WasmCallRefReg));
+ }
+
+ add(lir, ins);
+ assignWasmSafepoint(lir);
+
+ // WasmCall with WasmTable has two call instructions, and they both need a
+ // safepoint associated with them. Create a second safepoint here; the node
+ // otherwise does nothing, and codegen for it only marks the safepoint at the
+ // node.
+ if (ins->callee().which() == wasm::CalleeDesc::WasmTable) {
+ auto* adjunctSafepoint = new (alloc()) LWasmCallIndirectAdjunctSafepoint();
+ add(adjunctSafepoint);
+ assignWasmSafepoint(adjunctSafepoint);
+ lir->setAdjunctSafepoint(adjunctSafepoint);
+ }
+}
+
+void LIRGenerator::visitWasmCallCatchable(MWasmCallCatchable* ins) {
+ visitWasmCall(ins);
+}
+
+void LIRGenerator::visitWasmCallUncatchable(MWasmCallUncatchable* ins) {
+ visitWasmCall(ins);
+}
+
+void LIRGenerator::visitWasmCallLandingPrePad(MWasmCallLandingPrePad* ins) {
+ add(new (alloc()) LWasmCallLandingPrePad, ins);
+}
+
+void LIRGenerator::visitSetDOMProperty(MSetDOMProperty* ins) {
+ MDefinition* val = ins->value();
+
+ Register cxReg, objReg, privReg, valueReg;
+ GetTempRegForIntArg(0, 0, &cxReg);
+ GetTempRegForIntArg(1, 0, &objReg);
+ GetTempRegForIntArg(2, 0, &privReg);
+ GetTempRegForIntArg(3, 0, &valueReg);
+
+ // Keep using GetTempRegForIntArg, since we want to make sure we
+ // don't clobber registers we're already using.
+ Register tempReg1, tempReg2;
+ GetTempRegForIntArg(4, 0, &tempReg1);
+ mozilla::DebugOnly<bool> ok = GetTempRegForIntArg(5, 0, &tempReg2);
+ MOZ_ASSERT(ok, "How can we not have six temp registers?");
+
+ LSetDOMProperty* lir = new (alloc())
+ LSetDOMProperty(tempFixed(cxReg), useFixedAtStart(ins->object(), objReg),
+ useBoxFixedAtStart(val, tempReg1, tempReg2),
+ tempFixed(privReg), tempFixed(valueReg));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGetDOMProperty(MGetDOMProperty* ins) {
+ Register cxReg, objReg, privReg, valueReg;
+ GetTempRegForIntArg(0, 0, &cxReg);
+ GetTempRegForIntArg(1, 0, &objReg);
+ GetTempRegForIntArg(2, 0, &privReg);
+ mozilla::DebugOnly<bool> ok = GetTempRegForIntArg(3, 0, &valueReg);
+ MOZ_ASSERT(ok, "How can we not have four temp registers?");
+ LGetDOMProperty* lir = new (alloc())
+ LGetDOMProperty(tempFixed(cxReg), useFixedAtStart(ins->object(), objReg),
+ tempFixed(privReg), tempFixed(valueReg));
+
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGetDOMMember(MGetDOMMember* ins) {
+ MOZ_ASSERT(ins->isDomMovable(), "Members had better be movable");
+ // We wish we could assert that ins->domAliasSet() == JSJitInfo::AliasNone,
+ // but some MGetDOMMembers are for [Pure], not [Constant] properties, whose
+ // value can in fact change as a result of DOM setters and method calls.
+ MOZ_ASSERT(ins->domAliasSet() != JSJitInfo::AliasEverything,
+ "Member gets had better not alias the world");
+
+ MDefinition* obj = ins->object();
+ MOZ_ASSERT(obj->type() == MIRType::Object);
+
+ MIRType type = ins->type();
+
+ if (type == MIRType::Value) {
+ LGetDOMMemberV* lir = new (alloc()) LGetDOMMemberV(useRegisterAtStart(obj));
+ defineBox(lir, ins);
+ } else {
+ LGetDOMMemberT* lir =
+ new (alloc()) LGetDOMMemberT(useRegisterForTypedLoad(obj, type));
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitLoadDOMExpandoValue(MLoadDOMExpandoValue* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+ auto* lir =
+ new (alloc()) LLoadDOMExpandoValue(useRegisterAtStart(ins->proxy()));
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitLoadDOMExpandoValueGuardGeneration(
+ MLoadDOMExpandoValueGuardGeneration* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+ auto* lir = new (alloc())
+ LLoadDOMExpandoValueGuardGeneration(useRegisterAtStart(ins->proxy()));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitLoadDOMExpandoValueIgnoreGeneration(
+ MLoadDOMExpandoValueIgnoreGeneration* ins) {
+ MOZ_ASSERT(ins->proxy()->type() == MIRType::Object);
+ auto* lir = new (alloc())
+ LLoadDOMExpandoValueIgnoreGeneration(useRegisterAtStart(ins->proxy()));
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitGuardDOMExpandoMissingOrGuardShape(
+ MGuardDOMExpandoMissingOrGuardShape* ins) {
+ MOZ_ASSERT(ins->expando()->type() == MIRType::Value);
+ auto* lir = new (alloc())
+ LGuardDOMExpandoMissingOrGuardShape(useBox(ins->expando()), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->expando());
+}
+
+void LIRGenerator::visitIncrementWarmUpCounter(MIncrementWarmUpCounter* ins) {
+ LIncrementWarmUpCounter* lir = new (alloc()) LIncrementWarmUpCounter(temp());
+ add(lir, ins);
+}
+
+void LIRGenerator::visitLexicalCheck(MLexicalCheck* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Value);
+ LLexicalCheck* lir = new (alloc()) LLexicalCheck(useBox(input));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, input);
+}
+
+void LIRGenerator::visitThrowRuntimeLexicalError(
+ MThrowRuntimeLexicalError* ins) {
+ LThrowRuntimeLexicalError* lir = new (alloc()) LThrowRuntimeLexicalError();
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitThrowMsg(MThrowMsg* ins) {
+ LThrowMsg* lir = new (alloc()) LThrowMsg();
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGlobalDeclInstantiation(MGlobalDeclInstantiation* ins) {
+ LGlobalDeclInstantiation* lir = new (alloc()) LGlobalDeclInstantiation();
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitDebugger(MDebugger* ins) {
+ auto* lir = new (alloc()) LDebugger(tempFixed(CallTempReg0));
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+}
+
+void LIRGenerator::visitAtomicIsLockFree(MAtomicIsLockFree* ins) {
+ define(new (alloc()) LAtomicIsLockFree(useRegister(ins->input())), ins);
+}
+
+void LIRGenerator::visitCheckReturn(MCheckReturn* ins) {
+ MDefinition* retVal = ins->returnValue();
+ MDefinition* thisVal = ins->thisValue();
+ MOZ_ASSERT(retVal->type() == MIRType::Value);
+ MOZ_ASSERT(thisVal->type() == MIRType::Value);
+
+ auto* lir =
+ new (alloc()) LCheckReturn(useBoxAtStart(retVal), useBoxAtStart(thisVal));
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCheckIsObj(MCheckIsObj* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Value);
+
+ LCheckIsObj* lir = new (alloc()) LCheckIsObj(useBox(input));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCheckObjCoercible(MCheckObjCoercible* ins) {
+ MDefinition* checkVal = ins->checkValue();
+ MOZ_ASSERT(checkVal->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LCheckObjCoercible(useBoxAtStart(checkVal));
+ redefine(ins, checkVal);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCheckClassHeritage(MCheckClassHeritage* ins) {
+ MDefinition* heritage = ins->heritage();
+ MOZ_ASSERT(heritage->type() == MIRType::Value);
+
+ auto* lir =
+ new (alloc()) LCheckClassHeritage(useBox(heritage), temp(), temp());
+ redefine(ins, heritage);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCheckThis(MCheckThis* ins) {
+ MDefinition* thisValue = ins->thisValue();
+ MOZ_ASSERT(thisValue->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LCheckThis(useBoxAtStart(thisValue));
+ redefine(ins, thisValue);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCheckThisReinit(MCheckThisReinit* ins) {
+ MDefinition* thisValue = ins->thisValue();
+ MOZ_ASSERT(thisValue->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LCheckThisReinit(useBoxAtStart(thisValue));
+ redefine(ins, thisValue);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGenerator(MGenerator* ins) {
+ auto* lir =
+ new (alloc()) LGenerator(useRegisterAtStart(ins->callee()),
+ useRegisterAtStart(ins->environmentChain()),
+ useRegisterAtStart(ins->argsObject()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitAsyncResolve(MAsyncResolve* ins) {
+ auto* lir = new (alloc()) LAsyncResolve(useRegisterAtStart(ins->generator()),
+ useBoxAtStart(ins->valueOrReason()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitAsyncAwait(MAsyncAwait* ins) {
+ MOZ_ASSERT(ins->generator()->type() == MIRType::Object);
+ auto* lir = new (alloc()) LAsyncAwait(useBoxAtStart(ins->value()),
+ useRegisterAtStart(ins->generator()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCanSkipAwait(MCanSkipAwait* ins) {
+ auto* lir = new (alloc()) LCanSkipAwait(useBoxAtStart(ins->value()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitMaybeExtractAwaitValue(MMaybeExtractAwaitValue* ins) {
+ auto* lir = new (alloc()) LMaybeExtractAwaitValue(
+ useBoxAtStart(ins->value()), useRegisterAtStart(ins->canSkip()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitDebugCheckSelfHosted(MDebugCheckSelfHosted* ins) {
+ MDefinition* checkVal = ins->checkValue();
+ MOZ_ASSERT(checkVal->type() == MIRType::Value);
+
+ LDebugCheckSelfHosted* lir =
+ new (alloc()) LDebugCheckSelfHosted(useBoxAtStart(checkVal));
+ redefine(ins, checkVal);
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitFinishBoundFunctionInit(MFinishBoundFunctionInit* ins) {
+ auto lir = new (alloc()) LFinishBoundFunctionInit(
+ useRegister(ins->bound()), useRegister(ins->target()),
+ useRegister(ins->argCount()), temp(), temp());
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitIsPackedArray(MIsPackedArray* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+
+ auto lir = new (alloc()) LIsPackedArray(useRegister(ins->object()), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitGuardArrayIsPacked(MGuardArrayIsPacked* ins) {
+ MOZ_ASSERT(ins->array()->type() == MIRType::Object);
+
+ auto* lir = new (alloc())
+ LGuardArrayIsPacked(useRegister(ins->array()), temp(), temp());
+ assignSnapshot(lir, ins->bailoutKind());
+ add(lir, ins);
+ redefine(ins, ins->array());
+}
+
+void LIRGenerator::visitGetPrototypeOf(MGetPrototypeOf* ins) {
+ MOZ_ASSERT(ins->target()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ auto lir = new (alloc()) LGetPrototypeOf(useRegister(ins->target()));
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitObjectWithProto(MObjectWithProto* ins) {
+ MOZ_ASSERT(ins->prototype()->type() == MIRType::Value);
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LObjectWithProto(useBoxAtStart(ins->prototype()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitObjectStaticProto(MObjectStaticProto* ins) {
+ MOZ_ASSERT(ins->object()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ auto* lir =
+ new (alloc()) LObjectStaticProto(useRegisterAtStart(ins->object()));
+ define(lir, ins);
+};
+
+void LIRGenerator::visitBuiltinObject(MBuiltinObject* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LBuiltinObject();
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitReturn(MReturn* ret) {
+ return visitReturnImpl(ret->getOperand(0));
+}
+
+void LIRGenerator::visitGeneratorReturn(MGeneratorReturn* ret) {
+ return visitReturnImpl(ret->getOperand(0), true);
+}
+
+void LIRGenerator::visitSuperFunction(MSuperFunction* ins) {
+ MOZ_ASSERT(ins->callee()->type() == MIRType::Object);
+ MOZ_ASSERT(ins->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LSuperFunction(useRegister(ins->callee()), temp());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitInitHomeObject(MInitHomeObject* ins) {
+ MDefinition* function = ins->function();
+ MOZ_ASSERT(function->type() == MIRType::Object);
+
+ MDefinition* homeObject = ins->homeObject();
+ MOZ_ASSERT(homeObject->type() == MIRType::Value);
+
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ auto* lir = new (alloc())
+ LInitHomeObject(useRegisterAtStart(function), useBoxAtStart(homeObject));
+ redefine(ins, function);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitIsTypedArrayConstructor(MIsTypedArrayConstructor* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ auto* lir = new (alloc()) LIsTypedArrayConstructor(useRegister(object));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitLoadValueTag(MLoadValueTag* ins) {
+ MDefinition* value = ins->value();
+ MOZ_ASSERT(value->type() == MIRType::Value);
+
+ define(new (alloc()) LLoadValueTag(useBoxAtStart(value)), ins);
+}
+
+void LIRGenerator::visitGuardTagNotEqual(MGuardTagNotEqual* ins) {
+ MDefinition* lhs = ins->lhs();
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+
+ MDefinition* rhs = ins->rhs();
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+
+ auto* guard =
+ new (alloc()) LGuardTagNotEqual(useRegister(lhs), useRegister(rhs));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+}
+
+void LIRGenerator::visitLoadWrapperTarget(MLoadWrapperTarget* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ define(new (alloc()) LLoadWrapperTarget(useRegisterAtStart(object)), ins);
+}
+
+void LIRGenerator::visitGuardHasGetterSetter(MGuardHasGetterSetter* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ auto* guard = new (alloc())
+ LGuardHasGetterSetter(useRegisterAtStart(object), tempFixed(CallTempReg0),
+ tempFixed(CallTempReg1), tempFixed(CallTempReg2));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, object);
+}
+
+void LIRGenerator::visitGuardIsExtensible(MGuardIsExtensible* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ auto* guard = new (alloc()) LGuardIsExtensible(useRegister(object), temp());
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, object);
+}
+
+void LIRGenerator::visitGuardInt32IsNonNegative(MGuardInt32IsNonNegative* ins) {
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ auto* guard = new (alloc()) LGuardInt32IsNonNegative(useRegister(index));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, index);
+}
+
+void LIRGenerator::visitGuardInt32Range(MGuardInt32Range* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Int32);
+
+ auto* guard = new (alloc()) LGuardInt32Range(useRegister(input));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, input);
+}
+
+void LIRGenerator::visitGuardIndexIsNotDenseElement(
+ MGuardIndexIsNotDenseElement* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ LDefinition spectreTemp =
+ BoundsCheckNeedsSpectreTemp() ? temp() : LDefinition::BogusTemp();
+
+ auto* guard = new (alloc()) LGuardIndexIsNotDenseElement(
+ useRegister(object), useRegister(index), temp(), spectreTemp);
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, index);
+}
+
+void LIRGenerator::visitGuardIndexIsValidUpdateOrAdd(
+ MGuardIndexIsValidUpdateOrAdd* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ LDefinition spectreTemp =
+ BoundsCheckNeedsSpectreTemp() ? temp() : LDefinition::BogusTemp();
+
+ auto* guard = new (alloc()) LGuardIndexIsValidUpdateOrAdd(
+ useRegister(object), useRegister(index), temp(), spectreTemp);
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, index);
+}
+
+void LIRGenerator::visitCallAddOrUpdateSparseElement(
+ MCallAddOrUpdateSparseElement* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ MDefinition* value = ins->value();
+ MOZ_ASSERT(value->type() == MIRType::Value);
+
+ auto* lir = new (alloc()) LCallAddOrUpdateSparseElement(
+ useRegisterAtStart(object), useRegisterAtStart(index),
+ useBoxAtStart(value));
+ add(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCallGetSparseElement(MCallGetSparseElement* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LCallGetSparseElement(useRegisterAtStart(object),
+ useRegisterAtStart(index));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCallNativeGetElement(MCallNativeGetElement* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LCallNativeGetElement(useRegisterAtStart(object),
+ useRegisterAtStart(index));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCallNativeGetElementSuper(
+ MCallNativeGetElementSuper* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ MDefinition* receiver = ins->receiver();
+
+ auto* lir = new (alloc()) LCallNativeGetElementSuper(
+ useRegisterAtStart(object), useRegisterAtStart(index),
+ useBoxAtStart(receiver));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCallObjectHasSparseElement(
+ MCallObjectHasSparseElement* ins) {
+ MDefinition* object = ins->object();
+ MOZ_ASSERT(object->type() == MIRType::Object);
+
+ MDefinition* index = ins->index();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+
+ auto* lir = new (alloc()) LCallObjectHasSparseElement(
+ useRegisterAtStart(object), useRegisterAtStart(index),
+ tempFixed(CallTempReg0), tempFixed(CallTempReg1));
+ assignSnapshot(lir, ins->bailoutKind());
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitBigIntAsIntN(MBigIntAsIntN* ins) {
+ MOZ_ASSERT(ins->bits()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->input()->type() == MIRType::BigInt);
+
+ if (ins->bits()->isConstant()) {
+ int32_t bits = ins->bits()->toConstant()->toInt32();
+ if (bits == 64) {
+ auto* lir = new (alloc())
+ LBigIntAsIntN64(useRegister(ins->input()), temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+ if (bits == 32) {
+ auto* lir = new (alloc())
+ LBigIntAsIntN32(useRegister(ins->input()), temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+ }
+
+ auto* lir = new (alloc()) LBigIntAsIntN(useRegisterAtStart(ins->bits()),
+ useRegisterAtStart(ins->input()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitBigIntAsUintN(MBigIntAsUintN* ins) {
+ MOZ_ASSERT(ins->bits()->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->input()->type() == MIRType::BigInt);
+
+ if (ins->bits()->isConstant()) {
+ int32_t bits = ins->bits()->toConstant()->toInt32();
+ if (bits == 64) {
+ auto* lir = new (alloc())
+ LBigIntAsUintN64(useRegister(ins->input()), temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+ if (bits == 32) {
+ auto* lir = new (alloc())
+ LBigIntAsUintN32(useRegister(ins->input()), temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+ }
+
+ auto* lir = new (alloc()) LBigIntAsUintN(useRegisterAtStart(ins->bits()),
+ useRegisterAtStart(ins->input()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitGuardNonGCThing(MGuardNonGCThing* ins) {
+ MDefinition* input = ins->input();
+
+ auto* guard = new (alloc()) LGuardNonGCThing(useBox(input));
+ assignSnapshot(guard, ins->bailoutKind());
+ add(guard, ins);
+ redefine(ins, input);
+}
+
+void LIRGenerator::visitToHashableNonGCThing(MToHashableNonGCThing* ins) {
+ auto* lir =
+ new (alloc()) LToHashableNonGCThing(useBox(ins->input()), tempDouble());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitToHashableString(MToHashableString* ins) {
+ auto* lir = new (alloc()) LToHashableString(useRegister(ins->input()));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitToHashableValue(MToHashableValue* ins) {
+ auto* lir =
+ new (alloc()) LToHashableValue(useBox(ins->input()), tempDouble());
+ defineBox(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitHashNonGCThing(MHashNonGCThing* ins) {
+ auto* lir = new (alloc()) LHashNonGCThing(useBox(ins->input()), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitHashString(MHashString* ins) {
+ auto* lir = new (alloc()) LHashString(useRegister(ins->input()), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitHashSymbol(MHashSymbol* ins) {
+ auto* lir = new (alloc()) LHashSymbol(useRegister(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitHashBigInt(MHashBigInt* ins) {
+ auto* lir = new (alloc())
+ LHashBigInt(useRegister(ins->input()), temp(), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitHashObject(MHashObject* ins) {
+ auto* lir =
+ new (alloc()) LHashObject(useRegister(ins->set()), useBox(ins->input()),
+ temp(), temp(), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitHashValue(MHashValue* ins) {
+ auto* lir =
+ new (alloc()) LHashValue(useRegister(ins->set()), useBox(ins->input()),
+ temp(), temp(), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitSetObjectHasNonBigInt(MSetObjectHasNonBigInt* ins) {
+ auto* lir = new (alloc())
+ LSetObjectHasNonBigInt(useRegister(ins->set()), useBox(ins->value()),
+ useRegister(ins->hash()), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitSetObjectHasBigInt(MSetObjectHasBigInt* ins) {
+ auto* lir = new (alloc()) LSetObjectHasBigInt(
+ useRegister(ins->set()), useBox(ins->value()), useRegister(ins->hash()),
+ temp(), temp(), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitSetObjectHasValue(MSetObjectHasValue* ins) {
+ auto* lir = new (alloc()) LSetObjectHasValue(
+ useRegister(ins->set()), useBox(ins->value()), useRegister(ins->hash()),
+ temp(), temp(), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitSetObjectHasValueVMCall(MSetObjectHasValueVMCall* ins) {
+ auto* lir = new (alloc()) LSetObjectHasValueVMCall(
+ useRegisterAtStart(ins->set()), useBoxAtStart(ins->value()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitSetObjectSize(MSetObjectSize* ins) {
+ auto* lir = new (alloc()) LSetObjectSize(useRegisterAtStart(ins->set()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectHasNonBigInt(MMapObjectHasNonBigInt* ins) {
+ auto* lir = new (alloc())
+ LMapObjectHasNonBigInt(useRegister(ins->map()), useBox(ins->value()),
+ useRegister(ins->hash()), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectHasBigInt(MMapObjectHasBigInt* ins) {
+ auto* lir = new (alloc()) LMapObjectHasBigInt(
+ useRegister(ins->map()), useBox(ins->value()), useRegister(ins->hash()),
+ temp(), temp(), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectHasValue(MMapObjectHasValue* ins) {
+ auto* lir = new (alloc()) LMapObjectHasValue(
+ useRegister(ins->map()), useBox(ins->value()), useRegister(ins->hash()),
+ temp(), temp(), temp(), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectHasValueVMCall(MMapObjectHasValueVMCall* ins) {
+ auto* lir = new (alloc()) LMapObjectHasValueVMCall(
+ useRegisterAtStart(ins->map()), useBoxAtStart(ins->value()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectGetNonBigInt(MMapObjectGetNonBigInt* ins) {
+ auto* lir = new (alloc())
+ LMapObjectGetNonBigInt(useRegister(ins->map()), useBox(ins->value()),
+ useRegister(ins->hash()), temp(), temp());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectGetBigInt(MMapObjectGetBigInt* ins) {
+ auto* lir = new (alloc()) LMapObjectGetBigInt(
+ useRegister(ins->map()), useBox(ins->value()), useRegister(ins->hash()),
+ temp(), temp(), temp(), temp());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectGetValue(MMapObjectGetValue* ins) {
+ auto* lir = new (alloc()) LMapObjectGetValue(
+ useRegister(ins->map()), useBox(ins->value()), useRegister(ins->hash()),
+ temp(), temp(), temp(), temp());
+ defineBox(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectGetValueVMCall(MMapObjectGetValueVMCall* ins) {
+ auto* lir = new (alloc()) LMapObjectGetValueVMCall(
+ useRegisterAtStart(ins->map()), useBoxAtStart(ins->value()));
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitMapObjectSize(MMapObjectSize* ins) {
+ auto* lir = new (alloc()) LMapObjectSize(useRegisterAtStart(ins->map()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitConstant(MConstant* ins) {
+ if (!IsFloatingPointType(ins->type()) && ins->canEmitAtUses()) {
+ emitAtUses(ins);
+ return;
+ }
+
+ switch (ins->type()) {
+ case MIRType::Double:
+ define(new (alloc()) LDouble(ins->toDouble()), ins);
+ break;
+ case MIRType::Float32:
+ define(new (alloc()) LFloat32(ins->toFloat32()), ins);
+ break;
+ case MIRType::Boolean:
+ define(new (alloc()) LInteger(ins->toBoolean()), ins);
+ break;
+ case MIRType::Int32:
+ define(new (alloc()) LInteger(ins->toInt32()), ins);
+ break;
+ case MIRType::Int64:
+ defineInt64(new (alloc()) LInteger64(ins->toInt64()), ins);
+ break;
+ case MIRType::IntPtr:
+#ifdef JS_64BIT
+ defineInt64(new (alloc()) LInteger64(ins->toIntPtr()), ins);
+#else
+ define(new (alloc()) LInteger(ins->toIntPtr()), ins);
+#endif
+ break;
+ case MIRType::String:
+ define(new (alloc()) LPointer(ins->toString()), ins);
+ break;
+ case MIRType::Symbol:
+ define(new (alloc()) LPointer(ins->toSymbol()), ins);
+ break;
+ case MIRType::BigInt:
+ define(new (alloc()) LPointer(ins->toBigInt()), ins);
+ break;
+ case MIRType::Object:
+ define(new (alloc()) LPointer(&ins->toObject()), ins);
+ break;
+ case MIRType::Shape:
+ MOZ_ASSERT(ins->isEmittedAtUses());
+ break;
+ default:
+ // Constants of special types (undefined, null) should never flow into
+ // here directly. Operations blindly consuming them require a Box.
+ MOZ_CRASH("unexpected constant type");
+ }
+}
+
+void LIRGenerator::visitConstantProto(MConstantProto* ins) {
+ JSObject* obj = &ins->protoObject()->toConstant()->toObject();
+ define(new (alloc()) LPointer(obj), ins);
+}
+
+void LIRGenerator::visitWasmNullConstant(MWasmNullConstant* ins) {
+ define(new (alloc()) LWasmNullConstant(), ins);
+}
+
+void LIRGenerator::visitWasmFloatConstant(MWasmFloatConstant* ins) {
+ switch (ins->type()) {
+ case MIRType::Double:
+ define(new (alloc()) LDouble(ins->toDouble()), ins);
+ break;
+ case MIRType::Float32:
+ define(new (alloc()) LFloat32(ins->toFloat32()), ins);
+ break;
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ define(new (alloc()) LSimd128(ins->toSimd128()), ins);
+ break;
+#endif
+ default:
+ MOZ_CRASH("unexpected constant type");
+ }
+}
+
+#ifdef JS_JITSPEW
+static void SpewResumePoint(MBasicBlock* block, MInstruction* ins,
+ MResumePoint* resumePoint) {
+ Fprinter& out = JitSpewPrinter();
+ out.printf("Current resume point %p details:\n", (void*)resumePoint);
+ out.printf(" frame count: %u\n", resumePoint->frameCount());
+
+ if (ins) {
+ out.printf(" taken after: ");
+ ins->printName(out);
+ } else {
+ out.printf(" taken at block %u entry", block->id());
+ }
+ out.printf("\n");
+
+ out.printf(" pc: %p (script: %p, offset: %d)\n", (void*)resumePoint->pc(),
+ (void*)resumePoint->block()->info().script(),
+ int(resumePoint->block()->info().script()->pcToOffset(
+ resumePoint->pc())));
+
+ for (size_t i = 0, e = resumePoint->numOperands(); i < e; i++) {
+ MDefinition* in = resumePoint->getOperand(i);
+ out.printf(" slot%u: ", (unsigned)i);
+ in->printName(out);
+ out.printf("\n");
+ }
+}
+#endif
+
+void LIRGenerator::visitInstructionDispatch(MInstruction* ins) {
+#ifdef JS_CODEGEN_NONE
+ // Don't compile the switch-statement below so that we don't have to define
+ // the platform-specific visit* methods for the none-backend.
+ MOZ_CRASH();
+#else
+ switch (ins->op()) {
+# define MIR_OP(op) \
+ case MDefinition::Opcode::op: \
+ visit##op(ins->to##op()); \
+ break;
+ MIR_OPCODE_LIST(MIR_OP)
+# undef MIR_OP
+ default:
+ MOZ_CRASH("Invalid instruction");
+ }
+#endif
+}
+
+void LIRGeneratorShared::visitEmittedAtUses(MInstruction* ins) {
+ static_cast<LIRGenerator*>(this)->visitInstructionDispatch(ins);
+}
+
+bool LIRGenerator::visitInstruction(MInstruction* ins) {
+ MOZ_ASSERT(!errored());
+
+ if (ins->isRecoveredOnBailout()) {
+ MOZ_ASSERT(!JitOptions.disableRecoverIns);
+ return true;
+ }
+
+ if (!gen->ensureBallast()) {
+ return false;
+ }
+ visitInstructionDispatch(ins);
+
+ if (ins->resumePoint()) {
+ updateResumeState(ins);
+ }
+
+#ifdef DEBUG
+ ins->setInWorklistUnchecked();
+#endif
+
+ // If no safepoint was created, there's no need for an OSI point.
+ if (LOsiPoint* osiPoint = popOsiPoint()) {
+ add(osiPoint);
+ }
+
+ return !errored();
+}
+
+void LIRGenerator::definePhis() {
+ size_t lirIndex = 0;
+ MBasicBlock* block = current->mir();
+ for (MPhiIterator phi(block->phisBegin()); phi != block->phisEnd(); phi++) {
+ if (phi->type() == MIRType::Value) {
+ defineUntypedPhi(*phi, lirIndex);
+ lirIndex += BOX_PIECES;
+ } else if (phi->type() == MIRType::Int64) {
+ defineInt64Phi(*phi, lirIndex);
+ lirIndex += INT64_PIECES;
+ } else {
+ defineTypedPhi(*phi, lirIndex);
+ lirIndex += 1;
+ }
+ }
+}
+
+void LIRGenerator::updateResumeState(MInstruction* ins) {
+ lastResumePoint_ = ins->resumePoint();
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_IonSnapshots) && lastResumePoint_) {
+ SpewResumePoint(nullptr, ins, lastResumePoint_);
+ }
+#endif
+}
+
+void LIRGenerator::updateResumeState(MBasicBlock* block) {
+ // Note: RangeAnalysis can flag blocks as unreachable, but they are only
+ // removed iff GVN (including UCE) is enabled.
+ MOZ_ASSERT_IF(!mir()->compilingWasm() && !block->unreachable(),
+ block->entryResumePoint());
+ MOZ_ASSERT_IF(block->unreachable(), !mir()->optimizationInfo().gvnEnabled());
+ lastResumePoint_ = block->entryResumePoint();
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_IonSnapshots) && lastResumePoint_) {
+ SpewResumePoint(block, nullptr, lastResumePoint_);
+ }
+#endif
+}
+
+bool LIRGenerator::visitBlock(MBasicBlock* block) {
+ current = block->lir();
+ updateResumeState(block);
+
+ definePhis();
+
+ MOZ_ASSERT_IF(block->unreachable(), !mir()->optimizationInfo().gvnEnabled());
+ for (MInstructionIterator iter = block->begin(); *iter != block->lastIns();
+ iter++) {
+ if (!visitInstruction(*iter)) {
+ return false;
+ }
+ }
+
+ if (block->successorWithPhis()) {
+ // If we have a successor with phis, lower the phi input now that we
+ // are approaching the join point.
+ MBasicBlock* successor = block->successorWithPhis();
+ uint32_t position = block->positionInPhiSuccessor();
+ size_t lirIndex = 0;
+ for (MPhiIterator phi(successor->phisBegin()); phi != successor->phisEnd();
+ phi++) {
+ if (!gen->ensureBallast()) {
+ return false;
+ }
+
+ MDefinition* opd = phi->getOperand(position);
+ ensureDefined(opd);
+
+ MOZ_ASSERT(opd->type() == phi->type());
+
+ if (phi->type() == MIRType::Value) {
+ lowerUntypedPhiInput(*phi, position, successor->lir(), lirIndex);
+ lirIndex += BOX_PIECES;
+ } else if (phi->type() == MIRType::Int64) {
+ lowerInt64PhiInput(*phi, position, successor->lir(), lirIndex);
+ lirIndex += INT64_PIECES;
+ } else {
+ lowerTypedPhiInput(*phi, position, successor->lir(), lirIndex);
+ lirIndex += 1;
+ }
+ }
+ }
+
+ // Now emit the last instruction, which is some form of branch.
+ if (!visitInstruction(block->lastIns())) {
+ return false;
+ }
+
+ return true;
+}
+
+void LIRGenerator::visitNaNToZero(MNaNToZero* ins) {
+ MDefinition* input = ins->input();
+
+ if (ins->operandIsNeverNaN() && ins->operandIsNeverNegativeZero()) {
+ redefine(ins, input);
+ return;
+ }
+ LNaNToZero* lir =
+ new (alloc()) LNaNToZero(useRegisterAtStart(input), tempDouble());
+ defineReuseInput(lir, ins, 0);
+}
+
+bool LIRGenerator::generate() {
+ // Create all blocks and prep all phis beforehand.
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); block++) {
+ if (gen->shouldCancel("Lowering (preparation loop)")) {
+ return false;
+ }
+
+ if (!lirGraph_.initBlock(*block)) {
+ return false;
+ }
+ }
+
+ for (ReversePostorderIterator block(graph.rpoBegin());
+ block != graph.rpoEnd(); block++) {
+ if (gen->shouldCancel("Lowering (main loop)")) {
+ return false;
+ }
+
+ if (!visitBlock(*block)) {
+ return false;
+ }
+ }
+
+ lirGraph_.setArgumentSlotCount(maxargslots_);
+ return true;
+}
+
+void LIRGenerator::visitPhi(MPhi* phi) {
+ // Phi nodes are not lowered because they are only meaningful for the register
+ // allocator.
+ MOZ_CRASH("Unexpected Phi node during Lowering.");
+}
+
+void LIRGenerator::visitBeta(MBeta* beta) {
+ // Beta nodes are supposed to be removed before because they are
+ // only used to carry the range information for Range analysis
+ MOZ_CRASH("Unexpected Beta node during Lowering.");
+}
+
+void LIRGenerator::visitObjectState(MObjectState* objState) {
+ // ObjectState nodes are always recovered on bailouts
+ MOZ_CRASH("Unexpected ObjectState node during Lowering.");
+}
+
+void LIRGenerator::visitArrayState(MArrayState* objState) {
+ // ArrayState nodes are always recovered on bailouts
+ MOZ_CRASH("Unexpected ArrayState node during Lowering.");
+}
+
+void LIRGenerator::visitIonToWasmCall(MIonToWasmCall* ins) {
+ // The instruction needs a temp register:
+ // - that's not the FramePointer, since wasm is going to use it in the
+ // function.
+ // - that's not aliasing an input register.
+ LDefinition scratch = tempFixed(ABINonArgReg0);
+
+ // Note that since this is a LIR call instruction, regalloc will prevent
+ // the use*AtStart below from reusing any of the temporaries.
+
+ LInstruction* lir;
+ if (ins->type() == MIRType::Value) {
+ lir = allocateVariadic<LIonToWasmCallV>(ins->numOperands(), scratch);
+ } else if (ins->type() == MIRType::Int64) {
+ lir = allocateVariadic<LIonToWasmCallI64>(ins->numOperands(), scratch);
+ } else {
+ lir = allocateVariadic<LIonToWasmCall>(ins->numOperands(), scratch);
+ }
+ if (!lir) {
+ abort(AbortReason::Alloc, "OOM: LIRGenerator::visitIonToWasmCall");
+ return;
+ }
+
+ ABIArgGenerator abi;
+ for (unsigned i = 0; i < ins->numOperands(); i++) {
+ MDefinition* argDef = ins->getOperand(i);
+ ABIArg arg = abi.next(ToMIRType(argDef->type()));
+ switch (arg.kind()) {
+ case ABIArg::GPR:
+ case ABIArg::FPU:
+ lir->setOperand(i, useFixedAtStart(argDef, arg.reg()));
+ break;
+ case ABIArg::Stack:
+ lir->setOperand(i, useAtStart(argDef));
+ break;
+#ifdef JS_CODEGEN_REGISTER_PAIR
+ case ABIArg::GPR_PAIR:
+ MOZ_CRASH(
+ "no way to pass i64, and wasm uses hardfp for function calls");
+#endif
+ case ABIArg::Uninitialized:
+ MOZ_CRASH("Uninitialized ABIArg kind");
+ }
+ }
+
+ defineReturn(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmSelect(MWasmSelect* ins) {
+ MDefinition* condExpr = ins->condExpr();
+
+ // Pick off specific cases that we can do with LWasmCompareAndSelect to avoid
+ // generating a boolean that we then have to test again.
+ if (condExpr->isCompare() && condExpr->isEmittedAtUses()) {
+ MCompare* comp = condExpr->toCompare();
+ MCompare::CompareType compTy = comp->compareType();
+ if (canSpecializeWasmCompareAndSelect(compTy, ins->type())) {
+ JSOp jsop = comp->jsop();
+ // We don't currently generate any other JSOPs for the comparison, and if
+ // that changes, we want to know about it. Hence this assertion.
+ MOZ_ASSERT(jsop == JSOp::Eq || jsop == JSOp::Ne || jsop == JSOp::Lt ||
+ jsop == JSOp::Gt || jsop == JSOp::Le || jsop == JSOp::Ge);
+ MDefinition* lhs = comp->lhs();
+ MDefinition* rhs = comp->rhs();
+ jsop = ReorderComparison(jsop, &lhs, &rhs);
+ lowerWasmCompareAndSelect(ins, lhs, rhs, compTy, jsop);
+ return;
+ }
+ }
+ // Fall through to code that generates a boolean and selects on that.
+
+ if (ins->type() == MIRType::Int64) {
+ lowerWasmSelectI64(ins);
+ return;
+ }
+
+ lowerWasmSelectI(ins);
+}
+
+void LIRGenerator::visitWasmFence(MWasmFence* ins) {
+ add(new (alloc()) LWasmFence, ins);
+}
+
+void LIRGenerator::visitWasmLoadField(MWasmLoadField* ins) {
+ size_t offs = ins->offset();
+ LAllocation obj = useRegister(ins->obj());
+ MWideningOp wideningOp = ins->wideningOp();
+ if (ins->type() == MIRType::Int64) {
+ MOZ_RELEASE_ASSERT(wideningOp == MWideningOp::None);
+ defineInt64(new (alloc()) LWasmLoadSlotI64(obj, offs, ins->maybeTrap()),
+ ins);
+ } else {
+ define(new (alloc()) LWasmLoadSlot(obj, offs, ins->type(), wideningOp,
+ ins->maybeTrap()),
+ ins);
+ }
+}
+
+void LIRGenerator::visitWasmLoadFieldKA(MWasmLoadFieldKA* ins) {
+ size_t offs = ins->offset();
+ LAllocation obj = useRegister(ins->obj());
+ MWideningOp wideningOp = ins->wideningOp();
+ if (ins->type() == MIRType::Int64) {
+ MOZ_RELEASE_ASSERT(wideningOp == MWideningOp::None);
+ defineInt64(new (alloc()) LWasmLoadSlotI64(obj, offs, ins->maybeTrap()),
+ ins);
+ } else {
+ define(new (alloc()) LWasmLoadSlot(obj, offs, ins->type(), wideningOp,
+ ins->maybeTrap()),
+ ins);
+ }
+ add(new (alloc()) LKeepAliveObject(useKeepalive(ins->ka())), ins);
+}
+
+void LIRGenerator::visitWasmStoreFieldKA(MWasmStoreFieldKA* ins) {
+ MDefinition* value = ins->value();
+ size_t offs = ins->offset();
+ MNarrowingOp narrowingOp = ins->narrowingOp();
+ LAllocation obj = useRegister(ins->obj());
+ LInstruction* lir;
+ if (value->type() == MIRType::Int64) {
+ MOZ_RELEASE_ASSERT(narrowingOp == MNarrowingOp::None);
+ lir = new (alloc())
+ LWasmStoreSlotI64(useInt64Register(value), obj, offs, ins->maybeTrap());
+ } else {
+ lir = new (alloc())
+ LWasmStoreSlot(useRegister(value), obj, offs, value->type(),
+ narrowingOp, ins->maybeTrap());
+ }
+ add(lir, ins);
+ add(new (alloc()) LKeepAliveObject(useKeepalive(ins->ka())), ins);
+}
+
+void LIRGenerator::visitWasmStoreFieldRefKA(MWasmStoreFieldRefKA* ins) {
+ LAllocation instance = useRegister(ins->instance());
+ LAllocation valueAddr = useFixed(ins->valueAddr(), PreBarrierReg);
+ LAllocation value = useRegister(ins->value());
+ add(new (alloc()) LWasmStoreRef(instance, valueAddr, value, temp()), ins);
+ add(new (alloc()) LKeepAliveObject(useKeepalive(ins->ka())), ins);
+}
+
+void LIRGenerator::visitWasmGcObjectIsSubtypeOf(MWasmGcObjectIsSubtypeOf* ins) {
+ if (CanEmitAtUseForSingleTest(ins)) {
+ emitAtUses(ins);
+ return;
+ }
+
+ LAllocation object = useRegister(ins->object());
+ LAllocation superTypeDef = useRegister(ins->superTypeDef());
+ uint32_t subTypingDepth = ins->subTypingDepth();
+ LDefinition subTypeDepth = temp();
+ LDefinition scratch =
+ subTypingDepth >= wasm::MinSuperTypeVectorLength ? temp() : LDefinition();
+ define(new (alloc()) LWasmGcObjectIsSubtypeOf(object, superTypeDef,
+ subTypeDepth, scratch),
+ ins);
+}
+
+#ifdef FUZZING_JS_FUZZILLI
+void LIRGenerator::visitFuzzilliHash(MFuzzilliHash* ins) {
+ MDefinition* value = ins->getOperand(0);
+
+ if (value->type() == MIRType::Undefined || value->type() == MIRType::Null) {
+ define(new (alloc()) LFuzzilliHashT(LAllocation(), temp(), tempDouble()),
+ ins);
+ } else if (value->type() == MIRType::Int32 ||
+ value->type() == MIRType::Double ||
+ value->type() == MIRType::Float32 ||
+ value->type() == MIRType::Boolean ||
+ value->type() == MIRType::BigInt) {
+ define(new (alloc())
+ LFuzzilliHashT(useRegister(value), temp(), tempDouble()),
+ ins);
+ } else if (value->type() == MIRType::Object) {
+ LFuzzilliHashT* lir =
+ new (alloc()) LFuzzilliHashT(useRegister(value), temp(), tempDouble());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ } else if (value->type() == MIRType::Value) {
+ LFuzzilliHashV* lir =
+ new (alloc()) LFuzzilliHashV(useBox(value), temp(), tempDouble());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ } else {
+ define(new (alloc()) LInteger(0), ins);
+ }
+}
+
+void LIRGenerator::visitFuzzilliHashStore(MFuzzilliHashStore* ins) {
+ MDefinition* value = ins->getOperand(0);
+ MOZ_ASSERT(value->type() == MIRType::Int32);
+ add(new (alloc()) LFuzzilliHashStore(useRegister(value), temp(), temp()),
+ ins);
+}
+#endif
+
+static_assert(!std::is_polymorphic_v<LIRGenerator>,
+ "LIRGenerator should not have any virtual methods");
+
+#ifdef JS_CODEGEN_NONE
+void LIRGenerator::visitReturnImpl(MDefinition*, bool) { MOZ_CRASH(); }
+#endif
diff --git a/js/src/jit/Lowering.h b/js/src/jit/Lowering.h
new file mode 100644
index 0000000000..28a7c5173e
--- /dev/null
+++ b/js/src/jit/Lowering.h
@@ -0,0 +1,88 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Lowering_h
+#define jit_Lowering_h
+
+// This file declares the structures that are used for attaching LIR to a
+// MIRGraph.
+
+#include "jit/LIR.h"
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/Lowering-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/Lowering-x64.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/Lowering-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/Lowering-arm64.h"
+#elif defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/Lowering-mips32.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/Lowering-mips64.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/Lowering-loong64.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/Lowering-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/Lowering-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+namespace js {
+namespace jit {
+
+class LIRGenerator final : public LIRGeneratorSpecific {
+ void updateResumeState(MInstruction* ins);
+ void updateResumeState(MBasicBlock* block);
+
+ // The maximum depth, for framesizeclass determination.
+ uint32_t maxargslots_;
+
+ public:
+ LIRGenerator(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorSpecific(gen, graph, lirGraph), maxargslots_(0) {}
+
+ [[nodiscard]] bool generate();
+
+ private:
+ LBoxAllocation useBoxFixedAtStart(MDefinition* mir, Register reg1,
+ Register reg2) {
+ return useBoxFixed(mir, reg1, reg2, /* useAtStart = */ true);
+ }
+
+ LBoxAllocation useBoxFixedAtStart(MDefinition* mir, ValueOperand op);
+ LBoxAllocation useBoxAtStart(MDefinition* mir,
+ LUse::Policy policy = LUse::REGISTER);
+
+ void lowerBitOp(JSOp op, MBinaryInstruction* ins);
+ void lowerShiftOp(JSOp op, MShiftInstruction* ins);
+ LInstructionHelper<1, 1, 0>* allocateAbs(MAbs* ins, LAllocation input);
+ void definePhis();
+
+ [[nodiscard]] bool lowerCallArguments(MCall* call);
+
+ friend class LIRGeneratorShared;
+ void visitInstructionDispatch(MInstruction* ins);
+
+ void visitReturnImpl(MDefinition* def, bool isGenerator = false);
+
+ [[nodiscard]] bool visitInstruction(MInstruction* ins);
+ [[nodiscard]] bool visitBlock(MBasicBlock* block);
+
+#define MIR_OP(op) void visit##op(M##op* ins);
+ MIR_OPCODE_LIST(MIR_OP)
+#undef MIR_OP
+
+ template <class MWasmCallT>
+ void visitWasmCall(MWasmCallT ins);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Lowering_h */
diff --git a/js/src/jit/MIR.cpp b/js/src/jit/MIR.cpp
new file mode 100644
index 0000000000..a403b4ff83
--- /dev/null
+++ b/js/src/jit/MIR.cpp
@@ -0,0 +1,7195 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/MIR.h"
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/ScopeExit.h"
+
+#include <array>
+#include <utility>
+
+#include "jslibmath.h"
+#include "jsmath.h"
+#include "jsnum.h"
+
+#include "builtin/RegExp.h"
+#include "jit/AtomicOperations.h"
+#include "jit/CompileInfo.h"
+#include "jit/KnownClass.h"
+#include "jit/MIRGraph.h"
+#include "jit/RangeAnalysis.h"
+#include "jit/VMFunctions.h"
+#include "js/Conversions.h"
+#include "js/experimental/JitInfo.h" // JSJitInfo, JSTypedMethodJitInfo
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "util/Text.h"
+#include "util/Unicode.h"
+#include "vm/Iteration.h" // js::NativeIterator
+#include "vm/PlainObject.h" // js::PlainObject
+#include "vm/Uint8Clamped.h"
+#include "wasm/WasmCode.h"
+
+#include "vm/JSAtom-inl.h"
+#include "wasm/WasmInstance-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::ToInt32;
+
+using mozilla::CheckedInt;
+using mozilla::DebugOnly;
+using mozilla::IsFloat32Representable;
+using mozilla::IsNaN;
+using mozilla::IsPowerOfTwo;
+using mozilla::Maybe;
+using mozilla::NumbersAreIdentical;
+
+NON_GC_POINTER_TYPE_ASSERTIONS_GENERATED
+
+#ifdef DEBUG
+size_t MUse::index() const { return consumer()->indexOf(this); }
+#endif
+
+template <size_t Op>
+static void ConvertDefinitionToDouble(TempAllocator& alloc, MDefinition* def,
+ MInstruction* consumer) {
+ MInstruction* replace = MToDouble::New(alloc, def);
+ consumer->replaceOperand(Op, replace);
+ consumer->block()->insertBefore(consumer, replace);
+}
+
+template <size_t Arity, size_t Index>
+static void ConvertOperandToDouble(MAryInstruction<Arity>* def,
+ TempAllocator& alloc) {
+ static_assert(Index < Arity);
+ auto* operand = def->getOperand(Index);
+ if (operand->type() == MIRType::Float32) {
+ ConvertDefinitionToDouble<Index>(alloc, operand, def);
+ }
+}
+
+template <size_t Arity, size_t... ISeq>
+static void ConvertOperandsToDouble(MAryInstruction<Arity>* def,
+ TempAllocator& alloc,
+ std::index_sequence<ISeq...>) {
+ (ConvertOperandToDouble<Arity, ISeq>(def, alloc), ...);
+}
+
+template <size_t Arity>
+static void ConvertOperandsToDouble(MAryInstruction<Arity>* def,
+ TempAllocator& alloc) {
+ ConvertOperandsToDouble<Arity>(def, alloc, std::make_index_sequence<Arity>{});
+}
+
+template <size_t Arity, size_t... ISeq>
+static bool AllOperandsCanProduceFloat32(MAryInstruction<Arity>* def,
+ std::index_sequence<ISeq...>) {
+ return (def->getOperand(ISeq)->canProduceFloat32() && ...);
+}
+
+template <size_t Arity>
+static bool AllOperandsCanProduceFloat32(MAryInstruction<Arity>* def) {
+ return AllOperandsCanProduceFloat32<Arity>(def,
+ std::make_index_sequence<Arity>{});
+}
+
+static bool CheckUsesAreFloat32Consumers(const MInstruction* ins) {
+ if (ins->isImplicitlyUsed()) {
+ return false;
+ }
+ bool allConsumerUses = true;
+ for (MUseDefIterator use(ins); allConsumerUses && use; use++) {
+ allConsumerUses &= use.def()->canConsumeFloat32(use.use());
+ }
+ return allConsumerUses;
+}
+
+#ifdef JS_JITSPEW
+static const char* OpcodeName(MDefinition::Opcode op) {
+ static const char* const names[] = {
+# define NAME(x) #x,
+ MIR_OPCODE_LIST(NAME)
+# undef NAME
+ };
+ return names[unsigned(op)];
+}
+
+void MDefinition::PrintOpcodeName(GenericPrinter& out, Opcode op) {
+ const char* name = OpcodeName(op);
+ size_t len = strlen(name);
+ for (size_t i = 0; i < len; i++) {
+ out.printf("%c", unicode::ToLowerCase(name[i]));
+ }
+}
+
+uint32_t js::jit::GetMBasicBlockId(const MBasicBlock* block) {
+ return block->id();
+}
+#endif
+
+static MConstant* EvaluateInt64ConstantOperands(TempAllocator& alloc,
+ MBinaryInstruction* ins) {
+ MDefinition* left = ins->getOperand(0);
+ MDefinition* right = ins->getOperand(1);
+
+ if (!left->isConstant() || !right->isConstant()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(left->type() == MIRType::Int64);
+ MOZ_ASSERT(right->type() == MIRType::Int64);
+
+ int64_t lhs = left->toConstant()->toInt64();
+ int64_t rhs = right->toConstant()->toInt64();
+ int64_t ret;
+
+ switch (ins->op()) {
+ case MDefinition::Opcode::BitAnd:
+ ret = lhs & rhs;
+ break;
+ case MDefinition::Opcode::BitOr:
+ ret = lhs | rhs;
+ break;
+ case MDefinition::Opcode::BitXor:
+ ret = lhs ^ rhs;
+ break;
+ case MDefinition::Opcode::Lsh:
+ ret = lhs << (rhs & 0x3F);
+ break;
+ case MDefinition::Opcode::Rsh:
+ ret = lhs >> (rhs & 0x3F);
+ break;
+ case MDefinition::Opcode::Ursh:
+ ret = uint64_t(lhs) >> (uint64_t(rhs) & 0x3F);
+ break;
+ case MDefinition::Opcode::Add:
+ ret = lhs + rhs;
+ break;
+ case MDefinition::Opcode::Sub:
+ ret = lhs - rhs;
+ break;
+ case MDefinition::Opcode::Mul:
+ ret = lhs * rhs;
+ break;
+ case MDefinition::Opcode::Div:
+ if (rhs == 0) {
+ // Division by zero will trap at runtime.
+ return nullptr;
+ }
+ if (ins->toDiv()->isUnsigned()) {
+ ret = int64_t(uint64_t(lhs) / uint64_t(rhs));
+ } else if (lhs == INT64_MIN || rhs == -1) {
+ // Overflow will trap at runtime.
+ return nullptr;
+ } else {
+ ret = lhs / rhs;
+ }
+ break;
+ case MDefinition::Opcode::Mod:
+ if (rhs == 0) {
+ // Division by zero will trap at runtime.
+ return nullptr;
+ }
+ if (!ins->toMod()->isUnsigned() && (lhs < 0 || rhs < 0)) {
+ // Handle all negative values at runtime, for simplicity.
+ return nullptr;
+ }
+ ret = int64_t(uint64_t(lhs) % uint64_t(rhs));
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+
+ return MConstant::NewInt64(alloc, ret);
+}
+
+static MConstant* EvaluateConstantOperands(TempAllocator& alloc,
+ MBinaryInstruction* ins,
+ bool* ptypeChange = nullptr) {
+ MDefinition* left = ins->getOperand(0);
+ MDefinition* right = ins->getOperand(1);
+
+ MOZ_ASSERT(IsTypeRepresentableAsDouble(left->type()));
+ MOZ_ASSERT(IsTypeRepresentableAsDouble(right->type()));
+
+ if (!left->isConstant() || !right->isConstant()) {
+ return nullptr;
+ }
+
+ MConstant* lhs = left->toConstant();
+ MConstant* rhs = right->toConstant();
+ double ret = JS::GenericNaN();
+
+ switch (ins->op()) {
+ case MDefinition::Opcode::BitAnd:
+ ret = double(lhs->toInt32() & rhs->toInt32());
+ break;
+ case MDefinition::Opcode::BitOr:
+ ret = double(lhs->toInt32() | rhs->toInt32());
+ break;
+ case MDefinition::Opcode::BitXor:
+ ret = double(lhs->toInt32() ^ rhs->toInt32());
+ break;
+ case MDefinition::Opcode::Lsh:
+ ret = double(uint32_t(lhs->toInt32()) << (rhs->toInt32() & 0x1F));
+ break;
+ case MDefinition::Opcode::Rsh:
+ ret = double(lhs->toInt32() >> (rhs->toInt32() & 0x1F));
+ break;
+ case MDefinition::Opcode::Ursh:
+ ret = double(uint32_t(lhs->toInt32()) >> (rhs->toInt32() & 0x1F));
+ break;
+ case MDefinition::Opcode::Add:
+ ret = lhs->numberToDouble() + rhs->numberToDouble();
+ break;
+ case MDefinition::Opcode::Sub:
+ ret = lhs->numberToDouble() - rhs->numberToDouble();
+ break;
+ case MDefinition::Opcode::Mul:
+ ret = lhs->numberToDouble() * rhs->numberToDouble();
+ break;
+ case MDefinition::Opcode::Div:
+ if (ins->toDiv()->isUnsigned()) {
+ if (rhs->isInt32(0)) {
+ if (ins->toDiv()->trapOnError()) {
+ return nullptr;
+ }
+ ret = 0.0;
+ } else {
+ ret = double(uint32_t(lhs->toInt32()) / uint32_t(rhs->toInt32()));
+ }
+ } else {
+ ret = NumberDiv(lhs->numberToDouble(), rhs->numberToDouble());
+ }
+ break;
+ case MDefinition::Opcode::Mod:
+ if (ins->toMod()->isUnsigned()) {
+ if (rhs->isInt32(0)) {
+ if (ins->toMod()->trapOnError()) {
+ return nullptr;
+ }
+ ret = 0.0;
+ } else {
+ ret = double(uint32_t(lhs->toInt32()) % uint32_t(rhs->toInt32()));
+ }
+ } else {
+ ret = NumberMod(lhs->numberToDouble(), rhs->numberToDouble());
+ }
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+
+ if (ins->type() == MIRType::Float32) {
+ return MConstant::NewFloat32(alloc, float(ret));
+ }
+ if (ins->type() == MIRType::Double) {
+ return MConstant::New(alloc, DoubleValue(ret));
+ }
+
+ Value retVal;
+ retVal.setNumber(JS::CanonicalizeNaN(ret));
+
+ // If this was an int32 operation but the result isn't an int32 (for
+ // example, a division where the numerator isn't evenly divisible by the
+ // denominator), decline folding.
+ MOZ_ASSERT(ins->type() == MIRType::Int32);
+ if (!retVal.isInt32()) {
+ if (ptypeChange) {
+ *ptypeChange = true;
+ }
+ return nullptr;
+ }
+
+ return MConstant::New(alloc, retVal);
+}
+
+static MMul* EvaluateExactReciprocal(TempAllocator& alloc, MDiv* ins) {
+ // we should fold only when it is a floating point operation
+ if (!IsFloatingPointType(ins->type())) {
+ return nullptr;
+ }
+
+ MDefinition* left = ins->getOperand(0);
+ MDefinition* right = ins->getOperand(1);
+
+ if (!right->isConstant()) {
+ return nullptr;
+ }
+
+ int32_t num;
+ if (!mozilla::NumberIsInt32(right->toConstant()->numberToDouble(), &num)) {
+ return nullptr;
+ }
+
+ // check if rhs is a power of two
+ if (mozilla::Abs(num) & (mozilla::Abs(num) - 1)) {
+ return nullptr;
+ }
+
+ Value ret;
+ ret.setDouble(1.0 / double(num));
+
+ MConstant* foldedRhs;
+ if (ins->type() == MIRType::Float32) {
+ foldedRhs = MConstant::NewFloat32(alloc, ret.toDouble());
+ } else {
+ foldedRhs = MConstant::New(alloc, ret);
+ }
+
+ MOZ_ASSERT(foldedRhs->type() == ins->type());
+ ins->block()->insertBefore(ins, foldedRhs);
+
+ MMul* mul = MMul::New(alloc, left, foldedRhs, ins->type());
+ mul->setMustPreserveNaN(ins->mustPreserveNaN());
+ return mul;
+}
+
+#ifdef JS_JITSPEW
+const char* MDefinition::opName() const { return OpcodeName(op()); }
+
+void MDefinition::printName(GenericPrinter& out) const {
+ PrintOpcodeName(out, op());
+ out.printf("%u", id());
+}
+#endif
+
+HashNumber MDefinition::valueHash() const {
+ HashNumber out = HashNumber(op());
+ for (size_t i = 0, e = numOperands(); i < e; i++) {
+ out = addU32ToHash(out, getOperand(i)->id());
+ }
+ if (MDefinition* dep = dependency()) {
+ out = addU32ToHash(out, dep->id());
+ }
+ return out;
+}
+
+HashNumber MNullaryInstruction::valueHash() const {
+ HashNumber hash = HashNumber(op());
+ if (MDefinition* dep = dependency()) {
+ hash = addU32ToHash(hash, dep->id());
+ }
+ MOZ_ASSERT(hash == MDefinition::valueHash());
+ return hash;
+}
+
+HashNumber MUnaryInstruction::valueHash() const {
+ HashNumber hash = HashNumber(op());
+ hash = addU32ToHash(hash, getOperand(0)->id());
+ if (MDefinition* dep = dependency()) {
+ hash = addU32ToHash(hash, dep->id());
+ }
+ MOZ_ASSERT(hash == MDefinition::valueHash());
+ return hash;
+}
+
+HashNumber MBinaryInstruction::valueHash() const {
+ HashNumber hash = HashNumber(op());
+ hash = addU32ToHash(hash, getOperand(0)->id());
+ hash = addU32ToHash(hash, getOperand(1)->id());
+ if (MDefinition* dep = dependency()) {
+ hash = addU32ToHash(hash, dep->id());
+ }
+ MOZ_ASSERT(hash == MDefinition::valueHash());
+ return hash;
+}
+
+HashNumber MTernaryInstruction::valueHash() const {
+ HashNumber hash = HashNumber(op());
+ hash = addU32ToHash(hash, getOperand(0)->id());
+ hash = addU32ToHash(hash, getOperand(1)->id());
+ hash = addU32ToHash(hash, getOperand(2)->id());
+ if (MDefinition* dep = dependency()) {
+ hash = addU32ToHash(hash, dep->id());
+ }
+ MOZ_ASSERT(hash == MDefinition::valueHash());
+ return hash;
+}
+
+HashNumber MQuaternaryInstruction::valueHash() const {
+ HashNumber hash = HashNumber(op());
+ hash = addU32ToHash(hash, getOperand(0)->id());
+ hash = addU32ToHash(hash, getOperand(1)->id());
+ hash = addU32ToHash(hash, getOperand(2)->id());
+ hash = addU32ToHash(hash, getOperand(3)->id());
+ if (MDefinition* dep = dependency()) {
+ hash = addU32ToHash(hash, dep->id());
+ }
+ MOZ_ASSERT(hash == MDefinition::valueHash());
+ return hash;
+}
+
+const MDefinition* MDefinition::skipObjectGuards() const {
+ const MDefinition* result = this;
+ // These instructions don't modify the object and just guard specific
+ // properties.
+ while (true) {
+ if (result->isGuardShape()) {
+ result = result->toGuardShape()->object();
+ continue;
+ }
+ if (result->isGuardNullProto()) {
+ result = result->toGuardNullProto()->object();
+ continue;
+ }
+ if (result->isGuardProto()) {
+ result = result->toGuardProto()->object();
+ continue;
+ }
+
+ break;
+ }
+
+ return result;
+}
+
+bool MDefinition::congruentIfOperandsEqual(const MDefinition* ins) const {
+ if (op() != ins->op()) {
+ return false;
+ }
+
+ if (type() != ins->type()) {
+ return false;
+ }
+
+ if (isEffectful() || ins->isEffectful()) {
+ return false;
+ }
+
+ if (numOperands() != ins->numOperands()) {
+ return false;
+ }
+
+ for (size_t i = 0, e = numOperands(); i < e; i++) {
+ if (getOperand(i) != ins->getOperand(i)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+MDefinition* MDefinition::foldsTo(TempAllocator& alloc) {
+ // In the default case, there are no constants to fold.
+ return this;
+}
+
+bool MDefinition::mightBeMagicType() const {
+ if (IsMagicType(type())) {
+ return true;
+ }
+
+ if (MIRType::Value != type()) {
+ return false;
+ }
+
+ return true;
+}
+
+bool MDefinition::definitelyType(std::initializer_list<MIRType> types) const {
+#ifdef DEBUG
+ // Only support specialized, non-magic types.
+ auto isSpecializedNonMagic = [](MIRType type) {
+ return type <= MIRType::Object;
+ };
+#endif
+
+ MOZ_ASSERT(types.size() > 0);
+ MOZ_ASSERT(std::all_of(types.begin(), types.end(), isSpecializedNonMagic));
+
+ if (type() == MIRType::Value) {
+ return false;
+ }
+
+ return std::find(types.begin(), types.end(), type()) != types.end();
+}
+
+MDefinition* MInstruction::foldsToStore(TempAllocator& alloc) {
+ if (!dependency()) {
+ return nullptr;
+ }
+
+ MDefinition* store = dependency();
+ if (mightAlias(store) != AliasType::MustAlias) {
+ return nullptr;
+ }
+
+ if (!store->block()->dominates(block())) {
+ return nullptr;
+ }
+
+ MDefinition* value;
+ switch (store->op()) {
+ case Opcode::StoreFixedSlot:
+ value = store->toStoreFixedSlot()->value();
+ break;
+ case Opcode::StoreDynamicSlot:
+ value = store->toStoreDynamicSlot()->value();
+ break;
+ case Opcode::StoreElement:
+ value = store->toStoreElement()->value();
+ break;
+ default:
+ MOZ_CRASH("unknown store");
+ }
+
+ // If the type are matching then we return the value which is used as
+ // argument of the store.
+ if (value->type() != type()) {
+ // If we expect to read a type which is more generic than the type seen
+ // by the store, then we box the value used by the store.
+ if (type() != MIRType::Value) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(value->type() < MIRType::Value);
+ MBox* box = MBox::New(alloc, value);
+ value = box;
+ }
+
+ return value;
+}
+
+void MDefinition::analyzeEdgeCasesForward() {}
+
+void MDefinition::analyzeEdgeCasesBackward() {}
+
+void MInstruction::setResumePoint(MResumePoint* resumePoint) {
+ MOZ_ASSERT(!resumePoint_);
+ resumePoint_ = resumePoint;
+ resumePoint_->setInstruction(this);
+}
+
+void MInstruction::stealResumePoint(MInstruction* other) {
+ MResumePoint* resumePoint = other->resumePoint_;
+ other->resumePoint_ = nullptr;
+
+ resumePoint->resetInstruction();
+ setResumePoint(resumePoint);
+}
+
+void MInstruction::moveResumePointAsEntry() {
+ MOZ_ASSERT(isNop());
+ block()->clearEntryResumePoint();
+ block()->setEntryResumePoint(resumePoint_);
+ resumePoint_->resetInstruction();
+ resumePoint_ = nullptr;
+}
+
+void MInstruction::clearResumePoint() {
+ resumePoint_->resetInstruction();
+ block()->discardPreAllocatedResumePoint(resumePoint_);
+ resumePoint_ = nullptr;
+}
+
+MDefinition* MTest::foldsDoubleNegation(TempAllocator& alloc) {
+ MDefinition* op = getOperand(0);
+
+ if (op->isNot()) {
+ // If the operand of the Not is itself a Not, they cancel out.
+ MDefinition* opop = op->getOperand(0);
+ if (opop->isNot()) {
+ return MTest::New(alloc, opop->toNot()->input(), ifTrue(), ifFalse());
+ }
+ return MTest::New(alloc, op->toNot()->input(), ifFalse(), ifTrue());
+ }
+ return nullptr;
+}
+
+MDefinition* MTest::foldsConstant(TempAllocator& alloc) {
+ MDefinition* op = getOperand(0);
+ if (MConstant* opConst = op->maybeConstantValue()) {
+ bool b;
+ if (opConst->valueToBoolean(&b)) {
+ return MGoto::New(alloc, b ? ifTrue() : ifFalse());
+ }
+ }
+ return nullptr;
+}
+
+MDefinition* MTest::foldsTypes(TempAllocator& alloc) {
+ MDefinition* op = getOperand(0);
+
+ switch (op->type()) {
+ case MIRType::Undefined:
+ case MIRType::Null:
+ return MGoto::New(alloc, ifFalse());
+ case MIRType::Symbol:
+ return MGoto::New(alloc, ifTrue());
+ default:
+ break;
+ }
+ return nullptr;
+}
+
+class UsesIterator {
+ MDefinition* def_;
+
+ public:
+ explicit UsesIterator(MDefinition* def) : def_(def) {}
+ auto begin() const { return def_->usesBegin(); }
+ auto end() const { return def_->usesEnd(); }
+};
+
+static bool AllInstructionsDeadIfUnused(MBasicBlock* block) {
+ for (auto* ins : *block) {
+ // Skip trivial instructions.
+ if (ins->isNop() || ins->isGoto()) {
+ continue;
+ }
+
+ // All uses must be within the current block.
+ for (auto* use : UsesIterator(ins)) {
+ if (use->consumer()->block() != block) {
+ return false;
+ }
+ }
+
+ // All instructions within this block must be dead if unused.
+ if (!DeadIfUnused(ins)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+MDefinition* MTest::foldsNeedlessControlFlow(TempAllocator& alloc) {
+ // All instructions within both successors need be dead if unused.
+ if (!AllInstructionsDeadIfUnused(ifTrue()) ||
+ !AllInstructionsDeadIfUnused(ifFalse())) {
+ return nullptr;
+ }
+
+ // Both successors must have the same target successor.
+ if (ifTrue()->numSuccessors() != 1 || ifFalse()->numSuccessors() != 1) {
+ return nullptr;
+ }
+ if (ifTrue()->getSuccessor(0) != ifFalse()->getSuccessor(0)) {
+ return nullptr;
+ }
+
+ // The target successor's phis must be redundant. Redundant phis should have
+ // been removed in an earlier pass, so only check if any phis are present,
+ // which is a stronger condition.
+ if (ifTrue()->successorWithPhis()) {
+ return nullptr;
+ }
+
+ return MGoto::New(alloc, ifTrue());
+}
+
+MDefinition* MTest::foldsTo(TempAllocator& alloc) {
+ if (MDefinition* def = foldsDoubleNegation(alloc)) {
+ return def;
+ }
+
+ if (MDefinition* def = foldsConstant(alloc)) {
+ return def;
+ }
+
+ if (MDefinition* def = foldsTypes(alloc)) {
+ return def;
+ }
+
+ if (MDefinition* def = foldsNeedlessControlFlow(alloc)) {
+ return def;
+ }
+
+ return this;
+}
+
+AliasSet MThrow::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ExceptionState);
+}
+
+AliasSet MNewArrayDynamicLength::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ExceptionState);
+}
+
+AliasSet MNewTypedArrayDynamicLength::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ExceptionState);
+}
+
+#ifdef JS_JITSPEW
+void MDefinition::printOpcode(GenericPrinter& out) const {
+ PrintOpcodeName(out, op());
+ for (size_t j = 0, e = numOperands(); j < e; j++) {
+ out.printf(" ");
+ if (getUseFor(j)->hasProducer()) {
+ getOperand(j)->printName(out);
+ out.printf(":%s", StringFromMIRType(getOperand(j)->type()));
+ } else {
+ out.printf("(null)");
+ }
+ }
+}
+
+void MDefinition::dump(GenericPrinter& out) const {
+ printName(out);
+ out.printf(":%s", StringFromMIRType(type()));
+ out.printf(" = ");
+ printOpcode(out);
+ out.printf("\n");
+
+ if (isInstruction()) {
+ if (MResumePoint* resume = toInstruction()->resumePoint()) {
+ resume->dump(out);
+ }
+ }
+}
+
+void MDefinition::dump() const {
+ Fprinter out(stderr);
+ dump(out);
+ out.finish();
+}
+
+void MDefinition::dumpLocation(GenericPrinter& out) const {
+ MResumePoint* rp = nullptr;
+ const char* linkWord = nullptr;
+ if (isInstruction() && toInstruction()->resumePoint()) {
+ rp = toInstruction()->resumePoint();
+ linkWord = "at";
+ } else {
+ rp = block()->entryResumePoint();
+ linkWord = "after";
+ }
+
+ while (rp) {
+ JSScript* script = rp->block()->info().script();
+ uint32_t lineno = PCToLineNumber(rp->block()->info().script(), rp->pc());
+ out.printf(" %s %s:%u\n", linkWord, script->filename(), lineno);
+ rp = rp->caller();
+ linkWord = "in";
+ }
+}
+
+void MDefinition::dumpLocation() const {
+ Fprinter out(stderr);
+ dumpLocation(out);
+ out.finish();
+}
+#endif
+
+#ifdef DEBUG
+bool MDefinition::trackedSiteMatchesBlock(const BytecodeSite* site) const {
+ return site == block()->trackedSite();
+}
+#endif
+
+#if defined(DEBUG) || defined(JS_JITSPEW)
+size_t MDefinition::useCount() const {
+ size_t count = 0;
+ for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
+ count++;
+ }
+ return count;
+}
+
+size_t MDefinition::defUseCount() const {
+ size_t count = 0;
+ for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
+ if ((*i)->consumer()->isDefinition()) {
+ count++;
+ }
+ }
+ return count;
+}
+#endif
+
+bool MDefinition::hasOneUse() const {
+ MUseIterator i(uses_.begin());
+ if (i == uses_.end()) {
+ return false;
+ }
+ i++;
+ return i == uses_.end();
+}
+
+bool MDefinition::hasOneDefUse() const {
+ bool hasOneDefUse = false;
+ for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
+ if (!(*i)->consumer()->isDefinition()) {
+ continue;
+ }
+
+ // We already have a definition use. So 1+
+ if (hasOneDefUse) {
+ return false;
+ }
+
+ // We saw one definition. Loop to test if there is another.
+ hasOneDefUse = true;
+ }
+
+ return hasOneDefUse;
+}
+
+bool MDefinition::hasDefUses() const {
+ for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
+ if ((*i)->consumer()->isDefinition()) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+bool MDefinition::hasLiveDefUses() const {
+ for (MUseIterator i(uses_.begin()); i != uses_.end(); i++) {
+ MNode* ins = (*i)->consumer();
+ if (ins->isDefinition()) {
+ if (!ins->toDefinition()->isRecoveredOnBailout()) {
+ return true;
+ }
+ } else {
+ MOZ_ASSERT(ins->isResumePoint());
+ if (!ins->toResumePoint()->isRecoverableOperand(*i)) {
+ return true;
+ }
+ }
+ }
+
+ return false;
+}
+
+MDefinition* MDefinition::maybeSingleDefUse() const {
+ MUseDefIterator use(this);
+ if (!use) {
+ // No def-uses.
+ return nullptr;
+ }
+
+ MDefinition* useDef = use.def();
+
+ use++;
+ if (use) {
+ // More than one def-use.
+ return nullptr;
+ }
+
+ return useDef;
+}
+
+MDefinition* MDefinition::maybeMostRecentlyAddedDefUse() const {
+ MUseDefIterator use(this);
+ if (!use) {
+ // No def-uses.
+ return nullptr;
+ }
+
+ MDefinition* mostRecentUse = use.def();
+
+#ifdef DEBUG
+ // This function relies on addUse adding new uses to the front of the list.
+ // Check this invariant by asserting the next few uses are 'older'. Skip this
+ // for phis because setBackedge can add a new use for a loop phi even if the
+ // loop body has a use with an id greater than the loop phi's id.
+ if (!mostRecentUse->isPhi()) {
+ static constexpr size_t NumUsesToCheck = 3;
+ use++;
+ for (size_t i = 0; use && i < NumUsesToCheck; i++, use++) {
+ MOZ_ASSERT(use.def()->id() <= mostRecentUse->id());
+ }
+ }
+#endif
+
+ return mostRecentUse;
+}
+
+void MDefinition::replaceAllUsesWith(MDefinition* dom) {
+ for (size_t i = 0, e = numOperands(); i < e; ++i) {
+ getOperand(i)->setImplicitlyUsedUnchecked();
+ }
+
+ justReplaceAllUsesWith(dom);
+}
+
+void MDefinition::justReplaceAllUsesWith(MDefinition* dom) {
+ MOZ_ASSERT(dom != nullptr);
+ MOZ_ASSERT(dom != this);
+
+ // Carry over the fact the value has uses which are no longer inspectable
+ // with the graph.
+ if (isImplicitlyUsed()) {
+ dom->setImplicitlyUsedUnchecked();
+ }
+
+ for (MUseIterator i(usesBegin()), e(usesEnd()); i != e; ++i) {
+ i->setProducerUnchecked(dom);
+ }
+ dom->uses_.takeElements(uses_);
+}
+
+bool MDefinition::optimizeOutAllUses(TempAllocator& alloc) {
+ for (MUseIterator i(usesBegin()), e(usesEnd()); i != e;) {
+ MUse* use = *i++;
+ MConstant* constant = use->consumer()->block()->optimizedOutConstant(alloc);
+ if (!alloc.ensureBallast()) {
+ return false;
+ }
+
+ // Update the resume point operand to use the optimized-out constant.
+ use->setProducerUnchecked(constant);
+ constant->addUseUnchecked(use);
+ }
+
+ // Remove dangling pointers.
+ this->uses_.clear();
+ return true;
+}
+
+void MDefinition::replaceAllLiveUsesWith(MDefinition* dom) {
+ for (MUseIterator i(usesBegin()), e(usesEnd()); i != e;) {
+ MUse* use = *i++;
+ MNode* consumer = use->consumer();
+ if (consumer->isResumePoint()) {
+ continue;
+ }
+ if (consumer->isDefinition() &&
+ consumer->toDefinition()->isRecoveredOnBailout()) {
+ continue;
+ }
+
+ // Update the operand to use the dominating definition.
+ use->replaceProducer(dom);
+ }
+}
+
+MConstant* MConstant::New(TempAllocator& alloc, const Value& v) {
+ return new (alloc) MConstant(alloc, v);
+}
+
+MConstant* MConstant::New(TempAllocator::Fallible alloc, const Value& v) {
+ return new (alloc) MConstant(alloc.alloc, v);
+}
+
+MConstant* MConstant::NewFloat32(TempAllocator& alloc, double d) {
+ MOZ_ASSERT(IsNaN(d) || d == double(float(d)));
+ return new (alloc) MConstant(float(d));
+}
+
+MConstant* MConstant::NewInt64(TempAllocator& alloc, int64_t i) {
+ return new (alloc) MConstant(MIRType::Int64, i);
+}
+
+MConstant* MConstant::NewIntPtr(TempAllocator& alloc, intptr_t i) {
+ return new (alloc) MConstant(MIRType::IntPtr, i);
+}
+
+MConstant* MConstant::New(TempAllocator& alloc, const Value& v, MIRType type) {
+ if (type == MIRType::Float32) {
+ return NewFloat32(alloc, v.toNumber());
+ }
+ MConstant* res = New(alloc, v);
+ MOZ_ASSERT(res->type() == type);
+ return res;
+}
+
+MConstant* MConstant::NewObject(TempAllocator& alloc, JSObject* v) {
+ return new (alloc) MConstant(v);
+}
+
+MConstant* MConstant::NewShape(TempAllocator& alloc, Shape* s) {
+ return new (alloc) MConstant(s);
+}
+
+static MIRType MIRTypeFromValue(const js::Value& vp) {
+ if (vp.isDouble()) {
+ return MIRType::Double;
+ }
+ if (vp.isMagic()) {
+ switch (vp.whyMagic()) {
+ case JS_OPTIMIZED_OUT:
+ return MIRType::MagicOptimizedOut;
+ case JS_ELEMENTS_HOLE:
+ return MIRType::MagicHole;
+ case JS_IS_CONSTRUCTING:
+ return MIRType::MagicIsConstructing;
+ case JS_UNINITIALIZED_LEXICAL:
+ return MIRType::MagicUninitializedLexical;
+ default:
+ MOZ_ASSERT_UNREACHABLE("Unexpected magic constant");
+ }
+ }
+ return MIRTypeFromValueType(vp.extractNonDoubleType());
+}
+
+MConstant::MConstant(TempAllocator& alloc, const js::Value& vp)
+ : MNullaryInstruction(classOpcode) {
+ setResultType(MIRTypeFromValue(vp));
+
+ MOZ_ASSERT(payload_.asBits == 0);
+
+ switch (type()) {
+ case MIRType::Undefined:
+ case MIRType::Null:
+ break;
+ case MIRType::Boolean:
+ payload_.b = vp.toBoolean();
+ break;
+ case MIRType::Int32:
+ payload_.i32 = vp.toInt32();
+ break;
+ case MIRType::Double:
+ payload_.d = vp.toDouble();
+ break;
+ case MIRType::String:
+ MOZ_ASSERT(!IsInsideNursery(vp.toString()));
+ MOZ_ASSERT(vp.toString()->isLinear());
+ payload_.str = vp.toString();
+ break;
+ case MIRType::Symbol:
+ payload_.sym = vp.toSymbol();
+ break;
+ case MIRType::BigInt:
+ MOZ_ASSERT(!IsInsideNursery(vp.toBigInt()));
+ payload_.bi = vp.toBigInt();
+ break;
+ case MIRType::Object:
+ MOZ_ASSERT(!IsInsideNursery(&vp.toObject()));
+ payload_.obj = &vp.toObject();
+ break;
+ case MIRType::MagicOptimizedOut:
+ case MIRType::MagicHole:
+ case MIRType::MagicIsConstructing:
+ case MIRType::MagicUninitializedLexical:
+ break;
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+
+ setMovable();
+}
+
+MConstant::MConstant(JSObject* obj) : MNullaryInstruction(classOpcode) {
+ MOZ_ASSERT(!IsInsideNursery(obj));
+ setResultType(MIRType::Object);
+ payload_.obj = obj;
+ setMovable();
+}
+
+MConstant::MConstant(Shape* shape) : MNullaryInstruction(classOpcode) {
+ setResultType(MIRType::Shape);
+ payload_.shape = shape;
+ setMovable();
+}
+
+MConstant::MConstant(float f) : MNullaryInstruction(classOpcode) {
+ setResultType(MIRType::Float32);
+ payload_.f = f;
+ setMovable();
+}
+
+MConstant::MConstant(MIRType type, int64_t i)
+ : MNullaryInstruction(classOpcode) {
+ MOZ_ASSERT(type == MIRType::Int64 || type == MIRType::IntPtr);
+ setResultType(type);
+ if (type == MIRType::Int64) {
+ payload_.i64 = i;
+ } else {
+ payload_.iptr = i;
+ }
+ setMovable();
+}
+
+#ifdef DEBUG
+void MConstant::assertInitializedPayload() const {
+ // valueHash() and equals() expect the unused payload bits to be
+ // initialized to zero. Assert this in debug builds.
+
+ switch (type()) {
+ case MIRType::Int32:
+ case MIRType::Float32:
+# if MOZ_LITTLE_ENDIAN()
+ MOZ_ASSERT((payload_.asBits >> 32) == 0);
+# else
+ MOZ_ASSERT((payload_.asBits << 32) == 0);
+# endif
+ break;
+ case MIRType::Boolean:
+# if MOZ_LITTLE_ENDIAN()
+ MOZ_ASSERT((payload_.asBits >> 1) == 0);
+# else
+ MOZ_ASSERT((payload_.asBits & ~(1ULL << 56)) == 0);
+# endif
+ break;
+ case MIRType::Double:
+ case MIRType::Int64:
+ break;
+ case MIRType::String:
+ case MIRType::Object:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ case MIRType::IntPtr:
+ case MIRType::Shape:
+# if MOZ_LITTLE_ENDIAN()
+ MOZ_ASSERT_IF(JS_BITS_PER_WORD == 32, (payload_.asBits >> 32) == 0);
+# else
+ MOZ_ASSERT_IF(JS_BITS_PER_WORD == 32, (payload_.asBits << 32) == 0);
+# endif
+ break;
+ default:
+ MOZ_ASSERT(IsNullOrUndefined(type()) || IsMagicType(type()));
+ MOZ_ASSERT(payload_.asBits == 0);
+ break;
+ }
+}
+#endif
+
+static HashNumber ConstantValueHash(MIRType type, uint64_t payload) {
+ // Build a 64-bit value holding both the payload and the type.
+ static const size_t TypeBits = 8;
+ static const size_t TypeShift = 64 - TypeBits;
+ MOZ_ASSERT(uintptr_t(type) <= (1 << TypeBits) - 1);
+ uint64_t bits = (uint64_t(type) << TypeShift) ^ payload;
+
+ // Fold all 64 bits into the 32-bit result. It's tempting to just discard
+ // half of the bits, as this is just a hash, however there are many common
+ // patterns of values where only the low or the high bits vary, so
+ // discarding either side would lead to excessive hash collisions.
+ return (HashNumber)bits ^ (HashNumber)(bits >> 32);
+}
+
+HashNumber MConstant::valueHash() const {
+ static_assert(sizeof(Payload) == sizeof(uint64_t),
+ "Code below assumes payload fits in 64 bits");
+
+ assertInitializedPayload();
+ return ConstantValueHash(type(), payload_.asBits);
+}
+
+// We will in theory get some extra collisions here, but dealing with those
+// should be cheaper than doing the skipObjectGuards for the receiver object.
+HashNumber MConstantProto::valueHash() const {
+ return protoObject()->valueHash();
+}
+
+bool MConstant::congruentTo(const MDefinition* ins) const {
+ return ins->isConstant() && equals(ins->toConstant());
+}
+
+#ifdef JS_JITSPEW
+void MConstant::printOpcode(GenericPrinter& out) const {
+ PrintOpcodeName(out, op());
+ out.printf(" ");
+ switch (type()) {
+ case MIRType::Undefined:
+ out.printf("undefined");
+ break;
+ case MIRType::Null:
+ out.printf("null");
+ break;
+ case MIRType::Boolean:
+ out.printf(toBoolean() ? "true" : "false");
+ break;
+ case MIRType::Int32:
+ out.printf("0x%x", uint32_t(toInt32()));
+ break;
+ case MIRType::Int64:
+ out.printf("0x%" PRIx64, uint64_t(toInt64()));
+ break;
+ case MIRType::IntPtr:
+ out.printf("0x%" PRIxPTR, uintptr_t(toIntPtr()));
+ break;
+ case MIRType::Double:
+ out.printf("%.16g", toDouble());
+ break;
+ case MIRType::Float32: {
+ float val = toFloat32();
+ out.printf("%.16g", val);
+ break;
+ }
+ case MIRType::Object:
+ if (toObject().is<JSFunction>()) {
+ JSFunction* fun = &toObject().as<JSFunction>();
+ if (fun->displayAtom()) {
+ out.put("function ");
+ EscapedStringPrinter(out, fun->displayAtom(), 0);
+ } else {
+ out.put("unnamed function");
+ }
+ if (fun->hasBaseScript()) {
+ BaseScript* script = fun->baseScript();
+ out.printf(" (%s:%u)", script->filename() ? script->filename() : "",
+ script->lineno());
+ }
+ out.printf(" at %p", (void*)fun);
+ break;
+ }
+ out.printf("object %p (%s)", (void*)&toObject(),
+ toObject().getClass()->name);
+ break;
+ case MIRType::Symbol:
+ out.printf("symbol at %p", (void*)toSymbol());
+ break;
+ case MIRType::BigInt:
+ out.printf("BigInt at %p", (void*)toBigInt());
+ break;
+ case MIRType::String:
+ out.printf("string %p", (void*)toString());
+ break;
+ case MIRType::Shape:
+ out.printf("shape at %p", (void*)toShape());
+ break;
+ case MIRType::MagicHole:
+ out.printf("magic hole");
+ break;
+ case MIRType::MagicIsConstructing:
+ out.printf("magic is-constructing");
+ break;
+ case MIRType::MagicOptimizedOut:
+ out.printf("magic optimized-out");
+ break;
+ case MIRType::MagicUninitializedLexical:
+ out.printf("magic uninitialized-lexical");
+ break;
+ default:
+ MOZ_CRASH("unexpected type");
+ }
+}
+#endif
+
+bool MConstant::canProduceFloat32() const {
+ if (!isTypeRepresentableAsDouble()) {
+ return false;
+ }
+
+ if (type() == MIRType::Int32) {
+ return IsFloat32Representable(static_cast<double>(toInt32()));
+ }
+ if (type() == MIRType::Double) {
+ return IsFloat32Representable(toDouble());
+ }
+ MOZ_ASSERT(type() == MIRType::Float32);
+ return true;
+}
+
+Value MConstant::toJSValue() const {
+ // Wasm has types like int64 that cannot be stored as js::Value. It also
+ // doesn't want the NaN canonicalization enforced by js::Value.
+ MOZ_ASSERT(!IsCompilingWasm());
+
+ switch (type()) {
+ case MIRType::Undefined:
+ return UndefinedValue();
+ case MIRType::Null:
+ return NullValue();
+ case MIRType::Boolean:
+ return BooleanValue(toBoolean());
+ case MIRType::Int32:
+ return Int32Value(toInt32());
+ case MIRType::Double:
+ return DoubleValue(toDouble());
+ case MIRType::Float32:
+ return Float32Value(toFloat32());
+ case MIRType::String:
+ return StringValue(toString());
+ case MIRType::Symbol:
+ return SymbolValue(toSymbol());
+ case MIRType::BigInt:
+ return BigIntValue(toBigInt());
+ case MIRType::Object:
+ return ObjectValue(toObject());
+ case MIRType::Shape:
+ return PrivateGCThingValue(toShape());
+ case MIRType::MagicOptimizedOut:
+ return MagicValue(JS_OPTIMIZED_OUT);
+ case MIRType::MagicHole:
+ return MagicValue(JS_ELEMENTS_HOLE);
+ case MIRType::MagicIsConstructing:
+ return MagicValue(JS_IS_CONSTRUCTING);
+ case MIRType::MagicUninitializedLexical:
+ return MagicValue(JS_UNINITIALIZED_LEXICAL);
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+}
+
+bool MConstant::valueToBoolean(bool* res) const {
+ switch (type()) {
+ case MIRType::Boolean:
+ *res = toBoolean();
+ return true;
+ case MIRType::Int32:
+ *res = toInt32() != 0;
+ return true;
+ case MIRType::Int64:
+ *res = toInt64() != 0;
+ return true;
+ case MIRType::Double:
+ *res = !mozilla::IsNaN(toDouble()) && toDouble() != 0.0;
+ return true;
+ case MIRType::Float32:
+ *res = !mozilla::IsNaN(toFloat32()) && toFloat32() != 0.0f;
+ return true;
+ case MIRType::Null:
+ case MIRType::Undefined:
+ *res = false;
+ return true;
+ case MIRType::Symbol:
+ *res = true;
+ return true;
+ case MIRType::BigInt:
+ *res = !toBigInt()->isZero();
+ return true;
+ case MIRType::String:
+ *res = toString()->length() != 0;
+ return true;
+ case MIRType::Object:
+ // TODO(Warp): Lazy groups have been removed.
+ // We have to call EmulatesUndefined but that reads obj->group->clasp
+ // and so it's racy when the object has a lazy group. The main callers
+ // of this (MTest, MNot) already know how to fold the object case, so
+ // just give up.
+ return false;
+ default:
+ MOZ_ASSERT(IsMagicType(type()));
+ return false;
+ }
+}
+
+HashNumber MWasmFloatConstant::valueHash() const {
+#ifdef ENABLE_WASM_SIMD
+ return ConstantValueHash(type(), u.bits_[0] ^ u.bits_[1]);
+#else
+ return ConstantValueHash(type(), u.bits_[0]);
+#endif
+}
+
+bool MWasmFloatConstant::congruentTo(const MDefinition* ins) const {
+ return ins->isWasmFloatConstant() && type() == ins->type() &&
+#ifdef ENABLE_WASM_SIMD
+ u.bits_[1] == ins->toWasmFloatConstant()->u.bits_[1] &&
+#endif
+ u.bits_[0] == ins->toWasmFloatConstant()->u.bits_[0];
+}
+
+HashNumber MWasmNullConstant::valueHash() const {
+ return ConstantValueHash(MIRType::RefOrNull, 0);
+}
+
+#ifdef JS_JITSPEW
+void MControlInstruction::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ for (size_t j = 0; j < numSuccessors(); j++) {
+ if (getSuccessor(j)) {
+ out.printf(" block%u", getSuccessor(j)->id());
+ } else {
+ out.printf(" (null-to-be-patched)");
+ }
+ }
+}
+
+void MCompare::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" %s", CodeName(jsop()));
+}
+
+void MTypeOfIs::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" %s", CodeName(jsop()));
+
+ const char* name = "";
+ switch (jstype()) {
+ case JSTYPE_UNDEFINED:
+ name = "undefined";
+ break;
+ case JSTYPE_OBJECT:
+ name = "object";
+ break;
+ case JSTYPE_FUNCTION:
+ name = "function";
+ break;
+ case JSTYPE_STRING:
+ name = "string";
+ break;
+ case JSTYPE_NUMBER:
+ name = "number";
+ break;
+ case JSTYPE_BOOLEAN:
+ name = "boolean";
+ break;
+ case JSTYPE_SYMBOL:
+ name = "symbol";
+ break;
+ case JSTYPE_BIGINT:
+ name = "bigint";
+ break;
+# ifdef ENABLE_RECORD_TUPLE
+ case JSTYPE_RECORD:
+ case JSTYPE_TUPLE:
+# endif
+ case JSTYPE_LIMIT:
+ MOZ_CRASH("Unexpected type");
+ }
+ out.printf(" '%s'", name);
+}
+
+void MLoadUnboxedScalar::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" %s", Scalar::name(storageType()));
+}
+
+void MLoadDataViewElement::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" %s", Scalar::name(storageType()));
+}
+
+void MAssertRange::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.put(" ");
+ assertedRange()->dump(out);
+}
+
+void MNearbyInt::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ const char* roundingModeStr = nullptr;
+ switch (roundingMode_) {
+ case RoundingMode::Up:
+ roundingModeStr = "(up)";
+ break;
+ case RoundingMode::Down:
+ roundingModeStr = "(down)";
+ break;
+ case RoundingMode::NearestTiesToEven:
+ roundingModeStr = "(nearest ties even)";
+ break;
+ case RoundingMode::TowardsZero:
+ roundingModeStr = "(towards zero)";
+ break;
+ }
+ out.printf(" %s", roundingModeStr);
+}
+#endif
+
+AliasSet MRandom::getAliasSet() const { return AliasSet::Store(AliasSet::RNG); }
+
+MDefinition* MSign::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+ if (!input->isConstant() ||
+ !input->toConstant()->isTypeRepresentableAsDouble()) {
+ return this;
+ }
+
+ double in = input->toConstant()->numberToDouble();
+ double out = js::math_sign_impl(in);
+
+ if (type() == MIRType::Int32) {
+ // Decline folding if this is an int32 operation, but the result type
+ // isn't an int32.
+ Value outValue = NumberValue(out);
+ if (!outValue.isInt32()) {
+ return this;
+ }
+
+ return MConstant::New(alloc, outValue);
+ }
+
+ return MConstant::New(alloc, DoubleValue(out));
+}
+
+const char* MMathFunction::FunctionName(UnaryMathFunction function) {
+ return GetUnaryMathFunctionName(function);
+}
+
+#ifdef JS_JITSPEW
+void MMathFunction::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" %s", FunctionName(function()));
+}
+#endif
+
+MDefinition* MMathFunction::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+ if (!input->isConstant() ||
+ !input->toConstant()->isTypeRepresentableAsDouble()) {
+ return this;
+ }
+
+ UnaryMathFunctionType funPtr = GetUnaryMathFunctionPtr(function());
+
+ double in = input->toConstant()->numberToDouble();
+
+ // The function pointer call can't GC.
+ JS::AutoSuppressGCAnalysis nogc;
+ double out = funPtr(in);
+
+ if (input->type() == MIRType::Float32) {
+ return MConstant::NewFloat32(alloc, out);
+ }
+ return MConstant::New(alloc, DoubleValue(out));
+}
+
+MDefinition* MAtomicIsLockFree::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+ if (!input->isConstant() || input->type() != MIRType::Int32) {
+ return this;
+ }
+
+ int32_t i = input->toConstant()->toInt32();
+ return MConstant::New(alloc, BooleanValue(AtomicOperations::isLockfreeJS(i)));
+}
+
+// Define |THIS_SLOT| as part of this translation unit, as it is used to
+// specialized the parameterized |New| function calls introduced by
+// TRIVIAL_NEW_WRAPPERS.
+const int32_t MParameter::THIS_SLOT;
+
+#ifdef JS_JITSPEW
+void MParameter::printOpcode(GenericPrinter& out) const {
+ PrintOpcodeName(out, op());
+ if (index() == THIS_SLOT) {
+ out.printf(" THIS_SLOT");
+ } else {
+ out.printf(" %d", index());
+ }
+}
+#endif
+
+HashNumber MParameter::valueHash() const {
+ HashNumber hash = MDefinition::valueHash();
+ hash = addU32ToHash(hash, index_);
+ return hash;
+}
+
+bool MParameter::congruentTo(const MDefinition* ins) const {
+ if (!ins->isParameter()) {
+ return false;
+ }
+
+ return ins->toParameter()->index() == index_;
+}
+
+WrappedFunction::WrappedFunction(JSFunction* nativeFun, uint16_t nargs,
+ FunctionFlags flags)
+ : nativeFun_(nativeFun), nargs_(nargs), flags_(flags) {
+ MOZ_ASSERT_IF(nativeFun, isNativeWithoutJitEntry());
+
+#ifdef DEBUG
+ // If we are not running off-main thread we can assert that the
+ // metadata is consistent.
+ if (!CanUseExtraThreads() && nativeFun) {
+ MOZ_ASSERT(nativeFun->nargs() == nargs);
+
+ MOZ_ASSERT(nativeFun->isNativeWithoutJitEntry() ==
+ isNativeWithoutJitEntry());
+ MOZ_ASSERT(nativeFun->hasJitEntry() == hasJitEntry());
+ MOZ_ASSERT(nativeFun->isConstructor() == isConstructor());
+ MOZ_ASSERT(nativeFun->isClassConstructor() == isClassConstructor());
+ }
+#endif
+}
+
+MCall* MCall::New(TempAllocator& alloc, WrappedFunction* target, size_t maxArgc,
+ size_t numActualArgs, bool construct, bool ignoresReturnValue,
+ bool isDOMCall, mozilla::Maybe<DOMObjectKind> objectKind) {
+ MOZ_ASSERT(isDOMCall == objectKind.isSome());
+ MOZ_ASSERT(maxArgc >= numActualArgs);
+ MCall* ins;
+ if (isDOMCall) {
+ MOZ_ASSERT(!construct);
+ ins = new (alloc) MCallDOMNative(target, numActualArgs, *objectKind);
+ } else {
+ ins =
+ new (alloc) MCall(target, numActualArgs, construct, ignoresReturnValue);
+ }
+ if (!ins->init(alloc, maxArgc + NumNonArgumentOperands)) {
+ return nullptr;
+ }
+ return ins;
+}
+
+AliasSet MCallDOMNative::getAliasSet() const {
+ const JSJitInfo* jitInfo = getJitInfo();
+
+ // If we don't know anything about the types of our arguments, we have to
+ // assume that type-coercions can have side-effects, so we need to alias
+ // everything.
+ if (jitInfo->aliasSet() == JSJitInfo::AliasEverything ||
+ !jitInfo->isTypedMethodJitInfo()) {
+ return AliasSet::Store(AliasSet::Any);
+ }
+
+ uint32_t argIndex = 0;
+ const JSTypedMethodJitInfo* methodInfo =
+ reinterpret_cast<const JSTypedMethodJitInfo*>(jitInfo);
+ for (const JSJitInfo::ArgType* argType = methodInfo->argTypes;
+ *argType != JSJitInfo::ArgTypeListEnd; ++argType, ++argIndex) {
+ if (argIndex >= numActualArgs()) {
+ // Passing through undefined can't have side-effects
+ continue;
+ }
+ // getArg(0) is "this", so skip it
+ MDefinition* arg = getArg(argIndex + 1);
+ MIRType actualType = arg->type();
+ // The only way to reliably avoid side-effects given the information we
+ // have here is if we're passing in a known primitive value to an
+ // argument that expects a primitive value.
+ //
+ // XXXbz maybe we need to communicate better information. For example,
+ // a sequence argument will sort of unavoidably have side effects, while
+ // a typed array argument won't have any, but both are claimed to be
+ // JSJitInfo::Object. But if we do that, we need to watch out for our
+ // movability/DCE-ability bits: if we have an arg type that can reliably
+ // throw an exception on conversion, that might not affect our alias set
+ // per se, but it should prevent us being moved or DCE-ed, unless we
+ // know the incoming things match that arg type and won't throw.
+ //
+ if ((actualType == MIRType::Value || actualType == MIRType::Object) ||
+ (*argType & JSJitInfo::Object)) {
+ return AliasSet::Store(AliasSet::Any);
+ }
+ }
+
+ // We checked all the args, and they check out. So we only alias DOM
+ // mutations or alias nothing, depending on the alias set in the jitinfo.
+ if (jitInfo->aliasSet() == JSJitInfo::AliasNone) {
+ return AliasSet::None();
+ }
+
+ MOZ_ASSERT(jitInfo->aliasSet() == JSJitInfo::AliasDOMSets);
+ return AliasSet::Load(AliasSet::DOMProperty);
+}
+
+void MCallDOMNative::computeMovable() {
+ // We are movable if the jitinfo says we can be and if we're also not
+ // effectful. The jitinfo can't check for the latter, since it depends on
+ // the types of our arguments.
+ const JSJitInfo* jitInfo = getJitInfo();
+
+ MOZ_ASSERT_IF(jitInfo->isMovable,
+ jitInfo->aliasSet() != JSJitInfo::AliasEverything);
+
+ if (jitInfo->isMovable && !isEffectful()) {
+ setMovable();
+ }
+}
+
+bool MCallDOMNative::congruentTo(const MDefinition* ins) const {
+ if (!isMovable()) {
+ return false;
+ }
+
+ if (!ins->isCall()) {
+ return false;
+ }
+
+ const MCall* call = ins->toCall();
+
+ if (!call->isCallDOMNative()) {
+ return false;
+ }
+
+ if (getSingleTarget() != call->getSingleTarget()) {
+ return false;
+ }
+
+ if (isConstructing() != call->isConstructing()) {
+ return false;
+ }
+
+ if (numActualArgs() != call->numActualArgs()) {
+ return false;
+ }
+
+ if (!congruentIfOperandsEqual(call)) {
+ return false;
+ }
+
+ // The other call had better be movable at this point!
+ MOZ_ASSERT(call->isMovable());
+
+ return true;
+}
+
+const JSJitInfo* MCallDOMNative::getJitInfo() const {
+ MOZ_ASSERT(getSingleTarget()->hasJitInfo());
+ return getSingleTarget()->jitInfo();
+}
+
+MDefinition* MStringLength::foldsTo(TempAllocator& alloc) {
+ if (string()->isConstant()) {
+ JSString* str = string()->toConstant()->toString();
+ return MConstant::New(alloc, Int32Value(str->length()));
+ }
+
+ // MFromCharCode returns a one-element string.
+ if (string()->isFromCharCode()) {
+ return MConstant::New(alloc, Int32Value(1));
+ }
+
+ return this;
+}
+
+MDefinition* MConcat::foldsTo(TempAllocator& alloc) {
+ if (lhs()->isConstant() && lhs()->toConstant()->toString()->empty()) {
+ return rhs();
+ }
+
+ if (rhs()->isConstant() && rhs()->toConstant()->toString()->empty()) {
+ return lhs();
+ }
+
+ return this;
+}
+
+MDefinition* MCharCodeAt::foldsTo(TempAllocator& alloc) {
+ MDefinition* string = this->string();
+ if (!string->isConstant() && !string->isFromCharCode()) {
+ return this;
+ }
+
+ MDefinition* index = this->index();
+ if (index->isSpectreMaskIndex()) {
+ index = index->toSpectreMaskIndex()->index();
+ }
+ if (!index->isConstant()) {
+ return this;
+ }
+ int32_t idx = index->toConstant()->toInt32();
+
+ // Handle the pattern |s[idx].charCodeAt(0)|.
+ if (string->isFromCharCode()) {
+ if (idx != 0) {
+ return this;
+ }
+
+ // Simplify |CharCodeAt(FromCharCode(CharCodeAt(s, idx)), 0)| to just
+ // |CharCodeAt(s, idx)|.
+ auto* charCode = string->toFromCharCode()->code();
+ if (!charCode->isCharCodeAt()) {
+ return this;
+ }
+
+ return charCode;
+ }
+
+ JSLinearString* str = &string->toConstant()->toString()->asLinear();
+ if (idx < 0 || uint32_t(idx) >= str->length()) {
+ return this;
+ }
+
+ char16_t ch = str->latin1OrTwoByteChar(idx);
+ return MConstant::New(alloc, Int32Value(ch));
+}
+
+template <size_t Arity>
+[[nodiscard]] static bool EnsureFloatInputOrConvert(
+ MAryInstruction<Arity>* owner, TempAllocator& alloc) {
+ MOZ_ASSERT(!IsFloatingPointType(owner->type()),
+ "Floating point types must check consumers");
+
+ if (AllOperandsCanProduceFloat32(owner)) {
+ return true;
+ }
+ ConvertOperandsToDouble(owner, alloc);
+ return false;
+}
+
+template <size_t Arity>
+[[nodiscard]] static bool EnsureFloatConsumersAndInputOrConvert(
+ MAryInstruction<Arity>* owner, TempAllocator& alloc) {
+ MOZ_ASSERT(IsFloatingPointType(owner->type()),
+ "Integer types don't need to check consumers");
+
+ if (AllOperandsCanProduceFloat32(owner) &&
+ CheckUsesAreFloat32Consumers(owner)) {
+ return true;
+ }
+ ConvertOperandsToDouble(owner, alloc);
+ return false;
+}
+
+void MFloor::trySpecializeFloat32(TempAllocator& alloc) {
+ MOZ_ASSERT(type() == MIRType::Int32);
+ if (EnsureFloatInputOrConvert(this, alloc)) {
+ specialization_ = MIRType::Float32;
+ }
+}
+
+void MCeil::trySpecializeFloat32(TempAllocator& alloc) {
+ MOZ_ASSERT(type() == MIRType::Int32);
+ if (EnsureFloatInputOrConvert(this, alloc)) {
+ specialization_ = MIRType::Float32;
+ }
+}
+
+void MRound::trySpecializeFloat32(TempAllocator& alloc) {
+ MOZ_ASSERT(type() == MIRType::Int32);
+ if (EnsureFloatInputOrConvert(this, alloc)) {
+ specialization_ = MIRType::Float32;
+ }
+}
+
+void MTrunc::trySpecializeFloat32(TempAllocator& alloc) {
+ MOZ_ASSERT(type() == MIRType::Int32);
+ if (EnsureFloatInputOrConvert(this, alloc)) {
+ specialization_ = MIRType::Float32;
+ }
+}
+
+void MNearbyInt::trySpecializeFloat32(TempAllocator& alloc) {
+ if (EnsureFloatConsumersAndInputOrConvert(this, alloc)) {
+ specialization_ = MIRType::Float32;
+ setResultType(MIRType::Float32);
+ }
+}
+
+MGoto* MGoto::New(TempAllocator& alloc, MBasicBlock* target) {
+ return new (alloc) MGoto(target);
+}
+
+MGoto* MGoto::New(TempAllocator::Fallible alloc, MBasicBlock* target) {
+ MOZ_ASSERT(target);
+ return new (alloc) MGoto(target);
+}
+
+MGoto* MGoto::New(TempAllocator& alloc) { return new (alloc) MGoto(nullptr); }
+
+#ifdef JS_JITSPEW
+void MUnbox::printOpcode(GenericPrinter& out) const {
+ PrintOpcodeName(out, op());
+ out.printf(" ");
+ getOperand(0)->printName(out);
+ out.printf(" ");
+
+ switch (type()) {
+ case MIRType::Int32:
+ out.printf("to Int32");
+ break;
+ case MIRType::Double:
+ out.printf("to Double");
+ break;
+ case MIRType::Boolean:
+ out.printf("to Boolean");
+ break;
+ case MIRType::String:
+ out.printf("to String");
+ break;
+ case MIRType::Symbol:
+ out.printf("to Symbol");
+ break;
+ case MIRType::BigInt:
+ out.printf("to BigInt");
+ break;
+ case MIRType::Object:
+ out.printf("to Object");
+ break;
+ default:
+ break;
+ }
+
+ switch (mode()) {
+ case Fallible:
+ out.printf(" (fallible)");
+ break;
+ case Infallible:
+ out.printf(" (infallible)");
+ break;
+ default:
+ break;
+ }
+}
+#endif
+
+MDefinition* MUnbox::foldsTo(TempAllocator& alloc) {
+ if (input()->isBox()) {
+ MDefinition* unboxed = input()->toBox()->input();
+
+ // Fold MUnbox(MBox(x)) => x if types match.
+ if (unboxed->type() == type()) {
+ if (fallible()) {
+ unboxed->setImplicitlyUsedUnchecked();
+ }
+ return unboxed;
+ }
+
+ // Fold MUnbox(MBox(x)) => MToDouble(x) if possible.
+ if (type() == MIRType::Double &&
+ IsTypeRepresentableAsDouble(unboxed->type())) {
+ if (unboxed->isConstant()) {
+ return MConstant::New(
+ alloc, DoubleValue(unboxed->toConstant()->numberToDouble()));
+ }
+
+ return MToDouble::New(alloc, unboxed);
+ }
+
+ // MUnbox<Int32>(MBox<Double>(x)) will always fail, even if x can be
+ // represented as an Int32. Fold to avoid unnecessary bailouts.
+ if (type() == MIRType::Int32 && unboxed->type() == MIRType::Double) {
+ auto* folded = MToNumberInt32::New(alloc, unboxed,
+ IntConversionInputKind::NumbersOnly);
+ folded->setGuard();
+ return folded;
+ }
+ }
+
+ return this;
+}
+
+#ifdef DEBUG
+void MPhi::assertLoopPhi() const {
+ // getLoopPredecessorOperand and getLoopBackedgeOperand rely on these
+ // predecessors being at known indices.
+ if (block()->numPredecessors() == 2) {
+ MBasicBlock* pred = block()->getPredecessor(0);
+ MBasicBlock* back = block()->getPredecessor(1);
+ MOZ_ASSERT(pred == block()->loopPredecessor());
+ MOZ_ASSERT(pred->successorWithPhis() == block());
+ MOZ_ASSERT(pred->positionInPhiSuccessor() == 0);
+ MOZ_ASSERT(back == block()->backedge());
+ MOZ_ASSERT(back->successorWithPhis() == block());
+ MOZ_ASSERT(back->positionInPhiSuccessor() == 1);
+ } else {
+ // After we remove fake loop predecessors for loop headers that
+ // are only reachable via OSR, the only predecessor is the
+ // loop backedge.
+ MOZ_ASSERT(block()->numPredecessors() == 1);
+ MOZ_ASSERT(block()->graph().osrBlock());
+ MOZ_ASSERT(!block()->graph().canBuildDominators());
+ MBasicBlock* back = block()->getPredecessor(0);
+ MOZ_ASSERT(back == block()->backedge());
+ MOZ_ASSERT(back->successorWithPhis() == block());
+ MOZ_ASSERT(back->positionInPhiSuccessor() == 0);
+ }
+}
+#endif
+
+MDefinition* MPhi::getLoopPredecessorOperand() const {
+ // This should not be called after removing fake loop predecessors.
+ MOZ_ASSERT(block()->numPredecessors() == 2);
+ assertLoopPhi();
+ return getOperand(0);
+}
+
+MDefinition* MPhi::getLoopBackedgeOperand() const {
+ assertLoopPhi();
+ uint32_t idx = block()->numPredecessors() == 2 ? 1 : 0;
+ return getOperand(idx);
+}
+
+void MPhi::removeOperand(size_t index) {
+ MOZ_ASSERT(index < numOperands());
+ MOZ_ASSERT(getUseFor(index)->index() == index);
+ MOZ_ASSERT(getUseFor(index)->consumer() == this);
+
+ // If we have phi(..., a, b, c, d, ..., z) and we plan
+ // on removing a, then first shift downward so that we have
+ // phi(..., b, c, d, ..., z, z):
+ MUse* p = inputs_.begin() + index;
+ MUse* e = inputs_.end();
+ p->producer()->removeUse(p);
+ for (; p < e - 1; ++p) {
+ MDefinition* producer = (p + 1)->producer();
+ p->setProducerUnchecked(producer);
+ producer->replaceUse(p + 1, p);
+ }
+
+ // truncate the inputs_ list:
+ inputs_.popBack();
+}
+
+void MPhi::removeAllOperands() {
+ for (MUse& p : inputs_) {
+ p.producer()->removeUse(&p);
+ }
+ inputs_.clear();
+}
+
+MDefinition* MPhi::foldsTernary(TempAllocator& alloc) {
+ /* Look if this MPhi is a ternary construct.
+ * This is a very loose term as it actually only checks for
+ *
+ * MTest X
+ * / \
+ * ... ...
+ * \ /
+ * MPhi X Y
+ *
+ * Which we will simply call:
+ * x ? x : y or x ? y : x
+ */
+
+ if (numOperands() != 2) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(block()->numPredecessors() == 2);
+
+ MBasicBlock* pred = block()->immediateDominator();
+ if (!pred || !pred->lastIns()->isTest()) {
+ return nullptr;
+ }
+
+ MTest* test = pred->lastIns()->toTest();
+
+ // True branch may only dominate one edge of MPhi.
+ if (test->ifTrue()->dominates(block()->getPredecessor(0)) ==
+ test->ifTrue()->dominates(block()->getPredecessor(1))) {
+ return nullptr;
+ }
+
+ // False branch may only dominate one edge of MPhi.
+ if (test->ifFalse()->dominates(block()->getPredecessor(0)) ==
+ test->ifFalse()->dominates(block()->getPredecessor(1))) {
+ return nullptr;
+ }
+
+ // True and false branch must dominate different edges of MPhi.
+ if (test->ifTrue()->dominates(block()->getPredecessor(0)) ==
+ test->ifFalse()->dominates(block()->getPredecessor(0))) {
+ return nullptr;
+ }
+
+ // We found a ternary construct.
+ bool firstIsTrueBranch =
+ test->ifTrue()->dominates(block()->getPredecessor(0));
+ MDefinition* trueDef = firstIsTrueBranch ? getOperand(0) : getOperand(1);
+ MDefinition* falseDef = firstIsTrueBranch ? getOperand(1) : getOperand(0);
+
+ // Accept either
+ // testArg ? testArg : constant or
+ // testArg ? constant : testArg
+ if (!trueDef->isConstant() && !falseDef->isConstant()) {
+ return nullptr;
+ }
+
+ MConstant* c =
+ trueDef->isConstant() ? trueDef->toConstant() : falseDef->toConstant();
+ MDefinition* testArg = (trueDef == c) ? falseDef : trueDef;
+ if (testArg != test->input()) {
+ return nullptr;
+ }
+
+ // This check should be a tautology, except that the constant might be the
+ // result of the removal of a branch. In such case the domination scope of
+ // the block which is holding the constant might be incomplete. This
+ // condition is used to prevent doing this optimization based on incomplete
+ // information.
+ //
+ // As GVN removed a branch, it will update the dominations rules before
+ // trying to fold this MPhi again. Thus, this condition does not inhibit
+ // this optimization.
+ MBasicBlock* truePred = block()->getPredecessor(firstIsTrueBranch ? 0 : 1);
+ MBasicBlock* falsePred = block()->getPredecessor(firstIsTrueBranch ? 1 : 0);
+ if (!trueDef->block()->dominates(truePred) ||
+ !falseDef->block()->dominates(falsePred)) {
+ return nullptr;
+ }
+
+ // If testArg is an int32 type we can:
+ // - fold testArg ? testArg : 0 to testArg
+ // - fold testArg ? 0 : testArg to 0
+ if (testArg->type() == MIRType::Int32 && c->numberToDouble() == 0) {
+ testArg->setGuardRangeBailoutsUnchecked();
+
+ // When folding to the constant we need to hoist it.
+ if (trueDef == c && !c->block()->dominates(block())) {
+ c->block()->moveBefore(pred->lastIns(), c);
+ }
+ return trueDef;
+ }
+
+ // If testArg is an double type we can:
+ // - fold testArg ? testArg : 0.0 to MNaNToZero(testArg)
+ if (testArg->type() == MIRType::Double &&
+ mozilla::IsPositiveZero(c->numberToDouble()) && c != trueDef) {
+ MNaNToZero* replace = MNaNToZero::New(alloc, testArg);
+ test->block()->insertBefore(test, replace);
+ return replace;
+ }
+
+ // If testArg is a string type we can:
+ // - fold testArg ? testArg : "" to testArg
+ // - fold testArg ? "" : testArg to ""
+ if (testArg->type() == MIRType::String &&
+ c->toString() == GetJitContext()->runtime->emptyString()) {
+ // When folding to the constant we need to hoist it.
+ if (trueDef == c && !c->block()->dominates(block())) {
+ c->block()->moveBefore(pred->lastIns(), c);
+ }
+ return trueDef;
+ }
+
+ return nullptr;
+}
+
+MDefinition* MPhi::operandIfRedundant() {
+ if (inputs_.length() == 0) {
+ return nullptr;
+ }
+
+ // If this phi is redundant (e.g., phi(a,a) or b=phi(a,this)),
+ // returns the operand that it will always be equal to (a, in
+ // those two cases).
+ MDefinition* first = getOperand(0);
+ for (size_t i = 1, e = numOperands(); i < e; i++) {
+ MDefinition* op = getOperand(i);
+ if (op != first && op != this) {
+ return nullptr;
+ }
+ }
+ return first;
+}
+
+MDefinition* MPhi::foldsTo(TempAllocator& alloc) {
+ if (MDefinition* def = operandIfRedundant()) {
+ return def;
+ }
+
+ if (MDefinition* def = foldsTernary(alloc)) {
+ return def;
+ }
+
+ return this;
+}
+
+bool MPhi::congruentTo(const MDefinition* ins) const {
+ if (!ins->isPhi()) {
+ return false;
+ }
+
+ // Phis in different blocks may have different control conditions.
+ // For example, these phis:
+ //
+ // if (p)
+ // goto a
+ // a:
+ // t = phi(x, y)
+ //
+ // if (q)
+ // goto b
+ // b:
+ // s = phi(x, y)
+ //
+ // have identical operands, but they are not equvalent because t is
+ // effectively p?x:y and s is effectively q?x:y.
+ //
+ // For now, consider phis in different blocks incongruent.
+ if (ins->block() != block()) {
+ return false;
+ }
+
+ return congruentIfOperandsEqual(ins);
+}
+
+bool MPhi::updateForReplacement(MDefinition* def) {
+ // This function is called to fix the current Phi flags using it as a
+ // replacement of the other Phi instruction |def|.
+ //
+ // When dealing with usage analysis, any Use will replace all other values,
+ // such as Unused and Unknown. Unless both are Unused, the merge would be
+ // Unknown.
+ MPhi* other = def->toPhi();
+ if (usageAnalysis_ == PhiUsage::Used ||
+ other->usageAnalysis_ == PhiUsage::Used) {
+ usageAnalysis_ = PhiUsage::Used;
+ } else if (usageAnalysis_ != other->usageAnalysis_) {
+ // this == unused && other == unknown
+ // or this == unknown && other == unused
+ usageAnalysis_ = PhiUsage::Unknown;
+ } else {
+ // this == unused && other == unused
+ // or this == unknown && other = unknown
+ MOZ_ASSERT(usageAnalysis_ == PhiUsage::Unused ||
+ usageAnalysis_ == PhiUsage::Unknown);
+ MOZ_ASSERT(usageAnalysis_ == other->usageAnalysis_);
+ }
+ return true;
+}
+
+/* static */
+bool MPhi::markIteratorPhis(const PhiVector& iterators) {
+ // Find and mark phis that must transitively hold an iterator live.
+
+ Vector<MPhi*, 8, SystemAllocPolicy> worklist;
+
+ for (MPhi* iter : iterators) {
+ if (!iter->isInWorklist()) {
+ if (!worklist.append(iter)) {
+ return false;
+ }
+ iter->setInWorklist();
+ }
+ }
+
+ while (!worklist.empty()) {
+ MPhi* phi = worklist.popCopy();
+ phi->setNotInWorklist();
+
+ phi->setIterator();
+ phi->setImplicitlyUsedUnchecked();
+
+ for (MUseDefIterator iter(phi); iter; iter++) {
+ MDefinition* use = iter.def();
+ if (!use->isInWorklist() && use->isPhi() && !use->toPhi()->isIterator()) {
+ if (!worklist.append(use->toPhi())) {
+ return false;
+ }
+ use->setInWorklist();
+ }
+ }
+ }
+
+ return true;
+}
+
+bool MPhi::typeIncludes(MDefinition* def) {
+ MOZ_ASSERT(!IsMagicType(def->type()));
+
+ if (def->type() == MIRType::Int32 && this->type() == MIRType::Double) {
+ return true;
+ }
+
+ if (def->type() == MIRType::Value) {
+ // This phi must be able to be any value.
+ return this->type() == MIRType::Value;
+ }
+
+ return this->mightBeType(def->type());
+}
+
+void MCall::addArg(size_t argnum, MDefinition* arg) {
+ // The operand vector is initialized in reverse order by WarpBuilder.
+ // It cannot be checked for consistency until all arguments are added.
+ // FixedList doesn't initialize its elements, so do an unchecked init.
+ initOperand(argnum + NumNonArgumentOperands, arg);
+}
+
+static inline bool IsConstant(MDefinition* def, double v) {
+ if (!def->isConstant()) {
+ return false;
+ }
+
+ return NumbersAreIdentical(def->toConstant()->numberToDouble(), v);
+}
+
+MDefinition* MBinaryBitwiseInstruction::foldsTo(TempAllocator& alloc) {
+ // Identity operations are removed (for int32 only) in foldUnnecessaryBitop.
+
+ if (type() == MIRType::Int32) {
+ if (MDefinition* folded = EvaluateConstantOperands(alloc, this)) {
+ return folded;
+ }
+ } else if (type() == MIRType::Int64) {
+ if (MDefinition* folded = EvaluateInt64ConstantOperands(alloc, this)) {
+ return folded;
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MBinaryBitwiseInstruction::foldUnnecessaryBitop() {
+ // It's probably OK to perform this optimization only for int32, as it will
+ // have the greatest effect for asm.js code that is compiled with the JS
+ // pipeline, and that code will not see int64 values.
+
+ if (type() != MIRType::Int32) {
+ return this;
+ }
+
+ // Fold unsigned shift right operator when the second operand is zero and
+ // the only use is an unsigned modulo. Thus, the expression
+ // |(x >>> 0) % y| becomes |x % y|.
+ if (isUrsh() && IsUint32Type(this)) {
+ MDefinition* defUse = maybeSingleDefUse();
+ if (defUse && defUse->isMod() && defUse->toMod()->isUnsigned()) {
+ return getOperand(0);
+ }
+ }
+
+ // Eliminate bitwise operations that are no-ops when used on integer
+ // inputs, such as (x | 0).
+
+ MDefinition* lhs = getOperand(0);
+ MDefinition* rhs = getOperand(1);
+
+ if (IsConstant(lhs, 0)) {
+ return foldIfZero(0);
+ }
+
+ if (IsConstant(rhs, 0)) {
+ return foldIfZero(1);
+ }
+
+ if (IsConstant(lhs, -1)) {
+ return foldIfNegOne(0);
+ }
+
+ if (IsConstant(rhs, -1)) {
+ return foldIfNegOne(1);
+ }
+
+ if (lhs == rhs) {
+ return foldIfEqual();
+ }
+
+ if (maskMatchesRightRange) {
+ MOZ_ASSERT(lhs->isConstant());
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ return foldIfAllBitsSet(0);
+ }
+
+ if (maskMatchesLeftRange) {
+ MOZ_ASSERT(rhs->isConstant());
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+ return foldIfAllBitsSet(1);
+ }
+
+ return this;
+}
+
+static inline bool CanProduceNegativeZero(MDefinition* def) {
+ // Test if this instruction can produce negative zero even when bailing out
+ // and changing types.
+ switch (def->op()) {
+ case MDefinition::Opcode::Constant:
+ if (def->type() == MIRType::Double &&
+ def->toConstant()->toDouble() == -0.0) {
+ return true;
+ }
+ [[fallthrough]];
+ case MDefinition::Opcode::BitAnd:
+ case MDefinition::Opcode::BitOr:
+ case MDefinition::Opcode::BitXor:
+ case MDefinition::Opcode::BitNot:
+ case MDefinition::Opcode::Lsh:
+ case MDefinition::Opcode::Rsh:
+ return false;
+ default:
+ return true;
+ }
+}
+
+static inline bool NeedNegativeZeroCheck(MDefinition* def) {
+ if (def->isGuard() || def->isGuardRangeBailouts()) {
+ return true;
+ }
+
+ // Test if all uses have the same semantics for -0 and 0
+ for (MUseIterator use = def->usesBegin(); use != def->usesEnd(); use++) {
+ if (use->consumer()->isResumePoint()) {
+ return true;
+ }
+
+ MDefinition* use_def = use->consumer()->toDefinition();
+ switch (use_def->op()) {
+ case MDefinition::Opcode::Add: {
+ // If add is truncating -0 and 0 are observed as the same.
+ if (use_def->toAdd()->isTruncated()) {
+ break;
+ }
+
+ // x + y gives -0, when both x and y are -0
+
+ // Figure out the order in which the addition's operands will
+ // execute. EdgeCaseAnalysis::analyzeLate has renumbered the MIR
+ // definitions for us so that this just requires comparing ids.
+ MDefinition* first = use_def->toAdd()->lhs();
+ MDefinition* second = use_def->toAdd()->rhs();
+ if (first->id() > second->id()) {
+ std::swap(first, second);
+ }
+ // Negative zero checks can be removed on the first executed
+ // operand only if it is guaranteed the second executed operand
+ // will produce a value other than -0. While the second is
+ // typed as an int32, a bailout taken between execution of the
+ // operands may change that type and cause a -0 to flow to the
+ // second.
+ //
+ // There is no way to test whether there are any bailouts
+ // between execution of the operands, so remove negative
+ // zero checks from the first only if the second's type is
+ // independent from type changes that may occur after bailing.
+ if (def == first && CanProduceNegativeZero(second)) {
+ return true;
+ }
+
+ // The negative zero check can always be removed on the second
+ // executed operand; by the time this executes the first will have
+ // been evaluated as int32 and the addition's result cannot be -0.
+ break;
+ }
+ case MDefinition::Opcode::Sub: {
+ // If sub is truncating -0 and 0 are observed as the same
+ if (use_def->toSub()->isTruncated()) {
+ break;
+ }
+
+ // x + y gives -0, when x is -0 and y is 0
+
+ // We can remove the negative zero check on the rhs, only if we
+ // are sure the lhs isn't negative zero.
+
+ // The lhs is typed as integer (i.e. not -0.0), but it can bailout
+ // and change type. This should be fine if the lhs is executed
+ // first. However if the rhs is executed first, the lhs can bail,
+ // change type and become -0.0 while the rhs has already been
+ // optimized to not make a difference between zero and negative zero.
+ MDefinition* lhs = use_def->toSub()->lhs();
+ MDefinition* rhs = use_def->toSub()->rhs();
+ if (rhs->id() < lhs->id() && CanProduceNegativeZero(lhs)) {
+ return true;
+ }
+
+ [[fallthrough]];
+ }
+ case MDefinition::Opcode::StoreElement:
+ case MDefinition::Opcode::StoreHoleValueElement:
+ case MDefinition::Opcode::LoadElement:
+ case MDefinition::Opcode::LoadElementHole:
+ case MDefinition::Opcode::LoadUnboxedScalar:
+ case MDefinition::Opcode::LoadDataViewElement:
+ case MDefinition::Opcode::LoadTypedArrayElementHole:
+ case MDefinition::Opcode::CharCodeAt:
+ case MDefinition::Opcode::Mod:
+ case MDefinition::Opcode::InArray:
+ // Only allowed to remove check when definition is the second operand
+ if (use_def->getOperand(0) == def) {
+ return true;
+ }
+ for (size_t i = 2, e = use_def->numOperands(); i < e; i++) {
+ if (use_def->getOperand(i) == def) {
+ return true;
+ }
+ }
+ break;
+ case MDefinition::Opcode::BoundsCheck:
+ // Only allowed to remove check when definition is the first operand
+ if (use_def->toBoundsCheck()->getOperand(1) == def) {
+ return true;
+ }
+ break;
+ case MDefinition::Opcode::ToString:
+ case MDefinition::Opcode::FromCharCode:
+ case MDefinition::Opcode::FromCodePoint:
+ case MDefinition::Opcode::TableSwitch:
+ case MDefinition::Opcode::Compare:
+ case MDefinition::Opcode::BitAnd:
+ case MDefinition::Opcode::BitOr:
+ case MDefinition::Opcode::BitXor:
+ case MDefinition::Opcode::Abs:
+ case MDefinition::Opcode::TruncateToInt32:
+ // Always allowed to remove check. No matter which operand.
+ break;
+ case MDefinition::Opcode::StoreElementHole:
+ case MDefinition::Opcode::StoreTypedArrayElementHole:
+ case MDefinition::Opcode::PostWriteElementBarrier:
+ // Only allowed to remove check when definition is the third operand.
+ for (size_t i = 0, e = use_def->numOperands(); i < e; i++) {
+ if (i == 2) {
+ continue;
+ }
+ if (use_def->getOperand(i) == def) {
+ return true;
+ }
+ }
+ break;
+ default:
+ return true;
+ }
+ }
+ return false;
+}
+
+#ifdef JS_JITSPEW
+void MBinaryArithInstruction::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+
+ switch (type()) {
+ case MIRType::Int32:
+ if (isDiv()) {
+ out.printf(" [%s]", toDiv()->isUnsigned() ? "uint32" : "int32");
+ } else if (isMod()) {
+ out.printf(" [%s]", toMod()->isUnsigned() ? "uint32" : "int32");
+ } else {
+ out.printf(" [int32]");
+ }
+ break;
+ case MIRType::Int64:
+ if (isDiv()) {
+ out.printf(" [%s]", toDiv()->isUnsigned() ? "uint64" : "int64");
+ } else if (isMod()) {
+ out.printf(" [%s]", toMod()->isUnsigned() ? "uint64" : "int64");
+ } else {
+ out.printf(" [int64]");
+ }
+ break;
+ case MIRType::Float32:
+ out.printf(" [float]");
+ break;
+ case MIRType::Double:
+ out.printf(" [double]");
+ break;
+ default:
+ break;
+ }
+}
+#endif
+
+MDefinition* MRsh::foldsTo(TempAllocator& alloc) {
+ MDefinition* f = MBinaryBitwiseInstruction::foldsTo(alloc);
+
+ if (f != this) {
+ return f;
+ }
+
+ MDefinition* lhs = getOperand(0);
+ MDefinition* rhs = getOperand(1);
+
+ // It's probably OK to perform this optimization only for int32, as it will
+ // have the greatest effect for asm.js code that is compiled with the JS
+ // pipeline, and that code will not see int64 values.
+
+ if (!lhs->isLsh() || !rhs->isConstant() || rhs->type() != MIRType::Int32) {
+ return this;
+ }
+
+ if (!lhs->getOperand(1)->isConstant() ||
+ lhs->getOperand(1)->type() != MIRType::Int32) {
+ return this;
+ }
+
+ uint32_t shift = rhs->toConstant()->toInt32();
+ uint32_t shift_lhs = lhs->getOperand(1)->toConstant()->toInt32();
+ if (shift != shift_lhs) {
+ return this;
+ }
+
+ switch (shift) {
+ case 16:
+ return MSignExtendInt32::New(alloc, lhs->getOperand(0),
+ MSignExtendInt32::Half);
+ case 24:
+ return MSignExtendInt32::New(alloc, lhs->getOperand(0),
+ MSignExtendInt32::Byte);
+ }
+
+ return this;
+}
+
+MDefinition* MBinaryArithInstruction::foldsTo(TempAllocator& alloc) {
+ MOZ_ASSERT(IsNumberType(type()));
+
+ MDefinition* lhs = getOperand(0);
+ MDefinition* rhs = getOperand(1);
+
+ if (type() == MIRType::Int64) {
+ MOZ_ASSERT(!isTruncated());
+
+ if (MConstant* folded = EvaluateInt64ConstantOperands(alloc, this)) {
+ if (!folded->block()) {
+ block()->insertBefore(this, folded);
+ }
+ return folded;
+ }
+ if (isSub() || isDiv() || isMod()) {
+ return this;
+ }
+ if (rhs->isConstant() &&
+ rhs->toConstant()->toInt64() == int64_t(getIdentity())) {
+ return lhs;
+ }
+ if (lhs->isConstant() &&
+ lhs->toConstant()->toInt64() == int64_t(getIdentity())) {
+ return rhs;
+ }
+ return this;
+ }
+
+ if (MConstant* folded = EvaluateConstantOperands(alloc, this)) {
+ if (isTruncated()) {
+ if (!folded->block()) {
+ block()->insertBefore(this, folded);
+ }
+ if (folded->type() != MIRType::Int32) {
+ return MTruncateToInt32::New(alloc, folded);
+ }
+ }
+ return folded;
+ }
+
+ if (mustPreserveNaN_) {
+ return this;
+ }
+
+ // 0 + -0 = 0. So we can't remove addition
+ if (isAdd() && type() != MIRType::Int32) {
+ return this;
+ }
+
+ if (IsConstant(rhs, getIdentity())) {
+ if (isTruncated()) {
+ return MTruncateToInt32::New(alloc, lhs);
+ }
+ return lhs;
+ }
+
+ // subtraction isn't commutative. So we can't remove subtraction when lhs
+ // equals 0
+ if (isSub()) {
+ return this;
+ }
+
+ if (IsConstant(lhs, getIdentity())) {
+ if (isTruncated()) {
+ return MTruncateToInt32::New(alloc, rhs);
+ }
+ return rhs; // id op x => x
+ }
+
+ return this;
+}
+
+void MBinaryArithInstruction::trySpecializeFloat32(TempAllocator& alloc) {
+ MOZ_ASSERT(IsNumberType(type()));
+
+ // Do not use Float32 if we can use int32.
+ if (type() == MIRType::Int32) {
+ return;
+ }
+
+ if (EnsureFloatConsumersAndInputOrConvert(this, alloc)) {
+ setResultType(MIRType::Float32);
+ }
+}
+
+void MMinMax::trySpecializeFloat32(TempAllocator& alloc) {
+ if (type() == MIRType::Int32) {
+ return;
+ }
+
+ MDefinition* left = lhs();
+ MDefinition* right = rhs();
+
+ if ((left->canProduceFloat32() ||
+ (left->isMinMax() && left->type() == MIRType::Float32)) &&
+ (right->canProduceFloat32() ||
+ (right->isMinMax() && right->type() == MIRType::Float32))) {
+ setResultType(MIRType::Float32);
+ } else {
+ ConvertOperandsToDouble(this, alloc);
+ }
+}
+
+MDefinition* MMinMax::foldsTo(TempAllocator& alloc) {
+ MOZ_ASSERT(lhs()->type() == type());
+ MOZ_ASSERT(rhs()->type() == type());
+
+ if (lhs() == rhs()) {
+ return lhs();
+ }
+
+ // Fold min/max operations with same inputs.
+ if (lhs()->isMinMax() || rhs()->isMinMax()) {
+ auto* other = lhs()->isMinMax() ? lhs()->toMinMax() : rhs()->toMinMax();
+ auto* operand = lhs()->isMinMax() ? rhs() : lhs();
+
+ if (operand == other->lhs() || operand == other->rhs()) {
+ if (isMax() == other->isMax()) {
+ // min(x, min(x, y)) = min(x, y)
+ // max(x, max(x, y)) = max(x, y)
+ return other;
+ }
+ if (!IsFloatingPointType(type())) {
+ // When neither value is NaN:
+ // max(x, min(x, y)) = x
+ // min(x, max(x, y)) = x
+
+ // Ensure that any bailouts that we depend on to guarantee that |y| is
+ // Int32 are not removed.
+ auto* otherOp = operand == other->lhs() ? other->rhs() : other->lhs();
+ otherOp->setGuardRangeBailoutsUnchecked();
+
+ return operand;
+ }
+ }
+ }
+
+ if (!lhs()->isConstant() && !rhs()->isConstant()) {
+ return this;
+ }
+
+ auto foldConstants = [&alloc](MDefinition* lhs, MDefinition* rhs,
+ bool isMax) -> MConstant* {
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(lhs->toConstant()->isTypeRepresentableAsDouble());
+ MOZ_ASSERT(rhs->toConstant()->isTypeRepresentableAsDouble());
+
+ double lnum = lhs->toConstant()->numberToDouble();
+ double rnum = rhs->toConstant()->numberToDouble();
+
+ double result;
+ if (isMax) {
+ result = js::math_max_impl(lnum, rnum);
+ } else {
+ result = js::math_min_impl(lnum, rnum);
+ }
+
+ // The folded MConstant should maintain the same MIRType with the original
+ // inputs.
+ if (lhs->type() == MIRType::Int32) {
+ int32_t cast;
+ if (mozilla::NumberEqualsInt32(result, &cast)) {
+ return MConstant::New(alloc, Int32Value(cast));
+ }
+ return nullptr;
+ }
+ if (lhs->type() == MIRType::Float32) {
+ return MConstant::NewFloat32(alloc, result);
+ }
+ MOZ_ASSERT(lhs->type() == MIRType::Double);
+ return MConstant::New(alloc, DoubleValue(result));
+ };
+
+ // Directly apply math utility to compare the rhs() and lhs() when
+ // they are both constants.
+ if (lhs()->isConstant() && rhs()->isConstant()) {
+ if (!lhs()->toConstant()->isTypeRepresentableAsDouble() ||
+ !rhs()->toConstant()->isTypeRepresentableAsDouble()) {
+ return this;
+ }
+
+ if (auto* folded = foldConstants(lhs(), rhs(), isMax())) {
+ return folded;
+ }
+ }
+
+ MDefinition* operand = lhs()->isConstant() ? rhs() : lhs();
+ MConstant* constant =
+ lhs()->isConstant() ? lhs()->toConstant() : rhs()->toConstant();
+
+ if (operand->isToDouble() &&
+ operand->getOperand(0)->type() == MIRType::Int32) {
+ // min(int32, cte >= INT32_MAX) = int32
+ if (!isMax() && constant->isTypeRepresentableAsDouble() &&
+ constant->numberToDouble() >= INT32_MAX) {
+ MLimitedTruncate* limit = MLimitedTruncate::New(
+ alloc, operand->getOperand(0), TruncateKind::NoTruncate);
+ block()->insertBefore(this, limit);
+ MToDouble* toDouble = MToDouble::New(alloc, limit);
+ return toDouble;
+ }
+
+ // max(int32, cte <= INT32_MIN) = int32
+ if (isMax() && constant->isTypeRepresentableAsDouble() &&
+ constant->numberToDouble() <= INT32_MIN) {
+ MLimitedTruncate* limit = MLimitedTruncate::New(
+ alloc, operand->getOperand(0), TruncateKind::NoTruncate);
+ block()->insertBefore(this, limit);
+ MToDouble* toDouble = MToDouble::New(alloc, limit);
+ return toDouble;
+ }
+ }
+
+ auto foldLength = [](MDefinition* operand, MConstant* constant,
+ bool isMax) -> MDefinition* {
+ if ((operand->isArrayLength() || operand->isArrayBufferViewLength() ||
+ operand->isArgumentsLength() || operand->isStringLength()) &&
+ constant->type() == MIRType::Int32) {
+ // (Array|ArrayBufferView|Arguments|String)Length is always >= 0.
+ // max(array.length, cte <= 0) = array.length
+ // min(array.length, cte <= 0) = cte
+ if (constant->toInt32() <= 0) {
+ return isMax ? operand : constant;
+ }
+ }
+ return nullptr;
+ };
+
+ if (auto* folded = foldLength(operand, constant, isMax())) {
+ return folded;
+ }
+
+ // Attempt to fold nested min/max operations which are produced by
+ // self-hosted built-in functions.
+ if (operand->isMinMax()) {
+ auto* other = operand->toMinMax();
+ MOZ_ASSERT(other->lhs()->type() == type());
+ MOZ_ASSERT(other->rhs()->type() == type());
+
+ MConstant* otherConstant = nullptr;
+ MDefinition* otherOperand = nullptr;
+ if (other->lhs()->isConstant()) {
+ otherConstant = other->lhs()->toConstant();
+ otherOperand = other->rhs();
+ } else if (other->rhs()->isConstant()) {
+ otherConstant = other->rhs()->toConstant();
+ otherOperand = other->lhs();
+ }
+
+ if (otherConstant && constant->isTypeRepresentableAsDouble() &&
+ otherConstant->isTypeRepresentableAsDouble()) {
+ if (isMax() == other->isMax()) {
+ // Fold min(x, min(y, z)) to min(min(x, y), z) with constant min(x, y).
+ // Fold max(x, max(y, z)) to max(max(x, y), z) with constant max(x, y).
+ if (auto* left = foldConstants(constant, otherConstant, isMax())) {
+ block()->insertBefore(this, left);
+ return MMinMax::New(alloc, left, otherOperand, type(), isMax());
+ }
+ } else {
+ // Fold min(x, max(y, z)) to max(min(x, y), min(x, z)).
+ // Fold max(x, min(y, z)) to min(max(x, y), max(x, z)).
+ //
+ // But only do this when min(x, z) can also be simplified.
+ if (auto* right = foldLength(otherOperand, constant, isMax())) {
+ if (auto* left = foldConstants(constant, otherConstant, isMax())) {
+ block()->insertBefore(this, left);
+ return MMinMax::New(alloc, left, right, type(), !isMax());
+ }
+ }
+ }
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MPow::foldsConstant(TempAllocator& alloc) {
+ // Both `x` and `p` in `x^p` must be constants in order to precompute.
+ if (!input()->isConstant() || !power()->isConstant()) {
+ return nullptr;
+ }
+ if (!power()->toConstant()->isTypeRepresentableAsDouble()) {
+ return nullptr;
+ }
+ if (!input()->toConstant()->isTypeRepresentableAsDouble()) {
+ return nullptr;
+ }
+
+ double x = input()->toConstant()->numberToDouble();
+ double p = power()->toConstant()->numberToDouble();
+ double result = js::ecmaPow(x, p);
+ if (type() == MIRType::Int32) {
+ int32_t cast;
+ if (!mozilla::NumberIsInt32(result, &cast)) {
+ // Reject folding if the result isn't an int32, because we'll bail anyway.
+ return nullptr;
+ }
+ return MConstant::New(alloc, Int32Value(cast));
+ }
+ return MConstant::New(alloc, DoubleValue(result));
+}
+
+MDefinition* MPow::foldsConstantPower(TempAllocator& alloc) {
+ // If `p` in `x^p` isn't constant, we can't apply these folds.
+ if (!power()->isConstant()) {
+ return nullptr;
+ }
+ if (!power()->toConstant()->isTypeRepresentableAsDouble()) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(type() == MIRType::Double || type() == MIRType::Int32);
+
+ // NOTE: The optimizations must match the optimizations used in |js::ecmaPow|
+ // resp. |js::powi| to avoid differential testing issues.
+
+ double pow = power()->toConstant()->numberToDouble();
+
+ // Math.pow(x, 0.5) is a sqrt with edge-case detection.
+ if (pow == 0.5) {
+ MOZ_ASSERT(type() == MIRType::Double);
+ return MPowHalf::New(alloc, input());
+ }
+
+ // Math.pow(x, -0.5) == 1 / Math.pow(x, 0.5), even for edge cases.
+ if (pow == -0.5) {
+ MOZ_ASSERT(type() == MIRType::Double);
+ MPowHalf* half = MPowHalf::New(alloc, input());
+ block()->insertBefore(this, half);
+ MConstant* one = MConstant::New(alloc, DoubleValue(1.0));
+ block()->insertBefore(this, one);
+ return MDiv::New(alloc, one, half, MIRType::Double);
+ }
+
+ // Math.pow(x, 1) == x.
+ if (pow == 1.0) {
+ return input();
+ }
+
+ auto multiply = [this, &alloc](MDefinition* lhs, MDefinition* rhs) {
+ MMul* mul = MMul::New(alloc, lhs, rhs, type());
+ mul->setBailoutKind(bailoutKind());
+
+ // Multiplying the same number can't yield negative zero.
+ mul->setCanBeNegativeZero(lhs != rhs && canBeNegativeZero());
+ return mul;
+ };
+
+ // Math.pow(x, 2) == x*x.
+ if (pow == 2.0) {
+ return multiply(input(), input());
+ }
+
+ // Math.pow(x, 3) == x*x*x.
+ if (pow == 3.0) {
+ MMul* mul1 = multiply(input(), input());
+ block()->insertBefore(this, mul1);
+ return multiply(input(), mul1);
+ }
+
+ // Math.pow(x, 4) == y*y, where y = x*x.
+ if (pow == 4.0) {
+ MMul* y = multiply(input(), input());
+ block()->insertBefore(this, y);
+ return multiply(y, y);
+ }
+
+ // No optimization
+ return nullptr;
+}
+
+MDefinition* MPow::foldsTo(TempAllocator& alloc) {
+ if (MDefinition* def = foldsConstant(alloc)) {
+ return def;
+ }
+ if (MDefinition* def = foldsConstantPower(alloc)) {
+ return def;
+ }
+ return this;
+}
+
+MDefinition* MInt32ToIntPtr::foldsTo(TempAllocator& alloc) {
+ MDefinition* def = input();
+ if (def->isConstant()) {
+ int32_t i = def->toConstant()->toInt32();
+ return MConstant::NewIntPtr(alloc, intptr_t(i));
+ }
+
+ if (def->isNonNegativeIntPtrToInt32()) {
+ return def->toNonNegativeIntPtrToInt32()->input();
+ }
+
+ return this;
+}
+
+bool MAbs::fallible() const {
+ return !implicitTruncate_ && (!range() || !range()->hasInt32Bounds());
+}
+
+void MAbs::trySpecializeFloat32(TempAllocator& alloc) {
+ // Do not use Float32 if we can use int32.
+ if (input()->type() == MIRType::Int32) {
+ return;
+ }
+
+ if (EnsureFloatConsumersAndInputOrConvert(this, alloc)) {
+ setResultType(MIRType::Float32);
+ }
+}
+
+MDefinition* MDiv::foldsTo(TempAllocator& alloc) {
+ MOZ_ASSERT(IsNumberType(type()));
+
+ if (type() == MIRType::Int64) {
+ if (MDefinition* folded = EvaluateInt64ConstantOperands(alloc, this)) {
+ return folded;
+ }
+ return this;
+ }
+
+ if (MDefinition* folded = EvaluateConstantOperands(alloc, this)) {
+ return folded;
+ }
+
+ if (MDefinition* folded = EvaluateExactReciprocal(alloc, this)) {
+ return folded;
+ }
+
+ return this;
+}
+
+void MDiv::analyzeEdgeCasesForward() {
+ // This is only meaningful when doing integer division.
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ MOZ_ASSERT(lhs()->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs()->type() == MIRType::Int32);
+
+ // Try removing divide by zero check
+ if (rhs()->isConstant() && !rhs()->toConstant()->isInt32(0)) {
+ canBeDivideByZero_ = false;
+ }
+
+ // If lhs is a constant int != INT32_MIN, then
+ // negative overflow check can be skipped.
+ if (lhs()->isConstant() && !lhs()->toConstant()->isInt32(INT32_MIN)) {
+ canBeNegativeOverflow_ = false;
+ }
+
+ // If rhs is a constant int != -1, likewise.
+ if (rhs()->isConstant() && !rhs()->toConstant()->isInt32(-1)) {
+ canBeNegativeOverflow_ = false;
+ }
+
+ // If lhs is != 0, then negative zero check can be skipped.
+ if (lhs()->isConstant() && !lhs()->toConstant()->isInt32(0)) {
+ setCanBeNegativeZero(false);
+ }
+
+ // If rhs is >= 0, likewise.
+ if (rhs()->isConstant() && rhs()->type() == MIRType::Int32) {
+ if (rhs()->toConstant()->toInt32() >= 0) {
+ setCanBeNegativeZero(false);
+ }
+ }
+}
+
+void MDiv::analyzeEdgeCasesBackward() {
+ if (canBeNegativeZero() && !NeedNegativeZeroCheck(this)) {
+ setCanBeNegativeZero(false);
+ }
+}
+
+bool MDiv::fallible() const { return !isTruncated(); }
+
+MDefinition* MMod::foldsTo(TempAllocator& alloc) {
+ MOZ_ASSERT(IsNumberType(type()));
+
+ if (type() == MIRType::Int64) {
+ if (MDefinition* folded = EvaluateInt64ConstantOperands(alloc, this)) {
+ return folded;
+ }
+ } else {
+ if (MDefinition* folded = EvaluateConstantOperands(alloc, this)) {
+ return folded;
+ }
+ }
+ return this;
+}
+
+void MMod::analyzeEdgeCasesForward() {
+ // These optimizations make sense only for integer division
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ if (rhs()->isConstant() && !rhs()->toConstant()->isInt32(0)) {
+ canBeDivideByZero_ = false;
+ }
+
+ if (rhs()->isConstant()) {
+ int32_t n = rhs()->toConstant()->toInt32();
+ if (n > 0 && !IsPowerOfTwo(uint32_t(n))) {
+ canBePowerOfTwoDivisor_ = false;
+ }
+ }
+}
+
+bool MMod::fallible() const {
+ return !isTruncated() &&
+ (isUnsigned() || canBeDivideByZero() || canBeNegativeDividend());
+}
+
+void MMathFunction::trySpecializeFloat32(TempAllocator& alloc) {
+ if (EnsureFloatConsumersAndInputOrConvert(this, alloc)) {
+ setResultType(MIRType::Float32);
+ specialization_ = MIRType::Float32;
+ }
+}
+
+bool MMathFunction::isFloat32Commutative() const {
+ switch (function_) {
+ case UnaryMathFunction::Floor:
+ case UnaryMathFunction::Ceil:
+ case UnaryMathFunction::Round:
+ case UnaryMathFunction::Trunc:
+ return true;
+ default:
+ return false;
+ }
+}
+
+MHypot* MHypot::New(TempAllocator& alloc, const MDefinitionVector& vector) {
+ uint32_t length = vector.length();
+ MHypot* hypot = new (alloc) MHypot;
+ if (!hypot->init(alloc, length)) {
+ return nullptr;
+ }
+
+ for (uint32_t i = 0; i < length; ++i) {
+ hypot->initOperand(i, vector[i]);
+ }
+ return hypot;
+}
+
+bool MAdd::fallible() const {
+ // the add is fallible if range analysis does not say that it is finite, AND
+ // either the truncation analysis shows that there are non-truncated uses.
+ if (truncateKind() >= TruncateKind::IndirectTruncate) {
+ return false;
+ }
+ if (range() && range()->hasInt32Bounds()) {
+ return false;
+ }
+ return true;
+}
+
+bool MSub::fallible() const {
+ // see comment in MAdd::fallible()
+ if (truncateKind() >= TruncateKind::IndirectTruncate) {
+ return false;
+ }
+ if (range() && range()->hasInt32Bounds()) {
+ return false;
+ }
+ return true;
+}
+
+MDefinition* MSub::foldsTo(TempAllocator& alloc) {
+ MDefinition* out = MBinaryArithInstruction::foldsTo(alloc);
+ if (out != this) {
+ return out;
+ }
+
+ if (type() != MIRType::Int32) {
+ return this;
+ }
+
+ // Optimize X - X to 0. This optimization is only valid for Int32
+ // values. Subtracting a floating point value from itself returns
+ // NaN when the operand is either Infinity or NaN.
+ if (lhs() == rhs()) {
+ // Ensure that any bailouts that we depend on to guarantee that X
+ // is Int32 are not removed.
+ lhs()->setGuardRangeBailoutsUnchecked();
+ return MConstant::New(alloc, Int32Value(0));
+ }
+
+ return this;
+}
+
+MDefinition* MMul::foldsTo(TempAllocator& alloc) {
+ MDefinition* out = MBinaryArithInstruction::foldsTo(alloc);
+ if (out != this) {
+ return out;
+ }
+
+ if (type() != MIRType::Int32) {
+ return this;
+ }
+
+ if (lhs() == rhs()) {
+ setCanBeNegativeZero(false);
+ }
+
+ return this;
+}
+
+void MMul::analyzeEdgeCasesForward() {
+ // Try to remove the check for negative zero
+ // This only makes sense when using the integer multiplication
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ // If lhs is > 0, no need for negative zero check.
+ if (lhs()->isConstant() && lhs()->type() == MIRType::Int32) {
+ if (lhs()->toConstant()->toInt32() > 0) {
+ setCanBeNegativeZero(false);
+ }
+ }
+
+ // If rhs is > 0, likewise.
+ if (rhs()->isConstant() && rhs()->type() == MIRType::Int32) {
+ if (rhs()->toConstant()->toInt32() > 0) {
+ setCanBeNegativeZero(false);
+ }
+ }
+}
+
+void MMul::analyzeEdgeCasesBackward() {
+ if (canBeNegativeZero() && !NeedNegativeZeroCheck(this)) {
+ setCanBeNegativeZero(false);
+ }
+}
+
+bool MMul::updateForReplacement(MDefinition* ins_) {
+ MMul* ins = ins_->toMul();
+ bool negativeZero = canBeNegativeZero() || ins->canBeNegativeZero();
+ setCanBeNegativeZero(negativeZero);
+ // Remove the imul annotation when merging imul and normal multiplication.
+ if (mode_ == Integer && ins->mode() != Integer) {
+ mode_ = Normal;
+ }
+ return true;
+}
+
+bool MMul::canOverflow() const {
+ if (isTruncated()) {
+ return false;
+ }
+ return !range() || !range()->hasInt32Bounds();
+}
+
+bool MUrsh::fallible() const {
+ if (bailoutsDisabled()) {
+ return false;
+ }
+ return !range() || !range()->hasInt32Bounds();
+}
+
+MIRType MCompare::inputType() {
+ switch (compareType_) {
+ case Compare_Undefined:
+ return MIRType::Undefined;
+ case Compare_Null:
+ return MIRType::Null;
+ case Compare_UInt32:
+ case Compare_Int32:
+ return MIRType::Int32;
+ case Compare_UIntPtr:
+ return MIRType::IntPtr;
+ case Compare_Double:
+ return MIRType::Double;
+ case Compare_Float32:
+ return MIRType::Float32;
+ case Compare_String:
+ return MIRType::String;
+ case Compare_Symbol:
+ return MIRType::Symbol;
+ case Compare_Object:
+ return MIRType::Object;
+ case Compare_BigInt:
+ case Compare_BigInt_Int32:
+ case Compare_BigInt_Double:
+ case Compare_BigInt_String:
+ return MIRType::BigInt;
+ default:
+ MOZ_CRASH("No known conversion");
+ }
+}
+
+static inline bool MustBeUInt32(MDefinition* def, MDefinition** pwrapped) {
+ if (def->isUrsh()) {
+ *pwrapped = def->toUrsh()->lhs();
+ MDefinition* rhs = def->toUrsh()->rhs();
+ return def->toUrsh()->bailoutsDisabled() && rhs->maybeConstantValue() &&
+ rhs->maybeConstantValue()->isInt32(0);
+ }
+
+ if (MConstant* defConst = def->maybeConstantValue()) {
+ *pwrapped = defConst;
+ return defConst->type() == MIRType::Int32 && defConst->toInt32() >= 0;
+ }
+
+ *pwrapped = nullptr; // silence GCC warning
+ return false;
+}
+
+/* static */
+bool MBinaryInstruction::unsignedOperands(MDefinition* left,
+ MDefinition* right) {
+ MDefinition* replace;
+ if (!MustBeUInt32(left, &replace)) {
+ return false;
+ }
+ if (replace->type() != MIRType::Int32) {
+ return false;
+ }
+ if (!MustBeUInt32(right, &replace)) {
+ return false;
+ }
+ if (replace->type() != MIRType::Int32) {
+ return false;
+ }
+ return true;
+}
+
+bool MBinaryInstruction::unsignedOperands() {
+ return unsignedOperands(getOperand(0), getOperand(1));
+}
+
+void MBinaryInstruction::replaceWithUnsignedOperands() {
+ MOZ_ASSERT(unsignedOperands());
+
+ for (size_t i = 0; i < numOperands(); i++) {
+ MDefinition* replace;
+ MustBeUInt32(getOperand(i), &replace);
+ if (replace == getOperand(i)) {
+ continue;
+ }
+
+ getOperand(i)->setImplicitlyUsedUnchecked();
+ replaceOperand(i, replace);
+ }
+}
+
+MDefinition* MBitNot::foldsTo(TempAllocator& alloc) {
+ if (type() == MIRType::Int64) {
+ return this;
+ }
+ MOZ_ASSERT(type() == MIRType::Int32);
+
+ MDefinition* input = getOperand(0);
+
+ if (input->isConstant()) {
+ js::Value v = Int32Value(~(input->toConstant()->toInt32()));
+ return MConstant::New(alloc, v);
+ }
+
+ if (input->isBitNot()) {
+ MOZ_ASSERT(input->toBitNot()->type() == MIRType::Int32);
+ MOZ_ASSERT(input->toBitNot()->getOperand(0)->type() == MIRType::Int32);
+ return MTruncateToInt32::New(alloc,
+ input->toBitNot()->input()); // ~~x => x | 0
+ }
+
+ return this;
+}
+
+static void AssertKnownClass(TempAllocator& alloc, MInstruction* ins,
+ MDefinition* obj) {
+#ifdef DEBUG
+ const JSClass* clasp = GetObjectKnownJSClass(obj);
+ MOZ_ASSERT(clasp);
+
+ auto* assert = MAssertClass::New(alloc, obj, clasp);
+ ins->block()->insertBefore(ins, assert);
+#endif
+}
+
+MDefinition* MBoxNonStrictThis::foldsTo(TempAllocator& alloc) {
+ MDefinition* in = input();
+ if (in->isBox()) {
+ in = in->toBox()->input();
+ }
+
+ if (in->type() == MIRType::Object) {
+ return in;
+ }
+
+ return this;
+}
+
+AliasSet MLoadArgumentsObjectArg::getAliasSet() const {
+ return AliasSet::Load(AliasSet::Any);
+}
+
+AliasSet MLoadArgumentsObjectArgHole::getAliasSet() const {
+ return AliasSet::Load(AliasSet::Any);
+}
+
+AliasSet MInArgumentsObjectArg::getAliasSet() const {
+ // Loads |arguments.length|, but not the actual element, so we can use the
+ // same alias-set as MArgumentsObjectLength.
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+AliasSet MArgumentsObjectLength::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+bool MGuardArgumentsObjectFlags::congruentTo(const MDefinition* ins) const {
+ if (!ins->isGuardArgumentsObjectFlags() ||
+ ins->toGuardArgumentsObjectFlags()->flags() != flags()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MGuardArgumentsObjectFlags::getAliasSet() const {
+ // The flags are packed with the length in a fixed private slot.
+ return AliasSet::Load(AliasSet::FixedSlot);
+}
+
+MDefinition* MReturnFromCtor::foldsTo(TempAllocator& alloc) {
+ MDefinition* rval = value();
+ if (rval->isBox()) {
+ rval = rval->toBox()->input();
+ }
+
+ if (rval->type() == MIRType::Object) {
+ return rval;
+ }
+
+ if (rval->type() != MIRType::Value) {
+ return object();
+ }
+
+ return this;
+}
+
+MDefinition* MTypeOf::foldsTo(TempAllocator& alloc) {
+ MDefinition* unboxed = input();
+ if (unboxed->isBox()) {
+ unboxed = unboxed->toBox()->input();
+ }
+
+ JSType type;
+ switch (unboxed->type()) {
+ case MIRType::Double:
+ case MIRType::Float32:
+ case MIRType::Int32:
+ type = JSTYPE_NUMBER;
+ break;
+ case MIRType::String:
+ type = JSTYPE_STRING;
+ break;
+ case MIRType::Symbol:
+ type = JSTYPE_SYMBOL;
+ break;
+ case MIRType::BigInt:
+ type = JSTYPE_BIGINT;
+ break;
+ case MIRType::Null:
+ type = JSTYPE_OBJECT;
+ break;
+ case MIRType::Undefined:
+ type = JSTYPE_UNDEFINED;
+ break;
+ case MIRType::Boolean:
+ type = JSTYPE_BOOLEAN;
+ break;
+ case MIRType::Object: {
+ KnownClass known = GetObjectKnownClass(unboxed);
+ if (known != KnownClass::None) {
+ if (known == KnownClass::Function) {
+ type = JSTYPE_FUNCTION;
+ } else {
+ type = JSTYPE_OBJECT;
+ }
+
+ AssertKnownClass(alloc, this, unboxed);
+ break;
+ }
+ [[fallthrough]];
+ }
+ default:
+ return this;
+ }
+
+ return MConstant::New(alloc, Int32Value(static_cast<int32_t>(type)));
+}
+
+MDefinition* MTypeOfName::foldsTo(TempAllocator& alloc) {
+ MOZ_ASSERT(input()->type() == MIRType::Int32);
+
+ if (!input()->isConstant()) {
+ return this;
+ }
+
+ static_assert(JSTYPE_UNDEFINED == 0);
+
+ int32_t type = input()->toConstant()->toInt32();
+ MOZ_ASSERT(JSTYPE_UNDEFINED <= type && type < JSTYPE_LIMIT);
+
+ JSString* name =
+ TypeName(static_cast<JSType>(type), GetJitContext()->runtime->names());
+ return MConstant::New(alloc, StringValue(name));
+}
+
+MUrsh* MUrsh::NewWasm(TempAllocator& alloc, MDefinition* left,
+ MDefinition* right, MIRType type) {
+ MUrsh* ins = new (alloc) MUrsh(left, right, type);
+
+ // Since Ion has no UInt32 type, we use Int32 and we have a special
+ // exception to the type rules: we can return values in
+ // (INT32_MIN,UINT32_MAX] and still claim that we have an Int32 type
+ // without bailing out. This is necessary because Ion has no UInt32
+ // type and we can't have bailouts in wasm code.
+ ins->bailoutsDisabled_ = true;
+
+ return ins;
+}
+
+MResumePoint* MResumePoint::New(TempAllocator& alloc, MBasicBlock* block,
+ jsbytecode* pc, ResumeMode mode) {
+ MResumePoint* resume = new (alloc) MResumePoint(block, pc, mode);
+ if (!resume->init(alloc)) {
+ block->discardPreAllocatedResumePoint(resume);
+ return nullptr;
+ }
+ resume->inherit(block);
+ return resume;
+}
+
+MResumePoint::MResumePoint(MBasicBlock* block, jsbytecode* pc, ResumeMode mode)
+ : MNode(block, Kind::ResumePoint),
+ pc_(pc),
+ instruction_(nullptr),
+ mode_(mode) {
+ block->addResumePoint(this);
+}
+
+bool MResumePoint::init(TempAllocator& alloc) {
+ return operands_.init(alloc, block()->stackDepth());
+}
+
+MResumePoint* MResumePoint::caller() const {
+ return block()->callerResumePoint();
+}
+
+void MResumePoint::inherit(MBasicBlock* block) {
+ // FixedList doesn't initialize its elements, so do unchecked inits.
+ for (size_t i = 0; i < stackDepth(); i++) {
+ initOperand(i, block->getSlot(i));
+ }
+}
+
+void MResumePoint::addStore(TempAllocator& alloc, MDefinition* store,
+ const MResumePoint* cache) {
+ MOZ_ASSERT(block()->outerResumePoint() != this);
+ MOZ_ASSERT_IF(cache, !cache->stores_.empty());
+
+ if (cache && cache->stores_.begin()->operand == store) {
+ // If the last resume point had the same side-effect stack, then we can
+ // reuse the current side effect without cloning it. This is a simple
+ // way to share common context by making a spaghetti stack.
+ if (++cache->stores_.begin() == stores_.begin()) {
+ stores_.copy(cache->stores_);
+ return;
+ }
+ }
+
+ // Ensure that the store would not be deleted by DCE.
+ MOZ_ASSERT(store->isEffectful());
+
+ MStoreToRecover* top = new (alloc) MStoreToRecover(store);
+ stores_.push(top);
+}
+
+#ifdef JS_JITSPEW
+void MResumePoint::dump(GenericPrinter& out) const {
+ out.printf("resumepoint mode=");
+
+ switch (mode()) {
+ case ResumeMode::ResumeAt:
+ if (instruction_) {
+ out.printf("ResumeAt(%u)", instruction_->id());
+ } else {
+ out.printf("ResumeAt");
+ }
+ break;
+ default:
+ out.put(ResumeModeToString(mode()));
+ break;
+ }
+
+ if (MResumePoint* c = caller()) {
+ out.printf(" (caller in block%u)", c->block()->id());
+ }
+
+ for (size_t i = 0; i < numOperands(); i++) {
+ out.printf(" ");
+ if (operands_[i].hasProducer()) {
+ getOperand(i)->printName(out);
+ } else {
+ out.printf("(null)");
+ }
+ }
+ out.printf("\n");
+}
+
+void MResumePoint::dump() const {
+ Fprinter out(stderr);
+ dump(out);
+ out.finish();
+}
+#endif
+
+bool MResumePoint::isObservableOperand(MUse* u) const {
+ return isObservableOperand(indexOf(u));
+}
+
+bool MResumePoint::isObservableOperand(size_t index) const {
+ return block()->info().isObservableSlot(index);
+}
+
+bool MResumePoint::isRecoverableOperand(MUse* u) const {
+ return block()->info().isRecoverableOperand(indexOf(u));
+}
+
+MDefinition* MTruncateBigIntToInt64::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+
+ if (input->isBox()) {
+ input = input->getOperand(0);
+ }
+
+ // If the operand converts an I64 to BigInt, drop both conversions.
+ if (input->isInt64ToBigInt()) {
+ return input->getOperand(0);
+ }
+
+ // Fold this operation if the input operand is constant.
+ if (input->isConstant()) {
+ return MConstant::NewInt64(
+ alloc, BigInt::toInt64(input->toConstant()->toBigInt()));
+ }
+
+ return this;
+}
+
+MDefinition* MToInt64::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+
+ if (input->isBox()) {
+ input = input->getOperand(0);
+ }
+
+ // Unwrap MInt64ToBigInt: MToInt64(MInt64ToBigInt(int64)) = int64.
+ if (input->isInt64ToBigInt()) {
+ return input->getOperand(0);
+ }
+
+ // When the input is an Int64 already, just return it.
+ if (input->type() == MIRType::Int64) {
+ return input;
+ }
+
+ // Fold this operation if the input operand is constant.
+ if (input->isConstant()) {
+ switch (input->type()) {
+ case MIRType::Boolean:
+ return MConstant::NewInt64(alloc, input->toConstant()->toBoolean());
+ default:
+ break;
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MToNumberInt32::foldsTo(TempAllocator& alloc) {
+ // Fold this operation if the input operand is constant.
+ if (MConstant* cst = input()->maybeConstantValue()) {
+ switch (cst->type()) {
+ case MIRType::Null:
+ if (conversion() == IntConversionInputKind::Any) {
+ return MConstant::New(alloc, Int32Value(0));
+ }
+ break;
+ case MIRType::Boolean:
+ if (conversion() == IntConversionInputKind::Any ||
+ conversion() == IntConversionInputKind::NumbersOrBoolsOnly) {
+ return MConstant::New(alloc, Int32Value(cst->toBoolean()));
+ }
+ break;
+ case MIRType::Int32:
+ return MConstant::New(alloc, Int32Value(cst->toInt32()));
+ case MIRType::Float32:
+ case MIRType::Double:
+ int32_t ival;
+ // Only the value within the range of Int32 can be substituted as
+ // constant.
+ if (mozilla::NumberIsInt32(cst->numberToDouble(), &ival)) {
+ return MConstant::New(alloc, Int32Value(ival));
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ MDefinition* input = getOperand(0);
+ if (input->isBox()) {
+ input = input->toBox()->input();
+ }
+
+ // Do not fold the TruncateToInt32 node when the input is uint32 (e.g. ursh
+ // with a zero constant. Consider the test jit-test/tests/ion/bug1247880.js,
+ // where the relevant code is: |(imul(1, x >>> 0) % 2)|. The imul operator
+ // is folded to a MTruncateToInt32 node, which will result in this MIR:
+ // MMod(MTruncateToInt32(MUrsh(x, MConstant(0))), MConstant(2)). Note that
+ // the MUrsh node's type is int32 (since uint32 is not implemented), and
+ // that would fold the MTruncateToInt32 node. This will make the modulo
+ // unsigned, while is should have been signed.
+ if (input->type() == MIRType::Int32 && !IsUint32Type(input)) {
+ return input;
+ }
+
+ return this;
+}
+
+MDefinition* MToIntegerInt32::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+
+ // Fold this operation if the input operand is constant.
+ if (input->isConstant()) {
+ switch (input->type()) {
+ case MIRType::Undefined:
+ case MIRType::Null:
+ return MConstant::New(alloc, Int32Value(0));
+ case MIRType::Boolean:
+ return MConstant::New(alloc,
+ Int32Value(input->toConstant()->toBoolean()));
+ case MIRType::Int32:
+ return MConstant::New(alloc,
+ Int32Value(input->toConstant()->toInt32()));
+ case MIRType::Float32:
+ case MIRType::Double: {
+ double result = JS::ToInteger(input->toConstant()->numberToDouble());
+ int32_t ival;
+ // Only the value within the range of Int32 can be substituted as
+ // constant.
+ if (mozilla::NumberEqualsInt32(result, &ival)) {
+ return MConstant::New(alloc, Int32Value(ival));
+ }
+ break;
+ }
+ default:
+ break;
+ }
+ }
+
+ // See the comment in |MToNumberInt32::foldsTo|.
+ if (input->type() == MIRType::Int32 && !IsUint32Type(input)) {
+ return input;
+ }
+
+ return this;
+}
+
+void MToNumberInt32::analyzeEdgeCasesBackward() {
+ if (!NeedNegativeZeroCheck(this)) {
+ setNeedsNegativeZeroCheck(false);
+ }
+}
+
+MDefinition* MTruncateToInt32::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+ if (input->isBox()) {
+ input = input->getOperand(0);
+ }
+
+ // Do not fold the TruncateToInt32 node when the input is uint32 (e.g. ursh
+ // with a zero constant. Consider the test jit-test/tests/ion/bug1247880.js,
+ // where the relevant code is: |(imul(1, x >>> 0) % 2)|. The imul operator
+ // is folded to a MTruncateToInt32 node, which will result in this MIR:
+ // MMod(MTruncateToInt32(MUrsh(x, MConstant(0))), MConstant(2)). Note that
+ // the MUrsh node's type is int32 (since uint32 is not implemented), and
+ // that would fold the MTruncateToInt32 node. This will make the modulo
+ // unsigned, while is should have been signed.
+ if (input->type() == MIRType::Int32 && !IsUint32Type(input)) {
+ return input;
+ }
+
+ if (input->type() == MIRType::Double && input->isConstant()) {
+ int32_t ret = ToInt32(input->toConstant()->toDouble());
+ return MConstant::New(alloc, Int32Value(ret));
+ }
+
+ return this;
+}
+
+MDefinition* MWasmTruncateToInt32::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+ if (input->type() == MIRType::Int32) {
+ return input;
+ }
+
+ if (input->type() == MIRType::Double && input->isConstant()) {
+ double d = input->toConstant()->toDouble();
+ if (IsNaN(d)) {
+ return this;
+ }
+
+ if (!isUnsigned() && d <= double(INT32_MAX) && d >= double(INT32_MIN)) {
+ return MConstant::New(alloc, Int32Value(ToInt32(d)));
+ }
+
+ if (isUnsigned() && d <= double(UINT32_MAX) && d >= 0) {
+ return MConstant::New(alloc, Int32Value(ToInt32(d)));
+ }
+ }
+
+ if (input->type() == MIRType::Float32 && input->isConstant()) {
+ double f = double(input->toConstant()->toFloat32());
+ if (IsNaN(f)) {
+ return this;
+ }
+
+ if (!isUnsigned() && f <= double(INT32_MAX) && f >= double(INT32_MIN)) {
+ return MConstant::New(alloc, Int32Value(ToInt32(f)));
+ }
+
+ if (isUnsigned() && f <= double(UINT32_MAX) && f >= 0) {
+ return MConstant::New(alloc, Int32Value(ToInt32(f)));
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MWrapInt64ToInt32::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = this->input();
+ if (input->isConstant()) {
+ uint64_t c = input->toConstant()->toInt64();
+ int32_t output = bottomHalf() ? int32_t(c) : int32_t(c >> 32);
+ return MConstant::New(alloc, Int32Value(output));
+ }
+
+ return this;
+}
+
+MDefinition* MExtendInt32ToInt64::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = this->input();
+ if (input->isConstant()) {
+ int32_t c = input->toConstant()->toInt32();
+ int64_t res = isUnsigned() ? int64_t(uint32_t(c)) : int64_t(c);
+ return MConstant::NewInt64(alloc, res);
+ }
+
+ return this;
+}
+
+MDefinition* MSignExtendInt32::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = this->input();
+ if (input->isConstant()) {
+ int32_t c = input->toConstant()->toInt32();
+ int32_t res;
+ switch (mode_) {
+ case Byte:
+ res = int32_t(int8_t(c & 0xFF));
+ break;
+ case Half:
+ res = int32_t(int16_t(c & 0xFFFF));
+ break;
+ }
+ return MConstant::New(alloc, Int32Value(res));
+ }
+
+ return this;
+}
+
+MDefinition* MSignExtendInt64::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = this->input();
+ if (input->isConstant()) {
+ int64_t c = input->toConstant()->toInt64();
+ int64_t res;
+ switch (mode_) {
+ case Byte:
+ res = int64_t(int8_t(c & 0xFF));
+ break;
+ case Half:
+ res = int64_t(int16_t(c & 0xFFFF));
+ break;
+ case Word:
+ res = int64_t(int32_t(c & 0xFFFFFFFFU));
+ break;
+ }
+ return MConstant::NewInt64(alloc, res);
+ }
+
+ return this;
+}
+
+MDefinition* MToDouble::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+ if (input->isBox()) {
+ input = input->getOperand(0);
+ }
+
+ if (input->type() == MIRType::Double) {
+ return input;
+ }
+
+ if (input->isConstant() &&
+ input->toConstant()->isTypeRepresentableAsDouble()) {
+ return MConstant::New(alloc,
+ DoubleValue(input->toConstant()->numberToDouble()));
+ }
+
+ return this;
+}
+
+MDefinition* MToFloat32::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = getOperand(0);
+ if (input->isBox()) {
+ input = input->getOperand(0);
+ }
+
+ if (input->type() == MIRType::Float32) {
+ return input;
+ }
+
+ // If x is a Float32, Float32(Double(x)) == x
+ if (!mustPreserveNaN_ && input->isToDouble() &&
+ input->toToDouble()->input()->type() == MIRType::Float32) {
+ return input->toToDouble()->input();
+ }
+
+ if (input->isConstant() &&
+ input->toConstant()->isTypeRepresentableAsDouble()) {
+ return MConstant::NewFloat32(alloc,
+ float(input->toConstant()->numberToDouble()));
+ }
+
+ // Fold ToFloat32(ToDouble(int32)) to ToFloat32(int32).
+ if (input->isToDouble() &&
+ input->toToDouble()->input()->type() == MIRType::Int32) {
+ return MToFloat32::New(alloc, input->toToDouble()->input());
+ }
+
+ return this;
+}
+
+MDefinition* MToString::foldsTo(TempAllocator& alloc) {
+ MDefinition* in = input();
+ if (in->isBox()) {
+ in = in->getOperand(0);
+ }
+
+ if (in->type() == MIRType::String) {
+ return in;
+ }
+ return this;
+}
+
+MDefinition* MClampToUint8::foldsTo(TempAllocator& alloc) {
+ if (MConstant* inputConst = input()->maybeConstantValue()) {
+ if (inputConst->isTypeRepresentableAsDouble()) {
+ int32_t clamped = ClampDoubleToUint8(inputConst->numberToDouble());
+ return MConstant::New(alloc, Int32Value(clamped));
+ }
+ }
+ return this;
+}
+
+bool MCompare::tryFoldEqualOperands(bool* result) {
+ if (lhs() != rhs()) {
+ return false;
+ }
+
+ // Intuitively somebody would think that if lhs === rhs,
+ // then we can just return true. (Or false for !==)
+ // However NaN !== NaN is true! So we spend some time trying
+ // to eliminate this case.
+
+ if (!IsStrictEqualityOp(jsop())) {
+ return false;
+ }
+
+ MOZ_ASSERT(
+ compareType_ == Compare_Undefined || compareType_ == Compare_Null ||
+ compareType_ == Compare_Int32 || compareType_ == Compare_UInt32 ||
+ compareType_ == Compare_UInt64 || compareType_ == Compare_Double ||
+ compareType_ == Compare_Float32 || compareType_ == Compare_UIntPtr ||
+ compareType_ == Compare_String || compareType_ == Compare_Object ||
+ compareType_ == Compare_Symbol || compareType_ == Compare_BigInt ||
+ compareType_ == Compare_BigInt_Int32 ||
+ compareType_ == Compare_BigInt_Double ||
+ compareType_ == Compare_BigInt_String);
+
+ if (isDoubleComparison() || isFloat32Comparison()) {
+ if (!operandsAreNeverNaN()) {
+ return false;
+ }
+ }
+
+ lhs()->setGuardRangeBailoutsUnchecked();
+
+ *result = (jsop() == JSOp::StrictEq);
+ return true;
+}
+
+static JSType TypeOfName(JSLinearString* str) {
+ static constexpr std::array types = {
+ JSTYPE_UNDEFINED, JSTYPE_OBJECT, JSTYPE_FUNCTION, JSTYPE_STRING,
+ JSTYPE_NUMBER, JSTYPE_BOOLEAN, JSTYPE_SYMBOL, JSTYPE_BIGINT,
+#ifdef ENABLE_RECORD_TUPLE
+ JSTYPE_RECORD, JSTYPE_TUPLE,
+#endif
+ };
+ static_assert(types.size() == JSTYPE_LIMIT);
+
+ const JSAtomState& names = GetJitContext()->runtime->names();
+ for (auto type : types) {
+ if (EqualStrings(str, TypeName(type, names))) {
+ return type;
+ }
+ }
+ return JSTYPE_LIMIT;
+}
+
+static mozilla::Maybe<std::pair<MTypeOfName*, JSType>> IsTypeOfCompare(
+ MCompare* ins) {
+ if (!IsEqualityOp(ins->jsop())) {
+ return mozilla::Nothing();
+ }
+ if (ins->compareType() != MCompare::Compare_String) {
+ return mozilla::Nothing();
+ }
+
+ auto* lhs = ins->lhs();
+ auto* rhs = ins->rhs();
+
+ MOZ_ASSERT(ins->type() == MIRType::Boolean);
+ MOZ_ASSERT(lhs->type() == MIRType::String);
+ MOZ_ASSERT(rhs->type() == MIRType::String);
+
+ if (!lhs->isTypeOfName() && !rhs->isTypeOfName()) {
+ return mozilla::Nothing();
+ }
+ if (!lhs->isConstant() && !rhs->isConstant()) {
+ return mozilla::Nothing();
+ }
+
+ auto* typeOfName =
+ lhs->isTypeOfName() ? lhs->toTypeOfName() : rhs->toTypeOfName();
+ MOZ_ASSERT(typeOfName->input()->isTypeOf());
+
+ auto* constant = lhs->isConstant() ? lhs->toConstant() : rhs->toConstant();
+
+ JSType type = TypeOfName(&constant->toString()->asLinear());
+ return mozilla::Some(std::pair(typeOfName, type));
+}
+
+bool MCompare::tryFoldTypeOf(bool* result) {
+ auto typeOfPair = IsTypeOfCompare(this);
+ if (!typeOfPair) {
+ return false;
+ }
+ auto [typeOfName, type] = *typeOfPair;
+ auto* typeOf = typeOfName->input()->toTypeOf();
+
+ switch (type) {
+ case JSTYPE_BOOLEAN:
+ if (!typeOf->input()->mightBeType(MIRType::Boolean)) {
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+ }
+ break;
+ case JSTYPE_NUMBER:
+ if (!typeOf->input()->mightBeType(MIRType::Int32) &&
+ !typeOf->input()->mightBeType(MIRType::Float32) &&
+ !typeOf->input()->mightBeType(MIRType::Double)) {
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+ }
+ break;
+ case JSTYPE_STRING:
+ if (!typeOf->input()->mightBeType(MIRType::String)) {
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+ }
+ break;
+ case JSTYPE_SYMBOL:
+ if (!typeOf->input()->mightBeType(MIRType::Symbol)) {
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+ }
+ break;
+ case JSTYPE_BIGINT:
+ if (!typeOf->input()->mightBeType(MIRType::BigInt)) {
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+ }
+ break;
+ case JSTYPE_OBJECT:
+ if (!typeOf->input()->mightBeType(MIRType::Object) &&
+ !typeOf->input()->mightBeType(MIRType::Null)) {
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+ }
+ break;
+ case JSTYPE_UNDEFINED:
+ if (!typeOf->input()->mightBeType(MIRType::Object) &&
+ !typeOf->input()->mightBeType(MIRType::Undefined)) {
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+ }
+ break;
+ case JSTYPE_FUNCTION:
+ if (!typeOf->input()->mightBeType(MIRType::Object)) {
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+ }
+ break;
+ case JSTYPE_LIMIT:
+ *result = (jsop() == JSOp::StrictNe || jsop() == JSOp::Ne);
+ return true;
+#ifdef ENABLE_RECORD_TUPLE
+ case JSTYPE_RECORD:
+ case JSTYPE_TUPLE:
+ MOZ_CRASH("Records and Tuples are not supported yet.");
+#endif
+ }
+
+ return false;
+}
+
+bool MCompare::tryFold(bool* result) {
+ JSOp op = jsop();
+
+ if (tryFoldEqualOperands(result)) {
+ return true;
+ }
+
+ if (tryFoldTypeOf(result)) {
+ return true;
+ }
+
+ if (compareType_ == Compare_Null || compareType_ == Compare_Undefined) {
+ // The LHS is the value we want to test against null or undefined.
+ if (IsStrictEqualityOp(op)) {
+ if (lhs()->type() == inputType()) {
+ *result = (op == JSOp::StrictEq);
+ return true;
+ }
+ if (!lhs()->mightBeType(inputType())) {
+ *result = (op == JSOp::StrictNe);
+ return true;
+ }
+ } else {
+ MOZ_ASSERT(IsLooseEqualityOp(op));
+ if (IsNullOrUndefined(lhs()->type())) {
+ *result = (op == JSOp::Eq);
+ return true;
+ }
+ if (!lhs()->mightBeType(MIRType::Null) &&
+ !lhs()->mightBeType(MIRType::Undefined) &&
+ !lhs()->mightBeType(MIRType::Object)) {
+ *result = (op == JSOp::Ne);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ return false;
+}
+
+template <typename T>
+static bool FoldComparison(JSOp op, T left, T right) {
+ switch (op) {
+ case JSOp::Lt:
+ return left < right;
+ case JSOp::Le:
+ return left <= right;
+ case JSOp::Gt:
+ return left > right;
+ case JSOp::Ge:
+ return left >= right;
+ case JSOp::StrictEq:
+ case JSOp::Eq:
+ return left == right;
+ case JSOp::StrictNe:
+ case JSOp::Ne:
+ return left != right;
+ default:
+ MOZ_CRASH("Unexpected op.");
+ }
+}
+
+bool MCompare::evaluateConstantOperands(TempAllocator& alloc, bool* result) {
+ if (type() != MIRType::Boolean && type() != MIRType::Int32) {
+ return false;
+ }
+
+ MDefinition* left = getOperand(0);
+ MDefinition* right = getOperand(1);
+
+ if (compareType() == Compare_Double) {
+ // Optimize "MCompare MConstant (MToDouble SomethingInInt32Range).
+ // In most cases the MToDouble was added, because the constant is
+ // a double.
+ // e.g. v < 9007199254740991, where v is an int32 is always true.
+ if (!lhs()->isConstant() && !rhs()->isConstant()) {
+ return false;
+ }
+
+ MDefinition* operand = left->isConstant() ? right : left;
+ MConstant* constant =
+ left->isConstant() ? left->toConstant() : right->toConstant();
+ MOZ_ASSERT(constant->type() == MIRType::Double);
+ double cte = constant->toDouble();
+
+ if (operand->isToDouble() &&
+ operand->getOperand(0)->type() == MIRType::Int32) {
+ bool replaced = false;
+ switch (jsop_) {
+ case JSOp::Lt:
+ if (cte > INT32_MAX || cte < INT32_MIN) {
+ *result = !((constant == lhs()) ^ (cte < INT32_MIN));
+ replaced = true;
+ }
+ break;
+ case JSOp::Le:
+ if (constant == lhs()) {
+ if (cte > INT32_MAX || cte <= INT32_MIN) {
+ *result = (cte <= INT32_MIN);
+ replaced = true;
+ }
+ } else {
+ if (cte >= INT32_MAX || cte < INT32_MIN) {
+ *result = (cte >= INT32_MIN);
+ replaced = true;
+ }
+ }
+ break;
+ case JSOp::Gt:
+ if (cte > INT32_MAX || cte < INT32_MIN) {
+ *result = !((constant == rhs()) ^ (cte < INT32_MIN));
+ replaced = true;
+ }
+ break;
+ case JSOp::Ge:
+ if (constant == lhs()) {
+ if (cte >= INT32_MAX || cte < INT32_MIN) {
+ *result = (cte >= INT32_MAX);
+ replaced = true;
+ }
+ } else {
+ if (cte > INT32_MAX || cte <= INT32_MIN) {
+ *result = (cte <= INT32_MIN);
+ replaced = true;
+ }
+ }
+ break;
+ case JSOp::StrictEq: // Fall through.
+ case JSOp::Eq:
+ if (cte > INT32_MAX || cte < INT32_MIN) {
+ *result = false;
+ replaced = true;
+ }
+ break;
+ case JSOp::StrictNe: // Fall through.
+ case JSOp::Ne:
+ if (cte > INT32_MAX || cte < INT32_MIN) {
+ *result = true;
+ replaced = true;
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected op.");
+ }
+ if (replaced) {
+ MLimitedTruncate* limit = MLimitedTruncate::New(
+ alloc, operand->getOperand(0), TruncateKind::NoTruncate);
+ limit->setGuardUnchecked();
+ block()->insertBefore(this, limit);
+ return true;
+ }
+ }
+ }
+
+ if (!left->isConstant() || !right->isConstant()) {
+ return false;
+ }
+
+ MConstant* lhs = left->toConstant();
+ MConstant* rhs = right->toConstant();
+
+ // Fold away some String equality comparisons.
+ if (lhs->type() == MIRType::String && rhs->type() == MIRType::String) {
+ int32_t comp = 0; // Default to equal.
+ if (left != right) {
+ comp = CompareStrings(&lhs->toString()->asLinear(),
+ &rhs->toString()->asLinear());
+ }
+ *result = FoldComparison(jsop_, comp, 0);
+ return true;
+ }
+
+ if (compareType_ == Compare_UInt32) {
+ *result = FoldComparison(jsop_, uint32_t(lhs->toInt32()),
+ uint32_t(rhs->toInt32()));
+ return true;
+ }
+
+ if (compareType_ == Compare_Int64) {
+ *result = FoldComparison(jsop_, lhs->toInt64(), rhs->toInt64());
+ return true;
+ }
+
+ if (compareType_ == Compare_UInt64) {
+ *result = FoldComparison(jsop_, uint64_t(lhs->toInt64()),
+ uint64_t(rhs->toInt64()));
+ return true;
+ }
+
+ if (lhs->isTypeRepresentableAsDouble() &&
+ rhs->isTypeRepresentableAsDouble()) {
+ *result =
+ FoldComparison(jsop_, lhs->numberToDouble(), rhs->numberToDouble());
+ return true;
+ }
+
+ return false;
+}
+
+MDefinition* MCompare::tryFoldTypeOf(TempAllocator& alloc) {
+ auto typeOfPair = IsTypeOfCompare(this);
+ if (!typeOfPair) {
+ return this;
+ }
+ auto [typeOfName, type] = *typeOfPair;
+ auto* typeOf = typeOfName->input()->toTypeOf();
+
+ auto* input = typeOf->input();
+ MOZ_ASSERT(input->type() == MIRType::Value ||
+ input->type() == MIRType::Object);
+
+ // Constant typeof folding handles the other cases.
+ MOZ_ASSERT_IF(input->type() == MIRType::Object, type == JSTYPE_UNDEFINED ||
+ type == JSTYPE_OBJECT ||
+ type == JSTYPE_FUNCTION);
+
+ MOZ_ASSERT(type != JSTYPE_LIMIT, "unknown typeof strings folded earlier");
+
+ // If there's only a single use, assume this |typeof| is used in a simple
+ // comparison context.
+ //
+ // if (typeof thing === "number") { ... }
+ //
+ // It'll be compiled into something similar to:
+ //
+ // if (IsNumber(thing)) { ... }
+ //
+ // This heuristic can go wrong when repeated |typeof| are used in consecutive
+ // if-statements.
+ //
+ // if (typeof thing === "number") { ... }
+ // else if (typeof thing === "string") { ... }
+ // ... repeated for all possible types
+ //
+ // In that case it'd more efficient to emit MTypeOf compared to MTypeOfIs. We
+ // don't yet handle that case, because it'd require a separate optimization
+ // pass to correctly detect it.
+ if (typeOfName->hasOneUse()) {
+ return MTypeOfIs::New(alloc, input, jsop(), type);
+ }
+
+ MConstant* cst = MConstant::New(alloc, Int32Value(type));
+ block()->insertBefore(this, cst);
+
+ return MCompare::New(alloc, typeOf, cst, jsop(), MCompare::Compare_Int32);
+}
+
+MDefinition* MCompare::tryFoldCharCompare(TempAllocator& alloc) {
+ if (compareType() != Compare_String) {
+ return this;
+ }
+
+ MDefinition* left = lhs();
+ MOZ_ASSERT(left->type() == MIRType::String);
+
+ MDefinition* right = rhs();
+ MOZ_ASSERT(right->type() == MIRType::String);
+
+ // |str[i]| is compiled as |MFromCharCode(MCharCodeAt(str, i))|.
+ auto isCharAccess = [](MDefinition* ins) {
+ return ins->isFromCharCode() &&
+ ins->toFromCharCode()->input()->isCharCodeAt();
+ };
+
+ if (left->isConstant() || right->isConstant()) {
+ // Try to optimize |MConstant(string) <compare> (MFromCharCode MCharCodeAt)|
+ // as |MConstant(charcode) <compare> MCharCodeAt|.
+ MConstant* constant;
+ MDefinition* operand;
+ if (left->isConstant()) {
+ constant = left->toConstant();
+ operand = right;
+ } else {
+ constant = right->toConstant();
+ operand = left;
+ }
+
+ if (constant->toString()->length() != 1 || !isCharAccess(operand)) {
+ return this;
+ }
+
+ char16_t charCode = constant->toString()->asLinear().latin1OrTwoByteChar(0);
+ MConstant* charCodeConst = MConstant::New(alloc, Int32Value(charCode));
+ block()->insertBefore(this, charCodeConst);
+
+ MDefinition* charCodeAt = operand->toFromCharCode()->input();
+
+ if (left->isConstant()) {
+ left = charCodeConst;
+ right = charCodeAt;
+ } else {
+ left = charCodeAt;
+ right = charCodeConst;
+ }
+ } else if (isCharAccess(left) && isCharAccess(right)) {
+ // Try to optimize |(MFromCharCode MCharCodeAt) <compare> (MFromCharCode
+ // MCharCodeAt)| as |MCharCodeAt <compare> MCharCodeAt|.
+
+ left = left->toFromCharCode()->input();
+ right = right->toFromCharCode()->input();
+ } else {
+ return this;
+ }
+
+ return MCompare::New(alloc, left, right, jsop(), MCompare::Compare_Int32);
+}
+
+MDefinition* MCompare::tryFoldStringCompare(TempAllocator& alloc) {
+ if (compareType() != Compare_String) {
+ return this;
+ }
+
+ MDefinition* left = lhs();
+ MOZ_ASSERT(left->type() == MIRType::String);
+
+ MDefinition* right = rhs();
+ MOZ_ASSERT(right->type() == MIRType::String);
+
+ if (!left->isConstant() && !right->isConstant()) {
+ return this;
+ }
+
+ // Try to optimize |string <compare> MConstant("")| as |MStringLength(string)
+ // <compare> MConstant(0)|.
+
+ MConstant* constant =
+ left->isConstant() ? left->toConstant() : right->toConstant();
+ if (!constant->toString()->empty()) {
+ return this;
+ }
+
+ MDefinition* operand = left->isConstant() ? right : left;
+
+ auto* strLength = MStringLength::New(alloc, operand);
+ block()->insertBefore(this, strLength);
+
+ auto* zero = MConstant::New(alloc, Int32Value(0));
+ block()->insertBefore(this, zero);
+
+ if (left->isConstant()) {
+ left = zero;
+ right = strLength;
+ } else {
+ left = strLength;
+ right = zero;
+ }
+
+ return MCompare::New(alloc, left, right, jsop(), MCompare::Compare_Int32);
+}
+
+MDefinition* MCompare::tryFoldStringSubstring(TempAllocator& alloc) {
+ if (compareType() != Compare_String) {
+ return this;
+ }
+ if (!IsEqualityOp(jsop())) {
+ return this;
+ }
+
+ auto* left = lhs();
+ MOZ_ASSERT(left->type() == MIRType::String);
+
+ auto* right = rhs();
+ MOZ_ASSERT(right->type() == MIRType::String);
+
+ // One operand must be a constant string.
+ if (!left->isConstant() && !right->isConstant()) {
+ return this;
+ }
+
+ // The constant string must be non-empty.
+ auto* constant =
+ left->isConstant() ? left->toConstant() : right->toConstant();
+ if (constant->toString()->empty()) {
+ return this;
+ }
+
+ // The other operand must be a substring operation.
+ auto* operand = left->isConstant() ? right : left;
+ if (!operand->isSubstr()) {
+ return this;
+ }
+
+ // We want to match this pattern:
+ // Substr(string, Constant(0), Min(Constant(length), StringLength(string)))
+ auto* substr = operand->toSubstr();
+
+ auto isConstantZero = [](auto* def) {
+ return def->isConstant() && def->toConstant()->isInt32(0);
+ };
+
+ if (!isConstantZero(substr->begin())) {
+ return this;
+ }
+
+ auto* length = substr->length();
+ if (length->isBitOr()) {
+ // Unnecessary bit-ops haven't yet been removed.
+ auto* bitOr = length->toBitOr();
+ if (isConstantZero(bitOr->lhs())) {
+ length = bitOr->rhs();
+ } else if (isConstantZero(bitOr->rhs())) {
+ length = bitOr->lhs();
+ }
+ }
+ if (!length->isMinMax() || length->toMinMax()->isMax()) {
+ return this;
+ }
+
+ auto* min = length->toMinMax();
+ if (!min->lhs()->isConstant() && !min->rhs()->isConstant()) {
+ return this;
+ }
+
+ auto* minConstant = min->lhs()->isConstant() ? min->lhs()->toConstant()
+ : min->rhs()->toConstant();
+
+ auto* minOperand = min->lhs()->isConstant() ? min->rhs() : min->lhs();
+ if (!minOperand->isStringLength() ||
+ minOperand->toStringLength()->string() != substr->string()) {
+ return this;
+ }
+
+ static_assert(JSString::MAX_LENGTH < INT32_MAX,
+ "string length can be casted to int32_t");
+
+ // Ensure the string length matches the substring's length.
+ if (!minConstant->isInt32(int32_t(constant->toString()->length()))) {
+ return this;
+ }
+
+ // Now fold code like |str.substring(0, 2) == "aa"| to |str.startsWith("aa")|.
+
+ auto* startsWith = MStringStartsWith::New(alloc, substr->string(), constant);
+ if (jsop() == JSOp::Eq || jsop() == JSOp::StrictEq) {
+ return startsWith;
+ }
+
+ // Invert for inequality.
+ MOZ_ASSERT(jsop() == JSOp::Ne || jsop() == JSOp::StrictNe);
+
+ block()->insertBefore(this, startsWith);
+ return MNot::New(alloc, startsWith);
+}
+
+MDefinition* MCompare::tryFoldStringIndexOf(TempAllocator& alloc) {
+ if (compareType() != Compare_Int32) {
+ return this;
+ }
+ if (!IsEqualityOp(jsop())) {
+ return this;
+ }
+
+ auto* left = lhs();
+ MOZ_ASSERT(left->type() == MIRType::Int32);
+
+ auto* right = rhs();
+ MOZ_ASSERT(right->type() == MIRType::Int32);
+
+ // One operand must be a constant integer.
+ if (!left->isConstant() && !right->isConstant()) {
+ return this;
+ }
+
+ // The constant must be zero.
+ auto* constant =
+ left->isConstant() ? left->toConstant() : right->toConstant();
+ if (!constant->isInt32(0)) {
+ return this;
+ }
+
+ // The other operand must be an indexOf operation.
+ auto* operand = left->isConstant() ? right : left;
+ if (!operand->isStringIndexOf()) {
+ return this;
+ }
+
+ // Fold |str.indexOf(searchStr) == 0| to |str.startsWith(searchStr)|.
+
+ auto* indexOf = operand->toStringIndexOf();
+ auto* startsWith =
+ MStringStartsWith::New(alloc, indexOf->string(), indexOf->searchString());
+ if (jsop() == JSOp::Eq || jsop() == JSOp::StrictEq) {
+ return startsWith;
+ }
+
+ // Invert for inequality.
+ MOZ_ASSERT(jsop() == JSOp::Ne || jsop() == JSOp::StrictNe);
+
+ block()->insertBefore(this, startsWith);
+ return MNot::New(alloc, startsWith);
+}
+
+MDefinition* MCompare::foldsTo(TempAllocator& alloc) {
+ bool result;
+
+ if (tryFold(&result) || evaluateConstantOperands(alloc, &result)) {
+ if (type() == MIRType::Int32) {
+ return MConstant::New(alloc, Int32Value(result));
+ }
+
+ MOZ_ASSERT(type() == MIRType::Boolean);
+ return MConstant::New(alloc, BooleanValue(result));
+ }
+
+ if (MDefinition* folded = tryFoldTypeOf(alloc); folded != this) {
+ return folded;
+ }
+
+ if (MDefinition* folded = tryFoldCharCompare(alloc); folded != this) {
+ return folded;
+ }
+
+ if (MDefinition* folded = tryFoldStringCompare(alloc); folded != this) {
+ return folded;
+ }
+
+ if (MDefinition* folded = tryFoldStringSubstring(alloc); folded != this) {
+ return folded;
+ }
+
+ if (MDefinition* folded = tryFoldStringIndexOf(alloc); folded != this) {
+ return folded;
+ }
+
+ return this;
+}
+
+void MCompare::trySpecializeFloat32(TempAllocator& alloc) {
+ if (AllOperandsCanProduceFloat32(this) && compareType_ == Compare_Double) {
+ compareType_ = Compare_Float32;
+ } else {
+ ConvertOperandsToDouble(this, alloc);
+ }
+}
+
+MDefinition* MNot::foldsTo(TempAllocator& alloc) {
+ // Fold if the input is constant
+ if (MConstant* inputConst = input()->maybeConstantValue()) {
+ bool b;
+ if (inputConst->valueToBoolean(&b)) {
+ if (type() == MIRType::Int32 || type() == MIRType::Int64) {
+ return MConstant::New(alloc, Int32Value(!b));
+ }
+ return MConstant::New(alloc, BooleanValue(!b));
+ }
+ }
+
+ // If the operand of the Not is itself a Not, they cancel out. But we can't
+ // always convert Not(Not(x)) to x because that may loose the conversion to
+ // boolean. We can simplify Not(Not(Not(x))) to Not(x) though.
+ MDefinition* op = getOperand(0);
+ if (op->isNot()) {
+ MDefinition* opop = op->getOperand(0);
+ if (opop->isNot()) {
+ return opop;
+ }
+ }
+
+ // Not of an undefined or null value is always true
+ if (input()->type() == MIRType::Undefined ||
+ input()->type() == MIRType::Null) {
+ return MConstant::New(alloc, BooleanValue(true));
+ }
+
+ // Not of a symbol is always false.
+ if (input()->type() == MIRType::Symbol) {
+ return MConstant::New(alloc, BooleanValue(false));
+ }
+
+ return this;
+}
+
+void MNot::trySpecializeFloat32(TempAllocator& alloc) {
+ (void)EnsureFloatInputOrConvert(this, alloc);
+}
+
+#ifdef JS_JITSPEW
+void MBeta::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+
+ out.printf(" ");
+ comparison_->dump(out);
+}
+#endif
+
+AliasSet MCreateThis::getAliasSet() const {
+ return AliasSet::Load(AliasSet::Any);
+}
+
+bool MGetArgumentsObjectArg::congruentTo(const MDefinition* ins) const {
+ if (!ins->isGetArgumentsObjectArg()) {
+ return false;
+ }
+ if (ins->toGetArgumentsObjectArg()->argno() != argno()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MGetArgumentsObjectArg::getAliasSet() const {
+ return AliasSet::Load(AliasSet::Any);
+}
+
+AliasSet MSetArgumentsObjectArg::getAliasSet() const {
+ return AliasSet::Store(AliasSet::Any);
+}
+
+MObjectState::MObjectState(MObjectState* state)
+ : MVariadicInstruction(classOpcode),
+ numSlots_(state->numSlots_),
+ numFixedSlots_(state->numFixedSlots_) {
+ // This instruction is only used as a summary for bailout paths.
+ setResultType(MIRType::Object);
+ setRecoveredOnBailout();
+}
+
+MObjectState::MObjectState(JSObject* templateObject)
+ : MObjectState(templateObject->as<NativeObject>().shape()) {}
+
+MObjectState::MObjectState(const Shape* shape)
+ : MVariadicInstruction(classOpcode) {
+ // This instruction is only used as a summary for bailout paths.
+ setResultType(MIRType::Object);
+ setRecoveredOnBailout();
+
+ numSlots_ = shape->asShared().slotSpan();
+ numFixedSlots_ = shape->asShared().numFixedSlots();
+}
+
+/* static */
+JSObject* MObjectState::templateObjectOf(MDefinition* obj) {
+ // MNewPlainObject uses a shape constant, not an object.
+ MOZ_ASSERT(!obj->isNewPlainObject());
+
+ if (obj->isNewObject()) {
+ return obj->toNewObject()->templateObject();
+ } else if (obj->isNewCallObject()) {
+ return obj->toNewCallObject()->templateObject();
+ } else if (obj->isNewIterator()) {
+ return obj->toNewIterator()->templateObject();
+ }
+
+ MOZ_CRASH("unreachable");
+}
+
+bool MObjectState::init(TempAllocator& alloc, MDefinition* obj) {
+ if (!MVariadicInstruction::init(alloc, numSlots() + 1)) {
+ return false;
+ }
+ // +1, for the Object.
+ initOperand(0, obj);
+ return true;
+}
+
+void MObjectState::initFromTemplateObject(TempAllocator& alloc,
+ MDefinition* undefinedVal) {
+ if (object()->isNewPlainObject()) {
+ MOZ_ASSERT(object()->toNewPlainObject()->shape()->asShared().slotSpan() ==
+ numSlots());
+ for (size_t i = 0; i < numSlots(); i++) {
+ initSlot(i, undefinedVal);
+ }
+ return;
+ }
+
+ JSObject* templateObject = templateObjectOf(object());
+
+ // Initialize all the slots of the object state with the value contained in
+ // the template object. This is needed to account values which are baked in
+ // the template objects and not visible in IonMonkey, such as the
+ // uninitialized-lexical magic value of call objects.
+
+ MOZ_ASSERT(templateObject->is<NativeObject>());
+ NativeObject& nativeObject = templateObject->as<NativeObject>();
+ MOZ_ASSERT(nativeObject.slotSpan() == numSlots());
+
+ for (size_t i = 0; i < numSlots(); i++) {
+ Value val = nativeObject.getSlot(i);
+ MDefinition* def = undefinedVal;
+ if (!val.isUndefined()) {
+ MConstant* ins = MConstant::New(alloc, val);
+ block()->insertBefore(this, ins);
+ def = ins;
+ }
+ initSlot(i, def);
+ }
+}
+
+MObjectState* MObjectState::New(TempAllocator& alloc, MDefinition* obj) {
+ MObjectState* res;
+ if (obj->isNewPlainObject()) {
+ const Shape* shape = obj->toNewPlainObject()->shape();
+ res = new (alloc) MObjectState(shape);
+ } else {
+ JSObject* templateObject = templateObjectOf(obj);
+ MOZ_ASSERT(templateObject, "Unexpected object creation.");
+ res = new (alloc) MObjectState(templateObject);
+ }
+
+ if (!res || !res->init(alloc, obj)) {
+ return nullptr;
+ }
+ return res;
+}
+
+MObjectState* MObjectState::Copy(TempAllocator& alloc, MObjectState* state) {
+ MObjectState* res = new (alloc) MObjectState(state);
+ if (!res || !res->init(alloc, state->object())) {
+ return nullptr;
+ }
+ for (size_t i = 0; i < res->numSlots(); i++) {
+ res->initSlot(i, state->getSlot(i));
+ }
+ return res;
+}
+
+MArrayState::MArrayState(MDefinition* arr) : MVariadicInstruction(classOpcode) {
+ // This instruction is only used as a summary for bailout paths.
+ setResultType(MIRType::Object);
+ setRecoveredOnBailout();
+ if (arr->isNewArrayObject()) {
+ numElements_ = arr->toNewArrayObject()->length();
+ } else {
+ numElements_ = arr->toNewArray()->length();
+ }
+}
+
+bool MArrayState::init(TempAllocator& alloc, MDefinition* obj,
+ MDefinition* len) {
+ if (!MVariadicInstruction::init(alloc, numElements() + 2)) {
+ return false;
+ }
+ // +1, for the Array object.
+ initOperand(0, obj);
+ // +1, for the length value of the array.
+ initOperand(1, len);
+ return true;
+}
+
+void MArrayState::initFromTemplateObject(TempAllocator& alloc,
+ MDefinition* undefinedVal) {
+ for (size_t i = 0; i < numElements(); i++) {
+ initElement(i, undefinedVal);
+ }
+}
+
+MArrayState* MArrayState::New(TempAllocator& alloc, MDefinition* arr,
+ MDefinition* initLength) {
+ MArrayState* res = new (alloc) MArrayState(arr);
+ if (!res || !res->init(alloc, arr, initLength)) {
+ return nullptr;
+ }
+ return res;
+}
+
+MArrayState* MArrayState::Copy(TempAllocator& alloc, MArrayState* state) {
+ MDefinition* arr = state->array();
+ MDefinition* len = state->initializedLength();
+ MArrayState* res = new (alloc) MArrayState(arr);
+ if (!res || !res->init(alloc, arr, len)) {
+ return nullptr;
+ }
+ for (size_t i = 0; i < res->numElements(); i++) {
+ res->initElement(i, state->getElement(i));
+ }
+ return res;
+}
+
+MNewArray::MNewArray(uint32_t length, MConstant* templateConst,
+ gc::InitialHeap initialHeap, bool vmCall)
+ : MUnaryInstruction(classOpcode, templateConst),
+ length_(length),
+ initialHeap_(initialHeap),
+ vmCall_(vmCall) {
+ setResultType(MIRType::Object);
+}
+
+MDefinition::AliasType MLoadFixedSlot::mightAlias(
+ const MDefinition* def) const {
+ if (def->isStoreFixedSlot()) {
+ const MStoreFixedSlot* store = def->toStoreFixedSlot();
+ if (store->slot() != slot()) {
+ return AliasType::NoAlias;
+ }
+ if (store->object() != object()) {
+ return AliasType::MayAlias;
+ }
+ return AliasType::MustAlias;
+ }
+ return AliasType::MayAlias;
+}
+
+MDefinition* MLoadFixedSlot::foldsTo(TempAllocator& alloc) {
+ if (MDefinition* def = foldsToStore(alloc)) {
+ return def;
+ }
+
+ return this;
+}
+
+MDefinition::AliasType MLoadFixedSlotAndUnbox::mightAlias(
+ const MDefinition* def) const {
+ if (def->isStoreFixedSlot()) {
+ const MStoreFixedSlot* store = def->toStoreFixedSlot();
+ if (store->slot() != slot()) {
+ return AliasType::NoAlias;
+ }
+ if (store->object() != object()) {
+ return AliasType::MayAlias;
+ }
+ return AliasType::MustAlias;
+ }
+ return AliasType::MayAlias;
+}
+
+MDefinition* MLoadFixedSlotAndUnbox::foldsTo(TempAllocator& alloc) {
+ if (MDefinition* def = foldsToStore(alloc)) {
+ return def;
+ }
+
+ return this;
+}
+
+MDefinition* MWasmExtendU32Index::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = this->input();
+ if (input->isConstant()) {
+ return MConstant::NewInt64(
+ alloc, int64_t(uint32_t(input->toConstant()->toInt32())));
+ }
+
+ return this;
+}
+
+MDefinition* MWasmWrapU32Index::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = this->input();
+ if (input->isConstant()) {
+ return MConstant::New(
+ alloc, Int32Value(int32_t(uint32_t(input->toConstant()->toInt64()))));
+ }
+
+ return this;
+}
+
+// Some helpers for folding wasm and/or/xor on int32/64 values. Rather than
+// duplicating these for 32 and 64-bit values, all folding is done on 64-bit
+// values and masked for the 32-bit case.
+
+const uint64_t Low32Mask = uint64_t(0xFFFFFFFFULL);
+
+// Routines to check and disassemble values.
+
+static bool IsIntegralConstant(const MDefinition* def) {
+ return def->isConstant() &&
+ (def->type() == MIRType::Int32 || def->type() == MIRType::Int64);
+}
+
+static uint64_t GetIntegralConstant(const MDefinition* def) {
+ if (def->type() == MIRType::Int32) {
+ return uint64_t(def->toConstant()->toInt32()) & Low32Mask;
+ }
+ return uint64_t(def->toConstant()->toInt64());
+}
+
+static bool IsIntegralConstantZero(const MDefinition* def) {
+ return IsIntegralConstant(def) && GetIntegralConstant(def) == 0;
+}
+
+static bool IsIntegralConstantOnes(const MDefinition* def) {
+ uint64_t ones = def->type() == MIRType::Int32 ? Low32Mask : ~uint64_t(0);
+ return IsIntegralConstant(def) && GetIntegralConstant(def) == ones;
+}
+
+// Routines to create values.
+static MDefinition* ToIntegralConstant(TempAllocator& alloc, MIRType ty,
+ uint64_t val) {
+ switch (ty) {
+ case MIRType::Int32:
+ return MConstant::New(alloc,
+ Int32Value(int32_t(uint32_t(val & Low32Mask))));
+ case MIRType::Int64:
+ return MConstant::NewInt64(alloc, int64_t(val));
+ default:
+ MOZ_CRASH();
+ }
+}
+
+static MDefinition* ZeroOfType(TempAllocator& alloc, MIRType ty) {
+ return ToIntegralConstant(alloc, ty, 0);
+}
+
+static MDefinition* OnesOfType(TempAllocator& alloc, MIRType ty) {
+ return ToIntegralConstant(alloc, ty, ~uint64_t(0));
+}
+
+MDefinition* MWasmBinaryBitwise::foldsTo(TempAllocator& alloc) {
+ MOZ_ASSERT(op() == Opcode::WasmBinaryBitwise);
+ MOZ_ASSERT(type() == MIRType::Int32 || type() == MIRType::Int64);
+
+ MDefinition* argL = getOperand(0);
+ MDefinition* argR = getOperand(1);
+ MOZ_ASSERT(argL->type() == type() && argR->type() == type());
+
+ // The args are the same (SSA name)
+ if (argL == argR) {
+ switch (subOpcode()) {
+ case SubOpcode::And:
+ case SubOpcode::Or:
+ return argL;
+ case SubOpcode::Xor:
+ return ZeroOfType(alloc, type());
+ default:
+ MOZ_CRASH();
+ }
+ }
+
+ // Both args constant
+ if (IsIntegralConstant(argL) && IsIntegralConstant(argR)) {
+ uint64_t valL = GetIntegralConstant(argL);
+ uint64_t valR = GetIntegralConstant(argR);
+ uint64_t val = valL;
+ switch (subOpcode()) {
+ case SubOpcode::And:
+ val &= valR;
+ break;
+ case SubOpcode::Or:
+ val |= valR;
+ break;
+ case SubOpcode::Xor:
+ val ^= valR;
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ return ToIntegralConstant(alloc, type(), val);
+ }
+
+ // Left arg is zero
+ if (IsIntegralConstantZero(argL)) {
+ switch (subOpcode()) {
+ case SubOpcode::And:
+ return ZeroOfType(alloc, type());
+ case SubOpcode::Or:
+ case SubOpcode::Xor:
+ return argR;
+ default:
+ MOZ_CRASH();
+ }
+ }
+
+ // Right arg is zero
+ if (IsIntegralConstantZero(argR)) {
+ switch (subOpcode()) {
+ case SubOpcode::And:
+ return ZeroOfType(alloc, type());
+ case SubOpcode::Or:
+ case SubOpcode::Xor:
+ return argL;
+ default:
+ MOZ_CRASH();
+ }
+ }
+
+ // Left arg is ones
+ if (IsIntegralConstantOnes(argL)) {
+ switch (subOpcode()) {
+ case SubOpcode::And:
+ return argR;
+ case SubOpcode::Or:
+ return OnesOfType(alloc, type());
+ case SubOpcode::Xor:
+ return MBitNot::New(alloc, argR);
+ default:
+ MOZ_CRASH();
+ }
+ }
+
+ // Right arg is ones
+ if (IsIntegralConstantOnes(argR)) {
+ switch (subOpcode()) {
+ case SubOpcode::And:
+ return argL;
+ case SubOpcode::Or:
+ return OnesOfType(alloc, type());
+ case SubOpcode::Xor:
+ return MBitNot::New(alloc, argL);
+ default:
+ MOZ_CRASH();
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MWasmAddOffset::foldsTo(TempAllocator& alloc) {
+ MDefinition* baseArg = base();
+ if (!baseArg->isConstant()) {
+ return this;
+ }
+
+ if (baseArg->type() == MIRType::Int32) {
+ CheckedInt<uint32_t> ptr = baseArg->toConstant()->toInt32();
+ ptr += offset();
+ if (!ptr.isValid()) {
+ return this;
+ }
+ return MConstant::New(alloc, Int32Value(ptr.value()));
+ }
+
+ MOZ_ASSERT(baseArg->type() == MIRType::Int64);
+ CheckedInt<uint64_t> ptr = baseArg->toConstant()->toInt64();
+ ptr += offset();
+ if (!ptr.isValid()) {
+ return this;
+ }
+ return MConstant::NewInt64(alloc, ptr.value());
+}
+
+bool MWasmAlignmentCheck::congruentTo(const MDefinition* ins) const {
+ if (!ins->isWasmAlignmentCheck()) {
+ return false;
+ }
+ const MWasmAlignmentCheck* check = ins->toWasmAlignmentCheck();
+ return byteSize_ == check->byteSize() && congruentIfOperandsEqual(check);
+}
+
+MDefinition::AliasType MAsmJSLoadHeap::mightAlias(
+ const MDefinition* def) const {
+ if (def->isAsmJSStoreHeap()) {
+ const MAsmJSStoreHeap* store = def->toAsmJSStoreHeap();
+ if (store->accessType() != accessType()) {
+ return AliasType::MayAlias;
+ }
+ if (!base()->isConstant() || !store->base()->isConstant()) {
+ return AliasType::MayAlias;
+ }
+ const MConstant* otherBase = store->base()->toConstant();
+ if (base()->toConstant()->equals(otherBase)) {
+ return AliasType::MayAlias;
+ }
+ return AliasType::NoAlias;
+ }
+ return AliasType::MayAlias;
+}
+
+bool MAsmJSLoadHeap::congruentTo(const MDefinition* ins) const {
+ if (!ins->isAsmJSLoadHeap()) {
+ return false;
+ }
+ const MAsmJSLoadHeap* load = ins->toAsmJSLoadHeap();
+ return load->accessType() == accessType() && congruentIfOperandsEqual(load);
+}
+
+MDefinition::AliasType MWasmLoadGlobalVar::mightAlias(
+ const MDefinition* def) const {
+ if (def->isWasmStoreGlobalVar()) {
+ const MWasmStoreGlobalVar* store = def->toWasmStoreGlobalVar();
+ return store->globalDataOffset() == globalDataOffset_ ? AliasType::MayAlias
+ : AliasType::NoAlias;
+ }
+
+ return AliasType::MayAlias;
+}
+
+MDefinition::AliasType MWasmLoadGlobalCell::mightAlias(
+ const MDefinition* def) const {
+ if (def->isWasmStoreGlobalCell()) {
+ // No globals of different type can alias. See bug 1467415 comment 3.
+ if (type() != def->toWasmStoreGlobalCell()->value()->type()) {
+ return AliasType::NoAlias;
+ }
+
+ // We could do better here. We're dealing with two indirect globals.
+ // If at at least one of them is created in this module, then they
+ // can't alias -- in other words they can only alias if they are both
+ // imported. That would require having a flag on globals to indicate
+ // which are imported. See bug 1467415 comment 3, 4th rule.
+ }
+
+ return AliasType::MayAlias;
+}
+
+HashNumber MWasmLoadGlobalVar::valueHash() const {
+ // Same comment as in MWasmLoadGlobalVar::congruentTo() applies here.
+ HashNumber hash = MDefinition::valueHash();
+ hash = addU32ToHash(hash, globalDataOffset_);
+ return hash;
+}
+
+bool MWasmLoadGlobalVar::congruentTo(const MDefinition* ins) const {
+ if (!ins->isWasmLoadGlobalVar()) {
+ return false;
+ }
+
+ const MWasmLoadGlobalVar* other = ins->toWasmLoadGlobalVar();
+
+ // We don't need to consider the isConstant_ markings here, because
+ // equivalence of offsets implies equivalence of constness.
+ bool sameOffsets = globalDataOffset_ == other->globalDataOffset_;
+ MOZ_ASSERT_IF(sameOffsets, isConstant_ == other->isConstant_);
+
+ // We omit checking congruence of the operands. There is only one
+ // operand, the instance pointer, and it only ever has one value within the
+ // domain of optimization. If that should ever change then operand
+ // congruence checking should be reinstated.
+ return sameOffsets /* && congruentIfOperandsEqual(other) */;
+}
+
+MDefinition* MWasmLoadGlobalVar::foldsTo(TempAllocator& alloc) {
+ if (!dependency() || !dependency()->isWasmStoreGlobalVar()) {
+ return this;
+ }
+
+ MWasmStoreGlobalVar* store = dependency()->toWasmStoreGlobalVar();
+ if (!store->block()->dominates(block())) {
+ return this;
+ }
+
+ if (store->globalDataOffset() != globalDataOffset()) {
+ return this;
+ }
+
+ if (store->value()->type() != type()) {
+ return this;
+ }
+
+ return store->value();
+}
+
+bool MWasmLoadGlobalCell::congruentTo(const MDefinition* ins) const {
+ if (!ins->isWasmLoadGlobalCell()) {
+ return false;
+ }
+ const MWasmLoadGlobalCell* other = ins->toWasmLoadGlobalCell();
+ return congruentIfOperandsEqual(other);
+}
+
+#ifdef ENABLE_WASM_SIMD
+MDefinition* MWasmTernarySimd128::foldsTo(TempAllocator& alloc) {
+ if (simdOp() == wasm::SimdOp::V128Bitselect) {
+ if (v2()->op() == MDefinition::Opcode::WasmFloatConstant) {
+ int8_t shuffle[16];
+ if (specializeBitselectConstantMaskAsShuffle(shuffle)) {
+ return BuildWasmShuffleSimd128(alloc, shuffle, v0(), v1());
+ }
+ } else if (canRelaxBitselect()) {
+ return MWasmTernarySimd128::New(alloc, v0(), v1(), v2(),
+ wasm::SimdOp::I8x16RelaxedLaneSelect);
+ }
+ }
+ return this;
+}
+
+inline static bool MatchSpecificShift(MDefinition* instr,
+ wasm::SimdOp simdShiftOp,
+ int shiftValue) {
+ return instr->isWasmShiftSimd128() &&
+ instr->toWasmShiftSimd128()->simdOp() == simdShiftOp &&
+ instr->toWasmShiftSimd128()->rhs()->isConstant() &&
+ instr->toWasmShiftSimd128()->rhs()->toConstant()->toInt32() ==
+ shiftValue;
+}
+
+// Matches MIR subtree that represents PMADDUBSW instruction generated by
+// emscripten. The a and b parameters return subtrees that correspond
+// operands of the instruction, if match is found.
+static bool MatchPmaddubswSequence(MWasmBinarySimd128* lhs,
+ MWasmBinarySimd128* rhs, MDefinition** a,
+ MDefinition** b) {
+ MOZ_ASSERT(lhs->simdOp() == wasm::SimdOp::I16x8Mul &&
+ rhs->simdOp() == wasm::SimdOp::I16x8Mul);
+ // The emscripten/LLVM produced the following sequence for _mm_maddubs_epi16:
+ //
+ // return _mm_adds_epi16(
+ // _mm_mullo_epi16(
+ // _mm_and_si128(__a, _mm_set1_epi16(0x00FF)),
+ // _mm_srai_epi16(_mm_slli_epi16(__b, 8), 8)),
+ // _mm_mullo_epi16(_mm_srli_epi16(__a, 8), _mm_srai_epi16(__b, 8)));
+ //
+ // This will roughly correspond the following MIR:
+ // MWasmBinarySimd128[I16x8AddSatS]
+ // |-- lhs: MWasmBinarySimd128[I16x8Mul] (lhs)
+ // | |-- lhs: MWasmBinarySimd128WithConstant[V128And] (op0)
+ // | | |-- lhs: a
+ // | | -- rhs: SimdConstant::SplatX8(0x00FF)
+ // | -- rhs: MWasmShiftSimd128[I16x8ShrS] (op1)
+ // | |-- lhs: MWasmShiftSimd128[I16x8Shl]
+ // | | |-- lhs: b
+ // | | -- rhs: MConstant[8]
+ // | -- rhs: MConstant[8]
+ // -- rhs: MWasmBinarySimd128[I16x8Mul] (rhs)
+ // |-- lhs: MWasmShiftSimd128[I16x8ShrU] (op2)
+ // | |-- lhs: a
+ // | |-- rhs: MConstant[8]
+ // -- rhs: MWasmShiftSimd128[I16x8ShrS] (op3)
+ // |-- lhs: b
+ // -- rhs: MConstant[8]
+
+ // The I16x8AddSatS and I16x8Mul are commutative, so their operands
+ // may be swapped. Rearrange op0, op1, op2, op3 to be in the order
+ // noted above.
+ MDefinition *op0 = lhs->lhs(), *op1 = lhs->rhs(), *op2 = rhs->lhs(),
+ *op3 = rhs->rhs();
+ if (op1->isWasmBinarySimd128WithConstant()) {
+ // Move MWasmBinarySimd128WithConstant[V128And] as first operand in lhs.
+ std::swap(op0, op1);
+ } else if (op3->isWasmBinarySimd128WithConstant()) {
+ // Move MWasmBinarySimd128WithConstant[V128And] as first operand in rhs.
+ std::swap(op2, op3);
+ }
+ if (op2->isWasmBinarySimd128WithConstant()) {
+ // The lhs and rhs are swapped.
+ // Make MWasmBinarySimd128WithConstant[V128And] to be op0.
+ std::swap(op0, op2);
+ std::swap(op1, op3);
+ }
+ if (op2->isWasmShiftSimd128() &&
+ op2->toWasmShiftSimd128()->simdOp() == wasm::SimdOp::I16x8ShrS) {
+ // The op2 and op3 appears to be in wrong order, swap.
+ std::swap(op2, op3);
+ }
+
+ // Check all instructions SIMD code and constant values for assigned
+ // names op0, op1, op2, op3 (see diagram above).
+ const uint16_t const00FF[8] = {255, 255, 255, 255, 255, 255, 255, 255};
+ if (!op0->isWasmBinarySimd128WithConstant() ||
+ op0->toWasmBinarySimd128WithConstant()->simdOp() !=
+ wasm::SimdOp::V128And ||
+ memcmp(op0->toWasmBinarySimd128WithConstant()->rhs().bytes(), const00FF,
+ 16) != 0 ||
+ !MatchSpecificShift(op1, wasm::SimdOp::I16x8ShrS, 8) ||
+ !MatchSpecificShift(op2, wasm::SimdOp::I16x8ShrU, 8) ||
+ !MatchSpecificShift(op3, wasm::SimdOp::I16x8ShrS, 8) ||
+ !MatchSpecificShift(op1->toWasmShiftSimd128()->lhs(),
+ wasm::SimdOp::I16x8Shl, 8)) {
+ return false;
+ }
+
+ // Check if the instructions arguments that are subtrees match the
+ // a and b assignments. May depend on GVN behavior.
+ MDefinition* maybeA = op0->toWasmBinarySimd128WithConstant()->lhs();
+ MDefinition* maybeB = op3->toWasmShiftSimd128()->lhs();
+ if (maybeA != op2->toWasmShiftSimd128()->lhs() ||
+ maybeB != op1->toWasmShiftSimd128()->lhs()->toWasmShiftSimd128()->lhs()) {
+ return false;
+ }
+
+ *a = maybeA;
+ *b = maybeB;
+ return true;
+}
+
+MDefinition* MWasmBinarySimd128::foldsTo(TempAllocator& alloc) {
+ if (simdOp() == wasm::SimdOp::I8x16Swizzle && rhs()->isWasmFloatConstant()) {
+ // Specialize swizzle(v, constant) as shuffle(mask, v, zero) to trigger all
+ // our shuffle optimizations. We don't report this rewriting as the report
+ // will be overwritten by the subsequent shuffle analysis.
+ int8_t shuffleMask[16];
+ memcpy(shuffleMask, rhs()->toWasmFloatConstant()->toSimd128().bytes(), 16);
+ for (int i = 0; i < 16; i++) {
+ // Out-of-bounds lanes reference the zero vector; in many cases, the zero
+ // vector is removed by subsequent optimizations.
+ if (shuffleMask[i] < 0 || shuffleMask[i] > 15) {
+ shuffleMask[i] = 16;
+ }
+ }
+ MWasmFloatConstant* zero =
+ MWasmFloatConstant::NewSimd128(alloc, SimdConstant::SplatX4(0));
+ if (!zero) {
+ return nullptr;
+ }
+ block()->insertBefore(this, zero);
+ return BuildWasmShuffleSimd128(alloc, shuffleMask, lhs(), zero);
+ }
+
+ // Specialize var OP const / const OP var when possible.
+ //
+ // As the LIR layer can't directly handle v128 constants as part of its normal
+ // machinery we specialize some nodes here if they have single-use v128
+ // constant arguments. The purpose is to generate code that inlines the
+ // constant in the instruction stream, using either a rip-relative load+op or
+ // quickly-synthesized constant in a scratch on x64. There is a general
+ // assumption here that that is better than generating the constant into an
+ // allocatable register, since that register value could not be reused. (This
+ // ignores the possibility that the constant load could be hoisted).
+
+ if (lhs()->isWasmFloatConstant() != rhs()->isWasmFloatConstant() &&
+ specializeForConstantRhs()) {
+ if (isCommutative() && lhs()->isWasmFloatConstant() && lhs()->hasOneUse()) {
+ return MWasmBinarySimd128WithConstant::New(
+ alloc, rhs(), lhs()->toWasmFloatConstant()->toSimd128(), simdOp());
+ }
+
+ if (rhs()->isWasmFloatConstant() && rhs()->hasOneUse()) {
+ return MWasmBinarySimd128WithConstant::New(
+ alloc, lhs(), rhs()->toWasmFloatConstant()->toSimd128(), simdOp());
+ }
+ }
+
+ // Check special encoding for PMADDUBSW.
+ if (canPmaddubsw() && simdOp() == wasm::SimdOp::I16x8AddSatS &&
+ lhs()->isWasmBinarySimd128() && rhs()->isWasmBinarySimd128() &&
+ lhs()->toWasmBinarySimd128()->simdOp() == wasm::SimdOp::I16x8Mul &&
+ rhs()->toWasmBinarySimd128()->simdOp() == wasm::SimdOp::I16x8Mul) {
+ MDefinition *a, *b;
+ if (MatchPmaddubswSequence(lhs()->toWasmBinarySimd128(),
+ rhs()->toWasmBinarySimd128(), &a, &b)) {
+ return MWasmBinarySimd128::New(alloc, a, b, /* commutative = */ false,
+ wasm::SimdOp::MozPMADDUBSW);
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MWasmScalarToSimd128::foldsTo(TempAllocator& alloc) {
+# ifdef DEBUG
+ auto logging = mozilla::MakeScopeExit([&] {
+ js::wasm::ReportSimdAnalysis("scalar-to-simd128 -> constant folded");
+ });
+# endif
+ if (input()->isConstant()) {
+ MConstant* c = input()->toConstant();
+ switch (simdOp()) {
+ case wasm::SimdOp::I8x16Splat:
+ return MWasmFloatConstant::NewSimd128(
+ alloc, SimdConstant::SplatX16(c->toInt32()));
+ case wasm::SimdOp::I16x8Splat:
+ return MWasmFloatConstant::NewSimd128(
+ alloc, SimdConstant::SplatX8(c->toInt32()));
+ case wasm::SimdOp::I32x4Splat:
+ return MWasmFloatConstant::NewSimd128(
+ alloc, SimdConstant::SplatX4(c->toInt32()));
+ case wasm::SimdOp::I64x2Splat:
+ return MWasmFloatConstant::NewSimd128(
+ alloc, SimdConstant::SplatX2(c->toInt64()));
+ default:
+# ifdef DEBUG
+ logging.release();
+# endif
+ return this;
+ }
+ }
+ if (input()->isWasmFloatConstant()) {
+ MWasmFloatConstant* c = input()->toWasmFloatConstant();
+ switch (simdOp()) {
+ case wasm::SimdOp::F32x4Splat:
+ return MWasmFloatConstant::NewSimd128(
+ alloc, SimdConstant::SplatX4(c->toFloat32()));
+ case wasm::SimdOp::F64x2Splat:
+ return MWasmFloatConstant::NewSimd128(
+ alloc, SimdConstant::SplatX2(c->toDouble()));
+ default:
+# ifdef DEBUG
+ logging.release();
+# endif
+ return this;
+ }
+ }
+# ifdef DEBUG
+ logging.release();
+# endif
+ return this;
+}
+
+template <typename T>
+static bool AllTrue(const T& v) {
+ constexpr size_t count = sizeof(T) / sizeof(*v);
+ static_assert(count == 16 || count == 8 || count == 4 || count == 2);
+ bool result = true;
+ for (unsigned i = 0; i < count; i++) {
+ result = result && v[i] != 0;
+ }
+ return result;
+}
+
+template <typename T>
+static int32_t Bitmask(const T& v) {
+ constexpr size_t count = sizeof(T) / sizeof(*v);
+ constexpr size_t shift = 8 * sizeof(*v) - 1;
+ static_assert(shift == 7 || shift == 15 || shift == 31 || shift == 63);
+ int32_t result = 0;
+ for (unsigned i = 0; i < count; i++) {
+ result = result | int32_t(((v[i] >> shift) & 1) << i);
+ }
+ return result;
+}
+
+MDefinition* MWasmReduceSimd128::foldsTo(TempAllocator& alloc) {
+# ifdef DEBUG
+ auto logging = mozilla::MakeScopeExit([&] {
+ js::wasm::ReportSimdAnalysis("simd128-to-scalar -> constant folded");
+ });
+# endif
+ if (input()->isWasmFloatConstant()) {
+ SimdConstant c = input()->toWasmFloatConstant()->toSimd128();
+ int32_t i32Result = 0;
+ switch (simdOp()) {
+ case wasm::SimdOp::V128AnyTrue:
+ i32Result = !c.isZeroBits();
+ break;
+ case wasm::SimdOp::I8x16AllTrue:
+ i32Result = AllTrue(
+ SimdConstant::CreateSimd128((int8_t*)c.bytes()).asInt8x16());
+ break;
+ case wasm::SimdOp::I8x16Bitmask:
+ i32Result = Bitmask(
+ SimdConstant::CreateSimd128((int8_t*)c.bytes()).asInt8x16());
+ break;
+ case wasm::SimdOp::I16x8AllTrue:
+ i32Result = AllTrue(
+ SimdConstant::CreateSimd128((int16_t*)c.bytes()).asInt16x8());
+ break;
+ case wasm::SimdOp::I16x8Bitmask:
+ i32Result = Bitmask(
+ SimdConstant::CreateSimd128((int16_t*)c.bytes()).asInt16x8());
+ break;
+ case wasm::SimdOp::I32x4AllTrue:
+ i32Result = AllTrue(
+ SimdConstant::CreateSimd128((int32_t*)c.bytes()).asInt32x4());
+ break;
+ case wasm::SimdOp::I32x4Bitmask:
+ i32Result = Bitmask(
+ SimdConstant::CreateSimd128((int32_t*)c.bytes()).asInt32x4());
+ break;
+ case wasm::SimdOp::I64x2AllTrue:
+ i32Result = AllTrue(
+ SimdConstant::CreateSimd128((int64_t*)c.bytes()).asInt64x2());
+ break;
+ case wasm::SimdOp::I64x2Bitmask:
+ i32Result = Bitmask(
+ SimdConstant::CreateSimd128((int64_t*)c.bytes()).asInt64x2());
+ break;
+ case wasm::SimdOp::I8x16ExtractLaneS:
+ i32Result =
+ SimdConstant::CreateSimd128((int8_t*)c.bytes()).asInt8x16()[imm()];
+ break;
+ case wasm::SimdOp::I8x16ExtractLaneU:
+ i32Result = int32_t(SimdConstant::CreateSimd128((int8_t*)c.bytes())
+ .asInt8x16()[imm()]) &
+ 0xFF;
+ break;
+ case wasm::SimdOp::I16x8ExtractLaneS:
+ i32Result =
+ SimdConstant::CreateSimd128((int16_t*)c.bytes()).asInt16x8()[imm()];
+ break;
+ case wasm::SimdOp::I16x8ExtractLaneU:
+ i32Result = int32_t(SimdConstant::CreateSimd128((int16_t*)c.bytes())
+ .asInt16x8()[imm()]) &
+ 0xFFFF;
+ break;
+ case wasm::SimdOp::I32x4ExtractLane:
+ i32Result =
+ SimdConstant::CreateSimd128((int32_t*)c.bytes()).asInt32x4()[imm()];
+ break;
+ case wasm::SimdOp::I64x2ExtractLane:
+ return MConstant::NewInt64(
+ alloc, SimdConstant::CreateSimd128((int64_t*)c.bytes())
+ .asInt64x2()[imm()]);
+ case wasm::SimdOp::F32x4ExtractLane:
+ return MWasmFloatConstant::NewFloat32(
+ alloc, SimdConstant::CreateSimd128((float*)c.bytes())
+ .asFloat32x4()[imm()]);
+ case wasm::SimdOp::F64x2ExtractLane:
+ return MWasmFloatConstant::NewDouble(
+ alloc, SimdConstant::CreateSimd128((double*)c.bytes())
+ .asFloat64x2()[imm()]);
+ default:
+# ifdef DEBUG
+ logging.release();
+# endif
+ return this;
+ }
+ return MConstant::New(alloc, Int32Value(i32Result), MIRType::Int32);
+ }
+# ifdef DEBUG
+ logging.release();
+# endif
+ return this;
+}
+#endif // ENABLE_WASM_SIMD
+
+MDefinition::AliasType MLoadDynamicSlot::mightAlias(
+ const MDefinition* def) const {
+ if (def->isStoreDynamicSlot()) {
+ const MStoreDynamicSlot* store = def->toStoreDynamicSlot();
+ if (store->slot() != slot()) {
+ return AliasType::NoAlias;
+ }
+
+ if (store->slots() != slots()) {
+ return AliasType::MayAlias;
+ }
+
+ return AliasType::MustAlias;
+ }
+ return AliasType::MayAlias;
+}
+
+HashNumber MLoadDynamicSlot::valueHash() const {
+ HashNumber hash = MDefinition::valueHash();
+ hash = addU32ToHash(hash, slot_);
+ return hash;
+}
+
+MDefinition* MLoadDynamicSlot::foldsTo(TempAllocator& alloc) {
+ if (MDefinition* def = foldsToStore(alloc)) {
+ return def;
+ }
+
+ return this;
+}
+
+#ifdef JS_JITSPEW
+void MLoadDynamicSlot::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" (slot %u)", slot());
+}
+
+void MLoadDynamicSlotAndUnbox::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" (slot %zu)", slot());
+}
+
+void MStoreDynamicSlot::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" (slot %u)", slot());
+}
+
+void MLoadFixedSlot::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" (slot %zu)", slot());
+}
+
+void MLoadFixedSlotAndUnbox::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" (slot %zu)", slot());
+}
+
+void MStoreFixedSlot::printOpcode(GenericPrinter& out) const {
+ MDefinition::printOpcode(out);
+ out.printf(" (slot %zu)", slot());
+}
+#endif
+
+MDefinition* MGuardFunctionScript::foldsTo(TempAllocator& alloc) {
+ MDefinition* in = input();
+ if (in->isLambda() &&
+ in->toLambda()->templateFunction()->baseScript() == expected()) {
+ return in;
+ }
+ return this;
+}
+
+MDefinition* MFunctionEnvironment::foldsTo(TempAllocator& alloc) {
+ if (input()->isLambda()) {
+ return input()->toLambda()->environmentChain();
+ }
+ if (input()->isFunctionWithProto()) {
+ return input()->toFunctionWithProto()->environmentChain();
+ }
+ return this;
+}
+
+static bool AddIsANonZeroAdditionOf(MAdd* add, MDefinition* ins) {
+ if (add->lhs() != ins && add->rhs() != ins) {
+ return false;
+ }
+ MDefinition* other = (add->lhs() == ins) ? add->rhs() : add->lhs();
+ if (!IsNumberType(other->type())) {
+ return false;
+ }
+ if (!other->isConstant()) {
+ return false;
+ }
+ if (other->toConstant()->numberToDouble() == 0) {
+ return false;
+ }
+ return true;
+}
+
+// Skip over instructions that usually appear between the actual index
+// value being used and the MLoadElement.
+// They don't modify the index value in a meaningful way.
+static MDefinition* SkipUninterestingInstructions(MDefinition* ins) {
+ // Drop the MToNumberInt32 added by the TypePolicy for double and float
+ // values.
+ if (ins->isToNumberInt32()) {
+ return SkipUninterestingInstructions(ins->toToNumberInt32()->input());
+ }
+
+ // Ignore the bounds check, which don't modify the index.
+ if (ins->isBoundsCheck()) {
+ return SkipUninterestingInstructions(ins->toBoundsCheck()->index());
+ }
+
+ // Masking the index for Spectre-mitigation is not observable.
+ if (ins->isSpectreMaskIndex()) {
+ return SkipUninterestingInstructions(ins->toSpectreMaskIndex()->index());
+ }
+
+ return ins;
+}
+
+static bool DefinitelyDifferentValue(MDefinition* ins1, MDefinition* ins2) {
+ ins1 = SkipUninterestingInstructions(ins1);
+ ins2 = SkipUninterestingInstructions(ins2);
+
+ if (ins1 == ins2) {
+ return false;
+ }
+
+ // For constants check they are not equal.
+ if (ins1->isConstant() && ins2->isConstant()) {
+ MConstant* cst1 = ins1->toConstant();
+ MConstant* cst2 = ins2->toConstant();
+
+ if (!cst1->isTypeRepresentableAsDouble() ||
+ !cst2->isTypeRepresentableAsDouble()) {
+ return false;
+ }
+
+ // Be conservative and only allow values that fit into int32.
+ int32_t n1, n2;
+ if (!mozilla::NumberIsInt32(cst1->numberToDouble(), &n1) ||
+ !mozilla::NumberIsInt32(cst2->numberToDouble(), &n2)) {
+ return false;
+ }
+
+ return n1 != n2;
+ }
+
+ // Check if "ins1 = ins2 + cte", which would make both instructions
+ // have different values.
+ if (ins1->isAdd()) {
+ if (AddIsANonZeroAdditionOf(ins1->toAdd(), ins2)) {
+ return true;
+ }
+ }
+ if (ins2->isAdd()) {
+ if (AddIsANonZeroAdditionOf(ins2->toAdd(), ins1)) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+MDefinition::AliasType MLoadElement::mightAlias(const MDefinition* def) const {
+ if (def->isStoreElement()) {
+ const MStoreElement* store = def->toStoreElement();
+ if (store->index() != index()) {
+ if (DefinitelyDifferentValue(store->index(), index())) {
+ return AliasType::NoAlias;
+ }
+ return AliasType::MayAlias;
+ }
+
+ if (store->elements() != elements()) {
+ return AliasType::MayAlias;
+ }
+
+ return AliasType::MustAlias;
+ }
+ return AliasType::MayAlias;
+}
+
+MDefinition* MLoadElement::foldsTo(TempAllocator& alloc) {
+ if (MDefinition* def = foldsToStore(alloc)) {
+ return def;
+ }
+
+ return this;
+}
+
+MDefinition* MWasmUnsignedToDouble::foldsTo(TempAllocator& alloc) {
+ if (input()->isConstant()) {
+ return MConstant::New(
+ alloc, DoubleValue(uint32_t(input()->toConstant()->toInt32())));
+ }
+
+ return this;
+}
+
+MDefinition* MWasmUnsignedToFloat32::foldsTo(TempAllocator& alloc) {
+ if (input()->isConstant()) {
+ double dval = double(uint32_t(input()->toConstant()->toInt32()));
+ if (IsFloat32Representable(dval)) {
+ return MConstant::NewFloat32(alloc, float(dval));
+ }
+ }
+
+ return this;
+}
+
+MWasmCallCatchable* MWasmCallCatchable::New(TempAllocator& alloc,
+ const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee,
+ const Args& args,
+ uint32_t stackArgAreaSizeUnaligned,
+ const MWasmCallTryDesc& tryDesc,
+ MDefinition* tableIndexOrRef) {
+ MOZ_ASSERT(tryDesc.inTry);
+
+ MWasmCallCatchable* call = new (alloc) MWasmCallCatchable(
+ desc, callee, stackArgAreaSizeUnaligned, tryDesc.tryNoteIndex);
+
+ call->setSuccessor(FallthroughBranchIndex, tryDesc.fallthroughBlock);
+ call->setSuccessor(PrePadBranchIndex, tryDesc.prePadBlock);
+
+ MOZ_ASSERT_IF(callee.isTable() || callee.isFuncRef(), tableIndexOrRef);
+ if (!call->initWithArgs(alloc, call, args, tableIndexOrRef)) {
+ return nullptr;
+ }
+
+ return call;
+}
+
+MWasmCallUncatchable* MWasmCallUncatchable::New(
+ TempAllocator& alloc, const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee, const Args& args,
+ uint32_t stackArgAreaSizeUnaligned, MDefinition* tableIndexOrRef) {
+ MWasmCallUncatchable* call =
+ new (alloc) MWasmCallUncatchable(desc, callee, stackArgAreaSizeUnaligned);
+
+ MOZ_ASSERT_IF(callee.isTable() || callee.isFuncRef(), tableIndexOrRef);
+ if (!call->initWithArgs(alloc, call, args, tableIndexOrRef)) {
+ return nullptr;
+ }
+
+ return call;
+}
+
+MWasmCallUncatchable* MWasmCallUncatchable::NewBuiltinInstanceMethodCall(
+ TempAllocator& alloc, const wasm::CallSiteDesc& desc,
+ const wasm::SymbolicAddress builtin, wasm::FailureMode failureMode,
+ const ABIArg& instanceArg, const Args& args,
+ uint32_t stackArgAreaSizeUnaligned) {
+ auto callee = wasm::CalleeDesc::builtinInstanceMethod(builtin);
+ MWasmCallUncatchable* call = MWasmCallUncatchable::New(
+ alloc, desc, callee, args, stackArgAreaSizeUnaligned, nullptr);
+ if (!call) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(instanceArg != ABIArg());
+ call->instanceArg_ = instanceArg;
+ call->builtinMethodFailureMode_ = failureMode;
+ return call;
+}
+
+void MSqrt::trySpecializeFloat32(TempAllocator& alloc) {
+ if (EnsureFloatConsumersAndInputOrConvert(this, alloc)) {
+ setResultType(MIRType::Float32);
+ specialization_ = MIRType::Float32;
+ }
+}
+
+MDefinition* MClz::foldsTo(TempAllocator& alloc) {
+ if (num()->isConstant()) {
+ MConstant* c = num()->toConstant();
+ if (type() == MIRType::Int32) {
+ int32_t n = c->toInt32();
+ if (n == 0) {
+ return MConstant::New(alloc, Int32Value(32));
+ }
+ return MConstant::New(alloc,
+ Int32Value(mozilla::CountLeadingZeroes32(n)));
+ }
+ int64_t n = c->toInt64();
+ if (n == 0) {
+ return MConstant::NewInt64(alloc, int64_t(64));
+ }
+ return MConstant::NewInt64(alloc,
+ int64_t(mozilla::CountLeadingZeroes64(n)));
+ }
+
+ return this;
+}
+
+MDefinition* MCtz::foldsTo(TempAllocator& alloc) {
+ if (num()->isConstant()) {
+ MConstant* c = num()->toConstant();
+ if (type() == MIRType::Int32) {
+ int32_t n = num()->toConstant()->toInt32();
+ if (n == 0) {
+ return MConstant::New(alloc, Int32Value(32));
+ }
+ return MConstant::New(alloc,
+ Int32Value(mozilla::CountTrailingZeroes32(n)));
+ }
+ int64_t n = c->toInt64();
+ if (n == 0) {
+ return MConstant::NewInt64(alloc, int64_t(64));
+ }
+ return MConstant::NewInt64(alloc,
+ int64_t(mozilla::CountTrailingZeroes64(n)));
+ }
+
+ return this;
+}
+
+MDefinition* MPopcnt::foldsTo(TempAllocator& alloc) {
+ if (num()->isConstant()) {
+ MConstant* c = num()->toConstant();
+ if (type() == MIRType::Int32) {
+ int32_t n = num()->toConstant()->toInt32();
+ return MConstant::New(alloc, Int32Value(mozilla::CountPopulation32(n)));
+ }
+ int64_t n = c->toInt64();
+ return MConstant::NewInt64(alloc, int64_t(mozilla::CountPopulation64(n)));
+ }
+
+ return this;
+}
+
+MDefinition* MBoundsCheck::foldsTo(TempAllocator& alloc) {
+ if (type() == MIRType::Int32 && index()->isConstant() &&
+ length()->isConstant()) {
+ uint32_t len = length()->toConstant()->toInt32();
+ uint32_t idx = index()->toConstant()->toInt32();
+ if (idx + uint32_t(minimum()) < len && idx + uint32_t(maximum()) < len) {
+ return index();
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MTableSwitch::foldsTo(TempAllocator& alloc) {
+ MDefinition* op = getOperand(0);
+
+ // If we only have one successor, convert to a plain goto to the only
+ // successor. TableSwitch indices are numeric; other types will always go to
+ // the only successor.
+ if (numSuccessors() == 1 ||
+ (op->type() != MIRType::Value && !IsNumberType(op->type()))) {
+ return MGoto::New(alloc, getDefault());
+ }
+
+ if (MConstant* opConst = op->maybeConstantValue()) {
+ if (op->type() == MIRType::Int32) {
+ int32_t i = opConst->toInt32() - low_;
+ MBasicBlock* target;
+ if (size_t(i) < numCases()) {
+ target = getCase(size_t(i));
+ } else {
+ target = getDefault();
+ }
+ MOZ_ASSERT(target);
+ return MGoto::New(alloc, target);
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MArrayJoin::foldsTo(TempAllocator& alloc) {
+ MDefinition* arr = array();
+
+ if (!arr->isStringSplit()) {
+ return this;
+ }
+
+ setRecoveredOnBailout();
+ if (arr->hasLiveDefUses()) {
+ setNotRecoveredOnBailout();
+ return this;
+ }
+
+ // The MStringSplit won't generate any code.
+ arr->setRecoveredOnBailout();
+
+ // We're replacing foo.split(bar).join(baz) by
+ // foo.replace(bar, baz). MStringSplit could be recovered by
+ // a bailout. As we are removing its last use, and its result
+ // could be captured by a resume point, this MStringSplit will
+ // be executed on the bailout path.
+ MDefinition* string = arr->toStringSplit()->string();
+ MDefinition* pattern = arr->toStringSplit()->separator();
+ MDefinition* replacement = sep();
+
+ MStringReplace* substr =
+ MStringReplace::New(alloc, string, pattern, replacement);
+ substr->setFlatReplacement();
+ return substr;
+}
+
+MDefinition* MGetFirstDollarIndex::foldsTo(TempAllocator& alloc) {
+ MDefinition* strArg = str();
+ if (!strArg->isConstant()) {
+ return this;
+ }
+
+ JSLinearString* str = &strArg->toConstant()->toString()->asLinear();
+ int32_t index = GetFirstDollarIndexRawFlat(str);
+ return MConstant::New(alloc, Int32Value(index));
+}
+
+AliasSet MThrowRuntimeLexicalError::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ExceptionState);
+}
+
+AliasSet MSlots::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+MDefinition::AliasType MSlots::mightAlias(const MDefinition* store) const {
+ // ArrayPush only modifies object elements, but not object slots.
+ if (store->isArrayPush()) {
+ return AliasType::NoAlias;
+ }
+ return MInstruction::mightAlias(store);
+}
+
+AliasSet MElements::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MInitializedLength::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MSetInitializedLength::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ObjectFields);
+}
+
+AliasSet MArrayLength::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MSetArrayLength::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ObjectFields);
+}
+
+AliasSet MFunctionLength::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+AliasSet MFunctionName::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+AliasSet MArrayBufferByteLength::getAliasSet() const {
+ return AliasSet::Load(AliasSet::FixedSlot);
+}
+
+AliasSet MArrayBufferViewLength::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ArrayBufferViewLengthOrOffset);
+}
+
+AliasSet MArrayBufferViewByteOffset::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ArrayBufferViewLengthOrOffset);
+}
+
+AliasSet MArrayBufferViewElements::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MGuardHasAttachedArrayBuffer::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot);
+}
+
+AliasSet MArrayPush::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ObjectFields | AliasSet::Element);
+}
+
+MDefinition* MGuardNumberToIntPtrIndex::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = this->input();
+
+ if (input->isToDouble() && input->getOperand(0)->type() == MIRType::Int32) {
+ return MInt32ToIntPtr::New(alloc, input->getOperand(0));
+ }
+
+ if (!input->isConstant()) {
+ return this;
+ }
+
+ // Fold constant double representable as intptr to intptr.
+ int64_t ival;
+ if (!mozilla::NumberEqualsInt64(input->toConstant()->toDouble(), &ival)) {
+ // If not representable as an int64, this access is equal to an OOB access.
+ // So replace it with a known int64/intptr value which also produces an OOB
+ // access. If we don't support OOB accesses we have to bail out.
+ if (!supportOOB()) {
+ return this;
+ }
+ ival = -1;
+ }
+
+ if (ival < INTPTR_MIN || ival > INTPTR_MAX) {
+ return this;
+ }
+
+ return MConstant::NewIntPtr(alloc, intptr_t(ival));
+}
+
+MDefinition* MIsObject::foldsTo(TempAllocator& alloc) {
+ if (!object()->isBox()) {
+ return this;
+ }
+
+ MDefinition* unboxed = object()->getOperand(0);
+ if (unboxed->type() == MIRType::Object) {
+ return MConstant::New(alloc, BooleanValue(true));
+ }
+
+ return this;
+}
+
+MDefinition* MIsNullOrUndefined::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = value();
+ if (input->isBox()) {
+ input = input->toBox()->input();
+ }
+
+ if (input->definitelyType({MIRType::Null, MIRType::Undefined})) {
+ return MConstant::New(alloc, BooleanValue(true));
+ }
+
+ if (!input->mightBeType(MIRType::Null) &&
+ !input->mightBeType(MIRType::Undefined)) {
+ return MConstant::New(alloc, BooleanValue(false));
+ }
+
+ return this;
+}
+
+AliasSet MHomeObjectSuperBase::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+MDefinition* MGuardValue::foldsTo(TempAllocator& alloc) {
+ if (MConstant* cst = value()->maybeConstantValue()) {
+ if (cst->toJSValue() == expected()) {
+ return value();
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MGuardNullOrUndefined::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = value();
+ if (input->isBox()) {
+ input = input->toBox()->input();
+ }
+
+ if (input->definitelyType({MIRType::Null, MIRType::Undefined})) {
+ return value();
+ }
+
+ return this;
+}
+
+MDefinition* MGuardIsNotObject::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = value();
+ if (input->isBox()) {
+ input = input->toBox()->input();
+ }
+
+ if (!input->mightBeType(MIRType::Object)) {
+ return value();
+ }
+
+ return this;
+}
+
+MDefinition* MGuardObjectIdentity::foldsTo(TempAllocator& alloc) {
+ if (object()->isConstant() && expected()->isConstant()) {
+ JSObject* obj = &object()->toConstant()->toObject();
+ JSObject* other = &expected()->toConstant()->toObject();
+ if (!bailOnEquality()) {
+ if (obj == other) {
+ return object();
+ }
+ } else {
+ if (obj != other) {
+ return object();
+ }
+ }
+ }
+
+ if (!bailOnEquality() && object()->isNurseryObject() &&
+ expected()->isNurseryObject()) {
+ uint32_t objIndex = object()->toNurseryObject()->nurseryIndex();
+ uint32_t otherIndex = expected()->toNurseryObject()->nurseryIndex();
+ if (objIndex == otherIndex) {
+ return object();
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MGuardSpecificFunction::foldsTo(TempAllocator& alloc) {
+ if (function()->isConstant() && expected()->isConstant()) {
+ JSObject* fun = &function()->toConstant()->toObject();
+ JSObject* other = &expected()->toConstant()->toObject();
+ if (fun == other) {
+ return function();
+ }
+ }
+
+ if (function()->isNurseryObject() && expected()->isNurseryObject()) {
+ uint32_t funIndex = function()->toNurseryObject()->nurseryIndex();
+ uint32_t otherIndex = expected()->toNurseryObject()->nurseryIndex();
+ if (funIndex == otherIndex) {
+ return function();
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MGuardSpecificAtom::foldsTo(TempAllocator& alloc) {
+ if (str()->isConstant()) {
+ JSString* s = str()->toConstant()->toString();
+ if (s->isAtom()) {
+ JSAtom* cstAtom = &s->asAtom();
+ if (cstAtom == atom()) {
+ return str();
+ }
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MGuardSpecificSymbol::foldsTo(TempAllocator& alloc) {
+ if (symbol()->isConstant()) {
+ if (symbol()->toConstant()->toSymbol() == expected()) {
+ return symbol();
+ }
+ }
+
+ return this;
+}
+
+MDefinition* MGuardSpecificInt32::foldsTo(TempAllocator& alloc) {
+ if (num()->isConstant() && num()->toConstant()->isInt32(expected())) {
+ return num();
+ }
+ return this;
+}
+
+bool MCallBindVar::congruentTo(const MDefinition* ins) const {
+ if (!ins->isCallBindVar()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+bool MGuardShape::congruentTo(const MDefinition* ins) const {
+ if (!ins->isGuardShape()) {
+ return false;
+ }
+ if (shape() != ins->toGuardShape()->shape()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MGuardShape::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+MDefinition::AliasType MGuardShape::mightAlias(const MDefinition* store) const {
+ // These instructions only modify object elements, but not the shape.
+ if (store->isStoreElementHole() || store->isArrayPush()) {
+ return AliasType::NoAlias;
+ }
+ if (object()->isConstantProto()) {
+ const MDefinition* receiverObject =
+ object()->toConstantProto()->getReceiverObject();
+ switch (store->op()) {
+ case MDefinition::Opcode::StoreFixedSlot:
+ if (store->toStoreFixedSlot()->object()->skipObjectGuards() ==
+ receiverObject) {
+ return AliasType::NoAlias;
+ }
+ break;
+ case MDefinition::Opcode::StoreDynamicSlot:
+ if (store->toStoreDynamicSlot()
+ ->slots()
+ ->toSlots()
+ ->object()
+ ->skipObjectGuards() == receiverObject) {
+ return AliasType::NoAlias;
+ }
+ break;
+ case MDefinition::Opcode::AddAndStoreSlot:
+ if (store->toAddAndStoreSlot()->object()->skipObjectGuards() ==
+ receiverObject) {
+ return AliasType::NoAlias;
+ }
+ break;
+ case MDefinition::Opcode::AllocateAndStoreSlot:
+ if (store->toAllocateAndStoreSlot()->object()->skipObjectGuards() ==
+ receiverObject) {
+ return AliasType::NoAlias;
+ }
+ break;
+ case MDefinition::Opcode::MegamorphicStoreSlot:
+ if (store->toMegamorphicStoreSlot()->object()->skipObjectGuards() ==
+ receiverObject) {
+ return AliasType::NoAlias;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ return MInstruction::mightAlias(store);
+}
+
+AliasSet MGuardMultipleShapes::getAliasSet() const {
+ // Note: This instruction loads the elements of the ListObject used to
+ // store the list of shapes, but that object is internal and not exposed
+ // to script, so it doesn't have to be in the alias set.
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+MDefinition* MGuardIsNotProxy::foldsTo(TempAllocator& alloc) {
+ KnownClass known = GetObjectKnownClass(object());
+ if (known == KnownClass::None) {
+ return this;
+ }
+
+ MOZ_ASSERT(!GetObjectKnownJSClass(object())->isProxyObject());
+ AssertKnownClass(alloc, this, object());
+ return object();
+}
+
+AliasSet MMegamorphicLoadSlotByValue::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+MDefinition* MMegamorphicLoadSlotByValue::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = idVal();
+ if (input->isBox()) {
+ input = input->toBox()->input();
+ }
+
+ MDefinition* result = this;
+
+ if (input->isConstant()) {
+ MConstant* constant = input->toConstant();
+ if (constant->type() == MIRType::Symbol) {
+ PropertyKey id = PropertyKey::Symbol(constant->toSymbol());
+ result = MMegamorphicLoadSlot::New(alloc, object(), id);
+ }
+
+ if (constant->type() == MIRType::String) {
+ JSString* str = constant->toString();
+ if (str->isAtom() && !str->asAtom().isIndex()) {
+ PropertyKey id = PropertyKey::NonIntAtom(str);
+ result = MMegamorphicLoadSlot::New(alloc, object(), id);
+ }
+ }
+ }
+
+ if (result != this) {
+ result->setDependency(dependency());
+ }
+
+ return result;
+}
+
+bool MMegamorphicLoadSlot::congruentTo(const MDefinition* ins) const {
+ if (!ins->isMegamorphicLoadSlot()) {
+ return false;
+ }
+ if (ins->toMegamorphicLoadSlot()->name() != name()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MMegamorphicLoadSlot::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+bool MMegamorphicStoreSlot::congruentTo(const MDefinition* ins) const {
+ if (!ins->isMegamorphicStoreSlot()) {
+ return false;
+ }
+ if (ins->toMegamorphicStoreSlot()->name() != name()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MMegamorphicStoreSlot::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+bool MMegamorphicHasProp::congruentTo(const MDefinition* ins) const {
+ if (!ins->isMegamorphicHasProp()) {
+ return false;
+ }
+ if (ins->toMegamorphicHasProp()->hasOwn() != hasOwn()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MMegamorphicHasProp::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+bool MNurseryObject::congruentTo(const MDefinition* ins) const {
+ if (!ins->isNurseryObject()) {
+ return false;
+ }
+ return nurseryIndex() == ins->toNurseryObject()->nurseryIndex();
+}
+
+AliasSet MGuardFunctionIsNonBuiltinCtor::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+bool MGuardFunctionKind::congruentTo(const MDefinition* ins) const {
+ if (!ins->isGuardFunctionKind()) {
+ return false;
+ }
+ if (expected() != ins->toGuardFunctionKind()->expected()) {
+ return false;
+ }
+ if (bailOnEquality() != ins->toGuardFunctionKind()->bailOnEquality()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MGuardFunctionKind::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+bool MGuardFunctionScript::congruentTo(const MDefinition* ins) const {
+ if (!ins->isGuardFunctionScript()) {
+ return false;
+ }
+ if (expected() != ins->toGuardFunctionScript()->expected()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MGuardFunctionScript::getAliasSet() const {
+ // A JSFunction's BaseScript pointer is immutable. Relazification of
+ // self-hosted functions is an exception to this, but we don't use this
+ // guard for self-hosted functions.
+ MOZ_ASSERT(!flags_.isSelfHostedOrIntrinsic());
+ return AliasSet::None();
+}
+
+bool MGuardSpecificAtom::congruentTo(const MDefinition* ins) const {
+ if (!ins->isGuardSpecificAtom()) {
+ return false;
+ }
+ if (atom() != ins->toGuardSpecificAtom()->atom()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+MDefinition* MGuardStringToIndex::foldsTo(TempAllocator& alloc) {
+ if (!string()->isConstant()) {
+ return this;
+ }
+
+ JSString* str = string()->toConstant()->toString();
+
+ int32_t index = GetIndexFromString(str);
+ if (index < 0) {
+ return this;
+ }
+
+ return MConstant::New(alloc, Int32Value(index));
+}
+
+MDefinition* MGuardStringToInt32::foldsTo(TempAllocator& alloc) {
+ if (!string()->isConstant()) {
+ return this;
+ }
+
+ JSLinearString* str = &string()->toConstant()->toString()->asLinear();
+ double number = LinearStringToNumber(str);
+
+ int32_t n;
+ if (!mozilla::NumberIsInt32(number, &n)) {
+ return this;
+ }
+
+ return MConstant::New(alloc, Int32Value(n));
+}
+
+MDefinition* MGuardStringToDouble::foldsTo(TempAllocator& alloc) {
+ if (!string()->isConstant()) {
+ return this;
+ }
+
+ JSLinearString* str = &string()->toConstant()->toString()->asLinear();
+ double number = LinearStringToNumber(str);
+ return MConstant::New(alloc, DoubleValue(number));
+}
+
+AliasSet MGuardNoDenseElements::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MAllocateAndStoreSlot::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ObjectFields | AliasSet::DynamicSlot);
+}
+
+AliasSet MLoadDOMExpandoValue::getAliasSet() const {
+ return AliasSet::Load(AliasSet::DOMProxyExpando);
+}
+
+AliasSet MLoadDOMExpandoValueIgnoreGeneration::getAliasSet() const {
+ return AliasSet::Load(AliasSet::DOMProxyExpando);
+}
+
+bool MGuardDOMExpandoMissingOrGuardShape::congruentTo(
+ const MDefinition* ins) const {
+ if (!ins->isGuardDOMExpandoMissingOrGuardShape()) {
+ return false;
+ }
+ if (shape() != ins->toGuardDOMExpandoMissingOrGuardShape()->shape()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MGuardDOMExpandoMissingOrGuardShape::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+MDefinition* MGuardToClass::foldsTo(TempAllocator& alloc) {
+ const JSClass* clasp = GetObjectKnownJSClass(object());
+ if (!clasp || getClass() != clasp) {
+ return this;
+ }
+
+ AssertKnownClass(alloc, this, object());
+ return object();
+}
+
+MDefinition* MGuardToFunction::foldsTo(TempAllocator& alloc) {
+ if (GetObjectKnownClass(object()) != KnownClass::Function) {
+ return this;
+ }
+
+ AssertKnownClass(alloc, this, object());
+ return object();
+}
+
+MDefinition* MHasClass::foldsTo(TempAllocator& alloc) {
+ const JSClass* clasp = GetObjectKnownJSClass(object());
+ if (!clasp) {
+ return this;
+ }
+
+ AssertKnownClass(alloc, this, object());
+ return MConstant::New(alloc, BooleanValue(getClass() == clasp));
+}
+
+MDefinition* MIsCallable::foldsTo(TempAllocator& alloc) {
+ if (input()->type() != MIRType::Object) {
+ return this;
+ }
+
+ KnownClass known = GetObjectKnownClass(input());
+ if (known == KnownClass::None) {
+ return this;
+ }
+
+ AssertKnownClass(alloc, this, input());
+ return MConstant::New(alloc, BooleanValue(known == KnownClass::Function));
+}
+
+MDefinition* MIsArray::foldsTo(TempAllocator& alloc) {
+ if (input()->type() != MIRType::Object) {
+ return this;
+ }
+
+ KnownClass known = GetObjectKnownClass(input());
+ if (known == KnownClass::None) {
+ return this;
+ }
+
+ AssertKnownClass(alloc, this, input());
+ return MConstant::New(alloc, BooleanValue(known == KnownClass::Array));
+}
+
+AliasSet MObjectClassToString::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot);
+}
+
+MDefinition* MGuardIsNotArrayBufferMaybeShared::foldsTo(TempAllocator& alloc) {
+ switch (GetObjectKnownClass(object())) {
+ case KnownClass::PlainObject:
+ case KnownClass::Array:
+ case KnownClass::Function:
+ case KnownClass::RegExp:
+ case KnownClass::ArrayIterator:
+ case KnownClass::StringIterator:
+ case KnownClass::RegExpStringIterator: {
+ AssertKnownClass(alloc, this, object());
+ return object();
+ }
+ case KnownClass::None:
+ break;
+ }
+
+ return this;
+}
+
+MDefinition* MCheckIsObj::foldsTo(TempAllocator& alloc) {
+ if (!input()->isBox()) {
+ return this;
+ }
+
+ MDefinition* unboxed = input()->getOperand(0);
+ if (unboxed->type() == MIRType::Object) {
+ return unboxed;
+ }
+
+ return this;
+}
+
+static bool IsBoxedObject(MDefinition* def) {
+ MOZ_ASSERT(def->type() == MIRType::Value);
+
+ if (def->isBox()) {
+ return def->toBox()->input()->type() == MIRType::Object;
+ }
+
+ // Construct calls are always returning a boxed object.
+ //
+ // TODO: We should consider encoding this directly in the graph instead of
+ // having to special case it here.
+ if (def->isCall()) {
+ return def->toCall()->isConstructing();
+ }
+ if (def->isConstructArray()) {
+ return true;
+ }
+ if (def->isConstructArgs()) {
+ return true;
+ }
+
+ return false;
+}
+
+MDefinition* MCheckReturn::foldsTo(TempAllocator& alloc) {
+ auto* returnVal = returnValue();
+ if (!returnVal->isBox()) {
+ return this;
+ }
+
+ auto* unboxedReturnVal = returnVal->toBox()->input();
+ if (unboxedReturnVal->type() == MIRType::Object) {
+ return returnVal;
+ }
+
+ if (unboxedReturnVal->type() != MIRType::Undefined) {
+ return this;
+ }
+
+ auto* thisVal = thisValue();
+ if (IsBoxedObject(thisVal)) {
+ return thisVal;
+ }
+
+ return this;
+}
+
+MDefinition* MCheckThis::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = thisValue();
+ if (!input->isBox()) {
+ return this;
+ }
+
+ MDefinition* unboxed = input->getOperand(0);
+ if (unboxed->mightBeMagicType()) {
+ return this;
+ }
+
+ return input;
+}
+
+MDefinition* MCheckThisReinit::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = thisValue();
+ if (!input->isBox()) {
+ return this;
+ }
+
+ MDefinition* unboxed = input->getOperand(0);
+ if (unboxed->type() != MIRType::MagicUninitializedLexical) {
+ return this;
+ }
+
+ return input;
+}
+
+MDefinition* MCheckObjCoercible::foldsTo(TempAllocator& alloc) {
+ MDefinition* input = checkValue();
+ if (!input->isBox()) {
+ return this;
+ }
+
+ MDefinition* unboxed = input->getOperand(0);
+ if (unboxed->mightBeType(MIRType::Null) ||
+ unboxed->mightBeType(MIRType::Undefined)) {
+ return this;
+ }
+
+ return input;
+}
+
+AliasSet MCheckObjCoercible::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ExceptionState);
+}
+
+AliasSet MCheckReturn::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ExceptionState);
+}
+
+AliasSet MCheckThis::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ExceptionState);
+}
+
+AliasSet MCheckThisReinit::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ExceptionState);
+}
+
+AliasSet MIsPackedArray::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MGuardArrayIsPacked::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MSuperFunction::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MInitHomeObject::getAliasSet() const {
+ return AliasSet::Store(AliasSet::ObjectFields);
+}
+
+AliasSet MLoadWrapperTarget::getAliasSet() const {
+ return AliasSet::Load(AliasSet::Any);
+}
+
+AliasSet MGuardHasGetterSetter::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+bool MGuardHasGetterSetter::congruentTo(const MDefinition* ins) const {
+ if (!ins->isGuardHasGetterSetter()) {
+ return false;
+ }
+ if (ins->toGuardHasGetterSetter()->propId() != propId()) {
+ return false;
+ }
+ if (ins->toGuardHasGetterSetter()->getterSetter() != getterSetter()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+}
+
+AliasSet MGuardIsExtensible::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MGuardIndexIsNotDenseElement::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::Element);
+}
+
+AliasSet MGuardIndexIsValidUpdateOrAdd::getAliasSet() const {
+ return AliasSet::Load(AliasSet::ObjectFields);
+}
+
+AliasSet MCallObjectHasSparseElement::getAliasSet() const {
+ return AliasSet::Load(AliasSet::Element | AliasSet::ObjectFields |
+ AliasSet::FixedSlot | AliasSet::DynamicSlot);
+}
+
+MDefinition* MGuardInt32IsNonNegative::foldsTo(TempAllocator& alloc) {
+ MOZ_ASSERT(index()->type() == MIRType::Int32);
+
+ MDefinition* input = index();
+ if (!input->isConstant() || input->toConstant()->toInt32() < 0) {
+ return this;
+ }
+ return input;
+}
+
+MDefinition* MGuardInt32Range::foldsTo(TempAllocator& alloc) {
+ MOZ_ASSERT(input()->type() == MIRType::Int32);
+ MOZ_ASSERT(minimum() <= maximum());
+
+ MDefinition* in = input();
+ if (!in->isConstant()) {
+ return this;
+ }
+ int32_t cst = in->toConstant()->toInt32();
+ if (cst < minimum() || cst > maximum()) {
+ return this;
+ }
+ return in;
+}
+
+MDefinition* MGuardNonGCThing::foldsTo(TempAllocator& alloc) {
+ if (!input()->isBox()) {
+ return this;
+ }
+
+ MDefinition* unboxed = input()->getOperand(0);
+ if (!IsNonGCThing(unboxed->type())) {
+ return this;
+ }
+ return input();
+}
+
+AliasSet MSetObjectHasNonBigInt::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MSetObjectHasBigInt::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MSetObjectHasValue::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MSetObjectHasValueVMCall::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MSetObjectSize::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectHasNonBigInt::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectHasBigInt::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectHasValue::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectHasValueVMCall::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectGetNonBigInt::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectGetBigInt::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectGetValue::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectGetValueVMCall::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+AliasSet MMapObjectSize::getAliasSet() const {
+ return AliasSet::Load(AliasSet::MapOrSetHashTable);
+}
+
+MIonToWasmCall* MIonToWasmCall::New(TempAllocator& alloc,
+ WasmInstanceObject* instanceObj,
+ const wasm::FuncExport& funcExport) {
+ const wasm::FuncType& funcType =
+ instanceObj->instance().metadata().getFuncExportType(funcExport);
+ const wasm::ValTypeVector& results = funcType.results();
+ MIRType resultType = MIRType::Value;
+ // At the JS boundary some wasm types must be represented as a Value, and in
+ // addition a void return requires an Undefined value.
+ if (results.length() > 0 && !results[0].isEncodedAsJSValueOnEscape()) {
+ MOZ_ASSERT(results.length() == 1,
+ "multiple returns not implemented for inlined Wasm calls");
+ resultType = results[0].toMIRType();
+ }
+
+ auto* ins = new (alloc) MIonToWasmCall(instanceObj, resultType, funcExport);
+ if (!ins->init(alloc, funcType.args().length())) {
+ return nullptr;
+ }
+ return ins;
+}
+
+#ifdef DEBUG
+bool MIonToWasmCall::isConsistentFloat32Use(MUse* use) const {
+ const wasm::FuncType& funcType =
+ instance()->metadata().getFuncExportType(funcExport_);
+ return funcType.args()[use->index()].kind() == wasm::ValType::F32;
+}
+#endif
+
+MCreateInlinedArgumentsObject* MCreateInlinedArgumentsObject::New(
+ TempAllocator& alloc, MDefinition* callObj, MDefinition* callee,
+ MDefinitionVector& args, ArgumentsObject* templateObj) {
+ MCreateInlinedArgumentsObject* ins =
+ new (alloc) MCreateInlinedArgumentsObject(templateObj);
+
+ uint32_t argc = args.length();
+ MOZ_ASSERT(argc <= ArgumentsObject::MaxInlinedArgs);
+
+ if (!ins->init(alloc, argc + NumNonArgumentOperands)) {
+ return nullptr;
+ }
+
+ ins->initOperand(0, callObj);
+ ins->initOperand(1, callee);
+ for (uint32_t i = 0; i < argc; i++) {
+ ins->initOperand(i + NumNonArgumentOperands, args[i]);
+ }
+
+ return ins;
+}
+
+MGetInlinedArgument* MGetInlinedArgument::New(
+ TempAllocator& alloc, MDefinition* index,
+ MCreateInlinedArgumentsObject* args) {
+ MGetInlinedArgument* ins = new (alloc) MGetInlinedArgument();
+
+ uint32_t argc = args->numActuals();
+ MOZ_ASSERT(argc <= ArgumentsObject::MaxInlinedArgs);
+
+ if (!ins->init(alloc, argc + NumNonArgumentOperands)) {
+ return nullptr;
+ }
+
+ ins->initOperand(0, index);
+ for (uint32_t i = 0; i < argc; i++) {
+ ins->initOperand(i + NumNonArgumentOperands, args->getArg(i));
+ }
+
+ return ins;
+}
+
+MDefinition* MGetInlinedArgument::foldsTo(TempAllocator& alloc) {
+ MDefinition* indexDef = SkipUninterestingInstructions(index());
+ if (!indexDef->isConstant() || indexDef->type() != MIRType::Int32) {
+ return this;
+ }
+
+ int32_t indexConst = indexDef->toConstant()->toInt32();
+ if (indexConst < 0 || uint32_t(indexConst) >= numActuals()) {
+ return this;
+ }
+
+ MDefinition* arg = getArg(indexConst);
+ if (arg->type() != MIRType::Value) {
+ arg = MBox::New(alloc, arg);
+ }
+
+ return arg;
+}
+
+MGetInlinedArgumentHole* MGetInlinedArgumentHole::New(
+ TempAllocator& alloc, MDefinition* index,
+ MCreateInlinedArgumentsObject* args) {
+ auto* ins = new (alloc) MGetInlinedArgumentHole();
+
+ uint32_t argc = args->numActuals();
+ MOZ_ASSERT(argc <= ArgumentsObject::MaxInlinedArgs);
+
+ if (!ins->init(alloc, argc + NumNonArgumentOperands)) {
+ return nullptr;
+ }
+
+ ins->initOperand(0, index);
+ for (uint32_t i = 0; i < argc; i++) {
+ ins->initOperand(i + NumNonArgumentOperands, args->getArg(i));
+ }
+
+ return ins;
+}
+
+MDefinition* MGetInlinedArgumentHole::foldsTo(TempAllocator& alloc) {
+ MDefinition* indexDef = SkipUninterestingInstructions(index());
+ if (!indexDef->isConstant() || indexDef->type() != MIRType::Int32) {
+ return this;
+ }
+
+ int32_t indexConst = indexDef->toConstant()->toInt32();
+ if (indexConst < 0) {
+ return this;
+ }
+
+ MDefinition* arg;
+ if (uint32_t(indexConst) < numActuals()) {
+ arg = getArg(indexConst);
+
+ if (arg->type() != MIRType::Value) {
+ arg = MBox::New(alloc, arg);
+ }
+ } else {
+ auto* undefined = MConstant::New(alloc, UndefinedValue());
+ block()->insertBefore(this, undefined);
+
+ arg = MBox::New(alloc, undefined);
+ }
+
+ return arg;
+}
+
+MInlineArgumentsSlice* MInlineArgumentsSlice::New(
+ TempAllocator& alloc, MDefinition* begin, MDefinition* count,
+ MCreateInlinedArgumentsObject* args, JSObject* templateObj,
+ gc::InitialHeap initialHeap) {
+ auto* ins = new (alloc) MInlineArgumentsSlice(templateObj, initialHeap);
+
+ uint32_t argc = args->numActuals();
+ MOZ_ASSERT(argc <= ArgumentsObject::MaxInlinedArgs);
+
+ if (!ins->init(alloc, argc + NumNonArgumentOperands)) {
+ return nullptr;
+ }
+
+ ins->initOperand(0, begin);
+ ins->initOperand(1, count);
+ for (uint32_t i = 0; i < argc; i++) {
+ ins->initOperand(i + NumNonArgumentOperands, args->getArg(i));
+ }
+
+ return ins;
+}
+
+MDefinition* MNormalizeSliceTerm::foldsTo(TempAllocator& alloc) {
+ auto* length = this->length();
+ if (!length->isConstant() && !length->isArgumentsLength()) {
+ return this;
+ }
+
+ if (length->isConstant()) {
+ int32_t lengthConst = length->toConstant()->toInt32();
+ MOZ_ASSERT(lengthConst >= 0);
+
+ // Result is always zero when |length| is zero.
+ if (lengthConst == 0) {
+ return length;
+ }
+
+ auto* value = this->value();
+ if (value->isConstant()) {
+ int32_t valueConst = value->toConstant()->toInt32();
+
+ int32_t normalized;
+ if (valueConst < 0) {
+ normalized = std::max(valueConst + lengthConst, 0);
+ } else {
+ normalized = std::min(valueConst, lengthConst);
+ }
+
+ if (normalized == valueConst) {
+ return value;
+ }
+ if (normalized == lengthConst) {
+ return length;
+ }
+ return MConstant::New(alloc, Int32Value(normalized));
+ }
+
+ return this;
+ }
+
+ auto* value = this->value();
+ if (value->isConstant()) {
+ int32_t valueConst = value->toConstant()->toInt32();
+
+ // Minimum of |value| and |length|.
+ if (valueConst > 0) {
+ bool isMax = false;
+ return MMinMax::New(alloc, value, length, MIRType::Int32, isMax);
+ }
+
+ // Maximum of |value + length| and zero.
+ if (valueConst < 0) {
+ // Safe to truncate because |length| is never negative.
+ auto* add = MAdd::New(alloc, value, length, TruncateKind::Truncate);
+ block()->insertBefore(this, add);
+
+ auto* zero = MConstant::New(alloc, Int32Value(0));
+ block()->insertBefore(this, zero);
+
+ bool isMax = true;
+ return MMinMax::New(alloc, add, zero, MIRType::Int32, isMax);
+ }
+
+ // Directly return the value when it's zero.
+ return value;
+ }
+
+ // Normalizing MArgumentsLength is a no-op.
+ if (value->isArgumentsLength()) {
+ return value;
+ }
+
+ return this;
+}
+
+bool MWasmShiftSimd128::congruentTo(const MDefinition* ins) const {
+ return ins->toWasmShiftSimd128()->simdOp() == simdOp_ &&
+ congruentIfOperandsEqual(ins);
+}
+
+bool MWasmShuffleSimd128::congruentTo(const MDefinition* ins) const {
+ return ins->toWasmShuffleSimd128()->shuffle().equals(&shuffle_) &&
+ congruentIfOperandsEqual(ins);
+}
+
+bool MWasmUnarySimd128::congruentTo(const MDefinition* ins) const {
+ return ins->toWasmUnarySimd128()->simdOp() == simdOp_ &&
+ congruentIfOperandsEqual(ins);
+}
+
+#ifdef ENABLE_WASM_SIMD
+MWasmShuffleSimd128* jit::BuildWasmShuffleSimd128(TempAllocator& alloc,
+ const int8_t* control,
+ MDefinition* lhs,
+ MDefinition* rhs) {
+ SimdShuffle s =
+ AnalyzeSimdShuffle(SimdConstant::CreateX16(control), lhs, rhs);
+ switch (s.opd) {
+ case SimdShuffle::Operand::LEFT:
+ // When SimdShuffle::Operand is LEFT the right operand is not used,
+ // lose reference to rhs.
+ rhs = lhs;
+ break;
+ case SimdShuffle::Operand::RIGHT:
+ // When SimdShuffle::Operand is RIGHT the left operand is not used,
+ // lose reference to lhs.
+ lhs = rhs;
+ break;
+ default:
+ break;
+ }
+ return MWasmShuffleSimd128::New(alloc, lhs, rhs, s);
+}
+#endif // ENABLE_WASM_SIMD
diff --git a/js/src/jit/MIR.h b/js/src/jit/MIR.h
new file mode 100644
index 0000000000..0633a8598d
--- /dev/null
+++ b/js/src/jit/MIR.h
@@ -0,0 +1,11344 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * Everything needed to build actual MIR instructions: the actual opcodes and
+ * instructions, the instruction interface, and use chains.
+ */
+
+#ifndef jit_MIR_h
+#define jit_MIR_h
+
+#include "mozilla/Array.h"
+#ifdef JS_JITSPEW
+# include "mozilla/Attributes.h" // MOZ_STACK_CLASS
+#endif
+#include "mozilla/MacroForEach.h"
+#ifdef JS_JITSPEW
+# include "mozilla/Sprintf.h"
+# include "mozilla/Vector.h"
+#endif
+
+#include <algorithm>
+#include <initializer_list>
+
+#include "NamespaceImports.h"
+
+#include "jit/AtomicOp.h"
+#include "jit/FixedList.h"
+#include "jit/InlineList.h"
+#include "jit/JitAllocPolicy.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MIROpsGenerated.h"
+#include "jit/ShuffleAnalysis.h"
+#include "jit/TypeData.h"
+#include "jit/TypePolicy.h"
+#include "js/experimental/JitInfo.h" // JSJit{Getter,Setter}Op, JSJitInfo
+#include "js/HeapAPI.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "js/Value.h"
+#include "js/Vector.h"
+#include "vm/BigIntType.h"
+#include "vm/EnvironmentObject.h"
+#include "vm/FunctionFlags.h" // js::FunctionFlags
+#include "vm/JSContext.h"
+#include "vm/RegExpObject.h"
+#include "vm/TypedArrayObject.h"
+#include "wasm/WasmJS.h" // for WasmInstanceObject
+
+namespace JS {
+struct ExpandoAndGeneration;
+}
+
+namespace js {
+
+namespace wasm {
+class FuncExport;
+extern uint32_t MIRTypeToABIResultSize(jit::MIRType);
+} // namespace wasm
+
+class GenericPrinter;
+class NativeIteratorListHead;
+class StringObject;
+
+enum class UnaryMathFunction : uint8_t;
+
+bool CurrentThreadIsIonCompiling();
+
+namespace jit {
+
+#ifdef JS_JITSPEW
+// Helper for debug printing. Avoids creating a MIR.h <--> MIRGraph.h cycle.
+// Implementation of this needs to see inside `MBasicBlock`; that is possible
+// in MIR.cpp since it also includes MIRGraph.h, whereas this file does not.
+class MBasicBlock;
+uint32_t GetMBasicBlockId(const MBasicBlock* block);
+
+// Helper class for debug printing. This class allows `::getExtras` methods
+// to add strings to be printed, on a per-MIR-node basis. The strings are
+// copied into storage owned by this class when `::add` is called, so the
+// `::getExtras` methods do not need to be concerned about storage management.
+class MOZ_STACK_CLASS ExtrasCollector {
+ mozilla::Vector<UniqueChars, 4> strings_;
+
+ public:
+ // Add `str` to the collection. A copy, owned by this object, is made. In
+ // case of OOM the call has no effect.
+ void add(const char* str) {
+ UniqueChars dup = DuplicateString(str);
+ if (dup) {
+ (void)strings_.append(std::move(dup));
+ }
+ }
+ size_t count() const { return strings_.length(); }
+ UniqueChars get(size_t ix) { return std::move(strings_[ix]); }
+};
+#endif
+
+// Forward declarations of MIR types.
+#define FORWARD_DECLARE(op) class M##op;
+MIR_OPCODE_LIST(FORWARD_DECLARE)
+#undef FORWARD_DECLARE
+
+// MDefinition visitor which ignores non-overloaded visit functions.
+class MDefinitionVisitorDefaultNoop {
+ public:
+#define VISIT_INS(op) \
+ void visit##op(M##op*) {}
+ MIR_OPCODE_LIST(VISIT_INS)
+#undef VISIT_INS
+};
+
+class BytecodeSite;
+class CompactBufferWriter;
+class Range;
+
+#define MIR_FLAG_LIST(_) \
+ _(InWorklist) \
+ _(EmittedAtUses) \
+ _(Commutative) \
+ _(Movable) /* Allow passes like LICM to move this instruction */ \
+ _(Lowered) /* (Debug only) has a virtual register */ \
+ _(Guard) /* Not removable if uses == 0 */ \
+ \
+ /* Flag an instruction to be considered as a Guard if the instructions \
+ * bails out on some inputs. \
+ * \
+ * Some optimizations can replace an instruction, and leave its operands \
+ * unused. When the type information of the operand got used as a \
+ * predicate of the transformation, then we have to flag the operands as \
+ * GuardRangeBailouts. \
+ * \
+ * This flag prevents further optimization of instructions, which \
+ * might remove the run-time checks (bailout conditions) used as a \
+ * predicate of the previous transformation. \
+ */ \
+ _(GuardRangeBailouts) \
+ \
+ /* Some instructions have uses that aren't directly represented in the \
+ * graph, and need to be handled specially. As an example, this is used to \
+ * keep the flagged instruction in resume points, not substituting with an \
+ * UndefinedValue. This can be used by call inlining when a function \
+ * argument is not used by the inlined instructions. It is also used \
+ * to annotate instructions which were used in removed branches. \
+ */ \
+ _(ImplicitlyUsed) \
+ \
+ /* The instruction has been marked dead for lazy removal from resume \
+ * points. \
+ */ \
+ _(Unused) \
+ \
+ /* Marks if the current instruction should go to the bailout paths instead \
+ * of producing code as part of the control flow. This flag can only be set \
+ * on instructions which are only used by ResumePoint or by other flagged \
+ * instructions. \
+ */ \
+ _(RecoveredOnBailout) \
+ \
+ /* Some instructions might represent an object, but the memory of these \
+ * objects might be incomplete if we have not recovered all the stores which \
+ * were supposed to happen before. This flag is used to annotate \
+ * instructions which might return a pointer to a memory area which is not \
+ * yet fully initialized. This flag is used to ensure that stores are \
+ * executed before returning the value. \
+ */ \
+ _(IncompleteObject) \
+ \
+ /* For WebAssembly, there are functions with multiple results. Instead of \
+ * having the results defined by one call instruction, they are instead \
+ * captured in subsequent result capture instructions, because modelling \
+ * multi-value results in Ion is too complicated. However since they \
+ * capture ambient live registers, it would be an error to move an unrelated \
+ * instruction between the call and the result capture. This flag is used \
+ * to prevent code motion from moving instructions in invalid ways. \
+ */ \
+ _(CallResultCapture) \
+ \
+ /* The current instruction got discarded from the MIR Graph. This is useful \
+ * when we want to iterate over resume points and instructions, while \
+ * handling instructions which are discarded without reporting to the \
+ * iterator. \
+ */ \
+ _(Discarded)
+
+class MDefinition;
+class MInstruction;
+class MBasicBlock;
+class MNode;
+class MUse;
+class MPhi;
+class MIRGraph;
+class MResumePoint;
+class MControlInstruction;
+
+// Represents a use of a node.
+class MUse : public TempObject, public InlineListNode<MUse> {
+ // Grant access to setProducerUnchecked.
+ friend class MDefinition;
+ friend class MPhi;
+
+ MDefinition* producer_; // MDefinition that is being used.
+ MNode* consumer_; // The node that is using this operand.
+
+ // Low-level unchecked edit method for replaceAllUsesWith and
+ // MPhi::removeOperand. This doesn't update use lists!
+ // replaceAllUsesWith and MPhi::removeOperand do that manually.
+ void setProducerUnchecked(MDefinition* producer) {
+ MOZ_ASSERT(consumer_);
+ MOZ_ASSERT(producer_);
+ MOZ_ASSERT(producer);
+ producer_ = producer;
+ }
+
+ public:
+ // Default constructor for use in vectors.
+ MUse() : producer_(nullptr), consumer_(nullptr) {}
+
+ // Move constructor for use in vectors. When an MUse is moved, it stays
+ // in its containing use list.
+ MUse(MUse&& other)
+ : InlineListNode<MUse>(std::move(other)),
+ producer_(other.producer_),
+ consumer_(other.consumer_) {}
+
+ // Construct an MUse initialized with |producer| and |consumer|.
+ MUse(MDefinition* producer, MNode* consumer) {
+ initUnchecked(producer, consumer);
+ }
+
+ // Set this use, which was previously clear.
+ inline void init(MDefinition* producer, MNode* consumer);
+ // Like init, but works even when the use contains uninitialized data.
+ inline void initUnchecked(MDefinition* producer, MNode* consumer);
+ // Like initUnchecked, but set the producer to nullptr.
+ inline void initUncheckedWithoutProducer(MNode* consumer);
+ // Set this use, which was not previously clear.
+ inline void replaceProducer(MDefinition* producer);
+ // Clear this use.
+ inline void releaseProducer();
+
+ MDefinition* producer() const {
+ MOZ_ASSERT(producer_ != nullptr);
+ return producer_;
+ }
+ bool hasProducer() const { return producer_ != nullptr; }
+ MNode* consumer() const {
+ MOZ_ASSERT(consumer_ != nullptr);
+ return consumer_;
+ }
+
+#ifdef DEBUG
+ // Return the operand index of this MUse in its consumer. This is DEBUG-only
+ // as normal code should instead call indexOf on the cast consumer directly,
+ // to allow it to be devirtualized and inlined.
+ size_t index() const;
+#endif
+};
+
+using MUseIterator = InlineList<MUse>::iterator;
+
+// A node is an entry in the MIR graph. It has two kinds:
+// MInstruction: an instruction which appears in the IR stream.
+// MResumePoint: a list of instructions that correspond to the state of the
+// interpreter/Baseline stack.
+//
+// Nodes can hold references to MDefinitions. Each MDefinition has a list of
+// nodes holding such a reference (its use chain).
+class MNode : public TempObject {
+ protected:
+ enum class Kind { Definition = 0, ResumePoint };
+
+ private:
+ static const uintptr_t KindMask = 0x1;
+ uintptr_t blockAndKind_;
+
+ Kind kind() const { return Kind(blockAndKind_ & KindMask); }
+
+ protected:
+ explicit MNode(const MNode& other) : blockAndKind_(other.blockAndKind_) {}
+
+ MNode(MBasicBlock* block, Kind kind) { setBlockAndKind(block, kind); }
+
+ void setBlockAndKind(MBasicBlock* block, Kind kind) {
+ blockAndKind_ = uintptr_t(block) | uintptr_t(kind);
+ MOZ_ASSERT(this->block() == block);
+ }
+
+ MBasicBlock* definitionBlock() const {
+ MOZ_ASSERT(isDefinition());
+ static_assert(unsigned(Kind::Definition) == 0,
+ "Code below relies on low bit being 0");
+ return reinterpret_cast<MBasicBlock*>(blockAndKind_);
+ }
+ MBasicBlock* resumePointBlock() const {
+ MOZ_ASSERT(isResumePoint());
+ static_assert(unsigned(Kind::ResumePoint) == 1,
+ "Code below relies on low bit being 1");
+ // Use a subtraction: if the caller does block()->foo, the compiler
+ // will be able to fold it with the load.
+ return reinterpret_cast<MBasicBlock*>(blockAndKind_ - 1);
+ }
+
+ public:
+ // Returns the definition at a given operand.
+ virtual MDefinition* getOperand(size_t index) const = 0;
+ virtual size_t numOperands() const = 0;
+ virtual size_t indexOf(const MUse* u) const = 0;
+
+ bool isDefinition() const { return kind() == Kind::Definition; }
+ bool isResumePoint() const { return kind() == Kind::ResumePoint; }
+ MBasicBlock* block() const {
+ return reinterpret_cast<MBasicBlock*>(blockAndKind_ & ~KindMask);
+ }
+ MBasicBlock* caller() const;
+
+ // Sets an already set operand, updating use information. If you're looking
+ // for setOperand, this is probably what you want.
+ virtual void replaceOperand(size_t index, MDefinition* operand) = 0;
+
+ // Resets the operand to an uninitialized state, breaking the link
+ // with the previous operand's producer.
+ void releaseOperand(size_t index) { getUseFor(index)->releaseProducer(); }
+ bool hasOperand(size_t index) const {
+ return getUseFor(index)->hasProducer();
+ }
+
+ inline MDefinition* toDefinition();
+ inline MResumePoint* toResumePoint();
+
+ [[nodiscard]] virtual bool writeRecoverData(
+ CompactBufferWriter& writer) const;
+
+#ifdef JS_JITSPEW
+ virtual void dump(GenericPrinter& out) const = 0;
+ virtual void dump() const = 0;
+#endif
+
+ protected:
+ // Need visibility on getUseFor to avoid O(n^2) complexity.
+ friend void AssertBasicGraphCoherency(MIRGraph& graph, bool force);
+
+ // Gets the MUse corresponding to given operand.
+ virtual MUse* getUseFor(size_t index) = 0;
+ virtual const MUse* getUseFor(size_t index) const = 0;
+};
+
+class AliasSet {
+ private:
+ uint32_t flags_;
+
+ public:
+ enum Flag {
+ None_ = 0,
+ ObjectFields = 1 << 0, // shape, class, slots, length etc.
+ Element = 1 << 1, // A Value member of obj->elements or
+ // a typed object.
+ UnboxedElement = 1 << 2, // An unboxed scalar or reference member of
+ // typed object.
+ DynamicSlot = 1 << 3, // A Value member of obj->slots.
+ FixedSlot = 1 << 4, // A Value member of obj->fixedSlots().
+ DOMProperty = 1 << 5, // A DOM property
+ WasmGlobalVar = 1 << 6, // An asm.js/wasm private global var
+ WasmHeap = 1 << 7, // An asm.js/wasm heap load
+ WasmHeapMeta = 1 << 8, // The asm.js/wasm heap base pointer and
+ // bounds check limit, in Instance.
+ ArrayBufferViewLengthOrOffset =
+ 1 << 9, // An array buffer view's length or byteOffset
+ WasmGlobalCell = 1 << 10, // A wasm global cell
+ WasmTableElement = 1 << 11, // An element of a wasm table
+ WasmTableMeta = 1 << 12, // A wasm table elements pointer and
+ // length field, in instance data.
+ WasmStackResult = 1 << 13, // A stack result from the current function
+
+ // JSContext's exception state. This is used on instructions like MThrow
+ // or MNewArrayDynamicLength that throw exceptions (other than OOM) but have
+ // no other side effect, to ensure that they get their own up-to-date resume
+ // point. (This resume point will be used when constructing the Baseline
+ // frame during exception bailouts.)
+ ExceptionState = 1 << 14,
+
+ // Used for instructions that load the privateSlot of DOM proxies and
+ // the ExpandoAndGeneration.
+ DOMProxyExpando = 1 << 15,
+
+ // Hash table of a Map or Set object.
+ MapOrSetHashTable = 1 << 16,
+
+ // Internal state of the random number generator
+ RNG = 1 << 17,
+
+ // The pendingException slot on the wasm instance object.
+ WasmPendingException = 1 << 18,
+
+ // The fuzzilliHash slot
+ FuzzilliHash = 1 << 19,
+
+ // The WasmStructObject::inlineData_[..] storage area
+ WasmStructInlineDataArea = 1 << 20,
+
+ // The WasmStructObject::outlineData_ pointer only
+ WasmStructOutlineDataPointer = 1 << 21,
+
+ // The malloc'd block that WasmStructObject::outlineData_ points at
+ WasmStructOutlineDataArea = 1 << 22,
+
+ // The WasmArrayObject::numElements_ field
+ WasmArrayNumElements = 1 << 23,
+
+ // The WasmArrayObject::data_ pointer only
+ WasmArrayDataPointer = 1 << 24,
+
+ // The malloc'd block that WasmArrayObject::data_ points at
+ WasmArrayDataArea = 1 << 25,
+
+ Last = WasmArrayDataArea,
+
+ Any = Last | (Last - 1),
+ NumCategories = 26,
+
+ // Indicates load or store.
+ Store_ = 1 << 31
+ };
+
+ static_assert((1 << NumCategories) - 1 == Any,
+ "NumCategories must include all flags present in Any");
+
+ explicit AliasSet(uint32_t flags) : flags_(flags) {}
+
+ public:
+ inline bool isNone() const { return flags_ == None_; }
+ uint32_t flags() const { return flags_ & Any; }
+ inline bool isStore() const { return !!(flags_ & Store_); }
+ inline bool isLoad() const { return !isStore() && !isNone(); }
+ inline AliasSet operator|(const AliasSet& other) const {
+ return AliasSet(flags_ | other.flags_);
+ }
+ inline AliasSet operator&(const AliasSet& other) const {
+ return AliasSet(flags_ & other.flags_);
+ }
+ static AliasSet None() { return AliasSet(None_); }
+ static AliasSet Load(uint32_t flags) {
+ MOZ_ASSERT(flags && !(flags & Store_));
+ return AliasSet(flags);
+ }
+ static AliasSet Store(uint32_t flags) {
+ MOZ_ASSERT(flags && !(flags & Store_));
+ return AliasSet(flags | Store_);
+ }
+};
+
+typedef Vector<MDefinition*, 6, JitAllocPolicy> MDefinitionVector;
+typedef Vector<MInstruction*, 6, JitAllocPolicy> MInstructionVector;
+
+// When a floating-point value is used by nodes which would prefer to
+// receive integer inputs, we may be able to help by computing our result
+// into an integer directly.
+//
+// A value can be truncated in 4 differents ways:
+// 1. Ignore Infinities (x / 0 --> 0).
+// 2. Ignore overflow (INT_MIN / -1 == (INT_MAX + 1) --> INT_MIN)
+// 3. Ignore negative zeros. (-0 --> 0)
+// 4. Ignore remainder. (3 / 4 --> 0)
+//
+// Indirect truncation is used to represent that we are interested in the
+// truncated result, but only if it can safely flow into operations which
+// are computed modulo 2^32, such as (2) and (3). Infinities are not safe,
+// as they would have absorbed other math operations. Remainders are not
+// safe, as fractions can be scaled up by multiplication.
+//
+// Division is a particularly interesting node here because it covers all 4
+// cases even when its own operands are integers.
+//
+// Note that these enum values are ordered from least value-modifying to
+// most value-modifying, and code relies on this ordering.
+enum class TruncateKind {
+ // No correction.
+ NoTruncate = 0,
+ // An integer is desired, but we can't skip bailout checks.
+ TruncateAfterBailouts = 1,
+ // The value will be truncated after some arithmetic (see above).
+ IndirectTruncate = 2,
+ // Direct and infallible truncation to int32.
+ Truncate = 3
+};
+
+// An MDefinition is an SSA name.
+class MDefinition : public MNode {
+ friend class MBasicBlock;
+
+ public:
+ enum class Opcode : uint16_t {
+#define DEFINE_OPCODES(op) op,
+ MIR_OPCODE_LIST(DEFINE_OPCODES)
+#undef DEFINE_OPCODES
+ };
+
+ private:
+ InlineList<MUse> uses_; // Use chain.
+ uint32_t id_; // Instruction ID, which after block re-ordering
+ // is sorted within a basic block.
+ Opcode op_; // Opcode.
+ uint16_t flags_; // Bit flags.
+ Range* range_; // Any computed range for this def.
+ union {
+ MDefinition*
+ loadDependency_; // Implicit dependency (store, call, etc.) of this
+ // instruction. Used by alias analysis, GVN and LICM.
+ uint32_t virtualRegister_; // Used by lowering to map definitions to
+ // virtual registers.
+ };
+
+ // Track bailouts by storing the current pc in MIR instruction. Also used
+ // for profiling and keeping track of what the last known pc was.
+ const BytecodeSite* trackedSite_;
+
+ // If we generate a bailout path for this instruction, this is the
+ // bailout kind that will be encoded in the snapshot. When we bail out,
+ // FinishBailoutToBaseline may take action based on the bailout kind to
+ // prevent bailout loops. (For example, if an instruction bails out after
+ // being hoisted by LICM, we will disable LICM when recompiling the script.)
+ BailoutKind bailoutKind_;
+
+ MIRType resultType_; // Representation of result type.
+
+ private:
+ enum Flag {
+ None = 0,
+#define DEFINE_FLAG(flag) flag,
+ MIR_FLAG_LIST(DEFINE_FLAG)
+#undef DEFINE_FLAG
+ Total
+ };
+
+ bool hasFlags(uint32_t flags) const { return (flags_ & flags) == flags; }
+ void removeFlags(uint32_t flags) { flags_ &= ~flags; }
+ void setFlags(uint32_t flags) { flags_ |= flags; }
+
+ // Calling isDefinition or isResumePoint on MDefinition is unnecessary.
+ bool isDefinition() const = delete;
+ bool isResumePoint() const = delete;
+
+ protected:
+ void setInstructionBlock(MBasicBlock* block, const BytecodeSite* site) {
+ MOZ_ASSERT(isInstruction());
+ setBlockAndKind(block, Kind::Definition);
+ setTrackedSite(site);
+ }
+
+ void setPhiBlock(MBasicBlock* block) {
+ MOZ_ASSERT(isPhi());
+ setBlockAndKind(block, Kind::Definition);
+ }
+
+ static HashNumber addU32ToHash(HashNumber hash, uint32_t data) {
+ return data + (hash << 6) + (hash << 16) - hash;
+ }
+
+ public:
+ explicit MDefinition(Opcode op)
+ : MNode(nullptr, Kind::Definition),
+ id_(0),
+ op_(op),
+ flags_(0),
+ range_(nullptr),
+ loadDependency_(nullptr),
+ trackedSite_(nullptr),
+ bailoutKind_(BailoutKind::Unknown),
+ resultType_(MIRType::None) {}
+
+ // Copying a definition leaves the list of uses empty.
+ explicit MDefinition(const MDefinition& other)
+ : MNode(other),
+ id_(0),
+ op_(other.op_),
+ flags_(other.flags_),
+ range_(other.range_),
+ loadDependency_(other.loadDependency_),
+ trackedSite_(other.trackedSite_),
+ bailoutKind_(other.bailoutKind_),
+ resultType_(other.resultType_) {}
+
+ Opcode op() const { return op_; }
+
+#ifdef JS_JITSPEW
+ const char* opName() const;
+ void printName(GenericPrinter& out) const;
+ static void PrintOpcodeName(GenericPrinter& out, Opcode op);
+ virtual void printOpcode(GenericPrinter& out) const;
+ void dump(GenericPrinter& out) const override;
+ void dump() const override;
+ void dumpLocation(GenericPrinter& out) const;
+ void dumpLocation() const;
+ // Dump any other stuff the node wants to have printed in `extras`. The
+ // added strings are copied, with the `ExtrasCollector` taking ownership of
+ // the copies.
+ virtual void getExtras(ExtrasCollector* extras) {}
+#endif
+
+ // Also for LICM. Test whether this definition is likely to be a call, which
+ // would clobber all or many of the floating-point registers, such that
+ // hoisting floating-point constants out of containing loops isn't likely to
+ // be worthwhile.
+ virtual bool possiblyCalls() const { return false; }
+
+ MBasicBlock* block() const { return definitionBlock(); }
+
+ private:
+#ifdef DEBUG
+ bool trackedSiteMatchesBlock(const BytecodeSite* site) const;
+#endif
+
+ void setTrackedSite(const BytecodeSite* site) {
+ MOZ_ASSERT(site);
+ MOZ_ASSERT(trackedSiteMatchesBlock(site),
+ "tracked bytecode site should match block bytecode site");
+ trackedSite_ = site;
+ }
+
+ public:
+ const BytecodeSite* trackedSite() const {
+ MOZ_ASSERT(trackedSite_,
+ "missing tracked bytecode site; node not assigned to a block?");
+ MOZ_ASSERT(trackedSiteMatchesBlock(trackedSite_),
+ "tracked bytecode site should match block bytecode site");
+ return trackedSite_;
+ }
+
+ BailoutKind bailoutKind() const { return bailoutKind_; }
+ void setBailoutKind(BailoutKind kind) { bailoutKind_ = kind; }
+
+ // Return the range of this value, *before* any bailout checks. Contrast
+ // this with the type() method, and the Range constructor which takes an
+ // MDefinition*, which describe the value *after* any bailout checks.
+ //
+ // Warning: Range analysis is removing the bit-operations such as '| 0' at
+ // the end of the transformations. Using this function to analyse any
+ // operands after the truncate phase of the range analysis will lead to
+ // errors. Instead, one should define the collectRangeInfoPreTrunc() to set
+ // the right set of flags which are dependent on the range of the inputs.
+ Range* range() const {
+ MOZ_ASSERT(type() != MIRType::None);
+ return range_;
+ }
+ void setRange(Range* range) {
+ MOZ_ASSERT(type() != MIRType::None);
+ range_ = range;
+ }
+
+ virtual HashNumber valueHash() const;
+ virtual bool congruentTo(const MDefinition* ins) const { return false; }
+ const MDefinition* skipObjectGuards() const;
+ bool congruentIfOperandsEqual(const MDefinition* ins) const;
+ virtual MDefinition* foldsTo(TempAllocator& alloc);
+ virtual void analyzeEdgeCasesForward();
+ virtual void analyzeEdgeCasesBackward();
+
+ // |needTruncation| records the truncation kind of the results, such that it
+ // can be used to truncate the operands of this instruction. If
+ // |needTruncation| function returns true, then the |truncate| function is
+ // called on the same instruction to mutate the instruction, such as
+ // updating the return type, the range and the specialization of the
+ // instruction.
+ virtual bool needTruncation(TruncateKind kind) const;
+ virtual void truncate(TruncateKind kind);
+
+ // Determine what kind of truncate this node prefers for the operand at the
+ // given index.
+ virtual TruncateKind operandTruncateKind(size_t index) const;
+
+ // Compute an absolute or symbolic range for the value of this node.
+ virtual void computeRange(TempAllocator& alloc) {}
+
+ // Collect information from the pre-truncated ranges.
+ virtual void collectRangeInfoPreTrunc() {}
+
+ uint32_t id() const {
+ MOZ_ASSERT(block());
+ return id_;
+ }
+ void setId(uint32_t id) { id_ = id; }
+
+#define FLAG_ACCESSOR(flag) \
+ bool is##flag() const { \
+ static_assert(Flag::Total <= sizeof(flags_) * 8, \
+ "Flags should fit in flags_ field"); \
+ return hasFlags(1 << flag); \
+ } \
+ void set##flag() { \
+ MOZ_ASSERT(!hasFlags(1 << flag)); \
+ setFlags(1 << flag); \
+ } \
+ void setNot##flag() { \
+ MOZ_ASSERT(hasFlags(1 << flag)); \
+ removeFlags(1 << flag); \
+ } \
+ void set##flag##Unchecked() { setFlags(1 << flag); } \
+ void setNot##flag##Unchecked() { removeFlags(1 << flag); }
+
+ MIR_FLAG_LIST(FLAG_ACCESSOR)
+#undef FLAG_ACCESSOR
+
+ // Return the type of this value. This may be speculative, and enforced
+ // dynamically with the use of bailout checks. If all the bailout checks
+ // pass, the value will have this type.
+ //
+ // Unless this is an MUrsh that has bailouts disabled, which, as a special
+ // case, may return a value in (INT32_MAX,UINT32_MAX] even when its type()
+ // is MIRType::Int32.
+ MIRType type() const { return resultType_; }
+
+ bool mightBeType(MIRType type) const {
+ MOZ_ASSERT(type != MIRType::Value);
+
+ if (type == this->type()) {
+ return true;
+ }
+
+ if (this->type() == MIRType::Value) {
+ return true;
+ }
+
+ return false;
+ }
+
+ bool mightBeMagicType() const;
+
+ // Return true if the result-set types are a subset of the given types.
+ bool definitelyType(std::initializer_list<MIRType> types) const;
+
+ // Float32 specialization operations (see big comment in IonAnalysis before
+ // the Float32 specialization algorithm).
+ virtual bool isFloat32Commutative() const { return false; }
+ virtual bool canProduceFloat32() const { return false; }
+ virtual bool canConsumeFloat32(MUse* use) const { return false; }
+ virtual void trySpecializeFloat32(TempAllocator& alloc) {}
+#ifdef DEBUG
+ // Used during the pass that checks that Float32 flow into valid MDefinitions
+ virtual bool isConsistentFloat32Use(MUse* use) const {
+ return type() == MIRType::Float32 || canConsumeFloat32(use);
+ }
+#endif
+
+ // Returns the beginning of this definition's use chain.
+ MUseIterator usesBegin() const { return uses_.begin(); }
+
+ // Returns the end of this definition's use chain.
+ MUseIterator usesEnd() const { return uses_.end(); }
+
+ bool canEmitAtUses() const { return !isEmittedAtUses(); }
+
+ // Removes a use at the given position
+ void removeUse(MUse* use) { uses_.remove(use); }
+
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ // Number of uses of this instruction. This function is only available
+ // in DEBUG mode since it requires traversing the list. Most users should
+ // use hasUses() or hasOneUse() instead.
+ size_t useCount() const;
+
+ // Number of uses of this instruction (only counting MDefinitions, ignoring
+ // MResumePoints). This function is only available in DEBUG mode since it
+ // requires traversing the list. Most users should use hasUses() or
+ // hasOneUse() instead.
+ size_t defUseCount() const;
+#endif
+
+ // Test whether this MDefinition has exactly one use.
+ bool hasOneUse() const;
+
+ // Test whether this MDefinition has exactly one use.
+ // (only counting MDefinitions, ignoring MResumePoints)
+ bool hasOneDefUse() const;
+
+ // Test whether this MDefinition has at least one use.
+ // (only counting MDefinitions, ignoring MResumePoints)
+ bool hasDefUses() const;
+
+ // Test whether this MDefinition has at least one non-recovered use.
+ // (only counting MDefinitions, ignoring MResumePoints)
+ bool hasLiveDefUses() const;
+
+ bool hasUses() const { return !uses_.empty(); }
+
+ // If this MDefinition has a single use (ignoring MResumePoints), returns that
+ // use's definition. Else returns nullptr.
+ MDefinition* maybeSingleDefUse() const;
+
+ // Returns the most recently added use (ignoring MResumePoints) for this
+ // MDefinition. Returns nullptr if there are no uses. Note that this relies on
+ // addUse adding new uses to the front of the list, and should only be called
+ // during MIR building (before optimization passes make changes to the uses).
+ MDefinition* maybeMostRecentlyAddedDefUse() const;
+
+ void addUse(MUse* use) {
+ MOZ_ASSERT(use->producer() == this);
+ uses_.pushFront(use);
+ }
+ void addUseUnchecked(MUse* use) {
+ MOZ_ASSERT(use->producer() == this);
+ uses_.pushFrontUnchecked(use);
+ }
+ void replaceUse(MUse* old, MUse* now) {
+ MOZ_ASSERT(now->producer() == this);
+ uses_.replace(old, now);
+ }
+
+ // Replace the current instruction by a dominating instruction |dom| in all
+ // uses of the current instruction.
+ void replaceAllUsesWith(MDefinition* dom);
+
+ // Like replaceAllUsesWith, but doesn't set ImplicitlyUsed on |this|'s
+ // operands.
+ void justReplaceAllUsesWith(MDefinition* dom);
+
+ // Replace the current instruction by an optimized-out constant in all uses
+ // of the current instruction. Note, that optimized-out constant should not
+ // be observed, and thus they should not flow in any computation.
+ [[nodiscard]] bool optimizeOutAllUses(TempAllocator& alloc);
+
+ // Replace the current instruction by a dominating instruction |dom| in all
+ // instruction, but keep the current instruction for resume point and
+ // instruction which are recovered on bailouts.
+ void replaceAllLiveUsesWith(MDefinition* dom);
+
+ // Mark this instruction as having replaced all uses of ins, as during GVN,
+ // returning false if the replacement should not be performed. For use when
+ // GVN eliminates instructions which are not equivalent to one another.
+ [[nodiscard]] virtual bool updateForReplacement(MDefinition* ins) {
+ return true;
+ }
+
+ void setVirtualRegister(uint32_t vreg) {
+ virtualRegister_ = vreg;
+ setLoweredUnchecked();
+ }
+ uint32_t virtualRegister() const {
+ MOZ_ASSERT(isLowered());
+ return virtualRegister_;
+ }
+
+ public:
+ // Opcode testing and casts.
+ template <typename MIRType>
+ bool is() const {
+ return op() == MIRType::classOpcode;
+ }
+ template <typename MIRType>
+ MIRType* to() {
+ MOZ_ASSERT(this->is<MIRType>());
+ return static_cast<MIRType*>(this);
+ }
+ template <typename MIRType>
+ const MIRType* to() const {
+ MOZ_ASSERT(this->is<MIRType>());
+ return static_cast<const MIRType*>(this);
+ }
+#define OPCODE_CASTS(opcode) \
+ bool is##opcode() const { return this->is<M##opcode>(); } \
+ M##opcode* to##opcode() { return this->to<M##opcode>(); } \
+ const M##opcode* to##opcode() const { return this->to<M##opcode>(); }
+ MIR_OPCODE_LIST(OPCODE_CASTS)
+#undef OPCODE_CASTS
+
+ inline MConstant* maybeConstantValue();
+
+ inline MInstruction* toInstruction();
+ inline const MInstruction* toInstruction() const;
+ bool isInstruction() const { return !isPhi(); }
+
+ virtual bool isControlInstruction() const { return false; }
+ inline MControlInstruction* toControlInstruction();
+
+ void setResultType(MIRType type) { resultType_ = type; }
+ virtual AliasSet getAliasSet() const {
+ // Instructions are effectful by default.
+ return AliasSet::Store(AliasSet::Any);
+ }
+
+#ifdef DEBUG
+ bool hasDefaultAliasSet() const {
+ AliasSet set = getAliasSet();
+ return set.isStore() && set.flags() == AliasSet::Flag::Any;
+ }
+#endif
+
+ MDefinition* dependency() const {
+ if (getAliasSet().isStore()) {
+ return nullptr;
+ }
+ return loadDependency_;
+ }
+ void setDependency(MDefinition* dependency) {
+ MOZ_ASSERT(!getAliasSet().isStore());
+ loadDependency_ = dependency;
+ }
+ bool isEffectful() const { return getAliasSet().isStore(); }
+
+#ifdef DEBUG
+ bool needsResumePoint() const {
+ // Return whether this instruction should have its own resume point.
+ return isEffectful();
+ }
+#endif
+
+ enum class AliasType : uint32_t { NoAlias = 0, MayAlias = 1, MustAlias = 2 };
+ virtual AliasType mightAlias(const MDefinition* store) const {
+ // Return whether this load may depend on the specified store, given
+ // that the alias sets intersect. This may be refined to exclude
+ // possible aliasing in cases where alias set flags are too imprecise.
+ if (!(getAliasSet().flags() & store->getAliasSet().flags())) {
+ return AliasType::NoAlias;
+ }
+ MOZ_ASSERT(!isEffectful() && store->isEffectful());
+ return AliasType::MayAlias;
+ }
+
+ virtual bool canRecoverOnBailout() const { return false; }
+};
+
+// An MUseDefIterator walks over uses in a definition, skipping any use that is
+// not a definition. Items from the use list must not be deleted during
+// iteration.
+class MUseDefIterator {
+ const MDefinition* def_;
+ MUseIterator current_;
+
+ MUseIterator search(MUseIterator start) {
+ MUseIterator i(start);
+ for (; i != def_->usesEnd(); i++) {
+ if (i->consumer()->isDefinition()) {
+ return i;
+ }
+ }
+ return def_->usesEnd();
+ }
+
+ public:
+ explicit MUseDefIterator(const MDefinition* def)
+ : def_(def), current_(search(def->usesBegin())) {}
+
+ explicit operator bool() const { return current_ != def_->usesEnd(); }
+ MUseDefIterator operator++() {
+ MOZ_ASSERT(current_ != def_->usesEnd());
+ ++current_;
+ current_ = search(current_);
+ return *this;
+ }
+ MUseDefIterator operator++(int) {
+ MUseDefIterator old(*this);
+ operator++();
+ return old;
+ }
+ MUse* use() const { return *current_; }
+ MDefinition* def() const { return current_->consumer()->toDefinition(); }
+};
+
+// Helper class to check that GC pointers embedded in MIR instructions are not
+// in the nursery. Off-thread compilation and nursery GCs can happen in
+// parallel. Nursery pointers are handled with MNurseryObject and the
+// nurseryObjects lists in WarpSnapshot and IonScript.
+//
+// These GC things are rooted through the WarpSnapshot. Compacting GCs cancel
+// off-thread compilations.
+template <typename T>
+class CompilerGCPointer {
+ js::gc::Cell* ptr_;
+
+ public:
+ explicit CompilerGCPointer(T ptr) : ptr_(ptr) {
+ MOZ_ASSERT_IF(ptr, !IsInsideNursery(ptr));
+ MOZ_ASSERT_IF(!CurrentThreadIsIonCompiling(), TlsContext.get()->suppressGC);
+ }
+
+ operator T() const { return static_cast<T>(ptr_); }
+ T operator->() const { return static_cast<T>(ptr_); }
+
+ private:
+ CompilerGCPointer() = delete;
+ CompilerGCPointer(const CompilerGCPointer<T>&) = delete;
+ CompilerGCPointer<T>& operator=(const CompilerGCPointer<T>&) = delete;
+};
+
+using CompilerObject = CompilerGCPointer<JSObject*>;
+using CompilerNativeObject = CompilerGCPointer<NativeObject*>;
+using CompilerFunction = CompilerGCPointer<JSFunction*>;
+using CompilerBaseScript = CompilerGCPointer<BaseScript*>;
+using CompilerPropertyName = CompilerGCPointer<PropertyName*>;
+using CompilerShape = CompilerGCPointer<Shape*>;
+using CompilerGetterSetter = CompilerGCPointer<GetterSetter*>;
+
+// An instruction is an SSA name that is inserted into a basic block's IR
+// stream.
+class MInstruction : public MDefinition, public InlineListNode<MInstruction> {
+ MResumePoint* resumePoint_;
+
+ protected:
+ // All MInstructions are using the "MFoo::New(alloc)" notation instead of
+ // the TempObject new operator. This code redefines the new operator as
+ // protected, and delegates to the TempObject new operator. Thus, the
+ // following code prevents calls to "new(alloc) MFoo" outside the MFoo
+ // members.
+ inline void* operator new(size_t nbytes,
+ TempAllocator::Fallible view) noexcept(true) {
+ return TempObject::operator new(nbytes, view);
+ }
+ inline void* operator new(size_t nbytes, TempAllocator& alloc) {
+ return TempObject::operator new(nbytes, alloc);
+ }
+ template <class T>
+ inline void* operator new(size_t nbytes, T* pos) {
+ return TempObject::operator new(nbytes, pos);
+ }
+
+ public:
+ explicit MInstruction(Opcode op) : MDefinition(op), resumePoint_(nullptr) {}
+
+ // Copying an instruction leaves the resume point as empty.
+ explicit MInstruction(const MInstruction& other)
+ : MDefinition(other), resumePoint_(nullptr) {}
+
+ // Convenient function used for replacing a load by the value of the store
+ // if the types are match, and boxing the value if they do not match.
+ MDefinition* foldsToStore(TempAllocator& alloc);
+
+ void setResumePoint(MResumePoint* resumePoint);
+ void stealResumePoint(MInstruction* other);
+
+ void moveResumePointAsEntry();
+ void clearResumePoint();
+ MResumePoint* resumePoint() const { return resumePoint_; }
+
+ // For instructions which can be cloned with new inputs, with all other
+ // information being the same. clone() implementations do not need to worry
+ // about cloning generic MInstruction/MDefinition state like flags and
+ // resume points.
+ virtual bool canClone() const { return false; }
+ virtual MInstruction* clone(TempAllocator& alloc,
+ const MDefinitionVector& inputs) const {
+ MOZ_CRASH();
+ }
+
+ // Instructions needing to hook into type analysis should return a
+ // TypePolicy.
+ virtual const TypePolicy* typePolicy() = 0;
+ virtual MIRType typePolicySpecialization() = 0;
+};
+
+// Note: GenerateOpcodeFiles.py generates MOpcodesGenerated.h based on the
+// INSTRUCTION_HEADER* macros.
+#define INSTRUCTION_HEADER_WITHOUT_TYPEPOLICY(opcode) \
+ static const Opcode classOpcode = Opcode::opcode; \
+ using MThisOpcode = M##opcode;
+
+#define INSTRUCTION_HEADER(opcode) \
+ INSTRUCTION_HEADER_WITHOUT_TYPEPOLICY(opcode) \
+ virtual const TypePolicy* typePolicy() override; \
+ virtual MIRType typePolicySpecialization() override;
+
+#define ALLOW_CLONE(typename) \
+ bool canClone() const override { return true; } \
+ MInstruction* clone(TempAllocator& alloc, const MDefinitionVector& inputs) \
+ const override { \
+ MInstruction* res = new (alloc) typename(*this); \
+ for (size_t i = 0; i < numOperands(); i++) \
+ res->replaceOperand(i, inputs[i]); \
+ return res; \
+ }
+
+// Adds MFoo::New functions which are mirroring the arguments of the
+// constructors. Opcodes which are using this macro can be called with a
+// TempAllocator, or the fallible version of the TempAllocator.
+#define TRIVIAL_NEW_WRAPPERS \
+ template <typename... Args> \
+ static MThisOpcode* New(TempAllocator& alloc, Args&&... args) { \
+ return new (alloc) MThisOpcode(std::forward<Args>(args)...); \
+ } \
+ template <typename... Args> \
+ static MThisOpcode* New(TempAllocator::Fallible alloc, Args&&... args) { \
+ return new (alloc) MThisOpcode(std::forward<Args>(args)...); \
+ }
+
+// These macros are used as a syntactic sugar for writting getOperand
+// accessors. They are meant to be used in the body of MIR Instructions as
+// follows:
+//
+// public:
+// INSTRUCTION_HEADER(Foo)
+// NAMED_OPERANDS((0, lhs), (1, rhs))
+//
+// The above example defines 2 accessors, one named "lhs" accessing the first
+// operand, and a one named "rhs" accessing the second operand.
+#define NAMED_OPERAND_ACCESSOR(Index, Name) \
+ MDefinition* Name() const { return getOperand(Index); }
+#define NAMED_OPERAND_ACCESSOR_APPLY(Args) NAMED_OPERAND_ACCESSOR Args
+#define NAMED_OPERANDS(...) \
+ MOZ_FOR_EACH(NAMED_OPERAND_ACCESSOR_APPLY, (), (__VA_ARGS__))
+
+template <size_t Arity>
+class MAryInstruction : public MInstruction {
+ mozilla::Array<MUse, Arity> operands_;
+
+ protected:
+ MUse* getUseFor(size_t index) final { return &operands_[index]; }
+ const MUse* getUseFor(size_t index) const final { return &operands_[index]; }
+ void initOperand(size_t index, MDefinition* operand) {
+ operands_[index].init(operand, this);
+ }
+
+ public:
+ MDefinition* getOperand(size_t index) const final {
+ return operands_[index].producer();
+ }
+ size_t numOperands() const final { return Arity; }
+#ifdef DEBUG
+ static const size_t staticNumOperands = Arity;
+#endif
+ size_t indexOf(const MUse* u) const final {
+ MOZ_ASSERT(u >= &operands_[0]);
+ MOZ_ASSERT(u <= &operands_[numOperands() - 1]);
+ return u - &operands_[0];
+ }
+ void replaceOperand(size_t index, MDefinition* operand) final {
+ operands_[index].replaceProducer(operand);
+ }
+
+ explicit MAryInstruction(Opcode op) : MInstruction(op) {}
+
+ explicit MAryInstruction(const MAryInstruction<Arity>& other)
+ : MInstruction(other) {
+ for (int i = 0; i < (int)Arity;
+ i++) { // N.B. use |int| to avoid warnings when Arity == 0
+ operands_[i].init(other.operands_[i].producer(), this);
+ }
+ }
+};
+
+class MNullaryInstruction : public MAryInstruction<0>,
+ public NoTypePolicy::Data {
+ protected:
+ explicit MNullaryInstruction(Opcode op) : MAryInstruction(op) {}
+
+ HashNumber valueHash() const override;
+};
+
+class MUnaryInstruction : public MAryInstruction<1> {
+ protected:
+ MUnaryInstruction(Opcode op, MDefinition* ins) : MAryInstruction(op) {
+ initOperand(0, ins);
+ }
+
+ HashNumber valueHash() const override;
+
+ public:
+ NAMED_OPERANDS((0, input))
+};
+
+class MBinaryInstruction : public MAryInstruction<2> {
+ protected:
+ MBinaryInstruction(Opcode op, MDefinition* left, MDefinition* right)
+ : MAryInstruction(op) {
+ initOperand(0, left);
+ initOperand(1, right);
+ }
+
+ public:
+ NAMED_OPERANDS((0, lhs), (1, rhs))
+
+ protected:
+ HashNumber valueHash() const override;
+
+ bool binaryCongruentTo(const MDefinition* ins) const {
+ if (op() != ins->op()) {
+ return false;
+ }
+
+ if (type() != ins->type()) {
+ return false;
+ }
+
+ if (isEffectful() || ins->isEffectful()) {
+ return false;
+ }
+
+ const MDefinition* left = getOperand(0);
+ const MDefinition* right = getOperand(1);
+ if (isCommutative() && left->id() > right->id()) {
+ std::swap(left, right);
+ }
+
+ const MBinaryInstruction* bi = static_cast<const MBinaryInstruction*>(ins);
+ const MDefinition* insLeft = bi->getOperand(0);
+ const MDefinition* insRight = bi->getOperand(1);
+ if (bi->isCommutative() && insLeft->id() > insRight->id()) {
+ std::swap(insLeft, insRight);
+ }
+
+ return left == insLeft && right == insRight;
+ }
+
+ public:
+ // Return if the operands to this instruction are both unsigned.
+ static bool unsignedOperands(MDefinition* left, MDefinition* right);
+ bool unsignedOperands();
+
+ // Replace any wrapping operands with the underlying int32 operands
+ // in case of unsigned operands.
+ void replaceWithUnsignedOperands();
+};
+
+class MTernaryInstruction : public MAryInstruction<3> {
+ protected:
+ MTernaryInstruction(Opcode op, MDefinition* first, MDefinition* second,
+ MDefinition* third)
+ : MAryInstruction(op) {
+ initOperand(0, first);
+ initOperand(1, second);
+ initOperand(2, third);
+ }
+
+ HashNumber valueHash() const override;
+};
+
+class MQuaternaryInstruction : public MAryInstruction<4> {
+ protected:
+ MQuaternaryInstruction(Opcode op, MDefinition* first, MDefinition* second,
+ MDefinition* third, MDefinition* fourth)
+ : MAryInstruction(op) {
+ initOperand(0, first);
+ initOperand(1, second);
+ initOperand(2, third);
+ initOperand(3, fourth);
+ }
+
+ HashNumber valueHash() const override;
+};
+
+template <class T>
+class MVariadicT : public T {
+ FixedList<MUse> operands_;
+
+ protected:
+ explicit MVariadicT(typename T::Opcode op) : T(op) {}
+ [[nodiscard]] bool init(TempAllocator& alloc, size_t length) {
+ return operands_.init(alloc, length);
+ }
+ void initOperand(size_t index, MDefinition* operand) {
+ // FixedList doesn't initialize its elements, so do an unchecked init.
+ operands_[index].initUnchecked(operand, this);
+ }
+ MUse* getUseFor(size_t index) final { return &operands_[index]; }
+ const MUse* getUseFor(size_t index) const final { return &operands_[index]; }
+
+ // The MWasmCallBase mixin performs initialization for it's subclasses.
+ friend class MWasmCallBase;
+
+ public:
+ // Will assert if called before initialization.
+ MDefinition* getOperand(size_t index) const final {
+ return operands_[index].producer();
+ }
+ size_t numOperands() const final { return operands_.length(); }
+ size_t indexOf(const MUse* u) const final {
+ MOZ_ASSERT(u >= &operands_[0]);
+ MOZ_ASSERT(u <= &operands_[numOperands() - 1]);
+ return u - &operands_[0];
+ }
+ void replaceOperand(size_t index, MDefinition* operand) final {
+ operands_[index].replaceProducer(operand);
+ }
+};
+
+// An instruction with a variable number of operands. Note that the
+// MFoo::New constructor for variadic instructions fallibly
+// initializes the operands_ array and must be checked for OOM.
+using MVariadicInstruction = MVariadicT<MInstruction>;
+
+MIR_OPCODE_CLASS_GENERATED
+
+// Truncation barrier. This is intended for protecting its input against
+// follow-up truncation optimizations.
+class MLimitedTruncate : public MUnaryInstruction,
+ public ConvertToInt32Policy<0>::Data {
+ TruncateKind truncate_;
+ TruncateKind truncateLimit_;
+
+ MLimitedTruncate(MDefinition* input, TruncateKind limit)
+ : MUnaryInstruction(classOpcode, input),
+ truncate_(TruncateKind::NoTruncate),
+ truncateLimit_(limit) {
+ setResultType(MIRType::Int32);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(LimitedTruncate)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ void computeRange(TempAllocator& alloc) override;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+ TruncateKind truncateKind() const { return truncate_; }
+ void setTruncateKind(TruncateKind kind) { truncate_ = kind; }
+};
+
+// A constant js::Value.
+class MConstant : public MNullaryInstruction {
+ struct Payload {
+ union {
+ bool b;
+ int32_t i32;
+ int64_t i64;
+ intptr_t iptr;
+ float f;
+ double d;
+ JSString* str;
+ JS::Symbol* sym;
+ BigInt* bi;
+ JSObject* obj;
+ Shape* shape;
+ uint64_t asBits;
+ };
+ Payload() : asBits(0) {}
+ };
+
+ Payload payload_;
+
+ static_assert(sizeof(Payload) == sizeof(uint64_t),
+ "asBits must be big enough for all payload bits");
+
+#ifdef DEBUG
+ void assertInitializedPayload() const;
+#else
+ void assertInitializedPayload() const {}
+#endif
+
+ MConstant(TempAllocator& alloc, const Value& v);
+ explicit MConstant(JSObject* obj);
+ explicit MConstant(Shape* shape);
+ explicit MConstant(float f);
+ explicit MConstant(MIRType type, int64_t i);
+
+ public:
+ INSTRUCTION_HEADER(Constant)
+ static MConstant* New(TempAllocator& alloc, const Value& v);
+ static MConstant* New(TempAllocator::Fallible alloc, const Value& v);
+ static MConstant* New(TempAllocator& alloc, const Value& v, MIRType type);
+ static MConstant* NewFloat32(TempAllocator& alloc, double d);
+ static MConstant* NewInt64(TempAllocator& alloc, int64_t i);
+ static MConstant* NewIntPtr(TempAllocator& alloc, intptr_t i);
+ static MConstant* NewObject(TempAllocator& alloc, JSObject* v);
+ static MConstant* NewShape(TempAllocator& alloc, Shape* s);
+ static MConstant* Copy(TempAllocator& alloc, MConstant* src) {
+ return new (alloc) MConstant(*src);
+ }
+
+ // Try to convert this constant to boolean, similar to js::ToBoolean.
+ // Returns false if the type is MIRType::Magic* or MIRType::Object.
+ [[nodiscard]] bool valueToBoolean(bool* res) const;
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ HashNumber valueHash() const override;
+ bool congruentTo(const MDefinition* ins) const override;
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ [[nodiscard]] bool updateForReplacement(MDefinition* def) override {
+ MConstant* c = def->toConstant();
+ // During constant folding, we don't want to replace a float32
+ // value by a double value.
+ if (type() == MIRType::Float32) {
+ return c->type() == MIRType::Float32;
+ }
+ if (type() == MIRType::Double) {
+ return c->type() != MIRType::Float32;
+ }
+ return true;
+ }
+
+ void computeRange(TempAllocator& alloc) override;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+
+ bool canProduceFloat32() const override;
+
+ ALLOW_CLONE(MConstant)
+
+ bool equals(const MConstant* other) const {
+ assertInitializedPayload();
+ return type() == other->type() && payload_.asBits == other->payload_.asBits;
+ }
+
+ bool toBoolean() const {
+ MOZ_ASSERT(type() == MIRType::Boolean);
+ return payload_.b;
+ }
+ int32_t toInt32() const {
+ MOZ_ASSERT(type() == MIRType::Int32);
+ return payload_.i32;
+ }
+ int64_t toInt64() const {
+ MOZ_ASSERT(type() == MIRType::Int64);
+ return payload_.i64;
+ }
+ intptr_t toIntPtr() const {
+ MOZ_ASSERT(type() == MIRType::IntPtr);
+ return payload_.iptr;
+ }
+ bool isInt32(int32_t i) const {
+ return type() == MIRType::Int32 && payload_.i32 == i;
+ }
+ bool isInt64(int64_t i) const {
+ return type() == MIRType::Int64 && payload_.i64 == i;
+ }
+ const double& toDouble() const {
+ MOZ_ASSERT(type() == MIRType::Double);
+ return payload_.d;
+ }
+ const float& toFloat32() const {
+ MOZ_ASSERT(type() == MIRType::Float32);
+ return payload_.f;
+ }
+ JSString* toString() const {
+ MOZ_ASSERT(type() == MIRType::String);
+ return payload_.str;
+ }
+ JS::Symbol* toSymbol() const {
+ MOZ_ASSERT(type() == MIRType::Symbol);
+ return payload_.sym;
+ }
+ BigInt* toBigInt() const {
+ MOZ_ASSERT(type() == MIRType::BigInt);
+ return payload_.bi;
+ }
+ JSObject& toObject() const {
+ MOZ_ASSERT(type() == MIRType::Object);
+ return *payload_.obj;
+ }
+ JSObject* toObjectOrNull() const {
+ if (type() == MIRType::Object) {
+ return payload_.obj;
+ }
+ MOZ_ASSERT(type() == MIRType::Null);
+ return nullptr;
+ }
+ Shape* toShape() const {
+ MOZ_ASSERT(type() == MIRType::Shape);
+ return payload_.shape;
+ }
+
+ bool isTypeRepresentableAsDouble() const {
+ return IsTypeRepresentableAsDouble(type());
+ }
+ double numberToDouble() const {
+ MOZ_ASSERT(isTypeRepresentableAsDouble());
+ if (type() == MIRType::Int32) {
+ return toInt32();
+ }
+ if (type() == MIRType::Double) {
+ return toDouble();
+ }
+ return toFloat32();
+ }
+
+ // Convert this constant to a js::Value. Float32 constants will be stored
+ // as DoubleValue and NaNs are canonicalized. Callers must be careful: not
+ // all constants can be represented by js::Value (wasm supports int64).
+ Value toJSValue() const;
+};
+
+class MWasmNullConstant : public MNullaryInstruction {
+ explicit MWasmNullConstant() : MNullaryInstruction(classOpcode) {
+ setResultType(MIRType::RefOrNull);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmNullConstant)
+ TRIVIAL_NEW_WRAPPERS
+
+ HashNumber valueHash() const override;
+ bool congruentTo(const MDefinition* ins) const override {
+ return ins->isWasmNullConstant();
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ ALLOW_CLONE(MWasmNullConstant)
+};
+
+// Floating-point value as created by wasm. Just a constant value, used to
+// effectively inhibit all the MIR optimizations. This uses the same LIR nodes
+// as a MConstant of the same type would.
+class MWasmFloatConstant : public MNullaryInstruction {
+ union {
+ float f32_;
+ double f64_;
+#ifdef ENABLE_WASM_SIMD
+ int8_t s128_[16];
+ uint64_t bits_[2];
+#else
+ uint64_t bits_[1];
+#endif
+ } u;
+
+ explicit MWasmFloatConstant(MIRType type) : MNullaryInstruction(classOpcode) {
+ u.bits_[0] = 0;
+#ifdef ENABLE_WASM_SIMD
+ u.bits_[1] = 0;
+#endif
+ setResultType(type);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmFloatConstant)
+
+ static MWasmFloatConstant* NewDouble(TempAllocator& alloc, double d) {
+ auto* ret = new (alloc) MWasmFloatConstant(MIRType::Double);
+ ret->u.f64_ = d;
+ return ret;
+ }
+
+ static MWasmFloatConstant* NewFloat32(TempAllocator& alloc, float f) {
+ auto* ret = new (alloc) MWasmFloatConstant(MIRType::Float32);
+ ret->u.f32_ = f;
+ return ret;
+ }
+
+#ifdef ENABLE_WASM_SIMD
+ static MWasmFloatConstant* NewSimd128(TempAllocator& alloc,
+ const SimdConstant& s) {
+ auto* ret = new (alloc) MWasmFloatConstant(MIRType::Simd128);
+ memcpy(ret->u.s128_, s.bytes(), 16);
+ return ret;
+ }
+#endif
+
+ HashNumber valueHash() const override;
+ bool congruentTo(const MDefinition* ins) const override;
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ const double& toDouble() const {
+ MOZ_ASSERT(type() == MIRType::Double);
+ return u.f64_;
+ }
+ const float& toFloat32() const {
+ MOZ_ASSERT(type() == MIRType::Float32);
+ return u.f32_;
+ }
+#ifdef ENABLE_WASM_SIMD
+ const SimdConstant toSimd128() const {
+ MOZ_ASSERT(type() == MIRType::Simd128);
+ return SimdConstant::CreateX16(u.s128_);
+ }
+#endif
+#ifdef JS_JITSPEW
+ void getExtras(ExtrasCollector* extras) override {
+ char buf[64];
+ switch (type()) {
+ case MIRType::Float32:
+ SprintfLiteral(buf, "f32{%e}", (double)u.f32_);
+ break;
+ case MIRType::Double:
+ SprintfLiteral(buf, "f64{%e}", u.f64_);
+ break;
+# ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ SprintfLiteral(buf, "v128{[1]=%016llx:[0]=%016llx}",
+ (unsigned long long int)u.bits_[1],
+ (unsigned long long int)u.bits_[0]);
+ break;
+# endif
+ default:
+ SprintfLiteral(buf, "!!getExtras: missing case!!");
+ break;
+ }
+ extras->add(buf);
+ }
+#endif
+};
+
+class MParameter : public MNullaryInstruction {
+ int32_t index_;
+
+ explicit MParameter(int32_t index)
+ : MNullaryInstruction(classOpcode), index_(index) {
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Parameter)
+ TRIVIAL_NEW_WRAPPERS
+
+ static const int32_t THIS_SLOT = -1;
+ int32_t index() const { return index_; }
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+ HashNumber valueHash() const override;
+ bool congruentTo(const MDefinition* ins) const override;
+};
+
+class MControlInstruction : public MInstruction {
+ protected:
+ explicit MControlInstruction(Opcode op) : MInstruction(op) {}
+
+ public:
+ virtual size_t numSuccessors() const = 0;
+ virtual MBasicBlock* getSuccessor(size_t i) const = 0;
+ virtual void replaceSuccessor(size_t i, MBasicBlock* successor) = 0;
+
+ void initSuccessor(size_t i, MBasicBlock* successor) {
+ MOZ_ASSERT(!getSuccessor(i));
+ replaceSuccessor(i, successor);
+ }
+
+ bool isControlInstruction() const override { return true; }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+};
+
+class MTableSwitch final : public MControlInstruction,
+ public NoFloatPolicy<0>::Data {
+ // The successors of the tableswitch
+ // - First successor = the default case
+ // - Successors 2 and higher = the cases
+ Vector<MBasicBlock*, 0, JitAllocPolicy> successors_;
+ // Index into successors_ sorted on case index
+ Vector<size_t, 0, JitAllocPolicy> cases_;
+
+ MUse operand_;
+ int32_t low_;
+ int32_t high_;
+
+ void initOperand(size_t index, MDefinition* operand) {
+ MOZ_ASSERT(index == 0);
+ operand_.init(operand, this);
+ }
+
+ MTableSwitch(TempAllocator& alloc, MDefinition* ins, int32_t low,
+ int32_t high)
+ : MControlInstruction(classOpcode),
+ successors_(alloc),
+ cases_(alloc),
+ low_(low),
+ high_(high) {
+ initOperand(0, ins);
+ }
+
+ protected:
+ MUse* getUseFor(size_t index) override {
+ MOZ_ASSERT(index == 0);
+ return &operand_;
+ }
+
+ const MUse* getUseFor(size_t index) const override {
+ MOZ_ASSERT(index == 0);
+ return &operand_;
+ }
+
+ public:
+ INSTRUCTION_HEADER(TableSwitch)
+
+ static MTableSwitch* New(TempAllocator& alloc, MDefinition* ins, int32_t low,
+ int32_t high) {
+ return new (alloc) MTableSwitch(alloc, ins, low, high);
+ }
+
+ size_t numSuccessors() const override { return successors_.length(); }
+
+ [[nodiscard]] bool addSuccessor(MBasicBlock* successor, size_t* index) {
+ MOZ_ASSERT(successors_.length() < (size_t)(high_ - low_ + 2));
+ MOZ_ASSERT(!successors_.empty());
+ *index = successors_.length();
+ return successors_.append(successor);
+ }
+
+ MBasicBlock* getSuccessor(size_t i) const override {
+ MOZ_ASSERT(i < numSuccessors());
+ return successors_[i];
+ }
+
+ void replaceSuccessor(size_t i, MBasicBlock* successor) override {
+ MOZ_ASSERT(i < numSuccessors());
+ successors_[i] = successor;
+ }
+
+ int32_t low() const { return low_; }
+
+ int32_t high() const { return high_; }
+
+ MBasicBlock* getDefault() const { return getSuccessor(0); }
+
+ MBasicBlock* getCase(size_t i) const { return getSuccessor(cases_[i]); }
+
+ [[nodiscard]] bool addDefault(MBasicBlock* block, size_t* index = nullptr) {
+ MOZ_ASSERT(successors_.empty());
+ if (index) {
+ *index = 0;
+ }
+ return successors_.append(block);
+ }
+
+ [[nodiscard]] bool addCase(size_t successorIndex) {
+ return cases_.append(successorIndex);
+ }
+
+ size_t numCases() const { return high() - low() + 1; }
+
+ MDefinition* getOperand(size_t index) const override {
+ MOZ_ASSERT(index == 0);
+ return operand_.producer();
+ }
+
+ size_t numOperands() const override { return 1; }
+
+ size_t indexOf(const MUse* u) const final {
+ MOZ_ASSERT(u == getUseFor(0));
+ return 0;
+ }
+
+ void replaceOperand(size_t index, MDefinition* operand) final {
+ MOZ_ASSERT(index == 0);
+ operand_.replaceProducer(operand);
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+};
+
+template <size_t Arity, size_t Successors>
+class MAryControlInstruction : public MControlInstruction {
+ mozilla::Array<MUse, Arity> operands_;
+ mozilla::Array<MBasicBlock*, Successors> successors_;
+
+ protected:
+ explicit MAryControlInstruction(Opcode op) : MControlInstruction(op) {}
+ void setSuccessor(size_t index, MBasicBlock* successor) {
+ successors_[index] = successor;
+ }
+
+ MUse* getUseFor(size_t index) final { return &operands_[index]; }
+ const MUse* getUseFor(size_t index) const final { return &operands_[index]; }
+ void initOperand(size_t index, MDefinition* operand) {
+ operands_[index].init(operand, this);
+ }
+
+ public:
+ MDefinition* getOperand(size_t index) const final {
+ return operands_[index].producer();
+ }
+ size_t numOperands() const final { return Arity; }
+ size_t indexOf(const MUse* u) const final {
+ MOZ_ASSERT(u >= &operands_[0]);
+ MOZ_ASSERT(u <= &operands_[numOperands() - 1]);
+ return u - &operands_[0];
+ }
+ void replaceOperand(size_t index, MDefinition* operand) final {
+ operands_[index].replaceProducer(operand);
+ }
+ size_t numSuccessors() const final { return Successors; }
+ MBasicBlock* getSuccessor(size_t i) const final { return successors_[i]; }
+ void replaceSuccessor(size_t i, MBasicBlock* succ) final {
+ successors_[i] = succ;
+ }
+};
+
+template <size_t Successors>
+class MVariadicControlInstruction : public MVariadicT<MControlInstruction> {
+ mozilla::Array<MBasicBlock*, Successors> successors_;
+
+ protected:
+ explicit MVariadicControlInstruction(Opcode op)
+ : MVariadicT<MControlInstruction>(op) {}
+ void setSuccessor(size_t index, MBasicBlock* successor) {
+ successors_[index] = successor;
+ }
+
+ public:
+ size_t numSuccessors() const final { return Successors; }
+ MBasicBlock* getSuccessor(size_t i) const final { return successors_[i]; }
+ void replaceSuccessor(size_t i, MBasicBlock* succ) final {
+ successors_[i] = succ;
+ }
+};
+
+// Jump to the start of another basic block.
+class MGoto : public MAryControlInstruction<0, 1>, public NoTypePolicy::Data {
+ explicit MGoto(MBasicBlock* target) : MAryControlInstruction(classOpcode) {
+ setSuccessor(TargetIndex, target);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Goto)
+ static MGoto* New(TempAllocator& alloc, MBasicBlock* target);
+ static MGoto* New(TempAllocator::Fallible alloc, MBasicBlock* target);
+
+ // Variant that may patch the target later.
+ static MGoto* New(TempAllocator& alloc);
+
+ static constexpr size_t TargetIndex = 0;
+
+ MBasicBlock* target() { return getSuccessor(TargetIndex); }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+#ifdef JS_JITSPEW
+ void getExtras(ExtrasCollector* extras) override {
+ char buf[64];
+ SprintfLiteral(buf, "Block%u", GetMBasicBlockId(target()));
+ extras->add(buf);
+ }
+#endif
+};
+
+// Tests if the input instruction evaluates to true or false, and jumps to the
+// start of a corresponding basic block.
+class MTest : public MAryControlInstruction<1, 2>, public TestPolicy::Data {
+ // It is allowable to specify `trueBranch` or `falseBranch` as nullptr and
+ // patch it in later.
+ MTest(MDefinition* ins, MBasicBlock* trueBranch, MBasicBlock* falseBranch)
+ : MAryControlInstruction(classOpcode) {
+ initOperand(0, ins);
+ setSuccessor(TrueBranchIndex, trueBranch);
+ setSuccessor(FalseBranchIndex, falseBranch);
+ }
+
+ TypeDataList observedTypes_;
+
+ public:
+ INSTRUCTION_HEADER(Test)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, input))
+
+ const TypeDataList& observedTypes() const { return observedTypes_; }
+ void setObservedTypes(const TypeDataList& observed) {
+ observedTypes_ = observed;
+ }
+
+ static constexpr size_t TrueBranchIndex = 0;
+ static constexpr size_t FalseBranchIndex = 1;
+
+ MBasicBlock* ifTrue() const { return getSuccessor(TrueBranchIndex); }
+ MBasicBlock* ifFalse() const { return getSuccessor(FalseBranchIndex); }
+ MBasicBlock* branchSuccessor(BranchDirection dir) const {
+ return (dir == TRUE_BRANCH) ? ifTrue() : ifFalse();
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ MDefinition* foldsDoubleNegation(TempAllocator& alloc);
+ MDefinition* foldsConstant(TempAllocator& alloc);
+ MDefinition* foldsTypes(TempAllocator& alloc);
+ MDefinition* foldsNeedlessControlFlow(TempAllocator& alloc);
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+
+#ifdef JS_JITSPEW
+ void getExtras(ExtrasCollector* extras) override {
+ char buf[64];
+ SprintfLiteral(buf, "true->Block%u false->Block%u",
+ GetMBasicBlockId(ifTrue()), GetMBasicBlockId(ifFalse()));
+ extras->add(buf);
+ }
+#endif
+};
+
+// Returns from this function to the previous caller.
+class MReturn : public MAryControlInstruction<1, 0>,
+ public BoxInputsPolicy::Data {
+ explicit MReturn(MDefinition* ins) : MAryControlInstruction(classOpcode) {
+ initOperand(0, ins);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Return)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, input))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MNewArray : public MUnaryInstruction, public NoTypePolicy::Data {
+ private:
+ // Number of elements to allocate for the array.
+ uint32_t length_;
+
+ // Heap where the array should be allocated.
+ gc::InitialHeap initialHeap_;
+
+ bool vmCall_;
+
+ MNewArray(uint32_t length, MConstant* templateConst,
+ gc::InitialHeap initialHeap, bool vmCall = false);
+
+ public:
+ INSTRUCTION_HEADER(NewArray)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MNewArray* NewVM(TempAllocator& alloc, uint32_t length,
+ MConstant* templateConst,
+ gc::InitialHeap initialHeap) {
+ return new (alloc) MNewArray(length, templateConst, initialHeap, true);
+ }
+
+ uint32_t length() const { return length_; }
+
+ JSObject* templateObject() const {
+ return getOperand(0)->toConstant()->toObjectOrNull();
+ }
+
+ gc::InitialHeap initialHeap() const { return initialHeap_; }
+
+ bool isVMCall() const { return vmCall_; }
+
+ // NewArray is marked as non-effectful because all our allocations are
+ // either lazy when we are using "new Array(length)" or bounded by the
+ // script or the stack size when we are using "new Array(...)" or "[...]"
+ // notations. So we might have to allocate the array twice if we bail
+ // during the computation of the first element of the square braket
+ // notation.
+ virtual AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override {
+ // The template object can safely be used in the recover instruction
+ // because it can never be mutated by any other function execution.
+ return templateObject() != nullptr;
+ }
+};
+
+class MNewTypedArray : public MUnaryInstruction, public NoTypePolicy::Data {
+ gc::InitialHeap initialHeap_;
+
+ MNewTypedArray(MConstant* templateConst, gc::InitialHeap initialHeap)
+ : MUnaryInstruction(classOpcode, templateConst),
+ initialHeap_(initialHeap) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(NewTypedArray)
+ TRIVIAL_NEW_WRAPPERS
+
+ TypedArrayObject* templateObject() const {
+ return &getOperand(0)->toConstant()->toObject().as<TypedArrayObject>();
+ }
+
+ gc::InitialHeap initialHeap() const { return initialHeap_; }
+
+ virtual AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+class MNewObject : public MUnaryInstruction, public NoTypePolicy::Data {
+ public:
+ enum Mode { ObjectLiteral, ObjectCreate };
+
+ private:
+ gc::InitialHeap initialHeap_;
+ Mode mode_;
+ bool vmCall_;
+
+ MNewObject(MConstant* templateConst, gc::InitialHeap initialHeap, Mode mode,
+ bool vmCall = false)
+ : MUnaryInstruction(classOpcode, templateConst),
+ initialHeap_(initialHeap),
+ mode_(mode),
+ vmCall_(vmCall) {
+ if (mode == ObjectLiteral) {
+ MOZ_ASSERT(!templateObject());
+ } else {
+ MOZ_ASSERT(templateObject());
+ }
+ setResultType(MIRType::Object);
+
+ // The constant is kept separated in a MConstant, this way we can safely
+ // mark it during GC if we recover the object allocation. Otherwise, by
+ // making it emittedAtUses, we do not produce register allocations for
+ // it and inline its content inside the code produced by the
+ // CodeGenerator.
+ if (templateConst->toConstant()->type() == MIRType::Object) {
+ templateConst->setEmittedAtUses();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(NewObject)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MNewObject* NewVM(TempAllocator& alloc, MConstant* templateConst,
+ gc::InitialHeap initialHeap, Mode mode) {
+ return new (alloc) MNewObject(templateConst, initialHeap, mode, true);
+ }
+
+ Mode mode() const { return mode_; }
+
+ JSObject* templateObject() const {
+ return getOperand(0)->toConstant()->toObjectOrNull();
+ }
+
+ gc::InitialHeap initialHeap() const { return initialHeap_; }
+
+ bool isVMCall() const { return vmCall_; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override {
+ // The template object can safely be used in the recover instruction
+ // because it can never be mutated by any other function execution.
+ return templateObject() != nullptr;
+ }
+};
+
+class MNewPlainObject : public MUnaryInstruction, public NoTypePolicy::Data {
+ private:
+ uint32_t numFixedSlots_;
+ uint32_t numDynamicSlots_;
+ gc::AllocKind allocKind_;
+ gc::InitialHeap initialHeap_;
+
+ MNewPlainObject(MConstant* shapeConst, uint32_t numFixedSlots,
+ uint32_t numDynamicSlots, gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap)
+ : MUnaryInstruction(classOpcode, shapeConst),
+ numFixedSlots_(numFixedSlots),
+ numDynamicSlots_(numDynamicSlots),
+ allocKind_(allocKind),
+ initialHeap_(initialHeap) {
+ setResultType(MIRType::Object);
+
+ // The shape constant is kept separated in a MConstant. This way we can
+ // safely mark it during GC if we recover the object allocation. Otherwise,
+ // by making it emittedAtUses, we do not produce register allocations for it
+ // and inline its content inside the code produced by the CodeGenerator.
+ MOZ_ASSERT(shapeConst->toConstant()->type() == MIRType::Shape);
+ shapeConst->setEmittedAtUses();
+ }
+
+ public:
+ INSTRUCTION_HEADER(NewPlainObject)
+ TRIVIAL_NEW_WRAPPERS
+
+ const Shape* shape() const { return getOperand(0)->toConstant()->toShape(); }
+
+ uint32_t numFixedSlots() const { return numFixedSlots_; }
+ uint32_t numDynamicSlots() const { return numDynamicSlots_; }
+ gc::AllocKind allocKind() const { return allocKind_; }
+ gc::InitialHeap initialHeap() const { return initialHeap_; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MNewArrayObject : public MUnaryInstruction, public NoTypePolicy::Data {
+ private:
+ uint32_t length_;
+ gc::InitialHeap initialHeap_;
+
+ MNewArrayObject(TempAllocator& alloc, MConstant* shapeConst, uint32_t length,
+ gc::InitialHeap initialHeap)
+ : MUnaryInstruction(classOpcode, shapeConst),
+ length_(length),
+ initialHeap_(initialHeap) {
+ setResultType(MIRType::Object);
+ MOZ_ASSERT(shapeConst->toConstant()->type() == MIRType::Shape);
+ shapeConst->setEmittedAtUses();
+ }
+
+ public:
+ INSTRUCTION_HEADER(NewArrayObject)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MNewArrayObject* New(TempAllocator& alloc, MConstant* shapeConst,
+ uint32_t length, gc::InitialHeap initialHeap) {
+ return new (alloc) MNewArrayObject(alloc, shapeConst, length, initialHeap);
+ }
+
+ const Shape* shape() const { return getOperand(0)->toConstant()->toShape(); }
+
+ // See MNewArray::getAliasSet comment.
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ uint32_t length() const { return length_; }
+ gc::InitialHeap initialHeap() const { return initialHeap_; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+class MNewIterator : public MUnaryInstruction, public NoTypePolicy::Data {
+ public:
+ enum Type {
+ ArrayIterator,
+ StringIterator,
+ RegExpStringIterator,
+ };
+
+ private:
+ Type type_;
+
+ MNewIterator(MConstant* templateConst, Type type)
+ : MUnaryInstruction(classOpcode, templateConst), type_(type) {
+ setResultType(MIRType::Object);
+ templateConst->setEmittedAtUses();
+ }
+
+ public:
+ INSTRUCTION_HEADER(NewIterator)
+ TRIVIAL_NEW_WRAPPERS
+
+ Type type() const { return type_; }
+
+ JSObject* templateObject() {
+ return getOperand(0)->toConstant()->toObjectOrNull();
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+// Represent the content of all slots of an object. This instruction is not
+// lowered and is not used to generate code.
+class MObjectState : public MVariadicInstruction,
+ public NoFloatPolicyAfter<1>::Data {
+ private:
+ uint32_t numSlots_;
+ uint32_t numFixedSlots_;
+
+ explicit MObjectState(JSObject* templateObject);
+ explicit MObjectState(const Shape* shape);
+ explicit MObjectState(MObjectState* state);
+
+ [[nodiscard]] bool init(TempAllocator& alloc, MDefinition* obj);
+
+ void initSlot(uint32_t slot, MDefinition* def) { initOperand(slot + 1, def); }
+
+ public:
+ INSTRUCTION_HEADER(ObjectState)
+ NAMED_OPERANDS((0, object))
+
+ // Return the template object of any object creation which can be recovered
+ // on bailout.
+ static JSObject* templateObjectOf(MDefinition* obj);
+
+ static MObjectState* New(TempAllocator& alloc, MDefinition* obj);
+ static MObjectState* Copy(TempAllocator& alloc, MObjectState* state);
+
+ // As we might do read of uninitialized properties, we have to copy the
+ // initial values from the template object.
+ void initFromTemplateObject(TempAllocator& alloc, MDefinition* undefinedVal);
+
+ size_t numFixedSlots() const { return numFixedSlots_; }
+ size_t numSlots() const { return numSlots_; }
+
+ MDefinition* getSlot(uint32_t slot) const { return getOperand(slot + 1); }
+ void setSlot(uint32_t slot, MDefinition* def) {
+ replaceOperand(slot + 1, def);
+ }
+
+ bool hasFixedSlot(uint32_t slot) const {
+ return slot < numSlots() && slot < numFixedSlots();
+ }
+ MDefinition* getFixedSlot(uint32_t slot) const {
+ MOZ_ASSERT(slot < numFixedSlots());
+ return getSlot(slot);
+ }
+ void setFixedSlot(uint32_t slot, MDefinition* def) {
+ MOZ_ASSERT(slot < numFixedSlots());
+ setSlot(slot, def);
+ }
+
+ bool hasDynamicSlot(uint32_t slot) const {
+ return numFixedSlots() < numSlots() && slot < numSlots() - numFixedSlots();
+ }
+ MDefinition* getDynamicSlot(uint32_t slot) const {
+ return getSlot(slot + numFixedSlots());
+ }
+ void setDynamicSlot(uint32_t slot, MDefinition* def) {
+ setSlot(slot + numFixedSlots(), def);
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+// Represent the contents of all elements of an array. This instruction is not
+// lowered and is not used to generate code.
+class MArrayState : public MVariadicInstruction,
+ public NoFloatPolicyAfter<2>::Data {
+ private:
+ uint32_t numElements_;
+
+ explicit MArrayState(MDefinition* arr);
+
+ [[nodiscard]] bool init(TempAllocator& alloc, MDefinition* obj,
+ MDefinition* len);
+
+ void initElement(uint32_t index, MDefinition* def) {
+ initOperand(index + 2, def);
+ }
+
+ public:
+ INSTRUCTION_HEADER(ArrayState)
+ NAMED_OPERANDS((0, array), (1, initializedLength))
+
+ static MArrayState* New(TempAllocator& alloc, MDefinition* arr,
+ MDefinition* initLength);
+ static MArrayState* Copy(TempAllocator& alloc, MArrayState* state);
+
+ void initFromTemplateObject(TempAllocator& alloc, MDefinition* undefinedVal);
+
+ void setInitializedLength(MDefinition* def) { replaceOperand(1, def); }
+
+ size_t numElements() const { return numElements_; }
+
+ MDefinition* getElement(uint32_t index) const {
+ return getOperand(index + 2);
+ }
+ void setElement(uint32_t index, MDefinition* def) {
+ replaceOperand(index + 2, def);
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+// WrappedFunction stores information about a function that can safely be used
+// off-thread. In particular, a function's flags can be modified on the main
+// thread as functions are relazified and delazified, so we must be careful not
+// to access these flags off-thread.
+class WrappedFunction : public TempObject {
+ // If this is a native function without a JitEntry, the JSFunction*.
+ CompilerFunction nativeFun_;
+ uint16_t nargs_;
+ js::FunctionFlags flags_;
+
+ public:
+ WrappedFunction(JSFunction* nativeFun, uint16_t nargs, FunctionFlags flags);
+
+ // Note: When adding new accessors be sure to add consistency asserts
+ // to the constructor.
+
+ size_t nargs() const { return nargs_; }
+
+ bool isNativeWithoutJitEntry() const {
+ return flags_.isNativeWithoutJitEntry();
+ }
+ bool hasJitEntry() const { return flags_.hasJitEntry(); }
+ bool isConstructor() const { return flags_.isConstructor(); }
+ bool isClassConstructor() const { return flags_.isClassConstructor(); }
+
+ // These fields never change, they can be accessed off-main thread.
+ JSNative native() const {
+ MOZ_ASSERT(isNativeWithoutJitEntry());
+ return nativeFun_->nativeUnchecked();
+ }
+ bool hasJitInfo() const {
+ return flags_.isBuiltinNative() && nativeFun_->jitInfoUnchecked();
+ }
+ const JSJitInfo* jitInfo() const {
+ MOZ_ASSERT(hasJitInfo());
+ return nativeFun_->jitInfoUnchecked();
+ }
+
+ JSFunction* rawNativeJSFunction() const { return nativeFun_; }
+};
+
+enum class DOMObjectKind : uint8_t { Proxy, Native };
+
+class MCall : public MVariadicInstruction, public CallPolicy::Data {
+ private:
+ // The callee, this, and the actual arguments are all operands of MCall.
+ static const size_t CalleeOperandIndex = 0;
+ static const size_t NumNonArgumentOperands = 1;
+
+ protected:
+ // Monomorphic cache for MCalls with a single JSFunction target.
+ WrappedFunction* target_;
+
+ // Original value of argc from the bytecode.
+ uint32_t numActualArgs_;
+
+ // True if the call is for JSOp::New or JSOp::SuperCall.
+ bool construct_ : 1;
+
+ // True if the caller does not use the return value.
+ bool ignoresReturnValue_ : 1;
+
+ bool needsClassCheck_ : 1;
+ bool maybeCrossRealm_ : 1;
+ bool needsThisCheck_ : 1;
+
+ MCall(WrappedFunction* target, uint32_t numActualArgs, bool construct,
+ bool ignoresReturnValue)
+ : MVariadicInstruction(classOpcode),
+ target_(target),
+ numActualArgs_(numActualArgs),
+ construct_(construct),
+ ignoresReturnValue_(ignoresReturnValue),
+ needsClassCheck_(true),
+ maybeCrossRealm_(true),
+ needsThisCheck_(false) {
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Call)
+ static MCall* New(TempAllocator& alloc, WrappedFunction* target,
+ size_t maxArgc, size_t numActualArgs, bool construct,
+ bool ignoresReturnValue, bool isDOMCall,
+ mozilla::Maybe<DOMObjectKind> objectKind);
+
+ void initCallee(MDefinition* func) { initOperand(CalleeOperandIndex, func); }
+
+ bool needsClassCheck() const { return needsClassCheck_; }
+ void disableClassCheck() { needsClassCheck_ = false; }
+
+ bool maybeCrossRealm() const { return maybeCrossRealm_; }
+ void setNotCrossRealm() { maybeCrossRealm_ = false; }
+
+ bool needsThisCheck() const { return needsThisCheck_; }
+ void setNeedsThisCheck() {
+ MOZ_ASSERT(construct_);
+ needsThisCheck_ = true;
+ }
+
+ MDefinition* getCallee() const { return getOperand(CalleeOperandIndex); }
+ void replaceCallee(MInstruction* newfunc) {
+ replaceOperand(CalleeOperandIndex, newfunc);
+ }
+
+ void addArg(size_t argnum, MDefinition* arg);
+
+ MDefinition* getArg(uint32_t index) const {
+ return getOperand(NumNonArgumentOperands + index);
+ }
+
+ static size_t IndexOfThis() { return NumNonArgumentOperands; }
+ static size_t IndexOfArgument(size_t index) {
+ return NumNonArgumentOperands + index + 1; // +1 to skip |this|.
+ }
+ static size_t IndexOfStackArg(size_t index) {
+ return NumNonArgumentOperands + index;
+ }
+
+ // For monomorphic callsites.
+ WrappedFunction* getSingleTarget() const { return target_; }
+
+ bool isConstructing() const { return construct_; }
+
+ bool ignoresReturnValue() const { return ignoresReturnValue_; }
+
+ // The number of stack arguments is the max between the number of formal
+ // arguments and the number of actual arguments. The number of stack
+ // argument includes the |undefined| padding added in case of underflow.
+ // Includes |this|.
+ uint32_t numStackArgs() const {
+ return numOperands() - NumNonArgumentOperands;
+ }
+
+ // Does not include |this|.
+ uint32_t numActualArgs() const { return numActualArgs_; }
+
+ bool possiblyCalls() const override { return true; }
+
+ virtual bool isCallDOMNative() const { return false; }
+
+ // A method that can be called to tell the MCall to figure out whether it's
+ // movable or not. This can't be done in the constructor, because it
+ // depends on the arguments to the call, and those aren't passed to the
+ // constructor but are set up later via addArg.
+ virtual void computeMovable() {}
+};
+
+class MCallDOMNative : public MCall {
+ // A helper class for MCalls for DOM natives. Note that this is NOT
+ // actually a separate MIR op from MCall, because all sorts of places use
+ // isCall() to check for calls and all we really want is to overload a few
+ // virtual things from MCall.
+
+ DOMObjectKind objectKind_;
+
+ MCallDOMNative(WrappedFunction* target, uint32_t numActualArgs,
+ DOMObjectKind objectKind)
+ : MCall(target, numActualArgs, false, false), objectKind_(objectKind) {
+ MOZ_ASSERT(getJitInfo()->type() != JSJitInfo::InlinableNative);
+
+ // If our jitinfo is not marked eliminatable, that means that our C++
+ // implementation is fallible or that it never wants to be eliminated or
+ // that we have no hope of ever doing the sort of argument analysis that
+ // would allow us to detemine that we're side-effect-free. In the
+ // latter case we wouldn't get DCEd no matter what, but for the former
+ // two cases we have to explicitly say that we can't be DCEd.
+ if (!getJitInfo()->isEliminatable) {
+ setGuard();
+ }
+ }
+
+ friend MCall* MCall::New(TempAllocator& alloc, WrappedFunction* target,
+ size_t maxArgc, size_t numActualArgs, bool construct,
+ bool ignoresReturnValue, bool isDOMCall,
+ mozilla::Maybe<DOMObjectKind> objectKind);
+
+ const JSJitInfo* getJitInfo() const;
+
+ public:
+ DOMObjectKind objectKind() const { return objectKind_; }
+
+ virtual AliasSet getAliasSet() const override;
+
+ virtual bool congruentTo(const MDefinition* ins) const override;
+
+ virtual bool isCallDOMNative() const override { return true; }
+
+ virtual void computeMovable() override;
+};
+
+// fun.apply(self, arguments)
+class MApplyArgs : public MTernaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<1>,
+ BoxPolicy<2>>::Data {
+ // Single target from CacheIR, or nullptr
+ WrappedFunction* target_;
+ // Number of extra initial formals to skip.
+ uint32_t numExtraFormals_;
+ bool maybeCrossRealm_ = true;
+ bool ignoresReturnValue_ = false;
+
+ MApplyArgs(WrappedFunction* target, MDefinition* fun, MDefinition* argc,
+ MDefinition* self, uint32_t numExtraFormals = 0)
+ : MTernaryInstruction(classOpcode, fun, argc, self),
+ target_(target),
+ numExtraFormals_(numExtraFormals) {
+ MOZ_ASSERT(argc->type() == MIRType::Int32);
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(ApplyArgs)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, getFunction), (1, getArgc), (2, getThis))
+
+ WrappedFunction* getSingleTarget() const { return target_; }
+
+ uint32_t numExtraFormals() const { return numExtraFormals_; }
+
+ bool maybeCrossRealm() const { return maybeCrossRealm_; }
+ void setNotCrossRealm() { maybeCrossRealm_ = false; }
+
+ bool ignoresReturnValue() const { return ignoresReturnValue_; }
+ void setIgnoresReturnValue() { ignoresReturnValue_ = true; }
+
+ bool isConstructing() const { return false; }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+class MApplyArgsObj
+ : public MTernaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, ObjectPolicy<1>, BoxPolicy<2>>::Data {
+ WrappedFunction* target_;
+ bool maybeCrossRealm_ = true;
+ bool ignoresReturnValue_ = false;
+
+ MApplyArgsObj(WrappedFunction* target, MDefinition* fun, MDefinition* argsObj,
+ MDefinition* thisArg)
+ : MTernaryInstruction(classOpcode, fun, argsObj, thisArg),
+ target_(target) {
+ MOZ_ASSERT(argsObj->type() == MIRType::Object);
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(ApplyArgsObj)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, getFunction), (1, getArgsObj), (2, getThis))
+
+ WrappedFunction* getSingleTarget() const { return target_; }
+
+ bool maybeCrossRealm() const { return maybeCrossRealm_; }
+ void setNotCrossRealm() { maybeCrossRealm_ = false; }
+
+ bool ignoresReturnValue() const { return ignoresReturnValue_; }
+ void setIgnoresReturnValue() { ignoresReturnValue_ = true; }
+
+ bool isConstructing() const { return false; }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+// fun.apply(fn, array)
+class MApplyArray : public MTernaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, BoxPolicy<2>>::Data {
+ // Single target from CacheIR, or nullptr
+ WrappedFunction* target_;
+ bool maybeCrossRealm_ = true;
+ bool ignoresReturnValue_ = false;
+
+ MApplyArray(WrappedFunction* target, MDefinition* fun, MDefinition* elements,
+ MDefinition* self)
+ : MTernaryInstruction(classOpcode, fun, elements, self), target_(target) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(ApplyArray)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, getFunction), (1, getElements), (2, getThis))
+
+ WrappedFunction* getSingleTarget() const { return target_; }
+
+ bool maybeCrossRealm() const { return maybeCrossRealm_; }
+ void setNotCrossRealm() { maybeCrossRealm_ = false; }
+
+ bool ignoresReturnValue() const { return ignoresReturnValue_; }
+ void setIgnoresReturnValue() { ignoresReturnValue_ = true; }
+
+ bool isConstructing() const { return false; }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+// |new F(...arguments)| and |super(...arguments)|.
+class MConstructArgs : public MQuaternaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<1>,
+ BoxPolicy<2>, ObjectPolicy<3>>::Data {
+ // Single target from CacheIR, or nullptr
+ WrappedFunction* target_;
+ // Number of extra initial formals to skip.
+ uint32_t numExtraFormals_;
+ bool maybeCrossRealm_ = true;
+
+ MConstructArgs(WrappedFunction* target, MDefinition* fun, MDefinition* argc,
+ MDefinition* thisValue, MDefinition* newTarget,
+ uint32_t numExtraFormals = 0)
+ : MQuaternaryInstruction(classOpcode, fun, argc, thisValue, newTarget),
+ target_(target),
+ numExtraFormals_(numExtraFormals) {
+ MOZ_ASSERT(argc->type() == MIRType::Int32);
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(ConstructArgs)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, getFunction), (1, getArgc), (2, getThis),
+ (3, getNewTarget))
+
+ WrappedFunction* getSingleTarget() const { return target_; }
+
+ uint32_t numExtraFormals() const { return numExtraFormals_; }
+
+ bool maybeCrossRealm() const { return maybeCrossRealm_; }
+ void setNotCrossRealm() { maybeCrossRealm_ = false; }
+
+ bool ignoresReturnValue() const { return false; }
+ bool isConstructing() const { return true; }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+// |new F(...args)| and |super(...args)|.
+class MConstructArray
+ : public MQuaternaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, BoxPolicy<2>, ObjectPolicy<3>>::Data {
+ // Single target from CacheIR, or nullptr
+ WrappedFunction* target_;
+ bool maybeCrossRealm_ = true;
+ bool needsThisCheck_ = false;
+
+ MConstructArray(WrappedFunction* target, MDefinition* fun,
+ MDefinition* elements, MDefinition* thisValue,
+ MDefinition* newTarget)
+ : MQuaternaryInstruction(classOpcode, fun, elements, thisValue,
+ newTarget),
+ target_(target) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(ConstructArray)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, getFunction), (1, getElements), (2, getThis),
+ (3, getNewTarget))
+
+ WrappedFunction* getSingleTarget() const { return target_; }
+
+ bool maybeCrossRealm() const { return maybeCrossRealm_; }
+ void setNotCrossRealm() { maybeCrossRealm_ = false; }
+
+ bool needsThisCheck() const { return needsThisCheck_; }
+ void setNeedsThisCheck() { needsThisCheck_ = true; }
+
+ bool ignoresReturnValue() const { return false; }
+ bool isConstructing() const { return true; }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+class MBail : public MNullaryInstruction {
+ explicit MBail(BailoutKind kind) : MNullaryInstruction(classOpcode) {
+ setBailoutKind(kind);
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Bail)
+
+ static MBail* New(TempAllocator& alloc, BailoutKind kind) {
+ return new (alloc) MBail(kind);
+ }
+ static MBail* New(TempAllocator& alloc) {
+ return new (alloc) MBail(BailoutKind::Inevitable);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MUnreachable : public MAryControlInstruction<0, 0>,
+ public NoTypePolicy::Data {
+ MUnreachable() : MAryControlInstruction(classOpcode) {}
+
+ public:
+ INSTRUCTION_HEADER(Unreachable)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MAssertRecoveredOnBailout : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ bool mustBeRecovered_;
+
+ MAssertRecoveredOnBailout(MDefinition* ins, bool mustBeRecovered)
+ : MUnaryInstruction(classOpcode, ins), mustBeRecovered_(mustBeRecovered) {
+ setResultType(MIRType::Value);
+ setRecoveredOnBailout();
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(AssertRecoveredOnBailout)
+ TRIVIAL_NEW_WRAPPERS
+
+ // Needed to assert that float32 instructions are correctly recovered.
+ bool canConsumeFloat32(MUse* use) const override { return true; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+class MAssertFloat32 : public MUnaryInstruction, public NoTypePolicy::Data {
+ bool mustBeFloat32_;
+
+ MAssertFloat32(MDefinition* value, bool mustBeFloat32)
+ : MUnaryInstruction(classOpcode, value), mustBeFloat32_(mustBeFloat32) {}
+
+ public:
+ INSTRUCTION_HEADER(AssertFloat32)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool canConsumeFloat32(MUse* use) const override { return true; }
+
+ bool mustBeFloat32() const { return mustBeFloat32_; }
+};
+
+class MCompare : public MBinaryInstruction, public ComparePolicy::Data {
+ public:
+ enum CompareType {
+
+ // Anything compared to Undefined
+ Compare_Undefined,
+
+ // Anything compared to Null
+ Compare_Null,
+
+ // Int32 compared to Int32
+ // Boolean compared to Boolean
+ Compare_Int32,
+
+ // Int32 compared as unsigneds
+ Compare_UInt32,
+
+ // Int64 compared to Int64.
+ Compare_Int64,
+
+ // Int64 compared as unsigneds.
+ Compare_UInt64,
+
+ // IntPtr compared as unsigneds.
+ Compare_UIntPtr,
+
+ // Double compared to Double
+ Compare_Double,
+
+ // Float compared to Float
+ Compare_Float32,
+
+ // String compared to String
+ Compare_String,
+
+ // Symbol compared to Symbol
+ Compare_Symbol,
+
+ // Object compared to Object
+ Compare_Object,
+
+ // BigInt compared to BigInt
+ Compare_BigInt,
+
+ // BigInt compared to Int32
+ Compare_BigInt_Int32,
+
+ // BigInt compared to Double
+ Compare_BigInt_Double,
+
+ // BigInt compared to String
+ Compare_BigInt_String,
+
+ // Wasm Ref/AnyRef/NullRef compared to Ref/AnyRef/NullRef
+ Compare_RefOrNull,
+ };
+
+ private:
+ CompareType compareType_;
+ JSOp jsop_;
+ bool operandsAreNeverNaN_;
+
+ // When a floating-point comparison is converted to an integer comparison
+ // (when range analysis proves it safe), we need to convert the operands
+ // to integer as well.
+ bool truncateOperands_;
+
+ MCompare(MDefinition* left, MDefinition* right, JSOp jsop,
+ CompareType compareType)
+ : MBinaryInstruction(classOpcode, left, right),
+ compareType_(compareType),
+ jsop_(jsop),
+ operandsAreNeverNaN_(false),
+ truncateOperands_(false) {
+ setResultType(MIRType::Boolean);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Compare)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MCompare* NewWasm(TempAllocator& alloc, MDefinition* left,
+ MDefinition* right, JSOp jsop,
+ CompareType compareType) {
+ MOZ_ASSERT(compareType == Compare_Int32 || compareType == Compare_UInt32 ||
+ compareType == Compare_Int64 || compareType == Compare_UInt64 ||
+ compareType == Compare_Double ||
+ compareType == Compare_Float32 ||
+ compareType == Compare_RefOrNull);
+ auto* ins = MCompare::New(alloc, left, right, jsop, compareType);
+ ins->setResultType(MIRType::Int32);
+ return ins;
+ }
+
+ [[nodiscard]] bool tryFold(bool* result);
+ [[nodiscard]] bool evaluateConstantOperands(TempAllocator& alloc,
+ bool* result);
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ CompareType compareType() const { return compareType_; }
+ bool isInt32Comparison() const { return compareType() == Compare_Int32; }
+ bool isDoubleComparison() const { return compareType() == Compare_Double; }
+ bool isFloat32Comparison() const { return compareType() == Compare_Float32; }
+ bool isNumericComparison() const {
+ return isInt32Comparison() || isDoubleComparison() || isFloat32Comparison();
+ }
+ MIRType inputType();
+
+ JSOp jsop() const { return jsop_; }
+ bool operandsAreNeverNaN() const { return operandsAreNeverNaN_; }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+ void collectRangeInfoPreTrunc() override;
+
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+ bool isFloat32Commutative() const override { return true; }
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override {
+ // Both sides of the compare can be Float32
+ return compareType_ == Compare_Float32;
+ }
+#endif
+
+ ALLOW_CLONE(MCompare)
+
+ private:
+ [[nodiscard]] bool tryFoldEqualOperands(bool* result);
+ [[nodiscard]] bool tryFoldTypeOf(bool* result);
+ [[nodiscard]] MDefinition* tryFoldTypeOf(TempAllocator& alloc);
+ [[nodiscard]] MDefinition* tryFoldCharCompare(TempAllocator& alloc);
+ [[nodiscard]] MDefinition* tryFoldStringCompare(TempAllocator& alloc);
+ [[nodiscard]] MDefinition* tryFoldStringSubstring(TempAllocator& alloc);
+ [[nodiscard]] MDefinition* tryFoldStringIndexOf(TempAllocator& alloc);
+
+ public:
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!binaryCongruentTo(ins)) {
+ return false;
+ }
+ return compareType() == ins->toCompare()->compareType() &&
+ jsop() == ins->toCompare()->jsop();
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override {
+ switch (compareType_) {
+ case Compare_Undefined:
+ case Compare_Null:
+ case Compare_Int32:
+ case Compare_UInt32:
+ case Compare_Double:
+ case Compare_Float32:
+ case Compare_String:
+ case Compare_Symbol:
+ case Compare_Object:
+ case Compare_BigInt:
+ case Compare_BigInt_Int32:
+ case Compare_BigInt_Double:
+ case Compare_BigInt_String:
+ return true;
+
+ case Compare_Int64:
+ case Compare_UInt64:
+ case Compare_UIntPtr:
+ case Compare_RefOrNull:
+ return false;
+ }
+ MOZ_CRASH("unexpected compare type");
+ }
+
+#ifdef JS_JITSPEW
+ void getExtras(ExtrasCollector* extras) override {
+ const char* ty = nullptr;
+ switch (compareType_) {
+ case Compare_Undefined:
+ ty = "Undefined";
+ break;
+ case Compare_Null:
+ ty = "Null";
+ break;
+ case Compare_Int32:
+ ty = "Int32";
+ break;
+ case Compare_UInt32:
+ ty = "UInt32";
+ break;
+ case Compare_Int64:
+ ty = "Int64";
+ break;
+ case Compare_UInt64:
+ ty = "UInt64";
+ break;
+ case Compare_UIntPtr:
+ ty = "UIntPtr";
+ break;
+ case Compare_Double:
+ ty = "Double";
+ break;
+ case Compare_Float32:
+ ty = "Float32";
+ break;
+ case Compare_String:
+ ty = "String";
+ break;
+ case Compare_Symbol:
+ ty = "Symbol";
+ break;
+ case Compare_Object:
+ ty = "Object";
+ break;
+ case Compare_BigInt:
+ ty = "BigInt";
+ break;
+ case Compare_BigInt_Int32:
+ ty = "BigInt_Int32";
+ break;
+ case Compare_BigInt_Double:
+ ty = "BigInt_Double";
+ break;
+ case Compare_BigInt_String:
+ ty = "BigInt_String";
+ break;
+ case Compare_RefOrNull:
+ ty = "RefOrNull";
+ break;
+ default:
+ ty = "!!unknown!!";
+ break;
+ };
+ char buf[64];
+ SprintfLiteral(buf, "ty=%s jsop=%s", ty, CodeName(jsop()));
+ extras->add(buf);
+ }
+#endif
+};
+
+// Takes a typed value and returns an untyped value.
+class MBox : public MUnaryInstruction, public NoTypePolicy::Data {
+ explicit MBox(MDefinition* ins) : MUnaryInstruction(classOpcode, ins) {
+ // Cannot box a box.
+ MOZ_ASSERT(ins->type() != MIRType::Value);
+
+ setResultType(MIRType::Value);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Box)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ ALLOW_CLONE(MBox)
+};
+
+// Note: the op may have been inverted during lowering (to put constants in a
+// position where they can be immediates), so it is important to use the
+// lir->jsop() instead of the mir->jsop() when it is present.
+static inline Assembler::Condition JSOpToCondition(
+ MCompare::CompareType compareType, JSOp op) {
+ bool isSigned = (compareType != MCompare::Compare_UInt32 &&
+ compareType != MCompare::Compare_UInt64 &&
+ compareType != MCompare::Compare_UIntPtr);
+ return JSOpToCondition(op, isSigned);
+}
+
+// Takes a typed value and checks if it is a certain type. If so, the payload
+// is unpacked and returned as that type. Otherwise, it is considered a
+// deoptimization.
+class MUnbox final : public MUnaryInstruction, public BoxInputsPolicy::Data {
+ public:
+ enum Mode {
+ Fallible, // Check the type, and deoptimize if unexpected.
+ Infallible, // Type guard is not necessary.
+ };
+
+ private:
+ Mode mode_;
+
+ MUnbox(MDefinition* ins, MIRType type, Mode mode)
+ : MUnaryInstruction(classOpcode, ins), mode_(mode) {
+ // Only allow unboxing a non MIRType::Value when input and output types
+ // don't match. This is often used to force a bailout. Boxing happens
+ // during type analysis.
+ MOZ_ASSERT_IF(ins->type() != MIRType::Value, type != ins->type());
+
+ MOZ_ASSERT(type == MIRType::Boolean || type == MIRType::Int32 ||
+ type == MIRType::Double || type == MIRType::String ||
+ type == MIRType::Symbol || type == MIRType::BigInt ||
+ type == MIRType::Object);
+
+ setResultType(type);
+ setMovable();
+
+ if (mode_ == Fallible) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(Unbox)
+ TRIVIAL_NEW_WRAPPERS
+
+ Mode mode() const { return mode_; }
+ bool fallible() const { return mode() != Infallible; }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isUnbox() || ins->toUnbox()->mode() != mode()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ ALLOW_CLONE(MUnbox)
+};
+
+class MAssertRange : public MUnaryInstruction, public NoTypePolicy::Data {
+ // This is the range checked by the assertion. Don't confuse this with the
+ // range_ member or the range() accessor. Since MAssertRange doesn't return
+ // a value, it doesn't use those.
+ const Range* assertedRange_;
+
+ MAssertRange(MDefinition* ins, const Range* assertedRange)
+ : MUnaryInstruction(classOpcode, ins), assertedRange_(assertedRange) {
+ setGuard();
+ setResultType(MIRType::None);
+ }
+
+ public:
+ INSTRUCTION_HEADER(AssertRange)
+ TRIVIAL_NEW_WRAPPERS
+
+ const Range* assertedRange() const { return assertedRange_; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+};
+
+class MAssertClass : public MUnaryInstruction, public NoTypePolicy::Data {
+ const JSClass* class_;
+
+ MAssertClass(MDefinition* obj, const JSClass* clasp)
+ : MUnaryInstruction(classOpcode, obj), class_(clasp) {
+ MOZ_ASSERT(obj->type() == MIRType::Object);
+
+ setGuard();
+ setResultType(MIRType::None);
+ }
+
+ public:
+ INSTRUCTION_HEADER(AssertClass)
+ TRIVIAL_NEW_WRAPPERS
+
+ const JSClass* getClass() const { return class_; }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MAssertShape : public MUnaryInstruction, public NoTypePolicy::Data {
+ CompilerShape shape_;
+
+ MAssertShape(MDefinition* obj, Shape* shape)
+ : MUnaryInstruction(classOpcode, obj), shape_(shape) {
+ MOZ_ASSERT(obj->type() == MIRType::Object);
+
+ setGuard();
+ setResultType(MIRType::None);
+ }
+
+ public:
+ INSTRUCTION_HEADER(AssertShape)
+ TRIVIAL_NEW_WRAPPERS
+
+ const Shape* shape() const { return shape_; }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Eager initialization of arguments object.
+class MCreateArgumentsObject : public MUnaryInstruction,
+ public ObjectPolicy<0>::Data {
+ CompilerGCPointer<ArgumentsObject*> templateObj_;
+
+ MCreateArgumentsObject(MDefinition* callObj, ArgumentsObject* templateObj)
+ : MUnaryInstruction(classOpcode, callObj), templateObj_(templateObj) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(CreateArgumentsObject)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, getCallObject))
+
+ ArgumentsObject* templateObject() const { return templateObj_; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool possiblyCalls() const override { return true; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+// Eager initialization of arguments object for inlined function
+class MCreateInlinedArgumentsObject : public MVariadicInstruction,
+ public NoFloatPolicyAfter<0>::Data {
+ CompilerGCPointer<ArgumentsObject*> templateObj_;
+
+ explicit MCreateInlinedArgumentsObject(ArgumentsObject* templateObj)
+ : MVariadicInstruction(classOpcode), templateObj_(templateObj) {
+ setResultType(MIRType::Object);
+ }
+
+ static const size_t NumNonArgumentOperands = 2;
+
+ public:
+ INSTRUCTION_HEADER(CreateInlinedArgumentsObject)
+ static MCreateInlinedArgumentsObject* New(TempAllocator& alloc,
+ MDefinition* callObj,
+ MDefinition* callee,
+ MDefinitionVector& args,
+ ArgumentsObject* templateObj);
+ NAMED_OPERANDS((0, getCallObject), (1, getCallee))
+
+ ArgumentsObject* templateObject() const { return templateObj_; }
+
+ MDefinition* getArg(uint32_t idx) const {
+ return getOperand(idx + NumNonArgumentOperands);
+ }
+ uint32_t numActuals() const { return numOperands() - NumNonArgumentOperands; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool possiblyCalls() const override { return true; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+class MGetInlinedArgument
+ : public MVariadicInstruction,
+ public MixPolicy<UnboxedInt32Policy<0>, NoFloatPolicyAfter<1>>::Data {
+ MGetInlinedArgument() : MVariadicInstruction(classOpcode) {
+ setResultType(MIRType::Value);
+ }
+
+ static const size_t NumNonArgumentOperands = 1;
+
+ public:
+ INSTRUCTION_HEADER(GetInlinedArgument)
+ static MGetInlinedArgument* New(TempAllocator& alloc, MDefinition* index,
+ MCreateInlinedArgumentsObject* args);
+ NAMED_OPERANDS((0, index))
+
+ MDefinition* getArg(uint32_t idx) const {
+ return getOperand(idx + NumNonArgumentOperands);
+ }
+ uint32_t numActuals() const { return numOperands() - NumNonArgumentOperands; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+};
+
+class MGetInlinedArgumentHole
+ : public MVariadicInstruction,
+ public MixPolicy<UnboxedInt32Policy<0>, NoFloatPolicyAfter<1>>::Data {
+ MGetInlinedArgumentHole() : MVariadicInstruction(classOpcode) {
+ setGuard();
+ setResultType(MIRType::Value);
+ }
+
+ static const size_t NumNonArgumentOperands = 1;
+
+ public:
+ INSTRUCTION_HEADER(GetInlinedArgumentHole)
+ static MGetInlinedArgumentHole* New(TempAllocator& alloc, MDefinition* index,
+ MCreateInlinedArgumentsObject* args);
+ NAMED_OPERANDS((0, index))
+
+ MDefinition* getArg(uint32_t idx) const {
+ return getOperand(idx + NumNonArgumentOperands);
+ }
+ uint32_t numActuals() const { return numOperands() - NumNonArgumentOperands; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+};
+
+class MInlineArgumentsSlice
+ : public MVariadicInstruction,
+ public MixPolicy<UnboxedInt32Policy<0>, UnboxedInt32Policy<1>,
+ NoFloatPolicyAfter<2>>::Data {
+ JSObject* templateObj_;
+ gc::InitialHeap initialHeap_;
+
+ MInlineArgumentsSlice(JSObject* templateObj, gc::InitialHeap initialHeap)
+ : MVariadicInstruction(classOpcode),
+ templateObj_(templateObj),
+ initialHeap_(initialHeap) {
+ setResultType(MIRType::Object);
+ }
+
+ static const size_t NumNonArgumentOperands = 2;
+
+ public:
+ INSTRUCTION_HEADER(InlineArgumentsSlice)
+ static MInlineArgumentsSlice* New(TempAllocator& alloc, MDefinition* begin,
+ MDefinition* count,
+ MCreateInlinedArgumentsObject* args,
+ JSObject* templateObj,
+ gc::InitialHeap initialHeap);
+ NAMED_OPERANDS((0, begin), (1, count))
+
+ JSObject* templateObj() const { return templateObj_; }
+ gc::InitialHeap initialHeap() const { return initialHeap_; }
+
+ MDefinition* getArg(uint32_t idx) const {
+ return getOperand(idx + NumNonArgumentOperands);
+ }
+ uint32_t numActuals() const { return numOperands() - NumNonArgumentOperands; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+class MToFPInstruction : public MUnaryInstruction, public ToDoublePolicy::Data {
+ public:
+ // Types of values which can be converted.
+ enum ConversionKind { NonStringPrimitives, NumbersOnly };
+
+ private:
+ ConversionKind conversion_;
+
+ protected:
+ MToFPInstruction(Opcode op, MDefinition* def,
+ ConversionKind conversion = NonStringPrimitives)
+ : MUnaryInstruction(op, def), conversion_(conversion) {}
+
+ public:
+ ConversionKind conversion() const { return conversion_; }
+};
+
+// Converts a primitive (either typed or untyped) to a double. If the input is
+// not primitive at runtime, a bailout occurs.
+class MToDouble : public MToFPInstruction {
+ private:
+ TruncateKind implicitTruncate_;
+
+ explicit MToDouble(MDefinition* def,
+ ConversionKind conversion = NonStringPrimitives)
+ : MToFPInstruction(classOpcode, def, conversion),
+ implicitTruncate_(TruncateKind::NoTruncate) {
+ setResultType(MIRType::Double);
+ setMovable();
+
+ // Guard unless the conversion is known to be non-effectful & non-throwing.
+ if (!def->definitelyType({MIRType::Undefined, MIRType::Null,
+ MIRType::Boolean, MIRType::Int32, MIRType::Double,
+ MIRType::Float32, MIRType::String})) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(ToDouble)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isToDouble() || ins->toToDouble()->conversion() != conversion()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ void computeRange(TempAllocator& alloc) override;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+
+ TruncateKind truncateKind() const { return implicitTruncate_; }
+ void setTruncateKind(TruncateKind kind) {
+ implicitTruncate_ = std::max(implicitTruncate_, kind);
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override {
+ if (input()->type() == MIRType::Value) {
+ return false;
+ }
+ if (input()->type() == MIRType::Symbol) {
+ return false;
+ }
+ if (input()->type() == MIRType::BigInt) {
+ return false;
+ }
+
+ return true;
+ }
+
+ ALLOW_CLONE(MToDouble)
+};
+
+// Converts a primitive (either typed or untyped) to a float32. If the input is
+// not primitive at runtime, a bailout occurs.
+class MToFloat32 : public MToFPInstruction {
+ bool mustPreserveNaN_;
+
+ explicit MToFloat32(MDefinition* def,
+ ConversionKind conversion = NonStringPrimitives)
+ : MToFPInstruction(classOpcode, def, conversion),
+ mustPreserveNaN_(false) {
+ setResultType(MIRType::Float32);
+ setMovable();
+
+ // Guard unless the conversion is known to be non-effectful & non-throwing.
+ if (!def->definitelyType({MIRType::Undefined, MIRType::Null,
+ MIRType::Boolean, MIRType::Int32, MIRType::Double,
+ MIRType::Float32, MIRType::String})) {
+ setGuard();
+ }
+ }
+
+ explicit MToFloat32(MDefinition* def, bool mustPreserveNaN)
+ : MToFloat32(def) {
+ mustPreserveNaN_ = mustPreserveNaN;
+ }
+
+ public:
+ INSTRUCTION_HEADER(ToFloat32)
+ TRIVIAL_NEW_WRAPPERS
+
+ virtual MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!congruentIfOperandsEqual(ins)) {
+ return false;
+ }
+ auto* other = ins->toToFloat32();
+ return other->conversion() == conversion() &&
+ other->mustPreserveNaN_ == mustPreserveNaN_;
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ void computeRange(TempAllocator& alloc) override;
+
+ bool canConsumeFloat32(MUse* use) const override { return true; }
+ bool canProduceFloat32() const override { return true; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MToFloat32)
+};
+
+// Converts a uint32 to a float32 (coming from wasm).
+class MWasmUnsignedToFloat32 : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ explicit MWasmUnsignedToFloat32(MDefinition* def)
+ : MUnaryInstruction(classOpcode, def) {
+ setResultType(MIRType::Float32);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmUnsignedToFloat32)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool canProduceFloat32() const override { return true; }
+};
+
+class MWrapInt64ToInt32 : public MUnaryInstruction, public NoTypePolicy::Data {
+ bool bottomHalf_;
+
+ explicit MWrapInt64ToInt32(MDefinition* def, bool bottomHalf = true)
+ : MUnaryInstruction(classOpcode, def), bottomHalf_(bottomHalf) {
+ setResultType(MIRType::Int32);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WrapInt64ToInt32)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isWrapInt64ToInt32()) {
+ return false;
+ }
+ if (ins->toWrapInt64ToInt32()->bottomHalf() != bottomHalf()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool bottomHalf() const { return bottomHalf_; }
+};
+
+class MExtendInt32ToInt64 : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ bool isUnsigned_;
+
+ MExtendInt32ToInt64(MDefinition* def, bool isUnsigned)
+ : MUnaryInstruction(classOpcode, def), isUnsigned_(isUnsigned) {
+ setResultType(MIRType::Int64);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(ExtendInt32ToInt64)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool isUnsigned() const { return isUnsigned_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isExtendInt32ToInt64()) {
+ return false;
+ }
+ if (ins->toExtendInt32ToInt64()->isUnsigned_ != isUnsigned_) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// The same as MWasmTruncateToInt64 but with the Instance dependency.
+// It used only for arm now because on arm we need to call builtin to truncate
+// to i64.
+class MWasmBuiltinTruncateToInt64 : public MAryInstruction<2>,
+ public NoTypePolicy::Data {
+ TruncFlags flags_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MWasmBuiltinTruncateToInt64(MDefinition* def, MDefinition* instance,
+ TruncFlags flags,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MAryInstruction(classOpcode),
+ flags_(flags),
+ bytecodeOffset_(bytecodeOffset) {
+ initOperand(0, def);
+ initOperand(1, instance);
+
+ setResultType(MIRType::Int64);
+ setGuard(); // neither removable nor movable because of possible
+ // side-effects.
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBuiltinTruncateToInt64)
+ NAMED_OPERANDS((0, input), (1, instance));
+ TRIVIAL_NEW_WRAPPERS
+
+ bool isUnsigned() const { return flags_ & TRUNC_UNSIGNED; }
+ bool isSaturating() const { return flags_ & TRUNC_SATURATING; }
+ TruncFlags flags() const { return flags_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toWasmBuiltinTruncateToInt64()->flags() == flags_;
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MWasmTruncateToInt64 : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ TruncFlags flags_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MWasmTruncateToInt64(MDefinition* def, TruncFlags flags,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MUnaryInstruction(classOpcode, def),
+ flags_(flags),
+ bytecodeOffset_(bytecodeOffset) {
+ setResultType(MIRType::Int64);
+ setGuard(); // neither removable nor movable because of possible
+ // side-effects.
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmTruncateToInt64)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool isUnsigned() const { return flags_ & TRUNC_UNSIGNED; }
+ bool isSaturating() const { return flags_ & TRUNC_SATURATING; }
+ TruncFlags flags() const { return flags_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toWasmTruncateToInt64()->flags() == flags_;
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Truncate a value to an int32, with wasm semantics: this will trap when the
+// value is out of range.
+class MWasmTruncateToInt32 : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ TruncFlags flags_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ explicit MWasmTruncateToInt32(MDefinition* def, TruncFlags flags,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MUnaryInstruction(classOpcode, def),
+ flags_(flags),
+ bytecodeOffset_(bytecodeOffset) {
+ setResultType(MIRType::Int32);
+ setGuard(); // neither removable nor movable because of possible
+ // side-effects.
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmTruncateToInt32)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool isUnsigned() const { return flags_ & TRUNC_UNSIGNED; }
+ bool isSaturating() const { return flags_ & TRUNC_SATURATING; }
+ TruncFlags flags() const { return flags_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toWasmTruncateToInt32()->flags() == flags_;
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Converts an int32 value to intptr by sign-extending it.
+class MInt32ToIntPtr : public MUnaryInstruction,
+ public UnboxedInt32Policy<0>::Data {
+ bool canBeNegative_ = true;
+
+ explicit MInt32ToIntPtr(MDefinition* def)
+ : MUnaryInstruction(classOpcode, def) {
+ setResultType(MIRType::IntPtr);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Int32ToIntPtr)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool canBeNegative() const { return canBeNegative_; }
+ void setCanNotBeNegative() { canBeNegative_ = false; }
+
+ void computeRange(TempAllocator& alloc) override;
+ void collectRangeInfoPreTrunc() override;
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Converts an IntPtr value >= 0 to Int32. Bails out if the value > INT32_MAX.
+class MNonNegativeIntPtrToInt32 : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ explicit MNonNegativeIntPtrToInt32(MDefinition* def)
+ : MUnaryInstruction(classOpcode, def) {
+ MOZ_ASSERT(def->type() == MIRType::IntPtr);
+ setResultType(MIRType::Int32);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(NonNegativeIntPtrToInt32)
+ TRIVIAL_NEW_WRAPPERS
+
+ void computeRange(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Converts an IntPtr value to Double.
+class MIntPtrToDouble : public MUnaryInstruction, public NoTypePolicy::Data {
+ explicit MIntPtrToDouble(MDefinition* def)
+ : MUnaryInstruction(classOpcode, def) {
+ MOZ_ASSERT(def->type() == MIRType::IntPtr);
+ setResultType(MIRType::Double);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(IntPtrToDouble)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Subtracts (byteSize - 1) from the input value. Bails out if the result is
+// negative. This is used to implement bounds checks for DataView accesses.
+class MAdjustDataViewLength : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ const uint32_t byteSize_;
+
+ MAdjustDataViewLength(MDefinition* input, uint32_t byteSize)
+ : MUnaryInstruction(classOpcode, input), byteSize_(byteSize) {
+ MOZ_ASSERT(input->type() == MIRType::IntPtr);
+ MOZ_ASSERT(byteSize > 1);
+ setResultType(MIRType::IntPtr);
+ setMovable();
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(AdjustDataViewLength)
+ TRIVIAL_NEW_WRAPPERS
+
+ uint32_t byteSize() const { return byteSize_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isAdjustDataViewLength()) {
+ return false;
+ }
+ if (ins->toAdjustDataViewLength()->byteSize() != byteSize()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MInt64ToFloatingPoint : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ bool isUnsigned_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MInt64ToFloatingPoint(MDefinition* def, MIRType type,
+ wasm::BytecodeOffset bytecodeOffset, bool isUnsigned)
+ : MUnaryInstruction(classOpcode, def),
+ isUnsigned_(isUnsigned),
+ bytecodeOffset_(bytecodeOffset) {
+ MOZ_ASSERT(IsFloatingPointType(type));
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Int64ToFloatingPoint)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool isUnsigned() const { return isUnsigned_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isInt64ToFloatingPoint()) {
+ return false;
+ }
+ if (ins->toInt64ToFloatingPoint()->isUnsigned_ != isUnsigned_) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// It used only for arm now because on arm we need to call builtin to convert
+// i64 to float.
+class MBuiltinInt64ToFloatingPoint : public MAryInstruction<2>,
+ public NoTypePolicy::Data {
+ bool isUnsigned_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MBuiltinInt64ToFloatingPoint(MDefinition* def, MDefinition* instance,
+ MIRType type,
+ wasm::BytecodeOffset bytecodeOffset,
+ bool isUnsigned)
+ : MAryInstruction(classOpcode),
+ isUnsigned_(isUnsigned),
+ bytecodeOffset_(bytecodeOffset) {
+ MOZ_ASSERT(IsFloatingPointType(type));
+ initOperand(0, def);
+ initOperand(1, instance);
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(BuiltinInt64ToFloatingPoint)
+ NAMED_OPERANDS((0, input), (1, instance));
+ TRIVIAL_NEW_WRAPPERS
+
+ bool isUnsigned() const { return isUnsigned_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isBuiltinInt64ToFloatingPoint()) {
+ return false;
+ }
+ if (ins->toBuiltinInt64ToFloatingPoint()->isUnsigned_ != isUnsigned_) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Applies ECMA's ToNumber on a primitive (either typed or untyped) and expects
+// the result to be precisely representable as an Int32, otherwise bails.
+//
+// If the input is not primitive at runtime, a bailout occurs. If the input
+// cannot be converted to an int32 without loss (i.e. 5.5 or undefined) then a
+// bailout occurs.
+class MToNumberInt32 : public MUnaryInstruction, public ToInt32Policy::Data {
+ bool needsNegativeZeroCheck_;
+ IntConversionInputKind conversion_;
+
+ explicit MToNumberInt32(MDefinition* def, IntConversionInputKind conversion =
+ IntConversionInputKind::Any)
+ : MUnaryInstruction(classOpcode, def),
+ needsNegativeZeroCheck_(true),
+ conversion_(conversion) {
+ setResultType(MIRType::Int32);
+ setMovable();
+
+ // Guard unless the conversion is known to be non-effectful & non-throwing.
+ if (!def->definitelyType({MIRType::Undefined, MIRType::Null,
+ MIRType::Boolean, MIRType::Int32, MIRType::Double,
+ MIRType::Float32, MIRType::String})) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(ToNumberInt32)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ // this only has backwards information flow.
+ void analyzeEdgeCasesBackward() override;
+
+ bool needsNegativeZeroCheck() const { return needsNegativeZeroCheck_; }
+ void setNeedsNegativeZeroCheck(bool needsCheck) {
+ needsNegativeZeroCheck_ = needsCheck;
+ }
+
+ IntConversionInputKind conversion() const { return conversion_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isToNumberInt32() ||
+ ins->toToNumberInt32()->conversion() != conversion()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+ void collectRangeInfoPreTrunc() override;
+
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+
+ ALLOW_CLONE(MToNumberInt32)
+};
+
+// Applies ECMA's ToInteger on a primitive (either typed or untyped) and expects
+// the result to be precisely representable as an Int32, otherwise bails.
+//
+// NB: Negative zero doesn't lead to a bailout, but instead will be treated the
+// same as positive zero for this operation.
+//
+// If the input is not primitive at runtime, a bailout occurs. If the input
+// cannot be converted to an int32 without loss (i.e. 2e10 or Infinity) then a
+// bailout occurs.
+class MToIntegerInt32 : public MUnaryInstruction, public ToInt32Policy::Data {
+ explicit MToIntegerInt32(MDefinition* def)
+ : MUnaryInstruction(classOpcode, def) {
+ setResultType(MIRType::Int32);
+ setMovable();
+
+ // Guard unless the conversion is known to be non-effectful & non-throwing.
+ if (!def->definitelyType({MIRType::Undefined, MIRType::Null,
+ MIRType::Boolean, MIRType::Int32, MIRType::Double,
+ MIRType::Float32, MIRType::String})) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(ToIntegerInt32)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+
+ ALLOW_CLONE(MToIntegerInt32)
+};
+
+// Converts a value or typed input to a truncated int32, for use with bitwise
+// operations. This is an infallible ValueToECMAInt32.
+class MTruncateToInt32 : public MUnaryInstruction, public ToInt32Policy::Data {
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ explicit MTruncateToInt32(
+ MDefinition* def,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset())
+ : MUnaryInstruction(classOpcode, def), bytecodeOffset_(bytecodeOffset) {
+ setResultType(MIRType::Int32);
+ setMovable();
+
+ // Guard unless the conversion is known to be non-effectful & non-throwing.
+ if (mightHaveSideEffects(def)) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(TruncateToInt32)
+ TRIVIAL_NEW_WRAPPERS
+
+ static bool mightHaveSideEffects(MDefinition* def) {
+ return !def->definitelyType(
+ {MIRType::Undefined, MIRType::Null, MIRType::Boolean, MIRType::Int32,
+ MIRType::Double, MIRType::Float32, MIRType::String});
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ void computeRange(TempAllocator& alloc) override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override {
+ return input()->type() < MIRType::Symbol;
+ }
+
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ ALLOW_CLONE(MTruncateToInt32)
+};
+
+// It is like MTruncateToInt32 but with instance dependency.
+class MWasmBuiltinTruncateToInt32 : public MAryInstruction<2>,
+ public ToInt32Policy::Data {
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MWasmBuiltinTruncateToInt32(
+ MDefinition* def, MDefinition* instance,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset())
+ : MAryInstruction(classOpcode), bytecodeOffset_(bytecodeOffset) {
+ initOperand(0, def);
+ initOperand(1, instance);
+ setResultType(MIRType::Int32);
+ setMovable();
+
+ // Guard unless the conversion is known to be non-effectful & non-throwing.
+ if (MTruncateToInt32::mightHaveSideEffects(def)) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBuiltinTruncateToInt32)
+ NAMED_OPERANDS((0, input), (1, instance))
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ ALLOW_CLONE(MWasmBuiltinTruncateToInt32)
+};
+
+// Converts a primitive (either typed or untyped) to a BigInt. If the input is
+// not primitive at runtime, a bailout occurs.
+class MToBigInt : public MUnaryInstruction, public ToBigIntPolicy::Data {
+ private:
+ explicit MToBigInt(MDefinition* def) : MUnaryInstruction(classOpcode, def) {
+ setResultType(MIRType::BigInt);
+ setMovable();
+
+ // Guard unless the conversion is known to be non-effectful & non-throwing.
+ if (!def->definitelyType({MIRType::Boolean, MIRType::BigInt})) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(ToBigInt)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ ALLOW_CLONE(MToBigInt)
+};
+
+// Takes a Value or typed input and returns a suitable Int64 using the
+// ToBigInt algorithm, possibly calling out to the VM for string, etc inputs.
+class MToInt64 : public MUnaryInstruction, public ToInt64Policy::Data {
+ explicit MToInt64(MDefinition* def) : MUnaryInstruction(classOpcode, def) {
+ setResultType(MIRType::Int64);
+ setMovable();
+
+ // Guard unless the conversion is known to be non-effectful & non-throwing.
+ if (!def->definitelyType(
+ {MIRType::Boolean, MIRType::BigInt, MIRType::Int64})) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(ToInt64)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ ALLOW_CLONE(MToInt64)
+};
+
+// Takes a BigInt pointer and returns its toInt64 value.
+class MTruncateBigIntToInt64 : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ explicit MTruncateBigIntToInt64(MDefinition* def)
+ : MUnaryInstruction(classOpcode, def) {
+ MOZ_ASSERT(def->type() == MIRType::BigInt);
+ setResultType(MIRType::Int64);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(TruncateBigIntToInt64)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ ALLOW_CLONE(MTruncateBigIntToInt64)
+};
+
+// Takes an Int64 and returns a fresh BigInt pointer.
+class MInt64ToBigInt : public MUnaryInstruction, public NoTypePolicy::Data {
+ explicit MInt64ToBigInt(MDefinition* def)
+ : MUnaryInstruction(classOpcode, def) {
+ MOZ_ASSERT(def->type() == MIRType::Int64);
+ setResultType(MIRType::BigInt);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Int64ToBigInt)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ ALLOW_CLONE(MInt64ToBigInt)
+};
+
+// Converts any type to a string
+class MToString : public MUnaryInstruction, public ToStringPolicy::Data {
+ public:
+ // MToString has two modes for handling of object/symbol arguments: if the
+ // to-string conversion happens as part of another opcode, we have to bail out
+ // to Baseline. If the conversion is for a stand-alone JSOp we can support
+ // side-effects.
+ enum class SideEffectHandling { Bailout, Supported };
+
+ private:
+ SideEffectHandling sideEffects_;
+ bool mightHaveSideEffects_ = false;
+
+ MToString(MDefinition* def, SideEffectHandling sideEffects)
+ : MUnaryInstruction(classOpcode, def), sideEffects_(sideEffects) {
+ setResultType(MIRType::String);
+
+ if (!def->definitelyType({MIRType::Undefined, MIRType::Null,
+ MIRType::Boolean, MIRType::Int32, MIRType::Double,
+ MIRType::Float32, MIRType::String,
+ MIRType::BigInt})) {
+ mightHaveSideEffects_ = true;
+ }
+
+ // If this instruction is not effectful, mark it as movable and set the
+ // Guard flag if needed. If the operation is effectful it won't be
+ // optimized anyway so there's no need to set any flags.
+ if (!isEffectful()) {
+ setMovable();
+ // Objects might override toString; Symbol throws. We bailout in those
+ // cases and run side-effects in baseline instead.
+ if (mightHaveSideEffects_) {
+ setGuard();
+ }
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(ToString)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isToString()) {
+ return false;
+ }
+ if (sideEffects_ != ins->toToString()->sideEffects_) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override {
+ if (supportSideEffects() && mightHaveSideEffects_) {
+ return AliasSet::Store(AliasSet::Any);
+ }
+ return AliasSet::None();
+ }
+
+ bool mightHaveSideEffects() const { return mightHaveSideEffects_; }
+
+ bool supportSideEffects() const {
+ return sideEffects_ == SideEffectHandling::Supported;
+ }
+
+ bool needsSnapshot() const {
+ return sideEffects_ == SideEffectHandling::Bailout && mightHaveSideEffects_;
+ }
+
+ ALLOW_CLONE(MToString)
+};
+
+class MBitNot : public MUnaryInstruction, public BitwisePolicy::Data {
+ explicit MBitNot(MDefinition* input) : MUnaryInstruction(classOpcode, input) {
+ setResultType(MIRType::Int32);
+ if (input->type() == MIRType::Int64) {
+ setResultType(MIRType::Int64);
+ }
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(BitNot)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBitNot)
+};
+
+class MTypeOf : public MUnaryInstruction,
+ public BoxExceptPolicy<0, MIRType::Object>::Data {
+ explicit MTypeOf(MDefinition* def) : MUnaryInstruction(classOpcode, def) {
+ setResultType(MIRType::Int32);
+ setMovable();
+ }
+ TypeDataList observed_;
+
+ public:
+ INSTRUCTION_HEADER(TypeOf)
+ TRIVIAL_NEW_WRAPPERS
+
+ void setObservedTypes(const TypeDataList& observed) { observed_ = observed; }
+ bool hasObservedTypes() const { return observed_.count() > 0; }
+ const TypeDataList& observedTypes() const { return observed_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+class MTypeOfIs : public MUnaryInstruction, public NoTypePolicy::Data {
+ JSOp jsop_;
+ JSType jstype_;
+
+ MTypeOfIs(MDefinition* def, JSOp jsop, JSType jstype)
+ : MUnaryInstruction(classOpcode, def), jsop_(jsop), jstype_(jstype) {
+ MOZ_ASSERT(def->type() == MIRType::Object || def->type() == MIRType::Value);
+
+ setResultType(MIRType::Boolean);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(TypeOfIs)
+ TRIVIAL_NEW_WRAPPERS
+
+ JSOp jsop() const { return jsop_; }
+ JSType jstype() const { return jstype_; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!congruentIfOperandsEqual(ins)) {
+ return false;
+ }
+ return jsop() == ins->toTypeOfIs()->jsop() &&
+ jstype() == ins->toTypeOfIs()->jstype();
+ }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+};
+
+class MBinaryBitwiseInstruction : public MBinaryInstruction,
+ public BitwisePolicy::Data {
+ protected:
+ MBinaryBitwiseInstruction(Opcode op, MDefinition* left, MDefinition* right,
+ MIRType type)
+ : MBinaryInstruction(op, left, right),
+ maskMatchesLeftRange(false),
+ maskMatchesRightRange(false) {
+ MOZ_ASSERT(type == MIRType::Int32 || type == MIRType::Int64 ||
+ (isUrsh() && type == MIRType::Double));
+ setResultType(type);
+ setMovable();
+ }
+
+ bool maskMatchesLeftRange;
+ bool maskMatchesRightRange;
+
+ public:
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ MDefinition* foldUnnecessaryBitop();
+ virtual MDefinition* foldIfZero(size_t operand) = 0;
+ virtual MDefinition* foldIfNegOne(size_t operand) = 0;
+ virtual MDefinition* foldIfEqual() = 0;
+ virtual MDefinition* foldIfAllBitsSet(size_t operand) = 0;
+ void collectRangeInfoPreTrunc() override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return binaryCongruentTo(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ TruncateKind operandTruncateKind(size_t index) const override;
+};
+
+class MBitAnd : public MBinaryBitwiseInstruction {
+ MBitAnd(MDefinition* left, MDefinition* right, MIRType type)
+ : MBinaryBitwiseInstruction(classOpcode, left, right, type) {
+ setCommutative();
+ }
+
+ public:
+ INSTRUCTION_HEADER(BitAnd)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldIfZero(size_t operand) override {
+ return getOperand(operand); // 0 & x => 0;
+ }
+ MDefinition* foldIfNegOne(size_t operand) override {
+ return getOperand(1 - operand); // x & -1 => x
+ }
+ MDefinition* foldIfEqual() override {
+ return getOperand(0); // x & x => x;
+ }
+ MDefinition* foldIfAllBitsSet(size_t operand) override {
+ // e.g. for uint16: x & 0xffff => x;
+ return getOperand(1 - operand);
+ }
+ void computeRange(TempAllocator& alloc) override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBitAnd)
+};
+
+class MBitOr : public MBinaryBitwiseInstruction {
+ MBitOr(MDefinition* left, MDefinition* right, MIRType type)
+ : MBinaryBitwiseInstruction(classOpcode, left, right, type) {
+ setCommutative();
+ }
+
+ public:
+ INSTRUCTION_HEADER(BitOr)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldIfZero(size_t operand) override {
+ return getOperand(1 -
+ operand); // 0 | x => x, so if ith is 0, return (1-i)th
+ }
+ MDefinition* foldIfNegOne(size_t operand) override {
+ return getOperand(operand); // x | -1 => -1
+ }
+ MDefinition* foldIfEqual() override {
+ return getOperand(0); // x | x => x
+ }
+ MDefinition* foldIfAllBitsSet(size_t operand) override { return this; }
+ void computeRange(TempAllocator& alloc) override;
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBitOr)
+};
+
+class MBitXor : public MBinaryBitwiseInstruction {
+ MBitXor(MDefinition* left, MDefinition* right, MIRType type)
+ : MBinaryBitwiseInstruction(classOpcode, left, right, type) {
+ setCommutative();
+ }
+
+ public:
+ INSTRUCTION_HEADER(BitXor)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldIfZero(size_t operand) override {
+ return getOperand(1 - operand); // 0 ^ x => x
+ }
+ MDefinition* foldIfNegOne(size_t operand) override { return this; }
+ MDefinition* foldIfEqual() override { return this; }
+ MDefinition* foldIfAllBitsSet(size_t operand) override { return this; }
+ void computeRange(TempAllocator& alloc) override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBitXor)
+};
+
+class MShiftInstruction : public MBinaryBitwiseInstruction {
+ protected:
+ MShiftInstruction(Opcode op, MDefinition* left, MDefinition* right,
+ MIRType type)
+ : MBinaryBitwiseInstruction(op, left, right, type) {}
+
+ public:
+ MDefinition* foldIfNegOne(size_t operand) override { return this; }
+ MDefinition* foldIfEqual() override { return this; }
+ MDefinition* foldIfAllBitsSet(size_t operand) override { return this; }
+};
+
+class MLsh : public MShiftInstruction {
+ MLsh(MDefinition* left, MDefinition* right, MIRType type)
+ : MShiftInstruction(classOpcode, left, right, type) {}
+
+ public:
+ INSTRUCTION_HEADER(Lsh)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldIfZero(size_t operand) override {
+ // 0 << x => 0
+ // x << 0 => x
+ return getOperand(0);
+ }
+
+ void computeRange(TempAllocator& alloc) override;
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MLsh)
+};
+
+class MRsh : public MShiftInstruction {
+ MRsh(MDefinition* left, MDefinition* right, MIRType type)
+ : MShiftInstruction(classOpcode, left, right, type) {}
+
+ public:
+ INSTRUCTION_HEADER(Rsh)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldIfZero(size_t operand) override {
+ // 0 >> x => 0
+ // x >> 0 => x
+ return getOperand(0);
+ }
+ void computeRange(TempAllocator& alloc) override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ ALLOW_CLONE(MRsh)
+};
+
+class MUrsh : public MShiftInstruction {
+ bool bailoutsDisabled_;
+
+ MUrsh(MDefinition* left, MDefinition* right, MIRType type)
+ : MShiftInstruction(classOpcode, left, right, type),
+ bailoutsDisabled_(false) {}
+
+ public:
+ INSTRUCTION_HEADER(Ursh)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MUrsh* NewWasm(TempAllocator& alloc, MDefinition* left,
+ MDefinition* right, MIRType type);
+
+ MDefinition* foldIfZero(size_t operand) override {
+ // 0 >>> x => 0
+ if (operand == 0) {
+ return getOperand(0);
+ }
+
+ return this;
+ }
+
+ bool bailoutsDisabled() const { return bailoutsDisabled_; }
+
+ bool fallible() const;
+
+ void computeRange(TempAllocator& alloc) override;
+ void collectRangeInfoPreTrunc() override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MUrsh)
+};
+
+class MSignExtendInt32 : public MUnaryInstruction, public NoTypePolicy::Data {
+ public:
+ enum Mode { Byte, Half };
+
+ private:
+ Mode mode_;
+
+ MSignExtendInt32(MDefinition* op, Mode mode)
+ : MUnaryInstruction(classOpcode, op), mode_(mode) {
+ setResultType(MIRType::Int32);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(SignExtendInt32)
+ TRIVIAL_NEW_WRAPPERS
+
+ Mode mode() const { return mode_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!congruentIfOperandsEqual(ins)) {
+ return false;
+ }
+ return ins->isSignExtendInt32() && ins->toSignExtendInt32()->mode_ == mode_;
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MSignExtendInt32)
+};
+
+class MSignExtendInt64 : public MUnaryInstruction, public NoTypePolicy::Data {
+ public:
+ enum Mode { Byte, Half, Word };
+
+ private:
+ Mode mode_;
+
+ MSignExtendInt64(MDefinition* op, Mode mode)
+ : MUnaryInstruction(classOpcode, op), mode_(mode) {
+ setResultType(MIRType::Int64);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(SignExtendInt64)
+ TRIVIAL_NEW_WRAPPERS
+
+ Mode mode() const { return mode_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!congruentIfOperandsEqual(ins)) {
+ return false;
+ }
+ return ins->isSignExtendInt64() && ins->toSignExtendInt64()->mode_ == mode_;
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ ALLOW_CLONE(MSignExtendInt64)
+};
+
+class MBinaryArithInstruction : public MBinaryInstruction,
+ public ArithPolicy::Data {
+ // Implicit truncate flag is set by the truncate backward range analysis
+ // optimization phase, and by wasm pre-processing. It is used in
+ // NeedNegativeZeroCheck to check if the result of a multiplication needs to
+ // produce -0 double value, and for avoiding overflow checks.
+
+ // This optimization happens when the multiplication cannot be truncated
+ // even if all uses are truncating its result, such as when the range
+ // analysis detect a precision loss in the multiplication.
+ TruncateKind implicitTruncate_;
+
+ // Whether we must preserve NaN semantics, and in particular not fold
+ // (x op id) or (id op x) to x, or replace a division by a multiply of the
+ // exact reciprocal.
+ bool mustPreserveNaN_;
+
+ protected:
+ MBinaryArithInstruction(Opcode op, MDefinition* left, MDefinition* right,
+ MIRType type)
+ : MBinaryInstruction(op, left, right),
+ implicitTruncate_(TruncateKind::NoTruncate),
+ mustPreserveNaN_(false) {
+ MOZ_ASSERT(IsNumberType(type));
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ void setMustPreserveNaN(bool b) { mustPreserveNaN_ = b; }
+ bool mustPreserveNaN() const { return mustPreserveNaN_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ virtual double getIdentity() = 0;
+
+ void setSpecialization(MIRType type) {
+ MOZ_ASSERT(IsNumberType(type));
+ setResultType(type);
+ }
+
+ virtual void trySpecializeFloat32(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!binaryCongruentTo(ins)) {
+ return false;
+ }
+ const auto* other = static_cast<const MBinaryArithInstruction*>(ins);
+ return other->mustPreserveNaN_ == mustPreserveNaN_;
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool isTruncated() const {
+ return implicitTruncate_ == TruncateKind::Truncate;
+ }
+ TruncateKind truncateKind() const { return implicitTruncate_; }
+ void setTruncateKind(TruncateKind kind) {
+ implicitTruncate_ = std::max(implicitTruncate_, kind);
+ }
+};
+
+class MMinMax : public MBinaryInstruction, public ArithPolicy::Data {
+ bool isMax_;
+
+ MMinMax(MDefinition* left, MDefinition* right, MIRType type, bool isMax)
+ : MBinaryInstruction(classOpcode, left, right), isMax_(isMax) {
+ MOZ_ASSERT(IsNumberType(type));
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(MinMax)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MMinMax* NewWasm(TempAllocator& alloc, MDefinition* left,
+ MDefinition* right, MIRType type, bool isMax) {
+ return New(alloc, left, right, type, isMax);
+ }
+
+ bool isMax() const { return isMax_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!congruentIfOperandsEqual(ins)) {
+ return false;
+ }
+ const MMinMax* other = ins->toMinMax();
+ return other->isMax() == isMax();
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ void computeRange(TempAllocator& alloc) override;
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ bool isFloat32Commutative() const override { return true; }
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+
+ ALLOW_CLONE(MMinMax)
+};
+
+class MMinMaxArray : public MUnaryInstruction, public SingleObjectPolicy::Data {
+ bool isMax_;
+
+ MMinMaxArray(MDefinition* array, MIRType type, bool isMax)
+ : MUnaryInstruction(classOpcode, array), isMax_(isMax) {
+ MOZ_ASSERT(type == MIRType::Int32 || type == MIRType::Double);
+ setResultType(type);
+
+ // We can't DCE this, even if the result is unused, in case one of the
+ // elements of the array is an object with a `valueOf` function that
+ // must be called.
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(MinMaxArray)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, array))
+
+ bool isMax() const { return isMax_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isMinMaxArray() || ins->toMinMaxArray()->isMax() != isMax()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::Element);
+ }
+};
+
+class MAbs : public MUnaryInstruction, public ArithPolicy::Data {
+ bool implicitTruncate_;
+
+ MAbs(MDefinition* num, MIRType type)
+ : MUnaryInstruction(classOpcode, num), implicitTruncate_(false) {
+ MOZ_ASSERT(IsNumberType(type));
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Abs)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MAbs* NewWasm(TempAllocator& alloc, MDefinition* num, MIRType type) {
+ auto* ins = new (alloc) MAbs(num, type);
+ if (type == MIRType::Int32) {
+ ins->implicitTruncate_ = true;
+ }
+ return ins;
+ }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ bool fallible() const;
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+ bool isFloat32Commutative() const override { return true; }
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MAbs)
+};
+
+class MClz : public MUnaryInstruction, public BitwisePolicy::Data {
+ bool operandIsNeverZero_;
+
+ explicit MClz(MDefinition* num, MIRType type)
+ : MUnaryInstruction(classOpcode, num), operandIsNeverZero_(false) {
+ MOZ_ASSERT(IsIntType(type));
+ MOZ_ASSERT(IsNumberType(num->type()));
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Clz)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, num))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool operandIsNeverZero() const { return operandIsNeverZero_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ void computeRange(TempAllocator& alloc) override;
+ void collectRangeInfoPreTrunc() override;
+};
+
+class MCtz : public MUnaryInstruction, public BitwisePolicy::Data {
+ bool operandIsNeverZero_;
+
+ explicit MCtz(MDefinition* num, MIRType type)
+ : MUnaryInstruction(classOpcode, num), operandIsNeverZero_(false) {
+ MOZ_ASSERT(IsIntType(type));
+ MOZ_ASSERT(IsNumberType(num->type()));
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Ctz)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, num))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool operandIsNeverZero() const { return operandIsNeverZero_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ void computeRange(TempAllocator& alloc) override;
+ void collectRangeInfoPreTrunc() override;
+};
+
+class MPopcnt : public MUnaryInstruction, public BitwisePolicy::Data {
+ explicit MPopcnt(MDefinition* num, MIRType type)
+ : MUnaryInstruction(classOpcode, num) {
+ MOZ_ASSERT(IsNumberType(num->type()));
+ MOZ_ASSERT(IsIntType(type));
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Popcnt)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, num))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ void computeRange(TempAllocator& alloc) override;
+};
+
+// Inline implementation of Math.sqrt().
+class MSqrt : public MUnaryInstruction, public FloatingPointPolicy<0>::Data {
+ MSqrt(MDefinition* num, MIRType type) : MUnaryInstruction(classOpcode, num) {
+ setResultType(type);
+ specialization_ = type;
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Sqrt)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+
+ bool isFloat32Commutative() const override { return true; }
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MSqrt)
+};
+
+class MCopySign : public MBinaryInstruction, public NoTypePolicy::Data {
+ MCopySign(MDefinition* lhs, MDefinition* rhs, MIRType type)
+ : MBinaryInstruction(classOpcode, lhs, rhs) {
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(CopySign)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ ALLOW_CLONE(MCopySign)
+};
+
+// Inline implementation of Math.hypot().
+class MHypot : public MVariadicInstruction, public AllDoublePolicy::Data {
+ MHypot() : MVariadicInstruction(classOpcode) {
+ setResultType(MIRType::Double);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Hypot)
+ static MHypot* New(TempAllocator& alloc, const MDefinitionVector& vector);
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool possiblyCalls() const override { return true; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ bool canClone() const override { return true; }
+
+ MInstruction* clone(TempAllocator& alloc,
+ const MDefinitionVector& inputs) const override {
+ return MHypot::New(alloc, inputs);
+ }
+};
+
+// Inline implementation of Math.pow().
+//
+// Supports the following three specializations:
+//
+// 1. MPow(FloatingPoint, FloatingPoint) -> Double
+// - The most general mode, calls js::ecmaPow.
+// - Never performs a bailout.
+// 2. MPow(FloatingPoint, Int32) -> Double
+// - Optimization to call js::powi instead of js::ecmaPow.
+// - Never performs a bailout.
+// 3. MPow(Int32, Int32) -> Int32
+// - Performs the complete exponentiation operation in assembly code.
+// - Bails out if the result doesn't fit in Int32.
+class MPow : public MBinaryInstruction, public PowPolicy::Data {
+ // If true, the result is guaranteed to never be negative zero, as long as the
+ // power is a positive number.
+ bool canBeNegativeZero_;
+
+ MPow(MDefinition* input, MDefinition* power, MIRType specialization)
+ : MBinaryInstruction(classOpcode, input, power) {
+ MOZ_ASSERT(specialization == MIRType::Int32 ||
+ specialization == MIRType::Double);
+ setResultType(specialization);
+ setMovable();
+
+ // The result can't be negative zero if the base is an Int32 value.
+ canBeNegativeZero_ = input->type() != MIRType::Int32;
+ }
+
+ // Helpers for `foldsTo`
+ MDefinition* foldsConstant(TempAllocator& alloc);
+ MDefinition* foldsConstantPower(TempAllocator& alloc);
+
+ bool canBeNegativeZero() const { return canBeNegativeZero_; }
+
+ public:
+ INSTRUCTION_HEADER(Pow)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* input() const { return lhs(); }
+ MDefinition* power() const { return rhs(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool possiblyCalls() const override { return type() != MIRType::Int32; }
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ ALLOW_CLONE(MPow)
+};
+
+// Inline implementation of Math.pow(x, 0.5), which subtly differs from
+// Math.sqrt(x).
+class MPowHalf : public MUnaryInstruction, public DoublePolicy<0>::Data {
+ bool operandIsNeverNegativeInfinity_;
+ bool operandIsNeverNegativeZero_;
+ bool operandIsNeverNaN_;
+
+ explicit MPowHalf(MDefinition* input)
+ : MUnaryInstruction(classOpcode, input),
+ operandIsNeverNegativeInfinity_(false),
+ operandIsNeverNegativeZero_(false),
+ operandIsNeverNaN_(false) {
+ setResultType(MIRType::Double);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(PowHalf)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ bool operandIsNeverNegativeInfinity() const {
+ return operandIsNeverNegativeInfinity_;
+ }
+ bool operandIsNeverNegativeZero() const {
+ return operandIsNeverNegativeZero_;
+ }
+ bool operandIsNeverNaN() const { return operandIsNeverNaN_; }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void collectRangeInfoPreTrunc() override;
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MPowHalf)
+};
+
+class MSign : public MUnaryInstruction, public SignPolicy::Data {
+ private:
+ MSign(MDefinition* input, MIRType resultType)
+ : MUnaryInstruction(classOpcode, input) {
+ MOZ_ASSERT(IsNumberType(input->type()));
+ MOZ_ASSERT(resultType == MIRType::Int32 || resultType == MIRType::Double);
+ specialization_ = input->type();
+ setResultType(resultType);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Sign)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ void computeRange(TempAllocator& alloc) override;
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MSign)
+};
+
+class MMathFunction : public MUnaryInstruction,
+ public FloatingPointPolicy<0>::Data {
+ UnaryMathFunction function_;
+
+ // A nullptr cache means this function will neither access nor update the
+ // cache.
+ MMathFunction(MDefinition* input, UnaryMathFunction function)
+ : MUnaryInstruction(classOpcode, input), function_(function) {
+ setResultType(MIRType::Double);
+ specialization_ = MIRType::Double;
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(MathFunction)
+ TRIVIAL_NEW_WRAPPERS
+
+ UnaryMathFunction function() const { return function_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isMathFunction()) {
+ return false;
+ }
+ if (ins->toMathFunction()->function() != function()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool possiblyCalls() const override { return true; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ static const char* FunctionName(UnaryMathFunction function);
+
+ bool isFloat32Commutative() const override;
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+
+ void computeRange(TempAllocator& alloc) override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MMathFunction)
+};
+
+class MAdd : public MBinaryArithInstruction {
+ MAdd(MDefinition* left, MDefinition* right, MIRType type)
+ : MBinaryArithInstruction(classOpcode, left, right, type) {
+ setCommutative();
+ }
+
+ MAdd(MDefinition* left, MDefinition* right, TruncateKind truncateKind)
+ : MAdd(left, right, MIRType::Int32) {
+ setTruncateKind(truncateKind);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Add)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MAdd* NewWasm(TempAllocator& alloc, MDefinition* left,
+ MDefinition* right, MIRType type) {
+ auto* ret = new (alloc) MAdd(left, right, type);
+ if (type == MIRType::Int32) {
+ ret->setTruncateKind(TruncateKind::Truncate);
+ }
+ return ret;
+ }
+
+ bool isFloat32Commutative() const override { return true; }
+
+ double getIdentity() override { return 0; }
+
+ bool fallible() const;
+ void computeRange(TempAllocator& alloc) override;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MAdd)
+};
+
+class MSub : public MBinaryArithInstruction {
+ MSub(MDefinition* left, MDefinition* right, MIRType type)
+ : MBinaryArithInstruction(classOpcode, left, right, type) {}
+
+ public:
+ INSTRUCTION_HEADER(Sub)
+ TRIVIAL_NEW_WRAPPERS
+
+ static MSub* NewWasm(TempAllocator& alloc, MDefinition* left,
+ MDefinition* right, MIRType type, bool mustPreserveNaN) {
+ auto* ret = new (alloc) MSub(left, right, type);
+ ret->setMustPreserveNaN(mustPreserveNaN);
+ if (type == MIRType::Int32) {
+ ret->setTruncateKind(TruncateKind::Truncate);
+ }
+ return ret;
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ double getIdentity() override { return 0; }
+
+ bool isFloat32Commutative() const override { return true; }
+
+ bool fallible() const;
+ void computeRange(TempAllocator& alloc) override;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MSub)
+};
+
+class MMul : public MBinaryArithInstruction {
+ public:
+ enum Mode { Normal, Integer };
+
+ private:
+ // Annotation the result could be a negative zero
+ // and we need to guard this during execution.
+ bool canBeNegativeZero_;
+
+ Mode mode_;
+
+ MMul(MDefinition* left, MDefinition* right, MIRType type, Mode mode)
+ : MBinaryArithInstruction(classOpcode, left, right, type),
+ canBeNegativeZero_(true),
+ mode_(mode) {
+ setCommutative();
+ if (mode == Integer) {
+ // This implements the required behavior for Math.imul, which
+ // can never fail and always truncates its output to int32.
+ canBeNegativeZero_ = false;
+ setTruncateKind(TruncateKind::Truncate);
+ }
+ MOZ_ASSERT_IF(mode != Integer, mode == Normal);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Mul)
+
+ static MMul* New(TempAllocator& alloc, MDefinition* left, MDefinition* right,
+ MIRType type, Mode mode = Normal) {
+ return new (alloc) MMul(left, right, type, mode);
+ }
+ static MMul* NewWasm(TempAllocator& alloc, MDefinition* left,
+ MDefinition* right, MIRType type, Mode mode,
+ bool mustPreserveNaN) {
+ auto* ret = new (alloc) MMul(left, right, type, mode);
+ ret->setMustPreserveNaN(mustPreserveNaN);
+ return ret;
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ void analyzeEdgeCasesForward() override;
+ void analyzeEdgeCasesBackward() override;
+ void collectRangeInfoPreTrunc() override;
+
+ double getIdentity() override { return 1; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isMul()) {
+ return false;
+ }
+
+ const MMul* mul = ins->toMul();
+ if (canBeNegativeZero_ != mul->canBeNegativeZero()) {
+ return false;
+ }
+
+ if (mode_ != mul->mode()) {
+ return false;
+ }
+
+ if (mustPreserveNaN() != mul->mustPreserveNaN()) {
+ return false;
+ }
+
+ return binaryCongruentTo(ins);
+ }
+
+ bool canOverflow() const;
+
+ bool canBeNegativeZero() const { return canBeNegativeZero_; }
+ void setCanBeNegativeZero(bool negativeZero) {
+ canBeNegativeZero_ = negativeZero;
+ }
+
+ [[nodiscard]] bool updateForReplacement(MDefinition* ins) override;
+
+ bool fallible() const { return canBeNegativeZero_ || canOverflow(); }
+
+ bool isFloat32Commutative() const override { return true; }
+
+ void computeRange(TempAllocator& alloc) override;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+ Mode mode() const { return mode_; }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MMul)
+};
+
+class MDiv : public MBinaryArithInstruction {
+ bool canBeNegativeZero_;
+ bool canBeNegativeOverflow_;
+ bool canBeDivideByZero_;
+ bool canBeNegativeDividend_;
+ bool unsigned_; // If false, signedness will be derived from operands
+ bool trapOnError_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MDiv(MDefinition* left, MDefinition* right, MIRType type)
+ : MBinaryArithInstruction(classOpcode, left, right, type),
+ canBeNegativeZero_(true),
+ canBeNegativeOverflow_(true),
+ canBeDivideByZero_(true),
+ canBeNegativeDividend_(true),
+ unsigned_(false),
+ trapOnError_(false) {}
+
+ public:
+ INSTRUCTION_HEADER(Div)
+
+ static MDiv* New(TempAllocator& alloc, MDefinition* left, MDefinition* right,
+ MIRType type) {
+ return new (alloc) MDiv(left, right, type);
+ }
+ static MDiv* New(TempAllocator& alloc, MDefinition* left, MDefinition* right,
+ MIRType type, bool unsignd, bool trapOnError = false,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset(),
+ bool mustPreserveNaN = false) {
+ auto* div = new (alloc) MDiv(left, right, type);
+ div->unsigned_ = unsignd;
+ div->trapOnError_ = trapOnError;
+ div->bytecodeOffset_ = bytecodeOffset;
+ if (trapOnError) {
+ div->setGuard(); // not removable because of possible side-effects.
+ div->setNotMovable();
+ }
+ div->setMustPreserveNaN(mustPreserveNaN);
+ if (type == MIRType::Int32) {
+ div->setTruncateKind(TruncateKind::Truncate);
+ }
+ return div;
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ void analyzeEdgeCasesForward() override;
+ void analyzeEdgeCasesBackward() override;
+
+ double getIdentity() override { MOZ_CRASH("not used"); }
+
+ bool canBeNegativeZero() const { return canBeNegativeZero_; }
+ void setCanBeNegativeZero(bool negativeZero) {
+ canBeNegativeZero_ = negativeZero;
+ }
+
+ bool canBeNegativeOverflow() const { return canBeNegativeOverflow_; }
+
+ bool canBeDivideByZero() const { return canBeDivideByZero_; }
+
+ bool canBeNegativeDividend() const {
+ // "Dividend" is an ambiguous concept for unsigned truncated
+ // division, because of the truncation procedure:
+ // ((x>>>0)/2)|0, for example, gets transformed in
+ // MDiv::truncate into a node with lhs representing x (not
+ // x>>>0) and rhs representing the constant 2; in other words,
+ // the MIR node corresponds to "cast operands to unsigned and
+ // divide" operation. In this case, is the dividend x or is it
+ // x>>>0? In order to resolve such ambiguities, we disallow
+ // the usage of this method for unsigned division.
+ MOZ_ASSERT(!unsigned_);
+ return canBeNegativeDividend_;
+ }
+
+ bool isUnsigned() const { return unsigned_; }
+
+ bool isTruncatedIndirectly() const {
+ return truncateKind() >= TruncateKind::IndirectTruncate;
+ }
+
+ bool canTruncateInfinities() const { return isTruncated(); }
+ bool canTruncateRemainder() const { return isTruncated(); }
+ bool canTruncateOverflow() const {
+ return isTruncated() || isTruncatedIndirectly();
+ }
+ bool canTruncateNegativeZero() const {
+ return isTruncated() || isTruncatedIndirectly();
+ }
+
+ bool trapOnError() const { return trapOnError_; }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(bytecodeOffset_.isValid());
+ return bytecodeOffset_;
+ }
+
+ bool isFloat32Commutative() const override { return true; }
+
+ void computeRange(TempAllocator& alloc) override;
+ bool fallible() const;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+ void collectRangeInfoPreTrunc() override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!MBinaryArithInstruction::congruentTo(ins)) {
+ return false;
+ }
+ const MDiv* other = ins->toDiv();
+ MOZ_ASSERT(other->trapOnError() == trapOnError_);
+ return unsigned_ == other->isUnsigned();
+ }
+
+ ALLOW_CLONE(MDiv)
+};
+
+class MWasmBuiltinDivI64 : public MAryInstruction<3>, public ArithPolicy::Data {
+ bool canBeNegativeZero_;
+ bool canBeNegativeOverflow_;
+ bool canBeDivideByZero_;
+ bool canBeNegativeDividend_;
+ bool unsigned_; // If false, signedness will be derived from operands
+ bool trapOnError_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MWasmBuiltinDivI64(MDefinition* left, MDefinition* right,
+ MDefinition* instance)
+ : MAryInstruction(classOpcode),
+ canBeNegativeZero_(true),
+ canBeNegativeOverflow_(true),
+ canBeDivideByZero_(true),
+ canBeNegativeDividend_(true),
+ unsigned_(false),
+ trapOnError_(false) {
+ initOperand(0, left);
+ initOperand(1, right);
+ initOperand(2, instance);
+
+ setResultType(MIRType::Int64);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBuiltinDivI64)
+
+ NAMED_OPERANDS((0, lhs), (1, rhs), (2, instance))
+
+ static MWasmBuiltinDivI64* New(
+ TempAllocator& alloc, MDefinition* left, MDefinition* right,
+ MDefinition* instance, bool unsignd, bool trapOnError = false,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset()) {
+ auto* wasm64Div = new (alloc) MWasmBuiltinDivI64(left, right, instance);
+ wasm64Div->unsigned_ = unsignd;
+ wasm64Div->trapOnError_ = trapOnError;
+ wasm64Div->bytecodeOffset_ = bytecodeOffset;
+ if (trapOnError) {
+ wasm64Div->setGuard(); // not removable because of possible side-effects.
+ wasm64Div->setNotMovable();
+ }
+ return wasm64Div;
+ }
+
+ bool canBeNegativeZero() const { return canBeNegativeZero_; }
+ void setCanBeNegativeZero(bool negativeZero) {
+ canBeNegativeZero_ = negativeZero;
+ }
+
+ bool canBeNegativeOverflow() const { return canBeNegativeOverflow_; }
+
+ bool canBeDivideByZero() const { return canBeDivideByZero_; }
+
+ bool canBeNegativeDividend() const {
+ // "Dividend" is an ambiguous concept for unsigned truncated
+ // division, because of the truncation procedure:
+ // ((x>>>0)/2)|0, for example, gets transformed in
+ // MWasmDiv::truncate into a node with lhs representing x (not
+ // x>>>0) and rhs representing the constant 2; in other words,
+ // the MIR node corresponds to "cast operands to unsigned and
+ // divide" operation. In this case, is the dividend x or is it
+ // x>>>0? In order to resolve such ambiguities, we disallow
+ // the usage of this method for unsigned division.
+ MOZ_ASSERT(!unsigned_);
+ return canBeNegativeDividend_;
+ }
+
+ bool isUnsigned() const { return unsigned_; }
+
+ bool trapOnError() const { return trapOnError_; }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(bytecodeOffset_.isValid());
+ return bytecodeOffset_;
+ }
+
+ ALLOW_CLONE(MWasmBuiltinDivI64)
+};
+
+class MMod : public MBinaryArithInstruction {
+ bool unsigned_; // If false, signedness will be derived from operands
+ bool canBeNegativeDividend_;
+ bool canBePowerOfTwoDivisor_;
+ bool canBeDivideByZero_;
+ bool trapOnError_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MMod(MDefinition* left, MDefinition* right, MIRType type)
+ : MBinaryArithInstruction(classOpcode, left, right, type),
+ unsigned_(false),
+ canBeNegativeDividend_(true),
+ canBePowerOfTwoDivisor_(true),
+ canBeDivideByZero_(true),
+ trapOnError_(false) {}
+
+ public:
+ INSTRUCTION_HEADER(Mod)
+
+ static MMod* New(TempAllocator& alloc, MDefinition* left, MDefinition* right,
+ MIRType type) {
+ return new (alloc) MMod(left, right, type);
+ }
+ static MMod* New(
+ TempAllocator& alloc, MDefinition* left, MDefinition* right, MIRType type,
+ bool unsignd, bool trapOnError = false,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset()) {
+ auto* mod = new (alloc) MMod(left, right, type);
+ mod->unsigned_ = unsignd;
+ mod->trapOnError_ = trapOnError;
+ mod->bytecodeOffset_ = bytecodeOffset;
+ if (trapOnError) {
+ mod->setGuard(); // not removable because of possible side-effects.
+ mod->setNotMovable();
+ }
+ if (type == MIRType::Int32) {
+ mod->setTruncateKind(TruncateKind::Truncate);
+ }
+ return mod;
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ double getIdentity() override { MOZ_CRASH("not used"); }
+
+ bool canBeNegativeDividend() const {
+ MOZ_ASSERT(type() == MIRType::Int32 || type() == MIRType::Int64);
+ MOZ_ASSERT(!unsigned_);
+ return canBeNegativeDividend_;
+ }
+
+ bool canBeDivideByZero() const {
+ MOZ_ASSERT(type() == MIRType::Int32 || type() == MIRType::Int64);
+ return canBeDivideByZero_;
+ }
+
+ bool canBePowerOfTwoDivisor() const {
+ MOZ_ASSERT(type() == MIRType::Int32);
+ return canBePowerOfTwoDivisor_;
+ }
+
+ void analyzeEdgeCasesForward() override;
+
+ bool isUnsigned() const { return unsigned_; }
+
+ bool trapOnError() const { return trapOnError_; }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(bytecodeOffset_.isValid());
+ return bytecodeOffset_;
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ bool fallible() const;
+
+ void computeRange(TempAllocator& alloc) override;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+ void collectRangeInfoPreTrunc() override;
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return MBinaryArithInstruction::congruentTo(ins) &&
+ unsigned_ == ins->toMod()->isUnsigned();
+ }
+
+ bool possiblyCalls() const override { return type() == MIRType::Double; }
+
+ ALLOW_CLONE(MMod)
+};
+
+class MWasmBuiltinModD : public MAryInstruction<3>, public ArithPolicy::Data {
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MWasmBuiltinModD(MDefinition* left, MDefinition* right, MDefinition* instance,
+ MIRType type)
+ : MAryInstruction(classOpcode) {
+ initOperand(0, left);
+ initOperand(1, right);
+ initOperand(2, instance);
+
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBuiltinModD)
+ NAMED_OPERANDS((0, lhs), (1, rhs), (2, instance))
+
+ static MWasmBuiltinModD* New(
+ TempAllocator& alloc, MDefinition* left, MDefinition* right,
+ MDefinition* instance, MIRType type,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset()) {
+ auto* wasmBuiltinModD =
+ new (alloc) MWasmBuiltinModD(left, right, instance, type);
+ wasmBuiltinModD->bytecodeOffset_ = bytecodeOffset;
+ return wasmBuiltinModD;
+ }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(bytecodeOffset_.isValid());
+ return bytecodeOffset_;
+ }
+
+ ALLOW_CLONE(MWasmBuiltinModD)
+};
+
+class MWasmBuiltinModI64 : public MAryInstruction<3>, public ArithPolicy::Data {
+ bool unsigned_; // If false, signedness will be derived from operands
+ bool canBeNegativeDividend_;
+ bool canBeDivideByZero_;
+ bool trapOnError_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MWasmBuiltinModI64(MDefinition* left, MDefinition* right,
+ MDefinition* instance)
+ : MAryInstruction(classOpcode),
+ unsigned_(false),
+ canBeNegativeDividend_(true),
+ canBeDivideByZero_(true),
+ trapOnError_(false) {
+ initOperand(0, left);
+ initOperand(1, right);
+ initOperand(2, instance);
+
+ setResultType(MIRType::Int64);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBuiltinModI64)
+
+ NAMED_OPERANDS((0, lhs), (1, rhs), (2, instance))
+
+ static MWasmBuiltinModI64* New(
+ TempAllocator& alloc, MDefinition* left, MDefinition* right,
+ MDefinition* instance, bool unsignd, bool trapOnError = false,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset()) {
+ auto* mod = new (alloc) MWasmBuiltinModI64(left, right, instance);
+ mod->unsigned_ = unsignd;
+ mod->trapOnError_ = trapOnError;
+ mod->bytecodeOffset_ = bytecodeOffset;
+ if (trapOnError) {
+ mod->setGuard(); // not removable because of possible side-effects.
+ mod->setNotMovable();
+ }
+ return mod;
+ }
+
+ bool canBeNegativeDividend() const {
+ MOZ_ASSERT(!unsigned_);
+ return canBeNegativeDividend_;
+ }
+
+ bool canBeDivideByZero() const { return canBeDivideByZero_; }
+
+ bool isUnsigned() const { return unsigned_; }
+
+ bool trapOnError() const { return trapOnError_; }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(bytecodeOffset_.isValid());
+ return bytecodeOffset_;
+ }
+
+ ALLOW_CLONE(MWasmBuiltinModI64)
+};
+
+class MBigIntBinaryArithInstruction : public MBinaryInstruction,
+ public BigIntArithPolicy::Data {
+ protected:
+ MBigIntBinaryArithInstruction(Opcode op, MDefinition* left,
+ MDefinition* right)
+ : MBinaryInstruction(op, left, right) {
+ setResultType(MIRType::BigInt);
+ setMovable();
+ }
+
+ public:
+ bool congruentTo(const MDefinition* ins) const override {
+ return binaryCongruentTo(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MBigIntAdd : public MBigIntBinaryArithInstruction {
+ MBigIntAdd(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ setCommutative();
+
+ // Don't guard this instruction even though adding two BigInts can throw
+ // JSMSG_BIGINT_TOO_LARGE. This matches the behavior when adding too large
+ // strings in MConcat.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntAdd)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntAdd)
+};
+
+class MBigIntSub : public MBigIntBinaryArithInstruction {
+ MBigIntSub(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ // See MBigIntAdd for why we don't guard this instruction.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntSub)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntSub)
+};
+
+class MBigIntMul : public MBigIntBinaryArithInstruction {
+ MBigIntMul(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ setCommutative();
+
+ // See MBigIntAdd for why we don't guard this instruction.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntMul)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntMul)
+};
+
+class MBigIntDiv : public MBigIntBinaryArithInstruction {
+ bool canBeDivideByZero_;
+
+ MBigIntDiv(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ MOZ_ASSERT(right->type() == MIRType::BigInt);
+ canBeDivideByZero_ =
+ !right->isConstant() || right->toConstant()->toBigInt()->isZero();
+
+ // Throws when the divisor is zero.
+ if (canBeDivideByZero_) {
+ setGuard();
+ setNotMovable();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntDiv)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool canBeDivideByZero() const { return canBeDivideByZero_; }
+
+ AliasSet getAliasSet() const override {
+ if (canBeDivideByZero()) {
+ return AliasSet::Store(AliasSet::ExceptionState);
+ }
+ return AliasSet::None();
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return !canBeDivideByZero(); }
+
+ ALLOW_CLONE(MBigIntDiv)
+};
+
+class MBigIntMod : public MBigIntBinaryArithInstruction {
+ bool canBeDivideByZero_;
+
+ MBigIntMod(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ MOZ_ASSERT(right->type() == MIRType::BigInt);
+ canBeDivideByZero_ =
+ !right->isConstant() || right->toConstant()->toBigInt()->isZero();
+
+ // Throws when the divisor is zero.
+ if (canBeDivideByZero_) {
+ setGuard();
+ setNotMovable();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntMod)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool canBeDivideByZero() const { return canBeDivideByZero_; }
+
+ AliasSet getAliasSet() const override {
+ if (canBeDivideByZero()) {
+ return AliasSet::Store(AliasSet::ExceptionState);
+ }
+ return AliasSet::None();
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return !canBeDivideByZero(); }
+
+ ALLOW_CLONE(MBigIntMod)
+};
+
+class MBigIntPow : public MBigIntBinaryArithInstruction {
+ bool canBeNegativeExponent_;
+
+ MBigIntPow(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ MOZ_ASSERT(right->type() == MIRType::BigInt);
+ canBeNegativeExponent_ =
+ !right->isConstant() || right->toConstant()->toBigInt()->isNegative();
+
+ // Throws when the exponent is negative.
+ if (canBeNegativeExponent_) {
+ setGuard();
+ setNotMovable();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntPow)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool canBeNegativeExponent() const { return canBeNegativeExponent_; }
+
+ AliasSet getAliasSet() const override {
+ if (canBeNegativeExponent()) {
+ return AliasSet::Store(AliasSet::ExceptionState);
+ }
+ return AliasSet::None();
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return !canBeNegativeExponent(); }
+
+ ALLOW_CLONE(MBigIntPow)
+};
+
+class MBigIntBitAnd : public MBigIntBinaryArithInstruction {
+ MBigIntBitAnd(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ setCommutative();
+
+ // We don't need to guard this instruction because it can only fail on OOM.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntBitAnd)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntBitAnd)
+};
+
+class MBigIntBitOr : public MBigIntBinaryArithInstruction {
+ MBigIntBitOr(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ setCommutative();
+
+ // We don't need to guard this instruction because it can only fail on OOM.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntBitOr)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntBitOr)
+};
+
+class MBigIntBitXor : public MBigIntBinaryArithInstruction {
+ MBigIntBitXor(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ setCommutative();
+
+ // We don't need to guard this instruction because it can only fail on OOM.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntBitXor)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntBitXor)
+};
+
+class MBigIntLsh : public MBigIntBinaryArithInstruction {
+ MBigIntLsh(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ // See MBigIntAdd for why we don't guard this instruction.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntLsh)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntLsh)
+};
+
+class MBigIntRsh : public MBigIntBinaryArithInstruction {
+ MBigIntRsh(MDefinition* left, MDefinition* right)
+ : MBigIntBinaryArithInstruction(classOpcode, left, right) {
+ // See MBigIntAdd for why we don't guard this instruction.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntRsh)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntRsh)
+};
+
+class MBigIntUnaryArithInstruction : public MUnaryInstruction,
+ public BigIntArithPolicy::Data {
+ protected:
+ MBigIntUnaryArithInstruction(Opcode op, MDefinition* input)
+ : MUnaryInstruction(op, input) {
+ setResultType(MIRType::BigInt);
+ setMovable();
+ }
+
+ public:
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MBigIntIncrement : public MBigIntUnaryArithInstruction {
+ explicit MBigIntIncrement(MDefinition* input)
+ : MBigIntUnaryArithInstruction(classOpcode, input) {
+ // See MBigIntAdd for why we don't guard this instruction.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntIncrement)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntIncrement)
+};
+
+class MBigIntDecrement : public MBigIntUnaryArithInstruction {
+ explicit MBigIntDecrement(MDefinition* input)
+ : MBigIntUnaryArithInstruction(classOpcode, input) {
+ // See MBigIntAdd for why we don't guard this instruction.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntDecrement)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntDecrement)
+};
+
+class MBigIntNegate : public MBigIntUnaryArithInstruction {
+ explicit MBigIntNegate(MDefinition* input)
+ : MBigIntUnaryArithInstruction(classOpcode, input) {
+ // We don't need to guard this instruction because it can only fail on OOM.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntNegate)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntNegate)
+};
+
+class MBigIntBitNot : public MBigIntUnaryArithInstruction {
+ explicit MBigIntBitNot(MDefinition* input)
+ : MBigIntUnaryArithInstruction(classOpcode, input) {
+ // See MBigIntAdd for why we don't guard this instruction.
+ }
+
+ public:
+ INSTRUCTION_HEADER(BigIntBitNot)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MBigIntBitNot)
+};
+
+class MConcat : public MBinaryInstruction,
+ public MixPolicy<ConvertToStringPolicy<0>,
+ ConvertToStringPolicy<1>>::Data {
+ MConcat(MDefinition* left, MDefinition* right)
+ : MBinaryInstruction(classOpcode, left, right) {
+ // At least one input should be definitely string
+ MOZ_ASSERT(left->type() == MIRType::String ||
+ right->type() == MIRType::String);
+
+ setMovable();
+ setResultType(MIRType::String);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Concat)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MConcat)
+};
+
+class MStringConvertCase : public MUnaryInstruction,
+ public StringPolicy<0>::Data {
+ public:
+ enum Mode { LowerCase, UpperCase };
+
+ private:
+ Mode mode_;
+
+ MStringConvertCase(MDefinition* string, Mode mode)
+ : MUnaryInstruction(classOpcode, string), mode_(mode) {
+ setResultType(MIRType::String);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(StringConvertCase)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, string))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toStringConvertCase()->mode() == mode();
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool possiblyCalls() const override { return true; }
+ Mode mode() const { return mode_; }
+};
+
+// This is a 3 state flag used by FlagPhiInputsAsImplicitlyUsed to record and
+// propagate the information about the consumers of a Phi instruction. This is
+// then used to set ImplicitlyUsed flags on the inputs of such Phi instructions.
+enum class PhiUsage : uint8_t { Unknown, Unused, Used };
+
+using PhiVector = Vector<MPhi*, 4, JitAllocPolicy>;
+
+class MPhi final : public MDefinition,
+ public InlineListNode<MPhi>,
+ public NoTypePolicy::Data {
+ using InputVector = js::Vector<MUse, 2, JitAllocPolicy>;
+ InputVector inputs_;
+
+ TruncateKind truncateKind_;
+ bool triedToSpecialize_;
+ bool isIterator_;
+ bool canProduceFloat32_;
+ bool canConsumeFloat32_;
+ // Record the state of the data flow before any mutation made to the control
+ // flow, such that removing branches is properly accounted for.
+ PhiUsage usageAnalysis_;
+
+ protected:
+ MUse* getUseFor(size_t index) override {
+ MOZ_ASSERT(index < numOperands());
+ return &inputs_[index];
+ }
+ const MUse* getUseFor(size_t index) const override { return &inputs_[index]; }
+
+ public:
+ INSTRUCTION_HEADER_WITHOUT_TYPEPOLICY(Phi)
+ virtual const TypePolicy* typePolicy();
+ virtual MIRType typePolicySpecialization();
+
+ MPhi(TempAllocator& alloc, MIRType resultType)
+ : MDefinition(classOpcode),
+ inputs_(alloc),
+ truncateKind_(TruncateKind::NoTruncate),
+ triedToSpecialize_(false),
+ isIterator_(false),
+ canProduceFloat32_(false),
+ canConsumeFloat32_(false),
+ usageAnalysis_(PhiUsage::Unknown) {
+ setResultType(resultType);
+ }
+
+ static MPhi* New(TempAllocator& alloc, MIRType resultType = MIRType::Value) {
+ return new (alloc) MPhi(alloc, resultType);
+ }
+ static MPhi* New(TempAllocator::Fallible alloc,
+ MIRType resultType = MIRType::Value) {
+ return new (alloc) MPhi(alloc.alloc, resultType);
+ }
+
+ void removeOperand(size_t index);
+ void removeAllOperands();
+
+ MDefinition* getOperand(size_t index) const override {
+ return inputs_[index].producer();
+ }
+ size_t numOperands() const override { return inputs_.length(); }
+ size_t indexOf(const MUse* u) const final {
+ MOZ_ASSERT(u >= &inputs_[0]);
+ MOZ_ASSERT(u <= &inputs_[numOperands() - 1]);
+ return u - &inputs_[0];
+ }
+ void replaceOperand(size_t index, MDefinition* operand) final {
+ inputs_[index].replaceProducer(operand);
+ }
+ bool triedToSpecialize() const { return triedToSpecialize_; }
+ void specialize(MIRType type) {
+ triedToSpecialize_ = true;
+ setResultType(type);
+ }
+
+#ifdef DEBUG
+ // Assert that this is a phi in a loop header with a unique predecessor and
+ // a unique backedge.
+ void assertLoopPhi() const;
+#else
+ void assertLoopPhi() const {}
+#endif
+
+ // Assuming this phi is in a loop header with a unique loop entry, return
+ // the phi operand along the loop entry.
+ MDefinition* getLoopPredecessorOperand() const;
+
+ // Assuming this phi is in a loop header with a unique loop entry, return
+ // the phi operand along the loop backedge.
+ MDefinition* getLoopBackedgeOperand() const;
+
+ // Whether this phi's type already includes information for def.
+ bool typeIncludes(MDefinition* def);
+
+ // Mark all phis in |iterators|, and the phis they flow into, as having
+ // implicit uses.
+ [[nodiscard]] static bool markIteratorPhis(const PhiVector& iterators);
+
+ // Initializes the operands vector to the given capacity,
+ // permitting use of addInput() instead of addInputSlow().
+ [[nodiscard]] bool reserveLength(size_t length) {
+ return inputs_.reserve(length);
+ }
+
+ // Use only if capacity has been reserved by reserveLength
+ void addInput(MDefinition* ins) {
+ MOZ_ASSERT_IF(type() != MIRType::Value, ins->type() == type());
+ inputs_.infallibleEmplaceBack(ins, this);
+ }
+
+ // Appends a new input to the input vector. May perform reallocation.
+ // Prefer reserveLength() and addInput() instead, where possible.
+ [[nodiscard]] bool addInputSlow(MDefinition* ins) {
+ MOZ_ASSERT_IF(type() != MIRType::Value, ins->type() == type());
+ return inputs_.emplaceBack(ins, this);
+ }
+
+ // Appends a new input to the input vector. Infallible because
+ // we know the inputs fits in the vector's inline storage.
+ void addInlineInput(MDefinition* ins) {
+ MOZ_ASSERT(inputs_.length() < InputVector::InlineLength);
+ MOZ_ALWAYS_TRUE(addInputSlow(ins));
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ MDefinition* foldsTernary(TempAllocator& alloc);
+
+ bool congruentTo(const MDefinition* ins) const override;
+ bool updateForReplacement(MDefinition* def) override;
+
+ bool isIterator() const { return isIterator_; }
+ void setIterator() { isIterator_ = true; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+
+ MDefinition* operandIfRedundant();
+
+ bool canProduceFloat32() const override { return canProduceFloat32_; }
+
+ void setCanProduceFloat32(bool can) { canProduceFloat32_ = can; }
+
+ bool canConsumeFloat32(MUse* use) const override {
+ return canConsumeFloat32_;
+ }
+
+ void setCanConsumeFloat32(bool can) { canConsumeFloat32_ = can; }
+
+ TruncateKind operandTruncateKind(size_t index) const override;
+ bool needTruncation(TruncateKind kind) const override;
+ void truncate(TruncateKind kind) override;
+
+ PhiUsage getUsageAnalysis() const { return usageAnalysis_; }
+ void setUsageAnalysis(PhiUsage pu) {
+ MOZ_ASSERT(usageAnalysis_ == PhiUsage::Unknown);
+ usageAnalysis_ = pu;
+ MOZ_ASSERT(usageAnalysis_ != PhiUsage::Unknown);
+ }
+};
+
+// The goal of a Beta node is to split a def at a conditionally taken
+// branch, so that uses dominated by it have a different name.
+class MBeta : public MUnaryInstruction, public NoTypePolicy::Data {
+ private:
+ // This is the range induced by a comparison and branch in a preceding
+ // block. Note that this does not reflect any range constraints from
+ // the input value itself, so this value may differ from the range()
+ // range after it is computed.
+ const Range* comparison_;
+
+ MBeta(MDefinition* val, const Range* comp)
+ : MUnaryInstruction(classOpcode, val), comparison_(comp) {
+ setResultType(val->type());
+ }
+
+ public:
+ INSTRUCTION_HEADER(Beta)
+ TRIVIAL_NEW_WRAPPERS
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ void computeRange(TempAllocator& alloc) override;
+};
+
+// If input evaluates to false (i.e. it's NaN, 0 or -0), 0 is returned, else the
+// input is returned
+class MNaNToZero : public MUnaryInstruction, public DoublePolicy<0>::Data {
+ bool operandIsNeverNaN_;
+ bool operandIsNeverNegativeZero_;
+
+ explicit MNaNToZero(MDefinition* input)
+ : MUnaryInstruction(classOpcode, input),
+ operandIsNeverNaN_(false),
+ operandIsNeverNegativeZero_(false) {
+ setResultType(MIRType::Double);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(NaNToZero)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool operandIsNeverNaN() const { return operandIsNeverNaN_; }
+
+ bool operandIsNeverNegativeZero() const {
+ return operandIsNeverNegativeZero_;
+ }
+
+ void collectRangeInfoPreTrunc() override;
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ void computeRange(TempAllocator& alloc) override;
+
+ bool writeRecoverData(CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MNaNToZero)
+};
+
+// MIR representation of a Value on the OSR BaselineFrame.
+// The Value is indexed off of OsrFrameReg.
+class MOsrValue : public MUnaryInstruction, public NoTypePolicy::Data {
+ private:
+ ptrdiff_t frameOffset_;
+
+ MOsrValue(MOsrEntry* entry, ptrdiff_t frameOffset)
+ : MUnaryInstruction(classOpcode, entry), frameOffset_(frameOffset) {
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(OsrValue)
+ TRIVIAL_NEW_WRAPPERS
+
+ ptrdiff_t frameOffset() const { return frameOffset_; }
+
+ MOsrEntry* entry() { return getOperand(0)->toOsrEntry(); }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// MIR representation of a JSObject scope chain pointer on the OSR
+// BaselineFrame. The pointer is indexed off of OsrFrameReg.
+class MOsrEnvironmentChain : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ private:
+ explicit MOsrEnvironmentChain(MOsrEntry* entry)
+ : MUnaryInstruction(classOpcode, entry) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(OsrEnvironmentChain)
+ TRIVIAL_NEW_WRAPPERS
+
+ MOsrEntry* entry() { return getOperand(0)->toOsrEntry(); }
+};
+
+// MIR representation of a JSObject ArgumentsObject pointer on the OSR
+// BaselineFrame. The pointer is indexed off of OsrFrameReg.
+class MOsrArgumentsObject : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ private:
+ explicit MOsrArgumentsObject(MOsrEntry* entry)
+ : MUnaryInstruction(classOpcode, entry) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(OsrArgumentsObject)
+ TRIVIAL_NEW_WRAPPERS
+
+ MOsrEntry* entry() { return getOperand(0)->toOsrEntry(); }
+};
+
+// MIR representation of the return value on the OSR BaselineFrame.
+// The Value is indexed off of OsrFrameReg.
+class MOsrReturnValue : public MUnaryInstruction, public NoTypePolicy::Data {
+ private:
+ explicit MOsrReturnValue(MOsrEntry* entry)
+ : MUnaryInstruction(classOpcode, entry) {
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(OsrReturnValue)
+ TRIVIAL_NEW_WRAPPERS
+
+ MOsrEntry* entry() { return getOperand(0)->toOsrEntry(); }
+};
+
+class MBinaryCache : public MBinaryInstruction,
+ public MixPolicy<BoxPolicy<0>, BoxPolicy<1>>::Data {
+ explicit MBinaryCache(MDefinition* left, MDefinition* right, MIRType resType)
+ : MBinaryInstruction(classOpcode, left, right) {
+ setResultType(resType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(BinaryCache)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+// Check whether we need to fire the interrupt handler (in wasm code).
+class MWasmInterruptCheck : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MWasmInterruptCheck(MDefinition* instance,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MUnaryInstruction(classOpcode, instance),
+ bytecodeOffset_(bytecodeOffset) {
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmInterruptCheck)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, instance))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+};
+
+// Directly jumps to the indicated trap, leaving Wasm code and reporting a
+// runtime error.
+
+class MWasmTrap : public MAryControlInstruction<0, 0>,
+ public NoTypePolicy::Data {
+ wasm::Trap trap_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ explicit MWasmTrap(wasm::Trap trap, wasm::BytecodeOffset bytecodeOffset)
+ : MAryControlInstruction(classOpcode),
+ trap_(trap),
+ bytecodeOffset_(bytecodeOffset) {}
+
+ public:
+ INSTRUCTION_HEADER(WasmTrap)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ wasm::Trap trap() const { return trap_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+};
+
+// Checks if a value is JS_UNINITIALIZED_LEXICAL, bailout out if so, leaving
+// it to baseline to throw at the correct pc.
+class MLexicalCheck : public MUnaryInstruction, public BoxPolicy<0>::Data {
+ explicit MLexicalCheck(MDefinition* input)
+ : MUnaryInstruction(classOpcode, input) {
+ setResultType(MIRType::Value);
+ setMovable();
+ setGuard();
+
+ // If this instruction bails out, we will set a flag to prevent
+ // lexical checks in this script from being moved.
+ setBailoutKind(BailoutKind::UninitializedLexical);
+ }
+
+ public:
+ INSTRUCTION_HEADER(LexicalCheck)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+};
+
+// Unconditionally throw a known error number.
+class MThrowMsg : public MNullaryInstruction {
+ const ThrowMsgKind throwMsgKind_;
+
+ explicit MThrowMsg(ThrowMsgKind throwMsgKind)
+ : MNullaryInstruction(classOpcode), throwMsgKind_(throwMsgKind) {
+ setGuard();
+ setResultType(MIRType::None);
+ }
+
+ public:
+ INSTRUCTION_HEADER(ThrowMsg)
+ TRIVIAL_NEW_WRAPPERS
+
+ ThrowMsgKind throwMsgKind() const { return throwMsgKind_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::ExceptionState);
+ }
+};
+
+class MGetFirstDollarIndex : public MUnaryInstruction,
+ public StringPolicy<0>::Data {
+ explicit MGetFirstDollarIndex(MDefinition* str)
+ : MUnaryInstruction(classOpcode, str) {
+ setResultType(MIRType::Int32);
+
+ // Codegen assumes string length > 0. Don't allow LICM to move this
+ // before the .length > 1 check in RegExpReplace in RegExp.js.
+ MOZ_ASSERT(!isMovable());
+ }
+
+ public:
+ INSTRUCTION_HEADER(GetFirstDollarIndex)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, str))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+};
+
+class MStringReplace : public MTernaryInstruction,
+ public MixPolicy<StringPolicy<0>, StringPolicy<1>,
+ StringPolicy<2>>::Data {
+ private:
+ bool isFlatReplacement_;
+
+ MStringReplace(MDefinition* string, MDefinition* pattern,
+ MDefinition* replacement)
+ : MTernaryInstruction(classOpcode, string, pattern, replacement),
+ isFlatReplacement_(false) {
+ setMovable();
+ setResultType(MIRType::String);
+ }
+
+ public:
+ INSTRUCTION_HEADER(StringReplace)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, string), (1, pattern), (2, replacement))
+
+ void setFlatReplacement() {
+ MOZ_ASSERT(!isFlatReplacement_);
+ isFlatReplacement_ = true;
+ }
+
+ bool isFlatReplacement() const { return isFlatReplacement_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isStringReplace()) {
+ return false;
+ }
+ if (isFlatReplacement_ != ins->toStringReplace()->isFlatReplacement()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override {
+ if (isFlatReplacement_) {
+ MOZ_ASSERT(!pattern()->isRegExp());
+ return true;
+ }
+ return false;
+ }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+class MLambda : public MBinaryInstruction, public SingleObjectPolicy::Data {
+ MLambda(MDefinition* envChain, MConstant* cst)
+ : MBinaryInstruction(classOpcode, envChain, cst) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Lambda)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, environmentChain))
+
+ MConstant* functionOperand() const { return getOperand(1)->toConstant(); }
+ JSFunction* templateFunction() const {
+ return &functionOperand()->toObject().as<JSFunction>();
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+class MFunctionWithProto : public MTernaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, ObjectPolicy<1>,
+ ObjectPolicy<2>>::Data {
+ CompilerFunction fun_;
+
+ MFunctionWithProto(MDefinition* envChain, MDefinition* prototype,
+ MConstant* cst)
+ : MTernaryInstruction(classOpcode, envChain, prototype, cst),
+ fun_(&cst->toObject().as<JSFunction>()) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(FunctionWithProto)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, environmentChain), (1, prototype))
+
+ MConstant* functionOperand() const { return getOperand(2)->toConstant(); }
+ JSFunction* function() const { return fun_; }
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+class MGetNextEntryForIterator
+ : public MBinaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, ObjectPolicy<1>>::Data {
+ public:
+ enum Mode { Map, Set };
+
+ private:
+ Mode mode_;
+
+ explicit MGetNextEntryForIterator(MDefinition* iter, MDefinition* result,
+ Mode mode)
+ : MBinaryInstruction(classOpcode, iter, result), mode_(mode) {
+ setResultType(MIRType::Boolean);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GetNextEntryForIterator)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, iter), (1, result))
+
+ Mode mode() const { return mode_; }
+};
+
+// Convert a Double into an IntPtr value for accessing a TypedArray or DataView
+// element. If the input is non-finite, not an integer, negative, or outside the
+// IntPtr range, either bails out or produces a value which is known to trigger
+// an out-of-bounds access (this depends on the supportOOB flag).
+class MGuardNumberToIntPtrIndex : public MUnaryInstruction,
+ public DoublePolicy<0>::Data {
+ // If true, produce an out-of-bounds index for non-IntPtr doubles instead of
+ // bailing out.
+ const bool supportOOB_;
+
+ MGuardNumberToIntPtrIndex(MDefinition* def, bool supportOOB)
+ : MUnaryInstruction(classOpcode, def), supportOOB_(supportOOB) {
+ MOZ_ASSERT(def->type() == MIRType::Double);
+ setResultType(MIRType::IntPtr);
+ setMovable();
+ if (!supportOOB) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardNumberToIntPtrIndex)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool supportOOB() const { return supportOOB_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGuardNumberToIntPtrIndex()) {
+ return false;
+ }
+ if (ins->toGuardNumberToIntPtrIndex()->supportOOB() != supportOOB()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ ALLOW_CLONE(MGuardNumberToIntPtrIndex)
+};
+
+// Perform !-operation
+class MNot : public MUnaryInstruction, public TestPolicy::Data {
+ bool operandIsNeverNaN_;
+ TypeDataList observedTypes_;
+
+ explicit MNot(MDefinition* input)
+ : MUnaryInstruction(classOpcode, input), operandIsNeverNaN_(false) {
+ setResultType(MIRType::Boolean);
+ setMovable();
+ }
+
+ public:
+ static MNot* NewInt32(TempAllocator& alloc, MDefinition* input) {
+ MOZ_ASSERT(input->type() == MIRType::Int32 ||
+ input->type() == MIRType::Int64);
+ auto* ins = new (alloc) MNot(input);
+ ins->setResultType(MIRType::Int32);
+ return ins;
+ }
+
+ INSTRUCTION_HEADER(Not)
+ TRIVIAL_NEW_WRAPPERS
+
+ void setObservedTypes(const TypeDataList& observed) {
+ observedTypes_ = observed;
+ }
+ const TypeDataList& observedTypes() const { return observedTypes_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool operandIsNeverNaN() const { return operandIsNeverNaN_; }
+
+ virtual AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void collectRangeInfoPreTrunc() override;
+
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+ bool isFloat32Commutative() const override { return true; }
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+// Bailout if index + minimum < 0 or index + maximum >= length. The length used
+// in a bounds check must not be negative, or the wrong result may be computed
+// (unsigned comparisons may be used).
+class MBoundsCheck
+ : public MBinaryInstruction,
+ public MixPolicy<Int32OrIntPtrPolicy<0>, Int32OrIntPtrPolicy<1>>::Data {
+ // Range over which to perform the bounds check, may be modified by GVN.
+ int32_t minimum_;
+ int32_t maximum_;
+ bool fallible_;
+
+ MBoundsCheck(MDefinition* index, MDefinition* length)
+ : MBinaryInstruction(classOpcode, index, length),
+ minimum_(0),
+ maximum_(0),
+ fallible_(true) {
+ setGuard();
+ setMovable();
+ MOZ_ASSERT(index->type() == MIRType::Int32 ||
+ index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(index->type() == length->type());
+
+ // Returns the checked index.
+ setResultType(index->type());
+ }
+
+ public:
+ INSTRUCTION_HEADER(BoundsCheck)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, index), (1, length))
+
+ int32_t minimum() const { return minimum_; }
+ void setMinimum(int32_t n) {
+ MOZ_ASSERT(fallible_);
+ minimum_ = n;
+ }
+ int32_t maximum() const { return maximum_; }
+ void setMaximum(int32_t n) {
+ MOZ_ASSERT(fallible_);
+ maximum_ = n;
+ }
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isBoundsCheck()) {
+ return false;
+ }
+ const MBoundsCheck* other = ins->toBoundsCheck();
+ if (minimum() != other->minimum() || maximum() != other->maximum()) {
+ return false;
+ }
+ if (fallible() != other->fallible()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(other);
+ }
+ virtual AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+ bool fallible() const { return fallible_; }
+ void collectRangeInfoPreTrunc() override;
+
+ ALLOW_CLONE(MBoundsCheck)
+};
+
+// Bailout if index < minimum.
+class MBoundsCheckLower : public MUnaryInstruction,
+ public UnboxedInt32Policy<0>::Data {
+ int32_t minimum_;
+ bool fallible_;
+
+ explicit MBoundsCheckLower(MDefinition* index)
+ : MUnaryInstruction(classOpcode, index), minimum_(0), fallible_(true) {
+ setGuard();
+ setMovable();
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ }
+
+ public:
+ INSTRUCTION_HEADER(BoundsCheckLower)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, index))
+
+ int32_t minimum() const { return minimum_; }
+ void setMinimum(int32_t n) { minimum_ = n; }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool fallible() const { return fallible_; }
+ void collectRangeInfoPreTrunc() override;
+};
+
+class MSpectreMaskIndex
+ : public MBinaryInstruction,
+ public MixPolicy<Int32OrIntPtrPolicy<0>, Int32OrIntPtrPolicy<1>>::Data {
+ MSpectreMaskIndex(MDefinition* index, MDefinition* length)
+ : MBinaryInstruction(classOpcode, index, length) {
+ // Note: this instruction does not need setGuard(): if there are no uses
+ // it's fine for DCE to eliminate this instruction.
+ setMovable();
+ MOZ_ASSERT(index->type() == MIRType::Int32 ||
+ index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(index->type() == length->type());
+
+ // Returns the masked index.
+ setResultType(index->type());
+ }
+
+ public:
+ INSTRUCTION_HEADER(SpectreMaskIndex)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, index), (1, length))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ virtual AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+
+ ALLOW_CLONE(MSpectreMaskIndex)
+};
+
+// Load a value from a dense array's element vector. Bails out if the element is
+// a hole.
+class MLoadElement : public MBinaryInstruction, public NoTypePolicy::Data {
+ MLoadElement(MDefinition* elements, MDefinition* index)
+ : MBinaryInstruction(classOpcode, elements, index) {
+ // Uses may be optimized away based on this instruction's result
+ // type. This means it's invalid to DCE this instruction, as we
+ // have to invalidate when we read a hole.
+ setGuard();
+ setResultType(MIRType::Value);
+ setMovable();
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasType mightAlias(const MDefinition* store) const override;
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::Element);
+ }
+
+ ALLOW_CLONE(MLoadElement)
+};
+
+class MLoadElementAndUnbox : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ MUnbox::Mode mode_;
+
+ MLoadElementAndUnbox(MDefinition* elements, MDefinition* index,
+ MUnbox::Mode mode, MIRType type)
+ : MBinaryInstruction(classOpcode, elements, index), mode_(mode) {
+ setResultType(type);
+ setMovable();
+ if (mode_ == MUnbox::Fallible) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadElementAndUnbox)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index))
+
+ MUnbox::Mode mode() const { return mode_; }
+ bool fallible() const { return mode_ != MUnbox::Infallible; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadElementAndUnbox() ||
+ mode() != ins->toLoadElementAndUnbox()->mode()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::Element);
+ }
+
+ ALLOW_CLONE(MLoadElementAndUnbox);
+};
+
+// Load a value from the elements vector of a native object. If the index is
+// out-of-bounds, or the indexed slot has a hole, undefined is returned instead.
+class MLoadElementHole : public MTernaryInstruction, public NoTypePolicy::Data {
+ bool needsNegativeIntCheck_ = true;
+
+ MLoadElementHole(MDefinition* elements, MDefinition* index,
+ MDefinition* initLength)
+ : MTernaryInstruction(classOpcode, elements, index, initLength) {
+ setResultType(MIRType::Value);
+ setMovable();
+
+ // Set the guard flag to make sure we bail when we see a negative
+ // index. We can clear this flag (and needsNegativeIntCheck_) in
+ // collectRangeInfoPreTrunc.
+ setGuard();
+
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ MOZ_ASSERT(initLength->type() == MIRType::Int32);
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadElementHole)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, initLength))
+
+ bool needsNegativeIntCheck() const { return needsNegativeIntCheck_; }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadElementHole()) {
+ return false;
+ }
+ const MLoadElementHole* other = ins->toLoadElementHole();
+ if (needsNegativeIntCheck() != other->needsNegativeIntCheck()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(other);
+ }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::Element);
+ }
+ void collectRangeInfoPreTrunc() override;
+
+ ALLOW_CLONE(MLoadElementHole)
+};
+
+// Store a value to a dense array slots vector.
+class MStoreElement : public MTernaryInstruction,
+ public NoFloatPolicy<2>::Data {
+ bool needsHoleCheck_;
+ bool needsBarrier_;
+
+ MStoreElement(MDefinition* elements, MDefinition* index, MDefinition* value,
+ bool needsHoleCheck, bool needsBarrier)
+ : MTernaryInstruction(classOpcode, elements, index, value) {
+ needsHoleCheck_ = needsHoleCheck;
+ needsBarrier_ = needsBarrier;
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ MOZ_ASSERT(value->type() != MIRType::MagicHole);
+ }
+
+ public:
+ INSTRUCTION_HEADER(StoreElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, value))
+
+ static MStoreElement* NewUnbarriered(TempAllocator& alloc,
+ MDefinition* elements,
+ MDefinition* index, MDefinition* value,
+ bool needsHoleCheck) {
+ return new (alloc)
+ MStoreElement(elements, index, value, needsHoleCheck, false);
+ }
+
+ static MStoreElement* NewBarriered(TempAllocator& alloc,
+ MDefinition* elements, MDefinition* index,
+ MDefinition* value, bool needsHoleCheck) {
+ return new (alloc)
+ MStoreElement(elements, index, value, needsHoleCheck, true);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::Element);
+ }
+ bool needsBarrier() const { return needsBarrier_; }
+ bool needsHoleCheck() const { return needsHoleCheck_; }
+ bool fallible() const { return needsHoleCheck(); }
+
+ ALLOW_CLONE(MStoreElement)
+};
+
+// Stores MagicValue(JS_ELEMENTS_HOLE) and marks the elements as non-packed.
+class MStoreHoleValueElement : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ MStoreHoleValueElement(MDefinition* elements, MDefinition* index)
+ : MBinaryInstruction(classOpcode, elements, index) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ }
+
+ public:
+ INSTRUCTION_HEADER(StoreHoleValueElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index))
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::Element | AliasSet::ObjectFields);
+ }
+
+ ALLOW_CLONE(MStoreHoleValueElement)
+};
+
+// Like MStoreElement, but also supports index == initialized length. The
+// downside is that we cannot hoist the elements vector and bounds check, since
+// this instruction may update the (initialized) length and reallocate the
+// elements vector.
+class MStoreElementHole
+ : public MQuaternaryInstruction,
+ public MixPolicy<SingleObjectPolicy, NoFloatPolicy<3>>::Data {
+ MStoreElementHole(MDefinition* object, MDefinition* elements,
+ MDefinition* index, MDefinition* value)
+ : MQuaternaryInstruction(classOpcode, object, elements, index, value) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ MOZ_ASSERT(value->type() != MIRType::MagicHole);
+ }
+
+ public:
+ INSTRUCTION_HEADER(StoreElementHole)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, elements), (2, index), (3, value))
+
+ AliasSet getAliasSet() const override {
+ // StoreElementHole can update the initialized length, the array length
+ // or reallocate obj->elements.
+ return AliasSet::Store(AliasSet::ObjectFields | AliasSet::Element);
+ }
+
+ ALLOW_CLONE(MStoreElementHole)
+};
+
+// Array.prototype.pop or Array.prototype.shift on a dense array.
+class MArrayPopShift : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ public:
+ enum Mode { Pop, Shift };
+
+ private:
+ Mode mode_;
+
+ MArrayPopShift(MDefinition* object, Mode mode)
+ : MUnaryInstruction(classOpcode, object), mode_(mode) {
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(ArrayPopShift)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ bool mode() const { return mode_; }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::ObjectFields | AliasSet::Element);
+ }
+
+ ALLOW_CLONE(MArrayPopShift)
+};
+
+// All barriered operations - MCompareExchangeTypedArrayElement,
+// MExchangeTypedArrayElement, and MAtomicTypedArrayElementBinop, as
+// well as MLoadUnboxedScalar and MStoreUnboxedScalar when they are
+// marked as requiring a memory barrer - have the following
+// attributes:
+//
+// - Not movable
+// - Not removable
+// - Not congruent with any other instruction
+// - Effectful (they alias every TypedArray store)
+//
+// The intended effect of those constraints is to prevent all loads
+// and stores preceding the barriered operation from being moved to
+// after the barriered operation, and vice versa, and to prevent the
+// barriered operation from being removed or hoisted.
+
+enum MemoryBarrierRequirement {
+ DoesNotRequireMemoryBarrier,
+ DoesRequireMemoryBarrier
+};
+
+// Also see comments at MMemoryBarrierRequirement, above.
+
+// Load an unboxed scalar value from an array buffer view or other object.
+class MLoadUnboxedScalar : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ int32_t offsetAdjustment_ = 0;
+ Scalar::Type storageType_;
+ bool requiresBarrier_;
+
+ MLoadUnboxedScalar(
+ MDefinition* elements, MDefinition* index, Scalar::Type storageType,
+ MemoryBarrierRequirement requiresBarrier = DoesNotRequireMemoryBarrier)
+ : MBinaryInstruction(classOpcode, elements, index),
+ storageType_(storageType),
+ requiresBarrier_(requiresBarrier == DoesRequireMemoryBarrier) {
+ setResultType(MIRType::Value);
+ if (requiresBarrier_) {
+ setGuard(); // Not removable or movable
+ } else {
+ setMovable();
+ }
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(storageType >= 0 && storageType < Scalar::MaxTypedArrayViewType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadUnboxedScalar)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index))
+
+ Scalar::Type storageType() const { return storageType_; }
+ bool fallible() const {
+ // Bailout if the result does not fit in an int32.
+ return storageType_ == Scalar::Uint32 && type() == MIRType::Int32;
+ }
+ bool requiresMemoryBarrier() const { return requiresBarrier_; }
+ int32_t offsetAdjustment() const { return offsetAdjustment_; }
+ void setOffsetAdjustment(int32_t offsetAdjustment) {
+ offsetAdjustment_ = offsetAdjustment;
+ }
+ AliasSet getAliasSet() const override {
+ // When a barrier is needed make the instruction effectful by
+ // giving it a "store" effect.
+ if (requiresBarrier_) {
+ return AliasSet::Store(AliasSet::UnboxedElement);
+ }
+ return AliasSet::Load(AliasSet::UnboxedElement);
+ }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (requiresBarrier_) {
+ return false;
+ }
+ if (!ins->isLoadUnboxedScalar()) {
+ return false;
+ }
+ const MLoadUnboxedScalar* other = ins->toLoadUnboxedScalar();
+ if (storageType_ != other->storageType_) {
+ return false;
+ }
+ if (offsetAdjustment() != other->offsetAdjustment()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(other);
+ }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ void computeRange(TempAllocator& alloc) override;
+
+ bool canProduceFloat32() const override {
+ return storageType_ == Scalar::Float32;
+ }
+
+ ALLOW_CLONE(MLoadUnboxedScalar)
+};
+
+// Load an unboxed scalar value from a dataview object.
+class MLoadDataViewElement : public MTernaryInstruction,
+ public NoTypePolicy::Data {
+ Scalar::Type storageType_;
+
+ MLoadDataViewElement(MDefinition* elements, MDefinition* index,
+ MDefinition* littleEndian, Scalar::Type storageType)
+ : MTernaryInstruction(classOpcode, elements, index, littleEndian),
+ storageType_(storageType) {
+ setResultType(MIRType::Value);
+ setMovable();
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(littleEndian->type() == MIRType::Boolean);
+ MOZ_ASSERT(storageType >= 0 && storageType < Scalar::MaxTypedArrayViewType);
+ MOZ_ASSERT(Scalar::byteSize(storageType) > 1);
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadDataViewElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, littleEndian))
+
+ Scalar::Type storageType() const { return storageType_; }
+ bool fallible() const {
+ // Bailout if the result does not fit in an int32.
+ return storageType_ == Scalar::Uint32 && type() == MIRType::Int32;
+ }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::UnboxedElement);
+ }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadDataViewElement()) {
+ return false;
+ }
+ const MLoadDataViewElement* other = ins->toLoadDataViewElement();
+ if (storageType_ != other->storageType_) {
+ return false;
+ }
+ return congruentIfOperandsEqual(other);
+ }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ void computeRange(TempAllocator& alloc) override;
+
+ bool canProduceFloat32() const override {
+ return storageType_ == Scalar::Float32;
+ }
+
+ ALLOW_CLONE(MLoadDataViewElement)
+};
+
+// Load a value from a typed array. Out-of-bounds accesses are handled in-line.
+class MLoadTypedArrayElementHole : public MBinaryInstruction,
+ public SingleObjectPolicy::Data {
+ Scalar::Type arrayType_;
+ bool forceDouble_;
+
+ MLoadTypedArrayElementHole(MDefinition* object, MDefinition* index,
+ Scalar::Type arrayType, bool forceDouble)
+ : MBinaryInstruction(classOpcode, object, index),
+ arrayType_(arrayType),
+ forceDouble_(forceDouble) {
+ setResultType(MIRType::Value);
+ setMovable();
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(arrayType >= 0 && arrayType < Scalar::MaxTypedArrayViewType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadTypedArrayElementHole)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, index))
+
+ Scalar::Type arrayType() const { return arrayType_; }
+ bool forceDouble() const { return forceDouble_; }
+ bool fallible() const {
+ return arrayType_ == Scalar::Uint32 && !forceDouble_;
+ }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadTypedArrayElementHole()) {
+ return false;
+ }
+ const MLoadTypedArrayElementHole* other =
+ ins->toLoadTypedArrayElementHole();
+ if (arrayType() != other->arrayType()) {
+ return false;
+ }
+ if (forceDouble() != other->forceDouble()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(other);
+ }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::UnboxedElement | AliasSet::ObjectFields |
+ AliasSet::ArrayBufferViewLengthOrOffset);
+ }
+ bool canProduceFloat32() const override {
+ return arrayType_ == Scalar::Float32;
+ }
+
+ ALLOW_CLONE(MLoadTypedArrayElementHole)
+};
+
+// Base class for MIR ops that write unboxed scalar values.
+class StoreUnboxedScalarBase {
+ Scalar::Type writeType_;
+
+ protected:
+ explicit StoreUnboxedScalarBase(Scalar::Type writeType)
+ : writeType_(writeType) {
+ MOZ_ASSERT(isIntegerWrite() || isFloatWrite() || isBigIntWrite());
+ }
+
+ public:
+ Scalar::Type writeType() const { return writeType_; }
+ bool isByteWrite() const {
+ return writeType_ == Scalar::Int8 || writeType_ == Scalar::Uint8 ||
+ writeType_ == Scalar::Uint8Clamped;
+ }
+ bool isIntegerWrite() const {
+ return isByteWrite() || writeType_ == Scalar::Int16 ||
+ writeType_ == Scalar::Uint16 || writeType_ == Scalar::Int32 ||
+ writeType_ == Scalar::Uint32;
+ }
+ bool isFloatWrite() const {
+ return writeType_ == Scalar::Float32 || writeType_ == Scalar::Float64;
+ }
+ bool isBigIntWrite() const { return Scalar::isBigIntType(writeType_); }
+};
+
+// Store an unboxed scalar value to an array buffer view or other object.
+class MStoreUnboxedScalar : public MTernaryInstruction,
+ public StoreUnboxedScalarBase,
+ public StoreUnboxedScalarPolicy::Data {
+ bool requiresBarrier_;
+
+ MStoreUnboxedScalar(
+ MDefinition* elements, MDefinition* index, MDefinition* value,
+ Scalar::Type storageType,
+ MemoryBarrierRequirement requiresBarrier = DoesNotRequireMemoryBarrier)
+ : MTernaryInstruction(classOpcode, elements, index, value),
+ StoreUnboxedScalarBase(storageType),
+ requiresBarrier_(requiresBarrier == DoesRequireMemoryBarrier) {
+ if (requiresBarrier_) {
+ setGuard(); // Not removable or movable
+ }
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(storageType >= 0 && storageType < Scalar::MaxTypedArrayViewType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(StoreUnboxedScalar)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, value))
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::UnboxedElement);
+ }
+ bool requiresMemoryBarrier() const { return requiresBarrier_; }
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+ bool canConsumeFloat32(MUse* use) const override {
+ return use == getUseFor(2) && writeType() == Scalar::Float32;
+ }
+
+ ALLOW_CLONE(MStoreUnboxedScalar)
+};
+
+// Store an unboxed scalar value to a dataview object.
+class MStoreDataViewElement : public MQuaternaryInstruction,
+ public StoreUnboxedScalarBase,
+ public StoreDataViewElementPolicy::Data {
+ MStoreDataViewElement(MDefinition* elements, MDefinition* index,
+ MDefinition* value, MDefinition* littleEndian,
+ Scalar::Type storageType)
+ : MQuaternaryInstruction(classOpcode, elements, index, value,
+ littleEndian),
+ StoreUnboxedScalarBase(storageType) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(storageType >= 0 && storageType < Scalar::MaxTypedArrayViewType);
+ MOZ_ASSERT(Scalar::byteSize(storageType) > 1);
+ }
+
+ public:
+ INSTRUCTION_HEADER(StoreDataViewElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, value), (3, littleEndian))
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::UnboxedElement);
+ }
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+ bool canConsumeFloat32(MUse* use) const override {
+ return use == getUseFor(2) && writeType() == Scalar::Float32;
+ }
+
+ ALLOW_CLONE(MStoreDataViewElement)
+};
+
+class MStoreTypedArrayElementHole : public MQuaternaryInstruction,
+ public StoreUnboxedScalarBase,
+ public StoreTypedArrayHolePolicy::Data {
+ MStoreTypedArrayElementHole(MDefinition* elements, MDefinition* length,
+ MDefinition* index, MDefinition* value,
+ Scalar::Type arrayType)
+ : MQuaternaryInstruction(classOpcode, elements, length, index, value),
+ StoreUnboxedScalarBase(arrayType) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(length->type() == MIRType::IntPtr);
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(arrayType >= 0 && arrayType < Scalar::MaxTypedArrayViewType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(StoreTypedArrayElementHole)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, length), (2, index), (3, value))
+
+ Scalar::Type arrayType() const { return writeType(); }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::UnboxedElement);
+ }
+ TruncateKind operandTruncateKind(size_t index) const override;
+
+ bool canConsumeFloat32(MUse* use) const override {
+ return use == getUseFor(3) && arrayType() == Scalar::Float32;
+ }
+
+ ALLOW_CLONE(MStoreTypedArrayElementHole)
+};
+
+// Compute an "effective address", i.e., a compound computation of the form:
+// base + index * scale + displacement
+class MEffectiveAddress : public MBinaryInstruction, public NoTypePolicy::Data {
+ MEffectiveAddress(MDefinition* base, MDefinition* index, Scale scale,
+ int32_t displacement)
+ : MBinaryInstruction(classOpcode, base, index),
+ scale_(scale),
+ displacement_(displacement) {
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ setMovable();
+ setResultType(MIRType::Int32);
+ }
+
+ Scale scale_;
+ int32_t displacement_;
+
+ public:
+ INSTRUCTION_HEADER(EffectiveAddress)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* base() const { return lhs(); }
+ MDefinition* index() const { return rhs(); }
+ Scale scale() const { return scale_; }
+ int32_t displacement() const { return displacement_; }
+
+ ALLOW_CLONE(MEffectiveAddress)
+};
+
+// Clamp input to range [0, 255] for Uint8ClampedArray.
+class MClampToUint8 : public MUnaryInstruction, public ClampPolicy::Data {
+ explicit MClampToUint8(MDefinition* input)
+ : MUnaryInstruction(classOpcode, input) {
+ setResultType(MIRType::Int32);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(ClampToUint8)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ void computeRange(TempAllocator& alloc) override;
+
+ ALLOW_CLONE(MClampToUint8)
+};
+
+class MLoadFixedSlot : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ size_t slot_;
+
+ protected:
+ MLoadFixedSlot(MDefinition* obj, size_t slot)
+ : MUnaryInstruction(classOpcode, obj), slot_(slot) {
+ setResultType(MIRType::Value);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadFixedSlot)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ size_t slot() const { return slot_; }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadFixedSlot()) {
+ return false;
+ }
+ if (slot() != ins->toLoadFixedSlot()->slot()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::FixedSlot);
+ }
+
+ AliasType mightAlias(const MDefinition* store) const override;
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ ALLOW_CLONE(MLoadFixedSlot)
+};
+
+class MLoadFixedSlotAndUnbox : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ size_t slot_;
+ MUnbox::Mode mode_;
+
+ MLoadFixedSlotAndUnbox(MDefinition* obj, size_t slot, MUnbox::Mode mode,
+ MIRType type)
+ : MUnaryInstruction(classOpcode, obj), slot_(slot), mode_(mode) {
+ setResultType(type);
+ setMovable();
+ if (mode_ == MUnbox::Fallible) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadFixedSlotAndUnbox)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ size_t slot() const { return slot_; }
+ MUnbox::Mode mode() const { return mode_; }
+ bool fallible() const { return mode_ != MUnbox::Infallible; }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadFixedSlotAndUnbox() ||
+ slot() != ins->toLoadFixedSlotAndUnbox()->slot() ||
+ mode() != ins->toLoadFixedSlotAndUnbox()->mode()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::FixedSlot);
+ }
+
+ AliasType mightAlias(const MDefinition* store) const override;
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ ALLOW_CLONE(MLoadFixedSlotAndUnbox);
+};
+
+class MLoadDynamicSlotAndUnbox : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ size_t slot_;
+ MUnbox::Mode mode_;
+
+ MLoadDynamicSlotAndUnbox(MDefinition* slots, size_t slot, MUnbox::Mode mode,
+ MIRType type)
+ : MUnaryInstruction(classOpcode, slots), slot_(slot), mode_(mode) {
+ setResultType(type);
+ setMovable();
+ if (mode_ == MUnbox::Fallible) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadDynamicSlotAndUnbox)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, slots))
+
+ size_t slot() const { return slot_; }
+ MUnbox::Mode mode() const { return mode_; }
+ bool fallible() const { return mode_ != MUnbox::Infallible; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadDynamicSlotAndUnbox() ||
+ slot() != ins->toLoadDynamicSlotAndUnbox()->slot() ||
+ mode() != ins->toLoadDynamicSlotAndUnbox()->mode()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::DynamicSlot);
+ }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ ALLOW_CLONE(MLoadDynamicSlotAndUnbox);
+};
+
+class MStoreFixedSlot
+ : public MBinaryInstruction,
+ public MixPolicy<SingleObjectPolicy, NoFloatPolicy<1>>::Data {
+ bool needsBarrier_;
+ size_t slot_;
+
+ MStoreFixedSlot(MDefinition* obj, MDefinition* rval, size_t slot,
+ bool barrier)
+ : MBinaryInstruction(classOpcode, obj, rval),
+ needsBarrier_(barrier),
+ slot_(slot) {}
+
+ public:
+ INSTRUCTION_HEADER(StoreFixedSlot)
+ NAMED_OPERANDS((0, object), (1, value))
+
+ static MStoreFixedSlot* NewUnbarriered(TempAllocator& alloc, MDefinition* obj,
+ size_t slot, MDefinition* rval) {
+ return new (alloc) MStoreFixedSlot(obj, rval, slot, false);
+ }
+ static MStoreFixedSlot* NewBarriered(TempAllocator& alloc, MDefinition* obj,
+ size_t slot, MDefinition* rval) {
+ return new (alloc) MStoreFixedSlot(obj, rval, slot, true);
+ }
+
+ size_t slot() const { return slot_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::FixedSlot);
+ }
+ bool needsBarrier() const { return needsBarrier_; }
+ void setNeedsBarrier(bool needsBarrier = true) {
+ needsBarrier_ = needsBarrier;
+ }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ ALLOW_CLONE(MStoreFixedSlot)
+};
+
+class MGetPropertyCache : public MBinaryInstruction,
+ public MixPolicy<BoxExceptPolicy<0, MIRType::Object>,
+ CacheIdPolicy<1>>::Data {
+ MGetPropertyCache(MDefinition* obj, MDefinition* id)
+ : MBinaryInstruction(classOpcode, obj, id) {
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GetPropertyCache)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value), (1, idval))
+};
+
+class MGetPropSuperCache
+ : public MTernaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, BoxExceptPolicy<1, MIRType::Object>,
+ CacheIdPolicy<2>>::Data {
+ MGetPropSuperCache(MDefinition* obj, MDefinition* receiver, MDefinition* id)
+ : MTernaryInstruction(classOpcode, obj, receiver, id) {
+ setResultType(MIRType::Value);
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(GetPropSuperCache)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, receiver), (2, idval))
+};
+
+// Guard the object's proto is |expected|.
+class MGuardProto : public MBinaryInstruction, public SingleObjectPolicy::Data {
+ MGuardProto(MDefinition* obj, MDefinition* expected)
+ : MBinaryInstruction(classOpcode, obj, expected) {
+ MOZ_ASSERT(expected->isConstant() || expected->isNurseryObject());
+ setGuard();
+ setMovable();
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardProto)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, expected))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::ObjectFields);
+ }
+ AliasType mightAlias(const MDefinition* def) const override {
+ // These instructions never modify the [[Prototype]].
+ if (def->isAddAndStoreSlot() || def->isAllocateAndStoreSlot()) {
+ return AliasType::NoAlias;
+ }
+ return AliasType::MayAlias;
+ }
+};
+
+// Guard the object has no proto.
+class MGuardNullProto : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ explicit MGuardNullProto(MDefinition* obj)
+ : MUnaryInstruction(classOpcode, obj) {
+ setGuard();
+ setMovable();
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardNullProto)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::ObjectFields);
+ }
+ AliasType mightAlias(const MDefinition* def) const override {
+ // These instructions never modify the [[Prototype]].
+ if (def->isAddAndStoreSlot() || def->isAllocateAndStoreSlot()) {
+ return AliasType::NoAlias;
+ }
+ return AliasType::MayAlias;
+ }
+};
+
+// Guard on a specific Value.
+class MGuardValue : public MUnaryInstruction, public BoxInputsPolicy::Data {
+ Value expected_;
+
+ MGuardValue(MDefinition* val, const Value& expected)
+ : MUnaryInstruction(classOpcode, val), expected_(expected) {
+ MOZ_ASSERT(expected.isNullOrUndefined() || expected.isMagic() ||
+ expected.isPrivateGCThing());
+
+ setGuard();
+ setMovable();
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardValue)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value))
+
+ Value expected() const { return expected_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGuardValue()) {
+ return false;
+ }
+ if (expected() != ins->toGuardValue()->expected()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Guard on function flags
+class MGuardFunctionFlags : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ // At least one of the expected flags must be set, but not necessarily all
+ // expected flags.
+ uint16_t expectedFlags_;
+
+ // None of the unexpected flags must be set.
+ uint16_t unexpectedFlags_;
+
+ explicit MGuardFunctionFlags(MDefinition* fun, uint16_t expectedFlags,
+ uint16_t unexpectedFlags)
+ : MUnaryInstruction(classOpcode, fun),
+ expectedFlags_(expectedFlags),
+ unexpectedFlags_(unexpectedFlags) {
+ MOZ_ASSERT((expectedFlags & unexpectedFlags) == 0,
+ "Can't guard inconsistent flags");
+ MOZ_ASSERT((expectedFlags | unexpectedFlags) != 0,
+ "Can't guard zero flags");
+ setGuard();
+ setMovable();
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardFunctionFlags)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, function))
+
+ uint16_t expectedFlags() const { return expectedFlags_; };
+ uint16_t unexpectedFlags() const { return unexpectedFlags_; };
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGuardFunctionFlags()) {
+ return false;
+ }
+ if (expectedFlags() != ins->toGuardFunctionFlags()->expectedFlags()) {
+ return false;
+ }
+ if (unexpectedFlags() != ins->toGuardFunctionFlags()->unexpectedFlags()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::ObjectFields);
+ }
+};
+
+// Guard on an object's identity, inclusively or exclusively.
+class MGuardObjectIdentity : public MBinaryInstruction,
+ public SingleObjectPolicy::Data {
+ bool bailOnEquality_;
+
+ MGuardObjectIdentity(MDefinition* obj, MDefinition* expected,
+ bool bailOnEquality)
+ : MBinaryInstruction(classOpcode, obj, expected),
+ bailOnEquality_(bailOnEquality) {
+ MOZ_ASSERT(expected->isConstant() || expected->isNurseryObject());
+ setGuard();
+ setMovable();
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardObjectIdentity)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, expected))
+
+ bool bailOnEquality() const { return bailOnEquality_; }
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGuardObjectIdentity()) {
+ return false;
+ }
+ if (bailOnEquality() != ins->toGuardObjectIdentity()->bailOnEquality()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+// Guard on a specific JSFunction. Used instead of MGuardObjectIdentity,
+// so we can store some metadata related to the expected function.
+class MGuardSpecificFunction : public MBinaryInstruction,
+ public SingleObjectPolicy::Data {
+ uint16_t nargs_;
+ FunctionFlags flags_;
+
+ MGuardSpecificFunction(MDefinition* obj, MDefinition* expected,
+ uint16_t nargs, FunctionFlags flags)
+ : MBinaryInstruction(classOpcode, obj, expected),
+ nargs_(nargs),
+ flags_(flags) {
+ MOZ_ASSERT(expected->isConstant() || expected->isNurseryObject());
+ setGuard();
+ setMovable();
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardSpecificFunction)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, function), (1, expected))
+
+ uint16_t nargs() const { return nargs_; }
+ FunctionFlags flags() const { return flags_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGuardSpecificFunction()) {
+ return false;
+ }
+
+ auto* other = ins->toGuardSpecificFunction();
+ if (nargs() != other->nargs() ||
+ flags().toRaw() != other->flags().toRaw()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(other);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MGuardSpecificSymbol : public MUnaryInstruction,
+ public SymbolPolicy<0>::Data {
+ CompilerGCPointer<JS::Symbol*> expected_;
+
+ MGuardSpecificSymbol(MDefinition* symbol, JS::Symbol* expected)
+ : MUnaryInstruction(classOpcode, symbol), expected_(expected) {
+ setGuard();
+ setMovable();
+ setResultType(MIRType::Symbol);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardSpecificSymbol)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, symbol))
+
+ JS::Symbol* expected() const { return expected_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGuardSpecificSymbol()) {
+ return false;
+ }
+ if (expected() != ins->toGuardSpecificSymbol()->expected()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MGuardTagNotEqual
+ : public MBinaryInstruction,
+ public MixPolicy<UnboxedInt32Policy<0>, UnboxedInt32Policy<1>>::Data {
+ MGuardTagNotEqual(MDefinition* left, MDefinition* right)
+ : MBinaryInstruction(classOpcode, left, right) {
+ setGuard();
+ setMovable();
+ setCommutative();
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardTagNotEqual)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return binaryCongruentTo(ins);
+ }
+};
+
+// Load from vp[slot] (slots that are not inline in an object).
+class MLoadDynamicSlot : public MUnaryInstruction, public NoTypePolicy::Data {
+ uint32_t slot_;
+
+ MLoadDynamicSlot(MDefinition* slots, uint32_t slot)
+ : MUnaryInstruction(classOpcode, slots), slot_(slot) {
+ setResultType(MIRType::Value);
+ setMovable();
+ MOZ_ASSERT(slots->type() == MIRType::Slots);
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadDynamicSlot)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, slots))
+
+ uint32_t slot() const { return slot_; }
+
+ HashNumber valueHash() const override;
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadDynamicSlot()) {
+ return false;
+ }
+ if (slot() != ins->toLoadDynamicSlot()->slot()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ AliasSet getAliasSet() const override {
+ MOZ_ASSERT(slots()->type() == MIRType::Slots);
+ return AliasSet::Load(AliasSet::DynamicSlot);
+ }
+ AliasType mightAlias(const MDefinition* store) const override;
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ ALLOW_CLONE(MLoadDynamicSlot)
+};
+
+class MAddAndStoreSlot
+ : public MBinaryInstruction,
+ public MixPolicy<SingleObjectPolicy, BoxPolicy<1>>::Data {
+ public:
+ enum class Kind {
+ FixedSlot,
+ DynamicSlot,
+ };
+
+ private:
+ Kind kind_;
+ uint32_t slotOffset_;
+ CompilerShape shape_;
+
+ MAddAndStoreSlot(MDefinition* obj, MDefinition* value, Kind kind,
+ uint32_t slotOffset, Shape* shape)
+ : MBinaryInstruction(classOpcode, obj, value),
+ kind_(kind),
+ slotOffset_(slotOffset),
+ shape_(shape) {}
+
+ public:
+ INSTRUCTION_HEADER(AddAndStoreSlot)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, value))
+
+ Kind kind() const { return kind_; }
+ uint32_t slotOffset() const { return slotOffset_; }
+ Shape* shape() const { return shape_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::ObjectFields |
+ (kind() == Kind::FixedSlot ? AliasSet::FixedSlot
+ : AliasSet::DynamicSlot));
+ }
+};
+
+// Store to vp[slot] (slots that are not inline in an object).
+class MStoreDynamicSlot : public MBinaryInstruction,
+ public NoFloatPolicy<1>::Data {
+ uint32_t slot_;
+ bool needsBarrier_;
+
+ MStoreDynamicSlot(MDefinition* slots, uint32_t slot, MDefinition* value,
+ bool barrier)
+ : MBinaryInstruction(classOpcode, slots, value),
+ slot_(slot),
+ needsBarrier_(barrier) {
+ MOZ_ASSERT(slots->type() == MIRType::Slots);
+ }
+
+ public:
+ INSTRUCTION_HEADER(StoreDynamicSlot)
+ NAMED_OPERANDS((0, slots), (1, value))
+
+ static MStoreDynamicSlot* NewUnbarriered(TempAllocator& alloc,
+ MDefinition* slots, uint32_t slot,
+ MDefinition* value) {
+ return new (alloc) MStoreDynamicSlot(slots, slot, value, false);
+ }
+ static MStoreDynamicSlot* NewBarriered(TempAllocator& alloc,
+ MDefinition* slots, uint32_t slot,
+ MDefinition* value) {
+ return new (alloc) MStoreDynamicSlot(slots, slot, value, true);
+ }
+
+ uint32_t slot() const { return slot_; }
+ bool needsBarrier() const { return needsBarrier_; }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::DynamicSlot);
+ }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ ALLOW_CLONE(MStoreDynamicSlot)
+};
+
+class MSetPropertyCache : public MTernaryInstruction,
+ public MixPolicy<SingleObjectPolicy, CacheIdPolicy<1>,
+ NoFloatPolicy<2>>::Data {
+ bool strict_ : 1;
+
+ MSetPropertyCache(MDefinition* obj, MDefinition* id, MDefinition* value,
+ bool strict)
+ : MTernaryInstruction(classOpcode, obj, id, value), strict_(strict) {}
+
+ public:
+ INSTRUCTION_HEADER(SetPropertyCache)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, idval), (2, value))
+
+ bool strict() const { return strict_; }
+};
+
+class MMegamorphicSetElement : public MTernaryInstruction,
+ public MegamorphicSetElementPolicy::Data {
+ bool strict_;
+
+ MMegamorphicSetElement(MDefinition* object, MDefinition* index,
+ MDefinition* value, bool strict)
+ : MTernaryInstruction(classOpcode, object, index, value),
+ strict_(strict) {}
+
+ public:
+ INSTRUCTION_HEADER(MegamorphicSetElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, index), (2, value))
+
+ bool strict() const { return strict_; }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+class MSetDOMProperty : public MBinaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, BoxPolicy<1>>::Data {
+ const JSJitSetterOp func_;
+ Realm* setterRealm_;
+ DOMObjectKind objectKind_;
+
+ MSetDOMProperty(const JSJitSetterOp func, DOMObjectKind objectKind,
+ Realm* setterRealm, MDefinition* obj, MDefinition* val)
+ : MBinaryInstruction(classOpcode, obj, val),
+ func_(func),
+ setterRealm_(setterRealm),
+ objectKind_(objectKind) {}
+
+ public:
+ INSTRUCTION_HEADER(SetDOMProperty)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, value))
+
+ JSJitSetterOp fun() const { return func_; }
+ Realm* setterRealm() const { return setterRealm_; }
+ DOMObjectKind objectKind() const { return objectKind_; }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+class MGetDOMPropertyBase : public MVariadicInstruction,
+ public ObjectPolicy<0>::Data {
+ const JSJitInfo* info_;
+
+ protected:
+ MGetDOMPropertyBase(Opcode op, const JSJitInfo* jitinfo)
+ : MVariadicInstruction(op), info_(jitinfo) {
+ MOZ_ASSERT(jitinfo);
+ MOZ_ASSERT(jitinfo->type() == JSJitInfo::Getter);
+
+ // We are movable iff the jitinfo says we can be.
+ if (isDomMovable()) {
+ MOZ_ASSERT(jitinfo->aliasSet() != JSJitInfo::AliasEverything);
+ setMovable();
+ } else {
+ // If we're not movable, that means we shouldn't be DCEd either,
+ // because we might throw an exception when called, and getting rid
+ // of that is observable.
+ setGuard();
+ }
+
+ setResultType(MIRType::Value);
+ }
+
+ const JSJitInfo* info() const { return info_; }
+
+ [[nodiscard]] bool init(TempAllocator& alloc, MDefinition* obj,
+ MDefinition* guard, MDefinition* globalGuard) {
+ MOZ_ASSERT(obj);
+ // guard can be null.
+ // globalGuard can be null.
+ size_t operandCount = 1;
+ if (guard) {
+ ++operandCount;
+ }
+ if (globalGuard) {
+ ++operandCount;
+ }
+ if (!MVariadicInstruction::init(alloc, operandCount)) {
+ return false;
+ }
+ initOperand(0, obj);
+
+ size_t operandIndex = 1;
+ // Pin the guard, if we have one as an operand if we want to hoist later.
+ if (guard) {
+ initOperand(operandIndex++, guard);
+ }
+
+ // And the same for the global guard, if we have one.
+ if (globalGuard) {
+ initOperand(operandIndex, globalGuard);
+ }
+
+ return true;
+ }
+
+ public:
+ NAMED_OPERANDS((0, object))
+
+ JSJitGetterOp fun() const { return info_->getter; }
+ bool isInfallible() const { return info_->isInfallible; }
+ bool isDomMovable() const { return info_->isMovable; }
+ JSJitInfo::AliasSet domAliasSet() const { return info_->aliasSet(); }
+ size_t domMemberSlotIndex() const {
+ MOZ_ASSERT(info_->isAlwaysInSlot || info_->isLazilyCachedInSlot);
+ return info_->slotIndex;
+ }
+ bool valueMayBeInSlot() const { return info_->isLazilyCachedInSlot; }
+
+ bool baseCongruentTo(const MGetDOMPropertyBase* ins) const {
+ if (!isDomMovable()) {
+ return false;
+ }
+
+ // Checking the jitinfo is the same as checking the constant function
+ if (!(info() == ins->info())) {
+ return false;
+ }
+
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override {
+ JSJitInfo::AliasSet aliasSet = domAliasSet();
+ if (aliasSet == JSJitInfo::AliasNone) {
+ return AliasSet::None();
+ }
+ if (aliasSet == JSJitInfo::AliasDOMSets) {
+ return AliasSet::Load(AliasSet::DOMProperty);
+ }
+ MOZ_ASSERT(aliasSet == JSJitInfo::AliasEverything);
+ return AliasSet::Store(AliasSet::Any);
+ }
+};
+
+class MGetDOMProperty : public MGetDOMPropertyBase {
+ Realm* getterRealm_;
+ DOMObjectKind objectKind_;
+
+ MGetDOMProperty(const JSJitInfo* jitinfo, DOMObjectKind objectKind,
+ Realm* getterRealm)
+ : MGetDOMPropertyBase(classOpcode, jitinfo),
+ getterRealm_(getterRealm),
+ objectKind_(objectKind) {}
+
+ public:
+ INSTRUCTION_HEADER(GetDOMProperty)
+
+ static MGetDOMProperty* New(TempAllocator& alloc, const JSJitInfo* info,
+ DOMObjectKind objectKind, Realm* getterRealm,
+ MDefinition* obj, MDefinition* guard,
+ MDefinition* globalGuard) {
+ auto* res = new (alloc) MGetDOMProperty(info, objectKind, getterRealm);
+ if (!res || !res->init(alloc, obj, guard, globalGuard)) {
+ return nullptr;
+ }
+ return res;
+ }
+
+ Realm* getterRealm() const { return getterRealm_; }
+ DOMObjectKind objectKind() const { return objectKind_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGetDOMProperty()) {
+ return false;
+ }
+
+ if (ins->toGetDOMProperty()->getterRealm() != getterRealm()) {
+ return false;
+ }
+
+ return baseCongruentTo(ins->toGetDOMProperty());
+ }
+
+ bool possiblyCalls() const override { return true; }
+};
+
+class MGetDOMMember : public MGetDOMPropertyBase {
+ explicit MGetDOMMember(const JSJitInfo* jitinfo)
+ : MGetDOMPropertyBase(classOpcode, jitinfo) {
+ setResultType(MIRTypeFromValueType(jitinfo->returnType()));
+ }
+
+ public:
+ INSTRUCTION_HEADER(GetDOMMember)
+
+ static MGetDOMMember* New(TempAllocator& alloc, const JSJitInfo* info,
+ MDefinition* obj, MDefinition* guard,
+ MDefinition* globalGuard) {
+ auto* res = new (alloc) MGetDOMMember(info);
+ if (!res || !res->init(alloc, obj, guard, globalGuard)) {
+ return nullptr;
+ }
+ return res;
+ }
+
+ bool possiblyCalls() const override { return false; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGetDOMMember()) {
+ return false;
+ }
+
+ return baseCongruentTo(ins->toGetDOMMember());
+ }
+};
+
+class MLoadDOMExpandoValueGuardGeneration : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ JS::ExpandoAndGeneration* expandoAndGeneration_;
+ uint64_t generation_;
+
+ MLoadDOMExpandoValueGuardGeneration(
+ MDefinition* proxy, JS::ExpandoAndGeneration* expandoAndGeneration,
+ uint64_t generation)
+ : MUnaryInstruction(classOpcode, proxy),
+ expandoAndGeneration_(expandoAndGeneration),
+ generation_(generation) {
+ setGuard();
+ setMovable();
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(LoadDOMExpandoValueGuardGeneration)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, proxy))
+
+ JS::ExpandoAndGeneration* expandoAndGeneration() const {
+ return expandoAndGeneration_;
+ }
+ uint64_t generation() const { return generation_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isLoadDOMExpandoValueGuardGeneration()) {
+ return false;
+ }
+ const auto* other = ins->toLoadDOMExpandoValueGuardGeneration();
+ if (expandoAndGeneration() != other->expandoAndGeneration() ||
+ generation() != other->generation()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::DOMProxyExpando);
+ }
+};
+
+// Inlined assembly for Math.floor(double | float32) -> int32.
+class MFloor : public MUnaryInstruction, public FloatingPointPolicy<0>::Data {
+ explicit MFloor(MDefinition* num) : MUnaryInstruction(classOpcode, num) {
+ setResultType(MIRType::Int32);
+ specialization_ = MIRType::Double;
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Floor)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool isFloat32Commutative() const override { return true; }
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ void computeRange(TempAllocator& alloc) override;
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MFloor)
+};
+
+// Inlined assembly version for Math.ceil(double | float32) -> int32.
+class MCeil : public MUnaryInstruction, public FloatingPointPolicy<0>::Data {
+ explicit MCeil(MDefinition* num) : MUnaryInstruction(classOpcode, num) {
+ setResultType(MIRType::Int32);
+ specialization_ = MIRType::Double;
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Ceil)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool isFloat32Commutative() const override { return true; }
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ void computeRange(TempAllocator& alloc) override;
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MCeil)
+};
+
+// Inlined version of Math.round(double | float32) -> int32.
+class MRound : public MUnaryInstruction, public FloatingPointPolicy<0>::Data {
+ explicit MRound(MDefinition* num) : MUnaryInstruction(classOpcode, num) {
+ setResultType(MIRType::Int32);
+ specialization_ = MIRType::Double;
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Round)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool isFloat32Commutative() const override { return true; }
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MRound)
+};
+
+// Inlined version of Math.trunc(double | float32) -> int32.
+class MTrunc : public MUnaryInstruction, public FloatingPointPolicy<0>::Data {
+ explicit MTrunc(MDefinition* num) : MUnaryInstruction(classOpcode, num) {
+ setResultType(MIRType::Int32);
+ specialization_ = MIRType::Double;
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Trunc)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool isFloat32Commutative() const override { return true; }
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MTrunc)
+};
+
+// NearbyInt rounds the floating-point input to the nearest integer, according
+// to the RoundingMode.
+class MNearbyInt : public MUnaryInstruction,
+ public FloatingPointPolicy<0>::Data {
+ RoundingMode roundingMode_;
+
+ explicit MNearbyInt(MDefinition* num, MIRType resultType,
+ RoundingMode roundingMode)
+ : MUnaryInstruction(classOpcode, num), roundingMode_(roundingMode) {
+ MOZ_ASSERT(HasAssemblerSupport(roundingMode));
+
+ MOZ_ASSERT(IsFloatingPointType(resultType));
+ setResultType(resultType);
+ specialization_ = resultType;
+
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(NearbyInt)
+ TRIVIAL_NEW_WRAPPERS
+
+ static bool HasAssemblerSupport(RoundingMode mode) {
+ return Assembler::HasRoundInstruction(mode);
+ }
+
+ RoundingMode roundingMode() const { return roundingMode_; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool isFloat32Commutative() const override { return true; }
+ void trySpecializeFloat32(TempAllocator& alloc) override;
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+#endif
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toNearbyInt()->roundingMode() == roundingMode_;
+ }
+
+#ifdef JS_JITSPEW
+ void printOpcode(GenericPrinter& out) const override;
+#endif
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+
+ bool canRecoverOnBailout() const override {
+ switch (roundingMode_) {
+ case RoundingMode::Up:
+ case RoundingMode::Down:
+ case RoundingMode::TowardsZero:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ ALLOW_CLONE(MNearbyInt)
+};
+
+class MGetIteratorCache : public MUnaryInstruction,
+ public BoxExceptPolicy<0, MIRType::Object>::Data {
+ explicit MGetIteratorCache(MDefinition* val)
+ : MUnaryInstruction(classOpcode, val) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GetIteratorCache)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value))
+};
+
+// Implementation for 'in' operator using instruction cache
+class MInCache : public MBinaryInstruction,
+ public MixPolicy<CacheIdPolicy<0>, ObjectPolicy<1>>::Data {
+ MInCache(MDefinition* key, MDefinition* obj)
+ : MBinaryInstruction(classOpcode, key, obj) {
+ setResultType(MIRType::Boolean);
+ }
+
+ public:
+ INSTRUCTION_HEADER(InCache)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, key), (1, object))
+};
+
+// Test whether the index is in the array bounds or a hole.
+class MInArray : public MQuaternaryInstruction, public ObjectPolicy<3>::Data {
+ bool needsNegativeIntCheck_;
+
+ MInArray(MDefinition* elements, MDefinition* index, MDefinition* initLength,
+ MDefinition* object)
+ : MQuaternaryInstruction(classOpcode, elements, index, initLength,
+ object),
+ needsNegativeIntCheck_(true) {
+ setResultType(MIRType::Boolean);
+ setMovable();
+
+ // Set the guard flag to make sure we bail when we see a negative index.
+ // We can clear this flag (and needsNegativeIntCheck_) in
+ // collectRangeInfoPreTrunc.
+ setGuard();
+
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ MOZ_ASSERT(initLength->type() == MIRType::Int32);
+ }
+
+ public:
+ INSTRUCTION_HEADER(InArray)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, initLength), (3, object))
+
+ bool needsNegativeIntCheck() const { return needsNegativeIntCheck_; }
+ void collectRangeInfoPreTrunc() override;
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::Element);
+ }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isInArray()) {
+ return false;
+ }
+ const MInArray* other = ins->toInArray();
+ if (needsNegativeIntCheck() != other->needsNegativeIntCheck()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(other);
+ }
+};
+
+// Bail when the element is a hole.
+class MGuardElementNotHole : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ MGuardElementNotHole(MDefinition* elements, MDefinition* index)
+ : MBinaryInstruction(classOpcode, elements, index) {
+ setMovable();
+ setGuard();
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::Int32);
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardElementNotHole)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index))
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::Element);
+ }
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+};
+
+class MCheckPrivateFieldCache
+ : public MBinaryInstruction,
+ public MixPolicy<BoxExceptPolicy<0, MIRType::Object>,
+ CacheIdPolicy<1>>::Data {
+ MCheckPrivateFieldCache(MDefinition* obj, MDefinition* id)
+ : MBinaryInstruction(classOpcode, obj, id) {
+ setResultType(MIRType::Boolean);
+ }
+
+ public:
+ INSTRUCTION_HEADER(CheckPrivateFieldCache)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value), (1, idval))
+};
+
+class MHasOwnCache : public MBinaryInstruction,
+ public MixPolicy<BoxExceptPolicy<0, MIRType::Object>,
+ CacheIdPolicy<1>>::Data {
+ MHasOwnCache(MDefinition* obj, MDefinition* id)
+ : MBinaryInstruction(classOpcode, obj, id) {
+ setResultType(MIRType::Boolean);
+ }
+
+ public:
+ INSTRUCTION_HEADER(HasOwnCache)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value), (1, idval))
+};
+
+// Implementation for instanceof operator with specific rhs.
+class MInstanceOf : public MBinaryInstruction,
+ public MixPolicy<BoxExceptPolicy<0, MIRType::Object>,
+ ObjectPolicy<1>>::Data {
+ MInstanceOf(MDefinition* obj, MDefinition* proto)
+ : MBinaryInstruction(classOpcode, obj, proto) {
+ setResultType(MIRType::Boolean);
+ }
+
+ public:
+ INSTRUCTION_HEADER(InstanceOf)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+// Given a value being written to another object, update the generational store
+// buffer if the value is in the nursery and object is in the tenured heap.
+class MPostWriteBarrier : public MBinaryInstruction,
+ public ObjectPolicy<0>::Data {
+ MPostWriteBarrier(MDefinition* obj, MDefinition* value)
+ : MBinaryInstruction(classOpcode, obj, value) {
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(PostWriteBarrier)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, value))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override {
+ // During lowering, values that neither have object nor value MIR type
+ // are ignored, thus Float32 can show up at this point without any issue.
+ return use == getUseFor(1);
+ }
+#endif
+
+ ALLOW_CLONE(MPostWriteBarrier)
+};
+
+// Given a value being written to another object's elements at the specified
+// index, update the generational store buffer if the value is in the nursery
+// and object is in the tenured heap.
+class MPostWriteElementBarrier
+ : public MTernaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<2>>::Data {
+ MPostWriteElementBarrier(MDefinition* obj, MDefinition* value,
+ MDefinition* index)
+ : MTernaryInstruction(classOpcode, obj, value, index) {
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(PostWriteElementBarrier)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, value), (2, index))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override {
+ // During lowering, values that neither have object nor value MIR type
+ // are ignored, thus Float32 can show up at this point without any issue.
+ return use == getUseFor(1);
+ }
+#endif
+
+ ALLOW_CLONE(MPostWriteElementBarrier)
+};
+
+class MNewCallObject : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ public:
+ INSTRUCTION_HEADER(NewCallObject)
+ TRIVIAL_NEW_WRAPPERS
+
+ explicit MNewCallObject(MConstant* templateObj)
+ : MUnaryInstruction(classOpcode, templateObj) {
+ setResultType(MIRType::Object);
+ }
+
+ CallObject* templateObject() const {
+ return &getOperand(0)->toConstant()->toObject().as<CallObject>();
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+};
+
+class MNewStringObject : public MUnaryInstruction,
+ public ConvertToStringPolicy<0>::Data {
+ CompilerObject templateObj_;
+
+ MNewStringObject(MDefinition* input, JSObject* templateObj)
+ : MUnaryInstruction(classOpcode, input), templateObj_(templateObj) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ INSTRUCTION_HEADER(NewStringObject)
+ TRIVIAL_NEW_WRAPPERS
+
+ StringObject* templateObj() const;
+};
+
+// This is an alias for MLoadFixedSlot.
+class MEnclosingEnvironment : public MLoadFixedSlot {
+ explicit MEnclosingEnvironment(MDefinition* obj)
+ : MLoadFixedSlot(obj, EnvironmentObject::enclosingEnvironmentSlot()) {
+ setResultType(MIRType::Object);
+ }
+
+ public:
+ static MEnclosingEnvironment* New(TempAllocator& alloc, MDefinition* obj) {
+ return new (alloc) MEnclosingEnvironment(obj);
+ }
+
+ AliasSet getAliasSet() const override {
+ // EnvironmentObject reserved slots are immutable.
+ return AliasSet::None();
+ }
+};
+
+// This is an element of a spaghetti stack which is used to represent the memory
+// context which has to be restored in case of a bailout.
+struct MStoreToRecover : public TempObject,
+ public InlineSpaghettiStackNode<MStoreToRecover> {
+ MDefinition* operand;
+
+ explicit MStoreToRecover(MDefinition* operand) : operand(operand) {}
+};
+
+using MStoresToRecoverList = InlineSpaghettiStack<MStoreToRecover>;
+
+// A resume point contains the information needed to reconstruct the Baseline
+// Interpreter state from a position in Warp JIT code. A resume point is a
+// mapping of stack slots to MDefinitions.
+//
+// We capture stack state at critical points:
+// * (1) At the beginning of every basic block.
+// * (2) After every effectful operation.
+//
+// As long as these two properties are maintained, instructions can be moved,
+// hoisted, or, eliminated without problems, and ops without side effects do not
+// need to worry about capturing state at precisely the right point in time.
+//
+// Effectful instructions, of course, need to capture state after completion,
+// where the interpreter will not attempt to repeat the operation. For this,
+// ResumeAfter must be used. The state is attached directly to the effectful
+// instruction to ensure that no intermediate instructions could be injected
+// in between by a future analysis pass.
+//
+// During LIR construction, if an instruction can bail back to the interpreter,
+// we create an LSnapshot, which uses the last known resume point to request
+// register/stack assignments for every live value.
+class MResumePoint final : public MNode
+#ifdef DEBUG
+ ,
+ public InlineForwardListNode<MResumePoint>
+#endif
+{
+ private:
+ friend class MBasicBlock;
+ friend void AssertBasicGraphCoherency(MIRGraph& graph, bool force);
+
+ // List of stack slots needed to reconstruct the BaselineFrame.
+ FixedList<MUse> operands_;
+
+ // List of stores needed to reconstruct the content of objects which are
+ // emulated by EmulateStateOf variants.
+ MStoresToRecoverList stores_;
+
+ jsbytecode* pc_;
+ MInstruction* instruction_;
+ ResumeMode mode_;
+ bool isDiscarded_ = false;
+
+ MResumePoint(MBasicBlock* block, jsbytecode* pc, ResumeMode mode);
+ void inherit(MBasicBlock* state);
+
+ // Calling isDefinition or isResumePoint on MResumePoint is unnecessary.
+ bool isDefinition() const = delete;
+ bool isResumePoint() const = delete;
+
+ protected:
+ // Initializes operands_ to an empty array of a fixed length.
+ // The array may then be filled in by inherit().
+ [[nodiscard]] bool init(TempAllocator& alloc);
+
+ void clearOperand(size_t index) {
+ // FixedList doesn't initialize its elements, so do an unchecked init.
+ operands_[index].initUncheckedWithoutProducer(this);
+ }
+
+ MUse* getUseFor(size_t index) override { return &operands_[index]; }
+ const MUse* getUseFor(size_t index) const override {
+ return &operands_[index];
+ }
+
+ public:
+ static MResumePoint* New(TempAllocator& alloc, MBasicBlock* block,
+ jsbytecode* pc, ResumeMode mode);
+
+ MBasicBlock* block() const { return resumePointBlock(); }
+
+ size_t numAllocatedOperands() const { return operands_.length(); }
+ uint32_t stackDepth() const { return numAllocatedOperands(); }
+ size_t numOperands() const override { return numAllocatedOperands(); }
+ size_t indexOf(const MUse* u) const final {
+ MOZ_ASSERT(u >= &operands_[0]);
+ MOZ_ASSERT(u <= &operands_[numOperands() - 1]);
+ return u - &operands_[0];
+ }
+ void initOperand(size_t index, MDefinition* operand) {
+ // FixedList doesn't initialize its elements, so do an unchecked init.
+ operands_[index].initUnchecked(operand, this);
+ }
+ void replaceOperand(size_t index, MDefinition* operand) final {
+ operands_[index].replaceProducer(operand);
+ }
+
+ bool isObservableOperand(MUse* u) const;
+ bool isObservableOperand(size_t index) const;
+ bool isRecoverableOperand(MUse* u) const;
+
+ MDefinition* getOperand(size_t index) const override {
+ return operands_[index].producer();
+ }
+ jsbytecode* pc() const { return pc_; }
+ MResumePoint* caller() const;
+ uint32_t frameCount() const {
+ uint32_t count = 1;
+ for (MResumePoint* it = caller(); it; it = it->caller()) {
+ count++;
+ }
+ return count;
+ }
+ MInstruction* instruction() { return instruction_; }
+ void setInstruction(MInstruction* ins) {
+ MOZ_ASSERT(!instruction_);
+ instruction_ = ins;
+ }
+ void resetInstruction() {
+ MOZ_ASSERT(instruction_);
+ instruction_ = nullptr;
+ }
+ ResumeMode mode() const { return mode_; }
+
+ void releaseUses() {
+ for (size_t i = 0, e = numOperands(); i < e; i++) {
+ if (operands_[i].hasProducer()) {
+ operands_[i].releaseProducer();
+ }
+ }
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+
+ // Register a store instruction on the current resume point. This
+ // instruction would be recovered when we are bailing out. The |cache|
+ // argument can be any resume point, it is used to share memory if we are
+ // doing the same modification.
+ void addStore(TempAllocator& alloc, MDefinition* store,
+ const MResumePoint* cache = nullptr);
+
+ MStoresToRecoverList::iterator storesBegin() const { return stores_.begin(); }
+ MStoresToRecoverList::iterator storesEnd() const { return stores_.end(); }
+
+ void setDiscarded() { isDiscarded_ = true; }
+ bool isDiscarded() const { return isDiscarded_; }
+
+#ifdef JS_JITSPEW
+ virtual void dump(GenericPrinter& out) const override;
+ virtual void dump() const override;
+#endif
+};
+
+class MIsCallable : public MUnaryInstruction,
+ public BoxExceptPolicy<0, MIRType::Object>::Data {
+ explicit MIsCallable(MDefinition* object)
+ : MUnaryInstruction(classOpcode, object) {
+ setResultType(MIRType::Boolean);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(IsCallable)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MHasClass : public MUnaryInstruction, public SingleObjectPolicy::Data {
+ const JSClass* class_;
+
+ MHasClass(MDefinition* object, const JSClass* clasp)
+ : MUnaryInstruction(classOpcode, object), class_(clasp) {
+ MOZ_ASSERT(object->type() == MIRType::Object);
+ setResultType(MIRType::Boolean);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(HasClass)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ const JSClass* getClass() const { return class_; }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isHasClass()) {
+ return false;
+ }
+ if (getClass() != ins->toHasClass()->getClass()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+};
+
+class MGuardToClass : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ const JSClass* class_;
+
+ MGuardToClass(MDefinition* object, const JSClass* clasp)
+ : MUnaryInstruction(classOpcode, object), class_(clasp) {
+ MOZ_ASSERT(object->type() == MIRType::Object);
+ MOZ_ASSERT(!clasp->isJSFunction(), "Use MGuardToFunction instead");
+ setResultType(MIRType::Object);
+ setMovable();
+
+ // We will bail out if the class type is incorrect, so we need to ensure we
+ // don't eliminate this instruction
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardToClass)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ const JSClass* getClass() const { return class_; }
+ bool isArgumentsObjectClass() const {
+ return class_ == &MappedArgumentsObject::class_ ||
+ class_ == &UnmappedArgumentsObject::class_;
+ }
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGuardToClass()) {
+ return false;
+ }
+ if (getClass() != ins->toGuardToClass()->getClass()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+};
+
+class MGuardToFunction : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ explicit MGuardToFunction(MDefinition* object)
+ : MUnaryInstruction(classOpcode, object) {
+ MOZ_ASSERT(object->type() == MIRType::Object);
+ setResultType(MIRType::Object);
+ setMovable();
+
+ // We will bail out if the class type is incorrect, so we need to ensure we
+ // don't eliminate this instruction
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(GuardToFunction)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ if (!ins->isGuardToFunction()) {
+ return false;
+ }
+ return congruentIfOperandsEqual(ins);
+ }
+};
+
+// Note: we might call a proxy trap, so this instruction is effectful.
+class MIsArray : public MUnaryInstruction,
+ public BoxExceptPolicy<0, MIRType::Object>::Data {
+ explicit MIsArray(MDefinition* value)
+ : MUnaryInstruction(classOpcode, value) {
+ setResultType(MIRType::Boolean);
+ }
+
+ public:
+ INSTRUCTION_HEADER(IsArray)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value))
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+};
+
+class MIsTypedArray : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ bool possiblyWrapped_;
+
+ explicit MIsTypedArray(MDefinition* value, bool possiblyWrapped)
+ : MUnaryInstruction(classOpcode, value),
+ possiblyWrapped_(possiblyWrapped) {
+ setResultType(MIRType::Boolean);
+
+ if (possiblyWrapped) {
+ // Proxy checks may throw, so we're neither removable nor movable.
+ setGuard();
+ } else {
+ setMovable();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(IsTypedArray)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value))
+
+ bool isPossiblyWrapped() const { return possiblyWrapped_; }
+ AliasSet getAliasSet() const override {
+ if (isPossiblyWrapped()) {
+ return AliasSet::Store(AliasSet::Any);
+ }
+ return AliasSet::None();
+ }
+};
+
+// Allocate the generator object for a frame.
+class MGenerator : public MTernaryInstruction,
+ public MixPolicy<ObjectPolicy<0>, ObjectPolicy<1>>::Data {
+ explicit MGenerator(MDefinition* callee, MDefinition* environmentChain,
+ MDefinition* argsObject)
+ : MTernaryInstruction(classOpcode, callee, environmentChain, argsObject) {
+ setResultType(MIRType::Object);
+ };
+
+ public:
+ INSTRUCTION_HEADER(Generator)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, callee), (1, environmentChain), (2, argsObject))
+};
+
+class MMaybeExtractAwaitValue : public MBinaryInstruction,
+ public BoxPolicy<0>::Data {
+ explicit MMaybeExtractAwaitValue(MDefinition* value, MDefinition* canSkip)
+ : MBinaryInstruction(classOpcode, value, canSkip) {
+ setResultType(MIRType::Value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(MaybeExtractAwaitValue)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value), (1, canSkip))
+};
+
+class MAtomicIsLockFree : public MUnaryInstruction,
+ public ConvertToInt32Policy<0>::Data {
+ explicit MAtomicIsLockFree(MDefinition* value)
+ : MUnaryInstruction(classOpcode, value) {
+ setResultType(MIRType::Boolean);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(AtomicIsLockFree)
+ TRIVIAL_NEW_WRAPPERS
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ [[nodiscard]] bool writeRecoverData(
+ CompactBufferWriter& writer) const override;
+ bool canRecoverOnBailout() const override { return true; }
+
+ ALLOW_CLONE(MAtomicIsLockFree)
+};
+
+class MCompareExchangeTypedArrayElement
+ : public MQuaternaryInstruction,
+ public MixPolicy<TruncateToInt32OrToBigIntPolicy<2>,
+ TruncateToInt32OrToBigIntPolicy<3>>::Data {
+ Scalar::Type arrayType_;
+
+ explicit MCompareExchangeTypedArrayElement(MDefinition* elements,
+ MDefinition* index,
+ Scalar::Type arrayType,
+ MDefinition* oldval,
+ MDefinition* newval)
+ : MQuaternaryInstruction(classOpcode, elements, index, oldval, newval),
+ arrayType_(arrayType) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ setGuard(); // Not removable
+ }
+
+ public:
+ INSTRUCTION_HEADER(CompareExchangeTypedArrayElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, oldval), (3, newval))
+
+ bool isByteArray() const {
+ return (arrayType_ == Scalar::Int8 || arrayType_ == Scalar::Uint8);
+ }
+ Scalar::Type arrayType() const { return arrayType_; }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::UnboxedElement);
+ }
+};
+
+class MAtomicExchangeTypedArrayElement
+ : public MTernaryInstruction,
+ public TruncateToInt32OrToBigIntPolicy<2>::Data {
+ Scalar::Type arrayType_;
+
+ MAtomicExchangeTypedArrayElement(MDefinition* elements, MDefinition* index,
+ MDefinition* value, Scalar::Type arrayType)
+ : MTernaryInstruction(classOpcode, elements, index, value),
+ arrayType_(arrayType) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(arrayType <= Scalar::Uint32 || Scalar::isBigIntType(arrayType));
+ setGuard(); // Not removable
+ }
+
+ public:
+ INSTRUCTION_HEADER(AtomicExchangeTypedArrayElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, value))
+
+ bool isByteArray() const {
+ return (arrayType_ == Scalar::Int8 || arrayType_ == Scalar::Uint8);
+ }
+ Scalar::Type arrayType() const { return arrayType_; }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::UnboxedElement);
+ }
+};
+
+class MAtomicTypedArrayElementBinop
+ : public MTernaryInstruction,
+ public TruncateToInt32OrToBigIntPolicy<2>::Data {
+ private:
+ AtomicOp op_;
+ Scalar::Type arrayType_;
+ bool forEffect_;
+
+ explicit MAtomicTypedArrayElementBinop(AtomicOp op, MDefinition* elements,
+ MDefinition* index,
+ Scalar::Type arrayType,
+ MDefinition* value, bool forEffect)
+ : MTernaryInstruction(classOpcode, elements, index, value),
+ op_(op),
+ arrayType_(arrayType),
+ forEffect_(forEffect) {
+ MOZ_ASSERT(elements->type() == MIRType::Elements);
+ MOZ_ASSERT(index->type() == MIRType::IntPtr);
+ MOZ_ASSERT(arrayType <= Scalar::Uint32 || Scalar::isBigIntType(arrayType));
+ setGuard(); // Not removable
+ }
+
+ public:
+ INSTRUCTION_HEADER(AtomicTypedArrayElementBinop)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements), (1, index), (2, value))
+
+ bool isByteArray() const {
+ return (arrayType_ == Scalar::Int8 || arrayType_ == Scalar::Uint8);
+ }
+ AtomicOp operation() const { return op_; }
+ Scalar::Type arrayType() const { return arrayType_; }
+ bool isForEffect() const { return forEffect_; }
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::UnboxedElement);
+ }
+};
+
+class MDebugger : public MNullaryInstruction {
+ MDebugger() : MNullaryInstruction(classOpcode) {
+ setBailoutKind(BailoutKind::Debugger);
+ }
+
+ public:
+ INSTRUCTION_HEADER(Debugger)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+// Used to load the prototype of an object known to have
+// a static prototype.
+class MObjectStaticProto : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ explicit MObjectStaticProto(MDefinition* object)
+ : MUnaryInstruction(classOpcode, object) {
+ setResultType(MIRType::Object);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(ObjectStaticProto)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::ObjectFields);
+ }
+ AliasType mightAlias(const MDefinition* def) const override {
+ // These instructions never modify the [[Prototype]].
+ if (def->isAddAndStoreSlot() || def->isAllocateAndStoreSlot() ||
+ def->isStoreElementHole() || def->isArrayPush()) {
+ return AliasType::NoAlias;
+ }
+ return AliasType::MayAlias;
+ }
+};
+
+class MConstantProto : public MUnaryInstruction,
+ public SingleObjectPolicy::Data {
+ // NOTE: we're not going to actually use the underlying receiver object for
+ // anything. This is just here for giving extra information to MGuardShape
+ // to MGuardShape::mightAlias. Accordingly, we don't take it as an operand,
+ // but instead just keep a pointer to it. This means we need to ensure it's
+ // not discarded before we try to access it. If this is discarded, we
+ // basically just become an MConstant for the object's proto, which is fine.
+ MDefinition* receiverObject_;
+
+ explicit MConstantProto(MDefinition* protoObject, MDefinition* receiverObject)
+ : MUnaryInstruction(classOpcode, protoObject),
+ receiverObject_(receiverObject) {
+ MOZ_ASSERT(protoObject->isConstant());
+ setResultType(MIRType::Object);
+ setMovable();
+ }
+
+ ALLOW_CLONE(MConstantProto)
+
+ public:
+ INSTRUCTION_HEADER(ConstantProto)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, protoObject))
+
+ HashNumber valueHash() const override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ if (this == ins) {
+ return true;
+ }
+ const MDefinition* receiverObject = getReceiverObject();
+ return congruentIfOperandsEqual(ins) && receiverObject &&
+ receiverObject == ins->toConstantProto()->getReceiverObject();
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ const MDefinition* getReceiverObject() const {
+ if (receiverObject_->isDiscarded()) {
+ return nullptr;
+ }
+ return receiverObject_->skipObjectGuards();
+ }
+};
+
+// Flips the input's sign bit, independently of the rest of the number's
+// payload. Note this is different from multiplying by minus-one, which has
+// side-effects for e.g. NaNs.
+class MWasmNeg : public MUnaryInstruction, public NoTypePolicy::Data {
+ MWasmNeg(MDefinition* op, MIRType type) : MUnaryInstruction(classOpcode, op) {
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmNeg)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+// Machine-level bitwise AND/OR/XOR, avoiding all JS-level complexity embodied
+// in MBinaryBitwiseInstruction.
+class MWasmBinaryBitwise : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ public:
+ enum class SubOpcode { And, Or, Xor };
+
+ protected:
+ MWasmBinaryBitwise(MDefinition* left, MDefinition* right, MIRType type,
+ SubOpcode subOpcode)
+ : MBinaryInstruction(classOpcode, left, right), subOpcode_(subOpcode) {
+ MOZ_ASSERT(type == MIRType::Int32 || type == MIRType::Int64);
+ setResultType(type);
+ setMovable();
+ setCommutative();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBinaryBitwise)
+ TRIVIAL_NEW_WRAPPERS
+
+ SubOpcode subOpcode() const { return subOpcode_; }
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return ins->isWasmBinaryBitwise() &&
+ ins->toWasmBinaryBitwise()->subOpcode() == subOpcode() &&
+ binaryCongruentTo(ins);
+ }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+#ifdef JS_JITSPEW
+ void getExtras(ExtrasCollector* extras) override {
+ const char* what = "!!unknown!!";
+ switch (subOpcode()) {
+ case SubOpcode::And:
+ what = "And";
+ break;
+ case SubOpcode::Or:
+ what = "Or";
+ break;
+ case SubOpcode::Xor:
+ what = "Xor";
+ break;
+ }
+ extras->add(what);
+ }
+#endif
+
+ private:
+ SubOpcode subOpcode_;
+
+ ALLOW_CLONE(MWasmBinaryBitwise)
+};
+
+class MWasmLoadInstance : public MUnaryInstruction, public NoTypePolicy::Data {
+ uint32_t offset_;
+ AliasSet aliases_;
+
+ explicit MWasmLoadInstance(MDefinition* instance, uint32_t offset,
+ MIRType type, AliasSet aliases)
+ : MUnaryInstruction(classOpcode, instance),
+ offset_(offset),
+ aliases_(aliases) {
+ // Different instance data have different alias classes and only those
+ // classes are allowed.
+ MOZ_ASSERT(
+ aliases_.flags() == AliasSet::Load(AliasSet::WasmHeapMeta).flags() ||
+ aliases_.flags() == AliasSet::Load(AliasSet::WasmTableMeta).flags() ||
+ aliases_.flags() ==
+ AliasSet::Load(AliasSet::WasmPendingException).flags() ||
+ aliases_.flags() == AliasSet::None().flags());
+
+ // The only types supported at the moment.
+ MOZ_ASSERT(type == MIRType::Pointer || type == MIRType::Int32 ||
+ type == MIRType::Int64 || type == MIRType::RefOrNull);
+
+ setMovable();
+ setResultType(type);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmLoadInstance)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, instance))
+
+ uint32_t offset() const { return offset_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return op() == ins->op() &&
+ offset() == ins->toWasmLoadInstance()->offset() &&
+ type() == ins->type();
+ }
+
+ HashNumber valueHash() const override {
+ return addU32ToHash(HashNumber(op()), offset());
+ }
+
+ AliasSet getAliasSet() const override { return aliases_; }
+};
+
+class MWasmStoreInstance : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ uint32_t offset_;
+ AliasSet aliases_;
+
+ explicit MWasmStoreInstance(MDefinition* instance, MDefinition* value,
+ uint32_t offset, MIRType type, AliasSet aliases)
+ : MBinaryInstruction(classOpcode, instance, value),
+ offset_(offset),
+ aliases_(aliases) {
+ // Different instance data have different alias classes and only those
+ // classes are allowed.
+ MOZ_ASSERT(aliases_.flags() ==
+ AliasSet::Store(AliasSet::WasmPendingException).flags());
+
+ // The only types supported at the moment.
+ MOZ_ASSERT(type == MIRType::Pointer || type == MIRType::Int32 ||
+ type == MIRType::Int64 || type == MIRType::RefOrNull);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmStoreInstance)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, instance), (1, value))
+
+ uint32_t offset() const { return offset_; }
+
+ AliasSet getAliasSet() const override { return aliases_; }
+};
+
+class MWasmHeapBase : public MUnaryInstruction, public NoTypePolicy::Data {
+ AliasSet aliases_;
+
+ explicit MWasmHeapBase(MDefinition* instance, AliasSet aliases)
+ : MUnaryInstruction(classOpcode, instance), aliases_(aliases) {
+ setMovable();
+ setResultType(MIRType::Pointer);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmHeapBase)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, instance))
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return ins->isWasmHeapBase();
+ }
+
+ AliasSet getAliasSet() const override { return aliases_; }
+};
+
+// For memory32, bounds check nodes are of type Int32 on 32-bit systems for both
+// wasm and asm.js code, as well as on 64-bit systems for asm.js code and for
+// wasm code that is known to have a bounds check limit that fits into 32 bits.
+// They are of type Int64 only on 64-bit systems for wasm code with 4GB heaps.
+// There is no way for nodes of both types to be present in the same function.
+// Should this change, then BCE must be updated to take type into account.
+//
+// For memory64, bounds check nodes are always of type Int64.
+
+class MWasmBoundsCheck : public MBinaryInstruction, public NoTypePolicy::Data {
+ public:
+ enum Target {
+ // Linear memory at index zero, which is the only memory allowed so far.
+ Memory0,
+ // Everything else. Currently comprises tables, and arrays in the GC
+ // proposal.
+ Unknown
+ };
+
+ private:
+ wasm::BytecodeOffset bytecodeOffset_;
+ Target target_;
+
+ explicit MWasmBoundsCheck(MDefinition* index, MDefinition* boundsCheckLimit,
+ wasm::BytecodeOffset bytecodeOffset, Target target)
+ : MBinaryInstruction(classOpcode, index, boundsCheckLimit),
+ bytecodeOffset_(bytecodeOffset),
+ target_(target) {
+ MOZ_ASSERT(index->type() == boundsCheckLimit->type());
+
+ // Bounds check is effectful: it throws for OOB.
+ setGuard();
+
+ if (JitOptions.spectreIndexMasking) {
+ setResultType(index->type());
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBoundsCheck)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, index), (1, boundsCheckLimit))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool isMemory() const { return target_ == MWasmBoundsCheck::Memory0; }
+
+ bool isRedundant() const { return !isGuard(); }
+
+ void setRedundant() { setNotGuard(); }
+
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+};
+
+class MWasmAddOffset : public MUnaryInstruction, public NoTypePolicy::Data {
+ uint64_t offset_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ MWasmAddOffset(MDefinition* base, uint64_t offset,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MUnaryInstruction(classOpcode, base),
+ offset_(offset),
+ bytecodeOffset_(bytecodeOffset) {
+ setGuard();
+ MOZ_ASSERT(base->type() == MIRType::Int32 ||
+ base->type() == MIRType::Int64);
+ setResultType(base->type());
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmAddOffset)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, base))
+
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ uint64_t offset() const { return offset_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+};
+
+class MWasmAlignmentCheck : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ uint32_t byteSize_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ explicit MWasmAlignmentCheck(MDefinition* index, uint32_t byteSize,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MUnaryInstruction(classOpcode, index),
+ byteSize_(byteSize),
+ bytecodeOffset_(bytecodeOffset) {
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(byteSize));
+ // Alignment check is effectful: it throws for unaligned.
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmAlignmentCheck)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, index))
+
+ bool congruentTo(const MDefinition* ins) const override;
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ uint32_t byteSize() const { return byteSize_; }
+
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+};
+
+class MWasmLoad
+ : public MVariadicInstruction, // memoryBase is nullptr on some platforms
+ public NoTypePolicy::Data {
+ wasm::MemoryAccessDesc access_;
+
+ explicit MWasmLoad(const wasm::MemoryAccessDesc& access, MIRType resultType)
+ : MVariadicInstruction(classOpcode), access_(access) {
+ setGuard();
+ setResultType(resultType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmLoad)
+ NAMED_OPERANDS((0, base), (1, memoryBase));
+
+ static MWasmLoad* New(TempAllocator& alloc, MDefinition* memoryBase,
+ MDefinition* base, const wasm::MemoryAccessDesc& access,
+ MIRType resultType) {
+ MWasmLoad* load = new (alloc) MWasmLoad(access, resultType);
+ if (!load->init(alloc, 1 + !!memoryBase)) {
+ return nullptr;
+ }
+
+ load->initOperand(0, base);
+ if (memoryBase) {
+ load->initOperand(1, memoryBase);
+ }
+
+ return load;
+ }
+
+ const wasm::MemoryAccessDesc& access() const { return access_; }
+
+ AliasSet getAliasSet() const override {
+ // When a barrier is needed, make the instruction effectful by giving
+ // it a "store" effect.
+ if (access_.isAtomic()) {
+ return AliasSet::Store(AliasSet::WasmHeap);
+ }
+ return AliasSet::Load(AliasSet::WasmHeap);
+ }
+
+#ifdef JS_JITSPEW
+ void getExtras(ExtrasCollector* extras) override {
+ char buf[64];
+ SprintfLiteral(buf, "(offs=%lld)", (long long int)access().offset64());
+ extras->add(buf);
+ }
+#endif
+};
+
+class MWasmStore : public MVariadicInstruction, public NoTypePolicy::Data {
+ wasm::MemoryAccessDesc access_;
+
+ explicit MWasmStore(const wasm::MemoryAccessDesc& access)
+ : MVariadicInstruction(classOpcode), access_(access) {
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmStore)
+ NAMED_OPERANDS((0, base), (1, value), (2, memoryBase))
+
+ static MWasmStore* New(TempAllocator& alloc, MDefinition* memoryBase,
+ MDefinition* base,
+ const wasm::MemoryAccessDesc& access,
+ MDefinition* value) {
+ MWasmStore* store = new (alloc) MWasmStore(access);
+ if (!store->init(alloc, 2 + !!memoryBase)) {
+ return nullptr;
+ }
+
+ store->initOperand(0, base);
+ store->initOperand(1, value);
+ if (memoryBase) {
+ store->initOperand(2, memoryBase);
+ }
+
+ return store;
+ }
+
+ const wasm::MemoryAccessDesc& access() const { return access_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmHeap);
+ }
+
+#ifdef JS_JITSPEW
+ void getExtras(ExtrasCollector* extras) override {
+ char buf[64];
+ SprintfLiteral(buf, "(offs=%lld)", (long long int)access().offset64());
+ extras->add(buf);
+ }
+#endif
+};
+
+class MAsmJSMemoryAccess {
+ Scalar::Type accessType_;
+ bool needsBoundsCheck_;
+
+ public:
+ explicit MAsmJSMemoryAccess(Scalar::Type accessType)
+ : accessType_(accessType), needsBoundsCheck_(true) {
+ MOZ_ASSERT(accessType != Scalar::Uint8Clamped);
+ }
+
+ Scalar::Type accessType() const { return accessType_; }
+ unsigned byteSize() const { return TypedArrayElemSize(accessType()); }
+ bool needsBoundsCheck() const { return needsBoundsCheck_; }
+
+ wasm::MemoryAccessDesc access() const {
+ return wasm::MemoryAccessDesc(accessType_, Scalar::byteSize(accessType_), 0,
+ wasm::BytecodeOffset());
+ }
+
+ void removeBoundsCheck() { needsBoundsCheck_ = false; }
+};
+
+class MAsmJSLoadHeap
+ : public MVariadicInstruction, // 1 plus optional memoryBase and
+ // boundsCheckLimit
+ public MAsmJSMemoryAccess,
+ public NoTypePolicy::Data {
+ uint32_t memoryBaseIndex_;
+
+ explicit MAsmJSLoadHeap(uint32_t memoryBaseIndex, Scalar::Type accessType)
+ : MVariadicInstruction(classOpcode),
+ MAsmJSMemoryAccess(accessType),
+ memoryBaseIndex_(memoryBaseIndex) {
+ setResultType(ScalarTypeToMIRType(accessType));
+ }
+
+ public:
+ INSTRUCTION_HEADER(AsmJSLoadHeap)
+ NAMED_OPERANDS((0, base), (1, boundsCheckLimit))
+
+ static MAsmJSLoadHeap* New(TempAllocator& alloc, MDefinition* memoryBase,
+ MDefinition* base, MDefinition* boundsCheckLimit,
+ Scalar::Type accessType) {
+ uint32_t nextIndex = 2;
+ uint32_t memoryBaseIndex = memoryBase ? nextIndex++ : UINT32_MAX;
+
+ MAsmJSLoadHeap* load =
+ new (alloc) MAsmJSLoadHeap(memoryBaseIndex, accessType);
+ if (!load->init(alloc, nextIndex)) {
+ return nullptr;
+ }
+
+ load->initOperand(0, base);
+ load->initOperand(1, boundsCheckLimit);
+ if (memoryBase) {
+ load->initOperand(memoryBaseIndex, memoryBase);
+ }
+
+ return load;
+ }
+
+ bool hasMemoryBase() const { return memoryBaseIndex_ != UINT32_MAX; }
+ MDefinition* memoryBase() const {
+ MOZ_ASSERT(hasMemoryBase());
+ return getOperand(memoryBaseIndex_);
+ }
+
+ bool congruentTo(const MDefinition* ins) const override;
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::WasmHeap);
+ }
+ AliasType mightAlias(const MDefinition* def) const override;
+};
+
+class MAsmJSStoreHeap
+ : public MVariadicInstruction, // 2 plus optional memoryBase and
+ // boundsCheckLimit
+ public MAsmJSMemoryAccess,
+ public NoTypePolicy::Data {
+ uint32_t memoryBaseIndex_;
+
+ explicit MAsmJSStoreHeap(uint32_t memoryBaseIndex, Scalar::Type accessType)
+ : MVariadicInstruction(classOpcode),
+ MAsmJSMemoryAccess(accessType),
+ memoryBaseIndex_(memoryBaseIndex) {}
+
+ public:
+ INSTRUCTION_HEADER(AsmJSStoreHeap)
+ NAMED_OPERANDS((0, base), (1, value), (2, boundsCheckLimit))
+
+ static MAsmJSStoreHeap* New(TempAllocator& alloc, MDefinition* memoryBase,
+ MDefinition* base, MDefinition* boundsCheckLimit,
+ Scalar::Type accessType, MDefinition* v) {
+ uint32_t nextIndex = 3;
+ uint32_t memoryBaseIndex = memoryBase ? nextIndex++ : UINT32_MAX;
+
+ MAsmJSStoreHeap* store =
+ new (alloc) MAsmJSStoreHeap(memoryBaseIndex, accessType);
+ if (!store->init(alloc, nextIndex)) {
+ return nullptr;
+ }
+
+ store->initOperand(0, base);
+ store->initOperand(1, v);
+ store->initOperand(2, boundsCheckLimit);
+ if (memoryBase) {
+ store->initOperand(memoryBaseIndex, memoryBase);
+ }
+
+ return store;
+ }
+
+ bool hasMemoryBase() const { return memoryBaseIndex_ != UINT32_MAX; }
+ MDefinition* memoryBase() const {
+ MOZ_ASSERT(hasMemoryBase());
+ return getOperand(memoryBaseIndex_);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmHeap);
+ }
+};
+
+class MWasmCompareExchangeHeap : public MVariadicInstruction,
+ public NoTypePolicy::Data {
+ wasm::MemoryAccessDesc access_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ explicit MWasmCompareExchangeHeap(const wasm::MemoryAccessDesc& access,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MVariadicInstruction(classOpcode),
+ access_(access),
+ bytecodeOffset_(bytecodeOffset) {
+ setGuard(); // Not removable
+ setResultType(ScalarTypeToMIRType(access.type()));
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmCompareExchangeHeap)
+ NAMED_OPERANDS((0, base), (1, oldValue), (2, newValue), (3, instance),
+ (4, memoryBase))
+
+ static MWasmCompareExchangeHeap* New(TempAllocator& alloc,
+ wasm::BytecodeOffset bytecodeOffset,
+ MDefinition* memoryBase,
+ MDefinition* base,
+ const wasm::MemoryAccessDesc& access,
+ MDefinition* oldv, MDefinition* newv,
+ MDefinition* instance) {
+ MWasmCompareExchangeHeap* cas =
+ new (alloc) MWasmCompareExchangeHeap(access, bytecodeOffset);
+ if (!cas->init(alloc, 4 + !!memoryBase)) {
+ return nullptr;
+ }
+ cas->initOperand(0, base);
+ cas->initOperand(1, oldv);
+ cas->initOperand(2, newv);
+ cas->initOperand(3, instance);
+ if (memoryBase) {
+ cas->initOperand(4, memoryBase);
+ }
+ return cas;
+ }
+
+ const wasm::MemoryAccessDesc& access() const { return access_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmHeap);
+ }
+};
+
+class MWasmAtomicExchangeHeap : public MVariadicInstruction,
+ public NoTypePolicy::Data {
+ wasm::MemoryAccessDesc access_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ explicit MWasmAtomicExchangeHeap(const wasm::MemoryAccessDesc& access,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MVariadicInstruction(classOpcode),
+ access_(access),
+ bytecodeOffset_(bytecodeOffset) {
+ setGuard(); // Not removable
+ setResultType(ScalarTypeToMIRType(access.type()));
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmAtomicExchangeHeap)
+ NAMED_OPERANDS((0, base), (1, value), (2, instance), (3, memoryBase))
+
+ static MWasmAtomicExchangeHeap* New(TempAllocator& alloc,
+ wasm::BytecodeOffset bytecodeOffset,
+ MDefinition* memoryBase,
+ MDefinition* base,
+ const wasm::MemoryAccessDesc& access,
+ MDefinition* value,
+ MDefinition* instance) {
+ MWasmAtomicExchangeHeap* xchg =
+ new (alloc) MWasmAtomicExchangeHeap(access, bytecodeOffset);
+ if (!xchg->init(alloc, 3 + !!memoryBase)) {
+ return nullptr;
+ }
+
+ xchg->initOperand(0, base);
+ xchg->initOperand(1, value);
+ xchg->initOperand(2, instance);
+ if (memoryBase) {
+ xchg->initOperand(3, memoryBase);
+ }
+
+ return xchg;
+ }
+
+ const wasm::MemoryAccessDesc& access() const { return access_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmHeap);
+ }
+};
+
+class MWasmAtomicBinopHeap : public MVariadicInstruction,
+ public NoTypePolicy::Data {
+ AtomicOp op_;
+ wasm::MemoryAccessDesc access_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ explicit MWasmAtomicBinopHeap(AtomicOp op,
+ const wasm::MemoryAccessDesc& access,
+ wasm::BytecodeOffset bytecodeOffset)
+ : MVariadicInstruction(classOpcode),
+ op_(op),
+ access_(access),
+ bytecodeOffset_(bytecodeOffset) {
+ setGuard(); // Not removable
+ setResultType(ScalarTypeToMIRType(access.type()));
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmAtomicBinopHeap)
+ NAMED_OPERANDS((0, base), (1, value), (2, instance), (3, memoryBase))
+
+ static MWasmAtomicBinopHeap* New(TempAllocator& alloc,
+ wasm::BytecodeOffset bytecodeOffset,
+ AtomicOp op, MDefinition* memoryBase,
+ MDefinition* base,
+ const wasm::MemoryAccessDesc& access,
+ MDefinition* v, MDefinition* instance) {
+ MWasmAtomicBinopHeap* binop =
+ new (alloc) MWasmAtomicBinopHeap(op, access, bytecodeOffset);
+ if (!binop->init(alloc, 3 + !!memoryBase)) {
+ return nullptr;
+ }
+
+ binop->initOperand(0, base);
+ binop->initOperand(1, v);
+ binop->initOperand(2, instance);
+ if (memoryBase) {
+ binop->initOperand(3, memoryBase);
+ }
+
+ return binop;
+ }
+
+ AtomicOp operation() const { return op_; }
+ const wasm::MemoryAccessDesc& access() const { return access_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmHeap);
+ }
+};
+
+class MWasmLoadGlobalVar : public MUnaryInstruction, public NoTypePolicy::Data {
+ MWasmLoadGlobalVar(MIRType type, unsigned globalDataOffset, bool isConstant,
+ MDefinition* instance)
+ : MUnaryInstruction(classOpcode, instance),
+ globalDataOffset_(globalDataOffset),
+ isConstant_(isConstant) {
+ MOZ_ASSERT(IsNumberType(type) || type == MIRType::Simd128 ||
+ type == MIRType::Pointer || type == MIRType::RefOrNull);
+ setResultType(type);
+ setMovable();
+ }
+
+ unsigned globalDataOffset_;
+ bool isConstant_;
+
+ public:
+ INSTRUCTION_HEADER(WasmLoadGlobalVar)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, instance))
+
+ unsigned globalDataOffset() const { return globalDataOffset_; }
+
+ HashNumber valueHash() const override;
+ bool congruentTo(const MDefinition* ins) const override;
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ AliasSet getAliasSet() const override {
+ return isConstant_ ? AliasSet::None()
+ : AliasSet::Load(AliasSet::WasmGlobalVar);
+ }
+
+ AliasType mightAlias(const MDefinition* def) const override;
+};
+
+class MWasmLoadGlobalCell : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ MWasmLoadGlobalCell(MIRType type, MDefinition* cellPtr)
+ : MUnaryInstruction(classOpcode, cellPtr) {
+ setResultType(type);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmLoadGlobalCell)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, cellPtr))
+
+ // The default valueHash is good enough, because there are no non-operand
+ // fields.
+ bool congruentTo(const MDefinition* ins) const override;
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::WasmGlobalCell);
+ }
+
+ AliasType mightAlias(const MDefinition* def) const override;
+};
+
+class MWasmLoadTableElement : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ MWasmLoadTableElement(MDefinition* elements, MDefinition* index)
+ : MBinaryInstruction(classOpcode, elements, index) {
+ setResultType(MIRType::RefOrNull);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmLoadTableElement)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, elements))
+ NAMED_OPERANDS((1, index))
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::WasmTableElement);
+ }
+};
+
+class MWasmStoreGlobalVar : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ MWasmStoreGlobalVar(unsigned globalDataOffset, MDefinition* value,
+ MDefinition* instance)
+ : MBinaryInstruction(classOpcode, value, instance),
+ globalDataOffset_(globalDataOffset) {}
+
+ unsigned globalDataOffset_;
+
+ public:
+ INSTRUCTION_HEADER(WasmStoreGlobalVar)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value), (1, instance))
+
+ unsigned globalDataOffset() const { return globalDataOffset_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmGlobalVar);
+ }
+};
+
+class MWasmStoreGlobalCell : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ MWasmStoreGlobalCell(MDefinition* value, MDefinition* cellPtr)
+ : MBinaryInstruction(classOpcode, value, cellPtr) {}
+
+ public:
+ INSTRUCTION_HEADER(WasmStoreGlobalCell)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, value), (1, cellPtr))
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmGlobalCell);
+ }
+};
+
+class MWasmStoreStackResult : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ MWasmStoreStackResult(MDefinition* stackResultArea, uint32_t offset,
+ MDefinition* value)
+ : MBinaryInstruction(classOpcode, stackResultArea, value),
+ offset_(offset) {}
+
+ uint32_t offset_;
+
+ public:
+ INSTRUCTION_HEADER(WasmStoreStackResult)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, stackResultArea), (1, value))
+
+ uint32_t offset() const { return offset_; }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmStackResult);
+ }
+};
+
+// Represents a known-good derived pointer into an object or memory region (in
+// the most general sense) that will not move while the derived pointer is live.
+// The `offset` *must* be a valid offset into the object represented by `base`;
+// hence overflow in the address calculation will never be an issue.
+
+class MWasmDerivedPointer : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ MWasmDerivedPointer(MDefinition* base, size_t offset)
+ : MUnaryInstruction(classOpcode, base), offset_(offset) {
+ MOZ_ASSERT(offset <= INT32_MAX);
+ setResultType(MIRType::Pointer);
+ setMovable();
+ }
+
+ size_t offset_;
+
+ public:
+ INSTRUCTION_HEADER(WasmDerivedPointer)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, base))
+
+ size_t offset() const { return offset_; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toWasmDerivedPointer()->offset() == offset();
+ }
+
+#ifdef JS_JITSPEW
+ void getExtras(ExtrasCollector* extras) override {
+ char buf[64];
+ SprintfLiteral(buf, "(offs=%lld)", (long long int)offset_);
+ extras->add(buf);
+ }
+#endif
+
+ ALLOW_CLONE(MWasmDerivedPointer)
+};
+
+class MWasmDerivedIndexPointer : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ MWasmDerivedIndexPointer(MDefinition* base, MDefinition* index, Scale scale)
+ : MBinaryInstruction(classOpcode, base, index), scale_(scale) {
+ setResultType(MIRType::Pointer);
+ setMovable();
+ }
+
+ Scale scale_;
+
+ public:
+ INSTRUCTION_HEADER(WasmDerivedIndexPointer)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, base))
+ NAMED_OPERANDS((1, index))
+
+ Scale scale() const { return scale_; }
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toWasmDerivedIndexPointer()->scale() == scale();
+ }
+
+ ALLOW_CLONE(MWasmDerivedIndexPointer)
+};
+
+// Stores a reference to an address. This performs a pre-barrier on the address,
+// but not a post-barrier. A post-barrier must be performed separately, if it's
+// required.
+
+class MWasmStoreRef : public MAryInstruction<3>, public NoTypePolicy::Data {
+ AliasSet::Flag aliasSet_;
+
+ MWasmStoreRef(MDefinition* instance, MDefinition* valueAddr,
+ MDefinition* value, AliasSet::Flag aliasSet)
+ : MAryInstruction<3>(classOpcode), aliasSet_(aliasSet) {
+ MOZ_ASSERT(valueAddr->type() == MIRType::Pointer);
+ MOZ_ASSERT(value->type() == MIRType::RefOrNull);
+ initOperand(0, instance);
+ initOperand(1, valueAddr);
+ initOperand(2, value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmStoreRef)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, instance), (1, valueAddr), (2, value))
+
+ AliasSet getAliasSet() const override { return AliasSet::Store(aliasSet_); }
+};
+
+class MWasmParameter : public MNullaryInstruction {
+ ABIArg abi_;
+
+ MWasmParameter(ABIArg abi, MIRType mirType)
+ : MNullaryInstruction(classOpcode), abi_(abi) {
+ setResultType(mirType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmParameter)
+ TRIVIAL_NEW_WRAPPERS
+
+ ABIArg abi() const { return abi_; }
+};
+
+class MWasmReturn : public MAryControlInstruction<2, 0>,
+ public NoTypePolicy::Data {
+ MWasmReturn(MDefinition* ins, MDefinition* instance)
+ : MAryControlInstruction(classOpcode) {
+ initOperand(0, ins);
+ initOperand(1, instance);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmReturn)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+class MWasmReturnVoid : public MAryControlInstruction<1, 0>,
+ public NoTypePolicy::Data {
+ explicit MWasmReturnVoid(MDefinition* instance)
+ : MAryControlInstruction(classOpcode) {
+ initOperand(0, instance);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmReturnVoid)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+class MWasmStackArg : public MUnaryInstruction, public NoTypePolicy::Data {
+ MWasmStackArg(uint32_t spOffset, MDefinition* ins)
+ : MUnaryInstruction(classOpcode, ins), spOffset_(spOffset) {}
+
+ uint32_t spOffset_;
+
+ public:
+ INSTRUCTION_HEADER(WasmStackArg)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, arg))
+
+ uint32_t spOffset() const { return spOffset_; }
+ void incrementOffset(uint32_t inc) { spOffset_ += inc; }
+};
+
+template <typename Location>
+class MWasmResultBase : public MNullaryInstruction {
+ Location loc_;
+
+ protected:
+ MWasmResultBase(Opcode op, MIRType type, Location loc)
+ : MNullaryInstruction(op), loc_(loc) {
+ setResultType(type);
+ setCallResultCapture();
+ }
+
+ public:
+ Location loc() { return loc_; }
+};
+
+class MWasmRegisterResult : public MWasmResultBase<Register> {
+ MWasmRegisterResult(MIRType type, Register reg)
+ : MWasmResultBase(classOpcode, type, reg) {}
+
+ public:
+ INSTRUCTION_HEADER(WasmRegisterResult)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+class MWasmFloatRegisterResult : public MWasmResultBase<FloatRegister> {
+ MWasmFloatRegisterResult(MIRType type, FloatRegister reg)
+ : MWasmResultBase(classOpcode, type, reg) {}
+
+ public:
+ INSTRUCTION_HEADER(WasmFloatRegisterResult)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+class MWasmRegister64Result : public MWasmResultBase<Register64> {
+ explicit MWasmRegister64Result(Register64 reg)
+ : MWasmResultBase(classOpcode, MIRType::Int64, reg) {}
+
+ public:
+ INSTRUCTION_HEADER(WasmRegister64Result)
+ TRIVIAL_NEW_WRAPPERS
+};
+
+class MWasmStackResultArea : public MNullaryInstruction {
+ public:
+ class StackResult {
+ // Offset in bytes from lowest address of stack result area.
+ uint32_t offset_;
+ MIRType type_;
+
+ public:
+ StackResult() : type_(MIRType::Undefined) {}
+ StackResult(uint32_t offset, MIRType type) : offset_(offset), type_(type) {}
+
+ bool initialized() const { return type_ != MIRType::Undefined; }
+ uint32_t offset() const {
+ MOZ_ASSERT(initialized());
+ return offset_;
+ }
+ MIRType type() const {
+ MOZ_ASSERT(initialized());
+ return type_;
+ }
+ uint32_t endOffset() const {
+ return offset() + wasm::MIRTypeToABIResultSize(type());
+ }
+ };
+
+ private:
+ FixedList<StackResult> results_;
+ uint32_t base_;
+
+ explicit MWasmStackResultArea()
+ : MNullaryInstruction(classOpcode), base_(UINT32_MAX) {
+ setResultType(MIRType::StackResults);
+ }
+
+ void assertInitialized() const {
+ MOZ_ASSERT(results_.length() != 0);
+#ifdef DEBUG
+ for (size_t i = 0; i < results_.length(); i++) {
+ MOZ_ASSERT(results_[i].initialized());
+ }
+#endif
+ }
+
+ bool baseInitialized() const { return base_ != UINT32_MAX; }
+
+ public:
+ INSTRUCTION_HEADER(WasmStackResultArea)
+ TRIVIAL_NEW_WRAPPERS
+
+ [[nodiscard]] bool init(TempAllocator& alloc, size_t stackResultCount) {
+ MOZ_ASSERT(results_.length() == 0);
+ MOZ_ASSERT(stackResultCount > 0);
+ if (!results_.init(alloc, stackResultCount)) {
+ return false;
+ }
+ for (size_t n = 0; n < stackResultCount; n++) {
+ results_[n] = StackResult();
+ }
+ return true;
+ }
+
+ size_t resultCount() const { return results_.length(); }
+ const StackResult& result(size_t n) const {
+ MOZ_ASSERT(results_[n].initialized());
+ return results_[n];
+ }
+ void initResult(size_t n, const StackResult& loc) {
+ MOZ_ASSERT(!results_[n].initialized());
+ MOZ_ASSERT((n == 0) == (loc.offset() == 0));
+ MOZ_ASSERT_IF(n > 0, loc.offset() >= result(n - 1).endOffset());
+ results_[n] = loc;
+ }
+
+ uint32_t byteSize() const {
+ assertInitialized();
+ return result(resultCount() - 1).endOffset();
+ }
+
+ // Stack index indicating base of stack area.
+ uint32_t base() const {
+ MOZ_ASSERT(baseInitialized());
+ return base_;
+ }
+ void setBase(uint32_t base) {
+ MOZ_ASSERT(!baseInitialized());
+ base_ = base;
+ MOZ_ASSERT(baseInitialized());
+ }
+};
+
+class MWasmStackResult : public MUnaryInstruction, public NoTypePolicy::Data {
+ uint32_t resultIdx_;
+
+ MWasmStackResult(MWasmStackResultArea* resultArea, size_t idx)
+ : MUnaryInstruction(classOpcode, resultArea), resultIdx_(idx) {
+ setResultType(result().type());
+ setCallResultCapture();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmStackResult)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, resultArea))
+
+ const MWasmStackResultArea::StackResult& result() const {
+ return resultArea()->toWasmStackResultArea()->result(resultIdx_);
+ }
+};
+
+// Arguments for constructing a catchable wasm call inside of a try block.
+struct MWasmCallTryDesc {
+ bool inTry;
+ uint32_t relativeTryDepth;
+ size_t tryNoteIndex;
+ MBasicBlock* fallthroughBlock;
+ MBasicBlock* prePadBlock;
+
+ MWasmCallTryDesc()
+ : inTry(false),
+ relativeTryDepth(0),
+ tryNoteIndex(0),
+ fallthroughBlock(nullptr),
+ prePadBlock(nullptr) {}
+};
+
+// Mixin class for wasm calls that may or may not be catchable.
+class MWasmCallBase {
+ public:
+ struct Arg {
+ AnyRegister reg;
+ MDefinition* def;
+ Arg(AnyRegister reg, MDefinition* def) : reg(reg), def(def) {}
+ };
+ typedef Vector<Arg, 8, SystemAllocPolicy> Args;
+
+ protected:
+ wasm::CallSiteDesc desc_;
+ wasm::CalleeDesc callee_;
+ wasm::FailureMode builtinMethodFailureMode_;
+ FixedList<AnyRegister> argRegs_;
+ uint32_t stackArgAreaSizeUnaligned_;
+ ABIArg instanceArg_;
+ bool inTry_;
+ size_t tryNoteIndex_;
+
+ MWasmCallBase(const wasm::CallSiteDesc& desc, const wasm::CalleeDesc& callee,
+ uint32_t stackArgAreaSizeUnaligned, bool inTry,
+ size_t tryNoteIndex)
+ : desc_(desc),
+ callee_(callee),
+ builtinMethodFailureMode_(wasm::FailureMode::Infallible),
+ stackArgAreaSizeUnaligned_(stackArgAreaSizeUnaligned),
+ inTry_(inTry),
+ tryNoteIndex_(tryNoteIndex) {}
+
+ template <class MVariadicT>
+ [[nodiscard]] bool initWithArgs(TempAllocator& alloc, MVariadicT* ins,
+ const Args& args,
+ MDefinition* tableIndexOrRef) {
+ if (!argRegs_.init(alloc, args.length())) {
+ return false;
+ }
+ for (size_t i = 0; i < argRegs_.length(); i++) {
+ argRegs_[i] = args[i].reg;
+ }
+
+ if (!ins->init(alloc, argRegs_.length() + (tableIndexOrRef ? 1 : 0))) {
+ return false;
+ }
+ // FixedList doesn't initialize its elements, so do an unchecked init.
+ for (size_t i = 0; i < argRegs_.length(); i++) {
+ ins->initOperand(i, args[i].def);
+ }
+ if (tableIndexOrRef) {
+ ins->initOperand(argRegs_.length(), tableIndexOrRef);
+ }
+ return true;
+ }
+
+ public:
+ static bool IsWasmCall(MDefinition* def) {
+ return def->isWasmCallCatchable() || def->isWasmCallUncatchable();
+ }
+
+ size_t numArgs() const { return argRegs_.length(); }
+ AnyRegister registerForArg(size_t index) const {
+ MOZ_ASSERT(index < numArgs());
+ return argRegs_[index];
+ }
+ const wasm::CallSiteDesc& desc() const { return desc_; }
+ const wasm::CalleeDesc& callee() const { return callee_; }
+ wasm::FailureMode builtinMethodFailureMode() const {
+ MOZ_ASSERT(callee_.which() == wasm::CalleeDesc::BuiltinInstanceMethod);
+ return builtinMethodFailureMode_;
+ }
+ uint32_t stackArgAreaSizeUnaligned() const {
+ return stackArgAreaSizeUnaligned_;
+ }
+
+ const ABIArg& instanceArg() const { return instanceArg_; }
+
+ bool inTry() const { return inTry_; }
+ size_t tryNoteIndex() const { return tryNoteIndex_; }
+};
+
+// A wasm call that is catchable. This instruction is a control instruction,
+// and terminates the block it is on. A normal return will proceed in a the
+// fallthrough block. An exceptional return will unwind into the landing pad
+// block for this call. The landing pad block must begin with an
+// MWasmCallLandingPrePad.
+class MWasmCallCatchable final : public MVariadicControlInstruction<2>,
+ public MWasmCallBase,
+ public NoTypePolicy::Data {
+ MWasmCallCatchable(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee,
+ uint32_t stackArgAreaSizeUnaligned, size_t tryNoteIndex)
+ : MVariadicControlInstruction(classOpcode),
+ MWasmCallBase(desc, callee, stackArgAreaSizeUnaligned, true,
+ tryNoteIndex) {}
+
+ public:
+ INSTRUCTION_HEADER(WasmCallCatchable)
+
+ static MWasmCallCatchable* New(TempAllocator& alloc,
+ const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee,
+ const Args& args,
+ uint32_t stackArgAreaSizeUnaligned,
+ const MWasmCallTryDesc& tryDesc,
+ MDefinition* tableIndexOrRef = nullptr);
+
+ bool possiblyCalls() const override { return true; }
+
+ static const size_t FallthroughBranchIndex = 0;
+ static const size_t PrePadBranchIndex = 1;
+};
+
+// A wasm call that is not catchable. This instruction is not a control
+// instruction, and therefore is not a block terminator.
+class MWasmCallUncatchable final : public MVariadicInstruction,
+ public MWasmCallBase,
+ public NoTypePolicy::Data {
+ MWasmCallUncatchable(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee,
+ uint32_t stackArgAreaSizeUnaligned)
+ : MVariadicInstruction(classOpcode),
+ MWasmCallBase(desc, callee, stackArgAreaSizeUnaligned, false, 0) {}
+
+ public:
+ INSTRUCTION_HEADER(WasmCallUncatchable)
+
+ static MWasmCallUncatchable* New(TempAllocator& alloc,
+ const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee,
+ const Args& args,
+ uint32_t stackArgAreaSizeUnaligned,
+ MDefinition* tableIndexOrRef = nullptr);
+
+ static MWasmCallUncatchable* NewBuiltinInstanceMethodCall(
+ TempAllocator& alloc, const wasm::CallSiteDesc& desc,
+ const wasm::SymbolicAddress builtin, wasm::FailureMode failureMode,
+ const ABIArg& instanceArg, const Args& args,
+ uint32_t stackArgAreaSizeUnaligned);
+
+ bool possiblyCalls() const override { return true; }
+};
+
+// A marker instruction for a block which is the landing pad for a catchable
+// wasm call. This instruction does not emit any code, only filling in
+// metadata. This instruction must be the first instruction added to the
+// landing pad block.
+class MWasmCallLandingPrePad : public MNullaryInstruction {
+ // The block of the call that may unwind to this landing pad.
+ MBasicBlock* callBlock_;
+ // The index of the try note to initialize a landing pad for.
+ size_t tryNoteIndex_;
+
+ explicit MWasmCallLandingPrePad(MBasicBlock* callBlock, size_t tryNoteIndex)
+ : MNullaryInstruction(classOpcode),
+ callBlock_(callBlock),
+ tryNoteIndex_(tryNoteIndex) {
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmCallLandingPrePad)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ size_t tryNoteIndex() { return tryNoteIndex_; }
+ MBasicBlock* callBlock() { return callBlock_; }
+};
+
+class MWasmSelect : public MTernaryInstruction, public NoTypePolicy::Data {
+ MWasmSelect(MDefinition* trueExpr, MDefinition* falseExpr,
+ MDefinition* condExpr)
+ : MTernaryInstruction(classOpcode, trueExpr, falseExpr, condExpr) {
+ MOZ_ASSERT(condExpr->type() == MIRType::Int32);
+ MOZ_ASSERT(trueExpr->type() == falseExpr->type());
+ setResultType(trueExpr->type());
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmSelect)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, trueExpr), (1, falseExpr), (2, condExpr))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ ALLOW_CLONE(MWasmSelect)
+};
+
+class MWasmReinterpret : public MUnaryInstruction, public NoTypePolicy::Data {
+ MWasmReinterpret(MDefinition* val, MIRType toType)
+ : MUnaryInstruction(classOpcode, val) {
+ switch (val->type()) {
+ case MIRType::Int32:
+ MOZ_ASSERT(toType == MIRType::Float32);
+ break;
+ case MIRType::Float32:
+ MOZ_ASSERT(toType == MIRType::Int32);
+ break;
+ case MIRType::Double:
+ MOZ_ASSERT(toType == MIRType::Int64);
+ break;
+ case MIRType::Int64:
+ MOZ_ASSERT(toType == MIRType::Double);
+ break;
+ default:
+ MOZ_CRASH("unexpected reinterpret conversion");
+ }
+ setMovable();
+ setResultType(toType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmReinterpret)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+
+ ALLOW_CLONE(MWasmReinterpret)
+};
+
+class MRotate : public MBinaryInstruction, public NoTypePolicy::Data {
+ bool isLeftRotate_;
+
+ MRotate(MDefinition* input, MDefinition* count, MIRType type,
+ bool isLeftRotate)
+ : MBinaryInstruction(classOpcode, input, count),
+ isLeftRotate_(isLeftRotate) {
+ setMovable();
+ setResultType(type);
+ // Prevent reordering. Although there's no problem eliding call result
+ // definitions, there's also no need, as they cause no codegen.
+ setGuard();
+ }
+
+ public:
+ INSTRUCTION_HEADER(Rotate)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, input), (1, count))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toRotate()->isLeftRotate() == isLeftRotate_;
+ }
+
+ bool isLeftRotate() const { return isLeftRotate_; }
+
+ ALLOW_CLONE(MRotate)
+};
+
+// Wasm SIMD.
+//
+// See comment in WasmIonCompile.cpp for a justification for these nodes.
+
+// (v128, v128, v128) -> v128 effect-free operation.
+class MWasmTernarySimd128 : public MTernaryInstruction,
+ public NoTypePolicy::Data {
+ wasm::SimdOp simdOp_;
+
+ MWasmTernarySimd128(MDefinition* v0, MDefinition* v1, MDefinition* v2,
+ wasm::SimdOp simdOp)
+ : MTernaryInstruction(classOpcode, v0, v1, v2), simdOp_(simdOp) {
+ setMovable();
+ setResultType(MIRType::Simd128);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmTernarySimd128)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, v0), (1, v1), (2, v2))
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ // If the control mask of a bitselect allows the operation to be specialized
+ // as a shuffle and it is profitable to specialize it on this platform, return
+ // true and the appropriate shuffle mask.
+ bool specializeBitselectConstantMaskAsShuffle(int8_t shuffle[16]);
+ // Checks if more relaxed version of lane select can be used. It returns true
+ // if a bit mask input expected to be all 0s or 1s for entire 8-bit lanes,
+ // false otherwise.
+ bool canRelaxBitselect();
+#endif
+
+ wasm::SimdOp simdOp() const { return simdOp_; }
+
+ ALLOW_CLONE(MWasmTernarySimd128)
+};
+
+// (v128, v128) -> v128 effect-free operations.
+class MWasmBinarySimd128 : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ wasm::SimdOp simdOp_;
+
+ MWasmBinarySimd128(MDefinition* lhs, MDefinition* rhs, bool commutative,
+ wasm::SimdOp simdOp)
+ : MBinaryInstruction(classOpcode, lhs, rhs), simdOp_(simdOp) {
+ setMovable();
+ setResultType(MIRType::Simd128);
+ if (commutative) {
+ setCommutative();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBinarySimd128)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ return ins->toWasmBinarySimd128()->simdOp() == simdOp_ &&
+ congruentIfOperandsEqual(ins);
+ }
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+
+ // Checks if pmaddubsw operation is supported.
+ bool canPmaddubsw();
+#endif
+
+ wasm::SimdOp simdOp() const { return simdOp_; }
+
+ // Platform-dependent specialization.
+ bool specializeForConstantRhs();
+
+ ALLOW_CLONE(MWasmBinarySimd128)
+};
+
+// (v128, const) -> v128 effect-free operations.
+class MWasmBinarySimd128WithConstant : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ SimdConstant rhs_;
+ wasm::SimdOp simdOp_;
+
+ MWasmBinarySimd128WithConstant(MDefinition* lhs, const SimdConstant& rhs,
+ wasm::SimdOp simdOp)
+ : MUnaryInstruction(classOpcode, lhs), rhs_(rhs), simdOp_(simdOp) {
+ setMovable();
+ setResultType(MIRType::Simd128);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmBinarySimd128WithConstant)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ return ins->toWasmBinarySimd128WithConstant()->simdOp() == simdOp_ &&
+ congruentIfOperandsEqual(ins) &&
+ rhs_.bitwiseEqual(ins->toWasmBinarySimd128WithConstant()->rhs());
+ }
+
+ wasm::SimdOp simdOp() const { return simdOp_; }
+ MDefinition* lhs() const { return input(); }
+ const SimdConstant& rhs() const { return rhs_; }
+
+ ALLOW_CLONE(MWasmBinarySimd128WithConstant)
+};
+
+// (v128, scalar, imm) -> v128 effect-free operations.
+class MWasmReplaceLaneSimd128 : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ uint32_t laneIndex_;
+ wasm::SimdOp simdOp_;
+
+ MWasmReplaceLaneSimd128(MDefinition* lhs, MDefinition* rhs,
+ uint32_t laneIndex, wasm::SimdOp simdOp)
+ : MBinaryInstruction(classOpcode, lhs, rhs),
+ laneIndex_(laneIndex),
+ simdOp_(simdOp) {
+ setMovable();
+ setResultType(MIRType::Simd128);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmReplaceLaneSimd128)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ return ins->toWasmReplaceLaneSimd128()->simdOp() == simdOp_ &&
+ ins->toWasmReplaceLaneSimd128()->laneIndex() == laneIndex_ &&
+ congruentIfOperandsEqual(ins);
+ }
+
+ uint32_t laneIndex() const { return laneIndex_; }
+ wasm::SimdOp simdOp() const { return simdOp_; }
+
+ ALLOW_CLONE(MWasmReplaceLaneSimd128)
+};
+
+// (scalar) -> v128 effect-free operations.
+class MWasmScalarToSimd128 : public MUnaryInstruction,
+ public NoTypePolicy::Data {
+ wasm::SimdOp simdOp_;
+
+ MWasmScalarToSimd128(MDefinition* src, wasm::SimdOp simdOp)
+ : MUnaryInstruction(classOpcode, src), simdOp_(simdOp) {
+ setMovable();
+ setResultType(MIRType::Simd128);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmScalarToSimd128)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ return ins->toWasmScalarToSimd128()->simdOp() == simdOp_ &&
+ congruentIfOperandsEqual(ins);
+ }
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+#endif
+
+ wasm::SimdOp simdOp() const { return simdOp_; }
+
+ ALLOW_CLONE(MWasmScalarToSimd128)
+};
+
+// (v128, imm) -> scalar effect-free operations.
+class MWasmReduceSimd128 : public MUnaryInstruction, public NoTypePolicy::Data {
+ wasm::SimdOp simdOp_;
+ uint32_t imm_;
+
+ MWasmReduceSimd128(MDefinition* src, wasm::SimdOp simdOp, MIRType outType,
+ uint32_t imm)
+ : MUnaryInstruction(classOpcode, src), simdOp_(simdOp), imm_(imm) {
+ setMovable();
+ setResultType(outType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmReduceSimd128)
+ TRIVIAL_NEW_WRAPPERS
+
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+ bool congruentTo(const MDefinition* ins) const override {
+ return ins->toWasmReduceSimd128()->simdOp() == simdOp_ &&
+ ins->toWasmReduceSimd128()->imm() == imm_ &&
+ congruentIfOperandsEqual(ins);
+ }
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* foldsTo(TempAllocator& alloc) override;
+#endif
+
+ uint32_t imm() const { return imm_; }
+ wasm::SimdOp simdOp() const { return simdOp_; }
+
+ ALLOW_CLONE(MWasmReduceSimd128)
+};
+
+class MWasmLoadLaneSimd128
+ : public MVariadicInstruction, // memoryBase is nullptr on some platforms
+ public NoTypePolicy::Data {
+ wasm::MemoryAccessDesc access_;
+ uint32_t laneSize_;
+ uint32_t laneIndex_;
+ uint32_t memoryBaseIndex_;
+
+ MWasmLoadLaneSimd128(const wasm::MemoryAccessDesc& access, uint32_t laneSize,
+ uint32_t laneIndex, uint32_t memoryBaseIndex)
+ : MVariadicInstruction(classOpcode),
+ access_(access),
+ laneSize_(laneSize),
+ laneIndex_(laneIndex),
+ memoryBaseIndex_(memoryBaseIndex) {
+ MOZ_ASSERT(!access_.isAtomic());
+ setGuard();
+ setResultType(MIRType::Simd128);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmLoadLaneSimd128)
+ NAMED_OPERANDS((0, base), (1, value));
+
+ static MWasmLoadLaneSimd128* New(TempAllocator& alloc,
+ MDefinition* memoryBase, MDefinition* base,
+ const wasm::MemoryAccessDesc& access,
+ uint32_t laneSize, uint32_t laneIndex,
+ MDefinition* value) {
+ uint32_t nextIndex = 2;
+ uint32_t memoryBaseIndex = memoryBase ? nextIndex++ : UINT32_MAX;
+
+ MWasmLoadLaneSimd128* load = new (alloc)
+ MWasmLoadLaneSimd128(access, laneSize, laneIndex, memoryBaseIndex);
+ if (!load->init(alloc, nextIndex)) {
+ return nullptr;
+ }
+
+ load->initOperand(0, base);
+ load->initOperand(1, value);
+ if (memoryBase) {
+ load->initOperand(memoryBaseIndex, memoryBase);
+ }
+
+ return load;
+ }
+
+ const wasm::MemoryAccessDesc& access() const { return access_; }
+ uint32_t laneSize() const { return laneSize_; }
+ uint32_t laneIndex() const { return laneIndex_; }
+ bool hasMemoryBase() const { return memoryBaseIndex_ != UINT32_MAX; }
+ MDefinition* memoryBase() const {
+ MOZ_ASSERT(hasMemoryBase());
+ return getOperand(memoryBaseIndex_);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Load(AliasSet::WasmHeap);
+ }
+};
+
+class MWasmStoreLaneSimd128 : public MVariadicInstruction,
+ public NoTypePolicy::Data {
+ wasm::MemoryAccessDesc access_;
+ uint32_t laneSize_;
+ uint32_t laneIndex_;
+ uint32_t memoryBaseIndex_;
+
+ explicit MWasmStoreLaneSimd128(const wasm::MemoryAccessDesc& access,
+ uint32_t laneSize, uint32_t laneIndex,
+ uint32_t memoryBaseIndex)
+ : MVariadicInstruction(classOpcode),
+ access_(access),
+ laneSize_(laneSize),
+ laneIndex_(laneIndex),
+ memoryBaseIndex_(memoryBaseIndex) {
+ MOZ_ASSERT(!access_.isAtomic());
+ setGuard();
+ setResultType(MIRType::Simd128);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmStoreLaneSimd128)
+ NAMED_OPERANDS((0, base), (1, value))
+
+ static MWasmStoreLaneSimd128* New(TempAllocator& alloc,
+ MDefinition* memoryBase, MDefinition* base,
+ const wasm::MemoryAccessDesc& access,
+ uint32_t laneSize, uint32_t laneIndex,
+ MDefinition* value) {
+ uint32_t nextIndex = 2;
+ uint32_t memoryBaseIndex = memoryBase ? nextIndex++ : UINT32_MAX;
+
+ MWasmStoreLaneSimd128* store = new (alloc)
+ MWasmStoreLaneSimd128(access, laneSize, laneIndex, memoryBaseIndex);
+ if (!store->init(alloc, nextIndex)) {
+ return nullptr;
+ }
+
+ store->initOperand(0, base);
+ store->initOperand(1, value);
+ if (memoryBase) {
+ store->initOperand(memoryBaseIndex, memoryBase);
+ }
+
+ return store;
+ }
+
+ const wasm::MemoryAccessDesc& access() const { return access_; }
+ uint32_t laneSize() const { return laneSize_; }
+ uint32_t laneIndex() const { return laneIndex_; }
+ bool hasMemoryBase() const { return memoryBaseIndex_ != UINT32_MAX; }
+ MDefinition* memoryBase() const {
+ MOZ_ASSERT(hasMemoryBase());
+ return getOperand(memoryBaseIndex_);
+ }
+
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::WasmHeap);
+ }
+};
+
+// End Wasm SIMD
+
+// Used by MIR building to represent the bytecode result of an operation for
+// which an MBail was generated, to balance the basic block's MDefinition stack.
+class MUnreachableResult : public MNullaryInstruction {
+ explicit MUnreachableResult(MIRType type) : MNullaryInstruction(classOpcode) {
+ MOZ_ASSERT(type != MIRType::None);
+ setResultType(type);
+ }
+
+ public:
+ INSTRUCTION_HEADER(UnreachableResult)
+ TRIVIAL_NEW_WRAPPERS
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins);
+ }
+ AliasSet getAliasSet() const override { return AliasSet::None(); }
+};
+
+class MIonToWasmCall final : public MVariadicInstruction,
+ public NoTypePolicy::Data {
+ CompilerGCPointer<WasmInstanceObject*> instanceObj_;
+ const wasm::FuncExport& funcExport_;
+
+ MIonToWasmCall(WasmInstanceObject* instanceObj, MIRType resultType,
+ const wasm::FuncExport& funcExport)
+ : MVariadicInstruction(classOpcode),
+ instanceObj_(instanceObj),
+ funcExport_(funcExport) {
+ setResultType(resultType);
+ }
+
+ public:
+ INSTRUCTION_HEADER(IonToWasmCall);
+
+ static MIonToWasmCall* New(TempAllocator& alloc,
+ WasmInstanceObject* instanceObj,
+ const wasm::FuncExport& funcExport);
+
+ void initArg(size_t i, MDefinition* arg) { initOperand(i, arg); }
+
+ WasmInstanceObject* instanceObject() const { return instanceObj_; }
+ wasm::Instance* instance() const { return &instanceObj_->instance(); }
+ const wasm::FuncExport& funcExport() const { return funcExport_; }
+ bool possiblyCalls() const override { return true; }
+#ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override;
+#endif
+};
+
+// For accesses to wasm object fields, we need to be able to describe 8- and
+// 16-bit accesses. But MIRType can't represent those. Hence these two
+// supplemental enums, used for reading and writing fields respectively.
+
+// Indicates how to widen an 8- or 16-bit value (when it is read from memory).
+enum class MWideningOp : uint8_t { None, FromU16, FromS16, FromU8, FromS8 };
+
+// Indicates how to narrow a 32-bit value (when it is written to memory). The
+// operation is a simple truncate.
+enum class MNarrowingOp : uint8_t { None, To16, To8 };
+
+// Provide information about potential trap at the instruction machine code,
+// e.g. null pointer dereference.
+struct TrapSiteInfo {
+ wasm::BytecodeOffset offset;
+ explicit TrapSiteInfo(wasm::BytecodeOffset offset_) : offset(offset_) {}
+};
+
+typedef mozilla::Maybe<TrapSiteInfo> MaybeTrapSiteInfo;
+
+// Load an object field stored at a fixed offset from a base pointer. This
+// field may be any value type, including references. No barriers are
+// performed.
+class MWasmLoadField : public MUnaryInstruction, public NoTypePolicy::Data {
+ uint32_t offset_;
+ MWideningOp wideningOp_;
+ AliasSet aliases_;
+ MaybeTrapSiteInfo maybeTrap_;
+
+ MWasmLoadField(MDefinition* obj, uint32_t offset, MIRType type,
+ MWideningOp wideningOp, AliasSet aliases,
+ MaybeTrapSiteInfo maybeTrap = mozilla::Nothing())
+ : MUnaryInstruction(classOpcode, obj),
+ offset_(offset),
+ wideningOp_(wideningOp),
+ aliases_(aliases),
+ maybeTrap_(maybeTrap) {
+ // "if you want to widen the value when it is loaded, the destination type
+ // must be Int32".
+ MOZ_ASSERT_IF(wideningOp != MWideningOp::None, type == MIRType::Int32);
+ MOZ_ASSERT(
+ aliases.flags() ==
+ AliasSet::Load(AliasSet::WasmStructOutlineDataPointer).flags() ||
+ aliases.flags() ==
+ AliasSet::Load(AliasSet::WasmArrayNumElements).flags() ||
+ aliases.flags() ==
+ AliasSet::Load(AliasSet::WasmArrayDataPointer).flags() ||
+ aliases.flags() == AliasSet::Load(AliasSet::Any).flags());
+ setResultType(type);
+ if (maybeTrap_) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmLoadField)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, obj))
+
+ uint32_t offset() const { return offset_; }
+ MWideningOp wideningOp() const { return wideningOp_; }
+ MaybeTrapSiteInfo maybeTrap() const { return maybeTrap_; }
+ AliasSet getAliasSet() const override { return aliases_; }
+ bool congruentTo(const MDefinition* ins) const override {
+ // In the limited case where this insn is used to read
+ // WasmStructObject::outlineData_ (the field itself, not what it points
+ // at), we allow commoning up to happen. This is OK because
+ // WasmStructObject::outlineData_ is readonly for the life of the
+ // WasmStructObject.
+ if (!ins->isWasmLoadField()) {
+ return false;
+ }
+ const MWasmLoadField* other = ins->toWasmLoadField();
+ return ins->isWasmLoadField() && congruentIfOperandsEqual(ins) &&
+ offset() == other->offset() && wideningOp() == other->wideningOp() &&
+ getAliasSet().flags() == other->getAliasSet().flags() &&
+ getAliasSet().flags() ==
+ AliasSet::Load(AliasSet::WasmStructOutlineDataPointer).flags();
+ }
+};
+
+// Load a object field stored at a fixed offset from a base pointer. This
+// field may be any value type, including references. No barriers are
+// performed.
+//
+// This instruction takes a pointer to a second object `ka`, which it is
+// necessary to keep alive. It is expected that `ka` holds a reference to
+// `obj`, but this is not enforced and no code is generated to access `ka`.
+// This instruction extends the lifetime of `ka` so that it, and hence `obj`,
+// cannot be collected while `obj` is live. This is necessary if `obj` does
+// not point to a GC-managed object.
+class MWasmLoadFieldKA : public MBinaryInstruction, public NoTypePolicy::Data {
+ uint32_t offset_;
+ MWideningOp wideningOp_;
+ AliasSet aliases_;
+ MaybeTrapSiteInfo maybeTrap_;
+
+ MWasmLoadFieldKA(MDefinition* ka, MDefinition* obj, uint32_t offset,
+ MIRType type, MWideningOp wideningOp, AliasSet aliases,
+ MaybeTrapSiteInfo maybeTrap = mozilla::Nothing())
+ : MBinaryInstruction(classOpcode, ka, obj),
+ offset_(offset),
+ wideningOp_(wideningOp),
+ aliases_(aliases),
+ maybeTrap_(maybeTrap) {
+ MOZ_ASSERT_IF(wideningOp != MWideningOp::None, type == MIRType::Int32);
+ MOZ_ASSERT(
+ aliases.flags() ==
+ AliasSet::Load(AliasSet::WasmStructInlineDataArea).flags() ||
+ aliases.flags() ==
+ AliasSet::Load(AliasSet::WasmStructOutlineDataArea).flags() ||
+ aliases.flags() ==
+ AliasSet::Load(AliasSet::WasmArrayDataArea).flags() ||
+ aliases.flags() == AliasSet::Load(AliasSet::Any).flags());
+ setResultType(type);
+ if (maybeTrap_) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmLoadFieldKA)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, ka), (1, obj))
+
+ uint32_t offset() const { return offset_; }
+ MWideningOp wideningOp() const { return wideningOp_; }
+ MaybeTrapSiteInfo maybeTrap() const { return maybeTrap_; }
+
+ AliasSet getAliasSet() const override { return aliases_; }
+};
+
+// Store a value to an object field at a fixed offset from a base pointer.
+// This field may be any value type, _excluding_ references. References
+// _must_ use the 'Ref' variant of this instruction.
+//
+// This instruction takes a second object `ka` that must be kept alive, as
+// described for MWasmLoadFieldKA above.
+class MWasmStoreFieldKA : public MTernaryInstruction,
+ public NoTypePolicy::Data {
+ uint32_t offset_;
+ MNarrowingOp narrowingOp_;
+ AliasSet aliases_;
+ MaybeTrapSiteInfo maybeTrap_;
+
+ MWasmStoreFieldKA(MDefinition* ka, MDefinition* obj, uint32_t offset,
+ MDefinition* value, MNarrowingOp narrowingOp,
+ AliasSet aliases,
+ MaybeTrapSiteInfo maybeTrap = mozilla::Nothing())
+ : MTernaryInstruction(classOpcode, ka, obj, value),
+ offset_(offset),
+ narrowingOp_(narrowingOp),
+ aliases_(aliases),
+ maybeTrap_(maybeTrap) {
+ MOZ_ASSERT(value->type() != MIRType::RefOrNull);
+ // "if you want to narrow the value when it is stored, the source type
+ // must be Int32".
+ MOZ_ASSERT_IF(narrowingOp != MNarrowingOp::None,
+ value->type() == MIRType::Int32);
+ MOZ_ASSERT(
+ aliases.flags() ==
+ AliasSet::Store(AliasSet::WasmStructInlineDataArea).flags() ||
+ aliases.flags() ==
+ AliasSet::Store(AliasSet::WasmStructOutlineDataArea).flags() ||
+ aliases.flags() ==
+ AliasSet::Store(AliasSet::WasmArrayDataArea).flags() ||
+ aliases.flags() == AliasSet::Store(AliasSet::Any).flags());
+ if (maybeTrap_) {
+ setGuard();
+ }
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmStoreFieldKA)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, ka), (1, obj), (2, value))
+
+ uint32_t offset() const { return offset_; }
+ MNarrowingOp narrowingOp() const { return narrowingOp_; }
+ MaybeTrapSiteInfo maybeTrap() const { return maybeTrap_; }
+
+ AliasSet getAliasSet() const override { return aliases_; }
+};
+
+// Store a reference value to a location which (it is assumed) is within a
+// wasm object. This instruction emits a pre-barrier. A post barrier _must_
+// be performed separately.
+//
+// This instruction takes a second object `ka` that must be kept alive, as
+// described for MWasmLoadFieldKA above.
+class MWasmStoreFieldRefKA : public MAryInstruction<4>,
+ public NoTypePolicy::Data {
+ AliasSet aliases_;
+
+ MWasmStoreFieldRefKA(MDefinition* instance, MDefinition* ka,
+ MDefinition* valueAddr, MDefinition* value,
+ AliasSet aliases)
+ : MAryInstruction<4>(classOpcode), aliases_(aliases) {
+ MOZ_ASSERT(valueAddr->type() == MIRType::Pointer ||
+ valueAddr->type() == TargetWordMIRType());
+ MOZ_ASSERT(value->type() == MIRType::RefOrNull);
+ MOZ_ASSERT(
+ aliases.flags() ==
+ AliasSet::Store(AliasSet::WasmStructInlineDataArea).flags() ||
+ aliases.flags() ==
+ AliasSet::Store(AliasSet::WasmStructOutlineDataArea).flags() ||
+ aliases.flags() ==
+ AliasSet::Store(AliasSet::WasmArrayDataArea).flags() ||
+ aliases.flags() == AliasSet::Store(AliasSet::Any).flags());
+ initOperand(0, instance);
+ initOperand(1, ka);
+ initOperand(2, valueAddr);
+ initOperand(3, value);
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmStoreFieldRefKA)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, instance), (1, ka), (2, valueAddr), (3, value))
+
+ AliasSet getAliasSet() const override { return aliases_; }
+};
+
+// Tests if the WasmGcObject, `object`, is a subtype of `superTypeDef`. The
+// actual super type definition must be known at compile time, so that the
+// subtyping depth of super type depth can be used.
+class MWasmGcObjectIsSubtypeOf : public MBinaryInstruction,
+ public NoTypePolicy::Data {
+ uint32_t subTypingDepth_;
+ MWasmGcObjectIsSubtypeOf(MDefinition* object, MDefinition* superTypeDef,
+ uint32_t subTypingDepth)
+ : MBinaryInstruction(classOpcode, object, superTypeDef),
+ subTypingDepth_(subTypingDepth) {
+ setResultType(MIRType::Int32);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(WasmGcObjectIsSubtypeOf)
+ TRIVIAL_NEW_WRAPPERS
+ NAMED_OPERANDS((0, object), (1, superTypeDef))
+
+ uint32_t subTypingDepth() const { return subTypingDepth_; }
+
+ bool congruentTo(const MDefinition* ins) const override {
+ return congruentIfOperandsEqual(ins) &&
+ ins->toWasmGcObjectIsSubtypeOf()->subTypingDepth() ==
+ subTypingDepth();
+ }
+
+ HashNumber valueHash() const override {
+ return addU32ToHash(MBinaryInstruction::valueHash(), subTypingDepth());
+ }
+};
+
+#ifdef FUZZING_JS_FUZZILLI
+class MFuzzilliHash : public MUnaryInstruction, public NoTypePolicy::Data {
+ explicit MFuzzilliHash(MDefinition* obj)
+ : MUnaryInstruction(classOpcode, obj) {
+ setResultType(MIRType::Int32);
+ setMovable();
+ }
+
+ public:
+ INSTRUCTION_HEADER(FuzzilliHash);
+ TRIVIAL_NEW_WRAPPERS
+ ALLOW_CLONE(MFuzzilliHash)
+
+# ifdef DEBUG
+ bool isConsistentFloat32Use(MUse* use) const override { return true; }
+# endif
+
+ AliasSet getAliasSet() const override {
+ MDefinition* obj = getOperand(0);
+ if (obj->type() == MIRType::Object || obj->type() == MIRType::Value) {
+ return AliasSet::Load(AliasSet::ObjectFields | AliasSet::FixedSlot |
+ AliasSet::DynamicSlot | AliasSet::Element |
+ AliasSet::UnboxedElement);
+ }
+ return AliasSet::None();
+ }
+};
+
+class MFuzzilliHashStore : public MUnaryInstruction, public NoTypePolicy::Data {
+ explicit MFuzzilliHashStore(MDefinition* obj)
+ : MUnaryInstruction(classOpcode, obj) {
+ MOZ_ASSERT(obj->type() == MIRType::Int32);
+
+ setResultType(MIRType::None);
+ }
+
+ public:
+ INSTRUCTION_HEADER(FuzzilliHashStore);
+ TRIVIAL_NEW_WRAPPERS
+ ALLOW_CLONE(MFuzzilliHashStore)
+
+ // this is a store and hence effectful, however no other load can
+ // alias with the store
+ AliasSet getAliasSet() const override {
+ return AliasSet::Store(AliasSet::FuzzilliHash);
+ }
+};
+#endif
+
+#undef INSTRUCTION_HEADER
+
+void MUse::init(MDefinition* producer, MNode* consumer) {
+ MOZ_ASSERT(!consumer_, "Initializing MUse that already has a consumer");
+ MOZ_ASSERT(!producer_, "Initializing MUse that already has a producer");
+ initUnchecked(producer, consumer);
+}
+
+void MUse::initUnchecked(MDefinition* producer, MNode* consumer) {
+ MOZ_ASSERT(consumer, "Initializing to null consumer");
+ consumer_ = consumer;
+ producer_ = producer;
+ producer_->addUseUnchecked(this);
+}
+
+void MUse::initUncheckedWithoutProducer(MNode* consumer) {
+ MOZ_ASSERT(consumer, "Initializing to null consumer");
+ consumer_ = consumer;
+ producer_ = nullptr;
+}
+
+void MUse::replaceProducer(MDefinition* producer) {
+ MOZ_ASSERT(consumer_, "Resetting MUse without a consumer");
+ producer_->removeUse(this);
+ producer_ = producer;
+ producer_->addUse(this);
+}
+
+void MUse::releaseProducer() {
+ MOZ_ASSERT(consumer_, "Clearing MUse without a consumer");
+ producer_->removeUse(this);
+ producer_ = nullptr;
+}
+
+// Implement cast functions now that the compiler can see the inheritance.
+
+MDefinition* MNode::toDefinition() {
+ MOZ_ASSERT(isDefinition());
+ return (MDefinition*)this;
+}
+
+MResumePoint* MNode::toResumePoint() {
+ MOZ_ASSERT(isResumePoint());
+ return (MResumePoint*)this;
+}
+
+MInstruction* MDefinition::toInstruction() {
+ MOZ_ASSERT(!isPhi());
+ return (MInstruction*)this;
+}
+
+const MInstruction* MDefinition::toInstruction() const {
+ MOZ_ASSERT(!isPhi());
+ return (const MInstruction*)this;
+}
+
+MControlInstruction* MDefinition::toControlInstruction() {
+ MOZ_ASSERT(isControlInstruction());
+ return (MControlInstruction*)this;
+}
+
+MConstant* MDefinition::maybeConstantValue() {
+ MDefinition* op = this;
+ if (op->isBox()) {
+ op = op->toBox()->input();
+ }
+ if (op->isConstant()) {
+ return op->toConstant();
+ }
+ return nullptr;
+}
+
+#ifdef ENABLE_WASM_SIMD
+MWasmShuffleSimd128* BuildWasmShuffleSimd128(TempAllocator& alloc,
+ const int8_t* control,
+ MDefinition* lhs,
+ MDefinition* rhs);
+#endif // ENABLE_WASM_SIMD
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_MIR_h */
diff --git a/js/src/jit/MIRGenerator.h b/js/src/jit/MIRGenerator.h
new file mode 100644
index 0000000000..c91167175d
--- /dev/null
+++ b/js/src/jit/MIRGenerator.h
@@ -0,0 +1,176 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MIRGenerator_h
+#define jit_MIRGenerator_h
+
+// This file declares the data structures used to build a control-flow graph
+// containing MIR.
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Result.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/CompileInfo.h"
+#include "jit/CompileWrappers.h"
+#include "jit/JitAllocPolicy.h"
+#include "jit/JitContext.h"
+#include "jit/JitSpewer.h"
+#include "jit/PerfSpewer.h"
+#include "js/Utility.h"
+#include "vm/GeckoProfiler.h"
+
+namespace js {
+namespace jit {
+
+class JitRuntime;
+class MIRGraph;
+class OptimizationInfo;
+
+class MIRGenerator final {
+ public:
+ MIRGenerator(CompileRealm* realm, const JitCompileOptions& options,
+ TempAllocator* alloc, MIRGraph* graph,
+ const CompileInfo* outerInfo,
+ const OptimizationInfo* optimizationInfo);
+
+ void initMinWasmHeapLength(uint64_t init) { minWasmHeapLength_ = init; }
+
+ TempAllocator& alloc() { return *alloc_; }
+ MIRGraph& graph() { return *graph_; }
+ [[nodiscard]] bool ensureBallast() { return alloc().ensureBallast(); }
+ const JitRuntime* jitRuntime() const { return runtime->jitRuntime(); }
+ const CompileInfo& outerInfo() const { return *outerInfo_; }
+ const OptimizationInfo& optimizationInfo() const {
+ return *optimizationInfo_;
+ }
+ bool hasProfilingScripts() const {
+ return runtime && runtime->profilingScripts();
+ }
+
+ template <typename T>
+ T* allocate(size_t count = 1) {
+ size_t bytes;
+ if (MOZ_UNLIKELY(!CalculateAllocSize<T>(count, &bytes))) {
+ return nullptr;
+ }
+ return static_cast<T*>(alloc().allocate(bytes));
+ }
+
+ // Set an error state and prints a message. Returns false so errors can be
+ // propagated up.
+ mozilla::GenericErrorResult<AbortReason> abort(AbortReason r);
+ mozilla::GenericErrorResult<AbortReason> abort(AbortReason r,
+ const char* message, ...)
+ MOZ_FORMAT_PRINTF(3, 4);
+
+ mozilla::GenericErrorResult<AbortReason> abortFmt(AbortReason r,
+ const char* message,
+ va_list ap)
+ MOZ_FORMAT_PRINTF(3, 0);
+
+ // Collect the evaluation result of phases after WarpOracle, such that
+ // off-thread compilation can report what error got encountered.
+ void setOffThreadStatus(AbortReasonOr<Ok>&& result) {
+ MOZ_ASSERT(offThreadStatus_.isOk());
+ offThreadStatus_ = std::move(result);
+ }
+ const AbortReasonOr<Ok>& getOffThreadStatus() const {
+ return offThreadStatus_;
+ }
+
+ [[nodiscard]] bool instrumentedProfiling() {
+ if (!instrumentedProfilingIsCached_) {
+ instrumentedProfiling_ = runtime->geckoProfiler().enabled();
+ instrumentedProfilingIsCached_ = true;
+ }
+ return instrumentedProfiling_;
+ }
+
+ bool isProfilerInstrumentationEnabled() {
+ return !compilingWasm() && instrumentedProfiling();
+ }
+
+ gc::InitialHeap initialStringHeap() const {
+ return stringsCanBeInNursery_ ? gc::DefaultHeap : gc::TenuredHeap;
+ }
+
+ gc::InitialHeap initialBigIntHeap() const {
+ return bigIntsCanBeInNursery_ ? gc::DefaultHeap : gc::TenuredHeap;
+ }
+
+ // Whether the main thread is trying to cancel this build.
+ bool shouldCancel(const char* why) { return cancelBuild_; }
+ void cancel() { cancelBuild_ = true; }
+
+ bool compilingWasm() const { return outerInfo_->compilingWasm(); }
+
+ uint32_t wasmMaxStackArgBytes() const {
+ MOZ_ASSERT(compilingWasm());
+ return wasmMaxStackArgBytes_;
+ }
+ void initWasmMaxStackArgBytes(uint32_t n) {
+ MOZ_ASSERT(compilingWasm());
+ MOZ_ASSERT(wasmMaxStackArgBytes_ == 0);
+ wasmMaxStackArgBytes_ = n;
+ }
+ uint64_t minWasmHeapLength() const { return minWasmHeapLength_; }
+
+ void setNeedsOverrecursedCheck() { needsOverrecursedCheck_ = true; }
+ bool needsOverrecursedCheck() const { return needsOverrecursedCheck_; }
+
+ void setNeedsStaticStackAlignment() { needsStaticStackAlignment_ = true; }
+ bool needsStaticStackAlignment() const { return needsStaticStackAlignment_; }
+
+ public:
+ CompileRealm* realm;
+ CompileRuntime* runtime;
+
+ private:
+ // The CompileInfo for the outermost script.
+ const CompileInfo* outerInfo_;
+
+ const OptimizationInfo* optimizationInfo_;
+ TempAllocator* alloc_;
+ MIRGraph* graph_;
+ AbortReasonOr<Ok> offThreadStatus_;
+ mozilla::Atomic<bool, mozilla::Relaxed> cancelBuild_;
+
+ uint32_t wasmMaxStackArgBytes_;
+ bool needsOverrecursedCheck_;
+ bool needsStaticStackAlignment_;
+
+ bool instrumentedProfiling_;
+ bool instrumentedProfilingIsCached_;
+ bool stringsCanBeInNursery_;
+ bool bigIntsCanBeInNursery_;
+
+ uint64_t minWasmHeapLength_;
+
+ IonPerfSpewer wasmPerfSpewer_;
+
+ public:
+ IonPerfSpewer& perfSpewer() { return wasmPerfSpewer_; }
+
+ public:
+ const JitCompileOptions options;
+
+ private:
+ GraphSpewer gs_;
+
+ public:
+ GraphSpewer& graphSpewer() { return gs_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_MIRGenerator_h */
diff --git a/js/src/jit/MIRGraph.cpp b/js/src/jit/MIRGraph.cpp
new file mode 100644
index 0000000000..5b67c73553
--- /dev/null
+++ b/js/src/jit/MIRGraph.cpp
@@ -0,0 +1,1251 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/MIRGraph.h"
+
+#include "jit/CompileInfo.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitSpewer.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+
+using namespace js;
+using namespace js::jit;
+
+MIRGenerator::MIRGenerator(CompileRealm* realm,
+ const JitCompileOptions& options,
+ TempAllocator* alloc, MIRGraph* graph,
+ const CompileInfo* info,
+ const OptimizationInfo* optimizationInfo)
+ : realm(realm),
+ runtime(realm ? realm->runtime() : nullptr),
+ outerInfo_(info),
+ optimizationInfo_(optimizationInfo),
+ alloc_(alloc),
+ graph_(graph),
+ offThreadStatus_(Ok()),
+ cancelBuild_(false),
+ wasmMaxStackArgBytes_(0),
+ needsOverrecursedCheck_(false),
+ needsStaticStackAlignment_(false),
+ instrumentedProfiling_(false),
+ instrumentedProfilingIsCached_(false),
+ stringsCanBeInNursery_(realm ? realm->zone()->canNurseryAllocateStrings()
+ : false),
+ bigIntsCanBeInNursery_(realm ? realm->zone()->canNurseryAllocateBigInts()
+ : false),
+ minWasmHeapLength_(0),
+ options(options),
+ gs_(alloc) {}
+
+mozilla::GenericErrorResult<AbortReason> MIRGenerator::abort(AbortReason r) {
+ if (JitSpewEnabled(JitSpew_IonAbort)) {
+ switch (r) {
+ case AbortReason::Alloc:
+ JitSpew(JitSpew_IonAbort, "AbortReason::Alloc");
+ break;
+ case AbortReason::Disable:
+ JitSpew(JitSpew_IonAbort, "AbortReason::Disable");
+ break;
+ case AbortReason::Error:
+ JitSpew(JitSpew_IonAbort, "AbortReason::Error");
+ break;
+ case AbortReason::NoAbort:
+ MOZ_CRASH("Abort with AbortReason::NoAbort");
+ break;
+ }
+ }
+ return Err(std::move(r));
+}
+
+mozilla::GenericErrorResult<AbortReason> MIRGenerator::abortFmt(
+ AbortReason r, const char* message, va_list ap) {
+ JitSpewVA(JitSpew_IonAbort, message, ap);
+ return Err(std::move(r));
+}
+
+mozilla::GenericErrorResult<AbortReason> MIRGenerator::abort(
+ AbortReason r, const char* message, ...) {
+ va_list ap;
+ va_start(ap, message);
+ auto forward = abortFmt(r, message, ap);
+ va_end(ap);
+ return forward;
+}
+
+void MIRGraph::addBlock(MBasicBlock* block) {
+ MOZ_ASSERT(block);
+ block->setId(blockIdGen_++);
+ blocks_.pushBack(block);
+ numBlocks_++;
+}
+
+void MIRGraph::insertBlockAfter(MBasicBlock* at, MBasicBlock* block) {
+ block->setId(blockIdGen_++);
+ blocks_.insertAfter(at, block);
+ numBlocks_++;
+}
+
+void MIRGraph::insertBlockBefore(MBasicBlock* at, MBasicBlock* block) {
+ block->setId(blockIdGen_++);
+ blocks_.insertBefore(at, block);
+ numBlocks_++;
+}
+
+void MIRGraph::removeBlock(MBasicBlock* block) {
+ // Remove a block from the graph. It will also cleanup the block.
+
+ if (block == osrBlock_) {
+ osrBlock_ = nullptr;
+ }
+
+ if (returnAccumulator_) {
+ size_t i = 0;
+ while (i < returnAccumulator_->length()) {
+ if ((*returnAccumulator_)[i] == block) {
+ returnAccumulator_->erase(returnAccumulator_->begin() + i);
+ } else {
+ i++;
+ }
+ }
+ }
+
+ block->clear();
+ block->markAsDead();
+
+ if (block->isInList()) {
+ blocks_.remove(block);
+ numBlocks_--;
+ }
+}
+
+void MIRGraph::unmarkBlocks() {
+ for (MBasicBlockIterator i(blocks_.begin()); i != blocks_.end(); i++) {
+ i->unmark();
+ }
+}
+
+MBasicBlock* MBasicBlock::New(MIRGraph& graph, size_t stackDepth,
+ const CompileInfo& info, MBasicBlock* maybePred,
+ BytecodeSite* site, Kind kind) {
+ MOZ_ASSERT(site->pc() != nullptr);
+
+ MBasicBlock* block = new (graph.alloc()) MBasicBlock(graph, info, site, kind);
+ if (!block->init()) {
+ return nullptr;
+ }
+
+ if (!block->inherit(graph.alloc(), stackDepth, maybePred, 0)) {
+ return nullptr;
+ }
+
+ return block;
+}
+
+MBasicBlock* MBasicBlock::NewPopN(MIRGraph& graph, const CompileInfo& info,
+ MBasicBlock* pred, BytecodeSite* site,
+ Kind kind, uint32_t popped) {
+ MOZ_ASSERT(site->pc() != nullptr);
+
+ MBasicBlock* block = new (graph.alloc()) MBasicBlock(graph, info, site, kind);
+ if (!block->init()) {
+ return nullptr;
+ }
+
+ if (!block->inherit(graph.alloc(), pred->stackDepth(), pred, popped)) {
+ return nullptr;
+ }
+
+ return block;
+}
+
+MBasicBlock* MBasicBlock::NewPendingLoopHeader(MIRGraph& graph,
+ const CompileInfo& info,
+ MBasicBlock* pred,
+ BytecodeSite* site) {
+ MOZ_ASSERT(site->pc() != nullptr);
+
+ MBasicBlock* block =
+ new (graph.alloc()) MBasicBlock(graph, info, site, PENDING_LOOP_HEADER);
+ if (!block->init()) {
+ return nullptr;
+ }
+
+ if (!block->inherit(graph.alloc(), pred->stackDepth(), pred, 0)) {
+ return nullptr;
+ }
+
+ return block;
+}
+
+MBasicBlock* MBasicBlock::NewSplitEdge(MIRGraph& graph, MBasicBlock* pred,
+ size_t predEdgeIdx, MBasicBlock* succ) {
+ MBasicBlock* split = nullptr;
+ if (!succ->pc()) {
+ // The predecessor does not have a PC, this is a Wasm compilation.
+ split = MBasicBlock::New(graph, succ->info(), pred, SPLIT_EDGE);
+ if (!split) {
+ return nullptr;
+ }
+
+ // Insert the split edge block in-between.
+ split->end(MGoto::New(graph.alloc(), succ));
+ } else {
+ // The predecessor has a PC, this is a Warp compilation.
+ MResumePoint* succEntry = succ->entryResumePoint();
+
+ BytecodeSite* site =
+ new (graph.alloc()) BytecodeSite(succ->trackedTree(), succEntry->pc());
+ split =
+ new (graph.alloc()) MBasicBlock(graph, succ->info(), site, SPLIT_EDGE);
+
+ if (!split->init()) {
+ return nullptr;
+ }
+
+ // A split edge is used to simplify the graph to avoid having a
+ // predecessor with multiple successors as well as a successor with
+ // multiple predecessors. As instructions can be moved in this
+ // split-edge block, we need to give this block a resume point. To do
+ // so, we copy the entry resume points of the successor and filter the
+ // phis to keep inputs from the current edge.
+
+ // Propagate the caller resume point from the inherited block.
+ split->callerResumePoint_ = succ->callerResumePoint();
+
+ // Split-edge are created after the interpreter stack emulation. Thus,
+ // there is no need for creating slots.
+ split->stackPosition_ = succEntry->stackDepth();
+
+ // Create a resume point using our initial stack position.
+ MResumePoint* splitEntry = new (graph.alloc())
+ MResumePoint(split, succEntry->pc(), ResumeMode::ResumeAt);
+ if (!splitEntry->init(graph.alloc())) {
+ return nullptr;
+ }
+ split->entryResumePoint_ = splitEntry;
+
+ // Insert the split edge block in-between.
+ split->end(MGoto::New(graph.alloc(), succ));
+
+ // The target entry resume point might have phi operands, keep the
+ // operands of the phi coming from our edge.
+ size_t succEdgeIdx = succ->indexForPredecessor(pred);
+
+ for (size_t i = 0, e = splitEntry->numOperands(); i < e; i++) {
+ MDefinition* def = succEntry->getOperand(i);
+ // This early in the pipeline, we have no recover instructions in
+ // any entry resume point.
+ if (def->block() == succ) {
+ if (def->isPhi()) {
+ def = def->toPhi()->getOperand(succEdgeIdx);
+ } else {
+ // The phi-operand may already have been optimized out.
+ MOZ_ASSERT(def->isConstant());
+ MOZ_ASSERT(def->type() == MIRType::MagicOptimizedOut);
+
+ def = split->optimizedOutConstant(graph.alloc());
+ }
+ }
+
+ splitEntry->initOperand(i, def);
+ }
+
+ // This is done in the New variant for wasm, so we cannot keep this
+ // line below, where the rest of the graph is modified.
+ if (!split->predecessors_.append(pred)) {
+ return nullptr;
+ }
+ }
+
+ split->setLoopDepth(succ->loopDepth());
+
+ graph.insertBlockAfter(pred, split);
+
+ pred->replaceSuccessor(predEdgeIdx, split);
+ succ->replacePredecessor(pred, split);
+ return split;
+}
+
+MBasicBlock* MBasicBlock::New(MIRGraph& graph, const CompileInfo& info,
+ MBasicBlock* pred, Kind kind) {
+ BytecodeSite* site = new (graph.alloc()) BytecodeSite();
+ MBasicBlock* block = new (graph.alloc()) MBasicBlock(graph, info, site, kind);
+ if (!block->init()) {
+ return nullptr;
+ }
+
+ if (pred) {
+ block->stackPosition_ = pred->stackPosition_;
+
+ if (block->kind_ == PENDING_LOOP_HEADER) {
+ size_t nphis = block->stackPosition_;
+
+ size_t nfree = graph.phiFreeListLength();
+
+ TempAllocator& alloc = graph.alloc();
+ MPhi* phis = nullptr;
+ if (nphis > nfree) {
+ phis = alloc.allocateArray<MPhi>(nphis - nfree);
+ if (!phis) {
+ return nullptr;
+ }
+ }
+
+ // Note: Phis are inserted in the same order as the slots.
+ for (size_t i = 0; i < nphis; i++) {
+ MDefinition* predSlot = pred->getSlot(i);
+
+ MOZ_ASSERT(predSlot->type() != MIRType::Value);
+
+ MPhi* phi;
+ if (i < nfree) {
+ phi = graph.takePhiFromFreeList();
+ } else {
+ phi = phis + (i - nfree);
+ }
+ new (phi) MPhi(alloc, predSlot->type());
+
+ phi->addInlineInput(predSlot);
+
+ // Add append Phis in the block.
+ block->addPhi(phi);
+ block->setSlot(i, phi);
+ }
+ } else {
+ if (!block->ensureHasSlots(0)) {
+ return nullptr;
+ }
+ block->copySlots(pred);
+ }
+
+ if (!block->predecessors_.append(pred)) {
+ return nullptr;
+ }
+ }
+
+ return block;
+}
+
+// Create an empty and unreachable block which jumps to |header|. Used
+// when the normal entry into a loop is removed (but the loop is still
+// reachable due to OSR) to preserve the invariant that every loop
+// header has two predecessors, which is needed for building the
+// dominator tree. The new block is inserted immediately before the
+// header, which preserves the graph ordering (post-order/RPO). These
+// blocks will all be removed before lowering.
+MBasicBlock* MBasicBlock::NewFakeLoopPredecessor(MIRGraph& graph,
+ MBasicBlock* header) {
+ MOZ_ASSERT(graph.osrBlock());
+
+ MBasicBlock* backedge = header->backedge();
+ MBasicBlock* fake = MBasicBlock::New(graph, header->info(), nullptr,
+ MBasicBlock::FAKE_LOOP_PRED);
+ if (!fake) {
+ return nullptr;
+ }
+
+ graph.insertBlockBefore(header, fake);
+ fake->setUnreachable();
+
+ // Create fake defs to use as inputs for any phis in |header|.
+ for (MPhiIterator iter(header->phisBegin()), end(header->phisEnd());
+ iter != end; ++iter) {
+ MPhi* phi = *iter;
+ auto* fakeDef = MUnreachableResult::New(graph.alloc(), phi->type());
+ fake->add(fakeDef);
+ if (!phi->addInputSlow(fakeDef)) {
+ return nullptr;
+ }
+ }
+
+ fake->end(MGoto::New(graph.alloc(), header));
+
+ if (!header->addPredecessorWithoutPhis(fake)) {
+ return nullptr;
+ }
+
+ // The backedge is always the last predecessor, but we have added a
+ // new pred. Restore |backedge| as |header|'s loop backedge.
+ header->clearLoopHeader();
+ header->setLoopHeader(backedge);
+
+ return fake;
+}
+
+void MIRGraph::removeFakeLoopPredecessors() {
+ MOZ_ASSERT(osrBlock());
+ size_t id = 0;
+ for (ReversePostorderIterator it = rpoBegin(); it != rpoEnd();) {
+ MBasicBlock* block = *it++;
+ if (block->isFakeLoopPred()) {
+ MOZ_ASSERT(block->unreachable());
+ MBasicBlock* succ = block->getSingleSuccessor();
+ succ->removePredecessor(block);
+ removeBlock(block);
+ } else {
+ block->setId(id++);
+ }
+ }
+#ifdef DEBUG
+ canBuildDominators_ = false;
+#endif
+}
+
+MBasicBlock::MBasicBlock(MIRGraph& graph, const CompileInfo& info,
+ BytecodeSite* site, Kind kind)
+ : graph_(graph),
+ info_(info),
+ predecessors_(graph.alloc()),
+ stackPosition_(info_.firstStackSlot()),
+ id_(0),
+ domIndex_(0),
+ numDominated_(0),
+ lir_(nullptr),
+ callerResumePoint_(nullptr),
+ entryResumePoint_(nullptr),
+ outerResumePoint_(nullptr),
+ successorWithPhis_(nullptr),
+ positionInPhiSuccessor_(0),
+ loopDepth_(0),
+ kind_(kind),
+ mark_(false),
+ immediatelyDominated_(graph.alloc()),
+ immediateDominator_(nullptr),
+ trackedSite_(site),
+ lineno_(0u),
+ columnIndex_(0u) {
+ MOZ_ASSERT(trackedSite_, "trackedSite_ is non-nullptr");
+}
+
+bool MBasicBlock::init() { return slots_.init(graph_.alloc(), info_.nslots()); }
+
+bool MBasicBlock::increaseSlots(size_t num) {
+ return slots_.growBy(graph_.alloc(), num);
+}
+
+bool MBasicBlock::ensureHasSlots(size_t num) {
+ size_t depth = stackDepth() + num;
+ if (depth > nslots()) {
+ if (!increaseSlots(depth - nslots())) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void MBasicBlock::copySlots(MBasicBlock* from) {
+ MOZ_ASSERT(stackPosition_ <= from->stackPosition_);
+ MOZ_ASSERT(stackPosition_ <= nslots());
+
+ MDefinition** thisSlots = slots_.begin();
+ MDefinition** fromSlots = from->slots_.begin();
+ for (size_t i = 0, e = stackPosition_; i < e; ++i) {
+ thisSlots[i] = fromSlots[i];
+ }
+}
+
+bool MBasicBlock::inherit(TempAllocator& alloc, size_t stackDepth,
+ MBasicBlock* maybePred, uint32_t popped) {
+ MOZ_ASSERT_IF(maybePred, maybePred->stackDepth() == stackDepth);
+
+ MOZ_ASSERT(stackDepth >= popped);
+ stackDepth -= popped;
+ stackPosition_ = stackDepth;
+
+ if (maybePred && kind_ != PENDING_LOOP_HEADER) {
+ copySlots(maybePred);
+ }
+
+ MOZ_ASSERT(info_.nslots() >= stackPosition_);
+ MOZ_ASSERT(!entryResumePoint_);
+
+ // Propagate the caller resume point from the inherited block.
+ callerResumePoint_ = maybePred ? maybePred->callerResumePoint() : nullptr;
+
+ // Create a resume point using our initial stack state.
+ entryResumePoint_ =
+ new (alloc) MResumePoint(this, pc(), ResumeMode::ResumeAt);
+ if (!entryResumePoint_->init(alloc)) {
+ return false;
+ }
+
+ if (maybePred) {
+ if (!predecessors_.append(maybePred)) {
+ return false;
+ }
+
+ if (kind_ == PENDING_LOOP_HEADER) {
+ for (size_t i = 0; i < stackDepth; i++) {
+ MPhi* phi = MPhi::New(alloc.fallible());
+ if (!phi) {
+ return false;
+ }
+ phi->addInlineInput(maybePred->getSlot(i));
+ addPhi(phi);
+ setSlot(i, phi);
+ entryResumePoint()->initOperand(i, phi);
+ }
+ } else {
+ for (size_t i = 0; i < stackDepth; i++) {
+ entryResumePoint()->initOperand(i, getSlot(i));
+ }
+ }
+ } else {
+ /*
+ * Don't leave the operands uninitialized for the caller, as it may not
+ * initialize them later on.
+ */
+ for (size_t i = 0; i < stackDepth; i++) {
+ entryResumePoint()->clearOperand(i);
+ }
+ }
+
+ return true;
+}
+
+void MBasicBlock::inheritSlots(MBasicBlock* parent) {
+ stackPosition_ = parent->stackPosition_;
+ copySlots(parent);
+}
+
+bool MBasicBlock::initEntrySlots(TempAllocator& alloc) {
+ // Remove the previous resume point.
+ discardResumePoint(entryResumePoint_);
+
+ // Create a resume point using our initial stack state.
+ entryResumePoint_ =
+ MResumePoint::New(alloc, this, pc(), ResumeMode::ResumeAt);
+ if (!entryResumePoint_) {
+ return false;
+ }
+ return true;
+}
+
+MDefinition* MBasicBlock::environmentChain() {
+ return getSlot(info().environmentChainSlot());
+}
+
+MDefinition* MBasicBlock::argumentsObject() {
+ return getSlot(info().argsObjSlot());
+}
+
+void MBasicBlock::setEnvironmentChain(MDefinition* scopeObj) {
+ setSlot(info().environmentChainSlot(), scopeObj);
+}
+
+void MBasicBlock::setArgumentsObject(MDefinition* argsObj) {
+ setSlot(info().argsObjSlot(), argsObj);
+}
+
+void MBasicBlock::pick(int32_t depth) {
+ // pick takes a value and moves it to the top.
+ // pick(-2):
+ // A B C D E
+ // A B D C E [ swapAt(-2) ]
+ // A B D E C [ swapAt(-1) ]
+ for (; depth < 0; depth++) {
+ swapAt(depth);
+ }
+}
+
+void MBasicBlock::unpick(int32_t depth) {
+ // unpick takes the value on top of the stack and moves it under the depth-th
+ // element;
+ // unpick(-2):
+ // A B C D E
+ // A B C E D [ swapAt(-1) ]
+ // A B E C D [ swapAt(-2) ]
+ for (int32_t n = -1; n >= depth; n--) {
+ swapAt(n);
+ }
+}
+
+void MBasicBlock::swapAt(int32_t depth) {
+ uint32_t lhsDepth = stackPosition_ + depth - 1;
+ uint32_t rhsDepth = stackPosition_ + depth;
+
+ MDefinition* temp = slots_[lhsDepth];
+ slots_[lhsDepth] = slots_[rhsDepth];
+ slots_[rhsDepth] = temp;
+}
+
+void MBasicBlock::discardLastIns() { discard(lastIns()); }
+
+MConstant* MBasicBlock::optimizedOutConstant(TempAllocator& alloc) {
+ // If the first instruction is a MConstant(MagicValue(JS_OPTIMIZED_OUT))
+ // then reuse it.
+ MInstruction* ins = *begin();
+ if (ins->type() == MIRType::MagicOptimizedOut) {
+ return ins->toConstant();
+ }
+
+ MConstant* constant = MConstant::New(alloc, MagicValue(JS_OPTIMIZED_OUT));
+ insertBefore(ins, constant);
+ return constant;
+}
+
+void MBasicBlock::moveBefore(MInstruction* at, MInstruction* ins) {
+ // Remove |ins| from the current block.
+ MOZ_ASSERT(ins->block() == this);
+ instructions_.remove(ins);
+
+ // Insert into new block, which may be distinct.
+ // Uses and operands are untouched.
+ ins->setInstructionBlock(at->block(), at->trackedSite());
+ at->block()->instructions_.insertBefore(at, ins);
+}
+
+MInstruction* MBasicBlock::safeInsertTop(MDefinition* ins, IgnoreTop ignore) {
+ MOZ_ASSERT(graph().osrBlock() != this,
+ "We are not supposed to add any instruction in OSR blocks.");
+
+ // Beta nodes and interrupt checks are required to be located at the
+ // beginnings of basic blocks, so we must insert new instructions after any
+ // such instructions.
+ MInstructionIterator insertIter =
+ !ins || ins->isPhi() ? begin() : begin(ins->toInstruction());
+ while (insertIter->isBeta() || insertIter->isInterruptCheck() ||
+ insertIter->isConstant() || insertIter->isParameter() ||
+ (!(ignore & IgnoreRecover) && insertIter->isRecoveredOnBailout())) {
+ insertIter++;
+ }
+
+ return *insertIter;
+}
+
+void MBasicBlock::discardResumePoint(
+ MResumePoint* rp, ReferencesType refType /* = RefType_Default */) {
+ if (refType & RefType_DiscardOperands) {
+ rp->releaseUses();
+ }
+ rp->setDiscarded();
+#ifdef DEBUG
+ MResumePointIterator iter = resumePointsBegin();
+ while (*iter != rp) {
+ // We should reach it before reaching the end.
+ MOZ_ASSERT(iter != resumePointsEnd());
+ iter++;
+ }
+ resumePoints_.removeAt(iter);
+#endif
+}
+
+void MBasicBlock::prepareForDiscard(
+ MInstruction* ins, ReferencesType refType /* = RefType_Default */) {
+ // Only remove instructions from the same basic block. This is needed for
+ // correctly removing the resume point if any.
+ MOZ_ASSERT(ins->block() == this);
+
+ MResumePoint* rp = ins->resumePoint();
+ if ((refType & RefType_DiscardResumePoint) && rp) {
+ discardResumePoint(rp, refType);
+ }
+
+ // We need to assert that instructions have no uses after removing the their
+ // resume points operands as they could be captured by their own resume
+ // point.
+ MOZ_ASSERT_IF(refType & RefType_AssertNoUses, !ins->hasUses());
+
+ const uint32_t InstructionOperands =
+ RefType_DiscardOperands | RefType_DiscardInstruction;
+ if ((refType & InstructionOperands) == InstructionOperands) {
+ for (size_t i = 0, e = ins->numOperands(); i < e; i++) {
+ ins->releaseOperand(i);
+ }
+ }
+
+ ins->setDiscarded();
+}
+
+void MBasicBlock::discard(MInstruction* ins) {
+ prepareForDiscard(ins);
+ instructions_.remove(ins);
+}
+
+void MBasicBlock::discardIgnoreOperands(MInstruction* ins) {
+#ifdef DEBUG
+ for (size_t i = 0, e = ins->numOperands(); i < e; i++) {
+ MOZ_ASSERT(!ins->hasOperand(i));
+ }
+#endif
+
+ prepareForDiscard(ins, RefType_IgnoreOperands);
+ instructions_.remove(ins);
+}
+
+void MBasicBlock::discardAllInstructions() {
+ MInstructionIterator iter = begin();
+ discardAllInstructionsStartingAt(iter);
+}
+
+void MBasicBlock::discardAllInstructionsStartingAt(MInstructionIterator iter) {
+ while (iter != end()) {
+ // Discard operands and resume point operands and flag the instruction
+ // as discarded. Also we do not assert that we have no uses as blocks
+ // might be removed in reverse post order.
+ MInstruction* ins = *iter++;
+ prepareForDiscard(ins, RefType_DefaultNoAssert);
+ instructions_.remove(ins);
+ }
+}
+
+void MBasicBlock::discardAllPhis() {
+ for (MPhiIterator iter = phisBegin(); iter != phisEnd(); iter++) {
+ iter->removeAllOperands();
+ }
+
+ for (MBasicBlock** pred = predecessors_.begin(); pred != predecessors_.end();
+ pred++) {
+ (*pred)->clearSuccessorWithPhis();
+ }
+
+ phis_.clear();
+}
+
+void MBasicBlock::discardAllResumePoints(bool discardEntry) {
+ if (outerResumePoint_) {
+ clearOuterResumePoint();
+ }
+
+ if (discardEntry && entryResumePoint_) {
+ clearEntryResumePoint();
+ }
+
+#ifdef DEBUG
+ if (!entryResumePoint()) {
+ MOZ_ASSERT(resumePointsEmpty());
+ } else {
+ MResumePointIterator iter(resumePointsBegin());
+ MOZ_ASSERT(iter != resumePointsEnd());
+ iter++;
+ MOZ_ASSERT(iter == resumePointsEnd());
+ }
+#endif
+}
+
+void MBasicBlock::clear() {
+ discardAllInstructions();
+ discardAllResumePoints();
+ discardAllPhis();
+}
+
+void MBasicBlock::insertBefore(MInstruction* at, MInstruction* ins) {
+ MOZ_ASSERT(at->block() == this);
+ ins->setInstructionBlock(this, at->trackedSite());
+ graph().allocDefinitionId(ins);
+ instructions_.insertBefore(at, ins);
+}
+
+void MBasicBlock::insertAfter(MInstruction* at, MInstruction* ins) {
+ MOZ_ASSERT(at->block() == this);
+ ins->setInstructionBlock(this, at->trackedSite());
+ graph().allocDefinitionId(ins);
+ instructions_.insertAfter(at, ins);
+}
+
+void MBasicBlock::insertAtEnd(MInstruction* ins) {
+ if (hasLastIns()) {
+ insertBefore(lastIns(), ins);
+ } else {
+ add(ins);
+ }
+}
+
+void MBasicBlock::addPhi(MPhi* phi) {
+ phis_.pushBack(phi);
+ phi->setPhiBlock(this);
+ graph().allocDefinitionId(phi);
+}
+
+void MBasicBlock::discardPhi(MPhi* phi) {
+ MOZ_ASSERT(!phis_.empty());
+
+ phi->removeAllOperands();
+ phi->setDiscarded();
+
+ phis_.remove(phi);
+
+ if (phis_.empty()) {
+ for (MBasicBlock* pred : predecessors_) {
+ pred->clearSuccessorWithPhis();
+ }
+ }
+}
+
+void MBasicBlock::flagOperandsOfPrunedBranches(MInstruction* ins) {
+ // Find the previous resume point which would be used for bailing out.
+ MResumePoint* rp = nullptr;
+ for (MInstructionReverseIterator iter = rbegin(ins); iter != rend(); iter++) {
+ rp = iter->resumePoint();
+ if (rp) {
+ break;
+ }
+ }
+
+ // If none, take the entry resume point.
+ if (!rp) {
+ rp = entryResumePoint();
+ }
+
+ // The only blocks which do not have any entryResumePoint in Ion, are the
+ // SplitEdge blocks. SplitEdge blocks only have a Goto instruction before
+ // Range Analysis phase. In adjustInputs, we are manipulating instructions
+ // which have a TypePolicy. So, as a Goto has no operand and no type
+ // policy, the entry resume point should exist.
+ MOZ_ASSERT(rp);
+
+ // Flag all operands as being potentially used.
+ while (rp) {
+ for (size_t i = 0, end = rp->numOperands(); i < end; i++) {
+ rp->getOperand(i)->setImplicitlyUsedUnchecked();
+ }
+ rp = rp->caller();
+ }
+}
+
+bool MBasicBlock::addPredecessor(TempAllocator& alloc, MBasicBlock* pred) {
+ return addPredecessorPopN(alloc, pred, 0);
+}
+
+bool MBasicBlock::addPredecessorPopN(TempAllocator& alloc, MBasicBlock* pred,
+ uint32_t popped) {
+ MOZ_ASSERT(pred);
+ MOZ_ASSERT(predecessors_.length() > 0);
+
+ // Predecessors must be finished, and at the correct stack depth.
+ MOZ_ASSERT(pred->hasLastIns());
+ MOZ_ASSERT(pred->stackPosition_ == stackPosition_ + popped);
+
+ for (uint32_t i = 0, e = stackPosition_; i < e; ++i) {
+ MDefinition* mine = getSlot(i);
+ MDefinition* other = pred->getSlot(i);
+
+ if (mine != other) {
+ MIRType phiType = mine->type();
+ if (phiType != other->type()) {
+ phiType = MIRType::Value;
+ }
+
+ // If the current instruction is a phi, and it was created in this
+ // basic block, then we have already placed this phi and should
+ // instead append to its operands.
+ if (mine->isPhi() && mine->block() == this) {
+ MOZ_ASSERT(predecessors_.length());
+ MOZ_ASSERT(!mine->hasDefUses(),
+ "should only change type of newly created phis");
+ mine->setResultType(phiType);
+ if (!mine->toPhi()->addInputSlow(other)) {
+ return false;
+ }
+ } else {
+ // Otherwise, create a new phi node.
+ MPhi* phi = MPhi::New(alloc.fallible(), phiType);
+ if (!phi) {
+ return false;
+ }
+ addPhi(phi);
+
+ // Prime the phi for each predecessor, so input(x) comes from
+ // predecessor(x).
+ if (!phi->reserveLength(predecessors_.length() + 1)) {
+ return false;
+ }
+
+ for (size_t j = 0, numPreds = predecessors_.length(); j < numPreds;
+ ++j) {
+ MOZ_ASSERT(predecessors_[j]->getSlot(i) == mine);
+ phi->addInput(mine);
+ }
+ phi->addInput(other);
+
+ setSlot(i, phi);
+ if (entryResumePoint()) {
+ entryResumePoint()->replaceOperand(i, phi);
+ }
+ }
+ }
+ }
+
+ return predecessors_.append(pred);
+}
+
+bool MBasicBlock::addPredecessorSameInputsAs(MBasicBlock* pred,
+ MBasicBlock* existingPred) {
+ MOZ_ASSERT(pred);
+ MOZ_ASSERT(predecessors_.length() > 0);
+
+ // Predecessors must be finished, and at the correct stack depth.
+ MOZ_ASSERT(pred->hasLastIns());
+ MOZ_ASSERT(!pred->successorWithPhis());
+
+ if (!phisEmpty()) {
+ size_t existingPosition = indexForPredecessor(existingPred);
+ for (MPhiIterator iter = phisBegin(); iter != phisEnd(); iter++) {
+ if (!iter->addInputSlow(iter->getOperand(existingPosition))) {
+ return false;
+ }
+ }
+ }
+
+ if (!predecessors_.append(pred)) {
+ return false;
+ }
+ return true;
+}
+
+bool MBasicBlock::addPredecessorWithoutPhis(MBasicBlock* pred) {
+ // Predecessors must be finished.
+ MOZ_ASSERT(pred && pred->hasLastIns());
+ return predecessors_.append(pred);
+}
+
+bool MBasicBlock::addImmediatelyDominatedBlock(MBasicBlock* child) {
+ return immediatelyDominated_.append(child);
+}
+
+void MBasicBlock::removeImmediatelyDominatedBlock(MBasicBlock* child) {
+ for (size_t i = 0;; ++i) {
+ MOZ_ASSERT(i < immediatelyDominated_.length(),
+ "Dominated block to remove not present");
+ if (immediatelyDominated_[i] == child) {
+ immediatelyDominated_[i] = immediatelyDominated_.back();
+ immediatelyDominated_.popBack();
+ return;
+ }
+ }
+}
+
+bool MBasicBlock::setBackedge(MBasicBlock* pred) {
+ // Predecessors must be finished, and at the correct stack depth.
+ MOZ_ASSERT(hasLastIns());
+ MOZ_ASSERT(pred->hasLastIns());
+ MOZ_ASSERT(pred->stackDepth() == entryResumePoint()->stackDepth());
+
+ // We must be a pending loop header
+ MOZ_ASSERT(kind_ == PENDING_LOOP_HEADER);
+
+ // Add exit definitions to each corresponding phi at the entry.
+ if (!inheritPhisFromBackedge(pred)) {
+ return false;
+ }
+
+ // We are now a loop header proper
+ kind_ = LOOP_HEADER;
+
+ return predecessors_.append(pred);
+}
+
+bool MBasicBlock::setBackedgeWasm(MBasicBlock* pred, size_t paramCount) {
+ // Predecessors must be finished, and at the correct stack depth.
+ MOZ_ASSERT(hasLastIns());
+ MOZ_ASSERT(pred->hasLastIns());
+ MOZ_ASSERT(stackDepth() + paramCount == pred->stackDepth());
+
+ // We must be a pending loop header
+ MOZ_ASSERT(kind_ == PENDING_LOOP_HEADER);
+
+ // Add exit definitions to each corresponding phi at the entry.
+ // Note: Phis are inserted in the same order as the slots. (see
+ // MBasicBlock::New)
+ size_t slot = 0;
+ for (MPhiIterator phi = phisBegin(); phi != phisEnd(); phi++, slot++) {
+ MPhi* entryDef = *phi;
+ MDefinition* exitDef = pred->getSlot(slot);
+
+ // Assert that we already placed phis for each slot.
+ MOZ_ASSERT(entryDef->block() == this);
+
+ // Assert that the phi already has the correct type.
+ MOZ_ASSERT(entryDef->type() == exitDef->type());
+ MOZ_ASSERT(entryDef->type() != MIRType::Value);
+
+ if (entryDef == exitDef) {
+ // If the exit def is the same as the entry def, make a redundant
+ // phi. Since loop headers have exactly two incoming edges, we
+ // know that that's just the first input.
+ //
+ // Note that we eliminate later rather than now, to avoid any
+ // weirdness around pending continue edges which might still hold
+ // onto phis.
+ exitDef = entryDef->getOperand(0);
+ }
+
+ // Phis always have room for 2 operands, so this can't fail.
+ MOZ_ASSERT(phi->numOperands() == 1);
+ entryDef->addInlineInput(exitDef);
+
+ // Two cases here: phis that correspond to locals, and phis that correspond
+ // to loop parameters. Only phis for locals go in slots.
+ if (slot < stackDepth()) {
+ setSlot(slot, entryDef);
+ }
+ }
+
+ // We are now a loop header proper
+ kind_ = LOOP_HEADER;
+
+ return predecessors_.append(pred);
+}
+
+void MBasicBlock::clearLoopHeader() {
+ MOZ_ASSERT(isLoopHeader());
+ kind_ = NORMAL;
+}
+
+void MBasicBlock::setLoopHeader(MBasicBlock* newBackedge) {
+ MOZ_ASSERT(!isLoopHeader());
+ kind_ = LOOP_HEADER;
+
+ size_t numPreds = numPredecessors();
+ MOZ_ASSERT(numPreds != 0);
+
+ size_t lastIndex = numPreds - 1;
+ size_t oldIndex = 0;
+ for (;; ++oldIndex) {
+ MOZ_ASSERT(oldIndex < numPreds);
+ MBasicBlock* pred = getPredecessor(oldIndex);
+ if (pred == newBackedge) {
+ break;
+ }
+ }
+
+ // Set the loop backedge to be the last element in predecessors_.
+ std::swap(predecessors_[oldIndex], predecessors_[lastIndex]);
+
+ // If we have phis, reorder their operands accordingly.
+ if (!phisEmpty()) {
+ getPredecessor(oldIndex)->setSuccessorWithPhis(this, oldIndex);
+ getPredecessor(lastIndex)->setSuccessorWithPhis(this, lastIndex);
+ for (MPhiIterator iter(phisBegin()), end(phisEnd()); iter != end; ++iter) {
+ MPhi* phi = *iter;
+ MDefinition* last = phi->getOperand(oldIndex);
+ MDefinition* old = phi->getOperand(lastIndex);
+ phi->replaceOperand(oldIndex, old);
+ phi->replaceOperand(lastIndex, last);
+ }
+ }
+
+ MOZ_ASSERT(newBackedge->loopHeaderOfBackedge() == this);
+ MOZ_ASSERT(backedge() == newBackedge);
+}
+
+size_t MBasicBlock::getSuccessorIndex(MBasicBlock* block) const {
+ MOZ_ASSERT(lastIns());
+ for (size_t i = 0; i < numSuccessors(); i++) {
+ if (getSuccessor(i) == block) {
+ return i;
+ }
+ }
+ MOZ_CRASH("Invalid successor");
+}
+
+size_t MBasicBlock::getPredecessorIndex(MBasicBlock* block) const {
+ for (size_t i = 0, e = numPredecessors(); i < e; ++i) {
+ if (getPredecessor(i) == block) {
+ return i;
+ }
+ }
+ MOZ_CRASH("Invalid predecessor");
+}
+
+void MBasicBlock::replaceSuccessor(size_t pos, MBasicBlock* split) {
+ MOZ_ASSERT(lastIns());
+
+ // Note, during split-critical-edges, successors-with-phis is not yet set.
+ // During PAA, this case is handled before we enter.
+ MOZ_ASSERT_IF(successorWithPhis_, successorWithPhis_ != getSuccessor(pos));
+
+ lastIns()->replaceSuccessor(pos, split);
+}
+
+void MBasicBlock::replacePredecessor(MBasicBlock* old, MBasicBlock* split) {
+ for (size_t i = 0; i < numPredecessors(); i++) {
+ if (getPredecessor(i) == old) {
+ predecessors_[i] = split;
+
+#ifdef DEBUG
+ // The same block should not appear twice in the predecessor list.
+ for (size_t j = i; j < numPredecessors(); j++) {
+ MOZ_ASSERT(predecessors_[j] != old);
+ }
+#endif
+
+ return;
+ }
+ }
+
+ MOZ_CRASH("predecessor was not found");
+}
+
+void MBasicBlock::clearDominatorInfo() {
+ setImmediateDominator(nullptr);
+ immediatelyDominated_.clear();
+ numDominated_ = 0;
+}
+
+void MBasicBlock::removePredecessorWithoutPhiOperands(MBasicBlock* pred,
+ size_t predIndex) {
+ // If we're removing the last backedge, this is no longer a loop.
+ if (isLoopHeader() && hasUniqueBackedge() && backedge() == pred) {
+ clearLoopHeader();
+ }
+
+ // Adjust phis. Note that this can leave redundant phis behind.
+ // Don't adjust successorWithPhis() if we haven't constructed this
+ // information yet.
+ if (pred->successorWithPhis()) {
+ MOZ_ASSERT(pred->positionInPhiSuccessor() == predIndex);
+ pred->clearSuccessorWithPhis();
+ for (size_t j = predIndex + 1; j < numPredecessors(); j++) {
+ getPredecessor(j)->setSuccessorWithPhis(this, j - 1);
+ }
+ }
+
+ // Remove from pred list.
+ predecessors_.erase(predecessors_.begin() + predIndex);
+}
+
+void MBasicBlock::removePredecessor(MBasicBlock* pred) {
+ size_t predIndex = getPredecessorIndex(pred);
+
+ // Remove the phi operands.
+ for (MPhiIterator iter(phisBegin()), end(phisEnd()); iter != end; ++iter) {
+ iter->removeOperand(predIndex);
+ }
+
+ // Now we can call the underlying function, which expects that phi
+ // operands have been removed.
+ removePredecessorWithoutPhiOperands(pred, predIndex);
+}
+
+bool MBasicBlock::inheritPhisFromBackedge(MBasicBlock* backedge) {
+ // We must be a pending loop header
+ MOZ_ASSERT(kind_ == PENDING_LOOP_HEADER);
+
+ size_t stackDepth = entryResumePoint()->stackDepth();
+ for (size_t slot = 0; slot < stackDepth; slot++) {
+ // Get the value stack-slot of the back edge.
+ MDefinition* exitDef = backedge->getSlot(slot);
+
+ // Get the value of the loop header.
+ MDefinition* loopDef = entryResumePoint()->getOperand(slot);
+ if (loopDef->block() != this) {
+ // If we are finishing a pending loop header, then we need to ensure
+ // that all operands are phis. This is usualy the case, except for
+ // object/arrays build with generators, in which case we share the
+ // same allocations across all blocks.
+ MOZ_ASSERT(loopDef->block()->id() < id());
+ MOZ_ASSERT(loopDef == exitDef);
+ continue;
+ }
+
+ // Phis are allocated by NewPendingLoopHeader.
+ MPhi* entryDef = loopDef->toPhi();
+ MOZ_ASSERT(entryDef->block() == this);
+
+ if (entryDef == exitDef) {
+ // If the exit def is the same as the entry def, make a redundant
+ // phi. Since loop headers have exactly two incoming edges, we
+ // know that that's just the first input.
+ //
+ // Note that we eliminate later rather than now, to avoid any
+ // weirdness around pending continue edges which might still hold
+ // onto phis.
+ exitDef = entryDef->getOperand(0);
+ }
+
+ if (!entryDef->addInputSlow(exitDef)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+MTest* MBasicBlock::immediateDominatorBranch(BranchDirection* pdirection) {
+ *pdirection = FALSE_BRANCH;
+
+ if (numPredecessors() != 1) {
+ return nullptr;
+ }
+
+ MBasicBlock* dom = immediateDominator();
+ if (dom != getPredecessor(0)) {
+ return nullptr;
+ }
+
+ // Look for a trailing MTest branching to this block.
+ MInstruction* ins = dom->lastIns();
+ if (ins->isTest()) {
+ MTest* test = ins->toTest();
+
+ MOZ_ASSERT(test->ifTrue() == this || test->ifFalse() == this);
+ if (test->ifTrue() == this && test->ifFalse() == this) {
+ return nullptr;
+ }
+
+ *pdirection = (test->ifTrue() == this) ? TRUE_BRANCH : FALSE_BRANCH;
+ return test;
+ }
+
+ return nullptr;
+}
+
+void MBasicBlock::dumpStack(GenericPrinter& out) {
+#ifdef DEBUG
+ out.printf(" %-3s %-16s %-6s %-10s\n", "#", "name", "copyOf", "first/next");
+ out.printf("-------------------------------------------\n");
+ for (uint32_t i = 0; i < stackPosition_; i++) {
+ out.printf(" %-3u", i);
+ out.printf(" %-16p\n", (void*)slots_[i]);
+ }
+#endif
+}
+
+void MBasicBlock::dumpStack() {
+ Fprinter out(stderr);
+ dumpStack(out);
+ out.finish();
+}
+
+void MIRGraph::dump(GenericPrinter& out) {
+#ifdef JS_JITSPEW
+ for (MBasicBlockIterator iter(begin()); iter != end(); iter++) {
+ iter->dump(out);
+ out.printf("\n");
+ }
+#endif
+}
+
+void MIRGraph::dump() {
+ Fprinter out(stderr);
+ dump(out);
+ out.finish();
+}
+
+void MBasicBlock::dump(GenericPrinter& out) {
+#ifdef JS_JITSPEW
+ out.printf("block%u:%s%s%s\n", id(), isLoopHeader() ? " (loop header)" : "",
+ unreachable() ? " (unreachable)" : "",
+ isMarked() ? " (marked)" : "");
+ if (MResumePoint* resume = entryResumePoint()) {
+ resume->dump(out);
+ }
+ for (MPhiIterator iter(phisBegin()); iter != phisEnd(); iter++) {
+ iter->dump(out);
+ }
+ for (MInstructionIterator iter(begin()); iter != end(); iter++) {
+ iter->dump(out);
+ }
+#endif
+}
+
+void MBasicBlock::dump() {
+ Fprinter out(stderr);
+ dump(out);
+ out.finish();
+}
diff --git a/js/src/jit/MIRGraph.h b/js/src/jit/MIRGraph.h
new file mode 100644
index 0000000000..7b0aff0875
--- /dev/null
+++ b/js/src/jit/MIRGraph.h
@@ -0,0 +1,875 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MIRGraph_h
+#define jit_MIRGraph_h
+
+// This file declares the data structures used to build a control-flow graph
+// containing MIR.
+
+#include "jit/CompileInfo.h"
+#include "jit/FixedList.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitAllocPolicy.h"
+#include "jit/MIR.h"
+
+namespace js {
+namespace jit {
+
+class MBasicBlock;
+class MIRGraph;
+class MStart;
+
+class MDefinitionIterator;
+
+using MInstructionIterator = InlineListIterator<MInstruction>;
+using MInstructionReverseIterator = InlineListReverseIterator<MInstruction>;
+using MPhiIterator = InlineListIterator<MPhi>;
+
+#ifdef DEBUG
+typedef InlineForwardListIterator<MResumePoint> MResumePointIterator;
+#endif
+
+class LBlock;
+
+class MBasicBlock : public TempObject, public InlineListNode<MBasicBlock> {
+ public:
+ enum Kind {
+ NORMAL,
+ PENDING_LOOP_HEADER,
+ LOOP_HEADER,
+ SPLIT_EDGE,
+ FAKE_LOOP_PRED,
+ DEAD
+ };
+
+ private:
+ MBasicBlock(MIRGraph& graph, const CompileInfo& info, BytecodeSite* site,
+ Kind kind);
+ [[nodiscard]] bool init();
+ void copySlots(MBasicBlock* from);
+ [[nodiscard]] bool inherit(TempAllocator& alloc, size_t stackDepth,
+ MBasicBlock* maybePred, uint32_t popped);
+
+ // This block cannot be reached by any means.
+ bool unreachable_ = false;
+
+ // This block will unconditionally bail out.
+ bool alwaysBails_ = false;
+
+ // Pushes a copy of a local variable or argument.
+ void pushVariable(uint32_t slot) { push(slots_[slot]); }
+
+ // Sets a variable slot to the top of the stack, correctly creating copies
+ // as needed.
+ void setVariable(uint32_t slot) {
+ MOZ_ASSERT(stackPosition_ > info_.firstStackSlot());
+ setSlot(slot, slots_[stackPosition_ - 1]);
+ }
+
+ enum ReferencesType {
+ RefType_None = 0,
+
+ // Assert that the instruction is unused.
+ RefType_AssertNoUses = 1 << 0,
+
+ // Discard the operands of the resume point / instructions if the
+ // following flag are given too.
+ RefType_DiscardOperands = 1 << 1,
+ RefType_DiscardResumePoint = 1 << 2,
+ RefType_DiscardInstruction = 1 << 3,
+
+ // Discard operands of the instruction and its resume point.
+ RefType_DefaultNoAssert = RefType_DiscardOperands |
+ RefType_DiscardResumePoint |
+ RefType_DiscardInstruction,
+
+ // Discard everything and assert that the instruction is not used.
+ RefType_Default = RefType_AssertNoUses | RefType_DefaultNoAssert,
+
+ // Discard resume point operands only, without discarding the operands
+ // of the current instruction. Asserts that the instruction is unused.
+ RefType_IgnoreOperands = RefType_AssertNoUses | RefType_DiscardOperands |
+ RefType_DiscardResumePoint
+ };
+
+ void discardResumePoint(MResumePoint* rp,
+ ReferencesType refType = RefType_Default);
+
+ // Remove all references to an instruction such that it can be removed from
+ // the list of instruction, without keeping any dangling pointer to it. This
+ // includes the operands of the instruction, and the resume point if
+ // present.
+ void prepareForDiscard(MInstruction* ins,
+ ReferencesType refType = RefType_Default);
+
+ public:
+ ///////////////////////////////////////////////////////
+ ////////// BEGIN GRAPH BUILDING INSTRUCTIONS //////////
+ ///////////////////////////////////////////////////////
+
+ // Creates a new basic block for a MIR generator. If |pred| is not nullptr,
+ // its slots and stack depth are initialized from |pred|.
+ static MBasicBlock* New(MIRGraph& graph, size_t stackDepth,
+ const CompileInfo& info, MBasicBlock* maybePred,
+ BytecodeSite* site, Kind kind);
+ static MBasicBlock* New(MIRGraph& graph, const CompileInfo& info,
+ MBasicBlock* pred, Kind kind);
+ static MBasicBlock* NewPopN(MIRGraph& graph, const CompileInfo& info,
+ MBasicBlock* pred, BytecodeSite* site, Kind kind,
+ uint32_t popn);
+ static MBasicBlock* NewPendingLoopHeader(MIRGraph& graph,
+ const CompileInfo& info,
+ MBasicBlock* pred,
+ BytecodeSite* site);
+ static MBasicBlock* NewSplitEdge(MIRGraph& graph, MBasicBlock* pred,
+ size_t predEdgeIdx, MBasicBlock* succ);
+ static MBasicBlock* NewFakeLoopPredecessor(MIRGraph& graph,
+ MBasicBlock* header);
+
+ bool dominates(const MBasicBlock* other) const {
+ return other->domIndex() - domIndex() < numDominated();
+ }
+
+ void setId(uint32_t id) { id_ = id; }
+
+ // Mark this block (and only this block) as unreachable.
+ void setUnreachable() {
+ MOZ_ASSERT(!unreachable_);
+ setUnreachableUnchecked();
+ }
+ void setUnreachableUnchecked() { unreachable_ = true; }
+ bool unreachable() const { return unreachable_; }
+
+ void setAlwaysBails() { alwaysBails_ = true; }
+ bool alwaysBails() const { return alwaysBails_; }
+
+ // Move the definition to the top of the stack.
+ void pick(int32_t depth);
+
+ // Move the top of the stack definition under the depth-th stack value.
+ void unpick(int32_t depth);
+
+ // Exchange 2 stack slots at the defined depth
+ void swapAt(int32_t depth);
+
+ // Note: most of the methods below are hot. Do not un-inline them without
+ // measuring the impact.
+
+ // Gets the instruction associated with various slot types.
+ MDefinition* peek(int32_t depth) {
+ MOZ_ASSERT(depth < 0);
+ MOZ_ASSERT(stackPosition_ + depth >= info_.firstStackSlot());
+ return peekUnchecked(depth);
+ }
+
+ MDefinition* peekUnchecked(int32_t depth) {
+ MOZ_ASSERT(depth < 0);
+ return getSlot(stackPosition_ + depth);
+ }
+
+ MDefinition* environmentChain();
+ MDefinition* argumentsObject();
+
+ // Increase the number of slots available
+ [[nodiscard]] bool increaseSlots(size_t num);
+ [[nodiscard]] bool ensureHasSlots(size_t num);
+
+ // Initializes a slot value; must not be called for normal stack
+ // operations, as it will not create new SSA names for copies.
+ void initSlot(uint32_t slot, MDefinition* ins) {
+ slots_[slot] = ins;
+ if (entryResumePoint()) {
+ entryResumePoint()->initOperand(slot, ins);
+ }
+ }
+
+ // Sets the instruction associated with various slot types. The
+ // instruction must lie at the top of the stack.
+ void setLocal(uint32_t local) { setVariable(info_.localSlot(local)); }
+ void setArg(uint32_t arg) { setVariable(info_.argSlot(arg)); }
+ void setSlot(uint32_t slot, MDefinition* ins) { slots_[slot] = ins; }
+
+ // Tracks an instruction as being pushed onto the operand stack.
+ void push(MDefinition* ins) {
+ MOZ_ASSERT(stackPosition_ < nslots());
+ slots_[stackPosition_++] = ins;
+ }
+ void pushArg(uint32_t arg) { pushVariable(info_.argSlot(arg)); }
+ void pushLocal(uint32_t local) { pushVariable(info_.localSlot(local)); }
+ void pushSlot(uint32_t slot) { pushVariable(slot); }
+ void setEnvironmentChain(MDefinition* ins);
+ void setArgumentsObject(MDefinition* ins);
+
+ // Returns the top of the stack, then decrements the virtual stack pointer.
+ MDefinition* pop() {
+ MOZ_ASSERT(stackPosition_ > info_.firstStackSlot());
+ return slots_[--stackPosition_];
+ }
+ void popn(uint32_t n) {
+ MOZ_ASSERT(stackPosition_ - n >= info_.firstStackSlot());
+ MOZ_ASSERT(stackPosition_ >= stackPosition_ - n);
+ stackPosition_ -= n;
+ }
+
+ // Adds an instruction to this block's instruction list.
+ inline void add(MInstruction* ins);
+
+ // Marks the last instruction of the block; no further instructions
+ // can be added.
+ void end(MControlInstruction* ins) {
+ MOZ_ASSERT(!hasLastIns()); // Existing control instructions should be
+ // removed first.
+ MOZ_ASSERT(ins);
+ add(ins);
+ }
+
+ // Adds a phi instruction, but does not set successorWithPhis.
+ void addPhi(MPhi* phi);
+
+ // Adds a resume point to this block.
+ void addResumePoint(MResumePoint* resume) {
+#ifdef DEBUG
+ resumePoints_.pushFront(resume);
+#endif
+ }
+
+ // Discard pre-allocated resume point.
+ void discardPreAllocatedResumePoint(MResumePoint* resume) {
+ MOZ_ASSERT(!resume->instruction());
+ discardResumePoint(resume);
+ }
+
+ // Adds a predecessor. Every predecessor must have the same exit stack
+ // depth as the entry state to this block. Adding a predecessor
+ // automatically creates phi nodes and rewrites uses as needed.
+ [[nodiscard]] bool addPredecessor(TempAllocator& alloc, MBasicBlock* pred);
+ [[nodiscard]] bool addPredecessorPopN(TempAllocator& alloc, MBasicBlock* pred,
+ uint32_t popped);
+
+ // Add a predecessor which won't introduce any new phis to this block.
+ // This may be called after the contents of this block have been built.
+ [[nodiscard]] bool addPredecessorSameInputsAs(MBasicBlock* pred,
+ MBasicBlock* existingPred);
+
+ // Stranger utilities used for inlining.
+ [[nodiscard]] bool addPredecessorWithoutPhis(MBasicBlock* pred);
+ void inheritSlots(MBasicBlock* parent);
+ [[nodiscard]] bool initEntrySlots(TempAllocator& alloc);
+
+ // Replaces an edge for a given block with a new block. This is
+ // used for critical edge splitting.
+ //
+ // Note: If successorWithPhis is set, you must not be replacing it.
+ void replacePredecessor(MBasicBlock* old, MBasicBlock* split);
+ void replaceSuccessor(size_t pos, MBasicBlock* split);
+
+ // Removes `pred` from the predecessor list. If this block defines phis,
+ // removes the entry for `pred` and updates the indices of later entries.
+ // This may introduce redundant phis if the new block has fewer
+ // than two predecessors.
+ void removePredecessor(MBasicBlock* pred);
+
+ // A version of removePredecessor which expects that phi operands to
+ // |pred| have already been removed.
+ void removePredecessorWithoutPhiOperands(MBasicBlock* pred, size_t predIndex);
+
+ // Resets all the dominator info so that it can be recomputed.
+ void clearDominatorInfo();
+
+ // Sets a back edge. This places phi nodes and rewrites instructions within
+ // the current loop as necessary.
+ [[nodiscard]] bool setBackedge(MBasicBlock* block);
+ [[nodiscard]] bool setBackedgeWasm(MBasicBlock* block, size_t paramCount);
+
+ // Resets a LOOP_HEADER block to a NORMAL block. This is needed when
+ // optimizations remove the backedge.
+ void clearLoopHeader();
+
+ // Sets a block to a LOOP_HEADER block, with newBackedge as its backedge.
+ // This is needed when optimizations remove the normal entry to a loop
+ // with multiple entries.
+ void setLoopHeader(MBasicBlock* newBackedge);
+
+ // Propagates backedge slots into phis operands of the loop header.
+ [[nodiscard]] bool inheritPhisFromBackedge(MBasicBlock* backedge);
+
+ void insertBefore(MInstruction* at, MInstruction* ins);
+ void insertAfter(MInstruction* at, MInstruction* ins);
+
+ void insertAtEnd(MInstruction* ins);
+
+ // Move an instruction. Movement may cross block boundaries.
+ void moveBefore(MInstruction* at, MInstruction* ins);
+
+ enum IgnoreTop { IgnoreNone = 0, IgnoreRecover = 1 << 0 };
+
+ // Locate the top of the |block|, where it is safe to insert a new
+ // instruction.
+ MInstruction* safeInsertTop(MDefinition* ins = nullptr,
+ IgnoreTop ignore = IgnoreNone);
+
+ // Removes an instruction with the intention to discard it.
+ void discard(MInstruction* ins);
+ void discardLastIns();
+ void discardAllInstructions();
+ void discardAllInstructionsStartingAt(MInstructionIterator iter);
+ void discardAllPhis();
+ void discardAllResumePoints(bool discardEntry = true);
+ void clear();
+
+ // Same as |void discard(MInstruction* ins)| but assuming that
+ // all operands are already discarded.
+ void discardIgnoreOperands(MInstruction* ins);
+
+ // Discards a phi instruction and updates predecessor successorWithPhis.
+ void discardPhi(MPhi* phi);
+
+ // Some instruction which are guarding against some MIRType value, or
+ // against a type expectation should be considered as removing a potenatial
+ // branch where the guard does not hold. We need to register such
+ // instructions in order to do destructive optimizations correctly, such as
+ // Range Analysis.
+ void flagOperandsOfPrunedBranches(MInstruction* ins);
+
+ // Mark this block as having been removed from the graph.
+ void markAsDead() {
+ MOZ_ASSERT(kind_ != DEAD);
+ kind_ = DEAD;
+ }
+
+ ///////////////////////////////////////////////////////
+ /////////// END GRAPH BUILDING INSTRUCTIONS ///////////
+ ///////////////////////////////////////////////////////
+
+ MIRGraph& graph() { return graph_; }
+ const CompileInfo& info() const { return info_; }
+ jsbytecode* pc() const { return trackedSite_->pc(); }
+ uint32_t nslots() const { return slots_.length(); }
+ uint32_t id() const { return id_; }
+ uint32_t numPredecessors() const { return predecessors_.length(); }
+
+ uint32_t domIndex() const {
+ MOZ_ASSERT(!isDead());
+ return domIndex_;
+ }
+ void setDomIndex(uint32_t d) { domIndex_ = d; }
+
+ MBasicBlock* getPredecessor(uint32_t i) const { return predecessors_[i]; }
+ size_t indexForPredecessor(MBasicBlock* block) const {
+ // This should only be called before critical edge splitting.
+ MOZ_ASSERT(!block->successorWithPhis());
+
+ for (size_t i = 0; i < predecessors_.length(); i++) {
+ if (predecessors_[i] == block) {
+ return i;
+ }
+ }
+ MOZ_CRASH();
+ }
+ bool hasAnyIns() const { return !instructions_.empty(); }
+ bool hasLastIns() const {
+ return hasAnyIns() && instructions_.rbegin()->isControlInstruction();
+ }
+ MControlInstruction* lastIns() const {
+ MOZ_ASSERT(hasLastIns());
+ return instructions_.rbegin()->toControlInstruction();
+ }
+ // Find or allocate an optimized out constant.
+ MConstant* optimizedOutConstant(TempAllocator& alloc);
+ MPhiIterator phisBegin() const { return phis_.begin(); }
+ MPhiIterator phisBegin(MPhi* at) const { return phis_.begin(at); }
+ MPhiIterator phisEnd() const { return phis_.end(); }
+ bool phisEmpty() const { return phis_.empty(); }
+#ifdef DEBUG
+ MResumePointIterator resumePointsBegin() const {
+ return resumePoints_.begin();
+ }
+ MResumePointIterator resumePointsEnd() const { return resumePoints_.end(); }
+ bool resumePointsEmpty() const { return resumePoints_.empty(); }
+#endif
+ MInstructionIterator begin() { return instructions_.begin(); }
+ MInstructionIterator begin(MInstruction* at) {
+ MOZ_ASSERT(at->block() == this);
+ return instructions_.begin(at);
+ }
+ MInstructionIterator end() { return instructions_.end(); }
+ MInstructionReverseIterator rbegin() { return instructions_.rbegin(); }
+ MInstructionReverseIterator rbegin(MInstruction* at) {
+ MOZ_ASSERT(at->block() == this);
+ return instructions_.rbegin(at);
+ }
+ MInstructionReverseIterator rend() { return instructions_.rend(); }
+
+ bool isLoopHeader() const { return kind_ == LOOP_HEADER; }
+ bool isPendingLoopHeader() const { return kind_ == PENDING_LOOP_HEADER; }
+
+ bool hasUniqueBackedge() const {
+ MOZ_ASSERT(isLoopHeader());
+ MOZ_ASSERT(numPredecessors() >= 1);
+ if (numPredecessors() == 1 || numPredecessors() == 2) {
+ return true;
+ }
+ if (numPredecessors() == 3) {
+ // fixup block added by NewFakeLoopPredecessor
+ return getPredecessor(1)->numPredecessors() == 0;
+ }
+ return false;
+ }
+ MBasicBlock* backedge() const {
+ MOZ_ASSERT(hasUniqueBackedge());
+ return getPredecessor(numPredecessors() - 1);
+ }
+ MBasicBlock* loopHeaderOfBackedge() const {
+ MOZ_ASSERT(isLoopBackedge());
+ return getSuccessor(numSuccessors() - 1);
+ }
+ MBasicBlock* loopPredecessor() const {
+ MOZ_ASSERT(isLoopHeader());
+ return getPredecessor(0);
+ }
+ bool isLoopBackedge() const {
+ if (!numSuccessors()) {
+ return false;
+ }
+ MBasicBlock* lastSuccessor = getSuccessor(numSuccessors() - 1);
+ return lastSuccessor->isLoopHeader() &&
+ lastSuccessor->hasUniqueBackedge() &&
+ lastSuccessor->backedge() == this;
+ }
+ bool isSplitEdge() const { return kind_ == SPLIT_EDGE; }
+ bool isDead() const { return kind_ == DEAD; }
+ bool isFakeLoopPred() const { return kind_ == FAKE_LOOP_PRED; }
+
+ uint32_t stackDepth() const { return stackPosition_; }
+ bool isMarked() const { return mark_; }
+ void mark() {
+ MOZ_ASSERT(!mark_, "Marking already-marked block");
+ markUnchecked();
+ }
+ void markUnchecked() { mark_ = true; }
+ void unmark() {
+ MOZ_ASSERT(mark_, "Unarking unmarked block");
+ unmarkUnchecked();
+ }
+ void unmarkUnchecked() { mark_ = false; }
+
+ MBasicBlock* immediateDominator() const { return immediateDominator_; }
+
+ void setImmediateDominator(MBasicBlock* dom) { immediateDominator_ = dom; }
+
+ MTest* immediateDominatorBranch(BranchDirection* pdirection);
+
+ size_t numImmediatelyDominatedBlocks() const {
+ return immediatelyDominated_.length();
+ }
+
+ MBasicBlock* getImmediatelyDominatedBlock(size_t i) const {
+ return immediatelyDominated_[i];
+ }
+
+ MBasicBlock** immediatelyDominatedBlocksBegin() {
+ return immediatelyDominated_.begin();
+ }
+
+ MBasicBlock** immediatelyDominatedBlocksEnd() {
+ return immediatelyDominated_.end();
+ }
+
+ // Return the number of blocks dominated by this block. All blocks
+ // dominate at least themselves, so this will always be non-zero.
+ size_t numDominated() const {
+ MOZ_ASSERT(numDominated_ != 0);
+ return numDominated_;
+ }
+
+ void addNumDominated(size_t n) { numDominated_ += n; }
+
+ // Add |child| to this block's immediately-dominated set.
+ bool addImmediatelyDominatedBlock(MBasicBlock* child);
+
+ // Remove |child| from this block's immediately-dominated set.
+ void removeImmediatelyDominatedBlock(MBasicBlock* child);
+
+ // This function retrieves the internal instruction associated with a
+ // slot, and should not be used for normal stack operations. It is an
+ // internal helper that is also used to enhance spew.
+ MDefinition* getSlot(uint32_t index) {
+ MOZ_ASSERT(index < stackPosition_);
+ return slots_[index];
+ }
+
+ MResumePoint* entryResumePoint() const { return entryResumePoint_; }
+ void setEntryResumePoint(MResumePoint* rp) { entryResumePoint_ = rp; }
+ void clearEntryResumePoint() {
+ discardResumePoint(entryResumePoint_);
+ entryResumePoint_ = nullptr;
+ }
+ MResumePoint* outerResumePoint() const { return outerResumePoint_; }
+ void setOuterResumePoint(MResumePoint* outer) {
+ MOZ_ASSERT(!outerResumePoint_);
+ outerResumePoint_ = outer;
+ }
+ void clearOuterResumePoint() {
+ discardResumePoint(outerResumePoint_);
+ outerResumePoint_ = nullptr;
+ }
+ MResumePoint* callerResumePoint() const { return callerResumePoint_; }
+ void setCallerResumePoint(MResumePoint* caller) {
+ callerResumePoint_ = caller;
+ }
+
+ LBlock* lir() const { return lir_; }
+ void assignLir(LBlock* lir) {
+ MOZ_ASSERT(!lir_);
+ lir_ = lir;
+ }
+
+ MBasicBlock* successorWithPhis() const { return successorWithPhis_; }
+ uint32_t positionInPhiSuccessor() const {
+ MOZ_ASSERT(successorWithPhis());
+ return positionInPhiSuccessor_;
+ }
+ void setSuccessorWithPhis(MBasicBlock* successor, uint32_t id) {
+ successorWithPhis_ = successor;
+ positionInPhiSuccessor_ = id;
+ }
+ void clearSuccessorWithPhis() { successorWithPhis_ = nullptr; }
+ size_t numSuccessors() const {
+ MOZ_ASSERT(lastIns());
+ return lastIns()->numSuccessors();
+ }
+ MBasicBlock* getSuccessor(size_t index) const {
+ MOZ_ASSERT(lastIns());
+ return lastIns()->getSuccessor(index);
+ }
+ MBasicBlock* getSingleSuccessor() const {
+ MOZ_ASSERT(numSuccessors() == 1);
+ return getSuccessor(0);
+ }
+ size_t getSuccessorIndex(MBasicBlock*) const;
+ size_t getPredecessorIndex(MBasicBlock*) const;
+
+ void setLoopDepth(uint32_t loopDepth) { loopDepth_ = loopDepth; }
+ uint32_t loopDepth() const { return loopDepth_; }
+
+ void dumpStack(GenericPrinter& out);
+ void dumpStack();
+
+ void dump(GenericPrinter& out);
+ void dump();
+
+ BytecodeSite* trackedSite() const { return trackedSite_; }
+ InlineScriptTree* trackedTree() const { return trackedSite_->tree(); }
+
+ private:
+ MIRGraph& graph_;
+ const CompileInfo& info_; // Each block originates from a particular script.
+ InlineList<MInstruction> instructions_;
+ Vector<MBasicBlock*, 1, JitAllocPolicy> predecessors_;
+ InlineList<MPhi> phis_;
+ FixedList<MDefinition*> slots_;
+ uint32_t stackPosition_;
+ uint32_t id_;
+ uint32_t domIndex_; // Index in the dominator tree.
+ uint32_t numDominated_;
+ LBlock* lir_;
+
+ // Copy of a dominator block's outerResumePoint_ which holds the state of
+ // caller frame at the time of the call. If not null, this implies that this
+ // basic block corresponds to an inlined script.
+ MResumePoint* callerResumePoint_;
+
+ // Resume point holding baseline-like frame for the PC corresponding to the
+ // entry of this basic block.
+ MResumePoint* entryResumePoint_;
+
+ // Resume point holding baseline-like frame for the PC corresponding to the
+ // beginning of the call-site which is being inlined after this block.
+ MResumePoint* outerResumePoint_;
+
+#ifdef DEBUG
+ // Unordered list used to verify that all the resume points which are
+ // registered are correctly removed when a basic block is removed.
+ InlineForwardList<MResumePoint> resumePoints_;
+#endif
+
+ MBasicBlock* successorWithPhis_;
+ uint32_t positionInPhiSuccessor_;
+ uint32_t loopDepth_;
+ Kind kind_ : 8;
+
+ // Utility mark for traversal algorithms.
+ bool mark_;
+
+ Vector<MBasicBlock*, 1, JitAllocPolicy> immediatelyDominated_;
+ MBasicBlock* immediateDominator_;
+
+ // Track bailouts by storing the current pc in MIR instruction added at
+ // this cycle. This is also used for tracking calls and optimizations when
+ // profiling.
+ BytecodeSite* trackedSite_;
+
+ unsigned lineno_;
+ unsigned columnIndex_;
+
+ public:
+ void setLineno(unsigned l) { lineno_ = l; }
+ unsigned lineno() const { return lineno_; }
+ void setColumnIndex(unsigned c) { columnIndex_ = c; }
+ unsigned columnIndex() const { return columnIndex_; }
+};
+
+using MBasicBlockIterator = InlineListIterator<MBasicBlock>;
+using ReversePostorderIterator = InlineListIterator<MBasicBlock>;
+using PostorderIterator = InlineListReverseIterator<MBasicBlock>;
+
+typedef Vector<MBasicBlock*, 1, JitAllocPolicy> MIRGraphReturns;
+
+class MIRGraph {
+ InlineList<MBasicBlock> blocks_;
+ TempAllocator* alloc_;
+ MIRGraphReturns* returnAccumulator_;
+ uint32_t blockIdGen_;
+ uint32_t idGen_;
+ MBasicBlock* osrBlock_;
+
+ size_t numBlocks_;
+ bool hasTryBlock_;
+
+ InlineList<MPhi> phiFreeList_;
+ size_t phiFreeListLength_;
+
+ public:
+ explicit MIRGraph(TempAllocator* alloc)
+ : alloc_(alloc),
+ returnAccumulator_(nullptr),
+ blockIdGen_(0),
+ idGen_(0),
+ osrBlock_(nullptr),
+ numBlocks_(0),
+ hasTryBlock_(false),
+ phiFreeListLength_(0) {}
+
+ TempAllocator& alloc() const { return *alloc_; }
+
+ void addBlock(MBasicBlock* block);
+ void insertBlockAfter(MBasicBlock* at, MBasicBlock* block);
+ void insertBlockBefore(MBasicBlock* at, MBasicBlock* block);
+
+ void unmarkBlocks();
+
+ void setReturnAccumulator(MIRGraphReturns* accum) {
+ returnAccumulator_ = accum;
+ }
+ MIRGraphReturns* returnAccumulator() const { return returnAccumulator_; }
+
+ [[nodiscard]] bool addReturn(MBasicBlock* returnBlock) {
+ if (!returnAccumulator_) {
+ return true;
+ }
+
+ return returnAccumulator_->append(returnBlock);
+ }
+
+ MBasicBlock* entryBlock() { return *blocks_.begin(); }
+ MBasicBlockIterator begin() { return blocks_.begin(); }
+ MBasicBlockIterator begin(MBasicBlock* at) { return blocks_.begin(at); }
+ MBasicBlockIterator end() { return blocks_.end(); }
+ PostorderIterator poBegin() { return blocks_.rbegin(); }
+ PostorderIterator poBegin(MBasicBlock* at) { return blocks_.rbegin(at); }
+ PostorderIterator poEnd() { return blocks_.rend(); }
+ ReversePostorderIterator rpoBegin() { return blocks_.begin(); }
+ ReversePostorderIterator rpoBegin(MBasicBlock* at) {
+ return blocks_.begin(at);
+ }
+ ReversePostorderIterator rpoEnd() { return blocks_.end(); }
+ void removeBlock(MBasicBlock* block);
+ void moveBlockToEnd(MBasicBlock* block) {
+ blocks_.remove(block);
+ MOZ_ASSERT_IF(!blocks_.empty(), block->id());
+ blocks_.pushBack(block);
+ }
+ void moveBlockBefore(MBasicBlock* at, MBasicBlock* block) {
+ MOZ_ASSERT(block->id());
+ blocks_.remove(block);
+ blocks_.insertBefore(at, block);
+ }
+ void moveBlockAfter(MBasicBlock* at, MBasicBlock* block) {
+ MOZ_ASSERT(block->id());
+ blocks_.remove(block);
+ blocks_.insertAfter(at, block);
+ }
+ size_t numBlocks() const { return numBlocks_; }
+ uint32_t numBlockIds() const { return blockIdGen_; }
+ void allocDefinitionId(MDefinition* ins) { ins->setId(idGen_++); }
+ uint32_t getNumInstructionIds() { return idGen_; }
+ MResumePoint* entryResumePoint() { return entryBlock()->entryResumePoint(); }
+
+ void setOsrBlock(MBasicBlock* osrBlock) {
+ MOZ_ASSERT(!osrBlock_);
+ osrBlock_ = osrBlock;
+ }
+ MBasicBlock* osrBlock() const { return osrBlock_; }
+
+ MBasicBlock* osrPreHeaderBlock() const {
+ return osrBlock() ? osrBlock()->getSingleSuccessor() : nullptr;
+ }
+
+ bool hasTryBlock() const { return hasTryBlock_; }
+ void setHasTryBlock() { hasTryBlock_ = true; }
+
+ void dump(GenericPrinter& out);
+ void dump();
+
+ void addPhiToFreeList(MPhi* phi) {
+ phiFreeList_.pushBack(phi);
+ phiFreeListLength_++;
+ }
+ size_t phiFreeListLength() const { return phiFreeListLength_; }
+ MPhi* takePhiFromFreeList() {
+ MOZ_ASSERT(phiFreeListLength_ > 0);
+ phiFreeListLength_--;
+ return phiFreeList_.popBack();
+ }
+
+ void removeFakeLoopPredecessors();
+
+#ifdef DEBUG
+ // Dominators can't be built after we remove fake loop predecessors.
+ private:
+ bool canBuildDominators_ = true;
+
+ public:
+ bool canBuildDominators() const { return canBuildDominators_; }
+#endif
+};
+
+class MDefinitionIterator {
+ friend class MBasicBlock;
+ friend class MNodeIterator;
+
+ private:
+ MBasicBlock* block_;
+ MPhiIterator phiIter_;
+ MInstructionIterator iter_;
+
+ bool atPhi() const { return phiIter_ != block_->phisEnd(); }
+
+ MDefinition* getIns() {
+ if (atPhi()) {
+ return *phiIter_;
+ }
+ return *iter_;
+ }
+
+ bool more() const { return atPhi() || (*iter_) != block_->lastIns(); }
+
+ public:
+ explicit MDefinitionIterator(MBasicBlock* block)
+ : block_(block), phiIter_(block->phisBegin()), iter_(block->begin()) {}
+
+ MDefinitionIterator operator++() {
+ MOZ_ASSERT(more());
+ if (atPhi()) {
+ ++phiIter_;
+ } else {
+ ++iter_;
+ }
+ return *this;
+ }
+
+ MDefinitionIterator operator++(int) {
+ MDefinitionIterator old(*this);
+ operator++();
+ return old;
+ }
+
+ explicit operator bool() const { return more(); }
+
+ MDefinition* operator*() { return getIns(); }
+
+ MDefinition* operator->() { return getIns(); }
+};
+
+// Iterates on all resume points, phis, and instructions of a MBasicBlock.
+// Resume points are visited as long as they have not been discarded.
+class MNodeIterator {
+ private:
+ // If this is non-null, the resume point that we will visit next (unless
+ // it has been discarded). Initialized to the entry resume point.
+ // Otherwise, resume point of the most recently visited instruction.
+ MResumePoint* resumePoint_;
+
+ mozilla::DebugOnly<MInstruction*> lastInstruction_ = nullptr;
+
+ // Definition iterator which is one step ahead when visiting resume points.
+ // This is in order to avoid incrementing the iterator while it is settled
+ // on a discarded instruction.
+ MDefinitionIterator defIter_;
+
+ MBasicBlock* block() const { return defIter_.block_; }
+
+ bool atResumePoint() const {
+ MOZ_ASSERT_IF(lastInstruction_ && !lastInstruction_->isDiscarded(),
+ lastInstruction_->resumePoint() == resumePoint_);
+ return resumePoint_ && !resumePoint_->isDiscarded();
+ }
+
+ MNode* getNode() {
+ if (atResumePoint()) {
+ return resumePoint_;
+ }
+ return *defIter_;
+ }
+
+ void next() {
+ if (!atResumePoint()) {
+ if (defIter_->isInstruction()) {
+ resumePoint_ = defIter_->toInstruction()->resumePoint();
+ lastInstruction_ = defIter_->toInstruction();
+ }
+ defIter_++;
+ } else {
+ resumePoint_ = nullptr;
+ lastInstruction_ = nullptr;
+ }
+ }
+
+ bool more() const { return defIter_ || atResumePoint(); }
+
+ public:
+ explicit MNodeIterator(MBasicBlock* block)
+ : resumePoint_(block->entryResumePoint()), defIter_(block) {
+ MOZ_ASSERT(bool(block->entryResumePoint()) == atResumePoint());
+ }
+
+ MNodeIterator operator++(int) {
+ MNodeIterator old(*this);
+ if (more()) {
+ next();
+ }
+ return old;
+ }
+
+ explicit operator bool() const { return more(); }
+
+ MNode* operator*() { return getNode(); }
+
+ MNode* operator->() { return getNode(); }
+};
+
+void MBasicBlock::add(MInstruction* ins) {
+ MOZ_ASSERT(!hasLastIns());
+ ins->setInstructionBlock(this, trackedSite_);
+ graph().allocDefinitionId(ins);
+ instructions_.pushBack(ins);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_MIRGraph_h */
diff --git a/js/src/jit/MIROps.yaml b/js/src/jit/MIROps.yaml
new file mode 100644
index 0000000000..bf525cb625
--- /dev/null
+++ b/js/src/jit/MIROps.yaml
@@ -0,0 +1,2972 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# [SMDOC] MIR Opcodes
+# =======================
+# This file defines all MIR opcodes. It is parsed by GenerateMIRFiles.py
+# at build time to create MIROpsGenerated.h. Each opcode consists of a
+# name and a set of attributes that are described below. A few of the
+# attributes below allow setting the value to "custom", meaning the
+# method will be declared for the MIR op, but will need to be implemented
+# in C++ (typically done in MIR.cpp). Unless marked as required, attributes
+# are optional.
+#
+# name [required]
+# ====
+# Opcode name.
+# Possible values:
+# - opcode string: used as the name for MIR opcode.
+#
+# gen_boilerplate
+# ===============
+# Used to decide to generate MIR boilerplate.
+# - true (default): auto generate boilerplate for this MIR opcode
+# - false: do not generate boilerplate for this MIR opcode
+#
+# operands
+# ========
+# A list of operands for the MIR op class constructor. Each operand is a
+# MIR node. The operand kind is specified from the one of the kinds from
+# the MIRType enum in IonTypes.h. The specified types for the
+# operands will decide the type policy for the instruction.
+#
+# The naming of operands is how the NAMED_OPERANDS macro will define
+# its operands.
+#
+# For example:
+# object: Object
+# id: Value
+# value: Object
+#
+# Will result in an instruction having the type policy of:
+# MixPolicy<ObjectPolicy<0>, BoxPolicy<1>, ObjectPolicy<2>>
+# and a named operands definition that looks like the following:
+# NAMED_OPERANDS((0, object), (1, idValue), (2, value))
+#
+# - attribute not specified (default): no code generated
+# - operand list: MIRTypes (See MIRType in jit/IonTypes.h)
+#
+# arguments
+# =========
+# A list of non-MIR node arguments to the MIR op class constructor
+# that are passed along with the operands. The arguments require
+# both a name and a full type signature for each item in the list.
+#
+# For example:
+# templateObject: JSObject*
+# initialHeap: gc::InitialHeap
+#
+# For each argument a private variable declaration will be autogenerated
+# in the MIR op class, as well as simple accessor for that variable. If
+# the type of the variable is a GC pointer it will by automatically
+# wrapped by CompilerGCPointer. The above arguments list will result in
+# the following declarations and accessors:
+#
+# CompilerGCPointer<JSObject*> templateObject_;
+# gc::InitialHeap initialHeap_;
+#
+# JSObject* templateObject() const { return templateObject_; }
+# gc::InitialHeap initialHeap() const { return initialHeap_; }
+#
+# - attribute not specified (default): no code generated
+# - operand list: argument names and their full type signature
+#
+# type_policy
+# ============
+# If this attribute is present, then the type policy for that opcode will be
+# NoTypePolicy. This is used for opcode that should have no type policy.
+# - attribute not specified (default): no code generated, type policy
+# is based off of operands
+# - none: defines the type policy as opcode's NoTypePolicy
+#
+# result_type
+# ===========
+# Defines the result type of the MIR opcode.
+# - attribute not specified (default): no code is generated
+# - MIRType string: Will add a call to setResultType to the opcode constructor.
+# This will set the MIR opcodes result type to whatever the
+# specified MIRType is (See MIRType in jit/IonTypes.h).
+#
+# guard
+# =====
+# Set if the opcode is a guard instruction and is used for checks in optimizations
+# such as range analysis and value numbering.
+# - attribute not specified (default): no code generated
+# - true: adds setGuard to opcode constructor
+#
+# movable
+# =======
+# Defines the movable MIR flag for movable instructions. This is used for knowing
+# if we can hoist an instruction.
+# - attribute not specified (default): no code generated
+# - true: adds setMovable call in opcode constructor
+#
+# folds_to
+# ========
+# The foldsTo method is used for determining if an instruction can be folded into
+# simpler instruction or for constant folding, depending on its operands.
+# - attribute not specified (default): no code generated, no constants to fold
+# - custom: custom C++ implementation
+#
+# congruent_to
+# ============
+# Used by ValueNumbering to determine if two values are congruent.
+# - attribute not specified (default): no code generated, congruentTo(foo) returns
+# false
+# - if_operands_equal: congruentTo(foo) will return congruentIfOperandsEqual(foo)
+# - custom: custom C++ implementation
+#
+# alias_set
+# =========
+# Defines the getAliasSet function for a MIR op. The alias set is used for alias
+# analysis. The default alias set is Any.
+# - attribute not specified (default): no code generated, alias set is Any
+# - none: this is the most common case, this is will set the alias set to None.
+# - custom: custom C++ implementation in MIR.cpp
+#
+# possibly_calls
+# ==============
+# Defines if a opcode can possibly call.
+# - attribute not specified (default): no code generated, opcode does not call
+# - true: possiblyCalls returns true
+# - custom: custom C++ implementation
+#
+# compute_range
+# =============
+# Computes and sets the range value for a MIR node, which is then used in range
+# analysis.
+# - attribute not specified (default): no code generated, range is not set for node
+# - custom: custom C++ implementation in RangeAnalysis.cpp
+#
+# can_recover
+# ===========
+# Indicates whether this instruction can be recovered on bailout.
+# Possible values:
+# - attribute not specified (default): no code generated, canRecoverOnBailout
+# returns false
+# - true: canRecoverOnBailout returns true
+# - custom: canRecoverOnBailout has a custom C++ implementation
+# If the value is either 'true' or 'custom', writeRecoverData has a custom C++
+# implementation.
+#
+# clone
+# =====
+# Allows cloning for that MIR op.
+# - attribute not specified (default): no code generated
+# - true: allows cloning
+#
+
+# TODO(no-TI): try to remove this instruction.
+- name: Start
+
+# Instruction marking on entrypoint for on-stack replacement.
+# OSR may occur at loop headers (at JSOp::LoopHead).
+# There is at most one MOsrEntry per MIRGraph.
+- name: OsrEntry
+ result_type: Pointer
+
+- name: Nop
+ alias_set: none
+ clone: true
+
+- name: LimitedTruncate
+ gen_boilerplate: false
+
+- name: Constant
+ gen_boilerplate: false
+
+- name: WasmNullConstant
+ gen_boilerplate: false
+
+- name: WasmFloatConstant
+ gen_boilerplate: false
+
+- name: Parameter
+ gen_boilerplate: false
+
+- name: Callee
+ result_type: Object
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: IsConstructing
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: TableSwitch
+ gen_boilerplate: false
+
+- name: Goto
+ gen_boilerplate: false
+
+- name: Test
+ gen_boilerplate: false
+
+- name: Return
+ gen_boilerplate: false
+
+- name: Throw
+ operands:
+ ins: Value
+ alias_set: custom
+ possibly_calls: true
+
+- name: NewArray
+ gen_boilerplate: false
+
+- name: NewArrayDynamicLength
+ operands:
+ length: Int32
+ arguments:
+ templateObject: JSObject*
+ initialHeap: gc::InitialHeap
+ result_type: Object
+ # Need to throw if length is negative.
+ guard: true
+ # Throws if length is negative.
+ alias_set: custom
+
+- name: NewTypedArray
+ gen_boilerplate: false
+
+- name: NewTypedArrayDynamicLength
+ operands:
+ length: Int32
+ arguments:
+ templateObject: JSObject*
+ initialHeap: gc::InitialHeap
+ result_type: Object
+ guard: true
+ # Throws if length is negative.
+ alias_set: custom
+
+# Create a new TypedArray from an Array (or Array-like object) or a TypedArray.
+- name: NewTypedArrayFromArray
+ operands:
+ array: Object
+ arguments:
+ templateObject: JSObject*
+ initialHeap: gc::InitialHeap
+ result_type: Object
+ guard: true
+ possibly_calls: true
+
+# Create a new TypedArray from an ArrayBuffer (or SharedArrayBuffer).
+- name: NewTypedArrayFromArrayBuffer
+ operands:
+ arrayBuffer: Object
+ byteOffset: Value
+ length: Value
+ arguments:
+ templateObject: JSObject*
+ initialHeap: gc::InitialHeap
+ result_type: Object
+ guard: true
+ possibly_calls: true
+
+- name: NewObject
+ gen_boilerplate: false
+
+- name: NewPlainObject
+ gen_boilerplate: false
+
+- name: NewArrayObject
+ gen_boilerplate: false
+
+- name: NewIterator
+ gen_boilerplate: false
+
+- name: ObjectState
+ gen_boilerplate: false
+
+- name: ArrayState
+ gen_boilerplate: false
+
+# Setting __proto__ in an object literal.
+- name: MutateProto
+ operands:
+ object: Object
+ value: Value
+ result_type: None
+ possibly_calls: true
+
+- name: InitPropGetterSetter
+ operands:
+ object: Object
+ value: Object
+ arguments:
+ name: PropertyName*
+
+- name: InitElemGetterSetter
+ operands:
+ object: Object
+ id: Value
+ value: Object
+
+- name: Call
+ gen_boilerplate: false
+
+- name: ApplyArgs
+ gen_boilerplate: false
+
+- name: ApplyArgsObj
+ gen_boilerplate: false
+
+- name: ApplyArray
+ gen_boilerplate: false
+
+- name: ConstructArgs
+ gen_boilerplate: false
+
+- name: ConstructArray
+ gen_boilerplate: false
+
+- name: Bail
+ gen_boilerplate: false
+
+- name: Unreachable
+ gen_boilerplate: false
+
+# This op serves as a way to force the encoding of a snapshot, even if there
+# is no resume point using it. This is useful to run MAssertRecoveredOnBailout
+# assertions.
+- name: EncodeSnapshot
+ guard: true
+
+- name: AssertRecoveredOnBailout
+ gen_boilerplate: false
+
+- name: AssertFloat32
+ gen_boilerplate: false
+
+- name: Compare
+ gen_boilerplate: false
+
+- name: SameValueDouble
+ operands:
+ left: Double
+ right: Double
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ clone: true
+
+- name: SameValue
+ operands:
+ left: Value
+ right: Value
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ clone: true
+
+- name: Box
+ gen_boilerplate: false
+
+- name: Unbox
+ gen_boilerplate: false
+
+- name: AssertRange
+ gen_boilerplate: false
+
+- name: AssertClass
+ gen_boilerplate: false
+
+- name: AssertShape
+ gen_boilerplate: false
+
+# Caller-side allocation of |this| for |new|:
+# Constructs |this| when possible, else MagicValue(JS_IS_CONSTRUCTING).
+- name: CreateThis
+ operands:
+ callee: Object
+ newTarget: Object
+ result_type: Value
+ # Performs a property read from |newTarget| iff |newTarget| is a JSFunction
+ # with an own |.prototype| property.
+ alias_set: custom
+ possibly_calls: true
+
+- name: CreateArgumentsObject
+ gen_boilerplate: false
+
+- name: CreateInlinedArgumentsObject
+ gen_boilerplate: false
+
+- name: GetInlinedArgument
+ gen_boilerplate: false
+
+- name: GetInlinedArgumentHole
+ gen_boilerplate: false
+
+- name: GetArgumentsObjectArg
+ operands:
+ argsObject: Object
+ arguments:
+ argno: size_t
+ result_type: Value
+ congruent_to: custom
+ alias_set: custom
+
+- name: SetArgumentsObjectArg
+ operands:
+ argsObject: Object
+ value: Value
+ arguments:
+ argno: size_t
+ alias_set: custom
+
+# Load |arguments[index]| from a mapped or unmapped arguments object. Bails out
+# if any elements were overridden or deleted. Also bails out if the index is
+# out of bounds.
+- name: LoadArgumentsObjectArg
+ operands:
+ argsObject: Object
+ index: Int32
+ result_type: Value
+ guard: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+# Load |arguments[index]| from a mapped or unmapped arguments object. Bails out
+# if any elements were overridden or deleted. Returns undefined if the index is
+# out of bounds.
+- name: LoadArgumentsObjectArgHole
+ operands:
+ argsObject: Object
+ index: Int32
+ result_type: Value
+ guard: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: InArgumentsObjectArg
+ operands:
+ argsObject: Object
+ index: Int32
+ result_type: Boolean
+ guard: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+# Load |arguments.length|. Bails out if the length has been overriden.
+- name: ArgumentsObjectLength
+ operands:
+ argsObject: Object
+ result_type: Int32
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ # Even though the "length" property is lazily resolved, it acts similar to
+ # a normal property load, so we can treat this operation like any other
+ # property read.
+ alias_set: custom
+
+# Create an array from an arguments object.
+- name: ArrayFromArgumentsObject
+ operands:
+ argsObject: Object
+ arguments:
+ shape: Shape*
+ result_type: Object
+ possibly_calls: true
+
+# Guard that the given flags are not set on the arguments object.
+- name: GuardArgumentsObjectFlags
+ operands:
+ argsObject: Object
+ arguments:
+ flags: uint32_t
+ result_type: Object
+ movable: true
+ guard: true
+ congruent_to: custom
+ # The flags are packed with the length in a fixed private slot.
+ alias_set: custom
+
+# Given a MIRType::Value A and a MIRType::Object B:
+# If the Value may be safely unboxed to an Object, return Object(A).
+# Otherwise, return B.
+# Used to implement return behavior for inlined constructors.
+- name: ReturnFromCtor
+ operands:
+ value: Value
+ object: Object
+ result_type: Object
+ folds_to: custom
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: ToDouble
+ gen_boilerplate: false
+
+- name: ToFloat32
+ gen_boilerplate: false
+
+# Converts a uint32 to a double (coming from wasm).
+- name: WasmUnsignedToDouble
+ operands:
+ def: Int32
+ type_policy: none
+ result_type: Double
+ movable: true
+ folds_to: custom
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: WasmUnsignedToFloat32
+ gen_boilerplate: false
+
+- name: WrapInt64ToInt32
+ gen_boilerplate: false
+
+- name: ExtendInt32ToInt64
+ gen_boilerplate: false
+
+- name: WasmBuiltinTruncateToInt64
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt64
+ gen_boilerplate: false
+
+- name: WasmTruncateToInt32
+ gen_boilerplate: false
+
+# Store a JS Value that can't be represented as an AnyRef pointer into an
+# object that holds the value (opaquely) as such a pointer.
+- name: WasmBoxValue
+ operands:
+ def: Value
+ result_type: RefOrNull
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: WasmAnyRefFromJSObject
+ operands:
+ def: Object
+ type_policy: none
+ result_type: RefOrNull
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: Int32ToIntPtr
+ gen_boilerplate: false
+
+- name: NonNegativeIntPtrToInt32
+ gen_boilerplate: false
+
+- name: IntPtrToDouble
+ gen_boilerplate: false
+
+- name: AdjustDataViewLength
+ gen_boilerplate: false
+
+- name: Int64ToFloatingPoint
+ gen_boilerplate: false
+
+- name: BuiltinInt64ToFloatingPoint
+ gen_boilerplate: false
+
+- name: ToNumberInt32
+ gen_boilerplate: false
+
+- name: ToIntegerInt32
+ gen_boilerplate: false
+
+- name: TruncateToInt32
+ gen_boilerplate: false
+
+- name: WasmBuiltinTruncateToInt32
+ gen_boilerplate: false
+
+- name: ToBigInt
+ gen_boilerplate: false
+
+- name: ToInt64
+ gen_boilerplate: false
+
+- name: TruncateBigIntToInt64
+ gen_boilerplate: false
+
+- name: Int64ToBigInt
+ gen_boilerplate: false
+
+- name: ToString
+ gen_boilerplate: false
+
+- name: BitNot
+ gen_boilerplate: false
+
+- name: TypeOf
+ gen_boilerplate: false
+
+- name: TypeOfName
+ operands:
+ input: Int32
+ result_type: String
+ movable: true
+ folds_to: custom
+ congruent_to: if_operands_equal
+ alias_set: none
+ can_recover: true
+
+- name: TypeOfIs
+ gen_boilerplate: false
+
+- name: ToAsyncIter
+ operands:
+ iterator: Object
+ nextMethod: Value
+ result_type: Object
+
+- name: ToPropertyKeyCache
+ operands:
+ input: Value
+ result_type: Value
+
+- name: BitAnd
+ gen_boilerplate: false
+
+- name: BitOr
+ gen_boilerplate: false
+
+- name: BitXor
+ gen_boilerplate: false
+
+- name: Lsh
+ gen_boilerplate: false
+
+- name: Rsh
+ gen_boilerplate: false
+
+- name: Ursh
+ gen_boilerplate: false
+
+- name: SignExtendInt32
+ gen_boilerplate: false
+
+- name: SignExtendInt64
+ gen_boilerplate: false
+
+- name: MinMax
+ gen_boilerplate: false
+
+- name: MinMaxArray
+ gen_boilerplate: false
+
+- name: Abs
+ gen_boilerplate: false
+
+- name: Clz
+ gen_boilerplate: false
+
+- name: Ctz
+ gen_boilerplate: false
+
+- name: Popcnt
+ gen_boilerplate: false
+
+- name: Sqrt
+ gen_boilerplate: false
+
+- name: CopySign
+ gen_boilerplate: false
+
+# Inline implementation of atan2 (arctangent of y/x).
+- name: Atan2
+ operands:
+ y: Double
+ x: Double
+ result_type: Double
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ possibly_calls: true
+ can_recover: true
+ clone: true
+
+- name: Hypot
+ gen_boilerplate: false
+
+- name: Pow
+ gen_boilerplate: false
+
+- name: PowHalf
+ gen_boilerplate: false
+
+- name: Random
+ result_type: Double
+ alias_set: custom
+ possibly_calls: true
+ compute_range: custom
+ can_recover: custom
+ clone: true
+
+- name: Sign
+ gen_boilerplate: false
+
+- name: MathFunction
+ gen_boilerplate: false
+
+- name: Add
+ gen_boilerplate: false
+
+- name: Sub
+ gen_boilerplate: false
+
+- name: Mul
+ gen_boilerplate: false
+
+- name: Div
+ gen_boilerplate: false
+
+- name: WasmBuiltinDivI64
+ gen_boilerplate: false
+
+- name: Mod
+ gen_boilerplate: false
+
+- name: WasmBuiltinModD
+ gen_boilerplate: false
+
+- name: WasmBuiltinModI64
+ gen_boilerplate: false
+
+- name: BigIntAdd
+ gen_boilerplate: false
+
+- name: BigIntSub
+ gen_boilerplate: false
+
+- name: BigIntMul
+ gen_boilerplate: false
+
+- name: BigIntDiv
+ gen_boilerplate: false
+
+- name: BigIntMod
+ gen_boilerplate: false
+
+- name: BigIntPow
+ gen_boilerplate: false
+
+- name: BigIntBitAnd
+ gen_boilerplate: false
+
+- name: BigIntBitOr
+ gen_boilerplate: false
+
+- name: BigIntBitXor
+ gen_boilerplate: false
+
+- name: BigIntLsh
+ gen_boilerplate: false
+
+- name: BigIntRsh
+ gen_boilerplate: false
+
+- name: BigIntIncrement
+ gen_boilerplate: false
+
+- name: BigIntDecrement
+ gen_boilerplate: false
+
+- name: BigIntNegate
+ gen_boilerplate: false
+
+- name: BigIntBitNot
+ gen_boilerplate: false
+
+- name: Int32ToStringWithBase
+ operands:
+ input: Int32
+ base: Int32
+ result_type: String
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: NumberParseInt
+ operands:
+ string: String
+ radix: Int32
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ possibly_calls: true
+
+- name: DoubleParseInt
+ operands:
+ number: Double
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: Concat
+ gen_boilerplate: false
+
+- name: LinearizeForCharAccess
+ operands:
+ string: String
+ index: Int32
+ result_type: String
+ movable: true
+ congruent_to: if_operands_equal
+ # Strings are immutable, so there is no implicit dependency.
+ alias_set: none
+
+- name: CharCodeAt
+ operands:
+ string: String
+ index: Int32
+ result_type: Int32
+ movable: true
+ folds_to: custom
+ congruent_to: if_operands_equal
+ # Strings are immutable, so there is no implicit dependency.
+ alias_set: none
+ compute_range: custom
+ can_recover: true
+ clone: true
+
+# Similar to CharCodeAt, but also supports out-of-bounds access.
+- name: CharCodeAtMaybeOutOfBounds
+ operands:
+ string: String
+ index: Int32
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ # Strings are immutable, so there is no implicit dependency.
+ alias_set: none
+
+# Like CharCodeAtMaybeOutOfBounds, this operation also supports out-of-bounds access.
+- name: CharAtMaybeOutOfBounds
+ operands:
+ string: String
+ index: Int32
+ result_type: String
+ movable: true
+ congruent_to: if_operands_equal
+ # Strings are immutable, so there is no implicit dependency.
+ alias_set: none
+
+- name: FromCharCode
+ operands:
+ code: Int32
+ result_type: String
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ can_recover: true
+ clone: true
+
+- name: FromCodePoint
+ operands:
+ codePoint: Int32
+ result_type: String
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ clone: true
+
+- name: StringIndexOf
+ operands:
+ string: String
+ searchString: String
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ possibly_calls: true
+
+- name: StringStartsWith
+ operands:
+ string: String
+ searchString: String
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ possibly_calls: true
+
+- name: StringEndsWith
+ operands:
+ string: String
+ searchString: String
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ possibly_calls: true
+
+- name: StringConvertCase
+ gen_boilerplate: false
+
+- name: StringSplit
+ operands:
+ string: String
+ separator: String
+ result_type: Object
+ possibly_calls: true
+ # Although this instruction returns a new array, we don't have to mark
+ # it as store instruction, see also MNewArray.
+ alias_set: none
+ can_recover: true
+
+- name: BoxNonStrictThis
+ operands:
+ def: Value
+ arguments:
+ globalThis: JSObject*
+ result_type: Object
+ folds_to: custom
+ possibly_calls: true
+ # This instruction can allocate a new object for wrapped primitives, but
+ # has no effect on existing objects.
+ alias_set: none
+
+- name: ImplicitThis
+ operands:
+ envChain: Object
+ arguments:
+ name: PropertyName*
+ result_type: Value
+ possibly_calls: true
+
+- name: Phi
+ gen_boilerplate: false
+
+- name: Beta
+ gen_boilerplate: false
+
+- name: NaNToZero
+ gen_boilerplate: false
+
+- name: OsrValue
+ gen_boilerplate: false
+
+- name: OsrEnvironmentChain
+ gen_boilerplate: false
+
+- name: OsrArgumentsObject
+ gen_boilerplate: false
+
+- name: OsrReturnValue
+ gen_boilerplate: false
+
+- name: BinaryCache
+ gen_boilerplate: false
+
+- name: UnaryCache
+ operands:
+ input: Value
+ result_type: Value
+
+# Checks whether we need to fire the interrupt handler.
+- name: CheckOverRecursed
+ guard: true
+ alias_set: none
+
+
+# Check whether we need to fire the interrupt handler.
+- name: InterruptCheck
+ guard: true
+ alias_set: none
+
+- name: WasmInterruptCheck
+ gen_boilerplate: false
+
+- name: WasmTrap
+ gen_boilerplate: false
+
+- name: LexicalCheck
+ gen_boilerplate: false
+
+# Unconditionally throw an uninitialized let error.
+- name: ThrowRuntimeLexicalError
+ arguments:
+ errorNumber: unsigned
+ result_type: None
+ guard: true
+ alias_set: custom
+
+- name: ThrowMsg
+ gen_boilerplate: false
+
+# In the prologues of global and eval scripts, check for redeclarations and
+# initialize bindings.
+- name: GlobalDeclInstantiation
+ guard: true
+
+- name: RegExp
+ arguments:
+ source: RegExpObject*
+ hasShared: bool
+ result_type: Object
+ possibly_calls: true
+ alias_set: none
+
+- name: RegExpMatcher
+ operands:
+ regexp: Object
+ string: String
+ lastIndex: Int32
+ result_type: Value
+ possibly_calls: true
+ can_recover: true
+
+- name: RegExpSearcher
+ operands:
+ regexp: Object
+ string: String
+ lastIndex: Int32
+ result_type: Int32
+ possibly_calls: true
+ can_recover: true
+
+- name: RegExpTester
+ operands:
+ regexp: Object
+ string: String
+ lastIndex: Int32
+ result_type: Int32
+ possibly_calls: true
+ can_recover: true
+
+- name: RegExpPrototypeOptimizable
+ operands:
+ object: Object
+ result_type: Boolean
+ alias_set: none
+
+- name: RegExpInstanceOptimizable
+ operands:
+ object: Object
+ proto: Object
+ result_type: Boolean
+ alias_set: none
+
+- name: GetFirstDollarIndex
+ gen_boilerplate: false
+
+- name: StringReplace
+ gen_boilerplate: false
+
+- name: Substr
+ operands:
+ string: String
+ begin: Int32
+ length: Int32
+ result_type: String
+ congruent_to: if_operands_equal
+ alias_set: none
+ can_recover: true
+
+- name: ModuleMetadata
+ arguments:
+ module: JSObject*
+ result_type: Object
+
+- name: DynamicImport
+ operands:
+ specifier: Value
+ options: Value
+ result_type: Object
+
+- name: Lambda
+ gen_boilerplate: false
+
+- name: FunctionWithProto
+ gen_boilerplate: false
+
+- name: SetFunName
+ operands:
+ fun: Object
+ name: Value
+ arguments:
+ prefixKind: uint8_t
+ result_type: None
+ possibly_calls: true
+
+# Returns obj->slots.
+- name: Slots
+ operands:
+ object: Object
+ result_type: Slots
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ might_alias: custom
+ clone: true
+
+# Returns obj->elements.
+- name: Elements
+ operands:
+ object: Object
+ result_type: Elements
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ clone: true
+
+# Load the initialized length from an elements header.
+- name: InitializedLength
+ operands:
+ elements: Elements
+ type_policy: none
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ compute_range: custom
+ clone: true
+
+- name: SetInitializedLength
+ operands:
+ elements: Elements
+ index: Int32
+ type_policy: none
+ alias_set: custom
+ clone: true
+
+# Load the array length from an elements header.
+- name: ArrayLength
+ operands:
+ elements: Elements
+ type_policy: none
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ compute_range: custom
+ clone: true
+
+# Store to the length in an elements header. Note the input is an *index*, one
+# less than the desired length.
+- name: SetArrayLength
+ operands:
+ elements: Elements
+ index: Int32
+ type_policy: none
+ alias_set: custom
+ # By default no, unless built as a recovered instruction.
+ can_recover: custom
+
+# Load the function length. Bails for functions with lazy scripts or a
+# resolved "length" property.
+- name: FunctionLength
+ operands:
+ function: Object
+ result_type: Int32
+ guard: true
+ congruent_to: if_operands_equal
+ # Even though the "length" property is lazily resolved, it acts similar to
+ # a normal property load, so we can treat this operation like any other
+ # property read.
+ alias_set: custom
+
+# Load the function name. Bails for bound functions when the bound function
+# name prefix isn't present or functions with a resolved "name" property.
+- name: FunctionName
+ operands:
+ function: Object
+ result_type: String
+ guard: true
+ congruent_to: if_operands_equal
+ # Even though the "name" property is lazily resolved, it acts similar to
+ # a normal property load, so we can treat this operation like any other
+ # property read.
+ alias_set: custom
+
+- name: GetNextEntryForIterator
+ gen_boilerplate: false
+
+# Read the byte length of an array buffer as IntPtr.
+- name: ArrayBufferByteLength
+ operands:
+ object: Object
+ result_type: IntPtr
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+# Read the length of an array buffer view.
+- name: ArrayBufferViewLength
+ operands:
+ object: Object
+ result_type: IntPtr
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ compute_range: custom
+
+- name: ArrayBufferViewByteOffset
+ operands:
+ object: Object
+ result_type: IntPtr
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ compute_range: custom
+
+# Read the length of an array buffer view.
+- name: ArrayBufferViewElements
+ operands:
+ object: Object
+ result_type: Elements
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ clone: true
+
+# Return the element size of a typed array.
+- name: TypedArrayElementSize
+ operands:
+ object: Object
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ # Class is immutable. See also MHasClass.
+ alias_set: none
+ compute_range: custom
+
+# Guard an ArrayBufferView has an attached ArrayBuffer.
+- name: GuardHasAttachedArrayBuffer
+ operands:
+ object: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: GuardNumberToIntPtrIndex
+ gen_boilerplate: false
+
+- name: KeepAliveObject
+ operands:
+ object: Object
+ result_type: None
+ guard: true
+
+- name: Not
+ gen_boilerplate: false
+
+- name: BoundsCheck
+ gen_boilerplate: false
+
+- name: BoundsCheckLower
+ gen_boilerplate: false
+
+- name: SpectreMaskIndex
+ gen_boilerplate: false
+
+- name: LoadElement
+ gen_boilerplate: false
+
+- name: LoadElementAndUnbox
+ gen_boilerplate: false
+
+- name: LoadElementHole
+ gen_boilerplate: false
+
+- name: StoreElement
+ gen_boilerplate: false
+
+- name: StoreHoleValueElement
+ gen_boilerplate: false
+
+- name: StoreElementHole
+ gen_boilerplate: false
+
+- name: ArrayPopShift
+ gen_boilerplate: false
+
+# Array.prototype.push on a dense array. Returns the new array length.
+- name: ArrayPush
+ operands:
+ object: Object
+ value: Value
+ result_type: Int32
+ alias_set: custom
+ compute_range: custom
+ clone: true
+
+# Array.prototype.slice on a dense array.
+- name: ArraySlice
+ operands:
+ object: Object
+ begin: Int32
+ end: Int32
+ arguments:
+ templateObj: JSObject*
+ initialHeap: gc::InitialHeap
+ result_type: Object
+ possibly_calls: true
+
+# Array.prototype.slice on an arguments object.
+- name: ArgumentsSlice
+ operands:
+ object: Object
+ begin: Int32
+ end: Int32
+ arguments:
+ templateObj: JSObject*
+ initialHeap: gc::InitialHeap
+ result_type: Object
+ possibly_calls: true
+
+# Array.prototype.slice on an arguments object.
+- name: FrameArgumentsSlice
+ operands:
+ begin: Int32
+ count: Int32
+ arguments:
+ templateObj: JSObject*
+ initialHeap: gc::InitialHeap
+ result_type: Object
+ alias_set: none
+ possibly_calls: true
+
+# Array.prototype.slice on an inlined arguments object.
+- name: InlineArgumentsSlice
+ gen_boilerplate: false
+
+- name: NormalizeSliceTerm
+ operands:
+ value: Int32
+ length: Int32
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ folds_to: custom
+
+# MArrayJoin doesn't override |getAliasSet()|, because Array.prototype.join
+# might coerce the elements of the Array to strings. This coercion might
+# cause the evaluation of JavaScript code.
+- name: ArrayJoin
+ operands:
+ array: Object
+ sep: String
+ result_type: String
+ possibly_calls: true
+ folds_to: custom
+ # MArrayJoin doesn't override |getAliasSet()|, because Array.prototype.join
+ # might coerce the elements of the Array to strings. This coercion might
+ # cause the evaluation of JavaScript code.
+
+- name: LoadUnboxedScalar
+ gen_boilerplate: false
+
+- name: LoadDataViewElement
+ gen_boilerplate: false
+
+- name: LoadTypedArrayElementHole
+ gen_boilerplate: false
+
+- name: StoreUnboxedScalar
+ gen_boilerplate: false
+
+- name: StoreDataViewElement
+ gen_boilerplate: false
+
+- name: StoreTypedArrayElementHole
+ gen_boilerplate: false
+
+- name: EffectiveAddress
+ gen_boilerplate: false
+
+- name: ClampToUint8
+ gen_boilerplate: false
+
+- name: LoadFixedSlot
+ gen_boilerplate: false
+
+- name: LoadFixedSlotAndUnbox
+ gen_boilerplate: false
+
+- name: LoadDynamicSlotAndUnbox
+ gen_boilerplate: false
+
+- name: StoreFixedSlot
+ gen_boilerplate: false
+
+- name: GetPropertyCache
+ gen_boilerplate: false
+
+- name: HomeObjectSuperBase
+ operands:
+ homeObject: Object
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: GetPropSuperCache
+ gen_boilerplate: false
+
+- name: BindNameCache
+ operands:
+ envChain: Object
+ result_type: Object
+
+- name: CallBindVar
+ operands:
+ environmentChain: Object
+ result_type: Object
+ movable: true
+ congruent_to: custom
+ alias_set: none
+
+- name: GuardShape
+ operands:
+ object: Object
+ arguments:
+ shape: Shape*
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: custom
+ alias_set: custom
+ might_alias: custom
+
+- name: GuardMultipleShapes
+ operands:
+ object: Object
+ shapeList: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: GuardProto
+ gen_boilerplate: false
+
+- name: GuardNullProto
+ gen_boilerplate: false
+
+# Guard the object is a native object.
+- name: GuardIsNativeObject
+ operands:
+ object: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: GuardIsProxy
+ operands:
+ object: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: GuardIsNotDOMProxy
+ operands:
+ proxy: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: GuardIsNotProxy
+ operands:
+ object: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: ProxyGet
+ operands:
+ proxy: Object
+ arguments:
+ id: jsid
+ result_type: Value
+ possibly_calls: true
+
+- name: ProxyGetByValue
+ operands:
+ proxy: Object
+ idVal: Value
+ result_type: Value
+ possibly_calls: true
+
+- name: ProxyHasProp
+ operands:
+ proxy: Object
+ idVal: Value
+ arguments:
+ hasOwn: bool
+ result_type: Boolean
+ possibly_calls: true
+
+- name: ProxySet
+ operands:
+ proxy: Object
+ rhs: Value
+ arguments:
+ id: jsid
+ strict: bool
+ possibly_calls: true
+
+- name: ProxySetByValue
+ operands:
+ proxy: Object
+ idVal: Value
+ rhs: Value
+ arguments:
+ strict: bool
+ possibly_calls: true
+
+- name: CallSetArrayLength
+ operands:
+ obj: Object
+ rhs: Value
+ arguments:
+ strict: bool
+ possibly_calls: true
+
+- name: MegamorphicLoadSlot
+ operands:
+ object: Object
+ arguments:
+ name: PropertyKey
+ result_type: Value
+ # Bails when non-native or accessor properties are encountered, so we can't
+ # DCE this instruction.
+ guard: true
+ possibly_calls: true
+ congruent_to: custom
+ alias_set: custom
+
+- name: MegamorphicLoadSlotByValue
+ operands:
+ object: Object
+ idVal: Value
+ result_type: Value
+ # Bails when non-native or accessor properties are encountered, so we can't
+ # DCE this instruction.
+ guard: true
+ folds_to: custom
+ congruent_to: if_operands_equal
+ alias_set: custom
+ possibly_calls: true
+
+- name: MegamorphicStoreSlot
+ operands:
+ object: Object
+ rhs: Value
+ arguments:
+ name: PropertyName*
+ congruent_to: custom
+ alias_set: custom
+ possibly_calls: true
+
+- name: MegamorphicHasProp
+ operands:
+ object: Object
+ idVal: Value
+ arguments:
+ hasOwn: bool
+ result_type: Boolean
+ # Bails when non-native or accessor properties are encountered, so we can't
+ # DCE this instruction.
+ guard: true
+ congruent_to: custom
+ alias_set: custom
+ possibly_calls: true
+
+- name: GuardIsNotArrayBufferMaybeShared
+ operands:
+ object: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: GuardIsTypedArray
+ operands:
+ object: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+# Loads a specific JSObject* that was originally nursery-allocated.
+# See also WarpObjectField.
+- name: NurseryObject
+ arguments:
+ # Index in the Vector of objects stored in the WarpSnapshot.
+ nurseryIndex: uint32_t
+ result_type: Object
+ movable: true
+ congruent_to: custom
+ alias_set: none
+
+- name: GuardValue
+ gen_boilerplate: false
+
+- name: GuardNullOrUndefined
+ operands:
+ value: Value
+ result_type: Value
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: GuardIsNotObject
+ operands:
+ value: Value
+ result_type: Value
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: GuardFunctionFlags
+ gen_boilerplate: false
+
+- name: GuardFunctionIsNonBuiltinCtor
+ operands:
+ function: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: GuardFunctionKind
+ operands:
+ function: Object
+ arguments:
+ expected: FunctionFlags::FunctionKind
+ bailOnEquality: bool
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: custom
+ alias_set: custom
+
+- name: GuardFunctionScript
+ operands:
+ function: Object
+ arguments:
+ expected: BaseScript*
+ nargs: uint16_t
+ flags: FunctionFlags
+ result_type: Object
+ guard: true
+ movable: true
+ folds_to: custom
+ congruent_to: custom
+ # A JSFunction's BaseScript pointer is immutable. Relazification of
+ # self-hosted functions is an exception to this, but we don't use this
+ # guard for self-hosted functions.
+ alias_set: custom
+
+- name: GuardObjectIdentity
+ gen_boilerplate: false
+
+- name: GuardSpecificFunction
+ gen_boilerplate: false
+
+- name: GuardSpecificAtom
+ operands:
+ str: String
+ arguments:
+ atom: JSAtom*
+ result_type: String
+ guard: true
+ movable: true
+ congruent_to: custom
+ folds_to: custom
+ alias_set: none
+
+- name: GuardSpecificSymbol
+ gen_boilerplate: false
+
+- name: GuardSpecificInt32
+ operands:
+ num: Int32
+ arguments:
+ expected: int32_t
+ result_type: Int32
+ guard: true
+ movable: true
+ folds_to: custom
+ alias_set: none
+
+- name: GuardStringToIndex
+ operands:
+ string: String
+ result_type: Int32
+ # Mark as guard because this instruction must not be eliminated. For
+ # example, if the string is not an index the operation could change from a
+ # typed array load to a getter call.
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: GuardStringToInt32
+ operands:
+ string: String
+ result_type: Int32
+ # Mark as guard to prevent the issue described in MGuardStringToIndex's
+ # constructor.
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: GuardStringToDouble
+ operands:
+ string: String
+ result_type: Double
+ # Mark as guard to prevent the issue described in MGuardStringToIndex's
+ # constructor.
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: GuardNoDenseElements
+ operands:
+ object: Object
+ result_type: Object
+ guard: true
+ movable: true
+ alias_set: custom
+
+- name: GuardTagNotEqual
+ gen_boilerplate: false
+
+- name: LoadDynamicSlot
+ gen_boilerplate: false
+
+# Inline call to access a function's environment (scope chain).
+- name: FunctionEnvironment
+ operands:
+ function: Object
+ result_type: Object
+ movable: true
+ folds_to: custom
+ # A function's environment is fixed.
+ alias_set: none
+
+# Allocate a new BlockLexicalEnvironmentObject.
+- name: NewLexicalEnvironmentObject
+ operands:
+ templateObj: Object
+ result_type: Object
+ alias_set: none
+
+# Allocate a new ClassBodyEnvironmentObject.
+- name: NewClassBodyEnvironmentObject
+ operands:
+ templateObj: Object
+ result_type: Object
+ alias_set: none
+
+- name: NewVarEnvironmentObject
+ operands:
+ templateObj: Object
+ result_type: Object
+ alias_set: none
+
+- name: HomeObject
+ operands:
+ function: Object
+ result_type: Object
+ movable: true
+ # A function's [[HomeObject]] is fixed.
+ alias_set: none
+
+- name: AddAndStoreSlot
+ gen_boilerplate: false
+
+- name: AllocateAndStoreSlot
+ operands:
+ object: Object
+ value: Value
+ arguments:
+ slotOffset: uint32_t
+ shape: Shape*
+ numNewSlots: uint32_t
+ possibly_calls: true
+ alias_set: custom
+
+- name: AddSlotAndCallAddPropHook
+ operands:
+ object: Object
+ value: Value
+ arguments:
+ shape: Shape*
+ possibly_calls: true
+
+- name: StoreDynamicSlot
+ gen_boilerplate: false
+
+- name: GetNameCache
+ operands:
+ envObj: Object
+ result_type: Value
+
+- name: CallGetIntrinsicValue
+ arguments:
+ name: PropertyName*
+ result_type: Value
+ possibly_calls: true
+
+- name: DeleteProperty
+ operands:
+ value: Value
+ arguments:
+ name: PropertyName*
+ strict: bool
+ result_type: Boolean
+
+- name: DeleteElement
+ operands:
+ value: Value
+ index: Value
+ arguments:
+ strict: bool
+ result_type: Boolean
+
+- name: SetPropertyCache
+ gen_boilerplate: false
+
+- name: MegamorphicSetElement
+ gen_boilerplate: false
+
+- name: SetDOMProperty
+ gen_boilerplate: false
+
+- name: GetDOMProperty
+ gen_boilerplate: false
+
+- name: GetDOMMember
+ gen_boilerplate: false
+
+- name: ObjectToIterator
+ operands:
+ object: Object
+ arguments:
+ enumeratorsAddr: NativeIteratorListHead*
+ result_type:
+ Object
+
+- name: ValueToIterator
+ operands:
+ value: Value
+ result_type: Object
+
+# Load the private value expando from a DOM proxy. The target is stored in the
+# proxy object's private slot.
+# This is either an UndefinedValue (no expando), ObjectValue (the expando
+# object), or PrivateValue(ExpandoAndGeneration*).
+- name: LoadDOMExpandoValue
+ operands:
+ proxy: Object
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: LoadDOMExpandoValueGuardGeneration
+ gen_boilerplate: false
+
+- name: LoadDOMExpandoValueIgnoreGeneration
+ operands:
+ proxy: Object
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+# Takes an expando Value as input, then guards it's either UndefinedValue or
+# an object with the expected shape.
+- name: GuardDOMExpandoMissingOrGuardShape
+ operands:
+ expando: Value
+ arguments:
+ shape: Shape*
+ result_type: Value
+ guard: true
+ movable: true
+ congruent_to: custom
+ alias_set: custom
+
+- name: StringLength
+ operands:
+ string: String
+ result_type: Int32
+ movable: true
+ folds_to: custom
+ congruent_to: if_operands_equal
+ # The string |length| property is immutable, so there is no
+ # implicit dependency.
+ alias_set: none
+ compute_range: custom
+ can_recover: true
+ clone: true
+
+- name: Floor
+ gen_boilerplate: false
+
+- name: Ceil
+ gen_boilerplate: false
+
+- name: Round
+ gen_boilerplate: false
+
+- name: Trunc
+ gen_boilerplate: false
+
+- name: NearbyInt
+ gen_boilerplate: false
+
+- name: GetIteratorCache
+ gen_boilerplate: false
+
+- name: OptimizeSpreadCallCache
+ operands:
+ value: Value
+ result_type: Value
+
+- name: IteratorMore
+ operands:
+ iterator: Object
+ result_type: Value
+
+- name: IsNoIter
+ operands:
+ def: Object
+ result_type: Boolean
+ type_policy: none
+ movable : true
+ alias_set: none
+
+- name: IteratorEnd
+ operands:
+ iterator: Object
+
+- name: CloseIterCache
+ operands:
+ iter: Object
+ arguments:
+ completionKind: uint8_t
+ possibly_calls: true
+
+- name: InCache
+ gen_boilerplate: false
+
+- name: InArray
+ gen_boilerplate: false
+
+- name: GuardElementNotHole
+ gen_boilerplate: false
+
+- name: NewPrivateName
+ arguments:
+ name: JSAtom*
+ result_type: Symbol
+ possibly_calls: true
+
+- name: CheckPrivateFieldCache
+ gen_boilerplate: false
+
+- name: HasOwnCache
+ gen_boilerplate: false
+
+- name: InstanceOf
+ gen_boilerplate: false
+
+# Implementation for instanceof operator with unknown rhs.
+- name: InstanceOfCache
+ operands:
+ obj: Value
+ proto: Object
+ result_type: Boolean
+
+- name: ArgumentsLength
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ # Arguments |length| cannot be mutated by Ion Code.
+ alias_set: none
+ compute_range: custom
+ can_recover: true
+
+# This MIR instruction is used to get an argument from the actual arguments.
+- name: GetFrameArgument
+ operands:
+ index: Int32
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ # This instruction is never aliased, because ops like JSOp::SetArg don't
+ # write to the argument frames. We create an arguments object in that case.
+ alias_set: none
+
+# This MIR instruction is used to get an argument from the actual arguments.
+# Returns undefined if |index| is larger-or-equals to |length|. Bails out if
+# |index| is negative.
+- name: GetFrameArgumentHole
+ operands:
+ index: Int32
+ length: Int32
+ result_type: Value
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ # This instruction is never aliased, because ops like JSOp::SetArg don't
+ # write to the argument frames. We create an arguments object in that case.
+ alias_set: none
+
+- name: NewTarget
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: Rest
+ operands:
+ numActuals: Int32
+ arguments:
+ numFormals: unsigned
+ shape: Shape*
+ result_type: Object
+ possibly_calls: true
+ alias_set: none
+ can_recover: true
+
+- name: PostWriteBarrier
+ gen_boilerplate: false
+
+- name: PostWriteElementBarrier
+ gen_boilerplate: false
+
+- name: AssertCanElidePostWriteBarrier
+ operands:
+ object: Object
+ value: Value
+ result_type: None
+ guard: true
+ alias_set: none
+
+- name: NewNamedLambdaObject
+ arguments:
+ templateObj: NamedLambdaObject*
+ result_type: Object
+ alias_set: none
+
+- name: NewCallObject
+ gen_boilerplate: false
+
+- name: NewStringObject
+ gen_boilerplate: false
+
+- name: IsCallable
+ gen_boilerplate: false
+
+- name: IsConstructor
+ operands:
+ object: Object
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: IsCrossRealmArrayConstructor
+ operands:
+ object: Object
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: IsObject
+ operands:
+ object: Value
+ result_type: Boolean
+ movable: true
+ folds_to: custom
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: IsNullOrUndefined
+ operands:
+ value: Value
+ result_type: Boolean
+ movable: true
+ folds_to: custom
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: HasClass
+ gen_boilerplate: false
+
+- name: GuardToClass
+ gen_boilerplate: false
+
+- name: GuardToFunction
+ gen_boilerplate: false
+
+- name: IsArray
+ gen_boilerplate: false
+
+- name: IsTypedArray
+ gen_boilerplate: false
+
+- name: ObjectClassToString
+ operands:
+ object: Object
+ result_type: String
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ possibly_calls: true
+ # Tests @@toStringTag is neither present on this object nor on any object
+ # of the prototype chain.
+ alias_set: custom
+
+- name: CheckReturn
+ operands:
+ returnValue: Value
+ thisValue: Value
+ result_type: Value
+ guard: true
+ folds_to: custom
+ alias_set: custom
+
+- name: CheckThis
+ operands:
+ thisValue: Value
+ result_type: Value
+ guard: true
+ folds_to: custom
+ alias_set: custom
+
+- name: AsyncResolve
+ operands:
+ generator: Object
+ valueOrReason: Value
+ arguments:
+ resolveKind: AsyncFunctionResolveKind
+ result_type: Object
+
+# Returns from this function to the previous caller; this looks like a regular
+# Unary instruction and is used to lie to the MIR generator about suspending
+# ops like Yield/Await, which are emitted like returns, but MIR-Build like
+# regular instructions.
+- name: GeneratorReturn
+ operands:
+ input: Value
+ guard: true
+ alias_set: none
+
+- name: AsyncAwait
+ operands:
+ value: Value
+ generator: Object
+ result_type: Object
+
+- name: CheckThisReinit
+ operands:
+ thisValue: Value
+ result_type: Value
+ guard: true
+ folds_to: custom
+ alias_set: custom
+
+- name: Generator
+ gen_boilerplate: false
+
+- name: CanSkipAwait
+ operands:
+ value: Value
+ result_type: Boolean
+
+- name: MaybeExtractAwaitValue
+ gen_boilerplate: false
+
+- name: IncrementWarmUpCounter
+ arguments:
+ script: JSScript*
+ alias_set: none
+
+- name: AtomicIsLockFree
+ gen_boilerplate: false
+
+- name: CompareExchangeTypedArrayElement
+ gen_boilerplate: false
+
+- name: AtomicExchangeTypedArrayElement
+ gen_boilerplate: false
+
+- name: AtomicTypedArrayElementBinop
+ gen_boilerplate: false
+
+- name: Debugger
+ gen_boilerplate: false
+
+- name: CheckIsObj
+ operands:
+ value: Value
+ arguments:
+ checkKind: uint8_t
+ result_type: Object
+ guard: true
+ folds_to: custom
+ alias_set: none
+
+- name: CheckObjCoercible
+ operands:
+ checkValue: Value
+ result_type: Value
+ guard: true
+ folds_to: custom
+ # Throws on null or undefined.
+ alias_set: custom
+
+- name: CheckClassHeritage
+ operands:
+ heritage: Value
+ result_type: Value
+ guard: true
+
+- name: DebugCheckSelfHosted
+ operands:
+ checkValue: Value
+ result_type: Value
+ guard: true
+
+- name: FinishBoundFunctionInit
+ operands:
+ bound: Object
+ target: Object
+ argCount: Int32
+
+- name: IsPackedArray
+ operands:
+ object: Object
+ result_type: Boolean
+ movable: true
+ alias_set: custom
+
+- name: GuardArrayIsPacked
+ operands:
+ array: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: GetPrototypeOf
+ operands:
+ target: Object
+ result_type: Value
+ # May throw if target is a proxy.
+ guard: true
+
+- name: ObjectWithProto
+ operands:
+ prototype: Value
+ result_type: Object
+ # May throw if prototype is neither an object nor null.
+ guard: true
+ possibly_calls: true
+
+- name: ObjectStaticProto
+ gen_boilerplate: false
+
+# This is basically just a limited case of Constant, for objects which are
+# the prototype of another object and will be used for a GuardShape. It
+# includes a reference to the receiver object so we can eliminate redundant
+# shape guards.
+- name: ConstantProto
+ gen_boilerplate: false
+
+- name: BuiltinObject
+ arguments:
+ builtinObjectKind: BuiltinObjectKind
+ result_type: Object
+ possibly_calls: true
+
+- name: SuperFunction
+ operands:
+ callee: Object
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: InitHomeObject
+ operands:
+ function: Object
+ homeObject: Value
+ result_type: Object
+ alias_set: custom
+
+# Return true if the object is definitely a TypedArray constructor, but not
+# necessarily from the currently active realm. Return false if the object is
+# not a TypedArray constructor or if it's a wrapper.
+- name: IsTypedArrayConstructor
+ operands:
+ object: Object
+ result_type: Boolean
+ alias_set: none
+
+# Load the JSValueTag on all platforms except ARM64. See the comments in
+# MacroAssembler-arm64.h for the |cmpTag(Register, ImmTag)| method for why
+# ARM64 doesn't use the raw JSValueTag, but instead a modified tag value. That
+# modified tag value can't be directly compared against JSValueTag constants.
+- name: LoadValueTag
+ operands:
+ value: Value
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+# Load the target object from a proxy wrapper. The target is stored in the
+# proxy object's private slot.
+- name: LoadWrapperTarget
+ operands:
+ object: Object
+ result_type: Object
+ movable: true
+ congruent_to: if_operands_equal
+ # Can't use |AliasSet::None| because the target changes on navigation.
+ # TODO: Investigate using a narrower or a custom alias set.
+ alias_set: custom
+
+# Guard the accessor shape is present on the object or its prototype chain.
+- name: GuardHasGetterSetter
+ operands:
+ object: Object
+ arguments:
+ propId: jsid
+ getterSetter: GetterSetter*
+ result_type: Object
+ guard: true
+ movable: true
+ possibly_calls: true
+ congruent_to: custom
+ alias_set: custom
+
+- name: GuardIsExtensible
+ operands:
+ object: Object
+ result_type: Object
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: GuardInt32IsNonNegative
+ operands:
+ index: Int32
+ result_type: Int32
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: GuardInt32Range
+ operands:
+ input: Int32
+ arguments:
+ minimum: int32_t
+ maximum: int32_t
+ result_type: Int32
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+# Guard the input index is either greater than the dense initialized length of
+# an object, or a hole element.
+- name: GuardIndexIsNotDenseElement
+ operands:
+ object: Object
+ index: Int32
+ result_type: Int32
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+# Guard an array object's length can be updated successfully when adding an
+# element at the input index.
+- name: GuardIndexIsValidUpdateOrAdd
+ operands:
+ object: Object
+ index: Int32
+ result_type: Int32
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+# Add or update a sparse element of an ArrayObject or PlainObject. It's allowed
+# for the sparse element to be already present on the object. It may also be an
+# accessor property, so this instruction is always marked as effectful.
+- name: CallAddOrUpdateSparseElement
+ operands:
+ object: Object
+ index: Int32
+ value: Value
+ arguments:
+ strict: bool
+ possibly_calls: true
+
+# Get a sparse element from an ArrayObject or PlainObject, possibly by calling
+# an accessor property.
+- name: CallGetSparseElement
+ operands:
+ object: Object
+ index: Int32
+ result_type: Value
+ possibly_calls: true
+
+- name: CallNativeGetElement
+ operands:
+ object: Object
+ index: Int32
+ result_type: Value
+ possibly_calls: true
+
+- name: CallNativeGetElementSuper
+ operands:
+ object: Object
+ index: Int32
+ receiver: Value
+ result_type: Value
+ possibly_calls: true
+
+# Test if a native object has an own element (sparse or dense) at an index.
+- name: CallObjectHasSparseElement
+ operands:
+ object: Object
+ index: Int32
+ result_type: Boolean
+ guard: true
+ congruent_to: if_operands_equal
+ possibly_calls: true
+ alias_set: custom
+
+- name: BigIntAsIntN
+ operands:
+ bits: Int32
+ input: BigInt
+ result_type: BigInt
+ movable: true
+ congruent_to: if_operands_equal
+ possibly_calls: true
+ alias_set: none
+ can_recover: true
+ clone: true
+
+- name: BigIntAsUintN
+ operands:
+ bits: Int32
+ input: BigInt
+ result_type: BigInt
+ movable: true
+ congruent_to: if_operands_equal
+ possibly_calls: true
+ alias_set: none
+ can_recover: true
+ clone: true
+
+- name: GuardNonGCThing
+ operands:
+ input: Value
+ result_type: Value
+ guard: true
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ alias_set: none
+
+- name: ToHashableNonGCThing
+ operands:
+ input: Value
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: ToHashableString
+ operands:
+ input: String
+ result_type: String
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ possibly_calls: true
+
+- name: ToHashableValue
+ operands:
+ input: Value
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+ possibly_calls: true
+
+- name: HashNonGCThing
+ operands:
+ input: Value
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: HashString
+ operands:
+ input: String
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: HashSymbol
+ operands:
+ input: Symbol
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: HashBigInt
+ operands:
+ input: BigInt
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: none
+
+- name: HashObject
+ operands:
+ set: Object
+ input: Value
+ result_type: Int32
+ # In contrast to the previous hash operations, we can't move this
+ # instruction, because the hashcode is computed from the object's address,
+ # which can change when the object is moved by the GC.
+ movable: false
+ alias_set: none
+
+- name: HashValue
+ operands:
+ set: Object
+ input: Value
+ result_type: Int32
+ movable: false
+ alias_set: none
+
+- name: SetObjectHasNonBigInt
+ operands:
+ set: Object
+ value: Value
+ hash: Int32
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: SetObjectHasBigInt
+ operands:
+ set: Object
+ value: Value
+ hash: Int32
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: SetObjectHasValue
+ operands:
+ set: Object
+ value: Value
+ hash: Int32
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: SetObjectHasValueVMCall
+ operands:
+ set: Object
+ value: Value
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ possibly_calls: true
+
+- name: SetObjectSize
+ operands:
+ set: Object
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: MapObjectHasNonBigInt
+ operands:
+ map: Object
+ value: Value
+ hash: Int32
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: MapObjectHasBigInt
+ operands:
+ map: Object
+ value: Value
+ hash: Int32
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: MapObjectHasValue
+ operands:
+ map: Object
+ value: Value
+ hash: Int32
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: MapObjectHasValueVMCall
+ operands:
+ map: Object
+ value: Value
+ result_type: Boolean
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ possibly_calls: true
+
+- name: MapObjectGetNonBigInt
+ operands:
+ map: Object
+ value: Value
+ hash: Int32
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: MapObjectGetBigInt
+ operands:
+ map: Object
+ value: Value
+ hash: Int32
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: MapObjectGetValue
+ operands:
+ map: Object
+ value: Value
+ hash: Int32
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: MapObjectGetValueVMCall
+ operands:
+ map: Object
+ value: Value
+ result_type: Value
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+ possibly_calls: true
+
+- name: MapObjectSize
+ operands:
+ map: Object
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ alias_set: custom
+
+- name: WasmNeg
+ gen_boilerplate: false
+
+- name: WasmBinaryBitwise
+ gen_boilerplate: false
+
+- name: WasmLoadInstance
+ gen_boilerplate: false
+
+- name: WasmStoreInstance
+ gen_boilerplate: false
+
+- name: WasmHeapBase
+ gen_boilerplate: false
+
+- name: WasmBoundsCheck
+ gen_boilerplate: false
+
+- name: WasmExtendU32Index
+ operands:
+ input: Int32
+ result_type: Int64
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ type_policy: none
+ alias_set: none
+
+- name: WasmWrapU32Index
+ operands:
+ input: Int64
+ result_type: Int32
+ movable: true
+ congruent_to: if_operands_equal
+ folds_to: custom
+ type_policy: none
+ alias_set: none
+
+- name: WasmAddOffset
+ gen_boilerplate: false
+
+- name: WasmAlignmentCheck
+ gen_boilerplate: false
+
+- name: WasmLoad
+ gen_boilerplate: false
+
+- name: WasmStore
+ gen_boilerplate: false
+
+- name: AsmJSLoadHeap
+ gen_boilerplate: false
+
+- name: AsmJSStoreHeap
+ gen_boilerplate: false
+
+- name: WasmFence
+ guard: true
+ alias_set: none
+ clone: true
+
+- name: WasmCompareExchangeHeap
+ gen_boilerplate: false
+
+- name: WasmAtomicExchangeHeap
+ gen_boilerplate: false
+
+- name: WasmAtomicBinopHeap
+ gen_boilerplate: false
+
+- name: WasmLoadGlobalVar
+ gen_boilerplate: false
+
+- name: WasmLoadGlobalCell
+ gen_boilerplate: false
+
+- name: WasmLoadTableElement
+ gen_boilerplate: false
+
+- name: WasmStoreGlobalVar
+ gen_boilerplate: false
+
+- name: WasmStoreGlobalCell
+ gen_boilerplate: false
+
+- name: WasmStoreStackResult
+ gen_boilerplate: false
+
+- name: WasmDerivedPointer
+ gen_boilerplate: false
+
+- name: WasmDerivedIndexPointer
+ gen_boilerplate: false
+
+- name: WasmStoreRef
+ gen_boilerplate: false
+
+- name: WasmParameter
+ gen_boilerplate: false
+
+- name: WasmReturn
+ gen_boilerplate: false
+
+- name: WasmReturnVoid
+ gen_boilerplate: false
+
+- name: WasmStackArg
+ gen_boilerplate: false
+
+- name: WasmRegisterResult
+ gen_boilerplate: false
+
+- name: WasmFloatRegisterResult
+ gen_boilerplate: false
+
+- name: WasmRegister64Result
+ gen_boilerplate: false
+
+- name: WasmStackResultArea
+ gen_boilerplate: false
+
+- name: WasmStackResult
+ gen_boilerplate: false
+
+- name: WasmCallCatchable
+ gen_boilerplate: false
+
+- name: WasmCallUncatchable
+ gen_boilerplate: false
+
+- name: WasmCallLandingPrePad
+ gen_boilerplate: false
+
+- name: WasmSelect
+ gen_boilerplate: false
+
+- name: WasmReinterpret
+ gen_boilerplate: false
+
+- name: Rotate
+ gen_boilerplate: false
+
+- name: WasmBinarySimd128
+ gen_boilerplate: false
+
+- name: WasmBinarySimd128WithConstant
+ gen_boilerplate: false
+
+# (v128, i32) -> v128 effect-free shift operations.
+- name: WasmShiftSimd128
+ operands:
+ lhs: Simd128
+ rhs: Int32
+ arguments:
+ simdOp: wasm::SimdOp
+ type_policy: none
+ result_type: Simd128
+ movable: true
+ congruent_to: custom
+ alias_set: none
+ clone: true
+
+# (v128, v128, mask) -> v128 effect-free operation.
+- name: WasmShuffleSimd128
+ operands:
+ lhs: Simd128
+ rhs: Simd128
+ arguments:
+ shuffle: SimdShuffle
+ type_policy: none
+ result_type: Simd128
+ movable: true
+ congruent_to: custom
+ alias_set: none
+ clone: true
+
+- name: WasmReplaceLaneSimd128
+ gen_boilerplate: false
+
+- name: WasmUnarySimd128
+ operands:
+ src: Simd128
+ arguments:
+ simdOp: wasm::SimdOp
+ type_policy: none
+ result_type: Simd128
+ movable: true
+ congruent_to: custom
+ alias_set: none
+ clone: true
+
+- name: WasmTernarySimd128
+ gen_boilerplate: false
+
+- name: WasmScalarToSimd128
+ gen_boilerplate: false
+
+- name: WasmReduceSimd128
+ gen_boilerplate: false
+
+- name: WasmLoadLaneSimd128
+ gen_boilerplate: false
+
+- name: WasmStoreLaneSimd128
+ gen_boilerplate: false
+
+- name: UnreachableResult
+ gen_boilerplate: false
+
+- name: IonToWasmCall
+ gen_boilerplate: false
+
+- name: WasmLoadField
+ gen_boilerplate: false
+
+- name: WasmLoadFieldKA
+ gen_boilerplate: false
+
+- name: WasmStoreFieldKA
+ gen_boilerplate: false
+
+- name: WasmStoreFieldRefKA
+ gen_boilerplate: false
+
+- name: WasmGcObjectIsSubtypeOf
+ gen_boilerplate: false
+
+#ifdef FUZZING_JS_FUZZILLI
+- name: FuzzilliHash
+ gen_boilerplate: false
+
+- name: FuzzilliHashStore
+ gen_boilerplate: false
+#endif
diff --git a/js/src/jit/MachineState.h b/js/src/jit/MachineState.h
new file mode 100644
index 0000000000..63d04ae6ea
--- /dev/null
+++ b/js/src/jit/MachineState.h
@@ -0,0 +1,110 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MachineState_h
+#define jit_MachineState_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Variant.h"
+
+#include <stdint.h>
+
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+
+namespace js::jit {
+
+// Information needed to recover machine register state. This supports two
+// different modes:
+//
+// * Bailouts: all registers are pushed on the stack as part of the bailout
+// process, so MachineState simply points to these FPU/GPR arrays.
+// See RegisterDump and BailoutStack.
+//
+// * Safepoints: live registers are pushed on the stack before a VM call, so
+// MachineState stores the register sets and a pointer to the stack memory
+// where these registers were pushed. This is also used by exception bailouts.
+class MOZ_STACK_CLASS MachineState {
+ struct NullState {};
+
+ struct BailoutState {
+ RegisterDump::FPUArray& floatRegs;
+ RegisterDump::GPRArray& regs;
+
+ BailoutState(RegisterDump::FPUArray& floatRegs,
+ RegisterDump::GPRArray& regs)
+ : floatRegs(floatRegs), regs(regs) {}
+ };
+
+ struct SafepointState {
+ FloatRegisterSet floatRegs;
+ GeneralRegisterSet regs;
+ // Pointers to the start of the pushed |floatRegs| and |regs| on the stack.
+ // This is the value of the stack pointer right before the first register
+ // was pushed.
+ char* floatSpillBase;
+ uintptr_t* spillBase;
+
+ SafepointState(const FloatRegisterSet& floatRegs,
+ const GeneralRegisterSet& regs, char* floatSpillBase,
+ uintptr_t* spillBase)
+ : floatRegs(floatRegs),
+ regs(regs),
+ floatSpillBase(floatSpillBase),
+ spillBase(spillBase) {}
+ uintptr_t* addressOfRegister(Register reg) const;
+ char* addressOfRegister(FloatRegister reg) const;
+ };
+ using State = mozilla::Variant<NullState, BailoutState, SafepointState>;
+ State state_{NullState()};
+
+ public:
+ MachineState() = default;
+ MachineState(const MachineState& other) = default;
+ MachineState& operator=(const MachineState& other) = default;
+
+ static MachineState FromBailout(RegisterDump::GPRArray& regs,
+ RegisterDump::FPUArray& fpregs) {
+ MachineState res;
+ res.state_.emplace<BailoutState>(fpregs, regs);
+ return res;
+ }
+
+ static MachineState FromSafepoint(const FloatRegisterSet& floatRegs,
+ const GeneralRegisterSet& regs,
+ char* floatSpillBase,
+ uintptr_t* spillBase) {
+ MachineState res;
+ res.state_.emplace<SafepointState>(floatRegs, regs, floatSpillBase,
+ spillBase);
+ return res;
+ }
+
+ bool has(Register reg) const {
+ if (state_.is<BailoutState>()) {
+ return true;
+ }
+ return state_.as<SafepointState>().regs.hasRegisterIndex(reg);
+ }
+ bool has(FloatRegister reg) const {
+ if (state_.is<BailoutState>()) {
+ return true;
+ }
+ return state_.as<SafepointState>().floatRegs.hasRegisterIndex(reg);
+ }
+
+ uintptr_t read(Register reg) const;
+ template <typename T>
+ T read(FloatRegister reg) const;
+
+ // Used by moving GCs to update pointers.
+ void write(Register reg, uintptr_t value) const;
+};
+
+} // namespace js::jit
+
+#endif /* jit_MachineState_h */
diff --git a/js/src/jit/MacroAssembler-inl.h b/js/src/jit/MacroAssembler-inl.h
new file mode 100644
index 0000000000..0e0b85244f
--- /dev/null
+++ b/js/src/jit/MacroAssembler-inl.h
@@ -0,0 +1,1072 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MacroAssembler_inl_h
+#define jit_MacroAssembler_inl_h
+
+#include "jit/MacroAssembler.h"
+
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "gc/Zone.h"
+#include "jit/CalleeToken.h"
+#include "jit/CompileWrappers.h"
+#include "jit/JitFrames.h"
+#include "jit/JSJitFrameIter.h"
+#include "util/DifferentialTesting.h"
+#include "vm/BigIntType.h"
+#include "vm/JSObject.h"
+#include "vm/ProxyObject.h"
+#include "vm/Runtime.h"
+#include "vm/StringType.h"
+
+#include "jit/ABIFunctionList-inl.h"
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/MacroAssembler-x86-inl.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/MacroAssembler-x64-inl.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/MacroAssembler-arm-inl.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/MacroAssembler-arm64-inl.h"
+#elif defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/MacroAssembler-mips32-inl.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/MacroAssembler-mips64-inl.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/MacroAssembler-loong64-inl.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/MacroAssembler-wasm32-inl.h"
+#elif !defined(JS_CODEGEN_NONE)
+# error "Unknown architecture!"
+#endif
+
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+template <typename Sig>
+DynFn DynamicFunction(Sig fun) {
+ ABIFunctionSignature<Sig> sig;
+ return DynFn{sig.address(fun)};
+}
+
+// Helper for generatePreBarrier.
+inline DynFn JitPreWriteBarrier(MIRType type) {
+ switch (type) {
+ case MIRType::Value: {
+ using Fn = void (*)(JSRuntime * rt, Value * vp);
+ return DynamicFunction<Fn>(JitValuePreWriteBarrier);
+ }
+ case MIRType::String: {
+ using Fn = void (*)(JSRuntime * rt, JSString * *stringp);
+ return DynamicFunction<Fn>(JitStringPreWriteBarrier);
+ }
+ case MIRType::Object: {
+ using Fn = void (*)(JSRuntime * rt, JSObject * *objp);
+ return DynamicFunction<Fn>(JitObjectPreWriteBarrier);
+ }
+ case MIRType::Shape: {
+ using Fn = void (*)(JSRuntime * rt, Shape * *shapep);
+ return DynamicFunction<Fn>(JitShapePreWriteBarrier);
+ }
+ default:
+ MOZ_CRASH();
+ }
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// Stack manipulation functions.
+
+CodeOffset MacroAssembler::PushWithPatch(ImmWord word) {
+ framePushed_ += sizeof(word.value);
+ return pushWithPatch(word);
+}
+
+CodeOffset MacroAssembler::PushWithPatch(ImmPtr imm) {
+ return PushWithPatch(ImmWord(uintptr_t(imm.value)));
+}
+
+// ===============================================================
+// Simple call functions.
+
+void MacroAssembler::call(TrampolinePtr code) { call(ImmPtr(code.value)); }
+
+CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
+ const Register reg) {
+ CodeOffset l = call(reg);
+ append(desc, l);
+ return l;
+}
+
+CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
+ uint32_t funcIndex) {
+ CodeOffset l = callWithPatch();
+ append(desc, l, funcIndex);
+ return l;
+}
+
+void MacroAssembler::call(const wasm::CallSiteDesc& desc, wasm::Trap trap) {
+ CodeOffset l = callWithPatch();
+ append(desc, l, trap);
+}
+
+CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
+ wasm::SymbolicAddress imm) {
+ MOZ_ASSERT(wasm::NeedsBuiltinThunk(imm),
+ "only for functions which may appear in profiler");
+ CodeOffset raOffset = call(imm);
+ append(desc, raOffset);
+ return raOffset;
+}
+
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::passABIArg(Register reg) {
+ passABIArg(MoveOperand(reg), MoveOp::GENERAL);
+}
+
+void MacroAssembler::passABIArg(FloatRegister reg, MoveOp::Type type) {
+ passABIArg(MoveOperand(reg), type);
+}
+
+void MacroAssembler::callWithABI(DynFn fun, MoveOp::Type result,
+ CheckUnsafeCallWithABI check) {
+ AutoProfilerCallInstrumentation profiler(*this);
+ callWithABINoProfiler(fun.address, result, check);
+}
+
+template <typename Sig, Sig fun>
+void MacroAssembler::callWithABI(MoveOp::Type result,
+ CheckUnsafeCallWithABI check) {
+ ABIFunction<Sig, fun> abiFun;
+ AutoProfilerCallInstrumentation profiler(*this);
+ callWithABINoProfiler(abiFun.address(), result, check);
+}
+
+void MacroAssembler::callWithABI(Register fun, MoveOp::Type result) {
+ AutoProfilerCallInstrumentation profiler(*this);
+ callWithABINoProfiler(fun, result);
+}
+
+void MacroAssembler::callWithABI(const Address& fun, MoveOp::Type result) {
+ AutoProfilerCallInstrumentation profiler(*this);
+ callWithABINoProfiler(fun, result);
+}
+
+void MacroAssembler::appendSignatureType(MoveOp::Type type) {
+#ifdef JS_SIMULATOR
+ signature_ <<= ArgType_Shift;
+ switch (type) {
+ case MoveOp::GENERAL:
+ signature_ |= ArgType_General;
+ break;
+ case MoveOp::DOUBLE:
+ signature_ |= ArgType_Float64;
+ break;
+ case MoveOp::FLOAT32:
+ signature_ |= ArgType_Float32;
+ break;
+ default:
+ MOZ_CRASH("Invalid argument type");
+ }
+#endif
+}
+
+ABIFunctionType MacroAssembler::signature() const {
+#ifdef JS_SIMULATOR
+# ifdef DEBUG
+ switch (signature_) {
+ case Args_General0:
+ case Args_General1:
+ case Args_General2:
+ case Args_General3:
+ case Args_General4:
+ case Args_General5:
+ case Args_General6:
+ case Args_General7:
+ case Args_General8:
+ case Args_Double_None:
+ case Args_Int_Double:
+ case Args_Float32_Float32:
+ case Args_Int_Float32:
+ case Args_Double_Double:
+ case Args_Double_Int:
+ case Args_Double_DoubleInt:
+ case Args_Double_DoubleDouble:
+ case Args_Double_IntDouble:
+ case Args_Int_IntDouble:
+ case Args_Int_DoubleInt:
+ case Args_Int_DoubleIntInt:
+ case Args_Int_IntDoubleIntInt:
+ case Args_Double_DoubleDoubleDouble:
+ case Args_Double_DoubleDoubleDoubleDouble:
+ break;
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+# endif // DEBUG
+
+ return ABIFunctionType(signature_);
+#else
+ // No simulator enabled.
+ MOZ_CRASH("Only available for making calls within a simulator.");
+#endif
+}
+
+// ===============================================================
+// Jit Frames.
+
+uint32_t MacroAssembler::callJitNoProfiler(Register callee) {
+#ifdef JS_USE_LINK_REGISTER
+ // The return address is pushed by the callee.
+ call(callee);
+#else
+ callAndPushReturnAddress(callee);
+#endif
+ return currentOffset();
+}
+
+uint32_t MacroAssembler::callJit(Register callee) {
+ AutoProfilerCallInstrumentation profiler(*this);
+ uint32_t ret = callJitNoProfiler(callee);
+ return ret;
+}
+
+uint32_t MacroAssembler::callJit(JitCode* callee) {
+ AutoProfilerCallInstrumentation profiler(*this);
+ call(callee);
+ return currentOffset();
+}
+
+uint32_t MacroAssembler::callJit(TrampolinePtr code) {
+ AutoProfilerCallInstrumentation profiler(*this);
+ call(code);
+ return currentOffset();
+}
+
+uint32_t MacroAssembler::callJit(ImmPtr callee) {
+ AutoProfilerCallInstrumentation profiler(*this);
+ call(callee);
+ return currentOffset();
+}
+
+void MacroAssembler::pushFrameDescriptor(FrameType type) {
+ uint32_t descriptor = MakeFrameDescriptor(type);
+ push(Imm32(descriptor));
+}
+
+void MacroAssembler::PushFrameDescriptor(FrameType type) {
+ uint32_t descriptor = MakeFrameDescriptor(type);
+ Push(Imm32(descriptor));
+}
+
+void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type,
+ uint32_t argc) {
+ uint32_t descriptor = MakeFrameDescriptorForJitCall(type, argc);
+ push(Imm32(descriptor));
+}
+
+void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type,
+ uint32_t argc) {
+ uint32_t descriptor = MakeFrameDescriptorForJitCall(type, argc);
+ Push(Imm32(descriptor));
+}
+
+void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type,
+ Register argc,
+ Register scratch) {
+ if (argc != scratch) {
+ mov(argc, scratch);
+ }
+ lshift32(Imm32(NUMACTUALARGS_SHIFT), scratch);
+ or32(Imm32(int32_t(type)), scratch);
+ push(scratch);
+}
+
+void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type,
+ Register argc,
+ Register scratch) {
+ pushFrameDescriptorForJitCall(type, argc, scratch);
+ framePushed_ += sizeof(uintptr_t);
+}
+
+void MacroAssembler::loadNumActualArgs(Register framePtr, Register dest) {
+ loadPtr(Address(framePtr, JitFrameLayout::offsetOfDescriptor()), dest);
+ rshift32(Imm32(NUMACTUALARGS_SHIFT), dest);
+}
+
+void MacroAssembler::PushCalleeToken(Register callee, bool constructing) {
+ if (constructing) {
+ orPtr(Imm32(CalleeToken_FunctionConstructing), callee);
+ Push(callee);
+ andPtr(Imm32(uint32_t(CalleeTokenMask)), callee);
+ } else {
+ static_assert(CalleeToken_Function == 0,
+ "Non-constructing call requires no tagging");
+ Push(callee);
+ }
+}
+
+void MacroAssembler::loadFunctionFromCalleeToken(Address token, Register dest) {
+#ifdef DEBUG
+ Label ok;
+ loadPtr(token, dest);
+ andPtr(Imm32(uint32_t(~CalleeTokenMask)), dest);
+ branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_Function), &ok);
+ branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_FunctionConstructing),
+ &ok);
+ assumeUnreachable("Unexpected CalleeToken tag");
+ bind(&ok);
+#endif
+ loadPtr(token, dest);
+ andPtr(Imm32(uint32_t(CalleeTokenMask)), dest);
+}
+
+uint32_t MacroAssembler::buildFakeExitFrame(Register scratch) {
+ mozilla::DebugOnly<uint32_t> initialDepth = framePushed();
+
+ PushFrameDescriptor(FrameType::IonJS);
+ uint32_t retAddr = pushFakeReturnAddress(scratch);
+ Push(FramePointer);
+
+ MOZ_ASSERT(framePushed() == initialDepth + ExitFrameLayout::Size());
+ return retAddr;
+}
+
+// ===============================================================
+// Exit frame footer.
+
+void MacroAssembler::enterExitFrame(Register cxreg, Register scratch,
+ const VMFunctionData* f) {
+ MOZ_ASSERT(f);
+ linkExitFrame(cxreg, scratch);
+ // Push VMFunction pointer, to mark arguments.
+ Push(ImmPtr(f));
+}
+
+void MacroAssembler::enterFakeExitFrame(Register cxreg, Register scratch,
+ ExitFrameType type) {
+ linkExitFrame(cxreg, scratch);
+ Push(Imm32(int32_t(type)));
+}
+
+void MacroAssembler::enterFakeExitFrameForNative(Register cxreg,
+ Register scratch,
+ bool isConstructing) {
+ enterFakeExitFrame(cxreg, scratch,
+ isConstructing ? ExitFrameType::ConstructNative
+ : ExitFrameType::CallNative);
+}
+
+void MacroAssembler::leaveExitFrame(size_t extraFrame) {
+ freeStack(ExitFooterFrame::Size() + extraFrame);
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const ConstantOrRegister& src,
+ const ValueOperand& dest) {
+ if (src.constant()) {
+ moveValue(src.value(), dest);
+ return;
+ }
+
+ moveValue(src.reg(), dest);
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(ImmPtr imm, Register dest) {
+ addPtr(ImmWord(uintptr_t(imm.value)), dest);
+}
+
+// ===============================================================
+// Branch functions
+
+template <class L>
+void MacroAssembler::branchIfFalseBool(Register reg, L label) {
+ // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
+ branchTest32(Assembler::Zero, reg, Imm32(0xFF), label);
+}
+
+void MacroAssembler::branchIfTrueBool(Register reg, Label* label) {
+ // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
+ branchTest32(Assembler::NonZero, reg, Imm32(0xFF), label);
+}
+
+void MacroAssembler::branchIfRope(Register str, Label* label) {
+ Address flags(str, JSString::offsetOfFlags());
+ branchTest32(Assembler::Zero, flags, Imm32(JSString::LINEAR_BIT), label);
+}
+
+void MacroAssembler::branchIfNotRope(Register str, Label* label) {
+ Address flags(str, JSString::offsetOfFlags());
+ branchTest32(Assembler::NonZero, flags, Imm32(JSString::LINEAR_BIT), label);
+}
+
+void MacroAssembler::branchLatin1String(Register string, Label* label) {
+ branchTest32(Assembler::NonZero, Address(string, JSString::offsetOfFlags()),
+ Imm32(JSString::LATIN1_CHARS_BIT), label);
+}
+
+void MacroAssembler::branchTwoByteString(Register string, Label* label) {
+ branchTest32(Assembler::Zero, Address(string, JSString::offsetOfFlags()),
+ Imm32(JSString::LATIN1_CHARS_BIT), label);
+}
+
+void MacroAssembler::branchIfBigIntIsNegative(Register bigInt, Label* label) {
+ branchTest32(Assembler::NonZero, Address(bigInt, BigInt::offsetOfFlags()),
+ Imm32(BigInt::signBitMask()), label);
+}
+
+void MacroAssembler::branchIfBigIntIsNonNegative(Register bigInt,
+ Label* label) {
+ branchTest32(Assembler::Zero, Address(bigInt, BigInt::offsetOfFlags()),
+ Imm32(BigInt::signBitMask()), label);
+}
+
+void MacroAssembler::branchIfBigIntIsZero(Register bigInt, Label* label) {
+ branch32(Assembler::Equal, Address(bigInt, BigInt::offsetOfLength()),
+ Imm32(0), label);
+}
+
+void MacroAssembler::branchIfBigIntIsNonZero(Register bigInt, Label* label) {
+ branch32(Assembler::NotEqual, Address(bigInt, BigInt::offsetOfLength()),
+ Imm32(0), label);
+}
+
+void MacroAssembler::branchTestFunctionFlags(Register fun, uint32_t flags,
+ Condition cond, Label* label) {
+ Address address(fun, JSFunction::offsetOfFlagsAndArgCount());
+ branchTest32(cond, address, Imm32(flags), label);
+}
+
+void MacroAssembler::branchIfNotFunctionIsNonBuiltinCtor(Register fun,
+ Register scratch,
+ Label* label) {
+ // Guard the function has the BASESCRIPT and CONSTRUCTOR flags and does NOT
+ // have the SELF_HOSTED flag.
+ // This is equivalent to JSFunction::isNonBuiltinConstructor.
+ constexpr int32_t mask = FunctionFlags::BASESCRIPT |
+ FunctionFlags::SELF_HOSTED |
+ FunctionFlags::CONSTRUCTOR;
+ constexpr int32_t expected =
+ FunctionFlags::BASESCRIPT | FunctionFlags::CONSTRUCTOR;
+
+ load32(Address(fun, JSFunction::offsetOfFlagsAndArgCount()), scratch);
+ and32(Imm32(mask), scratch);
+ branch32(Assembler::NotEqual, scratch, Imm32(expected), label);
+}
+
+void MacroAssembler::branchIfFunctionHasNoJitEntry(Register fun,
+ bool isConstructing,
+ Label* label) {
+ uint16_t flags = FunctionFlags::HasJitEntryFlags(isConstructing);
+ branchTestFunctionFlags(fun, flags, Assembler::Zero, label);
+}
+
+void MacroAssembler::branchIfFunctionHasJitEntry(Register fun,
+ bool isConstructing,
+ Label* label) {
+ uint16_t flags = FunctionFlags::HasJitEntryFlags(isConstructing);
+ branchTestFunctionFlags(fun, flags, Assembler::NonZero, label);
+}
+
+void MacroAssembler::branchIfScriptHasJitScript(Register script, Label* label) {
+ static_assert(ScriptWarmUpData::JitScriptTag == 0,
+ "Code below depends on tag value");
+ branchTestPtr(Assembler::Zero,
+ Address(script, JSScript::offsetOfWarmUpData()),
+ Imm32(ScriptWarmUpData::TagMask), label);
+}
+
+void MacroAssembler::branchIfScriptHasNoJitScript(Register script,
+ Label* label) {
+ static_assert(ScriptWarmUpData::JitScriptTag == 0,
+ "Code below depends on tag value");
+ static_assert(BaseScript::offsetOfWarmUpData() ==
+ SelfHostedLazyScript::offsetOfWarmUpData(),
+ "SelfHostedLazyScript and BaseScript must use same layout for "
+ "warmUpData_");
+ branchTestPtr(Assembler::NonZero,
+ Address(script, JSScript::offsetOfWarmUpData()),
+ Imm32(ScriptWarmUpData::TagMask), label);
+}
+
+void MacroAssembler::loadJitScript(Register script, Register dest) {
+#ifdef DEBUG
+ Label ok;
+ branchIfScriptHasJitScript(script, &ok);
+ assumeUnreachable("Script has no JitScript!");
+ bind(&ok);
+#endif
+
+ static_assert(ScriptWarmUpData::JitScriptTag == 0,
+ "Code below depends on tag value");
+ loadPtr(Address(script, JSScript::offsetOfWarmUpData()), dest);
+}
+
+void MacroAssembler::loadFunctionArgCount(Register func, Register output) {
+ load32(Address(func, JSFunction::offsetOfFlagsAndArgCount()), output);
+ rshift32(Imm32(JSFunction::ArgCountShift), output);
+}
+
+void MacroAssembler::branchIfObjectEmulatesUndefined(Register objReg,
+ Register scratch,
+ Label* slowCheck,
+ Label* label) {
+ // The branches to out-of-line code here implement a conservative version
+ // of the JSObject::isWrapper test performed in EmulatesUndefined.
+ loadObjClassUnsafe(objReg, scratch);
+
+ branchTestClassIsProxy(true, scratch, slowCheck);
+
+ Address flags(scratch, JSClass::offsetOfFlags());
+ branchTest32(Assembler::NonZero, flags, Imm32(JSCLASS_EMULATES_UNDEFINED),
+ label);
+}
+
+void MacroAssembler::branchFunctionKind(Condition cond,
+ FunctionFlags::FunctionKind kind,
+ Register fun, Register scratch,
+ Label* label) {
+ Address address(fun, JSFunction::offsetOfFlagsAndArgCount());
+ load32(address, scratch);
+ and32(Imm32(FunctionFlags::FUNCTION_KIND_MASK), scratch);
+ branch32(cond, scratch, Imm32(kind), label);
+}
+
+void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
+ const JSClass* clasp, Register scratch,
+ Register spectreRegToZero,
+ Label* label) {
+ MOZ_ASSERT(obj != scratch);
+ MOZ_ASSERT(scratch != spectreRegToZero);
+
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp),
+ label);
+
+ if (JitOptions.spectreObjectMitigations) {
+ spectreZeroRegister(cond, scratch, spectreRegToZero);
+ }
+}
+
+void MacroAssembler::branchTestObjClassNoSpectreMitigations(
+ Condition cond, Register obj, const JSClass* clasp, Register scratch,
+ Label* label) {
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp),
+ label);
+}
+
+void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
+ const Address& clasp, Register scratch,
+ Register spectreRegToZero,
+ Label* label) {
+ MOZ_ASSERT(obj != scratch);
+ MOZ_ASSERT(scratch != spectreRegToZero);
+
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ loadPtr(Address(scratch, BaseShape::offsetOfClasp()), scratch);
+ branchPtr(cond, clasp, scratch, label);
+
+ if (JitOptions.spectreObjectMitigations) {
+ spectreZeroRegister(cond, scratch, spectreRegToZero);
+ }
+}
+
+void MacroAssembler::branchTestObjClassNoSpectreMitigations(
+ Condition cond, Register obj, const Address& clasp, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(obj != scratch);
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ loadPtr(Address(scratch, BaseShape::offsetOfClasp()), scratch);
+ branchPtr(cond, clasp, scratch, label);
+}
+
+void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
+ Register clasp, Register scratch,
+ Register spectreRegToZero,
+ Label* label) {
+ MOZ_ASSERT(obj != scratch);
+ MOZ_ASSERT(scratch != spectreRegToZero);
+
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ loadPtr(Address(scratch, BaseShape::offsetOfClasp()), scratch);
+ branchPtr(cond, clasp, scratch, label);
+
+ if (JitOptions.spectreObjectMitigations) {
+ spectreZeroRegister(cond, scratch, spectreRegToZero);
+ }
+}
+
+void MacroAssembler::branchTestClassIsFunction(Condition cond, Register clasp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ if (cond == Assembler::Equal) {
+ branchPtr(Assembler::Equal, clasp, ImmPtr(&FunctionClass), label);
+ branchPtr(Assembler::Equal, clasp, ImmPtr(&ExtendedFunctionClass), label);
+ return;
+ }
+
+ Label isFunction;
+ branchPtr(Assembler::Equal, clasp, ImmPtr(&FunctionClass), &isFunction);
+ branchPtr(Assembler::NotEqual, clasp, ImmPtr(&ExtendedFunctionClass), label);
+ bind(&isFunction);
+}
+
+void MacroAssembler::branchTestObjIsFunction(Condition cond, Register obj,
+ Register scratch,
+ Register spectreRegToZero,
+ Label* label) {
+ MOZ_ASSERT(scratch != spectreRegToZero);
+
+ branchTestObjIsFunctionNoSpectreMitigations(cond, obj, scratch, label);
+
+ if (JitOptions.spectreObjectMitigations) {
+ spectreZeroRegister(cond, scratch, spectreRegToZero);
+ }
+}
+
+void MacroAssembler::branchTestObjIsFunctionNoSpectreMitigations(
+ Condition cond, Register obj, Register scratch, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(obj != scratch);
+
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ loadPtr(Address(scratch, BaseShape::offsetOfClasp()), scratch);
+ branchTestClassIsFunction(cond, scratch, label);
+}
+
+void MacroAssembler::branchTestObjShape(Condition cond, Register obj,
+ const Shape* shape, Register scratch,
+ Register spectreRegToZero,
+ Label* label) {
+ MOZ_ASSERT(obj != scratch);
+ MOZ_ASSERT(spectreRegToZero != scratch);
+
+ if (JitOptions.spectreObjectMitigations) {
+ move32(Imm32(0), scratch);
+ }
+
+ branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape),
+ label);
+
+ if (JitOptions.spectreObjectMitigations) {
+ spectreMovePtr(cond, scratch, spectreRegToZero);
+ }
+}
+
+void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond,
+ Register obj,
+ const Shape* shape,
+ Label* label) {
+ branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape),
+ label);
+}
+
+void MacroAssembler::branchTestObjShape(Condition cond, Register obj,
+ Register shape, Register scratch,
+ Register spectreRegToZero,
+ Label* label) {
+ MOZ_ASSERT(obj != scratch);
+ MOZ_ASSERT(obj != shape);
+ MOZ_ASSERT(spectreRegToZero != scratch);
+
+ if (JitOptions.spectreObjectMitigations) {
+ move32(Imm32(0), scratch);
+ }
+
+ branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label);
+
+ if (JitOptions.spectreObjectMitigations) {
+ spectreMovePtr(cond, scratch, spectreRegToZero);
+ }
+}
+
+void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond,
+ Register obj,
+ Register shape,
+ Label* label) {
+ branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label);
+}
+
+void MacroAssembler::branchTestObjShapeUnsafe(Condition cond, Register obj,
+ Register shape, Label* label) {
+ branchTestObjShapeNoSpectreMitigations(cond, obj, shape, label);
+}
+
+void MacroAssembler::branchTestClassIsProxy(bool proxy, Register clasp,
+ Label* label) {
+ branchTest32(proxy ? Assembler::NonZero : Assembler::Zero,
+ Address(clasp, JSClass::offsetOfFlags()),
+ Imm32(JSCLASS_IS_PROXY), label);
+}
+
+void MacroAssembler::branchTestObjectIsProxy(bool proxy, Register object,
+ Register scratch, Label* label) {
+ constexpr uint32_t ShiftedMask = (Shape::kindMask() << Shape::kindShift());
+ static_assert(uint32_t(Shape::Kind::Proxy) == 0,
+ "branchTest32 below depends on proxy kind being 0");
+ loadPtr(Address(object, JSObject::offsetOfShape()), scratch);
+ branchTest32(proxy ? Assembler::Zero : Assembler::NonZero,
+ Address(scratch, Shape::offsetOfImmutableFlags()),
+ Imm32(ShiftedMask), label);
+}
+
+void MacroAssembler::branchTestProxyHandlerFamily(Condition cond,
+ Register proxy,
+ Register scratch,
+ const void* handlerp,
+ Label* label) {
+#ifdef DEBUG
+ Label ok;
+ branchTestObjectIsProxy(true, proxy, scratch, &ok);
+ assumeUnreachable("Expected ProxyObject in branchTestProxyHandlerFamily");
+ bind(&ok);
+#endif
+
+ Address handlerAddr(proxy, ProxyObject::offsetOfHandler());
+ loadPtr(handlerAddr, scratch);
+ Address familyAddr(scratch, BaseProxyHandler::offsetOfFamily());
+ branchPtr(cond, familyAddr, ImmPtr(handlerp), label);
+}
+
+void MacroAssembler::branchTestNeedsIncrementalBarrier(Condition cond,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ CompileZone* zone = realm()->zone();
+ const uint32_t* needsBarrierAddr = zone->addressOfNeedsIncrementalBarrier();
+ branchTest32(cond, AbsoluteAddress(needsBarrierAddr), Imm32(0x1), label);
+}
+
+void MacroAssembler::branchTestNeedsIncrementalBarrierAnyZone(
+ Condition cond, Label* label, Register scratch) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ if (maybeRealm_) {
+ branchTestNeedsIncrementalBarrier(cond, label);
+ } else {
+ // We are compiling the interpreter or another runtime-wide trampoline, so
+ // we have to load cx->zone.
+ loadPtr(AbsoluteAddress(runtime()->addressOfZone()), scratch);
+ Address needsBarrierAddr(scratch, Zone::offsetOfNeedsIncrementalBarrier());
+ branchTest32(cond, needsBarrierAddr, Imm32(0x1), label);
+ }
+}
+
+void MacroAssembler::branchTestMagicValue(Condition cond,
+ const ValueOperand& val,
+ JSWhyMagic why, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ branchTestValue(cond, val, MagicValue(why), label);
+}
+
+void MacroAssembler::branchDoubleNotInInt64Range(Address src, Register temp,
+ Label* fail) {
+ using mozilla::FloatingPoint;
+
+ // Tests if double is in [INT64_MIN; INT64_MAX] range
+ uint32_t EXPONENT_MASK = 0x7ff00000;
+ uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32;
+ uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 63)
+ << EXPONENT_SHIFT;
+
+ load32(Address(src.base, src.offset + sizeof(int32_t)), temp);
+ and32(Imm32(EXPONENT_MASK), temp);
+ branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
+}
+
+void MacroAssembler::branchDoubleNotInUInt64Range(Address src, Register temp,
+ Label* fail) {
+ using mozilla::FloatingPoint;
+
+ // Note: returns failure on -0.0
+ // Tests if double is in [0; UINT64_MAX] range
+ // Take the sign also in the equation. That way we can compare in one test?
+ uint32_t EXPONENT_MASK = 0xfff00000;
+ uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32;
+ uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 64)
+ << EXPONENT_SHIFT;
+
+ load32(Address(src.base, src.offset + sizeof(int32_t)), temp);
+ and32(Imm32(EXPONENT_MASK), temp);
+ branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
+}
+
+void MacroAssembler::branchFloat32NotInInt64Range(Address src, Register temp,
+ Label* fail) {
+ using mozilla::FloatingPoint;
+
+ // Tests if float is in [INT64_MIN; INT64_MAX] range
+ uint32_t EXPONENT_MASK = 0x7f800000;
+ uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift;
+ uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 63)
+ << EXPONENT_SHIFT;
+
+ load32(src, temp);
+ and32(Imm32(EXPONENT_MASK), temp);
+ branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
+}
+
+void MacroAssembler::branchFloat32NotInUInt64Range(Address src, Register temp,
+ Label* fail) {
+ using mozilla::FloatingPoint;
+
+ // Note: returns failure on -0.0
+ // Tests if float is in [0; UINT64_MAX] range
+ // Take the sign also in the equation. That way we can compare in one test?
+ uint32_t EXPONENT_MASK = 0xff800000;
+ uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift;
+ uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 64)
+ << EXPONENT_SHIFT;
+
+ load32(src, temp);
+ and32(Imm32(EXPONENT_MASK), temp);
+ branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
+}
+
+// ========================================================================
+// Canonicalization primitives.
+void MacroAssembler::canonicalizeFloat(FloatRegister reg) {
+ Label notNaN;
+ branchFloat(DoubleOrdered, reg, reg, &notNaN);
+ loadConstantFloat32(float(JS::GenericNaN()), reg);
+ bind(&notNaN);
+}
+
+void MacroAssembler::canonicalizeFloatIfDeterministic(FloatRegister reg) {
+ // See the comment in TypedArrayObjectTemplate::getElement.
+ if (js::SupportDifferentialTesting()) {
+ canonicalizeFloat(reg);
+ }
+}
+
+void MacroAssembler::canonicalizeDouble(FloatRegister reg) {
+ Label notNaN;
+ branchDouble(DoubleOrdered, reg, reg, &notNaN);
+ loadConstantDouble(JS::GenericNaN(), reg);
+ bind(&notNaN);
+}
+
+void MacroAssembler::canonicalizeDoubleIfDeterministic(FloatRegister reg) {
+ // See the comment in TypedArrayObjectTemplate::getElement.
+ if (js::SupportDifferentialTesting()) {
+ canonicalizeDouble(reg);
+ }
+}
+
+// ========================================================================
+// Memory access primitives.
+template <class T>
+void MacroAssembler::storeDouble(FloatRegister src, const T& dest) {
+ canonicalizeDoubleIfDeterministic(src);
+ storeUncanonicalizedDouble(src, dest);
+}
+
+template void MacroAssembler::storeDouble(FloatRegister src,
+ const Address& dest);
+template void MacroAssembler::storeDouble(FloatRegister src,
+ const BaseIndex& dest);
+
+template <class T>
+void MacroAssembler::boxDouble(FloatRegister src, const T& dest) {
+ storeDouble(src, dest);
+}
+
+template <class T>
+void MacroAssembler::storeFloat32(FloatRegister src, const T& dest) {
+ canonicalizeFloatIfDeterministic(src);
+ storeUncanonicalizedFloat32(src, dest);
+}
+
+template void MacroAssembler::storeFloat32(FloatRegister src,
+ const Address& dest);
+template void MacroAssembler::storeFloat32(FloatRegister src,
+ const BaseIndex& dest);
+
+template <typename T>
+void MacroAssembler::fallibleUnboxInt32(const T& src, Register dest,
+ Label* fail) {
+ // Int32Value can be unboxed efficiently with unboxInt32, so use that.
+ branchTestInt32(Assembler::NotEqual, src, fail);
+ unboxInt32(src, dest);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxBoolean(const T& src, Register dest,
+ Label* fail) {
+ // BooleanValue can be unboxed efficiently with unboxBoolean, so use that.
+ branchTestBoolean(Assembler::NotEqual, src, fail);
+ unboxBoolean(src, dest);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxObject(const T& src, Register dest,
+ Label* fail) {
+ fallibleUnboxPtr(src, dest, JSVAL_TYPE_OBJECT, fail);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxString(const T& src, Register dest,
+ Label* fail) {
+ fallibleUnboxPtr(src, dest, JSVAL_TYPE_STRING, fail);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxSymbol(const T& src, Register dest,
+ Label* fail) {
+ fallibleUnboxPtr(src, dest, JSVAL_TYPE_SYMBOL, fail);
+}
+
+template <typename T>
+void MacroAssembler::fallibleUnboxBigInt(const T& src, Register dest,
+ Label* fail) {
+ fallibleUnboxPtr(src, dest, JSVAL_TYPE_BIGINT, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+#ifndef JS_CODEGEN_ARM64
+
+template <typename T>
+void MacroAssembler::branchTestStackPtr(Condition cond, T t, Label* label) {
+ branchTestPtr(cond, getStackPointer(), t, label);
+}
+
+template <typename T>
+void MacroAssembler::branchStackPtr(Condition cond, T rhs, Label* label) {
+ branchPtr(cond, getStackPointer(), rhs, label);
+}
+
+template <typename T>
+void MacroAssembler::branchStackPtrRhs(Condition cond, T lhs, Label* label) {
+ branchPtr(cond, lhs, getStackPointer(), label);
+}
+
+template <typename T>
+void MacroAssembler::addToStackPtr(T t) {
+ addPtr(t, getStackPointer());
+}
+
+template <typename T>
+void MacroAssembler::addStackPtrTo(T t) {
+ addPtr(getStackPointer(), t);
+}
+
+void MacroAssembler::reserveStack(uint32_t amount) {
+ subFromStackPtr(Imm32(amount));
+ adjustFrame(amount);
+}
+#endif // !JS_CODEGEN_ARM64
+
+void MacroAssembler::loadObjClassUnsafe(Register obj, Register dest) {
+ loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
+ loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest);
+ loadPtr(Address(dest, BaseShape::offsetOfClasp()), dest);
+}
+
+template <typename EmitPreBarrier>
+void MacroAssembler::storeObjShape(Register shape, Register obj,
+ EmitPreBarrier emitPreBarrier) {
+ MOZ_ASSERT(shape != obj);
+ Address shapeAddr(obj, JSObject::offsetOfShape());
+ emitPreBarrier(*this, shapeAddr);
+ storePtr(shape, shapeAddr);
+}
+
+template <typename EmitPreBarrier>
+void MacroAssembler::storeObjShape(Shape* shape, Register obj,
+ EmitPreBarrier emitPreBarrier) {
+ Address shapeAddr(obj, JSObject::offsetOfShape());
+ emitPreBarrier(*this, shapeAddr);
+ storePtr(ImmGCPtr(shape), shapeAddr);
+}
+
+void MacroAssembler::loadObjProto(Register obj, Register dest) {
+ loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
+ loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest);
+ loadPtr(Address(dest, BaseShape::offsetOfProto()), dest);
+}
+
+void MacroAssembler::loadStringLength(Register str, Register dest) {
+ load32(Address(str, JSString::offsetOfLength()), dest);
+}
+
+void MacroAssembler::assertStackAlignment(uint32_t alignment,
+ int32_t offset /* = 0 */) {
+#ifdef DEBUG
+ Label ok, bad;
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment));
+
+ // Wrap around the offset to be a non-negative number.
+ offset %= alignment;
+ if (offset < 0) {
+ offset += alignment;
+ }
+
+ // Test if each bit from offset is set.
+ uint32_t off = offset;
+ while (off) {
+ uint32_t lowestBit = 1 << mozilla::CountTrailingZeroes32(off);
+ branchTestStackPtr(Assembler::Zero, Imm32(lowestBit), &bad);
+ off ^= lowestBit;
+ }
+
+ // Check that all remaining bits are zero.
+ branchTestStackPtr(Assembler::Zero, Imm32((alignment - 1) ^ offset), &ok);
+
+ bind(&bad);
+ breakpoint();
+ bind(&ok);
+#endif
+}
+
+void MacroAssembler::storeCallBoolResult(Register reg) {
+ convertBoolToInt32(ReturnReg, reg);
+}
+
+void MacroAssembler::storeCallInt32Result(Register reg) {
+#if JS_BITS_PER_WORD == 32
+ storeCallPointerResult(reg);
+#else
+ // Ensure the upper 32 bits are cleared.
+ move32(ReturnReg, reg);
+#endif
+}
+
+void MacroAssembler::storeCallResultValue(AnyRegister dest, JSValueType type) {
+ unboxValue(JSReturnOperand, dest, type);
+}
+
+void MacroAssembler::storeCallResultValue(TypedOrValueRegister dest) {
+ if (dest.hasValue()) {
+ storeCallResultValue(dest.valueReg());
+ } else {
+ storeCallResultValue(dest.typedReg(), ValueTypeFromMIRType(dest.type()));
+ }
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_MacroAssembler_inl_h */
diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp
new file mode 100644
index 0000000000..8f66c6218f
--- /dev/null
+++ b/js/src/jit/MacroAssembler.cpp
@@ -0,0 +1,6222 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/MacroAssembler-inl.h"
+
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/XorShift128PlusRNG.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "jit/AtomicOp.h"
+#include "jit/AtomicOperations.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineJIT.h"
+#include "jit/JitFrames.h"
+#include "jit/JitOptions.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitScript.h"
+#include "jit/MoveEmitter.h"
+#include "jit/ReciprocalMulConstants.h"
+#include "jit/SharedICHelpers.h"
+#include "jit/SharedICRegisters.h"
+#include "jit/Simulator.h"
+#include "jit/VMFunctions.h"
+#include "js/Conversions.h"
+#include "js/friend/DOMProxy.h" // JS::ExpandoAndGeneration
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "vm/ArgumentsObject.h"
+#include "vm/ArrayBufferViewObject.h"
+#include "vm/FunctionFlags.h" // js::FunctionFlags
+#include "vm/Iteration.h"
+#include "vm/JSContext.h"
+#include "vm/TypedArrayObject.h"
+#include "wasm/WasmBuiltins.h"
+#include "wasm/WasmCodegenTypes.h"
+#include "wasm/WasmInstanceData.h"
+#include "wasm/WasmMemory.h"
+#include "wasm/WasmValidate.h"
+
+#include "jit/TemplateObject-inl.h"
+#include "vm/BytecodeUtil-inl.h"
+#include "vm/Interpreter-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::GenericNaN;
+using JS::ToInt32;
+
+using mozilla::CheckedInt;
+
+TrampolinePtr MacroAssembler::preBarrierTrampoline(MIRType type) {
+ const JitRuntime* rt = runtime()->jitRuntime();
+ return rt->preBarrier(type);
+}
+
+template <typename S, typename T>
+static void StoreToTypedFloatArray(MacroAssembler& masm, int arrayType,
+ const S& value, const T& dest) {
+ switch (arrayType) {
+ case Scalar::Float32:
+ masm.storeFloat32(value, dest);
+ break;
+ case Scalar::Float64:
+ masm.storeDouble(value, dest);
+ break;
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+}
+
+void MacroAssembler::storeToTypedFloatArray(Scalar::Type arrayType,
+ FloatRegister value,
+ const BaseIndex& dest) {
+ StoreToTypedFloatArray(*this, arrayType, value, dest);
+}
+void MacroAssembler::storeToTypedFloatArray(Scalar::Type arrayType,
+ FloatRegister value,
+ const Address& dest) {
+ StoreToTypedFloatArray(*this, arrayType, value, dest);
+}
+
+template <typename S, typename T>
+static void StoreToTypedBigIntArray(MacroAssembler& masm,
+ Scalar::Type arrayType, const S& value,
+ const T& dest) {
+ MOZ_ASSERT(Scalar::isBigIntType(arrayType));
+ masm.store64(value, dest);
+}
+
+void MacroAssembler::storeToTypedBigIntArray(Scalar::Type arrayType,
+ Register64 value,
+ const BaseIndex& dest) {
+ StoreToTypedBigIntArray(*this, arrayType, value, dest);
+}
+void MacroAssembler::storeToTypedBigIntArray(Scalar::Type arrayType,
+ Register64 value,
+ const Address& dest) {
+ StoreToTypedBigIntArray(*this, arrayType, value, dest);
+}
+
+void MacroAssembler::boxUint32(Register source, ValueOperand dest,
+ Uint32Mode mode, Label* fail) {
+ switch (mode) {
+ // Fail if the value does not fit in an int32.
+ case Uint32Mode::FailOnDouble: {
+ branchTest32(Assembler::Signed, source, source, fail);
+ tagValue(JSVAL_TYPE_INT32, source, dest);
+ break;
+ }
+ case Uint32Mode::ForceDouble: {
+ // Always convert the value to double.
+ ScratchDoubleScope fpscratch(*this);
+ convertUInt32ToDouble(source, fpscratch);
+ boxDouble(fpscratch, dest, fpscratch);
+ break;
+ }
+ }
+}
+
+template <typename T>
+void MacroAssembler::loadFromTypedArray(Scalar::Type arrayType, const T& src,
+ AnyRegister dest, Register temp,
+ Label* fail) {
+ switch (arrayType) {
+ case Scalar::Int8:
+ load8SignExtend(src, dest.gpr());
+ break;
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ load8ZeroExtend(src, dest.gpr());
+ break;
+ case Scalar::Int16:
+ load16SignExtend(src, dest.gpr());
+ break;
+ case Scalar::Uint16:
+ load16ZeroExtend(src, dest.gpr());
+ break;
+ case Scalar::Int32:
+ load32(src, dest.gpr());
+ break;
+ case Scalar::Uint32:
+ if (dest.isFloat()) {
+ load32(src, temp);
+ convertUInt32ToDouble(temp, dest.fpu());
+ } else {
+ load32(src, dest.gpr());
+
+ // Bail out if the value doesn't fit into a signed int32 value. This
+ // is what allows MLoadUnboxedScalar to have a type() of
+ // MIRType::Int32 for UInt32 array loads.
+ branchTest32(Assembler::Signed, dest.gpr(), dest.gpr(), fail);
+ }
+ break;
+ case Scalar::Float32:
+ loadFloat32(src, dest.fpu());
+ canonicalizeFloat(dest.fpu());
+ break;
+ case Scalar::Float64:
+ loadDouble(src, dest.fpu());
+ canonicalizeDouble(dest.fpu());
+ break;
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+}
+
+template void MacroAssembler::loadFromTypedArray(Scalar::Type arrayType,
+ const Address& src,
+ AnyRegister dest,
+ Register temp, Label* fail);
+template void MacroAssembler::loadFromTypedArray(Scalar::Type arrayType,
+ const BaseIndex& src,
+ AnyRegister dest,
+ Register temp, Label* fail);
+
+template <typename T>
+void MacroAssembler::loadFromTypedArray(Scalar::Type arrayType, const T& src,
+ const ValueOperand& dest,
+ Uint32Mode uint32Mode, Register temp,
+ Label* fail) {
+ switch (arrayType) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ loadFromTypedArray(arrayType, src, AnyRegister(dest.scratchReg()),
+ InvalidReg, nullptr);
+ tagValue(JSVAL_TYPE_INT32, dest.scratchReg(), dest);
+ break;
+ case Scalar::Uint32:
+ // Don't clobber dest when we could fail, instead use temp.
+ load32(src, temp);
+ boxUint32(temp, dest, uint32Mode, fail);
+ break;
+ case Scalar::Float32: {
+ ScratchDoubleScope dscratch(*this);
+ FloatRegister fscratch = dscratch.asSingle();
+ loadFromTypedArray(arrayType, src, AnyRegister(fscratch),
+ dest.scratchReg(), nullptr);
+ convertFloat32ToDouble(fscratch, dscratch);
+ boxDouble(dscratch, dest, dscratch);
+ break;
+ }
+ case Scalar::Float64: {
+ ScratchDoubleScope fpscratch(*this);
+ loadFromTypedArray(arrayType, src, AnyRegister(fpscratch),
+ dest.scratchReg(), nullptr);
+ boxDouble(fpscratch, dest, fpscratch);
+ break;
+ }
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+}
+
+template void MacroAssembler::loadFromTypedArray(Scalar::Type arrayType,
+ const Address& src,
+ const ValueOperand& dest,
+ Uint32Mode uint32Mode,
+ Register temp, Label* fail);
+template void MacroAssembler::loadFromTypedArray(Scalar::Type arrayType,
+ const BaseIndex& src,
+ const ValueOperand& dest,
+ Uint32Mode uint32Mode,
+ Register temp, Label* fail);
+
+template <typename T>
+void MacroAssembler::loadFromTypedBigIntArray(Scalar::Type arrayType,
+ const T& src, Register bigInt,
+ Register64 temp) {
+ MOZ_ASSERT(Scalar::isBigIntType(arrayType));
+
+ load64(src, temp);
+ initializeBigInt64(arrayType, bigInt, temp);
+}
+
+template void MacroAssembler::loadFromTypedBigIntArray(Scalar::Type arrayType,
+ const Address& src,
+ Register bigInt,
+ Register64 temp);
+template void MacroAssembler::loadFromTypedBigIntArray(Scalar::Type arrayType,
+ const BaseIndex& src,
+ Register bigInt,
+ Register64 temp);
+
+// Inlined version of gc::CheckAllocatorState that checks the bare essentials
+// and bails for anything that cannot be handled with our jit allocators.
+void MacroAssembler::checkAllocatorState(Label* fail) {
+ // Don't execute the inline path if GC probes are built in.
+#ifdef JS_GC_PROBES
+ jump(fail);
+#endif
+
+#ifdef JS_GC_ZEAL
+ // Don't execute the inline path if gc zeal or tracing are active.
+ const uint32_t* ptrZealModeBits = runtime()->addressOfGCZealModeBits();
+ branch32(Assembler::NotEqual, AbsoluteAddress(ptrZealModeBits), Imm32(0),
+ fail);
+#endif
+
+ // Don't execute the inline path if the realm has an object metadata callback,
+ // as the metadata to use for the object may vary between executions of the
+ // op.
+ if (realm()->hasAllocationMetadataBuilder()) {
+ jump(fail);
+ }
+}
+
+bool MacroAssembler::shouldNurseryAllocate(gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap) {
+ // Note that Ion elides barriers on writes to objects known to be in the
+ // nursery, so any allocation that can be made into the nursery must be made
+ // into the nursery, even if the nursery is disabled. At runtime these will
+ // take the out-of-line path, which is required to insert a barrier for the
+ // initializing writes.
+ return IsNurseryAllocable(allocKind) && initialHeap != gc::TenuredHeap;
+}
+
+// Inline version of Nursery::allocateObject. If the object has dynamic slots,
+// this fills in the slots_ pointer.
+void MacroAssembler::nurseryAllocateObject(Register result, Register temp,
+ gc::AllocKind allocKind,
+ size_t nDynamicSlots, Label* fail,
+ const AllocSiteInput& allocSite) {
+ MOZ_ASSERT(IsNurseryAllocable(allocKind));
+
+ // We still need to allocate in the nursery, per the comment in
+ // shouldNurseryAllocate; however, we need to insert into the
+ // mallocedBuffers set, so bail to do the nursery allocation in the
+ // interpreter.
+ if (nDynamicSlots >= Nursery::MaxNurseryBufferSize / sizeof(Value)) {
+ jump(fail);
+ return;
+ }
+
+ // Check whether this allocation site needs pretenuring. This dynamic check
+ // only happens for baseline code.
+ if (allocSite.is<Register>()) {
+ Register site = allocSite.as<Register>();
+ branchTestPtr(Assembler::NonZero,
+ Address(site, gc::AllocSite::offsetOfScriptAndState()),
+ Imm32(gc::AllocSite::LONG_LIVED_BIT), fail);
+ }
+
+ // No explicit check for nursery.isEnabled() is needed, as the comparison
+ // with the nursery's end will always fail in such cases.
+ CompileZone* zone = realm()->zone();
+ size_t thingSize = gc::Arena::thingSize(allocKind);
+ size_t totalSize = thingSize;
+ if (nDynamicSlots) {
+ totalSize += ObjectSlots::allocSize(nDynamicSlots);
+ }
+ MOZ_ASSERT(totalSize < INT32_MAX);
+ MOZ_ASSERT(totalSize % gc::CellAlignBytes == 0);
+
+ bumpPointerAllocate(result, temp, fail, zone,
+ zone->addressOfNurseryPosition(),
+ zone->addressOfNurseryCurrentEnd(), JS::TraceKind::Object,
+ totalSize, allocSite);
+
+ if (nDynamicSlots) {
+ store32(Imm32(nDynamicSlots),
+ Address(result, thingSize + ObjectSlots::offsetOfCapacity()));
+ store32(
+ Imm32(0),
+ Address(result, thingSize + ObjectSlots::offsetOfDictionarySlotSpan()));
+ computeEffectiveAddress(
+ Address(result, thingSize + ObjectSlots::offsetOfSlots()), temp);
+ storePtr(temp, Address(result, NativeObject::offsetOfSlots()));
+ }
+}
+
+// Inlined version of FreeSpan::allocate. This does not fill in slots_.
+void MacroAssembler::freeListAllocate(Register result, Register temp,
+ gc::AllocKind allocKind, Label* fail) {
+ CompileZone* zone = realm()->zone();
+ int thingSize = int(gc::Arena::thingSize(allocKind));
+
+ Label fallback;
+ Label success;
+
+ // Load the first and last offsets of |zone|'s free list for |allocKind|.
+ // If there is no room remaining in the span, fall back to get the next one.
+ gc::FreeSpan** ptrFreeList = zone->addressOfFreeList(allocKind);
+ loadPtr(AbsoluteAddress(ptrFreeList), temp);
+ load16ZeroExtend(Address(temp, js::gc::FreeSpan::offsetOfFirst()), result);
+ load16ZeroExtend(Address(temp, js::gc::FreeSpan::offsetOfLast()), temp);
+ branch32(Assembler::AboveOrEqual, result, temp, &fallback);
+
+ // Bump the offset for the next allocation.
+ add32(Imm32(thingSize), result);
+ loadPtr(AbsoluteAddress(ptrFreeList), temp);
+ store16(result, Address(temp, js::gc::FreeSpan::offsetOfFirst()));
+ sub32(Imm32(thingSize), result);
+ addPtr(temp, result); // Turn the offset into a pointer.
+ jump(&success);
+
+ bind(&fallback);
+ // If there are no free spans left, we bail to finish the allocation. The
+ // interpreter will call the GC allocator to set up a new arena to allocate
+ // from, after which we can resume allocating in the jit.
+ branchTest32(Assembler::Zero, result, result, fail);
+ loadPtr(AbsoluteAddress(ptrFreeList), temp);
+ addPtr(temp, result); // Turn the offset into a pointer.
+ Push(result);
+ // Update the free list to point to the next span (which may be empty).
+ load32(Address(result, 0), result);
+ store32(result, Address(temp, js::gc::FreeSpan::offsetOfFirst()));
+ Pop(result);
+
+ bind(&success);
+
+ if (runtime()->geckoProfiler().enabled()) {
+ uint32_t* countAddress = zone->addressOfTenuredAllocCount();
+ movePtr(ImmPtr(countAddress), temp);
+ add32(Imm32(1), Address(temp, 0));
+ }
+}
+
+void MacroAssembler::callFreeStub(Register slots) {
+ // This register must match the one in JitRuntime::generateFreeStub.
+ const Register regSlots = CallTempReg0;
+
+ push(regSlots);
+ movePtr(slots, regSlots);
+ call(runtime()->jitRuntime()->freeStub());
+ pop(regSlots);
+}
+
+// Inlined equivalent of gc::AllocateObject, without failure case handling.
+void MacroAssembler::allocateObject(Register result, Register temp,
+ gc::AllocKind allocKind,
+ uint32_t nDynamicSlots,
+ gc::InitialHeap initialHeap, Label* fail,
+ const AllocSiteInput& allocSite) {
+ MOZ_ASSERT(gc::IsObjectAllocKind(allocKind));
+
+ checkAllocatorState(fail);
+
+ if (shouldNurseryAllocate(allocKind, initialHeap)) {
+ MOZ_ASSERT(initialHeap == gc::DefaultHeap);
+ return nurseryAllocateObject(result, temp, allocKind, nDynamicSlots, fail,
+ allocSite);
+ }
+
+ // Fall back to calling into the VM to allocate objects in the tenured heap
+ // that have dynamic slots.
+ if (nDynamicSlots) {
+ jump(fail);
+ return;
+ }
+
+ return freeListAllocate(result, temp, allocKind, fail);
+}
+
+void MacroAssembler::createGCObject(Register obj, Register temp,
+ const TemplateObject& templateObj,
+ gc::InitialHeap initialHeap, Label* fail,
+ bool initContents /* = true */) {
+ gc::AllocKind allocKind = templateObj.getAllocKind();
+ MOZ_ASSERT(gc::IsObjectAllocKind(allocKind));
+
+ uint32_t nDynamicSlots = 0;
+ if (templateObj.isNativeObject()) {
+ const TemplateNativeObject& ntemplate =
+ templateObj.asTemplateNativeObject();
+ nDynamicSlots = ntemplate.numDynamicSlots();
+ }
+
+ allocateObject(obj, temp, allocKind, nDynamicSlots, initialHeap, fail);
+ initGCThing(obj, temp, templateObj, initContents);
+}
+
+void MacroAssembler::createPlainGCObject(
+ Register result, Register shape, Register temp, Register temp2,
+ uint32_t numFixedSlots, uint32_t numDynamicSlots, gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap, Label* fail, const AllocSiteInput& allocSite,
+ bool initContents /* = true */) {
+ MOZ_ASSERT(gc::IsObjectAllocKind(allocKind));
+ MOZ_ASSERT(shape != temp, "shape can overlap with temp2, but not temp");
+
+ // Allocate object.
+ allocateObject(result, temp, allocKind, numDynamicSlots, initialHeap, fail,
+ allocSite);
+
+ // Initialize shape field.
+ storePtr(shape, Address(result, JSObject::offsetOfShape()));
+
+ // If the object has dynamic slots, allocateObject will initialize
+ // the slots field. If not, we must initialize it now.
+ if (numDynamicSlots == 0) {
+ storePtr(ImmPtr(emptyObjectSlots),
+ Address(result, NativeObject::offsetOfSlots()));
+ }
+
+ // Initialize elements field.
+ storePtr(ImmPtr(emptyObjectElements),
+ Address(result, NativeObject::offsetOfElements()));
+
+ // Initialize fixed slots.
+ if (initContents) {
+ fillSlotsWithUndefined(Address(result, NativeObject::getFixedSlotOffset(0)),
+ temp, 0, numFixedSlots);
+ }
+
+ // Initialize dynamic slots.
+ if (numDynamicSlots > 0) {
+ loadPtr(Address(result, NativeObject::offsetOfSlots()), temp2);
+ fillSlotsWithUndefined(Address(temp2, 0), temp, 0, numDynamicSlots);
+ }
+}
+
+void MacroAssembler::createArrayWithFixedElements(
+ Register result, Register shape, Register temp, uint32_t arrayLength,
+ uint32_t arrayCapacity, gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap, Label* fail, const AllocSiteInput& allocSite) {
+ MOZ_ASSERT(gc::IsObjectAllocKind(allocKind));
+ MOZ_ASSERT(shape != temp, "shape can overlap with temp2, but not temp");
+ MOZ_ASSERT(result != temp);
+
+ // This only supports allocating arrays with fixed elements and does not
+ // support any dynamic slots or elements.
+ MOZ_ASSERT(arrayCapacity >= arrayLength);
+ MOZ_ASSERT(gc::GetGCKindSlots(allocKind) >=
+ arrayCapacity + ObjectElements::VALUES_PER_HEADER);
+
+ // Allocate object.
+ allocateObject(result, temp, allocKind, 0, initialHeap, fail, allocSite);
+
+ // Initialize shape field.
+ storePtr(shape, Address(result, JSObject::offsetOfShape()));
+
+ // There are no dynamic slots.
+ storePtr(ImmPtr(emptyObjectSlots),
+ Address(result, NativeObject::offsetOfSlots()));
+
+ // Initialize elements pointer for fixed (inline) elements.
+ computeEffectiveAddress(
+ Address(result, NativeObject::offsetOfFixedElements()), temp);
+ storePtr(temp, Address(result, NativeObject::offsetOfElements()));
+
+ // Initialize elements header.
+ store32(Imm32(ObjectElements::FIXED),
+ Address(temp, ObjectElements::offsetOfFlags()));
+ store32(Imm32(0), Address(temp, ObjectElements::offsetOfInitializedLength()));
+ store32(Imm32(arrayCapacity),
+ Address(temp, ObjectElements::offsetOfCapacity()));
+ store32(Imm32(arrayLength), Address(temp, ObjectElements::offsetOfLength()));
+}
+
+// Inline version of Nursery::allocateString.
+void MacroAssembler::nurseryAllocateString(Register result, Register temp,
+ gc::AllocKind allocKind,
+ Label* fail) {
+ MOZ_ASSERT(IsNurseryAllocable(allocKind));
+
+ // No explicit check for nursery.isEnabled() is needed, as the comparison
+ // with the nursery's end will always fail in such cases.
+
+ CompileZone* zone = realm()->zone();
+ uint64_t* allocStrsPtr = &zone->zone()->nurseryAllocatedStrings.ref();
+ inc64(AbsoluteAddress(allocStrsPtr));
+ size_t thingSize = gc::Arena::thingSize(allocKind);
+
+ bumpPointerAllocate(result, temp, fail, zone,
+ zone->addressOfStringNurseryPosition(),
+ zone->addressOfStringNurseryCurrentEnd(),
+ JS::TraceKind::String, thingSize);
+}
+
+// Inline version of Nursery::allocateBigInt.
+void MacroAssembler::nurseryAllocateBigInt(Register result, Register temp,
+ Label* fail) {
+ MOZ_ASSERT(IsNurseryAllocable(gc::AllocKind::BIGINT));
+
+ // No explicit check for nursery.isEnabled() is needed, as the comparison
+ // with the nursery's end will always fail in such cases.
+
+ CompileZone* zone = realm()->zone();
+ size_t thingSize = gc::Arena::thingSize(gc::AllocKind::BIGINT);
+
+ bumpPointerAllocate(result, temp, fail, zone,
+ zone->addressOfBigIntNurseryPosition(),
+ zone->addressOfBigIntNurseryCurrentEnd(),
+ JS::TraceKind::BigInt, thingSize);
+}
+
+void MacroAssembler::bumpPointerAllocate(Register result, Register temp,
+ Label* fail, CompileZone* zone,
+ void* posAddr, const void* curEndAddr,
+ JS::TraceKind traceKind, uint32_t size,
+ const AllocSiteInput& allocSite) {
+ MOZ_ASSERT(size >= gc::MinCellSize);
+
+ uint32_t totalSize = size + Nursery::nurseryCellHeaderSize();
+ MOZ_ASSERT(totalSize < INT32_MAX, "Nursery allocation too large");
+ MOZ_ASSERT(totalSize % gc::CellAlignBytes == 0);
+
+ // The position (allocation pointer) and the end pointer are stored
+ // very close to each other -- specifically, easily within a 32 bit offset.
+ // Use relative offsets between them, to avoid 64-bit immediate loads.
+ //
+ // I tried to optimise this further by using an extra register to avoid
+ // the final subtraction and hopefully get some more instruction
+ // parallelism, but it made no difference.
+ movePtr(ImmPtr(posAddr), temp);
+ loadPtr(Address(temp, 0), result);
+ addPtr(Imm32(totalSize), result);
+ CheckedInt<int32_t> endOffset =
+ (CheckedInt<uintptr_t>(uintptr_t(curEndAddr)) -
+ CheckedInt<uintptr_t>(uintptr_t(posAddr)))
+ .toChecked<int32_t>();
+ MOZ_ASSERT(endOffset.isValid(), "Position and end pointers must be nearby");
+ branchPtr(Assembler::Below, Address(temp, endOffset.value()), result, fail);
+ storePtr(result, Address(temp, 0));
+ subPtr(Imm32(size), result);
+
+ if (runtime()->geckoProfiler().enabled()) {
+ uint32_t* countAddress = zone->addressOfNurseryAllocCount();
+ CheckedInt<int32_t> counterOffset =
+ (CheckedInt<uintptr_t>(uintptr_t(countAddress)) -
+ CheckedInt<uintptr_t>(uintptr_t(posAddr)))
+ .toChecked<int32_t>();
+ if (counterOffset.isValid()) {
+ add32(Imm32(1), Address(temp, counterOffset.value()));
+ } else {
+ movePtr(ImmPtr(countAddress), temp);
+ add32(Imm32(1), Address(temp, 0));
+ }
+ }
+
+ if (allocSite.is<gc::CatchAllAllocSite>()) {
+ // No allocation site supplied. This is the case when called from Warp, or
+ // from places that don't support pretenuring.
+ gc::CatchAllAllocSite siteKind = allocSite.as<gc::CatchAllAllocSite>();
+ storePtr(ImmWord(zone->nurseryCellHeader(traceKind, siteKind)),
+ Address(result, -js::Nursery::nurseryCellHeaderSize()));
+ } else {
+ // Update allocation site and store pointer in the nursery cell header. This
+ // is only used from baseline.
+ Register site = allocSite.as<Register>();
+ updateAllocSite(temp, result, zone, site);
+ // See NurseryCellHeader::MakeValue.
+ orPtr(Imm32(int32_t(traceKind)), site);
+ storePtr(site, Address(result, -js::Nursery::nurseryCellHeaderSize()));
+ }
+}
+
+// Update the allocation site in the same way as Nursery::allocateCell.
+void MacroAssembler::updateAllocSite(Register temp, Register result,
+ CompileZone* zone, Register site) {
+ Label done;
+
+ add32(Imm32(1), Address(site, gc::AllocSite::offsetOfNurseryAllocCount()));
+
+ branchPtr(Assembler::NotEqual,
+ Address(site, gc::AllocSite::offsetOfNextNurseryAllocated()),
+ ImmPtr(nullptr), &done);
+
+ loadPtr(AbsoluteAddress(zone->addressOfNurseryAllocatedSites()), temp);
+ storePtr(temp, Address(site, gc::AllocSite::offsetOfNextNurseryAllocated()));
+ storePtr(site, AbsoluteAddress(zone->addressOfNurseryAllocatedSites()));
+
+ bind(&done);
+}
+
+// Inlined equivalent of gc::AllocateString, jumping to fail if nursery
+// allocation requested but unsuccessful.
+void MacroAssembler::allocateString(Register result, Register temp,
+ gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap, Label* fail) {
+ MOZ_ASSERT(allocKind == gc::AllocKind::STRING ||
+ allocKind == gc::AllocKind::FAT_INLINE_STRING);
+
+ checkAllocatorState(fail);
+
+ if (shouldNurseryAllocate(allocKind, initialHeap)) {
+ MOZ_ASSERT(initialHeap == gc::DefaultHeap);
+ return nurseryAllocateString(result, temp, allocKind, fail);
+ }
+
+ freeListAllocate(result, temp, allocKind, fail);
+}
+
+void MacroAssembler::newGCString(Register result, Register temp,
+ gc::InitialHeap initialHeap, Label* fail) {
+ allocateString(result, temp, js::gc::AllocKind::STRING, initialHeap, fail);
+}
+
+void MacroAssembler::newGCFatInlineString(Register result, Register temp,
+ gc::InitialHeap initialHeap,
+ Label* fail) {
+ allocateString(result, temp, js::gc::AllocKind::FAT_INLINE_STRING,
+ initialHeap, fail);
+}
+
+void MacroAssembler::newGCBigInt(Register result, Register temp,
+ gc::InitialHeap initialHeap, Label* fail) {
+ checkAllocatorState(fail);
+
+ if (shouldNurseryAllocate(gc::AllocKind::BIGINT, initialHeap)) {
+ MOZ_ASSERT(initialHeap == gc::DefaultHeap);
+ return nurseryAllocateBigInt(result, temp, fail);
+ }
+
+ freeListAllocate(result, temp, gc::AllocKind::BIGINT, fail);
+}
+
+void MacroAssembler::copySlotsFromTemplate(
+ Register obj, const TemplateNativeObject& templateObj, uint32_t start,
+ uint32_t end) {
+ uint32_t nfixed = std::min(templateObj.numFixedSlots(), end);
+ for (unsigned i = start; i < nfixed; i++) {
+ // Template objects are not exposed to script and therefore immutable.
+ // However, regexp template objects are sometimes used directly (when
+ // the cloning is not observable), and therefore we can end up with a
+ // non-zero lastIndex. Detect this case here and just substitute 0, to
+ // avoid racing with the main thread updating this slot.
+ Value v;
+ if (templateObj.isRegExpObject() && i == RegExpObject::lastIndexSlot()) {
+ v = Int32Value(0);
+ } else {
+ v = templateObj.getSlot(i);
+ }
+ storeValue(v, Address(obj, NativeObject::getFixedSlotOffset(i)));
+ }
+}
+
+void MacroAssembler::fillSlotsWithConstantValue(Address base, Register temp,
+ uint32_t start, uint32_t end,
+ const Value& v) {
+ MOZ_ASSERT(v.isUndefined() || IsUninitializedLexical(v));
+
+ if (start >= end) {
+ return;
+ }
+
+#ifdef JS_NUNBOX32
+ // We only have a single spare register, so do the initialization as two
+ // strided writes of the tag and body.
+ Address addr = base;
+ move32(Imm32(v.toNunboxPayload()), temp);
+ for (unsigned i = start; i < end; ++i, addr.offset += sizeof(GCPtr<Value>)) {
+ store32(temp, ToPayload(addr));
+ }
+
+ addr = base;
+ move32(Imm32(v.toNunboxTag()), temp);
+ for (unsigned i = start; i < end; ++i, addr.offset += sizeof(GCPtr<Value>)) {
+ store32(temp, ToType(addr));
+ }
+#else
+ moveValue(v, ValueOperand(temp));
+ for (uint32_t i = start; i < end; ++i, base.offset += sizeof(GCPtr<Value>)) {
+ storePtr(temp, base);
+ }
+#endif
+}
+
+void MacroAssembler::fillSlotsWithUndefined(Address base, Register temp,
+ uint32_t start, uint32_t end) {
+ fillSlotsWithConstantValue(base, temp, start, end, UndefinedValue());
+}
+
+void MacroAssembler::fillSlotsWithUninitialized(Address base, Register temp,
+ uint32_t start, uint32_t end) {
+ fillSlotsWithConstantValue(base, temp, start, end,
+ MagicValue(JS_UNINITIALIZED_LEXICAL));
+}
+
+static std::pair<uint32_t, uint32_t> FindStartOfUninitializedAndUndefinedSlots(
+ const TemplateNativeObject& templateObj, uint32_t nslots) {
+ MOZ_ASSERT(nslots == templateObj.slotSpan());
+ MOZ_ASSERT(nslots > 0);
+
+ uint32_t first = nslots;
+ for (; first != 0; --first) {
+ if (templateObj.getSlot(first - 1) != UndefinedValue()) {
+ break;
+ }
+ }
+ uint32_t startOfUndefined = first;
+
+ if (first != 0 && IsUninitializedLexical(templateObj.getSlot(first - 1))) {
+ for (; first != 0; --first) {
+ if (!IsUninitializedLexical(templateObj.getSlot(first - 1))) {
+ break;
+ }
+ }
+ }
+ uint32_t startOfUninitialized = first;
+
+ return {startOfUninitialized, startOfUndefined};
+}
+
+void MacroAssembler::initTypedArraySlots(Register obj, Register temp,
+ Register lengthReg,
+ LiveRegisterSet liveRegs, Label* fail,
+ TypedArrayObject* templateObj,
+ TypedArrayLength lengthKind) {
+ MOZ_ASSERT(!templateObj->hasBuffer());
+
+ constexpr size_t dataSlotOffset = ArrayBufferViewObject::dataOffset();
+ constexpr size_t dataOffset = dataSlotOffset + sizeof(HeapSlot);
+
+ static_assert(
+ TypedArrayObject::FIXED_DATA_START == TypedArrayObject::DATA_SLOT + 1,
+ "fixed inline element data assumed to begin after the data slot");
+
+ static_assert(
+ TypedArrayObject::INLINE_BUFFER_LIMIT ==
+ JSObject::MAX_BYTE_SIZE - dataOffset,
+ "typed array inline buffer is limited by the maximum object byte size");
+
+ // Initialise data elements to zero.
+ size_t length = templateObj->length();
+ MOZ_ASSERT(length <= INT32_MAX,
+ "Template objects are only created for int32 lengths");
+ size_t nbytes = length * templateObj->bytesPerElement();
+
+ if (lengthKind == TypedArrayLength::Fixed &&
+ nbytes <= TypedArrayObject::INLINE_BUFFER_LIMIT) {
+ MOZ_ASSERT(dataOffset + nbytes <= templateObj->tenuredSizeOfThis());
+
+ // Store data elements inside the remaining JSObject slots.
+ computeEffectiveAddress(Address(obj, dataOffset), temp);
+ storePrivateValue(temp, Address(obj, dataSlotOffset));
+
+ // Write enough zero pointers into fixed data to zero every
+ // element. (This zeroes past the end of a byte count that's
+ // not a multiple of pointer size. That's okay, because fixed
+ // data is a count of 8-byte HeapSlots (i.e. <= pointer size),
+ // and we won't inline unless the desired memory fits in that
+ // space.)
+ static_assert(sizeof(HeapSlot) == 8, "Assumed 8 bytes alignment");
+
+ size_t numZeroPointers = ((nbytes + 7) & ~0x7) / sizeof(char*);
+ for (size_t i = 0; i < numZeroPointers; i++) {
+ storePtr(ImmWord(0), Address(obj, dataOffset + i * sizeof(char*)));
+ }
+ MOZ_ASSERT(nbytes > 0, "Zero-length TypedArrays need ZeroLengthArrayData");
+ } else {
+ if (lengthKind == TypedArrayLength::Fixed) {
+ move32(Imm32(length), lengthReg);
+ }
+
+ // Ensure volatile |obj| is saved across the call.
+ if (obj.volatile_()) {
+ liveRegs.addUnchecked(obj);
+ }
+
+ // Allocate a buffer on the heap to store the data elements.
+ PushRegsInMask(liveRegs);
+ using Fn = void (*)(JSContext * cx, TypedArrayObject * obj, int32_t count);
+ setupUnalignedABICall(temp);
+ loadJSContext(temp);
+ passABIArg(temp);
+ passABIArg(obj);
+ passABIArg(lengthReg);
+ callWithABI<Fn, AllocateAndInitTypedArrayBuffer>();
+ PopRegsInMask(liveRegs);
+
+ // Fail when data slot is UndefinedValue.
+ branchTestUndefined(Assembler::Equal, Address(obj, dataSlotOffset), fail);
+ }
+}
+
+void MacroAssembler::initGCSlots(Register obj, Register temp,
+ const TemplateNativeObject& templateObj) {
+ MOZ_ASSERT(!templateObj.isArrayObject());
+
+ // Slots of non-array objects are required to be initialized.
+ // Use the values currently in the template object.
+ uint32_t nslots = templateObj.slotSpan();
+ if (nslots == 0) {
+ return;
+ }
+
+ uint32_t nfixed = templateObj.numUsedFixedSlots();
+ uint32_t ndynamic = templateObj.numDynamicSlots();
+
+ // Attempt to group slot writes such that we minimize the amount of
+ // duplicated data we need to embed in code and load into registers. In
+ // general, most template object slots will be undefined except for any
+ // reserved slots. Since reserved slots come first, we split the object
+ // logically into independent non-UndefinedValue writes to the head and
+ // duplicated writes of UndefinedValue to the tail. For the majority of
+ // objects, the "tail" will be the entire slot range.
+ //
+ // The template object may be a CallObject, in which case we need to
+ // account for uninitialized lexical slots as well as undefined
+ // slots. Uninitialized lexical slots appears in CallObjects if the function
+ // has parameter expressions, in which case closed over parameters have
+ // TDZ. Uninitialized slots come before undefined slots in CallObjects.
+ auto [startOfUninitialized, startOfUndefined] =
+ FindStartOfUninitializedAndUndefinedSlots(templateObj, nslots);
+ MOZ_ASSERT(startOfUninitialized <= nfixed); // Reserved slots must be fixed.
+ MOZ_ASSERT(startOfUndefined >= startOfUninitialized);
+ MOZ_ASSERT_IF(!templateObj.isCallObject() &&
+ !templateObj.isBlockLexicalEnvironmentObject(),
+ startOfUninitialized == startOfUndefined);
+
+ // Copy over any preserved reserved slots.
+ copySlotsFromTemplate(obj, templateObj, 0, startOfUninitialized);
+
+ // Fill the rest of the fixed slots with undefined and uninitialized.
+ size_t offset = NativeObject::getFixedSlotOffset(startOfUninitialized);
+ fillSlotsWithUninitialized(Address(obj, offset), temp, startOfUninitialized,
+ std::min(startOfUndefined, nfixed));
+
+ if (startOfUndefined < nfixed) {
+ offset = NativeObject::getFixedSlotOffset(startOfUndefined);
+ fillSlotsWithUndefined(Address(obj, offset), temp, startOfUndefined,
+ nfixed);
+ }
+
+ if (ndynamic) {
+ // We are short one register to do this elegantly. Borrow the obj
+ // register briefly for our slots base address.
+ push(obj);
+ loadPtr(Address(obj, NativeObject::offsetOfSlots()), obj);
+
+ // Fill uninitialized slots if necessary. Otherwise initialize all
+ // slots to undefined.
+ if (startOfUndefined > nfixed) {
+ MOZ_ASSERT(startOfUninitialized != startOfUndefined);
+ fillSlotsWithUninitialized(Address(obj, 0), temp, 0,
+ startOfUndefined - nfixed);
+ size_t offset = (startOfUndefined - nfixed) * sizeof(Value);
+ fillSlotsWithUndefined(Address(obj, offset), temp,
+ startOfUndefined - nfixed, ndynamic);
+ } else {
+ fillSlotsWithUndefined(Address(obj, 0), temp, 0, ndynamic);
+ }
+
+ pop(obj);
+ }
+}
+
+void MacroAssembler::initGCThing(Register obj, Register temp,
+ const TemplateObject& templateObj,
+ bool initContents) {
+ // Fast initialization of an empty object returned by allocateObject().
+
+ storePtr(ImmGCPtr(templateObj.shape()),
+ Address(obj, JSObject::offsetOfShape()));
+
+ if (templateObj.isNativeObject()) {
+ const TemplateNativeObject& ntemplate =
+ templateObj.asTemplateNativeObject();
+ MOZ_ASSERT(!ntemplate.hasDynamicElements());
+
+ // If the object has dynamic slots, the slots member has already been
+ // filled in.
+ if (!ntemplate.hasDynamicSlots()) {
+ storePtr(ImmPtr(emptyObjectSlots),
+ Address(obj, NativeObject::offsetOfSlots()));
+ }
+
+ if (ntemplate.isArrayObject()) {
+ // Can't skip initializing reserved slots.
+ MOZ_ASSERT(initContents);
+
+ int elementsOffset = NativeObject::offsetOfFixedElements();
+
+ computeEffectiveAddress(Address(obj, elementsOffset), temp);
+ storePtr(temp, Address(obj, NativeObject::offsetOfElements()));
+
+ // Fill in the elements header.
+ store32(
+ Imm32(ntemplate.getDenseCapacity()),
+ Address(obj, elementsOffset + ObjectElements::offsetOfCapacity()));
+ store32(Imm32(ntemplate.getDenseInitializedLength()),
+ Address(obj, elementsOffset +
+ ObjectElements::offsetOfInitializedLength()));
+ store32(Imm32(ntemplate.getArrayLength()),
+ Address(obj, elementsOffset + ObjectElements::offsetOfLength()));
+ store32(Imm32(ObjectElements::FIXED),
+ Address(obj, elementsOffset + ObjectElements::offsetOfFlags()));
+ } else if (ntemplate.isArgumentsObject()) {
+ // The caller will initialize the reserved slots.
+ MOZ_ASSERT(!initContents);
+ storePtr(ImmPtr(emptyObjectElements),
+ Address(obj, NativeObject::offsetOfElements()));
+ } else {
+ // If the target type could be a TypedArray that maps shared memory
+ // then this would need to store emptyObjectElementsShared in that case.
+ MOZ_ASSERT(!ntemplate.isSharedMemory());
+
+ // Can't skip initializing reserved slots.
+ MOZ_ASSERT(initContents);
+
+ storePtr(ImmPtr(emptyObjectElements),
+ Address(obj, NativeObject::offsetOfElements()));
+
+ initGCSlots(obj, temp, ntemplate);
+ }
+ } else {
+ MOZ_CRASH("Unknown object");
+ }
+
+#ifdef JS_GC_PROBES
+ AllocatableRegisterSet regs(RegisterSet::Volatile());
+ LiveRegisterSet save(regs.asLiveSet());
+ PushRegsInMask(save);
+
+ regs.takeUnchecked(obj);
+ Register temp2 = regs.takeAnyGeneral();
+
+ using Fn = void (*)(JSObject * obj);
+ setupUnalignedABICall(temp2);
+ passABIArg(obj);
+ callWithABI<Fn, TraceCreateObject>();
+
+ PopRegsInMask(save);
+#endif
+}
+
+void MacroAssembler::compareStrings(JSOp op, Register left, Register right,
+ Register result, Label* fail) {
+ MOZ_ASSERT(left != result);
+ MOZ_ASSERT(right != result);
+ MOZ_ASSERT(IsEqualityOp(op) || IsRelationalOp(op));
+
+ Label notPointerEqual;
+ // If operands point to the same instance, the strings are trivially equal.
+ branchPtr(Assembler::NotEqual, left, right,
+ IsEqualityOp(op) ? &notPointerEqual : fail);
+ move32(Imm32(op == JSOp::Eq || op == JSOp::StrictEq || op == JSOp::Le ||
+ op == JSOp::Ge),
+ result);
+
+ if (IsEqualityOp(op)) {
+ Label done;
+ jump(&done);
+
+ bind(&notPointerEqual);
+
+ Label leftIsNotAtom;
+ Label setNotEqualResult;
+ // Atoms cannot be equal to each other if they point to different strings.
+ Imm32 atomBit(JSString::ATOM_BIT);
+ branchTest32(Assembler::Zero, Address(left, JSString::offsetOfFlags()),
+ atomBit, &leftIsNotAtom);
+ branchTest32(Assembler::NonZero, Address(right, JSString::offsetOfFlags()),
+ atomBit, &setNotEqualResult);
+
+ bind(&leftIsNotAtom);
+ // Strings of different length can never be equal.
+ loadStringLength(left, result);
+ branch32(Assembler::Equal, Address(right, JSString::offsetOfLength()),
+ result, fail);
+
+ bind(&setNotEqualResult);
+ move32(Imm32(op == JSOp::Ne || op == JSOp::StrictNe), result);
+
+ bind(&done);
+ }
+}
+
+void MacroAssembler::loadStringChars(Register str, Register dest,
+ CharEncoding encoding) {
+ MOZ_ASSERT(str != dest);
+
+ if (JitOptions.spectreStringMitigations) {
+ if (encoding == CharEncoding::Latin1) {
+ // If the string is a rope, zero the |str| register. The code below
+ // depends on str->flags so this should block speculative execution.
+ movePtr(ImmWord(0), dest);
+ test32MovePtr(Assembler::Zero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::LINEAR_BIT), dest, str);
+ } else {
+ // If we're loading TwoByte chars, there's an additional risk:
+ // if the string has Latin1 chars, we could read out-of-bounds. To
+ // prevent this, we check both the Linear and Latin1 bits. We don't
+ // have a scratch register, so we use these flags also to block
+ // speculative execution, similar to the use of 0 above.
+ MOZ_ASSERT(encoding == CharEncoding::TwoByte);
+ static constexpr uint32_t Mask =
+ JSString::LINEAR_BIT | JSString::LATIN1_CHARS_BIT;
+ static_assert(Mask < 1024,
+ "Mask should be a small, near-null value to ensure we "
+ "block speculative execution when it's used as string "
+ "pointer");
+ move32(Imm32(Mask), dest);
+ and32(Address(str, JSString::offsetOfFlags()), dest);
+ cmp32MovePtr(Assembler::NotEqual, dest, Imm32(JSString::LINEAR_BIT), dest,
+ str);
+ }
+ }
+
+ // Load the inline chars.
+ computeEffectiveAddress(Address(str, JSInlineString::offsetOfInlineStorage()),
+ dest);
+
+ // If it's not an inline string, load the non-inline chars. Use a
+ // conditional move to prevent speculative execution.
+ test32LoadPtr(Assembler::Zero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::INLINE_CHARS_BIT),
+ Address(str, JSString::offsetOfNonInlineChars()), dest);
+}
+
+void MacroAssembler::loadNonInlineStringChars(Register str, Register dest,
+ CharEncoding encoding) {
+ MOZ_ASSERT(str != dest);
+
+ if (JitOptions.spectreStringMitigations) {
+ // If the string is a rope, has inline chars, or has a different
+ // character encoding, set str to a near-null value to prevent
+ // speculative execution below (when reading str->nonInlineChars).
+
+ static constexpr uint32_t Mask = JSString::LINEAR_BIT |
+ JSString::INLINE_CHARS_BIT |
+ JSString::LATIN1_CHARS_BIT;
+ static_assert(Mask < 1024,
+ "Mask should be a small, near-null value to ensure we "
+ "block speculative execution when it's used as string "
+ "pointer");
+
+ uint32_t expectedBits = JSString::LINEAR_BIT;
+ if (encoding == CharEncoding::Latin1) {
+ expectedBits |= JSString::LATIN1_CHARS_BIT;
+ }
+
+ move32(Imm32(Mask), dest);
+ and32(Address(str, JSString::offsetOfFlags()), dest);
+
+ cmp32MovePtr(Assembler::NotEqual, dest, Imm32(expectedBits), dest, str);
+ }
+
+ loadPtr(Address(str, JSString::offsetOfNonInlineChars()), dest);
+}
+
+void MacroAssembler::storeNonInlineStringChars(Register chars, Register str) {
+ MOZ_ASSERT(chars != str);
+ storePtr(chars, Address(str, JSString::offsetOfNonInlineChars()));
+}
+
+void MacroAssembler::loadInlineStringCharsForStore(Register str,
+ Register dest) {
+ computeEffectiveAddress(Address(str, JSInlineString::offsetOfInlineStorage()),
+ dest);
+}
+
+void MacroAssembler::loadInlineStringChars(Register str, Register dest,
+ CharEncoding encoding) {
+ MOZ_ASSERT(str != dest);
+
+ if (JitOptions.spectreStringMitigations) {
+ // Making this Spectre-safe is a bit complicated: using
+ // computeEffectiveAddress and then zeroing the output register if
+ // non-inline is not sufficient: when the index is very large, it would
+ // allow reading |nullptr + index|. Just fall back to loadStringChars
+ // for now.
+ loadStringChars(str, dest, encoding);
+ } else {
+ computeEffectiveAddress(
+ Address(str, JSInlineString::offsetOfInlineStorage()), dest);
+ }
+}
+
+void MacroAssembler::loadRopeLeftChild(Register str, Register dest) {
+ MOZ_ASSERT(str != dest);
+
+ if (JitOptions.spectreStringMitigations) {
+ // Zero the output register if the input was not a rope.
+ movePtr(ImmWord(0), dest);
+ test32LoadPtr(Assembler::Zero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::LINEAR_BIT),
+ Address(str, JSRope::offsetOfLeft()), dest);
+ } else {
+ loadPtr(Address(str, JSRope::offsetOfLeft()), dest);
+ }
+}
+
+void MacroAssembler::loadRopeRightChild(Register str, Register dest) {
+ MOZ_ASSERT(str != dest);
+
+ if (JitOptions.spectreStringMitigations) {
+ // Zero the output register if the input was not a rope.
+ movePtr(ImmWord(0), dest);
+ test32LoadPtr(Assembler::Zero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::LINEAR_BIT),
+ Address(str, JSRope::offsetOfRight()), dest);
+ } else {
+ loadPtr(Address(str, JSRope::offsetOfRight()), dest);
+ }
+}
+
+void MacroAssembler::storeRopeChildren(Register left, Register right,
+ Register str) {
+ storePtr(left, Address(str, JSRope::offsetOfLeft()));
+ storePtr(right, Address(str, JSRope::offsetOfRight()));
+}
+
+void MacroAssembler::loadDependentStringBase(Register str, Register dest) {
+ MOZ_ASSERT(str != dest);
+
+ if (JitOptions.spectreStringMitigations) {
+ // If the string is not a dependent string, zero the |str| register.
+ // The code below loads str->base so this should block speculative
+ // execution.
+ movePtr(ImmWord(0), dest);
+ test32MovePtr(Assembler::Zero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::DEPENDENT_BIT), dest, str);
+ }
+
+ loadPtr(Address(str, JSDependentString::offsetOfBase()), dest);
+}
+
+void MacroAssembler::storeDependentStringBase(Register base, Register str) {
+ storePtr(base, Address(str, JSDependentString::offsetOfBase()));
+}
+
+void MacroAssembler::loadRopeChild(Register str, Register index,
+ Register output, Label* isLinear) {
+ // This follows JSString::getChar.
+ branchIfNotRope(str, isLinear);
+
+ loadRopeLeftChild(str, output);
+
+ // Check if the index is contained in the leftChild.
+ Label loadedChild;
+ branch32(Assembler::Above, Address(output, JSString::offsetOfLength()), index,
+ &loadedChild);
+
+ // The index must be in the rightChild.
+ loadRopeRightChild(str, output);
+
+ bind(&loadedChild);
+}
+
+void MacroAssembler::branchIfCanLoadStringChar(Register str, Register index,
+ Register scratch, Label* label) {
+ loadRopeChild(str, index, scratch, label);
+
+ // Branch if the left resp. right side is linear.
+ branchIfNotRope(scratch, label);
+}
+
+void MacroAssembler::branchIfNotCanLoadStringChar(Register str, Register index,
+ Register scratch,
+ Label* label) {
+ Label done;
+ loadRopeChild(str, index, scratch, &done);
+
+ // Branch if the left or right side is another rope.
+ branchIfRope(scratch, label);
+
+ bind(&done);
+}
+
+void MacroAssembler::loadStringChar(Register str, Register index,
+ Register output, Register scratch1,
+ Register scratch2, Label* fail) {
+ MOZ_ASSERT(str != output);
+ MOZ_ASSERT(str != index);
+ MOZ_ASSERT(index != output);
+ MOZ_ASSERT(output != scratch1);
+ MOZ_ASSERT(output != scratch2);
+
+ // Use scratch1 for the index (adjusted below).
+ move32(index, scratch1);
+ movePtr(str, output);
+
+ // This follows JSString::getChar.
+ Label notRope;
+ branchIfNotRope(str, &notRope);
+
+ loadRopeLeftChild(str, output);
+
+ // Check if the index is contained in the leftChild.
+ Label loadedChild, notInLeft;
+ spectreBoundsCheck32(scratch1, Address(output, JSString::offsetOfLength()),
+ scratch2, &notInLeft);
+ jump(&loadedChild);
+
+ // The index must be in the rightChild.
+ // index -= rope->leftChild()->length()
+ bind(&notInLeft);
+ sub32(Address(output, JSString::offsetOfLength()), scratch1);
+ loadRopeRightChild(str, output);
+
+ // If the left or right side is another rope, give up.
+ bind(&loadedChild);
+ branchIfRope(output, fail);
+
+ bind(&notRope);
+
+ Label isLatin1, done;
+ // We have to check the left/right side for ropes,
+ // because a TwoByte rope might have a Latin1 child.
+ branchLatin1String(output, &isLatin1);
+ loadStringChars(output, scratch2, CharEncoding::TwoByte);
+ loadChar(scratch2, scratch1, output, CharEncoding::TwoByte);
+ jump(&done);
+
+ bind(&isLatin1);
+ loadStringChars(output, scratch2, CharEncoding::Latin1);
+ loadChar(scratch2, scratch1, output, CharEncoding::Latin1);
+
+ bind(&done);
+}
+
+void MacroAssembler::loadStringIndexValue(Register str, Register dest,
+ Label* fail) {
+ MOZ_ASSERT(str != dest);
+
+ load32(Address(str, JSString::offsetOfFlags()), dest);
+
+ // Does not have a cached index value.
+ branchTest32(Assembler::Zero, dest, Imm32(JSString::INDEX_VALUE_BIT), fail);
+
+ // Extract the index.
+ rshift32(Imm32(JSString::INDEX_VALUE_SHIFT), dest);
+}
+
+void MacroAssembler::loadChar(Register chars, Register index, Register dest,
+ CharEncoding encoding, int32_t offset /* = 0 */) {
+ if (encoding == CharEncoding::Latin1) {
+ loadChar(BaseIndex(chars, index, TimesOne, offset), dest, encoding);
+ } else {
+ loadChar(BaseIndex(chars, index, TimesTwo, offset), dest, encoding);
+ }
+}
+
+void MacroAssembler::addToCharPtr(Register chars, Register index,
+ CharEncoding encoding) {
+ if (encoding == CharEncoding::Latin1) {
+ static_assert(sizeof(char) == 1,
+ "Latin-1 string index shouldn't need scaling");
+ addPtr(index, chars);
+ } else {
+ computeEffectiveAddress(BaseIndex(chars, index, TimesTwo), chars);
+ }
+}
+
+void MacroAssembler::loadStringFromUnit(Register unit, Register dest,
+ const StaticStrings& staticStrings) {
+ movePtr(ImmPtr(&staticStrings.unitStaticTable), dest);
+ loadPtr(BaseIndex(dest, unit, ScalePointer), dest);
+}
+
+void MacroAssembler::loadLengthTwoString(Register c1, Register c2,
+ Register dest,
+ const StaticStrings& staticStrings) {
+ // Compute (toSmallCharTable[c1] << SMALL_CHAR_BITS) + toSmallCharTable[c2]
+ // to obtain the index into `StaticStrings::length2StaticTable`.
+ static_assert(sizeof(StaticStrings::SmallChar) == 1);
+
+ movePtr(ImmPtr(&StaticStrings::toSmallCharTable.storage), dest);
+ load8ZeroExtend(BaseIndex(dest, c1, Scale::TimesOne), c1);
+ load8ZeroExtend(BaseIndex(dest, c2, Scale::TimesOne), c2);
+
+ lshift32(Imm32(StaticStrings::SMALL_CHAR_BITS), c1);
+ add32(c2, c1);
+
+ // Look up the string from the computed index.
+ movePtr(ImmPtr(&staticStrings.length2StaticTable), dest);
+ loadPtr(BaseIndex(dest, c1, ScalePointer), dest);
+}
+
+void MacroAssembler::loadInt32ToStringWithBase(
+ Register input, Register base, Register dest, Register scratch1,
+ Register scratch2, const StaticStrings& staticStrings,
+ const LiveRegisterSet& volatileRegs, Label* fail) {
+#ifdef DEBUG
+ Label baseBad, baseOk;
+ branch32(Assembler::LessThan, base, Imm32(2), &baseBad);
+ branch32(Assembler::LessThanOrEqual, base, Imm32(36), &baseOk);
+ bind(&baseBad);
+ assumeUnreachable("base must be in range [2, 36]");
+ bind(&baseOk);
+#endif
+
+ // Compute |"0123456789abcdefghijklmnopqrstuvwxyz"[r]|.
+ auto toChar = [this, base](Register r) {
+#ifdef DEBUG
+ Label ok;
+ branch32(Assembler::Below, r, base, &ok);
+ assumeUnreachable("bad digit");
+ bind(&ok);
+#else
+ // Silence unused lambda capture warning.
+ (void)base;
+#endif
+
+ Label done;
+ add32(Imm32('0'), r);
+ branch32(Assembler::BelowOrEqual, r, Imm32('9'), &done);
+ add32(Imm32('a' - '0' - 10), r);
+ bind(&done);
+ };
+
+ // Perform a "unit" lookup when |unsigned(input) < unsigned(base)|.
+ Label lengthTwo, done;
+ branch32(Assembler::AboveOrEqual, input, base, &lengthTwo);
+ {
+ move32(input, scratch1);
+ toChar(scratch1);
+
+ loadStringFromUnit(scratch1, dest, staticStrings);
+
+ jump(&done);
+ }
+ bind(&lengthTwo);
+
+ // Compute |base * base|.
+ move32(base, scratch1);
+ mul32(scratch1, scratch1);
+
+ // Perform a "length2" lookup when |unsigned(input) < unsigned(base * base)|.
+ branch32(Assembler::AboveOrEqual, input, scratch1, fail);
+ {
+ // Compute |scratch1 = input / base| and |scratch2 = input % base|.
+ move32(input, scratch1);
+ flexibleDivMod32(base, scratch1, scratch2, true, volatileRegs);
+
+ // Compute the digits of the divisor and remainder.
+ toChar(scratch1);
+ toChar(scratch2);
+
+ // Look up the 2-character digit string in the small-char table.
+ loadLengthTwoString(scratch1, scratch2, dest, staticStrings);
+ }
+ bind(&done);
+}
+
+void MacroAssembler::loadInt32ToStringWithBase(
+ Register input, int32_t base, Register dest, Register scratch1,
+ Register scratch2, const StaticStrings& staticStrings, Label* fail) {
+ MOZ_ASSERT(2 <= base && base <= 36, "base must be in range [2, 36]");
+
+ // Compute |"0123456789abcdefghijklmnopqrstuvwxyz"[r]|.
+ auto toChar = [this, base](Register r) {
+#ifdef DEBUG
+ Label ok;
+ branch32(Assembler::Below, r, Imm32(base), &ok);
+ assumeUnreachable("bad digit");
+ bind(&ok);
+#endif
+
+ if (base <= 10) {
+ add32(Imm32('0'), r);
+ } else {
+ Label done;
+ add32(Imm32('0'), r);
+ branch32(Assembler::BelowOrEqual, r, Imm32('9'), &done);
+ add32(Imm32('a' - '0' - 10), r);
+ bind(&done);
+ }
+ };
+
+ // Perform a "unit" lookup when |unsigned(input) < unsigned(base)|.
+ Label lengthTwo, done;
+ branch32(Assembler::AboveOrEqual, input, Imm32(base), &lengthTwo);
+ {
+ move32(input, scratch1);
+ toChar(scratch1);
+
+ loadStringFromUnit(scratch1, dest, staticStrings);
+
+ jump(&done);
+ }
+ bind(&lengthTwo);
+
+ // Perform a "length2" lookup when |unsigned(input) < unsigned(base * base)|.
+ branch32(Assembler::AboveOrEqual, input, Imm32(base * base), fail);
+ {
+ // Compute |scratch1 = input / base| and |scratch2 = input % base|.
+ if (mozilla::IsPowerOfTwo(uint32_t(base))) {
+ uint32_t shift = mozilla::FloorLog2(base);
+
+ move32(input, scratch1);
+ rshift32(Imm32(shift), scratch1);
+
+ move32(input, scratch2);
+ and32(Imm32((uint32_t(1) << shift) - 1), scratch2);
+ } else {
+ // The following code matches CodeGenerator::visitUDivOrModConstant()
+ // for x86-shared. Also see Hacker's Delight 2nd edition, chapter 10-8
+ // "Unsigned Division by 7" for the case when |rmc.multiplier| exceeds
+ // UINT32_MAX and we need to adjust the shift amount.
+
+ auto rmc = ReciprocalMulConstants::computeUnsignedDivisionConstants(base);
+
+ // We first compute |q = (M * n) >> 32), where M = rmc.multiplier.
+ mulHighUnsigned32(Imm32(rmc.multiplier), input, scratch1);
+
+ if (rmc.multiplier > UINT32_MAX) {
+ // M >= 2^32 and shift == 0 is impossible, as d >= 2 implies that
+ // ((M * n) >> (32 + shift)) >= n > floor(n/d) whenever n >= d,
+ // contradicting the proof of correctness in computeDivisionConstants.
+ MOZ_ASSERT(rmc.shiftAmount > 0);
+ MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 33));
+
+ // Compute |t = (n - q) / 2|.
+ move32(input, scratch2);
+ sub32(scratch1, scratch2);
+ rshift32(Imm32(1), scratch2);
+
+ // Compute |t = (n - q) / 2 + q = (n + q) / 2|.
+ add32(scratch2, scratch1);
+
+ // Finish the computation |q = floor(n / d)|.
+ rshift32(Imm32(rmc.shiftAmount - 1), scratch1);
+ } else {
+ rshift32(Imm32(rmc.shiftAmount), scratch1);
+ }
+
+ // Compute the remainder from |r = n - q * d|.
+ move32(scratch1, dest);
+ mul32(Imm32(base), dest);
+ move32(input, scratch2);
+ sub32(dest, scratch2);
+ }
+
+ // Compute the digits of the divisor and remainder.
+ toChar(scratch1);
+ toChar(scratch2);
+
+ // Look up the 2-character digit string in the small-char table.
+ loadLengthTwoString(scratch1, scratch2, dest, staticStrings);
+ }
+ bind(&done);
+}
+
+void MacroAssembler::loadBigIntDigits(Register bigInt, Register digits) {
+ MOZ_ASSERT(digits != bigInt);
+
+ // Load the inline digits.
+ computeEffectiveAddress(Address(bigInt, BigInt::offsetOfInlineDigits()),
+ digits);
+
+ // If inline digits aren't used, load the heap digits. Use a conditional move
+ // to prevent speculative execution.
+ cmp32LoadPtr(Assembler::Above, Address(bigInt, BigInt::offsetOfLength()),
+ Imm32(int32_t(BigInt::inlineDigitsLength())),
+ Address(bigInt, BigInt::offsetOfHeapDigits()), digits);
+}
+
+void MacroAssembler::loadBigInt64(Register bigInt, Register64 dest) {
+ // This code follows the implementation of |BigInt::toUint64()|. We're also
+ // using it for inline callers of |BigInt::toInt64()|, which works, because
+ // all supported Jit architectures use a two's complement representation for
+ // int64 values, which means the WrapToSigned call in toInt64() is a no-op.
+
+ Label done, nonZero;
+
+ branchIfBigIntIsNonZero(bigInt, &nonZero);
+ {
+ move64(Imm64(0), dest);
+ jump(&done);
+ }
+ bind(&nonZero);
+
+#ifdef JS_PUNBOX64
+ Register digits = dest.reg;
+#else
+ Register digits = dest.high;
+#endif
+
+ loadBigIntDigits(bigInt, digits);
+
+#if JS_PUNBOX64
+ // Load the first digit into the destination register.
+ load64(Address(digits, 0), dest);
+#else
+ // Load the first digit into the destination register's low value.
+ load32(Address(digits, 0), dest.low);
+
+ // And conditionally load the second digit into the high value register.
+ Label twoDigits, digitsDone;
+ branch32(Assembler::Above, Address(bigInt, BigInt::offsetOfLength()),
+ Imm32(1), &twoDigits);
+ {
+ move32(Imm32(0), dest.high);
+ jump(&digitsDone);
+ }
+ {
+ bind(&twoDigits);
+ load32(Address(digits, sizeof(BigInt::Digit)), dest.high);
+ }
+ bind(&digitsDone);
+#endif
+
+ branchTest32(Assembler::Zero, Address(bigInt, BigInt::offsetOfFlags()),
+ Imm32(BigInt::signBitMask()), &done);
+ neg64(dest);
+
+ bind(&done);
+}
+
+void MacroAssembler::loadFirstBigIntDigitOrZero(Register bigInt,
+ Register dest) {
+ Label done, nonZero;
+ branchIfBigIntIsNonZero(bigInt, &nonZero);
+ {
+ movePtr(ImmWord(0), dest);
+ jump(&done);
+ }
+ bind(&nonZero);
+
+ loadBigIntDigits(bigInt, dest);
+
+ // Load the first digit into the destination register.
+ loadPtr(Address(dest, 0), dest);
+
+ bind(&done);
+}
+
+void MacroAssembler::loadBigInt(Register bigInt, Register dest, Label* fail) {
+ Label done, nonZero;
+ branchIfBigIntIsNonZero(bigInt, &nonZero);
+ {
+ movePtr(ImmWord(0), dest);
+ jump(&done);
+ }
+ bind(&nonZero);
+
+ loadBigIntNonZero(bigInt, dest, fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::loadBigIntNonZero(Register bigInt, Register dest,
+ Label* fail) {
+ MOZ_ASSERT(bigInt != dest);
+
+#ifdef DEBUG
+ Label nonZero;
+ branchIfBigIntIsNonZero(bigInt, &nonZero);
+ assumeUnreachable("Unexpected zero BigInt");
+ bind(&nonZero);
+#endif
+
+ branch32(Assembler::Above, Address(bigInt, BigInt::offsetOfLength()),
+ Imm32(1), fail);
+
+ static_assert(BigInt::inlineDigitsLength() > 0,
+ "Single digit BigInts use inline storage");
+
+ // Load the first inline digit into the destination register.
+ loadPtr(Address(bigInt, BigInt::offsetOfInlineDigits()), dest);
+
+ // Return as a signed pointer.
+ bigIntDigitToSignedPtr(bigInt, dest, fail);
+}
+
+void MacroAssembler::bigIntDigitToSignedPtr(Register bigInt, Register digit,
+ Label* fail) {
+ // BigInt digits are stored as absolute numbers. Take the failure path when
+ // the digit can't be stored in intptr_t.
+ branchTestPtr(Assembler::Signed, digit, digit, fail);
+
+ // Negate |dest| when the BigInt is negative.
+ Label nonNegative;
+ branchIfBigIntIsNonNegative(bigInt, &nonNegative);
+ negPtr(digit);
+ bind(&nonNegative);
+}
+
+void MacroAssembler::loadBigIntAbsolute(Register bigInt, Register dest,
+ Label* fail) {
+ MOZ_ASSERT(bigInt != dest);
+
+ branch32(Assembler::Above, Address(bigInt, BigInt::offsetOfLength()),
+ Imm32(1), fail);
+
+ static_assert(BigInt::inlineDigitsLength() > 0,
+ "Single digit BigInts use inline storage");
+
+ // Load the first inline digit into the destination register.
+ movePtr(ImmWord(0), dest);
+ cmp32LoadPtr(Assembler::NotEqual, Address(bigInt, BigInt::offsetOfLength()),
+ Imm32(0), Address(bigInt, BigInt::offsetOfInlineDigits()), dest);
+}
+
+void MacroAssembler::initializeBigInt64(Scalar::Type type, Register bigInt,
+ Register64 val) {
+ MOZ_ASSERT(Scalar::isBigIntType(type));
+
+ store32(Imm32(0), Address(bigInt, BigInt::offsetOfFlags()));
+
+ Label done, nonZero;
+ branch64(Assembler::NotEqual, val, Imm64(0), &nonZero);
+ {
+ store32(Imm32(0), Address(bigInt, BigInt::offsetOfLength()));
+ jump(&done);
+ }
+ bind(&nonZero);
+
+ if (type == Scalar::BigInt64) {
+ // Set the sign-bit for negative values and then continue with the two's
+ // complement.
+ Label isPositive;
+ branch64(Assembler::GreaterThan, val, Imm64(0), &isPositive);
+ {
+ store32(Imm32(BigInt::signBitMask()),
+ Address(bigInt, BigInt::offsetOfFlags()));
+ neg64(val);
+ }
+ bind(&isPositive);
+ }
+
+ store32(Imm32(1), Address(bigInt, BigInt::offsetOfLength()));
+
+ static_assert(sizeof(BigInt::Digit) == sizeof(uintptr_t),
+ "BigInt Digit size matches uintptr_t, so there's a single "
+ "store on 64-bit and up to two stores on 32-bit");
+
+#ifndef JS_PUNBOX64
+ Label singleDigit;
+ branchTest32(Assembler::Zero, val.high, val.high, &singleDigit);
+ store32(Imm32(2), Address(bigInt, BigInt::offsetOfLength()));
+ bind(&singleDigit);
+
+ // We can perform a single store64 on 32-bit platforms, because inline
+ // storage can store at least two 32-bit integers.
+ static_assert(BigInt::inlineDigitsLength() >= 2,
+ "BigInt inline storage can store at least two digits");
+#endif
+
+ store64(val, Address(bigInt, js::BigInt::offsetOfInlineDigits()));
+
+ bind(&done);
+}
+
+void MacroAssembler::initializeBigInt(Register bigInt, Register val) {
+ store32(Imm32(0), Address(bigInt, BigInt::offsetOfFlags()));
+
+ Label done, nonZero;
+ branchTestPtr(Assembler::NonZero, val, val, &nonZero);
+ {
+ store32(Imm32(0), Address(bigInt, BigInt::offsetOfLength()));
+ jump(&done);
+ }
+ bind(&nonZero);
+
+ // Set the sign-bit for negative values and then continue with the two's
+ // complement.
+ Label isPositive;
+ branchTestPtr(Assembler::NotSigned, val, val, &isPositive);
+ {
+ store32(Imm32(BigInt::signBitMask()),
+ Address(bigInt, BigInt::offsetOfFlags()));
+ negPtr(val);
+ }
+ bind(&isPositive);
+
+ store32(Imm32(1), Address(bigInt, BigInt::offsetOfLength()));
+
+ static_assert(sizeof(BigInt::Digit) == sizeof(uintptr_t),
+ "BigInt Digit size matches uintptr_t");
+
+ storePtr(val, Address(bigInt, js::BigInt::offsetOfInlineDigits()));
+
+ bind(&done);
+}
+
+void MacroAssembler::initializeBigIntAbsolute(Register bigInt, Register val) {
+ store32(Imm32(0), Address(bigInt, BigInt::offsetOfFlags()));
+
+ Label done, nonZero;
+ branchTestPtr(Assembler::NonZero, val, val, &nonZero);
+ {
+ store32(Imm32(0), Address(bigInt, BigInt::offsetOfLength()));
+ jump(&done);
+ }
+ bind(&nonZero);
+
+ store32(Imm32(1), Address(bigInt, BigInt::offsetOfLength()));
+
+ static_assert(sizeof(BigInt::Digit) == sizeof(uintptr_t),
+ "BigInt Digit size matches uintptr_t");
+
+ storePtr(val, Address(bigInt, js::BigInt::offsetOfInlineDigits()));
+
+ bind(&done);
+}
+
+void MacroAssembler::copyBigIntWithInlineDigits(Register src, Register dest,
+ Register temp,
+ gc::InitialHeap initialHeap,
+ Label* fail) {
+ branch32(Assembler::Above, Address(src, BigInt::offsetOfLength()),
+ Imm32(int32_t(BigInt::inlineDigitsLength())), fail);
+
+ newGCBigInt(dest, temp, initialHeap, fail);
+
+ // Copy the sign-bit, but not any of the other bits used by the GC.
+ load32(Address(src, BigInt::offsetOfFlags()), temp);
+ and32(Imm32(BigInt::signBitMask()), temp);
+ store32(temp, Address(dest, BigInt::offsetOfFlags()));
+
+ // Copy the length.
+ load32(Address(src, BigInt::offsetOfLength()), temp);
+ store32(temp, Address(dest, BigInt::offsetOfLength()));
+
+ // Copy the digits.
+ Address srcDigits(src, js::BigInt::offsetOfInlineDigits());
+ Address destDigits(dest, js::BigInt::offsetOfInlineDigits());
+
+ for (size_t i = 0; i < BigInt::inlineDigitsLength(); i++) {
+ static_assert(sizeof(BigInt::Digit) == sizeof(uintptr_t),
+ "BigInt Digit size matches uintptr_t");
+
+ loadPtr(srcDigits, temp);
+ storePtr(temp, destDigits);
+
+ srcDigits = Address(src, srcDigits.offset + sizeof(BigInt::Digit));
+ destDigits = Address(dest, destDigits.offset + sizeof(BigInt::Digit));
+ }
+}
+
+void MacroAssembler::compareBigIntAndInt32(JSOp op, Register bigInt,
+ Register int32, Register scratch1,
+ Register scratch2, Label* ifTrue,
+ Label* ifFalse) {
+ MOZ_ASSERT(IsLooseEqualityOp(op) || IsRelationalOp(op));
+
+ static_assert(std::is_same_v<BigInt::Digit, uintptr_t>,
+ "BigInt digit can be loaded in a pointer-sized register");
+ static_assert(sizeof(BigInt::Digit) >= sizeof(uint32_t),
+ "BigInt digit stores at least an uint32");
+
+ // Test for too large numbers.
+ //
+ // If the absolute value of the BigInt can't be expressed in an uint32/uint64,
+ // the result of the comparison is a constant.
+ if (op == JSOp::Eq || op == JSOp::Ne) {
+ Label* tooLarge = op == JSOp::Eq ? ifFalse : ifTrue;
+ branch32(Assembler::GreaterThan,
+ Address(bigInt, BigInt::offsetOfDigitLength()), Imm32(1),
+ tooLarge);
+ } else {
+ Label doCompare;
+ branch32(Assembler::LessThanOrEqual,
+ Address(bigInt, BigInt::offsetOfDigitLength()), Imm32(1),
+ &doCompare);
+
+ // Still need to take the sign-bit into account for relational operations.
+ if (op == JSOp::Lt || op == JSOp::Le) {
+ branchIfBigIntIsNegative(bigInt, ifTrue);
+ jump(ifFalse);
+ } else {
+ branchIfBigIntIsNegative(bigInt, ifFalse);
+ jump(ifTrue);
+ }
+
+ bind(&doCompare);
+ }
+
+ // Test for mismatched signs and, if the signs are equal, load |abs(x)| in
+ // |scratch1| and |abs(y)| in |scratch2| and then compare the absolute numbers
+ // against each other.
+ {
+ // Jump to |ifTrue| resp. |ifFalse| if the BigInt is strictly less than
+ // resp. strictly greater than the int32 value, depending on the comparison
+ // operator.
+ Label* greaterThan;
+ Label* lessThan;
+ if (op == JSOp::Eq) {
+ greaterThan = ifFalse;
+ lessThan = ifFalse;
+ } else if (op == JSOp::Ne) {
+ greaterThan = ifTrue;
+ lessThan = ifTrue;
+ } else if (op == JSOp::Lt || op == JSOp::Le) {
+ greaterThan = ifFalse;
+ lessThan = ifTrue;
+ } else {
+ MOZ_ASSERT(op == JSOp::Gt || op == JSOp::Ge);
+ greaterThan = ifTrue;
+ lessThan = ifFalse;
+ }
+
+ // BigInt digits are always stored as an absolute number.
+ loadFirstBigIntDigitOrZero(bigInt, scratch1);
+
+ // Load the int32 into |scratch2| and negate it for negative numbers.
+ move32(int32, scratch2);
+
+ Label isNegative, doCompare;
+ branchIfBigIntIsNegative(bigInt, &isNegative);
+ branch32(Assembler::LessThan, int32, Imm32(0), greaterThan);
+ jump(&doCompare);
+
+ // We rely on |neg32(INT32_MIN)| staying INT32_MIN, because we're using an
+ // unsigned comparison below.
+ bind(&isNegative);
+ branch32(Assembler::GreaterThanOrEqual, int32, Imm32(0), lessThan);
+ neg32(scratch2);
+
+ // Not all supported platforms (e.g. MIPS64) zero-extend 32-bit operations,
+ // so we need to explicitly clear any high 32-bits.
+ move32ZeroExtendToPtr(scratch2, scratch2);
+
+ // Reverse the relational comparator for negative numbers.
+ // |-x < -y| <=> |+x > +y|.
+ // |-x ≤ -y| <=> |+x ≥ +y|.
+ // |-x > -y| <=> |+x < +y|.
+ // |-x ≥ -y| <=> |+x ≤ +y|.
+ JSOp reversed = ReverseCompareOp(op);
+ if (reversed != op) {
+ branchPtr(JSOpToCondition(reversed, /* isSigned = */ false), scratch1,
+ scratch2, ifTrue);
+ jump(ifFalse);
+ }
+
+ bind(&doCompare);
+ branchPtr(JSOpToCondition(op, /* isSigned = */ false), scratch1, scratch2,
+ ifTrue);
+ }
+}
+
+void MacroAssembler::equalBigInts(Register left, Register right, Register temp1,
+ Register temp2, Register temp3,
+ Register temp4, Label* notSameSign,
+ Label* notSameLength, Label* notSameDigit) {
+ MOZ_ASSERT(left != temp1);
+ MOZ_ASSERT(right != temp1);
+ MOZ_ASSERT(right != temp2);
+
+ // Jump to |notSameSign| when the sign aren't the same.
+ load32(Address(left, BigInt::offsetOfFlags()), temp1);
+ xor32(Address(right, BigInt::offsetOfFlags()), temp1);
+ branchTest32(Assembler::NonZero, temp1, Imm32(BigInt::signBitMask()),
+ notSameSign);
+
+ // Jump to |notSameLength| when the digits length is different.
+ load32(Address(right, BigInt::offsetOfLength()), temp1);
+ branch32(Assembler::NotEqual, Address(left, BigInt::offsetOfLength()), temp1,
+ notSameLength);
+
+ // Both BigInts have the same sign and the same number of digits. Loop
+ // over each digit, starting with the left-most one, and break from the
+ // loop when the first non-matching digit was found.
+
+ loadBigIntDigits(left, temp2);
+ loadBigIntDigits(right, temp3);
+
+ static_assert(sizeof(BigInt::Digit) == sizeof(void*),
+ "BigInt::Digit is pointer sized");
+
+ computeEffectiveAddress(BaseIndex(temp2, temp1, ScalePointer), temp2);
+ computeEffectiveAddress(BaseIndex(temp3, temp1, ScalePointer), temp3);
+
+ Label start, loop;
+ jump(&start);
+ bind(&loop);
+
+ subPtr(Imm32(sizeof(BigInt::Digit)), temp2);
+ subPtr(Imm32(sizeof(BigInt::Digit)), temp3);
+
+ loadPtr(Address(temp3, 0), temp4);
+ branchPtr(Assembler::NotEqual, Address(temp2, 0), temp4, notSameDigit);
+
+ bind(&start);
+ branchSub32(Assembler::NotSigned, Imm32(1), temp1, &loop);
+
+ // No different digits were found, both BigInts are equal to each other.
+}
+
+void MacroAssembler::typeOfObject(Register obj, Register scratch, Label* slow,
+ Label* isObject, Label* isCallable,
+ Label* isUndefined) {
+ loadObjClassUnsafe(obj, scratch);
+
+ // Proxies can emulate undefined and have complex isCallable behavior.
+ branchTestClassIsProxy(true, scratch, slow);
+
+ // JSFunctions are always callable.
+ branchTestClassIsFunction(Assembler::Equal, scratch, isCallable);
+
+ // Objects that emulate undefined.
+ Address flags(scratch, JSClass::offsetOfFlags());
+ branchTest32(Assembler::NonZero, flags, Imm32(JSCLASS_EMULATES_UNDEFINED),
+ isUndefined);
+
+ // Handle classes with a call hook.
+ branchPtr(Assembler::Equal, Address(scratch, offsetof(JSClass, cOps)),
+ ImmPtr(nullptr), isObject);
+
+ loadPtr(Address(scratch, offsetof(JSClass, cOps)), scratch);
+ branchPtr(Assembler::Equal, Address(scratch, offsetof(JSClassOps, call)),
+ ImmPtr(nullptr), isObject);
+
+ jump(isCallable);
+}
+
+void MacroAssembler::isCallableOrConstructor(bool isCallable, Register obj,
+ Register output, Label* isProxy) {
+ MOZ_ASSERT(obj != output);
+
+ Label notFunction, hasCOps, done;
+ loadObjClassUnsafe(obj, output);
+
+ // An object is callable iff:
+ // is<JSFunction>() || (getClass()->cOps && getClass()->cOps->call).
+ // An object is constructor iff:
+ // ((is<JSFunction>() && as<JSFunction>().isConstructor) ||
+ // (getClass()->cOps && getClass()->cOps->construct)).
+ branchTestClassIsFunction(Assembler::NotEqual, output, &notFunction);
+ if (isCallable) {
+ move32(Imm32(1), output);
+ } else {
+ static_assert(mozilla::IsPowerOfTwo(uint32_t(FunctionFlags::CONSTRUCTOR)),
+ "FunctionFlags::CONSTRUCTOR has only one bit set");
+
+ load32(Address(obj, JSFunction::offsetOfFlagsAndArgCount()), output);
+ rshift32(Imm32(mozilla::FloorLog2(uint32_t(FunctionFlags::CONSTRUCTOR))),
+ output);
+ and32(Imm32(1), output);
+ }
+ jump(&done);
+
+ bind(&notFunction);
+
+ // Just skim proxies off. Their notion of isCallable()/isConstructor() is
+ // more complicated.
+ branchTestClassIsProxy(true, output, isProxy);
+
+ branchPtr(Assembler::NonZero, Address(output, offsetof(JSClass, cOps)),
+ ImmPtr(nullptr), &hasCOps);
+ move32(Imm32(0), output);
+ jump(&done);
+
+ bind(&hasCOps);
+ loadPtr(Address(output, offsetof(JSClass, cOps)), output);
+ size_t opsOffset =
+ isCallable ? offsetof(JSClassOps, call) : offsetof(JSClassOps, construct);
+ cmpPtrSet(Assembler::NonZero, Address(output, opsOffset), ImmPtr(nullptr),
+ output);
+
+ bind(&done);
+}
+
+void MacroAssembler::loadJSContext(Register dest) {
+ movePtr(ImmPtr(runtime()->mainContextPtr()), dest);
+}
+
+static const uint8_t* ContextRealmPtr(CompileRuntime* rt) {
+ return (static_cast<const uint8_t*>(rt->mainContextPtr()) +
+ JSContext::offsetOfRealm());
+}
+
+void MacroAssembler::switchToRealm(Register realm) {
+ storePtr(realm, AbsoluteAddress(ContextRealmPtr(runtime())));
+}
+
+void MacroAssembler::switchToRealm(const void* realm, Register scratch) {
+ MOZ_ASSERT(realm);
+
+ movePtr(ImmPtr(realm), scratch);
+ switchToRealm(scratch);
+}
+
+void MacroAssembler::switchToObjectRealm(Register obj, Register scratch) {
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ loadPtr(Address(scratch, BaseShape::offsetOfRealm()), scratch);
+ switchToRealm(scratch);
+}
+
+void MacroAssembler::switchToBaselineFrameRealm(Register scratch) {
+ Address envChain(FramePointer,
+ BaselineFrame::reverseOffsetOfEnvironmentChain());
+ loadPtr(envChain, scratch);
+ switchToObjectRealm(scratch, scratch);
+}
+
+void MacroAssembler::switchToWasmInstanceRealm(Register scratch1,
+ Register scratch2) {
+ loadPtr(Address(InstanceReg, wasm::Instance::offsetOfCx()), scratch1);
+ loadPtr(Address(InstanceReg, wasm::Instance::offsetOfRealm()), scratch2);
+ storePtr(scratch2, Address(scratch1, JSContext::offsetOfRealm()));
+}
+
+void MacroAssembler::debugAssertContextRealm(const void* realm,
+ Register scratch) {
+#ifdef DEBUG
+ Label ok;
+ movePtr(ImmPtr(realm), scratch);
+ branchPtr(Assembler::Equal, AbsoluteAddress(ContextRealmPtr(runtime())),
+ scratch, &ok);
+ assumeUnreachable("Unexpected context realm");
+ bind(&ok);
+#endif
+}
+
+void MacroAssembler::setIsCrossRealmArrayConstructor(Register obj,
+ Register output) {
+#ifdef DEBUG
+ Label notProxy;
+ branchTestObjectIsProxy(false, obj, output, &notProxy);
+ assumeUnreachable("Unexpected proxy in setIsCrossRealmArrayConstructor");
+ bind(&notProxy);
+#endif
+
+ // The object's realm must not be cx->realm.
+ Label isFalse, done;
+ loadPtr(Address(obj, JSObject::offsetOfShape()), output);
+ loadPtr(Address(output, Shape::offsetOfBaseShape()), output);
+ loadPtr(Address(output, BaseShape::offsetOfRealm()), output);
+ branchPtr(Assembler::Equal, AbsoluteAddress(ContextRealmPtr(runtime())),
+ output, &isFalse);
+
+ // The object must be a function.
+ branchTestObjIsFunction(Assembler::NotEqual, obj, output, obj, &isFalse);
+
+ // The function must be the ArrayConstructor native.
+ branchPtr(Assembler::NotEqual,
+ Address(obj, JSFunction::offsetOfNativeOrEnv()),
+ ImmPtr(js::ArrayConstructor), &isFalse);
+
+ move32(Imm32(1), output);
+ jump(&done);
+
+ bind(&isFalse);
+ move32(Imm32(0), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::setIsDefinitelyTypedArrayConstructor(Register obj,
+ Register output) {
+ Label isFalse, isTrue, done;
+
+ // The object must be a function. (Wrappers are not supported.)
+ branchTestObjIsFunction(Assembler::NotEqual, obj, output, obj, &isFalse);
+
+ // Load the native into |output|.
+ loadPtr(Address(obj, JSFunction::offsetOfNativeOrEnv()), output);
+
+ auto branchIsTypedArrayCtor = [&](Scalar::Type type) {
+ // The function must be a TypedArrayConstructor native (from any realm).
+ JSNative constructor = TypedArrayConstructorNative(type);
+ branchPtr(Assembler::Equal, output, ImmPtr(constructor), &isTrue);
+ };
+
+#define TYPED_ARRAY_CONSTRUCTOR_NATIVE(_, T, N) \
+ branchIsTypedArrayCtor(Scalar::N);
+ JS_FOR_EACH_TYPED_ARRAY(TYPED_ARRAY_CONSTRUCTOR_NATIVE)
+#undef TYPED_ARRAY_CONSTRUCTOR_NATIVE
+
+ // Falls through to the false case.
+
+ bind(&isFalse);
+ move32(Imm32(0), output);
+ jump(&done);
+
+ bind(&isTrue);
+ move32(Imm32(1), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::loadMegamorphicCache(Register dest) {
+ movePtr(ImmPtr(runtime()->addressOfMegamorphicCache()), dest);
+}
+
+void MacroAssembler::loadStringToAtomCacheLastLookups(Register dest) {
+ uintptr_t cachePtr = uintptr_t(runtime()->addressOfStringToAtomCache());
+ void* offset = (void*)(cachePtr + StringToAtomCache::offsetOfLastLookups());
+ movePtr(ImmPtr(offset), dest);
+}
+
+void MacroAssembler::loadAtomOrSymbolAndHash(ValueOperand value, Register outId,
+ Register outHash,
+ Label* cacheMiss) {
+ Label isString, fatInline, done, nonAtom, atom, lastLookupAtom;
+
+ {
+ ScratchTagScope tag(*this, value);
+ splitTagForTest(value, tag);
+ branchTestString(Assembler::Equal, tag, &isString);
+
+ branchTestSymbol(Assembler::NotEqual, tag, cacheMiss);
+
+ unboxSymbol(value, outId);
+ load32(Address(outId, JS::Symbol::offsetOfHash()), outHash);
+ jump(&done);
+ }
+
+ bind(&isString);
+ unboxString(value, outId);
+ branchTest32(Assembler::Zero, Address(outId, JSString::offsetOfFlags()),
+ Imm32(JSString::ATOM_BIT), &nonAtom);
+
+ bind(&atom);
+ move32(Imm32(JSString::FAT_INLINE_MASK), outHash);
+ and32(Address(outId, JSString::offsetOfFlags()), outHash);
+
+ branch32(Assembler::Equal, outHash, Imm32(JSString::FAT_INLINE_MASK),
+ &fatInline);
+ load32(Address(outId, NormalAtom::offsetOfHash()), outHash);
+ jump(&done);
+ bind(&fatInline);
+ load32(Address(outId, FatInlineAtom::offsetOfHash()), outHash);
+ jump(&done);
+
+ bind(&nonAtom);
+ loadStringToAtomCacheLastLookups(outHash);
+
+ // Compare each entry in the StringToAtomCache's lastLookups_ array
+ size_t stringOffset = StringToAtomCache::LastLookup::offsetOfString();
+ branchPtr(Assembler::Equal, Address(outHash, stringOffset), outId,
+ &lastLookupAtom);
+ for (size_t i = 0; i < StringToAtomCache::NumLastLookups - 1; ++i) {
+ addPtr(Imm32(sizeof(StringToAtomCache::LastLookup)), outHash);
+ branchPtr(Assembler::Equal, Address(outHash, stringOffset), outId,
+ &lastLookupAtom);
+ }
+
+ // Couldn't find us in the cache, so fall back to the C++ call
+ jump(cacheMiss);
+
+ // We found a hit in the lastLookups_ array! Load the associated atom
+ // and jump back up to our usual atom handling code
+ bind(&lastLookupAtom);
+ size_t atomOffset = StringToAtomCache::LastLookup::offsetOfAtom();
+ loadPtr(Address(outHash, atomOffset), outId);
+ jump(&atom);
+
+ bind(&done);
+}
+
+void MacroAssembler::emitExtractValueFromMegamorphicCacheEntry(
+ Register obj, Register entry, Register scratch1, Register scratch2,
+ ValueOperand output, Label* cacheHit, Label* cacheMiss) {
+ Label isMissing, dynamicSlot, protoLoopHead, protoLoopTail;
+
+ // scratch2 = entry->numHops_
+ load8ZeroExtend(Address(entry, MegamorphicCache::Entry::offsetOfNumHops()),
+ scratch2);
+ // if (scratch2 == NumHopsForMissingOwnProperty) goto cacheMiss
+ branch32(Assembler::Equal, scratch2,
+ Imm32(MegamorphicCache::Entry::NumHopsForMissingOwnProperty),
+ cacheMiss);
+ // if (scratch2 == NumHopsForMissingProperty) goto isMissing
+ branch32(Assembler::Equal, scratch2,
+ Imm32(MegamorphicCache::Entry::NumHopsForMissingProperty),
+ &isMissing);
+
+ // NOTE: Where this is called, `output` can actually alias `obj`, and before
+ // the last cacheMiss branch above we can't write to `obj`, so we can't
+ // use `output`'s scratch register there. However a cache miss is impossible
+ // now, so we're free to use `output` as we like.
+ Register outputScratch = output.scratchReg();
+ if (!outputScratch.aliases(obj)) {
+ // We're okay with paying this very slight extra cost to avoid a potential
+ // footgun of writing to what callers understand as only an input register.
+ movePtr(obj, outputScratch);
+ }
+ branchTest32(Assembler::Zero, scratch2, scratch2, &protoLoopTail);
+ bind(&protoLoopHead);
+ loadObjProto(outputScratch, outputScratch);
+ branchSub32(Assembler::NonZero, Imm32(1), scratch2, &protoLoopHead);
+ bind(&protoLoopTail);
+
+ // scratch1 = outputScratch->numFixedSlots()
+ loadPtr(Address(outputScratch, JSObject::offsetOfShape()), scratch1);
+ load32(Address(scratch1, Shape::offsetOfImmutableFlags()), scratch1);
+ and32(Imm32(NativeShape::fixedSlotsMask()), scratch1);
+ rshift32(Imm32(NativeShape::fixedSlotsShift()), scratch1);
+
+ // scratch2 = entry->slot()
+ load16ZeroExtend(Address(entry, MegamorphicCache::Entry::offsetOfSlot()),
+ scratch2);
+ // if (scratch2 >= scratch1) goto dynamicSlot
+ branch32(Assembler::GreaterThanOrEqual, scratch2, scratch1, &dynamicSlot);
+
+ static_assert(sizeof(HeapSlot) == 8);
+ // output = outputScratch->fixedSlots()[scratch2]
+ loadValue(BaseValueIndex(outputScratch, scratch2, sizeof(NativeObject)),
+ output);
+ jump(cacheHit);
+
+ bind(&dynamicSlot);
+ // scratch2 -= scratch1
+ sub32(scratch1, scratch2);
+ // output = outputScratch->slots_[scratch2]
+ loadPtr(Address(outputScratch, NativeObject::offsetOfSlots()), outputScratch);
+ loadValue(BaseValueIndex(outputScratch, scratch2, 0), output);
+ jump(cacheHit);
+
+ bind(&isMissing);
+ // output = undefined
+ moveValue(UndefinedValue(), output);
+ jump(cacheHit);
+}
+
+void MacroAssembler::emitMegamorphicCacheLookupByValueCommon(
+ ValueOperand id, Register obj, Register scratch1, Register scratch2,
+ Register outEntryPtr, Label* cacheMiss) {
+ // A lot of this code is shared with emitMegamorphicCacheLookup. It would
+ // be nice to be able to avoid the duplication here, but due to a few
+ // differences like taking the id in a ValueOperand instead of being able
+ // to bake it in as an immediate, and only needing a Register for the output
+ // value, it seemed more awkward to read once it was deduplicated.
+
+ // outEntryPtr = obj->shape()
+ loadPtr(Address(obj, JSObject::offsetOfShape()), outEntryPtr);
+
+ movePtr(outEntryPtr, scratch2);
+
+ // outEntryPtr = (outEntryPtr >> 3) ^ (outEntryPtr >> 13) + idHash
+ rshiftPtr(Imm32(MegamorphicCache::ShapeHashShift1), outEntryPtr);
+ rshiftPtr(Imm32(MegamorphicCache::ShapeHashShift2), scratch2);
+ xorPtr(scratch2, outEntryPtr);
+
+ loadAtomOrSymbolAndHash(id, scratch1, scratch2, cacheMiss);
+ addPtr(scratch2, outEntryPtr);
+
+ // outEntryPtr %= MegamorphicCache::NumEntries
+ constexpr size_t cacheSize = MegamorphicCache::NumEntries;
+ static_assert(mozilla::IsPowerOfTwo(cacheSize));
+ size_t cacheMask = cacheSize - 1;
+ and32(Imm32(cacheMask), outEntryPtr);
+
+ loadMegamorphicCache(scratch2);
+ // outEntryPtr = &scratch2->entries_[outEntryPtr]
+ constexpr size_t entrySize = sizeof(MegamorphicCache::Entry);
+ static_assert(sizeof(void*) == 4 || entrySize == 24);
+ if constexpr (sizeof(void*) == 4) {
+ mul32(Imm32(entrySize), outEntryPtr);
+ computeEffectiveAddress(BaseIndex(scratch2, outEntryPtr, TimesOne,
+ MegamorphicCache::offsetOfEntries()),
+ outEntryPtr);
+ } else {
+ computeEffectiveAddress(BaseIndex(outEntryPtr, outEntryPtr, TimesTwo),
+ outEntryPtr);
+ computeEffectiveAddress(BaseIndex(scratch2, outEntryPtr, TimesEight,
+ MegamorphicCache::offsetOfEntries()),
+ outEntryPtr);
+ }
+
+ // if (outEntryPtr->key_ != scratch1) goto cacheMiss
+ branchPtr(Assembler::NotEqual,
+ Address(outEntryPtr, MegamorphicCache::Entry::offsetOfKey()),
+ scratch1, cacheMiss);
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch1);
+
+ // if (outEntryPtr->shape_ != scratch1) goto cacheMiss
+ branchPtr(Assembler::NotEqual,
+ Address(outEntryPtr, MegamorphicCache::Entry::offsetOfShape()),
+ scratch1, cacheMiss);
+
+ // scratch2 = scratch2->generation_
+ load16ZeroExtend(Address(scratch2, MegamorphicCache::offsetOfGeneration()),
+ scratch2);
+ load16ZeroExtend(
+ Address(outEntryPtr, MegamorphicCache::Entry::offsetOfGeneration()),
+ scratch1);
+ // if (outEntryPtr->generation_ != scratch2) goto cacheMiss
+ branch32(Assembler::NotEqual, scratch1, scratch2, cacheMiss);
+}
+
+void MacroAssembler::emitMegamorphicCacheLookup(
+ PropertyKey id, Register obj, Register scratch1, Register scratch2,
+ Register scratch3, ValueOperand output, Label* cacheHit) {
+ Label cacheMiss, isMissing, dynamicSlot, protoLoopHead, protoLoopTail;
+
+ // scratch1 = obj->shape()
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch1);
+
+ movePtr(scratch1, scratch2);
+ movePtr(scratch1, scratch3);
+
+ // scratch2 = (scratch1 >> 3) ^ (scratch1 >> 13) + hash(id)
+ rshiftPtr(Imm32(MegamorphicCache::ShapeHashShift1), scratch2);
+ rshiftPtr(Imm32(MegamorphicCache::ShapeHashShift2), scratch3);
+ xorPtr(scratch3, scratch2);
+ addPtr(Imm32(HashAtomOrSymbolPropertyKey(id)), scratch2);
+
+ // scratch2 %= MegamorphicCache::NumEntries
+ constexpr size_t cacheSize = MegamorphicCache::NumEntries;
+ static_assert(mozilla::IsPowerOfTwo(cacheSize));
+ size_t cacheMask = cacheSize - 1;
+ and32(Imm32(cacheMask), scratch2);
+
+ loadMegamorphicCache(scratch3);
+ // scratch2 = &scratch3->entries_[scratch2]
+ constexpr size_t entrySize = sizeof(MegamorphicCache::Entry);
+ static_assert(sizeof(void*) == 4 || entrySize == 24);
+ if constexpr (sizeof(void*) == 4) {
+ mul32(Imm32(entrySize), scratch2);
+ computeEffectiveAddress(BaseIndex(scratch3, scratch2, TimesOne,
+ MegamorphicCache::offsetOfEntries()),
+ scratch2);
+ } else {
+ computeEffectiveAddress(BaseIndex(scratch2, scratch2, TimesTwo), scratch2);
+ computeEffectiveAddress(BaseIndex(scratch3, scratch2, TimesEight,
+ MegamorphicCache::offsetOfEntries()),
+ scratch2);
+ }
+
+ // if (scratch2->shape_ != scratch1) goto cacheMiss
+ branchPtr(Assembler::NotEqual,
+ Address(scratch2, MegamorphicCache::Entry::offsetOfShape()),
+ scratch1, &cacheMiss);
+
+ // if (scratch2->key_ != id) goto cacheMiss
+ movePropertyKey(id, scratch1);
+ branchPtr(Assembler::NotEqual,
+ Address(scratch2, MegamorphicCache::Entry::offsetOfKey()), scratch1,
+ &cacheMiss);
+
+ // scratch3 = scratch3->generation_
+ load16ZeroExtend(Address(scratch3, MegamorphicCache::offsetOfGeneration()),
+ scratch3);
+ load16ZeroExtend(
+ Address(scratch2, MegamorphicCache::Entry::offsetOfGeneration()),
+ scratch1);
+ // if (scratch2->generation_ != scratch3) goto cacheMiss
+ branch32(Assembler::NotEqual, scratch1, scratch3, &cacheMiss);
+
+ emitExtractValueFromMegamorphicCacheEntry(obj, scratch2, scratch1, scratch3,
+ output, cacheHit, &cacheMiss);
+
+ bind(&cacheMiss);
+}
+
+void MacroAssembler::emitMegamorphicCacheLookupByValue(
+ ValueOperand id, Register obj, Register scratch1, Register scratch2,
+ Register scratch3, ValueOperand output, Label* cacheHit) {
+ Label cacheMiss;
+ emitMegamorphicCacheLookupByValueCommon(id, obj, scratch1, scratch2, scratch3,
+ &cacheMiss);
+ emitExtractValueFromMegamorphicCacheEntry(obj, scratch3, scratch1, scratch2,
+ output, cacheHit, &cacheMiss);
+ bind(&cacheMiss);
+}
+
+void MacroAssembler::emitMegamorphicCacheLookupExists(
+ ValueOperand id, Register obj, Register scratch1, Register scratch2,
+ Register scratch3, Register output, Label* cacheHit, bool hasOwn) {
+ Label cacheMiss, cacheHitFalse;
+ emitMegamorphicCacheLookupByValueCommon(id, obj, scratch1, scratch2, scratch3,
+ &cacheMiss);
+
+ // scratch3 = scratch3->numHops_
+ load8ZeroExtend(Address(scratch3, MegamorphicCache::Entry::offsetOfNumHops()),
+ scratch3);
+
+ branch32(Assembler::Equal, scratch3,
+ Imm32(MegamorphicCache::Entry::NumHopsForMissingProperty),
+ &cacheHitFalse);
+
+ if (hasOwn) {
+ branch32(Assembler::NotEqual, scratch3, Imm32(0), &cacheHitFalse);
+ } else {
+ branch32(Assembler::Equal, scratch3,
+ Imm32(MegamorphicCache::Entry::NumHopsForMissingOwnProperty),
+ &cacheMiss);
+ }
+
+ move32(Imm32(1), output);
+ jump(cacheHit);
+
+ bind(&cacheHitFalse);
+ xor32(output, output);
+ jump(cacheHit);
+
+ bind(&cacheMiss);
+}
+
+void MacroAssembler::guardNonNegativeIntPtrToInt32(Register reg, Label* fail) {
+#ifdef DEBUG
+ Label ok;
+ branchPtr(Assembler::NotSigned, reg, reg, &ok);
+ assumeUnreachable("Unexpected negative value");
+ bind(&ok);
+#endif
+
+#ifdef JS_64BIT
+ branchPtr(Assembler::Above, reg, Imm32(INT32_MAX), fail);
+#endif
+}
+
+void MacroAssembler::loadArrayBufferByteLengthIntPtr(Register obj,
+ Register output) {
+ Address slotAddr(obj, ArrayBufferObject::offsetOfByteLengthSlot());
+ loadPrivate(slotAddr, output);
+}
+
+void MacroAssembler::loadArrayBufferViewByteOffsetIntPtr(Register obj,
+ Register output) {
+ Address slotAddr(obj, ArrayBufferViewObject::byteOffsetOffset());
+ loadPrivate(slotAddr, output);
+}
+
+void MacroAssembler::loadArrayBufferViewLengthIntPtr(Register obj,
+ Register output) {
+ Address slotAddr(obj, ArrayBufferViewObject::lengthOffset());
+ loadPrivate(slotAddr, output);
+}
+
+void MacroAssembler::loadDOMExpandoValueGuardGeneration(
+ Register obj, ValueOperand output,
+ JS::ExpandoAndGeneration* expandoAndGeneration, uint64_t generation,
+ Label* fail) {
+ loadPtr(Address(obj, ProxyObject::offsetOfReservedSlots()),
+ output.scratchReg());
+ loadValue(Address(output.scratchReg(),
+ js::detail::ProxyReservedSlots::offsetOfPrivateSlot()),
+ output);
+
+ // Guard the ExpandoAndGeneration* matches the proxy's ExpandoAndGeneration
+ // privateSlot.
+ branchTestValue(Assembler::NotEqual, output,
+ PrivateValue(expandoAndGeneration), fail);
+
+ // Guard expandoAndGeneration->generation matches the expected generation.
+ Address generationAddr(output.payloadOrValueReg(),
+ JS::ExpandoAndGeneration::offsetOfGeneration());
+ branch64(Assembler::NotEqual, generationAddr, Imm64(generation), fail);
+
+ // Load expandoAndGeneration->expando into the output Value register.
+ loadValue(Address(output.payloadOrValueReg(),
+ JS::ExpandoAndGeneration::offsetOfExpando()),
+ output);
+}
+
+void MacroAssembler::loadJitActivation(Register dest) {
+ loadJSContext(dest);
+ loadPtr(Address(dest, offsetof(JSContext, activation_)), dest);
+}
+
+void MacroAssembler::guardSpecificAtom(Register str, JSAtom* atom,
+ Register scratch,
+ const LiveRegisterSet& volatileRegs,
+ Label* fail) {
+ Label done;
+ branchPtr(Assembler::Equal, str, ImmGCPtr(atom), &done);
+
+ // The pointers are not equal, so if the input string is also an atom it
+ // must be a different string.
+ branchTest32(Assembler::NonZero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::ATOM_BIT), fail);
+
+ // Check the length.
+ branch32(Assembler::NotEqual, Address(str, JSString::offsetOfLength()),
+ Imm32(atom->length()), fail);
+
+ // We have a non-atomized string with the same length. Call a helper
+ // function to do the comparison.
+ PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSString * str1, JSString * str2);
+ setupUnalignedABICall(scratch);
+ movePtr(ImmGCPtr(atom), scratch);
+ passABIArg(scratch);
+ passABIArg(str);
+ callWithABI<Fn, EqualStringsHelperPure>();
+ storeCallPointerResult(scratch);
+
+ MOZ_ASSERT(!volatileRegs.has(scratch));
+ PopRegsInMask(volatileRegs);
+ branchIfFalseBool(scratch, fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::guardStringToInt32(Register str, Register output,
+ Register scratch,
+ LiveRegisterSet volatileRegs,
+ Label* fail) {
+ Label vmCall, done;
+ // Use indexed value as fast path if possible.
+ loadStringIndexValue(str, output, &vmCall);
+ jump(&done);
+ {
+ bind(&vmCall);
+
+ // Reserve space for holding the result int32_t of the call. Use
+ // pointer-size to avoid misaligning the stack on 64-bit platforms.
+ reserveStack(sizeof(uintptr_t));
+ moveStackPtrTo(output);
+
+ volatileRegs.takeUnchecked(scratch);
+ if (output.volatile_()) {
+ volatileRegs.addUnchecked(output);
+ }
+ PushRegsInMask(volatileRegs);
+
+ using Fn = bool (*)(JSContext * cx, JSString * str, int32_t * result);
+ setupUnalignedABICall(scratch);
+ loadJSContext(scratch);
+ passABIArg(scratch);
+ passABIArg(str);
+ passABIArg(output);
+ callWithABI<Fn, GetInt32FromStringPure>();
+ storeCallPointerResult(scratch);
+
+ PopRegsInMask(volatileRegs);
+
+ Label ok;
+ branchIfTrueBool(scratch, &ok);
+ {
+ // OOM path, recovered by GetInt32FromStringPure.
+ //
+ // Use addToStackPtr instead of freeStack as freeStack tracks stack height
+ // flow-insensitively, and using it twice would confuse the stack height
+ // tracking.
+ addToStackPtr(Imm32(sizeof(uintptr_t)));
+ jump(fail);
+ }
+ bind(&ok);
+ load32(Address(output, 0), output);
+ freeStack(sizeof(uintptr_t));
+ }
+ bind(&done);
+}
+
+void MacroAssembler::generateBailoutTail(Register scratch,
+ Register bailoutInfo) {
+ Label bailoutFailed;
+ branchIfFalseBool(ReturnReg, &bailoutFailed);
+
+ // Finish bailing out to Baseline.
+ {
+ // Prepare a register set for use in this case.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT_IF(!IsHiddenSP(getStackPointer()),
+ !regs.has(AsRegister(getStackPointer())));
+ regs.take(bailoutInfo);
+
+ Register temp = regs.takeAny();
+
+#ifdef DEBUG
+ // Assert the stack pointer points to the JitFrameLayout header. Copying
+ // starts here.
+ Label ok;
+ loadPtr(Address(bailoutInfo, offsetof(BaselineBailoutInfo, incomingStack)),
+ temp);
+ branchStackPtr(Assembler::Equal, temp, &ok);
+ assumeUnreachable("Unexpected stack pointer value");
+ bind(&ok);
+#endif
+
+ Register copyCur = regs.takeAny();
+ Register copyEnd = regs.takeAny();
+
+ // Copy data onto stack.
+ loadPtr(Address(bailoutInfo, offsetof(BaselineBailoutInfo, copyStackTop)),
+ copyCur);
+ loadPtr(
+ Address(bailoutInfo, offsetof(BaselineBailoutInfo, copyStackBottom)),
+ copyEnd);
+ {
+ Label copyLoop;
+ Label endOfCopy;
+ bind(&copyLoop);
+ branchPtr(Assembler::BelowOrEqual, copyCur, copyEnd, &endOfCopy);
+ subPtr(Imm32(sizeof(uintptr_t)), copyCur);
+ subFromStackPtr(Imm32(sizeof(uintptr_t)));
+ loadPtr(Address(copyCur, 0), temp);
+ storePtr(temp, Address(getStackPointer(), 0));
+ jump(&copyLoop);
+ bind(&endOfCopy);
+ }
+
+ loadPtr(Address(bailoutInfo, offsetof(BaselineBailoutInfo, resumeFramePtr)),
+ FramePointer);
+
+ // Enter exit frame for the FinishBailoutToBaseline call.
+ pushFrameDescriptor(FrameType::BaselineJS);
+ push(Address(bailoutInfo, offsetof(BaselineBailoutInfo, resumeAddr)));
+ push(FramePointer);
+ // No GC things to mark on the stack, push a bare token.
+ loadJSContext(scratch);
+ enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ // Save needed values onto stack temporarily.
+ push(Address(bailoutInfo, offsetof(BaselineBailoutInfo, resumeAddr)));
+
+ // Call a stub to free allocated memory and create arguments objects.
+ using Fn = bool (*)(BaselineBailoutInfo * bailoutInfoArg);
+ setupUnalignedABICall(temp);
+ passABIArg(bailoutInfo);
+ callWithABI<Fn, FinishBailoutToBaseline>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+ branchIfFalseBool(ReturnReg, exceptionLabel());
+
+ // Restore values where they need to be and resume execution.
+ AllocatableGeneralRegisterSet enterRegs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!enterRegs.has(FramePointer));
+ Register jitcodeReg = enterRegs.takeAny();
+
+ pop(jitcodeReg);
+
+ // Discard exit frame.
+ addToStackPtr(Imm32(ExitFrameLayout::SizeWithFooter()));
+
+ jump(jitcodeReg);
+ }
+
+ bind(&bailoutFailed);
+ {
+ // jit::Bailout or jit::InvalidationBailout failed and returned false. The
+ // Ion frame has already been discarded and the stack pointer points to the
+ // JitFrameLayout header. Turn it into an ExitFrameLayout, similar to
+ // EnsureUnwoundJitExitFrame, and call the exception handler.
+ loadJSContext(scratch);
+ enterFakeExitFrame(scratch, scratch, ExitFrameType::UnwoundJit);
+ jump(exceptionLabel());
+ }
+}
+
+void MacroAssembler::loadJitCodeRaw(Register func, Register dest) {
+ static_assert(BaseScript::offsetOfJitCodeRaw() ==
+ SelfHostedLazyScript::offsetOfJitCodeRaw(),
+ "SelfHostedLazyScript and BaseScript must use same layout for "
+ "jitCodeRaw_");
+ loadPrivate(Address(func, JSFunction::offsetOfJitInfoOrScript()), dest);
+ loadPtr(Address(dest, BaseScript::offsetOfJitCodeRaw()), dest);
+}
+
+void MacroAssembler::loadBaselineJitCodeRaw(Register func, Register dest,
+ Label* failure) {
+ // Load JitScript
+ loadPrivate(Address(func, JSFunction::offsetOfJitInfoOrScript()), dest);
+ if (failure) {
+ branchIfScriptHasNoJitScript(dest, failure);
+ }
+ loadJitScript(dest, dest);
+
+ // Load BaselineScript
+ loadPtr(Address(dest, JitScript::offsetOfBaselineScript()), dest);
+ if (failure) {
+ static_assert(BaselineDisabledScript == 0x1);
+ branchPtr(Assembler::BelowOrEqual, dest, ImmWord(BaselineDisabledScript),
+ failure);
+ }
+
+ // Load Baseline jitcode
+ loadPtr(Address(dest, BaselineScript::offsetOfMethod()), dest);
+ loadPtr(Address(dest, JitCode::offsetOfCode()), dest);
+}
+
+void MacroAssembler::loadBaselineFramePtr(Register framePtr, Register dest) {
+ if (framePtr != dest) {
+ movePtr(framePtr, dest);
+ }
+ subPtr(Imm32(BaselineFrame::Size()), dest);
+}
+
+static const uint8_t* ContextInlinedICScriptPtr(CompileRuntime* rt) {
+ return (static_cast<const uint8_t*>(rt->mainContextPtr()) +
+ JSContext::offsetOfInlinedICScript());
+}
+
+void MacroAssembler::storeICScriptInJSContext(Register icScript) {
+ storePtr(icScript, AbsoluteAddress(ContextInlinedICScriptPtr(runtime())));
+}
+
+void MacroAssembler::handleFailure() {
+ // Re-entry code is irrelevant because the exception will leave the
+ // running function and never come back
+ TrampolinePtr excTail = runtime()->jitRuntime()->getExceptionTail();
+ jump(excTail);
+}
+
+void MacroAssembler::assumeUnreachable(const char* output) {
+#ifdef JS_MASM_VERBOSE
+ if (!IsCompilingWasm()) {
+ AllocatableRegisterSet regs(RegisterSet::Volatile());
+ LiveRegisterSet save(regs.asLiveSet());
+ PushRegsInMask(save);
+ Register temp = regs.takeAnyGeneral();
+
+ using Fn = void (*)(const char* output);
+ setupUnalignedABICall(temp);
+ movePtr(ImmPtr(output), temp);
+ passABIArg(temp);
+ callWithABI<Fn, AssumeUnreachable>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ PopRegsInMask(save);
+ }
+#endif
+
+ breakpoint();
+}
+
+void MacroAssembler::printf(const char* output) {
+#ifdef JS_MASM_VERBOSE
+ AllocatableRegisterSet regs(RegisterSet::Volatile());
+ LiveRegisterSet save(regs.asLiveSet());
+ PushRegsInMask(save);
+
+ Register temp = regs.takeAnyGeneral();
+
+ using Fn = void (*)(const char* output);
+ setupUnalignedABICall(temp);
+ movePtr(ImmPtr(output), temp);
+ passABIArg(temp);
+ callWithABI<Fn, Printf0>();
+
+ PopRegsInMask(save);
+#endif
+}
+
+void MacroAssembler::printf(const char* output, Register value) {
+#ifdef JS_MASM_VERBOSE
+ AllocatableRegisterSet regs(RegisterSet::Volatile());
+ LiveRegisterSet save(regs.asLiveSet());
+ PushRegsInMask(save);
+
+ regs.takeUnchecked(value);
+
+ Register temp = regs.takeAnyGeneral();
+
+ using Fn = void (*)(const char* output, uintptr_t value);
+ setupUnalignedABICall(temp);
+ movePtr(ImmPtr(output), temp);
+ passABIArg(temp);
+ passABIArg(value);
+ callWithABI<Fn, Printf1>();
+
+ PopRegsInMask(save);
+#endif
+}
+
+void MacroAssembler::convertInt32ValueToDouble(ValueOperand val) {
+ Label done;
+ branchTestInt32(Assembler::NotEqual, val, &done);
+ unboxInt32(val, val.scratchReg());
+ ScratchDoubleScope fpscratch(*this);
+ convertInt32ToDouble(val.scratchReg(), fpscratch);
+ boxDouble(fpscratch, val, fpscratch);
+ bind(&done);
+}
+
+void MacroAssembler::convertValueToFloatingPoint(ValueOperand value,
+ FloatRegister output,
+ Label* fail,
+ MIRType outputType) {
+ Label isDouble, isInt32, isBool, isNull, done;
+
+ {
+ ScratchTagScope tag(*this, value);
+ splitTagForTest(value, tag);
+
+ branchTestDouble(Assembler::Equal, tag, &isDouble);
+ branchTestInt32(Assembler::Equal, tag, &isInt32);
+ branchTestBoolean(Assembler::Equal, tag, &isBool);
+ branchTestNull(Assembler::Equal, tag, &isNull);
+ branchTestUndefined(Assembler::NotEqual, tag, fail);
+ }
+
+ // fall-through: undefined
+ loadConstantFloatingPoint(GenericNaN(), float(GenericNaN()), output,
+ outputType);
+ jump(&done);
+
+ bind(&isNull);
+ loadConstantFloatingPoint(0.0, 0.0f, output, outputType);
+ jump(&done);
+
+ bind(&isBool);
+ boolValueToFloatingPoint(value, output, outputType);
+ jump(&done);
+
+ bind(&isInt32);
+ int32ValueToFloatingPoint(value, output, outputType);
+ jump(&done);
+
+ // On some non-multiAlias platforms, unboxDouble may use the scratch register,
+ // so do not merge code paths here.
+ bind(&isDouble);
+ if (outputType == MIRType::Float32 && hasMultiAlias()) {
+ ScratchDoubleScope tmp(*this);
+ unboxDouble(value, tmp);
+ convertDoubleToFloat32(tmp, output);
+ } else {
+ FloatRegister tmp = output.asDouble();
+ unboxDouble(value, tmp);
+ if (outputType == MIRType::Float32) {
+ convertDoubleToFloat32(tmp, output);
+ }
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::outOfLineTruncateSlow(FloatRegister src, Register dest,
+ bool widenFloatToDouble,
+ bool compilingWasm,
+ wasm::BytecodeOffset callOffset) {
+ if (compilingWasm) {
+ Push(InstanceReg);
+ }
+ int32_t framePushedAfterInstance = framePushed();
+
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \
+ defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_LOONG64)
+ ScratchDoubleScope fpscratch(*this);
+ if (widenFloatToDouble) {
+ convertFloat32ToDouble(src, fpscratch);
+ src = fpscratch;
+ }
+#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ FloatRegister srcSingle;
+ if (widenFloatToDouble) {
+ MOZ_ASSERT(src.isSingle());
+ srcSingle = src;
+ src = src.asDouble();
+ Push(srcSingle);
+ convertFloat32ToDouble(srcSingle, src);
+ }
+#else
+ // Also see below
+ MOZ_CRASH("MacroAssembler platform hook: outOfLineTruncateSlow");
+#endif
+
+ MOZ_ASSERT(src.isDouble());
+
+ if (compilingWasm) {
+ int32_t instanceOffset = framePushed() - framePushedAfterInstance;
+ setupWasmABICall();
+ passABIArg(src, MoveOp::DOUBLE);
+ callWithABI(callOffset, wasm::SymbolicAddress::ToInt32,
+ mozilla::Some(instanceOffset));
+ } else {
+ using Fn = int32_t (*)(double);
+ setupUnalignedABICall(dest);
+ passABIArg(src, MoveOp::DOUBLE);
+ callWithABI<Fn, JS::ToInt32>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+ }
+ storeCallInt32Result(dest);
+
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \
+ defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_LOONG64)
+ // Nothing
+#elif defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+ if (widenFloatToDouble) {
+ Pop(srcSingle);
+ }
+#else
+ MOZ_CRASH("MacroAssembler platform hook: outOfLineTruncateSlow");
+#endif
+
+ if (compilingWasm) {
+ Pop(InstanceReg);
+ }
+}
+
+void MacroAssembler::convertDoubleToInt(FloatRegister src, Register output,
+ FloatRegister temp, Label* truncateFail,
+ Label* fail,
+ IntConversionBehavior behavior) {
+ switch (behavior) {
+ case IntConversionBehavior::Normal:
+ case IntConversionBehavior::NegativeZeroCheck:
+ convertDoubleToInt32(
+ src, output, fail,
+ behavior == IntConversionBehavior::NegativeZeroCheck);
+ break;
+ case IntConversionBehavior::Truncate:
+ branchTruncateDoubleMaybeModUint32(src, output,
+ truncateFail ? truncateFail : fail);
+ break;
+ case IntConversionBehavior::TruncateNoWrap:
+ branchTruncateDoubleToInt32(src, output,
+ truncateFail ? truncateFail : fail);
+ break;
+ case IntConversionBehavior::ClampToUint8:
+ // Clamping clobbers the input register, so use a temp.
+ if (src != temp) {
+ moveDouble(src, temp);
+ }
+ clampDoubleToUint8(temp, output);
+ break;
+ }
+}
+
+void MacroAssembler::convertValueToInt(
+ ValueOperand value, Label* handleStringEntry, Label* handleStringRejoin,
+ Label* truncateDoubleSlow, Register stringReg, FloatRegister temp,
+ Register output, Label* fail, IntConversionBehavior behavior,
+ IntConversionInputKind conversion) {
+ Label done, isInt32, isBool, isDouble, isNull, isString;
+
+ bool handleStrings = (behavior == IntConversionBehavior::Truncate ||
+ behavior == IntConversionBehavior::ClampToUint8) &&
+ handleStringEntry && handleStringRejoin;
+
+ MOZ_ASSERT_IF(handleStrings, conversion == IntConversionInputKind::Any);
+
+ {
+ ScratchTagScope tag(*this, value);
+ splitTagForTest(value, tag);
+
+ branchTestInt32(Equal, tag, &isInt32);
+ if (conversion == IntConversionInputKind::Any ||
+ conversion == IntConversionInputKind::NumbersOrBoolsOnly) {
+ branchTestBoolean(Equal, tag, &isBool);
+ }
+ branchTestDouble(Equal, tag, &isDouble);
+
+ if (conversion == IntConversionInputKind::Any) {
+ // If we are not truncating, we fail for anything that's not
+ // null. Otherwise we might be able to handle strings and undefined.
+ switch (behavior) {
+ case IntConversionBehavior::Normal:
+ case IntConversionBehavior::NegativeZeroCheck:
+ branchTestNull(Assembler::NotEqual, tag, fail);
+ break;
+
+ case IntConversionBehavior::Truncate:
+ case IntConversionBehavior::TruncateNoWrap:
+ case IntConversionBehavior::ClampToUint8:
+ branchTestNull(Equal, tag, &isNull);
+ if (handleStrings) {
+ branchTestString(Equal, tag, &isString);
+ }
+ branchTestUndefined(Assembler::NotEqual, tag, fail);
+ break;
+ }
+ } else {
+ jump(fail);
+ }
+ }
+
+ // The value is null or undefined in truncation contexts - just emit 0.
+ if (conversion == IntConversionInputKind::Any) {
+ if (isNull.used()) {
+ bind(&isNull);
+ }
+ mov(ImmWord(0), output);
+ jump(&done);
+ }
+
+ // |output| needs to be different from |stringReg| to load string indices.
+ bool handleStringIndices = handleStrings && output != stringReg;
+
+ // First try loading a string index. If that fails, try converting a string
+ // into a double, then jump to the double case.
+ Label handleStringIndex;
+ if (handleStrings) {
+ bind(&isString);
+ unboxString(value, stringReg);
+ if (handleStringIndices) {
+ loadStringIndexValue(stringReg, output, handleStringEntry);
+ jump(&handleStringIndex);
+ } else {
+ jump(handleStringEntry);
+ }
+ }
+
+ // Try converting double into integer.
+ if (isDouble.used() || handleStrings) {
+ if (isDouble.used()) {
+ bind(&isDouble);
+ unboxDouble(value, temp);
+ }
+
+ if (handleStrings) {
+ bind(handleStringRejoin);
+ }
+
+ convertDoubleToInt(temp, output, temp, truncateDoubleSlow, fail, behavior);
+ jump(&done);
+ }
+
+ // Just unbox a bool, the result is 0 or 1.
+ if (isBool.used()) {
+ bind(&isBool);
+ unboxBoolean(value, output);
+ jump(&done);
+ }
+
+ // Integers can be unboxed.
+ if (isInt32.used() || handleStringIndices) {
+ if (isInt32.used()) {
+ bind(&isInt32);
+ unboxInt32(value, output);
+ }
+
+ if (handleStringIndices) {
+ bind(&handleStringIndex);
+ }
+
+ if (behavior == IntConversionBehavior::ClampToUint8) {
+ clampIntToUint8(output);
+ }
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::finish() {
+ if (failureLabel_.used()) {
+ bind(&failureLabel_);
+ handleFailure();
+ }
+
+ MacroAssemblerSpecific::finish();
+
+ MOZ_RELEASE_ASSERT(
+ size() <= MaxCodeBytesPerProcess,
+ "AssemblerBuffer should ensure we don't exceed MaxCodeBytesPerProcess");
+
+ if (bytesNeeded() > MaxCodeBytesPerProcess) {
+ setOOM();
+ }
+}
+
+void MacroAssembler::link(JitCode* code) {
+ MOZ_ASSERT(!oom());
+ linkProfilerCallSites(code);
+}
+
+MacroAssembler::AutoProfilerCallInstrumentation::
+ AutoProfilerCallInstrumentation(MacroAssembler& masm) {
+ if (!masm.emitProfilingInstrumentation_) {
+ return;
+ }
+
+ Register reg = CallTempReg0;
+ Register reg2 = CallTempReg1;
+ masm.push(reg);
+ masm.push(reg2);
+
+ CodeOffset label = masm.movWithPatch(ImmWord(uintptr_t(-1)), reg);
+ masm.loadJSContext(reg2);
+ masm.loadPtr(Address(reg2, offsetof(JSContext, profilingActivation_)), reg2);
+ masm.storePtr(reg,
+ Address(reg2, JitActivation::offsetOfLastProfilingCallSite()));
+
+ masm.appendProfilerCallSite(label);
+
+ masm.pop(reg2);
+ masm.pop(reg);
+}
+
+void MacroAssembler::linkProfilerCallSites(JitCode* code) {
+ for (size_t i = 0; i < profilerCallSites_.length(); i++) {
+ CodeOffset offset = profilerCallSites_[i];
+ CodeLocationLabel location(code, offset);
+ PatchDataWithValueCheck(location, ImmPtr(location.raw()),
+ ImmPtr((void*)-1));
+ }
+}
+
+void MacroAssembler::alignJitStackBasedOnNArgs(Register nargs,
+ bool countIncludesThis) {
+ // The stack should already be aligned to the size of a value.
+ assertStackAlignment(sizeof(Value), 0);
+
+ static_assert(JitStackValueAlignment == 1 || JitStackValueAlignment == 2,
+ "JitStackValueAlignment is either 1 or 2.");
+ if (JitStackValueAlignment == 1) {
+ return;
+ }
+ // A jit frame is composed of the following:
+ //
+ // [padding?] [argN] .. [arg1] [this] [[argc] [callee] [descr] [raddr]]
+ // \________JitFrameLayout_________/
+ // (The stack grows this way --->)
+ //
+ // We want to ensure that |raddr|, the return address, is 16-byte aligned.
+ // (Note: if 8-byte alignment was sufficient, we would have already
+ // returned above.)
+
+ // JitFrameLayout does not affect the alignment, so we can ignore it.
+ static_assert(sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "JitFrameLayout doesn't affect stack alignment");
+
+ // Therefore, we need to ensure that |this| is aligned.
+ // This implies that |argN| must be aligned if N is even,
+ // and offset by |sizeof(Value)| if N is odd.
+
+ // Depending on the context of the caller, it may be easier to pass in a
+ // register that has already been modified to include |this|. If that is the
+ // case, we want to flip the direction of the test.
+ Assembler::Condition condition =
+ countIncludesThis ? Assembler::NonZero : Assembler::Zero;
+
+ Label alignmentIsOffset, end;
+ branchTestPtr(condition, nargs, Imm32(1), &alignmentIsOffset);
+
+ // |argN| should be aligned to 16 bytes.
+ andToStackPtr(Imm32(~(JitStackAlignment - 1)));
+ jump(&end);
+
+ // |argN| should be offset by 8 bytes from 16-byte alignment.
+ // We already know that it is 8-byte aligned, so the only possibilities are:
+ // a) It is 16-byte aligned, and we must offset it by 8 bytes.
+ // b) It is not 16-byte aligned, and therefore already has the right offset.
+ // Therefore, we test to see if it is 16-byte aligned, and adjust it if it is.
+ bind(&alignmentIsOffset);
+ branchTestStackPtr(Assembler::NonZero, Imm32(JitStackAlignment - 1), &end);
+ subFromStackPtr(Imm32(sizeof(Value)));
+
+ bind(&end);
+}
+
+void MacroAssembler::alignJitStackBasedOnNArgs(uint32_t argc,
+ bool countIncludesThis) {
+ // The stack should already be aligned to the size of a value.
+ assertStackAlignment(sizeof(Value), 0);
+
+ static_assert(JitStackValueAlignment == 1 || JitStackValueAlignment == 2,
+ "JitStackValueAlignment is either 1 or 2.");
+ if (JitStackValueAlignment == 1) {
+ return;
+ }
+
+ // See above for full explanation.
+ uint32_t nArgs = argc + !countIncludesThis;
+ if (nArgs % 2 == 0) {
+ // |argN| should be 16-byte aligned
+ andToStackPtr(Imm32(~(JitStackAlignment - 1)));
+ } else {
+ // |argN| must be 16-byte aligned if argc is even,
+ // and offset by 8 if argc is odd.
+ Label end;
+ branchTestStackPtr(Assembler::NonZero, Imm32(JitStackAlignment - 1), &end);
+ subFromStackPtr(Imm32(sizeof(Value)));
+ bind(&end);
+ assertStackAlignment(JitStackAlignment, sizeof(Value));
+ }
+}
+
+// ===============================================================
+
+MacroAssembler::MacroAssembler(TempAllocator& alloc,
+ CompileRuntime* maybeRuntime,
+ CompileRealm* maybeRealm)
+ : maybeRuntime_(maybeRuntime),
+ maybeRealm_(maybeRealm),
+ wasmMaxOffsetGuardLimit_(0),
+ framePushed_(0),
+#ifdef DEBUG
+ inCall_(false),
+#endif
+ dynamicAlignment_(false),
+ emitProfilingInstrumentation_(false) {
+ moveResolver_.setAllocator(alloc);
+}
+
+StackMacroAssembler::StackMacroAssembler(JSContext* cx, TempAllocator& alloc)
+ : MacroAssembler(alloc, CompileRuntime::get(cx->runtime()),
+ CompileRealm::get(cx->realm())) {}
+
+IonHeapMacroAssembler::IonHeapMacroAssembler(TempAllocator& alloc,
+ CompileRealm* realm)
+ : MacroAssembler(alloc, realm->runtime(), realm) {
+ MOZ_ASSERT(CurrentThreadIsIonCompiling());
+}
+
+WasmMacroAssembler::WasmMacroAssembler(TempAllocator& alloc, bool limitedSize)
+ : MacroAssembler(alloc) {
+#if defined(JS_CODEGEN_ARM64)
+ // Stubs + builtins + the baseline compiler all require the native SP,
+ // not the PSP.
+ SetStackPointer64(sp);
+#endif
+ if (!limitedSize) {
+ setUnlimitedBuffer();
+ }
+}
+
+WasmMacroAssembler::WasmMacroAssembler(TempAllocator& alloc,
+ const wasm::ModuleEnvironment& env,
+ bool limitedSize)
+ : MacroAssembler(alloc) {
+#if defined(JS_CODEGEN_ARM64)
+ // Stubs + builtins + the baseline compiler all require the native SP,
+ // not the PSP.
+ SetStackPointer64(sp);
+#endif
+ setWasmMaxOffsetGuardLimit(
+ wasm::GetMaxOffsetGuardLimit(env.hugeMemoryEnabled()));
+ if (!limitedSize) {
+ setUnlimitedBuffer();
+ }
+}
+
+bool MacroAssembler::icBuildOOLFakeExitFrame(void* fakeReturnAddr,
+ AutoSaveLiveRegisters& save) {
+ return buildOOLFakeExitFrame(fakeReturnAddr);
+}
+
+#ifndef JS_CODEGEN_ARM64
+void MacroAssembler::subFromStackPtr(Register reg) {
+ subPtr(reg, getStackPointer());
+}
+#endif // JS_CODEGEN_ARM64
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// Stack manipulation functions.
+
+void MacroAssembler::PushRegsInMask(LiveGeneralRegisterSet set) {
+ PushRegsInMask(LiveRegisterSet(set.set(), FloatRegisterSet()));
+}
+
+void MacroAssembler::PopRegsInMask(LiveRegisterSet set) {
+ PopRegsInMaskIgnore(set, LiveRegisterSet());
+}
+
+void MacroAssembler::PopRegsInMask(LiveGeneralRegisterSet set) {
+ PopRegsInMask(LiveRegisterSet(set.set(), FloatRegisterSet()));
+}
+
+void MacroAssembler::Push(PropertyKey key, Register scratchReg) {
+ if (key.isGCThing()) {
+ // If we're pushing a gcthing, then we can't just push the tagged key
+ // value since the GC won't have any idea that the push instruction
+ // carries a reference to a gcthing. Need to unpack the pointer,
+ // push it using ImmGCPtr, and then rematerialize the PropertyKey at
+ // runtime.
+
+ if (key.isString()) {
+ JSString* str = key.toString();
+ MOZ_ASSERT((uintptr_t(str) & PropertyKey::TypeMask) == 0);
+ static_assert(PropertyKey::StringTypeTag == 0,
+ "need to orPtr StringTypeTag if it's not 0");
+ Push(ImmGCPtr(str));
+ } else {
+ MOZ_ASSERT(key.isSymbol());
+ movePropertyKey(key, scratchReg);
+ Push(scratchReg);
+ }
+ } else {
+ MOZ_ASSERT(key.isInt());
+ Push(ImmWord(key.asRawBits()));
+ }
+}
+
+void MacroAssembler::movePropertyKey(PropertyKey key, Register dest) {
+ if (key.isGCThing()) {
+ // See comment in |Push(PropertyKey, ...)| above for an explanation.
+ if (key.isString()) {
+ JSString* str = key.toString();
+ MOZ_ASSERT((uintptr_t(str) & PropertyKey::TypeMask) == 0);
+ static_assert(PropertyKey::StringTypeTag == 0,
+ "need to orPtr JSID_TYPE_STRING tag if it's not 0");
+ movePtr(ImmGCPtr(str), dest);
+ } else {
+ MOZ_ASSERT(key.isSymbol());
+ JS::Symbol* sym = key.toSymbol();
+ movePtr(ImmGCPtr(sym), dest);
+ orPtr(Imm32(PropertyKey::SymbolTypeTag), dest);
+ }
+ } else {
+ MOZ_ASSERT(key.isInt());
+ movePtr(ImmWord(key.asRawBits()), dest);
+ }
+}
+
+void MacroAssembler::Push(TypedOrValueRegister v) {
+ if (v.hasValue()) {
+ Push(v.valueReg());
+ } else if (IsFloatingPointType(v.type())) {
+ FloatRegister reg = v.typedReg().fpu();
+ if (v.type() == MIRType::Float32) {
+ ScratchDoubleScope fpscratch(*this);
+ convertFloat32ToDouble(reg, fpscratch);
+ PushBoxed(fpscratch);
+ } else {
+ PushBoxed(reg);
+ }
+ } else {
+ Push(ValueTypeFromMIRType(v.type()), v.typedReg().gpr());
+ }
+}
+
+void MacroAssembler::Push(const ConstantOrRegister& v) {
+ if (v.constant()) {
+ Push(v.value());
+ } else {
+ Push(v.reg());
+ }
+}
+
+void MacroAssembler::Push(const Address& addr) {
+ push(addr);
+ framePushed_ += sizeof(uintptr_t);
+}
+
+void MacroAssembler::Push(const ValueOperand& val) {
+ pushValue(val);
+ framePushed_ += sizeof(Value);
+}
+
+void MacroAssembler::Push(const Value& val) {
+ pushValue(val);
+ framePushed_ += sizeof(Value);
+}
+
+void MacroAssembler::Push(JSValueType type, Register reg) {
+ pushValue(type, reg);
+ framePushed_ += sizeof(Value);
+}
+
+void MacroAssembler::Push(const Register64 reg) {
+#if JS_BITS_PER_WORD == 64
+ Push(reg.reg);
+#else
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Big-endian not supported.");
+ Push(reg.high);
+ Push(reg.low);
+#endif
+}
+
+void MacroAssembler::PushEmptyRooted(VMFunctionData::RootType rootType) {
+ switch (rootType) {
+ case VMFunctionData::RootNone:
+ MOZ_CRASH("Handle must have root type");
+ case VMFunctionData::RootObject:
+ case VMFunctionData::RootString:
+ case VMFunctionData::RootCell:
+ case VMFunctionData::RootBigInt:
+ Push(ImmPtr(nullptr));
+ break;
+ case VMFunctionData::RootValue:
+ Push(UndefinedValue());
+ break;
+ case VMFunctionData::RootId:
+ Push(ImmWord(JS::PropertyKey::Void().asRawBits()));
+ break;
+ }
+}
+
+void MacroAssembler::popRooted(VMFunctionData::RootType rootType,
+ Register cellReg, const ValueOperand& valueReg) {
+ switch (rootType) {
+ case VMFunctionData::RootNone:
+ MOZ_CRASH("Handle must have root type");
+ case VMFunctionData::RootObject:
+ case VMFunctionData::RootString:
+ case VMFunctionData::RootCell:
+ case VMFunctionData::RootId:
+ case VMFunctionData::RootBigInt:
+ Pop(cellReg);
+ break;
+ case VMFunctionData::RootValue:
+ Pop(valueReg);
+ break;
+ }
+}
+
+void MacroAssembler::adjustStack(int amount) {
+ if (amount > 0) {
+ freeStack(amount);
+ } else if (amount < 0) {
+ reserveStack(-amount);
+ }
+}
+
+void MacroAssembler::freeStack(uint32_t amount) {
+ MOZ_ASSERT(amount <= framePushed_);
+ if (amount) {
+ addToStackPtr(Imm32(amount));
+ }
+ framePushed_ -= amount;
+}
+
+void MacroAssembler::freeStack(Register amount) { addToStackPtr(amount); }
+
+// ===============================================================
+// ABI function calls.
+template <class ABIArgGeneratorT>
+void MacroAssembler::setupABICallHelper() {
+#ifdef DEBUG
+ MOZ_ASSERT(!inCall_);
+ inCall_ = true;
+#endif
+
+#ifdef JS_SIMULATOR
+ signature_ = 0;
+#endif
+
+ // Reinitialize the ABIArg generator.
+ abiArgs_ = ABIArgGeneratorT();
+
+#if defined(JS_CODEGEN_ARM)
+ // On ARM, we need to know what ABI we are using, either in the
+ // simulator, or based on the configure flags.
+# if defined(JS_SIMULATOR_ARM)
+ abiArgs_.setUseHardFp(UseHardFpABI());
+# elif defined(JS_CODEGEN_ARM_HARDFP)
+ abiArgs_.setUseHardFp(true);
+# else
+ abiArgs_.setUseHardFp(false);
+# endif
+#endif
+
+#if defined(JS_CODEGEN_MIPS32)
+ // On MIPS, the system ABI use general registers pairs to encode double
+ // arguments, after one or 2 integer-like arguments. Unfortunately, the
+ // Lowering phase is not capable to express it at the moment. So we enforce
+ // the system ABI here.
+ abiArgs_.enforceO32ABI();
+#endif
+}
+
+void MacroAssembler::setupNativeABICall() {
+ setupABICallHelper<ABIArgGenerator>();
+}
+
+void MacroAssembler::setupWasmABICall() {
+ MOZ_ASSERT(IsCompilingWasm(), "non-wasm should use setupAlignedABICall");
+ setupABICallHelper<WasmABIArgGenerator>();
+
+#if defined(JS_CODEGEN_ARM)
+ // The builtin thunk does the FP -> GPR moving on soft-FP, so
+ // use hard fp unconditionally.
+ abiArgs_.setUseHardFp(true);
+#endif
+ dynamicAlignment_ = false;
+}
+
+void MacroAssembler::setupAlignedABICall() {
+ MOZ_ASSERT(!IsCompilingWasm(), "wasm should use setupWasmABICall");
+ setupNativeABICall();
+ dynamicAlignment_ = false;
+}
+
+void MacroAssembler::passABIArg(const MoveOperand& from, MoveOp::Type type) {
+ MOZ_ASSERT(inCall_);
+ appendSignatureType(type);
+
+ ABIArg arg;
+ switch (type) {
+ case MoveOp::FLOAT32:
+ arg = abiArgs_.next(MIRType::Float32);
+ break;
+ case MoveOp::DOUBLE:
+ arg = abiArgs_.next(MIRType::Double);
+ break;
+ case MoveOp::GENERAL:
+ arg = abiArgs_.next(MIRType::Pointer);
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+
+ MoveOperand to(*this, arg);
+ if (from == to) {
+ return;
+ }
+
+ if (oom()) {
+ return;
+ }
+ propagateOOM(moveResolver_.addMove(from, to, type));
+}
+
+void MacroAssembler::callWithABINoProfiler(void* fun, MoveOp::Type result,
+ CheckUnsafeCallWithABI check) {
+ appendSignatureType(result);
+#ifdef JS_SIMULATOR
+ fun = Simulator::RedirectNativeFunction(fun, signature());
+#endif
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+
+#ifdef DEBUG
+ if (check == CheckUnsafeCallWithABI::Check) {
+ push(ReturnReg);
+ loadJSContext(ReturnReg);
+ Address flagAddr(ReturnReg, JSContext::offsetOfInUnsafeCallWithABI());
+ store32(Imm32(1), flagAddr);
+ pop(ReturnReg);
+ // On arm64, SP may be < PSP now (that's OK).
+ // eg testcase: tests/bug1375074.js
+ }
+#endif
+
+ call(ImmPtr(fun));
+
+ callWithABIPost(stackAdjust, result);
+
+#ifdef DEBUG
+ if (check == CheckUnsafeCallWithABI::Check) {
+ Label ok;
+ push(ReturnReg);
+ loadJSContext(ReturnReg);
+ Address flagAddr(ReturnReg, JSContext::offsetOfInUnsafeCallWithABI());
+ branch32(Assembler::Equal, flagAddr, Imm32(0), &ok);
+ assumeUnreachable("callWithABI: callee did not use AutoUnsafeCallWithABI");
+ bind(&ok);
+ pop(ReturnReg);
+ // On arm64, SP may be < PSP now (that's OK).
+ // eg testcase: tests/bug1375074.js
+ }
+#endif
+}
+
+CodeOffset MacroAssembler::callWithABI(wasm::BytecodeOffset bytecode,
+ wasm::SymbolicAddress imm,
+ mozilla::Maybe<int32_t> instanceOffset,
+ MoveOp::Type result) {
+ MOZ_ASSERT(wasm::NeedsBuiltinThunk(imm));
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust, /* callFromWasm = */ true);
+
+ // The instance register is used in builtin thunks and must be set.
+ if (instanceOffset) {
+ loadPtr(Address(getStackPointer(), *instanceOffset + stackAdjust),
+ InstanceReg);
+ } else {
+ MOZ_CRASH("instanceOffset is Nothing only for unsupported abi calls.");
+ }
+ CodeOffset raOffset = call(
+ wasm::CallSiteDesc(bytecode.offset(), wasm::CallSite::Symbolic), imm);
+
+ callWithABIPost(stackAdjust, result, /* callFromWasm = */ true);
+
+ return raOffset;
+}
+
+void MacroAssembler::callDebugWithABI(wasm::SymbolicAddress imm,
+ MoveOp::Type result) {
+ MOZ_ASSERT(!wasm::NeedsBuiltinThunk(imm));
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust, /* callFromWasm = */ false);
+ call(imm);
+ callWithABIPost(stackAdjust, result, /* callFromWasm = */ false);
+}
+
+// ===============================================================
+// Exit frame footer.
+
+void MacroAssembler::linkExitFrame(Register cxreg, Register scratch) {
+ loadPtr(Address(cxreg, JSContext::offsetOfActivation()), scratch);
+ storeStackPtr(Address(scratch, JitActivation::offsetOfPackedExitFP()));
+}
+
+// ===============================================================
+// Simple value-shuffling helpers, to hide MoveResolver verbosity
+// in common cases.
+
+void MacroAssembler::moveRegPair(Register src0, Register src1, Register dst0,
+ Register dst1, MoveOp::Type type) {
+ MoveResolver& moves = moveResolver();
+ if (src0 != dst0) {
+ propagateOOM(moves.addMove(MoveOperand(src0), MoveOperand(dst0), type));
+ }
+ if (src1 != dst1) {
+ propagateOOM(moves.addMove(MoveOperand(src1), MoveOperand(dst1), type));
+ }
+ propagateOOM(moves.resolve());
+ if (oom()) {
+ return;
+ }
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moves);
+ emitter.finish();
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::pow32(Register base, Register power, Register dest,
+ Register temp1, Register temp2, Label* onOver) {
+ // Inline int32-specialized implementation of js::powi with overflow
+ // detection.
+
+ move32(Imm32(1), dest); // result = 1
+
+ // x^y where x == 1 returns 1 for any y.
+ Label done;
+ branch32(Assembler::Equal, base, Imm32(1), &done);
+
+ move32(base, temp1); // runningSquare = x
+ move32(power, temp2); // n = y
+
+ // x^y where y < 0 returns a non-int32 value for any x != 1. Except when y is
+ // large enough so that the result is no longer representable as a double with
+ // fractional parts. We can't easily determine when y is too large, so we bail
+ // here.
+ // Note: it's important for this condition to match the code in CacheIR.cpp
+ // (CanAttachInt32Pow) to prevent failure loops.
+ Label start;
+ branchTest32(Assembler::NotSigned, power, power, &start);
+ jump(onOver);
+
+ Label loop;
+ bind(&loop);
+
+ // runningSquare *= runningSquare
+ branchMul32(Assembler::Overflow, temp1, temp1, onOver);
+
+ bind(&start);
+
+ // if ((n & 1) != 0) result *= runningSquare
+ Label even;
+ branchTest32(Assembler::Zero, temp2, Imm32(1), &even);
+ branchMul32(Assembler::Overflow, temp1, dest, onOver);
+ bind(&even);
+
+ // n >>= 1
+ // if (n == 0) return result
+ branchRshift32(Assembler::NonZero, Imm32(1), temp2, &loop);
+
+ bind(&done);
+}
+
+void MacroAssembler::signInt32(Register input, Register output) {
+ MOZ_ASSERT(input != output);
+
+ Label done;
+ move32(input, output);
+ rshift32Arithmetic(Imm32(31), output);
+ branch32(Assembler::LessThanOrEqual, input, Imm32(0), &done);
+ move32(Imm32(1), output);
+ bind(&done);
+}
+
+void MacroAssembler::signDouble(FloatRegister input, FloatRegister output) {
+ MOZ_ASSERT(input != output);
+
+ Label done, zeroOrNaN, negative;
+ loadConstantDouble(0.0, output);
+ branchDouble(Assembler::DoubleEqualOrUnordered, input, output, &zeroOrNaN);
+ branchDouble(Assembler::DoubleLessThan, input, output, &negative);
+
+ loadConstantDouble(1.0, output);
+ jump(&done);
+
+ bind(&negative);
+ loadConstantDouble(-1.0, output);
+ jump(&done);
+
+ bind(&zeroOrNaN);
+ moveDouble(input, output);
+
+ bind(&done);
+}
+
+void MacroAssembler::signDoubleToInt32(FloatRegister input, Register output,
+ FloatRegister temp, Label* fail) {
+ MOZ_ASSERT(input != temp);
+
+ Label done, zeroOrNaN, negative;
+ loadConstantDouble(0.0, temp);
+ branchDouble(Assembler::DoubleEqualOrUnordered, input, temp, &zeroOrNaN);
+ branchDouble(Assembler::DoubleLessThan, input, temp, &negative);
+
+ move32(Imm32(1), output);
+ jump(&done);
+
+ bind(&negative);
+ move32(Imm32(-1), output);
+ jump(&done);
+
+ // Fail for NaN and negative zero.
+ bind(&zeroOrNaN);
+ branchDouble(Assembler::DoubleUnordered, input, input, fail);
+
+ // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0
+ // is -Infinity instead of Infinity.
+ loadConstantDouble(1.0, temp);
+ divDouble(input, temp);
+ branchDouble(Assembler::DoubleLessThan, temp, input, fail);
+ move32(Imm32(0), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::randomDouble(Register rng, FloatRegister dest,
+ Register64 temp0, Register64 temp1) {
+ using mozilla::non_crypto::XorShift128PlusRNG;
+
+ static_assert(
+ sizeof(XorShift128PlusRNG) == 2 * sizeof(uint64_t),
+ "Code below assumes XorShift128PlusRNG contains two uint64_t values");
+
+ Address state0Addr(rng, XorShift128PlusRNG::offsetOfState0());
+ Address state1Addr(rng, XorShift128PlusRNG::offsetOfState1());
+
+ Register64 s0Reg = temp0;
+ Register64 s1Reg = temp1;
+
+ // uint64_t s1 = mState[0];
+ load64(state0Addr, s1Reg);
+
+ // s1 ^= s1 << 23;
+ move64(s1Reg, s0Reg);
+ lshift64(Imm32(23), s1Reg);
+ xor64(s0Reg, s1Reg);
+
+ // s1 ^= s1 >> 17
+ move64(s1Reg, s0Reg);
+ rshift64(Imm32(17), s1Reg);
+ xor64(s0Reg, s1Reg);
+
+ // const uint64_t s0 = mState[1];
+ load64(state1Addr, s0Reg);
+
+ // mState[0] = s0;
+ store64(s0Reg, state0Addr);
+
+ // s1 ^= s0
+ xor64(s0Reg, s1Reg);
+
+ // s1 ^= s0 >> 26
+ rshift64(Imm32(26), s0Reg);
+ xor64(s0Reg, s1Reg);
+
+ // mState[1] = s1
+ store64(s1Reg, state1Addr);
+
+ // s1 += mState[0]
+ load64(state0Addr, s0Reg);
+ add64(s0Reg, s1Reg);
+
+ // See comment in XorShift128PlusRNG::nextDouble().
+ static constexpr int MantissaBits =
+ mozilla::FloatingPoint<double>::kExponentShift + 1;
+ static constexpr double ScaleInv = double(1) / (1ULL << MantissaBits);
+
+ and64(Imm64((1ULL << MantissaBits) - 1), s1Reg);
+
+ // Note: we know s1Reg isn't signed after the and64 so we can use the faster
+ // convertInt64ToDouble instead of convertUInt64ToDouble.
+ convertInt64ToDouble(s1Reg, dest);
+
+ // dest *= ScaleInv
+ mulDoublePtr(ImmPtr(&ScaleInv), s0Reg.scratchReg(), dest);
+}
+
+void MacroAssembler::sameValueDouble(FloatRegister left, FloatRegister right,
+ FloatRegister temp, Register dest) {
+ Label nonEqual, isSameValue, isNotSameValue;
+ branchDouble(Assembler::DoubleNotEqualOrUnordered, left, right, &nonEqual);
+ {
+ // First, test for being equal to 0.0, which also includes -0.0.
+ loadConstantDouble(0.0, temp);
+ branchDouble(Assembler::DoubleNotEqual, left, temp, &isSameValue);
+
+ // The easiest way to distinguish -0.0 from 0.0 is that 1.0/-0.0
+ // is -Infinity instead of Infinity.
+ Label isNegInf;
+ loadConstantDouble(1.0, temp);
+ divDouble(left, temp);
+ branchDouble(Assembler::DoubleLessThan, temp, left, &isNegInf);
+ {
+ loadConstantDouble(1.0, temp);
+ divDouble(right, temp);
+ branchDouble(Assembler::DoubleGreaterThan, temp, right, &isSameValue);
+ jump(&isNotSameValue);
+ }
+ bind(&isNegInf);
+ {
+ loadConstantDouble(1.0, temp);
+ divDouble(right, temp);
+ branchDouble(Assembler::DoubleLessThan, temp, right, &isSameValue);
+ jump(&isNotSameValue);
+ }
+ }
+ bind(&nonEqual);
+ {
+ // Test if both values are NaN.
+ branchDouble(Assembler::DoubleOrdered, left, left, &isNotSameValue);
+ branchDouble(Assembler::DoubleOrdered, right, right, &isNotSameValue);
+ }
+
+ Label done;
+ bind(&isSameValue);
+ move32(Imm32(1), dest);
+ jump(&done);
+
+ bind(&isNotSameValue);
+ move32(Imm32(0), dest);
+
+ bind(&done);
+}
+
+void MacroAssembler::minMaxArrayInt32(Register array, Register result,
+ Register temp1, Register temp2,
+ Register temp3, bool isMax, Label* fail) {
+ // array must be a packed array. Load its elements.
+ Register elements = temp1;
+ loadPtr(Address(array, NativeObject::offsetOfElements()), elements);
+
+ // Load the length and guard that it is non-zero.
+ Address lengthAddr(elements, ObjectElements::offsetOfInitializedLength());
+ load32(lengthAddr, temp3);
+ branchTest32(Assembler::Zero, temp3, temp3, fail);
+
+ // Compute the address of the last element.
+ Register elementsEnd = temp2;
+ BaseObjectElementIndex elementsEndAddr(elements, temp3,
+ -int32_t(sizeof(Value)));
+ computeEffectiveAddress(elementsEndAddr, elementsEnd);
+
+ // Load the first element into result.
+ fallibleUnboxInt32(Address(elements, 0), result, fail);
+
+ Label loop, done;
+ bind(&loop);
+
+ // Check whether we're done.
+ branchPtr(Assembler::Equal, elements, elementsEnd, &done);
+
+ // If not, advance to the next element and load it.
+ addPtr(Imm32(sizeof(Value)), elements);
+ fallibleUnboxInt32(Address(elements, 0), temp3, fail);
+
+ // Update result if necessary.
+ Assembler::Condition cond =
+ isMax ? Assembler::GreaterThan : Assembler::LessThan;
+ cmp32Move32(cond, temp3, result, temp3, result);
+
+ jump(&loop);
+ bind(&done);
+}
+
+void MacroAssembler::minMaxArrayNumber(Register array, FloatRegister result,
+ FloatRegister floatTemp, Register temp1,
+ Register temp2, bool isMax,
+ Label* fail) {
+ // array must be a packed array. Load its elements.
+ Register elements = temp1;
+ loadPtr(Address(array, NativeObject::offsetOfElements()), elements);
+
+ // Load the length and check if the array is empty.
+ Label isEmpty;
+ Address lengthAddr(elements, ObjectElements::offsetOfInitializedLength());
+ load32(lengthAddr, temp2);
+ branchTest32(Assembler::Zero, temp2, temp2, &isEmpty);
+
+ // Compute the address of the last element.
+ Register elementsEnd = temp2;
+ BaseObjectElementIndex elementsEndAddr(elements, temp2,
+ -int32_t(sizeof(Value)));
+ computeEffectiveAddress(elementsEndAddr, elementsEnd);
+
+ // Load the first element into result.
+ ensureDouble(Address(elements, 0), result, fail);
+
+ Label loop, done;
+ bind(&loop);
+
+ // Check whether we're done.
+ branchPtr(Assembler::Equal, elements, elementsEnd, &done);
+
+ // If not, advance to the next element and load it into floatTemp.
+ addPtr(Imm32(sizeof(Value)), elements);
+ ensureDouble(Address(elements, 0), floatTemp, fail);
+
+ // Update result if necessary.
+ if (isMax) {
+ maxDouble(floatTemp, result, /* handleNaN = */ true);
+ } else {
+ minDouble(floatTemp, result, /* handleNaN = */ true);
+ }
+ jump(&loop);
+
+ // With no arguments, min/max return +Infinity/-Infinity respectively.
+ bind(&isEmpty);
+ if (isMax) {
+ loadConstantDouble(mozilla::NegativeInfinity<double>(), result);
+ } else {
+ loadConstantDouble(mozilla::PositiveInfinity<double>(), result);
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::branchIfNotRegExpPrototypeOptimizable(Register proto,
+ Register temp,
+ Label* fail) {
+ loadJSContext(temp);
+ loadPtr(Address(temp, JSContext::offsetOfRealm()), temp);
+ size_t offset = Realm::offsetOfRegExps() +
+ RegExpRealm::offsetOfOptimizableRegExpPrototypeShape();
+ loadPtr(Address(temp, offset), temp);
+ branchTestObjShapeUnsafe(Assembler::NotEqual, proto, temp, fail);
+}
+
+void MacroAssembler::branchIfNotRegExpInstanceOptimizable(Register regexp,
+ Register temp,
+ Label* label) {
+ loadJSContext(temp);
+ loadPtr(Address(temp, JSContext::offsetOfRealm()), temp);
+ size_t offset = Realm::offsetOfRegExps() +
+ RegExpRealm::offsetOfOptimizableRegExpInstanceShape();
+ loadPtr(Address(temp, offset), temp);
+ branchTestObjShapeUnsafe(Assembler::NotEqual, regexp, temp, label);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::loadFunctionLength(Register func,
+ Register funFlagsAndArgCount,
+ Register output, Label* slowPath) {
+#ifdef DEBUG
+ {
+ // These flags should already have been checked by caller.
+ Label ok;
+ uint32_t FlagsToCheck =
+ FunctionFlags::SELFHOSTLAZY | FunctionFlags::RESOLVED_LENGTH;
+ branchTest32(Assembler::Zero, funFlagsAndArgCount, Imm32(FlagsToCheck),
+ &ok);
+ assumeUnreachable("The function flags should already have been checked.");
+ bind(&ok);
+ }
+#endif // DEBUG
+
+ // NOTE: `funFlagsAndArgCount` and `output` must be allowed to alias.
+
+ // Load the target function's length.
+ Label isInterpreted, isBound, lengthLoaded;
+ branchTest32(Assembler::NonZero, funFlagsAndArgCount,
+ Imm32(FunctionFlags::BOUND_FUN), &isBound);
+ branchTest32(Assembler::NonZero, funFlagsAndArgCount,
+ Imm32(FunctionFlags::BASESCRIPT), &isInterpreted);
+ {
+ // The length property of a native function stored with the flags.
+ move32(funFlagsAndArgCount, output);
+ rshift32(Imm32(JSFunction::ArgCountShift), output);
+ jump(&lengthLoaded);
+ }
+ bind(&isBound);
+ {
+ // Load the length property of a bound function.
+ Address boundLength(func,
+ FunctionExtended::offsetOfBoundFunctionLengthSlot());
+ fallibleUnboxInt32(boundLength, output, slowPath);
+ jump(&lengthLoaded);
+ }
+ bind(&isInterpreted);
+ {
+ // Load the length property of an interpreted function.
+ loadPrivate(Address(func, JSFunction::offsetOfJitInfoOrScript()), output);
+ loadPtr(Address(output, JSScript::offsetOfSharedData()), output);
+ branchTestPtr(Assembler::Zero, output, output, slowPath);
+ loadPtr(Address(output, SharedImmutableScriptData::offsetOfISD()), output);
+ load16ZeroExtend(Address(output, ImmutableScriptData::offsetOfFunLength()),
+ output);
+ }
+ bind(&lengthLoaded);
+}
+
+void MacroAssembler::loadFunctionName(Register func, Register output,
+ ImmGCPtr emptyString, Label* slowPath) {
+ MOZ_ASSERT(func != output);
+
+ // Get the JSFunction flags.
+ load32(Address(func, JSFunction::offsetOfFlagsAndArgCount()), output);
+
+ // If the name was previously resolved, the name property may be shadowed.
+ branchTest32(Assembler::NonZero, output, Imm32(FunctionFlags::RESOLVED_NAME),
+ slowPath);
+
+ Label notBoundTarget, loadName;
+ branchTest32(Assembler::Zero, output, Imm32(FunctionFlags::BOUND_FUN),
+ &notBoundTarget);
+ {
+ // Call into the VM if the target's name atom doesn't contain the bound
+ // function prefix.
+ branchTest32(Assembler::Zero, output,
+ Imm32(FunctionFlags::HAS_BOUND_FUNCTION_NAME_PREFIX),
+ slowPath);
+
+ // Bound functions reuse HAS_GUESSED_ATOM for
+ // HAS_BOUND_FUNCTION_NAME_PREFIX, so skip the guessed atom check below.
+ static_assert(
+ FunctionFlags::HAS_BOUND_FUNCTION_NAME_PREFIX ==
+ FunctionFlags::HAS_GUESSED_ATOM,
+ "HAS_BOUND_FUNCTION_NAME_PREFIX is shared with HAS_GUESSED_ATOM");
+ jump(&loadName);
+ }
+ bind(&notBoundTarget);
+
+ Label noName, done;
+ branchTest32(Assembler::NonZero, output,
+ Imm32(FunctionFlags::HAS_GUESSED_ATOM), &noName);
+
+ bind(&loadName);
+ Address atomAddr(func, JSFunction::offsetOfAtom());
+ branchTestUndefined(Assembler::Equal, atomAddr, &noName);
+ unboxString(atomAddr, output);
+ jump(&done);
+
+ {
+ bind(&noName);
+
+ // An absent name property defaults to the empty string.
+ movePtr(emptyString, output);
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::assertFunctionIsExtended(Register func) {
+#ifdef DEBUG
+ Label extended;
+ branchTestFunctionFlags(func, FunctionFlags::EXTENDED, Assembler::NonZero,
+ &extended);
+ assumeUnreachable("Function is not extended");
+ bind(&extended);
+#endif
+}
+
+void MacroAssembler::branchTestType(Condition cond, Register tag,
+ JSValueType type, Label* label) {
+ switch (type) {
+ case JSVAL_TYPE_DOUBLE:
+ branchTestDouble(cond, tag, label);
+ break;
+ case JSVAL_TYPE_INT32:
+ branchTestInt32(cond, tag, label);
+ break;
+ case JSVAL_TYPE_BOOLEAN:
+ branchTestBoolean(cond, tag, label);
+ break;
+ case JSVAL_TYPE_UNDEFINED:
+ branchTestUndefined(cond, tag, label);
+ break;
+ case JSVAL_TYPE_NULL:
+ branchTestNull(cond, tag, label);
+ break;
+ case JSVAL_TYPE_MAGIC:
+ branchTestMagic(cond, tag, label);
+ break;
+ case JSVAL_TYPE_STRING:
+ branchTestString(cond, tag, label);
+ break;
+ case JSVAL_TYPE_SYMBOL:
+ branchTestSymbol(cond, tag, label);
+ break;
+ case JSVAL_TYPE_BIGINT:
+ branchTestBigInt(cond, tag, label);
+ break;
+ case JSVAL_TYPE_OBJECT:
+ branchTestObject(cond, tag, label);
+ break;
+ default:
+ MOZ_CRASH("Unexpected value type");
+ }
+}
+
+void MacroAssembler::branchTestObjShapeList(
+ Condition cond, Register obj, Register shapeElements, Register shapeScratch,
+ Register endScratch, Register spectreScratch, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ bool needSpectreMitigations = spectreScratch != InvalidReg;
+
+ Label done;
+ Label* onMatch = cond == Assembler::Equal ? label : &done;
+
+ // Load the object's shape pointer into shapeScratch, and prepare to compare
+ // it with the shapes in the list. On 64-bit, we box the shape. On 32-bit,
+ // we only have to compare the 32-bit payload.
+#ifdef JS_PUNBOX64
+ loadPtr(Address(obj, JSObject::offsetOfShape()), endScratch);
+ tagValue(JSVAL_TYPE_PRIVATE_GCTHING, endScratch, ValueOperand(shapeScratch));
+#else
+ loadPtr(Address(obj, JSObject::offsetOfShape()), shapeScratch);
+#endif
+
+ // Compute end pointer.
+ Address lengthAddr(shapeElements,
+ ObjectElements::offsetOfInitializedLength());
+ load32(lengthAddr, endScratch);
+ BaseObjectElementIndex endPtrAddr(shapeElements, endScratch);
+ computeEffectiveAddress(endPtrAddr, endScratch);
+
+ Label loop;
+ bind(&loop);
+
+ // Compare the object's shape with a shape from the list. Note that on 64-bit
+ // this includes the tag bits, but on 32-bit we only compare the low word of
+ // the value. This is fine because the list of shapes is never exposed and the
+ // tag is guaranteed to be PrivateGCThing.
+ if (needSpectreMitigations) {
+ move32(Imm32(0), spectreScratch);
+ }
+ branchPtr(Assembler::Equal, Address(shapeElements, 0), shapeScratch, onMatch);
+ if (needSpectreMitigations) {
+ spectreMovePtr(Assembler::Equal, spectreScratch, obj);
+ }
+
+ // Advance to next shape and loop if not finished.
+ addPtr(Imm32(sizeof(Value)), shapeElements);
+ branchPtr(Assembler::Below, shapeElements, endScratch, &loop);
+
+ if (cond == Assembler::NotEqual) {
+ jump(label);
+ bind(&done);
+ }
+}
+
+void MacroAssembler::branchTestObjCompartment(Condition cond, Register obj,
+ const Address& compartment,
+ Register scratch, Label* label) {
+ MOZ_ASSERT(obj != scratch);
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ loadPtr(Address(scratch, BaseShape::offsetOfRealm()), scratch);
+ loadPtr(Address(scratch, Realm::offsetOfCompartment()), scratch);
+ branchPtr(cond, compartment, scratch, label);
+}
+
+void MacroAssembler::branchTestObjCompartment(
+ Condition cond, Register obj, const JS::Compartment* compartment,
+ Register scratch, Label* label) {
+ MOZ_ASSERT(obj != scratch);
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
+ loadPtr(Address(scratch, BaseShape::offsetOfRealm()), scratch);
+ loadPtr(Address(scratch, Realm::offsetOfCompartment()), scratch);
+ branchPtr(cond, scratch, ImmPtr(compartment), label);
+}
+
+void MacroAssembler::branchIfNonNativeObj(Register obj, Register scratch,
+ Label* label) {
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ branchTest32(Assembler::Zero,
+ Address(scratch, Shape::offsetOfImmutableFlags()),
+ Imm32(Shape::isNativeBit()), label);
+}
+
+void MacroAssembler::branchIfObjectNotExtensible(Register obj, Register scratch,
+ Label* label) {
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+
+ // Spectre-style checks are not needed here because we do not interpret data
+ // based on this check.
+ static_assert(sizeof(ObjectFlags) == sizeof(uint16_t));
+ load16ZeroExtend(Address(scratch, Shape::offsetOfObjectFlags()), scratch);
+ branchTest32(Assembler::NonZero, scratch,
+ Imm32(uint32_t(ObjectFlag::NotExtensible)), label);
+}
+
+void MacroAssembler::wasmTrap(wasm::Trap trap,
+ wasm::BytecodeOffset bytecodeOffset) {
+ uint32_t trapOffset = wasmTrapInstruction().offset();
+ MOZ_ASSERT_IF(!oom(),
+ currentOffset() - trapOffset == WasmTrapInstructionLength);
+
+ append(trap, wasm::TrapSite(trapOffset, bytecodeOffset));
+}
+
+std::pair<CodeOffset, uint32_t> MacroAssembler::wasmReserveStackChecked(
+ uint32_t amount, wasm::BytecodeOffset trapOffset) {
+ if (amount > MAX_UNCHECKED_LEAF_FRAME_SIZE) {
+ // The frame is large. Don't bump sp until after the stack limit check so
+ // that the trap handler isn't called with a wild sp.
+ Label ok;
+ Register scratch = ABINonArgReg0;
+ moveStackPtrTo(scratch);
+
+ Label trap;
+ branchPtr(Assembler::Below, scratch, Imm32(amount), &trap);
+ subPtr(Imm32(amount), scratch);
+ branchPtr(Assembler::Below,
+ Address(InstanceReg, wasm::Instance::offsetOfStackLimit()),
+ scratch, &ok);
+
+ bind(&trap);
+ wasmTrap(wasm::Trap::StackOverflow, trapOffset);
+ CodeOffset trapInsnOffset = CodeOffset(currentOffset());
+
+ bind(&ok);
+ reserveStack(amount);
+ return std::pair<CodeOffset, uint32_t>(trapInsnOffset, 0);
+ }
+
+ reserveStack(amount);
+ Label ok;
+ branchStackPtrRhs(Assembler::Below,
+ Address(InstanceReg, wasm::Instance::offsetOfStackLimit()),
+ &ok);
+ wasmTrap(wasm::Trap::StackOverflow, trapOffset);
+ CodeOffset trapInsnOffset = CodeOffset(currentOffset());
+ bind(&ok);
+ return std::pair<CodeOffset, uint32_t>(trapInsnOffset, amount);
+}
+
+void MacroAssembler::loadWasmGlobalPtr(uint32_t globalDataOffset,
+ Register dest) {
+ loadPtr(Address(InstanceReg,
+ wasm::Instance::offsetOfGlobalArea() + globalDataOffset),
+ dest);
+}
+
+CodeOffset MacroAssembler::wasmCallImport(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee) {
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall));
+
+ // Load the callee, before the caller's registers are clobbered.
+ uint32_t globalDataOffset = callee.importGlobalDataOffset();
+ loadWasmGlobalPtr(
+ globalDataOffset + offsetof(wasm::FuncImportInstanceData, code),
+ ABINonArgReg0);
+
+#if !defined(JS_CODEGEN_NONE) && !defined(JS_CODEGEN_WASM32)
+ static_assert(ABINonArgReg0 != InstanceReg, "by constraint");
+#endif
+
+ // Switch to the callee's realm.
+ loadWasmGlobalPtr(
+ globalDataOffset + offsetof(wasm::FuncImportInstanceData, realm),
+ ABINonArgReg1);
+ loadPtr(Address(InstanceReg, wasm::Instance::offsetOfCx()), ABINonArgReg2);
+ storePtr(ABINonArgReg1, Address(ABINonArgReg2, JSContext::offsetOfRealm()));
+
+ // Switch to the callee's instance and pinned registers and make the call.
+ loadWasmGlobalPtr(
+ globalDataOffset + offsetof(wasm::FuncImportInstanceData, instance),
+ InstanceReg);
+
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCalleeInstanceOffsetBeforeCall));
+ loadWasmPinnedRegsFromInstance();
+
+ return call(desc, ABINonArgReg0);
+}
+
+CodeOffset MacroAssembler::wasmCallBuiltinInstanceMethod(
+ const wasm::CallSiteDesc& desc, const ABIArg& instanceArg,
+ wasm::SymbolicAddress builtin, wasm::FailureMode failureMode) {
+ MOZ_ASSERT(instanceArg != ABIArg());
+
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall));
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCalleeInstanceOffsetBeforeCall));
+
+ if (instanceArg.kind() == ABIArg::GPR) {
+ movePtr(InstanceReg, instanceArg.gpr());
+ } else if (instanceArg.kind() == ABIArg::Stack) {
+ storePtr(InstanceReg,
+ Address(getStackPointer(), instanceArg.offsetFromArgBase()));
+ } else {
+ MOZ_CRASH("Unknown abi passing style for pointer");
+ }
+
+ CodeOffset ret = call(desc, builtin);
+
+ if (failureMode != wasm::FailureMode::Infallible) {
+ Label noTrap;
+ switch (failureMode) {
+ case wasm::FailureMode::Infallible:
+ MOZ_CRASH();
+ case wasm::FailureMode::FailOnNegI32:
+ branchTest32(Assembler::NotSigned, ReturnReg, ReturnReg, &noTrap);
+ break;
+ case wasm::FailureMode::FailOnNullPtr:
+ branchTestPtr(Assembler::NonZero, ReturnReg, ReturnReg, &noTrap);
+ break;
+ case wasm::FailureMode::FailOnInvalidRef:
+ branchPtr(Assembler::NotEqual, ReturnReg,
+ ImmWord(uintptr_t(wasm::AnyRef::invalid().forCompiledCode())),
+ &noTrap);
+ break;
+ }
+ wasmTrap(wasm::Trap::ThrowReported,
+ wasm::BytecodeOffset(desc.lineOrBytecode()));
+ bind(&noTrap);
+ }
+
+ return ret;
+}
+
+CodeOffset MacroAssembler::asmCallIndirect(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee) {
+ MOZ_ASSERT(callee.which() == wasm::CalleeDesc::AsmJSTable);
+
+ const Register scratch = WasmTableCallScratchReg0;
+ const Register index = WasmTableCallIndexReg;
+
+ // Optimization opportunity: when offsetof(FunctionTableElem, code) == 0, as
+ // it is at present, we can probably generate better code here by folding
+ // the address computation into the load.
+
+ static_assert(sizeof(wasm::FunctionTableElem) == 8 ||
+ sizeof(wasm::FunctionTableElem) == 16,
+ "elements of function tables are two words");
+
+ // asm.js tables require no signature check, and have had their index
+ // masked into range and thus need no bounds check.
+ loadWasmGlobalPtr(callee.tableFunctionBaseGlobalDataOffset(), scratch);
+ if (sizeof(wasm::FunctionTableElem) == 8) {
+ computeEffectiveAddress(BaseIndex(scratch, index, TimesEight), scratch);
+ } else {
+ lshift32(Imm32(4), index);
+ addPtr(index, scratch);
+ }
+ loadPtr(Address(scratch, offsetof(wasm::FunctionTableElem, code)), scratch);
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall));
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCalleeInstanceOffsetBeforeCall));
+ return call(desc, scratch);
+}
+
+// In principle, call_indirect requires an expensive context switch to the
+// callee's instance and realm before the call and an almost equally expensive
+// switch back to the caller's ditto after. However, if the caller's instance
+// is the same as the callee's instance then no context switch is required, and
+// it only takes a compare-and-branch at run-time to test this - all values are
+// in registers already. We therefore generate two call paths, one for the fast
+// call without the context switch (which additionally avoids a null check) and
+// one for the slow call with the context switch.
+
+void MacroAssembler::wasmCallIndirect(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee,
+ Label* boundsCheckFailedLabel,
+ Label* nullCheckFailedLabel,
+ mozilla::Maybe<uint32_t> tableSize,
+ CodeOffset* fastCallOffset,
+ CodeOffset* slowCallOffset) {
+ static_assert(sizeof(wasm::FunctionTableElem) == 2 * sizeof(void*),
+ "Exactly two pointers or index scaling won't work correctly");
+ MOZ_ASSERT(callee.which() == wasm::CalleeDesc::WasmTable);
+
+ const int shift = sizeof(wasm::FunctionTableElem) == 8 ? 3 : 4;
+ wasm::BytecodeOffset trapOffset(desc.lineOrBytecode());
+ const Register calleeScratch = WasmTableCallScratchReg0;
+ const Register index = WasmTableCallIndexReg;
+
+ // Check the table index and throw if out-of-bounds.
+ //
+ // Frequently the table size is known, so optimize for that. Otherwise
+ // compare with a memory operand when that's possible. (There's little sense
+ // in hoisting the load of the bound into a register at a higher level and
+ // reusing that register, because a hoisted value would either have to be
+ // spilled and re-loaded before the next call_indirect, or would be abandoned
+ // because we could not trust that a hoisted value would not have changed.)
+
+ if (boundsCheckFailedLabel) {
+ if (tableSize.isSome()) {
+ branch32(Assembler::Condition::AboveOrEqual, index, Imm32(*tableSize),
+ boundsCheckFailedLabel);
+ } else {
+ branch32(Assembler::Condition::BelowOrEqual,
+ Address(InstanceReg, wasm::Instance::offsetOfGlobalArea() +
+ callee.tableLengthGlobalDataOffset()),
+ index, boundsCheckFailedLabel);
+ }
+ }
+
+ // Write the functype-id into the ABI functype-id register.
+
+ const wasm::CallIndirectId callIndirectId = callee.wasmTableSigId();
+ switch (callIndirectId.kind()) {
+ case wasm::CallIndirectIdKind::Global:
+ loadWasmGlobalPtr(callIndirectId.globalDataOffset(), WasmTableCallSigReg);
+ break;
+ case wasm::CallIndirectIdKind::Immediate:
+ move32(Imm32(callIndirectId.immediate()), WasmTableCallSigReg);
+ break;
+ case wasm::CallIndirectIdKind::AsmJS:
+ case wasm::CallIndirectIdKind::None:
+ break;
+ }
+
+ // Load the base pointer of the table and compute the address of the callee in
+ // the table.
+
+ loadWasmGlobalPtr(callee.tableFunctionBaseGlobalDataOffset(), calleeScratch);
+ shiftIndex32AndAdd(index, shift, calleeScratch);
+
+ // Load the callee instance and decide whether to take the fast path or the
+ // slow path.
+
+ Label fastCall;
+ Label done;
+ const Register newInstanceTemp = WasmTableCallScratchReg1;
+ loadPtr(Address(calleeScratch, offsetof(wasm::FunctionTableElem, instance)),
+ newInstanceTemp);
+ branchPtr(Assembler::Equal, InstanceReg, newInstanceTemp, &fastCall);
+
+ // Slow path: Save context, check for null, setup new context, call, restore
+ // context.
+ //
+ // TODO: The slow path could usefully be out-of-line and the test above would
+ // just fall through to the fast path. This keeps the fast-path code dense,
+ // and has correct static prediction for the branch (forward conditional
+ // branches predicted not taken, normally).
+
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall));
+ movePtr(newInstanceTemp, InstanceReg);
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCalleeInstanceOffsetBeforeCall));
+
+#ifdef WASM_HAS_HEAPREG
+ // Use the null pointer exception resulting from loading HeapReg from a null
+ // instance to handle a call to a null slot.
+ MOZ_ASSERT(nullCheckFailedLabel == nullptr);
+ loadWasmPinnedRegsFromInstance(mozilla::Some(trapOffset));
+#else
+ MOZ_ASSERT(nullCheckFailedLabel != nullptr);
+ branchTestPtr(Assembler::Zero, InstanceReg, InstanceReg,
+ nullCheckFailedLabel);
+
+ loadWasmPinnedRegsFromInstance();
+#endif
+ switchToWasmInstanceRealm(index, WasmTableCallScratchReg1);
+
+ loadPtr(Address(calleeScratch, offsetof(wasm::FunctionTableElem, code)),
+ calleeScratch);
+
+ *slowCallOffset = call(desc, calleeScratch);
+
+ // Restore registers and realm and join up with the fast path.
+
+ loadPtr(Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall),
+ InstanceReg);
+ loadWasmPinnedRegsFromInstance();
+ switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1);
+ jump(&done);
+
+ // Fast path: just load the code pointer and go. The instance and heap
+ // register are the same as in the caller, and nothing will be null.
+ //
+ // (In particular, the code pointer will not be null: if it were, the instance
+ // would have been null, and then it would not have been equivalent to our
+ // current instance. So no null check is needed on the fast path.)
+
+ bind(&fastCall);
+
+ loadPtr(Address(calleeScratch, offsetof(wasm::FunctionTableElem, code)),
+ calleeScratch);
+
+ // We use a different type of call site for the fast call since the instance
+ // slots in the frame do not have valid values.
+
+ wasm::CallSiteDesc newDesc(desc.lineOrBytecode(),
+ wasm::CallSiteDesc::IndirectFast);
+ *fastCallOffset = call(newDesc, calleeScratch);
+
+ bind(&done);
+}
+
+void MacroAssembler::wasmCallRef(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee,
+ CodeOffset* fastCallOffset,
+ CodeOffset* slowCallOffset) {
+ MOZ_ASSERT(callee.which() == wasm::CalleeDesc::FuncRef);
+ const Register calleeScratch = WasmCallRefCallScratchReg0;
+ const Register calleeFnObj = WasmCallRefReg;
+
+ // Load from the function's WASM_INSTANCE_SLOT extended slot, and decide
+ // whether to take the fast path or the slow path. Register this load
+ // instruction to be source of a trap -- null pointer check.
+
+ Label fastCall;
+ Label done;
+ const Register newInstanceTemp = WasmCallRefCallScratchReg1;
+ size_t instanceSlotOffset = FunctionExtended::offsetOfExtendedSlot(
+ FunctionExtended::WASM_INSTANCE_SLOT);
+ static_assert(FunctionExtended::WASM_INSTANCE_SLOT < wasm::NullPtrGuardSize);
+ wasm::BytecodeOffset trapOffset(desc.lineOrBytecode());
+ append(wasm::Trap::NullPointerDereference,
+ wasm::TrapSite(currentOffset(), trapOffset));
+ loadPtr(Address(calleeFnObj, instanceSlotOffset), newInstanceTemp);
+ branchPtr(Assembler::Equal, InstanceReg, newInstanceTemp, &fastCall);
+
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall));
+ movePtr(newInstanceTemp, InstanceReg);
+ storePtr(InstanceReg,
+ Address(getStackPointer(), WasmCalleeInstanceOffsetBeforeCall));
+
+ loadWasmPinnedRegsFromInstance();
+ switchToWasmInstanceRealm(WasmCallRefCallScratchReg0,
+ WasmCallRefCallScratchReg1);
+
+ // Get funcUncheckedCallEntry() from the function's
+ // WASM_FUNC_UNCHECKED_ENTRY_SLOT extended slot.
+ size_t uncheckedEntrySlotOffset = FunctionExtended::offsetOfExtendedSlot(
+ FunctionExtended::WASM_FUNC_UNCHECKED_ENTRY_SLOT);
+ loadPtr(Address(calleeFnObj, uncheckedEntrySlotOffset), calleeScratch);
+
+ *slowCallOffset = call(desc, calleeScratch);
+
+ // Restore registers and realm and back to this caller's.
+ loadPtr(Address(getStackPointer(), WasmCallerInstanceOffsetBeforeCall),
+ InstanceReg);
+ loadWasmPinnedRegsFromInstance();
+ switchToWasmInstanceRealm(ABINonArgReturnReg0, ABINonArgReturnReg1);
+ jump(&done);
+
+ // Fast path: just load WASM_FUNC_UNCHECKED_ENTRY_SLOT value and go.
+ // The instance and pinned registers are the same as in the caller.
+
+ bind(&fastCall);
+
+ loadPtr(Address(calleeFnObj, uncheckedEntrySlotOffset), calleeScratch);
+
+ // We use a different type of call site for the fast call since the instance
+ // slots in the frame do not have valid values.
+
+ wasm::CallSiteDesc newDesc(desc.lineOrBytecode(),
+ wasm::CallSiteDesc::FuncRefFast);
+ *fastCallOffset = call(newDesc, calleeScratch);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchWasmTypeDefIsSubtype(Register subTypeDef,
+ Register superTypeDef,
+ Register scratch,
+ uint32_t subTypingDepth,
+ Label* label, bool onSuccess) {
+ MOZ_ASSERT_IF(subTypingDepth >= wasm::MinSuperTypeVectorLength,
+ scratch != Register::Invalid());
+
+ // We generate just different enough code for 'is' subtype vs 'is not'
+ // subtype that we handle them separately.
+ if (onSuccess) {
+ Label failed;
+
+ // Fast path for the type defs being equal.
+ branchPtr(Assembler::Equal, subTypeDef, superTypeDef, label);
+
+ // Slower path for checking the supertype vector of `subTypeDef`. We don't
+ // need `subTypeDef` at this point, so we use it as a scratch for storing
+ // the super type vector and entry from it.
+ loadPtr(Address(subTypeDef, wasm::TypeDef::offsetOfSuperTypeVector()),
+ subTypeDef);
+
+ // Emit a bounds check if the super type depth may be out-of-bounds.
+ if (subTypingDepth >= wasm::MinSuperTypeVectorLength) {
+ // Slowest path for having a bounds check of the super type vector
+ load32(Address(subTypeDef, wasm::SuperTypeVector::offsetOfLength()),
+ scratch);
+ branch32(Assembler::LessThanOrEqual, scratch, Imm32(subTypingDepth),
+ &failed);
+ }
+
+ // Load the `subTypingDepth` entry from subTypeDef's super type vector. This
+ // will be `superTypeDef` if `subTypeDef` is indeed a subtype.
+ loadPtr(Address(subTypeDef, wasm::SuperTypeVector::offsetOfTypeDefInVector(
+ subTypingDepth)),
+ subTypeDef);
+ branchPtr(Assembler::Equal, subTypeDef, superTypeDef, label);
+
+ // Fallthrough to the failed case
+ bind(&failed);
+ return;
+ }
+
+ // Load the super type vector from subTypeDef
+ loadPtr(Address(subTypeDef, wasm::TypeDef::offsetOfSuperTypeVector()),
+ subTypeDef);
+
+ // Emit a bounds check if the super type depth may be out-of-bounds.
+ if (subTypingDepth >= wasm::MinSuperTypeVectorLength) {
+ load32(Address(subTypeDef, wasm::SuperTypeVector::offsetOfLength()),
+ scratch);
+ branch32(Assembler::LessThanOrEqual, scratch, Imm32(subTypingDepth), label);
+ }
+
+ // Load the `subTypingDepth` entry from subTypeDef's super type vector. This
+ // will be `superTypeDef` if `subTypeDef` is indeed a subtype.
+ loadPtr(Address(subTypeDef, wasm::SuperTypeVector::offsetOfTypeDefInVector(
+ subTypingDepth)),
+ subTypeDef);
+ branchPtr(Assembler::NotEqual, subTypeDef, superTypeDef, label);
+ // Fallthrough to the success case
+}
+
+void MacroAssembler::nopPatchableToCall(const wasm::CallSiteDesc& desc) {
+ CodeOffset offset = nopPatchableToCall();
+ append(desc, offset);
+}
+
+void MacroAssembler::emitPreBarrierFastPath(JSRuntime* rt, MIRType type,
+ Register temp1, Register temp2,
+ Register temp3, Label* noBarrier) {
+ MOZ_ASSERT(temp1 != PreBarrierReg);
+ MOZ_ASSERT(temp2 != PreBarrierReg);
+ MOZ_ASSERT(temp3 != PreBarrierReg);
+
+ // Load the GC thing in temp1.
+ if (type == MIRType::Value) {
+ unboxGCThingForGCBarrier(Address(PreBarrierReg, 0), temp1);
+ } else {
+ MOZ_ASSERT(type == MIRType::Object || type == MIRType::String ||
+ type == MIRType::Shape);
+ loadPtr(Address(PreBarrierReg, 0), temp1);
+ }
+
+#ifdef DEBUG
+ // The caller should have checked for null pointers.
+ Label nonZero;
+ branchTestPtr(Assembler::NonZero, temp1, temp1, &nonZero);
+ assumeUnreachable("JIT pre-barrier: unexpected nullptr");
+ bind(&nonZero);
+#endif
+
+ // Load the chunk address in temp2.
+ movePtr(temp1, temp2);
+ andPtr(Imm32(int32_t(~gc::ChunkMask)), temp2);
+
+ // If the GC thing is in the nursery, we don't need to barrier it.
+ if (type == MIRType::Value || type == MIRType::Object ||
+ type == MIRType::String) {
+ branchPtr(Assembler::NotEqual, Address(temp2, gc::ChunkStoreBufferOffset),
+ ImmWord(0), noBarrier);
+ } else {
+#ifdef DEBUG
+ Label isTenured;
+ branchPtr(Assembler::Equal, Address(temp2, gc::ChunkStoreBufferOffset),
+ ImmWord(0), &isTenured);
+ assumeUnreachable("JIT pre-barrier: unexpected nursery pointer");
+ bind(&isTenured);
+#endif
+ }
+
+ // Determine the bit index and store in temp1.
+ //
+ // bit = (addr & js::gc::ChunkMask) / js::gc::CellBytesPerMarkBit +
+ // static_cast<uint32_t>(colorBit);
+ static_assert(gc::CellBytesPerMarkBit == 8,
+ "Calculation below relies on this");
+ static_assert(size_t(gc::ColorBit::BlackBit) == 0,
+ "Calculation below relies on this");
+ andPtr(Imm32(gc::ChunkMask), temp1);
+ rshiftPtr(Imm32(3), temp1);
+
+ static_assert(gc::MarkBitmapWordBits == JS_BITS_PER_WORD,
+ "Calculation below relies on this");
+
+ // Load the bitmap word in temp2.
+ //
+ // word = chunk.bitmap[bit / MarkBitmapWordBits];
+
+ // Fold the adjustment for the fact that arenas don't start at the beginning
+ // of the chunk into the offset to the chunk bitmap.
+ const size_t firstArenaAdjustment = gc::FirstArenaAdjustmentBits / CHAR_BIT;
+ const intptr_t offset =
+ intptr_t(gc::ChunkMarkBitmapOffset) - intptr_t(firstArenaAdjustment);
+
+ movePtr(temp1, temp3);
+#if JS_BITS_PER_WORD == 64
+ rshiftPtr(Imm32(6), temp1);
+ loadPtr(BaseIndex(temp2, temp1, TimesEight, offset), temp2);
+#else
+ rshiftPtr(Imm32(5), temp1);
+ loadPtr(BaseIndex(temp2, temp1, TimesFour, offset), temp2);
+#endif
+
+ // Load the mask in temp1.
+ //
+ // mask = uintptr_t(1) << (bit % MarkBitmapWordBits);
+ andPtr(Imm32(gc::MarkBitmapWordBits - 1), temp3);
+ move32(Imm32(1), temp1);
+#ifdef JS_CODEGEN_X64
+ MOZ_ASSERT(temp3 == rcx);
+ shlq_cl(temp1);
+#elif JS_CODEGEN_X86
+ MOZ_ASSERT(temp3 == ecx);
+ shll_cl(temp1);
+#elif JS_CODEGEN_ARM
+ ma_lsl(temp3, temp1, temp1);
+#elif JS_CODEGEN_ARM64
+ Lsl(ARMRegister(temp1, 64), ARMRegister(temp1, 64), ARMRegister(temp3, 64));
+#elif JS_CODEGEN_MIPS32
+ ma_sll(temp1, temp1, temp3);
+#elif JS_CODEGEN_MIPS64
+ ma_dsll(temp1, temp1, temp3);
+#elif JS_CODEGEN_LOONG64
+ as_sll_d(temp1, temp1, temp3);
+#elif JS_CODEGEN_WASM32
+ MOZ_CRASH();
+#elif JS_CODEGEN_NONE
+ MOZ_CRASH();
+#else
+# error "Unknown architecture"
+#endif
+
+ // No barrier is needed if the bit is set, |word & mask != 0|.
+ branchTestPtr(Assembler::NonZero, temp2, temp1, noBarrier);
+}
+
+// ========================================================================
+// JS atomic operations.
+
+void MacroAssembler::atomicIsLockFreeJS(Register value, Register output) {
+ // Keep this in sync with isLockfreeJS() in jit/AtomicOperations.h.
+ static_assert(AtomicOperations::isLockfreeJS(1)); // Implementation artifact
+ static_assert(AtomicOperations::isLockfreeJS(2)); // Implementation artifact
+ static_assert(AtomicOperations::isLockfreeJS(4)); // Spec requirement
+ static_assert(AtomicOperations::isLockfreeJS(8)); // Implementation artifact
+
+ Label done;
+ move32(Imm32(1), output);
+ branch32(Assembler::Equal, value, Imm32(8), &done);
+ branch32(Assembler::Equal, value, Imm32(4), &done);
+ branch32(Assembler::Equal, value, Imm32(2), &done);
+ branch32(Assembler::Equal, value, Imm32(1), &done);
+ move32(Imm32(0), output);
+ bind(&done);
+}
+
+// ========================================================================
+// Spectre Mitigations.
+
+void MacroAssembler::spectreMaskIndex32(Register index, Register length,
+ Register output) {
+ MOZ_ASSERT(JitOptions.spectreIndexMasking);
+ MOZ_ASSERT(length != output);
+ MOZ_ASSERT(index != output);
+
+ move32(Imm32(0), output);
+ cmp32Move32(Assembler::Below, index, length, index, output);
+}
+
+void MacroAssembler::spectreMaskIndex32(Register index, const Address& length,
+ Register output) {
+ MOZ_ASSERT(JitOptions.spectreIndexMasking);
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != output);
+ MOZ_ASSERT(index != output);
+
+ move32(Imm32(0), output);
+ cmp32Move32(Assembler::Below, index, length, index, output);
+}
+
+void MacroAssembler::spectreMaskIndexPtr(Register index, Register length,
+ Register output) {
+ MOZ_ASSERT(JitOptions.spectreIndexMasking);
+ MOZ_ASSERT(length != output);
+ MOZ_ASSERT(index != output);
+
+ movePtr(ImmWord(0), output);
+ cmpPtrMovePtr(Assembler::Below, index, length, index, output);
+}
+
+void MacroAssembler::spectreMaskIndexPtr(Register index, const Address& length,
+ Register output) {
+ MOZ_ASSERT(JitOptions.spectreIndexMasking);
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != output);
+ MOZ_ASSERT(index != output);
+
+ movePtr(ImmWord(0), output);
+ cmpPtrMovePtr(Assembler::Below, index, length, index, output);
+}
+
+void MacroAssembler::boundsCheck32PowerOfTwo(Register index, uint32_t length,
+ Label* failure) {
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(length));
+ branch32(Assembler::AboveOrEqual, index, Imm32(length), failure);
+
+ // Note: it's fine to clobber the input register, as this is a no-op: it
+ // only affects speculative execution.
+ if (JitOptions.spectreIndexMasking) {
+ and32(Imm32(length - 1), index);
+ }
+}
+
+void MacroAssembler::loadWasmPinnedRegsFromInstance(
+ mozilla::Maybe<wasm::BytecodeOffset> trapOffset) {
+#ifdef WASM_HAS_HEAPREG
+ static_assert(wasm::Instance::offsetOfMemoryBase() < 4096,
+ "We count only on the low page being inaccessible");
+ if (trapOffset) {
+ append(wasm::Trap::IndirectCallToNull,
+ wasm::TrapSite(currentOffset(), *trapOffset));
+ }
+ loadPtr(Address(InstanceReg, wasm::Instance::offsetOfMemoryBase()), HeapReg);
+#else
+ MOZ_ASSERT(!trapOffset);
+#endif
+}
+
+//}}} check_macroassembler_style
+
+#ifdef JS_64BIT
+void MacroAssembler::debugAssertCanonicalInt32(Register r) {
+# ifdef DEBUG
+ if (!js::jit::JitOptions.lessDebugCode) {
+# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM64)
+ Label ok;
+ branchPtr(Assembler::BelowOrEqual, r, ImmWord(UINT32_MAX), &ok);
+ breakpoint();
+ bind(&ok);
+# elif defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64)
+ Label ok;
+ ScratchRegisterScope scratch(asMasm());
+ move32SignExtendToPtr(r, scratch);
+ branchPtr(Assembler::Equal, r, scratch, &ok);
+ breakpoint();
+ bind(&ok);
+# else
+ MOZ_CRASH("IMPLEMENT ME");
+# endif
+ }
+# endif
+}
+#endif
+
+void MacroAssembler::memoryBarrierBefore(const Synchronization& sync) {
+ memoryBarrier(sync.barrierBefore);
+}
+
+void MacroAssembler::memoryBarrierAfter(const Synchronization& sync) {
+ memoryBarrier(sync.barrierAfter);
+}
+
+void MacroAssembler::debugAssertIsObject(const ValueOperand& val) {
+#ifdef DEBUG
+ Label ok;
+ branchTestObject(Assembler::Equal, val, &ok);
+ assumeUnreachable("Expected an object!");
+ bind(&ok);
+#endif
+}
+
+void MacroAssembler::debugAssertObjHasFixedSlots(Register obj,
+ Register scratch) {
+#ifdef DEBUG
+ Label hasFixedSlots;
+ loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
+ branchTest32(Assembler::NonZero,
+ Address(scratch, Shape::offsetOfImmutableFlags()),
+ Imm32(NativeShape::fixedSlotsMask()), &hasFixedSlots);
+ assumeUnreachable("Expected a fixed slot");
+ bind(&hasFixedSlots);
+#endif
+}
+
+void MacroAssembler::debugAssertObjectHasClass(Register obj, Register scratch,
+ const JSClass* clasp) {
+#ifdef DEBUG
+ Label done;
+ branchTestObjClassNoSpectreMitigations(Assembler::Equal, obj, clasp, scratch,
+ &done);
+ assumeUnreachable("Class check failed");
+ bind(&done);
+#endif
+}
+
+void MacroAssembler::branchArrayIsNotPacked(Register array, Register temp1,
+ Register temp2, Label* label) {
+ loadPtr(Address(array, NativeObject::offsetOfElements()), temp1);
+
+ // Test length == initializedLength.
+ Address initLength(temp1, ObjectElements::offsetOfInitializedLength());
+ load32(Address(temp1, ObjectElements::offsetOfLength()), temp2);
+ branch32(Assembler::NotEqual, initLength, temp2, label);
+
+ // Test the NON_PACKED flag.
+ Address flags(temp1, ObjectElements::offsetOfFlags());
+ branchTest32(Assembler::NonZero, flags, Imm32(ObjectElements::NON_PACKED),
+ label);
+}
+
+void MacroAssembler::setIsPackedArray(Register obj, Register output,
+ Register temp) {
+ // Ensure it's an ArrayObject.
+ Label notPackedArray;
+ branchTestObjClass(Assembler::NotEqual, obj, &ArrayObject::class_, temp, obj,
+ &notPackedArray);
+
+ branchArrayIsNotPacked(obj, temp, output, &notPackedArray);
+
+ Label done;
+ move32(Imm32(1), output);
+ jump(&done);
+
+ bind(&notPackedArray);
+ move32(Imm32(0), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::packedArrayPop(Register array, ValueOperand output,
+ Register temp1, Register temp2,
+ Label* fail) {
+ // Load obj->elements in temp1.
+ loadPtr(Address(array, NativeObject::offsetOfElements()), temp1);
+
+ // Check flags.
+ static constexpr uint32_t UnhandledFlags =
+ ObjectElements::Flags::NON_PACKED |
+ ObjectElements::Flags::NONWRITABLE_ARRAY_LENGTH |
+ ObjectElements::Flags::NOT_EXTENSIBLE |
+ ObjectElements::Flags::MAYBE_IN_ITERATION;
+ Address flags(temp1, ObjectElements::offsetOfFlags());
+ branchTest32(Assembler::NonZero, flags, Imm32(UnhandledFlags), fail);
+
+ // Load length in temp2. Ensure length == initializedLength.
+ Address lengthAddr(temp1, ObjectElements::offsetOfLength());
+ Address initLengthAddr(temp1, ObjectElements::offsetOfInitializedLength());
+ load32(lengthAddr, temp2);
+ branch32(Assembler::NotEqual, initLengthAddr, temp2, fail);
+
+ // Result is |undefined| if length == 0.
+ Label notEmpty, done;
+ branchTest32(Assembler::NonZero, temp2, temp2, &notEmpty);
+ {
+ moveValue(UndefinedValue(), output);
+ jump(&done);
+ }
+
+ bind(&notEmpty);
+
+ // Load the last element.
+ sub32(Imm32(1), temp2);
+ BaseObjectElementIndex elementAddr(temp1, temp2);
+ loadValue(elementAddr, output);
+
+ // Pre-barrier the element because we're removing it from the array.
+ EmitPreBarrier(*this, elementAddr, MIRType::Value);
+
+ // Update length and initializedLength.
+ store32(temp2, lengthAddr);
+ store32(temp2, initLengthAddr);
+
+ bind(&done);
+}
+
+void MacroAssembler::packedArrayShift(Register array, ValueOperand output,
+ Register temp1, Register temp2,
+ LiveRegisterSet volatileRegs,
+ Label* fail) {
+ // Load obj->elements in temp1.
+ loadPtr(Address(array, NativeObject::offsetOfElements()), temp1);
+
+ // Check flags.
+ static constexpr uint32_t UnhandledFlags =
+ ObjectElements::Flags::NON_PACKED |
+ ObjectElements::Flags::NONWRITABLE_ARRAY_LENGTH |
+ ObjectElements::Flags::NOT_EXTENSIBLE |
+ ObjectElements::Flags::MAYBE_IN_ITERATION;
+ Address flags(temp1, ObjectElements::offsetOfFlags());
+ branchTest32(Assembler::NonZero, flags, Imm32(UnhandledFlags), fail);
+
+ // Load length in temp2. Ensure length == initializedLength.
+ Address lengthAddr(temp1, ObjectElements::offsetOfLength());
+ Address initLengthAddr(temp1, ObjectElements::offsetOfInitializedLength());
+ load32(lengthAddr, temp2);
+ branch32(Assembler::NotEqual, initLengthAddr, temp2, fail);
+
+ // Result is |undefined| if length == 0.
+ Label notEmpty, done;
+ branchTest32(Assembler::NonZero, temp2, temp2, &notEmpty);
+ {
+ moveValue(UndefinedValue(), output);
+ jump(&done);
+ }
+
+ bind(&notEmpty);
+
+ // Load the first element.
+ Address elementAddr(temp1, 0);
+ loadValue(elementAddr, output);
+
+ // Move the other elements and update the initializedLength/length. This will
+ // also trigger pre-barriers.
+ {
+ // Ensure output is in volatileRegs. Don't preserve temp1 and temp2.
+ volatileRegs.takeUnchecked(temp1);
+ volatileRegs.takeUnchecked(temp2);
+ if (output.hasVolatileReg()) {
+ volatileRegs.addUnchecked(output);
+ }
+
+ PushRegsInMask(volatileRegs);
+
+ using Fn = void (*)(ArrayObject * arr);
+ setupUnalignedABICall(temp1);
+ passABIArg(array);
+ callWithABI<Fn, ArrayShiftMoveElements>();
+
+ PopRegsInMask(volatileRegs);
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::loadArgumentsObjectElement(Register obj, Register index,
+ ValueOperand output,
+ Register temp, Label* fail) {
+ Register temp2 = output.scratchReg();
+
+ // Get initial length value.
+ unboxInt32(Address(obj, ArgumentsObject::getInitialLengthSlotOffset()), temp);
+
+ // Ensure no overridden elements.
+ branchTest32(Assembler::NonZero, temp,
+ Imm32(ArgumentsObject::ELEMENT_OVERRIDDEN_BIT), fail);
+
+ // Bounds check.
+ rshift32(Imm32(ArgumentsObject::PACKED_BITS_COUNT), temp);
+ spectreBoundsCheck32(index, temp, temp2, fail);
+
+ // Load ArgumentsData.
+ loadPrivate(Address(obj, ArgumentsObject::getDataSlotOffset()), temp);
+
+ // Guard the argument is not a FORWARD_TO_CALL_SLOT MagicValue.
+ BaseValueIndex argValue(temp, index, ArgumentsData::offsetOfArgs());
+ branchTestMagic(Assembler::Equal, argValue, fail);
+ loadValue(argValue, output);
+}
+
+void MacroAssembler::loadArgumentsObjectElementHole(Register obj,
+ Register index,
+ ValueOperand output,
+ Register temp,
+ Label* fail) {
+ Register temp2 = output.scratchReg();
+
+ // Get initial length value.
+ unboxInt32(Address(obj, ArgumentsObject::getInitialLengthSlotOffset()), temp);
+
+ // Ensure no overridden elements.
+ branchTest32(Assembler::NonZero, temp,
+ Imm32(ArgumentsObject::ELEMENT_OVERRIDDEN_BIT), fail);
+
+ // Bounds check.
+ Label outOfBounds, done;
+ rshift32(Imm32(ArgumentsObject::PACKED_BITS_COUNT), temp);
+ spectreBoundsCheck32(index, temp, temp2, &outOfBounds);
+
+ // Load ArgumentsData.
+ loadPrivate(Address(obj, ArgumentsObject::getDataSlotOffset()), temp);
+
+ // Guard the argument is not a FORWARD_TO_CALL_SLOT MagicValue.
+ BaseValueIndex argValue(temp, index, ArgumentsData::offsetOfArgs());
+ branchTestMagic(Assembler::Equal, argValue, fail);
+ loadValue(argValue, output);
+ jump(&done);
+
+ bind(&outOfBounds);
+ branch32(Assembler::LessThan, index, Imm32(0), fail);
+ moveValue(UndefinedValue(), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::loadArgumentsObjectElementExists(
+ Register obj, Register index, Register output, Register temp, Label* fail) {
+ // Ensure the index is non-negative.
+ branch32(Assembler::LessThan, index, Imm32(0), fail);
+
+ // Get initial length value.
+ unboxInt32(Address(obj, ArgumentsObject::getInitialLengthSlotOffset()), temp);
+
+ // Ensure no overridden or deleted elements.
+ branchTest32(Assembler::NonZero, temp,
+ Imm32(ArgumentsObject::ELEMENT_OVERRIDDEN_BIT), fail);
+
+ // Compare index against the length.
+ rshift32(Imm32(ArgumentsObject::PACKED_BITS_COUNT), temp);
+ cmp32Set(Assembler::LessThan, index, temp, output);
+}
+
+void MacroAssembler::loadArgumentsObjectLength(Register obj, Register output,
+ Label* fail) {
+ // Get initial length value.
+ unboxInt32(Address(obj, ArgumentsObject::getInitialLengthSlotOffset()),
+ output);
+
+ // Test if length has been overridden.
+ branchTest32(Assembler::NonZero, output,
+ Imm32(ArgumentsObject::LENGTH_OVERRIDDEN_BIT), fail);
+
+ // Shift out arguments length and return it.
+ rshift32(Imm32(ArgumentsObject::PACKED_BITS_COUNT), output);
+}
+
+void MacroAssembler::branchTestArgumentsObjectFlags(Register obj, Register temp,
+ uint32_t flags,
+ Condition cond,
+ Label* label) {
+ MOZ_ASSERT((flags & ~ArgumentsObject::PACKED_BITS_MASK) == 0);
+
+ // Get initial length value.
+ unboxInt32(Address(obj, ArgumentsObject::getInitialLengthSlotOffset()), temp);
+
+ // Test flags.
+ branchTest32(cond, temp, Imm32(flags), label);
+}
+
+static constexpr bool ValidateSizeRange(Scalar::Type from, Scalar::Type to) {
+ for (Scalar::Type type = from; type < to; type = Scalar::Type(type + 1)) {
+ if (TypedArrayElemSize(type) != TypedArrayElemSize(from)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+void MacroAssembler::typedArrayElementSize(Register obj, Register output) {
+ static_assert(Scalar::Int8 == 0, "Int8 is the first typed array class");
+ static_assert(
+ (Scalar::BigUint64 - Scalar::Int8) == Scalar::MaxTypedArrayViewType - 1,
+ "BigUint64 is the last typed array class");
+
+ Label one, two, four, eight, done;
+
+ loadObjClassUnsafe(obj, output);
+
+ static_assert(ValidateSizeRange(Scalar::Int8, Scalar::Int16),
+ "element size is one in [Int8, Int16)");
+ branchPtr(Assembler::Below, output,
+ ImmPtr(TypedArrayObject::classForType(Scalar::Int16)), &one);
+
+ static_assert(ValidateSizeRange(Scalar::Int16, Scalar::Int32),
+ "element size is two in [Int16, Int32)");
+ branchPtr(Assembler::Below, output,
+ ImmPtr(TypedArrayObject::classForType(Scalar::Int32)), &two);
+
+ static_assert(ValidateSizeRange(Scalar::Int32, Scalar::Float64),
+ "element size is four in [Int32, Float64)");
+ branchPtr(Assembler::Below, output,
+ ImmPtr(TypedArrayObject::classForType(Scalar::Float64)), &four);
+
+ static_assert(ValidateSizeRange(Scalar::Float64, Scalar::Uint8Clamped),
+ "element size is eight in [Float64, Uint8Clamped)");
+ branchPtr(Assembler::Below, output,
+ ImmPtr(TypedArrayObject::classForType(Scalar::Uint8Clamped)),
+ &eight);
+
+ static_assert(ValidateSizeRange(Scalar::Uint8Clamped, Scalar::BigInt64),
+ "element size is one in [Uint8Clamped, BigInt64)");
+ branchPtr(Assembler::Below, output,
+ ImmPtr(TypedArrayObject::classForType(Scalar::BigInt64)), &one);
+
+ static_assert(
+ ValidateSizeRange(Scalar::BigInt64, Scalar::MaxTypedArrayViewType),
+ "element size is eight in [BigInt64, MaxTypedArrayViewType)");
+ // Fall through for BigInt64 and BigUint64
+
+ bind(&eight);
+ move32(Imm32(8), output);
+ jump(&done);
+
+ bind(&four);
+ move32(Imm32(4), output);
+ jump(&done);
+
+ bind(&two);
+ move32(Imm32(2), output);
+ jump(&done);
+
+ bind(&one);
+ move32(Imm32(1), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchIfClassIsNotTypedArray(Register clasp,
+ Label* notTypedArray) {
+ static_assert(Scalar::Int8 == 0, "Int8 is the first typed array class");
+ const JSClass* firstTypedArrayClass =
+ TypedArrayObject::classForType(Scalar::Int8);
+
+ static_assert(
+ (Scalar::BigUint64 - Scalar::Int8) == Scalar::MaxTypedArrayViewType - 1,
+ "BigUint64 is the last typed array class");
+ const JSClass* lastTypedArrayClass =
+ TypedArrayObject::classForType(Scalar::BigUint64);
+
+ branchPtr(Assembler::Below, clasp, ImmPtr(firstTypedArrayClass),
+ notTypedArray);
+ branchPtr(Assembler::Above, clasp, ImmPtr(lastTypedArrayClass),
+ notTypedArray);
+}
+
+void MacroAssembler::branchIfHasDetachedArrayBuffer(Register obj, Register temp,
+ Label* label) {
+ // Inline implementation of ArrayBufferViewObject::hasDetachedBuffer().
+
+ // Load obj->elements in temp.
+ loadPtr(Address(obj, NativeObject::offsetOfElements()), temp);
+
+ // Shared buffers can't be detached.
+ Label done;
+ branchTest32(Assembler::NonZero,
+ Address(temp, ObjectElements::offsetOfFlags()),
+ Imm32(ObjectElements::SHARED_MEMORY), &done);
+
+ // An ArrayBufferView with a null buffer has never had its buffer exposed to
+ // become detached.
+ fallibleUnboxObject(Address(obj, ArrayBufferViewObject::bufferOffset()), temp,
+ &done);
+
+ // Load the ArrayBuffer flags and branch if the detached flag is set.
+ unboxInt32(Address(temp, ArrayBufferObject::offsetOfFlagsSlot()), temp);
+ branchTest32(Assembler::NonZero, temp, Imm32(ArrayBufferObject::DETACHED),
+ label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchIfNativeIteratorNotReusable(Register ni,
+ Label* notReusable) {
+ // See NativeIterator::isReusable.
+ Address flagsAddr(ni, NativeIterator::offsetOfFlagsAndCount());
+
+#ifdef DEBUG
+ Label niIsInitialized;
+ branchTest32(Assembler::NonZero, flagsAddr,
+ Imm32(NativeIterator::Flags::Initialized), &niIsInitialized);
+ assumeUnreachable(
+ "Expected a NativeIterator that's been completely "
+ "initialized");
+ bind(&niIsInitialized);
+#endif
+
+ branchTest32(Assembler::NonZero, flagsAddr,
+ Imm32(NativeIterator::Flags::NotReusable), notReusable);
+}
+
+static void LoadNativeIterator(MacroAssembler& masm, Register obj,
+ Register dest) {
+ MOZ_ASSERT(obj != dest);
+
+#ifdef DEBUG
+ // Assert we have a PropertyIteratorObject.
+ Label ok;
+ masm.branchTestObjClass(Assembler::Equal, obj,
+ &PropertyIteratorObject::class_, dest, obj, &ok);
+ masm.assumeUnreachable("Expected PropertyIteratorObject!");
+ masm.bind(&ok);
+#endif
+
+ // Load NativeIterator object.
+ Address slotAddr(obj, PropertyIteratorObject::offsetOfIteratorSlot());
+ masm.loadPrivate(slotAddr, dest);
+}
+
+// The ShapeCachePtr may be used to cache an iterator for for-in. Return that
+// iterator in |dest| if:
+// - the shape cache pointer exists and stores a native iterator
+// - the iterator is reusable
+// - the iterated object has no dense elements
+// - the shapes of each object on the proto chain of |obj| match the cached
+// shapes
+// - the proto chain has no dense elements
+// Otherwise, jump to |failure|.
+void MacroAssembler::maybeLoadIteratorFromShape(Register obj, Register dest,
+ Register temp, Register temp2,
+ Register temp3,
+ Label* failure) {
+ // Register usage:
+ // obj: always contains the input object
+ // temp: walks the obj->shape->baseshape->proto->shape->... chain
+ // temp2: points to the native iterator. Incremented to walk the shapes array.
+ // temp3: scratch space
+ // dest: stores the resulting PropertyIteratorObject on success
+
+ Label success;
+ Register shapeAndProto = temp;
+ Register nativeIterator = temp2;
+
+ // Load ShapeCache from shape.
+ loadPtr(Address(obj, JSObject::offsetOfShape()), shapeAndProto);
+ loadPtr(Address(shapeAndProto, Shape::offsetOfCachePtr()), dest);
+
+ // Check if it's an iterator.
+ movePtr(dest, temp3);
+ andPtr(Imm32(ShapeCachePtr::MASK), temp3);
+ branch32(Assembler::NotEqual, temp3, Imm32(ShapeCachePtr::ITERATOR), failure);
+
+ // If we've cached an iterator, |obj| must be a native object.
+#ifdef DEBUG
+ Label nonNative;
+ branchIfNonNativeObj(obj, temp3, &nonNative);
+#endif
+
+ // Verify that |obj| has no dense elements.
+ loadPtr(Address(obj, NativeObject::offsetOfElements()), temp3);
+ branch32(Assembler::NotEqual,
+ Address(temp3, ObjectElements::offsetOfInitializedLength()),
+ Imm32(0), failure);
+
+ // Clear tag bits from iterator object. |dest| is now valid.
+ // Load the native iterator and verify that it's reusable.
+ andPtr(Imm32(~ShapeCachePtr::MASK), dest);
+ LoadNativeIterator(*this, dest, nativeIterator);
+ branchIfNativeIteratorNotReusable(nativeIterator, failure);
+
+ // We have to compare the shapes in the native iterator with the shapes on the
+ // proto chain to ensure the cached iterator is still valid. The shape array
+ // always starts at a fixed offset from the base of the NativeIterator, so
+ // instead of using an instruction outside the loop to initialize a pointer to
+ // the shapes array, we can bake it into the offset and reuse the pointer to
+ // the NativeIterator. We add |sizeof(Shape*)| to start at the second shape.
+ // (The first shape corresponds to the object itself. We don't have to check
+ // it, because we got the iterator via the shape.)
+ size_t nativeIteratorProtoShapeOffset =
+ NativeIterator::offsetOfFirstShape() + sizeof(Shape*);
+
+ // Loop over the proto chain. At the head of the loop, |shape| is the shape of
+ // the current object, and |iteratorShapes| points to the expected shape of
+ // its proto.
+ Label protoLoop;
+ bind(&protoLoop);
+
+ // Load the proto. If the proto is null, then we're done.
+ loadPtr(Address(shapeAndProto, Shape::offsetOfBaseShape()), shapeAndProto);
+ loadPtr(Address(shapeAndProto, BaseShape::offsetOfProto()), shapeAndProto);
+ branchPtr(Assembler::Equal, shapeAndProto, ImmPtr(nullptr), &success);
+
+#ifdef DEBUG
+ // We have guarded every shape up until this point, so we know that the proto
+ // is a native object.
+ branchIfNonNativeObj(shapeAndProto, temp3, &nonNative);
+#endif
+
+ // Verify that the proto has no dense elements.
+ loadPtr(Address(shapeAndProto, NativeObject::offsetOfElements()), temp3);
+ branch32(Assembler::NotEqual,
+ Address(temp3, ObjectElements::offsetOfInitializedLength()),
+ Imm32(0), failure);
+
+ // Compare the shape of the proto to the expected shape.
+ loadPtr(Address(shapeAndProto, JSObject::offsetOfShape()), shapeAndProto);
+ loadPtr(Address(nativeIterator, nativeIteratorProtoShapeOffset), temp3);
+ branchPtr(Assembler::NotEqual, shapeAndProto, temp3, failure);
+
+ // Increment |iteratorShapes| and jump back to the top of the loop.
+ addPtr(Imm32(sizeof(Shape*)), nativeIterator);
+ jump(&protoLoop);
+
+#ifdef DEBUG
+ bind(&nonNative);
+ assumeUnreachable("Expected NativeObject in maybeLoadIteratorFromShape");
+#endif
+
+ bind(&success);
+}
+
+void MacroAssembler::iteratorMore(Register obj, ValueOperand output,
+ Register temp) {
+ Label done;
+ Register outputScratch = output.scratchReg();
+ LoadNativeIterator(*this, obj, outputScratch);
+
+ // If propertyCursor_ < propertiesEnd_, load the next string and advance
+ // the cursor. Otherwise return MagicValue(JS_NO_ITER_VALUE).
+ Label iterDone;
+ Address cursorAddr(outputScratch, NativeIterator::offsetOfPropertyCursor());
+ Address cursorEndAddr(outputScratch, NativeIterator::offsetOfPropertiesEnd());
+ loadPtr(cursorAddr, temp);
+ branchPtr(Assembler::BelowOrEqual, cursorEndAddr, temp, &iterDone);
+
+ // Get next string.
+ loadPtr(Address(temp, 0), temp);
+
+ // Increase the cursor.
+ addPtr(Imm32(sizeof(GCPtr<JSLinearString*>)), cursorAddr);
+
+ tagValue(JSVAL_TYPE_STRING, temp, output);
+ jump(&done);
+
+ bind(&iterDone);
+ moveValue(MagicValue(JS_NO_ITER_VALUE), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::iteratorClose(Register obj, Register temp1, Register temp2,
+ Register temp3) {
+ LoadNativeIterator(*this, obj, temp1);
+
+ // The shared iterator used for for-in with null/undefined is immutable and
+ // unlinked. See NativeIterator::isEmptyIteratorSingleton.
+ Label done;
+ branchTest32(Assembler::NonZero,
+ Address(temp1, NativeIterator::offsetOfFlagsAndCount()),
+ Imm32(NativeIterator::Flags::IsEmptyIteratorSingleton), &done);
+
+ // Clear active bit.
+ and32(Imm32(~NativeIterator::Flags::Active),
+ Address(temp1, NativeIterator::offsetOfFlagsAndCount()));
+
+ // Clear objectBeingIterated.
+ Address iterObjAddr(temp1, NativeIterator::offsetOfObjectBeingIterated());
+ guardedCallPreBarrierAnyZone(iterObjAddr, MIRType::Object, temp2);
+ storePtr(ImmPtr(nullptr), iterObjAddr);
+
+ // Reset property cursor.
+ loadPtr(Address(temp1, NativeIterator::offsetOfShapesEnd()), temp2);
+ storePtr(temp2, Address(temp1, NativeIterator::offsetOfPropertyCursor()));
+
+ // Unlink from the iterator list.
+ const Register next = temp2;
+ const Register prev = temp3;
+ loadPtr(Address(temp1, NativeIterator::offsetOfNext()), next);
+ loadPtr(Address(temp1, NativeIterator::offsetOfPrev()), prev);
+ storePtr(prev, Address(next, NativeIterator::offsetOfPrev()));
+ storePtr(next, Address(prev, NativeIterator::offsetOfNext()));
+#ifdef DEBUG
+ storePtr(ImmPtr(nullptr), Address(temp1, NativeIterator::offsetOfNext()));
+ storePtr(ImmPtr(nullptr), Address(temp1, NativeIterator::offsetOfPrev()));
+#endif
+
+ bind(&done);
+}
+
+void MacroAssembler::registerIterator(Register enumeratorsList, Register iter,
+ Register temp) {
+ // iter->next = list
+ storePtr(enumeratorsList, Address(iter, NativeIterator::offsetOfNext()));
+
+ // iter->prev = list->prev
+ loadPtr(Address(enumeratorsList, NativeIterator::offsetOfPrev()), temp);
+ storePtr(temp, Address(iter, NativeIterator::offsetOfPrev()));
+
+ // list->prev->next = iter
+ storePtr(iter, Address(temp, NativeIterator::offsetOfNext()));
+
+ // list->prev = iter
+ storePtr(iter, Address(enumeratorsList, NativeIterator::offsetOfPrev()));
+}
+
+void MacroAssembler::toHashableNonGCThing(ValueOperand value,
+ ValueOperand result,
+ FloatRegister tempFloat) {
+ // Inline implementation of |HashableValue::setValue()|.
+
+#ifdef DEBUG
+ Label ok;
+ branchTestGCThing(Assembler::NotEqual, value, &ok);
+ assumeUnreachable("Unexpected GC thing");
+ bind(&ok);
+#endif
+
+ Label useInput, done;
+ branchTestDouble(Assembler::NotEqual, value, &useInput);
+ {
+ Register int32 = result.scratchReg();
+ unboxDouble(value, tempFloat);
+
+ // Normalize int32-valued doubles to int32 and negative zero to +0.
+ Label canonicalize;
+ convertDoubleToInt32(tempFloat, int32, &canonicalize, false);
+ {
+ tagValue(JSVAL_TYPE_INT32, int32, result);
+ jump(&done);
+ }
+ bind(&canonicalize);
+ {
+ // Normalize the sign bit of a NaN.
+ branchDouble(Assembler::DoubleOrdered, tempFloat, tempFloat, &useInput);
+ moveValue(JS::NaNValue(), result);
+ jump(&done);
+ }
+ }
+
+ bind(&useInput);
+ moveValue(value, result);
+
+ bind(&done);
+}
+
+void MacroAssembler::toHashableValue(ValueOperand value, ValueOperand result,
+ FloatRegister tempFloat,
+ Label* atomizeString, Label* tagString) {
+ // Inline implementation of |HashableValue::setValue()|.
+
+ ScratchTagScope tag(*this, value);
+ splitTagForTest(value, tag);
+
+ Label notString, useInput, done;
+ branchTestString(Assembler::NotEqual, tag, &notString);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ Register str = result.scratchReg();
+ unboxString(value, str);
+
+ branchTest32(Assembler::NonZero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::ATOM_BIT), &useInput);
+
+ jump(atomizeString);
+ bind(tagString);
+
+ tagValue(JSVAL_TYPE_STRING, str, result);
+ jump(&done);
+ }
+ bind(&notString);
+ branchTestDouble(Assembler::NotEqual, tag, &useInput);
+ {
+ ScratchTagScopeRelease _(&tag);
+
+ Register int32 = result.scratchReg();
+ unboxDouble(value, tempFloat);
+
+ Label canonicalize;
+ convertDoubleToInt32(tempFloat, int32, &canonicalize, false);
+ {
+ tagValue(JSVAL_TYPE_INT32, int32, result);
+ jump(&done);
+ }
+ bind(&canonicalize);
+ {
+ branchDouble(Assembler::DoubleOrdered, tempFloat, tempFloat, &useInput);
+ moveValue(JS::NaNValue(), result);
+ jump(&done);
+ }
+ }
+
+ bind(&useInput);
+ moveValue(value, result);
+
+ bind(&done);
+}
+
+void MacroAssembler::scrambleHashCode(Register result) {
+ // Inline implementation of |mozilla::ScrambleHashCode()|.
+
+ mul32(Imm32(mozilla::kGoldenRatioU32), result);
+}
+
+void MacroAssembler::prepareHashNonGCThing(ValueOperand value, Register result,
+ Register temp) {
+ // Inline implementation of |OrderedHashTable::prepareHash()| and
+ // |mozilla::HashGeneric(v.asRawBits())|.
+
+#ifdef DEBUG
+ Label ok;
+ branchTestGCThing(Assembler::NotEqual, value, &ok);
+ assumeUnreachable("Unexpected GC thing");
+ bind(&ok);
+#endif
+
+ // uint32_t v1 = static_cast<uint32_t>(aValue);
+#ifdef JS_PUNBOX64
+ move64To32(value.toRegister64(), result);
+#else
+ move32(value.payloadReg(), result);
+#endif
+
+ // uint32_t v2 = static_cast<uint32_t>(static_cast<uint64_t>(aValue) >> 32);
+#ifdef JS_PUNBOX64
+ auto r64 = Register64(temp);
+ move64(value.toRegister64(), r64);
+ rshift64Arithmetic(Imm32(32), r64);
+#else
+ // TODO: This seems like a bug in mozilla::detail::AddUintptrToHash().
+ // The uint64_t input is first converted to uintptr_t and then back to
+ // uint64_t. But |uint64_t(uintptr_t(bits))| actually only clears the high
+ // bits, so this computation:
+ //
+ // aValue = uintptr_t(bits)
+ // v2 = static_cast<uint32_t>(static_cast<uint64_t>(aValue) >> 32)
+ //
+ // really just sets |v2 = 0|. And that means the xor-operation in AddU32ToHash
+ // can be optimized away, because |x ^ 0 = x|.
+ //
+ // Filed as bug 1718516.
+#endif
+
+ // mozilla::WrappingMultiply(kGoldenRatioU32, RotateLeft5(aHash) ^ aValue);
+ // with |aHash = 0| and |aValue = v1|.
+ mul32(Imm32(mozilla::kGoldenRatioU32), result);
+
+ // mozilla::WrappingMultiply(kGoldenRatioU32, RotateLeft5(aHash) ^ aValue);
+ // with |aHash = <above hash>| and |aValue = v2|.
+ rotateLeft(Imm32(5), result, result);
+#ifdef JS_PUNBOX64
+ xor32(temp, result);
+#endif
+
+ // Combine |mul32| and |scrambleHashCode| by directly multiplying with
+ // |kGoldenRatioU32 * kGoldenRatioU32|.
+ //
+ // mul32(Imm32(mozilla::kGoldenRatioU32), result);
+ //
+ // scrambleHashCode(result);
+ mul32(Imm32(mozilla::kGoldenRatioU32 * mozilla::kGoldenRatioU32), result);
+}
+
+void MacroAssembler::prepareHashString(Register str, Register result,
+ Register temp) {
+ // Inline implementation of |OrderedHashTable::prepareHash()| and
+ // |JSAtom::hash()|.
+
+#ifdef DEBUG
+ Label ok;
+ branchTest32(Assembler::NonZero, Address(str, JSString::offsetOfFlags()),
+ Imm32(JSString::ATOM_BIT), &ok);
+ assumeUnreachable("Unexpected non-atom string");
+ bind(&ok);
+#endif
+
+ move32(Imm32(JSString::FAT_INLINE_MASK), temp);
+ and32(Address(str, JSString::offsetOfFlags()), temp);
+
+ // Set |result| to 1 for FatInlineAtoms.
+ move32(Imm32(0), result);
+ cmp32Set(Assembler::Equal, temp, Imm32(JSString::FAT_INLINE_MASK), result);
+
+ // Use a computed load for branch-free code.
+
+ static_assert(FatInlineAtom::offsetOfHash() > NormalAtom::offsetOfHash());
+
+ constexpr size_t offsetDiff =
+ FatInlineAtom::offsetOfHash() - NormalAtom::offsetOfHash();
+ static_assert(mozilla::IsPowerOfTwo(offsetDiff));
+
+ uint8_t shift = mozilla::FloorLog2Size(offsetDiff);
+ if (IsShiftInScaleRange(shift)) {
+ load32(
+ BaseIndex(str, result, ShiftToScale(shift), NormalAtom::offsetOfHash()),
+ result);
+ } else {
+ lshift32(Imm32(shift), result);
+ load32(BaseIndex(str, result, TimesOne, NormalAtom::offsetOfHash()),
+ result);
+ }
+
+ scrambleHashCode(result);
+}
+
+void MacroAssembler::prepareHashSymbol(Register sym, Register result) {
+ // Inline implementation of |OrderedHashTable::prepareHash()| and
+ // |Symbol::hash()|.
+
+ load32(Address(sym, JS::Symbol::offsetOfHash()), result);
+
+ scrambleHashCode(result);
+}
+
+void MacroAssembler::prepareHashBigInt(Register bigInt, Register result,
+ Register temp1, Register temp2,
+ Register temp3) {
+ // Inline implementation of |OrderedHashTable::prepareHash()| and
+ // |BigInt::hash()|.
+
+ // Inline implementation of |mozilla::AddU32ToHash()|.
+ auto addU32ToHash = [&](auto toAdd) {
+ rotateLeft(Imm32(5), result, result);
+ xor32(toAdd, result);
+ mul32(Imm32(mozilla::kGoldenRatioU32), result);
+ };
+
+ move32(Imm32(0), result);
+
+ // Inline |mozilla::HashBytes()|.
+
+ load32(Address(bigInt, BigInt::offsetOfLength()), temp1);
+ loadBigIntDigits(bigInt, temp2);
+
+ Label start, loop;
+ jump(&start);
+ bind(&loop);
+
+ {
+ // Compute |AddToHash(AddToHash(hash, data), sizeof(Digit))|.
+#if defined(JS_CODEGEN_MIPS64)
+ // Hash the lower 32-bits.
+ addU32ToHash(Address(temp2, 0));
+
+ // Hash the upper 32-bits.
+ addU32ToHash(Address(temp2, sizeof(int32_t)));
+#elif JS_PUNBOX64
+ // Use a single 64-bit load on non-MIPS64 platforms.
+ loadPtr(Address(temp2, 0), temp3);
+
+ // Hash the lower 32-bits.
+ addU32ToHash(temp3);
+
+ // Hash the upper 32-bits.
+ rshiftPtr(Imm32(32), temp3);
+ addU32ToHash(temp3);
+#else
+ addU32ToHash(Address(temp2, 0));
+#endif
+ }
+ addPtr(Imm32(sizeof(BigInt::Digit)), temp2);
+
+ bind(&start);
+ branchSub32(Assembler::NotSigned, Imm32(1), temp1, &loop);
+
+ // Compute |mozilla::AddToHash(h, isNegative())|.
+ {
+ static_assert(mozilla::IsPowerOfTwo(BigInt::signBitMask()));
+
+ load32(Address(bigInt, BigInt::offsetOfFlags()), temp1);
+ and32(Imm32(BigInt::signBitMask()), temp1);
+ rshift32(Imm32(mozilla::FloorLog2(BigInt::signBitMask())), temp1);
+
+ addU32ToHash(temp1);
+ }
+
+ scrambleHashCode(result);
+}
+
+void MacroAssembler::prepareHashObject(Register setObj, ValueOperand value,
+ Register result, Register temp1,
+ Register temp2, Register temp3,
+ Register temp4) {
+#ifdef JS_PUNBOX64
+ // Inline implementation of |OrderedHashTable::prepareHash()| and
+ // |HashCodeScrambler::scramble(v.asRawBits())|.
+
+ // Load the |ValueSet| or |ValueMap|.
+ static_assert(SetObject::getDataSlotOffset() ==
+ MapObject::getDataSlotOffset());
+ loadPrivate(Address(setObj, SetObject::getDataSlotOffset()), temp1);
+
+ // Load |HashCodeScrambler::mK0| and |HashCodeScrambler::mK0|.
+ static_assert(ValueSet::offsetOfImplHcsK0() == ValueMap::offsetOfImplHcsK0());
+ static_assert(ValueSet::offsetOfImplHcsK1() == ValueMap::offsetOfImplHcsK1());
+ auto k0 = Register64(temp1);
+ auto k1 = Register64(temp2);
+ load64(Address(temp1, ValueSet::offsetOfImplHcsK1()), k1);
+ load64(Address(temp1, ValueSet::offsetOfImplHcsK0()), k0);
+
+ // Hash numbers are 32-bit values, so only hash the lower double-word.
+ static_assert(sizeof(mozilla::HashNumber) == 4);
+ move32To64ZeroExtend(value.valueReg(), Register64(result));
+
+ // Inline implementation of |SipHasher::sipHash()|.
+ auto m = Register64(result);
+ auto v0 = Register64(temp3);
+ auto v1 = Register64(temp4);
+ auto v2 = k0;
+ auto v3 = k1;
+
+ auto sipRound = [&]() {
+ // mV0 = WrappingAdd(mV0, mV1);
+ add64(v1, v0);
+
+ // mV1 = RotateLeft(mV1, 13);
+ rotateLeft64(Imm32(13), v1, v1, InvalidReg);
+
+ // mV1 ^= mV0;
+ xor64(v0, v1);
+
+ // mV0 = RotateLeft(mV0, 32);
+ rotateLeft64(Imm32(32), v0, v0, InvalidReg);
+
+ // mV2 = WrappingAdd(mV2, mV3);
+ add64(v3, v2);
+
+ // mV3 = RotateLeft(mV3, 16);
+ rotateLeft64(Imm32(16), v3, v3, InvalidReg);
+
+ // mV3 ^= mV2;
+ xor64(v2, v3);
+
+ // mV0 = WrappingAdd(mV0, mV3);
+ add64(v3, v0);
+
+ // mV3 = RotateLeft(mV3, 21);
+ rotateLeft64(Imm32(21), v3, v3, InvalidReg);
+
+ // mV3 ^= mV0;
+ xor64(v0, v3);
+
+ // mV2 = WrappingAdd(mV2, mV1);
+ add64(v1, v2);
+
+ // mV1 = RotateLeft(mV1, 17);
+ rotateLeft64(Imm32(17), v1, v1, InvalidReg);
+
+ // mV1 ^= mV2;
+ xor64(v2, v1);
+
+ // mV2 = RotateLeft(mV2, 32);
+ rotateLeft64(Imm32(32), v2, v2, InvalidReg);
+ };
+
+ // 1. Initialization.
+ // mV0 = aK0 ^ UINT64_C(0x736f6d6570736575);
+ move64(Imm64(0x736f6d6570736575), v0);
+ xor64(k0, v0);
+
+ // mV1 = aK1 ^ UINT64_C(0x646f72616e646f6d);
+ move64(Imm64(0x646f72616e646f6d), v1);
+ xor64(k1, v1);
+
+ // mV2 = aK0 ^ UINT64_C(0x6c7967656e657261);
+ MOZ_ASSERT(v2 == k0);
+ xor64(Imm64(0x6c7967656e657261), v2);
+
+ // mV3 = aK1 ^ UINT64_C(0x7465646279746573);
+ MOZ_ASSERT(v3 == k1);
+ xor64(Imm64(0x7465646279746573), v3);
+
+ // 2. Compression.
+ // mV3 ^= aM;
+ xor64(m, v3);
+
+ // sipRound();
+ sipRound();
+
+ // mV0 ^= aM;
+ xor64(m, v0);
+
+ // 3. Finalization.
+ // mV2 ^= 0xff;
+ xor64(Imm64(0xff), v2);
+
+ // for (int i = 0; i < 3; i++) sipRound();
+ for (int i = 0; i < 3; i++) {
+ sipRound();
+ }
+
+ // return mV0 ^ mV1 ^ mV2 ^ mV3;
+ xor64(v1, v0);
+ xor64(v2, v3);
+ xor64(v3, v0);
+
+ move64To32(v0, result);
+
+ scrambleHashCode(result);
+#else
+ MOZ_CRASH("Not implemented");
+#endif
+}
+
+void MacroAssembler::prepareHashValue(Register setObj, ValueOperand value,
+ Register result, Register temp1,
+ Register temp2, Register temp3,
+ Register temp4) {
+ Label isString, isObject, isSymbol, isBigInt;
+ {
+ ScratchTagScope tag(*this, value);
+ splitTagForTest(value, tag);
+
+ branchTestString(Assembler::Equal, tag, &isString);
+ branchTestObject(Assembler::Equal, tag, &isObject);
+ branchTestSymbol(Assembler::Equal, tag, &isSymbol);
+ branchTestBigInt(Assembler::Equal, tag, &isBigInt);
+ }
+
+ Label done;
+ {
+ prepareHashNonGCThing(value, result, temp1);
+ jump(&done);
+ }
+ bind(&isString);
+ {
+ unboxString(value, temp1);
+ prepareHashString(temp1, result, temp2);
+ jump(&done);
+ }
+ bind(&isObject);
+ {
+ prepareHashObject(setObj, value, result, temp1, temp2, temp3, temp4);
+ jump(&done);
+ }
+ bind(&isSymbol);
+ {
+ unboxSymbol(value, temp1);
+ prepareHashSymbol(temp1, result);
+ jump(&done);
+ }
+ bind(&isBigInt);
+ {
+ unboxBigInt(value, temp1);
+ prepareHashBigInt(temp1, result, temp2, temp3, temp4);
+
+ // Fallthrough to |done|.
+ }
+
+ bind(&done);
+}
+
+template <typename OrderedHashTable>
+void MacroAssembler::orderedHashTableLookup(Register setOrMapObj,
+ ValueOperand value, Register hash,
+ Register entryTemp, Register temp1,
+ Register temp2, Register temp3,
+ Register temp4, Label* found,
+ IsBigInt isBigInt) {
+ // Inline implementation of |OrderedHashTable::lookup()|.
+
+ MOZ_ASSERT_IF(isBigInt == IsBigInt::No, temp3 == InvalidReg);
+ MOZ_ASSERT_IF(isBigInt == IsBigInt::No, temp4 == InvalidReg);
+
+#ifdef DEBUG
+ Label ok;
+ if (isBigInt == IsBigInt::No) {
+ branchTestBigInt(Assembler::NotEqual, value, &ok);
+ assumeUnreachable("Unexpected BigInt");
+ } else if (isBigInt == IsBigInt::Yes) {
+ branchTestBigInt(Assembler::Equal, value, &ok);
+ assumeUnreachable("Unexpected non-BigInt");
+ }
+ bind(&ok);
+#endif
+
+#ifdef DEBUG
+ PushRegsInMask(LiveRegisterSet(RegisterSet::Volatile()));
+
+ pushValue(value);
+ moveStackPtrTo(temp2);
+
+ setupUnalignedABICall(temp1);
+ loadJSContext(temp1);
+ passABIArg(temp1);
+ passABIArg(setOrMapObj);
+ passABIArg(temp2);
+ passABIArg(hash);
+
+ if constexpr (std::is_same_v<OrderedHashTable, ValueSet>) {
+ using Fn =
+ void (*)(JSContext*, SetObject*, const Value*, mozilla::HashNumber);
+ callWithABI<Fn, jit::AssertSetObjectHash>();
+ } else {
+ using Fn =
+ void (*)(JSContext*, MapObject*, const Value*, mozilla::HashNumber);
+ callWithABI<Fn, jit::AssertMapObjectHash>();
+ }
+
+ popValue(value);
+ PopRegsInMask(LiveRegisterSet(RegisterSet::Volatile()));
+#endif
+
+ // Load the |ValueSet| or |ValueMap|.
+ static_assert(SetObject::getDataSlotOffset() ==
+ MapObject::getDataSlotOffset());
+ loadPrivate(Address(setOrMapObj, SetObject::getDataSlotOffset()), temp1);
+
+ // Load the bucket.
+ move32(hash, entryTemp);
+ load32(Address(temp1, OrderedHashTable::offsetOfImplHashShift()), temp2);
+ flexibleRshift32(temp2, entryTemp);
+
+ loadPtr(Address(temp1, OrderedHashTable::offsetOfImplHashTable()), temp2);
+ loadPtr(BaseIndex(temp2, entryTemp, ScalePointer), entryTemp);
+
+ // Search for a match in this bucket.
+ Label start, loop;
+ jump(&start);
+ bind(&loop);
+ {
+ // Inline implementation of |HashableValue::operator==|.
+
+ static_assert(OrderedHashTable::offsetOfImplDataElement() == 0,
+ "offsetof(Data, element) is 0");
+ auto keyAddr = Address(entryTemp, OrderedHashTable::offsetOfEntryKey());
+
+ if (isBigInt == IsBigInt::No) {
+ // Two HashableValues are equal if they have equal bits.
+ branch64(Assembler::Equal, keyAddr, value.toRegister64(), found);
+ } else {
+#ifdef JS_PUNBOX64
+ auto key = ValueOperand(temp1);
+#else
+ auto key = ValueOperand(temp1, temp2);
+#endif
+
+ loadValue(keyAddr, key);
+
+ // Two HashableValues are equal if they have equal bits.
+ branch64(Assembler::Equal, key.toRegister64(), value.toRegister64(),
+ found);
+
+ // BigInt values are considered equal if they represent the same
+ // mathematical value.
+ Label next;
+ fallibleUnboxBigInt(key, temp2, &next);
+ if (isBigInt == IsBigInt::Yes) {
+ unboxBigInt(value, temp1);
+ } else {
+ fallibleUnboxBigInt(value, temp1, &next);
+ }
+ equalBigInts(temp1, temp2, temp3, temp4, temp1, temp2, &next, &next,
+ &next);
+ jump(found);
+ bind(&next);
+ }
+ }
+ loadPtr(Address(entryTemp, OrderedHashTable::offsetOfImplDataChain()),
+ entryTemp);
+ bind(&start);
+ branchTestPtr(Assembler::NonZero, entryTemp, entryTemp, &loop);
+}
+
+void MacroAssembler::setObjectHas(Register setObj, ValueOperand value,
+ Register hash, Register result,
+ Register temp1, Register temp2,
+ Register temp3, Register temp4,
+ IsBigInt isBigInt) {
+ Label found;
+ orderedHashTableLookup<ValueSet>(setObj, value, hash, result, temp1, temp2,
+ temp3, temp4, &found, isBigInt);
+
+ Label done;
+ move32(Imm32(0), result);
+ jump(&done);
+
+ bind(&found);
+ move32(Imm32(1), result);
+ bind(&done);
+}
+
+void MacroAssembler::mapObjectHas(Register mapObj, ValueOperand value,
+ Register hash, Register result,
+ Register temp1, Register temp2,
+ Register temp3, Register temp4,
+ IsBigInt isBigInt) {
+ Label found;
+ orderedHashTableLookup<ValueMap>(mapObj, value, hash, result, temp1, temp2,
+ temp3, temp4, &found, isBigInt);
+
+ Label done;
+ move32(Imm32(0), result);
+ jump(&done);
+
+ bind(&found);
+ move32(Imm32(1), result);
+ bind(&done);
+}
+
+void MacroAssembler::mapObjectGet(Register mapObj, ValueOperand value,
+ Register hash, ValueOperand result,
+ Register temp1, Register temp2,
+ Register temp3, Register temp4,
+ Register temp5, IsBigInt isBigInt) {
+ Label found;
+ orderedHashTableLookup<ValueMap>(mapObj, value, hash, temp1, temp2, temp3,
+ temp4, temp5, &found, isBigInt);
+
+ Label done;
+ moveValue(UndefinedValue(), result);
+ jump(&done);
+
+ // |temp1| holds the found entry.
+ bind(&found);
+ loadValue(Address(temp1, ValueMap::Entry::offsetOfValue()), result);
+
+ bind(&done);
+}
+
+template <typename OrderedHashTable>
+void MacroAssembler::loadOrderedHashTableCount(Register setOrMapObj,
+ Register result) {
+ // Inline implementation of |OrderedHashTable::count()|.
+
+ // Load the |ValueSet| or |ValueMap|.
+ static_assert(SetObject::getDataSlotOffset() ==
+ MapObject::getDataSlotOffset());
+ loadPrivate(Address(setOrMapObj, SetObject::getDataSlotOffset()), result);
+
+ // Load the live count.
+ load32(Address(result, OrderedHashTable::offsetOfImplLiveCount()), result);
+}
+
+void MacroAssembler::loadSetObjectSize(Register setObj, Register result) {
+ loadOrderedHashTableCount<ValueSet>(setObj, result);
+}
+
+void MacroAssembler::loadMapObjectSize(Register mapObj, Register result) {
+ loadOrderedHashTableCount<ValueMap>(mapObj, result);
+}
+
+// Can't push large frames blindly on windows, so we must touch frame memory
+// incrementally, with no more than 4096 - 1 bytes between touches.
+//
+// This is used across all platforms for simplicity.
+void MacroAssembler::touchFrameValues(Register numStackValues,
+ Register scratch1, Register scratch2) {
+ const size_t FRAME_TOUCH_INCREMENT = 2048;
+ static_assert(FRAME_TOUCH_INCREMENT < 4096 - 1,
+ "Frame increment is too large");
+
+ moveStackPtrTo(scratch2);
+ mov(numStackValues, scratch1);
+ lshiftPtr(Imm32(3), scratch1);
+ subPtr(scratch1, scratch2);
+ {
+ moveStackPtrTo(scratch1);
+ subPtr(Imm32(FRAME_TOUCH_INCREMENT), scratch1);
+
+ Label touchFrameLoop;
+ Label touchFrameLoopEnd;
+ bind(&touchFrameLoop);
+ branchPtr(Assembler::Below, scratch1, scratch2, &touchFrameLoopEnd);
+ store32(Imm32(0), Address(scratch1, 0));
+ subPtr(Imm32(FRAME_TOUCH_INCREMENT), scratch1);
+ jump(&touchFrameLoop);
+ bind(&touchFrameLoopEnd);
+ }
+}
+
+namespace js {
+namespace jit {
+
+#ifdef DEBUG
+template <class RegisterType>
+AutoGenericRegisterScope<RegisterType>::AutoGenericRegisterScope(
+ MacroAssembler& masm, RegisterType reg)
+ : RegisterType(reg), masm_(masm), released_(false) {
+ masm.debugTrackedRegisters_.add(reg);
+}
+
+template AutoGenericRegisterScope<Register>::AutoGenericRegisterScope(
+ MacroAssembler& masm, Register reg);
+template AutoGenericRegisterScope<FloatRegister>::AutoGenericRegisterScope(
+ MacroAssembler& masm, FloatRegister reg);
+#endif // DEBUG
+
+#ifdef DEBUG
+template <class RegisterType>
+AutoGenericRegisterScope<RegisterType>::~AutoGenericRegisterScope() {
+ if (!released_) {
+ release();
+ }
+}
+
+template AutoGenericRegisterScope<Register>::~AutoGenericRegisterScope();
+template AutoGenericRegisterScope<FloatRegister>::~AutoGenericRegisterScope();
+
+template <class RegisterType>
+void AutoGenericRegisterScope<RegisterType>::release() {
+ MOZ_ASSERT(!released_);
+ released_ = true;
+ const RegisterType& reg = *dynamic_cast<RegisterType*>(this);
+ masm_.debugTrackedRegisters_.take(reg);
+}
+
+template void AutoGenericRegisterScope<Register>::release();
+template void AutoGenericRegisterScope<FloatRegister>::release();
+
+template <class RegisterType>
+void AutoGenericRegisterScope<RegisterType>::reacquire() {
+ MOZ_ASSERT(released_);
+ released_ = false;
+ const RegisterType& reg = *dynamic_cast<RegisterType*>(this);
+ masm_.debugTrackedRegisters_.add(reg);
+}
+
+template void AutoGenericRegisterScope<Register>::reacquire();
+template void AutoGenericRegisterScope<FloatRegister>::reacquire();
+
+#endif // DEBUG
+
+} // namespace jit
+
+} // namespace js
diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h
new file mode 100644
index 0000000000..7aba76d0e7
--- /dev/null
+++ b/js/src/jit/MacroAssembler.h
@@ -0,0 +1,5522 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MacroAssembler_h
+#define jit_MacroAssembler_h
+
+#include "mozilla/EndianUtils.h"
+#include "mozilla/MacroForEach.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/Variant.h"
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/MacroAssembler-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/MacroAssembler-x64.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/MacroAssembler-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/MacroAssembler-arm64.h"
+#elif defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/MacroAssembler-mips32.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/MacroAssembler-mips64.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/MacroAssembler-loong64.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/MacroAssembler-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/MacroAssembler-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+#include "jit/ABIArgGenerator.h"
+#include "jit/ABIFunctions.h"
+#include "jit/AtomicOp.h"
+#include "jit/IonTypes.h"
+#include "jit/MoveResolver.h"
+#include "jit/VMFunctions.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "util/Memory.h"
+#include "vm/FunctionFlags.h"
+#include "vm/Opcodes.h"
+#include "wasm/WasmCodegenTypes.h"
+#include "wasm/WasmFrame.h"
+
+// [SMDOC] MacroAssembler multi-platform overview
+//
+// * How to read/write MacroAssembler method declarations:
+//
+// The following macros are made to avoid #ifdef around each method declarations
+// of the Macro Assembler, and they are also used as an hint on the location of
+// the implementations of each method. For example, the following declaration
+//
+// void Pop(FloatRegister t) DEFINED_ON(x86_shared, arm);
+//
+// suggests the MacroAssembler::Pop(FloatRegister) method is implemented in
+// x86-shared/MacroAssembler-x86-shared.h, and also in arm/MacroAssembler-arm.h.
+//
+// - If there is no annotation, then there is only one generic definition in
+// MacroAssembler.cpp.
+//
+// - If the declaration is "inline", then the method definition(s) would be in
+// the "-inl.h" variant of the same file(s).
+//
+// The script check_macroassembler_style.py (which runs on every build) is
+// used to verify that method definitions match the annotation on the method
+// declarations. If there is any difference, then you either forgot to define
+// the method in one of the macro assembler, or you forgot to update the
+// annotation of the macro assembler declaration.
+//
+// Some convenient short-cuts are used to avoid repeating the same list of
+// architectures on each method declaration, such as PER_ARCH and
+// PER_SHARED_ARCH.
+//
+// Functions that are architecture-agnostic and are the same for all
+// architectures, that it's necessary to define inline *in this header* to
+// avoid used-before-defined warnings/errors that would occur if the
+// definitions were in MacroAssembler-inl.h, should use the OOL_IN_HEADER
+// marker at end of the declaration:
+//
+// inline uint32_t framePushed() const OOL_IN_HEADER;
+//
+// Such functions should then be defined immediately after MacroAssembler's
+// definition, for example:
+//
+// //{{{ check_macroassembler_style
+// inline uint32_t
+// MacroAssembler::framePushed() const
+// {
+// return framePushed_;
+// }
+// ////}}} check_macroassembler_style
+
+#define ALL_ARCH mips32, mips64, arm, arm64, x86, x64, loong64, wasm32
+#define ALL_SHARED_ARCH arm, arm64, loong64, x86_shared, mips_shared, wasm32
+
+// * How this macro works:
+//
+// DEFINED_ON is a macro which check if, for the current architecture, the
+// method is defined on the macro assembler or not.
+//
+// For each architecture, we have a macro named DEFINED_ON_arch. This macro is
+// empty if this is not the current architecture. Otherwise it must be either
+// set to "define" or "crash" (only used for the none target so far).
+//
+// The DEFINED_ON macro maps the list of architecture names given as arguments
+// to a list of macro names. For example,
+//
+// DEFINED_ON(arm, x86_shared)
+//
+// is expanded to
+//
+// DEFINED_ON_none DEFINED_ON_arm DEFINED_ON_x86_shared
+//
+// which are later expanded on ARM, x86, x64 by DEFINED_ON_EXPAND_ARCH_RESULTS
+// to
+//
+// define
+//
+// or if the JIT is disabled or set to no architecture to
+//
+// crash
+//
+// or to nothing, if the current architecture is not listed in the list of
+// arguments of DEFINED_ON. Note, only one of the DEFINED_ON_arch macro
+// contributes to the non-empty result, which is the macro of the current
+// architecture if it is listed in the arguments of DEFINED_ON.
+//
+// This result is appended to DEFINED_ON_RESULT_ before expanding the macro,
+// which results in either no annotation, a MOZ_CRASH(), or a "= delete"
+// annotation on the method declaration.
+
+#define DEFINED_ON_x86
+#define DEFINED_ON_x64
+#define DEFINED_ON_x86_shared
+#define DEFINED_ON_arm
+#define DEFINED_ON_arm64
+#define DEFINED_ON_mips32
+#define DEFINED_ON_mips64
+#define DEFINED_ON_mips_shared
+#define DEFINED_ON_loong64
+#define DEFINED_ON_wasm32
+#define DEFINED_ON_none
+
+// Specialize for each architecture.
+#if defined(JS_CODEGEN_X86)
+# undef DEFINED_ON_x86
+# define DEFINED_ON_x86 define
+# undef DEFINED_ON_x86_shared
+# define DEFINED_ON_x86_shared define
+#elif defined(JS_CODEGEN_X64)
+# undef DEFINED_ON_x64
+# define DEFINED_ON_x64 define
+# undef DEFINED_ON_x86_shared
+# define DEFINED_ON_x86_shared define
+#elif defined(JS_CODEGEN_ARM)
+# undef DEFINED_ON_arm
+# define DEFINED_ON_arm define
+#elif defined(JS_CODEGEN_ARM64)
+# undef DEFINED_ON_arm64
+# define DEFINED_ON_arm64 define
+#elif defined(JS_CODEGEN_MIPS32)
+# undef DEFINED_ON_mips32
+# define DEFINED_ON_mips32 define
+# undef DEFINED_ON_mips_shared
+# define DEFINED_ON_mips_shared define
+#elif defined(JS_CODEGEN_MIPS64)
+# undef DEFINED_ON_mips64
+# define DEFINED_ON_mips64 define
+# undef DEFINED_ON_mips_shared
+# define DEFINED_ON_mips_shared define
+#elif defined(JS_CODEGEN_LOONG64)
+# undef DEFINED_ON_loong64
+# define DEFINED_ON_loong64 define
+#elif defined(JS_CODEGEN_WASM32)
+# undef DEFINED_ON_wasm32
+# define DEFINED_ON_wasm32 define
+#elif defined(JS_CODEGEN_NONE)
+# undef DEFINED_ON_none
+# define DEFINED_ON_none crash
+#else
+# error "Unknown architecture!"
+#endif
+
+#define DEFINED_ON_RESULT_crash \
+ { MOZ_CRASH(); }
+#define DEFINED_ON_RESULT_define
+#define DEFINED_ON_RESULT_ = delete
+
+#define DEFINED_ON_DISPATCH_RESULT_2(Macro, Result) Macro##Result
+#define DEFINED_ON_DISPATCH_RESULT(...) \
+ DEFINED_ON_DISPATCH_RESULT_2(DEFINED_ON_RESULT_, __VA_ARGS__)
+
+// We need to let the evaluation of MOZ_FOR_EACH terminates.
+#define DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult) \
+ DEFINED_ON_DISPATCH_RESULT ParenResult
+#define DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult) \
+ DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult)
+#define DEFINED_ON_EXPAND_ARCH_RESULTS(ParenResult) \
+ DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult)
+
+#define DEFINED_ON_FWDARCH(Arch) DEFINED_ON_##Arch
+#define DEFINED_ON_MAP_ON_ARCHS(ArchList) \
+ DEFINED_ON_EXPAND_ARCH_RESULTS( \
+ (MOZ_FOR_EACH(DEFINED_ON_FWDARCH, (), ArchList)))
+
+#define DEFINED_ON(...) DEFINED_ON_MAP_ON_ARCHS((none, __VA_ARGS__))
+
+#define PER_ARCH DEFINED_ON(ALL_ARCH)
+#define PER_SHARED_ARCH DEFINED_ON(ALL_SHARED_ARCH)
+#define OOL_IN_HEADER
+
+namespace JS {
+struct ExpandoAndGeneration;
+}
+
+namespace js {
+
+class StaticStrings;
+class TypedArrayObject;
+
+namespace wasm {
+class CalleeDesc;
+class CallSiteDesc;
+class BytecodeOffset;
+class MemoryAccessDesc;
+
+struct ModuleEnvironment;
+
+enum class FailureMode : uint8_t;
+enum class SimdOp;
+enum class SymbolicAddress;
+enum class Trap;
+} // namespace wasm
+
+namespace jit {
+
+// Defined in JitFrames.h
+enum class ExitFrameType : uint8_t;
+
+class AutoSaveLiveRegisters;
+class CompileZone;
+class TemplateNativeObject;
+class TemplateObject;
+
+enum class CheckUnsafeCallWithABI {
+ // Require the callee to use AutoUnsafeCallWithABI.
+ Check,
+
+ // We pushed an exit frame so this callWithABI can safely GC and walk the
+ // stack.
+ DontCheckHasExitFrame,
+
+ // Don't check this callWithABI uses AutoUnsafeCallWithABI, for instance
+ // because we're calling a simple helper function (like malloc or js_free)
+ // that we can't change and/or that we know won't GC.
+ DontCheckOther,
+};
+
+// This is a global function made to create the DynFn type in a controlled
+// environment which would check if the function signature has been registered
+// as an ABI function signature.
+template <typename Sig>
+static inline DynFn DynamicFunction(Sig fun);
+
+enum class CharEncoding { Latin1, TwoByte };
+
+constexpr uint32_t WasmCallerInstanceOffsetBeforeCall =
+ wasm::FrameWithInstances::callerInstanceOffsetWithoutFrame();
+constexpr uint32_t WasmCalleeInstanceOffsetBeforeCall =
+ wasm::FrameWithInstances::calleeInstanceOffsetWithoutFrame();
+
+// Allocation sites may be passed to GC thing allocation methods either via a
+// register (for baseline compilation) or an enum indicating one of the
+// catch-all allocation sites (for optimized compilation).
+struct AllocSiteInput
+ : public mozilla::Variant<Register, gc::CatchAllAllocSite> {
+ using Base = mozilla::Variant<Register, gc::CatchAllAllocSite>;
+ AllocSiteInput() : Base(gc::CatchAllAllocSite::Unknown) {}
+ explicit AllocSiteInput(gc::CatchAllAllocSite catchAll) : Base(catchAll) {}
+ explicit AllocSiteInput(Register reg) : Base(reg) {}
+};
+
+// [SMDOC] Code generation invariants (incomplete)
+//
+// ## 64-bit GPRs carrying 32-bit values
+//
+// At least at the end of every JS or Wasm operation (= SpiderMonkey bytecode or
+// Wasm bytecode; this is necessarily a little vague), if a 64-bit GPR has a
+// 32-bit value, then the upper 32 bits of the register may be predictable in
+// accordance with platform-specific rules, as follows.
+//
+// - On x64 and arm64, the upper bits are zero
+// - On mips64 and loongarch64 the upper bits are the sign extension of the
+// lower bits
+// - (On risc-v we have no rule, having no port yet. Sign extension is the most
+// likely rule, but "unpredictable" is an option.)
+//
+// In most cases no extra work needs to be done to maintain the invariant:
+//
+// - 32-bit operations on x64 and arm64 zero-extend the result to 64 bits.
+// These operations ignore the upper bits of the inputs.
+// - 32-bit operations on mips64 sign-extend the result to 64 bits (even many
+// that are labeled as "unsigned", eg ADDU, though not all, eg LU).
+// Additionally, the inputs to many 32-bit operations must be properly
+// sign-extended to avoid "unpredictable" behavior, and our simulators check
+// that inputs conform.
+// - (32-bit operations on risc-v and loongarch64 sign-extend, much as mips, but
+// appear to ignore the upper bits of the inputs.)
+//
+// The upshot of these invariants is, among other things, that:
+//
+// - No code needs to be generated when a 32-bit value is extended to 64 bits
+// or a 64-bit value is wrapped to 32 bits, if the upper bits are known to be
+// correct because they resulted from an operation that produced them
+// predictably.
+// - Literal loads must be careful to avoid instructions that might extend the
+// literal in the wrong way.
+// - Code that produces values using intermediate values with non-canonical
+// extensions must extend according to platform conventions before being
+// "done".
+//
+// All optimizations are necessarily platform-specific and should only be used
+// in platform-specific code. We may add architectures in the future that do
+// not follow the patterns of the few architectures we already have.
+//
+// Also see MacroAssembler::debugAssertCanonicalInt32().
+
+// The public entrypoint for emitting assembly. Note that a MacroAssembler can
+// use cx->lifoAlloc, so take care not to interleave masm use with other
+// lifoAlloc use if one will be destroyed before the other.
+class MacroAssembler : public MacroAssemblerSpecific {
+ private:
+ // Information about the current JSRuntime. This is nullptr only for Wasm
+ // compilations.
+ CompileRuntime* maybeRuntime_ = nullptr;
+
+ // Information about the current Realm. This is nullptr for Wasm compilations
+ // and when compiling JitRuntime trampolines.
+ CompileRealm* maybeRealm_ = nullptr;
+
+ // Labels for handling exceptions and failures.
+ NonAssertingLabel failureLabel_;
+
+ protected:
+ // Constructor is protected. Use one of the derived classes!
+ explicit MacroAssembler(TempAllocator& alloc,
+ CompileRuntime* maybeRuntime = nullptr,
+ CompileRealm* maybeRealm = nullptr);
+
+ public:
+ MoveResolver& moveResolver() {
+ // As an optimization, the MoveResolver is a persistent data structure
+ // shared between visitors in the CodeGenerator. This assertion
+ // checks that state is not leaking from visitor to visitor
+ // via an unresolved addMove().
+ MOZ_ASSERT(moveResolver_.hasNoPendingMoves());
+ return moveResolver_;
+ }
+
+ size_t instructionsSize() const { return size(); }
+
+ CompileRealm* realm() const {
+ MOZ_ASSERT(maybeRealm_);
+ return maybeRealm_;
+ }
+ CompileRuntime* runtime() const {
+ MOZ_ASSERT(maybeRuntime_);
+ return maybeRuntime_;
+ }
+
+#ifdef JS_HAS_HIDDEN_SP
+ void Push(RegisterOrSP reg);
+#endif
+
+#ifdef ENABLE_WASM_SIMD
+ // `op` should be a shift operation. Return true if a variable-width shift
+ // operation on this architecture should pre-mask the shift count, and if so,
+ // return the mask in `*mask`.
+ static bool MustMaskShiftCountSimd128(wasm::SimdOp op, int32_t* mask);
+#endif
+
+ private:
+ // The value returned by GetMaxOffsetGuardLimit() in WasmTypes.h
+ uint32_t wasmMaxOffsetGuardLimit_;
+
+ public:
+ uint32_t wasmMaxOffsetGuardLimit() const { return wasmMaxOffsetGuardLimit_; }
+ void setWasmMaxOffsetGuardLimit(uint32_t limit) {
+ wasmMaxOffsetGuardLimit_ = limit;
+ }
+
+ //{{{ check_macroassembler_decl_style
+ public:
+ // ===============================================================
+ // MacroAssembler high-level usage.
+
+ // Flushes the assembly buffer, on platforms that need it.
+ void flush() PER_SHARED_ARCH;
+
+ // Add a comment that is visible in the pretty printed assembly code.
+ void comment(const char* msg) PER_SHARED_ARCH;
+
+ // ===============================================================
+ // Frame manipulation functions.
+
+ inline uint32_t framePushed() const OOL_IN_HEADER;
+ inline void setFramePushed(uint32_t framePushed) OOL_IN_HEADER;
+ inline void adjustFrame(int32_t value) OOL_IN_HEADER;
+
+ // Adjust the frame, to account for implicit modification of the stack
+ // pointer, such that callee can remove arguments on the behalf of the
+ // caller.
+ inline void implicitPop(uint32_t bytes) OOL_IN_HEADER;
+
+ private:
+ // This field is used to statically (at compilation time) emulate a frame
+ // pointer by keeping track of stack manipulations.
+ //
+ // It is maintained by all stack manipulation functions below.
+ uint32_t framePushed_;
+
+ public:
+ // ===============================================================
+ // Stack manipulation functions -- sets of registers.
+
+ // Approximately speaking, the following routines must use the same memory
+ // layout. Any inconsistencies will certainly lead to crashing in generated
+ // code:
+ //
+ // MacroAssembler::PushRegsInMaskSizeInBytes
+ // MacroAssembler::PushRegsInMask
+ // MacroAssembler::storeRegsInMask
+ // MacroAssembler::PopRegsInMask
+ // MacroAssembler::PopRegsInMaskIgnore
+ // FloatRegister::getRegisterDumpOffsetInBytes
+ // (no class) PushRegisterDump
+ // (union) RegisterContent
+ // JitRuntime::generateInvalidator
+ // JitRuntime::generateBailoutHandler
+ // JSJitFrameIter::machineState
+ //
+ // To be more exact, the invariants are:
+ //
+ // * The save area is conceptually viewed as starting at a highest address
+ // (really, at "highest address - 1") and working down to some lower
+ // address.
+ //
+ // * PushRegsInMask, storeRegsInMask and PopRegsInMask{Ignore} must use
+ // exactly the same memory layout, when starting from the abovementioned
+ // highest address.
+ //
+ // * PushRegsInMaskSizeInBytes must produce a value which is exactly equal
+ // to the change in the machine's stack pointer register as a result of
+ // calling PushRegsInMask or PopRegsInMask{Ignore}. This value must be at
+ // least uintptr_t-aligned on the target, and may be more aligned than that.
+ //
+ // * PushRegsInMaskSizeInBytes must produce a value which is greater than or
+ // equal to the amount of space used by storeRegsInMask.
+ //
+ // * Hence, regardless of whether the save area is created with
+ // storeRegsInMask or PushRegsInMask, it is guaranteed to fit inside an
+ // area of size calculated by PushRegsInMaskSizeInBytes.
+ //
+ // * For the `ignore` argument of PopRegsInMaskIgnore, equality checking
+ // for the floating point/SIMD registers is done on the basis of the
+ // underlying physical register, regardless of width. For example, if the
+ // to-restore set contains v17 (the SIMD register with encoding 17) and
+ // the ignore set contains d17 (the double register with encoding 17) then
+ // no part of the physical register with encoding 17 will be restored.
+ // (This is probably not true on arm32, since that has aliased float32
+ // registers; but none of our other targets do.)
+ //
+ // * {Push,store}RegsInMask/storeRegsInMask are further constrained as
+ // follows: when given the argument AllFloatRegisters, the resulting
+ // memory area must contain exactly all the SIMD/FP registers for the
+ // target at their widest width (that we care about). [We have no targets
+ // where the SIMD registers and FP register sets are disjoint.] They must
+ // be packed end-to-end with no holes, with the register with the lowest
+ // encoding number (0), as returned by FloatRegister::encoding(), at the
+ // abovementioned highest address, register 1 just below that, etc.
+ //
+ // Furthermore the sizeof(RegisterContent) must equal the size of a SIMD
+ // register in the abovementioned array.
+ //
+ // Furthermore the value returned by
+ // FloatRegister::getRegisterDumpOffsetInBytes must be a correct index
+ // into the abovementioned array. Given the constraints, the only correct
+ // value is `reg.encoding() * sizeof(RegisterContent)`.
+ //
+ // Note that some of the routines listed above are JS-only, and do not support
+ // SIMD registers. They are otherwise part of the same equivalence class.
+ // Register spilling for e.g. OOL VM calls is implemented using
+ // PushRegsInMask, and recovered on bailout using machineState. This requires
+ // the same layout to be used in machineState, and therefore in all other code
+ // that can spill registers that are recovered on bailout. Implementations of
+ // JitRuntime::generate{Invalidator,BailoutHandler} should either call
+ // PushRegsInMask, or check carefully to be sure that they generate the same
+ // layout.
+
+ // The size of the area used by PushRegsInMask.
+ size_t PushRegsInMaskSizeInBytes(LiveRegisterSet set)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ void PushRegsInMask(LiveRegisterSet set)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+ void PushRegsInMask(LiveGeneralRegisterSet set);
+
+ // Like PushRegsInMask, but instead of pushing the registers, store them to
+ // |dest|. |dest| should point to the end of the reserved space, so the
+ // first register will be stored at |dest.offset - sizeof(register)|. It is
+ // required that |dest.offset| is at least as large as the value computed by
+ // PushRegsInMaskSizeInBytes for this |set|. In other words, |dest.base|
+ // must point to either the lowest address in the save area, or some address
+ // below that.
+ void storeRegsInMask(LiveRegisterSet set, Address dest, Register scratch)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ void PopRegsInMask(LiveRegisterSet set);
+ void PopRegsInMask(LiveGeneralRegisterSet set);
+ void PopRegsInMaskIgnore(LiveRegisterSet set, LiveRegisterSet ignore)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ // ===============================================================
+ // Stack manipulation functions -- single registers/values.
+
+ void Push(const Operand op) DEFINED_ON(x86_shared);
+ void Push(Register reg) PER_SHARED_ARCH;
+ void Push(Register reg1, Register reg2, Register reg3, Register reg4)
+ DEFINED_ON(arm64);
+ void Push(const Imm32 imm) PER_SHARED_ARCH;
+ void Push(const ImmWord imm) PER_SHARED_ARCH;
+ void Push(const ImmPtr imm) PER_SHARED_ARCH;
+ void Push(const ImmGCPtr ptr) PER_SHARED_ARCH;
+ void Push(FloatRegister reg) PER_SHARED_ARCH;
+ void PushBoxed(FloatRegister reg) PER_ARCH;
+ void PushFlags() DEFINED_ON(x86_shared);
+ void Push(PropertyKey key, Register scratchReg);
+ void Push(const Address& addr);
+ void Push(TypedOrValueRegister v);
+ void Push(const ConstantOrRegister& v);
+ void Push(const ValueOperand& val);
+ void Push(const Value& val);
+ void Push(JSValueType type, Register reg);
+ void Push(const Register64 reg);
+ void PushEmptyRooted(VMFunctionData::RootType rootType);
+ inline CodeOffset PushWithPatch(ImmWord word);
+ inline CodeOffset PushWithPatch(ImmPtr imm);
+
+ void Pop(const Operand op) DEFINED_ON(x86_shared);
+ void Pop(Register reg) PER_SHARED_ARCH;
+ void Pop(FloatRegister t) PER_SHARED_ARCH;
+ void Pop(const ValueOperand& val) PER_SHARED_ARCH;
+ void PopFlags() DEFINED_ON(x86_shared);
+ void PopStackPtr() DEFINED_ON(arm, mips_shared, x86_shared, loong64, wasm32);
+ void popRooted(VMFunctionData::RootType rootType, Register cellReg,
+ const ValueOperand& valueReg);
+
+ // Move the stack pointer based on the requested amount.
+ void adjustStack(int amount);
+ void freeStack(uint32_t amount);
+
+ // Warning: This method does not update the framePushed() counter.
+ void freeStack(Register amount);
+
+ private:
+ // ===============================================================
+ // Register allocation fields.
+#ifdef DEBUG
+ friend AutoRegisterScope;
+ friend AutoFloatRegisterScope;
+ // Used to track register scopes for debug builds.
+ // Manipulated by the AutoGenericRegisterScope class.
+ AllocatableRegisterSet debugTrackedRegisters_;
+#endif // DEBUG
+
+ public:
+ // ===============================================================
+ // Simple call functions.
+
+ // The returned CodeOffset is the assembler offset for the instruction
+ // immediately following the call; that is, for the return point.
+ CodeOffset call(Register reg) PER_SHARED_ARCH;
+ CodeOffset call(Label* label) PER_SHARED_ARCH;
+
+ void call(const Address& addr) PER_SHARED_ARCH;
+ void call(ImmWord imm) PER_SHARED_ARCH;
+ // Call a target native function, which is neither traceable nor movable.
+ void call(ImmPtr imm) PER_SHARED_ARCH;
+ CodeOffset call(wasm::SymbolicAddress imm) PER_SHARED_ARCH;
+ inline CodeOffset call(const wasm::CallSiteDesc& desc,
+ wasm::SymbolicAddress imm);
+
+ // Call a target JitCode, which must be traceable, and may be movable.
+ void call(JitCode* c) PER_SHARED_ARCH;
+
+ inline void call(TrampolinePtr code);
+
+ inline CodeOffset call(const wasm::CallSiteDesc& desc, const Register reg);
+ inline CodeOffset call(const wasm::CallSiteDesc& desc, uint32_t funcDefIndex);
+ inline void call(const wasm::CallSiteDesc& desc, wasm::Trap trap);
+
+ CodeOffset callWithPatch() PER_SHARED_ARCH;
+ void patchCall(uint32_t callerOffset, uint32_t calleeOffset) PER_SHARED_ARCH;
+
+ // Push the return address and make a call. On platforms where this function
+ // is not defined, push the link register (pushReturnAddress) at the entry
+ // point of the callee.
+ void callAndPushReturnAddress(Register reg) DEFINED_ON(x86_shared);
+ void callAndPushReturnAddress(Label* label) DEFINED_ON(x86_shared);
+
+ // These do not adjust framePushed().
+ void pushReturnAddress() DEFINED_ON(mips_shared, arm, arm64, loong64, wasm32);
+ void popReturnAddress() DEFINED_ON(mips_shared, arm, arm64, loong64, wasm32);
+
+ // Useful for dealing with two-valued returns.
+ void moveRegPair(Register src0, Register src1, Register dst0, Register dst1,
+ MoveOp::Type type = MoveOp::GENERAL);
+
+ public:
+ // ===============================================================
+ // Patchable near/far jumps.
+
+ // "Far jumps" provide the ability to jump to any uint32_t offset from any
+ // other uint32_t offset without using a constant pool (thus returning a
+ // simple CodeOffset instead of a CodeOffsetJump).
+ CodeOffset farJumpWithPatch() PER_SHARED_ARCH;
+ void patchFarJump(CodeOffset farJump, uint32_t targetOffset) PER_SHARED_ARCH;
+
+ // Emit a nop that can be patched to and from a nop and a call with int32
+ // relative displacement.
+ CodeOffset nopPatchableToCall() PER_SHARED_ARCH;
+ void nopPatchableToCall(const wasm::CallSiteDesc& desc);
+ static void patchNopToCall(uint8_t* callsite,
+ uint8_t* target) PER_SHARED_ARCH;
+ static void patchCallToNop(uint8_t* callsite) PER_SHARED_ARCH;
+
+ // These methods are like movWithPatch/PatchDataWithValueCheck but allow
+ // using pc-relative addressing on certain platforms (RIP-relative LEA on x64,
+ // ADR instruction on arm64).
+ //
+ // Note: "Near" applies to ARM64 where the target must be within 1 MB (this is
+ // release-asserted).
+ CodeOffset moveNearAddressWithPatch(Register dest)
+ DEFINED_ON(x86, x64, arm, arm64, loong64, wasm32, mips_shared);
+ static void patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target)
+ DEFINED_ON(x86, x64, arm, arm64, loong64, wasm32, mips_shared);
+
+ public:
+ // ===============================================================
+ // [SMDOC] JIT-to-C++ Function Calls (callWithABI)
+ //
+ // callWithABI is used to make a call using the standard C/C++ system ABI.
+ //
+ // callWithABI is a low level interface for making calls, as such every call
+ // made with callWithABI should be organized with 6 steps: spilling live
+ // registers, aligning the stack, listing arguments of the called function,
+ // calling a function pointer, extracting the returned value and restoring
+ // live registers.
+ //
+ // A more detailed example of the six stages:
+ //
+ // 1) Saving of registers that are live. This will vary depending on which
+ // SpiderMonkey compiler you are working on. Registers that shouldn't be
+ // restored can be excluded.
+ //
+ // LiveRegisterSet volatileRegs(...);
+ // volatileRegs.take(scratch);
+ // masm.PushRegsInMask(volatileRegs);
+ //
+ // 2) Align the stack to perform the call with the correct stack alignment.
+ //
+ // When the stack pointer alignment is unknown and cannot be corrected
+ // when generating the code, setupUnalignedABICall must be used to
+ // dynamically align the stack pointer to the expectation of the ABI.
+ // When the stack pointer is known at JIT compilation time, the stack can
+ // be fixed manually and setupAlignedABICall and setupWasmABICall can be
+ // used.
+ //
+ // setupWasmABICall is a special case of setupAlignedABICall as
+ // SpiderMonkey's WebAssembly implementation mostly follow the system
+ // ABI, except for float/double arguments, which always use floating
+ // point registers, even if this is not supported by the system ABI.
+ //
+ // masm.setupUnalignedABICall(scratch);
+ //
+ // 3) Passing arguments. Arguments are passed left-to-right.
+ //
+ // masm.passABIArg(scratch);
+ // masm.passABIArg(FloatOp0, MoveOp::Double);
+ //
+ // Note how float register arguments are annotated with MoveOp::Double.
+ //
+ // Concerning stack-relative address, see the note on passABIArg.
+ //
+ // 4) Make the call:
+ //
+ // using Fn = int32_t (*)(int32_t)
+ // masm.callWithABI<Fn, Callee>();
+ //
+ // In the case where the call returns a double, that needs to be
+ // indicated to the callWithABI like this:
+ //
+ // using Fn = double (*)(int32_t)
+ // masm.callWithABI<Fn, Callee>(MoveOp::DOUBLE);
+ //
+ // There are overloads to allow calls to registers and addresses.
+ //
+ // 5) Take care of the result
+ //
+ // masm.storeCallPointerResult(scratch1);
+ // masm.storeCallBoolResult(scratch1);
+ // masm.storeCallInt32Result(scratch1);
+ // masm.storeCallFloatResult(scratch1);
+ //
+ // 6) Restore the potentially clobbered volatile registers
+ //
+ // masm.PopRegsInMask(volatileRegs);
+ //
+ // If expecting a returned value, this call should use
+ // PopRegsInMaskIgnore to filter out the registers which are containing
+ // the returned value.
+ //
+ // Unless an exit frame is pushed prior to the setupABICall, the callee
+ // should not GC. To ensure this is the case callWithABI is instrumented to
+ // make sure that in the default case callees are annotated with an
+ // AutoUnsafeCallWithABI on the stack.
+ //
+ // A callWithABI can opt out of checking, if for example it is known there
+ // is an exit frame, or the callee is known not to GC.
+ //
+ // If your callee needs to be able to GC, consider using a VMFunction, or
+ // create a fake exit frame, and instrument the TraceJitExitFrame
+ // accordingly.
+
+ // Setup a call to C/C++ code, given the assumption that the framePushed
+ // accurately defines the state of the stack, and that the top of the stack
+ // was properly aligned. Note that this only supports cdecl.
+ //
+ // As a rule of thumb, this can be used in CodeGenerator but not in CacheIR or
+ // Baseline code (because the stack is not aligned to ABIStackAlignment).
+ void setupAlignedABICall();
+
+ // As setupAlignedABICall, but for WebAssembly native ABI calls, which pass
+ // through a builtin thunk that uses the wasm ABI. All the wasm ABI calls
+ // can be native, since we always know the stack alignment a priori.
+ void setupWasmABICall();
+
+ // Setup an ABI call for when the alignment is not known. This may need a
+ // scratch register.
+ void setupUnalignedABICall(Register scratch) PER_ARCH;
+
+ // Arguments must be assigned to a C/C++ call in order. They are moved
+ // in parallel immediately before performing the call. This process may
+ // temporarily use more stack, in which case esp-relative addresses will be
+ // automatically adjusted. It is extremely important that esp-relative
+ // addresses are computed *after* setupABICall(). Furthermore, no
+ // operations should be emitted while setting arguments.
+ void passABIArg(const MoveOperand& from, MoveOp::Type type);
+ inline void passABIArg(Register reg);
+ inline void passABIArg(FloatRegister reg, MoveOp::Type type);
+
+ inline void callWithABI(
+ DynFn fun, MoveOp::Type result = MoveOp::GENERAL,
+ CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check);
+ template <typename Sig, Sig fun>
+ inline void callWithABI(
+ MoveOp::Type result = MoveOp::GENERAL,
+ CheckUnsafeCallWithABI check = CheckUnsafeCallWithABI::Check);
+ inline void callWithABI(Register fun, MoveOp::Type result = MoveOp::GENERAL);
+ inline void callWithABI(const Address& fun,
+ MoveOp::Type result = MoveOp::GENERAL);
+
+ CodeOffset callWithABI(wasm::BytecodeOffset offset, wasm::SymbolicAddress fun,
+ mozilla::Maybe<int32_t> instanceOffset,
+ MoveOp::Type result = MoveOp::GENERAL);
+ void callDebugWithABI(wasm::SymbolicAddress fun,
+ MoveOp::Type result = MoveOp::GENERAL);
+
+ private:
+ // Reinitialize the variables which have to be cleared before making a call
+ // with callWithABI.
+ template <class ABIArgGeneratorT>
+ void setupABICallHelper();
+
+ // Reinitialize the variables which have to be cleared before making a call
+ // with native abi.
+ void setupNativeABICall();
+
+ // Reserve the stack and resolve the arguments move.
+ void callWithABIPre(uint32_t* stackAdjust,
+ bool callFromWasm = false) PER_ARCH;
+
+ // Emits a call to a C/C++ function, resolving all argument moves.
+ void callWithABINoProfiler(void* fun, MoveOp::Type result,
+ CheckUnsafeCallWithABI check);
+ void callWithABINoProfiler(Register fun, MoveOp::Type result) PER_ARCH;
+ void callWithABINoProfiler(const Address& fun, MoveOp::Type result) PER_ARCH;
+
+ // Restore the stack to its state before the setup function call.
+ void callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool callFromWasm = false) PER_ARCH;
+
+ // Create the signature to be able to decode the arguments of a native
+ // function, when calling a function within the simulator.
+ inline void appendSignatureType(MoveOp::Type type);
+ inline ABIFunctionType signature() const;
+
+ // Private variables used to handle moves between registers given as
+ // arguments to passABIArg and the list of ABI registers expected for the
+ // signature of the function.
+ MoveResolver moveResolver_;
+
+ // Architecture specific implementation which specify how registers & stack
+ // offsets are used for calling a function.
+ ABIArgGenerator abiArgs_;
+
+#ifdef DEBUG
+ // Flag use to assert that we use ABI function in the right context.
+ bool inCall_;
+#endif
+
+ // If set by setupUnalignedABICall then callWithABI will pop the stack
+ // register which is on the stack.
+ bool dynamicAlignment_;
+
+#ifdef JS_SIMULATOR
+ // The signature is used to accumulate all types of arguments which are used
+ // by the caller. This is used by the simulators to decode the arguments
+ // properly, and cast the function pointer to the right type.
+ uint32_t signature_;
+#endif
+
+ public:
+ // ===============================================================
+ // Jit Frames.
+ //
+ // These functions are used to build the content of the Jit frames. See
+ // CommonFrameLayout class, and all its derivatives. The content should be
+ // pushed in the opposite order as the fields of the structures, such that
+ // the structures can be used to interpret the content of the stack.
+
+ // Call the Jit function, and push the return address (or let the callee
+ // push the return address).
+ //
+ // These functions return the offset of the return address, in order to use
+ // the return address to index the safepoints, which are used to list all
+ // live registers.
+ inline uint32_t callJitNoProfiler(Register callee);
+ inline uint32_t callJit(Register callee);
+ inline uint32_t callJit(JitCode* code);
+ inline uint32_t callJit(TrampolinePtr code);
+ inline uint32_t callJit(ImmPtr callee);
+
+ // The frame descriptor is the second field of all Jit frames, pushed before
+ // calling the Jit function. See CommonFrameLayout::descriptor_.
+ inline void pushFrameDescriptor(FrameType type);
+ inline void PushFrameDescriptor(FrameType type);
+
+ // For JitFrameLayout, the descriptor also stores the number of arguments
+ // passed by the caller. See MakeFrameDescriptorForJitCall.
+ inline void pushFrameDescriptorForJitCall(FrameType type, uint32_t argc);
+ inline void pushFrameDescriptorForJitCall(FrameType type, Register argc,
+ Register scratch);
+ inline void PushFrameDescriptorForJitCall(FrameType type, uint32_t argc);
+ inline void PushFrameDescriptorForJitCall(FrameType type, Register argc,
+ Register scratch);
+
+ // Load the number of actual arguments from the frame's JitFrameLayout.
+ inline void loadNumActualArgs(Register framePtr, Register dest);
+
+ // Push the callee token of a JSFunction which pointer is stored in the
+ // |callee| register. The callee token is packed with a |constructing| flag
+ // which correspond to the fact that the JS function is called with "new" or
+ // not.
+ inline void PushCalleeToken(Register callee, bool constructing);
+
+ // Unpack a callee token located at the |token| address, and return the
+ // JSFunction pointer in the |dest| register.
+ inline void loadFunctionFromCalleeToken(Address token, Register dest);
+
+ // This function emulates a call by pushing an exit frame on the stack,
+ // except that the fake-function is inlined within the body of the caller.
+ //
+ // This function assumes that the current frame is an IonJS frame.
+ //
+ // This function returns the offset of the /fake/ return address, in order to
+ // use the return address to index the safepoints, which are used to list all
+ // live registers.
+ //
+ // This function should be balanced with a call to adjustStack, to pop the
+ // exit frame and emulate the return statement of the inlined function.
+ inline uint32_t buildFakeExitFrame(Register scratch);
+
+ private:
+ // This function is used by buildFakeExitFrame to push a fake return address
+ // on the stack. This fake return address should never be used for resuming
+ // any execution, and can even be an invalid pointer into the instruction
+ // stream, as long as it does not alias any other.
+ uint32_t pushFakeReturnAddress(Register scratch) PER_SHARED_ARCH;
+
+ public:
+ // ===============================================================
+ // Exit frame footer.
+ //
+ // When calling outside the Jit we push an exit frame. To mark the stack
+ // correctly, we have to push additional information, called the Exit frame
+ // footer, which is used to identify how the stack is marked.
+ //
+ // See JitFrames.h, and TraceJitExitFrame in JitFrames.cpp.
+
+ // Push stub code and the VMFunctionData pointer.
+ inline void enterExitFrame(Register cxreg, Register scratch,
+ const VMFunctionData* f);
+
+ // Push an exit frame token to identify which fake exit frame this footer
+ // corresponds to.
+ inline void enterFakeExitFrame(Register cxreg, Register scratch,
+ ExitFrameType type);
+
+ // Push an exit frame token for a native call.
+ inline void enterFakeExitFrameForNative(Register cxreg, Register scratch,
+ bool isConstructing);
+
+ // Pop ExitFrame footer in addition to the extra frame.
+ inline void leaveExitFrame(size_t extraFrame = 0);
+
+ private:
+ // Save the top of the stack into JitActivation::packedExitFP of the
+ // current thread, which should be the location of the latest exit frame.
+ void linkExitFrame(Register cxreg, Register scratch);
+
+ public:
+ // ===============================================================
+ // Move instructions
+
+ inline void move64(Imm64 imm, Register64 dest) PER_ARCH;
+ inline void move64(Register64 src, Register64 dest) PER_ARCH;
+
+ inline void moveFloat32ToGPR(FloatRegister src,
+ Register dest) PER_SHARED_ARCH;
+ inline void moveGPRToFloat32(Register src,
+ FloatRegister dest) PER_SHARED_ARCH;
+
+ inline void moveDoubleToGPR64(FloatRegister src, Register64 dest) PER_ARCH;
+ inline void moveGPR64ToDouble(Register64 src, FloatRegister dest) PER_ARCH;
+
+ inline void move8SignExtend(Register src, Register dest) PER_SHARED_ARCH;
+ inline void move16SignExtend(Register src, Register dest) PER_SHARED_ARCH;
+
+ // move64To32 will clear the high bits of `dest` on 64-bit systems.
+ inline void move64To32(Register64 src, Register dest) PER_ARCH;
+
+ inline void move32To64ZeroExtend(Register src, Register64 dest) PER_ARCH;
+
+ inline void move8To64SignExtend(Register src, Register64 dest) PER_ARCH;
+ inline void move16To64SignExtend(Register src, Register64 dest) PER_ARCH;
+ inline void move32To64SignExtend(Register src, Register64 dest) PER_ARCH;
+
+ inline void move32SignExtendToPtr(Register src, Register dest) PER_ARCH;
+ inline void move32ZeroExtendToPtr(Register src, Register dest) PER_ARCH;
+
+ // Copy a constant, typed-register, or a ValueOperand into a ValueOperand
+ // destination.
+ inline void moveValue(const ConstantOrRegister& src,
+ const ValueOperand& dest);
+ void moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) PER_ARCH;
+ void moveValue(const ValueOperand& src, const ValueOperand& dest) PER_ARCH;
+ void moveValue(const Value& src, const ValueOperand& dest) PER_ARCH;
+
+ void movePropertyKey(PropertyKey key, Register dest);
+
+ // ===============================================================
+ // Load instructions
+
+ inline void load32SignExtendToPtr(const Address& src, Register dest) PER_ARCH;
+
+ inline void loadAbiReturnAddress(Register dest) PER_SHARED_ARCH;
+
+ public:
+ // ===============================================================
+ // Logical instructions
+
+ inline void not32(Register reg) PER_SHARED_ARCH;
+ inline void notPtr(Register reg) PER_ARCH;
+
+ inline void and32(Register src, Register dest) PER_SHARED_ARCH;
+ inline void and32(Imm32 imm, Register dest) PER_SHARED_ARCH;
+ inline void and32(Imm32 imm, Register src, Register dest) DEFINED_ON(arm64);
+ inline void and32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
+ inline void and32(const Address& src, Register dest) PER_SHARED_ARCH;
+
+ inline void andPtr(Register src, Register dest) PER_ARCH;
+ inline void andPtr(Imm32 imm, Register dest) PER_ARCH;
+
+ inline void and64(Imm64 imm, Register64 dest) PER_ARCH;
+ inline void or64(Imm64 imm, Register64 dest) PER_ARCH;
+ inline void xor64(Imm64 imm, Register64 dest) PER_ARCH;
+
+ inline void or32(Register src, Register dest) PER_SHARED_ARCH;
+ inline void or32(Imm32 imm, Register dest) PER_SHARED_ARCH;
+ inline void or32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
+
+ inline void orPtr(Register src, Register dest) PER_ARCH;
+ inline void orPtr(Imm32 imm, Register dest) PER_ARCH;
+
+ inline void and64(Register64 src, Register64 dest) PER_ARCH;
+ inline void or64(Register64 src, Register64 dest) PER_ARCH;
+ inline void xor64(Register64 src, Register64 dest) PER_ARCH;
+
+ inline void xor32(Register src, Register dest) PER_SHARED_ARCH;
+ inline void xor32(Imm32 imm, Register dest) PER_SHARED_ARCH;
+ inline void xor32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
+ inline void xor32(const Address& src, Register dest) PER_SHARED_ARCH;
+
+ inline void xorPtr(Register src, Register dest) PER_ARCH;
+ inline void xorPtr(Imm32 imm, Register dest) PER_ARCH;
+
+ inline void and64(const Operand& src, Register64 dest)
+ DEFINED_ON(x64, mips64, loong64);
+ inline void or64(const Operand& src, Register64 dest)
+ DEFINED_ON(x64, mips64, loong64);
+ inline void xor64(const Operand& src, Register64 dest)
+ DEFINED_ON(x64, mips64, loong64);
+
+ // ===============================================================
+ // Swap instructions
+
+ // Swap the two lower bytes and sign extend the result to 32-bit.
+ inline void byteSwap16SignExtend(Register reg) PER_SHARED_ARCH;
+
+ // Swap the two lower bytes and zero extend the result to 32-bit.
+ inline void byteSwap16ZeroExtend(Register reg) PER_SHARED_ARCH;
+
+ // Swap all four bytes in a 32-bit integer.
+ inline void byteSwap32(Register reg) PER_SHARED_ARCH;
+
+ // Swap all eight bytes in a 64-bit integer.
+ inline void byteSwap64(Register64 reg) PER_ARCH;
+
+ // ===============================================================
+ // Arithmetic functions
+
+ inline void add32(Register src, Register dest) PER_SHARED_ARCH;
+ inline void add32(Imm32 imm, Register dest) PER_SHARED_ARCH;
+ inline void add32(Imm32 imm, const Address& dest) PER_SHARED_ARCH;
+ inline void add32(Imm32 imm, const AbsoluteAddress& dest)
+ DEFINED_ON(x86_shared);
+
+ inline void addPtr(Register src, Register dest) PER_ARCH;
+ inline void addPtr(Register src1, Register src2, Register dest)
+ DEFINED_ON(arm64);
+ inline void addPtr(Imm32 imm, Register dest) PER_ARCH;
+ inline void addPtr(Imm32 imm, Register src, Register dest) DEFINED_ON(arm64);
+ inline void addPtr(ImmWord imm, Register dest) PER_ARCH;
+ inline void addPtr(ImmPtr imm, Register dest);
+ inline void addPtr(Imm32 imm, const Address& dest)
+ DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64, wasm32);
+ inline void addPtr(Imm32 imm, const AbsoluteAddress& dest)
+ DEFINED_ON(x86, x64);
+ inline void addPtr(const Address& src, Register dest)
+ DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64, wasm32);
+
+ inline void add64(Register64 src, Register64 dest) PER_ARCH;
+ inline void add64(Imm32 imm, Register64 dest) PER_ARCH;
+ inline void add64(Imm64 imm, Register64 dest) PER_ARCH;
+ inline void add64(const Operand& src, Register64 dest)
+ DEFINED_ON(x64, mips64, loong64);
+
+ inline void addFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+
+ // Compute dest=SP-imm where dest is a pointer registers and not SP. The
+ // offset returned from sub32FromStackPtrWithPatch() must be passed to
+ // patchSub32FromStackPtr().
+ inline CodeOffset sub32FromStackPtrWithPatch(Register dest) PER_ARCH;
+ inline void patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) PER_ARCH;
+
+ inline void addDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+ inline void addConstantDouble(double d, FloatRegister dest) DEFINED_ON(x86);
+
+ inline void sub32(const Address& src, Register dest) PER_SHARED_ARCH;
+ inline void sub32(Register src, Register dest) PER_SHARED_ARCH;
+ inline void sub32(Imm32 imm, Register dest) PER_SHARED_ARCH;
+
+ inline void subPtr(Register src, Register dest) PER_ARCH;
+ inline void subPtr(Register src, const Address& dest)
+ DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64, wasm32);
+ inline void subPtr(Imm32 imm, Register dest) PER_ARCH;
+ inline void subPtr(ImmWord imm, Register dest) DEFINED_ON(x64);
+ inline void subPtr(const Address& addr, Register dest)
+ DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64, wasm32);
+
+ inline void sub64(Register64 src, Register64 dest) PER_ARCH;
+ inline void sub64(Imm64 imm, Register64 dest) PER_ARCH;
+ inline void sub64(const Operand& src, Register64 dest)
+ DEFINED_ON(x64, mips64, loong64);
+
+ inline void subFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+
+ inline void subDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+
+ inline void mul32(Register rhs, Register srcDest) PER_SHARED_ARCH;
+ inline void mul32(Imm32 imm, Register srcDest) PER_SHARED_ARCH;
+
+ inline void mul32(Register src1, Register src2, Register dest, Label* onOver)
+ DEFINED_ON(arm64);
+
+ // Return the high word of the unsigned multiplication into |dest|.
+ inline void mulHighUnsigned32(Imm32 imm, Register src,
+ Register dest) PER_ARCH;
+
+ inline void mulPtr(Register rhs, Register srcDest) PER_ARCH;
+
+ inline void mul64(const Operand& src, const Register64& dest) DEFINED_ON(x64);
+ inline void mul64(const Operand& src, const Register64& dest,
+ const Register temp) DEFINED_ON(x64, mips64, loong64);
+ inline void mul64(Imm64 imm, const Register64& dest) PER_ARCH;
+ inline void mul64(Imm64 imm, const Register64& dest, const Register temp)
+ DEFINED_ON(x86, x64, arm, mips32, mips64, loong64);
+ inline void mul64(const Register64& src, const Register64& dest,
+ const Register temp) PER_ARCH;
+ inline void mul64(const Register64& src1, const Register64& src2,
+ const Register64& dest) DEFINED_ON(arm64);
+ inline void mul64(Imm64 src1, const Register64& src2, const Register64& dest)
+ DEFINED_ON(arm64);
+
+ inline void mulBy3(Register src, Register dest) PER_ARCH;
+
+ inline void mulFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+ inline void mulDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+
+ inline void mulDoublePtr(ImmPtr imm, Register temp, FloatRegister dest)
+ DEFINED_ON(mips_shared, arm, arm64, x86, x64, loong64, wasm32);
+
+ // Perform an integer division, returning the integer part rounded toward
+ // zero. rhs must not be zero, and the division must not overflow.
+ //
+ // On ARM, the chip must have hardware division instructions.
+ inline void quotient32(Register rhs, Register srcDest, bool isUnsigned)
+ DEFINED_ON(mips_shared, arm, arm64, loong64, wasm32);
+
+ // As above, but srcDest must be eax and tempEdx must be edx.
+ inline void quotient32(Register rhs, Register srcDest, Register tempEdx,
+ bool isUnsigned) DEFINED_ON(x86_shared);
+
+ // Perform an integer division, returning the remainder part.
+ // rhs must not be zero, and the division must not overflow.
+ //
+ // On ARM, the chip must have hardware division instructions.
+ inline void remainder32(Register rhs, Register srcDest, bool isUnsigned)
+ DEFINED_ON(mips_shared, arm, arm64, loong64, wasm32);
+
+ // As above, but srcDest must be eax and tempEdx must be edx.
+ inline void remainder32(Register rhs, Register srcDest, Register tempEdx,
+ bool isUnsigned) DEFINED_ON(x86_shared);
+
+ // Perform an integer division, returning the integer part rounded toward
+ // zero. rhs must not be zero, and the division must not overflow.
+ //
+ // This variant preserves registers, and doesn't require hardware division
+ // instructions on ARM (will call out to a runtime routine).
+ //
+ // rhs is preserved, srdDest is clobbered.
+ void flexibleRemainder32(Register rhs, Register srcDest, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs)
+ DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64, wasm32);
+
+ // Perform an integer division, returning the integer part rounded toward
+ // zero. rhs must not be zero, and the division must not overflow.
+ //
+ // This variant preserves registers, and doesn't require hardware division
+ // instructions on ARM (will call out to a runtime routine).
+ //
+ // rhs is preserved, srdDest is clobbered.
+ void flexibleQuotient32(Register rhs, Register srcDest, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs)
+ DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64);
+
+ // Perform an integer division, returning the integer part rounded toward
+ // zero. rhs must not be zero, and the division must not overflow. The
+ // remainder is stored into the third argument register here.
+ //
+ // This variant preserves registers, and doesn't require hardware division
+ // instructions on ARM (will call out to a runtime routine).
+ //
+ // rhs is preserved, srdDest and remOutput are clobbered.
+ void flexibleDivMod32(Register rhs, Register srcDest, Register remOutput,
+ bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs)
+ DEFINED_ON(mips_shared, arm, arm64, x86_shared, loong64, wasm32);
+
+ inline void divFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+ inline void divDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+
+ inline void inc64(AbsoluteAddress dest) PER_ARCH;
+
+ inline void neg32(Register reg) PER_SHARED_ARCH;
+ inline void neg64(Register64 reg) PER_ARCH;
+ inline void negPtr(Register reg) PER_ARCH;
+
+ inline void negateFloat(FloatRegister reg) PER_SHARED_ARCH;
+
+ inline void negateDouble(FloatRegister reg) PER_SHARED_ARCH;
+
+ inline void abs32(Register src, Register dest) PER_SHARED_ARCH;
+ inline void absFloat32(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+ inline void absDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+
+ inline void sqrtFloat32(FloatRegister src,
+ FloatRegister dest) PER_SHARED_ARCH;
+ inline void sqrtDouble(FloatRegister src, FloatRegister dest) PER_SHARED_ARCH;
+
+ void floorFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) PER_SHARED_ARCH;
+ void floorDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) PER_SHARED_ARCH;
+
+ void ceilFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) PER_SHARED_ARCH;
+ void ceilDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) PER_SHARED_ARCH;
+
+ void roundFloat32ToInt32(FloatRegister src, Register dest, FloatRegister temp,
+ Label* fail) PER_SHARED_ARCH;
+ void roundDoubleToInt32(FloatRegister src, Register dest, FloatRegister temp,
+ Label* fail) PER_SHARED_ARCH;
+
+ void truncFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) PER_SHARED_ARCH;
+ void truncDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) PER_SHARED_ARCH;
+
+ void nearbyIntDouble(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) PER_SHARED_ARCH;
+ void nearbyIntFloat32(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) PER_SHARED_ARCH;
+
+ void signInt32(Register input, Register output);
+ void signDouble(FloatRegister input, FloatRegister output);
+ void signDoubleToInt32(FloatRegister input, Register output,
+ FloatRegister temp, Label* fail);
+
+ void copySignDouble(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) PER_SHARED_ARCH;
+ void copySignFloat32(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) DEFINED_ON(x86_shared, arm64);
+
+ // Returns a random double in range [0, 1) in |dest|. The |rng| register must
+ // hold a pointer to a mozilla::non_crypto::XorShift128PlusRNG.
+ void randomDouble(Register rng, FloatRegister dest, Register64 temp0,
+ Register64 temp1);
+
+ // srcDest = {min,max}{Float32,Double}(srcDest, other)
+ // For min and max, handle NaN specially if handleNaN is true.
+
+ inline void minFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) PER_SHARED_ARCH;
+ inline void minDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) PER_SHARED_ARCH;
+
+ inline void maxFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) PER_SHARED_ARCH;
+ inline void maxDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) PER_SHARED_ARCH;
+
+ void minMaxArrayInt32(Register array, Register result, Register temp1,
+ Register temp2, Register temp3, bool isMax,
+ Label* fail);
+ void minMaxArrayNumber(Register array, FloatRegister result,
+ FloatRegister floatTemp, Register temp1,
+ Register temp2, bool isMax, Label* fail);
+
+ // Compute |pow(base, power)| and store the result in |dest|. If the result
+ // exceeds the int32 range, jumps to |onOver|.
+ // |base| and |power| are preserved, the other input registers are clobbered.
+ void pow32(Register base, Register power, Register dest, Register temp1,
+ Register temp2, Label* onOver);
+
+ void sameValueDouble(FloatRegister left, FloatRegister right,
+ FloatRegister temp, Register dest);
+
+ void branchIfNotRegExpPrototypeOptimizable(Register proto, Register temp,
+ Label* label);
+ void branchIfNotRegExpInstanceOptimizable(Register regexp, Register temp,
+ Label* label);
+
+ // ===============================================================
+ // Shift functions
+
+ // For shift-by-register there may be platform-specific variations, for
+ // example, x86 will perform the shift mod 32 but ARM will perform the shift
+ // mod 256.
+ //
+ // For shift-by-immediate the platform assembler may restrict the immediate,
+ // for example, the ARM assembler requires the count for 32-bit shifts to be
+ // in the range [0,31].
+
+ inline void lshift32(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
+ inline void rshift32(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
+ inline void rshift32Arithmetic(Imm32 shift, Register srcDest) PER_SHARED_ARCH;
+
+ inline void lshiftPtr(Imm32 imm, Register dest) PER_ARCH;
+ inline void rshiftPtr(Imm32 imm, Register dest) PER_ARCH;
+ inline void rshiftPtr(Imm32 imm, Register src, Register dest)
+ DEFINED_ON(arm64);
+ inline void rshiftPtrArithmetic(Imm32 imm, Register dest) PER_ARCH;
+
+ inline void lshift64(Imm32 imm, Register64 dest) PER_ARCH;
+ inline void rshift64(Imm32 imm, Register64 dest) PER_ARCH;
+ inline void rshift64Arithmetic(Imm32 imm, Register64 dest) PER_ARCH;
+
+ // On x86_shared these have the constraint that shift must be in CL.
+ inline void lshift32(Register shift, Register srcDest) PER_SHARED_ARCH;
+ inline void rshift32(Register shift, Register srcDest) PER_SHARED_ARCH;
+ inline void rshift32Arithmetic(Register shift,
+ Register srcDest) PER_SHARED_ARCH;
+ inline void lshiftPtr(Register shift, Register srcDest) PER_ARCH;
+ inline void rshiftPtr(Register shift, Register srcDest) PER_ARCH;
+
+ // These variants do not have the above constraint, but may emit some extra
+ // instructions on x86_shared. They also handle shift >= 32 consistently by
+ // masking with 0x1F (either explicitly or relying on the hardware to do
+ // that).
+ inline void flexibleLshift32(Register shift,
+ Register srcDest) PER_SHARED_ARCH;
+ inline void flexibleRshift32(Register shift,
+ Register srcDest) PER_SHARED_ARCH;
+ inline void flexibleRshift32Arithmetic(Register shift,
+ Register srcDest) PER_SHARED_ARCH;
+
+ inline void lshift64(Register shift, Register64 srcDest) PER_ARCH;
+ inline void rshift64(Register shift, Register64 srcDest) PER_ARCH;
+ inline void rshift64Arithmetic(Register shift, Register64 srcDest) PER_ARCH;
+
+ // ===============================================================
+ // Rotation functions
+ // Note: - on x86 and x64 the count register must be in CL.
+ // - on x64 the temp register should be InvalidReg.
+
+ inline void rotateLeft(Imm32 count, Register input,
+ Register dest) PER_SHARED_ARCH;
+ inline void rotateLeft(Register count, Register input,
+ Register dest) PER_SHARED_ARCH;
+ inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest)
+ DEFINED_ON(x64);
+ inline void rotateLeft64(Register count, Register64 input, Register64 dest)
+ DEFINED_ON(x64);
+ inline void rotateLeft64(Imm32 count, Register64 input, Register64 dest,
+ Register temp) PER_ARCH;
+ inline void rotateLeft64(Register count, Register64 input, Register64 dest,
+ Register temp) PER_ARCH;
+
+ inline void rotateRight(Imm32 count, Register input,
+ Register dest) PER_SHARED_ARCH;
+ inline void rotateRight(Register count, Register input,
+ Register dest) PER_SHARED_ARCH;
+ inline void rotateRight64(Imm32 count, Register64 input, Register64 dest)
+ DEFINED_ON(x64);
+ inline void rotateRight64(Register count, Register64 input, Register64 dest)
+ DEFINED_ON(x64);
+ inline void rotateRight64(Imm32 count, Register64 input, Register64 dest,
+ Register temp) PER_ARCH;
+ inline void rotateRight64(Register count, Register64 input, Register64 dest,
+ Register temp) PER_ARCH;
+
+ // ===============================================================
+ // Bit counting functions
+
+ // knownNotZero may be true only if the src is known not to be zero.
+ inline void clz32(Register src, Register dest,
+ bool knownNotZero) PER_SHARED_ARCH;
+ inline void ctz32(Register src, Register dest,
+ bool knownNotZero) PER_SHARED_ARCH;
+
+ inline void clz64(Register64 src, Register dest) PER_ARCH;
+ inline void ctz64(Register64 src, Register dest) PER_ARCH;
+
+ // On x86_shared, temp may be Invalid only if the chip has the POPCNT
+ // instruction. On ARM, temp may never be Invalid.
+ inline void popcnt32(Register src, Register dest,
+ Register temp) PER_SHARED_ARCH;
+
+ // temp may be invalid only if the chip has the POPCNT instruction.
+ inline void popcnt64(Register64 src, Register64 dest, Register temp) PER_ARCH;
+
+ // ===============================================================
+ // Condition functions
+
+ inline void cmp8Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) PER_SHARED_ARCH;
+
+ inline void cmp16Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) PER_SHARED_ARCH;
+
+ template <typename T1, typename T2>
+ inline void cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest)
+ DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64, wasm32);
+
+ // Only the NotEqual and Equal conditions are allowed.
+ inline void cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) PER_ARCH;
+
+ template <typename T1, typename T2>
+ inline void cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) PER_ARCH;
+
+ // ===============================================================
+ // Branch functions
+
+ inline void branch8(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ // Compares the byte in |lhs| against |rhs| using a 8-bit comparison on
+ // x86/x64 or a 32-bit comparison (all other platforms). The caller should
+ // ensure |rhs| is a zero- resp. sign-extended byte value for cross-platform
+ // compatible code.
+ inline void branch8(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branch16(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ template <class L>
+ inline void branch32(Condition cond, Register lhs, Register rhs,
+ L label) PER_SHARED_ARCH;
+ template <class L>
+ inline void branch32(Condition cond, Register lhs, Imm32 rhs,
+ L label) PER_SHARED_ARCH;
+
+ inline void branch32(Condition cond, Register lhs, const Address& rhs,
+ Label* label) DEFINED_ON(arm64);
+
+ inline void branch32(Condition cond, const Address& lhs, Register rhs,
+ Label* label) PER_SHARED_ARCH;
+ inline void branch32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branch32(Condition cond, const AbsoluteAddress& lhs, Register rhs,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+ inline void branch32(Condition cond, const AbsoluteAddress& lhs, Imm32 rhs,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ inline void branch32(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) DEFINED_ON(arm, x86_shared);
+ inline void branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branch32(Condition cond, const Operand& lhs, Register rhs,
+ Label* label) DEFINED_ON(x86_shared);
+ inline void branch32(Condition cond, const Operand& lhs, Imm32 rhs,
+ Label* label) DEFINED_ON(x86_shared);
+
+ inline void branch32(Condition cond, wasm::SymbolicAddress lhs, Imm32 rhs,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ // The supported condition are Equal, NotEqual, LessThan(orEqual),
+ // GreaterThan(orEqual), Below(orEqual) and Above(orEqual). When a fail label
+ // is not defined it will fall through to next instruction, else jump to the
+ // fail label.
+ inline void branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail = nullptr) PER_ARCH;
+ inline void branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail = nullptr) PER_ARCH;
+ // Only the NotEqual and Equal conditions are allowed for the branch64
+ // variants with Address as lhs.
+ inline void branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) PER_ARCH;
+ inline void branch64(Condition cond, const Address& lhs, Register64 rhs,
+ Label* label) PER_ARCH;
+
+ // Compare the value at |lhs| with the value at |rhs|. The scratch
+ // register *must not* be the base of |lhs| or |rhs|.
+ inline void branch64(Condition cond, const Address& lhs, const Address& rhs,
+ Register scratch, Label* label) PER_ARCH;
+
+ template <class L>
+ inline void branchPtr(Condition cond, Register lhs, Register rhs,
+ L label) PER_SHARED_ARCH;
+ inline void branchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchPtr(Condition cond, Register lhs, ImmPtr rhs,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchPtr(Condition cond, Register lhs, ImmWord rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ template <class L>
+ inline void branchPtr(Condition cond, const Address& lhs, Register rhs,
+ L label) PER_SHARED_ARCH;
+ inline void branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branchPtr(Condition cond, const BaseIndex& lhs, ImmWord rhs,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchPtr(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+ inline void branchPtr(Condition cond, const AbsoluteAddress& lhs, ImmWord rhs,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ inline void branchPtr(Condition cond, wasm::SymbolicAddress lhs, Register rhs,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ // Given a pointer to a GC Cell, retrieve the StoreBuffer pointer from its
+ // chunk header, or nullptr if it is in the tenured heap.
+ void loadStoreBuffer(Register ptr, Register buffer) PER_ARCH;
+
+ void branchPtrInNurseryChunk(Condition cond, Register ptr, Register temp,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+ void branchPtrInNurseryChunk(Condition cond, const Address& address,
+ Register temp, Label* label) DEFINED_ON(x86);
+ void branchValueIsNurseryCell(Condition cond, const Address& address,
+ Register temp, Label* label) PER_ARCH;
+ void branchValueIsNurseryCell(Condition cond, ValueOperand value,
+ Register temp, Label* label) PER_ARCH;
+
+ // This function compares a Value (lhs) which is having a private pointer
+ // boxed inside a js::Value, with a raw pointer (rhs).
+ inline void branchPrivatePtr(Condition cond, const Address& lhs, Register rhs,
+ Label* label) PER_ARCH;
+
+ inline void branchFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) PER_SHARED_ARCH;
+
+ // Truncate a double/float32 to int32 and when it doesn't fit an int32 it will
+ // jump to the failure label. This particular variant is allowed to return the
+ // value module 2**32, which isn't implemented on all architectures. E.g. the
+ // x64 variants will do this only in the int64_t range.
+ inline void branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest, Label* fail)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+ inline void branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest, Label* fail)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ // Truncate a double/float32 to intptr and when it doesn't fit jump to the
+ // failure label.
+ inline void branchTruncateFloat32ToPtr(FloatRegister src, Register dest,
+ Label* fail) DEFINED_ON(x86, x64);
+ inline void branchTruncateDoubleToPtr(FloatRegister src, Register dest,
+ Label* fail) DEFINED_ON(x86, x64);
+
+ // Truncate a double/float32 to int32 and when it doesn't fit jump to the
+ // failure label.
+ inline void branchTruncateFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+ inline void branchTruncateDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ inline void branchDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) PER_SHARED_ARCH;
+
+ inline void branchDoubleNotInInt64Range(Address src, Register temp,
+ Label* fail);
+ inline void branchDoubleNotInUInt64Range(Address src, Register temp,
+ Label* fail);
+ inline void branchFloat32NotInInt64Range(Address src, Register temp,
+ Label* fail);
+ inline void branchFloat32NotInUInt64Range(Address src, Register temp,
+ Label* fail);
+
+ template <typename T>
+ inline void branchAdd32(Condition cond, T src, Register dest,
+ Label* label) PER_SHARED_ARCH;
+ template <typename T>
+ inline void branchSub32(Condition cond, T src, Register dest,
+ Label* label) PER_SHARED_ARCH;
+ template <typename T>
+ inline void branchMul32(Condition cond, T src, Register dest,
+ Label* label) PER_SHARED_ARCH;
+ template <typename T>
+ inline void branchRshift32(Condition cond, T src, Register dest,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branchNeg32(Condition cond, Register reg,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branchAdd64(Condition cond, Imm64 imm, Register64 dest,
+ Label* label) DEFINED_ON(x86, arm, wasm32);
+
+ template <typename T>
+ inline void branchAddPtr(Condition cond, T src, Register dest,
+ Label* label) PER_SHARED_ARCH;
+
+ template <typename T>
+ inline void branchSubPtr(Condition cond, T src, Register dest,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branchMulPtr(Condition cond, Register src, Register dest,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ template <class L>
+ inline void branchTest32(Condition cond, Register lhs, Register rhs,
+ L label) PER_SHARED_ARCH;
+ template <class L>
+ inline void branchTest32(Condition cond, Register lhs, Imm32 rhs,
+ L label) PER_SHARED_ARCH;
+ inline void branchTest32(Condition cond, const Address& lhs, Imm32 rhh,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ template <class L>
+ inline void branchTestPtr(Condition cond, Register lhs, Register rhs,
+ L label) PER_SHARED_ARCH;
+ inline void branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) PER_SHARED_ARCH;
+
+ template <class L>
+ inline void branchTest64(Condition cond, Register64 lhs, Register64 rhs,
+ Register temp, L label) PER_ARCH;
+
+ // Branches to |label| if |reg| is false. |reg| should be a C++ bool.
+ template <class L>
+ inline void branchIfFalseBool(Register reg, L label);
+
+ // Branches to |label| if |reg| is true. |reg| should be a C++ bool.
+ inline void branchIfTrueBool(Register reg, Label* label);
+
+ inline void branchIfRope(Register str, Label* label);
+ inline void branchIfNotRope(Register str, Label* label);
+
+ inline void branchLatin1String(Register string, Label* label);
+ inline void branchTwoByteString(Register string, Label* label);
+
+ inline void branchIfBigIntIsNegative(Register bigInt, Label* label);
+ inline void branchIfBigIntIsNonNegative(Register bigInt, Label* label);
+ inline void branchIfBigIntIsZero(Register bigInt, Label* label);
+ inline void branchIfBigIntIsNonZero(Register bigInt, Label* label);
+
+ inline void branchTestFunctionFlags(Register fun, uint32_t flags,
+ Condition cond, Label* label);
+
+ inline void branchIfNotFunctionIsNonBuiltinCtor(Register fun,
+ Register scratch,
+ Label* label);
+
+ inline void branchIfFunctionHasNoJitEntry(Register fun, bool isConstructing,
+ Label* label);
+ inline void branchIfFunctionHasJitEntry(Register fun, bool isConstructing,
+ Label* label);
+
+ inline void branchIfScriptHasJitScript(Register script, Label* label);
+ inline void branchIfScriptHasNoJitScript(Register script, Label* label);
+ inline void loadJitScript(Register script, Register dest);
+
+ // Loads the function's argument count.
+ inline void loadFunctionArgCount(Register func, Register output);
+
+ // Loads the function length. This handles interpreted, native, and bound
+ // functions. The caller is responsible for checking that INTERPRETED_LAZY and
+ // RESOLVED_LENGTH flags are not set.
+ void loadFunctionLength(Register func, Register funFlagsAndArgCount,
+ Register output, Label* slowPath);
+
+ // Loads the function name. This handles interpreted, native, and bound
+ // functions.
+ void loadFunctionName(Register func, Register output, ImmGCPtr emptyString,
+ Label* slowPath);
+
+ void assertFunctionIsExtended(Register func);
+
+ inline void branchFunctionKind(Condition cond,
+ FunctionFlags::FunctionKind kind, Register fun,
+ Register scratch, Label* label);
+
+ inline void branchIfObjectEmulatesUndefined(Register objReg, Register scratch,
+ Label* slowCheck, Label* label);
+
+ // For all methods below: spectreRegToZero is a register that will be zeroed
+ // on speculatively executed code paths (when the branch should be taken but
+ // branch prediction speculates it isn't). Usually this will be the object
+ // register but the caller may pass a different register.
+
+ inline void branchTestObjClass(Condition cond, Register obj,
+ const JSClass* clasp, Register scratch,
+ Register spectreRegToZero, Label* label);
+ inline void branchTestObjClassNoSpectreMitigations(Condition cond,
+ Register obj,
+ const JSClass* clasp,
+ Register scratch,
+ Label* label);
+
+ inline void branchTestObjClass(Condition cond, Register obj,
+ const Address& clasp, Register scratch,
+ Register spectreRegToZero, Label* label);
+ inline void branchTestObjClassNoSpectreMitigations(Condition cond,
+ Register obj,
+ const Address& clasp,
+ Register scratch,
+ Label* label);
+
+ inline void branchTestObjClass(Condition cond, Register obj, Register clasp,
+ Register scratch, Register spectreRegToZero,
+ Label* label);
+
+ inline void branchTestObjShape(Condition cond, Register obj,
+ const Shape* shape, Register scratch,
+ Register spectreRegToZero, Label* label);
+ inline void branchTestObjShapeNoSpectreMitigations(Condition cond,
+ Register obj,
+ const Shape* shape,
+ Label* label);
+
+ void branchTestObjShapeList(Condition cond, Register obj,
+ Register shapeElements, Register shapeScratch,
+ Register endScratch, Register spectreScratch,
+ Label* label);
+
+ inline void branchTestClassIsFunction(Condition cond, Register clasp,
+ Label* label);
+ inline void branchTestObjIsFunction(Condition cond, Register obj,
+ Register scratch,
+ Register spectreRegToZero, Label* label);
+ inline void branchTestObjIsFunctionNoSpectreMitigations(Condition cond,
+ Register obj,
+ Register scratch,
+ Label* label);
+
+ inline void branchTestObjShape(Condition cond, Register obj, Register shape,
+ Register scratch, Register spectreRegToZero,
+ Label* label);
+ inline void branchTestObjShapeNoSpectreMitigations(Condition cond,
+ Register obj,
+ Register shape,
+ Label* label);
+
+ // TODO: audit/fix callers to be Spectre safe.
+ inline void branchTestObjShapeUnsafe(Condition cond, Register obj,
+ Register shape, Label* label);
+
+ void branchTestObjCompartment(Condition cond, Register obj,
+ const Address& compartment, Register scratch,
+ Label* label);
+ void branchTestObjCompartment(Condition cond, Register obj,
+ const JS::Compartment* compartment,
+ Register scratch, Label* label);
+
+ void branchIfNonNativeObj(Register obj, Register scratch, Label* label);
+
+ void branchIfObjectNotExtensible(Register obj, Register scratch,
+ Label* label);
+
+ inline void branchTestClassIsProxy(bool proxy, Register clasp, Label* label);
+
+ inline void branchTestObjectIsProxy(bool proxy, Register object,
+ Register scratch, Label* label);
+
+ inline void branchTestProxyHandlerFamily(Condition cond, Register proxy,
+ Register scratch,
+ const void* handlerp, Label* label);
+
+ inline void branchTestNeedsIncrementalBarrier(Condition cond, Label* label);
+ inline void branchTestNeedsIncrementalBarrierAnyZone(Condition cond,
+ Label* label,
+ Register scratch);
+
+ // Perform a type-test on a tag of a Value (32bits boxing), or the tagged
+ // value (64bits boxing).
+ inline void branchTestUndefined(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestInt32(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestDouble(Condition cond, Register tag, Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+ inline void branchTestNumber(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestBoolean(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestString(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestSymbol(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestBigInt(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestNull(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestObject(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestPrimitive(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestMagic(Condition cond, Register tag,
+ Label* label) PER_SHARED_ARCH;
+ void branchTestType(Condition cond, Register tag, JSValueType type,
+ Label* label);
+
+ // Perform a type-test on a Value, addressed by Address or BaseIndex, or
+ // loaded into ValueOperand.
+ // BaseIndex and ValueOperand variants clobber the ScratchReg on x64.
+ // All Variants clobber the ScratchReg on arm64.
+ inline void branchTestUndefined(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestUndefined(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestUndefined(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestInt32(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestInt32(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestInt32(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestDouble(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestDouble(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestDouble(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestNumber(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestBoolean(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestBoolean(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestBoolean(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestString(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestString(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestString(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestSymbol(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestSymbol(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestSymbol(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestBigInt(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestNull(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestNull(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestNull(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ // Clobbers the ScratchReg on x64.
+ inline void branchTestObject(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestObject(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestObject(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestGCThing(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestGCThing(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestGCThing(Condition cond, const ValueOperand& value,
+ Label* label) PER_SHARED_ARCH;
+
+ inline void branchTestPrimitive(Condition cond, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestMagic(Condition cond, const Address& address,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestMagic(Condition cond, const BaseIndex& address,
+ Label* label) PER_SHARED_ARCH;
+ template <class L>
+ inline void branchTestMagic(Condition cond, const ValueOperand& value,
+ L label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ inline void branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) PER_ARCH;
+
+ inline void branchTestMagicValue(Condition cond, const ValueOperand& val,
+ JSWhyMagic why, Label* label);
+
+ void branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) PER_ARCH;
+
+ inline void branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) PER_ARCH;
+
+ // Checks if given Value is evaluated to true or false in a condition.
+ // The type of the value should match the type of the method.
+ inline void branchTestInt32Truthy(bool truthy, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, x86_shared, wasm32);
+ inline void branchTestDoubleTruthy(bool truthy, FloatRegister reg,
+ Label* label) PER_SHARED_ARCH;
+ inline void branchTestBooleanTruthy(bool truthy, const ValueOperand& value,
+ Label* label) PER_ARCH;
+ inline void branchTestStringTruthy(bool truthy, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+ inline void branchTestBigIntTruthy(bool truthy, const ValueOperand& value,
+ Label* label)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, wasm32, x86_shared);
+
+ // Create an unconditional branch to the address given as argument.
+ inline void branchToComputedAddress(const BaseIndex& address) PER_ARCH;
+
+ private:
+ template <typename T, typename S, typename L>
+ inline void branchPtrImpl(Condition cond, const T& lhs, const S& rhs, L label)
+ DEFINED_ON(x86_shared);
+
+ void branchPtrInNurseryChunkImpl(Condition cond, Register ptr, Label* label)
+ DEFINED_ON(x86);
+ template <typename T>
+ void branchValueIsNurseryCellImpl(Condition cond, const T& value,
+ Register temp, Label* label)
+ DEFINED_ON(arm64, x64, mips64, loong64);
+
+ template <typename T>
+ inline void branchTestUndefinedImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestInt32Impl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestDoubleImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestNumberImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestBooleanImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestStringImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestSymbolImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestBigIntImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestNullImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestObjectImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T>
+ inline void branchTestGCThingImpl(Condition cond, const T& t,
+ Label* label) PER_SHARED_ARCH;
+ template <typename T>
+ inline void branchTestPrimitiveImpl(Condition cond, const T& t, Label* label)
+ DEFINED_ON(arm, arm64, x86_shared);
+ template <typename T, class L>
+ inline void branchTestMagicImpl(Condition cond, const T& t, L label)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ public:
+ template <typename T>
+ inline void testNumberSet(Condition cond, const T& src,
+ Register dest) PER_SHARED_ARCH;
+ template <typename T>
+ inline void testBooleanSet(Condition cond, const T& src,
+ Register dest) PER_SHARED_ARCH;
+ template <typename T>
+ inline void testStringSet(Condition cond, const T& src,
+ Register dest) PER_SHARED_ARCH;
+ template <typename T>
+ inline void testSymbolSet(Condition cond, const T& src,
+ Register dest) PER_SHARED_ARCH;
+ template <typename T>
+ inline void testBigIntSet(Condition cond, const T& src,
+ Register dest) PER_SHARED_ARCH;
+
+ public:
+ // The fallibleUnbox* methods below combine a Value type check with an unbox.
+ // Especially on 64-bit platforms this can be implemented more efficiently
+ // than a separate branch + unbox.
+ //
+ // |src| and |dest| can be the same register, but |dest| may hold garbage on
+ // failure.
+ inline void fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) PER_ARCH;
+ inline void fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) PER_ARCH;
+ inline void fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) PER_ARCH;
+ template <typename T>
+ inline void fallibleUnboxInt32(const T& src, Register dest, Label* fail);
+ template <typename T>
+ inline void fallibleUnboxBoolean(const T& src, Register dest, Label* fail);
+ template <typename T>
+ inline void fallibleUnboxObject(const T& src, Register dest, Label* fail);
+ template <typename T>
+ inline void fallibleUnboxString(const T& src, Register dest, Label* fail);
+ template <typename T>
+ inline void fallibleUnboxSymbol(const T& src, Register dest, Label* fail);
+ template <typename T>
+ inline void fallibleUnboxBigInt(const T& src, Register dest, Label* fail);
+
+ inline void cmp32Move32(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest)
+ DEFINED_ON(arm, arm64, loong64, wasm32, mips_shared, x86_shared);
+
+ inline void cmp32Move32(Condition cond, Register lhs, const Address& rhs,
+ Register src, Register dest)
+ DEFINED_ON(arm, arm64, loong64, wasm32, mips_shared, x86_shared);
+
+ inline void cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) PER_ARCH;
+
+ inline void cmpPtrMovePtr(Condition cond, Register lhs, const Address& rhs,
+ Register src, Register dest) PER_ARCH;
+
+ inline void cmp32Load32(Condition cond, Register lhs, const Address& rhs,
+ const Address& src, Register dest)
+ DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared);
+
+ inline void cmp32Load32(Condition cond, Register lhs, Register rhs,
+ const Address& src, Register dest)
+ DEFINED_ON(arm, arm64, loong64, mips_shared, x86_shared);
+
+ inline void cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest)
+ DEFINED_ON(arm, arm64, loong64, wasm32, mips_shared, x86, x64);
+
+ inline void cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest)
+ DEFINED_ON(arm, arm64, loong64, wasm32, mips_shared, x86, x64);
+
+ inline void test32LoadPtr(Condition cond, const Address& addr, Imm32 mask,
+ const Address& src, Register dest)
+ DEFINED_ON(arm, arm64, loong64, wasm32, mips_shared, x86, x64);
+
+ inline void test32MovePtr(Condition cond, const Address& addr, Imm32 mask,
+ Register src, Register dest)
+ DEFINED_ON(arm, arm64, loong64, wasm32, mips_shared, x86, x64);
+
+ // Conditional move for Spectre mitigations.
+ inline void spectreMovePtr(Condition cond, Register src, Register dest)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ // Zeroes dest if the condition is true.
+ inline void spectreZeroRegister(Condition cond, Register scratch,
+ Register dest)
+ DEFINED_ON(arm, arm64, mips_shared, x86_shared, loong64, wasm32);
+
+ // Performs a bounds check and zeroes the index register if out-of-bounds
+ // (to mitigate Spectre).
+ private:
+ inline void spectreBoundsCheck32(Register index, const Operand& length,
+ Register maybeScratch, Label* failure)
+ DEFINED_ON(x86);
+
+ public:
+ inline void spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch, Label* failure)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+ inline void spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch, Label* failure)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ inline void spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch, Label* failure)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+ inline void spectreBoundsCheckPtr(Register index, const Address& length,
+ Register maybeScratch, Label* failure)
+ DEFINED_ON(arm, arm64, mips_shared, x86, x64, loong64, wasm32);
+
+ // ========================================================================
+ // Canonicalization primitives.
+ inline void canonicalizeDouble(FloatRegister reg);
+ inline void canonicalizeDoubleIfDeterministic(FloatRegister reg);
+
+ inline void canonicalizeFloat(FloatRegister reg);
+ inline void canonicalizeFloatIfDeterministic(FloatRegister reg);
+
+ public:
+ // ========================================================================
+ // Memory access primitives.
+ inline void storeUncanonicalizedDouble(FloatRegister src, const Address& dest)
+ DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64, wasm32);
+ inline void storeUncanonicalizedDouble(FloatRegister src,
+ const BaseIndex& dest)
+ DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64, wasm32);
+ inline void storeUncanonicalizedDouble(FloatRegister src, const Operand& dest)
+ DEFINED_ON(x86_shared);
+
+ template <class T>
+ inline void storeDouble(FloatRegister src, const T& dest);
+
+ template <class T>
+ inline void boxDouble(FloatRegister src, const T& dest);
+
+ using MacroAssemblerSpecific::boxDouble;
+
+ inline void storeUncanonicalizedFloat32(FloatRegister src,
+ const Address& dest)
+ DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64, wasm32);
+ inline void storeUncanonicalizedFloat32(FloatRegister src,
+ const BaseIndex& dest)
+ DEFINED_ON(x86_shared, arm, arm64, mips32, mips64, loong64, wasm32);
+ inline void storeUncanonicalizedFloat32(FloatRegister src,
+ const Operand& dest)
+ DEFINED_ON(x86_shared);
+
+ template <class T>
+ inline void storeFloat32(FloatRegister src, const T& dest);
+
+ template <typename T>
+ void storeUnboxedValue(const ConstantOrRegister& value, MIRType valueType,
+ const T& dest) PER_ARCH;
+
+ inline void memoryBarrier(MemoryBarrierBits barrier) PER_SHARED_ARCH;
+
+ public:
+ // ========================================================================
+ // Wasm SIMD
+ //
+ // Naming is "operationSimd128" when operate on the whole vector, otherwise
+ // it's "operation<Type><Size>x<Lanes>".
+ //
+ // For microarchitectural reasons we can in principle get a performance win by
+ // using int or float specific instructions in the operationSimd128 case when
+ // we know that subsequent operations on the result are int or float oriented.
+ // In practice, we don't care about that yet.
+ //
+ // The order of operations here follows those in the SIMD overview document,
+ // https://github.com/WebAssembly/simd/blob/master/proposals/simd/SIMD.md.
+ //
+ // Since we must target Intel SSE indefinitely and SSE is one-address or
+ // two-address, the x86 porting interfaces are nearly all one-address or
+ // two-address. Likewise there are two-address ARM64 interfaces to support
+ // the baseline compiler. But there are also three-address ARM64 interfaces
+ // as the ARM64 Ion back-end can use those. In the future, they may support
+ // AVX2 or similar for x86.
+ //
+ // Conventions for argument order and naming and semantics:
+ // - Condition codes come first.
+ // - Other immediates (masks, shift counts) come next.
+ // - Operands come next:
+ // - For a binary two-address operator where the left-hand-side has the
+ // same type as the result, one register parameter is normally named
+ // `lhsDest` and is both the left-hand side and destination; the other
+ // parameter is named `rhs` and is the right-hand side. `rhs` comes
+ // first, `lhsDest` second. `rhs` and `lhsDest` may be the same register
+ // (if rhs is a register).
+ // - For a binary three-address operator the order is `lhs`, `rhs`, `dest`,
+ // and generally these registers may be the same.
+ // - For a unary operator, the input is named `src` and the output is named
+ // `dest`. `src` comes first, `dest` second. `src` and `dest` may be
+ // the same register (if `src` is a register).
+ // - Temp registers follow operands and are named `temp` if there's only one,
+ // otherwise `temp1`, `temp2`, etc regardless of type. GPR temps precede
+ // FPU temps. If there are several temps then they must be distinct
+ // registers, and they must be distinct from the operand registers unless
+ // noted.
+
+ // Moves
+
+ inline void moveSimd128(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Constants
+
+ inline void loadConstantSimd128(const SimdConstant& v, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Splat
+
+ inline void splatX16(Register src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void splatX16(uint32_t srcLane, FloatRegister src, FloatRegister dest)
+ DEFINED_ON(arm64);
+
+ inline void splatX8(Register src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void splatX8(uint32_t srcLane, FloatRegister src, FloatRegister dest)
+ DEFINED_ON(arm64);
+
+ inline void splatX4(Register src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void splatX4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void splatX2(Register64 src, FloatRegister dest)
+ DEFINED_ON(x86, x64, arm64);
+
+ inline void splatX2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Extract lane as scalar. Float extraction does not canonicalize the value.
+
+ inline void extractLaneInt8x16(uint32_t lane, FloatRegister src,
+ Register dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtractLaneInt8x16(uint32_t lane, FloatRegister src,
+ Register dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extractLaneInt16x8(uint32_t lane, FloatRegister src,
+ Register dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtractLaneInt16x8(uint32_t lane, FloatRegister src,
+ Register dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extractLaneInt32x4(uint32_t lane, FloatRegister src,
+ Register dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void extractLaneInt64x2(uint32_t lane, FloatRegister src,
+ Register64 dest) DEFINED_ON(x86, x64, arm64);
+
+ inline void extractLaneFloat32x4(uint32_t lane, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extractLaneFloat64x2(uint32_t lane, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Replace lane value
+
+ inline void replaceLaneInt8x16(unsigned lane, FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void replaceLaneInt8x16(unsigned lane, Register rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void replaceLaneInt16x8(unsigned lane, FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void replaceLaneInt16x8(unsigned lane, Register rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void replaceLaneInt32x4(unsigned lane, FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void replaceLaneInt32x4(unsigned lane, Register rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void replaceLaneInt64x2(unsigned lane, FloatRegister lhs,
+ Register64 rhs, FloatRegister dest)
+ DEFINED_ON(x86, x64);
+
+ inline void replaceLaneInt64x2(unsigned lane, Register64 rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86, x64, arm64);
+
+ inline void replaceLaneFloat32x4(unsigned lane, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ inline void replaceLaneFloat32x4(unsigned lane, FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void replaceLaneFloat64x2(unsigned lane, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ inline void replaceLaneFloat64x2(unsigned lane, FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Shuffle - blend and permute with immediate indices, and its many
+ // specializations. Lane values other than those mentioned are illegal.
+
+ // lane values 0..31
+ inline void shuffleInt8x16(const uint8_t lanes[16], FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void shuffleInt8x16(const uint8_t lanes[16], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Lane values must be 0 (select from lhs) or FF (select from rhs).
+ // The behavior is undefined for lane values that are neither 0 nor FF.
+ // on x86_shared: it is required that lhs == dest.
+ inline void blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest,
+ FloatRegister temp) DEFINED_ON(x86_shared);
+
+ // Lane values must be 0 (select from lhs) or FF (select from rhs).
+ // The behavior is undefined for lane values that are neither 0 nor FF.
+ inline void blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(arm64);
+
+ // Lane values must be 0 (select from lhs) or FFFF (select from rhs).
+ // The behavior is undefined for lane values that are neither 0 nor FFFF.
+ // on x86_shared: it is required that lhs == dest.
+ inline void blendInt16x8(const uint16_t lanes[8], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Mask lane values must be ~0 or 0. The former selects from lhs and the
+ // latter from rhs.
+ // The implementation works effectively for I8x16, I16x8, I32x4, and I64x2.
+ inline void laneSelectSimd128(FloatRegister mask, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void interleaveHighInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void interleaveHighInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void interleaveHighInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void interleaveHighInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void interleaveLowInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void interleaveLowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void interleaveLowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void interleaveLowInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Permute - permute with immediate indices.
+
+ // lane values 0..15
+ inline void permuteInt8x16(const uint8_t lanes[16], FloatRegister src,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ // lane values 0..7
+ inline void permuteInt16x8(const uint16_t lanes[8], FloatRegister src,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ // lane values 0..3 [sic].
+ inline void permuteHighInt16x8(const uint16_t lanes[4], FloatRegister src,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // lane values 0..3.
+ inline void permuteLowInt16x8(const uint16_t lanes[4], FloatRegister src,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // lane values 0..3
+ inline void permuteInt32x4(const uint32_t lanes[4], FloatRegister src,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ // Funnel shift by immediate count:
+ // low_16_bytes_of((lhs ++ rhs) >> shift*8), shift must be < 16
+ inline void concatAndRightShiftSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, uint32_t shift)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Rotate right by immediate count:
+ // low_16_bytes_of((src ++ src) >> shift*8), shift must be < 16
+ inline void rotateRightSimd128(FloatRegister src, FloatRegister dest,
+ uint32_t shift) DEFINED_ON(arm64);
+
+ // Shift bytes with immediate count, shifting in zeroes. Shift count 0..15.
+
+ inline void leftShiftSimd128(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void rightShiftSimd128(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Reverse bytes in lanes.
+
+ inline void reverseInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void reverseInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void reverseInt64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Swizzle - permute with variable indices. `rhs` holds the lanes parameter.
+
+ inline void swizzleInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void swizzleInt8x16Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Integer Add
+
+ inline void addInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void addInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void addInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void addInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void addInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void addInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void addInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void addInt64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Integer Subtract
+
+ inline void subInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void subInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void subInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void subInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void subInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void subInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void subInt64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void subInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ // Integer Multiply
+
+ inline void mulInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void mulInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void mulInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void mulInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // On x86_shared, it is required lhs == dest
+ inline void mulInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp)
+ DEFINED_ON(x86_shared);
+
+ inline void mulInt64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest, FloatRegister temp)
+ DEFINED_ON(x86_shared);
+
+ inline void mulInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) DEFINED_ON(arm64);
+
+ // Note for the extMul opcodes, the NxM designation is for the input lanes;
+ // the output lanes are twice as wide.
+ inline void extMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void q15MulrSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Integer Negate
+
+ inline void negInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void negInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void negInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void negInt64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Saturating integer add
+
+ inline void addSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void addSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedAddSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedAddSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void addSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void addSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedAddSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedAddSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Saturating integer subtract
+
+ inline void subSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void subSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedSubSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedSubSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void subSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void subSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedSubSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedSubSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Lane-wise integer minimum
+
+ inline void minInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void minInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedMinInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedMinInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void minInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void minInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedMinInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedMinInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void minInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void minInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedMinInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedMinInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Lane-wise integer maximum
+
+ inline void maxInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void maxInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedMaxInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedMaxInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void maxInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void maxInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedMaxInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedMaxInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void maxInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void maxInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedMaxInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedMaxInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Lane-wise integer rounding average
+
+ inline void unsignedAverageInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedAverageInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Lane-wise integer absolute value
+
+ inline void absInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void absInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void absInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void absInt64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Left shift by scalar. Immediates and variable shifts must have been
+ // masked; shifts of zero will work but may or may not generate code.
+
+ inline void leftShiftInt8x16(Register rhs, FloatRegister lhsDest,
+ FloatRegister temp) DEFINED_ON(x86_shared);
+
+ inline void leftShiftInt8x16(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void leftShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void leftShiftInt16x8(Register rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ inline void leftShiftInt16x8(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void leftShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void leftShiftInt32x4(Register rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ inline void leftShiftInt32x4(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void leftShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void leftShiftInt64x2(Register rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ inline void leftShiftInt64x2(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void leftShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Right shift by scalar. Immediates and variable shifts must have been
+ // masked; shifts of zero will work but may or may not generate code.
+
+ inline void rightShiftInt8x16(Register rhs, FloatRegister lhsDest,
+ FloatRegister temp) DEFINED_ON(x86_shared);
+
+ inline void rightShiftInt8x16(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void rightShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedRightShiftInt8x16(Register rhs, FloatRegister lhsDest,
+ FloatRegister temp)
+ DEFINED_ON(x86_shared);
+
+ inline void unsignedRightShiftInt8x16(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void unsignedRightShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void rightShiftInt16x8(Register rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ inline void rightShiftInt16x8(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void rightShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedRightShiftInt16x8(Register rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ inline void unsignedRightShiftInt16x8(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void unsignedRightShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void rightShiftInt32x4(Register rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ inline void rightShiftInt32x4(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void rightShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedRightShiftInt32x4(Register rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ inline void unsignedRightShiftInt32x4(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void unsignedRightShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void rightShiftInt64x2(Register rhs, FloatRegister lhsDest,
+ FloatRegister temp) DEFINED_ON(x86_shared);
+
+ inline void rightShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void rightShiftInt64x2(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void unsignedRightShiftInt64x2(Register rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ inline void unsignedRightShiftInt64x2(FloatRegister lhs, Register rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void unsignedRightShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Sign replication operation
+
+ inline void signReplicationInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ inline void signReplicationInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ inline void signReplicationInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ inline void signReplicationInt64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ // Bitwise and, or, xor, not
+
+ inline void bitwiseAndSimd128(FloatRegister rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void bitwiseAndSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void bitwiseAndSimd128(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void bitwiseOrSimd128(FloatRegister rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void bitwiseOrSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void bitwiseOrSimd128(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void bitwiseXorSimd128(FloatRegister rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void bitwiseXorSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void bitwiseXorSimd128(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void bitwiseNotSimd128(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Bitwise AND with compliment: dest = lhs & ~rhs, note only arm64 can do it.
+ inline void bitwiseAndNotSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister lhsDest) DEFINED_ON(arm64);
+
+ // Bitwise AND with complement: dest = ~lhs & rhs, note this is not what Wasm
+ // wants but what the x86 hardware offers. Hence the name.
+
+ inline void bitwiseNotAndSimd128(FloatRegister rhs, FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void bitwiseNotAndSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared);
+
+ // Bitwise select
+
+ inline void bitwiseSelectSimd128(FloatRegister mask, FloatRegister onTrue,
+ FloatRegister onFalse, FloatRegister dest,
+ FloatRegister temp) DEFINED_ON(x86_shared);
+
+ inline void bitwiseSelectSimd128(FloatRegister onTrue, FloatRegister onFalse,
+ FloatRegister maskDest) DEFINED_ON(arm64);
+
+ // Population count
+
+ inline void popcntInt8x16(FloatRegister src, FloatRegister dest,
+ FloatRegister temp) DEFINED_ON(x86_shared);
+
+ inline void popcntInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(arm64);
+
+ // Any lane true, ie, any bit set
+
+ inline void anyTrueSimd128(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // All lanes true
+
+ inline void allTrueInt8x16(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void allTrueInt16x8(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void allTrueInt32x4(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void allTrueInt64x2(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Bitmask, ie extract and compress high bits of all lanes
+
+ inline void bitmaskInt8x16(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared);
+
+ inline void bitmaskInt8x16(FloatRegister src, Register dest,
+ FloatRegister temp) DEFINED_ON(arm64);
+
+ inline void bitmaskInt16x8(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared);
+
+ inline void bitmaskInt16x8(FloatRegister src, Register dest,
+ FloatRegister temp) DEFINED_ON(arm64);
+
+ inline void bitmaskInt32x4(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared);
+
+ inline void bitmaskInt32x4(FloatRegister src, Register dest,
+ FloatRegister temp) DEFINED_ON(arm64);
+
+ inline void bitmaskInt64x2(FloatRegister src, Register dest)
+ DEFINED_ON(x86_shared);
+
+ inline void bitmaskInt64x2(FloatRegister src, Register dest,
+ FloatRegister temp) DEFINED_ON(arm64);
+
+ // Comparisons (integer and floating-point)
+
+ inline void compareInt8x16(Assembler::Condition cond, FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // On x86_shared, limited to !=, ==, <=, >
+ inline void compareInt8x16(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ // On arm64, use any integer comparison condition.
+ inline void compareInt8x16(Assembler::Condition cond, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void compareInt16x8(Assembler::Condition cond, FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void compareInt16x8(Assembler::Condition cond, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // On x86_shared, limited to !=, ==, <=, >
+ inline void compareInt16x8(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ // On x86_shared, limited to !=, ==, <=, >
+ inline void compareInt32x4(Assembler::Condition cond, FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void compareInt32x4(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ // On arm64, use any integer comparison condition.
+ inline void compareInt32x4(Assembler::Condition cond, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void compareForEqualityInt64x2(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ inline void compareForOrderingInt64x2(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2)
+ DEFINED_ON(x86_shared);
+
+ inline void compareInt64x2(Assembler::Condition cond, FloatRegister rhs,
+ FloatRegister lhsDest) DEFINED_ON(arm64);
+
+ inline void compareInt64x2(Assembler::Condition cond, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(arm64);
+
+ inline void compareFloat32x4(Assembler::Condition cond, FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // On x86_shared, limited to ==, !=, <, <=
+ inline void compareFloat32x4(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ // On x86_shared, limited to ==, !=, <, <=
+ // On arm64, use any float-point comparison condition.
+ inline void compareFloat32x4(Assembler::Condition cond, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void compareFloat64x2(Assembler::Condition cond, FloatRegister rhs,
+ FloatRegister lhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // On x86_shared, limited to ==, !=, <, <=
+ inline void compareFloat64x2(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ // On x86_shared, limited to ==, !=, <, <=
+ // On arm64, use any float-point comparison condition.
+ inline void compareFloat64x2(Assembler::Condition cond, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Load
+
+ inline void loadUnalignedSimd128(const Operand& src, FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ inline void loadUnalignedSimd128(const Address& src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void loadUnalignedSimd128(const BaseIndex& src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Store
+
+ inline void storeUnalignedSimd128(FloatRegister src, const Address& dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void storeUnalignedSimd128(FloatRegister src, const BaseIndex& dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Floating point negation
+
+ inline void negFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void negFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Floating point absolute value
+
+ inline void absFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void absFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // NaN-propagating minimum
+
+ inline void minFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) DEFINED_ON(x86_shared);
+
+ inline void minFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
+ DEFINED_ON(arm64);
+
+ inline void minFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void minFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) DEFINED_ON(x86_shared);
+
+ inline void minFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
+ DEFINED_ON(arm64);
+
+ inline void minFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ // NaN-propagating maximum
+
+ inline void maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) DEFINED_ON(x86_shared);
+
+ inline void maxFloat32x4(FloatRegister rhs, FloatRegister lhsDest)
+ DEFINED_ON(arm64);
+
+ inline void maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ inline void maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) DEFINED_ON(x86_shared);
+
+ inline void maxFloat64x2(FloatRegister rhs, FloatRegister lhsDest)
+ DEFINED_ON(arm64);
+
+ inline void maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(arm64);
+
+ // Floating add
+
+ inline void addFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void addFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void addFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void addFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Floating subtract
+
+ inline void subFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void subFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void subFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void subFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Floating division
+
+ inline void divFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void divFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void divFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void divFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Floating Multiply
+
+ inline void mulFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void mulFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void mulFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void mulFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ // Pairwise add
+
+ inline void extAddPairwiseInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtAddPairwiseInt8x16(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void extAddPairwiseInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedExtAddPairwiseInt16x8(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Floating square root
+
+ inline void sqrtFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void sqrtFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Integer to floating point with rounding
+
+ inline void convertInt32x4ToFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedConvertInt32x4ToFloat32x4(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void convertInt32x4ToFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedConvertInt32x4ToFloat64x2(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Floating point to integer with saturation
+
+ inline void truncSatFloat32x4ToInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
+ FloatRegister dest,
+ FloatRegister temp)
+ DEFINED_ON(x86_shared);
+
+ inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(arm64);
+
+ inline void truncSatFloat64x2ToInt32x4(FloatRegister src, FloatRegister dest,
+ FloatRegister temp)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src,
+ FloatRegister dest,
+ FloatRegister temp)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void truncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedTruncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void truncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedTruncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Floating point narrowing
+
+ inline void convertFloat64x2ToFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Floating point widening
+
+ inline void convertFloat32x4ToFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Integer to integer narrowing
+
+ inline void narrowInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void narrowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedNarrowInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedNarrowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void narrowInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void narrowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedNarrowInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void unsignedNarrowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Integer to integer widening
+
+ inline void widenLowInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void widenHighInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedWidenLowInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedWidenHighInt8x16(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void widenLowInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void widenHighInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedWidenLowInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedWidenHighInt16x8(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void widenLowInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedWidenLowInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void widenHighInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void unsignedWidenHighInt32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Compare-based minimum/maximum
+ //
+ // On x86, the signature is (rhsDest, lhs); on arm64 it is (rhs, lhsDest).
+ //
+ // The masm preprocessor can't deal with multiple declarations with identical
+ // signatures even if they are on different platforms, hence the weird
+ // argument names.
+
+ inline void pseudoMinFloat32x4(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void pseudoMinFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void pseudoMinFloat64x2(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void pseudoMinFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void pseudoMaxFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void pseudoMaxFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Widening/pairwise integer dot product
+
+ inline void widenDotInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared, arm64);
+
+ inline void widenDotInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) DEFINED_ON(x86_shared);
+
+ inline void dotInt8x16Int7x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void dotInt8x16Int7x16ThenAdd(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared);
+
+ inline void dotInt8x16Int7x16ThenAdd(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp)
+ DEFINED_ON(arm64);
+
+ inline void dotBFloat16x8ThenAdd(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Floating point rounding
+
+ inline void ceilFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void ceilFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void floorFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void floorFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void truncFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void truncFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void nearestFloat32x4(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void nearestFloat64x2(FloatRegister src, FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ // Floating multiply-accumulate: srcDest [+-]= src1 * src2
+
+ inline void fmaFloat32x4(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) DEFINED_ON(x86_shared, arm64);
+
+ inline void fnmaFloat32x4(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void fmaFloat64x2(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) DEFINED_ON(x86_shared, arm64);
+
+ inline void fnmaFloat64x2(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void minFloat32x4Relaxed(FloatRegister src, FloatRegister srcDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void minFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void maxFloat32x4Relaxed(FloatRegister src, FloatRegister srcDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void maxFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void minFloat64x2Relaxed(FloatRegister src, FloatRegister srcDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void minFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void maxFloat64x2Relaxed(FloatRegister src, FloatRegister srcDest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void maxFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ inline void q15MulrInt16x8Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest)
+ DEFINED_ON(x86_shared, arm64);
+
+ public:
+ // ========================================================================
+ // Truncate floating point.
+
+ // Undefined behaviour when truncation is outside Int64 range.
+ // Needs a temp register if SSE3 is not present.
+ inline void truncateFloat32ToInt64(Address src, Address dest, Register temp)
+ DEFINED_ON(x86_shared);
+ inline void truncateFloat32ToUInt64(Address src, Address dest, Register temp,
+ FloatRegister floatTemp)
+ DEFINED_ON(x86, x64);
+ inline void truncateDoubleToInt64(Address src, Address dest, Register temp)
+ DEFINED_ON(x86_shared);
+ inline void truncateDoubleToUInt64(Address src, Address dest, Register temp,
+ FloatRegister floatTemp)
+ DEFINED_ON(x86, x64);
+
+ public:
+ // ========================================================================
+ // Convert floating point.
+
+ // temp required on x86 and x64; must be undefined on mips64 and loong64.
+ void convertUInt64ToFloat32(Register64 src, FloatRegister dest, Register temp)
+ DEFINED_ON(arm64, mips64, loong64, wasm32, x64, x86);
+
+ void convertInt64ToFloat32(Register64 src, FloatRegister dest)
+ DEFINED_ON(arm64, mips64, loong64, wasm32, x64, x86);
+
+ bool convertUInt64ToDoubleNeedsTemp() PER_ARCH;
+
+ // temp required when convertUInt64ToDoubleNeedsTemp() returns true.
+ void convertUInt64ToDouble(Register64 src, FloatRegister dest,
+ Register temp) PER_ARCH;
+
+ void convertInt64ToDouble(Register64 src, FloatRegister dest) PER_ARCH;
+
+ void convertIntPtrToDouble(Register src, FloatRegister dest) PER_ARCH;
+
+ public:
+ // ========================================================================
+ // wasm support
+
+ CodeOffset wasmTrapInstruction() PER_SHARED_ARCH;
+
+ void wasmTrap(wasm::Trap trap, wasm::BytecodeOffset bytecodeOffset);
+
+ // Load all pinned regs via InstanceReg. If the trapOffset is something,
+ // give the first load a trap descriptor with type IndirectCallToNull, so that
+ // a null instance will cause a trap.
+ void loadWasmPinnedRegsFromInstance(
+ mozilla::Maybe<wasm::BytecodeOffset> trapOffset = mozilla::Nothing());
+
+ // Returns a pair: the offset of the undefined (trapping) instruction, and
+ // the number of extra bytes of stack allocated prior to the trap
+ // instruction proper.
+ std::pair<CodeOffset, uint32_t> wasmReserveStackChecked(
+ uint32_t amount, wasm::BytecodeOffset trapOffset);
+
+ // Emit a bounds check against the wasm heap limit, jumping to 'ok' if 'cond'
+ // holds; this can be the label either of the access or of the trap. The
+ // label should name a code position greater than the position of the bounds
+ // check.
+ //
+ // If JitOptions.spectreMaskIndex is true, a no-op speculation barrier is
+ // emitted in the code stream after the check to prevent an OOB access from
+ // being executed speculatively. (On current tier-1 platforms the barrier is
+ // a conditional saturation of 'index' to 'boundsCheckLimit', using the same
+ // condition as the check.) If the condition is such that the bounds check
+ // branches out of line to the trap, the barrier will actually be executed
+ // when the bounds check passes.
+ //
+ // On 32-bit systems for both wasm and asm.js, and on 64-bit systems for
+ // asm.js, heap lengths are limited to 2GB. On 64-bit systems for wasm,
+ // 32-bit heap lengths are limited to 4GB, and 64-bit heap lengths will be
+ // limited to something much larger.
+
+ void wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* ok)
+ DEFINED_ON(arm, arm64, mips32, mips64, x86_shared, loong64, wasm32);
+
+ void wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* ok)
+ DEFINED_ON(arm, arm64, mips32, mips64, x86_shared, loong64, wasm32);
+
+ void wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok)
+ DEFINED_ON(arm64, mips64, x64, x86, arm, loong64, wasm32);
+
+ void wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok)
+ DEFINED_ON(arm64, mips64, x64, x86, arm, loong64, wasm32);
+
+ // Each wasm load/store instruction appends its own wasm::Trap::OutOfBounds.
+ void wasmLoad(const wasm::MemoryAccessDesc& access, Operand srcAddr,
+ AnyRegister out) DEFINED_ON(x86, x64);
+ void wasmLoadI64(const wasm::MemoryAccessDesc& access, Operand srcAddr,
+ Register64 out) DEFINED_ON(x86, x64);
+ void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
+ Operand dstAddr) DEFINED_ON(x86, x64);
+ void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
+ Operand dstAddr) DEFINED_ON(x86);
+
+ // For all the ARM/MIPS/LOONG64 wasmLoad and wasmStore functions below, `ptr`
+ // MUST equal `ptrScratch`, and that register will be updated based on
+ // conditions listed below (where it is only mentioned as `ptr`).
+
+ // `ptr` will be updated if access.offset() != 0 or access.type() ==
+ // Scalar::Int64.
+ void wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, AnyRegister output)
+ DEFINED_ON(arm, loong64, mips_shared);
+ void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, Register64 output)
+ DEFINED_ON(arm, mips32, mips64, loong64);
+ void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
+ Register memoryBase, Register ptr, Register ptrScratch)
+ DEFINED_ON(arm, loong64, mips_shared);
+ void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
+ Register memoryBase, Register ptr, Register ptrScratch)
+ DEFINED_ON(arm, mips32, mips64, loong64);
+
+ // These accept general memoryBase + ptr + offset (in `access`); the offset is
+ // always smaller than the guard region. They will insert an additional add
+ // if the offset is nonzero, and of course that add may require a temporary
+ // register for the offset if the offset is large, and instructions to set it
+ // up.
+ void wasmLoad(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, AnyRegister output) DEFINED_ON(arm64);
+ void wasmLoadI64(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register64 output) DEFINED_ON(arm64);
+ void wasmStore(const wasm::MemoryAccessDesc& access, AnyRegister value,
+ Register memoryBase, Register ptr) DEFINED_ON(arm64);
+ void wasmStoreI64(const wasm::MemoryAccessDesc& access, Register64 value,
+ Register memoryBase, Register ptr) DEFINED_ON(arm64);
+
+ // `ptr` will always be updated.
+ void wasmUnalignedLoad(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register output, Register tmp)
+ DEFINED_ON(mips32, mips64);
+
+ // MIPS: `ptr` will always be updated.
+ void wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, FloatRegister output,
+ Register tmp1) DEFINED_ON(mips32, mips64);
+
+ // `ptr` will always be updated.
+ void wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output,
+ Register tmp) DEFINED_ON(mips32, mips64);
+
+ // MIPS: `ptr` will always be updated.
+ void wasmUnalignedStore(const wasm::MemoryAccessDesc& access, Register value,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register tmp)
+ DEFINED_ON(mips32, mips64);
+
+ // `ptr` will always be updated.
+ void wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access,
+ FloatRegister floatValue, Register memoryBase,
+ Register ptr, Register ptrScratch, Register tmp)
+ DEFINED_ON(mips32, mips64);
+
+ // `ptr` will always be updated.
+ void wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch, Register tmp)
+ DEFINED_ON(mips32, mips64);
+
+ // wasm specific methods, used in both the wasm baseline compiler and ion.
+
+ // The truncate-to-int32 methods do not bind the rejoin label; clients must
+ // do so if oolWasmTruncateCheckF64ToI32() can jump to it.
+ void wasmTruncateDoubleToUInt32(FloatRegister input, Register output,
+ bool isSaturating, Label* oolEntry) PER_ARCH;
+ void wasmTruncateDoubleToInt32(FloatRegister input, Register output,
+ bool isSaturating,
+ Label* oolEntry) PER_SHARED_ARCH;
+ void oolWasmTruncateCheckF64ToI32(FloatRegister input, Register output,
+ TruncFlags flags, wasm::BytecodeOffset off,
+ Label* rejoin)
+ DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64, wasm32);
+
+ void wasmTruncateFloat32ToUInt32(FloatRegister input, Register output,
+ bool isSaturating, Label* oolEntry) PER_ARCH;
+ void wasmTruncateFloat32ToInt32(FloatRegister input, Register output,
+ bool isSaturating,
+ Label* oolEntry) PER_SHARED_ARCH;
+ void oolWasmTruncateCheckF32ToI32(FloatRegister input, Register output,
+ TruncFlags flags, wasm::BytecodeOffset off,
+ Label* rejoin)
+ DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64, wasm32);
+
+ // The truncate-to-int64 methods will always bind the `oolRejoin` label
+ // after the last emitted instruction.
+ void wasmTruncateDoubleToInt64(FloatRegister input, Register64 output,
+ bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble)
+ DEFINED_ON(arm64, x86, x64, mips64, loong64, wasm32);
+ void wasmTruncateDoubleToUInt64(FloatRegister input, Register64 output,
+ bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble)
+ DEFINED_ON(arm64, x86, x64, mips64, loong64, wasm32);
+ void oolWasmTruncateCheckF64ToI64(FloatRegister input, Register64 output,
+ TruncFlags flags, wasm::BytecodeOffset off,
+ Label* rejoin)
+ DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64, wasm32);
+
+ void wasmTruncateFloat32ToInt64(FloatRegister input, Register64 output,
+ bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble)
+ DEFINED_ON(arm64, x86, x64, mips64, loong64, wasm32);
+ void wasmTruncateFloat32ToUInt64(FloatRegister input, Register64 output,
+ bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble)
+ DEFINED_ON(arm64, x86, x64, mips64, loong64, wasm32);
+ void oolWasmTruncateCheckF32ToI64(FloatRegister input, Register64 output,
+ TruncFlags flags, wasm::BytecodeOffset off,
+ Label* rejoin)
+ DEFINED_ON(arm, arm64, x86_shared, mips_shared, loong64, wasm32);
+
+ void loadWasmGlobalPtr(uint32_t globalDataOffset, Register dest);
+
+ // This function takes care of loading the callee's instance and pinned regs
+ // but it is the caller's responsibility to save/restore instance or pinned
+ // regs.
+ CodeOffset wasmCallImport(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee);
+
+ // WasmTableCallIndexReg must contain the index of the indirect call. This is
+ // for wasm calls only.
+ //
+ // Indirect calls use a dual-path mechanism where a run-time test determines
+ // whether a context switch is needed (slow path) or not (fast path). This
+ // gives rise to two call instructions, both of which need safe points. As
+ // per normal, the call offsets are the code offsets at the end of the call
+ // instructions (the return points).
+ //
+ // `boundsCheckFailedLabel` is non-null iff a bounds check is required.
+ // `nullCheckFailedLabel` is non-null only on platforms that can't fold the
+ // null check into the rest of the call instructions.
+ void wasmCallIndirect(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee,
+ Label* boundsCheckFailedLabel,
+ Label* nullCheckFailedLabel,
+ mozilla::Maybe<uint32_t> tableSize,
+ CodeOffset* fastCallOffset, CodeOffset* slowCallOffset);
+
+ // This function takes care of loading the callee's instance and address from
+ // pinned reg.
+ void wasmCallRef(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee, CodeOffset* fastCallOffset,
+ CodeOffset* slowCallOffset);
+
+ // WasmTableCallIndexReg must contain the index of the indirect call.
+ // This is for asm.js calls only.
+ CodeOffset asmCallIndirect(const wasm::CallSiteDesc& desc,
+ const wasm::CalleeDesc& callee);
+
+ // This function takes care of loading the pointer to the current instance
+ // as the implicit first argument. It preserves instance and pinned registers.
+ // (instance & pinned regs are non-volatile registers in the system ABI).
+ CodeOffset wasmCallBuiltinInstanceMethod(const wasm::CallSiteDesc& desc,
+ const ABIArg& instanceArg,
+ wasm::SymbolicAddress builtin,
+ wasm::FailureMode failureMode);
+
+ // Perform a subtype check that `subTypeDef` is a subtype of `superTypeDef`
+ // branching to label depending on `onSuccess`. This method is a
+ // specialization of the general `wasm::TypeDef::isSubTypeOf` method for the
+ // case where the `superTypeDef` is statically known, which is the case for
+ // all wasm instructions.
+ //
+ // `scratch` is required iff the `subTypeDepth` is >=
+ // wasm::MinSuperTypeVectorLength. `subTypeDef` is clobbered by this method.
+ // `superTypeDef` is preserved.
+ void branchWasmTypeDefIsSubtype(Register subTypeDef, Register superTypeDef,
+ Register scratch, uint32_t subTypeDepth,
+ Label* label, bool onSuccess);
+
+ // Compute ptr += (indexTemp32 << shift) where shift can be any value < 32.
+ // May destroy indexTemp32. The value of indexTemp32 must be positive, and it
+ // is implementation-defined what happens if bits are lost or the value
+ // becomes negative through the shift. On 64-bit systems, the high 32 bits of
+ // indexTemp32 must be zero, not garbage.
+ void shiftIndex32AndAdd(Register indexTemp32, int shift,
+ Register pointer) PER_SHARED_ARCH;
+
+ // The System ABI frequently states that the high bits of a 64-bit register
+ // that holds a 32-bit return value are unpredictable, and C++ compilers will
+ // indeed generate code that leaves garbage in the upper bits.
+ //
+ // Adjust the contents of the 64-bit register `r` to conform to our internal
+ // convention, which requires predictable high bits. In practice, this means
+ // that the 32-bit value will be zero-extended or sign-extended to 64 bits as
+ // appropriate for the platform.
+ void widenInt32(Register r) DEFINED_ON(arm64, x64, mips64, loong64);
+
+ // As enterFakeExitFrame(), but using register conventions appropriate for
+ // wasm stubs.
+ void enterFakeExitFrameForWasm(Register cxreg, Register scratch,
+ ExitFrameType type) PER_SHARED_ARCH;
+
+ public:
+ // ========================================================================
+ // Barrier functions.
+
+ void emitPreBarrierFastPath(JSRuntime* rt, MIRType type, Register temp1,
+ Register temp2, Register temp3, Label* noBarrier);
+
+ public:
+ // ========================================================================
+ // Clamping functions.
+
+ inline void clampIntToUint8(Register reg) PER_SHARED_ARCH;
+
+ public:
+ // ========================================================================
+ // Primitive atomic operations.
+ //
+ // If the access is from JS and the eventual destination of the result is a
+ // js::Value, it's probably best to use the JS-specific versions of these,
+ // see further below.
+ //
+ // Temp registers must be defined unless otherwise noted in the per-function
+ // constraints.
+
+ // 8-bit, 16-bit, and 32-bit wide operations.
+ //
+ // The 8-bit and 16-bit operations zero-extend or sign-extend the result to
+ // 32 bits, according to `type`. On 64-bit systems, the upper 32 bits of the
+ // result will be zero on some platforms (eg, on x64) and will be the sign
+ // extension of the lower bits on other platforms (eg, MIPS).
+
+ // CompareExchange with memory. Return the value that was in memory,
+ // whether we wrote or not.
+ //
+ // x86-shared: `output` must be eax.
+ // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
+ // and 16-bit wide operations.
+
+ void compareExchange(Scalar::Type type, const Synchronization& sync,
+ const Address& mem, Register expected,
+ Register replacement, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void compareExchange(Scalar::Type type, const Synchronization& sync,
+ const BaseIndex& mem, Register expected,
+ Register replacement, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void compareExchange(Scalar::Type type, const Synchronization& sync,
+ const Address& mem, Register expected,
+ Register replacement, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+ DEFINED_ON(mips_shared, loong64);
+
+ void compareExchange(Scalar::Type type, const Synchronization& sync,
+ const BaseIndex& mem, Register expected,
+ Register replacement, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+ DEFINED_ON(mips_shared, loong64);
+
+ // x86: `expected` and `output` must be edx:eax; `replacement` is ecx:ebx.
+ // x64: `output` must be rax.
+ // ARM: Registers must be distinct; `replacement` and `output` must be
+ // (even,odd) pairs.
+
+ void compareExchange64(const Synchronization& sync, const Address& mem,
+ Register64 expected, Register64 replacement,
+ Register64 output)
+ DEFINED_ON(arm, arm64, x64, x86, mips64, loong64);
+
+ void compareExchange64(const Synchronization& sync, const BaseIndex& mem,
+ Register64 expected, Register64 replacement,
+ Register64 output)
+ DEFINED_ON(arm, arm64, x64, x86, mips64, loong64);
+
+ // Exchange with memory. Return the value initially in memory.
+ // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
+ // and 16-bit wide operations.
+
+ void atomicExchange(Scalar::Type type, const Synchronization& sync,
+ const Address& mem, Register value, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicExchange(Scalar::Type type, const Synchronization& sync,
+ const BaseIndex& mem, Register value, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicExchange(Scalar::Type type, const Synchronization& sync,
+ const Address& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+ DEFINED_ON(mips_shared, loong64);
+
+ void atomicExchange(Scalar::Type type, const Synchronization& sync,
+ const BaseIndex& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register output)
+ DEFINED_ON(mips_shared, loong64);
+
+ // x86: `value` must be ecx:ebx; `output` must be edx:eax.
+ // ARM: `value` and `output` must be distinct and (even,odd) pairs.
+ // ARM64: `value` and `output` must be distinct.
+
+ void atomicExchange64(const Synchronization& sync, const Address& mem,
+ Register64 value, Register64 output)
+ DEFINED_ON(arm, arm64, x64, x86, mips64, loong64);
+
+ void atomicExchange64(const Synchronization& sync, const BaseIndex& mem,
+ Register64 value, Register64 output)
+ DEFINED_ON(arm, arm64, x64, x86, mips64, loong64);
+
+ // Read-modify-write with memory. Return the value in memory before the
+ // operation.
+ //
+ // x86-shared:
+ // For 8-bit operations, `value` and `output` must have a byte subregister.
+ // For Add and Sub, `temp` must be invalid.
+ // For And, Or, and Xor, `output` must be eax and `temp` must have a byte
+ // subregister.
+ //
+ // ARM: Registers `value` and `output` must differ.
+ // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
+ // and 16-bit wide operations; `value` and `output` must differ.
+
+ void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, Register value, const Address& mem,
+ Register temp, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, Imm32 value, const Address& mem,
+ Register temp, Register output) DEFINED_ON(x86_shared);
+
+ void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, Register value, const BaseIndex& mem,
+ Register temp, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, Imm32 value, const BaseIndex& mem,
+ Register temp, Register output) DEFINED_ON(x86_shared);
+
+ void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp, Register maskTemp,
+ Register output) DEFINED_ON(mips_shared, loong64);
+
+ void atomicFetchOp(Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp, Register maskTemp,
+ Register output) DEFINED_ON(mips_shared, loong64);
+
+ // x86:
+ // `temp` must be ecx:ebx; `output` must be edx:eax.
+ // x64:
+ // For Add and Sub, `temp` is ignored.
+ // For And, Or, and Xor, `output` must be rax.
+ // ARM:
+ // `temp` and `output` must be (even,odd) pairs and distinct from `value`.
+ // ARM64:
+ // Registers `value`, `temp`, and `output` must all differ.
+
+ void atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem, Register64 temp,
+ Register64 output)
+ DEFINED_ON(arm, arm64, x64, mips64, loong64);
+
+ void atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ const Address& value, const Address& mem,
+ Register64 temp, Register64 output) DEFINED_ON(x86);
+
+ void atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem, Register64 temp,
+ Register64 output)
+ DEFINED_ON(arm, arm64, x64, mips64, loong64);
+
+ void atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ const Address& value, const BaseIndex& mem,
+ Register64 temp, Register64 output) DEFINED_ON(x86);
+
+ // x64:
+ // `value` can be any register.
+ // ARM:
+ // `temp` must be an (even,odd) pair and distinct from `value`.
+ // ARM64:
+ // Registers `value` and `temp` must differ.
+
+ void atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem) DEFINED_ON(x64);
+
+ void atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem, Register64 temp)
+ DEFINED_ON(arm, arm64, mips64, loong64);
+
+ void atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem) DEFINED_ON(x64);
+
+ void atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem, Register64 temp)
+ DEFINED_ON(arm, arm64, mips64, loong64);
+
+ // 64-bit atomic load. On 64-bit systems, use regular load with
+ // Synchronization::Load, not this method.
+ //
+ // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
+ // ARM: `output` must be (even,odd) pair.
+
+ void atomicLoad64(const Synchronization& sync, const Address& mem,
+ Register64 temp, Register64 output) DEFINED_ON(x86);
+
+ void atomicLoad64(const Synchronization& sync, const BaseIndex& mem,
+ Register64 temp, Register64 output) DEFINED_ON(x86);
+
+ void atomicLoad64(const Synchronization& sync, const Address& mem,
+ Register64 output) DEFINED_ON(arm);
+
+ void atomicLoad64(const Synchronization& sync, const BaseIndex& mem,
+ Register64 output) DEFINED_ON(arm);
+
+ // 64-bit atomic store. On 64-bit systems, use regular store with
+ // Synchronization::Store, not this method.
+ //
+ // x86: `value` must be ecx:ebx; `temp` must be edx:eax.
+ // ARM: `value` and `temp` must be (even,odd) pairs.
+
+ void atomicStore64(const Synchronization& sync, const Address& mem,
+ Register64 value, Register64 temp) DEFINED_ON(x86, arm);
+
+ void atomicStore64(const Synchronization& sync, const BaseIndex& mem,
+ Register64 value, Register64 temp) DEFINED_ON(x86, arm);
+
+ // ========================================================================
+ // Wasm atomic operations.
+ //
+ // Constraints, when omitted, are exactly as for the primitive operations
+ // above.
+
+ void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register expected,
+ Register replacement, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register expected,
+ Register replacement, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register expected,
+ Register replacement, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) DEFINED_ON(mips_shared, loong64);
+
+ void wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register expected,
+ Register replacement, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) DEFINED_ON(mips_shared, loong64);
+
+ void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register value, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register value, Register output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output)
+ DEFINED_ON(mips_shared, loong64);
+
+ void wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output)
+ DEFINED_ON(mips_shared, loong64);
+
+ void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const Address& mem, Register temp,
+ Register output) DEFINED_ON(arm, arm64, x86_shared);
+
+ void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Imm32 value, const Address& mem, Register temp,
+ Register output) DEFINED_ON(x86_shared);
+
+ void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const BaseIndex& mem, Register temp,
+ Register output) DEFINED_ON(arm, arm64, x86_shared);
+
+ void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Imm32 value, const BaseIndex& mem, Register temp,
+ Register output) DEFINED_ON(x86_shared);
+
+ void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const Address& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) DEFINED_ON(mips_shared, loong64);
+
+ void wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output)
+ DEFINED_ON(mips_shared, loong64);
+
+ // Read-modify-write with memory. Return no value.
+ //
+ // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
+ // and 16-bit wide operations.
+
+ void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const Address& mem, Register temp)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Imm32 value, const Address& mem, Register temp)
+ DEFINED_ON(x86_shared);
+
+ void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const BaseIndex& mem, Register temp)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Imm32 value, const BaseIndex& mem, Register temp)
+ DEFINED_ON(x86_shared);
+
+ void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) DEFINED_ON(mips_shared, loong64);
+
+ void wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) DEFINED_ON(mips_shared, loong64);
+
+ // 64-bit wide operations.
+
+ // 64-bit atomic load. On 64-bit systems, use regular wasm load with
+ // Synchronization::Load, not this method.
+ //
+ // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
+ // ARM: `temp` should be invalid; `output` must be (even,odd) pair.
+ // MIPS32: `temp` should be invalid.
+
+ void wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 temp, Register64 output)
+ DEFINED_ON(arm, mips32, x86, wasm32);
+
+ void wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) DEFINED_ON(arm, mips32, x86, wasm32);
+
+ // x86: `expected` must be the same as `output`, and must be edx:eax.
+ // x86: `replacement` must be ecx:ebx.
+ // x64: `output` must be rax.
+ // ARM: Registers must be distinct; `replacement` and `output` must be
+ // (even,odd) pairs.
+ // ARM64: The base register in `mem` must not overlap `output`.
+ // MIPS: Registers must be distinct.
+
+ void wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 expected,
+ Register64 replacement,
+ Register64 output) PER_ARCH;
+
+ void wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 expected,
+ Register64 replacement,
+ Register64 output) PER_ARCH;
+
+ // x86: `value` must be ecx:ebx; `output` must be edx:eax.
+ // ARM: Registers must be distinct; `value` and `output` must be (even,odd)
+ // pairs.
+ // MIPS: Registers must be distinct.
+
+ void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 value,
+ Register64 output) PER_ARCH;
+
+ void wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 value,
+ Register64 output) PER_ARCH;
+
+ // x86: `output` must be edx:eax, `temp` must be ecx:ebx.
+ // x64: For And, Or, and Xor `output` must be rax.
+ // ARM: Registers must be distinct; `temp` and `output` must be (even,odd)
+ // pairs.
+ // MIPS: Registers must be distinct.
+ // MIPS32: `temp` should be invalid.
+
+ void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp, Register64 output)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, x64);
+
+ void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp, Register64 output)
+ DEFINED_ON(arm, arm64, mips32, mips64, loong64, x64);
+
+ void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ const Address& value, const Address& mem,
+ Register64 temp, Register64 output) DEFINED_ON(x86);
+
+ void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ const Address& value, const BaseIndex& mem,
+ Register64 temp, Register64 output) DEFINED_ON(x86);
+
+ // Here `value` can be any register.
+
+ void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register64 value, const BaseIndex& mem)
+ DEFINED_ON(x64);
+
+ void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp) DEFINED_ON(arm64);
+
+ // ========================================================================
+ // JS atomic operations.
+ //
+ // Here the arrayType must be a type that is valid for JS. As of 2017 that
+ // is an 8-bit, 16-bit, or 32-bit integer type.
+ //
+ // If arrayType is Scalar::Uint32 then:
+ //
+ // - `output` must be a float register
+ // - if the operation takes one temp register then `temp` must be defined
+ // - if the operation takes two temp registers then `temp2` must be defined.
+ //
+ // Otherwise `output` must be a GPR and `temp`/`temp2` should be InvalidReg.
+ // (`temp1` must always be valid.)
+ //
+ // For additional register constraints, see the primitive 32-bit operations
+ // and/or wasm operations above.
+
+ void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
+ const Address& mem, Register expected,
+ Register replacement, Register temp,
+ AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);
+
+ void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
+ const BaseIndex& mem, Register expected,
+ Register replacement, Register temp,
+ AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);
+
+ void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
+ const Address& mem, Register expected,
+ Register replacement, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp,
+ AnyRegister output) DEFINED_ON(mips_shared, loong64);
+
+ void compareExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
+ const BaseIndex& mem, Register expected,
+ Register replacement, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp,
+ AnyRegister output) DEFINED_ON(mips_shared, loong64);
+
+ void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
+ const Address& mem, Register value, Register temp,
+ AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
+ const BaseIndex& mem, Register value, Register temp,
+ AnyRegister output) DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
+ const Address& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp, Register temp,
+ AnyRegister output) DEFINED_ON(mips_shared, loong64);
+
+ void atomicExchangeJS(Scalar::Type arrayType, const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp, AnyRegister output)
+ DEFINED_ON(mips_shared, loong64);
+
+ void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Register value, const Address& mem,
+ Register temp1, Register temp2, AnyRegister output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Register value, const BaseIndex& mem,
+ Register temp1, Register temp2, AnyRegister output)
+ DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Imm32 value, const Address& mem,
+ Register temp1, Register temp2, AnyRegister output)
+ DEFINED_ON(x86_shared);
+
+ void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Imm32 value, const BaseIndex& mem,
+ Register temp1, Register temp2, AnyRegister output)
+ DEFINED_ON(x86_shared);
+
+ void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp, AnyRegister output)
+ DEFINED_ON(mips_shared, loong64);
+
+ void atomicFetchOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp, AnyRegister output)
+ DEFINED_ON(mips_shared, loong64);
+
+ void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Register value, const Address& mem,
+ Register temp) DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Register value, const BaseIndex& mem,
+ Register temp) DEFINED_ON(arm, arm64, x86_shared);
+
+ void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Imm32 value, const Address& mem,
+ Register temp) DEFINED_ON(x86_shared);
+
+ void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Imm32 value, const BaseIndex& mem,
+ Register temp) DEFINED_ON(x86_shared);
+
+ void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) DEFINED_ON(mips_shared, loong64);
+
+ void atomicEffectOpJS(Scalar::Type arrayType, const Synchronization& sync,
+ AtomicOp op, Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) DEFINED_ON(mips_shared, loong64);
+
+ void atomicIsLockFreeJS(Register value, Register output);
+
+ // ========================================================================
+ // Spectre Mitigations.
+ //
+ // Spectre attacks are side-channel attacks based on cache pollution or
+ // slow-execution of some instructions. We have multiple spectre mitigations
+ // possible:
+ //
+ // - Stop speculative executions, with memory barriers. Memory barriers
+ // force all branches depending on loads to be resolved, and thus
+ // resolve all miss-speculated paths.
+ //
+ // - Use conditional move instructions. Some CPUs have a branch predictor,
+ // and not a flag predictor. In such cases, using a conditional move
+ // instruction to zero some pointer/index is enough to add a
+ // data-dependency which prevents any futher executions until the load is
+ // resolved.
+
+ void spectreMaskIndex32(Register index, Register length, Register output);
+ void spectreMaskIndex32(Register index, const Address& length,
+ Register output);
+ void spectreMaskIndexPtr(Register index, Register length, Register output);
+ void spectreMaskIndexPtr(Register index, const Address& length,
+ Register output);
+
+ // The length must be a power of two. Performs a bounds check and Spectre
+ // index masking.
+ void boundsCheck32PowerOfTwo(Register index, uint32_t length, Label* failure);
+
+ void speculationBarrier() PER_SHARED_ARCH;
+
+ //}}} check_macroassembler_decl_style
+ public:
+ // Unsafe here means the caller is responsible for Spectre mitigations if
+ // needed. Prefer branchTestObjClass or one of the other masm helpers!
+ inline void loadObjClassUnsafe(Register obj, Register dest);
+
+ template <typename EmitPreBarrier>
+ inline void storeObjShape(Register shape, Register obj,
+ EmitPreBarrier emitPreBarrier);
+ template <typename EmitPreBarrier>
+ inline void storeObjShape(Shape* shape, Register obj,
+ EmitPreBarrier emitPreBarrier);
+
+ inline void loadObjProto(Register obj, Register dest);
+
+ inline void loadStringLength(Register str, Register dest);
+
+ void loadStringChars(Register str, Register dest, CharEncoding encoding);
+
+ void loadNonInlineStringChars(Register str, Register dest,
+ CharEncoding encoding);
+ void loadNonInlineStringCharsForStore(Register str, Register dest);
+ void storeNonInlineStringChars(Register chars, Register str);
+
+ void loadInlineStringChars(Register str, Register dest,
+ CharEncoding encoding);
+ void loadInlineStringCharsForStore(Register str, Register dest);
+
+ private:
+ void loadRopeChild(Register str, Register index, Register output,
+ Label* isLinear);
+
+ public:
+ void branchIfCanLoadStringChar(Register str, Register index, Register scratch,
+ Label* label);
+ void branchIfNotCanLoadStringChar(Register str, Register index,
+ Register scratch, Label* label);
+
+ void loadStringChar(Register str, Register index, Register output,
+ Register scratch1, Register scratch2, Label* fail);
+
+ void loadRopeLeftChild(Register str, Register dest);
+ void loadRopeRightChild(Register str, Register dest);
+ void storeRopeChildren(Register left, Register right, Register str);
+
+ void loadDependentStringBase(Register str, Register dest);
+ void storeDependentStringBase(Register base, Register str);
+
+ void loadStringIndexValue(Register str, Register dest, Label* fail);
+
+ /**
+ * Store the character in |src| to |dest|.
+ */
+ template <typename T>
+ void storeChar(const T& src, Address dest, CharEncoding encoding) {
+ if (encoding == CharEncoding::Latin1) {
+ store8(src, dest);
+ } else {
+ store16(src, dest);
+ }
+ }
+
+ /**
+ * Load the character at |src| into |dest|.
+ */
+ template <typename T>
+ void loadChar(const T& src, Register dest, CharEncoding encoding) {
+ if (encoding == CharEncoding::Latin1) {
+ load8ZeroExtend(src, dest);
+ } else {
+ load16ZeroExtend(src, dest);
+ }
+ }
+
+ /**
+ * Load the character at |chars[index + offset]| into |dest|. The optional
+ * offset argument is not scaled to the character encoding.
+ */
+ void loadChar(Register chars, Register index, Register dest,
+ CharEncoding encoding, int32_t offset = 0);
+
+ /**
+ * Add |index| to |chars| so that |chars| now points at |chars[index]|.
+ */
+ void addToCharPtr(Register chars, Register index, CharEncoding encoding);
+
+ private:
+ void loadStringFromUnit(Register unit, Register dest,
+ const StaticStrings& staticStrings);
+ void loadLengthTwoString(Register c1, Register c2, Register dest,
+ const StaticStrings& staticStrings);
+
+ public:
+ /**
+ * Load the string representation of |input| in base |base|. Jumps to |fail|
+ * when the string representation needs to be allocated dynamically.
+ */
+ void loadInt32ToStringWithBase(Register input, Register base, Register dest,
+ Register scratch1, Register scratch2,
+ const StaticStrings& staticStrings,
+ const LiveRegisterSet& volatileRegs,
+ Label* fail);
+ void loadInt32ToStringWithBase(Register input, int32_t base, Register dest,
+ Register scratch1, Register scratch2,
+ const StaticStrings& staticStrings,
+ Label* fail);
+
+ /**
+ * Load the BigInt digits from |bigInt| into |digits|.
+ */
+ void loadBigIntDigits(Register bigInt, Register digits);
+
+ /**
+ * Load the first [u]int64 value from |bigInt| into |dest|.
+ */
+ void loadBigInt64(Register bigInt, Register64 dest);
+
+ /**
+ * Load the first digit from |bigInt| into |dest|. Handles the case when the
+ * BigInt digits length is zero.
+ *
+ * Note: A BigInt digit is a pointer-sized value.
+ */
+ void loadFirstBigIntDigitOrZero(Register bigInt, Register dest);
+
+ /**
+ * Load the number stored in |bigInt| into |dest|. Handles the case when the
+ * BigInt digits length is zero. Jumps to |fail| when the number can't be
+ * saved into a single pointer-sized register.
+ */
+ void loadBigInt(Register bigInt, Register dest, Label* fail);
+
+ /**
+ * Load the number stored in |bigInt| into |dest|. Doesn't handle the case
+ * when the BigInt digits length is zero. Jumps to |fail| when the number
+ * can't be saved into a single pointer-sized register.
+ */
+ void loadBigIntNonZero(Register bigInt, Register dest, Label* fail);
+
+ /**
+ * Load the absolute number stored in |bigInt| into |dest|. Handles the case
+ * when the BigInt digits length is zero. Jumps to |fail| when the number
+ * can't be saved into a single pointer-sized register.
+ */
+ void loadBigIntAbsolute(Register bigInt, Register dest, Label* fail);
+
+ /**
+ * In-place modifies the BigInt digit to a signed pointer-sized value. Jumps
+ * to |fail| when the digit exceeds the representable range.
+ */
+ void bigIntDigitToSignedPtr(Register bigInt, Register digit, Label* fail);
+
+ /**
+ * Initialize a BigInt from |val|. Clobbers |val|!
+ */
+ void initializeBigInt64(Scalar::Type type, Register bigInt, Register64 val);
+
+ /**
+ * Initialize a BigInt from the signed, pointer-sized register |val|.
+ * Clobbers |val|!
+ */
+ void initializeBigInt(Register bigInt, Register val);
+
+ /**
+ * Initialize a BigInt from the pointer-sized register |val|.
+ */
+ void initializeBigIntAbsolute(Register bigInt, Register val);
+
+ /**
+ * Copy a BigInt. Jumps to |fail| on allocation failure or when the BigInt
+ * digits need to be heap allocated.
+ */
+ void copyBigIntWithInlineDigits(Register src, Register dest, Register temp,
+ gc::InitialHeap initialHeap, Label* fail);
+
+ /**
+ * Compare a BigInt and an Int32 value. Falls through to the false case.
+ */
+ void compareBigIntAndInt32(JSOp op, Register bigInt, Register int32,
+ Register scratch1, Register scratch2,
+ Label* ifTrue, Label* ifFalse);
+
+ /**
+ * Compare two BigInts for equality. Falls through if both BigInts are equal
+ * to each other.
+ *
+ * - When we jump to |notSameLength|, |temp1| holds the length of the right
+ * operand.
+ * - When we jump to |notSameDigit|, |temp2| points to the current digit of
+ * the left operand and |temp4| holds the current digit of the right
+ * operand.
+ */
+ void equalBigInts(Register left, Register right, Register temp1,
+ Register temp2, Register temp3, Register temp4,
+ Label* notSameSign, Label* notSameLength,
+ Label* notSameDigit);
+
+ void loadJSContext(Register dest);
+
+ void switchToRealm(Register realm);
+ void switchToRealm(const void* realm, Register scratch);
+ void switchToObjectRealm(Register obj, Register scratch);
+ void switchToBaselineFrameRealm(Register scratch);
+ void switchToWasmInstanceRealm(Register scratch1, Register scratch2);
+ void debugAssertContextRealm(const void* realm, Register scratch);
+
+ void loadJitActivation(Register dest);
+
+ void guardSpecificAtom(Register str, JSAtom* atom, Register scratch,
+ const LiveRegisterSet& volatileRegs, Label* fail);
+
+ void guardStringToInt32(Register str, Register output, Register scratch,
+ LiveRegisterSet volatileRegs, Label* fail);
+
+ template <typename T>
+ void loadTypedOrValue(const T& src, TypedOrValueRegister dest) {
+ if (dest.hasValue()) {
+ loadValue(src, dest.valueReg());
+ } else {
+ loadUnboxedValue(src, dest.type(), dest.typedReg());
+ }
+ }
+
+ template <typename T>
+ void storeTypedOrValue(TypedOrValueRegister src, const T& dest) {
+ if (src.hasValue()) {
+ storeValue(src.valueReg(), dest);
+ } else if (IsFloatingPointType(src.type())) {
+ FloatRegister reg = src.typedReg().fpu();
+ if (src.type() == MIRType::Float32) {
+ ScratchDoubleScope fpscratch(*this);
+ convertFloat32ToDouble(reg, fpscratch);
+ boxDouble(fpscratch, dest);
+ } else {
+ boxDouble(reg, dest);
+ }
+ } else {
+ storeValue(ValueTypeFromMIRType(src.type()), src.typedReg().gpr(), dest);
+ }
+ }
+
+ template <typename T>
+ void storeConstantOrRegister(const ConstantOrRegister& src, const T& dest) {
+ if (src.constant()) {
+ storeValue(src.value(), dest);
+ } else {
+ storeTypedOrValue(src.reg(), dest);
+ }
+ }
+
+ void storeCallPointerResult(Register reg) {
+ if (reg != ReturnReg) {
+ mov(ReturnReg, reg);
+ }
+ }
+
+ inline void storeCallBoolResult(Register reg);
+ inline void storeCallInt32Result(Register reg);
+
+ void storeCallFloatResult(FloatRegister reg) {
+ if (reg != ReturnDoubleReg) {
+ moveDouble(ReturnDoubleReg, reg);
+ }
+ }
+
+ inline void storeCallResultValue(AnyRegister dest, JSValueType type);
+
+ void storeCallResultValue(ValueOperand dest) {
+#if defined(JS_NUNBOX32)
+ // reshuffle the return registers used for a call result to store into
+ // dest, using ReturnReg as a scratch register if necessary. This must
+ // only be called after returning from a call, at a point when the
+ // return register is not live. XXX would be better to allow wrappers
+ // to store the return value to different places.
+ if (dest.typeReg() == JSReturnReg_Data) {
+ if (dest.payloadReg() == JSReturnReg_Type) {
+ // swap the two registers.
+ mov(JSReturnReg_Type, ReturnReg);
+ mov(JSReturnReg_Data, JSReturnReg_Type);
+ mov(ReturnReg, JSReturnReg_Data);
+ } else {
+ mov(JSReturnReg_Data, dest.payloadReg());
+ mov(JSReturnReg_Type, dest.typeReg());
+ }
+ } else {
+ mov(JSReturnReg_Type, dest.typeReg());
+ mov(JSReturnReg_Data, dest.payloadReg());
+ }
+#elif defined(JS_PUNBOX64)
+ if (dest.valueReg() != JSReturnReg) {
+ mov(JSReturnReg, dest.valueReg());
+ }
+#else
+# error "Bad architecture"
+#endif
+ }
+
+ inline void storeCallResultValue(TypedOrValueRegister dest);
+
+ private:
+ TrampolinePtr preBarrierTrampoline(MIRType type);
+
+ template <typename T>
+ void unguardedCallPreBarrier(const T& address, MIRType type) {
+ Label done;
+ if (type == MIRType::Value) {
+ branchTestGCThing(Assembler::NotEqual, address, &done);
+ } else if (type == MIRType::Object || type == MIRType::String) {
+ branchPtr(Assembler::Equal, address, ImmWord(0), &done);
+ }
+
+ Push(PreBarrierReg);
+ computeEffectiveAddress(address, PreBarrierReg);
+
+ TrampolinePtr preBarrier = preBarrierTrampoline(type);
+
+ call(preBarrier);
+ Pop(PreBarrierReg);
+ // On arm64, SP may be < PSP now (that's OK).
+ // eg testcase: tests/auto-regress/bug702915.js
+ bind(&done);
+ }
+
+ public:
+ template <typename T>
+ void guardedCallPreBarrier(const T& address, MIRType type) {
+ Label done;
+ branchTestNeedsIncrementalBarrier(Assembler::Zero, &done);
+ unguardedCallPreBarrier(address, type);
+ bind(&done);
+ }
+
+ // Like guardedCallPreBarrier, but unlike guardedCallPreBarrier this can be
+ // called from runtime-wide trampolines because it loads cx->zone (instead of
+ // baking in the current Zone) if JitContext::realm is nullptr.
+ template <typename T>
+ void guardedCallPreBarrierAnyZone(const T& address, MIRType type,
+ Register scratch) {
+ Label done;
+ branchTestNeedsIncrementalBarrierAnyZone(Assembler::Zero, &done, scratch);
+ unguardedCallPreBarrier(address, type);
+ bind(&done);
+ }
+
+ enum class Uint32Mode { FailOnDouble, ForceDouble };
+
+ void boxUint32(Register source, ValueOperand dest, Uint32Mode uint32Mode,
+ Label* fail);
+
+ template <typename T>
+ void loadFromTypedArray(Scalar::Type arrayType, const T& src,
+ AnyRegister dest, Register temp, Label* fail);
+
+ template <typename T>
+ void loadFromTypedArray(Scalar::Type arrayType, const T& src,
+ const ValueOperand& dest, Uint32Mode uint32Mode,
+ Register temp, Label* fail);
+
+ template <typename T>
+ void loadFromTypedBigIntArray(Scalar::Type arrayType, const T& src,
+ Register bigInt, Register64 temp);
+
+ template <typename S, typename T>
+ void storeToTypedIntArray(Scalar::Type arrayType, const S& value,
+ const T& dest) {
+ switch (arrayType) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ store8(value, dest);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ store16(value, dest);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ store32(value, dest);
+ break;
+ default:
+ MOZ_CRASH("Invalid typed array type");
+ }
+ }
+
+ void storeToTypedFloatArray(Scalar::Type arrayType, FloatRegister value,
+ const BaseIndex& dest);
+ void storeToTypedFloatArray(Scalar::Type arrayType, FloatRegister value,
+ const Address& dest);
+
+ void storeToTypedBigIntArray(Scalar::Type arrayType, Register64 value,
+ const BaseIndex& dest);
+ void storeToTypedBigIntArray(Scalar::Type arrayType, Register64 value,
+ const Address& dest);
+
+ void memoryBarrierBefore(const Synchronization& sync);
+ void memoryBarrierAfter(const Synchronization& sync);
+
+ void debugAssertIsObject(const ValueOperand& val);
+ void debugAssertObjHasFixedSlots(Register obj, Register scratch);
+
+ void debugAssertObjectHasClass(Register obj, Register scratch,
+ const JSClass* clasp);
+
+ void branchArrayIsNotPacked(Register array, Register temp1, Register temp2,
+ Label* label);
+
+ void setIsPackedArray(Register obj, Register output, Register temp);
+
+ void packedArrayPop(Register array, ValueOperand output, Register temp1,
+ Register temp2, Label* fail);
+ void packedArrayShift(Register array, ValueOperand output, Register temp1,
+ Register temp2, LiveRegisterSet volatileRegs,
+ Label* fail);
+
+ void loadArgumentsObjectElement(Register obj, Register index,
+ ValueOperand output, Register temp,
+ Label* fail);
+ void loadArgumentsObjectElementHole(Register obj, Register index,
+ ValueOperand output, Register temp,
+ Label* fail);
+ void loadArgumentsObjectElementExists(Register obj, Register index,
+ Register output, Register temp,
+ Label* fail);
+
+ void loadArgumentsObjectLength(Register obj, Register output, Label* fail);
+
+ void branchTestArgumentsObjectFlags(Register obj, Register temp,
+ uint32_t flags, Condition cond,
+ Label* label);
+
+ void typedArrayElementSize(Register obj, Register output);
+ void branchIfClassIsNotTypedArray(Register clasp, Label* notTypedArray);
+
+ void branchIfHasDetachedArrayBuffer(Register obj, Register temp,
+ Label* label);
+
+ void branchIfNativeIteratorNotReusable(Register ni, Label* notReusable);
+
+ void maybeLoadIteratorFromShape(Register obj, Register dest, Register temp,
+ Register temp2, Register temp3,
+ Label* failure);
+
+ void iteratorMore(Register obj, ValueOperand output, Register temp);
+ void iteratorClose(Register obj, Register temp1, Register temp2,
+ Register temp3);
+ void registerIterator(Register enumeratorsList, Register iter, Register temp);
+
+ void toHashableNonGCThing(ValueOperand value, ValueOperand result,
+ FloatRegister tempFloat);
+
+ void toHashableValue(ValueOperand value, ValueOperand result,
+ FloatRegister tempFloat, Label* atomizeString,
+ Label* tagString);
+
+ private:
+ void scrambleHashCode(Register result);
+
+ public:
+ void prepareHashNonGCThing(ValueOperand value, Register result,
+ Register temp);
+ void prepareHashString(Register str, Register result, Register temp);
+ void prepareHashSymbol(Register sym, Register result);
+ void prepareHashBigInt(Register bigInt, Register result, Register temp1,
+ Register temp2, Register temp3);
+ void prepareHashObject(Register setObj, ValueOperand value, Register result,
+ Register temp1, Register temp2, Register temp3,
+ Register temp4);
+ void prepareHashValue(Register setObj, ValueOperand value, Register result,
+ Register temp1, Register temp2, Register temp3,
+ Register temp4);
+
+ private:
+ enum class IsBigInt { No, Yes, Maybe };
+
+ /**
+ * Search for a value in a OrderedHashTable.
+ *
+ * When we jump to |found|, |entryTemp| holds the found hashtable entry.
+ */
+ template <typename OrderedHashTable>
+ void orderedHashTableLookup(Register setOrMapObj, ValueOperand value,
+ Register hash, Register entryTemp, Register temp1,
+ Register temp3, Register temp4, Register temp5,
+ Label* found, IsBigInt isBigInt);
+
+ void setObjectHas(Register setObj, ValueOperand value, Register hash,
+ Register result, Register temp1, Register temp2,
+ Register temp3, Register temp4, IsBigInt isBigInt);
+
+ void mapObjectHas(Register mapObj, ValueOperand value, Register hash,
+ Register result, Register temp1, Register temp2,
+ Register temp3, Register temp4, IsBigInt isBigInt);
+
+ void mapObjectGet(Register mapObj, ValueOperand value, Register hash,
+ ValueOperand result, Register temp1, Register temp2,
+ Register temp3, Register temp4, Register temp5,
+ IsBigInt isBigInt);
+
+ public:
+ void setObjectHasNonBigInt(Register setObj, ValueOperand value, Register hash,
+ Register result, Register temp1, Register temp2) {
+ return setObjectHas(setObj, value, hash, result, temp1, temp2, InvalidReg,
+ InvalidReg, IsBigInt::No);
+ }
+ void setObjectHasBigInt(Register setObj, ValueOperand value, Register hash,
+ Register result, Register temp1, Register temp2,
+ Register temp3, Register temp4) {
+ return setObjectHas(setObj, value, hash, result, temp1, temp2, temp3, temp4,
+ IsBigInt::Yes);
+ }
+ void setObjectHasValue(Register setObj, ValueOperand value, Register hash,
+ Register result, Register temp1, Register temp2,
+ Register temp3, Register temp4) {
+ return setObjectHas(setObj, value, hash, result, temp1, temp2, temp3, temp4,
+ IsBigInt::Maybe);
+ }
+
+ void mapObjectHasNonBigInt(Register mapObj, ValueOperand value, Register hash,
+ Register result, Register temp1, Register temp2) {
+ return mapObjectHas(mapObj, value, hash, result, temp1, temp2, InvalidReg,
+ InvalidReg, IsBigInt::No);
+ }
+ void mapObjectHasBigInt(Register mapObj, ValueOperand value, Register hash,
+ Register result, Register temp1, Register temp2,
+ Register temp3, Register temp4) {
+ return mapObjectHas(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
+ IsBigInt::Yes);
+ }
+ void mapObjectHasValue(Register mapObj, ValueOperand value, Register hash,
+ Register result, Register temp1, Register temp2,
+ Register temp3, Register temp4) {
+ return mapObjectHas(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
+ IsBigInt::Maybe);
+ }
+
+ void mapObjectGetNonBigInt(Register mapObj, ValueOperand value, Register hash,
+ ValueOperand result, Register temp1,
+ Register temp2, Register temp3) {
+ return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3,
+ InvalidReg, InvalidReg, IsBigInt::No);
+ }
+ void mapObjectGetBigInt(Register mapObj, ValueOperand value, Register hash,
+ ValueOperand result, Register temp1, Register temp2,
+ Register temp3, Register temp4, Register temp5) {
+ return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
+ temp5, IsBigInt::Yes);
+ }
+ void mapObjectGetValue(Register mapObj, ValueOperand value, Register hash,
+ ValueOperand result, Register temp1, Register temp2,
+ Register temp3, Register temp4, Register temp5) {
+ return mapObjectGet(mapObj, value, hash, result, temp1, temp2, temp3, temp4,
+ temp5, IsBigInt::Maybe);
+ }
+
+ private:
+ template <typename OrderedHashTable>
+ void loadOrderedHashTableCount(Register setOrMapObj, Register result);
+
+ public:
+ void loadSetObjectSize(Register setObj, Register result);
+ void loadMapObjectSize(Register mapObj, Register result);
+
+ // Inline version of js_TypedArray_uint8_clamp_double.
+ // This function clobbers the input register.
+ void clampDoubleToUint8(FloatRegister input, Register output) PER_ARCH;
+
+ using MacroAssemblerSpecific::ensureDouble;
+
+ template <typename S>
+ void ensureDouble(const S& source, FloatRegister dest, Label* failure) {
+ Label isDouble, done;
+ branchTestDouble(Assembler::Equal, source, &isDouble);
+ branchTestInt32(Assembler::NotEqual, source, failure);
+
+ convertInt32ToDouble(source, dest);
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+ }
+
+ // Inline allocation.
+ private:
+ void checkAllocatorState(Label* fail);
+ bool shouldNurseryAllocate(gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap);
+ void nurseryAllocateObject(
+ Register result, Register temp, gc::AllocKind allocKind,
+ size_t nDynamicSlots, Label* fail,
+ const AllocSiteInput& allocSite = AllocSiteInput());
+ void bumpPointerAllocate(Register result, Register temp, Label* fail,
+ CompileZone* zone, void* posAddr,
+ const void* curEddAddr, JS::TraceKind traceKind,
+ uint32_t size,
+ const AllocSiteInput& allocSite = AllocSiteInput());
+ void updateAllocSite(Register temp, Register result, CompileZone* zone,
+ Register site);
+
+ void freeListAllocate(Register result, Register temp, gc::AllocKind allocKind,
+ Label* fail);
+ void allocateObject(Register result, Register temp, gc::AllocKind allocKind,
+ uint32_t nDynamicSlots, gc::InitialHeap initialHeap,
+ Label* fail,
+ const AllocSiteInput& allocSite = AllocSiteInput());
+ void nurseryAllocateString(Register result, Register temp,
+ gc::AllocKind allocKind, Label* fail);
+ void allocateString(Register result, Register temp, gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap, Label* fail);
+ void nurseryAllocateBigInt(Register result, Register temp, Label* fail);
+ void copySlotsFromTemplate(Register obj,
+ const TemplateNativeObject& templateObj,
+ uint32_t start, uint32_t end);
+ void fillSlotsWithConstantValue(Address addr, Register temp, uint32_t start,
+ uint32_t end, const Value& v);
+ void fillSlotsWithUndefined(Address addr, Register temp, uint32_t start,
+ uint32_t end);
+ void fillSlotsWithUninitialized(Address addr, Register temp, uint32_t start,
+ uint32_t end);
+
+ void initGCSlots(Register obj, Register temp,
+ const TemplateNativeObject& templateObj);
+
+ public:
+ void callFreeStub(Register slots);
+ void createGCObject(Register result, Register temp,
+ const TemplateObject& templateObj,
+ gc::InitialHeap initialHeap, Label* fail,
+ bool initContents = true);
+
+ void createPlainGCObject(Register result, Register shape, Register temp,
+ Register temp2, uint32_t numFixedSlots,
+ uint32_t numDynamicSlots, gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap, Label* fail,
+ const AllocSiteInput& allocSite,
+ bool initContents = true);
+
+ void createArrayWithFixedElements(
+ Register result, Register shape, Register temp, uint32_t arrayLength,
+ uint32_t arrayCapacity, gc::AllocKind allocKind,
+ gc::InitialHeap initialHeap, Label* fail,
+ const AllocSiteInput& allocSite = AllocSiteInput());
+
+ void initGCThing(Register obj, Register temp,
+ const TemplateObject& templateObj, bool initContents = true);
+
+ enum class TypedArrayLength { Fixed, Dynamic };
+
+ void initTypedArraySlots(Register obj, Register temp, Register lengthReg,
+ LiveRegisterSet liveRegs, Label* fail,
+ TypedArrayObject* templateObj,
+ TypedArrayLength lengthKind);
+
+ void newGCString(Register result, Register temp, gc::InitialHeap initialHeap,
+ Label* fail);
+ void newGCFatInlineString(Register result, Register temp,
+ gc::InitialHeap initialHeap, Label* fail);
+
+ void newGCBigInt(Register result, Register temp, gc::InitialHeap initialHeap,
+ Label* fail);
+
+ // Compares two strings for equality based on the JSOP.
+ // This checks for identical pointers, atoms and length and fails for
+ // everything else.
+ void compareStrings(JSOp op, Register left, Register right, Register result,
+ Label* fail);
+
+ // Result of the typeof operation. Falls back to slow-path for proxies.
+ void typeOfObject(Register objReg, Register scratch, Label* slow,
+ Label* isObject, Label* isCallable, Label* isUndefined);
+
+ // Implementation of IsCallable. Doesn't handle proxies.
+ void isCallable(Register obj, Register output, Label* isProxy) {
+ isCallableOrConstructor(true, obj, output, isProxy);
+ }
+ void isConstructor(Register obj, Register output, Label* isProxy) {
+ isCallableOrConstructor(false, obj, output, isProxy);
+ }
+
+ void setIsCrossRealmArrayConstructor(Register obj, Register output);
+
+ void setIsDefinitelyTypedArrayConstructor(Register obj, Register output);
+
+ void loadMegamorphicCache(Register dest);
+ void loadStringToAtomCacheLastLookups(Register dest);
+
+ void loadAtomOrSymbolAndHash(ValueOperand value, Register outId,
+ Register outHash, Label* cacheMiss);
+
+ void emitExtractValueFromMegamorphicCacheEntry(
+ Register obj, Register entry, Register scratch1, Register scratch2,
+ ValueOperand output, Label* cacheHit, Label* cacheMiss);
+
+ void emitMegamorphicCacheLookupByValueCommon(ValueOperand id, Register obj,
+ Register scratch1,
+ Register scratch2,
+ Register outEntryPtr,
+ Label* cacheMiss);
+
+ void emitMegamorphicCacheLookup(PropertyKey id, Register obj,
+ Register scratch1, Register scratch2,
+ Register scratch3, ValueOperand output,
+ Label* cacheHit);
+
+ void emitMegamorphicCacheLookupByValue(ValueOperand id, Register obj,
+ Register scratch1, Register scratch2,
+ Register scratch3, ValueOperand output,
+ Label* cacheHit);
+
+ void emitMegamorphicCacheLookupExists(ValueOperand id, Register obj,
+ Register scratch1, Register scratch2,
+ Register scratch3, Register output,
+ Label* cacheHit, bool hasOwn);
+
+ void loadDOMExpandoValueGuardGeneration(
+ Register obj, ValueOperand output,
+ JS::ExpandoAndGeneration* expandoAndGeneration, uint64_t generation,
+ Label* fail);
+
+ void guardNonNegativeIntPtrToInt32(Register reg, Label* fail);
+
+ void loadArrayBufferByteLengthIntPtr(Register obj, Register output);
+ void loadArrayBufferViewByteOffsetIntPtr(Register obj, Register output);
+ void loadArrayBufferViewLengthIntPtr(Register obj, Register output);
+
+ private:
+ void isCallableOrConstructor(bool isCallable, Register obj, Register output,
+ Label* isProxy);
+
+ public:
+ // Generates code used to complete a bailout.
+ void generateBailoutTail(Register scratch, Register bailoutInfo);
+
+ public:
+#ifndef JS_CODEGEN_ARM64
+ // StackPointer manipulation functions.
+ // On ARM64, the StackPointer is implemented as two synchronized registers.
+ // Code shared across platforms must use these functions to be valid.
+ template <typename T>
+ inline void addToStackPtr(T t);
+ template <typename T>
+ inline void addStackPtrTo(T t);
+
+ void subFromStackPtr(Imm32 imm32)
+ DEFINED_ON(mips32, mips64, loong64, wasm32, arm, x86, x64);
+ void subFromStackPtr(Register reg);
+
+ template <typename T>
+ void subStackPtrFrom(T t) {
+ subPtr(getStackPointer(), t);
+ }
+
+ template <typename T>
+ void andToStackPtr(T t) {
+ andPtr(t, getStackPointer());
+ }
+
+ template <typename T>
+ void moveToStackPtr(T t) {
+ movePtr(t, getStackPointer());
+ }
+ template <typename T>
+ void moveStackPtrTo(T t) {
+ movePtr(getStackPointer(), t);
+ }
+
+ template <typename T>
+ void loadStackPtr(T t) {
+ loadPtr(t, getStackPointer());
+ }
+ template <typename T>
+ void storeStackPtr(T t) {
+ storePtr(getStackPointer(), t);
+ }
+
+ // StackPointer testing functions.
+ // On ARM64, sp can function as the zero register depending on context.
+ // Code shared across platforms must use these functions to be valid.
+ template <typename T>
+ inline void branchTestStackPtr(Condition cond, T t, Label* label);
+ template <typename T>
+ inline void branchStackPtr(Condition cond, T rhs, Label* label);
+ template <typename T>
+ inline void branchStackPtrRhs(Condition cond, T lhs, Label* label);
+
+ // Move the stack pointer based on the requested amount.
+ inline void reserveStack(uint32_t amount);
+#else // !JS_CODEGEN_ARM64
+ void reserveStack(uint32_t amount);
+#endif
+
+ public:
+ void enableProfilingInstrumentation() {
+ emitProfilingInstrumentation_ = true;
+ }
+
+ private:
+ // This class is used to surround call sites throughout the assembler. This
+ // is used by callWithABI, and callJit functions, except if suffixed by
+ // NoProfiler.
+ class MOZ_RAII AutoProfilerCallInstrumentation {
+ public:
+ explicit AutoProfilerCallInstrumentation(MacroAssembler& masm);
+ ~AutoProfilerCallInstrumentation() = default;
+ };
+ friend class AutoProfilerCallInstrumentation;
+
+ void appendProfilerCallSite(CodeOffset label) {
+ propagateOOM(profilerCallSites_.append(label));
+ }
+
+ // Fix up the code pointers to be written for locations where profilerCallSite
+ // emitted moves of RIP to a register.
+ void linkProfilerCallSites(JitCode* code);
+
+ // This field is used to manage profiling instrumentation output. If
+ // provided and enabled, then instrumentation will be emitted around call
+ // sites.
+ bool emitProfilingInstrumentation_;
+
+ // Record locations of the call sites.
+ Vector<CodeOffset, 0, SystemAllocPolicy> profilerCallSites_;
+
+ public:
+ void loadJitCodeRaw(Register func, Register dest);
+ void loadBaselineJitCodeRaw(Register func, Register dest,
+ Label* failure = nullptr);
+ void storeICScriptInJSContext(Register icScript);
+
+ void loadBaselineFramePtr(Register framePtr, Register dest);
+
+ void pushBaselineFramePtr(Register framePtr, Register scratch) {
+ loadBaselineFramePtr(framePtr, scratch);
+ push(scratch);
+ }
+
+ void PushBaselineFramePtr(Register framePtr, Register scratch) {
+ loadBaselineFramePtr(framePtr, scratch);
+ Push(scratch);
+ }
+
+ using MacroAssemblerSpecific::movePtr;
+
+ void movePtr(TrampolinePtr ptr, Register dest) {
+ movePtr(ImmPtr(ptr.value), dest);
+ }
+
+ private:
+ void handleFailure();
+
+ public:
+ Label* exceptionLabel() {
+ // Exceptions are currently handled the same way as sequential failures.
+ return &failureLabel_;
+ }
+
+ Label* failureLabel() { return &failureLabel_; }
+
+ void finish();
+ void link(JitCode* code);
+
+ void assumeUnreachable(const char* output);
+
+ void printf(const char* output);
+ void printf(const char* output, Register value);
+
+#define DISPATCH_FLOATING_POINT_OP(method, type, arg1d, arg1f, arg2) \
+ MOZ_ASSERT(IsFloatingPointType(type)); \
+ if (type == MIRType::Double) \
+ method##Double(arg1d, arg2); \
+ else \
+ method##Float32(arg1f, arg2);
+
+ void loadConstantFloatingPoint(double d, float f, FloatRegister dest,
+ MIRType destType) {
+ DISPATCH_FLOATING_POINT_OP(loadConstant, destType, d, f, dest);
+ }
+ void boolValueToFloatingPoint(ValueOperand value, FloatRegister dest,
+ MIRType destType) {
+ DISPATCH_FLOATING_POINT_OP(boolValueTo, destType, value, value, dest);
+ }
+ void int32ValueToFloatingPoint(ValueOperand value, FloatRegister dest,
+ MIRType destType) {
+ DISPATCH_FLOATING_POINT_OP(int32ValueTo, destType, value, value, dest);
+ }
+ void convertInt32ToFloatingPoint(Register src, FloatRegister dest,
+ MIRType destType) {
+ DISPATCH_FLOATING_POINT_OP(convertInt32To, destType, src, src, dest);
+ }
+
+#undef DISPATCH_FLOATING_POINT_OP
+
+ void convertValueToFloatingPoint(ValueOperand value, FloatRegister output,
+ Label* fail, MIRType outputType);
+
+ void outOfLineTruncateSlow(FloatRegister src, Register dest,
+ bool widenFloatToDouble, bool compilingWasm,
+ wasm::BytecodeOffset callOffset);
+
+ void convertInt32ValueToDouble(ValueOperand val);
+
+ void convertValueToDouble(ValueOperand value, FloatRegister output,
+ Label* fail) {
+ convertValueToFloatingPoint(value, output, fail, MIRType::Double);
+ }
+
+ void convertValueToFloat(ValueOperand value, FloatRegister output,
+ Label* fail) {
+ convertValueToFloatingPoint(value, output, fail, MIRType::Float32);
+ }
+
+ //
+ // Functions for converting values to int.
+ //
+ void convertDoubleToInt(FloatRegister src, Register output,
+ FloatRegister temp, Label* truncateFail, Label* fail,
+ IntConversionBehavior behavior);
+
+ // Strings may be handled by providing labels to jump to when the behavior
+ // is truncation or clamping. The subroutine, usually an OOL call, is
+ // passed the unboxed string in |stringReg| and should convert it to a
+ // double store into |temp|.
+ void convertValueToInt(
+ ValueOperand value, Label* handleStringEntry, Label* handleStringRejoin,
+ Label* truncateDoubleSlow, Register stringReg, FloatRegister temp,
+ Register output, Label* fail, IntConversionBehavior behavior,
+ IntConversionInputKind conversion = IntConversionInputKind::Any);
+
+ // This carries over the MToNumberInt32 operation on the ValueOperand
+ // input; see comment at the top of this class.
+ void convertValueToInt32(
+ ValueOperand value, FloatRegister temp, Register output, Label* fail,
+ bool negativeZeroCheck,
+ IntConversionInputKind conversion = IntConversionInputKind::Any) {
+ convertValueToInt(
+ value, nullptr, nullptr, nullptr, InvalidReg, temp, output, fail,
+ negativeZeroCheck ? IntConversionBehavior::NegativeZeroCheck
+ : IntConversionBehavior::Normal,
+ conversion);
+ }
+
+ // This carries over the MTruncateToInt32 operation on the ValueOperand
+ // input; see the comment at the top of this class.
+ void truncateValueToInt32(ValueOperand value, Label* handleStringEntry,
+ Label* handleStringRejoin,
+ Label* truncateDoubleSlow, Register stringReg,
+ FloatRegister temp, Register output, Label* fail) {
+ convertValueToInt(value, handleStringEntry, handleStringRejoin,
+ truncateDoubleSlow, stringReg, temp, output, fail,
+ IntConversionBehavior::Truncate);
+ }
+
+ void truncateValueToInt32(ValueOperand value, FloatRegister temp,
+ Register output, Label* fail) {
+ truncateValueToInt32(value, nullptr, nullptr, nullptr, InvalidReg, temp,
+ output, fail);
+ }
+
+ // Truncates, i.e. removes any fractional parts, but doesn't wrap around to
+ // the int32 range.
+ void truncateNoWrapValueToInt32(ValueOperand value, FloatRegister temp,
+ Register output, Label* truncateDoubleSlow,
+ Label* fail) {
+ convertValueToInt(value, nullptr, nullptr, truncateDoubleSlow, InvalidReg,
+ temp, output, fail,
+ IntConversionBehavior::TruncateNoWrap);
+ }
+
+ // Convenience functions for clamping values to uint8.
+ void clampValueToUint8(ValueOperand value, Label* handleStringEntry,
+ Label* handleStringRejoin, Register stringReg,
+ FloatRegister temp, Register output, Label* fail) {
+ convertValueToInt(value, handleStringEntry, handleStringRejoin, nullptr,
+ stringReg, temp, output, fail,
+ IntConversionBehavior::ClampToUint8);
+ }
+
+ [[nodiscard]] bool icBuildOOLFakeExitFrame(void* fakeReturnAddr,
+ AutoSaveLiveRegisters& save);
+
+ // Align the stack pointer based on the number of arguments which are pushed
+ // on the stack, such that the JitFrameLayout would be correctly aligned on
+ // the JitStackAlignment.
+ void alignJitStackBasedOnNArgs(Register nargs, bool countIncludesThis);
+ void alignJitStackBasedOnNArgs(uint32_t argc, bool countIncludesThis);
+
+ inline void assertStackAlignment(uint32_t alignment, int32_t offset = 0);
+
+ void touchFrameValues(Register numStackValues, Register scratch1,
+ Register scratch2);
+
+#ifdef JS_64BIT
+ // See comment block "64-bit GPRs carrying 32-bit values" above. This asserts
+ // that the high bits of the register are appropriate for the architecture and
+ // the value in the low bits.
+ void debugAssertCanonicalInt32(Register r);
+#endif
+};
+
+// StackMacroAssembler checks no GC will happen while it's on the stack.
+class MOZ_RAII StackMacroAssembler : public MacroAssembler {
+ JS::AutoCheckCannotGC nogc;
+
+ public:
+ StackMacroAssembler(JSContext* cx, TempAllocator& alloc);
+};
+
+// WasmMacroAssembler does not contain GC pointers, so it doesn't need the no-GC
+// checking StackMacroAssembler has.
+class MOZ_RAII WasmMacroAssembler : public MacroAssembler {
+ public:
+ explicit WasmMacroAssembler(TempAllocator& alloc, bool limitedSize = true);
+ explicit WasmMacroAssembler(TempAllocator& alloc,
+ const wasm::ModuleEnvironment& env,
+ bool limitedSize = true);
+ ~WasmMacroAssembler() { assertNoGCThings(); }
+};
+
+// Heap-allocated MacroAssembler used for Ion off-thread code generation.
+// GC cancels off-thread compilations.
+class IonHeapMacroAssembler : public MacroAssembler {
+ public:
+ IonHeapMacroAssembler(TempAllocator& alloc, CompileRealm* realm);
+};
+
+//{{{ check_macroassembler_style
+inline uint32_t MacroAssembler::framePushed() const { return framePushed_; }
+
+inline void MacroAssembler::setFramePushed(uint32_t framePushed) {
+ framePushed_ = framePushed;
+}
+
+inline void MacroAssembler::adjustFrame(int32_t value) {
+ MOZ_ASSERT_IF(value < 0, framePushed_ >= uint32_t(-value));
+ setFramePushed(framePushed_ + value);
+}
+
+inline void MacroAssembler::implicitPop(uint32_t bytes) {
+ MOZ_ASSERT(bytes % sizeof(intptr_t) == 0);
+ MOZ_ASSERT(bytes <= INT32_MAX);
+ adjustFrame(-int32_t(bytes));
+}
+//}}} check_macroassembler_style
+
+static inline Assembler::DoubleCondition JSOpToDoubleCondition(JSOp op) {
+ switch (op) {
+ case JSOp::Eq:
+ case JSOp::StrictEq:
+ return Assembler::DoubleEqual;
+ case JSOp::Ne:
+ case JSOp::StrictNe:
+ return Assembler::DoubleNotEqualOrUnordered;
+ case JSOp::Lt:
+ return Assembler::DoubleLessThan;
+ case JSOp::Le:
+ return Assembler::DoubleLessThanOrEqual;
+ case JSOp::Gt:
+ return Assembler::DoubleGreaterThan;
+ case JSOp::Ge:
+ return Assembler::DoubleGreaterThanOrEqual;
+ default:
+ MOZ_CRASH("Unexpected comparison operation");
+ }
+}
+
+// Note: the op may have been inverted during lowering (to put constants in a
+// position where they can be immediates), so it is important to use the
+// lir->jsop() instead of the mir->jsop() when it is present.
+static inline Assembler::Condition JSOpToCondition(JSOp op, bool isSigned) {
+ if (isSigned) {
+ switch (op) {
+ case JSOp::Eq:
+ case JSOp::StrictEq:
+ return Assembler::Equal;
+ case JSOp::Ne:
+ case JSOp::StrictNe:
+ return Assembler::NotEqual;
+ case JSOp::Lt:
+ return Assembler::LessThan;
+ case JSOp::Le:
+ return Assembler::LessThanOrEqual;
+ case JSOp::Gt:
+ return Assembler::GreaterThan;
+ case JSOp::Ge:
+ return Assembler::GreaterThanOrEqual;
+ default:
+ MOZ_CRASH("Unrecognized comparison operation");
+ }
+ } else {
+ switch (op) {
+ case JSOp::Eq:
+ case JSOp::StrictEq:
+ return Assembler::Equal;
+ case JSOp::Ne:
+ case JSOp::StrictNe:
+ return Assembler::NotEqual;
+ case JSOp::Lt:
+ return Assembler::Below;
+ case JSOp::Le:
+ return Assembler::BelowOrEqual;
+ case JSOp::Gt:
+ return Assembler::Above;
+ case JSOp::Ge:
+ return Assembler::AboveOrEqual;
+ default:
+ MOZ_CRASH("Unrecognized comparison operation");
+ }
+ }
+}
+
+static inline size_t StackDecrementForCall(uint32_t alignment,
+ size_t bytesAlreadyPushed,
+ size_t bytesToPush) {
+ return bytesToPush +
+ ComputeByteAlignment(bytesAlreadyPushed + bytesToPush, alignment);
+}
+
+// Helper for generatePreBarrier.
+inline DynFn JitPreWriteBarrier(MIRType type);
+} // namespace jit
+
+} // namespace js
+
+#endif /* jit_MacroAssembler_h */
diff --git a/js/src/jit/MoveEmitter.h b/js/src/jit/MoveEmitter.h
new file mode 100644
index 0000000000..a661fd75b7
--- /dev/null
+++ b/js/src/jit/MoveEmitter.h
@@ -0,0 +1,30 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MoveEmitter_h
+#define jit_MoveEmitter_h
+
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+# include "jit/x86-shared/MoveEmitter-x86-shared.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/MoveEmitter-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/MoveEmitter-arm64.h"
+#elif defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/MoveEmitter-mips32.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/MoveEmitter-mips64.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/MoveEmitter-loong64.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/MoveEmitter-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/MoveEmitter-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+#endif /* jit_MoveEmitter_h */
diff --git a/js/src/jit/MoveResolver.cpp b/js/src/jit/MoveResolver.cpp
new file mode 100644
index 0000000000..2d47b32c35
--- /dev/null
+++ b/js/src/jit/MoveResolver.cpp
@@ -0,0 +1,443 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/MoveResolver.h"
+
+#include "mozilla/ScopeExit.h"
+
+#include "jit/MacroAssembler.h"
+#include "jit/RegisterSets.h"
+
+using namespace js;
+using namespace js::jit;
+
+MoveOperand::MoveOperand(MacroAssembler& masm, const ABIArg& arg) : disp_(0) {
+ switch (arg.kind()) {
+ case ABIArg::GPR:
+ kind_ = REG;
+ code_ = arg.gpr().code();
+ break;
+#ifdef JS_CODEGEN_REGISTER_PAIR
+ case ABIArg::GPR_PAIR:
+ kind_ = REG_PAIR;
+ code_ = arg.evenGpr().code();
+ MOZ_ASSERT(code_ % 2 == 0);
+ MOZ_ASSERT(code_ + 1 == arg.oddGpr().code());
+ break;
+#endif
+ case ABIArg::FPU:
+ kind_ = FLOAT_REG;
+ code_ = arg.fpu().code();
+ break;
+ case ABIArg::Stack:
+ kind_ = MEMORY;
+ if (IsHiddenSP(masm.getStackPointer())) {
+ MOZ_CRASH(
+ "Hidden SP cannot be represented as register code on this "
+ "platform");
+ } else {
+ code_ = AsRegister(masm.getStackPointer()).code();
+ }
+ disp_ = arg.offsetFromArgBase();
+ break;
+ case ABIArg::Uninitialized:
+ MOZ_CRASH("Uninitialized ABIArg kind");
+ }
+}
+
+MoveResolver::MoveResolver() : numCycles_(0), curCycles_(0) {}
+
+void MoveResolver::resetState() {
+ numCycles_ = 0;
+ curCycles_ = 0;
+}
+
+bool MoveResolver::addMove(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type) {
+ // Assert that we're not doing no-op moves.
+ MOZ_ASSERT(!(from == to));
+ PendingMove* pm = movePool_.allocate(from, to, type);
+ if (!pm) {
+ return false;
+ }
+ pending_.pushBack(pm);
+ return true;
+}
+
+// Given move (A -> B), this function attempts to find any move (B -> *) in the
+// pending move list, and returns the first one.
+MoveResolver::PendingMove* MoveResolver::findBlockingMove(
+ const PendingMove* last) {
+ for (PendingMoveIterator iter = pending_.begin(); iter != pending_.end();
+ iter++) {
+ PendingMove* other = *iter;
+
+ if (other->from().aliases(last->to())) {
+ // We now have pairs in the form (A -> X) (X -> y). The second pair
+ // blocks the move in the first pair, so return it.
+ return other;
+ }
+ }
+
+ // No blocking moves found.
+ return nullptr;
+}
+
+// Given move (A -> B), this function attempts to find any move (B -> *) in the
+// move list iterator, and returns the first one.
+// N.B. It is unclear if a single move can complete more than one cycle, so to
+// be conservative, this function operates on iterators, so the caller can
+// process all instructions that start a cycle.
+MoveResolver::PendingMove* MoveResolver::findCycledMove(
+ PendingMoveIterator* iter, PendingMoveIterator end,
+ const PendingMove* last) {
+ for (; *iter != end; (*iter)++) {
+ PendingMove* other = **iter;
+ if (other->from().aliases(last->to())) {
+ // We now have pairs in the form (A -> X) (X -> y). The second pair
+ // blocks the move in the first pair, so return it.
+ (*iter)++;
+ return other;
+ }
+ }
+ // No blocking moves found.
+ return nullptr;
+}
+
+#ifdef JS_CODEGEN_ARM
+static inline bool MoveIsDouble(const MoveOperand& move) {
+ if (!move.isFloatReg()) {
+ return false;
+ }
+ return move.floatReg().isDouble();
+}
+#endif
+
+#ifdef JS_CODEGEN_ARM
+static inline bool MoveIsSingle(const MoveOperand& move) {
+ if (!move.isFloatReg()) {
+ return false;
+ }
+ return move.floatReg().isSingle();
+}
+#endif
+
+#ifdef JS_CODEGEN_ARM
+bool MoveResolver::isDoubleAliasedAsSingle(const MoveOperand& move) {
+ if (!MoveIsDouble(move)) {
+ return false;
+ }
+
+ for (auto iter = pending_.begin(); iter != pending_.end(); ++iter) {
+ PendingMove* other = *iter;
+ if (other->from().aliases(move) && MoveIsSingle(other->from())) {
+ return true;
+ }
+ if (other->to().aliases(move) && MoveIsSingle(other->to())) {
+ return true;
+ }
+ }
+ return false;
+}
+#endif
+
+#ifdef JS_CODEGEN_ARM
+static MoveOperand SplitIntoLowerHalf(const MoveOperand& move) {
+ if (MoveIsDouble(move)) {
+ FloatRegister lowerSingle = move.floatReg().asSingle();
+ return MoveOperand(lowerSingle);
+ }
+
+ MOZ_ASSERT(move.isMemoryOrEffectiveAddress());
+ return move;
+}
+#endif
+
+#ifdef JS_CODEGEN_ARM
+static MoveOperand SplitIntoUpperHalf(const MoveOperand& move) {
+ if (MoveIsDouble(move)) {
+ FloatRegister lowerSingle = move.floatReg().asSingle();
+ FloatRegister upperSingle =
+ VFPRegister(lowerSingle.code() + 1, VFPRegister::Single);
+ return MoveOperand(upperSingle);
+ }
+
+ MOZ_ASSERT(move.isMemoryOrEffectiveAddress());
+ return MoveOperand(move.base(), move.disp() + sizeof(float));
+}
+#endif
+
+// Resolves the pending_ list to a list in orderedMoves_.
+bool MoveResolver::resolve() {
+ resetState();
+ orderedMoves_.clear();
+
+ // Upon return from this function, the pending_ list must be cleared.
+ auto clearPending = mozilla::MakeScopeExit([this]() { pending_.clear(); });
+
+#ifdef JS_CODEGEN_ARM
+ // Some of ARM's double registers alias two of its single registers,
+ // but the algorithm below assumes that every register can participate
+ // in at most one cycle. To satisfy the algorithm, any double registers
+ // that may conflict are split into their single-register halves.
+ //
+ // This logic is only applicable because ARM only uses registers d0-d15,
+ // all of which alias s0-s31. Double registers d16-d31 are unused.
+ // Therefore there is never a double move that cannot be split.
+ // If this changes in the future, the algorithm will have to be fixed.
+
+ bool splitDoubles = false;
+ for (auto iter = pending_.begin(); iter != pending_.end(); ++iter) {
+ PendingMove* pm = *iter;
+
+ if (isDoubleAliasedAsSingle(pm->from()) ||
+ isDoubleAliasedAsSingle(pm->to())) {
+ splitDoubles = true;
+ break;
+ }
+ }
+
+ if (splitDoubles) {
+ for (auto iter = pending_.begin(); iter != pending_.end(); ++iter) {
+ PendingMove* pm = *iter;
+
+ if (!MoveIsDouble(pm->from()) && !MoveIsDouble(pm->to())) {
+ continue;
+ }
+
+ MoveOperand fromLower = SplitIntoLowerHalf(pm->from());
+ MoveOperand toLower = SplitIntoLowerHalf(pm->to());
+
+ PendingMove* lower =
+ movePool_.allocate(fromLower, toLower, MoveOp::FLOAT32);
+ if (!lower) {
+ return false;
+ }
+
+ // Insert the new node before the current position to not affect
+ // iteration.
+ pending_.insertBefore(pm, lower);
+
+ // Overwrite pm in place for the upper move. Iteration proceeds as normal.
+ MoveOperand fromUpper = SplitIntoUpperHalf(pm->from());
+ MoveOperand toUpper = SplitIntoUpperHalf(pm->to());
+ pm->overwrite(fromUpper, toUpper, MoveOp::FLOAT32);
+ }
+ }
+#endif
+
+ InlineList<PendingMove> stack;
+
+ // This is a depth-first-search without recursion, which tries to find
+ // cycles in a list of moves.
+ //
+ // Algorithm.
+ //
+ // S = Traversal stack.
+ // P = Pending move list.
+ // O = Ordered list of moves.
+ //
+ // As long as there are pending moves in P:
+ // Let |root| be any pending move removed from P
+ // Add |root| to the traversal stack.
+ // As long as S is not empty:
+ // Let |L| be the most recent move added to S.
+ //
+ // Find any pending move M whose source is L's destination, thus
+ // preventing L's move until M has completed.
+ //
+ // If a move M was found,
+ // Remove M from the pending list.
+ // If M's destination is |root|,
+ // Annotate M and |root| as cycles.
+ // Add M to S.
+ // do not Add M to O, since M may have other conflictors in P
+ // that have not yet been processed.
+ // Otherwise,
+ // Add M to S.
+ // Otherwise,
+ // Remove L from S.
+ // Add L to O.
+ //
+ while (!pending_.empty()) {
+ PendingMove* pm = pending_.popBack();
+
+ // Add this pending move to the cycle detection stack.
+ stack.pushBack(pm);
+
+ while (!stack.empty()) {
+ PendingMove* blocking = findBlockingMove(stack.peekBack());
+
+ if (blocking) {
+ PendingMoveIterator stackiter = stack.begin();
+ PendingMove* cycled = findCycledMove(&stackiter, stack.end(), blocking);
+ if (cycled) {
+ // Find the cycle's start.
+ // We annotate cycles at each move in the cycle, and
+ // assert that we do not find two cycles in one move chain
+ // traversal (which would indicate two moves to the same
+ // destination).
+ // Since there can be more than one cycle, find them all.
+ do {
+ cycled->setCycleEnd(curCycles_);
+ cycled = findCycledMove(&stackiter, stack.end(), blocking);
+ } while (cycled);
+
+ blocking->setCycleBegin(pm->type(), curCycles_);
+ curCycles_++;
+ pending_.remove(blocking);
+ stack.pushBack(blocking);
+ } else {
+ // This is a new link in the move chain, so keep
+ // searching for a cycle.
+ pending_.remove(blocking);
+ stack.pushBack(blocking);
+ }
+ } else {
+ // Otherwise, pop the last move on the search stack because it's
+ // complete and not participating in a cycle. The resulting
+ // move can safely be added to the ordered move list.
+ PendingMove* done = stack.popBack();
+ if (!addOrderedMove(*done)) {
+ return false;
+ }
+ movePool_.free(done);
+ }
+ }
+ // If the current queue is empty, it is certain that there are
+ // all previous cycles cannot conflict with future cycles,
+ // so re-set the counter of pending cycles, while keeping a high-water mark.
+ if (numCycles_ < curCycles_) {
+ numCycles_ = curCycles_;
+ }
+ curCycles_ = 0;
+ }
+
+ return true;
+}
+
+bool MoveResolver::addOrderedMove(const MoveOp& move) {
+ // Sometimes the register allocator generates move groups where multiple
+ // moves have the same source. Try to optimize these cases when the source
+ // is in memory and the target of one of the moves is in a register.
+ MOZ_ASSERT(!move.from().aliases(move.to()));
+
+ if (!move.from().isMemory() || move.isCycleBegin() || move.isCycleEnd()) {
+ return orderedMoves_.append(move);
+ }
+
+ // Look for an earlier move with the same source, where no intervening move
+ // touches either the source or destination of the new move.
+ for (int i = orderedMoves_.length() - 1; i >= 0; i--) {
+ const MoveOp& existing = orderedMoves_[i];
+
+ if (existing.from() == move.from() && !existing.to().aliases(move.to()) &&
+ existing.type() == move.type() && !existing.isCycleBegin() &&
+ !existing.isCycleEnd()) {
+ MoveOp* after = orderedMoves_.begin() + i + 1;
+ if (existing.to().isGeneralReg() || existing.to().isFloatReg()) {
+ MoveOp nmove(existing.to(), move.to(), move.type());
+ return orderedMoves_.insert(after, nmove);
+ } else if (move.to().isGeneralReg() || move.to().isFloatReg()) {
+ MoveOp nmove(move.to(), existing.to(), move.type());
+ orderedMoves_[i] = move;
+ return orderedMoves_.insert(after, nmove);
+ }
+ }
+
+ if (existing.aliases(move)) {
+ break;
+ }
+ }
+
+ return orderedMoves_.append(move);
+}
+
+void MoveResolver::reorderMove(size_t from, size_t to) {
+ MOZ_ASSERT(from != to);
+
+ MoveOp op = orderedMoves_[from];
+ if (from < to) {
+ for (size_t i = from; i < to; i++) {
+ orderedMoves_[i] = orderedMoves_[i + 1];
+ }
+ } else {
+ for (size_t i = from; i > to; i--) {
+ orderedMoves_[i] = orderedMoves_[i - 1];
+ }
+ }
+ orderedMoves_[to] = op;
+}
+
+void MoveResolver::sortMemoryToMemoryMoves() {
+ // Try to reorder memory->memory moves so that they are executed right
+ // before a move that clobbers some register. This will allow the move
+ // emitter to use that clobbered register as a scratch register for the
+ // memory->memory move, if necessary.
+ for (size_t i = 0; i < orderedMoves_.length(); i++) {
+ const MoveOp& base = orderedMoves_[i];
+ if (!base.from().isMemory() || !base.to().isMemory()) {
+ continue;
+ }
+ if (base.type() != MoveOp::GENERAL && base.type() != MoveOp::INT32) {
+ continue;
+ }
+
+ // Look for an earlier move clobbering a register.
+ bool found = false;
+ for (int j = i - 1; j >= 0; j--) {
+ const MoveOp& previous = orderedMoves_[j];
+ if (previous.aliases(base) || previous.isCycleBegin() ||
+ previous.isCycleEnd()) {
+ break;
+ }
+
+ if (previous.to().isGeneralReg()) {
+ reorderMove(i, j);
+ found = true;
+ break;
+ }
+ }
+ if (found) {
+ continue;
+ }
+
+ // Look for a later move clobbering a register.
+ if (i + 1 < orderedMoves_.length()) {
+ bool found = false, skippedRegisterUse = false;
+ for (size_t j = i + 1; j < orderedMoves_.length(); j++) {
+ const MoveOp& later = orderedMoves_[j];
+ if (later.aliases(base) || later.isCycleBegin() || later.isCycleEnd()) {
+ break;
+ }
+
+ if (later.to().isGeneralReg()) {
+ if (skippedRegisterUse) {
+ reorderMove(i, j);
+ found = true;
+ } else {
+ // There is no move that uses a register between the
+ // original memory->memory move and this move that
+ // clobbers a register. The move should already be able
+ // to use a scratch register, so don't shift anything
+ // around.
+ }
+ break;
+ }
+
+ if (later.from().isGeneralReg()) {
+ skippedRegisterUse = true;
+ }
+ }
+
+ if (found) {
+ // Redo the search for memory->memory moves at the current
+ // index, so we don't skip the move just shifted back.
+ i--;
+ }
+ }
+ }
+}
diff --git a/js/src/jit/MoveResolver.h b/js/src/jit/MoveResolver.h
new file mode 100644
index 0000000000..4c0585c0eb
--- /dev/null
+++ b/js/src/jit/MoveResolver.h
@@ -0,0 +1,308 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MoveResolver_h
+#define jit_MoveResolver_h
+
+#include "jit/InlineList.h"
+#include "jit/JitAllocPolicy.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "jit/shared/Assembler-shared.h"
+
+namespace js {
+namespace jit {
+
+class MacroAssembler;
+
+// This is similar to Operand, but carries more information. We're also not
+// guaranteed that Operand looks like this on all ISAs.
+class MoveOperand {
+ public:
+ enum Kind {
+ // A register in the "integer", aka "general purpose", class.
+ REG,
+#ifdef JS_CODEGEN_REGISTER_PAIR
+ // Two consecutive "integer" register (aka "general purpose"). The even
+ // register contains the lower part, the odd register has the high bits
+ // of the content.
+ REG_PAIR,
+#endif
+ // A register in the "float" register class.
+ FLOAT_REG,
+ // A memory region.
+ MEMORY,
+ // The address of a memory region.
+ EFFECTIVE_ADDRESS
+ };
+
+ private:
+ Kind kind_;
+ uint32_t code_;
+ int32_t disp_;
+
+ public:
+ MoveOperand() = delete;
+ explicit MoveOperand(Register reg)
+ : kind_(REG), code_(reg.code()), disp_(0) {}
+ explicit MoveOperand(FloatRegister reg)
+ : kind_(FLOAT_REG), code_(reg.code()), disp_(0) {}
+ MoveOperand(Register reg, int32_t disp, Kind kind = MEMORY)
+ : kind_(kind), code_(reg.code()), disp_(disp) {
+ MOZ_ASSERT(isMemoryOrEffectiveAddress());
+
+ // With a zero offset, this is a plain reg-to-reg move.
+ if (disp == 0 && kind_ == EFFECTIVE_ADDRESS) {
+ kind_ = REG;
+ }
+ }
+ explicit MoveOperand(const Address& addr, Kind kind = MEMORY)
+ : MoveOperand(AsRegister(addr.base), addr.offset, kind) {}
+ MoveOperand(MacroAssembler& masm, const ABIArg& arg);
+ MoveOperand(const MoveOperand& other) = default;
+ bool isFloatReg() const { return kind_ == FLOAT_REG; }
+ bool isGeneralReg() const { return kind_ == REG; }
+ bool isGeneralRegPair() const {
+#ifdef JS_CODEGEN_REGISTER_PAIR
+ return kind_ == REG_PAIR;
+#else
+ return false;
+#endif
+ }
+ bool isMemory() const { return kind_ == MEMORY; }
+ bool isEffectiveAddress() const { return kind_ == EFFECTIVE_ADDRESS; }
+ bool isMemoryOrEffectiveAddress() const {
+ return isMemory() || isEffectiveAddress();
+ }
+ Register reg() const {
+ MOZ_ASSERT(isGeneralReg());
+ return Register::FromCode(code_);
+ }
+ Register evenReg() const {
+ MOZ_ASSERT(isGeneralRegPair());
+ return Register::FromCode(code_);
+ }
+ Register oddReg() const {
+ MOZ_ASSERT(isGeneralRegPair());
+ return Register::FromCode(code_ + 1);
+ }
+ FloatRegister floatReg() const {
+ MOZ_ASSERT(isFloatReg());
+ return FloatRegister::FromCode(code_);
+ }
+ Register base() const {
+ MOZ_ASSERT(isMemoryOrEffectiveAddress());
+ return Register::FromCode(code_);
+ }
+ int32_t disp() const {
+ MOZ_ASSERT(isMemoryOrEffectiveAddress());
+ return disp_;
+ }
+
+ bool aliases(MoveOperand other) const {
+ // These are not handled presently, but MEMORY and EFFECTIVE_ADDRESS
+ // only appear in controlled circumstances in the trampoline code
+ // which ensures these cases never come up.
+
+ MOZ_ASSERT_IF(isMemoryOrEffectiveAddress() && other.isGeneralReg(),
+ base() != other.reg());
+ MOZ_ASSERT_IF(other.isMemoryOrEffectiveAddress() && isGeneralReg(),
+ other.base() != reg());
+
+ // Check if one of the operand is a registe rpair, in which case, we
+ // have to check any other register, or register pair.
+ if (isGeneralRegPair() || other.isGeneralRegPair()) {
+ if (isGeneralRegPair() && other.isGeneralRegPair()) {
+ // Assume that register pairs are aligned on even registers.
+ MOZ_ASSERT(!evenReg().aliases(other.oddReg()));
+ MOZ_ASSERT(!oddReg().aliases(other.evenReg()));
+ // Pair of registers are composed of consecutive registers, thus
+ // if the first registers are aliased, then the second registers
+ // are aliased too.
+ MOZ_ASSERT(evenReg().aliases(other.evenReg()) ==
+ oddReg().aliases(other.oddReg()));
+ return evenReg().aliases(other.evenReg());
+ } else if (other.isGeneralReg()) {
+ MOZ_ASSERT(isGeneralRegPair());
+ return evenReg().aliases(other.reg()) || oddReg().aliases(other.reg());
+ } else if (isGeneralReg()) {
+ MOZ_ASSERT(other.isGeneralRegPair());
+ return other.evenReg().aliases(reg()) || other.oddReg().aliases(reg());
+ }
+ return false;
+ }
+
+ if (kind_ != other.kind_) {
+ return false;
+ }
+ if (kind_ == FLOAT_REG) {
+ return floatReg().aliases(other.floatReg());
+ }
+ if (code_ != other.code_) {
+ return false;
+ }
+ if (isMemoryOrEffectiveAddress()) {
+ return disp_ == other.disp_;
+ }
+ return true;
+ }
+
+ bool operator==(const MoveOperand& other) const {
+ if (kind_ != other.kind_) {
+ return false;
+ }
+ if (code_ != other.code_) {
+ return false;
+ }
+ if (isMemoryOrEffectiveAddress()) {
+ return disp_ == other.disp_;
+ }
+ return true;
+ }
+ bool operator!=(const MoveOperand& other) const { return !operator==(other); }
+};
+
+// This represents a move operation.
+class MoveOp {
+ protected:
+ MoveOperand from_;
+ MoveOperand to_;
+ bool cycleBegin_;
+ bool cycleEnd_;
+ int cycleBeginSlot_;
+ int cycleEndSlot_;
+
+ public:
+ enum Type { GENERAL, INT32, FLOAT32, DOUBLE, SIMD128 };
+
+ protected:
+ Type type_;
+
+ // If cycleBegin_ is true, endCycleType_ is the type of the move at the end
+ // of the cycle. For example, given these moves:
+ // INT32 move a -> b
+ // GENERAL move b -> a
+ // the move resolver starts by copying b into a temporary location, so that
+ // the last move can read it. This copy needs to use use type GENERAL.
+ Type endCycleType_;
+
+ public:
+ MoveOp() = delete;
+ MoveOp(const MoveOperand& from, const MoveOperand& to, Type type)
+ : from_(from),
+ to_(to),
+ cycleBegin_(false),
+ cycleEnd_(false),
+ cycleBeginSlot_(-1),
+ cycleEndSlot_(-1),
+ type_(type),
+ endCycleType_(GENERAL) // initialize to silence UBSan warning
+ {}
+
+ bool isCycleBegin() const { return cycleBegin_; }
+ bool isCycleEnd() const { return cycleEnd_; }
+ uint32_t cycleBeginSlot() const {
+ MOZ_ASSERT(cycleBeginSlot_ != -1);
+ return cycleBeginSlot_;
+ }
+ uint32_t cycleEndSlot() const {
+ MOZ_ASSERT(cycleEndSlot_ != -1);
+ return cycleEndSlot_;
+ }
+ const MoveOperand& from() const { return from_; }
+ const MoveOperand& to() const { return to_; }
+ Type type() const { return type_; }
+ Type endCycleType() const {
+ MOZ_ASSERT(isCycleBegin());
+ return endCycleType_;
+ }
+ bool aliases(const MoveOperand& op) const {
+ return from().aliases(op) || to().aliases(op);
+ }
+ bool aliases(const MoveOp& other) const {
+ return aliases(other.from()) || aliases(other.to());
+ }
+#ifdef JS_CODEGEN_ARM
+ void overwrite(MoveOperand& from, MoveOperand& to, Type type) {
+ from_ = from;
+ to_ = to;
+ type_ = type;
+ }
+#endif
+};
+
+class MoveResolver {
+ private:
+ struct PendingMove : public MoveOp,
+ public TempObject,
+ public InlineListNode<PendingMove> {
+ PendingMove() = delete;
+
+ PendingMove(const MoveOperand& from, const MoveOperand& to, Type type)
+ : MoveOp(from, to, type) {}
+
+ void setCycleBegin(Type endCycleType, int cycleSlot) {
+ MOZ_ASSERT(!cycleBegin_);
+ cycleBegin_ = true;
+ cycleBeginSlot_ = cycleSlot;
+ endCycleType_ = endCycleType;
+ }
+ void setCycleEnd(int cycleSlot) {
+ MOZ_ASSERT(!cycleEnd_);
+ cycleEnd_ = true;
+ cycleEndSlot_ = cycleSlot;
+ }
+ };
+
+ using PendingMoveIterator = InlineList<MoveResolver::PendingMove>::iterator;
+
+ js::Vector<MoveOp, 16, SystemAllocPolicy> orderedMoves_;
+ int numCycles_;
+ int curCycles_;
+ TempObjectPool<PendingMove> movePool_;
+
+ InlineList<PendingMove> pending_;
+
+ PendingMove* findBlockingMove(const PendingMove* last);
+ PendingMove* findCycledMove(PendingMoveIterator* stack,
+ PendingMoveIterator end,
+ const PendingMove* first);
+ [[nodiscard]] bool addOrderedMove(const MoveOp& move);
+ void reorderMove(size_t from, size_t to);
+
+ // Internal reset function. Does not clear lists.
+ void resetState();
+
+#ifdef JS_CODEGEN_ARM
+ bool isDoubleAliasedAsSingle(const MoveOperand& move);
+#endif
+
+ public:
+ MoveResolver();
+
+ // Resolves a move group into two lists of ordered moves. These moves must
+ // be executed in the order provided. Some moves may indicate that they
+ // participate in a cycle. For every cycle there are two such moves, and it
+ // is guaranteed that cycles do not nest inside each other in the list.
+ //
+ // After calling addMove() for each parallel move, resolve() performs the
+ // cycle resolution algorithm. Calling addMove() again resets the resolver.
+ [[nodiscard]] bool addMove(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type);
+ [[nodiscard]] bool resolve();
+ void sortMemoryToMemoryMoves();
+
+ size_t numMoves() const { return orderedMoves_.length(); }
+ const MoveOp& getMove(size_t i) const { return orderedMoves_[i]; }
+ uint32_t numCycles() const { return numCycles_; }
+ bool hasNoPendingMoves() const { return pending_.empty(); }
+ void setAllocator(TempAllocator& alloc) { movePool_.setAllocator(alloc); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_MoveResolver_h */
diff --git a/js/src/jit/PcScriptCache.h b/js/src/jit/PcScriptCache.h
new file mode 100644
index 0000000000..c83c479c85
--- /dev/null
+++ b/js/src/jit/PcScriptCache.h
@@ -0,0 +1,88 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_PcScriptCache_h
+#define jit_PcScriptCache_h
+
+#include "mozilla/Array.h"
+#include "js/TypeDecls.h"
+#include "vm/Runtime.h"
+
+// Defines a fixed-size hash table solely for the purpose of caching
+// jit::GetPcScript(). One cache is attached to each JSRuntime; it functions as
+// if cleared on GC.
+
+namespace js {
+namespace jit {
+
+struct PcScriptCacheEntry {
+ uint8_t* returnAddress; // Key into the hash table.
+ jsbytecode* pc; // Cached PC.
+ JSScript* script; // Cached script.
+};
+
+struct PcScriptCache {
+ private:
+ static const uint32_t Length = 73;
+
+ // GC number at the time the cache was filled or created.
+ // Storing and checking against this number allows us to not bother
+ // clearing this cache on every GC -- only when actually necessary.
+ uint64_t gcNumber;
+
+ // List of cache entries.
+ mozilla::Array<PcScriptCacheEntry, Length> entries;
+
+ public:
+ explicit PcScriptCache(uint64_t gcNumber) { clear(gcNumber); }
+
+ void clear(uint64_t gcNumber) {
+ for (uint32_t i = 0; i < Length; i++) {
+ entries[i].returnAddress = nullptr;
+ }
+ this->gcNumber = gcNumber;
+ }
+
+ // Get a value from the cache. May perform lazy allocation.
+ [[nodiscard]] bool get(JSRuntime* rt, uint32_t hash, uint8_t* addr,
+ JSScript** scriptRes, jsbytecode** pcRes) {
+ // If a GC occurred, lazily clear the cache now.
+ if (gcNumber != rt->gc.gcNumber()) {
+ clear(rt->gc.gcNumber());
+ return false;
+ }
+
+ if (entries[hash].returnAddress != addr) {
+ return false;
+ }
+
+ *scriptRes = entries[hash].script;
+ if (pcRes) {
+ *pcRes = entries[hash].pc;
+ }
+
+ return true;
+ }
+
+ void add(uint32_t hash, uint8_t* addr, jsbytecode* pc, JSScript* script) {
+ MOZ_ASSERT(addr);
+ MOZ_ASSERT(pc);
+ MOZ_ASSERT(script);
+ entries[hash].returnAddress = addr;
+ entries[hash].pc = pc;
+ entries[hash].script = script;
+ }
+
+ static uint32_t Hash(uint8_t* addr) {
+ uint32_t key = (uint32_t)((uintptr_t)addr);
+ return ((key >> 3) * 2654435761u) % Length;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_PcScriptCache_h */
diff --git a/js/src/jit/PerfSpewer.cpp b/js/src/jit/PerfSpewer.cpp
new file mode 100644
index 0000000000..4ebf384ce5
--- /dev/null
+++ b/js/src/jit/PerfSpewer.cpp
@@ -0,0 +1,719 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/IntegerPrintfMacros.h"
+#include "mozilla/Printf.h"
+
+#if defined(JS_ION_PERF) && defined(XP_UNIX)
+# include <fcntl.h>
+# include <sys/mman.h>
+# include <sys/stat.h>
+# include <unistd.h>
+#endif
+
+#if defined(JS_ION_PERF) && defined(XP_LINUX) && !defined(ANDROID) && \
+ defined(__GLIBC__)
+# include <dlfcn.h>
+# include <sys/syscall.h>
+# include <sys/types.h>
+# include <unistd.h>
+# define gettid() static_cast<pid_t>(syscall(__NR_gettid))
+#endif
+#include "jit/PerfSpewer.h"
+
+#include <atomic>
+
+#include "jit/Jitdump.h"
+#include "jit/JitSpewer.h"
+#include "jit/LIR.h"
+#include "jit/MIR.h"
+#include "js/JitCodeAPI.h"
+#include "js/Printf.h"
+#include "vm/BytecodeUtil.h"
+#include "vm/MutexIDs.h"
+
+using namespace js;
+using namespace js::jit;
+
+#define PERF_MODE_NONE 1
+#define PERF_MODE_FUNC 2
+#define PERF_MODE_SRC 3
+#define PERF_MODE_IR 4
+
+static std::atomic<bool> geckoProfiling = false;
+
+static uint32_t PerfMode = 0;
+
+// Mutex to guard access to the profiler vectors and jitdump file if perf
+// profiling is enabled.
+static js::Mutex PerfMutex(mutexid::PerfSpewer);
+
+static PersistentRooted<GCVector<JitCode*, 0, js::SystemAllocPolicy>>
+ jitCodeVector;
+static ProfilerJitCodeVector profilerData;
+
+static bool IsGeckoProfiling() { return geckoProfiling; }
+#ifdef JS_ION_PERF
+static UniqueChars spew_dir;
+static FILE* JitDumpFilePtr = nullptr;
+static void* mmap_address = nullptr;
+static bool IsPerfProfiling() { return JitDumpFilePtr != nullptr; }
+#endif
+
+namespace {
+struct MOZ_RAII AutoLockPerfSpewer {
+ AutoLockPerfSpewer() { PerfMutex.lock(); }
+ ~AutoLockPerfSpewer() {
+#ifdef JS_ION_PERF
+ if (JitDumpFilePtr) {
+ fflush(JitDumpFilePtr);
+ }
+#endif
+ PerfMutex.unlock();
+ }
+};
+} // namespace
+
+#ifdef JS_ION_PERF
+static uint64_t GetMonotonicTimestamp() {
+ struct timespec ts = {};
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return ts.tv_sec * 1000000000 + ts.tv_nsec;
+}
+
+// values are from /usr/include/elf.h
+static uint32_t GetMachineEncoding() {
+# if defined(JS_CODEGEN_X86)
+ return 3; // EM_386
+# elif defined(JS_CODEGEN_X64)
+ return 62; // EM_X86_64
+# elif defined(JS_CODEGEN_ARM)
+ return 40; // EM_ARM
+# elif defined(JS_CODEGEN_ARM64)
+ return 183; // EM_AARCH64
+# elif defined(JS_CODEGEN_MIPS32)
+ return 8; // EM_MIPS
+# elif defined(JS_CODEGEN_MIPS64)
+ return 8; // EM_MIPS
+# else
+ return 0; // Unsupported
+# endif
+}
+
+static void WriteToJitDumpFile(const void* addr, uint32_t size,
+ AutoLockPerfSpewer& lock) {
+ MOZ_RELEASE_ASSERT(JitDumpFilePtr);
+ size_t rv = fwrite(addr, 1, size, JitDumpFilePtr);
+ MOZ_RELEASE_ASSERT(rv == size);
+}
+
+static void WriteJitDumpDebugEntry(uint64_t addr, const char* filename,
+ uint32_t lineno, uint32_t colno,
+ AutoLockPerfSpewer& lock) {
+ JitDumpDebugEntry entry = {addr, lineno, colno};
+ WriteToJitDumpFile(&entry, sizeof(entry), lock);
+ WriteToJitDumpFile(filename, strlen(filename) + 1, lock);
+}
+
+static bool FileExists(const char* filename) {
+ // We don't currently dump external resources to disk.
+ if (strncmp(filename, "http", 4) == 0) {
+ return false;
+ }
+
+ struct stat buf = {};
+ return stat(filename, &buf) == 0;
+}
+
+static void writeJitDumpHeader(AutoLockPerfSpewer& lock) {
+ JitDumpHeader header = {};
+ header.magic = 0x4A695444;
+ header.version = 1;
+ header.total_size = sizeof(header);
+ header.elf_mach = GetMachineEncoding();
+ header.pad1 = 0;
+ header.pid = getpid();
+ header.timestamp = GetMonotonicTimestamp();
+ header.flags = 0;
+
+ WriteToJitDumpFile(&header, sizeof(header), lock);
+}
+
+static bool openJitDump() {
+ if (JitDumpFilePtr) {
+ return true;
+ }
+ AutoLockPerfSpewer lock;
+
+ const ssize_t bufferSize = 256;
+ char filenameBuffer[bufferSize];
+
+ // We want to write absolute filenames into the debug info or else we get the
+ // filenames in the perf report
+ if (getenv("PERF_SPEW_DIR")) {
+ char* env_dir = getenv("PERF_SPEW_DIR");
+ if (env_dir[0] == '/') {
+ spew_dir = JS_smprintf("%s", env_dir);
+ } else {
+ spew_dir = JS_smprintf("%s/%s", get_current_dir_name(), env_dir);
+ }
+ } else {
+ fprintf(stderr, "Please define PERF_SPEW_DIR as an output directory.\n");
+ return false;
+ }
+
+ if (snprintf(filenameBuffer, bufferSize, "%s/jit-%d.dump", spew_dir.get(),
+ getpid()) >= bufferSize) {
+ return false;
+ }
+
+ MOZ_ASSERT(!JitDumpFilePtr);
+
+ int fd = open(filenameBuffer, O_CREAT | O_TRUNC | O_RDWR, 0666);
+ JitDumpFilePtr = fdopen(fd, "w+");
+
+ if (!JitDumpFilePtr) {
+ return false;
+ }
+
+ // We need to mmap the jitdump file for perf to find it.
+ long page_size = sysconf(_SC_PAGESIZE);
+ mmap_address =
+ mmap(nullptr, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0);
+ if (mmap_address == MAP_FAILED) {
+ PerfMode = PERF_MODE_NONE;
+ return false;
+ }
+
+ writeJitDumpHeader(lock);
+ return true;
+}
+
+void js::jit::CheckPerf() {
+ static bool PerfChecked = false;
+
+ if (!PerfChecked) {
+ const char* env = getenv("IONPERF");
+ if (env == nullptr) {
+ PerfMode = PERF_MODE_NONE;
+ fprintf(stderr,
+ "Warning: JIT perf reporting requires IONPERF set to \"func\" "
+ ", \"src\" or \"ir\". ");
+ fprintf(stderr, "Perf mapping will be deactivated.\n");
+ } else if (!strcmp(env, "src")) {
+ PerfMode = PERF_MODE_SRC;
+ } else if (!strcmp(env, "ir")) {
+ PerfMode = PERF_MODE_IR;
+ } else if (!strcmp(env, "func")) {
+ PerfMode = PERF_MODE_FUNC;
+ } else {
+ fprintf(stderr, "Use IONPERF=func to record at function granularity\n");
+ fprintf(stderr,
+ "Use IONPERF=ir to record and annotate assembly with IR\n");
+ fprintf(stderr,
+ "Use IONPERF=src to record and annotate assembly with source, if "
+ "available locally\n");
+ exit(0);
+ }
+
+ if (PerfMode != PERF_MODE_NONE) {
+ if (openJitDump()) {
+ PerfChecked = true;
+ return;
+ }
+
+ fprintf(stderr, "Failed to open perf map file. Disabling IONPERF.\n");
+ PerfMode = PERF_MODE_NONE;
+ }
+ PerfChecked = true;
+ }
+}
+#endif
+
+static void DisablePerfSpewer(AutoLockPerfSpewer& lock) {
+ fprintf(stderr, "Warning: Disabling PerfSpewer.");
+
+ geckoProfiling = false;
+ PerfMode = 0;
+#ifdef JS_ION_PERF
+ long page_size = sysconf(_SC_PAGESIZE);
+ munmap(mmap_address, page_size);
+ fclose(JitDumpFilePtr);
+ JitDumpFilePtr = nullptr;
+#endif
+}
+
+void js::jit::ResetPerfSpewer(bool enabled) {
+ AutoLockPerfSpewer lock;
+
+ profilerData.clear();
+ jitCodeVector.clear();
+ geckoProfiling = enabled;
+}
+
+static JS::JitCodeRecord* CreateProfilerEntry(AutoLockPerfSpewer& lock) {
+ if (!profilerData.growBy(1)) {
+ DisablePerfSpewer(lock);
+ return nullptr;
+ }
+ return &profilerData.back();
+}
+
+static JS::JitCodeIRInfo* CreateProfilerIREntry(JS::JitCodeRecord* record,
+ AutoLockPerfSpewer& lock) {
+ if (!record) {
+ return nullptr;
+ }
+
+ if (!record->irInfo.growBy(1)) {
+ DisablePerfSpewer(lock);
+ return nullptr;
+ }
+ return &record->irInfo.back();
+}
+
+static JS::JitCodeSourceInfo* CreateProfilerSourceEntry(
+ JS::JitCodeRecord* record, AutoLockPerfSpewer& lock) {
+ if (!record) {
+ return nullptr;
+ }
+
+ if (!record->sourceInfo.growBy(1)) {
+ DisablePerfSpewer(lock);
+ return nullptr;
+ }
+ return &record->sourceInfo.back();
+}
+
+JS::JitOpcodeDictionary::JitOpcodeDictionary() {
+ MOZ_ASSERT(JS_IsInitialized());
+
+#define COPY_JSOP_OPCODE(name, ...) \
+ if (!baselineDictionary.append(JS_smprintf(#name))) { \
+ return; \
+ }
+ FOR_EACH_OPCODE(COPY_JSOP_OPCODE)
+#undef COPY_JSOP_OPCODE
+
+#define COPY_LIR_OPCODE(name, ...) \
+ if (!ionDictionary.append(JS_smprintf(#name))) { \
+ return; \
+ }
+ LIR_OPCODE_LIST(COPY_LIR_OPCODE)
+#undef COPY_LIR_OPCODE
+
+#define COPY_CACHEIR_OPCODE(name, ...) \
+ if (!icDictionary.append(JS_smprintf(#name))) { \
+ return; \
+ }
+ CACHE_IR_OPS(COPY_CACHEIR_OPCODE)
+#undef COPY_CACHEIR_OPCODE
+}
+
+// API to access JitCode data for the Gecko Profiler.
+void JS::JitCodeIterator::getDataForIndex(size_t IteratorIndex) {
+ if (IteratorIndex >= profilerData.length()) {
+ data = nullptr;
+ } else {
+ data = &profilerData[IteratorIndex];
+ }
+}
+
+JS::JitCodeIterator::JitCodeIterator() : iteratorIndex(0) {
+ MOZ_ASSERT(JS_IsInitialized());
+ PerfMutex.lock();
+ getDataForIndex(0);
+}
+
+JS::JitCodeIterator::~JitCodeIterator() { PerfMutex.unlock(); }
+
+static bool PerfSrcEnabled() {
+ return PerfMode == PERF_MODE_SRC || geckoProfiling;
+}
+
+static bool PerfIREnabled() {
+ return PerfMode == PERF_MODE_IR || geckoProfiling;
+}
+
+static bool PerfFuncEnabled() {
+ return PerfMode == PERF_MODE_FUNC || geckoProfiling;
+}
+
+bool js::jit::PerfEnabled() {
+ return PerfSrcEnabled() || PerfIREnabled() || PerfFuncEnabled();
+}
+
+void InlineCachePerfSpewer::recordInstruction(MacroAssembler& masm,
+ CacheOp op) {
+ if (!PerfIREnabled()) {
+ return;
+ }
+ AutoLockPerfSpewer lock;
+
+ OpcodeEntry entry;
+ entry.opcode = static_cast<unsigned>(op);
+ masm.bind(&entry.addr);
+
+ if (!opcodes_.append(entry)) {
+ opcodes_.clear();
+ DisablePerfSpewer(lock);
+ }
+}
+
+void IonPerfSpewer::recordInstruction(MacroAssembler& masm, LNode::Opcode op) {
+ if (!PerfIREnabled()) {
+ return;
+ }
+ AutoLockPerfSpewer lock;
+
+ OpcodeEntry entry;
+ entry.opcode = static_cast<unsigned>(op);
+ masm.bind(&entry.addr);
+
+ if (!opcodes_.append(entry)) {
+ opcodes_.clear();
+ DisablePerfSpewer(lock);
+ }
+}
+
+void BaselinePerfSpewer::recordInstruction(MacroAssembler& masm, JSOp op) {
+ if (!PerfIREnabled()) {
+ return;
+ }
+ AutoLockPerfSpewer lock;
+
+ OpcodeEntry entry;
+ masm.bind(&entry.addr);
+ entry.opcode = static_cast<unsigned>(op);
+
+ if (!opcodes_.append(entry)) {
+ opcodes_.clear();
+ DisablePerfSpewer(lock);
+ }
+}
+
+const char* BaselinePerfSpewer::CodeName(unsigned op) {
+ return js::CodeName(static_cast<JSOp>(op));
+}
+const char* IonPerfSpewer::CodeName(unsigned op) {
+ return js::jit::LIRCodeName(static_cast<LNode::Opcode>(op));
+}
+const char* InlineCachePerfSpewer::CodeName(unsigned op) {
+ return js::jit::CacheIRCodeName(static_cast<CacheOp>(op));
+}
+
+void PerfSpewer::CollectJitCodeInfo(UniqueChars& function_name, JitCode* code,
+ JS::JitCodeRecord* profilerRecord,
+ AutoLockPerfSpewer& lock) {
+ // Hold the JitCode objects here so they are not GC'd during profiling.
+ if (IsGeckoProfiling()) {
+ if (!jitCodeVector.append(code)) {
+ DisablePerfSpewer(lock);
+ }
+ }
+
+ CollectJitCodeInfo(function_name, reinterpret_cast<void*>(code->raw()),
+ code->instructionsSize(), profilerRecord, lock);
+}
+
+void PerfSpewer::CollectJitCodeInfo(UniqueChars& function_name, void* code_addr,
+ uint64_t code_size,
+ JS::JitCodeRecord* profilerRecord,
+ AutoLockPerfSpewer& lock) {
+#ifdef JS_ION_PERF
+ static uint64_t codeIndex = 1;
+
+ if (IsPerfProfiling()) {
+ JitDumpLoadRecord record = {};
+
+ record.header.id = JIT_CODE_LOAD;
+ record.header.total_size =
+ sizeof(record) + strlen(function_name.get()) + 1 + code_size;
+ record.header.timestamp = GetMonotonicTimestamp();
+ record.pid = getpid();
+ record.tid = gettid();
+ record.vma = uint64_t(code_addr);
+ record.code_addr = uint64_t(code_addr);
+ record.code_size = code_size;
+ record.code_index = codeIndex++;
+
+ WriteToJitDumpFile(&record, sizeof(record), lock);
+ WriteToJitDumpFile(function_name.get(), strlen(function_name.get()) + 1,
+ lock);
+ WriteToJitDumpFile(code_addr, code_size, lock);
+ }
+#endif
+
+ if (IsGeckoProfiling()) {
+ profilerRecord->instructionSize = code_size;
+ profilerRecord->code_addr = uint64_t(code_addr);
+ profilerRecord->functionName = std::move(function_name);
+ }
+}
+
+void PerfSpewer::saveJitCodeIRInfo(const char* desc, JitCode* code,
+ JS::JitCodeRecord* profilerRecord,
+ AutoLockPerfSpewer& lock) {
+#ifdef JS_ION_PERF
+ static uint32_t filenameCounter = 0;
+ UniqueChars scriptFilename;
+ FILE* scriptFile = nullptr;
+
+ if (IsPerfProfiling()) {
+ scriptFilename = JS_smprintf("%s/jitdump-script-%u.%u.txt", spew_dir.get(),
+ filenameCounter++, getpid());
+ scriptFile = fopen(scriptFilename.get(), "w");
+ if (!scriptFile) {
+ DisablePerfSpewer(lock);
+ return;
+ }
+
+ JitDumpDebugRecord debug_record = {};
+ uint64_t n_records = opcodes_.length();
+
+ debug_record.header.id = JIT_CODE_DEBUG_INFO;
+ debug_record.header.total_size =
+ sizeof(debug_record) + n_records * (sizeof(JitDumpDebugEntry) +
+ strlen(scriptFilename.get()) + 1);
+ debug_record.header.timestamp = GetMonotonicTimestamp();
+ debug_record.code_addr = uint64_t(code->raw());
+ debug_record.nr_entry = n_records;
+
+ WriteToJitDumpFile(&debug_record, sizeof(debug_record), lock);
+ }
+#endif
+
+ if (profilerRecord) {
+ profilerRecord->tier = GetTier();
+ }
+
+ for (size_t i = 0; i < opcodes_.length(); i++) {
+ OpcodeEntry& entry = opcodes_[i];
+ if (JS::JitCodeIRInfo* irInfo =
+ CreateProfilerIREntry(profilerRecord, lock)) {
+ irInfo->offset = entry.addr.offset();
+ irInfo->opcode = entry.opcode;
+ }
+
+#ifdef JS_ION_PERF
+ if (IsPerfProfiling()) {
+ fprintf(scriptFile, "%s\n", CodeName(entry.opcode));
+ uint64_t addr = uint64_t(code->raw()) + entry.addr.offset();
+ uint64_t lineno = i + 1;
+ WriteJitDumpDebugEntry(addr, scriptFilename.get(), lineno, 0, lock);
+ }
+#endif
+ }
+ opcodes_.clear();
+
+#ifdef JS_ION_PERF
+ if (desc && IsPerfProfiling()) {
+ // Add the desc as the last line in the file so as to not confuse objdump
+ fprintf(scriptFile, "%s\n", desc);
+ fclose(scriptFile);
+ }
+#endif
+}
+
+void PerfSpewer::saveJitCodeSourceInfo(JSScript* script, JitCode* code,
+ JS::JitCodeRecord* profilerRecord,
+ AutoLockPerfSpewer& lock) {
+ const char* filename = script->filename();
+ if (!filename) {
+ return;
+ }
+
+#ifdef JS_ION_PERF
+ bool perfProfiling = IsPerfProfiling() && FileExists(filename);
+
+ // If we are using perf, we need to know the number of debug entries ahead of
+ // time for the header.
+ if (perfProfiling) {
+ JitDumpDebugRecord debug_record = {};
+ uint64_t n_records = 0;
+
+ for (SrcNoteIterator iter(script->notes()); !iter.atEnd(); ++iter) {
+ const auto* const sn = *iter;
+ switch (sn->type()) {
+ case SrcNoteType::SetLine:
+ case SrcNoteType::NewLine:
+ case SrcNoteType::ColSpan:
+ if (sn->delta() > 0) {
+ n_records++;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ // Nothing to do
+ if (n_records == 0) {
+ return;
+ }
+
+ debug_record.header.id = JIT_CODE_DEBUG_INFO;
+ debug_record.header.total_size =
+ sizeof(debug_record) +
+ n_records * (sizeof(JitDumpDebugEntry) + strlen(filename) + 1);
+
+ debug_record.header.timestamp = GetMonotonicTimestamp();
+ debug_record.code_addr = uint64_t(code->raw());
+ debug_record.nr_entry = n_records;
+
+ WriteToJitDumpFile(&debug_record, sizeof(debug_record), lock);
+ }
+#endif
+
+ uint32_t lineno = script->lineno();
+ uint32_t colno = script->column();
+ uint64_t offset = 0;
+ for (SrcNoteIterator iter(script->notes()); !iter.atEnd(); ++iter) {
+ const auto* sn = *iter;
+ offset += sn->delta();
+
+ SrcNoteType type = sn->type();
+ if (type == SrcNoteType::SetLine) {
+ lineno = SrcNote::SetLine::getLine(sn, script->lineno());
+ colno = 0;
+ } else if (type == SrcNoteType::NewLine) {
+ lineno++;
+ colno = 0;
+ } else if (type == SrcNoteType::ColSpan) {
+ colno += SrcNote::ColSpan::getSpan(sn);
+ } else {
+ continue;
+ }
+
+ // Don't add entries that won't change the offset
+ if (sn->delta() <= 0) {
+ continue;
+ }
+
+ if (JS::JitCodeSourceInfo* srcInfo =
+ CreateProfilerSourceEntry(profilerRecord, lock)) {
+ srcInfo->offset = offset;
+ srcInfo->lineno = lineno;
+ srcInfo->colno = colno;
+ srcInfo->filename = JS_smprintf("%s", filename);
+ }
+
+#ifdef JS_ION_PERF
+ if (perfProfiling) {
+ WriteJitDumpDebugEntry(uint64_t(code->raw()) + offset, filename, lineno,
+ colno, lock);
+ }
+#endif
+ }
+}
+
+static UniqueChars GetFunctionDesc(bool ion, JSContext* cx, JSScript* script) {
+ UniqueChars funName;
+ if (script->function() && script->function()->displayAtom()) {
+ funName = AtomToPrintableString(cx, script->function()->displayAtom());
+ }
+
+ return JS_smprintf("%s: %s (%s:%u:%u)", ion ? "Ion" : "Baseline",
+ funName ? funName.get() : "*", script->filename(),
+ script->lineno(), script->column());
+}
+
+void IonPerfSpewer::saveProfile(JSContext* cx, JSScript* script,
+ JitCode* code) {
+ if (!PerfEnabled()) {
+ return;
+ }
+ AutoLockPerfSpewer lock;
+
+ JS::JitCodeRecord* profilerRecord = CreateProfilerEntry(lock);
+
+ UniqueChars desc = GetFunctionDesc(/*ion = */ true, cx, script);
+ if (PerfIREnabled()) {
+ saveJitCodeIRInfo(desc.get(), code, profilerRecord, lock);
+ }
+
+ CollectJitCodeInfo(desc, code, profilerRecord, lock);
+}
+
+void BaselinePerfSpewer::saveProfile(JSContext* cx, JSScript* script,
+ JitCode* code) {
+ if (!PerfEnabled()) {
+ return;
+ }
+ AutoLockPerfSpewer lock;
+
+ JS::JitCodeRecord* profilerRecord = CreateProfilerEntry(lock);
+
+ UniqueChars desc = GetFunctionDesc(/*ion = */ false, cx, script);
+ if (PerfIREnabled()) {
+ saveJitCodeIRInfo(desc.get(), code, profilerRecord, lock);
+ } else if (PerfSrcEnabled()) {
+ saveJitCodeSourceInfo(script, code, profilerRecord, lock);
+ }
+
+ CollectJitCodeInfo(desc, code, profilerRecord, lock);
+}
+
+void InlineCachePerfSpewer::saveProfile(JitCode* code, const char* name) {
+ if (!PerfEnabled()) {
+ return;
+ }
+ AutoLockPerfSpewer lock;
+
+ JS::JitCodeRecord* profilerRecord = CreateProfilerEntry(lock);
+
+ UniqueChars desc = JS_smprintf("IC: %s", name);
+ if (PerfIREnabled()) {
+ saveJitCodeIRInfo(desc.get(), code, profilerRecord, lock);
+ }
+
+ CollectJitCodeInfo(desc, code, profilerRecord, lock);
+}
+
+void js::jit::CollectPerfSpewerJitCodeProfile(JitCode* code, const char* msg) {
+ if (!code || !PerfEnabled()) {
+ return;
+ }
+
+ size_t size = code->instructionsSize();
+ if (size > 0) {
+ AutoLockPerfSpewer lock;
+
+ JS::JitCodeRecord* profilerRecord = CreateProfilerEntry(lock);
+ UniqueChars desc = JS_smprintf("%s", msg);
+ PerfSpewer::CollectJitCodeInfo(desc, code, profilerRecord, lock);
+ }
+}
+
+void js::jit::CollectPerfSpewerWasmMap(uintptr_t base, uintptr_t size,
+ const char* filename,
+ const char* annotation) {
+ if (size == 0U || !PerfEnabled()) {
+ return;
+ }
+ AutoLockPerfSpewer lock;
+
+ JS::JitCodeRecord* profilerRecord = CreateProfilerEntry(lock);
+ UniqueChars desc = JS_smprintf("%s: Function %s", filename, annotation);
+ PerfSpewer::CollectJitCodeInfo(desc, reinterpret_cast<void*>(base),
+ uint64_t(size), profilerRecord, lock);
+}
+
+void js::jit::CollectPerfSpewerWasmFunctionMap(uintptr_t base, uintptr_t size,
+ const char* filename,
+ unsigned lineno,
+ const char* funcName) {
+ if (size == 0U || !PerfEnabled()) {
+ return;
+ }
+ AutoLockPerfSpewer lock;
+
+ JS::JitCodeRecord* profilerRecord = CreateProfilerEntry(lock);
+ UniqueChars desc =
+ JS_smprintf("%s:%u: Function %s", filename, lineno, funcName);
+ PerfSpewer::CollectJitCodeInfo(desc, reinterpret_cast<void*>(base),
+ uint64_t(size), profilerRecord, lock);
+}
diff --git a/js/src/jit/PerfSpewer.h b/js/src/jit/PerfSpewer.h
new file mode 100644
index 0000000000..cd9c9ae0f6
--- /dev/null
+++ b/js/src/jit/PerfSpewer.h
@@ -0,0 +1,112 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_PerfSpewer_h
+#define jit_PerfSpewer_h
+
+#ifdef JS_ION_PERF
+# include <stdio.h>
+#endif
+#include "jit/CacheIR.h"
+#include "jit/JitCode.h"
+#include "jit/Label.h"
+#include "jit/LIR.h"
+#include "js/AllocPolicy.h"
+#include "js/JitCodeAPI.h"
+#include "js/Vector.h"
+#include "vm/JSScript.h"
+
+namespace {
+struct AutoLockPerfSpewer;
+}
+
+namespace js::jit {
+
+using ProfilerJitCodeVector = Vector<JS::JitCodeRecord, 0, SystemAllocPolicy>;
+
+#ifdef JS_ION_PERF
+void CheckPerf();
+#else
+inline void CheckPerf() {}
+#endif
+void ResetPerfSpewer(bool enabled);
+
+class MBasicBlock;
+class MacroAssembler;
+
+bool PerfEnabled();
+
+class PerfSpewer {
+ protected:
+ struct OpcodeEntry {
+ Label addr;
+ unsigned opcode = 0;
+ };
+ Vector<OpcodeEntry, 0, SystemAllocPolicy> opcodes_;
+
+ uint32_t lir_opcode_length = 0;
+ uint32_t js_opcode_length = 0;
+
+ virtual JS::JitTier GetTier() { return JS::JitTier::Other; }
+
+ virtual const char* CodeName(unsigned op) = 0;
+
+ public:
+ PerfSpewer() = default;
+
+ void saveJitCodeIRInfo(const char* desc, JitCode* code,
+ JS::JitCodeRecord* profilerRecord,
+ AutoLockPerfSpewer& lock);
+ void saveJitCodeSourceInfo(JSScript* script, JitCode* code,
+ JS::JitCodeRecord* record,
+ AutoLockPerfSpewer& lock);
+
+ static void CollectJitCodeInfo(UniqueChars& function_name, JitCode* code,
+ JS::JitCodeRecord*, AutoLockPerfSpewer& lock);
+ static void CollectJitCodeInfo(UniqueChars& function_name, void* code_addr,
+ uint64_t code_size,
+ JS::JitCodeRecord* profilerRecord,
+ AutoLockPerfSpewer& lock);
+};
+
+void CollectPerfSpewerJitCodeProfile(JitCode* code, const char* msg);
+
+void CollectPerfSpewerWasmMap(uintptr_t base, uintptr_t size,
+ const char* filename, const char* annotation);
+void CollectPerfSpewerWasmFunctionMap(uintptr_t base, uintptr_t size,
+ const char* filename, unsigned lineno,
+ const char* funcName);
+
+class IonPerfSpewer : public PerfSpewer {
+ JS::JitTier GetTier() override { return JS::JitTier::Ion; }
+ virtual const char* CodeName(unsigned op) override;
+
+ public:
+ void recordInstruction(MacroAssembler& masm, LNode::Opcode op);
+ void saveProfile(JSContext* cx, JSScript* script, JitCode* code);
+};
+
+class BaselinePerfSpewer : public PerfSpewer {
+ JS::JitTier GetTier() override { return JS::JitTier::Baseline; }
+ virtual const char* CodeName(unsigned op) override;
+
+ public:
+ void recordInstruction(MacroAssembler& masm, JSOp op);
+ void saveProfile(JSContext* cx, JSScript* script, JitCode* code);
+};
+
+class InlineCachePerfSpewer : public PerfSpewer {
+ JS::JitTier GetTier() override { return JS::JitTier::IC; }
+ virtual const char* CodeName(unsigned op) override;
+
+ public:
+ void recordInstruction(MacroAssembler& masm, CacheOp op);
+ void saveProfile(JitCode* code, const char* name);
+};
+
+} // namespace js::jit
+
+#endif /* jit_PerfSpewer_h */
diff --git a/js/src/jit/ProcessExecutableMemory.cpp b/js/src/jit/ProcessExecutableMemory.cpp
new file mode 100644
index 0000000000..00eff4d544
--- /dev/null
+++ b/js/src/jit/ProcessExecutableMemory.cpp
@@ -0,0 +1,846 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ProcessExecutableMemory.h"
+
+#include "mozilla/Array.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/TaggedAnonymousMemory.h"
+#include "mozilla/XorShift128PlusRNG.h"
+
+#include <errno.h>
+
+#include "jsfriendapi.h"
+#include "jsmath.h"
+
+#include "gc/Memory.h"
+#include "jit/FlushICache.h" // js::jit::FlushICache
+#include "jit/JitOptions.h"
+#include "threading/LockGuard.h"
+#include "threading/Mutex.h"
+#include "util/Memory.h"
+#include "util/Poison.h"
+#include "util/WindowsWrapper.h"
+#include "vm/MutexIDs.h"
+
+#ifdef XP_WIN
+# include "mozilla/StackWalk_windows.h"
+# include "mozilla/WindowsVersion.h"
+#elif defined(__wasi__)
+# if defined(JS_CODEGEN_WASM32)
+# include <cstdlib>
+# else
+// Nothing.
+# endif
+#else
+# include <sys/mman.h>
+# include <unistd.h>
+#endif
+
+#ifdef MOZ_VALGRIND
+# include <valgrind/valgrind.h>
+#endif
+
+using namespace js;
+using namespace js::jit;
+
+#ifdef XP_WIN
+# if defined(HAVE_64BIT_BUILD)
+# define NEED_JIT_UNWIND_HANDLING
+# endif
+
+static void* ComputeRandomAllocationAddress() {
+ /*
+ * Inspiration is V8's OS::Allocate in platform-win32.cc.
+ *
+ * VirtualAlloc takes 64K chunks out of the virtual address space, so we
+ * keep 16b alignment.
+ *
+ * x86: V8 comments say that keeping addresses in the [64MiB, 1GiB) range
+ * tries to avoid system default DLL mapping space. In the end, we get 13
+ * bits of randomness in our selection.
+ * x64: [2GiB, 4TiB), with 25 bits of randomness.
+ */
+# ifdef HAVE_64BIT_BUILD
+ static const uintptr_t base = 0x0000000080000000;
+ static const uintptr_t mask = 0x000003ffffff0000;
+# elif defined(_M_IX86) || defined(__i386__)
+ static const uintptr_t base = 0x04000000;
+ static const uintptr_t mask = 0x3fff0000;
+# else
+# error "Unsupported architecture"
+# endif
+
+ uint64_t rand = js::GenerateRandomSeed();
+ return (void*)(base | (rand & mask));
+}
+
+# ifdef NEED_JIT_UNWIND_HANDLING
+static js::JitExceptionHandler sJitExceptionHandler;
+# endif
+
+JS_PUBLIC_API void js::SetJitExceptionHandler(JitExceptionHandler handler) {
+# ifdef NEED_JIT_UNWIND_HANDLING
+ MOZ_ASSERT(!sJitExceptionHandler);
+ sJitExceptionHandler = handler;
+# else
+ // Just do nothing if unwind handling is disabled.
+# endif
+}
+
+# ifdef NEED_JIT_UNWIND_HANDLING
+# if defined(_M_ARM64)
+// See the ".xdata records" section of
+// https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
+// These records can have various fields present or absent depending on the
+// bits set in the header. Our struct will use one 32-bit slot for unwind codes,
+// and no slots for epilog scopes.
+struct UnwindInfo {
+ uint32_t functionLength : 18;
+ uint32_t version : 2;
+ uint32_t hasExceptionHandler : 1;
+ uint32_t packedEpilog : 1;
+ uint32_t epilogCount : 5;
+ uint32_t codeWords : 5;
+ uint8_t unwindCodes[4];
+ uint32_t exceptionHandler;
+};
+static const unsigned ThunkLength = 20;
+# else
+// From documentation for UNWIND_INFO on
+// http://msdn.microsoft.com/en-us/library/ddssxxy8.aspx
+struct UnwindInfo {
+ uint8_t version : 3;
+ uint8_t flags : 5;
+ uint8_t sizeOfPrologue;
+ uint8_t countOfUnwindCodes;
+ uint8_t frameRegister : 4;
+ uint8_t frameOffset : 4;
+ ULONG exceptionHandler;
+};
+static const unsigned ThunkLength = 12;
+# endif
+
+struct ExceptionHandlerRecord {
+ RUNTIME_FUNCTION runtimeFunction;
+ UnwindInfo unwindInfo;
+ uint8_t thunk[ThunkLength];
+};
+
+// This function must match the function pointer type PEXCEPTION_HANDLER
+// mentioned in:
+// http://msdn.microsoft.com/en-us/library/ssa62fwe.aspx.
+// This type is rather elusive in documentation; Wine is the best I've found:
+// http://source.winehq.org/source/include/winnt.h
+static DWORD ExceptionHandler(PEXCEPTION_RECORD exceptionRecord,
+ _EXCEPTION_REGISTRATION_RECORD*, PCONTEXT context,
+ _EXCEPTION_REGISTRATION_RECORD**) {
+ return sJitExceptionHandler(exceptionRecord, context);
+}
+
+PRUNTIME_FUNCTION RuntimeFunctionCallback(DWORD64 ControlPc, PVOID Context);
+
+// For an explanation of the problem being solved here, see
+// SetJitExceptionFilter in jsfriendapi.h.
+static bool RegisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
+ if (!VirtualAlloc(p, pageSize, MEM_COMMIT, PAGE_READWRITE)) {
+ MOZ_CRASH();
+ }
+
+ ExceptionHandlerRecord* r = reinterpret_cast<ExceptionHandlerRecord*>(p);
+ void* handler = JS_FUNC_TO_DATA_PTR(void*, ExceptionHandler);
+
+ // Because the .xdata format on ARM64 can only encode sizes up to 1M (much
+ // too small for our JIT code regions), we register a function table callback
+ // to provide RUNTIME_FUNCTIONs at runtime. Windows doesn't seem to care about
+ // the size fields on RUNTIME_FUNCTIONs that are created in this way, so the
+ // same RUNTIME_FUNCTION can work for any address in the region. We'll set up
+ // a generic one now and the callback can just return a pointer to it.
+
+ // All these fields are specified to be offsets from the base of the
+ // executable code (which is 'p'), even if they have 'Address' in their
+ // names. In particular, exceptionHandler is a ULONG offset which is a
+ // 32-bit integer. Since 'p' can be farther than INT32_MAX away from
+ // sJitExceptionHandler, we must generate a little thunk inside the
+ // record. The record is put on its own page so that we can take away write
+ // access to protect against accidental clobbering.
+
+# if defined(_M_ARM64)
+ r->runtimeFunction.BeginAddress = pageSize;
+ r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindInfo);
+ static_assert(offsetof(ExceptionHandlerRecord, unwindInfo) % 4 == 0,
+ "The ARM64 .pdata format requires that exception information "
+ "RVAs be 4-byte aligned.");
+
+ memset(&r->unwindInfo, 0, sizeof(r->unwindInfo));
+ r->unwindInfo.hasExceptionHandler = true;
+ r->unwindInfo.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);
+
+ // Use a fake unwind code to make the Windows unwinder do _something_. If the
+ // PC and SP both stay unchanged, we'll fail the unwinder's sanity checks and
+ // it won't call our exception handler.
+ r->unwindInfo.codeWords = 1; // one 32-bit word gives us up to 4 codes
+ r->unwindInfo.unwindCodes[0] =
+ 0b00000001; // alloc_s small stack of size 1*16
+ r->unwindInfo.unwindCodes[1] = 0b11100100; // end
+
+ uint32_t* thunk = (uint32_t*)r->thunk;
+ uint16_t* addr = (uint16_t*)&handler;
+
+ // xip0/r16 should be safe to clobber: Windows just used it to call our thunk.
+ const uint8_t reg = 16;
+
+ // Say `handler` is 0x4444333322221111, then:
+ thunk[0] = 0xd2800000 | addr[0] << 5 | reg; // mov xip0, 1111
+ thunk[1] = 0xf2a00000 | addr[1] << 5 | reg; // movk xip0, 2222 lsl #0x10
+ thunk[2] = 0xf2c00000 | addr[2] << 5 | reg; // movk xip0, 3333 lsl #0x20
+ thunk[3] = 0xf2e00000 | addr[3] << 5 | reg; // movk xip0, 4444 lsl #0x30
+ thunk[4] = 0xd61f0000 | reg << 5; // br xip0
+# else
+ r->runtimeFunction.BeginAddress = pageSize;
+ r->runtimeFunction.EndAddress = (DWORD)bytes;
+ r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindInfo);
+
+ r->unwindInfo.version = 1;
+ r->unwindInfo.flags = UNW_FLAG_EHANDLER;
+ r->unwindInfo.sizeOfPrologue = 0;
+ r->unwindInfo.countOfUnwindCodes = 0;
+ r->unwindInfo.frameRegister = 0;
+ r->unwindInfo.frameOffset = 0;
+ r->unwindInfo.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);
+
+ // mov imm64, rax
+ r->thunk[0] = 0x48;
+ r->thunk[1] = 0xb8;
+ memcpy(&r->thunk[2], &handler, 8);
+
+ // jmp rax
+ r->thunk[10] = 0xff;
+ r->thunk[11] = 0xe0;
+# endif
+
+ DWORD oldProtect;
+ if (!VirtualProtect(p, pageSize, PAGE_EXECUTE_READ, &oldProtect)) {
+ MOZ_CRASH();
+ }
+
+ // XXX NB: The profiler believes this function is only called from the main
+ // thread. If that ever becomes untrue, the profiler must be updated
+ // immediately.
+ AutoSuppressStackWalking suppress;
+ return RtlInstallFunctionTableCallback((DWORD64)p | 0x3, (DWORD64)p, bytes,
+ RuntimeFunctionCallback, NULL, NULL);
+}
+
+static void UnregisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
+ // There's no such thing as RtlUninstallFunctionTableCallback, so there's
+ // nothing to do here.
+}
+# endif
+
+static void* ReserveProcessExecutableMemory(size_t bytes) {
+# ifdef NEED_JIT_UNWIND_HANDLING
+ size_t pageSize = gc::SystemPageSize();
+ if (sJitExceptionHandler) {
+ bytes += pageSize;
+ }
+# endif
+
+ void* p = nullptr;
+ for (size_t i = 0; i < 10; i++) {
+ void* randomAddr = ComputeRandomAllocationAddress();
+ p = VirtualAlloc(randomAddr, bytes, MEM_RESERVE, PAGE_NOACCESS);
+ if (p) {
+ break;
+ }
+ }
+
+ if (!p) {
+ // Try again without randomization.
+ p = VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_NOACCESS);
+ if (!p) {
+ return nullptr;
+ }
+ }
+
+# ifdef NEED_JIT_UNWIND_HANDLING
+ if (sJitExceptionHandler) {
+ if (!RegisterExecutableMemory(p, bytes, pageSize)) {
+ VirtualFree(p, 0, MEM_RELEASE);
+ return nullptr;
+ }
+
+ p = (uint8_t*)p + pageSize;
+ bytes -= pageSize;
+ }
+
+ RegisterJitCodeRegion((uint8_t*)p, bytes);
+# endif
+
+ return p;
+}
+
+static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
+# ifdef NEED_JIT_UNWIND_HANDLING
+ UnregisterJitCodeRegion((uint8_t*)addr, bytes);
+
+ if (sJitExceptionHandler) {
+ size_t pageSize = gc::SystemPageSize();
+ addr = (uint8_t*)addr - pageSize;
+ UnregisterExecutableMemory(addr, bytes, pageSize);
+ }
+# endif
+
+ VirtualFree(addr, 0, MEM_RELEASE);
+}
+
+static DWORD ProtectionSettingToFlags(ProtectionSetting protection) {
+ switch (protection) {
+ case ProtectionSetting::Protected:
+ return PAGE_NOACCESS;
+ case ProtectionSetting::Writable:
+ return PAGE_READWRITE;
+ case ProtectionSetting::Executable:
+ return PAGE_EXECUTE_READ;
+ }
+ MOZ_CRASH();
+}
+
+[[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
+ ProtectionSetting protection) {
+ void* p = VirtualAlloc(addr, bytes, MEM_COMMIT,
+ ProtectionSettingToFlags(protection));
+ if (!p) {
+ return false;
+ }
+ MOZ_RELEASE_ASSERT(p == addr);
+ return true;
+}
+
+static void DecommitPages(void* addr, size_t bytes) {
+ if (!VirtualFree(addr, bytes, MEM_DECOMMIT)) {
+ MOZ_CRASH("DecommitPages failed");
+ }
+}
+#elif defined(__wasi__)
+# if defined(JS_CODEGEN_WASM32)
+static void* ReserveProcessExecutableMemory(size_t bytes) {
+ return malloc(bytes);
+}
+
+static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
+ free(addr);
+}
+
+[[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
+ ProtectionSetting protection) {
+ return true;
+}
+
+static void DecommitPages(void* addr, size_t bytes) {}
+
+# else
+static void* ReserveProcessExecutableMemory(size_t bytes) {
+ MOZ_CRASH("NYI for WASI.");
+ return nullptr;
+}
+static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
+ MOZ_CRASH("NYI for WASI.");
+}
+[[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
+ ProtectionSetting protection) {
+ MOZ_CRASH("NYI for WASI.");
+ return false;
+}
+static void DecommitPages(void* addr, size_t bytes) {
+ MOZ_CRASH("NYI for WASI.");
+}
+# endif
+#else // !XP_WIN && !__wasi__
+# ifndef MAP_NORESERVE
+# define MAP_NORESERVE 0
+# endif
+
+static void* ComputeRandomAllocationAddress() {
+# ifdef __OpenBSD__
+ // OpenBSD already has random mmap and the idea that all x64 cpus
+ // have 48-bit address space is not correct. Returning nullptr
+ // allows OpenBSD do to the right thing.
+ return nullptr;
+# else
+ uint64_t rand = js::GenerateRandomSeed();
+
+# ifdef HAVE_64BIT_BUILD
+ // x64 CPUs have a 48-bit address space and on some platforms the OS will
+ // give us access to 47 bits, so to be safe we right shift by 18 to leave
+ // 46 bits.
+ rand >>= 18;
+# else
+ // On 32-bit, right shift by 34 to leave 30 bits, range [0, 1GiB). Then add
+ // 512MiB to get range [512MiB, 1.5GiB), or [0x20000000, 0x60000000). This
+ // is based on V8 comments in platform-posix.cc saying this range is
+ // relatively unpopulated across a variety of kernels.
+ rand >>= 34;
+ rand += 512 * 1024 * 1024;
+# endif
+
+ // Ensure page alignment.
+ uintptr_t mask = ~uintptr_t(gc::SystemPageSize() - 1);
+ return (void*)uintptr_t(rand & mask);
+# endif
+}
+
+static void* ReserveProcessExecutableMemory(size_t bytes) {
+ // Note that randomAddr is just a hint: if the address is not available
+ // mmap will pick a different address.
+ void* randomAddr = ComputeRandomAllocationAddress();
+ void* p = MozTaggedAnonymousMmap(randomAddr, bytes, PROT_NONE,
+ MAP_NORESERVE | MAP_PRIVATE | MAP_ANON, -1,
+ 0, "js-executable-memory");
+ if (p == MAP_FAILED) {
+ return nullptr;
+ }
+ return p;
+}
+
+static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
+ mozilla::DebugOnly<int> result = munmap(addr, bytes);
+ MOZ_ASSERT(!result || errno == ENOMEM);
+}
+
+static unsigned ProtectionSettingToFlags(ProtectionSetting protection) {
+# ifdef MOZ_VALGRIND
+ // If we're configured for Valgrind and running on it, use a slacker
+ // scheme that doesn't change execute permissions, since doing so causes
+ // Valgrind a lot of extra overhead re-JITting code that loses and later
+ // regains execute permission. See bug 1338179.
+ if (RUNNING_ON_VALGRIND) {
+ switch (protection) {
+ case ProtectionSetting::Protected:
+ return PROT_NONE;
+ case ProtectionSetting::Writable:
+ return PROT_READ | PROT_WRITE | PROT_EXEC;
+ case ProtectionSetting::Executable:
+ return PROT_READ | PROT_EXEC;
+ }
+ MOZ_CRASH();
+ }
+ // If we get here, we're configured for Valgrind but not running on
+ // it, so use the standard scheme.
+# endif
+ switch (protection) {
+ case ProtectionSetting::Protected:
+ return PROT_NONE;
+ case ProtectionSetting::Writable:
+ return PROT_READ | PROT_WRITE;
+ case ProtectionSetting::Executable:
+ return PROT_READ | PROT_EXEC;
+ }
+ MOZ_CRASH();
+}
+
+[[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
+ ProtectionSetting protection) {
+ void* p = MozTaggedAnonymousMmap(
+ addr, bytes, ProtectionSettingToFlags(protection),
+ MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0, "js-executable-memory");
+ if (p == MAP_FAILED) {
+ return false;
+ }
+ MOZ_RELEASE_ASSERT(p == addr);
+ return true;
+}
+
+static void DecommitPages(void* addr, size_t bytes) {
+ // Use mmap with MAP_FIXED and PROT_NONE. Inspired by jemalloc's
+ // pages_decommit.
+ void* p = MozTaggedAnonymousMmap(addr, bytes, PROT_NONE,
+ MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0,
+ "js-executable-memory");
+ MOZ_RELEASE_ASSERT(addr == p);
+}
+#endif
+
+template <size_t NumBits>
+class PageBitSet {
+ using WordType = uint32_t;
+ static const size_t BitsPerWord = sizeof(WordType) * 8;
+
+ static_assert((NumBits % BitsPerWord) == 0,
+ "NumBits must be a multiple of BitsPerWord");
+ static const size_t NumWords = NumBits / BitsPerWord;
+
+ mozilla::Array<WordType, NumWords> words_;
+
+ uint32_t indexToWord(uint32_t index) const {
+ MOZ_ASSERT(index < NumBits);
+ return index / BitsPerWord;
+ }
+ WordType indexToBit(uint32_t index) const {
+ MOZ_ASSERT(index < NumBits);
+ return WordType(1) << (index % BitsPerWord);
+ }
+
+ public:
+ void init() { mozilla::PodArrayZero(words_); }
+ bool contains(size_t index) const {
+ uint32_t word = indexToWord(index);
+ return words_[word] & indexToBit(index);
+ }
+ void insert(size_t index) {
+ MOZ_ASSERT(!contains(index));
+ uint32_t word = indexToWord(index);
+ words_[word] |= indexToBit(index);
+ }
+ void remove(size_t index) {
+ MOZ_ASSERT(contains(index));
+ uint32_t word = indexToWord(index);
+ words_[word] &= ~indexToBit(index);
+ }
+
+#ifdef DEBUG
+ bool empty() const {
+ for (size_t i = 0; i < NumWords; i++) {
+ if (words_[i] != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+#endif
+};
+
+// Per-process executable memory allocator. It reserves a block of memory of
+// MaxCodeBytesPerProcess bytes, then allocates/deallocates pages from that.
+//
+// This has a number of benefits compared to raw mmap/VirtualAlloc:
+//
+// * More resillient against certain attacks.
+//
+// * Behaves more consistently across platforms: it avoids the 64K granularity
+// issues on Windows, for instance.
+//
+// * On x64, near jumps can be used for jumps to other JIT pages.
+//
+// * On Win64, we have to register the exception handler only once (at process
+// startup). This saves some memory and avoids RtlAddFunctionTable profiler
+// deadlocks.
+class ProcessExecutableMemory {
+ static_assert(
+ (MaxCodeBytesPerProcess % ExecutableCodePageSize) == 0,
+ "MaxCodeBytesPerProcess must be a multiple of ExecutableCodePageSize");
+ static const size_t MaxCodePages =
+ MaxCodeBytesPerProcess / ExecutableCodePageSize;
+
+ // Start of the MaxCodeBytesPerProcess memory block or nullptr if
+ // uninitialized. Note that this is NOT guaranteed to be aligned to
+ // ExecutableCodePageSize.
+ uint8_t* base_;
+
+ // The fields below should only be accessed while we hold the lock.
+ Mutex lock_ MOZ_UNANNOTATED;
+
+ // pagesAllocated_ is an Atomic so that bytesAllocated does not have to
+ // take the lock.
+ mozilla::Atomic<size_t, mozilla::ReleaseAcquire> pagesAllocated_;
+
+ // Page where we should try to allocate next.
+ size_t cursor_;
+
+ mozilla::Maybe<mozilla::non_crypto::XorShift128PlusRNG> rng_;
+ PageBitSet<MaxCodePages> pages_;
+
+ public:
+ ProcessExecutableMemory()
+ : base_(nullptr),
+ lock_(mutexid::ProcessExecutableRegion),
+ pagesAllocated_(0),
+ cursor_(0),
+ rng_(),
+ pages_() {}
+
+ [[nodiscard]] bool init() {
+ pages_.init();
+
+ MOZ_RELEASE_ASSERT(!initialized());
+ MOZ_RELEASE_ASSERT(HasJitBackend());
+ MOZ_RELEASE_ASSERT(gc::SystemPageSize() <= ExecutableCodePageSize);
+
+ void* p = ReserveProcessExecutableMemory(MaxCodeBytesPerProcess);
+ if (!p) {
+ return false;
+ }
+
+ base_ = static_cast<uint8_t*>(p);
+
+ mozilla::Array<uint64_t, 2> seed;
+ GenerateXorShift128PlusSeed(seed);
+ rng_.emplace(seed[0], seed[1]);
+ return true;
+ }
+
+ uint8_t* base() const { return base_; }
+
+ bool initialized() const { return base_ != nullptr; }
+
+ size_t bytesAllocated() const {
+ MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
+ return pagesAllocated_ * ExecutableCodePageSize;
+ }
+
+ void release() {
+ MOZ_ASSERT(initialized());
+ MOZ_ASSERT(pages_.empty());
+ MOZ_ASSERT(pagesAllocated_ == 0);
+ DeallocateProcessExecutableMemory(base_, MaxCodeBytesPerProcess);
+ base_ = nullptr;
+ rng_.reset();
+ MOZ_ASSERT(!initialized());
+ }
+
+ void assertValidAddress(void* p, size_t bytes) const {
+ MOZ_RELEASE_ASSERT(p >= base_ &&
+ uintptr_t(p) + bytes <=
+ uintptr_t(base_) + MaxCodeBytesPerProcess);
+ }
+
+ bool containsAddress(const void* p) const {
+ return p >= base_ &&
+ uintptr_t(p) < uintptr_t(base_) + MaxCodeBytesPerProcess;
+ }
+
+ void* allocate(size_t bytes, ProtectionSetting protection,
+ MemCheckKind checkKind);
+ void deallocate(void* addr, size_t bytes, bool decommit);
+};
+
+void* ProcessExecutableMemory::allocate(size_t bytes,
+ ProtectionSetting protection,
+ MemCheckKind checkKind) {
+ MOZ_ASSERT(initialized());
+ MOZ_ASSERT(HasJitBackend());
+ MOZ_ASSERT(bytes > 0);
+ MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);
+
+ size_t numPages = bytes / ExecutableCodePageSize;
+
+ // Take the lock and try to allocate.
+ void* p = nullptr;
+ {
+ LockGuard<Mutex> guard(lock_);
+ MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
+
+ // Check if we have enough pages available.
+ if (pagesAllocated_ + numPages >= MaxCodePages) {
+ return nullptr;
+ }
+
+ MOZ_ASSERT(bytes <= MaxCodeBytesPerProcess);
+
+ // Maybe skip a page to make allocations less predictable.
+ size_t page = cursor_ + (rng_.ref().next() % 2);
+
+ for (size_t i = 0; i < MaxCodePages; i++) {
+ // Make sure page + numPages - 1 is a valid index.
+ if (page + numPages > MaxCodePages) {
+ page = 0;
+ }
+
+ bool available = true;
+ for (size_t j = 0; j < numPages; j++) {
+ if (pages_.contains(page + j)) {
+ available = false;
+ break;
+ }
+ }
+ if (!available) {
+ page++;
+ continue;
+ }
+
+ // Mark the pages as unavailable.
+ for (size_t j = 0; j < numPages; j++) {
+ pages_.insert(page + j);
+ }
+
+ pagesAllocated_ += numPages;
+ MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
+
+ // If we allocated a small number of pages, move cursor_ to the
+ // next page. We don't do this for larger allocations to avoid
+ // skipping a large number of small holes.
+ if (numPages <= 2) {
+ cursor_ = page + numPages;
+ }
+
+ p = base_ + page * ExecutableCodePageSize;
+ break;
+ }
+ if (!p) {
+ return nullptr;
+ }
+ }
+
+ // Commit the pages after releasing the lock.
+ if (!CommitPages(p, bytes, protection)) {
+ deallocate(p, bytes, /* decommit = */ false);
+ return nullptr;
+ }
+
+ SetMemCheckKind(p, bytes, checkKind);
+
+ return p;
+}
+
+void ProcessExecutableMemory::deallocate(void* addr, size_t bytes,
+ bool decommit) {
+ MOZ_ASSERT(initialized());
+ MOZ_ASSERT(addr);
+ MOZ_ASSERT((uintptr_t(addr) % gc::SystemPageSize()) == 0);
+ MOZ_ASSERT(bytes > 0);
+ MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);
+
+ assertValidAddress(addr, bytes);
+
+ size_t firstPage =
+ (static_cast<uint8_t*>(addr) - base_) / ExecutableCodePageSize;
+ size_t numPages = bytes / ExecutableCodePageSize;
+
+ // Decommit before taking the lock.
+ MOZ_MAKE_MEM_NOACCESS(addr, bytes);
+ if (decommit) {
+ DecommitPages(addr, bytes);
+ }
+
+ LockGuard<Mutex> guard(lock_);
+ MOZ_ASSERT(numPages <= pagesAllocated_);
+ pagesAllocated_ -= numPages;
+
+ for (size_t i = 0; i < numPages; i++) {
+ pages_.remove(firstPage + i);
+ }
+
+ // Move the cursor back so we can reuse pages instead of fragmenting the
+ // whole region.
+ if (firstPage < cursor_) {
+ cursor_ = firstPage;
+ }
+}
+
+static ProcessExecutableMemory execMemory;
+
+void* js::jit::AllocateExecutableMemory(size_t bytes,
+ ProtectionSetting protection,
+ MemCheckKind checkKind) {
+ return execMemory.allocate(bytes, protection, checkKind);
+}
+
+void js::jit::DeallocateExecutableMemory(void* addr, size_t bytes) {
+ execMemory.deallocate(addr, bytes, /* decommit = */ true);
+}
+
+bool js::jit::InitProcessExecutableMemory() { return execMemory.init(); }
+
+void js::jit::ReleaseProcessExecutableMemory() { execMemory.release(); }
+
+size_t js::jit::LikelyAvailableExecutableMemory() {
+ // Round down available memory to the closest MB.
+ return MaxCodeBytesPerProcess -
+ AlignBytes(execMemory.bytesAllocated(), 0x100000U);
+}
+
+bool js::jit::CanLikelyAllocateMoreExecutableMemory() {
+ // Use a 8 MB buffer.
+ static const size_t BufferSize = 8 * 1024 * 1024;
+
+ MOZ_ASSERT(execMemory.bytesAllocated() <= MaxCodeBytesPerProcess);
+
+ return execMemory.bytesAllocated() + BufferSize <= MaxCodeBytesPerProcess;
+}
+
+bool js::jit::AddressIsInExecutableMemory(const void* p) {
+ return execMemory.containsAddress(p);
+}
+
+bool js::jit::ReprotectRegion(void* start, size_t size,
+ ProtectionSetting protection,
+ MustFlushICache flushICache) {
+#if defined(JS_CODEGEN_WASM32)
+ return true;
+#endif
+
+ // Flush ICache when making code executable, before we modify |size|.
+ if (flushICache == MustFlushICache::Yes) {
+ MOZ_ASSERT(protection == ProtectionSetting::Executable);
+ jit::FlushICache(start, size);
+ }
+
+ // Calculate the start of the page containing this region,
+ // and account for this extra memory within size.
+ size_t pageSize = gc::SystemPageSize();
+ intptr_t startPtr = reinterpret_cast<intptr_t>(start);
+ intptr_t pageStartPtr = startPtr & ~(pageSize - 1);
+ void* pageStart = reinterpret_cast<void*>(pageStartPtr);
+ size += (startPtr - pageStartPtr);
+
+ // Round size up
+ size += (pageSize - 1);
+ size &= ~(pageSize - 1);
+
+ MOZ_ASSERT((uintptr_t(pageStart) % pageSize) == 0);
+
+ execMemory.assertValidAddress(pageStart, size);
+
+ // On weak memory systems, make sure new code is visible on all cores before
+ // addresses of the code are made public. Now is the latest moment in time
+ // when we can do that, and we're assuming that every other thread that has
+ // written into the memory that is being reprotected here has synchronized
+ // with this thread in such a way that the memory writes have become visible
+ // and we therefore only need to execute the fence once here. See bug 1529933
+ // for a longer discussion of why this is both necessary and sufficient.
+ //
+ // We use the C++ fence here -- and not AtomicOperations::fenceSeqCst() --
+ // primarily because ReprotectRegion will be called while we construct our own
+ // jitted atomics. But the C++ fence is sufficient and correct, too.
+#ifdef __wasi__
+ MOZ_CRASH("NYI FOR WASI.");
+#else
+ std::atomic_thread_fence(std::memory_order_seq_cst);
+
+# ifdef XP_WIN
+ DWORD oldProtect;
+ DWORD flags = ProtectionSettingToFlags(protection);
+ if (!VirtualProtect(pageStart, size, flags, &oldProtect)) {
+ return false;
+ }
+# else
+ unsigned flags = ProtectionSettingToFlags(protection);
+ if (mprotect(pageStart, size, flags)) {
+ return false;
+ }
+# endif
+#endif // __wasi__
+
+ execMemory.assertValidAddress(pageStart, size);
+ return true;
+}
+
+#if defined(XP_WIN) && defined(NEED_JIT_UNWIND_HANDLING)
+static PRUNTIME_FUNCTION RuntimeFunctionCallback(DWORD64 ControlPc,
+ PVOID Context) {
+ MOZ_ASSERT(sJitExceptionHandler);
+
+ // RegisterExecutableMemory already set up the runtime function in the
+ // exception-data page preceding the allocation.
+ uint8_t* p = execMemory.base();
+ if (!p) {
+ return nullptr;
+ }
+ return (PRUNTIME_FUNCTION)(p - gc::SystemPageSize());
+}
+#endif
diff --git a/js/src/jit/ProcessExecutableMemory.h b/js/src/jit/ProcessExecutableMemory.h
new file mode 100644
index 0000000000..51747634f3
--- /dev/null
+++ b/js/src/jit/ProcessExecutableMemory.h
@@ -0,0 +1,109 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ProcessExecutableMemory_h
+#define jit_ProcessExecutableMemory_h
+
+#include "util/Poison.h"
+
+namespace js {
+namespace jit {
+
+// Limit on the number of bytes of executable memory to prevent JIT spraying
+// attacks.
+#if JS_BITS_PER_WORD == 32
+static const size_t MaxCodeBytesPerProcess = 140 * 1024 * 1024;
+#else
+// This is the largest number which satisfies various alignment static
+// asserts that is <= INT32_MAX. The INT32_MAX limit is required for making a
+// single call to RtlInstallFunctionTableCallback(). (This limit could be
+// relaxed in the future by making multiple calls.)
+static const size_t MaxCodeBytesPerProcess = 2044 * 1024 * 1024;
+#endif
+
+// Limit on the number of bytes of code memory per buffer. This limit comes
+// about because we encode an unresolved relative unconditional branch during
+// assembly as a branch instruction that carries the absolute offset of the next
+// branch instruction in the chain of branches that all reference the same
+// unresolved label. For this architecture to work, no branch instruction may
+// lie at an offset greater than the maximum forward branch distance. This is
+// true on both ARM and ARM64.
+//
+// Notably, even though we know that the offsets thus encoded are always
+// positive offsets, we use only the positive part of the signed range of the
+// branch offset.
+//
+// On ARM-32, we are limited by BOffImm::IsInRange(), which checks that the
+// offset is no greater than 2^25-4 in the offset's 26-bit signed field.
+//
+// On ARM-64, we are limited by Instruction::ImmBranchMaxForwardOffset(), which
+// checks that the offset is no greater than 2^27-4 in the offset's 28-bit
+// signed field.
+//
+// On MIPS, there are no limitations because the assembler has to implement
+// jump chaining to be effective at all (jump offsets are quite small).
+//
+// On x86 and x64, there are no limitations here because the assembler
+// MOZ_CRASHes if the 32-bit offset is exceeded.
+
+#if defined(JS_CODEGEN_ARM)
+static const size_t MaxCodeBytesPerBuffer = (1 << 25) - 4;
+#elif defined(JS_CODEGEN_ARM64)
+static const size_t MaxCodeBytesPerBuffer = (1 << 27) - 4;
+#else
+static const size_t MaxCodeBytesPerBuffer = MaxCodeBytesPerProcess;
+#endif
+
+// Executable code is allocated in 64K chunks. ExecutableAllocator uses pools
+// that are at least this big. Code we allocate does not necessarily have 64K
+// alignment though.
+static const size_t ExecutableCodePageSize = 64 * 1024;
+
+enum class ProtectionSetting {
+ Protected, // Not readable, writable, or executable.
+ Writable,
+ Executable,
+};
+
+/// Whether the instruction cache must be flushed
+
+enum class MustFlushICache { No, Yes };
+
+[[nodiscard]] extern bool ReprotectRegion(void* start, size_t size,
+ ProtectionSetting protection,
+ MustFlushICache flushICache);
+
+// Functions called at process start-up/shutdown to initialize/release the
+// executable memory region.
+[[nodiscard]] extern bool InitProcessExecutableMemory();
+extern void ReleaseProcessExecutableMemory();
+
+// Allocate/deallocate executable pages.
+extern void* AllocateExecutableMemory(size_t bytes,
+ ProtectionSetting protection,
+ MemCheckKind checkKind);
+extern void DeallocateExecutableMemory(void* addr, size_t bytes);
+
+// Returns true if we can allocate a few more MB of executable code without
+// hitting our code limit. This function can be used to stop compiling things
+// that are optional (like Baseline and Ion code) when we're about to reach the
+// limit, so we are less likely to OOM or crash. Note that the limit is
+// per-process, so other threads can also allocate code after we call this
+// function.
+extern bool CanLikelyAllocateMoreExecutableMemory();
+
+// Returns a rough guess of how much executable memory remains available,
+// rounded down to MB limit. Note this can fluctuate as other threads within
+// the process allocate executable memory.
+extern size_t LikelyAvailableExecutableMemory();
+
+// Returns whether |p| is stored in the executable code buffer.
+extern bool AddressIsInExecutableMemory(const void* p);
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_ProcessExecutableMemory_h
diff --git a/js/src/jit/RangeAnalysis.cpp b/js/src/jit/RangeAnalysis.cpp
new file mode 100644
index 0000000000..2cbf0ccc7a
--- /dev/null
+++ b/js/src/jit/RangeAnalysis.cpp
@@ -0,0 +1,3697 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/RangeAnalysis.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+
+#include "jsmath.h"
+
+#include "jit/CompileInfo.h"
+#include "jit/IonAnalysis.h"
+#include "jit/JitSpewer.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "js/Conversions.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "util/CheckedArithmetic.h"
+#include "vm/ArgumentsObject.h"
+#include "vm/TypedArrayObject.h"
+#include "vm/Uint8Clamped.h"
+
+#include "vm/BytecodeUtil-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::GenericNaN;
+using JS::ToInt32;
+using mozilla::Abs;
+using mozilla::CountLeadingZeroes32;
+using mozilla::ExponentComponent;
+using mozilla::FloorLog2;
+using mozilla::IsInfinite;
+using mozilla::IsNaN;
+using mozilla::IsNegativeZero;
+using mozilla::NegativeInfinity;
+using mozilla::NumberEqualsInt32;
+using mozilla::PositiveInfinity;
+
+// [SMDOC] IonMonkey Range Analysis
+//
+// This algorithm is based on the paper "Eliminating Range Checks Using
+// Static Single Assignment Form" by Gough and Klaren.
+//
+// We associate a range object with each SSA name, and the ranges are consulted
+// in order to determine whether overflow is possible for arithmetic
+// computations.
+//
+// An important source of range information that requires care to take
+// advantage of is conditional control flow. Consider the code below:
+//
+// if (x < 0) {
+// y = x + 2000000000;
+// } else {
+// if (x < 1000000000) {
+// y = x * 2;
+// } else {
+// y = x - 3000000000;
+// }
+// }
+//
+// The arithmetic operations in this code cannot overflow, but it is not
+// sufficient to simply associate each name with a range, since the information
+// differs between basic blocks. The traditional dataflow approach would be
+// associate ranges with (name, basic block) pairs. This solution is not
+// satisfying, since we lose the benefit of SSA form: in SSA form, each
+// definition has a unique name, so there is no need to track information about
+// the control flow of the program.
+//
+// The approach used here is to add a new form of pseudo operation called a
+// beta node, which associates range information with a value. These beta
+// instructions take one argument and additionally have an auxiliary constant
+// range associated with them. Operationally, beta nodes are just copies, but
+// the invariant expressed by beta node copies is that the output will fall
+// inside the range given by the beta node. Gough and Klaeren refer to SSA
+// extended with these beta nodes as XSA form. The following shows the example
+// code transformed into XSA form:
+//
+// if (x < 0) {
+// x1 = Beta(x, [INT_MIN, -1]);
+// y1 = x1 + 2000000000;
+// } else {
+// x2 = Beta(x, [0, INT_MAX]);
+// if (x2 < 1000000000) {
+// x3 = Beta(x2, [INT_MIN, 999999999]);
+// y2 = x3*2;
+// } else {
+// x4 = Beta(x2, [1000000000, INT_MAX]);
+// y3 = x4 - 3000000000;
+// }
+// y4 = Phi(y2, y3);
+// }
+// y = Phi(y1, y4);
+//
+// We insert beta nodes for the purposes of range analysis (they might also be
+// usefully used for other forms of bounds check elimination) and remove them
+// after range analysis is performed. The remaining compiler phases do not ever
+// encounter beta nodes.
+
+static bool IsDominatedUse(MBasicBlock* block, MUse* use) {
+ MNode* n = use->consumer();
+ bool isPhi = n->isDefinition() && n->toDefinition()->isPhi();
+
+ if (isPhi) {
+ MPhi* phi = n->toDefinition()->toPhi();
+ return block->dominates(phi->block()->getPredecessor(phi->indexOf(use)));
+ }
+
+ return block->dominates(n->block());
+}
+
+static inline void SpewRange(MDefinition* def) {
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_Range) && def->type() != MIRType::None &&
+ def->range()) {
+ JitSpewHeader(JitSpew_Range);
+ Fprinter& out = JitSpewPrinter();
+ out.printf(" ");
+ def->printName(out);
+ out.printf(" has range ");
+ def->range()->dump(out);
+ out.printf("\n");
+ }
+#endif
+}
+
+#ifdef JS_JITSPEW
+static const char* TruncateKindString(TruncateKind kind) {
+ switch (kind) {
+ case TruncateKind::NoTruncate:
+ return "NoTruncate";
+ case TruncateKind::TruncateAfterBailouts:
+ return "TruncateAfterBailouts";
+ case TruncateKind::IndirectTruncate:
+ return "IndirectTruncate";
+ case TruncateKind::Truncate:
+ return "Truncate";
+ default:
+ MOZ_CRASH("Unknown truncate kind.");
+ }
+}
+
+static inline void SpewTruncate(MDefinition* def, TruncateKind kind,
+ bool shouldClone) {
+ if (JitSpewEnabled(JitSpew_Range)) {
+ JitSpewHeader(JitSpew_Range);
+ Fprinter& out = JitSpewPrinter();
+ out.printf(" ");
+ out.printf("truncating ");
+ def->printName(out);
+ out.printf(" (kind: %s, clone: %d)\n", TruncateKindString(kind),
+ shouldClone);
+ }
+}
+#else
+static inline void SpewTruncate(MDefinition* def, TruncateKind kind,
+ bool shouldClone) {}
+#endif
+
+TempAllocator& RangeAnalysis::alloc() const { return graph_.alloc(); }
+
+void RangeAnalysis::replaceDominatedUsesWith(MDefinition* orig,
+ MDefinition* dom,
+ MBasicBlock* block) {
+ for (MUseIterator i(orig->usesBegin()); i != orig->usesEnd();) {
+ MUse* use = *i++;
+ if (use->consumer() != dom && IsDominatedUse(block, use)) {
+ use->replaceProducer(dom);
+ }
+ }
+}
+
+bool RangeAnalysis::addBetaNodes() {
+ JitSpew(JitSpew_Range, "Adding beta nodes");
+
+ for (PostorderIterator i(graph_.poBegin()); i != graph_.poEnd(); i++) {
+ MBasicBlock* block = *i;
+ JitSpew(JitSpew_Range, "Looking at block %u", block->id());
+
+ BranchDirection branch_dir;
+ MTest* test = block->immediateDominatorBranch(&branch_dir);
+
+ if (!test || !test->getOperand(0)->isCompare()) {
+ continue;
+ }
+
+ MCompare* compare = test->getOperand(0)->toCompare();
+
+ if (!compare->isNumericComparison()) {
+ continue;
+ }
+
+ // TODO: support unsigned comparisons
+ if (compare->compareType() == MCompare::Compare_UInt32) {
+ continue;
+ }
+
+ // isNumericComparison should return false for UIntPtr.
+ MOZ_ASSERT(compare->compareType() != MCompare::Compare_UIntPtr);
+
+ MDefinition* left = compare->getOperand(0);
+ MDefinition* right = compare->getOperand(1);
+ double bound;
+ double conservativeLower = NegativeInfinity<double>();
+ double conservativeUpper = PositiveInfinity<double>();
+ MDefinition* val = nullptr;
+
+ JSOp jsop = compare->jsop();
+
+ if (branch_dir == FALSE_BRANCH) {
+ jsop = NegateCompareOp(jsop);
+ conservativeLower = GenericNaN();
+ conservativeUpper = GenericNaN();
+ }
+
+ MConstant* leftConst = left->maybeConstantValue();
+ MConstant* rightConst = right->maybeConstantValue();
+ if (leftConst && leftConst->isTypeRepresentableAsDouble()) {
+ bound = leftConst->numberToDouble();
+ val = right;
+ jsop = ReverseCompareOp(jsop);
+ } else if (rightConst && rightConst->isTypeRepresentableAsDouble()) {
+ bound = rightConst->numberToDouble();
+ val = left;
+ } else if (left->type() == MIRType::Int32 &&
+ right->type() == MIRType::Int32) {
+ MDefinition* smaller = nullptr;
+ MDefinition* greater = nullptr;
+ if (jsop == JSOp::Lt) {
+ smaller = left;
+ greater = right;
+ } else if (jsop == JSOp::Gt) {
+ smaller = right;
+ greater = left;
+ }
+ if (smaller && greater) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ MBeta* beta;
+ beta = MBeta::New(
+ alloc(), smaller,
+ Range::NewInt32Range(alloc(), JSVAL_INT_MIN, JSVAL_INT_MAX - 1));
+ block->insertBefore(*block->begin(), beta);
+ replaceDominatedUsesWith(smaller, beta, block);
+ JitSpew(JitSpew_Range, " Adding beta node for smaller %u",
+ smaller->id());
+ beta = MBeta::New(
+ alloc(), greater,
+ Range::NewInt32Range(alloc(), JSVAL_INT_MIN + 1, JSVAL_INT_MAX));
+ block->insertBefore(*block->begin(), beta);
+ replaceDominatedUsesWith(greater, beta, block);
+ JitSpew(JitSpew_Range, " Adding beta node for greater %u",
+ greater->id());
+ }
+ continue;
+ } else {
+ continue;
+ }
+
+ // At this point, one of the operands if the compare is a constant, and
+ // val is the other operand.
+ MOZ_ASSERT(val);
+
+ Range comp;
+ switch (jsop) {
+ case JSOp::Le:
+ comp.setDouble(conservativeLower, bound);
+ break;
+ case JSOp::Lt:
+ // For integers, if x < c, the upper bound of x is c-1.
+ if (val->type() == MIRType::Int32) {
+ int32_t intbound;
+ if (NumberEqualsInt32(bound, &intbound) &&
+ SafeSub(intbound, 1, &intbound)) {
+ bound = intbound;
+ }
+ }
+ comp.setDouble(conservativeLower, bound);
+
+ // Negative zero is not less than zero.
+ if (bound == 0) {
+ comp.refineToExcludeNegativeZero();
+ }
+ break;
+ case JSOp::Ge:
+ comp.setDouble(bound, conservativeUpper);
+ break;
+ case JSOp::Gt:
+ // For integers, if x > c, the lower bound of x is c+1.
+ if (val->type() == MIRType::Int32) {
+ int32_t intbound;
+ if (NumberEqualsInt32(bound, &intbound) &&
+ SafeAdd(intbound, 1, &intbound)) {
+ bound = intbound;
+ }
+ }
+ comp.setDouble(bound, conservativeUpper);
+
+ // Negative zero is not greater than zero.
+ if (bound == 0) {
+ comp.refineToExcludeNegativeZero();
+ }
+ break;
+ case JSOp::StrictEq:
+ case JSOp::Eq:
+ comp.setDouble(bound, bound);
+ break;
+ case JSOp::StrictNe:
+ case JSOp::Ne:
+ // Negative zero is not not-equal to zero.
+ if (bound == 0) {
+ comp.refineToExcludeNegativeZero();
+ break;
+ }
+ continue; // well, we could have
+ // [-\inf, bound-1] U [bound+1, \inf] but we only use
+ // contiguous ranges.
+ default:
+ continue;
+ }
+
+ if (JitSpewEnabled(JitSpew_Range)) {
+ JitSpewHeader(JitSpew_Range);
+ Fprinter& out = JitSpewPrinter();
+ out.printf(" Adding beta node for %u with range ", val->id());
+ comp.dump(out);
+ out.printf("\n");
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ MBeta* beta = MBeta::New(alloc(), val, new (alloc()) Range(comp));
+ block->insertBefore(*block->begin(), beta);
+ replaceDominatedUsesWith(val, beta, block);
+ }
+
+ return true;
+}
+
+bool RangeAnalysis::removeBetaNodes() {
+ JitSpew(JitSpew_Range, "Removing beta nodes");
+
+ for (PostorderIterator i(graph_.poBegin()); i != graph_.poEnd(); i++) {
+ MBasicBlock* block = *i;
+ for (MDefinitionIterator iter(*i); iter;) {
+ MDefinition* def = *iter++;
+ if (def->isBeta()) {
+ auto* beta = def->toBeta();
+ MDefinition* op = beta->input();
+ JitSpew(JitSpew_Range, " Removing beta node %u for %u", beta->id(),
+ op->id());
+ beta->justReplaceAllUsesWith(op);
+ block->discard(beta);
+ } else {
+ // We only place Beta nodes at the beginning of basic
+ // blocks, so if we see something else, we can move on
+ // to the next block.
+ break;
+ }
+ }
+ }
+ return true;
+}
+
+void SymbolicBound::dump(GenericPrinter& out) const {
+ if (loop) {
+ out.printf("[loop] ");
+ }
+ sum.dump(out);
+}
+
+void SymbolicBound::dump() const {
+ Fprinter out(stderr);
+ dump(out);
+ out.printf("\n");
+ out.finish();
+}
+
+// Test whether the given range's exponent tells us anything that its lower
+// and upper bound values don't.
+static bool IsExponentInteresting(const Range* r) {
+ // If it lacks either a lower or upper bound, the exponent is interesting.
+ if (!r->hasInt32Bounds()) {
+ return true;
+ }
+
+ // Otherwise if there's no fractional part, the lower and upper bounds,
+ // which are integers, are perfectly precise.
+ if (!r->canHaveFractionalPart()) {
+ return false;
+ }
+
+ // Otherwise, if the bounds are conservatively rounded across a power-of-two
+ // boundary, the exponent may imply a tighter range.
+ return FloorLog2(std::max(Abs(r->lower()), Abs(r->upper()))) > r->exponent();
+}
+
+void Range::dump(GenericPrinter& out) const {
+ assertInvariants();
+
+ // Floating-point or Integer subset.
+ if (canHaveFractionalPart_) {
+ out.printf("F");
+ } else {
+ out.printf("I");
+ }
+
+ out.printf("[");
+
+ if (!hasInt32LowerBound_) {
+ out.printf("?");
+ } else {
+ out.printf("%d", lower_);
+ }
+ if (symbolicLower_) {
+ out.printf(" {");
+ symbolicLower_->dump(out);
+ out.printf("}");
+ }
+
+ out.printf(", ");
+
+ if (!hasInt32UpperBound_) {
+ out.printf("?");
+ } else {
+ out.printf("%d", upper_);
+ }
+ if (symbolicUpper_) {
+ out.printf(" {");
+ symbolicUpper_->dump(out);
+ out.printf("}");
+ }
+
+ out.printf("]");
+
+ bool includesNaN = max_exponent_ == IncludesInfinityAndNaN;
+ bool includesNegativeInfinity =
+ max_exponent_ >= IncludesInfinity && !hasInt32LowerBound_;
+ bool includesPositiveInfinity =
+ max_exponent_ >= IncludesInfinity && !hasInt32UpperBound_;
+ bool includesNegativeZero = canBeNegativeZero_;
+
+ if (includesNaN || includesNegativeInfinity || includesPositiveInfinity ||
+ includesNegativeZero) {
+ out.printf(" (");
+ bool first = true;
+ if (includesNaN) {
+ if (first) {
+ first = false;
+ } else {
+ out.printf(" ");
+ }
+ out.printf("U NaN");
+ }
+ if (includesNegativeInfinity) {
+ if (first) {
+ first = false;
+ } else {
+ out.printf(" ");
+ }
+ out.printf("U -Infinity");
+ }
+ if (includesPositiveInfinity) {
+ if (first) {
+ first = false;
+ } else {
+ out.printf(" ");
+ }
+ out.printf("U Infinity");
+ }
+ if (includesNegativeZero) {
+ if (first) {
+ first = false;
+ } else {
+ out.printf(" ");
+ }
+ out.printf("U -0");
+ }
+ out.printf(")");
+ }
+ if (max_exponent_ < IncludesInfinity && IsExponentInteresting(this)) {
+ out.printf(" (< pow(2, %d+1))", max_exponent_);
+ }
+}
+
+void Range::dump() const {
+ Fprinter out(stderr);
+ dump(out);
+ out.printf("\n");
+ out.finish();
+}
+
+Range* Range::intersect(TempAllocator& alloc, const Range* lhs,
+ const Range* rhs, bool* emptyRange) {
+ *emptyRange = false;
+
+ if (!lhs && !rhs) {
+ return nullptr;
+ }
+
+ if (!lhs) {
+ return new (alloc) Range(*rhs);
+ }
+ if (!rhs) {
+ return new (alloc) Range(*lhs);
+ }
+
+ int32_t newLower = std::max(lhs->lower_, rhs->lower_);
+ int32_t newUpper = std::min(lhs->upper_, rhs->upper_);
+
+ // If upper < lower, then we have conflicting constraints. Consider:
+ //
+ // if (x < 0) {
+ // if (x > 0) {
+ // [Some code.]
+ // }
+ // }
+ //
+ // In this case, the block is unreachable.
+ if (newUpper < newLower) {
+ // If both ranges can be NaN, the result can still be NaN.
+ if (!lhs->canBeNaN() || !rhs->canBeNaN()) {
+ *emptyRange = true;
+ }
+ return nullptr;
+ }
+
+ bool newHasInt32LowerBound =
+ lhs->hasInt32LowerBound_ || rhs->hasInt32LowerBound_;
+ bool newHasInt32UpperBound =
+ lhs->hasInt32UpperBound_ || rhs->hasInt32UpperBound_;
+
+ FractionalPartFlag newCanHaveFractionalPart = FractionalPartFlag(
+ lhs->canHaveFractionalPart_ && rhs->canHaveFractionalPart_);
+ NegativeZeroFlag newMayIncludeNegativeZero =
+ NegativeZeroFlag(lhs->canBeNegativeZero_ && rhs->canBeNegativeZero_);
+
+ uint16_t newExponent = std::min(lhs->max_exponent_, rhs->max_exponent_);
+
+ // NaN is a special value which is neither greater than infinity or less than
+ // negative infinity. When we intersect two ranges like [?, 0] and [0, ?], we
+ // can end up thinking we have both a lower and upper bound, even though NaN
+ // is still possible. In this case, just be conservative, since any case where
+ // we can have NaN is not especially interesting.
+ if (newHasInt32LowerBound && newHasInt32UpperBound &&
+ newExponent == IncludesInfinityAndNaN) {
+ return nullptr;
+ }
+
+ // If one of the ranges has a fractional part and the other doesn't, it's
+ // possible that we will have computed a newExponent that's more precise
+ // than our newLower and newUpper. This is unusual, so we handle it here
+ // instead of in optimize().
+ //
+ // For example, consider the range F[0,1.5]. Range analysis represents the
+ // lower and upper bound as integers, so we'd actually have
+ // F[0,2] (< pow(2, 0+1)). In this case, the exponent gives us a slightly
+ // more precise upper bound than the integer upper bound.
+ //
+ // When intersecting such a range with an integer range, the fractional part
+ // of the range is dropped. The max exponent of 0 remains valid, so the
+ // upper bound needs to be adjusted to 1.
+ //
+ // When intersecting F[0,2] (< pow(2, 0+1)) with a range like F[2,4],
+ // the naive intersection is I[2,2], but since the max exponent tells us
+ // that the value is always less than 2, the intersection is actually empty.
+ if (lhs->canHaveFractionalPart() != rhs->canHaveFractionalPart() ||
+ (lhs->canHaveFractionalPart() && newHasInt32LowerBound &&
+ newHasInt32UpperBound && newLower == newUpper)) {
+ refineInt32BoundsByExponent(newExponent, &newLower, &newHasInt32LowerBound,
+ &newUpper, &newHasInt32UpperBound);
+
+ // If we're intersecting two ranges that don't overlap, this could also
+ // push the bounds past each other, since the actual intersection is
+ // the empty set.
+ if (newLower > newUpper) {
+ *emptyRange = true;
+ return nullptr;
+ }
+ }
+
+ return new (alloc)
+ Range(newLower, newHasInt32LowerBound, newUpper, newHasInt32UpperBound,
+ newCanHaveFractionalPart, newMayIncludeNegativeZero, newExponent);
+}
+
+void Range::unionWith(const Range* other) {
+ int32_t newLower = std::min(lower_, other->lower_);
+ int32_t newUpper = std::max(upper_, other->upper_);
+
+ bool newHasInt32LowerBound =
+ hasInt32LowerBound_ && other->hasInt32LowerBound_;
+ bool newHasInt32UpperBound =
+ hasInt32UpperBound_ && other->hasInt32UpperBound_;
+
+ FractionalPartFlag newCanHaveFractionalPart = FractionalPartFlag(
+ canHaveFractionalPart_ || other->canHaveFractionalPart_);
+ NegativeZeroFlag newMayIncludeNegativeZero =
+ NegativeZeroFlag(canBeNegativeZero_ || other->canBeNegativeZero_);
+
+ uint16_t newExponent = std::max(max_exponent_, other->max_exponent_);
+
+ rawInitialize(newLower, newHasInt32LowerBound, newUpper,
+ newHasInt32UpperBound, newCanHaveFractionalPart,
+ newMayIncludeNegativeZero, newExponent);
+}
+
+Range::Range(const MDefinition* def)
+ : symbolicLower_(nullptr), symbolicUpper_(nullptr) {
+ if (const Range* other = def->range()) {
+ // The instruction has range information; use it.
+ *this = *other;
+
+ // Simulate the effect of converting the value to its type.
+ // Note: we cannot clamp here, since ranges aren't allowed to shrink
+ // and truncation can increase range again. So doing wrapAround to
+ // mimick a possible truncation.
+ switch (def->type()) {
+ case MIRType::Int32:
+ // MToNumberInt32 cannot truncate. So we can safely clamp.
+ if (def->isToNumberInt32()) {
+ clampToInt32();
+ } else {
+ wrapAroundToInt32();
+ }
+ break;
+ case MIRType::Boolean:
+ wrapAroundToBoolean();
+ break;
+ case MIRType::None:
+ MOZ_CRASH("Asking for the range of an instruction with no value");
+ default:
+ break;
+ }
+ } else {
+ // Otherwise just use type information. We can trust the type here
+ // because we don't care what value the instruction actually produces,
+ // but what value we might get after we get past the bailouts.
+ switch (def->type()) {
+ case MIRType::Int32:
+ setInt32(JSVAL_INT_MIN, JSVAL_INT_MAX);
+ break;
+ case MIRType::Boolean:
+ setInt32(0, 1);
+ break;
+ case MIRType::None:
+ MOZ_CRASH("Asking for the range of an instruction with no value");
+ default:
+ setUnknown();
+ break;
+ }
+ }
+
+ // As a special case, MUrsh is permitted to claim a result type of
+ // MIRType::Int32 while actually returning values in [0,UINT32_MAX] without
+ // bailouts. If range analysis hasn't ruled out values in
+ // (INT32_MAX,UINT32_MAX], set the range to be conservatively correct for
+ // use as either a uint32 or an int32.
+ if (!hasInt32UpperBound() && def->isUrsh() &&
+ def->toUrsh()->bailoutsDisabled() && def->type() != MIRType::Int64) {
+ lower_ = INT32_MIN;
+ }
+
+ assertInvariants();
+}
+
+static uint16_t ExponentImpliedByDouble(double d) {
+ // Handle the special values.
+ if (IsNaN(d)) {
+ return Range::IncludesInfinityAndNaN;
+ }
+ if (IsInfinite(d)) {
+ return Range::IncludesInfinity;
+ }
+
+ // Otherwise take the exponent part and clamp it at zero, since the Range
+ // class doesn't track fractional ranges.
+ return uint16_t(std::max(int_fast16_t(0), ExponentComponent(d)));
+}
+
+void Range::setDouble(double l, double h) {
+ MOZ_ASSERT(!(l > h));
+
+ // Infer lower_, upper_, hasInt32LowerBound_, and hasInt32UpperBound_.
+ if (l >= INT32_MIN && l <= INT32_MAX) {
+ lower_ = int32_t(::floor(l));
+ hasInt32LowerBound_ = true;
+ } else if (l >= INT32_MAX) {
+ lower_ = INT32_MAX;
+ hasInt32LowerBound_ = true;
+ } else {
+ lower_ = INT32_MIN;
+ hasInt32LowerBound_ = false;
+ }
+ if (h >= INT32_MIN && h <= INT32_MAX) {
+ upper_ = int32_t(::ceil(h));
+ hasInt32UpperBound_ = true;
+ } else if (h <= INT32_MIN) {
+ upper_ = INT32_MIN;
+ hasInt32UpperBound_ = true;
+ } else {
+ upper_ = INT32_MAX;
+ hasInt32UpperBound_ = false;
+ }
+
+ // Infer max_exponent_.
+ uint16_t lExp = ExponentImpliedByDouble(l);
+ uint16_t hExp = ExponentImpliedByDouble(h);
+ max_exponent_ = std::max(lExp, hExp);
+
+ canHaveFractionalPart_ = ExcludesFractionalParts;
+ canBeNegativeZero_ = ExcludesNegativeZero;
+
+ // Infer the canHaveFractionalPart_ setting. We can have a
+ // fractional part if the range crosses through the neighborhood of zero. We
+ // won't have a fractional value if the value is always beyond the point at
+ // which double precision can't represent fractional values.
+ uint16_t minExp = std::min(lExp, hExp);
+ bool includesNegative = IsNaN(l) || l < 0;
+ bool includesPositive = IsNaN(h) || h > 0;
+ bool crossesZero = includesNegative && includesPositive;
+ if (crossesZero || minExp < MaxTruncatableExponent) {
+ canHaveFractionalPart_ = IncludesFractionalParts;
+ }
+
+ // Infer the canBeNegativeZero_ setting. We can have a negative zero if
+ // either bound is zero.
+ if (!(l > 0) && !(h < 0)) {
+ canBeNegativeZero_ = IncludesNegativeZero;
+ }
+
+ optimize();
+}
+
+void Range::setDoubleSingleton(double d) {
+ setDouble(d, d);
+
+ // The above setDouble call is for comparisons, and treats negative zero
+ // as equal to zero. We're aiming for a minimum range, so we can clear the
+ // negative zero flag if the value isn't actually negative zero.
+ if (!IsNegativeZero(d)) {
+ canBeNegativeZero_ = ExcludesNegativeZero;
+ }
+
+ assertInvariants();
+}
+
+static inline bool MissingAnyInt32Bounds(const Range* lhs, const Range* rhs) {
+ return !lhs->hasInt32Bounds() || !rhs->hasInt32Bounds();
+}
+
+Range* Range::add(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ int64_t l = (int64_t)lhs->lower_ + (int64_t)rhs->lower_;
+ if (!lhs->hasInt32LowerBound() || !rhs->hasInt32LowerBound()) {
+ l = NoInt32LowerBound;
+ }
+
+ int64_t h = (int64_t)lhs->upper_ + (int64_t)rhs->upper_;
+ if (!lhs->hasInt32UpperBound() || !rhs->hasInt32UpperBound()) {
+ h = NoInt32UpperBound;
+ }
+
+ // The exponent is at most one greater than the greater of the operands'
+ // exponents, except for NaN and infinity cases.
+ uint16_t e = std::max(lhs->max_exponent_, rhs->max_exponent_);
+ if (e <= Range::MaxFiniteExponent) {
+ ++e;
+ }
+
+ // Infinity + -Infinity is NaN.
+ if (lhs->canBeInfiniteOrNaN() && rhs->canBeInfiniteOrNaN()) {
+ e = Range::IncludesInfinityAndNaN;
+ }
+
+ return new (alloc) Range(
+ l, h,
+ FractionalPartFlag(lhs->canHaveFractionalPart() ||
+ rhs->canHaveFractionalPart()),
+ NegativeZeroFlag(lhs->canBeNegativeZero() && rhs->canBeNegativeZero()),
+ e);
+}
+
+Range* Range::sub(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ int64_t l = (int64_t)lhs->lower_ - (int64_t)rhs->upper_;
+ if (!lhs->hasInt32LowerBound() || !rhs->hasInt32UpperBound()) {
+ l = NoInt32LowerBound;
+ }
+
+ int64_t h = (int64_t)lhs->upper_ - (int64_t)rhs->lower_;
+ if (!lhs->hasInt32UpperBound() || !rhs->hasInt32LowerBound()) {
+ h = NoInt32UpperBound;
+ }
+
+ // The exponent is at most one greater than the greater of the operands'
+ // exponents, except for NaN and infinity cases.
+ uint16_t e = std::max(lhs->max_exponent_, rhs->max_exponent_);
+ if (e <= Range::MaxFiniteExponent) {
+ ++e;
+ }
+
+ // Infinity - Infinity is NaN.
+ if (lhs->canBeInfiniteOrNaN() && rhs->canBeInfiniteOrNaN()) {
+ e = Range::IncludesInfinityAndNaN;
+ }
+
+ return new (alloc)
+ Range(l, h,
+ FractionalPartFlag(lhs->canHaveFractionalPart() ||
+ rhs->canHaveFractionalPart()),
+ NegativeZeroFlag(lhs->canBeNegativeZero() && rhs->canBeZero()), e);
+}
+
+Range* Range::and_(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ MOZ_ASSERT(lhs->isInt32());
+ MOZ_ASSERT(rhs->isInt32());
+
+ // If both numbers can be negative, result can be negative in the whole range
+ if (lhs->lower() < 0 && rhs->lower() < 0) {
+ return Range::NewInt32Range(alloc, INT32_MIN,
+ std::max(lhs->upper(), rhs->upper()));
+ }
+
+ // Only one of both numbers can be negative.
+ // - result can't be negative
+ // - Upper bound is minimum of both upper range,
+ int32_t lower = 0;
+ int32_t upper = std::min(lhs->upper(), rhs->upper());
+
+ // EXCEPT when upper bound of non negative number is max value,
+ // because negative value can return the whole max value.
+ // -1 & 5 = 5
+ if (lhs->lower() < 0) {
+ upper = rhs->upper();
+ }
+ if (rhs->lower() < 0) {
+ upper = lhs->upper();
+ }
+
+ return Range::NewInt32Range(alloc, lower, upper);
+}
+
+Range* Range::or_(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ MOZ_ASSERT(lhs->isInt32());
+ MOZ_ASSERT(rhs->isInt32());
+ // When one operand is always 0 or always -1, it's a special case where we
+ // can compute a fully precise result. Handling these up front also
+ // protects the code below from calling CountLeadingZeroes32 with a zero
+ // operand or from shifting an int32_t by 32.
+ if (lhs->lower() == lhs->upper()) {
+ if (lhs->lower() == 0) {
+ return new (alloc) Range(*rhs);
+ }
+ if (lhs->lower() == -1) {
+ return new (alloc) Range(*lhs);
+ }
+ }
+ if (rhs->lower() == rhs->upper()) {
+ if (rhs->lower() == 0) {
+ return new (alloc) Range(*lhs);
+ }
+ if (rhs->lower() == -1) {
+ return new (alloc) Range(*rhs);
+ }
+ }
+
+ // The code below uses CountLeadingZeroes32, which has undefined behavior
+ // if its operand is 0. We rely on the code above to protect it.
+ MOZ_ASSERT_IF(lhs->lower() >= 0, lhs->upper() != 0);
+ MOZ_ASSERT_IF(rhs->lower() >= 0, rhs->upper() != 0);
+ MOZ_ASSERT_IF(lhs->upper() < 0, lhs->lower() != -1);
+ MOZ_ASSERT_IF(rhs->upper() < 0, rhs->lower() != -1);
+
+ int32_t lower = INT32_MIN;
+ int32_t upper = INT32_MAX;
+
+ if (lhs->lower() >= 0 && rhs->lower() >= 0) {
+ // Both operands are non-negative, so the result won't be less than either.
+ lower = std::max(lhs->lower(), rhs->lower());
+ // The result will have leading zeros where both operands have leading
+ // zeros. CountLeadingZeroes32 of a non-negative int32 will at least be 1 to
+ // account for the bit of sign.
+ upper = int32_t(UINT32_MAX >> std::min(CountLeadingZeroes32(lhs->upper()),
+ CountLeadingZeroes32(rhs->upper())));
+ } else {
+ // The result will have leading ones where either operand has leading ones.
+ if (lhs->upper() < 0) {
+ unsigned leadingOnes = CountLeadingZeroes32(~lhs->lower());
+ lower = std::max(lower, ~int32_t(UINT32_MAX >> leadingOnes));
+ upper = -1;
+ }
+ if (rhs->upper() < 0) {
+ unsigned leadingOnes = CountLeadingZeroes32(~rhs->lower());
+ lower = std::max(lower, ~int32_t(UINT32_MAX >> leadingOnes));
+ upper = -1;
+ }
+ }
+
+ return Range::NewInt32Range(alloc, lower, upper);
+}
+
+Range* Range::xor_(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ MOZ_ASSERT(lhs->isInt32());
+ MOZ_ASSERT(rhs->isInt32());
+ int32_t lhsLower = lhs->lower();
+ int32_t lhsUpper = lhs->upper();
+ int32_t rhsLower = rhs->lower();
+ int32_t rhsUpper = rhs->upper();
+ bool invertAfter = false;
+
+ // If either operand is negative, bitwise-negate it, and arrange to negate
+ // the result; ~((~x)^y) == x^y. If both are negative the negations on the
+ // result cancel each other out; effectively this is (~x)^(~y) == x^y.
+ // These transformations reduce the number of cases we have to handle below.
+ if (lhsUpper < 0) {
+ lhsLower = ~lhsLower;
+ lhsUpper = ~lhsUpper;
+ std::swap(lhsLower, lhsUpper);
+ invertAfter = !invertAfter;
+ }
+ if (rhsUpper < 0) {
+ rhsLower = ~rhsLower;
+ rhsUpper = ~rhsUpper;
+ std::swap(rhsLower, rhsUpper);
+ invertAfter = !invertAfter;
+ }
+
+ // Handle cases where lhs or rhs is always zero specially, because they're
+ // easy cases where we can be perfectly precise, and because it protects the
+ // CountLeadingZeroes32 calls below from seeing 0 operands, which would be
+ // undefined behavior.
+ int32_t lower = INT32_MIN;
+ int32_t upper = INT32_MAX;
+ if (lhsLower == 0 && lhsUpper == 0) {
+ upper = rhsUpper;
+ lower = rhsLower;
+ } else if (rhsLower == 0 && rhsUpper == 0) {
+ upper = lhsUpper;
+ lower = lhsLower;
+ } else if (lhsLower >= 0 && rhsLower >= 0) {
+ // Both operands are non-negative. The result will be non-negative.
+ lower = 0;
+ // To compute the upper value, take each operand's upper value and
+ // set all bits that don't correspond to leading zero bits in the
+ // other to one. For each one, this gives an upper bound for the
+ // result, so we can take the minimum between the two.
+ unsigned lhsLeadingZeros = CountLeadingZeroes32(lhsUpper);
+ unsigned rhsLeadingZeros = CountLeadingZeroes32(rhsUpper);
+ upper = std::min(rhsUpper | int32_t(UINT32_MAX >> lhsLeadingZeros),
+ lhsUpper | int32_t(UINT32_MAX >> rhsLeadingZeros));
+ }
+
+ // If we bitwise-negated one (but not both) of the operands above, apply the
+ // bitwise-negate to the result, completing ~((~x)^y) == x^y.
+ if (invertAfter) {
+ lower = ~lower;
+ upper = ~upper;
+ std::swap(lower, upper);
+ }
+
+ return Range::NewInt32Range(alloc, lower, upper);
+}
+
+Range* Range::not_(TempAllocator& alloc, const Range* op) {
+ MOZ_ASSERT(op->isInt32());
+ return Range::NewInt32Range(alloc, ~op->upper(), ~op->lower());
+}
+
+Range* Range::mul(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ FractionalPartFlag newCanHaveFractionalPart = FractionalPartFlag(
+ lhs->canHaveFractionalPart_ || rhs->canHaveFractionalPart_);
+
+ NegativeZeroFlag newMayIncludeNegativeZero = NegativeZeroFlag(
+ (lhs->canHaveSignBitSet() && rhs->canBeFiniteNonNegative()) ||
+ (rhs->canHaveSignBitSet() && lhs->canBeFiniteNonNegative()));
+
+ uint16_t exponent;
+ if (!lhs->canBeInfiniteOrNaN() && !rhs->canBeInfiniteOrNaN()) {
+ // Two finite values.
+ exponent = lhs->numBits() + rhs->numBits() - 1;
+ if (exponent > Range::MaxFiniteExponent) {
+ exponent = Range::IncludesInfinity;
+ }
+ } else if (!lhs->canBeNaN() && !rhs->canBeNaN() &&
+ !(lhs->canBeZero() && rhs->canBeInfiniteOrNaN()) &&
+ !(rhs->canBeZero() && lhs->canBeInfiniteOrNaN())) {
+ // Two values that multiplied together won't produce a NaN.
+ exponent = Range::IncludesInfinity;
+ } else {
+ // Could be anything.
+ exponent = Range::IncludesInfinityAndNaN;
+ }
+
+ if (MissingAnyInt32Bounds(lhs, rhs)) {
+ return new (alloc)
+ Range(NoInt32LowerBound, NoInt32UpperBound, newCanHaveFractionalPart,
+ newMayIncludeNegativeZero, exponent);
+ }
+ int64_t a = (int64_t)lhs->lower() * (int64_t)rhs->lower();
+ int64_t b = (int64_t)lhs->lower() * (int64_t)rhs->upper();
+ int64_t c = (int64_t)lhs->upper() * (int64_t)rhs->lower();
+ int64_t d = (int64_t)lhs->upper() * (int64_t)rhs->upper();
+ return new (alloc)
+ Range(std::min(std::min(a, b), std::min(c, d)),
+ std::max(std::max(a, b), std::max(c, d)), newCanHaveFractionalPart,
+ newMayIncludeNegativeZero, exponent);
+}
+
+Range* Range::lsh(TempAllocator& alloc, const Range* lhs, int32_t c) {
+ MOZ_ASSERT(lhs->isInt32());
+ int32_t shift = c & 0x1f;
+
+ // If the shift doesn't loose bits or shift bits into the sign bit, we
+ // can simply compute the correct range by shifting.
+ if ((int32_t)((uint32_t)lhs->lower() << shift << 1 >> shift >> 1) ==
+ lhs->lower() &&
+ (int32_t)((uint32_t)lhs->upper() << shift << 1 >> shift >> 1) ==
+ lhs->upper()) {
+ return Range::NewInt32Range(alloc, uint32_t(lhs->lower()) << shift,
+ uint32_t(lhs->upper()) << shift);
+ }
+
+ return Range::NewInt32Range(alloc, INT32_MIN, INT32_MAX);
+}
+
+Range* Range::rsh(TempAllocator& alloc, const Range* lhs, int32_t c) {
+ MOZ_ASSERT(lhs->isInt32());
+ int32_t shift = c & 0x1f;
+ return Range::NewInt32Range(alloc, lhs->lower() >> shift,
+ lhs->upper() >> shift);
+}
+
+Range* Range::ursh(TempAllocator& alloc, const Range* lhs, int32_t c) {
+ // ursh's left operand is uint32, not int32, but for range analysis we
+ // currently approximate it as int32. We assume here that the range has
+ // already been adjusted accordingly by our callers.
+ MOZ_ASSERT(lhs->isInt32());
+
+ int32_t shift = c & 0x1f;
+
+ // If the value is always non-negative or always negative, we can simply
+ // compute the correct range by shifting.
+ if (lhs->isFiniteNonNegative() || lhs->isFiniteNegative()) {
+ return Range::NewUInt32Range(alloc, uint32_t(lhs->lower()) >> shift,
+ uint32_t(lhs->upper()) >> shift);
+ }
+
+ // Otherwise return the most general range after the shift.
+ return Range::NewUInt32Range(alloc, 0, UINT32_MAX >> shift);
+}
+
+Range* Range::lsh(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ MOZ_ASSERT(lhs->isInt32());
+ MOZ_ASSERT(rhs->isInt32());
+ return Range::NewInt32Range(alloc, INT32_MIN, INT32_MAX);
+}
+
+Range* Range::rsh(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ MOZ_ASSERT(lhs->isInt32());
+ MOZ_ASSERT(rhs->isInt32());
+
+ // Canonicalize the shift range to 0 to 31.
+ int32_t shiftLower = rhs->lower();
+ int32_t shiftUpper = rhs->upper();
+ if ((int64_t(shiftUpper) - int64_t(shiftLower)) >= 31) {
+ shiftLower = 0;
+ shiftUpper = 31;
+ } else {
+ shiftLower &= 0x1f;
+ shiftUpper &= 0x1f;
+ if (shiftLower > shiftUpper) {
+ shiftLower = 0;
+ shiftUpper = 31;
+ }
+ }
+ MOZ_ASSERT(shiftLower >= 0 && shiftUpper <= 31);
+
+ // The lhs bounds are signed, thus the minimum is either the lower bound
+ // shift by the smallest shift if negative or the lower bound shifted by the
+ // biggest shift otherwise. And the opposite for the maximum.
+ int32_t lhsLower = lhs->lower();
+ int32_t min = lhsLower < 0 ? lhsLower >> shiftLower : lhsLower >> shiftUpper;
+ int32_t lhsUpper = lhs->upper();
+ int32_t max = lhsUpper >= 0 ? lhsUpper >> shiftLower : lhsUpper >> shiftUpper;
+
+ return Range::NewInt32Range(alloc, min, max);
+}
+
+Range* Range::ursh(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ // ursh's left operand is uint32, not int32, but for range analysis we
+ // currently approximate it as int32. We assume here that the range has
+ // already been adjusted accordingly by our callers.
+ MOZ_ASSERT(lhs->isInt32());
+ MOZ_ASSERT(rhs->isInt32());
+ return Range::NewUInt32Range(
+ alloc, 0, lhs->isFiniteNonNegative() ? lhs->upper() : UINT32_MAX);
+}
+
+Range* Range::abs(TempAllocator& alloc, const Range* op) {
+ int32_t l = op->lower_;
+ int32_t u = op->upper_;
+ FractionalPartFlag canHaveFractionalPart = op->canHaveFractionalPart_;
+
+ // Abs never produces a negative zero.
+ NegativeZeroFlag canBeNegativeZero = ExcludesNegativeZero;
+
+ return new (alloc) Range(
+ std::max(std::max(int32_t(0), l), u == INT32_MIN ? INT32_MAX : -u), true,
+ std::max(std::max(int32_t(0), u), l == INT32_MIN ? INT32_MAX : -l),
+ op->hasInt32Bounds() && l != INT32_MIN, canHaveFractionalPart,
+ canBeNegativeZero, op->max_exponent_);
+}
+
+Range* Range::min(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ // If either operand is NaN, the result is NaN.
+ if (lhs->canBeNaN() || rhs->canBeNaN()) {
+ return nullptr;
+ }
+
+ FractionalPartFlag newCanHaveFractionalPart = FractionalPartFlag(
+ lhs->canHaveFractionalPart_ || rhs->canHaveFractionalPart_);
+ NegativeZeroFlag newMayIncludeNegativeZero =
+ NegativeZeroFlag(lhs->canBeNegativeZero_ || rhs->canBeNegativeZero_);
+
+ return new (alloc) Range(std::min(lhs->lower_, rhs->lower_),
+ lhs->hasInt32LowerBound_ && rhs->hasInt32LowerBound_,
+ std::min(lhs->upper_, rhs->upper_),
+ lhs->hasInt32UpperBound_ || rhs->hasInt32UpperBound_,
+ newCanHaveFractionalPart, newMayIncludeNegativeZero,
+ std::max(lhs->max_exponent_, rhs->max_exponent_));
+}
+
+Range* Range::max(TempAllocator& alloc, const Range* lhs, const Range* rhs) {
+ // If either operand is NaN, the result is NaN.
+ if (lhs->canBeNaN() || rhs->canBeNaN()) {
+ return nullptr;
+ }
+
+ FractionalPartFlag newCanHaveFractionalPart = FractionalPartFlag(
+ lhs->canHaveFractionalPart_ || rhs->canHaveFractionalPart_);
+ NegativeZeroFlag newMayIncludeNegativeZero =
+ NegativeZeroFlag(lhs->canBeNegativeZero_ || rhs->canBeNegativeZero_);
+
+ return new (alloc) Range(std::max(lhs->lower_, rhs->lower_),
+ lhs->hasInt32LowerBound_ || rhs->hasInt32LowerBound_,
+ std::max(lhs->upper_, rhs->upper_),
+ lhs->hasInt32UpperBound_ && rhs->hasInt32UpperBound_,
+ newCanHaveFractionalPart, newMayIncludeNegativeZero,
+ std::max(lhs->max_exponent_, rhs->max_exponent_));
+}
+
+Range* Range::floor(TempAllocator& alloc, const Range* op) {
+ Range* copy = new (alloc) Range(*op);
+ // Decrement lower bound of copy range if op have a factional part and lower
+ // bound is Int32 defined. Also we avoid to decrement when op have a
+ // fractional part but lower_ >= JSVAL_INT_MAX.
+ if (op->canHaveFractionalPart() && op->hasInt32LowerBound()) {
+ copy->setLowerInit(int64_t(copy->lower_) - 1);
+ }
+
+ // Also refine max_exponent_ because floor may have decremented int value
+ // If we've got int32 defined bounds, just deduce it using defined bounds.
+ // But, if we don't have those, value's max_exponent_ may have changed.
+ // Because we're looking to maintain an over estimation, if we can,
+ // we increment it.
+ if (copy->hasInt32Bounds())
+ copy->max_exponent_ = copy->exponentImpliedByInt32Bounds();
+ else if (copy->max_exponent_ < MaxFiniteExponent)
+ copy->max_exponent_++;
+
+ copy->canHaveFractionalPart_ = ExcludesFractionalParts;
+ copy->assertInvariants();
+ return copy;
+}
+
+Range* Range::ceil(TempAllocator& alloc, const Range* op) {
+ Range* copy = new (alloc) Range(*op);
+
+ // We need to refine max_exponent_ because ceil may have incremented the int
+ // value. If we have got int32 bounds defined, just deduce it using the
+ // defined bounds. Else we can just increment its value, as we are looking to
+ // maintain an over estimation.
+ if (copy->hasInt32Bounds()) {
+ copy->max_exponent_ = copy->exponentImpliedByInt32Bounds();
+ } else if (copy->max_exponent_ < MaxFiniteExponent) {
+ copy->max_exponent_++;
+ }
+
+ // If the range is definitely above 0 or below -1, we don't need to include
+ // -0; otherwise we do.
+
+ copy->canBeNegativeZero_ = ((copy->lower_ > 0) || (copy->upper_ <= -1))
+ ? copy->canBeNegativeZero_
+ : IncludesNegativeZero;
+
+ copy->canHaveFractionalPart_ = ExcludesFractionalParts;
+ copy->assertInvariants();
+ return copy;
+}
+
+Range* Range::sign(TempAllocator& alloc, const Range* op) {
+ if (op->canBeNaN()) {
+ return nullptr;
+ }
+
+ return new (alloc) Range(std::max(std::min(op->lower_, 1), -1),
+ std::max(std::min(op->upper_, 1), -1),
+ Range::ExcludesFractionalParts,
+ NegativeZeroFlag(op->canBeNegativeZero()), 0);
+}
+
+Range* Range::NaNToZero(TempAllocator& alloc, const Range* op) {
+ Range* copy = new (alloc) Range(*op);
+ if (copy->canBeNaN()) {
+ copy->max_exponent_ = Range::IncludesInfinity;
+ if (!copy->canBeZero()) {
+ Range zero;
+ zero.setDoubleSingleton(0);
+ copy->unionWith(&zero);
+ }
+ }
+ copy->refineToExcludeNegativeZero();
+ return copy;
+}
+
+Range* Range::toIntegerInt32(TempAllocator& alloc, const Range* op) {
+ return Range::NaNToZero(alloc, Range::ceil(alloc, op));
+}
+
+bool Range::negativeZeroMul(const Range* lhs, const Range* rhs) {
+ // The result can only be negative zero if both sides are finite and they
+ // have differing signs.
+ return (lhs->canHaveSignBitSet() && rhs->canBeFiniteNonNegative()) ||
+ (rhs->canHaveSignBitSet() && lhs->canBeFiniteNonNegative());
+}
+
+bool Range::update(const Range* other) {
+ bool changed = lower_ != other->lower_ ||
+ hasInt32LowerBound_ != other->hasInt32LowerBound_ ||
+ upper_ != other->upper_ ||
+ hasInt32UpperBound_ != other->hasInt32UpperBound_ ||
+ canHaveFractionalPart_ != other->canHaveFractionalPart_ ||
+ canBeNegativeZero_ != other->canBeNegativeZero_ ||
+ max_exponent_ != other->max_exponent_;
+ if (changed) {
+ lower_ = other->lower_;
+ hasInt32LowerBound_ = other->hasInt32LowerBound_;
+ upper_ = other->upper_;
+ hasInt32UpperBound_ = other->hasInt32UpperBound_;
+ canHaveFractionalPart_ = other->canHaveFractionalPart_;
+ canBeNegativeZero_ = other->canBeNegativeZero_;
+ max_exponent_ = other->max_exponent_;
+ assertInvariants();
+ }
+
+ return changed;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Range Computation for MIR Nodes
+///////////////////////////////////////////////////////////////////////////////
+
+void MPhi::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32 && type() != MIRType::Double) {
+ return;
+ }
+
+ Range* range = nullptr;
+ for (size_t i = 0, e = numOperands(); i < e; i++) {
+ if (getOperand(i)->block()->unreachable()) {
+ JitSpew(JitSpew_Range, "Ignoring unreachable input %u",
+ getOperand(i)->id());
+ continue;
+ }
+
+ // Peek at the pre-bailout range so we can take a short-cut; if any of
+ // the operands has an unknown range, this phi has an unknown range.
+ if (!getOperand(i)->range()) {
+ return;
+ }
+
+ Range input(getOperand(i));
+
+ if (range) {
+ range->unionWith(&input);
+ } else {
+ range = new (alloc) Range(input);
+ }
+ }
+
+ setRange(range);
+}
+
+void MBeta::computeRange(TempAllocator& alloc) {
+ bool emptyRange = false;
+
+ Range opRange(getOperand(0));
+ Range* range = Range::intersect(alloc, &opRange, comparison_, &emptyRange);
+ if (emptyRange) {
+ JitSpew(JitSpew_Range, "Marking block for inst %u unreachable", id());
+ block()->setUnreachableUnchecked();
+ } else {
+ setRange(range);
+ }
+}
+
+void MConstant::computeRange(TempAllocator& alloc) {
+ if (isTypeRepresentableAsDouble()) {
+ double d = numberToDouble();
+ setRange(Range::NewDoubleSingletonRange(alloc, d));
+ } else if (type() == MIRType::Boolean) {
+ bool b = toBoolean();
+ setRange(Range::NewInt32Range(alloc, b, b));
+ }
+}
+
+void MCharCodeAt::computeRange(TempAllocator& alloc) {
+ // ECMA 262 says that the integer will be non-negative and at most 65535.
+ setRange(Range::NewInt32Range(alloc, 0, 65535));
+}
+
+void MClampToUint8::computeRange(TempAllocator& alloc) {
+ setRange(Range::NewUInt32Range(alloc, 0, 255));
+}
+
+void MBitAnd::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ left.wrapAroundToInt32();
+ right.wrapAroundToInt32();
+
+ setRange(Range::and_(alloc, &left, &right));
+}
+
+void MBitOr::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ left.wrapAroundToInt32();
+ right.wrapAroundToInt32();
+
+ setRange(Range::or_(alloc, &left, &right));
+}
+
+void MBitXor::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ left.wrapAroundToInt32();
+ right.wrapAroundToInt32();
+
+ setRange(Range::xor_(alloc, &left, &right));
+}
+
+void MBitNot::computeRange(TempAllocator& alloc) {
+ if (type() == MIRType::Int64) {
+ return;
+ }
+ MOZ_ASSERT(type() == MIRType::Int32);
+
+ Range op(getOperand(0));
+ op.wrapAroundToInt32();
+
+ setRange(Range::not_(alloc, &op));
+}
+
+void MLsh::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ left.wrapAroundToInt32();
+
+ MConstant* rhsConst = getOperand(1)->maybeConstantValue();
+ if (rhsConst && rhsConst->type() == MIRType::Int32) {
+ int32_t c = rhsConst->toInt32();
+ setRange(Range::lsh(alloc, &left, c));
+ return;
+ }
+
+ right.wrapAroundToShiftCount();
+ setRange(Range::lsh(alloc, &left, &right));
+}
+
+void MRsh::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ left.wrapAroundToInt32();
+
+ MConstant* rhsConst = getOperand(1)->maybeConstantValue();
+ if (rhsConst && rhsConst->type() == MIRType::Int32) {
+ int32_t c = rhsConst->toInt32();
+ setRange(Range::rsh(alloc, &left, c));
+ return;
+ }
+
+ right.wrapAroundToShiftCount();
+ setRange(Range::rsh(alloc, &left, &right));
+}
+
+void MUrsh::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+
+ // ursh can be thought of as converting its left operand to uint32, or it
+ // can be thought of as converting its left operand to int32, and then
+ // reinterpreting the int32 bits as a uint32 value. Both approaches yield
+ // the same result. Since we lack support for full uint32 ranges, we use
+ // the second interpretation, though it does cause us to be conservative.
+ left.wrapAroundToInt32();
+ right.wrapAroundToShiftCount();
+
+ MConstant* rhsConst = getOperand(1)->maybeConstantValue();
+ if (rhsConst && rhsConst->type() == MIRType::Int32) {
+ int32_t c = rhsConst->toInt32();
+ setRange(Range::ursh(alloc, &left, c));
+ } else {
+ setRange(Range::ursh(alloc, &left, &right));
+ }
+
+ MOZ_ASSERT(range()->lower() >= 0);
+}
+
+void MAbs::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32 && type() != MIRType::Double) {
+ return;
+ }
+
+ Range other(getOperand(0));
+ Range* next = Range::abs(alloc, &other);
+ if (implicitTruncate_) {
+ next->wrapAroundToInt32();
+ }
+ setRange(next);
+}
+
+void MFloor::computeRange(TempAllocator& alloc) {
+ Range other(getOperand(0));
+ setRange(Range::floor(alloc, &other));
+}
+
+void MCeil::computeRange(TempAllocator& alloc) {
+ Range other(getOperand(0));
+ setRange(Range::ceil(alloc, &other));
+}
+
+void MClz::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+ setRange(Range::NewUInt32Range(alloc, 0, 32));
+}
+
+void MCtz::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+ setRange(Range::NewUInt32Range(alloc, 0, 32));
+}
+
+void MPopcnt::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32) {
+ return;
+ }
+ setRange(Range::NewUInt32Range(alloc, 0, 32));
+}
+
+void MMinMax::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32 && type() != MIRType::Double) {
+ return;
+ }
+
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ setRange(isMax() ? Range::max(alloc, &left, &right)
+ : Range::min(alloc, &left, &right));
+}
+
+void MAdd::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32 && type() != MIRType::Double) {
+ return;
+ }
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ Range* next = Range::add(alloc, &left, &right);
+ if (isTruncated()) {
+ next->wrapAroundToInt32();
+ }
+ setRange(next);
+}
+
+void MSub::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32 && type() != MIRType::Double) {
+ return;
+ }
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ Range* next = Range::sub(alloc, &left, &right);
+ if (isTruncated()) {
+ next->wrapAroundToInt32();
+ }
+ setRange(next);
+}
+
+void MMul::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32 && type() != MIRType::Double) {
+ return;
+ }
+ Range left(getOperand(0));
+ Range right(getOperand(1));
+ if (canBeNegativeZero()) {
+ canBeNegativeZero_ = Range::negativeZeroMul(&left, &right);
+ }
+ Range* next = Range::mul(alloc, &left, &right);
+ if (!next->canBeNegativeZero()) {
+ canBeNegativeZero_ = false;
+ }
+ // Truncated multiplications could overflow in both directions
+ if (isTruncated()) {
+ next->wrapAroundToInt32();
+ }
+ setRange(next);
+}
+
+void MMod::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32 && type() != MIRType::Double) {
+ return;
+ }
+ Range lhs(getOperand(0));
+ Range rhs(getOperand(1));
+
+ // If either operand is a NaN, the result is NaN. This also conservatively
+ // handles Infinity cases.
+ if (!lhs.hasInt32Bounds() || !rhs.hasInt32Bounds()) {
+ return;
+ }
+
+ // If RHS can be zero, the result can be NaN.
+ if (rhs.lower() <= 0 && rhs.upper() >= 0) {
+ return;
+ }
+
+ // If both operands are non-negative integers, we can optimize this to an
+ // unsigned mod.
+ if (type() == MIRType::Int32 && rhs.lower() > 0) {
+ bool hasDoubles = lhs.lower() < 0 || lhs.canHaveFractionalPart() ||
+ rhs.canHaveFractionalPart();
+ // It is not possible to check that lhs.lower() >= 0, since the range
+ // of a ursh with rhs a 0 constant is wrapped around the int32 range in
+ // Range::Range(). However, IsUint32Type() will only return true for
+ // nodes that lie in the range [0, UINT32_MAX].
+ bool hasUint32s =
+ IsUint32Type(getOperand(0)) &&
+ getOperand(1)->type() == MIRType::Int32 &&
+ (IsUint32Type(getOperand(1)) || getOperand(1)->isConstant());
+ if (!hasDoubles || hasUint32s) {
+ unsigned_ = true;
+ }
+ }
+
+ // For unsigned mod, we have to convert both operands to unsigned.
+ // Note that we handled the case of a zero rhs above.
+ if (unsigned_) {
+ // The result of an unsigned mod will never be unsigned-greater than
+ // either operand.
+ uint32_t lhsBound = std::max<uint32_t>(lhs.lower(), lhs.upper());
+ uint32_t rhsBound = std::max<uint32_t>(rhs.lower(), rhs.upper());
+
+ // If either range crosses through -1 as a signed value, it could be
+ // the maximum unsigned value when interpreted as unsigned. If the range
+ // doesn't include -1, then the simple max value we computed above is
+ // correct.
+ if (lhs.lower() <= -1 && lhs.upper() >= -1) {
+ lhsBound = UINT32_MAX;
+ }
+ if (rhs.lower() <= -1 && rhs.upper() >= -1) {
+ rhsBound = UINT32_MAX;
+ }
+
+ // The result will never be equal to the rhs, and we shouldn't have
+ // any rounding to worry about.
+ MOZ_ASSERT(!lhs.canHaveFractionalPart() && !rhs.canHaveFractionalPart());
+ --rhsBound;
+
+ // This gives us two upper bounds, so we can take the best one.
+ setRange(Range::NewUInt32Range(alloc, 0, std::min(lhsBound, rhsBound)));
+ return;
+ }
+
+ // Math.abs(lhs % rhs) == Math.abs(lhs) % Math.abs(rhs).
+ // First, the absolute value of the result will always be less than the
+ // absolute value of rhs. (And if rhs is zero, the result is NaN).
+ int64_t a = Abs<int64_t>(rhs.lower());
+ int64_t b = Abs<int64_t>(rhs.upper());
+ if (a == 0 && b == 0) {
+ return;
+ }
+ int64_t rhsAbsBound = std::max(a, b);
+
+ // If the value is known to be integer, less-than abs(rhs) is equivalent
+ // to less-than-or-equal abs(rhs)-1. This is important for being able to
+ // say that the result of x%256 is an 8-bit unsigned number.
+ if (!lhs.canHaveFractionalPart() && !rhs.canHaveFractionalPart()) {
+ --rhsAbsBound;
+ }
+
+ // Next, the absolute value of the result will never be greater than the
+ // absolute value of lhs.
+ int64_t lhsAbsBound =
+ std::max(Abs<int64_t>(lhs.lower()), Abs<int64_t>(lhs.upper()));
+
+ // This gives us two upper bounds, so we can take the best one.
+ int64_t absBound = std::min(lhsAbsBound, rhsAbsBound);
+
+ // Now consider the sign of the result.
+ // If lhs is non-negative, the result will be non-negative.
+ // If lhs is non-positive, the result will be non-positive.
+ int64_t lower = lhs.lower() >= 0 ? 0 : -absBound;
+ int64_t upper = lhs.upper() <= 0 ? 0 : absBound;
+
+ Range::FractionalPartFlag newCanHaveFractionalPart =
+ Range::FractionalPartFlag(lhs.canHaveFractionalPart() ||
+ rhs.canHaveFractionalPart());
+
+ // If the lhs can have the sign bit set and we can return a zero, it'll be a
+ // negative zero.
+ Range::NegativeZeroFlag newMayIncludeNegativeZero =
+ Range::NegativeZeroFlag(lhs.canHaveSignBitSet());
+
+ setRange(new (alloc) Range(lower, upper, newCanHaveFractionalPart,
+ newMayIncludeNegativeZero,
+ std::min(lhs.exponent(), rhs.exponent())));
+}
+
+void MDiv::computeRange(TempAllocator& alloc) {
+ if (type() != MIRType::Int32 && type() != MIRType::Double) {
+ return;
+ }
+ Range lhs(getOperand(0));
+ Range rhs(getOperand(1));
+
+ // If either operand is a NaN, the result is NaN. This also conservatively
+ // handles Infinity cases.
+ if (!lhs.hasInt32Bounds() || !rhs.hasInt32Bounds()) {
+ return;
+ }
+
+ // Something simple for now: When dividing by a positive rhs, the result
+ // won't be further from zero than lhs.
+ if (lhs.lower() >= 0 && rhs.lower() >= 1) {
+ setRange(new (alloc) Range(0, lhs.upper(), Range::IncludesFractionalParts,
+ Range::IncludesNegativeZero, lhs.exponent()));
+ } else if (unsigned_ && rhs.lower() >= 1) {
+ // We shouldn't set the unsigned flag if the inputs can have
+ // fractional parts.
+ MOZ_ASSERT(!lhs.canHaveFractionalPart() && !rhs.canHaveFractionalPart());
+ // We shouldn't set the unsigned flag if the inputs can be
+ // negative zero.
+ MOZ_ASSERT(!lhs.canBeNegativeZero() && !rhs.canBeNegativeZero());
+ // Unsigned division by a non-zero rhs will return a uint32 value.
+ setRange(Range::NewUInt32Range(alloc, 0, UINT32_MAX));
+ }
+}
+
+void MSqrt::computeRange(TempAllocator& alloc) {
+ Range input(getOperand(0));
+
+ // If either operand is a NaN, the result is NaN. This also conservatively
+ // handles Infinity cases.
+ if (!input.hasInt32Bounds()) {
+ return;
+ }
+
+ // Sqrt of a negative non-zero value is NaN.
+ if (input.lower() < 0) {
+ return;
+ }
+
+ // Something simple for now: When taking the sqrt of a positive value, the
+ // result won't be further from zero than the input.
+ // And, sqrt of an integer may have a fractional part.
+ setRange(new (alloc) Range(0, input.upper(), Range::IncludesFractionalParts,
+ input.canBeNegativeZero(), input.exponent()));
+}
+
+void MToDouble::computeRange(TempAllocator& alloc) {
+ setRange(new (alloc) Range(getOperand(0)));
+}
+
+void MToFloat32::computeRange(TempAllocator& alloc) {}
+
+void MTruncateToInt32::computeRange(TempAllocator& alloc) {
+ Range* output = new (alloc) Range(getOperand(0));
+ output->wrapAroundToInt32();
+ setRange(output);
+}
+
+void MToNumberInt32::computeRange(TempAllocator& alloc) {
+ // No clamping since this computes the range *before* bailouts.
+ setRange(new (alloc) Range(getOperand(0)));
+}
+
+void MToIntegerInt32::computeRange(TempAllocator& alloc) {
+ Range other(input());
+ setRange(Range::toIntegerInt32(alloc, &other));
+}
+
+void MLimitedTruncate::computeRange(TempAllocator& alloc) {
+ Range* output = new (alloc) Range(input());
+ setRange(output);
+}
+
+static Range* GetArrayBufferViewRange(TempAllocator& alloc, Scalar::Type type) {
+ switch (type) {
+ case Scalar::Uint8Clamped:
+ case Scalar::Uint8:
+ return Range::NewUInt32Range(alloc, 0, UINT8_MAX);
+ case Scalar::Uint16:
+ return Range::NewUInt32Range(alloc, 0, UINT16_MAX);
+ case Scalar::Uint32:
+ return Range::NewUInt32Range(alloc, 0, UINT32_MAX);
+
+ case Scalar::Int8:
+ return Range::NewInt32Range(alloc, INT8_MIN, INT8_MAX);
+ case Scalar::Int16:
+ return Range::NewInt32Range(alloc, INT16_MIN, INT16_MAX);
+ case Scalar::Int32:
+ return Range::NewInt32Range(alloc, INT32_MIN, INT32_MAX);
+
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::Int64:
+ case Scalar::Simd128:
+ case Scalar::Float32:
+ case Scalar::Float64:
+ case Scalar::MaxTypedArrayViewType:
+ break;
+ }
+ return nullptr;
+}
+
+void MLoadUnboxedScalar::computeRange(TempAllocator& alloc) {
+ // We have an Int32 type and if this is a UInt32 load it may produce a value
+ // outside of our range, but we have a bailout to handle those cases.
+ setRange(GetArrayBufferViewRange(alloc, storageType()));
+}
+
+void MLoadDataViewElement::computeRange(TempAllocator& alloc) {
+ // We have an Int32 type and if this is a UInt32 load it may produce a value
+ // outside of our range, but we have a bailout to handle those cases.
+ setRange(GetArrayBufferViewRange(alloc, storageType()));
+}
+
+void MArrayLength::computeRange(TempAllocator& alloc) {
+ // Array lengths can go up to UINT32_MAX. We will bail out if the array
+ // length > INT32_MAX.
+ MOZ_ASSERT(type() == MIRType::Int32);
+ setRange(Range::NewUInt32Range(alloc, 0, INT32_MAX));
+}
+
+void MInitializedLength::computeRange(TempAllocator& alloc) {
+ setRange(
+ Range::NewUInt32Range(alloc, 0, NativeObject::MAX_DENSE_ELEMENTS_COUNT));
+}
+
+void MArrayBufferViewLength::computeRange(TempAllocator& alloc) {
+ if constexpr (ArrayBufferObject::MaxByteLength <= INT32_MAX) {
+ setRange(Range::NewUInt32Range(alloc, 0, INT32_MAX));
+ }
+}
+
+void MArrayBufferViewByteOffset::computeRange(TempAllocator& alloc) {
+ if constexpr (ArrayBufferObject::MaxByteLength <= INT32_MAX) {
+ setRange(Range::NewUInt32Range(alloc, 0, INT32_MAX));
+ }
+}
+
+void MTypedArrayElementSize::computeRange(TempAllocator& alloc) {
+ constexpr auto MaxTypedArraySize = sizeof(double);
+
+#define ASSERT_MAX_SIZE(_, T, N) \
+ static_assert(sizeof(T) <= MaxTypedArraySize, \
+ "unexpected typed array type exceeding 64-bits storage");
+ JS_FOR_EACH_TYPED_ARRAY(ASSERT_MAX_SIZE)
+#undef ASSERT_MAX_SIZE
+
+ setRange(Range::NewUInt32Range(alloc, 0, MaxTypedArraySize));
+}
+
+void MStringLength::computeRange(TempAllocator& alloc) {
+ static_assert(JSString::MAX_LENGTH <= UINT32_MAX,
+ "NewUInt32Range requires a uint32 value");
+ setRange(Range::NewUInt32Range(alloc, 0, JSString::MAX_LENGTH));
+}
+
+void MArgumentsLength::computeRange(TempAllocator& alloc) {
+ // This is is a conservative upper bound on what |TooManyActualArguments|
+ // checks. If exceeded, Ion will not be entered in the first place.
+ static_assert(ARGS_LENGTH_MAX <= UINT32_MAX,
+ "NewUInt32Range requires a uint32 value");
+ setRange(Range::NewUInt32Range(alloc, 0, ARGS_LENGTH_MAX));
+}
+
+void MBoundsCheck::computeRange(TempAllocator& alloc) {
+ // Just transfer the incoming index range to the output. The length() is
+ // also interesting, but it is handled as a bailout check, and we're
+ // computing a pre-bailout range here.
+ setRange(new (alloc) Range(index()));
+}
+
+void MSpectreMaskIndex::computeRange(TempAllocator& alloc) {
+ // Just transfer the incoming index range to the output for now.
+ setRange(new (alloc) Range(index()));
+}
+
+void MInt32ToIntPtr::computeRange(TempAllocator& alloc) {
+ setRange(new (alloc) Range(input()));
+}
+
+void MNonNegativeIntPtrToInt32::computeRange(TempAllocator& alloc) {
+ // We will bail out if the IntPtr value > INT32_MAX.
+ setRange(Range::NewUInt32Range(alloc, 0, INT32_MAX));
+}
+
+void MArrayPush::computeRange(TempAllocator& alloc) {
+ // MArrayPush returns the new array length. It bails out if the new length
+ // doesn't fit in an Int32.
+ MOZ_ASSERT(type() == MIRType::Int32);
+ setRange(Range::NewUInt32Range(alloc, 0, INT32_MAX));
+}
+
+void MMathFunction::computeRange(TempAllocator& alloc) {
+ Range opRange(getOperand(0));
+ switch (function()) {
+ case UnaryMathFunction::Sin:
+ case UnaryMathFunction::Cos:
+ if (!opRange.canBeInfiniteOrNaN()) {
+ setRange(Range::NewDoubleRange(alloc, -1.0, 1.0));
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+void MSign::computeRange(TempAllocator& alloc) {
+ Range opRange(getOperand(0));
+ setRange(Range::sign(alloc, &opRange));
+}
+
+void MRandom::computeRange(TempAllocator& alloc) {
+ Range* r = Range::NewDoubleRange(alloc, 0.0, 1.0);
+
+ // Random never returns negative zero.
+ r->refineToExcludeNegativeZero();
+
+ setRange(r);
+}
+
+void MNaNToZero::computeRange(TempAllocator& alloc) {
+ Range other(input());
+ setRange(Range::NaNToZero(alloc, &other));
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Range Analysis
+///////////////////////////////////////////////////////////////////////////////
+
+static BranchDirection NegateBranchDirection(BranchDirection dir) {
+ return (dir == FALSE_BRANCH) ? TRUE_BRANCH : FALSE_BRANCH;
+}
+
+bool RangeAnalysis::analyzeLoop(MBasicBlock* header) {
+ MOZ_ASSERT(header->hasUniqueBackedge());
+
+ // Try to compute an upper bound on the number of times the loop backedge
+ // will be taken. Look for tests that dominate the backedge and which have
+ // an edge leaving the loop body.
+ MBasicBlock* backedge = header->backedge();
+
+ // Ignore trivial infinite loops.
+ if (backedge == header) {
+ return true;
+ }
+
+ bool canOsr;
+ size_t numBlocks = MarkLoopBlocks(graph_, header, &canOsr);
+
+ // Ignore broken loops.
+ if (numBlocks == 0) {
+ return true;
+ }
+
+ LoopIterationBound* iterationBound = nullptr;
+
+ MBasicBlock* block = backedge;
+ do {
+ BranchDirection direction;
+ MTest* branch = block->immediateDominatorBranch(&direction);
+
+ if (block == block->immediateDominator()) {
+ break;
+ }
+
+ block = block->immediateDominator();
+
+ if (branch) {
+ direction = NegateBranchDirection(direction);
+ MBasicBlock* otherBlock = branch->branchSuccessor(direction);
+ if (!otherBlock->isMarked()) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ iterationBound = analyzeLoopIterationCount(header, branch, direction);
+ if (iterationBound) {
+ break;
+ }
+ }
+ }
+ } while (block != header);
+
+ if (!iterationBound) {
+ UnmarkLoopBlocks(graph_, header);
+ return true;
+ }
+
+ if (!loopIterationBounds.append(iterationBound)) {
+ return false;
+ }
+
+#ifdef DEBUG
+ if (JitSpewEnabled(JitSpew_Range)) {
+ Sprinter sp(GetJitContext()->cx);
+ if (!sp.init()) {
+ return false;
+ }
+ iterationBound->boundSum.dump(sp);
+ JitSpew(JitSpew_Range, "computed symbolic bound on backedges: %s",
+ sp.string());
+ }
+#endif
+
+ // Try to compute symbolic bounds for the phi nodes at the head of this
+ // loop, expressed in terms of the iteration bound just computed.
+
+ for (MPhiIterator iter(header->phisBegin()); iter != header->phisEnd();
+ iter++) {
+ analyzeLoopPhi(iterationBound, *iter);
+ }
+
+ if (!mir->compilingWasm() && !mir->outerInfo().hadBoundsCheckBailout()) {
+ // Try to hoist any bounds checks from the loop using symbolic bounds.
+
+ Vector<MBoundsCheck*, 0, JitAllocPolicy> hoistedChecks(alloc());
+
+ for (ReversePostorderIterator iter(graph_.rpoBegin(header));
+ iter != graph_.rpoEnd(); iter++) {
+ MBasicBlock* block = *iter;
+ if (!block->isMarked()) {
+ continue;
+ }
+
+ for (MDefinitionIterator iter(block); iter; iter++) {
+ MDefinition* def = *iter;
+ if (def->isBoundsCheck() && def->isMovable()) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ if (tryHoistBoundsCheck(header, def->toBoundsCheck())) {
+ if (!hoistedChecks.append(def->toBoundsCheck())) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+
+ // Note: replace all uses of the original bounds check with the
+ // actual index. This is usually done during bounds check elimination,
+ // but in this case it's safe to do it here since the load/store is
+ // definitely not loop-invariant, so we will never move it before
+ // one of the bounds checks we just added.
+ for (size_t i = 0; i < hoistedChecks.length(); i++) {
+ MBoundsCheck* ins = hoistedChecks[i];
+ ins->replaceAllUsesWith(ins->index());
+ ins->block()->discard(ins);
+ }
+ }
+
+ UnmarkLoopBlocks(graph_, header);
+ return true;
+}
+
+// Unbox beta nodes in order to hoist instruction properly, and not be limited
+// by the beta nodes which are added after each branch.
+static inline MDefinition* DefinitionOrBetaInputDefinition(MDefinition* ins) {
+ while (ins->isBeta()) {
+ ins = ins->toBeta()->input();
+ }
+ return ins;
+}
+
+LoopIterationBound* RangeAnalysis::analyzeLoopIterationCount(
+ MBasicBlock* header, MTest* test, BranchDirection direction) {
+ SimpleLinearSum lhs(nullptr, 0);
+ MDefinition* rhs;
+ bool lessEqual;
+ if (!ExtractLinearInequality(test, direction, &lhs, &rhs, &lessEqual)) {
+ return nullptr;
+ }
+
+ // Ensure the rhs is a loop invariant term.
+ if (rhs && rhs->block()->isMarked()) {
+ if (lhs.term && lhs.term->block()->isMarked()) {
+ return nullptr;
+ }
+ MDefinition* temp = lhs.term;
+ lhs.term = rhs;
+ rhs = temp;
+ if (!SafeSub(0, lhs.constant, &lhs.constant)) {
+ return nullptr;
+ }
+ lessEqual = !lessEqual;
+ }
+
+ MOZ_ASSERT_IF(rhs, !rhs->block()->isMarked());
+
+ // Ensure the lhs is a phi node from the start of the loop body.
+ if (!lhs.term || !lhs.term->isPhi() || lhs.term->block() != header) {
+ return nullptr;
+ }
+
+ // Check that the value of the lhs changes by a constant amount with each
+ // loop iteration. This requires that the lhs be written in every loop
+ // iteration with a value that is a constant difference from its value at
+ // the start of the iteration.
+
+ if (lhs.term->toPhi()->numOperands() != 2) {
+ return nullptr;
+ }
+
+ // The first operand of the phi should be the lhs' value at the start of
+ // the first executed iteration, and not a value written which could
+ // replace the second operand below during the middle of execution.
+ MDefinition* lhsInitial = lhs.term->toPhi()->getLoopPredecessorOperand();
+ if (lhsInitial->block()->isMarked()) {
+ return nullptr;
+ }
+
+ // The second operand of the phi should be a value written by an add/sub
+ // in every loop iteration, i.e. in a block which dominates the backedge.
+ MDefinition* lhsWrite = DefinitionOrBetaInputDefinition(
+ lhs.term->toPhi()->getLoopBackedgeOperand());
+ if (!lhsWrite->isAdd() && !lhsWrite->isSub()) {
+ return nullptr;
+ }
+ if (!lhsWrite->block()->isMarked()) {
+ return nullptr;
+ }
+ MBasicBlock* bb = header->backedge();
+ for (; bb != lhsWrite->block() && bb != header;
+ bb = bb->immediateDominator()) {
+ }
+ if (bb != lhsWrite->block()) {
+ return nullptr;
+ }
+
+ SimpleLinearSum lhsModified = ExtractLinearSum(lhsWrite);
+
+ // Check that the value of the lhs at the backedge is of the form
+ // 'old(lhs) + N'. We can be sure that old(lhs) is the value at the start
+ // of the iteration, and not that written to lhs in a previous iteration,
+ // as such a previous value could not appear directly in the addition:
+ // it could not be stored in lhs as the lhs add/sub executes in every
+ // iteration, and if it were stored in another variable its use here would
+ // be as an operand to a phi node for that variable.
+ if (lhsModified.term != lhs.term) {
+ return nullptr;
+ }
+
+ LinearSum iterationBound(alloc());
+ LinearSum currentIteration(alloc());
+
+ if (lhsModified.constant == 1 && !lessEqual) {
+ // The value of lhs is 'initial(lhs) + iterCount' and this will end
+ // execution of the loop if 'lhs + lhsN >= rhs'. Thus, an upper bound
+ // on the number of backedges executed is:
+ //
+ // initial(lhs) + iterCount + lhsN == rhs
+ // iterCount == rhsN - initial(lhs) - lhsN
+
+ if (rhs) {
+ if (!iterationBound.add(rhs, 1)) {
+ return nullptr;
+ }
+ }
+ if (!iterationBound.add(lhsInitial, -1)) {
+ return nullptr;
+ }
+
+ int32_t lhsConstant;
+ if (!SafeSub(0, lhs.constant, &lhsConstant)) {
+ return nullptr;
+ }
+ if (!iterationBound.add(lhsConstant)) {
+ return nullptr;
+ }
+
+ if (!currentIteration.add(lhs.term, 1)) {
+ return nullptr;
+ }
+ if (!currentIteration.add(lhsInitial, -1)) {
+ return nullptr;
+ }
+ } else if (lhsModified.constant == -1 && lessEqual) {
+ // The value of lhs is 'initial(lhs) - iterCount'. Similar to the above
+ // case, an upper bound on the number of backedges executed is:
+ //
+ // initial(lhs) - iterCount + lhsN == rhs
+ // iterCount == initial(lhs) - rhs + lhsN
+
+ if (!iterationBound.add(lhsInitial, 1)) {
+ return nullptr;
+ }
+ if (rhs) {
+ if (!iterationBound.add(rhs, -1)) {
+ return nullptr;
+ }
+ }
+ if (!iterationBound.add(lhs.constant)) {
+ return nullptr;
+ }
+
+ if (!currentIteration.add(lhsInitial, 1)) {
+ return nullptr;
+ }
+ if (!currentIteration.add(lhs.term, -1)) {
+ return nullptr;
+ }
+ } else {
+ return nullptr;
+ }
+
+ return new (alloc())
+ LoopIterationBound(header, test, iterationBound, currentIteration);
+}
+
+void RangeAnalysis::analyzeLoopPhi(LoopIterationBound* loopBound, MPhi* phi) {
+ // Given a bound on the number of backedges taken, compute an upper and
+ // lower bound for a phi node that may change by a constant amount each
+ // iteration. Unlike for the case when computing the iteration bound
+ // itself, the phi does not need to change the same amount every iteration,
+ // but is required to change at most N and be either nondecreasing or
+ // nonincreasing.
+
+ MOZ_ASSERT(phi->numOperands() == 2);
+
+ MDefinition* initial = phi->getLoopPredecessorOperand();
+ if (initial->block()->isMarked()) {
+ return;
+ }
+
+ SimpleLinearSum modified =
+ ExtractLinearSum(phi->getLoopBackedgeOperand(), MathSpace::Infinite);
+
+ if (modified.term != phi || modified.constant == 0) {
+ return;
+ }
+
+ if (!phi->range()) {
+ phi->setRange(new (alloc()) Range(phi));
+ }
+
+ LinearSum initialSum(alloc());
+ if (!initialSum.add(initial, 1)) {
+ return;
+ }
+
+ // The phi may change by N each iteration, and is either nondecreasing or
+ // nonincreasing. initial(phi) is either a lower or upper bound for the
+ // phi, and initial(phi) + loopBound * N is either an upper or lower bound,
+ // at all points within the loop, provided that loopBound >= 0.
+ //
+ // We are more interested, however, in the bound for phi at points
+ // dominated by the loop bound's test; if the test dominates e.g. a bounds
+ // check we want to hoist from the loop, using the value of the phi at the
+ // head of the loop for this will usually be too imprecise to hoist the
+ // check. These points will execute only if the backedge executes at least
+ // one more time (as the test passed and the test dominates the backedge),
+ // so we know both that loopBound >= 1 and that the phi's value has changed
+ // at most loopBound - 1 times. Thus, another upper or lower bound for the
+ // phi is initial(phi) + (loopBound - 1) * N, without requiring us to
+ // ensure that loopBound >= 0.
+
+ LinearSum limitSum(loopBound->boundSum);
+ if (!limitSum.multiply(modified.constant) || !limitSum.add(initialSum)) {
+ return;
+ }
+
+ int32_t negativeConstant;
+ if (!SafeSub(0, modified.constant, &negativeConstant) ||
+ !limitSum.add(negativeConstant)) {
+ return;
+ }
+
+ Range* initRange = initial->range();
+ if (modified.constant > 0) {
+ if (initRange && initRange->hasInt32LowerBound()) {
+ phi->range()->refineLower(initRange->lower());
+ }
+ phi->range()->setSymbolicLower(
+ SymbolicBound::New(alloc(), nullptr, initialSum));
+ phi->range()->setSymbolicUpper(
+ SymbolicBound::New(alloc(), loopBound, limitSum));
+ } else {
+ if (initRange && initRange->hasInt32UpperBound()) {
+ phi->range()->refineUpper(initRange->upper());
+ }
+ phi->range()->setSymbolicUpper(
+ SymbolicBound::New(alloc(), nullptr, initialSum));
+ phi->range()->setSymbolicLower(
+ SymbolicBound::New(alloc(), loopBound, limitSum));
+ }
+
+ JitSpew(JitSpew_Range, "added symbolic range on %u", phi->id());
+ SpewRange(phi);
+}
+
+// Whether bound is valid at the specified bounds check instruction in a loop,
+// and may be used to hoist ins.
+static inline bool SymbolicBoundIsValid(MBasicBlock* header, MBoundsCheck* ins,
+ const SymbolicBound* bound) {
+ if (!bound->loop) {
+ return true;
+ }
+ if (ins->block() == header) {
+ return false;
+ }
+ MBasicBlock* bb = ins->block()->immediateDominator();
+ while (bb != header && bb != bound->loop->test->block()) {
+ bb = bb->immediateDominator();
+ }
+ return bb == bound->loop->test->block();
+}
+
+bool RangeAnalysis::tryHoistBoundsCheck(MBasicBlock* header,
+ MBoundsCheck* ins) {
+ // The bounds check's length must be loop invariant or a constant.
+ MDefinition* length = DefinitionOrBetaInputDefinition(ins->length());
+ if (length->block()->isMarked() && !length->isConstant()) {
+ return false;
+ }
+
+ // The bounds check's index should not be loop invariant (else we would
+ // already have hoisted it during LICM).
+ SimpleLinearSum index = ExtractLinearSum(ins->index());
+ if (!index.term || !index.term->block()->isMarked()) {
+ return false;
+ }
+
+ // Check for a symbolic lower and upper bound on the index. If either
+ // condition depends on an iteration bound for the loop, only hoist if
+ // the bounds check is dominated by the iteration bound's test.
+ if (!index.term->range()) {
+ return false;
+ }
+ const SymbolicBound* lower = index.term->range()->symbolicLower();
+ if (!lower || !SymbolicBoundIsValid(header, ins, lower)) {
+ return false;
+ }
+ const SymbolicBound* upper = index.term->range()->symbolicUpper();
+ if (!upper || !SymbolicBoundIsValid(header, ins, upper)) {
+ return false;
+ }
+
+ MBasicBlock* preLoop = header->loopPredecessor();
+ MOZ_ASSERT(!preLoop->isMarked());
+
+ MDefinition* lowerTerm = ConvertLinearSum(alloc(), preLoop, lower->sum,
+ BailoutKind::HoistBoundsCheck);
+ if (!lowerTerm) {
+ return false;
+ }
+
+ MDefinition* upperTerm = ConvertLinearSum(alloc(), preLoop, upper->sum,
+ BailoutKind::HoistBoundsCheck);
+ if (!upperTerm) {
+ return false;
+ }
+
+ // We are checking that index + indexConstant >= 0, and know that
+ // index >= lowerTerm + lowerConstant. Thus, check that:
+ //
+ // lowerTerm + lowerConstant + indexConstant >= 0
+ // lowerTerm >= -lowerConstant - indexConstant
+
+ int32_t lowerConstant = 0;
+ if (!SafeSub(lowerConstant, index.constant, &lowerConstant)) {
+ return false;
+ }
+ if (!SafeSub(lowerConstant, lower->sum.constant(), &lowerConstant)) {
+ return false;
+ }
+
+ // We are checking that index < boundsLength, and know that
+ // index <= upperTerm + upperConstant. Thus, check that:
+ //
+ // upperTerm + upperConstant < boundsLength
+
+ int32_t upperConstant = index.constant;
+ if (!SafeAdd(upper->sum.constant(), upperConstant, &upperConstant)) {
+ return false;
+ }
+
+ // Hoist the loop invariant lower bounds checks.
+ MBoundsCheckLower* lowerCheck = MBoundsCheckLower::New(alloc(), lowerTerm);
+ lowerCheck->setMinimum(lowerConstant);
+ lowerCheck->computeRange(alloc());
+ lowerCheck->collectRangeInfoPreTrunc();
+ lowerCheck->setBailoutKind(BailoutKind::HoistBoundsCheck);
+ preLoop->insertBefore(preLoop->lastIns(), lowerCheck);
+
+ // A common pattern for iterating over typed arrays is this:
+ //
+ // for (var i = 0; i < ta.length; i++) {
+ // use ta[i];
+ // }
+ //
+ // Here |upperTerm| (= ta.length) is a NonNegativeIntPtrToInt32 instruction.
+ // Unwrap this if |length| is also an IntPtr so that we don't add an
+ // unnecessary bounds check and Int32ToIntPtr below.
+ if (upperTerm->isNonNegativeIntPtrToInt32() &&
+ length->type() == MIRType::IntPtr) {
+ upperTerm = upperTerm->toNonNegativeIntPtrToInt32()->input();
+ }
+
+ // Hoist the loop invariant upper bounds checks.
+ if (upperTerm != length || upperConstant >= 0) {
+ // Hoist the bound check's length if it isn't already loop invariant.
+ if (length->block()->isMarked()) {
+ MOZ_ASSERT(length->isConstant());
+ MInstruction* lengthIns = length->toInstruction();
+ lengthIns->block()->moveBefore(preLoop->lastIns(), lengthIns);
+ }
+
+ // If the length is IntPtr, convert the upperTerm to that as well for the
+ // bounds check.
+ if (length->type() == MIRType::IntPtr &&
+ upperTerm->type() == MIRType::Int32) {
+ upperTerm = MInt32ToIntPtr::New(alloc(), upperTerm);
+ upperTerm->computeRange(alloc());
+ upperTerm->collectRangeInfoPreTrunc();
+ preLoop->insertBefore(preLoop->lastIns(), upperTerm->toInstruction());
+ }
+
+ MBoundsCheck* upperCheck = MBoundsCheck::New(alloc(), upperTerm, length);
+ upperCheck->setMinimum(upperConstant);
+ upperCheck->setMaximum(upperConstant);
+ upperCheck->computeRange(alloc());
+ upperCheck->collectRangeInfoPreTrunc();
+ upperCheck->setBailoutKind(BailoutKind::HoistBoundsCheck);
+ preLoop->insertBefore(preLoop->lastIns(), upperCheck);
+ }
+
+ return true;
+}
+
+bool RangeAnalysis::analyze() {
+ JitSpew(JitSpew_Range, "Doing range propagation");
+
+ for (ReversePostorderIterator iter(graph_.rpoBegin());
+ iter != graph_.rpoEnd(); iter++) {
+ MBasicBlock* block = *iter;
+ // No blocks are supposed to be unreachable, except when we have an OSR
+ // block, in which case the Value Numbering phase add fixup blocks which
+ // are unreachable.
+ MOZ_ASSERT(!block->unreachable() || graph_.osrBlock());
+
+ // If the block's immediate dominator is unreachable, the block is
+ // unreachable. Iterating in RPO, we'll always see the immediate
+ // dominator before the block.
+ if (block->immediateDominator()->unreachable()) {
+ block->setUnreachableUnchecked();
+ continue;
+ }
+
+ for (MDefinitionIterator iter(block); iter; iter++) {
+ MDefinition* def = *iter;
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ def->computeRange(alloc());
+ JitSpew(JitSpew_Range, "computing range on %u", def->id());
+ SpewRange(def);
+ }
+
+ // Beta node range analysis may have marked this block unreachable. If
+ // so, it's no longer interesting to continue processing it.
+ if (block->unreachable()) {
+ continue;
+ }
+
+ if (block->isLoopHeader()) {
+ if (!analyzeLoop(block)) {
+ return false;
+ }
+ }
+
+ // First pass at collecting range info - while the beta nodes are still
+ // around and before truncation.
+ for (MInstructionIterator iter(block->begin()); iter != block->end();
+ iter++) {
+ iter->collectRangeInfoPreTrunc();
+ }
+ }
+
+ return true;
+}
+
+bool RangeAnalysis::addRangeAssertions() {
+ if (!JitOptions.checkRangeAnalysis) {
+ return true;
+ }
+
+ // Check the computed range for this instruction, if the option is set. Note
+ // that this code is quite invasive; it adds numerous additional
+ // instructions for each MInstruction with a computed range, and it uses
+ // registers, so it also affects register allocation.
+ for (ReversePostorderIterator iter(graph_.rpoBegin());
+ iter != graph_.rpoEnd(); iter++) {
+ MBasicBlock* block = *iter;
+
+ // Do not add assertions in unreachable blocks.
+ if (block->unreachable()) {
+ continue;
+ }
+
+ for (MDefinitionIterator iter(block); iter; iter++) {
+ MDefinition* ins = *iter;
+
+ // Perform range checking for all numeric and numeric-like types.
+ if (!IsNumberType(ins->type()) && ins->type() != MIRType::Boolean &&
+ ins->type() != MIRType::Value && ins->type() != MIRType::IntPtr) {
+ continue;
+ }
+
+ // MIsNoIter is fused with the MTest that follows it and emitted as
+ // LIsNoIterAndBranch. Skip it to avoid complicating MIsNoIter
+ // lowering.
+ if (ins->isIsNoIter()) {
+ continue;
+ }
+
+ Range r(ins);
+
+ MOZ_ASSERT_IF(ins->type() == MIRType::Int64, r.isUnknown());
+
+ // Don't insert assertions if there's nothing interesting to assert.
+ if (r.isUnknown() ||
+ (ins->type() == MIRType::Int32 && r.isUnknownInt32())) {
+ continue;
+ }
+
+ // Don't add a use to an instruction that is recovered on bailout.
+ if (ins->isRecoveredOnBailout()) {
+ continue;
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ MAssertRange* guard =
+ MAssertRange::New(alloc(), ins, new (alloc()) Range(r));
+
+ // Beta nodes and interrupt checks are required to be located at the
+ // beginnings of basic blocks, so we must insert range assertions
+ // after any such instructions.
+ MInstruction* insertAt = nullptr;
+ if (block->graph().osrBlock() == block) {
+ insertAt = ins->toInstruction();
+ } else {
+ insertAt = block->safeInsertTop(ins);
+ }
+
+ if (insertAt == *iter) {
+ block->insertAfter(insertAt, guard);
+ } else {
+ block->insertBefore(insertAt, guard);
+ }
+ }
+ }
+
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Range based Truncation
+///////////////////////////////////////////////////////////////////////////////
+
+void Range::clampToInt32() {
+ if (isInt32()) {
+ return;
+ }
+ int32_t l = hasInt32LowerBound() ? lower() : JSVAL_INT_MIN;
+ int32_t h = hasInt32UpperBound() ? upper() : JSVAL_INT_MAX;
+ setInt32(l, h);
+}
+
+void Range::wrapAroundToInt32() {
+ if (!hasInt32Bounds()) {
+ setInt32(JSVAL_INT_MIN, JSVAL_INT_MAX);
+ } else if (canHaveFractionalPart()) {
+ // Clearing the fractional field may provide an opportunity to refine
+ // lower_ or upper_.
+ canHaveFractionalPart_ = ExcludesFractionalParts;
+ canBeNegativeZero_ = ExcludesNegativeZero;
+ refineInt32BoundsByExponent(max_exponent_, &lower_, &hasInt32LowerBound_,
+ &upper_, &hasInt32UpperBound_);
+
+ assertInvariants();
+ } else {
+ // If nothing else, we can clear the negative zero flag.
+ canBeNegativeZero_ = ExcludesNegativeZero;
+ }
+ MOZ_ASSERT(isInt32());
+}
+
+void Range::wrapAroundToShiftCount() {
+ wrapAroundToInt32();
+ if (lower() < 0 || upper() >= 32) {
+ setInt32(0, 31);
+ }
+}
+
+void Range::wrapAroundToBoolean() {
+ wrapAroundToInt32();
+ if (!isBoolean()) {
+ setInt32(0, 1);
+ }
+ MOZ_ASSERT(isBoolean());
+}
+
+bool MDefinition::needTruncation(TruncateKind kind) const {
+ // No procedure defined for truncating this instruction.
+ return false;
+}
+
+void MDefinition::truncate(TruncateKind kind) {
+ MOZ_CRASH("No procedure defined for truncating this instruction.");
+}
+
+bool MConstant::needTruncation(TruncateKind kind) const {
+ return IsFloatingPointType(type());
+}
+
+void MConstant::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+
+ // Truncate the double to int, since all uses truncates it.
+ int32_t res = ToInt32(numberToDouble());
+ payload_.asBits = 0;
+ payload_.i32 = res;
+ setResultType(MIRType::Int32);
+ if (range()) {
+ range()->setInt32(res, res);
+ }
+}
+
+bool MPhi::needTruncation(TruncateKind kind) const {
+ if (type() == MIRType::Double || type() == MIRType::Int32) {
+ return true;
+ }
+
+ return false;
+}
+
+void MPhi::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+ truncateKind_ = kind;
+ setResultType(MIRType::Int32);
+ if (kind >= TruncateKind::IndirectTruncate && range()) {
+ range()->wrapAroundToInt32();
+ }
+}
+
+bool MAdd::needTruncation(TruncateKind kind) const {
+ return type() == MIRType::Double || type() == MIRType::Int32;
+}
+
+void MAdd::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+
+ // Remember analysis, needed for fallible checks.
+ setTruncateKind(kind);
+
+ setSpecialization(MIRType::Int32);
+ if (truncateKind() >= TruncateKind::IndirectTruncate && range()) {
+ range()->wrapAroundToInt32();
+ }
+}
+
+bool MSub::needTruncation(TruncateKind kind) const {
+ return type() == MIRType::Double || type() == MIRType::Int32;
+}
+
+void MSub::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+
+ // Remember analysis, needed for fallible checks.
+ setTruncateKind(kind);
+ setSpecialization(MIRType::Int32);
+ if (truncateKind() >= TruncateKind::IndirectTruncate && range()) {
+ range()->wrapAroundToInt32();
+ }
+}
+
+bool MMul::needTruncation(TruncateKind kind) const {
+ return type() == MIRType::Double || type() == MIRType::Int32;
+}
+
+void MMul::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+
+ // Remember analysis, needed for fallible checks.
+ setTruncateKind(kind);
+ setSpecialization(MIRType::Int32);
+ if (truncateKind() >= TruncateKind::IndirectTruncate) {
+ setCanBeNegativeZero(false);
+ if (range()) {
+ range()->wrapAroundToInt32();
+ }
+ }
+}
+
+bool MDiv::needTruncation(TruncateKind kind) const {
+ return type() == MIRType::Double || type() == MIRType::Int32;
+}
+
+void MDiv::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+
+ // Remember analysis, needed for fallible checks.
+ setTruncateKind(kind);
+ setSpecialization(MIRType::Int32);
+
+ // Divisions where the lhs and rhs are unsigned and the result is
+ // truncated can be lowered more efficiently.
+ if (unsignedOperands()) {
+ replaceWithUnsignedOperands();
+ unsigned_ = true;
+ }
+}
+
+bool MMod::needTruncation(TruncateKind kind) const {
+ return type() == MIRType::Double || type() == MIRType::Int32;
+}
+
+void MMod::truncate(TruncateKind kind) {
+ // As for division, handle unsigned modulus with a truncated result.
+ MOZ_ASSERT(needTruncation(kind));
+
+ // Remember analysis, needed for fallible checks.
+ setTruncateKind(kind);
+ setSpecialization(MIRType::Int32);
+
+ if (unsignedOperands()) {
+ replaceWithUnsignedOperands();
+ unsigned_ = true;
+ }
+}
+
+bool MToDouble::needTruncation(TruncateKind kind) const {
+ MOZ_ASSERT(type() == MIRType::Double);
+ return true;
+}
+
+void MToDouble::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+ setTruncateKind(kind);
+
+ // We use the return type to flag that this MToDouble should be replaced by
+ // a MTruncateToInt32 when modifying the graph.
+ setResultType(MIRType::Int32);
+ if (truncateKind() >= TruncateKind::IndirectTruncate) {
+ if (range()) {
+ range()->wrapAroundToInt32();
+ }
+ }
+}
+
+bool MLimitedTruncate::needTruncation(TruncateKind kind) const { return true; }
+
+void MLimitedTruncate::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+ setTruncateKind(kind);
+ setResultType(MIRType::Int32);
+ if (kind >= TruncateKind::IndirectTruncate && range()) {
+ range()->wrapAroundToInt32();
+ }
+}
+
+bool MCompare::needTruncation(TruncateKind kind) const {
+ // If we're compiling wasm, don't try to optimize the comparison type, as
+ // the code presumably is already using the type it wants. Also, wasm
+ // doesn't support bailouts, so we woudn't be able to rely on
+ // TruncateAfterBailouts to convert our inputs.
+ if (block()->info().compilingWasm()) {
+ return false;
+ }
+
+ if (!isDoubleComparison()) {
+ return false;
+ }
+
+ // If both operands are naturally in the int32 range, we can convert from
+ // a double comparison to being an int32 comparison.
+ if (!Range(lhs()).isInt32() || !Range(rhs()).isInt32()) {
+ return false;
+ }
+
+ return true;
+}
+
+void MCompare::truncate(TruncateKind kind) {
+ MOZ_ASSERT(needTruncation(kind));
+ compareType_ = Compare_Int32;
+
+ // Truncating the operands won't change their value because we don't force a
+ // truncation, but it will change their type, which we need because we
+ // now expect integer inputs.
+ truncateOperands_ = true;
+}
+
+TruncateKind MDefinition::operandTruncateKind(size_t index) const {
+ // Generic routine: We don't know anything.
+ return TruncateKind::NoTruncate;
+}
+
+TruncateKind MPhi::operandTruncateKind(size_t index) const {
+ // The truncation applied to a phi is effectively applied to the phi's
+ // operands.
+ return truncateKind_;
+}
+
+TruncateKind MTruncateToInt32::operandTruncateKind(size_t index) const {
+ // This operator is an explicit truncate to int32.
+ return TruncateKind::Truncate;
+}
+
+TruncateKind MBinaryBitwiseInstruction::operandTruncateKind(
+ size_t index) const {
+ // The bitwise operators truncate to int32.
+ return TruncateKind::Truncate;
+}
+
+TruncateKind MLimitedTruncate::operandTruncateKind(size_t index) const {
+ return std::min(truncateKind(), truncateLimit_);
+}
+
+TruncateKind MAdd::operandTruncateKind(size_t index) const {
+ // This operator is doing some arithmetic. If its result is truncated,
+ // it's an indirect truncate for its operands.
+ return std::min(truncateKind(), TruncateKind::IndirectTruncate);
+}
+
+TruncateKind MSub::operandTruncateKind(size_t index) const {
+ // See the comment in MAdd::operandTruncateKind.
+ return std::min(truncateKind(), TruncateKind::IndirectTruncate);
+}
+
+TruncateKind MMul::operandTruncateKind(size_t index) const {
+ // See the comment in MAdd::operandTruncateKind.
+ return std::min(truncateKind(), TruncateKind::IndirectTruncate);
+}
+
+TruncateKind MToDouble::operandTruncateKind(size_t index) const {
+ // MToDouble propagates its truncate kind to its operand.
+ return truncateKind();
+}
+
+TruncateKind MStoreUnboxedScalar::operandTruncateKind(size_t index) const {
+ // An integer store truncates the stored value.
+ return (index == 2 && isIntegerWrite()) ? TruncateKind::Truncate
+ : TruncateKind::NoTruncate;
+}
+
+TruncateKind MStoreDataViewElement::operandTruncateKind(size_t index) const {
+ // An integer store truncates the stored value.
+ return (index == 2 && isIntegerWrite()) ? TruncateKind::Truncate
+ : TruncateKind::NoTruncate;
+}
+
+TruncateKind MStoreTypedArrayElementHole::operandTruncateKind(
+ size_t index) const {
+ // An integer store truncates the stored value.
+ return (index == 3 && isIntegerWrite()) ? TruncateKind::Truncate
+ : TruncateKind::NoTruncate;
+}
+
+TruncateKind MDiv::operandTruncateKind(size_t index) const {
+ return std::min(truncateKind(), TruncateKind::TruncateAfterBailouts);
+}
+
+TruncateKind MMod::operandTruncateKind(size_t index) const {
+ return std::min(truncateKind(), TruncateKind::TruncateAfterBailouts);
+}
+
+TruncateKind MCompare::operandTruncateKind(size_t index) const {
+ // If we're doing an int32 comparison on operands which were previously
+ // floating-point, convert them!
+ MOZ_ASSERT_IF(truncateOperands_, isInt32Comparison());
+ return truncateOperands_ ? TruncateKind::TruncateAfterBailouts
+ : TruncateKind::NoTruncate;
+}
+
+static bool TruncateTest(TempAllocator& alloc, MTest* test) {
+ // If all possible inputs to the test are either int32 or boolean,
+ // convert those inputs to int32 so that an int32 test can be performed.
+
+ if (test->input()->type() != MIRType::Value) {
+ return true;
+ }
+
+ if (!test->input()->isPhi() || !test->input()->hasOneDefUse() ||
+ test->input()->isImplicitlyUsed()) {
+ return true;
+ }
+
+ MPhi* phi = test->input()->toPhi();
+ for (size_t i = 0; i < phi->numOperands(); i++) {
+ MDefinition* def = phi->getOperand(i);
+ if (!def->isBox()) {
+ return true;
+ }
+ MDefinition* inner = def->getOperand(0);
+ if (inner->type() != MIRType::Boolean && inner->type() != MIRType::Int32) {
+ return true;
+ }
+ }
+
+ for (size_t i = 0; i < phi->numOperands(); i++) {
+ MDefinition* inner = phi->getOperand(i)->getOperand(0);
+ if (inner->type() != MIRType::Int32) {
+ if (!alloc.ensureBallast()) {
+ return false;
+ }
+ MBasicBlock* block = inner->block();
+ inner = MToNumberInt32::New(alloc, inner);
+ block->insertBefore(block->lastIns(), inner->toInstruction());
+ }
+ MOZ_ASSERT(inner->type() == MIRType::Int32);
+ phi->replaceOperand(i, inner);
+ }
+
+ phi->setResultType(MIRType::Int32);
+ return true;
+}
+
+// Truncating instruction result is an optimization which implies
+// knowing all uses of an instruction. This implies that if one of
+// the uses got removed, then Range Analysis is not be allowed to do
+// any modification which can change the result, especially if the
+// result can be observed.
+//
+// This corner can easily be understood with UCE examples, but it
+// might also happen with type inference assumptions. Note: Type
+// inference is implicitly branches where other types might be
+// flowing into.
+static bool CloneForDeadBranches(TempAllocator& alloc,
+ MInstruction* candidate) {
+ // Compare returns a boolean so it doesn't have to be recovered on bailout
+ // because the output would remain correct.
+ if (candidate->isCompare()) {
+ return true;
+ }
+
+ MOZ_ASSERT(candidate->canClone());
+ if (!alloc.ensureBallast()) {
+ return false;
+ }
+
+ MDefinitionVector operands(alloc);
+ size_t end = candidate->numOperands();
+ if (!operands.reserve(end)) {
+ return false;
+ }
+ for (size_t i = 0; i < end; ++i) {
+ operands.infallibleAppend(candidate->getOperand(i));
+ }
+
+ MInstruction* clone = candidate->clone(alloc, operands);
+ if (!clone) {
+ return false;
+ }
+ clone->setRange(nullptr);
+
+ // Set ImplicitlyUsed flag on the cloned instruction in order to chain recover
+ // instruction for the bailout path.
+ clone->setImplicitlyUsedUnchecked();
+
+ candidate->block()->insertBefore(candidate, clone);
+
+ if (!candidate->maybeConstantValue()) {
+ MOZ_ASSERT(clone->canRecoverOnBailout());
+ clone->setRecoveredOnBailout();
+ }
+
+ // Replace the candidate by its recovered on bailout clone within recovered
+ // instructions and resume points operands.
+ for (MUseIterator i(candidate->usesBegin()); i != candidate->usesEnd();) {
+ MUse* use = *i++;
+ MNode* ins = use->consumer();
+ if (ins->isDefinition() && !ins->toDefinition()->isRecoveredOnBailout()) {
+ continue;
+ }
+
+ use->replaceProducer(clone);
+ }
+
+ return true;
+}
+
+// Examine all the users of |candidate| and determine the most aggressive
+// truncate kind that satisfies all of them.
+static TruncateKind ComputeRequestedTruncateKind(MDefinition* candidate,
+ bool* shouldClone) {
+ bool isCapturedResult =
+ false; // Check if used by a recovered instruction or a resume point.
+ bool isObservableResult =
+ false; // Check if it can be read from another frame.
+ bool isRecoverableResult = true; // Check if it can safely be reconstructed.
+ bool isImplicitlyUsed = candidate->isImplicitlyUsed();
+ bool hasTryBlock = candidate->block()->graph().hasTryBlock();
+
+ TruncateKind kind = TruncateKind::Truncate;
+ for (MUseIterator use(candidate->usesBegin()); use != candidate->usesEnd();
+ use++) {
+ if (use->consumer()->isResumePoint()) {
+ // Truncation is a destructive optimization, as such, we need to pay
+ // attention to removed branches and prevent optimization
+ // destructive optimizations if we have no alternative. (see
+ // ImplicitlyUsed flag)
+ isCapturedResult = true;
+ isObservableResult =
+ isObservableResult ||
+ use->consumer()->toResumePoint()->isObservableOperand(*use);
+ isRecoverableResult =
+ isRecoverableResult &&
+ use->consumer()->toResumePoint()->isRecoverableOperand(*use);
+ continue;
+ }
+
+ MDefinition* consumer = use->consumer()->toDefinition();
+ if (consumer->isRecoveredOnBailout()) {
+ isCapturedResult = true;
+ isImplicitlyUsed = isImplicitlyUsed || consumer->isImplicitlyUsed();
+ continue;
+ }
+
+ TruncateKind consumerKind =
+ consumer->operandTruncateKind(consumer->indexOf(*use));
+ kind = std::min(kind, consumerKind);
+ if (kind == TruncateKind::NoTruncate) {
+ break;
+ }
+ }
+
+ // We cannot do full trunction on guarded instructions.
+ if (candidate->isGuard() || candidate->isGuardRangeBailouts()) {
+ kind = std::min(kind, TruncateKind::TruncateAfterBailouts);
+ }
+
+ // If the value naturally produces an int32 value (before bailout checks)
+ // that needs no conversion, we don't have to worry about resume points
+ // seeing truncated values.
+ bool needsConversion = !candidate->range() || !candidate->range()->isInt32();
+
+ // If the instruction is explicitly truncated (not indirectly) by all its
+ // uses and if it is not implicitly used, then we can safely encode its
+ // truncated result as part of the resume point operands. This is safe,
+ // because even if we resume with a truncated double, the next baseline
+ // instruction operating on this instruction is going to be a no-op.
+ //
+ // Note, that if the result can be observed from another frame, then this
+ // optimization is not safe. Similarly, if this function contains a try
+ // block, the result could be observed from a catch block, which we do
+ // not compile.
+ bool safeToConvert = kind == TruncateKind::Truncate && !isImplicitlyUsed &&
+ !isObservableResult && !hasTryBlock;
+
+ // If the candidate instruction appears as operand of a resume point or a
+ // recover instruction, and we have to truncate its result, then we might
+ // have to either recover the result during the bailout, or avoid the
+ // truncation.
+ if (isCapturedResult && needsConversion && !safeToConvert) {
+ // If the result can be recovered from all the resume points (not needed
+ // for iterating over the inlined frames), and this instruction can be
+ // recovered on bailout, then we can clone it and use the cloned
+ // instruction to encode the recover instruction. Otherwise, we should
+ // keep the original result and bailout if the value is not in the int32
+ // range.
+ if (!JitOptions.disableRecoverIns && isRecoverableResult &&
+ candidate->canRecoverOnBailout()) {
+ *shouldClone = true;
+ } else {
+ kind = std::min(kind, TruncateKind::TruncateAfterBailouts);
+ }
+ }
+
+ return kind;
+}
+
+static TruncateKind ComputeTruncateKind(MDefinition* candidate,
+ bool* shouldClone) {
+ // Compare operations might coerce its inputs to int32 if the ranges are
+ // correct. So we do not need to check if all uses are coerced.
+ if (candidate->isCompare()) {
+ return TruncateKind::TruncateAfterBailouts;
+ }
+
+ // Set truncated flag if range analysis ensure that it has no
+ // rounding errors and no fractional part. Note that we can't use
+ // the MDefinition Range constructor, because we need to know if
+ // the value will have rounding errors before any bailout checks.
+ const Range* r = candidate->range();
+ bool canHaveRoundingErrors = !r || r->canHaveRoundingErrors();
+
+ // Special case integer division and modulo: a/b can be infinite, and a%b
+ // can be NaN but cannot actually have rounding errors induced by truncation.
+ if ((candidate->isDiv() || candidate->isMod()) &&
+ candidate->type() == MIRType::Int32) {
+ canHaveRoundingErrors = false;
+ }
+
+ if (canHaveRoundingErrors) {
+ return TruncateKind::NoTruncate;
+ }
+
+ // Ensure all observable uses are truncated.
+ return ComputeRequestedTruncateKind(candidate, shouldClone);
+}
+
+static void RemoveTruncatesOnOutput(MDefinition* truncated) {
+ // Compare returns a boolean so it doen't have any output truncates.
+ if (truncated->isCompare()) {
+ return;
+ }
+
+ MOZ_ASSERT(truncated->type() == MIRType::Int32);
+ MOZ_ASSERT(Range(truncated).isInt32());
+
+ for (MUseDefIterator use(truncated); use; use++) {
+ MDefinition* def = use.def();
+ if (!def->isTruncateToInt32() || !def->isToNumberInt32()) {
+ continue;
+ }
+
+ def->replaceAllUsesWith(truncated);
+ }
+}
+
+void RangeAnalysis::adjustTruncatedInputs(MDefinition* truncated) {
+ MBasicBlock* block = truncated->block();
+ for (size_t i = 0, e = truncated->numOperands(); i < e; i++) {
+ TruncateKind kind = truncated->operandTruncateKind(i);
+ if (kind == TruncateKind::NoTruncate) {
+ continue;
+ }
+
+ MDefinition* input = truncated->getOperand(i);
+ if (input->type() == MIRType::Int32) {
+ continue;
+ }
+
+ if (input->isToDouble() && input->getOperand(0)->type() == MIRType::Int32) {
+ truncated->replaceOperand(i, input->getOperand(0));
+ } else {
+ MInstruction* op;
+ if (kind == TruncateKind::TruncateAfterBailouts) {
+ MOZ_ASSERT(!mir->outerInfo().hadEagerTruncationBailout());
+ op = MToNumberInt32::New(alloc(), truncated->getOperand(i));
+ op->setBailoutKind(BailoutKind::EagerTruncation);
+ } else {
+ op = MTruncateToInt32::New(alloc(), truncated->getOperand(i));
+ }
+
+ if (truncated->isPhi()) {
+ MBasicBlock* pred = block->getPredecessor(i);
+ pred->insertBefore(pred->lastIns(), op);
+ } else {
+ block->insertBefore(truncated->toInstruction(), op);
+ }
+ truncated->replaceOperand(i, op);
+ }
+ }
+
+ if (truncated->isToDouble()) {
+ truncated->replaceAllUsesWith(truncated->toToDouble()->getOperand(0));
+ block->discard(truncated->toToDouble());
+ }
+}
+
+bool RangeAnalysis::canTruncate(MDefinition* def, TruncateKind kind) const {
+ if (kind == TruncateKind::NoTruncate) {
+ return false;
+ }
+
+ // Range Analysis is sometimes eager to do optimizations, even if we
+ // are not able to truncate an instruction. In such case, we
+ // speculatively compile the instruction to an int32 instruction
+ // while adding a guard. This is what is implied by
+ // TruncateAfterBailout.
+ //
+ // If a previous compilation was invalidated because a speculative
+ // truncation bailed out, we no longer attempt to make this kind of
+ // eager optimization.
+ if (mir->outerInfo().hadEagerTruncationBailout()) {
+ if (kind == TruncateKind::TruncateAfterBailouts) {
+ return false;
+ }
+ // MDiv and MMod always require TruncateAfterBailout for their operands.
+ // See MDiv::operandTruncateKind and MMod::operandTruncateKind.
+ if (def->isDiv() || def->isMod()) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// Iterate backward on all instruction and attempt to truncate operations for
+// each instruction which respect the following list of predicates: Has been
+// analyzed by range analysis, the range has no rounding errors, all uses cases
+// are truncating the result.
+//
+// If the truncation of the operation is successful, then the instruction is
+// queue for later updating the graph to restore the type correctness by
+// converting the operands that need to be truncated.
+//
+// We iterate backward because it is likely that a truncated operation truncates
+// some of its operands.
+bool RangeAnalysis::truncate() {
+ JitSpew(JitSpew_Range, "Do range-base truncation (backward loop)");
+
+ // Automatic truncation is disabled for wasm because the truncation logic
+ // is based on IonMonkey which assumes that we can bailout if the truncation
+ // logic fails. As wasm code has no bailout mechanism, it is safer to avoid
+ // any automatic truncations.
+ MOZ_ASSERT(!mir->compilingWasm());
+
+ Vector<MDefinition*, 16, SystemAllocPolicy> worklist;
+
+ for (PostorderIterator block(graph_.poBegin()); block != graph_.poEnd();
+ block++) {
+ for (MInstructionReverseIterator iter(block->rbegin());
+ iter != block->rend(); iter++) {
+ if (iter->isRecoveredOnBailout()) {
+ continue;
+ }
+
+ if (iter->type() == MIRType::None) {
+ if (iter->isTest()) {
+ if (!TruncateTest(alloc(), iter->toTest())) {
+ return false;
+ }
+ }
+ continue;
+ }
+
+ // Remember all bitop instructions for folding after range analysis.
+ switch (iter->op()) {
+ case MDefinition::Opcode::BitAnd:
+ case MDefinition::Opcode::BitOr:
+ case MDefinition::Opcode::BitXor:
+ case MDefinition::Opcode::Lsh:
+ case MDefinition::Opcode::Rsh:
+ case MDefinition::Opcode::Ursh:
+ if (!bitops.append(static_cast<MBinaryBitwiseInstruction*>(*iter))) {
+ return false;
+ }
+ break;
+ default:;
+ }
+
+ bool shouldClone = false;
+ TruncateKind kind = ComputeTruncateKind(*iter, &shouldClone);
+
+ // Truncate this instruction if possible.
+ if (!canTruncate(*iter, kind) || !iter->needTruncation(kind)) {
+ continue;
+ }
+
+ SpewTruncate(*iter, kind, shouldClone);
+
+ // If needed, clone the current instruction for keeping it for the
+ // bailout path. This give us the ability to truncate instructions
+ // even after the removal of branches.
+ if (shouldClone && !CloneForDeadBranches(alloc(), *iter)) {
+ return false;
+ }
+
+ // TruncateAfterBailouts keeps the bailout code as-is and
+ // continues with truncated operations, with the expectation
+ // that we are unlikely to bail out. If we do bail out, then we
+ // will set a flag in FinishBailoutToBaseline to prevent eager
+ // truncation when we recompile, to avoid bailout loops.
+ if (kind == TruncateKind::TruncateAfterBailouts) {
+ iter->setBailoutKind(BailoutKind::EagerTruncation);
+ }
+
+ iter->truncate(kind);
+
+ // Delay updates of inputs/outputs to avoid creating node which
+ // would be removed by the truncation of the next operations.
+ iter->setInWorklist();
+ if (!worklist.append(*iter)) {
+ return false;
+ }
+ }
+ for (MPhiIterator iter(block->phisBegin()), end(block->phisEnd());
+ iter != end; ++iter) {
+ bool shouldClone = false;
+ TruncateKind kind = ComputeTruncateKind(*iter, &shouldClone);
+
+ // Truncate this phi if possible.
+ if (shouldClone || !canTruncate(*iter, kind) ||
+ !iter->needTruncation(kind)) {
+ continue;
+ }
+
+ SpewTruncate(*iter, kind, shouldClone);
+
+ iter->truncate(kind);
+
+ // Delay updates of inputs/outputs to avoid creating node which
+ // would be removed by the truncation of the next operations.
+ iter->setInWorklist();
+ if (!worklist.append(*iter)) {
+ return false;
+ }
+ }
+ }
+
+ // Update inputs/outputs of truncated instructions.
+ JitSpew(JitSpew_Range, "Do graph type fixup (dequeue)");
+ while (!worklist.empty()) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ MDefinition* def = worklist.popCopy();
+ def->setNotInWorklist();
+ RemoveTruncatesOnOutput(def);
+ adjustTruncatedInputs(def);
+ }
+
+ return true;
+}
+
+bool RangeAnalysis::removeUnnecessaryBitops() {
+ JitSpew(JitSpew_Range, "Begin (removeUnnecessaryBitops)");
+ // Note: This operation change the semantic of the program in a way which
+ // uniquely works with Int32, Recover Instructions added by the Sink phase
+ // expects the MIR Graph to still have a valid flow as-if they were double
+ // operations instead of Int32 operations. Thus, this phase should be
+ // executed after the Sink phase, and before DCE.
+
+ // Fold any unnecessary bitops in the graph, such as (x | 0) on an integer
+ // input. This is done after range analysis rather than during GVN as the
+ // presence of the bitop can change which instructions are truncated.
+ for (size_t i = 0; i < bitops.length(); i++) {
+ MBinaryBitwiseInstruction* ins = bitops[i];
+ if (ins->isRecoveredOnBailout()) {
+ continue;
+ }
+
+ MDefinition* folded = ins->foldUnnecessaryBitop();
+ if (folded != ins) {
+ ins->replaceAllLiveUsesWith(folded);
+ ins->setRecoveredOnBailout();
+ }
+ }
+
+ bitops.clear();
+ return true;
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// Collect Range information of operands
+///////////////////////////////////////////////////////////////////////////////
+
+void MInArray::collectRangeInfoPreTrunc() {
+ Range indexRange(index());
+ if (indexRange.isFiniteNonNegative()) {
+ needsNegativeIntCheck_ = false;
+ setNotGuard();
+ }
+}
+
+void MLoadElementHole::collectRangeInfoPreTrunc() {
+ Range indexRange(index());
+ if (indexRange.isFiniteNonNegative()) {
+ needsNegativeIntCheck_ = false;
+ setNotGuard();
+ }
+}
+
+void MInt32ToIntPtr::collectRangeInfoPreTrunc() {
+ Range inputRange(input());
+ if (inputRange.isFiniteNonNegative()) {
+ canBeNegative_ = false;
+ }
+}
+
+void MClz::collectRangeInfoPreTrunc() {
+ Range inputRange(input());
+ if (!inputRange.canBeZero()) {
+ operandIsNeverZero_ = true;
+ }
+}
+
+void MCtz::collectRangeInfoPreTrunc() {
+ Range inputRange(input());
+ if (!inputRange.canBeZero()) {
+ operandIsNeverZero_ = true;
+ }
+}
+
+void MDiv::collectRangeInfoPreTrunc() {
+ Range lhsRange(lhs());
+ Range rhsRange(rhs());
+
+ // Test if Dividend is non-negative.
+ if (lhsRange.isFiniteNonNegative()) {
+ canBeNegativeDividend_ = false;
+ }
+
+ // Try removing divide by zero check.
+ if (!rhsRange.canBeZero()) {
+ canBeDivideByZero_ = false;
+ }
+
+ // If lhsRange does not contain INT32_MIN in its range,
+ // negative overflow check can be skipped.
+ if (!lhsRange.contains(INT32_MIN)) {
+ canBeNegativeOverflow_ = false;
+ }
+
+ // If rhsRange does not contain -1 likewise.
+ if (!rhsRange.contains(-1)) {
+ canBeNegativeOverflow_ = false;
+ }
+
+ // If lhsRange does not contain a zero,
+ // negative zero check can be skipped.
+ if (!lhsRange.canBeZero()) {
+ canBeNegativeZero_ = false;
+ }
+
+ // If rhsRange >= 0 negative zero check can be skipped.
+ if (rhsRange.isFiniteNonNegative()) {
+ canBeNegativeZero_ = false;
+ }
+
+ if (fallible()) {
+ setGuardRangeBailoutsUnchecked();
+ }
+}
+
+void MMul::collectRangeInfoPreTrunc() {
+ Range lhsRange(lhs());
+ Range rhsRange(rhs());
+
+ // If lhsRange contains only positive then we can skip negative zero check.
+ if (lhsRange.isFiniteNonNegative() && !lhsRange.canBeZero()) {
+ setCanBeNegativeZero(false);
+ }
+
+ // Likewise rhsRange.
+ if (rhsRange.isFiniteNonNegative() && !rhsRange.canBeZero()) {
+ setCanBeNegativeZero(false);
+ }
+
+ // If rhsRange and lhsRange contain Non-negative integers only,
+ // We skip negative zero check.
+ if (rhsRange.isFiniteNonNegative() && lhsRange.isFiniteNonNegative()) {
+ setCanBeNegativeZero(false);
+ }
+
+ // If rhsRange and lhsRange < 0. Then we skip negative zero check.
+ if (rhsRange.isFiniteNegative() && lhsRange.isFiniteNegative()) {
+ setCanBeNegativeZero(false);
+ }
+}
+
+void MMod::collectRangeInfoPreTrunc() {
+ Range lhsRange(lhs());
+ Range rhsRange(rhs());
+ if (lhsRange.isFiniteNonNegative()) {
+ canBeNegativeDividend_ = false;
+ }
+ if (!rhsRange.canBeZero()) {
+ canBeDivideByZero_ = false;
+ }
+ if (type() == MIRType::Int32 && fallible()) {
+ setGuardRangeBailoutsUnchecked();
+ }
+}
+
+void MToNumberInt32::collectRangeInfoPreTrunc() {
+ Range inputRange(input());
+ if (!inputRange.canBeNegativeZero()) {
+ needsNegativeZeroCheck_ = false;
+ }
+}
+
+void MBoundsCheck::collectRangeInfoPreTrunc() {
+ Range indexRange(index());
+ Range lengthRange(length());
+ if (!indexRange.hasInt32LowerBound() || !indexRange.hasInt32UpperBound()) {
+ return;
+ }
+ if (!lengthRange.hasInt32LowerBound() || lengthRange.canBeNaN()) {
+ return;
+ }
+
+ int64_t indexLower = indexRange.lower();
+ int64_t indexUpper = indexRange.upper();
+ int64_t lengthLower = lengthRange.lower();
+ int64_t min = minimum();
+ int64_t max = maximum();
+
+ if (indexLower + min >= 0 && indexUpper + max < lengthLower) {
+ fallible_ = false;
+ }
+}
+
+void MBoundsCheckLower::collectRangeInfoPreTrunc() {
+ Range indexRange(index());
+ if (indexRange.hasInt32LowerBound() && indexRange.lower() >= minimum_) {
+ fallible_ = false;
+ }
+}
+
+void MCompare::collectRangeInfoPreTrunc() {
+ if (!Range(lhs()).canBeNaN() && !Range(rhs()).canBeNaN()) {
+ operandsAreNeverNaN_ = true;
+ }
+}
+
+void MNot::collectRangeInfoPreTrunc() {
+ if (!Range(input()).canBeNaN()) {
+ operandIsNeverNaN_ = true;
+ }
+}
+
+void MPowHalf::collectRangeInfoPreTrunc() {
+ Range inputRange(input());
+ if (!inputRange.canBeInfiniteOrNaN() || inputRange.hasInt32LowerBound()) {
+ operandIsNeverNegativeInfinity_ = true;
+ }
+ if (!inputRange.canBeNegativeZero()) {
+ operandIsNeverNegativeZero_ = true;
+ }
+ if (!inputRange.canBeNaN()) {
+ operandIsNeverNaN_ = true;
+ }
+}
+
+void MUrsh::collectRangeInfoPreTrunc() {
+ if (type() == MIRType::Int64) {
+ return;
+ }
+
+ Range lhsRange(lhs()), rhsRange(rhs());
+
+ // As in MUrsh::computeRange(), convert the inputs.
+ lhsRange.wrapAroundToInt32();
+ rhsRange.wrapAroundToShiftCount();
+
+ // If the most significant bit of our result is always going to be zero,
+ // we can optimize by disabling bailout checks for enforcing an int32 range.
+ if (lhsRange.lower() >= 0 || rhsRange.lower() >= 1) {
+ bailoutsDisabled_ = true;
+ }
+}
+
+static bool DoesMaskMatchRange(int32_t mask, Range& range) {
+ // Check if range is positive, because the bitand operator in `(-3) & 0xff`
+ // can't be eliminated.
+ if (range.lower() >= 0) {
+ MOZ_ASSERT(range.isInt32());
+ // Check that the mask value has all bits set given the range upper bound.
+ // Note that the upper bound does not have to be exactly the mask value. For
+ // example, consider `x & 0xfff` where `x` is a uint8. That expression can
+ // still be optimized to `x`.
+ int bits = 1 + FloorLog2(range.upper());
+ uint32_t maskNeeded = (bits == 32) ? 0xffffffff : (uint32_t(1) << bits) - 1;
+ if ((mask & maskNeeded) == maskNeeded) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void MBinaryBitwiseInstruction::collectRangeInfoPreTrunc() {
+ Range lhsRange(lhs());
+ Range rhsRange(rhs());
+
+ if (lhs()->isConstant() && lhs()->type() == MIRType::Int32 &&
+ DoesMaskMatchRange(lhs()->toConstant()->toInt32(), rhsRange)) {
+ maskMatchesRightRange = true;
+ }
+
+ if (rhs()->isConstant() && rhs()->type() == MIRType::Int32 &&
+ DoesMaskMatchRange(rhs()->toConstant()->toInt32(), lhsRange)) {
+ maskMatchesLeftRange = true;
+ }
+}
+
+void MNaNToZero::collectRangeInfoPreTrunc() {
+ Range inputRange(input());
+
+ if (!inputRange.canBeNaN()) {
+ operandIsNeverNaN_ = true;
+ }
+ if (!inputRange.canBeNegativeZero()) {
+ operandIsNeverNegativeZero_ = true;
+ }
+}
+
+bool RangeAnalysis::prepareForUCE(bool* shouldRemoveDeadCode) {
+ *shouldRemoveDeadCode = false;
+
+ for (ReversePostorderIterator iter(graph_.rpoBegin());
+ iter != graph_.rpoEnd(); iter++) {
+ MBasicBlock* block = *iter;
+
+ if (!block->unreachable()) {
+ continue;
+ }
+
+ // Filter out unreachable fake entries.
+ if (block->numPredecessors() == 0) {
+ // Ignore fixup blocks added by the Value Numbering phase, in order
+ // to keep the dominator tree as-is when we have OSR Block which are
+ // no longer reachable from the main entry point of the graph.
+ MOZ_ASSERT(graph_.osrBlock());
+ continue;
+ }
+
+ MControlInstruction* cond = block->getPredecessor(0)->lastIns();
+ if (!cond->isTest()) {
+ continue;
+ }
+
+ // Replace the condition of the test control instruction by a constant
+ // chosen based which of the successors has the unreachable flag which is
+ // added by MBeta::computeRange on its own block.
+ MTest* test = cond->toTest();
+ MDefinition* condition = test->input();
+
+ // If the false-branch is unreachable, then the test condition must be true.
+ // If the true-branch is unreachable, then the test condition must be false.
+ MOZ_ASSERT(block == test->ifTrue() || block == test->ifFalse());
+ bool value = block == test->ifFalse();
+ MConstant* constant =
+ MConstant::New(alloc().fallible(), BooleanValue(value));
+ if (!constant) {
+ return false;
+ }
+
+ condition->setGuardRangeBailoutsUnchecked();
+
+ test->block()->insertBefore(test, constant);
+
+ test->replaceOperand(0, constant);
+ JitSpew(JitSpew_Range,
+ "Update condition of %u to reflect unreachable branches.",
+ test->id());
+
+ *shouldRemoveDeadCode = true;
+ }
+
+ return tryRemovingGuards();
+}
+
+bool RangeAnalysis::tryRemovingGuards() {
+ MDefinitionVector guards(alloc());
+
+ for (ReversePostorderIterator block = graph_.rpoBegin();
+ block != graph_.rpoEnd(); block++) {
+ for (MDefinitionIterator iter(*block); iter; iter++) {
+ if (!iter->isGuardRangeBailouts()) {
+ continue;
+ }
+
+ iter->setInWorklist();
+ if (!guards.append(*iter)) {
+ return false;
+ }
+ }
+ }
+
+ // Flag all fallible instructions which were indirectly used in the
+ // computation of the condition, such that we do not ignore
+ // bailout-paths which are used to shrink the input range of the
+ // operands of the condition.
+ for (size_t i = 0; i < guards.length(); i++) {
+ MDefinition* guard = guards[i];
+
+ // If this ins is a guard even without guardRangeBailouts,
+ // there is no reason in trying to hoist the guardRangeBailouts check.
+ guard->setNotGuardRangeBailouts();
+ if (!DeadIfUnused(guard)) {
+ guard->setGuardRangeBailouts();
+ continue;
+ }
+ guard->setGuardRangeBailouts();
+
+ if (!guard->isPhi()) {
+ if (!guard->range()) {
+ continue;
+ }
+
+ // Filter the range of the instruction based on its MIRType.
+ Range typeFilteredRange(guard);
+
+ // If the output range is updated by adding the inner range,
+ // then the MIRType act as an effectful filter. As we do not know if
+ // this filtered Range might change or not the result of the
+ // previous comparison, we have to keep this instruction as a guard
+ // because it has to bailout in order to restrict the Range to its
+ // MIRType.
+ if (typeFilteredRange.update(guard->range())) {
+ continue;
+ }
+ }
+
+ guard->setNotGuardRangeBailouts();
+
+ // Propagate the guard to its operands.
+ for (size_t op = 0, e = guard->numOperands(); op < e; op++) {
+ MDefinition* operand = guard->getOperand(op);
+
+ // Already marked.
+ if (operand->isInWorklist()) {
+ continue;
+ }
+
+ MOZ_ASSERT(!operand->isGuardRangeBailouts());
+
+ operand->setInWorklist();
+ operand->setGuardRangeBailouts();
+ if (!guards.append(operand)) {
+ return false;
+ }
+ }
+ }
+
+ for (size_t i = 0; i < guards.length(); i++) {
+ MDefinition* guard = guards[i];
+ guard->setNotInWorklist();
+ }
+
+ return true;
+}
diff --git a/js/src/jit/RangeAnalysis.h b/js/src/jit/RangeAnalysis.h
new file mode 100644
index 0000000000..3c4eef4010
--- /dev/null
+++ b/js/src/jit/RangeAnalysis.h
@@ -0,0 +1,683 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_RangeAnalysis_h
+#define jit_RangeAnalysis_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+#include <stdint.h>
+
+#include "jit/IonAnalysis.h"
+#include "jit/IonTypes.h"
+#include "jit/JitAllocPolicy.h"
+#include "js/AllocPolicy.h"
+#include "js/Value.h"
+#include "js/Vector.h"
+
+namespace js {
+
+class GenericPrinter;
+
+namespace jit {
+
+class MBasicBlock;
+class MBinaryBitwiseInstruction;
+class MBoundsCheck;
+class MDefinition;
+class MIRGenerator;
+class MIRGraph;
+class MPhi;
+class MTest;
+
+enum class TruncateKind;
+
+// An upper bound computed on the number of backedges a loop will take.
+// This count only includes backedges taken while running Ion code: for OSR
+// loops, this will exclude iterations that executed in the interpreter or in
+// baseline compiled code.
+struct LoopIterationBound : public TempObject {
+ // Loop for which this bound applies.
+ MBasicBlock* header;
+
+ // Test from which this bound was derived; after executing exactly 'bound'
+ // times this test will exit the loop. Code in the loop body which this
+ // test dominates (will include the backedge) will execute at most 'bound'
+ // times. Other code in the loop will execute at most '1 + Max(bound, 0)'
+ // times.
+ MTest* test;
+
+ // Symbolic bound computed for the number of backedge executions. The terms
+ // in this bound are all loop invariant.
+ LinearSum boundSum;
+
+ // Linear sum for the number of iterations already executed, at the start
+ // of the loop header. This will use loop invariant terms and header phis.
+ LinearSum currentSum;
+
+ LoopIterationBound(MBasicBlock* header, MTest* test,
+ const LinearSum& boundSum, const LinearSum& currentSum)
+ : header(header),
+ test(test),
+ boundSum(boundSum),
+ currentSum(currentSum) {}
+};
+
+typedef Vector<LoopIterationBound*, 0, SystemAllocPolicy>
+ LoopIterationBoundVector;
+
+// A symbolic upper or lower bound computed for a term.
+struct SymbolicBound : public TempObject {
+ private:
+ SymbolicBound(LoopIterationBound* loop, const LinearSum& sum)
+ : loop(loop), sum(sum) {}
+
+ public:
+ // Any loop iteration bound from which this was derived.
+ //
+ // If non-nullptr, then 'sum' is only valid within the loop body, at
+ // points dominated by the loop bound's test (see LoopIterationBound).
+ //
+ // If nullptr, then 'sum' is always valid.
+ LoopIterationBound* loop;
+
+ static SymbolicBound* New(TempAllocator& alloc, LoopIterationBound* loop,
+ const LinearSum& sum) {
+ return new (alloc) SymbolicBound(loop, sum);
+ }
+
+ // Computed symbolic bound, see above.
+ LinearSum sum;
+
+ void dump(GenericPrinter& out) const;
+ void dump() const;
+};
+
+class RangeAnalysis {
+ protected:
+ bool blockDominates(MBasicBlock* b, MBasicBlock* b2);
+ void replaceDominatedUsesWith(MDefinition* orig, MDefinition* dom,
+ MBasicBlock* block);
+
+ protected:
+ MIRGenerator* mir;
+ MIRGraph& graph_;
+ Vector<MBinaryBitwiseInstruction*, 16, SystemAllocPolicy> bitops;
+
+ TempAllocator& alloc() const;
+
+ public:
+ RangeAnalysis(MIRGenerator* mir, MIRGraph& graph) : mir(mir), graph_(graph) {}
+ [[nodiscard]] bool addBetaNodes();
+ [[nodiscard]] bool analyze();
+ [[nodiscard]] bool addRangeAssertions();
+ [[nodiscard]] bool removeBetaNodes();
+ [[nodiscard]] bool prepareForUCE(bool* shouldRemoveDeadCode);
+ [[nodiscard]] bool tryRemovingGuards();
+ [[nodiscard]] bool truncate();
+ [[nodiscard]] bool removeUnnecessaryBitops();
+
+ bool canTruncate(MDefinition* def, TruncateKind kind) const;
+ void adjustTruncatedInputs(MDefinition* def);
+
+ // Any iteration bounds discovered for loops in the graph.
+ LoopIterationBoundVector loopIterationBounds;
+
+ private:
+ [[nodiscard]] bool analyzeLoop(MBasicBlock* header);
+ LoopIterationBound* analyzeLoopIterationCount(MBasicBlock* header,
+ MTest* test,
+ BranchDirection direction);
+ void analyzeLoopPhi(LoopIterationBound* loopBound, MPhi* phi);
+ [[nodiscard]] bool tryHoistBoundsCheck(MBasicBlock* header,
+ MBoundsCheck* ins);
+};
+
+class Range : public TempObject {
+ public:
+ // Int32 are signed. INT32_MAX is pow(2,31)-1 and INT32_MIN is -pow(2,31),
+ // so the greatest exponent we need is 31.
+ static const uint16_t MaxInt32Exponent = 31;
+
+ // UInt32 are unsigned. UINT32_MAX is pow(2,32)-1, so it's the greatest
+ // value that has an exponent of 31.
+ static const uint16_t MaxUInt32Exponent = 31;
+
+ // Maximal exponenent under which we have no precission loss on double
+ // operations. Double has 52 bits of mantissa, so 2^52+1 cannot be
+ // represented without loss.
+ static const uint16_t MaxTruncatableExponent =
+ mozilla::FloatingPoint<double>::kExponentShift;
+
+ // Maximum exponent for finite values.
+ static const uint16_t MaxFiniteExponent =
+ mozilla::FloatingPoint<double>::kExponentBias;
+
+ // An special exponent value representing all non-NaN values. This
+ // includes finite values and the infinities.
+ static const uint16_t IncludesInfinity = MaxFiniteExponent + 1;
+
+ // An special exponent value representing all possible double-precision
+ // values. This includes finite values, the infinities, and NaNs.
+ static const uint16_t IncludesInfinityAndNaN = UINT16_MAX;
+
+ // This range class uses int32_t ranges, but has several interfaces which
+ // use int64_t, which either holds an int32_t value, or one of the following
+ // special values which mean a value which is beyond the int32 range,
+ // potentially including infinity or NaN. These special values are
+ // guaranteed to compare greater, and less than, respectively, any int32_t
+ // value.
+ static const int64_t NoInt32UpperBound = int64_t(JSVAL_INT_MAX) + 1;
+ static const int64_t NoInt32LowerBound = int64_t(JSVAL_INT_MIN) - 1;
+
+ enum FractionalPartFlag : bool {
+ ExcludesFractionalParts = false,
+ IncludesFractionalParts = true
+ };
+ enum NegativeZeroFlag : bool {
+ ExcludesNegativeZero = false,
+ IncludesNegativeZero = true
+ };
+
+ private:
+ // Absolute ranges.
+ //
+ // We represent ranges where the endpoints can be in the set:
+ // {-infty} U [INT_MIN, INT_MAX] U {infty}. A bound of +/-
+ // infty means that the value may have overflowed in that
+ // direction. When computing the range of an integer
+ // instruction, the ranges of the operands can be clamped to
+ // [INT_MIN, INT_MAX], since if they had overflowed they would
+ // no longer be integers. This is important for optimizations
+ // and somewhat subtle.
+ //
+ // N.B.: All of the operations that compute new ranges based
+ // on existing ranges will ignore the hasInt32*Bound_ flags of the
+ // input ranges; that is, they implicitly clamp the ranges of
+ // the inputs to [INT_MIN, INT_MAX]. Therefore, while our range might
+ // be unbounded (and could overflow), when using this information to
+ // propagate through other ranges, we disregard this fact; if that code
+ // executes, then the overflow did not occur, so we may safely assume
+ // that the range is [INT_MIN, INT_MAX] instead.
+ //
+ // To facilitate this trick, we maintain the invariants that:
+ // 1) hasInt32LowerBound_ == false implies lower_ == JSVAL_INT_MIN
+ // 2) hasInt32UpperBound_ == false implies upper_ == JSVAL_INT_MAX
+ //
+ // As a second and less precise range analysis, we represent the maximal
+ // exponent taken by a value. The exponent is calculated by taking the
+ // absolute value and looking at the position of the highest bit. All
+ // exponent computation have to be over-estimations of the actual result. On
+ // the Int32 this over approximation is rectified.
+
+ MOZ_INIT_OUTSIDE_CTOR int32_t lower_;
+ MOZ_INIT_OUTSIDE_CTOR int32_t upper_;
+
+ MOZ_INIT_OUTSIDE_CTOR bool hasInt32LowerBound_;
+ MOZ_INIT_OUTSIDE_CTOR bool hasInt32UpperBound_;
+
+ MOZ_INIT_OUTSIDE_CTOR FractionalPartFlag canHaveFractionalPart_ : 1;
+ MOZ_INIT_OUTSIDE_CTOR NegativeZeroFlag canBeNegativeZero_ : 1;
+ MOZ_INIT_OUTSIDE_CTOR uint16_t max_exponent_;
+
+ // Any symbolic lower or upper bound computed for this term.
+ const SymbolicBound* symbolicLower_;
+ const SymbolicBound* symbolicUpper_;
+
+ // This function simply makes several MOZ_ASSERTs to verify the internal
+ // consistency of this range.
+ void assertInvariants() const {
+ // Basic sanity :).
+ MOZ_ASSERT(lower_ <= upper_);
+
+ // When hasInt32LowerBound_ or hasInt32UpperBound_ are false, we set
+ // lower_ and upper_ to these specific values as it simplifies the
+ // implementation in some places.
+ MOZ_ASSERT_IF(!hasInt32LowerBound_, lower_ == JSVAL_INT_MIN);
+ MOZ_ASSERT_IF(!hasInt32UpperBound_, upper_ == JSVAL_INT_MAX);
+
+ // max_exponent_ must be one of three possible things.
+ MOZ_ASSERT(max_exponent_ <= MaxFiniteExponent ||
+ max_exponent_ == IncludesInfinity ||
+ max_exponent_ == IncludesInfinityAndNaN);
+
+ // Forbid the max_exponent_ field from implying better bounds for
+ // lower_/upper_ fields. We have to add 1 to the max_exponent_ when
+ // canHaveFractionalPart_ is true in order to accomodate
+ // fractional offsets. For example, 2147483647.9 is greater than
+ // INT32_MAX, so a range containing that value will have
+ // hasInt32UpperBound_ set to false, however that value also has
+ // exponent 30, which is strictly less than MaxInt32Exponent. For
+ // another example, 1.9 has an exponent of 0 but requires upper_ to be
+ // at least 2, which has exponent 1.
+ mozilla::DebugOnly<uint32_t> adjustedExponent =
+ max_exponent_ + (canHaveFractionalPart_ ? 1 : 0);
+ MOZ_ASSERT_IF(!hasInt32LowerBound_ || !hasInt32UpperBound_,
+ adjustedExponent >= MaxInt32Exponent);
+ MOZ_ASSERT(adjustedExponent >= mozilla::FloorLog2(mozilla::Abs(upper_)));
+ MOZ_ASSERT(adjustedExponent >= mozilla::FloorLog2(mozilla::Abs(lower_)));
+
+ // The following are essentially static assertions, but FloorLog2 isn't
+ // trivially suitable for constexpr :(.
+ MOZ_ASSERT(mozilla::FloorLog2(JSVAL_INT_MIN) == MaxInt32Exponent);
+ MOZ_ASSERT(mozilla::FloorLog2(JSVAL_INT_MAX) == 30);
+ MOZ_ASSERT(mozilla::FloorLog2(UINT32_MAX) == MaxUInt32Exponent);
+ MOZ_ASSERT(mozilla::FloorLog2(0) == 0);
+ }
+
+ // Set the lower_ and hasInt32LowerBound_ values.
+ void setLowerInit(int64_t x) {
+ if (x > JSVAL_INT_MAX) {
+ lower_ = JSVAL_INT_MAX;
+ hasInt32LowerBound_ = true;
+ } else if (x < JSVAL_INT_MIN) {
+ lower_ = JSVAL_INT_MIN;
+ hasInt32LowerBound_ = false;
+ } else {
+ lower_ = int32_t(x);
+ hasInt32LowerBound_ = true;
+ }
+ }
+ // Set the upper_ and hasInt32UpperBound_ values.
+ void setUpperInit(int64_t x) {
+ if (x > JSVAL_INT_MAX) {
+ upper_ = JSVAL_INT_MAX;
+ hasInt32UpperBound_ = false;
+ } else if (x < JSVAL_INT_MIN) {
+ upper_ = JSVAL_INT_MIN;
+ hasInt32UpperBound_ = true;
+ } else {
+ upper_ = int32_t(x);
+ hasInt32UpperBound_ = true;
+ }
+ }
+
+ // Compute the least exponent value that would be compatible with the
+ // values of lower() and upper().
+ //
+ // Note:
+ // exponent of JSVAL_INT_MIN == 31
+ // exponent of JSVAL_INT_MAX == 30
+ uint16_t exponentImpliedByInt32Bounds() const {
+ // The number of bits needed to encode |max| is the power of 2 plus one.
+ uint32_t max = std::max(mozilla::Abs(lower()), mozilla::Abs(upper()));
+ uint16_t result = mozilla::FloorLog2(max);
+ MOZ_ASSERT(result ==
+ (max == 0 ? 0 : mozilla::ExponentComponent(double(max))));
+ return result;
+ }
+
+ // When converting a range which contains fractional values to a range
+ // containing only integers, the old max_exponent_ value may imply a better
+ // lower and/or upper bound than was previously available, because they no
+ // longer need to be conservative about fractional offsets and the ends of
+ // the range.
+ //
+ // Given an exponent value and pointers to the lower and upper bound values,
+ // this function refines the lower and upper bound values to the tighest
+ // bound for integer values implied by the exponent.
+ static void refineInt32BoundsByExponent(uint16_t e, int32_t* l, bool* lb,
+ int32_t* h, bool* hb) {
+ if (e < MaxInt32Exponent) {
+ // pow(2, max_exponent_+1)-1 to compute a maximum absolute value.
+ int32_t limit = (uint32_t(1) << (e + 1)) - 1;
+ *h = std::min(*h, limit);
+ *l = std::max(*l, -limit);
+ *hb = true;
+ *lb = true;
+ }
+ }
+
+ // If the value of any of the fields implies a stronger possible value for
+ // any other field, update that field to the stronger value. The range must
+ // be completely valid before and it is guaranteed to be kept valid.
+ void optimize() {
+ assertInvariants();
+
+ if (hasInt32Bounds()) {
+ // Examine lower() and upper(), and if they imply a better exponent
+ // bound than max_exponent_, set that value as the new
+ // max_exponent_.
+ uint16_t newExponent = exponentImpliedByInt32Bounds();
+ if (newExponent < max_exponent_) {
+ max_exponent_ = newExponent;
+ assertInvariants();
+ }
+
+ // If we have a completely precise range, the value is an integer,
+ // since we can only represent integers.
+ if (canHaveFractionalPart_ && lower_ == upper_) {
+ canHaveFractionalPart_ = ExcludesFractionalParts;
+ assertInvariants();
+ }
+ }
+
+ // If the range doesn't include zero, it doesn't include negative zero.
+ if (canBeNegativeZero_ && !canBeZero()) {
+ canBeNegativeZero_ = ExcludesNegativeZero;
+ assertInvariants();
+ }
+ }
+
+ // Set the range fields to the given raw values.
+ void rawInitialize(int32_t l, bool lb, int32_t h, bool hb,
+ FractionalPartFlag canHaveFractionalPart,
+ NegativeZeroFlag canBeNegativeZero, uint16_t e) {
+ lower_ = l;
+ upper_ = h;
+ hasInt32LowerBound_ = lb;
+ hasInt32UpperBound_ = hb;
+ canHaveFractionalPart_ = canHaveFractionalPart;
+ canBeNegativeZero_ = canBeNegativeZero;
+ max_exponent_ = e;
+ optimize();
+ }
+
+ // Construct a range from the given raw values.
+ Range(int32_t l, bool lb, int32_t h, bool hb,
+ FractionalPartFlag canHaveFractionalPart,
+ NegativeZeroFlag canBeNegativeZero, uint16_t e)
+ : symbolicLower_(nullptr), symbolicUpper_(nullptr) {
+ rawInitialize(l, lb, h, hb, canHaveFractionalPart, canBeNegativeZero, e);
+ }
+
+ public:
+ Range() : symbolicLower_(nullptr), symbolicUpper_(nullptr) { setUnknown(); }
+
+ Range(int64_t l, int64_t h, FractionalPartFlag canHaveFractionalPart,
+ NegativeZeroFlag canBeNegativeZero, uint16_t e)
+ : symbolicLower_(nullptr), symbolicUpper_(nullptr) {
+ set(l, h, canHaveFractionalPart, canBeNegativeZero, e);
+ }
+
+ Range(const Range& other)
+ : lower_(other.lower_),
+ upper_(other.upper_),
+ hasInt32LowerBound_(other.hasInt32LowerBound_),
+ hasInt32UpperBound_(other.hasInt32UpperBound_),
+ canHaveFractionalPart_(other.canHaveFractionalPart_),
+ canBeNegativeZero_(other.canBeNegativeZero_),
+ max_exponent_(other.max_exponent_),
+ symbolicLower_(nullptr),
+ symbolicUpper_(nullptr) {
+ assertInvariants();
+ }
+
+ // Construct a range from the given MDefinition. This differs from the
+ // MDefinition's range() method in that it describes the range of values
+ // *after* any bailout checks.
+ explicit Range(const MDefinition* def);
+
+ static Range* NewInt32Range(TempAllocator& alloc, int32_t l, int32_t h) {
+ return new (alloc) Range(l, h, ExcludesFractionalParts,
+ ExcludesNegativeZero, MaxInt32Exponent);
+ }
+
+ // Construct an int32 range containing just i. This is just a convenience
+ // wrapper around NewInt32Range.
+ static Range* NewInt32SingletonRange(TempAllocator& alloc, int32_t i) {
+ return NewInt32Range(alloc, i, i);
+ }
+
+ static Range* NewUInt32Range(TempAllocator& alloc, uint32_t l, uint32_t h) {
+ // For now, just pass them to the constructor as int64_t values.
+ // They'll become unbounded if they're not in the int32_t range.
+ return new (alloc) Range(l, h, ExcludesFractionalParts,
+ ExcludesNegativeZero, MaxUInt32Exponent);
+ }
+
+ // Construct a range containing values >= l and <= h. Note that this
+ // function treats negative zero as equal to zero, as >= and <= do. If the
+ // range includes zero, it is assumed to include negative zero too.
+ static Range* NewDoubleRange(TempAllocator& alloc, double l, double h) {
+ if (mozilla::IsNaN(l) && mozilla::IsNaN(h)) {
+ return nullptr;
+ }
+
+ Range* r = new (alloc) Range();
+ r->setDouble(l, h);
+ return r;
+ }
+
+ // Construct the strictest possible range containing d, or null if d is NaN.
+ // This function treats negative zero as distinct from zero, since this
+ // makes the strictest possible range containin zero a range which
+ // contains one value rather than two.
+ static Range* NewDoubleSingletonRange(TempAllocator& alloc, double d) {
+ if (mozilla::IsNaN(d)) {
+ return nullptr;
+ }
+
+ Range* r = new (alloc) Range();
+ r->setDoubleSingleton(d);
+ return r;
+ }
+
+ void dump(GenericPrinter& out) const;
+ void dump() const;
+ [[nodiscard]] bool update(const Range* other);
+
+ // Unlike the other operations, unionWith is an in-place
+ // modification. This is to avoid a bunch of useless extra
+ // copying when chaining together unions when handling Phi
+ // nodes.
+ void unionWith(const Range* other);
+ static Range* intersect(TempAllocator& alloc, const Range* lhs,
+ const Range* rhs, bool* emptyRange);
+ static Range* add(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* sub(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* mul(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* and_(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* or_(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* xor_(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* not_(TempAllocator& alloc, const Range* op);
+ static Range* lsh(TempAllocator& alloc, const Range* lhs, int32_t c);
+ static Range* rsh(TempAllocator& alloc, const Range* lhs, int32_t c);
+ static Range* ursh(TempAllocator& alloc, const Range* lhs, int32_t c);
+ static Range* lsh(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* rsh(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* ursh(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* abs(TempAllocator& alloc, const Range* op);
+ static Range* min(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* max(TempAllocator& alloc, const Range* lhs, const Range* rhs);
+ static Range* floor(TempAllocator& alloc, const Range* op);
+ static Range* ceil(TempAllocator& alloc, const Range* op);
+ static Range* sign(TempAllocator& alloc, const Range* op);
+ static Range* NaNToZero(TempAllocator& alloc, const Range* op);
+ static Range* toIntegerInt32(TempAllocator& alloc, const Range* op);
+
+ [[nodiscard]] static bool negativeZeroMul(const Range* lhs, const Range* rhs);
+
+ bool isUnknownInt32() const {
+ return isInt32() && lower() == INT32_MIN && upper() == INT32_MAX;
+ }
+
+ bool isUnknown() const {
+ return !hasInt32LowerBound_ && !hasInt32UpperBound_ &&
+ canHaveFractionalPart_ && canBeNegativeZero_ &&
+ max_exponent_ == IncludesInfinityAndNaN;
+ }
+
+ bool hasInt32LowerBound() const { return hasInt32LowerBound_; }
+ bool hasInt32UpperBound() const { return hasInt32UpperBound_; }
+
+ // Test whether the value is known to be within [INT32_MIN,INT32_MAX].
+ // Note that this does not necessarily mean the value is an integer.
+ bool hasInt32Bounds() const {
+ return hasInt32LowerBound() && hasInt32UpperBound();
+ }
+
+ // Test whether the value is known to be representable as an int32.
+ bool isInt32() const {
+ return hasInt32Bounds() && !canHaveFractionalPart_ && !canBeNegativeZero_;
+ }
+
+ // Test whether the given value is known to be either 0 or 1.
+ bool isBoolean() const {
+ return lower() >= 0 && upper() <= 1 && !canHaveFractionalPart_ &&
+ !canBeNegativeZero_;
+ }
+
+ bool canHaveRoundingErrors() const {
+ return canHaveFractionalPart_ || canBeNegativeZero_ ||
+ max_exponent_ >= MaxTruncatableExponent;
+ }
+
+ // Test if an integer x belongs to the range.
+ bool contains(int32_t x) const { return x >= lower_ && x <= upper_; }
+
+ // Test whether the range contains zero (of either sign).
+ bool canBeZero() const { return contains(0); }
+
+ // Test whether the range contains NaN values.
+ bool canBeNaN() const { return max_exponent_ == IncludesInfinityAndNaN; }
+
+ // Test whether the range contains infinities or NaN values.
+ bool canBeInfiniteOrNaN() const { return max_exponent_ >= IncludesInfinity; }
+
+ FractionalPartFlag canHaveFractionalPart() const {
+ return canHaveFractionalPart_;
+ }
+
+ NegativeZeroFlag canBeNegativeZero() const { return canBeNegativeZero_; }
+
+ uint16_t exponent() const {
+ MOZ_ASSERT(!canBeInfiniteOrNaN());
+ return max_exponent_;
+ }
+
+ uint16_t numBits() const {
+ return exponent() + 1; // 2^0 -> 1
+ }
+
+ // Return the lower bound. Asserts that the value has an int32 bound.
+ int32_t lower() const {
+ MOZ_ASSERT(hasInt32LowerBound());
+ return lower_;
+ }
+
+ // Return the upper bound. Asserts that the value has an int32 bound.
+ int32_t upper() const {
+ MOZ_ASSERT(hasInt32UpperBound());
+ return upper_;
+ }
+
+ // Test whether all values in this range can are finite and negative.
+ bool isFiniteNegative() const { return upper_ < 0 && !canBeInfiniteOrNaN(); }
+
+ // Test whether all values in this range can are finite and non-negative.
+ bool isFiniteNonNegative() const {
+ return lower_ >= 0 && !canBeInfiniteOrNaN();
+ }
+
+ // Test whether a value in this range can possibly be a finite
+ // negative value. Note that "negative zero" is not considered negative.
+ bool canBeFiniteNegative() const { return lower_ < 0; }
+
+ // Test whether a value in this range can possibly be a finite
+ // non-negative value.
+ bool canBeFiniteNonNegative() const { return upper_ >= 0; }
+
+ // Test whether a value in this range can have the sign bit set (not
+ // counting NaN, where the sign bit is meaningless).
+ bool canHaveSignBitSet() const {
+ return !hasInt32LowerBound() || canBeFiniteNegative() ||
+ canBeNegativeZero();
+ }
+
+ // Set this range to have a lower bound not less than x.
+ void refineLower(int32_t x) {
+ assertInvariants();
+ hasInt32LowerBound_ = true;
+ lower_ = std::max(lower_, x);
+ optimize();
+ }
+
+ // Set this range to have an upper bound not greater than x.
+ void refineUpper(int32_t x) {
+ assertInvariants();
+ hasInt32UpperBound_ = true;
+ upper_ = std::min(upper_, x);
+ optimize();
+ }
+
+ // Set this range to exclude negative zero.
+ void refineToExcludeNegativeZero() {
+ assertInvariants();
+ canBeNegativeZero_ = ExcludesNegativeZero;
+ optimize();
+ }
+
+ void setInt32(int32_t l, int32_t h) {
+ hasInt32LowerBound_ = true;
+ hasInt32UpperBound_ = true;
+ lower_ = l;
+ upper_ = h;
+ canHaveFractionalPart_ = ExcludesFractionalParts;
+ canBeNegativeZero_ = ExcludesNegativeZero;
+ max_exponent_ = exponentImpliedByInt32Bounds();
+ assertInvariants();
+ }
+
+ // Set this range to include values >= l and <= h. Note that this
+ // function treats negative zero as equal to zero, as >= and <= do. If the
+ // range includes zero, it is assumed to include negative zero too.
+ void setDouble(double l, double h);
+
+ // Set this range to the narrowest possible range containing d.
+ // This function treats negative zero as distinct from zero, since this
+ // makes the narrowest possible range containin zero a range which
+ // contains one value rather than two.
+ void setDoubleSingleton(double d);
+
+ void setUnknown() {
+ set(NoInt32LowerBound, NoInt32UpperBound, IncludesFractionalParts,
+ IncludesNegativeZero, IncludesInfinityAndNaN);
+ MOZ_ASSERT(isUnknown());
+ }
+
+ void set(int64_t l, int64_t h, FractionalPartFlag canHaveFractionalPart,
+ NegativeZeroFlag canBeNegativeZero, uint16_t e) {
+ max_exponent_ = e;
+ canHaveFractionalPart_ = canHaveFractionalPart;
+ canBeNegativeZero_ = canBeNegativeZero;
+ setLowerInit(l);
+ setUpperInit(h);
+ optimize();
+ }
+
+ // Make the lower end of this range at least INT32_MIN, and make
+ // the upper end of this range at most INT32_MAX.
+ void clampToInt32();
+
+ // If this range exceeds int32_t range, at either or both ends, change
+ // it to int32_t range. Otherwise do nothing.
+ void wrapAroundToInt32();
+
+ // If this range exceeds [0, 32) range, at either or both ends, change
+ // it to the [0, 32) range. Otherwise do nothing.
+ void wrapAroundToShiftCount();
+
+ // If this range exceeds [0, 1] range, at either or both ends, change
+ // it to the [0, 1] range. Otherwise do nothing.
+ void wrapAroundToBoolean();
+
+ const SymbolicBound* symbolicLower() const { return symbolicLower_; }
+ const SymbolicBound* symbolicUpper() const { return symbolicUpper_; }
+
+ void setSymbolicLower(SymbolicBound* bound) { symbolicLower_ = bound; }
+ void setSymbolicUpper(SymbolicBound* bound) { symbolicUpper_ = bound; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_RangeAnalysis_h */
diff --git a/js/src/jit/ReciprocalMulConstants.cpp b/js/src/jit/ReciprocalMulConstants.cpp
new file mode 100644
index 0000000000..956c2e62d9
--- /dev/null
+++ b/js/src/jit/ReciprocalMulConstants.cpp
@@ -0,0 +1,94 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ReciprocalMulConstants.h"
+
+#include "mozilla/Assertions.h"
+
+using namespace js::jit;
+
+ReciprocalMulConstants ReciprocalMulConstants::computeDivisionConstants(
+ uint32_t d, int maxLog) {
+ MOZ_ASSERT(maxLog >= 2 && maxLog <= 32);
+ // In what follows, 0 < d < 2^maxLog and d is not a power of 2.
+ MOZ_ASSERT(d < (uint64_t(1) << maxLog) && (d & (d - 1)) != 0);
+
+ // Speeding up division by non power-of-2 constants is possible by
+ // calculating, during compilation, a value M such that high-order
+ // bits of M*n correspond to the result of the division of n by d.
+ // No value of M can serve this purpose for arbitrarily big values
+ // of n but, for optimizing integer division, we're just concerned
+ // with values of n whose absolute value is bounded (by fitting in
+ // an integer type, say). With this in mind, we'll find a constant
+ // M as above that works for -2^maxLog <= n < 2^maxLog; maxLog can
+ // then be 31 for signed division or 32 for unsigned division.
+ //
+ // The original presentation of this technique appears in Hacker's
+ // Delight, a book by Henry S. Warren, Jr.. A proof of correctness
+ // for our version follows; we'll denote maxLog by L in the proof,
+ // for conciseness.
+ //
+ // Formally, for |d| < 2^L, we'll compute two magic values M and s
+ // in the ranges 0 <= M < 2^(L+1) and 0 <= s <= L such that
+ // (M * n) >> (32 + s) = floor(n/d) if 0 <= n < 2^L
+ // (M * n) >> (32 + s) = ceil(n/d) - 1 if -2^L <= n < 0.
+ //
+ // Define p = 32 + s, M = ceil(2^p/d), and assume that s satisfies
+ // M - 2^p/d <= 2^(p-L)/d. (1)
+ // (Observe that p = CeilLog32(d) + L satisfies this, as the right
+ // side of (1) is at least one in this case). Then,
+ //
+ // a) If p <= CeilLog32(d) + L, then M < 2^(L+1) - 1.
+ // Proof: Indeed, M is monotone in p and, for p equal to the above
+ // value, the bounds 2^L > d >= 2^(p-L-1) + 1 readily imply that
+ // 2^p / d < 2^p/(d - 1) * (d - 1)/d
+ // <= 2^(L+1) * (1 - 1/d) < 2^(L+1) - 2.
+ // The claim follows by applying the ceiling function.
+ //
+ // b) For any 0 <= n < 2^L, floor(Mn/2^p) = floor(n/d).
+ // Proof: Put x = floor(Mn/2^p); it's the unique integer for which
+ // Mn/2^p - 1 < x <= Mn/2^p. (2)
+ // Using M >= 2^p/d on the LHS and (1) on the RHS, we get
+ // n/d - 1 < x <= n/d + n/(2^L d) < n/d + 1/d.
+ // Since x is an integer, it's not in the interval (n/d, (n+1)/d),
+ // and so n/d - 1 < x <= n/d, which implies x = floor(n/d).
+ //
+ // c) For any -2^L <= n < 0, floor(Mn/2^p) + 1 = ceil(n/d).
+ // Proof: The proof is similar. Equation (2) holds as above. Using
+ // M > 2^p/d (d isn't a power of 2) on the RHS and (1) on the LHS,
+ // n/d + n/(2^L d) - 1 < x < n/d.
+ // Using n >= -2^L and summing 1,
+ // n/d - 1/d < x + 1 < n/d + 1.
+ // Since x + 1 is an integer, this implies n/d <= x + 1 < n/d + 1.
+ // In other words, x + 1 = ceil(n/d).
+ //
+ // Condition (1) isn't necessary for the existence of M and s with
+ // the properties above. Hacker's Delight provides a slightly less
+ // restrictive condition when d >= 196611, at the cost of a 3-page
+ // proof of correctness, for the case L = 31.
+ //
+ // Note that, since d*M - 2^p = d - (2^p)%d, (1) can be written as
+ // 2^(p-L) >= d - (2^p)%d.
+ // In order to avoid overflow in the (2^p) % d calculation, we can
+ // compute it as (2^p-1) % d + 1, where 2^p-1 can then be computed
+ // without overflow as UINT64_MAX >> (64-p).
+
+ // We now compute the least p >= 32 with the property above...
+ int32_t p = 32;
+ while ((uint64_t(1) << (p - maxLog)) + (UINT64_MAX >> (64 - p)) % d + 1 < d) {
+ p++;
+ }
+
+ // ...and the corresponding M. For either the signed (L=31) or the
+ // unsigned (L=32) case, this value can be too large (cf. item a).
+ // Codegen can still multiply by M by multiplying by (M - 2^L) and
+ // adjusting the value afterwards, if this is the case.
+ ReciprocalMulConstants rmc;
+ rmc.multiplier = (UINT64_MAX >> (64 - p)) / d + 1;
+ rmc.shiftAmount = p - 32;
+
+ return rmc;
+}
diff --git a/js/src/jit/ReciprocalMulConstants.h b/js/src/jit/ReciprocalMulConstants.h
new file mode 100644
index 0000000000..d305a4caa5
--- /dev/null
+++ b/js/src/jit/ReciprocalMulConstants.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ReciprocalMulConstants_h
+#define jit_ReciprocalMulConstants_h
+
+#include <stdint.h>
+
+namespace js::jit {
+
+struct ReciprocalMulConstants {
+ int64_t multiplier;
+ int32_t shiftAmount;
+
+ static ReciprocalMulConstants computeSignedDivisionConstants(uint32_t d) {
+ return computeDivisionConstants(d, 31);
+ }
+
+ static ReciprocalMulConstants computeUnsignedDivisionConstants(uint32_t d) {
+ return computeDivisionConstants(d, 32);
+ }
+
+ private:
+ static ReciprocalMulConstants computeDivisionConstants(uint32_t d,
+ int maxLog);
+};
+
+} // namespace js::jit
+
+#endif /* jit_ReciprocalMulConstants_h */
diff --git a/js/src/jit/Recover.cpp b/js/src/jit/Recover.cpp
new file mode 100644
index 0000000000..8eab95253e
--- /dev/null
+++ b/js/src/jit/Recover.cpp
@@ -0,0 +1,2126 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Recover.h"
+
+#include "jsmath.h"
+
+#include "builtin/Object.h"
+#include "builtin/RegExp.h"
+#include "builtin/String.h"
+#include "jit/Bailouts.h"
+#include "jit/CompileInfo.h"
+#include "jit/Ion.h"
+#include "jit/JitSpewer.h"
+#include "jit/JSJitFrameIter.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "jit/VMFunctions.h"
+#include "util/DifferentialTesting.h"
+#include "vm/BigIntType.h"
+#include "vm/EqualityOperations.h"
+#include "vm/Interpreter.h"
+#include "vm/Iteration.h"
+#include "vm/JSContext.h"
+#include "vm/JSObject.h"
+#include "vm/PlainObject.h" // js::PlainObject
+#include "vm/StringType.h"
+
+#include "vm/Interpreter-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+bool MNode::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_CRASH("This instruction is not serializable");
+}
+
+void RInstruction::readRecoverData(CompactBufferReader& reader,
+ RInstructionStorage* raw) {
+ uint32_t op = reader.readUnsigned();
+ switch (Opcode(op)) {
+#define MATCH_OPCODES_(op) \
+ case Recover_##op: \
+ static_assert(sizeof(R##op) <= sizeof(RInstructionStorage), \
+ "storage space must be big enough to store R" #op); \
+ static_assert(alignof(R##op) <= alignof(RInstructionStorage), \
+ "storage space must be aligned adequate to store R" #op); \
+ new (raw->addr()) R##op(reader); \
+ break;
+
+ RECOVER_OPCODE_LIST(MATCH_OPCODES_)
+#undef MATCH_OPCODES_
+
+ case Recover_Invalid:
+ default:
+ MOZ_CRASH("Bad decoding of the previous instruction?");
+ }
+}
+
+bool MResumePoint::writeRecoverData(CompactBufferWriter& writer) const {
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_ResumePoint));
+
+ MBasicBlock* bb = block();
+ bool hasFun = bb->info().hasFunMaybeLazy();
+ uint32_t nargs = bb->info().nargs();
+ JSScript* script = bb->info().script();
+ uint32_t exprStack = stackDepth() - bb->info().ninvoke();
+
+#ifdef DEBUG
+ // Ensure that all snapshot which are encoded can safely be used for
+ // bailouts.
+ uint32_t numIntermediate = NumIntermediateValues(mode());
+ if (JSContext* cx = GetJitContext()->cx) {
+ if (!AssertBailoutStackDepth(cx, script, pc(), mode(),
+ exprStack - numIntermediate)) {
+ return false;
+ }
+ }
+#endif
+
+ uint32_t formalArgs = CountArgSlots(script, hasFun, nargs);
+
+ // Test if we honor the maximum of arguments at all times. This is a sanity
+ // check and not an algorithm limit. So check might be a bit too loose. +4
+ // to account for scope chain, return value, this value and maybe
+ // arguments_object.
+ MOZ_ASSERT(formalArgs < SNAPSHOT_MAX_NARGS + 4);
+
+#ifdef JS_JITSPEW
+ uint32_t implicit = StartArgSlot(script);
+#endif
+ uint32_t nallocs = formalArgs + script->nfixed() + exprStack;
+
+ JitSpew(JitSpew_IonSnapshots,
+ "Starting frame; implicit %u, formals %u, fixed %zu, exprs %u",
+ implicit, formalArgs - implicit, script->nfixed(), exprStack);
+
+ uint32_t pcOff = script->pcToOffset(pc());
+ JitSpew(JitSpew_IonSnapshots, "Writing pc offset %u, mode %s, nslots %u",
+ pcOff, ResumeModeToString(mode()), nallocs);
+
+ uint32_t pcOffAndMode =
+ (pcOff << RResumePoint::PCOffsetShift) | uint32_t(mode());
+ MOZ_RELEASE_ASSERT((pcOffAndMode >> RResumePoint::PCOffsetShift) == pcOff,
+ "pcOff doesn't fit in pcOffAndMode");
+ writer.writeUnsigned(pcOffAndMode);
+
+ writer.writeUnsigned(nallocs);
+ return true;
+}
+
+RResumePoint::RResumePoint(CompactBufferReader& reader) {
+ pcOffsetAndMode_ = reader.readUnsigned();
+ numOperands_ = reader.readUnsigned();
+ JitSpew(JitSpew_IonSnapshots,
+ "Read RResumePoint (pc offset %u, mode %s, nslots %u)", pcOffset(),
+ ResumeModeToString(mode()), numOperands_);
+}
+
+bool RResumePoint::recover(JSContext* cx, SnapshotIterator& iter) const {
+ MOZ_CRASH("This instruction is not recoverable.");
+}
+
+bool MBitNot::writeRecoverData(CompactBufferWriter& writer) const {
+ // 64-bit int bitnots exist only when compiling wasm; they exist neither for
+ // JS nor asm.js. So we don't expect them here.
+ MOZ_ASSERT(type() != MIRType::Int64);
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BitNot));
+ return true;
+}
+
+RBitNot::RBitNot(CompactBufferReader& reader) {}
+
+bool RBitNot::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue operand(cx, iter.read());
+ RootedValue result(cx);
+
+ if (!js::BitNot(cx, &operand, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBitAnd::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BitAnd));
+ return true;
+}
+
+RBitAnd::RBitAnd(CompactBufferReader& reader) {}
+
+bool RBitAnd::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+
+ if (!js::BitAnd(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBitOr::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BitOr));
+ return true;
+}
+
+RBitOr::RBitOr(CompactBufferReader& reader) {}
+
+bool RBitOr::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+
+ if (!js::BitOr(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBitXor::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BitXor));
+ return true;
+}
+
+RBitXor::RBitXor(CompactBufferReader& reader) {}
+
+bool RBitXor::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ if (!js::BitXor(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MLsh::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Lsh));
+ return true;
+}
+
+RLsh::RLsh(CompactBufferReader& reader) {}
+
+bool RLsh::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+
+ if (!js::BitLsh(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MRsh::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Rsh));
+ return true;
+}
+
+RRsh::RRsh(CompactBufferReader& reader) {}
+
+bool RRsh::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+
+ if (!js::BitRsh(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MUrsh::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Ursh));
+ return true;
+}
+
+RUrsh::RUrsh(CompactBufferReader& reader) {}
+
+bool RUrsh::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+
+ RootedValue result(cx);
+ if (!js::UrshValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MSignExtendInt32::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_SignExtendInt32));
+ MOZ_ASSERT(Mode(uint8_t(mode_)) == mode_);
+ writer.writeByte(uint8_t(mode_));
+ return true;
+}
+
+RSignExtendInt32::RSignExtendInt32(CompactBufferReader& reader) {
+ mode_ = reader.readByte();
+}
+
+bool RSignExtendInt32::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue operand(cx, iter.read());
+
+ int32_t i;
+ if (!ToInt32(cx, operand, &i)) {
+ return false;
+ }
+
+ int32_t result;
+ switch (MSignExtendInt32::Mode(mode_)) {
+ case MSignExtendInt32::Byte:
+ result = static_cast<int8_t>(i);
+ break;
+ case MSignExtendInt32::Half:
+ result = static_cast<int16_t>(i);
+ break;
+ }
+
+ iter.storeInstructionResult(JS::Int32Value(result));
+ return true;
+}
+
+bool MAdd::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Add));
+ writer.writeByte(type() == MIRType::Float32);
+ return true;
+}
+
+RAdd::RAdd(CompactBufferReader& reader) {
+ isFloatOperation_ = reader.readByte();
+}
+
+bool RAdd::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+ if (!js::AddValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ // MIRType::Float32 is a specialization embedding the fact that the result is
+ // rounded to a Float32.
+ if (isFloatOperation_ && !RoundFloat32(cx, result, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MSub::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Sub));
+ writer.writeByte(type() == MIRType::Float32);
+ return true;
+}
+
+RSub::RSub(CompactBufferReader& reader) {
+ isFloatOperation_ = reader.readByte();
+}
+
+bool RSub::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+ if (!js::SubValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ // MIRType::Float32 is a specialization embedding the fact that the result is
+ // rounded to a Float32.
+ if (isFloatOperation_ && !RoundFloat32(cx, result, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MMul::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Mul));
+ writer.writeByte(type() == MIRType::Float32);
+ MOZ_ASSERT(Mode(uint8_t(mode_)) == mode_);
+ writer.writeByte(uint8_t(mode_));
+ return true;
+}
+
+RMul::RMul(CompactBufferReader& reader) {
+ isFloatOperation_ = reader.readByte();
+ mode_ = reader.readByte();
+}
+
+bool RMul::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ if (MMul::Mode(mode_) == MMul::Normal) {
+ if (!js::MulValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ // MIRType::Float32 is a specialization embedding the fact that the
+ // result is rounded to a Float32.
+ if (isFloatOperation_ && !RoundFloat32(cx, result, &result)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(MMul::Mode(mode_) == MMul::Integer);
+ if (!js::math_imul_handle(cx, lhs, rhs, &result)) {
+ return false;
+ }
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MDiv::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Div));
+ writer.writeByte(type() == MIRType::Float32);
+ return true;
+}
+
+RDiv::RDiv(CompactBufferReader& reader) {
+ isFloatOperation_ = reader.readByte();
+}
+
+bool RDiv::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ if (!js::DivValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ // MIRType::Float32 is a specialization embedding the fact that the result is
+ // rounded to a Float32.
+ if (isFloatOperation_ && !RoundFloat32(cx, result, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MMod::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Mod));
+ return true;
+}
+
+RMod::RMod(CompactBufferReader& reader) {}
+
+bool RMod::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+ if (!js::ModValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MNot::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Not));
+ return true;
+}
+
+RNot::RNot(CompactBufferReader& reader) {}
+
+bool RNot::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue v(cx, iter.read());
+ RootedValue result(cx);
+
+ result.setBoolean(!ToBoolean(v));
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntAdd::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntAdd));
+ return true;
+}
+
+RBigIntAdd::RBigIntAdd(CompactBufferReader& reader) {}
+
+bool RBigIntAdd::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ if (!js::AddValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntSub::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntSub));
+ return true;
+}
+
+RBigIntSub::RBigIntSub(CompactBufferReader& reader) {}
+
+bool RBigIntSub::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ if (!js::SubValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntMul::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntMul));
+ return true;
+}
+
+RBigIntMul::RBigIntMul(CompactBufferReader& reader) {}
+
+bool RBigIntMul::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ if (!js::MulValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntDiv::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntDiv));
+ return true;
+}
+
+RBigIntDiv::RBigIntDiv(CompactBufferReader& reader) {}
+
+bool RBigIntDiv::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ MOZ_ASSERT(!rhs.toBigInt()->isZero(),
+ "division by zero throws and therefore can't be recovered");
+ if (!js::DivValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntMod::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntMod));
+ return true;
+}
+
+RBigIntMod::RBigIntMod(CompactBufferReader& reader) {}
+
+bool RBigIntMod::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ MOZ_ASSERT(!rhs.toBigInt()->isZero(),
+ "division by zero throws and therefore can't be recovered");
+ if (!js::ModValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntPow::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntPow));
+ return true;
+}
+
+RBigIntPow::RBigIntPow(CompactBufferReader& reader) {}
+
+bool RBigIntPow::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ MOZ_ASSERT(!rhs.toBigInt()->isNegative(),
+ "negative exponent throws and therefore can't be recovered");
+ if (!js::PowValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntBitAnd::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntBitAnd));
+ return true;
+}
+
+RBigIntBitAnd::RBigIntBitAnd(CompactBufferReader& reader) {}
+
+bool RBigIntBitAnd::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ if (!js::BitAnd(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntBitOr::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntBitOr));
+ return true;
+}
+
+RBigIntBitOr::RBigIntBitOr(CompactBufferReader& reader) {}
+
+bool RBigIntBitOr::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ if (!js::BitOr(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntBitXor::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntBitXor));
+ return true;
+}
+
+RBigIntBitXor::RBigIntBitXor(CompactBufferReader& reader) {}
+
+bool RBigIntBitXor::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ if (!js::BitXor(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntLsh::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntLsh));
+ return true;
+}
+
+RBigIntLsh::RBigIntLsh(CompactBufferReader& reader) {}
+
+bool RBigIntLsh::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ if (!js::BitLsh(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntRsh::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntRsh));
+ return true;
+}
+
+RBigIntRsh::RBigIntRsh(CompactBufferReader& reader) {}
+
+bool RBigIntRsh::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(lhs.isBigInt() && rhs.isBigInt());
+ if (!js::BitRsh(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntIncrement::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntIncrement));
+ return true;
+}
+
+RBigIntIncrement::RBigIntIncrement(CompactBufferReader& reader) {}
+
+bool RBigIntIncrement::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue operand(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(operand.isBigInt());
+ if (!js::IncOperation(cx, operand, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntDecrement::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntDecrement));
+ return true;
+}
+
+RBigIntDecrement::RBigIntDecrement(CompactBufferReader& reader) {}
+
+bool RBigIntDecrement::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue operand(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(operand.isBigInt());
+ if (!js::DecOperation(cx, operand, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntNegate::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntNegate));
+ return true;
+}
+
+RBigIntNegate::RBigIntNegate(CompactBufferReader& reader) {}
+
+bool RBigIntNegate::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue operand(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(operand.isBigInt());
+ if (!js::NegOperation(cx, &operand, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MBigIntBitNot::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntBitNot));
+ return true;
+}
+
+RBigIntBitNot::RBigIntBitNot(CompactBufferReader& reader) {}
+
+bool RBigIntBitNot::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue operand(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(operand.isBigInt());
+ if (!js::BitNot(cx, &operand, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MCompare::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Compare));
+
+ static_assert(sizeof(JSOp) == sizeof(uint8_t));
+ writer.writeByte(uint8_t(jsop_));
+ return true;
+}
+
+RCompare::RCompare(CompactBufferReader& reader) {
+ jsop_ = JSOp(reader.readByte());
+
+ MOZ_ASSERT(IsEqualityOp(jsop_) || IsRelationalOp(jsop_));
+}
+
+bool RCompare::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+
+ bool result;
+ switch (jsop_) {
+ case JSOp::Eq:
+ case JSOp::Ne:
+ if (!js::LooselyEqual(cx, lhs, rhs, &result)) {
+ return false;
+ }
+ if (jsop_ == JSOp::Ne) {
+ result = !result;
+ }
+ break;
+ case JSOp::StrictEq:
+ case JSOp::StrictNe:
+ if (!StrictlyEqual(cx, lhs, rhs, &result)) {
+ return false;
+ }
+ if (jsop_ == JSOp::StrictNe) {
+ result = !result;
+ }
+ break;
+ case JSOp::Lt:
+ if (!js::LessThan(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+ break;
+ case JSOp::Le:
+ if (!js::LessThanOrEqual(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+ break;
+ case JSOp::Gt:
+ if (!js::GreaterThan(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+ break;
+ case JSOp::Ge:
+ if (!js::GreaterThanOrEqual(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected op.");
+ }
+
+ iter.storeInstructionResult(BooleanValue(result));
+ return true;
+}
+
+bool MConcat::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Concat));
+ return true;
+}
+
+RConcat::RConcat(CompactBufferReader& reader) {}
+
+bool RConcat::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue lhs(cx, iter.read());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(!lhs.isObject() && !rhs.isObject());
+ if (!js::AddValues(cx, &lhs, &rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+RStringLength::RStringLength(CompactBufferReader& reader) {}
+
+bool RStringLength::recover(JSContext* cx, SnapshotIterator& iter) const {
+ JSString* string = iter.read().toString();
+
+ static_assert(JSString::MAX_LENGTH <= INT32_MAX,
+ "Can cast string length to int32_t");
+
+ iter.storeInstructionResult(Int32Value(int32_t(string->length())));
+ return true;
+}
+
+bool MStringLength::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_StringLength));
+ return true;
+}
+
+bool MArgumentsLength::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_ArgumentsLength));
+ return true;
+}
+
+RArgumentsLength::RArgumentsLength(CompactBufferReader& reader) {}
+
+bool RArgumentsLength::recover(JSContext* cx, SnapshotIterator& iter) const {
+ uintptr_t numActualArgs = iter.frame()->numActualArgs();
+
+ static_assert(ARGS_LENGTH_MAX <= INT32_MAX,
+ "Can cast arguments count to int32_t");
+ MOZ_ASSERT(numActualArgs <= ARGS_LENGTH_MAX);
+
+ iter.storeInstructionResult(JS::Int32Value(int32_t(numActualArgs)));
+ return true;
+}
+
+bool MFloor::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Floor));
+ return true;
+}
+
+RFloor::RFloor(CompactBufferReader& reader) {}
+
+bool RFloor::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+ double result = js::math_floor_impl(num);
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MCeil::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Ceil));
+ return true;
+}
+
+RCeil::RCeil(CompactBufferReader& reader) {}
+
+bool RCeil::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+ double result = js::math_ceil_impl(num);
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MRound::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Round));
+ return true;
+}
+
+RRound::RRound(CompactBufferReader& reader) {}
+
+bool RRound::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+ double result = js::math_round_impl(num);
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MTrunc::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Trunc));
+ return true;
+}
+
+RTrunc::RTrunc(CompactBufferReader& reader) {}
+
+bool RTrunc::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+ double result = js::math_trunc_impl(num);
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MCharCodeAt::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_CharCodeAt));
+ return true;
+}
+
+RCharCodeAt::RCharCodeAt(CompactBufferReader& reader) {}
+
+bool RCharCodeAt::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedString lhs(cx, iter.read().toString());
+ RootedValue rhs(cx, iter.read());
+ RootedValue result(cx);
+
+ if (!js::str_charCodeAt_impl(cx, lhs, rhs, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MFromCharCode::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_FromCharCode));
+ return true;
+}
+
+RFromCharCode::RFromCharCode(CompactBufferReader& reader) {}
+
+bool RFromCharCode::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue operand(cx, iter.read());
+ RootedValue result(cx);
+
+ MOZ_ASSERT(!operand.isObject());
+ if (!js::str_fromCharCode_one_arg(cx, operand, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MPow::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Pow));
+ return true;
+}
+
+RPow::RPow(CompactBufferReader& reader) {}
+
+bool RPow::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double base = iter.read().toNumber();
+ double power = iter.read().toNumber();
+ double result = ecmaPow(base, power);
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MPowHalf::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_PowHalf));
+ return true;
+}
+
+RPowHalf::RPowHalf(CompactBufferReader& reader) {}
+
+bool RPowHalf::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double base = iter.read().toNumber();
+ double power = 0.5;
+ double result = ecmaPow(base, power);
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MMinMax::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_MinMax));
+ writer.writeByte(isMax_);
+ return true;
+}
+
+RMinMax::RMinMax(CompactBufferReader& reader) { isMax_ = reader.readByte(); }
+
+bool RMinMax::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double x = iter.read().toNumber();
+ double y = iter.read().toNumber();
+
+ double result;
+ if (isMax_) {
+ result = js::math_max_impl(x, y);
+ } else {
+ result = js::math_min_impl(x, y);
+ }
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MAbs::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Abs));
+ return true;
+}
+
+RAbs::RAbs(CompactBufferReader& reader) {}
+
+bool RAbs::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+ double result = js::math_abs_impl(num);
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MSqrt::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Sqrt));
+ writer.writeByte(type() == MIRType::Float32);
+ return true;
+}
+
+RSqrt::RSqrt(CompactBufferReader& reader) {
+ isFloatOperation_ = reader.readByte();
+}
+
+bool RSqrt::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+ double result = js::math_sqrt_impl(num);
+
+ // MIRType::Float32 is a specialization embedding the fact that the result is
+ // rounded to a Float32.
+ if (isFloatOperation_) {
+ result = js::RoundFloat32(result);
+ }
+
+ iter.storeInstructionResult(DoubleValue(result));
+ return true;
+}
+
+bool MAtan2::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Atan2));
+ return true;
+}
+
+RAtan2::RAtan2(CompactBufferReader& reader) {}
+
+bool RAtan2::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double y = iter.read().toNumber();
+ double x = iter.read().toNumber();
+ double result = js::ecmaAtan2(y, x);
+
+ iter.storeInstructionResult(DoubleValue(result));
+ return true;
+}
+
+bool MHypot::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Hypot));
+ writer.writeUnsigned(uint32_t(numOperands()));
+ return true;
+}
+
+RHypot::RHypot(CompactBufferReader& reader)
+ : numOperands_(reader.readUnsigned()) {}
+
+bool RHypot::recover(JSContext* cx, SnapshotIterator& iter) const {
+ JS::RootedValueVector vec(cx);
+
+ if (!vec.reserve(numOperands_)) {
+ return false;
+ }
+
+ for (uint32_t i = 0; i < numOperands_; ++i) {
+ vec.infallibleAppend(iter.read());
+ }
+
+ RootedValue result(cx);
+
+ if (!js::math_hypot_handle(cx, vec, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MNearbyInt::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ switch (roundingMode_) {
+ case RoundingMode::Up:
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Ceil));
+ return true;
+ case RoundingMode::Down:
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Floor));
+ return true;
+ case RoundingMode::TowardsZero:
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Trunc));
+ return true;
+ default:
+ MOZ_CRASH("Unsupported rounding mode.");
+ }
+}
+
+RNearbyInt::RNearbyInt(CompactBufferReader& reader) {
+ roundingMode_ = reader.readByte();
+}
+
+bool RNearbyInt::recover(JSContext* cx, SnapshotIterator& iter) const {
+ MOZ_CRASH("Unsupported rounding mode.");
+}
+
+bool MSign::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Sign));
+ return true;
+}
+
+RSign::RSign(CompactBufferReader& reader) {}
+
+bool RSign::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+ double result = js::math_sign_impl(num);
+
+ iter.storeInstructionResult(NumberValue(result));
+ return true;
+}
+
+bool MMathFunction::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ switch (function_) {
+ case UnaryMathFunction::Ceil:
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Ceil));
+ return true;
+ case UnaryMathFunction::Floor:
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Floor));
+ return true;
+ case UnaryMathFunction::Round:
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Round));
+ return true;
+ case UnaryMathFunction::Trunc:
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Trunc));
+ return true;
+ case UnaryMathFunction::Log:
+ case UnaryMathFunction::Sin:
+ case UnaryMathFunction::Cos:
+ case UnaryMathFunction::Exp:
+ case UnaryMathFunction::Tan:
+ case UnaryMathFunction::ACos:
+ case UnaryMathFunction::ASin:
+ case UnaryMathFunction::ATan:
+ case UnaryMathFunction::Log10:
+ case UnaryMathFunction::Log2:
+ case UnaryMathFunction::Log1P:
+ case UnaryMathFunction::ExpM1:
+ case UnaryMathFunction::CosH:
+ case UnaryMathFunction::SinH:
+ case UnaryMathFunction::TanH:
+ case UnaryMathFunction::ACosH:
+ case UnaryMathFunction::ASinH:
+ case UnaryMathFunction::ATanH:
+ case UnaryMathFunction::Cbrt:
+ static_assert(sizeof(UnaryMathFunction) == sizeof(uint8_t));
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_MathFunction));
+ writer.writeByte(uint8_t(function_));
+ return true;
+ }
+ MOZ_CRASH("Unknown math function.");
+}
+
+RMathFunction::RMathFunction(CompactBufferReader& reader) {
+ function_ = UnaryMathFunction(reader.readByte());
+}
+
+bool RMathFunction::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+
+ double result;
+ switch (function_) {
+ case UnaryMathFunction::Log:
+ result = js::math_log_impl(num);
+ break;
+ case UnaryMathFunction::Sin:
+ result = js::math_sin_impl(num);
+ break;
+ case UnaryMathFunction::Cos:
+ result = js::math_cos_impl(num);
+ break;
+ case UnaryMathFunction::Exp:
+ result = js::math_exp_impl(num);
+ break;
+ case UnaryMathFunction::Tan:
+ result = js::math_tan_impl(num);
+ break;
+ case UnaryMathFunction::ACos:
+ result = js::math_acos_impl(num);
+ break;
+ case UnaryMathFunction::ASin:
+ result = js::math_asin_impl(num);
+ break;
+ case UnaryMathFunction::ATan:
+ result = js::math_atan_impl(num);
+ break;
+ case UnaryMathFunction::Log10:
+ result = js::math_log10_impl(num);
+ break;
+ case UnaryMathFunction::Log2:
+ result = js::math_log2_impl(num);
+ break;
+ case UnaryMathFunction::Log1P:
+ result = js::math_log1p_impl(num);
+ break;
+ case UnaryMathFunction::ExpM1:
+ result = js::math_expm1_impl(num);
+ break;
+ case UnaryMathFunction::CosH:
+ result = js::math_cosh_impl(num);
+ break;
+ case UnaryMathFunction::SinH:
+ result = js::math_sinh_impl(num);
+ break;
+ case UnaryMathFunction::TanH:
+ result = js::math_tanh_impl(num);
+ break;
+ case UnaryMathFunction::ACosH:
+ result = js::math_acosh_impl(num);
+ break;
+ case UnaryMathFunction::ASinH:
+ result = js::math_asinh_impl(num);
+ break;
+ case UnaryMathFunction::ATanH:
+ result = js::math_atanh_impl(num);
+ break;
+ case UnaryMathFunction::Cbrt:
+ result = js::math_cbrt_impl(num);
+ break;
+
+ case UnaryMathFunction::Trunc:
+ case UnaryMathFunction::Floor:
+ case UnaryMathFunction::Ceil:
+ case UnaryMathFunction::Round:
+ // These have their own recover instructions.
+ MOZ_CRASH("Unexpected rounding math function.");
+ }
+
+ iter.storeInstructionResult(DoubleValue(result));
+ return true;
+}
+
+bool MRandom::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(this->canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Random));
+ return true;
+}
+
+bool MRandom::canRecoverOnBailout() const {
+ return !js::SupportDifferentialTesting();
+}
+
+RRandom::RRandom(CompactBufferReader& reader) {}
+
+bool RRandom::recover(JSContext* cx, SnapshotIterator& iter) const {
+ iter.storeInstructionResult(DoubleValue(math_random_impl(cx)));
+ return true;
+}
+
+bool MStringSplit::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_StringSplit));
+ return true;
+}
+
+RStringSplit::RStringSplit(CompactBufferReader& reader) {}
+
+bool RStringSplit::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedString str(cx, iter.read().toString());
+ RootedString sep(cx, iter.read().toString());
+
+ JSObject* res = StringSplitString(cx, str, sep, INT32_MAX);
+ if (!res) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*res));
+ return true;
+}
+
+bool MNaNToZero::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_NaNToZero));
+ return true;
+}
+
+RNaNToZero::RNaNToZero(CompactBufferReader& reader) {}
+
+bool RNaNToZero::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double v = iter.read().toNumber();
+ if (mozilla::IsNaN(v) || mozilla::IsNegativeZero(v)) {
+ v = 0.0;
+ }
+
+ iter.storeInstructionResult(DoubleValue(v));
+ return true;
+}
+
+bool MRegExpMatcher::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_RegExpMatcher));
+ return true;
+}
+
+RRegExpMatcher::RRegExpMatcher(CompactBufferReader& reader) {}
+
+bool RRegExpMatcher::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject regexp(cx, &iter.read().toObject());
+ RootedString input(cx, iter.read().toString());
+ int32_t lastIndex = iter.read().toInt32();
+
+ RootedValue result(cx);
+ if (!RegExpMatcherRaw(cx, regexp, input, lastIndex, nullptr, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(result);
+ return true;
+}
+
+bool MRegExpSearcher::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_RegExpSearcher));
+ return true;
+}
+
+RRegExpSearcher::RRegExpSearcher(CompactBufferReader& reader) {}
+
+bool RRegExpSearcher::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject regexp(cx, &iter.read().toObject());
+ RootedString input(cx, iter.read().toString());
+ int32_t lastIndex = iter.read().toInt32();
+
+ int32_t result;
+ if (!RegExpSearcherRaw(cx, regexp, input, lastIndex, nullptr, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(Int32Value(result));
+ return true;
+}
+
+bool MRegExpTester::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_RegExpTester));
+ return true;
+}
+
+RRegExpTester::RRegExpTester(CompactBufferReader& reader) {}
+
+bool RRegExpTester::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedString string(cx, iter.read().toString());
+ RootedObject regexp(cx, &iter.read().toObject());
+ int32_t lastIndex = iter.read().toInt32();
+ int32_t endIndex;
+
+ if (!js::RegExpTesterRaw(cx, regexp, string, lastIndex, &endIndex)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(Int32Value(endIndex));
+ return true;
+}
+
+bool MTypeOf::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_TypeOf));
+ return true;
+}
+
+RTypeOf::RTypeOf(CompactBufferReader& reader) {}
+
+bool RTypeOf::recover(JSContext* cx, SnapshotIterator& iter) const {
+ JS::Value v = iter.read();
+
+ iter.storeInstructionResult(Int32Value(TypeOfValue(v)));
+ return true;
+}
+
+bool MTypeOfName::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_TypeOfName));
+ return true;
+}
+
+RTypeOfName::RTypeOfName(CompactBufferReader& reader) {}
+
+bool RTypeOfName::recover(JSContext* cx, SnapshotIterator& iter) const {
+ int32_t type = iter.read().toInt32();
+ MOZ_ASSERT(JSTYPE_UNDEFINED <= type && type < JSTYPE_LIMIT);
+
+ JSString* name = TypeName(JSType(type), *cx->runtime()->commonNames);
+ iter.storeInstructionResult(StringValue(name));
+ return true;
+}
+
+bool MToDouble::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_ToDouble));
+ return true;
+}
+
+RToDouble::RToDouble(CompactBufferReader& reader) {}
+
+bool RToDouble::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue v(cx, iter.read());
+
+ MOZ_ASSERT(!v.isObject());
+ MOZ_ASSERT(!v.isSymbol());
+ MOZ_ASSERT(!v.isBigInt());
+
+ double dbl;
+ if (!ToNumber(cx, v, &dbl)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(DoubleValue(dbl));
+ return true;
+}
+
+bool MToFloat32::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_ToFloat32));
+ return true;
+}
+
+RToFloat32::RToFloat32(CompactBufferReader& reader) {}
+
+bool RToFloat32::recover(JSContext* cx, SnapshotIterator& iter) const {
+ double num = iter.read().toNumber();
+ double result = js::RoundFloat32(num);
+
+ iter.storeInstructionResult(DoubleValue(result));
+ return true;
+}
+
+bool MTruncateToInt32::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_TruncateToInt32));
+ return true;
+}
+
+RTruncateToInt32::RTruncateToInt32(CompactBufferReader& reader) {}
+
+bool RTruncateToInt32::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue value(cx, iter.read());
+
+ int32_t trunc;
+ if (!JS::ToInt32(cx, value, &trunc)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(Int32Value(trunc));
+ return true;
+}
+
+bool MNewObject::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_NewObject));
+
+ // Recover instructions are only supported if we have a template object.
+ MOZ_ASSERT(mode_ == MNewObject::ObjectCreate);
+ return true;
+}
+
+RNewObject::RNewObject(CompactBufferReader& reader) {}
+
+bool RNewObject::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject templateObject(cx, &iter.read().toObject());
+
+ // See CodeGenerator::visitNewObjectVMCall.
+ // Note that recover instructions are only used if mode == ObjectCreate.
+ JSObject* resultObject =
+ ObjectCreateWithTemplate(cx, templateObject.as<PlainObject>());
+ if (!resultObject) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*resultObject));
+ return true;
+}
+
+bool MNewPlainObject::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_NewPlainObject));
+
+ MOZ_ASSERT(gc::AllocKind(uint8_t(allocKind_)) == allocKind_);
+ writer.writeByte(uint8_t(allocKind_));
+ MOZ_ASSERT(gc::InitialHeap(uint8_t(initialHeap_)) == initialHeap_);
+ writer.writeByte(uint8_t(initialHeap_));
+ return true;
+}
+
+RNewPlainObject::RNewPlainObject(CompactBufferReader& reader) {
+ allocKind_ = gc::AllocKind(reader.readByte());
+ MOZ_ASSERT(gc::IsValidAllocKind(allocKind_));
+ initialHeap_ = gc::InitialHeap(reader.readByte());
+ MOZ_ASSERT(initialHeap_ == gc::DefaultHeap ||
+ initialHeap_ == gc::TenuredHeap);
+}
+
+bool RNewPlainObject::recover(JSContext* cx, SnapshotIterator& iter) const {
+ Rooted<SharedShape*> shape(cx,
+ &iter.read().toGCCellPtr().as<Shape>().asShared());
+
+ // See CodeGenerator::visitNewPlainObject.
+ JSObject* resultObject =
+ NewPlainObjectOptimizedFallback(cx, shape, allocKind_, initialHeap_);
+ if (!resultObject) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*resultObject));
+ return true;
+}
+
+bool MNewArrayObject::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_NewArrayObject));
+
+ writer.writeUnsigned(length_);
+ MOZ_ASSERT(gc::InitialHeap(uint8_t(initialHeap_)) == initialHeap_);
+ writer.writeByte(uint8_t(initialHeap_));
+ return true;
+}
+
+RNewArrayObject::RNewArrayObject(CompactBufferReader& reader) {
+ length_ = reader.readUnsigned();
+ initialHeap_ = gc::InitialHeap(reader.readByte());
+ MOZ_ASSERT(initialHeap_ == gc::DefaultHeap ||
+ initialHeap_ == gc::TenuredHeap);
+}
+
+bool RNewArrayObject::recover(JSContext* cx, SnapshotIterator& iter) const {
+ iter.read(); // Skip unused shape field.
+
+ NewObjectKind kind =
+ initialHeap_ == gc::TenuredHeap ? TenuredObject : GenericObject;
+ JSObject* array = NewArrayOperation(cx, length_, kind);
+ if (!array) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*array));
+ return true;
+}
+
+bool MNewTypedArray::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_NewTypedArray));
+ return true;
+}
+
+RNewTypedArray::RNewTypedArray(CompactBufferReader& reader) {}
+
+bool RNewTypedArray::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject templateObject(cx, &iter.read().toObject());
+
+ size_t length = templateObject.as<TypedArrayObject>()->length();
+ MOZ_ASSERT(length <= INT32_MAX,
+ "Template objects are only created for int32 lengths");
+
+ JSObject* resultObject =
+ NewTypedArrayWithTemplateAndLength(cx, templateObject, length);
+ if (!resultObject) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*resultObject));
+ return true;
+}
+
+bool MNewArray::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_NewArray));
+ writer.writeUnsigned(length());
+ return true;
+}
+
+RNewArray::RNewArray(CompactBufferReader& reader) {
+ count_ = reader.readUnsigned();
+}
+
+bool RNewArray::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject templateObject(cx, &iter.read().toObject());
+ Rooted<Shape*> shape(cx, templateObject->shape());
+
+ ArrayObject* resultObject = NewArrayWithShape(cx, count_, shape);
+ if (!resultObject) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*resultObject));
+ return true;
+}
+
+bool MNewIterator::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_NewIterator));
+ writer.writeByte(type_);
+ return true;
+}
+
+RNewIterator::RNewIterator(CompactBufferReader& reader) {
+ type_ = reader.readByte();
+}
+
+bool RNewIterator::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject templateObject(cx, &iter.read().toObject());
+
+ JSObject* resultObject = nullptr;
+ switch (MNewIterator::Type(type_)) {
+ case MNewIterator::ArrayIterator:
+ resultObject = NewArrayIterator(cx);
+ break;
+ case MNewIterator::StringIterator:
+ resultObject = NewStringIterator(cx);
+ break;
+ case MNewIterator::RegExpStringIterator:
+ resultObject = NewRegExpStringIterator(cx);
+ break;
+ }
+
+ if (!resultObject) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*resultObject));
+ return true;
+}
+
+bool MLambda::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Lambda));
+ return true;
+}
+
+RLambda::RLambda(CompactBufferReader& reader) {}
+
+bool RLambda::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject scopeChain(cx, &iter.read().toObject());
+ RootedFunction fun(cx, &iter.read().toObject().as<JSFunction>());
+
+ JSObject* resultObject = js::Lambda(cx, fun, scopeChain);
+ if (!resultObject) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*resultObject));
+ return true;
+}
+
+bool MFunctionWithProto::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_FunctionWithProto));
+ return true;
+}
+
+RFunctionWithProto::RFunctionWithProto(CompactBufferReader& reader) {}
+
+bool RFunctionWithProto::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject scopeChain(cx, &iter.read().toObject());
+ RootedObject prototype(cx, &iter.read().toObject());
+ RootedFunction fun(cx, &iter.read().toObject().as<JSFunction>());
+
+ JSObject* resultObject =
+ js::FunWithProtoOperation(cx, fun, scopeChain, prototype);
+ if (!resultObject) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*resultObject));
+ return true;
+}
+
+bool MNewCallObject::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_NewCallObject));
+ return true;
+}
+
+RNewCallObject::RNewCallObject(CompactBufferReader& reader) {}
+
+bool RNewCallObject::recover(JSContext* cx, SnapshotIterator& iter) const {
+ Rooted<CallObject*> templateObj(cx, &iter.read().toObject().as<CallObject>());
+
+ Rooted<SharedShape*> shape(cx, templateObj->sharedShape());
+
+ JSObject* resultObject = CallObject::createWithShape(cx, shape);
+ if (!resultObject) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*resultObject));
+ return true;
+}
+
+bool MObjectState::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_ObjectState));
+ writer.writeUnsigned(numSlots());
+ return true;
+}
+
+RObjectState::RObjectState(CompactBufferReader& reader) {
+ numSlots_ = reader.readUnsigned();
+}
+
+bool RObjectState::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedObject object(cx, &iter.read().toObject());
+ Handle<NativeObject*> nativeObject = object.as<NativeObject>();
+ MOZ_ASSERT(nativeObject->slotSpan() == numSlots());
+
+ for (size_t i = 0; i < numSlots(); i++) {
+ Value val = iter.read();
+ nativeObject->setSlot(i, val);
+ }
+
+ iter.storeInstructionResult(ObjectValue(*object));
+ return true;
+}
+
+bool MArrayState::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_ArrayState));
+ writer.writeUnsigned(numElements());
+ return true;
+}
+
+RArrayState::RArrayState(CompactBufferReader& reader) {
+ numElements_ = reader.readUnsigned();
+}
+
+bool RArrayState::recover(JSContext* cx, SnapshotIterator& iter) const {
+ ArrayObject* object = &iter.read().toObject().as<ArrayObject>();
+ uint32_t initLength = iter.read().toInt32();
+
+ MOZ_ASSERT(object->getDenseInitializedLength() == 0,
+ "initDenseElement call below relies on this");
+ object->setDenseInitializedLength(initLength);
+
+ for (size_t index = 0; index < numElements(); index++) {
+ Value val = iter.read();
+
+ if (index >= initLength) {
+ MOZ_ASSERT(val.isUndefined());
+ continue;
+ }
+
+ object->initDenseElement(index, val);
+ }
+
+ iter.storeInstructionResult(ObjectValue(*object));
+ return true;
+}
+
+bool MSetArrayLength::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ // For simplicity, we capture directly the object instead of the elements
+ // pointer.
+ MOZ_ASSERT(elements()->type() != MIRType::Elements);
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_SetArrayLength));
+ return true;
+}
+
+bool MSetArrayLength::canRecoverOnBailout() const {
+ return isRecoveredOnBailout();
+}
+
+RSetArrayLength::RSetArrayLength(CompactBufferReader& reader) {}
+
+bool RSetArrayLength::recover(JSContext* cx, SnapshotIterator& iter) const {
+ Rooted<ArrayObject*> obj(cx, &iter.read().toObject().as<ArrayObject>());
+ RootedValue len(cx, iter.read());
+
+ RootedId id(cx, NameToId(cx->names().length));
+ Rooted<PropertyDescriptor> desc(
+ cx, PropertyDescriptor::Data(len, JS::PropertyAttribute::Writable));
+ ObjectOpResult error;
+ if (!ArraySetLength(cx, obj, id, desc, error)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*obj));
+ return true;
+}
+
+bool MAssertRecoveredOnBailout::writeRecoverData(
+ CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ MOZ_RELEASE_ASSERT(input()->isRecoveredOnBailout() == mustBeRecovered_,
+ "assertRecoveredOnBailout failed during compilation");
+ writer.writeUnsigned(
+ uint32_t(RInstruction::Recover_AssertRecoveredOnBailout));
+ return true;
+}
+
+RAssertRecoveredOnBailout::RAssertRecoveredOnBailout(
+ CompactBufferReader& reader) {}
+
+bool RAssertRecoveredOnBailout::recover(JSContext* cx,
+ SnapshotIterator& iter) const {
+ iter.read(); // skip the unused operand.
+ iter.storeInstructionResult(UndefinedValue());
+ return true;
+}
+
+bool MStringReplace::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_StringReplace));
+ writer.writeByte(isFlatReplacement_);
+ return true;
+}
+
+RStringReplace::RStringReplace(CompactBufferReader& reader) {
+ isFlatReplacement_ = reader.readByte();
+}
+
+bool RStringReplace::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedString string(cx, iter.read().toString());
+ RootedString pattern(cx, iter.read().toString());
+ RootedString replace(cx, iter.read().toString());
+
+ JSString* result =
+ isFlatReplacement_
+ ? js::StringFlatReplaceString(cx, string, pattern, replace)
+ : js::str_replace_string_raw(cx, string, pattern, replace);
+
+ if (!result) {
+ return false;
+ }
+
+ iter.storeInstructionResult(StringValue(result));
+ return true;
+}
+
+bool MSubstr::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Substr));
+ return true;
+}
+
+RSubstr::RSubstr(CompactBufferReader& reader) {}
+
+bool RSubstr::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedString str(cx, iter.read().toString());
+ int32_t begin = iter.read().toInt32();
+ int32_t length = iter.read().toInt32();
+
+ JSString* result = SubstringKernel(cx, str, begin, length);
+ if (!result) {
+ return false;
+ }
+
+ iter.storeInstructionResult(StringValue(result));
+ return true;
+}
+
+bool MAtomicIsLockFree::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_AtomicIsLockFree));
+ return true;
+}
+
+RAtomicIsLockFree::RAtomicIsLockFree(CompactBufferReader& reader) {}
+
+bool RAtomicIsLockFree::recover(JSContext* cx, SnapshotIterator& iter) const {
+ RootedValue operand(cx, iter.read());
+ MOZ_ASSERT(operand.isInt32());
+
+ int32_t result;
+ if (!js::AtomicIsLockFree(cx, operand, &result)) {
+ return false;
+ }
+
+ iter.storeInstructionResult(Int32Value(result));
+ return true;
+}
+
+bool MBigIntAsIntN::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntAsIntN));
+ return true;
+}
+
+RBigIntAsIntN::RBigIntAsIntN(CompactBufferReader& reader) {}
+
+bool RBigIntAsIntN::recover(JSContext* cx, SnapshotIterator& iter) const {
+ int32_t bits = iter.read().toInt32();
+ RootedBigInt input(cx, iter.read().toBigInt());
+
+ MOZ_ASSERT(bits >= 0);
+ BigInt* result = BigInt::asIntN(cx, input, bits);
+ if (!result) {
+ return false;
+ }
+
+ iter.storeInstructionResult(JS::BigIntValue(result));
+ return true;
+}
+
+bool MBigIntAsUintN::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_BigIntAsUintN));
+ return true;
+}
+
+RBigIntAsUintN::RBigIntAsUintN(CompactBufferReader& reader) {}
+
+bool RBigIntAsUintN::recover(JSContext* cx, SnapshotIterator& iter) const {
+ int32_t bits = iter.read().toInt32();
+ RootedBigInt input(cx, iter.read().toBigInt());
+
+ MOZ_ASSERT(bits >= 0);
+ BigInt* result = BigInt::asUintN(cx, input, bits);
+ if (!result) {
+ return false;
+ }
+
+ iter.storeInstructionResult(JS::BigIntValue(result));
+ return true;
+}
+
+bool MCreateArgumentsObject::writeRecoverData(
+ CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_CreateArgumentsObject));
+ return true;
+}
+
+RCreateArgumentsObject::RCreateArgumentsObject(CompactBufferReader& reader) {}
+
+bool RCreateArgumentsObject::recover(JSContext* cx,
+ SnapshotIterator& iter) const {
+ RootedObject callObject(cx, &iter.read().toObject());
+ RootedObject result(
+ cx, ArgumentsObject::createForIon(cx, iter.frame(), callObject));
+ if (!result) {
+ return false;
+ }
+
+ iter.storeInstructionResult(JS::ObjectValue(*result));
+ return true;
+}
+
+bool MCreateInlinedArgumentsObject::writeRecoverData(
+ CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(
+ uint32_t(RInstruction::Recover_CreateInlinedArgumentsObject));
+ writer.writeUnsigned(numActuals());
+ return true;
+}
+
+RCreateInlinedArgumentsObject::RCreateInlinedArgumentsObject(
+ CompactBufferReader& reader) {
+ numActuals_ = reader.readUnsigned();
+}
+
+bool RCreateInlinedArgumentsObject::recover(JSContext* cx,
+ SnapshotIterator& iter) const {
+ RootedObject callObject(cx, &iter.read().toObject());
+ RootedFunction callee(cx, &iter.read().toObject().as<JSFunction>());
+
+ JS::RootedValueArray<ArgumentsObject::MaxInlinedArgs> argsArray(cx);
+ for (uint32_t i = 0; i < numActuals_; i++) {
+ argsArray[i].set(iter.read());
+ }
+
+ ArgumentsObject* result = ArgumentsObject::createFromValueArray(
+ cx, argsArray, callee, callObject, numActuals_);
+ if (!result) {
+ return false;
+ }
+
+ iter.storeInstructionResult(JS::ObjectValue(*result));
+ return true;
+}
+
+bool MRest::writeRecoverData(CompactBufferWriter& writer) const {
+ MOZ_ASSERT(canRecoverOnBailout());
+ writer.writeUnsigned(uint32_t(RInstruction::Recover_Rest));
+ writer.writeUnsigned(numFormals());
+ return true;
+}
+
+RRest::RRest(CompactBufferReader& reader) {
+ numFormals_ = reader.readUnsigned();
+}
+
+bool RRest::recover(JSContext* cx, SnapshotIterator& iter) const {
+ JitFrameLayout* frame = iter.frame();
+
+ uint32_t numActuals = iter.read().toInt32();
+ MOZ_ASSERT(numActuals == frame->numActualArgs());
+
+ uint32_t numFormals = numFormals_;
+
+ uint32_t length = std::max(numActuals, numFormals) - numFormals;
+ Value* src = frame->actualArgs() + numFormals;
+ JSObject* rest = jit::InitRestParameter(cx, length, src, nullptr);
+ if (!rest) {
+ return false;
+ }
+
+ iter.storeInstructionResult(ObjectValue(*rest));
+ return true;
+}
diff --git a/js/src/jit/Recover.h b/js/src/jit/Recover.h
new file mode 100644
index 0000000000..352b41cea3
--- /dev/null
+++ b/js/src/jit/Recover.h
@@ -0,0 +1,973 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Recover_h
+#define jit_Recover_h
+
+#include "mozilla/Attributes.h"
+
+#include "jit/MIR.h"
+#include "jit/Snapshots.h"
+
+namespace js {
+namespace jit {
+
+// [SMDOC] IonMonkey Recover Instructions
+//
+// This file contains all recover instructions.
+//
+// A recover instruction is an equivalent of a MIR instruction which is executed
+// before the reconstruction of a baseline frame. Recover instructions are used
+// by resume points to fill the value which are not produced by the code
+// compiled by IonMonkey. For example, if a value is optimized away by
+// IonMonkey, but required by Baseline, then we should have a recover
+// instruction to fill the missing baseline frame slot.
+//
+// Recover instructions are executed either during a bailout, or under a call
+// when the stack frame is introspected. If the stack is introspected, then any
+// use of recover instruction must lead to an invalidation of the code.
+//
+// For each MIR instruction where |canRecoverOnBailout| might return true, we
+// have a RInstruction of the same name.
+//
+// Recover instructions are encoded by the code generator into a compact buffer
+// (RecoverWriter). The MIR instruction method |writeRecoverData| should write a
+// tag in the |CompactBufferWriter| which is used by
+// |RInstruction::readRecoverData| to dispatch to the right Recover
+// instruction. Then |writeRecoverData| writes any local fields which are
+// necessary for the execution of the |recover| method. These fields are decoded
+// by the Recover instruction constructor which has a |CompactBufferReader| as
+// argument. The constructor of the Recover instruction should follow the same
+// sequence as the |writeRecoverData| method of the MIR instruction.
+//
+// Recover instructions are decoded by the |SnapshotIterator| (RecoverReader),
+// which is given as argument of the |recover| methods, in order to read the
+// operands. The number of operands read should be the same as the result of
+// |numOperands|, which corresponds to the number of operands of the MIR
+// instruction. Operands should be decoded in the same order as the operands of
+// the MIR instruction.
+//
+// The result of the |recover| method should either be a failure, or a value
+// stored on the |SnapshotIterator|, by using the |storeInstructionResult|
+// method.
+
+#define RECOVER_OPCODE_LIST(_) \
+ _(ResumePoint) \
+ _(BitNot) \
+ _(BitAnd) \
+ _(BitOr) \
+ _(BitXor) \
+ _(Lsh) \
+ _(Rsh) \
+ _(Ursh) \
+ _(SignExtendInt32) \
+ _(Add) \
+ _(Sub) \
+ _(Mul) \
+ _(Div) \
+ _(Mod) \
+ _(Not) \
+ _(BigIntAdd) \
+ _(BigIntSub) \
+ _(BigIntMul) \
+ _(BigIntDiv) \
+ _(BigIntMod) \
+ _(BigIntPow) \
+ _(BigIntBitAnd) \
+ _(BigIntBitOr) \
+ _(BigIntBitXor) \
+ _(BigIntLsh) \
+ _(BigIntRsh) \
+ _(BigIntIncrement) \
+ _(BigIntDecrement) \
+ _(BigIntNegate) \
+ _(BigIntBitNot) \
+ _(Compare) \
+ _(Concat) \
+ _(StringLength) \
+ _(ArgumentsLength) \
+ _(Floor) \
+ _(Ceil) \
+ _(Round) \
+ _(Trunc) \
+ _(CharCodeAt) \
+ _(FromCharCode) \
+ _(Pow) \
+ _(PowHalf) \
+ _(MinMax) \
+ _(Abs) \
+ _(Sqrt) \
+ _(Atan2) \
+ _(Hypot) \
+ _(NearbyInt) \
+ _(Sign) \
+ _(MathFunction) \
+ _(Random) \
+ _(StringSplit) \
+ _(NaNToZero) \
+ _(RegExpMatcher) \
+ _(RegExpSearcher) \
+ _(RegExpTester) \
+ _(StringReplace) \
+ _(Substr) \
+ _(TypeOf) \
+ _(TypeOfName) \
+ _(ToDouble) \
+ _(ToFloat32) \
+ _(TruncateToInt32) \
+ _(NewObject) \
+ _(NewPlainObject) \
+ _(NewArrayObject) \
+ _(NewTypedArray) \
+ _(NewArray) \
+ _(NewIterator) \
+ _(NewCallObject) \
+ _(Lambda) \
+ _(FunctionWithProto) \
+ _(ObjectState) \
+ _(ArrayState) \
+ _(SetArrayLength) \
+ _(AtomicIsLockFree) \
+ _(BigIntAsIntN) \
+ _(BigIntAsUintN) \
+ _(CreateArgumentsObject) \
+ _(CreateInlinedArgumentsObject) \
+ _(Rest) \
+ _(AssertRecoveredOnBailout)
+
+class RResumePoint;
+class SnapshotIterator;
+
+class MOZ_NON_PARAM RInstruction {
+ public:
+ enum Opcode {
+#define DEFINE_OPCODES_(op) Recover_##op,
+ RECOVER_OPCODE_LIST(DEFINE_OPCODES_)
+#undef DEFINE_OPCODES_
+ Recover_Invalid
+ };
+
+ virtual Opcode opcode() const = 0;
+
+ // As opposed to the MIR, there is no need to add more methods as every
+ // other instruction is well abstracted under the "recover" method.
+ bool isResumePoint() const { return opcode() == Recover_ResumePoint; }
+ inline const RResumePoint* toResumePoint() const;
+
+ // Call the copy constructor of a specific RInstruction, to do a copy of the
+ // RInstruction content.
+ virtual void cloneInto(RInstructionStorage* raw) const = 0;
+
+ // Number of allocations which are encoded in the Snapshot for recovering
+ // the current instruction.
+ virtual uint32_t numOperands() const = 0;
+
+ // Function used to recover the value computed by this instruction. This
+ // function reads its arguments from the allocations listed on the snapshot
+ // iterator and stores its returned value on the snapshot iterator too.
+ [[nodiscard]] virtual bool recover(JSContext* cx,
+ SnapshotIterator& iter) const = 0;
+
+ // Decode an RInstruction on top of the reserved storage space, based on the
+ // tag written by the writeRecoverData function of the corresponding MIR
+ // instruction.
+ static void readRecoverData(CompactBufferReader& reader,
+ RInstructionStorage* raw);
+};
+
+#define RINSTRUCTION_HEADER_(op) \
+ private: \
+ friend class RInstruction; \
+ explicit R##op(CompactBufferReader& reader); \
+ explicit R##op(const R##op& src) = default; \
+ \
+ public: \
+ Opcode opcode() const override { return RInstruction::Recover_##op; } \
+ void cloneInto(RInstructionStorage* raw) const override { \
+ new (raw->addr()) R##op(*this); \
+ }
+
+#define RINSTRUCTION_HEADER_NUM_OP_MAIN(op, numOp) \
+ RINSTRUCTION_HEADER_(op) \
+ uint32_t numOperands() const override { return numOp; }
+
+#ifdef DEBUG
+# define RINSTRUCTION_HEADER_NUM_OP_(op, numOp) \
+ RINSTRUCTION_HEADER_NUM_OP_MAIN(op, numOp) \
+ static_assert( \
+ M##op::staticNumOperands == numOp, \
+ "The recover instructions's numOperands should equal to the " \
+ "MIR's numOperands");
+#else
+# define RINSTRUCTION_HEADER_NUM_OP_(op, numOp) \
+ RINSTRUCTION_HEADER_NUM_OP_MAIN(op, numOp)
+#endif
+
+class RResumePoint final : public RInstruction {
+ private:
+ uint32_t pcOffsetAndMode_; // Offset from script->code and ResumeMode.
+ uint32_t numOperands_; // Number of slots.
+
+ public:
+ RINSTRUCTION_HEADER_(ResumePoint)
+
+ // Used to encode/decode pcOffsetAndMode_.
+ static constexpr uint32_t PCOffsetShift = 4;
+ static constexpr uint32_t ResumeModeMask = 0b1111;
+ static_assert(uint32_t(ResumeMode::Last) <= ResumeModeMask);
+
+ uint32_t pcOffset() const { return pcOffsetAndMode_ >> PCOffsetShift; }
+ ResumeMode mode() const {
+ return ResumeMode(pcOffsetAndMode_ & ResumeModeMask);
+ }
+
+ uint32_t numOperands() const override { return numOperands_; }
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBitNot final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BitNot, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBitAnd final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BitAnd, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBitOr final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BitOr, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBitXor final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BitXor, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RLsh final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Lsh, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RRsh final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Rsh, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RUrsh final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Ursh, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RSignExtendInt32 final : public RInstruction {
+ private:
+ uint8_t mode_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(SignExtendInt32, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RAdd final : public RInstruction {
+ private:
+ bool isFloatOperation_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Add, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RSub final : public RInstruction {
+ private:
+ bool isFloatOperation_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Sub, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RMul final : public RInstruction {
+ private:
+ bool isFloatOperation_;
+ uint8_t mode_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Mul, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RDiv final : public RInstruction {
+ private:
+ bool isFloatOperation_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Div, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RMod final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Mod, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNot final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Not, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntAdd final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntAdd, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntSub final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntSub, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntMul final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntMul, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntDiv final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntDiv, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntMod final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntMod, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntPow final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntPow, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntBitAnd final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntBitAnd, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntBitOr final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntBitOr, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntBitXor final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntBitXor, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntLsh final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntLsh, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntRsh final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntRsh, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntIncrement final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntIncrement, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntDecrement final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntDecrement, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntNegate final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntNegate, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntBitNot final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntBitNot, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RCompare final : public RInstruction {
+ JSOp jsop_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Compare, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RConcat final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Concat, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RStringLength final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(StringLength, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RArgumentsLength final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(ArgumentsLength, 0)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RFloor final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Floor, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RCeil final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Ceil, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RRound final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Round, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RTrunc final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Trunc, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RCharCodeAt final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(CharCodeAt, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RFromCharCode final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(FromCharCode, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RPow final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Pow, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RPowHalf final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(PowHalf, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RMinMax final : public RInstruction {
+ private:
+ bool isMax_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(MinMax, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RAbs final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Abs, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RSqrt final : public RInstruction {
+ private:
+ bool isFloatOperation_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Sqrt, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RAtan2 final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Atan2, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RHypot final : public RInstruction {
+ private:
+ uint32_t numOperands_;
+
+ public:
+ RINSTRUCTION_HEADER_(Hypot)
+
+ uint32_t numOperands() const override { return numOperands_; }
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNearbyInt final : public RInstruction {
+ private:
+ uint8_t roundingMode_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NearbyInt, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RSign final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Sign, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RMathFunction final : public RInstruction {
+ private:
+ UnaryMathFunction function_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(MathFunction, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RRandom final : public RInstruction {
+ RINSTRUCTION_HEADER_NUM_OP_(Random, 0)
+ public:
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RStringSplit final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(StringSplit, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNaNToZero final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NaNToZero, 1);
+
+ bool recover(JSContext* cx, SnapshotIterator& iter) const override;
+};
+
+class RRegExpMatcher final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(RegExpMatcher, 3)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RRegExpSearcher final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(RegExpSearcher, 3)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RRegExpTester final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(RegExpTester, 3)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RStringReplace final : public RInstruction {
+ private:
+ bool isFlatReplacement_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(StringReplace, 3)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RSubstr final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Substr, 3)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RTypeOf final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(TypeOf, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RTypeOfName final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(TypeOfName, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RToDouble final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(ToDouble, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RToFloat32 final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(ToFloat32, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RTruncateToInt32 final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(TruncateToInt32, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNewObject final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NewObject, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNewPlainObject final : public RInstruction {
+ private:
+ gc::AllocKind allocKind_;
+ gc::InitialHeap initialHeap_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NewPlainObject, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNewArrayObject final : public RInstruction {
+ private:
+ uint32_t length_;
+ gc::InitialHeap initialHeap_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NewArrayObject, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNewTypedArray final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NewTypedArray, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNewArray final : public RInstruction {
+ private:
+ uint32_t count_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NewArray, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNewIterator final : public RInstruction {
+ private:
+ uint8_t type_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NewIterator, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RLambda final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Lambda, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RFunctionWithProto final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(FunctionWithProto, 3)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RNewCallObject final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(NewCallObject, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RObjectState final : public RInstruction {
+ private:
+ uint32_t numSlots_; // Number of slots.
+
+ public:
+ RINSTRUCTION_HEADER_(ObjectState)
+
+ uint32_t numSlots() const { return numSlots_; }
+ uint32_t numOperands() const override {
+ // +1 for the object.
+ return numSlots() + 1;
+ }
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RArrayState final : public RInstruction {
+ private:
+ uint32_t numElements_;
+
+ public:
+ RINSTRUCTION_HEADER_(ArrayState)
+
+ uint32_t numElements() const { return numElements_; }
+ uint32_t numOperands() const override {
+ // +1 for the array.
+ // +1 for the initalized length.
+ return numElements() + 2;
+ }
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RSetArrayLength final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(SetArrayLength, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RAtomicIsLockFree final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(AtomicIsLockFree, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntAsIntN final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntAsIntN, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RBigIntAsUintN final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(BigIntAsUintN, 2)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RCreateArgumentsObject final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(CreateArgumentsObject, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RCreateInlinedArgumentsObject final : public RInstruction {
+ private:
+ uint32_t numActuals_;
+
+ public:
+ RINSTRUCTION_HEADER_(CreateInlinedArgumentsObject)
+
+ uint32_t numActuals() const { return numActuals_; }
+ uint32_t numOperands() const override {
+ // +1 for the callObj.
+ // +1 for the callee.
+ return numActuals() + 2;
+ }
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RRest final : public RInstruction {
+ uint32_t numFormals_;
+
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(Rest, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+class RAssertRecoveredOnBailout final : public RInstruction {
+ public:
+ RINSTRUCTION_HEADER_NUM_OP_(AssertRecoveredOnBailout, 1)
+
+ [[nodiscard]] bool recover(JSContext* cx,
+ SnapshotIterator& iter) const override;
+};
+
+#undef RINSTRUCTION_HEADER_
+#undef RINSTRUCTION_HEADER_NUM_OP_
+#undef RINSTRUCTION_HEADER_NUM_OP_MAIN
+
+const RResumePoint* RInstruction::toResumePoint() const {
+ MOZ_ASSERT(isResumePoint());
+ return static_cast<const RResumePoint*>(this);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Recover_h */
diff --git a/js/src/jit/RegisterAllocator.cpp b/js/src/jit/RegisterAllocator.cpp
new file mode 100644
index 0000000000..c572e4bc2a
--- /dev/null
+++ b/js/src/jit/RegisterAllocator.cpp
@@ -0,0 +1,669 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/RegisterAllocator.h"
+
+using namespace js;
+using namespace js::jit;
+
+#ifdef DEBUG
+bool AllocationIntegrityState::record() {
+ // Ignore repeated record() calls.
+ if (!instructions.empty()) {
+ return true;
+ }
+
+ if (!instructions.appendN(InstructionInfo(), graph.numInstructions())) {
+ return false;
+ }
+
+ if (!virtualRegisters.appendN((LDefinition*)nullptr,
+ graph.numVirtualRegisters())) {
+ return false;
+ }
+
+ if (!blocks.reserve(graph.numBlocks())) {
+ return false;
+ }
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ blocks.infallibleAppend(BlockInfo());
+ LBlock* block = graph.getBlock(i);
+ MOZ_ASSERT(block->mir()->id() == i);
+
+ BlockInfo& blockInfo = blocks[i];
+ if (!blockInfo.phis.reserve(block->numPhis())) {
+ return false;
+ }
+
+ for (size_t j = 0; j < block->numPhis(); j++) {
+ blockInfo.phis.infallibleAppend(InstructionInfo());
+ InstructionInfo& info = blockInfo.phis[j];
+ LPhi* phi = block->getPhi(j);
+ MOZ_ASSERT(phi->numDefs() == 1);
+ uint32_t vreg = phi->getDef(0)->virtualRegister();
+ virtualRegisters[vreg] = phi->getDef(0);
+ if (!info.outputs.append(*phi->getDef(0))) {
+ return false;
+ }
+ for (size_t k = 0, kend = phi->numOperands(); k < kend; k++) {
+ if (!info.inputs.append(*phi->getOperand(k))) {
+ return false;
+ }
+ }
+ }
+
+ for (LInstructionIterator iter = block->begin(); iter != block->end();
+ iter++) {
+ LInstruction* ins = *iter;
+ InstructionInfo& info = instructions[ins->id()];
+
+ for (size_t k = 0; k < ins->numTemps(); k++) {
+ if (!ins->getTemp(k)->isBogusTemp()) {
+ uint32_t vreg = ins->getTemp(k)->virtualRegister();
+ virtualRegisters[vreg] = ins->getTemp(k);
+ }
+ if (!info.temps.append(*ins->getTemp(k))) {
+ return false;
+ }
+ }
+ for (size_t k = 0; k < ins->numDefs(); k++) {
+ if (!ins->getDef(k)->isBogusTemp()) {
+ uint32_t vreg = ins->getDef(k)->virtualRegister();
+ virtualRegisters[vreg] = ins->getDef(k);
+ }
+ if (!info.outputs.append(*ins->getDef(k))) {
+ return false;
+ }
+ }
+ for (LInstruction::InputIterator alloc(*ins); alloc.more();
+ alloc.next()) {
+ if (!info.inputs.append(**alloc)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+bool AllocationIntegrityState::check() {
+ MOZ_ASSERT(!instructions.empty());
+
+# ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_RegAlloc)) {
+ dump();
+ }
+# endif
+ for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) {
+ LBlock* block = graph.getBlock(blockIndex);
+
+ // Check that all instruction inputs and outputs have been assigned an
+ // allocation.
+ for (LInstructionIterator iter = block->begin(); iter != block->end();
+ iter++) {
+ LInstruction* ins = *iter;
+
+ for (LInstruction::InputIterator alloc(*ins); alloc.more();
+ alloc.next()) {
+ MOZ_ASSERT(!alloc->isUse());
+ }
+
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ LDefinition* def = ins->getDef(i);
+ MOZ_ASSERT(!def->output()->isUse());
+
+ LDefinition oldDef = instructions[ins->id()].outputs[i];
+ MOZ_ASSERT_IF(
+ oldDef.policy() == LDefinition::MUST_REUSE_INPUT,
+ *def->output() == *ins->getOperand(oldDef.getReusedInput()));
+ }
+
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ LDefinition* temp = ins->getTemp(i);
+ MOZ_ASSERT_IF(!temp->isBogusTemp(), temp->output()->isRegister());
+
+ LDefinition oldTemp = instructions[ins->id()].temps[i];
+ MOZ_ASSERT_IF(
+ oldTemp.policy() == LDefinition::MUST_REUSE_INPUT,
+ *temp->output() == *ins->getOperand(oldTemp.getReusedInput()));
+ }
+ }
+ }
+
+ // Check that the register assignment and move groups preserve the original
+ // semantics of the virtual registers. Each virtual register has a single
+ // write (owing to the SSA representation), but the allocation may move the
+ // written value around between registers and memory locations along
+ // different paths through the script.
+ //
+ // For each use of an allocation, follow the physical value which is read
+ // backward through the script, along all paths to the value's virtual
+ // register's definition.
+ for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) {
+ LBlock* block = graph.getBlock(blockIndex);
+ for (LInstructionIterator iter = block->begin(); iter != block->end();
+ iter++) {
+ LInstruction* ins = *iter;
+ const InstructionInfo& info = instructions[ins->id()];
+
+ LSafepoint* safepoint = ins->safepoint();
+ if (safepoint) {
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ if (ins->getTemp(i)->isBogusTemp()) {
+ continue;
+ }
+ uint32_t vreg = info.temps[i].virtualRegister();
+ LAllocation* alloc = ins->getTemp(i)->output();
+ checkSafepointAllocation(ins, vreg, *alloc);
+ }
+ MOZ_ASSERT_IF(ins->isCall(), safepoint->liveRegs().emptyFloat() &&
+ safepoint->liveRegs().emptyGeneral());
+ }
+
+ size_t inputIndex = 0;
+ for (LInstruction::InputIterator alloc(*ins); alloc.more();
+ inputIndex++, alloc.next()) {
+ LAllocation oldInput = info.inputs[inputIndex];
+ if (!oldInput.isUse()) {
+ continue;
+ }
+
+ uint32_t vreg = oldInput.toUse()->virtualRegister();
+
+ if (safepoint && !oldInput.toUse()->usedAtStart()) {
+ checkSafepointAllocation(ins, vreg, **alloc);
+ }
+
+ // Temps must never alias inputs (even at-start uses) unless explicitly
+ // requested.
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ if (ins->getTemp(i)->isBogusTemp()) {
+ continue;
+ }
+ LAllocation* tempAlloc = ins->getTemp(i)->output();
+
+ // Fixed uses and fixed temps are allowed to alias.
+ if (oldInput.toUse()->isFixedRegister() && info.temps[i].isFixed()) {
+ continue;
+ }
+
+ // MUST_REUSE_INPUT temps will alias their input.
+ if (info.temps[i].policy() == LDefinition::MUST_REUSE_INPUT &&
+ info.temps[i].getReusedInput() == inputIndex) {
+ continue;
+ }
+
+ MOZ_ASSERT(!tempAlloc->aliases(**alloc));
+ }
+
+ // Start checking at the previous instruction, in case this
+ // instruction reuses its input register for an output.
+ LInstructionReverseIterator riter = block->rbegin(ins);
+ riter++;
+ if (!checkIntegrity(block, *riter, vreg, **alloc)) {
+ return false;
+ }
+
+ while (!worklist.empty()) {
+ IntegrityItem item = worklist.popCopy();
+ if (!checkIntegrity(item.block, *item.block->rbegin(), item.vreg,
+ item.alloc)) {
+ return false;
+ }
+ }
+ }
+ }
+ }
+
+ return true;
+}
+
+bool AllocationIntegrityState::checkIntegrity(LBlock* block, LInstruction* ins,
+ uint32_t vreg,
+ LAllocation alloc) {
+ for (LInstructionReverseIterator iter(block->rbegin(ins));
+ iter != block->rend(); iter++) {
+ ins = *iter;
+
+ // Follow values through assignments in move groups. All assignments in
+ // a move group are considered to happen simultaneously, so stop after
+ // the first matching move is found.
+ if (ins->isMoveGroup()) {
+ LMoveGroup* group = ins->toMoveGroup();
+ for (int i = group->numMoves() - 1; i >= 0; i--) {
+ if (group->getMove(i).to() == alloc) {
+ alloc = group->getMove(i).from();
+ break;
+ }
+ }
+ }
+
+ const InstructionInfo& info = instructions[ins->id()];
+
+ // Make sure the physical location being tracked is not clobbered by
+ // another instruction, and that if the originating vreg definition is
+ // found that it is writing to the tracked location.
+
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ LDefinition* def = ins->getDef(i);
+ if (def->isBogusTemp()) {
+ continue;
+ }
+ if (info.outputs[i].virtualRegister() == vreg) {
+# ifdef JS_JITSPEW
+ // If the following assertion is about to fail, print some useful info.
+ if (!(*def->output() == alloc) && JitSpewEnabled(JitSpew_RegAlloc)) {
+ CodePosition input(ins->id(), CodePosition::INPUT);
+ CodePosition output(ins->id(), CodePosition::OUTPUT);
+ JitSpew(JitSpew_RegAlloc,
+ "Instruction at %u-%u, output number %u:", input.bits(),
+ output.bits(), unsigned(i));
+ JitSpew(JitSpew_RegAlloc,
+ " Error: conflicting allocations: %s vs %s",
+ (*def->output()).toString().get(), alloc.toString().get());
+ }
+# endif
+ MOZ_ASSERT(*def->output() == alloc);
+
+ // Found the original definition, done scanning.
+ return true;
+ } else {
+ MOZ_ASSERT(*def->output() != alloc);
+ }
+ }
+
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ LDefinition* temp = ins->getTemp(i);
+ if (!temp->isBogusTemp()) {
+ MOZ_ASSERT(*temp->output() != alloc);
+ }
+ }
+
+ if (ins->safepoint()) {
+ checkSafepointAllocation(ins, vreg, alloc);
+ }
+ }
+
+ // Phis are effectless, but change the vreg we are tracking. Check if there
+ // is one which produced this vreg. We need to follow back through the phi
+ // inputs as it is not guaranteed the register allocator filled in physical
+ // allocations for the inputs and outputs of the phis.
+ for (size_t i = 0; i < block->numPhis(); i++) {
+ const InstructionInfo& info = blocks[block->mir()->id()].phis[i];
+ LPhi* phi = block->getPhi(i);
+ if (info.outputs[0].virtualRegister() == vreg) {
+ for (size_t j = 0, jend = phi->numOperands(); j < jend; j++) {
+ uint32_t newvreg = info.inputs[j].toUse()->virtualRegister();
+ LBlock* predecessor = block->mir()->getPredecessor(j)->lir();
+ if (!addPredecessor(predecessor, newvreg, alloc)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ // No phi which defined the vreg we are tracking, follow back through all
+ // predecessors with the existing vreg.
+ for (size_t i = 0, iend = block->mir()->numPredecessors(); i < iend; i++) {
+ LBlock* predecessor = block->mir()->getPredecessor(i)->lir();
+ if (!addPredecessor(predecessor, vreg, alloc)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void AllocationIntegrityState::checkSafepointAllocation(LInstruction* ins,
+ uint32_t vreg,
+ LAllocation alloc) {
+ LSafepoint* safepoint = ins->safepoint();
+ MOZ_ASSERT(safepoint);
+
+ if (ins->isCall() && alloc.isRegister()) {
+ return;
+ }
+
+ if (alloc.isRegister()) {
+ MOZ_ASSERT(safepoint->liveRegs().has(alloc.toRegister()));
+ }
+
+ // The |this| argument slot is implicitly included in all safepoints.
+ if (alloc.isArgument() &&
+ alloc.toArgument()->index() < THIS_FRAME_ARGSLOT + sizeof(Value)) {
+ return;
+ }
+
+ LDefinition::Type type = virtualRegisters[vreg]
+ ? virtualRegisters[vreg]->type()
+ : LDefinition::GENERAL;
+
+ switch (type) {
+ case LDefinition::OBJECT:
+ MOZ_ASSERT(safepoint->hasGcPointer(alloc));
+ break;
+ case LDefinition::STACKRESULTS:
+ MOZ_ASSERT(safepoint->hasAllGcPointersFromStackArea(alloc));
+ break;
+ case LDefinition::SLOTS:
+ MOZ_ASSERT(safepoint->hasSlotsOrElementsPointer(alloc));
+ break;
+# ifdef JS_NUNBOX32
+ // Do not assert that safepoint information for nunbox types is complete,
+ // as if a vreg for a value's components are copied in multiple places
+ // then the safepoint information may not reflect all copies. All copies
+ // of payloads must be reflected, however, for generational GC.
+ case LDefinition::TYPE:
+ break;
+ case LDefinition::PAYLOAD:
+ MOZ_ASSERT(safepoint->hasNunboxPayload(alloc));
+ break;
+# else
+ case LDefinition::BOX:
+ MOZ_ASSERT(safepoint->hasBoxedValue(alloc));
+ break;
+# endif
+ default:
+ break;
+ }
+}
+
+bool AllocationIntegrityState::addPredecessor(LBlock* block, uint32_t vreg,
+ LAllocation alloc) {
+ // There is no need to reanalyze if we have already seen this predecessor.
+ // We share the seen allocations across analysis of each use, as there will
+ // likely be common ground between different uses of the same vreg.
+ IntegrityItem item;
+ item.block = block;
+ item.vreg = vreg;
+ item.alloc = alloc;
+ item.index = seen.count();
+
+ IntegrityItemSet::AddPtr p = seen.lookupForAdd(item);
+ if (p) {
+ return true;
+ }
+ if (!seen.add(p, item)) {
+ return false;
+ }
+
+ return worklist.append(item);
+}
+
+void AllocationIntegrityState::dump() {
+# ifdef JS_JITSPEW
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ JitSpew(JitSpew_RegAlloc, "Register Allocation Integrity State:");
+
+ for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) {
+ LBlock* block = graph.getBlock(blockIndex);
+ MBasicBlock* mir = block->mir();
+
+ JitSpewHeader(JitSpew_RegAlloc);
+ JitSpewCont(JitSpew_RegAlloc, " Block %lu",
+ static_cast<unsigned long>(blockIndex));
+ for (size_t i = 0; i < mir->numSuccessors(); i++) {
+ JitSpewCont(JitSpew_RegAlloc, " [successor %u]",
+ mir->getSuccessor(i)->id());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+
+ for (size_t i = 0; i < block->numPhis(); i++) {
+ const InstructionInfo& info = blocks[blockIndex].phis[i];
+ LPhi* phi = block->getPhi(i);
+ CodePosition input(block->getPhi(0)->id(), CodePosition::INPUT);
+ CodePosition output(block->getPhi(block->numPhis() - 1)->id(),
+ CodePosition::OUTPUT);
+
+ JitSpewHeader(JitSpew_RegAlloc);
+ JitSpewCont(JitSpew_RegAlloc, " %u-%u Phi [def %s] ", input.bits(),
+ output.bits(), phi->getDef(0)->toString().get());
+ for (size_t j = 0; j < phi->numOperands(); j++) {
+ JitSpewCont(JitSpew_RegAlloc, " [use %s]",
+ info.inputs[j].toString().get());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ }
+
+ for (LInstructionIterator iter = block->begin(); iter != block->end();
+ iter++) {
+ LInstruction* ins = *iter;
+ const InstructionInfo& info = instructions[ins->id()];
+
+ CodePosition input(ins->id(), CodePosition::INPUT);
+ CodePosition output(ins->id(), CodePosition::OUTPUT);
+
+ JitSpewHeader(JitSpew_RegAlloc);
+ JitSpewCont(JitSpew_RegAlloc, " ");
+ if (input != CodePosition::MIN) {
+ JitSpewCont(JitSpew_RegAlloc, "%u-%u ", input.bits(), output.bits());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "%s", ins->opName());
+
+ if (ins->isMoveGroup()) {
+ LMoveGroup* group = ins->toMoveGroup();
+ for (int i = group->numMoves() - 1; i >= 0; i--) {
+ JitSpewCont(JitSpew_RegAlloc, " [%s <- %s]",
+ group->getMove(i).to().toString().get(),
+ group->getMove(i).from().toString().get());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ continue;
+ }
+
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ JitSpewCont(JitSpew_RegAlloc, " [def %s]",
+ ins->getDef(i)->toString().get());
+ }
+
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ LDefinition* temp = ins->getTemp(i);
+ if (!temp->isBogusTemp()) {
+ JitSpewCont(JitSpew_RegAlloc, " [temp v%u %s]",
+ info.temps[i].virtualRegister(), temp->toString().get());
+ }
+ }
+
+ size_t index = 0;
+ for (LInstruction::InputIterator alloc(*ins); alloc.more();
+ alloc.next()) {
+ JitSpewCont(JitSpew_RegAlloc, " [use %s",
+ info.inputs[index++].toString().get());
+ if (!alloc->isConstant()) {
+ JitSpewCont(JitSpew_RegAlloc, " %s", alloc->toString().get());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "]");
+ }
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ }
+ }
+
+ // Print discovered allocations at the ends of blocks, in the order they
+ // were discovered.
+
+ Vector<IntegrityItem, 20, SystemAllocPolicy> seenOrdered;
+ if (!seenOrdered.appendN(IntegrityItem(), seen.count())) {
+ fprintf(stderr, "OOM while dumping allocations\n");
+ return;
+ }
+
+ for (IntegrityItemSet::Enum iter(seen); !iter.empty(); iter.popFront()) {
+ IntegrityItem item = iter.front();
+ seenOrdered[item.index] = item;
+ }
+
+ if (!seenOrdered.empty()) {
+ fprintf(stderr, "Intermediate Allocations:\n");
+
+ for (size_t i = 0; i < seenOrdered.length(); i++) {
+ IntegrityItem item = seenOrdered[i];
+ fprintf(stderr, " block %u reg v%u alloc %s\n", item.block->mir()->id(),
+ item.vreg, item.alloc.toString().get());
+ }
+ }
+
+ fprintf(stderr, "\n");
+# endif
+}
+#endif // DEBUG
+
+const CodePosition CodePosition::MAX(UINT_MAX);
+const CodePosition CodePosition::MIN(0);
+
+bool RegisterAllocator::init() {
+ if (!insData.init(mir, graph.numInstructions())) {
+ return false;
+ }
+
+ if (!entryPositions.reserve(graph.numBlocks()) ||
+ !exitPositions.reserve(graph.numBlocks())) {
+ return false;
+ }
+
+ for (size_t i = 0; i < graph.numBlocks(); i++) {
+ LBlock* block = graph.getBlock(i);
+ for (LInstructionIterator ins = block->begin(); ins != block->end();
+ ins++) {
+ insData[ins->id()] = *ins;
+ }
+ for (size_t j = 0; j < block->numPhis(); j++) {
+ LPhi* phi = block->getPhi(j);
+ insData[phi->id()] = phi;
+ }
+
+ CodePosition entry =
+ block->numPhis() != 0
+ ? CodePosition(block->getPhi(0)->id(), CodePosition::INPUT)
+ : inputOf(block->firstInstructionWithId());
+ CodePosition exit = outputOf(block->lastInstructionWithId());
+
+ MOZ_ASSERT(block->mir()->id() == i);
+ entryPositions.infallibleAppend(entry);
+ exitPositions.infallibleAppend(exit);
+ }
+
+ return true;
+}
+
+LMoveGroup* RegisterAllocator::getInputMoveGroup(LInstruction* ins) {
+ MOZ_ASSERT(!ins->fixReuseMoves());
+ if (ins->inputMoves()) {
+ return ins->inputMoves();
+ }
+
+ LMoveGroup* moves = LMoveGroup::New(alloc());
+ ins->setInputMoves(moves);
+ ins->block()->insertBefore(ins, moves);
+ return moves;
+}
+
+LMoveGroup* RegisterAllocator::getFixReuseMoveGroup(LInstruction* ins) {
+ if (ins->fixReuseMoves()) {
+ return ins->fixReuseMoves();
+ }
+
+ LMoveGroup* moves = LMoveGroup::New(alloc());
+ ins->setFixReuseMoves(moves);
+ ins->block()->insertBefore(ins, moves);
+ return moves;
+}
+
+LMoveGroup* RegisterAllocator::getMoveGroupAfter(LInstruction* ins) {
+ if (ins->movesAfter()) {
+ return ins->movesAfter();
+ }
+
+ LMoveGroup* moves = LMoveGroup::New(alloc());
+ ins->setMovesAfter(moves);
+
+ ins->block()->insertAfter(ins, moves);
+ return moves;
+}
+
+void RegisterAllocator::dumpInstructions(const char* who) {
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_RegAlloc, "LIR instructions %s", who);
+
+ for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); blockIndex++) {
+ LBlock* block = graph.getBlock(blockIndex);
+ MBasicBlock* mir = block->mir();
+
+ JitSpewHeader(JitSpew_RegAlloc);
+ JitSpewCont(JitSpew_RegAlloc, " Block %lu",
+ static_cast<unsigned long>(blockIndex));
+ for (size_t i = 0; i < mir->numSuccessors(); i++) {
+ JitSpewCont(JitSpew_RegAlloc, " [successor %u]",
+ mir->getSuccessor(i)->id());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+
+ for (size_t i = 0; i < block->numPhis(); i++) {
+ LPhi* phi = block->getPhi(i);
+
+ JitSpewHeader(JitSpew_RegAlloc);
+ JitSpewCont(JitSpew_RegAlloc, " %u-%u Phi [def %s]",
+ inputOf(phi).bits(), outputOf(phi).bits(),
+ phi->getDef(0)->toString().get());
+ for (size_t j = 0; j < phi->numOperands(); j++) {
+ JitSpewCont(JitSpew_RegAlloc, " [use %s]",
+ phi->getOperand(j)->toString().get());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ }
+
+ for (LInstructionIterator iter = block->begin(); iter != block->end();
+ iter++) {
+ LInstruction* ins = *iter;
+
+ JitSpewHeader(JitSpew_RegAlloc);
+ JitSpewCont(JitSpew_RegAlloc, " ");
+ if (ins->id() != 0) {
+ JitSpewCont(JitSpew_RegAlloc, "%u-%u ", inputOf(ins).bits(),
+ outputOf(ins).bits());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "%s", ins->opName());
+
+ if (ins->isMoveGroup()) {
+ LMoveGroup* group = ins->toMoveGroup();
+ for (int i = group->numMoves() - 1; i >= 0; i--) {
+ // Use two printfs, as LAllocation::toString is not reentant.
+ JitSpewCont(JitSpew_RegAlloc, " [%s",
+ group->getMove(i).to().toString().get());
+ JitSpewCont(JitSpew_RegAlloc, " <- %s]",
+ group->getMove(i).from().toString().get());
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ continue;
+ }
+
+ for (size_t i = 0; i < ins->numDefs(); i++) {
+ JitSpewCont(JitSpew_RegAlloc, " [def %s]",
+ ins->getDef(i)->toString().get());
+ }
+
+ for (size_t i = 0; i < ins->numTemps(); i++) {
+ LDefinition* temp = ins->getTemp(i);
+ if (!temp->isBogusTemp()) {
+ JitSpewCont(JitSpew_RegAlloc, " [temp %s]", temp->toString().get());
+ }
+ }
+
+ for (LInstruction::InputIterator alloc(*ins); alloc.more();
+ alloc.next()) {
+ if (!alloc->isBogus()) {
+ JitSpewCont(JitSpew_RegAlloc, " [use %s]", alloc->toString().get());
+ }
+ }
+
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+ }
+ }
+ JitSpewCont(JitSpew_RegAlloc, "\n");
+#endif // JS_JITSPEW
+}
diff --git a/js/src/jit/RegisterAllocator.h b/js/src/jit/RegisterAllocator.h
new file mode 100644
index 0000000000..b2ce66bdde
--- /dev/null
+++ b/js/src/jit/RegisterAllocator.h
@@ -0,0 +1,313 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_RegisterAllocator_h
+#define jit_RegisterAllocator_h
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/LIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+// Generic structures and functions for use by register allocators.
+
+namespace js {
+namespace jit {
+
+class LIRGenerator;
+
+#ifdef DEBUG
+// Structure for running a liveness analysis on a finished register allocation.
+// This analysis can be used for two purposes:
+//
+// - Check the integrity of the allocation, i.e. that the reads and writes of
+// physical values preserve the semantics of the original virtual registers.
+//
+// - Populate safepoints with live registers, GC thing and value data, to
+// streamline the process of prototyping new allocators.
+struct AllocationIntegrityState {
+ explicit AllocationIntegrityState(LIRGraph& graph) : graph(graph) {}
+
+ // Record all virtual registers in the graph. This must be called before
+ // register allocation, to pick up the original LUses.
+ [[nodiscard]] bool record();
+
+ // Perform the liveness analysis on the graph, and assert on an invalid
+ // allocation. This must be called after register allocation, to pick up
+ // all assigned physical values.
+ [[nodiscard]] bool check();
+
+ private:
+ LIRGraph& graph;
+
+ // For all instructions and phis in the graph, keep track of the virtual
+ // registers for all inputs and outputs of the nodes. These are overwritten
+ // in place during register allocation. This information is kept on the
+ // side rather than in the instructions and phis themselves to avoid
+ // debug-builds-only bloat in the size of the involved structures.
+
+ struct InstructionInfo {
+ Vector<LAllocation, 2, SystemAllocPolicy> inputs;
+ Vector<LDefinition, 0, SystemAllocPolicy> temps;
+ Vector<LDefinition, 1, SystemAllocPolicy> outputs;
+
+ InstructionInfo() = default;
+
+ InstructionInfo(const InstructionInfo& o) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!inputs.appendAll(o.inputs) || !temps.appendAll(o.temps) ||
+ !outputs.appendAll(o.outputs)) {
+ oomUnsafe.crash("InstructionInfo::InstructionInfo");
+ }
+ }
+ };
+ Vector<InstructionInfo, 0, SystemAllocPolicy> instructions;
+
+ struct BlockInfo {
+ Vector<InstructionInfo, 5, SystemAllocPolicy> phis;
+ BlockInfo() = default;
+ BlockInfo(const BlockInfo& o) {
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!phis.appendAll(o.phis)) {
+ oomUnsafe.crash("BlockInfo::BlockInfo");
+ }
+ }
+ };
+ Vector<BlockInfo, 0, SystemAllocPolicy> blocks;
+
+ Vector<LDefinition*, 20, SystemAllocPolicy> virtualRegisters;
+
+ // Describes a correspondence that should hold at the end of a block.
+ // The value which was written to vreg in the original LIR should be
+ // physically stored in alloc after the register allocation.
+ struct IntegrityItem {
+ LBlock* block;
+ uint32_t vreg;
+ LAllocation alloc;
+
+ // Order of insertion into seen, for sorting.
+ uint32_t index;
+
+ using Lookup = IntegrityItem;
+ static HashNumber hash(const IntegrityItem& item) {
+ HashNumber hash = item.alloc.hash();
+ hash = mozilla::RotateLeft(hash, 4) ^ item.vreg;
+ hash = mozilla::RotateLeft(hash, 4) ^ HashNumber(item.block->mir()->id());
+ return hash;
+ }
+ static bool match(const IntegrityItem& one, const IntegrityItem& two) {
+ return one.block == two.block && one.vreg == two.vreg &&
+ one.alloc == two.alloc;
+ }
+ };
+
+ // Items still to be processed.
+ Vector<IntegrityItem, 10, SystemAllocPolicy> worklist;
+
+ // Set of all items that have already been processed.
+ typedef HashSet<IntegrityItem, IntegrityItem, SystemAllocPolicy>
+ IntegrityItemSet;
+ IntegrityItemSet seen;
+
+ [[nodiscard]] bool checkIntegrity(LBlock* block, LInstruction* ins,
+ uint32_t vreg, LAllocation alloc);
+ void checkSafepointAllocation(LInstruction* ins, uint32_t vreg,
+ LAllocation alloc);
+ [[nodiscard]] bool addPredecessor(LBlock* block, uint32_t vreg,
+ LAllocation alloc);
+
+ void dump();
+};
+#endif // DEBUG
+
+// Represents with better-than-instruction precision a position in the
+// instruction stream.
+//
+// An issue comes up when performing register allocation as to how to represent
+// information such as "this register is only needed for the input of
+// this instruction, it can be clobbered in the output". Just having ranges
+// of instruction IDs is insufficiently expressive to denote all possibilities.
+// This class solves this issue by associating an extra bit with the instruction
+// ID which indicates whether the position is the input half or output half of
+// an instruction.
+class CodePosition {
+ private:
+ constexpr explicit CodePosition(uint32_t bits) : bits_(bits) {}
+
+ static const unsigned int INSTRUCTION_SHIFT = 1;
+ static const unsigned int SUBPOSITION_MASK = 1;
+ uint32_t bits_;
+
+ public:
+ static const CodePosition MAX;
+ static const CodePosition MIN;
+
+ // This is the half of the instruction this code position represents, as
+ // described in the huge comment above.
+ enum SubPosition { INPUT, OUTPUT };
+
+ constexpr CodePosition() : bits_(0) {}
+
+ CodePosition(uint32_t instruction, SubPosition where) {
+ MOZ_ASSERT(instruction < 0x80000000u);
+ MOZ_ASSERT(((uint32_t)where & SUBPOSITION_MASK) == (uint32_t)where);
+ bits_ = (instruction << INSTRUCTION_SHIFT) | (uint32_t)where;
+ }
+
+ uint32_t ins() const { return bits_ >> INSTRUCTION_SHIFT; }
+
+ uint32_t bits() const { return bits_; }
+
+ SubPosition subpos() const { return (SubPosition)(bits_ & SUBPOSITION_MASK); }
+
+ bool operator<(CodePosition other) const { return bits_ < other.bits_; }
+
+ bool operator<=(CodePosition other) const { return bits_ <= other.bits_; }
+
+ bool operator!=(CodePosition other) const { return bits_ != other.bits_; }
+
+ bool operator==(CodePosition other) const { return bits_ == other.bits_; }
+
+ bool operator>(CodePosition other) const { return bits_ > other.bits_; }
+
+ bool operator>=(CodePosition other) const { return bits_ >= other.bits_; }
+
+ uint32_t operator-(CodePosition other) const {
+ MOZ_ASSERT(bits_ >= other.bits_);
+ return bits_ - other.bits_;
+ }
+
+ CodePosition previous() const {
+ MOZ_ASSERT(*this != MIN);
+ return CodePosition(bits_ - 1);
+ }
+ CodePosition next() const {
+ MOZ_ASSERT(*this != MAX);
+ return CodePosition(bits_ + 1);
+ }
+};
+
+// Structure to track all moves inserted next to instructions in a graph.
+class InstructionDataMap {
+ FixedList<LNode*> insData_;
+
+ public:
+ InstructionDataMap() : insData_() {}
+
+ [[nodiscard]] bool init(MIRGenerator* gen, uint32_t numInstructions) {
+ if (!insData_.init(gen->alloc(), numInstructions)) {
+ return false;
+ }
+ memset(&insData_[0], 0, sizeof(LNode*) * numInstructions);
+ return true;
+ }
+
+ LNode*& operator[](CodePosition pos) { return operator[](pos.ins()); }
+ LNode* const& operator[](CodePosition pos) const {
+ return operator[](pos.ins());
+ }
+ LNode*& operator[](uint32_t ins) { return insData_[ins]; }
+ LNode* const& operator[](uint32_t ins) const { return insData_[ins]; }
+};
+
+// Common superclass for register allocators.
+class RegisterAllocator {
+ void operator=(const RegisterAllocator&) = delete;
+ RegisterAllocator(const RegisterAllocator&) = delete;
+
+ protected:
+ // Context
+ MIRGenerator* mir;
+ LIRGenerator* lir;
+ LIRGraph& graph;
+
+ // Pool of all registers that should be considered allocateable
+ AllocatableRegisterSet allRegisters_;
+
+ // Computed data
+ InstructionDataMap insData;
+ Vector<CodePosition, 12, SystemAllocPolicy> entryPositions;
+ Vector<CodePosition, 12, SystemAllocPolicy> exitPositions;
+
+ RegisterAllocator(MIRGenerator* mir, LIRGenerator* lir, LIRGraph& graph)
+ : mir(mir), lir(lir), graph(graph), allRegisters_(RegisterSet::All()) {
+ MOZ_ASSERT(!allRegisters_.has(FramePointer));
+ if (mir->compilingWasm()) {
+ takeWasmRegisters(allRegisters_);
+ }
+ }
+
+ [[nodiscard]] bool init();
+
+ TempAllocator& alloc() const { return mir->alloc(); }
+
+ CodePosition outputOf(const LNode* ins) const {
+ return ins->isPhi() ? outputOf(ins->toPhi())
+ : outputOf(ins->toInstruction());
+ }
+ CodePosition outputOf(const LPhi* ins) const {
+ // All phis in a block write their outputs after all of them have
+ // read their inputs. Consequently, it doesn't make sense to talk
+ // about code positions in the middle of a series of phis.
+ LBlock* block = ins->block();
+ return CodePosition(block->getPhi(block->numPhis() - 1)->id(),
+ CodePosition::OUTPUT);
+ }
+ CodePosition outputOf(const LInstruction* ins) const {
+ return CodePosition(ins->id(), CodePosition::OUTPUT);
+ }
+ CodePosition inputOf(const LNode* ins) const {
+ return ins->isPhi() ? inputOf(ins->toPhi()) : inputOf(ins->toInstruction());
+ }
+ CodePosition inputOf(const LPhi* ins) const {
+ // All phis in a block read their inputs before any of them write their
+ // outputs. Consequently, it doesn't make sense to talk about code
+ // positions in the middle of a series of phis.
+ return CodePosition(ins->block()->getPhi(0)->id(), CodePosition::INPUT);
+ }
+ CodePosition inputOf(const LInstruction* ins) const {
+ return CodePosition(ins->id(), CodePosition::INPUT);
+ }
+ CodePosition entryOf(const LBlock* block) {
+ return entryPositions[block->mir()->id()];
+ }
+ CodePosition exitOf(const LBlock* block) {
+ return exitPositions[block->mir()->id()];
+ }
+
+ LMoveGroup* getInputMoveGroup(LInstruction* ins);
+ LMoveGroup* getFixReuseMoveGroup(LInstruction* ins);
+ LMoveGroup* getMoveGroupAfter(LInstruction* ins);
+
+ // Atomic group helper. See comments in BacktrackingAllocator.cpp.
+ CodePosition minimalDefEnd(LNode* ins) const;
+
+ void dumpInstructions(const char* who);
+
+ public:
+ template <typename TakeableSet>
+ static void takeWasmRegisters(TakeableSet& regs) {
+#if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_ARM) || \
+ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_MIPS32) || \
+ defined(JS_CODEGEN_MIPS64) || defined(JS_CODEGEN_LOONG64)
+ regs.take(HeapReg);
+#endif
+ MOZ_ASSERT(!regs.has(FramePointer));
+ }
+};
+
+static inline AnyRegister GetFixedRegister(const LDefinition* def,
+ const LUse* use) {
+ return def->isFloatReg()
+ ? AnyRegister(FloatRegister::FromCode(use->registerCode()))
+ : AnyRegister(Register::FromCode(use->registerCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_RegisterAllocator_h */
diff --git a/js/src/jit/RegisterSets.h b/js/src/jit/RegisterSets.h
new file mode 100644
index 0000000000..4212a2d682
--- /dev/null
+++ b/js/src/jit/RegisterSets.h
@@ -0,0 +1,1333 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_RegisterSets_h
+#define jit_RegisterSets_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Variant.h"
+
+#include <new>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/IonTypes.h"
+#include "jit/Registers.h"
+#include "js/Value.h"
+
+namespace js {
+namespace jit {
+
+struct AnyRegister {
+ using Code = uint8_t;
+
+ static const uint8_t Total = Registers::Total + FloatRegisters::Total;
+ static const uint8_t FirstFloatReg = Registers::Total;
+ static const uint8_t Invalid = UINT8_MAX;
+
+ static_assert(size_t(Registers::Total) + FloatRegisters::Total <= UINT8_MAX,
+ "Number of registers must fit in uint8_t");
+
+ private:
+ Code code_;
+
+ public:
+ AnyRegister() : code_(Invalid) {}
+
+ explicit AnyRegister(Register gpr) { code_ = gpr.code(); }
+ explicit AnyRegister(FloatRegister fpu) {
+ code_ = fpu.code() + Registers::Total;
+ }
+ static AnyRegister FromCode(uint8_t i) {
+ MOZ_ASSERT(i < Total);
+ AnyRegister r;
+ r.code_ = i;
+ return r;
+ }
+ bool isFloat() const {
+ MOZ_ASSERT(isValid());
+ return code_ >= Registers::Total;
+ }
+ Register gpr() const {
+ MOZ_ASSERT(!isFloat());
+ return Register::FromCode(code_);
+ }
+ FloatRegister fpu() const {
+ MOZ_ASSERT(isFloat());
+ return FloatRegister::FromCode(code_ - Registers::Total);
+ }
+ bool operator==(AnyRegister other) const {
+ // We don't need the operands to be valid to test for equality.
+ return code_ == other.code_;
+ }
+ bool operator!=(AnyRegister other) const {
+ // We don't need the operands to be valid to test for equality.
+ return code_ != other.code_;
+ }
+ const char* name() const { return isFloat() ? fpu().name() : gpr().name(); }
+ Code code() const {
+ MOZ_ASSERT(isValid());
+ return code_;
+ }
+ bool volatile_() const {
+ return isFloat() ? fpu().volatile_() : gpr().volatile_();
+ }
+ AnyRegister aliased(uint8_t aliasIdx) const {
+ AnyRegister ret;
+ if (isFloat()) {
+ ret = AnyRegister(fpu().aliased(aliasIdx));
+ } else {
+ ret = AnyRegister(gpr().aliased(aliasIdx));
+ }
+ MOZ_ASSERT_IF(aliasIdx == 0, ret == *this);
+ return ret;
+ }
+ uint8_t numAliased() const {
+ if (isFloat()) {
+ return fpu().numAliased();
+ }
+ return gpr().numAliased();
+ }
+ bool aliases(const AnyRegister& other) const {
+ if (isFloat() && other.isFloat()) {
+ return fpu().aliases(other.fpu());
+ }
+ if (!isFloat() && !other.isFloat()) {
+ return gpr().aliases(other.gpr());
+ }
+ return false;
+ }
+ // do the two registers hold the same type of data (e.g. both float32, both
+ // gpr)
+ bool isCompatibleReg(const AnyRegister other) const {
+ if (isFloat() && other.isFloat()) {
+ return fpu().equiv(other.fpu());
+ }
+ if (!isFloat() && !other.isFloat()) {
+ return true;
+ }
+ return false;
+ }
+ bool isValid() const { return code_ != Invalid; }
+};
+
+// Registers to hold a boxed value. Uses one register on 64 bit
+// platforms, two registers on 32 bit platforms.
+class ValueOperand {
+#if defined(JS_NUNBOX32)
+ Register type_;
+ Register payload_;
+
+ public:
+ constexpr ValueOperand(Register type, Register payload)
+ : type_(type), payload_(payload) {}
+
+ constexpr Register typeReg() const { return type_; }
+ constexpr Register payloadReg() const { return payload_; }
+ constexpr Register64 toRegister64() const {
+ return Register64(typeReg(), payloadReg());
+ }
+ constexpr bool aliases(Register reg) const {
+ return type_ == reg || payload_ == reg;
+ }
+ constexpr Register payloadOrValueReg() const { return payloadReg(); }
+ bool hasVolatileReg() const {
+ return type_.volatile_() || payload_.volatile_();
+ }
+ constexpr bool operator==(const ValueOperand& o) const {
+ return type_ == o.type_ && payload_ == o.payload_;
+ }
+ constexpr bool operator!=(const ValueOperand& o) const {
+ return !(*this == o);
+ }
+
+#elif defined(JS_PUNBOX64)
+ Register value_;
+
+ public:
+ explicit constexpr ValueOperand(Register value) : value_(value) {}
+
+ constexpr Register valueReg() const { return value_; }
+ constexpr Register64 toRegister64() const { return Register64(valueReg()); }
+ constexpr bool aliases(Register reg) const { return value_ == reg; }
+ constexpr Register payloadOrValueReg() const { return valueReg(); }
+ bool hasVolatileReg() const { return value_.volatile_(); }
+ constexpr bool operator==(const ValueOperand& o) const {
+ return value_ == o.value_;
+ }
+ constexpr bool operator!=(const ValueOperand& o) const {
+ return !(*this == o);
+ }
+#endif
+
+ constexpr Register scratchReg() const { return payloadOrValueReg(); }
+
+ ValueOperand() = default;
+};
+
+// Registers to hold either either a typed or untyped value.
+class TypedOrValueRegister {
+ // Type of value being stored.
+ MIRType type_;
+
+ union U {
+ AnyRegister::Code typed;
+#if defined(JS_PUNBOX64)
+ Register::Code value;
+#elif defined(JS_NUNBOX32)
+ struct {
+ Register::Code valueType;
+ Register::Code valuePayload;
+ } s;
+#else
+# error "Bad architecture"
+#endif
+ } data;
+
+ public:
+ TypedOrValueRegister() = default;
+
+ TypedOrValueRegister(MIRType type, AnyRegister reg) : type_(type) {
+ data.typed = reg.code();
+ }
+
+ MOZ_IMPLICIT TypedOrValueRegister(ValueOperand value)
+ : type_(MIRType::Value) {
+#if defined(JS_PUNBOX64)
+ data.value = value.valueReg().code();
+#elif defined(JS_NUNBOX32)
+ data.s.valueType = value.typeReg().code();
+ data.s.valuePayload = value.payloadReg().code();
+#else
+# error "Bad architecture"
+#endif
+ }
+
+ MIRType type() const { return type_; }
+
+ bool hasTyped() const {
+ return type() != MIRType::None && type() != MIRType::Value;
+ }
+
+ bool hasValue() const { return type() == MIRType::Value; }
+
+ AnyRegister typedReg() const {
+ MOZ_ASSERT(hasTyped());
+ return AnyRegister::FromCode(data.typed);
+ }
+
+ ValueOperand valueReg() const {
+ MOZ_ASSERT(hasValue());
+#if defined(JS_PUNBOX64)
+ return ValueOperand(Register::FromCode(data.value));
+#elif defined(JS_NUNBOX32)
+ return ValueOperand(Register::FromCode(data.s.valueType),
+ Register::FromCode(data.s.valuePayload));
+#else
+# error "Bad architecture"
+#endif
+ }
+
+ AnyRegister scratchReg() {
+ if (hasValue()) {
+ return AnyRegister(valueReg().scratchReg());
+ }
+ return typedReg();
+ }
+};
+
+// A constant value, or registers to hold a typed/untyped value.
+class ConstantOrRegister {
+ // Whether a constant value is being stored.
+ bool constant_;
+
+ // Space to hold either a Value or a TypedOrValueRegister.
+ union U {
+ JS::Value constant;
+ TypedOrValueRegister reg;
+
+ // |constant| has a non-trivial constructor and therefore MUST be
+ // placement-new'd into existence.
+ MOZ_PUSH_DISABLE_NONTRIVIAL_UNION_WARNINGS
+ U() {}
+ MOZ_POP_DISABLE_NONTRIVIAL_UNION_WARNINGS
+ } data;
+
+ public:
+ ConstantOrRegister() = delete;
+
+ MOZ_IMPLICIT ConstantOrRegister(const JS::Value& value) : constant_(true) {
+ MOZ_ASSERT(constant());
+ new (&data.constant) JS::Value(value);
+ }
+
+ MOZ_IMPLICIT ConstantOrRegister(TypedOrValueRegister reg) : constant_(false) {
+ MOZ_ASSERT(!constant());
+ new (&data.reg) TypedOrValueRegister(reg);
+ }
+
+ bool constant() const { return constant_; }
+
+ JS::Value value() const {
+ MOZ_ASSERT(constant());
+ return data.constant;
+ }
+
+ const TypedOrValueRegister& reg() const {
+ MOZ_ASSERT(!constant());
+ return data.reg;
+ }
+};
+
+template <typename T>
+class TypedRegisterSet {
+ public:
+ using RegType = T;
+ using SetType = typename T::SetType;
+
+ private:
+ SetType bits_;
+
+ public:
+ explicit constexpr TypedRegisterSet(SetType bits) : bits_(bits) {}
+
+ constexpr TypedRegisterSet() : bits_(0) {}
+ constexpr TypedRegisterSet(const TypedRegisterSet<T>& set)
+ : bits_(set.bits_) {}
+
+ static inline TypedRegisterSet All() {
+ return TypedRegisterSet(T::Codes::AllocatableMask);
+ }
+ static inline TypedRegisterSet Intersect(const TypedRegisterSet& lhs,
+ const TypedRegisterSet& rhs) {
+ return TypedRegisterSet(lhs.bits_ & rhs.bits_);
+ }
+ static inline TypedRegisterSet Union(const TypedRegisterSet& lhs,
+ const TypedRegisterSet& rhs) {
+ return TypedRegisterSet(lhs.bits_ | rhs.bits_);
+ }
+ static inline TypedRegisterSet Not(const TypedRegisterSet& in) {
+ return TypedRegisterSet(~in.bits_ & T::Codes::AllocatableMask);
+ }
+ static inline TypedRegisterSet Subtract(const TypedRegisterSet& lhs,
+ const TypedRegisterSet& rhs) {
+ return TypedRegisterSet(lhs.bits_ & ~rhs.bits_);
+ }
+ static inline TypedRegisterSet VolatileNot(const TypedRegisterSet& in) {
+ const SetType allocatableVolatile =
+ T::Codes::AllocatableMask & T::Codes::VolatileMask;
+ return TypedRegisterSet(~in.bits_ & allocatableVolatile);
+ }
+ static inline TypedRegisterSet Volatile() {
+ return TypedRegisterSet(T::Codes::AllocatableMask & T::Codes::VolatileMask);
+ }
+ static inline TypedRegisterSet NonVolatile() {
+ return TypedRegisterSet(T::Codes::AllocatableMask &
+ T::Codes::NonVolatileMask);
+ }
+
+ bool empty() const { return !bits_; }
+ void clear() { bits_ = 0; }
+
+ bool hasRegisterIndex(T reg) const {
+ return !!(bits_ & (SetType(1) << reg.code()));
+ }
+ bool hasAllocatable(T reg) const {
+ return !(~bits_ & reg.alignedOrDominatedAliasedSet());
+ }
+
+ void addRegisterIndex(T reg) { bits_ |= (SetType(1) << reg.code()); }
+ void addAllocatable(T reg) { bits_ |= reg.alignedOrDominatedAliasedSet(); }
+
+ void takeRegisterIndex(T reg) { bits_ &= ~(SetType(1) << reg.code()); }
+ void takeAllocatable(T reg) { bits_ &= ~reg.alignedOrDominatedAliasedSet(); }
+
+ static constexpr RegTypeName DefaultType = RegType::DefaultType;
+
+ template <RegTypeName Name>
+ SetType allLive() const {
+ return T::template LiveAsIndexableSet<Name>(bits_);
+ }
+ template <RegTypeName Name>
+ SetType allAllocatable() const {
+ return T::template AllocatableAsIndexableSet<Name>(bits_);
+ }
+
+ static RegType FirstRegister(SetType set) {
+ return RegType::FromCode(RegType::FirstBit(set));
+ }
+ static RegType LastRegister(SetType set) {
+ return RegType::FromCode(RegType::LastBit(set));
+ }
+
+ SetType bits() const { return bits_; }
+ uint32_t size() const { return T::SetSize(bits_); }
+ bool operator==(const TypedRegisterSet<T>& other) const {
+ return other.bits_ == bits_;
+ }
+ TypedRegisterSet<T> reduceSetForPush() const {
+ return T::ReduceSetForPush(*this);
+ }
+ uint32_t getPushSizeInBytes() const { return T::GetPushSizeInBytes(*this); }
+
+ size_t offsetOfPushedRegister(RegType reg) const {
+ MOZ_ASSERT(hasRegisterIndex(reg));
+ return T::OffsetOfPushedRegister(bits(), reg);
+ }
+};
+
+using GeneralRegisterSet = TypedRegisterSet<Register>;
+using FloatRegisterSet = TypedRegisterSet<FloatRegister>;
+
+class AnyRegisterIterator;
+
+class RegisterSet {
+ GeneralRegisterSet gpr_;
+ FloatRegisterSet fpu_;
+
+ friend class AnyRegisterIterator;
+
+ public:
+ RegisterSet() = default;
+ constexpr RegisterSet(const GeneralRegisterSet& gpr,
+ const FloatRegisterSet& fpu)
+ : gpr_(gpr), fpu_(fpu) {}
+ static inline RegisterSet All() {
+ return RegisterSet(GeneralRegisterSet::All(), FloatRegisterSet::All());
+ }
+ static inline RegisterSet Intersect(const RegisterSet& lhs,
+ const RegisterSet& rhs) {
+ return RegisterSet(GeneralRegisterSet::Intersect(lhs.gpr_, rhs.gpr_),
+ FloatRegisterSet::Intersect(lhs.fpu_, rhs.fpu_));
+ }
+ static inline RegisterSet Union(const RegisterSet& lhs,
+ const RegisterSet& rhs) {
+ return RegisterSet(GeneralRegisterSet::Union(lhs.gpr_, rhs.gpr_),
+ FloatRegisterSet::Union(lhs.fpu_, rhs.fpu_));
+ }
+ static inline RegisterSet Not(const RegisterSet& in) {
+ return RegisterSet(GeneralRegisterSet::Not(in.gpr_),
+ FloatRegisterSet::Not(in.fpu_));
+ }
+ static inline RegisterSet VolatileNot(const RegisterSet& in) {
+ return RegisterSet(GeneralRegisterSet::VolatileNot(in.gpr_),
+ FloatRegisterSet::VolatileNot(in.fpu_));
+ }
+ static inline RegisterSet Volatile() {
+ return RegisterSet(GeneralRegisterSet::Volatile(),
+ FloatRegisterSet::Volatile());
+ }
+
+ bool empty() const { return fpu_.empty() && gpr_.empty(); }
+ void clear() {
+ fpu_.clear();
+ gpr_.clear();
+ }
+ bool emptyGeneral() const { return gpr_.empty(); }
+ bool emptyFloat() const { return fpu_.empty(); }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::GPR;
+
+ constexpr GeneralRegisterSet gprs() const { return gpr_; }
+ GeneralRegisterSet& gprs() { return gpr_; }
+ constexpr FloatRegisterSet fpus() const { return fpu_; }
+ FloatRegisterSet& fpus() { return fpu_; }
+ bool operator==(const RegisterSet& other) const {
+ return other.gpr_ == gpr_ && other.fpu_ == fpu_;
+ }
+};
+
+// [SMDOC] JIT Register-Set overview
+//
+// There are 2 use cases for register sets:
+//
+// 1. To serve as a pool of allocatable register. This is useful for working
+// on the code produced by some stub where free registers are available, or
+// when we can release some registers.
+//
+// 2. To serve as a list of typed registers. This is useful for working with
+// live registers and to manipulate them with the proper instructions. This
+// is used by the register allocator to fill the Safepoints.
+//
+// These 2 uses cases can be used on top of 3 different backend representation
+// of register sets, which are either GeneralRegisterSet, FloatRegisterSet, or
+// RegisterSet (for both). These classes are used to store the bit sets to
+// represent each register.
+//
+// Each use case defines an Accessor class, such as AllocatableSetAccessor or
+// LiveSetAccessor, which is parameterized with the type of the register
+// set. These accessors are in charge of manipulating the register set in a
+// consistent way.
+//
+// The RegSetCommonInterface class is used to wrap the accessors with convenient
+// shortcuts which are based on the accessors.
+//
+// Then, to avoid to many levels of complexity while using these interfaces,
+// shortcut templates are created to make it easy to distinguish between a
+// register set used for allocating registers, or a register set used for making
+// a collection of allocated (live) registers.
+//
+// This separation exists to prevent mixing LiveSet and AllocatableSet
+// manipulations of the same register set, and ensure safety while avoiding
+// false positive.
+
+template <typename RegisterSet>
+class AllocatableSet;
+
+template <typename RegisterSet>
+class LiveSet;
+
+// [SMDOC] JIT Register-Set (Allocatable)
+//
+// Base accessors classes have the minimal set of raw methods to manipulate the
+// register set given as parameter in a consistent manner. These methods are:
+//
+// - all<Type>: Returns a bit-set of all the register of a specific type
+// which are present.
+//
+// - has: Returns if all the bits needed to take a register are present.
+//
+// - takeUnchecked: Subtracts the bits used to represent the register in the
+// register set.
+//
+// - addUnchecked: Adds the bits used to represent the register in the
+// register set.
+
+// The AllocatableSet accessors are used to make a pool of unused
+// registers. Taking or adding registers should consider the aliasing rules of
+// the architecture. For example, on ARM, the following piece of code should
+// work fine, knowing that the double register |d0| is composed of float
+// registers |s0| and |s1|:
+//
+// AllocatableFloatRegisterSet regs;
+// regs.add(s0);
+// regs.add(s1);
+// // d0 is now available.
+// regs.take(d0);
+//
+// These accessors are useful for allocating registers within the functions used
+// to generate stubs, trampolines, and inline caches (BaselineIC, IonCache).
+template <typename Set>
+class AllocatableSetAccessors {
+ public:
+ using RegSet = Set;
+ using RegType = typename RegSet::RegType;
+ using SetType = typename RegSet::SetType;
+
+ protected:
+ RegSet set_;
+
+ template <RegTypeName Name>
+ SetType all() const {
+ return set_.template allAllocatable<Name>();
+ }
+
+ public:
+ AllocatableSetAccessors() : set_() {}
+ explicit constexpr AllocatableSetAccessors(SetType set) : set_(set) {}
+ explicit constexpr AllocatableSetAccessors(RegSet set) : set_(set) {}
+
+ bool has(RegType reg) const { return set_.hasAllocatable(reg); }
+
+ template <RegTypeName Name>
+ bool hasAny(RegType reg) const {
+ return all<Name>() != 0;
+ }
+
+ void addUnchecked(RegType reg) { set_.addAllocatable(reg); }
+
+ void takeUnchecked(RegType reg) { set_.takeAllocatable(reg); }
+};
+
+// Specialization of the AllocatableSet accessors for the RegisterSet aggregate.
+template <>
+class AllocatableSetAccessors<RegisterSet> {
+ public:
+ using RegSet = RegisterSet;
+ using RegType = AnyRegister;
+ using SetType = char;
+
+ protected:
+ RegisterSet set_;
+
+ template <RegTypeName Name>
+ GeneralRegisterSet::SetType allGpr() const {
+ return set_.gprs().allAllocatable<Name>();
+ }
+ template <RegTypeName Name>
+ FloatRegisterSet::SetType allFpu() const {
+ return set_.fpus().allAllocatable<Name>();
+ }
+
+ public:
+ AllocatableSetAccessors() : set_() {}
+ explicit constexpr AllocatableSetAccessors(SetType) = delete;
+ explicit constexpr AllocatableSetAccessors(RegisterSet set) : set_(set) {}
+
+ bool has(Register reg) const { return set_.gprs().hasAllocatable(reg); }
+ bool has(FloatRegister reg) const { return set_.fpus().hasAllocatable(reg); }
+
+ void addUnchecked(Register reg) { set_.gprs().addAllocatable(reg); }
+ void addUnchecked(FloatRegister reg) { set_.fpus().addAllocatable(reg); }
+
+ void takeUnchecked(Register reg) { set_.gprs().takeAllocatable(reg); }
+ void takeUnchecked(FloatRegister reg) { set_.fpus().takeAllocatable(reg); }
+};
+
+// [SMDOC] JIT Register-Set (Live)
+//
+// The LiveSet accessors are used to collect a list of allocated
+// registers. Taking or adding a register should *not* consider the aliases, as
+// we care about interpreting the registers with the correct type. For example,
+// on x64, where one float registers can be interpreted as an Simd128, a Double,
+// or a Float, adding xmm0 as an Simd128, does not make the register available
+// as a Double.
+//
+// LiveFloatRegisterSet regs;
+// regs.add(xmm0.asSimd128());
+// regs.take(xmm0); // Assert!
+//
+// These accessors are useful for recording the result of a register allocator,
+// such as what the Backtracking allocator do on the Safepoints.
+template <typename Set>
+class LiveSetAccessors {
+ public:
+ using RegSet = Set;
+ using RegType = typename RegSet::RegType;
+ using SetType = typename RegSet::SetType;
+
+ protected:
+ RegSet set_;
+
+ template <RegTypeName Name>
+ SetType all() const {
+ return set_.template allLive<Name>();
+ }
+
+ public:
+ LiveSetAccessors() : set_() {}
+ explicit constexpr LiveSetAccessors(SetType set) : set_(set) {}
+ explicit constexpr LiveSetAccessors(RegSet set) : set_(set) {}
+
+ bool has(RegType reg) const { return set_.hasRegisterIndex(reg); }
+
+ void addUnchecked(RegType reg) { set_.addRegisterIndex(reg); }
+
+ void takeUnchecked(RegType reg) { set_.takeRegisterIndex(reg); }
+};
+
+// Specialization of the LiveSet accessors for the RegisterSet aggregate.
+template <>
+class LiveSetAccessors<RegisterSet> {
+ public:
+ using RegSet = RegisterSet;
+ using RegType = AnyRegister;
+ using SetType = char;
+
+ protected:
+ RegisterSet set_;
+
+ template <RegTypeName Name>
+ GeneralRegisterSet::SetType allGpr() const {
+ return set_.gprs().allLive<Name>();
+ }
+ template <RegTypeName Name>
+ FloatRegisterSet::SetType allFpu() const {
+ return set_.fpus().allLive<Name>();
+ }
+
+ public:
+ LiveSetAccessors() : set_() {}
+ explicit constexpr LiveSetAccessors(SetType) = delete;
+ explicit constexpr LiveSetAccessors(RegisterSet set) : set_(set) {}
+
+ bool has(Register reg) const { return set_.gprs().hasRegisterIndex(reg); }
+ bool has(FloatRegister reg) const {
+ return set_.fpus().hasRegisterIndex(reg);
+ }
+
+ void addUnchecked(Register reg) { set_.gprs().addRegisterIndex(reg); }
+ void addUnchecked(FloatRegister reg) { set_.fpus().addRegisterIndex(reg); }
+
+ void takeUnchecked(Register reg) { set_.gprs().takeRegisterIndex(reg); }
+ void takeUnchecked(FloatRegister reg) { set_.fpus().takeRegisterIndex(reg); }
+};
+
+#define DEFINE_ACCESSOR_CONSTRUCTORS_(REGSET) \
+ typedef typename Parent::RegSet RegSet; \
+ typedef typename Parent::RegType RegType; \
+ typedef typename Parent::SetType SetType; \
+ \
+ constexpr REGSET() : Parent() {} \
+ explicit constexpr REGSET(SetType set) : Parent(set) {} \
+ explicit constexpr REGSET(RegSet set) : Parent(set) {}
+
+// This class adds checked accessors on top of the unchecked variants defined by
+// AllocatableSet and LiveSet accessors. Also it defines interface which are
+// specialized to the register set implementation, such as |getAny| and
+// |takeAny| variants.
+template <class Accessors, typename Set>
+class SpecializedRegSet : public Accessors {
+ using Parent = Accessors;
+
+ public:
+ DEFINE_ACCESSOR_CONSTRUCTORS_(SpecializedRegSet)
+
+ SetType bits() const { return this->Parent::set_.bits(); }
+
+ using Parent::has;
+
+ using Parent::addUnchecked;
+ void add(RegType reg) {
+ MOZ_ASSERT(!this->has(reg));
+ addUnchecked(reg);
+ }
+
+ using Parent::takeUnchecked;
+ void take(RegType reg) {
+ MOZ_ASSERT(this->has(reg));
+ takeUnchecked(reg);
+ }
+
+ template <RegTypeName Name>
+ bool hasAny() const {
+ return Parent::template all<Name>() != 0;
+ }
+
+ template <RegTypeName Name = RegSet::DefaultType>
+ RegType getFirst() const {
+ SetType set = Parent::template all<Name>();
+ MOZ_ASSERT(set);
+ return RegSet::FirstRegister(set);
+ }
+ template <RegTypeName Name = RegSet::DefaultType>
+ RegType getLast() const {
+ SetType set = Parent::template all<Name>();
+ MOZ_ASSERT(set);
+ return RegSet::LastRegister(set);
+ }
+ template <RegTypeName Name = RegSet::DefaultType>
+ RegType getAny() const {
+ // The choice of first or last here is mostly arbitrary, as they are
+ // about the same speed on popular architectures. We choose first, as
+ // it has the advantage of using the "lower" registers more often. These
+ // registers are sometimes more efficient (e.g. optimized encodings for
+ // EAX on x86).
+ return getFirst<Name>();
+ }
+
+ template <RegTypeName Name = RegSet::DefaultType>
+ RegType getAnyExcluding(RegType preclude) {
+ if (!this->has(preclude)) {
+ return getAny<Name>();
+ }
+
+ take(preclude);
+ RegType result = getAny<Name>();
+ add(preclude);
+ return result;
+ }
+
+ template <RegTypeName Name = RegSet::DefaultType>
+ RegType takeAny() {
+ RegType reg = getAny<Name>();
+ take(reg);
+ return reg;
+ }
+ template <RegTypeName Name = RegSet::DefaultType>
+ RegType takeFirst() {
+ RegType reg = getFirst<Name>();
+ take(reg);
+ return reg;
+ }
+ template <RegTypeName Name = RegSet::DefaultType>
+ RegType takeLast() {
+ RegType reg = getLast<Name>();
+ take(reg);
+ return reg;
+ }
+
+ ValueOperand takeAnyValue() {
+#if defined(JS_NUNBOX32)
+ return ValueOperand(takeAny<RegTypeName::GPR>(),
+ takeAny<RegTypeName::GPR>());
+#elif defined(JS_PUNBOX64)
+ return ValueOperand(takeAny<RegTypeName::GPR>());
+#else
+# error "Bad architecture"
+#endif
+ }
+
+ bool aliases(ValueOperand v) const {
+#ifdef JS_NUNBOX32
+ return this->has(v.typeReg()) || this->has(v.payloadReg());
+#else
+ return this->has(v.valueReg());
+#endif
+ }
+
+ template <RegTypeName Name = RegSet::DefaultType>
+ RegType takeAnyExcluding(RegType preclude) {
+ RegType reg = getAnyExcluding<Name>(preclude);
+ take(reg);
+ return reg;
+ }
+};
+
+// Specialization of the accessors for the RegisterSet aggregate.
+template <class Accessors>
+class SpecializedRegSet<Accessors, RegisterSet> : public Accessors {
+ using Parent = Accessors;
+
+ public:
+ DEFINE_ACCESSOR_CONSTRUCTORS_(SpecializedRegSet)
+
+ GeneralRegisterSet gprs() const { return this->Parent::set_.gprs(); }
+ GeneralRegisterSet& gprs() { return this->Parent::set_.gprs(); }
+ FloatRegisterSet fpus() const { return this->Parent::set_.fpus(); }
+ FloatRegisterSet& fpus() { return this->Parent::set_.fpus(); }
+
+ bool emptyGeneral() const { return this->Parent::set_.emptyGeneral(); }
+ bool emptyFloat() const { return this->Parent::set_.emptyFloat(); }
+
+ using Parent::has;
+ bool has(AnyRegister reg) const {
+ return reg.isFloat() ? this->has(reg.fpu()) : this->has(reg.gpr());
+ }
+
+ template <RegTypeName Name>
+ bool hasAny() const {
+ if (Name == RegTypeName::GPR) {
+ return Parent::template allGpr<RegTypeName::GPR>() != 0;
+ }
+ return Parent::template allFpu<Name>() != 0;
+ }
+
+ using Parent::addUnchecked;
+ void addUnchecked(AnyRegister reg) {
+ if (reg.isFloat()) {
+ addUnchecked(reg.fpu());
+ } else {
+ addUnchecked(reg.gpr());
+ }
+ }
+
+ void add(Register reg) {
+ MOZ_ASSERT(!this->has(reg));
+ addUnchecked(reg);
+ }
+ void add(FloatRegister reg) {
+ MOZ_ASSERT(!this->has(reg));
+ addUnchecked(reg);
+ }
+ void add(AnyRegister reg) {
+ if (reg.isFloat()) {
+ add(reg.fpu());
+ } else {
+ add(reg.gpr());
+ }
+ }
+
+ using Parent::takeUnchecked;
+ void takeUnchecked(AnyRegister reg) {
+ if (reg.isFloat()) {
+ takeUnchecked(reg.fpu());
+ } else {
+ takeUnchecked(reg.gpr());
+ }
+ }
+
+ void take(Register reg) {
+#ifdef DEBUG
+ bool hasReg = this->has(reg);
+ MOZ_ASSERT(hasReg);
+#endif
+ takeUnchecked(reg);
+ }
+ void take(FloatRegister reg) {
+ MOZ_ASSERT(this->has(reg));
+ takeUnchecked(reg);
+ }
+ void take(AnyRegister reg) {
+ if (reg.isFloat()) {
+ take(reg.fpu());
+ } else {
+ take(reg.gpr());
+ }
+ }
+
+ Register getAnyGeneral() const {
+ GeneralRegisterSet::SetType set =
+ Parent::template allGpr<RegTypeName::GPR>();
+ MOZ_ASSERT(set);
+ return GeneralRegisterSet::FirstRegister(set);
+ }
+ template <RegTypeName Name = RegTypeName::Float64>
+ FloatRegister getAnyFloat() const {
+ FloatRegisterSet::SetType set = Parent::template allFpu<Name>();
+ MOZ_ASSERT(set);
+ return FloatRegisterSet::FirstRegister(set);
+ }
+
+ Register takeAnyGeneral() {
+ Register reg = getAnyGeneral();
+ take(reg);
+ return reg;
+ }
+ template <RegTypeName Name = RegTypeName::Float64>
+ FloatRegister takeAnyFloat() {
+ FloatRegister reg = getAnyFloat<Name>();
+ take(reg);
+ return reg;
+ }
+ ValueOperand takeAnyValue() {
+#if defined(JS_NUNBOX32)
+ return ValueOperand(takeAnyGeneral(), takeAnyGeneral());
+#elif defined(JS_PUNBOX64)
+ return ValueOperand(takeAnyGeneral());
+#else
+# error "Bad architecture"
+#endif
+ }
+};
+
+// Interface which is common to all register set implementations. It overloads
+// |add|, |take| and |takeUnchecked| methods for types such as |ValueOperand|,
+// |TypedOrValueRegister|, and |Register64|.
+template <class Accessors, typename Set>
+class CommonRegSet : public SpecializedRegSet<Accessors, Set> {
+ typedef SpecializedRegSet<Accessors, Set> Parent;
+
+ public:
+ DEFINE_ACCESSOR_CONSTRUCTORS_(CommonRegSet)
+
+ RegSet set() const { return this->Parent::set_; }
+ RegSet& set() { return this->Parent::set_; }
+
+ bool empty() const { return this->Parent::set_.empty(); }
+ void clear() { this->Parent::set_.clear(); }
+
+ using Parent::add;
+ void add(ValueOperand value) {
+#if defined(JS_NUNBOX32)
+ add(value.payloadReg());
+ add(value.typeReg());
+#elif defined(JS_PUNBOX64)
+ add(value.valueReg());
+#else
+# error "Bad architecture"
+#endif
+ }
+ void add(Register64 reg) {
+#if JS_BITS_PER_WORD == 32
+ add(reg.high);
+ add(reg.low);
+#else
+ add(reg.reg);
+#endif
+ }
+
+ using Parent::addUnchecked;
+ void addUnchecked(ValueOperand value) {
+#if defined(JS_NUNBOX32)
+ addUnchecked(value.payloadReg());
+ addUnchecked(value.typeReg());
+#elif defined(JS_PUNBOX64)
+ addUnchecked(value.valueReg());
+#else
+# error "Bad architecture"
+#endif
+ }
+ void addUnchecked(Register64 reg) {
+#if JS_BITS_PER_WORD == 32
+ take(reg.high);
+ take(reg.low);
+#else
+ take(reg.reg);
+#endif
+ }
+
+ void add(TypedOrValueRegister reg) {
+ if (reg.hasValue()) {
+ add(reg.valueReg());
+ } else if (reg.hasTyped()) {
+ add(reg.typedReg());
+ }
+ }
+
+ using Parent::take;
+ void take(ValueOperand value) {
+#if defined(JS_NUNBOX32)
+ take(value.payloadReg());
+ take(value.typeReg());
+#elif defined(JS_PUNBOX64)
+ take(value.valueReg());
+#else
+# error "Bad architecture"
+#endif
+ }
+ void take(TypedOrValueRegister reg) {
+ if (reg.hasValue()) {
+ take(reg.valueReg());
+ } else if (reg.hasTyped()) {
+ take(reg.typedReg());
+ }
+ }
+ void take(Register64 reg) {
+#if JS_BITS_PER_WORD == 32
+ take(reg.high);
+ take(reg.low);
+#else
+ take(reg.reg);
+#endif
+ }
+
+ using Parent::takeUnchecked;
+ void takeUnchecked(ValueOperand value) {
+#if defined(JS_NUNBOX32)
+ takeUnchecked(value.payloadReg());
+ takeUnchecked(value.typeReg());
+#elif defined(JS_PUNBOX64)
+ takeUnchecked(value.valueReg());
+#else
+# error "Bad architecture"
+#endif
+ }
+ void takeUnchecked(TypedOrValueRegister reg) {
+ if (reg.hasValue()) {
+ takeUnchecked(reg.valueReg());
+ } else if (reg.hasTyped()) {
+ takeUnchecked(reg.typedReg());
+ }
+ }
+ void takeUnchecked(Register64 reg) {
+#if JS_BITS_PER_WORD == 32
+ takeUnchecked(reg.high);
+ takeUnchecked(reg.low);
+#else
+ takeUnchecked(reg.reg);
+#endif
+ }
+};
+
+// These classes do not provide any additional members, they only use their
+// constructors to forward to the common interface for all register sets. The
+// only benefit of these classes is to provide user friendly names.
+template <typename Set>
+class LiveSet : public CommonRegSet<LiveSetAccessors<Set>, Set> {
+ typedef CommonRegSet<LiveSetAccessors<Set>, Set> Parent;
+
+ public:
+ DEFINE_ACCESSOR_CONSTRUCTORS_(LiveSet)
+};
+
+template <typename Set>
+class AllocatableSet : public CommonRegSet<AllocatableSetAccessors<Set>, Set> {
+ typedef CommonRegSet<AllocatableSetAccessors<Set>, Set> Parent;
+
+ public:
+ DEFINE_ACCESSOR_CONSTRUCTORS_(AllocatableSet)
+
+ LiveSet<Set> asLiveSet() const { return LiveSet<Set>(this->set()); }
+};
+
+#define DEFINE_ACCESSOR_CONSTRUCTORS_FOR_REGISTERSET_(REGSET) \
+ typedef Parent::RegSet RegSet; \
+ typedef Parent::RegType RegType; \
+ typedef Parent::SetType SetType; \
+ \
+ constexpr REGSET() : Parent() {} \
+ explicit constexpr REGSET(SetType) = delete; \
+ explicit constexpr REGSET(RegSet set) : Parent(set) {} \
+ constexpr REGSET(GeneralRegisterSet gpr, FloatRegisterSet fpu) \
+ : Parent(RegisterSet(gpr, fpu)) {} \
+ REGSET(REGSET<GeneralRegisterSet> gpr, REGSET<FloatRegisterSet> fpu) \
+ : Parent(RegisterSet(gpr.set(), fpu.set())) {}
+
+template <>
+class LiveSet<RegisterSet>
+ : public CommonRegSet<LiveSetAccessors<RegisterSet>, RegisterSet> {
+ // Note: We have to provide a qualified name for LiveSetAccessors, as it is
+ // interpreted as being the specialized class name inherited from the parent
+ // class specialization.
+ typedef CommonRegSet<jit::LiveSetAccessors<RegisterSet>, RegisterSet> Parent;
+
+ public:
+ DEFINE_ACCESSOR_CONSTRUCTORS_FOR_REGISTERSET_(LiveSet)
+};
+
+template <>
+class AllocatableSet<RegisterSet>
+ : public CommonRegSet<AllocatableSetAccessors<RegisterSet>, RegisterSet> {
+ // Note: We have to provide a qualified name for AllocatableSetAccessors, as
+ // it is interpreted as being the specialized class name inherited from the
+ // parent class specialization.
+ typedef CommonRegSet<jit::AllocatableSetAccessors<RegisterSet>, RegisterSet>
+ Parent;
+
+ public:
+ DEFINE_ACCESSOR_CONSTRUCTORS_FOR_REGISTERSET_(AllocatableSet)
+
+ LiveSet<RegisterSet> asLiveSet() const {
+ return LiveSet<RegisterSet>(this->set());
+ }
+};
+
+#undef DEFINE_ACCESSOR_CONSTRUCTORS_FOR_REGISTERSET_
+#undef DEFINE_ACCESSOR_CONSTRUCTORS_
+
+using AllocatableGeneralRegisterSet = AllocatableSet<GeneralRegisterSet>;
+using AllocatableFloatRegisterSet = AllocatableSet<FloatRegisterSet>;
+using AllocatableRegisterSet = AllocatableSet<RegisterSet>;
+
+using LiveGeneralRegisterSet = LiveSet<GeneralRegisterSet>;
+using LiveFloatRegisterSet = LiveSet<FloatRegisterSet>;
+using LiveRegisterSet = LiveSet<RegisterSet>;
+
+// iterates in whatever order happens to be convenient.
+// Use TypedRegisterBackwardIterator or TypedRegisterForwardIterator if a
+// specific order is required.
+template <typename T>
+class TypedRegisterIterator {
+ LiveSet<TypedRegisterSet<T>> regset_;
+
+ public:
+ explicit TypedRegisterIterator(TypedRegisterSet<T> regset)
+ : regset_(regset) {}
+ explicit TypedRegisterIterator(LiveSet<TypedRegisterSet<T>> regset)
+ : regset_(regset) {}
+ TypedRegisterIterator(const TypedRegisterIterator& other)
+ : regset_(other.regset_) {}
+
+ bool more() const { return !regset_.empty(); }
+ TypedRegisterIterator<T>& operator++() {
+ regset_.template takeAny<RegTypeName::Any>();
+ return *this;
+ }
+ T operator*() const { return regset_.template getAny<RegTypeName::Any>(); }
+};
+
+// iterates backwards, that is, rn to r0
+template <typename T>
+class TypedRegisterBackwardIterator {
+ LiveSet<TypedRegisterSet<T>> regset_;
+
+ public:
+ explicit TypedRegisterBackwardIterator(TypedRegisterSet<T> regset)
+ : regset_(regset) {}
+ explicit TypedRegisterBackwardIterator(LiveSet<TypedRegisterSet<T>> regset)
+ : regset_(regset) {}
+ TypedRegisterBackwardIterator(const TypedRegisterBackwardIterator& other)
+ : regset_(other.regset_) {}
+
+ bool more() const { return !regset_.empty(); }
+ TypedRegisterBackwardIterator<T>& operator++() {
+ regset_.template takeLast<RegTypeName::Any>();
+ return *this;
+ }
+ T operator*() const { return regset_.template getLast<RegTypeName::Any>(); }
+};
+
+// iterates forwards, that is r0 to rn
+template <typename T>
+class TypedRegisterForwardIterator {
+ LiveSet<TypedRegisterSet<T>> regset_;
+
+ public:
+ explicit TypedRegisterForwardIterator(TypedRegisterSet<T> regset)
+ : regset_(regset) {}
+ explicit TypedRegisterForwardIterator(LiveSet<TypedRegisterSet<T>> regset)
+ : regset_(regset) {}
+ TypedRegisterForwardIterator(const TypedRegisterForwardIterator& other)
+ : regset_(other.regset_) {}
+
+ bool more() const { return !regset_.empty(); }
+ TypedRegisterForwardIterator<T>& operator++() {
+ regset_.template takeFirst<RegTypeName::Any>();
+ return *this;
+ }
+ T operator*() const { return regset_.template getFirst<RegTypeName::Any>(); }
+};
+
+using GeneralRegisterIterator = TypedRegisterIterator<Register>;
+using FloatRegisterIterator = TypedRegisterIterator<FloatRegister>;
+using GeneralRegisterBackwardIterator = TypedRegisterBackwardIterator<Register>;
+using FloatRegisterBackwardIterator =
+ TypedRegisterBackwardIterator<FloatRegister>;
+using GeneralRegisterForwardIterator = TypedRegisterForwardIterator<Register>;
+using FloatRegisterForwardIterator =
+ TypedRegisterForwardIterator<FloatRegister>;
+
+class AnyRegisterIterator {
+ GeneralRegisterIterator geniter_;
+ FloatRegisterIterator floatiter_;
+
+ public:
+ AnyRegisterIterator()
+ : geniter_(GeneralRegisterSet::All()),
+ floatiter_(FloatRegisterSet::All()) {}
+ AnyRegisterIterator(GeneralRegisterSet genset, FloatRegisterSet floatset)
+ : geniter_(genset), floatiter_(floatset) {}
+ explicit AnyRegisterIterator(const RegisterSet& set)
+ : geniter_(set.gpr_), floatiter_(set.fpu_) {}
+ explicit AnyRegisterIterator(const LiveSet<RegisterSet>& set)
+ : geniter_(set.gprs()), floatiter_(set.fpus()) {}
+ AnyRegisterIterator(const AnyRegisterIterator& other) = default;
+ bool more() const { return geniter_.more() || floatiter_.more(); }
+ AnyRegisterIterator& operator++() {
+ if (geniter_.more()) {
+ ++geniter_;
+ } else {
+ ++floatiter_;
+ }
+ return *this;
+ }
+ AnyRegister operator*() const {
+ if (geniter_.more()) {
+ return AnyRegister(*geniter_);
+ }
+ return AnyRegister(*floatiter_);
+ }
+};
+
+class ABIArg {
+ public:
+ enum Kind {
+ GPR,
+#ifdef JS_CODEGEN_REGISTER_PAIR
+ GPR_PAIR,
+#endif
+ FPU,
+ Stack,
+ Uninitialized = -1
+ };
+
+ private:
+ Kind kind_;
+ union {
+ Register::Code gpr_;
+ FloatRegister::Code fpu_;
+ uint32_t offset_;
+ } u;
+
+ public:
+ ABIArg() : kind_(Uninitialized) { u.offset_ = -1; }
+ explicit ABIArg(Register gpr) : kind_(GPR) { u.gpr_ = gpr.code(); }
+ explicit ABIArg(Register gprLow, Register gprHigh) {
+#if defined(JS_CODEGEN_REGISTER_PAIR)
+ kind_ = GPR_PAIR;
+#else
+ MOZ_CRASH("Unsupported type of ABI argument.");
+#endif
+ u.gpr_ = gprLow.code();
+ MOZ_ASSERT(u.gpr_ % 2 == 0);
+ MOZ_ASSERT(u.gpr_ + 1 == gprHigh.code());
+ }
+ explicit ABIArg(FloatRegister fpu) : kind_(FPU) { u.fpu_ = fpu.code(); }
+ explicit ABIArg(uint32_t offset) : kind_(Stack) { u.offset_ = offset; }
+
+ Kind kind() const {
+ MOZ_ASSERT(kind_ != Uninitialized);
+ return kind_;
+ }
+#ifdef JS_CODEGEN_REGISTER_PAIR
+ bool isGeneralRegPair() const { return kind() == GPR_PAIR; }
+#else
+ bool isGeneralRegPair() const { return false; }
+#endif
+
+ Register gpr() const {
+ MOZ_ASSERT(kind() == GPR);
+ return Register::FromCode(u.gpr_);
+ }
+ Register64 gpr64() const {
+#ifdef JS_PUNBOX64
+ return Register64(gpr());
+#else
+ return Register64(oddGpr(), evenGpr());
+#endif
+ }
+ Register evenGpr() const {
+ MOZ_ASSERT(isGeneralRegPair());
+ return Register::FromCode(u.gpr_);
+ }
+ Register oddGpr() const {
+ MOZ_ASSERT(isGeneralRegPair());
+ return Register::FromCode(u.gpr_ + 1);
+ }
+ FloatRegister fpu() const {
+ MOZ_ASSERT(kind() == FPU);
+ return FloatRegister::FromCode(u.fpu_);
+ }
+ uint32_t offsetFromArgBase() const {
+ MOZ_ASSERT(kind() == Stack);
+ return u.offset_;
+ }
+
+ bool argInRegister() const { return kind() != Stack; }
+ AnyRegister reg() const {
+ return kind() == GPR ? AnyRegister(gpr()) : AnyRegister(fpu());
+ }
+
+ bool operator==(const ABIArg& rhs) const {
+ if (kind_ != rhs.kind_) {
+ return false;
+ }
+
+ switch (kind_) {
+ case GPR:
+ return u.gpr_ == rhs.u.gpr_;
+#if defined(JS_CODEGEN_REGISTER_PAIR)
+ case GPR_PAIR:
+ return u.gpr_ == rhs.u.gpr_;
+#endif
+ case FPU:
+ return u.fpu_ == rhs.u.fpu_;
+ case Stack:
+ return u.offset_ == rhs.u.offset_;
+ case Uninitialized:
+ return true;
+ }
+ MOZ_CRASH("Invalid value for ABIArg kind");
+ }
+
+ bool operator!=(const ABIArg& rhs) const { return !(*this == rhs); }
+};
+
+// Get the set of registers which should be saved by a block of code which
+// clobbers all registers besides |unused|, but does not clobber floating point
+// registers.
+inline LiveGeneralRegisterSet SavedNonVolatileRegisters(
+ const AllocatableGeneralRegisterSet& unused) {
+ LiveGeneralRegisterSet result;
+
+ for (GeneralRegisterIterator iter(GeneralRegisterSet::NonVolatile());
+ iter.more(); ++iter) {
+ Register reg = *iter;
+ if (!unused.has(reg)) {
+ result.add(reg);
+ }
+ }
+
+ // Some platforms require the link register to be saved, if calls can be made.
+#if defined(JS_CODEGEN_ARM)
+ result.add(Register::FromCode(Registers::lr));
+#elif defined(JS_CODEGEN_ARM64)
+ result.add(Register::FromCode(Registers::lr));
+#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+ result.add(Register::FromCode(Registers::ra));
+#elif defined(JS_CODEGEN_LOONG64)
+ result.add(Register::FromCode(Registers::ra));
+#endif
+
+ return result;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_RegisterSets_h */
diff --git a/js/src/jit/Registers.h b/js/src/jit/Registers.h
new file mode 100644
index 0000000000..c384d6ccf4
--- /dev/null
+++ b/js/src/jit/Registers.h
@@ -0,0 +1,297 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Registers_h
+#define jit_Registers_h
+
+#include "mozilla/Array.h"
+
+#include "jit/IonTypes.h"
+#if defined(JS_CODEGEN_X86) || defined(JS_CODEGEN_X64)
+# include "jit/x86-shared/Architecture-x86-shared.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/Architecture-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/Architecture-arm64.h"
+#elif defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/Architecture-mips32.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/Architecture-mips64.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/Architecture-loong64.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/Architecture-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/Architecture-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+namespace js {
+namespace jit {
+
+struct Register {
+ using Codes = Registers;
+ using Encoding = Codes::Encoding;
+ using Code = Codes::Code;
+ using SetType = Codes::SetType;
+
+ Encoding reg_;
+ explicit constexpr Register(Encoding e) : reg_(e) {}
+ Register() : reg_(Encoding(Codes::Invalid)) {}
+
+ static Register FromCode(Code i) {
+ MOZ_ASSERT(i < Registers::Total);
+ Register r{Encoding(i)};
+ return r;
+ }
+ static Register FromName(const char* name) {
+ Code code = Registers::FromName(name);
+ Register r{Encoding(code)};
+ return r;
+ }
+ constexpr static Register Invalid() {
+ Register r{Encoding(Codes::Invalid)};
+ return r;
+ }
+ constexpr Code code() const { return Code(reg_); }
+ Encoding encoding() const {
+ MOZ_ASSERT(Code(reg_) < Registers::Total);
+ return reg_;
+ }
+ const char* name() const { return Registers::GetName(code()); }
+ constexpr bool operator==(Register other) const { return reg_ == other.reg_; }
+ constexpr bool operator!=(Register other) const { return reg_ != other.reg_; }
+ bool volatile_() const {
+ return !!((SetType(1) << code()) & Registers::VolatileMask);
+ }
+ bool aliases(const Register& other) const { return reg_ == other.reg_; }
+ uint32_t numAliased() const { return 1; }
+
+ Register aliased(uint32_t aliasIdx) const {
+ MOZ_ASSERT(aliasIdx == 0);
+ return *this;
+ }
+
+ SetType alignedOrDominatedAliasedSet() const { return SetType(1) << code(); }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::GPR;
+
+ template <RegTypeName = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return SetType(0);
+ }
+
+ static uint32_t SetSize(SetType x) { return Codes::SetSize(x); }
+ static uint32_t FirstBit(SetType x) { return Codes::FirstBit(x); }
+ static uint32_t LastBit(SetType x) { return Codes::LastBit(x); }
+
+ // Returns the offset of |reg| on the stack, assuming all registers in |set|
+ // were pushed in order (e.g. by |PushRegsInMask|). This is computed by
+ // clearing the lower bits (registers that were pushed later).
+ static size_t OffsetOfPushedRegister(SetType set, Register reg) {
+ return sizeof(Codes::RegisterContent) * Codes::SetSize(set >> reg.code());
+ }
+};
+
+// Architectures where the stack pointer is not a plain register with a standard
+// register encoding must define JS_HAS_HIDDEN_SP and HiddenSPEncoding.
+
+#ifdef JS_HAS_HIDDEN_SP
+struct RegisterOrSP {
+ // The register code -- but possibly one that cannot be represented as a bit
+ // position in a 32-bit vector.
+ uint32_t code;
+
+ explicit RegisterOrSP(uint32_t code) : code(code) {}
+ explicit RegisterOrSP(Register r) : code(r.code()) {}
+};
+
+static inline bool IsHiddenSP(RegisterOrSP r) {
+ return r.code == HiddenSPEncoding;
+}
+
+static inline Register AsRegister(RegisterOrSP r) {
+ MOZ_ASSERT(!IsHiddenSP(r));
+ return Register::FromCode(r.code);
+}
+
+static inline Register AsRegister(Register r) { return r; }
+
+inline bool operator==(Register r, RegisterOrSP e) {
+ return r.code() == e.code;
+}
+
+inline bool operator!=(Register r, RegisterOrSP e) { return !(r == e); }
+
+inline bool operator==(RegisterOrSP e, Register r) { return r == e; }
+
+inline bool operator!=(RegisterOrSP e, Register r) { return r != e; }
+
+inline bool operator==(RegisterOrSP lhs, RegisterOrSP rhs) {
+ return lhs.code == rhs.code;
+}
+
+inline bool operator!=(RegisterOrSP lhs, RegisterOrSP rhs) {
+ return !(lhs == rhs);
+}
+#else
+// On platforms where there's nothing special about SP, make RegisterOrSP be
+// just Register, and return false for IsHiddenSP(r) for any r so that we use
+// "normal" code for handling the SP. This reduces ifdeffery throughout the
+// jit.
+using RegisterOrSP = Register;
+
+static inline bool IsHiddenSP(RegisterOrSP r) { return false; }
+
+static inline Register AsRegister(RegisterOrSP r) { return r; }
+#endif
+
+template <>
+inline Register::SetType Register::LiveAsIndexableSet<RegTypeName::GPR>(
+ SetType set) {
+ return set;
+}
+
+template <>
+inline Register::SetType Register::LiveAsIndexableSet<RegTypeName::Any>(
+ SetType set) {
+ return set;
+}
+
+template <>
+inline Register::SetType Register::AllocatableAsIndexableSet<RegTypeName::GPR>(
+ SetType set) {
+ return set;
+}
+
+#if JS_BITS_PER_WORD == 32
+// Note, some platform code depends on INT64LOW_OFFSET being zero.
+static const uint32_t INT64LOW_OFFSET = 0 * sizeof(int32_t);
+static const uint32_t INT64HIGH_OFFSET = 1 * sizeof(int32_t);
+#endif
+
+struct Register64 {
+#ifdef JS_PUNBOX64
+ Register reg;
+#else
+ Register high;
+ Register low;
+#endif
+
+#ifdef JS_PUNBOX64
+ explicit constexpr Register64(Register r) : reg(r) {}
+ constexpr bool operator==(Register64 other) const { return reg == other.reg; }
+ constexpr bool operator!=(Register64 other) const { return reg != other.reg; }
+ Register scratchReg() { return reg; }
+ static Register64 Invalid() { return Register64(Register::Invalid()); }
+#else
+ constexpr Register64(Register h, Register l) : high(h), low(l) {}
+ constexpr bool operator==(Register64 other) const {
+ return high == other.high && low == other.low;
+ }
+ constexpr bool operator!=(Register64 other) const {
+ return high != other.high || low != other.low;
+ }
+ Register scratchReg() { return high; }
+ static Register64 Invalid() {
+ return Register64(Register::Invalid(), Register::Invalid());
+ }
+#endif
+};
+
+class RegisterDump {
+ public:
+ typedef mozilla::Array<Registers::RegisterContent, Registers::Total> GPRArray;
+ typedef mozilla::Array<FloatRegisters::RegisterContent,
+ FloatRegisters::TotalPhys>
+ FPUArray;
+
+ protected: // Silence Clang warning.
+ GPRArray regs_;
+ FPUArray fpregs_;
+
+ public:
+ static size_t offsetOfRegister(Register reg) {
+ return offsetof(RegisterDump, regs_) + reg.code() * sizeof(uintptr_t);
+ }
+ static size_t offsetOfRegister(FloatRegister reg) {
+ return offsetof(RegisterDump, fpregs_) + reg.getRegisterDumpOffsetInBytes();
+ }
+};
+
+// Class for mapping each register to an offset.
+class RegisterOffsets {
+ mozilla::Array<uint32_t, Registers::Total> offsets_;
+
+ // Sentinel value representing an uninitialized offset.
+ static constexpr uint32_t InvalidOffset = UINT32_MAX;
+
+ public:
+ RegisterOffsets() {
+ for (size_t i = 0; i < Registers::Total; i++) {
+ offsets_[i] = InvalidOffset;
+ }
+ }
+
+ RegisterOffsets(const RegisterOffsets&) = delete;
+ void operator=(const RegisterOffsets&) = delete;
+
+ bool hasOffset(Register reg) const {
+ return offsets_[reg.code()] != InvalidOffset;
+ }
+ uint32_t getOffset(Register reg) const {
+ MOZ_ASSERT(hasOffset(reg));
+ return offsets_[reg.code()];
+ }
+ void setOffset(Register reg, size_t offset) {
+ MOZ_ASSERT(offset < InvalidOffset);
+ offsets_[reg.code()] = uint32_t(offset);
+ }
+};
+
+class MacroAssembler;
+
+// Declares a register as owned within the scope of the object.
+// In debug mode, owned register state is tracked within the MacroAssembler,
+// and an assert will fire if ownership is conflicting.
+// In contrast to ARM64's UseScratchRegisterScope, this class has no overhead
+// in non-debug builds.
+template <class RegisterType>
+struct AutoGenericRegisterScope : public RegisterType {
+ // Prevent MacroAssembler templates from creating copies,
+ // which causes the destructor to fire more than once.
+ AutoGenericRegisterScope(const AutoGenericRegisterScope& other) = delete;
+
+#ifdef DEBUG
+ MacroAssembler& masm_;
+ bool released_;
+ explicit AutoGenericRegisterScope(MacroAssembler& masm, RegisterType reg);
+ ~AutoGenericRegisterScope();
+ void release();
+ void reacquire();
+#else
+ constexpr explicit AutoGenericRegisterScope(MacroAssembler& masm,
+ RegisterType reg)
+ : RegisterType(reg) {}
+ void release() {}
+ void reacquire() {}
+#endif
+};
+
+using AutoRegisterScope = AutoGenericRegisterScope<Register>;
+using AutoFloatRegisterScope = AutoGenericRegisterScope<FloatRegister>;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Registers_h */
diff --git a/js/src/jit/RematerializedFrame-inl.h b/js/src/jit/RematerializedFrame-inl.h
new file mode 100644
index 0000000000..390aaa35d0
--- /dev/null
+++ b/js/src/jit/RematerializedFrame-inl.h
@@ -0,0 +1,23 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_RematerializedFrame_inl_h
+#define jit_RematerializedFrame_inl_h
+
+#include "jit/RematerializedFrame.h"
+
+#include "mozilla/Assertions.h" // MOZ_ASSERT
+
+#include "vm/JSScript.h" // JSScript
+
+#include "vm/JSScript-inl.h" // JSScript::isDebuggee
+
+inline void js::jit::RematerializedFrame::unsetIsDebuggee() {
+ MOZ_ASSERT(!script()->isDebuggee());
+ isDebuggee_ = false;
+}
+
+#endif // jit_RematerializedFrame_inl_h
diff --git a/js/src/jit/RematerializedFrame.cpp b/js/src/jit/RematerializedFrame.cpp
new file mode 100644
index 0000000000..098eb36356
--- /dev/null
+++ b/js/src/jit/RematerializedFrame.cpp
@@ -0,0 +1,221 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/RematerializedFrame.h"
+
+#include <algorithm>
+#include <utility>
+
+#include "jit/Bailouts.h"
+#include "jit/JSJitFrameIter.h"
+#include "js/friend/DumpFunctions.h" // js::DumpValue
+#include "vm/ArgumentsObject.h"
+
+#include "vm/EnvironmentObject-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace jit;
+
+struct CopyValueToRematerializedFrame {
+ Value* slots;
+
+ explicit CopyValueToRematerializedFrame(Value* slots) : slots(slots) {}
+
+ void operator()(const Value& v) { *slots++ = v; }
+};
+
+RematerializedFrame::RematerializedFrame(JSContext* cx, uint8_t* top,
+ unsigned numActualArgs,
+ InlineFrameIterator& iter,
+ MaybeReadFallback& fallback)
+ : prevUpToDate_(false),
+ isDebuggee_(iter.script()->isDebuggee()),
+ hasInitialEnv_(false),
+ isConstructing_(iter.isConstructing()),
+ hasCachedSavedFrame_(false),
+ top_(top),
+ pc_(iter.pc()),
+ frameNo_(iter.frameNo()),
+ numActualArgs_(numActualArgs),
+ script_(iter.script()),
+ envChain_(nullptr),
+ argsObj_(nullptr) {
+ if (iter.isFunctionFrame()) {
+ callee_ = iter.callee(fallback);
+ } else {
+ callee_ = nullptr;
+ }
+
+ CopyValueToRematerializedFrame op(slots_);
+ iter.readFrameArgsAndLocals(cx, op, op, &envChain_, &hasInitialEnv_,
+ &returnValue_, &argsObj_, &thisArgument_,
+ ReadFrame_Actuals, fallback);
+}
+
+/* static */
+RematerializedFrame* RematerializedFrame::New(JSContext* cx, uint8_t* top,
+ InlineFrameIterator& iter,
+ MaybeReadFallback& fallback) {
+ unsigned numFormals =
+ iter.isFunctionFrame() ? iter.calleeTemplate()->nargs() : 0;
+ unsigned argSlots = std::max(numFormals, iter.numActualArgs());
+ unsigned extraSlots = argSlots + iter.script()->nfixed();
+
+ // One Value slot is included in sizeof(RematerializedFrame), so we can
+ // reduce the extra slot count by one. However, if there are zero slot
+ // allocations total, then reducing the slots by one will lead to
+ // the memory allocation being smaller than sizeof(RematerializedFrame).
+ if (extraSlots > 0) {
+ extraSlots -= 1;
+ }
+
+ RematerializedFrame* buf =
+ cx->pod_calloc_with_extra<RematerializedFrame, Value>(extraSlots);
+ if (!buf) {
+ return nullptr;
+ }
+
+ return new (buf)
+ RematerializedFrame(cx, top, iter.numActualArgs(), iter, fallback);
+}
+
+/* static */
+bool RematerializedFrame::RematerializeInlineFrames(
+ JSContext* cx, uint8_t* top, InlineFrameIterator& iter,
+ MaybeReadFallback& fallback, RematerializedFrameVector& frames) {
+ Rooted<RematerializedFrameVector> tempFrames(cx,
+ RematerializedFrameVector(cx));
+ if (!tempFrames.resize(iter.frameCount())) {
+ return false;
+ }
+
+ while (true) {
+ size_t frameNo = iter.frameNo();
+ tempFrames[frameNo].reset(
+ RematerializedFrame::New(cx, top, iter, fallback));
+ if (!tempFrames[frameNo]) {
+ return false;
+ }
+ if (tempFrames[frameNo]->environmentChain()) {
+ if (!EnsureHasEnvironmentObjects(cx, tempFrames[frameNo].get().get())) {
+ return false;
+ }
+ }
+
+ if (!iter.more()) {
+ break;
+ }
+ ++iter;
+ }
+
+ frames = std::move(tempFrames.get());
+ return true;
+}
+
+CallObject& RematerializedFrame::callObj() const {
+ MOZ_ASSERT(hasInitialEnvironment());
+ MOZ_ASSERT(callee()->needsCallObject());
+
+ JSObject* env = environmentChain();
+ while (!env->is<CallObject>()) {
+ env = env->enclosingEnvironment();
+ }
+ return env->as<CallObject>();
+}
+
+bool RematerializedFrame::initFunctionEnvironmentObjects(JSContext* cx) {
+ return js::InitFunctionEnvironmentObjects(cx, this);
+}
+
+bool RematerializedFrame::pushVarEnvironment(JSContext* cx,
+ Handle<Scope*> scope) {
+ return js::PushVarEnvironmentObject(cx, scope, this);
+}
+
+void RematerializedFrame::trace(JSTracer* trc) {
+ TraceRoot(trc, &script_, "remat ion frame script");
+ TraceRoot(trc, &envChain_, "remat ion frame env chain");
+ if (callee_) {
+ TraceRoot(trc, &callee_, "remat ion frame callee");
+ }
+ if (argsObj_) {
+ TraceRoot(trc, &argsObj_, "remat ion frame argsobj");
+ }
+ TraceRoot(trc, &returnValue_, "remat ion frame return value");
+ TraceRoot(trc, &thisArgument_, "remat ion frame this");
+ TraceRootRange(trc, numArgSlots() + script_->nfixed(), slots_,
+ "remat ion frame stack");
+}
+
+void RematerializedFrame::dump() {
+ fprintf(stderr, " Rematerialized Ion Frame%s\n",
+ inlined() ? " (inlined)" : "");
+ if (isFunctionFrame()) {
+ fprintf(stderr, " callee fun: ");
+#ifdef DEBUG
+ DumpValue(ObjectValue(*callee()));
+#else
+ fprintf(stderr, "?\n");
+#endif
+ } else {
+ fprintf(stderr, " global frame, no callee\n");
+ }
+
+ fprintf(stderr, " file %s line %u offset %zu\n", script()->filename(),
+ script()->lineno(), script()->pcToOffset(pc()));
+
+ fprintf(stderr, " script = %p\n", (void*)script());
+
+ if (isFunctionFrame()) {
+ fprintf(stderr, " env chain: ");
+#ifdef DEBUG
+ DumpValue(ObjectValue(*environmentChain()));
+#else
+ fprintf(stderr, "?\n");
+#endif
+
+ if (hasArgsObj()) {
+ fprintf(stderr, " args obj: ");
+#ifdef DEBUG
+ DumpValue(ObjectValue(argsObj()));
+#else
+ fprintf(stderr, "?\n");
+#endif
+ }
+
+ fprintf(stderr, " this: ");
+#ifdef DEBUG
+ DumpValue(thisArgument());
+#else
+ fprintf(stderr, "?\n");
+#endif
+
+ for (unsigned i = 0; i < numActualArgs(); i++) {
+ if (i < numFormalArgs()) {
+ fprintf(stderr, " formal (arg %u): ", i);
+ } else {
+ fprintf(stderr, " overflown (arg %u): ", i);
+ }
+#ifdef DEBUG
+ DumpValue(argv()[i]);
+#else
+ fprintf(stderr, "?\n");
+#endif
+ }
+
+ for (unsigned i = 0; i < script()->nfixed(); i++) {
+ fprintf(stderr, " local %u: ", i);
+#ifdef DEBUG
+ DumpValue(locals()[i]);
+#else
+ fprintf(stderr, "?\n");
+#endif
+ }
+ }
+
+ fputc('\n', stderr);
+}
diff --git a/js/src/jit/RematerializedFrame.h b/js/src/jit/RematerializedFrame.h
new file mode 100644
index 0000000000..89a431180d
--- /dev/null
+++ b/js/src/jit/RematerializedFrame.h
@@ -0,0 +1,219 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_RematerializedFrame_h
+#define jit_RematerializedFrame_h
+
+#include "mozilla/Assertions.h"
+
+#include <algorithm>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jstypes.h"
+
+#include "jit/JitFrames.h"
+#include "jit/ScriptFromCalleeToken.h"
+#include "js/GCVector.h"
+#include "js/TypeDecls.h"
+#include "js/UniquePtr.h"
+#include "js/Value.h"
+#include "vm/JSFunction.h"
+#include "vm/JSScript.h"
+#include "vm/Stack.h"
+
+class JS_PUBLIC_API JSTracer;
+
+namespace js {
+
+class ArgumentsObject;
+class CallObject;
+
+namespace jit {
+
+class InlineFrameIterator;
+struct MaybeReadFallback;
+
+// RematerializedFrame: An optimized frame that has been rematerialized with
+// values read out of Snapshots.
+//
+// If the Debugger API tries to inspect or modify an IonMonkey frame, much of
+// the information it expects to find in a frame is missing: function calls may
+// have been inlined, variables may have been optimized out, and so on. So when
+// this happens, SpiderMonkey builds one or more Rematerialized frames from the
+// IonMonkey frame, using the snapshot metadata built by Ion to reconstruct the
+// missing parts. The Rematerialized frames are now the authority on the state
+// of those frames, and the Ion frame is ignored: stack iterators ignore the Ion
+// frame, producing the Rematerialized frames in their stead; and when control
+// returns to the Ion frame, we pop it, rebuild Baseline frames from the
+// Rematerialized frames, and resume execution in Baseline.
+class RematerializedFrame {
+ // See DebugScopes::updateLiveScopes.
+ bool prevUpToDate_;
+
+ // Propagated to the Baseline frame once this is popped.
+ bool isDebuggee_;
+
+ // Has an initial environment has been pushed on the environment chain for
+ // function frames that need a CallObject or eval frames that need a
+ // VarEnvironmentObject?
+ bool hasInitialEnv_;
+
+ // Is this frame constructing?
+ bool isConstructing_;
+
+ // If true, this frame has been on the stack when
+ // |js::SavedStacks::saveCurrentStack| was called, and so there is a
+ // |js::SavedFrame| object cached for this frame.
+ bool hasCachedSavedFrame_;
+
+ // The fp of the top frame associated with this possibly inlined frame.
+ uint8_t* top_;
+
+ // The bytecode at the time of rematerialization.
+ jsbytecode* pc_;
+
+ size_t frameNo_;
+ unsigned numActualArgs_;
+
+ JSScript* script_;
+ JSObject* envChain_;
+ JSFunction* callee_;
+ ArgumentsObject* argsObj_;
+
+ Value returnValue_;
+ Value thisArgument_;
+ Value slots_[1];
+
+ RematerializedFrame(JSContext* cx, uint8_t* top, unsigned numActualArgs,
+ InlineFrameIterator& iter, MaybeReadFallback& fallback);
+
+ public:
+ static RematerializedFrame* New(JSContext* cx, uint8_t* top,
+ InlineFrameIterator& iter,
+ MaybeReadFallback& fallback);
+
+ // RematerializedFrame are allocated on non-GC heap, so use GCVector and
+ // UniquePtr to ensure they are traced and cleaned up correctly.
+ using RematerializedFrameVector = GCVector<UniquePtr<RematerializedFrame>>;
+
+ // Rematerialize all remaining frames pointed to by |iter| into |frames|
+ // in older-to-younger order, e.g., frames[0] is the oldest frame.
+ [[nodiscard]] static bool RematerializeInlineFrames(
+ JSContext* cx, uint8_t* top, InlineFrameIterator& iter,
+ MaybeReadFallback& fallback, RematerializedFrameVector& frames);
+
+ bool prevUpToDate() const { return prevUpToDate_; }
+ void setPrevUpToDate() { prevUpToDate_ = true; }
+ void unsetPrevUpToDate() { prevUpToDate_ = false; }
+
+ bool isDebuggee() const { return isDebuggee_; }
+ void setIsDebuggee() { isDebuggee_ = true; }
+ inline void unsetIsDebuggee();
+
+ uint8_t* top() const { return top_; }
+ JSScript* outerScript() const {
+ JitFrameLayout* jsFrame = (JitFrameLayout*)top_;
+ return ScriptFromCalleeToken(jsFrame->calleeToken());
+ }
+ jsbytecode* pc() const { return pc_; }
+ size_t frameNo() const { return frameNo_; }
+ bool inlined() const { return frameNo_ > 0; }
+
+ JSObject* environmentChain() const { return envChain_; }
+
+ template <typename SpecificEnvironment>
+ void pushOnEnvironmentChain(SpecificEnvironment& env) {
+ MOZ_ASSERT(*environmentChain() == env.enclosingEnvironment());
+ envChain_ = &env;
+ if (IsFrameInitialEnvironment(this, env)) {
+ hasInitialEnv_ = true;
+ }
+ }
+
+ template <typename SpecificEnvironment>
+ void popOffEnvironmentChain() {
+ MOZ_ASSERT(envChain_->is<SpecificEnvironment>());
+ envChain_ = &envChain_->as<SpecificEnvironment>().enclosingEnvironment();
+ }
+
+ [[nodiscard]] bool initFunctionEnvironmentObjects(JSContext* cx);
+ [[nodiscard]] bool pushVarEnvironment(JSContext* cx, Handle<Scope*> scope);
+
+ bool hasInitialEnvironment() const { return hasInitialEnv_; }
+ CallObject& callObj() const;
+
+ bool hasArgsObj() const { return !!argsObj_; }
+ ArgumentsObject& argsObj() const {
+ MOZ_ASSERT(hasArgsObj());
+ MOZ_ASSERT(script()->needsArgsObj());
+ return *argsObj_;
+ }
+
+ bool isFunctionFrame() const { return script_->isFunction(); }
+ bool isGlobalFrame() const { return script_->isGlobalCode(); }
+ bool isModuleFrame() const { return script_->isModule(); }
+
+ JSScript* script() const { return script_; }
+ JSFunction* callee() const {
+ MOZ_ASSERT(isFunctionFrame());
+ MOZ_ASSERT(callee_);
+ return callee_;
+ }
+ Value calleev() const { return ObjectValue(*callee()); }
+ Value& thisArgument() { return thisArgument_; }
+
+ bool isConstructing() const { return isConstructing_; }
+
+ bool hasCachedSavedFrame() const { return hasCachedSavedFrame_; }
+
+ void setHasCachedSavedFrame() { hasCachedSavedFrame_ = true; }
+
+ void clearHasCachedSavedFrame() { hasCachedSavedFrame_ = false; }
+
+ unsigned numFormalArgs() const {
+ return isFunctionFrame() ? callee()->nargs() : 0;
+ }
+ unsigned numActualArgs() const { return numActualArgs_; }
+ unsigned numArgSlots() const {
+ return (std::max)(numFormalArgs(), numActualArgs());
+ }
+
+ Value* argv() { return slots_; }
+ Value* locals() { return slots_ + numArgSlots(); }
+
+ Value& unaliasedLocal(unsigned i) {
+ MOZ_ASSERT(i < script()->nfixed());
+ return locals()[i];
+ }
+ Value& unaliasedFormal(unsigned i,
+ MaybeCheckAliasing checkAliasing = CHECK_ALIASING) {
+ MOZ_ASSERT(i < numFormalArgs());
+ MOZ_ASSERT_IF(checkAliasing, !script()->argsObjAliasesFormals() &&
+ !script()->formalIsAliased(i));
+ return argv()[i];
+ }
+ Value& unaliasedActual(unsigned i,
+ MaybeCheckAliasing checkAliasing = CHECK_ALIASING) {
+ MOZ_ASSERT(i < numActualArgs());
+ MOZ_ASSERT_IF(checkAliasing, !script()->argsObjAliasesFormals());
+ MOZ_ASSERT_IF(checkAliasing && i < numFormalArgs(),
+ !script()->formalIsAliased(i));
+ return argv()[i];
+ }
+
+ void setReturnValue(const Value& value) { returnValue_ = value; }
+
+ Value& returnValue() { return returnValue_; }
+
+ void trace(JSTracer* trc);
+ void dump();
+};
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_RematerializedFrame_h
diff --git a/js/src/jit/SafepointIndex-inl.h b/js/src/jit/SafepointIndex-inl.h
new file mode 100644
index 0000000000..45e54314a3
--- /dev/null
+++ b/js/src/jit/SafepointIndex-inl.h
@@ -0,0 +1,22 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_SafepointIndex_inl_h
+#define jit_SafepointIndex_inl_h
+
+#include "jit/SafepointIndex.h"
+
+#include "jit/LIR.h"
+
+namespace js::jit {
+
+inline SafepointIndex::SafepointIndex(const CodegenSafepointIndex& csi)
+ : displacement_(csi.displacement()),
+ safepointOffset_(csi.safepoint()->offset()) {}
+
+} // namespace js::jit
+
+#endif /* jit_SafepointIndex_inl_h */
diff --git a/js/src/jit/SafepointIndex.cpp b/js/src/jit/SafepointIndex.cpp
new file mode 100644
index 0000000000..4e486133cc
--- /dev/null
+++ b/js/src/jit/SafepointIndex.cpp
@@ -0,0 +1,20 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/SafepointIndex-inl.h"
+
+#include "jit/MacroAssembler.h"
+
+namespace js::jit {
+
+uint32_t OsiIndex::returnPointDisplacement() const {
+ // In general, pointer arithmetic on code is bad, but in this case,
+ // getting the return address from a call instruction, stepping over pools
+ // would be wrong.
+ return callPointDisplacement_ + Assembler::PatchWrite_NearCallSize();
+}
+
+} // namespace js::jit
diff --git a/js/src/jit/SafepointIndex.h b/js/src/jit/SafepointIndex.h
new file mode 100644
index 0000000000..ab8ebcb785
--- /dev/null
+++ b/js/src/jit/SafepointIndex.h
@@ -0,0 +1,76 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_SafepointIndex_h
+#define jit_SafepointIndex_h
+
+#include "mozilla/Assertions.h"
+
+#include <stdint.h>
+
+#include "jit/IonTypes.h"
+
+namespace js {
+namespace jit {
+
+class LSafepoint;
+class CodegenSafepointIndex;
+
+// Two-tuple that lets you look up the safepoint entry given the
+// displacement of a call instruction within the JIT code.
+class SafepointIndex {
+ // The displacement is the distance from the first byte of the JIT'd code
+ // to the return address (of the call that the safepoint was generated for).
+ uint32_t displacement_ = 0;
+
+ // Offset within the safepoint buffer.
+ uint32_t safepointOffset_ = 0;
+
+ public:
+ inline explicit SafepointIndex(const CodegenSafepointIndex& csi);
+
+ uint32_t displacement() const { return displacement_; }
+ uint32_t safepointOffset() const { return safepointOffset_; }
+};
+
+class CodegenSafepointIndex {
+ uint32_t displacement_ = 0;
+
+ LSafepoint* safepoint_ = nullptr;
+
+ public:
+ CodegenSafepointIndex(uint32_t displacement, LSafepoint* safepoint)
+ : displacement_(displacement), safepoint_(safepoint) {}
+
+ LSafepoint* safepoint() const { return safepoint_; }
+ uint32_t displacement() const { return displacement_; }
+
+ inline SnapshotOffset snapshotOffset() const;
+ inline bool hasSnapshotOffset() const;
+};
+
+// The OSI point is patched to a call instruction. Therefore, the
+// returnPoint for an OSI call is the address immediately following that
+// call instruction. The displacement of that point within the assembly
+// buffer is the |returnPointDisplacement|.
+class OsiIndex {
+ uint32_t callPointDisplacement_;
+ uint32_t snapshotOffset_;
+
+ public:
+ OsiIndex(uint32_t callPointDisplacement, uint32_t snapshotOffset)
+ : callPointDisplacement_(callPointDisplacement),
+ snapshotOffset_(snapshotOffset) {}
+
+ uint32_t returnPointDisplacement() const;
+ uint32_t callPointDisplacement() const { return callPointDisplacement_; }
+ uint32_t snapshotOffset() const { return snapshotOffset_; }
+};
+
+} /* namespace jit */
+} /* namespace js */
+
+#endif /* jit_SafepointIndex_h */
diff --git a/js/src/jit/Safepoints.cpp b/js/src/jit/Safepoints.cpp
new file mode 100644
index 0000000000..7dc06ca886
--- /dev/null
+++ b/js/src/jit/Safepoints.cpp
@@ -0,0 +1,559 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Safepoints.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/BitSet.h"
+#include "jit/IonScript.h"
+#include "jit/JitSpewer.h"
+#include "jit/LIR.h"
+#include "jit/SafepointIndex.h"
+
+using namespace js;
+using namespace jit;
+
+using mozilla::FloorLog2;
+
+SafepointWriter::SafepointWriter(uint32_t localSlotsSize,
+ uint32_t argumentsSize)
+ : localSlots_((localSlotsSize / sizeof(intptr_t)) +
+ 1), // Stack slot counts are inclusive.
+ argumentSlots_(argumentsSize / sizeof(intptr_t)) {}
+
+bool SafepointWriter::init(TempAllocator& alloc) {
+ return localSlots_.init(alloc) && argumentSlots_.init(alloc);
+}
+
+uint32_t SafepointWriter::startEntry() {
+ JitSpew(JitSpew_Safepoints,
+ "Encoding safepoint (position %zu):", stream_.length());
+ return uint32_t(stream_.length());
+}
+
+void SafepointWriter::writeOsiCallPointOffset(uint32_t osiCallPointOffset) {
+ stream_.writeUnsigned(osiCallPointOffset);
+}
+
+static void WriteRegisterMask(CompactBufferWriter& stream,
+ PackedRegisterMask bits) {
+ if (sizeof(PackedRegisterMask) == 1) {
+ stream.writeByte(bits);
+ } else {
+ MOZ_ASSERT(sizeof(PackedRegisterMask) <= 4);
+ stream.writeUnsigned(bits);
+ }
+}
+
+static PackedRegisterMask ReadRegisterMask(CompactBufferReader& stream) {
+ if (sizeof(PackedRegisterMask) == 1) {
+ return stream.readByte();
+ }
+ MOZ_ASSERT(sizeof(PackedRegisterMask) <= 4);
+ return stream.readUnsigned();
+}
+
+static void WriteFloatRegisterMask(CompactBufferWriter& stream,
+ FloatRegisters::SetType bits) {
+ switch (sizeof(FloatRegisters::SetType)) {
+#ifdef JS_CODEGEN_ARM64
+ case 16:
+ stream.writeUnsigned64(bits.low());
+ stream.writeUnsigned64(bits.high());
+ break;
+#else
+ case 1:
+ stream.writeByte(bits);
+ break;
+ case 4:
+ stream.writeUnsigned(bits);
+ break;
+ case 8:
+ stream.writeUnsigned64(bits);
+ break;
+#endif
+ default:
+ MOZ_CRASH("WriteFloatRegisterMask: unexpected size");
+ }
+}
+
+static FloatRegisters::SetType ReadFloatRegisterMask(
+ CompactBufferReader& stream) {
+ switch (sizeof(FloatRegisters::SetType)) {
+#ifdef JS_CODEGEN_ARM64
+ case 16: {
+ uint64_t low = stream.readUnsigned64();
+ uint64_t high = stream.readUnsigned64();
+ return Bitset128(high, low);
+ }
+#else
+ case 1:
+ return stream.readByte();
+ case 2:
+ case 3:
+ case 4:
+ return stream.readUnsigned();
+ case 8:
+ return stream.readUnsigned64();
+#endif
+ default:
+ MOZ_CRASH("ReadFloatRegisterMask: unexpected size");
+ }
+}
+
+void SafepointWriter::writeGcRegs(LSafepoint* safepoint) {
+ LiveGeneralRegisterSet gc(safepoint->gcRegs());
+ LiveGeneralRegisterSet spilledGpr(safepoint->liveRegs().gprs());
+ LiveFloatRegisterSet spilledFloat(safepoint->liveRegs().fpus());
+ LiveGeneralRegisterSet slots(safepoint->slotsOrElementsRegs());
+ LiveGeneralRegisterSet valueRegs;
+
+ WriteRegisterMask(stream_, spilledGpr.bits());
+ if (!spilledGpr.empty()) {
+ WriteRegisterMask(stream_, gc.bits());
+ WriteRegisterMask(stream_, slots.bits());
+
+#ifdef JS_PUNBOX64
+ valueRegs = safepoint->valueRegs();
+ WriteRegisterMask(stream_, valueRegs.bits());
+#endif
+ }
+
+ // GC registers are a subset of the spilled registers.
+ MOZ_ASSERT((valueRegs.bits() & ~spilledGpr.bits()) == 0);
+ MOZ_ASSERT((gc.bits() & ~spilledGpr.bits()) == 0);
+
+ WriteFloatRegisterMask(stream_, spilledFloat.bits());
+
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_Safepoints)) {
+ for (GeneralRegisterForwardIterator iter(spilledGpr); iter.more(); ++iter) {
+ const char* type = gc.has(*iter) ? "gc"
+ : slots.has(*iter) ? "slots"
+ : valueRegs.has(*iter) ? "value"
+ : "any";
+ JitSpew(JitSpew_Safepoints, " %s reg: %s", type, (*iter).name());
+ }
+ for (FloatRegisterForwardIterator iter(spilledFloat); iter.more(); ++iter) {
+ JitSpew(JitSpew_Safepoints, " float reg: %s", (*iter).name());
+ }
+ }
+#endif
+}
+
+static void WriteBitset(const BitSet& set, CompactBufferWriter& stream) {
+ size_t count = set.rawLength();
+ const uint32_t* words = set.raw();
+ for (size_t i = 0; i < count; i++) {
+ stream.writeUnsigned(words[i]);
+ }
+}
+
+static void MapSlotsToBitset(BitSet& stackSet, BitSet& argumentSet,
+ CompactBufferWriter& stream,
+ const LSafepoint::SlotList& slots) {
+ stackSet.clear();
+ argumentSet.clear();
+
+ for (uint32_t i = 0; i < slots.length(); i++) {
+ // Slots are represented at a distance from |fp|. We divide by the
+ // pointer size, since we only care about pointer-sized/aligned slots
+ // here.
+ MOZ_ASSERT(slots[i].slot % sizeof(intptr_t) == 0);
+ size_t index = slots[i].slot / sizeof(intptr_t);
+ (slots[i].stack ? stackSet : argumentSet).insert(index);
+ }
+
+ WriteBitset(stackSet, stream);
+ WriteBitset(argumentSet, stream);
+}
+
+void SafepointWriter::writeGcSlots(LSafepoint* safepoint) {
+ LSafepoint::SlotList& slots = safepoint->gcSlots();
+
+#ifdef JS_JITSPEW
+ for (uint32_t i = 0; i < slots.length(); i++) {
+ JitSpew(JitSpew_Safepoints, " gc slot: %u", slots[i].slot);
+ }
+#endif
+
+ MapSlotsToBitset(localSlots_, argumentSlots_, stream_, slots);
+}
+
+void SafepointWriter::writeSlotsOrElementsSlots(LSafepoint* safepoint) {
+ LSafepoint::SlotList& slots = safepoint->slotsOrElementsSlots();
+
+ stream_.writeUnsigned(slots.length());
+
+ for (uint32_t i = 0; i < slots.length(); i++) {
+ if (!slots[i].stack) {
+ MOZ_CRASH();
+ }
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_Safepoints, " slots/elements slot: %u", slots[i].slot);
+#endif
+ stream_.writeUnsigned(slots[i].slot);
+ }
+}
+
+#ifdef JS_PUNBOX64
+void SafepointWriter::writeValueSlots(LSafepoint* safepoint) {
+ LSafepoint::SlotList& slots = safepoint->valueSlots();
+
+# ifdef JS_JITSPEW
+ for (uint32_t i = 0; i < slots.length(); i++) {
+ JitSpew(JitSpew_Safepoints, " gc value: %u", slots[i].slot);
+ }
+# endif
+
+ MapSlotsToBitset(localSlots_, argumentSlots_, stream_, slots);
+}
+#endif
+
+#if defined(JS_JITSPEW) && defined(JS_NUNBOX32)
+static void DumpNunboxPart(const LAllocation& a) {
+ Fprinter& out = JitSpewPrinter();
+ if (a.isStackSlot()) {
+ out.printf("stack %d", a.toStackSlot()->slot());
+ } else if (a.isArgument()) {
+ out.printf("arg %d", a.toArgument()->index());
+ } else {
+ out.printf("reg %s", a.toGeneralReg()->reg().name());
+ }
+}
+#endif // DEBUG
+
+// Nunbox part encoding:
+//
+// Reg = 000
+// Stack = 001
+// Arg = 010
+//
+// [vwu] nentries:
+// uint16_t: tttp ppXX XXXY YYYY
+//
+// If ttt = Reg, type is reg XXXXX
+// If ppp = Reg, payload is reg YYYYY
+//
+// If ttt != Reg, type is:
+// XXXXX if not 11111, otherwise followed by [vwu]
+// If ppp != Reg, payload is:
+// YYYYY if not 11111, otherwise followed by [vwu]
+//
+enum NunboxPartKind { Part_Reg, Part_Stack, Part_Arg };
+
+static const uint32_t PART_KIND_BITS = 3;
+static const uint32_t PART_KIND_MASK = (1 << PART_KIND_BITS) - 1;
+static const uint32_t PART_INFO_BITS = 5;
+static const uint32_t PART_INFO_MASK = (1 << PART_INFO_BITS) - 1;
+
+static const uint32_t MAX_INFO_VALUE = (1 << PART_INFO_BITS) - 1;
+static const uint32_t TYPE_KIND_SHIFT = 16 - PART_KIND_BITS;
+static const uint32_t PAYLOAD_KIND_SHIFT = TYPE_KIND_SHIFT - PART_KIND_BITS;
+static const uint32_t TYPE_INFO_SHIFT = PAYLOAD_KIND_SHIFT - PART_INFO_BITS;
+static const uint32_t PAYLOAD_INFO_SHIFT = TYPE_INFO_SHIFT - PART_INFO_BITS;
+
+static_assert(PAYLOAD_INFO_SHIFT == 0);
+
+#ifdef JS_NUNBOX32
+static inline NunboxPartKind AllocationToPartKind(const LAllocation& a) {
+ if (a.isRegister()) {
+ return Part_Reg;
+ }
+ if (a.isStackSlot()) {
+ return Part_Stack;
+ }
+ MOZ_ASSERT(a.isArgument());
+ return Part_Arg;
+}
+
+// gcc 4.5 doesn't actually inline CanEncodeInfoInHeader when only
+// using the "inline" keyword, and miscompiles the function as well
+// when doing block reordering with branch prediction information.
+// See bug 799295 comment 71.
+static MOZ_ALWAYS_INLINE bool CanEncodeInfoInHeader(const LAllocation& a,
+ uint32_t* out) {
+ if (a.isGeneralReg()) {
+ *out = a.toGeneralReg()->reg().code();
+ return true;
+ }
+
+ if (a.isStackSlot()) {
+ *out = a.toStackSlot()->slot();
+ } else {
+ *out = a.toArgument()->index();
+ }
+
+ return *out < MAX_INFO_VALUE;
+}
+
+void SafepointWriter::writeNunboxParts(LSafepoint* safepoint) {
+ LSafepoint::NunboxList& entries = safepoint->nunboxParts();
+
+# ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_Safepoints)) {
+ for (uint32_t i = 0; i < entries.length(); i++) {
+ SafepointNunboxEntry& entry = entries[i];
+ if (entry.type.isUse() || entry.payload.isUse()) {
+ continue;
+ }
+ JitSpewHeader(JitSpew_Safepoints);
+ Fprinter& out = JitSpewPrinter();
+ out.printf(" nunbox (type in ");
+ DumpNunboxPart(entry.type);
+ out.printf(", payload in ");
+ DumpNunboxPart(entry.payload);
+ out.printf(")\n");
+ }
+ }
+# endif
+
+ // Safepoints are permitted to have partially filled in entries for nunboxes,
+ // provided that only the type is live and not the payload. Omit these from
+ // the written safepoint.
+
+ size_t pos = stream_.length();
+ stream_.writeUnsigned(entries.length());
+
+ size_t count = 0;
+ for (size_t i = 0; i < entries.length(); i++) {
+ SafepointNunboxEntry& entry = entries[i];
+
+ if (entry.payload.isUse()) {
+ // No allocation associated with the payload.
+ continue;
+ }
+
+ if (entry.type.isUse()) {
+ // No allocation associated with the type. Look for another
+ // safepoint entry with an allocation for the type.
+ entry.type = safepoint->findTypeAllocation(entry.typeVreg);
+ if (entry.type.isUse()) {
+ continue;
+ }
+ }
+
+ count++;
+
+ uint16_t header = 0;
+
+ header |= (AllocationToPartKind(entry.type) << TYPE_KIND_SHIFT);
+ header |= (AllocationToPartKind(entry.payload) << PAYLOAD_KIND_SHIFT);
+
+ uint32_t typeVal;
+ bool typeExtra = !CanEncodeInfoInHeader(entry.type, &typeVal);
+ if (!typeExtra) {
+ header |= (typeVal << TYPE_INFO_SHIFT);
+ } else {
+ header |= (MAX_INFO_VALUE << TYPE_INFO_SHIFT);
+ }
+
+ uint32_t payloadVal;
+ bool payloadExtra = !CanEncodeInfoInHeader(entry.payload, &payloadVal);
+ if (!payloadExtra) {
+ header |= (payloadVal << PAYLOAD_INFO_SHIFT);
+ } else {
+ header |= (MAX_INFO_VALUE << PAYLOAD_INFO_SHIFT);
+ }
+
+ stream_.writeFixedUint16_t(header);
+ if (typeExtra) {
+ stream_.writeUnsigned(typeVal);
+ }
+ if (payloadExtra) {
+ stream_.writeUnsigned(payloadVal);
+ }
+ }
+
+ // Update the stream with the actual number of safepoint entries written.
+ stream_.writeUnsignedAt(pos, count, entries.length());
+}
+#endif
+
+void SafepointWriter::encode(LSafepoint* safepoint) {
+ uint32_t safepointOffset = startEntry();
+
+ MOZ_ASSERT(safepoint->osiCallPointOffset());
+
+ writeOsiCallPointOffset(safepoint->osiCallPointOffset());
+ writeGcRegs(safepoint);
+ writeGcSlots(safepoint);
+
+#ifdef JS_PUNBOX64
+ writeValueSlots(safepoint);
+#else
+ writeNunboxParts(safepoint);
+#endif
+
+ writeSlotsOrElementsSlots(safepoint);
+
+ endEntry();
+ safepoint->setOffset(safepointOffset);
+}
+
+void SafepointWriter::endEntry() {
+ JitSpew(JitSpew_Safepoints, " -- entry ended at %u",
+ uint32_t(stream_.length()));
+}
+
+SafepointReader::SafepointReader(IonScript* script, const SafepointIndex* si)
+ : stream_(script->safepoints() + si->safepointOffset(),
+ script->safepoints() + script->safepointsSize()),
+ localSlots_((script->localSlotsSize() / sizeof(intptr_t)) +
+ 1), // Stack slot counts are inclusive.
+ argumentSlots_(script->argumentSlotsSize() / sizeof(intptr_t)),
+ nunboxSlotsRemaining_(0),
+ slotsOrElementsSlotsRemaining_(0) {
+ osiCallPointOffset_ = stream_.readUnsigned();
+
+ // gcSpills is a subset of allGprSpills.
+ allGprSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
+ if (allGprSpills_.empty()) {
+ gcSpills_ = allGprSpills_;
+ valueSpills_ = allGprSpills_;
+ slotsOrElementsSpills_ = allGprSpills_;
+ } else {
+ gcSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
+ slotsOrElementsSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
+#ifdef JS_PUNBOX64
+ valueSpills_ = GeneralRegisterSet(ReadRegisterMask(stream_));
+#endif
+ }
+
+ allFloatSpills_ = FloatRegisterSet(ReadFloatRegisterMask(stream_));
+
+ advanceFromGcRegs();
+}
+
+uint32_t SafepointReader::osiReturnPointOffset() const {
+ return osiCallPointOffset_ + Assembler::PatchWrite_NearCallSize();
+}
+
+CodeLocationLabel SafepointReader::InvalidationPatchPoint(
+ IonScript* script, const SafepointIndex* si) {
+ SafepointReader reader(script, si);
+
+ return CodeLocationLabel(script->method(),
+ CodeOffset(reader.osiCallPointOffset()));
+}
+
+void SafepointReader::advanceFromGcRegs() {
+ currentSlotChunk_ = 0;
+ nextSlotChunkNumber_ = 0;
+ currentSlotsAreStack_ = true;
+}
+
+bool SafepointReader::getSlotFromBitmap(SafepointSlotEntry* entry) {
+ while (currentSlotChunk_ == 0) {
+ // Are there any more chunks to read?
+ if (currentSlotsAreStack_) {
+ if (nextSlotChunkNumber_ == BitSet::RawLengthForBits(localSlots_)) {
+ nextSlotChunkNumber_ = 0;
+ currentSlotsAreStack_ = false;
+ continue;
+ }
+ } else if (nextSlotChunkNumber_ ==
+ BitSet::RawLengthForBits(argumentSlots_)) {
+ return false;
+ }
+
+ // Yes, read the next chunk.
+ currentSlotChunk_ = stream_.readUnsigned();
+ nextSlotChunkNumber_++;
+ }
+
+ // The current chunk still has bits in it, so get the next bit, then mask
+ // it out of the slot chunk.
+ uint32_t bit = FloorLog2(currentSlotChunk_);
+ currentSlotChunk_ &= ~(1 << bit);
+
+ // Return the slot, and re-scale it by the pointer size, reversing the
+ // transformation in MapSlotsToBitset.
+ entry->stack = currentSlotsAreStack_;
+ entry->slot = (((nextSlotChunkNumber_ - 1) * BitSet::BitsPerWord) + bit) *
+ sizeof(intptr_t);
+ return true;
+}
+
+bool SafepointReader::getGcSlot(SafepointSlotEntry* entry) {
+ if (getSlotFromBitmap(entry)) {
+ return true;
+ }
+ advanceFromGcSlots();
+ return false;
+}
+
+void SafepointReader::advanceFromGcSlots() {
+ // No, reset the counter.
+ currentSlotChunk_ = 0;
+ nextSlotChunkNumber_ = 0;
+ currentSlotsAreStack_ = true;
+#ifdef JS_NUNBOX32
+ // Nunbox slots are next.
+ nunboxSlotsRemaining_ = stream_.readUnsigned();
+#else
+ // Value slots are next.
+#endif
+}
+
+bool SafepointReader::getValueSlot(SafepointSlotEntry* entry) {
+ if (getSlotFromBitmap(entry)) {
+ return true;
+ }
+ advanceFromNunboxOrValueSlots();
+ return false;
+}
+
+static inline LAllocation PartFromStream(CompactBufferReader& stream,
+ NunboxPartKind kind, uint32_t info) {
+ if (kind == Part_Reg) {
+ return LGeneralReg(Register::FromCode(info));
+ }
+
+ if (info == MAX_INFO_VALUE) {
+ info = stream.readUnsigned();
+ }
+
+ if (kind == Part_Stack) {
+ return LStackSlot(info);
+ }
+
+ MOZ_ASSERT(kind == Part_Arg);
+ return LArgument(info);
+}
+
+bool SafepointReader::getNunboxSlot(LAllocation* type, LAllocation* payload) {
+ if (!nunboxSlotsRemaining_--) {
+ advanceFromNunboxOrValueSlots();
+ return false;
+ }
+
+ uint16_t header = stream_.readFixedUint16_t();
+ NunboxPartKind typeKind =
+ (NunboxPartKind)((header >> TYPE_KIND_SHIFT) & PART_KIND_MASK);
+ NunboxPartKind payloadKind =
+ (NunboxPartKind)((header >> PAYLOAD_KIND_SHIFT) & PART_KIND_MASK);
+ uint32_t typeInfo = (header >> TYPE_INFO_SHIFT) & PART_INFO_MASK;
+ uint32_t payloadInfo = (header >> PAYLOAD_INFO_SHIFT) & PART_INFO_MASK;
+
+ *type = PartFromStream(stream_, typeKind, typeInfo);
+ *payload = PartFromStream(stream_, payloadKind, payloadInfo);
+ return true;
+}
+
+void SafepointReader::advanceFromNunboxOrValueSlots() {
+ slotsOrElementsSlotsRemaining_ = stream_.readUnsigned();
+}
+
+bool SafepointReader::getSlotsOrElementsSlot(SafepointSlotEntry* entry) {
+ if (!slotsOrElementsSlotsRemaining_--) {
+ return false;
+ }
+ entry->stack = true;
+ entry->slot = stream_.readUnsigned();
+ return true;
+}
diff --git a/js/src/jit/Safepoints.h b/js/src/jit/Safepoints.h
new file mode 100644
index 0000000000..90f827e738
--- /dev/null
+++ b/js/src/jit/Safepoints.h
@@ -0,0 +1,129 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Safepoints_h
+#define jit_Safepoints_h
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/BitSet.h"
+#include "jit/CompactBuffer.h"
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+class CodeLocationLabel;
+class IonScript;
+class SafepointIndex;
+struct SafepointSlotEntry;
+class TempAllocator;
+
+class LAllocation;
+class LSafepoint;
+
+static const uint32_t INVALID_SAFEPOINT_OFFSET = uint32_t(-1);
+
+class SafepointWriter {
+ CompactBufferWriter stream_;
+ BitSet localSlots_;
+ BitSet argumentSlots_;
+
+ public:
+ explicit SafepointWriter(uint32_t localSlotsSize, uint32_t argumentsSize);
+ [[nodiscard]] bool init(TempAllocator& alloc);
+
+ private:
+ // A safepoint entry is written in the order these functions appear.
+ uint32_t startEntry();
+
+ void writeOsiCallPointOffset(uint32_t osiPointOffset);
+ void writeGcRegs(LSafepoint* safepoint);
+ void writeGcSlots(LSafepoint* safepoint);
+
+ void writeSlotsOrElementsSlots(LSafepoint* safepoint);
+
+#ifdef JS_PUNBOX64
+ void writeValueSlots(LSafepoint* safepoint);
+#else
+ void writeNunboxParts(LSafepoint* safepoint);
+#endif
+
+ void endEntry();
+
+ public:
+ void encode(LSafepoint* safepoint);
+
+ size_t size() const { return stream_.length(); }
+ const uint8_t* buffer() const { return stream_.buffer(); }
+ bool oom() const { return stream_.oom(); }
+};
+
+class SafepointReader {
+ CompactBufferReader stream_;
+ uint32_t localSlots_;
+ uint32_t argumentSlots_;
+ uint32_t currentSlotChunk_;
+ bool currentSlotsAreStack_;
+ uint32_t nextSlotChunkNumber_;
+ uint32_t osiCallPointOffset_;
+ GeneralRegisterSet gcSpills_;
+ GeneralRegisterSet valueSpills_;
+ GeneralRegisterSet slotsOrElementsSpills_;
+ GeneralRegisterSet allGprSpills_;
+ FloatRegisterSet allFloatSpills_;
+ uint32_t nunboxSlotsRemaining_;
+ uint32_t slotsOrElementsSlotsRemaining_;
+
+ private:
+ void advanceFromGcRegs();
+ void advanceFromGcSlots();
+ void advanceFromNunboxOrValueSlots();
+ [[nodiscard]] bool getSlotFromBitmap(SafepointSlotEntry* entry);
+
+ public:
+ SafepointReader(IonScript* script, const SafepointIndex* si);
+
+ static CodeLocationLabel InvalidationPatchPoint(IonScript* script,
+ const SafepointIndex* si);
+
+ uint32_t osiCallPointOffset() const { return osiCallPointOffset_; }
+ LiveGeneralRegisterSet gcSpills() const {
+ return LiveGeneralRegisterSet(gcSpills_);
+ }
+ LiveGeneralRegisterSet slotsOrElementsSpills() const {
+ return LiveGeneralRegisterSet(slotsOrElementsSpills_);
+ }
+ LiveGeneralRegisterSet valueSpills() const {
+ return LiveGeneralRegisterSet(valueSpills_);
+ }
+ LiveGeneralRegisterSet allGprSpills() const {
+ return LiveGeneralRegisterSet(allGprSpills_);
+ }
+ LiveFloatRegisterSet allFloatSpills() const {
+ return LiveFloatRegisterSet(allFloatSpills_);
+ }
+ uint32_t osiReturnPointOffset() const;
+
+ // Returns true if a slot was read, false if there are no more slots.
+ [[nodiscard]] bool getGcSlot(SafepointSlotEntry* entry);
+
+ // Returns true if a slot was read, false if there are no more value slots.
+ [[nodiscard]] bool getValueSlot(SafepointSlotEntry* entry);
+
+ // Returns true if a nunbox slot was read, false if there are no more
+ // nunbox slots.
+ [[nodiscard]] bool getNunboxSlot(LAllocation* type, LAllocation* payload);
+
+ // Returns true if a slot was read, false if there are no more slots.
+ [[nodiscard]] bool getSlotsOrElementsSlot(SafepointSlotEntry* entry);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Safepoints_h */
diff --git a/js/src/jit/ScalarReplacement.cpp b/js/src/jit/ScalarReplacement.cpp
new file mode 100644
index 0000000000..c95469a2d8
--- /dev/null
+++ b/js/src/jit/ScalarReplacement.cpp
@@ -0,0 +1,3071 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ScalarReplacement.h"
+
+#include "jit/IonAnalysis.h"
+#include "jit/JitSpewer.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "jit/WarpBuilderShared.h"
+#include "js/Vector.h"
+#include "vm/ArgumentsObject.h"
+
+#include "gc/ObjectKind-inl.h"
+
+namespace js {
+namespace jit {
+
+template <typename MemoryView>
+class EmulateStateOf {
+ private:
+ using BlockState = typename MemoryView::BlockState;
+
+ MIRGenerator* mir_;
+ MIRGraph& graph_;
+
+ // Block state at the entrance of all basic blocks.
+ Vector<BlockState*, 8, SystemAllocPolicy> states_;
+
+ public:
+ EmulateStateOf(MIRGenerator* mir, MIRGraph& graph)
+ : mir_(mir), graph_(graph) {}
+
+ bool run(MemoryView& view);
+};
+
+template <typename MemoryView>
+bool EmulateStateOf<MemoryView>::run(MemoryView& view) {
+ // Initialize the current block state of each block to an unknown state.
+ if (!states_.appendN(nullptr, graph_.numBlocks())) {
+ return false;
+ }
+
+ // Initialize the first block which needs to be traversed in RPO.
+ MBasicBlock* startBlock = view.startingBlock();
+ if (!view.initStartingState(&states_[startBlock->id()])) {
+ return false;
+ }
+
+ // Iterate over each basic block which has a valid entry state, and merge
+ // the state in the successor blocks.
+ for (ReversePostorderIterator block = graph_.rpoBegin(startBlock);
+ block != graph_.rpoEnd(); block++) {
+ if (mir_->shouldCancel(MemoryView::phaseName)) {
+ return false;
+ }
+
+ // Get the block state as the result of the merge of all predecessors
+ // which have already been visited in RPO. This means that backedges
+ // are not yet merged into the loop.
+ BlockState* state = states_[block->id()];
+ if (!state) {
+ continue;
+ }
+ view.setEntryBlockState(state);
+
+ // Iterates over resume points, phi and instructions.
+ for (MNodeIterator iter(*block); iter;) {
+ // Increment the iterator before visiting the instruction, as the
+ // visit function might discard itself from the basic block.
+ MNode* ins = *iter++;
+ if (ins->isDefinition()) {
+ MDefinition* def = ins->toDefinition();
+ switch (def->op()) {
+#define MIR_OP(op) \
+ case MDefinition::Opcode::op: \
+ view.visit##op(def->to##op()); \
+ break;
+ MIR_OPCODE_LIST(MIR_OP)
+#undef MIR_OP
+ }
+ } else {
+ view.visitResumePoint(ins->toResumePoint());
+ }
+ if (!graph_.alloc().ensureBallast()) {
+ return false;
+ }
+ if (view.oom()) {
+ return false;
+ }
+ }
+
+ // For each successor, merge the current state into the state of the
+ // successors.
+ for (size_t s = 0; s < block->numSuccessors(); s++) {
+ MBasicBlock* succ = block->getSuccessor(s);
+ if (!view.mergeIntoSuccessorState(*block, succ, &states_[succ->id()])) {
+ return false;
+ }
+ }
+ }
+
+ states_.clear();
+ return true;
+}
+
+static inline bool IsOptimizableObjectInstruction(MInstruction* ins) {
+ return ins->isNewObject() || ins->isNewPlainObject() ||
+ ins->isNewCallObject() || ins->isNewIterator();
+}
+
+static bool PhiOperandEqualTo(MDefinition* operand, MInstruction* newObject) {
+ if (operand == newObject) {
+ return true;
+ }
+
+ switch (operand->op()) {
+ case MDefinition::Opcode::GuardShape:
+ return PhiOperandEqualTo(operand->toGuardShape()->input(), newObject);
+
+ case MDefinition::Opcode::GuardToClass:
+ return PhiOperandEqualTo(operand->toGuardToClass()->input(), newObject);
+
+ case MDefinition::Opcode::CheckIsObj:
+ return PhiOperandEqualTo(operand->toCheckIsObj()->input(), newObject);
+
+ case MDefinition::Opcode::Unbox:
+ return PhiOperandEqualTo(operand->toUnbox()->input(), newObject);
+
+ default:
+ return false;
+ }
+}
+
+// Return true if all phi operands are equal to |newObject|.
+static bool PhiOperandsEqualTo(MPhi* phi, MInstruction* newObject) {
+ MOZ_ASSERT(IsOptimizableObjectInstruction(newObject));
+
+ for (size_t i = 0, e = phi->numOperands(); i < e; i++) {
+ if (!PhiOperandEqualTo(phi->getOperand(i), newObject)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+static bool IsObjectEscaped(MDefinition* ins, MInstruction* newObject,
+ const Shape* shapeDefault = nullptr);
+
+// Returns False if the lambda is not escaped and if it is optimizable by
+// ScalarReplacementOfObject.
+static bool IsLambdaEscaped(MInstruction* ins, MInstruction* lambda,
+ MInstruction* newObject, const Shape* shape) {
+ MOZ_ASSERT(lambda->isLambda() || lambda->isFunctionWithProto());
+ MOZ_ASSERT(IsOptimizableObjectInstruction(newObject));
+ JitSpewDef(JitSpew_Escape, "Check lambda\n", ins);
+ JitSpewIndent spewIndent(JitSpew_Escape);
+
+ // The scope chain is not escaped if none of the Lambdas which are
+ // capturing it are escaped.
+ for (MUseIterator i(ins->usesBegin()); i != ins->usesEnd(); i++) {
+ MNode* consumer = (*i)->consumer();
+ if (!consumer->isDefinition()) {
+ // Cannot optimize if it is observable from fun.arguments or others.
+ if (!consumer->toResumePoint()->isRecoverableOperand(*i)) {
+ JitSpew(JitSpew_Escape, "Observable lambda cannot be recovered");
+ return true;
+ }
+ continue;
+ }
+
+ MDefinition* def = consumer->toDefinition();
+ switch (def->op()) {
+ case MDefinition::Opcode::GuardToFunction: {
+ auto* guard = def->toGuardToFunction();
+ if (IsLambdaEscaped(guard, lambda, newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::GuardFunctionScript: {
+ auto* guard = def->toGuardFunctionScript();
+ BaseScript* actual;
+ if (lambda->isLambda()) {
+ actual = lambda->toLambda()->templateFunction()->baseScript();
+ } else {
+ actual = lambda->toFunctionWithProto()->function()->baseScript();
+ }
+ if (actual != guard->expected()) {
+ JitSpewDef(JitSpew_Escape, "has a non-matching script guard\n",
+ guard);
+ return true;
+ }
+ if (IsLambdaEscaped(guard, lambda, newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::FunctionEnvironment: {
+ if (IsObjectEscaped(def->toFunctionEnvironment(), newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ default:
+ JitSpewDef(JitSpew_Escape, "is escaped by\n", def);
+ return true;
+ }
+ }
+ JitSpew(JitSpew_Escape, "Lambda is not escaped");
+ return false;
+}
+
+static bool IsLambdaEscaped(MInstruction* lambda, MInstruction* newObject,
+ const Shape* shape) {
+ return IsLambdaEscaped(lambda, lambda, newObject, shape);
+}
+
+// Returns False if the object is not escaped and if it is optimizable by
+// ScalarReplacementOfObject.
+//
+// For the moment, this code is dumb as it only supports objects which are not
+// changing shape.
+static bool IsObjectEscaped(MDefinition* ins, MInstruction* newObject,
+ const Shape* shapeDefault) {
+ MOZ_ASSERT(ins->type() == MIRType::Object || ins->isPhi());
+ MOZ_ASSERT(IsOptimizableObjectInstruction(newObject));
+
+ JitSpewDef(JitSpew_Escape, "Check object\n", ins);
+ JitSpewIndent spewIndent(JitSpew_Escape);
+
+ const Shape* shape = shapeDefault;
+ if (!shape) {
+ if (ins->isNewPlainObject()) {
+ shape = ins->toNewPlainObject()->shape();
+ } else if (JSObject* templateObj = MObjectState::templateObjectOf(ins)) {
+ shape = templateObj->shape();
+ }
+ }
+
+ if (!shape) {
+ JitSpew(JitSpew_Escape, "No shape defined.");
+ return true;
+ }
+
+ // Check if the object is escaped. If the object is not the first argument
+ // of either a known Store / Load, then we consider it as escaped. This is a
+ // cheap and conservative escape analysis.
+ for (MUseIterator i(ins->usesBegin()); i != ins->usesEnd(); i++) {
+ MNode* consumer = (*i)->consumer();
+ if (!consumer->isDefinition()) {
+ // Cannot optimize if it is observable from fun.arguments or others.
+ if (!consumer->toResumePoint()->isRecoverableOperand(*i)) {
+ JitSpew(JitSpew_Escape, "Observable object cannot be recovered");
+ return true;
+ }
+ continue;
+ }
+
+ MDefinition* def = consumer->toDefinition();
+ switch (def->op()) {
+ case MDefinition::Opcode::StoreFixedSlot:
+ case MDefinition::Opcode::LoadFixedSlot:
+ // Not escaped if it is the first argument.
+ if (def->indexOf(*i) == 0) {
+ break;
+ }
+
+ JitSpewDef(JitSpew_Escape, "is escaped by\n", def);
+ return true;
+
+ case MDefinition::Opcode::PostWriteBarrier:
+ break;
+
+ case MDefinition::Opcode::Slots: {
+#ifdef DEBUG
+ // Assert that MSlots are only used by MStoreDynamicSlot and
+ // MLoadDynamicSlot.
+ MSlots* ins = def->toSlots();
+ MOZ_ASSERT(ins->object() != 0);
+ for (MUseIterator i(ins->usesBegin()); i != ins->usesEnd(); i++) {
+ // toDefinition should normally never fail, since they don't get
+ // captured by resume points.
+ MDefinition* def = (*i)->consumer()->toDefinition();
+ MOZ_ASSERT(def->op() == MDefinition::Opcode::StoreDynamicSlot ||
+ def->op() == MDefinition::Opcode::LoadDynamicSlot);
+ }
+#endif
+ break;
+ }
+
+ case MDefinition::Opcode::GuardShape: {
+ MGuardShape* guard = def->toGuardShape();
+ MOZ_ASSERT(!ins->isGuardShape());
+ if (shape != guard->shape()) {
+ JitSpewDef(JitSpew_Escape, "has a non-matching guard shape\n", guard);
+ return true;
+ }
+ if (IsObjectEscaped(def->toInstruction(), newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::GuardToClass: {
+ MGuardToClass* guard = def->toGuardToClass();
+ if (!shape || shape->getObjectClass() != guard->getClass()) {
+ JitSpewDef(JitSpew_Escape, "has a non-matching class guard\n", guard);
+ return true;
+ }
+ if (IsObjectEscaped(def->toInstruction(), newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::CheckIsObj: {
+ if (IsObjectEscaped(def->toInstruction(), newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::Unbox: {
+ if (def->type() != MIRType::Object) {
+ JitSpewDef(JitSpew_Escape, "has an invalid unbox\n", def);
+ return true;
+ }
+ if (IsObjectEscaped(def->toInstruction(), newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::Lambda:
+ case MDefinition::Opcode::FunctionWithProto: {
+ if (IsLambdaEscaped(def->toInstruction(), newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::Phi: {
+ auto* phi = def->toPhi();
+ if (!PhiOperandsEqualTo(phi, newObject)) {
+ JitSpewDef(JitSpew_Escape, "has different phi operands\n", def);
+ return true;
+ }
+ if (IsObjectEscaped(phi, newObject, shape)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::Compare: {
+ bool canFold;
+ if (!def->toCompare()->tryFold(&canFold)) {
+ JitSpewDef(JitSpew_Escape, "has an unsupported compare\n", def);
+ return true;
+ }
+ break;
+ }
+
+ // Doesn't escape the object.
+ case MDefinition::Opcode::IsObject:
+ break;
+
+ // This instruction is a no-op used to verify that scalar replacement
+ // is working as expected in jit-test.
+ case MDefinition::Opcode::AssertRecoveredOnBailout:
+ break;
+
+ // This is just a special flavor of constant which lets us optimize
+ // out some guards in certain circumstances. We'll turn this into a
+ // regular constant later.
+ case MDefinition::Opcode::ConstantProto:
+ break;
+
+ default:
+ JitSpewDef(JitSpew_Escape, "is escaped by\n", def);
+ return true;
+ }
+ }
+
+ JitSpew(JitSpew_Escape, "Object is not escaped");
+ return false;
+}
+
+class ObjectMemoryView : public MDefinitionVisitorDefaultNoop {
+ public:
+ using BlockState = MObjectState;
+ static const char phaseName[];
+
+ private:
+ TempAllocator& alloc_;
+ MConstant* undefinedVal_;
+ MInstruction* obj_;
+ MBasicBlock* startBlock_;
+ BlockState* state_;
+
+ // Used to improve the memory usage by sharing common modification.
+ const MResumePoint* lastResumePoint_;
+
+ bool oom_;
+
+ public:
+ ObjectMemoryView(TempAllocator& alloc, MInstruction* obj);
+
+ MBasicBlock* startingBlock();
+ bool initStartingState(BlockState** pState);
+
+ void setEntryBlockState(BlockState* state);
+ bool mergeIntoSuccessorState(MBasicBlock* curr, MBasicBlock* succ,
+ BlockState** pSuccState);
+
+#ifdef DEBUG
+ void assertSuccess();
+#else
+ void assertSuccess() {}
+#endif
+
+ bool oom() const { return oom_; }
+
+ private:
+ MDefinition* functionForCallObject(MDefinition* ins);
+
+ public:
+ void visitResumePoint(MResumePoint* rp);
+ void visitObjectState(MObjectState* ins);
+ void visitStoreFixedSlot(MStoreFixedSlot* ins);
+ void visitLoadFixedSlot(MLoadFixedSlot* ins);
+ void visitPostWriteBarrier(MPostWriteBarrier* ins);
+ void visitStoreDynamicSlot(MStoreDynamicSlot* ins);
+ void visitLoadDynamicSlot(MLoadDynamicSlot* ins);
+ void visitGuardShape(MGuardShape* ins);
+ void visitGuardToClass(MGuardToClass* ins);
+ void visitCheckIsObj(MCheckIsObj* ins);
+ void visitUnbox(MUnbox* ins);
+ void visitFunctionEnvironment(MFunctionEnvironment* ins);
+ void visitGuardToFunction(MGuardToFunction* ins);
+ void visitGuardFunctionScript(MGuardFunctionScript* ins);
+ void visitLambda(MLambda* ins);
+ void visitFunctionWithProto(MFunctionWithProto* ins);
+ void visitPhi(MPhi* ins);
+ void visitCompare(MCompare* ins);
+ void visitConstantProto(MConstantProto* ins);
+ void visitIsObject(MIsObject* ins);
+};
+
+/* static */ const char ObjectMemoryView::phaseName[] =
+ "Scalar Replacement of Object";
+
+ObjectMemoryView::ObjectMemoryView(TempAllocator& alloc, MInstruction* obj)
+ : alloc_(alloc),
+ undefinedVal_(nullptr),
+ obj_(obj),
+ startBlock_(obj->block()),
+ state_(nullptr),
+ lastResumePoint_(nullptr),
+ oom_(false) {
+ // Annotate snapshots RValue such that we recover the store first.
+ obj_->setIncompleteObject();
+
+ // Annotate the instruction such that we do not replace it by a
+ // Magic(JS_OPTIMIZED_OUT) in case of removed uses.
+ obj_->setImplicitlyUsedUnchecked();
+}
+
+MBasicBlock* ObjectMemoryView::startingBlock() { return startBlock_; }
+
+bool ObjectMemoryView::initStartingState(BlockState** pState) {
+ // Uninitialized slots have an "undefined" value.
+ undefinedVal_ = MConstant::New(alloc_, UndefinedValue());
+ startBlock_->insertBefore(obj_, undefinedVal_);
+
+ // Create a new block state and insert at it at the location of the new
+ // object.
+ BlockState* state = BlockState::New(alloc_, obj_);
+ if (!state) {
+ return false;
+ }
+
+ startBlock_->insertAfter(obj_, state);
+
+ // Initialize the properties of the object state.
+ state->initFromTemplateObject(alloc_, undefinedVal_);
+
+ // Hold out of resume point until it is visited.
+ state->setInWorklist();
+
+ *pState = state;
+ return true;
+}
+
+void ObjectMemoryView::setEntryBlockState(BlockState* state) { state_ = state; }
+
+bool ObjectMemoryView::mergeIntoSuccessorState(MBasicBlock* curr,
+ MBasicBlock* succ,
+ BlockState** pSuccState) {
+ BlockState* succState = *pSuccState;
+
+ // When a block has no state yet, create an empty one for the
+ // successor.
+ if (!succState) {
+ // If the successor is not dominated then the object cannot flow
+ // in this basic block without a Phi. We know that no Phi exist
+ // in non-dominated successors as the conservative escaped
+ // analysis fails otherwise. Such condition can succeed if the
+ // successor is a join at the end of a if-block and the object
+ // only exists within the branch.
+ if (!startBlock_->dominates(succ)) {
+ return true;
+ }
+
+ // If there is only one predecessor, carry over the last state of the
+ // block to the successor. As the block state is immutable, if the
+ // current block has multiple successors, they will share the same entry
+ // state.
+ if (succ->numPredecessors() <= 1 || !state_->numSlots()) {
+ *pSuccState = state_;
+ return true;
+ }
+
+ // If we have multiple predecessors, then we allocate one Phi node for
+ // each predecessor, and create a new block state which only has phi
+ // nodes. These would later be removed by the removal of redundant phi
+ // nodes.
+ succState = BlockState::Copy(alloc_, state_);
+ if (!succState) {
+ return false;
+ }
+
+ size_t numPreds = succ->numPredecessors();
+ for (size_t slot = 0; slot < state_->numSlots(); slot++) {
+ MPhi* phi = MPhi::New(alloc_.fallible());
+ if (!phi || !phi->reserveLength(numPreds)) {
+ return false;
+ }
+
+ // Fill the input of the successors Phi with undefined
+ // values, and each block later fills the Phi inputs.
+ for (size_t p = 0; p < numPreds; p++) {
+ phi->addInput(undefinedVal_);
+ }
+
+ // Add Phi in the list of Phis of the basic block.
+ succ->addPhi(phi);
+ succState->setSlot(slot, phi);
+ }
+
+ // Insert the newly created block state instruction at the beginning
+ // of the successor block, after all the phi nodes. Note that it
+ // would be captured by the entry resume point of the successor
+ // block.
+ succ->insertBefore(succ->safeInsertTop(), succState);
+ *pSuccState = succState;
+ }
+
+ MOZ_ASSERT_IF(succ == startBlock_, startBlock_->isLoopHeader());
+ if (succ->numPredecessors() > 1 && succState->numSlots() &&
+ succ != startBlock_) {
+ // We need to re-compute successorWithPhis as the previous EliminatePhis
+ // phase might have removed all the Phis from the successor block.
+ size_t currIndex;
+ MOZ_ASSERT(!succ->phisEmpty());
+ if (curr->successorWithPhis()) {
+ MOZ_ASSERT(curr->successorWithPhis() == succ);
+ currIndex = curr->positionInPhiSuccessor();
+ } else {
+ currIndex = succ->indexForPredecessor(curr);
+ curr->setSuccessorWithPhis(succ, currIndex);
+ }
+ MOZ_ASSERT(succ->getPredecessor(currIndex) == curr);
+
+ // Copy the current slot states to the index of current block in all the
+ // Phi created during the first visit of the successor.
+ for (size_t slot = 0; slot < state_->numSlots(); slot++) {
+ MPhi* phi = succState->getSlot(slot)->toPhi();
+ phi->replaceOperand(currIndex, state_->getSlot(slot));
+ }
+ }
+
+ return true;
+}
+
+#ifdef DEBUG
+void ObjectMemoryView::assertSuccess() {
+ for (MUseIterator i(obj_->usesBegin()); i != obj_->usesEnd(); i++) {
+ MNode* ins = (*i)->consumer();
+ MDefinition* def = nullptr;
+
+ // Resume points have been replaced by the object state.
+ if (ins->isResumePoint() ||
+ (def = ins->toDefinition())->isRecoveredOnBailout()) {
+ MOZ_ASSERT(obj_->isIncompleteObject());
+ continue;
+ }
+
+ // The only remaining uses would be removed by DCE, which will also
+ // recover the object on bailouts.
+ MOZ_ASSERT(def->isSlots() || def->isLambda() || def->isFunctionWithProto());
+ MOZ_ASSERT(!def->hasDefUses());
+ }
+}
+#endif
+
+void ObjectMemoryView::visitResumePoint(MResumePoint* rp) {
+ // As long as the MObjectState is not yet seen next to the allocation, we do
+ // not patch the resume point to recover the side effects.
+ if (!state_->isInWorklist()) {
+ rp->addStore(alloc_, state_, lastResumePoint_);
+ lastResumePoint_ = rp;
+ }
+}
+
+void ObjectMemoryView::visitObjectState(MObjectState* ins) {
+ if (ins->isInWorklist()) {
+ ins->setNotInWorklist();
+ }
+}
+
+void ObjectMemoryView::visitStoreFixedSlot(MStoreFixedSlot* ins) {
+ // Skip stores made on other objects.
+ if (ins->object() != obj_) {
+ return;
+ }
+
+ // Clone the state and update the slot value.
+ if (state_->hasFixedSlot(ins->slot())) {
+ state_ = BlockState::Copy(alloc_, state_);
+ if (!state_) {
+ oom_ = true;
+ return;
+ }
+
+ state_->setFixedSlot(ins->slot(), ins->value());
+ ins->block()->insertBefore(ins->toInstruction(), state_);
+ } else {
+ // UnsafeSetReserveSlot can access baked-in slots which are guarded by
+ // conditions, which are not seen by the escape analysis.
+ MBail* bailout = MBail::New(alloc_, BailoutKind::Inevitable);
+ ins->block()->insertBefore(ins, bailout);
+ }
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitLoadFixedSlot(MLoadFixedSlot* ins) {
+ // Skip loads made on other objects.
+ if (ins->object() != obj_) {
+ return;
+ }
+
+ // Replace load by the slot value.
+ if (state_->hasFixedSlot(ins->slot())) {
+ ins->replaceAllUsesWith(state_->getFixedSlot(ins->slot()));
+ } else {
+ // UnsafeGetReserveSlot can access baked-in slots which are guarded by
+ // conditions, which are not seen by the escape analysis.
+ MBail* bailout = MBail::New(alloc_, BailoutKind::Inevitable);
+ ins->block()->insertBefore(ins, bailout);
+ ins->replaceAllUsesWith(undefinedVal_);
+ }
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitPostWriteBarrier(MPostWriteBarrier* ins) {
+ // Skip loads made on other objects.
+ if (ins->object() != obj_) {
+ return;
+ }
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitStoreDynamicSlot(MStoreDynamicSlot* ins) {
+ // Skip stores made on other objects.
+ MSlots* slots = ins->slots()->toSlots();
+ if (slots->object() != obj_) {
+ // Guard objects are replaced when they are visited.
+ MOZ_ASSERT(!slots->object()->isGuardShape() ||
+ slots->object()->toGuardShape()->object() != obj_);
+ return;
+ }
+
+ // Clone the state and update the slot value.
+ if (state_->hasDynamicSlot(ins->slot())) {
+ state_ = BlockState::Copy(alloc_, state_);
+ if (!state_) {
+ oom_ = true;
+ return;
+ }
+
+ state_->setDynamicSlot(ins->slot(), ins->value());
+ ins->block()->insertBefore(ins->toInstruction(), state_);
+ } else {
+ // UnsafeSetReserveSlot can access baked-in slots which are guarded by
+ // conditions, which are not seen by the escape analysis.
+ MBail* bailout = MBail::New(alloc_, BailoutKind::Inevitable);
+ ins->block()->insertBefore(ins, bailout);
+ }
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitLoadDynamicSlot(MLoadDynamicSlot* ins) {
+ // Skip loads made on other objects.
+ MSlots* slots = ins->slots()->toSlots();
+ if (slots->object() != obj_) {
+ // Guard objects are replaced when they are visited.
+ MOZ_ASSERT(!slots->object()->isGuardShape() ||
+ slots->object()->toGuardShape()->object() != obj_);
+ return;
+ }
+
+ // Replace load by the slot value.
+ if (state_->hasDynamicSlot(ins->slot())) {
+ ins->replaceAllUsesWith(state_->getDynamicSlot(ins->slot()));
+ } else {
+ // UnsafeGetReserveSlot can access baked-in slots which are guarded by
+ // conditions, which are not seen by the escape analysis.
+ MBail* bailout = MBail::New(alloc_, BailoutKind::Inevitable);
+ ins->block()->insertBefore(ins, bailout);
+ ins->replaceAllUsesWith(undefinedVal_);
+ }
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitGuardShape(MGuardShape* ins) {
+ // Skip guards on other objects.
+ if (ins->object() != obj_) {
+ return;
+ }
+
+ // Replace the guard by its object.
+ ins->replaceAllUsesWith(obj_);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitGuardToClass(MGuardToClass* ins) {
+ // Skip guards on other objects.
+ if (ins->object() != obj_) {
+ return;
+ }
+
+ // Replace the guard by its object.
+ ins->replaceAllUsesWith(obj_);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitCheckIsObj(MCheckIsObj* ins) {
+ // Skip checks on other objects.
+ if (ins->input() != obj_) {
+ return;
+ }
+
+ // Replace the check by its object.
+ ins->replaceAllUsesWith(obj_);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitUnbox(MUnbox* ins) {
+ // Skip unrelated unboxes.
+ if (ins->input() != obj_) {
+ return;
+ }
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ // Replace the unbox with the object.
+ ins->replaceAllUsesWith(obj_);
+
+ // Remove the unbox.
+ ins->block()->discard(ins);
+}
+
+MDefinition* ObjectMemoryView::functionForCallObject(MDefinition* ins) {
+ // Return early when we don't replace MNewCallObject.
+ if (!obj_->isNewCallObject()) {
+ return nullptr;
+ }
+
+ // Unwrap instructions until we found either MLambda or MFunctionWithProto.
+ // Return the function instruction if their environment chain matches the
+ // MNewCallObject we're about to replace.
+ while (true) {
+ switch (ins->op()) {
+ case MDefinition::Opcode::Lambda: {
+ if (ins->toLambda()->environmentChain() == obj_) {
+ return ins;
+ }
+ return nullptr;
+ }
+ case MDefinition::Opcode::FunctionWithProto: {
+ if (ins->toFunctionWithProto()->environmentChain() == obj_) {
+ return ins;
+ }
+ return nullptr;
+ }
+ case MDefinition::Opcode::FunctionEnvironment:
+ ins = ins->toFunctionEnvironment()->function();
+ break;
+ case MDefinition::Opcode::GuardToFunction:
+ ins = ins->toGuardToFunction()->object();
+ break;
+ case MDefinition::Opcode::GuardFunctionScript:
+ ins = ins->toGuardFunctionScript()->function();
+ break;
+ default:
+ return nullptr;
+ }
+ }
+}
+
+void ObjectMemoryView::visitFunctionEnvironment(MFunctionEnvironment* ins) {
+ // Skip function environment which are not aliases of the NewCallObject.
+ if (!functionForCallObject(ins)) {
+ return;
+ }
+
+ // Replace the function environment by the scope chain of the lambda.
+ ins->replaceAllUsesWith(obj_);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitGuardToFunction(MGuardToFunction* ins) {
+ // Skip guards on other objects.
+ auto* function = functionForCallObject(ins);
+ if (!function) {
+ return;
+ }
+
+ // Replace the guard by its object.
+ ins->replaceAllUsesWith(function);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitGuardFunctionScript(MGuardFunctionScript* ins) {
+ // Skip guards on other objects.
+ auto* function = functionForCallObject(ins);
+ if (!function) {
+ return;
+ }
+
+ // Replace the guard by its object.
+ ins->replaceAllUsesWith(function);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitLambda(MLambda* ins) {
+ if (ins->environmentChain() != obj_) {
+ return;
+ }
+
+ // In order to recover the lambda we need to recover the scope chain, as the
+ // lambda is holding it.
+ ins->setIncompleteObject();
+}
+
+void ObjectMemoryView::visitFunctionWithProto(MFunctionWithProto* ins) {
+ if (ins->environmentChain() != obj_) {
+ return;
+ }
+
+ ins->setIncompleteObject();
+}
+
+void ObjectMemoryView::visitPhi(MPhi* ins) {
+ // Skip phis on other objects.
+ if (!PhiOperandsEqualTo(ins, obj_)) {
+ return;
+ }
+
+ // Replace the phi by its object.
+ ins->replaceAllUsesWith(obj_);
+
+ // Remove original instruction.
+ ins->block()->discardPhi(ins);
+}
+
+void ObjectMemoryView::visitCompare(MCompare* ins) {
+ // Skip unrelated comparisons.
+ if (ins->lhs() != obj_ && ins->rhs() != obj_) {
+ return;
+ }
+
+ bool folded;
+ MOZ_ALWAYS_TRUE(ins->tryFold(&folded));
+
+ auto* cst = MConstant::New(alloc_, BooleanValue(folded));
+ ins->block()->insertBefore(ins, cst);
+
+ // Replace the comparison with a constant.
+ ins->replaceAllUsesWith(cst);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitConstantProto(MConstantProto* ins) {
+ if (ins->getReceiverObject() != obj_) {
+ return;
+ }
+
+ auto* cst = ins->protoObject();
+ ins->replaceAllUsesWith(cst);
+ ins->block()->discard(ins);
+}
+
+void ObjectMemoryView::visitIsObject(MIsObject* ins) {
+ // Skip unrelated tests.
+ if (ins->input() != obj_) {
+ return;
+ }
+
+ auto* cst = MConstant::New(alloc_, BooleanValue(true));
+ ins->block()->insertBefore(ins, cst);
+
+ // Replace the test with a constant.
+ ins->replaceAllUsesWith(cst);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+static bool IndexOf(MDefinition* ins, int32_t* res) {
+ MOZ_ASSERT(ins->isLoadElement() || ins->isStoreElement());
+ MDefinition* indexDef = ins->getOperand(1); // ins->index();
+ if (indexDef->isSpectreMaskIndex()) {
+ indexDef = indexDef->toSpectreMaskIndex()->index();
+ }
+ if (indexDef->isBoundsCheck()) {
+ indexDef = indexDef->toBoundsCheck()->index();
+ }
+ if (indexDef->isToNumberInt32()) {
+ indexDef = indexDef->toToNumberInt32()->getOperand(0);
+ }
+ MConstant* indexDefConst = indexDef->maybeConstantValue();
+ if (!indexDefConst || indexDefConst->type() != MIRType::Int32) {
+ return false;
+ }
+ *res = indexDefConst->toInt32();
+ return true;
+}
+
+static inline bool IsOptimizableArrayInstruction(MInstruction* ins) {
+ return ins->isNewArray() || ins->isNewArrayObject();
+}
+
+// We don't support storing holes when doing scalar replacement, so any
+// optimizable MNewArrayObject instruction is guaranteed to be packed.
+static inline bool IsPackedArray(MInstruction* ins) {
+ return ins->isNewArrayObject();
+}
+
+// Returns False if the elements is not escaped and if it is optimizable by
+// ScalarReplacementOfArray.
+static bool IsElementEscaped(MDefinition* def, MInstruction* newArray,
+ uint32_t arraySize) {
+ MOZ_ASSERT(def->isElements());
+ MOZ_ASSERT(IsOptimizableArrayInstruction(newArray));
+
+ JitSpewDef(JitSpew_Escape, "Check elements\n", def);
+ JitSpewIndent spewIndent(JitSpew_Escape);
+
+ for (MUseIterator i(def->usesBegin()); i != def->usesEnd(); i++) {
+ // The MIRType::Elements cannot be captured in a resume point as
+ // it does not represent a value allocation.
+ MDefinition* access = (*i)->consumer()->toDefinition();
+
+ switch (access->op()) {
+ case MDefinition::Opcode::LoadElement: {
+ MOZ_ASSERT(access->toLoadElement()->elements() == def);
+
+ // If the index is not a constant then this index can alias
+ // all others. We do not handle this case.
+ int32_t index;
+ if (!IndexOf(access, &index)) {
+ JitSpewDef(JitSpew_Escape,
+ "has a load element with a non-trivial index\n", access);
+ return true;
+ }
+ if (index < 0 || arraySize <= uint32_t(index)) {
+ JitSpewDef(JitSpew_Escape,
+ "has a load element with an out-of-bound index\n", access);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::StoreElement: {
+ MStoreElement* storeElem = access->toStoreElement();
+ MOZ_ASSERT(storeElem->elements() == def);
+
+ // StoreElement must bail out if it stores to a hole, in case
+ // there is a setter on the prototype chain. If this StoreElement
+ // might store to a hole, we can't scalar-replace it.
+ if (storeElem->needsHoleCheck()) {
+ JitSpewDef(JitSpew_Escape, "has a store element with a hole check\n",
+ storeElem);
+ return true;
+ }
+
+ // If the index is not a constant then this index can alias
+ // all others. We do not handle this case.
+ int32_t index;
+ if (!IndexOf(storeElem, &index)) {
+ JitSpewDef(JitSpew_Escape,
+ "has a store element with a non-trivial index\n",
+ storeElem);
+ return true;
+ }
+ if (index < 0 || arraySize <= uint32_t(index)) {
+ JitSpewDef(JitSpew_Escape,
+ "has a store element with an out-of-bound index\n",
+ storeElem);
+ return true;
+ }
+
+ // Dense element holes are written using MStoreHoleValueElement instead
+ // of MStoreElement.
+ MOZ_ASSERT(storeElem->value()->type() != MIRType::MagicHole);
+ break;
+ }
+
+ case MDefinition::Opcode::SetInitializedLength:
+ MOZ_ASSERT(access->toSetInitializedLength()->elements() == def);
+ break;
+
+ case MDefinition::Opcode::InitializedLength:
+ MOZ_ASSERT(access->toInitializedLength()->elements() == def);
+ break;
+
+ case MDefinition::Opcode::ArrayLength:
+ MOZ_ASSERT(access->toArrayLength()->elements() == def);
+ break;
+
+ case MDefinition::Opcode::ApplyArray:
+ MOZ_ASSERT(access->toApplyArray()->getElements() == def);
+ if (!IsPackedArray(newArray)) {
+ JitSpewDef(JitSpew_Escape, "is not guaranteed to be packed\n",
+ access);
+ return true;
+ }
+ break;
+
+ case MDefinition::Opcode::ConstructArray:
+ MOZ_ASSERT(access->toConstructArray()->getElements() == def);
+ if (!IsPackedArray(newArray)) {
+ JitSpewDef(JitSpew_Escape, "is not guaranteed to be packed\n",
+ access);
+ return true;
+ }
+ break;
+
+ default:
+ JitSpewDef(JitSpew_Escape, "is escaped by\n", access);
+ return true;
+ }
+ }
+ JitSpew(JitSpew_Escape, "Elements is not escaped");
+ return false;
+}
+
+// Returns False if the array is not escaped and if it is optimizable by
+// ScalarReplacementOfArray.
+//
+// For the moment, this code is dumb as it only supports arrays which are not
+// changing length, with only access with known constants.
+static bool IsArrayEscaped(MInstruction* ins, MInstruction* newArray) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+ MOZ_ASSERT(IsOptimizableArrayInstruction(newArray));
+
+ JitSpewDef(JitSpew_Escape, "Check array\n", ins);
+ JitSpewIndent spewIndent(JitSpew_Escape);
+
+ const Shape* shape;
+ uint32_t length;
+ if (newArray->isNewArrayObject()) {
+ length = newArray->toNewArrayObject()->length();
+ shape = newArray->toNewArrayObject()->shape();
+ } else {
+ length = newArray->toNewArray()->length();
+ JSObject* templateObject = newArray->toNewArray()->templateObject();
+ if (!templateObject) {
+ JitSpew(JitSpew_Escape, "No template object defined.");
+ return true;
+ }
+ shape = templateObject->shape();
+ }
+
+ if (length >= 16) {
+ JitSpew(JitSpew_Escape, "Array has too many elements");
+ return true;
+ }
+
+ // Check if the object is escaped. If the object is not the first argument
+ // of either a known Store / Load, then we consider it as escaped. This is a
+ // cheap and conservative escape analysis.
+ for (MUseIterator i(ins->usesBegin()); i != ins->usesEnd(); i++) {
+ MNode* consumer = (*i)->consumer();
+ if (!consumer->isDefinition()) {
+ // Cannot optimize if it is observable from fun.arguments or others.
+ if (!consumer->toResumePoint()->isRecoverableOperand(*i)) {
+ JitSpew(JitSpew_Escape, "Observable array cannot be recovered");
+ return true;
+ }
+ continue;
+ }
+
+ MDefinition* def = consumer->toDefinition();
+ switch (def->op()) {
+ case MDefinition::Opcode::Elements: {
+ MElements* elem = def->toElements();
+ MOZ_ASSERT(elem->object() == ins);
+ if (IsElementEscaped(elem, newArray, length)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", elem);
+ return true;
+ }
+
+ break;
+ }
+
+ case MDefinition::Opcode::GuardShape: {
+ MGuardShape* guard = def->toGuardShape();
+ if (shape != guard->shape()) {
+ JitSpewDef(JitSpew_Escape, "has a non-matching guard shape\n", guard);
+ return true;
+ }
+ if (IsArrayEscaped(guard, newArray)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+
+ break;
+ }
+
+ case MDefinition::Opcode::GuardToClass: {
+ MGuardToClass* guard = def->toGuardToClass();
+ if (shape->getObjectClass() != guard->getClass()) {
+ JitSpewDef(JitSpew_Escape, "has a non-matching class guard\n", guard);
+ return true;
+ }
+ if (IsArrayEscaped(guard, newArray)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+
+ break;
+ }
+
+ case MDefinition::Opcode::GuardArrayIsPacked: {
+ auto* guard = def->toGuardArrayIsPacked();
+ if (!IsPackedArray(newArray)) {
+ JitSpewDef(JitSpew_Escape, "is not guaranteed to be packed\n", def);
+ return true;
+ }
+ if (IsArrayEscaped(guard, newArray)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::Unbox: {
+ if (def->type() != MIRType::Object) {
+ JitSpewDef(JitSpew_Escape, "has an invalid unbox\n", def);
+ return true;
+ }
+ if (IsArrayEscaped(def->toInstruction(), newArray)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ // This instruction is supported for |JSOp::OptimizeSpreadCall|.
+ case MDefinition::Opcode::Compare: {
+ bool canFold;
+ if (!def->toCompare()->tryFold(&canFold)) {
+ JitSpewDef(JitSpew_Escape, "has an unsupported compare\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::PostWriteBarrier:
+ case MDefinition::Opcode::PostWriteElementBarrier:
+ break;
+
+ // This instruction is a no-op used to verify that scalar replacement
+ // is working as expected in jit-test.
+ case MDefinition::Opcode::AssertRecoveredOnBailout:
+ break;
+
+ default:
+ JitSpewDef(JitSpew_Escape, "is escaped by\n", def);
+ return true;
+ }
+ }
+
+ JitSpew(JitSpew_Escape, "Array is not escaped");
+ return false;
+}
+
+// This class replaces every MStoreElement and MSetInitializedLength by an
+// MArrayState which emulates the content of the array. All MLoadElement,
+// MInitializedLength and MArrayLength are replaced by the corresponding value.
+//
+// In order to restore the value of the array correctly in case of bailouts, we
+// replace all reference of the allocation by the MArrayState definition.
+class ArrayMemoryView : public MDefinitionVisitorDefaultNoop {
+ public:
+ using BlockState = MArrayState;
+ static const char* phaseName;
+
+ private:
+ TempAllocator& alloc_;
+ MConstant* undefinedVal_;
+ MConstant* length_;
+ MInstruction* arr_;
+ MBasicBlock* startBlock_;
+ BlockState* state_;
+
+ // Used to improve the memory usage by sharing common modification.
+ const MResumePoint* lastResumePoint_;
+
+ bool oom_;
+
+ public:
+ ArrayMemoryView(TempAllocator& alloc, MInstruction* arr);
+
+ MBasicBlock* startingBlock();
+ bool initStartingState(BlockState** pState);
+
+ void setEntryBlockState(BlockState* state);
+ bool mergeIntoSuccessorState(MBasicBlock* curr, MBasicBlock* succ,
+ BlockState** pSuccState);
+
+#ifdef DEBUG
+ void assertSuccess();
+#else
+ void assertSuccess() {}
+#endif
+
+ bool oom() const { return oom_; }
+
+ private:
+ bool isArrayStateElements(MDefinition* elements);
+ void discardInstruction(MInstruction* ins, MDefinition* elements);
+
+ public:
+ void visitResumePoint(MResumePoint* rp);
+ void visitArrayState(MArrayState* ins);
+ void visitStoreElement(MStoreElement* ins);
+ void visitLoadElement(MLoadElement* ins);
+ void visitSetInitializedLength(MSetInitializedLength* ins);
+ void visitInitializedLength(MInitializedLength* ins);
+ void visitArrayLength(MArrayLength* ins);
+ void visitPostWriteBarrier(MPostWriteBarrier* ins);
+ void visitPostWriteElementBarrier(MPostWriteElementBarrier* ins);
+ void visitGuardShape(MGuardShape* ins);
+ void visitGuardToClass(MGuardToClass* ins);
+ void visitGuardArrayIsPacked(MGuardArrayIsPacked* ins);
+ void visitUnbox(MUnbox* ins);
+ void visitCompare(MCompare* ins);
+ void visitApplyArray(MApplyArray* ins);
+ void visitConstructArray(MConstructArray* ins);
+};
+
+const char* ArrayMemoryView::phaseName = "Scalar Replacement of Array";
+
+ArrayMemoryView::ArrayMemoryView(TempAllocator& alloc, MInstruction* arr)
+ : alloc_(alloc),
+ undefinedVal_(nullptr),
+ length_(nullptr),
+ arr_(arr),
+ startBlock_(arr->block()),
+ state_(nullptr),
+ lastResumePoint_(nullptr),
+ oom_(false) {
+ // Annotate snapshots RValue such that we recover the store first.
+ arr_->setIncompleteObject();
+
+ // Annotate the instruction such that we do not replace it by a
+ // Magic(JS_OPTIMIZED_OUT) in case of removed uses.
+ arr_->setImplicitlyUsedUnchecked();
+}
+
+MBasicBlock* ArrayMemoryView::startingBlock() { return startBlock_; }
+
+bool ArrayMemoryView::initStartingState(BlockState** pState) {
+ // Uninitialized elements have an "undefined" value.
+ undefinedVal_ = MConstant::New(alloc_, UndefinedValue());
+ MConstant* initLength = MConstant::New(alloc_, Int32Value(0));
+ arr_->block()->insertBefore(arr_, undefinedVal_);
+ arr_->block()->insertBefore(arr_, initLength);
+
+ // Create a new block state and insert at it at the location of the new array.
+ BlockState* state = BlockState::New(alloc_, arr_, initLength);
+ if (!state) {
+ return false;
+ }
+
+ startBlock_->insertAfter(arr_, state);
+
+ // Initialize the elements of the array state.
+ state->initFromTemplateObject(alloc_, undefinedVal_);
+
+ // Hold out of resume point until it is visited.
+ state->setInWorklist();
+
+ *pState = state;
+ return true;
+}
+
+void ArrayMemoryView::setEntryBlockState(BlockState* state) { state_ = state; }
+
+bool ArrayMemoryView::mergeIntoSuccessorState(MBasicBlock* curr,
+ MBasicBlock* succ,
+ BlockState** pSuccState) {
+ BlockState* succState = *pSuccState;
+
+ // When a block has no state yet, create an empty one for the
+ // successor.
+ if (!succState) {
+ // If the successor is not dominated then the array cannot flow
+ // in this basic block without a Phi. We know that no Phi exist
+ // in non-dominated successors as the conservative escaped
+ // analysis fails otherwise. Such condition can succeed if the
+ // successor is a join at the end of a if-block and the array
+ // only exists within the branch.
+ if (!startBlock_->dominates(succ)) {
+ return true;
+ }
+
+ // If there is only one predecessor, carry over the last state of the
+ // block to the successor. As the block state is immutable, if the
+ // current block has multiple successors, they will share the same entry
+ // state.
+ if (succ->numPredecessors() <= 1 || !state_->numElements()) {
+ *pSuccState = state_;
+ return true;
+ }
+
+ // If we have multiple predecessors, then we allocate one Phi node for
+ // each predecessor, and create a new block state which only has phi
+ // nodes. These would later be removed by the removal of redundant phi
+ // nodes.
+ succState = BlockState::Copy(alloc_, state_);
+ if (!succState) {
+ return false;
+ }
+
+ size_t numPreds = succ->numPredecessors();
+ for (size_t index = 0; index < state_->numElements(); index++) {
+ MPhi* phi = MPhi::New(alloc_.fallible());
+ if (!phi || !phi->reserveLength(numPreds)) {
+ return false;
+ }
+
+ // Fill the input of the successors Phi with undefined
+ // values, and each block later fills the Phi inputs.
+ for (size_t p = 0; p < numPreds; p++) {
+ phi->addInput(undefinedVal_);
+ }
+
+ // Add Phi in the list of Phis of the basic block.
+ succ->addPhi(phi);
+ succState->setElement(index, phi);
+ }
+
+ // Insert the newly created block state instruction at the beginning
+ // of the successor block, after all the phi nodes. Note that it
+ // would be captured by the entry resume point of the successor
+ // block.
+ succ->insertBefore(succ->safeInsertTop(), succState);
+ *pSuccState = succState;
+ }
+
+ MOZ_ASSERT_IF(succ == startBlock_, startBlock_->isLoopHeader());
+ if (succ->numPredecessors() > 1 && succState->numElements() &&
+ succ != startBlock_) {
+ // We need to re-compute successorWithPhis as the previous EliminatePhis
+ // phase might have removed all the Phis from the successor block.
+ size_t currIndex;
+ MOZ_ASSERT(!succ->phisEmpty());
+ if (curr->successorWithPhis()) {
+ MOZ_ASSERT(curr->successorWithPhis() == succ);
+ currIndex = curr->positionInPhiSuccessor();
+ } else {
+ currIndex = succ->indexForPredecessor(curr);
+ curr->setSuccessorWithPhis(succ, currIndex);
+ }
+ MOZ_ASSERT(succ->getPredecessor(currIndex) == curr);
+
+ // Copy the current element states to the index of current block in all
+ // the Phi created during the first visit of the successor.
+ for (size_t index = 0; index < state_->numElements(); index++) {
+ MPhi* phi = succState->getElement(index)->toPhi();
+ phi->replaceOperand(currIndex, state_->getElement(index));
+ }
+ }
+
+ return true;
+}
+
+#ifdef DEBUG
+void ArrayMemoryView::assertSuccess() { MOZ_ASSERT(!arr_->hasLiveDefUses()); }
+#endif
+
+void ArrayMemoryView::visitResumePoint(MResumePoint* rp) {
+ // As long as the MArrayState is not yet seen next to the allocation, we do
+ // not patch the resume point to recover the side effects.
+ if (!state_->isInWorklist()) {
+ rp->addStore(alloc_, state_, lastResumePoint_);
+ lastResumePoint_ = rp;
+ }
+}
+
+void ArrayMemoryView::visitArrayState(MArrayState* ins) {
+ if (ins->isInWorklist()) {
+ ins->setNotInWorklist();
+ }
+}
+
+bool ArrayMemoryView::isArrayStateElements(MDefinition* elements) {
+ return elements->isElements() && elements->toElements()->object() == arr_;
+}
+
+void ArrayMemoryView::discardInstruction(MInstruction* ins,
+ MDefinition* elements) {
+ MOZ_ASSERT(elements->isElements());
+ ins->block()->discard(ins);
+ if (!elements->hasLiveDefUses()) {
+ elements->block()->discard(elements->toInstruction());
+ }
+}
+
+void ArrayMemoryView::visitStoreElement(MStoreElement* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->elements();
+ if (!isArrayStateElements(elements)) {
+ return;
+ }
+
+ // Register value of the setter in the state.
+ int32_t index;
+ MOZ_ALWAYS_TRUE(IndexOf(ins, &index));
+ state_ = BlockState::Copy(alloc_, state_);
+ if (!state_) {
+ oom_ = true;
+ return;
+ }
+
+ state_->setElement(index, ins->value());
+ ins->block()->insertBefore(ins, state_);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+void ArrayMemoryView::visitLoadElement(MLoadElement* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->elements();
+ if (!isArrayStateElements(elements)) {
+ return;
+ }
+
+ // Replace by the value contained at the index.
+ int32_t index;
+ MOZ_ALWAYS_TRUE(IndexOf(ins, &index));
+
+ // The only way to store a hole value in a new array is with
+ // StoreHoleValueElement, which IsElementEscaped does not allow.
+ // Therefore, we do not have to do a hole check.
+ MDefinition* element = state_->getElement(index);
+ MOZ_ASSERT(element->type() != MIRType::MagicHole);
+
+ ins->replaceAllUsesWith(element);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+void ArrayMemoryView::visitSetInitializedLength(MSetInitializedLength* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->elements();
+ if (!isArrayStateElements(elements)) {
+ return;
+ }
+
+ // Replace by the new initialized length. Note that the argument of
+ // MSetInitializedLength is the last index and not the initialized length.
+ // To obtain the length, we need to add 1 to it, and thus we need to create
+ // a new constant that we register in the ArrayState.
+ state_ = BlockState::Copy(alloc_, state_);
+ if (!state_) {
+ oom_ = true;
+ return;
+ }
+
+ int32_t initLengthValue = ins->index()->maybeConstantValue()->toInt32() + 1;
+ MConstant* initLength = MConstant::New(alloc_, Int32Value(initLengthValue));
+ ins->block()->insertBefore(ins, initLength);
+ ins->block()->insertBefore(ins, state_);
+ state_->setInitializedLength(initLength);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+void ArrayMemoryView::visitInitializedLength(MInitializedLength* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->elements();
+ if (!isArrayStateElements(elements)) {
+ return;
+ }
+
+ // Replace by the value of the length.
+ ins->replaceAllUsesWith(state_->initializedLength());
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+void ArrayMemoryView::visitArrayLength(MArrayLength* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->elements();
+ if (!isArrayStateElements(elements)) {
+ return;
+ }
+
+ // Replace by the value of the length.
+ if (!length_) {
+ length_ = MConstant::New(alloc_, Int32Value(state_->numElements()));
+ arr_->block()->insertBefore(arr_, length_);
+ }
+ ins->replaceAllUsesWith(length_);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+void ArrayMemoryView::visitPostWriteBarrier(MPostWriteBarrier* ins) {
+ // Skip barriers on other objects.
+ if (ins->object() != arr_) {
+ return;
+ }
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArrayMemoryView::visitPostWriteElementBarrier(
+ MPostWriteElementBarrier* ins) {
+ // Skip barriers on other objects.
+ if (ins->object() != arr_) {
+ return;
+ }
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArrayMemoryView::visitGuardShape(MGuardShape* ins) {
+ // Skip guards on other objects.
+ if (ins->object() != arr_) {
+ return;
+ }
+
+ // Replace the guard by its object.
+ ins->replaceAllUsesWith(arr_);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArrayMemoryView::visitGuardToClass(MGuardToClass* ins) {
+ // Skip guards on other objects.
+ if (ins->object() != arr_) {
+ return;
+ }
+
+ // Replace the guard by its object.
+ ins->replaceAllUsesWith(arr_);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArrayMemoryView::visitGuardArrayIsPacked(MGuardArrayIsPacked* ins) {
+ // Skip guards on other objects.
+ if (ins->array() != arr_) {
+ return;
+ }
+
+ // Replace the guard by its object.
+ ins->replaceAllUsesWith(arr_);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArrayMemoryView::visitUnbox(MUnbox* ins) {
+ // Skip unrelated unboxes.
+ if (ins->getOperand(0) != arr_) {
+ return;
+ }
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ // Replace the unbox with the array object.
+ ins->replaceAllUsesWith(arr_);
+
+ // Remove the unbox.
+ ins->block()->discard(ins);
+}
+
+void ArrayMemoryView::visitCompare(MCompare* ins) {
+ // Skip unrelated comparisons.
+ if (ins->lhs() != arr_ && ins->rhs() != arr_) {
+ return;
+ }
+
+ bool folded;
+ MOZ_ALWAYS_TRUE(ins->tryFold(&folded));
+
+ auto* cst = MConstant::New(alloc_, BooleanValue(folded));
+ ins->block()->insertBefore(ins, cst);
+
+ // Replace the comparison with a constant.
+ ins->replaceAllUsesWith(cst);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArrayMemoryView::visitApplyArray(MApplyArray* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->getElements();
+ if (!isArrayStateElements(elements)) {
+ return;
+ }
+
+ uint32_t numElements = state_->numElements();
+
+ CallInfo callInfo(alloc_, /*constructing=*/false, ins->ignoresReturnValue());
+ if (!callInfo.initForApplyArray(ins->getFunction(), ins->getThis(),
+ numElements)) {
+ oom_ = true;
+ return;
+ }
+
+ for (uint32_t i = 0; i < numElements; i++) {
+ auto* element = state_->getElement(i);
+ MOZ_ASSERT(element->type() != MIRType::MagicHole);
+
+ callInfo.initArg(i, element);
+ }
+
+ auto addUndefined = [this]() { return undefinedVal_; };
+
+ bool needsThisCheck = false;
+ bool isDOMCall = false;
+ auto* call = MakeCall(alloc_, addUndefined, callInfo, needsThisCheck,
+ ins->getSingleTarget(), isDOMCall);
+ if (!call) {
+ oom_ = true;
+ return;
+ }
+ if (!ins->maybeCrossRealm()) {
+ call->setNotCrossRealm();
+ }
+
+ ins->block()->insertBefore(ins, call);
+ ins->replaceAllUsesWith(call);
+
+ call->stealResumePoint(ins);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+void ArrayMemoryView::visitConstructArray(MConstructArray* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->getElements();
+ if (!isArrayStateElements(elements)) {
+ return;
+ }
+
+ uint32_t numElements = state_->numElements();
+
+ CallInfo callInfo(alloc_, /*constructing=*/true, ins->ignoresReturnValue());
+ if (!callInfo.initForConstructArray(ins->getFunction(), ins->getThis(),
+ ins->getNewTarget(), numElements)) {
+ oom_ = true;
+ return;
+ }
+
+ for (uint32_t i = 0; i < numElements; i++) {
+ auto* element = state_->getElement(i);
+ MOZ_ASSERT(element->type() != MIRType::MagicHole);
+
+ callInfo.initArg(i, element);
+ }
+
+ auto addUndefined = [this]() { return undefinedVal_; };
+
+ bool needsThisCheck = ins->needsThisCheck();
+ bool isDOMCall = false;
+ auto* call = MakeCall(alloc_, addUndefined, callInfo, needsThisCheck,
+ ins->getSingleTarget(), isDOMCall);
+ if (!call) {
+ oom_ = true;
+ return;
+ }
+ if (!ins->maybeCrossRealm()) {
+ call->setNotCrossRealm();
+ }
+
+ ins->block()->insertBefore(ins, call);
+ ins->replaceAllUsesWith(call);
+
+ call->stealResumePoint(ins);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+static inline bool IsOptimizableArgumentsInstruction(MInstruction* ins) {
+ return ins->isCreateArgumentsObject() ||
+ ins->isCreateInlinedArgumentsObject();
+}
+
+class ArgumentsReplacer : public MDefinitionVisitorDefaultNoop {
+ private:
+ MIRGenerator* mir_;
+ MIRGraph& graph_;
+ MInstruction* args_;
+
+ bool oom_ = false;
+
+ TempAllocator& alloc() { return graph_.alloc(); }
+
+ bool isInlinedArguments() const {
+ return args_->isCreateInlinedArgumentsObject();
+ }
+
+ MNewArrayObject* inlineArgsArray(MInstruction* ins, Shape* shape,
+ uint32_t begin, uint32_t count);
+
+ void visitGuardToClass(MGuardToClass* ins);
+ void visitGuardProto(MGuardProto* ins);
+ void visitGuardArgumentsObjectFlags(MGuardArgumentsObjectFlags* ins);
+ void visitUnbox(MUnbox* ins);
+ void visitGetArgumentsObjectArg(MGetArgumentsObjectArg* ins);
+ void visitLoadArgumentsObjectArg(MLoadArgumentsObjectArg* ins);
+ void visitLoadArgumentsObjectArgHole(MLoadArgumentsObjectArgHole* ins);
+ void visitInArgumentsObjectArg(MInArgumentsObjectArg* ins);
+ void visitArgumentsObjectLength(MArgumentsObjectLength* ins);
+ void visitApplyArgsObj(MApplyArgsObj* ins);
+ void visitArrayFromArgumentsObject(MArrayFromArgumentsObject* ins);
+ void visitArgumentsSlice(MArgumentsSlice* ins);
+ void visitLoadFixedSlot(MLoadFixedSlot* ins);
+
+ bool oom() const { return oom_; }
+
+ public:
+ ArgumentsReplacer(MIRGenerator* mir, MIRGraph& graph, MInstruction* args)
+ : mir_(mir), graph_(graph), args_(args) {
+ MOZ_ASSERT(IsOptimizableArgumentsInstruction(args_));
+ }
+
+ bool escapes(MInstruction* ins, bool guardedForMapped = false);
+ bool run();
+ void assertSuccess();
+};
+
+// Returns false if the arguments object does not escape.
+bool ArgumentsReplacer::escapes(MInstruction* ins, bool guardedForMapped) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ JitSpewDef(JitSpew_Escape, "Check arguments object\n", ins);
+ JitSpewIndent spewIndent(JitSpew_Escape);
+
+ // We can replace inlined arguments in scripts with OSR entries, but
+ // the outermost arguments object has already been allocated before
+ // we enter via OSR and can't be replaced.
+ if (ins->isCreateArgumentsObject() && graph_.osrBlock()) {
+ JitSpew(JitSpew_Escape, "Can't replace outermost OSR arguments");
+ return true;
+ }
+
+ // Check all uses to see whether they can be supported without
+ // allocating an ArgumentsObject.
+ for (MUseIterator i(ins->usesBegin()); i != ins->usesEnd(); i++) {
+ MNode* consumer = (*i)->consumer();
+
+ // If a resume point can observe this instruction, we can only optimize
+ // if it is recoverable.
+ if (consumer->isResumePoint()) {
+ if (!consumer->toResumePoint()->isRecoverableOperand(*i)) {
+ JitSpew(JitSpew_Escape, "Observable args object cannot be recovered");
+ return true;
+ }
+ continue;
+ }
+
+ MDefinition* def = consumer->toDefinition();
+ switch (def->op()) {
+ case MDefinition::Opcode::GuardToClass: {
+ MGuardToClass* guard = def->toGuardToClass();
+ if (!guard->isArgumentsObjectClass()) {
+ JitSpewDef(JitSpew_Escape, "has a non-matching class guard\n", guard);
+ return true;
+ }
+ bool isMapped = guard->getClass() == &MappedArgumentsObject::class_;
+ if (escapes(guard, isMapped)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::GuardProto: {
+ if (escapes(def->toInstruction(), guardedForMapped)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::GuardArgumentsObjectFlags: {
+ if (escapes(def->toInstruction(), guardedForMapped)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::Unbox: {
+ if (def->type() != MIRType::Object) {
+ JitSpewDef(JitSpew_Escape, "has an invalid unbox\n", def);
+ return true;
+ }
+ if (escapes(def->toInstruction())) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::LoadFixedSlot: {
+ MLoadFixedSlot* load = def->toLoadFixedSlot();
+
+ // We can replace arguments.callee.
+ if (load->slot() == ArgumentsObject::CALLEE_SLOT) {
+ MOZ_ASSERT(guardedForMapped);
+ continue;
+ }
+ JitSpew(JitSpew_Escape, "is escaped by unsupported LoadFixedSlot\n");
+ return true;
+ }
+
+ case MDefinition::Opcode::ApplyArgsObj: {
+ if (ins == def->toApplyArgsObj()->getThis()) {
+ JitSpew(JitSpew_Escape, "is escaped as |this| arg of ApplyArgsObj\n");
+ return true;
+ }
+ MOZ_ASSERT(ins == def->toApplyArgsObj()->getArgsObj());
+ break;
+ }
+
+ // This is a replaceable consumer.
+ case MDefinition::Opcode::ArgumentsObjectLength:
+ case MDefinition::Opcode::GetArgumentsObjectArg:
+ case MDefinition::Opcode::LoadArgumentsObjectArg:
+ case MDefinition::Opcode::LoadArgumentsObjectArgHole:
+ case MDefinition::Opcode::InArgumentsObjectArg:
+ case MDefinition::Opcode::ArrayFromArgumentsObject:
+ case MDefinition::Opcode::ArgumentsSlice:
+ break;
+
+ // This instruction is a no-op used to test that scalar replacement
+ // is working as expected.
+ case MDefinition::Opcode::AssertRecoveredOnBailout:
+ break;
+
+ default:
+ JitSpewDef(JitSpew_Escape, "is escaped by\n", def);
+ return true;
+ }
+ }
+
+ JitSpew(JitSpew_Escape, "ArgumentsObject is not escaped");
+ return false;
+}
+
+// Replacing the arguments object is simpler than replacing an object
+// or array, because the arguments object does not change state.
+bool ArgumentsReplacer::run() {
+ MBasicBlock* startBlock = args_->block();
+
+ // Iterate over each basic block.
+ for (ReversePostorderIterator block = graph_.rpoBegin(startBlock);
+ block != graph_.rpoEnd(); block++) {
+ if (mir_->shouldCancel("Scalar replacement of Arguments Object")) {
+ return false;
+ }
+
+ // Iterates over phis and instructions.
+ // We do not have to visit resume points. Any resume points that capture
+ // the argument object will be handled by the Sink pass.
+ for (MDefinitionIterator iter(*block); iter;) {
+ // Increment the iterator before visiting the instruction, as the
+ // visit function might discard itself from the basic block.
+ MDefinition* def = *iter++;
+ switch (def->op()) {
+#define MIR_OP(op) \
+ case MDefinition::Opcode::op: \
+ visit##op(def->to##op()); \
+ break;
+ MIR_OPCODE_LIST(MIR_OP)
+#undef MIR_OP
+ }
+ if (!graph_.alloc().ensureBallast()) {
+ return false;
+ }
+ if (oom()) {
+ return false;
+ }
+ }
+ }
+
+ assertSuccess();
+ return true;
+}
+
+void ArgumentsReplacer::assertSuccess() {
+ MOZ_ASSERT(args_->canRecoverOnBailout());
+ MOZ_ASSERT(!args_->hasLiveDefUses());
+}
+
+void ArgumentsReplacer::visitGuardToClass(MGuardToClass* ins) {
+ // Skip guards on other objects.
+ if (ins->object() != args_) {
+ return;
+ }
+ MOZ_ASSERT(ins->isArgumentsObjectClass());
+
+ // Replace the guard with the args object.
+ ins->replaceAllUsesWith(args_);
+
+ // Remove the guard.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitGuardProto(MGuardProto* ins) {
+ // Skip guards on other objects.
+ if (ins->object() != args_) {
+ return;
+ }
+
+ // The prototype can only be changed through explicit operations, for example
+ // by calling |Reflect.setPrototype|. We have already determined that the args
+ // object doesn't escape, so its prototype can't be mutated.
+
+ // Replace the guard with the args object.
+ ins->replaceAllUsesWith(args_);
+
+ // Remove the guard.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitGuardArgumentsObjectFlags(
+ MGuardArgumentsObjectFlags* ins) {
+ // Skip other arguments objects.
+ if (ins->argsObject() != args_) {
+ return;
+ }
+
+#ifdef DEBUG
+ // Each *_OVERRIDDEN_BIT can only be set by setting or deleting a
+ // property of the args object. We have already determined that the
+ // args object doesn't escape, so its properties can't be mutated.
+ //
+ // FORWARDED_ARGUMENTS_BIT is set if any mapped argument is closed
+ // over, which is an immutable property of the script. Because we
+ // are replacing the args object for a known script, we can check
+ // the flag once, which is done when we first attach the CacheIR,
+ // and rely on it. (Note that this wouldn't be true if we didn't
+ // know the origin of args_, because it could be passed in from
+ // another function.)
+ uint32_t supportedBits = ArgumentsObject::LENGTH_OVERRIDDEN_BIT |
+ ArgumentsObject::ITERATOR_OVERRIDDEN_BIT |
+ ArgumentsObject::ELEMENT_OVERRIDDEN_BIT |
+ ArgumentsObject::CALLEE_OVERRIDDEN_BIT |
+ ArgumentsObject::FORWARDED_ARGUMENTS_BIT;
+
+ MOZ_ASSERT((ins->flags() & ~supportedBits) == 0);
+ MOZ_ASSERT_IF(ins->flags() & ArgumentsObject::FORWARDED_ARGUMENTS_BIT,
+ !args_->block()->info().anyFormalIsForwarded());
+#endif
+
+ // Replace the guard with the args object.
+ ins->replaceAllUsesWith(args_);
+
+ // Remove the guard.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitUnbox(MUnbox* ins) {
+ // Skip unrelated unboxes.
+ if (ins->getOperand(0) != args_) {
+ return;
+ }
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ // Replace the unbox with the args object.
+ ins->replaceAllUsesWith(args_);
+
+ // Remove the unbox.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitGetArgumentsObjectArg(
+ MGetArgumentsObjectArg* ins) {
+ // Skip other arguments objects.
+ if (ins->argsObject() != args_) {
+ return;
+ }
+
+ // We don't support setting arguments in ArgumentsReplacer::escapes,
+ // so we can load the initial value of the argument without worrying
+ // about it being stale.
+ MDefinition* getArg;
+ if (isInlinedArguments()) {
+ // Inlined frames have direct access to the actual arguments.
+ auto* actualArgs = args_->toCreateInlinedArgumentsObject();
+ if (ins->argno() < actualArgs->numActuals()) {
+ getArg = actualArgs->getArg(ins->argno());
+ } else {
+ // Omitted arguments are not mapped to the arguments object, and
+ // will always be undefined.
+ auto* undef = MConstant::New(alloc(), UndefinedValue());
+ ins->block()->insertBefore(ins, undef);
+ getArg = undef;
+ }
+ } else {
+ // Load the argument from the frame.
+ auto* index = MConstant::New(alloc(), Int32Value(ins->argno()));
+ ins->block()->insertBefore(ins, index);
+
+ auto* loadArg = MGetFrameArgument::New(alloc(), index);
+ ins->block()->insertBefore(ins, loadArg);
+ getArg = loadArg;
+ }
+ ins->replaceAllUsesWith(getArg);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitLoadArgumentsObjectArg(
+ MLoadArgumentsObjectArg* ins) {
+ // Skip other arguments objects.
+ if (ins->argsObject() != args_) {
+ return;
+ }
+
+ MDefinition* index = ins->index();
+
+ MInstruction* loadArg;
+ if (isInlinedArguments()) {
+ auto* actualArgs = args_->toCreateInlinedArgumentsObject();
+
+ // Insert bounds check.
+ auto* length =
+ MConstant::New(alloc(), Int32Value(actualArgs->numActuals()));
+ ins->block()->insertBefore(ins, length);
+
+ MInstruction* check = MBoundsCheck::New(alloc(), index, length);
+ check->setBailoutKind(ins->bailoutKind());
+ ins->block()->insertBefore(ins, check);
+
+ if (mir_->outerInfo().hadBoundsCheckBailout()) {
+ check->setNotMovable();
+ }
+
+ loadArg = MGetInlinedArgument::New(alloc(), check, actualArgs);
+ if (!loadArg) {
+ oom_ = true;
+ return;
+ }
+ } else {
+ // Insert bounds check.
+ auto* length = MArgumentsLength::New(alloc());
+ ins->block()->insertBefore(ins, length);
+
+ MInstruction* check = MBoundsCheck::New(alloc(), index, length);
+ check->setBailoutKind(ins->bailoutKind());
+ ins->block()->insertBefore(ins, check);
+
+ if (mir_->outerInfo().hadBoundsCheckBailout()) {
+ check->setNotMovable();
+ }
+
+ if (JitOptions.spectreIndexMasking) {
+ check = MSpectreMaskIndex::New(alloc(), check, length);
+ ins->block()->insertBefore(ins, check);
+ }
+
+ loadArg = MGetFrameArgument::New(alloc(), check);
+ }
+ ins->block()->insertBefore(ins, loadArg);
+ ins->replaceAllUsesWith(loadArg);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitLoadArgumentsObjectArgHole(
+ MLoadArgumentsObjectArgHole* ins) {
+ // Skip other arguments objects.
+ if (ins->argsObject() != args_) {
+ return;
+ }
+
+ MDefinition* index = ins->index();
+
+ MInstruction* loadArg;
+ if (isInlinedArguments()) {
+ auto* actualArgs = args_->toCreateInlinedArgumentsObject();
+
+ loadArg = MGetInlinedArgumentHole::New(alloc(), index, actualArgs);
+ if (!loadArg) {
+ oom_ = true;
+ return;
+ }
+ } else {
+ auto* length = MArgumentsLength::New(alloc());
+ ins->block()->insertBefore(ins, length);
+
+ loadArg = MGetFrameArgumentHole::New(alloc(), index, length);
+ }
+ loadArg->setBailoutKind(ins->bailoutKind());
+ ins->block()->insertBefore(ins, loadArg);
+ ins->replaceAllUsesWith(loadArg);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitInArgumentsObjectArg(MInArgumentsObjectArg* ins) {
+ // Skip other arguments objects.
+ if (ins->argsObject() != args_) {
+ return;
+ }
+
+ MDefinition* index = ins->index();
+
+ // Ensure the index is non-negative.
+ auto* guardedIndex = MGuardInt32IsNonNegative::New(alloc(), index);
+ guardedIndex->setBailoutKind(ins->bailoutKind());
+ ins->block()->insertBefore(ins, guardedIndex);
+
+ MInstruction* length;
+ if (isInlinedArguments()) {
+ uint32_t argc = args_->toCreateInlinedArgumentsObject()->numActuals();
+ length = MConstant::New(alloc(), Int32Value(argc));
+ } else {
+ length = MArgumentsLength::New(alloc());
+ }
+ ins->block()->insertBefore(ins, length);
+
+ auto* compare = MCompare::New(alloc(), guardedIndex, length, JSOp::Lt,
+ MCompare::Compare_Int32);
+ ins->block()->insertBefore(ins, compare);
+ ins->replaceAllUsesWith(compare);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitArgumentsObjectLength(
+ MArgumentsObjectLength* ins) {
+ // Skip other arguments objects.
+ if (ins->argsObject() != args_) {
+ return;
+ }
+
+ MInstruction* length;
+ if (isInlinedArguments()) {
+ uint32_t argc = args_->toCreateInlinedArgumentsObject()->numActuals();
+ length = MConstant::New(alloc(), Int32Value(argc));
+ } else {
+ length = MArgumentsLength::New(alloc());
+ }
+ ins->block()->insertBefore(ins, length);
+ ins->replaceAllUsesWith(length);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitApplyArgsObj(MApplyArgsObj* ins) {
+ // Skip other arguments objects.
+ if (ins->getArgsObj() != args_) {
+ return;
+ }
+
+ MInstruction* newIns;
+ if (isInlinedArguments()) {
+ auto* actualArgs = args_->toCreateInlinedArgumentsObject();
+ CallInfo callInfo(alloc(), /*constructing=*/false,
+ ins->ignoresReturnValue());
+
+ callInfo.initForApplyInlinedArgs(ins->getFunction(), ins->getThis(),
+ actualArgs->numActuals());
+ for (uint32_t i = 0; i < actualArgs->numActuals(); i++) {
+ callInfo.initArg(i, actualArgs->getArg(i));
+ }
+
+ auto addUndefined = [this, &ins]() -> MConstant* {
+ MConstant* undef = MConstant::New(alloc(), UndefinedValue());
+ ins->block()->insertBefore(ins, undef);
+ return undef;
+ };
+
+ bool needsThisCheck = false;
+ bool isDOMCall = false;
+ auto* call = MakeCall(alloc(), addUndefined, callInfo, needsThisCheck,
+ ins->getSingleTarget(), isDOMCall);
+ if (!call) {
+ oom_ = true;
+ return;
+ }
+ if (!ins->maybeCrossRealm()) {
+ call->setNotCrossRealm();
+ }
+ newIns = call;
+ } else {
+ auto* numArgs = MArgumentsLength::New(alloc());
+ ins->block()->insertBefore(ins, numArgs);
+
+ // TODO: Should we rename MApplyArgs?
+ auto* apply = MApplyArgs::New(alloc(), ins->getSingleTarget(),
+ ins->getFunction(), numArgs, ins->getThis());
+ apply->setBailoutKind(ins->bailoutKind());
+ if (!ins->maybeCrossRealm()) {
+ apply->setNotCrossRealm();
+ }
+ if (ins->ignoresReturnValue()) {
+ apply->setIgnoresReturnValue();
+ }
+ newIns = apply;
+ }
+
+ ins->block()->insertBefore(ins, newIns);
+ ins->replaceAllUsesWith(newIns);
+
+ newIns->stealResumePoint(ins);
+ ins->block()->discard(ins);
+}
+
+MNewArrayObject* ArgumentsReplacer::inlineArgsArray(MInstruction* ins,
+ Shape* shape,
+ uint32_t begin,
+ uint32_t count) {
+ auto* actualArgs = args_->toCreateInlinedArgumentsObject();
+
+ // Contrary to |WarpBuilder::build_Rest()|, we can always create
+ // MNewArrayObject, because we're guaranteed to have a shape and all
+ // arguments can be stored into fixed elements.
+ static_assert(
+ gc::CanUseFixedElementsForArray(ArgumentsObject::MaxInlinedArgs));
+
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ // Allocate an array of the correct size.
+ auto* shapeConstant = MConstant::NewShape(alloc(), shape);
+ ins->block()->insertBefore(ins, shapeConstant);
+
+ auto* newArray = MNewArrayObject::New(alloc(), shapeConstant, count, heap);
+ ins->block()->insertBefore(ins, newArray);
+
+ if (count) {
+ auto* elements = MElements::New(alloc(), newArray);
+ ins->block()->insertBefore(ins, elements);
+
+ MConstant* index = nullptr;
+ for (uint32_t i = 0; i < count; i++) {
+ index = MConstant::New(alloc(), Int32Value(i));
+ ins->block()->insertBefore(ins, index);
+
+ MDefinition* arg = actualArgs->getArg(begin + i);
+ auto* store = MStoreElement::NewUnbarriered(alloc(), elements, index, arg,
+ /* needsHoleCheck = */ false);
+ ins->block()->insertBefore(ins, store);
+
+ auto* barrier = MPostWriteBarrier::New(alloc(), newArray, arg);
+ ins->block()->insertBefore(ins, barrier);
+ }
+
+ auto* initLength = MSetInitializedLength::New(alloc(), elements, index);
+ ins->block()->insertBefore(ins, initLength);
+ }
+
+ return newArray;
+}
+
+void ArgumentsReplacer::visitArrayFromArgumentsObject(
+ MArrayFromArgumentsObject* ins) {
+ // Skip other arguments objects.
+ if (ins->argsObject() != args_) {
+ return;
+ }
+
+ // We can only replace `arguments` because we've verified that the `arguments`
+ // object hasn't been modified in any way. This implies that the arguments
+ // stored in the stack frame haven't been changed either.
+ //
+ // The idea to replace `arguments` in spread calls `f(...arguments)` is now as
+ // follows:
+ // We replace |MArrayFromArgumentsObject| with the identical instructions we
+ // emit when building a rest-array object, cf. |WarpBuilder::build_Rest()|. In
+ // a next step, scalar replacement will then replace these new instructions
+ // themselves.
+
+ Shape* shape = ins->shape();
+ MOZ_ASSERT(shape);
+
+ MDefinition* replacement;
+ if (isInlinedArguments()) {
+ auto* actualArgs = args_->toCreateInlinedArgumentsObject();
+ uint32_t numActuals = actualArgs->numActuals();
+ MOZ_ASSERT(numActuals <= ArgumentsObject::MaxInlinedArgs);
+
+ replacement = inlineArgsArray(ins, shape, 0, numActuals);
+ } else {
+ // We can use |MRest| to read all arguments, because we've guaranteed that
+ // the arguments stored in the stack frame haven't changed; see the comment
+ // at the start of this method.
+
+ auto* numActuals = MArgumentsLength::New(alloc());
+ ins->block()->insertBefore(ins, numActuals);
+
+ // Set |numFormals| to zero to read all arguments, including any formals.
+ uint32_t numFormals = 0;
+
+ auto* rest = MRest::New(alloc(), numActuals, numFormals, shape);
+ ins->block()->insertBefore(ins, rest);
+
+ replacement = rest;
+ }
+
+ ins->replaceAllUsesWith(replacement);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+static uint32_t NormalizeSlice(MDefinition* def, uint32_t length) {
+ int32_t value = def->toConstant()->toInt32();
+ if (value < 0) {
+ return std::max(int32_t(uint32_t(value) + length), 0);
+ }
+ return std::min(uint32_t(value), length);
+}
+
+void ArgumentsReplacer::visitArgumentsSlice(MArgumentsSlice* ins) {
+ // Skip other arguments objects.
+ if (ins->object() != args_) {
+ return;
+ }
+
+ // Optimise the common pattern |Array.prototype.slice.call(arguments, begin)|,
+ // where |begin| is a non-negative, constant int32.
+ //
+ // An absent end-index is replaced by |arguments.length|, so we try to match
+ // |Array.prototype.slice.call(arguments, begin, arguments.length)|.
+ if (isInlinedArguments()) {
+ // When this is an inlined arguments, |arguments.length| has been replaced
+ // by a constant.
+ if (ins->begin()->isConstant() && ins->end()->isConstant()) {
+ auto* actualArgs = args_->toCreateInlinedArgumentsObject();
+ uint32_t numActuals = actualArgs->numActuals();
+ MOZ_ASSERT(numActuals <= ArgumentsObject::MaxInlinedArgs);
+
+ uint32_t begin = NormalizeSlice(ins->begin(), numActuals);
+ uint32_t end = NormalizeSlice(ins->end(), numActuals);
+ uint32_t count = end > begin ? end - begin : 0;
+ MOZ_ASSERT(count <= numActuals);
+
+ Shape* shape = ins->templateObj()->shape();
+ auto* newArray = inlineArgsArray(ins, shape, begin, count);
+
+ ins->replaceAllUsesWith(newArray);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+ return;
+ }
+ } else {
+ // Otherwise |arguments.length| is emitted as MArgumentsLength.
+ if (ins->begin()->isConstant() && ins->end()->isArgumentsLength()) {
+ int32_t begin = ins->begin()->toConstant()->toInt32();
+ if (begin >= 0) {
+ auto* numActuals = MArgumentsLength::New(alloc());
+ ins->block()->insertBefore(ins, numActuals);
+
+ // Set |numFormals| to read all arguments starting at |begin|.
+ uint32_t numFormals = begin;
+
+ Shape* shape = ins->templateObj()->shape();
+
+ // Use MRest because it can be scalar replaced, which enables further
+ // optimizations.
+ auto* rest = MRest::New(alloc(), numActuals, numFormals, shape);
+ ins->block()->insertBefore(ins, rest);
+
+ ins->replaceAllUsesWith(rest);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+ return;
+ }
+ }
+ }
+
+ MInstruction* numArgs;
+ if (isInlinedArguments()) {
+ uint32_t argc = args_->toCreateInlinedArgumentsObject()->numActuals();
+ numArgs = MConstant::New(alloc(), Int32Value(argc));
+ } else {
+ numArgs = MArgumentsLength::New(alloc());
+ }
+ ins->block()->insertBefore(ins, numArgs);
+
+ auto* begin = MNormalizeSliceTerm::New(alloc(), ins->begin(), numArgs);
+ ins->block()->insertBefore(ins, begin);
+
+ auto* end = MNormalizeSliceTerm::New(alloc(), ins->end(), numArgs);
+ ins->block()->insertBefore(ins, end);
+
+ bool isMax = false;
+ auto* beginMin = MMinMax::New(alloc(), begin, end, MIRType::Int32, isMax);
+ ins->block()->insertBefore(ins, beginMin);
+
+ // Safe to truncate because both operands are positive and end >= beginMin.
+ auto* count = MSub::New(alloc(), end, beginMin, MIRType::Int32);
+ count->setTruncateKind(TruncateKind::Truncate);
+ ins->block()->insertBefore(ins, count);
+
+ MInstruction* replacement;
+ if (isInlinedArguments()) {
+ auto* actualArgs = args_->toCreateInlinedArgumentsObject();
+ replacement =
+ MInlineArgumentsSlice::New(alloc(), beginMin, count, actualArgs,
+ ins->templateObj(), ins->initialHeap());
+ } else {
+ replacement = MFrameArgumentsSlice::New(
+ alloc(), beginMin, count, ins->templateObj(), ins->initialHeap());
+ }
+ ins->block()->insertBefore(ins, replacement);
+
+ ins->replaceAllUsesWith(replacement);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void ArgumentsReplacer::visitLoadFixedSlot(MLoadFixedSlot* ins) {
+ // Skip other arguments objects.
+ if (ins->object() != args_) {
+ return;
+ }
+
+ MOZ_ASSERT(ins->slot() == ArgumentsObject::CALLEE_SLOT);
+
+ MDefinition* replacement;
+ if (isInlinedArguments()) {
+ replacement = args_->toCreateInlinedArgumentsObject()->getCallee();
+ } else {
+ auto* callee = MCallee::New(alloc());
+ ins->block()->insertBefore(ins, callee);
+ replacement = callee;
+ }
+ ins->replaceAllUsesWith(replacement);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+static inline bool IsOptimizableRestInstruction(MInstruction* ins) {
+ return ins->isRest();
+}
+
+class RestReplacer : public MDefinitionVisitorDefaultNoop {
+ private:
+ MIRGenerator* mir_;
+ MIRGraph& graph_;
+ MInstruction* rest_;
+
+ TempAllocator& alloc() { return graph_.alloc(); }
+ MRest* rest() const { return rest_->toRest(); }
+
+ bool isRestElements(MDefinition* elements);
+ void discardInstruction(MInstruction* ins, MDefinition* elements);
+ MDefinition* restLength(MInstruction* ins);
+ void visitLength(MInstruction* ins, MDefinition* elements);
+
+ void visitGuardToClass(MGuardToClass* ins);
+ void visitGuardShape(MGuardShape* ins);
+ void visitGuardArrayIsPacked(MGuardArrayIsPacked* ins);
+ void visitUnbox(MUnbox* ins);
+ void visitCompare(MCompare* ins);
+ void visitLoadElement(MLoadElement* ins);
+ void visitArrayLength(MArrayLength* ins);
+ void visitInitializedLength(MInitializedLength* ins);
+ void visitApplyArray(MApplyArray* ins);
+ void visitConstructArray(MConstructArray* ins);
+
+ bool escapes(MElements* ins);
+
+ public:
+ RestReplacer(MIRGenerator* mir, MIRGraph& graph, MInstruction* rest)
+ : mir_(mir), graph_(graph), rest_(rest) {
+ MOZ_ASSERT(IsOptimizableRestInstruction(rest_));
+ }
+
+ bool escapes(MInstruction* ins);
+ bool run();
+ void assertSuccess();
+};
+
+// Returns false if the rest array object does not escape.
+bool RestReplacer::escapes(MInstruction* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ JitSpewDef(JitSpew_Escape, "Check rest array\n", ins);
+ JitSpewIndent spewIndent(JitSpew_Escape);
+
+ // We can replace rest arrays in scripts with OSR entries, but the outermost
+ // rest object has already been allocated before we enter via OSR and can't be
+ // replaced.
+ // See also the same restriction when replacing |arguments|.
+ if (graph_.osrBlock()) {
+ JitSpew(JitSpew_Escape, "Can't replace outermost OSR rest array");
+ return true;
+ }
+
+ // Check all uses to see whether they can be supported without allocating an
+ // ArrayObject for the rest parameter.
+ for (MUseIterator i(ins->usesBegin()); i != ins->usesEnd(); i++) {
+ MNode* consumer = (*i)->consumer();
+
+ // If a resume point can observe this instruction, we can only optimize
+ // if it is recoverable.
+ if (consumer->isResumePoint()) {
+ if (!consumer->toResumePoint()->isRecoverableOperand(*i)) {
+ JitSpew(JitSpew_Escape, "Observable rest array cannot be recovered");
+ return true;
+ }
+ continue;
+ }
+
+ MDefinition* def = consumer->toDefinition();
+ switch (def->op()) {
+ case MDefinition::Opcode::Elements: {
+ auto* elem = def->toElements();
+ MOZ_ASSERT(elem->object() == ins);
+ if (escapes(elem)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::GuardShape: {
+ const Shape* shape = rest()->shape();
+ if (!shape) {
+ JitSpew(JitSpew_Escape, "No shape defined.");
+ return true;
+ }
+
+ auto* guard = def->toGuardShape();
+ if (shape != guard->shape()) {
+ JitSpewDef(JitSpew_Escape, "has a non-matching guard shape\n", def);
+ return true;
+ }
+ if (escapes(guard)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::GuardToClass: {
+ auto* guard = def->toGuardToClass();
+ if (guard->getClass() != &ArrayObject::class_) {
+ JitSpewDef(JitSpew_Escape, "has a non-matching class guard\n", def);
+ return true;
+ }
+ if (escapes(guard)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::GuardArrayIsPacked: {
+ // Rest arrays are always packed as long as they aren't modified.
+ auto* guard = def->toGuardArrayIsPacked();
+ if (escapes(guard)) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ case MDefinition::Opcode::Unbox: {
+ if (def->type() != MIRType::Object) {
+ JitSpewDef(JitSpew_Escape, "has an invalid unbox\n", def);
+ return true;
+ }
+ if (escapes(def->toInstruction())) {
+ JitSpewDef(JitSpew_Escape, "is indirectly escaped by\n", def);
+ return true;
+ }
+ break;
+ }
+
+ // This instruction is supported for |JSOp::OptimizeSpreadCall|.
+ case MDefinition::Opcode::Compare: {
+ bool canFold;
+ if (!def->toCompare()->tryFold(&canFold)) {
+ JitSpewDef(JitSpew_Escape, "has an unsupported compare\n", def);
+ return true;
+ }
+ break;
+ }
+
+ // This instruction is a no-op used to test that scalar replacement is
+ // working as expected.
+ case MDefinition::Opcode::AssertRecoveredOnBailout:
+ break;
+
+ default:
+ JitSpewDef(JitSpew_Escape, "is escaped by\n", def);
+ return true;
+ }
+ }
+
+ JitSpew(JitSpew_Escape, "Rest array object is not escaped");
+ return false;
+}
+
+bool RestReplacer::escapes(MElements* ins) {
+ JitSpewDef(JitSpew_Escape, "Check rest array elements\n", ins);
+ JitSpewIndent spewIndent(JitSpew_Escape);
+
+ for (MUseIterator i(ins->usesBegin()); i != ins->usesEnd(); i++) {
+ // The MIRType::Elements cannot be captured in a resume point as it does not
+ // represent a value allocation.
+ MDefinition* def = (*i)->consumer()->toDefinition();
+
+ switch (def->op()) {
+ case MDefinition::Opcode::LoadElement:
+ MOZ_ASSERT(def->toLoadElement()->elements() == ins);
+ break;
+
+ case MDefinition::Opcode::ArrayLength:
+ MOZ_ASSERT(def->toArrayLength()->elements() == ins);
+ break;
+
+ case MDefinition::Opcode::InitializedLength:
+ MOZ_ASSERT(def->toInitializedLength()->elements() == ins);
+ break;
+
+ case MDefinition::Opcode::ApplyArray:
+ MOZ_ASSERT(def->toApplyArray()->getElements() == ins);
+ break;
+
+ case MDefinition::Opcode::ConstructArray:
+ MOZ_ASSERT(def->toConstructArray()->getElements() == ins);
+ break;
+
+ default:
+ JitSpewDef(JitSpew_Escape, "is escaped by\n", def);
+ return true;
+ }
+ }
+
+ JitSpew(JitSpew_Escape, "Rest array object is not escaped");
+ return false;
+}
+
+// Replacing the rest array object is simpler than replacing an object or array,
+// because the rest array object does not change state.
+bool RestReplacer::run() {
+ MBasicBlock* startBlock = rest_->block();
+
+ // Iterate over each basic block.
+ for (ReversePostorderIterator block = graph_.rpoBegin(startBlock);
+ block != graph_.rpoEnd(); block++) {
+ if (mir_->shouldCancel("Scalar replacement of rest array object")) {
+ return false;
+ }
+
+ // Iterates over phis and instructions.
+ // We do not have to visit resume points. Any resume points that capture the
+ // rest array object will be handled by the Sink pass.
+ for (MDefinitionIterator iter(*block); iter;) {
+ // Increment the iterator before visiting the instruction, as the visit
+ // function might discard itself from the basic block.
+ MDefinition* def = *iter++;
+ switch (def->op()) {
+#define MIR_OP(op) \
+ case MDefinition::Opcode::op: \
+ visit##op(def->to##op()); \
+ break;
+ MIR_OPCODE_LIST(MIR_OP)
+#undef MIR_OP
+ }
+ if (!graph_.alloc().ensureBallast()) {
+ return false;
+ }
+ }
+ }
+
+ assertSuccess();
+ return true;
+}
+
+void RestReplacer::assertSuccess() {
+ MOZ_ASSERT(rest_->canRecoverOnBailout());
+ MOZ_ASSERT(!rest_->hasLiveDefUses());
+}
+
+bool RestReplacer::isRestElements(MDefinition* elements) {
+ return elements->isElements() && elements->toElements()->object() == rest_;
+}
+
+void RestReplacer::discardInstruction(MInstruction* ins,
+ MDefinition* elements) {
+ MOZ_ASSERT(elements->isElements());
+ ins->block()->discard(ins);
+ if (!elements->hasLiveDefUses()) {
+ elements->block()->discard(elements->toInstruction());
+ }
+}
+
+void RestReplacer::visitGuardToClass(MGuardToClass* ins) {
+ // Skip guards on other objects.
+ if (ins->object() != rest_) {
+ return;
+ }
+ MOZ_ASSERT(ins->getClass() == &ArrayObject::class_);
+
+ // Replace the guard with the array object.
+ ins->replaceAllUsesWith(rest_);
+
+ // Remove the guard.
+ ins->block()->discard(ins);
+}
+
+void RestReplacer::visitGuardShape(MGuardShape* ins) {
+ // Skip guards on other objects.
+ if (ins->object() != rest_) {
+ return;
+ }
+
+ // Replace the guard with the array object.
+ ins->replaceAllUsesWith(rest_);
+
+ // Remove the guard.
+ ins->block()->discard(ins);
+}
+
+void RestReplacer::visitGuardArrayIsPacked(MGuardArrayIsPacked* ins) {
+ // Skip guards on other objects.
+ if (ins->array() != rest_) {
+ return;
+ }
+
+ // Replace the guard by its object.
+ ins->replaceAllUsesWith(rest_);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void RestReplacer::visitUnbox(MUnbox* ins) {
+ // Skip unrelated unboxes.
+ if (ins->input() != rest_) {
+ return;
+ }
+ MOZ_ASSERT(ins->type() == MIRType::Object);
+
+ // Replace the unbox with the array object.
+ ins->replaceAllUsesWith(rest_);
+
+ // Remove the unbox.
+ ins->block()->discard(ins);
+}
+
+void RestReplacer::visitCompare(MCompare* ins) {
+ // Skip unrelated comparisons.
+ if (ins->lhs() != rest_ && ins->rhs() != rest_) {
+ return;
+ }
+
+ bool folded;
+ MOZ_ALWAYS_TRUE(ins->tryFold(&folded));
+
+ auto* cst = MConstant::New(alloc(), BooleanValue(folded));
+ ins->block()->insertBefore(ins, cst);
+
+ // Replace the comparison with a constant.
+ ins->replaceAllUsesWith(cst);
+
+ // Remove original instruction.
+ ins->block()->discard(ins);
+}
+
+void RestReplacer::visitLoadElement(MLoadElement* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->elements();
+ if (!isRestElements(elements)) {
+ return;
+ }
+
+ MDefinition* index = ins->index();
+
+ // Adjust the index to skip any extra formals.
+ if (uint32_t formals = rest()->numFormals()) {
+ auto* numFormals = MConstant::New(alloc(), Int32Value(formals));
+ ins->block()->insertBefore(ins, numFormals);
+
+ auto* add = MAdd::New(alloc(), index, numFormals, TruncateKind::Truncate);
+ ins->block()->insertBefore(ins, add);
+
+ index = add;
+ }
+
+ auto* loadArg = MGetFrameArgument::New(alloc(), index);
+
+ ins->block()->insertBefore(ins, loadArg);
+ ins->replaceAllUsesWith(loadArg);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+MDefinition* RestReplacer::restLength(MInstruction* ins) {
+ // Compute |Math.max(numActuals - numFormals, 0)| for the rest array length.
+
+ auto* numActuals = rest()->numActuals();
+
+ if (uint32_t formals = rest()->numFormals()) {
+ auto* numFormals = MConstant::New(alloc(), Int32Value(formals));
+ ins->block()->insertBefore(ins, numFormals);
+
+ auto* length = MSub::New(alloc(), numActuals, numFormals, MIRType::Int32);
+ length->setTruncateKind(TruncateKind::Truncate);
+ ins->block()->insertBefore(ins, length);
+
+ auto* zero = MConstant::New(alloc(), Int32Value(0));
+ ins->block()->insertBefore(ins, zero);
+
+ bool isMax = true;
+ auto* minmax = MMinMax::New(alloc(), length, zero, MIRType::Int32, isMax);
+ ins->block()->insertBefore(ins, minmax);
+
+ return minmax;
+ }
+
+ return numActuals;
+}
+
+void RestReplacer::visitLength(MInstruction* ins, MDefinition* elements) {
+ MOZ_ASSERT(ins->isArrayLength() || ins->isInitializedLength());
+
+ // Skip other array objects.
+ if (!isRestElements(elements)) {
+ return;
+ }
+
+ MDefinition* replacement = restLength(ins);
+
+ ins->replaceAllUsesWith(replacement);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+void RestReplacer::visitArrayLength(MArrayLength* ins) {
+ visitLength(ins, ins->elements());
+}
+
+void RestReplacer::visitInitializedLength(MInitializedLength* ins) {
+ // The initialized length of a rest array is equal to its length.
+ visitLength(ins, ins->elements());
+}
+
+void RestReplacer::visitApplyArray(MApplyArray* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->getElements();
+ if (!isRestElements(elements)) {
+ return;
+ }
+
+ auto* numActuals = restLength(ins);
+
+ auto* apply =
+ MApplyArgs::New(alloc(), ins->getSingleTarget(), ins->getFunction(),
+ numActuals, ins->getThis(), rest()->numFormals());
+ apply->setBailoutKind(ins->bailoutKind());
+ if (!ins->maybeCrossRealm()) {
+ apply->setNotCrossRealm();
+ }
+ if (ins->ignoresReturnValue()) {
+ apply->setIgnoresReturnValue();
+ }
+ ins->block()->insertBefore(ins, apply);
+
+ ins->replaceAllUsesWith(apply);
+
+ apply->stealResumePoint(ins);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+void RestReplacer::visitConstructArray(MConstructArray* ins) {
+ // Skip other array objects.
+ MDefinition* elements = ins->getElements();
+ if (!isRestElements(elements)) {
+ return;
+ }
+
+ auto* numActuals = restLength(ins);
+
+ auto* construct = MConstructArgs::New(
+ alloc(), ins->getSingleTarget(), ins->getFunction(), numActuals,
+ ins->getThis(), ins->getNewTarget(), rest()->numFormals());
+ construct->setBailoutKind(ins->bailoutKind());
+ if (!ins->maybeCrossRealm()) {
+ construct->setNotCrossRealm();
+ }
+
+ ins->block()->insertBefore(ins, construct);
+ ins->replaceAllUsesWith(construct);
+
+ construct->stealResumePoint(ins);
+
+ // Remove original instruction.
+ discardInstruction(ins, elements);
+}
+
+bool ScalarReplacement(MIRGenerator* mir, MIRGraph& graph) {
+ JitSpew(JitSpew_Escape, "Begin (ScalarReplacement)");
+
+ EmulateStateOf<ObjectMemoryView> replaceObject(mir, graph);
+ EmulateStateOf<ArrayMemoryView> replaceArray(mir, graph);
+ bool addedPhi = false;
+
+ for (ReversePostorderIterator block = graph.rpoBegin();
+ block != graph.rpoEnd(); block++) {
+ if (mir->shouldCancel("Scalar Replacement (main loop)")) {
+ return false;
+ }
+
+ for (MInstructionIterator ins = block->begin(); ins != block->end();
+ ins++) {
+ if (IsOptimizableObjectInstruction(*ins) &&
+ !IsObjectEscaped(*ins, *ins)) {
+ ObjectMemoryView view(graph.alloc(), *ins);
+ if (!replaceObject.run(view)) {
+ return false;
+ }
+ view.assertSuccess();
+ addedPhi = true;
+ continue;
+ }
+
+ if (IsOptimizableArrayInstruction(*ins) && !IsArrayEscaped(*ins, *ins)) {
+ ArrayMemoryView view(graph.alloc(), *ins);
+ if (!replaceArray.run(view)) {
+ return false;
+ }
+ view.assertSuccess();
+ addedPhi = true;
+ continue;
+ }
+
+ if (IsOptimizableArgumentsInstruction(*ins)) {
+ ArgumentsReplacer replacer(mir, graph, *ins);
+ if (replacer.escapes(*ins)) {
+ continue;
+ }
+ if (!replacer.run()) {
+ return false;
+ }
+ continue;
+ }
+
+ if (IsOptimizableRestInstruction(*ins)) {
+ RestReplacer replacer(mir, graph, *ins);
+ if (replacer.escapes(*ins)) {
+ continue;
+ }
+ if (!replacer.run()) {
+ return false;
+ }
+ continue;
+ }
+ }
+ }
+
+ if (addedPhi) {
+ // Phis added by Scalar Replacement are only redundant Phis which are
+ // not directly captured by any resume point but only by the MDefinition
+ // state. The conservative observability only focuses on Phis which are
+ // not used as resume points operands.
+ AssertExtendedGraphCoherency(graph);
+ if (!EliminatePhis(mir, graph, ConservativeObservability)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+} /* namespace jit */
+} /* namespace js */
diff --git a/js/src/jit/ScalarReplacement.h b/js/src/jit/ScalarReplacement.h
new file mode 100644
index 0000000000..1b46604aa4
--- /dev/null
+++ b/js/src/jit/ScalarReplacement.h
@@ -0,0 +1,22 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file declares scalar replacement of objects transformation.
+#ifndef jit_ScalarReplacement_h
+#define jit_ScalarReplacement_h
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+class MIRGraph;
+
+[[nodiscard]] bool ScalarReplacement(MIRGenerator* mir, MIRGraph& graph);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ScalarReplacement_h */
diff --git a/js/src/jit/ScalarTypeUtils.h b/js/src/jit/ScalarTypeUtils.h
new file mode 100644
index 0000000000..19e904d9b8
--- /dev/null
+++ b/js/src/jit/ScalarTypeUtils.h
@@ -0,0 +1,41 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ScalarTypeUtils_h
+#define jit_ScalarTypeUtils_h
+
+#include "mozilla/CheckedInt.h"
+
+#include <stdint.h>
+
+#include "js/ScalarType.h"
+
+namespace js {
+namespace jit {
+
+// Compute |index * Scalar::byteSize(type) + offsetAdjustment|. If this doesn't
+// overflow and is non-negative, return true and store the result in *offset.
+// If the computation overflows or the result is negative, false is returned and
+// *offset is left unchanged.
+[[nodiscard]] inline bool ArrayOffsetFitsInInt32(int32_t index,
+ Scalar::Type type,
+ int32_t offsetAdjustment,
+ int32_t* offset) {
+ mozilla::CheckedInt<int32_t> val = index;
+ val *= Scalar::byteSize(type);
+ val += offsetAdjustment;
+ if (!val.isValid() || val.value() < 0) {
+ return false;
+ }
+
+ *offset = val.value();
+ return true;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ScalarTypeUtils_h */
diff --git a/js/src/jit/ScriptFromCalleeToken.h b/js/src/jit/ScriptFromCalleeToken.h
new file mode 100644
index 0000000000..a4642893a6
--- /dev/null
+++ b/js/src/jit/ScriptFromCalleeToken.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ScriptFromCalleeToken_h
+#define jit_ScriptFromCalleeToken_h
+
+#include "mozilla/Assertions.h"
+
+#include "jit/CalleeToken.h"
+#include "js/TypeDecls.h"
+#include "vm/JSFunction.h"
+
+namespace js::jit {
+
+static inline JSScript* ScriptFromCalleeToken(CalleeToken token) {
+ switch (GetCalleeTokenTag(token)) {
+ case CalleeToken_Script:
+ return CalleeTokenToScript(token);
+ case CalleeToken_Function:
+ case CalleeToken_FunctionConstructing:
+ return CalleeTokenToFunction(token)->nonLazyScript();
+ }
+ MOZ_CRASH("invalid callee token tag");
+}
+
+JSScript* MaybeForwardedScriptFromCalleeToken(CalleeToken token);
+
+} /* namespace js::jit */
+
+#endif /* jit_ScriptFromCalleeToken_h */
diff --git a/js/src/jit/SharedICHelpers-inl.h b/js/src/jit/SharedICHelpers-inl.h
new file mode 100644
index 0000000000..ee257283ca
--- /dev/null
+++ b/js/src/jit/SharedICHelpers-inl.h
@@ -0,0 +1,34 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_SharedICHelpers_inl_h
+#define jit_SharedICHelpers_inl_h
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/SharedICHelpers-x86-inl.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/SharedICHelpers-x64-inl.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/SharedICHelpers-arm-inl.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/SharedICHelpers-arm64-inl.h"
+#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+# include "jit/mips-shared/SharedICHelpers-mips-shared-inl.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/SharedICHelpers-loong64-inl.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/SharedICHelpers-wasm32-inl.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/SharedICHelpers-none-inl.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+namespace js {
+namespace jit {} // namespace jit
+} // namespace js
+
+#endif /* jit_SharedICHelpers_inl_h */
diff --git a/js/src/jit/SharedICHelpers.h b/js/src/jit/SharedICHelpers.h
new file mode 100644
index 0000000000..95da74f4ca
--- /dev/null
+++ b/js/src/jit/SharedICHelpers.h
@@ -0,0 +1,34 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_SharedICHelpers_h
+#define jit_SharedICHelpers_h
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/SharedICHelpers-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/SharedICHelpers-x64.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/SharedICHelpers-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/SharedICHelpers-arm64.h"
+#elif defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
+# include "jit/mips-shared/SharedICHelpers-mips-shared.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/SharedICHelpers-loong64.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/SharedICHelpers-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/SharedICHelpers-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+namespace js {
+namespace jit {} // namespace jit
+} // namespace js
+
+#endif /* jit_SharedICHelpers_h */
diff --git a/js/src/jit/SharedICRegisters.h b/js/src/jit/SharedICRegisters.h
new file mode 100644
index 0000000000..9905e6728d
--- /dev/null
+++ b/js/src/jit/SharedICRegisters.h
@@ -0,0 +1,36 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_SharedICRegisters_h
+#define jit_SharedICRegisters_h
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/SharedICRegisters-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/SharedICRegisters-x64.h"
+#elif defined(JS_CODEGEN_ARM)
+# include "jit/arm/SharedICRegisters-arm.h"
+#elif defined(JS_CODEGEN_ARM64)
+# include "jit/arm64/SharedICRegisters-arm64.h"
+#elif defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/SharedICRegisters-mips32.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/SharedICRegisters-mips64.h"
+#elif defined(JS_CODEGEN_LOONG64)
+# include "jit/loong64/SharedICRegisters-loong64.h"
+#elif defined(JS_CODEGEN_WASM32)
+# include "jit/wasm32/SharedICRegisters-wasm32.h"
+#elif defined(JS_CODEGEN_NONE)
+# include "jit/none/SharedICRegisters-none.h"
+#else
+# error "Unknown architecture!"
+#endif
+
+namespace js {
+namespace jit {} // namespace jit
+} // namespace js
+
+#endif /* jit_SharedICRegisters_h */
diff --git a/js/src/jit/ShuffleAnalysis.cpp b/js/src/jit/ShuffleAnalysis.cpp
new file mode 100644
index 0000000000..20158b18d4
--- /dev/null
+++ b/js/src/jit/ShuffleAnalysis.cpp
@@ -0,0 +1,747 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ShuffleAnalysis.h"
+#include "jit/MIR.h"
+
+using namespace js;
+using namespace jit;
+
+using mozilla::Maybe;
+using mozilla::Nothing;
+using mozilla::Some;
+
+#ifdef ENABLE_WASM_SIMD
+
+// Specialization analysis for SIMD operations. This is still x86-centric but
+// generalizes fairly easily to other architectures.
+
+// Optimization of v8x16.shuffle. The general byte shuffle+blend is very
+// expensive (equivalent to at least a dozen instructions), and we want to avoid
+// that if we can. So look for special cases - there are many.
+//
+// The strategy is to sort the operation into one of three buckets depending
+// on the shuffle pattern and inputs:
+//
+// - single operand; shuffles on these values are rotations, reversals,
+// transpositions, and general permutations
+// - single-operand-with-interesting-constant (especially zero); shuffles on
+// these values are often byte shift or scatter operations
+// - dual operand; shuffles on these operations are blends, catenated
+// shifts, and (in the worst case) general shuffle+blends
+//
+// We're not trying to solve the general problem, only to lower reasonably
+// expressed patterns that express common operations. Producers that produce
+// dense and convoluted patterns will end up with the general byte shuffle.
+// Producers that produce simpler patterns that easily map to hardware will
+// get faster code.
+//
+// In particular, these matchers do not try to combine transformations, so a
+// shuffle that optimally is lowered to rotate + permute32x4 + rotate, say, is
+// usually going to end up as a general byte shuffle.
+
+// Reduce a 0..31 byte mask to a 0..15 word mask if possible and if so return
+// true, updating *control.
+static bool ByteMaskToWordMask(SimdConstant* control) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ int16_t controlWords[8];
+ for (int i = 0; i < 16; i += 2) {
+ if (!((lanes[i] & 1) == 0 && lanes[i + 1] == lanes[i] + 1)) {
+ return false;
+ }
+ controlWords[i / 2] = int16_t(lanes[i] / 2);
+ }
+ *control = SimdConstant::CreateX8(controlWords);
+ return true;
+}
+
+// Reduce a 0..31 byte mask to a 0..7 dword mask if possible and if so return
+// true, updating *control.
+static bool ByteMaskToDWordMask(SimdConstant* control) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ int32_t controlDWords[4];
+ for (int i = 0; i < 16; i += 4) {
+ if (!((lanes[i] & 3) == 0 && lanes[i + 1] == lanes[i] + 1 &&
+ lanes[i + 2] == lanes[i] + 2 && lanes[i + 3] == lanes[i] + 3)) {
+ return false;
+ }
+ controlDWords[i / 4] = lanes[i] / 4;
+ }
+ *control = SimdConstant::CreateX4(controlDWords);
+ return true;
+}
+
+// Reduce a 0..31 byte mask to a 0..3 qword mask if possible and if so return
+// true, updating *control.
+static bool ByteMaskToQWordMask(SimdConstant* control) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ int64_t controlQWords[2];
+ for (int i = 0; i < 16; i += 8) {
+ if (!((lanes[i] & 7) == 0 && lanes[i + 1] == lanes[i] + 1 &&
+ lanes[i + 2] == lanes[i] + 2 && lanes[i + 3] == lanes[i] + 3 &&
+ lanes[i + 4] == lanes[i] + 4 && lanes[i + 5] == lanes[i] + 5 &&
+ lanes[i + 6] == lanes[i] + 6 && lanes[i + 7] == lanes[i] + 7)) {
+ return false;
+ }
+ controlQWords[i / 8] = lanes[i] / 8;
+ }
+ *control = SimdConstant::CreateX2(controlQWords);
+ return true;
+}
+
+// Skip across consecutive values in lanes starting at i, returning the index
+// after the last element. Lane values must be <= len-1 ("masked").
+//
+// Since every element is a 1-element run, the return value is never the same as
+// the starting i.
+template <typename T>
+static int ScanIncreasingMasked(const T* lanes, int i) {
+ int len = int(16 / sizeof(T));
+ MOZ_ASSERT(i < len);
+ MOZ_ASSERT(lanes[i] <= len - 1);
+ i++;
+ while (i < len && lanes[i] == lanes[i - 1] + 1) {
+ MOZ_ASSERT(lanes[i] <= len - 1);
+ i++;
+ }
+ return i;
+}
+
+// Skip across consecutive values in lanes starting at i, returning the index
+// after the last element. Lane values must be <= len*2-1 ("unmasked"); the
+// values len-1 and len are not considered consecutive.
+//
+// Since every element is a 1-element run, the return value is never the same as
+// the starting i.
+template <typename T>
+static int ScanIncreasingUnmasked(const T* lanes, int i) {
+ int len = int(16 / sizeof(T));
+ MOZ_ASSERT(i < len);
+ if (lanes[i] < len) {
+ i++;
+ while (i < len && lanes[i] < len && lanes[i - 1] == lanes[i] - 1) {
+ i++;
+ }
+ } else {
+ i++;
+ while (i < len && lanes[i] >= len && lanes[i - 1] == lanes[i] - 1) {
+ i++;
+ }
+ }
+ return i;
+}
+
+// Skip lanes that equal v starting at i, returning the index just beyond the
+// last of those. There is no requirement that the initial lanes[i] == v.
+template <typename T>
+static int ScanConstant(const T* lanes, int v, int i) {
+ int len = int(16 / sizeof(T));
+ MOZ_ASSERT(i <= len);
+ while (i < len && lanes[i] == v) {
+ i++;
+ }
+ return i;
+}
+
+// Mask lane values denoting rhs elements into lhs elements.
+template <typename T>
+static void MaskLanes(T* result, const T* input) {
+ int len = int(16 / sizeof(T));
+ for (int i = 0; i < len; i++) {
+ result[i] = input[i] & (len - 1);
+ }
+}
+
+// Apply a transformation to each lane value.
+template <typename T>
+static void MapLanes(T* result, const T* input, int (*f)(int)) {
+ // Hazard analysis trips on "IndirectCall: f" error.
+ // Suppress the check -- `f` is expected to be trivial here.
+ JS::AutoSuppressGCAnalysis nogc;
+
+ int len = int(16 / sizeof(T));
+ for (int i = 0; i < len; i++) {
+ result[i] = f(input[i]);
+ }
+}
+
+// Recognize an identity permutation, assuming lanes is masked.
+template <typename T>
+static bool IsIdentity(const T* lanes) {
+ return ScanIncreasingMasked(lanes, 0) == int(16 / sizeof(T));
+}
+
+// Recognize part of an identity permutation starting at start, with
+// the first value of the permutation expected to be bias.
+template <typename T>
+static bool IsIdentity(const T* lanes, int start, int len, int bias) {
+ if (lanes[start] != bias) {
+ return false;
+ }
+ for (int i = start + 1; i < start + len; i++) {
+ if (lanes[i] != lanes[i - 1] + 1) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// We can permute by dwords if the mask is reducible to a dword mask, and in
+// this case a single PSHUFD is enough.
+static bool TryPermute32x4(SimdConstant* control) {
+ SimdConstant tmp = *control;
+ if (!ByteMaskToDWordMask(&tmp)) {
+ return false;
+ }
+ *control = tmp;
+ return true;
+}
+
+// Can we perform a byte rotate right? We can use PALIGNR. The shift count is
+// just lanes[0], and *control is unchanged.
+static bool TryRotateRight8x16(SimdConstant* control) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ // Look for the end of the first run of consecutive bytes.
+ int i = ScanIncreasingMasked(lanes, 0);
+
+ // First run must start at a value s.t. we have a rotate if all remaining
+ // bytes are a run.
+ if (lanes[0] != 16 - i) {
+ return false;
+ }
+
+ // If we reached the end of the vector, we're done.
+ if (i == 16) {
+ return true;
+ }
+
+ // Second run must start at source lane zero.
+ if (lanes[i] != 0) {
+ return false;
+ }
+
+ // Second run must end at the end of the lane vector.
+ return ScanIncreasingMasked(lanes, i) == 16;
+}
+
+// We can permute by words if the mask is reducible to a word mask.
+static bool TryPermute16x8(SimdConstant* control) {
+ SimdConstant tmp = *control;
+ if (!ByteMaskToWordMask(&tmp)) {
+ return false;
+ }
+ *control = tmp;
+ return true;
+}
+
+// A single word lane is copied into all the other lanes: PSHUF*W + PSHUFD.
+static bool TryBroadcast16x8(SimdConstant* control) {
+ SimdConstant tmp = *control;
+ if (!ByteMaskToWordMask(&tmp)) {
+ return false;
+ }
+ const SimdConstant::I16x8& lanes = tmp.asInt16x8();
+ if (ScanConstant(lanes, lanes[0], 0) < 8) {
+ return false;
+ }
+ *control = tmp;
+ return true;
+}
+
+// A single byte lane is copied int all the other lanes: PUNPCK*BW + PSHUF*W +
+// PSHUFD.
+static bool TryBroadcast8x16(SimdConstant* control) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ return ScanConstant(lanes, lanes[0], 0) >= 16;
+}
+
+template <int N>
+static bool TryReverse(SimdConstant* control) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ for (int i = 0; i < 16; i++) {
+ if (lanes[i] != (i ^ (N - 1))) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Look for permutations of a single operand.
+static SimdPermuteOp AnalyzePermute(SimdConstant* control) {
+ // Lane indices are input-agnostic for single-operand permutations.
+ SimdConstant::I8x16 controlBytes;
+ MaskLanes(controlBytes, control->asInt8x16());
+
+ // Get rid of no-ops immediately, so nobody else needs to check.
+ if (IsIdentity(controlBytes)) {
+ return SimdPermuteOp::MOVE;
+ }
+
+ // Default control is the masked bytes.
+ *control = SimdConstant::CreateX16(controlBytes);
+
+ // Analysis order matters here and is architecture-dependent or even
+ // microarchitecture-dependent: ideally the cheapest implementation first.
+ // The Intel manual says that the cost of a PSHUFB is about five other
+ // operations, so make that our cutoff.
+ //
+ // Word, dword, and qword reversals are handled optimally by general permutes.
+ //
+ // Byte reversals are probably best left to PSHUFB, no alternative rendition
+ // seems to reliably go below five instructions. (Discuss.)
+ //
+ // Word swaps within doublewords and dword swaps within quadwords are handled
+ // optimally by general permutes.
+ //
+ // Dword and qword broadcasts are handled by dword permute.
+
+ if (TryPermute32x4(control)) {
+ return SimdPermuteOp::PERMUTE_32x4;
+ }
+ if (TryRotateRight8x16(control)) {
+ return SimdPermuteOp::ROTATE_RIGHT_8x16;
+ }
+ if (TryBroadcast16x8(control)) {
+ return SimdPermuteOp::BROADCAST_16x8;
+ }
+ if (TryPermute16x8(control)) {
+ return SimdPermuteOp::PERMUTE_16x8;
+ }
+ if (TryBroadcast8x16(control)) {
+ return SimdPermuteOp::BROADCAST_8x16;
+ }
+ if (TryReverse<2>(control)) {
+ return SimdPermuteOp::REVERSE_16x8;
+ }
+ if (TryReverse<4>(control)) {
+ return SimdPermuteOp::REVERSE_32x4;
+ }
+ if (TryReverse<8>(control)) {
+ return SimdPermuteOp::REVERSE_64x2;
+ }
+
+ // TODO: (From v8) Unzip and transpose generally have renditions that slightly
+ // beat a general permute (three or four instructions)
+ //
+ // TODO: (From MacroAssemblerX86Shared::ShuffleX4): MOVLHPS and MOVHLPS can be
+ // used when merging two values.
+
+ // The default operation is to permute bytes with the default control.
+ return SimdPermuteOp::PERMUTE_8x16;
+}
+
+// Can we shift the bytes left or right by a constant? A shift is a run of
+// lanes from the rhs (which is zero) on one end and a run of values from the
+// lhs on the other end.
+static Maybe<SimdPermuteOp> TryShift8x16(SimdConstant* control) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+
+ // Represent all zero lanes by 16
+ SimdConstant::I8x16 zeroesMasked;
+ MapLanes(zeroesMasked, lanes, [](int x) -> int { return x >= 16 ? 16 : x; });
+
+ int i = ScanConstant(zeroesMasked, 16, 0);
+ int shiftLeft = i;
+ if (shiftLeft > 0 && lanes[shiftLeft] != 0) {
+ return Nothing();
+ }
+
+ i = ScanIncreasingUnmasked(zeroesMasked, i);
+ int shiftRight = 16 - i;
+ if (shiftRight > 0 && lanes[i - 1] != 15) {
+ return Nothing();
+ }
+
+ i = ScanConstant(zeroesMasked, 16, i);
+ if (i < 16 || (shiftRight > 0 && shiftLeft > 0) ||
+ (shiftRight == 0 && shiftLeft == 0)) {
+ return Nothing();
+ }
+
+ if (shiftRight) {
+ *control = SimdConstant::SplatX16((int8_t)shiftRight);
+ return Some(SimdPermuteOp::SHIFT_RIGHT_8x16);
+ }
+ *control = SimdConstant::SplatX16((int8_t)shiftLeft);
+ return Some(SimdPermuteOp::SHIFT_LEFT_8x16);
+}
+
+static Maybe<SimdPermuteOp> AnalyzeShuffleWithZero(SimdConstant* control) {
+ Maybe<SimdPermuteOp> op;
+ op = TryShift8x16(control);
+ if (op) {
+ return op;
+ }
+
+ // TODO: Optimization opportunity? A byte-blend-with-zero is just a CONST;
+ // PAND. This may beat the general byte blend code below.
+ return Nothing();
+}
+
+// Concat: if the result is the suffix (high bytes) of the rhs in front of a
+// prefix (low bytes) of the lhs then this is PALIGNR; ditto if the operands are
+// swapped.
+static Maybe<SimdShuffleOp> TryConcatRightShift8x16(SimdConstant* control,
+ bool* swapOperands) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ int i = ScanIncreasingUnmasked(lanes, 0);
+ MOZ_ASSERT(i < 16, "Single-operand run should have been handled elswhere");
+ // First run must end with 15 % 16
+ if ((lanes[i - 1] & 15) != 15) {
+ return Nothing();
+ }
+ // Second run must start with 0 % 16
+ if ((lanes[i] & 15) != 0) {
+ return Nothing();
+ }
+ // The two runs must come from different inputs
+ if ((lanes[i] & 16) == (lanes[i - 1] & 16)) {
+ return Nothing();
+ }
+ int suffixLength = i;
+
+ i = ScanIncreasingUnmasked(lanes, i);
+ // Must end at the left end
+ if (i != 16) {
+ return Nothing();
+ }
+
+ // If the suffix is from the lhs then swap the operands
+ if (lanes[0] < 16) {
+ *swapOperands = !*swapOperands;
+ }
+ *control = SimdConstant::SplatX16((int8_t)suffixLength);
+ return Some(SimdShuffleOp::CONCAT_RIGHT_SHIFT_8x16);
+}
+
+// Blend words: if we pick words from both operands without a pattern but all
+// the input words stay in their position then this is PBLENDW (immediate mask);
+// this also handles all larger sizes on x64.
+static Maybe<SimdShuffleOp> TryBlendInt16x8(SimdConstant* control) {
+ SimdConstant tmp(*control);
+ if (!ByteMaskToWordMask(&tmp)) {
+ return Nothing();
+ }
+ SimdConstant::I16x8 masked;
+ MaskLanes(masked, tmp.asInt16x8());
+ if (!IsIdentity(masked)) {
+ return Nothing();
+ }
+ SimdConstant::I16x8 mapped;
+ MapLanes(mapped, tmp.asInt16x8(),
+ [](int x) -> int { return x < 8 ? 0 : -1; });
+ *control = SimdConstant::CreateX8(mapped);
+ return Some(SimdShuffleOp::BLEND_16x8);
+}
+
+// Blend bytes: if we pick bytes ditto then this is a byte blend, which can be
+// handled with a CONST, PAND, PANDNOT, and POR.
+//
+// TODO: Optimization opportunity? If we pick all but one lanes from one with at
+// most one from the other then it could be a MOV + PEXRB + PINSRB (also if this
+// element is not in its source location).
+static Maybe<SimdShuffleOp> TryBlendInt8x16(SimdConstant* control) {
+ SimdConstant::I8x16 masked;
+ MaskLanes(masked, control->asInt8x16());
+ if (!IsIdentity(masked)) {
+ return Nothing();
+ }
+ SimdConstant::I8x16 mapped;
+ MapLanes(mapped, control->asInt8x16(),
+ [](int x) -> int { return x < 16 ? 0 : -1; });
+ *control = SimdConstant::CreateX16(mapped);
+ return Some(SimdShuffleOp::BLEND_8x16);
+}
+
+template <typename T>
+static bool MatchInterleave(const T* lanes, int lhs, int rhs, int len) {
+ for (int i = 0; i < len; i++) {
+ if (lanes[i * 2] != lhs + i || lanes[i * 2 + 1] != rhs + i) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Unpack/interleave:
+// - if we interleave the low (bytes/words/doublewords) of the inputs into
+// the output then this is UNPCKL*W (possibly with a swap of operands).
+// - if we interleave the high ditto then it is UNPCKH*W (ditto)
+template <typename T>
+static Maybe<SimdShuffleOp> TryInterleave(const T* lanes, int lhs, int rhs,
+ bool* swapOperands,
+ SimdShuffleOp lowOp,
+ SimdShuffleOp highOp) {
+ int len = int(32 / (sizeof(T) * 4));
+ if (MatchInterleave(lanes, lhs, rhs, len)) {
+ return Some(lowOp);
+ }
+ if (MatchInterleave(lanes, rhs, lhs, len)) {
+ *swapOperands = !*swapOperands;
+ return Some(lowOp);
+ }
+ if (MatchInterleave(lanes, lhs + len, rhs + len, len)) {
+ return Some(highOp);
+ }
+ if (MatchInterleave(lanes, rhs + len, lhs + len, len)) {
+ *swapOperands = !*swapOperands;
+ return Some(highOp);
+ }
+ return Nothing();
+}
+
+static Maybe<SimdShuffleOp> TryInterleave64x2(SimdConstant* control,
+ bool* swapOperands) {
+ SimdConstant tmp = *control;
+ if (!ByteMaskToQWordMask(&tmp)) {
+ return Nothing();
+ }
+ const SimdConstant::I64x2& lanes = tmp.asInt64x2();
+ return TryInterleave(lanes, 0, 2, swapOperands,
+ SimdShuffleOp::INTERLEAVE_LOW_64x2,
+ SimdShuffleOp::INTERLEAVE_HIGH_64x2);
+}
+
+static Maybe<SimdShuffleOp> TryInterleave32x4(SimdConstant* control,
+ bool* swapOperands) {
+ SimdConstant tmp = *control;
+ if (!ByteMaskToDWordMask(&tmp)) {
+ return Nothing();
+ }
+ const SimdConstant::I32x4& lanes = tmp.asInt32x4();
+ return TryInterleave(lanes, 0, 4, swapOperands,
+ SimdShuffleOp::INTERLEAVE_LOW_32x4,
+ SimdShuffleOp::INTERLEAVE_HIGH_32x4);
+}
+
+static Maybe<SimdShuffleOp> TryInterleave16x8(SimdConstant* control,
+ bool* swapOperands) {
+ SimdConstant tmp = *control;
+ if (!ByteMaskToWordMask(&tmp)) {
+ return Nothing();
+ }
+ const SimdConstant::I16x8& lanes = tmp.asInt16x8();
+ return TryInterleave(lanes, 0, 8, swapOperands,
+ SimdShuffleOp::INTERLEAVE_LOW_16x8,
+ SimdShuffleOp::INTERLEAVE_HIGH_16x8);
+}
+
+static Maybe<SimdShuffleOp> TryInterleave8x16(SimdConstant* control,
+ bool* swapOperands) {
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ return TryInterleave(lanes, 0, 16, swapOperands,
+ SimdShuffleOp::INTERLEAVE_LOW_8x16,
+ SimdShuffleOp::INTERLEAVE_HIGH_8x16);
+}
+
+static SimdShuffleOp AnalyzeTwoArgShuffle(SimdConstant* control,
+ bool* swapOperands) {
+ Maybe<SimdShuffleOp> op;
+ op = TryConcatRightShift8x16(control, swapOperands);
+ if (!op) {
+ op = TryBlendInt16x8(control);
+ }
+ if (!op) {
+ op = TryBlendInt8x16(control);
+ }
+ if (!op) {
+ op = TryInterleave64x2(control, swapOperands);
+ }
+ if (!op) {
+ op = TryInterleave32x4(control, swapOperands);
+ }
+ if (!op) {
+ op = TryInterleave16x8(control, swapOperands);
+ }
+ if (!op) {
+ op = TryInterleave8x16(control, swapOperands);
+ }
+ if (!op) {
+ op = Some(SimdShuffleOp::SHUFFLE_BLEND_8x16);
+ }
+ return *op;
+}
+
+// Reorder the operands if that seems useful, notably, move a constant to the
+// right hand side. Rewrites the control to account for any move.
+static bool MaybeReorderShuffleOperands(MDefinition** lhs, MDefinition** rhs,
+ SimdConstant* control) {
+ if ((*lhs)->isWasmFloatConstant()) {
+ MDefinition* tmp = *lhs;
+ *lhs = *rhs;
+ *rhs = tmp;
+
+ int8_t controlBytes[16];
+ const SimdConstant::I8x16& lanes = control->asInt8x16();
+ for (unsigned i = 0; i < 16; i++) {
+ controlBytes[i] = int8_t(lanes[i] ^ 16);
+ }
+ *control = SimdConstant::CreateX16(controlBytes);
+
+ return true;
+ }
+ return false;
+}
+
+# ifdef DEBUG
+static const SimdShuffle& ReportShuffleSpecialization(const SimdShuffle& s) {
+ switch (s.opd) {
+ case SimdShuffle::Operand::BOTH:
+ case SimdShuffle::Operand::BOTH_SWAPPED:
+ switch (*s.shuffleOp) {
+ case SimdShuffleOp::SHUFFLE_BLEND_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> shuffle+blend 8x16");
+ break;
+ case SimdShuffleOp::BLEND_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> blend 8x16");
+ break;
+ case SimdShuffleOp::BLEND_16x8:
+ js::wasm::ReportSimdAnalysis("shuffle -> blend 16x8");
+ break;
+ case SimdShuffleOp::CONCAT_RIGHT_SHIFT_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> concat+shift-right 8x16");
+ break;
+ case SimdShuffleOp::INTERLEAVE_HIGH_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> interleave-high 8x16");
+ break;
+ case SimdShuffleOp::INTERLEAVE_HIGH_16x8:
+ js::wasm::ReportSimdAnalysis("shuffle -> interleave-high 16x8");
+ break;
+ case SimdShuffleOp::INTERLEAVE_HIGH_32x4:
+ js::wasm::ReportSimdAnalysis("shuffle -> interleave-high 32x4");
+ break;
+ case SimdShuffleOp::INTERLEAVE_HIGH_64x2:
+ js::wasm::ReportSimdAnalysis("shuffle -> interleave-high 64x2");
+ break;
+ case SimdShuffleOp::INTERLEAVE_LOW_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> interleave-low 8x16");
+ break;
+ case SimdShuffleOp::INTERLEAVE_LOW_16x8:
+ js::wasm::ReportSimdAnalysis("shuffle -> interleave-low 16x8");
+ break;
+ case SimdShuffleOp::INTERLEAVE_LOW_32x4:
+ js::wasm::ReportSimdAnalysis("shuffle -> interleave-low 32x4");
+ break;
+ case SimdShuffleOp::INTERLEAVE_LOW_64x2:
+ js::wasm::ReportSimdAnalysis("shuffle -> interleave-low 64x2");
+ break;
+ default:
+ MOZ_CRASH("Unexpected shuffle op");
+ }
+ break;
+ case SimdShuffle::Operand::LEFT:
+ case SimdShuffle::Operand::RIGHT:
+ switch (*s.permuteOp) {
+ case SimdPermuteOp::BROADCAST_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> broadcast 8x16");
+ break;
+ case SimdPermuteOp::BROADCAST_16x8:
+ js::wasm::ReportSimdAnalysis("shuffle -> broadcast 16x8");
+ break;
+ case SimdPermuteOp::MOVE:
+ js::wasm::ReportSimdAnalysis("shuffle -> move");
+ break;
+ case SimdPermuteOp::REVERSE_16x8:
+ js::wasm::ReportSimdAnalysis(
+ "shuffle -> reverse bytes in 16-bit lanes");
+ break;
+ case SimdPermuteOp::REVERSE_32x4:
+ js::wasm::ReportSimdAnalysis(
+ "shuffle -> reverse bytes in 32-bit lanes");
+ break;
+ case SimdPermuteOp::REVERSE_64x2:
+ js::wasm::ReportSimdAnalysis(
+ "shuffle -> reverse bytes in 64-bit lanes");
+ break;
+ case SimdPermuteOp::PERMUTE_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> permute 8x16");
+ break;
+ case SimdPermuteOp::PERMUTE_16x8:
+ js::wasm::ReportSimdAnalysis("shuffle -> permute 16x8");
+ break;
+ case SimdPermuteOp::PERMUTE_32x4:
+ js::wasm::ReportSimdAnalysis("shuffle -> permute 32x4");
+ break;
+ case SimdPermuteOp::ROTATE_RIGHT_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> rotate-right 8x16");
+ break;
+ case SimdPermuteOp::SHIFT_LEFT_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> shift-left 8x16");
+ break;
+ case SimdPermuteOp::SHIFT_RIGHT_8x16:
+ js::wasm::ReportSimdAnalysis("shuffle -> shift-right 8x16");
+ break;
+ default:
+ MOZ_CRASH("Unexpected permute op");
+ }
+ break;
+ }
+ return s;
+}
+# endif // DEBUG
+
+SimdShuffle jit::AnalyzeSimdShuffle(SimdConstant control, MDefinition* lhs,
+ MDefinition* rhs) {
+# ifdef DEBUG
+# define R(s) ReportShuffleSpecialization(s)
+# else
+# define R(s) (s)
+# endif
+
+ // If only one of the inputs is used, determine which.
+ bool useLeft = true;
+ bool useRight = true;
+ if (lhs == rhs) {
+ useRight = false;
+ } else {
+ bool allAbove = true;
+ bool allBelow = true;
+ const SimdConstant::I8x16& lanes = control.asInt8x16();
+ for (int8_t i : lanes) {
+ allAbove = allAbove && i >= 16;
+ allBelow = allBelow && i < 16;
+ }
+ if (allAbove) {
+ useLeft = false;
+ } else if (allBelow) {
+ useRight = false;
+ }
+ }
+
+ // Deal with one-ignored-input.
+ if (!(useLeft && useRight)) {
+ SimdPermuteOp op = AnalyzePermute(&control);
+ return R(SimdShuffle::permute(
+ useLeft ? SimdShuffle::Operand::LEFT : SimdShuffle::Operand::RIGHT,
+ control, op));
+ }
+
+ // Move constants to rhs.
+ bool swapOperands = MaybeReorderShuffleOperands(&lhs, &rhs, &control);
+
+ // Deal with constant rhs.
+ if (rhs->isWasmFloatConstant()) {
+ SimdConstant rhsConstant = rhs->toWasmFloatConstant()->toSimd128();
+ if (rhsConstant.isZeroBits()) {
+ Maybe<SimdPermuteOp> op = AnalyzeShuffleWithZero(&control);
+ if (op) {
+ return R(SimdShuffle::permute(swapOperands ? SimdShuffle::Operand::RIGHT
+ : SimdShuffle::Operand::LEFT,
+ control, *op));
+ }
+ }
+ }
+
+ // Two operands both of which are used. If there's one constant operand it is
+ // now on the rhs.
+ SimdShuffleOp op = AnalyzeTwoArgShuffle(&control, &swapOperands);
+ return R(SimdShuffle::shuffle(swapOperands
+ ? SimdShuffle::Operand::BOTH_SWAPPED
+ : SimdShuffle::Operand::BOTH,
+ control, op));
+# undef R
+}
+
+#endif // ENABLE_WASM_SIMD
diff --git a/js/src/jit/ShuffleAnalysis.h b/js/src/jit/ShuffleAnalysis.h
new file mode 100644
index 0000000000..84133ccf5a
--- /dev/null
+++ b/js/src/jit/ShuffleAnalysis.h
@@ -0,0 +1,147 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ShuffleAnalysis_h
+#define jit_ShuffleAnalysis_h
+
+#include "jit/IonTypes.h"
+
+namespace js {
+namespace jit {
+
+class MDefinition;
+
+// Permutation operations. NOTE: these may still be x86-centric, but the set
+// can accomodate operations from other architectures.
+//
+// The "low-order" byte is in lane 0 of an 8x16 datum, the "high-order" byte
+// in lane 15. The low-order byte is also the "rightmost". In wasm, the
+// constant (v128.const i8x16 0 1 2 ... 15) has 0 in the low-order byte and 15
+// in the high-order byte.
+enum class SimdPermuteOp {
+ // A single byte lane is copied into all the other byte lanes. control_[0]
+ // has the source lane.
+ BROADCAST_8x16,
+
+ // A single word lane is copied into all the other word lanes. control_[0]
+ // has the source lane.
+ BROADCAST_16x8,
+
+ // Copy input to output.
+ MOVE,
+
+ // control_ has bytes in range 0..15 s.t. control_[i] holds the source lane
+ // for output lane i.
+ PERMUTE_8x16,
+
+ // control_ has int16s in range 0..7, as for 8x16. In addition, the high
+ // byte of control_[0] has flags detailing the operation, values taken
+ // from the Perm16x8Action enum below.
+ PERMUTE_16x8,
+
+ // control_ has int32s in range 0..3, as for 8x16.
+ PERMUTE_32x4,
+
+ // control_[0] has the number of places to rotate by.
+ ROTATE_RIGHT_8x16,
+
+ // Zeroes are shifted into high-order bytes and low-order bytes are lost.
+ // control_[0] has the number of places to shift by.
+ SHIFT_RIGHT_8x16,
+
+ // Zeroes are shifted into low-order bytes and high-order bytes are lost.
+ // control_[0] has the number of places to shift by.
+ SHIFT_LEFT_8x16,
+
+ // Reverse bytes of 16-bit lanes.
+ REVERSE_16x8,
+
+ // Reverse bytes of 32-bit lanes.
+ REVERSE_32x4,
+
+ // Reverse bytes of 64-bit lanes.
+ REVERSE_64x2,
+};
+
+// Shuffle operations. NOTE: these may still be x86-centric, but the set can
+// accomodate operations from other architectures.
+enum class SimdShuffleOp {
+ // Blend bytes. control_ has the blend mask as an I8x16: 0 to select from
+ // the lhs, -1 to select from the rhs.
+ BLEND_8x16,
+
+ // Blend words. control_ has the blend mask as an I16x8: 0 to select from
+ // the lhs, -1 to select from the rhs.
+ BLEND_16x8,
+
+ // Concat the lhs in front of the rhs and shift right by bytes, extracting
+ // the low 16 bytes; control_[0] has the shift count.
+ CONCAT_RIGHT_SHIFT_8x16,
+
+ // Interleave qwords/dwords/words/bytes from high/low halves of operands.
+ // The low-order item in the result comes from the lhs, then the next from
+ // the rhs, and so on. control_ is ignored.
+ INTERLEAVE_HIGH_8x16,
+ INTERLEAVE_HIGH_16x8,
+ INTERLEAVE_HIGH_32x4,
+ INTERLEAVE_HIGH_64x2,
+ INTERLEAVE_LOW_8x16,
+ INTERLEAVE_LOW_16x8,
+ INTERLEAVE_LOW_32x4,
+ INTERLEAVE_LOW_64x2,
+
+ // Fully general shuffle+blend. control_ has the shuffle mask.
+ SHUFFLE_BLEND_8x16,
+};
+
+// Representation of the result of the shuffle analysis.
+struct SimdShuffle {
+ enum class Operand {
+ // Both inputs, in the original lhs-rhs order
+ BOTH,
+ // Both inputs, but in rhs-lhs order
+ BOTH_SWAPPED,
+ // Only the lhs input
+ LEFT,
+ // Only the rhs input
+ RIGHT,
+ };
+
+ Operand opd;
+ SimdConstant control;
+ mozilla::Maybe<SimdPermuteOp> permuteOp; // Single operands
+ mozilla::Maybe<SimdShuffleOp> shuffleOp; // Double operands
+
+ static SimdShuffle permute(Operand opd, SimdConstant control,
+ SimdPermuteOp op) {
+ MOZ_ASSERT(opd == Operand::LEFT || opd == Operand::RIGHT);
+ SimdShuffle s{opd, control, mozilla::Some(op), mozilla::Nothing()};
+ return s;
+ }
+
+ static SimdShuffle shuffle(Operand opd, SimdConstant control,
+ SimdShuffleOp op) {
+ MOZ_ASSERT(opd == Operand::BOTH || opd == Operand::BOTH_SWAPPED);
+ SimdShuffle s{opd, control, mozilla::Nothing(), mozilla::Some(op)};
+ return s;
+ }
+
+ bool equals(const SimdShuffle* other) const {
+ return permuteOp == other->permuteOp && shuffleOp == other->shuffleOp &&
+ opd == other->opd && control.bitwiseEqual(other->control);
+ }
+};
+
+#ifdef ENABLE_WASM_SIMD
+
+SimdShuffle AnalyzeSimdShuffle(SimdConstant control, MDefinition* lhs,
+ MDefinition* rhs);
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_ShuffleAnalysis_h
diff --git a/js/src/jit/Simulator.h b/js/src/jit/Simulator.h
new file mode 100644
index 0000000000..babe2b9c19
--- /dev/null
+++ b/js/src/jit/Simulator.h
@@ -0,0 +1,30 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Simulator_h
+#define jit_Simulator_h
+
+#if defined(JS_SIMULATOR_ARM)
+# include "jit/arm/Simulator-arm.h"
+#elif defined(JS_SIMULATOR_ARM64)
+# include "jit/arm64/vixl/Simulator-vixl.h"
+#elif defined(JS_SIMULATOR_MIPS32)
+# include "jit/mips32/Simulator-mips32.h"
+#elif defined(JS_SIMULATOR_MIPS64)
+# include "jit/mips64/Simulator-mips64.h"
+#elif defined(JS_SIMULATOR_LOONG64)
+# include "jit/loong64/Simulator-loong64.h"
+#elif defined(JS_SIMULATOR)
+# error "Unexpected simulator platform"
+#endif
+
+#if defined(JS_SIMULATOR_ARM64)
+namespace js::jit {
+using Simulator = vixl::Simulator;
+}
+#endif
+
+#endif /* jit_Simulator_h */
diff --git a/js/src/jit/Sink.cpp b/js/src/jit/Sink.cpp
new file mode 100644
index 0000000000..36977fb93d
--- /dev/null
+++ b/js/src/jit/Sink.cpp
@@ -0,0 +1,255 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Sink.h"
+
+#include "jit/IonOptimizationLevels.h"
+#include "jit/JitSpewer.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+namespace js {
+namespace jit {
+
+// Given the last found common dominator and a new definition to dominate, the
+// CommonDominator function returns the basic block which dominate the last
+// common dominator and the definition. If no such block exists, then this
+// functions return null.
+static MBasicBlock* CommonDominator(MBasicBlock* commonDominator,
+ MBasicBlock* defBlock) {
+ // This is the first instruction visited, record its basic block as being
+ // the only interesting one.
+ if (!commonDominator) {
+ return defBlock;
+ }
+
+ // Iterate on immediate dominators of the known common dominator to find a
+ // block which dominates all previous uses as well as this instruction.
+ while (!commonDominator->dominates(defBlock)) {
+ MBasicBlock* nextBlock = commonDominator->immediateDominator();
+ // All uses are dominated, so, this cannot happen unless the graph
+ // coherency is not respected.
+ MOZ_ASSERT(commonDominator != nextBlock);
+ commonDominator = nextBlock;
+ }
+
+ return commonDominator;
+}
+
+bool Sink(MIRGenerator* mir, MIRGraph& graph) {
+ JitSpew(JitSpew_Sink, "Begin");
+ TempAllocator& alloc = graph.alloc();
+ bool sinkEnabled = mir->optimizationInfo().sinkEnabled();
+
+ for (PostorderIterator block = graph.poBegin(); block != graph.poEnd();
+ block++) {
+ if (mir->shouldCancel("Sink")) {
+ return false;
+ }
+
+ for (MInstructionReverseIterator iter = block->rbegin();
+ iter != block->rend();) {
+ MInstruction* ins = *iter++;
+
+ // Only instructions which can be recovered on bailout can be moved
+ // into the bailout paths.
+ if (ins->isGuard() || ins->isGuardRangeBailouts() ||
+ ins->isRecoveredOnBailout() || !ins->canRecoverOnBailout()) {
+ continue;
+ }
+
+ // Compute a common dominator for all uses of the current
+ // instruction.
+ bool hasLiveUses = false;
+ bool hasUses = false;
+ MBasicBlock* usesDominator = nullptr;
+ for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e; i++) {
+ hasUses = true;
+ MNode* consumerNode = (*i)->consumer();
+ if (consumerNode->isResumePoint()) {
+ if (!consumerNode->toResumePoint()->isRecoverableOperand(*i)) {
+ hasLiveUses = true;
+ }
+ continue;
+ }
+
+ MDefinition* consumer = consumerNode->toDefinition();
+ if (consumer->isRecoveredOnBailout()) {
+ continue;
+ }
+
+ hasLiveUses = true;
+
+ // If the instruction is a Phi, then we should dominate the
+ // predecessor from which the value is coming from.
+ MBasicBlock* consumerBlock = consumer->block();
+ if (consumer->isPhi()) {
+ consumerBlock = consumerBlock->getPredecessor(consumer->indexOf(*i));
+ }
+
+ usesDominator = CommonDominator(usesDominator, consumerBlock);
+ if (usesDominator == *block) {
+ break;
+ }
+ }
+
+ // Leave this instruction for DCE.
+ if (!hasUses) {
+ continue;
+ }
+
+ // We have no uses, so sink this instruction in all the bailout
+ // paths.
+ if (!hasLiveUses) {
+ MOZ_ASSERT(!usesDominator);
+ ins->setRecoveredOnBailout();
+ JitSpewDef(JitSpew_Sink,
+ " No live uses, recover the instruction on bailout\n", ins);
+ continue;
+ }
+
+ // This guard is temporarly moved here as the above code deals with
+ // Dead Code elimination, which got moved into this Sink phase, as
+ // the Dead Code elimination used to move instructions with no-live
+ // uses to the bailout path.
+ if (!sinkEnabled) {
+ continue;
+ }
+
+ // To move an effectful instruction, we would have to verify that the
+ // side-effect is not observed. In the mean time, we just inhibit
+ // this optimization on effectful instructions.
+ if (ins->isEffectful()) {
+ continue;
+ }
+
+ // If all the uses are under a loop, we might not want to work
+ // against LICM by moving everything back into the loop, but if the
+ // loop is it-self inside an if, then we still want to move the
+ // computation under this if statement.
+ while (block->loopDepth() < usesDominator->loopDepth()) {
+ MOZ_ASSERT(usesDominator != usesDominator->immediateDominator());
+ usesDominator = usesDominator->immediateDominator();
+ }
+
+ // Only move instructions if there is a branch between the dominator
+ // of the uses and the original instruction. This prevent moving the
+ // computation of the arguments into an inline function if there is
+ // no major win.
+ MBasicBlock* lastJoin = usesDominator;
+ while (*block != lastJoin && lastJoin->numPredecessors() == 1) {
+ MOZ_ASSERT(lastJoin != lastJoin->immediateDominator());
+ MBasicBlock* next = lastJoin->immediateDominator();
+ if (next->numSuccessors() > 1) {
+ break;
+ }
+ lastJoin = next;
+ }
+ if (*block == lastJoin) {
+ continue;
+ }
+
+ // Skip to the next instruction if we cannot find a common dominator
+ // for all the uses of this instruction, or if the common dominator
+ // correspond to the block of the current instruction.
+ if (!usesDominator || usesDominator == *block) {
+ continue;
+ }
+
+ // Only instruction which can be recovered on bailout and which are
+ // sinkable can be moved into blocks which are below while filling
+ // the resume points with a clone which is recovered on bailout.
+
+ // If the instruction has live uses and if it is clonable, then we
+ // can clone the instruction for all non-dominated uses and move the
+ // instruction into the block which is dominating all live uses.
+ if (!ins->canClone()) {
+ continue;
+ }
+
+ // If the block is a split-edge block, which is created for folding
+ // test conditions, then the block has no resume point and has
+ // multiple predecessors. In such case, we cannot safely move
+ // bailing instruction to these blocks as we have no way to bailout.
+ if (!usesDominator->entryResumePoint() &&
+ usesDominator->numPredecessors() != 1) {
+ continue;
+ }
+
+ JitSpewDef(JitSpew_Sink, " Can Clone & Recover, sink instruction\n",
+ ins);
+ JitSpew(JitSpew_Sink, " into Block %u", usesDominator->id());
+
+ // Copy the arguments and clone the instruction.
+ MDefinitionVector operands(alloc);
+ for (size_t i = 0, end = ins->numOperands(); i < end; i++) {
+ if (!operands.append(ins->getOperand(i))) {
+ return false;
+ }
+ }
+
+ MInstruction* clone = ins->clone(alloc, operands);
+ if (!clone) {
+ return false;
+ }
+ ins->block()->insertBefore(ins, clone);
+ clone->setRecoveredOnBailout();
+
+ // We should not update the producer of the entry resume point, as
+ // it cannot refer to any instruction within the basic block excepts
+ // for Phi nodes.
+ MResumePoint* entry = usesDominator->entryResumePoint();
+
+ // Replace the instruction by its clone in all the resume points /
+ // recovered-on-bailout instructions which are not in blocks which
+ // are dominated by the usesDominator block.
+ for (MUseIterator i(ins->usesBegin()), e(ins->usesEnd()); i != e;) {
+ MUse* use = *i++;
+ MNode* consumer = use->consumer();
+
+ // If the consumer is a Phi, then we look for the index of the
+ // use to find the corresponding predecessor block, which is
+ // then used as the consumer block.
+ MBasicBlock* consumerBlock = consumer->block();
+ if (consumer->isDefinition() && consumer->toDefinition()->isPhi()) {
+ consumerBlock = consumerBlock->getPredecessor(
+ consumer->toDefinition()->toPhi()->indexOf(use));
+ }
+
+ // Keep the current instruction for all dominated uses, except
+ // for the entry resume point of the block in which the
+ // instruction would be moved into.
+ if (usesDominator->dominates(consumerBlock) &&
+ (!consumer->isResumePoint() ||
+ consumer->toResumePoint() != entry)) {
+ continue;
+ }
+
+ use->replaceProducer(clone);
+ }
+
+ // As we move this instruction in a different block, we should
+ // verify that we do not carry over a resume point which would refer
+ // to an outdated state of the control flow.
+ if (ins->resumePoint()) {
+ ins->clearResumePoint();
+ }
+
+ // Now, that all uses which are not dominated by usesDominator are
+ // using the cloned instruction, we can safely move the instruction
+ // into the usesDominator block.
+ MInstruction* at =
+ usesDominator->safeInsertTop(nullptr, MBasicBlock::IgnoreRecover);
+ block->moveBefore(at, ins);
+ }
+ }
+
+ return true;
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/Sink.h b/js/src/jit/Sink.h
new file mode 100644
index 0000000000..0e714ed3fb
--- /dev/null
+++ b/js/src/jit/Sink.h
@@ -0,0 +1,22 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file declares sink transformation.
+#ifndef jit_Sink_h
+#define jit_Sink_h
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+class MIRGraph;
+
+[[nodiscard]] bool Sink(MIRGenerator* mir, MIRGraph& graph);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Sink_h */
diff --git a/js/src/jit/Snapshots.cpp b/js/src/jit/Snapshots.cpp
new file mode 100644
index 0000000000..c5de08d362
--- /dev/null
+++ b/js/src/jit/Snapshots.cpp
@@ -0,0 +1,605 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Snapshots.h"
+
+#include "jit/JitSpewer.h"
+#ifdef TRACK_SNAPSHOTS
+# include "jit/LIR.h"
+#endif
+#include "jit/MIR.h"
+#include "jit/Recover.h"
+#include "vm/Printer.h"
+
+using namespace js;
+using namespace js::jit;
+
+// [SMDOC] IonMonkey Snapshot encoding
+//
+// Encodings:
+// [ptr] A fixed-size pointer.
+// [vwu] A variable-width unsigned integer.
+// [vws] A variable-width signed integer.
+// [u8] An 8-bit unsigned integer.
+// [u8'] An 8-bit unsigned integer which is potentially extended with packed
+// data.
+// [u8"] Packed data which is stored and packed in the previous [u8'].
+// [vwu*] A list of variable-width unsigned integers.
+// [pld] Payload of Recover Value Allocation:
+// PAYLOAD_NONE:
+// There is no payload.
+//
+// PAYLOAD_INDEX:
+// [vwu] Index, such as the constant pool index.
+//
+// PAYLOAD_STACK_OFFSET:
+// [vws] Stack offset based on the base of the Ion frame.
+//
+// PAYLOAD_GPR:
+// [u8] Code of the general register.
+//
+// PAYLOAD_FPU:
+// [u8] Code of the FPU register.
+//
+// PAYLOAD_PACKED_TAG:
+// [u8"] Bits 5-7: JSValueType is encoded on the low bits of the Mode
+// of the RValueAllocation.
+//
+// Snapshot header:
+//
+// [vwu] bits ((n+1)-31]: recover instruction offset
+// bits [0,n): bailout kind (n = SNAPSHOT_BAILOUTKIND_BITS)
+//
+// Snapshot body, repeated "frame count" times, from oldest frame to newest
+// frame. Note that the first frame doesn't have the "parent PC" field.
+//
+// [ptr] Debug only: JSScript*
+// [vwu] pc offset
+// [vwu] # of RVA's indexes, including nargs
+// [vwu*] List of indexes to R(ecover)ValueAllocation table. Contains
+// nargs + nfixed + stackDepth items.
+//
+// Recover value allocations are encoded at the end of the Snapshot buffer, and
+// they are padded on ALLOCATION_TABLE_ALIGNMENT. The encoding of each
+// allocation is determined by the RValueAllocation::Layout, which can be
+// obtained from the RValueAllocation::Mode with layoutFromMode function. The
+// layout structure list the type of payload which are used to serialized /
+// deserialized / dumped the content of the allocations.
+//
+// R(ecover)ValueAllocation items:
+// [u8'] Mode, which defines the type of the payload as well as the
+// interpretation.
+// [pld] first payload (packed tag, index, stack offset, register, ...)
+// [pld] second payload (register, stack offset, none)
+//
+// Modes:
+// CONSTANT [INDEX]
+// Index into the constant pool.
+//
+// CST_UNDEFINED []
+// Constant value which correspond to the "undefined" JS value.
+//
+// CST_NULL []
+// Constant value which correspond to the "null" JS value.
+//
+// DOUBLE_REG [FPU_REG]
+// Double value stored in a FPU register.
+//
+// ANY_FLOAT_REG [FPU_REG]
+// Any Float value (float32, simd) stored in a FPU register.
+//
+// ANY_FLOAT_STACK [STACK_OFFSET]
+// Any Float value (float32, simd) stored on the stack.
+//
+// UNTYPED_REG [GPR_REG]
+// UNTYPED_STACK [STACK_OFFSET]
+// UNTYPED_REG_REG [GPR_REG, GPR_REG]
+// UNTYPED_REG_STACK [GPR_REG, STACK_OFFSET]
+// UNTYPED_STACK_REG [STACK_OFFSET, GPR_REG]
+// UNTYPED_STACK_STACK [STACK_OFFSET, STACK_OFFSET]
+// Value with dynamically known type. On 32 bits architecture, the
+// first register/stack-offset correspond to the holder of the type,
+// and the second correspond to the payload of the JS Value.
+//
+// RECOVER_INSTRUCTION [INDEX]
+// Index into the list of recovered instruction results.
+//
+// RI_WITH_DEFAULT_CST [INDEX] [INDEX]
+// The first payload is the index into the list of recovered
+// instruction results. The second payload is the index in the
+// constant pool.
+//
+// TYPED_REG [PACKED_TAG, GPR_REG]:
+// Value with statically known type, which payload is stored in a
+// register.
+//
+// TYPED_STACK [PACKED_TAG, STACK_OFFSET]:
+// Value with statically known type, which payload is stored at an
+// offset on the stack.
+//
+
+const RValueAllocation::Layout& RValueAllocation::layoutFromMode(Mode mode) {
+ switch (mode) {
+ case CONSTANT: {
+ static const RValueAllocation::Layout layout = {PAYLOAD_INDEX,
+ PAYLOAD_NONE, "constant"};
+ return layout;
+ }
+
+ case CST_UNDEFINED: {
+ static const RValueAllocation::Layout layout = {
+ PAYLOAD_NONE, PAYLOAD_NONE, "undefined"};
+ return layout;
+ }
+
+ case CST_NULL: {
+ static const RValueAllocation::Layout layout = {PAYLOAD_NONE,
+ PAYLOAD_NONE, "null"};
+ return layout;
+ }
+
+ case DOUBLE_REG: {
+ static const RValueAllocation::Layout layout = {PAYLOAD_FPU, PAYLOAD_NONE,
+ "double"};
+ return layout;
+ }
+ case ANY_FLOAT_REG: {
+ static const RValueAllocation::Layout layout = {PAYLOAD_FPU, PAYLOAD_NONE,
+ "float register content"};
+ return layout;
+ }
+ case ANY_FLOAT_STACK: {
+ static const RValueAllocation::Layout layout = {
+ PAYLOAD_STACK_OFFSET, PAYLOAD_NONE, "float register content"};
+ return layout;
+ }
+#if defined(JS_NUNBOX32)
+ case UNTYPED_REG_REG: {
+ static const RValueAllocation::Layout layout = {PAYLOAD_GPR, PAYLOAD_GPR,
+ "value"};
+ return layout;
+ }
+ case UNTYPED_REG_STACK: {
+ static const RValueAllocation::Layout layout = {
+ PAYLOAD_GPR, PAYLOAD_STACK_OFFSET, "value"};
+ return layout;
+ }
+ case UNTYPED_STACK_REG: {
+ static const RValueAllocation::Layout layout = {PAYLOAD_STACK_OFFSET,
+ PAYLOAD_GPR, "value"};
+ return layout;
+ }
+ case UNTYPED_STACK_STACK: {
+ static const RValueAllocation::Layout layout = {
+ PAYLOAD_STACK_OFFSET, PAYLOAD_STACK_OFFSET, "value"};
+ return layout;
+ }
+#elif defined(JS_PUNBOX64)
+ case UNTYPED_REG: {
+ static const RValueAllocation::Layout layout = {PAYLOAD_GPR, PAYLOAD_NONE,
+ "value"};
+ return layout;
+ }
+ case UNTYPED_STACK: {
+ static const RValueAllocation::Layout layout = {PAYLOAD_STACK_OFFSET,
+ PAYLOAD_NONE, "value"};
+ return layout;
+ }
+#endif
+ case RECOVER_INSTRUCTION: {
+ static const RValueAllocation::Layout layout = {
+ PAYLOAD_INDEX, PAYLOAD_NONE, "instruction"};
+ return layout;
+ }
+ case RI_WITH_DEFAULT_CST: {
+ static const RValueAllocation::Layout layout = {
+ PAYLOAD_INDEX, PAYLOAD_INDEX, "instruction with default"};
+ return layout;
+ }
+
+ default: {
+ static const RValueAllocation::Layout regLayout = {
+ PAYLOAD_PACKED_TAG, PAYLOAD_GPR, "typed value"};
+
+ static const RValueAllocation::Layout stackLayout = {
+ PAYLOAD_PACKED_TAG, PAYLOAD_STACK_OFFSET, "typed value"};
+
+ if (mode >= TYPED_REG_MIN && mode <= TYPED_REG_MAX) {
+ return regLayout;
+ }
+ if (mode >= TYPED_STACK_MIN && mode <= TYPED_STACK_MAX) {
+ return stackLayout;
+ }
+ }
+ }
+
+ MOZ_CRASH_UNSAFE_PRINTF("Unexpected mode: 0x%x", uint32_t(mode));
+}
+
+// Pad serialized RValueAllocations by a multiple of X bytes in the allocation
+// buffer. By padding serialized value allocations, we are building an
+// indexable table of elements of X bytes, and thus we can safely divide any
+// offset within the buffer by X to obtain an index.
+//
+// By padding, we are loosing space within the allocation buffer, but we
+// multiple by X the number of indexes that we can store on one byte in each
+// snapshots.
+//
+// Some value allocations are taking more than X bytes to be encoded, in which
+// case we will pad to a multiple of X, and we are wasting indexes. The choice
+// of X should be balanced between the wasted padding of serialized value
+// allocation, and the saving made in snapshot indexes.
+static const size_t ALLOCATION_TABLE_ALIGNMENT = 2; /* bytes */
+
+void RValueAllocation::readPayload(CompactBufferReader& reader,
+ PayloadType type, uint8_t* mode,
+ Payload* p) {
+ switch (type) {
+ case PAYLOAD_NONE:
+ break;
+ case PAYLOAD_INDEX:
+ p->index = reader.readUnsigned();
+ break;
+ case PAYLOAD_STACK_OFFSET:
+ p->stackOffset = reader.readSigned();
+ break;
+ case PAYLOAD_GPR:
+ p->gpr = Register::FromCode(reader.readByte());
+ break;
+ case PAYLOAD_FPU:
+ p->fpu.data = reader.readByte();
+ break;
+ case PAYLOAD_PACKED_TAG:
+ p->type = JSValueType(*mode & PACKED_TAG_MASK);
+ *mode = *mode & ~PACKED_TAG_MASK;
+ break;
+ }
+}
+
+RValueAllocation RValueAllocation::read(CompactBufferReader& reader) {
+ uint8_t mode = reader.readByte();
+ const Layout& layout = layoutFromMode(Mode(mode & MODE_BITS_MASK));
+ Payload arg1, arg2;
+
+ readPayload(reader, layout.type1, &mode, &arg1);
+ readPayload(reader, layout.type2, &mode, &arg2);
+ return RValueAllocation(Mode(mode), arg1, arg2);
+}
+
+void RValueAllocation::writePayload(CompactBufferWriter& writer,
+ PayloadType type, Payload p) {
+ switch (type) {
+ case PAYLOAD_NONE:
+ break;
+ case PAYLOAD_INDEX:
+ writer.writeUnsigned(p.index);
+ break;
+ case PAYLOAD_STACK_OFFSET:
+ writer.writeSigned(p.stackOffset);
+ break;
+ case PAYLOAD_GPR:
+ static_assert(Registers::Total <= 0x100,
+ "Not enough bytes to encode all registers.");
+ writer.writeByte(p.gpr.code());
+ break;
+ case PAYLOAD_FPU:
+ static_assert(FloatRegisters::Total <= 0x100,
+ "Not enough bytes to encode all float registers.");
+ writer.writeByte(p.fpu.code());
+ break;
+ case PAYLOAD_PACKED_TAG: {
+ // This code assumes that the PACKED_TAG payload is following the
+ // writeByte of the mode.
+ if (!writer.oom()) {
+ MOZ_ASSERT(writer.length());
+ uint8_t* mode = writer.buffer() + (writer.length() - 1);
+ MOZ_ASSERT((*mode & PACKED_TAG_MASK) == 0 &&
+ (p.type & ~PACKED_TAG_MASK) == 0);
+ *mode = *mode | p.type;
+ }
+ break;
+ }
+ }
+}
+
+void RValueAllocation::writePadding(CompactBufferWriter& writer) {
+ // Write 0x7f in all padding bytes.
+ while (writer.length() % ALLOCATION_TABLE_ALIGNMENT) {
+ writer.writeByte(0x7f);
+ }
+}
+
+void RValueAllocation::write(CompactBufferWriter& writer) const {
+ const Layout& layout = layoutFromMode(mode());
+ MOZ_ASSERT(layout.type2 != PAYLOAD_PACKED_TAG);
+ MOZ_ASSERT(writer.length() % ALLOCATION_TABLE_ALIGNMENT == 0);
+
+ writer.writeByte(mode_);
+ writePayload(writer, layout.type1, arg1_);
+ writePayload(writer, layout.type2, arg2_);
+ writePadding(writer);
+}
+
+HashNumber RValueAllocation::hash() const {
+ HashNumber res = 0;
+ res = HashNumber(mode_);
+ res = arg1_.index + (res << 6) + (res << 16) - res;
+ res = arg2_.index + (res << 6) + (res << 16) - res;
+ return res;
+}
+
+#ifdef JS_JITSPEW
+void RValueAllocation::dumpPayload(GenericPrinter& out, PayloadType type,
+ Payload p) {
+ switch (type) {
+ case PAYLOAD_NONE:
+ break;
+ case PAYLOAD_INDEX:
+ out.printf("index %u", p.index);
+ break;
+ case PAYLOAD_STACK_OFFSET:
+ out.printf("stack %d", p.stackOffset);
+ break;
+ case PAYLOAD_GPR:
+ out.printf("reg %s", p.gpr.name());
+ break;
+ case PAYLOAD_FPU:
+ out.printf("reg %s", p.fpu.name());
+ break;
+ case PAYLOAD_PACKED_TAG:
+ out.printf("%s", ValTypeToString(p.type));
+ break;
+ }
+}
+
+void RValueAllocation::dump(GenericPrinter& out) const {
+ const Layout& layout = layoutFromMode(mode());
+ out.printf("%s", layout.name);
+
+ if (layout.type1 != PAYLOAD_NONE) {
+ out.printf(" (");
+ }
+ dumpPayload(out, layout.type1, arg1_);
+ if (layout.type2 != PAYLOAD_NONE) {
+ out.printf(", ");
+ }
+ dumpPayload(out, layout.type2, arg2_);
+ if (layout.type1 != PAYLOAD_NONE) {
+ out.printf(")");
+ }
+}
+#endif // JS_JITSPEW
+
+SnapshotReader::SnapshotReader(const uint8_t* snapshots, uint32_t offset,
+ uint32_t RVATableSize, uint32_t listSize)
+ : reader_(snapshots + offset, snapshots + listSize),
+ allocReader_(snapshots + listSize, snapshots + listSize + RVATableSize),
+ allocTable_(snapshots + listSize),
+ allocRead_(0) {
+ if (!snapshots) {
+ return;
+ }
+ JitSpew(JitSpew_IonSnapshots, "Creating snapshot reader");
+ readSnapshotHeader();
+}
+
+#define COMPUTE_SHIFT_AFTER_(name) (name##_BITS + name##_SHIFT)
+#define COMPUTE_MASK_(name) ((uint32_t(1 << name##_BITS) - 1) << name##_SHIFT)
+
+// Details of snapshot header packing.
+static const uint32_t SNAPSHOT_BAILOUTKIND_SHIFT = 0;
+static const uint32_t SNAPSHOT_BAILOUTKIND_BITS = 6;
+static const uint32_t SNAPSHOT_BAILOUTKIND_MASK =
+ COMPUTE_MASK_(SNAPSHOT_BAILOUTKIND);
+
+static_assert((1 << SNAPSHOT_BAILOUTKIND_BITS) - 1 >=
+ uint8_t(BailoutKind::Limit),
+ "Not enough bits for BailoutKinds");
+
+static const uint32_t SNAPSHOT_ROFFSET_SHIFT =
+ COMPUTE_SHIFT_AFTER_(SNAPSHOT_BAILOUTKIND);
+static const uint32_t SNAPSHOT_ROFFSET_BITS = 32 - SNAPSHOT_ROFFSET_SHIFT;
+static const uint32_t SNAPSHOT_ROFFSET_MASK = COMPUTE_MASK_(SNAPSHOT_ROFFSET);
+
+#undef COMPUTE_MASK_
+#undef COMPUTE_SHIFT_AFTER_
+
+void SnapshotReader::readSnapshotHeader() {
+ uint32_t bits = reader_.readUnsigned();
+
+ bailoutKind_ = BailoutKind((bits & SNAPSHOT_BAILOUTKIND_MASK) >>
+ SNAPSHOT_BAILOUTKIND_SHIFT);
+ recoverOffset_ = (bits & SNAPSHOT_ROFFSET_MASK) >> SNAPSHOT_ROFFSET_SHIFT;
+
+ JitSpew(JitSpew_IonSnapshots, "Read snapshot header with bailout kind %u",
+ uint32_t(bailoutKind_));
+
+#ifdef TRACK_SNAPSHOTS
+ readTrackSnapshot();
+#endif
+}
+
+#ifdef TRACK_SNAPSHOTS
+void SnapshotReader::readTrackSnapshot() {
+ pcOpcode_ = reader_.readUnsigned();
+ mirOpcode_ = reader_.readUnsigned();
+ mirId_ = reader_.readUnsigned();
+ lirOpcode_ = reader_.readUnsigned();
+ lirId_ = reader_.readUnsigned();
+}
+
+void SnapshotReader::spewBailingFrom() const {
+# ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_IonBailouts)) {
+ JitSpewHeader(JitSpew_IonBailouts);
+ Fprinter& out = JitSpewPrinter();
+ out.printf(" bailing from bytecode: %s, MIR: ", CodeName(JSOp(pcOpcode_)));
+ MDefinition::PrintOpcodeName(out, MDefinition::Opcode(mirOpcode_));
+ out.printf(" [%u], LIR: ", mirId_);
+ LInstruction::printName(out, LInstruction::Opcode(lirOpcode_));
+ out.printf(" [%u]", lirId_);
+ out.printf("\n");
+ }
+# endif
+}
+#endif
+
+uint32_t SnapshotReader::readAllocationIndex() {
+ allocRead_++;
+ return reader_.readUnsigned();
+}
+
+RValueAllocation SnapshotReader::readAllocation() {
+ JitSpew(JitSpew_IonSnapshots, "Reading slot %u", allocRead_);
+ uint32_t offset = readAllocationIndex() * ALLOCATION_TABLE_ALIGNMENT;
+ allocReader_.seek(allocTable_, offset);
+ return RValueAllocation::read(allocReader_);
+}
+
+SnapshotWriter::SnapshotWriter()
+ // Based on the measurements made in Bug 962555 comment 20, this length
+ // should be enough to prevent the reallocation of the hash table for at
+ // least half of the compilations.
+ : allocMap_(32) {}
+
+RecoverReader::RecoverReader(SnapshotReader& snapshot, const uint8_t* recovers,
+ uint32_t size)
+ : reader_(nullptr, nullptr), numInstructions_(0), numInstructionsRead_(0) {
+ if (!recovers) {
+ return;
+ }
+ reader_ =
+ CompactBufferReader(recovers + snapshot.recoverOffset(), recovers + size);
+ readRecoverHeader();
+ readInstruction();
+}
+
+RecoverReader::RecoverReader(const RecoverReader& rr)
+ : reader_(rr.reader_),
+ numInstructions_(rr.numInstructions_),
+ numInstructionsRead_(rr.numInstructionsRead_) {
+ if (reader_.currentPosition()) {
+ rr.instruction()->cloneInto(&rawData_);
+ }
+}
+
+RecoverReader& RecoverReader::operator=(const RecoverReader& rr) {
+ reader_ = rr.reader_;
+ numInstructions_ = rr.numInstructions_;
+ numInstructionsRead_ = rr.numInstructionsRead_;
+ if (reader_.currentPosition()) {
+ rr.instruction()->cloneInto(&rawData_);
+ }
+ return *this;
+}
+
+void RecoverReader::readRecoverHeader() {
+ numInstructions_ = reader_.readUnsigned();
+ MOZ_ASSERT(numInstructions_);
+
+ JitSpew(JitSpew_IonSnapshots, "Read recover header with instructionCount %u",
+ numInstructions_);
+}
+
+void RecoverReader::readInstruction() {
+ MOZ_ASSERT(moreInstructions());
+ RInstruction::readRecoverData(reader_, &rawData_);
+ numInstructionsRead_++;
+}
+
+SnapshotOffset SnapshotWriter::startSnapshot(RecoverOffset recoverOffset,
+ BailoutKind kind) {
+ lastStart_ = writer_.length();
+ allocWritten_ = 0;
+
+ JitSpew(JitSpew_IonSnapshots,
+ "starting snapshot with recover offset %u, bailout kind %u",
+ recoverOffset, uint32_t(kind));
+
+ MOZ_ASSERT(uint32_t(kind) < (1 << SNAPSHOT_BAILOUTKIND_BITS));
+ MOZ_ASSERT(recoverOffset < (1 << SNAPSHOT_ROFFSET_BITS));
+ uint32_t bits = (uint32_t(kind) << SNAPSHOT_BAILOUTKIND_SHIFT) |
+ (recoverOffset << SNAPSHOT_ROFFSET_SHIFT);
+
+ writer_.writeUnsigned(bits);
+ return lastStart_;
+}
+
+#ifdef TRACK_SNAPSHOTS
+void SnapshotWriter::trackSnapshot(uint32_t pcOpcode, uint32_t mirOpcode,
+ uint32_t mirId, uint32_t lirOpcode,
+ uint32_t lirId) {
+ writer_.writeUnsigned(pcOpcode);
+ writer_.writeUnsigned(mirOpcode);
+ writer_.writeUnsigned(mirId);
+ writer_.writeUnsigned(lirOpcode);
+ writer_.writeUnsigned(lirId);
+}
+#endif
+
+bool SnapshotWriter::add(const RValueAllocation& alloc) {
+ uint32_t offset;
+ RValueAllocMap::AddPtr p = allocMap_.lookupForAdd(alloc);
+ if (!p) {
+ offset = allocWriter_.length();
+ alloc.write(allocWriter_);
+ if (!allocMap_.add(p, alloc, offset)) {
+ allocWriter_.setOOM();
+ return false;
+ }
+ } else {
+ offset = p->value();
+ }
+
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_IonSnapshots)) {
+ JitSpewHeader(JitSpew_IonSnapshots);
+ Fprinter& out = JitSpewPrinter();
+ out.printf(" slot %u (%u): ", allocWritten_, offset);
+ alloc.dump(out);
+ out.printf("\n");
+ }
+#endif
+
+ allocWritten_++;
+ writer_.writeUnsigned(offset / ALLOCATION_TABLE_ALIGNMENT);
+ return true;
+}
+
+void SnapshotWriter::endSnapshot() {
+ // Place a sentinel for asserting on the other end.
+#ifdef DEBUG
+ writer_.writeSigned(-1);
+#endif
+
+ JitSpew(JitSpew_IonSnapshots,
+ "ending snapshot total size: %u bytes (start %u)",
+ uint32_t(writer_.length() - lastStart_), lastStart_);
+}
+
+RecoverOffset RecoverWriter::startRecover(uint32_t instructionCount) {
+ MOZ_ASSERT(instructionCount);
+ instructionCount_ = instructionCount;
+ instructionsWritten_ = 0;
+
+ JitSpew(JitSpew_IonSnapshots, "starting recover with %u instruction(s)",
+ instructionCount);
+
+ RecoverOffset recoverOffset = writer_.length();
+ writer_.writeUnsigned(instructionCount);
+ return recoverOffset;
+}
+
+void RecoverWriter::writeInstruction(const MNode* rp) {
+ if (!rp->writeRecoverData(writer_)) {
+ writer_.setOOM();
+ }
+ instructionsWritten_++;
+}
+
+void RecoverWriter::endRecover() {
+ MOZ_ASSERT(instructionCount_ == instructionsWritten_);
+}
diff --git a/js/src/jit/Snapshots.h b/js/src/jit/Snapshots.h
new file mode 100644
index 0000000000..9125b87b0f
--- /dev/null
+++ b/js/src/jit/Snapshots.h
@@ -0,0 +1,529 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_Snapshot_h
+#define jit_Snapshot_h
+
+#include "mozilla/Attributes.h"
+
+#include "jit/CompactBuffer.h"
+#include "jit/IonTypes.h"
+#include "jit/Registers.h"
+#include "js/AllocPolicy.h"
+#include "js/HashTable.h"
+#include "js/TypeDecls.h"
+
+namespace js {
+class GenericPrinter;
+
+namespace jit {
+
+class RValueAllocation;
+
+// A Recover Value Allocation mirror what is known at compiled time as being the
+// MIRType and the LAllocation. This is read out of the snapshot to recover the
+// value which would be there if this frame was an interpreter frame instead of
+// an Ion frame.
+//
+// It is used with the SnapshotIterator to recover a Value from the stack,
+// spilled registers or the list of constant of the compiled script.
+//
+// Unit tests are located in jsapi-tests/testJitRValueAlloc.cpp.
+class RValueAllocation {
+ public:
+ // See RValueAllocation encoding in Snapshots.cpp
+ enum Mode {
+ CONSTANT = 0x00,
+ CST_UNDEFINED = 0x01,
+ CST_NULL = 0x02,
+ DOUBLE_REG = 0x03,
+ ANY_FLOAT_REG = 0x04,
+ ANY_FLOAT_STACK = 0x05,
+#if defined(JS_NUNBOX32)
+ UNTYPED_REG_REG = 0x06,
+ UNTYPED_REG_STACK = 0x07,
+ UNTYPED_STACK_REG = 0x08,
+ UNTYPED_STACK_STACK = 0x09,
+#elif defined(JS_PUNBOX64)
+ UNTYPED_REG = 0x06,
+ UNTYPED_STACK = 0x07,
+#endif
+
+ // Recover instructions.
+ RECOVER_INSTRUCTION = 0x0a,
+ RI_WITH_DEFAULT_CST = 0x0b,
+
+ // The JSValueType is packed in the Mode.
+ TYPED_REG_MIN = 0x10,
+ TYPED_REG_MAX = 0x1f,
+ TYPED_REG = TYPED_REG_MIN,
+
+ // The JSValueType is packed in the Mode.
+ TYPED_STACK_MIN = 0x20,
+ TYPED_STACK_MAX = 0x2f,
+ TYPED_STACK = TYPED_STACK_MIN,
+
+ // This mask can be used with any other valid mode. When this flag is
+ // set on the mode, this inform the snapshot iterator that even if the
+ // allocation is readable, the content of if might be incomplete unless
+ // all side-effects are executed.
+ RECOVER_SIDE_EFFECT_MASK = 0x80,
+
+ // This mask represents the set of bits which can be used to encode a
+ // value in a snapshot. The mode is used to determine how to interpret
+ // the union of values and how to pack the value in memory.
+ MODE_BITS_MASK = 0x17f,
+
+ INVALID = 0x100,
+ };
+
+ enum { PACKED_TAG_MASK = 0x0f };
+
+ // See Payload encoding in Snapshots.cpp
+ enum PayloadType {
+ PAYLOAD_NONE,
+ PAYLOAD_INDEX,
+ PAYLOAD_STACK_OFFSET,
+ PAYLOAD_GPR,
+ PAYLOAD_FPU,
+ PAYLOAD_PACKED_TAG
+ };
+
+ struct Layout {
+ PayloadType type1;
+ PayloadType type2;
+ const char* name;
+ };
+
+ private:
+ Mode mode_;
+
+ // Additional information to recover the content of the allocation.
+ struct FloatRegisterBits {
+ uint32_t data;
+ bool operator==(const FloatRegisterBits& other) const {
+ return data == other.data;
+ }
+ uint32_t code() const { return data; }
+ const char* name() const {
+ FloatRegister tmp = FloatRegister::FromCode(data);
+ return tmp.name();
+ }
+ };
+
+ union Payload {
+ uint32_t index;
+ int32_t stackOffset;
+ Register gpr;
+ FloatRegisterBits fpu;
+ JSValueType type;
+
+ Payload() : index(0) {
+ static_assert(sizeof(index) == sizeof(Payload),
+ "All Payload bits are initialized.");
+ }
+ };
+
+ Payload arg1_;
+ Payload arg2_;
+
+ static Payload payloadOfIndex(uint32_t index) {
+ Payload p;
+ p.index = index;
+ return p;
+ }
+ static Payload payloadOfStackOffset(int32_t offset) {
+ Payload p;
+ p.stackOffset = offset;
+ return p;
+ }
+ static Payload payloadOfRegister(Register reg) {
+ Payload p;
+ p.gpr = reg;
+ return p;
+ }
+ static Payload payloadOfFloatRegister(FloatRegister reg) {
+ Payload p;
+ FloatRegisterBits b;
+ b.data = reg.code();
+ p.fpu = b;
+ return p;
+ }
+ static Payload payloadOfValueType(JSValueType type) {
+ Payload p;
+ p.type = type;
+ return p;
+ }
+
+ static const Layout& layoutFromMode(Mode mode);
+
+ static void readPayload(CompactBufferReader& reader, PayloadType t,
+ uint8_t* mode, Payload* p);
+ static void writePayload(CompactBufferWriter& writer, PayloadType t,
+ Payload p);
+ static void writePadding(CompactBufferWriter& writer);
+#ifdef JS_JITSPEW
+ static void dumpPayload(GenericPrinter& out, PayloadType t, Payload p);
+#endif
+ static bool equalPayloads(PayloadType t, Payload lhs, Payload rhs);
+
+ RValueAllocation(Mode mode, Payload a1, Payload a2)
+ : mode_(mode), arg1_(a1), arg2_(a2) {}
+
+ RValueAllocation(Mode mode, Payload a1) : mode_(mode), arg1_(a1) {
+ arg2_.index = 0;
+ }
+
+ explicit RValueAllocation(Mode mode) : mode_(mode) {
+ arg1_.index = 0;
+ arg2_.index = 0;
+ }
+
+ public:
+ RValueAllocation() : mode_(INVALID) {
+ arg1_.index = 0;
+ arg2_.index = 0;
+ }
+
+ // DOUBLE_REG
+ static RValueAllocation Double(FloatRegister reg) {
+ return RValueAllocation(DOUBLE_REG, payloadOfFloatRegister(reg));
+ }
+
+ // ANY_FLOAT_REG or ANY_FLOAT_STACK
+ static RValueAllocation AnyFloat(FloatRegister reg) {
+ return RValueAllocation(ANY_FLOAT_REG, payloadOfFloatRegister(reg));
+ }
+ static RValueAllocation AnyFloat(int32_t offset) {
+ return RValueAllocation(ANY_FLOAT_STACK, payloadOfStackOffset(offset));
+ }
+
+ // TYPED_REG or TYPED_STACK
+ static RValueAllocation Typed(JSValueType type, Register reg) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE && type != JSVAL_TYPE_MAGIC &&
+ type != JSVAL_TYPE_NULL && type != JSVAL_TYPE_UNDEFINED);
+ return RValueAllocation(TYPED_REG, payloadOfValueType(type),
+ payloadOfRegister(reg));
+ }
+ static RValueAllocation Typed(JSValueType type, int32_t offset) {
+ MOZ_ASSERT(type != JSVAL_TYPE_MAGIC && type != JSVAL_TYPE_NULL &&
+ type != JSVAL_TYPE_UNDEFINED);
+ return RValueAllocation(TYPED_STACK, payloadOfValueType(type),
+ payloadOfStackOffset(offset));
+ }
+
+ // UNTYPED
+#if defined(JS_NUNBOX32)
+ static RValueAllocation Untyped(Register type, Register payload) {
+ return RValueAllocation(UNTYPED_REG_REG, payloadOfRegister(type),
+ payloadOfRegister(payload));
+ }
+
+ static RValueAllocation Untyped(Register type, int32_t payloadStackOffset) {
+ return RValueAllocation(UNTYPED_REG_STACK, payloadOfRegister(type),
+ payloadOfStackOffset(payloadStackOffset));
+ }
+
+ static RValueAllocation Untyped(int32_t typeStackOffset, Register payload) {
+ return RValueAllocation(UNTYPED_STACK_REG,
+ payloadOfStackOffset(typeStackOffset),
+ payloadOfRegister(payload));
+ }
+
+ static RValueAllocation Untyped(int32_t typeStackOffset,
+ int32_t payloadStackOffset) {
+ return RValueAllocation(UNTYPED_STACK_STACK,
+ payloadOfStackOffset(typeStackOffset),
+ payloadOfStackOffset(payloadStackOffset));
+ }
+
+#elif defined(JS_PUNBOX64)
+ static RValueAllocation Untyped(Register reg) {
+ return RValueAllocation(UNTYPED_REG, payloadOfRegister(reg));
+ }
+
+ static RValueAllocation Untyped(int32_t stackOffset) {
+ return RValueAllocation(UNTYPED_STACK, payloadOfStackOffset(stackOffset));
+ }
+#endif
+
+ // common constants.
+ static RValueAllocation Undefined() {
+ return RValueAllocation(CST_UNDEFINED);
+ }
+ static RValueAllocation Null() { return RValueAllocation(CST_NULL); }
+
+ // CONSTANT's index
+ static RValueAllocation ConstantPool(uint32_t index) {
+ return RValueAllocation(CONSTANT, payloadOfIndex(index));
+ }
+
+ // Recover instruction's index
+ static RValueAllocation RecoverInstruction(uint32_t index) {
+ return RValueAllocation(RECOVER_INSTRUCTION, payloadOfIndex(index));
+ }
+ static RValueAllocation RecoverInstruction(uint32_t riIndex,
+ uint32_t cstIndex) {
+ return RValueAllocation(RI_WITH_DEFAULT_CST, payloadOfIndex(riIndex),
+ payloadOfIndex(cstIndex));
+ }
+
+ void setNeedSideEffect() {
+ MOZ_ASSERT(!needSideEffect() && mode_ != INVALID);
+ mode_ = Mode(mode_ | RECOVER_SIDE_EFFECT_MASK);
+ }
+
+ void writeHeader(CompactBufferWriter& writer, JSValueType type,
+ uint32_t regCode) const;
+
+ public:
+ static RValueAllocation read(CompactBufferReader& reader);
+ void write(CompactBufferWriter& writer) const;
+
+ public:
+ bool valid() const { return mode_ != INVALID; }
+ Mode mode() const { return Mode(mode_ & MODE_BITS_MASK); }
+ bool needSideEffect() const { return mode_ & RECOVER_SIDE_EFFECT_MASK; }
+
+ uint32_t index() const {
+ MOZ_ASSERT(layoutFromMode(mode()).type1 == PAYLOAD_INDEX);
+ return arg1_.index;
+ }
+ int32_t stackOffset() const {
+ MOZ_ASSERT(layoutFromMode(mode()).type1 == PAYLOAD_STACK_OFFSET);
+ return arg1_.stackOffset;
+ }
+ Register reg() const {
+ MOZ_ASSERT(layoutFromMode(mode()).type1 == PAYLOAD_GPR);
+ return arg1_.gpr;
+ }
+ FloatRegister fpuReg() const {
+ MOZ_ASSERT(layoutFromMode(mode()).type1 == PAYLOAD_FPU);
+ FloatRegisterBits b = arg1_.fpu;
+ return FloatRegister::FromCode(b.data);
+ }
+ JSValueType knownType() const {
+ MOZ_ASSERT(layoutFromMode(mode()).type1 == PAYLOAD_PACKED_TAG);
+ return arg1_.type;
+ }
+
+ uint32_t index2() const {
+ MOZ_ASSERT(layoutFromMode(mode()).type2 == PAYLOAD_INDEX);
+ return arg2_.index;
+ }
+ int32_t stackOffset2() const {
+ MOZ_ASSERT(layoutFromMode(mode()).type2 == PAYLOAD_STACK_OFFSET);
+ return arg2_.stackOffset;
+ }
+ Register reg2() const {
+ MOZ_ASSERT(layoutFromMode(mode()).type2 == PAYLOAD_GPR);
+ return arg2_.gpr;
+ }
+
+ public:
+#ifdef JS_JITSPEW
+ void dump(GenericPrinter& out) const;
+#endif
+
+ bool operator==(const RValueAllocation& rhs) const {
+ // Note, this equality compares the verbatim content of the payload,
+ // which is made possible because we ensure that the payload content is
+ // fully initialized during the creation.
+ static_assert(sizeof(int32_t) == sizeof(Payload),
+ "All Payload bits are compared.");
+ return mode_ == rhs.mode_ && arg1_.index == rhs.arg1_.index &&
+ arg2_.index == rhs.arg2_.index;
+ }
+
+ HashNumber hash() const;
+
+ struct Hasher {
+ using Key = RValueAllocation;
+ using Lookup = Key;
+ static HashNumber hash(const Lookup& v) { return v.hash(); }
+ static bool match(const Key& k, const Lookup& l) { return k == l; }
+ };
+};
+
+class RecoverWriter;
+
+// Collects snapshots in a contiguous buffer, which is copied into IonScript
+// memory after code generation.
+class SnapshotWriter {
+ CompactBufferWriter writer_;
+ CompactBufferWriter allocWriter_;
+
+ // Map RValueAllocations to an offset in the allocWriter_ buffer. This is
+ // useful as value allocations are repeated frequently.
+ using RVA = RValueAllocation;
+ typedef HashMap<RVA, uint32_t, RVA::Hasher, SystemAllocPolicy> RValueAllocMap;
+ RValueAllocMap allocMap_;
+
+ // This is only used to assert sanity.
+ uint32_t allocWritten_;
+
+ // Used to report size of the snapshot in the spew messages.
+ SnapshotOffset lastStart_;
+
+ public:
+ SnapshotWriter();
+
+ SnapshotOffset startSnapshot(RecoverOffset recoverOffset, BailoutKind kind);
+#ifdef TRACK_SNAPSHOTS
+ void trackSnapshot(uint32_t pcOpcode, uint32_t mirOpcode, uint32_t mirId,
+ uint32_t lirOpcode, uint32_t lirId);
+#endif
+ [[nodiscard]] bool add(const RValueAllocation& slot);
+
+ uint32_t allocWritten() const { return allocWritten_; }
+ void endSnapshot();
+
+ bool oom() const {
+ return writer_.oom() || writer_.length() >= MAX_BUFFER_SIZE ||
+ allocWriter_.oom() || allocWriter_.length() >= MAX_BUFFER_SIZE;
+ }
+
+ size_t listSize() const { return writer_.length(); }
+ const uint8_t* listBuffer() const { return writer_.buffer(); }
+
+ size_t RVATableSize() const { return allocWriter_.length(); }
+ const uint8_t* RVATableBuffer() const { return allocWriter_.buffer(); }
+};
+
+class MNode;
+
+class RecoverWriter {
+ CompactBufferWriter writer_;
+
+ uint32_t instructionCount_;
+ uint32_t instructionsWritten_;
+
+ public:
+ SnapshotOffset startRecover(uint32_t instructionCount);
+
+ void writeInstruction(const MNode* rp);
+
+ void endRecover();
+
+ size_t size() const { return writer_.length(); }
+ const uint8_t* buffer() const { return writer_.buffer(); }
+
+ bool oom() const {
+ return writer_.oom() || writer_.length() >= MAX_BUFFER_SIZE;
+ }
+};
+
+class RecoverReader;
+
+// A snapshot reader reads the entries out of the compressed snapshot buffer in
+// a script. These entries describe the equivalent interpreter frames at a given
+// position in JIT code. Each entry is an Ion's value allocations, used to
+// recover the corresponding Value from an Ion frame.
+class SnapshotReader {
+ CompactBufferReader reader_;
+ CompactBufferReader allocReader_;
+ const uint8_t* allocTable_;
+
+ BailoutKind bailoutKind_;
+ uint32_t allocRead_; // Number of slots that have been read.
+ RecoverOffset recoverOffset_; // Offset of the recover instructions.
+
+#ifdef TRACK_SNAPSHOTS
+ private:
+ uint32_t pcOpcode_;
+ uint32_t mirOpcode_;
+ uint32_t mirId_;
+ uint32_t lirOpcode_;
+ uint32_t lirId_;
+
+ public:
+ void readTrackSnapshot();
+ void spewBailingFrom() const;
+#endif
+
+ private:
+ void readSnapshotHeader();
+ uint32_t readAllocationIndex();
+
+ public:
+ SnapshotReader(const uint8_t* snapshots, uint32_t offset,
+ uint32_t RVATableSize, uint32_t listSize);
+
+ RValueAllocation readAllocation();
+ void skipAllocation() { readAllocationIndex(); }
+
+ BailoutKind bailoutKind() const { return bailoutKind_; }
+ RecoverOffset recoverOffset() const { return recoverOffset_; }
+
+ uint32_t numAllocationsRead() const { return allocRead_; }
+ void resetNumAllocationsRead() { allocRead_ = 0; }
+};
+
+class MOZ_NON_PARAM RInstructionStorage {
+ static constexpr size_t Size = 4 * sizeof(uint32_t);
+
+ // This presumes all RInstructionStorage are safely void*-alignable.
+ // RInstruction::readRecoverData asserts that no RInstruction subclass
+ // has stricter alignment requirements than RInstructionStorage.
+ static constexpr size_t Alignment = alignof(void*);
+
+ alignas(Alignment) unsigned char mem[Size];
+
+ public:
+ const void* addr() const { return mem; }
+ void* addr() { return mem; }
+
+ RInstructionStorage() = default;
+
+ // Making a copy of raw bytes holding a RInstruction instance would be a
+ // strict aliasing violation: see bug 1269319 for an instance of bytewise
+ // copying having caused crashes.
+ RInstructionStorage(const RInstructionStorage&) = delete;
+ RInstructionStorage& operator=(const RInstructionStorage& other) = delete;
+};
+
+class RInstruction;
+
+class RecoverReader {
+ CompactBufferReader reader_;
+
+ // Number of encoded instructions.
+ uint32_t numInstructions_;
+
+ // Number of instruction read.
+ uint32_t numInstructionsRead_;
+
+ // Space is reserved as part of the RecoverReader to avoid allocations of
+ // data which is needed to decode the current instruction.
+ RInstructionStorage rawData_;
+
+ private:
+ void readRecoverHeader();
+ void readInstruction();
+
+ public:
+ RecoverReader(SnapshotReader& snapshot, const uint8_t* recovers,
+ uint32_t size);
+ explicit RecoverReader(const RecoverReader& rr);
+ RecoverReader& operator=(const RecoverReader& rr);
+
+ uint32_t numInstructions() const { return numInstructions_; }
+ uint32_t numInstructionsRead() const { return numInstructionsRead_; }
+
+ bool moreInstructions() const {
+ return numInstructionsRead_ < numInstructions_;
+ }
+ void nextInstruction() { readInstruction(); }
+
+ const RInstruction* instruction() const {
+ return reinterpret_cast<const RInstruction*>(rawData_.addr());
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_Snapshot_h */
diff --git a/js/src/jit/StackSlotAllocator.h b/js/src/jit/StackSlotAllocator.h
new file mode 100644
index 0000000000..74a2ab18bd
--- /dev/null
+++ b/js/src/jit/StackSlotAllocator.h
@@ -0,0 +1,133 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_StackSlotAllocator_h
+#define jit_StackSlotAllocator_h
+
+#include "jit/Registers.h"
+
+namespace js {
+namespace jit {
+
+class StackSlotAllocator {
+ js::Vector<uint32_t, 4, SystemAllocPolicy> normalSlots;
+ js::Vector<uint32_t, 4, SystemAllocPolicy> doubleSlots;
+ uint32_t height_;
+
+ void addAvailableSlot(uint32_t index) {
+ // Ignoring OOM here (and below) is fine; it just means the stack slot
+ // will be unused.
+ (void)normalSlots.append(index);
+ }
+ void addAvailableDoubleSlot(uint32_t index) {
+ (void)doubleSlots.append(index);
+ }
+
+ uint32_t allocateQuadSlot() {
+ // This relies on the fact that any architecture specific
+ // alignment of the stack pointer is done a priori.
+ if (height_ % 8 != 0) {
+ addAvailableSlot(height_ += 4);
+ }
+ if (height_ % 16 != 0) {
+ addAvailableDoubleSlot(height_ += 8);
+ }
+ return height_ += 16;
+ }
+ uint32_t allocateDoubleSlot() {
+ if (!doubleSlots.empty()) {
+ return doubleSlots.popCopy();
+ }
+ if (height_ % 8 != 0) {
+ addAvailableSlot(height_ += 4);
+ }
+ return height_ += 8;
+ }
+ uint32_t allocateSlot() {
+ if (!normalSlots.empty()) {
+ return normalSlots.popCopy();
+ }
+ if (!doubleSlots.empty()) {
+ uint32_t index = doubleSlots.popCopy();
+ addAvailableSlot(index - 4);
+ return index;
+ }
+ return height_ += 4;
+ }
+
+ public:
+ StackSlotAllocator() : height_(0) {}
+
+ void allocateStackArea(LStackArea* alloc) {
+ uint32_t size = alloc->size();
+
+ MOZ_ASSERT(size % 4 == 0);
+ switch (alloc->alignment()) {
+ case 8:
+ if ((height_ + size) % 8 != 0) {
+ addAvailableSlot(height_ += 4);
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected stack results area alignment");
+ }
+ MOZ_ASSERT((height_ + size) % alloc->alignment() == 0);
+
+ height_ += size;
+ alloc->setBase(height_);
+ }
+
+ static uint32_t width(LDefinition::Type type) {
+ switch (type) {
+#if JS_BITS_PER_WORD == 32
+ case LDefinition::GENERAL:
+ case LDefinition::OBJECT:
+ case LDefinition::SLOTS:
+#endif
+#ifdef JS_NUNBOX32
+ case LDefinition::TYPE:
+ case LDefinition::PAYLOAD:
+#endif
+ case LDefinition::INT32:
+ case LDefinition::FLOAT32:
+ return 4;
+#if JS_BITS_PER_WORD == 64
+ case LDefinition::GENERAL:
+ case LDefinition::OBJECT:
+ case LDefinition::SLOTS:
+#endif
+#ifdef JS_PUNBOX64
+ case LDefinition::BOX:
+#endif
+ case LDefinition::DOUBLE:
+ return 8;
+ case LDefinition::SIMD128:
+ return 16;
+ case LDefinition::STACKRESULTS:
+ MOZ_CRASH("Stack results area must be allocated manually");
+ }
+ MOZ_CRASH("Unknown slot type");
+ }
+
+ uint32_t allocateSlot(LDefinition::Type type) {
+ switch (width(type)) {
+ case 4:
+ return allocateSlot();
+ case 8:
+ return allocateDoubleSlot();
+ case 16:
+ return allocateQuadSlot();
+ }
+ MOZ_CRASH("Unknown slot width");
+ }
+
+ uint32_t stackHeight() const { return height_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_StackSlotAllocator_h */
diff --git a/js/src/jit/TemplateObject-inl.h b/js/src/jit/TemplateObject-inl.h
new file mode 100644
index 0000000000..3bfeb8e72d
--- /dev/null
+++ b/js/src/jit/TemplateObject-inl.h
@@ -0,0 +1,126 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_TemplateObject_inl_h
+#define jit_TemplateObject_inl_h
+
+#include "jit/TemplateObject.h"
+
+#include "vm/EnvironmentObject.h"
+#include "vm/PlainObject.h" // js::PlainObject
+#include "vm/RegExpObject.h"
+
+namespace js {
+namespace jit {
+
+inline gc::AllocKind TemplateObject::getAllocKind() const {
+ return obj_->asTenured().getAllocKind();
+}
+
+inline bool TemplateObject::isNativeObject() const {
+ return obj_->is<NativeObject>();
+}
+
+inline bool TemplateObject::isArrayObject() const {
+ return obj_->is<ArrayObject>();
+}
+
+inline bool TemplateObject::isArgumentsObject() const {
+ return obj_->is<ArgumentsObject>();
+}
+
+inline bool TemplateObject::isTypedArrayObject() const {
+ return obj_->is<TypedArrayObject>();
+}
+
+inline bool TemplateObject::isRegExpObject() const {
+ return obj_->is<RegExpObject>();
+}
+
+inline bool TemplateObject::isCallObject() const {
+ return obj_->is<CallObject>();
+}
+
+inline bool TemplateObject::isBlockLexicalEnvironmentObject() const {
+ return obj_->is<BlockLexicalEnvironmentObject>();
+}
+
+inline bool TemplateObject::isPlainObject() const {
+ return obj_->is<PlainObject>();
+}
+
+inline gc::Cell* TemplateObject::shape() const {
+ Shape* shape = obj_->shape();
+ MOZ_ASSERT(!shape->isDictionary());
+ return shape;
+}
+
+inline const TemplateNativeObject& TemplateObject::asTemplateNativeObject()
+ const {
+ MOZ_ASSERT(isNativeObject());
+ return *static_cast<const TemplateNativeObject*>(this);
+}
+
+inline bool TemplateNativeObject::hasDynamicSlots() const {
+ return asNativeObject().hasDynamicSlots();
+}
+
+inline uint32_t TemplateNativeObject::numDynamicSlots() const {
+ return asNativeObject().numDynamicSlots();
+}
+
+inline uint32_t TemplateNativeObject::numUsedFixedSlots() const {
+ return asNativeObject().numUsedFixedSlots();
+}
+
+inline uint32_t TemplateNativeObject::numFixedSlots() const {
+ return asNativeObject().numFixedSlots();
+}
+
+inline uint32_t TemplateNativeObject::slotSpan() const {
+ return asNativeObject().sharedShape()->slotSpan();
+}
+
+inline Value TemplateNativeObject::getSlot(uint32_t i) const {
+ return asNativeObject().getSlot(i);
+}
+
+inline const Value* TemplateNativeObject::getDenseElements() const {
+ return asNativeObject().getDenseElements();
+}
+
+#ifdef DEBUG
+inline bool TemplateNativeObject::isSharedMemory() const {
+ return asNativeObject().isSharedMemory();
+}
+#endif
+
+inline uint32_t TemplateNativeObject::getDenseCapacity() const {
+ return asNativeObject().getDenseCapacity();
+}
+
+inline uint32_t TemplateNativeObject::getDenseInitializedLength() const {
+ return asNativeObject().getDenseInitializedLength();
+}
+
+inline uint32_t TemplateNativeObject::getArrayLength() const {
+ return obj_->as<ArrayObject>().length();
+}
+
+inline bool TemplateNativeObject::hasDynamicElements() const {
+ return asNativeObject().hasDynamicElements();
+}
+
+inline gc::Cell* TemplateNativeObject::regExpShared() const {
+ RegExpObject* regexp = &obj_->as<RegExpObject>();
+ MOZ_ASSERT(regexp->hasShared());
+ return regexp->getShared();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_TemplateObject_inl_h */
diff --git a/js/src/jit/TemplateObject.h b/js/src/jit/TemplateObject.h
new file mode 100644
index 0000000000..3620b94738
--- /dev/null
+++ b/js/src/jit/TemplateObject.h
@@ -0,0 +1,77 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_TemplateObject_h
+#define jit_TemplateObject_h
+
+#include "vm/NativeObject.h"
+#include "vm/Shape.h"
+
+namespace js {
+namespace jit {
+
+class TemplateNativeObject;
+
+// Wrapper for template objects. This should only expose methods that can be
+// safely called off-thread without racing with the main thread.
+class TemplateObject {
+ protected:
+ JSObject* obj_;
+
+ public:
+ explicit TemplateObject(JSObject* obj) : obj_(obj) {}
+
+ inline gc::AllocKind getAllocKind() const;
+
+ // The following methods rely on the object's group->clasp. This is safe
+ // to read off-thread for template objects.
+ inline bool isNativeObject() const;
+ inline const TemplateNativeObject& asTemplateNativeObject() const;
+ inline bool isArrayObject() const;
+ inline bool isArgumentsObject() const;
+ inline bool isTypedArrayObject() const;
+ inline bool isRegExpObject() const;
+ inline bool isCallObject() const;
+ inline bool isBlockLexicalEnvironmentObject() const;
+ inline bool isPlainObject() const;
+
+ // The shape should not change. This is true for template objects because
+ // they're never exposed to arbitrary script.
+ inline gc::Cell* shape() const;
+};
+
+class TemplateNativeObject : public TemplateObject {
+ protected:
+ NativeObject& asNativeObject() const { return obj_->as<NativeObject>(); }
+
+ public:
+ // Reading slot counts and object slots is safe, as long as we don't touch
+ // the BaseShape (it can change when we create a ShapeTable for the shape).
+ inline bool hasDynamicSlots() const;
+ inline uint32_t numDynamicSlots() const;
+ inline uint32_t numUsedFixedSlots() const;
+ inline uint32_t numFixedSlots() const;
+ inline uint32_t slotSpan() const;
+ inline Value getSlot(uint32_t i) const;
+
+ // Reading ObjectElements fields is safe, except for the flags.
+ // isSharedMemory is an exception: it's debug-only and not called on arrays.
+#ifdef DEBUG
+ inline bool isSharedMemory() const;
+#endif
+ inline uint32_t getDenseCapacity() const;
+ inline uint32_t getDenseInitializedLength() const;
+ inline uint32_t getArrayLength() const;
+ inline bool hasDynamicElements() const;
+ inline const Value* getDenseElements() const;
+
+ inline gc::Cell* regExpShared() const;
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_TemplateObject_h */
diff --git a/js/src/jit/Trampoline.cpp b/js/src/jit/Trampoline.cpp
new file mode 100644
index 0000000000..b6a270d777
--- /dev/null
+++ b/js/src/jit/Trampoline.cpp
@@ -0,0 +1,253 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include <initializer_list>
+
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "vm/JitActivation.h"
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void JitRuntime::generateExceptionTailStub(MacroAssembler& masm,
+ Label* profilerExitTail,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateExceptionTailStub");
+
+ exceptionTailOffset_ = startTrampolineCode(masm);
+
+ masm.bind(masm.failureLabel());
+ masm.handleFailureWithHandlerTail(profilerExitTail, bailoutTail);
+}
+
+void JitRuntime::generateProfilerExitFrameTailStub(MacroAssembler& masm,
+ Label* profilerExitTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateProfilerExitFrameTailStub");
+
+ profilerExitFrameTailOffset_ = startTrampolineCode(masm);
+ masm.bind(profilerExitTail);
+
+ static constexpr size_t CallerFPOffset =
+ CommonFrameLayout::offsetOfCallerFramePtr();
+
+ // Assert the caller frame's type is one of the types we expect.
+ auto emitAssertPrevFrameType = [&masm](
+ Register framePtr, Register scratch,
+ std::initializer_list<FrameType> types) {
+#ifdef DEBUG
+ masm.loadPtr(Address(framePtr, CommonFrameLayout::offsetOfDescriptor()),
+ scratch);
+ masm.and32(Imm32(FRAMETYPE_MASK), scratch);
+
+ Label checkOk;
+ for (FrameType type : types) {
+ masm.branch32(Assembler::Equal, scratch, Imm32(type), &checkOk);
+ }
+ masm.assumeUnreachable("Unexpected previous frame");
+ masm.bind(&checkOk);
+#else
+ (void)masm;
+#endif
+ };
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.take(JSReturnOperand);
+ Register scratch = regs.takeAny();
+
+ // The code generated below expects that the current frame pointer points
+ // to an Ion or Baseline frame, at the state it would be immediately before
+ // the frame epilogue and ret(). Thus, after this stub's business is done, it
+ // restores the frame pointer and stack pointer, then executes a ret() and
+ // returns directly to the caller frame, on behalf of the callee script that
+ // jumped to this code.
+ //
+ // Thus the expected state is:
+ //
+ // [JitFrameLayout] <-- FramePointer
+ // [frame contents] <-- StackPointer
+ //
+ // The generated jitcode is responsible for overwriting the
+ // jitActivation->lastProfilingFrame field with a pointer to the previous
+ // Ion or Baseline jit-frame that was pushed before this one. It is also
+ // responsible for overwriting jitActivation->lastProfilingCallSite with
+ // the return address into that frame.
+ //
+ // So this jitcode is responsible for "walking up" the jit stack, finding
+ // the previous Ion or Baseline JS frame, and storing its address and the
+ // return address into the appropriate fields on the current jitActivation.
+ //
+ // There are a fixed number of different path types that can lead to the
+ // current frame, which is either a Baseline or Ion frame:
+ //
+ // <Baseline-Or-Ion>
+ // ^
+ // |
+ // ^--- Ion (or Baseline JSOp::Resume)
+ // |
+ // ^--- Baseline Stub <---- Baseline
+ // |
+ // ^--- IonICCall <---- Ion
+ // |
+ // ^--- Arguments Rectifier
+ // | ^
+ // | |
+ // | ^--- Ion
+ // | |
+ // | ^--- Baseline Stub <---- Baseline
+ // | |
+ // | ^--- Entry Frame (CppToJSJit or WasmToJSJit)
+ // |
+ // ^--- Entry Frame (CppToJSJit or WasmToJSJit)
+ //
+ // NOTE: Keep this in sync with JSJitProfilingFrameIterator::moveToNextFrame!
+
+ Register actReg = regs.takeAny();
+ masm.loadJSContext(actReg);
+ masm.loadPtr(Address(actReg, offsetof(JSContext, profilingActivation_)),
+ actReg);
+
+ Address lastProfilingFrame(actReg,
+ JitActivation::offsetOfLastProfilingFrame());
+ Address lastProfilingCallSite(actReg,
+ JitActivation::offsetOfLastProfilingCallSite());
+
+#ifdef DEBUG
+ // Ensure that frame we are exiting is current lastProfilingFrame
+ {
+ masm.loadPtr(lastProfilingFrame, scratch);
+ Label checkOk;
+ masm.branchPtr(Assembler::Equal, scratch, ImmWord(0), &checkOk);
+ masm.branchPtr(Assembler::Equal, FramePointer, scratch, &checkOk);
+ masm.assumeUnreachable(
+ "Mismatch between stored lastProfilingFrame and current frame "
+ "pointer.");
+ masm.bind(&checkOk);
+ }
+#endif
+
+ // Move FP into a scratch register and use that scratch register below, to
+ // allow unwrapping rectifier frames without clobbering FP.
+ Register fpScratch = regs.takeAny();
+ masm.mov(FramePointer, fpScratch);
+
+ Label again;
+ masm.bind(&again);
+
+ // Load the frame descriptor into |scratch|, figure out what to do depending
+ // on its type.
+ masm.loadPtr(Address(fpScratch, JitFrameLayout::offsetOfDescriptor()),
+ scratch);
+ masm.and32(Imm32(FRAMETYPE_MASK), scratch);
+
+ // Handling of each case is dependent on FrameDescriptor.type
+ Label handle_BaselineOrIonJS;
+ Label handle_BaselineStub;
+ Label handle_Rectifier;
+ Label handle_IonICCall;
+ Label handle_Entry;
+
+ // We check for IonJS and BaselineStub first because these are the most common
+ // types. Calls from Baseline are usually from a BaselineStub frame.
+ masm.branch32(Assembler::Equal, scratch, Imm32(FrameType::IonJS),
+ &handle_BaselineOrIonJS);
+ masm.branch32(Assembler::Equal, scratch, Imm32(FrameType::BaselineStub),
+ &handle_BaselineStub);
+ masm.branch32(Assembler::Equal, scratch, Imm32(FrameType::Rectifier),
+ &handle_Rectifier);
+ masm.branch32(Assembler::Equal, scratch, Imm32(FrameType::CppToJSJit),
+ &handle_Entry);
+ masm.branch32(Assembler::Equal, scratch, Imm32(FrameType::BaselineJS),
+ &handle_BaselineOrIonJS);
+ masm.branch32(Assembler::Equal, scratch, Imm32(FrameType::IonICCall),
+ &handle_IonICCall);
+ masm.branch32(Assembler::Equal, scratch, Imm32(FrameType::WasmToJSJit),
+ &handle_Entry);
+
+ masm.assumeUnreachable(
+ "Invalid caller frame type when returning from a JIT frame.");
+
+ masm.bind(&handle_BaselineOrIonJS);
+ {
+ // Returning directly to a Baseline or Ion frame.
+
+ // lastProfilingCallSite := ReturnAddress
+ masm.loadPtr(Address(fpScratch, JitFrameLayout::offsetOfReturnAddress()),
+ scratch);
+ masm.storePtr(scratch, lastProfilingCallSite);
+
+ // lastProfilingFrame := CallerFrame
+ masm.loadPtr(Address(fpScratch, CallerFPOffset), scratch);
+ masm.storePtr(scratch, lastProfilingFrame);
+
+ masm.moveToStackPtr(FramePointer);
+ masm.pop(FramePointer);
+ masm.ret();
+ }
+
+ // Shared implementation for BaselineStub and IonICCall frames.
+ auto emitHandleStubFrame = [&](FrameType expectedPrevType) {
+ // Load pointer to stub frame and assert type of its caller frame.
+ masm.loadPtr(Address(fpScratch, CallerFPOffset), fpScratch);
+ emitAssertPrevFrameType(fpScratch, scratch, {expectedPrevType});
+
+ // lastProfilingCallSite := StubFrame.ReturnAddress
+ masm.loadPtr(Address(fpScratch, CommonFrameLayout::offsetOfReturnAddress()),
+ scratch);
+ masm.storePtr(scratch, lastProfilingCallSite);
+
+ // lastProfilingFrame := StubFrame.CallerFrame
+ masm.loadPtr(Address(fpScratch, CallerFPOffset), scratch);
+ masm.storePtr(scratch, lastProfilingFrame);
+
+ masm.moveToStackPtr(FramePointer);
+ masm.pop(FramePointer);
+ masm.ret();
+ };
+
+ masm.bind(&handle_BaselineStub);
+ {
+ // BaselineJS => BaselineStub frame.
+ emitHandleStubFrame(FrameType::BaselineJS);
+ }
+
+ masm.bind(&handle_IonICCall);
+ {
+ // IonJS => IonICCall frame.
+ emitHandleStubFrame(FrameType::IonJS);
+ }
+
+ masm.bind(&handle_Rectifier);
+ {
+ // There can be multiple previous frame types so just "unwrap" the arguments
+ // rectifier frame and try again.
+ masm.loadPtr(Address(fpScratch, CallerFPOffset), fpScratch);
+ emitAssertPrevFrameType(fpScratch, scratch,
+ {FrameType::IonJS, FrameType::BaselineStub,
+ FrameType::CppToJSJit, FrameType::WasmToJSJit});
+ masm.jump(&again);
+ }
+
+ masm.bind(&handle_Entry);
+ {
+ // FrameType::CppToJSJit / FrameType::WasmToJSJit
+ //
+ // A fast-path wasm->jit transition frame is an entry frame from the point
+ // of view of the JIT.
+ // Store null into both fields.
+ masm.movePtr(ImmPtr(nullptr), scratch);
+ masm.storePtr(scratch, lastProfilingCallSite);
+ masm.storePtr(scratch, lastProfilingFrame);
+
+ masm.moveToStackPtr(FramePointer);
+ masm.pop(FramePointer);
+ masm.ret();
+ }
+}
diff --git a/js/src/jit/TrialInlining.cpp b/js/src/jit/TrialInlining.cpp
new file mode 100644
index 0000000000..7c97388735
--- /dev/null
+++ b/js/src/jit/TrialInlining.cpp
@@ -0,0 +1,851 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/TrialInlining.h"
+
+#include "jit/BaselineCacheIRCompiler.h"
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineIC.h"
+#include "jit/CacheIRCloner.h"
+#include "jit/CacheIRHealth.h"
+#include "jit/CacheIRWriter.h"
+#include "jit/Ion.h" // TooManyFormalArguments
+
+#include "vm/BytecodeLocation-inl.h"
+
+using mozilla::Maybe;
+
+namespace js {
+namespace jit {
+
+bool DoTrialInlining(JSContext* cx, BaselineFrame* frame) {
+ RootedScript script(cx, frame->script());
+ ICScript* icScript = frame->icScript();
+ bool isRecursive = icScript->depth() > 0;
+
+#ifdef JS_CACHEIR_SPEW
+ if (cx->spewer().enabled(cx, script, SpewChannel::CacheIRHealthReport)) {
+ for (uint32_t i = 0; i < icScript->numICEntries(); i++) {
+ ICEntry& entry = icScript->icEntry(i);
+ ICFallbackStub* fallbackStub = icScript->fallbackStub(i);
+
+ // If the IC is megamorphic or generic, then we have already
+ // spewed the IC report on transition.
+ if (!(uint8_t(fallbackStub->state().mode()) > 0)) {
+ jit::ICStub* stub = entry.firstStub();
+ bool sawNonZeroCount = false;
+ while (!stub->isFallback()) {
+ uint32_t count = stub->enteredCount();
+ if (count > 0 && sawNonZeroCount) {
+ CacheIRHealth cih;
+ cih.healthReportForIC(cx, &entry, fallbackStub, script,
+ SpewContext::TrialInlining);
+ break;
+ }
+
+ if (count > 0 && !sawNonZeroCount) {
+ sawNonZeroCount = true;
+ }
+
+ stub = stub->toCacheIRStub()->next();
+ }
+ }
+ }
+ }
+#endif
+
+ if (!script->canIonCompile()) {
+ return true;
+ }
+
+ // Baseline shouldn't attempt trial inlining in scripts that are too large.
+ MOZ_ASSERT_IF(JitOptions.limitScriptSize,
+ script->length() <= JitOptions.ionMaxScriptSize);
+
+ const uint32_t MAX_INLINING_DEPTH = 4;
+ if (icScript->depth() > MAX_INLINING_DEPTH) {
+ return true;
+ }
+
+ InliningRoot* root = isRecursive ? icScript->inliningRoot()
+ : script->jitScript()->inliningRoot();
+ if (JitSpewEnabled(JitSpew_WarpTrialInlining)) {
+ // Eagerly create the inlining root when it's used in the spew output.
+ if (!root) {
+ MOZ_ASSERT(!isRecursive);
+ root = script->jitScript()->getOrCreateInliningRoot(cx, script);
+ if (!root) {
+ return false;
+ }
+ }
+ }
+
+ JitSpew(JitSpew_WarpTrialInlining,
+ "Trial inlining for %s script %s:%u:%u (%p) (inliningRoot=%p)",
+ (isRecursive ? "inner" : "outer"), script->filename(),
+ script->lineno(), script->column(), frame->script(), root);
+ JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
+
+ TrialInliner inliner(cx, script, icScript);
+ return inliner.tryInlining();
+}
+
+void TrialInliner::cloneSharedPrefix(ICCacheIRStub* stub,
+ const uint8_t* endOfPrefix,
+ CacheIRWriter& writer) {
+ CacheIRReader reader(stub->stubInfo());
+ CacheIRCloner cloner(stub);
+ while (reader.currentPosition() < endOfPrefix) {
+ CacheOp op = reader.readOp();
+ cloner.cloneOp(op, reader, writer);
+ }
+}
+
+bool TrialInliner::replaceICStub(ICEntry& entry, ICFallbackStub* fallback,
+ CacheIRWriter& writer, CacheKind kind) {
+ MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate);
+
+ fallback->discardStubs(cx(), &entry);
+
+ // Note: AttachBaselineCacheIRStub never throws an exception.
+ ICAttachResult result = AttachBaselineCacheIRStub(
+ cx(), writer, kind, script_, icScript_, fallback, "TrialInline");
+ if (result == ICAttachResult::Attached) {
+ MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Inlined);
+ return true;
+ }
+
+ MOZ_ASSERT(fallback->trialInliningState() == TrialInliningState::Candidate);
+ icScript_->removeInlinedChild(fallback->pcOffset());
+
+ if (result == ICAttachResult::OOM) {
+ ReportOutOfMemory(cx());
+ return false;
+ }
+
+ // We failed to attach a new IC stub due to CacheIR size limits. Disable trial
+ // inlining for this location and return true.
+ MOZ_ASSERT(result == ICAttachResult::TooLarge);
+ fallback->setTrialInliningState(TrialInliningState::Failure);
+ return true;
+}
+
+ICCacheIRStub* TrialInliner::maybeSingleStub(const ICEntry& entry) {
+ // Look for a single non-fallback stub followed by stubs with entered-count 0.
+ // Allow one optimized stub before the fallback stub to support the
+ // CallIRGenerator::emitCalleeGuard optimization where we first try a
+ // GuardSpecificFunction guard before falling back to GuardFunctionHasScript.
+ ICStub* stub = entry.firstStub();
+ if (stub->isFallback()) {
+ return nullptr;
+ }
+ ICStub* next = stub->toCacheIRStub()->next();
+ if (next->enteredCount() != 0) {
+ return nullptr;
+ }
+
+ ICFallbackStub* fallback = nullptr;
+ if (next->isFallback()) {
+ fallback = next->toFallbackStub();
+ } else {
+ ICStub* nextNext = next->toCacheIRStub()->next();
+ if (!nextNext->isFallback() || nextNext->enteredCount() != 0) {
+ return nullptr;
+ }
+ fallback = nextNext->toFallbackStub();
+ }
+
+ if (fallback->trialInliningState() != TrialInliningState::Candidate) {
+ return nullptr;
+ }
+
+ return stub->toCacheIRStub();
+}
+
+Maybe<InlinableOpData> FindInlinableOpData(ICCacheIRStub* stub,
+ BytecodeLocation loc) {
+ if (loc.isInvokeOp()) {
+ Maybe<InlinableCallData> call = FindInlinableCallData(stub);
+ if (call.isSome()) {
+ return call;
+ }
+ }
+ if (loc.isGetPropOp() || loc.isGetElemOp()) {
+ Maybe<InlinableGetterData> getter = FindInlinableGetterData(stub);
+ if (getter.isSome()) {
+ return getter;
+ }
+ }
+ if (loc.isSetPropOp()) {
+ Maybe<InlinableSetterData> setter = FindInlinableSetterData(stub);
+ if (setter.isSome()) {
+ return setter;
+ }
+ }
+ return mozilla::Nothing();
+}
+
+Maybe<InlinableCallData> FindInlinableCallData(ICCacheIRStub* stub) {
+ Maybe<InlinableCallData> data;
+
+ const CacheIRStubInfo* stubInfo = stub->stubInfo();
+ const uint8_t* stubData = stub->stubDataStart();
+
+ ObjOperandId calleeGuardOperand;
+ CallFlags flags;
+ JSFunction* target = nullptr;
+
+ CacheIRReader reader(stubInfo);
+ while (reader.more()) {
+ const uint8_t* opStart = reader.currentPosition();
+
+ CacheOp op = reader.readOp();
+ CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
+ uint32_t argLength = opInfo.argLength;
+ mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
+
+ switch (op) {
+ case CacheOp::GuardSpecificFunction: {
+ // If we see a guard, remember which operand we are guarding.
+ MOZ_ASSERT(data.isNothing());
+ calleeGuardOperand = reader.objOperandId();
+ uint32_t targetOffset = reader.stubOffset();
+ (void)reader.stubOffset(); // nargsAndFlags
+ uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset);
+ target = reinterpret_cast<JSFunction*>(rawTarget);
+ break;
+ }
+ case CacheOp::GuardFunctionScript: {
+ MOZ_ASSERT(data.isNothing());
+ calleeGuardOperand = reader.objOperandId();
+ uint32_t targetOffset = reader.stubOffset();
+ uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, targetOffset);
+ target = reinterpret_cast<BaseScript*>(rawTarget)->function();
+ (void)reader.stubOffset(); // nargsAndFlags
+ break;
+ }
+ case CacheOp::CallScriptedFunction: {
+ // If we see a call, check if `callee` is the previously guarded
+ // operand. If it is, we know the target and can inline.
+ ObjOperandId calleeOperand = reader.objOperandId();
+ mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId();
+ flags = reader.callFlags();
+ mozilla::DebugOnly<uint32_t> argcFixed = reader.uint32Immediate();
+ MOZ_ASSERT(argcFixed <= MaxUnrolledArgCopy);
+
+ if (calleeOperand == calleeGuardOperand) {
+ MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0);
+ MOZ_ASSERT(data.isNothing());
+ data.emplace();
+ data->endOfSharedPrefix = opStart;
+ }
+ break;
+ }
+ case CacheOp::CallInlinedFunction: {
+ ObjOperandId calleeOperand = reader.objOperandId();
+ mozilla::DebugOnly<Int32OperandId> argcId = reader.int32OperandId();
+ uint32_t icScriptOffset = reader.stubOffset();
+ flags = reader.callFlags();
+ mozilla::DebugOnly<uint32_t> argcFixed = reader.uint32Immediate();
+ MOZ_ASSERT(argcFixed <= MaxUnrolledArgCopy);
+
+ if (calleeOperand == calleeGuardOperand) {
+ MOZ_ASSERT(static_cast<OperandId&>(argcId).id() == 0);
+ MOZ_ASSERT(data.isNothing());
+ data.emplace();
+ data->endOfSharedPrefix = opStart;
+ uintptr_t rawICScript =
+ stubInfo->getStubRawWord(stubData, icScriptOffset);
+ data->icScript = reinterpret_cast<ICScript*>(rawICScript);
+ }
+ break;
+ }
+ default:
+ if (!opInfo.transpile) {
+ return mozilla::Nothing();
+ }
+ if (data.isSome()) {
+ MOZ_ASSERT(op == CacheOp::ReturnFromIC);
+ }
+ reader.skip(argLength);
+ break;
+ }
+ MOZ_ASSERT(argStart + argLength == reader.currentPosition());
+ }
+
+ if (data.isSome()) {
+ // Warp only supports inlining Standard and FunCall calls.
+ if (flags.getArgFormat() != CallFlags::Standard &&
+ flags.getArgFormat() != CallFlags::FunCall) {
+ return mozilla::Nothing();
+ }
+ data->calleeOperand = calleeGuardOperand;
+ data->callFlags = flags;
+ data->target = target;
+ }
+ return data;
+}
+
+Maybe<InlinableGetterData> FindInlinableGetterData(ICCacheIRStub* stub) {
+ Maybe<InlinableGetterData> data;
+
+ const CacheIRStubInfo* stubInfo = stub->stubInfo();
+ const uint8_t* stubData = stub->stubDataStart();
+
+ CacheIRReader reader(stubInfo);
+ while (reader.more()) {
+ const uint8_t* opStart = reader.currentPosition();
+
+ CacheOp op = reader.readOp();
+ CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
+ uint32_t argLength = opInfo.argLength;
+ mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
+
+ switch (op) {
+ case CacheOp::CallScriptedGetterResult: {
+ data.emplace();
+ data->receiverOperand = reader.valOperandId();
+
+ uint32_t getterOffset = reader.stubOffset();
+ uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, getterOffset);
+ data->target = reinterpret_cast<JSFunction*>(rawTarget);
+
+ data->sameRealm = reader.readBool();
+ (void)reader.stubOffset(); // nargsAndFlags
+
+ data->endOfSharedPrefix = opStart;
+ break;
+ }
+ case CacheOp::CallInlinedGetterResult: {
+ data.emplace();
+ data->receiverOperand = reader.valOperandId();
+
+ uint32_t getterOffset = reader.stubOffset();
+ uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, getterOffset);
+ data->target = reinterpret_cast<JSFunction*>(rawTarget);
+
+ uint32_t icScriptOffset = reader.stubOffset();
+ uintptr_t rawICScript =
+ stubInfo->getStubRawWord(stubData, icScriptOffset);
+ data->icScript = reinterpret_cast<ICScript*>(rawICScript);
+
+ data->sameRealm = reader.readBool();
+ (void)reader.stubOffset(); // nargsAndFlags
+
+ data->endOfSharedPrefix = opStart;
+ break;
+ }
+ default:
+ if (!opInfo.transpile) {
+ return mozilla::Nothing();
+ }
+ if (data.isSome()) {
+ MOZ_ASSERT(op == CacheOp::ReturnFromIC);
+ }
+ reader.skip(argLength);
+ break;
+ }
+ MOZ_ASSERT(argStart + argLength == reader.currentPosition());
+ }
+
+ return data;
+}
+
+Maybe<InlinableSetterData> FindInlinableSetterData(ICCacheIRStub* stub) {
+ Maybe<InlinableSetterData> data;
+
+ const CacheIRStubInfo* stubInfo = stub->stubInfo();
+ const uint8_t* stubData = stub->stubDataStart();
+
+ CacheIRReader reader(stubInfo);
+ while (reader.more()) {
+ const uint8_t* opStart = reader.currentPosition();
+
+ CacheOp op = reader.readOp();
+ CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
+ uint32_t argLength = opInfo.argLength;
+ mozilla::DebugOnly<const uint8_t*> argStart = reader.currentPosition();
+
+ switch (op) {
+ case CacheOp::CallScriptedSetter: {
+ data.emplace();
+ data->receiverOperand = reader.objOperandId();
+
+ uint32_t setterOffset = reader.stubOffset();
+ uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, setterOffset);
+ data->target = reinterpret_cast<JSFunction*>(rawTarget);
+
+ data->rhsOperand = reader.valOperandId();
+ data->sameRealm = reader.readBool();
+ (void)reader.stubOffset(); // nargsAndFlags
+
+ data->endOfSharedPrefix = opStart;
+ break;
+ }
+ case CacheOp::CallInlinedSetter: {
+ data.emplace();
+ data->receiverOperand = reader.objOperandId();
+
+ uint32_t setterOffset = reader.stubOffset();
+ uintptr_t rawTarget = stubInfo->getStubRawWord(stubData, setterOffset);
+ data->target = reinterpret_cast<JSFunction*>(rawTarget);
+
+ data->rhsOperand = reader.valOperandId();
+
+ uint32_t icScriptOffset = reader.stubOffset();
+ uintptr_t rawICScript =
+ stubInfo->getStubRawWord(stubData, icScriptOffset);
+ data->icScript = reinterpret_cast<ICScript*>(rawICScript);
+
+ data->sameRealm = reader.readBool();
+ (void)reader.stubOffset(); // nargsAndFlags
+
+ data->endOfSharedPrefix = opStart;
+ break;
+ }
+ default:
+ if (!opInfo.transpile) {
+ return mozilla::Nothing();
+ }
+ if (data.isSome()) {
+ MOZ_ASSERT(op == CacheOp::ReturnFromIC);
+ }
+ reader.skip(argLength);
+ break;
+ }
+ MOZ_ASSERT(argStart + argLength == reader.currentPosition());
+ }
+
+ return data;
+}
+
+// Return the maximum number of actual arguments that will be passed to the
+// target function. This may be an overapproximation, for example when inlining
+// js::fun_call we may omit an argument.
+static uint32_t GetMaxCalleeNumActuals(BytecodeLocation loc) {
+ switch (loc.getOp()) {
+ case JSOp::GetProp:
+ case JSOp::GetElem:
+ // Getters do not pass arguments.
+ return 0;
+
+ case JSOp::SetProp:
+ case JSOp::StrictSetProp:
+ // Setters pass 1 argument.
+ return 1;
+
+ case JSOp::Call:
+ case JSOp::CallContent:
+ case JSOp::CallIgnoresRv:
+ case JSOp::CallIter:
+ case JSOp::CallContentIter:
+ case JSOp::New:
+ case JSOp::NewContent:
+ case JSOp::SuperCall:
+ return loc.getCallArgc();
+
+ default:
+ MOZ_CRASH("Unsupported op");
+ }
+}
+
+/*static*/
+bool TrialInliner::canInline(JSFunction* target, HandleScript caller,
+ BytecodeLocation loc) {
+ if (!target->hasJitScript()) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: no JIT script");
+ return false;
+ }
+ JSScript* script = target->nonLazyScript();
+ if (!script->jitScript()->hasBaselineScript()) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: no BaselineScript");
+ return false;
+ }
+ if (script->uninlineable()) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: uninlineable flag");
+ return false;
+ }
+ if (!script->canIonCompile()) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: can't ion-compile");
+ return false;
+ }
+ if (script->isDebuggee()) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: is debuggee");
+ return false;
+ }
+ // Don't inline cross-realm calls.
+ if (target->realm() != caller->realm()) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: cross-realm call");
+ return false;
+ }
+
+ uint32_t maxCalleeNumActuals = GetMaxCalleeNumActuals(loc);
+ if (script->needsArgsObj() &&
+ maxCalleeNumActuals > ArgumentsObject::MaxInlinedArgs) {
+ JitSpew(JitSpew_WarpTrialInlining,
+ "SKIP: needs arguments object with %u actual args (maximum %u)",
+ maxCalleeNumActuals, ArgumentsObject::MaxInlinedArgs);
+ return false;
+ }
+
+ if (TooManyFormalArguments(target->nargs())) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: Too many formal arguments: %u",
+ unsigned(target->nargs()));
+ return false;
+ }
+
+ if (TooManyFormalArguments(maxCalleeNumActuals)) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: argc too large: %u",
+ unsigned(loc.getCallArgc()));
+ return false;
+ }
+
+ return true;
+}
+
+bool TrialInliner::shouldInline(JSFunction* target, ICCacheIRStub* stub,
+ BytecodeLocation loc) {
+#ifdef JS_JITSPEW
+ BaseScript* baseScript =
+ target->hasBaseScript() ? target->baseScript() : nullptr;
+ JitSpew(JitSpew_WarpTrialInlining,
+ "Inlining candidate JSOp::%s (offset=%u): callee script %s:%u:%u",
+ CodeName(loc.getOp()), loc.bytecodeToOffset(script_),
+ baseScript ? baseScript->filename() : "<not-scripted>",
+ baseScript ? baseScript->lineno() : 0,
+ baseScript ? baseScript->column() : 0);
+ JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
+#endif
+
+ if (!canInline(target, script_, loc)) {
+ return false;
+ }
+
+ // Don't inline (direct) recursive calls. This still allows recursion if
+ // called through another function (f => g => f).
+ JSScript* targetScript = target->nonLazyScript();
+ if (script_ == targetScript) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: recursion");
+ return false;
+ }
+
+ // Don't inline if the callee has a loop that was hot enough to enter Warp
+ // via OSR. This helps prevent getting stuck in Baseline code for a long time.
+ if (targetScript->jitScript()->hadIonOSR()) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: had OSR");
+ return false;
+ }
+
+ // Ensure the total bytecode size does not exceed ionMaxScriptSize.
+ size_t newTotalSize =
+ inliningRootTotalBytecodeSize() + targetScript->length();
+ if (newTotalSize > JitOptions.ionMaxScriptSize) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: total size too big");
+ return false;
+ }
+
+ uint32_t entryCount = stub->enteredCount();
+ if (entryCount < JitOptions.inliningEntryThreshold) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: Entry count is %u (minimum %u)",
+ unsigned(entryCount), unsigned(JitOptions.inliningEntryThreshold));
+ return false;
+ }
+
+ if (!JitOptions.isSmallFunction(targetScript)) {
+ if (!targetScript->isInlinableLargeFunction()) {
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: Length is %u (maximum %u)",
+ unsigned(targetScript->length()),
+ unsigned(JitOptions.smallFunctionMaxBytecodeLength));
+ return false;
+ }
+
+ JitSpew(JitSpew_WarpTrialInlining,
+ "INFO: Ignored length (%u) of InlinableLargeFunction",
+ unsigned(targetScript->length()));
+ }
+
+ return true;
+}
+
+ICScript* TrialInliner::createInlinedICScript(JSFunction* target,
+ BytecodeLocation loc) {
+ MOZ_ASSERT(target->hasJitEntry());
+ MOZ_ASSERT(target->hasJitScript());
+
+ InliningRoot* root = getOrCreateInliningRoot();
+ if (!root) {
+ return nullptr;
+ }
+
+ JSScript* targetScript = target->baseScript()->asJSScript();
+
+ // We don't have to check for overflow here because we have already
+ // successfully allocated an ICScript with this number of entries
+ // when creating the JitScript for the target function, and we
+ // checked for overflow then.
+ uint32_t fallbackStubsOffset =
+ sizeof(ICScript) + targetScript->numICEntries() * sizeof(ICEntry);
+ uint32_t allocSize = fallbackStubsOffset +
+ targetScript->numICEntries() * sizeof(ICFallbackStub);
+
+ void* raw = cx()->pod_malloc<uint8_t>(allocSize);
+ MOZ_ASSERT(uintptr_t(raw) % alignof(ICScript) == 0);
+ if (!raw) {
+ return nullptr;
+ }
+
+ uint32_t initialWarmUpCount = JitOptions.trialInliningInitialWarmUpCount;
+
+ uint32_t depth = icScript_->depth() + 1;
+ UniquePtr<ICScript> inlinedICScript(new (raw) ICScript(
+ initialWarmUpCount, fallbackStubsOffset, allocSize, depth, root));
+
+ inlinedICScript->initICEntries(cx(), targetScript);
+
+ uint32_t pcOffset = loc.bytecodeToOffset(script_);
+ ICScript* result = inlinedICScript.get();
+ if (!icScript_->addInlinedChild(cx(), std::move(inlinedICScript), pcOffset)) {
+ return nullptr;
+ }
+ MOZ_ASSERT(result->numICEntries() == targetScript->numICEntries());
+
+ root->addToTotalBytecodeSize(targetScript->length());
+
+ JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
+ JitSpew(JitSpew_WarpTrialInlining,
+ "SUCCESS: Outer ICScript: %p Inner ICScript: %p", icScript_, result);
+
+ return result;
+}
+
+bool TrialInliner::maybeInlineCall(ICEntry& entry, ICFallbackStub* fallback,
+ BytecodeLocation loc) {
+ ICCacheIRStub* stub = maybeSingleStub(entry);
+ if (!stub) {
+#ifdef JS_JITSPEW
+ if (fallback->numOptimizedStubs() > 1) {
+ JitSpew(JitSpew_WarpTrialInlining,
+ "Inlining candidate JSOp::%s (offset=%u):", CodeName(loc.getOp()),
+ fallback->pcOffset());
+ JitSpewIndent spewIndent(JitSpew_WarpTrialInlining);
+ JitSpew(JitSpew_WarpTrialInlining, "SKIP: Polymorphic (%u stubs)",
+ (unsigned)fallback->numOptimizedStubs());
+ }
+#endif
+ return true;
+ }
+
+ MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
+
+ // Look for a CallScriptedFunction with a known target.
+ Maybe<InlinableCallData> data = FindInlinableCallData(stub);
+ if (data.isNothing()) {
+ return true;
+ }
+
+ MOZ_ASSERT(!data->icScript);
+
+ // Decide whether to inline the target.
+ if (!shouldInline(data->target, stub, loc)) {
+ return true;
+ }
+
+ ICScript* newICScript = createInlinedICScript(data->target, loc);
+ if (!newICScript) {
+ return false;
+ }
+
+ CacheIRWriter writer(cx());
+ Int32OperandId argcId(writer.setInputOperandId(0));
+ cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
+
+ writer.callInlinedFunction(data->calleeOperand, argcId, newICScript,
+ data->callFlags,
+ ClampFixedArgc(loc.getCallArgc()));
+ writer.returnFromIC();
+
+ return replaceICStub(entry, fallback, writer, CacheKind::Call);
+}
+
+bool TrialInliner::maybeInlineGetter(ICEntry& entry, ICFallbackStub* fallback,
+ BytecodeLocation loc, CacheKind kind) {
+ ICCacheIRStub* stub = maybeSingleStub(entry);
+ if (!stub) {
+ return true;
+ }
+
+ MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
+
+ Maybe<InlinableGetterData> data = FindInlinableGetterData(stub);
+ if (data.isNothing()) {
+ return true;
+ }
+
+ MOZ_ASSERT(!data->icScript);
+
+ // Decide whether to inline the target.
+ if (!shouldInline(data->target, stub, loc)) {
+ return true;
+ }
+
+ ICScript* newICScript = createInlinedICScript(data->target, loc);
+ if (!newICScript) {
+ return false;
+ }
+
+ CacheIRWriter writer(cx());
+ ValOperandId valId(writer.setInputOperandId(0));
+ if (kind == CacheKind::GetElem) {
+ // Register the key operand.
+ writer.setInputOperandId(1);
+ }
+ cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
+
+ writer.callInlinedGetterResult(data->receiverOperand, data->target,
+ newICScript, data->sameRealm);
+ writer.returnFromIC();
+
+ return replaceICStub(entry, fallback, writer, kind);
+}
+
+bool TrialInliner::maybeInlineSetter(ICEntry& entry, ICFallbackStub* fallback,
+ BytecodeLocation loc, CacheKind kind) {
+ ICCacheIRStub* stub = maybeSingleStub(entry);
+ if (!stub) {
+ return true;
+ }
+
+ MOZ_ASSERT(!icScript_->hasInlinedChild(fallback->pcOffset()));
+
+ Maybe<InlinableSetterData> data = FindInlinableSetterData(stub);
+ if (data.isNothing()) {
+ return true;
+ }
+
+ MOZ_ASSERT(!data->icScript);
+
+ // Decide whether to inline the target.
+ if (!shouldInline(data->target, stub, loc)) {
+ return true;
+ }
+
+ ICScript* newICScript = createInlinedICScript(data->target, loc);
+ if (!newICScript) {
+ return false;
+ }
+
+ CacheIRWriter writer(cx());
+ ValOperandId objValId(writer.setInputOperandId(0));
+ ValOperandId rhsValId(writer.setInputOperandId(1));
+ cloneSharedPrefix(stub, data->endOfSharedPrefix, writer);
+
+ writer.callInlinedSetter(data->receiverOperand, data->target,
+ data->rhsOperand, newICScript, data->sameRealm);
+ writer.returnFromIC();
+
+ return replaceICStub(entry, fallback, writer, kind);
+}
+
+bool TrialInliner::tryInlining() {
+ uint32_t numICEntries = icScript_->numICEntries();
+ BytecodeLocation startLoc = script_->location();
+
+ for (uint32_t icIndex = 0; icIndex < numICEntries; icIndex++) {
+ ICEntry& entry = icScript_->icEntry(icIndex);
+ ICFallbackStub* fallback = icScript_->fallbackStub(icIndex);
+
+ if (!TryFoldingStubs(cx(), fallback, script_, icScript_)) {
+ return false;
+ }
+
+ BytecodeLocation loc =
+ startLoc + BytecodeLocationOffset(fallback->pcOffset());
+ JSOp op = loc.getOp();
+ switch (op) {
+ case JSOp::Call:
+ case JSOp::CallContent:
+ case JSOp::CallIgnoresRv:
+ case JSOp::CallIter:
+ case JSOp::CallContentIter:
+ case JSOp::New:
+ case JSOp::NewContent:
+ case JSOp::SuperCall:
+ if (!maybeInlineCall(entry, fallback, loc)) {
+ return false;
+ }
+ break;
+ case JSOp::GetProp:
+ if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetProp)) {
+ return false;
+ }
+ break;
+ case JSOp::GetElem:
+ if (!maybeInlineGetter(entry, fallback, loc, CacheKind::GetElem)) {
+ return false;
+ }
+ break;
+ case JSOp::SetProp:
+ case JSOp::StrictSetProp:
+ if (!maybeInlineSetter(entry, fallback, loc, CacheKind::SetProp)) {
+ return false;
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ return true;
+}
+
+InliningRoot* TrialInliner::maybeGetInliningRoot() const {
+ if (auto* root = icScript_->inliningRoot()) {
+ return root;
+ }
+
+ MOZ_ASSERT(!icScript_->isInlined());
+ return script_->jitScript()->inliningRoot();
+}
+
+InliningRoot* TrialInliner::getOrCreateInliningRoot() {
+ if (auto* root = maybeGetInliningRoot()) {
+ return root;
+ }
+ return script_->jitScript()->getOrCreateInliningRoot(cx(), script_);
+}
+
+size_t TrialInliner::inliningRootTotalBytecodeSize() const {
+ if (auto* root = maybeGetInliningRoot()) {
+ return root->totalBytecodeSize();
+ }
+ return script_->length();
+}
+
+bool InliningRoot::addInlinedScript(UniquePtr<ICScript> icScript) {
+ return inlinedScripts_.append(std::move(icScript));
+}
+
+void InliningRoot::trace(JSTracer* trc) {
+ TraceEdge(trc, &owningScript_, "inlining-root-owning-script");
+ for (auto& inlinedScript : inlinedScripts_) {
+ inlinedScript->trace(trc);
+ }
+}
+
+void InliningRoot::purgeOptimizedStubs(Zone* zone) {
+ for (auto& inlinedScript : inlinedScripts_) {
+ inlinedScript->purgeOptimizedStubs(zone);
+ }
+}
+
+void InliningRoot::resetWarmUpCounts(uint32_t count) {
+ for (auto& inlinedScript : inlinedScripts_) {
+ inlinedScript->resetWarmUpCount(count);
+ }
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/TrialInlining.h b/js/src/jit/TrialInlining.h
new file mode 100644
index 0000000000..fa6ecf57ae
--- /dev/null
+++ b/js/src/jit/TrialInlining.h
@@ -0,0 +1,187 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_TrialInlining_h
+#define jit_TrialInlining_h
+
+#include "mozilla/Attributes.h"
+#include "mozilla/Maybe.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jstypes.h"
+#include "NamespaceImports.h"
+
+#include "gc/Barrier.h"
+#include "jit/CacheIR.h"
+#include "jit/ICStubSpace.h"
+#include "js/RootingAPI.h"
+#include "js/TypeDecls.h"
+#include "js/UniquePtr.h"
+#include "js/Vector.h"
+#include "vm/JSScript.h"
+
+/*
+ * [SMDOC] Trial Inlining
+ *
+ * WarpBuilder relies on transpiling CacheIR. When inlining scripted
+ * functions in WarpBuilder, we want our ICs to be as monomorphic as
+ * possible. Functions with multiple callers complicate this. An IC in
+ * such a function might be monomorphic for any given caller, but
+ * polymorphic overall. This make the input to WarpBuilder less precise.
+ *
+ * To solve this problem, we do trial inlining. During baseline
+ * execution, we identify call sites for which it would be useful to
+ * have more precise inlining data. For each such call site, we
+ * allocate a fresh ICScript and replace the existing call IC with a
+ * new specialized IC that invokes the callee using the new
+ * ICScript. Other callers of the callee will continue using the
+ * default ICScript. When we eventually Warp-compile the script, we
+ * can generate code for the callee using the IC information in our
+ * private ICScript, which is specialized for its caller.
+ *
+ * The same approach can be used to inline recursively.
+ */
+
+class JS_PUBLIC_API JSTracer;
+struct JS_PUBLIC_API JSContext;
+
+class JSFunction;
+
+namespace JS {
+class Zone;
+}
+
+namespace js {
+
+class BytecodeLocation;
+
+namespace jit {
+
+class BaselineFrame;
+class CacheIRWriter;
+class ICCacheIRStub;
+class ICEntry;
+class ICFallbackStub;
+class ICScript;
+
+/*
+ * An InliningRoot is owned by a JitScript. In turn, it owns the set
+ * of ICScripts that are candidates for being inlined in that JitScript.
+ */
+class InliningRoot {
+ public:
+ explicit InliningRoot(JSContext* cx, JSScript* owningScript)
+ : owningScript_(owningScript),
+ inlinedScripts_(cx),
+ totalBytecodeSize_(owningScript->length()) {}
+
+ JitScriptICStubSpace* jitScriptStubSpace() { return &jitScriptStubSpace_; }
+
+ void trace(JSTracer* trc);
+
+ bool addInlinedScript(js::UniquePtr<ICScript> icScript);
+
+ uint32_t numInlinedScripts() const { return inlinedScripts_.length(); }
+
+ void purgeOptimizedStubs(Zone* zone);
+ void resetWarmUpCounts(uint32_t count);
+
+ JSScript* owningScript() const { return owningScript_; }
+
+ size_t totalBytecodeSize() const { return totalBytecodeSize_; }
+
+ void addToTotalBytecodeSize(size_t size) { totalBytecodeSize_ += size; }
+
+ private:
+ JitScriptICStubSpace jitScriptStubSpace_ = {};
+ HeapPtr<JSScript*> owningScript_;
+ js::Vector<js::UniquePtr<ICScript>> inlinedScripts_;
+
+ // Bytecode size of outer script and all inlined scripts.
+ size_t totalBytecodeSize_;
+};
+
+class InlinableOpData {
+ public:
+ JSFunction* target = nullptr;
+ ICScript* icScript = nullptr;
+ const uint8_t* endOfSharedPrefix = nullptr;
+};
+
+class InlinableCallData : public InlinableOpData {
+ public:
+ ObjOperandId calleeOperand;
+ CallFlags callFlags;
+};
+
+class InlinableGetterData : public InlinableOpData {
+ public:
+ ValOperandId receiverOperand;
+ bool sameRealm = false;
+};
+
+class InlinableSetterData : public InlinableOpData {
+ public:
+ ObjOperandId receiverOperand;
+ ValOperandId rhsOperand;
+ bool sameRealm = false;
+};
+
+mozilla::Maybe<InlinableOpData> FindInlinableOpData(ICCacheIRStub* stub,
+ BytecodeLocation loc);
+
+mozilla::Maybe<InlinableCallData> FindInlinableCallData(ICCacheIRStub* stub);
+mozilla::Maybe<InlinableGetterData> FindInlinableGetterData(
+ ICCacheIRStub* stub);
+mozilla::Maybe<InlinableSetterData> FindInlinableSetterData(
+ ICCacheIRStub* stub);
+
+class MOZ_RAII TrialInliner {
+ public:
+ TrialInliner(JSContext* cx, HandleScript script, ICScript* icScript)
+ : cx_(cx), script_(script), icScript_(icScript) {}
+
+ JSContext* cx() { return cx_; }
+
+ [[nodiscard]] bool tryInlining();
+ [[nodiscard]] bool maybeInlineCall(ICEntry& entry, ICFallbackStub* fallback,
+ BytecodeLocation loc);
+ [[nodiscard]] bool maybeInlineGetter(ICEntry& entry, ICFallbackStub* fallback,
+ BytecodeLocation loc, CacheKind kind);
+ [[nodiscard]] bool maybeInlineSetter(ICEntry& entry, ICFallbackStub* fallback,
+ BytecodeLocation loc, CacheKind kind);
+
+ static bool canInline(JSFunction* target, HandleScript caller,
+ BytecodeLocation loc);
+
+ private:
+ ICCacheIRStub* maybeSingleStub(const ICEntry& entry);
+ void cloneSharedPrefix(ICCacheIRStub* stub, const uint8_t* endOfPrefix,
+ CacheIRWriter& writer);
+ ICScript* createInlinedICScript(JSFunction* target, BytecodeLocation loc);
+ [[nodiscard]] bool replaceICStub(ICEntry& entry, ICFallbackStub* fallback,
+ CacheIRWriter& writer, CacheKind kind);
+
+ bool shouldInline(JSFunction* target, ICCacheIRStub* stub,
+ BytecodeLocation loc);
+
+ InliningRoot* getOrCreateInliningRoot();
+ InliningRoot* maybeGetInliningRoot() const;
+ size_t inliningRootTotalBytecodeSize() const;
+
+ JSContext* cx_;
+ HandleScript script_;
+ ICScript* icScript_;
+};
+
+bool DoTrialInlining(JSContext* cx, BaselineFrame* frame);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_TrialInlining_h */
diff --git a/js/src/jit/TypeData.h b/js/src/jit/TypeData.h
new file mode 100644
index 0000000000..4a05895e87
--- /dev/null
+++ b/js/src/jit/TypeData.h
@@ -0,0 +1,54 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_TypeData_h
+#define jit_TypeData_h
+
+#include "js/Value.h"
+
+namespace js {
+namespace jit {
+
+class TypeData {
+ JSValueType type_;
+
+ public:
+ TypeData() : type_(JSVAL_TYPE_UNKNOWN) {}
+ explicit TypeData(JSValueType type) : type_(type) {}
+
+ JSValueType type() const { return type_; }
+ bool hasData() const { return type_ != JSVAL_TYPE_UNKNOWN; }
+};
+
+class TypeDataList {
+ const static size_t MaxLength = 6;
+
+ uint8_t count_ = 0;
+ TypeData typeData_[MaxLength];
+
+ public:
+ TypeDataList() {}
+
+ uint8_t count() const { return count_; }
+
+ void addTypeData(TypeData data) {
+ MOZ_ASSERT(count_ < MaxLength);
+ MOZ_ASSERT(!typeData_[count_].hasData());
+ typeData_[count_++] = data;
+ }
+ TypeData get(uint32_t idx) const {
+ MOZ_ASSERT(idx < count_);
+ return typeData_[idx];
+ }
+
+ const TypeData* begin() const { return &typeData_[0]; }
+ const TypeData* end() const { return begin() + count_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_TypeData_h */
diff --git a/js/src/jit/TypePolicy.cpp b/js/src/jit/TypePolicy.cpp
new file mode 100644
index 0000000000..31b478452e
--- /dev/null
+++ b/js/src/jit/TypePolicy.cpp
@@ -0,0 +1,1139 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/TypePolicy.h"
+
+#include "jit/JitAllocPolicy.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+using namespace js;
+using namespace js::jit;
+
+static void EnsureOperandNotFloat32(TempAllocator& alloc, MInstruction* def,
+ unsigned op) {
+ MDefinition* in = def->getOperand(op);
+ if (in->type() == MIRType::Float32) {
+ MToDouble* replace = MToDouble::New(alloc, in);
+ def->block()->insertBefore(def, replace);
+ if (def->isRecoveredOnBailout()) {
+ replace->setRecoveredOnBailout();
+ }
+ def->replaceOperand(op, replace);
+ }
+}
+
+template <class T>
+[[nodiscard]] static bool ConvertOperand(TempAllocator& alloc,
+ MInstruction* def, unsigned op,
+ MIRType expected) {
+ MDefinition* in = def->getOperand(op);
+ if (in->type() == expected) {
+ return true;
+ }
+
+ auto* replace = T::New(alloc, in);
+ def->block()->insertBefore(def, replace);
+ def->replaceOperand(op, replace);
+
+ return replace->typePolicy()->adjustInputs(alloc, replace);
+}
+
+[[nodiscard]] static bool UnboxOperand(TempAllocator& alloc, MInstruction* def,
+ unsigned op, MIRType expected) {
+ MDefinition* in = def->getOperand(op);
+ if (in->type() == expected) {
+ return true;
+ }
+
+ auto* replace = MUnbox::New(alloc, in, expected, MUnbox::Fallible);
+ replace->setBailoutKind(BailoutKind::TypePolicy);
+ def->block()->insertBefore(def, replace);
+ def->replaceOperand(op, replace);
+
+ return replace->typePolicy()->adjustInputs(alloc, replace);
+}
+
+MDefinition* js::jit::AlwaysBoxAt(TempAllocator& alloc, MInstruction* at,
+ MDefinition* operand) {
+ MDefinition* boxedOperand = operand;
+ // Replace Float32 by double
+ if (operand->type() == MIRType::Float32) {
+ MInstruction* replace = MToDouble::New(alloc, operand);
+ at->block()->insertBefore(at, replace);
+ boxedOperand = replace;
+ }
+ MBox* box = MBox::New(alloc, boxedOperand);
+ at->block()->insertBefore(at, box);
+ return box;
+}
+
+static MDefinition* BoxAt(TempAllocator& alloc, MInstruction* at,
+ MDefinition* operand) {
+ if (operand->isUnbox()) {
+ return operand->toUnbox()->input();
+ }
+ return AlwaysBoxAt(alloc, at, operand);
+}
+
+bool BoxInputsPolicy::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ for (size_t i = 0, e = ins->numOperands(); i < e; i++) {
+ MDefinition* in = ins->getOperand(i);
+ if (in->type() == MIRType::Value) {
+ continue;
+ }
+ ins->replaceOperand(i, BoxAt(alloc, ins, in));
+ }
+ return true;
+}
+
+bool ArithPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) const {
+ MOZ_ASSERT(IsNumberType(ins->type()));
+ MOZ_ASSERT(ins->type() == MIRType::Double || ins->type() == MIRType::Int32 ||
+ ins->type() == MIRType::Float32);
+
+ for (size_t i = 0, e = ins->numOperands(); i < e; i++) {
+ MDefinition* in = ins->getOperand(i);
+ if (in->type() == ins->type()) {
+ continue;
+ }
+
+ MInstruction* replace;
+
+ if (ins->type() == MIRType::Double) {
+ replace = MToDouble::New(alloc, in);
+ } else if (ins->type() == MIRType::Float32) {
+ replace = MToFloat32::New(alloc, in);
+ } else {
+ replace = MToNumberInt32::New(alloc, in);
+ }
+
+ replace->setBailoutKind(BailoutKind::TypePolicy);
+ ins->block()->insertBefore(ins, replace);
+ ins->replaceOperand(i, replace);
+
+ if (!replace->typePolicy()->adjustInputs(alloc, replace)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool BigIntArithPolicy::adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const {
+ MOZ_ASSERT(ins->type() == MIRType::BigInt);
+
+ for (size_t i = 0, e = ins->numOperands(); i < e; i++) {
+ if (!ConvertOperand<MToBigInt>(alloc, ins, i, MIRType::BigInt)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool AllDoublePolicy::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ for (size_t i = 0, e = ins->numOperands(); i < e; i++) {
+ MDefinition* in = ins->getOperand(i);
+ if (in->type() == MIRType::Double) {
+ continue;
+ }
+
+ if (!alloc.ensureBallast()) {
+ return false;
+ }
+ MInstruction* replace = MToDouble::New(alloc, in);
+
+ ins->block()->insertBefore(ins, replace);
+ ins->replaceOperand(i, replace);
+
+ if (!replace->typePolicy()->adjustInputs(alloc, replace)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool ComparePolicy::adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const {
+ MOZ_ASSERT(def->isCompare());
+ MCompare* compare = def->toCompare();
+
+ // Convert Float32 operands to doubles
+ for (size_t i = 0; i < 2; i++) {
+ MDefinition* in = def->getOperand(i);
+ if (in->type() == MIRType::Float32) {
+ MInstruction* replace = MToDouble::New(alloc, in);
+ def->block()->insertBefore(def, replace);
+ def->replaceOperand(i, replace);
+ }
+ }
+
+ auto replaceOperand = [&](size_t index, auto* replace) {
+ def->block()->insertBefore(def, replace);
+ def->replaceOperand(index, replace);
+ return replace->typePolicy()->adjustInputs(alloc, replace);
+ };
+
+ if (compare->compareType() == MCompare::Compare_Undefined ||
+ compare->compareType() == MCompare::Compare_Null) {
+ // Nothing to do for undefined and null, lowering handles all types.
+ return true;
+ }
+
+ if (compare->compareType() == MCompare::Compare_UIntPtr) {
+ MOZ_ASSERT(compare->lhs()->type() == MIRType::IntPtr);
+ MOZ_ASSERT(compare->rhs()->type() == MIRType::IntPtr);
+ return true;
+ }
+
+ // Compare_BigInt_Int32 specialization is done for "BigInt <cmp> Int32".
+ // Compare_BigInt_Double specialization is done for "BigInt <cmp> Double".
+ // Compare_BigInt_String specialization is done for "BigInt <cmp> String".
+ if (compare->compareType() == MCompare::Compare_BigInt_Int32 ||
+ compare->compareType() == MCompare::Compare_BigInt_Double ||
+ compare->compareType() == MCompare::Compare_BigInt_String) {
+ if (MDefinition* in = def->getOperand(0); in->type() != MIRType::BigInt) {
+ auto* replace =
+ MUnbox::New(alloc, in, MIRType::BigInt, MUnbox::Infallible);
+ if (!replaceOperand(0, replace)) {
+ return false;
+ }
+ }
+
+ MDefinition* in = def->getOperand(1);
+
+ MInstruction* replace = nullptr;
+ if (compare->compareType() == MCompare::Compare_BigInt_Int32) {
+ if (in->type() != MIRType::Int32) {
+ replace = MToNumberInt32::New(
+ alloc, in, IntConversionInputKind::NumbersOrBoolsOnly);
+ }
+ } else if (compare->compareType() == MCompare::Compare_BigInt_Double) {
+ if (in->type() != MIRType::Double) {
+ replace = MToDouble::New(alloc, in, MToFPInstruction::NumbersOnly);
+ }
+ } else {
+ MOZ_ASSERT(compare->compareType() == MCompare::Compare_BigInt_String);
+ if (in->type() != MIRType::String) {
+ replace = MUnbox::New(alloc, in, MIRType::String, MUnbox::Infallible);
+ }
+ }
+
+ if (replace) {
+ if (!replaceOperand(1, replace)) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ // Convert all inputs to the right input type
+ MIRType type = compare->inputType();
+ MOZ_ASSERT(type == MIRType::Int32 || type == MIRType::Double ||
+ type == MIRType::Float32 || type == MIRType::Object ||
+ type == MIRType::String || type == MIRType::Symbol ||
+ type == MIRType::BigInt);
+ for (size_t i = 0; i < 2; i++) {
+ MDefinition* in = def->getOperand(i);
+ if (in->type() == type) {
+ continue;
+ }
+
+ MInstruction* replace;
+
+ switch (type) {
+ case MIRType::Double:
+ replace = MToDouble::New(alloc, in, MToFPInstruction::NumbersOnly);
+ break;
+ case MIRType::Float32:
+ replace = MToFloat32::New(alloc, in, MToFPInstruction::NumbersOnly);
+ break;
+ case MIRType::Int32: {
+ IntConversionInputKind convert = IntConversionInputKind::NumbersOnly;
+ replace = MToNumberInt32::New(alloc, in, convert);
+ break;
+ }
+ case MIRType::Object:
+ replace = MUnbox::New(alloc, in, MIRType::Object, MUnbox::Infallible);
+ break;
+ case MIRType::String:
+ replace = MUnbox::New(alloc, in, MIRType::String, MUnbox::Infallible);
+ break;
+ case MIRType::Symbol:
+ replace = MUnbox::New(alloc, in, MIRType::Symbol, MUnbox::Infallible);
+ break;
+ case MIRType::BigInt:
+ replace = MUnbox::New(alloc, in, MIRType::BigInt, MUnbox::Infallible);
+ break;
+ default:
+ MOZ_CRASH("Unknown compare specialization");
+ }
+
+ if (!replaceOperand(i, replace)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool TestPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) const {
+ MDefinition* op = ins->getOperand(0);
+ switch (op->type()) {
+ case MIRType::Value:
+ case MIRType::Null:
+ case MIRType::Undefined:
+ case MIRType::Boolean:
+ case MIRType::Int32:
+ case MIRType::Double:
+ case MIRType::Float32:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ case MIRType::Object:
+ break;
+
+ case MIRType::String: {
+ MStringLength* length = MStringLength::New(alloc, op);
+ ins->block()->insertBefore(ins, length);
+ ins->replaceOperand(0, length);
+ break;
+ }
+
+ default:
+ MOZ_ASSERT(IsMagicType(op->type()));
+ ins->replaceOperand(0, BoxAt(alloc, ins, op));
+ break;
+ }
+ return true;
+}
+
+bool BitwisePolicy::adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const {
+ MOZ_ASSERT(ins->type() == MIRType::Int32 || ins->type() == MIRType::Double);
+
+ // This policy works for both unary and binary bitwise operations.
+ for (size_t i = 0, e = ins->numOperands(); i < e; i++) {
+ if (!ConvertOperand<MTruncateToInt32>(alloc, ins, i, MIRType::Int32)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool PowPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) const {
+ MOZ_ASSERT(ins->type() == MIRType::Int32 || ins->type() == MIRType::Double);
+
+ if (ins->type() == MIRType::Int32) {
+ // Both operands must be int32.
+ return UnboxedInt32Policy<0>::staticAdjustInputs(alloc, ins) &&
+ UnboxedInt32Policy<1>::staticAdjustInputs(alloc, ins);
+ }
+
+ // Otherwise, input must be a double.
+ if (!DoublePolicy<0>::staticAdjustInputs(alloc, ins)) {
+ return false;
+ }
+
+ // Power may be an int32 or a double. Integers receive a faster path.
+ MDefinition* power = ins->toPow()->power();
+ if (power->isToDouble()) {
+ MDefinition* input = power->toToDouble()->input();
+ if (input->type() == MIRType::Int32) {
+ ins->replaceOperand(1, input);
+ return true;
+ }
+ }
+ return DoublePolicy<1>::staticAdjustInputs(alloc, ins);
+}
+
+bool SignPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) const {
+ MOZ_ASSERT(ins->isSign());
+ MIRType specialization = ins->typePolicySpecialization();
+
+ // MSign is specialized for int32 input types.
+ if (specialization == MIRType::Int32) {
+ return UnboxedInt32Policy<0>::staticAdjustInputs(alloc, ins);
+ }
+
+ // Otherwise convert input to double.
+ MOZ_ASSERT(IsFloatingPointType(specialization));
+ return DoublePolicy<0>::staticAdjustInputs(alloc, ins);
+}
+
+template <unsigned Op>
+bool SymbolPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ return UnboxOperand(alloc, ins, Op, MIRType::Symbol);
+}
+
+template bool SymbolPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+
+template <unsigned Op>
+bool StringPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ return UnboxOperand(alloc, ins, Op, MIRType::String);
+}
+
+template bool StringPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool StringPolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool StringPolicy<2>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+
+template <unsigned Op>
+bool ConvertToStringPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MDefinition* in = ins->getOperand(Op);
+ if (in->type() == MIRType::String) {
+ return true;
+ }
+
+ MToString* replace =
+ MToString::New(alloc, in, MToString::SideEffectHandling::Bailout);
+ ins->block()->insertBefore(ins, replace);
+ ins->replaceOperand(Op, replace);
+
+ return ToStringPolicy::staticAdjustInputs(alloc, replace);
+}
+
+template bool ConvertToStringPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool ConvertToStringPolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool ConvertToStringPolicy<2>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+
+template <unsigned Op>
+bool BigIntPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ return UnboxOperand(alloc, ins, Op, MIRType::BigInt);
+}
+
+template bool BigIntPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool BigIntPolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+
+template <unsigned Op>
+bool UnboxedInt32Policy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def) {
+ return UnboxOperand(alloc, def, Op, MIRType::Int32);
+}
+
+template bool UnboxedInt32Policy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool UnboxedInt32Policy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool UnboxedInt32Policy<2>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool UnboxedInt32Policy<3>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+
+template <unsigned Op>
+bool Int32OrIntPtrPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def) {
+ MDefinition* in = def->getOperand(Op);
+ if (in->type() == MIRType::IntPtr) {
+ return true;
+ }
+
+ return UnboxedInt32Policy<Op>::staticAdjustInputs(alloc, def);
+}
+
+template bool Int32OrIntPtrPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool Int32OrIntPtrPolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+
+template <unsigned Op>
+bool ConvertToInt32Policy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def) {
+ return ConvertOperand<MToNumberInt32>(alloc, def, Op, MIRType::Int32);
+}
+
+template bool ConvertToInt32Policy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+
+template <unsigned Op>
+bool TruncateToInt32OrToBigIntPolicy<Op>::staticAdjustInputs(
+ TempAllocator& alloc, MInstruction* def) {
+ MOZ_ASSERT(def->isCompareExchangeTypedArrayElement() ||
+ def->isAtomicExchangeTypedArrayElement() ||
+ def->isAtomicTypedArrayElementBinop());
+
+ Scalar::Type type;
+ if (def->isCompareExchangeTypedArrayElement()) {
+ type = def->toCompareExchangeTypedArrayElement()->arrayType();
+ } else if (def->isAtomicExchangeTypedArrayElement()) {
+ type = def->toAtomicExchangeTypedArrayElement()->arrayType();
+ } else {
+ type = def->toAtomicTypedArrayElementBinop()->arrayType();
+ }
+
+ if (Scalar::isBigIntType(type)) {
+ return ConvertOperand<MToBigInt>(alloc, def, Op, MIRType::BigInt);
+ }
+ return ConvertOperand<MTruncateToInt32>(alloc, def, Op, MIRType::Int32);
+}
+
+template bool TruncateToInt32OrToBigIntPolicy<2>::staticAdjustInputs(
+ TempAllocator& alloc, MInstruction* def);
+template bool TruncateToInt32OrToBigIntPolicy<3>::staticAdjustInputs(
+ TempAllocator& alloc, MInstruction* def);
+
+template <unsigned Op>
+bool DoublePolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def) {
+ return ConvertOperand<MToDouble>(alloc, def, Op, MIRType::Double);
+}
+
+template bool DoublePolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool DoublePolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+
+template <unsigned Op>
+bool Float32Policy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def) {
+ return ConvertOperand<MToFloat32>(alloc, def, Op, MIRType::Float32);
+}
+
+template bool Float32Policy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool Float32Policy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool Float32Policy<2>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+
+template <unsigned Op>
+bool FloatingPointPolicy<Op>::adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const {
+ MIRType policyType = def->typePolicySpecialization();
+ if (policyType == MIRType::Double) {
+ return DoublePolicy<Op>::staticAdjustInputs(alloc, def);
+ }
+ return Float32Policy<Op>::staticAdjustInputs(alloc, def);
+}
+
+template bool FloatingPointPolicy<0>::adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const;
+
+template <unsigned Op>
+bool NoFloatPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def) {
+ EnsureOperandNotFloat32(alloc, def, Op);
+ return true;
+}
+
+template bool NoFloatPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool NoFloatPolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool NoFloatPolicy<2>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool NoFloatPolicy<3>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+
+template <unsigned FirstOp>
+bool NoFloatPolicyAfter<FirstOp>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def) {
+ for (size_t op = FirstOp, e = def->numOperands(); op < e; op++) {
+ EnsureOperandNotFloat32(alloc, def, op);
+ }
+ return true;
+}
+
+template bool NoFloatPolicyAfter<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool NoFloatPolicyAfter<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+template bool NoFloatPolicyAfter<2>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+
+template <unsigned Op>
+bool BoxPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MDefinition* in = ins->getOperand(Op);
+ if (in->type() == MIRType::Value) {
+ return true;
+ }
+
+ ins->replaceOperand(Op, BoxAt(alloc, ins, in));
+ return true;
+}
+
+template bool BoxPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool BoxPolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool BoxPolicy<2>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+
+template <unsigned Op, MIRType Type>
+bool BoxExceptPolicy<Op, Type>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MDefinition* in = ins->getOperand(Op);
+ if (in->type() == Type) {
+ return true;
+ }
+ return BoxPolicy<Op>::staticAdjustInputs(alloc, ins);
+}
+
+template bool BoxExceptPolicy<0, MIRType::Object>::staticAdjustInputs(
+ TempAllocator& alloc, MInstruction* ins);
+
+template <unsigned Op>
+bool CacheIdPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MDefinition* in = ins->getOperand(Op);
+ switch (in->type()) {
+ case MIRType::Int32:
+ case MIRType::String:
+ case MIRType::Symbol:
+ return true;
+ default:
+ return BoxPolicy<Op>::staticAdjustInputs(alloc, ins);
+ }
+}
+
+template bool CacheIdPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool CacheIdPolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+
+bool ToDoublePolicy::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MOZ_ASSERT(ins->isToDouble() || ins->isToFloat32());
+
+ MDefinition* in = ins->getOperand(0);
+ MToFPInstruction::ConversionKind conversion;
+ if (ins->isToDouble()) {
+ conversion = ins->toToDouble()->conversion();
+ } else {
+ conversion = ins->toToFloat32()->conversion();
+ }
+
+ switch (in->type()) {
+ case MIRType::Int32:
+ case MIRType::Float32:
+ case MIRType::Double:
+ case MIRType::Value:
+ // No need for boxing for these types.
+ return true;
+ case MIRType::Null:
+ // No need for boxing, when we will convert.
+ if (conversion == MToFPInstruction::NonStringPrimitives) {
+ return true;
+ }
+ break;
+ case MIRType::Undefined:
+ case MIRType::Boolean:
+ // No need for boxing, when we will convert.
+ if (conversion == MToFPInstruction::NonStringPrimitives) {
+ return true;
+ }
+ break;
+ case MIRType::Object:
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ // Objects might be effectful. Symbols and BigInts give TypeError.
+ break;
+ default:
+ break;
+ }
+
+ in = BoxAt(alloc, ins, in);
+ ins->replaceOperand(0, in);
+ return true;
+}
+
+bool ToInt32Policy::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MOZ_ASSERT(ins->isToNumberInt32() || ins->isTruncateToInt32() ||
+ ins->isToIntegerInt32());
+
+ IntConversionInputKind conversion = IntConversionInputKind::Any;
+ if (ins->isToNumberInt32()) {
+ conversion = ins->toToNumberInt32()->conversion();
+ }
+
+ MDefinition* in = ins->getOperand(0);
+ switch (in->type()) {
+ case MIRType::Int32:
+ case MIRType::Float32:
+ case MIRType::Double:
+ case MIRType::Value:
+ // No need for boxing for these types.
+ return true;
+ case MIRType::Undefined:
+ // No need for boxing when truncating.
+ // Also no need for boxing when performing ToInteger, because
+ // ToInteger(undefined) = ToInteger(NaN) = 0.
+ if (ins->isTruncateToInt32() || ins->isToIntegerInt32()) {
+ return true;
+ }
+ break;
+ case MIRType::Null:
+ // No need for boxing, when we will convert.
+ if (conversion == IntConversionInputKind::Any) {
+ return true;
+ }
+ break;
+ case MIRType::Boolean:
+ // No need for boxing, when we will convert.
+ if (conversion == IntConversionInputKind::Any) {
+ return true;
+ }
+ if (conversion == IntConversionInputKind::NumbersOrBoolsOnly) {
+ return true;
+ }
+ break;
+ case MIRType::Object:
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ // Objects might be effectful. Symbols and BigInts give TypeError.
+ break;
+ default:
+ break;
+ }
+
+ in = BoxAt(alloc, ins, in);
+ ins->replaceOperand(0, in);
+ return true;
+}
+
+bool ToBigIntPolicy::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MOZ_ASSERT(ins->isToBigInt());
+
+ MDefinition* in = ins->getOperand(0);
+ switch (in->type()) {
+ case MIRType::BigInt:
+ case MIRType::Value:
+ // No need for boxing for these types.
+ return true;
+ default:
+ // Any other types need to be boxed.
+ break;
+ }
+
+ in = BoxAt(alloc, ins, in);
+ ins->replaceOperand(0, in);
+ return true;
+}
+
+bool ToStringPolicy::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MOZ_ASSERT(ins->isToString());
+
+ MIRType type = ins->getOperand(0)->type();
+ if (type == MIRType::Object || type == MIRType::Symbol ||
+ type == MIRType::BigInt) {
+ ins->replaceOperand(0, BoxAt(alloc, ins, ins->getOperand(0)));
+ return true;
+ }
+
+ // TODO remove the following line once 966957 has landed
+ EnsureOperandNotFloat32(alloc, ins, 0);
+
+ return true;
+}
+
+bool ToInt64Policy::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MOZ_ASSERT(ins->isToInt64());
+
+ MDefinition* input = ins->getOperand(0);
+ MIRType type = input->type();
+
+ switch (type) {
+ case MIRType::BigInt: {
+ auto* replace = MTruncateBigIntToInt64::New(alloc, input);
+ ins->block()->insertBefore(ins, replace);
+ ins->replaceOperand(0, replace);
+ break;
+ }
+ // No need for boxing for these types, because they are handled specially
+ // when this instruction is lowered to LIR.
+ case MIRType::Boolean:
+ case MIRType::String:
+ case MIRType::Int64:
+ case MIRType::Value:
+ break;
+ default:
+ ins->replaceOperand(0, BoxAt(alloc, ins, ins->getOperand(0)));
+ break;
+ }
+
+ return true;
+}
+
+template <unsigned Op>
+bool ObjectPolicy<Op>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ MOZ_ASSERT(ins->getOperand(Op)->type() != MIRType::Slots);
+ MOZ_ASSERT(ins->getOperand(Op)->type() != MIRType::Elements);
+
+ return UnboxOperand(alloc, ins, Op, MIRType::Object);
+}
+
+template bool ObjectPolicy<0>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool ObjectPolicy<1>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool ObjectPolicy<2>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+template bool ObjectPolicy<3>::staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+
+bool CallPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) const {
+ MCall* call = ins->toCall();
+
+ MDefinition* func = call->getCallee();
+ if (func->type() != MIRType::Object) {
+ MInstruction* unbox =
+ MUnbox::New(alloc, func, MIRType::Object, MUnbox::Fallible);
+ unbox->setBailoutKind(BailoutKind::TypePolicy);
+ call->block()->insertBefore(call, unbox);
+ call->replaceCallee(unbox);
+
+ if (!unbox->typePolicy()->adjustInputs(alloc, unbox)) {
+ return false;
+ }
+ }
+
+ for (uint32_t i = 0; i < call->numStackArgs(); i++) {
+ if (!alloc.ensureBallast()) {
+ return false;
+ }
+ EnsureOperandNotFloat32(alloc, call, MCall::IndexOfStackArg(i));
+ }
+
+ return true;
+}
+
+bool MegamorphicSetElementPolicy::adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const {
+ // The first operand should be an object.
+ if (!SingleObjectPolicy::staticAdjustInputs(alloc, ins)) {
+ return false;
+ }
+
+ // Box the index and value operands.
+ for (size_t i = 1, e = ins->numOperands(); i < e; i++) {
+ MDefinition* in = ins->getOperand(i);
+ if (in->type() == MIRType::Value) {
+ continue;
+ }
+ ins->replaceOperand(i, BoxAt(alloc, ins, in));
+ }
+ return true;
+}
+
+bool StoreUnboxedScalarPolicy::adjustValueInput(TempAllocator& alloc,
+ MInstruction* ins,
+ Scalar::Type writeType,
+ MDefinition* value,
+ int valueOperand) {
+ if (Scalar::isBigIntType(writeType)) {
+ if (value->type() == MIRType::BigInt) {
+ return true;
+ }
+
+ auto* replace = MToBigInt::New(alloc, value);
+ ins->block()->insertBefore(ins, replace);
+ ins->replaceOperand(valueOperand, replace);
+
+ return replace->typePolicy()->adjustInputs(alloc, replace);
+ }
+
+ MDefinition* curValue = value;
+ // First, ensure the value is int32, boolean, double or Value.
+ // The conversion is based on TypedArrayObjectTemplate::setElementTail.
+ switch (value->type()) {
+ case MIRType::Int32:
+ case MIRType::Double:
+ case MIRType::Float32:
+ case MIRType::Boolean:
+ case MIRType::Value:
+ break;
+ case MIRType::Null:
+ value->setImplicitlyUsedUnchecked();
+ value = MConstant::New(alloc, Int32Value(0));
+ ins->block()->insertBefore(ins, value->toInstruction());
+ break;
+ case MIRType::Undefined:
+ value->setImplicitlyUsedUnchecked();
+ value = MConstant::New(alloc, JS::NaNValue());
+ ins->block()->insertBefore(ins, value->toInstruction());
+ break;
+ case MIRType::Object:
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ value = BoxAt(alloc, ins, value);
+ break;
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+
+ if (value != curValue) {
+ ins->replaceOperand(valueOperand, value);
+ curValue = value;
+ }
+
+ MOZ_ASSERT(
+ value->type() == MIRType::Int32 || value->type() == MIRType::Boolean ||
+ value->type() == MIRType::Double || value->type() == MIRType::Float32 ||
+ value->type() == MIRType::Value);
+
+ switch (writeType) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ if (value->type() != MIRType::Int32) {
+ value = MTruncateToInt32::New(alloc, value);
+ ins->block()->insertBefore(ins, value->toInstruction());
+ }
+ break;
+ case Scalar::Uint8Clamped:
+ // The transpiler should have inserted MClampToUint8.
+ MOZ_ASSERT(value->type() == MIRType::Int32);
+ break;
+ case Scalar::Float32:
+ if (value->type() != MIRType::Float32) {
+ value = MToFloat32::New(alloc, value);
+ ins->block()->insertBefore(ins, value->toInstruction());
+ }
+ break;
+ case Scalar::Float64:
+ if (value->type() != MIRType::Double) {
+ value = MToDouble::New(alloc, value);
+ ins->block()->insertBefore(ins, value->toInstruction());
+ }
+ break;
+ default:
+ MOZ_CRASH("Invalid array type");
+ }
+
+ if (value != curValue) {
+ ins->replaceOperand(valueOperand, value);
+ }
+
+ return true;
+}
+
+bool StoreUnboxedScalarPolicy::adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const {
+ MStoreUnboxedScalar* store = ins->toStoreUnboxedScalar();
+ MOZ_ASSERT(store->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(store->index()->type() == MIRType::IntPtr);
+
+ return adjustValueInput(alloc, store, store->writeType(), store->value(), 2);
+}
+
+bool StoreDataViewElementPolicy::adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const {
+ auto* store = ins->toStoreDataViewElement();
+ MOZ_ASSERT(store->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(store->index()->type() == MIRType::IntPtr);
+ MOZ_ASSERT(store->littleEndian()->type() == MIRType::Boolean);
+
+ return StoreUnboxedScalarPolicy::adjustValueInput(
+ alloc, ins, store->writeType(), store->value(), 2);
+}
+
+bool StoreTypedArrayHolePolicy::adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const {
+ MStoreTypedArrayElementHole* store = ins->toStoreTypedArrayElementHole();
+ MOZ_ASSERT(store->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(store->index()->type() == MIRType::IntPtr);
+ MOZ_ASSERT(store->length()->type() == MIRType::IntPtr);
+
+ return StoreUnboxedScalarPolicy::adjustValueInput(
+ alloc, ins, store->arrayType(), store->value(), 3);
+}
+
+bool ClampPolicy::adjustInputs(TempAllocator& alloc, MInstruction* ins) const {
+ MDefinition* in = ins->toClampToUint8()->input();
+
+ switch (in->type()) {
+ case MIRType::Int32:
+ case MIRType::Double:
+ case MIRType::Value:
+ break;
+ default:
+ ins->replaceOperand(0, BoxAt(alloc, ins, in));
+ break;
+ }
+
+ return true;
+}
+
+// Lists of all TypePolicy specializations which are used by MIR Instructions.
+#define TYPE_POLICY_LIST(_) \
+ _(AllDoublePolicy) \
+ _(ArithPolicy) \
+ _(BigIntArithPolicy) \
+ _(BitwisePolicy) \
+ _(BoxInputsPolicy) \
+ _(CallPolicy) \
+ _(MegamorphicSetElementPolicy) \
+ _(ClampPolicy) \
+ _(ComparePolicy) \
+ _(PowPolicy) \
+ _(SignPolicy) \
+ _(StoreDataViewElementPolicy) \
+ _(StoreTypedArrayHolePolicy) \
+ _(StoreUnboxedScalarPolicy) \
+ _(TestPolicy) \
+ _(ToDoublePolicy) \
+ _(ToInt32Policy) \
+ _(ToBigIntPolicy) \
+ _(ToStringPolicy) \
+ _(ToInt64Policy)
+
+#define TEMPLATE_TYPE_POLICY_LIST(_) \
+ _(BigIntPolicy<0>) \
+ _(BoxExceptPolicy<0, MIRType::Object>) \
+ _(BoxPolicy<0>) \
+ _(ConvertToInt32Policy<0>) \
+ _(ConvertToStringPolicy<0>) \
+ _(ConvertToStringPolicy<2>) \
+ _(DoublePolicy<0>) \
+ _(FloatingPointPolicy<0>) \
+ _(UnboxedInt32Policy<0>) \
+ _(UnboxedInt32Policy<1>) \
+ _(TruncateToInt32OrToBigIntPolicy<2>) \
+ _(MixPolicy<ObjectPolicy<0>, StringPolicy<1>, BoxPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, BoxPolicy<1>, BoxPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, BoxPolicy<1>, ObjectPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, BoxPolicy<1>, UnboxedInt32Policy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<1>, BoxPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<1>, UnboxedInt32Policy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, BoxPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, ObjectPolicy<1>, UnboxedInt32Policy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, ObjectPolicy<1>, ObjectPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, ObjectPolicy<1>, BoxPolicy<2>>) \
+ _(MixPolicy<StringPolicy<0>, UnboxedInt32Policy<1>, UnboxedInt32Policy<2>>) \
+ _(MixPolicy<StringPolicy<0>, ObjectPolicy<1>, StringPolicy<2>>) \
+ _(MixPolicy<StringPolicy<0>, StringPolicy<1>, StringPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, StringPolicy<1>, UnboxedInt32Policy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<1>, BoxPolicy<2>, \
+ ObjectPolicy<3>>) \
+ _(MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<1>, UnboxedInt32Policy<2>, \
+ UnboxedInt32Policy<3>>) \
+ _(MixPolicy<TruncateToInt32OrToBigIntPolicy<2>, \
+ TruncateToInt32OrToBigIntPolicy<3>>) \
+ _(MixPolicy<ObjectPolicy<0>, CacheIdPolicy<1>, NoFloatPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, BoxExceptPolicy<1, MIRType::Object>, \
+ CacheIdPolicy<2>>) \
+ _(MixPolicy<BoxPolicy<0>, ObjectPolicy<1>>) \
+ _(MixPolicy<ConvertToStringPolicy<0>, ConvertToStringPolicy<1>>) \
+ _(MixPolicy<ConvertToStringPolicy<0>, ObjectPolicy<1>>) \
+ _(MixPolicy<DoublePolicy<0>, DoublePolicy<1>>) \
+ _(MixPolicy<UnboxedInt32Policy<0>, UnboxedInt32Policy<1>>) \
+ _(MixPolicy<Int32OrIntPtrPolicy<0>, Int32OrIntPtrPolicy<1>>) \
+ _(MixPolicy<ObjectPolicy<0>, BoxPolicy<1>>) \
+ _(MixPolicy<BoxExceptPolicy<0, MIRType::Object>, CacheIdPolicy<1>>) \
+ _(MixPolicy<CacheIdPolicy<0>, ObjectPolicy<1>>) \
+ _(MixPolicy<ObjectPolicy<0>, ConvertToStringPolicy<1>>) \
+ _(MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<1>>) \
+ _(MixPolicy<ObjectPolicy<0>, UnboxedInt32Policy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, NoFloatPolicy<1>>) \
+ _(MixPolicy<ObjectPolicy<0>, NoFloatPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<0>, NoFloatPolicy<3>>) \
+ _(MixPolicy<ObjectPolicy<0>, ObjectPolicy<1>>) \
+ _(MixPolicy<ObjectPolicy<0>, StringPolicy<1>>) \
+ _(MixPolicy<ObjectPolicy<0>, ConvertToStringPolicy<2>>) \
+ _(MixPolicy<ObjectPolicy<1>, ConvertToStringPolicy<0>>) \
+ _(MixPolicy<StringPolicy<0>, UnboxedInt32Policy<1>>) \
+ _(MixPolicy<StringPolicy<0>, StringPolicy<1>>) \
+ _(MixPolicy<BoxPolicy<0>, BoxPolicy<1>>) \
+ _(MixPolicy<ObjectPolicy<0>, BoxPolicy<2>, ObjectPolicy<3>>) \
+ _(MixPolicy<BoxExceptPolicy<0, MIRType::Object>, ObjectPolicy<1>>) \
+ _(MixPolicy<UnboxedInt32Policy<0>, BigIntPolicy<1>>) \
+ _(MixPolicy<UnboxedInt32Policy<0>, NoFloatPolicyAfter<1>>) \
+ _(MixPolicy<UnboxedInt32Policy<0>, UnboxedInt32Policy<1>, \
+ NoFloatPolicyAfter<2>>) \
+ _(NoFloatPolicy<0>) \
+ _(NoFloatPolicy<1>) \
+ _(NoFloatPolicy<2>) \
+ _(NoFloatPolicyAfter<0>) \
+ _(NoFloatPolicyAfter<1>) \
+ _(NoFloatPolicyAfter<2>) \
+ _(ObjectPolicy<0>) \
+ _(ObjectPolicy<1>) \
+ _(ObjectPolicy<3>) \
+ _(StringPolicy<0>) \
+ _(SymbolPolicy<0>)
+
+namespace js {
+namespace jit {
+
+// Define for all used TypePolicy specialization, the definition for
+// |TypePolicy::Data::thisTypePolicy|. This function returns one constant
+// instance of the TypePolicy which is shared among all MIR Instructions of the
+// same type.
+//
+// This Macro use __VA_ARGS__ to account for commas of template parameters.
+#define DEFINE_TYPE_POLICY_SINGLETON_INSTANCES_(...) \
+ const TypePolicy* __VA_ARGS__::Data::thisTypePolicy() { \
+ static constexpr __VA_ARGS__ singletonType; \
+ return &singletonType; \
+ }
+
+TYPE_POLICY_LIST(DEFINE_TYPE_POLICY_SINGLETON_INSTANCES_)
+TEMPLATE_TYPE_POLICY_LIST(template <> DEFINE_TYPE_POLICY_SINGLETON_INSTANCES_)
+#undef DEFINE_TYPE_POLICY_SINGLETON_INSTANCES_
+
+} // namespace jit
+} // namespace js
+
+namespace {
+
+// For extra-good measure in case an unqualified use is ever introduced. (The
+// main use in the macro below is explicitly qualified so as not to consult
+// this scope and find this function.)
+inline TypePolicy* thisTypePolicy() = delete;
+
+static MIRType thisTypeSpecialization() {
+ MOZ_CRASH("TypeSpecialization lacks definition of thisTypeSpecialization.");
+}
+
+} // namespace
+
+// For each MIR Instruction, this macro define the |typePolicy| method which is
+// using the |thisTypePolicy| method. The |thisTypePolicy| method is either a
+// member of the MIR Instruction, such as with MGetElementCache, a member
+// inherited from the TypePolicy::Data structure, or a member inherited from
+// NoTypePolicy if the MIR instruction has no type policy.
+#define DEFINE_MIR_TYPEPOLICY_MEMBERS_(op) \
+ const TypePolicy* js::jit::M##op::typePolicy() { \
+ return M##op::thisTypePolicy(); \
+ } \
+ \
+ MIRType js::jit::M##op::typePolicySpecialization() { \
+ return thisTypeSpecialization(); \
+ }
+
+MIR_OPCODE_LIST(DEFINE_MIR_TYPEPOLICY_MEMBERS_)
+#undef DEFINE_MIR_TYPEPOLICY_MEMBERS_
diff --git a/js/src/jit/TypePolicy.h b/js/src/jit/TypePolicy.h
new file mode 100644
index 0000000000..5511fa53dd
--- /dev/null
+++ b/js/src/jit/TypePolicy.h
@@ -0,0 +1,543 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_TypePolicy_h
+#define jit_TypePolicy_h
+
+#include "jit/IonTypes.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+namespace js {
+namespace jit {
+
+class MInstruction;
+class MDefinition;
+class TempAllocator;
+
+extern MDefinition* AlwaysBoxAt(TempAllocator& alloc, MInstruction* at,
+ MDefinition* operand);
+
+// A type policy directs the type analysis phases, which insert conversion,
+// boxing, unboxing, and type changes as necessary.
+class TypePolicy {
+ public:
+ // Analyze the inputs of the instruction and perform one of the following
+ // actions for each input:
+ // * Nothing; the input already type-checks.
+ // * If untyped, optionally ask the input to try and specialize its value.
+ // * Replace the operand with a conversion instruction.
+ // * Insert an unconditional deoptimization (no conversion possible).
+ [[nodiscard]] virtual bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const = 0;
+};
+
+struct TypeSpecializationData {
+ protected:
+ // Specifies three levels of specialization:
+ // - < Value. This input is expected and required.
+ // - == None. This op should not be specialized.
+ MIRType specialization_;
+
+ MIRType thisTypeSpecialization() { return specialization_; }
+
+ public:
+ MIRType specialization() const { return specialization_; }
+};
+
+#define EMPTY_DATA_ \
+ struct Data { \
+ static const TypePolicy* thisTypePolicy(); \
+ }
+
+#define INHERIT_DATA_(DATA_TYPE) \
+ struct Data : public DATA_TYPE { \
+ static const TypePolicy* thisTypePolicy(); \
+ }
+
+#define SPECIALIZATION_DATA_ INHERIT_DATA_(TypeSpecializationData)
+
+class NoTypePolicy {
+ public:
+ struct Data {
+ static const TypePolicy* thisTypePolicy() { return nullptr; }
+ };
+};
+
+class BoxInputsPolicy final : public TypePolicy {
+ public:
+ constexpr BoxInputsPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+class ArithPolicy final : public TypePolicy {
+ public:
+ constexpr ArithPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override;
+};
+
+class BigIntArithPolicy final : public TypePolicy {
+ public:
+ constexpr BigIntArithPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override;
+};
+
+class AllDoublePolicy final : public TypePolicy {
+ public:
+ constexpr AllDoublePolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+class BitwisePolicy final : public TypePolicy {
+ public:
+ constexpr BitwisePolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override;
+};
+
+class ComparePolicy final : public TypePolicy {
+ public:
+ constexpr ComparePolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override;
+};
+
+// Policy for MTest instructions.
+class TestPolicy final : public TypePolicy {
+ public:
+ constexpr TestPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override;
+};
+
+class CallPolicy final : public TypePolicy {
+ public:
+ constexpr CallPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override;
+};
+
+// Policy for MPow:
+//
+// * If return type is MIRType::Double, we need (Double, Double) or
+// (Double, Int32) operands.
+// * If return type is MIRType::Int32, we need (Int32, Int32) operands.
+class PowPolicy final : public TypePolicy {
+ public:
+ constexpr PowPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override;
+};
+
+// Policy for MSign. Operand is either Double or Int32.
+class SignPolicy final : public TypePolicy {
+ public:
+ constexpr SignPolicy() = default;
+ SPECIALIZATION_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override;
+};
+
+// Expect a symbol for operand Op. If the input is a Value, it is unboxed.
+template <unsigned Op>
+class SymbolPolicy final : public TypePolicy {
+ public:
+ constexpr SymbolPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expect a string for operand Op. If the input is a Value, it is unboxed.
+template <unsigned Op>
+class StringPolicy final : public TypePolicy {
+ public:
+ constexpr StringPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expect a string for operand Op. Else a ToString instruction is inserted.
+template <unsigned Op>
+class ConvertToStringPolicy final : public TypePolicy {
+ public:
+ constexpr ConvertToStringPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expect a BigInt for operand Op. If the input is a Value, it is unboxed.
+template <unsigned Op>
+class BigIntPolicy final : public TypePolicy {
+ public:
+ constexpr BigIntPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expects either an Int32 or a boxed Int32 for operand Op; may unbox if needed.
+template <unsigned Op>
+class UnboxedInt32Policy final : private TypePolicy {
+ public:
+ constexpr UnboxedInt32Policy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expects either an Int32 or IntPtr for operand Op.
+template <unsigned Op>
+class Int32OrIntPtrPolicy final : private TypePolicy {
+ public:
+ constexpr Int32OrIntPtrPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expect an Int for operand Op. Else a ToInt32 instruction is inserted.
+template <unsigned Op>
+class ConvertToInt32Policy final : public TypePolicy {
+ public:
+ constexpr ConvertToInt32Policy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expect either an Int or BigInt for operand Op. Else a TruncateToInt32 or
+// ToBigInt instruction is inserted.
+template <unsigned Op>
+class TruncateToInt32OrToBigIntPolicy final : public TypePolicy {
+ public:
+ constexpr TruncateToInt32OrToBigIntPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expect a double for operand Op. If the input is a Value, it is unboxed.
+template <unsigned Op>
+class DoublePolicy final : public TypePolicy {
+ public:
+ constexpr DoublePolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expect a float32 for operand Op. If the input is a Value, it is unboxed.
+template <unsigned Op>
+class Float32Policy final : public TypePolicy {
+ public:
+ constexpr Float32Policy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Expect a float32 OR a double for operand Op, but will prioritize Float32
+// if the result type is set as such. If the input is a Value, it is unboxed.
+template <unsigned Op>
+class FloatingPointPolicy final : public TypePolicy {
+ public:
+ constexpr FloatingPointPolicy() = default;
+ SPECIALIZATION_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override;
+};
+
+template <unsigned Op>
+class NoFloatPolicy final : public TypePolicy {
+ public:
+ constexpr NoFloatPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Policy for guarding variadic instructions such as object / array state
+// instructions.
+template <unsigned FirstOp>
+class NoFloatPolicyAfter final : public TypePolicy {
+ public:
+ constexpr NoFloatPolicyAfter() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override {
+ return staticAdjustInputs(alloc, ins);
+ }
+};
+
+// Box objects or strings as an input to a ToDouble instruction.
+class ToDoublePolicy final : public TypePolicy {
+ public:
+ constexpr ToDoublePolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Box objects, strings and undefined as input to a ToInt32 instruction.
+class ToInt32Policy final : public TypePolicy {
+ public:
+ constexpr ToInt32Policy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Box any non-BigInts as input to a ToBigInt instruction.
+class ToBigIntPolicy final : public TypePolicy {
+ public:
+ constexpr ToBigIntPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Box objects as input to a ToString instruction.
+class ToStringPolicy final : public TypePolicy {
+ public:
+ constexpr ToStringPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* def);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override {
+ return staticAdjustInputs(alloc, def);
+ }
+};
+
+// Box non-Boolean, non-String, non-BigInt as input to a ToInt64 instruction.
+class ToInt64Policy final : public TypePolicy {
+ public:
+ constexpr ToInt64Policy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override {
+ return staticAdjustInputs(alloc, ins);
+ }
+};
+
+template <unsigned Op>
+class ObjectPolicy final : public TypePolicy {
+ public:
+ constexpr ObjectPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override {
+ return staticAdjustInputs(alloc, ins);
+ }
+};
+
+// Single-object input. If the input is a Value, it is unboxed. If it is
+// a primitive, we use ValueToNonNullObject.
+using SingleObjectPolicy = ObjectPolicy<0>;
+
+template <unsigned Op>
+class BoxPolicy final : public TypePolicy {
+ public:
+ constexpr BoxPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override {
+ return staticAdjustInputs(alloc, ins);
+ }
+};
+
+// Boxes everything except inputs of type Type.
+template <unsigned Op, MIRType Type>
+class BoxExceptPolicy final : public TypePolicy {
+ public:
+ constexpr BoxExceptPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override {
+ return staticAdjustInputs(alloc, ins);
+ }
+};
+
+// Box if not a typical property id (string, symbol, int32).
+template <unsigned Op>
+class CacheIdPolicy final : public TypePolicy {
+ public:
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins);
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override {
+ return staticAdjustInputs(alloc, ins);
+ }
+};
+
+// Combine multiple policies.
+template <class... Policies>
+class MixPolicy final : public TypePolicy {
+ public:
+ constexpr MixPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] static bool staticAdjustInputs(TempAllocator& alloc,
+ MInstruction* ins) {
+ return (Policies::staticAdjustInputs(alloc, ins) && ...);
+ }
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override {
+ return staticAdjustInputs(alloc, ins);
+ }
+};
+
+class MegamorphicSetElementPolicy final : public TypePolicy {
+ public:
+ constexpr MegamorphicSetElementPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* def) const override;
+};
+
+class StoreDataViewElementPolicy;
+class StoreTypedArrayHolePolicy;
+
+class StoreUnboxedScalarPolicy : public TypePolicy {
+ private:
+ constexpr StoreUnboxedScalarPolicy() = default;
+ [[nodiscard]] static bool adjustValueInput(TempAllocator& alloc,
+ MInstruction* ins,
+ Scalar::Type arrayType,
+ MDefinition* value,
+ int valueOperand);
+
+ friend class StoreDataViewElementPolicy;
+ friend class StoreTypedArrayHolePolicy;
+
+ public:
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override;
+};
+
+class StoreDataViewElementPolicy final : public StoreUnboxedScalarPolicy {
+ public:
+ constexpr StoreDataViewElementPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override;
+};
+
+class StoreTypedArrayHolePolicy final : public StoreUnboxedScalarPolicy {
+ public:
+ constexpr StoreTypedArrayHolePolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override;
+};
+
+// Accepts integers and doubles. Everything else is boxed.
+class ClampPolicy final : public TypePolicy {
+ public:
+ constexpr ClampPolicy() = default;
+ EMPTY_DATA_;
+ [[nodiscard]] bool adjustInputs(TempAllocator& alloc,
+ MInstruction* ins) const override;
+};
+
+#undef SPECIALIZATION_DATA_
+#undef INHERIT_DATA_
+#undef EMPTY_DATA_
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_TypePolicy_h */
diff --git a/js/src/jit/VMFunctionList-inl.h b/js/src/jit/VMFunctionList-inl.h
new file mode 100644
index 0000000000..b19e6d18e8
--- /dev/null
+++ b/js/src/jit/VMFunctionList-inl.h
@@ -0,0 +1,371 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_VMFunctionList_inl_h
+#define jit_VMFunctionList_inl_h
+
+#include "builtin/Eval.h"
+#include "builtin/ModuleObject.h" // js::GetOrCreateModuleMetaObject
+#include "builtin/Object.h" // js::ObjectCreateWithTemplate
+#include "builtin/Promise.h" // js::AsyncFunctionAwait
+#include "builtin/RegExp.h"
+#include "builtin/String.h"
+#include "builtin/TestingFunctions.h"
+#include "jit/BaselineIC.h"
+#include "jit/Ion.h"
+#include "jit/IonIC.h"
+#include "jit/TrialInlining.h"
+#include "jit/VMFunctions.h"
+#include "vm/AsyncFunction.h"
+#include "vm/AsyncIteration.h"
+#include "vm/BigIntType.h"
+#include "vm/EqualityOperations.h"
+#include "vm/Interpreter.h"
+#include "vm/Iteration.h"
+#include "vm/TypedArrayObject.h"
+
+#include "jit/BaselineFrame-inl.h"
+#include "vm/Interpreter-inl.h"
+
+namespace js {
+namespace jit {
+
+#ifdef FUZZING_JS_FUZZILLI
+# define VMFUNCTION_FUZZILLI_LIST(_) \
+ _(FuzzilliHashObject, js::FuzzilliHashObject) \
+ _(FuzzilliHashObjectInl, js::FuzzilliHashObjectInl)
+#else
+# define VMFUNCTION_FUZZILLI_LIST(_)
+#endif
+
+// List of all VM functions to be used with callVM. Each entry stores the name
+// (must be unique, used for the VMFunctionId enum and profiling) and the C++
+// function to be called. This list must be sorted on the name field.
+#define VMFUNCTION_LIST(_) \
+ _(AddOrUpdateSparseElementHelper, js::AddOrUpdateSparseElementHelper) \
+ _(AddSlotAndCallAddPropHook, js::AddSlotAndCallAddPropHook) \
+ _(ArgumentsObjectCreateForInlinedIon, \
+ js::ArgumentsObject::createForInlinedIon) \
+ _(ArgumentsObjectCreateForIon, js::ArgumentsObject::createForIon) \
+ _(ArgumentsSliceDense, js::ArgumentsSliceDense) \
+ _(ArrayConstructorOneArg, js::ArrayConstructorOneArg) \
+ _(ArrayFromArgumentsObject, js::ArrayFromArgumentsObject) \
+ _(ArrayJoin, js::jit::ArrayJoin) \
+ _(ArraySliceDense, js::ArraySliceDense) \
+ _(AsyncFunctionAwait, js::AsyncFunctionAwait) \
+ _(AsyncFunctionResolve, js::AsyncFunctionResolve) \
+ _(AtomicsAdd64, js::jit::AtomicsAdd64) \
+ _(AtomicsAnd64, js::jit::AtomicsAnd64) \
+ _(AtomicsCompareExchange64, js::jit::AtomicsCompareExchange64) \
+ _(AtomicsExchange64, js::jit::AtomicsExchange64) \
+ _(AtomicsLoad64, js::jit::AtomicsLoad64) \
+ _(AtomicsOr64, js::jit::AtomicsOr64) \
+ _(AtomicsSub64, js::jit::AtomicsSub64) \
+ _(AtomicsXor64, js::jit::AtomicsXor64) \
+ _(AtomizeString, js::AtomizeString) \
+ _(BaselineCompileFromBaselineInterpreter, \
+ js::jit::BaselineCompileFromBaselineInterpreter) \
+ _(BaselineDebugPrologue, js::jit::DebugPrologue) \
+ _(BaselineGetFunctionThis, js::jit::BaselineGetFunctionThis) \
+ _(BigIntAdd, JS::BigInt::add) \
+ _(BigIntAsIntN, js::jit::BigIntAsIntN) \
+ _(BigIntAsUintN, js::jit::BigIntAsUintN) \
+ _(BigIntBitAnd, JS::BigInt::bitAnd) \
+ _(BigIntBitNot, JS::BigInt::bitNot) \
+ _(BigIntBitOr, JS::BigInt::bitOr) \
+ _(BigIntBitXor, JS::BigInt::bitXor) \
+ _(BigIntDec, JS::BigInt::dec) \
+ _(BigIntDiv, JS::BigInt::div) \
+ _(BigIntInc, JS::BigInt::inc) \
+ _(BigIntLeftShift, JS::BigInt::lsh) \
+ _(BigIntMod, JS::BigInt::mod) \
+ _(BigIntMul, JS::BigInt::mul) \
+ _(BigIntNeg, JS::BigInt::neg) \
+ _(BigIntPow, JS::BigInt::pow) \
+ _(BigIntRightShift, JS::BigInt::rsh) \
+ _(BigIntStringEqual, \
+ js::jit::BigIntStringEqual<js::jit::EqualityKind::Equal>) \
+ _(BigIntStringGreaterThanOrEqual, \
+ js::jit::BigIntStringCompare<js::jit::ComparisonKind::GreaterThanOrEqual>) \
+ _(BigIntStringLessThan, \
+ js::jit::BigIntStringCompare<js::jit::ComparisonKind::LessThan>) \
+ _(BigIntStringNotEqual, \
+ js::jit::BigIntStringEqual<js::jit::EqualityKind::NotEqual>) \
+ _(BigIntSub, JS::BigInt::sub) \
+ _(BindVarOperation, js::BindVarOperation) \
+ _(BlockLexicalEnvironmentObjectCreateWithoutEnclosing, \
+ js::BlockLexicalEnvironmentObject::createWithoutEnclosing) \
+ _(BoxBoxableValue, js::wasm::BoxBoxableValue) \
+ _(BoxNonStrictThis, js::BoxNonStrictThis) \
+ _(BuiltinObjectOperation, js::BuiltinObjectOperation) \
+ _(CallDOMGetter, js::jit::CallDOMGetter) \
+ _(CallDOMSetter, js::jit::CallDOMSetter) \
+ _(CallNativeGetter, js::jit::CallNativeGetter) \
+ _(CallNativeSetter, js::jit::CallNativeSetter) \
+ _(CallObjectCreateWithShape, js::CallObject::createWithShape) \
+ _(CanSkipAwait, js::CanSkipAwait) \
+ _(CharCodeAt, js::jit::CharCodeAt) \
+ _(CheckClassHeritageOperation, js::CheckClassHeritageOperation) \
+ _(CheckOverRecursed, js::jit::CheckOverRecursed) \
+ _(CheckOverRecursedBaseline, js::jit::CheckOverRecursedBaseline) \
+ _(CheckPrivateFieldOperation, js::CheckPrivateFieldOperation) \
+ _(ClassBodyLexicalEnvironmentObjectCreateWithoutEnclosing, \
+ js::ClassBodyLexicalEnvironmentObject::createWithoutEnclosing) \
+ _(CloneRegExpObject, js::CloneRegExpObject) \
+ _(CloseIterOperation, js::CloseIterOperation) \
+ _(ConcatStrings, js::ConcatStrings<CanGC>) \
+ _(CreateAsyncFromSyncIterator, js::CreateAsyncFromSyncIterator) \
+ _(CreateBigIntFromInt64, js::jit::CreateBigIntFromInt64) \
+ _(CreateBigIntFromUint64, js::jit::CreateBigIntFromUint64) \
+ _(CreateGenerator, js::jit::CreateGenerator) \
+ _(CreateGeneratorFromFrame, js::jit::CreateGeneratorFromFrame) \
+ _(CreateThisFromIC, js::jit::CreateThisFromIC) \
+ _(CreateThisFromIon, js::jit::CreateThisFromIon) \
+ _(DebugAfterYield, js::jit::DebugAfterYield) \
+ _(DebugEpilogueOnBaselineReturn, js::jit::DebugEpilogueOnBaselineReturn) \
+ _(DebugLeaveLexicalEnv, js::jit::DebugLeaveLexicalEnv) \
+ _(DebugLeaveThenFreshenLexicalEnv, js::jit::DebugLeaveThenFreshenLexicalEnv) \
+ _(DebugLeaveThenPopLexicalEnv, js::jit::DebugLeaveThenPopLexicalEnv) \
+ _(DebugLeaveThenRecreateLexicalEnv, \
+ js::jit::DebugLeaveThenRecreateLexicalEnv) \
+ _(Debug_CheckSelfHosted, js::Debug_CheckSelfHosted) \
+ _(DelElemOperationNonStrict, js::DelElemOperation<false>) \
+ _(DelElemOperationStrict, js::DelElemOperation<true>) \
+ _(DelPropOperationNonStrict, js::DelPropOperation<false>) \
+ _(DelPropOperationStrict, js::DelPropOperation<true>) \
+ _(DeleteNameOperation, js::DeleteNameOperation) \
+ _(DoCallFallback, js::jit::DoCallFallback) \
+ _(DoConcatStringObject, js::jit::DoConcatStringObject) \
+ _(DoSpreadCallFallback, js::jit::DoSpreadCallFallback) \
+ _(DoStringToInt64, js::jit::DoStringToInt64) \
+ _(DoTrialInlining, js::jit::DoTrialInlining) \
+ _(EnterWith, js::jit::EnterWith) \
+ _(ExtractAwaitValue, js::ExtractAwaitValue) \
+ _(FinalSuspend, js::jit::FinalSuspend) \
+ _(FinishBoundFunctionInit, JSFunction::finishBoundFunctionInit) \
+ _(FreshenLexicalEnv, js::jit::FreshenLexicalEnv) \
+ _(FunWithProtoOperation, js::FunWithProtoOperation) \
+ VMFUNCTION_FUZZILLI_LIST(_) \
+ _(GeneratorThrowOrReturn, js::jit::GeneratorThrowOrReturn) \
+ _(GetAndClearException, js::GetAndClearException) \
+ _(GetFirstDollarIndexRaw, js::GetFirstDollarIndexRaw) \
+ _(GetImportOperation, js::GetImportOperation) \
+ _(GetIntrinsicValue, js::jit::GetIntrinsicValue) \
+ _(GetIterator, js::GetIterator) \
+ _(GetNonSyntacticGlobalThis, js::GetNonSyntacticGlobalThis) \
+ _(GetOrCreateModuleMetaObject, js::GetOrCreateModuleMetaObject) \
+ _(GetPrototypeOf, js::jit::GetPrototypeOf) \
+ _(GetSparseElementHelper, js::GetSparseElementHelper) \
+ _(GlobalDeclInstantiationFromIon, js::jit::GlobalDeclInstantiationFromIon) \
+ _(GlobalOrEvalDeclInstantiation, js::GlobalOrEvalDeclInstantiation) \
+ _(HandleDebugTrap, js::jit::HandleDebugTrap) \
+ _(ImplicitThisOperation, js::ImplicitThisOperation) \
+ _(ImportMetaOperation, js::ImportMetaOperation) \
+ _(InitElemGetterSetterOperation, js::InitElemGetterSetterOperation) \
+ _(InitFunctionEnvironmentObjects, js::jit::InitFunctionEnvironmentObjects) \
+ _(InitPropGetterSetterOperation, js::InitPropGetterSetterOperation) \
+ _(InitRestParameter, js::jit::InitRestParameter) \
+ _(Int32ToString, js::Int32ToString<CanGC>) \
+ _(Int32ToStringWithBase, js::Int32ToStringWithBase) \
+ _(InterpretResume, js::jit::InterpretResume) \
+ _(InterruptCheck, js::jit::InterruptCheck) \
+ _(InvokeFunction, js::jit::InvokeFunction) \
+ _(IonBinaryArithICUpdate, js::jit::IonBinaryArithIC::update) \
+ _(IonBindNameICUpdate, js::jit::IonBindNameIC::update) \
+ _(IonCheckPrivateFieldICUpdate, js::jit::IonCheckPrivateFieldIC::update) \
+ _(IonCloseIterICUpdate, js::jit::IonCloseIterIC::update) \
+ _(IonCompareICUpdate, js::jit::IonCompareIC::update) \
+ _(IonCompileScriptForBaselineAtEntry, \
+ js::jit::IonCompileScriptForBaselineAtEntry) \
+ _(IonCompileScriptForBaselineOSR, js::jit::IonCompileScriptForBaselineOSR) \
+ _(IonGetIteratorICUpdate, js::jit::IonGetIteratorIC::update) \
+ _(IonGetNameICUpdate, js::jit::IonGetNameIC::update) \
+ _(IonGetPropSuperICUpdate, js::jit::IonGetPropSuperIC::update) \
+ _(IonGetPropertyICUpdate, js::jit::IonGetPropertyIC::update) \
+ _(IonHasOwnICUpdate, js::jit::IonHasOwnIC::update) \
+ _(IonInICUpdate, js::jit::IonInIC::update) \
+ _(IonInstanceOfICUpdate, js::jit::IonInstanceOfIC::update) \
+ _(IonOptimizeSpreadCallICUpdate, js::jit::IonOptimizeSpreadCallIC::update) \
+ _(IonSetPropertyICUpdate, js::jit::IonSetPropertyIC::update) \
+ _(IonToPropertyKeyICUpdate, js::jit::IonToPropertyKeyIC::update) \
+ _(IonUnaryArithICUpdate, js::jit::IonUnaryArithIC::update) \
+ _(IsArrayFromJit, js::IsArrayFromJit) \
+ _(IsPossiblyWrappedTypedArray, js::jit::IsPossiblyWrappedTypedArray) \
+ _(IsPrototypeOf, js::IsPrototypeOf) \
+ _(Lambda, js::Lambda) \
+ _(LeaveWith, js::jit::LeaveWith) \
+ _(LinearizeForCharAccess, js::jit::LinearizeForCharAccess) \
+ _(LoadAliasedDebugVar, js::LoadAliasedDebugVar) \
+ _(MapObjectGet, js::jit::MapObjectGet) \
+ _(MapObjectHas, js::jit::MapObjectHas) \
+ _(MutatePrototype, js::jit::MutatePrototype) \
+ _(NamedLambdaObjectCreateWithoutEnclosing, \
+ js::NamedLambdaObject::createWithoutEnclosing) \
+ _(NativeGetElement, js::NativeGetElement) \
+ _(NewArgumentsObject, js::jit::NewArgumentsObject) \
+ _(NewArrayIterator, js::NewArrayIterator) \
+ _(NewArrayObjectBaselineFallback, js::NewArrayObjectBaselineFallback) \
+ _(NewArrayObjectEnsureDenseInitLength, \
+ js::jit::NewArrayObjectEnsureDenseInitLength) \
+ _(NewArrayObjectOptimzedFallback, js::NewArrayObjectOptimizedFallback) \
+ _(NewArrayOperation, js::NewArrayOperation) \
+ _(NewArrayWithShape, js::NewArrayWithShape) \
+ _(NewObjectOperation, js::NewObjectOperation) \
+ _(NewPlainObjectBaselineFallback, js::NewPlainObjectBaselineFallback) \
+ _(NewPlainObjectOptimizedFallback, js::NewPlainObjectOptimizedFallback) \
+ _(NewPrivateName, js::NewPrivateName) \
+ _(NewRegExpStringIterator, js::NewRegExpStringIterator) \
+ _(NewStringIterator, js::NewStringIterator) \
+ _(NewStringObject, js::jit::NewStringObject) \
+ _(NewTypedArrayWithTemplateAndArray, js::NewTypedArrayWithTemplateAndArray) \
+ _(NewTypedArrayWithTemplateAndBuffer, \
+ js::NewTypedArrayWithTemplateAndBuffer) \
+ _(NewTypedArrayWithTemplateAndLength, \
+ js::NewTypedArrayWithTemplateAndLength) \
+ _(NormalSuspend, js::jit::NormalSuspend) \
+ _(NumberParseInt, js::NumberParseInt) \
+ _(NumberToString, js::NumberToString<CanGC>) \
+ _(ObjectCreateWithTemplate, js::ObjectCreateWithTemplate) \
+ _(ObjectWithProtoOperation, js::ObjectWithProtoOperation) \
+ _(OnDebuggerStatement, js::jit::OnDebuggerStatement) \
+ _(ProxyGetProperty, js::ProxyGetProperty) \
+ _(ProxyGetPropertyByValue, js::ProxyGetPropertyByValue) \
+ _(ProxyHas, js::ProxyHas) \
+ _(ProxyHasOwn, js::ProxyHasOwn) \
+ _(ProxySetProperty, js::ProxySetProperty) \
+ _(ProxySetPropertyByValue, js::ProxySetPropertyByValue) \
+ _(PushClassBodyEnv, js::jit::PushClassBodyEnv) \
+ _(PushLexicalEnv, js::jit::PushLexicalEnv) \
+ _(PushVarEnv, js::jit::PushVarEnv) \
+ _(RecreateLexicalEnv, js::jit::RecreateLexicalEnv) \
+ _(RegExpMatcherRaw, js::RegExpMatcherRaw) \
+ _(RegExpSearcherRaw, js::RegExpSearcherRaw) \
+ _(RegExpTesterRaw, js::RegExpTesterRaw) \
+ _(SameValue, js::SameValue) \
+ _(SetArrayLength, js::jit::SetArrayLength) \
+ _(SetElementMegamorphic, js::jit::SetElementMegamorphic) \
+ _(SetElementSuper, js::SetElementSuper) \
+ _(SetFunctionName, js::SetFunctionName) \
+ _(SetIntrinsicOperation, js::SetIntrinsicOperation) \
+ _(SetObjectHas, js::jit::SetObjectHas) \
+ _(SetPropertySuper, js::SetPropertySuper) \
+ _(StartDynamicModuleImport, js::StartDynamicModuleImport) \
+ _(StringBigIntGreaterThanOrEqual, \
+ js::jit::StringBigIntCompare<js::jit::ComparisonKind::GreaterThanOrEqual>) \
+ _(StringBigIntLessThan, \
+ js::jit::StringBigIntCompare<js::jit::ComparisonKind::LessThan>) \
+ _(StringEndsWith, js::StringEndsWith) \
+ _(StringFlatReplaceString, js::StringFlatReplaceString) \
+ _(StringFromCharCode, js::jit::StringFromCharCode) \
+ _(StringFromCodePoint, js::jit::StringFromCodePoint) \
+ _(StringIndexOf, js::StringIndexOf) \
+ _(StringReplace, js::jit::StringReplace) \
+ _(StringSplitString, js::StringSplitString) \
+ _(StringStartsWith, js::StringStartsWith) \
+ _(StringToLowerCase, js::StringToLowerCase) \
+ _(StringToNumber, js::StringToNumber) \
+ _(StringToUpperCase, js::StringToUpperCase) \
+ _(StringsCompareGreaterThanOrEquals, \
+ js::jit::StringsCompare<ComparisonKind::GreaterThanOrEqual>) \
+ _(StringsCompareLessThan, js::jit::StringsCompare<ComparisonKind::LessThan>) \
+ _(StringsEqual, js::jit::StringsEqual<js::jit::EqualityKind::Equal>) \
+ _(StringsNotEqual, js::jit::StringsEqual<js::jit::EqualityKind::NotEqual>) \
+ _(SubstringKernel, js::SubstringKernel) \
+ _(ThrowBadDerivedReturnOrUninitializedThis, \
+ js::jit::ThrowBadDerivedReturnOrUninitializedThis) \
+ _(ThrowCheckIsObject, js::ThrowCheckIsObject) \
+ _(ThrowInitializedThis, js::ThrowInitializedThis) \
+ _(ThrowMsgOperation, js::ThrowMsgOperation) \
+ _(ThrowObjectCoercible, js::ThrowObjectCoercible) \
+ _(ThrowOperation, js::ThrowOperation) \
+ _(ThrowRuntimeLexicalError, js::jit::ThrowRuntimeLexicalError) \
+ _(ThrowUninitializedThis, js::ThrowUninitializedThis) \
+ _(ToBigInt, js::ToBigInt) \
+ _(ToStringSlow, js::ToStringSlow<CanGC>) \
+ _(ValueToIterator, js::ValueToIterator) \
+ _(VarEnvironmentObjectCreateWithoutEnclosing, \
+ js::VarEnvironmentObject::createWithoutEnclosing)
+
+// The list below is for tail calls. The third argument specifies the number of
+// non-argument Values the VM wrapper should pop from the stack. This is used
+// for Baseline ICs.
+//
+// This list is required to be alphabetized.
+#define TAIL_CALL_VMFUNCTION_LIST(_) \
+ _(DoBinaryArithFallback, js::jit::DoBinaryArithFallback, 2) \
+ _(DoBindNameFallback, js::jit::DoBindNameFallback, 0) \
+ _(DoCheckPrivateFieldFallback, js::jit::DoCheckPrivateFieldFallback, 2) \
+ _(DoCloseIterFallback, js::jit::DoCloseIterFallback, 0) \
+ _(DoCompareFallback, js::jit::DoCompareFallback, 2) \
+ _(DoConcatStringObject, js::jit::DoConcatStringObject, 2) \
+ _(DoGetElemFallback, js::jit::DoGetElemFallback, 2) \
+ _(DoGetElemSuperFallback, js::jit::DoGetElemSuperFallback, 3) \
+ _(DoGetIntrinsicFallback, js::jit::DoGetIntrinsicFallback, 0) \
+ _(DoGetIteratorFallback, js::jit::DoGetIteratorFallback, 1) \
+ _(DoGetNameFallback, js::jit::DoGetNameFallback, 0) \
+ _(DoGetPropFallback, js::jit::DoGetPropFallback, 1) \
+ _(DoGetPropSuperFallback, js::jit::DoGetPropSuperFallback, 0) \
+ _(DoHasOwnFallback, js::jit::DoHasOwnFallback, 2) \
+ _(DoInFallback, js::jit::DoInFallback, 2) \
+ _(DoInstanceOfFallback, js::jit::DoInstanceOfFallback, 2) \
+ _(DoNewArrayFallback, js::jit::DoNewArrayFallback, 0) \
+ _(DoNewObjectFallback, js::jit::DoNewObjectFallback, 0) \
+ _(DoOptimizeSpreadCallFallback, js::jit::DoOptimizeSpreadCallFallback, 0) \
+ _(DoRestFallback, js::jit::DoRestFallback, 0) \
+ _(DoSetElemFallback, js::jit::DoSetElemFallback, 2) \
+ _(DoSetPropFallback, js::jit::DoSetPropFallback, 1) \
+ _(DoToBoolFallback, js::jit::DoToBoolFallback, 0) \
+ _(DoToPropertyKeyFallback, js::jit::DoToPropertyKeyFallback, 0) \
+ _(DoTypeOfFallback, js::jit::DoTypeOfFallback, 0) \
+ _(DoUnaryArithFallback, js::jit::DoUnaryArithFallback, 1)
+
+#define DEF_ID(name, ...) name,
+enum class VMFunctionId { VMFUNCTION_LIST(DEF_ID) Count };
+enum class TailCallVMFunctionId { TAIL_CALL_VMFUNCTION_LIST(DEF_ID) Count };
+#undef DEF_ID
+
+// Define the VMFunctionToId template to map from signature + function to
+// the VMFunctionId. This lets us verify the consumer/codegen code matches
+// the C++ signature.
+template <typename Function, Function fun>
+struct VMFunctionToId; // Error here? Update VMFUNCTION_LIST?
+
+template <typename Function, Function fun>
+struct TailCallVMFunctionToId; // Error here? Update TAIL_CALL_VMFUNCTION_LIST?
+
+// GCC warns when the signature does not have matching attributes (for example
+// [[nodiscard]]). Squelch this warning to avoid a GCC-only footgun.
+#if MOZ_IS_GCC
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wignored-attributes"
+#endif
+
+// Note: the use of ::fp instead of fp is intentional to enforce use of
+// fully-qualified names in the list above.
+#define DEF_TEMPLATE(name, fp) \
+ template <> \
+ struct VMFunctionToId<decltype(&(::fp)), ::fp> { \
+ static constexpr VMFunctionId id = VMFunctionId::name; \
+ };
+VMFUNCTION_LIST(DEF_TEMPLATE)
+#undef DEF_TEMPLATE
+
+#define DEF_TEMPLATE(name, fp, valuesToPop) \
+ template <> \
+ struct TailCallVMFunctionToId<decltype(&(::fp)), ::fp> { \
+ static constexpr TailCallVMFunctionId id = TailCallVMFunctionId::name; \
+ };
+TAIL_CALL_VMFUNCTION_LIST(DEF_TEMPLATE)
+#undef DEF_TEMPLATE
+
+#if MOZ_IS_GCC
+# pragma GCC diagnostic pop
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_VMFunctionList_inl_h
diff --git a/js/src/jit/VMFunctions.cpp b/js/src/jit/VMFunctions.cpp
new file mode 100644
index 0000000000..e6d9f94a98
--- /dev/null
+++ b/js/src/jit/VMFunctions.cpp
@@ -0,0 +1,2834 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/VMFunctions.h"
+
+#include "mozilla/FloatingPoint.h"
+
+#include "builtin/MapObject.h"
+#include "builtin/String.h"
+#include "ds/OrderedHashTable.h"
+#include "gc/Cell.h"
+#include "gc/GC.h"
+#include "jit/arm/Simulator-arm.h"
+#include "jit/AtomicOperations.h"
+#include "jit/BaselineIC.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/mips32/Simulator-mips32.h"
+#include "jit/mips64/Simulator-mips64.h"
+#include "jit/Simulator.h"
+#include "js/experimental/JitInfo.h"
+#include "js/friend/ErrorMessages.h" // js::GetErrorMessage, JSMSG_*
+#include "js/friend/StackLimits.h" // js::AutoCheckRecursionLimit
+#include "js/friend/WindowProxy.h" // js::IsWindow
+#include "js/Printf.h"
+#include "js/TraceKind.h"
+#include "vm/ArrayObject.h"
+#include "vm/Compartment.h"
+#include "vm/Interpreter.h"
+#include "vm/JSAtom.h"
+#include "vm/PlainObject.h" // js::PlainObject
+#include "vm/SelfHosting.h"
+#include "vm/StaticStrings.h"
+#include "vm/TypedArrayObject.h"
+#include "wasm/WasmGcObject.h"
+
+#include "debugger/DebugAPI-inl.h"
+#include "jit/BaselineFrame-inl.h"
+#include "jit/VMFunctionList-inl.h"
+#include "vm/Interpreter-inl.h"
+#include "vm/JSScript-inl.h"
+#include "vm/NativeObject-inl.h"
+#include "vm/PlainObject-inl.h" // js::CreateThis
+#include "vm/StringObject-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+namespace js {
+
+class ArgumentsObject;
+class NamedLambdaObject;
+class AsyncFunctionGeneratorObject;
+class RegExpObject;
+
+namespace jit {
+
+struct IonOsrTempData;
+
+struct PopValues {
+ uint8_t numValues;
+
+ explicit constexpr PopValues(uint8_t numValues) : numValues(numValues) {}
+};
+
+template <class>
+struct ReturnTypeToDataType { /* Unexpected return type for a VMFunction. */
+};
+template <>
+struct ReturnTypeToDataType<void> {
+ static const DataType result = Type_Void;
+};
+template <>
+struct ReturnTypeToDataType<bool> {
+ static const DataType result = Type_Bool;
+};
+template <class T>
+struct ReturnTypeToDataType<T*> {
+ // Assume by default that any pointer return types are cells.
+ static_assert(std::is_base_of_v<gc::Cell, T>);
+
+ static const DataType result = Type_Cell;
+};
+
+// Convert argument types to properties of the argument known by the jit.
+template <class T>
+struct TypeToArgProperties {
+ static const uint32_t result =
+ (sizeof(T) <= sizeof(void*) ? VMFunctionData::Word
+ : VMFunctionData::Double);
+};
+template <>
+struct TypeToArgProperties<const Value&> {
+ static const uint32_t result =
+ TypeToArgProperties<Value>::result | VMFunctionData::ByRef;
+};
+template <>
+struct TypeToArgProperties<HandleValue> {
+ static const uint32_t result =
+ TypeToArgProperties<Value>::result | VMFunctionData::ByRef;
+};
+template <>
+struct TypeToArgProperties<MutableHandleValue> {
+ static const uint32_t result =
+ TypeToArgProperties<Value>::result | VMFunctionData::ByRef;
+};
+template <>
+struct TypeToArgProperties<HandleId> {
+ static const uint32_t result =
+ TypeToArgProperties<jsid>::result | VMFunctionData::ByRef;
+};
+template <class T>
+struct TypeToArgProperties<Handle<T*>> {
+ // Assume by default that any pointer handle types are cells.
+ static_assert(std::is_base_of_v<gc::Cell, T>);
+
+ static const uint32_t result =
+ TypeToArgProperties<T*>::result | VMFunctionData::ByRef;
+};
+template <class T>
+struct TypeToArgProperties<Handle<T>> {
+ // Fail for Handle types that aren't specialized above.
+};
+
+// Convert argument type to whether or not it should be passed in a float
+// register on platforms that have them, like x64.
+template <class T>
+struct TypeToPassInFloatReg {
+ static const uint32_t result = 0;
+};
+template <>
+struct TypeToPassInFloatReg<double> {
+ static const uint32_t result = 1;
+};
+
+// Convert argument types to root types used by the gc, see TraceJitExitFrame.
+template <class T>
+struct TypeToRootType {
+ static const uint32_t result = VMFunctionData::RootNone;
+};
+template <>
+struct TypeToRootType<HandleValue> {
+ static const uint32_t result = VMFunctionData::RootValue;
+};
+template <>
+struct TypeToRootType<MutableHandleValue> {
+ static const uint32_t result = VMFunctionData::RootValue;
+};
+template <>
+struct TypeToRootType<HandleId> {
+ static const uint32_t result = VMFunctionData::RootId;
+};
+template <class T>
+struct TypeToRootType<Handle<T*>> {
+ // Assume by default that any pointer types are cells.
+ static_assert(std::is_base_of_v<gc::Cell, T>);
+
+ static constexpr uint32_t rootType() {
+ using JS::TraceKind;
+
+ switch (JS::MapTypeToTraceKind<T>::kind) {
+ case TraceKind::Object:
+ return VMFunctionData::RootObject;
+ case TraceKind::BigInt:
+ return VMFunctionData::RootBigInt;
+ case TraceKind::String:
+ return VMFunctionData::RootString;
+ case TraceKind::Shape:
+ case TraceKind::Script:
+ case TraceKind::Scope:
+ return VMFunctionData::RootCell;
+ case TraceKind::Symbol:
+ case TraceKind::BaseShape:
+ case TraceKind::Null:
+ case TraceKind::JitCode:
+ case TraceKind::RegExpShared:
+ case TraceKind::GetterSetter:
+ case TraceKind::PropMap:
+ MOZ_CRASH("Unexpected trace kind");
+ }
+ }
+
+ static constexpr uint32_t result = rootType();
+};
+template <class T>
+struct TypeToRootType<Handle<T>> {
+ // Fail for Handle types that aren't specialized above.
+};
+
+template <class>
+struct OutParamToDataType {
+ static const DataType result = Type_Void;
+};
+template <class T>
+struct OutParamToDataType<const T*> {
+ // Const pointers can't be output parameters.
+ static const DataType result = Type_Void;
+};
+template <>
+struct OutParamToDataType<uint64_t*> {
+ // Already used as an input type, so it can't be used as an output param.
+ static const DataType result = Type_Void;
+};
+template <>
+struct OutParamToDataType<JSObject*> {
+ // Already used as an input type, so it can't be used as an output param.
+ static const DataType result = Type_Void;
+};
+template <>
+struct OutParamToDataType<JSString*> {
+ // Already used as an input type, so it can't be used as an output param.
+ static const DataType result = Type_Void;
+};
+template <>
+struct OutParamToDataType<BaselineFrame*> {
+ // Already used as an input type, so it can't be used as an output param.
+ static const DataType result = Type_Void;
+};
+template <>
+struct OutParamToDataType<gc::AllocSite*> {
+ // Already used as an input type, so it can't be used as an output param.
+ static const DataType result = Type_Void;
+};
+template <>
+struct OutParamToDataType<Value*> {
+ static const DataType result = Type_Value;
+};
+template <>
+struct OutParamToDataType<int*> {
+ static const DataType result = Type_Int32;
+};
+template <>
+struct OutParamToDataType<uint32_t*> {
+ static const DataType result = Type_Int32;
+};
+template <>
+struct OutParamToDataType<bool*> {
+ static const DataType result = Type_Bool;
+};
+template <>
+struct OutParamToDataType<double*> {
+ static const DataType result = Type_Double;
+};
+template <class T>
+struct OutParamToDataType<T*> {
+ // Fail for pointer types that aren't specialized above.
+};
+template <class T>
+struct OutParamToDataType<T**> {
+ static const DataType result = Type_Pointer;
+};
+template <class T>
+struct OutParamToDataType<MutableHandle<T>> {
+ static const DataType result = Type_Handle;
+};
+
+template <class>
+struct OutParamToRootType {
+ static const VMFunctionData::RootType result = VMFunctionData::RootNone;
+};
+template <>
+struct OutParamToRootType<MutableHandleValue> {
+ static const VMFunctionData::RootType result = VMFunctionData::RootValue;
+};
+template <>
+struct OutParamToRootType<MutableHandleObject> {
+ static const VMFunctionData::RootType result = VMFunctionData::RootObject;
+};
+template <>
+struct OutParamToRootType<MutableHandleString> {
+ static const VMFunctionData::RootType result = VMFunctionData::RootString;
+};
+template <>
+struct OutParamToRootType<MutableHandleBigInt> {
+ static const VMFunctionData::RootType result = VMFunctionData::RootBigInt;
+};
+
+// Construct a bit mask from a list of types. The mask is constructed as an OR
+// of the mask produced for each argument. The result of each argument is
+// shifted by its index, such that the result of the first argument is on the
+// low bits of the mask, and the result of the last argument in part of the
+// high bits of the mask.
+template <template <typename> class Each, typename ResultType, size_t Shift,
+ typename... Args>
+struct BitMask;
+
+template <template <typename> class Each, typename ResultType, size_t Shift>
+struct BitMask<Each, ResultType, Shift> {
+ static constexpr ResultType result = ResultType();
+};
+
+template <template <typename> class Each, typename ResultType, size_t Shift,
+ typename HeadType, typename... TailTypes>
+struct BitMask<Each, ResultType, Shift, HeadType, TailTypes...> {
+ static_assert(ResultType(Each<HeadType>::result) < (1 << Shift),
+ "not enough bits reserved by the shift for individual results");
+ static_assert(sizeof...(TailTypes) < (8 * sizeof(ResultType) / Shift),
+ "not enough bits in the result type to store all bit masks");
+
+ static constexpr ResultType result =
+ ResultType(Each<HeadType>::result) |
+ (BitMask<Each, ResultType, Shift, TailTypes...>::result << Shift);
+};
+
+// Helper template to build the VMFunctionData for a function.
+template <typename... Args>
+struct VMFunctionDataHelper;
+
+template <class R, typename... Args>
+struct VMFunctionDataHelper<R (*)(JSContext*, Args...)>
+ : public VMFunctionData {
+ using Fun = R (*)(JSContext*, Args...);
+
+ static constexpr DataType returnType() {
+ return ReturnTypeToDataType<R>::result;
+ }
+ static constexpr DataType outParam() {
+ return OutParamToDataType<typename LastArg<Args...>::Type>::result;
+ }
+ static constexpr RootType outParamRootType() {
+ return OutParamToRootType<typename LastArg<Args...>::Type>::result;
+ }
+ static constexpr size_t NbArgs() { return sizeof...(Args); }
+ static constexpr size_t explicitArgs() {
+ return NbArgs() - (outParam() != Type_Void ? 1 : 0);
+ }
+ static constexpr uint32_t argumentProperties() {
+ return BitMask<TypeToArgProperties, uint32_t, 2, Args...>::result;
+ }
+ static constexpr uint32_t argumentPassedInFloatRegs() {
+ return BitMask<TypeToPassInFloatReg, uint32_t, 2, Args...>::result;
+ }
+ static constexpr uint64_t argumentRootTypes() {
+ return BitMask<TypeToRootType, uint64_t, 3, Args...>::result;
+ }
+ constexpr explicit VMFunctionDataHelper(const char* name)
+ : VMFunctionData(name, explicitArgs(), argumentProperties(),
+ argumentPassedInFloatRegs(), argumentRootTypes(),
+ outParam(), outParamRootType(), returnType(),
+ /* extraValuesToPop = */ 0, NonTailCall) {}
+ constexpr explicit VMFunctionDataHelper(const char* name,
+ MaybeTailCall expectTailCall,
+ PopValues extraValuesToPop)
+ : VMFunctionData(name, explicitArgs(), argumentProperties(),
+ argumentPassedInFloatRegs(), argumentRootTypes(),
+ outParam(), outParamRootType(), returnType(),
+ extraValuesToPop.numValues, expectTailCall) {}
+};
+
+// GCC warns when the signature does not have matching attributes (for example
+// [[nodiscard]]). Squelch this warning to avoid a GCC-only footgun.
+#if MOZ_IS_GCC
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wignored-attributes"
+#endif
+
+// Generate VMFunctionData array.
+static constexpr VMFunctionData vmFunctions[] = {
+#define DEF_VMFUNCTION(name, fp) VMFunctionDataHelper<decltype(&(::fp))>(#name),
+ VMFUNCTION_LIST(DEF_VMFUNCTION)
+#undef DEF_VMFUNCTION
+};
+static constexpr VMFunctionData tailCallVMFunctions[] = {
+#define DEF_VMFUNCTION(name, fp, valuesToPop) \
+ VMFunctionDataHelper<decltype(&(::fp))>(#name, TailCall, \
+ PopValues(valuesToPop)),
+ TAIL_CALL_VMFUNCTION_LIST(DEF_VMFUNCTION)
+#undef DEF_VMFUNCTION
+};
+
+#if MOZ_IS_GCC
+# pragma GCC diagnostic pop
+#endif
+
+// Generate arrays storing C++ function pointers. These pointers are not stored
+// in VMFunctionData because there's no good way to cast them to void* in
+// constexpr code. Compilers are smart enough to treat the const array below as
+// constexpr.
+#define DEF_VMFUNCTION(name, fp, ...) (void*)(::fp),
+static void* const vmFunctionTargets[] = {VMFUNCTION_LIST(DEF_VMFUNCTION)};
+static void* const tailCallVMFunctionTargets[] = {
+ TAIL_CALL_VMFUNCTION_LIST(DEF_VMFUNCTION)};
+#undef DEF_VMFUNCTION
+
+const VMFunctionData& GetVMFunction(VMFunctionId id) {
+ return vmFunctions[size_t(id)];
+}
+const VMFunctionData& GetVMFunction(TailCallVMFunctionId id) {
+ return tailCallVMFunctions[size_t(id)];
+}
+
+static DynFn GetVMFunctionTarget(VMFunctionId id) {
+ return DynFn{vmFunctionTargets[size_t(id)]};
+}
+
+static DynFn GetVMFunctionTarget(TailCallVMFunctionId id) {
+ return DynFn{tailCallVMFunctionTargets[size_t(id)]};
+}
+
+template <typename IdT>
+bool JitRuntime::generateVMWrappers(JSContext* cx, MacroAssembler& masm,
+ VMWrapperOffsets& offsets) {
+ // Generate all VM function wrappers.
+
+ static constexpr size_t NumVMFunctions = size_t(IdT::Count);
+
+ if (!offsets.reserve(NumVMFunctions)) {
+ return false;
+ }
+
+#ifdef DEBUG
+ const char* lastName = nullptr;
+#endif
+
+ for (size_t i = 0; i < NumVMFunctions; i++) {
+ IdT id = IdT(i);
+ const VMFunctionData& fun = GetVMFunction(id);
+
+#ifdef DEBUG
+ // Assert the list is sorted by name.
+ if (lastName) {
+ MOZ_ASSERT(strcmp(lastName, fun.name()) < 0,
+ "VM function list must be sorted by name");
+ }
+ lastName = fun.name();
+#endif
+
+ JitSpew(JitSpew_Codegen, "# VM function wrapper (%s)", fun.name());
+
+ uint32_t offset;
+ if (!generateVMWrapper(cx, masm, fun, GetVMFunctionTarget(id), &offset)) {
+ return false;
+ }
+
+ MOZ_ASSERT(offsets.length() == size_t(id));
+ offsets.infallibleAppend(offset);
+ }
+
+ return true;
+};
+
+bool JitRuntime::generateVMWrappers(JSContext* cx, MacroAssembler& masm) {
+ if (!generateVMWrappers<VMFunctionId>(cx, masm, functionWrapperOffsets_)) {
+ return false;
+ }
+
+ if (!generateVMWrappers<TailCallVMFunctionId>(
+ cx, masm, tailCallFunctionWrapperOffsets_)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool InvokeFunction(JSContext* cx, HandleObject obj, bool constructing,
+ bool ignoresReturnValue, uint32_t argc, Value* argv,
+ MutableHandleValue rval) {
+ RootedExternalValueArray argvRoot(cx, argc + 1 + constructing, argv);
+
+ // Data in the argument vector is arranged for a JIT -> JIT call.
+ RootedValue thisv(cx, argv[0]);
+ Value* argvWithoutThis = argv + 1;
+
+ RootedValue fval(cx, ObjectValue(*obj));
+ if (constructing) {
+ if (!IsConstructor(fval)) {
+ ReportValueError(cx, JSMSG_NOT_CONSTRUCTOR, JSDVG_IGNORE_STACK, fval,
+ nullptr);
+ return false;
+ }
+
+ ConstructArgs cargs(cx);
+ if (!cargs.init(cx, argc)) {
+ return false;
+ }
+
+ for (uint32_t i = 0; i < argc; i++) {
+ cargs[i].set(argvWithoutThis[i]);
+ }
+
+ RootedValue newTarget(cx, argvWithoutThis[argc]);
+
+ // See CreateThisFromIon for why this can be NullValue.
+ if (thisv.isNull()) {
+ thisv.setMagic(JS_IS_CONSTRUCTING);
+ }
+
+ // If |this| hasn't been created, or is JS_UNINITIALIZED_LEXICAL,
+ // we can use normal construction code without creating an extraneous
+ // object.
+ if (thisv.isMagic()) {
+ MOZ_ASSERT(thisv.whyMagic() == JS_IS_CONSTRUCTING ||
+ thisv.whyMagic() == JS_UNINITIALIZED_LEXICAL);
+
+ RootedObject obj(cx);
+ if (!Construct(cx, fval, cargs, newTarget, &obj)) {
+ return false;
+ }
+
+ rval.setObject(*obj);
+ return true;
+ }
+
+ // Otherwise the default |this| has already been created. We could
+ // almost perform a *call* at this point, but we'd break |new.target|
+ // in the function. So in this one weird case we call a one-off
+ // construction path that *won't* set |this| to JS_IS_CONSTRUCTING.
+ return InternalConstructWithProvidedThis(cx, fval, thisv, cargs, newTarget,
+ rval);
+ }
+
+ InvokeArgsMaybeIgnoresReturnValue args(cx);
+ if (!args.init(cx, argc, ignoresReturnValue)) {
+ return false;
+ }
+
+ for (size_t i = 0; i < argc; i++) {
+ args[i].set(argvWithoutThis[i]);
+ }
+
+ return Call(cx, fval, thisv, args, rval);
+}
+
+void* GetContextSensitiveInterpreterStub() {
+ return TlsContext.get()->runtime()->jitRuntime()->interpreterStub().value;
+}
+
+bool InvokeFromInterpreterStub(JSContext* cx,
+ InterpreterStubExitFrameLayout* frame) {
+ JitFrameLayout* jsFrame = frame->jsFrame();
+ CalleeToken token = jsFrame->calleeToken();
+
+ Value* argv = jsFrame->thisAndActualArgs();
+ uint32_t numActualArgs = jsFrame->numActualArgs();
+ bool constructing = CalleeTokenIsConstructing(token);
+ RootedFunction fun(cx, CalleeTokenToFunction(token));
+
+ // Ensure new.target immediately follows the actual arguments (the arguments
+ // rectifier added padding).
+ if (constructing && numActualArgs < fun->nargs()) {
+ argv[1 + numActualArgs] = argv[1 + fun->nargs()];
+ }
+
+ RootedValue rval(cx);
+ if (!InvokeFunction(cx, fun, constructing,
+ /* ignoresReturnValue = */ false, numActualArgs, argv,
+ &rval)) {
+ return false;
+ }
+
+ // Overwrite |this| with the return value.
+ argv[0] = rval;
+ return true;
+}
+
+static bool CheckOverRecursedImpl(JSContext* cx, size_t extra) {
+ // We just failed the jitStackLimit check. There are two possible reasons:
+ // 1) jitStackLimit was the real stack limit and we're over-recursed
+ // 2) jitStackLimit was set to JS::NativeStackLimitMin by
+ // JSContext::requestInterrupt and we need to call
+ // JSContext::handleInterrupt.
+
+ // This handles 1).
+#ifdef JS_SIMULATOR
+ if (cx->simulator()->overRecursedWithExtra(extra)) {
+ ReportOverRecursed(cx);
+ return false;
+ }
+#else
+ AutoCheckRecursionLimit recursion(cx);
+ if (!recursion.checkWithExtra(cx, extra)) {
+ return false;
+ }
+#endif
+
+ // This handles 2).
+ gc::MaybeVerifyBarriers(cx);
+ return cx->handleInterrupt();
+}
+
+bool CheckOverRecursed(JSContext* cx) { return CheckOverRecursedImpl(cx, 0); }
+
+bool CheckOverRecursedBaseline(JSContext* cx, BaselineFrame* frame) {
+ // The stack check in Baseline happens before pushing locals so we have to
+ // account for that by including script->nslots() in the C++ recursion check.
+ size_t extra = frame->script()->nslots() * sizeof(Value);
+ return CheckOverRecursedImpl(cx, extra);
+}
+
+bool MutatePrototype(JSContext* cx, Handle<PlainObject*> obj,
+ HandleValue value) {
+ if (!value.isObjectOrNull()) {
+ return true;
+ }
+
+ RootedObject newProto(cx, value.toObjectOrNull());
+ return SetPrototype(cx, obj, newProto);
+}
+
+template <EqualityKind Kind>
+bool StringsEqual(JSContext* cx, HandleString lhs, HandleString rhs,
+ bool* res) {
+ if (!js::EqualStrings(cx, lhs, rhs, res)) {
+ return false;
+ }
+ if (Kind != EqualityKind::Equal) {
+ *res = !*res;
+ }
+ return true;
+}
+
+template bool StringsEqual<EqualityKind::Equal>(JSContext* cx, HandleString lhs,
+ HandleString rhs, bool* res);
+template bool StringsEqual<EqualityKind::NotEqual>(JSContext* cx,
+ HandleString lhs,
+ HandleString rhs, bool* res);
+
+template <ComparisonKind Kind>
+bool StringsCompare(JSContext* cx, HandleString lhs, HandleString rhs,
+ bool* res) {
+ int32_t result;
+ if (!js::CompareStrings(cx, lhs, rhs, &result)) {
+ return false;
+ }
+ if (Kind == ComparisonKind::LessThan) {
+ *res = result < 0;
+ } else {
+ *res = result >= 0;
+ }
+ return true;
+}
+
+template bool StringsCompare<ComparisonKind::LessThan>(JSContext* cx,
+ HandleString lhs,
+ HandleString rhs,
+ bool* res);
+template bool StringsCompare<ComparisonKind::GreaterThanOrEqual>(
+ JSContext* cx, HandleString lhs, HandleString rhs, bool* res);
+
+bool ArrayPushDensePure(JSContext* cx, ArrayObject* arr, Value* v) {
+ AutoUnsafeCallWithABI unsafe;
+
+ // Shape guards guarantee that the input is an extensible ArrayObject, which
+ // has a writable "length" property and has no other indexed properties.
+ MOZ_ASSERT(arr->isExtensible());
+ MOZ_ASSERT(arr->lengthIsWritable());
+ MOZ_ASSERT(!arr->isIndexed());
+
+ // Length must fit in an int32 because we guard against overflow before
+ // calling this VM function.
+ uint32_t index = arr->length();
+ MOZ_ASSERT(index < uint32_t(INT32_MAX));
+
+ DenseElementResult result = arr->setOrExtendDenseElements(cx, index, v, 1);
+ if (result == DenseElementResult::Failure) {
+ cx->recoverFromOutOfMemory();
+ }
+ return result == DenseElementResult::Success;
+}
+
+JSString* ArrayJoin(JSContext* cx, HandleObject array, HandleString sep) {
+ JS::RootedValueArray<3> argv(cx);
+ argv[0].setUndefined();
+ argv[1].setObject(*array);
+ argv[2].setString(sep);
+ if (!js::array_join(cx, 1, argv.begin())) {
+ return nullptr;
+ }
+ return argv[0].toString();
+}
+
+bool SetArrayLength(JSContext* cx, HandleObject obj, HandleValue value,
+ bool strict) {
+ Handle<ArrayObject*> array = obj.as<ArrayObject>();
+
+ RootedId id(cx, NameToId(cx->names().length));
+ ObjectOpResult result;
+
+ // SetArrayLength is called by IC stubs for SetProp and SetElem on arrays'
+ // "length" property.
+ //
+ // ArraySetLength below coerces |value| before checking for length being
+ // writable, and in the case of illegal values, will throw RangeError even
+ // when "length" is not writable. This is incorrect observable behavior,
+ // as a regular [[Set]] operation will check for "length" being
+ // writable before attempting any assignment.
+ //
+ // So, perform ArraySetLength if and only if "length" is writable.
+ if (array->lengthIsWritable()) {
+ Rooted<PropertyDescriptor> desc(
+ cx, PropertyDescriptor::Data(value, JS::PropertyAttribute::Writable));
+ if (!ArraySetLength(cx, array, id, desc, result)) {
+ return false;
+ }
+ } else {
+ MOZ_ALWAYS_TRUE(result.fail(JSMSG_READ_ONLY));
+ }
+
+ return result.checkStrictModeError(cx, obj, id, strict);
+}
+
+bool CharCodeAt(JSContext* cx, HandleString str, int32_t index,
+ uint32_t* code) {
+ char16_t c;
+ if (!str->getChar(cx, index, &c)) {
+ return false;
+ }
+ *code = c;
+ return true;
+}
+
+JSLinearString* StringFromCharCode(JSContext* cx, int32_t code) {
+ char16_t c = char16_t(code);
+
+ if (StaticStrings::hasUnit(c)) {
+ return cx->staticStrings().getUnit(c);
+ }
+
+ return NewStringCopyNDontDeflate<CanGC>(cx, &c, 1);
+}
+
+JSLinearString* StringFromCharCodeNoGC(JSContext* cx, int32_t code) {
+ AutoUnsafeCallWithABI unsafe;
+
+ char16_t c = char16_t(code);
+
+ if (StaticStrings::hasUnit(c)) {
+ return cx->staticStrings().getUnit(c);
+ }
+
+ return NewStringCopyNDontDeflate<NoGC>(cx, &c, 1);
+}
+
+JSString* StringFromCodePoint(JSContext* cx, int32_t codePoint) {
+ RootedValue rval(cx, Int32Value(codePoint));
+ if (!str_fromCodePoint_one_arg(cx, rval, &rval)) {
+ return nullptr;
+ }
+
+ return rval.toString();
+}
+
+JSLinearString* LinearizeForCharAccessPure(JSString* str) {
+ AutoUnsafeCallWithABI unsafe;
+
+ // Should only be called on ropes.
+ MOZ_ASSERT(str->isRope());
+
+ // ensureLinear is intentionally called with a nullptr to avoid OOM reporting.
+ return str->ensureLinear(nullptr);
+}
+
+JSLinearString* LinearizeForCharAccess(JSContext* cx, JSString* str) {
+ // Should only be called on ropes.
+ MOZ_ASSERT(str->isRope());
+
+ return str->ensureLinear(cx);
+}
+
+bool SetProperty(JSContext* cx, HandleObject obj, Handle<PropertyName*> name,
+ HandleValue value, bool strict, jsbytecode* pc) {
+ RootedId id(cx, NameToId(name));
+
+ RootedValue receiver(cx, ObjectValue(*obj));
+ ObjectOpResult result;
+ if (MOZ_LIKELY(!obj->getOpsSetProperty())) {
+ JSOp op = JSOp(*pc);
+ if (op == JSOp::SetName || op == JSOp::StrictSetName ||
+ op == JSOp::SetGName || op == JSOp::StrictSetGName) {
+ if (!NativeSetProperty<Unqualified>(cx, obj.as<NativeObject>(), id, value,
+ receiver, result)) {
+ return false;
+ }
+ } else {
+ if (!NativeSetProperty<Qualified>(cx, obj.as<NativeObject>(), id, value,
+ receiver, result)) {
+ return false;
+ }
+ }
+ } else {
+ if (!SetProperty(cx, obj, id, value, receiver, result)) {
+ return false;
+ }
+ }
+ return result.checkStrictModeError(cx, obj, id, strict);
+}
+
+bool InterruptCheck(JSContext* cx) {
+ gc::MaybeVerifyBarriers(cx);
+
+ return CheckForInterrupt(cx);
+}
+
+JSObject* NewStringObject(JSContext* cx, HandleString str) {
+ return StringObject::create(cx, str);
+}
+
+bool OperatorIn(JSContext* cx, HandleValue key, HandleObject obj, bool* out) {
+ RootedId id(cx);
+ return ToPropertyKey(cx, key, &id) && HasProperty(cx, obj, id, out);
+}
+
+bool GetIntrinsicValue(JSContext* cx, Handle<PropertyName*> name,
+ MutableHandleValue rval) {
+ return GlobalObject::getIntrinsicValue(cx, cx->global(), name, rval);
+}
+
+bool CreateThisFromIC(JSContext* cx, HandleObject callee,
+ HandleObject newTarget, MutableHandleValue rval) {
+ HandleFunction fun = callee.as<JSFunction>();
+ MOZ_ASSERT(fun->isInterpreted());
+ MOZ_ASSERT(fun->isConstructor());
+ MOZ_ASSERT(cx->realm() == fun->realm(),
+ "Realm switching happens before creating this");
+
+ // CreateThis expects rval to be this magic value.
+ rval.set(MagicValue(JS_IS_CONSTRUCTING));
+
+ if (!js::CreateThis(cx, fun, newTarget, GenericObject, rval)) {
+ return false;
+ }
+
+ MOZ_ASSERT_IF(rval.isObject(), fun->realm() == rval.toObject().nonCCWRealm());
+ return true;
+}
+
+bool CreateThisFromIon(JSContext* cx, HandleObject callee,
+ HandleObject newTarget, MutableHandleValue rval) {
+ // Return JS_IS_CONSTRUCTING for cases not supported by the inline call path.
+ rval.set(MagicValue(JS_IS_CONSTRUCTING));
+
+ if (!callee->is<JSFunction>()) {
+ return true;
+ }
+
+ HandleFunction fun = callee.as<JSFunction>();
+ if (!fun->isInterpreted() || !fun->isConstructor()) {
+ return true;
+ }
+
+ // If newTarget is not a function or is a function with a possibly-getter
+ // .prototype property, return NullValue to signal to LCallGeneric that it has
+ // to take the slow path. Note that we return NullValue instead of a
+ // MagicValue only because it's easier and faster to check for in JIT code
+ // (if we returned a MagicValue, JIT code would have to check both the type
+ // tag and the JSWhyMagic payload).
+ if (!fun->constructorNeedsUninitializedThis()) {
+ if (!newTarget->is<JSFunction>()) {
+ rval.setNull();
+ return true;
+ }
+ JSFunction* newTargetFun = &newTarget->as<JSFunction>();
+ if (!newTargetFun->hasNonConfigurablePrototypeDataProperty()) {
+ rval.setNull();
+ return true;
+ }
+ }
+
+ AutoRealm ar(cx, fun);
+ if (!js::CreateThis(cx, fun, newTarget, GenericObject, rval)) {
+ return false;
+ }
+
+ MOZ_ASSERT_IF(rval.isObject(), fun->realm() == rval.toObject().nonCCWRealm());
+ return true;
+}
+
+void PostWriteBarrier(JSRuntime* rt, js::gc::Cell* cell) {
+ AutoUnsafeCallWithABI unsafe;
+ MOZ_ASSERT(!IsInsideNursery(cell));
+ rt->gc.storeBuffer().putWholeCell(cell);
+}
+
+static const size_t MAX_WHOLE_CELL_BUFFER_SIZE = 4096;
+
+template <IndexInBounds InBounds>
+void PostWriteElementBarrier(JSRuntime* rt, JSObject* obj, int32_t index) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!IsInsideNursery(obj));
+
+ if (InBounds == IndexInBounds::Yes) {
+ MOZ_ASSERT(uint32_t(index) <
+ obj->as<NativeObject>().getDenseInitializedLength());
+ } else {
+ if (MOZ_UNLIKELY(!obj->is<NativeObject>() || index < 0 ||
+ uint32_t(index) >=
+ NativeObject::MAX_DENSE_ELEMENTS_COUNT)) {
+ rt->gc.storeBuffer().putWholeCell(obj);
+ return;
+ }
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ if (nobj->isInWholeCellBuffer()) {
+ return;
+ }
+
+ if (nobj->getDenseInitializedLength() > MAX_WHOLE_CELL_BUFFER_SIZE
+#ifdef JS_GC_ZEAL
+ || rt->hasZealMode(gc::ZealMode::ElementsBarrier)
+#endif
+ ) {
+ rt->gc.storeBuffer().putSlot(nobj, HeapSlot::Element,
+ nobj->unshiftedIndex(index), 1);
+ return;
+ }
+
+ rt->gc.storeBuffer().putWholeCell(obj);
+}
+
+template void PostWriteElementBarrier<IndexInBounds::Yes>(JSRuntime* rt,
+ JSObject* obj,
+ int32_t index);
+
+template void PostWriteElementBarrier<IndexInBounds::Maybe>(JSRuntime* rt,
+ JSObject* obj,
+ int32_t index);
+
+void PostGlobalWriteBarrier(JSRuntime* rt, GlobalObject* obj) {
+ MOZ_ASSERT(obj->JSObject::is<GlobalObject>());
+
+ if (!obj->realm()->globalWriteBarriered) {
+ PostWriteBarrier(rt, obj);
+ obj->realm()->globalWriteBarriered = 1;
+ }
+}
+
+bool GetInt32FromStringPure(JSContext* cx, JSString* str, int32_t* result) {
+ // We shouldn't GC here as this is called directly from IC code.
+ AutoUnsafeCallWithABI unsafe;
+
+ double d;
+ if (!StringToNumberPure(cx, str, &d)) {
+ return false;
+ }
+
+ return mozilla::NumberIsInt32(d, result);
+}
+
+int32_t GetIndexFromString(JSString* str) {
+ // We shouldn't GC here as this is called directly from IC code.
+ AutoUnsafeCallWithABI unsafe;
+
+ if (!str->isLinear()) {
+ return -1;
+ }
+
+ uint32_t index = UINT32_MAX; // Initialize this to appease Valgrind.
+ if (!str->asLinear().isIndex(&index) || index > INT32_MAX) {
+ return -1;
+ }
+
+ return int32_t(index);
+}
+
+JSObject* WrapObjectPure(JSContext* cx, JSObject* obj) {
+ // IC code calls this directly so we shouldn't GC.
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(obj);
+ MOZ_ASSERT(cx->compartment() != obj->compartment());
+
+ // From: Compartment::getNonWrapperObjectForCurrentCompartment
+ // Note that if the object is same-compartment, but has been wrapped into a
+ // different compartment, we need to unwrap it and return the bare same-
+ // compartment object. Note again that windows are always wrapped by a
+ // WindowProxy even when same-compartment so take care not to strip this
+ // particular wrapper.
+ obj = UncheckedUnwrap(obj, /* stopAtWindowProxy = */ true);
+ if (cx->compartment() == obj->compartment()) {
+ MOZ_ASSERT(!IsWindow(obj));
+ JS::ExposeObjectToActiveJS(obj);
+ return obj;
+ }
+
+ // Try to Lookup an existing wrapper for this object. We assume that
+ // if we can find such a wrapper, not calling preWrap is correct.
+ if (ObjectWrapperMap::Ptr p = cx->compartment()->lookupWrapper(obj)) {
+ JSObject* wrapped = p->value().get();
+
+ // Ensure the wrapper is still exposed.
+ JS::ExposeObjectToActiveJS(wrapped);
+ return wrapped;
+ }
+
+ return nullptr;
+}
+
+bool DebugPrologue(JSContext* cx, BaselineFrame* frame) {
+ return DebugAPI::onEnterFrame(cx, frame);
+}
+
+bool DebugEpilogueOnBaselineReturn(JSContext* cx, BaselineFrame* frame,
+ const jsbytecode* pc) {
+ if (!DebugEpilogue(cx, frame, pc, true)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool DebugEpilogue(JSContext* cx, BaselineFrame* frame, const jsbytecode* pc,
+ bool ok) {
+ // If DebugAPI::onLeaveFrame returns |true| we have to return the frame's
+ // return value. If it returns |false|, the debugger threw an exception.
+ // In both cases we have to pop debug scopes.
+ ok = DebugAPI::onLeaveFrame(cx, frame, pc, ok);
+
+ // Unwind to the outermost environment.
+ EnvironmentIter ei(cx, frame, pc);
+ UnwindAllEnvironmentsInFrame(cx, ei);
+
+ if (!ok) {
+ // Pop this frame by updating packedExitFP, so that the exception
+ // handling code will start at the previous frame.
+ JitFrameLayout* prefix = frame->framePrefix();
+ EnsureUnwoundJitExitFrame(cx->activation()->asJit(), prefix);
+ return false;
+ }
+
+ return true;
+}
+
+void FrameIsDebuggeeCheck(BaselineFrame* frame) {
+ AutoUnsafeCallWithABI unsafe;
+ if (frame->script()->isDebuggee()) {
+ frame->setIsDebuggee();
+ }
+}
+
+JSObject* CreateGeneratorFromFrame(JSContext* cx, BaselineFrame* frame) {
+ return AbstractGeneratorObject::createFromFrame(cx, frame);
+}
+
+JSObject* CreateGenerator(JSContext* cx, HandleFunction callee,
+ HandleScript script, HandleObject environmentChain,
+ HandleObject args) {
+ Rooted<ArgumentsObject*> argsObj(
+ cx, args ? &args->as<ArgumentsObject>() : nullptr);
+ return AbstractGeneratorObject::create(cx, callee, script, environmentChain,
+ argsObj);
+}
+
+bool NormalSuspend(JSContext* cx, HandleObject obj, BaselineFrame* frame,
+ uint32_t frameSize, const jsbytecode* pc) {
+ MOZ_ASSERT(JSOp(*pc) == JSOp::InitialYield || JSOp(*pc) == JSOp::Yield ||
+ JSOp(*pc) == JSOp::Await);
+
+ // Minus one because we don't want to include the return value.
+ uint32_t numSlots = frame->numValueSlots(frameSize) - 1;
+ MOZ_ASSERT(numSlots >= frame->script()->nfixed());
+ return AbstractGeneratorObject::suspend(cx, obj, frame, pc, numSlots);
+}
+
+bool FinalSuspend(JSContext* cx, HandleObject obj, const jsbytecode* pc) {
+ MOZ_ASSERT(JSOp(*pc) == JSOp::FinalYieldRval);
+ AbstractGeneratorObject::finalSuspend(obj);
+ return true;
+}
+
+bool InterpretResume(JSContext* cx, HandleObject obj, Value* stackValues,
+ MutableHandleValue rval) {
+ MOZ_ASSERT(obj->is<AbstractGeneratorObject>());
+
+ // The |stackValues| argument points to the JSOp::Resume operands on the
+ // native stack. Because the stack grows down, these values are:
+ //
+ // [resumeKind, argument, generator, ..]
+
+ MOZ_ASSERT(stackValues[2].toObject() == *obj);
+
+ GeneratorResumeKind resumeKind = IntToResumeKind(stackValues[0].toInt32());
+ JSAtom* kind = ResumeKindToAtom(cx, resumeKind);
+
+ FixedInvokeArgs<3> args(cx);
+
+ args[0].setObject(*obj);
+ args[1].set(stackValues[1]);
+ args[2].setString(kind);
+
+ return CallSelfHostedFunction(cx, cx->names().InterpretGeneratorResume,
+ UndefinedHandleValue, args, rval);
+}
+
+bool DebugAfterYield(JSContext* cx, BaselineFrame* frame) {
+ // The BaselineFrame has just been constructed by JSOp::Resume in the
+ // caller. We need to set its debuggee flag as necessary.
+ //
+ // If a breakpoint is set on JSOp::AfterYield, or stepping is enabled,
+ // we may already have done this work. Don't fire onEnterFrame again.
+ if (frame->script()->isDebuggee() && !frame->isDebuggee()) {
+ frame->setIsDebuggee();
+ return DebugAPI::onResumeFrame(cx, frame);
+ }
+
+ return true;
+}
+
+bool GeneratorThrowOrReturn(JSContext* cx, BaselineFrame* frame,
+ Handle<AbstractGeneratorObject*> genObj,
+ HandleValue arg, int32_t resumeKindArg) {
+ GeneratorResumeKind resumeKind = IntToResumeKind(resumeKindArg);
+ MOZ_ALWAYS_FALSE(
+ js::GeneratorThrowOrReturn(cx, frame, genObj, arg, resumeKind));
+ return false;
+}
+
+bool GlobalDeclInstantiationFromIon(JSContext* cx, HandleScript script,
+ const jsbytecode* pc) {
+ MOZ_ASSERT(!script->hasNonSyntacticScope());
+
+ RootedObject envChain(cx, &cx->global()->lexicalEnvironment());
+ GCThingIndex lastFun = GET_GCTHING_INDEX(pc);
+
+ return GlobalOrEvalDeclInstantiation(cx, envChain, script, lastFun);
+}
+
+bool InitFunctionEnvironmentObjects(JSContext* cx, BaselineFrame* frame) {
+ return frame->initFunctionEnvironmentObjects(cx);
+}
+
+bool NewArgumentsObject(JSContext* cx, BaselineFrame* frame,
+ MutableHandleValue res) {
+ ArgumentsObject* obj = ArgumentsObject::createExpected(cx, frame);
+ if (!obj) {
+ return false;
+ }
+ res.setObject(*obj);
+ return true;
+}
+
+ArrayObject* NewArrayObjectEnsureDenseInitLength(JSContext* cx, int32_t count) {
+ MOZ_ASSERT(count >= 0);
+
+ auto* array = NewDenseFullyAllocatedArray(cx, count);
+ if (!array) {
+ return nullptr;
+ }
+ array->ensureDenseInitializedLength(0, count);
+
+ return array;
+}
+
+JSObject* InitRestParameter(JSContext* cx, uint32_t length, Value* rest,
+ HandleObject objRes) {
+ if (objRes) {
+ Handle<ArrayObject*> arrRes = objRes.as<ArrayObject>();
+ MOZ_ASSERT(arrRes->getDenseInitializedLength() == 0);
+
+ // Fast path: we managed to allocate the array inline; initialize the
+ // slots.
+ if (length > 0) {
+ if (!arrRes->ensureElements(cx, length)) {
+ return nullptr;
+ }
+ arrRes->initDenseElements(rest, length);
+ arrRes->setLength(length);
+ }
+ return arrRes;
+ }
+
+ return NewDenseCopiedArray(cx, length, rest);
+}
+
+bool HandleDebugTrap(JSContext* cx, BaselineFrame* frame,
+ const uint8_t* retAddr) {
+ RootedScript script(cx, frame->script());
+ jsbytecode* pc;
+ if (frame->runningInInterpreter()) {
+ pc = frame->interpreterPC();
+ } else {
+ BaselineScript* blScript = script->baselineScript();
+ pc = blScript->retAddrEntryFromReturnAddress(retAddr).pc(script);
+ }
+
+ // The Baseline Interpreter calls HandleDebugTrap for every op when the script
+ // is in step mode or has breakpoints. The Baseline Compiler can toggle
+ // breakpoints more granularly for specific bytecode PCs.
+ if (frame->runningInInterpreter()) {
+ MOZ_ASSERT(DebugAPI::hasAnyBreakpointsOrStepMode(script));
+ } else {
+ MOZ_ASSERT(DebugAPI::stepModeEnabled(script) ||
+ DebugAPI::hasBreakpointsAt(script, pc));
+ }
+
+ if (JSOp(*pc) == JSOp::AfterYield) {
+ // JSOp::AfterYield will set the frame's debuggee flag and call the
+ // onEnterFrame handler, but if we set a breakpoint there we have to do
+ // it now.
+ MOZ_ASSERT(!frame->isDebuggee());
+
+ if (!DebugAfterYield(cx, frame)) {
+ return false;
+ }
+
+ // If the frame is not a debuggee we're done. This can happen, for instance,
+ // if the onEnterFrame hook called removeDebuggee.
+ if (!frame->isDebuggee()) {
+ return true;
+ }
+ }
+
+ MOZ_ASSERT(frame->isDebuggee());
+
+ if (DebugAPI::stepModeEnabled(script) && !DebugAPI::onSingleStep(cx)) {
+ return false;
+ }
+
+ if (DebugAPI::hasBreakpointsAt(script, pc) && !DebugAPI::onTrap(cx)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool OnDebuggerStatement(JSContext* cx, BaselineFrame* frame) {
+ return DebugAPI::onDebuggerStatement(cx, frame);
+}
+
+bool GlobalHasLiveOnDebuggerStatement(JSContext* cx) {
+ AutoUnsafeCallWithABI unsafe;
+ return cx->realm()->isDebuggee() &&
+ DebugAPI::hasDebuggerStatementHook(cx->global());
+}
+
+bool PushLexicalEnv(JSContext* cx, BaselineFrame* frame,
+ Handle<LexicalScope*> scope) {
+ return frame->pushLexicalEnvironment(cx, scope);
+}
+
+bool DebugLeaveThenPopLexicalEnv(JSContext* cx, BaselineFrame* frame,
+ const jsbytecode* pc) {
+ MOZ_ALWAYS_TRUE(DebugLeaveLexicalEnv(cx, frame, pc));
+ frame->popOffEnvironmentChain<ScopedLexicalEnvironmentObject>();
+ return true;
+}
+
+bool FreshenLexicalEnv(JSContext* cx, BaselineFrame* frame) {
+ return frame->freshenLexicalEnvironment(cx);
+}
+
+bool DebugLeaveThenFreshenLexicalEnv(JSContext* cx, BaselineFrame* frame,
+ const jsbytecode* pc) {
+ MOZ_ALWAYS_TRUE(DebugLeaveLexicalEnv(cx, frame, pc));
+ return frame->freshenLexicalEnvironment(cx);
+}
+
+bool RecreateLexicalEnv(JSContext* cx, BaselineFrame* frame) {
+ return frame->recreateLexicalEnvironment(cx);
+}
+
+bool DebugLeaveThenRecreateLexicalEnv(JSContext* cx, BaselineFrame* frame,
+ const jsbytecode* pc) {
+ MOZ_ALWAYS_TRUE(DebugLeaveLexicalEnv(cx, frame, pc));
+ return frame->recreateLexicalEnvironment(cx);
+}
+
+bool DebugLeaveLexicalEnv(JSContext* cx, BaselineFrame* frame,
+ const jsbytecode* pc) {
+ MOZ_ASSERT_IF(!frame->runningInInterpreter(),
+ frame->script()->baselineScript()->hasDebugInstrumentation());
+ if (cx->realm()->isDebuggee()) {
+ DebugEnvironments::onPopLexical(cx, frame, pc);
+ }
+ return true;
+}
+
+bool PushClassBodyEnv(JSContext* cx, BaselineFrame* frame,
+ Handle<ClassBodyScope*> scope) {
+ return frame->pushClassBodyEnvironment(cx, scope);
+}
+
+bool PushVarEnv(JSContext* cx, BaselineFrame* frame, Handle<Scope*> scope) {
+ return frame->pushVarEnvironment(cx, scope);
+}
+
+bool EnterWith(JSContext* cx, BaselineFrame* frame, HandleValue val,
+ Handle<WithScope*> templ) {
+ return EnterWithOperation(cx, frame, val, templ);
+}
+
+bool LeaveWith(JSContext* cx, BaselineFrame* frame) {
+ if (MOZ_UNLIKELY(frame->isDebuggee())) {
+ DebugEnvironments::onPopWith(frame);
+ }
+ frame->popOffEnvironmentChain<WithEnvironmentObject>();
+ return true;
+}
+
+bool InitBaselineFrameForOsr(BaselineFrame* frame,
+ InterpreterFrame* interpFrame,
+ uint32_t numStackValues) {
+ return frame->initForOsr(interpFrame, numStackValues);
+}
+
+JSString* StringReplace(JSContext* cx, HandleString string,
+ HandleString pattern, HandleString repl) {
+ MOZ_ASSERT(string);
+ MOZ_ASSERT(pattern);
+ MOZ_ASSERT(repl);
+
+ return str_replace_string_raw(cx, string, pattern, repl);
+}
+
+void AssertValidBigIntPtr(JSContext* cx, JS::BigInt* bi) {
+ AutoUnsafeCallWithABI unsafe;
+ // FIXME: check runtime?
+ MOZ_ASSERT(cx->zone() == bi->zone());
+ MOZ_ASSERT(bi->isAligned());
+ MOZ_ASSERT(bi->getAllocKind() == gc::AllocKind::BIGINT);
+}
+
+void AssertValidObjectPtr(JSContext* cx, JSObject* obj) {
+ AutoUnsafeCallWithABI unsafe;
+#ifdef DEBUG
+ // Check what we can, so that we'll hopefully assert/crash if we get a
+ // bogus object (pointer).
+ MOZ_ASSERT(obj->compartment() == cx->compartment());
+ MOZ_ASSERT(obj->zoneFromAnyThread() == cx->zone());
+ MOZ_ASSERT(obj->runtimeFromMainThread() == cx->runtime());
+
+ if (obj->isTenured()) {
+ MOZ_ASSERT(obj->isAligned());
+ gc::AllocKind kind = obj->asTenured().getAllocKind();
+ MOZ_ASSERT(gc::IsObjectAllocKind(kind));
+ }
+#endif
+}
+
+void AssertValidStringPtr(JSContext* cx, JSString* str) {
+ AutoUnsafeCallWithABI unsafe;
+#ifdef DEBUG
+ // We can't closely inspect strings from another runtime.
+ if (str->runtimeFromAnyThread() != cx->runtime()) {
+ MOZ_ASSERT(str->isPermanentAtom());
+ return;
+ }
+
+ if (str->isAtom()) {
+ MOZ_ASSERT(str->zone()->isAtomsZone());
+ } else {
+ MOZ_ASSERT(str->zone() == cx->zone());
+ }
+
+ MOZ_ASSERT(str->isAligned());
+ MOZ_ASSERT(str->length() <= JSString::MAX_LENGTH);
+
+ gc::AllocKind kind = str->getAllocKind();
+ if (str->isFatInline()) {
+ MOZ_ASSERT(kind == gc::AllocKind::FAT_INLINE_STRING ||
+ kind == gc::AllocKind::FAT_INLINE_ATOM);
+ } else if (str->isExternal()) {
+ MOZ_ASSERT(kind == gc::AllocKind::EXTERNAL_STRING);
+ } else if (str->isAtom()) {
+ MOZ_ASSERT(kind == gc::AllocKind::ATOM);
+ } else if (str->isLinear()) {
+ MOZ_ASSERT(kind == gc::AllocKind::STRING ||
+ kind == gc::AllocKind::FAT_INLINE_STRING);
+ } else {
+ MOZ_ASSERT(kind == gc::AllocKind::STRING);
+ }
+#endif
+}
+
+void AssertValidSymbolPtr(JSContext* cx, JS::Symbol* sym) {
+ AutoUnsafeCallWithABI unsafe;
+
+ // We can't closely inspect symbols from another runtime.
+ if (sym->runtimeFromAnyThread() != cx->runtime()) {
+ MOZ_ASSERT(sym->isWellKnownSymbol());
+ return;
+ }
+
+ MOZ_ASSERT(sym->zone()->isAtomsZone());
+ MOZ_ASSERT(sym->isAligned());
+ if (JSAtom* desc = sym->description()) {
+ AssertValidStringPtr(cx, desc);
+ }
+
+ MOZ_ASSERT(sym->getAllocKind() == gc::AllocKind::SYMBOL);
+}
+
+void AssertValidValue(JSContext* cx, Value* v) {
+ AutoUnsafeCallWithABI unsafe;
+ if (v->isObject()) {
+ AssertValidObjectPtr(cx, &v->toObject());
+ } else if (v->isString()) {
+ AssertValidStringPtr(cx, v->toString());
+ } else if (v->isSymbol()) {
+ AssertValidSymbolPtr(cx, v->toSymbol());
+ } else if (v->isBigInt()) {
+ AssertValidBigIntPtr(cx, v->toBigInt());
+ }
+}
+
+bool ObjectIsCallable(JSObject* obj) {
+ AutoUnsafeCallWithABI unsafe;
+ return obj->isCallable();
+}
+
+bool ObjectIsConstructor(JSObject* obj) {
+ AutoUnsafeCallWithABI unsafe;
+ return obj->isConstructor();
+}
+
+void JitValuePreWriteBarrier(JSRuntime* rt, Value* vp) {
+ AutoUnsafeCallWithABI unsafe;
+ MOZ_ASSERT(vp->isGCThing());
+ MOZ_ASSERT(!vp->toGCThing()->isMarkedBlack());
+ gc::ValuePreWriteBarrier(*vp);
+}
+
+void JitStringPreWriteBarrier(JSRuntime* rt, JSString** stringp) {
+ AutoUnsafeCallWithABI unsafe;
+ MOZ_ASSERT(*stringp);
+ MOZ_ASSERT(!(*stringp)->isMarkedBlack());
+ gc::PreWriteBarrier(*stringp);
+}
+
+void JitObjectPreWriteBarrier(JSRuntime* rt, JSObject** objp) {
+ AutoUnsafeCallWithABI unsafe;
+ MOZ_ASSERT(*objp);
+ MOZ_ASSERT(!(*objp)->isMarkedBlack());
+ gc::PreWriteBarrier(*objp);
+}
+
+void JitShapePreWriteBarrier(JSRuntime* rt, Shape** shapep) {
+ AutoUnsafeCallWithABI unsafe;
+ MOZ_ASSERT(!(*shapep)->isMarkedBlack());
+ gc::PreWriteBarrier(*shapep);
+}
+
+bool ThrowRuntimeLexicalError(JSContext* cx, unsigned errorNumber) {
+ ScriptFrameIter iter(cx);
+ RootedScript script(cx, iter.script());
+ ReportRuntimeLexicalError(cx, errorNumber, script, iter.pc());
+ return false;
+}
+
+bool ThrowBadDerivedReturnOrUninitializedThis(JSContext* cx, HandleValue v) {
+ MOZ_ASSERT(!v.isObject());
+ if (v.isUndefined()) {
+ return js::ThrowUninitializedThis(cx);
+ }
+
+ ReportValueError(cx, JSMSG_BAD_DERIVED_RETURN, JSDVG_IGNORE_STACK, v,
+ nullptr);
+ return false;
+}
+
+bool BaselineGetFunctionThis(JSContext* cx, BaselineFrame* frame,
+ MutableHandleValue res) {
+ return GetFunctionThis(cx, frame, res);
+}
+
+bool CallNativeGetter(JSContext* cx, HandleFunction callee,
+ HandleValue receiver, MutableHandleValue result) {
+ AutoRealm ar(cx, callee);
+
+ MOZ_ASSERT(callee->isNativeFun());
+ JSNative natfun = callee->native();
+
+ JS::RootedValueArray<2> vp(cx);
+ vp[0].setObject(*callee.get());
+ vp[1].set(receiver);
+
+ if (!natfun(cx, 0, vp.begin())) {
+ return false;
+ }
+
+ result.set(vp[0]);
+ return true;
+}
+
+bool CallDOMGetter(JSContext* cx, const JSJitInfo* info, HandleObject obj,
+ MutableHandleValue result) {
+ MOZ_ASSERT(info->type() == JSJitInfo::Getter);
+ MOZ_ASSERT(obj->is<NativeObject>());
+ MOZ_ASSERT(obj->getClass()->isDOMClass());
+
+#ifdef DEBUG
+ DOMInstanceClassHasProtoAtDepth instanceChecker =
+ cx->runtime()->DOMcallbacks->instanceClassMatchesProto;
+ MOZ_ASSERT(instanceChecker(obj->getClass(), info->protoID, info->depth));
+#endif
+
+ // Loading DOM_OBJECT_SLOT, which must be the first slot.
+ JS::Value val = JS::GetReservedSlot(obj, 0);
+ JSJitGetterOp getter = info->getter;
+ return getter(cx, obj, val.toPrivate(), JSJitGetterCallArgs(result));
+}
+
+bool CallNativeSetter(JSContext* cx, HandleFunction callee, HandleObject obj,
+ HandleValue rhs) {
+ AutoRealm ar(cx, callee);
+
+ MOZ_ASSERT(callee->isNativeFun());
+ JSNative natfun = callee->native();
+
+ JS::RootedValueArray<3> vp(cx);
+ vp[0].setObject(*callee.get());
+ vp[1].setObject(*obj.get());
+ vp[2].set(rhs);
+
+ return natfun(cx, 1, vp.begin());
+}
+
+bool CallDOMSetter(JSContext* cx, const JSJitInfo* info, HandleObject obj,
+ HandleValue value) {
+ MOZ_ASSERT(info->type() == JSJitInfo::Setter);
+ MOZ_ASSERT(obj->is<NativeObject>());
+ MOZ_ASSERT(obj->getClass()->isDOMClass());
+
+#ifdef DEBUG
+ DOMInstanceClassHasProtoAtDepth instanceChecker =
+ cx->runtime()->DOMcallbacks->instanceClassMatchesProto;
+ MOZ_ASSERT(instanceChecker(obj->getClass(), info->protoID, info->depth));
+#endif
+
+ // Loading DOM_OBJECT_SLOT, which must be the first slot.
+ JS::Value val = JS::GetReservedSlot(obj, 0);
+ JSJitSetterOp setter = info->setter;
+
+ RootedValue v(cx, value);
+ return setter(cx, obj, val.toPrivate(), JSJitSetterCallArgs(&v));
+}
+
+bool EqualStringsHelperPure(JSString* str1, JSString* str2) {
+ // IC code calls this directly so we shouldn't GC.
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(str1->isAtom());
+ MOZ_ASSERT(!str2->isAtom());
+ MOZ_ASSERT(str1->length() == str2->length());
+
+ // ensureLinear is intentionally called with a nullptr to avoid OOM
+ // reporting; if it fails, we will continue to the next stub.
+ JSLinearString* str2Linear = str2->ensureLinear(nullptr);
+ if (!str2Linear) {
+ return false;
+ }
+
+ return EqualChars(&str1->asLinear(), str2Linear);
+}
+
+static bool MaybeTypedArrayIndexString(jsid id) {
+ MOZ_ASSERT(id.isAtom() || id.isSymbol());
+
+ if (MOZ_LIKELY(id.isAtom())) {
+ JSAtom* str = id.toAtom();
+ if (str->length() > 0) {
+ // Only check the first character because we want this function to be
+ // fast.
+ return CanStartTypedArrayIndex(str->latin1OrTwoByteChar(0));
+ }
+ }
+ return false;
+}
+
+static void VerifyCacheEntry(JSContext* cx, NativeObject* obj, PropertyKey key,
+ const MegamorphicCache::Entry& entry) {
+#ifdef DEBUG
+ if (entry.isMissingProperty()) {
+ NativeObject* pobj;
+ PropertyResult prop;
+ MOZ_ASSERT(LookupPropertyPure(cx, obj, key, &pobj, &prop));
+ MOZ_ASSERT(prop.isNotFound());
+ return;
+ }
+ if (entry.isMissingOwnProperty()) {
+ MOZ_ASSERT(!obj->containsPure(key));
+ return;
+ }
+ MOZ_ASSERT(entry.isDataProperty());
+ for (size_t i = 0, numHops = entry.numHops(); i < numHops; i++) {
+ MOZ_ASSERT(!obj->containsPure(key));
+ obj = &obj->staticPrototype()->as<NativeObject>();
+ }
+ mozilla::Maybe<PropertyInfo> prop = obj->lookupPure(key);
+ MOZ_ASSERT(prop.isSome());
+ MOZ_ASSERT(prop->isDataProperty());
+ MOZ_ASSERT(prop->slot() == entry.slot());
+#endif
+}
+
+static MOZ_ALWAYS_INLINE bool GetNativeDataPropertyPureFallback(
+ JSContext* cx, JSObject* obj, jsid id, Value* vp,
+ MegamorphicCache::Entry* entry) {
+ NativeObject* nobj = &obj->as<NativeObject>();
+ Shape* receiverShape = obj->shape();
+ MegamorphicCache& cache = cx->caches().megamorphicCache;
+
+ MOZ_ASSERT_IF(JitOptions.enableWatchtowerMegamorphic, entry);
+
+ size_t numHops = 0;
+ while (true) {
+ MOZ_ASSERT(!nobj->getOpsLookupProperty());
+
+ uint32_t index;
+ if (PropMap* map = nobj->shape()->lookup(cx, id, &index)) {
+ PropertyInfo prop = map->getPropertyInfo(index);
+ if (!prop.isDataProperty()) {
+ return false;
+ }
+ if (JitOptions.enableWatchtowerMegamorphic) {
+ cache.initEntryForDataProperty(entry, receiverShape, id, numHops,
+ prop.slot());
+ }
+ *vp = nobj->getSlot(prop.slot());
+ return true;
+ }
+
+ // Property not found. Watch out for Class hooks and TypedArrays.
+ if (MOZ_UNLIKELY(!nobj->is<PlainObject>())) {
+ if (ClassMayResolveId(cx->names(), nobj->getClass(), id, nobj)) {
+ return false;
+ }
+
+ // Don't skip past TypedArrayObjects if the id can be a TypedArray index.
+ if (nobj->is<TypedArrayObject>()) {
+ if (MaybeTypedArrayIndexString(id)) {
+ return false;
+ }
+ }
+ }
+
+ JSObject* proto = nobj->staticPrototype();
+ if (!proto) {
+ if (JitOptions.enableWatchtowerMegamorphic) {
+ cache.initEntryForMissingProperty(entry, receiverShape, id);
+ }
+ vp->setUndefined();
+ return true;
+ }
+
+ if (!proto->is<NativeObject>()) {
+ return false;
+ }
+ nobj = &proto->as<NativeObject>();
+ numHops++;
+ }
+}
+
+static MOZ_ALWAYS_INLINE bool GetNativeDataPropertyPure(JSContext* cx,
+ JSObject* obj, jsid id,
+ Value* vp) {
+ // Fast path used by megamorphic IC stubs. Unlike our other property
+ // lookup paths, this is optimized to be as fast as possible for simple
+ // data property lookups.
+
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(id.isAtom() || id.isSymbol());
+
+ Shape* receiverShape = obj->shape();
+ MegamorphicCache& cache = cx->caches().megamorphicCache;
+ MegamorphicCache::Entry* entry = nullptr;
+ if (JitOptions.enableWatchtowerMegamorphic &&
+ cache.lookup(receiverShape, id, &entry)) {
+ NativeObject* nobj = &obj->as<NativeObject>();
+ VerifyCacheEntry(cx, nobj, id, *entry);
+ if (entry->isDataProperty()) {
+ for (size_t i = 0, numHops = entry->numHops(); i < numHops; i++) {
+ nobj = &nobj->staticPrototype()->as<NativeObject>();
+ }
+ *vp = nobj->getSlot(entry->slot());
+ return true;
+ }
+ if (entry->isMissingProperty()) {
+ vp->setUndefined();
+ return true;
+ }
+ MOZ_ASSERT(entry->isMissingOwnProperty());
+ }
+
+ if (!obj->is<NativeObject>()) {
+ return false;
+ }
+
+ return GetNativeDataPropertyPureFallback(cx, obj, id, vp, entry);
+}
+
+bool GetNativeDataPropertyPure(JSContext* cx, JSObject* obj, PropertyName* name,
+ Value* vp) {
+ return GetNativeDataPropertyPure(cx, obj, NameToId(name), vp);
+}
+
+bool GetNativeDataPropertyPureFallback(JSContext* cx, JSObject* obj,
+ PropertyKey id, Value* vp) {
+ AutoUnsafeCallWithABI unsafe;
+
+ Shape* receiverShape = obj->shape();
+ MegamorphicCache& cache = cx->caches().megamorphicCache;
+ MegamorphicCache::Entry* entry = nullptr;
+ if (JitOptions.enableWatchtowerMegamorphic) {
+ cache.lookup(receiverShape, id, &entry);
+ }
+ return GetNativeDataPropertyPureFallback(cx, obj, id, vp, entry);
+}
+
+static MOZ_ALWAYS_INLINE bool ValueToAtomOrSymbolPure(JSContext* cx,
+ const Value& idVal,
+ jsid* id) {
+ if (MOZ_LIKELY(idVal.isString())) {
+ JSString* s = idVal.toString();
+ JSAtom* atom;
+ if (s->isAtom()) {
+ atom = &s->asAtom();
+ } else {
+ atom = AtomizeString(cx, s);
+ if (!atom) {
+ cx->recoverFromOutOfMemory();
+ return false;
+ }
+ }
+
+ // Watch out for integer ids because they may be stored in dense elements.
+ static_assert(PropertyKey::IntMin == 0);
+ static_assert(NativeObject::MAX_DENSE_ELEMENTS_COUNT < PropertyKey::IntMax,
+ "All dense elements must have integer jsids");
+ uint32_t index;
+ if (MOZ_UNLIKELY(atom->isIndex(&index) && index <= PropertyKey::IntMax)) {
+ return false;
+ }
+
+ *id = PropertyKey::NonIntAtom(atom);
+ return true;
+ }
+
+ if (idVal.isSymbol()) {
+ *id = PropertyKey::Symbol(idVal.toSymbol());
+ return true;
+ }
+
+ return false;
+}
+
+bool GetNativeDataPropertyByValuePure(JSContext* cx, JSObject* obj, Value* vp) {
+ AutoUnsafeCallWithABI unsafe;
+
+ // vp[0] contains the id, result will be stored in vp[1].
+ Value idVal = vp[0];
+ jsid id;
+ if (!ValueToAtomOrSymbolPure(cx, idVal, &id)) {
+ return false;
+ }
+
+ Value* res = vp + 1;
+ return GetNativeDataPropertyPure(cx, obj, id, res);
+}
+
+bool SetNativeDataPropertyPure(JSContext* cx, JSObject* obj, PropertyName* name,
+ Value* val) {
+ AutoUnsafeCallWithABI unsafe;
+
+ if (MOZ_UNLIKELY(!obj->is<NativeObject>())) {
+ return false;
+ }
+
+ NativeObject* nobj = &obj->as<NativeObject>();
+ uint32_t index;
+ PropMap* map = nobj->shape()->lookup(cx, NameToId(name), &index);
+ if (!map) {
+ return false;
+ }
+
+ PropertyInfo prop = map->getPropertyInfo(index);
+ if (!prop.isDataProperty() || !prop.writable()) {
+ return false;
+ }
+
+ nobj->setSlot(prop.slot(), *val);
+ return true;
+}
+
+bool ObjectHasGetterSetterPure(JSContext* cx, JSObject* objArg, jsid id,
+ GetterSetter* getterSetter) {
+ AutoUnsafeCallWithABI unsafe;
+
+ // Window objects may require outerizing (passing the WindowProxy to the
+ // getter/setter), so we don't support them here.
+ if (MOZ_UNLIKELY(!objArg->is<NativeObject>() || IsWindow(objArg))) {
+ return false;
+ }
+
+ NativeObject* nobj = &objArg->as<NativeObject>();
+
+ while (true) {
+ uint32_t index;
+ if (PropMap* map = nobj->shape()->lookup(cx, id, &index)) {
+ PropertyInfo prop = map->getPropertyInfo(index);
+ if (!prop.isAccessorProperty()) {
+ return false;
+ }
+ GetterSetter* actualGetterSetter = nobj->getGetterSetter(prop);
+ if (actualGetterSetter == getterSetter) {
+ return true;
+ }
+ return (actualGetterSetter->getter() == getterSetter->getter() &&
+ actualGetterSetter->setter() == getterSetter->setter());
+ }
+
+ // Property not found. Watch out for Class hooks.
+ if (!nobj->is<PlainObject>()) {
+ if (ClassMayResolveId(cx->names(), nobj->getClass(), id, nobj)) {
+ return false;
+ }
+ }
+
+ JSObject* proto = nobj->staticPrototype();
+ if (!proto) {
+ return false;
+ }
+
+ if (!proto->is<NativeObject>()) {
+ return false;
+ }
+ nobj = &proto->as<NativeObject>();
+ }
+}
+
+template <bool HasOwn>
+bool HasNativeDataPropertyPure(JSContext* cx, JSObject* obj, Value* vp) {
+ AutoUnsafeCallWithABI unsafe;
+
+ // vp[0] contains the id, result will be stored in vp[1].
+ Value idVal = vp[0];
+ jsid id;
+ if (!ValueToAtomOrSymbolPure(cx, idVal, &id)) {
+ return false;
+ }
+
+ Shape* receiverShape = obj->shape();
+ MegamorphicCache& cache = cx->caches().megamorphicCache;
+ MegamorphicCache::Entry* entry;
+ if (JitOptions.enableWatchtowerMegamorphic &&
+ cache.lookup(receiverShape, id, &entry)) {
+ VerifyCacheEntry(cx, &obj->as<NativeObject>(), id, *entry);
+ if (entry->isDataProperty()) {
+ vp[1].setBoolean(HasOwn ? entry->numHops() == 0 : true);
+ return true;
+ }
+ if (HasOwn || entry->isMissingProperty()) {
+ vp[1].setBoolean(false);
+ return true;
+ }
+ MOZ_ASSERT(!HasOwn);
+ MOZ_ASSERT(entry->isMissingOwnProperty());
+ }
+
+ size_t numHops = 0;
+ do {
+ if (MOZ_UNLIKELY(!obj->is<NativeObject>())) {
+ return false;
+ }
+
+ MOZ_ASSERT(!obj->getOpsLookupProperty());
+
+ uint32_t index;
+ if (PropMap* map =
+ obj->as<NativeObject>().shape()->lookup(cx, id, &index)) {
+ if (JitOptions.enableWatchtowerMegamorphic) {
+ PropertyInfo prop = map->getPropertyInfo(index);
+ if (prop.isDataProperty()) {
+ cache.initEntryForDataProperty(entry, receiverShape, id, numHops,
+ prop.slot());
+ }
+ }
+ vp[1].setBoolean(true);
+ return true;
+ }
+
+ // Property not found. Watch out for Class hooks and TypedArrays.
+ if (MOZ_UNLIKELY(!obj->is<PlainObject>())) {
+ // Fail if there's a resolve hook, unless the mayResolve hook tells us
+ // the resolve hook won't define a property with this id.
+ if (ClassMayResolveId(cx->names(), obj->getClass(), id, obj)) {
+ return false;
+ }
+
+ // Don't skip past TypedArrayObjects if the id can be a TypedArray
+ // index.
+ if (obj->is<TypedArrayObject>()) {
+ if (MaybeTypedArrayIndexString(id)) {
+ return false;
+ }
+ }
+ }
+
+ // If implementing Object.hasOwnProperty, don't follow protochain.
+ if constexpr (HasOwn) {
+ break;
+ }
+
+ // Get prototype. Objects that may allow dynamic prototypes are already
+ // filtered out above.
+ obj = obj->staticPrototype();
+ numHops++;
+ } while (obj);
+
+ // Missing property.
+ if (JitOptions.enableWatchtowerMegamorphic) {
+ if constexpr (HasOwn) {
+ cache.initEntryForMissingOwnProperty(entry, receiverShape, id);
+ } else {
+ cache.initEntryForMissingProperty(entry, receiverShape, id);
+ }
+ }
+ vp[1].setBoolean(false);
+ return true;
+}
+
+template bool HasNativeDataPropertyPure<true>(JSContext* cx, JSObject* obj,
+ Value* vp);
+
+template bool HasNativeDataPropertyPure<false>(JSContext* cx, JSObject* obj,
+ Value* vp);
+
+bool HasNativeElementPure(JSContext* cx, NativeObject* obj, int32_t index,
+ Value* vp) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(obj->is<NativeObject>());
+ MOZ_ASSERT(!obj->getOpsHasProperty());
+ MOZ_ASSERT(!obj->getOpsLookupProperty());
+ MOZ_ASSERT(!obj->getOpsGetOwnPropertyDescriptor());
+
+ if (MOZ_UNLIKELY(index < 0)) {
+ return false;
+ }
+
+ if (obj->containsDenseElement(index)) {
+ vp[0].setBoolean(true);
+ return true;
+ }
+
+ jsid id = PropertyKey::Int(index);
+ uint32_t unused;
+ if (obj->shape()->lookup(cx, id, &unused)) {
+ vp[0].setBoolean(true);
+ return true;
+ }
+
+ // Fail if there's a resolve hook, unless the mayResolve hook tells
+ // us the resolve hook won't define a property with this id.
+ if (MOZ_UNLIKELY(ClassMayResolveId(cx->names(), obj->getClass(), id, obj))) {
+ return false;
+ }
+ // TypedArrayObject are also native and contain indexed properties.
+ if (MOZ_UNLIKELY(obj->is<TypedArrayObject>())) {
+ size_t length = obj->as<TypedArrayObject>().length();
+ vp[0].setBoolean(uint32_t(index) < length);
+ return true;
+ }
+
+ vp[0].setBoolean(false);
+ return true;
+}
+
+// Fast path for setting/adding a plain object property. This is the common case
+// for megamorphic SetProp/SetElem.
+static bool TryAddOrSetPlainObjectProperty(JSContext* cx,
+ Handle<PlainObject*> obj,
+ HandleValue keyVal,
+ HandleValue value, bool* optimized) {
+ MOZ_ASSERT(!*optimized);
+
+ // The key must be a string or symbol so that we don't have to handle dense
+ // elements here.
+ PropertyKey key;
+ if (!ValueToAtomOrSymbolPure(cx, keyVal, &key)) {
+ return true;
+ }
+
+ // Fast path for changing a data property.
+ uint32_t index;
+ if (PropMap* map = obj->shape()->lookup(cx, key, &index)) {
+ PropertyInfo prop = map->getPropertyInfo(index);
+ if (!prop.isDataProperty() || !prop.writable()) {
+ return true;
+ }
+ obj->setSlot(prop.slot(), value);
+ *optimized = true;
+ return true;
+ }
+
+ // Don't support "__proto__". This lets us take advantage of the
+ // hasNonWritableOrAccessorPropExclProto optimization below.
+ if (MOZ_UNLIKELY(!obj->isExtensible() || key.isAtom(cx->names().proto))) {
+ return true;
+ }
+
+ // Ensure the proto chain contains only plain objects. Deoptimize for accessor
+ // properties and non-writable data properties (we can't shadow non-writable
+ // properties).
+ JSObject* proto = obj->staticPrototype();
+ while (proto) {
+ if (!proto->is<PlainObject>()) {
+ return true;
+ }
+ PlainObject* plainProto = &proto->as<PlainObject>();
+ if (plainProto->hasNonWritableOrAccessorPropExclProto()) {
+ uint32_t index;
+ if (PropMap* map = plainProto->shape()->lookup(cx, key, &index)) {
+ PropertyInfo prop = map->getPropertyInfo(index);
+ if (!prop.isDataProperty() || !prop.writable()) {
+ return true;
+ }
+ break;
+ }
+ }
+ proto = plainProto->staticPrototype();
+ }
+
+#ifdef DEBUG
+ // At this point either the property is missing or it's a writable data
+ // property on the proto chain that we can shadow.
+ {
+ NativeObject* holder = nullptr;
+ PropertyResult prop;
+ MOZ_ASSERT(LookupPropertyPure(cx, obj, key, &holder, &prop));
+ MOZ_ASSERT(obj != holder);
+ MOZ_ASSERT_IF(prop.isFound(), prop.isNativeProperty() &&
+ prop.propertyInfo().isDataProperty() &&
+ prop.propertyInfo().writable());
+ }
+#endif
+
+ *optimized = true;
+ Rooted<PropertyKey> keyRoot(cx, key);
+ return AddDataPropertyToPlainObject(cx, obj, keyRoot, value);
+}
+
+bool SetElementMegamorphic(JSContext* cx, HandleObject obj, HandleValue index,
+ HandleValue value, HandleValue receiver,
+ bool strict) {
+ if (obj->is<PlainObject>()) {
+ bool optimized = false;
+ if (!TryAddOrSetPlainObjectProperty(cx, obj.as<PlainObject>(), index, value,
+ &optimized)) {
+ return false;
+ }
+ if (optimized) {
+ return true;
+ }
+ }
+ return SetObjectElementWithReceiver(cx, obj, index, value, receiver, strict);
+}
+
+void HandleCodeCoverageAtPC(BaselineFrame* frame, jsbytecode* pc) {
+ AutoUnsafeCallWithABI unsafe(UnsafeABIStrictness::AllowPendingExceptions);
+
+ MOZ_ASSERT(frame->runningInInterpreter());
+
+ JSScript* script = frame->script();
+ MOZ_ASSERT(pc == script->main() || BytecodeIsJumpTarget(JSOp(*pc)));
+
+ if (!script->hasScriptCounts()) {
+ if (!script->realm()->collectCoverageForDebug()) {
+ return;
+ }
+ JSContext* cx = script->runtimeFromMainThread()->mainContextFromOwnThread();
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!script->initScriptCounts(cx)) {
+ oomUnsafe.crash("initScriptCounts");
+ }
+ }
+
+ PCCounts* counts = script->maybeGetPCCounts(pc);
+ MOZ_ASSERT(counts);
+ counts->numExec()++;
+}
+
+void HandleCodeCoverageAtPrologue(BaselineFrame* frame) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(frame->runningInInterpreter());
+
+ JSScript* script = frame->script();
+ jsbytecode* main = script->main();
+ if (!BytecodeIsJumpTarget(JSOp(*main))) {
+ HandleCodeCoverageAtPC(frame, main);
+ }
+}
+
+JSString* TypeOfNameObject(JSObject* obj, JSRuntime* rt) {
+ AutoUnsafeCallWithABI unsafe;
+ JSType type = js::TypeOfObject(obj);
+ return TypeName(type, *rt->commonNames);
+}
+
+bool GetPrototypeOf(JSContext* cx, HandleObject target,
+ MutableHandleValue rval) {
+ MOZ_ASSERT(target->hasDynamicPrototype());
+
+ RootedObject proto(cx);
+ if (!GetPrototype(cx, target, &proto)) {
+ return false;
+ }
+ rval.setObjectOrNull(proto);
+ return true;
+}
+
+static JSString* ConvertObjectToStringForConcat(JSContext* cx,
+ HandleValue obj) {
+ MOZ_ASSERT(obj.isObject());
+ RootedValue rootedObj(cx, obj);
+ if (!ToPrimitive(cx, &rootedObj)) {
+ return nullptr;
+ }
+ return ToString<CanGC>(cx, rootedObj);
+}
+
+bool DoConcatStringObject(JSContext* cx, HandleValue lhs, HandleValue rhs,
+ MutableHandleValue res) {
+ JSString* lstr = nullptr;
+ JSString* rstr = nullptr;
+
+ if (lhs.isString()) {
+ // Convert rhs first.
+ MOZ_ASSERT(lhs.isString() && rhs.isObject());
+ rstr = ConvertObjectToStringForConcat(cx, rhs);
+ if (!rstr) {
+ return false;
+ }
+
+ // lhs is already string.
+ lstr = lhs.toString();
+ } else {
+ MOZ_ASSERT(rhs.isString() && lhs.isObject());
+ // Convert lhs first.
+ lstr = ConvertObjectToStringForConcat(cx, lhs);
+ if (!lstr) {
+ return false;
+ }
+
+ // rhs is already string.
+ rstr = rhs.toString();
+ }
+
+ JSString* str = ConcatStrings<NoGC>(cx, lstr, rstr);
+ if (!str) {
+ RootedString nlstr(cx, lstr), nrstr(cx, rstr);
+ str = ConcatStrings<CanGC>(cx, nlstr, nrstr);
+ if (!str) {
+ return false;
+ }
+ }
+
+ res.setString(str);
+ return true;
+}
+
+bool IsPossiblyWrappedTypedArray(JSContext* cx, JSObject* obj, bool* result) {
+ JSObject* unwrapped = CheckedUnwrapDynamic(obj, cx);
+ if (!unwrapped) {
+ ReportAccessDenied(cx);
+ return false;
+ }
+
+ *result = unwrapped->is<TypedArrayObject>();
+ return true;
+}
+
+// Called from CreateDependentString::generateFallback.
+void* AllocateDependentString(JSContext* cx) {
+ AutoUnsafeCallWithABI unsafe;
+ return cx->newCell<JSDependentString, NoGC>(js::gc::DefaultHeap);
+}
+void* AllocateFatInlineString(JSContext* cx) {
+ AutoUnsafeCallWithABI unsafe;
+ return cx->newCell<JSFatInlineString, NoGC>(js::gc::DefaultHeap);
+}
+
+// Called to allocate a BigInt if inline allocation failed.
+void* AllocateBigIntNoGC(JSContext* cx, bool requestMinorGC) {
+ AutoUnsafeCallWithABI unsafe;
+
+ if (requestMinorGC) {
+ cx->nursery().requestMinorGC(JS::GCReason::OUT_OF_NURSERY);
+ }
+
+ return cx->newCell<JS::BigInt, NoGC>(js::gc::TenuredHeap);
+}
+
+void AllocateAndInitTypedArrayBuffer(JSContext* cx, TypedArrayObject* obj,
+ int32_t count) {
+ AutoUnsafeCallWithABI unsafe;
+
+ // Initialize the data slot to UndefinedValue to signal to our JIT caller that
+ // the allocation failed if the slot isn't overwritten below.
+ obj->initFixedSlot(TypedArrayObject::DATA_SLOT, UndefinedValue());
+
+ // Negative numbers or zero will bail out to the slow path, which in turn will
+ // raise an invalid argument exception or create a correct object with zero
+ // elements.
+ constexpr size_t maxByteLength = TypedArrayObject::MaxByteLength;
+ if (count <= 0 || size_t(count) > maxByteLength / obj->bytesPerElement()) {
+ obj->setFixedSlot(TypedArrayObject::LENGTH_SLOT, PrivateValue(size_t(0)));
+ return;
+ }
+
+ obj->setFixedSlot(TypedArrayObject::LENGTH_SLOT, PrivateValue(count));
+
+ size_t nbytes = size_t(count) * obj->bytesPerElement();
+ MOZ_ASSERT(nbytes <= maxByteLength);
+ nbytes = RoundUp(nbytes, sizeof(Value));
+
+ void* buf = cx->nursery().allocateZeroedBuffer(obj, nbytes,
+ js::ArrayBufferContentsArena);
+ if (buf) {
+ InitReservedSlot(obj, TypedArrayObject::DATA_SLOT, buf, nbytes,
+ MemoryUse::TypedArrayElements);
+ }
+}
+
+void* CreateMatchResultFallbackFunc(JSContext* cx, gc::AllocKind kind,
+ size_t nDynamicSlots) {
+ AutoUnsafeCallWithABI unsafe;
+ return cx->newCell<ArrayObject, NoGC>(kind, nDynamicSlots, gc::DefaultHeap,
+ &ArrayObject::class_);
+}
+
+#ifdef JS_GC_PROBES
+void TraceCreateObject(JSObject* obj) {
+ AutoUnsafeCallWithABI unsafe;
+ js::gc::gcprobes::CreateObject(obj);
+}
+#endif
+
+#if JS_BITS_PER_WORD == 32
+BigInt* CreateBigIntFromInt64(JSContext* cx, uint32_t low, uint32_t high) {
+ uint64_t n = (static_cast<uint64_t>(high) << 32) + low;
+ return js::BigInt::createFromInt64(cx, n);
+}
+
+BigInt* CreateBigIntFromUint64(JSContext* cx, uint32_t low, uint32_t high) {
+ uint64_t n = (static_cast<uint64_t>(high) << 32) + low;
+ return js::BigInt::createFromUint64(cx, n);
+}
+#else
+BigInt* CreateBigIntFromInt64(JSContext* cx, uint64_t i64) {
+ return js::BigInt::createFromInt64(cx, i64);
+}
+
+BigInt* CreateBigIntFromUint64(JSContext* cx, uint64_t i64) {
+ return js::BigInt::createFromUint64(cx, i64);
+}
+#endif
+
+bool DoStringToInt64(JSContext* cx, HandleString str, uint64_t* res) {
+ BigInt* bi;
+ JS_TRY_VAR_OR_RETURN_FALSE(cx, bi, js::StringToBigInt(cx, str));
+
+ if (!bi) {
+ JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr,
+ JSMSG_BIGINT_INVALID_SYNTAX);
+ return false;
+ }
+
+ *res = js::BigInt::toUint64(bi);
+ return true;
+}
+
+template <EqualityKind Kind>
+bool BigIntEqual(BigInt* x, BigInt* y) {
+ AutoUnsafeCallWithABI unsafe;
+ bool res = BigInt::equal(x, y);
+ if (Kind != EqualityKind::Equal) {
+ res = !res;
+ }
+ return res;
+}
+
+template bool BigIntEqual<EqualityKind::Equal>(BigInt* x, BigInt* y);
+template bool BigIntEqual<EqualityKind::NotEqual>(BigInt* x, BigInt* y);
+
+template <ComparisonKind Kind>
+bool BigIntCompare(BigInt* x, BigInt* y) {
+ AutoUnsafeCallWithABI unsafe;
+ bool res = BigInt::lessThan(x, y);
+ if (Kind != ComparisonKind::LessThan) {
+ res = !res;
+ }
+ return res;
+}
+
+template bool BigIntCompare<ComparisonKind::LessThan>(BigInt* x, BigInt* y);
+template bool BigIntCompare<ComparisonKind::GreaterThanOrEqual>(BigInt* x,
+ BigInt* y);
+
+template <EqualityKind Kind>
+bool BigIntNumberEqual(BigInt* x, double y) {
+ AutoUnsafeCallWithABI unsafe;
+ bool res = BigInt::equal(x, y);
+ if (Kind != EqualityKind::Equal) {
+ res = !res;
+ }
+ return res;
+}
+
+template bool BigIntNumberEqual<EqualityKind::Equal>(BigInt* x, double y);
+template bool BigIntNumberEqual<EqualityKind::NotEqual>(BigInt* x, double y);
+
+template <ComparisonKind Kind>
+bool BigIntNumberCompare(BigInt* x, double y) {
+ AutoUnsafeCallWithABI unsafe;
+ mozilla::Maybe<bool> res = BigInt::lessThan(x, y);
+ if (Kind == ComparisonKind::LessThan) {
+ return res.valueOr(false);
+ }
+ return !res.valueOr(true);
+}
+
+template bool BigIntNumberCompare<ComparisonKind::LessThan>(BigInt* x,
+ double y);
+template bool BigIntNumberCompare<ComparisonKind::GreaterThanOrEqual>(BigInt* x,
+ double y);
+
+template <ComparisonKind Kind>
+bool NumberBigIntCompare(double x, BigInt* y) {
+ AutoUnsafeCallWithABI unsafe;
+ mozilla::Maybe<bool> res = BigInt::lessThan(x, y);
+ if (Kind == ComparisonKind::LessThan) {
+ return res.valueOr(false);
+ }
+ return !res.valueOr(true);
+}
+
+template bool NumberBigIntCompare<ComparisonKind::LessThan>(double x,
+ BigInt* y);
+template bool NumberBigIntCompare<ComparisonKind::GreaterThanOrEqual>(
+ double x, BigInt* y);
+
+template <EqualityKind Kind>
+bool BigIntStringEqual(JSContext* cx, HandleBigInt x, HandleString y,
+ bool* res) {
+ JS_TRY_VAR_OR_RETURN_FALSE(cx, *res, BigInt::equal(cx, x, y));
+ if (Kind != EqualityKind::Equal) {
+ *res = !*res;
+ }
+ return true;
+}
+
+template bool BigIntStringEqual<EqualityKind::Equal>(JSContext* cx,
+ HandleBigInt x,
+ HandleString y, bool* res);
+template bool BigIntStringEqual<EqualityKind::NotEqual>(JSContext* cx,
+ HandleBigInt x,
+ HandleString y,
+ bool* res);
+
+template <ComparisonKind Kind>
+bool BigIntStringCompare(JSContext* cx, HandleBigInt x, HandleString y,
+ bool* res) {
+ mozilla::Maybe<bool> result;
+ if (!BigInt::lessThan(cx, x, y, result)) {
+ return false;
+ }
+ if (Kind == ComparisonKind::LessThan) {
+ *res = result.valueOr(false);
+ } else {
+ *res = !result.valueOr(true);
+ }
+ return true;
+}
+
+template bool BigIntStringCompare<ComparisonKind::LessThan>(JSContext* cx,
+ HandleBigInt x,
+ HandleString y,
+ bool* res);
+template bool BigIntStringCompare<ComparisonKind::GreaterThanOrEqual>(
+ JSContext* cx, HandleBigInt x, HandleString y, bool* res);
+
+template <ComparisonKind Kind>
+bool StringBigIntCompare(JSContext* cx, HandleString x, HandleBigInt y,
+ bool* res) {
+ mozilla::Maybe<bool> result;
+ if (!BigInt::lessThan(cx, x, y, result)) {
+ return false;
+ }
+ if (Kind == ComparisonKind::LessThan) {
+ *res = result.valueOr(false);
+ } else {
+ *res = !result.valueOr(true);
+ }
+ return true;
+}
+
+template bool StringBigIntCompare<ComparisonKind::LessThan>(JSContext* cx,
+ HandleString x,
+ HandleBigInt y,
+ bool* res);
+template bool StringBigIntCompare<ComparisonKind::GreaterThanOrEqual>(
+ JSContext* cx, HandleString x, HandleBigInt y, bool* res);
+
+BigInt* BigIntAsIntN(JSContext* cx, HandleBigInt x, int32_t bits) {
+ MOZ_ASSERT(bits >= 0);
+ return BigInt::asIntN(cx, x, uint64_t(bits));
+}
+
+BigInt* BigIntAsUintN(JSContext* cx, HandleBigInt x, int32_t bits) {
+ MOZ_ASSERT(bits >= 0);
+ return BigInt::asUintN(cx, x, uint64_t(bits));
+}
+
+template <typename T>
+static int32_t AtomicsCompareExchange(TypedArrayObject* typedArray,
+ size_t index, int32_t expected,
+ int32_t replacement) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ SharedMem<T*> addr = typedArray->dataPointerEither().cast<T*>();
+ return jit::AtomicOperations::compareExchangeSeqCst(addr + index, T(expected),
+ T(replacement));
+}
+
+AtomicsCompareExchangeFn AtomicsCompareExchange(Scalar::Type elementType) {
+ switch (elementType) {
+ case Scalar::Int8:
+ return AtomicsCompareExchange<int8_t>;
+ case Scalar::Uint8:
+ return AtomicsCompareExchange<uint8_t>;
+ case Scalar::Int16:
+ return AtomicsCompareExchange<int16_t>;
+ case Scalar::Uint16:
+ return AtomicsCompareExchange<uint16_t>;
+ case Scalar::Int32:
+ return AtomicsCompareExchange<int32_t>;
+ case Scalar::Uint32:
+ return AtomicsCompareExchange<uint32_t>;
+ default:
+ MOZ_CRASH("Unexpected TypedArray type");
+ }
+}
+
+template <typename T>
+static int32_t AtomicsExchange(TypedArrayObject* typedArray, size_t index,
+ int32_t value) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ SharedMem<T*> addr = typedArray->dataPointerEither().cast<T*>();
+ return jit::AtomicOperations::exchangeSeqCst(addr + index, T(value));
+}
+
+AtomicsReadWriteModifyFn AtomicsExchange(Scalar::Type elementType) {
+ switch (elementType) {
+ case Scalar::Int8:
+ return AtomicsExchange<int8_t>;
+ case Scalar::Uint8:
+ return AtomicsExchange<uint8_t>;
+ case Scalar::Int16:
+ return AtomicsExchange<int16_t>;
+ case Scalar::Uint16:
+ return AtomicsExchange<uint16_t>;
+ case Scalar::Int32:
+ return AtomicsExchange<int32_t>;
+ case Scalar::Uint32:
+ return AtomicsExchange<uint32_t>;
+ default:
+ MOZ_CRASH("Unexpected TypedArray type");
+ }
+}
+
+template <typename T>
+static int32_t AtomicsAdd(TypedArrayObject* typedArray, size_t index,
+ int32_t value) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ SharedMem<T*> addr = typedArray->dataPointerEither().cast<T*>();
+ return jit::AtomicOperations::fetchAddSeqCst(addr + index, T(value));
+}
+
+AtomicsReadWriteModifyFn AtomicsAdd(Scalar::Type elementType) {
+ switch (elementType) {
+ case Scalar::Int8:
+ return AtomicsAdd<int8_t>;
+ case Scalar::Uint8:
+ return AtomicsAdd<uint8_t>;
+ case Scalar::Int16:
+ return AtomicsAdd<int16_t>;
+ case Scalar::Uint16:
+ return AtomicsAdd<uint16_t>;
+ case Scalar::Int32:
+ return AtomicsAdd<int32_t>;
+ case Scalar::Uint32:
+ return AtomicsAdd<uint32_t>;
+ default:
+ MOZ_CRASH("Unexpected TypedArray type");
+ }
+}
+
+template <typename T>
+static int32_t AtomicsSub(TypedArrayObject* typedArray, size_t index,
+ int32_t value) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ SharedMem<T*> addr = typedArray->dataPointerEither().cast<T*>();
+ return jit::AtomicOperations::fetchSubSeqCst(addr + index, T(value));
+}
+
+AtomicsReadWriteModifyFn AtomicsSub(Scalar::Type elementType) {
+ switch (elementType) {
+ case Scalar::Int8:
+ return AtomicsSub<int8_t>;
+ case Scalar::Uint8:
+ return AtomicsSub<uint8_t>;
+ case Scalar::Int16:
+ return AtomicsSub<int16_t>;
+ case Scalar::Uint16:
+ return AtomicsSub<uint16_t>;
+ case Scalar::Int32:
+ return AtomicsSub<int32_t>;
+ case Scalar::Uint32:
+ return AtomicsSub<uint32_t>;
+ default:
+ MOZ_CRASH("Unexpected TypedArray type");
+ }
+}
+
+template <typename T>
+static int32_t AtomicsAnd(TypedArrayObject* typedArray, size_t index,
+ int32_t value) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ SharedMem<T*> addr = typedArray->dataPointerEither().cast<T*>();
+ return jit::AtomicOperations::fetchAndSeqCst(addr + index, T(value));
+}
+
+AtomicsReadWriteModifyFn AtomicsAnd(Scalar::Type elementType) {
+ switch (elementType) {
+ case Scalar::Int8:
+ return AtomicsAnd<int8_t>;
+ case Scalar::Uint8:
+ return AtomicsAnd<uint8_t>;
+ case Scalar::Int16:
+ return AtomicsAnd<int16_t>;
+ case Scalar::Uint16:
+ return AtomicsAnd<uint16_t>;
+ case Scalar::Int32:
+ return AtomicsAnd<int32_t>;
+ case Scalar::Uint32:
+ return AtomicsAnd<uint32_t>;
+ default:
+ MOZ_CRASH("Unexpected TypedArray type");
+ }
+}
+
+template <typename T>
+static int32_t AtomicsOr(TypedArrayObject* typedArray, size_t index,
+ int32_t value) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ SharedMem<T*> addr = typedArray->dataPointerEither().cast<T*>();
+ return jit::AtomicOperations::fetchOrSeqCst(addr + index, T(value));
+}
+
+AtomicsReadWriteModifyFn AtomicsOr(Scalar::Type elementType) {
+ switch (elementType) {
+ case Scalar::Int8:
+ return AtomicsOr<int8_t>;
+ case Scalar::Uint8:
+ return AtomicsOr<uint8_t>;
+ case Scalar::Int16:
+ return AtomicsOr<int16_t>;
+ case Scalar::Uint16:
+ return AtomicsOr<uint16_t>;
+ case Scalar::Int32:
+ return AtomicsOr<int32_t>;
+ case Scalar::Uint32:
+ return AtomicsOr<uint32_t>;
+ default:
+ MOZ_CRASH("Unexpected TypedArray type");
+ }
+}
+
+template <typename T>
+static int32_t AtomicsXor(TypedArrayObject* typedArray, size_t index,
+ int32_t value) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ SharedMem<T*> addr = typedArray->dataPointerEither().cast<T*>();
+ return jit::AtomicOperations::fetchXorSeqCst(addr + index, T(value));
+}
+
+AtomicsReadWriteModifyFn AtomicsXor(Scalar::Type elementType) {
+ switch (elementType) {
+ case Scalar::Int8:
+ return AtomicsXor<int8_t>;
+ case Scalar::Uint8:
+ return AtomicsXor<uint8_t>;
+ case Scalar::Int16:
+ return AtomicsXor<int16_t>;
+ case Scalar::Uint16:
+ return AtomicsXor<uint16_t>;
+ case Scalar::Int32:
+ return AtomicsXor<int32_t>;
+ case Scalar::Uint32:
+ return AtomicsXor<uint32_t>;
+ default:
+ MOZ_CRASH("Unexpected TypedArray type");
+ }
+}
+
+template <typename AtomicOp, typename... Args>
+static BigInt* AtomicAccess64(JSContext* cx, TypedArrayObject* typedArray,
+ size_t index, AtomicOp op, Args... args) {
+ MOZ_ASSERT(Scalar::isBigIntType(typedArray->type()));
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ if (typedArray->type() == Scalar::BigInt64) {
+ SharedMem<int64_t*> addr = typedArray->dataPointerEither().cast<int64_t*>();
+ int64_t v = op(addr + index, BigInt::toInt64(args)...);
+ return BigInt::createFromInt64(cx, v);
+ }
+
+ SharedMem<uint64_t*> addr = typedArray->dataPointerEither().cast<uint64_t*>();
+ uint64_t v = op(addr + index, BigInt::toUint64(args)...);
+ return BigInt::createFromUint64(cx, v);
+}
+
+template <typename AtomicOp, typename... Args>
+static auto AtomicAccess64(TypedArrayObject* typedArray, size_t index,
+ AtomicOp op, Args... args) {
+ MOZ_ASSERT(Scalar::isBigIntType(typedArray->type()));
+ MOZ_ASSERT(!typedArray->hasDetachedBuffer());
+ MOZ_ASSERT(index < typedArray->length());
+
+ if (typedArray->type() == Scalar::BigInt64) {
+ SharedMem<int64_t*> addr = typedArray->dataPointerEither().cast<int64_t*>();
+ return op(addr + index, BigInt::toInt64(args)...);
+ }
+
+ SharedMem<uint64_t*> addr = typedArray->dataPointerEither().cast<uint64_t*>();
+ return op(addr + index, BigInt::toUint64(args)...);
+}
+
+BigInt* AtomicsLoad64(JSContext* cx, TypedArrayObject* typedArray,
+ size_t index) {
+ return AtomicAccess64(cx, typedArray, index, [](auto addr) {
+ return jit::AtomicOperations::loadSeqCst(addr);
+ });
+}
+
+void AtomicsStore64(TypedArrayObject* typedArray, size_t index,
+ const BigInt* value) {
+ AutoUnsafeCallWithABI unsafe;
+
+ AtomicAccess64(
+ typedArray, index,
+ [](auto addr, auto val) {
+ jit::AtomicOperations::storeSeqCst(addr, val);
+ },
+ value);
+}
+
+BigInt* AtomicsCompareExchange64(JSContext* cx, TypedArrayObject* typedArray,
+ size_t index, const BigInt* expected,
+ const BigInt* replacement) {
+ return AtomicAccess64(
+ cx, typedArray, index,
+ [](auto addr, auto oldval, auto newval) {
+ return jit::AtomicOperations::compareExchangeSeqCst(addr, oldval,
+ newval);
+ },
+ expected, replacement);
+}
+
+BigInt* AtomicsExchange64(JSContext* cx, TypedArrayObject* typedArray,
+ size_t index, const BigInt* value) {
+ return AtomicAccess64(
+ cx, typedArray, index,
+ [](auto addr, auto val) {
+ return jit::AtomicOperations::exchangeSeqCst(addr, val);
+ },
+ value);
+}
+
+BigInt* AtomicsAdd64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value) {
+ return AtomicAccess64(
+ cx, typedArray, index,
+ [](auto addr, auto val) {
+ return jit::AtomicOperations::fetchAddSeqCst(addr, val);
+ },
+ value);
+}
+
+BigInt* AtomicsAnd64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value) {
+ return AtomicAccess64(
+ cx, typedArray, index,
+ [](auto addr, auto val) {
+ return jit::AtomicOperations::fetchAndSeqCst(addr, val);
+ },
+ value);
+}
+
+BigInt* AtomicsOr64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value) {
+ return AtomicAccess64(
+ cx, typedArray, index,
+ [](auto addr, auto val) {
+ return jit::AtomicOperations::fetchOrSeqCst(addr, val);
+ },
+ value);
+}
+
+BigInt* AtomicsSub64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value) {
+ return AtomicAccess64(
+ cx, typedArray, index,
+ [](auto addr, auto val) {
+ return jit::AtomicOperations::fetchSubSeqCst(addr, val);
+ },
+ value);
+}
+
+BigInt* AtomicsXor64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value) {
+ return AtomicAccess64(
+ cx, typedArray, index,
+ [](auto addr, auto val) {
+ return jit::AtomicOperations::fetchXorSeqCst(addr, val);
+ },
+ value);
+}
+
+JSAtom* AtomizeStringNoGC(JSContext* cx, JSString* str) {
+ // IC code calls this directly so we shouldn't GC.
+ AutoUnsafeCallWithABI unsafe;
+
+ JSAtom* atom = AtomizeString(cx, str);
+ if (!atom) {
+ cx->recoverFromOutOfMemory();
+ return nullptr;
+ }
+
+ return atom;
+}
+
+bool SetObjectHas(JSContext* cx, HandleObject obj, HandleValue key,
+ bool* rval) {
+ return SetObject::has(cx, obj, key, rval);
+}
+
+bool MapObjectHas(JSContext* cx, HandleObject obj, HandleValue key,
+ bool* rval) {
+ return MapObject::has(cx, obj, key, rval);
+}
+
+bool MapObjectGet(JSContext* cx, HandleObject obj, HandleValue key,
+ MutableHandleValue rval) {
+ return MapObject::get(cx, obj, key, rval);
+}
+
+#ifdef DEBUG
+template <class OrderedHashTable>
+static mozilla::HashNumber HashValue(JSContext* cx, OrderedHashTable* hashTable,
+ const Value* value) {
+ RootedValue rootedValue(cx, *value);
+ HashableValue hashable;
+ MOZ_ALWAYS_TRUE(hashable.setValue(cx, rootedValue));
+
+ return hashTable->hash(hashable);
+}
+#endif
+
+void AssertSetObjectHash(JSContext* cx, SetObject* obj, const Value* value,
+ mozilla::HashNumber actualHash) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(actualHash == HashValue(cx, obj->getData(), value));
+}
+
+void AssertMapObjectHash(JSContext* cx, MapObject* obj, const Value* value,
+ mozilla::HashNumber actualHash) {
+ AutoUnsafeCallWithABI unsafe;
+
+ MOZ_ASSERT(actualHash == HashValue(cx, obj->getData(), value));
+}
+
+void AssumeUnreachable(const char* output) {
+ MOZ_ReportAssertionFailure(output, __FILE__, __LINE__);
+}
+
+void Printf0(const char* output) {
+ AutoUnsafeCallWithABI unsafe;
+
+ // Use stderr instead of stdout because this is only used for debug
+ // output. stderr is less likely to interfere with the program's normal
+ // output, and it's always unbuffered.
+ fprintf(stderr, "%s", output);
+}
+
+void Printf1(const char* output, uintptr_t value) {
+ AutoUnsafeCallWithABI unsafe;
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ js::UniqueChars line = JS_sprintf_append(nullptr, output, value);
+ if (!line) {
+ oomUnsafe.crash("OOM at masm.printf");
+ }
+ fprintf(stderr, "%s", line.get());
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/VMFunctions.h b/js/src/jit/VMFunctions.h
new file mode 100644
index 0000000000..3acfb2e95e
--- /dev/null
+++ b/js/src/jit/VMFunctions.h
@@ -0,0 +1,704 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_VMFunctions_h
+#define jit_VMFunctions_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/HashFunctions.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jstypes.h"
+#include "NamespaceImports.h"
+
+#include "gc/AllocKind.h"
+#include "js/ScalarType.h"
+#include "js/TypeDecls.h"
+
+class JSJitInfo;
+class JSLinearString;
+
+namespace js {
+
+class AbstractGeneratorObject;
+class ArrayObject;
+class GlobalObject;
+class InterpreterFrame;
+class LexicalScope;
+class ClassBodyScope;
+class MapObject;
+class NativeObject;
+class PlainObject;
+class PropertyName;
+class SetObject;
+class Shape;
+class TypedArrayObject;
+class WithScope;
+
+namespace gc {
+
+struct Cell;
+
+}
+
+namespace jit {
+
+class BaselineFrame;
+class InterpreterStubExitFrameLayout;
+
+enum DataType : uint8_t {
+ Type_Void,
+ Type_Bool,
+ Type_Int32,
+ Type_Double,
+ Type_Pointer,
+ Type_Cell,
+ Type_Value,
+ Type_Handle
+};
+
+enum MaybeTailCall : bool { TailCall, NonTailCall };
+
+// [SMDOC] JIT-to-C++ Function Calls. (callVM)
+//
+// Sometimes it is easier to reuse C++ code by calling VM's functions. Calling a
+// function from the VM can be achieved with the use of callWithABI but this is
+// discouraged when the called functions might trigger exceptions and/or
+// garbage collections which are expecting to walk the stack. VMFunctions and
+// callVM are interfaces provided to handle the exception handling and register
+// the stack end (JITActivation) such that walking the stack is made possible.
+//
+// VMFunctionData is a structure which contains the necessary information needed
+// for generating a trampoline function to make a call (with generateVMWrapper)
+// and to root the arguments of the function (in TraceJitExitFrame).
+// VMFunctionData is created with the VMFunctionDataHelper template, which
+// infers the VMFunctionData fields from the function signature. The rooting and
+// trampoline code is therefore determined by the arguments of a function and
+// their locations in the signature of a function.
+//
+// VM functions all expect a JSContext* as first argument. This argument is
+// implicitly provided by the trampoline code (in generateVMWrapper) and used
+// for creating new objects or reporting errors. If your function does not make
+// use of a JSContext* argument, then you might probably use a callWithABI
+// call.
+//
+// Functions described using the VMFunction system must conform to a simple
+// protocol: the return type must have a special "failure" value (for example,
+// false for bool, or nullptr for Objects). If the function is designed to
+// return a value that does not meet this requirement - such as
+// object-or-nullptr, or an integer, an optional, final outParam can be
+// specified. In this case, the return type must be boolean to indicate
+// failure.
+//
+// JIT Code usage:
+//
+// Different JIT compilers in SpiderMonkey have their own implementations of
+// callVM to call VM functions. However, the general shape of them is that
+// arguments (excluding the JSContext or trailing out-param) are pushed on to
+// the stack from right to left (rightmost argument is pushed first).
+//
+// Regardless of return value protocol being used (final outParam, or return
+// value) the generated trampolines ensure the return value ends up in
+// JSReturnOperand, ReturnReg or ReturnDoubleReg.
+//
+// Example:
+//
+// The details will differ slightly between the different compilers in
+// SpiderMonkey, but the general shape of our usage looks like this:
+//
+// Suppose we have a function Foo:
+//
+// bool Foo(JSContext* cx, HandleObject x, HandleId y,
+// MutableHandleValue z);
+//
+// This function returns true on success, and z is the outparam return value.
+//
+// A VM function wrapper for this can be created by adding an entry to
+// VM_FUNCTION_LIST in VMFunctionList-inl.h:
+//
+// _(Foo, js::Foo)
+//
+// In the compiler code the call would then be issued like this:
+//
+// masm.Push(id);
+// masm.Push(obj);
+//
+// using Fn = bool (*)(JSContext*, HandleObject, HandleId,
+// MutableHandleValue);
+// if (!callVM<Fn, js::Foo>()) {
+// return false;
+// }
+//
+// After this, the result value is in the return value register.
+
+// Data for a VM function. All VMFunctionDatas are stored in a constexpr array.
+struct VMFunctionData {
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ // Informative name of the wrapped function. The name should not be present
+ // in release builds in order to save memory.
+ const char* name_;
+#endif
+
+ // Note: a maximum of seven root types is supported.
+ enum RootType : uint8_t {
+ RootNone = 0,
+ RootObject,
+ RootString,
+ RootId,
+ RootValue,
+ RootCell,
+ RootBigInt
+ };
+
+ // Contains an combination of enumerated types used by the gc for marking
+ // arguments of the VM wrapper.
+ uint64_t argumentRootTypes;
+
+ enum ArgProperties {
+ WordByValue = 0,
+ DoubleByValue = 1,
+ WordByRef = 2,
+ DoubleByRef = 3,
+ // BitMask version.
+ Word = 0,
+ Double = 1,
+ ByRef = 2
+ };
+
+ // Contains properties about the first 16 arguments.
+ uint32_t argumentProperties;
+
+ // Which arguments should be passed in float register on platforms that
+ // have them.
+ uint32_t argumentPassedInFloatRegs;
+
+ // Number of arguments expected, excluding JSContext * as an implicit
+ // first argument and an outparam as a possible implicit final argument.
+ uint8_t explicitArgs;
+
+ // The root type of the out param if outParam == Type_Handle.
+ RootType outParamRootType;
+
+ // The outparam may be any Type_*, and must be the final argument to the
+ // function, if not Void. outParam != Void implies that the return type
+ // has a boolean failure mode.
+ DataType outParam;
+
+ // Type returned by the C function and used by the VMFunction wrapper to
+ // check for failures of the C function. Valid failure/return types are
+ // boolean and object pointers which are asserted inside the VMFunction
+ // constructor. If the C function use an outparam (!= Type_Void), then
+ // the only valid failure/return type is boolean -- object pointers are
+ // pointless because the wrapper will only use it to compare it against
+ // nullptr before discarding its value.
+ DataType returnType;
+
+ // Number of Values the VM wrapper should pop from the stack when it returns.
+ // Used by baseline IC stubs so that they can use tail calls to call the VM
+ // wrapper.
+ uint8_t extraValuesToPop;
+
+ // On some architectures, called functions need to explicitly push their
+ // return address, for a tail call, there is nothing to push, so tail-callness
+ // needs to be known at compile time.
+ MaybeTailCall expectTailCall;
+
+ uint32_t argc() const {
+ // JSContext * + args + (OutParam? *)
+ return 1 + explicitArgc() + ((outParam == Type_Void) ? 0 : 1);
+ }
+
+ DataType failType() const { return returnType; }
+
+ // Whether this function returns anything more than a boolean flag for
+ // failures.
+ bool returnsData() const {
+ return returnType == Type_Cell || outParam != Type_Void;
+ }
+
+ ArgProperties argProperties(uint32_t explicitArg) const {
+ return ArgProperties((argumentProperties >> (2 * explicitArg)) & 3);
+ }
+
+ RootType argRootType(uint32_t explicitArg) const {
+ return RootType((argumentRootTypes >> (3 * explicitArg)) & 7);
+ }
+
+ bool argPassedInFloatReg(uint32_t explicitArg) const {
+ return ((argumentPassedInFloatRegs >> explicitArg) & 1) == 1;
+ }
+
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ const char* name() const { return name_; }
+#endif
+
+ // Return the stack size consumed by explicit arguments.
+ size_t explicitStackSlots() const {
+ size_t stackSlots = explicitArgs;
+
+ // Fetch all double-word flags of explicit arguments.
+ uint32_t n = ((1 << (explicitArgs * 2)) - 1) // = Explicit argument mask.
+ & 0x55555555 // = Mask double-size args.
+ & argumentProperties;
+
+ // Add the number of double-word flags. (expect a few loop
+ // iteration)
+ while (n) {
+ stackSlots++;
+ n &= n - 1;
+ }
+ return stackSlots;
+ }
+
+ // Double-size argument which are passed by value are taking the space
+ // of 2 C arguments. This function is used to compute the number of
+ // argument expected by the C function. This is not the same as
+ // explicitStackSlots because reference to stack slots may take one less
+ // register in the total count.
+ size_t explicitArgc() const {
+ size_t stackSlots = explicitArgs;
+
+ // Fetch all explicit arguments.
+ uint32_t n = ((1 << (explicitArgs * 2)) - 1) // = Explicit argument mask.
+ & argumentProperties;
+
+ // Filter double-size arguments (0x5 = 0b0101) and remove (& ~)
+ // arguments passed by reference (0b1010 >> 1 == 0b0101).
+ n = (n & 0x55555555) & ~(n >> 1);
+
+ // Add the number of double-word transfered by value. (expect a few
+ // loop iteration)
+ while (n) {
+ stackSlots++;
+ n &= n - 1;
+ }
+ return stackSlots;
+ }
+
+ size_t doubleByRefArgs() const {
+ size_t count = 0;
+
+ // Fetch all explicit arguments.
+ uint32_t n = ((1 << (explicitArgs * 2)) - 1) // = Explicit argument mask.
+ & argumentProperties;
+
+ // Filter double-size arguments (0x5 = 0b0101) and take (&) only
+ // arguments passed by reference (0b1010 >> 1 == 0b0101).
+ n = (n & 0x55555555) & (n >> 1);
+
+ // Add the number of double-word transfered by refference. (expect a
+ // few loop iterations)
+ while (n) {
+ count++;
+ n &= n - 1;
+ }
+ return count;
+ }
+
+ constexpr VMFunctionData(const char* name, uint32_t explicitArgs,
+ uint32_t argumentProperties,
+ uint32_t argumentPassedInFloatRegs,
+ uint64_t argRootTypes, DataType outParam,
+ RootType outParamRootType, DataType returnType,
+ uint8_t extraValuesToPop = 0,
+ MaybeTailCall expectTailCall = NonTailCall)
+ :
+#if defined(DEBUG) || defined(JS_JITSPEW)
+ name_(name),
+#endif
+ argumentRootTypes(argRootTypes),
+ argumentProperties(argumentProperties),
+ argumentPassedInFloatRegs(argumentPassedInFloatRegs),
+ explicitArgs(explicitArgs),
+ outParamRootType(outParamRootType),
+ outParam(outParam),
+ returnType(returnType),
+ extraValuesToPop(extraValuesToPop),
+ expectTailCall(expectTailCall) {
+ // Check for valid failure/return type.
+ MOZ_ASSERT_IF(outParam != Type_Void,
+ returnType == Type_Void || returnType == Type_Bool);
+ MOZ_ASSERT(returnType == Type_Void || returnType == Type_Bool ||
+ returnType == Type_Cell);
+ }
+
+ constexpr VMFunctionData(const VMFunctionData& o) = default;
+};
+
+// Extract the last element of a list of types.
+template <typename... ArgTypes>
+struct LastArg;
+
+template <>
+struct LastArg<> {
+ using Type = void;
+};
+
+template <typename HeadType>
+struct LastArg<HeadType> {
+ using Type = HeadType;
+};
+
+template <typename HeadType, typename... TailTypes>
+struct LastArg<HeadType, TailTypes...> {
+ using Type = typename LastArg<TailTypes...>::Type;
+};
+
+[[nodiscard]] bool InvokeFunction(JSContext* cx, HandleObject obj0,
+ bool constructing, bool ignoresReturnValue,
+ uint32_t argc, Value* argv,
+ MutableHandleValue rval);
+
+bool InvokeFromInterpreterStub(JSContext* cx,
+ InterpreterStubExitFrameLayout* frame);
+void* GetContextSensitiveInterpreterStub();
+
+bool CheckOverRecursed(JSContext* cx);
+bool CheckOverRecursedBaseline(JSContext* cx, BaselineFrame* frame);
+
+[[nodiscard]] bool MutatePrototype(JSContext* cx, Handle<PlainObject*> obj,
+ HandleValue value);
+
+enum class EqualityKind : bool { NotEqual, Equal };
+
+template <EqualityKind Kind>
+bool StringsEqual(JSContext* cx, HandleString lhs, HandleString rhs, bool* res);
+
+enum class ComparisonKind : bool { GreaterThanOrEqual, LessThan };
+
+template <ComparisonKind Kind>
+bool StringsCompare(JSContext* cx, HandleString lhs, HandleString rhs,
+ bool* res);
+
+[[nodiscard]] bool ArrayPushDensePure(JSContext* cx, ArrayObject* arr,
+ Value* v);
+JSString* ArrayJoin(JSContext* cx, HandleObject array, HandleString sep);
+[[nodiscard]] bool SetArrayLength(JSContext* cx, HandleObject obj,
+ HandleValue value, bool strict);
+
+[[nodiscard]] bool CharCodeAt(JSContext* cx, HandleString str, int32_t index,
+ uint32_t* code);
+JSLinearString* StringFromCharCode(JSContext* cx, int32_t code);
+JSLinearString* StringFromCharCodeNoGC(JSContext* cx, int32_t code);
+JSString* StringFromCodePoint(JSContext* cx, int32_t codePoint);
+JSLinearString* LinearizeForCharAccessPure(JSString* str);
+JSLinearString* LinearizeForCharAccess(JSContext* cx, JSString* str);
+
+[[nodiscard]] bool SetProperty(JSContext* cx, HandleObject obj,
+ Handle<PropertyName*> name, HandleValue value,
+ bool strict, jsbytecode* pc);
+
+[[nodiscard]] bool InterruptCheck(JSContext* cx);
+
+JSObject* NewStringObject(JSContext* cx, HandleString str);
+
+bool OperatorIn(JSContext* cx, HandleValue key, HandleObject obj, bool* out);
+
+[[nodiscard]] bool GetIntrinsicValue(JSContext* cx, Handle<PropertyName*> name,
+ MutableHandleValue rval);
+
+[[nodiscard]] bool CreateThisFromIC(JSContext* cx, HandleObject callee,
+ HandleObject newTarget,
+ MutableHandleValue rval);
+[[nodiscard]] bool CreateThisFromIon(JSContext* cx, HandleObject callee,
+ HandleObject newTarget,
+ MutableHandleValue rval);
+
+void PostWriteBarrier(JSRuntime* rt, js::gc::Cell* cell);
+void PostGlobalWriteBarrier(JSRuntime* rt, GlobalObject* obj);
+
+enum class IndexInBounds { Yes, Maybe };
+
+template <IndexInBounds InBounds>
+void PostWriteElementBarrier(JSRuntime* rt, JSObject* obj, int32_t index);
+
+// If |str| represents an int32, assign it to |result| and return true.
+// Otherwise return false.
+bool GetInt32FromStringPure(JSContext* cx, JSString* str, int32_t* result);
+
+// If |str| is an index in the range [0, INT32_MAX], return it. If the string
+// is not an index in this range, return -1.
+int32_t GetIndexFromString(JSString* str);
+
+JSObject* WrapObjectPure(JSContext* cx, JSObject* obj);
+
+[[nodiscard]] bool DebugPrologue(JSContext* cx, BaselineFrame* frame);
+[[nodiscard]] bool DebugEpilogue(JSContext* cx, BaselineFrame* frame,
+ const jsbytecode* pc, bool ok);
+[[nodiscard]] bool DebugEpilogueOnBaselineReturn(JSContext* cx,
+ BaselineFrame* frame,
+ const jsbytecode* pc);
+void FrameIsDebuggeeCheck(BaselineFrame* frame);
+
+JSObject* CreateGeneratorFromFrame(JSContext* cx, BaselineFrame* frame);
+JSObject* CreateGenerator(JSContext* cx, HandleFunction, HandleScript,
+ HandleObject, HandleObject);
+
+[[nodiscard]] bool NormalSuspend(JSContext* cx, HandleObject obj,
+ BaselineFrame* frame, uint32_t frameSize,
+ const jsbytecode* pc);
+[[nodiscard]] bool FinalSuspend(JSContext* cx, HandleObject obj,
+ const jsbytecode* pc);
+[[nodiscard]] bool InterpretResume(JSContext* cx, HandleObject obj,
+ Value* stackValues, MutableHandleValue rval);
+[[nodiscard]] bool DebugAfterYield(JSContext* cx, BaselineFrame* frame);
+[[nodiscard]] bool GeneratorThrowOrReturn(
+ JSContext* cx, BaselineFrame* frame,
+ Handle<AbstractGeneratorObject*> genObj, HandleValue arg,
+ int32_t resumeKindArg);
+
+[[nodiscard]] bool GlobalDeclInstantiationFromIon(JSContext* cx,
+ HandleScript script,
+ const jsbytecode* pc);
+[[nodiscard]] bool InitFunctionEnvironmentObjects(JSContext* cx,
+ BaselineFrame* frame);
+
+[[nodiscard]] bool NewArgumentsObject(JSContext* cx, BaselineFrame* frame,
+ MutableHandleValue res);
+
+ArrayObject* NewArrayObjectEnsureDenseInitLength(JSContext* cx, int32_t count);
+
+JSObject* InitRestParameter(JSContext* cx, uint32_t length, Value* rest,
+ HandleObject res);
+
+[[nodiscard]] bool HandleDebugTrap(JSContext* cx, BaselineFrame* frame,
+ const uint8_t* retAddr);
+[[nodiscard]] bool OnDebuggerStatement(JSContext* cx, BaselineFrame* frame);
+[[nodiscard]] bool GlobalHasLiveOnDebuggerStatement(JSContext* cx);
+
+[[nodiscard]] bool EnterWith(JSContext* cx, BaselineFrame* frame,
+ HandleValue val, Handle<WithScope*> templ);
+[[nodiscard]] bool LeaveWith(JSContext* cx, BaselineFrame* frame);
+
+[[nodiscard]] bool PushLexicalEnv(JSContext* cx, BaselineFrame* frame,
+ Handle<LexicalScope*> scope);
+[[nodiscard]] bool PushClassBodyEnv(JSContext* cx, BaselineFrame* frame,
+ Handle<ClassBodyScope*> scope);
+[[nodiscard]] bool DebugLeaveThenPopLexicalEnv(JSContext* cx,
+ BaselineFrame* frame,
+ const jsbytecode* pc);
+[[nodiscard]] bool FreshenLexicalEnv(JSContext* cx, BaselineFrame* frame);
+[[nodiscard]] bool DebugLeaveThenFreshenLexicalEnv(JSContext* cx,
+ BaselineFrame* frame,
+ const jsbytecode* pc);
+[[nodiscard]] bool RecreateLexicalEnv(JSContext* cx, BaselineFrame* frame);
+[[nodiscard]] bool DebugLeaveThenRecreateLexicalEnv(JSContext* cx,
+ BaselineFrame* frame,
+ const jsbytecode* pc);
+[[nodiscard]] bool DebugLeaveLexicalEnv(JSContext* cx, BaselineFrame* frame,
+ const jsbytecode* pc);
+
+[[nodiscard]] bool PushVarEnv(JSContext* cx, BaselineFrame* frame,
+ Handle<Scope*> scope);
+
+[[nodiscard]] bool InitBaselineFrameForOsr(BaselineFrame* frame,
+ InterpreterFrame* interpFrame,
+ uint32_t numStackValues);
+
+JSString* StringReplace(JSContext* cx, HandleString string,
+ HandleString pattern, HandleString repl);
+
+void AssertValidBigIntPtr(JSContext* cx, JS::BigInt* bi);
+void AssertValidObjectPtr(JSContext* cx, JSObject* obj);
+void AssertValidStringPtr(JSContext* cx, JSString* str);
+void AssertValidSymbolPtr(JSContext* cx, JS::Symbol* sym);
+void AssertValidValue(JSContext* cx, Value* v);
+
+void JitValuePreWriteBarrier(JSRuntime* rt, Value* vp);
+void JitStringPreWriteBarrier(JSRuntime* rt, JSString** stringp);
+void JitObjectPreWriteBarrier(JSRuntime* rt, JSObject** objp);
+void JitShapePreWriteBarrier(JSRuntime* rt, Shape** shapep);
+
+bool ObjectIsCallable(JSObject* obj);
+bool ObjectIsConstructor(JSObject* obj);
+
+[[nodiscard]] bool ThrowRuntimeLexicalError(JSContext* cx,
+ unsigned errorNumber);
+
+[[nodiscard]] bool ThrowBadDerivedReturnOrUninitializedThis(JSContext* cx,
+ HandleValue v);
+
+[[nodiscard]] bool BaselineGetFunctionThis(JSContext* cx, BaselineFrame* frame,
+ MutableHandleValue res);
+
+[[nodiscard]] bool CallNativeGetter(JSContext* cx, HandleFunction callee,
+ HandleValue receiver,
+ MutableHandleValue result);
+
+bool CallDOMGetter(JSContext* cx, const JSJitInfo* jitInfo, HandleObject obj,
+ MutableHandleValue result);
+
+bool CallDOMSetter(JSContext* cx, const JSJitInfo* jitInfo, HandleObject obj,
+ HandleValue value);
+
+[[nodiscard]] bool CallNativeSetter(JSContext* cx, HandleFunction callee,
+ HandleObject obj, HandleValue rhs);
+
+[[nodiscard]] bool EqualStringsHelperPure(JSString* str1, JSString* str2);
+
+void HandleCodeCoverageAtPC(BaselineFrame* frame, jsbytecode* pc);
+void HandleCodeCoverageAtPrologue(BaselineFrame* frame);
+
+bool GetNativeDataPropertyPure(JSContext* cx, JSObject* obj, PropertyName* name,
+ Value* vp);
+
+bool GetNativeDataPropertyPureFallback(JSContext* cx, JSObject* obj,
+ PropertyKey id, Value* vp);
+
+bool GetNativeDataPropertyByValuePure(JSContext* cx, JSObject* obj, Value* vp);
+
+template <bool HasOwn>
+bool HasNativeDataPropertyPure(JSContext* cx, JSObject* obj, Value* vp);
+
+bool HasNativeElementPure(JSContext* cx, NativeObject* obj, int32_t index,
+ Value* vp);
+
+bool SetNativeDataPropertyPure(JSContext* cx, JSObject* obj, PropertyName* name,
+ Value* val);
+
+bool ObjectHasGetterSetterPure(JSContext* cx, JSObject* objArg, jsid id,
+ GetterSetter* getterSetter);
+
+bool SetElementMegamorphic(JSContext* cx, HandleObject obj, HandleValue index,
+ HandleValue value, HandleValue receiver,
+ bool strict);
+
+JSString* TypeOfNameObject(JSObject* obj, JSRuntime* rt);
+
+bool GetPrototypeOf(JSContext* cx, HandleObject target,
+ MutableHandleValue rval);
+
+bool DoConcatStringObject(JSContext* cx, HandleValue lhs, HandleValue rhs,
+ MutableHandleValue res);
+
+bool IsPossiblyWrappedTypedArray(JSContext* cx, JSObject* obj, bool* result);
+
+void* AllocateDependentString(JSContext* cx);
+void* AllocateFatInlineString(JSContext* cx);
+void* AllocateBigIntNoGC(JSContext* cx, bool requestMinorGC);
+void AllocateAndInitTypedArrayBuffer(JSContext* cx, TypedArrayObject* obj,
+ int32_t count);
+
+void* CreateMatchResultFallbackFunc(JSContext* cx, gc::AllocKind kind,
+ size_t nDynamicSlots);
+#ifdef JS_GC_PROBES
+void TraceCreateObject(JSObject* obj);
+#endif
+
+bool DoStringToInt64(JSContext* cx, HandleString str, uint64_t* res);
+
+#if JS_BITS_PER_WORD == 32
+BigInt* CreateBigIntFromInt64(JSContext* cx, uint32_t low, uint32_t high);
+BigInt* CreateBigIntFromUint64(JSContext* cx, uint32_t low, uint32_t high);
+#else
+BigInt* CreateBigIntFromInt64(JSContext* cx, uint64_t i64);
+BigInt* CreateBigIntFromUint64(JSContext* cx, uint64_t i64);
+#endif
+
+template <EqualityKind Kind>
+bool BigIntEqual(BigInt* x, BigInt* y);
+
+template <ComparisonKind Kind>
+bool BigIntCompare(BigInt* x, BigInt* y);
+
+template <EqualityKind Kind>
+bool BigIntNumberEqual(BigInt* x, double y);
+
+template <ComparisonKind Kind>
+bool BigIntNumberCompare(BigInt* x, double y);
+
+template <ComparisonKind Kind>
+bool NumberBigIntCompare(double x, BigInt* y);
+
+template <EqualityKind Kind>
+bool BigIntStringEqual(JSContext* cx, HandleBigInt x, HandleString y,
+ bool* res);
+
+template <ComparisonKind Kind>
+bool BigIntStringCompare(JSContext* cx, HandleBigInt x, HandleString y,
+ bool* res);
+
+template <ComparisonKind Kind>
+bool StringBigIntCompare(JSContext* cx, HandleString x, HandleBigInt y,
+ bool* res);
+
+BigInt* BigIntAsIntN(JSContext* cx, HandleBigInt x, int32_t bits);
+BigInt* BigIntAsUintN(JSContext* cx, HandleBigInt x, int32_t bits);
+
+using AtomicsCompareExchangeFn = int32_t (*)(TypedArrayObject*, size_t, int32_t,
+ int32_t);
+
+using AtomicsReadWriteModifyFn = int32_t (*)(TypedArrayObject*, size_t,
+ int32_t);
+
+AtomicsCompareExchangeFn AtomicsCompareExchange(Scalar::Type elementType);
+AtomicsReadWriteModifyFn AtomicsExchange(Scalar::Type elementType);
+AtomicsReadWriteModifyFn AtomicsAdd(Scalar::Type elementType);
+AtomicsReadWriteModifyFn AtomicsSub(Scalar::Type elementType);
+AtomicsReadWriteModifyFn AtomicsAnd(Scalar::Type elementType);
+AtomicsReadWriteModifyFn AtomicsOr(Scalar::Type elementType);
+AtomicsReadWriteModifyFn AtomicsXor(Scalar::Type elementType);
+
+BigInt* AtomicsLoad64(JSContext* cx, TypedArrayObject* typedArray,
+ size_t index);
+
+void AtomicsStore64(TypedArrayObject* typedArray, size_t index,
+ const BigInt* value);
+
+BigInt* AtomicsCompareExchange64(JSContext* cx, TypedArrayObject* typedArray,
+ size_t index, const BigInt* expected,
+ const BigInt* replacement);
+
+BigInt* AtomicsExchange64(JSContext* cx, TypedArrayObject* typedArray,
+ size_t index, const BigInt* value);
+
+BigInt* AtomicsAdd64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value);
+BigInt* AtomicsAnd64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value);
+BigInt* AtomicsOr64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value);
+BigInt* AtomicsSub64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value);
+BigInt* AtomicsXor64(JSContext* cx, TypedArrayObject* typedArray, size_t index,
+ const BigInt* value);
+
+JSAtom* AtomizeStringNoGC(JSContext* cx, JSString* str);
+
+bool SetObjectHas(JSContext* cx, HandleObject obj, HandleValue key, bool* rval);
+bool MapObjectHas(JSContext* cx, HandleObject obj, HandleValue key, bool* rval);
+bool MapObjectGet(JSContext* cx, HandleObject obj, HandleValue key,
+ MutableHandleValue rval);
+
+void AssertSetObjectHash(JSContext* cx, SetObject* obj, const Value* value,
+ mozilla::HashNumber actualHash);
+void AssertMapObjectHash(JSContext* cx, MapObject* obj, const Value* value,
+ mozilla::HashNumber actualHash);
+
+// Functions used when JS_MASM_VERBOSE is enabled.
+void AssumeUnreachable(const char* output);
+void Printf0(const char* output);
+void Printf1(const char* output, uintptr_t value);
+
+enum class TailCallVMFunctionId;
+enum class VMFunctionId;
+
+extern const VMFunctionData& GetVMFunction(VMFunctionId id);
+extern const VMFunctionData& GetVMFunction(TailCallVMFunctionId id);
+
+} // namespace jit
+} // namespace js
+
+#if defined(JS_CODEGEN_ARM)
+extern "C" {
+extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int);
+extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int);
+}
+#endif
+
+#endif /* jit_VMFunctions_h */
diff --git a/js/src/jit/ValueNumbering.cpp b/js/src/jit/ValueNumbering.cpp
new file mode 100644
index 0000000000..9a5e49d9d1
--- /dev/null
+++ b/js/src/jit/ValueNumbering.cpp
@@ -0,0 +1,1335 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/ValueNumbering.h"
+
+#include "jit/IonAnalysis.h"
+#include "jit/JitSpewer.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+using namespace js;
+using namespace js::jit;
+
+/*
+ * [SMDOC] IonMonkey Value Numbering
+ *
+ * Some notes on the main algorithm here:
+ * - The SSA identifier id() is the value number. We do replaceAllUsesWith as
+ * we go, so there's always at most one visible value with a given number.
+ *
+ * - Consequently, the GVN algorithm is effectively pessimistic. This means it
+ * is not as powerful as an optimistic GVN would be, but it is simpler and
+ * faster.
+ *
+ * - We iterate in RPO, so that when visiting a block, we've already optimized
+ * and hashed all values in dominating blocks. With occasional exceptions,
+ * this allows us to do everything in a single pass.
+ *
+ * - When we do use multiple passes, we just re-run the algorithm on the whole
+ * graph instead of doing sparse propagation. This is a tradeoff to keep the
+ * algorithm simpler and lighter on inputs that don't have a lot of
+ * interesting unreachable blocks or degenerate loop induction variables, at
+ * the expense of being slower on inputs that do. The loop for this always
+ * terminates, because it only iterates when code is or will be removed, so
+ * eventually it must stop iterating.
+ *
+ * - Values are not immediately removed from the hash set when they go out of
+ * scope. Instead, we check for dominance after a lookup. If the dominance
+ * check fails, the value is removed.
+ */
+
+HashNumber ValueNumberer::VisibleValues::ValueHasher::hash(Lookup ins) {
+ return ins->valueHash();
+}
+
+// Test whether two MDefinitions are congruent.
+bool ValueNumberer::VisibleValues::ValueHasher::match(Key k, Lookup l) {
+ // If one of the instructions depends on a store, and the other instruction
+ // does not depend on the same store, the instructions are not congruent.
+ if (k->dependency() != l->dependency()) {
+ return false;
+ }
+
+ bool congruent =
+ k->congruentTo(l); // Ask the values themselves what they think.
+#ifdef JS_JITSPEW
+ if (congruent != l->congruentTo(k)) {
+ JitSpew(
+ JitSpew_GVN,
+ " congruentTo relation is not symmetric between %s%u and %s%u!!",
+ k->opName(), k->id(), l->opName(), l->id());
+ }
+#endif
+ return congruent;
+}
+
+void ValueNumberer::VisibleValues::ValueHasher::rekey(Key& k, Key newKey) {
+ k = newKey;
+}
+
+ValueNumberer::VisibleValues::VisibleValues(TempAllocator& alloc)
+ : set_(alloc) {}
+
+// Look up the first entry for |def|.
+ValueNumberer::VisibleValues::Ptr ValueNumberer::VisibleValues::findLeader(
+ const MDefinition* def) const {
+ return set_.lookup(def);
+}
+
+// Look up the first entry for |def|.
+ValueNumberer::VisibleValues::AddPtr
+ValueNumberer::VisibleValues::findLeaderForAdd(MDefinition* def) {
+ return set_.lookupForAdd(def);
+}
+
+// Insert a value into the set.
+bool ValueNumberer::VisibleValues::add(AddPtr p, MDefinition* def) {
+ return set_.add(p, def);
+}
+
+// Insert a value onto the set overwriting any existing entry.
+void ValueNumberer::VisibleValues::overwrite(AddPtr p, MDefinition* def) {
+ set_.replaceKey(p, def);
+}
+
+// |def| will be discarded, so remove it from any sets.
+void ValueNumberer::VisibleValues::forget(const MDefinition* def) {
+ Ptr p = set_.lookup(def);
+ if (p && *p == def) {
+ set_.remove(p);
+ }
+}
+
+// Clear all state.
+void ValueNumberer::VisibleValues::clear() { set_.clear(); }
+
+#ifdef DEBUG
+// Test whether |def| is in the set.
+bool ValueNumberer::VisibleValues::has(const MDefinition* def) const {
+ Ptr p = set_.lookup(def);
+ return p && *p == def;
+}
+#endif
+
+// Call MDefinition::justReplaceAllUsesWith, and add some GVN-specific asserts.
+static void ReplaceAllUsesWith(MDefinition* from, MDefinition* to) {
+ MOZ_ASSERT(from != to, "GVN shouldn't try to replace a value with itself");
+ MOZ_ASSERT(from->type() == to->type(), "Def replacement has different type");
+ MOZ_ASSERT(!to->isDiscarded(),
+ "GVN replaces an instruction by a removed instruction");
+
+ // We don't need the extra setting of ImplicitlyUsed flags that the regular
+ // replaceAllUsesWith does because we do it ourselves.
+ from->justReplaceAllUsesWith(to);
+}
+
+// Test whether |succ| is a successor of |block|.
+static bool HasSuccessor(const MControlInstruction* block,
+ const MBasicBlock* succ) {
+ for (size_t i = 0, e = block->numSuccessors(); i != e; ++i) {
+ if (block->getSuccessor(i) == succ) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// Given a block which has had predecessors removed but is still reachable, test
+// whether the block's new dominator will be closer than its old one and whether
+// it will expose potential optimization opportunities.
+static MBasicBlock* ComputeNewDominator(MBasicBlock* block, MBasicBlock* old) {
+ MBasicBlock* now = block->getPredecessor(0);
+ for (size_t i = 1, e = block->numPredecessors(); i < e; ++i) {
+ MBasicBlock* pred = block->getPredecessor(i);
+ // Note that dominators haven't been recomputed yet, so we have to check
+ // whether now dominates pred, not block.
+ while (!now->dominates(pred)) {
+ MBasicBlock* next = now->immediateDominator();
+ if (next == old) {
+ return old;
+ }
+ if (next == now) {
+ MOZ_ASSERT(block == old,
+ "Non-self-dominating block became self-dominating");
+ return block;
+ }
+ now = next;
+ }
+ }
+ MOZ_ASSERT(old != block || old != now,
+ "Missed self-dominating block staying self-dominating");
+ return now;
+}
+
+// Test for any defs which look potentially interesting to GVN.
+static bool BlockHasInterestingDefs(MBasicBlock* block) {
+ return !block->phisEmpty() || *block->begin() != block->lastIns();
+}
+
+// Walk up the dominator tree from |block| to the root and test for any defs
+// which look potentially interesting to GVN.
+static bool ScanDominatorsForDefs(MBasicBlock* block) {
+ for (MBasicBlock* i = block;;) {
+ if (BlockHasInterestingDefs(block)) {
+ return true;
+ }
+
+ MBasicBlock* immediateDominator = i->immediateDominator();
+ if (immediateDominator == i) {
+ break;
+ }
+ i = immediateDominator;
+ }
+ return false;
+}
+
+// Walk up the dominator tree from |now| to |old| and test for any defs which
+// look potentially interesting to GVN.
+static bool ScanDominatorsForDefs(MBasicBlock* now, MBasicBlock* old) {
+ MOZ_ASSERT(old->dominates(now),
+ "Refined dominator not dominated by old dominator");
+
+ for (MBasicBlock* i = now; i != old; i = i->immediateDominator()) {
+ if (BlockHasInterestingDefs(i)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// Given a block which has had predecessors removed but is still reachable, test
+// whether the block's new dominator will be closer than its old one and whether
+// it will expose potential optimization opportunities.
+static bool IsDominatorRefined(MBasicBlock* block) {
+ MBasicBlock* old = block->immediateDominator();
+ MBasicBlock* now = ComputeNewDominator(block, old);
+
+ // If this block is just a goto and it doesn't dominate its destination,
+ // removing its predecessors won't refine the dominators of anything
+ // interesting.
+ MControlInstruction* control = block->lastIns();
+ if (*block->begin() == control && block->phisEmpty() && control->isGoto() &&
+ !block->dominates(control->toGoto()->target())) {
+ return false;
+ }
+
+ // We've computed block's new dominator. Test whether there are any
+ // newly-dominating definitions which look interesting.
+ if (block == old) {
+ return block != now && ScanDominatorsForDefs(now);
+ }
+ MOZ_ASSERT(block != now, "Non-self-dominating block became self-dominating");
+ return ScanDominatorsForDefs(now, old);
+}
+
+// |def| has just had one of its users release it. If it's now dead, enqueue it
+// for discarding, otherwise just make note of it.
+bool ValueNumberer::handleUseReleased(MDefinition* def,
+ ImplicitUseOption implicitUseOption) {
+ if (IsDiscardable(def)) {
+ values_.forget(def);
+ if (!deadDefs_.append(def)) {
+ return false;
+ }
+ } else {
+ if (implicitUseOption == SetImplicitUse) {
+ def->setImplicitlyUsedUnchecked();
+ }
+ }
+ return true;
+}
+
+// Discard |def| and anything in its use-def subtree which is no longer needed.
+bool ValueNumberer::discardDefsRecursively(MDefinition* def,
+ AllowEffectful allowEffectful) {
+ MOZ_ASSERT(deadDefs_.empty(), "deadDefs_ not cleared");
+
+ return discardDef(def, allowEffectful) && processDeadDefs();
+}
+
+// Assuming |resume| is unreachable, release its operands.
+// It might be nice to integrate this code with prepareForDiscard, however GVN
+// needs it to call handleUseReleased so that it can observe when a definition
+// becomes unused, so it isn't trivial to do.
+bool ValueNumberer::releaseResumePointOperands(MResumePoint* resume) {
+ for (size_t i = 0, e = resume->numOperands(); i < e; ++i) {
+ if (!resume->hasOperand(i)) {
+ continue;
+ }
+ MDefinition* op = resume->getOperand(i);
+ resume->releaseOperand(i);
+
+ // We set the ImplicitlyUsed flag when removing resume point operands,
+ // because even though we may think we're certain that a particular
+ // branch might not be taken, the type information might be incomplete.
+ if (!handleUseReleased(op, SetImplicitUse)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Assuming |phi| is dead, release and remove its operands. If an operand
+// becomes dead, push it to the discard worklist.
+bool ValueNumberer::releaseAndRemovePhiOperands(MPhi* phi) {
+ // MPhi saves operands in a vector so we iterate in reverse.
+ for (int o = phi->numOperands() - 1; o >= 0; --o) {
+ MDefinition* op = phi->getOperand(o);
+ phi->removeOperand(o);
+ if (!handleUseReleased(op, DontSetImplicitUse)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Assuming |def| is dead, release its operands. If an operand becomes dead,
+// push it to the discard worklist.
+bool ValueNumberer::releaseOperands(MDefinition* def) {
+ for (size_t o = 0, e = def->numOperands(); o < e; ++o) {
+ MDefinition* op = def->getOperand(o);
+ def->releaseOperand(o);
+ if (!handleUseReleased(op, DontSetImplicitUse)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Discard |def| and mine its operands for any subsequently dead defs.
+bool ValueNumberer::discardDef(MDefinition* def,
+ AllowEffectful allowEffectful) {
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_GVN, " Discarding %s %s%u",
+ def->block()->isMarked() ? "unreachable" : "dead", def->opName(),
+ def->id());
+#endif
+#ifdef DEBUG
+ MOZ_ASSERT(def != nextDef_, "Invalidating the MDefinition iterator");
+ if (def->block()->isMarked()) {
+ MOZ_ASSERT(!def->hasUses(), "Discarding def that still has uses");
+ } else {
+ MOZ_ASSERT(allowEffectful == AllowEffectful::Yes
+ ? IsDiscardableAllowEffectful(def)
+ : IsDiscardable(def),
+ "Discarding non-discardable definition");
+ MOZ_ASSERT(!values_.has(def), "Discarding a definition still in the set");
+ }
+#endif
+
+ MBasicBlock* block = def->block();
+ if (def->isPhi()) {
+ MPhi* phi = def->toPhi();
+ if (!releaseAndRemovePhiOperands(phi)) {
+ return false;
+ }
+ block->discardPhi(phi);
+ } else {
+ MInstruction* ins = def->toInstruction();
+ if (MResumePoint* resume = ins->resumePoint()) {
+ if (!releaseResumePointOperands(resume)) {
+ return false;
+ }
+ }
+ if (!releaseOperands(ins)) {
+ return false;
+ }
+ block->discardIgnoreOperands(ins);
+ }
+
+ // If that was the last definition in the block, it can be safely removed
+ // from the graph.
+ if (block->phisEmpty() && block->begin() == block->end()) {
+ MOZ_ASSERT(block->isMarked(),
+ "Reachable block lacks at least a control instruction");
+
+ // As a special case, don't remove a block which is a dominator tree
+ // root so that we don't invalidate the iterator in visitGraph. We'll
+ // check for this and remove it later.
+ if (block->immediateDominator() != block) {
+ JitSpew(JitSpew_GVN, " Block block%u is now empty; discarding",
+ block->id());
+ graph_.removeBlock(block);
+ blocksRemoved_ = true;
+ } else {
+ JitSpew(JitSpew_GVN,
+ " Dominator root block%u is now empty; will discard later",
+ block->id());
+ }
+ }
+
+ return true;
+}
+
+// Recursively discard all the defs on the deadDefs_ worklist.
+bool ValueNumberer::processDeadDefs() {
+ MDefinition* nextDef = nextDef_;
+ while (!deadDefs_.empty()) {
+ MDefinition* def = deadDefs_.popCopy();
+
+ // Don't invalidate the MDefinition iterator. This is what we're going
+ // to visit next, so we won't miss anything.
+ if (def == nextDef) {
+ continue;
+ }
+
+ if (!discardDef(def)) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// Test whether |block|, which is a loop header, has any predecessors other than
+// |loopPred|, the loop predecessor, which it doesn't dominate.
+static bool hasNonDominatingPredecessor(MBasicBlock* block,
+ MBasicBlock* loopPred) {
+ MOZ_ASSERT(block->isLoopHeader());
+ MOZ_ASSERT(block->loopPredecessor() == loopPred);
+
+ for (uint32_t i = 0, e = block->numPredecessors(); i < e; ++i) {
+ MBasicBlock* pred = block->getPredecessor(i);
+ if (pred != loopPred && !block->dominates(pred)) {
+ return true;
+ }
+ }
+ return false;
+}
+
+// A loop is about to be made reachable only through an OSR entry into one of
+// its nested loops. Fix everything up.
+bool ValueNumberer::fixupOSROnlyLoop(MBasicBlock* block) {
+ // Create an empty and unreachable(!) block which jumps to |block|. This
+ // allows |block| to remain marked as a loop header, so we don't have to
+ // worry about moving a different block into place as the new loop header,
+ // which is hard, especially if the OSR is into a nested loop. Doing all
+ // that would produce slightly more optimal code, but this is so
+ // extraordinarily rare that it isn't worth the complexity.
+ MBasicBlock* fake = MBasicBlock::NewFakeLoopPredecessor(graph_, block);
+ if (!fake) {
+ return false;
+ }
+ fake->setImmediateDominator(fake);
+ fake->addNumDominated(1);
+ fake->setDomIndex(fake->id());
+
+ JitSpew(JitSpew_GVN, " Created fake block%u", fake->id());
+ hasOSRFixups_ = true;
+ return true;
+}
+
+// Remove the CFG edge between |pred| and |block|, after releasing the phi
+// operands on that edge and discarding any definitions consequently made dead.
+bool ValueNumberer::removePredecessorAndDoDCE(MBasicBlock* block,
+ MBasicBlock* pred,
+ size_t predIndex) {
+ MOZ_ASSERT(
+ !block->isMarked(),
+ "Block marked unreachable should have predecessors removed already");
+
+ // Before removing the predecessor edge, scan the phi operands for that edge
+ // for dead code before they get removed.
+ MOZ_ASSERT(nextDef_ == nullptr);
+ for (MPhiIterator iter(block->phisBegin()), end(block->phisEnd());
+ iter != end;) {
+ MPhi* phi = *iter++;
+ MOZ_ASSERT(!values_.has(phi),
+ "Visited phi in block having predecessor removed");
+ MOZ_ASSERT(!phi->isGuard());
+
+ MDefinition* op = phi->getOperand(predIndex);
+ phi->removeOperand(predIndex);
+
+ nextDef_ = iter != end ? *iter : nullptr;
+ if (!handleUseReleased(op, DontSetImplicitUse) || !processDeadDefs()) {
+ return false;
+ }
+
+ // If |nextDef_| became dead while we had it pinned, advance the
+ // iterator and discard it now.
+ while (nextDef_ && !nextDef_->hasUses() &&
+ !nextDef_->isGuardRangeBailouts()) {
+ phi = nextDef_->toPhi();
+ iter++;
+ nextDef_ = iter != end ? *iter : nullptr;
+ if (!discardDefsRecursively(phi)) {
+ return false;
+ }
+ }
+ }
+ nextDef_ = nullptr;
+
+ block->removePredecessorWithoutPhiOperands(pred, predIndex);
+ return true;
+}
+
+// Remove the CFG edge between |pred| and |block|, and if this makes |block|
+// unreachable, mark it so, and remove the rest of its incoming edges too. And
+// discard any instructions made dead by the entailed release of any phi
+// operands.
+bool ValueNumberer::removePredecessorAndCleanUp(MBasicBlock* block,
+ MBasicBlock* pred) {
+ MOZ_ASSERT(!block->isMarked(),
+ "Removing predecessor on block already marked unreachable");
+
+ // We'll be removing a predecessor, so anything we know about phis in this
+ // block will be wrong.
+ for (MPhiIterator iter(block->phisBegin()), end(block->phisEnd());
+ iter != end; ++iter) {
+ values_.forget(*iter);
+ }
+
+ // If this is a loop header, test whether it will become an unreachable
+ // loop, or whether it needs special OSR-related fixups.
+ bool isUnreachableLoop = false;
+ if (block->isLoopHeader()) {
+ if (block->loopPredecessor() == pred) {
+ if (MOZ_UNLIKELY(hasNonDominatingPredecessor(block, pred))) {
+ JitSpew(JitSpew_GVN,
+ " "
+ "Loop with header block%u is now only reachable through an "
+ "OSR entry into the middle of the loop!!",
+ block->id());
+ } else {
+ // Deleting the entry into the loop makes the loop unreachable.
+ isUnreachableLoop = true;
+ JitSpew(JitSpew_GVN,
+ " "
+ "Loop with header block%u is no longer reachable",
+ block->id());
+ }
+#ifdef JS_JITSPEW
+ } else if (block->hasUniqueBackedge() && block->backedge() == pred) {
+ JitSpew(JitSpew_GVN, " Loop with header block%u is no longer a loop",
+ block->id());
+#endif
+ }
+ }
+
+ // Actually remove the CFG edge.
+ if (!removePredecessorAndDoDCE(block, pred,
+ block->getPredecessorIndex(pred))) {
+ return false;
+ }
+
+ // We've now edited the CFG; check to see if |block| became unreachable.
+ if (block->numPredecessors() == 0 || isUnreachableLoop) {
+ JitSpew(JitSpew_GVN, " Disconnecting block%u", block->id());
+
+ // Remove |block| from its dominator parent's subtree. This is the only
+ // immediately-dominated-block information we need to update, because
+ // everything dominated by this block is about to be swept away.
+ MBasicBlock* parent = block->immediateDominator();
+ if (parent != block) {
+ parent->removeImmediatelyDominatedBlock(block);
+ }
+
+ // Completely disconnect it from the CFG. We do this now rather than
+ // just doing it later when we arrive there in visitUnreachableBlock
+ // so that we don't leave a partially broken loop sitting around. This
+ // also lets visitUnreachableBlock assert that numPredecessors() == 0,
+ // which is a nice invariant.
+ if (block->isLoopHeader()) {
+ block->clearLoopHeader();
+ }
+ for (size_t i = 0, e = block->numPredecessors(); i < e; ++i) {
+ if (!removePredecessorAndDoDCE(block, block->getPredecessor(i), i)) {
+ return false;
+ }
+ }
+
+ // Clear out the resume point operands, as they can hold things that
+ // don't appear to dominate them live.
+ if (MResumePoint* resume = block->entryResumePoint()) {
+ if (!releaseResumePointOperands(resume) || !processDeadDefs()) {
+ return false;
+ }
+ if (MResumePoint* outer = block->outerResumePoint()) {
+ if (!releaseResumePointOperands(outer) || !processDeadDefs()) {
+ return false;
+ }
+ }
+ MOZ_ASSERT(nextDef_ == nullptr);
+ for (MInstructionIterator iter(block->begin()), end(block->end());
+ iter != end;) {
+ MInstruction* ins = *iter++;
+ nextDef_ = iter != end ? *iter : nullptr;
+ if (MResumePoint* resume = ins->resumePoint()) {
+ if (!releaseResumePointOperands(resume) || !processDeadDefs()) {
+ return false;
+ }
+ }
+ }
+ nextDef_ = nullptr;
+ } else {
+#ifdef DEBUG
+ MOZ_ASSERT(block->outerResumePoint() == nullptr,
+ "Outer resume point in block without an entry resume point");
+ for (MInstructionIterator iter(block->begin()), end(block->end());
+ iter != end; ++iter) {
+ MOZ_ASSERT(iter->resumePoint() == nullptr,
+ "Instruction with resume point in block without entry "
+ "resume point");
+ }
+#endif
+ }
+
+ // Use the mark to note that we've already removed all its predecessors,
+ // and we know it's unreachable.
+ block->mark();
+ }
+
+ return true;
+}
+
+// Return a simplified form of |def|, if we can.
+MDefinition* ValueNumberer::simplified(MDefinition* def) const {
+ return def->foldsTo(graph_.alloc());
+}
+
+// If an equivalent and dominating value already exists in the set, return it.
+// Otherwise insert |def| into the set and return it.
+MDefinition* ValueNumberer::leader(MDefinition* def) {
+ // If the value isn't suitable for eliminating, don't bother hashing it. The
+ // convention is that congruentTo returns false for node kinds that wish to
+ // opt out of redundance elimination.
+ // TODO: It'd be nice to clean up that convention (bug 1031406).
+ if (!def->isEffectful() && def->congruentTo(def)) {
+ // Look for a match.
+ VisibleValues::AddPtr p = values_.findLeaderForAdd(def);
+ if (p) {
+ MDefinition* rep = *p;
+ if (!rep->isDiscarded() && rep->block()->dominates(def->block())) {
+ // We found a dominating congruent value.
+ return rep;
+ }
+
+ // The congruent value doesn't dominate. It never will again in this
+ // dominator tree, so overwrite it.
+ values_.overwrite(p, def);
+ } else {
+ // No match. Add a new entry.
+ if (!values_.add(p, def)) {
+ return nullptr;
+ }
+ }
+
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_GVN, " Recording %s%u", def->opName(), def->id());
+#endif
+ }
+
+ return def;
+}
+
+// Test whether |phi| is dominated by a congruent phi.
+bool ValueNumberer::hasLeader(const MPhi* phi,
+ const MBasicBlock* phiBlock) const {
+ if (VisibleValues::Ptr p = values_.findLeader(phi)) {
+ const MDefinition* rep = *p;
+ return rep != phi && rep->block()->dominates(phiBlock);
+ }
+ return false;
+}
+
+// Test whether there are any phis in |header| which are newly optimizable, as a
+// result of optimizations done inside the loop. This is not a sparse approach,
+// but restarting is rare enough in practice. Termination is ensured by
+// discarding the phi triggering the iteration.
+bool ValueNumberer::loopHasOptimizablePhi(MBasicBlock* header) const {
+ // If the header is unreachable, don't bother re-optimizing it.
+ if (header->isMarked()) {
+ return false;
+ }
+
+ // Rescan the phis for any that can be simplified, since they may be reading
+ // values from backedges.
+ for (MPhiIterator iter(header->phisBegin()), end(header->phisEnd());
+ iter != end; ++iter) {
+ MPhi* phi = *iter;
+ MOZ_ASSERT_IF(!phi->hasUses(), !DeadIfUnused(phi));
+
+ if (phi->operandIfRedundant() || hasLeader(phi, header)) {
+ return true; // Phi can be simplified.
+ }
+ }
+ return false;
+}
+
+// Visit |def|.
+bool ValueNumberer::visitDefinition(MDefinition* def) {
+ // Nop does not fit in any of the previous optimization, as its only purpose
+ // is to reduce the register pressure by keeping additional resume
+ // point. Still, there is no need consecutive list of MNop instructions, and
+ // this will slow down every other iteration on the Graph.
+ if (def->isNop()) {
+ MNop* nop = def->toNop();
+ MBasicBlock* block = nop->block();
+
+ // We look backward to know if we can remove the previous Nop, we do not
+ // look forward as we would not benefit from the folding made by GVN.
+ MInstructionReverseIterator iter = ++block->rbegin(nop);
+
+ // This nop is at the beginning of the basic block, just replace the
+ // resume point of the basic block by the one from the resume point.
+ if (iter == block->rend()) {
+ JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id());
+ nop->moveResumePointAsEntry();
+ block->discard(nop);
+ return true;
+ }
+
+ // The previous instruction is also a Nop, no need to keep it anymore.
+ MInstruction* prev = *iter;
+ if (prev->isNop()) {
+ JitSpew(JitSpew_GVN, " Removing Nop%u", prev->id());
+ block->discard(prev);
+ return true;
+ }
+
+ // The Nop is introduced to capture the result and make sure the operands
+ // are not live anymore when there are no further uses. Though when
+ // all operands are still needed the Nop doesn't decrease the liveness
+ // and can get removed.
+ MResumePoint* rp = nop->resumePoint();
+ if (rp && rp->numOperands() > 0 &&
+ rp->getOperand(rp->numOperands() - 1) == prev &&
+ !nop->block()->lastIns()->isThrow() &&
+ !prev->isAssertRecoveredOnBailout()) {
+ size_t numOperandsLive = 0;
+ for (size_t j = 0; j < prev->numOperands(); j++) {
+ for (size_t i = 0; i < rp->numOperands(); i++) {
+ if (prev->getOperand(j) == rp->getOperand(i)) {
+ numOperandsLive++;
+ break;
+ }
+ }
+ }
+
+ if (numOperandsLive == prev->numOperands()) {
+ JitSpew(JitSpew_GVN, " Removing Nop%u", nop->id());
+ block->discard(nop);
+ }
+ }
+
+ return true;
+ }
+
+ // Skip optimizations on instructions which are recovered on bailout, to
+ // avoid mixing instructions which are recovered on bailouts with
+ // instructions which are not.
+ if (def->isRecoveredOnBailout()) {
+ return true;
+ }
+
+ // If this instruction has a dependency() into an unreachable block, we'll
+ // need to update AliasAnalysis.
+ MDefinition* dep = def->dependency();
+ if (dep != nullptr && (dep->isDiscarded() || dep->block()->isDead())) {
+ JitSpew(JitSpew_GVN, " AliasAnalysis invalidated");
+ if (updateAliasAnalysis_ && !dependenciesBroken_) {
+ // TODO: Recomputing alias-analysis could theoretically expose more
+ // GVN opportunities.
+ JitSpew(JitSpew_GVN, " Will recompute!");
+ dependenciesBroken_ = true;
+ }
+ // Temporarily clear its dependency, to protect foldsTo, which may
+ // wish to use the dependency to do store-to-load forwarding.
+ def->setDependency(def->toInstruction());
+ } else {
+ dep = nullptr;
+ }
+
+ // Look for a simplified form of |def|.
+ MDefinition* sim = simplified(def);
+ if (sim != def) {
+ if (sim == nullptr) {
+ return false;
+ }
+
+ bool isNewInstruction = sim->block() == nullptr;
+
+ // If |sim| doesn't belong to a block, insert it next to |def|.
+ if (isNewInstruction) {
+ // A new |sim| node mustn't be effectful when |def| wasn't effectful.
+ MOZ_ASSERT((def->isEffectful() && sim->isEffectful()) ||
+ !sim->isEffectful());
+
+ // If both instructions are effectful, |sim| must have stolen the resume
+ // point of |def| when it's a new instruction.
+ MOZ_ASSERT_IF(def->isEffectful() && sim->isEffectful(),
+ !def->toInstruction()->resumePoint() &&
+ sim->toInstruction()->resumePoint());
+
+ def->block()->insertAfter(def->toInstruction(), sim->toInstruction());
+ }
+
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_GVN, " Folded %s%u to %s%u", def->opName(), def->id(),
+ sim->opName(), sim->id());
+#endif
+ MOZ_ASSERT(!sim->isDiscarded());
+ ReplaceAllUsesWith(def, sim);
+
+ // The node's foldsTo said |def| can be replaced by |rep|. If |def| is a
+ // guard, then either |rep| is also a guard, or a guard isn't actually
+ // needed, so we can clear |def|'s guard flag and let it be discarded.
+ def->setNotGuardUnchecked();
+
+ if (def->isGuardRangeBailouts()) {
+ sim->setGuardRangeBailoutsUnchecked();
+ }
+
+ if (sim->bailoutKind() == BailoutKind::Unknown) {
+ sim->setBailoutKind(def->bailoutKind());
+ }
+
+ // Discard |def| if it's now unused. Similar to guards, we allow to replace
+ // effectful instructions when the node's foldsTo method said |def| can be
+ // replaced.
+ if (DeadIfUnusedAllowEffectful(def)) {
+ if (!discardDefsRecursively(def, AllowEffectful::Yes)) {
+ return false;
+ }
+
+ // If that ended up discarding |sim|, then we're done here.
+ if (sim->isDiscarded()) {
+ return true;
+ }
+ }
+
+ if (!rerun_ && def->isPhi() && !sim->isPhi()) {
+ rerun_ = true;
+ JitSpew(JitSpew_GVN,
+ " Replacing phi%u may have enabled cascading optimisations; "
+ "will re-run",
+ def->id());
+ }
+
+ // Otherwise, procede to optimize with |sim| in place of |def|.
+ def = sim;
+
+ // If the simplified instruction was already part of the graph, then we
+ // probably already visited and optimized this instruction.
+ if (!isNewInstruction) {
+ return true;
+ }
+ }
+
+ // Now that foldsTo is done, re-enable the original dependency. Even though
+ // it may be pointing into a discarded block, it's still valid for the
+ // purposes of detecting congruent loads.
+ if (dep != nullptr) {
+ def->setDependency(dep);
+ }
+
+ // Look for a dominating def which makes |def| redundant.
+ MDefinition* rep = leader(def);
+ if (rep != def) {
+ if (rep == nullptr) {
+ return false;
+ }
+ if (rep->updateForReplacement(def)) {
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_GVN, " Replacing %s%u with %s%u", def->opName(),
+ def->id(), rep->opName(), rep->id());
+#endif
+ ReplaceAllUsesWith(def, rep);
+
+ // The node's congruentTo said |def| is congruent to |rep|, and it's
+ // dominated by |rep|. If |def| is a guard, it's covered by |rep|,
+ // so we can clear |def|'s guard flag and let it be discarded.
+ def->setNotGuardUnchecked();
+
+ if (DeadIfUnused(def)) {
+ // discardDef should not add anything to the deadDefs, as the
+ // redundant operation should have the same input operands.
+ mozilla::DebugOnly<bool> r = discardDef(def);
+ MOZ_ASSERT(
+ r,
+ "discardDef shouldn't have tried to add anything to the worklist, "
+ "so it shouldn't have failed");
+ MOZ_ASSERT(deadDefs_.empty(),
+ "discardDef shouldn't have added anything to the worklist");
+ }
+ def = rep;
+ }
+ }
+
+ return true;
+}
+
+// Visit the control instruction at the end of |block|.
+bool ValueNumberer::visitControlInstruction(MBasicBlock* block) {
+ // Look for a simplified form of the control instruction.
+ MControlInstruction* control = block->lastIns();
+ MDefinition* rep = simplified(control);
+ if (rep == control) {
+ return true;
+ }
+
+ if (rep == nullptr) {
+ return false;
+ }
+
+ MControlInstruction* newControl = rep->toControlInstruction();
+ MOZ_ASSERT(!newControl->block(),
+ "Control instruction replacement shouldn't already be in a block");
+#ifdef JS_JITSPEW
+ JitSpew(JitSpew_GVN, " Folded control instruction %s%u to %s%u",
+ control->opName(), control->id(), newControl->opName(),
+ graph_.getNumInstructionIds());
+#endif
+
+ // If the simplification removes any CFG edges, update the CFG and remove
+ // any blocks that become dead.
+ size_t oldNumSuccs = control->numSuccessors();
+ size_t newNumSuccs = newControl->numSuccessors();
+ if (newNumSuccs != oldNumSuccs) {
+ MOZ_ASSERT(newNumSuccs < oldNumSuccs,
+ "New control instruction has too many successors");
+ for (size_t i = 0; i != oldNumSuccs; ++i) {
+ MBasicBlock* succ = control->getSuccessor(i);
+ if (HasSuccessor(newControl, succ)) {
+ continue;
+ }
+ if (succ->isMarked()) {
+ continue;
+ }
+ if (!removePredecessorAndCleanUp(succ, block)) {
+ return false;
+ }
+ if (succ->isMarked()) {
+ continue;
+ }
+ if (!rerun_) {
+ if (!remainingBlocks_.append(succ)) {
+ return false;
+ }
+ }
+ }
+ }
+
+ if (!releaseOperands(control)) {
+ return false;
+ }
+ block->discardIgnoreOperands(control);
+ block->end(newControl);
+ if (block->entryResumePoint() && newNumSuccs != oldNumSuccs) {
+ block->flagOperandsOfPrunedBranches(newControl);
+ }
+ return processDeadDefs();
+}
+
+// |block| is unreachable. Mine it for opportunities to delete more dead
+// code, and then discard it.
+bool ValueNumberer::visitUnreachableBlock(MBasicBlock* block) {
+ JitSpew(JitSpew_GVN, " Visiting unreachable block%u%s%s%s", block->id(),
+ block->isLoopHeader() ? " (loop header)" : "",
+ block->isSplitEdge() ? " (split edge)" : "",
+ block->immediateDominator() == block ? " (dominator root)" : "");
+
+ MOZ_ASSERT(block->isMarked(),
+ "Visiting unmarked (and therefore reachable?) block");
+ MOZ_ASSERT(block->numPredecessors() == 0,
+ "Block marked unreachable still has predecessors");
+ MOZ_ASSERT(block != graph_.entryBlock(), "Removing normal entry block");
+ MOZ_ASSERT(block != graph_.osrBlock(), "Removing OSR entry block");
+ MOZ_ASSERT(deadDefs_.empty(), "deadDefs_ not cleared");
+
+ // Disconnect all outgoing CFG edges.
+ for (size_t i = 0, e = block->numSuccessors(); i < e; ++i) {
+ MBasicBlock* succ = block->getSuccessor(i);
+ if (succ->isDead() || succ->isMarked()) {
+ continue;
+ }
+ if (!removePredecessorAndCleanUp(succ, block)) {
+ return false;
+ }
+ if (succ->isMarked()) {
+ continue;
+ }
+ // |succ| is still reachable. Make a note of it so that we can scan
+ // it for interesting dominator tree changes later.
+ if (!rerun_) {
+ if (!remainingBlocks_.append(succ)) {
+ return false;
+ }
+ }
+ }
+
+ // Discard any instructions with no uses. The remaining instructions will be
+ // discarded when their last use is discarded.
+ MOZ_ASSERT(nextDef_ == nullptr);
+ for (MDefinitionIterator iter(block); iter;) {
+ MDefinition* def = *iter++;
+ if (def->hasUses()) {
+ continue;
+ }
+ nextDef_ = iter ? *iter : nullptr;
+ if (!discardDefsRecursively(def)) {
+ return false;
+ }
+ }
+
+ nextDef_ = nullptr;
+ MControlInstruction* control = block->lastIns();
+ return discardDefsRecursively(control);
+}
+
+// Visit all the phis and instructions |block|.
+bool ValueNumberer::visitBlock(MBasicBlock* block) {
+ MOZ_ASSERT(!block->isMarked(), "Blocks marked unreachable during GVN");
+ MOZ_ASSERT(!block->isDead(), "Block to visit is already dead");
+
+ JitSpew(JitSpew_GVN, " Visiting block%u", block->id());
+
+ // Visit the definitions in the block top-down.
+ MOZ_ASSERT(nextDef_ == nullptr);
+ for (MDefinitionIterator iter(block); iter;) {
+ if (!graph_.alloc().ensureBallast()) {
+ return false;
+ }
+ MDefinition* def = *iter++;
+
+ // Remember where our iterator is so that we don't invalidate it.
+ nextDef_ = iter ? *iter : nullptr;
+
+ // If the definition is dead, discard it.
+ if (IsDiscardable(def)) {
+ if (!discardDefsRecursively(def)) {
+ return false;
+ }
+ continue;
+ }
+
+ if (!visitDefinition(def)) {
+ return false;
+ }
+ }
+ nextDef_ = nullptr;
+
+ if (!graph_.alloc().ensureBallast()) {
+ return false;
+ }
+
+ return visitControlInstruction(block);
+}
+
+// Visit all the blocks dominated by dominatorRoot.
+bool ValueNumberer::visitDominatorTree(MBasicBlock* dominatorRoot) {
+ JitSpew(JitSpew_GVN,
+ " Visiting dominator tree (with %" PRIu64
+ " blocks) rooted at block%u%s",
+ uint64_t(dominatorRoot->numDominated()), dominatorRoot->id(),
+ dominatorRoot == graph_.entryBlock() ? " (normal entry block)"
+ : dominatorRoot == graph_.osrBlock() ? " (OSR entry block)"
+ : dominatorRoot->numPredecessors() == 0
+ ? " (odd unreachable block)"
+ : " (merge point from normal entry and OSR entry)");
+ MOZ_ASSERT(dominatorRoot->immediateDominator() == dominatorRoot,
+ "root is not a dominator tree root");
+
+ // Visit all blocks dominated by dominatorRoot, in RPO. This has the nice
+ // property that we'll always visit a block before any block it dominates,
+ // so we can make a single pass through the list and see every full
+ // redundance.
+ size_t numVisited = 0;
+ size_t numDiscarded = 0;
+ for (ReversePostorderIterator iter(graph_.rpoBegin(dominatorRoot));;) {
+ MOZ_ASSERT(iter != graph_.rpoEnd(), "Inconsistent dominator information");
+ MBasicBlock* block = *iter++;
+ // We're only visiting blocks in dominatorRoot's tree right now.
+ if (!dominatorRoot->dominates(block)) {
+ continue;
+ }
+
+ // If this is a loop backedge, remember the header, as we may not be able
+ // to find it after we simplify the block.
+ MBasicBlock* header =
+ block->isLoopBackedge() ? block->loopHeaderOfBackedge() : nullptr;
+
+ if (block->isMarked()) {
+ // This block has become unreachable; handle it specially.
+ if (!visitUnreachableBlock(block)) {
+ return false;
+ }
+ ++numDiscarded;
+ } else {
+ // Visit the block!
+ if (!visitBlock(block)) {
+ return false;
+ }
+ ++numVisited;
+ }
+
+ // If the block is/was a loop backedge, check to see if the block that
+ // is/was its header has optimizable phis, which would want a re-run.
+ if (!rerun_ && header && loopHasOptimizablePhi(header)) {
+ JitSpew(JitSpew_GVN,
+ " Loop phi in block%u can now be optimized; will re-run GVN!",
+ header->id());
+ rerun_ = true;
+ remainingBlocks_.clear();
+ }
+
+ MOZ_ASSERT(numVisited <= dominatorRoot->numDominated() - numDiscarded,
+ "Visited blocks too many times");
+ if (numVisited >= dominatorRoot->numDominated() - numDiscarded) {
+ break;
+ }
+ }
+
+ totalNumVisited_ += numVisited;
+ values_.clear();
+ return true;
+}
+
+// Visit all the blocks in the graph.
+bool ValueNumberer::visitGraph() {
+ // Due to OSR blocks, the set of blocks dominated by a blocks may not be
+ // contiguous in the RPO. Do a separate traversal for each dominator tree
+ // root. There's always the main entry, and sometimes there's an OSR entry,
+ // and then there are the roots formed where the OSR paths merge with the
+ // main entry paths.
+ for (ReversePostorderIterator iter(graph_.rpoBegin());;) {
+ MOZ_ASSERT(iter != graph_.rpoEnd(), "Inconsistent dominator information");
+ MBasicBlock* block = *iter;
+ if (block->immediateDominator() == block) {
+ if (!visitDominatorTree(block)) {
+ return false;
+ }
+
+ // Normally unreachable blocks would be removed by now, but if this
+ // block is a dominator tree root, it has been special-cased and left
+ // in place in order to avoid invalidating our iterator. Now that
+ // we've finished the tree, increment the iterator, and then if it's
+ // marked for removal, remove it.
+ ++iter;
+ if (block->isMarked()) {
+ JitSpew(JitSpew_GVN, " Discarding dominator root block%u",
+ block->id());
+ MOZ_ASSERT(
+ block->begin() == block->end(),
+ "Unreachable dominator tree root has instructions after tree walk");
+ MOZ_ASSERT(block->phisEmpty(),
+ "Unreachable dominator tree root has phis after tree walk");
+ graph_.removeBlock(block);
+ blocksRemoved_ = true;
+ }
+
+ MOZ_ASSERT(totalNumVisited_ <= graph_.numBlocks(),
+ "Visited blocks too many times");
+ if (totalNumVisited_ >= graph_.numBlocks()) {
+ break;
+ }
+ } else {
+ // This block a dominator tree root. Proceed to the next one.
+ ++iter;
+ }
+ }
+ totalNumVisited_ = 0;
+ return true;
+}
+
+bool ValueNumberer::insertOSRFixups() {
+ ReversePostorderIterator end(graph_.end());
+ for (ReversePostorderIterator iter(graph_.begin()); iter != end;) {
+ MBasicBlock* block = *iter++;
+
+ // Only add fixup block above for loops which can be reached from OSR.
+ if (!block->isLoopHeader()) {
+ continue;
+ }
+
+ // If the loop header is not self-dominated, then this loop does not
+ // have to deal with a second entry point, so there is no need to add a
+ // second entry point with a fixup block.
+ if (block->immediateDominator() != block) {
+ continue;
+ }
+
+ if (!fixupOSROnlyLoop(block)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+// OSR fixups serve the purpose of representing the non-OSR entry into a loop
+// when the only real entry is an OSR entry into the middle. However, if the
+// entry into the middle is subsequently folded away, the loop may actually
+// have become unreachable. Mark-and-sweep all blocks to remove all such code.
+bool ValueNumberer::cleanupOSRFixups() {
+ // Mark.
+ Vector<MBasicBlock*, 0, JitAllocPolicy> worklist(graph_.alloc());
+ unsigned numMarked = 2;
+ graph_.entryBlock()->mark();
+ graph_.osrBlock()->mark();
+ if (!worklist.append(graph_.entryBlock()) ||
+ !worklist.append(graph_.osrBlock())) {
+ return false;
+ }
+ while (!worklist.empty()) {
+ MBasicBlock* block = worklist.popCopy();
+ for (size_t i = 0, e = block->numSuccessors(); i != e; ++i) {
+ MBasicBlock* succ = block->getSuccessor(i);
+ if (!succ->isMarked()) {
+ ++numMarked;
+ succ->mark();
+ if (!worklist.append(succ)) {
+ return false;
+ }
+ } else if (succ->isLoopHeader() && succ->loopPredecessor() == block &&
+ succ->numPredecessors() == 3) {
+ // Unmark fixup blocks if the loop predecessor is marked after
+ // the loop header.
+ succ->getPredecessor(1)->unmarkUnchecked();
+ }
+ }
+
+ // OSR fixup blocks are needed if and only if the loop header is
+ // reachable from its backedge (via the OSR block) and not from its
+ // original loop predecessor.
+ //
+ // Thus OSR fixup blocks are removed if the loop header is not
+ // reachable, or if the loop header is reachable from both its backedge
+ // and its original loop predecessor.
+ if (block->isLoopHeader()) {
+ MBasicBlock* maybeFixupBlock = nullptr;
+ if (block->numPredecessors() == 2) {
+ maybeFixupBlock = block->getPredecessor(0);
+ } else {
+ MOZ_ASSERT(block->numPredecessors() == 3);
+ if (!block->loopPredecessor()->isMarked()) {
+ maybeFixupBlock = block->getPredecessor(1);
+ }
+ }
+
+ if (maybeFixupBlock && !maybeFixupBlock->isMarked() &&
+ maybeFixupBlock->numPredecessors() == 0) {
+ MOZ_ASSERT(maybeFixupBlock->numSuccessors() == 1,
+ "OSR fixup block should have exactly one successor");
+ MOZ_ASSERT(maybeFixupBlock != graph_.entryBlock(),
+ "OSR fixup block shouldn't be the entry block");
+ MOZ_ASSERT(maybeFixupBlock != graph_.osrBlock(),
+ "OSR fixup block shouldn't be the OSR entry block");
+ maybeFixupBlock->mark();
+ }
+ }
+ }
+
+ // And sweep.
+ return RemoveUnmarkedBlocks(mir_, graph_, numMarked);
+}
+
+ValueNumberer::ValueNumberer(MIRGenerator* mir, MIRGraph& graph)
+ : mir_(mir),
+ graph_(graph),
+ // Initialize the value set. It's tempting to pass in a length that is a
+ // function of graph_.getNumInstructionIds(). But if we start out with a
+ // large capacity, it will be far larger than the actual element count for
+ // most of the pass, so when we remove elements, it would often think it
+ // needs to compact itself. Empirically, just letting the HashTable grow
+ // as needed on its own seems to work pretty well.
+ values_(graph.alloc()),
+ deadDefs_(graph.alloc()),
+ remainingBlocks_(graph.alloc()),
+ nextDef_(nullptr),
+ totalNumVisited_(0),
+ rerun_(false),
+ blocksRemoved_(false),
+ updateAliasAnalysis_(false),
+ dependenciesBroken_(false),
+ hasOSRFixups_(false) {}
+
+bool ValueNumberer::run(UpdateAliasAnalysisFlag updateAliasAnalysis) {
+ updateAliasAnalysis_ = updateAliasAnalysis == UpdateAliasAnalysis;
+
+ JitSpew(JitSpew_GVN, "Running GVN on graph (with %" PRIu64 " blocks)",
+ uint64_t(graph_.numBlocks()));
+
+ // Adding fixup blocks only make sense iff we have a second entry point into
+ // the graph which cannot be reached any more from the entry point.
+ if (graph_.osrBlock()) {
+ if (!insertOSRFixups()) {
+ return false;
+ }
+ }
+
+ // Top level non-sparse iteration loop. If an iteration performs a
+ // significant change, such as discarding a block which changes the
+ // dominator tree and may enable more optimization, this loop takes another
+ // iteration.
+ int runs = 0;
+ for (;;) {
+ if (!visitGraph()) {
+ return false;
+ }
+
+ // Test whether any block which was not removed but which had at least
+ // one predecessor removed will have a new dominator parent.
+ while (!remainingBlocks_.empty()) {
+ MBasicBlock* block = remainingBlocks_.popCopy();
+ if (!block->isDead() && IsDominatorRefined(block)) {
+ JitSpew(JitSpew_GVN,
+ " Dominator for block%u can now be refined; will re-run GVN!",
+ block->id());
+ rerun_ = true;
+ remainingBlocks_.clear();
+ break;
+ }
+ }
+
+ if (blocksRemoved_) {
+ if (!AccountForCFGChanges(mir_, graph_, dependenciesBroken_,
+ /* underValueNumberer = */ true)) {
+ return false;
+ }
+
+ blocksRemoved_ = false;
+ dependenciesBroken_ = false;
+ }
+
+ if (mir_->shouldCancel("GVN (outer loop)")) {
+ return false;
+ }
+
+ // If no further opportunities have been discovered, we're done.
+ if (!rerun_) {
+ break;
+ }
+
+ rerun_ = false;
+
+ // Enforce an arbitrary iteration limit. This is rarely reached, and
+ // isn't even strictly necessary, as the algorithm is guaranteed to
+ // terminate on its own in a finite amount of time (since every time we
+ // re-run we discard the construct which triggered the re-run), but it
+ // does help avoid slow compile times on pathological code.
+ ++runs;
+ if (runs == 6) {
+ JitSpew(JitSpew_GVN, "Re-run cutoff of %d reached. Terminating GVN!",
+ runs);
+ break;
+ }
+
+ JitSpew(JitSpew_GVN,
+ "Re-running GVN on graph (run %d, now with %" PRIu64 " blocks)",
+ runs, uint64_t(graph_.numBlocks()));
+ }
+
+ if (MOZ_UNLIKELY(hasOSRFixups_)) {
+ if (!cleanupOSRFixups()) {
+ return false;
+ }
+ hasOSRFixups_ = false;
+ }
+
+ return true;
+}
diff --git a/js/src/jit/ValueNumbering.h b/js/src/jit/ValueNumbering.h
new file mode 100644
index 0000000000..472d31ecce
--- /dev/null
+++ b/js/src/jit/ValueNumbering.h
@@ -0,0 +1,123 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_ValueNumbering_h
+#define jit_ValueNumbering_h
+
+#include "jit/JitAllocPolicy.h"
+#include "js/HashTable.h"
+#include "js/Vector.h"
+
+namespace js {
+namespace jit {
+
+class MDefinition;
+class MBasicBlock;
+class MIRGraph;
+class MPhi;
+class MIRGenerator;
+class MResumePoint;
+
+class ValueNumberer {
+ // Value numbering data.
+ class VisibleValues {
+ // Hash policy for ValueSet.
+ struct ValueHasher {
+ using Lookup = const MDefinition*;
+ using Key = MDefinition*;
+ static HashNumber hash(Lookup ins);
+ static bool match(Key k, Lookup l);
+ static void rekey(Key& k, Key newKey);
+ };
+
+ typedef HashSet<MDefinition*, ValueHasher, JitAllocPolicy> ValueSet;
+
+ ValueSet set_; // Set of visible values
+
+ public:
+ explicit VisibleValues(TempAllocator& alloc);
+
+ using Ptr = ValueSet::Ptr;
+ using AddPtr = ValueSet::AddPtr;
+
+ Ptr findLeader(const MDefinition* def) const;
+ AddPtr findLeaderForAdd(MDefinition* def);
+ [[nodiscard]] bool add(AddPtr p, MDefinition* def);
+ void overwrite(AddPtr p, MDefinition* def);
+ void forget(const MDefinition* def);
+ void clear();
+#ifdef DEBUG
+ bool has(const MDefinition* def) const;
+#endif
+ };
+
+ typedef Vector<MBasicBlock*, 4, JitAllocPolicy> BlockWorklist;
+ typedef Vector<MDefinition*, 4, JitAllocPolicy> DefWorklist;
+
+ MIRGenerator* const mir_;
+ MIRGraph& graph_;
+ VisibleValues values_; // Numbered values
+ DefWorklist deadDefs_; // Worklist for deleting values
+ BlockWorklist remainingBlocks_; // Blocks remaining with fewer preds
+ MDefinition* nextDef_; // The next definition; don't discard
+ size_t totalNumVisited_; // The number of blocks visited
+ bool rerun_; // Should we run another GVN iteration?
+ bool blocksRemoved_; // Have any blocks been removed?
+ bool updateAliasAnalysis_; // Do we care about AliasAnalysis?
+ bool dependenciesBroken_; // Have we broken AliasAnalysis?
+ bool hasOSRFixups_; // Have we created any OSR fixup blocks?
+
+ enum ImplicitUseOption { DontSetImplicitUse, SetImplicitUse };
+ enum class AllowEffectful : bool { No, Yes };
+
+ [[nodiscard]] bool handleUseReleased(MDefinition* def,
+ ImplicitUseOption implicitUseOption);
+ [[nodiscard]] bool discardDefsRecursively(
+ MDefinition* def, AllowEffectful allowEffectful = AllowEffectful::No);
+ [[nodiscard]] bool releaseResumePointOperands(MResumePoint* resume);
+ [[nodiscard]] bool releaseAndRemovePhiOperands(MPhi* phi);
+ [[nodiscard]] bool releaseOperands(MDefinition* def);
+ [[nodiscard]] bool discardDef(
+ MDefinition* def, AllowEffectful allowEffectful = AllowEffectful::No);
+ [[nodiscard]] bool processDeadDefs();
+
+ [[nodiscard]] bool fixupOSROnlyLoop(MBasicBlock* block);
+ [[nodiscard]] bool removePredecessorAndDoDCE(MBasicBlock* block,
+ MBasicBlock* pred,
+ size_t predIndex);
+ [[nodiscard]] bool removePredecessorAndCleanUp(MBasicBlock* block,
+ MBasicBlock* pred);
+
+ MDefinition* simplified(MDefinition* def) const;
+ MDefinition* leader(MDefinition* def);
+ bool hasLeader(const MPhi* phi, const MBasicBlock* phiBlock) const;
+ bool loopHasOptimizablePhi(MBasicBlock* header) const;
+
+ [[nodiscard]] bool visitDefinition(MDefinition* def);
+ [[nodiscard]] bool visitControlInstruction(MBasicBlock* block);
+ [[nodiscard]] bool visitUnreachableBlock(MBasicBlock* block);
+ [[nodiscard]] bool visitBlock(MBasicBlock* block);
+ [[nodiscard]] bool visitDominatorTree(MBasicBlock* root);
+ [[nodiscard]] bool visitGraph();
+
+ [[nodiscard]] bool insertOSRFixups();
+ [[nodiscard]] bool cleanupOSRFixups();
+
+ public:
+ ValueNumberer(MIRGenerator* mir, MIRGraph& graph);
+
+ enum UpdateAliasAnalysisFlag { DontUpdateAliasAnalysis, UpdateAliasAnalysis };
+
+ // Optimize the graph, performing expression simplification and
+ // canonicalization, eliminating statically fully-redundant expressions,
+ // deleting dead instructions, and removing unreachable blocks.
+ [[nodiscard]] bool run(UpdateAliasAnalysisFlag updateAliasAnalysis);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_ValueNumbering_h */
diff --git a/js/src/jit/WarpBuilder.cpp b/js/src/jit/WarpBuilder.cpp
new file mode 100644
index 0000000000..9b31912e52
--- /dev/null
+++ b/js/src/jit/WarpBuilder.cpp
@@ -0,0 +1,3589 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/WarpBuilder.h"
+
+#include "mozilla/DebugOnly.h"
+
+#include "jit/BaselineFrame.h"
+#include "jit/CacheIR.h"
+#include "jit/CompileInfo.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "jit/WarpCacheIRTranspiler.h"
+#include "jit/WarpSnapshot.h"
+#include "js/friend/ErrorMessages.h" // JSMSG_BAD_CONST_ASSIGN
+#include "vm/GeneratorObject.h"
+#include "vm/Interpreter.h"
+#include "vm/Opcodes.h"
+
+#include "gc/ObjectKind-inl.h"
+#include "vm/BytecodeIterator-inl.h"
+#include "vm/BytecodeLocation-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// Used for building the outermost script.
+WarpBuilder::WarpBuilder(WarpSnapshot& snapshot, MIRGenerator& mirGen,
+ WarpCompilation* warpCompilation)
+ : WarpBuilderShared(snapshot, mirGen, nullptr),
+ warpCompilation_(warpCompilation),
+ graph_(mirGen.graph()),
+ info_(mirGen.outerInfo()),
+ scriptSnapshot_(snapshot.rootScript()),
+ script_(snapshot.rootScript()->script()),
+ loopStack_(mirGen.alloc()) {
+ opSnapshotIter_ = scriptSnapshot_->opSnapshots().getFirst();
+}
+
+// Used for building inlined scripts.
+WarpBuilder::WarpBuilder(WarpBuilder* caller, WarpScriptSnapshot* snapshot,
+ CompileInfo& compileInfo, CallInfo* inlineCallInfo,
+ MResumePoint* callerResumePoint)
+ : WarpBuilderShared(caller->snapshot(), caller->mirGen(), nullptr),
+ warpCompilation_(caller->warpCompilation()),
+ graph_(caller->mirGen().graph()),
+ info_(compileInfo),
+ scriptSnapshot_(snapshot),
+ script_(snapshot->script()),
+ loopStack_(caller->mirGen().alloc()),
+ callerBuilder_(caller),
+ callerResumePoint_(callerResumePoint),
+ inlineCallInfo_(inlineCallInfo) {
+ opSnapshotIter_ = snapshot->opSnapshots().getFirst();
+}
+
+BytecodeSite* WarpBuilder::newBytecodeSite(BytecodeLocation loc) {
+ jsbytecode* pc = loc.toRawBytecode();
+ MOZ_ASSERT(info().inlineScriptTree()->script()->containsPC(pc));
+ return new (alloc()) BytecodeSite(info().inlineScriptTree(), pc);
+}
+
+const WarpOpSnapshot* WarpBuilder::getOpSnapshotImpl(
+ BytecodeLocation loc, WarpOpSnapshot::Kind kind) {
+ uint32_t offset = loc.bytecodeToOffset(script_);
+
+ // Skip snapshots until we get to a snapshot with offset >= offset. This is
+ // a loop because WarpBuilder can skip unreachable bytecode ops.
+ while (opSnapshotIter_ && opSnapshotIter_->offset() < offset) {
+ opSnapshotIter_ = opSnapshotIter_->getNext();
+ }
+
+ if (!opSnapshotIter_ || opSnapshotIter_->offset() != offset ||
+ opSnapshotIter_->kind() != kind) {
+ return nullptr;
+ }
+
+ return opSnapshotIter_;
+}
+
+void WarpBuilder::initBlock(MBasicBlock* block) {
+ graph().addBlock(block);
+
+ block->setLoopDepth(loopDepth());
+
+ current = block;
+}
+
+bool WarpBuilder::startNewBlock(MBasicBlock* predecessor, BytecodeLocation loc,
+ size_t numToPop) {
+ MBasicBlock* block =
+ MBasicBlock::NewPopN(graph(), info(), predecessor, newBytecodeSite(loc),
+ MBasicBlock::NORMAL, numToPop);
+ if (!block) {
+ return false;
+ }
+
+ initBlock(block);
+ return true;
+}
+
+bool WarpBuilder::startNewEntryBlock(size_t stackDepth, BytecodeLocation loc) {
+ MBasicBlock* block =
+ MBasicBlock::New(graph(), stackDepth, info(), /* maybePred = */ nullptr,
+ newBytecodeSite(loc), MBasicBlock::NORMAL);
+ if (!block) {
+ return false;
+ }
+
+ initBlock(block);
+ return true;
+}
+
+bool WarpBuilder::startNewLoopHeaderBlock(BytecodeLocation loopHead) {
+ MBasicBlock* header = MBasicBlock::NewPendingLoopHeader(
+ graph(), info(), current, newBytecodeSite(loopHead));
+ if (!header) {
+ return false;
+ }
+
+ initBlock(header);
+ return loopStack_.emplaceBack(header);
+}
+
+bool WarpBuilder::startNewOsrPreHeaderBlock(BytecodeLocation loopHead) {
+ MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
+ MOZ_ASSERT(loopHead.toRawBytecode() == info().osrPc());
+
+ // Create two blocks:
+ // * The OSR entry block. This is always the graph's second block and has no
+ // predecessors. This is the entry point for OSR from the Baseline JIT.
+ // * The OSR preheader block. This has two predecessors: the OSR entry block
+ // and the current block.
+
+ MBasicBlock* pred = current;
+
+ // Create the OSR entry block.
+ if (!startNewEntryBlock(pred->stackDepth(), loopHead)) {
+ return false;
+ }
+
+ MBasicBlock* osrBlock = current;
+ graph().setOsrBlock(osrBlock);
+ graph().moveBlockAfter(*graph().begin(), osrBlock);
+
+ MOsrEntry* entry = MOsrEntry::New(alloc());
+ osrBlock->add(entry);
+
+ // Initialize environment chain.
+ {
+ uint32_t slot = info().environmentChainSlot();
+ MInstruction* envv;
+ if (usesEnvironmentChain()) {
+ envv = MOsrEnvironmentChain::New(alloc(), entry);
+ } else {
+ // Use an undefined value if the script does not need its environment
+ // chain, to match the main entry point.
+ envv = MConstant::New(alloc(), UndefinedValue());
+ }
+ osrBlock->add(envv);
+ osrBlock->initSlot(slot, envv);
+ }
+
+ // Initialize return value.
+ {
+ MInstruction* returnValue;
+ if (!script_->noScriptRval()) {
+ returnValue = MOsrReturnValue::New(alloc(), entry);
+ } else {
+ returnValue = MConstant::New(alloc(), UndefinedValue());
+ }
+ osrBlock->add(returnValue);
+ osrBlock->initSlot(info().returnValueSlot(), returnValue);
+ }
+
+ // Initialize arguments object.
+ MInstruction* argsObj = nullptr;
+ if (info().needsArgsObj()) {
+ argsObj = MOsrArgumentsObject::New(alloc(), entry);
+ osrBlock->add(argsObj);
+ osrBlock->initSlot(info().argsObjSlot(), argsObj);
+ }
+
+ if (info().hasFunMaybeLazy()) {
+ // Initialize |this| parameter.
+ MParameter* thisv = MParameter::New(alloc(), MParameter::THIS_SLOT);
+ osrBlock->add(thisv);
+ osrBlock->initSlot(info().thisSlot(), thisv);
+
+ // Initialize arguments. There are three cases:
+ //
+ // 1) There's no ArgumentsObject or it doesn't alias formals. In this case
+ // we can just use the frame's argument slot.
+ // 2) The ArgumentsObject aliases formals and the argument is stored in the
+ // CallObject. Use |undefined| because we can't load from the arguments
+ // object and code will use the CallObject anyway.
+ // 3) The ArgumentsObject aliases formals and the argument isn't stored in
+ // the CallObject. We have to load it from the ArgumentsObject.
+ for (uint32_t i = 0; i < info().nargs(); i++) {
+ uint32_t slot = info().argSlotUnchecked(i);
+ MInstruction* osrv;
+ if (!info().argsObjAliasesFormals()) {
+ osrv = MParameter::New(alloc().fallible(), i);
+ } else if (script_->formalIsAliased(i)) {
+ osrv = MConstant::New(alloc().fallible(), UndefinedValue());
+ } else {
+ osrv = MGetArgumentsObjectArg::New(alloc().fallible(), argsObj, i);
+ }
+ if (!osrv) {
+ return false;
+ }
+ current->add(osrv);
+ current->initSlot(slot, osrv);
+ }
+ }
+
+ // Initialize locals.
+ uint32_t nlocals = info().nlocals();
+ for (uint32_t i = 0; i < nlocals; i++) {
+ uint32_t slot = info().localSlot(i);
+ ptrdiff_t offset = BaselineFrame::reverseOffsetOfLocal(i);
+ MOsrValue* osrv = MOsrValue::New(alloc().fallible(), entry, offset);
+ if (!osrv) {
+ return false;
+ }
+ current->add(osrv);
+ current->initSlot(slot, osrv);
+ }
+
+ // Initialize expression stack slots.
+ uint32_t numStackSlots = current->stackDepth() - info().firstStackSlot();
+ for (uint32_t i = 0; i < numStackSlots; i++) {
+ uint32_t slot = info().stackSlot(i);
+ ptrdiff_t offset = BaselineFrame::reverseOffsetOfLocal(nlocals + i);
+ MOsrValue* osrv = MOsrValue::New(alloc().fallible(), entry, offset);
+ if (!osrv) {
+ return false;
+ }
+ current->add(osrv);
+ current->initSlot(slot, osrv);
+ }
+
+ MStart* start = MStart::New(alloc());
+ current->add(start);
+
+ // Note: phi specialization can add type guard instructions to the OSR entry
+ // block if needed. See TypeAnalyzer::shouldSpecializeOsrPhis.
+
+ // Create the preheader block, with the predecessor block and OSR block as
+ // predecessors.
+ if (!startNewBlock(pred, loopHead)) {
+ return false;
+ }
+
+ pred->end(MGoto::New(alloc(), current));
+ osrBlock->end(MGoto::New(alloc(), current));
+
+ if (!current->addPredecessor(alloc(), osrBlock)) {
+ return false;
+ }
+
+ return true;
+}
+
+bool WarpBuilder::addPendingEdge(BytecodeLocation target, MBasicBlock* block,
+ uint32_t successor, uint32_t numToPop) {
+ MOZ_ASSERT(successor < block->lastIns()->numSuccessors());
+ MOZ_ASSERT(numToPop <= block->stackDepth());
+
+ jsbytecode* targetPC = target.toRawBytecode();
+ PendingEdgesMap::AddPtr p = pendingEdges_.lookupForAdd(targetPC);
+ if (p) {
+ return p->value().emplaceBack(block, successor, numToPop);
+ }
+
+ PendingEdges edges;
+ static_assert(PendingEdges::InlineLength >= 1,
+ "Appending one element should be infallible");
+ MOZ_ALWAYS_TRUE(edges.emplaceBack(block, successor, numToPop));
+
+ return pendingEdges_.add(p, targetPC, std::move(edges));
+}
+
+bool WarpBuilder::build() {
+ if (!buildPrologue()) {
+ return false;
+ }
+
+ if (!buildBody()) {
+ return false;
+ }
+
+ if (!MPhi::markIteratorPhis(*iterators())) {
+ return false;
+ }
+
+ MOZ_ASSERT_IF(info().osrPc(), graph().osrBlock());
+ MOZ_ASSERT(loopStack_.empty());
+ MOZ_ASSERT(loopDepth() == 0);
+
+ return true;
+}
+
+bool WarpBuilder::buildInline() {
+ if (!buildInlinePrologue()) {
+ return false;
+ }
+
+ if (!buildBody()) {
+ return false;
+ }
+
+ MOZ_ASSERT(loopStack_.empty());
+ return true;
+}
+
+MInstruction* WarpBuilder::buildNamedLambdaEnv(MDefinition* callee,
+ MDefinition* env,
+ NamedLambdaObject* templateObj) {
+ MOZ_ASSERT(!templateObj->hasDynamicSlots());
+
+ MInstruction* namedLambda = MNewNamedLambdaObject::New(alloc(), templateObj);
+ current->add(namedLambda);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barriers.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), namedLambda, env));
+ current->add(
+ MAssertCanElidePostWriteBarrier::New(alloc(), namedLambda, callee));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here:
+ // the object will be allocated in the nursery if possible, and if the
+ // tenured heap is used instead, a minor collection will have been performed
+ // that moved env/callee to the tenured heap.
+ size_t enclosingSlot = NamedLambdaObject::enclosingEnvironmentSlot();
+ size_t lambdaSlot = NamedLambdaObject::lambdaSlot();
+ current->add(MStoreFixedSlot::NewUnbarriered(alloc(), namedLambda,
+ enclosingSlot, env));
+ current->add(MStoreFixedSlot::NewUnbarriered(alloc(), namedLambda, lambdaSlot,
+ callee));
+
+ return namedLambda;
+}
+
+MInstruction* WarpBuilder::buildCallObject(MDefinition* callee,
+ MDefinition* env,
+ CallObject* templateObj) {
+ MConstant* templateCst = constant(ObjectValue(*templateObj));
+
+ MNewCallObject* callObj = MNewCallObject::New(alloc(), templateCst);
+ current->add(callObj);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barriers.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), callObj, env));
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), callObj, callee));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ size_t enclosingSlot = CallObject::enclosingEnvironmentSlot();
+ size_t calleeSlot = CallObject::calleeSlot();
+ current->add(
+ MStoreFixedSlot::NewUnbarriered(alloc(), callObj, enclosingSlot, env));
+ current->add(
+ MStoreFixedSlot::NewUnbarriered(alloc(), callObj, calleeSlot, callee));
+
+ // Copy closed-over argument slots if there aren't parameter expressions.
+ MSlots* slots = nullptr;
+ for (PositionalFormalParameterIter fi(script_); fi; fi++) {
+ if (!fi.closedOver()) {
+ continue;
+ }
+
+ if (!alloc().ensureBallast()) {
+ return nullptr;
+ }
+
+ uint32_t slot = fi.location().slot();
+ uint32_t formal = fi.argumentSlot();
+ uint32_t numFixedSlots = templateObj->numFixedSlots();
+ MDefinition* param;
+ if (script_->functionHasParameterExprs()) {
+ param = constant(MagicValue(JS_UNINITIALIZED_LEXICAL));
+ } else {
+ param = current->getSlot(info().argSlotUnchecked(formal));
+ }
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), callObj, param));
+#endif
+
+ if (slot >= numFixedSlots) {
+ if (!slots) {
+ slots = MSlots::New(alloc(), callObj);
+ current->add(slots);
+ }
+ uint32_t dynamicSlot = slot - numFixedSlots;
+ current->add(MStoreDynamicSlot::NewUnbarriered(alloc(), slots,
+ dynamicSlot, param));
+ } else {
+ current->add(
+ MStoreFixedSlot::NewUnbarriered(alloc(), callObj, slot, param));
+ }
+ }
+
+ return callObj;
+}
+
+bool WarpBuilder::buildEnvironmentChain() {
+ const WarpEnvironment& env = scriptSnapshot()->environment();
+
+ if (env.is<NoEnvironment>()) {
+ return true;
+ }
+
+ MInstruction* envDef = env.match(
+ [](const NoEnvironment&) -> MInstruction* {
+ MOZ_CRASH("Already handled");
+ },
+ [this](JSObject* obj) -> MInstruction* {
+ return constant(ObjectValue(*obj));
+ },
+ [this](const FunctionEnvironment& env) -> MInstruction* {
+ MDefinition* callee = getCallee();
+ MInstruction* envDef = MFunctionEnvironment::New(alloc(), callee);
+ current->add(envDef);
+ if (NamedLambdaObject* obj = env.namedLambdaTemplate) {
+ envDef = buildNamedLambdaEnv(callee, envDef, obj);
+ }
+ if (CallObject* obj = env.callObjectTemplate) {
+ envDef = buildCallObject(callee, envDef, obj);
+ if (!envDef) {
+ return nullptr;
+ }
+ }
+ return envDef;
+ });
+ if (!envDef) {
+ return false;
+ }
+
+ // Update the environment slot from UndefinedValue only after the initial
+ // environment is created so that bailout doesn't see a partial environment.
+ // See: |BaselineStackBuilder::buildBaselineFrame|
+ current->setEnvironmentChain(envDef);
+ return true;
+}
+
+bool WarpBuilder::buildPrologue() {
+ BytecodeLocation startLoc(script_, script_->code());
+ if (!startNewEntryBlock(info().firstStackSlot(), startLoc)) {
+ return false;
+ }
+
+ if (info().hasFunMaybeLazy()) {
+ // Initialize |this|.
+ MParameter* param = MParameter::New(alloc(), MParameter::THIS_SLOT);
+ current->add(param);
+ current->initSlot(info().thisSlot(), param);
+
+ // Initialize arguments.
+ for (uint32_t i = 0; i < info().nargs(); i++) {
+ MParameter* param = MParameter::New(alloc().fallible(), i);
+ if (!param) {
+ return false;
+ }
+ current->add(param);
+ current->initSlot(info().argSlotUnchecked(i), param);
+ }
+ }
+
+ MConstant* undef = constant(UndefinedValue());
+
+ // Initialize local slots.
+ for (uint32_t i = 0; i < info().nlocals(); i++) {
+ current->initSlot(info().localSlot(i), undef);
+ }
+
+ // Initialize the environment chain, return value, and arguments object slots.
+ current->initSlot(info().environmentChainSlot(), undef);
+ current->initSlot(info().returnValueSlot(), undef);
+ if (info().needsArgsObj()) {
+ current->initSlot(info().argsObjSlot(), undef);
+ }
+
+ current->add(MStart::New(alloc()));
+
+ // Guard against over-recursion.
+ MCheckOverRecursed* check = MCheckOverRecursed::New(alloc());
+ current->add(check);
+
+ if (!buildEnvironmentChain()) {
+ return false;
+ }
+
+#ifdef JS_CACHEIR_SPEW
+ if (snapshot().needsFinalWarmUpCount()) {
+ MIncrementWarmUpCounter* ins =
+ MIncrementWarmUpCounter::New(alloc(), script_);
+ current->add(ins);
+ }
+#endif
+
+ return true;
+}
+
+bool WarpBuilder::buildInlinePrologue() {
+ // Generate entry block.
+ BytecodeLocation startLoc(script_, script_->code());
+ if (!startNewEntryBlock(info().firstStackSlot(), startLoc)) {
+ return false;
+ }
+ current->setCallerResumePoint(callerResumePoint());
+
+ // Connect the entry block to the last block in the caller's graph.
+ MBasicBlock* pred = callerBuilder()->current;
+ MOZ_ASSERT(pred == callerResumePoint()->block());
+
+ pred->end(MGoto::New(alloc(), current));
+ if (!current->addPredecessorWithoutPhis(pred)) {
+ return false;
+ }
+
+ MConstant* undef = constant(UndefinedValue());
+
+ // Initialize env chain slot to Undefined. It's set later by
+ // |buildEnvironmentChain|.
+ current->initSlot(info().environmentChainSlot(), undef);
+
+ // Initialize |return value| slot.
+ current->initSlot(info().returnValueSlot(), undef);
+
+ // Initialize |arguments| slot if needed.
+ if (info().needsArgsObj()) {
+ current->initSlot(info().argsObjSlot(), undef);
+ }
+
+ // Initialize |this| slot.
+ current->initSlot(info().thisSlot(), inlineCallInfo()->thisArg());
+
+ uint32_t callerArgs = inlineCallInfo()->argc();
+ uint32_t actualArgs = info().nargs();
+ uint32_t passedArgs = std::min<uint32_t>(callerArgs, actualArgs);
+
+ // Initialize actually set arguments.
+ for (uint32_t i = 0; i < passedArgs; i++) {
+ MDefinition* arg = inlineCallInfo()->getArg(i);
+ current->initSlot(info().argSlotUnchecked(i), arg);
+ }
+
+ // Pass undefined for missing arguments.
+ for (uint32_t i = passedArgs; i < actualArgs; i++) {
+ current->initSlot(info().argSlotUnchecked(i), undef);
+ }
+
+ // Initialize local slots.
+ for (uint32_t i = 0; i < info().nlocals(); i++) {
+ current->initSlot(info().localSlot(i), undef);
+ }
+
+ MOZ_ASSERT(current->entryResumePoint()->stackDepth() == info().totalSlots());
+
+ if (!buildEnvironmentChain()) {
+ return false;
+ }
+
+ return true;
+}
+
+#ifdef DEBUG
+// In debug builds, after compiling a bytecode op, this class is used to check
+// that all values popped by this opcode either:
+//
+// (1) Have the ImplicitlyUsed flag set on them.
+// (2) Have more uses than before compiling this op (the value is
+// used as operand of a new MIR instruction).
+//
+// This is used to catch problems where WarpBuilder pops a value without
+// adding any SSA uses and doesn't call setImplicitlyUsedUnchecked on it.
+class MOZ_RAII WarpPoppedValueUseChecker {
+ Vector<MDefinition*, 4, SystemAllocPolicy> popped_;
+ Vector<size_t, 4, SystemAllocPolicy> poppedUses_;
+ MBasicBlock* current_;
+ BytecodeLocation loc_;
+
+ public:
+ WarpPoppedValueUseChecker(MBasicBlock* current, BytecodeLocation loc)
+ : current_(current), loc_(loc) {}
+
+ [[nodiscard]] bool init() {
+ // Don't require SSA uses for values popped by these ops.
+ switch (loc_.getOp()) {
+ case JSOp::Pop:
+ case JSOp::PopN:
+ case JSOp::DupAt:
+ case JSOp::Dup:
+ case JSOp::Dup2:
+ case JSOp::Pick:
+ case JSOp::Unpick:
+ case JSOp::Swap:
+ case JSOp::SetArg:
+ case JSOp::SetLocal:
+ case JSOp::InitLexical:
+ case JSOp::SetRval:
+ case JSOp::Void:
+ // Basic stack/local/argument management opcodes.
+ return true;
+
+ case JSOp::Case:
+ case JSOp::Default:
+ // These ops have to pop the switch value when branching but don't
+ // actually use it.
+ return true;
+
+ default:
+ break;
+ }
+
+ unsigned nuses = loc_.useCount();
+
+ for (unsigned i = 0; i < nuses; i++) {
+ MDefinition* def = current_->peek(-int32_t(i + 1));
+ if (!popped_.append(def) || !poppedUses_.append(def->defUseCount())) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ void checkAfterOp() {
+ for (size_t i = 0; i < popped_.length(); i++) {
+ // First value popped by JSOp::EndIter is not used at all, it's similar
+ // to JSOp::Pop above.
+ if (loc_.is(JSOp::EndIter) && i == 0) {
+ continue;
+ }
+ MOZ_ASSERT(popped_[i]->isImplicitlyUsed() ||
+ popped_[i]->defUseCount() > poppedUses_[i]);
+ }
+ }
+};
+#endif
+
+bool WarpBuilder::buildBody() {
+ for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
+ if (mirGen().shouldCancel("WarpBuilder (opcode loop)")) {
+ return false;
+ }
+
+ // Skip unreachable ops (for example code after a 'return' or 'throw') until
+ // we get to the next jump target.
+ if (hasTerminatedBlock()) {
+ // Finish any "broken" loops with an unreachable backedge. For example:
+ //
+ // do {
+ // ...
+ // return;
+ // ...
+ // } while (x);
+ //
+ // This loop never actually loops.
+ if (loc.isBackedge() && !loopStack_.empty()) {
+ BytecodeLocation loopHead(script_, loopStack_.back().header()->pc());
+ if (loc.isBackedgeForLoophead(loopHead)) {
+ decLoopDepth();
+ loopStack_.popBack();
+ }
+ }
+ if (!loc.isJumpTarget()) {
+ continue;
+ }
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+#ifdef DEBUG
+ WarpPoppedValueUseChecker useChecker(current, loc);
+ if (!useChecker.init()) {
+ return false;
+ }
+#endif
+
+ JSOp op = loc.getOp();
+
+#define BUILD_OP(OP, ...) \
+ case JSOp::OP: \
+ if (MOZ_UNLIKELY(!this->build_##OP(loc))) { \
+ return false; \
+ } \
+ break;
+ switch (op) { FOR_EACH_OPCODE(BUILD_OP) }
+#undef BUILD_OP
+
+#ifdef DEBUG
+ useChecker.checkAfterOp();
+#endif
+ }
+
+ return true;
+}
+
+#define DEF_OP(OP) \
+ bool WarpBuilder::build_##OP(BytecodeLocation) { \
+ MOZ_CRASH("Unsupported op"); \
+ }
+WARP_UNSUPPORTED_OPCODE_LIST(DEF_OP)
+#undef DEF_OP
+
+bool WarpBuilder::build_Nop(BytecodeLocation) { return true; }
+
+bool WarpBuilder::build_NopDestructuring(BytecodeLocation) { return true; }
+
+bool WarpBuilder::build_TryDestructuring(BytecodeLocation) {
+ // Set the hasTryBlock flag to turn off optimizations that eliminate dead
+ // resume points operands because the exception handler code for
+ // TryNoteKind::Destructuring is effectively a (specialized) catch-block.
+ graph().setHasTryBlock();
+ return true;
+}
+
+bool WarpBuilder::build_Lineno(BytecodeLocation) { return true; }
+
+bool WarpBuilder::build_DebugLeaveLexicalEnv(BytecodeLocation) { return true; }
+
+bool WarpBuilder::build_Undefined(BytecodeLocation) {
+ pushConstant(UndefinedValue());
+ return true;
+}
+
+bool WarpBuilder::build_Void(BytecodeLocation) {
+ current->pop();
+ pushConstant(UndefinedValue());
+ return true;
+}
+
+bool WarpBuilder::build_Null(BytecodeLocation) {
+ pushConstant(NullValue());
+ return true;
+}
+
+bool WarpBuilder::build_Hole(BytecodeLocation) {
+ pushConstant(MagicValue(JS_ELEMENTS_HOLE));
+ return true;
+}
+
+bool WarpBuilder::build_Uninitialized(BytecodeLocation) {
+ pushConstant(MagicValue(JS_UNINITIALIZED_LEXICAL));
+ return true;
+}
+
+bool WarpBuilder::build_IsConstructing(BytecodeLocation) {
+ pushConstant(MagicValue(JS_IS_CONSTRUCTING));
+ return true;
+}
+
+bool WarpBuilder::build_False(BytecodeLocation) {
+ pushConstant(BooleanValue(false));
+ return true;
+}
+
+bool WarpBuilder::build_True(BytecodeLocation) {
+ pushConstant(BooleanValue(true));
+ return true;
+}
+
+bool WarpBuilder::build_Pop(BytecodeLocation) {
+ current->pop();
+ return true;
+}
+
+bool WarpBuilder::build_PopN(BytecodeLocation loc) {
+ for (uint32_t i = 0, n = loc.getPopCount(); i < n; i++) {
+ current->pop();
+ }
+ return true;
+}
+
+bool WarpBuilder::build_Dup(BytecodeLocation) {
+ current->pushSlot(current->stackDepth() - 1);
+ return true;
+}
+
+bool WarpBuilder::build_Dup2(BytecodeLocation) {
+ uint32_t lhsSlot = current->stackDepth() - 2;
+ uint32_t rhsSlot = current->stackDepth() - 1;
+ current->pushSlot(lhsSlot);
+ current->pushSlot(rhsSlot);
+ return true;
+}
+
+bool WarpBuilder::build_DupAt(BytecodeLocation loc) {
+ current->pushSlot(current->stackDepth() - 1 - loc.getDupAtIndex());
+ return true;
+}
+
+bool WarpBuilder::build_Swap(BytecodeLocation) {
+ current->swapAt(-1);
+ return true;
+}
+
+bool WarpBuilder::build_Pick(BytecodeLocation loc) {
+ int32_t depth = -int32_t(loc.getPickDepth());
+ current->pick(depth);
+ return true;
+}
+
+bool WarpBuilder::build_Unpick(BytecodeLocation loc) {
+ int32_t depth = -int32_t(loc.getUnpickDepth());
+ current->unpick(depth);
+ return true;
+}
+
+bool WarpBuilder::build_Zero(BytecodeLocation) {
+ pushConstant(Int32Value(0));
+ return true;
+}
+
+bool WarpBuilder::build_One(BytecodeLocation) {
+ pushConstant(Int32Value(1));
+ return true;
+}
+
+bool WarpBuilder::build_Int8(BytecodeLocation loc) {
+ pushConstant(Int32Value(loc.getInt8()));
+ return true;
+}
+
+bool WarpBuilder::build_Uint16(BytecodeLocation loc) {
+ pushConstant(Int32Value(loc.getUint16()));
+ return true;
+}
+
+bool WarpBuilder::build_Uint24(BytecodeLocation loc) {
+ pushConstant(Int32Value(loc.getUint24()));
+ return true;
+}
+
+bool WarpBuilder::build_Int32(BytecodeLocation loc) {
+ pushConstant(Int32Value(loc.getInt32()));
+ return true;
+}
+
+bool WarpBuilder::build_Double(BytecodeLocation loc) {
+ pushConstant(loc.getInlineValue());
+ return true;
+}
+
+bool WarpBuilder::build_BigInt(BytecodeLocation loc) {
+ BigInt* bi = loc.getBigInt(script_);
+ pushConstant(BigIntValue(bi));
+ return true;
+}
+
+bool WarpBuilder::build_String(BytecodeLocation loc) {
+ JSString* str = loc.getString(script_);
+ pushConstant(StringValue(str));
+ return true;
+}
+
+bool WarpBuilder::build_Symbol(BytecodeLocation loc) {
+ uint32_t which = loc.getSymbolIndex();
+ JS::Symbol* sym = mirGen().runtime->wellKnownSymbols().get(which);
+ pushConstant(SymbolValue(sym));
+ return true;
+}
+
+bool WarpBuilder::build_RegExp(BytecodeLocation loc) {
+ RegExpObject* reObj = loc.getRegExp(script_);
+
+ auto* snapshot = getOpSnapshot<WarpRegExp>(loc);
+
+ MRegExp* regexp = MRegExp::New(alloc(), reObj, snapshot->hasShared());
+ current->add(regexp);
+ current->push(regexp);
+
+ return true;
+}
+
+bool WarpBuilder::build_Return(BytecodeLocation) {
+ MDefinition* def = current->pop();
+
+ MReturn* ret = MReturn::New(alloc(), def);
+ current->end(ret);
+
+ if (!graph().addReturn(current)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_RetRval(BytecodeLocation) {
+ MDefinition* rval;
+ if (script_->noScriptRval()) {
+ rval = constant(UndefinedValue());
+ } else {
+ rval = current->getSlot(info().returnValueSlot());
+ }
+
+ MReturn* ret = MReturn::New(alloc(), rval);
+ current->end(ret);
+
+ if (!graph().addReturn(current)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_SetRval(BytecodeLocation) {
+ MOZ_ASSERT(!script_->noScriptRval());
+ MDefinition* rval = current->pop();
+ current->setSlot(info().returnValueSlot(), rval);
+ return true;
+}
+
+bool WarpBuilder::build_GetRval(BytecodeLocation) {
+ MOZ_ASSERT(!script_->noScriptRval());
+ MDefinition* rval = current->getSlot(info().returnValueSlot());
+ current->push(rval);
+ return true;
+}
+
+bool WarpBuilder::build_GetLocal(BytecodeLocation loc) {
+ current->pushLocal(loc.local());
+ return true;
+}
+
+bool WarpBuilder::build_SetLocal(BytecodeLocation loc) {
+ current->setLocal(loc.local());
+ return true;
+}
+
+bool WarpBuilder::build_InitLexical(BytecodeLocation loc) {
+ current->setLocal(loc.local());
+ return true;
+}
+
+bool WarpBuilder::build_GetArg(BytecodeLocation loc) {
+ uint32_t arg = loc.arg();
+ if (info().argsObjAliasesFormals()) {
+ MDefinition* argsObj = current->argumentsObject();
+ auto* getArg = MGetArgumentsObjectArg::New(alloc(), argsObj, arg);
+ current->add(getArg);
+ current->push(getArg);
+ } else {
+ current->pushArg(arg);
+ }
+ return true;
+}
+
+bool WarpBuilder::build_SetArg(BytecodeLocation loc) {
+ MOZ_ASSERT(script_->jitScript()->modifiesArguments());
+
+ uint32_t arg = loc.arg();
+ MDefinition* val = current->peek(-1);
+
+ if (!info().argsObjAliasesFormals()) {
+ // Either |arguments| is never referenced within this function, or
+ // it doesn't map to the actual arguments values. Either way, we
+ // don't need to worry about synchronizing the argument values
+ // when writing to them.
+ current->setArg(arg);
+ return true;
+ }
+
+ // If an arguments object is in use, and it aliases formals, then all SetArgs
+ // must go through the arguments object.
+ MDefinition* argsObj = current->argumentsObject();
+ current->add(MPostWriteBarrier::New(alloc(), argsObj, val));
+ auto* ins = MSetArgumentsObjectArg::New(alloc(), argsObj, val, arg);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_ToNumeric(BytecodeLocation loc) {
+ return buildUnaryOp(loc);
+}
+
+bool WarpBuilder::buildUnaryOp(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ return buildIC(loc, CacheKind::UnaryArith, {value});
+}
+
+bool WarpBuilder::build_Inc(BytecodeLocation loc) { return buildUnaryOp(loc); }
+
+bool WarpBuilder::build_Dec(BytecodeLocation loc) { return buildUnaryOp(loc); }
+
+bool WarpBuilder::build_Pos(BytecodeLocation loc) { return buildUnaryOp(loc); }
+
+bool WarpBuilder::build_Neg(BytecodeLocation loc) { return buildUnaryOp(loc); }
+
+bool WarpBuilder::build_BitNot(BytecodeLocation loc) {
+ return buildUnaryOp(loc);
+}
+
+bool WarpBuilder::buildBinaryOp(BytecodeLocation loc) {
+ MDefinition* right = current->pop();
+ MDefinition* left = current->pop();
+ return buildIC(loc, CacheKind::BinaryArith, {left, right});
+}
+
+bool WarpBuilder::build_Add(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Sub(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Mul(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Div(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Mod(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Pow(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_BitAnd(BytecodeLocation loc) {
+ return buildBinaryOp(loc);
+}
+
+bool WarpBuilder::build_BitOr(BytecodeLocation loc) {
+ return buildBinaryOp(loc);
+}
+
+bool WarpBuilder::build_BitXor(BytecodeLocation loc) {
+ return buildBinaryOp(loc);
+}
+
+bool WarpBuilder::build_Lsh(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Rsh(BytecodeLocation loc) { return buildBinaryOp(loc); }
+
+bool WarpBuilder::build_Ursh(BytecodeLocation loc) {
+ return buildBinaryOp(loc);
+}
+
+bool WarpBuilder::buildCompareOp(BytecodeLocation loc) {
+ MDefinition* right = current->pop();
+ MDefinition* left = current->pop();
+ return buildIC(loc, CacheKind::Compare, {left, right});
+}
+
+bool WarpBuilder::build_Eq(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Ne(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Lt(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Le(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Gt(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_Ge(BytecodeLocation loc) { return buildCompareOp(loc); }
+
+bool WarpBuilder::build_StrictEq(BytecodeLocation loc) {
+ return buildCompareOp(loc);
+}
+
+bool WarpBuilder::build_StrictNe(BytecodeLocation loc) {
+ return buildCompareOp(loc);
+}
+
+// Returns true iff the MTest added for |op| has a true-target corresponding
+// with the join point in the bytecode.
+static bool TestTrueTargetIsJoinPoint(JSOp op) {
+ switch (op) {
+ case JSOp::JumpIfTrue:
+ case JSOp::Or:
+ case JSOp::Case:
+ return true;
+
+ case JSOp::JumpIfFalse:
+ case JSOp::And:
+ case JSOp::Coalesce:
+ return false;
+
+ default:
+ MOZ_CRASH("Unexpected op");
+ }
+}
+
+bool WarpBuilder::build_JumpTarget(BytecodeLocation loc) {
+ PendingEdgesMap::Ptr p = pendingEdges_.lookup(loc.toRawBytecode());
+ if (!p) {
+ // No (reachable) jumps so this is just a no-op.
+ return true;
+ }
+
+ PendingEdges edges(std::move(p->value()));
+ pendingEdges_.remove(p);
+
+ MOZ_ASSERT(!edges.empty());
+
+ // Create join block if there's fall-through from the previous bytecode op.
+ if (!hasTerminatedBlock()) {
+ MBasicBlock* pred = current;
+ if (!startNewBlock(pred, loc)) {
+ return false;
+ }
+ pred->end(MGoto::New(alloc(), current));
+ }
+
+ for (const PendingEdge& edge : edges) {
+ MBasicBlock* source = edge.block();
+ uint32_t numToPop = edge.numToPop();
+
+ if (hasTerminatedBlock()) {
+ if (!startNewBlock(source, loc, numToPop)) {
+ return false;
+ }
+ } else {
+ MOZ_ASSERT(source->stackDepth() - numToPop == current->stackDepth());
+ if (!current->addPredecessorPopN(alloc(), source, numToPop)) {
+ return false;
+ }
+ }
+
+ MOZ_ASSERT(source->lastIns()->isTest() || source->lastIns()->isGoto() ||
+ source->lastIns()->isTableSwitch());
+ source->lastIns()->initSuccessor(edge.successor(), current);
+ }
+
+ MOZ_ASSERT(!hasTerminatedBlock());
+ return true;
+}
+
+bool WarpBuilder::addIteratorLoopPhis(BytecodeLocation loopHead) {
+ // When unwinding the stack for a thrown exception, the exception handler must
+ // close live iterators. For ForIn and Destructuring loops, the exception
+ // handler needs access to values on the stack. To prevent them from being
+ // optimized away (and replaced with the JS_OPTIMIZED_OUT MagicValue), we need
+ // to mark the phis (and phis they flow into) as having implicit uses.
+ // See ProcessTryNotes in vm/Interpreter.cpp and CloseLiveIteratorIon in
+ // jit/JitFrames.cpp
+
+ MOZ_ASSERT(current->stackDepth() >= info().firstStackSlot());
+
+ bool emptyStack = current->stackDepth() == info().firstStackSlot();
+ if (emptyStack) {
+ return true;
+ }
+
+ jsbytecode* loopHeadPC = loopHead.toRawBytecode();
+
+ for (TryNoteIterAllNoGC tni(script_, loopHeadPC); !tni.done(); ++tni) {
+ const TryNote& tn = **tni;
+
+ // Stop if we reach an outer loop because outer loops were already
+ // processed when we visited their loop headers.
+ if (tn.isLoop()) {
+ BytecodeLocation tnStart = script_->offsetToLocation(tn.start);
+ if (tnStart != loopHead) {
+ MOZ_ASSERT(tnStart.is(JSOp::LoopHead));
+ MOZ_ASSERT(tnStart < loopHead);
+ return true;
+ }
+ }
+
+ switch (tn.kind()) {
+ case TryNoteKind::Destructuring:
+ case TryNoteKind::ForIn: {
+ // For for-in loops we add the iterator object to iterators(). For
+ // destructuring loops we add the "done" value that's on top of the
+ // stack and used in the exception handler.
+ MOZ_ASSERT(tn.stackDepth >= 1);
+ uint32_t slot = info().stackSlot(tn.stackDepth - 1);
+ MPhi* phi = current->getSlot(slot)->toPhi();
+ if (!iterators()->append(phi)) {
+ return false;
+ }
+ break;
+ }
+ case TryNoteKind::Loop:
+ case TryNoteKind::ForOf:
+ // Regular loops do not have iterators to close. ForOf loops handle
+ // unwinding using catch blocks.
+ break;
+ default:
+ break;
+ }
+ }
+
+ return true;
+}
+
+bool WarpBuilder::build_LoopHead(BytecodeLocation loc) {
+ // All loops have the following bytecode structure:
+ //
+ // LoopHead
+ // ...
+ // JumpIfTrue/Goto to LoopHead
+
+ if (hasTerminatedBlock()) {
+ // The whole loop is unreachable.
+ return true;
+ }
+
+ // Handle OSR from Baseline JIT code.
+ if (loc.toRawBytecode() == info().osrPc()) {
+ if (!startNewOsrPreHeaderBlock(loc)) {
+ return false;
+ }
+ }
+
+ incLoopDepth();
+
+ MBasicBlock* pred = current;
+ if (!startNewLoopHeaderBlock(loc)) {
+ return false;
+ }
+
+ pred->end(MGoto::New(alloc(), current));
+
+ if (!addIteratorLoopPhis(loc)) {
+ return false;
+ }
+
+ MInterruptCheck* check = MInterruptCheck::New(alloc());
+ current->add(check);
+
+#ifdef JS_CACHEIR_SPEW
+ if (snapshot().needsFinalWarmUpCount()) {
+ MIncrementWarmUpCounter* ins =
+ MIncrementWarmUpCounter::New(alloc(), script_);
+ current->add(ins);
+ }
+#endif
+
+ return true;
+}
+
+bool WarpBuilder::buildTestOp(BytecodeLocation loc) {
+ MDefinition* originalValue = current->peek(-1);
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ // If we have CacheIR, we can use it to refine the input. Note that
+ // the transpiler doesn't generate any control instructions. Instead,
+ // we fall through and generate them below.
+ MDefinition* value = current->pop();
+ if (!TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {value})) {
+ return false;
+ }
+ }
+
+ if (loc.isBackedge()) {
+ return buildTestBackedge(loc);
+ }
+
+ JSOp op = loc.getOp();
+ BytecodeLocation target1 = loc.next();
+ BytecodeLocation target2 = loc.getJumpTarget();
+
+ if (TestTrueTargetIsJoinPoint(op)) {
+ std::swap(target1, target2);
+ }
+
+ MDefinition* value = current->pop();
+
+ // JSOp::And and JSOp::Or leave the top stack value unchanged. The
+ // top stack value may have been converted to bool by a transpiled
+ // ToBool IC, so we push the original value.
+ bool mustKeepCondition = (op == JSOp::And || op == JSOp::Or);
+ if (mustKeepCondition) {
+ current->push(originalValue);
+ }
+
+ // If this op always branches to the same location we treat this as a
+ // JSOp::Goto.
+ if (target1 == target2) {
+ value->setImplicitlyUsedUnchecked();
+ return buildForwardGoto(target1);
+ }
+
+ MTest* test = MTest::New(alloc(), value, /* ifTrue = */ nullptr,
+ /* ifFalse = */ nullptr);
+ current->end(test);
+
+ // JSOp::Case must pop a second value on the true-branch (the input to the
+ // switch-statement).
+ uint32_t numToPop = (loc.getOp() == JSOp::Case) ? 1 : 0;
+
+ if (!addPendingEdge(target1, current, MTest::TrueBranchIndex, numToPop)) {
+ return false;
+ }
+ if (!addPendingEdge(target2, current, MTest::FalseBranchIndex)) {
+ return false;
+ }
+
+ if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
+ test->setObservedTypes(typesSnapshot->list());
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::buildTestBackedge(BytecodeLocation loc) {
+ MOZ_ASSERT(loc.is(JSOp::JumpIfTrue));
+ MOZ_ASSERT(loopDepth() > 0);
+
+ MDefinition* value = current->pop();
+
+ BytecodeLocation loopHead = loc.getJumpTarget();
+ MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
+
+ BytecodeLocation successor = loc.next();
+
+ // We can finish the loop now. Use the loophead pc instead of the current pc
+ // because the stack depth at the start of that op matches the current stack
+ // depth (after popping our operand).
+ MBasicBlock* pred = current;
+ if (!startNewBlock(current, loopHead)) {
+ return false;
+ }
+
+ MTest* test = MTest::New(alloc(), value, /* ifTrue = */ current,
+ /* ifFalse = */ nullptr);
+ pred->end(test);
+
+ if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
+ test->setObservedTypes(typesSnapshot->list());
+ }
+
+ if (!addPendingEdge(successor, pred, MTest::FalseBranchIndex)) {
+ return false;
+ }
+
+ return buildBackedge();
+}
+
+bool WarpBuilder::build_JumpIfFalse(BytecodeLocation loc) {
+ return buildTestOp(loc);
+}
+
+bool WarpBuilder::build_JumpIfTrue(BytecodeLocation loc) {
+ return buildTestOp(loc);
+}
+
+bool WarpBuilder::build_And(BytecodeLocation loc) { return buildTestOp(loc); }
+
+bool WarpBuilder::build_Or(BytecodeLocation loc) { return buildTestOp(loc); }
+
+bool WarpBuilder::build_Case(BytecodeLocation loc) { return buildTestOp(loc); }
+
+bool WarpBuilder::build_Default(BytecodeLocation loc) {
+ current->pop();
+ return buildForwardGoto(loc.getJumpTarget());
+}
+
+bool WarpBuilder::build_Coalesce(BytecodeLocation loc) {
+ BytecodeLocation target1 = loc.next();
+ BytecodeLocation target2 = loc.getJumpTarget();
+ MOZ_ASSERT(target2 > target1);
+
+ MDefinition* value = current->peek(-1);
+
+ MInstruction* isNullOrUndefined = MIsNullOrUndefined::New(alloc(), value);
+ current->add(isNullOrUndefined);
+
+ current->end(MTest::New(alloc(), isNullOrUndefined, /* ifTrue = */ nullptr,
+ /* ifFalse = */ nullptr));
+
+ if (!addPendingEdge(target1, current, MTest::TrueBranchIndex)) {
+ return false;
+ }
+ if (!addPendingEdge(target2, current, MTest::FalseBranchIndex)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::buildBackedge() {
+ decLoopDepth();
+
+ MBasicBlock* header = loopStack_.popCopy().header();
+ current->end(MGoto::New(alloc(), header));
+
+ if (!header->setBackedge(current)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::buildForwardGoto(BytecodeLocation target) {
+ current->end(MGoto::New(alloc(), nullptr));
+
+ if (!addPendingEdge(target, current, MGoto::TargetIndex)) {
+ return false;
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_Goto(BytecodeLocation loc) {
+ if (loc.isBackedge()) {
+ return buildBackedge();
+ }
+
+ return buildForwardGoto(loc.getJumpTarget());
+}
+
+bool WarpBuilder::build_IsNullOrUndefined(BytecodeLocation loc) {
+ MDefinition* value = current->peek(-1);
+ auto* isNullOrUndef = MIsNullOrUndefined::New(alloc(), value);
+ current->add(isNullOrUndef);
+ current->push(isNullOrUndef);
+ return true;
+}
+
+bool WarpBuilder::build_DebugCheckSelfHosted(BytecodeLocation loc) {
+#ifdef DEBUG
+ MDefinition* val = current->pop();
+ MDebugCheckSelfHosted* check = MDebugCheckSelfHosted::New(alloc(), val);
+ current->add(check);
+ current->push(check);
+ if (!resumeAfter(check, loc)) {
+ return false;
+ }
+#endif
+ return true;
+}
+
+bool WarpBuilder::build_DynamicImport(BytecodeLocation loc) {
+ MDefinition* options = current->pop();
+ MDefinition* specifier = current->pop();
+ MDynamicImport* ins = MDynamicImport::New(alloc(), specifier, options);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_Not(BytecodeLocation loc) {
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ // If we have CacheIR, we can use it to refine the input before
+ // emitting the MNot.
+ MDefinition* value = current->pop();
+ if (!TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {value})) {
+ return false;
+ }
+ }
+
+ MDefinition* value = current->pop();
+ MNot* ins = MNot::New(alloc(), value);
+ current->add(ins);
+ current->push(ins);
+
+ if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
+ ins->setObservedTypes(typesSnapshot->list());
+ }
+
+ return true;
+}
+
+bool WarpBuilder::build_ToString(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+
+ if (value->type() == MIRType::String) {
+ value->setImplicitlyUsedUnchecked();
+ current->push(value);
+ return true;
+ }
+
+ MToString* ins =
+ MToString::New(alloc(), value, MToString::SideEffectHandling::Supported);
+ current->add(ins);
+ current->push(ins);
+ if (ins->isEffectful()) {
+ return resumeAfter(ins, loc);
+ }
+ return true;
+}
+
+bool WarpBuilder::usesEnvironmentChain() const {
+ return script_->jitScript()->usesEnvironmentChain();
+}
+
+bool WarpBuilder::build_GlobalOrEvalDeclInstantiation(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->isForEval(), "Eval scripts not supported");
+ auto* redeclCheck = MGlobalDeclInstantiation::New(alloc());
+ current->add(redeclCheck);
+ return resumeAfter(redeclCheck, loc);
+}
+
+bool WarpBuilder::build_BindVar(BytecodeLocation) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* env = current->environmentChain();
+ MCallBindVar* ins = MCallBindVar::New(alloc(), env);
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_MutateProto(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ MDefinition* obj = current->peek(-1);
+ MMutateProto* mutate = MMutateProto::New(alloc(), obj, value);
+ current->add(mutate);
+ return resumeAfter(mutate, loc);
+}
+
+MDefinition* WarpBuilder::getCallee() {
+ if (inlineCallInfo()) {
+ return inlineCallInfo()->callee();
+ }
+
+ MInstruction* callee = MCallee::New(alloc());
+ current->add(callee);
+ return callee;
+}
+
+bool WarpBuilder::build_Callee(BytecodeLocation) {
+ MDefinition* callee = getCallee();
+ current->push(callee);
+ return true;
+}
+
+bool WarpBuilder::build_ToAsyncIter(BytecodeLocation loc) {
+ MDefinition* nextMethod = current->pop();
+ MDefinition* iterator = current->pop();
+ MToAsyncIter* ins = MToAsyncIter::New(alloc(), iterator, nextMethod);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_ToPropertyKey(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ return buildIC(loc, CacheKind::ToPropertyKey, {value});
+}
+
+bool WarpBuilder::build_Typeof(BytecodeLocation loc) {
+ MDefinition* input = current->pop();
+
+ if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
+ auto* typeOf = MTypeOf::New(alloc(), input);
+ typeOf->setObservedTypes(typesSnapshot->list());
+ current->add(typeOf);
+
+ auto* ins = MTypeOfName::New(alloc(), typeOf);
+ current->add(ins);
+ current->push(ins);
+ return true;
+ }
+
+ return buildIC(loc, CacheKind::TypeOf, {input});
+}
+
+bool WarpBuilder::build_TypeofExpr(BytecodeLocation loc) {
+ return build_Typeof(loc);
+}
+
+bool WarpBuilder::build_Arguments(BytecodeLocation loc) {
+ auto* snapshot = getOpSnapshot<WarpArguments>(loc);
+ MOZ_ASSERT(info().needsArgsObj());
+ MOZ_ASSERT(snapshot);
+
+ ArgumentsObject* templateObj = snapshot->templateObj();
+ MDefinition* env = current->environmentChain();
+
+ MInstruction* argsObj;
+ if (inlineCallInfo()) {
+ argsObj = MCreateInlinedArgumentsObject::New(
+ alloc(), env, getCallee(), inlineCallInfo()->argv(), templateObj);
+ if (!argsObj) {
+ return false;
+ }
+ } else {
+ argsObj = MCreateArgumentsObject::New(alloc(), env, templateObj);
+ }
+ current->add(argsObj);
+ current->setArgumentsObject(argsObj);
+ current->push(argsObj);
+
+ return true;
+}
+
+bool WarpBuilder::build_ObjWithProto(BytecodeLocation loc) {
+ MDefinition* proto = current->pop();
+ MInstruction* ins = MObjectWithProto::New(alloc(), proto);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+MDefinition* WarpBuilder::walkEnvironmentChain(uint32_t numHops) {
+ MDefinition* env = current->environmentChain();
+
+ for (uint32_t i = 0; i < numHops; i++) {
+ if (!alloc().ensureBallast()) {
+ return nullptr;
+ }
+
+ MInstruction* ins = MEnclosingEnvironment::New(alloc(), env);
+ current->add(ins);
+ env = ins;
+ }
+
+ return env;
+}
+
+bool WarpBuilder::build_GetAliasedVar(BytecodeLocation loc) {
+ EnvironmentCoordinate ec = loc.getEnvironmentCoordinate();
+ MDefinition* obj = walkEnvironmentChain(ec.hops());
+ if (!obj) {
+ return false;
+ }
+
+ MInstruction* load;
+ if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
+ load = MLoadFixedSlot::New(alloc(), obj, ec.slot());
+ } else {
+ MInstruction* slots = MSlots::New(alloc(), obj);
+ current->add(slots);
+
+ uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
+ load = MLoadDynamicSlot::New(alloc(), slots, slot);
+ }
+
+ current->add(load);
+ current->push(load);
+ return true;
+}
+
+bool WarpBuilder::build_SetAliasedVar(BytecodeLocation loc) {
+ EnvironmentCoordinate ec = loc.getEnvironmentCoordinate();
+ MDefinition* val = current->peek(-1);
+ MDefinition* obj = walkEnvironmentChain(ec.hops());
+ if (!obj) {
+ return false;
+ }
+
+ current->add(MPostWriteBarrier::New(alloc(), obj, val));
+
+ MInstruction* store;
+ if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
+ store = MStoreFixedSlot::NewBarriered(alloc(), obj, ec.slot(), val);
+ } else {
+ MInstruction* slots = MSlots::New(alloc(), obj);
+ current->add(slots);
+
+ uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
+ store = MStoreDynamicSlot::NewBarriered(alloc(), slots, slot, val);
+ }
+
+ current->add(store);
+ return resumeAfter(store, loc);
+}
+
+bool WarpBuilder::build_InitAliasedLexical(BytecodeLocation loc) {
+ return build_SetAliasedVar(loc);
+}
+
+bool WarpBuilder::build_EnvCallee(BytecodeLocation loc) {
+ uint32_t numHops = loc.getEnvCalleeNumHops();
+ MDefinition* env = walkEnvironmentChain(numHops);
+ if (!env) {
+ return false;
+ }
+
+ auto* callee = MLoadFixedSlot::New(alloc(), env, CallObject::calleeSlot());
+ current->add(callee);
+ current->push(callee);
+ return true;
+}
+
+bool WarpBuilder::build_Iter(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ return buildIC(loc, CacheKind::GetIterator, {obj});
+}
+
+bool WarpBuilder::build_MoreIter(BytecodeLocation loc) {
+ MDefinition* iter = current->peek(-1);
+ MInstruction* ins = MIteratorMore::New(alloc(), iter);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_EndIter(BytecodeLocation loc) {
+ current->pop(); // Iterator value is not used.
+ MDefinition* iter = current->pop();
+ MInstruction* ins = MIteratorEnd::New(alloc(), iter);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CloseIter(BytecodeLocation loc) {
+ MDefinition* iter = current->pop();
+ return buildIC(loc, CacheKind::CloseIter, {iter});
+}
+
+bool WarpBuilder::build_IsNoIter(BytecodeLocation) {
+ MDefinition* def = current->peek(-1);
+ MOZ_ASSERT(def->isIteratorMore());
+ MInstruction* ins = MIsNoIter::New(alloc(), def);
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::transpileCall(BytecodeLocation loc,
+ const WarpCacheIR* cacheIRSnapshot,
+ CallInfo* callInfo) {
+ // Synthesize the constant number of arguments for this call op.
+ auto* argc = MConstant::New(alloc(), Int32Value(callInfo->argc()));
+ current->add(argc);
+
+ return TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {argc}, callInfo);
+}
+
+void WarpBuilder::buildCreateThis(CallInfo& callInfo) {
+ MOZ_ASSERT(callInfo.constructing());
+
+ // Inline the this-object allocation on the caller-side.
+ MDefinition* callee = callInfo.callee();
+ MDefinition* newTarget = callInfo.getNewTarget();
+ auto* createThis = MCreateThis::New(alloc(), callee, newTarget);
+ current->add(createThis);
+ callInfo.thisArg()->setImplicitlyUsedUnchecked();
+ callInfo.setThis(createThis);
+}
+
+bool WarpBuilder::buildCallOp(BytecodeLocation loc) {
+ uint32_t argc = loc.getCallArgc();
+ JSOp op = loc.getOp();
+ bool constructing = IsConstructOp(op);
+ bool ignoresReturnValue = (op == JSOp::CallIgnoresRv || loc.resultIsPopped());
+
+ CallInfo callInfo(alloc(), constructing, ignoresReturnValue);
+ if (!callInfo.init(current, argc)) {
+ return false;
+ }
+
+ if (const auto* inliningSnapshot = getOpSnapshot<WarpInlinedCall>(loc)) {
+ // Transpile the CacheIR to generate the correct guards before
+ // inlining. In this case, CacheOp::CallInlinedFunction updates
+ // the CallInfo, but does not generate a call.
+ callInfo.markAsInlined();
+ if (!transpileCall(loc, inliningSnapshot->cacheIRSnapshot(), &callInfo)) {
+ return false;
+ }
+
+ // Generate the body of the inlined function.
+ return buildInlinedCall(loc, inliningSnapshot, callInfo);
+ }
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ return transpileCall(loc, cacheIRSnapshot, &callInfo);
+ }
+
+ if (getOpSnapshot<WarpBailout>(loc)) {
+ callInfo.setImplicitlyUsedUnchecked();
+ return buildBailoutForColdIC(loc, CacheKind::Call);
+ }
+
+ bool needsThisCheck = false;
+ if (callInfo.constructing()) {
+ buildCreateThis(callInfo);
+ needsThisCheck = true;
+ }
+
+ MCall* call = makeCall(callInfo, needsThisCheck);
+ if (!call) {
+ return false;
+ }
+
+ current->add(call);
+ current->push(call);
+ return resumeAfter(call, loc);
+}
+
+bool WarpBuilder::build_Call(BytecodeLocation loc) { return buildCallOp(loc); }
+
+bool WarpBuilder::build_CallContent(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_CallIgnoresRv(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_CallIter(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_CallContentIter(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_New(BytecodeLocation loc) { return buildCallOp(loc); }
+
+bool WarpBuilder::build_NewContent(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_SuperCall(BytecodeLocation loc) {
+ return buildCallOp(loc);
+}
+
+bool WarpBuilder::build_FunctionThis(BytecodeLocation loc) {
+ MOZ_ASSERT(info().hasFunMaybeLazy());
+
+ if (script_->strict()) {
+ // No need to wrap primitive |this| in strict mode.
+ current->pushSlot(info().thisSlot());
+ return true;
+ }
+
+ MOZ_ASSERT(!script_->hasNonSyntacticScope(),
+ "WarpOracle should have aborted compilation");
+
+ MDefinition* def = current->getSlot(info().thisSlot());
+ JSObject* globalThis = snapshot().globalLexicalEnvThis();
+
+ auto* thisObj = MBoxNonStrictThis::New(alloc(), def, globalThis);
+ current->add(thisObj);
+ current->push(thisObj);
+
+ return true;
+}
+
+bool WarpBuilder::build_GlobalThis(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+ JSObject* obj = snapshot().globalLexicalEnvThis();
+ pushConstant(ObjectValue(*obj));
+ return true;
+}
+
+MConstant* WarpBuilder::globalLexicalEnvConstant() {
+ JSObject* globalLexical = snapshot().globalLexicalEnv();
+ return constant(ObjectValue(*globalLexical));
+}
+
+bool WarpBuilder::build_GetName(BytecodeLocation loc) {
+ MDefinition* env = current->environmentChain();
+ return buildIC(loc, CacheKind::GetName, {env});
+}
+
+bool WarpBuilder::build_GetGName(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ // Try to optimize undefined/NaN/Infinity.
+ PropertyName* name = loc.getPropertyName(script_);
+ const JSAtomState& names = mirGen().runtime->names();
+
+ if (name == names.undefined) {
+ pushConstant(UndefinedValue());
+ return true;
+ }
+ if (name == names.NaN) {
+ pushConstant(JS::NaNValue());
+ return true;
+ }
+ if (name == names.Infinity) {
+ pushConstant(JS::InfinityValue());
+ return true;
+ }
+
+ MDefinition* env = globalLexicalEnvConstant();
+ return buildIC(loc, CacheKind::GetName, {env});
+}
+
+bool WarpBuilder::build_BindName(BytecodeLocation loc) {
+ MDefinition* env = current->environmentChain();
+ return buildIC(loc, CacheKind::BindName, {env});
+}
+
+bool WarpBuilder::build_BindGName(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ if (const auto* snapshot = getOpSnapshot<WarpBindGName>(loc)) {
+ JSObject* globalEnv = snapshot->globalEnv();
+ pushConstant(ObjectValue(*globalEnv));
+ return true;
+ }
+
+ MDefinition* env = globalLexicalEnvConstant();
+ return buildIC(loc, CacheKind::BindName, {env});
+}
+
+bool WarpBuilder::build_GetProp(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ return buildIC(loc, CacheKind::GetProp, {val});
+}
+
+bool WarpBuilder::build_GetElem(BytecodeLocation loc) {
+ MDefinition* id = current->pop();
+ MDefinition* val = current->pop();
+ return buildIC(loc, CacheKind::GetElem, {val, id});
+}
+
+bool WarpBuilder::build_SetProp(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* obj = current->pop();
+ current->push(val);
+ return buildIC(loc, CacheKind::SetProp, {obj, val});
+}
+
+bool WarpBuilder::build_StrictSetProp(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_SetName(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_StrictSetName(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_SetGName(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_StrictSetGName(BytecodeLocation loc) {
+ return build_SetProp(loc);
+}
+
+bool WarpBuilder::build_InitGLexical(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+
+ MDefinition* globalLexical = globalLexicalEnvConstant();
+ MDefinition* val = current->peek(-1);
+
+ return buildIC(loc, CacheKind::SetProp, {globalLexical, val});
+}
+
+bool WarpBuilder::build_SetElem(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* id = current->pop();
+ MDefinition* obj = current->pop();
+ current->push(val);
+ return buildIC(loc, CacheKind::SetElem, {obj, id, val});
+}
+
+bool WarpBuilder::build_StrictSetElem(BytecodeLocation loc) {
+ return build_SetElem(loc);
+}
+
+bool WarpBuilder::build_DelProp(BytecodeLocation loc) {
+ PropertyName* name = loc.getPropertyName(script_);
+ MDefinition* obj = current->pop();
+ bool strict = loc.getOp() == JSOp::StrictDelProp;
+
+ MInstruction* ins = MDeleteProperty::New(alloc(), obj, name, strict);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_StrictDelProp(BytecodeLocation loc) {
+ return build_DelProp(loc);
+}
+
+bool WarpBuilder::build_DelElem(BytecodeLocation loc) {
+ MDefinition* id = current->pop();
+ MDefinition* obj = current->pop();
+ bool strict = loc.getOp() == JSOp::StrictDelElem;
+
+ MInstruction* ins = MDeleteElement::New(alloc(), obj, id, strict);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_StrictDelElem(BytecodeLocation loc) {
+ return build_DelElem(loc);
+}
+
+bool WarpBuilder::build_SetFunName(BytecodeLocation loc) {
+ FunctionPrefixKind prefixKind = loc.getFunctionPrefixKind();
+ MDefinition* name = current->pop();
+ MDefinition* fun = current->pop();
+
+ MSetFunName* ins = MSetFunName::New(alloc(), fun, name, uint8_t(prefixKind));
+ current->add(ins);
+ current->push(fun);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_PushLexicalEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* env = current->environmentChain();
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_PushClassBodyEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpClassBodyEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* env = current->environmentChain();
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* ins = MNewClassBodyEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_PopLexicalEnv(BytecodeLocation) {
+ MDefinition* enclosingEnv = walkEnvironmentChain(1);
+ if (!enclosingEnv) {
+ return false;
+ }
+ current->setEnvironmentChain(enclosingEnv);
+ return true;
+}
+
+bool WarpBuilder::build_FreshenLexicalEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* enclosingEnv = walkEnvironmentChain(1);
+ if (!enclosingEnv) {
+ return false;
+ }
+
+ MDefinition* env = current->environmentChain();
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* templateObj = snapshot->templateObj();
+ auto* scope = &templateObj->scope();
+ MOZ_ASSERT(scope->hasEnvironment());
+
+ auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(
+ MAssertCanElidePostWriteBarrier::New(alloc(), ins, enclosingEnv));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(),
+ enclosingEnv));
+
+ // Copy environment slots.
+ MSlots* envSlots = nullptr;
+ MSlots* slots = nullptr;
+ for (BindingIter iter(scope); iter; iter++) {
+ auto loc = iter.location();
+ if (loc.kind() != BindingLocation::Kind::Environment) {
+ MOZ_ASSERT(loc.kind() == BindingLocation::Kind::Frame);
+ continue;
+ }
+
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ uint32_t slot = loc.slot();
+ uint32_t numFixedSlots = templateObj->numFixedSlots();
+ if (slot >= numFixedSlots) {
+ if (!envSlots) {
+ envSlots = MSlots::New(alloc(), env);
+ current->add(envSlots);
+ }
+ if (!slots) {
+ slots = MSlots::New(alloc(), ins);
+ current->add(slots);
+ }
+
+ uint32_t dynamicSlot = slot - numFixedSlots;
+
+ auto* load = MLoadDynamicSlot::New(alloc(), envSlots, dynamicSlot);
+ current->add(load);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, load));
+#endif
+
+ current->add(
+ MStoreDynamicSlot::NewUnbarriered(alloc(), slots, dynamicSlot, load));
+ } else {
+ auto* load = MLoadFixedSlot::New(alloc(), env, slot);
+ current->add(load);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, load));
+#endif
+
+ current->add(MStoreFixedSlot::NewUnbarriered(alloc(), ins, slot, load));
+ }
+ }
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_RecreateLexicalEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* enclosingEnv = walkEnvironmentChain(1);
+ if (!enclosingEnv) {
+ return false;
+ }
+
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(
+ MAssertCanElidePostWriteBarrier::New(alloc(), ins, enclosingEnv));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(),
+ enclosingEnv));
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_PushVarEnv(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ const auto* snapshot = getOpSnapshot<WarpVarEnvironment>(loc);
+ MOZ_ASSERT(snapshot);
+
+ MDefinition* env = current->environmentChain();
+ MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
+
+ auto* ins = MNewVarEnvironmentObject::New(alloc(), templateCst);
+ current->add(ins);
+
+#ifdef DEBUG
+ // Assert in debug mode we can elide the post write barrier.
+ current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
+#endif
+
+ // Initialize the object's reserved slots. No post barrier is needed here,
+ // for the same reason as in buildNamedLambdaEnv.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
+
+ current->setEnvironmentChain(ins);
+ return true;
+}
+
+bool WarpBuilder::build_ImplicitThis(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ PropertyName* name = loc.getPropertyName(script_);
+ MDefinition* env = current->environmentChain();
+
+ auto* ins = MImplicitThis::New(alloc(), env, name);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CheckClassHeritage(BytecodeLocation loc) {
+ MDefinition* def = current->pop();
+ auto* ins = MCheckClassHeritage::New(alloc(), def);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CheckThis(BytecodeLocation loc) {
+ MDefinition* def = current->pop();
+ auto* ins = MCheckThis::New(alloc(), def);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CheckThisReinit(BytecodeLocation loc) {
+ MDefinition* def = current->pop();
+ auto* ins = MCheckThisReinit::New(alloc(), def);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_Generator(BytecodeLocation loc) {
+ MDefinition* callee = getCallee();
+ MDefinition* environmentChain = current->environmentChain();
+ MDefinition* argsObj = info().needsArgsObj() ? current->argumentsObject()
+ : constant(Int32Value(0));
+
+ MGenerator* generator =
+ MGenerator::New(alloc(), callee, environmentChain, argsObj);
+
+ current->add(generator);
+ current->push(generator);
+ return resumeAfter(generator, loc);
+}
+
+bool WarpBuilder::build_AfterYield(BytecodeLocation loc) {
+ // Unreachable blocks don't need to generate a bail.
+ if (hasTerminatedBlock()) {
+ return true;
+ }
+
+ // This comes after a yield, which we generate as a return,
+ // so we know this should be unreachable code.
+ //
+ // We emit an unreachable bail for this, which will assert if we
+ // ever execute this.
+ //
+ // An Unreachable bail, instead of MUnreachable, because MUnreachable
+ // is a control instruction, and injecting it in the middle of a block
+ // causes various graph state assertions to fail.
+ MBail* bail = MBail::New(alloc(), BailoutKind::Unreachable);
+ current->add(bail);
+
+ return true;
+}
+
+bool WarpBuilder::build_FinalYieldRval(BytecodeLocation loc) {
+ MDefinition* gen = current->pop();
+
+ auto setSlotNull = [this, gen](size_t slot) {
+ auto* ins = MStoreFixedSlot::NewBarriered(alloc(), gen, slot,
+ constant(NullValue()));
+ current->add(ins);
+ };
+
+ // Close the generator
+ setSlotNull(AbstractGeneratorObject::calleeSlot());
+ setSlotNull(AbstractGeneratorObject::envChainSlot());
+ setSlotNull(AbstractGeneratorObject::argsObjectSlot());
+ setSlotNull(AbstractGeneratorObject::stackStorageSlot());
+ setSlotNull(AbstractGeneratorObject::resumeIndexSlot());
+
+ // Return
+ return build_RetRval(loc);
+}
+
+bool WarpBuilder::build_AsyncResolve(BytecodeLocation loc) {
+ MDefinition* generator = current->pop();
+ MDefinition* valueOrReason = current->pop();
+ auto resolveKind = loc.getAsyncFunctionResolveKind();
+
+ MAsyncResolve* resolve =
+ MAsyncResolve::New(alloc(), generator, valueOrReason, resolveKind);
+ current->add(resolve);
+ current->push(resolve);
+ return resumeAfter(resolve, loc);
+}
+
+bool WarpBuilder::build_ResumeKind(BytecodeLocation loc) {
+ GeneratorResumeKind resumeKind = loc.resumeKind();
+
+ current->push(constant(Int32Value(static_cast<int32_t>(resumeKind))));
+ return true;
+}
+
+bool WarpBuilder::build_CheckResumeKind(BytecodeLocation loc) {
+ // Outside of `yield*`, this is normally unreachable code in Warp,
+ // so we just manipulate the stack appropriately to ensure correct
+ // MIR generation.
+ //
+ // However, `yield*` emits a forced generator return which can be
+ // warp compiled, so in order to correctly handle these semantics
+ // we also generate a bailout, so that the forced generator return
+ // runs in baseline.
+ MDefinition* resumeKind = current->pop();
+ MDefinition* gen = current->pop();
+ MDefinition* rval = current->peek(-1);
+
+ // Mark operands as implicitly used.
+ resumeKind->setImplicitlyUsedUnchecked();
+ gen->setImplicitlyUsedUnchecked();
+ rval->setImplicitlyUsedUnchecked();
+
+ // Bail out if we encounter CheckResumeKind.
+ MBail* bail = MBail::New(alloc(), BailoutKind::Inevitable);
+ current->add(bail);
+ current->setAlwaysBails();
+
+ return true;
+}
+
+bool WarpBuilder::build_CanSkipAwait(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+
+ MCanSkipAwait* canSkip = MCanSkipAwait::New(alloc(), val);
+ current->add(canSkip);
+
+ current->push(val);
+ current->push(canSkip);
+
+ return resumeAfter(canSkip, loc);
+}
+
+bool WarpBuilder::build_MaybeExtractAwaitValue(BytecodeLocation loc) {
+ MDefinition* canSkip = current->pop();
+ MDefinition* value = current->pop();
+
+ MMaybeExtractAwaitValue* extracted =
+ MMaybeExtractAwaitValue::New(alloc(), value, canSkip);
+ current->add(extracted);
+
+ current->push(extracted);
+ current->push(canSkip);
+
+ return resumeAfter(extracted, loc);
+}
+
+bool WarpBuilder::build_InitialYield(BytecodeLocation loc) {
+ MDefinition* gen = current->pop();
+ return buildSuspend(loc, gen, gen);
+}
+
+bool WarpBuilder::build_Await(BytecodeLocation loc) {
+ MDefinition* gen = current->pop();
+ MDefinition* promiseOrGenerator = current->pop();
+
+ return buildSuspend(loc, gen, promiseOrGenerator);
+}
+bool WarpBuilder::build_Yield(BytecodeLocation loc) { return build_Await(loc); }
+
+bool WarpBuilder::buildSuspend(BytecodeLocation loc, MDefinition* gen,
+ MDefinition* retVal) {
+ // If required, unbox the generator object explicitly and infallibly.
+ //
+ // This is done to avoid fuzz-bugs where ApplyTypeInformation does the
+ // unboxing, and generates fallible unboxes which can lead to torn object
+ // state due to `bailAfter`.
+ MDefinition* genObj = gen;
+ if (genObj->type() != MIRType::Object) {
+ auto* unbox =
+ MUnbox::New(alloc(), gen, MIRType::Object, MUnbox::Mode::Infallible);
+ current->add(unbox);
+
+ genObj = unbox;
+ }
+
+ int32_t slotsToCopy = current->stackDepth() - info().firstLocalSlot();
+ MOZ_ASSERT(slotsToCopy >= 0);
+ if (slotsToCopy > 0) {
+ auto* arrayObj = MLoadFixedSlotAndUnbox::New(
+ alloc(), genObj, AbstractGeneratorObject::stackStorageSlot(),
+ MUnbox::Mode::Infallible, MIRType::Object);
+ current->add(arrayObj);
+
+ auto* stackStorage = MElements::New(alloc(), arrayObj);
+ current->add(stackStorage);
+
+ for (int32_t i = 0; i < slotsToCopy; i++) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+ // Use peekUnchecked because we're also writing out the argument slots
+ int32_t peek = -slotsToCopy + i;
+ MDefinition* stackElem = current->peekUnchecked(peek);
+ auto* store = MStoreElement::NewUnbarriered(
+ alloc(), stackStorage, constant(Int32Value(i)), stackElem,
+ /* needsHoleCheck = */ false);
+
+ current->add(store);
+ current->add(MPostWriteBarrier::New(alloc(), arrayObj, stackElem));
+ }
+
+ auto* len = constant(Int32Value(slotsToCopy - 1));
+
+ auto* setInitLength =
+ MSetInitializedLength::New(alloc(), stackStorage, len);
+ current->add(setInitLength);
+
+ auto* setLength = MSetArrayLength::New(alloc(), stackStorage, len);
+ current->add(setLength);
+ }
+
+ // Update Generator Object state
+ uint32_t resumeIndex = loc.getResumeIndex();
+
+ // This store is unbarriered, as it's only ever storing an integer, and as
+ // such doesn't partake of object tracing.
+ current->add(MStoreFixedSlot::NewUnbarriered(
+ alloc(), genObj, AbstractGeneratorObject::resumeIndexSlot(),
+ constant(Int32Value(resumeIndex))));
+
+ // This store is barriered because it stores an object value.
+ current->add(MStoreFixedSlot::NewBarriered(
+ alloc(), genObj, AbstractGeneratorObject::envChainSlot(),
+ current->environmentChain()));
+
+ current->add(
+ MPostWriteBarrier::New(alloc(), genObj, current->environmentChain()));
+
+ // GeneratorReturn will return from the method, however to support MIR
+ // generation isn't treated like the end of a block
+ MGeneratorReturn* ret = MGeneratorReturn::New(alloc(), retVal);
+ current->add(ret);
+
+ // To ensure the rest of the MIR generation looks correct, fill the stack with
+ // the appropriately typed MUnreachable's for the stack pushes from this
+ // opcode.
+ auto* unreachableResumeKind =
+ MUnreachableResult::New(alloc(), MIRType::Int32);
+ current->add(unreachableResumeKind);
+ current->push(unreachableResumeKind);
+
+ auto* unreachableGenerator =
+ MUnreachableResult::New(alloc(), MIRType::Object);
+ current->add(unreachableGenerator);
+ current->push(unreachableGenerator);
+
+ auto* unreachableRval = MUnreachableResult::New(alloc(), MIRType::Value);
+ current->add(unreachableRval);
+ current->push(unreachableRval);
+
+ return true;
+}
+
+bool WarpBuilder::build_AsyncAwait(BytecodeLocation loc) {
+ MDefinition* gen = current->pop();
+ MDefinition* value = current->pop();
+
+ MAsyncAwait* asyncAwait = MAsyncAwait::New(alloc(), value, gen);
+ current->add(asyncAwait);
+ current->push(asyncAwait);
+ return resumeAfter(asyncAwait, loc);
+}
+
+bool WarpBuilder::build_CheckReturn(BytecodeLocation loc) {
+ MOZ_ASSERT(!script_->noScriptRval());
+
+ MDefinition* returnValue = current->getSlot(info().returnValueSlot());
+ MDefinition* thisValue = current->pop();
+
+ auto* ins = MCheckReturn::New(alloc(), returnValue, thisValue);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+void WarpBuilder::buildCheckLexicalOp(BytecodeLocation loc) {
+ JSOp op = loc.getOp();
+ MOZ_ASSERT(op == JSOp::CheckLexical || op == JSOp::CheckAliasedLexical);
+
+ MDefinition* input = current->pop();
+ MInstruction* lexicalCheck = MLexicalCheck::New(alloc(), input);
+ current->add(lexicalCheck);
+ current->push(lexicalCheck);
+
+ if (snapshot().bailoutInfo().failedLexicalCheck()) {
+ lexicalCheck->setNotMovable();
+ }
+
+ if (op == JSOp::CheckLexical) {
+ // Set the local slot so that a subsequent GetLocal without a CheckLexical
+ // (the frontend can elide lexical checks) doesn't let a definition with
+ // MIRType::MagicUninitializedLexical escape to arbitrary MIR instructions.
+ // Note that in this case the GetLocal would be unreachable because we throw
+ // an exception here, but we still generate MIR instructions for it.
+ uint32_t slot = info().localSlot(loc.local());
+ current->setSlot(slot, lexicalCheck);
+ }
+}
+
+bool WarpBuilder::build_CheckLexical(BytecodeLocation loc) {
+ buildCheckLexicalOp(loc);
+ return true;
+}
+
+bool WarpBuilder::build_CheckAliasedLexical(BytecodeLocation loc) {
+ buildCheckLexicalOp(loc);
+ return true;
+}
+
+bool WarpBuilder::build_InitHomeObject(BytecodeLocation loc) {
+ MDefinition* homeObject = current->pop();
+ MDefinition* function = current->pop();
+
+ current->add(MPostWriteBarrier::New(alloc(), function, homeObject));
+
+ auto* ins = MInitHomeObject::New(alloc(), function, homeObject);
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_SuperBase(BytecodeLocation) {
+ MDefinition* callee = current->pop();
+
+ auto* homeObject = MHomeObject::New(alloc(), callee);
+ current->add(homeObject);
+
+ auto* superBase = MHomeObjectSuperBase::New(alloc(), homeObject);
+ current->add(superBase);
+ current->push(superBase);
+ return true;
+}
+
+bool WarpBuilder::build_SuperFun(BytecodeLocation) {
+ MDefinition* callee = current->pop();
+ auto* ins = MSuperFunction::New(alloc(), callee);
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_BuiltinObject(BytecodeLocation loc) {
+ if (auto* snapshot = getOpSnapshot<WarpBuiltinObject>(loc)) {
+ JSObject* builtin = snapshot->builtin();
+ pushConstant(ObjectValue(*builtin));
+ return true;
+ }
+
+ auto kind = loc.getBuiltinObjectKind();
+ auto* ins = MBuiltinObject::New(alloc(), kind);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_GetIntrinsic(BytecodeLocation loc) {
+ if (auto* snapshot = getOpSnapshot<WarpGetIntrinsic>(loc)) {
+ Value intrinsic = snapshot->intrinsic();
+ pushConstant(intrinsic);
+ return true;
+ }
+
+ PropertyName* name = loc.getPropertyName(script_);
+ MCallGetIntrinsicValue* ins = MCallGetIntrinsicValue::New(alloc(), name);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_ImportMeta(BytecodeLocation loc) {
+ ModuleObject* moduleObj = scriptSnapshot()->moduleObject();
+ MOZ_ASSERT(moduleObj);
+
+ MModuleMetadata* ins = MModuleMetadata::New(alloc(), moduleObj);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_CallSiteObj(BytecodeLocation loc) {
+ return build_Object(loc);
+}
+
+bool WarpBuilder::build_NewArray(BytecodeLocation loc) {
+ return buildIC(loc, CacheKind::NewArray, {});
+}
+
+bool WarpBuilder::build_NewObject(BytecodeLocation loc) {
+ return buildIC(loc, CacheKind::NewObject, {});
+}
+
+bool WarpBuilder::build_NewInit(BytecodeLocation loc) {
+ return build_NewObject(loc);
+}
+
+bool WarpBuilder::build_Object(BytecodeLocation loc) {
+ JSObject* obj = loc.getObject(script_);
+ MConstant* objConst = constant(ObjectValue(*obj));
+
+ current->push(objConst);
+ return true;
+}
+
+bool WarpBuilder::buildInitPropGetterSetterOp(BytecodeLocation loc) {
+ PropertyName* name = loc.getPropertyName(script_);
+ MDefinition* value = current->pop();
+ MDefinition* obj = current->peek(-1);
+
+ auto* ins = MInitPropGetterSetter::New(alloc(), obj, value, name);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_InitPropGetter(BytecodeLocation loc) {
+ return buildInitPropGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitPropSetter(BytecodeLocation loc) {
+ return buildInitPropGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenPropGetter(BytecodeLocation loc) {
+ return buildInitPropGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenPropSetter(BytecodeLocation loc) {
+ return buildInitPropGetterSetterOp(loc);
+}
+
+bool WarpBuilder::buildInitElemGetterSetterOp(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ MDefinition* id = current->pop();
+ MDefinition* obj = current->peek(-1);
+
+ auto* ins = MInitElemGetterSetter::New(alloc(), obj, id, value);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_InitElemGetter(BytecodeLocation loc) {
+ return buildInitElemGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitElemSetter(BytecodeLocation loc) {
+ return buildInitElemGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenElemGetter(BytecodeLocation loc) {
+ return buildInitElemGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenElemSetter(BytecodeLocation loc) {
+ return buildInitElemGetterSetterOp(loc);
+}
+
+bool WarpBuilder::build_In(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ MDefinition* id = current->pop();
+ return buildIC(loc, CacheKind::In, {id, obj});
+}
+
+bool WarpBuilder::build_HasOwn(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ MDefinition* id = current->pop();
+ return buildIC(loc, CacheKind::HasOwn, {id, obj});
+}
+
+bool WarpBuilder::build_CheckPrivateField(BytecodeLocation loc) {
+ MDefinition* id = current->peek(-1);
+ MDefinition* obj = current->peek(-2);
+ return buildIC(loc, CacheKind::CheckPrivateField, {obj, id});
+}
+
+bool WarpBuilder::build_NewPrivateName(BytecodeLocation loc) {
+ JSAtom* name = loc.getAtom(script_);
+
+ auto* ins = MNewPrivateName::New(alloc(), name);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_Instanceof(BytecodeLocation loc) {
+ MDefinition* rhs = current->pop();
+ MDefinition* obj = current->pop();
+ return buildIC(loc, CacheKind::InstanceOf, {obj, rhs});
+}
+
+bool WarpBuilder::build_NewTarget(BytecodeLocation loc) {
+ MOZ_ASSERT(script_->isFunction());
+ MOZ_ASSERT(info().hasFunMaybeLazy());
+ MOZ_ASSERT(!scriptSnapshot()->isArrowFunction());
+
+ if (inlineCallInfo()) {
+ if (inlineCallInfo()->constructing()) {
+ current->push(inlineCallInfo()->getNewTarget());
+ } else {
+ pushConstant(UndefinedValue());
+ }
+ return true;
+ }
+
+ MNewTarget* ins = MNewTarget::New(alloc());
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_CheckIsObj(BytecodeLocation loc) {
+ CheckIsObjectKind kind = loc.getCheckIsObjectKind();
+
+ MDefinition* toCheck = current->peek(-1);
+ if (toCheck->type() == MIRType::Object) {
+ toCheck->setImplicitlyUsedUnchecked();
+ return true;
+ }
+
+ MDefinition* val = current->pop();
+ MCheckIsObj* ins = MCheckIsObj::New(alloc(), val, uint8_t(kind));
+ current->add(ins);
+ current->push(ins);
+ return true;
+}
+
+bool WarpBuilder::build_CheckObjCoercible(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MCheckObjCoercible* ins = MCheckObjCoercible::New(alloc(), val);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+MInstruction* WarpBuilder::buildLoadSlot(MDefinition* obj,
+ uint32_t numFixedSlots,
+ uint32_t slot) {
+ if (slot < numFixedSlots) {
+ MLoadFixedSlot* load = MLoadFixedSlot::New(alloc(), obj, slot);
+ current->add(load);
+ return load;
+ }
+
+ MSlots* slots = MSlots::New(alloc(), obj);
+ current->add(slots);
+
+ MLoadDynamicSlot* load =
+ MLoadDynamicSlot::New(alloc(), slots, slot - numFixedSlots);
+ current->add(load);
+ return load;
+}
+
+bool WarpBuilder::build_GetImport(BytecodeLocation loc) {
+ auto* snapshot = getOpSnapshot<WarpGetImport>(loc);
+
+ ModuleEnvironmentObject* targetEnv = snapshot->targetEnv();
+
+ // Load the target environment slot.
+ MConstant* obj = constant(ObjectValue(*targetEnv));
+ auto* load = buildLoadSlot(obj, snapshot->numFixedSlots(), snapshot->slot());
+
+ if (snapshot->needsLexicalCheck()) {
+ // TODO: IonBuilder has code to mark non-movable. See buildCheckLexicalOp.
+ MInstruction* lexicalCheck = MLexicalCheck::New(alloc(), load);
+ current->add(lexicalCheck);
+ current->push(lexicalCheck);
+ } else {
+ current->push(load);
+ }
+
+ return true;
+}
+
+bool WarpBuilder::build_GetPropSuper(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ MDefinition* receiver = current->pop();
+ return buildIC(loc, CacheKind::GetPropSuper, {obj, receiver});
+}
+
+bool WarpBuilder::build_GetElemSuper(BytecodeLocation loc) {
+ MDefinition* obj = current->pop();
+ MDefinition* id = current->pop();
+ MDefinition* receiver = current->pop();
+ return buildIC(loc, CacheKind::GetElemSuper, {obj, id, receiver});
+}
+
+bool WarpBuilder::build_InitProp(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* obj = current->peek(-1);
+ return buildIC(loc, CacheKind::SetProp, {obj, val});
+}
+
+bool WarpBuilder::build_InitLockedProp(BytecodeLocation loc) {
+ return build_InitProp(loc);
+}
+
+bool WarpBuilder::build_InitHiddenProp(BytecodeLocation loc) {
+ return build_InitProp(loc);
+}
+
+bool WarpBuilder::build_InitElem(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* id = current->pop();
+ MDefinition* obj = current->peek(-1);
+ return buildIC(loc, CacheKind::SetElem, {obj, id, val});
+}
+
+bool WarpBuilder::build_InitLockedElem(BytecodeLocation loc) {
+ return build_InitElem(loc);
+}
+
+bool WarpBuilder::build_InitHiddenElem(BytecodeLocation loc) {
+ return build_InitElem(loc);
+}
+
+bool WarpBuilder::build_InitElemArray(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* obj = current->peek(-1);
+
+ // Note: getInitElemArrayIndex asserts the index fits in int32_t.
+ uint32_t index = loc.getInitElemArrayIndex();
+ MConstant* indexConst = constant(Int32Value(index));
+
+ // Note: InitArrayElemOperation asserts the index does not exceed the array's
+ // dense element capacity.
+
+ auto* elements = MElements::New(alloc(), obj);
+ current->add(elements);
+
+ if (val->type() == MIRType::MagicHole) {
+ val->setImplicitlyUsedUnchecked();
+ auto* store = MStoreHoleValueElement::New(alloc(), elements, indexConst);
+ current->add(store);
+ } else {
+ current->add(MPostWriteBarrier::New(alloc(), obj, val));
+ auto* store =
+ MStoreElement::NewUnbarriered(alloc(), elements, indexConst, val,
+ /* needsHoleCheck = */ false);
+ current->add(store);
+ }
+
+ auto* setLength = MSetInitializedLength::New(alloc(), elements, indexConst);
+ current->add(setLength);
+
+ return resumeAfter(setLength, loc);
+}
+
+bool WarpBuilder::build_InitElemInc(BytecodeLocation loc) {
+ MDefinition* val = current->pop();
+ MDefinition* index = current->pop();
+ MDefinition* obj = current->peek(-1);
+
+ // Push index + 1.
+ MConstant* constOne = constant(Int32Value(1));
+ MAdd* nextIndex = MAdd::New(alloc(), index, constOne, TruncateKind::Truncate);
+ current->add(nextIndex);
+ current->push(nextIndex);
+
+ return buildIC(loc, CacheKind::SetElem, {obj, index, val});
+}
+
+bool WarpBuilder::build_Lambda(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* env = current->environmentChain();
+
+ JSFunction* fun = loc.getFunction(script_);
+ MConstant* funConst = constant(ObjectValue(*fun));
+
+ auto* ins = MLambda::New(alloc(), env, funConst);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_FunWithProto(BytecodeLocation loc) {
+ MOZ_ASSERT(usesEnvironmentChain());
+
+ MDefinition* proto = current->pop();
+ MDefinition* env = current->environmentChain();
+
+ JSFunction* fun = loc.getFunction(script_);
+ MConstant* funConst = constant(ObjectValue(*fun));
+
+ auto* ins = MFunctionWithProto::New(alloc(), env, proto, funConst);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+}
+
+bool WarpBuilder::build_SpreadCall(BytecodeLocation loc) {
+ bool constructing = false;
+ CallInfo callInfo(alloc(), constructing, loc.resultIsPopped());
+ callInfo.initForSpreadCall(current);
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ return transpileCall(loc, cacheIRSnapshot, &callInfo);
+ }
+
+ bool needsThisCheck = false;
+ MInstruction* call = makeSpreadCall(callInfo, needsThisCheck);
+ if (!call) {
+ return false;
+ }
+ call->setBailoutKind(BailoutKind::TooManyArguments);
+ current->add(call);
+ current->push(call);
+ return resumeAfter(call, loc);
+}
+
+bool WarpBuilder::build_SpreadNew(BytecodeLocation loc) {
+ bool constructing = true;
+ CallInfo callInfo(alloc(), constructing, loc.resultIsPopped());
+ callInfo.initForSpreadCall(current);
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ return transpileCall(loc, cacheIRSnapshot, &callInfo);
+ }
+
+ buildCreateThis(callInfo);
+
+ bool needsThisCheck = true;
+ MInstruction* call = makeSpreadCall(callInfo, needsThisCheck);
+ if (!call) {
+ return false;
+ }
+ call->setBailoutKind(BailoutKind::TooManyArguments);
+ current->add(call);
+ current->push(call);
+ return resumeAfter(call, loc);
+}
+
+bool WarpBuilder::build_SpreadSuperCall(BytecodeLocation loc) {
+ return build_SpreadNew(loc);
+}
+
+bool WarpBuilder::build_OptimizeSpreadCall(BytecodeLocation loc) {
+ MDefinition* value = current->pop();
+ return buildIC(loc, CacheKind::OptimizeSpreadCall, {value});
+}
+
+bool WarpBuilder::build_Debugger(BytecodeLocation loc) {
+ // The |debugger;| statement will bail out to Baseline if the realm is a
+ // debuggee realm with an onDebuggerStatement hook.
+ MDebugger* debugger = MDebugger::New(alloc());
+ current->add(debugger);
+ return resumeAfter(debugger, loc);
+}
+
+bool WarpBuilder::build_TableSwitch(BytecodeLocation loc) {
+ int32_t low = loc.getTableSwitchLow();
+ int32_t high = loc.getTableSwitchHigh();
+ size_t numCases = high - low + 1;
+
+ MDefinition* input = current->pop();
+ MTableSwitch* tableswitch = MTableSwitch::New(alloc(), input, low, high);
+ current->end(tableswitch);
+
+ // Table mapping from target bytecode offset to MTableSwitch successor index.
+ // This prevents adding multiple predecessor/successor edges to the same
+ // target block, which isn't valid in MIR.
+ using TargetToSuccessorMap =
+ InlineMap<uint32_t, uint32_t, 8, DefaultHasher<uint32_t>,
+ SystemAllocPolicy>;
+ TargetToSuccessorMap targetToSuccessor;
+
+ // Create |default| edge.
+ {
+ BytecodeLocation defaultLoc = loc.getTableSwitchDefaultTarget();
+ uint32_t defaultOffset = defaultLoc.bytecodeToOffset(script_);
+
+ size_t index;
+ if (!tableswitch->addDefault(nullptr, &index)) {
+ return false;
+ }
+ if (!addPendingEdge(defaultLoc, current, index)) {
+ return false;
+ }
+ if (!targetToSuccessor.put(defaultOffset, index)) {
+ return false;
+ }
+ }
+
+ // Add all cases.
+ for (size_t i = 0; i < numCases; i++) {
+ BytecodeLocation caseLoc = loc.getTableSwitchCaseTarget(script_, i);
+ uint32_t caseOffset = caseLoc.bytecodeToOffset(script_);
+
+ size_t index;
+ if (auto p = targetToSuccessor.lookupForAdd(caseOffset)) {
+ index = p->value();
+ } else {
+ if (!tableswitch->addSuccessor(nullptr, &index)) {
+ return false;
+ }
+ if (!addPendingEdge(caseLoc, current, index)) {
+ return false;
+ }
+ if (!targetToSuccessor.add(p, caseOffset, index)) {
+ return false;
+ }
+ }
+ if (!tableswitch->addCase(index)) {
+ return false;
+ }
+ }
+
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_Rest(BytecodeLocation loc) {
+ auto* snapshot = getOpSnapshot<WarpRest>(loc);
+ Shape* shape = snapshot ? snapshot->shape() : nullptr;
+
+ // NOTE: Keep this code in sync with |ArgumentsReplacer|.
+
+ if (inlineCallInfo()) {
+ // If we are inlining, we know the actual arguments.
+ unsigned numActuals = inlineCallInfo()->argc();
+ unsigned numFormals = info().nargs() - 1;
+ unsigned numRest = numActuals > numFormals ? numActuals - numFormals : 0;
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ // Allocate an array of the correct size.
+ MInstruction* newArray;
+ if (shape && gc::CanUseFixedElementsForArray(numRest)) {
+ auto* shapeConstant = MConstant::NewShape(alloc(), shape);
+ current->add(shapeConstant);
+ newArray = MNewArrayObject::New(alloc(), shapeConstant, numRest, heap);
+ } else {
+ MConstant* templateConst = constant(NullValue());
+ newArray = MNewArray::NewVM(alloc(), numRest, templateConst, heap);
+ }
+ current->add(newArray);
+ current->push(newArray);
+
+ if (numRest == 0) {
+ // No more updating to do.
+ return true;
+ }
+
+ MElements* elements = MElements::New(alloc(), newArray);
+ current->add(elements);
+
+ // Unroll the argument copy loop. We don't need to do any bounds or hole
+ // checking here.
+ MConstant* index = nullptr;
+ for (uint32_t i = numFormals; i < numActuals; i++) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ index = MConstant::New(alloc(), Int32Value(i - numFormals));
+ current->add(index);
+
+ MDefinition* arg = inlineCallInfo()->argv()[i];
+ MStoreElement* store =
+ MStoreElement::NewUnbarriered(alloc(), elements, index, arg,
+ /* needsHoleCheck = */ false);
+ current->add(store);
+ current->add(MPostWriteBarrier::New(alloc(), newArray, arg));
+ }
+
+ // Update the initialized length for all the (necessarily non-hole)
+ // elements added.
+ MSetInitializedLength* initLength =
+ MSetInitializedLength::New(alloc(), elements, index);
+ current->add(initLength);
+
+ return true;
+ }
+
+ MArgumentsLength* numActuals = MArgumentsLength::New(alloc());
+ current->add(numActuals);
+
+ // Pass in the number of actual arguments, the number of formals (not
+ // including the rest parameter slot itself), and the shape.
+ unsigned numFormals = info().nargs() - 1;
+ MRest* rest = MRest::New(alloc(), numActuals, numFormals, shape);
+ current->add(rest);
+ current->push(rest);
+ return true;
+}
+
+bool WarpBuilder::build_Try(BytecodeLocation loc) {
+ graph().setHasTryBlock();
+
+ MBasicBlock* pred = current;
+ if (!startNewBlock(pred, loc.next())) {
+ return false;
+ }
+
+ pred->end(MGoto::New(alloc(), current));
+ return true;
+}
+
+bool WarpBuilder::build_Finally(BytecodeLocation loc) {
+ MOZ_ASSERT(graph().hasTryBlock());
+ return true;
+}
+
+bool WarpBuilder::build_Exception(BytecodeLocation) {
+ MOZ_CRASH("Unreachable because we skip catch-blocks");
+}
+
+bool WarpBuilder::build_Throw(BytecodeLocation loc) {
+ MDefinition* def = current->pop();
+
+ MThrow* ins = MThrow::New(alloc(), def);
+ current->add(ins);
+ if (!resumeAfter(ins, loc)) {
+ return false;
+ }
+
+ // Terminate the block.
+ current->end(MUnreachable::New(alloc()));
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_ThrowSetConst(BytecodeLocation loc) {
+ auto* ins = MThrowRuntimeLexicalError::New(alloc(), JSMSG_BAD_CONST_ASSIGN);
+ current->add(ins);
+ if (!resumeAfter(ins, loc)) {
+ return false;
+ }
+
+ // Terminate the block.
+ current->end(MUnreachable::New(alloc()));
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::build_ThrowMsg(BytecodeLocation loc) {
+ auto* ins = MThrowMsg::New(alloc(), loc.throwMsgKind());
+ current->add(ins);
+ if (!resumeAfter(ins, loc)) {
+ return false;
+ }
+
+ // Terminate the block.
+ current->end(MUnreachable::New(alloc()));
+ setTerminatedBlock();
+ return true;
+}
+
+bool WarpBuilder::buildIC(BytecodeLocation loc, CacheKind kind,
+ std::initializer_list<MDefinition*> inputs) {
+ MOZ_ASSERT(loc.opHasIC());
+
+ mozilla::DebugOnly<size_t> numInputs = inputs.size();
+ MOZ_ASSERT(numInputs == NumInputsForCacheKind(kind));
+
+ if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
+ return TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, inputs);
+ }
+
+ if (getOpSnapshot<WarpBailout>(loc)) {
+ for (MDefinition* input : inputs) {
+ input->setImplicitlyUsedUnchecked();
+ }
+ return buildBailoutForColdIC(loc, kind);
+ }
+
+ if (const auto* inliningSnapshot = getOpSnapshot<WarpInlinedCall>(loc)) {
+ // The CallInfo will be initialized by the transpiler.
+ bool ignoresRval = BytecodeIsPopped(loc.toRawBytecode());
+ CallInfo callInfo(alloc(), /*constructing =*/false, ignoresRval);
+ callInfo.markAsInlined();
+
+ if (!TranspileCacheIRToMIR(this, loc, inliningSnapshot->cacheIRSnapshot(),
+ inputs, &callInfo)) {
+ return false;
+ }
+ return buildInlinedCall(loc, inliningSnapshot, callInfo);
+ }
+
+ // Work around std::initializer_list not defining operator[].
+ auto getInput = [&](size_t index) -> MDefinition* {
+ MOZ_ASSERT(index < numInputs);
+ return inputs.begin()[index];
+ };
+
+ switch (kind) {
+ case CacheKind::UnaryArith: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MUnaryCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::ToPropertyKey: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MToPropertyKeyCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::BinaryArith: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins =
+ MBinaryCache::New(alloc(), getInput(0), getInput(1), MIRType::Value);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::Compare: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins = MBinaryCache::New(alloc(), getInput(0), getInput(1),
+ MIRType::Boolean);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::In: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins = MInCache::New(alloc(), getInput(0), getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::HasOwn: {
+ MOZ_ASSERT(numInputs == 2);
+ // Note: the MHasOwnCache constructor takes obj/id instead of id/obj.
+ auto* ins = MHasOwnCache::New(alloc(), getInput(1), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::CheckPrivateField: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins =
+ MCheckPrivateFieldCache::New(alloc(), getInput(0), getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::InstanceOf: {
+ MOZ_ASSERT(numInputs == 2);
+ auto* ins = MInstanceOfCache::New(alloc(), getInput(0), getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::BindName: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MBindNameCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetIterator: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MGetIteratorCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetName: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MGetNameCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetProp: {
+ MOZ_ASSERT(numInputs == 1);
+ PropertyName* name = loc.getPropertyName(script_);
+ MConstant* id = constant(StringValue(name));
+ MDefinition* val = getInput(0);
+ auto* ins = MGetPropertyCache::New(alloc(), val, id);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetElem: {
+ MOZ_ASSERT(numInputs == 2);
+ MDefinition* val = getInput(0);
+ auto* ins = MGetPropertyCache::New(alloc(), val, getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::SetProp: {
+ MOZ_ASSERT(numInputs == 2);
+ PropertyName* name = loc.getPropertyName(script_);
+ MConstant* id = constant(StringValue(name));
+ bool strict = loc.isStrictSetOp();
+ auto* ins =
+ MSetPropertyCache::New(alloc(), getInput(0), id, getInput(1), strict);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::SetElem: {
+ MOZ_ASSERT(numInputs == 3);
+ bool strict = loc.isStrictSetOp();
+ auto* ins = MSetPropertyCache::New(alloc(), getInput(0), getInput(1),
+ getInput(2), strict);
+ current->add(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetPropSuper: {
+ MOZ_ASSERT(numInputs == 2);
+ PropertyName* name = loc.getPropertyName(script_);
+ MConstant* id = constant(StringValue(name));
+ auto* ins =
+ MGetPropSuperCache::New(alloc(), getInput(0), getInput(1), id);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetElemSuper: {
+ MOZ_ASSERT(numInputs == 3);
+ // Note: CacheIR expects obj/id/receiver but MGetPropSuperCache takes
+ // obj/receiver/id so swap the last two inputs.
+ auto* ins = MGetPropSuperCache::New(alloc(), getInput(0), getInput(2),
+ getInput(1));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::OptimizeSpreadCall: {
+ MOZ_ASSERT(numInputs == 1);
+ auto* ins = MOptimizeSpreadCallCache::New(alloc(), getInput(0));
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::TypeOf: {
+ // Note: Warp does not have a TypeOf IC, it just inlines the operation.
+ MOZ_ASSERT(numInputs == 1);
+ auto* typeOf = MTypeOf::New(alloc(), getInput(0));
+ current->add(typeOf);
+
+ auto* ins = MTypeOfName::New(alloc(), typeOf);
+ current->add(ins);
+ current->push(ins);
+ return true;
+ }
+ case CacheKind::NewObject: {
+ auto* templateConst = constant(NullValue());
+ MNewObject* ins = MNewObject::NewVM(
+ alloc(), templateConst, gc::DefaultHeap, MNewObject::ObjectLiteral);
+ current->add(ins);
+ current->push(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::NewArray: {
+ uint32_t length = loc.getNewArrayLength();
+ MConstant* templateConst = constant(NullValue());
+ MNewArray* ins =
+ MNewArray::NewVM(alloc(), length, templateConst, gc::DefaultHeap);
+ current->add(ins);
+ current->push(ins);
+ return true;
+ }
+ case CacheKind::CloseIter: {
+ MOZ_ASSERT(numInputs == 1);
+ static_assert(sizeof(CompletionKind) == sizeof(uint8_t));
+ CompletionKind kind = loc.getCompletionKind();
+ auto* ins = MCloseIterCache::New(alloc(), getInput(0), uint8_t(kind));
+ current->add(ins);
+ return resumeAfter(ins, loc);
+ }
+ case CacheKind::GetIntrinsic:
+ case CacheKind::ToBool:
+ case CacheKind::Call:
+ // We're currently not using an IC or transpiling CacheIR for these kinds.
+ MOZ_CRASH("Unexpected kind");
+ }
+
+ return true;
+}
+
+bool WarpBuilder::buildBailoutForColdIC(BytecodeLocation loc, CacheKind kind) {
+ MOZ_ASSERT(loc.opHasIC());
+
+ MBail* bail = MBail::New(alloc(), BailoutKind::FirstExecution);
+ current->add(bail);
+ current->setAlwaysBails();
+
+ MIRType resultType;
+ switch (kind) {
+ case CacheKind::UnaryArith:
+ case CacheKind::BinaryArith:
+ case CacheKind::GetName:
+ case CacheKind::GetProp:
+ case CacheKind::GetElem:
+ case CacheKind::GetPropSuper:
+ case CacheKind::GetElemSuper:
+ case CacheKind::GetIntrinsic:
+ case CacheKind::Call:
+ case CacheKind::ToPropertyKey:
+ case CacheKind::OptimizeSpreadCall:
+ resultType = MIRType::Value;
+ break;
+ case CacheKind::BindName:
+ case CacheKind::GetIterator:
+ case CacheKind::NewArray:
+ case CacheKind::NewObject:
+ resultType = MIRType::Object;
+ break;
+ case CacheKind::TypeOf:
+ resultType = MIRType::String;
+ break;
+ case CacheKind::ToBool:
+ case CacheKind::Compare:
+ case CacheKind::In:
+ case CacheKind::HasOwn:
+ case CacheKind::CheckPrivateField:
+ case CacheKind::InstanceOf:
+ resultType = MIRType::Boolean;
+ break;
+ case CacheKind::SetProp:
+ case CacheKind::SetElem:
+ case CacheKind::CloseIter:
+ return true; // No result.
+ }
+
+ auto* ins = MUnreachableResult::New(alloc(), resultType);
+ current->add(ins);
+ current->push(ins);
+
+ return true;
+}
+
+class MOZ_RAII AutoAccumulateReturns {
+ MIRGraph& graph_;
+ MIRGraphReturns* prev_;
+
+ public:
+ AutoAccumulateReturns(MIRGraph& graph, MIRGraphReturns& returns)
+ : graph_(graph) {
+ prev_ = graph_.returnAccumulator();
+ graph_.setReturnAccumulator(&returns);
+ }
+ ~AutoAccumulateReturns() { graph_.setReturnAccumulator(prev_); }
+};
+
+bool WarpBuilder::buildInlinedCall(BytecodeLocation loc,
+ const WarpInlinedCall* inlineSnapshot,
+ CallInfo& callInfo) {
+ jsbytecode* pc = loc.toRawBytecode();
+
+ if (callInfo.isSetter()) {
+ // build_SetProp pushes the rhs argument onto the stack. Remove it
+ // in preparation for pushCallStack.
+ current->pop();
+ }
+
+ callInfo.setImplicitlyUsedUnchecked();
+
+ // Capture formals in the outer resume point.
+ if (!callInfo.pushCallStack(current)) {
+ return false;
+ }
+ MResumePoint* outerResumePoint =
+ MResumePoint::New(alloc(), current, pc, callInfo.inliningResumeMode());
+ if (!outerResumePoint) {
+ return false;
+ }
+ current->setOuterResumePoint(outerResumePoint);
+
+ // Pop formals again, except leave |callee| on stack for duration of call.
+ callInfo.popCallStack(current);
+ current->push(callInfo.callee());
+
+ // Build the graph.
+ CompileInfo* calleeCompileInfo = inlineSnapshot->info();
+ MIRGraphReturns returns(alloc());
+ AutoAccumulateReturns aar(graph(), returns);
+ WarpBuilder inlineBuilder(this, inlineSnapshot->scriptSnapshot(),
+ *calleeCompileInfo, &callInfo, outerResumePoint);
+ if (!inlineBuilder.buildInline()) {
+ // Note: Inlining only aborts on OOM. If inlining would fail for
+ // any other reason, we detect it in advance and don't inline.
+ return false;
+ }
+
+ // We mark scripts as uninlineable in BytecodeAnalysis if we cannot
+ // reach a return statement (without going through a catch/finally).
+ MOZ_ASSERT(!returns.empty());
+
+ // Create return block
+ BytecodeLocation postCall = loc.next();
+ MBasicBlock* prev = current;
+ if (!startNewEntryBlock(prev->stackDepth(), postCall)) {
+ return false;
+ }
+ // Restore previous value of callerResumePoint.
+ current->setCallerResumePoint(callerResumePoint());
+ current->inheritSlots(prev);
+
+ // Pop |callee|.
+ current->pop();
+
+ // Accumulate return values.
+ MDefinition* returnValue =
+ patchInlinedReturns(calleeCompileInfo, callInfo, returns, current);
+ if (!returnValue) {
+ return false;
+ }
+ current->push(returnValue);
+
+ // Initialize entry slots
+ if (!current->initEntrySlots(alloc())) {
+ return false;
+ }
+
+ return true;
+}
+
+MDefinition* WarpBuilder::patchInlinedReturns(CompileInfo* calleeCompileInfo,
+ CallInfo& callInfo,
+ MIRGraphReturns& exits,
+ MBasicBlock* returnBlock) {
+ if (exits.length() == 1) {
+ return patchInlinedReturn(calleeCompileInfo, callInfo, exits[0],
+ returnBlock);
+ }
+
+ // Accumulate multiple returns with a phi.
+ MPhi* phi = MPhi::New(alloc());
+ if (!phi->reserveLength(exits.length())) {
+ return nullptr;
+ }
+
+ for (auto* exit : exits) {
+ MDefinition* rdef =
+ patchInlinedReturn(calleeCompileInfo, callInfo, exit, returnBlock);
+ if (!rdef) {
+ return nullptr;
+ }
+ phi->addInput(rdef);
+ }
+ returnBlock->addPhi(phi);
+ return phi;
+}
+
+MDefinition* WarpBuilder::patchInlinedReturn(CompileInfo* calleeCompileInfo,
+ CallInfo& callInfo,
+ MBasicBlock* exit,
+ MBasicBlock* returnBlock) {
+ // Replace the MReturn in the exit block with an MGoto branching to
+ // the return block.
+ MDefinition* rdef = exit->lastIns()->toReturn()->input();
+ exit->discardLastIns();
+
+ // Constructors must be patched by the caller to always return an object.
+ // Derived class constructors contain extra bytecode to ensure an object
+ // is always returned, so no additional patching is needed.
+ if (callInfo.constructing() &&
+ !calleeCompileInfo->isDerivedClassConstructor()) {
+ auto* filter = MReturnFromCtor::New(alloc(), rdef, callInfo.thisArg());
+ exit->add(filter);
+ rdef = filter;
+ } else if (callInfo.isSetter()) {
+ // Setters return the rhs argument, not whatever value is returned.
+ rdef = callInfo.getArg(0);
+ }
+
+ exit->end(MGoto::New(alloc(), returnBlock));
+ if (!returnBlock->addPredecessorWithoutPhis(exit)) {
+ return nullptr;
+ }
+
+ return rdef;
+}
diff --git a/js/src/jit/WarpBuilder.h b/js/src/jit/WarpBuilder.h
new file mode 100644
index 0000000000..2a92984f64
--- /dev/null
+++ b/js/src/jit/WarpBuilder.h
@@ -0,0 +1,323 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_WarpBuilder_h
+#define jit_WarpBuilder_h
+
+#include <initializer_list>
+
+#include "ds/InlineTable.h"
+#include "jit/JitContext.h"
+#include "jit/MIR.h"
+#include "jit/WarpBuilderShared.h"
+#include "jit/WarpSnapshot.h"
+#include "vm/Opcodes.h"
+
+namespace js {
+namespace jit {
+
+// JSOps not yet supported by WarpBuilder. See warning at the end of the list.
+#define WARP_UNSUPPORTED_OPCODE_LIST(_) \
+ /* Intentionally not implemented */ \
+ _(ForceInterpreter) \
+ /* With */ \
+ _(EnterWith) \
+ _(LeaveWith) \
+ /* Eval */ \
+ _(Eval) \
+ _(StrictEval) \
+ _(SpreadEval) \
+ _(StrictSpreadEval) \
+ /* Super */ \
+ _(SetPropSuper) \
+ _(SetElemSuper) \
+ _(StrictSetPropSuper) \
+ _(StrictSetElemSuper) \
+ /* Compound assignment */ \
+ _(GetBoundName) \
+ /* Generators / Async (bug 1317690) */ \
+ _(IsGenClosing) \
+ _(Resume) \
+ /* Misc */ \
+ _(DelName) \
+ _(SetIntrinsic) \
+ /* Private Fields */ \
+ _(GetAliasedDebugVar) \
+ /* Non-syntactic scope */ \
+ _(NonSyntacticGlobalThis) \
+ /* Records and Tuples */ \
+ IF_RECORD_TUPLE(_(InitRecord)) \
+ IF_RECORD_TUPLE(_(AddRecordProperty)) \
+ IF_RECORD_TUPLE(_(AddRecordSpread)) \
+ IF_RECORD_TUPLE(_(FinishRecord)) \
+ IF_RECORD_TUPLE(_(InitTuple)) \
+ IF_RECORD_TUPLE(_(AddTupleElement)) \
+ IF_RECORD_TUPLE(_(FinishTuple)) \
+ // === !! WARNING WARNING WARNING !! ===
+ // Do you really want to sacrifice performance by not implementing this
+ // operation in the optimizing compiler?
+
+class MIRGenerator;
+class MIRGraph;
+class WarpSnapshot;
+
+enum class CacheKind : uint8_t;
+
+// [SMDOC] Control Flow handling in WarpBuilder.
+//
+// WarpBuilder traverses the script's bytecode and compiles each instruction to
+// corresponding MIR instructions. Handling control flow bytecode ops requires
+// some special machinery:
+//
+// Forward branches
+// ----------------
+// Most branches in the bytecode are forward branches to a JSOp::JumpTarget
+// instruction that we have not inspected yet. We compile them in two phases:
+//
+// 1) When compiling the source instruction: the MBasicBlock is terminated
+// with a control instruction that has a nullptr successor block. We also add
+// a PendingEdge instance to the PendingEdges list for the target bytecode
+// location.
+//
+// 2) When finally compiling the JSOp::JumpTarget: WarpBuilder::build_JumpTarget
+// creates the target block and uses the list of PendingEdges to 'link' the
+// blocks.
+//
+// Loops
+// -----
+// Loops may be nested within other loops, so each WarpBuilder has a LoopState
+// stack. This is used to link the backedge to the loop's header block.
+//
+// Unreachable/dead code
+// ---------------------
+// Some bytecode instructions never fall through to the next instruction, for
+// example JSOp::Return, JSOp::Goto, or JSOp::Throw. Code after such
+// instructions is guaranteed to be dead so WarpBuilder skips it until it gets
+// to a jump target instruction with pending edges.
+//
+// Note: The frontend may generate unnecessary JSOp::JumpTarget instructions we
+// can ignore when they have no incoming pending edges.
+//
+// Try-catch
+// ---------
+// WarpBuilder supports scripts with try-catch by only compiling the try-block
+// and bailing out (to the Baseline Interpreter) from the exception handler
+// whenever we need to execute the catch-block.
+//
+// Because we don't compile the catch-block and the code after the try-catch may
+// only be reachable via the catch-block, Baseline's BytecodeAnalysis ensures
+// Baseline does not attempt OSR into Warp at loops that are only reachable via
+// catch/finally blocks.
+//
+// Finally-blocks are compiled by WarpBuilder, but when we have to enter a
+// finally-block from the exception handler, we bail out to the Baseline
+// Interpreter.
+
+// PendingEdge is used whenever a block is terminated with a forward branch in
+// the bytecode. When we reach the jump target we use this information to link
+// the block to the jump target's block.
+class PendingEdge {
+ MBasicBlock* block_;
+ uint32_t successor_;
+ uint8_t numToPop_;
+
+ public:
+ PendingEdge(MBasicBlock* block, uint32_t successor, uint32_t numToPop)
+ : block_(block), successor_(successor), numToPop_(numToPop) {
+ MOZ_ASSERT(numToPop_ == numToPop, "value must fit in field");
+ }
+
+ MBasicBlock* block() const { return block_; }
+ uint32_t successor() const { return successor_; }
+ uint8_t numToPop() const { return numToPop_; }
+};
+
+// PendingEdgesMap maps a bytecode instruction to a Vector of PendingEdges
+// targeting it. We use InlineMap<> for this because most of the time there are
+// only a few pending edges but there can be many when switch-statements are
+// involved.
+using PendingEdges = Vector<PendingEdge, 2, SystemAllocPolicy>;
+using PendingEdgesMap =
+ InlineMap<jsbytecode*, PendingEdges, 8, PointerHasher<jsbytecode*>,
+ SystemAllocPolicy>;
+
+// LoopState stores information about a loop that's being compiled to MIR.
+class LoopState {
+ MBasicBlock* header_ = nullptr;
+
+ public:
+ explicit LoopState(MBasicBlock* header) : header_(header) {}
+
+ MBasicBlock* header() const { return header_; }
+};
+using LoopStateStack = Vector<LoopState, 4, JitAllocPolicy>;
+
+// Data that is shared across all WarpBuilders for a given compilation.
+class MOZ_STACK_CLASS WarpCompilation {
+ // The total loop depth, including loops in the caller while
+ // compiling inlined functions.
+ uint32_t loopDepth_ = 0;
+
+ // Loop phis for iterators that need to be kept alive.
+ PhiVector iterators_;
+
+ public:
+ explicit WarpCompilation(TempAllocator& alloc) : iterators_(alloc) {}
+
+ uint32_t loopDepth() const { return loopDepth_; }
+ void incLoopDepth() { loopDepth_++; }
+ void decLoopDepth() {
+ MOZ_ASSERT(loopDepth() > 0);
+ loopDepth_--;
+ }
+
+ PhiVector* iterators() { return &iterators_; }
+};
+
+// WarpBuilder builds a MIR graph from WarpSnapshot. Unlike WarpOracle,
+// WarpBuilder can run off-thread.
+class MOZ_STACK_CLASS WarpBuilder : public WarpBuilderShared {
+ WarpCompilation* warpCompilation_;
+ MIRGraph& graph_;
+ const CompileInfo& info_;
+ const WarpScriptSnapshot* scriptSnapshot_;
+ JSScript* script_;
+
+ // Pointer to a WarpOpSnapshot or nullptr if we reached the end of the list.
+ // Because bytecode is compiled from first to last instruction (and
+ // WarpOpSnapshot is sorted the same way), the iterator always moves forward.
+ const WarpOpSnapshot* opSnapshotIter_ = nullptr;
+
+ // Note: loopStack_ is builder-specific. loopStack_.length is the
+ // depth relative to the current script. The overall loop depth is
+ // stored in the WarpCompilation.
+ LoopStateStack loopStack_;
+ PendingEdgesMap pendingEdges_;
+
+ // These are only initialized when building an inlined script.
+ WarpBuilder* callerBuilder_ = nullptr;
+ MResumePoint* callerResumePoint_ = nullptr;
+ CallInfo* inlineCallInfo_ = nullptr;
+
+ WarpCompilation* warpCompilation() const { return warpCompilation_; }
+ MIRGraph& graph() { return graph_; }
+ const CompileInfo& info() const { return info_; }
+ const WarpScriptSnapshot* scriptSnapshot() const { return scriptSnapshot_; }
+
+ uint32_t loopDepth() const { return warpCompilation_->loopDepth(); }
+ void incLoopDepth() { warpCompilation_->incLoopDepth(); }
+ void decLoopDepth() { warpCompilation_->decLoopDepth(); }
+ PhiVector* iterators() { return warpCompilation_->iterators(); }
+
+ WarpBuilder* callerBuilder() const { return callerBuilder_; }
+ MResumePoint* callerResumePoint() const { return callerResumePoint_; }
+
+ BytecodeSite* newBytecodeSite(BytecodeLocation loc);
+
+ const WarpOpSnapshot* getOpSnapshotImpl(BytecodeLocation loc,
+ WarpOpSnapshot::Kind kind);
+
+ template <typename T>
+ const T* getOpSnapshot(BytecodeLocation loc) {
+ const WarpOpSnapshot* snapshot = getOpSnapshotImpl(loc, T::ThisKind);
+ return snapshot ? snapshot->as<T>() : nullptr;
+ }
+
+ void initBlock(MBasicBlock* block);
+ [[nodiscard]] bool startNewEntryBlock(size_t stackDepth,
+ BytecodeLocation loc);
+ [[nodiscard]] bool startNewBlock(MBasicBlock* predecessor,
+ BytecodeLocation loc, size_t numToPop = 0);
+ [[nodiscard]] bool startNewLoopHeaderBlock(BytecodeLocation loopHead);
+ [[nodiscard]] bool startNewOsrPreHeaderBlock(BytecodeLocation loopHead);
+
+ bool hasTerminatedBlock() const { return current == nullptr; }
+ void setTerminatedBlock() { current = nullptr; }
+
+ [[nodiscard]] bool addPendingEdge(BytecodeLocation target, MBasicBlock* block,
+ uint32_t successor, uint32_t numToPop = 0);
+ [[nodiscard]] bool buildForwardGoto(BytecodeLocation target);
+ [[nodiscard]] bool buildBackedge();
+ [[nodiscard]] bool buildTestBackedge(BytecodeLocation loc);
+
+ [[nodiscard]] bool addIteratorLoopPhis(BytecodeLocation loopHead);
+
+ [[nodiscard]] bool buildPrologue();
+ [[nodiscard]] bool buildBody();
+
+ [[nodiscard]] bool buildInlinePrologue();
+
+ [[nodiscard]] bool buildIC(BytecodeLocation loc, CacheKind kind,
+ std::initializer_list<MDefinition*> inputs);
+ [[nodiscard]] bool buildBailoutForColdIC(BytecodeLocation loc,
+ CacheKind kind);
+
+ [[nodiscard]] bool buildEnvironmentChain();
+ MInstruction* buildNamedLambdaEnv(MDefinition* callee, MDefinition* env,
+ NamedLambdaObject* templateObj);
+ MInstruction* buildCallObject(MDefinition* callee, MDefinition* env,
+ CallObject* templateObj);
+ MInstruction* buildLoadSlot(MDefinition* obj, uint32_t numFixedSlots,
+ uint32_t slot);
+
+ MConstant* globalLexicalEnvConstant();
+ MDefinition* getCallee();
+
+ [[nodiscard]] bool buildUnaryOp(BytecodeLocation loc);
+ [[nodiscard]] bool buildBinaryOp(BytecodeLocation loc);
+ [[nodiscard]] bool buildCompareOp(BytecodeLocation loc);
+ [[nodiscard]] bool buildTestOp(BytecodeLocation loc);
+ [[nodiscard]] bool buildCallOp(BytecodeLocation loc);
+
+ [[nodiscard]] bool buildInitPropGetterSetterOp(BytecodeLocation loc);
+ [[nodiscard]] bool buildInitElemGetterSetterOp(BytecodeLocation loc);
+
+ [[nodiscard]] bool buildSuspend(BytecodeLocation loc, MDefinition* gen,
+ MDefinition* retVal);
+
+ void buildCheckLexicalOp(BytecodeLocation loc);
+
+ bool usesEnvironmentChain() const;
+ MDefinition* walkEnvironmentChain(uint32_t numHops);
+
+ void buildCreateThis(CallInfo& callInfo);
+
+ [[nodiscard]] bool transpileCall(BytecodeLocation loc,
+ const WarpCacheIR* cacheIRSnapshot,
+ CallInfo* callInfo);
+
+ [[nodiscard]] bool buildInlinedCall(BytecodeLocation loc,
+ const WarpInlinedCall* snapshot,
+ CallInfo& callInfo);
+
+ MDefinition* patchInlinedReturns(CompileInfo* calleeCompileInfo,
+ CallInfo& callInfo, MIRGraphReturns& exits,
+ MBasicBlock* returnBlock);
+ MDefinition* patchInlinedReturn(CompileInfo* calleeCompileInfo,
+ CallInfo& callInfo, MBasicBlock* exit,
+ MBasicBlock* returnBlock);
+
+#define BUILD_OP(OP, ...) [[nodiscard]] bool build_##OP(BytecodeLocation loc);
+ FOR_EACH_OPCODE(BUILD_OP)
+#undef BUILD_OP
+
+ public:
+ WarpBuilder(WarpSnapshot& snapshot, MIRGenerator& mirGen,
+ WarpCompilation* warpCompilation);
+ WarpBuilder(WarpBuilder* caller, WarpScriptSnapshot* snapshot,
+ CompileInfo& compileInfo, CallInfo* inlineCallInfo,
+ MResumePoint* callerResumePoint);
+
+ [[nodiscard]] bool build();
+ [[nodiscard]] bool buildInline();
+
+ CallInfo* inlineCallInfo() const { return inlineCallInfo_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_WarpBuilder_h */
diff --git a/js/src/jit/WarpBuilderShared.cpp b/js/src/jit/WarpBuilderShared.cpp
new file mode 100644
index 0000000000..50d25e628a
--- /dev/null
+++ b/js/src/jit/WarpBuilderShared.cpp
@@ -0,0 +1,99 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/WarpBuilderShared.h"
+
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+using namespace js;
+using namespace js::jit;
+
+WarpBuilderShared::WarpBuilderShared(WarpSnapshot& snapshot,
+ MIRGenerator& mirGen,
+ MBasicBlock* current_)
+ : snapshot_(snapshot),
+ mirGen_(mirGen),
+ alloc_(mirGen.alloc()),
+ current(current_) {}
+
+bool WarpBuilderShared::resumeAfter(MInstruction* ins, BytecodeLocation loc) {
+ // resumeAfter should only be used with effectful instructions. The only
+ // exception is MInt64ToBigInt, it's used to convert the result of a call into
+ // Wasm code so we attach the resume point to that instead of to the call.
+ MOZ_ASSERT(ins->isEffectful() || ins->isInt64ToBigInt());
+ MOZ_ASSERT(!ins->isMovable());
+
+ MResumePoint* resumePoint = MResumePoint::New(
+ alloc(), ins->block(), loc.toRawBytecode(), ResumeMode::ResumeAfter);
+ if (!resumePoint) {
+ return false;
+ }
+
+ ins->setResumePoint(resumePoint);
+ return true;
+}
+
+MConstant* WarpBuilderShared::constant(const Value& v) {
+ MOZ_ASSERT_IF(v.isString(), v.toString()->isLinear());
+ MOZ_ASSERT_IF(v.isGCThing(), !IsInsideNursery(v.toGCThing()));
+
+ MConstant* cst = MConstant::New(alloc(), v);
+ current->add(cst);
+ return cst;
+}
+
+void WarpBuilderShared::pushConstant(const Value& v) {
+ MConstant* cst = constant(v);
+ current->push(cst);
+}
+
+MCall* WarpBuilderShared::makeCall(CallInfo& callInfo, bool needsThisCheck,
+ WrappedFunction* target, bool isDOMCall) {
+ auto addUndefined = [this]() -> MConstant* {
+ return constant(UndefinedValue());
+ };
+
+ return MakeCall(alloc(), addUndefined, callInfo, needsThisCheck, target,
+ isDOMCall);
+}
+
+MInstruction* WarpBuilderShared::makeSpreadCall(CallInfo& callInfo,
+ bool needsThisCheck,
+ bool isSameRealm,
+ WrappedFunction* target) {
+ MOZ_ASSERT(callInfo.argFormat() == CallInfo::ArgFormat::Array);
+ MOZ_ASSERT_IF(needsThisCheck, !target);
+
+ // Load dense elements of the argument array.
+ MElements* elements = MElements::New(alloc(), callInfo.arrayArg());
+ current->add(elements);
+
+ if (callInfo.constructing()) {
+ auto* construct =
+ MConstructArray::New(alloc(), target, callInfo.callee(), elements,
+ callInfo.thisArg(), callInfo.getNewTarget());
+ if (isSameRealm) {
+ construct->setNotCrossRealm();
+ }
+ if (needsThisCheck) {
+ construct->setNeedsThisCheck();
+ }
+ return construct;
+ }
+
+ auto* apply = MApplyArray::New(alloc(), target, callInfo.callee(), elements,
+ callInfo.thisArg());
+
+ if (callInfo.ignoresReturnValue()) {
+ apply->setIgnoresReturnValue();
+ }
+ if (isSameRealm) {
+ apply->setNotCrossRealm();
+ }
+ MOZ_ASSERT(!needsThisCheck);
+ return apply;
+}
diff --git a/js/src/jit/WarpBuilderShared.h b/js/src/jit/WarpBuilderShared.h
new file mode 100644
index 0000000000..8f6431b57b
--- /dev/null
+++ b/js/src/jit/WarpBuilderShared.h
@@ -0,0 +1,409 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_WarpBuilderShared_h
+#define jit_WarpBuilderShared_h
+
+#include "mozilla/Attributes.h"
+#include "mozilla/Maybe.h"
+
+#include "jit/MIRGraph.h"
+#include "js/Value.h"
+
+namespace js {
+
+class BytecodeLocation;
+
+namespace jit {
+
+class MBasicBlock;
+class MCall;
+class MConstant;
+class MInstruction;
+class MIRGenerator;
+class TempAllocator;
+class WarpSnapshot;
+class WrappedFunction;
+
+// Helper class to manage call state.
+class MOZ_STACK_CLASS CallInfo {
+ MDefinition* callee_ = nullptr;
+ MDefinition* thisArg_ = nullptr;
+ MDefinition* newTargetArg_ = nullptr;
+ MDefinitionVector args_;
+
+ bool constructing_;
+
+ // True if the caller does not use the return value.
+ bool ignoresReturnValue_;
+
+ bool inlined_ = false;
+ bool setter_ = false;
+
+ public:
+ // For normal calls and FunCall we can shuffle around definitions in
+ // the CallInfo and use a normal MCall. For others, we need to use a
+ // specialized call.
+ enum class ArgFormat {
+ Standard,
+ Array,
+ FunApplyArgsObj,
+ };
+
+ private:
+ ArgFormat argFormat_ = ArgFormat::Standard;
+ mozilla::Maybe<ResumeMode> inliningMode_;
+
+ public:
+ CallInfo(TempAllocator& alloc, bool constructing, bool ignoresReturnValue,
+ jsbytecode* pc = nullptr)
+ : args_(alloc),
+ constructing_(constructing),
+ ignoresReturnValue_(ignoresReturnValue) {}
+
+ [[nodiscard]] bool init(MBasicBlock* current, uint32_t argc) {
+ MOZ_ASSERT(args_.empty());
+
+ // Get the arguments in the right order
+ if (!args_.reserve(argc)) {
+ return false;
+ }
+
+ if (constructing()) {
+ setNewTarget(current->pop());
+ }
+
+ for (int32_t i = argc; i > 0; i--) {
+ args_.infallibleAppend(current->peek(-i));
+ }
+ current->popn(argc);
+
+ // Get |this| and |callee|
+ setThis(current->pop());
+ setCallee(current->pop());
+
+ return true;
+ }
+
+ void initForSpreadCall(MBasicBlock* current) {
+ MOZ_ASSERT(args_.empty());
+
+ if (constructing()) {
+ setNewTarget(current->pop());
+ }
+
+ // Spread calls have one argument, an Array object containing the args.
+ static_assert(decltype(args_)::InlineLength >= 1,
+ "Appending one argument should be infallible");
+ MOZ_ALWAYS_TRUE(args_.append(current->pop()));
+
+ // Get |this| and |callee|
+ setThis(current->pop());
+ setCallee(current->pop());
+
+ argFormat_ = ArgFormat::Array;
+ }
+
+ void initForGetterCall(MDefinition* callee, MDefinition* thisVal) {
+ MOZ_ASSERT(args_.empty());
+ setCallee(callee);
+ setThis(thisVal);
+ }
+ void initForSetterCall(MDefinition* callee, MDefinition* thisVal,
+ MDefinition* rhs) {
+ MOZ_ASSERT(args_.empty());
+ markAsSetter();
+ setCallee(callee);
+ setThis(thisVal);
+ static_assert(decltype(args_)::InlineLength >= 1,
+ "Appending one argument should be infallible");
+ MOZ_ALWAYS_TRUE(args_.append(rhs));
+ }
+
+ void initForApplyInlinedArgs(MDefinition* callee, MDefinition* thisVal,
+ uint32_t numActuals) {
+ MOZ_ASSERT(args_.empty());
+ MOZ_ASSERT(!constructing_);
+
+ setCallee(callee);
+ setThis(thisVal);
+
+ MOZ_ASSERT(numActuals <= ArgumentsObject::MaxInlinedArgs);
+ static_assert(
+ ArgumentsObject::MaxInlinedArgs <= decltype(args_)::InlineLength,
+ "Actual arguments can be infallibly stored inline");
+ MOZ_ALWAYS_TRUE(args_.reserve(numActuals));
+ }
+
+ [[nodiscard]] bool initForApplyArray(MDefinition* callee,
+ MDefinition* thisVal,
+ uint32_t numActuals) {
+ MOZ_ASSERT(args_.empty());
+ MOZ_ASSERT(!constructing_);
+
+ setCallee(callee);
+ setThis(thisVal);
+
+ return args_.reserve(numActuals);
+ }
+
+ [[nodiscard]] bool initForConstructArray(MDefinition* callee,
+ MDefinition* thisVal,
+ MDefinition* newTarget,
+ uint32_t numActuals) {
+ MOZ_ASSERT(args_.empty());
+ MOZ_ASSERT(constructing_);
+
+ setCallee(callee);
+ setThis(thisVal);
+ setNewTarget(newTarget);
+
+ return args_.reserve(numActuals);
+ }
+
+ void initForCloseIter(MDefinition* iter, MDefinition* callee) {
+ MOZ_ASSERT(args_.empty());
+ setCallee(callee);
+ setThis(iter);
+ }
+
+ void popCallStack(MBasicBlock* current) { current->popn(numFormals()); }
+
+ [[nodiscard]] bool pushCallStack(MBasicBlock* current) {
+ current->push(callee());
+ current->push(thisArg());
+
+ for (uint32_t i = 0; i < argc(); i++) {
+ current->push(getArg(i));
+ }
+
+ if (constructing()) {
+ current->push(getNewTarget());
+ }
+
+ return true;
+ }
+
+ uint32_t argc() const { return args_.length(); }
+ uint32_t numFormals() const { return argc() + 2 + constructing(); }
+
+ [[nodiscard]] bool setArgs(const MDefinitionVector& args) {
+ MOZ_ASSERT(args_.empty());
+ return args_.appendAll(args);
+ }
+
+ MDefinitionVector& argv() { return args_; }
+
+ const MDefinitionVector& argv() const { return args_; }
+
+ MDefinition* getArg(uint32_t i) const {
+ MOZ_ASSERT(i < argc());
+ return args_[i];
+ }
+
+ void initArg(uint32_t i, MDefinition* def) {
+ MOZ_ASSERT(i == argc());
+ args_.infallibleAppend(def);
+ }
+
+ void setArg(uint32_t i, MDefinition* def) {
+ MOZ_ASSERT(i < argc());
+ args_[i] = def;
+ }
+
+ void removeArg(uint32_t i) { args_.erase(&args_[i]); }
+
+ MDefinition* thisArg() const {
+ MOZ_ASSERT(thisArg_);
+ return thisArg_;
+ }
+
+ void setThis(MDefinition* thisArg) { thisArg_ = thisArg; }
+
+ bool constructing() const { return constructing_; }
+
+ bool ignoresReturnValue() const { return ignoresReturnValue_; }
+
+ void setNewTarget(MDefinition* newTarget) {
+ MOZ_ASSERT(constructing());
+ newTargetArg_ = newTarget;
+ }
+ MDefinition* getNewTarget() const {
+ MOZ_ASSERT(newTargetArg_);
+ return newTargetArg_;
+ }
+
+ bool isSetter() const { return setter_; }
+ void markAsSetter() { setter_ = true; }
+
+ bool isInlined() const { return inlined_; }
+ void markAsInlined() { inlined_ = true; }
+
+ ResumeMode inliningResumeMode() const {
+ MOZ_ASSERT(isInlined());
+ return *inliningMode_;
+ }
+
+ void setInliningResumeMode(ResumeMode mode) {
+ MOZ_ASSERT(isInlined());
+ MOZ_ASSERT(inliningMode_.isNothing());
+ inliningMode_.emplace(mode);
+ }
+
+ MDefinition* callee() const {
+ MOZ_ASSERT(callee_);
+ return callee_;
+ }
+
+ void setCallee(MDefinition* callee) { callee_ = callee; }
+
+ template <typename Fun>
+ void forEachCallOperand(Fun& f) {
+ f(callee_);
+ f(thisArg_);
+ if (newTargetArg_) {
+ f(newTargetArg_);
+ }
+ for (uint32_t i = 0; i < argc(); i++) {
+ f(getArg(i));
+ }
+ }
+
+ void setImplicitlyUsedUnchecked() {
+ auto setFlag = [](MDefinition* def) { def->setImplicitlyUsedUnchecked(); };
+ forEachCallOperand(setFlag);
+ }
+
+ ArgFormat argFormat() const { return argFormat_; }
+ void setArgFormat(ArgFormat argFormat) { argFormat_ = argFormat; }
+
+ MDefinition* arrayArg() const {
+ MOZ_ASSERT(argFormat_ == ArgFormat::Array);
+ // The array argument for a spread call or FunApply is always the last
+ // argument.
+ return getArg(argc() - 1);
+ }
+};
+
+template <typename Undef>
+MCall* MakeCall(TempAllocator& alloc, Undef addUndefined, CallInfo& callInfo,
+ bool needsThisCheck, WrappedFunction* target, bool isDOMCall) {
+ MOZ_ASSERT(callInfo.argFormat() == CallInfo::ArgFormat::Standard);
+ MOZ_ASSERT_IF(needsThisCheck, !target);
+ MOZ_ASSERT_IF(isDOMCall, target->jitInfo()->type() == JSJitInfo::Method);
+
+ mozilla::Maybe<DOMObjectKind> objKind;
+ if (isDOMCall) {
+ const JSClass* clasp = callInfo.thisArg()->toGuardToClass()->getClass();
+ MOZ_ASSERT(clasp->isDOMClass());
+ if (clasp->isNativeObject()) {
+ objKind.emplace(DOMObjectKind::Native);
+ } else {
+ MOZ_ASSERT(clasp->isProxyObject());
+ objKind.emplace(DOMObjectKind::Proxy);
+ }
+ }
+
+ uint32_t targetArgs = callInfo.argc();
+
+ // Collect number of missing arguments provided that the target is
+ // scripted. Native functions are passed an explicit 'argc' parameter.
+ if (target && target->hasJitEntry()) {
+ targetArgs = std::max<uint32_t>(target->nargs(), callInfo.argc());
+ }
+
+ MCall* call =
+ MCall::New(alloc, target, targetArgs + 1 + callInfo.constructing(),
+ callInfo.argc(), callInfo.constructing(),
+ callInfo.ignoresReturnValue(), isDOMCall, objKind);
+ if (!call) {
+ return nullptr;
+ }
+
+ if (callInfo.constructing()) {
+ // Note: setThis should have been done by the caller of makeCall.
+ if (needsThisCheck) {
+ call->setNeedsThisCheck();
+ }
+
+ // Pass |new.target|
+ call->addArg(targetArgs + 1, callInfo.getNewTarget());
+ }
+
+ // Explicitly pad any missing arguments with |undefined|.
+ // This permits skipping the argumentsRectifier.
+ MOZ_ASSERT_IF(target && targetArgs > callInfo.argc(), target->hasJitEntry());
+
+ MConstant* undef = nullptr;
+ for (uint32_t i = targetArgs; i > callInfo.argc(); i--) {
+ if (!undef) {
+ undef = addUndefined();
+ }
+ if (!alloc.ensureBallast()) {
+ return nullptr;
+ }
+ call->addArg(i, undef);
+ }
+
+ // Add explicit arguments.
+ // Skip addArg(0) because it is reserved for |this|.
+ for (int32_t i = callInfo.argc() - 1; i >= 0; i--) {
+ call->addArg(i + 1, callInfo.getArg(i));
+ }
+
+ if (isDOMCall) {
+ // Now that we've told it about all the args, compute whether it's movable
+ call->computeMovable();
+ }
+
+ // Pass |this| and callee.
+ call->addArg(0, callInfo.thisArg());
+ call->initCallee(callInfo.callee());
+
+ if (target) {
+ // The callee must be a JSFunction so we don't need a Class check.
+ call->disableClassCheck();
+ }
+
+ return call;
+}
+
+// Base class for code sharing between WarpBuilder and WarpCacheIRTranspiler.
+// Because this code is used by WarpCacheIRTranspiler we should
+// generally assume that we only have access to the current basic block.
+class WarpBuilderShared {
+ WarpSnapshot& snapshot_;
+ MIRGenerator& mirGen_;
+ TempAllocator& alloc_;
+
+ protected:
+ MBasicBlock* current;
+
+ WarpBuilderShared(WarpSnapshot& snapshot, MIRGenerator& mirGen,
+ MBasicBlock* current_);
+
+ [[nodiscard]] bool resumeAfter(MInstruction* ins, BytecodeLocation loc);
+
+ MConstant* constant(const JS::Value& v);
+ void pushConstant(const JS::Value& v);
+
+ MCall* makeCall(CallInfo& callInfo, bool needsThisCheck,
+ WrappedFunction* target = nullptr, bool isDOMCall = false);
+ MInstruction* makeSpreadCall(CallInfo& callInfo, bool needsThisCheck,
+ bool isSameRealm = false,
+ WrappedFunction* target = nullptr);
+
+ public:
+ MBasicBlock* currentBlock() const { return current; }
+ WarpSnapshot& snapshot() const { return snapshot_; }
+ MIRGenerator& mirGen() { return mirGen_; }
+ TempAllocator& alloc() { return alloc_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif
diff --git a/js/src/jit/WarpCacheIRTranspiler.cpp b/js/src/jit/WarpCacheIRTranspiler.cpp
new file mode 100644
index 0000000000..e0f066901f
--- /dev/null
+++ b/js/src/jit/WarpCacheIRTranspiler.cpp
@@ -0,0 +1,5574 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/WarpCacheIRTranspiler.h"
+
+#include "mozilla/Casting.h"
+#include "mozilla/Maybe.h"
+
+#include "jsmath.h"
+
+#include "builtin/DataViewObject.h"
+#include "builtin/MapObject.h"
+#include "jit/AtomicOp.h"
+#include "jit/CacheIR.h"
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIROpsGenerated.h"
+#include "jit/CacheIRReader.h"
+#include "jit/LIR.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "jit/WarpBuilder.h"
+#include "jit/WarpBuilderShared.h"
+#include "jit/WarpSnapshot.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "vm/ArgumentsObject.h"
+#include "vm/BytecodeLocation.h"
+#include "wasm/WasmCode.h"
+
+#include "gc/ObjectKind-inl.h"
+#include "vm/NativeObject-inl.h"
+#include "wasm/WasmInstance-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// The CacheIR transpiler generates MIR from Baseline CacheIR.
+class MOZ_RAII WarpCacheIRTranspiler : public WarpBuilderShared {
+ WarpBuilder* builder_;
+ BytecodeLocation loc_;
+ const CacheIRStubInfo* stubInfo_;
+ const uint8_t* stubData_;
+
+ // Vector mapping OperandId to corresponding MDefinition.
+ using MDefinitionStackVector = Vector<MDefinition*, 8, SystemAllocPolicy>;
+ MDefinitionStackVector operands_;
+
+ CallInfo* callInfo_;
+
+ // Array mapping call arguments to OperandId.
+ using ArgumentKindArray =
+ mozilla::EnumeratedArray<ArgumentKind, ArgumentKind::NumKinds, OperandId>;
+ ArgumentKindArray argumentOperandIds_;
+
+ void setArgumentId(ArgumentKind kind, OperandId id) {
+ MOZ_ASSERT(kind != ArgumentKind::Callee);
+ MOZ_ASSERT(!argumentOperandIds_[kind].valid());
+ argumentOperandIds_[kind] = id;
+ }
+
+ void updateArgumentsFromOperands();
+
+#ifdef DEBUG
+ // Used to assert that there is only one effectful instruction
+ // per stub. And that this instruction has a resume point.
+ MInstruction* effectful_ = nullptr;
+ bool pushedResult_ = false;
+#endif
+
+ inline void addUnchecked(MInstruction* ins) {
+ current->add(ins);
+
+ // If we have not set a more specific bailout kind, mark this instruction
+ // as transpiled CacheIR. If one of these instructions bails out, we
+ // expect to hit the baseline fallback stub and invalidate the Warp script
+ // in tryAttach.
+ if (ins->bailoutKind() == BailoutKind::Unknown) {
+ ins->setBailoutKind(BailoutKind::TranspiledCacheIR);
+ }
+ }
+
+ inline void add(MInstruction* ins) {
+ MOZ_ASSERT(!ins->isEffectful());
+ addUnchecked(ins);
+ }
+
+ inline void addEffectful(MInstruction* ins) {
+ MOZ_ASSERT(ins->isEffectful());
+ MOZ_ASSERT(!effectful_, "Can only have one effectful instruction");
+ addUnchecked(ins);
+#ifdef DEBUG
+ effectful_ = ins;
+#endif
+ }
+
+ // Bypasses all checks in addEffectful. Only used for testing functions.
+ inline void addEffectfulUnsafe(MInstruction* ins) {
+ MOZ_ASSERT(ins->isEffectful());
+ addUnchecked(ins);
+ }
+
+ [[nodiscard]] bool resumeAfterUnchecked(MInstruction* ins) {
+ return WarpBuilderShared::resumeAfter(ins, loc_);
+ }
+ [[nodiscard]] bool resumeAfter(MInstruction* ins) {
+ MOZ_ASSERT(effectful_ == ins);
+ return resumeAfterUnchecked(ins);
+ }
+
+ // CacheIR instructions writing to the IC's result register (the *Result
+ // instructions) must call this to push the result onto the virtual stack.
+ void pushResult(MDefinition* result) {
+ MOZ_ASSERT(!pushedResult_, "Can't have more than one result");
+ current->push(result);
+#ifdef DEBUG
+ pushedResult_ = true;
+#endif
+ }
+
+ MDefinition* getOperand(OperandId id) const { return operands_[id.id()]; }
+
+ void setOperand(OperandId id, MDefinition* def) { operands_[id.id()] = def; }
+
+ [[nodiscard]] bool defineOperand(OperandId id, MDefinition* def) {
+ MOZ_ASSERT(id.id() == operands_.length());
+ return operands_.append(def);
+ }
+
+ uintptr_t readStubWord(uint32_t offset) {
+ return stubInfo_->getStubRawWord(stubData_, offset);
+ }
+
+ Shape* shapeStubField(uint32_t offset) {
+ return reinterpret_cast<Shape*>(readStubWord(offset));
+ }
+ GetterSetter* getterSetterStubField(uint32_t offset) {
+ return reinterpret_cast<GetterSetter*>(readStubWord(offset));
+ }
+ const JSClass* classStubField(uint32_t offset) {
+ return reinterpret_cast<const JSClass*>(readStubWord(offset));
+ }
+ JSString* stringStubField(uint32_t offset) {
+ return reinterpret_cast<JSString*>(readStubWord(offset));
+ }
+ JS::Symbol* symbolStubField(uint32_t offset) {
+ return reinterpret_cast<JS::Symbol*>(readStubWord(offset));
+ }
+ BaseScript* baseScriptStubField(uint32_t offset) {
+ return reinterpret_cast<BaseScript*>(readStubWord(offset));
+ }
+ const JSJitInfo* jitInfoStubField(uint32_t offset) {
+ return reinterpret_cast<const JSJitInfo*>(readStubWord(offset));
+ }
+ JS::ExpandoAndGeneration* expandoAndGenerationField(uint32_t offset) {
+ return reinterpret_cast<JS::ExpandoAndGeneration*>(readStubWord(offset));
+ }
+ const wasm::FuncExport* wasmFuncExportField(uint32_t offset) {
+ return reinterpret_cast<const wasm::FuncExport*>(readStubWord(offset));
+ }
+ NativeIteratorListHead* nativeIteratorListHeadStubField(uint32_t offset) {
+ return reinterpret_cast<NativeIteratorListHead*>(readStubWord(offset));
+ }
+ gc::InitialHeap allocSiteInitialHeapField(uint32_t offset) {
+ uintptr_t word = readStubWord(offset);
+ MOZ_ASSERT(word == uintptr_t(gc::DefaultHeap) ||
+ word == uintptr_t(gc::TenuredHeap));
+ return gc::InitialHeap(word);
+ }
+ const void* rawPointerField(uint32_t offset) {
+ return reinterpret_cast<const void*>(readStubWord(offset));
+ }
+ jsid idStubField(uint32_t offset) {
+ return jsid::fromRawBits(readStubWord(offset));
+ }
+ int32_t int32StubField(uint32_t offset) {
+ return static_cast<int32_t>(readStubWord(offset));
+ }
+ uint32_t uint32StubField(uint32_t offset) {
+ return static_cast<uint32_t>(readStubWord(offset));
+ }
+ uint64_t uint64StubField(uint32_t offset) {
+ return static_cast<uint64_t>(stubInfo_->getStubRawInt64(stubData_, offset));
+ }
+ Value valueStubField(uint32_t offset) {
+ uint64_t raw =
+ static_cast<uint64_t>(stubInfo_->getStubRawInt64(stubData_, offset));
+ return Value::fromRawBits(raw);
+ }
+ double doubleStubField(uint32_t offset) {
+ uint64_t raw =
+ static_cast<uint64_t>(stubInfo_->getStubRawInt64(stubData_, offset));
+ return mozilla::BitwiseCast<double>(raw);
+ }
+
+ // This must only be called when the caller knows the object is tenured and
+ // not a nursery index.
+ JSObject* tenuredObjectStubField(uint32_t offset) {
+ WarpObjectField field = WarpObjectField::fromData(readStubWord(offset));
+ return field.toObject();
+ }
+
+ // Returns either MConstant or MNurseryIndex. See WarpObjectField.
+ MInstruction* objectStubField(uint32_t offset);
+
+ const JSClass* classForGuardClassKind(GuardClassKind kind);
+
+ [[nodiscard]] bool emitGuardTo(ValOperandId inputId, MIRType type);
+
+ [[nodiscard]] bool emitToString(OperandId inputId, StringOperandId resultId);
+
+ template <typename T>
+ [[nodiscard]] bool emitDoubleBinaryArithResult(NumberOperandId lhsId,
+ NumberOperandId rhsId);
+
+ template <typename T>
+ [[nodiscard]] bool emitInt32BinaryArithResult(Int32OperandId lhsId,
+ Int32OperandId rhsId);
+
+ template <typename T>
+ [[nodiscard]] bool emitBigIntBinaryArithResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId);
+
+ template <typename T>
+ [[nodiscard]] bool emitBigIntBinaryArithEffectfulResult(
+ BigIntOperandId lhsId, BigIntOperandId rhsId);
+
+ template <typename T>
+ [[nodiscard]] bool emitBigIntUnaryArithResult(BigIntOperandId inputId);
+
+ [[nodiscard]] bool emitCompareResult(JSOp op, OperandId lhsId,
+ OperandId rhsId,
+ MCompare::CompareType compareType);
+
+ [[nodiscard]] bool emitTruthyResult(OperandId inputId);
+
+ [[nodiscard]] bool emitNewIteratorResult(MNewIterator::Type type,
+ uint32_t templateObjectOffset);
+
+ MInstruction* addBoundsCheck(MDefinition* index, MDefinition* length);
+
+ [[nodiscard]] MInstruction* convertToBoolean(MDefinition* input);
+
+ bool emitAddAndStoreSlotShared(MAddAndStoreSlot::Kind kind,
+ ObjOperandId objId, uint32_t offsetOffset,
+ ValOperandId rhsId, uint32_t newShapeOffset);
+
+ void addDataViewData(MDefinition* obj, Scalar::Type type,
+ MDefinition** offset, MInstruction** elements);
+
+ [[nodiscard]] bool emitAtomicsBinaryOp(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect, AtomicOp op);
+
+ [[nodiscard]] bool emitLoadArgumentSlot(ValOperandId resultId,
+ uint32_t slotIndex);
+
+ // Calls are either Native (native function without a JitEntry),
+ // a DOM Native (native function with a JitInfo OpType::Method),
+ // or Scripted (scripted function or native function with a JitEntry).
+ enum class CallKind { Native, DOM, Scripted };
+
+ [[nodiscard]] bool updateCallInfo(MDefinition* callee, CallFlags flags);
+
+ [[nodiscard]] bool emitCallFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ mozilla::Maybe<ObjOperandId> thisObjId,
+ CallFlags flags, CallKind kind);
+ [[nodiscard]] bool emitFunApplyArgsObj(WrappedFunction* wrappedTarget,
+ CallFlags flags);
+
+ MDefinition* convertWasmArg(MDefinition* arg, wasm::ValType::Kind kind);
+
+ WrappedFunction* maybeWrappedFunction(MDefinition* callee, CallKind kind,
+ uint16_t nargs, FunctionFlags flags);
+ WrappedFunction* maybeCallTarget(MDefinition* callee, CallKind kind);
+
+ bool maybeCreateThis(MDefinition* callee, CallFlags flags, CallKind kind);
+
+ [[nodiscard]] bool emitCallGetterResult(CallKind kind,
+ ValOperandId receiverId,
+ uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset);
+ [[nodiscard]] bool emitCallSetter(CallKind kind, ObjOperandId receiverId,
+ uint32_t setterOffset, ValOperandId rhsId,
+ bool sameRealm,
+ uint32_t nargsAndFlagsOffset);
+
+ CACHE_IR_TRANSPILER_GENERATED
+
+ public:
+ WarpCacheIRTranspiler(WarpBuilder* builder, BytecodeLocation loc,
+ CallInfo* callInfo, const WarpCacheIR* cacheIRSnapshot)
+ : WarpBuilderShared(builder->snapshot(), builder->mirGen(),
+ builder->currentBlock()),
+ builder_(builder),
+ loc_(loc),
+ stubInfo_(cacheIRSnapshot->stubInfo()),
+ stubData_(cacheIRSnapshot->stubData()),
+ callInfo_(callInfo) {}
+
+ [[nodiscard]] bool transpile(std::initializer_list<MDefinition*> inputs);
+};
+
+bool WarpCacheIRTranspiler::transpile(
+ std::initializer_list<MDefinition*> inputs) {
+ if (!operands_.append(inputs.begin(), inputs.end())) {
+ return false;
+ }
+
+ CacheIRReader reader(stubInfo_);
+ do {
+ CacheOp op = reader.readOp();
+ switch (op) {
+#define DEFINE_OP(op, ...) \
+ case CacheOp::op: \
+ if (!emit##op(reader)) { \
+ return false; \
+ } \
+ break;
+ CACHE_IR_TRANSPILER_OPS(DEFINE_OP)
+#undef DEFINE_OP
+
+ default:
+ fprintf(stderr, "Unsupported op: %s\n", CacheIROpNames[size_t(op)]);
+ MOZ_CRASH("Unsupported op");
+ }
+ } while (reader.more());
+
+ // Effectful instructions should have a resume point. MIonToWasmCall is an
+ // exception: we can attach the resume point to the MInt64ToBigInt instruction
+ // instead.
+ MOZ_ASSERT_IF(effectful_,
+ effectful_->resumePoint() || effectful_->isIonToWasmCall());
+ return true;
+}
+
+MInstruction* WarpCacheIRTranspiler::objectStubField(uint32_t offset) {
+ WarpObjectField field = WarpObjectField::fromData(readStubWord(offset));
+
+ if (field.isNurseryIndex()) {
+ auto* ins = MNurseryObject::New(alloc(), field.toNurseryIndex());
+ add(ins);
+ return ins;
+ }
+
+ auto* ins = MConstant::NewObject(alloc(), field.toObject());
+ add(ins);
+ return ins;
+}
+
+bool WarpCacheIRTranspiler::emitGuardClass(ObjOperandId objId,
+ GuardClassKind kind) {
+ MDefinition* def = getOperand(objId);
+
+ MInstruction* ins;
+ if (kind == GuardClassKind::JSFunction) {
+ ins = MGuardToFunction::New(alloc(), def);
+ } else {
+ const JSClass* classp = classForGuardClassKind(kind);
+ ins = MGuardToClass::New(alloc(), def, classp);
+ }
+
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+const JSClass* WarpCacheIRTranspiler::classForGuardClassKind(
+ GuardClassKind kind) {
+ switch (kind) {
+ case GuardClassKind::Array:
+ return &ArrayObject::class_;
+ case GuardClassKind::PlainObject:
+ return &PlainObject::class_;
+ case GuardClassKind::ArrayBuffer:
+ return &ArrayBufferObject::class_;
+ case GuardClassKind::SharedArrayBuffer:
+ return &SharedArrayBufferObject::class_;
+ case GuardClassKind::DataView:
+ return &DataViewObject::class_;
+ case GuardClassKind::MappedArguments:
+ return &MappedArgumentsObject::class_;
+ case GuardClassKind::UnmappedArguments:
+ return &UnmappedArgumentsObject::class_;
+ case GuardClassKind::WindowProxy:
+ return mirGen().runtime->maybeWindowProxyClass();
+ case GuardClassKind::Set:
+ return &SetObject::class_;
+ case GuardClassKind::Map:
+ return &MapObject::class_;
+ case GuardClassKind::JSFunction:
+ break;
+ }
+ MOZ_CRASH("unexpected kind");
+}
+
+bool WarpCacheIRTranspiler::emitGuardAnyClass(ObjOperandId objId,
+ uint32_t claspOffset) {
+ MDefinition* def = getOperand(objId);
+ const JSClass* classp = classStubField(claspOffset);
+
+ auto* ins = MGuardToClass::New(alloc(), def, classp);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardShape(ObjOperandId objId,
+ uint32_t shapeOffset) {
+ MDefinition* def = getOperand(objId);
+ Shape* shape = shapeStubField(shapeOffset);
+
+ auto* ins = MGuardShape::New(alloc(), def, shape);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardMultipleShapes(ObjOperandId objId,
+ uint32_t shapesOffset) {
+ MDefinition* def = getOperand(objId);
+ MInstruction* shapeList = objectStubField(shapesOffset);
+
+ auto* ins = MGuardMultipleShapes::New(alloc(), def, shapeList);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardNullProto(ObjOperandId objId) {
+ MDefinition* def = getOperand(objId);
+
+ auto* ins = MGuardNullProto::New(alloc(), def);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsNativeObject(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardIsNativeObject::New(alloc(), obj);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsProxy(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardIsProxy::New(alloc(), obj);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsNotProxy(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardIsNotProxy::New(alloc(), obj);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsNotDOMProxy(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardIsNotDOMProxy::New(alloc(), obj);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardHasGetterSetter(
+ ObjOperandId objId, uint32_t idOffset, uint32_t getterSetterOffset) {
+ MDefinition* obj = getOperand(objId);
+ jsid id = idStubField(idOffset);
+ GetterSetter* gs = getterSetterStubField(getterSetterOffset);
+
+ auto* ins = MGuardHasGetterSetter::New(alloc(), obj, id, gs);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitProxyGetResult(ObjOperandId objId,
+ uint32_t idOffset) {
+ MDefinition* obj = getOperand(objId);
+ jsid id = idStubField(idOffset);
+
+ auto* ins = MProxyGet::New(alloc(), obj, id);
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitProxyGetByValueResult(ObjOperandId objId,
+ ValOperandId idId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* id = getOperand(idId);
+
+ auto* ins = MProxyGetByValue::New(alloc(), obj, id);
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitProxyHasPropResult(ObjOperandId objId,
+ ValOperandId idId,
+ bool hasOwn) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* id = getOperand(idId);
+
+ auto* ins = MProxyHasProp::New(alloc(), obj, id, hasOwn);
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitProxySet(ObjOperandId objId, uint32_t idOffset,
+ ValOperandId rhsId, bool strict) {
+ MDefinition* obj = getOperand(objId);
+ jsid id = idStubField(idOffset);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MProxySet::New(alloc(), obj, rhs, id, strict);
+ addEffectful(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitProxySetByValue(ObjOperandId objId,
+ ValOperandId idId,
+ ValOperandId rhsId,
+ bool strict) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* id = getOperand(idId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MProxySetByValue::New(alloc(), obj, id, rhs, strict);
+ addEffectful(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitCallSetArrayLength(ObjOperandId objId,
+ bool strict,
+ ValOperandId rhsId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MCallSetArrayLength::New(alloc(), obj, rhs, strict);
+ addEffectful(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitCallDOMGetterResult(ObjOperandId objId,
+ uint32_t jitInfoOffset) {
+ MDefinition* obj = getOperand(objId);
+ const JSJitInfo* jitInfo = jitInfoStubField(jitInfoOffset);
+
+ MInstruction* ins;
+ if (jitInfo->isAlwaysInSlot) {
+ ins = MGetDOMMember::New(alloc(), jitInfo, obj, nullptr, nullptr);
+ } else {
+ // TODO(post-Warp): realms, guard operands (movable?).
+ ins = MGetDOMProperty::New(alloc(), jitInfo, DOMObjectKind::Native,
+ (JS::Realm*)mirGen().realm->realmPtr(), obj,
+ nullptr, nullptr);
+ }
+
+ if (!ins) {
+ return false;
+ }
+
+ if (ins->isEffectful()) {
+ addEffectful(ins);
+ pushResult(ins);
+ return resumeAfter(ins);
+ }
+
+ add(ins);
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCallDOMSetter(ObjOperandId objId,
+ uint32_t jitInfoOffset,
+ ValOperandId rhsId) {
+ MDefinition* obj = getOperand(objId);
+ const JSJitInfo* jitInfo = jitInfoStubField(jitInfoOffset);
+ MDefinition* value = getOperand(rhsId);
+
+ MOZ_ASSERT(jitInfo->type() == JSJitInfo::Setter);
+ auto* set =
+ MSetDOMProperty::New(alloc(), jitInfo->setter, DOMObjectKind::Native,
+ (JS::Realm*)mirGen().realm->realmPtr(), obj, value);
+ addEffectful(set);
+ return resumeAfter(set);
+}
+
+bool WarpCacheIRTranspiler::emitLoadDOMExpandoValue(ObjOperandId objId,
+ ValOperandId resultId) {
+ MDefinition* proxy = getOperand(objId);
+
+ auto* ins = MLoadDOMExpandoValue::New(alloc(), proxy);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadDOMExpandoValueGuardGeneration(
+ ObjOperandId objId, uint32_t expandoAndGenerationOffset,
+ uint32_t generationOffset, ValOperandId resultId) {
+ MDefinition* proxy = getOperand(objId);
+ JS::ExpandoAndGeneration* expandoAndGeneration =
+ expandoAndGenerationField(expandoAndGenerationOffset);
+ uint64_t generation = uint64StubField(generationOffset);
+
+ auto* ins = MLoadDOMExpandoValueGuardGeneration::New(
+ alloc(), proxy, expandoAndGeneration, generation);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadDOMExpandoValueIgnoreGeneration(
+ ObjOperandId objId, ValOperandId resultId) {
+ MDefinition* proxy = getOperand(objId);
+
+ auto* ins = MLoadDOMExpandoValueIgnoreGeneration::New(alloc(), proxy);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardDOMExpandoMissingOrGuardShape(
+ ValOperandId expandoId, uint32_t shapeOffset) {
+ MDefinition* expando = getOperand(expandoId);
+ Shape* shape = shapeStubField(shapeOffset);
+
+ auto* ins = MGuardDOMExpandoMissingOrGuardShape::New(alloc(), expando, shape);
+ add(ins);
+
+ setOperand(expandoId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMegamorphicLoadSlotResult(ObjOperandId objId,
+ uint32_t nameOffset) {
+ MDefinition* obj = getOperand(objId);
+ PropertyName* name = stringStubField(nameOffset)->asAtom().asPropertyName();
+
+ auto* ins = MMegamorphicLoadSlot::New(alloc(), obj, NameToId(name));
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMegamorphicLoadSlotByValueResult(
+ ObjOperandId objId, ValOperandId idId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* id = getOperand(idId);
+
+ auto* ins = MMegamorphicLoadSlotByValue::New(alloc(), obj, id);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMegamorphicStoreSlot(ObjOperandId objId,
+ uint32_t nameOffset,
+ ValOperandId rhsId) {
+ MDefinition* obj = getOperand(objId);
+ PropertyName* name = stringStubField(nameOffset)->asAtom().asPropertyName();
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MMegamorphicStoreSlot::New(alloc(), obj, rhs, name);
+ addEffectful(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitMegamorphicHasPropResult(ObjOperandId objId,
+ ValOperandId idId,
+ bool hasOwn) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* id = getOperand(idId);
+
+ auto* ins = MMegamorphicHasProp::New(alloc(), obj, id, hasOwn);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMegamorphicSetElement(ObjOperandId objId,
+ ValOperandId idId,
+ ValOperandId rhsId,
+ bool strict) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* id = getOperand(idId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MMegamorphicSetElement::New(alloc(), obj, id, rhs, strict);
+ addEffectful(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitObjectToIteratorResult(
+ ObjOperandId objId, uint32_t enumeratorsAddrOffset) {
+ MDefinition* obj = getOperand(objId);
+ NativeIteratorListHead* enumeratorsAddr =
+ nativeIteratorListHeadStubField(enumeratorsAddrOffset);
+
+ auto* ins = MObjectToIterator::New(alloc(), obj, enumeratorsAddr);
+ addEffectful(ins);
+ pushResult(ins);
+ if (!resumeAfter(ins)) {
+ return false;
+ }
+
+ auto* barrier = MPostWriteBarrier::New(alloc(), ins, obj);
+ add(barrier);
+
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitValueToIteratorResult(ValOperandId valId) {
+ MDefinition* val = getOperand(valId);
+
+ auto* ins = MValueToIterator::New(alloc(), val);
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsNotArrayBufferMaybeShared(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardIsNotArrayBufferMaybeShared::New(alloc(), obj);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsTypedArray(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardIsTypedArray::New(alloc(), obj);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardProto(ObjOperandId objId,
+ uint32_t protoOffset) {
+ MDefinition* def = getOperand(objId);
+ MDefinition* proto = objectStubField(protoOffset);
+
+ auto* ins = MGuardProto::New(alloc(), def, proto);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardDynamicSlotIsSpecificObject(
+ ObjOperandId objId, ObjOperandId expectedId, uint32_t slotOffset) {
+ size_t slotIndex = int32StubField(slotOffset);
+ MDefinition* obj = getOperand(objId);
+ MDefinition* expected = getOperand(expectedId);
+
+ auto* slots = MSlots::New(alloc(), obj);
+ add(slots);
+
+ auto* load = MLoadDynamicSlot::New(alloc(), slots, slotIndex);
+ add(load);
+
+ auto* unbox = MUnbox::New(alloc(), load, MIRType::Object, MUnbox::Fallible);
+ add(unbox);
+
+ auto* guard = MGuardObjectIdentity::New(alloc(), unbox, expected,
+ /* bailOnEquality = */ false);
+ add(guard);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadDynamicSlot(ValOperandId resultId,
+ ObjOperandId objId,
+ uint32_t slotOffset) {
+ size_t slotIndex = int32StubField(slotOffset);
+ MDefinition* obj = getOperand(objId);
+
+ auto* slots = MSlots::New(alloc(), obj);
+ add(slots);
+
+ auto* load = MLoadDynamicSlot::New(alloc(), slots, slotIndex);
+ add(load);
+
+ return defineOperand(resultId, load);
+}
+
+bool WarpCacheIRTranspiler::emitGuardDynamicSlotIsNotObject(
+ ObjOperandId objId, uint32_t slotOffset) {
+ size_t slotIndex = int32StubField(slotOffset);
+ MDefinition* obj = getOperand(objId);
+
+ auto* slots = MSlots::New(alloc(), obj);
+ add(slots);
+
+ auto* load = MLoadDynamicSlot::New(alloc(), slots, slotIndex);
+ add(load);
+
+ auto* guard = MGuardIsNotObject::New(alloc(), load);
+ add(guard);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardFixedSlotValue(ObjOperandId objId,
+ uint32_t offsetOffset,
+ uint32_t valOffset) {
+ MDefinition* obj = getOperand(objId);
+
+ size_t offset = int32StubField(offsetOffset);
+ Value val = valueStubField(valOffset);
+ MOZ_ASSERT(val.isPrivateGCThing());
+
+ uint32_t slotIndex = NativeObject::getFixedSlotIndexFromOffset(offset);
+
+ auto* load = MLoadFixedSlot::New(alloc(), obj, slotIndex);
+ add(load);
+
+ auto* guard = MGuardValue::New(alloc(), load, val);
+ add(guard);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardDynamicSlotValue(ObjOperandId objId,
+ uint32_t offsetOffset,
+ uint32_t valOffset) {
+ MDefinition* obj = getOperand(objId);
+
+ size_t offset = int32StubField(offsetOffset);
+ Value val = valueStubField(valOffset);
+ MOZ_ASSERT(val.isPrivateGCThing());
+
+ size_t slotIndex = NativeObject::getDynamicSlotIndexFromOffset(offset);
+
+ auto* slots = MSlots::New(alloc(), obj);
+ add(slots);
+
+ auto* load = MLoadDynamicSlot::New(alloc(), slots, slotIndex);
+ add(load);
+
+ auto* guard = MGuardValue::New(alloc(), load, val);
+ add(guard);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardSpecificAtom(StringOperandId strId,
+ uint32_t expectedOffset) {
+ MDefinition* str = getOperand(strId);
+ JSString* expected = stringStubField(expectedOffset);
+
+ auto* ins = MGuardSpecificAtom::New(alloc(), str, &expected->asAtom());
+ add(ins);
+
+ setOperand(strId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardSpecificSymbol(SymbolOperandId symId,
+ uint32_t expectedOffset) {
+ MDefinition* symbol = getOperand(symId);
+ JS::Symbol* expected = symbolStubField(expectedOffset);
+
+ auto* ins = MGuardSpecificSymbol::New(alloc(), symbol, expected);
+ add(ins);
+
+ setOperand(symId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardSpecificInt32(Int32OperandId numId,
+ int32_t expected) {
+ MDefinition* num = getOperand(numId);
+
+ auto* ins = MGuardSpecificInt32::New(alloc(), num, expected);
+ add(ins);
+
+ setOperand(numId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardSpecificObject(ObjOperandId objId,
+ uint32_t expectedOffset) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* expected = objectStubField(expectedOffset);
+
+ auto* ins = MGuardObjectIdentity::New(alloc(), obj, expected,
+ /* bailOnEquality = */ false);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardSpecificFunction(
+ ObjOperandId objId, uint32_t expectedOffset, uint32_t nargsAndFlagsOffset) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* expected = objectStubField(expectedOffset);
+ uint32_t nargsAndFlags = uint32StubField(nargsAndFlagsOffset);
+
+ uint16_t nargs = nargsAndFlags >> 16;
+ FunctionFlags flags = FunctionFlags(uint16_t(nargsAndFlags));
+
+ auto* ins = MGuardSpecificFunction::New(alloc(), obj, expected, nargs, flags);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardFunctionScript(
+ ObjOperandId funId, uint32_t expectedOffset, uint32_t nargsAndFlagsOffset) {
+ MDefinition* fun = getOperand(funId);
+ BaseScript* expected = baseScriptStubField(expectedOffset);
+ uint32_t nargsAndFlags = uint32StubField(nargsAndFlagsOffset);
+
+ uint16_t nargs = nargsAndFlags >> 16;
+ FunctionFlags flags = FunctionFlags(uint16_t(nargsAndFlags));
+
+ auto* ins = MGuardFunctionScript::New(alloc(), fun, expected, nargs, flags);
+ add(ins);
+
+ setOperand(funId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardStringToIndex(StringOperandId strId,
+ Int32OperandId resultId) {
+ MDefinition* str = getOperand(strId);
+
+ auto* ins = MGuardStringToIndex::New(alloc(), str);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardStringToInt32(StringOperandId strId,
+ Int32OperandId resultId) {
+ MDefinition* str = getOperand(strId);
+
+ auto* ins = MGuardStringToInt32::New(alloc(), str);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardStringToNumber(StringOperandId strId,
+ NumberOperandId resultId) {
+ MDefinition* str = getOperand(strId);
+
+ auto* ins = MGuardStringToDouble::New(alloc(), str);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardNoDenseElements(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardNoDenseElements::New(alloc(), obj);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardFunctionHasJitEntry(ObjOperandId funId,
+ bool constructing) {
+ MDefinition* fun = getOperand(funId);
+ uint16_t expectedFlags = FunctionFlags::HasJitEntryFlags(constructing);
+ uint16_t unexpectedFlags = 0;
+
+ auto* ins =
+ MGuardFunctionFlags::New(alloc(), fun, expectedFlags, unexpectedFlags);
+ add(ins);
+
+ setOperand(funId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardFunctionHasNoJitEntry(ObjOperandId funId) {
+ MDefinition* fun = getOperand(funId);
+ uint16_t expectedFlags = 0;
+ uint16_t unexpectedFlags =
+ FunctionFlags::HasJitEntryFlags(/*isConstructing=*/false);
+
+ auto* ins =
+ MGuardFunctionFlags::New(alloc(), fun, expectedFlags, unexpectedFlags);
+ add(ins);
+
+ setOperand(funId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardFunctionIsNonBuiltinCtor(
+ ObjOperandId funId) {
+ MDefinition* fun = getOperand(funId);
+
+ auto* ins = MGuardFunctionIsNonBuiltinCtor::New(alloc(), fun);
+ add(ins);
+
+ setOperand(funId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardFunctionIsConstructor(ObjOperandId funId) {
+ MDefinition* fun = getOperand(funId);
+ uint16_t expectedFlags = FunctionFlags::CONSTRUCTOR;
+ uint16_t unexpectedFlags = 0;
+
+ auto* ins =
+ MGuardFunctionFlags::New(alloc(), fun, expectedFlags, unexpectedFlags);
+ add(ins);
+
+ setOperand(funId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardNotClassConstructor(ObjOperandId funId) {
+ MDefinition* fun = getOperand(funId);
+
+ auto* ins =
+ MGuardFunctionKind::New(alloc(), fun, FunctionFlags::ClassConstructor,
+ /*bailOnEquality=*/true);
+ add(ins);
+
+ setOperand(funId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardArrayIsPacked(ObjOperandId arrayId) {
+ MDefinition* array = getOperand(arrayId);
+
+ auto* ins = MGuardArrayIsPacked::New(alloc(), array);
+ add(ins);
+
+ setOperand(arrayId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardArgumentsObjectFlags(ObjOperandId objId,
+ uint8_t flags) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardArgumentsObjectFlags::New(alloc(), obj, flags);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardNonDoubleType(ValOperandId inputId,
+ ValueType type) {
+ switch (type) {
+ case ValueType::String:
+ case ValueType::Symbol:
+ case ValueType::BigInt:
+ case ValueType::Int32:
+ case ValueType::Boolean:
+ return emitGuardTo(inputId, MIRTypeFromValueType(JSValueType(type)));
+ case ValueType::Undefined:
+ return emitGuardIsUndefined(inputId);
+ case ValueType::Null:
+ return emitGuardIsNull(inputId);
+ case ValueType::Double:
+ case ValueType::Magic:
+ case ValueType::PrivateGCThing:
+ case ValueType::Object:
+#ifdef ENABLE_RECORD_TUPLE
+ case ValueType::ExtendedPrimitive:
+#endif
+ break;
+ }
+
+ MOZ_CRASH("unexpected type");
+}
+
+bool WarpCacheIRTranspiler::emitGuardTo(ValOperandId inputId, MIRType type) {
+ MDefinition* def = getOperand(inputId);
+ if (def->type() == type) {
+ return true;
+ }
+
+ auto* ins = MUnbox::New(alloc(), def, type, MUnbox::Fallible);
+ add(ins);
+
+ setOperand(inputId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardToObject(ValOperandId inputId) {
+ return emitGuardTo(inputId, MIRType::Object);
+}
+
+bool WarpCacheIRTranspiler::emitGuardToString(ValOperandId inputId) {
+ return emitGuardTo(inputId, MIRType::String);
+}
+
+bool WarpCacheIRTranspiler::emitGuardToSymbol(ValOperandId inputId) {
+ return emitGuardTo(inputId, MIRType::Symbol);
+}
+
+bool WarpCacheIRTranspiler::emitGuardToBigInt(ValOperandId inputId) {
+ return emitGuardTo(inputId, MIRType::BigInt);
+}
+
+bool WarpCacheIRTranspiler::emitGuardToBoolean(ValOperandId inputId) {
+ return emitGuardTo(inputId, MIRType::Boolean);
+}
+
+bool WarpCacheIRTranspiler::emitGuardToInt32(ValOperandId inputId) {
+ return emitGuardTo(inputId, MIRType::Int32);
+}
+
+bool WarpCacheIRTranspiler::emitGuardBooleanToInt32(ValOperandId inputId,
+ Int32OperandId resultId) {
+ MDefinition* input = getOperand(inputId);
+
+ MDefinition* boolean;
+ if (input->type() == MIRType::Boolean) {
+ boolean = input;
+ } else {
+ auto* unbox =
+ MUnbox::New(alloc(), input, MIRType::Boolean, MUnbox::Fallible);
+ add(unbox);
+ boolean = unbox;
+ }
+
+ auto* ins = MToIntegerInt32::New(alloc(), boolean);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsNumber(ValOperandId inputId) {
+ // Prefer MToDouble because it gets further optimizations downstream.
+ MDefinition* def = getOperand(inputId);
+ if (def->type() == MIRType::Int32) {
+ auto* ins = MToDouble::New(alloc(), def);
+ add(ins);
+
+ setOperand(inputId, ins);
+ return true;
+ }
+
+ // MIRType::Double also implies int32 in Ion.
+ return emitGuardTo(inputId, MIRType::Double);
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsNullOrUndefined(ValOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+ if (input->type() == MIRType::Null || input->type() == MIRType::Undefined) {
+ return true;
+ }
+
+ auto* ins = MGuardNullOrUndefined::New(alloc(), input);
+ add(ins);
+
+ setOperand(inputId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsNull(ValOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+ if (input->type() == MIRType::Null) {
+ return true;
+ }
+
+ auto* ins = MGuardValue::New(alloc(), input, NullValue());
+ add(ins);
+ setOperand(inputId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsUndefined(ValOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+ if (input->type() == MIRType::Undefined) {
+ return true;
+ }
+
+ auto* ins = MGuardValue::New(alloc(), input, UndefinedValue());
+ add(ins);
+ setOperand(inputId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsExtensible(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardIsExtensible::New(alloc(), obj);
+ add(ins);
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardInt32IsNonNegative(
+ Int32OperandId indexId) {
+ MDefinition* index = getOperand(indexId);
+
+ auto* ins = MGuardInt32IsNonNegative::New(alloc(), index);
+ add(ins);
+ setOperand(indexId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIndexIsNotDenseElement(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* ins = MGuardIndexIsNotDenseElement::New(alloc(), obj, index);
+ add(ins);
+ setOperand(indexId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIndexIsValidUpdateOrAdd(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* ins = MGuardIndexIsValidUpdateOrAdd::New(alloc(), obj, index);
+ add(ins);
+ setOperand(indexId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCallAddOrUpdateSparseElementHelper(
+ ObjOperandId objId, Int32OperandId idId, ValOperandId rhsId, bool strict) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* id = getOperand(idId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MCallAddOrUpdateSparseElement::New(alloc(), obj, id, rhs, strict);
+ addEffectful(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardTagNotEqual(ValueTagOperandId lhsId,
+ ValueTagOperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MGuardTagNotEqual::New(alloc(), lhs, rhs);
+ add(ins);
+
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardToInt32Index(ValOperandId inputId,
+ Int32OperandId resultId) {
+ MDefinition* input = getOperand(inputId);
+ auto* ins =
+ MToNumberInt32::New(alloc(), input, IntConversionInputKind::NumbersOnly);
+
+ // ToPropertyKey(-0) is "0", so we can silently convert -0 to 0 here.
+ ins->setNeedsNegativeZeroCheck(false);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitTruncateDoubleToUInt32(
+ NumberOperandId inputId, Int32OperandId resultId) {
+ MDefinition* input = getOperand(inputId);
+ auto* ins = MTruncateToInt32::New(alloc(), input);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardToInt32ModUint32(ValOperandId valId,
+ Int32OperandId resultId) {
+ MDefinition* input = getOperand(valId);
+ auto* ins = MTruncateToInt32::New(alloc(), input);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardToUint8Clamped(ValOperandId valId,
+ Int32OperandId resultId) {
+ MDefinition* input = getOperand(valId);
+ auto* ins = MClampToUint8::New(alloc(), input);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitToString(OperandId inputId,
+ StringOperandId resultId) {
+ MDefinition* input = getOperand(inputId);
+ auto* ins =
+ MToString::New(alloc(), input, MToString::SideEffectHandling::Bailout);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitInt32ToIntPtr(Int32OperandId inputId,
+ IntPtrOperandId resultId) {
+ MDefinition* input = getOperand(inputId);
+ auto* ins = MInt32ToIntPtr::New(alloc(), input);
+ add(ins);
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitGuardNumberToIntPtrIndex(
+ NumberOperandId inputId, bool supportOOB, IntPtrOperandId resultId) {
+ MDefinition* input = getOperand(inputId);
+ auto* ins = MGuardNumberToIntPtrIndex::New(alloc(), input, supportOOB);
+ add(ins);
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitCallInt32ToString(Int32OperandId inputId,
+ StringOperandId resultId) {
+ return emitToString(inputId, resultId);
+}
+
+bool WarpCacheIRTranspiler::emitCallNumberToString(NumberOperandId inputId,
+ StringOperandId resultId) {
+ return emitToString(inputId, resultId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32ToStringWithBaseResult(
+ Int32OperandId inputId, Int32OperandId baseId) {
+ MDefinition* input = getOperand(inputId);
+ MDefinition* base = getOperand(baseId);
+
+ auto* guardedBase = MGuardInt32Range::New(alloc(), base, 2, 36);
+ add(guardedBase);
+
+ auto* ins = MInt32ToStringWithBase::New(alloc(), input, guardedBase);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitBooleanToString(BooleanOperandId inputId,
+ StringOperandId resultId) {
+ return emitToString(inputId, resultId);
+}
+
+bool WarpCacheIRTranspiler::emitBooleanToNumber(BooleanOperandId inputId,
+ NumberOperandId resultId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MToDouble::New(alloc(), input);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadInt32Result(Int32OperandId valId) {
+ MDefinition* val = getOperand(valId);
+ MOZ_ASSERT(val->type() == MIRType::Int32);
+ pushResult(val);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadDoubleResult(NumberOperandId valId) {
+ MDefinition* val = getOperand(valId);
+ MOZ_ASSERT(val->type() == MIRType::Double);
+ pushResult(val);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadBigIntResult(BigIntOperandId valId) {
+ MDefinition* val = getOperand(valId);
+ MOZ_ASSERT(val->type() == MIRType::BigInt);
+ pushResult(val);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadObjectResult(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+ MOZ_ASSERT(obj->type() == MIRType::Object);
+ pushResult(obj);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadStringResult(StringOperandId strId) {
+ MDefinition* str = getOperand(strId);
+ MOZ_ASSERT(str->type() == MIRType::String);
+ pushResult(str);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadSymbolResult(SymbolOperandId symId) {
+ MDefinition* sym = getOperand(symId);
+ MOZ_ASSERT(sym->type() == MIRType::Symbol);
+ pushResult(sym);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadUndefinedResult() {
+ pushResult(constant(UndefinedValue()));
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadBooleanResult(bool val) {
+ pushResult(constant(BooleanValue(val)));
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadInt32Constant(uint32_t valOffset,
+ Int32OperandId resultId) {
+ int32_t val = int32StubField(valOffset);
+ auto* valConst = constant(Int32Value(val));
+ return defineOperand(resultId, valConst);
+}
+
+bool WarpCacheIRTranspiler::emitLoadDoubleConstant(uint32_t valOffset,
+ NumberOperandId resultId) {
+ double val = doubleStubField(valOffset);
+ auto* valConst = constant(DoubleValue(val));
+ return defineOperand(resultId, valConst);
+}
+
+bool WarpCacheIRTranspiler::emitLoadBooleanConstant(bool val,
+ BooleanOperandId resultId) {
+ auto* valConst = constant(BooleanValue(val));
+ return defineOperand(resultId, valConst);
+}
+
+bool WarpCacheIRTranspiler::emitLoadUndefined(ValOperandId resultId) {
+ auto* valConst = constant(UndefinedValue());
+ return defineOperand(resultId, valConst);
+}
+
+bool WarpCacheIRTranspiler::emitLoadConstantString(uint32_t strOffset,
+ StringOperandId resultId) {
+ JSString* val = stringStubField(strOffset);
+ auto* valConst = constant(StringValue(val));
+ return defineOperand(resultId, valConst);
+}
+
+bool WarpCacheIRTranspiler::emitLoadConstantStringResult(uint32_t strOffset) {
+ JSString* val = stringStubField(strOffset);
+ auto* valConst = constant(StringValue(val));
+ pushResult(valConst);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadTypeOfObjectResult(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+ auto* typeOf = MTypeOf::New(alloc(), obj);
+ add(typeOf);
+
+ auto* ins = MTypeOfName::New(alloc(), typeOf);
+ add(ins);
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadEnclosingEnvironment(
+ ObjOperandId objId, ObjOperandId resultId) {
+ MDefinition* env = getOperand(objId);
+ auto* ins = MEnclosingEnvironment::New(alloc(), env);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadObject(ObjOperandId resultId,
+ uint32_t objOffset) {
+ MInstruction* ins = objectStubField(objOffset);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadProtoObject(ObjOperandId resultId,
+ uint32_t objOffset,
+ ObjOperandId receiverObjId) {
+ MInstruction* ins = objectStubField(objOffset);
+ if (ins->isConstant()) {
+ MDefinition* receiverObj = getOperand(receiverObjId);
+
+ ins = MConstantProto::New(alloc(), ins, receiverObj);
+ add(ins);
+ }
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadProto(ObjOperandId objId,
+ ObjOperandId resultId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MObjectStaticProto::New(alloc(), obj);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadInstanceOfObjectResult(
+ ValOperandId lhsId, ObjOperandId protoId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* proto = getOperand(protoId);
+
+ auto* instanceOf = MInstanceOf::New(alloc(), lhs, proto);
+ addEffectful(instanceOf);
+
+ pushResult(instanceOf);
+ return resumeAfter(instanceOf);
+}
+
+bool WarpCacheIRTranspiler::emitLoadValueTag(ValOperandId valId,
+ ValueTagOperandId resultId) {
+ MDefinition* val = getOperand(valId);
+
+ auto* ins = MLoadValueTag::New(alloc(), val);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadDynamicSlotResult(ObjOperandId objId,
+ uint32_t offsetOffset) {
+ int32_t offset = int32StubField(offsetOffset);
+
+ MDefinition* obj = getOperand(objId);
+ size_t slotIndex = NativeObject::getDynamicSlotIndexFromOffset(offset);
+
+ auto* slots = MSlots::New(alloc(), obj);
+ add(slots);
+
+ auto* load = MLoadDynamicSlot::New(alloc(), slots, slotIndex);
+ add(load);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadFixedSlot(ValOperandId resultId,
+ ObjOperandId objId,
+ uint32_t offsetOffset) {
+ MDefinition* obj = getOperand(objId);
+
+ size_t offset = int32StubField(offsetOffset);
+ uint32_t slotIndex = NativeObject::getFixedSlotIndexFromOffset(offset);
+
+ auto* load = MLoadFixedSlot::New(alloc(), obj, slotIndex);
+ add(load);
+
+ return defineOperand(resultId, load);
+}
+
+bool WarpCacheIRTranspiler::emitLoadFixedSlotResult(ObjOperandId objId,
+ uint32_t offsetOffset) {
+ int32_t offset = int32StubField(offsetOffset);
+
+ MDefinition* obj = getOperand(objId);
+ uint32_t slotIndex = NativeObject::getFixedSlotIndexFromOffset(offset);
+
+ auto* load = MLoadFixedSlot::New(alloc(), obj, slotIndex);
+ add(load);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadFixedSlotTypedResult(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValueType type) {
+ int32_t offset = int32StubField(offsetOffset);
+
+ MDefinition* obj = getOperand(objId);
+ uint32_t slotIndex = NativeObject::getFixedSlotIndexFromOffset(offset);
+
+ auto* load = MLoadFixedSlot::New(alloc(), obj, slotIndex);
+ load->setResultType(MIRTypeFromValueType(JSValueType(type)));
+ add(load);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardIsNotUninitializedLexical(
+ ValOperandId valId) {
+ MDefinition* val = getOperand(valId);
+
+ auto* lexicalCheck = MLexicalCheck::New(alloc(), val);
+ add(lexicalCheck);
+
+ if (snapshot().bailoutInfo().failedLexicalCheck()) {
+ lexicalCheck->setNotMovable();
+ }
+
+ setOperand(valId, lexicalCheck);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadInt32ArrayLengthResult(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* length = MArrayLength::New(alloc(), elements);
+ add(length);
+
+ pushResult(length);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadInt32ArrayLength(ObjOperandId objId,
+ Int32OperandId resultId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* length = MArrayLength::New(alloc(), elements);
+ add(length);
+
+ return defineOperand(resultId, length);
+}
+
+bool WarpCacheIRTranspiler::emitLoadArgumentsObjectArgResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* load = MLoadArgumentsObjectArg::New(alloc(), obj, index);
+ add(load);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadArgumentsObjectArgHoleResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* load = MLoadArgumentsObjectArgHole::New(alloc(), obj, index);
+ add(load);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadArgumentsObjectArgExistsResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* ins = MInArgumentsObjectArg::New(alloc(), obj, index);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadArgumentsObjectLengthResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* length = MArgumentsObjectLength::New(alloc(), obj);
+ add(length);
+
+ pushResult(length);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadArgumentsObjectLength(
+ ObjOperandId objId, Int32OperandId resultId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* length = MArgumentsObjectLength::New(alloc(), obj);
+ add(length);
+
+ return defineOperand(resultId, length);
+}
+
+bool WarpCacheIRTranspiler::emitArrayFromArgumentsObjectResult(
+ ObjOperandId objId, uint32_t shapeOffset) {
+ MDefinition* obj = getOperand(objId);
+ Shape* shape = shapeStubField(shapeOffset);
+ MOZ_ASSERT(shape);
+
+ auto* array = MArrayFromArgumentsObject::New(alloc(), obj, shape);
+ addEffectful(array);
+
+ pushResult(array);
+ return resumeAfter(array);
+}
+
+bool WarpCacheIRTranspiler::emitLoadFunctionLengthResult(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* length = MFunctionLength::New(alloc(), obj);
+ add(length);
+
+ pushResult(length);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadFunctionNameResult(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* name = MFunctionName::New(alloc(), obj);
+ add(name);
+
+ pushResult(name);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadArrayBufferByteLengthInt32Result(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* length = MArrayBufferByteLength::New(alloc(), obj);
+ add(length);
+
+ auto* lengthInt32 = MNonNegativeIntPtrToInt32::New(alloc(), length);
+ add(lengthInt32);
+
+ pushResult(lengthInt32);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadArrayBufferByteLengthDoubleResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* length = MArrayBufferByteLength::New(alloc(), obj);
+ add(length);
+
+ auto* lengthDouble = MIntPtrToDouble::New(alloc(), length);
+ add(lengthDouble);
+
+ pushResult(lengthDouble);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadArrayBufferViewLengthInt32Result(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ // Use a separate instruction for converting the length to Int32, so that we
+ // can fold the MArrayBufferViewLength instruction with length instructions
+ // added for bounds checks.
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ auto* lengthInt32 = MNonNegativeIntPtrToInt32::New(alloc(), length);
+ add(lengthInt32);
+
+ pushResult(lengthInt32);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadArrayBufferViewLengthDoubleResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ auto* lengthDouble = MIntPtrToDouble::New(alloc(), length);
+ add(lengthDouble);
+
+ pushResult(lengthDouble);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadStringLengthResult(StringOperandId strId) {
+ MDefinition* str = getOperand(strId);
+
+ auto* length = MStringLength::New(alloc(), str);
+ add(length);
+
+ pushResult(length);
+ return true;
+}
+
+MInstruction* WarpCacheIRTranspiler::addBoundsCheck(MDefinition* index,
+ MDefinition* length) {
+ MInstruction* check = MBoundsCheck::New(alloc(), index, length);
+ add(check);
+
+ if (snapshot().bailoutInfo().failedBoundsCheck()) {
+ check->setNotMovable();
+ }
+
+ if (JitOptions.spectreIndexMasking) {
+ // Use a separate MIR instruction for the index masking. Doing this as
+ // part of MBoundsCheck would be unsound because bounds checks can be
+ // optimized or eliminated completely. Consider this:
+ //
+ // for (var i = 0; i < x; i++)
+ // res = arr[i];
+ //
+ // If we can prove |x < arr.length|, we are able to eliminate the bounds
+ // check, but we should not get rid of the index masking because the
+ // |i < x| branch could still be mispredicted.
+ //
+ // Using a separate instruction lets us eliminate the bounds check
+ // without affecting the index masking.
+ check = MSpectreMaskIndex::New(alloc(), check, length);
+ add(check);
+ }
+
+ return check;
+}
+
+bool WarpCacheIRTranspiler::emitLoadDenseElementResult(ObjOperandId objId,
+ Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* length = MInitializedLength::New(alloc(), elements);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* load = MLoadElement::New(alloc(), elements, index);
+ add(load);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadDenseElementHoleResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* length = MInitializedLength::New(alloc(), elements);
+ add(length);
+
+ auto* load = MLoadElementHole::New(alloc(), elements, index, length);
+ add(load);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCallGetSparseElementResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* call = MCallGetSparseElement::New(alloc(), obj, index);
+ addEffectful(call);
+
+ pushResult(call);
+ return resumeAfter(call);
+}
+
+bool WarpCacheIRTranspiler::emitCallNativeGetElementResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* call = MCallNativeGetElement::New(alloc(), obj, index);
+ addEffectful(call);
+
+ pushResult(call);
+ return resumeAfter(call);
+}
+
+bool WarpCacheIRTranspiler::emitCallNativeGetElementSuperResult(
+ ObjOperandId objId, Int32OperandId indexId, ValOperandId receiverId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+ MDefinition* receiver = getOperand(receiverId);
+
+ auto* call = MCallNativeGetElementSuper::New(alloc(), obj, index, receiver);
+ addEffectful(call);
+
+ pushResult(call);
+ return resumeAfter(call);
+}
+
+bool WarpCacheIRTranspiler::emitLoadDenseElementExistsResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ // Get the elements vector.
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* length = MInitializedLength::New(alloc(), elements);
+ add(length);
+
+ // Check if id < initLength.
+ index = addBoundsCheck(index, length);
+
+ // And check elem[id] is not a hole.
+ auto* guard = MGuardElementNotHole::New(alloc(), elements, index);
+ add(guard);
+
+ pushResult(constant(BooleanValue(true)));
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadDenseElementHoleExistsResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ // Get the elements vector.
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* length = MInitializedLength::New(alloc(), elements);
+ add(length);
+
+ // Check if id < initLength and elem[id] not a hole.
+ auto* ins = MInArray::New(alloc(), elements, index, length, obj);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCallObjectHasSparseElementResult(
+ ObjOperandId objId, Int32OperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* ins = MCallObjectHasSparseElement::New(alloc(), obj, index);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadTypedArrayElementExistsResult(
+ ObjOperandId objId, IntPtrOperandId indexId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ // Unsigned comparison to catch negative indices.
+ auto* ins = MCompare::New(alloc(), index, length, JSOp::Lt,
+ MCompare::Compare_UIntPtr);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+static MIRType MIRTypeForArrayBufferViewRead(Scalar::Type arrayType,
+ bool forceDoubleForUint32) {
+ switch (arrayType) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Uint8Clamped:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ return MIRType::Int32;
+ case Scalar::Uint32:
+ return forceDoubleForUint32 ? MIRType::Double : MIRType::Int32;
+ case Scalar::Float32:
+ return MIRType::Float32;
+ case Scalar::Float64:
+ return MIRType::Double;
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ return MIRType::BigInt;
+ default:
+ break;
+ }
+ MOZ_CRASH("Unknown typed array type");
+}
+
+bool WarpCacheIRTranspiler::emitLoadTypedArrayElementResult(
+ ObjOperandId objId, IntPtrOperandId indexId, Scalar::Type elementType,
+ bool handleOOB, bool forceDoubleForUint32) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ if (handleOOB) {
+ auto* load = MLoadTypedArrayElementHole::New(
+ alloc(), obj, index, elementType, forceDoubleForUint32);
+ add(load);
+
+ pushResult(load);
+ return true;
+ }
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* elements = MArrayBufferViewElements::New(alloc(), obj);
+ add(elements);
+
+ auto* load = MLoadUnboxedScalar::New(alloc(), elements, index, elementType);
+ load->setResultType(
+ MIRTypeForArrayBufferViewRead(elementType, forceDoubleForUint32));
+ add(load);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLinearizeForCharAccess(
+ StringOperandId strId, Int32OperandId indexId, StringOperandId resultId) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* ins = MLinearizeForCharAccess::New(alloc(), str, index);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitLoadStringCharResult(StringOperandId strId,
+ Int32OperandId indexId,
+ bool handleOOB) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* index = getOperand(indexId);
+
+ if (handleOOB) {
+ auto* ins = MCharAtMaybeOutOfBounds::New(alloc(), str, index);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+ }
+
+ auto* length = MStringLength::New(alloc(), str);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* charCode = MCharCodeAt::New(alloc(), str, index);
+ add(charCode);
+
+ auto* fromCharCode = MFromCharCode::New(alloc(), charCode);
+ add(fromCharCode);
+
+ pushResult(fromCharCode);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadStringCharCodeResult(StringOperandId strId,
+ Int32OperandId indexId,
+ bool handleOOB) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* index = getOperand(indexId);
+
+ if (handleOOB) {
+ auto* ins = MCharCodeAtMaybeOutOfBounds::New(alloc(), str, index);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+ }
+
+ auto* length = MStringLength::New(alloc(), str);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* charCode = MCharCodeAt::New(alloc(), str, index);
+ add(charCode);
+
+ pushResult(charCode);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitNewStringObjectResult(
+ uint32_t templateObjectOffset, StringOperandId strId) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+ MDefinition* string = getOperand(strId);
+
+ auto* obj = MNewStringObject::New(alloc(), string, templateObj);
+ addEffectful(obj);
+
+ pushResult(obj);
+ return resumeAfter(obj);
+}
+
+bool WarpCacheIRTranspiler::emitStringFromCharCodeResult(
+ Int32OperandId codeId) {
+ MDefinition* code = getOperand(codeId);
+
+ auto* fromCharCode = MFromCharCode::New(alloc(), code);
+ add(fromCharCode);
+
+ pushResult(fromCharCode);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStringFromCodePointResult(
+ Int32OperandId codeId) {
+ MDefinition* code = getOperand(codeId);
+
+ auto* fromCodePoint = MFromCodePoint::New(alloc(), code);
+ add(fromCodePoint);
+
+ pushResult(fromCodePoint);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStringIndexOfResult(
+ StringOperandId strId, StringOperandId searchStrId) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* searchStr = getOperand(searchStrId);
+
+ auto* indexOf = MStringIndexOf::New(alloc(), str, searchStr);
+ add(indexOf);
+
+ pushResult(indexOf);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStringStartsWithResult(
+ StringOperandId strId, StringOperandId searchStrId) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* searchStr = getOperand(searchStrId);
+
+ auto* startsWith = MStringStartsWith::New(alloc(), str, searchStr);
+ add(startsWith);
+
+ pushResult(startsWith);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStringEndsWithResult(
+ StringOperandId strId, StringOperandId searchStrId) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* searchStr = getOperand(searchStrId);
+
+ auto* endsWith = MStringEndsWith::New(alloc(), str, searchStr);
+ add(endsWith);
+
+ pushResult(endsWith);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStringToLowerCaseResult(StringOperandId strId) {
+ MDefinition* str = getOperand(strId);
+
+ auto* convert =
+ MStringConvertCase::New(alloc(), str, MStringConvertCase::LowerCase);
+ add(convert);
+
+ pushResult(convert);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStringToUpperCaseResult(StringOperandId strId) {
+ MDefinition* str = getOperand(strId);
+
+ auto* convert =
+ MStringConvertCase::New(alloc(), str, MStringConvertCase::UpperCase);
+ add(convert);
+
+ pushResult(convert);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStoreDynamicSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId) {
+ int32_t offset = int32StubField(offsetOffset);
+
+ MDefinition* obj = getOperand(objId);
+ size_t slotIndex = NativeObject::getDynamicSlotIndexFromOffset(offset);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* barrier = MPostWriteBarrier::New(alloc(), obj, rhs);
+ add(barrier);
+
+ auto* slots = MSlots::New(alloc(), obj);
+ add(slots);
+
+ auto* store = MStoreDynamicSlot::NewBarriered(alloc(), slots, slotIndex, rhs);
+ addEffectful(store);
+ return resumeAfter(store);
+}
+
+bool WarpCacheIRTranspiler::emitStoreFixedSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId) {
+ int32_t offset = int32StubField(offsetOffset);
+
+ MDefinition* obj = getOperand(objId);
+ size_t slotIndex = NativeObject::getFixedSlotIndexFromOffset(offset);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* barrier = MPostWriteBarrier::New(alloc(), obj, rhs);
+ add(barrier);
+
+ auto* store = MStoreFixedSlot::NewBarriered(alloc(), obj, slotIndex, rhs);
+ addEffectful(store);
+ return resumeAfter(store);
+}
+
+bool WarpCacheIRTranspiler::emitStoreFixedSlotUndefinedResult(
+ ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId) {
+ int32_t offset = int32StubField(offsetOffset);
+
+ MDefinition* obj = getOperand(objId);
+ size_t slotIndex = NativeObject::getFixedSlotIndexFromOffset(offset);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* barrier = MPostWriteBarrier::New(alloc(), obj, rhs);
+ add(barrier);
+
+ auto* store = MStoreFixedSlot::NewBarriered(alloc(), obj, slotIndex, rhs);
+ addEffectful(store);
+
+ auto* undef = constant(UndefinedValue());
+ pushResult(undef);
+
+ return resumeAfter(store);
+}
+
+bool WarpCacheIRTranspiler::emitAddAndStoreSlotShared(
+ MAddAndStoreSlot::Kind kind, ObjOperandId objId, uint32_t offsetOffset,
+ ValOperandId rhsId, uint32_t newShapeOffset) {
+ int32_t offset = int32StubField(offsetOffset);
+ Shape* shape = shapeStubField(newShapeOffset);
+
+ MDefinition* obj = getOperand(objId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* barrier = MPostWriteBarrier::New(alloc(), obj, rhs);
+ add(barrier);
+
+ auto* addAndStore =
+ MAddAndStoreSlot::New(alloc(), obj, rhs, kind, offset, shape);
+ addEffectful(addAndStore);
+
+ return resumeAfter(addAndStore);
+}
+
+bool WarpCacheIRTranspiler::emitAddAndStoreFixedSlot(ObjOperandId objId,
+ uint32_t offsetOffset,
+ ValOperandId rhsId,
+ uint32_t newShapeOffset) {
+ return emitAddAndStoreSlotShared(MAddAndStoreSlot::Kind::FixedSlot, objId,
+ offsetOffset, rhsId, newShapeOffset);
+}
+
+bool WarpCacheIRTranspiler::emitAddAndStoreDynamicSlot(
+ ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset) {
+ return emitAddAndStoreSlotShared(MAddAndStoreSlot::Kind::DynamicSlot, objId,
+ offsetOffset, rhsId, newShapeOffset);
+}
+
+bool WarpCacheIRTranspiler::emitAllocateAndStoreDynamicSlot(
+ ObjOperandId objId, uint32_t offsetOffset, ValOperandId rhsId,
+ uint32_t newShapeOffset, uint32_t numNewSlotsOffset) {
+ int32_t offset = int32StubField(offsetOffset);
+ Shape* shape = shapeStubField(newShapeOffset);
+ uint32_t numNewSlots = uint32StubField(numNewSlotsOffset);
+
+ MDefinition* obj = getOperand(objId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* barrier = MPostWriteBarrier::New(alloc(), obj, rhs);
+ add(barrier);
+
+ auto* allocateAndStore =
+ MAllocateAndStoreSlot::New(alloc(), obj, rhs, offset, shape, numNewSlots);
+ addEffectful(allocateAndStore);
+
+ return resumeAfter(allocateAndStore);
+}
+
+bool WarpCacheIRTranspiler::emitAddSlotAndCallAddPropHook(
+ ObjOperandId objId, ValOperandId rhsId, uint32_t newShapeOffset) {
+ Shape* shape = shapeStubField(newShapeOffset);
+ MDefinition* obj = getOperand(objId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* addProp = MAddSlotAndCallAddPropHook::New(alloc(), obj, rhs, shape);
+ addEffectful(addProp);
+
+ return resumeAfter(addProp);
+}
+
+bool WarpCacheIRTranspiler::emitStoreDenseElement(ObjOperandId objId,
+ Int32OperandId indexId,
+ ValOperandId rhsId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* length = MInitializedLength::New(alloc(), elements);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* barrier = MPostWriteElementBarrier::New(alloc(), obj, rhs, index);
+ add(barrier);
+
+ bool needsHoleCheck = true;
+ auto* store = MStoreElement::NewBarriered(alloc(), elements, index, rhs,
+ needsHoleCheck);
+ addEffectful(store);
+ return resumeAfter(store);
+}
+
+bool WarpCacheIRTranspiler::emitStoreDenseElementHole(ObjOperandId objId,
+ Int32OperandId indexId,
+ ValOperandId rhsId,
+ bool handleAdd) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* barrier = MPostWriteElementBarrier::New(alloc(), obj, rhs, index);
+ add(barrier);
+
+ MInstruction* store;
+ if (handleAdd) {
+ // TODO(post-Warp): Consider changing MStoreElementHole to match IC code.
+ store = MStoreElementHole::New(alloc(), obj, elements, index, rhs);
+ } else {
+ auto* length = MInitializedLength::New(alloc(), elements);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ bool needsHoleCheck = false;
+ store = MStoreElement::NewBarriered(alloc(), elements, index, rhs,
+ needsHoleCheck);
+ }
+ addEffectful(store);
+
+ return resumeAfter(store);
+}
+
+bool WarpCacheIRTranspiler::emitStoreTypedArrayElement(ObjOperandId objId,
+ Scalar::Type elementType,
+ IntPtrOperandId indexId,
+ uint32_t rhsId,
+ bool handleOOB) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+ MDefinition* rhs = getOperand(ValOperandId(rhsId));
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ if (!handleOOB) {
+ // MStoreTypedArrayElementHole does the bounds checking.
+ index = addBoundsCheck(index, length);
+ }
+
+ auto* elements = MArrayBufferViewElements::New(alloc(), obj);
+ add(elements);
+
+ MInstruction* store;
+ if (handleOOB) {
+ store = MStoreTypedArrayElementHole::New(alloc(), elements, length, index,
+ rhs, elementType);
+ } else {
+ store =
+ MStoreUnboxedScalar::New(alloc(), elements, index, rhs, elementType);
+ }
+ addEffectful(store);
+ return resumeAfter(store);
+}
+
+void WarpCacheIRTranspiler::addDataViewData(MDefinition* obj, Scalar::Type type,
+ MDefinition** offset,
+ MInstruction** elements) {
+ MInstruction* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ // Adjust the length to account for accesses near the end of the dataview.
+ if (size_t byteSize = Scalar::byteSize(type); byteSize > 1) {
+ // To ensure |0 <= offset && offset + byteSize <= length|, first adjust the
+ // length by subtracting |byteSize - 1| (bailing out if that becomes
+ // negative).
+ length = MAdjustDataViewLength::New(alloc(), length, byteSize);
+ add(length);
+ }
+
+ *offset = addBoundsCheck(*offset, length);
+
+ *elements = MArrayBufferViewElements::New(alloc(), obj);
+ add(*elements);
+}
+
+bool WarpCacheIRTranspiler::emitLoadDataViewValueResult(
+ ObjOperandId objId, IntPtrOperandId offsetId,
+ BooleanOperandId littleEndianId, Scalar::Type elementType,
+ bool forceDoubleForUint32) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* offset = getOperand(offsetId);
+ MDefinition* littleEndian = getOperand(littleEndianId);
+
+ // Add bounds check and get the DataViewObject's elements.
+ MInstruction* elements;
+ addDataViewData(obj, elementType, &offset, &elements);
+
+ // Load the element.
+ MInstruction* load;
+ if (Scalar::byteSize(elementType) == 1) {
+ load = MLoadUnboxedScalar::New(alloc(), elements, offset, elementType);
+ } else {
+ load = MLoadDataViewElement::New(alloc(), elements, offset, littleEndian,
+ elementType);
+ }
+ add(load);
+
+ MIRType knownType =
+ MIRTypeForArrayBufferViewRead(elementType, forceDoubleForUint32);
+ load->setResultType(knownType);
+
+ pushResult(load);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStoreDataViewValueResult(
+ ObjOperandId objId, IntPtrOperandId offsetId, uint32_t valueId,
+ BooleanOperandId littleEndianId, Scalar::Type elementType) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* offset = getOperand(offsetId);
+ MDefinition* value = getOperand(ValOperandId(valueId));
+ MDefinition* littleEndian = getOperand(littleEndianId);
+
+ // Add bounds check and get the DataViewObject's elements.
+ MInstruction* elements;
+ addDataViewData(obj, elementType, &offset, &elements);
+
+ // Store the element.
+ MInstruction* store;
+ if (Scalar::byteSize(elementType) == 1) {
+ store =
+ MStoreUnboxedScalar::New(alloc(), elements, offset, value, elementType);
+ } else {
+ store = MStoreDataViewElement::New(alloc(), elements, offset, value,
+ littleEndian, elementType);
+ }
+ addEffectful(store);
+
+ pushResult(constant(UndefinedValue()));
+
+ return resumeAfter(store);
+}
+
+bool WarpCacheIRTranspiler::emitInt32IncResult(Int32OperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* constOne = MConstant::New(alloc(), Int32Value(1));
+ add(constOne);
+
+ auto* ins = MAdd::New(alloc(), input, constOne, MIRType::Int32);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitDoubleIncResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* constOne = MConstant::New(alloc(), DoubleValue(1.0));
+ add(constOne);
+
+ auto* ins = MAdd::New(alloc(), input, constOne, MIRType::Double);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitInt32DecResult(Int32OperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* constOne = MConstant::New(alloc(), Int32Value(1));
+ add(constOne);
+
+ auto* ins = MSub::New(alloc(), input, constOne, MIRType::Int32);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitDoubleDecResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* constOne = MConstant::New(alloc(), DoubleValue(1.0));
+ add(constOne);
+
+ auto* ins = MSub::New(alloc(), input, constOne, MIRType::Double);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitInt32NegationResult(Int32OperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* constNegOne = MConstant::New(alloc(), Int32Value(-1));
+ add(constNegOne);
+
+ auto* ins = MMul::New(alloc(), input, constNegOne, MIRType::Int32);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitDoubleNegationResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* constNegOne = MConstant::New(alloc(), DoubleValue(-1.0));
+ add(constNegOne);
+
+ auto* ins = MMul::New(alloc(), input, constNegOne, MIRType::Double);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitInt32NotResult(Int32OperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MBitNot::New(alloc(), input);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+template <typename T>
+bool WarpCacheIRTranspiler::emitDoubleBinaryArithResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = T::New(alloc(), lhs, rhs, MIRType::Double);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitDoubleAddResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ return emitDoubleBinaryArithResult<MAdd>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitDoubleSubResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ return emitDoubleBinaryArithResult<MSub>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitDoubleMulResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ return emitDoubleBinaryArithResult<MMul>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitDoubleDivResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ return emitDoubleBinaryArithResult<MDiv>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitDoubleModResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ return emitDoubleBinaryArithResult<MMod>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitDoublePowResult(NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ return emitDoubleBinaryArithResult<MPow>(lhsId, rhsId);
+}
+
+template <typename T>
+bool WarpCacheIRTranspiler::emitInt32BinaryArithResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = T::New(alloc(), lhs, rhs, MIRType::Int32);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitInt32AddResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MAdd>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32SubResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MSub>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32MulResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MMul>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32DivResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MDiv>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32ModResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MMod>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32PowResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MPow>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32BitOrResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MBitOr>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32BitXorResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MBitXor>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32BitAndResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MBitAnd>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32LeftShiftResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MLsh>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32RightShiftResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitInt32BinaryArithResult<MRsh>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitInt32URightShiftResult(Int32OperandId lhsId,
+ Int32OperandId rhsId,
+ bool forceDouble) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ MIRType specialization = forceDouble ? MIRType::Double : MIRType::Int32;
+ auto* ins = MUrsh::New(alloc(), lhs, rhs, specialization);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+template <typename T>
+bool WarpCacheIRTranspiler::emitBigIntBinaryArithResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = T::New(alloc(), lhs, rhs);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitBigIntAddResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithResult<MBigIntAdd>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntSubResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithResult<MBigIntSub>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntMulResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithResult<MBigIntMul>(lhsId, rhsId);
+}
+
+template <typename T>
+bool WarpCacheIRTranspiler::emitBigIntBinaryArithEffectfulResult(
+ BigIntOperandId lhsId, BigIntOperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = T::New(alloc(), lhs, rhs);
+
+ if (ins->isEffectful()) {
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+ }
+
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitBigIntDivResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithEffectfulResult<MBigIntDiv>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntModResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithEffectfulResult<MBigIntMod>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntPowResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithEffectfulResult<MBigIntPow>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntBitAndResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithResult<MBigIntBitAnd>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntBitOrResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithResult<MBigIntBitOr>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntBitXorResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithResult<MBigIntBitXor>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntLeftShiftResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithResult<MBigIntLsh>(lhsId, rhsId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntRightShiftResult(BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitBigIntBinaryArithResult<MBigIntRsh>(lhsId, rhsId);
+}
+
+template <typename T>
+bool WarpCacheIRTranspiler::emitBigIntUnaryArithResult(
+ BigIntOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = T::New(alloc(), input);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitBigIntIncResult(BigIntOperandId inputId) {
+ return emitBigIntUnaryArithResult<MBigIntIncrement>(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntDecResult(BigIntOperandId inputId) {
+ return emitBigIntUnaryArithResult<MBigIntDecrement>(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntNegationResult(BigIntOperandId inputId) {
+ return emitBigIntUnaryArithResult<MBigIntNegate>(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitBigIntNotResult(BigIntOperandId inputId) {
+ return emitBigIntUnaryArithResult<MBigIntBitNot>(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitCallStringConcatResult(StringOperandId lhsId,
+ StringOperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MConcat::New(alloc(), lhs, rhs);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCompareResult(
+ JSOp op, OperandId lhsId, OperandId rhsId,
+ MCompare::CompareType compareType) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MCompare::New(alloc(), lhs, rhs, op, compareType);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCompareInt32Result(JSOp op,
+ Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_Int32);
+}
+
+bool WarpCacheIRTranspiler::emitCompareDoubleResult(JSOp op,
+ NumberOperandId lhsId,
+ NumberOperandId rhsId) {
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_Double);
+}
+
+bool WarpCacheIRTranspiler::emitCompareObjectResult(JSOp op, ObjOperandId lhsId,
+ ObjOperandId rhsId) {
+ MOZ_ASSERT(IsEqualityOp(op));
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_Object);
+}
+
+bool WarpCacheIRTranspiler::emitCompareStringResult(JSOp op,
+ StringOperandId lhsId,
+ StringOperandId rhsId) {
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_String);
+}
+
+bool WarpCacheIRTranspiler::emitCompareSymbolResult(JSOp op,
+ SymbolOperandId lhsId,
+ SymbolOperandId rhsId) {
+ MOZ_ASSERT(IsEqualityOp(op));
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_Symbol);
+}
+
+bool WarpCacheIRTranspiler::emitCompareBigIntResult(JSOp op,
+ BigIntOperandId lhsId,
+ BigIntOperandId rhsId) {
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_BigInt);
+}
+
+bool WarpCacheIRTranspiler::emitCompareBigIntInt32Result(JSOp op,
+ BigIntOperandId lhsId,
+ Int32OperandId rhsId) {
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_BigInt_Int32);
+}
+
+bool WarpCacheIRTranspiler::emitCompareBigIntNumberResult(
+ JSOp op, BigIntOperandId lhsId, NumberOperandId rhsId) {
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_BigInt_Double);
+}
+
+bool WarpCacheIRTranspiler::emitCompareBigIntStringResult(
+ JSOp op, BigIntOperandId lhsId, StringOperandId rhsId) {
+ return emitCompareResult(op, lhsId, rhsId, MCompare::Compare_BigInt_String);
+}
+
+bool WarpCacheIRTranspiler::emitCompareNullUndefinedResult(
+ JSOp op, bool isUndefined, ValOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ MOZ_ASSERT(IsEqualityOp(op));
+
+ // A previously emitted guard ensures that one side of the comparison
+ // is null or undefined.
+ MDefinition* cst =
+ isUndefined ? constant(UndefinedValue()) : constant(NullValue());
+ auto compareType =
+ isUndefined ? MCompare::Compare_Undefined : MCompare::Compare_Null;
+ auto* ins = MCompare::New(alloc(), input, cst, op, compareType);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCompareDoubleSameValueResult(
+ NumberOperandId lhsId, NumberOperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* sameValue = MSameValueDouble::New(alloc(), lhs, rhs);
+ add(sameValue);
+
+ pushResult(sameValue);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitSameValueResult(ValOperandId lhsId,
+ ValOperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* sameValue = MSameValue::New(alloc(), lhs, rhs);
+ add(sameValue);
+
+ pushResult(sameValue);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitIndirectTruncateInt32Result(
+ Int32OperandId valId) {
+ MDefinition* val = getOperand(valId);
+ MOZ_ASSERT(val->type() == MIRType::Int32);
+
+ auto* truncate =
+ MLimitedTruncate::New(alloc(), val, TruncateKind::IndirectTruncate);
+ add(truncate);
+
+ pushResult(truncate);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathHypot2NumberResult(
+ NumberOperandId firstId, NumberOperandId secondId) {
+ MDefinitionVector vector(alloc());
+ if (!vector.reserve(2)) {
+ return false;
+ }
+
+ vector.infallibleAppend(getOperand(firstId));
+ vector.infallibleAppend(getOperand(secondId));
+
+ auto* ins = MHypot::New(alloc(), vector);
+ if (!ins) {
+ return false;
+ }
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathHypot3NumberResult(
+ NumberOperandId firstId, NumberOperandId secondId,
+ NumberOperandId thirdId) {
+ MDefinitionVector vector(alloc());
+ if (!vector.reserve(3)) {
+ return false;
+ }
+
+ vector.infallibleAppend(getOperand(firstId));
+ vector.infallibleAppend(getOperand(secondId));
+ vector.infallibleAppend(getOperand(thirdId));
+
+ auto* ins = MHypot::New(alloc(), vector);
+ if (!ins) {
+ return false;
+ }
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathHypot4NumberResult(
+ NumberOperandId firstId, NumberOperandId secondId, NumberOperandId thirdId,
+ NumberOperandId fourthId) {
+ MDefinitionVector vector(alloc());
+ if (!vector.reserve(4)) {
+ return false;
+ }
+
+ vector.infallibleAppend(getOperand(firstId));
+ vector.infallibleAppend(getOperand(secondId));
+ vector.infallibleAppend(getOperand(thirdId));
+ vector.infallibleAppend(getOperand(fourthId));
+
+ auto* ins = MHypot::New(alloc(), vector);
+ if (!ins) {
+ return false;
+ }
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathRandomResult(uint32_t rngOffset) {
+#ifdef DEBUG
+ // CodeGenerator uses CompileRealm::addressOfRandomNumberGenerator. Assert it
+ // matches the RNG pointer stored in the stub field.
+ const void* rng = rawPointerField(rngOffset);
+ MOZ_ASSERT(rng == mirGen().realm->addressOfRandomNumberGenerator());
+#endif
+
+ auto* ins = MRandom::New(alloc());
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitInt32MinMax(bool isMax, Int32OperandId firstId,
+ Int32OperandId secondId,
+ Int32OperandId resultId) {
+ MDefinition* first = getOperand(firstId);
+ MDefinition* second = getOperand(secondId);
+
+ auto* ins = MMinMax::New(alloc(), first, second, MIRType::Int32, isMax);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitNumberMinMax(bool isMax,
+ NumberOperandId firstId,
+ NumberOperandId secondId,
+ NumberOperandId resultId) {
+ MDefinition* first = getOperand(firstId);
+ MDefinition* second = getOperand(secondId);
+
+ auto* ins = MMinMax::New(alloc(), first, second, MIRType::Double, isMax);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+bool WarpCacheIRTranspiler::emitInt32MinMaxArrayResult(ObjOperandId arrayId,
+ bool isMax) {
+ MDefinition* array = getOperand(arrayId);
+
+ auto* ins = MMinMaxArray::New(alloc(), array, MIRType::Int32, isMax);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitNumberMinMaxArrayResult(ObjOperandId arrayId,
+ bool isMax) {
+ MDefinition* array = getOperand(arrayId);
+
+ auto* ins = MMinMaxArray::New(alloc(), array, MIRType::Double, isMax);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathAbsInt32Result(Int32OperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MAbs::New(alloc(), input, MIRType::Int32);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathAbsNumberResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MAbs::New(alloc(), input, MIRType::Double);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathClz32Result(Int32OperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MClz::New(alloc(), input, MIRType::Int32);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathSignInt32Result(Int32OperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MSign::New(alloc(), input, MIRType::Int32);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathSignNumberResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MSign::New(alloc(), input, MIRType::Double);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathSignNumberToInt32Result(
+ NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MSign::New(alloc(), input, MIRType::Int32);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathImulResult(Int32OperandId lhsId,
+ Int32OperandId rhsId) {
+ MDefinition* lhs = getOperand(lhsId);
+ MDefinition* rhs = getOperand(rhsId);
+
+ auto* ins = MMul::New(alloc(), lhs, rhs, MIRType::Int32, MMul::Integer);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathFloorToInt32Result(
+ NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MFloor::New(alloc(), input);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathCeilToInt32Result(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MCeil::New(alloc(), input);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathTruncToInt32Result(
+ NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MTrunc::New(alloc(), input);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathRoundToInt32Result(
+ NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MRound::New(alloc(), input);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathSqrtNumberResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MSqrt::New(alloc(), input, MIRType::Double);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathFRoundNumberResult(
+ NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MToFloat32::New(alloc(), input);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathAtan2NumberResult(NumberOperandId yId,
+ NumberOperandId xId) {
+ MDefinition* y = getOperand(yId);
+ MDefinition* x = getOperand(xId);
+
+ auto* ins = MAtan2::New(alloc(), y, x);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathFunctionNumberResult(
+ NumberOperandId inputId, UnaryMathFunction fun) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* ins = MMathFunction::New(alloc(), input, fun);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathFloorNumberResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ MInstruction* ins;
+ if (MNearbyInt::HasAssemblerSupport(RoundingMode::Down)) {
+ ins = MNearbyInt::New(alloc(), input, MIRType::Double, RoundingMode::Down);
+ } else {
+ ins = MMathFunction::New(alloc(), input, UnaryMathFunction::Floor);
+ }
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathCeilNumberResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ MInstruction* ins;
+ if (MNearbyInt::HasAssemblerSupport(RoundingMode::Up)) {
+ ins = MNearbyInt::New(alloc(), input, MIRType::Double, RoundingMode::Up);
+ } else {
+ ins = MMathFunction::New(alloc(), input, UnaryMathFunction::Ceil);
+ }
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMathTruncNumberResult(NumberOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ MInstruction* ins;
+ if (MNearbyInt::HasAssemblerSupport(RoundingMode::TowardsZero)) {
+ ins = MNearbyInt::New(alloc(), input, MIRType::Double,
+ RoundingMode::TowardsZero);
+ } else {
+ ins = MMathFunction::New(alloc(), input, UnaryMathFunction::Trunc);
+ }
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitNumberParseIntResult(StringOperandId strId,
+ Int32OperandId radixId) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* radix = getOperand(radixId);
+
+ auto* ins = MNumberParseInt::New(alloc(), str, radix);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitDoubleParseIntResult(NumberOperandId numId) {
+ MDefinition* num = getOperand(numId);
+
+ auto* ins = MDoubleParseInt::New(alloc(), num);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitObjectToStringResult(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MObjectClassToString::New(alloc(), obj);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitReflectGetPrototypeOfResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGetPrototypeOf::New(alloc(), obj);
+ addEffectful(ins);
+ pushResult(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitArrayPush(ObjOperandId objId,
+ ValOperandId rhsId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* value = getOperand(rhsId);
+
+ auto* elements = MElements::New(alloc(), obj);
+ add(elements);
+
+ auto* initLength = MInitializedLength::New(alloc(), elements);
+ add(initLength);
+
+ auto* barrier =
+ MPostWriteElementBarrier::New(alloc(), obj, value, initLength);
+ add(barrier);
+
+ auto* ins = MArrayPush::New(alloc(), obj, value);
+ addEffectful(ins);
+ pushResult(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitArrayJoinResult(ObjOperandId objId,
+ StringOperandId sepId) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* sep = getOperand(sepId);
+
+ auto* join = MArrayJoin::New(alloc(), obj, sep);
+ addEffectful(join);
+
+ pushResult(join);
+ return resumeAfter(join);
+}
+
+bool WarpCacheIRTranspiler::emitPackedArrayPopResult(ObjOperandId arrayId) {
+ MDefinition* array = getOperand(arrayId);
+
+ auto* ins = MArrayPopShift::New(alloc(), array, MArrayPopShift::Pop);
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitPackedArrayShiftResult(ObjOperandId arrayId) {
+ MDefinition* array = getOperand(arrayId);
+
+ auto* ins = MArrayPopShift::New(alloc(), array, MArrayPopShift::Shift);
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitPackedArraySliceResult(
+ uint32_t templateObjectOffset, ObjOperandId arrayId, Int32OperandId beginId,
+ Int32OperandId endId) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+
+ MDefinition* array = getOperand(arrayId);
+ MDefinition* begin = getOperand(beginId);
+ MDefinition* end = getOperand(endId);
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ auto* ins = MArraySlice::New(alloc(), array, begin, end, templateObj, heap);
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitArgumentsSliceResult(
+ uint32_t templateObjectOffset, ObjOperandId argsId, Int32OperandId beginId,
+ Int32OperandId endId) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+
+ MDefinition* args = getOperand(argsId);
+ MDefinition* begin = getOperand(beginId);
+ MDefinition* end = getOperand(endId);
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ auto* ins =
+ MArgumentsSlice::New(alloc(), args, begin, end, templateObj, heap);
+ addEffectful(ins);
+
+ pushResult(ins);
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitHasClassResult(ObjOperandId objId,
+ uint32_t claspOffset) {
+ MDefinition* obj = getOperand(objId);
+ const JSClass* clasp = classStubField(claspOffset);
+
+ auto* hasClass = MHasClass::New(alloc(), obj, clasp);
+ add(hasClass);
+
+ pushResult(hasClass);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCallRegExpMatcherResult(
+ ObjOperandId regexpId, StringOperandId inputId,
+ Int32OperandId lastIndexId) {
+ MDefinition* regexp = getOperand(regexpId);
+ MDefinition* input = getOperand(inputId);
+ MDefinition* lastIndex = getOperand(lastIndexId);
+
+ auto* matcher = MRegExpMatcher::New(alloc(), regexp, input, lastIndex);
+ addEffectful(matcher);
+ pushResult(matcher);
+
+ return resumeAfter(matcher);
+}
+
+bool WarpCacheIRTranspiler::emitCallRegExpSearcherResult(
+ ObjOperandId regexpId, StringOperandId inputId,
+ Int32OperandId lastIndexId) {
+ MDefinition* regexp = getOperand(regexpId);
+ MDefinition* input = getOperand(inputId);
+ MDefinition* lastIndex = getOperand(lastIndexId);
+
+ auto* searcher = MRegExpSearcher::New(alloc(), regexp, input, lastIndex);
+ addEffectful(searcher);
+ pushResult(searcher);
+
+ return resumeAfter(searcher);
+}
+
+bool WarpCacheIRTranspiler::emitCallRegExpTesterResult(
+ ObjOperandId regexpId, StringOperandId inputId,
+ Int32OperandId lastIndexId) {
+ MDefinition* regexp = getOperand(regexpId);
+ MDefinition* input = getOperand(inputId);
+ MDefinition* lastIndex = getOperand(lastIndexId);
+
+ auto* tester = MRegExpTester::New(alloc(), regexp, input, lastIndex);
+ addEffectful(tester);
+ pushResult(tester);
+
+ return resumeAfter(tester);
+}
+
+MInstruction* WarpCacheIRTranspiler::convertToBoolean(MDefinition* input) {
+ // Convert to bool with the '!!' idiom.
+ //
+ // The FoldTests and GVN passes both specifically handle this pattern. If you
+ // change this code, make sure to update FoldTests and GVN, too.
+
+ auto* resultInverted = MNot::New(alloc(), input);
+ add(resultInverted);
+ auto* result = MNot::New(alloc(), resultInverted);
+ add(result);
+
+ return result;
+}
+
+bool WarpCacheIRTranspiler::emitRegExpFlagResult(ObjOperandId regexpId,
+ int32_t flagsMask) {
+ MDefinition* regexp = getOperand(regexpId);
+
+ auto* flags = MLoadFixedSlot::New(alloc(), regexp, RegExpObject::flagsSlot());
+ flags->setResultType(MIRType::Int32);
+ add(flags);
+
+ auto* mask = MConstant::New(alloc(), Int32Value(flagsMask));
+ add(mask);
+
+ auto* maskedFlag = MBitAnd::New(alloc(), flags, mask, MIRType::Int32);
+ add(maskedFlag);
+
+ auto* result = convertToBoolean(maskedFlag);
+
+ pushResult(result);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCallSubstringKernelResult(
+ StringOperandId strId, Int32OperandId beginId, Int32OperandId lengthId) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* begin = getOperand(beginId);
+ MDefinition* length = getOperand(lengthId);
+
+ auto* substr = MSubstr::New(alloc(), str, begin, length);
+ add(substr);
+
+ pushResult(substr);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStringReplaceStringResult(
+ StringOperandId strId, StringOperandId patternId,
+ StringOperandId replacementId) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* pattern = getOperand(patternId);
+ MDefinition* replacement = getOperand(replacementId);
+
+ auto* replace = MStringReplace::New(alloc(), str, pattern, replacement);
+ add(replace);
+
+ pushResult(replace);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitStringSplitStringResult(
+ StringOperandId strId, StringOperandId separatorId) {
+ MDefinition* str = getOperand(strId);
+ MDefinition* separator = getOperand(separatorId);
+
+ auto* split = MStringSplit::New(alloc(), str, separator);
+ add(split);
+
+ pushResult(split);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitRegExpPrototypeOptimizableResult(
+ ObjOperandId protoId) {
+ MDefinition* proto = getOperand(protoId);
+
+ auto* optimizable = MRegExpPrototypeOptimizable::New(alloc(), proto);
+ add(optimizable);
+
+ pushResult(optimizable);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitRegExpInstanceOptimizableResult(
+ ObjOperandId regexpId, ObjOperandId protoId) {
+ MDefinition* regexp = getOperand(regexpId);
+ MDefinition* proto = getOperand(protoId);
+
+ auto* optimizable = MRegExpInstanceOptimizable::New(alloc(), regexp, proto);
+ add(optimizable);
+
+ pushResult(optimizable);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGetFirstDollarIndexResult(
+ StringOperandId strId) {
+ MDefinition* str = getOperand(strId);
+
+ auto* firstDollarIndex = MGetFirstDollarIndex::New(alloc(), str);
+ add(firstDollarIndex);
+
+ pushResult(firstDollarIndex);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitIsArrayResult(ValOperandId inputId) {
+ MDefinition* value = getOperand(inputId);
+
+ auto* isArray = MIsArray::New(alloc(), value);
+ addEffectful(isArray);
+ pushResult(isArray);
+
+ return resumeAfter(isArray);
+}
+
+bool WarpCacheIRTranspiler::emitIsObjectResult(ValOperandId inputId) {
+ MDefinition* value = getOperand(inputId);
+
+ if (value->type() == MIRType::Object) {
+ pushResult(constant(BooleanValue(true)));
+ } else {
+ auto* isObject = MIsObject::New(alloc(), value);
+ add(isObject);
+ pushResult(isObject);
+ }
+
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitIsPackedArrayResult(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* isPackedArray = MIsPackedArray::New(alloc(), obj);
+ add(isPackedArray);
+
+ pushResult(isPackedArray);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitIsCallableResult(ValOperandId inputId) {
+ MDefinition* value = getOperand(inputId);
+
+ auto* isCallable = MIsCallable::New(alloc(), value);
+ add(isCallable);
+
+ pushResult(isCallable);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitIsConstructorResult(ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* isConstructor = MIsConstructor::New(alloc(), obj);
+ add(isConstructor);
+
+ pushResult(isConstructor);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitIsCrossRealmArrayConstructorResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MIsCrossRealmArrayConstructor::New(alloc(), obj);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitIsTypedArrayResult(ObjOperandId objId,
+ bool isPossiblyWrapped) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MIsTypedArray::New(alloc(), obj, isPossiblyWrapped);
+ if (isPossiblyWrapped) {
+ addEffectful(ins);
+ } else {
+ add(ins);
+ }
+
+ pushResult(ins);
+
+ if (isPossiblyWrapped) {
+ if (!resumeAfter(ins)) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitArrayBufferViewByteOffsetInt32Result(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* byteOffset = MArrayBufferViewByteOffset::New(alloc(), obj);
+ add(byteOffset);
+
+ auto* byteOffsetInt32 = MNonNegativeIntPtrToInt32::New(alloc(), byteOffset);
+ add(byteOffsetInt32);
+
+ pushResult(byteOffsetInt32);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitArrayBufferViewByteOffsetDoubleResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* byteOffset = MArrayBufferViewByteOffset::New(alloc(), obj);
+ add(byteOffset);
+
+ auto* byteOffsetDouble = MIntPtrToDouble::New(alloc(), byteOffset);
+ add(byteOffsetDouble);
+
+ pushResult(byteOffsetDouble);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitTypedArrayByteLengthInt32Result(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ auto* lengthInt32 = MNonNegativeIntPtrToInt32::New(alloc(), length);
+ add(lengthInt32);
+
+ auto* size = MTypedArrayElementSize::New(alloc(), obj);
+ add(size);
+
+ auto* mul = MMul::New(alloc(), lengthInt32, size, MIRType::Int32);
+ mul->setCanBeNegativeZero(false);
+ add(mul);
+
+ pushResult(mul);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitTypedArrayByteLengthDoubleResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ auto* lengthDouble = MIntPtrToDouble::New(alloc(), length);
+ add(lengthDouble);
+
+ auto* size = MTypedArrayElementSize::New(alloc(), obj);
+ add(size);
+
+ auto* mul = MMul::New(alloc(), lengthDouble, size, MIRType::Double);
+ mul->setCanBeNegativeZero(false);
+ add(mul);
+
+ pushResult(mul);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitTypedArrayElementSizeResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MTypedArrayElementSize::New(alloc(), obj);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardHasAttachedArrayBuffer(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MGuardHasAttachedArrayBuffer::New(alloc(), obj);
+ add(ins);
+
+ setOperand(objId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitIsTypedArrayConstructorResult(
+ ObjOperandId objId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MIsTypedArrayConstructor::New(alloc(), obj);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGetNextMapSetEntryForIteratorResult(
+ ObjOperandId iterId, ObjOperandId resultArrId, bool isMap) {
+ MDefinition* iter = getOperand(iterId);
+ MDefinition* resultArr = getOperand(resultArrId);
+
+ MGetNextEntryForIterator::Mode mode =
+ isMap ? MGetNextEntryForIterator::Map : MGetNextEntryForIterator::Set;
+ auto* ins = MGetNextEntryForIterator::New(alloc(), iter, resultArr, mode);
+ addEffectful(ins);
+ pushResult(ins);
+
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitFrameIsConstructingResult() {
+ if (const CallInfo* callInfo = builder_->inlineCallInfo()) {
+ auto* ins = constant(BooleanValue(callInfo->constructing()));
+ pushResult(ins);
+ return true;
+ }
+
+ auto* ins = MIsConstructing::New(alloc());
+ add(ins);
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitFinishBoundFunctionInitResult(
+ ObjOperandId boundId, ObjOperandId targetId, Int32OperandId argCountId) {
+ MDefinition* bound = getOperand(boundId);
+ MDefinition* target = getOperand(targetId);
+ MDefinition* argCount = getOperand(argCountId);
+
+ auto* ins = MFinishBoundFunctionInit::New(alloc(), bound, target, argCount);
+ addEffectful(ins);
+
+ pushResult(constant(UndefinedValue()));
+ return resumeAfter(ins);
+}
+
+bool WarpCacheIRTranspiler::emitNewIteratorResult(
+ MNewIterator::Type type, uint32_t templateObjectOffset) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+
+ auto* templateConst = constant(ObjectValue(*templateObj));
+ auto* iter = MNewIterator::New(alloc(), templateConst, type);
+ add(iter);
+
+ pushResult(iter);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitNewArrayIteratorResult(
+ uint32_t templateObjectOffset) {
+ return emitNewIteratorResult(MNewIterator::ArrayIterator,
+ templateObjectOffset);
+}
+
+bool WarpCacheIRTranspiler::emitNewStringIteratorResult(
+ uint32_t templateObjectOffset) {
+ return emitNewIteratorResult(MNewIterator::StringIterator,
+ templateObjectOffset);
+}
+
+bool WarpCacheIRTranspiler::emitNewRegExpStringIteratorResult(
+ uint32_t templateObjectOffset) {
+ return emitNewIteratorResult(MNewIterator::RegExpStringIterator,
+ templateObjectOffset);
+}
+
+bool WarpCacheIRTranspiler::emitObjectCreateResult(
+ uint32_t templateObjectOffset) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+
+ auto* templateConst = constant(ObjectValue(*templateObj));
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+ auto* obj =
+ MNewObject::New(alloc(), templateConst, heap, MNewObject::ObjectCreate);
+ addEffectful(obj);
+
+ pushResult(obj);
+ return resumeAfter(obj);
+}
+
+bool WarpCacheIRTranspiler::emitNewArrayFromLengthResult(
+ uint32_t templateObjectOffset, Int32OperandId lengthId) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+ MDefinition* length = getOperand(lengthId);
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ if (length->isConstant()) {
+ int32_t lenInt32 = length->toConstant()->toInt32();
+ if (lenInt32 >= 0 &&
+ uint32_t(lenInt32) == templateObj->as<ArrayObject>().length()) {
+ uint32_t len = uint32_t(lenInt32);
+ auto* templateConst = constant(ObjectValue(*templateObj));
+
+ size_t inlineLength =
+ gc::GetGCKindSlots(templateObj->asTenured().getAllocKind()) -
+ ObjectElements::VALUES_PER_HEADER;
+
+ MNewArray* obj;
+ if (len > inlineLength) {
+ obj = MNewArray::NewVM(alloc(), len, templateConst, heap);
+ } else {
+ obj = MNewArray::New(alloc(), len, templateConst, heap);
+ }
+ add(obj);
+ pushResult(obj);
+ return true;
+ }
+ }
+
+ auto* obj = MNewArrayDynamicLength::New(alloc(), length, templateObj, heap);
+ addEffectful(obj);
+ pushResult(obj);
+ return resumeAfter(obj);
+}
+
+bool WarpCacheIRTranspiler::emitNewTypedArrayFromLengthResult(
+ uint32_t templateObjectOffset, Int32OperandId lengthId) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+ MDefinition* length = getOperand(lengthId);
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ if (length->isConstant()) {
+ int32_t len = length->toConstant()->toInt32();
+ if (len > 0 &&
+ uint32_t(len) == templateObj->as<TypedArrayObject>().length()) {
+ auto* templateConst = constant(ObjectValue(*templateObj));
+ auto* obj = MNewTypedArray::New(alloc(), templateConst, heap);
+ add(obj);
+ pushResult(obj);
+ return true;
+ }
+ }
+
+ auto* obj =
+ MNewTypedArrayDynamicLength::New(alloc(), length, templateObj, heap);
+ addEffectful(obj);
+ pushResult(obj);
+ return resumeAfter(obj);
+}
+
+bool WarpCacheIRTranspiler::emitNewTypedArrayFromArrayBufferResult(
+ uint32_t templateObjectOffset, ObjOperandId bufferId,
+ ValOperandId byteOffsetId, ValOperandId lengthId) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+ MDefinition* buffer = getOperand(bufferId);
+ MDefinition* byteOffset = getOperand(byteOffsetId);
+ MDefinition* length = getOperand(lengthId);
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ auto* obj = MNewTypedArrayFromArrayBuffer::New(alloc(), buffer, byteOffset,
+ length, templateObj, heap);
+ addEffectful(obj);
+
+ pushResult(obj);
+ return resumeAfter(obj);
+}
+
+bool WarpCacheIRTranspiler::emitNewTypedArrayFromArrayResult(
+ uint32_t templateObjectOffset, ObjOperandId arrayId) {
+ JSObject* templateObj = tenuredObjectStubField(templateObjectOffset);
+ MDefinition* array = getOperand(arrayId);
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ auto* obj = MNewTypedArrayFromArray::New(alloc(), array, templateObj, heap);
+ addEffectful(obj);
+
+ pushResult(obj);
+ return resumeAfter(obj);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsCompareExchangeResult(
+ ObjOperandId objId, IntPtrOperandId indexId, uint32_t expectedId,
+ uint32_t replacementId, Scalar::Type elementType) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+ MDefinition* expected = getOperand(ValOperandId(expectedId));
+ MDefinition* replacement = getOperand(ValOperandId(replacementId));
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* elements = MArrayBufferViewElements::New(alloc(), obj);
+ add(elements);
+
+ bool forceDoubleForUint32 = true;
+ MIRType knownType =
+ MIRTypeForArrayBufferViewRead(elementType, forceDoubleForUint32);
+
+ auto* cas = MCompareExchangeTypedArrayElement::New(
+ alloc(), elements, index, elementType, expected, replacement);
+ cas->setResultType(knownType);
+ addEffectful(cas);
+
+ pushResult(cas);
+ return resumeAfter(cas);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsExchangeResult(
+ ObjOperandId objId, IntPtrOperandId indexId, uint32_t valueId,
+ Scalar::Type elementType) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+ MDefinition* value = getOperand(ValOperandId(valueId));
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* elements = MArrayBufferViewElements::New(alloc(), obj);
+ add(elements);
+
+ bool forceDoubleForUint32 = true;
+ MIRType knownType =
+ MIRTypeForArrayBufferViewRead(elementType, forceDoubleForUint32);
+
+ auto* exchange = MAtomicExchangeTypedArrayElement::New(
+ alloc(), elements, index, value, elementType);
+ exchange->setResultType(knownType);
+ addEffectful(exchange);
+
+ pushResult(exchange);
+ return resumeAfter(exchange);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsBinaryOp(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect, AtomicOp op) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+ MDefinition* value = getOperand(ValOperandId(valueId));
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* elements = MArrayBufferViewElements::New(alloc(), obj);
+ add(elements);
+
+ bool forceDoubleForUint32 = true;
+ MIRType knownType =
+ MIRTypeForArrayBufferViewRead(elementType, forceDoubleForUint32);
+
+ auto* binop = MAtomicTypedArrayElementBinop::New(
+ alloc(), op, elements, index, elementType, value, forEffect);
+ if (!forEffect) {
+ binop->setResultType(knownType);
+ }
+ addEffectful(binop);
+
+ if (!forEffect) {
+ pushResult(binop);
+ } else {
+ pushResult(constant(UndefinedValue()));
+ }
+ return resumeAfter(binop);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsAddResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ return emitAtomicsBinaryOp(objId, indexId, valueId, elementType, forEffect,
+ AtomicFetchAddOp);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsSubResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ return emitAtomicsBinaryOp(objId, indexId, valueId, elementType, forEffect,
+ AtomicFetchSubOp);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsAndResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ return emitAtomicsBinaryOp(objId, indexId, valueId, elementType, forEffect,
+ AtomicFetchAndOp);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsOrResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ return emitAtomicsBinaryOp(objId, indexId, valueId, elementType, forEffect,
+ AtomicFetchOrOp);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsXorResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType,
+ bool forEffect) {
+ return emitAtomicsBinaryOp(objId, indexId, valueId, elementType, forEffect,
+ AtomicFetchXorOp);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsLoadResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ Scalar::Type elementType) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* elements = MArrayBufferViewElements::New(alloc(), obj);
+ add(elements);
+
+ bool forceDoubleForUint32 = true;
+ MIRType knownType =
+ MIRTypeForArrayBufferViewRead(elementType, forceDoubleForUint32);
+
+ auto* load = MLoadUnboxedScalar::New(alloc(), elements, index, elementType,
+ DoesRequireMemoryBarrier);
+ load->setResultType(knownType);
+ addEffectful(load);
+
+ pushResult(load);
+ return resumeAfter(load);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsStoreResult(ObjOperandId objId,
+ IntPtrOperandId indexId,
+ uint32_t valueId,
+ Scalar::Type elementType) {
+ MDefinition* obj = getOperand(objId);
+ MDefinition* index = getOperand(indexId);
+ MDefinition* value = getOperand(ValOperandId(valueId));
+
+ auto* length = MArrayBufferViewLength::New(alloc(), obj);
+ add(length);
+
+ index = addBoundsCheck(index, length);
+
+ auto* elements = MArrayBufferViewElements::New(alloc(), obj);
+ add(elements);
+
+ auto* store = MStoreUnboxedScalar::New(alloc(), elements, index, value,
+ elementType, DoesRequireMemoryBarrier);
+ addEffectful(store);
+
+ pushResult(value);
+ return resumeAfter(store);
+}
+
+bool WarpCacheIRTranspiler::emitAtomicsIsLockFreeResult(
+ Int32OperandId valueId) {
+ MDefinition* value = getOperand(valueId);
+
+ auto* ilf = MAtomicIsLockFree::New(alloc(), value);
+ add(ilf);
+
+ pushResult(ilf);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitBigIntAsIntNResult(Int32OperandId bitsId,
+ BigIntOperandId bigIntId) {
+ MDefinition* bits = getOperand(bitsId);
+ MDefinition* bigInt = getOperand(bigIntId);
+
+ auto* ins = MBigIntAsIntN::New(alloc(), bits, bigInt);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitBigIntAsUintNResult(Int32OperandId bitsId,
+ BigIntOperandId bigIntId) {
+ MDefinition* bits = getOperand(bitsId);
+ MDefinition* bigInt = getOperand(bigIntId);
+
+ auto* ins = MBigIntAsUintN::New(alloc(), bits, bigInt);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardToNonGCThing(ValOperandId inputId) {
+ MDefinition* def = getOperand(inputId);
+ if (IsNonGCThing(def->type())) {
+ return true;
+ }
+
+ auto* ins = MGuardNonGCThing::New(alloc(), def);
+ add(ins);
+
+ setOperand(inputId, ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitSetHasNonGCThingResult(ObjOperandId setId,
+ ValOperandId valId) {
+ MDefinition* set = getOperand(setId);
+ MDefinition* val = getOperand(valId);
+
+ auto* hashValue = MToHashableNonGCThing::New(alloc(), val);
+ add(hashValue);
+
+ auto* hash = MHashNonGCThing::New(alloc(), hashValue);
+ add(hash);
+
+ auto* ins = MSetObjectHasNonBigInt::New(alloc(), set, hashValue, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitSetHasStringResult(ObjOperandId setId,
+ StringOperandId strId) {
+ MDefinition* set = getOperand(setId);
+ MDefinition* str = getOperand(strId);
+
+ auto* hashValue = MToHashableString::New(alloc(), str);
+ add(hashValue);
+
+ auto* hash = MHashString::New(alloc(), hashValue);
+ add(hash);
+
+ auto* ins = MSetObjectHasNonBigInt::New(alloc(), set, hashValue, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitSetHasSymbolResult(ObjOperandId setId,
+ SymbolOperandId symId) {
+ MDefinition* set = getOperand(setId);
+ MDefinition* sym = getOperand(symId);
+
+ auto* hash = MHashSymbol::New(alloc(), sym);
+ add(hash);
+
+ auto* ins = MSetObjectHasNonBigInt::New(alloc(), set, sym, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitSetHasBigIntResult(ObjOperandId setId,
+ BigIntOperandId bigIntId) {
+ MDefinition* set = getOperand(setId);
+ MDefinition* bigInt = getOperand(bigIntId);
+
+ auto* hash = MHashBigInt::New(alloc(), bigInt);
+ add(hash);
+
+ auto* ins = MSetObjectHasBigInt::New(alloc(), set, bigInt, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitSetHasObjectResult(ObjOperandId setId,
+ ObjOperandId objId) {
+ MDefinition* set = getOperand(setId);
+ MDefinition* obj = getOperand(objId);
+
+ auto* hash = MHashObject::New(alloc(), set, obj);
+ add(hash);
+
+ auto* ins = MSetObjectHasNonBigInt::New(alloc(), set, obj, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitSetHasResult(ObjOperandId setId,
+ ValOperandId valId) {
+ MDefinition* set = getOperand(setId);
+ MDefinition* val = getOperand(valId);
+
+#ifdef JS_PUNBOX64
+ auto* hashValue = MToHashableValue::New(alloc(), val);
+ add(hashValue);
+
+ auto* hash = MHashValue::New(alloc(), set, hashValue);
+ add(hash);
+
+ auto* ins = MSetObjectHasValue::New(alloc(), set, hashValue, hash);
+ add(ins);
+#else
+ auto* ins = MSetObjectHasValueVMCall::New(alloc(), set, val);
+ add(ins);
+#endif
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitSetSizeResult(ObjOperandId setId) {
+ MDefinition* set = getOperand(setId);
+
+ auto* ins = MSetObjectSize::New(alloc(), set);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapHasNonGCThingResult(ObjOperandId mapId,
+ ValOperandId valId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* val = getOperand(valId);
+
+ auto* hashValue = MToHashableNonGCThing::New(alloc(), val);
+ add(hashValue);
+
+ auto* hash = MHashNonGCThing::New(alloc(), hashValue);
+ add(hash);
+
+ auto* ins = MMapObjectHasNonBigInt::New(alloc(), map, hashValue, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapHasStringResult(ObjOperandId mapId,
+ StringOperandId strId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* str = getOperand(strId);
+
+ auto* hashValue = MToHashableString::New(alloc(), str);
+ add(hashValue);
+
+ auto* hash = MHashString::New(alloc(), hashValue);
+ add(hash);
+
+ auto* ins = MMapObjectHasNonBigInt::New(alloc(), map, hashValue, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapHasSymbolResult(ObjOperandId mapId,
+ SymbolOperandId symId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* sym = getOperand(symId);
+
+ auto* hash = MHashSymbol::New(alloc(), sym);
+ add(hash);
+
+ auto* ins = MMapObjectHasNonBigInt::New(alloc(), map, sym, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapHasBigIntResult(ObjOperandId mapId,
+ BigIntOperandId bigIntId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* bigInt = getOperand(bigIntId);
+
+ auto* hash = MHashBigInt::New(alloc(), bigInt);
+ add(hash);
+
+ auto* ins = MMapObjectHasBigInt::New(alloc(), map, bigInt, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapHasObjectResult(ObjOperandId mapId,
+ ObjOperandId objId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* obj = getOperand(objId);
+
+ auto* hash = MHashObject::New(alloc(), map, obj);
+ add(hash);
+
+ auto* ins = MMapObjectHasNonBigInt::New(alloc(), map, obj, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapHasResult(ObjOperandId mapId,
+ ValOperandId valId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* val = getOperand(valId);
+
+#ifdef JS_PUNBOX64
+ auto* hashValue = MToHashableValue::New(alloc(), val);
+ add(hashValue);
+
+ auto* hash = MHashValue::New(alloc(), map, hashValue);
+ add(hash);
+
+ auto* ins = MMapObjectHasValue::New(alloc(), map, hashValue, hash);
+ add(ins);
+#else
+ auto* ins = MMapObjectHasValueVMCall::New(alloc(), map, val);
+ add(ins);
+#endif
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapGetNonGCThingResult(ObjOperandId mapId,
+ ValOperandId valId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* val = getOperand(valId);
+
+ auto* hashValue = MToHashableNonGCThing::New(alloc(), val);
+ add(hashValue);
+
+ auto* hash = MHashNonGCThing::New(alloc(), hashValue);
+ add(hash);
+
+ auto* ins = MMapObjectGetNonBigInt::New(alloc(), map, hashValue, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapGetStringResult(ObjOperandId mapId,
+ StringOperandId strId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* str = getOperand(strId);
+
+ auto* hashValue = MToHashableString::New(alloc(), str);
+ add(hashValue);
+
+ auto* hash = MHashString::New(alloc(), hashValue);
+ add(hash);
+
+ auto* ins = MMapObjectGetNonBigInt::New(alloc(), map, hashValue, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapGetSymbolResult(ObjOperandId mapId,
+ SymbolOperandId symId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* sym = getOperand(symId);
+
+ auto* hash = MHashSymbol::New(alloc(), sym);
+ add(hash);
+
+ auto* ins = MMapObjectGetNonBigInt::New(alloc(), map, sym, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapGetBigIntResult(ObjOperandId mapId,
+ BigIntOperandId bigIntId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* bigInt = getOperand(bigIntId);
+
+ auto* hash = MHashBigInt::New(alloc(), bigInt);
+ add(hash);
+
+ auto* ins = MMapObjectGetBigInt::New(alloc(), map, bigInt, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapGetObjectResult(ObjOperandId mapId,
+ ObjOperandId objId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* obj = getOperand(objId);
+
+ auto* hash = MHashObject::New(alloc(), map, obj);
+ add(hash);
+
+ auto* ins = MMapObjectGetNonBigInt::New(alloc(), map, obj, hash);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapGetResult(ObjOperandId mapId,
+ ValOperandId valId) {
+ MDefinition* map = getOperand(mapId);
+ MDefinition* val = getOperand(valId);
+
+#ifdef JS_PUNBOX64
+ auto* hashValue = MToHashableValue::New(alloc(), val);
+ add(hashValue);
+
+ auto* hash = MHashValue::New(alloc(), map, hashValue);
+ add(hash);
+
+ auto* ins = MMapObjectGetValue::New(alloc(), map, hashValue, hash);
+ add(ins);
+#else
+ auto* ins = MMapObjectGetValueVMCall::New(alloc(), map, val);
+ add(ins);
+#endif
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitMapSizeResult(ObjOperandId mapId) {
+ MDefinition* map = getOperand(mapId);
+
+ auto* ins = MMapObjectSize::New(alloc(), map);
+ add(ins);
+
+ pushResult(ins);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitTruthyResult(OperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+
+ auto* result = convertToBoolean(input);
+
+ pushResult(result);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadInt32TruthyResult(ValOperandId inputId) {
+ return emitTruthyResult(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitLoadDoubleTruthyResult(
+ NumberOperandId inputId) {
+ return emitTruthyResult(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitLoadStringTruthyResult(
+ StringOperandId inputId) {
+ return emitTruthyResult(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitLoadObjectTruthyResult(ObjOperandId inputId) {
+ return emitTruthyResult(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitLoadBigIntTruthyResult(
+ BigIntOperandId inputId) {
+ return emitTruthyResult(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitLoadValueTruthyResult(ValOperandId inputId) {
+ return emitTruthyResult(inputId);
+}
+
+bool WarpCacheIRTranspiler::emitLoadOperandResult(ValOperandId inputId) {
+ MDefinition* input = getOperand(inputId);
+ pushResult(input);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitLoadWrapperTarget(ObjOperandId objId,
+ ObjOperandId resultId) {
+ MDefinition* obj = getOperand(objId);
+
+ auto* ins = MLoadWrapperTarget::New(alloc(), obj);
+ add(ins);
+
+ return defineOperand(resultId, ins);
+}
+
+// When we transpile a call, we may generate guards for some
+// arguments. To make sure the call instruction depends on those
+// guards, when the transpiler creates an operand for an argument, we
+// register the OperandId of that argument in argumentIds_. (See
+// emitLoadArgumentSlot.) Before generating the call, we update the
+// CallInfo to use the appropriate value from operands_.
+// Note: The callee is an explicit argument to the call op, and is
+// tracked separately.
+void WarpCacheIRTranspiler::updateArgumentsFromOperands() {
+ for (uint32_t i = 0; i < uint32_t(ArgumentKind::NumKinds); i++) {
+ ArgumentKind kind = ArgumentKind(i);
+ OperandId id = argumentOperandIds_[kind];
+ if (id.valid()) {
+ switch (kind) {
+ case ArgumentKind::This:
+ callInfo_->setThis(getOperand(id));
+ break;
+ case ArgumentKind::NewTarget:
+ callInfo_->setNewTarget(getOperand(id));
+ break;
+ case ArgumentKind::Arg0:
+ callInfo_->setArg(0, getOperand(id));
+ break;
+ case ArgumentKind::Arg1:
+ callInfo_->setArg(1, getOperand(id));
+ break;
+ case ArgumentKind::Arg2:
+ callInfo_->setArg(2, getOperand(id));
+ break;
+ case ArgumentKind::Arg3:
+ callInfo_->setArg(3, getOperand(id));
+ break;
+ case ArgumentKind::Arg4:
+ callInfo_->setArg(4, getOperand(id));
+ break;
+ case ArgumentKind::Arg5:
+ callInfo_->setArg(5, getOperand(id));
+ break;
+ case ArgumentKind::Arg6:
+ callInfo_->setArg(6, getOperand(id));
+ break;
+ case ArgumentKind::Arg7:
+ callInfo_->setArg(7, getOperand(id));
+ break;
+ case ArgumentKind::Callee:
+ case ArgumentKind::NumKinds:
+ MOZ_CRASH("Unexpected argument kind");
+ }
+ }
+ }
+}
+
+bool WarpCacheIRTranspiler::emitLoadArgumentSlot(ValOperandId resultId,
+ uint32_t slotIndex) {
+ // Reverse of GetIndexOfArgument.
+
+ // Layout:
+ // NewTarget | Args.. (reversed) | ThisValue | Callee
+ // 0 | ArgC .. Arg1 Arg0 (+1) | argc (+1) | argc + 1 (+ 1)
+ // ^ (if constructing)
+
+ // NewTarget (optional)
+ if (callInfo_->constructing()) {
+ if (slotIndex == 0) {
+ setArgumentId(ArgumentKind::NewTarget, resultId);
+ return defineOperand(resultId, callInfo_->getNewTarget());
+ }
+
+ slotIndex -= 1; // Adjust slot index to match non-constructing calls.
+ }
+
+ // Args..
+ if (slotIndex < callInfo_->argc()) {
+ uint32_t arg = callInfo_->argc() - 1 - slotIndex;
+ ArgumentKind kind = ArgumentKindForArgIndex(arg);
+ MOZ_ASSERT(kind < ArgumentKind::NumKinds);
+ setArgumentId(kind, resultId);
+ return defineOperand(resultId, callInfo_->getArg(arg));
+ }
+
+ // ThisValue
+ if (slotIndex == callInfo_->argc()) {
+ setArgumentId(ArgumentKind::This, resultId);
+ return defineOperand(resultId, callInfo_->thisArg());
+ }
+
+ // Callee
+ MOZ_ASSERT(slotIndex == callInfo_->argc() + 1);
+ return defineOperand(resultId, callInfo_->callee());
+}
+
+bool WarpCacheIRTranspiler::emitLoadArgumentFixedSlot(ValOperandId resultId,
+ uint8_t slotIndex) {
+ return emitLoadArgumentSlot(resultId, slotIndex);
+}
+
+bool WarpCacheIRTranspiler::emitLoadArgumentDynamicSlot(ValOperandId resultId,
+ Int32OperandId argcId,
+ uint8_t slotIndex) {
+#ifdef DEBUG
+ MDefinition* argc = getOperand(argcId);
+ MOZ_ASSERT(argc->toConstant()->toInt32() ==
+ static_cast<int32_t>(callInfo_->argc()));
+#endif
+
+ return emitLoadArgumentSlot(resultId, callInfo_->argc() + slotIndex);
+}
+
+WrappedFunction* WarpCacheIRTranspiler::maybeWrappedFunction(
+ MDefinition* callee, CallKind kind, uint16_t nargs, FunctionFlags flags) {
+ MOZ_ASSERT(callee->isConstant() || callee->isNurseryObject());
+
+ // If this is a native without a JitEntry, WrappedFunction needs to know the
+ // target JSFunction.
+ // TODO: support nursery-allocated natives with WrappedFunction, maybe by
+ // storing the JSNative in the Baseline stub like flags/nargs.
+ bool isNative = flags.isNativeWithoutJitEntry();
+ if (isNative && !callee->isConstant()) {
+ return nullptr;
+ }
+
+ JSFunction* nativeTarget = nullptr;
+ if (isNative) {
+ nativeTarget = &callee->toConstant()->toObject().as<JSFunction>();
+ }
+
+ WrappedFunction* wrappedTarget =
+ new (alloc()) WrappedFunction(nativeTarget, nargs, flags);
+ MOZ_ASSERT_IF(kind == CallKind::Native || kind == CallKind::DOM,
+ wrappedTarget->isNativeWithoutJitEntry());
+ MOZ_ASSERT_IF(kind == CallKind::Scripted, wrappedTarget->hasJitEntry());
+ return wrappedTarget;
+}
+
+WrappedFunction* WarpCacheIRTranspiler::maybeCallTarget(MDefinition* callee,
+ CallKind kind) {
+ // CacheIR emits the following for specialized calls:
+ // GuardSpecificFunction <callee> <func> ..
+ // Call(Native|Scripted)Function <callee> ..
+ // or:
+ // GuardClass <callee> ..
+ // GuardFunctionScript <callee> <script> ..
+ // CallScriptedFunction <callee> ..
+ //
+ // We can use the <func> JSFunction or <script> BaseScript to specialize this
+ // call.
+ if (callee->isGuardSpecificFunction()) {
+ auto* guard = callee->toGuardSpecificFunction();
+ return maybeWrappedFunction(guard->expected(), kind, guard->nargs(),
+ guard->flags());
+ }
+ if (callee->isGuardFunctionScript()) {
+ MOZ_ASSERT(kind == CallKind::Scripted);
+ auto* guard = callee->toGuardFunctionScript();
+ WrappedFunction* wrappedTarget = new (alloc()) WrappedFunction(
+ /* nativeFun = */ nullptr, guard->nargs(), guard->flags());
+ MOZ_ASSERT(wrappedTarget->hasJitEntry());
+ return wrappedTarget;
+ }
+ return nullptr;
+}
+
+// If it is possible to use MCall for this call, update callInfo_ to use
+// the correct arguments. Otherwise, update the ArgFormat of callInfo_.
+bool WarpCacheIRTranspiler::updateCallInfo(MDefinition* callee,
+ CallFlags flags) {
+ // The transpilation will add various guards to the callee.
+ // We replace the callee referenced by the CallInfo, so that
+ // the resulting call instruction depends on these guards.
+ callInfo_->setCallee(callee);
+
+ // The transpilation may also add guards to other arguments.
+ // We replace those arguments in the CallInfo here.
+ updateArgumentsFromOperands();
+
+ switch (flags.getArgFormat()) {
+ case CallFlags::Standard:
+ MOZ_ASSERT(callInfo_->argFormat() == CallInfo::ArgFormat::Standard);
+ break;
+ case CallFlags::Spread:
+ MOZ_ASSERT(callInfo_->argFormat() == CallInfo::ArgFormat::Array);
+ break;
+ case CallFlags::FunCall:
+ // Note: We already changed the callee to the target
+ // function instead of the |call| function.
+ MOZ_ASSERT(!callInfo_->constructing());
+ MOZ_ASSERT(callInfo_->argFormat() == CallInfo::ArgFormat::Standard);
+
+ if (callInfo_->argc() == 0) {
+ // Special case for fun.call() with no arguments.
+ auto* undef = constant(UndefinedValue());
+ callInfo_->setThis(undef);
+ } else {
+ // The first argument for |call| is the new this value.
+ callInfo_->setThis(callInfo_->getArg(0));
+
+ // Shift down all other arguments by removing the first.
+ callInfo_->removeArg(0);
+ }
+ break;
+ case CallFlags::FunApplyArgsObj:
+ MOZ_ASSERT(!callInfo_->constructing());
+ MOZ_ASSERT(callInfo_->argFormat() == CallInfo::ArgFormat::Standard);
+
+ callInfo_->setArgFormat(CallInfo::ArgFormat::FunApplyArgsObj);
+ break;
+ case CallFlags::FunApplyArray: {
+ MDefinition* argFunc = callInfo_->thisArg();
+ MDefinition* argThis = callInfo_->getArg(0);
+ callInfo_->setCallee(argFunc);
+ callInfo_->setThis(argThis);
+ callInfo_->setArgFormat(CallInfo::ArgFormat::Array);
+ break;
+ }
+ default:
+ MOZ_CRASH("Unsupported arg format");
+ }
+ return true;
+}
+
+// Returns true if we are generating a call to CreateThisFromIon and
+// must check its return value.
+bool WarpCacheIRTranspiler::maybeCreateThis(MDefinition* callee,
+ CallFlags flags, CallKind kind) {
+ MOZ_ASSERT(kind != CallKind::DOM, "DOM functions are not constructors");
+ MDefinition* thisArg = callInfo_->thisArg();
+
+ if (kind == CallKind::Native) {
+ // Native functions keep the is-constructing MagicValue as |this|.
+ // If one of the arguments uses spread syntax this can be a loop phi with
+ // MIRType::Value.
+ MOZ_ASSERT(thisArg->type() == MIRType::MagicIsConstructing ||
+ thisArg->isPhi());
+ return false;
+ }
+ MOZ_ASSERT(kind == CallKind::Scripted);
+
+ if (thisArg->isNewPlainObject()) {
+ // We have already updated |this| based on MetaScriptedThisShape. We do
+ // not need to generate a check.
+ return false;
+ }
+ if (flags.needsUninitializedThis()) {
+ MConstant* uninit = constant(MagicValue(JS_UNINITIALIZED_LEXICAL));
+ thisArg->setImplicitlyUsedUnchecked();
+ callInfo_->setThis(uninit);
+ return false;
+ }
+ // See the Native case above.
+ MOZ_ASSERT(thisArg->type() == MIRType::MagicIsConstructing ||
+ thisArg->isPhi());
+
+ MDefinition* newTarget = callInfo_->getNewTarget();
+ auto* createThis = MCreateThis::New(alloc(), callee, newTarget);
+ add(createThis);
+
+ thisArg->setImplicitlyUsedUnchecked();
+ callInfo_->setThis(createThis);
+
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCallFunction(
+ ObjOperandId calleeId, Int32OperandId argcId,
+ mozilla::Maybe<ObjOperandId> thisObjId, CallFlags flags, CallKind kind) {
+ MDefinition* callee = getOperand(calleeId);
+#ifdef DEBUG
+ MDefinition* argc = getOperand(argcId);
+ MOZ_ASSERT(argc->toConstant()->toInt32() ==
+ static_cast<int32_t>(callInfo_->argc()));
+#endif
+
+ if (!updateCallInfo(callee, flags)) {
+ return false;
+ }
+
+ if (kind == CallKind::DOM) {
+ MOZ_ASSERT(flags.getArgFormat() == CallFlags::Standard);
+ // For DOM calls |this| has a class guard.
+ MDefinition* thisObj = getOperand(*thisObjId);
+ callInfo_->setThis(thisObj);
+ }
+
+ WrappedFunction* wrappedTarget = maybeCallTarget(callee, kind);
+
+ bool needsThisCheck = false;
+ if (callInfo_->constructing()) {
+ MOZ_ASSERT(flags.isConstructing());
+ needsThisCheck = maybeCreateThis(callee, flags, kind);
+ if (needsThisCheck) {
+ wrappedTarget = nullptr;
+ }
+ }
+
+ switch (callInfo_->argFormat()) {
+ case CallInfo::ArgFormat::Standard: {
+ MCall* call = makeCall(*callInfo_, needsThisCheck, wrappedTarget,
+ kind == CallKind::DOM);
+ if (!call) {
+ return false;
+ }
+
+ if (flags.isSameRealm()) {
+ call->setNotCrossRealm();
+ }
+
+ if (call->isEffectful()) {
+ addEffectful(call);
+ pushResult(call);
+ return resumeAfter(call);
+ }
+
+ MOZ_ASSERT(kind == CallKind::DOM);
+ add(call);
+ pushResult(call);
+ return true;
+ }
+ case CallInfo::ArgFormat::Array: {
+ MInstruction* call = makeSpreadCall(*callInfo_, needsThisCheck,
+ flags.isSameRealm(), wrappedTarget);
+ if (!call) {
+ return false;
+ }
+ addEffectful(call);
+ pushResult(call);
+
+ return resumeAfter(call);
+ }
+ case CallInfo::ArgFormat::FunApplyArgsObj: {
+ return emitFunApplyArgsObj(wrappedTarget, flags);
+ }
+ }
+ MOZ_CRASH("unreachable");
+}
+
+bool WarpCacheIRTranspiler::emitFunApplyArgsObj(WrappedFunction* wrappedTarget,
+ CallFlags flags) {
+ MOZ_ASSERT(!callInfo_->constructing());
+
+ MDefinition* callee = callInfo_->thisArg();
+ MDefinition* thisArg = callInfo_->getArg(0);
+ MDefinition* argsObj = callInfo_->getArg(1);
+
+ MApplyArgsObj* apply =
+ MApplyArgsObj::New(alloc(), wrappedTarget, callee, argsObj, thisArg);
+
+ if (flags.isSameRealm()) {
+ apply->setNotCrossRealm();
+ }
+ if (callInfo_->ignoresReturnValue()) {
+ apply->setIgnoresReturnValue();
+ }
+
+ addEffectful(apply);
+ pushResult(apply);
+
+ return resumeAfter(apply);
+}
+
+#ifndef JS_SIMULATOR
+bool WarpCacheIRTranspiler::emitCallNativeFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed,
+ bool ignoresReturnValue) {
+ // Instead of ignoresReturnValue we use CallInfo::ignoresReturnValue.
+ return emitCallFunction(calleeId, argcId, mozilla::Nothing(), flags,
+ CallKind::Native);
+}
+
+bool WarpCacheIRTranspiler::emitCallDOMFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ ObjOperandId thisObjId,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ return emitCallFunction(calleeId, argcId, mozilla::Some(thisObjId), flags,
+ CallKind::DOM);
+}
+#else
+bool WarpCacheIRTranspiler::emitCallNativeFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed,
+ uint32_t targetOffset) {
+ return emitCallFunction(calleeId, argcId, mozilla::Nothing(), flags,
+ CallKind::Native);
+}
+
+bool WarpCacheIRTranspiler::emitCallDOMFunction(
+ ObjOperandId calleeId, Int32OperandId argcId, ObjOperandId thisObjId,
+ CallFlags flags, uint32_t argcFixed, uint32_t targetOffset) {
+ return emitCallFunction(calleeId, argcId, mozilla::Some(thisObjId), flags,
+ CallKind::DOM);
+}
+#endif
+
+bool WarpCacheIRTranspiler::emitCallScriptedFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ return emitCallFunction(calleeId, argcId, mozilla::Nothing(), flags,
+ CallKind::Scripted);
+}
+
+bool WarpCacheIRTranspiler::emitCallInlinedFunction(ObjOperandId calleeId,
+ Int32OperandId argcId,
+ uint32_t icScriptOffset,
+ CallFlags flags,
+ uint32_t argcFixed) {
+ if (callInfo_->isInlined()) {
+ // We are transpiling to generate the correct guards. We also
+ // update the CallInfo to use the correct arguments. Code for the
+ // inlined function itself will be generated in
+ // WarpBuilder::buildInlinedCall.
+ MDefinition* callee = getOperand(calleeId);
+ if (!updateCallInfo(callee, flags)) {
+ return false;
+ }
+ if (callInfo_->constructing()) {
+ MOZ_ASSERT(flags.isConstructing());
+
+ // We call maybeCreateThis to update |this|, but inlined constructors
+ // never need a VM call. CallIRGenerator::getThisForScripted ensures that
+ // we don't attach a specialized stub unless we have a template object or
+ // know that the constructor needs uninitialized this.
+ MOZ_ALWAYS_FALSE(maybeCreateThis(callee, flags, CallKind::Scripted));
+ mozilla::DebugOnly<MDefinition*> thisArg = callInfo_->thisArg();
+ MOZ_ASSERT(thisArg->isNewPlainObject() ||
+ thisArg->type() == MIRType::MagicUninitializedLexical);
+ }
+
+ if (flags.getArgFormat() == CallFlags::FunCall) {
+ callInfo_->setInliningResumeMode(ResumeMode::InlinedFunCall);
+ } else {
+ MOZ_ASSERT(flags.getArgFormat() == CallFlags::Standard);
+ callInfo_->setInliningResumeMode(ResumeMode::InlinedStandardCall);
+ }
+
+ switch (callInfo_->argFormat()) {
+ case CallInfo::ArgFormat::Standard:
+ break;
+ default:
+ MOZ_CRASH("Unsupported arg format");
+ }
+ return true;
+ }
+ return emitCallFunction(calleeId, argcId, mozilla::Nothing(), flags,
+ CallKind::Scripted);
+}
+
+bool WarpCacheIRTranspiler::emitCallWasmFunction(
+ ObjOperandId calleeId, Int32OperandId argcId, CallFlags flags,
+ uint32_t argcFixed, uint32_t funcExportOffset, uint32_t instanceOffset) {
+ MDefinition* callee = getOperand(calleeId);
+#ifdef DEBUG
+ MDefinition* argc = getOperand(argcId);
+ MOZ_ASSERT(argc->toConstant()->toInt32() ==
+ static_cast<int32_t>(callInfo_->argc()));
+#endif
+ JSObject* instanceObject = tenuredObjectStubField(instanceOffset);
+ auto* wasmInstanceObj = &instanceObject->as<WasmInstanceObject>();
+ const wasm::FuncExport* funcExport = wasmFuncExportField(funcExportOffset);
+ const wasm::FuncType& sig =
+ wasmInstanceObj->instance().metadata().getFuncExportType(*funcExport);
+
+ if (!updateCallInfo(callee, flags)) {
+ return false;
+ }
+
+ static_assert(wasm::MaxArgsForJitInlineCall <= MaxNumLInstructionOperands,
+ "arguments must fit in LIR operands");
+ MOZ_ASSERT(sig.args().length() <= wasm::MaxArgsForJitInlineCall);
+
+ MOZ_ASSERT(callInfo_->argFormat() == CallInfo::ArgFormat::Standard);
+
+ auto* call = MIonToWasmCall::New(alloc(), wasmInstanceObj, *funcExport);
+ if (!call) {
+ return false;
+ }
+
+ mozilla::Maybe<MDefinition*> undefined;
+ for (size_t i = 0; i < sig.args().length(); i++) {
+ if (!alloc().ensureBallast()) {
+ return false;
+ }
+
+ MDefinition* arg;
+ if (i < callInfo_->argc()) {
+ arg = callInfo_->getArg(i);
+ } else {
+ if (!undefined) {
+ undefined.emplace(constant(UndefinedValue()));
+ }
+ arg = convertWasmArg(*undefined, sig.args()[i].kind());
+ }
+ call->initArg(i, arg);
+ }
+
+ addEffectful(call);
+
+ // Add any post-function call conversions that are necessary.
+ MInstruction* postConversion = call;
+ const wasm::ValTypeVector& results = sig.results();
+ MOZ_ASSERT(results.length() <= 1, "Multi-value returns not supported.");
+ if (results.length() == 0) {
+ // No results to convert.
+ } else {
+ switch (results[0].kind()) {
+ case wasm::ValType::I64:
+ // JS expects a BigInt from I64 types.
+ postConversion = MInt64ToBigInt::New(alloc(), call);
+
+ // Make non-movable so we can attach a resume point.
+ postConversion->setNotMovable();
+
+ add(postConversion);
+ break;
+ default:
+ // No spectre.index_masking of i32 results required, as the generated
+ // stub takes care of that.
+ break;
+ }
+ }
+
+ // The resume point has to be attached to the post-conversion instruction
+ // (if present) instead of to the call. This way, if the call triggers an
+ // invalidation bailout, we will have the BigInt value on the Baseline stack.
+ // Potential alternative solution: attach the resume point to the call and
+ // have bailouts turn the Int64 value into a BigInt, maybe with a recover
+ // instruction.
+ pushResult(postConversion);
+ return resumeAfterUnchecked(postConversion);
+}
+
+MDefinition* WarpCacheIRTranspiler::convertWasmArg(MDefinition* arg,
+ wasm::ValType::Kind kind) {
+ // An invariant in this code is that any type conversion operation that has
+ // externally visible effects, such as invoking valueOf on an object argument,
+ // must bailout so that we don't have to worry about replaying effects during
+ // argument conversion.
+ MInstruction* conversion = nullptr;
+ switch (kind) {
+ case wasm::ValType::I32:
+ conversion = MTruncateToInt32::New(alloc(), arg);
+ break;
+ case wasm::ValType::I64:
+ conversion = MToInt64::New(alloc(), arg);
+ break;
+ case wasm::ValType::F32:
+ conversion = MToFloat32::New(alloc(), arg);
+ break;
+ case wasm::ValType::F64:
+ conversion = MToDouble::New(alloc(), arg);
+ break;
+ case wasm::ValType::V128:
+ MOZ_CRASH("Unexpected type for Wasm JitEntry");
+ case wasm::ValType::Ref:
+ // Transform the JS representation into an AnyRef representation.
+ // The resulting type is MIRType::RefOrNull. These cases are all
+ // effect-free.
+ switch (arg->type()) {
+ case MIRType::Object:
+ conversion = MWasmAnyRefFromJSObject::New(alloc(), arg);
+ break;
+ case MIRType::Null:
+ arg->setImplicitlyUsedUnchecked();
+ conversion = MWasmNullConstant::New(alloc());
+ break;
+ default:
+ conversion = MWasmBoxValue::New(alloc(), arg);
+ break;
+ }
+ break;
+ }
+
+ add(conversion);
+ return conversion;
+}
+
+bool WarpCacheIRTranspiler::emitGuardWasmArg(ValOperandId argId,
+ wasm::ValType::Kind kind) {
+ MDefinition* arg = getOperand(argId);
+ MDefinition* conversion = convertWasmArg(arg, kind);
+
+ setOperand(argId, conversion);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCallGetterResult(CallKind kind,
+ ValOperandId receiverId,
+ uint32_t getterOffset,
+ bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ MDefinition* receiver = getOperand(receiverId);
+ MDefinition* getter = objectStubField(getterOffset);
+ uint32_t nargsAndFlags = uint32StubField(nargsAndFlagsOffset);
+
+ uint16_t nargs = nargsAndFlags >> 16;
+ FunctionFlags flags = FunctionFlags(uint16_t(nargsAndFlags));
+ WrappedFunction* wrappedTarget =
+ maybeWrappedFunction(getter, kind, nargs, flags);
+
+ bool ignoresRval = loc_.resultIsPopped();
+ CallInfo callInfo(alloc(), /* constructing = */ false, ignoresRval);
+ callInfo.initForGetterCall(getter, receiver);
+
+ MCall* call = makeCall(callInfo, /* needsThisCheck = */ false, wrappedTarget);
+ if (!call) {
+ return false;
+ }
+
+ if (sameRealm) {
+ call->setNotCrossRealm();
+ }
+
+ addEffectful(call);
+ pushResult(call);
+
+ return resumeAfter(call);
+}
+
+bool WarpCacheIRTranspiler::emitCallScriptedGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ return emitCallGetterResult(CallKind::Scripted, receiverId, getterOffset,
+ sameRealm, nargsAndFlagsOffset);
+}
+
+bool WarpCacheIRTranspiler::emitCallInlinedGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, uint32_t icScriptOffset,
+ bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ if (callInfo_) {
+ MOZ_ASSERT(callInfo_->isInlined());
+ // We are transpiling to generate the correct guards. We also update the
+ // CallInfo to use the correct arguments. Code for the inlined getter
+ // itself will be generated in WarpBuilder::buildInlinedCall.
+ MDefinition* receiver = getOperand(receiverId);
+ MDefinition* getter = objectStubField(getterOffset);
+ callInfo_->initForGetterCall(getter, receiver);
+ callInfo_->setInliningResumeMode(ResumeMode::InlinedAccessor);
+
+ // Make sure there's enough room to push the arguments on the stack.
+ if (!current->ensureHasSlots(2)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ return emitCallGetterResult(CallKind::Scripted, receiverId, getterOffset,
+ sameRealm, nargsAndFlagsOffset);
+}
+
+bool WarpCacheIRTranspiler::emitCallNativeGetterResult(
+ ValOperandId receiverId, uint32_t getterOffset, bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ return emitCallGetterResult(CallKind::Native, receiverId, getterOffset,
+ sameRealm, nargsAndFlagsOffset);
+}
+
+bool WarpCacheIRTranspiler::emitCallSetter(CallKind kind,
+ ObjOperandId receiverId,
+ uint32_t setterOffset,
+ ValOperandId rhsId, bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ MDefinition* receiver = getOperand(receiverId);
+ MDefinition* setter = objectStubField(setterOffset);
+ MDefinition* rhs = getOperand(rhsId);
+ uint32_t nargsAndFlags = uint32StubField(nargsAndFlagsOffset);
+
+ uint16_t nargs = nargsAndFlags >> 16;
+ FunctionFlags flags = FunctionFlags(uint16_t(nargsAndFlags));
+ WrappedFunction* wrappedTarget =
+ maybeWrappedFunction(setter, kind, nargs, flags);
+
+ CallInfo callInfo(alloc(), /* constructing = */ false,
+ /* ignoresReturnValue = */ true);
+ callInfo.initForSetterCall(setter, receiver, rhs);
+
+ MCall* call = makeCall(callInfo, /* needsThisCheck = */ false, wrappedTarget);
+ if (!call) {
+ return false;
+ }
+
+ if (sameRealm) {
+ call->setNotCrossRealm();
+ }
+
+ addEffectful(call);
+ return resumeAfter(call);
+}
+
+bool WarpCacheIRTranspiler::emitCallScriptedSetter(
+ ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId,
+ bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ return emitCallSetter(CallKind::Scripted, receiverId, setterOffset, rhsId,
+ sameRealm, nargsAndFlagsOffset);
+}
+
+bool WarpCacheIRTranspiler::emitCallInlinedSetter(
+ ObjOperandId receiverId, uint32_t setterOffset, ValOperandId rhsId,
+ uint32_t icScriptOffset, bool sameRealm, uint32_t nargsAndFlagsOffset) {
+ if (callInfo_) {
+ MOZ_ASSERT(callInfo_->isInlined());
+ // We are transpiling to generate the correct guards. We also update the
+ // CallInfo to use the correct arguments. Code for the inlined setter
+ // itself will be generated in WarpBuilder::buildInlinedCall.
+ MDefinition* receiver = getOperand(receiverId);
+ MDefinition* setter = objectStubField(setterOffset);
+ MDefinition* rhs = getOperand(rhsId);
+ callInfo_->initForSetterCall(setter, receiver, rhs);
+ callInfo_->setInliningResumeMode(ResumeMode::InlinedAccessor);
+
+ // Make sure there's enough room to push the arguments on the stack.
+ if (!current->ensureHasSlots(3)) {
+ return false;
+ }
+
+ return true;
+ }
+
+ return emitCallSetter(CallKind::Scripted, receiverId, setterOffset, rhsId,
+ sameRealm, nargsAndFlagsOffset);
+}
+
+bool WarpCacheIRTranspiler::emitCallNativeSetter(ObjOperandId receiverId,
+ uint32_t setterOffset,
+ ValOperandId rhsId,
+ bool sameRealm,
+ uint32_t nargsAndFlagsOffset) {
+ return emitCallSetter(CallKind::Native, receiverId, setterOffset, rhsId,
+ sameRealm, nargsAndFlagsOffset);
+}
+
+bool WarpCacheIRTranspiler::emitMetaScriptedThisShape(
+ uint32_t thisShapeOffset) {
+ SharedShape* shape = &shapeStubField(thisShapeOffset)->asShared();
+ MOZ_ASSERT(shape->getObjectClass() == &PlainObject::class_);
+
+ MConstant* shapeConst = MConstant::NewShape(alloc(), shape);
+ add(shapeConst);
+
+ // TODO: support pre-tenuring.
+ gc::InitialHeap heap = gc::DefaultHeap;
+
+ uint32_t numFixedSlots = shape->numFixedSlots();
+ uint32_t numDynamicSlots = NativeObject::calculateDynamicSlots(shape);
+ gc::AllocKind kind = gc::GetGCObjectKind(numFixedSlots);
+ MOZ_ASSERT(gc::CanChangeToBackgroundAllocKind(kind, &PlainObject::class_));
+ kind = gc::ForegroundToBackgroundAllocKind(kind);
+
+ auto* createThis = MNewPlainObject::New(alloc(), shapeConst, numFixedSlots,
+ numDynamicSlots, kind, heap);
+ add(createThis);
+
+ callInfo_->thisArg()->setImplicitlyUsedUnchecked();
+ callInfo_->setThis(createThis);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitReturnFromIC() { return true; }
+
+bool WarpCacheIRTranspiler::emitBailout() {
+ auto* bail = MBail::New(alloc());
+ add(bail);
+
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitAssertRecoveredOnBailoutResult(
+ ValOperandId valId, bool mustBeRecovered) {
+ MDefinition* val = getOperand(valId);
+
+ // Don't assert for recovered instructions when recovering is disabled.
+ if (JitOptions.disableRecoverIns) {
+ pushResult(constant(UndefinedValue()));
+ return true;
+ }
+
+ if (JitOptions.checkRangeAnalysis) {
+ // If we are checking the range of all instructions, then the guards
+ // inserted by Range Analysis prevent the use of recover instruction. Thus,
+ // we just disable these checks.
+ pushResult(constant(UndefinedValue()));
+ return true;
+ }
+
+ auto* assert = MAssertRecoveredOnBailout::New(alloc(), val, mustBeRecovered);
+ addEffectfulUnsafe(assert);
+ current->push(assert);
+
+ // Create an instruction sequence which implies that the argument of the
+ // assertRecoveredOnBailout function would be encoded at least in one
+ // Snapshot.
+ auto* nop = MNop::New(alloc());
+ add(nop);
+
+ auto* resumePoint = MResumePoint::New(
+ alloc(), nop->block(), loc_.toRawBytecode(), ResumeMode::ResumeAfter);
+ if (!resumePoint) {
+ return false;
+ }
+ nop->setResumePoint(resumePoint);
+
+ auto* encode = MEncodeSnapshot::New(alloc());
+ addEffectfulUnsafe(encode);
+
+ current->pop();
+
+ pushResult(constant(UndefinedValue()));
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitGuardNoAllocationMetadataBuilder(
+ uint32_t builderAddrOffset) {
+ // This is a no-op because we discard all JIT code when set an allocation
+ // metadata callback.
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitNewPlainObjectResult(uint32_t numFixedSlots,
+ uint32_t numDynamicSlots,
+ gc::AllocKind allocKind,
+ uint32_t shapeOffset,
+ uint32_t siteOffset) {
+ Shape* shape = shapeStubField(shapeOffset);
+ gc::InitialHeap heap = allocSiteInitialHeapField(siteOffset);
+
+ auto* shapeConstant = MConstant::NewShape(alloc(), shape);
+ add(shapeConstant);
+
+ auto* obj = MNewPlainObject::New(alloc(), shapeConstant, numFixedSlots,
+ numDynamicSlots, allocKind, heap);
+ add(obj);
+
+ pushResult(obj);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitNewArrayObjectResult(uint32_t length,
+ uint32_t shapeOffset,
+ uint32_t siteOffset) {
+ Shape* shape = shapeStubField(shapeOffset);
+ gc::InitialHeap heap = allocSiteInitialHeapField(siteOffset);
+
+ auto* shapeConstant = MConstant::NewShape(alloc(), shape);
+ add(shapeConstant);
+
+ auto* obj = MNewArrayObject::New(alloc(), shapeConstant, length, heap);
+ add(obj);
+
+ pushResult(obj);
+ return true;
+}
+
+bool WarpCacheIRTranspiler::emitCloseIterScriptedResult(ObjOperandId iterId,
+ ObjOperandId calleeId,
+ CompletionKind kind,
+ uint32_t calleeNargs) {
+ MDefinition* iter = getOperand(iterId);
+ MDefinition* callee = getOperand(calleeId);
+
+ WrappedFunction* wrappedTarget = maybeCallTarget(callee, CallKind::Scripted);
+ MOZ_ASSERT(wrappedTarget);
+ MOZ_ASSERT(wrappedTarget->nargs() == calleeNargs);
+ MOZ_ASSERT(wrappedTarget->hasJitEntry());
+
+ bool constructing = false;
+ bool ignoresRval = false;
+ bool needsThisCheck = false;
+ bool isDOMCall = false;
+ CallInfo callInfo(alloc(), constructing, ignoresRval);
+ callInfo.initForCloseIter(iter, callee);
+ MCall* call = makeCall(callInfo, needsThisCheck, wrappedTarget, isDOMCall);
+ if (!call) {
+ return false;
+ }
+ addEffectful(call);
+ if (kind == CompletionKind::Throw) {
+ return resumeAfter(call);
+ }
+
+ // If we bail out here, after the call but before the CheckIsObj, we
+ // can't simply resume in the baseline interpreter. If we resume
+ // after the CloseIter, we won't check the return value. If we
+ // resume at the CloseIter, we will call the |return| method twice.
+ // Instead, we use a special resume mode that captures the
+ // intermediate value, and then checks that it's an object while
+ // bailing out.
+ current->push(call);
+ MResumePoint* resumePoint =
+ MResumePoint::New(alloc(), current, loc_.toRawBytecode(),
+ ResumeMode::ResumeAfterCheckIsObject);
+ if (!resumePoint) {
+ return false;
+ }
+ call->setResumePoint(resumePoint);
+ current->pop();
+
+ MCheckIsObj* check = MCheckIsObj::New(
+ alloc(), call, uint8_t(CheckIsObjectKind::IteratorReturn));
+ add(check);
+
+ return true;
+}
+
+#ifdef FUZZING_JS_FUZZILLI
+bool WarpCacheIRTranspiler::emitFuzzilliHashResult(ValOperandId valId) {
+ MDefinition* input = getOperand(valId);
+
+ auto* hash = MFuzzilliHash::New(alloc(), input);
+ add(hash);
+
+ auto* store = MFuzzilliHashStore::New(alloc(), hash);
+ addEffectful(store);
+ pushResult(constant(UndefinedValue()));
+
+ return resumeAfter(store);
+}
+#endif
+
+static void MaybeSetImplicitlyUsed(uint32_t numInstructionIdsBefore,
+ MDefinition* input) {
+ // When building MIR from bytecode, for each MDefinition that's an operand to
+ // a bytecode instruction, we must either add an SSA use or set the
+ // ImplicitlyUsed flag on that definition. The ImplicitlyUsed flag prevents
+ // the backend from optimizing-out values that will be used by Baseline after
+ // a bailout.
+ //
+ // WarpBuilder uses WarpPoppedValueUseChecker to assert this invariant in
+ // debug builds.
+ //
+ // This function is responsible for setting the ImplicitlyUsed flag for an
+ // input when using the transpiler. It looks at the input's most recent use
+ // and if that's an instruction that was added while transpiling this JSOp
+ // (based on the MIR instruction id) we don't set the ImplicitlyUsed flag.
+
+ if (input->isImplicitlyUsed()) {
+ // Nothing to do.
+ return;
+ }
+
+ // If the most recent use of 'input' is an instruction we just added, there is
+ // nothing to do.
+ MDefinition* inputUse = input->maybeMostRecentlyAddedDefUse();
+ if (inputUse && inputUse->id() >= numInstructionIdsBefore) {
+ return;
+ }
+
+ // The transpiler didn't add a use for 'input'.
+ input->setImplicitlyUsed();
+}
+
+bool jit::TranspileCacheIRToMIR(WarpBuilder* builder, BytecodeLocation loc,
+ const WarpCacheIR* cacheIRSnapshot,
+ std::initializer_list<MDefinition*> inputs,
+ CallInfo* maybeCallInfo) {
+ uint32_t numInstructionIdsBefore =
+ builder->mirGen().graph().getNumInstructionIds();
+
+ WarpCacheIRTranspiler transpiler(builder, loc, maybeCallInfo,
+ cacheIRSnapshot);
+ if (!transpiler.transpile(inputs)) {
+ return false;
+ }
+
+ for (MDefinition* input : inputs) {
+ MaybeSetImplicitlyUsed(numInstructionIdsBefore, input);
+ }
+
+ if (maybeCallInfo) {
+ auto maybeSetFlag = [numInstructionIdsBefore](MDefinition* def) {
+ MaybeSetImplicitlyUsed(numInstructionIdsBefore, def);
+ };
+ maybeCallInfo->forEachCallOperand(maybeSetFlag);
+ }
+
+ return true;
+}
diff --git a/js/src/jit/WarpCacheIRTranspiler.h b/js/src/jit/WarpCacheIRTranspiler.h
new file mode 100644
index 0000000000..19022b9e1a
--- /dev/null
+++ b/js/src/jit/WarpCacheIRTranspiler.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_WarpCacheIRTranspiler_h
+#define jit_WarpCacheIRTranspiler_h
+
+#include <initializer_list>
+
+namespace js {
+
+class BytecodeLocation;
+
+namespace jit {
+
+class CallInfo;
+class MDefinition;
+class WarpBuilder;
+class WarpCacheIR;
+
+// Generate MIR from a Baseline ICStub's CacheIR.
+[[nodiscard]] bool TranspileCacheIRToMIR(
+ WarpBuilder* builder, BytecodeLocation loc,
+ const WarpCacheIR* cacheIRSnapshot,
+ std::initializer_list<MDefinition*> inputs,
+ CallInfo* maybeCallInfo = nullptr);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_WarpCacheIRTranspiler_h */
diff --git a/js/src/jit/WarpOracle.cpp b/js/src/jit/WarpOracle.cpp
new file mode 100644
index 0000000000..6e2c0ce12f
--- /dev/null
+++ b/js/src/jit/WarpOracle.cpp
@@ -0,0 +1,1171 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/WarpOracle.h"
+
+#include "mozilla/ScopeExit.h"
+
+#include <algorithm>
+
+#include "jit/CacheIR.h"
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIRReader.h"
+#include "jit/CompileInfo.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitRealm.h"
+#include "jit/JitScript.h"
+#include "jit/JitSpewer.h"
+#include "jit/MIRGenerator.h"
+#include "jit/TrialInlining.h"
+#include "jit/TypeData.h"
+#include "jit/WarpBuilder.h"
+#include "util/DifferentialTesting.h"
+#include "vm/BuiltinObjectKind.h"
+#include "vm/BytecodeIterator.h"
+#include "vm/BytecodeLocation.h"
+
+#include "jit/InlineScriptTree-inl.h"
+#include "vm/BytecodeIterator-inl.h"
+#include "vm/BytecodeLocation-inl.h"
+#include "vm/EnvironmentObject-inl.h"
+#include "vm/Interpreter-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::Maybe;
+
+// WarpScriptOracle creates a WarpScriptSnapshot for a single JSScript. Note
+// that a single WarpOracle can use multiple WarpScriptOracles when scripts are
+// inlined.
+class MOZ_STACK_CLASS WarpScriptOracle {
+ JSContext* cx_;
+ WarpOracle* oracle_;
+ MIRGenerator& mirGen_;
+ TempAllocator& alloc_;
+ HandleScript script_;
+ const CompileInfo* info_;
+ ICScript* icScript_;
+
+ // Index of the next ICEntry for getICEntry. This assumes the script's
+ // bytecode is processed from first to last instruction.
+ uint32_t icEntryIndex_ = 0;
+
+ template <typename... Args>
+ mozilla::GenericErrorResult<AbortReason> abort(Args&&... args) {
+ return oracle_->abort(script_, args...);
+ }
+
+ WarpEnvironment createEnvironment();
+ AbortReasonOr<Ok> maybeInlineIC(WarpOpSnapshotList& snapshots,
+ BytecodeLocation loc);
+ AbortReasonOr<bool> maybeInlineCall(WarpOpSnapshotList& snapshots,
+ BytecodeLocation loc, ICCacheIRStub* stub,
+ ICFallbackStub* fallbackStub,
+ uint8_t* stubDataCopy);
+ AbortReasonOr<bool> maybeInlinePolymorphicTypes(WarpOpSnapshotList& snapshots,
+ BytecodeLocation loc,
+ ICCacheIRStub* firstStub,
+ ICFallbackStub* fallbackStub);
+ [[nodiscard]] bool replaceNurseryAndAllocSitePointers(
+ ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo,
+ uint8_t* stubDataCopy);
+
+ public:
+ WarpScriptOracle(JSContext* cx, WarpOracle* oracle, HandleScript script,
+ const CompileInfo* info, ICScript* icScript)
+ : cx_(cx),
+ oracle_(oracle),
+ mirGen_(oracle->mirGen()),
+ alloc_(mirGen_.alloc()),
+ script_(script),
+ info_(info),
+ icScript_(icScript) {}
+
+ AbortReasonOr<WarpScriptSnapshot*> createScriptSnapshot();
+
+ ICEntry& getICEntryAndFallback(BytecodeLocation loc,
+ ICFallbackStub** fallback);
+};
+
+WarpOracle::WarpOracle(JSContext* cx, MIRGenerator& mirGen,
+ HandleScript outerScript)
+ : cx_(cx),
+ mirGen_(mirGen),
+ alloc_(mirGen.alloc()),
+ outerScript_(outerScript) {}
+
+mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script,
+ AbortReason r) {
+ auto res = mirGen_.abort(r);
+ JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename());
+ return res;
+}
+
+mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script,
+ AbortReason r,
+ const char* message,
+ ...) {
+ va_list ap;
+ va_start(ap, message);
+ auto res = mirGen_.abortFmt(r, message, ap);
+ va_end(ap);
+ JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename());
+ return res;
+}
+
+void WarpOracle::addScriptSnapshot(WarpScriptSnapshot* scriptSnapshot) {
+ scriptSnapshots_.insertBack(scriptSnapshot);
+}
+
+AbortReasonOr<WarpSnapshot*> WarpOracle::createSnapshot() {
+#ifdef JS_JITSPEW
+ const char* mode;
+ if (outerScript_->hasIonScript()) {
+ mode = "Recompiling";
+ } else {
+ mode = "Compiling";
+ }
+ JitSpew(JitSpew_IonScripts,
+ "Warp %s script %s:%u:%u (%p) (warmup-counter=%" PRIu32 ",%s%s)",
+ mode, outerScript_->filename(), outerScript_->lineno(),
+ outerScript_->column(), static_cast<JSScript*>(outerScript_),
+ outerScript_->getWarmUpCount(),
+ outerScript_->isGenerator() ? " isGenerator" : "",
+ outerScript_->isAsync() ? " isAsync" : "");
+#endif
+
+ MOZ_ASSERT(outerScript_->hasJitScript());
+ ICScript* icScript = outerScript_->jitScript()->icScript();
+ WarpScriptOracle scriptOracle(cx_, this, outerScript_, &mirGen_.outerInfo(),
+ icScript);
+
+ WarpScriptSnapshot* scriptSnapshot;
+ MOZ_TRY_VAR(scriptSnapshot, scriptOracle.createScriptSnapshot());
+
+ // Insert the outermost scriptSnapshot at the front of the list.
+ scriptSnapshots_.insertFront(scriptSnapshot);
+
+ bool recordFinalWarmUpCount = false;
+#ifdef JS_CACHEIR_SPEW
+ recordFinalWarmUpCount = outerScript_->needsFinalWarmUpCount();
+#endif
+
+ auto* snapshot = new (alloc_.fallible())
+ WarpSnapshot(cx_, alloc_, std::move(scriptSnapshots_), bailoutInfo_,
+ recordFinalWarmUpCount);
+ if (!snapshot) {
+ return abort(outerScript_, AbortReason::Alloc);
+ }
+
+ if (!snapshot->nurseryObjects().appendAll(nurseryObjects_)) {
+ return abort(outerScript_, AbortReason::Alloc);
+ }
+
+#ifdef JS_JITSPEW
+ if (JitSpewEnabled(JitSpew_WarpSnapshots)) {
+ Fprinter& out = JitSpewPrinter();
+ snapshot->dump(out);
+ }
+#endif
+
+#ifdef DEBUG
+ // When transpiled CacheIR bails out, we do not want to recompile
+ // with the exact same data and get caught in an invalidation loop.
+ //
+ // To avoid this, we store a hash of the stub pointers and entry
+ // counts in this snapshot, save that hash in the JitScript if we
+ // have a TranspiledCacheIR bailout, and assert that the hash has
+ // changed when we recompile.
+ //
+ // Note: this assertion catches potential performance issues.
+ // Failing this assertion is not a correctness/security problem.
+ // We therefore ignore cases involving resource exhaustion (OOM,
+ // stack overflow, etc), or stubs purged by GC.
+ HashNumber hash = icScript->hash();
+ if (outerScript_->jitScript()->hasFailedICHash()) {
+ HashNumber oldHash = outerScript_->jitScript()->getFailedICHash();
+ MOZ_ASSERT_IF(hash == oldHash && !js::SupportDifferentialTesting(),
+ cx_->hadResourceExhaustion());
+ }
+ snapshot->setICHash(hash);
+#endif
+
+ return snapshot;
+}
+
+template <typename T, typename... Args>
+[[nodiscard]] static bool AddOpSnapshot(TempAllocator& alloc,
+ WarpOpSnapshotList& snapshots,
+ uint32_t offset, Args&&... args) {
+ T* snapshot = new (alloc.fallible()) T(offset, std::forward<Args>(args)...);
+ if (!snapshot) {
+ return false;
+ }
+
+ snapshots.insertBack(snapshot);
+ return true;
+}
+
+[[nodiscard]] static bool AddWarpGetImport(TempAllocator& alloc,
+ WarpOpSnapshotList& snapshots,
+ uint32_t offset, JSScript* script,
+ PropertyName* name) {
+ ModuleEnvironmentObject* env = GetModuleEnvironmentForScript(script);
+ MOZ_ASSERT(env);
+
+ mozilla::Maybe<PropertyInfo> prop;
+ ModuleEnvironmentObject* targetEnv;
+ MOZ_ALWAYS_TRUE(env->lookupImport(NameToId(name), &targetEnv, &prop));
+
+ uint32_t numFixedSlots = targetEnv->numFixedSlots();
+ uint32_t slot = prop->slot();
+
+ // In the rare case where this import hasn't been initialized already (we have
+ // an import cycle where modules reference each other's imports), we need a
+ // check.
+ bool needsLexicalCheck =
+ targetEnv->getSlot(slot).isMagic(JS_UNINITIALIZED_LEXICAL);
+
+ return AddOpSnapshot<WarpGetImport>(alloc, snapshots, offset, targetEnv,
+ numFixedSlots, slot, needsLexicalCheck);
+}
+
+ICEntry& WarpScriptOracle::getICEntryAndFallback(BytecodeLocation loc,
+ ICFallbackStub** fallback) {
+ const uint32_t offset = loc.bytecodeToOffset(script_);
+
+ do {
+ *fallback = icScript_->fallbackStub(icEntryIndex_);
+ icEntryIndex_++;
+ } while ((*fallback)->pcOffset() < offset);
+
+ MOZ_ASSERT((*fallback)->pcOffset() == offset);
+ return icScript_->icEntry(icEntryIndex_ - 1);
+}
+
+WarpEnvironment WarpScriptOracle::createEnvironment() {
+ // Don't do anything if the script doesn't use the environment chain.
+ // Always make an environment chain if the script needs an arguments object
+ // because ArgumentsObject construction requires the environment chain to be
+ // passed in.
+ if (!script_->jitScript()->usesEnvironmentChain() &&
+ !script_->needsArgsObj()) {
+ return WarpEnvironment(NoEnvironment());
+ }
+
+ if (script_->isModule()) {
+ ModuleObject* module = script_->module();
+ JSObject* obj = &module->initialEnvironment();
+ return WarpEnvironment(ConstantObjectEnvironment(obj));
+ }
+
+ JSFunction* fun = script_->function();
+ if (!fun) {
+ // For global scripts without a non-syntactic global scope, the environment
+ // chain is the global lexical environment.
+ MOZ_ASSERT(!script_->isForEval());
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+ JSObject* obj = &script_->global().lexicalEnvironment();
+ return WarpEnvironment(ConstantObjectEnvironment(obj));
+ }
+
+ JSObject* templateEnv = script_->jitScript()->templateEnvironment();
+
+ CallObject* callObjectTemplate = nullptr;
+ if (fun->needsCallObject()) {
+ callObjectTemplate = &templateEnv->as<CallObject>();
+ }
+
+ NamedLambdaObject* namedLambdaTemplate = nullptr;
+ if (fun->needsNamedLambdaEnvironment()) {
+ if (callObjectTemplate) {
+ templateEnv = templateEnv->enclosingEnvironment();
+ }
+ namedLambdaTemplate = &templateEnv->as<NamedLambdaObject>();
+ }
+
+ return WarpEnvironment(
+ FunctionEnvironment(callObjectTemplate, namedLambdaTemplate));
+}
+
+AbortReasonOr<WarpScriptSnapshot*> WarpScriptOracle::createScriptSnapshot() {
+ MOZ_ASSERT(script_->hasJitScript());
+
+ if (!script_->jitScript()->ensureHasCachedIonData(cx_, script_)) {
+ return abort(AbortReason::Error);
+ }
+
+ if (script_->failedBoundsCheck()) {
+ oracle_->bailoutInfo().setFailedBoundsCheck();
+ }
+ if (script_->failedLexicalCheck()) {
+ oracle_->bailoutInfo().setFailedLexicalCheck();
+ }
+
+ WarpEnvironment environment = createEnvironment();
+
+ // Unfortunately LinkedList<> asserts the list is empty in its destructor.
+ // Clear the list if we abort compilation.
+ WarpOpSnapshotList opSnapshots;
+ auto autoClearOpSnapshots =
+ mozilla::MakeScopeExit([&] { opSnapshots.clear(); });
+
+ ModuleObject* moduleObject = nullptr;
+
+ // Analyze the bytecode. Abort compilation for unsupported ops and create
+ // WarpOpSnapshots.
+ for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
+ JSOp op = loc.getOp();
+ uint32_t offset = loc.bytecodeToOffset(script_);
+ switch (op) {
+ case JSOp::Arguments: {
+ MOZ_ASSERT(script_->needsArgsObj());
+ bool mapped = script_->hasMappedArgsObj();
+ ArgumentsObject* templateObj =
+ script_->global().maybeArgumentsTemplateObject(mapped);
+ if (!AddOpSnapshot<WarpArguments>(alloc_, opSnapshots, offset,
+ templateObj)) {
+ return abort(AbortReason::Alloc);
+ }
+ break;
+ }
+ case JSOp::RegExp: {
+ bool hasShared = loc.getRegExp(script_)->hasShared();
+ if (!AddOpSnapshot<WarpRegExp>(alloc_, opSnapshots, offset,
+ hasShared)) {
+ return abort(AbortReason::Alloc);
+ }
+ break;
+ }
+
+ case JSOp::FunctionThis:
+ if (!script_->strict() && script_->hasNonSyntacticScope()) {
+ // Abort because MBoxNonStrictThis doesn't support non-syntactic
+ // scopes (a deprecated SpiderMonkey mechanism). If this becomes an
+ // issue we could support it by refactoring GetFunctionThis to not
+ // take a frame pointer and then call that.
+ return abort(AbortReason::Disable,
+ "JSOp::FunctionThis with non-syntactic scope");
+ }
+ break;
+
+ case JSOp::GlobalThis:
+ MOZ_ASSERT(!script_->hasNonSyntacticScope());
+ break;
+
+ case JSOp::BuiltinObject: {
+ // If we already resolved this built-in we can bake it in.
+ auto kind = loc.getBuiltinObjectKind();
+ if (JSObject* proto = MaybeGetBuiltinObject(cx_->global(), kind)) {
+ if (!AddOpSnapshot<WarpBuiltinObject>(alloc_, opSnapshots, offset,
+ proto)) {
+ return abort(AbortReason::Alloc);
+ }
+ }
+ break;
+ }
+
+ case JSOp::GetIntrinsic: {
+ // If we already cloned this intrinsic we can bake it in.
+ // NOTE: When the initializer runs in a content global, we also have to
+ // worry about nursery objects. These quickly tenure and stay that
+ // way so this is only a temporary problem.
+ PropertyName* name = loc.getPropertyName(script_);
+ Value val;
+ if (cx_->global()->maybeGetIntrinsicValue(name, &val, cx_) &&
+ JS::GCPolicy<Value>::isTenured(val)) {
+ if (!AddOpSnapshot<WarpGetIntrinsic>(alloc_, opSnapshots, offset,
+ val)) {
+ return abort(AbortReason::Alloc);
+ }
+ }
+ break;
+ }
+
+ case JSOp::ImportMeta: {
+ if (!moduleObject) {
+ moduleObject = GetModuleObjectForScript(script_);
+ MOZ_ASSERT(moduleObject->isTenured());
+ }
+ break;
+ }
+
+ case JSOp::GetImport: {
+ PropertyName* name = loc.getPropertyName(script_);
+ if (!AddWarpGetImport(alloc_, opSnapshots, offset, script_, name)) {
+ return abort(AbortReason::Alloc);
+ }
+ break;
+ }
+
+ case JSOp::Lambda: {
+ JSFunction* fun = loc.getFunction(script_);
+ if (IsAsmJSModule(fun)) {
+ return abort(AbortReason::Disable, "asm.js module function lambda");
+ }
+ break;
+ }
+
+ case JSOp::GetElemSuper: {
+#if defined(JS_CODEGEN_X86)
+ // x86 does not have enough registers.
+ return abort(AbortReason::Disable,
+ "GetElemSuper is not supported on x86");
+#else
+ MOZ_TRY(maybeInlineIC(opSnapshots, loc));
+ break;
+#endif
+ }
+
+ case JSOp::Rest: {
+ if (Shape* shape =
+ script_->global().maybeArrayShapeWithDefaultProto()) {
+ if (!AddOpSnapshot<WarpRest>(alloc_, opSnapshots, offset, shape)) {
+ return abort(AbortReason::Alloc);
+ }
+ }
+ break;
+ }
+
+ case JSOp::BindGName: {
+ Rooted<GlobalObject*> global(cx_, &script_->global());
+ Rooted<PropertyName*> name(cx_, loc.getPropertyName(script_));
+ if (JSObject* env = MaybeOptimizeBindGlobalName(cx_, global, name)) {
+ MOZ_ASSERT(env->isTenured());
+ if (!AddOpSnapshot<WarpBindGName>(alloc_, opSnapshots, offset, env)) {
+ return abort(AbortReason::Alloc);
+ }
+ } else {
+ MOZ_TRY(maybeInlineIC(opSnapshots, loc));
+ }
+ break;
+ }
+
+ case JSOp::PushVarEnv: {
+ Rooted<VarScope*> scope(cx_, &loc.getScope(script_)->as<VarScope>());
+
+ auto* templateObj =
+ VarEnvironmentObject::createTemplateObject(cx_, scope);
+ if (!templateObj) {
+ return abort(AbortReason::Alloc);
+ }
+ MOZ_ASSERT(templateObj->isTenured());
+
+ if (!AddOpSnapshot<WarpVarEnvironment>(alloc_, opSnapshots, offset,
+ templateObj)) {
+ return abort(AbortReason::Alloc);
+ }
+ break;
+ }
+
+ case JSOp::PushLexicalEnv:
+ case JSOp::FreshenLexicalEnv:
+ case JSOp::RecreateLexicalEnv: {
+ Rooted<LexicalScope*> scope(cx_,
+ &loc.getScope(script_)->as<LexicalScope>());
+
+ auto* templateObj =
+ BlockLexicalEnvironmentObject::createTemplateObject(cx_, scope);
+ if (!templateObj) {
+ return abort(AbortReason::Alloc);
+ }
+ MOZ_ASSERT(templateObj->isTenured());
+
+ if (!AddOpSnapshot<WarpLexicalEnvironment>(alloc_, opSnapshots, offset,
+ templateObj)) {
+ return abort(AbortReason::Alloc);
+ }
+ break;
+ }
+
+ case JSOp::PushClassBodyEnv: {
+ Rooted<ClassBodyScope*> scope(
+ cx_, &loc.getScope(script_)->as<ClassBodyScope>());
+
+ auto* templateObj =
+ ClassBodyLexicalEnvironmentObject::createTemplateObject(cx_, scope);
+ if (!templateObj) {
+ return abort(AbortReason::Alloc);
+ }
+ MOZ_ASSERT(templateObj->isTenured());
+
+ if (!AddOpSnapshot<WarpClassBodyEnvironment>(alloc_, opSnapshots,
+ offset, templateObj)) {
+ return abort(AbortReason::Alloc);
+ }
+ break;
+ }
+
+ case JSOp::GetName:
+ case JSOp::GetGName:
+ case JSOp::GetProp:
+ case JSOp::GetElem:
+ case JSOp::SetProp:
+ case JSOp::StrictSetProp:
+ case JSOp::Call:
+ case JSOp::CallContent:
+ case JSOp::CallIgnoresRv:
+ case JSOp::CallIter:
+ case JSOp::CallContentIter:
+ case JSOp::New:
+ case JSOp::NewContent:
+ case JSOp::SuperCall:
+ case JSOp::SpreadCall:
+ case JSOp::SpreadNew:
+ case JSOp::SpreadSuperCall:
+ case JSOp::ToNumeric:
+ case JSOp::Pos:
+ case JSOp::Inc:
+ case JSOp::Dec:
+ case JSOp::Neg:
+ case JSOp::BitNot:
+ case JSOp::Iter:
+ case JSOp::Eq:
+ case JSOp::Ne:
+ case JSOp::Lt:
+ case JSOp::Le:
+ case JSOp::Gt:
+ case JSOp::Ge:
+ case JSOp::StrictEq:
+ case JSOp::StrictNe:
+ case JSOp::BindName:
+ case JSOp::Add:
+ case JSOp::Sub:
+ case JSOp::Mul:
+ case JSOp::Div:
+ case JSOp::Mod:
+ case JSOp::Pow:
+ case JSOp::BitAnd:
+ case JSOp::BitOr:
+ case JSOp::BitXor:
+ case JSOp::Lsh:
+ case JSOp::Rsh:
+ case JSOp::Ursh:
+ case JSOp::In:
+ case JSOp::HasOwn:
+ case JSOp::CheckPrivateField:
+ case JSOp::Instanceof:
+ case JSOp::GetPropSuper:
+ case JSOp::InitProp:
+ case JSOp::InitLockedProp:
+ case JSOp::InitHiddenProp:
+ case JSOp::InitElem:
+ case JSOp::InitHiddenElem:
+ case JSOp::InitLockedElem:
+ case JSOp::InitElemInc:
+ case JSOp::SetName:
+ case JSOp::StrictSetName:
+ case JSOp::SetGName:
+ case JSOp::StrictSetGName:
+ case JSOp::InitGLexical:
+ case JSOp::SetElem:
+ case JSOp::StrictSetElem:
+ case JSOp::ToPropertyKey:
+ case JSOp::OptimizeSpreadCall:
+ case JSOp::Typeof:
+ case JSOp::TypeofExpr:
+ case JSOp::NewObject:
+ case JSOp::NewInit:
+ case JSOp::NewArray:
+ case JSOp::JumpIfFalse:
+ case JSOp::JumpIfTrue:
+ case JSOp::And:
+ case JSOp::Or:
+ case JSOp::Not:
+ case JSOp::CloseIter:
+ MOZ_TRY(maybeInlineIC(opSnapshots, loc));
+ break;
+
+ case JSOp::Nop:
+ case JSOp::NopDestructuring:
+ case JSOp::TryDestructuring:
+ case JSOp::Lineno:
+ case JSOp::DebugLeaveLexicalEnv:
+ case JSOp::Undefined:
+ case JSOp::Void:
+ case JSOp::Null:
+ case JSOp::Hole:
+ case JSOp::Uninitialized:
+ case JSOp::IsConstructing:
+ case JSOp::False:
+ case JSOp::True:
+ case JSOp::Zero:
+ case JSOp::One:
+ case JSOp::Int8:
+ case JSOp::Uint16:
+ case JSOp::Uint24:
+ case JSOp::Int32:
+ case JSOp::Double:
+ case JSOp::BigInt:
+ case JSOp::String:
+ case JSOp::Symbol:
+ case JSOp::Pop:
+ case JSOp::PopN:
+ case JSOp::Dup:
+ case JSOp::Dup2:
+ case JSOp::DupAt:
+ case JSOp::Swap:
+ case JSOp::Pick:
+ case JSOp::Unpick:
+ case JSOp::GetLocal:
+ case JSOp::SetLocal:
+ case JSOp::InitLexical:
+ case JSOp::GetArg:
+ case JSOp::SetArg:
+ case JSOp::JumpTarget:
+ case JSOp::LoopHead:
+ case JSOp::Case:
+ case JSOp::Default:
+ case JSOp::Coalesce:
+ case JSOp::Goto:
+ case JSOp::DebugCheckSelfHosted:
+ case JSOp::DynamicImport:
+ case JSOp::ToString:
+ case JSOp::GlobalOrEvalDeclInstantiation:
+ case JSOp::BindVar:
+ case JSOp::MutateProto:
+ case JSOp::Callee:
+ case JSOp::ToAsyncIter:
+ case JSOp::ObjWithProto:
+ case JSOp::GetAliasedVar:
+ case JSOp::SetAliasedVar:
+ case JSOp::InitAliasedLexical:
+ case JSOp::EnvCallee:
+ case JSOp::MoreIter:
+ case JSOp::EndIter:
+ case JSOp::IsNoIter:
+ case JSOp::IsNullOrUndefined:
+ case JSOp::DelProp:
+ case JSOp::StrictDelProp:
+ case JSOp::DelElem:
+ case JSOp::StrictDelElem:
+ case JSOp::SetFunName:
+ case JSOp::PopLexicalEnv:
+ case JSOp::ImplicitThis:
+ case JSOp::CheckClassHeritage:
+ case JSOp::CheckThis:
+ case JSOp::CheckThisReinit:
+ case JSOp::Generator:
+ case JSOp::AfterYield:
+ case JSOp::FinalYieldRval:
+ case JSOp::AsyncResolve:
+ case JSOp::CheckResumeKind:
+ case JSOp::CanSkipAwait:
+ case JSOp::MaybeExtractAwaitValue:
+ case JSOp::AsyncAwait:
+ case JSOp::Await:
+ case JSOp::CheckReturn:
+ case JSOp::CheckLexical:
+ case JSOp::CheckAliasedLexical:
+ case JSOp::InitHomeObject:
+ case JSOp::SuperBase:
+ case JSOp::SuperFun:
+ case JSOp::InitElemArray:
+ case JSOp::InitPropGetter:
+ case JSOp::InitPropSetter:
+ case JSOp::InitHiddenPropGetter:
+ case JSOp::InitHiddenPropSetter:
+ case JSOp::InitElemGetter:
+ case JSOp::InitElemSetter:
+ case JSOp::InitHiddenElemGetter:
+ case JSOp::InitHiddenElemSetter:
+ case JSOp::NewTarget:
+ case JSOp::Object:
+ case JSOp::CallSiteObj:
+ case JSOp::CheckIsObj:
+ case JSOp::CheckObjCoercible:
+ case JSOp::FunWithProto:
+ case JSOp::Debugger:
+ case JSOp::TableSwitch:
+ case JSOp::Exception:
+ case JSOp::Throw:
+ case JSOp::ThrowSetConst:
+ case JSOp::SetRval:
+ case JSOp::GetRval:
+ case JSOp::Return:
+ case JSOp::RetRval:
+ case JSOp::InitialYield:
+ case JSOp::Yield:
+ case JSOp::ResumeKind:
+ case JSOp::ThrowMsg:
+ case JSOp::Try:
+ case JSOp::Finally:
+ case JSOp::NewPrivateName:
+ // Supported by WarpBuilder. Nothing to do.
+ break;
+
+ // Unsupported ops. Don't use a 'default' here, we want to trigger a
+ // compiler warning when adding a new JSOp.
+#define DEF_CASE(OP) case JSOp::OP:
+ WARP_UNSUPPORTED_OPCODE_LIST(DEF_CASE)
+#undef DEF_CASE
+#ifdef DEBUG
+ return abort(AbortReason::Disable, "Unsupported opcode: %s",
+ CodeName(op));
+#else
+ return abort(AbortReason::Disable, "Unsupported opcode: %u",
+ uint8_t(op));
+#endif
+ }
+ }
+
+ auto* scriptSnapshot = new (alloc_.fallible()) WarpScriptSnapshot(
+ script_, environment, std::move(opSnapshots), moduleObject);
+ if (!scriptSnapshot) {
+ return abort(AbortReason::Alloc);
+ }
+
+ autoClearOpSnapshots.release();
+ return scriptSnapshot;
+}
+
+static void LineNumberAndColumn(HandleScript script, BytecodeLocation loc,
+ unsigned* line, unsigned* column) {
+#ifdef DEBUG
+ *line = PCToLineNumber(script, loc.toRawBytecode(), column);
+#else
+ *line = script->lineno();
+ *column = script->column();
+#endif
+}
+
+AbortReasonOr<Ok> WarpScriptOracle::maybeInlineIC(WarpOpSnapshotList& snapshots,
+ BytecodeLocation loc) {
+ // Do one of the following:
+ //
+ // * If the Baseline IC has a single ICStub we can inline, add a WarpCacheIR
+ // snapshot to transpile it to MIR.
+ //
+ // * If that single ICStub is a call IC with a known target, instead add a
+ // WarpInline snapshot to transpile the guards to MIR and inline the target.
+ //
+ // * If the Baseline IC is cold (never executed), add a WarpBailout snapshot
+ // so that we can collect information in Baseline.
+ //
+ // * Else, don't add a snapshot and rely on WarpBuilder adding an Ion IC.
+
+ MOZ_ASSERT(loc.opHasIC());
+
+ // Don't create snapshots when testing ICs.
+ if (JitOptions.forceInlineCaches) {
+ return Ok();
+ }
+
+ ICFallbackStub* fallbackStub;
+ const ICEntry& entry = getICEntryAndFallback(loc, &fallbackStub);
+ ICStub* firstStub = entry.firstStub();
+
+ uint32_t offset = loc.bytecodeToOffset(script_);
+
+ // Clear the used-by-transpiler flag on the IC. It can still be set from a
+ // previous compilation because we don't clear the flag on every IC when
+ // invalidating.
+ fallbackStub->clearUsedByTranspiler();
+
+ if (firstStub == fallbackStub) {
+ [[maybe_unused]] unsigned line, column;
+ LineNumberAndColumn(script_, loc, &line, &column);
+
+ // No optimized stubs.
+ JitSpew(JitSpew_WarpTranspiler,
+ "fallback stub (entered-count: %" PRIu32
+ ") for JSOp::%s @ %s:%u:%u",
+ fallbackStub->enteredCount(), CodeName(loc.getOp()),
+ script_->filename(), line, column);
+
+ // If the fallback stub was used but there's no optimized stub, use an IC.
+ if (fallbackStub->enteredCount() != 0) {
+ return Ok();
+ }
+
+ // Cold IC. Bailout to collect information.
+ if (!AddOpSnapshot<WarpBailout>(alloc_, snapshots, offset)) {
+ return abort(AbortReason::Alloc);
+ }
+ return Ok();
+ }
+
+ ICCacheIRStub* stub = firstStub->toCacheIRStub();
+
+ // Don't transpile if there are other stubs with entered-count > 0. Counters
+ // are reset when a new stub is attached so this means the stub that was added
+ // most recently didn't handle all cases.
+ // If this code is changed, ICScript::hash may also need changing.
+ bool firstStubHandlesAllCases = true;
+ for (ICStub* next = stub->next(); next; next = next->maybeNext()) {
+ if (next->enteredCount() != 0) {
+ firstStubHandlesAllCases = false;
+ break;
+ }
+ }
+
+ if (!firstStubHandlesAllCases) {
+ // In some polymorphic cases, we can generate better code than the
+ // default fallback if we know the observed types of the operands
+ // and their relative frequency.
+ if (ICSupportsPolymorphicTypeData(loc.getOp()) &&
+ fallbackStub->enteredCount() == 0) {
+ bool inlinedPolymorphicTypes = false;
+ MOZ_TRY_VAR(
+ inlinedPolymorphicTypes,
+ maybeInlinePolymorphicTypes(snapshots, loc, stub, fallbackStub));
+ if (inlinedPolymorphicTypes) {
+ return Ok();
+ }
+ }
+
+ [[maybe_unused]] unsigned line, column;
+ LineNumberAndColumn(script_, loc, &line, &column);
+
+ JitSpew(JitSpew_WarpTranspiler,
+ "multiple active stubs for JSOp::%s @ %s:%u:%u",
+ CodeName(loc.getOp()), script_->filename(), line, column);
+ return Ok();
+ }
+
+ const CacheIRStubInfo* stubInfo = stub->stubInfo();
+ const uint8_t* stubData = stub->stubDataStart();
+
+ // Only create a snapshot if all opcodes are supported by the transpiler.
+ CacheIRReader reader(stubInfo);
+ while (reader.more()) {
+ CacheOp op = reader.readOp();
+ CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
+ reader.skip(opInfo.argLength);
+
+ if (!opInfo.transpile) {
+ [[maybe_unused]] unsigned line, column;
+ LineNumberAndColumn(script_, loc, &line, &column);
+
+ MOZ_ASSERT(
+ fallbackStub->trialInliningState() != TrialInliningState::Inlined,
+ "Trial-inlined stub not supported by transpiler");
+
+ // Unsupported CacheIR opcode.
+ JitSpew(JitSpew_WarpTranspiler,
+ "unsupported CacheIR opcode %s for JSOp::%s @ %s:%u:%u",
+ CacheIROpNames[size_t(op)], CodeName(loc.getOp()),
+ script_->filename(), line, column);
+ return Ok();
+ }
+
+ // While on the main thread, ensure code stubs exist for ops that require
+ // them.
+ switch (op) {
+ case CacheOp::CallRegExpMatcherResult:
+ if (!cx_->realm()->jitRealm()->ensureRegExpMatcherStubExists(cx_)) {
+ return abort(AbortReason::Error);
+ }
+ break;
+ case CacheOp::CallRegExpSearcherResult:
+ if (!cx_->realm()->jitRealm()->ensureRegExpSearcherStubExists(cx_)) {
+ return abort(AbortReason::Error);
+ }
+ break;
+ case CacheOp::CallRegExpTesterResult:
+ if (!cx_->realm()->jitRealm()->ensureRegExpTesterStubExists(cx_)) {
+ return abort(AbortReason::Error);
+ }
+ break;
+ default:
+ break;
+ }
+ }
+
+ // Copy the ICStub data to protect against the stub being unlinked or mutated.
+ // We don't need to copy the CacheIRStubInfo: because we store and trace the
+ // stub's JitCode*, the baselineCacheIRStubCodes_ map in JitZone will keep it
+ // alive.
+ uint8_t* stubDataCopy = nullptr;
+ size_t bytesNeeded = stubInfo->stubDataSize();
+ if (bytesNeeded > 0) {
+ stubDataCopy = alloc_.allocateArray<uint8_t>(bytesNeeded);
+ if (!stubDataCopy) {
+ return abort(AbortReason::Alloc);
+ }
+
+ // Note: nursery pointers are handled below so we don't need to trigger any
+ // GC barriers and can do a bitwise copy.
+ std::copy_n(stubData, bytesNeeded, stubDataCopy);
+
+ if (!replaceNurseryAndAllocSitePointers(stub, stubInfo, stubDataCopy)) {
+ return abort(AbortReason::Alloc);
+ }
+ }
+
+ JitCode* jitCode = stub->jitCode();
+
+ if (fallbackStub->trialInliningState() == TrialInliningState::Inlined) {
+ bool inlinedCall;
+ MOZ_TRY_VAR(inlinedCall, maybeInlineCall(snapshots, loc, stub, fallbackStub,
+ stubDataCopy));
+ if (inlinedCall) {
+ return Ok();
+ }
+ }
+
+ if (!AddOpSnapshot<WarpCacheIR>(alloc_, snapshots, offset, jitCode, stubInfo,
+ stubDataCopy)) {
+ return abort(AbortReason::Alloc);
+ }
+
+ fallbackStub->setUsedByTranspiler();
+
+ return Ok();
+}
+
+AbortReasonOr<bool> WarpScriptOracle::maybeInlineCall(
+ WarpOpSnapshotList& snapshots, BytecodeLocation loc, ICCacheIRStub* stub,
+ ICFallbackStub* fallbackStub, uint8_t* stubDataCopy) {
+ Maybe<InlinableOpData> inlineData = FindInlinableOpData(stub, loc);
+ if (inlineData.isNothing() || !inlineData->icScript) {
+ return false;
+ }
+
+ RootedFunction targetFunction(cx_, inlineData->target);
+ if (!TrialInliner::canInline(targetFunction, script_, loc)) {
+ return false;
+ }
+
+ RootedScript targetScript(cx_, targetFunction->nonLazyScript());
+ ICScript* icScript = inlineData->icScript;
+
+ // Add the inlined script to the inline script tree.
+ LifoAlloc* lifoAlloc = alloc_.lifoAlloc();
+ InlineScriptTree* inlineScriptTree = info_->inlineScriptTree()->addCallee(
+ &alloc_, loc.toRawBytecode(), targetScript);
+ if (!inlineScriptTree) {
+ return abort(AbortReason::Alloc);
+ }
+
+ // Create a CompileInfo for the inlined script.
+ jsbytecode* osrPc = nullptr;
+ bool needsArgsObj = targetScript->needsArgsObj();
+ CompileInfo* info = lifoAlloc->new_<CompileInfo>(
+ mirGen_.runtime, targetScript, targetFunction, osrPc, needsArgsObj,
+ inlineScriptTree);
+ if (!info) {
+ return abort(AbortReason::Alloc);
+ }
+
+ // Take a snapshot of the CacheIR.
+ uint32_t offset = loc.bytecodeToOffset(script_);
+ JitCode* jitCode = stub->jitCode();
+ const CacheIRStubInfo* stubInfo = stub->stubInfo();
+ WarpCacheIR* cacheIRSnapshot = new (alloc_.fallible())
+ WarpCacheIR(offset, jitCode, stubInfo, stubDataCopy);
+ if (!cacheIRSnapshot) {
+ return abort(AbortReason::Alloc);
+ }
+
+ // Take a snapshot of the inlined script (which may do more
+ // inlining recursively).
+ WarpScriptOracle scriptOracle(cx_, oracle_, targetScript, info, icScript);
+
+ AbortReasonOr<WarpScriptSnapshot*> maybeScriptSnapshot =
+ scriptOracle.createScriptSnapshot();
+
+ if (maybeScriptSnapshot.isErr()) {
+ JitSpew(JitSpew_WarpTranspiler, "Can't create snapshot for JSOp::%s",
+ CodeName(loc.getOp()));
+
+ switch (maybeScriptSnapshot.unwrapErr()) {
+ case AbortReason::Disable: {
+ // If the target script can't be warp-compiled, mark it as
+ // uninlineable, clean up, and fall through to the non-inlined path.
+ ICEntry* entry = icScript_->icEntryForStub(fallbackStub);
+ fallbackStub->setTrialInliningState(TrialInliningState::Failure);
+ fallbackStub->unlinkStub(cx_->zone(), entry, /*prev=*/nullptr, stub);
+ targetScript->setUninlineable();
+ info_->inlineScriptTree()->removeCallee(inlineScriptTree);
+ icScript_->removeInlinedChild(loc.bytecodeToOffset(script_));
+ return false;
+ }
+ case AbortReason::Error:
+ case AbortReason::Alloc:
+ return Err(maybeScriptSnapshot.unwrapErr());
+ default:
+ MOZ_CRASH("Unexpected abort reason");
+ }
+ }
+
+ WarpScriptSnapshot* scriptSnapshot = maybeScriptSnapshot.unwrap();
+ oracle_->addScriptSnapshot(scriptSnapshot);
+
+ if (!AddOpSnapshot<WarpInlinedCall>(alloc_, snapshots, offset,
+ cacheIRSnapshot, scriptSnapshot, info)) {
+ return abort(AbortReason::Alloc);
+ }
+ fallbackStub->setUsedByTranspiler();
+ return true;
+}
+
+struct TypeFrequency {
+ TypeData typeData_;
+ uint32_t successCount_;
+ TypeFrequency(TypeData typeData, uint32_t successCount)
+ : typeData_(typeData), successCount_(successCount) {}
+
+ // Sort highest frequency first.
+ bool operator<(const TypeFrequency& other) const {
+ return other.successCount_ < successCount_;
+ }
+};
+
+AbortReasonOr<bool> WarpScriptOracle::maybeInlinePolymorphicTypes(
+ WarpOpSnapshotList& snapshots, BytecodeLocation loc,
+ ICCacheIRStub* firstStub, ICFallbackStub* fallbackStub) {
+ MOZ_ASSERT(ICSupportsPolymorphicTypeData(loc.getOp()));
+
+ // We use polymorphic type data if there are multiple active stubs,
+ // all of which have type data available.
+ Vector<TypeFrequency, 6, SystemAllocPolicy> candidates;
+ for (ICStub* stub = firstStub; !stub->isFallback();
+ stub = stub->maybeNext()) {
+ ICCacheIRStub* cacheIRStub = stub->toCacheIRStub();
+ uint32_t successCount =
+ cacheIRStub->enteredCount() - cacheIRStub->next()->enteredCount();
+ if (successCount == 0) {
+ continue;
+ }
+ TypeData types = cacheIRStub->typeData();
+ if (!types.hasData()) {
+ return false;
+ }
+ if (!candidates.append(TypeFrequency(types, successCount))) {
+ return abort(AbortReason::Alloc);
+ }
+ }
+ if (candidates.length() < 2) {
+ return false;
+ }
+
+ // Sort candidates by success frequency.
+ std::sort(candidates.begin(), candidates.end());
+
+ TypeDataList list;
+ for (auto& candidate : candidates) {
+ list.addTypeData(candidate.typeData_);
+ }
+
+ uint32_t offset = loc.bytecodeToOffset(script_);
+ if (!AddOpSnapshot<WarpPolymorphicTypes>(alloc_, snapshots, offset, list)) {
+ return abort(AbortReason::Alloc);
+ }
+
+ return true;
+}
+
+bool WarpScriptOracle::replaceNurseryAndAllocSitePointers(
+ ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo,
+ uint8_t* stubDataCopy) {
+ // If the stub data contains nursery object pointers, replace them with the
+ // corresponding nursery index. See WarpObjectField.
+ //
+ // If the stub data contains allocation site pointers replace them with the
+ // initial heap to use, because the site's state may be mutated by the main
+ // thread while we are compiling.
+ //
+ // Also asserts non-object fields don't contain nursery pointers.
+
+ uint32_t field = 0;
+ size_t offset = 0;
+ while (true) {
+ StubField::Type fieldType = stubInfo->fieldType(field);
+ switch (fieldType) {
+ case StubField::Type::RawInt32:
+ case StubField::Type::RawPointer:
+ case StubField::Type::RawInt64:
+ case StubField::Type::Double:
+ break;
+ case StubField::Type::Shape:
+ static_assert(std::is_convertible_v<Shape*, gc::TenuredCell*>,
+ "Code assumes shapes are tenured");
+ break;
+ case StubField::Type::GetterSetter:
+ static_assert(std::is_convertible_v<GetterSetter*, gc::TenuredCell*>,
+ "Code assumes GetterSetters are tenured");
+ break;
+ case StubField::Type::Symbol:
+ static_assert(std::is_convertible_v<JS::Symbol*, gc::TenuredCell*>,
+ "Code assumes symbols are tenured");
+ break;
+ case StubField::Type::BaseScript:
+ static_assert(std::is_convertible_v<BaseScript*, gc::TenuredCell*>,
+ "Code assumes scripts are tenured");
+ break;
+ case StubField::Type::JSObject: {
+ JSObject* obj =
+ stubInfo->getStubField<ICCacheIRStub, JSObject*>(stub, offset);
+ if (IsInsideNursery(obj)) {
+ uint32_t nurseryIndex;
+ if (!oracle_->registerNurseryObject(obj, &nurseryIndex)) {
+ return false;
+ }
+ uintptr_t oldWord = WarpObjectField::fromObject(obj).rawData();
+ uintptr_t newWord =
+ WarpObjectField::fromNurseryIndex(nurseryIndex).rawData();
+ stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord);
+ }
+ break;
+ }
+ case StubField::Type::String: {
+#ifdef DEBUG
+ JSString* str =
+ stubInfo->getStubField<ICCacheIRStub, JSString*>(stub, offset);
+ MOZ_ASSERT(!IsInsideNursery(str));
+#endif
+ break;
+ }
+ case StubField::Type::Id: {
+#ifdef DEBUG
+ // jsid never contains nursery-allocated things.
+ jsid id = stubInfo->getStubField<ICCacheIRStub, jsid>(stub, offset);
+ MOZ_ASSERT_IF(id.isGCThing(),
+ !IsInsideNursery(id.toGCCellPtr().asCell()));
+#endif
+ break;
+ }
+ case StubField::Type::Value: {
+#ifdef DEBUG
+ Value v =
+ stubInfo->getStubField<ICCacheIRStub, JS::Value>(stub, offset);
+ MOZ_ASSERT_IF(v.isGCThing(), !IsInsideNursery(v.toGCThing()));
+#endif
+ break;
+ }
+ case StubField::Type::AllocSite: {
+ uintptr_t oldWord = stubInfo->getStubRawWord(stub, offset);
+ auto* site = reinterpret_cast<gc::AllocSite*>(oldWord);
+ gc::InitialHeap initialHeap = site->initialHeap();
+ uintptr_t newWord = uintptr_t(initialHeap);
+ stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord);
+ break;
+ }
+ case StubField::Type::Limit:
+ return true; // Done.
+ }
+ field++;
+ offset += StubField::sizeInBytes(fieldType);
+ }
+}
+
+bool WarpOracle::registerNurseryObject(JSObject* obj, uint32_t* nurseryIndex) {
+ MOZ_ASSERT(IsInsideNursery(obj));
+
+ auto p = nurseryObjectsMap_.lookupForAdd(obj);
+ if (p) {
+ *nurseryIndex = p->value();
+ return true;
+ }
+
+ if (!nurseryObjects_.append(obj)) {
+ return false;
+ }
+ *nurseryIndex = nurseryObjects_.length() - 1;
+ return nurseryObjectsMap_.add(p, obj, *nurseryIndex);
+}
diff --git a/js/src/jit/WarpOracle.h b/js/src/jit/WarpOracle.h
new file mode 100644
index 0000000000..970cf325ac
--- /dev/null
+++ b/js/src/jit/WarpOracle.h
@@ -0,0 +1,61 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_WarpOracle_h
+#define jit_WarpOracle_h
+
+#include "jit/JitAllocPolicy.h"
+#include "jit/JitContext.h"
+#include "jit/WarpSnapshot.h"
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+
+// WarpOracle creates a WarpSnapshot data structure that's used by WarpBuilder
+// to generate the MIR graph off-thread.
+class MOZ_STACK_CLASS WarpOracle {
+ JSContext* cx_;
+ MIRGenerator& mirGen_;
+ TempAllocator& alloc_;
+ HandleScript outerScript_;
+ WarpBailoutInfo bailoutInfo_;
+ WarpScriptSnapshotList scriptSnapshots_;
+
+ // List of nursery objects to copy to the snapshot. See WarpObjectField.
+ // The HashMap is used to de-duplicate the Vector. It maps each object to the
+ // corresponding nursery index (index into the Vector).
+ // Note: this stores raw object pointers because WarpOracle can't GC.
+ Vector<JSObject*, 8, SystemAllocPolicy> nurseryObjects_;
+ using NurseryObjectsMap =
+ HashMap<JSObject*, uint32_t, DefaultHasher<JSObject*>, SystemAllocPolicy>;
+ NurseryObjectsMap nurseryObjectsMap_;
+
+ public:
+ WarpOracle(JSContext* cx, MIRGenerator& mirGen, HandleScript outerScript);
+ ~WarpOracle() { scriptSnapshots_.clear(); }
+
+ MIRGenerator& mirGen() { return mirGen_; }
+ WarpBailoutInfo& bailoutInfo() { return bailoutInfo_; }
+
+ [[nodiscard]] bool registerNurseryObject(JSObject* obj,
+ uint32_t* nurseryIndex);
+
+ AbortReasonOr<WarpSnapshot*> createSnapshot();
+
+ mozilla::GenericErrorResult<AbortReason> abort(HandleScript script,
+ AbortReason r);
+ mozilla::GenericErrorResult<AbortReason> abort(HandleScript script,
+ AbortReason r,
+ const char* message, ...);
+ void addScriptSnapshot(WarpScriptSnapshot* scriptSnapshot);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_WarpOracle_h */
diff --git a/js/src/jit/WarpSnapshot.cpp b/js/src/jit/WarpSnapshot.cpp
new file mode 100644
index 0000000000..ceb024f96c
--- /dev/null
+++ b/js/src/jit/WarpSnapshot.cpp
@@ -0,0 +1,402 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/WarpSnapshot.h"
+
+#include "mozilla/DebugOnly.h"
+
+#include <type_traits>
+
+#include "jit/CacheIRCompiler.h"
+#include "jit/CacheIRSpewer.h"
+#include "vm/EnvironmentObject.h"
+#include "vm/GetterSetter.h"
+#include "vm/GlobalObject.h"
+#include "vm/JSContext.h"
+#include "vm/Printer.h"
+
+using namespace js;
+using namespace js::jit;
+
+static_assert(!std::is_polymorphic_v<WarpOpSnapshot>,
+ "WarpOpSnapshot should not have any virtual methods");
+
+WarpSnapshot::WarpSnapshot(JSContext* cx, TempAllocator& alloc,
+ WarpScriptSnapshotList&& scriptSnapshots,
+ const WarpBailoutInfo& bailoutInfo,
+ bool needsFinalWarmUpCount)
+ : scriptSnapshots_(std::move(scriptSnapshots)),
+ globalLexicalEnv_(&cx->global()->lexicalEnvironment()),
+ globalLexicalEnvThis_(globalLexicalEnv_->thisObject()),
+ bailoutInfo_(bailoutInfo),
+ nurseryObjects_(alloc) {
+#ifdef JS_CACHEIR_SPEW
+ needsFinalWarmUpCount_ = needsFinalWarmUpCount;
+#endif
+}
+
+WarpScriptSnapshot::WarpScriptSnapshot(JSScript* script,
+ const WarpEnvironment& env,
+ WarpOpSnapshotList&& opSnapshots,
+ ModuleObject* moduleObject)
+ : script_(script),
+ environment_(env),
+ opSnapshots_(std::move(opSnapshots)),
+ moduleObject_(moduleObject),
+ isArrowFunction_(script->isFunction() && script->function()->isArrow()) {}
+
+#ifdef JS_JITSPEW
+void WarpSnapshot::dump() const {
+ Fprinter out(stderr);
+ dump(out);
+}
+
+void WarpSnapshot::dump(GenericPrinter& out) const {
+ out.printf("WarpSnapshot (0x%p)\n", this);
+ out.printf("------------------------------\n");
+ out.printf("globalLexicalEnv: 0x%p\n", globalLexicalEnv());
+ out.printf("globalLexicalEnvThis: 0x%p\n", globalLexicalEnvThis());
+ out.printf("failedBoundsCheck: %u\n", bailoutInfo().failedBoundsCheck());
+ out.printf("failedLexicalCheck: %u\n", bailoutInfo().failedLexicalCheck());
+ out.printf("\n");
+
+ out.printf("Nursery objects (%u):\n", unsigned(nurseryObjects_.length()));
+ for (size_t i = 0; i < nurseryObjects_.length(); i++) {
+ out.printf(" %u: 0x%p\n", unsigned(i), nurseryObjects_[i]);
+ }
+ out.printf("\n");
+
+ for (auto* scriptSnapshot : scriptSnapshots_) {
+ scriptSnapshot->dump(out);
+ }
+}
+
+void WarpScriptSnapshot::dump(GenericPrinter& out) const {
+ out.printf("WarpScriptSnapshot (0x%p)\n", this);
+ out.printf("------------------------------\n");
+ out.printf("Script: %s:%u:%u (0x%p)\n", script_->filename(),
+ script_->lineno(), script_->column(),
+ static_cast<JSScript*>(script_));
+ out.printf(" moduleObject: 0x%p\n", moduleObject());
+ out.printf(" isArrowFunction: %u\n", isArrowFunction());
+
+ out.printf(" environment: ");
+ environment_.match(
+ [&](const NoEnvironment&) { out.printf("None\n"); },
+ [&](JSObject* obj) { out.printf("Object: 0x%p\n", obj); },
+ [&](const FunctionEnvironment& env) {
+ out.printf(
+ "Function: callobject template 0x%p, named lambda template: 0x%p\n",
+ static_cast<JSObject*>(env.callObjectTemplate),
+ static_cast<JSObject*>(env.namedLambdaTemplate));
+ });
+
+ out.printf("\n");
+ for (const WarpOpSnapshot* snapshot : opSnapshots()) {
+ snapshot->dump(out, script_);
+ out.printf("\n");
+ }
+}
+
+static const char* OpSnapshotKindString(WarpOpSnapshot::Kind kind) {
+ static const char* const names[] = {
+# define NAME(x) # x,
+ WARP_OP_SNAPSHOT_LIST(NAME)
+# undef NAME
+ };
+ return names[unsigned(kind)];
+}
+
+void WarpOpSnapshot::dump(GenericPrinter& out, JSScript* script) const {
+ jsbytecode* pc = script->offsetToPC(offset_);
+ out.printf(" %s (offset %u, JSOp::%s)\n", OpSnapshotKindString(kind_),
+ offset_, CodeName(JSOp(*pc)));
+
+ // Dispatch to dumpData() methods.
+ switch (kind_) {
+# define DUMP(kind) \
+ case Kind::kind: \
+ as<kind>()->dumpData(out); \
+ break;
+ WARP_OP_SNAPSHOT_LIST(DUMP)
+# undef DUMP
+ }
+}
+
+void WarpArguments::dumpData(GenericPrinter& out) const {
+ out.printf(" template: 0x%p\n", templateObj());
+}
+
+void WarpRegExp::dumpData(GenericPrinter& out) const {
+ out.printf(" hasShared: %u\n", hasShared());
+}
+
+void WarpBuiltinObject::dumpData(GenericPrinter& out) const {
+ out.printf(" builtin: 0x%p\n", builtin());
+}
+
+void WarpGetIntrinsic::dumpData(GenericPrinter& out) const {
+ out.printf(" intrinsic: 0x%016" PRIx64 "\n", intrinsic().asRawBits());
+}
+
+void WarpGetImport::dumpData(GenericPrinter& out) const {
+ out.printf(" targetEnv: 0x%p\n", targetEnv());
+ out.printf(" numFixedSlots: %u\n", numFixedSlots());
+ out.printf(" slot: %u\n", slot());
+ out.printf(" needsLexicalCheck: %u\n", needsLexicalCheck());
+}
+
+void WarpRest::dumpData(GenericPrinter& out) const {
+ out.printf(" shape: 0x%p\n", shape());
+}
+
+void WarpBindGName::dumpData(GenericPrinter& out) const {
+ out.printf(" globalEnv: 0x%p\n", globalEnv());
+}
+
+void WarpVarEnvironment::dumpData(GenericPrinter& out) const {
+ out.printf(" template: 0x%p\n", templateObj());
+}
+
+void WarpLexicalEnvironment::dumpData(GenericPrinter& out) const {
+ out.printf(" template: 0x%p\n", templateObj());
+}
+
+void WarpClassBodyEnvironment::dumpData(GenericPrinter& out) const {
+ out.printf(" template: 0x%p\n", templateObj());
+}
+
+void WarpBailout::dumpData(GenericPrinter& out) const {
+ // No fields.
+}
+
+void WarpCacheIR::dumpData(GenericPrinter& out) const {
+ out.printf(" stubCode: 0x%p\n", static_cast<JitCode*>(stubCode_));
+ out.printf(" stubInfo: 0x%p\n", stubInfo_);
+ out.printf(" stubData: 0x%p\n", stubData_);
+# ifdef JS_CACHEIR_SPEW
+ out.printf(" IR:\n");
+ SpewCacheIROps(out, " ", stubInfo_);
+# else
+ out.printf("(CacheIR spew unavailable)\n");
+# endif
+}
+
+void WarpInlinedCall::dumpData(GenericPrinter& out) const {
+ out.printf(" scriptSnapshot: 0x%p\n", scriptSnapshot_);
+ out.printf(" info: 0x%p\n", info_);
+ cacheIRSnapshot_->dumpData(out);
+}
+
+void WarpPolymorphicTypes::dumpData(GenericPrinter& out) const {
+ out.printf(" types:\n");
+ for (auto& typeData : list_) {
+ out.printf(" %s\n", ValTypeToString(typeData.type()));
+ }
+}
+
+#endif // JS_JITSPEW
+
+template <typename T>
+static void TraceWarpGCPtr(JSTracer* trc, const WarpGCPtr<T>& thing,
+ const char* name) {
+ T thingRaw = thing;
+ TraceManuallyBarrieredEdge(trc, &thingRaw, name);
+ MOZ_ASSERT(static_cast<T>(thing) == thingRaw, "Unexpected moving GC!");
+}
+
+void WarpSnapshot::trace(JSTracer* trc) {
+ // Nursery objects can be tenured in parallel with Warp compilation.
+ // Note: don't use TraceWarpGCPtr here as that asserts non-moving.
+ for (size_t i = 0; i < nurseryObjects_.length(); i++) {
+ TraceManuallyBarrieredEdge(trc, &nurseryObjects_[i], "warp-nursery-object");
+ }
+
+ // Other GC things are not in the nursery.
+ if (trc->runtime()->heapState() == JS::HeapState::MinorCollecting) {
+ return;
+ }
+
+ for (auto* script : scriptSnapshots_) {
+ script->trace(trc);
+ }
+ TraceWarpGCPtr(trc, globalLexicalEnv_, "warp-lexical");
+ TraceWarpGCPtr(trc, globalLexicalEnvThis_, "warp-lexicalthis");
+}
+
+void WarpScriptSnapshot::trace(JSTracer* trc) {
+ TraceWarpGCPtr(trc, script_, "warp-script");
+
+ environment_.match(
+ [](const NoEnvironment&) {},
+ [trc](WarpGCPtr<JSObject*>& obj) {
+ TraceWarpGCPtr(trc, obj, "warp-env-object");
+ },
+ [trc](FunctionEnvironment& env) {
+ if (env.callObjectTemplate) {
+ TraceWarpGCPtr(trc, env.callObjectTemplate, "warp-env-callobject");
+ }
+ if (env.namedLambdaTemplate) {
+ TraceWarpGCPtr(trc, env.namedLambdaTemplate, "warp-env-namedlambda");
+ }
+ });
+
+ for (WarpOpSnapshot* snapshot : opSnapshots_) {
+ snapshot->trace(trc);
+ }
+
+ if (moduleObject_) {
+ TraceWarpGCPtr(trc, moduleObject_, "warp-module-obj");
+ }
+}
+
+void WarpOpSnapshot::trace(JSTracer* trc) {
+ // Dispatch to traceData() methods.
+ switch (kind_) {
+#define TRACE(kind) \
+ case Kind::kind: \
+ as<kind>()->traceData(trc); \
+ break;
+ WARP_OP_SNAPSHOT_LIST(TRACE)
+#undef TRACE
+ }
+}
+
+void WarpArguments::traceData(JSTracer* trc) {
+ if (templateObj_) {
+ TraceWarpGCPtr(trc, templateObj_, "warp-args-template");
+ }
+}
+
+void WarpRegExp::traceData(JSTracer* trc) {
+ // No GC pointers.
+}
+
+void WarpBuiltinObject::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, builtin_, "warp-builtin-object");
+}
+
+void WarpGetIntrinsic::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, intrinsic_, "warp-intrinsic");
+}
+
+void WarpGetImport::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, targetEnv_, "warp-import-env");
+}
+
+void WarpRest::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, shape_, "warp-rest-shape");
+}
+
+void WarpBindGName::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, globalEnv_, "warp-bindgname-globalenv");
+}
+
+void WarpVarEnvironment::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, templateObj_, "warp-varenv-template");
+}
+
+void WarpLexicalEnvironment::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, templateObj_, "warp-lexenv-template");
+}
+
+void WarpClassBodyEnvironment::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, templateObj_, "warp-classbodyenv-template");
+}
+
+void WarpBailout::traceData(JSTracer* trc) {
+ // No GC pointers.
+}
+
+void WarpPolymorphicTypes::traceData(JSTracer* trc) {
+ // No GC pointers.
+}
+
+template <typename T>
+static void TraceWarpStubPtr(JSTracer* trc, uintptr_t word, const char* name) {
+ T* ptr = reinterpret_cast<T*>(word);
+ TraceWarpGCPtr(trc, WarpGCPtr<T*>(ptr), name);
+}
+
+void WarpCacheIR::traceData(JSTracer* trc) {
+ TraceWarpGCPtr(trc, stubCode_, "warp-stub-code");
+ if (stubData_) {
+ uint32_t field = 0;
+ size_t offset = 0;
+ while (true) {
+ StubField::Type fieldType = stubInfo_->fieldType(field);
+ switch (fieldType) {
+ case StubField::Type::RawInt32:
+ case StubField::Type::RawPointer:
+ case StubField::Type::RawInt64:
+ case StubField::Type::Double:
+ break;
+ case StubField::Type::Shape: {
+ uintptr_t word = stubInfo_->getStubRawWord(stubData_, offset);
+ TraceWarpStubPtr<Shape>(trc, word, "warp-cacheir-shape");
+ break;
+ }
+ case StubField::Type::GetterSetter: {
+ uintptr_t word = stubInfo_->getStubRawWord(stubData_, offset);
+ TraceWarpStubPtr<GetterSetter>(trc, word,
+ "warp-cacheir-getter-setter");
+ break;
+ }
+ case StubField::Type::JSObject: {
+ uintptr_t word = stubInfo_->getStubRawWord(stubData_, offset);
+ WarpObjectField field = WarpObjectField::fromData(word);
+ if (!field.isNurseryIndex()) {
+ TraceWarpStubPtr<JSObject>(trc, word, "warp-cacheir-object");
+ }
+ break;
+ }
+ case StubField::Type::Symbol: {
+ uintptr_t word = stubInfo_->getStubRawWord(stubData_, offset);
+ TraceWarpStubPtr<JS::Symbol>(trc, word, "warp-cacheir-symbol");
+ break;
+ }
+ case StubField::Type::String: {
+ uintptr_t word = stubInfo_->getStubRawWord(stubData_, offset);
+ TraceWarpStubPtr<JSString>(trc, word, "warp-cacheir-string");
+ break;
+ }
+ case StubField::Type::BaseScript: {
+ uintptr_t word = stubInfo_->getStubRawWord(stubData_, offset);
+ TraceWarpStubPtr<BaseScript>(trc, word, "warp-cacheir-script");
+ break;
+ }
+ case StubField::Type::Id: {
+ uintptr_t word = stubInfo_->getStubRawWord(stubData_, offset);
+ jsid id = jsid::fromRawBits(word);
+ TraceWarpGCPtr(trc, WarpGCPtr<jsid>(id), "warp-cacheir-jsid");
+ break;
+ }
+ case StubField::Type::Value: {
+ uint64_t data = stubInfo_->getStubRawInt64(stubData_, offset);
+ Value val = Value::fromRawBits(data);
+ TraceWarpGCPtr(trc, WarpGCPtr<Value>(val), "warp-cacheir-value");
+ break;
+ }
+ case StubField::Type::AllocSite: {
+ mozilla::DebugOnly<uintptr_t> word =
+ stubInfo_->getStubRawWord(stubData_, offset);
+ MOZ_ASSERT(word == uintptr_t(gc::DefaultHeap) ||
+ word == uintptr_t(gc::TenuredHeap));
+ break;
+ }
+ case StubField::Type::Limit:
+ return; // Done.
+ }
+ field++;
+ offset += StubField::sizeInBytes(fieldType);
+ }
+ }
+}
+
+void WarpInlinedCall::traceData(JSTracer* trc) {
+ // Note: scriptSnapshot_ is traced through WarpSnapshot.
+ cacheIRSnapshot_->trace(trc);
+}
diff --git a/js/src/jit/WarpSnapshot.h b/js/src/jit/WarpSnapshot.h
new file mode 100644
index 0000000000..f78a20ab3f
--- /dev/null
+++ b/js/src/jit/WarpSnapshot.h
@@ -0,0 +1,624 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_WarpSnapshot_h
+#define jit_WarpSnapshot_h
+
+#include "mozilla/LinkedList.h"
+#include "mozilla/Variant.h"
+
+#include "builtin/ModuleObject.h"
+#include "gc/Policy.h"
+#include "jit/JitAllocPolicy.h"
+#include "jit/JitContext.h"
+#include "jit/TypeData.h"
+#include "vm/EnvironmentObject.h"
+#include "vm/FunctionFlags.h" // js::FunctionFlags
+
+namespace js {
+
+class ArgumentsObject;
+class CallObject;
+class GlobalLexicalEnvironmentObject;
+class LexicalEnvironmentObject;
+class ModuleEnvironmentObject;
+class NamedLambdaObject;
+
+namespace jit {
+
+class CacheIRStubInfo;
+class CompileInfo;
+class WarpScriptSnapshot;
+
+#define WARP_OP_SNAPSHOT_LIST(_) \
+ _(WarpArguments) \
+ _(WarpRegExp) \
+ _(WarpBuiltinObject) \
+ _(WarpGetIntrinsic) \
+ _(WarpGetImport) \
+ _(WarpRest) \
+ _(WarpBindGName) \
+ _(WarpVarEnvironment) \
+ _(WarpLexicalEnvironment) \
+ _(WarpClassBodyEnvironment) \
+ _(WarpBailout) \
+ _(WarpCacheIR) \
+ _(WarpInlinedCall) \
+ _(WarpPolymorphicTypes)
+
+// Wrapper for GC things stored in WarpSnapshot. Asserts the GC pointer is not
+// nursery-allocated. These pointers must be traced using TraceWarpGCPtr.
+template <typename T>
+class WarpGCPtr {
+ // Note: no pre-barrier is needed because this is a constant. No post-barrier
+ // is needed because the value is always tenured.
+ const T ptr_;
+
+ public:
+ explicit WarpGCPtr(const T& ptr) : ptr_(ptr) {
+ MOZ_ASSERT(JS::GCPolicy<T>::isTenured(ptr),
+ "WarpSnapshot pointers must be tenured");
+ }
+ WarpGCPtr(const WarpGCPtr<T>& other) = default;
+
+ operator T() const { return ptr_; }
+ T operator->() const { return ptr_; }
+
+ private:
+ WarpGCPtr() = delete;
+ void operator=(WarpGCPtr<T>& other) = delete;
+};
+
+// WarpOpSnapshot is the base class for data attached to a single bytecode op by
+// WarpOracle. This is typically data that WarpBuilder can't read off-thread
+// without racing.
+class WarpOpSnapshot : public TempObject,
+ public mozilla::LinkedListElement<WarpOpSnapshot> {
+ public:
+ enum class Kind : uint16_t {
+#define DEF_KIND(KIND) KIND,
+ WARP_OP_SNAPSHOT_LIST(DEF_KIND)
+#undef DEF_KIND
+ };
+
+ private:
+ // Bytecode offset.
+ uint32_t offset_ = 0;
+
+ Kind kind_;
+
+ protected:
+ WarpOpSnapshot(Kind kind, uint32_t offset) : offset_(offset), kind_(kind) {}
+
+ public:
+ uint32_t offset() const { return offset_; }
+ Kind kind() const { return kind_; }
+
+ template <typename T>
+ const T* as() const {
+ MOZ_ASSERT(kind_ == T::ThisKind);
+ return static_cast<const T*>(this);
+ }
+
+ template <typename T>
+ T* as() {
+ MOZ_ASSERT(kind_ == T::ThisKind);
+ return static_cast<T*>(this);
+ }
+
+ void trace(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dump(GenericPrinter& out, JSScript* script) const;
+#endif
+};
+
+using WarpOpSnapshotList = mozilla::LinkedList<WarpOpSnapshot>;
+
+// Template object for JSOp::Arguments.
+class WarpArguments : public WarpOpSnapshot {
+ // Note: this can be nullptr if the realm has no template object yet.
+ WarpGCPtr<ArgumentsObject*> templateObj_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpArguments;
+
+ WarpArguments(uint32_t offset, ArgumentsObject* templateObj)
+ : WarpOpSnapshot(ThisKind, offset), templateObj_(templateObj) {}
+ ArgumentsObject* templateObj() const { return templateObj_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// The "has RegExpShared" state for JSOp::RegExp's template object.
+class WarpRegExp : public WarpOpSnapshot {
+ bool hasShared_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpRegExp;
+
+ WarpRegExp(uint32_t offset, bool hasShared)
+ : WarpOpSnapshot(ThisKind, offset), hasShared_(hasShared) {}
+ bool hasShared() const { return hasShared_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// The object for JSOp::BuiltinObject if it exists at compile-time.
+class WarpBuiltinObject : public WarpOpSnapshot {
+ WarpGCPtr<JSObject*> builtin_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpBuiltinObject;
+
+ WarpBuiltinObject(uint32_t offset, JSObject* builtin)
+ : WarpOpSnapshot(ThisKind, offset), builtin_(builtin) {
+ MOZ_ASSERT(builtin);
+ }
+ JSObject* builtin() const { return builtin_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// The intrinsic for JSOp::GetIntrinsic if it exists at compile-time.
+class WarpGetIntrinsic : public WarpOpSnapshot {
+ WarpGCPtr<Value> intrinsic_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpGetIntrinsic;
+
+ WarpGetIntrinsic(uint32_t offset, const Value& intrinsic)
+ : WarpOpSnapshot(ThisKind, offset), intrinsic_(intrinsic) {}
+ Value intrinsic() const { return intrinsic_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Target module environment and slot information for JSOp::GetImport.
+class WarpGetImport : public WarpOpSnapshot {
+ WarpGCPtr<ModuleEnvironmentObject*> targetEnv_;
+ uint32_t numFixedSlots_;
+ uint32_t slot_;
+ bool needsLexicalCheck_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpGetImport;
+
+ WarpGetImport(uint32_t offset, ModuleEnvironmentObject* targetEnv,
+ uint32_t numFixedSlots, uint32_t slot, bool needsLexicalCheck)
+ : WarpOpSnapshot(ThisKind, offset),
+ targetEnv_(targetEnv),
+ numFixedSlots_(numFixedSlots),
+ slot_(slot),
+ needsLexicalCheck_(needsLexicalCheck) {}
+ ModuleEnvironmentObject* targetEnv() const { return targetEnv_; }
+ uint32_t numFixedSlots() const { return numFixedSlots_; }
+ uint32_t slot() const { return slot_; }
+ bool needsLexicalCheck() const { return needsLexicalCheck_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Informs WarpBuilder that an IC site is cold and execution should bail out.
+class WarpBailout : public WarpOpSnapshot {
+ public:
+ static constexpr Kind ThisKind = Kind::WarpBailout;
+
+ explicit WarpBailout(uint32_t offset) : WarpOpSnapshot(ThisKind, offset) {}
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Information from a Baseline IC stub.
+class WarpCacheIR : public WarpOpSnapshot {
+ // Baseline stub code. Stored here to keep the CacheIRStubInfo alive.
+ WarpGCPtr<JitCode*> stubCode_;
+ const CacheIRStubInfo* stubInfo_;
+
+ // Copied Baseline stub data. Allocated in the same LifoAlloc.
+ const uint8_t* stubData_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpCacheIR;
+
+ WarpCacheIR(uint32_t offset, JitCode* stubCode,
+ const CacheIRStubInfo* stubInfo, const uint8_t* stubData)
+ : WarpOpSnapshot(ThisKind, offset),
+ stubCode_(stubCode),
+ stubInfo_(stubInfo),
+ stubData_(stubData) {}
+
+ const CacheIRStubInfo* stubInfo() const { return stubInfo_; }
+ const uint8_t* stubData() const { return stubData_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// [SMDOC] Warp Nursery Object support
+//
+// CacheIR stub data can contain nursery allocated objects. This can happen for
+// example for GuardSpecificObject/GuardSpecificFunction or GuardProto.
+//
+// To support nursery GCs in parallel with off-thread compilation, we use the
+// following mechanism:
+//
+// * When WarpOracle copies stub data, it builds a Vector of nursery objects.
+// The nursery object pointers in the stub data are replaced with the
+// corresponding index into this Vector.
+// See WarpScriptOracle::replaceNurseryPointers.
+//
+// * The Vector is copied to the snapshot and, at the end of compilation, to
+// the IonScript. The Vector is only accessed on the main thread.
+//
+// * The MIR backend never accesses the raw JSObject*. Instead, it uses
+// MNurseryObject which will load the object at runtime from the IonScript.
+//
+// WarpObjectField is a helper class to encode/decode a stub data field that
+// either stores an object or a nursery index.
+class WarpObjectField {
+ // This is a nursery index if the low bit is set. Else it's a JSObject*.
+ static constexpr uintptr_t NurseryIndexTag = 0x1;
+ static constexpr uintptr_t NurseryIndexShift = 1;
+
+ uintptr_t data_;
+
+ explicit WarpObjectField(uintptr_t data) : data_(data) {}
+
+ public:
+ static WarpObjectField fromData(uintptr_t data) {
+ return WarpObjectField(data);
+ }
+ static WarpObjectField fromObject(JSObject* obj) {
+ return WarpObjectField(uintptr_t(obj));
+ }
+ static WarpObjectField fromNurseryIndex(uint32_t index) {
+ uintptr_t data = (uintptr_t(index) << NurseryIndexShift) | NurseryIndexTag;
+ return WarpObjectField(data);
+ }
+
+ uintptr_t rawData() const { return data_; }
+
+ bool isNurseryIndex() const { return (data_ & NurseryIndexTag) != 0; }
+
+ uint32_t toNurseryIndex() const {
+ MOZ_ASSERT(isNurseryIndex());
+ return data_ >> NurseryIndexShift;
+ }
+
+ JSObject* toObject() const {
+ MOZ_ASSERT(!isNurseryIndex());
+ return reinterpret_cast<JSObject*>(data_);
+ }
+};
+
+// Information for inlining a scripted call IC.
+class WarpInlinedCall : public WarpOpSnapshot {
+ // Used for generating the correct guards.
+ WarpCacheIR* cacheIRSnapshot_;
+
+ // Used for generating the inlined code.
+ WarpScriptSnapshot* scriptSnapshot_;
+ CompileInfo* info_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpInlinedCall;
+
+ WarpInlinedCall(uint32_t offset, WarpCacheIR* cacheIRSnapshot,
+ WarpScriptSnapshot* scriptSnapshot, CompileInfo* info)
+ : WarpOpSnapshot(ThisKind, offset),
+ cacheIRSnapshot_(cacheIRSnapshot),
+ scriptSnapshot_(scriptSnapshot),
+ info_(info) {}
+
+ WarpCacheIR* cacheIRSnapshot() const { return cacheIRSnapshot_; }
+ WarpScriptSnapshot* scriptSnapshot() const { return scriptSnapshot_; }
+ CompileInfo* info() const { return info_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Information for inlining an ordered set of types
+class WarpPolymorphicTypes : public WarpOpSnapshot {
+ TypeDataList list_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpPolymorphicTypes;
+
+ WarpPolymorphicTypes(uint32_t offset, TypeDataList list)
+ : WarpOpSnapshot(ThisKind, offset), list_(list) {}
+
+ const TypeDataList& list() const { return list_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Shape for JSOp::Rest.
+class WarpRest : public WarpOpSnapshot {
+ WarpGCPtr<Shape*> shape_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpRest;
+
+ WarpRest(uint32_t offset, Shape* shape)
+ : WarpOpSnapshot(ThisKind, offset), shape_(shape) {}
+
+ Shape* shape() const { return shape_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Global environment for BindGName
+class WarpBindGName : public WarpOpSnapshot {
+ WarpGCPtr<JSObject*> globalEnv_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpBindGName;
+
+ WarpBindGName(uint32_t offset, JSObject* globalEnv)
+ : WarpOpSnapshot(ThisKind, offset), globalEnv_(globalEnv) {}
+
+ JSObject* globalEnv() const { return globalEnv_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Block environment for PushVarEnv
+class WarpVarEnvironment : public WarpOpSnapshot {
+ WarpGCPtr<VarEnvironmentObject*> templateObj_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpVarEnvironment;
+
+ WarpVarEnvironment(uint32_t offset, VarEnvironmentObject* templateObj)
+ : WarpOpSnapshot(ThisKind, offset), templateObj_(templateObj) {}
+
+ VarEnvironmentObject* templateObj() const { return templateObj_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Block environment for PushLexicalEnv, FreshenLexicalEnv, RecreateLexicalEnv
+class WarpLexicalEnvironment : public WarpOpSnapshot {
+ WarpGCPtr<BlockLexicalEnvironmentObject*> templateObj_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpLexicalEnvironment;
+
+ WarpLexicalEnvironment(uint32_t offset,
+ BlockLexicalEnvironmentObject* templateObj)
+ : WarpOpSnapshot(ThisKind, offset), templateObj_(templateObj) {}
+
+ BlockLexicalEnvironmentObject* templateObj() const { return templateObj_; }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+// Class body lexical environment for PushClassBodyEnv
+class WarpClassBodyEnvironment : public WarpOpSnapshot {
+ WarpGCPtr<ClassBodyLexicalEnvironmentObject*> templateObj_;
+
+ public:
+ static constexpr Kind ThisKind = Kind::WarpClassBodyEnvironment;
+
+ WarpClassBodyEnvironment(uint32_t offset,
+ ClassBodyLexicalEnvironmentObject* templateObj)
+ : WarpOpSnapshot(ThisKind, offset), templateObj_(templateObj) {}
+
+ ClassBodyLexicalEnvironmentObject* templateObj() const {
+ return templateObj_;
+ }
+
+ void traceData(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dumpData(GenericPrinter& out) const;
+#endif
+};
+
+struct NoEnvironment {};
+using ConstantObjectEnvironment = WarpGCPtr<JSObject*>;
+struct FunctionEnvironment {
+ WarpGCPtr<CallObject*> callObjectTemplate;
+ WarpGCPtr<NamedLambdaObject*> namedLambdaTemplate;
+
+ public:
+ FunctionEnvironment(CallObject* callObjectTemplate,
+ NamedLambdaObject* namedLambdaTemplate)
+ : callObjectTemplate(callObjectTemplate),
+ namedLambdaTemplate(namedLambdaTemplate) {}
+};
+
+// Snapshot data for the environment object(s) created in the script's prologue.
+//
+// One of:
+//
+// * NoEnvironment: No environment object should be set. Leave the slot
+// initialized to |undefined|.
+//
+// * ConstantObjectEnvironment: Use this JSObject* as environment object.
+//
+// * FunctionEnvironment: Use the callee's environment chain. Optionally
+// allocate a new NamedLambdaObject and/or CallObject based on
+// namedLambdaTemplate and callObjectTemplate.
+using WarpEnvironment =
+ mozilla::Variant<NoEnvironment, ConstantObjectEnvironment,
+ FunctionEnvironment>;
+
+// Snapshot data for a single JSScript.
+class WarpScriptSnapshot
+ : public TempObject,
+ public mozilla::LinkedListElement<WarpScriptSnapshot> {
+ WarpGCPtr<JSScript*> script_;
+ WarpEnvironment environment_;
+ WarpOpSnapshotList opSnapshots_;
+
+ // If the script has a JSOp::ImportMeta op, this is the module to bake in.
+ WarpGCPtr<ModuleObject*> moduleObject_;
+
+ // Whether this script is for an arrow function.
+ bool isArrowFunction_;
+
+ public:
+ WarpScriptSnapshot(JSScript* script, const WarpEnvironment& env,
+ WarpOpSnapshotList&& opSnapshots,
+ ModuleObject* moduleObject);
+
+ JSScript* script() const { return script_; }
+ const WarpEnvironment& environment() const { return environment_; }
+ const WarpOpSnapshotList& opSnapshots() const { return opSnapshots_; }
+ ModuleObject* moduleObject() const { return moduleObject_; }
+
+ bool isArrowFunction() const { return isArrowFunction_; }
+
+ void trace(JSTracer* trc);
+
+#ifdef JS_JITSPEW
+ void dump(GenericPrinter& out) const;
+#endif
+};
+
+// Captures information from previous bailouts to prevent bailout/recompile
+// loops.
+class WarpBailoutInfo {
+ // True if any script in the compilation has the failedBoundsCheck flag. In
+ // this case mark bounds checks as non-movable to prevent hoisting them in
+ // TryEliminateBoundsCheck.
+ bool failedBoundsCheck_ = false;
+
+ // True if any script in the compilation has the failedLexicalCheck flag. In
+ // this case mark lexical checks as non-movable.
+ bool failedLexicalCheck_ = false;
+
+ public:
+ bool failedBoundsCheck() const { return failedBoundsCheck_; }
+ void setFailedBoundsCheck() { failedBoundsCheck_ = true; }
+
+ bool failedLexicalCheck() const { return failedLexicalCheck_; }
+ void setFailedLexicalCheck() { failedLexicalCheck_ = true; }
+};
+
+using WarpScriptSnapshotList = mozilla::LinkedList<WarpScriptSnapshot>;
+
+// Data allocated by WarpOracle on the main thread that's used off-thread by
+// WarpBuilder to build the MIR graph.
+class WarpSnapshot : public TempObject {
+ // The scripts being compiled.
+ WarpScriptSnapshotList scriptSnapshots_;
+
+ // The global lexical environment and its thisObject(). We don't inline
+ // cross-realm calls so this can be stored once per snapshot.
+ WarpGCPtr<GlobalLexicalEnvironmentObject*> globalLexicalEnv_;
+ WarpGCPtr<JSObject*> globalLexicalEnvThis_;
+
+ const WarpBailoutInfo bailoutInfo_;
+
+ // List of (originally) nursery-allocated objects. Must only be accessed on
+ // the main thread. See also WarpObjectField.
+ using NurseryObjectVector = Vector<JSObject*, 0, JitAllocPolicy>;
+ NurseryObjectVector nurseryObjects_;
+
+#ifdef JS_CACHEIR_SPEW
+ bool needsFinalWarmUpCount_ = false;
+#endif
+
+#ifdef DEBUG
+ // A hash of the stub pointers and entry counts for each of the ICs
+ // in this snapshot.
+ mozilla::HashNumber icHash_ = 0;
+#endif
+
+ public:
+ explicit WarpSnapshot(JSContext* cx, TempAllocator& alloc,
+ WarpScriptSnapshotList&& scriptSnapshots,
+ const WarpBailoutInfo& bailoutInfo,
+ bool recordWarmUpCount);
+
+ WarpScriptSnapshot* rootScript() { return scriptSnapshots_.getFirst(); }
+ const WarpScriptSnapshotList& scripts() const { return scriptSnapshots_; }
+
+ GlobalLexicalEnvironmentObject* globalLexicalEnv() const {
+ return globalLexicalEnv_;
+ }
+ JSObject* globalLexicalEnvThis() const { return globalLexicalEnvThis_; }
+
+ void trace(JSTracer* trc);
+
+ const WarpBailoutInfo& bailoutInfo() const { return bailoutInfo_; }
+
+ NurseryObjectVector& nurseryObjects() { return nurseryObjects_; }
+ const NurseryObjectVector& nurseryObjects() const { return nurseryObjects_; }
+
+#ifdef DEBUG
+ mozilla::HashNumber icHash() const { return icHash_; }
+ void setICHash(mozilla::HashNumber hash) { icHash_ = hash; }
+#endif
+
+#ifdef JS_JITSPEW
+ void dump() const;
+ void dump(GenericPrinter& out) const;
+#endif
+
+#ifdef JS_CACHEIR_SPEW
+ bool needsFinalWarmUpCount() const { return needsFinalWarmUpCount_; }
+#endif
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_WarpSnapshot_h */
diff --git a/js/src/jit/WasmBCE.cpp b/js/src/jit/WasmBCE.cpp
new file mode 100644
index 0000000000..f855aab41d
--- /dev/null
+++ b/js/src/jit/WasmBCE.cpp
@@ -0,0 +1,139 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/WasmBCE.h"
+
+#include "jit/JitSpewer.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+
+using namespace js;
+using namespace js::jit;
+
+typedef js::HashMap<uint32_t, MDefinition*, DefaultHasher<uint32_t>,
+ SystemAllocPolicy>
+ LastSeenMap;
+
+// The Wasm Bounds Check Elimination (BCE) pass looks for bounds checks
+// on SSA values that have already been checked. (in the same block or in a
+// dominating block). These bounds checks are redundant and thus eliminated.
+//
+// Note: This is safe in the presense of dynamic memory sizes as long as they
+// can ONLY GROW. If we allow SHRINKING the heap, this pass should be
+// RECONSIDERED.
+//
+// TODO (dbounov): Are there a lot of cases where there is no single dominating
+// check, but a set of checks that together dominate a redundant check?
+//
+// TODO (dbounov): Generalize to constant additions relative to one base
+bool jit::EliminateBoundsChecks(MIRGenerator* mir, MIRGraph& graph) {
+ JitSpew(JitSpew_WasmBCE, "Begin");
+ // Map for dominating block where a given definition was checked
+ LastSeenMap lastSeen;
+
+ for (ReversePostorderIterator bIter(graph.rpoBegin());
+ bIter != graph.rpoEnd(); bIter++) {
+ MBasicBlock* block = *bIter;
+ for (MDefinitionIterator dIter(block); dIter;) {
+ MDefinition* def = *dIter++;
+
+ switch (def->op()) {
+ case MDefinition::Opcode::WasmBoundsCheck: {
+ MWasmBoundsCheck* bc = def->toWasmBoundsCheck();
+ MDefinition* addr = bc->index();
+
+ // We only support bounds check elimination on wasm memory, not
+ // tables. See bug 1625891.
+ if (!bc->isMemory()) {
+ continue;
+ }
+
+ // Eliminate constant-address memory bounds checks to addresses below
+ // the heap minimum.
+ //
+ // The payload of the MConstant will be Double if the constant
+ // result is above 2^31-1, but we don't care about that for BCE.
+
+ if (addr->isConstant() &&
+ ((addr->toConstant()->type() == MIRType::Int32 &&
+ uint64_t(addr->toConstant()->toInt32()) <
+ mir->minWasmHeapLength()) ||
+ (addr->toConstant()->type() == MIRType::Int64 &&
+ uint64_t(addr->toConstant()->toInt64()) <
+ mir->minWasmHeapLength()))) {
+ bc->setRedundant();
+ if (JitOptions.spectreIndexMasking) {
+ bc->replaceAllUsesWith(addr);
+ } else {
+ MOZ_ASSERT(!bc->hasUses());
+ }
+ } else {
+ LastSeenMap::AddPtr ptr = lastSeen.lookupForAdd(addr->id());
+ if (ptr) {
+ MDefinition* prevCheckOrPhi = ptr->value();
+ if (prevCheckOrPhi->block()->dominates(block)) {
+ bc->setRedundant();
+ if (JitOptions.spectreIndexMasking) {
+ bc->replaceAllUsesWith(prevCheckOrPhi);
+ } else {
+ MOZ_ASSERT(!bc->hasUses());
+ }
+ }
+ } else {
+ if (!lastSeen.add(ptr, addr->id(), def)) {
+ return false;
+ }
+ }
+ }
+ break;
+ }
+ case MDefinition::Opcode::Phi: {
+ MPhi* phi = def->toPhi();
+ bool phiChecked = true;
+
+ MOZ_ASSERT(phi->numOperands() > 0);
+
+ // If all incoming values to a phi node are safe (i.e. have a
+ // check that dominates this block) then we can consider this
+ // phi node checked.
+ //
+ // Note that any phi that is part of a cycle
+ // will not be "safe" since the value coming on the backedge
+ // cannot be in lastSeen because its block hasn't been traversed yet.
+ for (int i = 0, nOps = phi->numOperands(); i < nOps; i++) {
+ MDefinition* src = phi->getOperand(i);
+
+ if (JitOptions.spectreIndexMasking) {
+ if (src->isWasmBoundsCheck()) {
+ src = src->toWasmBoundsCheck()->index();
+ }
+ } else {
+ MOZ_ASSERT(!src->isWasmBoundsCheck());
+ }
+
+ LastSeenMap::Ptr checkPtr = lastSeen.lookup(src->id());
+ if (!checkPtr || !checkPtr->value()->block()->dominates(block)) {
+ phiChecked = false;
+ break;
+ }
+ }
+
+ if (phiChecked) {
+ if (!lastSeen.put(def->id(), def)) {
+ return false;
+ }
+ }
+
+ break;
+ }
+ default:
+ break;
+ }
+ }
+ }
+
+ return true;
+}
diff --git a/js/src/jit/WasmBCE.h b/js/src/jit/WasmBCE.h
new file mode 100644
index 0000000000..8f8a15f067
--- /dev/null
+++ b/js/src/jit/WasmBCE.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ *
+ * Copyright 2016 Mozilla Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef jit_wasmbce_h
+#define jit_wasmbce_h
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+class MIRGraph;
+
+bool EliminateBoundsChecks(MIRGenerator* mir, MIRGraph& graph);
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_wasmbce_h */
diff --git a/js/src/jit/XrayJitInfo.cpp b/js/src/jit/XrayJitInfo.cpp
new file mode 100644
index 0000000000..70e00f4296
--- /dev/null
+++ b/js/src/jit/XrayJitInfo.cpp
@@ -0,0 +1,17 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "js/friend/XrayJitInfo.h"
+
+#include "jit/CacheIRGenerator.h" // js::jit::GetXrayJitInfo
+
+using JS::XrayJitInfo;
+
+static XrayJitInfo* gXrayJitInfo = nullptr;
+
+void JS::SetXrayJitInfo(XrayJitInfo* info) { gXrayJitInfo = info; }
+
+XrayJitInfo* js::jit::GetXrayJitInfo() { return gXrayJitInfo; }
diff --git a/js/src/jit/arm/Architecture-arm.cpp b/js/src/jit/arm/Architecture-arm.cpp
new file mode 100644
index 0000000000..d4c5026705
--- /dev/null
+++ b/js/src/jit/arm/Architecture-arm.cpp
@@ -0,0 +1,540 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm/Architecture-arm.h"
+
+#if !defined(JS_SIMULATOR_ARM) && !defined(__APPLE__)
+# include <elf.h>
+#endif
+
+#include <fcntl.h>
+#ifdef XP_UNIX
+# include <unistd.h>
+#endif
+
+#if defined(XP_IOS)
+# include <libkern/OSCacheControl.h>
+#endif
+
+#include "jit/arm/Assembler-arm.h"
+#include "jit/arm/Simulator-arm.h"
+#include "jit/FlushICache.h" // js::jit::FlushICache
+#include "jit/RegisterSets.h"
+
+#if !defined(__linux__) || defined(ANDROID) || defined(JS_SIMULATOR_ARM)
+// The Android NDK and B2G do not include the hwcap.h kernel header, and it is
+// not defined when building the simulator, so inline the header defines we
+// need.
+# define HWCAP_VFP (1 << 6)
+# define HWCAP_NEON (1 << 12)
+# define HWCAP_VFPv3 (1 << 13)
+# define HWCAP_VFPv3D16 (1 << 14) /* also set for VFPv4-D16 */
+# define HWCAP_VFPv4 (1 << 16)
+# define HWCAP_IDIVA (1 << 17)
+# define HWCAP_IDIVT (1 << 18)
+# define HWCAP_VFPD32 (1 << 19) /* set if VFP has 32 regs (not 16) */
+# define AT_HWCAP 16
+#else
+# include <asm/hwcap.h>
+# if !defined(HWCAP_IDIVA)
+# define HWCAP_IDIVA (1 << 17)
+# endif
+# if !defined(HWCAP_VFPD32)
+# define HWCAP_VFPD32 (1 << 19) /* set if VFP has 32 regs (not 16) */
+# endif
+#endif
+
+namespace js {
+namespace jit {
+
+// Parse the Linux kernel cpuinfo features. This is also used to parse the
+// override features which has some extensions: 'armv7', 'align' and 'hardfp'.
+static uint32_t ParseARMCpuFeatures(const char* features,
+ bool override = false) {
+ uint32_t flags = 0;
+
+ // For ease of running tests we want it to be the default to fixup faults.
+ bool fixupAlignmentFault = true;
+
+ for (;;) {
+ char ch = *features;
+ if (!ch) {
+ // End of string.
+ break;
+ }
+ if (ch == ' ' || ch == ',') {
+ // Skip separator characters.
+ features++;
+ continue;
+ }
+ // Find the end of the token.
+ const char* end = features + 1;
+ for (;; end++) {
+ ch = *end;
+ if (!ch || ch == ' ' || ch == ',') {
+ break;
+ }
+ }
+ size_t count = end - features;
+ if (count == 3 && strncmp(features, "vfp", 3) == 0) {
+ flags |= HWCAP_VFP;
+ } else if (count == 5 && strncmp(features, "vfpv2", 5) == 0) {
+ flags |= HWCAP_VFP; // vfpv2 is the same as vfp
+ } else if (count == 4 && strncmp(features, "neon", 4) == 0) {
+ flags |= HWCAP_NEON;
+ } else if (count == 5 && strncmp(features, "vfpv3", 5) == 0) {
+ flags |= HWCAP_VFPv3;
+ } else if (count == 8 && strncmp(features, "vfpv3d16", 8) == 0) {
+ flags |= HWCAP_VFPv3D16;
+ } else if (count == 5 && strncmp(features, "vfpv4", 5) == 0) {
+ flags |= HWCAP_VFPv4;
+ } else if (count == 5 && strncmp(features, "idiva", 5) == 0) {
+ flags |= HWCAP_IDIVA;
+ } else if (count == 5 && strncmp(features, "idivt", 5) == 0) {
+ flags |= HWCAP_IDIVT;
+ } else if (count == 6 && strncmp(features, "vfpd32", 6) == 0) {
+ flags |= HWCAP_VFPD32;
+ } else if (count == 5 && strncmp(features, "armv7", 5) == 0) {
+ flags |= HWCAP_ARMv7;
+ } else if (count == 5 && strncmp(features, "align", 5) == 0) {
+ flags |= HWCAP_ALIGNMENT_FAULT | HWCAP_FIXUP_FAULT;
+#if defined(JS_SIMULATOR_ARM)
+ } else if (count == 7 && strncmp(features, "nofixup", 7) == 0) {
+ fixupAlignmentFault = false;
+ } else if (count == 6 && strncmp(features, "hardfp", 6) == 0) {
+ flags |= HWCAP_USE_HARDFP_ABI;
+#endif
+ } else if (override) {
+ fprintf(stderr, "Warning: unexpected ARM feature at: %s\n", features);
+ }
+ features = end;
+ }
+
+ if (!fixupAlignmentFault) {
+ flags &= ~HWCAP_FIXUP_FAULT;
+ }
+
+ return flags;
+}
+
+static uint32_t CanonicalizeARMHwCapFlags(uint32_t flags) {
+ // Canonicalize the flags. These rules are also applied to the features
+ // supplied for simulation.
+
+ // VFPv3 is a subset of VFPv4, force this if the input string omits it.
+ if (flags & HWCAP_VFPv4) {
+ flags |= HWCAP_VFPv3;
+ }
+
+ // The VFPv3 feature is expected when the VFPv3D16 is reported, but add it
+ // just in case of a kernel difference in feature reporting.
+ if (flags & HWCAP_VFPv3D16) {
+ flags |= HWCAP_VFPv3;
+ }
+
+ // VFPv2 is a subset of VFPv3, force this if the input string omits it. VFPv2
+ // is just an alias for VFP.
+ if (flags & HWCAP_VFPv3) {
+ flags |= HWCAP_VFP;
+ }
+
+ // If we have Neon we have floating point.
+ if (flags & HWCAP_NEON) {
+ flags |= HWCAP_VFP;
+ }
+
+ // If VFPv3 or Neon is supported then this must be an ARMv7.
+ if (flags & (HWCAP_VFPv3 | HWCAP_NEON)) {
+ flags |= HWCAP_ARMv7;
+ }
+
+ // Some old kernels report VFP and not VFPv3, but if ARMv7 then it must be
+ // VFPv3.
+ if ((flags & HWCAP_VFP) && (flags & HWCAP_ARMv7)) {
+ flags |= HWCAP_VFPv3;
+ }
+
+ // Older kernels do not implement the HWCAP_VFPD32 flag.
+ if ((flags & HWCAP_VFPv3) && !(flags & HWCAP_VFPv3D16)) {
+ flags |= HWCAP_VFPD32;
+ }
+
+ return flags;
+}
+
+#if !defined(JS_SIMULATOR_ARM) && (defined(__linux__) || defined(ANDROID))
+static bool forceDoubleCacheFlush = false;
+#endif
+
+// The override flags parsed from the ARMHWCAP environment variable or from the
+// --arm-hwcap js shell argument. They are stable after startup: there is no
+// longer a programmatic way of setting these from JS.
+volatile uint32_t armHwCapFlags = HWCAP_UNINITIALIZED;
+
+bool CPUFlagsHaveBeenComputed() { return armHwCapFlags != HWCAP_UNINITIALIZED; }
+
+static const char* gArmHwCapString = nullptr;
+
+void SetARMHwCapFlagsString(const char* armHwCap) {
+ MOZ_ASSERT(!CPUFlagsHaveBeenComputed());
+ gArmHwCapString = armHwCap;
+}
+
+static void ParseARMHwCapFlags(const char* armHwCap) {
+ MOZ_ASSERT(armHwCap);
+
+ if (strstr(armHwCap, "help")) {
+ fflush(NULL);
+ printf(
+ "\n"
+ "usage: ARMHWCAP=option,option,option,... where options can be:\n"
+ "\n"
+ " vfp \n"
+ " neon \n"
+ " vfpv3 \n"
+ " vfpv3d16 \n"
+ " vfpv4 \n"
+ " idiva \n"
+ " idivt \n"
+ " vfpd32 \n"
+ " armv7 \n"
+ " align - unaligned accesses will trap and be emulated\n"
+#ifdef JS_SIMULATOR_ARM
+ " nofixup - disable emulation of unaligned accesses\n"
+ " hardfp \n"
+#endif
+ "\n");
+ exit(0);
+ /*NOTREACHED*/
+ }
+
+ uint32_t flags = ParseARMCpuFeatures(armHwCap, /* override = */ true);
+
+#ifdef JS_CODEGEN_ARM_HARDFP
+ flags |= HWCAP_USE_HARDFP_ABI;
+#endif
+
+ armHwCapFlags = CanonicalizeARMHwCapFlags(flags);
+ JitSpew(JitSpew_Codegen, "ARM HWCAP: 0x%x\n", armHwCapFlags);
+}
+
+void InitARMFlags() {
+ MOZ_RELEASE_ASSERT(armHwCapFlags == HWCAP_UNINITIALIZED);
+
+ if (const char* env = getenv("ARMHWCAP")) {
+ ParseARMHwCapFlags(env);
+ return;
+ }
+
+ if (gArmHwCapString) {
+ ParseARMHwCapFlags(gArmHwCapString);
+ return;
+ }
+
+ uint32_t flags = 0;
+#ifdef JS_SIMULATOR_ARM
+ // HWCAP_FIXUP_FAULT is on by default even if HWCAP_ALIGNMENT_FAULT is
+ // not on by default, because some memory access instructions always fault.
+ // Notably, this is true for floating point accesses.
+ flags = HWCAP_ARMv7 | HWCAP_VFP | HWCAP_VFPv3 | HWCAP_VFPv4 | HWCAP_NEON |
+ HWCAP_IDIVA | HWCAP_FIXUP_FAULT;
+#else
+
+# if defined(__linux__) || defined(ANDROID)
+ // This includes Android and B2G.
+ bool readAuxv = false;
+ int fd = open("/proc/self/auxv", O_RDONLY);
+ if (fd > 0) {
+ struct {
+ uint32_t a_type;
+ uint32_t a_val;
+ } aux;
+ while (read(fd, &aux, sizeof(aux))) {
+ if (aux.a_type == AT_HWCAP) {
+ flags = aux.a_val;
+ readAuxv = true;
+ break;
+ }
+ }
+ close(fd);
+ }
+
+ FILE* fp = fopen("/proc/cpuinfo", "r");
+ if (fp) {
+ char buf[1024] = {};
+ size_t len = fread(buf, sizeof(char), sizeof(buf) - 1, fp);
+ fclose(fp);
+ buf[len] = '\0';
+
+ // Read the cpuinfo Features if the auxv is not available.
+ if (!readAuxv) {
+ char* featureList = strstr(buf, "Features");
+ if (featureList) {
+ if (char* featuresEnd = strstr(featureList, "\n")) {
+ *featuresEnd = '\0';
+ }
+ flags = ParseARMCpuFeatures(featureList + 8);
+ }
+ if (strstr(buf, "ARMv7")) {
+ flags |= HWCAP_ARMv7;
+ }
+ }
+
+ // The exynos7420 cpu (EU galaxy S6 (Note)) has a bug where sometimes
+ // flushing doesn't invalidate the instruction cache. As a result we force
+ // it by calling the cacheFlush twice on different start addresses.
+ char* exynos7420 = strstr(buf, "Exynos7420");
+ if (exynos7420) {
+ forceDoubleCacheFlush = true;
+ }
+ }
+# endif
+
+ // If compiled to use specialized features then these features can be
+ // assumed to be present otherwise the compiler would fail to run.
+
+# ifdef JS_CODEGEN_ARM_HARDFP
+ // Compiled to use the hardfp ABI.
+ flags |= HWCAP_USE_HARDFP_ABI;
+# endif
+
+# if defined(__VFP_FP__) && !defined(__SOFTFP__)
+ // Compiled to use VFP instructions so assume VFP support.
+ flags |= HWCAP_VFP;
+# endif
+
+# if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__)
+ // Compiled to use ARMv7 instructions so assume the ARMv7 arch.
+ flags |= HWCAP_ARMv7;
+# endif
+
+# if defined(__APPLE__)
+# if defined(__ARM_NEON__)
+ flags |= HWCAP_NEON;
+# endif
+# if defined(__ARMVFPV3__)
+ flags |= HWCAP_VFPv3 | HWCAP_VFPD32
+# endif
+# endif
+
+#endif // JS_SIMULATOR_ARM
+
+ armHwCapFlags = CanonicalizeARMHwCapFlags(flags);
+
+ JitSpew(JitSpew_Codegen, "ARM HWCAP: 0x%x\n", armHwCapFlags);
+ return;
+}
+
+uint32_t GetARMFlags() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags;
+}
+
+bool HasNEON() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_NEON;
+}
+
+bool HasARMv7() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_ARMv7;
+}
+
+bool HasMOVWT() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_ARMv7;
+}
+
+bool HasLDSTREXBHD() {
+ // These are really available from ARMv6K and later, but why bother?
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_ARMv7;
+}
+
+bool HasDMBDSBISB() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_ARMv7;
+}
+
+bool HasVFPv3() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_VFPv3;
+}
+
+bool HasVFP() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_VFP;
+}
+
+bool Has32DP() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_VFPD32;
+}
+
+bool HasIDIV() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_IDIVA;
+}
+
+// This is defined in the header and inlined when not using the simulator.
+#ifdef JS_SIMULATOR_ARM
+bool UseHardFpABI() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_USE_HARDFP_ABI;
+}
+#endif
+
+Registers::Code Registers::FromName(const char* name) {
+ // Check for some register aliases first.
+ if (strcmp(name, "ip") == 0) {
+ return ip;
+ }
+ if (strcmp(name, "r13") == 0) {
+ return r13;
+ }
+ if (strcmp(name, "lr") == 0) {
+ return lr;
+ }
+ if (strcmp(name, "r15") == 0) {
+ return r15;
+ }
+
+ for (size_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(i), name) == 0) {
+ return Code(i);
+ }
+ }
+
+ return Invalid;
+}
+
+FloatRegisters::Code FloatRegisters::FromName(const char* name) {
+ for (size_t i = 0; i < TotalSingle; ++i) {
+ if (strcmp(GetSingleName(Encoding(i)), name) == 0) {
+ return VFPRegister(i, VFPRegister::Single).code();
+ }
+ }
+ for (size_t i = 0; i < TotalDouble; ++i) {
+ if (strcmp(GetDoubleName(Encoding(i)), name) == 0) {
+ return VFPRegister(i, VFPRegister::Double).code();
+ }
+ }
+
+ return Invalid;
+}
+
+FloatRegisterSet VFPRegister::ReduceSetForPush(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ LiveFloatRegisterSet mod;
+ for (FloatRegisterIterator iter(s); iter.more(); ++iter) {
+ if ((*iter).isSingle()) {
+ // Add in just this float.
+ mod.addUnchecked(*iter);
+ } else if ((*iter).id() < 16) {
+ // A double with an overlay, add in both floats.
+ mod.addUnchecked((*iter).singleOverlay(0));
+ mod.addUnchecked((*iter).singleOverlay(1));
+ } else {
+ // Add in the lone double in the range 16-31.
+ mod.addUnchecked(*iter);
+ }
+ }
+ return mod.set();
+}
+
+uint32_t VFPRegister::GetPushSizeInBytes(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ FloatRegisterSet ss = s.reduceSetForPush();
+ uint64_t bits = ss.bits();
+ uint32_t ret = mozilla::CountPopulation32(bits & 0xffffffff) * sizeof(float);
+ ret += mozilla::CountPopulation32(bits >> 32) * sizeof(double);
+ return ret;
+}
+uint32_t VFPRegister::getRegisterDumpOffsetInBytes() {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ if (isSingle()) {
+ return id() * sizeof(float);
+ }
+ if (isDouble()) {
+ return id() * sizeof(double);
+ }
+ MOZ_CRASH("not Single or Double");
+}
+
+uint32_t FloatRegisters::ActualTotalPhys() {
+ if (Has32DP()) {
+ return 32;
+ }
+ return 16;
+}
+
+void FlushICache(void* code, size_t size) {
+#if defined(JS_SIMULATOR_ARM)
+ js::jit::SimulatorProcess::FlushICache(code, size);
+
+#elif (defined(__linux__) || defined(ANDROID)) && defined(__GNUC__)
+ void* end = (void*)(reinterpret_cast<char*>(code) + size);
+ asm volatile(
+ "push {r7}\n"
+ "mov r0, %0\n"
+ "mov r1, %1\n"
+ "mov r7, #0xf0000\n"
+ "add r7, r7, #0x2\n"
+ "mov r2, #0x0\n"
+ "svc 0x0\n"
+ "pop {r7}\n"
+ :
+ : "r"(code), "r"(end)
+ : "r0", "r1", "r2");
+
+ if (forceDoubleCacheFlush) {
+ void* start = (void*)((uintptr_t)code + 1);
+ asm volatile(
+ "push {r7}\n"
+ "mov r0, %0\n"
+ "mov r1, %1\n"
+ "mov r7, #0xf0000\n"
+ "add r7, r7, #0x2\n"
+ "mov r2, #0x0\n"
+ "svc 0x0\n"
+ "pop {r7}\n"
+ :
+ : "r"(start), "r"(end)
+ : "r0", "r1", "r2");
+ }
+
+#elif defined(__FreeBSD__) || defined(__NetBSD__)
+ __clear_cache(code, reinterpret_cast<char*>(code) + size);
+
+#elif defined(XP_IOS)
+ sys_icache_invalidate(code, size);
+
+#else
+# error "Unexpected platform"
+#endif
+}
+
+void FlushExecutionContext() {
+#ifndef JS_SIMULATOR_ARM
+ // Ensure that any instructions already in the pipeline are discarded and
+ // reloaded from the icache.
+ asm volatile("isb\n" : : : "memory");
+#else
+ // We assume the icache flushing routines on other platforms take care of this
+#endif
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/arm/Architecture-arm.h b/js/src/jit/arm/Architecture-arm.h
new file mode 100644
index 0000000000..34b3072e6c
--- /dev/null
+++ b/js/src/jit/arm/Architecture-arm.h
@@ -0,0 +1,735 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_Architecture_arm_h
+#define jit_arm_Architecture_arm_h
+
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+#include <limits.h>
+#include <stdint.h>
+
+#include "jit/shared/Architecture-shared.h"
+
+#include "js/Utility.h"
+
+// GCC versions 4.6 and above define __ARM_PCS_VFP to denote a hard-float
+// ABI target. The iOS toolchain doesn't define anything specific here,
+// but iOS always supports VFP.
+#if defined(__ARM_PCS_VFP) || defined(XP_IOS)
+# define JS_CODEGEN_ARM_HARDFP
+#endif
+
+namespace js {
+namespace jit {
+
+// These offsets are specific to nunboxing, and capture offsets into the
+// components of a js::Value.
+static const int32_t NUNBOX32_TYPE_OFFSET = 4;
+static const int32_t NUNBOX32_PAYLOAD_OFFSET = 0;
+
+static const uint32_t ShadowStackSpace = 0;
+
+// How far forward/back can a jump go? Provide a generous buffer for thunks.
+static const uint32_t JumpImmediateRange = 20 * 1024 * 1024;
+
+class Registers {
+ public:
+ enum RegisterID {
+ r0 = 0,
+ r1,
+ r2,
+ r3,
+ r4,
+ r5,
+ r6,
+ r7,
+ r8,
+ r9,
+ r10,
+ r11,
+ fp = r11,
+ r12,
+ ip = r12,
+ r13,
+ sp = r13,
+ r14,
+ lr = r14,
+ r15,
+ pc = r15,
+ invalid_reg
+ };
+ typedef uint8_t Code;
+ typedef RegisterID Encoding;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ uintptr_t r;
+ };
+
+ static const char* GetName(Code code) {
+ MOZ_ASSERT(code < Total);
+ static const char* const Names[] = {"r0", "r1", "r2", "r3", "r4", "r5",
+ "r6", "r7", "r8", "r9", "r10", "r11",
+ "r12", "sp", "r14", "pc"};
+ return Names[code];
+ }
+ static const char* GetName(Encoding i) { return GetName(Code(i)); }
+
+ static Code FromName(const char* name);
+
+ static const Encoding StackPointer = sp;
+ static const Encoding Invalid = invalid_reg;
+
+ static const uint32_t Total = 16;
+ static const uint32_t Allocatable = 13;
+
+ typedef uint32_t SetType;
+
+ static const SetType AllMask = (1 << Total) - 1;
+ static const SetType ArgRegMask =
+ (1 << r0) | (1 << r1) | (1 << r2) | (1 << r3);
+
+ static const SetType VolatileMask =
+ (1 << r0) | (1 << r1) | (1 << Registers::r2) |
+ (1 << Registers::r3)
+#if defined(XP_IOS)
+ // per
+ // https://developer.apple.com/library/ios/documentation/Xcode/Conceptual/iPhoneOSABIReference/Articles/ARMv6FunctionCallingConventions.html#//apple_ref/doc/uid/TP40009021-SW4
+ | (1 << Registers::r9)
+#endif
+ ;
+
+ static const SetType NonVolatileMask =
+ (1 << Registers::r4) | (1 << Registers::r5) | (1 << Registers::r6) |
+ (1 << Registers::r7) | (1 << Registers::r8) |
+#if !defined(XP_IOS)
+ (1 << Registers::r9) |
+#endif
+ (1 << Registers::r10) | (1 << Registers::r11) | (1 << Registers::r12) |
+ (1 << Registers::r14);
+
+ static const SetType WrapperMask = VolatileMask | // = arguments
+ (1 << Registers::r4) | // = outReg
+ (1 << Registers::r5); // = argBase
+
+ static const SetType SingleByteRegs = VolatileMask | NonVolatileMask;
+
+ static const SetType NonAllocatableMask =
+ (1 << Registers::sp) | (1 << Registers::r12) | // r12 = ip = scratch
+ (1 << Registers::lr) | (1 << Registers::pc) | (1 << Registers::fp);
+
+ // Registers returned from a JS -> JS call.
+ static const SetType JSCallMask = (1 << Registers::r2) | (1 << Registers::r3);
+
+ // Registers returned from a JS -> C call.
+ static const SetType CallMask =
+ (1 << Registers::r0) |
+ (1 << Registers::r1); // Used for double-size returns.
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return mozilla::CountPopulation32(x);
+ }
+ static uint32_t FirstBit(SetType x) {
+ return mozilla::CountTrailingZeroes32(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ return 31 - mozilla::CountLeadingZeroes32(x);
+ }
+};
+
+// Smallest integer type that can hold a register bitmask.
+typedef uint16_t PackedRegisterMask;
+
+class FloatRegisters {
+ public:
+ enum FPRegisterID {
+ s0,
+ s1,
+ s2,
+ s3,
+ s4,
+ s5,
+ s6,
+ s7,
+ s8,
+ s9,
+ s10,
+ s11,
+ s12,
+ s13,
+ s14,
+ s15,
+ s16,
+ s17,
+ s18,
+ s19,
+ s20,
+ s21,
+ s22,
+ s23,
+ s24,
+ s25,
+ s26,
+ s27,
+ s28,
+ s29,
+ s30,
+ s31,
+ d0,
+ d1,
+ d2,
+ d3,
+ d4,
+ d5,
+ d6,
+ d7,
+ d8,
+ d9,
+ d10,
+ d11,
+ d12,
+ d13,
+ d14,
+ d15,
+ d16,
+ d17,
+ d18,
+ d19,
+ d20,
+ d21,
+ d22,
+ d23,
+ d24,
+ d25,
+ d26,
+ d27,
+ d28,
+ d29,
+ d30,
+ d31,
+ invalid_freg
+ };
+
+ typedef uint32_t Code;
+ typedef FPRegisterID Encoding;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ double d;
+ };
+
+ static const char* GetDoubleName(Encoding code) {
+ static const char* const Names[] = {
+ "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
+ "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15",
+ "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23",
+ "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"};
+ return Names[code];
+ }
+ static const char* GetSingleName(Encoding code) {
+ static const char* const Names[] = {
+ "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
+ "s8", "s9", "s10", "s11", "s12", "s13", "s14", "s15",
+ "s16", "s17", "s18", "s19", "s20", "s21", "s22", "s23",
+ "s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31"};
+ return Names[code];
+ }
+
+ static Code FromName(const char* name);
+
+ static const Encoding Invalid = invalid_freg;
+ static const uint32_t Total = 48;
+ static const uint32_t TotalDouble = 16;
+ static const uint32_t TotalSingle = 32;
+ static const uint32_t Allocatable = 45;
+ // There are only 32 places that we can put values.
+ static const uint32_t TotalPhys = 32;
+ static uint32_t ActualTotalPhys();
+
+ /* clang-format off */
+ // ARM float registers overlap in a way that for 1 double registers, in the
+ // range d0-d15, we have 2 singles register in the range s0-s31. d16-d31
+ // have no single register aliases. The aliasing rule state that d{n}
+ // aliases s{2n} and s{2n+1}, for n in [0 .. 15].
+ //
+ // The register set is used to represent either allocatable register or live
+ // registers. The register maps d0-d15 and s0-s31 to a single bit each. The
+ // registers d16-d31 are not used at the moment.
+ //
+ // uuuu uuuu uuuu uuuu dddd dddd dddd dddd ssss ssss ssss ssss ssss ssss ssss ssss
+ // ^ ^ ^ ^
+ // '-- d15 d0 --' '-- s31 s0 --'
+ //
+ // LiveSet are handled by adding the bit of each register without
+ // considering the aliases.
+ //
+ // AllocatableSet are handled by adding and removing the bit of each
+ // aligned-or-dominated-aliased registers.
+ //
+ // ...0...00... : s{2n}, s{2n+1} and d{n} are not available
+ // ...1...01... : s{2n} is available (*)
+ // ...0...10... : s{2n+1} is available
+ // ...1...11... : s{2n}, s{2n+1} and d{n} are available
+ //
+ // (*) Note that d{n} bit is set, but is not available because s{2n+1} bit
+ // is not set, which is required as d{n} dominates s{2n+1}. The d{n} bit is
+ // set, because s{2n} is aligned.
+ //
+ // | d{n} |
+ // | s{2n+1} | s{2n} |
+ //
+ /* clang-format on */
+ typedef uint64_t SetType;
+ static const SetType AllSingleMask = (1ull << TotalSingle) - 1;
+ static const SetType AllDoubleMask = ((1ull << TotalDouble) - 1)
+ << TotalSingle;
+ static const SetType AllMask = AllDoubleMask | AllSingleMask;
+
+ // d15 is the ScratchFloatReg.
+ static const SetType NonVolatileDoubleMask =
+ ((1ULL << d8) | (1ULL << d9) | (1ULL << d10) | (1ULL << d11) |
+ (1ULL << d12) | (1ULL << d13) | (1ULL << d14));
+ // s30 and s31 alias d15.
+ static const SetType NonVolatileMask =
+ (NonVolatileDoubleMask |
+ ((1 << s16) | (1 << s17) | (1 << s18) | (1 << s19) | (1 << s20) |
+ (1 << s21) | (1 << s22) | (1 << s23) | (1 << s24) | (1 << s25) |
+ (1 << s26) | (1 << s27) | (1 << s28) | (1 << s29) | (1 << s30)));
+
+ static const SetType VolatileMask = AllMask & ~NonVolatileMask;
+ static const SetType VolatileDoubleMask =
+ AllDoubleMask & ~NonVolatileDoubleMask;
+
+ static const SetType WrapperMask = VolatileMask;
+
+ // d15 is the ARM scratch float register.
+ // s30 and s31 alias d15.
+ static const SetType NonAllocatableMask =
+ ((1ULL << d15)) | (1ULL << s30) | (1ULL << s31);
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+};
+
+static const uint32_t SpillSlotSize =
+ std::max(sizeof(Registers::RegisterContent),
+ sizeof(FloatRegisters::RegisterContent));
+
+template <typename T>
+class TypedRegisterSet;
+
+class VFPRegister {
+ public:
+ // What type of data is being stored in this register? UInt / Int are
+ // specifically for vcvt, where we need to know how the data is supposed to
+ // be converted.
+ enum RegType : uint8_t { Single = 0x0, Double = 0x1, UInt = 0x2, Int = 0x3 };
+
+ typedef FloatRegisters Codes;
+ typedef Codes::Code Code;
+ typedef Codes::Encoding Encoding;
+
+ // Bitfields below are all uint32_t to make sure MSVC packs them correctly.
+ public:
+ // ARM doesn't have more than 32 registers of each type, so 5 bits should
+ // suffice.
+ uint32_t code_ : 5;
+
+ protected:
+ uint32_t kind : 2;
+ uint32_t _isInvalid : 1;
+ uint32_t _isMissing : 1;
+
+ public:
+ constexpr VFPRegister(uint32_t r, RegType k)
+ : code_(Code(r)), kind(k), _isInvalid(false), _isMissing(false) {}
+ constexpr VFPRegister()
+ : code_(Code(0)), kind(Double), _isInvalid(true), _isMissing(false) {}
+
+ constexpr VFPRegister(RegType k, uint32_t id, bool invalid, bool missing)
+ : code_(Code(id)), kind(k), _isInvalid(invalid), _isMissing(missing) {}
+
+ explicit constexpr VFPRegister(Code id)
+ : code_(id), kind(Double), _isInvalid(false), _isMissing(false) {}
+ bool operator==(const VFPRegister& other) const {
+ return kind == other.kind && code_ == other.code_ &&
+ isInvalid() == other.isInvalid();
+ }
+ bool operator!=(const VFPRegister& other) const { return !operator==(other); }
+
+ bool isSingle() const { return kind == Single; }
+ bool isDouble() const { return kind == Double; }
+ bool isSimd128() const { return false; }
+ bool isFloat() const { return (kind == Double) || (kind == Single); }
+ bool isInt() const { return (kind == UInt) || (kind == Int); }
+ bool isSInt() const { return kind == Int; }
+ bool isUInt() const { return kind == UInt; }
+ bool equiv(const VFPRegister& other) const { return other.kind == kind; }
+ size_t size() const { return (kind == Double) ? 8 : 4; }
+ bool isInvalid() const { return _isInvalid; }
+ bool isMissing() const {
+ MOZ_ASSERT(!_isInvalid);
+ return _isMissing;
+ }
+
+ VFPRegister doubleOverlay(unsigned int which = 0) const;
+ VFPRegister singleOverlay(unsigned int which = 0) const;
+ VFPRegister sintOverlay(unsigned int which = 0) const;
+ VFPRegister uintOverlay(unsigned int which = 0) const;
+
+ VFPRegister asSingle() const { return singleOverlay(); }
+ VFPRegister asDouble() const { return doubleOverlay(); }
+ VFPRegister asSimd128() const { MOZ_CRASH("NYI"); }
+
+ struct VFPRegIndexSplit;
+ VFPRegIndexSplit encode();
+
+ // For serializing values.
+ struct VFPRegIndexSplit {
+ const uint32_t block : 4;
+ const uint32_t bit : 1;
+
+ private:
+ friend VFPRegIndexSplit js::jit::VFPRegister::encode();
+
+ VFPRegIndexSplit(uint32_t block_, uint32_t bit_)
+ : block(block_), bit(bit_) {
+ MOZ_ASSERT(block == block_);
+ MOZ_ASSERT(bit == bit_);
+ }
+ };
+
+ Code code() const {
+ MOZ_ASSERT(!_isInvalid && !_isMissing);
+ // This should only be used in areas where we only have doubles and
+ // singles.
+ MOZ_ASSERT(isFloat());
+ return Code(code_ | (kind << 5));
+ }
+ Encoding encoding() const {
+ MOZ_ASSERT(!_isInvalid && !_isMissing);
+ return Encoding(code_);
+ }
+ uint32_t id() const { return code_; }
+ static VFPRegister FromCode(uint32_t i) {
+ uint32_t code = i & 31;
+ uint32_t kind = i >> 5;
+ return VFPRegister(code, RegType(kind));
+ }
+ bool volatile_() const {
+ if (isDouble()) {
+ return !!((1ULL << (code_ >> 1)) & FloatRegisters::VolatileMask);
+ }
+ return !!((1ULL << code_) & FloatRegisters::VolatileMask);
+ }
+ const char* name() const {
+ if (isDouble()) {
+ return FloatRegisters::GetDoubleName(Encoding(code_));
+ }
+ return FloatRegisters::GetSingleName(Encoding(code_));
+ }
+ bool aliases(const VFPRegister& other) {
+ if (kind == other.kind) {
+ return code_ == other.code_;
+ }
+ return doubleOverlay() == other.doubleOverlay();
+ }
+ static const int NumAliasedDoubles = 16;
+ uint32_t numAliased() const {
+ if (isDouble()) {
+ if (code_ < NumAliasedDoubles) {
+ return 3;
+ }
+ return 1;
+ }
+ return 2;
+ }
+
+ VFPRegister aliased(uint32_t aliasIdx) {
+ if (aliasIdx == 0) {
+ return *this;
+ }
+ if (isDouble()) {
+ MOZ_ASSERT(code_ < NumAliasedDoubles);
+ MOZ_ASSERT(aliasIdx <= 2);
+ return singleOverlay(aliasIdx - 1);
+ }
+ MOZ_ASSERT(aliasIdx == 1);
+ return doubleOverlay(aliasIdx - 1);
+ }
+ uint32_t numAlignedAliased() const {
+ if (isDouble()) {
+ if (code_ < NumAliasedDoubles) {
+ return 2;
+ }
+ return 1;
+ }
+ // s1 has 0 other aligned aliases, 1 total.
+ // s0 has 1 other aligned aliase, 2 total.
+ return 2 - (code_ & 1);
+ }
+ // | d0 |
+ // | s0 | s1 |
+ // If we've stored s0 and s1 in memory, we also want to say that d0 is
+ // stored there, but it is only stored at the location where it is aligned
+ // e.g. at s0, not s1.
+ VFPRegister alignedAliased(uint32_t aliasIdx) {
+ if (aliasIdx == 0) {
+ return *this;
+ }
+ MOZ_ASSERT(aliasIdx == 1);
+ if (isDouble()) {
+ MOZ_ASSERT(code_ < NumAliasedDoubles);
+ return singleOverlay(aliasIdx - 1);
+ }
+ MOZ_ASSERT((code_ & 1) == 0);
+ return doubleOverlay(aliasIdx - 1);
+ }
+
+ typedef FloatRegisters::SetType SetType;
+
+ // This function is used to ensure that Register set can take all Single
+ // registers, even if we are taking a mix of either double or single
+ // registers.
+ //
+ // s0.alignedOrDominatedAliasedSet() == s0 | d0.
+ // s1.alignedOrDominatedAliasedSet() == s1.
+ // d0.alignedOrDominatedAliasedSet() == s0 | s1 | d0.
+ //
+ // This way the Allocatable register set does not have to do any arithmetics
+ // to know if a register is available or not, as we have the following
+ // relations:
+ //
+ // d0.alignedOrDominatedAliasedSet() ==
+ // s0.alignedOrDominatedAliasedSet() | s1.alignedOrDominatedAliasedSet()
+ //
+ // s0.alignedOrDominatedAliasedSet() & s1.alignedOrDominatedAliasedSet() == 0
+ //
+ SetType alignedOrDominatedAliasedSet() const {
+ if (isSingle()) {
+ if (code_ % 2 != 0) {
+ return SetType(1) << code_;
+ }
+ return (SetType(1) << code_) | (SetType(1) << (32 + code_ / 2));
+ }
+
+ MOZ_ASSERT(isDouble());
+ return (SetType(0b11) << (code_ * 2)) | (SetType(1) << (32 + code_));
+ }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return SetType(0);
+ }
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
+ return mozilla::CountPopulation32(x);
+ }
+ static Code FromName(const char* name) {
+ return FloatRegisters::FromName(name);
+ }
+ static TypedRegisterSet<VFPRegister> ReduceSetForPush(
+ const TypedRegisterSet<VFPRegister>& s);
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<VFPRegister>& s);
+ uint32_t getRegisterDumpOffsetInBytes();
+ static uint32_t FirstBit(SetType x) {
+ return mozilla::CountTrailingZeroes64(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ return 63 - mozilla::CountLeadingZeroes64(x);
+ }
+};
+
+template <>
+inline VFPRegister::SetType
+VFPRegister::LiveAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline VFPRegister::SetType
+VFPRegister::LiveAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ return set & FloatRegisters::AllDoubleMask;
+}
+
+template <>
+inline VFPRegister::SetType VFPRegister::LiveAsIndexableSet<RegTypeName::Any>(
+ SetType set) {
+ return set;
+}
+
+template <>
+inline VFPRegister::SetType
+VFPRegister::AllocatableAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ // Single registers are not dominating any smaller registers, thus masking
+ // is enough to convert an allocatable set into a set of register list all
+ // single register available.
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline VFPRegister::SetType
+VFPRegister::AllocatableAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ /* clang-format off */
+ // An allocatable float register set is represented as follow:
+ //
+ // uuuu uuuu uuuu uuuu dddd dddd dddd dddd ssss ssss ssss ssss ssss ssss ssss ssss
+ // ^ ^ ^ ^
+ // '-- d15 d0 --' '-- s31 s0 --'
+ //
+ // ...0...00... : s{2n}, s{2n+1} and d{n} are not available
+ // ...1...01... : s{2n} is available
+ // ...0...10... : s{2n+1} is available
+ // ...1...11... : s{2n}, s{2n+1} and d{n} are available
+ //
+ // The goal of this function is to return the set of double registers which
+ // are available as an indexable bit set. This implies that iff a double bit
+ // is set in the returned set, then the register is available.
+ //
+ // To do so, this functions converts the 32 bits set of single registers
+ // into a 16 bits set of equivalent double registers. Then, we mask out
+ // double registers which do not have all the single register that compose
+ // them. As d{n} bit is set when s{2n} is available, we only need to take
+ // s{2n+1} into account.
+ /* clang-format on */
+
+ // Convert s7s6s5s4 s3s2s1s0 into s7s5s3s1, for all s0-s31.
+ SetType s2d = AllocatableAsIndexableSet<RegTypeName::Float32>(set);
+ static_assert(FloatRegisters::TotalSingle == 32, "Wrong mask");
+ s2d = (0xaaaaaaaa & s2d) >> 1; // Filter s{2n+1} registers.
+ // Group adjacent bits as follow:
+ // 0.0.s3.s1 == ((0.s3.0.s1) >> 1 | (0.s3.0.s1)) & 0b0011;
+ s2d = ((s2d >> 1) | s2d) & 0x33333333; // 0a0b --> 00ab
+ s2d = ((s2d >> 2) | s2d) & 0x0f0f0f0f; // 00ab00cd --> 0000abcd
+ s2d = ((s2d >> 4) | s2d) & 0x00ff00ff;
+ s2d = ((s2d >> 8) | s2d) & 0x0000ffff;
+ // Move the s7s5s3s1 to the aliased double positions.
+ s2d = s2d << FloatRegisters::TotalSingle;
+
+ // Note: We currently do not use any representation for d16-d31.
+ static_assert(FloatRegisters::TotalDouble == 16,
+ "d16-d31 do not have a single register mapping");
+
+ // Filter out any double register which are not allocatable due to
+ // non-aligned dominated single registers.
+ return set & s2d;
+}
+
+// The only floating point register set that we work with are the VFP Registers.
+typedef VFPRegister FloatRegister;
+
+uint32_t GetARMFlags();
+bool HasARMv7();
+bool HasMOVWT();
+bool HasLDSTREXBHD(); // {LD,ST}REX{B,H,D}
+bool HasDMBDSBISB(); // DMB, DSB, and ISB
+bool HasVFPv3();
+bool HasVFP();
+bool Has32DP();
+bool HasIDIV();
+bool HasNEON();
+
+extern volatile uint32_t armHwCapFlags;
+
+// Not part of the HWCAP flag, but we need to know these and these bits are not
+// used. Define these here so that their use can be inlined by the simulator.
+
+// A bit to flag when signaled alignment faults are to be fixed up.
+#define HWCAP_FIXUP_FAULT (1 << 24)
+
+// A bit to flag when the flags are uninitialized, so they can be atomically
+// set.
+#define HWCAP_UNINITIALIZED (1 << 25)
+
+// A bit to flag when alignment faults are enabled and signal.
+#define HWCAP_ALIGNMENT_FAULT (1 << 26)
+
+// A bit to flag the use of the hardfp ABI.
+#define HWCAP_USE_HARDFP_ABI (1 << 27)
+
+// A bit to flag the use of the ARMv7 arch, otherwise ARMv6.
+#define HWCAP_ARMv7 (1 << 28)
+
+// Top three bits are reserved, do not use them.
+
+// Returns true when cpu alignment faults are enabled and signaled, and thus we
+// should ensure loads and stores are aligned.
+inline bool HasAlignmentFault() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_ALIGNMENT_FAULT;
+}
+
+#ifdef JS_SIMULATOR_ARM
+// Returns true when cpu alignment faults will be fixed up by the
+// "operating system", which functionality we will emulate.
+inline bool FixupFault() {
+ MOZ_ASSERT(armHwCapFlags != HWCAP_UNINITIALIZED);
+ return armHwCapFlags & HWCAP_FIXUP_FAULT;
+}
+#endif
+
+// Arm/D32 has double registers that can NOT be treated as float32 and this
+// requires some dances in lowering.
+inline bool hasUnaliasedDouble() { return Has32DP(); }
+
+// On ARM, Dn aliases both S2n and S2n+1, so if you need to convert a float32 to
+// a double as a temporary, you need a temporary double register.
+inline bool hasMultiAlias() { return true; }
+
+// InitARMFlags is called from the JitContext constructor to read the hardware
+// flags. The call is a no-op after the first call, or if the JS shell has
+// already set the flags (it has a command line switch for this, see
+// ParseARMHwCapFlags).
+//
+// If the environment variable ARMHWCAP is set then the flags are read from it
+// instead; see ParseARMHwCapFlags.
+void InitARMFlags();
+
+// Register a string denoting ARM hardware flags. During engine initialization,
+// these flags will then be used instead of the actual hardware capabilities.
+// This must be called before JS_Init and the passed string's buffer must
+// outlive the JS_Init call.
+void SetARMHwCapFlagsString(const char* armHwCap);
+
+// Retrive the ARM hardware flags at a bitmask. They must have been set.
+uint32_t GetARMFlags();
+
+// If the simulator is used then the ABI choice is dynamic. Otherwise the ABI is
+// static and useHardFpABI is inlined so that unused branches can be optimized
+// away.
+#ifdef JS_SIMULATOR_ARM
+bool UseHardFpABI();
+#else
+static inline bool UseHardFpABI() {
+# if defined(JS_CODEGEN_ARM_HARDFP)
+ return true;
+# else
+ return false;
+# endif
+}
+#endif
+
+// In order to handle SoftFp ABI calls, we need to be able to express that we
+// have ABIArg which are represented by pair of general purpose registers.
+#define JS_CODEGEN_REGISTER_PAIR 1
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_Architecture_arm_h */
diff --git a/js/src/jit/arm/Assembler-arm.cpp b/js/src/jit/arm/Assembler-arm.cpp
new file mode 100644
index 0000000000..a1213b6f21
--- /dev/null
+++ b/js/src/jit/arm/Assembler-arm.cpp
@@ -0,0 +1,2832 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm/Assembler-arm.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+#include "mozilla/Sprintf.h"
+
+#include <type_traits>
+
+#include "gc/Marking.h"
+#include "jit/arm/disasm/Disasm-arm.h"
+#include "jit/arm/MacroAssembler-arm.h"
+#include "jit/AutoWritableJitCode.h"
+#include "jit/ExecutableAllocator.h"
+#include "jit/MacroAssembler.h"
+#include "vm/Realm.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::CountLeadingZeroes32;
+using mozilla::DebugOnly;
+
+using LabelDoc = DisassemblerSpew::LabelDoc;
+using LiteralDoc = DisassemblerSpew::LiteralDoc;
+
+void dbg_break() {}
+
+// The ABIArgGenerator is used for making system ABI calls and for inter-wasm
+// calls. The system ABI can either be SoftFp or HardFp, and inter-wasm calls
+// are always HardFp calls. The initialization defaults to HardFp, and the ABI
+// choice is made before any system ABI calls with the method "setUseHardFp".
+ABIArgGenerator::ABIArgGenerator()
+ : intRegIndex_(0),
+ floatRegIndex_(0),
+ stackOffset_(0),
+ current_(),
+ useHardFp_(true) {}
+
+// See the "Parameter Passing" section of the "Procedure Call Standard for the
+// ARM Architecture" documentation.
+ABIArg ABIArgGenerator::softNext(MIRType type) {
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ if (intRegIndex_ == NumIntArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint32_t);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_));
+ intRegIndex_++;
+ break;
+ case MIRType::Int64:
+ // Make sure to use an even register index. Increase to next even number
+ // when odd.
+ intRegIndex_ = (intRegIndex_ + 1) & ~1;
+ if (intRegIndex_ == NumIntArgRegs) {
+ // Align the stack on 8 bytes.
+ static const uint32_t align = sizeof(uint64_t) - 1;
+ stackOffset_ = (stackOffset_ + align) & ~align;
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_),
+ Register::FromCode(intRegIndex_ + 1));
+ intRegIndex_ += 2;
+ break;
+ case MIRType::Float32:
+ if (intRegIndex_ == NumIntArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint32_t);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_));
+ intRegIndex_++;
+ break;
+ case MIRType::Double:
+ // Make sure to use an even register index. Increase to next even number
+ // when odd.
+ intRegIndex_ = (intRegIndex_ + 1) & ~1;
+ if (intRegIndex_ == NumIntArgRegs) {
+ // Align the stack on 8 bytes.
+ static const uint32_t align = sizeof(double) - 1;
+ stackOffset_ = (stackOffset_ + align) & ~align;
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(double);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_),
+ Register::FromCode(intRegIndex_ + 1));
+ intRegIndex_ += 2;
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+
+ return current_;
+}
+
+ABIArg ABIArgGenerator::hardNext(MIRType type) {
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ if (intRegIndex_ == NumIntArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint32_t);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_));
+ intRegIndex_++;
+ break;
+ case MIRType::Int64:
+ // Make sure to use an even register index. Increase to next even number
+ // when odd.
+ intRegIndex_ = (intRegIndex_ + 1) & ~1;
+ if (intRegIndex_ == NumIntArgRegs) {
+ // Align the stack on 8 bytes.
+ static const uint32_t align = sizeof(uint64_t) - 1;
+ stackOffset_ = (stackOffset_ + align) & ~align;
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_),
+ Register::FromCode(intRegIndex_ + 1));
+ intRegIndex_ += 2;
+ break;
+ case MIRType::Float32:
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint32_t);
+ break;
+ }
+ current_ = ABIArg(VFPRegister(floatRegIndex_, VFPRegister::Single));
+ floatRegIndex_++;
+ break;
+ case MIRType::Double:
+ // Double register are composed of 2 float registers, thus we have to
+ // skip any float register which cannot be used in a pair of float
+ // registers in which a double value can be stored.
+ floatRegIndex_ = (floatRegIndex_ + 1) & ~1;
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ static const uint32_t align = sizeof(double) - 1;
+ stackOffset_ = (stackOffset_ + align) & ~align;
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ break;
+ }
+ current_ = ABIArg(VFPRegister(floatRegIndex_ >> 1, VFPRegister::Double));
+ floatRegIndex_ += 2;
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+
+ return current_;
+}
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+ if (useHardFp_) {
+ return hardNext(type);
+ }
+ return softNext(type);
+}
+
+bool js::jit::IsUnaligned(const wasm::MemoryAccessDesc& access) {
+ if (!access.align()) {
+ return false;
+ }
+
+ if (access.type() == Scalar::Float64 && access.align() >= 4) {
+ return false;
+ }
+
+ return access.align() < access.byteSize();
+}
+
+// Encode a standard register when it is being used as src1, the dest, and an
+// extra register. These should never be called with an InvalidReg.
+uint32_t js::jit::RT(Register r) {
+ MOZ_ASSERT((r.code() & ~0xf) == 0);
+ return r.code() << 12;
+}
+
+uint32_t js::jit::RN(Register r) {
+ MOZ_ASSERT((r.code() & ~0xf) == 0);
+ return r.code() << 16;
+}
+
+uint32_t js::jit::RD(Register r) {
+ MOZ_ASSERT((r.code() & ~0xf) == 0);
+ return r.code() << 12;
+}
+
+uint32_t js::jit::RM(Register r) {
+ MOZ_ASSERT((r.code() & ~0xf) == 0);
+ return r.code() << 8;
+}
+
+// Encode a standard register when it is being used as src1, the dest, and an
+// extra register. For these, an InvalidReg is used to indicate a optional
+// register that has been omitted.
+uint32_t js::jit::maybeRT(Register r) {
+ if (r == InvalidReg) {
+ return 0;
+ }
+
+ MOZ_ASSERT((r.code() & ~0xf) == 0);
+ return r.code() << 12;
+}
+
+uint32_t js::jit::maybeRN(Register r) {
+ if (r == InvalidReg) {
+ return 0;
+ }
+
+ MOZ_ASSERT((r.code() & ~0xf) == 0);
+ return r.code() << 16;
+}
+
+uint32_t js::jit::maybeRD(Register r) {
+ if (r == InvalidReg) {
+ return 0;
+ }
+
+ MOZ_ASSERT((r.code() & ~0xf) == 0);
+ return r.code() << 12;
+}
+
+Register js::jit::toRD(Instruction i) {
+ return Register::FromCode((i.encode() >> 12) & 0xf);
+}
+Register js::jit::toR(Instruction i) {
+ return Register::FromCode(i.encode() & 0xf);
+}
+
+Register js::jit::toRM(Instruction i) {
+ return Register::FromCode((i.encode() >> 8) & 0xf);
+}
+
+Register js::jit::toRN(Instruction i) {
+ return Register::FromCode((i.encode() >> 16) & 0xf);
+}
+
+uint32_t js::jit::VD(VFPRegister vr) {
+ if (vr.isMissing()) {
+ return 0;
+ }
+
+ // Bits 15,14,13,12, 22.
+ VFPRegister::VFPRegIndexSplit s = vr.encode();
+ return s.bit << 22 | s.block << 12;
+}
+uint32_t js::jit::VN(VFPRegister vr) {
+ if (vr.isMissing()) {
+ return 0;
+ }
+
+ // Bits 19,18,17,16, 7.
+ VFPRegister::VFPRegIndexSplit s = vr.encode();
+ return s.bit << 7 | s.block << 16;
+}
+uint32_t js::jit::VM(VFPRegister vr) {
+ if (vr.isMissing()) {
+ return 0;
+ }
+
+ // Bits 5, 3,2,1,0.
+ VFPRegister::VFPRegIndexSplit s = vr.encode();
+ return s.bit << 5 | s.block;
+}
+
+VFPRegister::VFPRegIndexSplit jit::VFPRegister::encode() {
+ MOZ_ASSERT(!_isInvalid);
+
+ switch (kind) {
+ case Double:
+ return VFPRegIndexSplit(code_ & 0xf, code_ >> 4);
+ case Single:
+ return VFPRegIndexSplit(code_ >> 1, code_ & 1);
+ default:
+ // VFP register treated as an integer, NOT a gpr.
+ return VFPRegIndexSplit(code_ >> 1, code_ & 1);
+ }
+}
+
+bool InstDTR::IsTHIS(const Instruction& i) {
+ return (i.encode() & IsDTRMask) == (uint32_t)IsDTR;
+}
+
+InstDTR* InstDTR::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstDTR*)&i;
+ }
+ return nullptr;
+}
+
+bool InstLDR::IsTHIS(const Instruction& i) {
+ return (i.encode() & IsDTRMask) == (uint32_t)IsDTR;
+}
+
+InstLDR* InstLDR::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstLDR*)&i;
+ }
+ return nullptr;
+}
+
+InstNOP* InstNOP::AsTHIS(Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstNOP*)&i;
+ }
+ return nullptr;
+}
+
+bool InstNOP::IsTHIS(const Instruction& i) {
+ return (i.encode() & 0x0fffffff) == NopInst;
+}
+
+bool InstBranchReg::IsTHIS(const Instruction& i) {
+ return InstBXReg::IsTHIS(i) || InstBLXReg::IsTHIS(i);
+}
+
+InstBranchReg* InstBranchReg::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstBranchReg*)&i;
+ }
+ return nullptr;
+}
+void InstBranchReg::extractDest(Register* dest) { *dest = toR(*this); }
+bool InstBranchReg::checkDest(Register dest) { return dest == toR(*this); }
+
+bool InstBranchImm::IsTHIS(const Instruction& i) {
+ return InstBImm::IsTHIS(i) || InstBLImm::IsTHIS(i);
+}
+
+InstBranchImm* InstBranchImm::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstBranchImm*)&i;
+ }
+ return nullptr;
+}
+
+void InstBranchImm::extractImm(BOffImm* dest) { *dest = BOffImm(*this); }
+
+bool InstBXReg::IsTHIS(const Instruction& i) {
+ return (i.encode() & IsBRegMask) == IsBX;
+}
+
+InstBXReg* InstBXReg::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstBXReg*)&i;
+ }
+ return nullptr;
+}
+
+bool InstBLXReg::IsTHIS(const Instruction& i) {
+ return (i.encode() & IsBRegMask) == IsBLX;
+}
+InstBLXReg* InstBLXReg::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstBLXReg*)&i;
+ }
+ return nullptr;
+}
+
+bool InstBImm::IsTHIS(const Instruction& i) {
+ return (i.encode() & IsBImmMask) == IsB;
+}
+InstBImm* InstBImm::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstBImm*)&i;
+ }
+ return nullptr;
+}
+
+bool InstBLImm::IsTHIS(const Instruction& i) {
+ return (i.encode() & IsBImmMask) == IsBL;
+}
+InstBLImm* InstBLImm::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstBLImm*)&i;
+ }
+ return nullptr;
+}
+
+bool InstMovWT::IsTHIS(Instruction& i) {
+ return InstMovW::IsTHIS(i) || InstMovT::IsTHIS(i);
+}
+InstMovWT* InstMovWT::AsTHIS(Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstMovWT*)&i;
+ }
+ return nullptr;
+}
+
+void InstMovWT::extractImm(Imm16* imm) { *imm = Imm16(*this); }
+bool InstMovWT::checkImm(Imm16 imm) {
+ return imm.decode() == Imm16(*this).decode();
+}
+
+void InstMovWT::extractDest(Register* dest) { *dest = toRD(*this); }
+bool InstMovWT::checkDest(Register dest) { return dest == toRD(*this); }
+
+bool InstMovW::IsTHIS(const Instruction& i) {
+ return (i.encode() & IsWTMask) == IsW;
+}
+
+InstMovW* InstMovW::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstMovW*)&i;
+ }
+ return nullptr;
+}
+InstMovT* InstMovT::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstMovT*)&i;
+ }
+ return nullptr;
+}
+
+bool InstMovT::IsTHIS(const Instruction& i) {
+ return (i.encode() & IsWTMask) == IsT;
+}
+
+InstALU* InstALU::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstALU*)&i;
+ }
+ return nullptr;
+}
+bool InstALU::IsTHIS(const Instruction& i) {
+ return (i.encode() & ALUMask) == 0;
+}
+void InstALU::extractOp(ALUOp* ret) { *ret = ALUOp(encode() & (0xf << 21)); }
+bool InstALU::checkOp(ALUOp op) {
+ ALUOp mine;
+ extractOp(&mine);
+ return mine == op;
+}
+void InstALU::extractDest(Register* ret) { *ret = toRD(*this); }
+bool InstALU::checkDest(Register rd) { return rd == toRD(*this); }
+void InstALU::extractOp1(Register* ret) { *ret = toRN(*this); }
+bool InstALU::checkOp1(Register rn) { return rn == toRN(*this); }
+Operand2 InstALU::extractOp2() { return Operand2(encode()); }
+
+InstCMP* InstCMP::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstCMP*)&i;
+ }
+ return nullptr;
+}
+
+bool InstCMP::IsTHIS(const Instruction& i) {
+ return InstALU::IsTHIS(i) && InstALU::AsTHIS(i)->checkDest(r0) &&
+ InstALU::AsTHIS(i)->checkOp(OpCmp);
+}
+
+InstMOV* InstMOV::AsTHIS(const Instruction& i) {
+ if (IsTHIS(i)) {
+ return (InstMOV*)&i;
+ }
+ return nullptr;
+}
+
+bool InstMOV::IsTHIS(const Instruction& i) {
+ return InstALU::IsTHIS(i) && InstALU::AsTHIS(i)->checkOp1(r0) &&
+ InstALU::AsTHIS(i)->checkOp(OpMov);
+}
+
+Op2Reg Operand2::toOp2Reg() const { return *(Op2Reg*)this; }
+
+Imm16::Imm16(Instruction& inst)
+ : lower_(inst.encode() & 0xfff),
+ upper_(inst.encode() >> 16),
+ invalid_(0xfff) {}
+
+Imm16::Imm16(uint32_t imm)
+ : lower_(imm & 0xfff), pad_(0), upper_((imm >> 12) & 0xf), invalid_(0) {
+ MOZ_ASSERT(decode() == imm);
+}
+
+Imm16::Imm16() : invalid_(0xfff) {}
+
+void Assembler::finish() {
+ flush();
+ MOZ_ASSERT(!isFinished);
+ isFinished = true;
+}
+
+bool Assembler::appendRawCode(const uint8_t* code, size_t numBytes) {
+ flush();
+ return m_buffer.appendRawCode(code, numBytes);
+}
+
+bool Assembler::reserve(size_t size) {
+ // This buffer uses fixed-size chunks so there's no point in reserving
+ // now vs. on-demand.
+ return !oom();
+}
+
+bool Assembler::swapBuffer(wasm::Bytes& bytes) {
+ // For now, specialize to the one use case. As long as wasm::Bytes is a
+ // Vector, not a linked-list of chunks, there's not much we can do other
+ // than copy.
+ MOZ_ASSERT(bytes.empty());
+ if (!bytes.resize(bytesNeeded())) {
+ return false;
+ }
+ m_buffer.executableCopy(bytes.begin());
+ return true;
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ MOZ_ASSERT(isFinished);
+ m_buffer.executableCopy(buffer);
+}
+
+class RelocationIterator {
+ CompactBufferReader reader_;
+ // Offset in bytes.
+ uint32_t offset_;
+
+ public:
+ explicit RelocationIterator(CompactBufferReader& reader) : reader_(reader) {}
+
+ bool read() {
+ if (!reader_.more()) {
+ return false;
+ }
+ offset_ = reader_.readUnsigned();
+ return true;
+ }
+
+ uint32_t offset() const { return offset_; }
+};
+
+template <class Iter>
+const uint32_t* Assembler::GetCF32Target(Iter* iter) {
+ Instruction* inst1 = iter->cur();
+
+ if (inst1->is<InstBranchImm>()) {
+ // See if we have a simple case, b #offset.
+ BOffImm imm;
+ InstBranchImm* jumpB = inst1->as<InstBranchImm>();
+ jumpB->extractImm(&imm);
+ return imm.getDest(inst1)->raw();
+ }
+
+ if (inst1->is<InstMovW>()) {
+ // See if we have the complex case:
+ // movw r_temp, #imm1
+ // movt r_temp, #imm2
+ // bx r_temp
+ // OR
+ // movw r_temp, #imm1
+ // movt r_temp, #imm2
+ // str pc, [sp]
+ // bx r_temp
+
+ Imm16 targ_bot;
+ Imm16 targ_top;
+ Register temp;
+
+ // Extract both the temp register and the bottom immediate.
+ InstMovW* bottom = inst1->as<InstMovW>();
+ bottom->extractImm(&targ_bot);
+ bottom->extractDest(&temp);
+
+ // Extract the top part of the immediate.
+ Instruction* inst2 = iter->next();
+ MOZ_ASSERT(inst2->is<InstMovT>());
+ InstMovT* top = inst2->as<InstMovT>();
+ top->extractImm(&targ_top);
+
+ // Make sure they are being loaded into the same register.
+ MOZ_ASSERT(top->checkDest(temp));
+
+ // Make sure we're branching to the same register.
+#ifdef DEBUG
+ // A toggled call sometimes has a NOP instead of a branch for the third
+ // instruction. No way to assert that it's valid in that situation.
+ Instruction* inst3 = iter->next();
+ if (!inst3->is<InstNOP>()) {
+ InstBranchReg* realBranch = nullptr;
+ if (inst3->is<InstBranchReg>()) {
+ realBranch = inst3->as<InstBranchReg>();
+ } else {
+ Instruction* inst4 = iter->next();
+ realBranch = inst4->as<InstBranchReg>();
+ }
+ MOZ_ASSERT(realBranch->checkDest(temp));
+ }
+#endif
+
+ uint32_t* dest = (uint32_t*)(targ_bot.decode() | (targ_top.decode() << 16));
+ return dest;
+ }
+
+ if (inst1->is<InstLDR>()) {
+ return *(uint32_t**)inst1->as<InstLDR>()->dest();
+ }
+
+ MOZ_CRASH("unsupported branch relocation");
+}
+
+uintptr_t Assembler::GetPointer(uint8_t* instPtr) {
+ InstructionIterator iter((Instruction*)instPtr);
+ uintptr_t ret = (uintptr_t)GetPtr32Target(iter, nullptr, nullptr);
+ return ret;
+}
+
+const uint32_t* Assembler::GetPtr32Target(InstructionIterator start,
+ Register* dest, RelocStyle* style) {
+ Instruction* load1 = start.cur();
+ Instruction* load2 = start.next();
+
+ if (load1->is<InstMovW>() && load2->is<InstMovT>()) {
+ if (style) {
+ *style = L_MOVWT;
+ }
+
+ // See if we have the complex case:
+ // movw r_temp, #imm1
+ // movt r_temp, #imm2
+
+ Imm16 targ_bot;
+ Imm16 targ_top;
+ Register temp;
+
+ // Extract both the temp register and the bottom immediate.
+ InstMovW* bottom = load1->as<InstMovW>();
+ bottom->extractImm(&targ_bot);
+ bottom->extractDest(&temp);
+
+ // Extract the top part of the immediate.
+ InstMovT* top = load2->as<InstMovT>();
+ top->extractImm(&targ_top);
+
+ // Make sure they are being loaded into the same register.
+ MOZ_ASSERT(top->checkDest(temp));
+
+ if (dest) {
+ *dest = temp;
+ }
+
+ uint32_t* value =
+ (uint32_t*)(targ_bot.decode() | (targ_top.decode() << 16));
+ return value;
+ }
+
+ if (load1->is<InstLDR>()) {
+ if (style) {
+ *style = L_LDR;
+ }
+ if (dest) {
+ *dest = toRD(*load1);
+ }
+ return *(uint32_t**)load1->as<InstLDR>()->dest();
+ }
+
+ MOZ_CRASH("unsupported relocation");
+}
+
+static JitCode* CodeFromJump(InstructionIterator* jump) {
+ uint8_t* target = (uint8_t*)Assembler::GetCF32Target(jump);
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ RelocationIterator iter(reader);
+ while (iter.read()) {
+ InstructionIterator institer((Instruction*)(code->raw() + iter.offset()));
+ JitCode* child = CodeFromJump(&institer);
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ }
+}
+
+static void TraceOneDataRelocation(JSTracer* trc,
+ mozilla::Maybe<AutoWritableJitCode>& awjc,
+ JitCode* code, InstructionIterator iter) {
+ Register dest;
+ Assembler::RelocStyle rs;
+ const void* prior = Assembler::GetPtr32Target(iter, &dest, &rs);
+ void* ptr = const_cast<void*>(prior);
+
+ // No barrier needed since these are constants.
+ TraceManuallyBarrieredGenericPointerEdge(
+ trc, reinterpret_cast<gc::Cell**>(&ptr), "jit-masm-ptr");
+
+ if (ptr != prior) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+
+ MacroAssemblerARM::ma_mov_patch(Imm32(int32_t(ptr)), dest,
+ Assembler::Always, rs, iter);
+ }
+}
+
+/* static */
+void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ InstructionIterator iter((Instruction*)(code->raw() + offset));
+ TraceOneDataRelocation(trc, awjc, code, iter);
+ }
+}
+
+void Assembler::copyJumpRelocationTable(uint8_t* dest) {
+ if (jumpRelocations_.length()) {
+ memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
+ }
+}
+
+void Assembler::copyDataRelocationTable(uint8_t* dest) {
+ if (dataRelocations_.length()) {
+ memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
+ }
+}
+
+void Assembler::processCodeLabels(uint8_t* rawCode) {
+ for (const CodeLabel& label : codeLabels_) {
+ Bind(rawCode, label);
+ }
+}
+
+void Assembler::writeCodePointer(CodeLabel* label) {
+ m_buffer.assertNoPoolAndNoNops();
+ BufferOffset off = writeInst(-1);
+ label->patchAt()->bind(off.getOffset());
+}
+
+void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) {
+ size_t offset = label.patchAt().offset();
+ size_t target = label.target().offset();
+ *reinterpret_cast<const void**>(rawCode + offset) = rawCode + target;
+}
+
+Assembler::Condition Assembler::InvertCondition(Condition cond) {
+ const uint32_t ConditionInversionBit = 0x10000000;
+ return Condition(ConditionInversionBit ^ cond);
+}
+
+Assembler::Condition Assembler::UnsignedCondition(Condition cond) {
+ switch (cond) {
+ case Zero:
+ case NonZero:
+ return cond;
+ case LessThan:
+ case Below:
+ return Below;
+ case LessThanOrEqual:
+ case BelowOrEqual:
+ return BelowOrEqual;
+ case GreaterThan:
+ case Above:
+ return Above;
+ case AboveOrEqual:
+ case GreaterThanOrEqual:
+ return AboveOrEqual;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+Assembler::Condition Assembler::ConditionWithoutEqual(Condition cond) {
+ switch (cond) {
+ case LessThan:
+ case LessThanOrEqual:
+ return LessThan;
+ case Below:
+ case BelowOrEqual:
+ return Below;
+ case GreaterThan:
+ case GreaterThanOrEqual:
+ return GreaterThan;
+ case Above:
+ case AboveOrEqual:
+ return Above;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+Assembler::DoubleCondition Assembler::InvertCondition(DoubleCondition cond) {
+ const uint32_t ConditionInversionBit = 0x10000000;
+ return DoubleCondition(ConditionInversionBit ^ cond);
+}
+
+Imm8::TwoImm8mData Imm8::EncodeTwoImms(uint32_t imm) {
+ // In the ideal case, we are looking for a number that (in binary) looks
+ // like:
+ // 0b((00)*)n_1((00)*)n_2((00)*)
+ // left n1 mid n2
+ // where both n_1 and n_2 fit into 8 bits.
+ // Since this is being done with rotates, we also need to handle the case
+ // that one of these numbers is in fact split between the left and right
+ // sides, in which case the constant will look like:
+ // 0bn_1a((00)*)n_2((00)*)n_1b
+ // n1a mid n2 rgh n1b
+ // Also remember, values are rotated by multiples of two, and left, mid or
+ // right can have length zero.
+ uint32_t imm1, imm2;
+ int left = CountLeadingZeroes32(imm) & 0x1E;
+ uint32_t no_n1 = imm & ~(0xff << (24 - left));
+
+ // Not technically needed: this case only happens if we can encode as a
+ // single imm8m. There is a perfectly reasonable encoding in this case, but
+ // we shouldn't encourage people to do things like this.
+ if (no_n1 == 0) {
+ return TwoImm8mData();
+ }
+
+ int mid = CountLeadingZeroes32(no_n1) & 0x1E;
+ uint32_t no_n2 =
+ no_n1 & ~((0xff << ((24 - mid) & 0x1f)) | 0xff >> ((8 + mid) & 0x1f));
+
+ if (no_n2 == 0) {
+ // We hit the easy case, no wraparound.
+ // Note: a single constant *may* look like this.
+ int imm1shift = left + 8;
+ int imm2shift = mid + 8;
+ imm1 = (imm >> (32 - imm1shift)) & 0xff;
+ if (imm2shift >= 32) {
+ imm2shift = 0;
+ // This assert does not always hold, in fact, this would lead to
+ // some incredibly subtle bugs.
+ // assert((imm & 0xff) == no_n1);
+ imm2 = no_n1;
+ } else {
+ imm2 = ((imm >> (32 - imm2shift)) | (imm << imm2shift)) & 0xff;
+ MOZ_ASSERT(((no_n1 >> (32 - imm2shift)) | (no_n1 << imm2shift)) == imm2);
+ }
+ MOZ_ASSERT((imm1shift & 0x1) == 0);
+ MOZ_ASSERT((imm2shift & 0x1) == 0);
+ return TwoImm8mData(datastore::Imm8mData(imm1, imm1shift >> 1),
+ datastore::Imm8mData(imm2, imm2shift >> 1));
+ }
+
+ // Either it wraps, or it does not fit. If we initially chopped off more
+ // than 8 bits, then it won't fit.
+ if (left >= 8) {
+ return TwoImm8mData();
+ }
+
+ int right = 32 - (CountLeadingZeroes32(no_n2) & 30);
+ // All remaining set bits *must* fit into the lower 8 bits.
+ // The right == 8 case should be handled by the previous case.
+ if (right > 8) {
+ return TwoImm8mData();
+ }
+
+ // Make sure the initial bits that we removed for no_n1 fit into the
+ // 8-(32-right) leftmost bits.
+ if (((imm & (0xff << (24 - left))) << (8 - right)) != 0) {
+ // BUT we may have removed more bits than we needed to for no_n1
+ // 0x04104001 e.g. we can encode 0x104 with a single op, then 0x04000001
+ // with a second, but we try to encode 0x0410000 and find that we need a
+ // second op for 0x4000, and 0x1 cannot be included in the encoding of
+ // 0x04100000.
+ no_n1 = imm & ~((0xff >> (8 - right)) | (0xff << (24 + right)));
+ mid = CountLeadingZeroes32(no_n1) & 30;
+ no_n2 = no_n1 & ~((0xff << ((24 - mid) & 31)) | 0xff >> ((8 + mid) & 31));
+ if (no_n2 != 0) {
+ return TwoImm8mData();
+ }
+ }
+
+ // Now assemble all of this information into a two coherent constants it is
+ // a rotate right from the lower 8 bits.
+ int imm1shift = 8 - right;
+ imm1 = 0xff & ((imm << imm1shift) | (imm >> (32 - imm1shift)));
+ MOZ_ASSERT((imm1shift & ~0x1e) == 0);
+ // left + 8 + mid is the position of the leftmost bit of n_2.
+ // We needed to rotate 0x000000ab right by 8 in order to get 0xab000000,
+ // then shift again by the leftmost bit in order to get the constant that we
+ // care about.
+ int imm2shift = mid + 8;
+ imm2 = ((imm >> (32 - imm2shift)) | (imm << imm2shift)) & 0xff;
+ MOZ_ASSERT((imm1shift & 0x1) == 0);
+ MOZ_ASSERT((imm2shift & 0x1) == 0);
+ return TwoImm8mData(datastore::Imm8mData(imm1, imm1shift >> 1),
+ datastore::Imm8mData(imm2, imm2shift >> 1));
+}
+
+ALUOp jit::ALUNeg(ALUOp op, Register dest, Register scratch, Imm32* imm,
+ Register* negDest) {
+ // Find an alternate ALUOp to get the job done, and use a different imm.
+ *negDest = dest;
+ switch (op) {
+ case OpMov:
+ *imm = Imm32(~imm->value);
+ return OpMvn;
+ case OpMvn:
+ *imm = Imm32(~imm->value);
+ return OpMov;
+ case OpAnd:
+ *imm = Imm32(~imm->value);
+ return OpBic;
+ case OpBic:
+ *imm = Imm32(~imm->value);
+ return OpAnd;
+ case OpAdd:
+ *imm = Imm32(-imm->value);
+ return OpSub;
+ case OpSub:
+ *imm = Imm32(-imm->value);
+ return OpAdd;
+ case OpCmp:
+ *imm = Imm32(-imm->value);
+ return OpCmn;
+ case OpCmn:
+ *imm = Imm32(-imm->value);
+ return OpCmp;
+ case OpTst:
+ MOZ_ASSERT(dest == InvalidReg);
+ *imm = Imm32(~imm->value);
+ *negDest = scratch;
+ return OpBic;
+ // orr has orn on thumb2 only.
+ default:
+ return OpInvalid;
+ }
+}
+
+bool jit::can_dbl(ALUOp op) {
+ // Some instructions can't be processed as two separate instructions such as
+ // and, and possibly add (when we're setting ccodes). There is also some
+ // hilarity with *reading* condition codes. For example, adc dest, src1,
+ // 0xfff; (add with carry) can be split up into adc dest, src1, 0xf00; add
+ // dest, dest, 0xff, since "reading" the condition code increments the
+ // result by one conditionally, that only needs to be done on one of the two
+ // instructions.
+ switch (op) {
+ case OpBic:
+ case OpAdd:
+ case OpSub:
+ case OpEor:
+ case OpOrr:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool jit::condsAreSafe(ALUOp op) {
+ // Even when we are setting condition codes, sometimes we can get away with
+ // splitting an operation into two. For example, if our immediate is
+ // 0x00ff00ff, and the operation is eors we can split this in half, since x
+ // ^ 0x00ff0000 ^ 0x000000ff should set all of its condition codes exactly
+ // the same as x ^ 0x00ff00ff. However, if the operation were adds, we
+ // cannot split this in half. If the source on the add is 0xfff00ff0, the
+ // result sholud be 0xef10ef, but do we set the overflow bit or not?
+ // Depending on which half is performed first (0x00ff0000 or 0x000000ff) the
+ // V bit will be set differently, and *not* updating the V bit would be
+ // wrong. Theoretically, the following should work:
+ // adds r0, r1, 0x00ff0000;
+ // addsvs r0, r1, 0x000000ff;
+ // addvc r0, r1, 0x000000ff;
+ // But this is 3 instructions, and at that point, we might as well use
+ // something else.
+ switch (op) {
+ case OpBic:
+ case OpOrr:
+ case OpEor:
+ return true;
+ default:
+ return false;
+ }
+}
+
+ALUOp jit::getDestVariant(ALUOp op) {
+ // All of the compare operations are dest-less variants of a standard
+ // operation. Given the dest-less variant, return the dest-ful variant.
+ switch (op) {
+ case OpCmp:
+ return OpSub;
+ case OpCmn:
+ return OpAdd;
+ case OpTst:
+ return OpAnd;
+ case OpTeq:
+ return OpEor;
+ default:
+ return op;
+ }
+}
+
+O2RegImmShift jit::O2Reg(Register r) { return O2RegImmShift(r, LSL, 0); }
+
+O2RegImmShift jit::lsl(Register r, int amt) {
+ MOZ_ASSERT(0 <= amt && amt <= 31);
+ return O2RegImmShift(r, LSL, amt);
+}
+
+O2RegImmShift jit::lsr(Register r, int amt) {
+ MOZ_ASSERT(1 <= amt && amt <= 32);
+ return O2RegImmShift(r, LSR, amt);
+}
+
+O2RegImmShift jit::ror(Register r, int amt) {
+ MOZ_ASSERT(1 <= amt && amt <= 31);
+ return O2RegImmShift(r, ROR, amt);
+}
+O2RegImmShift jit::rol(Register r, int amt) {
+ MOZ_ASSERT(1 <= amt && amt <= 31);
+ return O2RegImmShift(r, ROR, 32 - amt);
+}
+
+O2RegImmShift jit::asr(Register r, int amt) {
+ MOZ_ASSERT(1 <= amt && amt <= 32);
+ return O2RegImmShift(r, ASR, amt);
+}
+
+O2RegRegShift jit::lsl(Register r, Register amt) {
+ return O2RegRegShift(r, LSL, amt);
+}
+
+O2RegRegShift jit::lsr(Register r, Register amt) {
+ return O2RegRegShift(r, LSR, amt);
+}
+
+O2RegRegShift jit::ror(Register r, Register amt) {
+ return O2RegRegShift(r, ROR, amt);
+}
+
+O2RegRegShift jit::asr(Register r, Register amt) {
+ return O2RegRegShift(r, ASR, amt);
+}
+
+static js::jit::DoubleEncoder doubleEncoder;
+
+/* static */
+const js::jit::VFPImm js::jit::VFPImm::One(0x3FF00000);
+
+js::jit::VFPImm::VFPImm(uint32_t top) {
+ data_ = -1;
+ datastore::Imm8VFPImmData tmp;
+ if (doubleEncoder.lookup(top, &tmp)) {
+ data_ = tmp.encode();
+ }
+}
+
+BOffImm::BOffImm(const Instruction& inst) : data_(inst.encode() & 0x00ffffff) {}
+
+Instruction* BOffImm::getDest(Instruction* src) const {
+ // TODO: It is probably worthwhile to verify that src is actually a branch.
+ // NOTE: This does not explicitly shift the offset of the destination left by
+ // 2, since it is indexing into an array of instruction sized objects.
+ return &src[((int32_t(data_) << 8) >> 8) + 2];
+}
+
+const js::jit::DoubleEncoder::DoubleEntry js::jit::DoubleEncoder::table[256] = {
+#include "jit/arm/DoubleEntryTable.tbl"
+};
+
+// VFPRegister implementation
+VFPRegister VFPRegister::doubleOverlay(unsigned int which) const {
+ MOZ_ASSERT(!_isInvalid);
+ MOZ_ASSERT(which == 0);
+ if (kind != Double) {
+ return VFPRegister(code_ >> 1, Double);
+ }
+ return *this;
+}
+VFPRegister VFPRegister::singleOverlay(unsigned int which) const {
+ MOZ_ASSERT(!_isInvalid);
+ if (kind == Double) {
+ // There are no corresponding float registers for d16-d31.
+ MOZ_ASSERT(code_ < 16);
+ MOZ_ASSERT(which < 2);
+ return VFPRegister((code_ << 1) + which, Single);
+ }
+ MOZ_ASSERT(which == 0);
+ return VFPRegister(code_, Single);
+}
+
+static_assert(
+ FloatRegisters::TotalDouble <= 16,
+ "We assume that every Double register also has an Integer personality");
+
+VFPRegister VFPRegister::sintOverlay(unsigned int which) const {
+ MOZ_ASSERT(!_isInvalid);
+ if (kind == Double) {
+ // There are no corresponding float registers for d16-d31.
+ MOZ_ASSERT(code_ < 16);
+ MOZ_ASSERT(which < 2);
+ return VFPRegister((code_ << 1) + which, Int);
+ }
+ MOZ_ASSERT(which == 0);
+ return VFPRegister(code_, Int);
+}
+VFPRegister VFPRegister::uintOverlay(unsigned int which) const {
+ MOZ_ASSERT(!_isInvalid);
+ if (kind == Double) {
+ // There are no corresponding float registers for d16-d31.
+ MOZ_ASSERT(code_ < 16);
+ MOZ_ASSERT(which < 2);
+ return VFPRegister((code_ << 1) + which, UInt);
+ }
+ MOZ_ASSERT(which == 0);
+ return VFPRegister(code_, UInt);
+}
+
+bool Assembler::oom() const {
+ return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() ||
+ dataRelocations_.oom();
+}
+
+// Size of the instruction stream, in bytes. Including pools. This function
+// expects all pools that need to be placed have been placed. If they haven't
+// then we need to go an flush the pools :(
+size_t Assembler::size() const { return m_buffer.size(); }
+// Size of the relocation table, in bytes.
+size_t Assembler::jumpRelocationTableBytes() const {
+ return jumpRelocations_.length();
+}
+size_t Assembler::dataRelocationTableBytes() const {
+ return dataRelocations_.length();
+}
+
+// Size of the data table, in bytes.
+size_t Assembler::bytesNeeded() const {
+ return size() + jumpRelocationTableBytes() + dataRelocationTableBytes();
+}
+
+// Allocate memory for a branch instruction, it will be overwritten
+// subsequently and should not be disassembled.
+
+BufferOffset Assembler::allocBranchInst() {
+ return m_buffer.putInt(Always | InstNOP::NopInst);
+}
+
+void Assembler::WriteInstStatic(uint32_t x, uint32_t* dest) {
+ MOZ_ASSERT(dest != nullptr);
+ *dest = x;
+}
+
+void Assembler::haltingAlign(int alignment) {
+ // HLT with payload 0xBAAD
+ m_buffer.align(alignment, 0xE1000070 | (0xBAA << 8) | 0xD);
+}
+
+void Assembler::nopAlign(int alignment) { m_buffer.align(alignment); }
+
+BufferOffset Assembler::as_nop() { return writeInst(0xe320f000); }
+
+static uint32_t EncodeAlu(Register dest, Register src1, Operand2 op2, ALUOp op,
+ SBit s, Assembler::Condition c) {
+ return (int)op | (int)s | (int)c | op2.encode() |
+ ((dest == InvalidReg) ? 0 : RD(dest)) |
+ ((src1 == InvalidReg) ? 0 : RN(src1));
+}
+
+BufferOffset Assembler::as_alu(Register dest, Register src1, Operand2 op2,
+ ALUOp op, SBit s, Condition c) {
+ return writeInst(EncodeAlu(dest, src1, op2, op, s, c));
+}
+
+BufferOffset Assembler::as_mov(Register dest, Operand2 op2, SBit s,
+ Condition c) {
+ return as_alu(dest, InvalidReg, op2, OpMov, s, c);
+}
+
+/* static */
+void Assembler::as_alu_patch(Register dest, Register src1, Operand2 op2,
+ ALUOp op, SBit s, Condition c, uint32_t* pos) {
+ WriteInstStatic(EncodeAlu(dest, src1, op2, op, s, c), pos);
+}
+
+/* static */
+void Assembler::as_mov_patch(Register dest, Operand2 op2, SBit s, Condition c,
+ uint32_t* pos) {
+ as_alu_patch(dest, InvalidReg, op2, OpMov, s, c, pos);
+}
+
+BufferOffset Assembler::as_mvn(Register dest, Operand2 op2, SBit s,
+ Condition c) {
+ return as_alu(dest, InvalidReg, op2, OpMvn, s, c);
+}
+
+// Logical operations.
+BufferOffset Assembler::as_and(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpAnd, s, c);
+}
+BufferOffset Assembler::as_bic(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpBic, s, c);
+}
+BufferOffset Assembler::as_eor(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpEor, s, c);
+}
+BufferOffset Assembler::as_orr(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpOrr, s, c);
+}
+
+// Reverse byte operations.
+BufferOffset Assembler::as_rev(Register dest, Register src, Condition c) {
+ return writeInst((int)c | 0b0000'0110'1011'1111'0000'1111'0011'0000 |
+ RD(dest) | src.code());
+}
+BufferOffset Assembler::as_rev16(Register dest, Register src, Condition c) {
+ return writeInst((int)c | 0b0000'0110'1011'1111'0000'1111'1011'0000 |
+ RD(dest) | src.code());
+}
+BufferOffset Assembler::as_revsh(Register dest, Register src, Condition c) {
+ return writeInst((int)c | 0b0000'0110'1111'1111'0000'1111'1011'0000 |
+ RD(dest) | src.code());
+}
+
+// Mathematical operations.
+BufferOffset Assembler::as_adc(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpAdc, s, c);
+}
+BufferOffset Assembler::as_add(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpAdd, s, c);
+}
+BufferOffset Assembler::as_sbc(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpSbc, s, c);
+}
+BufferOffset Assembler::as_sub(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpSub, s, c);
+}
+BufferOffset Assembler::as_rsb(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpRsb, s, c);
+}
+BufferOffset Assembler::as_rsc(Register dest, Register src1, Operand2 op2,
+ SBit s, Condition c) {
+ return as_alu(dest, src1, op2, OpRsc, s, c);
+}
+
+// Test operations.
+BufferOffset Assembler::as_cmn(Register src1, Operand2 op2, Condition c) {
+ return as_alu(InvalidReg, src1, op2, OpCmn, SetCC, c);
+}
+BufferOffset Assembler::as_cmp(Register src1, Operand2 op2, Condition c) {
+ return as_alu(InvalidReg, src1, op2, OpCmp, SetCC, c);
+}
+BufferOffset Assembler::as_teq(Register src1, Operand2 op2, Condition c) {
+ return as_alu(InvalidReg, src1, op2, OpTeq, SetCC, c);
+}
+BufferOffset Assembler::as_tst(Register src1, Operand2 op2, Condition c) {
+ return as_alu(InvalidReg, src1, op2, OpTst, SetCC, c);
+}
+
+static constexpr Register NoAddend{Registers::pc};
+
+static const int SignExtend = 0x06000070;
+
+enum SignExtend {
+ SxSxtb = 10 << 20,
+ SxSxth = 11 << 20,
+ SxUxtb = 14 << 20,
+ SxUxth = 15 << 20
+};
+
+// Sign extension operations.
+BufferOffset Assembler::as_sxtb(Register dest, Register src, int rotate,
+ Condition c) {
+ return writeInst((int)c | SignExtend | SxSxtb | RN(NoAddend) | RD(dest) |
+ ((rotate & 3) << 10) | src.code());
+}
+BufferOffset Assembler::as_sxth(Register dest, Register src, int rotate,
+ Condition c) {
+ return writeInst((int)c | SignExtend | SxSxth | RN(NoAddend) | RD(dest) |
+ ((rotate & 3) << 10) | src.code());
+}
+BufferOffset Assembler::as_uxtb(Register dest, Register src, int rotate,
+ Condition c) {
+ return writeInst((int)c | SignExtend | SxUxtb | RN(NoAddend) | RD(dest) |
+ ((rotate & 3) << 10) | src.code());
+}
+BufferOffset Assembler::as_uxth(Register dest, Register src, int rotate,
+ Condition c) {
+ return writeInst((int)c | SignExtend | SxUxth | RN(NoAddend) | RD(dest) |
+ ((rotate & 3) << 10) | src.code());
+}
+
+static uint32_t EncodeMovW(Register dest, Imm16 imm, Assembler::Condition c) {
+ MOZ_ASSERT(HasMOVWT());
+ return 0x03000000 | c | imm.encode() | RD(dest);
+}
+
+static uint32_t EncodeMovT(Register dest, Imm16 imm, Assembler::Condition c) {
+ MOZ_ASSERT(HasMOVWT());
+ return 0x03400000 | c | imm.encode() | RD(dest);
+}
+
+// Not quite ALU worthy, but these are useful none the less. These also have
+// the isue of these being formatted completly differently from the standard ALU
+// operations.
+BufferOffset Assembler::as_movw(Register dest, Imm16 imm, Condition c) {
+ return writeInst(EncodeMovW(dest, imm, c));
+}
+
+/* static */
+void Assembler::as_movw_patch(Register dest, Imm16 imm, Condition c,
+ Instruction* pos) {
+ WriteInstStatic(EncodeMovW(dest, imm, c), (uint32_t*)pos);
+}
+
+BufferOffset Assembler::as_movt(Register dest, Imm16 imm, Condition c) {
+ return writeInst(EncodeMovT(dest, imm, c));
+}
+
+/* static */
+void Assembler::as_movt_patch(Register dest, Imm16 imm, Condition c,
+ Instruction* pos) {
+ WriteInstStatic(EncodeMovT(dest, imm, c), (uint32_t*)pos);
+}
+
+static const int mull_tag = 0x90;
+
+BufferOffset Assembler::as_genmul(Register dhi, Register dlo, Register rm,
+ Register rn, MULOp op, SBit s, Condition c) {
+ return writeInst(RN(dhi) | maybeRD(dlo) | RM(rm) | rn.code() | op | s | c |
+ mull_tag);
+}
+BufferOffset Assembler::as_mul(Register dest, Register src1, Register src2,
+ SBit s, Condition c) {
+ return as_genmul(dest, InvalidReg, src1, src2, OpmMul, s, c);
+}
+BufferOffset Assembler::as_mla(Register dest, Register acc, Register src1,
+ Register src2, SBit s, Condition c) {
+ return as_genmul(dest, acc, src1, src2, OpmMla, s, c);
+}
+BufferOffset Assembler::as_umaal(Register destHI, Register destLO,
+ Register src1, Register src2, Condition c) {
+ return as_genmul(destHI, destLO, src1, src2, OpmUmaal, LeaveCC, c);
+}
+BufferOffset Assembler::as_mls(Register dest, Register acc, Register src1,
+ Register src2, Condition c) {
+ return as_genmul(dest, acc, src1, src2, OpmMls, LeaveCC, c);
+}
+
+BufferOffset Assembler::as_umull(Register destHI, Register destLO,
+ Register src1, Register src2, SBit s,
+ Condition c) {
+ return as_genmul(destHI, destLO, src1, src2, OpmUmull, s, c);
+}
+
+BufferOffset Assembler::as_umlal(Register destHI, Register destLO,
+ Register src1, Register src2, SBit s,
+ Condition c) {
+ return as_genmul(destHI, destLO, src1, src2, OpmUmlal, s, c);
+}
+
+BufferOffset Assembler::as_smull(Register destHI, Register destLO,
+ Register src1, Register src2, SBit s,
+ Condition c) {
+ return as_genmul(destHI, destLO, src1, src2, OpmSmull, s, c);
+}
+
+BufferOffset Assembler::as_smlal(Register destHI, Register destLO,
+ Register src1, Register src2, SBit s,
+ Condition c) {
+ return as_genmul(destHI, destLO, src1, src2, OpmSmlal, s, c);
+}
+
+BufferOffset Assembler::as_sdiv(Register rd, Register rn, Register rm,
+ Condition c) {
+ return writeInst(0x0710f010 | c | RN(rd) | RM(rm) | rn.code());
+}
+
+BufferOffset Assembler::as_udiv(Register rd, Register rn, Register rm,
+ Condition c) {
+ return writeInst(0x0730f010 | c | RN(rd) | RM(rm) | rn.code());
+}
+
+BufferOffset Assembler::as_clz(Register dest, Register src, Condition c) {
+ MOZ_ASSERT(src != pc && dest != pc);
+ return writeInst(RD(dest) | src.code() | c | 0x016f0f10);
+}
+
+// Data transfer instructions: ldr, str, ldrb, strb. Using an int to
+// differentiate between 8 bits and 32 bits is overkill, but meh.
+
+static uint32_t EncodeDtr(LoadStore ls, int size, Index mode, Register rt,
+ DTRAddr addr, Assembler::Condition c) {
+ MOZ_ASSERT(mode == Offset || (rt != addr.getBase() && pc != addr.getBase()));
+ MOZ_ASSERT(size == 32 || size == 8);
+ return 0x04000000 | ls | (size == 8 ? 0x00400000 : 0) | mode | c | RT(rt) |
+ addr.encode();
+}
+
+BufferOffset Assembler::as_dtr(LoadStore ls, int size, Index mode, Register rt,
+ DTRAddr addr, Condition c) {
+ return writeInst(EncodeDtr(ls, size, mode, rt, addr, c));
+}
+
+/* static */
+void Assembler::as_dtr_patch(LoadStore ls, int size, Index mode, Register rt,
+ DTRAddr addr, Condition c, uint32_t* dest) {
+ WriteInstStatic(EncodeDtr(ls, size, mode, rt, addr, c), dest);
+}
+
+class PoolHintData {
+ public:
+ enum LoadType {
+ // Set 0 to bogus, since that is the value most likely to be
+ // accidentally left somewhere.
+ PoolBOGUS = 0,
+ PoolDTR = 1,
+ PoolBranch = 2,
+ PoolVDTR = 3
+ };
+
+ private:
+ uint32_t index_ : 16;
+ uint32_t cond_ : 4;
+ uint32_t loadType_ : 2;
+ uint32_t destReg_ : 5;
+ uint32_t destType_ : 1;
+ uint32_t ONES : 4;
+
+ static const uint32_t ExpectedOnes = 0xfu;
+
+ public:
+ void init(uint32_t index, Assembler::Condition cond, LoadType lt,
+ Register destReg) {
+ index_ = index;
+ MOZ_ASSERT(index_ == index);
+ cond_ = cond >> 28;
+ MOZ_ASSERT(cond_ == cond >> 28);
+ loadType_ = lt;
+ ONES = ExpectedOnes;
+ destReg_ = destReg.code();
+ destType_ = 0;
+ }
+ void init(uint32_t index, Assembler::Condition cond, LoadType lt,
+ const VFPRegister& destReg) {
+ MOZ_ASSERT(destReg.isFloat());
+ index_ = index;
+ MOZ_ASSERT(index_ == index);
+ cond_ = cond >> 28;
+ MOZ_ASSERT(cond_ == cond >> 28);
+ loadType_ = lt;
+ ONES = ExpectedOnes;
+ destReg_ = destReg.id();
+ destType_ = destReg.isDouble();
+ }
+ Assembler::Condition getCond() const {
+ return Assembler::Condition(cond_ << 28);
+ }
+
+ Register getReg() const { return Register::FromCode(destReg_); }
+ VFPRegister getVFPReg() const {
+ VFPRegister r = VFPRegister(
+ destReg_, destType_ ? VFPRegister::Double : VFPRegister::Single);
+ return r;
+ }
+
+ int32_t getIndex() const { return index_; }
+ void setIndex(uint32_t index) {
+ MOZ_ASSERT(ONES == ExpectedOnes && loadType_ != PoolBOGUS);
+ index_ = index;
+ MOZ_ASSERT(index_ == index);
+ }
+
+ LoadType getLoadType() const {
+ // If this *was* a PoolBranch, but the branch has already been bound
+ // then this isn't going to look like a real poolhintdata, but we still
+ // want to lie about it so everyone knows it *used* to be a branch.
+ if (ONES != ExpectedOnes) {
+ return PoolHintData::PoolBranch;
+ }
+ return static_cast<LoadType>(loadType_);
+ }
+
+ bool isValidPoolHint() const {
+ // Most instructions cannot have a condition that is 0xf. Notable
+ // exceptions are blx and the entire NEON instruction set. For the
+ // purposes of pool loads, and possibly patched branches, the possible
+ // instructions are ldr and b, neither of which can have a condition
+ // code of 0xf.
+ return ONES == ExpectedOnes;
+ }
+};
+
+union PoolHintPun {
+ PoolHintData phd;
+ uint32_t raw;
+};
+
+// Handles all of the other integral data transferring functions: ldrsb, ldrsh,
+// ldrd, etc. The size is given in bits.
+BufferOffset Assembler::as_extdtr(LoadStore ls, int size, bool IsSigned,
+ Index mode, Register rt, EDtrAddr addr,
+ Condition c) {
+ int extra_bits2 = 0;
+ int extra_bits1 = 0;
+ switch (size) {
+ case 8:
+ MOZ_ASSERT(IsSigned);
+ MOZ_ASSERT(ls != IsStore);
+ extra_bits1 = 0x1;
+ extra_bits2 = 0x2;
+ break;
+ case 16:
+ // 'case 32' doesn't need to be handled, it is handled by the default
+ // ldr/str.
+ extra_bits2 = 0x01;
+ extra_bits1 = (ls == IsStore) ? 0 : 1;
+ if (IsSigned) {
+ MOZ_ASSERT(ls != IsStore);
+ extra_bits2 |= 0x2;
+ }
+ break;
+ case 64:
+ extra_bits2 = (ls == IsStore) ? 0x3 : 0x2;
+ extra_bits1 = 0;
+ break;
+ default:
+ MOZ_CRASH("unexpected size in as_extdtr");
+ }
+ return writeInst(extra_bits2 << 5 | extra_bits1 << 20 | 0x90 | addr.encode() |
+ RT(rt) | mode | c);
+}
+
+BufferOffset Assembler::as_dtm(LoadStore ls, Register rn, uint32_t mask,
+ DTMMode mode, DTMWriteBack wb, Condition c) {
+ return writeInst(0x08000000 | RN(rn) | ls | mode | mask | c | wb);
+}
+
+BufferOffset Assembler::allocLiteralLoadEntry(
+ size_t numInst, unsigned numPoolEntries, PoolHintPun& php, uint8_t* data,
+ const LiteralDoc& doc, ARMBuffer::PoolEntry* pe, bool loadToPC) {
+ uint8_t* inst = (uint8_t*)&php.raw;
+
+ MOZ_ASSERT(inst);
+ MOZ_ASSERT(numInst == 1); // Or fix the disassembly
+
+ BufferOffset offs =
+ m_buffer.allocEntry(numInst, numPoolEntries, inst, data, pe);
+ propagateOOM(offs.assigned());
+#ifdef JS_DISASM_ARM
+ Instruction* instruction = m_buffer.getInstOrNull(offs);
+ if (instruction) {
+ spewLiteralLoad(php, loadToPC, instruction, doc);
+ }
+#endif
+ return offs;
+}
+
+// This is also used for instructions that might be resolved into branches,
+// or might not. If dest==pc then it is effectively a branch.
+
+BufferOffset Assembler::as_Imm32Pool(Register dest, uint32_t value,
+ Condition c) {
+ PoolHintPun php;
+ php.phd.init(0, c, PoolHintData::PoolDTR, dest);
+ BufferOffset offs = allocLiteralLoadEntry(
+ 1, 1, php, (uint8_t*)&value, LiteralDoc(value), nullptr, dest == pc);
+ return offs;
+}
+
+/* static */
+void Assembler::WritePoolEntry(Instruction* addr, Condition c, uint32_t data) {
+ MOZ_ASSERT(addr->is<InstLDR>());
+ *addr->as<InstLDR>()->dest() = data;
+ MOZ_ASSERT(addr->extractCond() == c);
+}
+
+BufferOffset Assembler::as_FImm64Pool(VFPRegister dest, double d, Condition c) {
+ MOZ_ASSERT(dest.isDouble());
+ PoolHintPun php;
+ php.phd.init(0, c, PoolHintData::PoolVDTR, dest);
+ return allocLiteralLoadEntry(1, 2, php, (uint8_t*)&d, LiteralDoc(d));
+}
+
+BufferOffset Assembler::as_FImm32Pool(VFPRegister dest, float f, Condition c) {
+ // Insert floats into the double pool as they have the same limitations on
+ // immediate offset. This wastes 4 bytes padding per float. An alternative
+ // would be to have a separate pool for floats.
+ MOZ_ASSERT(dest.isSingle());
+ PoolHintPun php;
+ php.phd.init(0, c, PoolHintData::PoolVDTR, dest);
+ return allocLiteralLoadEntry(1, 1, php, (uint8_t*)&f, LiteralDoc(f));
+}
+
+// Pool callbacks stuff:
+void Assembler::InsertIndexIntoTag(uint8_t* load_, uint32_t index) {
+ uint32_t* load = (uint32_t*)load_;
+ PoolHintPun php;
+ php.raw = *load;
+ php.phd.setIndex(index);
+ *load = php.raw;
+}
+
+// patchConstantPoolLoad takes the address of the instruction that wants to be
+// patched, and the address of the start of the constant pool, and figures
+// things out from there.
+void Assembler::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr) {
+ PoolHintData data = *(PoolHintData*)loadAddr;
+ uint32_t* instAddr = (uint32_t*)loadAddr;
+ int offset = (char*)constPoolAddr - (char*)loadAddr;
+ switch (data.getLoadType()) {
+ case PoolHintData::PoolBOGUS:
+ MOZ_CRASH("bogus load type!");
+ case PoolHintData::PoolDTR:
+ Assembler::as_dtr_patch(
+ IsLoad, 32, Offset, data.getReg(),
+ DTRAddr(pc, DtrOffImm(offset + 4 * data.getIndex() - 8)),
+ data.getCond(), instAddr);
+ break;
+ case PoolHintData::PoolBranch:
+ // Either this used to be a poolBranch, and the label was already bound,
+ // so it was replaced with a real branch, or this may happen in the
+ // future. If this is going to happen in the future, then the actual
+ // bits that are written here don't matter (except the condition code,
+ // since that is always preserved across patchings) but if it does not
+ // get bound later, then we want to make sure this is a load from the
+ // pool entry (and the pool entry should be nullptr so it will crash).
+ if (data.isValidPoolHint()) {
+ Assembler::as_dtr_patch(
+ IsLoad, 32, Offset, pc,
+ DTRAddr(pc, DtrOffImm(offset + 4 * data.getIndex() - 8)),
+ data.getCond(), instAddr);
+ }
+ break;
+ case PoolHintData::PoolVDTR: {
+ VFPRegister dest = data.getVFPReg();
+ int32_t imm = offset + (data.getIndex() * 4) - 8;
+ MOZ_ASSERT(-1024 < imm && imm < 1024);
+ Assembler::as_vdtr_patch(IsLoad, dest, VFPAddr(pc, VFPOffImm(imm)),
+ data.getCond(), instAddr);
+ break;
+ }
+ }
+}
+
+// Atomic instruction stuff:
+
+BufferOffset Assembler::as_ldrexd(Register rt, Register rt2, Register rn,
+ Condition c) {
+ MOZ_ASSERT(!(rt.code() & 1) && rt2.code() == rt.code() + 1);
+ MOZ_ASSERT(rt.code() != 14 && rn.code() != 15);
+ return writeInst(0x01b00f9f | (int)c | RT(rt) | RN(rn));
+}
+
+BufferOffset Assembler::as_ldrex(Register rt, Register rn, Condition c) {
+ MOZ_ASSERT(rt.code() != 15 && rn.code() != 15);
+ return writeInst(0x01900f9f | (int)c | RT(rt) | RN(rn));
+}
+
+BufferOffset Assembler::as_ldrexh(Register rt, Register rn, Condition c) {
+ MOZ_ASSERT(rt.code() != 15 && rn.code() != 15);
+ return writeInst(0x01f00f9f | (int)c | RT(rt) | RN(rn));
+}
+
+BufferOffset Assembler::as_ldrexb(Register rt, Register rn, Condition c) {
+ MOZ_ASSERT(rt.code() != 15 && rn.code() != 15);
+ return writeInst(0x01d00f9f | (int)c | RT(rt) | RN(rn));
+}
+
+BufferOffset Assembler::as_strexd(Register rd, Register rt, Register rt2,
+ Register rn, Condition c) {
+ MOZ_ASSERT(!(rt.code() & 1) && rt2.code() == rt.code() + 1);
+ MOZ_ASSERT(rt.code() != 14 && rn.code() != 15 && rd.code() != 15);
+ MOZ_ASSERT(rd != rn && rd != rt && rd != rt2);
+ return writeInst(0x01a00f90 | (int)c | RD(rd) | RN(rn) | rt.code());
+}
+
+BufferOffset Assembler::as_strex(Register rd, Register rt, Register rn,
+ Condition c) {
+ MOZ_ASSERT(rd != rn && rd != rt); // True restriction on Cortex-A7 (RPi2)
+ return writeInst(0x01800f90 | (int)c | RD(rd) | RN(rn) | rt.code());
+}
+
+BufferOffset Assembler::as_strexh(Register rd, Register rt, Register rn,
+ Condition c) {
+ MOZ_ASSERT(rd != rn && rd != rt); // True restriction on Cortex-A7 (RPi2)
+ return writeInst(0x01e00f90 | (int)c | RD(rd) | RN(rn) | rt.code());
+}
+
+BufferOffset Assembler::as_strexb(Register rd, Register rt, Register rn,
+ Condition c) {
+ MOZ_ASSERT(rd != rn && rd != rt); // True restriction on Cortex-A7 (RPi2)
+ return writeInst(0x01c00f90 | (int)c | RD(rd) | RN(rn) | rt.code());
+}
+
+BufferOffset Assembler::as_clrex() { return writeInst(0xf57ff01f); }
+
+// Memory barrier stuff:
+
+BufferOffset Assembler::as_dmb(BarrierOption option) {
+ return writeInst(0xf57ff050U | (int)option);
+}
+BufferOffset Assembler::as_dsb(BarrierOption option) {
+ return writeInst(0xf57ff040U | (int)option);
+}
+BufferOffset Assembler::as_isb() {
+ return writeInst(0xf57ff06fU); // option == SY
+}
+BufferOffset Assembler::as_dsb_trap() {
+ // DSB is "mcr 15, 0, r0, c7, c10, 4".
+ // See eg https://bugs.kde.org/show_bug.cgi?id=228060.
+ // ARMv7 manual, "VMSA CP15 c7 register summary".
+ // Flagged as "legacy" starting with ARMv8, may be disabled on chip, see
+ // ARMv8 manual E2.7.3 and G3.18.16.
+ return writeInst(0xee070f9a);
+}
+BufferOffset Assembler::as_dmb_trap() {
+ // DMB is "mcr 15, 0, r0, c7, c10, 5".
+ // ARMv7 manual, "VMSA CP15 c7 register summary".
+ // Flagged as "legacy" starting with ARMv8, may be disabled on chip, see
+ // ARMv8 manual E2.7.3 and G3.18.16.
+ return writeInst(0xee070fba);
+}
+BufferOffset Assembler::as_isb_trap() {
+ // ISB is "mcr 15, 0, r0, c7, c5, 4".
+ // ARMv7 manual, "VMSA CP15 c7 register summary".
+ // Flagged as "legacy" starting with ARMv8, may be disabled on chip, see
+ // ARMv8 manual E2.7.3 and G3.18.16.
+ return writeInst(0xee070f94);
+}
+
+BufferOffset Assembler::as_csdb() {
+ // NOP (see as_nop) on architectures where this instruction is not defined.
+ //
+ // https://developer.arm.com/-/media/developer/pdf/Cache_Speculation_Side-channels_22Feb18.pdf
+ // CSDB A32: 1110_0011_0010_0000_1111_0000_0001_0100
+ return writeInst(0xe320f000 | 0x14);
+}
+
+// Control flow stuff:
+
+// bx can *only* branch to a register, never to an immediate.
+BufferOffset Assembler::as_bx(Register r, Condition c) {
+ BufferOffset ret = writeInst(((int)c) | OpBx | r.code());
+ return ret;
+}
+
+void Assembler::WritePoolGuard(BufferOffset branch, Instruction* dest,
+ BufferOffset afterPool) {
+ BOffImm off = afterPool.diffB<BOffImm>(branch);
+ if (off.isInvalid()) {
+ MOZ_CRASH("BOffImm invalid");
+ }
+ *dest = InstBImm(off, Always);
+}
+
+// Branch can branch to an immediate *or* to a register.
+// Branches to immediates are pc relative, branches to registers are absolute.
+BufferOffset Assembler::as_b(BOffImm off, Condition c, Label* documentation) {
+ return writeBranchInst(((int)c) | OpB | off.encode(),
+ refLabel(documentation));
+}
+
+BufferOffset Assembler::as_b(Label* l, Condition c) {
+ if (l->bound()) {
+ // Note only one instruction is emitted here, the NOP is overwritten.
+ BufferOffset ret = allocBranchInst();
+ if (oom()) {
+ return BufferOffset();
+ }
+
+ BOffImm offset = BufferOffset(l).diffB<BOffImm>(ret);
+ MOZ_RELEASE_ASSERT(!offset.isInvalid(),
+ "Buffer size limit should prevent this");
+ as_b(offset, c, ret);
+#ifdef JS_DISASM_ARM
+ spewBranch(m_buffer.getInstOrNull(ret), refLabel(l));
+#endif
+ return ret;
+ }
+
+ if (oom()) {
+ return BufferOffset();
+ }
+
+ BufferOffset ret;
+ if (l->used()) {
+ int32_t old = l->offset();
+ MOZ_RELEASE_ASSERT(BOffImm::IsInRange(old),
+ "Buffer size limit should prevent this");
+ ret = as_b(BOffImm(old), c, l);
+ } else {
+ BOffImm inv;
+ ret = as_b(inv, c, l);
+ }
+
+ if (oom()) {
+ return BufferOffset();
+ }
+
+ l->use(ret.getOffset());
+ return ret;
+}
+
+BufferOffset Assembler::as_b(BOffImm off, Condition c, BufferOffset inst) {
+ // JS_DISASM_ARM NOTE: Can't disassemble here, because numerous callers use
+ // this to patchup old code. Must disassemble in caller where it makes sense.
+ // Not many callers.
+ *editSrc(inst) = InstBImm(off, c);
+ return inst;
+}
+
+// blx can go to either an immediate or a register.
+// When blx'ing to a register, we change processor state depending on the low
+// bit of the register when blx'ing to an immediate, we *always* change
+// processor state.
+
+BufferOffset Assembler::as_blx(Register r, Condition c) {
+ return writeInst(((int)c) | OpBlx | r.code());
+}
+
+// bl can only branch to an pc-relative immediate offset
+// It cannot change the processor state.
+BufferOffset Assembler::as_bl(BOffImm off, Condition c, Label* documentation) {
+ return writeBranchInst(((int)c) | OpBl | off.encode(),
+ refLabel(documentation));
+}
+
+BufferOffset Assembler::as_bl(Label* l, Condition c) {
+ if (l->bound()) {
+ // Note only one instruction is emitted here, the NOP is overwritten.
+ BufferOffset ret = allocBranchInst();
+ if (oom()) {
+ return BufferOffset();
+ }
+
+ BOffImm offset = BufferOffset(l).diffB<BOffImm>(ret);
+ MOZ_RELEASE_ASSERT(!offset.isInvalid(),
+ "Buffer size limit should prevent this");
+
+ as_bl(offset, c, ret);
+#ifdef JS_DISASM_ARM
+ spewBranch(m_buffer.getInstOrNull(ret), refLabel(l));
+#endif
+ return ret;
+ }
+
+ if (oom()) {
+ return BufferOffset();
+ }
+
+ BufferOffset ret;
+ // See if the list was empty.
+ if (l->used()) {
+ int32_t old = l->offset();
+ MOZ_RELEASE_ASSERT(BOffImm::IsInRange(old),
+ "Buffer size limit should prevent this");
+ ret = as_bl(BOffImm(old), c, l);
+ } else {
+ BOffImm inv;
+ ret = as_bl(inv, c, l);
+ }
+
+ if (oom()) {
+ return BufferOffset();
+ }
+
+ l->use(ret.getOffset());
+ return ret;
+}
+
+BufferOffset Assembler::as_bl(BOffImm off, Condition c, BufferOffset inst) {
+ *editSrc(inst) = InstBLImm(off, c);
+ return inst;
+}
+
+BufferOffset Assembler::as_mrs(Register r, Condition c) {
+ return writeInst(0x010f0000 | int(c) | RD(r));
+}
+
+BufferOffset Assembler::as_msr(Register r, Condition c) {
+ // Hardcode the 'mask' field to 0b11 for now. It is bits 18 and 19, which
+ // are the two high bits of the 'c' in this constant.
+ MOZ_ASSERT((r.code() & ~0xf) == 0);
+ return writeInst(0x012cf000 | int(c) | r.code());
+}
+
+// VFP instructions!
+enum vfp_tags { VfpTag = 0x0C000A00, VfpArith = 0x02000000 };
+
+BufferOffset Assembler::writeVFPInst(vfp_size sz, uint32_t blob) {
+ MOZ_ASSERT((sz & blob) == 0);
+ MOZ_ASSERT((VfpTag & blob) == 0);
+ return writeInst(VfpTag | std::underlying_type_t<vfp_size>(sz) | blob);
+}
+
+/* static */
+void Assembler::WriteVFPInstStatic(vfp_size sz, uint32_t blob, uint32_t* dest) {
+ MOZ_ASSERT((sz & blob) == 0);
+ MOZ_ASSERT((VfpTag & blob) == 0);
+ WriteInstStatic(VfpTag | std::underlying_type_t<vfp_size>(sz) | blob, dest);
+}
+
+// Unityped variants: all registers hold the same (ieee754 single/double)
+// notably not included are vcvt; vmov vd, #imm; vmov rt, vn.
+BufferOffset Assembler::as_vfp_float(VFPRegister vd, VFPRegister vn,
+ VFPRegister vm, VFPOp op, Condition c) {
+ // Make sure we believe that all of our operands are the same kind.
+ MOZ_ASSERT_IF(!vn.isMissing(), vd.equiv(vn));
+ MOZ_ASSERT_IF(!vm.isMissing(), vd.equiv(vm));
+ vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
+ return writeVFPInst(sz, VD(vd) | VN(vn) | VM(vm) | op | VfpArith | c);
+}
+
+BufferOffset Assembler::as_vadd(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c) {
+ return as_vfp_float(vd, vn, vm, OpvAdd, c);
+}
+
+BufferOffset Assembler::as_vdiv(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c) {
+ return as_vfp_float(vd, vn, vm, OpvDiv, c);
+}
+
+BufferOffset Assembler::as_vmul(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c) {
+ return as_vfp_float(vd, vn, vm, OpvMul, c);
+}
+
+BufferOffset Assembler::as_vnmul(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c) {
+ return as_vfp_float(vd, vn, vm, OpvMul, c);
+}
+
+BufferOffset Assembler::as_vnmla(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c) {
+ MOZ_CRASH("Feature NYI");
+}
+
+BufferOffset Assembler::as_vnmls(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c) {
+ MOZ_CRASH("Feature NYI");
+}
+
+BufferOffset Assembler::as_vneg(VFPRegister vd, VFPRegister vm, Condition c) {
+ return as_vfp_float(vd, NoVFPRegister, vm, OpvNeg, c);
+}
+
+BufferOffset Assembler::as_vsqrt(VFPRegister vd, VFPRegister vm, Condition c) {
+ return as_vfp_float(vd, NoVFPRegister, vm, OpvSqrt, c);
+}
+
+BufferOffset Assembler::as_vabs(VFPRegister vd, VFPRegister vm, Condition c) {
+ return as_vfp_float(vd, NoVFPRegister, vm, OpvAbs, c);
+}
+
+BufferOffset Assembler::as_vsub(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c) {
+ return as_vfp_float(vd, vn, vm, OpvSub, c);
+}
+
+BufferOffset Assembler::as_vcmp(VFPRegister vd, VFPRegister vm, Condition c) {
+ return as_vfp_float(vd, NoVFPRegister, vm, OpvCmp, c);
+}
+
+BufferOffset Assembler::as_vcmpz(VFPRegister vd, Condition c) {
+ return as_vfp_float(vd, NoVFPRegister, NoVFPRegister, OpvCmpz, c);
+}
+
+// Specifically, a move between two same sized-registers.
+BufferOffset Assembler::as_vmov(VFPRegister vd, VFPRegister vsrc, Condition c) {
+ return as_vfp_float(vd, NoVFPRegister, vsrc, OpvMov, c);
+}
+
+// Transfer between Core and VFP.
+
+// Unlike the next function, moving between the core registers and vfp registers
+// can't be *that* properly typed. Namely, since I don't want to munge the type
+// VFPRegister to also include core registers. Thus, the core and vfp registers
+// are passed in based on their type, and src/dest is determined by the
+// float2core.
+
+BufferOffset Assembler::as_vxfer(Register vt1, Register vt2, VFPRegister vm,
+ FloatToCore_ f2c, Condition c, int idx) {
+ vfp_size sz = IsSingle;
+ if (vm.isDouble()) {
+ // Technically, this can be done with a vmov à la ARM ARM under vmov
+ // however, that requires at least an extra bit saying if the operation
+ // should be performed on the lower or upper half of the double. Moving
+ // a single to/from 2N/2N+1 isn't equivalent, since there are 32 single
+ // registers, and 32 double registers so there is no way to encode the
+ // last 16 double registers.
+ sz = IsDouble;
+ MOZ_ASSERT(idx == 0 || idx == 1);
+ // If we are transferring a single half of the double then it must be
+ // moving a VFP reg to a core reg.
+ MOZ_ASSERT_IF(vt2 == InvalidReg, f2c == FloatToCore);
+ idx = idx << 21;
+ } else {
+ MOZ_ASSERT(idx == 0);
+ }
+
+ if (vt2 == InvalidReg) {
+ return writeVFPInst(sz, WordTransfer |
+ std::underlying_type_t<FloatToCore_>(f2c) |
+ std::underlying_type_t<Condition>(c) | RT(vt1) |
+ maybeRN(vt2) | VN(vm) | idx);
+ }
+
+ // We are doing a 64 bit transfer.
+ return writeVFPInst(sz, DoubleTransfer |
+ std::underlying_type_t<FloatToCore_>(f2c) |
+ std::underlying_type_t<Condition>(c) | RT(vt1) |
+ maybeRN(vt2) | VM(vm) | idx);
+}
+
+enum vcvt_destFloatness { VcvtToInteger = 1 << 18, VcvtToFloat = 0 << 18 };
+enum vcvt_toZero {
+ VcvtToZero =
+ 1 << 7, // Use the default rounding mode, which rounds truncates.
+ VcvtToFPSCR = 0 << 7 // Use whatever rounding mode the fpscr specifies.
+};
+enum vcvt_Signedness {
+ VcvtToSigned = 1 << 16,
+ VcvtToUnsigned = 0 << 16,
+ VcvtFromSigned = 1 << 7,
+ VcvtFromUnsigned = 0 << 7
+};
+
+// Our encoding actually allows just the src and the dest (and their types) to
+// uniquely specify the encoding that we are going to use.
+BufferOffset Assembler::as_vcvt(VFPRegister vd, VFPRegister vm, bool useFPSCR,
+ Condition c) {
+ // Unlike other cases, the source and dest types cannot be the same.
+ MOZ_ASSERT(!vd.equiv(vm));
+ vfp_size sz = IsDouble;
+ if (vd.isFloat() && vm.isFloat()) {
+ // Doing a float -> float conversion.
+ if (vm.isSingle()) {
+ sz = IsSingle;
+ }
+ return writeVFPInst(sz, c | 0x02B700C0 | VM(vm) | VD(vd));
+ }
+
+ // At least one of the registers should be a float.
+ vcvt_destFloatness destFloat;
+ vcvt_Signedness opSign;
+ vcvt_toZero doToZero = VcvtToFPSCR;
+ MOZ_ASSERT(vd.isFloat() || vm.isFloat());
+ if (vd.isSingle() || vm.isSingle()) {
+ sz = IsSingle;
+ }
+
+ if (vd.isFloat()) {
+ destFloat = VcvtToFloat;
+ opSign = (vm.isSInt()) ? VcvtFromSigned : VcvtFromUnsigned;
+ } else {
+ destFloat = VcvtToInteger;
+ opSign = (vd.isSInt()) ? VcvtToSigned : VcvtToUnsigned;
+ doToZero = useFPSCR ? VcvtToFPSCR : VcvtToZero;
+ }
+ return writeVFPInst(
+ sz, c | 0x02B80040 | VD(vd) | VM(vm) | destFloat | opSign | doToZero);
+}
+
+BufferOffset Assembler::as_vcvtFixed(VFPRegister vd, bool isSigned,
+ uint32_t fixedPoint, bool toFixed,
+ Condition c) {
+ MOZ_ASSERT(vd.isFloat());
+ uint32_t sx = 0x1;
+ vfp_size sf = vd.isDouble() ? IsDouble : IsSingle;
+ int32_t imm5 = fixedPoint;
+ imm5 = (sx ? 32 : 16) - imm5;
+ MOZ_ASSERT(imm5 >= 0);
+ imm5 = imm5 >> 1 | (imm5 & 1) << 5;
+ return writeVFPInst(sf, 0x02BA0040 | VD(vd) | toFixed << 18 | sx << 7 |
+ (!isSigned) << 16 | imm5 | c);
+}
+
+// Transfer between VFP and memory.
+static uint32_t EncodeVdtr(LoadStore ls, VFPRegister vd, VFPAddr addr,
+ Assembler::Condition c) {
+ return ls | 0x01000000 | addr.encode() | VD(vd) | c;
+}
+
+BufferOffset Assembler::as_vdtr(
+ LoadStore ls, VFPRegister vd, VFPAddr addr,
+ Condition c /* vfp doesn't have a wb option */) {
+ vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
+ return writeVFPInst(sz, EncodeVdtr(ls, vd, addr, c));
+}
+
+/* static */
+void Assembler::as_vdtr_patch(LoadStore ls, VFPRegister vd, VFPAddr addr,
+ Condition c, uint32_t* dest) {
+ vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
+ WriteVFPInstStatic(sz, EncodeVdtr(ls, vd, addr, c), dest);
+}
+
+// VFP's ldm/stm work differently from the standard arm ones. You can only
+// transfer a range.
+
+BufferOffset Assembler::as_vdtm(LoadStore st, Register rn, VFPRegister vd,
+ int length,
+ /* also has update conditions */ Condition c) {
+ MOZ_ASSERT(length <= 16 && length >= 0);
+ vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
+
+ if (vd.isDouble()) {
+ length *= 2;
+ }
+
+ return writeVFPInst(sz, dtmLoadStore | RN(rn) | VD(vd) | length | dtmMode |
+ dtmUpdate | dtmCond);
+}
+
+BufferOffset Assembler::as_vldr_unaligned(VFPRegister vd, Register rn) {
+ MOZ_ASSERT(HasNEON());
+ if (vd.isDouble()) {
+ // vld1 (multiple single elements) with align=0, size=3, numregs=1
+ return writeInst(0xF42007CF | RN(rn) | VD(vd));
+ }
+ // vld1 (single element to single lane) with index=0, size=2
+ MOZ_ASSERT(vd.isFloat());
+ MOZ_ASSERT((vd.code() & 1) == 0);
+ return writeInst(0xF4A0080F | RN(rn) | VD(vd.asDouble()));
+}
+
+BufferOffset Assembler::as_vstr_unaligned(VFPRegister vd, Register rn) {
+ MOZ_ASSERT(HasNEON());
+ if (vd.isDouble()) {
+ // vst1 (multiple single elements) with align=0, size=3, numregs=1
+ return writeInst(0xF40007CF | RN(rn) | VD(vd));
+ }
+ // vst1 (single element from one lane) with index=0, size=2
+ MOZ_ASSERT(vd.isFloat());
+ MOZ_ASSERT((vd.code() & 1) == 0);
+ return writeInst(0xF480080F | RN(rn) | VD(vd.asDouble()));
+}
+
+BufferOffset Assembler::as_vimm(VFPRegister vd, VFPImm imm, Condition c) {
+ MOZ_ASSERT(imm.isValid());
+ vfp_size sz = vd.isDouble() ? IsDouble : IsSingle;
+ return writeVFPInst(sz, c | imm.encode() | VD(vd) | 0x02B00000);
+}
+
+BufferOffset Assembler::as_vmrs(Register r, Condition c) {
+ return writeInst(c | 0x0ef10a10 | RT(r));
+}
+
+BufferOffset Assembler::as_vmsr(Register r, Condition c) {
+ return writeInst(c | 0x0ee10a10 | RT(r));
+}
+
+bool Assembler::nextLink(BufferOffset b, BufferOffset* next) {
+ Instruction branch = *editSrc(b);
+ MOZ_ASSERT(branch.is<InstBranchImm>());
+
+ BOffImm destOff;
+ branch.as<InstBranchImm>()->extractImm(&destOff);
+ if (destOff.isInvalid()) {
+ return false;
+ }
+
+ // Propagate the next link back to the caller, by constructing a new
+ // BufferOffset into the space they provided.
+ new (next) BufferOffset(destOff.decode());
+ return true;
+}
+
+void Assembler::bind(Label* label, BufferOffset boff) {
+#ifdef JS_DISASM_ARM
+ spew_.spewBind(label);
+#endif
+ if (oom()) {
+ // Ensure we always bind the label. This matches what we do on
+ // x86/x64 and silences the assert in ~Label.
+ label->bind(0);
+ return;
+ }
+
+ if (label->used()) {
+ bool more;
+ // If our caller didn't give us an explicit target to bind to then we
+ // want to bind to the location of the next instruction.
+ BufferOffset dest = boff.assigned() ? boff : nextOffset();
+ BufferOffset b(label);
+ do {
+ BufferOffset next;
+ more = nextLink(b, &next);
+ Instruction branch = *editSrc(b);
+ Condition c = branch.extractCond();
+ BOffImm offset = dest.diffB<BOffImm>(b);
+ MOZ_RELEASE_ASSERT(!offset.isInvalid(),
+ "Buffer size limit should prevent this");
+ if (branch.is<InstBImm>()) {
+ as_b(offset, c, b);
+ } else if (branch.is<InstBLImm>()) {
+ as_bl(offset, c, b);
+ } else {
+ MOZ_CRASH("crazy fixup!");
+ }
+ b = next;
+ } while (more);
+ }
+ label->bind(nextOffset().getOffset());
+ MOZ_ASSERT(!oom());
+}
+
+void Assembler::retarget(Label* label, Label* target) {
+#ifdef JS_DISASM_ARM
+ spew_.spewRetarget(label, target);
+#endif
+ if (label->used() && !oom()) {
+ if (target->bound()) {
+ bind(label, BufferOffset(target));
+ } else if (target->used()) {
+ // The target is not bound but used. Prepend label's branch list
+ // onto target's.
+ BufferOffset labelBranchOffset(label);
+ BufferOffset next;
+
+ // Find the head of the use chain for label.
+ while (nextLink(labelBranchOffset, &next)) {
+ labelBranchOffset = next;
+ }
+
+ // Then patch the head of label's use chain to the tail of target's
+ // use chain, prepending the entire use chain of target.
+ Instruction branch = *editSrc(labelBranchOffset);
+ Condition c = branch.extractCond();
+ int32_t prev = target->offset();
+ target->use(label->offset());
+ if (branch.is<InstBImm>()) {
+ as_b(BOffImm(prev), c, labelBranchOffset);
+ } else if (branch.is<InstBLImm>()) {
+ as_bl(BOffImm(prev), c, labelBranchOffset);
+ } else {
+ MOZ_CRASH("crazy fixup!");
+ }
+ } else {
+ // The target is unbound and unused. We can just take the head of
+ // the list hanging off of label, and dump that into target.
+ target->use(label->offset());
+ }
+ }
+ label->reset();
+}
+
+static int stopBKPT = -1;
+void Assembler::as_bkpt() {
+ // This is a count of how many times a breakpoint instruction has been
+ // generated. It is embedded into the instruction for debugging
+ // purposes. Gdb will print "bkpt xxx" when you attempt to dissassemble a
+ // breakpoint with the number xxx embedded into it. If this breakpoint is
+ // being hit, then you can run (in gdb):
+ // >b dbg_break
+ // >b main
+ // >commands
+ // >set stopBKPT = xxx
+ // >c
+ // >end
+ // which will set a breakpoint on the function dbg_break above set a
+ // scripted breakpoint on main that will set the (otherwise unmodified)
+ // value to the number of the breakpoint, so dbg_break will actuall be
+ // called and finally, when you run the executable, execution will halt when
+ // that breakpoint is generated.
+ static int hit = 0;
+ if (stopBKPT == hit) {
+ dbg_break();
+ }
+ writeInst(0xe1200070 | (hit & 0xf) | ((hit & 0xfff0) << 4));
+ hit++;
+}
+
+BufferOffset Assembler::as_illegal_trap() {
+ // Encoding of the permanently-undefined 'udf' instruction, with the imm16
+ // set to 0.
+ return writeInst(0xe7f000f0);
+}
+
+void Assembler::flushBuffer() { m_buffer.flushPool(); }
+
+void Assembler::enterNoPool(size_t maxInst) { m_buffer.enterNoPool(maxInst); }
+
+void Assembler::leaveNoPool() { m_buffer.leaveNoPool(); }
+
+void Assembler::enterNoNops() { m_buffer.enterNoNops(); }
+
+void Assembler::leaveNoNops() { m_buffer.leaveNoNops(); }
+
+struct PoolHeader : Instruction {
+ struct Header {
+ // The size should take into account the pool header.
+ // The size is in units of Instruction (4 bytes), not byte.
+ uint32_t size : 15;
+ uint32_t isNatural : 1;
+ uint32_t ONES : 16;
+
+ Header(int size_, bool isNatural_)
+ : size(size_), isNatural(isNatural_), ONES(0xffff) {}
+
+ explicit Header(const Instruction* i) {
+ static_assert(sizeof(Header) == sizeof(uint32_t));
+ memcpy(this, i, sizeof(Header));
+ MOZ_ASSERT(ONES == 0xffff);
+ }
+
+ uint32_t raw() const {
+ static_assert(sizeof(Header) == sizeof(uint32_t));
+ uint32_t dest;
+ memcpy(&dest, this, sizeof(Header));
+ return dest;
+ }
+ };
+
+ PoolHeader(int size_, bool isNatural_)
+ : Instruction(Header(size_, isNatural_).raw(), true) {}
+
+ uint32_t size() const {
+ Header tmp(this);
+ return tmp.size;
+ }
+ uint32_t isNatural() const {
+ Header tmp(this);
+ return tmp.isNatural;
+ }
+
+ static bool IsTHIS(const Instruction& i) {
+ return (*i.raw() & 0xffff0000) == 0xffff0000;
+ }
+ static const PoolHeader* AsTHIS(const Instruction& i) {
+ if (!IsTHIS(i)) {
+ return nullptr;
+ }
+ return static_cast<const PoolHeader*>(&i);
+ }
+};
+
+void Assembler::WritePoolHeader(uint8_t* start, Pool* p, bool isNatural) {
+ static_assert(sizeof(PoolHeader) == 4,
+ "PoolHandler must have the correct size.");
+ uint8_t* pool = start + 4;
+ // Go through the usual rigmarole to get the size of the pool.
+ pool += p->getPoolSize();
+ uint32_t size = pool - start;
+ MOZ_ASSERT((size & 3) == 0);
+ size = size >> 2;
+ MOZ_ASSERT(size < (1 << 15));
+ PoolHeader header(size, isNatural);
+ *(PoolHeader*)start = header;
+}
+
+// The size of an arbitrary 32-bit call in the instruction stream. On ARM this
+// sequence is |pc = ldr pc - 4; imm32| given that we never reach the imm32.
+uint32_t Assembler::PatchWrite_NearCallSize() { return sizeof(uint32_t); }
+
+void Assembler::PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall) {
+ Instruction* inst = (Instruction*)start.raw();
+ // Overwrite whatever instruction used to be here with a call. Since the
+ // destination is in the same function, it will be within range of the
+ // 24 << 2 byte bl instruction.
+ uint8_t* dest = toCall.raw();
+ new (inst) InstBLImm(BOffImm(dest - (uint8_t*)inst), Always);
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue) {
+ Instruction* ptr = reinterpret_cast<Instruction*>(label.raw());
+
+ Register dest;
+ Assembler::RelocStyle rs;
+
+ {
+ InstructionIterator iter(ptr);
+ DebugOnly<const uint32_t*> val = GetPtr32Target(iter, &dest, &rs);
+ MOZ_ASSERT(uint32_t((const uint32_t*)val) == uint32_t(expectedValue.value));
+ }
+
+ // Patch over actual instructions.
+ {
+ InstructionIterator iter(ptr);
+ MacroAssembler::ma_mov_patch(Imm32(int32_t(newValue.value)), dest, Always,
+ rs, iter);
+ }
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ ImmPtr newValue, ImmPtr expectedValue) {
+ PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value),
+ PatchedImmPtr(expectedValue.value));
+}
+
+// This just stomps over memory with 32 bits of raw data. Its purpose is to
+// overwrite the call of JITed code with 32 bits worth of an offset. This will
+// is only meant to function on code that has been invalidated, so it should be
+// totally safe. Since that instruction will never be executed again, a ICache
+// flush should not be necessary
+void Assembler::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) {
+ // Raw is going to be the return address.
+ uint32_t* raw = (uint32_t*)label.raw();
+ // Overwrite the 4 bytes before the return address, which will end up being
+ // the call instruction.
+ *(raw - 1) = imm.value;
+}
+
+uint8_t* Assembler::NextInstruction(uint8_t* inst_, uint32_t* count) {
+ if (count != nullptr) {
+ *count += sizeof(Instruction);
+ }
+
+ InstructionIterator iter(reinterpret_cast<Instruction*>(inst_));
+ return reinterpret_cast<uint8_t*>(iter.next());
+}
+
+static bool InstIsGuard(Instruction* inst, const PoolHeader** ph) {
+ Assembler::Condition c = inst->extractCond();
+ if (c != Assembler::Always) {
+ return false;
+ }
+ if (!(inst->is<InstBXReg>() || inst->is<InstBImm>())) {
+ return false;
+ }
+ // See if the next instruction is a pool header.
+ *ph = (inst + 1)->as<const PoolHeader>();
+ return *ph != nullptr;
+}
+
+static bool InstIsGuard(BufferInstructionIterator& iter,
+ const PoolHeader** ph) {
+ Instruction* inst = iter.cur();
+ Assembler::Condition c = inst->extractCond();
+ if (c != Assembler::Always) {
+ return false;
+ }
+ if (!(inst->is<InstBXReg>() || inst->is<InstBImm>())) {
+ return false;
+ }
+ // See if the next instruction is a pool header.
+ *ph = iter.peek()->as<const PoolHeader>();
+ return *ph != nullptr;
+}
+
+template <class T>
+static bool InstIsBNop(const T& iter) {
+ // In some special situations, it is necessary to insert a NOP into the
+ // instruction stream that nobody knows about, since nobody should know
+ // about it, make sure it gets skipped when Instruction::next() is called.
+ // this generates a very specific nop, namely a branch to the next
+ // instruction.
+ const Instruction* cur = iter.cur();
+ Assembler::Condition c = cur->extractCond();
+ if (c != Assembler::Always) {
+ return false;
+ }
+ if (!cur->is<InstBImm>()) {
+ return false;
+ }
+ InstBImm* b = cur->as<InstBImm>();
+ BOffImm offset;
+ b->extractImm(&offset);
+ return offset.decode() == 4;
+}
+
+Instruction* InstructionIterator::maybeSkipAutomaticInstructions() {
+ // If the current instruction was automatically-inserted, skip past it.
+ const PoolHeader* ph;
+
+ // Loop until an intentionally-placed instruction is found.
+ while (true) {
+ if (InstIsGuard(cur(), &ph)) {
+ // Don't skip a natural guard.
+ if (ph->isNatural()) {
+ return cur();
+ }
+ advanceRaw(1 + ph->size());
+ } else if (InstIsBNop<InstructionIterator>(*this)) {
+ advanceRaw(1);
+ } else {
+ return cur();
+ }
+ }
+}
+
+Instruction* BufferInstructionIterator::maybeSkipAutomaticInstructions() {
+ const PoolHeader* ph;
+ // If this is a guard, and the next instruction is a header, always work
+ // around the pool. If it isn't a guard, then start looking ahead.
+ if (InstIsGuard(*this, &ph)) {
+ // Don't skip a natural guard.
+ if (ph->isNatural()) {
+ return cur();
+ }
+ advance(sizeof(Instruction) * ph->size());
+ return next();
+ }
+ if (InstIsBNop<BufferInstructionIterator>(*this)) {
+ return next();
+ }
+ return cur();
+}
+
+// Cases to be handled:
+// 1) no pools or branches in sight => return this+1
+// 2) branch to next instruction => return this+2, because a nop needed to be
+// inserted into the stream.
+// 3) this+1 is an artificial guard for a pool => return first instruction
+// after the pool
+// 4) this+1 is a natural guard => return the branch
+// 5) this is a branch, right before a pool => return first instruction after
+// the pool
+// in assembly form:
+// 1) add r0, r0, r0 <= this
+// add r1, r1, r1 <= returned value
+// add r2, r2, r2
+//
+// 2) add r0, r0, r0 <= this
+// b foo
+// foo:
+// add r2, r2, r2 <= returned value
+//
+// 3) add r0, r0, r0 <= this
+// b after_pool;
+// .word 0xffff0002 # bit 15 being 0 indicates that the branch was not
+// # requested by the assembler
+// 0xdeadbeef # the 2 indicates that there is 1 pool entry, and the
+// # pool header
+// add r4, r4, r4 <= returned value
+// 4) add r0, r0, r0 <= this
+// b after_pool <= returned value
+// .word 0xffff8002 # bit 15 being 1 indicates that the branch was
+// # requested by the assembler
+// 0xdeadbeef
+// add r4, r4, r4
+// 5) b after_pool <= this
+// .word 0xffff8002 # bit 15 has no bearing on the returned value
+// 0xdeadbeef
+// add r4, r4, r4 <= returned value
+
+Instruction* InstructionIterator::next() {
+ const PoolHeader* ph;
+
+ // If the current instruction is followed by a pool header,
+ // move past the current instruction and the pool.
+ if (InstIsGuard(cur(), &ph)) {
+ advanceRaw(1 + ph->size());
+ return maybeSkipAutomaticInstructions();
+ }
+
+ // The next instruction is then known to not be a PoolHeader.
+ advanceRaw(1);
+ return maybeSkipAutomaticInstructions();
+}
+
+void Assembler::ToggleToJmp(CodeLocationLabel inst_) {
+ uint32_t* ptr = (uint32_t*)inst_.raw();
+
+ DebugOnly<Instruction*> inst = (Instruction*)inst_.raw();
+ MOZ_ASSERT(inst->is<InstCMP>());
+
+ // Zero bits 20-27, then set 24-27 to be correct for a branch.
+ // 20-23 will be party of the B's immediate, and should be 0.
+ *ptr = (*ptr & ~(0xff << 20)) | (0xa0 << 20);
+}
+
+void Assembler::ToggleToCmp(CodeLocationLabel inst_) {
+ uint32_t* ptr = (uint32_t*)inst_.raw();
+
+ DebugOnly<Instruction*> inst = (Instruction*)inst_.raw();
+ MOZ_ASSERT(inst->is<InstBImm>());
+
+ // Ensure that this masking operation doesn't affect the offset of the
+ // branch instruction when it gets toggled back.
+ MOZ_ASSERT((*ptr & (0xf << 20)) == 0);
+
+ // Also make sure that the CMP is valid. Part of having a valid CMP is that
+ // all of the bits describing the destination in most ALU instructions are
+ // all unset (looks like it is encoding r0).
+ MOZ_ASSERT(toRD(*inst) == r0);
+
+ // Zero out bits 20-27, then set them to be correct for a compare.
+ *ptr = (*ptr & ~(0xff << 20)) | (0x35 << 20);
+}
+
+void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) {
+ InstructionIterator iter(reinterpret_cast<Instruction*>(inst_.raw()));
+ MOZ_ASSERT(iter.cur()->is<InstMovW>() || iter.cur()->is<InstLDR>());
+
+ if (iter.cur()->is<InstMovW>()) {
+ // If it looks like the start of a movw/movt sequence, then make sure we
+ // have all of it (and advance the iterator past the full sequence).
+ iter.next();
+ MOZ_ASSERT(iter.cur()->is<InstMovT>());
+ }
+
+ iter.next();
+ MOZ_ASSERT(iter.cur()->is<InstNOP>() || iter.cur()->is<InstBLXReg>());
+
+ if (enabled == iter.cur()->is<InstBLXReg>()) {
+ // Nothing to do.
+ return;
+ }
+
+ Instruction* inst = iter.cur();
+
+ if (enabled) {
+ *inst = InstBLXReg(ScratchRegister, Always);
+ } else {
+ *inst = InstNOP();
+ }
+}
+
+size_t Assembler::ToggledCallSize(uint8_t* code) {
+ InstructionIterator iter(reinterpret_cast<Instruction*>(code));
+ MOZ_ASSERT(iter.cur()->is<InstMovW>() || iter.cur()->is<InstLDR>());
+
+ if (iter.cur()->is<InstMovW>()) {
+ // If it looks like the start of a movw/movt sequence, then make sure we
+ // have all of it (and advance the iterator past the full sequence).
+ iter.next();
+ MOZ_ASSERT(iter.cur()->is<InstMovT>());
+ }
+
+ iter.next();
+ MOZ_ASSERT(iter.cur()->is<InstNOP>() || iter.cur()->is<InstBLXReg>());
+ return uintptr_t(iter.cur()) + 4 - uintptr_t(code);
+}
+
+uint32_t Assembler::NopFill = 0;
+
+uint32_t Assembler::GetNopFill() {
+ static bool isSet = false;
+ if (!isSet) {
+ char* fillStr = getenv("ARM_ASM_NOP_FILL");
+ uint32_t fill;
+ if (fillStr && sscanf(fillStr, "%u", &fill) == 1) {
+ NopFill = fill;
+ }
+ if (NopFill > 8) {
+ MOZ_CRASH("Nop fill > 8 is not supported");
+ }
+ isSet = true;
+ }
+ return NopFill;
+}
+
+uint32_t Assembler::AsmPoolMaxOffset = 1024;
+
+uint32_t Assembler::GetPoolMaxOffset() {
+ static bool isSet = false;
+ if (!isSet) {
+ char* poolMaxOffsetStr = getenv("ASM_POOL_MAX_OFFSET");
+ uint32_t poolMaxOffset;
+ if (poolMaxOffsetStr &&
+ sscanf(poolMaxOffsetStr, "%u", &poolMaxOffset) == 1) {
+ AsmPoolMaxOffset = poolMaxOffset;
+ }
+ isSet = true;
+ }
+ return AsmPoolMaxOffset;
+}
+
+SecondScratchRegisterScope::SecondScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, masm.getSecondScratchReg()) {}
+
+#ifdef JS_DISASM_ARM
+
+/* static */
+void Assembler::disassembleInstruction(const Instruction* i,
+ DisasmBuffer& buffer) {
+ disasm::NameConverter converter;
+ disasm::Disassembler dasm(converter);
+ uint8_t* loc = reinterpret_cast<uint8_t*>(const_cast<uint32_t*>(i->raw()));
+ dasm.InstructionDecode(buffer, loc);
+}
+
+void Assembler::initDisassembler() {
+ // The line is normally laid out like this:
+ //
+ // xxxxxxxx ldr r, op ; comment
+ //
+ // where xx...x is the instruction bit pattern.
+ //
+ // Labels are laid out by themselves to line up with the instructions above
+ // and below:
+ //
+ // nnnn:
+ //
+ // Branch targets are normally on the same line as the branch instruction,
+ // but when they cannot be they will be on a line by themselves, indented
+ // significantly:
+ //
+ // -> label
+
+ spew_.setLabelIndent(" "); // 10
+ spew_.setTargetIndent(" "); // 20
+}
+
+void Assembler::finishDisassembler() { spew_.spewOrphans(); }
+
+// Labels are named as they are encountered by adding names to a
+// table, using the Label address as the key. This is made tricky by
+// the (memory for) Label objects being reused, but reused label
+// objects are recognizable from being marked as not used or not
+// bound. See spew_.refLabel().
+//
+// In a number of cases there is no information about the target, and
+// we just end up printing "patchable constant load to PC". This is
+// true especially for jumps to bailout handlers (which have no
+// names). See allocLiteralLoadEntry() and its callers. In some cases
+// (loop back edges) some information about the intended target may be
+// propagated from higher levels, and if so it's printed here.
+
+void Assembler::spew(Instruction* i) {
+ if (spew_.isDisabled() || !i) {
+ return;
+ }
+
+ DisasmBuffer buffer;
+ disassembleInstruction(i, buffer);
+ spew_.spew("%s", buffer.start());
+}
+
+// If a target label is known, always print that and do not attempt to
+// disassemble the branch operands, as they will often be encoding
+// metainformation (pointers for a chain of jump instructions), and
+// not actual branch targets.
+
+void Assembler::spewBranch(Instruction* i, const LabelDoc& target) {
+ if (spew_.isDisabled() || !i) {
+ return;
+ }
+
+ DisasmBuffer buffer;
+ disassembleInstruction(i, buffer);
+
+ char labelBuf[128];
+ labelBuf[0] = 0;
+
+ bool haveTarget = target.valid;
+ if (!haveTarget) {
+ SprintfLiteral(labelBuf, " -> (link-time target)");
+ }
+
+ if (InstBranchImm::IsTHIS(*i)) {
+ InstBranchImm* bimm = InstBranchImm::AsTHIS(*i);
+ BOffImm destOff;
+ bimm->extractImm(&destOff);
+ if (destOff.isInvalid() || haveTarget) {
+ // The target information in the instruction is likely garbage, so remove
+ // it. The target label will in any case be printed if we have it.
+ //
+ // The format of the instruction disassembly is [0-9a-f]{8}\s+\S+\s+.*,
+ // where the \S+ string is the opcode. Strip everything after the opcode,
+ // and attach the label if we have it.
+ int i;
+ for (i = 8; i < buffer.length() && buffer[i] == ' '; i++) {
+ }
+ for (; i < buffer.length() && buffer[i] != ' '; i++) {
+ }
+ buffer[i] = 0;
+ if (haveTarget) {
+ SprintfLiteral(labelBuf, " -> %d%s", target.doc,
+ !target.bound ? "f" : "");
+ haveTarget = false;
+ }
+ }
+ }
+ spew_.spew("%s%s", buffer.start(), labelBuf);
+
+ if (haveTarget) {
+ spew_.spewRef(target);
+ }
+}
+
+void Assembler::spewLiteralLoad(PoolHintPun& php, bool loadToPC,
+ const Instruction* i, const LiteralDoc& doc) {
+ if (spew_.isDisabled()) {
+ return;
+ }
+
+ char litbuf[2048];
+ spew_.formatLiteral(doc, litbuf, sizeof(litbuf));
+
+ // See patchConstantPoolLoad, above. We assemble the instruction into a
+ // buffer with a zero offset, as documentation, but the offset will be
+ // patched later.
+
+ uint32_t inst;
+ PoolHintData& data = php.phd;
+ switch (php.phd.getLoadType()) {
+ case PoolHintData::PoolDTR:
+ Assembler::as_dtr_patch(IsLoad, 32, Offset, data.getReg(),
+ DTRAddr(pc, DtrOffImm(0)), data.getCond(), &inst);
+ break;
+ case PoolHintData::PoolBranch:
+ if (data.isValidPoolHint()) {
+ Assembler::as_dtr_patch(IsLoad, 32, Offset, pc,
+ DTRAddr(pc, DtrOffImm(0)), data.getCond(),
+ &inst);
+ }
+ break;
+ case PoolHintData::PoolVDTR:
+ Assembler::as_vdtr_patch(IsLoad, data.getVFPReg(),
+ VFPAddr(pc, VFPOffImm(0)), data.getCond(),
+ &inst);
+ break;
+
+ default:
+ MOZ_CRASH();
+ }
+
+ DisasmBuffer buffer;
+ disasm::NameConverter converter;
+ disasm::Disassembler dasm(converter);
+ dasm.InstructionDecode(buffer, reinterpret_cast<uint8_t*>(&inst));
+ spew_.spew("%s ; .const %s", buffer.start(), litbuf);
+}
+
+#endif // JS_DISASM_ARM
diff --git a/js/src/jit/arm/Assembler-arm.h b/js/src/jit/arm/Assembler-arm.h
new file mode 100644
index 0000000000..a941601ed3
--- /dev/null
+++ b/js/src/jit/arm/Assembler-arm.h
@@ -0,0 +1,2291 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_Assembler_arm_h
+#define jit_arm_Assembler_arm_h
+
+#include "mozilla/Attributes.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+#include <iterator>
+#include <type_traits>
+
+#include "jit/arm/Architecture-arm.h"
+#include "jit/arm/disasm/Disasm-arm.h"
+#include "jit/CompactBuffer.h"
+#include "jit/JitCode.h"
+#include "jit/shared/Assembler-shared.h"
+#include "jit/shared/Disassembler-shared.h"
+#include "jit/shared/IonAssemblerBufferWithConstantPools.h"
+#include "wasm/WasmTypeDecls.h"
+
+union PoolHintPun;
+
+namespace js {
+namespace jit {
+
+using LiteralDoc = DisassemblerSpew::LiteralDoc;
+using LabelDoc = DisassemblerSpew::LabelDoc;
+
+// NOTE: there are duplicates in this list! Sometimes we want to specifically
+// refer to the link register as a link register (bl lr is much clearer than bl
+// r14). HOWEVER, this register can easily be a gpr when it is not busy holding
+// the return address.
+static constexpr Register r0{Registers::r0};
+static constexpr Register r1{Registers::r1};
+static constexpr Register r2{Registers::r2};
+static constexpr Register r3{Registers::r3};
+static constexpr Register r4{Registers::r4};
+static constexpr Register r5{Registers::r5};
+static constexpr Register r6{Registers::r6};
+static constexpr Register r7{Registers::r7};
+static constexpr Register r8{Registers::r8};
+static constexpr Register r9{Registers::r9};
+static constexpr Register r10{Registers::r10};
+static constexpr Register r11{Registers::r11};
+static constexpr Register r12{Registers::ip};
+static constexpr Register ip{Registers::ip};
+static constexpr Register sp{Registers::sp};
+static constexpr Register r14{Registers::lr};
+static constexpr Register lr{Registers::lr};
+static constexpr Register pc{Registers::pc};
+
+static constexpr Register ScratchRegister{Registers::ip};
+
+// Helper class for ScratchRegister usage. Asserts that only one piece
+// of code thinks it has exclusive ownership of the scratch register.
+struct ScratchRegisterScope : public AutoRegisterScope {
+ explicit ScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, ScratchRegister) {}
+};
+
+struct SecondScratchRegisterScope : public AutoRegisterScope {
+ explicit SecondScratchRegisterScope(MacroAssembler& masm);
+};
+
+static constexpr Register OsrFrameReg = r3;
+static constexpr Register CallTempReg0 = r5;
+static constexpr Register CallTempReg1 = r6;
+static constexpr Register CallTempReg2 = r7;
+static constexpr Register CallTempReg3 = r8;
+static constexpr Register CallTempReg4 = r0;
+static constexpr Register CallTempReg5 = r1;
+
+static constexpr Register IntArgReg0 = r0;
+static constexpr Register IntArgReg1 = r1;
+static constexpr Register IntArgReg2 = r2;
+static constexpr Register IntArgReg3 = r3;
+static constexpr Register HeapReg = r10;
+static constexpr Register CallTempNonArgRegs[] = {r5, r6, r7, r8};
+static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+// These register assignments for the 64-bit atomic ops are frequently too
+// constraining, but we have no way of expressing looser constraints to the
+// register allocator.
+
+// CompareExchange: Any two odd/even pairs would do for `new` and `out`, and any
+// pair would do for `old`, so long as none of them overlap.
+
+static constexpr Register CmpXchgOldLo = r4;
+static constexpr Register CmpXchgOldHi = r5;
+static constexpr Register64 CmpXchgOld64 =
+ Register64(CmpXchgOldHi, CmpXchgOldLo);
+static constexpr Register CmpXchgNewLo = IntArgReg2;
+static constexpr Register CmpXchgNewHi = IntArgReg3;
+static constexpr Register64 CmpXchgNew64 =
+ Register64(CmpXchgNewHi, CmpXchgNewLo);
+static constexpr Register CmpXchgOutLo = IntArgReg0;
+static constexpr Register CmpXchgOutHi = IntArgReg1;
+static constexpr Register64 CmpXchgOut64 =
+ Register64(CmpXchgOutHi, CmpXchgOutLo);
+
+// Exchange: Any two non-equal odd/even pairs would do for `new` and `out`.
+
+static constexpr Register XchgNewLo = IntArgReg2;
+static constexpr Register XchgNewHi = IntArgReg3;
+static constexpr Register64 XchgNew64 = Register64(XchgNewHi, XchgNewLo);
+static constexpr Register XchgOutLo = IntArgReg0;
+static constexpr Register XchgOutHi = IntArgReg1;
+
+// Atomic rmw operations: Any two odd/even pairs would do for `tmp` and `out`,
+// and any pair would do for `val`, so long as none of them overlap.
+
+static constexpr Register FetchOpValLo = r4;
+static constexpr Register FetchOpValHi = r5;
+static constexpr Register64 FetchOpVal64 =
+ Register64(FetchOpValHi, FetchOpValLo);
+static constexpr Register FetchOpTmpLo = IntArgReg2;
+static constexpr Register FetchOpTmpHi = IntArgReg3;
+static constexpr Register64 FetchOpTmp64 =
+ Register64(FetchOpTmpHi, FetchOpTmpLo);
+static constexpr Register FetchOpOutLo = IntArgReg0;
+static constexpr Register FetchOpOutHi = IntArgReg1;
+static constexpr Register64 FetchOpOut64 =
+ Register64(FetchOpOutHi, FetchOpOutLo);
+
+class ABIArgGenerator {
+ unsigned intRegIndex_;
+ unsigned floatRegIndex_;
+ uint32_t stackOffset_;
+ ABIArg current_;
+
+ // ARM can either use HardFp (use float registers for float arguments), or
+ // SoftFp (use general registers for float arguments) ABI. We keep this
+ // switch as a runtime switch because wasm always use the HardFp back-end
+ // while the calls to native functions have to use the one provided by the
+ // system.
+ bool useHardFp_;
+
+ ABIArg softNext(MIRType argType);
+ ABIArg hardNext(MIRType argType);
+
+ public:
+ ABIArgGenerator();
+
+ void setUseHardFp(bool useHardFp) {
+ MOZ_ASSERT(intRegIndex_ == 0 && floatRegIndex_ == 0);
+ useHardFp_ = useHardFp;
+ }
+ ABIArg next(MIRType argType);
+ ABIArg& current() { return current_; }
+ uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+ void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+};
+
+bool IsUnaligned(const wasm::MemoryAccessDesc& access);
+
+// These registers may be volatile or nonvolatile.
+static constexpr Register ABINonArgReg0 = r4;
+static constexpr Register ABINonArgReg1 = r5;
+static constexpr Register ABINonArgReg2 = r6;
+static constexpr Register ABINonArgReg3 = r7;
+
+// This register may be volatile or nonvolatile. Avoid d15 which is the
+// ScratchDoubleReg_.
+static constexpr FloatRegister ABINonArgDoubleReg{FloatRegisters::d8,
+ VFPRegister::Double};
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = r4;
+static constexpr Register ABINonArgReturnReg1 = r5;
+static constexpr Register ABINonVolatileReg = r6;
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = lr;
+
+// Instance pointer argument register for WebAssembly functions. This must not
+// alias any other register used for passing function arguments or return
+// values. Preserved by WebAssembly functions.
+static constexpr Register InstanceReg = r9;
+
+// Registers used for wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = r5;
+
+static constexpr Register PreBarrierReg = r1;
+
+static constexpr Register InterpreterPCReg = r9;
+
+static constexpr Register InvalidReg{Registers::invalid_reg};
+static constexpr FloatRegister InvalidFloatReg;
+
+static constexpr Register JSReturnReg_Type = r3;
+static constexpr Register JSReturnReg_Data = r2;
+static constexpr Register StackPointer = sp;
+static constexpr Register FramePointer = r11;
+static constexpr Register ReturnReg = r0;
+static constexpr Register64 ReturnReg64(r1, r0);
+
+// The attribute '__value_in_regs' alters the calling convention of a function
+// so that a structure of up to four elements can be returned via the argument
+// registers rather than being written to memory.
+static constexpr Register ReturnRegVal0 = IntArgReg0;
+static constexpr Register ReturnRegVal1 = IntArgReg1;
+static constexpr Register ReturnRegVal2 = IntArgReg2;
+static constexpr Register ReturnRegVal3 = IntArgReg3;
+
+static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::d0,
+ VFPRegister::Single};
+static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::d0,
+ VFPRegister::Double};
+static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg;
+static constexpr FloatRegister ScratchFloat32Reg_ = {FloatRegisters::s30,
+ VFPRegister::Single};
+static constexpr FloatRegister ScratchDoubleReg_ = {FloatRegisters::d15,
+ VFPRegister::Double};
+static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg;
+static constexpr FloatRegister ScratchUIntReg = {FloatRegisters::d15,
+ VFPRegister::UInt};
+static constexpr FloatRegister ScratchIntReg = {FloatRegisters::d15,
+ VFPRegister::Int};
+
+// Do not reference ScratchFloat32Reg_ directly, use ScratchFloat32Scope
+// instead.
+struct ScratchFloat32Scope : public AutoFloatRegisterScope {
+ explicit ScratchFloat32Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {}
+};
+
+// Do not reference ScratchDoubleReg_ directly, use ScratchDoubleScope instead.
+struct ScratchDoubleScope : public AutoFloatRegisterScope {
+ explicit ScratchDoubleScope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {}
+};
+
+// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registerd used in RegExpTester instruction (do not use ReturnReg).
+static constexpr Register RegExpTesterRegExpReg = CallTempReg0;
+static constexpr Register RegExpTesterStringReg = CallTempReg1;
+static constexpr Register RegExpTesterLastIndexReg = CallTempReg2;
+
+static constexpr FloatRegister d0 = {FloatRegisters::d0, VFPRegister::Double};
+static constexpr FloatRegister d1 = {FloatRegisters::d1, VFPRegister::Double};
+static constexpr FloatRegister d2 = {FloatRegisters::d2, VFPRegister::Double};
+static constexpr FloatRegister d3 = {FloatRegisters::d3, VFPRegister::Double};
+static constexpr FloatRegister d4 = {FloatRegisters::d4, VFPRegister::Double};
+static constexpr FloatRegister d5 = {FloatRegisters::d5, VFPRegister::Double};
+static constexpr FloatRegister d6 = {FloatRegisters::d6, VFPRegister::Double};
+static constexpr FloatRegister d7 = {FloatRegisters::d7, VFPRegister::Double};
+static constexpr FloatRegister d8 = {FloatRegisters::d8, VFPRegister::Double};
+static constexpr FloatRegister d9 = {FloatRegisters::d9, VFPRegister::Double};
+static constexpr FloatRegister d10 = {FloatRegisters::d10, VFPRegister::Double};
+static constexpr FloatRegister d11 = {FloatRegisters::d11, VFPRegister::Double};
+static constexpr FloatRegister d12 = {FloatRegisters::d12, VFPRegister::Double};
+static constexpr FloatRegister d13 = {FloatRegisters::d13, VFPRegister::Double};
+static constexpr FloatRegister d14 = {FloatRegisters::d14, VFPRegister::Double};
+static constexpr FloatRegister d15 = {FloatRegisters::d15, VFPRegister::Double};
+
+// For maximal awesomeness, 8 should be sufficent. ldrd/strd (dual-register
+// load/store) operate in a single cycle when the address they are dealing with
+// is 8 byte aligned. Also, the ARM abi wants the stack to be 8 byte aligned at
+// function boundaries. I'm trying to make sure this is always true.
+static constexpr uint32_t ABIStackAlignment = 8;
+static constexpr uint32_t CodeAlignment = 8;
+static constexpr uint32_t JitStackAlignment = 8;
+
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+ JitStackValueAlignment >= 1,
+ "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+static constexpr uint32_t SimdMemoryAlignment = 8;
+
+static_assert(CodeAlignment % SimdMemoryAlignment == 0,
+ "Code alignment should be larger than any of the alignments "
+ "which are used for "
+ "the constant sections of the code buffer. Thus it should be "
+ "larger than the "
+ "alignment for SIMD constants.");
+
+static_assert(JitStackAlignment % SimdMemoryAlignment == 0,
+ "Stack alignment should be larger than any of the alignments "
+ "which are used for "
+ "spilled values. Thus it should be larger than the alignment "
+ "for SIMD accesses.");
+
+static const uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static const uint32_t WasmTrapInstructionLength = 4;
+
+// See comments in wasm::GenerateFunctionPrologue. The difference between these
+// is the size of the largest callable prologue on the platform.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+static const Scale ScalePointer = TimesFour;
+
+class Instruction;
+class InstBranchImm;
+uint32_t RM(Register r);
+uint32_t RS(Register r);
+uint32_t RD(Register r);
+uint32_t RT(Register r);
+uint32_t RN(Register r);
+
+uint32_t maybeRD(Register r);
+uint32_t maybeRT(Register r);
+uint32_t maybeRN(Register r);
+
+Register toRN(Instruction i);
+Register toRM(Instruction i);
+Register toRD(Instruction i);
+Register toR(Instruction i);
+
+class VFPRegister;
+uint32_t VD(VFPRegister vr);
+uint32_t VN(VFPRegister vr);
+uint32_t VM(VFPRegister vr);
+
+// For being passed into the generic vfp instruction generator when there is an
+// instruction that only takes two registers.
+static constexpr VFPRegister NoVFPRegister(VFPRegister::Double, 0, false, true);
+
+struct ImmTag : public Imm32 {
+ explicit ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {}
+};
+
+struct ImmType : public ImmTag {
+ explicit ImmType(JSValueType type) : ImmTag(JSVAL_TYPE_TO_TAG(type)) {}
+};
+
+enum Index {
+ Offset = 0 << 21 | 1 << 24,
+ PreIndex = 1 << 21 | 1 << 24,
+ PostIndex = 0 << 21 | 0 << 24
+ // The docs were rather unclear on this. It sounds like
+ // 1 << 21 | 0 << 24 encodes dtrt.
+};
+
+enum IsImmOp2_ { IsImmOp2 = 1 << 25, IsNotImmOp2 = 0 << 25 };
+enum IsImmDTR_ { IsImmDTR = 0 << 25, IsNotImmDTR = 1 << 25 };
+// For the extra memory operations, ldrd, ldrsb, ldrh.
+enum IsImmEDTR_ { IsImmEDTR = 1 << 22, IsNotImmEDTR = 0 << 22 };
+
+enum ShiftType {
+ LSL = 0, // << 5
+ LSR = 1, // << 5
+ ASR = 2, // << 5
+ ROR = 3, // << 5
+ RRX = ROR // RRX is encoded as ROR with a 0 offset.
+};
+
+// Modes for STM/LDM. Names are the suffixes applied to the instruction.
+enum DTMMode {
+ A = 0 << 24, // empty / after
+ B = 1 << 24, // full / before
+ D = 0 << 23, // decrement
+ I = 1 << 23, // increment
+ DA = D | A,
+ DB = D | B,
+ IA = I | A,
+ IB = I | B
+};
+
+enum DTMWriteBack { WriteBack = 1 << 21, NoWriteBack = 0 << 21 };
+
+// Condition code updating mode.
+enum SBit {
+ SetCC = 1 << 20, // Set condition code.
+ LeaveCC = 0 << 20 // Leave condition code unchanged.
+};
+
+enum LoadStore { IsLoad = 1 << 20, IsStore = 0 << 20 };
+
+// You almost never want to use this directly. Instead, you wantto pass in a
+// signed constant, and let this bit be implicitly set for you. This is however,
+// necessary if we want a negative index.
+enum IsUp_ { IsUp = 1 << 23, IsDown = 0 << 23 };
+enum ALUOp {
+ OpMov = 0xd << 21,
+ OpMvn = 0xf << 21,
+ OpAnd = 0x0 << 21,
+ OpBic = 0xe << 21,
+ OpEor = 0x1 << 21,
+ OpOrr = 0xc << 21,
+ OpAdc = 0x5 << 21,
+ OpAdd = 0x4 << 21,
+ OpSbc = 0x6 << 21,
+ OpSub = 0x2 << 21,
+ OpRsb = 0x3 << 21,
+ OpRsc = 0x7 << 21,
+ OpCmn = 0xb << 21,
+ OpCmp = 0xa << 21,
+ OpTeq = 0x9 << 21,
+ OpTst = 0x8 << 21,
+ OpInvalid = -1
+};
+
+enum MULOp {
+ OpmMul = 0 << 21,
+ OpmMla = 1 << 21,
+ OpmUmaal = 2 << 21,
+ OpmMls = 3 << 21,
+ OpmUmull = 4 << 21,
+ OpmUmlal = 5 << 21,
+ OpmSmull = 6 << 21,
+ OpmSmlal = 7 << 21
+};
+enum BranchTag {
+ OpB = 0x0a000000,
+ OpBMask = 0x0f000000,
+ OpBDestMask = 0x00ffffff,
+ OpBl = 0x0b000000,
+ OpBlx = 0x012fff30,
+ OpBx = 0x012fff10
+};
+
+// Just like ALUOp, but for the vfp instruction set.
+enum VFPOp {
+ OpvMul = 0x2 << 20,
+ OpvAdd = 0x3 << 20,
+ OpvSub = 0x3 << 20 | 0x1 << 6,
+ OpvDiv = 0x8 << 20,
+ OpvMov = 0xB << 20 | 0x1 << 6,
+ OpvAbs = 0xB << 20 | 0x3 << 6,
+ OpvNeg = 0xB << 20 | 0x1 << 6 | 0x1 << 16,
+ OpvSqrt = 0xB << 20 | 0x3 << 6 | 0x1 << 16,
+ OpvCmp = 0xB << 20 | 0x1 << 6 | 0x4 << 16,
+ OpvCmpz = 0xB << 20 | 0x1 << 6 | 0x5 << 16
+};
+
+// Negate the operation, AND negate the immediate that we were passed in.
+ALUOp ALUNeg(ALUOp op, Register dest, Register scratch, Imm32* imm,
+ Register* negDest);
+bool can_dbl(ALUOp op);
+bool condsAreSafe(ALUOp op);
+
+// If there is a variant of op that has a dest (think cmp/sub) return that
+// variant of it.
+ALUOp getDestVariant(ALUOp op);
+
+static constexpr ValueOperand JSReturnOperand{JSReturnReg_Type,
+ JSReturnReg_Data};
+static const ValueOperand softfpReturnOperand = ValueOperand(r1, r0);
+
+// All of these classes exist solely to shuffle data into the various operands.
+// For example Operand2 can be an imm8, a register-shifted-by-a-constant or a
+// register-shifted-by-a-register. We represent this in C++ by having a base
+// class Operand2, which just stores the 32 bits of data as they will be encoded
+// in the instruction. You cannot directly create an Operand2 since it is
+// tricky, and not entirely sane to do so. Instead, you create one of its child
+// classes, e.g. Imm8. Imm8's constructor takes a single integer argument. Imm8
+// will verify that its argument can be encoded as an ARM 12 bit imm8, encode it
+// using an Imm8data, and finally call its parent's (Operand2) constructor with
+// the Imm8data. The Operand2 constructor will then call the Imm8data's encode()
+// function to extract the raw bits from it.
+//
+// In the future, we should be able to extract data from the Operand2 by asking
+// it for its component Imm8data structures. The reason this is so horribly
+// round-about is we wanted to have Imm8 and RegisterShiftedRegister inherit
+// directly from Operand2 but have all of them take up only a single word of
+// storage. We also wanted to avoid passing around raw integers at all since
+// they are error prone.
+class Op2Reg;
+class O2RegImmShift;
+class O2RegRegShift;
+
+namespace datastore {
+
+class Reg {
+ // The "second register".
+ uint32_t rm_ : 4;
+ // Do we get another register for shifting.
+ uint32_t rrs_ : 1;
+ uint32_t type_ : 2;
+ // We'd like this to be a more sensible encoding, but that would need to be
+ // a struct and that would not pack :(
+ uint32_t shiftAmount_ : 5;
+
+ protected:
+ // Mark as a protected field to avoid unused private field warnings.
+ uint32_t pad_ : 20;
+
+ public:
+ Reg(uint32_t rm, ShiftType type, uint32_t rsr, uint32_t shiftAmount)
+ : rm_(rm), rrs_(rsr), type_(type), shiftAmount_(shiftAmount), pad_(0) {}
+ explicit Reg(const Op2Reg& op) { memcpy(this, &op, sizeof(*this)); }
+
+ uint32_t shiftAmount() const { return shiftAmount_; }
+
+ uint32_t encode() const {
+ return rm_ | (rrs_ << 4) | (type_ << 5) | (shiftAmount_ << 7);
+ }
+};
+
+// Op2 has a mode labelled "<imm8m>", which is arm's magical immediate encoding.
+// Some instructions actually get 8 bits of data, which is called Imm8Data
+// below. These should have edit distance > 1, but this is how it is for now.
+class Imm8mData {
+ uint32_t data_ : 8;
+ uint32_t rot_ : 4;
+
+ protected:
+ // Mark as a protected field to avoid unused private field warnings.
+ uint32_t buff_ : 19;
+
+ private:
+ // Throw in an extra bit that will be 1 if we can't encode this properly.
+ // if we can encode it properly, a simple "|" will still suffice to meld it
+ // into the instruction.
+ uint32_t invalid_ : 1;
+
+ public:
+ // Default constructor makes an invalid immediate.
+ Imm8mData() : data_(0xff), rot_(0xf), buff_(0), invalid_(true) {}
+
+ Imm8mData(uint32_t data, uint32_t rot)
+ : data_(data), rot_(rot), buff_(0), invalid_(false) {
+ MOZ_ASSERT(data == data_);
+ MOZ_ASSERT(rot == rot_);
+ }
+
+ bool invalid() const { return invalid_; }
+
+ uint32_t encode() const {
+ MOZ_ASSERT(!invalid_);
+ return data_ | (rot_ << 8);
+ };
+};
+
+class Imm8Data {
+ uint32_t imm4L_ : 4;
+
+ protected:
+ // Mark as a protected field to avoid unused private field warnings.
+ uint32_t pad_ : 4;
+
+ private:
+ uint32_t imm4H_ : 4;
+
+ public:
+ explicit Imm8Data(uint32_t imm) : imm4L_(imm & 0xf), imm4H_(imm >> 4) {
+ MOZ_ASSERT(imm <= 0xff);
+ }
+
+ uint32_t encode() const { return imm4L_ | (imm4H_ << 8); };
+};
+
+// VLDR/VSTR take an 8 bit offset, which is implicitly left shifted by 2.
+class Imm8VFPOffData {
+ uint32_t data_;
+
+ public:
+ explicit Imm8VFPOffData(uint32_t imm) : data_(imm) {
+ MOZ_ASSERT((imm & ~(0xff)) == 0);
+ }
+ uint32_t encode() const { return data_; };
+};
+
+// ARM can magically encode 256 very special immediates to be moved into a
+// register.
+struct Imm8VFPImmData {
+ // This structure's members are public and it has no constructor to
+ // initialize them, for a very special reason. Were this structure to
+ // have a constructor, the initialization for DoubleEncoder's internal
+ // table (see below) would require a rather large static constructor on
+ // some of our supported compilers. The known solution to this is to mark
+ // the constructor constexpr, but, again, some of our supported
+ // compilers don't support constexpr! So we are reduced to public
+ // members and eschewing a constructor in hopes that the initialization
+ // of DoubleEncoder's table is correct.
+ uint32_t imm4L : 4;
+ uint32_t imm4H : 4;
+ int32_t isInvalid : 24;
+
+ uint32_t encode() const {
+ // This assert is an attempting at ensuring that we don't create random
+ // instances of this structure and then asking to encode() it.
+ MOZ_ASSERT(isInvalid == 0);
+ return imm4L | (imm4H << 16);
+ };
+};
+
+class Imm12Data {
+ uint32_t data_ : 12;
+
+ public:
+ explicit Imm12Data(uint32_t imm) : data_(imm) { MOZ_ASSERT(data_ == imm); }
+
+ uint32_t encode() const { return data_; }
+};
+
+class RIS {
+ uint32_t shiftAmount_ : 5;
+
+ public:
+ explicit RIS(uint32_t imm) : shiftAmount_(imm) {
+ MOZ_ASSERT(shiftAmount_ == imm);
+ }
+
+ explicit RIS(Reg r) : shiftAmount_(r.shiftAmount()) {}
+
+ uint32_t encode() const { return shiftAmount_; }
+};
+
+class RRS {
+ protected:
+ // Mark as a protected field to avoid unused private field warnings.
+ uint32_t mustZero_ : 1;
+
+ private:
+ // The register that holds the shift amount.
+ uint32_t rs_ : 4;
+
+ public:
+ explicit RRS(uint32_t rs) : rs_(rs) { MOZ_ASSERT(rs_ == rs); }
+
+ uint32_t encode() const { return rs_ << 1; }
+};
+
+} // namespace datastore
+
+class MacroAssemblerARM;
+class Operand;
+
+class Operand2 {
+ friend class Operand;
+ friend class MacroAssemblerARM;
+ friend class InstALU;
+
+ uint32_t oper_ : 31;
+ uint32_t invalid_ : 1;
+
+ protected:
+ explicit Operand2(datastore::Imm8mData base)
+ : oper_(base.invalid() ? -1 : (base.encode() | uint32_t(IsImmOp2))),
+ invalid_(base.invalid()) {}
+
+ explicit Operand2(datastore::Reg base)
+ : oper_(base.encode() | uint32_t(IsNotImmOp2)), invalid_(false) {}
+
+ private:
+ explicit Operand2(uint32_t blob) : oper_(blob), invalid_(false) {}
+
+ public:
+ bool isO2Reg() const { return !(oper_ & IsImmOp2); }
+
+ Op2Reg toOp2Reg() const;
+
+ bool isImm8() const { return oper_ & IsImmOp2; }
+
+ bool invalid() const { return invalid_; }
+
+ uint32_t encode() const { return oper_; }
+};
+
+class Imm8 : public Operand2 {
+ public:
+ explicit Imm8(uint32_t imm) : Operand2(EncodeImm(imm)) {}
+
+ static datastore::Imm8mData EncodeImm(uint32_t imm) {
+ // RotateLeft below may not be called with a shift of zero.
+ if (imm <= 0xFF) {
+ return datastore::Imm8mData(imm, 0);
+ }
+
+ // An encodable integer has a maximum of 8 contiguous set bits,
+ // with an optional wrapped left rotation to even bit positions.
+ for (int rot = 1; rot < 16; rot++) {
+ uint32_t rotimm = mozilla::RotateLeft(imm, rot * 2);
+ if (rotimm <= 0xFF) {
+ return datastore::Imm8mData(rotimm, rot);
+ }
+ }
+ return datastore::Imm8mData();
+ }
+
+ // Pair template?
+ struct TwoImm8mData {
+ datastore::Imm8mData fst_, snd_;
+
+ TwoImm8mData() = default;
+
+ TwoImm8mData(datastore::Imm8mData fst, datastore::Imm8mData snd)
+ : fst_(fst), snd_(snd) {}
+
+ datastore::Imm8mData fst() const { return fst_; }
+ datastore::Imm8mData snd() const { return snd_; }
+ };
+
+ static TwoImm8mData EncodeTwoImms(uint32_t);
+};
+
+class Op2Reg : public Operand2 {
+ public:
+ explicit Op2Reg(Register rm, ShiftType type, datastore::RIS shiftImm)
+ : Operand2(datastore::Reg(rm.code(), type, 0, shiftImm.encode())) {}
+
+ explicit Op2Reg(Register rm, ShiftType type, datastore::RRS shiftReg)
+ : Operand2(datastore::Reg(rm.code(), type, 1, shiftReg.encode())) {}
+};
+
+static_assert(sizeof(Op2Reg) == sizeof(datastore::Reg),
+ "datastore::Reg(const Op2Reg&) constructor relies on Reg/Op2Reg "
+ "having same size");
+
+class O2RegImmShift : public Op2Reg {
+ public:
+ explicit O2RegImmShift(Register rn, ShiftType type, uint32_t shift)
+ : Op2Reg(rn, type, datastore::RIS(shift)) {}
+};
+
+class O2RegRegShift : public Op2Reg {
+ public:
+ explicit O2RegRegShift(Register rn, ShiftType type, Register rs)
+ : Op2Reg(rn, type, datastore::RRS(rs.code())) {}
+};
+
+O2RegImmShift O2Reg(Register r);
+O2RegImmShift lsl(Register r, int amt);
+O2RegImmShift lsr(Register r, int amt);
+O2RegImmShift asr(Register r, int amt);
+O2RegImmShift rol(Register r, int amt);
+O2RegImmShift ror(Register r, int amt);
+
+O2RegRegShift lsl(Register r, Register amt);
+O2RegRegShift lsr(Register r, Register amt);
+O2RegRegShift asr(Register r, Register amt);
+O2RegRegShift ror(Register r, Register amt);
+
+// An offset from a register to be used for ldr/str. This should include the
+// sign bit, since ARM has "signed-magnitude" offsets. That is it encodes an
+// unsigned offset, then the instruction specifies if the offset is positive or
+// negative. The +/- bit is necessary if the instruction set wants to be able to
+// have a negative register offset e.g. ldr pc, [r1,-r2];
+class DtrOff {
+ uint32_t data_;
+
+ protected:
+ explicit DtrOff(datastore::Imm12Data immdata, IsUp_ iu)
+ : data_(immdata.encode() | uint32_t(IsImmDTR) | uint32_t(iu)) {}
+
+ explicit DtrOff(datastore::Reg reg, IsUp_ iu = IsUp)
+ : data_(reg.encode() | uint32_t(IsNotImmDTR) | iu) {}
+
+ public:
+ uint32_t encode() const { return data_; }
+};
+
+class DtrOffImm : public DtrOff {
+ public:
+ explicit DtrOffImm(int32_t imm)
+ : DtrOff(datastore::Imm12Data(mozilla::Abs(imm)),
+ imm >= 0 ? IsUp : IsDown) {
+ MOZ_ASSERT(mozilla::Abs(imm) < 4096);
+ }
+};
+
+class DtrOffReg : public DtrOff {
+ // These are designed to be called by a constructor of a subclass.
+ // Constructing the necessary RIS/RRS structures is annoying.
+
+ protected:
+ explicit DtrOffReg(Register rn, ShiftType type, datastore::RIS shiftImm,
+ IsUp_ iu = IsUp)
+ : DtrOff(datastore::Reg(rn.code(), type, 0, shiftImm.encode()), iu) {}
+
+ explicit DtrOffReg(Register rn, ShiftType type, datastore::RRS shiftReg,
+ IsUp_ iu = IsUp)
+ : DtrOff(datastore::Reg(rn.code(), type, 1, shiftReg.encode()), iu) {}
+};
+
+class DtrRegImmShift : public DtrOffReg {
+ public:
+ explicit DtrRegImmShift(Register rn, ShiftType type, uint32_t shift,
+ IsUp_ iu = IsUp)
+ : DtrOffReg(rn, type, datastore::RIS(shift), iu) {}
+};
+
+class DtrRegRegShift : public DtrOffReg {
+ public:
+ explicit DtrRegRegShift(Register rn, ShiftType type, Register rs,
+ IsUp_ iu = IsUp)
+ : DtrOffReg(rn, type, datastore::RRS(rs.code()), iu) {}
+};
+
+// We will frequently want to bundle a register with its offset so that we have
+// an "operand" to a load instruction.
+class DTRAddr {
+ friend class Operand;
+
+ uint32_t data_;
+
+ public:
+ explicit DTRAddr(Register reg, DtrOff dtr)
+ : data_(dtr.encode() | (reg.code() << 16)) {}
+
+ uint32_t encode() const { return data_; }
+
+ Register getBase() const { return Register::FromCode((data_ >> 16) & 0xf); }
+};
+
+// Offsets for the extended data transfer instructions:
+// ldrsh, ldrd, ldrsb, etc.
+class EDtrOff {
+ uint32_t data_;
+
+ protected:
+ explicit EDtrOff(datastore::Imm8Data imm8, IsUp_ iu = IsUp)
+ : data_(imm8.encode() | IsImmEDTR | uint32_t(iu)) {}
+
+ explicit EDtrOff(Register rm, IsUp_ iu = IsUp)
+ : data_(rm.code() | IsNotImmEDTR | iu) {}
+
+ public:
+ uint32_t encode() const { return data_; }
+};
+
+class EDtrOffImm : public EDtrOff {
+ public:
+ explicit EDtrOffImm(int32_t imm)
+ : EDtrOff(datastore::Imm8Data(mozilla::Abs(imm)),
+ (imm >= 0) ? IsUp : IsDown) {
+ MOZ_ASSERT(mozilla::Abs(imm) < 256);
+ }
+};
+
+// This is the most-derived class, since the extended data transfer instructions
+// don't support any sort of modifying the "index" operand.
+class EDtrOffReg : public EDtrOff {
+ public:
+ explicit EDtrOffReg(Register rm) : EDtrOff(rm) {}
+};
+
+class EDtrAddr {
+ uint32_t data_;
+
+ public:
+ explicit EDtrAddr(Register r, EDtrOff off) : data_(RN(r) | off.encode()) {}
+
+ uint32_t encode() const { return data_; }
+#ifdef DEBUG
+ Register maybeOffsetRegister() const {
+ if (data_ & IsImmEDTR) {
+ return InvalidReg;
+ }
+ return Register::FromCode(data_ & 0xf);
+ }
+#endif
+};
+
+class VFPOff {
+ uint32_t data_;
+
+ protected:
+ explicit VFPOff(datastore::Imm8VFPOffData imm, IsUp_ isup)
+ : data_(imm.encode() | uint32_t(isup)) {}
+
+ public:
+ uint32_t encode() const { return data_; }
+};
+
+class VFPOffImm : public VFPOff {
+ public:
+ explicit VFPOffImm(int32_t imm)
+ : VFPOff(datastore::Imm8VFPOffData(mozilla::Abs(imm) / 4),
+ imm < 0 ? IsDown : IsUp) {
+ MOZ_ASSERT(mozilla::Abs(imm) <= 255 * 4);
+ }
+};
+
+class VFPAddr {
+ friend class Operand;
+
+ uint32_t data_;
+
+ public:
+ explicit VFPAddr(Register base, VFPOff off)
+ : data_(RN(base) | off.encode()) {}
+
+ uint32_t encode() const { return data_; }
+};
+
+class VFPImm {
+ uint32_t data_;
+
+ public:
+ explicit VFPImm(uint32_t topWordOfDouble);
+
+ static const VFPImm One;
+
+ uint32_t encode() const { return data_; }
+ bool isValid() const { return data_ != (~0U); }
+};
+
+// A BOffImm is an immediate that is used for branches. Namely, it is the offset
+// that will be encoded in the branch instruction. This is the only sane way of
+// constructing a branch.
+class BOffImm {
+ friend class InstBranchImm;
+
+ uint32_t data_;
+
+ public:
+ explicit BOffImm(int offset) : data_((offset - 8) >> 2 & 0x00ffffff) {
+ MOZ_ASSERT((offset & 0x3) == 0);
+ if (!IsInRange(offset)) {
+ MOZ_CRASH("BOffImm offset out of range");
+ }
+ }
+
+ explicit BOffImm() : data_(INVALID) {}
+
+ private:
+ explicit BOffImm(const Instruction& inst);
+
+ public:
+ static const uint32_t INVALID = 0x00800000;
+
+ uint32_t encode() const { return data_; }
+ int32_t decode() const { return ((int32_t(data_) << 8) >> 6) + 8; }
+
+ static bool IsInRange(int offset) {
+ if ((offset - 8) < -33554432) {
+ return false;
+ }
+ if ((offset - 8) > 33554428) {
+ return false;
+ }
+ return true;
+ }
+
+ bool isInvalid() const { return data_ == INVALID; }
+ Instruction* getDest(Instruction* src) const;
+};
+
+class Imm16 {
+ uint32_t lower_ : 12;
+
+ protected:
+ // Mark as a protected field to avoid unused private field warnings.
+ uint32_t pad_ : 4;
+
+ private:
+ uint32_t upper_ : 4;
+ uint32_t invalid_ : 12;
+
+ public:
+ explicit Imm16();
+ explicit Imm16(uint32_t imm);
+ explicit Imm16(Instruction& inst);
+
+ uint32_t encode() const { return lower_ | (upper_ << 16); }
+ uint32_t decode() const { return lower_ | (upper_ << 12); }
+
+ bool isInvalid() const { return invalid_; }
+};
+
+// I would preffer that these do not exist, since there are essentially no
+// instructions that would ever take more than one of these, however, the MIR
+// wants to only have one type of arguments to functions, so bugger.
+class Operand {
+ // The encoding of registers is the same for OP2, DTR and EDTR yet the type
+ // system doesn't let us express this, so choices must be made.
+ public:
+ enum class Tag : uint8_t { OP2, MEM, FOP };
+
+ private:
+ uint32_t tag_ : 8;
+ uint32_t reg_ : 5;
+ int32_t offset_;
+
+ protected:
+ Operand(Tag tag, uint32_t regCode, int32_t offset)
+ : tag_(static_cast<uint32_t>(tag)), reg_(regCode), offset_(offset) {}
+
+ public:
+ explicit Operand(Register reg) : Operand(Tag::OP2, reg.code(), 0) {}
+
+ explicit Operand(FloatRegister freg) : Operand(Tag::FOP, freg.code(), 0) {}
+
+ explicit Operand(Register base, Imm32 off)
+ : Operand(Tag::MEM, base.code(), off.value) {}
+
+ explicit Operand(Register base, int32_t off)
+ : Operand(Tag::MEM, base.code(), off) {}
+
+ explicit Operand(const Address& addr)
+ : Operand(Tag::MEM, addr.base.code(), addr.offset) {}
+
+ public:
+ Tag tag() const { return static_cast<Tag>(tag_); }
+
+ Operand2 toOp2() const {
+ MOZ_ASSERT(tag() == Tag::OP2);
+ return O2Reg(Register::FromCode(reg_));
+ }
+
+ Register toReg() const {
+ MOZ_ASSERT(tag() == Tag::OP2);
+ return Register::FromCode(reg_);
+ }
+
+ Address toAddress() const {
+ MOZ_ASSERT(tag() == Tag::MEM);
+ return Address(Register::FromCode(reg_), offset_);
+ }
+ int32_t disp() const {
+ MOZ_ASSERT(tag() == Tag::MEM);
+ return offset_;
+ }
+
+ int32_t base() const {
+ MOZ_ASSERT(tag() == Tag::MEM);
+ return reg_;
+ }
+ Register baseReg() const {
+ MOZ_ASSERT(tag() == Tag::MEM);
+ return Register::FromCode(reg_);
+ }
+ DTRAddr toDTRAddr() const {
+ MOZ_ASSERT(tag() == Tag::MEM);
+ return DTRAddr(baseReg(), DtrOffImm(offset_));
+ }
+ VFPAddr toVFPAddr() const {
+ MOZ_ASSERT(tag() == Tag::MEM);
+ return VFPAddr(baseReg(), VFPOffImm(offset_));
+ }
+};
+
+inline Imm32 Imm64::firstHalf() const { return low(); }
+
+inline Imm32 Imm64::secondHalf() const { return hi(); }
+
+class InstructionIterator {
+ private:
+ Instruction* inst_;
+
+ public:
+ explicit InstructionIterator(Instruction* inst) : inst_(inst) {
+ maybeSkipAutomaticInstructions();
+ }
+
+ // Advances to the next intentionally-inserted instruction.
+ Instruction* next();
+
+ // Advances past any automatically-inserted instructions.
+ Instruction* maybeSkipAutomaticInstructions();
+
+ Instruction* cur() const { return inst_; }
+
+ protected:
+ // Advances past the given number of instruction-length bytes.
+ inline void advanceRaw(ptrdiff_t instructions = 1);
+};
+
+class Assembler;
+typedef js::jit::AssemblerBufferWithConstantPools<1024, 4, Instruction,
+ Assembler>
+ ARMBuffer;
+
+class Assembler : public AssemblerShared {
+ public:
+ // ARM conditional constants:
+ enum ARMCondition : uint32_t {
+ EQ = 0x00000000, // Zero
+ NE = 0x10000000, // Non-zero
+ CS = 0x20000000,
+ CC = 0x30000000,
+ MI = 0x40000000,
+ PL = 0x50000000,
+ VS = 0x60000000,
+ VC = 0x70000000,
+ HI = 0x80000000,
+ LS = 0x90000000,
+ GE = 0xa0000000,
+ LT = 0xb0000000,
+ GT = 0xc0000000,
+ LE = 0xd0000000,
+ AL = 0xe0000000
+ };
+
+ enum Condition : uint32_t {
+ Equal = EQ,
+ NotEqual = NE,
+ Above = HI,
+ AboveOrEqual = CS,
+ Below = CC,
+ BelowOrEqual = LS,
+ GreaterThan = GT,
+ GreaterThanOrEqual = GE,
+ LessThan = LT,
+ LessThanOrEqual = LE,
+ Overflow = VS,
+ CarrySet = CS,
+ CarryClear = CC,
+ Signed = MI,
+ NotSigned = PL,
+ Zero = EQ,
+ NonZero = NE,
+ Always = AL,
+
+ VFP_NotEqualOrUnordered = NE,
+ VFP_Equal = EQ,
+ VFP_Unordered = VS,
+ VFP_NotUnordered = VC,
+ VFP_GreaterThanOrEqualOrUnordered = CS,
+ VFP_GreaterThanOrEqual = GE,
+ VFP_GreaterThanOrUnordered = HI,
+ VFP_GreaterThan = GT,
+ VFP_LessThanOrEqualOrUnordered = LE,
+ VFP_LessThanOrEqual = LS,
+ VFP_LessThanOrUnordered = LT,
+ VFP_LessThan = CC // MI is valid too.
+ };
+
+ // Bit set when a DoubleCondition does not map to a single ARM condition.
+ // The macro assembler has to special-case these conditions, or else
+ // ConditionFromDoubleCondition will complain.
+ static const int DoubleConditionBitSpecial = 0x1;
+
+ enum DoubleCondition : uint32_t {
+ // These conditions will only evaluate to true if the comparison is
+ // ordered - i.e. neither operand is NaN.
+ DoubleOrdered = VFP_NotUnordered,
+ DoubleEqual = VFP_Equal,
+ DoubleNotEqual = VFP_NotEqualOrUnordered | DoubleConditionBitSpecial,
+ DoubleGreaterThan = VFP_GreaterThan,
+ DoubleGreaterThanOrEqual = VFP_GreaterThanOrEqual,
+ DoubleLessThan = VFP_LessThan,
+ DoubleLessThanOrEqual = VFP_LessThanOrEqual,
+ // If either operand is NaN, these conditions always evaluate to true.
+ DoubleUnordered = VFP_Unordered,
+ DoubleEqualOrUnordered = VFP_Equal | DoubleConditionBitSpecial,
+ DoubleNotEqualOrUnordered = VFP_NotEqualOrUnordered,
+ DoubleGreaterThanOrUnordered = VFP_GreaterThanOrUnordered,
+ DoubleGreaterThanOrEqualOrUnordered = VFP_GreaterThanOrEqualOrUnordered,
+ DoubleLessThanOrUnordered = VFP_LessThanOrUnordered,
+ DoubleLessThanOrEqualOrUnordered = VFP_LessThanOrEqualOrUnordered
+ };
+
+ Condition getCondition(uint32_t inst) {
+ return (Condition)(0xf0000000 & inst);
+ }
+ static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
+ MOZ_ASSERT(!(cond & DoubleConditionBitSpecial));
+ return static_cast<Condition>(cond);
+ }
+
+ enum BarrierOption {
+ BarrierSY = 15, // Full system barrier
+ BarrierST = 14 // StoreStore barrier
+ };
+
+ // This should be protected, but since CodeGenerator wants to use it, it
+ // needs to go out here :(
+
+ BufferOffset nextOffset() { return m_buffer.nextOffset(); }
+
+ protected:
+ // Shim around AssemblerBufferWithConstantPools::allocEntry.
+ BufferOffset allocLiteralLoadEntry(size_t numInst, unsigned numPoolEntries,
+ PoolHintPun& php, uint8_t* data,
+ const LiteralDoc& doc = LiteralDoc(),
+ ARMBuffer::PoolEntry* pe = nullptr,
+ bool loadToPC = false);
+
+ Instruction* editSrc(BufferOffset bo) { return m_buffer.getInst(bo); }
+
+#ifdef JS_DISASM_ARM
+ typedef disasm::EmbeddedVector<char, disasm::ReasonableBufferSize>
+ DisasmBuffer;
+
+ static void disassembleInstruction(const Instruction* i,
+ DisasmBuffer& buffer);
+
+ void initDisassembler();
+ void finishDisassembler();
+ void spew(Instruction* i);
+ void spewBranch(Instruction* i, const LabelDoc& target);
+ void spewLiteralLoad(PoolHintPun& php, bool loadToPC, const Instruction* offs,
+ const LiteralDoc& doc);
+#endif
+
+ public:
+ void resetCounter();
+ static uint32_t NopFill;
+ static uint32_t GetNopFill();
+ static uint32_t AsmPoolMaxOffset;
+ static uint32_t GetPoolMaxOffset();
+
+ protected:
+ // Structure for fixing up pc-relative loads/jumps when a the machine code
+ // gets moved (executable copy, gc, etc.).
+ class RelativePatch {
+ void* target_;
+ RelocationKind kind_;
+
+ public:
+ RelativePatch(void* target, RelocationKind kind)
+ : target_(target), kind_(kind) {}
+ void* target() const { return target_; }
+ RelocationKind kind() const { return kind_; }
+ };
+
+ // TODO: this should actually be a pool-like object. It is currently a big
+ // hack, and probably shouldn't exist.
+ js::Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;
+
+ CompactBufferWriter jumpRelocations_;
+ CompactBufferWriter dataRelocations_;
+
+ ARMBuffer m_buffer;
+
+#ifdef JS_DISASM_ARM
+ DisassemblerSpew spew_;
+#endif
+
+ public:
+ // For the alignment fill use NOP: 0x0320f000 or (Always | InstNOP::NopInst).
+ // For the nopFill use a branch to the next instruction: 0xeaffffff.
+ Assembler()
+ : m_buffer(1, 1, 8, GetPoolMaxOffset(), 8, 0xe320f000, 0xeaffffff,
+ GetNopFill()),
+ isFinished(false),
+ dtmActive(false),
+ dtmCond(Always) {
+#ifdef JS_DISASM_ARM
+ initDisassembler();
+#endif
+ }
+
+ ~Assembler() {
+#ifdef JS_DISASM_ARM
+ finishDisassembler();
+#endif
+ }
+
+ void setUnlimitedBuffer() { m_buffer.setUnlimited(); }
+
+ static Condition InvertCondition(Condition cond);
+ static Condition UnsignedCondition(Condition cond);
+ static Condition ConditionWithoutEqual(Condition cond);
+
+ static DoubleCondition InvertCondition(DoubleCondition cond);
+
+ void writeDataRelocation(BufferOffset offset, ImmGCPtr ptr) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (ptr.value) {
+ if (gc::IsInsideNursery(ptr.value)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(offset.getOffset());
+ }
+ }
+
+ enum RelocBranchStyle { B_MOVWT, B_LDR_BX, B_LDR, B_MOVW_ADD };
+
+ enum RelocStyle { L_MOVWT, L_LDR };
+
+ public:
+ // Given the start of a Control Flow sequence, grab the value that is
+ // finally branched to given the start of a function that loads an address
+ // into a register get the address that ends up in the register.
+ template <class Iter>
+ static const uint32_t* GetCF32Target(Iter* iter);
+
+ static uintptr_t GetPointer(uint8_t*);
+ static const uint32_t* GetPtr32Target(InstructionIterator iter,
+ Register* dest = nullptr,
+ RelocStyle* rs = nullptr);
+
+ bool oom() const;
+
+ void setPrinter(Sprinter* sp) {
+#ifdef JS_DISASM_ARM
+ spew_.setPrinter(sp);
+#endif
+ }
+
+ Register getStackPointer() const { return StackPointer; }
+
+ private:
+ bool isFinished;
+
+ protected:
+ LabelDoc refLabel(const Label* label) {
+#ifdef JS_DISASM_ARM
+ return spew_.refLabel(label);
+#else
+ return LabelDoc();
+#endif
+ }
+
+ public:
+ void finish();
+ bool appendRawCode(const uint8_t* code, size_t numBytes);
+ bool reserve(size_t size);
+ bool swapBuffer(wasm::Bytes& bytes);
+ void copyJumpRelocationTable(uint8_t* dest);
+ void copyDataRelocationTable(uint8_t* dest);
+
+ // Size of the instruction stream, in bytes, after pools are flushed.
+ size_t size() const;
+ // Size of the jump relocation table, in bytes.
+ size_t jumpRelocationTableBytes() const;
+ size_t dataRelocationTableBytes() const;
+
+ // Size of the data table, in bytes.
+ size_t bytesNeeded() const;
+
+ // Write a single instruction into the instruction stream. Very hot,
+ // inlined for performance
+ MOZ_ALWAYS_INLINE BufferOffset writeInst(uint32_t x) {
+ MOZ_ASSERT(hasCreator());
+ BufferOffset offs = m_buffer.putInt(x);
+#ifdef JS_DISASM_ARM
+ spew(m_buffer.getInstOrNull(offs));
+#endif
+ return offs;
+ }
+
+ // As above, but also mark the instruction as a branch. Very hot, inlined
+ // for performance
+ MOZ_ALWAYS_INLINE BufferOffset
+ writeBranchInst(uint32_t x, const LabelDoc& documentation) {
+ BufferOffset offs = m_buffer.putInt(x);
+#ifdef JS_DISASM_ARM
+ spewBranch(m_buffer.getInstOrNull(offs), documentation);
+#endif
+ return offs;
+ }
+
+ // Write a placeholder NOP for a branch into the instruction stream
+ // (in order to adjust assembler addresses and mark it as a branch), it will
+ // be overwritten subsequently.
+ BufferOffset allocBranchInst();
+
+ // A static variant for the cases where we don't want to have an assembler
+ // object.
+ static void WriteInstStatic(uint32_t x, uint32_t* dest);
+
+ public:
+ void writeCodePointer(CodeLabel* label);
+
+ void haltingAlign(int alignment);
+ void nopAlign(int alignment);
+ BufferOffset as_nop();
+ BufferOffset as_alu(Register dest, Register src1, Operand2 op2, ALUOp op,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_mov(Register dest, Operand2 op2, SBit s = LeaveCC,
+ Condition c = Always);
+ BufferOffset as_mvn(Register dest, Operand2 op2, SBit s = LeaveCC,
+ Condition c = Always);
+
+ static void as_alu_patch(Register dest, Register src1, Operand2 op2, ALUOp op,
+ SBit s, Condition c, uint32_t* pos);
+ static void as_mov_patch(Register dest, Operand2 op2, SBit s, Condition c,
+ uint32_t* pos);
+
+ // Logical operations:
+ BufferOffset as_and(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_bic(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_eor(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_orr(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ // Reverse byte operations:
+ BufferOffset as_rev(Register dest, Register src, Condition c = Always);
+ BufferOffset as_rev16(Register dest, Register src, Condition c = Always);
+ BufferOffset as_revsh(Register dest, Register src, Condition c = Always);
+ // Mathematical operations:
+ BufferOffset as_adc(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_add(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_sbc(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_sub(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_rsb(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_rsc(Register dest, Register src1, Operand2 op2,
+ SBit s = LeaveCC, Condition c = Always);
+ // Test operations:
+ BufferOffset as_cmn(Register src1, Operand2 op2, Condition c = Always);
+ BufferOffset as_cmp(Register src1, Operand2 op2, Condition c = Always);
+ BufferOffset as_teq(Register src1, Operand2 op2, Condition c = Always);
+ BufferOffset as_tst(Register src1, Operand2 op2, Condition c = Always);
+
+ // Sign extension operations:
+ BufferOffset as_sxtb(Register dest, Register src, int rotate,
+ Condition c = Always);
+ BufferOffset as_sxth(Register dest, Register src, int rotate,
+ Condition c = Always);
+ BufferOffset as_uxtb(Register dest, Register src, int rotate,
+ Condition c = Always);
+ BufferOffset as_uxth(Register dest, Register src, int rotate,
+ Condition c = Always);
+
+ // Not quite ALU worthy, but useful none the less: These also have the issue
+ // of these being formatted completly differently from the standard ALU
+ // operations.
+ BufferOffset as_movw(Register dest, Imm16 imm, Condition c = Always);
+ BufferOffset as_movt(Register dest, Imm16 imm, Condition c = Always);
+
+ static void as_movw_patch(Register dest, Imm16 imm, Condition c,
+ Instruction* pos);
+ static void as_movt_patch(Register dest, Imm16 imm, Condition c,
+ Instruction* pos);
+
+ BufferOffset as_genmul(Register d1, Register d2, Register rm, Register rn,
+ MULOp op, SBit s, Condition c = Always);
+ BufferOffset as_mul(Register dest, Register src1, Register src2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_mla(Register dest, Register acc, Register src1, Register src2,
+ SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_umaal(Register dest1, Register dest2, Register src1,
+ Register src2, Condition c = Always);
+ BufferOffset as_mls(Register dest, Register acc, Register src1, Register src2,
+ Condition c = Always);
+ BufferOffset as_umull(Register dest1, Register dest2, Register src1,
+ Register src2, SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_umlal(Register dest1, Register dest2, Register src1,
+ Register src2, SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_smull(Register dest1, Register dest2, Register src1,
+ Register src2, SBit s = LeaveCC, Condition c = Always);
+ BufferOffset as_smlal(Register dest1, Register dest2, Register src1,
+ Register src2, SBit s = LeaveCC, Condition c = Always);
+
+ BufferOffset as_sdiv(Register dest, Register num, Register div,
+ Condition c = Always);
+ BufferOffset as_udiv(Register dest, Register num, Register div,
+ Condition c = Always);
+ BufferOffset as_clz(Register dest, Register src, Condition c = Always);
+
+ // Data transfer instructions: ldr, str, ldrb, strb.
+ // Using an int to differentiate between 8 bits and 32 bits is overkill.
+ BufferOffset as_dtr(LoadStore ls, int size, Index mode, Register rt,
+ DTRAddr addr, Condition c = Always);
+
+ static void as_dtr_patch(LoadStore ls, int size, Index mode, Register rt,
+ DTRAddr addr, Condition c, uint32_t* dest);
+
+ // Handles all of the other integral data transferring functions:
+ // ldrsb, ldrsh, ldrd, etc. The size is given in bits.
+ BufferOffset as_extdtr(LoadStore ls, int size, bool IsSigned, Index mode,
+ Register rt, EDtrAddr addr, Condition c = Always);
+
+ BufferOffset as_dtm(LoadStore ls, Register rn, uint32_t mask, DTMMode mode,
+ DTMWriteBack wb, Condition c = Always);
+
+ // Overwrite a pool entry with new data.
+ static void WritePoolEntry(Instruction* addr, Condition c, uint32_t data);
+
+ // Load a 32 bit immediate from a pool into a register.
+ BufferOffset as_Imm32Pool(Register dest, uint32_t value,
+ Condition c = Always);
+
+ // Load a 64 bit floating point immediate from a pool into a register.
+ BufferOffset as_FImm64Pool(VFPRegister dest, double value,
+ Condition c = Always);
+ // Load a 32 bit floating point immediate from a pool into a register.
+ BufferOffset as_FImm32Pool(VFPRegister dest, float value,
+ Condition c = Always);
+
+ // Atomic instructions: ldrexd, ldrex, ldrexh, ldrexb, strexd, strex, strexh,
+ // strexb.
+ //
+ // The doubleword, halfword, and byte versions are available from ARMv6K
+ // forward.
+ //
+ // The word versions are available from ARMv6 forward and can be used to
+ // implement the halfword and byte versions on older systems.
+
+ // LDREXD rt, rt2, [rn]. Constraint: rt even register, rt2=rt+1.
+ BufferOffset as_ldrexd(Register rt, Register rt2, Register rn,
+ Condition c = Always);
+
+ // LDREX rt, [rn]
+ BufferOffset as_ldrex(Register rt, Register rn, Condition c = Always);
+ BufferOffset as_ldrexh(Register rt, Register rn, Condition c = Always);
+ BufferOffset as_ldrexb(Register rt, Register rn, Condition c = Always);
+
+ // STREXD rd, rt, rt2, [rn]. Constraint: rt even register, rt2=rt+1.
+ BufferOffset as_strexd(Register rd, Register rt, Register rt2, Register rn,
+ Condition c = Always);
+
+ // STREX rd, rt, [rn]. Constraint: rd != rn, rd != rt.
+ BufferOffset as_strex(Register rd, Register rt, Register rn,
+ Condition c = Always);
+ BufferOffset as_strexh(Register rd, Register rt, Register rn,
+ Condition c = Always);
+ BufferOffset as_strexb(Register rd, Register rt, Register rn,
+ Condition c = Always);
+
+ // CLREX
+ BufferOffset as_clrex();
+
+ // Memory synchronization.
+ // These are available from ARMv7 forward.
+ BufferOffset as_dmb(BarrierOption option = BarrierSY);
+ BufferOffset as_dsb(BarrierOption option = BarrierSY);
+ BufferOffset as_isb();
+
+ // Memory synchronization for architectures before ARMv7.
+ BufferOffset as_dsb_trap();
+ BufferOffset as_dmb_trap();
+ BufferOffset as_isb_trap();
+
+ // Speculation barrier
+ BufferOffset as_csdb();
+
+ // Control flow stuff:
+
+ // bx can *only* branch to a register never to an immediate.
+ BufferOffset as_bx(Register r, Condition c = Always);
+
+ // Branch can branch to an immediate *or* to a register. Branches to
+ // immediates are pc relative, branches to registers are absolute.
+ BufferOffset as_b(BOffImm off, Condition c, Label* documentation = nullptr);
+
+ BufferOffset as_b(Label* l, Condition c = Always);
+ BufferOffset as_b(BOffImm off, Condition c, BufferOffset inst);
+
+ // blx can go to either an immediate or a register. When blx'ing to a
+ // register, we change processor mode depending on the low bit of the
+ // register when blx'ing to an immediate, we *always* change processor
+ // state.
+ BufferOffset as_blx(Label* l);
+
+ BufferOffset as_blx(Register r, Condition c = Always);
+ BufferOffset as_bl(BOffImm off, Condition c, Label* documentation = nullptr);
+ // bl can only branch+link to an immediate, never to a register it never
+ // changes processor state.
+ BufferOffset as_bl();
+ // bl #imm can have a condition code, blx #imm cannot.
+ // blx reg can be conditional.
+ BufferOffset as_bl(Label* l, Condition c);
+ BufferOffset as_bl(BOffImm off, Condition c, BufferOffset inst);
+
+ BufferOffset as_mrs(Register r, Condition c = Always);
+ BufferOffset as_msr(Register r, Condition c = Always);
+
+ // VFP instructions!
+ private:
+ enum vfp_size { IsDouble = 1 << 8, IsSingle = 0 << 8 };
+
+ BufferOffset writeVFPInst(vfp_size sz, uint32_t blob);
+
+ static void WriteVFPInstStatic(vfp_size sz, uint32_t blob, uint32_t* dest);
+
+ // Unityped variants: all registers hold the same (ieee754 single/double)
+ // notably not included are vcvt; vmov vd, #imm; vmov rt, vn.
+ BufferOffset as_vfp_float(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ VFPOp op, Condition c = Always);
+
+ public:
+ BufferOffset as_vadd(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c = Always);
+ BufferOffset as_vdiv(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c = Always);
+ BufferOffset as_vmul(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c = Always);
+ BufferOffset as_vnmul(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c = Always);
+ BufferOffset as_vnmla(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c = Always);
+ BufferOffset as_vnmls(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c = Always);
+ BufferOffset as_vneg(VFPRegister vd, VFPRegister vm, Condition c = Always);
+ BufferOffset as_vsqrt(VFPRegister vd, VFPRegister vm, Condition c = Always);
+ BufferOffset as_vabs(VFPRegister vd, VFPRegister vm, Condition c = Always);
+ BufferOffset as_vsub(VFPRegister vd, VFPRegister vn, VFPRegister vm,
+ Condition c = Always);
+ BufferOffset as_vcmp(VFPRegister vd, VFPRegister vm, Condition c = Always);
+ BufferOffset as_vcmpz(VFPRegister vd, Condition c = Always);
+
+ // Specifically, a move between two same sized-registers.
+ BufferOffset as_vmov(VFPRegister vd, VFPRegister vsrc, Condition c = Always);
+
+ // Transfer between Core and VFP.
+ enum FloatToCore_ { FloatToCore = 1 << 20, CoreToFloat = 0 << 20 };
+
+ private:
+ enum VFPXferSize { WordTransfer = 0x02000010, DoubleTransfer = 0x00400010 };
+
+ public:
+ // Unlike the next function, moving between the core registers and vfp
+ // registers can't be *that* properly typed. Namely, since I don't want to
+ // munge the type VFPRegister to also include core registers. Thus, the core
+ // and vfp registers are passed in based on their type, and src/dest is
+ // determined by the float2core.
+
+ BufferOffset as_vxfer(Register vt1, Register vt2, VFPRegister vm,
+ FloatToCore_ f2c, Condition c = Always, int idx = 0);
+
+ // Our encoding actually allows just the src and the dest (and their types)
+ // to uniquely specify the encoding that we are going to use.
+ BufferOffset as_vcvt(VFPRegister vd, VFPRegister vm, bool useFPSCR = false,
+ Condition c = Always);
+
+ // Hard coded to a 32 bit fixed width result for now.
+ BufferOffset as_vcvtFixed(VFPRegister vd, bool isSigned, uint32_t fixedPoint,
+ bool toFixed, Condition c = Always);
+
+ // Transfer between VFP and memory.
+ BufferOffset as_vdtr(LoadStore ls, VFPRegister vd, VFPAddr addr,
+ Condition c = Always /* vfp doesn't have a wb option*/);
+
+ static void as_vdtr_patch(LoadStore ls, VFPRegister vd, VFPAddr addr,
+ Condition c /* vfp doesn't have a wb option */,
+ uint32_t* dest);
+
+ // VFP's ldm/stm work differently from the standard arm ones. You can only
+ // transfer a range.
+
+ BufferOffset as_vdtm(LoadStore st, Register rn, VFPRegister vd, int length,
+ /* also has update conditions */ Condition c = Always);
+
+ // vldr/vstr variants that handle unaligned accesses. These encode as NEON
+ // single-element instructions and can only be used if NEON is available.
+ // Here, vd must be tagged as a float or double register.
+ BufferOffset as_vldr_unaligned(VFPRegister vd, Register rn);
+ BufferOffset as_vstr_unaligned(VFPRegister vd, Register rn);
+
+ BufferOffset as_vimm(VFPRegister vd, VFPImm imm, Condition c = Always);
+
+ BufferOffset as_vmrs(Register r, Condition c = Always);
+ BufferOffset as_vmsr(Register r, Condition c = Always);
+
+ // Label operations.
+ bool nextLink(BufferOffset b, BufferOffset* next);
+ void bind(Label* label, BufferOffset boff = BufferOffset());
+ uint32_t currentOffset() { return nextOffset().getOffset(); }
+ void retarget(Label* label, Label* target);
+ // I'm going to pretend this doesn't exist for now.
+ void retarget(Label* label, void* target, RelocationKind reloc);
+
+ static void Bind(uint8_t* rawCode, const CodeLabel& label);
+
+ void as_bkpt();
+ BufferOffset as_illegal_trap();
+
+ public:
+ static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+ static void TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ void assertNoGCThings() const {
+#ifdef DEBUG
+ MOZ_ASSERT(dataRelocations_.length() == 0);
+ for (auto& j : jumps_) {
+ MOZ_ASSERT(j.kind() == RelocationKind::HARDCODED);
+ }
+#endif
+ }
+
+ static bool SupportsFloatingPoint() { return HasVFP(); }
+ static bool SupportsUnalignedAccesses() { return HasARMv7(); }
+ // Note, returning false here is technically wrong, but one has to go via the
+ // as_vldr_unaligned and as_vstr_unaligned instructions to get proper behavior
+ // and those are NEON-specific and have to be asked for specifically.
+ static bool SupportsFastUnalignedFPAccesses() { return false; }
+
+ static bool HasRoundInstruction(RoundingMode mode) { return false; }
+
+ protected:
+ void addPendingJump(BufferOffset src, ImmPtr target, RelocationKind kind) {
+ enoughMemory_ &= jumps_.append(RelativePatch(target.value, kind));
+ if (kind == RelocationKind::JITCODE) {
+ jumpRelocations_.writeUnsigned(src.getOffset());
+ }
+ }
+
+ public:
+ // The buffer is about to be linked, make sure any constant pools or excess
+ // bookkeeping has been flushed to the instruction stream.
+ void flush() {
+ MOZ_ASSERT(!isFinished);
+ m_buffer.flushPool();
+ return;
+ }
+
+ void comment(const char* msg) {
+#ifdef JS_DISASM_ARM
+ spew_.spew("; %s", msg);
+#endif
+ }
+
+ // Copy the assembly code to the given buffer, and perform any pending
+ // relocations relying on the target address.
+ void executableCopy(uint8_t* buffer);
+
+ // Actual assembly emitting functions.
+
+ // Since I can't think of a reasonable default for the mode, I'm going to
+ // leave it as a required argument.
+ void startDataTransferM(LoadStore ls, Register rm, DTMMode mode,
+ DTMWriteBack update = NoWriteBack,
+ Condition c = Always) {
+ MOZ_ASSERT(!dtmActive);
+ dtmUpdate = update;
+ dtmBase = rm;
+ dtmLoadStore = ls;
+ dtmLastReg = -1;
+ dtmRegBitField = 0;
+ dtmActive = 1;
+ dtmCond = c;
+ dtmMode = mode;
+ }
+
+ void transferReg(Register rn) {
+ MOZ_ASSERT(dtmActive);
+ MOZ_ASSERT(rn.code() > dtmLastReg);
+ dtmRegBitField |= 1 << rn.code();
+ if (dtmLoadStore == IsLoad && rn.code() == 13 && dtmBase.code() == 13) {
+ MOZ_CRASH("ARM Spec says this is invalid");
+ }
+ }
+ void finishDataTransfer() {
+ dtmActive = false;
+ as_dtm(dtmLoadStore, dtmBase, dtmRegBitField, dtmMode, dtmUpdate, dtmCond);
+ }
+
+ void startFloatTransferM(LoadStore ls, Register rm, DTMMode mode,
+ DTMWriteBack update = NoWriteBack,
+ Condition c = Always) {
+ MOZ_ASSERT(!dtmActive);
+ dtmActive = true;
+ dtmUpdate = update;
+ dtmLoadStore = ls;
+ dtmBase = rm;
+ dtmCond = c;
+ dtmLastReg = -1;
+ dtmMode = mode;
+ dtmDelta = 0;
+ }
+ void transferFloatReg(VFPRegister rn) {
+ if (dtmLastReg == -1) {
+ vdtmFirstReg = rn.code();
+ } else {
+ if (dtmDelta == 0) {
+ dtmDelta = rn.code() - dtmLastReg;
+ MOZ_ASSERT(dtmDelta == 1 || dtmDelta == -1);
+ }
+ MOZ_ASSERT(dtmLastReg >= 0);
+ MOZ_ASSERT(rn.code() == unsigned(dtmLastReg) + dtmDelta);
+ }
+
+ dtmLastReg = rn.code();
+ }
+ void finishFloatTransfer() {
+ MOZ_ASSERT(dtmActive);
+ dtmActive = false;
+ MOZ_ASSERT(dtmLastReg != -1);
+ dtmDelta = dtmDelta ? dtmDelta : 1;
+ // The operand for the vstr/vldr instruction is the lowest register in the
+ // range.
+ int low = std::min(dtmLastReg, vdtmFirstReg);
+ int high = std::max(dtmLastReg, vdtmFirstReg);
+ // Fencepost problem.
+ int len = high - low + 1;
+ // vdtm can only transfer 16 registers at once. If we need to transfer
+ // more, then either hoops are necessary, or we need to be updating the
+ // register.
+ MOZ_ASSERT_IF(len > 16, dtmUpdate == WriteBack);
+
+ int adjustLow = dtmLoadStore == IsStore ? 0 : 1;
+ int adjustHigh = dtmLoadStore == IsStore ? -1 : 0;
+ while (len > 0) {
+ // Limit the instruction to 16 registers.
+ int curLen = std::min(len, 16);
+ // If it is a store, we want to start at the high end and move down
+ // (e.g. vpush d16-d31; vpush d0-d15).
+ int curStart = (dtmLoadStore == IsStore) ? high - curLen + 1 : low;
+ as_vdtm(dtmLoadStore, dtmBase,
+ VFPRegister(FloatRegister::FromCode(curStart)), curLen, dtmCond);
+ // Update the bounds.
+ low += adjustLow * curLen;
+ high += adjustHigh * curLen;
+ // Update the length parameter.
+ len -= curLen;
+ }
+ }
+
+ private:
+ int dtmRegBitField;
+ int vdtmFirstReg;
+ int dtmLastReg;
+ int dtmDelta;
+ Register dtmBase;
+ DTMWriteBack dtmUpdate;
+ DTMMode dtmMode;
+ LoadStore dtmLoadStore;
+ bool dtmActive;
+ Condition dtmCond;
+
+ public:
+ enum {
+ PadForAlign8 = (int)0x00,
+ PadForAlign16 = (int)0x0000,
+ PadForAlign32 = (int)0xe12fff7f // 'bkpt 0xffff'
+ };
+
+ // API for speaking with the IonAssemblerBufferWithConstantPools generate an
+ // initial placeholder instruction that we want to later fix up.
+ static void InsertIndexIntoTag(uint8_t* load, uint32_t index);
+
+ // Take the stub value that was written in before, and write in an actual
+ // load using the index we'd computed previously as well as the address of
+ // the pool start.
+ static void PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr);
+
+ // We're not tracking short-range branches for ARM for now.
+ static void PatchShortRangeBranchToVeneer(ARMBuffer*, unsigned rangeIdx,
+ BufferOffset deadline,
+ BufferOffset veneer) {
+ MOZ_CRASH();
+ }
+ // END API
+
+ // Move our entire pool into the instruction stream. This is to force an
+ // opportunistic dump of the pool, prefferably when it is more convenient to
+ // do a dump.
+ void flushBuffer();
+ void enterNoPool(size_t maxInst);
+ void leaveNoPool();
+ void enterNoNops();
+ void leaveNoNops();
+
+ static void WritePoolHeader(uint8_t* start, Pool* p, bool isNatural);
+ static void WritePoolGuard(BufferOffset branch, Instruction* inst,
+ BufferOffset dest);
+
+ static uint32_t PatchWrite_NearCallSize();
+ static uint32_t NopSize() { return 4; }
+ static void PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall);
+ static void PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue);
+ static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
+ ImmPtr expectedValue);
+ static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm);
+
+ static uint32_t AlignDoubleArg(uint32_t offset) { return (offset + 1) & ~1; }
+ static uint8_t* NextInstruction(uint8_t* instruction,
+ uint32_t* count = nullptr);
+
+ // Toggle a jmp or cmp emitted by toggledJump().
+ static void ToggleToJmp(CodeLocationLabel inst_);
+ static void ToggleToCmp(CodeLocationLabel inst_);
+
+ static size_t ToggledCallSize(uint8_t* code);
+ static void ToggleCall(CodeLocationLabel inst_, bool enabled);
+
+ void processCodeLabels(uint8_t* rawCode);
+
+ void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
+ const Disassembler::HeapAccess& heapAccess) {
+ // Implement this if we implement a disassembler.
+ }
+}; // Assembler
+
+// An Instruction is a structure for both encoding and decoding any and all ARM
+// instructions. Many classes have not been implemented thus far.
+class Instruction {
+ uint32_t data;
+
+ protected:
+ // This is not for defaulting to always, this is for instructions that
+ // cannot be made conditional, and have the usually invalid 4b1111 cond
+ // field.
+ explicit Instruction(uint32_t data_, bool fake = false)
+ : data(data_ | 0xf0000000) {
+ MOZ_ASSERT(fake || ((data_ & 0xf0000000) == 0));
+ }
+ // Standard constructor.
+ Instruction(uint32_t data_, Assembler::Condition c)
+ : data(data_ | (uint32_t)c) {
+ MOZ_ASSERT((data_ & 0xf0000000) == 0);
+ }
+ // You should never create an instruction directly. You should create a more
+ // specific instruction which will eventually call one of these constructors
+ // for you.
+ public:
+ uint32_t encode() const { return data; }
+ // Check if this instruction is really a particular case.
+ template <class C>
+ bool is() const {
+ return C::IsTHIS(*this);
+ }
+
+ // Safely get a more specific variant of this pointer.
+ template <class C>
+ C* as() const {
+ return C::AsTHIS(*this);
+ }
+
+ const Instruction& operator=(Instruction src) {
+ data = src.data;
+ return *this;
+ }
+ // Since almost all instructions have condition codes, the condition code
+ // extractor resides in the base class.
+ Assembler::Condition extractCond() const {
+ MOZ_ASSERT(data >> 28 != 0xf,
+ "The instruction does not have condition code");
+ return (Assembler::Condition)(data & 0xf0000000);
+ }
+
+ // Sometimes, an api wants a uint32_t (or a pointer to it) rather than an
+ // instruction. raw() just coerces this into a pointer to a uint32_t.
+ const uint32_t* raw() const { return &data; }
+ uint32_t size() const { return 4; }
+}; // Instruction
+
+// Make sure that it is the right size.
+static_assert(sizeof(Instruction) == 4);
+
+inline void InstructionIterator::advanceRaw(ptrdiff_t instructions) {
+ inst_ = inst_ + instructions;
+}
+
+// Data Transfer Instructions.
+class InstDTR : public Instruction {
+ public:
+ enum IsByte_ { IsByte = 0x00400000, IsWord = 0x00000000 };
+ static const int IsDTR = 0x04000000;
+ static const int IsDTRMask = 0x0c000000;
+
+ // TODO: Replace the initialization with something that is safer.
+ InstDTR(LoadStore ls, IsByte_ ib, Index mode, Register rt, DTRAddr addr,
+ Assembler::Condition c)
+ : Instruction(std::underlying_type_t<LoadStore>(ls) |
+ std::underlying_type_t<IsByte_>(ib) |
+ std::underlying_type_t<Index>(mode) | RT(rt) |
+ addr.encode() | IsDTR,
+ c) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstDTR* AsTHIS(const Instruction& i);
+};
+static_assert(sizeof(InstDTR) == sizeof(Instruction));
+
+class InstLDR : public InstDTR {
+ public:
+ InstLDR(Index mode, Register rt, DTRAddr addr, Assembler::Condition c)
+ : InstDTR(IsLoad, IsWord, mode, rt, addr, c) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstLDR* AsTHIS(const Instruction& i);
+
+ int32_t signedOffset() const {
+ int32_t offset = encode() & 0xfff;
+ if (IsUp_(encode() & IsUp) != IsUp) {
+ return -offset;
+ }
+ return offset;
+ }
+ uint32_t* dest() const {
+ int32_t offset = signedOffset();
+ // When patching the load in PatchConstantPoolLoad, we ensure that the
+ // offset is a multiple of 4, offset by 8 bytes from the actual
+ // location. Indeed, when the base register is PC, ARM's 3 stages
+ // pipeline design makes it that PC is off by 8 bytes (= 2 *
+ // sizeof(uint32*)) when we actually executed it.
+ MOZ_ASSERT(offset % 4 == 0);
+ offset >>= 2;
+ return (uint32_t*)raw() + offset + 2;
+ }
+};
+static_assert(sizeof(InstDTR) == sizeof(InstLDR));
+
+class InstNOP : public Instruction {
+ public:
+ static const uint32_t NopInst = 0x0320f000;
+
+ InstNOP() : Instruction(NopInst, Assembler::Always) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstNOP* AsTHIS(Instruction& i);
+};
+
+// Branching to a register, or calling a register
+class InstBranchReg : public Instruction {
+ protected:
+ // Don't use BranchTag yourself, use a derived instruction.
+ enum BranchTag { IsBX = 0x012fff10, IsBLX = 0x012fff30 };
+
+ static const uint32_t IsBRegMask = 0x0ffffff0;
+
+ InstBranchReg(BranchTag tag, Register rm, Assembler::Condition c)
+ : Instruction(tag | rm.code(), c) {}
+
+ public:
+ static bool IsTHIS(const Instruction& i);
+ static InstBranchReg* AsTHIS(const Instruction& i);
+
+ // Get the register that is being branched to
+ void extractDest(Register* dest);
+ // Make sure we are branching to a pre-known register
+ bool checkDest(Register dest);
+};
+static_assert(sizeof(InstBranchReg) == sizeof(Instruction));
+
+// Branching to an immediate offset, or calling an immediate offset
+class InstBranchImm : public Instruction {
+ protected:
+ enum BranchTag { IsB = 0x0a000000, IsBL = 0x0b000000 };
+
+ static const uint32_t IsBImmMask = 0x0f000000;
+
+ InstBranchImm(BranchTag tag, BOffImm off, Assembler::Condition c)
+ : Instruction(tag | off.encode(), c) {}
+
+ public:
+ static bool IsTHIS(const Instruction& i);
+ static InstBranchImm* AsTHIS(const Instruction& i);
+
+ void extractImm(BOffImm* dest);
+};
+static_assert(sizeof(InstBranchImm) == sizeof(Instruction));
+
+// Very specific branching instructions.
+class InstBXReg : public InstBranchReg {
+ public:
+ static bool IsTHIS(const Instruction& i);
+ static InstBXReg* AsTHIS(const Instruction& i);
+};
+
+class InstBLXReg : public InstBranchReg {
+ public:
+ InstBLXReg(Register reg, Assembler::Condition c)
+ : InstBranchReg(IsBLX, reg, c) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstBLXReg* AsTHIS(const Instruction& i);
+};
+
+class InstBImm : public InstBranchImm {
+ public:
+ InstBImm(BOffImm off, Assembler::Condition c) : InstBranchImm(IsB, off, c) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstBImm* AsTHIS(const Instruction& i);
+};
+
+class InstBLImm : public InstBranchImm {
+ public:
+ InstBLImm(BOffImm off, Assembler::Condition c)
+ : InstBranchImm(IsBL, off, c) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstBLImm* AsTHIS(const Instruction& i);
+};
+
+// Both movw and movt. The layout of both the immediate and the destination
+// register is the same so the code is being shared.
+class InstMovWT : public Instruction {
+ protected:
+ enum WT { IsW = 0x03000000, IsT = 0x03400000 };
+ static const uint32_t IsWTMask = 0x0ff00000;
+
+ InstMovWT(Register rd, Imm16 imm, WT wt, Assembler::Condition c)
+ : Instruction(RD(rd) | imm.encode() | wt, c) {}
+
+ public:
+ void extractImm(Imm16* dest);
+ void extractDest(Register* dest);
+ bool checkImm(Imm16 dest);
+ bool checkDest(Register dest);
+
+ static bool IsTHIS(Instruction& i);
+ static InstMovWT* AsTHIS(Instruction& i);
+};
+static_assert(sizeof(InstMovWT) == sizeof(Instruction));
+
+class InstMovW : public InstMovWT {
+ public:
+ InstMovW(Register rd, Imm16 imm, Assembler::Condition c)
+ : InstMovWT(rd, imm, IsW, c) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstMovW* AsTHIS(const Instruction& i);
+};
+
+class InstMovT : public InstMovWT {
+ public:
+ InstMovT(Register rd, Imm16 imm, Assembler::Condition c)
+ : InstMovWT(rd, imm, IsT, c) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstMovT* AsTHIS(const Instruction& i);
+};
+
+class InstALU : public Instruction {
+ static const int32_t ALUMask = 0xc << 24;
+
+ public:
+ InstALU(Register rd, Register rn, Operand2 op2, ALUOp op, SBit s,
+ Assembler::Condition c)
+ : Instruction(maybeRD(rd) | maybeRN(rn) | op2.encode() | op | s, c) {}
+
+ static bool IsTHIS(const Instruction& i);
+ static InstALU* AsTHIS(const Instruction& i);
+
+ void extractOp(ALUOp* ret);
+ bool checkOp(ALUOp op);
+ void extractDest(Register* ret);
+ bool checkDest(Register rd);
+ void extractOp1(Register* ret);
+ bool checkOp1(Register rn);
+ Operand2 extractOp2();
+};
+
+class InstCMP : public InstALU {
+ public:
+ static bool IsTHIS(const Instruction& i);
+ static InstCMP* AsTHIS(const Instruction& i);
+};
+
+class InstMOV : public InstALU {
+ public:
+ static bool IsTHIS(const Instruction& i);
+ static InstMOV* AsTHIS(const Instruction& i);
+};
+
+// Compile-time iterator over instructions, with a safe interface that
+// references not-necessarily-linear Instructions by linear BufferOffset.
+class BufferInstructionIterator
+ : public ARMBuffer::AssemblerBufferInstIterator {
+ public:
+ BufferInstructionIterator(BufferOffset bo, ARMBuffer* buffer)
+ : ARMBuffer::AssemblerBufferInstIterator(bo, buffer) {}
+
+ // Advances the buffer to the next intentionally-inserted instruction.
+ Instruction* next() {
+ advance(cur()->size());
+ maybeSkipAutomaticInstructions();
+ return cur();
+ }
+
+ // Advances the BufferOffset past any automatically-inserted instructions.
+ Instruction* maybeSkipAutomaticInstructions();
+};
+
+static const uint32_t NumIntArgRegs = 4;
+
+// There are 16 *float* registers available for arguments
+// If doubles are used, only half the number of registers are available.
+static const uint32_t NumFloatArgRegs = 16;
+
+static inline bool GetIntArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs,
+ Register* out) {
+ if (usedIntArgs >= NumIntArgRegs) {
+ return false;
+ }
+
+ *out = Register::FromCode(usedIntArgs);
+ return true;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs, Register* out) {
+ if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) {
+ return true;
+ }
+
+ // Unfortunately, we have to assume things about the point at which
+ // GetIntArgReg returns false, because we need to know how many registers it
+ // can allocate.
+ usedIntArgs -= NumIntArgRegs;
+ if (usedIntArgs >= NumCallTempNonArgRegs) {
+ return false;
+ }
+
+ *out = CallTempNonArgRegs[usedIntArgs];
+ return true;
+}
+
+#if defined(JS_CODEGEN_ARM_HARDFP) || defined(JS_SIMULATOR_ARM)
+
+static inline bool GetFloat32ArgReg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs,
+ FloatRegister* out) {
+ MOZ_ASSERT(UseHardFpABI());
+ if (usedFloatArgs >= NumFloatArgRegs) {
+ return false;
+ }
+ *out = VFPRegister(usedFloatArgs, VFPRegister::Single);
+ return true;
+}
+static inline bool GetDoubleArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs,
+ FloatRegister* out) {
+ MOZ_ASSERT(UseHardFpABI());
+ MOZ_ASSERT((usedFloatArgs % 2) == 0);
+ if (usedFloatArgs >= NumFloatArgRegs) {
+ return false;
+ }
+ *out = VFPRegister(usedFloatArgs >> 1, VFPRegister::Double);
+ return true;
+}
+
+#endif
+
+class DoubleEncoder {
+ struct DoubleEntry {
+ uint32_t dblTop;
+ datastore::Imm8VFPImmData data;
+ };
+
+ static const DoubleEntry table[256];
+
+ public:
+ bool lookup(uint32_t top, datastore::Imm8VFPImmData* ret) const {
+ for (int i = 0; i < 256; i++) {
+ if (table[i].dblTop == top) {
+ *ret = table[i].data;
+ return true;
+ }
+ }
+ return false;
+ }
+};
+
+// Forbids nop filling for testing purposes. Not nestable.
+class AutoForbidNops {
+ protected:
+ Assembler* masm_;
+
+ public:
+ explicit AutoForbidNops(Assembler* masm) : masm_(masm) {
+ masm_->enterNoNops();
+ }
+ ~AutoForbidNops() { masm_->leaveNoNops(); }
+};
+
+class AutoForbidPoolsAndNops : public AutoForbidNops {
+ public:
+ // The maxInst argument is the maximum number of word sized instructions
+ // that will be allocated within this context. It is used to determine if
+ // the pool needs to be dumped before entering this content. The debug code
+ // checks that no more than maxInst instructions are actually allocated.
+ //
+ // Allocation of pool entries is not supported within this content so the
+ // code can not use large integers or float constants etc.
+ AutoForbidPoolsAndNops(Assembler* masm, size_t maxInst)
+ : AutoForbidNops(masm) {
+ masm_->enterNoPool(maxInst);
+ }
+
+ ~AutoForbidPoolsAndNops() { masm_->leaveNoPool(); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_Assembler_arm_h */
diff --git a/js/src/jit/arm/CodeGenerator-arm.cpp b/js/src/jit/arm/CodeGenerator-arm.cpp
new file mode 100644
index 0000000000..89e600c95b
--- /dev/null
+++ b/js/src/jit/arm/CodeGenerator-arm.cpp
@@ -0,0 +1,3154 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm/CodeGenerator-arm.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+
+#include <iterator>
+
+#include "jsnum.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitRuntime.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "js/Conversions.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+#include "vm/Shape.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::GenericNaN;
+using JS::ToInt32;
+using mozilla::DebugOnly;
+using mozilla::FloorLog2;
+using mozilla::NegativeInfinity;
+
+// shared
+CodeGeneratorARM::CodeGeneratorARM(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorShared(gen, graph, masm) {}
+
+Register64 CodeGeneratorARM::ToOperandOrRegister64(
+ const LInt64Allocation input) {
+ return ToRegister64(input);
+}
+
+void CodeGeneratorARM::emitBranch(Assembler::Condition cond,
+ MBasicBlock* mirTrue, MBasicBlock* mirFalse) {
+ if (isNextBlock(mirFalse->lir())) {
+ jumpToBlock(mirTrue, cond);
+ } else {
+ jumpToBlock(mirFalse, Assembler::InvertCondition(cond));
+ jumpToBlock(mirTrue);
+ }
+}
+
+void OutOfLineBailout::accept(CodeGeneratorARM* codegen) {
+ codegen->visitOutOfLineBailout(this);
+}
+
+void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
+ const LAllocation* opd = test->getOperand(0);
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ // Test the operand
+ masm.as_cmp(ToRegister(opd), Imm8(0));
+
+ if (isNextBlock(ifFalse->lir())) {
+ jumpToBlock(ifTrue, Assembler::NonZero);
+ } else if (isNextBlock(ifTrue->lir())) {
+ jumpToBlock(ifFalse, Assembler::Zero);
+ } else {
+ jumpToBlock(ifFalse, Assembler::Zero);
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitCompare(LCompare* comp) {
+ Assembler::Condition cond =
+ JSOpToCondition(comp->mir()->compareType(), comp->jsop());
+ const LAllocation* left = comp->getOperand(0);
+ const LAllocation* right = comp->getOperand(1);
+ const LDefinition* def = comp->getDef(0);
+
+ ScratchRegisterScope scratch(masm);
+
+ if (right->isConstant()) {
+ masm.ma_cmp(ToRegister(left), Imm32(ToInt32(right)), scratch);
+ } else if (right->isRegister()) {
+ masm.ma_cmp(ToRegister(left), ToRegister(right));
+ } else {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.ma_cmp(ToRegister(left), Operand(ToAddress(right)), scratch, scratch2);
+ }
+ masm.ma_mov(Imm32(0), ToRegister(def));
+ masm.ma_mov(Imm32(1), ToRegister(def), cond);
+}
+
+void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
+ Assembler::Condition cond =
+ JSOpToCondition(comp->cmpMir()->compareType(), comp->jsop());
+ const LAllocation* left = comp->left();
+ const LAllocation* right = comp->right();
+
+ ScratchRegisterScope scratch(masm);
+
+ if (right->isConstant()) {
+ masm.ma_cmp(ToRegister(left), Imm32(ToInt32(right)), scratch);
+ } else if (right->isRegister()) {
+ masm.ma_cmp(ToRegister(left), ToRegister(right));
+ } else {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.ma_cmp(ToRegister(left), Operand(ToAddress(right)), scratch, scratch2);
+ }
+ emitBranch(cond, comp->ifTrue(), comp->ifFalse());
+}
+
+bool CodeGeneratorARM::generateOutOfLineCode() {
+ if (!CodeGeneratorShared::generateOutOfLineCode()) {
+ return false;
+ }
+
+ if (deoptLabel_.used()) {
+ // All non-table-based bailouts will go here.
+ masm.bind(&deoptLabel_);
+
+ // Push the frame size, so the handler can recover the IonScript.
+ masm.push(Imm32(frameSize()));
+
+ TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
+ masm.jump(handler);
+ }
+
+ return !masm.oom();
+}
+
+void CodeGeneratorARM::bailoutIf(Assembler::Condition condition,
+ LSnapshot* snapshot) {
+ encode(snapshot);
+
+ InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
+ OutOfLineBailout* ool =
+ new (alloc()) OutOfLineBailout(snapshot, masm.framePushed());
+
+ // All bailout code is associated with the bytecodeSite of the block we are
+ // bailing out from.
+ addOutOfLineCode(ool,
+ new (alloc()) BytecodeSite(tree, tree->script()->code()));
+
+ masm.ma_b(ool->entry(), condition);
+}
+
+void CodeGeneratorARM::bailoutFrom(Label* label, LSnapshot* snapshot) {
+ MOZ_ASSERT_IF(!masm.oom(), label->used());
+ MOZ_ASSERT_IF(!masm.oom(), !label->bound());
+
+ encode(snapshot);
+
+ InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
+ OutOfLineBailout* ool =
+ new (alloc()) OutOfLineBailout(snapshot, masm.framePushed());
+
+ // All bailout code is associated with the bytecodeSite of the block we are
+ // bailing out from.
+ addOutOfLineCode(ool,
+ new (alloc()) BytecodeSite(tree, tree->script()->code()));
+
+ masm.retarget(label, ool->entry());
+}
+
+void CodeGeneratorARM::bailout(LSnapshot* snapshot) {
+ Label label;
+ masm.ma_b(&label);
+ bailoutFrom(&label, snapshot);
+}
+
+void CodeGeneratorARM::visitOutOfLineBailout(OutOfLineBailout* ool) {
+ masm.push(Imm32(ool->snapshot()->snapshotOffset()));
+ masm.ma_b(&deoptLabel_);
+}
+
+void CodeGenerator::visitMinMaxD(LMinMaxD* ins) {
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+
+ MOZ_ASSERT(first == ToFloatRegister(ins->output()));
+
+ if (ins->mir()->isMax()) {
+ masm.maxDouble(second, first, true);
+ } else {
+ masm.minDouble(second, first, true);
+ }
+}
+
+void CodeGenerator::visitMinMaxF(LMinMaxF* ins) {
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+
+ MOZ_ASSERT(first == ToFloatRegister(ins->output()));
+
+ if (ins->mir()->isMax()) {
+ masm.maxFloat32(second, first, true);
+ } else {
+ masm.minFloat32(second, first, true);
+ }
+}
+
+void CodeGenerator::visitAddI(LAddI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ ScratchRegisterScope scratch(masm);
+
+ if (rhs->isConstant()) {
+ masm.ma_add(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest), scratch,
+ SetCC);
+ } else if (rhs->isRegister()) {
+ masm.ma_add(ToRegister(lhs), ToRegister(rhs), ToRegister(dest), SetCC);
+ } else {
+ masm.ma_add(ToRegister(lhs), Operand(ToAddress(rhs)), ToRegister(dest),
+ SetCC);
+ }
+
+ if (ins->snapshot()) {
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitAddI64(LAddI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void CodeGenerator::visitSubI(LSubI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ ScratchRegisterScope scratch(masm);
+
+ if (rhs->isConstant()) {
+ masm.ma_sub(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest), scratch,
+ SetCC);
+ } else if (rhs->isRegister()) {
+ masm.ma_sub(ToRegister(lhs), ToRegister(rhs), ToRegister(dest), SetCC);
+ } else {
+ masm.ma_sub(ToRegister(lhs), Operand(ToAddress(rhs)), ToRegister(dest),
+ SetCC);
+ }
+
+ if (ins->snapshot()) {
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitSubI64(LSubI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void CodeGenerator::visitMulI(LMulI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+ MMul* mul = ins->mir();
+ MOZ_ASSERT_IF(mul->mode() == MMul::Integer,
+ !mul->canBeNegativeZero() && !mul->canOverflow());
+
+ if (rhs->isConstant()) {
+ // Bailout when this condition is met.
+ Assembler::Condition c = Assembler::Overflow;
+ // Bailout on -0.0
+ int32_t constant = ToInt32(rhs);
+ if (mul->canBeNegativeZero() && constant <= 0) {
+ Assembler::Condition bailoutCond =
+ (constant == 0) ? Assembler::LessThan : Assembler::Equal;
+ masm.as_cmp(ToRegister(lhs), Imm8(0));
+ bailoutIf(bailoutCond, ins->snapshot());
+ }
+ // TODO: move these to ma_mul.
+ switch (constant) {
+ case -1:
+ masm.as_rsb(ToRegister(dest), ToRegister(lhs), Imm8(0), SetCC);
+ break;
+ case 0:
+ masm.ma_mov(Imm32(0), ToRegister(dest));
+ return; // Escape overflow check;
+ case 1:
+ // Nop
+ masm.ma_mov(ToRegister(lhs), ToRegister(dest));
+ return; // Escape overflow check;
+ case 2:
+ masm.ma_add(ToRegister(lhs), ToRegister(lhs), ToRegister(dest), SetCC);
+ // Overflow is handled later.
+ break;
+ default: {
+ bool handled = false;
+ if (constant > 0) {
+ // Try shift and add sequences for a positive constant.
+ if (!mul->canOverflow()) {
+ // If it cannot overflow, we can do lots of optimizations.
+ Register src = ToRegister(lhs);
+ uint32_t shift = FloorLog2(constant);
+ uint32_t rest = constant - (1 << shift);
+ // See if the constant has one bit set, meaning it can be
+ // encoded as a bitshift.
+ if ((1 << shift) == constant) {
+ masm.ma_lsl(Imm32(shift), src, ToRegister(dest));
+ handled = true;
+ } else {
+ // If the constant cannot be encoded as (1 << C1), see
+ // if it can be encoded as (1 << C1) | (1 << C2), which
+ // can be computed using an add and a shift.
+ uint32_t shift_rest = FloorLog2(rest);
+ if ((1u << shift_rest) == rest) {
+ masm.as_add(ToRegister(dest), src,
+ lsl(src, shift - shift_rest));
+ if (shift_rest != 0) {
+ masm.ma_lsl(Imm32(shift_rest), ToRegister(dest),
+ ToRegister(dest));
+ }
+ handled = true;
+ }
+ }
+ } else if (ToRegister(lhs) != ToRegister(dest)) {
+ // To stay on the safe side, only optimize things that are a
+ // power of 2.
+
+ uint32_t shift = FloorLog2(constant);
+ if ((1 << shift) == constant) {
+ // dest = lhs * pow(2,shift)
+ masm.ma_lsl(Imm32(shift), ToRegister(lhs), ToRegister(dest));
+ // At runtime, check (lhs == dest >> shift), if this
+ // does not hold, some bits were lost due to overflow,
+ // and the computation should be resumed as a double.
+ masm.as_cmp(ToRegister(lhs), asr(ToRegister(dest), shift));
+ c = Assembler::NotEqual;
+ handled = true;
+ }
+ }
+ }
+
+ if (!handled) {
+ ScratchRegisterScope scratch(masm);
+ if (mul->canOverflow()) {
+ c = masm.ma_check_mul(ToRegister(lhs), Imm32(ToInt32(rhs)),
+ ToRegister(dest), scratch, c);
+ } else {
+ masm.ma_mul(ToRegister(lhs), Imm32(ToInt32(rhs)), ToRegister(dest),
+ scratch);
+ }
+ }
+ }
+ }
+ // Bailout on overflow.
+ if (mul->canOverflow()) {
+ bailoutIf(c, ins->snapshot());
+ }
+ } else {
+ Assembler::Condition c = Assembler::Overflow;
+
+ if (mul->canOverflow()) {
+ ScratchRegisterScope scratch(masm);
+ c = masm.ma_check_mul(ToRegister(lhs), ToRegister(rhs), ToRegister(dest),
+ scratch, c);
+ } else {
+ masm.ma_mul(ToRegister(lhs), ToRegister(rhs), ToRegister(dest));
+ }
+
+ // Bailout on overflow.
+ if (mul->canOverflow()) {
+ bailoutIf(c, ins->snapshot());
+ }
+
+ if (mul->canBeNegativeZero()) {
+ Label done;
+ masm.as_cmp(ToRegister(dest), Imm8(0));
+ masm.ma_b(&done, Assembler::NotEqual);
+
+ // Result is -0 if lhs or rhs is negative.
+ masm.ma_cmn(ToRegister(lhs), ToRegister(rhs));
+ bailoutIf(Assembler::Signed, ins->snapshot());
+
+ masm.bind(&done);
+ }
+ }
+}
+
+void CodeGenerator::visitMulI64(LMulI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
+
+ MOZ_ASSERT(ToRegister64(lhs) == ToOutRegister64(lir));
+
+ if (IsConstant(rhs)) {
+ int64_t constant = ToInt64(rhs);
+ switch (constant) {
+ case -1:
+ masm.neg64(ToRegister64(lhs));
+ return;
+ case 0:
+ masm.xor64(ToRegister64(lhs), ToRegister64(lhs));
+ return;
+ case 1:
+ // nop
+ return;
+ case 2:
+ masm.add64(ToRegister64(lhs), ToRegister64(lhs));
+ return;
+ default:
+ if (constant > 0) {
+ // Use shift if constant is power of 2.
+ int32_t shift = mozilla::FloorLog2(constant);
+ if (int64_t(1) << shift == constant) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ return;
+ }
+ }
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
+ }
+ } else {
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp);
+ }
+}
+
+void CodeGeneratorARM::divICommon(MDiv* mir, Register lhs, Register rhs,
+ Register output, LSnapshot* snapshot,
+ Label& done) {
+ ScratchRegisterScope scratch(masm);
+
+ if (mir->canBeNegativeOverflow()) {
+ // Handle INT32_MIN / -1;
+ // The integer division will give INT32_MIN, but we want -(double)INT32_MIN.
+
+ // Sets EQ if lhs == INT32_MIN.
+ masm.ma_cmp(lhs, Imm32(INT32_MIN), scratch);
+ // If EQ (LHS == INT32_MIN), sets EQ if rhs == -1.
+ masm.ma_cmp(rhs, Imm32(-1), scratch, Assembler::Equal);
+ if (mir->canTruncateOverflow()) {
+ if (mir->trapOnError()) {
+ Label ok;
+ masm.ma_b(&ok, Assembler::NotEqual);
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
+ masm.bind(&ok);
+ } else {
+ // (-INT32_MIN)|0 = INT32_MIN
+ Label skip;
+ masm.ma_b(&skip, Assembler::NotEqual);
+ masm.ma_mov(Imm32(INT32_MIN), output);
+ masm.ma_b(&done);
+ masm.bind(&skip);
+ }
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Equal, snapshot);
+ }
+ }
+
+ // Handle divide by zero.
+ if (mir->canBeDivideByZero()) {
+ masm.as_cmp(rhs, Imm8(0));
+ if (mir->canTruncateInfinities()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.ma_b(&nonZero, Assembler::NotEqual);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ // Infinity|0 == 0
+ Label skip;
+ masm.ma_b(&skip, Assembler::NotEqual);
+ masm.ma_mov(Imm32(0), output);
+ masm.ma_b(&done);
+ masm.bind(&skip);
+ }
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Equal, snapshot);
+ }
+ }
+
+ // Handle negative 0.
+ if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
+ Label nonzero;
+ masm.as_cmp(lhs, Imm8(0));
+ masm.ma_b(&nonzero, Assembler::NotEqual);
+ masm.as_cmp(rhs, Imm8(0));
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::LessThan, snapshot);
+ masm.bind(&nonzero);
+ }
+}
+
+void CodeGenerator::visitDivI(LDivI* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register temp = ToRegister(ins->getTemp(0));
+ Register output = ToRegister(ins->output());
+ MDiv* mir = ins->mir();
+
+ Label done;
+ divICommon(mir, lhs, rhs, output, ins->snapshot(), done);
+
+ if (mir->canTruncateRemainder()) {
+ masm.ma_sdiv(lhs, rhs, output);
+ } else {
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_sdiv(lhs, rhs, temp);
+ masm.ma_mul(temp, rhs, scratch);
+ masm.ma_cmp(lhs, scratch);
+ }
+ bailoutIf(Assembler::NotEqual, ins->snapshot());
+ masm.ma_mov(temp, output);
+ }
+
+ masm.bind(&done);
+}
+
+extern "C" {
+extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int);
+extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int);
+}
+
+void CodeGenerator::visitSoftDivI(LSoftDivI* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+ MDiv* mir = ins->mir();
+
+ Label done;
+ divICommon(mir, lhs, rhs, output, ins->snapshot(), done);
+
+ if (gen->compilingWasm()) {
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs);
+ masm.passABIArg(rhs);
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::aeabi_idivmod,
+ mozilla::Some(instanceOffset));
+ masm.Pop(InstanceReg);
+ } else {
+ using Fn = int64_t (*)(int, int);
+ masm.setupAlignedABICall();
+ masm.passABIArg(lhs);
+ masm.passABIArg(rhs);
+ masm.callWithABI<Fn, __aeabi_idivmod>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+ }
+
+ // idivmod returns the quotient in r0, and the remainder in r1.
+ if (!mir->canTruncateRemainder()) {
+ MOZ_ASSERT(mir->fallible());
+ masm.as_cmp(r1, Imm8(0));
+ bailoutIf(Assembler::NonZero, ins->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) {
+ MDiv* mir = ins->mir();
+ Register lhs = ToRegister(ins->numerator());
+ Register output = ToRegister(ins->output());
+ int32_t shift = ins->shift();
+
+ if (shift == 0) {
+ masm.ma_mov(lhs, output);
+ return;
+ }
+
+ if (!mir->isTruncated()) {
+ // If the remainder is != 0, bailout since this must be a double.
+ {
+ // The bailout code also needs the scratch register.
+ // Here it is only used as a dummy target to set CC flags.
+ ScratchRegisterScope scratch(masm);
+ masm.as_mov(scratch, lsl(lhs, 32 - shift), SetCC);
+ }
+ bailoutIf(Assembler::NonZero, ins->snapshot());
+ }
+
+ if (!mir->canBeNegativeDividend()) {
+ // Numerator is unsigned, so needs no adjusting. Do the shift.
+ masm.as_mov(output, asr(lhs, shift));
+ return;
+ }
+
+ // Adjust the value so that shifting produces a correctly rounded result
+ // when the numerator is negative. See 10-1 "Signed Division by a Known
+ // Power of 2" in Henry S. Warren, Jr.'s Hacker's Delight.
+ ScratchRegisterScope scratch(masm);
+
+ if (shift > 1) {
+ masm.as_mov(scratch, asr(lhs, 31));
+ masm.as_add(scratch, lhs, lsr(scratch, 32 - shift));
+ } else {
+ masm.as_add(scratch, lhs, lsr(lhs, 32 - shift));
+ }
+
+ // Do the shift.
+ masm.as_mov(output, asr(scratch, shift));
+}
+
+void CodeGeneratorARM::modICommon(MMod* mir, Register lhs, Register rhs,
+ Register output, LSnapshot* snapshot,
+ Label& done) {
+ // X % 0 is bad because it will give garbage (or abort), when it should give
+ // NaN.
+
+ if (mir->canBeDivideByZero()) {
+ masm.as_cmp(rhs, Imm8(0));
+ if (mir->isTruncated()) {
+ Label nonZero;
+ masm.ma_b(&nonZero, Assembler::NotEqual);
+ if (mir->trapOnError()) {
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ } else {
+ // NaN|0 == 0
+ masm.ma_mov(Imm32(0), output);
+ masm.ma_b(&done);
+ }
+ masm.bind(&nonZero);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Equal, snapshot);
+ }
+ }
+}
+
+void CodeGenerator::visitModI(LModI* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+ MMod* mir = ins->mir();
+
+ // Contrary to other architectures (notably x86) INT_MIN % -1 doesn't need to
+ // be handled separately. |ma_smod| computes the remainder using the |SDIV|
+ // and the |MLS| instructions. On overflow, |SDIV| truncates the result to
+ // 32-bit and returns INT_MIN, see ARM Architecture Reference Manual, SDIV
+ // instruction.
+ //
+ // mls(INT_MIN, sdiv(INT_MIN, -1), -1)
+ // = INT_MIN - (sdiv(INT_MIN, -1) * -1)
+ // = INT_MIN - (INT_MIN * -1)
+ // = INT_MIN - INT_MIN
+ // = 0
+ //
+ // And a zero remainder with a negative dividend is already handled below.
+
+ Label done;
+ modICommon(mir, lhs, rhs, output, ins->snapshot(), done);
+
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_smod(lhs, rhs, output, scratch);
+ }
+
+ // If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0.
+ if (mir->canBeNegativeDividend()) {
+ if (mir->isTruncated()) {
+ // -0.0|0 == 0
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ // See if X < 0
+ masm.as_cmp(output, Imm8(0));
+ masm.ma_b(&done, Assembler::NotEqual);
+ masm.as_cmp(lhs, Imm8(0));
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ }
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitSoftModI(LSoftModI* ins) {
+ // Extract the registers from this instruction.
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+ Register callTemp = ToRegister(ins->callTemp());
+ MMod* mir = ins->mir();
+ Label done;
+
+ // Save the lhs in case we end up with a 0 that should be a -0.0 because lhs <
+ // 0.
+ MOZ_ASSERT(callTemp != lhs);
+ MOZ_ASSERT(callTemp != rhs);
+ masm.ma_mov(lhs, callTemp);
+
+ // Prevent INT_MIN % -1.
+ //
+ // |aeabi_idivmod| is allowed to return any arbitrary value when called with
+ // |(INT_MIN, -1)|, see "Run-time ABI for the ARM architecture manual". Most
+ // implementations perform a non-trapping signed integer division and
+ // return the expected result, i.e. INT_MIN. But since we can't rely on this
+ // behavior, handle this case separately here.
+ if (mir->canBeNegativeDividend()) {
+ {
+ ScratchRegisterScope scratch(masm);
+ // Sets EQ if lhs == INT_MIN
+ masm.ma_cmp(lhs, Imm32(INT_MIN), scratch);
+ // If EQ (LHS == INT_MIN), sets EQ if rhs == -1
+ masm.ma_cmp(rhs, Imm32(-1), scratch, Assembler::Equal);
+ }
+ if (mir->isTruncated()) {
+ // (INT_MIN % -1)|0 == 0
+ Label skip;
+ masm.ma_b(&skip, Assembler::NotEqual);
+ masm.ma_mov(Imm32(0), output);
+ masm.ma_b(&done);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Equal, ins->snapshot());
+ }
+ }
+
+ modICommon(mir, lhs, rhs, output, ins->snapshot(), done);
+
+ if (gen->compilingWasm()) {
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs);
+ masm.passABIArg(rhs);
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::aeabi_idivmod,
+ mozilla::Some(instanceOffset));
+ masm.Pop(InstanceReg);
+ } else {
+ using Fn = int64_t (*)(int, int);
+ masm.setupAlignedABICall();
+ masm.passABIArg(lhs);
+ masm.passABIArg(rhs);
+ masm.callWithABI<Fn, __aeabi_idivmod>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+ }
+
+ MOZ_ASSERT(r1 != output);
+ masm.move32(r1, output);
+
+ // If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0
+ if (mir->canBeNegativeDividend()) {
+ if (mir->isTruncated()) {
+ // -0.0|0 == 0
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ // See if X < 0
+ masm.as_cmp(output, Imm8(0));
+ masm.ma_b(&done, Assembler::NotEqual);
+ masm.as_cmp(callTemp, Imm8(0));
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ }
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) {
+ Register in = ToRegister(ins->getOperand(0));
+ Register out = ToRegister(ins->getDef(0));
+ MMod* mir = ins->mir();
+ Label fin;
+ // bug 739870, jbramley has a different sequence that may help with speed
+ // here.
+
+ masm.ma_mov(in, out, SetCC);
+ masm.ma_b(&fin, Assembler::Zero);
+ masm.as_rsb(out, out, Imm8(0), LeaveCC, Assembler::Signed);
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_and(Imm32((1 << ins->shift()) - 1), out, scratch);
+ }
+ masm.as_rsb(out, out, Imm8(0), SetCC, Assembler::Signed);
+ if (mir->canBeNegativeDividend()) {
+ if (!mir->isTruncated()) {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ } else {
+ // -0|0 == 0
+ }
+ }
+ masm.bind(&fin);
+}
+
+void CodeGenerator::visitModMaskI(LModMaskI* ins) {
+ Register src = ToRegister(ins->getOperand(0));
+ Register dest = ToRegister(ins->getDef(0));
+ Register tmp1 = ToRegister(ins->getTemp(0));
+ Register tmp2 = ToRegister(ins->getTemp(1));
+ MMod* mir = ins->mir();
+
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+
+ masm.ma_mod_mask(src, dest, tmp1, tmp2, scratch, scratch2, ins->shift());
+
+ if (mir->canBeNegativeDividend()) {
+ if (!mir->isTruncated()) {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ } else {
+ // -0|0 == 0
+ }
+ }
+}
+
+void CodeGeneratorARM::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ if (HasIDIV()) {
+ masm.ma_sdiv(dividend, divisor, /* result= */ dividend);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+
+ return;
+ }
+
+ // idivmod returns the quotient in r0, and the remainder in r1.
+ MOZ_ASSERT(dividend == r0);
+ MOZ_ASSERT(divisor == r1);
+
+ LiveRegisterSet volatileRegs = liveVolatileRegs(ins);
+ volatileRegs.takeUnchecked(dividend);
+ volatileRegs.takeUnchecked(divisor);
+ volatileRegs.takeUnchecked(output);
+
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = int64_t (*)(int, int);
+ masm.setupUnalignedABICall(output);
+ masm.passABIArg(dividend);
+ masm.passABIArg(divisor);
+ masm.callWithABI<Fn, __aeabi_idivmod>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.PopRegsInMask(volatileRegs);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorARM::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ if (HasIDIV()) {
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_smod(dividend, divisor, /* result= */ dividend, scratch);
+ }
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+
+ return;
+ }
+
+ // idivmod returns the quotient in r0, and the remainder in r1.
+ MOZ_ASSERT(dividend == r0);
+ MOZ_ASSERT(divisor == r1);
+
+ LiveRegisterSet volatileRegs = liveVolatileRegs(ins);
+ volatileRegs.takeUnchecked(dividend);
+ volatileRegs.takeUnchecked(divisor);
+ volatileRegs.takeUnchecked(output);
+
+ masm.PushRegsInMask(volatileRegs);
+
+ using Fn = int64_t (*)(int, int);
+ masm.setupUnalignedABICall(output);
+ masm.passABIArg(dividend);
+ masm.passABIArg(divisor);
+ masm.callWithABI<Fn, __aeabi_idivmod>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.PopRegsInMask(volatileRegs);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, dividend, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, divisor);
+}
+
+void CodeGenerator::visitBitNotI(LBitNotI* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ const LDefinition* dest = ins->getDef(0);
+ // This will not actually be true on arm. We can not an imm8m in order to
+ // get a wider range of numbers
+ MOZ_ASSERT(!input->isConstant());
+
+ masm.ma_mvn(ToRegister(input), ToRegister(dest));
+}
+
+void CodeGenerator::visitBitOpI(LBitOpI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ ScratchRegisterScope scratch(masm);
+
+ // All of these bitops should be either imm32's, or integer registers.
+ switch (ins->bitop()) {
+ case JSOp::BitOr:
+ if (rhs->isConstant()) {
+ masm.ma_orr(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
+ scratch);
+ } else {
+ masm.ma_orr(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
+ }
+ break;
+ case JSOp::BitXor:
+ if (rhs->isConstant()) {
+ masm.ma_eor(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
+ scratch);
+ } else {
+ masm.ma_eor(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
+ }
+ break;
+ case JSOp::BitAnd:
+ if (rhs->isConstant()) {
+ masm.ma_and(Imm32(ToInt32(rhs)), ToRegister(lhs), ToRegister(dest),
+ scratch);
+ } else {
+ masm.ma_and(ToRegister(rhs), ToRegister(lhs), ToRegister(dest));
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitShiftI(LShiftI* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ const LAllocation* rhs = ins->rhs();
+ Register dest = ToRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ if (shift) {
+ masm.ma_lsl(Imm32(shift), lhs, dest);
+ } else {
+ masm.ma_mov(lhs, dest);
+ }
+ break;
+ case JSOp::Rsh:
+ if (shift) {
+ masm.ma_asr(Imm32(shift), lhs, dest);
+ } else {
+ masm.ma_mov(lhs, dest);
+ }
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.ma_lsr(Imm32(shift), lhs, dest);
+ } else {
+ // x >>> 0 can overflow.
+ masm.ma_mov(lhs, dest);
+ if (ins->mir()->toUrsh()->fallible()) {
+ masm.as_cmp(dest, Imm8(0));
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+ }
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ } else {
+ // The shift amounts should be AND'ed into the 0-31 range since arm
+ // shifts by the lower byte of the register (it will attempt to shift by
+ // 250 if you ask it to).
+ masm.as_and(dest, ToRegister(rhs), Imm8(0x1F));
+
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ masm.ma_lsl(dest, lhs, dest);
+ break;
+ case JSOp::Rsh:
+ masm.ma_asr(dest, lhs, dest);
+ break;
+ case JSOp::Ursh:
+ masm.ma_lsr(dest, lhs, dest);
+ if (ins->mir()->toUrsh()->fallible()) {
+ // x >>> 0 can overflow.
+ masm.as_cmp(dest, Imm8(0));
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ }
+}
+
+void CodeGenerator::visitUrshD(LUrshD* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register temp = ToRegister(ins->temp());
+
+ const LAllocation* rhs = ins->rhs();
+ FloatRegister out = ToFloatRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ if (shift) {
+ masm.ma_lsr(Imm32(shift), lhs, temp);
+ } else {
+ masm.ma_mov(lhs, temp);
+ }
+ } else {
+ masm.as_and(temp, ToRegister(rhs), Imm8(0x1F));
+ masm.ma_lsr(temp, lhs, temp);
+ }
+
+ masm.convertUInt32ToDouble(temp, out);
+}
+
+void CodeGenerator::visitClzI(LClzI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.clz32(input, output, /* knownNotZero = */ false);
+}
+
+void CodeGenerator::visitCtzI(LCtzI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.ctz32(input, output, /* knownNotZero = */ false);
+}
+
+void CodeGenerator::visitPopcntI(LPopcntI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ Register tmp = ToRegister(ins->temp0());
+
+ masm.popcnt32(input, output, tmp);
+}
+
+void CodeGenerator::visitPowHalfD(LPowHalfD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+ ScratchDoubleScope scratch(masm);
+
+ Label done;
+
+ // Masm.pow(-Infinity, 0.5) == Infinity.
+ masm.loadConstantDouble(NegativeInfinity<double>(), scratch);
+ masm.compareDouble(input, scratch);
+ masm.ma_vneg(scratch, output, Assembler::Equal);
+ masm.ma_b(&done, Assembler::Equal);
+
+ // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
+ // Adding 0 converts any -0 to 0.
+ masm.loadConstantDouble(0.0, scratch);
+ masm.ma_vadd(scratch, input, output);
+ masm.ma_vsqrt(output, output);
+
+ masm.bind(&done);
+}
+
+MoveOperand CodeGeneratorARM::toMoveOperand(LAllocation a) const {
+ if (a.isGeneralReg()) {
+ return MoveOperand(ToRegister(a));
+ }
+ if (a.isFloatReg()) {
+ return MoveOperand(ToFloatRegister(a));
+ }
+ MoveOperand::Kind kind =
+ a.isStackArea() ? MoveOperand::EFFECTIVE_ADDRESS : MoveOperand::MEMORY;
+ Address addr = ToAddress(a);
+ MOZ_ASSERT((addr.offset & 3) == 0);
+ return MoveOperand(addr, kind);
+}
+
+class js::jit::OutOfLineTableSwitch
+ : public OutOfLineCodeBase<CodeGeneratorARM> {
+ MTableSwitch* mir_;
+ Vector<CodeLabel, 8, JitAllocPolicy> codeLabels_;
+
+ void accept(CodeGeneratorARM* codegen) override {
+ codegen->visitOutOfLineTableSwitch(this);
+ }
+
+ public:
+ OutOfLineTableSwitch(TempAllocator& alloc, MTableSwitch* mir)
+ : mir_(mir), codeLabels_(alloc) {}
+
+ MTableSwitch* mir() const { return mir_; }
+
+ bool addCodeLabel(CodeLabel label) { return codeLabels_.append(label); }
+ CodeLabel codeLabel(unsigned i) { return codeLabels_[i]; }
+};
+
+void CodeGeneratorARM::visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool) {
+ MTableSwitch* mir = ool->mir();
+
+ size_t numCases = mir->numCases();
+ for (size_t i = 0; i < numCases; i++) {
+ LBlock* caseblock =
+ skipTrivialBlocks(mir->getCase(numCases - 1 - i))->lir();
+ Label* caseheader = caseblock->label();
+ uint32_t caseoffset = caseheader->offset();
+
+ // The entries of the jump table need to be absolute addresses and thus
+ // must be patched after codegen is finished.
+ CodeLabel cl = ool->codeLabel(i);
+ cl.target()->bind(caseoffset);
+ masm.addCodeLabel(cl);
+ }
+}
+
+void CodeGeneratorARM::emitTableSwitchDispatch(MTableSwitch* mir,
+ Register index, Register base) {
+ // The code generated by this is utter hax.
+ // The end result looks something like:
+ // SUBS index, input, #base
+ // RSBSPL index, index, #max
+ // LDRPL pc, pc, index lsl 2
+ // B default
+
+ // If the range of targets in N through M, we first subtract off the lowest
+ // case (N), which both shifts the arguments into the range 0 to (M - N)
+ // with and sets the MInus flag if the argument was out of range on the low
+ // end.
+
+ // Then we a reverse subtract with the size of the jump table, which will
+ // reverse the order of range (It is size through 0, rather than 0 through
+ // size). The main purpose of this is that we set the same flag as the lower
+ // bound check for the upper bound check. Lastly, we do this conditionally
+ // on the previous check succeeding.
+
+ // Then we conditionally load the pc offset by the (reversed) index (times
+ // the address size) into the pc, which branches to the correct case. NOTE:
+ // when we go to read the pc, the value that we get back is the pc of the
+ // current instruction *PLUS 8*. This means that ldr foo, [pc, +0] reads
+ // $pc+8. In other words, there is an empty word after the branch into the
+ // switch table before the table actually starts. Since the only other
+ // unhandled case is the default case (both out of range high and out of
+ // range low) I then insert a branch to default case into the extra slot,
+ // which ensures we don't attempt to execute the address table.
+ Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
+
+ ScratchRegisterScope scratch(masm);
+
+ int32_t cases = mir->numCases();
+ // Lower value with low value.
+ masm.ma_sub(index, Imm32(mir->low()), index, scratch, SetCC);
+ masm.ma_rsb(index, Imm32(cases - 1), index, scratch, SetCC,
+ Assembler::NotSigned);
+ // Inhibit pools within the following sequence because we are indexing into
+ // a pc relative table. The region will have one instruction for ma_ldr, one
+ // for ma_b, and each table case takes one word.
+ AutoForbidPoolsAndNops afp(&masm, 1 + 1 + cases);
+ masm.ma_ldr(DTRAddr(pc, DtrRegImmShift(index, LSL, 2)), pc, Offset,
+ Assembler::NotSigned);
+ masm.ma_b(defaultcase);
+
+ // To fill in the CodeLabels for the case entries, we need to first generate
+ // the case entries (we don't yet know their offsets in the instruction
+ // stream).
+ OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(alloc(), mir);
+ for (int32_t i = 0; i < cases; i++) {
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ masm.propagateOOM(ool->addCodeLabel(cl));
+ }
+ addOutOfLineCode(ool, mir);
+}
+
+void CodeGenerator::visitMathD(LMathD* math) {
+ FloatRegister src1 = ToFloatRegister(math->getOperand(0));
+ FloatRegister src2 = ToFloatRegister(math->getOperand(1));
+ FloatRegister output = ToFloatRegister(math->getDef(0));
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.ma_vadd(src1, src2, output);
+ break;
+ case JSOp::Sub:
+ masm.ma_vsub(src1, src2, output);
+ break;
+ case JSOp::Mul:
+ masm.ma_vmul(src1, src2, output);
+ break;
+ case JSOp::Div:
+ masm.ma_vdiv(src1, src2, output);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitMathF(LMathF* math) {
+ FloatRegister src1 = ToFloatRegister(math->getOperand(0));
+ FloatRegister src2 = ToFloatRegister(math->getOperand(1));
+ FloatRegister output = ToFloatRegister(math->getDef(0));
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.ma_vadd_f32(src1, src2, output);
+ break;
+ case JSOp::Sub:
+ masm.ma_vsub_f32(src1, src2, output);
+ break;
+ case JSOp::Mul:
+ masm.ma_vmul_f32(src1, src2, output);
+ break;
+ case JSOp::Div:
+ masm.ma_vdiv_f32(src1, src2, output);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+ emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* ins) {
+ emitTruncateDouble(ToFloatRegister(ins->getOperand(0)),
+ ToRegister(ins->getDef(0)), ins->mir());
+}
+
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+ emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+ LWasmBuiltinTruncateFToInt32* ins) {
+ emitTruncateFloat32(ToFloatRegister(ins->getOperand(0)),
+ ToRegister(ins->getDef(0)), ins->mir());
+}
+
+ValueOperand CodeGeneratorARM::ToValue(LInstruction* ins, size_t pos) {
+ Register typeReg = ToRegister(ins->getOperand(pos + TYPE_INDEX));
+ Register payloadReg = ToRegister(ins->getOperand(pos + PAYLOAD_INDEX));
+ return ValueOperand(typeReg, payloadReg);
+}
+
+ValueOperand CodeGeneratorARM::ToTempValue(LInstruction* ins, size_t pos) {
+ Register typeReg = ToRegister(ins->getTemp(pos + TYPE_INDEX));
+ Register payloadReg = ToRegister(ins->getTemp(pos + PAYLOAD_INDEX));
+ return ValueOperand(typeReg, payloadReg);
+}
+
+void CodeGenerator::visitValue(LValue* value) {
+ const ValueOperand out = ToOutValue(value);
+
+ masm.moveValue(value->value(), out);
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+ const LDefinition* type = box->getDef(TYPE_INDEX);
+
+ MOZ_ASSERT(!box->getOperand(0)->isConstant());
+
+ // On arm, the input operand and the output payload have the same virtual
+ // register. All that needs to be written is the type tag for the type
+ // definition.
+ masm.ma_mov(Imm32(MIRTypeToTag(box->type())), ToRegister(type));
+}
+
+void CodeGenerator::visitBoxFloatingPoint(LBoxFloatingPoint* box) {
+ const AnyRegister in = ToAnyRegister(box->getOperand(0));
+ const ValueOperand out = ToOutValue(box);
+
+ masm.moveValue(TypedOrValueRegister(box->type(), in), out);
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+ // Note that for unbox, the type and payload indexes are switched on the
+ // inputs.
+ MUnbox* mir = unbox->mir();
+ Register type = ToRegister(unbox->type());
+ Register payload = ToRegister(unbox->payload());
+ Register output = ToRegister(unbox->output());
+
+ mozilla::Maybe<ScratchRegisterScope> scratch;
+ scratch.emplace(masm);
+
+ JSValueTag tag = MIRTypeToTag(mir->type());
+ if (mir->fallible()) {
+ masm.ma_cmp(type, Imm32(tag), *scratch);
+ bailoutIf(Assembler::NotEqual, unbox->snapshot());
+ } else {
+#ifdef DEBUG
+ Label ok;
+ masm.ma_cmp(type, Imm32(tag), *scratch);
+ masm.ma_b(&ok, Assembler::Equal);
+ scratch.reset();
+ masm.assumeUnreachable("Infallible unbox type mismatch");
+ masm.bind(&ok);
+#endif
+ }
+
+ // Note: If spectreValueMasking is disabled, then this instruction will
+ // default to a no-op as long as the lowering allocate the same register for
+ // the output and the payload.
+ masm.unboxNonDouble(ValueOperand(type, payload), output,
+ ValueTypeFromMIRType(mir->type()));
+}
+
+void CodeGenerator::visitDouble(LDouble* ins) {
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantDouble(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitFloat32(LFloat32* ins) {
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantFloat32(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGeneratorARM::splitTagForTest(const ValueOperand& value,
+ ScratchTagScope& tag) {
+ MOZ_ASSERT(value.typeReg() == tag);
+}
+
+void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) {
+ const LAllocation* opd = test->input();
+ masm.ma_vcmpz(ToFloatRegister(opd));
+ masm.as_vmrs(pc);
+
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+ // If the compare set the 0 bit, then the result is definitely false.
+ jumpToBlock(ifFalse, Assembler::Zero);
+ // It is also false if one of the operands is NAN, which is shown as
+ // Overflow.
+ jumpToBlock(ifFalse, Assembler::Overflow);
+ jumpToBlock(ifTrue);
+}
+
+void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) {
+ const LAllocation* opd = test->input();
+ masm.ma_vcmpz_f32(ToFloatRegister(opd));
+ masm.as_vmrs(pc);
+
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+ // If the compare set the 0 bit, then the result is definitely false.
+ jumpToBlock(ifFalse, Assembler::Zero);
+ // It is also false if one of the operands is NAN, which is shown as
+ // Overflow.
+ jumpToBlock(ifFalse, Assembler::Overflow);
+ jumpToBlock(ifTrue);
+}
+
+void CodeGenerator::visitCompareD(LCompareD* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+ masm.compareDouble(lhs, rhs);
+ masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
+ ToRegister(comp->output()));
+}
+
+void CodeGenerator::visitCompareF(LCompareF* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+ masm.compareFloat(lhs, rhs);
+ masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
+ ToRegister(comp->output()));
+}
+
+void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ masm.compareDouble(lhs, rhs);
+ emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
+ comp->ifFalse());
+}
+
+void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ masm.compareFloat(lhs, rhs);
+ emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
+ comp->ifFalse());
+}
+
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
+ // LBitAndAndBranch only represents single-word ANDs, hence it can't be
+ // 64-bit here.
+ MOZ_ASSERT(!baab->is64());
+ Register regL = ToRegister(baab->left());
+ if (baab->right()->isConstant()) {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_tst(regL, Imm32(ToInt32(baab->right())), scratch);
+ } else {
+ masm.ma_tst(regL, ToRegister(baab->right()));
+ }
+ emitBranch(baab->cond(), baab->ifTrue(), baab->ifFalse());
+}
+
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+ masm.convertUInt32ToDouble(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+ masm.convertUInt32ToFloat32(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitNotI(LNotI* ins) {
+ // It is hard to optimize !x, so just do it the basic way for now.
+ masm.as_cmp(ToRegister(ins->input()), Imm8(0));
+ masm.emitSet(Assembler::Equal, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register output = ToRegister(lir->output());
+
+ masm.ma_orr(input.low, input.high, output);
+ masm.as_cmp(output, Imm8(0));
+ masm.emitSet(Assembler::Equal, output);
+}
+
+void CodeGenerator::visitNotD(LNotD* ins) {
+ // Since this operation is not, we want to set a bit if the double is
+ // falsey, which means 0.0, -0.0 or NaN. When comparing with 0, an input of
+ // 0 will set the Z bit (30) and NaN will set the V bit (28) of the APSR.
+ FloatRegister opd = ToFloatRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+
+ // Do the compare.
+ masm.ma_vcmpz(opd);
+ // TODO There are three variations here to compare performance-wise.
+ bool nocond = true;
+ if (nocond) {
+ // Load the value into the dest register.
+ masm.as_vmrs(dest);
+ masm.ma_lsr(Imm32(28), dest, dest);
+ // 28 + 2 = 30
+ masm.ma_alu(dest, lsr(dest, 2), dest, OpOrr);
+ masm.as_and(dest, dest, Imm8(1));
+ } else {
+ masm.as_vmrs(pc);
+ masm.ma_mov(Imm32(0), dest);
+ masm.ma_mov(Imm32(1), dest, Assembler::Equal);
+ masm.ma_mov(Imm32(1), dest, Assembler::Overflow);
+ }
+}
+
+void CodeGenerator::visitNotF(LNotF* ins) {
+ // Since this operation is not, we want to set a bit if the double is
+ // falsey, which means 0.0, -0.0 or NaN. When comparing with 0, an input of
+ // 0 will set the Z bit (30) and NaN will set the V bit (28) of the APSR.
+ FloatRegister opd = ToFloatRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+
+ // Do the compare.
+ masm.ma_vcmpz_f32(opd);
+ // TODO There are three variations here to compare performance-wise.
+ bool nocond = true;
+ if (nocond) {
+ // Load the value into the dest register.
+ masm.as_vmrs(dest);
+ masm.ma_lsr(Imm32(28), dest, dest);
+ // 28 + 2 = 30
+ masm.ma_alu(dest, lsr(dest, 2), dest, OpOrr);
+ masm.as_and(dest, dest, Imm8(1));
+ } else {
+ masm.as_vmrs(pc);
+ masm.ma_mov(Imm32(0), dest);
+ masm.ma_mov(Imm32(1), dest, Assembler::Equal);
+ masm.ma_mov(Imm32(1), dest, Assembler::Overflow);
+ }
+}
+
+void CodeGeneratorARM::generateInvalidateEpilogue() {
+ // Ensure that there is enough space in the buffer for the OsiPoint patching
+ // to occur. Otherwise, we could overwrite the invalidation epilogue.
+ for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) {
+ masm.nop();
+ }
+
+ masm.bind(&invalidate_);
+
+ // Push the return address of the point that we bailed out at onto the stack.
+ masm.Push(lr);
+
+ // Push the Ion script onto the stack (when we determine what that pointer
+ // is).
+ invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
+
+ // Jump to the invalidator which will replace the current frame.
+ TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
+ masm.jump(thunk);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement(
+ LCompareExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register temp =
+ lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
+
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, temp, output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, temp, output);
+ }
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement(
+ LAtomicExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register temp =
+ lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
+
+ Register value = ToRegister(lir->value());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
+ output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
+ output);
+ }
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop(
+ LAtomicTypedArrayElementBinop* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register elements = ToRegister(lir->elements());
+ Register flagTemp = ToRegister(lir->temp1());
+ Register outTemp =
+ lir->temp2()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp2());
+ Register value = ToRegister(lir->value());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, flagTemp, outTemp,
+ output);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, flagTemp, outTemp,
+ output);
+ }
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(
+ LAtomicTypedArrayElementBinopForEffect* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register flagTemp = ToRegister(lir->flagTemp());
+ Register value = ToRegister(lir->value());
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, flagTemp);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, flagTemp);
+ }
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.atomicLoad64(Synchronization::Load(), source, temp64);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.atomicLoad64(Synchronization::Load(), source, temp64);
+ }
+
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.atomicStore64(Synchronization::Store(), dest, temp1, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.atomicStore64(Synchronization::Store(), dest, temp1, temp2);
+ }
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register64 temp3 = ToRegister64(lir->temp3());
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(oldval, temp1);
+ masm.loadBigInt64(newval, temp2);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, temp2, temp3);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, temp2, temp3);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp3, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 temp64 = Register64(temp2, out);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(value, temp64);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchange64(Synchronization::Full(), dest, temp64, temp1);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchange64(Synchronization::Full(), dest, temp64, temp1);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp1, out, temp2);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register64 temp3 = ToRegister64(lir->temp3());
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest, temp2,
+ temp3);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest, temp2,
+ temp3);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp3, out, temp2.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ }
+}
+
+void CodeGenerator::visitWasmSelect(LWasmSelect* ins) {
+ MIRType mirType = ins->mir()->type();
+
+ Register cond = ToRegister(ins->condExpr());
+ masm.as_cmp(cond, Imm8(0));
+
+ if (mirType == MIRType::Int32 || mirType == MIRType::RefOrNull) {
+ Register falseExpr = ToRegister(ins->falseExpr());
+ Register out = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->trueExpr()) == out,
+ "true expr input is reused for output");
+ masm.ma_mov(falseExpr, out, LeaveCC, Assembler::Zero);
+ return;
+ }
+
+ FloatRegister out = ToFloatRegister(ins->output());
+ MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out,
+ "true expr input is reused for output");
+
+ FloatRegister falseExpr = ToFloatRegister(ins->falseExpr());
+
+ if (mirType == MIRType::Double) {
+ masm.moveDouble(falseExpr, out, Assembler::Zero);
+ } else if (mirType == MIRType::Float32) {
+ masm.moveFloat32(falseExpr, out, Assembler::Zero);
+ } else {
+ MOZ_CRASH("unhandled type in visitWasmSelect!");
+ }
+}
+
+// We expect to handle only the case where compare is {U,}Int32 and select is
+// {U,}Int32, and the "true" input is reused for the output.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
+ ins->compareType() == MCompare::Compare_UInt32;
+ bool selIs32bit = ins->mir()->type() == MIRType::Int32;
+
+ MOZ_RELEASE_ASSERT(
+ cmpIs32bit && selIs32bit,
+ "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
+
+ Register trueExprAndDest = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
+ "true expr input is reused for output");
+
+ Assembler::Condition cond = Assembler::InvertCondition(
+ JSOpToCondition(ins->compareType(), ins->jsop()));
+ const LAllocation* rhs = ins->rightExpr();
+ const LAllocation* falseExpr = ins->ifFalseExpr();
+ Register lhs = ToRegister(ins->leftExpr());
+
+ masm.cmp32Move32(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+}
+
+void CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MWasmReinterpret* ins = lir->mir();
+
+ MIRType to = ins->type();
+ DebugOnly<MIRType> from = ins->input()->type();
+
+ switch (to) {
+ case MIRType::Int32:
+ MOZ_ASSERT(static_cast<MIRType>(from) == MIRType::Float32);
+ masm.ma_vxfer(ToFloatRegister(lir->input()), ToRegister(lir->output()));
+ break;
+ case MIRType::Float32:
+ MOZ_ASSERT(static_cast<MIRType>(from) == MIRType::Int32);
+ masm.ma_vxfer(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+ break;
+ case MIRType::Double:
+ case MIRType::Int64:
+ MOZ_CRASH("not handled by this LIR opcode");
+ default:
+ MOZ_CRASH("unexpected WasmReinterpret");
+ }
+}
+
+void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) {
+ const MAsmJSLoadHeap* mir = ins->mir();
+
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ bool isSigned;
+ int size;
+ bool isFloat = false;
+ switch (mir->accessType()) {
+ case Scalar::Int8:
+ isSigned = true;
+ size = 8;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ size = 8;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ size = 16;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ size = 16;
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ isSigned = true;
+ size = 32;
+ break;
+ case Scalar::Float64:
+ isFloat = true;
+ size = 64;
+ break;
+ case Scalar::Float32:
+ isFloat = true;
+ size = 32;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ if (ptr->isConstant()) {
+ MOZ_ASSERT(!mir->needsBoundsCheck());
+ int32_t ptrImm = ptr->toConstant()->toInt32();
+ MOZ_ASSERT(ptrImm >= 0);
+ if (isFloat) {
+ ScratchRegisterScope scratch(masm);
+ VFPRegister vd(ToFloatRegister(ins->output()));
+ if (size == 32) {
+ masm.ma_vldr(Address(HeapReg, ptrImm), vd.singleOverlay(), scratch,
+ Assembler::Always);
+ } else {
+ masm.ma_vldr(Address(HeapReg, ptrImm), vd, scratch, Assembler::Always);
+ }
+ } else {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_dataTransferN(IsLoad, size, isSigned, HeapReg, Imm32(ptrImm),
+ ToRegister(ins->output()), scratch, Offset,
+ Assembler::Always);
+ }
+ } else {
+ Register ptrReg = ToRegister(ptr);
+ if (isFloat) {
+ FloatRegister output = ToFloatRegister(ins->output());
+ if (size == 32) {
+ output = output.singleOverlay();
+ }
+
+ Assembler::Condition cond = Assembler::Always;
+ if (mir->needsBoundsCheck()) {
+ Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
+ masm.as_cmp(ptrReg, O2Reg(boundsCheckLimitReg));
+ if (size == 32) {
+ masm.ma_vimm_f32(GenericNaN(), output, Assembler::AboveOrEqual);
+ } else {
+ masm.ma_vimm(GenericNaN(), output, Assembler::AboveOrEqual);
+ }
+ cond = Assembler::Below;
+ }
+
+ ScratchRegisterScope scratch(masm);
+ masm.ma_vldr(output, HeapReg, ptrReg, scratch, 0, cond);
+ } else {
+ Register output = ToRegister(ins->output());
+
+ Assembler::Condition cond = Assembler::Always;
+ if (mir->needsBoundsCheck()) {
+ Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
+ masm.as_cmp(ptrReg, O2Reg(boundsCheckLimitReg));
+ masm.ma_mov(Imm32(0), output, Assembler::AboveOrEqual);
+ cond = Assembler::Below;
+ }
+
+ ScratchRegisterScope scratch(masm);
+ masm.ma_dataTransferN(IsLoad, size, isSigned, HeapReg, ptrReg, output,
+ scratch, Offset, cond);
+ }
+ }
+}
+
+void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) {
+ MOZ_ASSERT(ins->instance()->isBogus());
+ masm.movePtr(HeapReg, ToRegister(ins->output()));
+}
+
+template <typename T>
+void CodeGeneratorARM::emitWasmLoad(T* lir) {
+ const MWasmLoad* mir = lir->mir();
+ MIRType resultType = mir->type();
+ Register ptr;
+
+ if (mir->access().offset() || mir->access().type() == Scalar::Int64) {
+ ptr = ToRegister(lir->ptrCopy());
+ } else {
+ MOZ_ASSERT(lir->ptrCopy()->isBogusTemp());
+ ptr = ToRegister(lir->ptr());
+ }
+
+ if (resultType == MIRType::Int64) {
+ masm.wasmLoadI64(mir->access(), HeapReg, ptr, ptr, ToOutRegister64(lir));
+ } else {
+ masm.wasmLoad(mir->access(), HeapReg, ptr, ptr,
+ ToAnyRegister(lir->output()));
+ }
+}
+
+void CodeGenerator::visitWasmLoad(LWasmLoad* lir) { emitWasmLoad(lir); }
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir) { emitWasmLoad(lir); }
+
+void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register base = ToRegister(lir->base());
+ Register out = ToRegister(lir->output());
+
+ ScratchRegisterScope scratch(masm);
+ masm.ma_add(base, Imm32(mir->offset()), out, scratch, SetCC);
+ OutOfLineAbortingWasmTrap* ool = new (alloc())
+ OutOfLineAbortingWasmTrap(mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
+ addOutOfLineCode(ool, mir);
+ masm.ma_b(ool->entry(), Assembler::CarrySet);
+}
+
+void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register64 base = ToRegister64(lir->base());
+ Register64 out = ToOutRegister64(lir);
+ MOZ_ASSERT(base.low != out.high && base.high != out.low);
+
+ ScratchRegisterScope scratch(masm);
+ masm.ma_add(base.low, Imm32(mir->offset()), out.low, scratch, SetCC);
+ masm.ma_adc(base.high, Imm32(mir->offset() >> 32), out.high, scratch, SetCC);
+ OutOfLineAbortingWasmTrap* ool = new (alloc())
+ OutOfLineAbortingWasmTrap(mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
+ addOutOfLineCode(ool, mir);
+ masm.ma_b(ool->entry(), Assembler::CarrySet);
+}
+
+template <typename T>
+void CodeGeneratorARM::emitWasmStore(T* lir) {
+ const MWasmStore* mir = lir->mir();
+ Scalar::Type accessType = mir->access().type();
+ Register ptr;
+
+ // Maybe add the offset.
+ if (mir->access().offset() || accessType == Scalar::Int64) {
+ ptr = ToRegister(lir->ptrCopy());
+ } else {
+ MOZ_ASSERT(lir->ptrCopy()->isBogusTemp());
+ ptr = ToRegister(lir->ptr());
+ }
+
+ if (accessType == Scalar::Int64) {
+ masm.wasmStoreI64(mir->access(),
+ ToRegister64(lir->getInt64Operand(lir->ValueIndex)),
+ HeapReg, ptr, ptr);
+ } else {
+ masm.wasmStore(mir->access(),
+ ToAnyRegister(lir->getOperand(lir->ValueIndex)), HeapReg,
+ ptr, ptr);
+ }
+}
+
+void CodeGenerator::visitWasmStore(LWasmStore* lir) { emitWasmStore(lir); }
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir) {
+ emitWasmStore(lir);
+}
+
+void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) {
+ const MAsmJSStoreHeap* mir = ins->mir();
+
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ bool isSigned;
+ int size;
+ bool isFloat = false;
+ switch (mir->accessType()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ isSigned = false;
+ size = 8;
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ isSigned = false;
+ size = 16;
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ isSigned = true;
+ size = 32;
+ break;
+ case Scalar::Float64:
+ isFloat = true;
+ size = 64;
+ break;
+ case Scalar::Float32:
+ isFloat = true;
+ size = 32;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ if (ptr->isConstant()) {
+ MOZ_ASSERT(!mir->needsBoundsCheck());
+ int32_t ptrImm = ptr->toConstant()->toInt32();
+ MOZ_ASSERT(ptrImm >= 0);
+ if (isFloat) {
+ VFPRegister vd(ToFloatRegister(ins->value()));
+ Address addr(HeapReg, ptrImm);
+ if (size == 32) {
+ masm.storeFloat32(vd, addr);
+ } else {
+ masm.storeDouble(vd, addr);
+ }
+ } else {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_dataTransferN(IsStore, size, isSigned, HeapReg, Imm32(ptrImm),
+ ToRegister(ins->value()), scratch, Offset,
+ Assembler::Always);
+ }
+ } else {
+ Register ptrReg = ToRegister(ptr);
+
+ Assembler::Condition cond = Assembler::Always;
+ if (mir->needsBoundsCheck()) {
+ Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
+ masm.as_cmp(ptrReg, O2Reg(boundsCheckLimitReg));
+ cond = Assembler::Below;
+ }
+
+ if (isFloat) {
+ ScratchRegisterScope scratch(masm);
+ FloatRegister value = ToFloatRegister(ins->value());
+ if (size == 32) {
+ value = value.singleOverlay();
+ }
+
+ masm.ma_vstr(value, HeapReg, ptrReg, scratch, 0, Assembler::Below);
+ } else {
+ ScratchRegisterScope scratch(masm);
+ Register value = ToRegister(ins->value());
+ masm.ma_dataTransferN(IsStore, size, isSigned, HeapReg, ptrReg, value,
+ scratch, Offset, cond);
+ }
+ }
+}
+
+void CodeGenerator::visitWasmCompareExchangeHeap(
+ LWasmCompareExchangeHeap* ins) {
+ MWasmCompareExchangeHeap* mir = ins->mir();
+
+ const LAllocation* ptr = ins->ptr();
+ Register ptrReg = ToRegister(ptr);
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Register oldval = ToRegister(ins->oldValue());
+ Register newval = ToRegister(ins->newValue());
+ Register out = ToRegister(ins->output());
+
+ masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval, out);
+}
+
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+ MWasmAtomicExchangeHeap* mir = ins->mir();
+
+ Register ptrReg = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ Register output = ToRegister(ins->output());
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ masm.wasmAtomicExchange(mir->access(), srcAddr, value, output);
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ MOZ_ASSERT(mir->hasUses());
+
+ Register ptrReg = ToRegister(ins->ptr());
+ Register flagTemp = ToRegister(ins->flagTemp());
+ Register output = ToRegister(ins->output());
+ const LAllocation* value = ins->value();
+ AtomicOp op = mir->operation();
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ masm.wasmAtomicFetchOp(mir->access(), op, ToRegister(value), srcAddr,
+ flagTemp, output);
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+ LWasmAtomicBinopHeapForEffect* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ MOZ_ASSERT(!mir->hasUses());
+
+ Register ptrReg = ToRegister(ins->ptr());
+ Register flagTemp = ToRegister(ins->flagTemp());
+ const LAllocation* value = ins->value();
+ AtomicOp op = mir->operation();
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ masm.wasmAtomicEffectOp(mir->access(), op, ToRegister(value), srcAddr,
+ flagTemp);
+}
+
+void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(StackPointer, mir->spOffset());
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+
+ if (ins->arg()->isConstant()) {
+ masm.ma_mov(Imm32(ToInt32(ins->arg())), scratch);
+ masm.ma_str(scratch, dst, scratch2);
+ } else {
+ if (ins->arg()->isGeneralReg()) {
+ masm.ma_str(ToRegister(ins->arg()), dst, scratch);
+ } else {
+ masm.ma_vstr(ToFloatRegister(ins->arg()), dst, scratch);
+ }
+ }
+}
+
+void CodeGenerator::visitUDiv(LUDiv* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+
+ Label done;
+ generateUDivModZeroCheck(rhs, output, &done, ins->snapshot(), ins->mir());
+
+ masm.ma_udiv(lhs, rhs, output);
+
+ // Check for large unsigned result - represent as double.
+ if (!ins->mir()->isTruncated()) {
+ MOZ_ASSERT(ins->mir()->fallible());
+ masm.as_cmp(output, Imm8(0));
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+ }
+
+ // Check for non-zero remainder if not truncating to int.
+ if (!ins->mir()->canTruncateRemainder()) {
+ MOZ_ASSERT(ins->mir()->fallible());
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_mul(rhs, output, scratch);
+ masm.ma_cmp(scratch, lhs);
+ }
+ bailoutIf(Assembler::NotEqual, ins->snapshot());
+ }
+
+ if (done.used()) {
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitUMod(LUMod* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+
+ Label done;
+ generateUDivModZeroCheck(rhs, output, &done, ins->snapshot(), ins->mir());
+
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_umod(lhs, rhs, output, scratch);
+ }
+
+ // Check for large unsigned result - represent as double.
+ if (!ins->mir()->isTruncated()) {
+ MOZ_ASSERT(ins->mir()->fallible());
+ masm.as_cmp(output, Imm8(0));
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+ }
+
+ if (done.used()) {
+ masm.bind(&done);
+ }
+}
+
+template <class T>
+void CodeGeneratorARM::generateUDivModZeroCheck(Register rhs, Register output,
+ Label* done,
+ LSnapshot* snapshot, T* mir) {
+ if (!mir) {
+ return;
+ }
+ if (mir->canBeDivideByZero()) {
+ masm.as_cmp(rhs, Imm8(0));
+ if (mir->isTruncated()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.ma_b(&nonZero, Assembler::NotEqual);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ Label skip;
+ masm.ma_b(&skip, Assembler::NotEqual);
+ // Infinity|0 == 0
+ masm.ma_mov(Imm32(0), output);
+ masm.ma_b(done);
+ masm.bind(&skip);
+ }
+ } else {
+ // Bailout for divide by zero
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Equal, snapshot);
+ }
+ }
+}
+
+void CodeGenerator::visitSoftUDivOrMod(LSoftUDivOrMod* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+
+ MOZ_ASSERT(lhs == r0);
+ MOZ_ASSERT(rhs == r1);
+ MOZ_ASSERT(output == r0);
+
+ Label done;
+ MDiv* div = ins->mir()->isDiv() ? ins->mir()->toDiv() : nullptr;
+ MMod* mod = !div ? ins->mir()->toMod() : nullptr;
+
+ generateUDivModZeroCheck(rhs, output, &done, ins->snapshot(), div);
+ generateUDivModZeroCheck(rhs, output, &done, ins->snapshot(), mod);
+
+ if (gen->compilingWasm()) {
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs);
+ masm.passABIArg(rhs);
+ wasm::BytecodeOffset bytecodeOffset =
+ (div ? div->bytecodeOffset() : mod->bytecodeOffset());
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ masm.callWithABI(bytecodeOffset, wasm::SymbolicAddress::aeabi_uidivmod,
+ mozilla::Some(instanceOffset));
+ masm.Pop(InstanceReg);
+ } else {
+ using Fn = int64_t (*)(int, int);
+ masm.setupAlignedABICall();
+ masm.passABIArg(lhs);
+ masm.passABIArg(rhs);
+ masm.callWithABI<Fn, __aeabi_uidivmod>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+ }
+
+ if (mod) {
+ MOZ_ASSERT(output == r0, "output should not be r1 for mod");
+ masm.move32(r1, output);
+ }
+
+ // uidivmod returns the quotient in r0, and the remainder in r1.
+ if (div && !div->canTruncateRemainder()) {
+ MOZ_ASSERT(div->fallible());
+ masm.as_cmp(r1, Imm8(0));
+ bailoutIf(Assembler::NonZero, ins->snapshot());
+ }
+
+ // Bailout for big unsigned results
+ if ((div && !div->isTruncated()) || (mod && !mod->isTruncated())) {
+ DebugOnly<bool> isFallible =
+ (div && div->fallible()) || (mod && mod->fallible());
+ MOZ_ASSERT(isFallible);
+ masm.as_cmp(output, Imm8(0));
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins) {
+ const MEffectiveAddress* mir = ins->mir();
+ Register base = ToRegister(ins->base());
+ Register index = ToRegister(ins->index());
+ Register output = ToRegister(ins->output());
+
+ ScratchRegisterScope scratch(masm);
+
+ masm.as_add(output, base, lsl(index, mir->scale()));
+ masm.ma_add(Imm32(mir->displacement()), output, scratch);
+}
+
+void CodeGenerator::visitNegI(LNegI* ins) {
+ Register input = ToRegister(ins->input());
+ masm.ma_neg(input, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitNegI64(LNegI64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ MOZ_ASSERT(input == ToOutRegister64(ins));
+ masm.neg64(input);
+}
+
+void CodeGenerator::visitNegD(LNegD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ masm.ma_vneg(input, ToFloatRegister(ins->output()));
+}
+
+void CodeGenerator::visitNegF(LNegF* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ masm.ma_vneg_f32(input, ToFloatRegister(ins->output()));
+}
+
+void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) {
+ masm.memoryBarrier(ins->type());
+}
+
+void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) {
+ auto input = ToFloatRegister(lir->input());
+ auto output = ToRegister(lir->output());
+
+ MWasmTruncateToInt32* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ OutOfLineWasmTruncateCheck* ool = nullptr;
+ Label* oolEntry = nullptr;
+ if (!lir->mir()->isSaturating()) {
+ ool = new (alloc())
+ OutOfLineWasmTruncateCheck(mir, input, Register::Invalid());
+ addOutOfLineCode(ool, mir);
+ oolEntry = ool->entry();
+ }
+
+ masm.wasmTruncateToInt32(input, output, fromType, mir->isUnsigned(),
+ mir->isSaturating(), oolEntry);
+
+ if (!lir->mir()->isSaturating()) {
+ masm.bind(ool->rejoin());
+ }
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MOZ_ASSERT(ToRegister(lir->instance()) == InstanceReg);
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister inputDouble = input;
+ Register64 output = ToOutRegister64(lir);
+
+ MWasmBuiltinTruncateToInt64* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ OutOfLineWasmTruncateCheck* ool = nullptr;
+ if (!lir->mir()->isSaturating()) {
+ ool = new (alloc())
+ OutOfLineWasmTruncateCheck(mir, input, Register64::Invalid());
+ addOutOfLineCode(ool, mir);
+ }
+
+ ScratchDoubleScope fpscratch(masm);
+ if (fromType == MIRType::Float32) {
+ inputDouble = fpscratch;
+ masm.convertFloat32ToDouble(input, inputDouble);
+ }
+
+ masm.Push(input);
+
+ masm.setupWasmABICall();
+ masm.passABIArg(inputDouble, MoveOp::DOUBLE);
+
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ if (lir->mir()->isSaturating()) {
+ if (lir->mir()->isUnsigned()) {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::SaturatingTruncateDoubleToUint64,
+ mozilla::Some(instanceOffset));
+ } else {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::SaturatingTruncateDoubleToInt64,
+ mozilla::Some(instanceOffset));
+ }
+ } else {
+ if (lir->mir()->isUnsigned()) {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::TruncateDoubleToUint64,
+ mozilla::Some(instanceOffset));
+ } else {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::TruncateDoubleToInt64,
+ mozilla::Some(instanceOffset));
+ }
+ }
+
+ masm.Pop(input);
+ masm.Pop(InstanceReg);
+
+ // TruncateDoubleTo{UI,I}nt64 returns 0x8000000000000000 to indicate
+ // exceptional results, so check for that and produce the appropriate
+ // traps. The Saturating form always returns a normal value and never
+ // needs traps.
+ if (!lir->mir()->isSaturating()) {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_cmp(output.high, Imm32(0x80000000), scratch);
+ masm.as_cmp(output.low, Imm8(0x00000000), Assembler::Equal);
+ masm.ma_b(ool->entry(), Assembler::Equal);
+
+ masm.bind(ool->rejoin());
+ }
+
+ MOZ_ASSERT(ReturnReg64 == output);
+}
+
+void CodeGeneratorARM::visitOutOfLineWasmTruncateCheck(
+ OutOfLineWasmTruncateCheck* ool) {
+ // On ARM, saturating truncation codegen handles saturating itself rather than
+ // relying on out-of-line fixup code.
+ if (ool->isSaturating()) {
+ return;
+ }
+
+ masm.outOfLineWasmTruncateToIntCheck(ool->input(), ool->fromType(),
+ ool->toType(), ool->isUnsigned(),
+ ool->rejoin(), ool->bytecodeOffset());
+}
+
+void CodeGenerator::visitInt64ToFloatingPointCall(
+ LInt64ToFloatingPointCall* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MOZ_ASSERT(ToRegister(lir->getOperand(LInt64ToFloatingPointCall::Instance)) ==
+ InstanceReg);
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+
+ MBuiltinInt64ToFloatingPoint* mir = lir->mir();
+ MIRType toType = mir->type();
+
+ masm.setupWasmABICall();
+ masm.passABIArg(input.high);
+ masm.passABIArg(input.low);
+
+ bool isUnsigned = mir->isUnsigned();
+ wasm::SymbolicAddress callee =
+ toType == MIRType::Float32
+ ? (isUnsigned ? wasm::SymbolicAddress::Uint64ToFloat32
+ : wasm::SymbolicAddress::Int64ToFloat32)
+ : (isUnsigned ? wasm::SymbolicAddress::Uint64ToDouble
+ : wasm::SymbolicAddress::Int64ToDouble);
+
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ MoveOp::Type result =
+ toType == MIRType::Float32 ? MoveOp::FLOAT32 : MoveOp::DOUBLE;
+ masm.callWithABI(mir->bytecodeOffset(), callee, mozilla::Some(instanceOffset),
+ result);
+
+ DebugOnly<FloatRegister> output(ToFloatRegister(lir->output()));
+ MOZ_ASSERT_IF(toType == MIRType::Double, output.value == ReturnDoubleReg);
+ MOZ_ASSERT_IF(toType == MIRType::Float32, output.value == ReturnFloat32Reg);
+
+ masm.Pop(InstanceReg);
+}
+
+void CodeGenerator::visitCopySignF(LCopySignF* ins) {
+ FloatRegister lhs = ToFloatRegister(ins->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(ins->getOperand(1));
+ FloatRegister output = ToFloatRegister(ins->getDef(0));
+
+ Register lhsi = ToRegister(ins->getTemp(0));
+ Register rhsi = ToRegister(ins->getTemp(1));
+
+ masm.ma_vxfer(lhs, lhsi);
+ masm.ma_vxfer(rhs, rhsi);
+
+ ScratchRegisterScope scratch(masm);
+
+ // Clear lhs's sign.
+ masm.ma_and(Imm32(INT32_MAX), lhsi, lhsi, scratch);
+
+ // Keep rhs's sign.
+ masm.ma_and(Imm32(INT32_MIN), rhsi, rhsi, scratch);
+
+ // Combine.
+ masm.ma_orr(lhsi, rhsi, rhsi);
+
+ masm.ma_vxfer(rhsi, output);
+}
+
+void CodeGenerator::visitCopySignD(LCopySignD* ins) {
+ FloatRegister lhs = ToFloatRegister(ins->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(ins->getOperand(1));
+ FloatRegister output = ToFloatRegister(ins->getDef(0));
+
+ Register lhsi = ToRegister(ins->getTemp(0));
+ Register rhsi = ToRegister(ins->getTemp(1));
+
+ // Manipulate high words of double inputs.
+ masm.as_vxfer(lhsi, InvalidReg, lhs, Assembler::FloatToCore,
+ Assembler::Always, 1);
+ masm.as_vxfer(rhsi, InvalidReg, rhs, Assembler::FloatToCore,
+ Assembler::Always, 1);
+
+ ScratchRegisterScope scratch(masm);
+
+ // Clear lhs's sign.
+ masm.ma_and(Imm32(INT32_MAX), lhsi, lhsi, scratch);
+
+ // Keep rhs's sign.
+ masm.ma_and(Imm32(INT32_MIN), rhsi, rhsi, scratch);
+
+ // Combine.
+ masm.ma_orr(lhsi, rhsi, rhsi);
+
+ // Reconstruct the output.
+ masm.as_vxfer(lhsi, InvalidReg, lhs, Assembler::FloatToCore,
+ Assembler::Always, 0);
+ masm.ma_vxfer(lhsi, rhsi, output);
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ const LInt64Allocation& input = lir->getInt64Operand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ masm.move32(ToRegister(input.low()), output);
+ } else {
+ masm.move32(ToRegister(input.high()), output);
+ }
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ Register64 output = ToOutRegister64(lir);
+ MOZ_ASSERT(ToRegister(lir->input()) == output.low);
+
+ if (lir->mir()->isUnsigned()) {
+ masm.ma_mov(Imm32(0), output.high);
+ } else {
+ masm.ma_asr(Imm32(31), output.low, output.high);
+ }
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ switch (lir->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.move8SignExtend(input.low, output.low);
+ break;
+ case MSignExtendInt64::Half:
+ masm.move16SignExtend(input.low, output.low);
+ break;
+ case MSignExtendInt64::Word:
+ masm.move32(input.low, output.low);
+ break;
+ }
+ masm.ma_asr(Imm32(31), output.low, output.high);
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index*) {
+ MOZ_CRASH("64-bit only");
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ // Generates no code on this platform because we just return the low part of
+ // the input register pair.
+ MOZ_ASSERT(ToRegister(lir->input()) == ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MOZ_ASSERT(ToRegister(lir->getOperand(LDivOrModI64::Instance)) ==
+ InstanceReg);
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ Register64 lhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Lhs));
+ Register64 rhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Rhs));
+ Register64 output = ToOutRegister64(lir);
+
+ MOZ_ASSERT(output == ReturnReg64);
+
+ Label done;
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ // We can use InstanceReg as temp register because we preserved it
+ // before.
+ masm.branchTest64(Assembler::NonZero, rhs, rhs, InstanceReg, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ auto* mir = lir->mir();
+
+ // Handle an integer overflow exception from INT64_MIN / -1.
+ if (lir->canBeNegativeOverflow()) {
+ Label notmin;
+ masm.branch64(Assembler::NotEqual, lhs, Imm64(INT64_MIN), &notmin);
+ masm.branch64(Assembler::NotEqual, rhs, Imm64(-1), &notmin);
+ if (mir->isWasmBuiltinModI64()) {
+ masm.xor64(output, output);
+ } else {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ masm.jump(&done);
+ masm.bind(&notmin);
+ }
+
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs.high);
+ masm.passABIArg(lhs.low);
+ masm.passABIArg(rhs.high);
+ masm.passABIArg(rhs.low);
+
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ if (mir->isWasmBuiltinModI64()) {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::ModI64,
+ mozilla::Some(instanceOffset));
+ } else {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::DivI64,
+ mozilla::Some(instanceOffset));
+ }
+
+ MOZ_ASSERT(ReturnReg64 == output);
+
+ masm.bind(&done);
+ masm.Pop(InstanceReg);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MOZ_ASSERT(ToRegister(lir->getOperand(LDivOrModI64::Instance)) ==
+ InstanceReg);
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ Register64 lhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Lhs));
+ Register64 rhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Rhs));
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ReturnReg64);
+
+ // Prevent divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ // We can use InstanceReg as temp register because we preserved it
+ // before.
+ masm.branchTest64(Assembler::NonZero, rhs, rhs, InstanceReg, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs.high);
+ masm.passABIArg(lhs.low);
+ masm.passABIArg(rhs.high);
+ masm.passABIArg(rhs.low);
+
+ MDefinition* mir = lir->mir();
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ if (mir->isWasmBuiltinModI64()) {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::UModI64,
+ mozilla::Some(instanceOffset));
+ } else {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::UDivI64,
+ mozilla::Some(instanceOffset));
+ }
+ masm.Pop(InstanceReg);
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+ MCompare* mir = lir->mir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register64 lhsRegs = ToRegister64(lhs);
+ Register output = ToRegister(lir->output());
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition condition = JSOpToCondition(lir->jsop(), isSigned);
+ Label done;
+
+ masm.move32(Imm32(1), output);
+
+ if (IsConstant(rhs)) {
+ Imm64 imm = Imm64(ToInt64(rhs));
+ masm.branch64(condition, lhsRegs, imm, &done);
+ } else {
+ Register64 rhsRegs = ToRegister64(rhs);
+ masm.branch64(condition, lhsRegs, rhsRegs, &done);
+ }
+
+ masm.move32(Imm32(0), output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+ MCompare* mir = lir->cmpMir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register64 lhsRegs = ToRegister64(lhs);
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition condition = JSOpToCondition(lir->jsop(), isSigned);
+
+ Label* trueLabel = getJumpLabelForBranch(lir->ifTrue());
+ Label* falseLabel = getJumpLabelForBranch(lir->ifFalse());
+
+ if (isNextBlock(lir->ifFalse()->lir())) {
+ falseLabel = nullptr;
+ } else if (isNextBlock(lir->ifTrue()->lir())) {
+ condition = Assembler::InvertCondition(condition);
+ trueLabel = falseLabel;
+ falseLabel = nullptr;
+ }
+
+ if (IsConstant(rhs)) {
+ Imm64 imm = Imm64(ToInt64(rhs));
+ masm.branch64(condition, lhsRegs, imm, trueLabel, falseLabel);
+ } else {
+ Register64 rhsRegs = ToRegister64(rhs);
+ masm.branch64(condition, lhsRegs, rhsRegs, trueLabel, falseLabel);
+ }
+}
+
+void CodeGenerator::visitShiftI64(LShiftI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LShiftI64::Lhs);
+ LAllocation* rhs = lir->getOperand(LShiftI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (rhs->isConstant()) {
+ int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ if (shift) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::Rsh:
+ if (shift) {
+ masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.rshift64(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ return;
+ }
+
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ masm.lshift64(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::Rsh:
+ masm.rshift64Arithmetic(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::Ursh:
+ masm.rshift64(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+}
+
+void CodeGenerator::visitBitOpI64(LBitOpI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LBitOpI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LBitOpI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ switch (lir->bitop()) {
+ case JSOp::BitOr:
+ if (IsConstant(rhs)) {
+ masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.or64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::BitXor:
+ if (IsConstant(rhs)) {
+ masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.xor64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::BitAnd:
+ if (IsConstant(rhs)) {
+ masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.and64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitRotateI64(LRotateI64* lir) {
+ MRotate* mir = lir->mir();
+ LAllocation* count = lir->count();
+
+ Register64 input = ToRegister64(lir->input());
+ Register64 output = ToOutRegister64(lir);
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+
+ if (count->isConstant()) {
+ int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F);
+ if (!c) {
+ masm.move64(input, output);
+ return;
+ }
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft64(Imm32(c), input, output, temp);
+ } else {
+ masm.rotateRight64(Imm32(c), input, output, temp);
+ }
+ } else {
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft64(ToRegister(count), input, output, temp);
+ } else {
+ masm.rotateRight64(ToRegister(count), input, output, temp);
+ }
+ }
+}
+
+void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(StackPointer, mir->spOffset());
+ if (IsConstant(ins->arg())) {
+ masm.store64(Imm64(ToInt64(ins->arg())), dst);
+ } else {
+ masm.store64(ToRegister64(ins->arg()), dst);
+ }
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+ Register cond = ToRegister(lir->condExpr());
+ const LInt64Allocation falseExpr = lir->falseExpr();
+
+ Register64 out = ToOutRegister64(lir);
+ MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
+ "true expr is reused for input");
+
+ masm.as_cmp(cond, Imm8(0));
+ if (falseExpr.low().isRegister()) {
+ masm.ma_mov(ToRegister(falseExpr.low()), out.low, LeaveCC,
+ Assembler::Equal);
+ masm.ma_mov(ToRegister(falseExpr.high()), out.high, LeaveCC,
+ Assembler::Equal);
+ } else {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_ldr(ToAddress(falseExpr.low()), out.low, scratch, Offset,
+ Assembler::Equal);
+ masm.ma_ldr(ToAddress(falseExpr.high()), out.high, scratch, Offset,
+ Assembler::Equal);
+ }
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ masm.ma_vxfer(input.low, input.high, output);
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+ FloatRegister input = ToFloatRegister(lir->getOperand(0));
+ Register64 output = ToOutRegister64(lir);
+
+ masm.ma_vxfer(input, output.low, output.high);
+}
+
+void CodeGenerator::visitPopcntI64(LPopcntI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ Register temp = ToRegister(lir->getTemp(0));
+
+ masm.popcnt64(input, output, temp);
+}
+
+void CodeGenerator::visitClzI64(LClzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+
+ masm.clz64(input, output.low);
+ masm.move32(Imm32(0), output.high);
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+
+ masm.ctz64(input, output.low);
+ masm.move32(Imm32(0), output.high);
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ MOZ_ASSERT(input == ToOutRegister64(lir));
+ masm.ma_mvn(input.high, input.high);
+ masm.ma_mvn(input.low, input.low);
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+
+ masm.as_cmp(input.high, Imm8(0));
+ jumpToBlock(lir->ifTrue(), Assembler::NonZero);
+ masm.as_cmp(input.low, Imm8(0));
+ emitBranch(Assembler::NonZero, lir->ifTrue(), lir->ifFalse());
+}
+
+void CodeGenerator::visitWasmAtomicLoadI64(LWasmAtomicLoadI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 output = ToOutRegister64(lir);
+ Register64 tmp(InvalidReg, InvalidReg);
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, lir->mir()->access().offset());
+ masm.wasmAtomicLoad64(lir->mir()->access(), addr, tmp, output);
+}
+
+void CodeGenerator::visitWasmAtomicStoreI64(LWasmAtomicStoreI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 tmp(ToRegister(lir->tmpHigh()), ToRegister(lir->tmpLow()));
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, lir->mir()->access().offset());
+ masm.wasmAtomicExchange64(lir->mir()->access(), addr, value, tmp);
+}
+
+void CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 expected = ToRegister64(lir->expected());
+ Register64 replacement = ToRegister64(lir->replacement());
+ Register64 out = ToOutRegister64(lir);
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, lir->mir()->access().offset());
+ masm.wasmCompareExchange64(lir->mir()->access(), addr, expected, replacement,
+ out);
+}
+
+void CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 out = ToOutRegister64(lir);
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, lir->access().offset());
+ Register64 tmp(ToRegister(lir->tmpHigh()), ToRegister(lir->tmpLow()));
+ masm.wasmAtomicFetchOp64(lir->access(), lir->operation(), value, addr, tmp,
+ out);
+}
+
+void CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 out = ToOutRegister64(lir);
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, lir->access().offset());
+ masm.wasmAtomicExchange64(lir->access(), addr, value, out);
+}
+
+void CodeGenerator::visitNearbyInt(LNearbyInt*) { MOZ_CRASH("NYI"); }
+
+void CodeGenerator::visitNearbyIntF(LNearbyIntF*) { MOZ_CRASH("NYI"); }
+
+void CodeGenerator::visitSimd128(LSimd128* ins) { MOZ_CRASH("No SIMD"); }
+
+void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmBinarySimd128WithConstant(
+ LWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmVariableShiftSimd128(
+ LWasmVariableShiftSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmConstantShiftSimd128(
+ LWasmConstantShiftSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmSignReplicationSimd128(
+ LWasmSignReplicationSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
+ LWasmReplaceInt64LaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceAndBranchSimd128(
+ LWasmReduceAndBranchSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceSimd128ToInt64(
+ LWasmReduceSimd128ToInt64* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
diff --git a/js/src/jit/arm/CodeGenerator-arm.h b/js/src/jit/arm/CodeGenerator-arm.h
new file mode 100644
index 0000000000..f7cf2b263e
--- /dev/null
+++ b/js/src/jit/arm/CodeGenerator-arm.h
@@ -0,0 +1,172 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_CodeGenerator_arm_h
+#define jit_arm_CodeGenerator_arm_h
+
+#include "jit/arm/Assembler-arm.h"
+#include "jit/shared/CodeGenerator-shared.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorARM;
+class OutOfLineBailout;
+class OutOfLineTableSwitch;
+
+using OutOfLineWasmTruncateCheck =
+ OutOfLineWasmTruncateCheckBase<CodeGeneratorARM>;
+
+class CodeGeneratorARM : public CodeGeneratorShared {
+ friend class MoveResolverARM;
+
+ protected:
+ CodeGeneratorARM(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
+
+ NonAssertingLabel deoptLabel_;
+
+ MoveOperand toMoveOperand(LAllocation a) const;
+
+ void bailoutIf(Assembler::Condition condition, LSnapshot* snapshot);
+ void bailoutFrom(Label* label, LSnapshot* snapshot);
+ void bailout(LSnapshot* snapshot);
+
+ template <typename T1, typename T2>
+ void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.cmpPtr(lhs, rhs);
+ bailoutIf(c, snapshot);
+ }
+ void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs,
+ LSnapshot* snapshot) {
+ masm.testPtr(lhs, rhs);
+ bailoutIf(c, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.cmp32(lhs, rhs);
+ bailoutIf(c, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutTest32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.test32(lhs, rhs);
+ bailoutIf(c, snapshot);
+ }
+ void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) {
+ masm.test32(reg, Imm32(0xFF));
+ bailoutIf(Assembler::Zero, snapshot);
+ }
+
+ template <class T>
+ void generateUDivModZeroCheck(Register rhs, Register output, Label* done,
+ LSnapshot* snapshot, T* mir);
+
+ bool generateOutOfLineCode();
+
+ // Emits a branch that directs control flow to the true block if |cond| is
+ // true, and the false block if |cond| is false.
+ void emitBranch(Assembler::Condition cond, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse);
+
+ void testNullEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ cond = masm.testNull(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+ void testUndefinedEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ cond = masm.testUndefined(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+ void testObjectEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ cond = masm.testObject(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+ void testZeroEmitBranch(Assembler::Condition cond, Register reg,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ masm.cmpPtr(reg, ImmWord(0));
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+
+ void emitTableSwitchDispatch(MTableSwitch* mir, Register index,
+ Register base);
+
+ void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+
+ template <typename T>
+ void emitWasmLoad(T* ins);
+ template <typename T>
+ void emitWasmUnalignedLoad(T* ins);
+ template <typename T>
+ void emitWasmStore(T* ins);
+ template <typename T>
+ void emitWasmUnalignedStore(T* ins);
+
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ Register64 ToOperandOrRegister64(const LInt64Allocation input);
+
+ // Functions for LTestVAndBranch.
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag);
+
+ void divICommon(MDiv* mir, Register lhs, Register rhs, Register output,
+ LSnapshot* snapshot, Label& done);
+ void modICommon(MMod* mir, Register lhs, Register rhs, Register output,
+ LSnapshot* snapshot, Label& done);
+
+ void generateInvalidateEpilogue();
+
+ // Generating a result.
+ template <typename S, typename T>
+ void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
+ const S& value, const T& mem,
+ Register flagTemp, Register outTemp,
+ AnyRegister output);
+
+ // Generating no result.
+ template <typename S, typename T>
+ void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
+ const S& value, const T& mem,
+ Register flagTemp);
+
+ public:
+ void visitOutOfLineBailout(OutOfLineBailout* ool);
+ void visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool);
+ void visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool);
+};
+
+typedef CodeGeneratorARM CodeGeneratorSpecific;
+
+// An out-of-line bailout thunk.
+class OutOfLineBailout : public OutOfLineCodeBase<CodeGeneratorARM> {
+ protected: // Silence Clang warning.
+ LSnapshot* snapshot_;
+ uint32_t frameSize_;
+
+ public:
+ OutOfLineBailout(LSnapshot* snapshot, uint32_t frameSize)
+ : snapshot_(snapshot), frameSize_(frameSize) {}
+
+ void accept(CodeGeneratorARM* codegen) override;
+
+ LSnapshot* snapshot() const { return snapshot_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_CodeGenerator_arm_h */
diff --git a/js/src/jit/arm/DoubleEntryTable.tbl b/js/src/jit/arm/DoubleEntryTable.tbl
new file mode 100644
index 0000000000..2e9e8c4a34
--- /dev/null
+++ b/js/src/jit/arm/DoubleEntryTable.tbl
@@ -0,0 +1,257 @@
+/* THIS FILE IS AUTOMATICALLY GENERATED BY gen-double-encode-table.py. */
+ { 0x40000000, { 0, 0, 0 } },
+ { 0x40010000, { 1, 0, 0 } },
+ { 0x40020000, { 2, 0, 0 } },
+ { 0x40030000, { 3, 0, 0 } },
+ { 0x40040000, { 4, 0, 0 } },
+ { 0x40050000, { 5, 0, 0 } },
+ { 0x40060000, { 6, 0, 0 } },
+ { 0x40070000, { 7, 0, 0 } },
+ { 0x40080000, { 8, 0, 0 } },
+ { 0x40090000, { 9, 0, 0 } },
+ { 0x400a0000, { 10, 0, 0 } },
+ { 0x400b0000, { 11, 0, 0 } },
+ { 0x400c0000, { 12, 0, 0 } },
+ { 0x400d0000, { 13, 0, 0 } },
+ { 0x400e0000, { 14, 0, 0 } },
+ { 0x400f0000, { 15, 0, 0 } },
+ { 0x40100000, { 0, 1, 0 } },
+ { 0x40110000, { 1, 1, 0 } },
+ { 0x40120000, { 2, 1, 0 } },
+ { 0x40130000, { 3, 1, 0 } },
+ { 0x40140000, { 4, 1, 0 } },
+ { 0x40150000, { 5, 1, 0 } },
+ { 0x40160000, { 6, 1, 0 } },
+ { 0x40170000, { 7, 1, 0 } },
+ { 0x40180000, { 8, 1, 0 } },
+ { 0x40190000, { 9, 1, 0 } },
+ { 0x401a0000, { 10, 1, 0 } },
+ { 0x401b0000, { 11, 1, 0 } },
+ { 0x401c0000, { 12, 1, 0 } },
+ { 0x401d0000, { 13, 1, 0 } },
+ { 0x401e0000, { 14, 1, 0 } },
+ { 0x401f0000, { 15, 1, 0 } },
+ { 0x40200000, { 0, 2, 0 } },
+ { 0x40210000, { 1, 2, 0 } },
+ { 0x40220000, { 2, 2, 0 } },
+ { 0x40230000, { 3, 2, 0 } },
+ { 0x40240000, { 4, 2, 0 } },
+ { 0x40250000, { 5, 2, 0 } },
+ { 0x40260000, { 6, 2, 0 } },
+ { 0x40270000, { 7, 2, 0 } },
+ { 0x40280000, { 8, 2, 0 } },
+ { 0x40290000, { 9, 2, 0 } },
+ { 0x402a0000, { 10, 2, 0 } },
+ { 0x402b0000, { 11, 2, 0 } },
+ { 0x402c0000, { 12, 2, 0 } },
+ { 0x402d0000, { 13, 2, 0 } },
+ { 0x402e0000, { 14, 2, 0 } },
+ { 0x402f0000, { 15, 2, 0 } },
+ { 0x40300000, { 0, 3, 0 } },
+ { 0x40310000, { 1, 3, 0 } },
+ { 0x40320000, { 2, 3, 0 } },
+ { 0x40330000, { 3, 3, 0 } },
+ { 0x40340000, { 4, 3, 0 } },
+ { 0x40350000, { 5, 3, 0 } },
+ { 0x40360000, { 6, 3, 0 } },
+ { 0x40370000, { 7, 3, 0 } },
+ { 0x40380000, { 8, 3, 0 } },
+ { 0x40390000, { 9, 3, 0 } },
+ { 0x403a0000, { 10, 3, 0 } },
+ { 0x403b0000, { 11, 3, 0 } },
+ { 0x403c0000, { 12, 3, 0 } },
+ { 0x403d0000, { 13, 3, 0 } },
+ { 0x403e0000, { 14, 3, 0 } },
+ { 0x403f0000, { 15, 3, 0 } },
+ { 0x3fc00000, { 0, 4, 0 } },
+ { 0x3fc10000, { 1, 4, 0 } },
+ { 0x3fc20000, { 2, 4, 0 } },
+ { 0x3fc30000, { 3, 4, 0 } },
+ { 0x3fc40000, { 4, 4, 0 } },
+ { 0x3fc50000, { 5, 4, 0 } },
+ { 0x3fc60000, { 6, 4, 0 } },
+ { 0x3fc70000, { 7, 4, 0 } },
+ { 0x3fc80000, { 8, 4, 0 } },
+ { 0x3fc90000, { 9, 4, 0 } },
+ { 0x3fca0000, { 10, 4, 0 } },
+ { 0x3fcb0000, { 11, 4, 0 } },
+ { 0x3fcc0000, { 12, 4, 0 } },
+ { 0x3fcd0000, { 13, 4, 0 } },
+ { 0x3fce0000, { 14, 4, 0 } },
+ { 0x3fcf0000, { 15, 4, 0 } },
+ { 0x3fd00000, { 0, 5, 0 } },
+ { 0x3fd10000, { 1, 5, 0 } },
+ { 0x3fd20000, { 2, 5, 0 } },
+ { 0x3fd30000, { 3, 5, 0 } },
+ { 0x3fd40000, { 4, 5, 0 } },
+ { 0x3fd50000, { 5, 5, 0 } },
+ { 0x3fd60000, { 6, 5, 0 } },
+ { 0x3fd70000, { 7, 5, 0 } },
+ { 0x3fd80000, { 8, 5, 0 } },
+ { 0x3fd90000, { 9, 5, 0 } },
+ { 0x3fda0000, { 10, 5, 0 } },
+ { 0x3fdb0000, { 11, 5, 0 } },
+ { 0x3fdc0000, { 12, 5, 0 } },
+ { 0x3fdd0000, { 13, 5, 0 } },
+ { 0x3fde0000, { 14, 5, 0 } },
+ { 0x3fdf0000, { 15, 5, 0 } },
+ { 0x3fe00000, { 0, 6, 0 } },
+ { 0x3fe10000, { 1, 6, 0 } },
+ { 0x3fe20000, { 2, 6, 0 } },
+ { 0x3fe30000, { 3, 6, 0 } },
+ { 0x3fe40000, { 4, 6, 0 } },
+ { 0x3fe50000, { 5, 6, 0 } },
+ { 0x3fe60000, { 6, 6, 0 } },
+ { 0x3fe70000, { 7, 6, 0 } },
+ { 0x3fe80000, { 8, 6, 0 } },
+ { 0x3fe90000, { 9, 6, 0 } },
+ { 0x3fea0000, { 10, 6, 0 } },
+ { 0x3feb0000, { 11, 6, 0 } },
+ { 0x3fec0000, { 12, 6, 0 } },
+ { 0x3fed0000, { 13, 6, 0 } },
+ { 0x3fee0000, { 14, 6, 0 } },
+ { 0x3fef0000, { 15, 6, 0 } },
+ { 0x3ff00000, { 0, 7, 0 } },
+ { 0x3ff10000, { 1, 7, 0 } },
+ { 0x3ff20000, { 2, 7, 0 } },
+ { 0x3ff30000, { 3, 7, 0 } },
+ { 0x3ff40000, { 4, 7, 0 } },
+ { 0x3ff50000, { 5, 7, 0 } },
+ { 0x3ff60000, { 6, 7, 0 } },
+ { 0x3ff70000, { 7, 7, 0 } },
+ { 0x3ff80000, { 8, 7, 0 } },
+ { 0x3ff90000, { 9, 7, 0 } },
+ { 0x3ffa0000, { 10, 7, 0 } },
+ { 0x3ffb0000, { 11, 7, 0 } },
+ { 0x3ffc0000, { 12, 7, 0 } },
+ { 0x3ffd0000, { 13, 7, 0 } },
+ { 0x3ffe0000, { 14, 7, 0 } },
+ { 0x3fff0000, { 15, 7, 0 } },
+ { 0xc0000000, { 0, 8, 0 } },
+ { 0xc0010000, { 1, 8, 0 } },
+ { 0xc0020000, { 2, 8, 0 } },
+ { 0xc0030000, { 3, 8, 0 } },
+ { 0xc0040000, { 4, 8, 0 } },
+ { 0xc0050000, { 5, 8, 0 } },
+ { 0xc0060000, { 6, 8, 0 } },
+ { 0xc0070000, { 7, 8, 0 } },
+ { 0xc0080000, { 8, 8, 0 } },
+ { 0xc0090000, { 9, 8, 0 } },
+ { 0xc00a0000, { 10, 8, 0 } },
+ { 0xc00b0000, { 11, 8, 0 } },
+ { 0xc00c0000, { 12, 8, 0 } },
+ { 0xc00d0000, { 13, 8, 0 } },
+ { 0xc00e0000, { 14, 8, 0 } },
+ { 0xc00f0000, { 15, 8, 0 } },
+ { 0xc0100000, { 0, 9, 0 } },
+ { 0xc0110000, { 1, 9, 0 } },
+ { 0xc0120000, { 2, 9, 0 } },
+ { 0xc0130000, { 3, 9, 0 } },
+ { 0xc0140000, { 4, 9, 0 } },
+ { 0xc0150000, { 5, 9, 0 } },
+ { 0xc0160000, { 6, 9, 0 } },
+ { 0xc0170000, { 7, 9, 0 } },
+ { 0xc0180000, { 8, 9, 0 } },
+ { 0xc0190000, { 9, 9, 0 } },
+ { 0xc01a0000, { 10, 9, 0 } },
+ { 0xc01b0000, { 11, 9, 0 } },
+ { 0xc01c0000, { 12, 9, 0 } },
+ { 0xc01d0000, { 13, 9, 0 } },
+ { 0xc01e0000, { 14, 9, 0 } },
+ { 0xc01f0000, { 15, 9, 0 } },
+ { 0xc0200000, { 0, 10, 0 } },
+ { 0xc0210000, { 1, 10, 0 } },
+ { 0xc0220000, { 2, 10, 0 } },
+ { 0xc0230000, { 3, 10, 0 } },
+ { 0xc0240000, { 4, 10, 0 } },
+ { 0xc0250000, { 5, 10, 0 } },
+ { 0xc0260000, { 6, 10, 0 } },
+ { 0xc0270000, { 7, 10, 0 } },
+ { 0xc0280000, { 8, 10, 0 } },
+ { 0xc0290000, { 9, 10, 0 } },
+ { 0xc02a0000, { 10, 10, 0 } },
+ { 0xc02b0000, { 11, 10, 0 } },
+ { 0xc02c0000, { 12, 10, 0 } },
+ { 0xc02d0000, { 13, 10, 0 } },
+ { 0xc02e0000, { 14, 10, 0 } },
+ { 0xc02f0000, { 15, 10, 0 } },
+ { 0xc0300000, { 0, 11, 0 } },
+ { 0xc0310000, { 1, 11, 0 } },
+ { 0xc0320000, { 2, 11, 0 } },
+ { 0xc0330000, { 3, 11, 0 } },
+ { 0xc0340000, { 4, 11, 0 } },
+ { 0xc0350000, { 5, 11, 0 } },
+ { 0xc0360000, { 6, 11, 0 } },
+ { 0xc0370000, { 7, 11, 0 } },
+ { 0xc0380000, { 8, 11, 0 } },
+ { 0xc0390000, { 9, 11, 0 } },
+ { 0xc03a0000, { 10, 11, 0 } },
+ { 0xc03b0000, { 11, 11, 0 } },
+ { 0xc03c0000, { 12, 11, 0 } },
+ { 0xc03d0000, { 13, 11, 0 } },
+ { 0xc03e0000, { 14, 11, 0 } },
+ { 0xc03f0000, { 15, 11, 0 } },
+ { 0xbfc00000, { 0, 12, 0 } },
+ { 0xbfc10000, { 1, 12, 0 } },
+ { 0xbfc20000, { 2, 12, 0 } },
+ { 0xbfc30000, { 3, 12, 0 } },
+ { 0xbfc40000, { 4, 12, 0 } },
+ { 0xbfc50000, { 5, 12, 0 } },
+ { 0xbfc60000, { 6, 12, 0 } },
+ { 0xbfc70000, { 7, 12, 0 } },
+ { 0xbfc80000, { 8, 12, 0 } },
+ { 0xbfc90000, { 9, 12, 0 } },
+ { 0xbfca0000, { 10, 12, 0 } },
+ { 0xbfcb0000, { 11, 12, 0 } },
+ { 0xbfcc0000, { 12, 12, 0 } },
+ { 0xbfcd0000, { 13, 12, 0 } },
+ { 0xbfce0000, { 14, 12, 0 } },
+ { 0xbfcf0000, { 15, 12, 0 } },
+ { 0xbfd00000, { 0, 13, 0 } },
+ { 0xbfd10000, { 1, 13, 0 } },
+ { 0xbfd20000, { 2, 13, 0 } },
+ { 0xbfd30000, { 3, 13, 0 } },
+ { 0xbfd40000, { 4, 13, 0 } },
+ { 0xbfd50000, { 5, 13, 0 } },
+ { 0xbfd60000, { 6, 13, 0 } },
+ { 0xbfd70000, { 7, 13, 0 } },
+ { 0xbfd80000, { 8, 13, 0 } },
+ { 0xbfd90000, { 9, 13, 0 } },
+ { 0xbfda0000, { 10, 13, 0 } },
+ { 0xbfdb0000, { 11, 13, 0 } },
+ { 0xbfdc0000, { 12, 13, 0 } },
+ { 0xbfdd0000, { 13, 13, 0 } },
+ { 0xbfde0000, { 14, 13, 0 } },
+ { 0xbfdf0000, { 15, 13, 0 } },
+ { 0xbfe00000, { 0, 14, 0 } },
+ { 0xbfe10000, { 1, 14, 0 } },
+ { 0xbfe20000, { 2, 14, 0 } },
+ { 0xbfe30000, { 3, 14, 0 } },
+ { 0xbfe40000, { 4, 14, 0 } },
+ { 0xbfe50000, { 5, 14, 0 } },
+ { 0xbfe60000, { 6, 14, 0 } },
+ { 0xbfe70000, { 7, 14, 0 } },
+ { 0xbfe80000, { 8, 14, 0 } },
+ { 0xbfe90000, { 9, 14, 0 } },
+ { 0xbfea0000, { 10, 14, 0 } },
+ { 0xbfeb0000, { 11, 14, 0 } },
+ { 0xbfec0000, { 12, 14, 0 } },
+ { 0xbfed0000, { 13, 14, 0 } },
+ { 0xbfee0000, { 14, 14, 0 } },
+ { 0xbfef0000, { 15, 14, 0 } },
+ { 0xbff00000, { 0, 15, 0 } },
+ { 0xbff10000, { 1, 15, 0 } },
+ { 0xbff20000, { 2, 15, 0 } },
+ { 0xbff30000, { 3, 15, 0 } },
+ { 0xbff40000, { 4, 15, 0 } },
+ { 0xbff50000, { 5, 15, 0 } },
+ { 0xbff60000, { 6, 15, 0 } },
+ { 0xbff70000, { 7, 15, 0 } },
+ { 0xbff80000, { 8, 15, 0 } },
+ { 0xbff90000, { 9, 15, 0 } },
+ { 0xbffa0000, { 10, 15, 0 } },
+ { 0xbffb0000, { 11, 15, 0 } },
+ { 0xbffc0000, { 12, 15, 0 } },
+ { 0xbffd0000, { 13, 15, 0 } },
+ { 0xbffe0000, { 14, 15, 0 } },
+ { 0xbfff0000, { 15, 15, 0 } },
diff --git a/js/src/jit/arm/LIR-arm.h b/js/src/jit/arm/LIR-arm.h
new file mode 100644
index 0000000000..395b285c93
--- /dev/null
+++ b/js/src/jit/arm/LIR-arm.h
@@ -0,0 +1,511 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_LIR_arm_h
+#define jit_arm_LIR_arm_h
+
+namespace js {
+namespace jit {
+
+class LBoxFloatingPoint : public LInstructionHelper<2, 1, 1> {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(BoxFloatingPoint);
+
+ LBoxFloatingPoint(const LAllocation& in, const LDefinition& temp,
+ MIRType type)
+ : LInstructionHelper(classOpcode), type_(type) {
+ setOperand(0, in);
+ setTemp(0, temp);
+ }
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+class LUnbox : public LInstructionHelper<1, 2, 0> {
+ public:
+ LIR_HEADER(Unbox);
+
+ LUnbox() : LInstructionHelper(classOpcode) {}
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+ const LAllocation* payload() { return getOperand(0); }
+ const LAllocation* type() { return getOperand(1); }
+ const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LInstructionHelper<1, 2, 0> {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint);
+
+ static const size_t Input = 0;
+
+ LUnboxFloatingPoint(const LBoxAllocation& input, MIRType type)
+ : LInstructionHelper(classOpcode), type_(type) {
+ setBoxOperand(Input, input);
+ }
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToDouble)
+
+ explicit LWasmUint32ToDouble(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToFloat32)
+
+ explicit LWasmUint32ToFloat32(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+class LDivI : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivI);
+
+ LDivI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LDivOrModI64
+ : public LCallInstructionHelper<INT64_PIECES, INT64_PIECES * 2 + 1, 0> {
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+ static const size_t Instance = 2 * INT64_PIECES;
+
+ LDivOrModI64(const LInt64Allocation& lhs, const LInt64Allocation& rhs,
+ const LAllocation& instance)
+ : LCallInstructionHelper(classOpcode) {
+ setInt64Operand(Lhs, lhs);
+ setInt64Operand(Rhs, rhs);
+ setOperand(Instance, instance);
+ }
+
+ MDefinition* mir() const {
+ MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64());
+ return mir_;
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->canBeDivideByZero();
+ }
+ return mir_->toWasmBuiltinDivI64()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->canBeNegativeDividend();
+ }
+ return mir_->toWasmBuiltinDivI64()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64());
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->bytecodeOffset();
+ }
+ return mir_->toWasmBuiltinDivI64()->bytecodeOffset();
+ }
+};
+
+class LUDivOrModI64
+ : public LCallInstructionHelper<INT64_PIECES, INT64_PIECES * 2 + 1, 0> {
+ public:
+ LIR_HEADER(UDivOrModI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+ static const size_t Instance = 2 * INT64_PIECES;
+
+ LUDivOrModI64(const LInt64Allocation& lhs, const LInt64Allocation& rhs,
+ const LAllocation& instance)
+ : LCallInstructionHelper(classOpcode) {
+ setInt64Operand(Lhs, lhs);
+ setInt64Operand(Rhs, rhs);
+ setOperand(Instance, instance);
+ }
+
+ MDefinition* mir() const {
+ MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64());
+ return mir_;
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->canBeDivideByZero();
+ }
+ return mir_->toWasmBuiltinDivI64()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->canBeNegativeDividend();
+ }
+ return mir_->toWasmBuiltinDivI64()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64());
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->bytecodeOffset();
+ }
+ return mir_->toWasmBuiltinDivI64()->bytecodeOffset();
+ }
+};
+
+// LSoftDivI is a software divide for ARM cores that don't support a hardware
+// divide instruction, implemented as a C++ native call.
+class LSoftDivI : public LBinaryCallInstructionHelper<1, 0> {
+ public:
+ LIR_HEADER(SoftDivI);
+
+ LSoftDivI(const LAllocation& lhs, const LAllocation& rhs)
+ : LBinaryCallInstructionHelper(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ }
+
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LDivPowTwoI : public LInstructionHelper<1, 1, 0> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(DivPowTwoI)
+
+ LDivPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+
+ int32_t shift() { return shift_; }
+
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LModI : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(ModI);
+
+ LModI(const LAllocation& lhs, const LAllocation& rhs)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ }
+
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LSoftModI : public LBinaryCallInstructionHelper<1, 1> {
+ public:
+ LIR_HEADER(SoftModI);
+
+ LSoftModI(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryCallInstructionHelper(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* callTemp() { return getTemp(0); }
+
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModPowTwoI : public LInstructionHelper<1, 1, 0> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModPowTwoI);
+ int32_t shift() { return shift_; }
+
+ LModPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ }
+
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModMaskI : public LInstructionHelper<1, 1, 2> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModMaskI);
+
+ LModMaskI(const LAllocation& lhs, const LDefinition& temp1,
+ const LDefinition& temp2, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ int32_t shift() const { return shift_; }
+
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+// Takes a tableswitch with an integer to decide.
+class LTableSwitch : public LInstructionHelper<0, 1, 1> {
+ public:
+ LIR_HEADER(TableSwitch);
+
+ LTableSwitch(const LAllocation& in, const LDefinition& inputCopy,
+ MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setTemp(0, inputCopy);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+
+ const LAllocation* index() { return getOperand(0); }
+ const LDefinition* tempInt() { return getTemp(0); }
+ // This is added to share the same CodeGenerator prefixes.
+ const LDefinition* tempPointer() { return nullptr; }
+};
+
+// Takes a tableswitch with an integer to decide.
+class LTableSwitchV : public LInstructionHelper<0, BOX_PIECES, 2> {
+ public:
+ LIR_HEADER(TableSwitchV);
+
+ LTableSwitchV(const LBoxAllocation& input, const LDefinition& inputCopy,
+ const LDefinition& floatCopy, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setBoxOperand(InputValue, input);
+ setTemp(0, inputCopy);
+ setTemp(1, floatCopy);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+
+ static const size_t InputValue = 0;
+
+ const LDefinition* tempInt() { return getTemp(0); }
+ const LDefinition* tempFloat() { return getTemp(1); }
+ const LDefinition* tempPointer() { return nullptr; }
+};
+
+class LMulI : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(MulI);
+
+ LMulI() : LBinaryMath(classOpcode) {}
+
+ MMul* mir() { return mir_->toMul(); }
+};
+
+class LUDiv : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(UDiv);
+
+ LUDiv() : LBinaryMath(classOpcode) {}
+
+ MDiv* mir() { return mir_->toDiv(); }
+};
+
+class LUMod : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(UMod);
+
+ LUMod() : LBinaryMath(classOpcode) {}
+
+ MMod* mir() { return mir_->toMod(); }
+};
+
+class LSoftUDivOrMod : public LBinaryCallInstructionHelper<1, 0> {
+ public:
+ LIR_HEADER(SoftUDivOrMod);
+
+ LSoftUDivOrMod(const LAllocation& lhs, const LAllocation& rhs)
+ : LBinaryCallInstructionHelper(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ }
+
+ MInstruction* mir() { return mir_->toInstruction(); }
+};
+
+class LWasmTruncateToInt64 : public LCallInstructionHelper<INT64_PIECES, 2, 0> {
+ static const size_t Input = 0;
+ static const size_t Instance = 1;
+
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ LWasmTruncateToInt64(const LAllocation& in, const LAllocation& instance)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(Input, in);
+ setOperand(Instance, instance);
+ }
+
+ LAllocation* input() { return getOperand(Input); }
+ LAllocation* instance() { return getOperand(Instance); }
+
+ MWasmBuiltinTruncateToInt64* mir() const {
+ return mir_->toWasmBuiltinTruncateToInt64();
+ }
+};
+
+class LInt64ToFloatingPointCall
+ : public LCallInstructionHelper<1, INT64_PIECES + 1, 0> {
+ public:
+ LIR_HEADER(Int64ToFloatingPointCall);
+
+ static const size_t Input = 0;
+ static const size_t Instance = INT64_PIECES;
+
+ LInt64ToFloatingPointCall(const LInt64Allocation& in,
+ const LAllocation& instance)
+ : LCallInstructionHelper(classOpcode) {
+ setInt64Operand(Input, in);
+ setOperand(Instance, instance);
+ }
+
+ LAllocation* input() { return getOperand(Input); }
+ LAllocation* instance() { return getOperand(Instance); }
+
+ MBuiltinInt64ToFloatingPoint* mir() const {
+ return mir_->toBuiltinInt64ToFloatingPoint();
+ }
+};
+
+class LWasmAtomicLoadI64 : public LInstructionHelper<INT64_PIECES, 1, 0> {
+ public:
+ LIR_HEADER(WasmAtomicLoadI64);
+
+ explicit LWasmAtomicLoadI64(const LAllocation& ptr)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ }
+
+ MWasmLoad* mir() const { return mir_->toWasmLoad(); }
+ const LAllocation* ptr() { return getOperand(0); }
+};
+
+class LWasmAtomicStoreI64 : public LInstructionHelper<0, 1 + INT64_PIECES, 2> {
+ public:
+ LIR_HEADER(WasmAtomicStoreI64);
+
+ LWasmAtomicStoreI64(const LAllocation& ptr, const LInt64Allocation& value,
+ const LDefinition& tmpLow, const LDefinition& tmpHigh)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ setTemp(0, tmpLow);
+ setTemp(1, tmpHigh);
+ }
+
+ MWasmStore* mir() const { return mir_->toWasmStore(); }
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation value() { return getInt64Operand(1); }
+ const LDefinition* tmpLow() { return getTemp(0); }
+ const LDefinition* tmpHigh() { return getTemp(1); }
+};
+
+class LWasmCompareExchangeI64
+ : public LInstructionHelper<INT64_PIECES, 1 + 2 * INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(WasmCompareExchangeI64);
+
+ LWasmCompareExchangeI64(const LAllocation& ptr,
+ const LInt64Allocation& expected,
+ const LInt64Allocation& replacement)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, expected);
+ setInt64Operand(1 + INT64_PIECES, replacement);
+ }
+
+ MWasmCompareExchangeHeap* mir() const {
+ return mir_->toWasmCompareExchangeHeap();
+ }
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation expected() { return getInt64Operand(1); }
+ const LInt64Allocation replacement() {
+ return getInt64Operand(1 + INT64_PIECES);
+ }
+};
+
+class LWasmAtomicBinopI64
+ : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES, 2> {
+ const wasm::MemoryAccessDesc& access_;
+ AtomicOp op_;
+
+ public:
+ LIR_HEADER(WasmAtomicBinopI64);
+
+ LWasmAtomicBinopI64(const LAllocation& ptr, const LInt64Allocation& value,
+ const LDefinition& tmpLow, const LDefinition& tmpHigh,
+ const wasm::MemoryAccessDesc& access, AtomicOp op)
+ : LInstructionHelper(classOpcode), access_(access), op_(op) {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ setTemp(0, tmpLow);
+ setTemp(1, tmpHigh);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation value() { return getInt64Operand(1); }
+ const wasm::MemoryAccessDesc& access() { return access_; }
+ AtomicOp operation() const { return op_; }
+ const LDefinition* tmpLow() { return getTemp(0); }
+ const LDefinition* tmpHigh() { return getTemp(1); }
+};
+
+class LWasmAtomicExchangeI64
+ : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES, 0> {
+ const wasm::MemoryAccessDesc& access_;
+
+ public:
+ LIR_HEADER(WasmAtomicExchangeI64);
+
+ LWasmAtomicExchangeI64(const LAllocation& ptr, const LInt64Allocation& value,
+ const wasm::MemoryAccessDesc& access)
+ : LInstructionHelper(classOpcode), access_(access) {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation value() { return getInt64Operand(1); }
+ const wasm::MemoryAccessDesc& access() { return access_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_LIR_arm_h */
diff --git a/js/src/jit/arm/Lowering-arm.cpp b/js/src/jit/arm/Lowering-arm.cpp
new file mode 100644
index 0000000000..e384ee7911
--- /dev/null
+++ b/js/src/jit/arm/Lowering-arm.cpp
@@ -0,0 +1,1223 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm/Lowering-arm.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/arm/Assembler-arm.h"
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::FloorLog2;
+
+LBoxAllocation LIRGeneratorARM::useBoxFixed(MDefinition* mir, Register reg1,
+ Register reg2, bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+ MOZ_ASSERT(reg1 != reg2);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart),
+ LUse(reg2, VirtualRegisterOfPayload(mir), useAtStart));
+}
+
+LAllocation LIRGeneratorARM::useByteOpRegister(MDefinition* mir) {
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorARM::useByteOpRegisterAtStart(MDefinition* mir) {
+ return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorARM::useByteOpRegisterOrNonDoubleConstant(
+ MDefinition* mir) {
+ return useRegisterOrNonDoubleConstant(mir);
+}
+
+LDefinition LIRGeneratorARM::tempByteOpRegister() { return temp(); }
+
+void LIRGenerator::visitBox(MBox* box) {
+ MDefinition* inner = box->getOperand(0);
+
+ // If the box wrapped a double, it needs a new register.
+ if (IsFloatingPointType(inner->type())) {
+ defineBox(new (alloc()) LBoxFloatingPoint(
+ useRegisterAtStart(inner), tempCopy(inner, 0), inner->type()),
+ box);
+ return;
+ }
+
+ if (box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (inner->isConstant()) {
+ defineBox(new (alloc()) LValue(inner->toConstant()->toJSValue()), box);
+ return;
+ }
+
+ LBox* lir = new (alloc()) LBox(use(inner), inner->type());
+
+ // Otherwise, we should not define a new register for the payload portion
+ // of the output, so bypass defineBox().
+ uint32_t vreg = getVirtualRegister();
+
+ // Note that because we're using BogusTemp(), we do not change the type of
+ // the definition. We also do not define the first output as "TYPE",
+ // because it has no corresponding payload at (vreg + 1). Also note that
+ // although we copy the input's original type for the payload half of the
+ // definition, this is only for clarity. BogusTemp() definitions are
+ // ignored.
+ lir->setDef(0, LDefinition(vreg, LDefinition::GENERAL));
+ lir->setDef(1, LDefinition::BogusTemp());
+ box->setVirtualRegister(vreg);
+ add(lir);
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+ MDefinition* inner = unbox->getOperand(0);
+
+ // An unbox on arm reads in a type tag (either in memory or a register) and
+ // a payload. Unlike most instructions consuming a box, we ask for the type
+ // second, so that the result can re-use the first input.
+ MOZ_ASSERT(inner->type() == MIRType::Value);
+
+ ensureDefined(inner);
+
+ if (IsFloatingPointType(unbox->type())) {
+ LUnboxFloatingPoint* lir =
+ new (alloc()) LUnboxFloatingPoint(useBox(inner), unbox->type());
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+ define(lir, unbox);
+ return;
+ }
+
+ // Swap the order we use the box pieces so we can re-use the payload register.
+ LUnbox* lir = new (alloc()) LUnbox;
+ lir->setOperand(0, usePayloadInRegisterAtStart(inner));
+ lir->setOperand(1, useType(inner, LUse::REGISTER));
+
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+
+ // Types and payloads form two separate intervals. If the type becomes dead
+ // before the payload, it could be used as a Value without the type being
+ // recoverable. Unbox's purpose is to eagerly kill the definition of a type
+ // tag, so keeping both alive (for the purpose of gcmaps) is unappealing.
+ // Instead, we create a new virtual register.
+ defineReuseInput(lir, unbox, 0);
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new (alloc()) LReturn(isGenerator);
+ ins->setOperand(0, LUse(JSReturnReg_Type));
+ ins->setOperand(1, LUse(JSReturnReg_Data));
+ fillBoxUses(ins, 0, opd);
+ add(ins);
+}
+
+void LIRGeneratorARM::defineInt64Phi(MPhi* phi, size_t lirIndex) {
+ LPhi* low = current->getPhi(lirIndex + INT64LOW_INDEX);
+ LPhi* high = current->getPhi(lirIndex + INT64HIGH_INDEX);
+
+ uint32_t lowVreg = getVirtualRegister();
+
+ phi->setVirtualRegister(lowVreg);
+
+ uint32_t highVreg = getVirtualRegister();
+ MOZ_ASSERT(lowVreg + INT64HIGH_INDEX == highVreg + INT64LOW_INDEX);
+
+ low->setDef(0, LDefinition(lowVreg, LDefinition::INT32));
+ high->setDef(0, LDefinition(highVreg, LDefinition::INT32));
+ annotate(high);
+ annotate(low);
+}
+
+void LIRGeneratorARM::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ MDefinition* operand = phi->getOperand(inputPosition);
+ LPhi* low = block->getPhi(lirIndex + INT64LOW_INDEX);
+ LPhi* high = block->getPhi(lirIndex + INT64HIGH_INDEX);
+ low->setOperand(inputPosition,
+ LUse(operand->virtualRegister() + INT64LOW_INDEX, LUse::ANY));
+ high->setOperand(
+ inputPosition,
+ LUse(operand->virtualRegister() + INT64HIGH_INDEX, LUse::ANY));
+}
+
+// x = !y
+void LIRGeneratorARM::lowerForALU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(
+ 0, ins->snapshot() ? useRegister(input) : useRegisterAtStart(input));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+// z = x+y
+void LIRGeneratorARM::lowerForALU(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ // Some operations depend on checking inputs after writing the result, e.g.
+ // MulI, but only for bail out paths so useAtStart when no bailouts.
+ ins->setOperand(0,
+ ins->snapshot() ? useRegister(lhs) : useRegisterAtStart(lhs));
+ ins->setOperand(1, ins->snapshot() ? useRegisterOrConstant(rhs)
+ : useRegisterOrConstantAtStart(rhs));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+void LIRGeneratorARM::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+ MDefinition* input) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorARM::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, useInt64OrConstant(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorARM::lowerForMulInt64(LMulI64* ins, MMul* mir,
+ MDefinition* lhs, MDefinition* rhs) {
+ bool needsTemp = true;
+
+ if (rhs->isConstant()) {
+ int64_t constant = rhs->toConstant()->toInt64();
+ int32_t shift = mozilla::FloorLog2(constant);
+ // See special cases in CodeGeneratorARM::visitMulI64
+ if (constant >= -1 && constant <= 2) {
+ needsTemp = false;
+ }
+ if (constant > 0 && int64_t(1) << shift == constant) {
+ needsTemp = false;
+ }
+ }
+
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, useInt64OrConstant(rhs));
+ if (needsTemp) {
+ ins->setTemp(0, temp());
+ }
+
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorARM::lowerForCompareI64AndBranch(MTest* mir, MCompare* comp,
+ JSOp op, MDefinition* left,
+ MDefinition* right,
+ MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ LCompareI64AndBranch* lir = new (alloc())
+ LCompareI64AndBranch(comp, op, useInt64Register(left),
+ useInt64OrConstant(right), ifTrue, ifFalse);
+ add(lir, mir);
+}
+
+void LIRGeneratorARM::lowerForFPU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(0, useRegisterAtStart(input));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template <size_t Temps>
+void LIRGeneratorARM::lowerForFPU(LInstructionHelper<1, 2, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegisterAtStart(lhs));
+ ins->setOperand(1, useRegisterAtStart(rhs));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template void LIRGeneratorARM::lowerForFPU(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+template void LIRGeneratorARM::lowerForFPU(LInstructionHelper<1, 2, 1>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+
+void LIRGeneratorARM::lowerForBitAndAndBranch(LBitAndAndBranch* baab,
+ MInstruction* mir,
+ MDefinition* lhs,
+ MDefinition* rhs) {
+ baab->setOperand(0, useRegisterAtStart(lhs));
+ baab->setOperand(1, useRegisterOrConstantAtStart(rhs));
+ add(baab, mir);
+}
+
+void LIRGeneratorARM::lowerWasmBuiltinTruncateToInt32(
+ MWasmBuiltinTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ if (opd->type() == MIRType::Double) {
+ define(new (alloc()) LWasmBuiltinTruncateDToInt32(
+ useRegister(opd), useFixedAtStart(ins->instance(), InstanceReg),
+ LDefinition::BogusTemp()),
+ ins);
+ return;
+ }
+
+ define(new (alloc()) LWasmBuiltinTruncateFToInt32(
+ useRegister(opd), useFixedAtStart(ins->instance(), InstanceReg),
+ LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGeneratorARM::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ MDefinition* operand = phi->getOperand(inputPosition);
+ LPhi* type = block->getPhi(lirIndex + VREG_TYPE_OFFSET);
+ LPhi* payload = block->getPhi(lirIndex + VREG_DATA_OFFSET);
+ type->setOperand(
+ inputPosition,
+ LUse(operand->virtualRegister() + VREG_TYPE_OFFSET, LUse::ANY));
+ payload->setOperand(inputPosition,
+ LUse(VirtualRegisterOfPayload(operand), LUse::ANY));
+}
+
+void LIRGeneratorARM::lowerForShift(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegisterOrConstant(rhs));
+ define(ins, mir);
+}
+
+template <size_t Temps>
+void LIRGeneratorARM::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ if (mir->isRotate() && !rhs->isConstant()) {
+ ins->setTemp(0, temp());
+ }
+
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setOperand(INT64_PIECES, useRegisterOrConstant(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+template void LIRGeneratorARM::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+template void LIRGeneratorARM::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 1>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+void LIRGeneratorARM::lowerDivI(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDiv(div);
+ return;
+ }
+
+ // Division instructions are slow. Division by constant denominators can be
+ // rewritten to use other instructions.
+ if (div->rhs()->isConstant()) {
+ int32_t rhs = div->rhs()->toConstant()->toInt32();
+ // Check for division by a positive power of two, which is an easy and
+ // important case to optimize. Note that other optimizations are also
+ // possible; division by negative powers of two can be optimized in a
+ // similar manner as positive powers of two, and division by other
+ // constants can be optimized by a reciprocal multiplication technique.
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LDivPowTwoI* lir =
+ new (alloc()) LDivPowTwoI(useRegisterAtStart(div->lhs()), shift);
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+ }
+
+ if (HasIDIV()) {
+ LDivI* lir = new (alloc())
+ LDivI(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+
+ LSoftDivI* lir = new (alloc()) LSoftDivI(useFixedAtStart(div->lhs(), r0),
+ useFixedAtStart(div->rhs(), r1));
+
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+
+ defineReturn(lir, div);
+}
+
+void LIRGeneratorARM::lowerNegI(MInstruction* ins, MDefinition* input) {
+ define(new (alloc()) LNegI(useRegisterAtStart(input)), ins);
+}
+
+void LIRGeneratorARM::lowerNegI64(MInstruction* ins, MDefinition* input) {
+ // Reuse the input. Define + use-at-start would create risk that the output
+ // uses the same register pair as the input but in reverse order. Reusing
+ // probably has less spilling than the alternative, define + use.
+ defineInt64ReuseInput(new (alloc()) LNegI64(useInt64RegisterAtStart(input)),
+ ins, 0);
+}
+
+void LIRGenerator::visitAbs(MAbs* ins) {
+ define(allocateAbs(ins, useRegisterAtStart(ins->input())), ins);
+}
+
+void LIRGeneratorARM::lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs) {
+ LMulI* lir = new (alloc()) LMulI;
+ if (mul->fallible()) {
+ assignSnapshot(lir, mul->bailoutKind());
+ }
+ lowerForALU(lir, mul, lhs, rhs);
+}
+
+void LIRGeneratorARM::lowerModI(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUMod(mod);
+ return;
+ }
+
+ if (mod->rhs()->isConstant()) {
+ int32_t rhs = mod->rhs()->toConstant()->toInt32();
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LModPowTwoI* lir =
+ new (alloc()) LModPowTwoI(useRegister(mod->lhs()), shift);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ }
+ if (shift < 31 && (1 << (shift + 1)) - 1 == rhs) {
+ MOZ_ASSERT(rhs);
+ LModMaskI* lir = new (alloc())
+ LModMaskI(useRegister(mod->lhs()), temp(), temp(), shift + 1);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ }
+ }
+
+ if (HasIDIV()) {
+ LModI* lir =
+ new (alloc()) LModI(useRegister(mod->lhs()), useRegister(mod->rhs()));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ }
+
+ LSoftModI* lir =
+ new (alloc()) LSoftModI(useFixedAtStart(mod->lhs(), r0),
+ useFixedAtStart(mod->rhs(), r1), tempFixed(r2));
+
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+
+ defineReturn(lir, mod);
+}
+
+void LIRGeneratorARM::lowerDivI64(MDiv* div) {
+ MOZ_CRASH("We use MWasmBuiltinDivI64 instead.");
+}
+
+void LIRGeneratorARM::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ if (div->isUnsigned()) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useInt64RegisterAtStart(div->lhs()),
+ useInt64RegisterAtStart(div->rhs()),
+ useFixedAtStart(div->instance(), InstanceReg));
+ defineReturn(lir, div);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+ useInt64RegisterAtStart(div->lhs()), useInt64RegisterAtStart(div->rhs()),
+ useFixedAtStart(div->instance(), InstanceReg));
+ defineReturn(lir, div);
+}
+
+void LIRGeneratorARM::lowerModI64(MMod* mod) {
+ MOZ_CRASH("We use MWasmBuiltinModI64 instead.");
+}
+
+void LIRGeneratorARM::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ if (mod->isUnsigned()) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useInt64RegisterAtStart(mod->lhs()),
+ useInt64RegisterAtStart(mod->rhs()),
+ useFixedAtStart(mod->instance(), InstanceReg));
+ defineReturn(lir, mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+ useInt64RegisterAtStart(mod->lhs()), useInt64RegisterAtStart(mod->rhs()),
+ useFixedAtStart(mod->instance(), InstanceReg));
+ defineReturn(lir, mod);
+}
+
+void LIRGeneratorARM::lowerUDivI64(MDiv* div) {
+ MOZ_CRASH("We use MWasmBuiltinDivI64 instead.");
+}
+
+void LIRGeneratorARM::lowerUModI64(MMod* mod) {
+ MOZ_CRASH("We use MWasmBuiltinModI64 instead.");
+}
+
+void LIRGenerator::visitPowHalf(MPowHalf* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Double);
+ LPowHalfD* lir = new (alloc()) LPowHalfD(useRegisterAtStart(input));
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGeneratorARM::lowerWasmSelectI(MWasmSelect* select) {
+ auto* lir = new (alloc())
+ LWasmSelect(useRegisterAtStart(select->trueExpr()),
+ useAny(select->falseExpr()), useRegister(select->condExpr()));
+ defineReuseInput(lir, select, LWasmSelect::TrueExprIndex);
+}
+
+void LIRGeneratorARM::lowerWasmSelectI64(MWasmSelect* select) {
+ auto* lir = new (alloc()) LWasmSelectI64(
+ useInt64RegisterAtStart(select->trueExpr()),
+ useInt64(select->falseExpr()), useRegister(select->condExpr()));
+ defineInt64ReuseInput(lir, select, LWasmSelectI64::TrueExprIndex);
+}
+
+LTableSwitch* LIRGeneratorARM::newLTableSwitch(const LAllocation& in,
+ const LDefinition& inputCopy,
+ MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitch(in, inputCopy, tableswitch);
+}
+
+LTableSwitchV* LIRGeneratorARM::newLTableSwitchV(MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitchV(useBox(tableswitch->getOperand(0)), temp(),
+ tempDouble(), tableswitch);
+}
+
+void LIRGeneratorARM::lowerUrshD(MUrsh* mir) {
+ MDefinition* lhs = mir->lhs();
+ MDefinition* rhs = mir->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+
+ LUrshD* lir = new (alloc())
+ LUrshD(useRegister(lhs), useRegisterOrConstant(rhs), temp());
+ define(lir, mir);
+}
+
+void LIRGeneratorARM::lowerPowOfTwoI(MPow* mir) {
+ int32_t base = mir->input()->toConstant()->toInt32();
+ MDefinition* power = mir->power();
+
+ auto* lir = new (alloc()) LPowOfTwoI(useRegister(power), base);
+ assignSnapshot(lir, mir->bailoutKind());
+ define(lir, mir);
+}
+
+void LIRGeneratorARM::lowerBigIntLsh(MBigIntLsh* ins) {
+ auto* lir = new (alloc()) LBigIntLsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorARM::lowerBigIntRsh(MBigIntRsh* ins) {
+ auto* lir = new (alloc()) LBigIntRsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorARM::lowerBigIntDiv(MBigIntDiv* ins) {
+ LDefinition temp1, temp2;
+ if (HasIDIV()) {
+ temp1 = temp();
+ temp2 = temp();
+ } else {
+ temp1 = tempFixed(r0);
+ temp2 = tempFixed(r1);
+ }
+ auto* lir = new (alloc()) LBigIntDiv(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorARM::lowerBigIntMod(MBigIntMod* ins) {
+ LDefinition temp1, temp2;
+ if (HasIDIV()) {
+ temp1 = temp();
+ temp2 = temp();
+ } else {
+ temp1 = tempFixed(r0);
+ temp2 = tempFixed(r1);
+ }
+ auto* lir = new (alloc()) LBigIntMod(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmNeg(MWasmNeg* ins) {
+ if (ins->type() == MIRType::Int32) {
+ define(new (alloc()) LNegI(useRegisterAtStart(ins->input())), ins);
+ } else if (ins->type() == MIRType::Float32) {
+ define(new (alloc()) LNegF(useRegisterAtStart(ins->input())), ins);
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ define(new (alloc()) LNegD(useRegisterAtStart(ins->input())), ins);
+ }
+}
+
+void LIRGeneratorARM::lowerUDiv(MDiv* div) {
+ MDefinition* lhs = div->getOperand(0);
+ MDefinition* rhs = div->getOperand(1);
+
+ if (HasIDIV()) {
+ LUDiv* lir = new (alloc()) LUDiv;
+ lir->setOperand(0, useRegister(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+
+ LSoftUDivOrMod* lir = new (alloc())
+ LSoftUDivOrMod(useFixedAtStart(lhs, r0), useFixedAtStart(rhs, r1));
+
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+
+ defineReturn(lir, div);
+}
+
+void LIRGeneratorARM::lowerUMod(MMod* mod) {
+ MDefinition* lhs = mod->getOperand(0);
+ MDefinition* rhs = mod->getOperand(1);
+
+ if (HasIDIV()) {
+ LUMod* lir = new (alloc()) LUMod;
+ lir->setOperand(0, useRegister(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ }
+
+ LSoftUDivOrMod* lir = new (alloc())
+ LSoftUDivOrMod(useFixedAtStart(lhs, r0), useFixedAtStart(rhs, r1));
+
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+
+ defineReturn(lir, mod);
+}
+
+void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToDouble* lir =
+ new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToFloat32* lir =
+ new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) {
+ auto* lir = new (alloc()) LWasmHeapBase(LAllocation());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ if (ins->access().type() == Scalar::Int64 && ins->access().isAtomic()) {
+ auto* lir = new (alloc()) LWasmAtomicLoadI64(useRegisterAtStart(base));
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(IntArgReg1)),
+ LAllocation(AnyRegister(IntArgReg0))));
+ return;
+ }
+
+ LAllocation ptr = useRegisterAtStart(base);
+
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new (alloc()) LWasmLoadI64(ptr);
+ if (ins->access().offset() || ins->access().type() == Scalar::Int64) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+ defineInt64(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LWasmLoad(ptr);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmStore(MWasmStore* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ if (ins->access().type() == Scalar::Int64 && ins->access().isAtomic()) {
+ auto* lir = new (alloc()) LWasmAtomicStoreI64(
+ useRegister(base),
+ useInt64Fixed(ins->value(), Register64(IntArgReg1, IntArgReg0)),
+ tempFixed(IntArgReg2), tempFixed(IntArgReg3));
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation ptr = useRegisterAtStart(base);
+
+ if (ins->value()->type() == MIRType::Int64) {
+ LInt64Allocation value = useInt64RegisterAtStart(ins->value());
+ auto* lir = new (alloc()) LWasmStoreI64(ptr, value);
+ if (ins->access().offset() || ins->access().type() == Scalar::Int64) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation value = useRegisterAtStart(ins->value());
+ auto* lir = new (alloc()) LWasmStore(ptr, value);
+
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ add(lir, ins);
+}
+
+void LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ // For the ARM it is best to keep the 'base' in a register if a bounds check
+ // is needed.
+ LAllocation baseAlloc;
+ LAllocation limitAlloc;
+
+ if (base->isConstant() && !ins->needsBoundsCheck()) {
+ // A bounds check is only skipped for a positive index.
+ MOZ_ASSERT(base->toConstant()->toInt32() >= 0);
+ baseAlloc = LAllocation(base->toConstant());
+ } else {
+ baseAlloc = useRegisterAtStart(base);
+ if (ins->needsBoundsCheck()) {
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT(boundsCheckLimit->type() == MIRType::Int32);
+ limitAlloc = useRegisterAtStart(boundsCheckLimit);
+ }
+ }
+
+ define(new (alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc, LAllocation()),
+ ins);
+}
+
+void LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ LAllocation baseAlloc;
+ LAllocation limitAlloc;
+
+ if (base->isConstant() && !ins->needsBoundsCheck()) {
+ MOZ_ASSERT(base->toConstant()->toInt32() >= 0);
+ baseAlloc = LAllocation(base->toConstant());
+ } else {
+ baseAlloc = useRegisterAtStart(base);
+ if (ins->needsBoundsCheck()) {
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT(boundsCheckLimit->type() == MIRType::Int32);
+ limitAlloc = useRegisterAtStart(boundsCheckLimit);
+ }
+ }
+
+ add(new (alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()),
+ limitAlloc, LAllocation()),
+ ins);
+}
+
+void LIRGeneratorARM::lowerTruncateDToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double);
+
+ define(new (alloc())
+ LTruncateDToInt32(useRegister(opd), LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGeneratorARM::lowerTruncateFToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Float32);
+
+ define(new (alloc())
+ LTruncateFToInt32(useRegister(opd), LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(HasLDSTREXBHD());
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ // The two register pairs must be distinct.
+ LInt64Definition temp1 = tempInt64Fixed(Register64(IntArgReg3, IntArgReg2));
+ LDefinition temp2 = tempFixed(IntArgReg1);
+
+ auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64(
+ elements, index, value, temp1, temp2);
+ defineFixed(lir, ins, LAllocation(AnyRegister(IntArgReg0)));
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32);
+
+ // If the target is a floating register then we need a temp at the
+ // CodeGenerator level for creating the result.
+
+ LDefinition tempDef = LDefinition::BogusTemp();
+ if (ins->arrayType() == Scalar::Uint32) {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ tempDef = temp();
+ }
+
+ LAtomicExchangeTypedArrayElement* lir = new (alloc())
+ LAtomicExchangeTypedArrayElement(elements, index, value, tempDef);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ // Wasm additionally pins the value register to `FetchOpVal64`, but it's
+ // unclear why this was deemed necessary.
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64Fixed(FetchOpTmp64);
+
+ if (ins->isForEffect()) {
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+ elements, index, value, temp1, temp2);
+ add(lir, ins);
+ return;
+ }
+
+ LInt64Definition temp3 = tempInt64Fixed(FetchOpOut64);
+
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinop64(
+ elements, index, value, temp1, temp2, temp3);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ if (ins->isForEffect()) {
+ LAtomicTypedArrayElementBinopForEffect* lir = new (alloc())
+ LAtomicTypedArrayElementBinopForEffect(elements, index, value,
+ /* flagTemp= */ temp());
+ add(lir, ins);
+ return;
+ }
+
+ // For a Uint32Array with a known double result we need a temp for
+ // the intermediate output.
+ //
+ // Optimization opportunity (bug 1077317): We can do better by
+ // allowing 'value' to remain as an imm32 if it is small enough to
+ // fit in an instruction.
+
+ LDefinition flagTemp = temp();
+ LDefinition outTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ outTemp = temp();
+ }
+
+ // On arm, map flagTemp to temp1 and outTemp to temp2, at least for now.
+
+ LAtomicTypedArrayElementBinop* lir = new (alloc())
+ LAtomicTypedArrayElementBinop(elements, index, value, flagTemp, outTemp);
+ define(lir, ins);
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation newval = useRegister(ins->newval());
+ const LAllocation oldval = useRegister(ins->oldval());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ // The three register pairs must be distinct.
+ LInt64Definition temp1 = tempInt64Fixed(CmpXchgOld64);
+ LInt64Definition temp2 = tempInt64Fixed(CmpXchgNew64);
+ LInt64Definition temp3 = tempInt64Fixed(CmpXchgOut64);
+
+ auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+ elements, index, oldval, newval, temp1, temp2, temp3);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ // If the target is a floating register then we need a temp at the
+ // CodeGenerator level for creating the result.
+ //
+ // Optimization opportunity (bug 1077317): We could do better by
+ // allowing oldval to remain an immediate, if it is small enough
+ // to fit in an instruction.
+
+ LDefinition tempDef = LDefinition::BogusTemp();
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ tempDef = temp();
+ }
+
+ LCompareExchangeTypedArrayElement* lir =
+ new (alloc()) LCompareExchangeTypedArrayElement(elements, index, oldval,
+ newval, tempDef);
+
+ define(lir, ins);
+}
+
+void LIRGeneratorARM::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+ auto* lir = new (alloc())
+ LAtomicLoad64(elements, index, temp(),
+ tempInt64Fixed(Register64(IntArgReg1, IntArgReg0)));
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorARM::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value = useRegister(ins->value());
+ LInt64Definition temp1 = tempInt64Fixed(Register64(IntArgReg1, IntArgReg0));
+ LInt64Definition temp2 = tempInt64Fixed(Register64(IntArgReg3, IntArgReg2));
+
+ add(new (alloc()) LAtomicStore64(elements, index, value, temp1, temp2), ins);
+}
+
+void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ if (ins->access().type() == Scalar::Int64) {
+ // The three register pairs must be distinct.
+ auto* lir = new (alloc()) LWasmCompareExchangeI64(
+ useRegister(base), useInt64Fixed(ins->oldValue(), CmpXchgOld64),
+ useInt64Fixed(ins->newValue(), CmpXchgNew64));
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(CmpXchgOutHi)),
+ LAllocation(AnyRegister(CmpXchgOutLo))));
+ return;
+ }
+
+ MOZ_ASSERT(ins->access().type() < Scalar::Float32);
+ MOZ_ASSERT(HasLDSTREXBHD(), "by HasPlatformSupport() constraints");
+
+ LWasmCompareExchangeHeap* lir = new (alloc())
+ LWasmCompareExchangeHeap(useRegister(base), useRegister(ins->oldValue()),
+ useRegister(ins->newValue()));
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
+ MOZ_ASSERT(ins->base()->type() == MIRType::Int32);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmAtomicExchangeI64(
+ useRegister(ins->base()), useInt64Fixed(ins->value(), XchgNew64),
+ ins->access());
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(XchgOutHi)),
+ LAllocation(AnyRegister(XchgOutLo))));
+ return;
+ }
+
+ MOZ_ASSERT(ins->access().type() < Scalar::Float32);
+ MOZ_ASSERT(HasLDSTREXBHD(), "by HasPlatformSupport() constraints");
+
+ const LAllocation base = useRegister(ins->base());
+ const LAllocation value = useRegister(ins->value());
+ define(new (alloc()) LWasmAtomicExchangeHeap(base, value), ins);
+}
+
+void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmAtomicBinopI64(
+ useRegister(ins->base()), useInt64Fixed(ins->value(), FetchOpVal64),
+ tempFixed(FetchOpTmpLo), tempFixed(FetchOpTmpHi), ins->access(),
+ ins->operation());
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(FetchOpOutHi)),
+ LAllocation(AnyRegister(FetchOpOutLo))));
+ return;
+ }
+
+ MOZ_ASSERT(ins->access().type() < Scalar::Float32);
+ MOZ_ASSERT(HasLDSTREXBHD(), "by HasPlatformSupport() constraints");
+
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ if (!ins->hasUses()) {
+ LWasmAtomicBinopHeapForEffect* lir =
+ new (alloc()) LWasmAtomicBinopHeapForEffect(useRegister(base),
+ useRegister(ins->value()),
+ /* flagTemp= */ temp());
+ add(lir, ins);
+ return;
+ }
+
+ LWasmAtomicBinopHeap* lir = new (alloc())
+ LWasmAtomicBinopHeap(useRegister(base), useRegister(ins->value()),
+ /* temp = */ LDefinition::BogusTemp(),
+ /* flagTemp= */ temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitSubstr(MSubstr* ins) {
+ LSubstr* lir = new (alloc())
+ LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
+ useRegister(ins->length()), temp(), temp(), tempByteOpRegister());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use MWasmTruncateToInt64 for arm");
+}
+
+void LIRGeneratorARM::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MDefinition* opd = ins->input();
+ MDefinition* instance = ins->instance();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ defineReturn(new (alloc())
+ LWasmTruncateToInt64(useRegisterAtStart(opd),
+ useFixedAtStart(instance, InstanceReg)),
+ ins);
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+ MOZ_CRASH("We use BuiltinInt64ToFloatingPoint instead.");
+}
+
+void LIRGeneratorARM::lowerBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_ASSERT(ins->type() == MIRType::Double || ins->type() == MIRType::Float32);
+
+ auto* lir = new (alloc())
+ LInt64ToFloatingPointCall(useInt64RegisterAtStart(ins->input()),
+ useFixedAtStart(ins->instance(), InstanceReg));
+ defineReturn(lir, ins);
+}
+
+void LIRGenerator::visitCopySign(MCopySign* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(IsFloatingPointType(lhs->type()));
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(lhs->type() == ins->type());
+
+ LInstructionHelper<1, 2, 2>* lir;
+ if (lhs->type() == MIRType::Double) {
+ lir = new (alloc()) LCopySignD();
+ } else {
+ lir = new (alloc()) LCopySignF();
+ }
+
+ lir->setTemp(0, temp());
+ lir->setTemp(1, temp());
+
+ lowerForFPU(lir, ins, lhs, rhs);
+}
+
+void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
+ auto* lir =
+ new (alloc()) LExtendInt32ToInt64(useRegisterAtStart(ins->input()));
+ defineInt64(lir, ins);
+
+ LDefinition def(LDefinition::GENERAL, LDefinition::MUST_REUSE_INPUT);
+ def.setReusedInput(0);
+ def.setVirtualRegister(ins->virtualRegister());
+
+ lir->setDef(0, def);
+}
+
+void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
+ defineInt64(new (alloc())
+ LSignExtendInt64(useInt64RegisterAtStart(ins->input())),
+ ins);
+}
+
+// On arm we specialize the only cases where compare is {U,}Int32 and select
+// is {U,}Int32.
+bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
+ MCompare::CompareType compTy, MIRType insTy) {
+ return insTy == MIRType::Int32 && (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32);
+}
+
+void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
+ MDefinition* lhs,
+ MDefinition* rhs,
+ MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+ auto* lir = new (alloc()) LWasmCompareAndSelect(
+ useRegister(lhs), useRegister(rhs), compTy, jsop,
+ useRegisterAtStart(ins->trueExpr()), useRegister(ins->falseExpr()));
+ defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
+}
+
+void LIRGenerator::visitWasmTernarySimd128(MWasmTernarySimd128* ins) {
+ MOZ_CRASH("ternary SIMD NYI");
+}
+
+void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) {
+ MOZ_CRASH("binary SIMD NYI");
+}
+
+#ifdef ENABLE_WASM_SIMD
+bool MWasmTernarySimd128::specializeBitselectConstantMaskAsShuffle(
+ int8_t shuffle[16]) {
+ return false;
+}
+bool MWasmTernarySimd128::canRelaxBitselect() { return false; }
+
+bool MWasmBinarySimd128::canPmaddubsw() { return false; }
+#endif
+
+bool MWasmBinarySimd128::specializeForConstantRhs() {
+ // Probably many we want to do here
+ return false;
+}
+
+void LIRGenerator::visitWasmBinarySimd128WithConstant(
+ MWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("binary SIMD with constant NYI");
+}
+
+void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) {
+ MOZ_CRASH("shift SIMD NYI");
+}
+
+void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) {
+ MOZ_CRASH("shuffle SIMD NYI");
+}
+
+void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH("replace-lane SIMD NYI");
+}
+
+void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) {
+ MOZ_CRASH("scalar-to-SIMD NYI");
+}
+
+void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) {
+ MOZ_CRASH("unary SIMD NYI");
+}
+
+void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) {
+ MOZ_CRASH("reduce-SIMD NYI");
+}
+
+void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH("load-lane SIMD NYI");
+}
+
+void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH("store-lane SIMD NYI");
+}
diff --git a/js/src/jit/arm/Lowering-arm.h b/js/src/jit/arm/Lowering-arm.h
new file mode 100644
index 0000000000..3f03d22941
--- /dev/null
+++ b/js/src/jit/arm/Lowering-arm.h
@@ -0,0 +1,118 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_Lowering_arm_h
+#define jit_arm_Lowering_arm_h
+
+#include "jit/shared/Lowering-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorARM : public LIRGeneratorShared {
+ protected:
+ LIRGeneratorARM(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorShared(gen, graph, lirGraph) {}
+
+ // Returns a box allocation with type set to reg1 and payload set to reg2.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2,
+ bool useAtStart = false);
+
+ // x86 has constraints on what registers can be formatted for 1-byte
+ // stores and loads; on ARM all registers are okay.
+ LAllocation useByteOpRegister(MDefinition* mir);
+ LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+ LDefinition tempByteOpRegister();
+
+ inline LDefinition tempToUnbox() { return LDefinition::BogusTemp(); }
+
+ bool needTempForPostBarrier() { return false; }
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void defineInt64Phi(MPhi* phi, size_t lirIndex);
+
+ void lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerUrshD(MUrsh* mir);
+
+ void lowerPowOfTwoI(MPow* mir);
+
+ void lowerForALU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* input);
+ void lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* input);
+ void lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+ void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs,
+ MDefinition* rhs);
+ template <size_t Temps>
+ void lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForCompareI64AndBranch(MTest* mir, MCompare* comp, JSOp op,
+ MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse);
+
+ void lowerForFPU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* src);
+ template <size_t Temps>
+ void lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+ void lowerForBitAndAndBranch(LBitAndAndBranch* baab, MInstruction* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins);
+ void lowerTruncateDToInt32(MTruncateToInt32* ins);
+ void lowerTruncateFToInt32(MTruncateToInt32* ins);
+ void lowerDivI(MDiv* div);
+ void lowerModI(MMod* mod);
+ void lowerDivI64(MDiv* div);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+ void lowerModI64(MMod* mod);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+ void lowerUDivI64(MDiv* div);
+ void lowerUModI64(MMod* mod);
+ void lowerNegI(MInstruction* ins, MDefinition* input);
+ void lowerNegI64(MInstruction* ins, MDefinition* input);
+ void lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs);
+ void lowerUDiv(MDiv* div);
+ void lowerUMod(MMod* mod);
+ void lowerWasmSelectI(MWasmSelect* select);
+ void lowerWasmSelectI64(MWasmSelect* select);
+
+ void lowerBigIntLsh(MBigIntLsh* ins);
+ void lowerBigIntRsh(MBigIntRsh* ins);
+ void lowerBigIntDiv(MBigIntDiv* ins);
+ void lowerBigIntMod(MBigIntMod* ins);
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+ void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+
+ LTableSwitch* newLTableSwitch(const LAllocation& in,
+ const LDefinition& inputCopy,
+ MTableSwitch* ins);
+ LTableSwitchV* newLTableSwitchV(MTableSwitch* ins);
+
+ void lowerPhi(MPhi* phi);
+};
+
+typedef LIRGeneratorARM LIRGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_Lowering_arm_h */
diff --git a/js/src/jit/arm/MacroAssembler-arm-inl.h b/js/src/jit/arm/MacroAssembler-arm-inl.h
new file mode 100644
index 0000000000..94d323207e
--- /dev/null
+++ b/js/src/jit/arm/MacroAssembler-arm-inl.h
@@ -0,0 +1,2582 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_MacroAssembler_arm_inl_h
+#define jit_arm_MacroAssembler_arm_inl_h
+
+#include "jit/arm/MacroAssembler-arm.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+ move32(src.low, dest.low);
+ move32(src.high, dest.high);
+}
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+ move32(Imm32(imm.value & 0xFFFFFFFFL), dest.low);
+ move32(Imm32((imm.value >> 32) & 0xFFFFFFFFL), dest.high);
+}
+
+void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
+ ma_vxfer(src, dest);
+}
+
+void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
+ ma_vxfer(src, dest);
+}
+
+void MacroAssembler::move8SignExtend(Register src, Register dest) {
+ as_sxtb(dest, src, 0);
+}
+
+void MacroAssembler::move16SignExtend(Register src, Register dest) {
+ as_sxth(dest, src, 0);
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ ma_vxfer(src, dest.low, dest.high);
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ ma_vxfer(src.low, src.high, dest);
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+ if (src.low != dest) {
+ move32(src.low, dest);
+ }
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ if (src != dest.low) {
+ move32(src, dest.low);
+ }
+ move32(Imm32(0), dest.high);
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ as_sxtb(dest.low, src, 0);
+ ma_asr(Imm32(31), dest.low, dest.high);
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ as_sxth(dest.low, src, 0);
+ ma_asr(Imm32(31), dest.low, dest.high);
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ if (src != dest.low) {
+ move32(src, dest.low);
+ }
+ ma_asr(Imm32(31), dest.low, dest.high);
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ move32(src, dest);
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ move32(src, dest);
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ load32(src, dest);
+}
+
+void MacroAssembler::loadAbiReturnAddress(Register dest) { movePtr(lr, dest); }
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::not32(Register reg) { ma_mvn(reg, reg); }
+
+void MacroAssembler::notPtr(Register reg) { ma_mvn(reg, reg); }
+
+void MacroAssembler::and32(Register src, Register dest) {
+ ma_and(src, dest, SetCC);
+}
+
+void MacroAssembler::and32(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_and(imm, dest, scratch, SetCC);
+}
+
+void MacroAssembler::and32(Imm32 imm, const Address& dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(dest, scratch, scratch2);
+ ma_and(imm, scratch, scratch2);
+ ma_str(scratch, dest, scratch2);
+}
+
+void MacroAssembler::and32(const Address& src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(src, scratch, scratch2);
+ ma_and(scratch, dest, SetCC);
+}
+
+void MacroAssembler::andPtr(Register src, Register dest) { ma_and(src, dest); }
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_and(imm, dest, scratch);
+}
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+ if (imm.low().value != int32_t(0xFFFFFFFF)) {
+ and32(imm.low(), dest.low);
+ }
+ if (imm.hi().value != int32_t(0xFFFFFFFF)) {
+ and32(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+ if (imm.low().value) {
+ or32(imm.low(), dest.low);
+ }
+ if (imm.hi().value) {
+ or32(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+ if (imm.low().value) {
+ xor32(imm.low(), dest.low);
+ }
+ if (imm.hi().value) {
+ xor32(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::or32(Register src, Register dest) { ma_orr(src, dest); }
+
+void MacroAssembler::or32(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_orr(imm, dest, scratch);
+}
+
+void MacroAssembler::or32(Imm32 imm, const Address& dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(dest, scratch, scratch2);
+ ma_orr(imm, scratch, scratch2);
+ ma_str(scratch, dest, scratch2);
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) { ma_orr(src, dest); }
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_orr(imm, dest, scratch);
+}
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+ and32(src.low, dest.low);
+ and32(src.high, dest.high);
+}
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+ or32(src.low, dest.low);
+ or32(src.high, dest.high);
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+ ma_eor(src.low, dest.low);
+ ma_eor(src.high, dest.high);
+}
+
+void MacroAssembler::xor32(Register src, Register dest) {
+ ma_eor(src, dest, SetCC);
+}
+
+void MacroAssembler::xor32(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_eor(imm, dest, scratch, SetCC);
+}
+
+void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(dest, scratch, scratch2);
+ ma_eor(imm, scratch, scratch2);
+ ma_str(scratch, dest, scratch2);
+}
+
+void MacroAssembler::xor32(const Address& src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(src, scratch, scratch2);
+ ma_eor(scratch, dest, SetCC);
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) { ma_eor(src, dest); }
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_eor(imm, dest, scratch);
+}
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap16SignExtend(Register reg) { as_revsh(reg, reg); }
+
+void MacroAssembler::byteSwap16ZeroExtend(Register reg) {
+ as_rev16(reg, reg);
+ as_uxth(reg, reg, 0);
+}
+
+void MacroAssembler::byteSwap32(Register reg) { as_rev(reg, reg); }
+
+void MacroAssembler::byteSwap64(Register64 reg) {
+ as_rev(reg.high, reg.high);
+ as_rev(reg.low, reg.low);
+
+ ScratchRegisterScope scratch(*this);
+ ma_mov(reg.high, scratch);
+ ma_mov(reg.low, reg.high);
+ ma_mov(scratch, reg.low);
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::add32(Register src, Register dest) {
+ ma_add(src, dest, SetCC);
+}
+
+void MacroAssembler::add32(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_add(imm, dest, scratch, SetCC);
+}
+
+void MacroAssembler::add32(Imm32 imm, const Address& dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(dest, scratch, scratch2);
+ ma_add(imm, scratch, scratch2, SetCC);
+ ma_str(scratch, dest, scratch2);
+}
+
+void MacroAssembler::addPtr(Register src, Register dest) { ma_add(src, dest); }
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_add(imm, dest, scratch);
+}
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+ addPtr(Imm32(imm.value), dest);
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(dest, scratch, scratch2);
+ ma_add(imm, scratch, scratch2);
+ ma_str(scratch, dest, scratch2);
+}
+
+void MacroAssembler::addPtr(const Address& src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(src, scratch, scratch2);
+ ma_add(scratch, dest, SetCC);
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+ ma_add(src.low, dest.low, SetCC);
+ ma_adc(src.high, dest.high);
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_add(imm, dest.low, scratch, SetCC);
+ as_adc(dest.high, dest.high, Imm8(0), LeaveCC);
+}
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_add(imm.low(), dest.low, scratch, SetCC);
+ ma_adc(imm.hi(), dest.high, scratch, LeaveCC);
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ ScratchRegisterScope scratch(*this);
+ CodeOffset offs = CodeOffset(currentOffset());
+ ma_movPatchable(Imm32(0), scratch, Always);
+ ma_sub(getStackPointer(), scratch, dest);
+ return offs;
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ ScratchRegisterScope scratch(*this);
+ BufferInstructionIterator iter(BufferOffset(offset.offset()), &m_buffer);
+ iter.maybeSkipAutomaticInstructions();
+ ma_mov_patch(imm, scratch, Always, HasMOVWT() ? L_MOVWT : L_LDR, iter);
+}
+
+void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
+ ma_vadd(dest, src, dest);
+}
+
+void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
+ ma_vadd_f32(dest, src, dest);
+}
+
+void MacroAssembler::sub32(Register src, Register dest) {
+ ma_sub(src, dest, SetCC);
+}
+
+void MacroAssembler::sub32(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_sub(imm, dest, scratch, SetCC);
+}
+
+void MacroAssembler::sub32(const Address& src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(src, scratch, scratch2);
+ ma_sub(scratch, dest, SetCC);
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) { ma_sub(src, dest); }
+
+void MacroAssembler::subPtr(Register src, const Address& dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(dest, scratch, scratch2);
+ ma_sub(src, scratch);
+ ma_str(scratch, dest, scratch2);
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_sub(imm, dest, scratch);
+}
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(addr, scratch, scratch2);
+ ma_sub(scratch, dest);
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+ ma_sub(src.low, dest.low, SetCC);
+ ma_sbc(src.high, dest.high, LeaveCC);
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_sub(imm.low(), dest.low, scratch, SetCC);
+ ma_sbc(imm.hi(), dest.high, scratch, LeaveCC);
+}
+
+void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
+ ma_vsub(dest, src, dest);
+}
+
+void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
+ ma_vsub_f32(dest, src, dest);
+}
+
+void MacroAssembler::mul32(Register rhs, Register srcDest) {
+ as_mul(srcDest, srcDest, rhs);
+}
+
+void MacroAssembler::mul32(Imm32 imm, Register srcDest) {
+ ScratchRegisterScope scratch(*this);
+ move32(imm, scratch);
+ mul32(scratch, srcDest);
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_umull(src, imm, dest, scratch, scratch);
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+ as_mul(srcDest, srcDest, rhs);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+ // LOW32 = LOW(LOW(dest) * LOW(imm));
+ // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
+ // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
+ // + HIGH(LOW(dest) * LOW(imm)) [carry]
+
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ // HIGH(dest) = LOW(HIGH(dest) * LOW(imm));
+ ma_mov(Imm32(imm.value & 0xFFFFFFFFL), scratch);
+ as_mul(dest.high, dest.high, scratch);
+
+ // high:low = LOW(dest) * LOW(imm);
+ as_umull(scratch2, scratch, dest.low, scratch);
+
+ // HIGH(dest) += high;
+ as_add(dest.high, dest.high, O2Reg(scratch2));
+
+ // HIGH(dest) += LOW(LOW(dest) * HIGH(imm));
+ if (((imm.value >> 32) & 0xFFFFFFFFL) == 5) {
+ as_add(scratch2, dest.low, lsl(dest.low, 2));
+ } else {
+ MOZ_CRASH("Not supported imm");
+ }
+ as_add(dest.high, dest.high, O2Reg(scratch2));
+
+ // LOW(dest) = low;
+ ma_mov(scratch, dest.low);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
+ const Register temp) {
+ // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
+ // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
+ // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
+ // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
+
+ MOZ_ASSERT(temp != dest.high && temp != dest.low);
+
+ // Compute mul64
+ ScratchRegisterScope scratch(*this);
+ ma_mul(dest.high, imm.low(), dest.high, scratch); // (2)
+ ma_mul(dest.low, imm.hi(), temp, scratch); // (3)
+ ma_add(dest.high, temp, temp);
+ ma_umull(dest.low, imm.low(), dest.high, dest.low, scratch); // (4) + (1)
+ ma_add(temp, dest.high, dest.high);
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
+ // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
+ // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
+ // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
+
+ MOZ_ASSERT(dest != src);
+ MOZ_ASSERT(dest.low != src.high && dest.high != src.low);
+
+ // Compute mul64
+ ma_mul(dest.high, src.low, dest.high); // (2)
+ ma_mul(src.high, dest.low, temp); // (3)
+ ma_add(dest.high, temp, temp);
+ ma_umull(dest.low, src.low, dest.high, dest.low); // (4) + (1)
+ ma_add(temp, dest.high, dest.high);
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+ as_add(dest, src, lsl(src, 1));
+}
+
+void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
+ ma_vmul_f32(dest, src, dest);
+}
+
+void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
+ ma_vmul(dest, src, dest);
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+ FloatRegister dest) {
+ ScratchRegisterScope scratch(*this);
+ ScratchDoubleScope scratchDouble(*this);
+
+ movePtr(imm, scratch);
+ ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), scratchDouble);
+ mulDouble(scratchDouble, dest);
+}
+
+void MacroAssembler::quotient32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ MOZ_ASSERT(HasIDIV());
+ if (isUnsigned) {
+ ma_udiv(srcDest, rhs, srcDest);
+ } else {
+ ma_sdiv(srcDest, rhs, srcDest);
+ }
+}
+
+void MacroAssembler::remainder32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ MOZ_ASSERT(HasIDIV());
+
+ ScratchRegisterScope scratch(*this);
+ if (isUnsigned) {
+ ma_umod(srcDest, rhs, srcDest, scratch);
+ } else {
+ ma_smod(srcDest, rhs, srcDest, scratch);
+ }
+}
+
+void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
+ ma_vdiv_f32(dest, src, dest);
+}
+
+void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
+ ma_vdiv(dest, src, dest);
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+ ScratchRegisterScope scratch(*this);
+
+ ma_strd(r0, r1, EDtrAddr(sp, EDtrOffImm(-8)), PreIndex);
+
+ ma_mov(Imm32((int32_t)dest.addr), scratch);
+ ma_ldrd(EDtrAddr(scratch, EDtrOffImm(0)), r0, r1);
+
+ as_add(r0, r0, Imm8(1), SetCC);
+ as_adc(r1, r1, Imm8(0), LeaveCC);
+
+ ma_strd(r0, r1, EDtrAddr(scratch, EDtrOffImm(0)));
+ ma_ldrd(EDtrAddr(sp, EDtrOffImm(8)), r0, r1, PostIndex);
+}
+
+void MacroAssembler::neg32(Register reg) { ma_neg(reg, reg, SetCC); }
+
+void MacroAssembler::neg64(Register64 reg) {
+ as_rsb(reg.low, reg.low, Imm8(0), SetCC);
+ as_rsc(reg.high, reg.high, Imm8(0));
+}
+
+void MacroAssembler::negPtr(Register reg) { neg32(reg); }
+
+void MacroAssembler::negateDouble(FloatRegister reg) { ma_vneg(reg, reg); }
+
+void MacroAssembler::negateFloat(FloatRegister reg) { ma_vneg_f32(reg, reg); }
+
+void MacroAssembler::abs32(Register src, Register dest) {
+ as_cmp(src, Imm8(0));
+ as_rsb(dest, src, Imm8(0), LeaveCC, LessThan);
+ if (dest != src) {
+ as_mov(dest, O2Reg(src), LeaveCC, GreaterThanOrEqual);
+ }
+}
+
+void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
+ ma_vabs_f32(src, dest);
+}
+
+void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
+ ma_vabs(src, dest);
+}
+
+void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
+ ma_vsqrt_f32(src, dest);
+}
+
+void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
+ ma_vsqrt(src, dest);
+}
+
+void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxFloat32(srcDest, other, handleNaN, false);
+}
+
+void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxDouble(srcDest, other, handleNaN, false);
+}
+
+void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxFloat32(srcDest, other, handleNaN, true);
+}
+
+void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxDouble(srcDest, other, handleNaN, true);
+}
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ ma_lsl(imm, dest, dest);
+}
+
+void MacroAssembler::lshiftPtr(Register src, Register dest) {
+ ma_lsl(src, dest, dest);
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ if (imm.value == 0) {
+ return;
+ }
+
+ if (imm.value < 32) {
+ as_mov(dest.high, lsl(dest.high, imm.value));
+ as_orr(dest.high, dest.high, lsr(dest.low, 32 - imm.value));
+ as_mov(dest.low, lsl(dest.low, imm.value));
+ } else {
+ as_mov(dest.high, lsl(dest.low, imm.value - 32));
+ ma_mov(Imm32(0), dest.low);
+ }
+}
+
+void MacroAssembler::lshift64(Register unmaskedShift, Register64 dest) {
+ // dest.high = dest.high << shift | dest.low << shift - 32 | dest.low >> 32 -
+ // shift Note: one of the two dest.low shift will always yield zero due to
+ // negative shift.
+
+ ScratchRegisterScope shift(*this);
+ as_and(shift, unmaskedShift, Imm8(0x3f));
+ as_mov(dest.high, lsl(dest.high, shift));
+ as_sub(shift, shift, Imm8(32));
+ as_orr(dest.high, dest.high, lsl(dest.low, shift));
+ ma_neg(shift, shift);
+ as_orr(dest.high, dest.high, lsr(dest.low, shift));
+ as_and(shift, unmaskedShift, Imm8(0x3f));
+ as_mov(dest.low, lsl(dest.low, shift));
+}
+
+void MacroAssembler::lshift32(Register src, Register dest) {
+ ma_lsl(src, dest, dest);
+}
+
+void MacroAssembler::flexibleLshift32(Register src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ as_and(scratch, src, Imm8(0x1F));
+ lshift32(scratch, dest);
+}
+
+void MacroAssembler::lshift32(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ lshiftPtr(imm, dest);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ if (imm.value) {
+ ma_lsr(imm, dest, dest);
+ }
+}
+
+void MacroAssembler::rshiftPtr(Register src, Register dest) {
+ ma_lsr(src, dest, dest);
+}
+
+void MacroAssembler::rshift32(Register src, Register dest) {
+ ma_lsr(src, dest, dest);
+}
+
+void MacroAssembler::flexibleRshift32(Register src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ as_and(scratch, src, Imm8(0x1F));
+ rshift32(scratch, dest);
+}
+
+void MacroAssembler::rshift32(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ rshiftPtr(imm, dest);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ if (imm.value) {
+ ma_asr(imm, dest, dest);
+ }
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ if (!imm.value) {
+ return;
+ }
+
+ if (imm.value < 32) {
+ as_mov(dest.low, lsr(dest.low, imm.value));
+ as_orr(dest.low, dest.low, lsl(dest.high, 32 - imm.value));
+ as_mov(dest.high, asr(dest.high, imm.value));
+ } else if (imm.value == 32) {
+ as_mov(dest.low, O2Reg(dest.high));
+ as_mov(dest.high, asr(dest.high, 31));
+ } else {
+ as_mov(dest.low, asr(dest.high, imm.value - 32));
+ as_mov(dest.high, asr(dest.high, 31));
+ }
+}
+
+void MacroAssembler::rshift64Arithmetic(Register unmaskedShift,
+ Register64 dest) {
+ Label proceed;
+
+ // dest.low = dest.low >>> shift | dest.high <<< 32 - shift
+ // if (shift - 32 >= 0)
+ // dest.low |= dest.high >>> shift - 32
+ // Note: Negative shifts yield a zero as result, except for the signed
+ // right shift. Therefore we need to test for it and only do it if
+ // it isn't negative.
+ ScratchRegisterScope shift(*this);
+
+ as_and(shift, unmaskedShift, Imm8(0x3f));
+ as_mov(dest.low, lsr(dest.low, shift));
+ as_rsb(shift, shift, Imm8(32));
+ as_orr(dest.low, dest.low, lsl(dest.high, shift));
+ ma_neg(shift, shift, SetCC);
+ ma_b(&proceed, Signed);
+
+ as_orr(dest.low, dest.low, asr(dest.high, shift));
+
+ bind(&proceed);
+ as_and(shift, unmaskedShift, Imm8(0x3f));
+ as_mov(dest.high, asr(dest.high, shift));
+}
+
+void MacroAssembler::rshift32Arithmetic(Register src, Register dest) {
+ ma_asr(src, dest, dest);
+}
+
+void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ rshiftPtrArithmetic(imm, dest);
+}
+
+void MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ as_and(scratch, src, Imm8(0x1F));
+ rshift32Arithmetic(scratch, dest);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ if (!imm.value) {
+ return;
+ }
+
+ if (imm.value < 32) {
+ as_mov(dest.low, lsr(dest.low, imm.value));
+ as_orr(dest.low, dest.low, lsl(dest.high, 32 - imm.value));
+ as_mov(dest.high, lsr(dest.high, imm.value));
+ } else if (imm.value == 32) {
+ ma_mov(dest.high, dest.low);
+ ma_mov(Imm32(0), dest.high);
+ } else {
+ ma_lsr(Imm32(imm.value - 32), dest.high, dest.low);
+ ma_mov(Imm32(0), dest.high);
+ }
+}
+
+void MacroAssembler::rshift64(Register unmaskedShift, Register64 dest) {
+ // dest.low = dest.low >> shift | dest.high >> shift - 32 | dest.high << 32 -
+ // shift Note: one of the two dest.high shifts will always yield zero due to
+ // negative shift.
+
+ ScratchRegisterScope shift(*this);
+ as_and(shift, unmaskedShift, Imm8(0x3f));
+ as_mov(dest.low, lsr(dest.low, shift));
+ as_sub(shift, shift, Imm8(32));
+ as_orr(dest.low, dest.low, lsr(dest.high, shift));
+ ma_neg(shift, shift);
+ as_orr(dest.low, dest.low, lsl(dest.high, shift));
+ as_and(shift, unmaskedShift, Imm8(0x3f));
+ as_mov(dest.high, lsr(dest.high, shift));
+}
+
+// ===============================================================
+// Rotate functions
+void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
+ if (count.value) {
+ ma_rol(count, input, dest);
+ } else {
+ ma_mov(input, dest);
+ }
+}
+
+void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ ma_rol(count, input, dest, scratch);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ MOZ_ASSERT(input.low != dest.high && input.high != dest.low);
+
+ int32_t amount = count.value & 0x3f;
+ if (amount > 32) {
+ rotateRight64(Imm32(64 - amount), input, dest, temp);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ if (amount == 0) {
+ ma_mov(input.low, dest.low);
+ ma_mov(input.high, dest.high);
+ } else if (amount == 32) {
+ ma_mov(input.low, scratch);
+ ma_mov(input.high, dest.low);
+ ma_mov(scratch, dest.high);
+ } else {
+ MOZ_ASSERT(0 < amount && amount < 32);
+ ma_mov(dest.high, scratch);
+ as_mov(dest.high, lsl(dest.high, amount));
+ as_orr(dest.high, dest.high, lsr(dest.low, 32 - amount));
+ as_mov(dest.low, lsl(dest.low, amount));
+ as_orr(dest.low, dest.low, lsr(scratch, 32 - amount));
+ }
+ }
+}
+
+void MacroAssembler::rotateLeft64(Register shift, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(shift != temp);
+ MOZ_ASSERT(src == dest);
+ MOZ_ASSERT(temp != src.low && temp != src.high);
+ MOZ_ASSERT(shift != src.low && shift != src.high);
+ MOZ_ASSERT(temp != InvalidReg);
+
+ ScratchRegisterScope shift_value(*this);
+ Label high, done;
+
+ ma_mov(src.high, temp);
+ as_and(shift_value, shift, Imm8(0x3f));
+ as_cmp(shift_value, Imm8(32));
+ ma_b(&high, GreaterThanOrEqual);
+
+ // high = high << shift | low >> 32 - shift
+ // low = low << shift | high >> 32 - shift
+ as_mov(dest.high, lsl(src.high, shift_value));
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_orr(dest.high, dest.high, lsr(src.low, shift_value));
+
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_mov(dest.low, lsl(src.low, shift_value));
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_orr(dest.low, dest.low, lsr(temp, shift_value));
+
+ ma_b(&done);
+
+ // A 32 - 64 shift is a 0 - 32 shift in the other direction.
+ bind(&high);
+ as_rsb(shift_value, shift_value, Imm8(64));
+
+ as_mov(dest.high, lsr(src.high, shift_value));
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_orr(dest.high, dest.high, lsl(src.low, shift_value));
+
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_mov(dest.low, lsr(src.low, shift_value));
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_orr(dest.low, dest.low, lsl(temp, shift_value));
+
+ bind(&done);
+}
+
+void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
+ if (count.value) {
+ ma_ror(count, input, dest);
+ } else {
+ ma_mov(input, dest);
+ }
+}
+
+void MacroAssembler::rotateRight(Register count, Register input,
+ Register dest) {
+ ma_ror(count, input, dest);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ MOZ_ASSERT(input.low != dest.high && input.high != dest.low);
+
+ int32_t amount = count.value & 0x3f;
+ if (amount > 32) {
+ rotateLeft64(Imm32(64 - amount), input, dest, temp);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ if (amount == 0) {
+ ma_mov(input.low, dest.low);
+ ma_mov(input.high, dest.high);
+ } else if (amount == 32) {
+ ma_mov(input.low, scratch);
+ ma_mov(input.high, dest.low);
+ ma_mov(scratch, dest.high);
+ } else {
+ MOZ_ASSERT(0 < amount && amount < 32);
+ ma_mov(dest.high, scratch);
+ as_mov(dest.high, lsr(dest.high, amount));
+ as_orr(dest.high, dest.high, lsl(dest.low, 32 - amount));
+ as_mov(dest.low, lsr(dest.low, amount));
+ as_orr(dest.low, dest.low, lsl(scratch, 32 - amount));
+ }
+ }
+}
+
+void MacroAssembler::rotateRight64(Register shift, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(shift != temp);
+ MOZ_ASSERT(src == dest);
+ MOZ_ASSERT(temp != src.low && temp != src.high);
+ MOZ_ASSERT(shift != src.low && shift != src.high);
+ MOZ_ASSERT(temp != InvalidReg);
+
+ ScratchRegisterScope shift_value(*this);
+ Label high, done;
+
+ ma_mov(src.high, temp);
+ as_and(shift_value, shift, Imm8(0x3f));
+ as_cmp(shift_value, Imm8(32));
+ ma_b(&high, GreaterThanOrEqual);
+
+ // high = high >> shift | low << 32 - shift
+ // low = low >> shift | high << 32 - shift
+ as_mov(dest.high, lsr(src.high, shift_value));
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_orr(dest.high, dest.high, lsl(src.low, shift_value));
+
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_mov(dest.low, lsr(src.low, shift_value));
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_orr(dest.low, dest.low, lsl(temp, shift_value));
+
+ ma_b(&done);
+
+ // A 32 - 64 shift is a 0 - 32 shift in the other direction.
+ bind(&high);
+ as_rsb(shift_value, shift_value, Imm8(64));
+
+ as_mov(dest.high, lsl(src.high, shift_value));
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_orr(dest.high, dest.high, lsr(src.low, shift_value));
+
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_mov(dest.low, lsl(src.low, shift_value));
+ as_rsb(shift_value, shift_value, Imm8(32));
+ as_orr(dest.low, dest.low, lsr(temp, shift_value));
+
+ bind(&done);
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ // Inlined calls to load8{Zero,Sign}Extend() and cmp32Set() to acquire
+ // exclusive access to scratch registers.
+
+ bool isSigned;
+ Imm32 imm(0);
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ isSigned = false;
+ imm = Imm32(uint8_t(rhs.value));
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ isSigned = true;
+ imm = Imm32(int8_t(rhs.value));
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+
+ ma_dataTransferN(IsLoad, 8, isSigned, lhs.base, Imm32(lhs.offset), scratch,
+ scratch2);
+ ma_cmp(scratch, imm, scratch2);
+ emitSet(cond, dest);
+}
+
+void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ // Inlined calls to load16{Zero,Sign}Extend() and cmp32Set() to acquire
+ // exclusive access to scratch registers.
+
+ bool isSigned;
+ Imm32 imm(0);
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ isSigned = false;
+ imm = Imm32(uint16_t(rhs.value));
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ isSigned = true;
+ imm = Imm32(int16_t(rhs.value));
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+
+ ma_dataTransferN(IsLoad, 16, isSigned, lhs.base, Imm32(lhs.offset), scratch,
+ scratch2);
+ ma_cmp(scratch, imm, scratch2);
+ emitSet(cond, dest);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ cmp32(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ Label success, done;
+
+ branch64(cond, lhs, rhs, &success);
+ move32(Imm32(0), dest);
+ jump(&done);
+ bind(&success);
+ move32(Imm32(1), dest);
+ bind(&done);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ cmpPtr(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
+ ma_clz(src, dest);
+}
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+
+ ma_clz(src.high, scratch);
+ as_cmp(scratch, Imm8(32));
+ ma_mov(scratch, dest, LeaveCC, NotEqual);
+ ma_clz(src.low, dest, Equal);
+ as_add(dest, dest, Imm8(32), LeaveCC, Equal);
+}
+
+void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
+ ScratchRegisterScope scratch(*this);
+ ma_ctz(src, dest, scratch);
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+ Label done, high;
+ as_cmp(src.low, Imm8(0));
+ ma_b(&high, Equal);
+
+ ctz32(src.low, dest, /* knownNotZero = */ true);
+ ma_b(&done);
+
+ bind(&high);
+ ctz32(src.high, dest, /* knownNotZero = */ false);
+ as_add(dest, dest, Imm8(32));
+
+ bind(&done);
+}
+
+void MacroAssembler::popcnt32(Register input, Register output, Register tmp) {
+ // Equivalent to GCC output of mozilla::CountPopulation32()
+
+ ScratchRegisterScope scratch(*this);
+
+ if (input != output) {
+ ma_mov(input, output);
+ }
+ as_mov(tmp, asr(output, 1));
+ ma_and(Imm32(0x55555555), tmp, scratch);
+ ma_sub(output, tmp, output);
+ as_mov(tmp, asr(output, 2));
+ ma_mov(Imm32(0x33333333), scratch);
+ ma_and(scratch, output);
+ ma_and(scratch, tmp);
+ ma_add(output, tmp, output);
+ as_add(output, output, lsr(output, 4));
+ ma_and(Imm32(0xF0F0F0F), output, scratch);
+ as_add(output, output, lsl(output, 8));
+ as_add(output, output, lsl(output, 16));
+ as_mov(output, asr(output, 24));
+}
+
+void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register tmp) {
+ MOZ_ASSERT(dest.low != tmp);
+ MOZ_ASSERT(dest.high != tmp);
+ MOZ_ASSERT(dest.low != dest.high);
+ // The source and destination can overlap. Therefore make sure we don't
+ // clobber the source before we have the data.
+ if (dest.low != src.high) {
+ popcnt32(src.low, dest.low, tmp);
+ popcnt32(src.high, dest.high, tmp);
+ } else {
+ MOZ_ASSERT(dest.high != src.high);
+ popcnt32(src.low, dest.high, tmp);
+ popcnt32(src.high, dest.low, tmp);
+ }
+ ma_add(dest.high, dest.low);
+ ma_mov(Imm32(0), dest.high);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ // Inlined calls to load8{Zero,Sign}Extend() and branch32() to acquire
+ // exclusive access to scratch registers.
+
+ bool isSigned;
+ Imm32 imm(0);
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ isSigned = false;
+ imm = Imm32(uint8_t(rhs.value));
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ isSigned = true;
+ imm = Imm32(int8_t(rhs.value));
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+
+ ma_dataTransferN(IsLoad, 8, isSigned, lhs.base, Imm32(lhs.offset), scratch,
+ scratch2);
+ ma_cmp(scratch, imm, scratch2);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ // Inlined calls to load8{Zero,Sign}Extend() and branch32() to acquire
+ // exclusive access to scratch registers.
+
+ bool isSigned;
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ isSigned = false;
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ isSigned = true;
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+
+ if (isSigned) {
+ Register index = lhs.index;
+
+ // ARMv7 does not have LSL on an index register with an extended load.
+ if (lhs.scale != TimesOne) {
+ ma_lsl(Imm32::ShiftOf(lhs.scale), index, scratch);
+ index = scratch;
+ }
+
+ if (lhs.offset != 0) {
+ if (index != scratch) {
+ ma_mov(index, scratch);
+ index = scratch;
+ }
+ ma_add(Imm32(lhs.offset), index, scratch2);
+ }
+ ma_ldrsb(EDtrAddr(lhs.base, EDtrOffReg(index)), scratch);
+ } else {
+ Register base = lhs.base;
+ uint32_t scale = Imm32::ShiftOf(lhs.scale).value;
+
+ if (lhs.offset == 0) {
+ ma_ldrb(DTRAddr(base, DtrRegImmShift(lhs.index, LSL, scale)), scratch);
+ } else {
+ ma_add(base, Imm32(lhs.offset), scratch, scratch2);
+ ma_ldrb(DTRAddr(scratch, DtrRegImmShift(lhs.index, LSL, scale)), scratch);
+ }
+ }
+
+ ma_cmp(scratch, rhs);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ // Inlined calls to load16{Zero,Sign}Extend() and branch32() to acquire
+ // exclusive access to scratch registers.
+
+ bool isSigned;
+ Imm32 imm(0);
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ isSigned = false;
+ imm = Imm32(uint16_t(rhs.value));
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ isSigned = true;
+ imm = Imm32(int16_t(rhs.value));
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+
+ ma_dataTransferN(IsLoad, 16, isSigned, lhs.base, Imm32(lhs.offset), scratch,
+ scratch2);
+ ma_cmp(scratch, imm, scratch2);
+ ma_b(label, cond);
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ ma_cmp(lhs, rhs);
+ ma_b(label, cond);
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ ScratchRegisterScope scratch(*this);
+
+ ma_cmp(lhs, rhs, scratch);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(lhs, scratch, scratch2);
+ ma_cmp(scratch, rhs);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ ma_ldr(lhs, scratch, scratch2);
+ ma_cmp(scratch, rhs, scratch2);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+
+ // Load into scratch.
+ movePtr(ImmWord(uintptr_t(lhs.addr)), scratch);
+ ma_ldr(DTRAddr(scratch, DtrOffImm(0)), scratch);
+
+ ma_cmp(scratch, rhs);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ // Load into scratch.
+ movePtr(ImmWord(uintptr_t(lhs.addr)), scratch);
+ ma_ldr(DTRAddr(scratch, DtrOffImm(0)), scratch);
+
+ ma_cmp(scratch, rhs, scratch2);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ {
+ ScratchRegisterScope scratch(*this);
+
+ Register base = lhs.base;
+ uint32_t scale = Imm32::ShiftOf(lhs.scale).value;
+
+ // Load lhs into scratch2.
+ if (lhs.offset != 0) {
+ ma_add(base, Imm32(lhs.offset), scratch, scratch2);
+ ma_ldr(DTRAddr(scratch, DtrRegImmShift(lhs.index, LSL, scale)), scratch2);
+ } else {
+ ma_ldr(DTRAddr(base, DtrRegImmShift(lhs.index, LSL, scale)), scratch2);
+ }
+ }
+ branch32(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs,
+ Register rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ {
+ ScratchRegisterScope scratch(*this);
+
+ Register base = lhs.base;
+ uint32_t scale = Imm32::ShiftOf(lhs.scale).value;
+
+ // Load lhs into scratch2.
+ if (lhs.offset != 0) {
+ ma_add(base, Imm32(lhs.offset), scratch, scratch2);
+ ma_ldr(DTRAddr(scratch, DtrRegImmShift(lhs.index, LSL, scale)), scratch2);
+ } else {
+ ma_ldr(DTRAddr(base, DtrRegImmShift(lhs.index, LSL, scale)), scratch2);
+ }
+ }
+ branch32(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
+ Imm32 rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+
+ movePtr(lhs, scratch);
+ ma_ldr(DTRAddr(scratch, DtrOffImm(0)), scratch);
+
+ ma_cmp(scratch, rhs, scratch2);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ Label done;
+
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, val.firstHalf(), &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, val.firstHalf(), label);
+ }
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)),
+ val.secondHalf(), label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ Label done;
+
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, rhs.low, &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, rhs.low, label);
+ }
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), rhs.high,
+ label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ Label done;
+
+ load32(rhs, scratch);
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, scratch, &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, scratch, label);
+ }
+
+ load32(Address(rhs.base, rhs.offset + sizeof(uint32_t)), scratch);
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), scratch,
+ label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ bool fallthrough = false;
+ Label fallthroughLabel;
+
+ if (!fail) {
+ fail = &fallthroughLabel;
+ fallthrough = true;
+ }
+
+ switch (cond) {
+ case Assembler::Equal:
+ branch32(Assembler::NotEqual, lhs.low, val.low(), fail);
+ branch32(Assembler::Equal, lhs.high, val.hi(), success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ case Assembler::NotEqual:
+ branch32(Assembler::NotEqual, lhs.low, val.low(), success);
+ branch32(Assembler::NotEqual, lhs.high, val.hi(), success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual: {
+ Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
+ Assembler::Condition cond2 =
+ Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
+ Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
+
+ cmp32(lhs.high, val.hi());
+ ma_b(success, cond1);
+ ma_b(fail, cond2);
+ cmp32(lhs.low, val.low());
+ ma_b(success, cond3);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Condition code not supported");
+ break;
+ }
+
+ if (fallthrough) {
+ bind(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ bool fallthrough = false;
+ Label fallthroughLabel;
+
+ if (!fail) {
+ fail = &fallthroughLabel;
+ fallthrough = true;
+ }
+
+ switch (cond) {
+ case Assembler::Equal:
+ branch32(Assembler::NotEqual, lhs.low, rhs.low, fail);
+ branch32(Assembler::Equal, lhs.high, rhs.high, success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ case Assembler::NotEqual:
+ branch32(Assembler::NotEqual, lhs.low, rhs.low, success);
+ branch32(Assembler::NotEqual, lhs.high, rhs.high, success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual: {
+ Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
+ Assembler::Condition cond2 =
+ Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
+ Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
+
+ cmp32(lhs.high, rhs.high);
+ ma_b(success, cond1);
+ ma_b(fail, cond2);
+ cmp32(lhs.low, rhs.low);
+ ma_b(success, cond3);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Condition code not supported");
+ break;
+ }
+
+ if (fallthrough) {
+ bind(fail);
+ }
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ branch32(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ branch32(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
+ Label* label) {
+ branchPtr(cond, lhs, ImmWord(uintptr_t(rhs.value)), label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ movePtr(rhs, scratch);
+ branchPtr(cond, lhs, scratch, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
+ Label* label) {
+ branch32(cond, lhs, Imm32(rhs.value), label);
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
+ L label) {
+ branch32(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
+ Label* label) {
+ branchPtr(cond, lhs, ImmWord(uintptr_t(rhs.value)), label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ ImmWord rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+ Register rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ ImmWord rhs, Label* label) {
+ branch32(cond, lhs, Imm32(rhs.value), label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ Register rhs, Label* label) {
+ branch32(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ branchPtr(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ compareFloat(lhs, rhs);
+
+ if (cond == DoubleNotEqual) {
+ // Force the unordered cases not to jump.
+ Label unordered;
+ ma_b(&unordered, VFP_Unordered);
+ ma_b(label, VFP_NotEqualOrUnordered);
+ bind(&unordered);
+ return;
+ }
+
+ if (cond == DoubleEqualOrUnordered) {
+ ma_b(label, VFP_Unordered);
+ ma_b(label, VFP_Equal);
+ return;
+ }
+
+ ma_b(label, ConditionFromDoubleCondition(cond));
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ branchTruncateFloat32ToInt32(src, dest, fail);
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ ScratchFloat32Scope scratchFloat32(*this);
+ ScratchRegisterScope scratch(*this);
+
+ ma_vcvt_F32_I32(src, scratchFloat32.sintOverlay());
+ ma_vxfer(scratchFloat32, dest);
+ ma_cmp(dest, Imm32(0x7fffffff), scratch);
+ ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::NotEqual);
+ ma_b(fail, Assembler::Equal);
+}
+
+void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ compareDouble(lhs, rhs);
+
+ if (cond == DoubleNotEqual) {
+ // Force the unordered cases not to jump.
+ Label unordered;
+ ma_b(&unordered, VFP_Unordered);
+ ma_b(label, VFP_NotEqualOrUnordered);
+ bind(&unordered);
+ return;
+ }
+
+ if (cond == DoubleEqualOrUnordered) {
+ ma_b(label, VFP_Unordered);
+ ma_b(label, VFP_Equal);
+ return;
+ }
+
+ ma_b(label, ConditionFromDoubleCondition(cond));
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ branchTruncateDoubleToInt32(src, dest, fail);
+}
+
+// There are two options for implementing branchTruncateDoubleToInt32:
+//
+// 1. Convert the floating point value to an integer, if it did not fit, then it
+// was clamped to INT_MIN/INT_MAX, and we can test it. NOTE: if the value
+// really was supposed to be INT_MAX / INT_MIN then it will be wrong.
+//
+// 2. Convert the floating point value to an integer, if it did not fit, then it
+// set one or two bits in the fpcsr. Check those.
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ ScratchDoubleScope scratchDouble(*this);
+ FloatRegister scratchSIntReg = scratchDouble.sintOverlay();
+ ScratchRegisterScope scratch(*this);
+
+ ma_vcvt_F64_I32(src, scratchSIntReg);
+ ma_vxfer(scratchSIntReg, dest);
+ ma_cmp(dest, Imm32(0x7fffffff), scratch);
+ ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::NotEqual);
+ ma_b(fail, Assembler::Equal);
+}
+
+template <typename T>
+void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
+ Label* label) {
+ add32(src, dest);
+ as_b(label, cond);
+}
+
+template <typename T>
+void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
+ Label* label) {
+ sub32(src, dest);
+ j(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Overflow);
+ ScratchRegisterScope scratch(*this);
+ Assembler::Condition overflow_cond =
+ ma_check_mul(src, dest, dest, scratch, cond);
+ j(overflow_cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ rshift32(src, dest);
+ branch32(cond == Zero ? Equal : NotEqual, dest, Imm32(0), label);
+}
+
+void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
+ MOZ_ASSERT(cond == Overflow);
+ neg32(reg);
+ j(cond, label);
+}
+
+void MacroAssembler::branchAdd64(Condition cond, Imm64 imm, Register64 dest,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ ma_add(imm.low(), dest.low, scratch, SetCC);
+ ma_adc(imm.hi(), dest.high, scratch, SetCC);
+ j(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ branchAdd32(cond, src, dest, label);
+}
+
+template <typename T>
+void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ branchSub32(cond, src, dest, label);
+}
+
+void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
+ Label* label) {
+ branchMul32(cond, src, dest, label);
+}
+
+void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ ma_sub(rhs, lhs, scratch, SetCC);
+ as_b(label, cond);
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ // x86 likes test foo, foo rather than cmp foo, #0.
+ // Convert the former into the latter.
+ if (lhs == rhs && (cond == Zero || cond == NonZero)) {
+ as_cmp(lhs, Imm8(0));
+ } else {
+ ma_tst(lhs, rhs);
+ }
+ ma_b(label, cond);
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ ScratchRegisterScope scratch(*this);
+ ma_tst(lhs, rhs, scratch);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ load32(lhs, scratch2);
+ branchTest32(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ load32(lhs, scratch2);
+ branchTest32(cond, scratch2, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ branchTest32(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ branchTest32(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
+ Imm32 rhs, Label* label) {
+ branchTest32(cond, lhs, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ if (cond == Assembler::Zero || cond == Assembler::NonZero) {
+ ScratchRegisterScope scratch(*this);
+
+ MOZ_ASSERT(lhs.low == rhs.low);
+ MOZ_ASSERT(lhs.high == rhs.high);
+ ma_orr(lhs.low, lhs.high, scratch);
+ branchTestPtr(cond, scratch, scratch, label);
+ } else if (cond == Assembler::Signed || cond == Assembler::NotSigned) {
+ branchTest32(cond, lhs.high, rhs.high, label);
+ } else {
+ MOZ_CRASH("Unsupported condition");
+ }
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
+ Label* label) {
+ branchTestUndefinedImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
+ Label* label) {
+ branchTestUndefinedImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const BaseIndex& address,
+ Label* label) {
+ branchTestUndefinedImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestUndefinedImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestUndefinedImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testUndefined(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, Register tag,
+ Label* label) {
+ branchTestInt32Impl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
+ Label* label) {
+ branchTestInt32Impl(cond, address, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestInt32Impl(cond, address, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestInt32Impl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestInt32Impl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testInt32(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestInt32Truthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition c = testInt32Truthy(truthy, value);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, Register tag,
+ Label* label) {
+ branchTestDoubleImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
+ Label* label) {
+ branchTestDoubleImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestDoubleImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestDoubleImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestDoubleImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testDouble(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestDoubleTruthy(bool truthy, FloatRegister reg,
+ Label* label) {
+ Condition c = testDoubleTruthy(truthy, reg);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, Register tag,
+ Label* label) {
+ branchTestNumberImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestNumberImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestNumberImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testNumber(cond, t);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
+ Label* label) {
+ branchTestBooleanImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
+ Label* label) {
+ branchTestBooleanImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestBooleanImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestBooleanImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestBooleanImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testBoolean(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition c = testBooleanTruthy(truthy, value);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestString(Condition cond, Register tag,
+ Label* label) {
+ branchTestStringImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const Address& address,
+ Label* label) {
+ branchTestStringImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestStringImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestStringImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestStringImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testString(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestStringTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition c = testStringTruthy(truthy, value);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
+ Label* label) {
+ branchTestSymbolImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
+ Label* label) {
+ branchTestSymbolImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestSymbolImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestSymbolImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestSymbolImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testSymbol(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
+ Label* label) {
+ branchTestBigIntImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
+ Label* label) {
+ branchTestBigIntImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestBigIntImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestBigIntImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestBigIntImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testBigInt(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestBigIntTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition c = testBigIntTruthy(truthy, value);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, Register tag,
+ Label* label) {
+ branchTestNullImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const Address& address,
+ Label* label) {
+ branchTestNullImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestNullImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestNullImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestNullImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testNull(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, Register tag,
+ Label* label) {
+ branchTestObjectImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const Address& address,
+ Label* label) {
+ branchTestObjectImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestObjectImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestObjectImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestObjectImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testObject(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestGCThingImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testGCThing(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
+ Label* label) {
+ branchTestPrimitiveImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestPrimitiveImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestPrimitiveImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testPrimitive(cond, t);
+ ma_b(label, c);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, Register tag,
+ Label* label) {
+ branchTestMagicImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
+ Label* label) {
+ branchTestMagicImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestMagicImpl(cond, address, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
+ L label) {
+ branchTestMagicImpl(cond, value, label);
+}
+
+template <typename T, class L>
+void MacroAssembler::branchTestMagicImpl(Condition cond, const T& t, L label) {
+ cond = testMagic(cond, t);
+ ma_b(label, cond);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label notMagic;
+ if (cond == Assembler::Equal) {
+ branchTestMagic(Assembler::NotEqual, valaddr, &notMagic);
+ } else {
+ branchTestMagic(Assembler::NotEqual, valaddr, label);
+ }
+
+ branch32(cond, ToPayload(valaddr), Imm32(why), label);
+ bind(&notMagic);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label notSameValue;
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, ToType(lhs), rhs.typeReg(), &notSameValue);
+ } else {
+ branch32(Assembler::NotEqual, ToType(lhs), rhs.typeReg(), label);
+ }
+
+ branch32(cond, ToPayload(lhs), rhs.payloadReg(), label);
+ bind(&notSameValue);
+}
+
+template <typename T>
+void MacroAssembler::testNumberSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testNumber(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testBooleanSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testBoolean(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testStringSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testString(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testSymbolSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testSymbol(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testBigIntSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testBigInt(cond, src);
+ emitSet(cond, dest);
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
+ MOZ_ASSERT(
+ addr.offset == 0,
+ "NYI: offsets from pc should be shifted by the number of instructions.");
+
+ Register base = addr.base;
+ uint32_t scale = Imm32::ShiftOf(addr.scale).value;
+
+ ma_ldr(DTRAddr(base, DtrRegImmShift(addr.index, LSL, scale)), pc);
+
+ if (base == pc) {
+ // When loading from pc, the pc is shifted to the next instruction, we
+ // add one extra instruction to accomodate for this shifted offset.
+ breakpoint();
+ }
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmp32(lhs, rhs);
+ ma_mov(src, dest, LeaveCC, cond);
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ cmp32(lhs, rhs);
+ ma_mov(src, dest, LeaveCC, cond);
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+ ma_ldr(rhs, scratch, scratch2);
+ cmp32Move32(cond, lhs, scratch, src, dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmp32Move32(cond, lhs, rhs, src, dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ cmp32Move32(cond, lhs, rhs, src, dest);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
+ const Address& rhs, const Address& src,
+ Register dest) {
+ // This is never used, but must be present to facilitate linking on arm.
+ MOZ_CRASH("No known use cases");
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
+ const Address& src, Register dest) {
+ // This is never used, but must be present to facilitate linking on arm.
+ MOZ_CRASH("No known use cases");
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ cmp32(lhs, rhs);
+ ScratchRegisterScope scratch(*this);
+ ma_ldr(src, dest, scratch, Offset, cond);
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+ Imm32 mask, const Address& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ ScratchRegisterScope scratch(*this);
+ ma_ldr(src, dest, scratch, Offset, cond);
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+ Imm32 mask, Register src, Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ ma_mov(src, dest, LeaveCC, cond);
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+ Register dest) {
+ ma_mov(src, dest, LeaveCC, cond);
+}
+
+void MacroAssembler::spectreZeroRegister(Condition cond, Register,
+ Register dest) {
+ ma_mov(Imm32(0), dest, cond);
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(length != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ ma_mov(Imm32(0), index, Assembler::BelowOrEqual);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ ma_mov(Imm32(0), index, Assembler::BelowOrEqual);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ spectreBoundsCheck32(index, length, maybeScratch, failure);
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+ const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ spectreBoundsCheck32(index, length, maybeScratch, failure);
+}
+
+// ========================================================================
+// Memory access primitives.
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const Address& addr) {
+ ScratchRegisterScope scratch(*this);
+ ma_vstr(src, addr, scratch);
+}
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const BaseIndex& addr) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+ uint32_t scale = Imm32::ShiftOf(addr.scale).value;
+ ma_vstr(src, addr.base, addr.index, scratch, scratch2, scale, addr.offset);
+}
+
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const Address& addr) {
+ ScratchRegisterScope scratch(*this);
+ ma_vstr(src.asSingle(), addr, scratch);
+}
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const BaseIndex& addr) {
+ ScratchRegisterScope scratch(*this);
+ SecondScratchRegisterScope scratch2(*this);
+ uint32_t scale = Imm32::ShiftOf(addr.scale).value;
+ ma_vstr(src.asSingle(), addr.base, addr.index, scratch, scratch2, scale,
+ addr.offset);
+}
+
+void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) {
+ // On ARMv6 the optional argument (BarrierST, etc) is ignored.
+ if (barrier == (MembarStoreStore | MembarSynchronizing)) {
+ ma_dsb(BarrierST);
+ } else if (barrier & MembarSynchronizing) {
+ ma_dsb();
+ } else if (barrier == MembarStoreStore) {
+ ma_dmb(BarrierST);
+ } else if (barrier) {
+ ma_dmb();
+ }
+}
+
+// ===============================================================
+// Clamping functions.
+
+void MacroAssembler::clampIntToUint8(Register reg) {
+ // Look at (reg >> 8) if it is 0, then reg shouldn't be clamped if it is
+ // <0, then we want to clamp to 0, otherwise, we wish to clamp to 255
+ ScratchRegisterScope scratch(*this);
+ as_mov(scratch, asr(reg, 8), SetCC);
+ ma_mov(Imm32(0xff), reg, NotEqual);
+ ma_mov(Imm32(0), reg, Signed);
+}
+
+template <typename T>
+void MacroAssemblerARMCompat::fallibleUnboxPtrImpl(const T& src, Register dest,
+ JSValueType type,
+ Label* fail) {
+ switch (type) {
+ case JSVAL_TYPE_OBJECT:
+ asMasm().branchTestObject(Assembler::NotEqual, src, fail);
+ break;
+ case JSVAL_TYPE_STRING:
+ asMasm().branchTestString(Assembler::NotEqual, src, fail);
+ break;
+ case JSVAL_TYPE_SYMBOL:
+ asMasm().branchTestSymbol(Assembler::NotEqual, src, fail);
+ break;
+ case JSVAL_TYPE_BIGINT:
+ asMasm().branchTestBigInt(Assembler::NotEqual, src, fail);
+ break;
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+ unboxNonDouble(src, dest, type);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(src, dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(src, dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(src, dest, type, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+void MacroAssemblerARMCompat::incrementInt32Value(const Address& addr) {
+ asMasm().add32(Imm32(1), ToPayload(addr));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_MacroAssembler_arm_inl_h */
diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp
new file mode 100644
index 0000000000..5c3205a6db
--- /dev/null
+++ b/js/src/jit/arm/MacroAssembler-arm.cpp
@@ -0,0 +1,6375 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm/MacroAssembler-arm.h"
+
+#include "mozilla/Casting.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+
+#include "jsmath.h"
+
+#include "jit/arm/Simulator-arm.h"
+#include "jit/AtomicOp.h"
+#include "jit/AtomicOperations.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MoveEmitter.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "util/Memory.h"
+#include "vm/BigIntType.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/StringType.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace jit;
+
+using mozilla::Abs;
+using mozilla::BitwiseCast;
+using mozilla::DebugOnly;
+using mozilla::IsPositiveZero;
+using mozilla::Maybe;
+
+bool isValueDTRDCandidate(ValueOperand& val) {
+ // In order to be used for a DTRD memory function, the two target registers
+ // need to be a) Adjacent, with the tag larger than the payload, and b)
+ // Aligned to a multiple of two.
+ if ((val.typeReg().code() != (val.payloadReg().code() + 1))) {
+ return false;
+ }
+ if ((val.payloadReg().code() & 1) != 0) {
+ return false;
+ }
+ return true;
+}
+
+void MacroAssemblerARM::convertBoolToInt32(Register source, Register dest) {
+ // Note that C++ bool is only 1 byte, so zero extend it to clear the
+ // higher-order bits.
+ as_and(dest, source, Imm8(0xff));
+}
+
+void MacroAssemblerARM::convertInt32ToDouble(Register src,
+ FloatRegister dest_) {
+ // Direct conversions aren't possible.
+ VFPRegister dest = VFPRegister(dest_);
+ as_vxfer(src, InvalidReg, dest.sintOverlay(), CoreToFloat);
+ as_vcvt(dest, dest.sintOverlay());
+}
+
+void MacroAssemblerARM::convertInt32ToDouble(const Address& src,
+ FloatRegister dest) {
+ ScratchDoubleScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_vldr(src, scratch, scratch2);
+ as_vcvt(dest, VFPRegister(scratch).sintOverlay());
+}
+
+void MacroAssemblerARM::convertInt32ToDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ Register base = src.base;
+ uint32_t scale = Imm32::ShiftOf(src.scale).value;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (src.offset != 0) {
+ ma_add(base, Imm32(src.offset), scratch, scratch2);
+ base = scratch;
+ }
+ ma_ldr(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), scratch);
+ convertInt32ToDouble(scratch, dest);
+}
+
+void MacroAssemblerARM::convertUInt32ToDouble(Register src,
+ FloatRegister dest_) {
+ // Direct conversions aren't possible.
+ VFPRegister dest = VFPRegister(dest_);
+ as_vxfer(src, InvalidReg, dest.uintOverlay(), CoreToFloat);
+ as_vcvt(dest, dest.uintOverlay());
+}
+
+static const double TO_DOUBLE_HIGH_SCALE = 0x100000000;
+
+void MacroAssemblerARM::convertUInt32ToFloat32(Register src,
+ FloatRegister dest_) {
+ // Direct conversions aren't possible.
+ VFPRegister dest = VFPRegister(dest_);
+ as_vxfer(src, InvalidReg, dest.uintOverlay(), CoreToFloat);
+ as_vcvt(VFPRegister(dest).singleOverlay(), dest.uintOverlay());
+}
+
+void MacroAssemblerARM::convertDoubleToFloat32(FloatRegister src,
+ FloatRegister dest,
+ Condition c) {
+ as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src), false, c);
+}
+
+// Checks whether a double is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void MacroAssemblerARM::convertDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail,
+ bool negativeZeroCheck) {
+ // Convert the floating point value to an integer, if it did not fit, then
+ // when we convert it *back* to a float, it will have a different value,
+ // which we can test.
+ ScratchDoubleScope scratchDouble(asMasm());
+ ScratchRegisterScope scratch(asMasm());
+
+ FloatRegister scratchSIntReg = scratchDouble.sintOverlay();
+
+ ma_vcvt_F64_I32(src, scratchSIntReg);
+ // Move the value into the dest register.
+ ma_vxfer(scratchSIntReg, dest);
+ ma_vcvt_I32_F64(scratchSIntReg, scratchDouble);
+ ma_vcmp(src, scratchDouble);
+ as_vmrs(pc);
+ ma_b(fail, Assembler::VFP_NotEqualOrUnordered);
+
+ if (negativeZeroCheck) {
+ as_cmp(dest, Imm8(0));
+ // Test and bail for -0.0, when integer result is 0. Move the top word
+ // of the double into the output reg, if it is non-zero, then the
+ // original value was -0.0.
+ as_vxfer(dest, InvalidReg, src, FloatToCore, Assembler::Equal, 1);
+ ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::Equal);
+ ma_b(fail, Assembler::Equal);
+ }
+}
+
+// Checks whether a float32 is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void MacroAssemblerARM::convertFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail,
+ bool negativeZeroCheck) {
+ // Converting the floating point value to an integer and then converting it
+ // back to a float32 would not work, as float to int32 conversions are
+ // clamping (e.g. float(INT32_MAX + 1) would get converted into INT32_MAX
+ // and then back to float(INT32_MAX + 1)). If this ever happens, we just
+ // bail out.
+ ScratchFloat32Scope scratchFloat(asMasm());
+ ScratchRegisterScope scratch(asMasm());
+
+ FloatRegister ScratchSIntReg = scratchFloat.sintOverlay();
+ ma_vcvt_F32_I32(src, ScratchSIntReg);
+
+ // Store the result
+ ma_vxfer(ScratchSIntReg, dest);
+
+ ma_vcvt_I32_F32(ScratchSIntReg, scratchFloat);
+ ma_vcmp(src, scratchFloat);
+ as_vmrs(pc);
+ ma_b(fail, Assembler::VFP_NotEqualOrUnordered);
+
+ // Bail out in the clamped cases.
+ ma_cmp(dest, Imm32(0x7fffffff), scratch);
+ ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::NotEqual);
+ ma_b(fail, Assembler::Equal);
+
+ if (negativeZeroCheck) {
+ as_cmp(dest, Imm8(0));
+ // Test and bail for -0.0, when integer result is 0. Move the float into
+ // the output reg, and if it is non-zero then the original value was
+ // -0.0
+ as_vxfer(dest, InvalidReg, VFPRegister(src).singleOverlay(), FloatToCore,
+ Assembler::Equal, 0);
+ ma_cmp(dest, Imm32(0x80000000), scratch, Assembler::Equal);
+ ma_b(fail, Assembler::Equal);
+ }
+}
+
+void MacroAssemblerARM::convertFloat32ToDouble(FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(dest.isDouble());
+ MOZ_ASSERT(src.isSingle());
+ as_vcvt(VFPRegister(dest), VFPRegister(src).singleOverlay());
+}
+
+void MacroAssemblerARM::convertInt32ToFloat32(Register src,
+ FloatRegister dest) {
+ // Direct conversions aren't possible.
+ as_vxfer(src, InvalidReg, dest.sintOverlay(), CoreToFloat);
+ as_vcvt(dest.singleOverlay(), dest.sintOverlay());
+}
+
+void MacroAssemblerARM::convertInt32ToFloat32(const Address& src,
+ FloatRegister dest) {
+ ScratchFloat32Scope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_vldr(src, scratch, scratch2);
+ as_vcvt(dest, VFPRegister(scratch).sintOverlay());
+}
+
+bool MacroAssemblerARM::alu_dbl(Register src1, Imm32 imm, Register dest,
+ ALUOp op, SBit s, Condition c) {
+ if ((s == SetCC && !condsAreSafe(op)) || !can_dbl(op)) {
+ return false;
+ }
+
+ ALUOp interop = getDestVariant(op);
+ Imm8::TwoImm8mData both = Imm8::EncodeTwoImms(imm.value);
+ if (both.fst().invalid()) {
+ return false;
+ }
+
+ // For the most part, there is no good reason to set the condition codes for
+ // the first instruction. We can do better things if the second instruction
+ // doesn't have a dest, such as check for overflow by doing first operation
+ // don't do second operation if first operation overflowed. This preserves
+ // the overflow condition code. Unfortunately, it is horribly brittle.
+ as_alu(dest, src1, Operand2(both.fst()), interop, LeaveCC, c);
+ as_alu(dest, dest, Operand2(both.snd()), op, s, c);
+ return true;
+}
+
+void MacroAssemblerARM::ma_alu(Register src1, Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, ALUOp op, SBit s,
+ Condition c) {
+ // ma_mov should be used for moves.
+ MOZ_ASSERT(op != OpMov);
+ MOZ_ASSERT(op != OpMvn);
+ MOZ_ASSERT(src1 != scratch);
+
+ // As it turns out, if you ask for a compare-like instruction you *probably*
+ // want it to set condition codes.
+ MOZ_ASSERT_IF(dest == InvalidReg, s == SetCC);
+
+ // The operator gives us the ability to determine how this can be used.
+ Imm8 imm8 = Imm8(imm.value);
+ // One instruction: If we can encode it using an imm8m, then do so.
+ if (!imm8.invalid()) {
+ as_alu(dest, src1, imm8, op, s, c);
+ return;
+ }
+
+ // One instruction, negated:
+ Imm32 negImm = imm;
+ Register negDest;
+ ALUOp negOp = ALUNeg(op, dest, scratch, &negImm, &negDest);
+ Imm8 negImm8 = Imm8(negImm.value);
+ // 'add r1, r2, -15' can be replaced with 'sub r1, r2, 15'.
+ // The dest can be replaced (InvalidReg => scratch).
+ // This is useful if we wish to negate tst. tst has an invalid (aka not
+ // used) dest, but its negation bic requires a dest.
+ if (negOp != OpInvalid && !negImm8.invalid()) {
+ as_alu(negDest, src1, negImm8, negOp, s, c);
+ return;
+ }
+
+ // Start by attempting to generate a two instruction form. Some things
+ // cannot be made into two-inst forms correctly. Namely, adds dest, src,
+ // 0xffff. Since we want the condition codes (and don't know which ones
+ // will be checked), we need to assume that the overflow flag will be
+ // checked and add{,s} dest, src, 0xff00; add{,s} dest, dest, 0xff is not
+ // guaranteed to set the overflof flag the same as the (theoretical) one
+ // instruction variant.
+ if (alu_dbl(src1, imm, dest, op, s, c)) {
+ return;
+ }
+
+ // And try with its negative.
+ if (negOp != OpInvalid && alu_dbl(src1, negImm, negDest, negOp, s, c)) {
+ return;
+ }
+
+ ma_mov(imm, scratch, c);
+ as_alu(dest, src1, O2Reg(scratch), op, s, c);
+}
+
+void MacroAssemblerARM::ma_alu(Register src1, Operand op2, Register dest,
+ ALUOp op, SBit s, Assembler::Condition c) {
+ MOZ_ASSERT(op2.tag() == Operand::Tag::OP2);
+ as_alu(dest, src1, op2.toOp2(), op, s, c);
+}
+
+void MacroAssemblerARM::ma_alu(Register src1, Operand2 op2, Register dest,
+ ALUOp op, SBit s, Condition c) {
+ as_alu(dest, src1, op2, op, s, c);
+}
+
+void MacroAssemblerARM::ma_nop() { as_nop(); }
+
+BufferOffset MacroAssemblerARM::ma_movPatchable(Imm32 imm_, Register dest,
+ Assembler::Condition c) {
+ int32_t imm = imm_.value;
+ if (HasMOVWT()) {
+ BufferOffset offset = as_movw(dest, Imm16(imm & 0xffff), c);
+ as_movt(dest, Imm16(imm >> 16 & 0xffff), c);
+ return offset;
+ } else {
+ return as_Imm32Pool(dest, imm, c);
+ }
+}
+
+BufferOffset MacroAssemblerARM::ma_movPatchable(ImmPtr imm, Register dest,
+ Assembler::Condition c) {
+ return ma_movPatchable(Imm32(int32_t(imm.value)), dest, c);
+}
+
+/* static */
+template <class Iter>
+void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
+ Assembler::Condition c, RelocStyle rs,
+ Iter iter) {
+ // The current instruction must be an actual instruction,
+ // not automatically-inserted boilerplate.
+ MOZ_ASSERT(iter.cur());
+ MOZ_ASSERT(iter.cur() == iter.maybeSkipAutomaticInstructions());
+
+ int32_t imm = imm32.value;
+ switch (rs) {
+ case L_MOVWT:
+ Assembler::as_movw_patch(dest, Imm16(imm & 0xffff), c, iter.cur());
+ Assembler::as_movt_patch(dest, Imm16(imm >> 16 & 0xffff), c, iter.next());
+ break;
+ case L_LDR:
+ Assembler::WritePoolEntry(iter.cur(), c, imm);
+ break;
+ }
+}
+
+template void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
+ Assembler::Condition c,
+ RelocStyle rs,
+ InstructionIterator iter);
+template void MacroAssemblerARM::ma_mov_patch(Imm32 imm32, Register dest,
+ Assembler::Condition c,
+ RelocStyle rs,
+ BufferInstructionIterator iter);
+
+void MacroAssemblerARM::ma_mov(Register src, Register dest, SBit s,
+ Assembler::Condition c) {
+ if (s == SetCC || dest != src) {
+ as_mov(dest, O2Reg(src), s, c);
+ }
+}
+
+void MacroAssemblerARM::ma_mov(Imm32 imm, Register dest,
+ Assembler::Condition c) {
+ // Try mov with Imm8 operand.
+ Imm8 imm8 = Imm8(imm.value);
+ if (!imm8.invalid()) {
+ as_alu(dest, InvalidReg, imm8, OpMov, LeaveCC, c);
+ return;
+ }
+
+ // Try mvn with Imm8 operand.
+ Imm8 negImm8 = Imm8(~imm.value);
+ if (!negImm8.invalid()) {
+ as_alu(dest, InvalidReg, negImm8, OpMvn, LeaveCC, c);
+ return;
+ }
+
+ // Try movw/movt.
+ if (HasMOVWT()) {
+ // ARMv7 supports movw/movt. movw zero-extends its 16 bit argument,
+ // so we can set the register this way. movt leaves the bottom 16
+ // bits in tact, so we always need a movw.
+ as_movw(dest, Imm16(imm.value & 0xffff), c);
+ if (uint32_t(imm.value) >> 16) {
+ as_movt(dest, Imm16(uint32_t(imm.value) >> 16), c);
+ }
+ return;
+ }
+
+ // If we don't have movw/movt, we need a load.
+ as_Imm32Pool(dest, imm.value, c);
+}
+
+void MacroAssemblerARM::ma_mov(ImmWord imm, Register dest,
+ Assembler::Condition c) {
+ ma_mov(Imm32(imm.value), dest, c);
+}
+
+void MacroAssemblerARM::ma_mov(ImmGCPtr ptr, Register dest) {
+ BufferOffset offset =
+ ma_movPatchable(Imm32(uintptr_t(ptr.value)), dest, Always);
+ writeDataRelocation(offset, ptr);
+}
+
+// Shifts (just a move with a shifting op2)
+void MacroAssemblerARM::ma_lsl(Imm32 shift, Register src, Register dst) {
+ as_mov(dst, lsl(src, shift.value));
+}
+
+void MacroAssemblerARM::ma_lsr(Imm32 shift, Register src, Register dst) {
+ as_mov(dst, lsr(src, shift.value));
+}
+
+void MacroAssemblerARM::ma_asr(Imm32 shift, Register src, Register dst) {
+ as_mov(dst, asr(src, shift.value));
+}
+
+void MacroAssemblerARM::ma_ror(Imm32 shift, Register src, Register dst) {
+ as_mov(dst, ror(src, shift.value));
+}
+
+void MacroAssemblerARM::ma_rol(Imm32 shift, Register src, Register dst) {
+ as_mov(dst, rol(src, shift.value));
+}
+
+// Shifts (just a move with a shifting op2)
+void MacroAssemblerARM::ma_lsl(Register shift, Register src, Register dst) {
+ as_mov(dst, lsl(src, shift));
+}
+
+void MacroAssemblerARM::ma_lsr(Register shift, Register src, Register dst) {
+ as_mov(dst, lsr(src, shift));
+}
+
+void MacroAssemblerARM::ma_asr(Register shift, Register src, Register dst) {
+ as_mov(dst, asr(src, shift));
+}
+
+void MacroAssemblerARM::ma_ror(Register shift, Register src, Register dst) {
+ as_mov(dst, ror(src, shift));
+}
+
+void MacroAssemblerARM::ma_rol(Register shift, Register src, Register dst,
+ AutoRegisterScope& scratch) {
+ as_rsb(scratch, shift, Imm8(32));
+ as_mov(dst, ror(src, scratch));
+}
+
+// Move not (dest <- ~src)
+void MacroAssemblerARM::ma_mvn(Register src1, Register dest, SBit s,
+ Assembler::Condition c) {
+ as_alu(dest, InvalidReg, O2Reg(src1), OpMvn, s, c);
+}
+
+// Negate (dest <- -src), src is a register, rather than a general op2.
+void MacroAssemblerARM::ma_neg(Register src1, Register dest, SBit s,
+ Assembler::Condition c) {
+ as_rsb(dest, src1, Imm8(0), s, c);
+}
+
+void MacroAssemblerARM::ma_neg(Register64 src, Register64 dest) {
+ as_rsb(dest.low, src.low, Imm8(0), SetCC);
+ as_rsc(dest.high, src.high, Imm8(0));
+}
+
+// And.
+void MacroAssemblerARM::ma_and(Register src, Register dest, SBit s,
+ Assembler::Condition c) {
+ ma_and(dest, src, dest);
+}
+
+void MacroAssemblerARM::ma_and(Register src1, Register src2, Register dest,
+ SBit s, Assembler::Condition c) {
+ as_and(dest, src1, O2Reg(src2), s, c);
+}
+
+void MacroAssemblerARM::ma_and(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Assembler::Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpAnd, s, c);
+}
+
+void MacroAssemblerARM::ma_and(Imm32 imm, Register src1, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Assembler::Condition c) {
+ ma_alu(src1, imm, dest, scratch, OpAnd, s, c);
+}
+
+// Bit clear (dest <- dest & ~imm) or (dest <- src1 & ~src2).
+void MacroAssemblerARM::ma_bic(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Assembler::Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpBic, s, c);
+}
+
+// Exclusive or.
+void MacroAssemblerARM::ma_eor(Register src, Register dest, SBit s,
+ Assembler::Condition c) {
+ ma_eor(dest, src, dest, s, c);
+}
+
+void MacroAssemblerARM::ma_eor(Register src1, Register src2, Register dest,
+ SBit s, Assembler::Condition c) {
+ as_eor(dest, src1, O2Reg(src2), s, c);
+}
+
+void MacroAssemblerARM::ma_eor(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Assembler::Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpEor, s, c);
+}
+
+void MacroAssemblerARM::ma_eor(Imm32 imm, Register src1, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Assembler::Condition c) {
+ ma_alu(src1, imm, dest, scratch, OpEor, s, c);
+}
+
+// Or.
+void MacroAssemblerARM::ma_orr(Register src, Register dest, SBit s,
+ Assembler::Condition c) {
+ ma_orr(dest, src, dest, s, c);
+}
+
+void MacroAssemblerARM::ma_orr(Register src1, Register src2, Register dest,
+ SBit s, Assembler::Condition c) {
+ as_orr(dest, src1, O2Reg(src2), s, c);
+}
+
+void MacroAssemblerARM::ma_orr(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Assembler::Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpOrr, s, c);
+}
+
+void MacroAssemblerARM::ma_orr(Imm32 imm, Register src1, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Assembler::Condition c) {
+ ma_alu(src1, imm, dest, scratch, OpOrr, s, c);
+}
+
+// Arithmetic-based ops.
+// Add with carry.
+void MacroAssemblerARM::ma_adc(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpAdc, s, c);
+}
+
+void MacroAssemblerARM::ma_adc(Register src, Register dest, SBit s,
+ Condition c) {
+ as_alu(dest, dest, O2Reg(src), OpAdc, s, c);
+}
+
+void MacroAssemblerARM::ma_adc(Register src1, Register src2, Register dest,
+ SBit s, Condition c) {
+ as_alu(dest, src1, O2Reg(src2), OpAdc, s, c);
+}
+
+void MacroAssemblerARM::ma_adc(Register src1, Imm32 op, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(src1, op, dest, scratch, OpAdc, s, c);
+}
+
+// Add.
+void MacroAssemblerARM::ma_add(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpAdd, s, c);
+}
+
+void MacroAssemblerARM::ma_add(Register src1, Register dest, SBit s,
+ Condition c) {
+ ma_alu(dest, O2Reg(src1), dest, OpAdd, s, c);
+}
+
+void MacroAssemblerARM::ma_add(Register src1, Register src2, Register dest,
+ SBit s, Condition c) {
+ as_alu(dest, src1, O2Reg(src2), OpAdd, s, c);
+}
+
+void MacroAssemblerARM::ma_add(Register src1, Operand op, Register dest, SBit s,
+ Condition c) {
+ ma_alu(src1, op, dest, OpAdd, s, c);
+}
+
+void MacroAssemblerARM::ma_add(Register src1, Imm32 op, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(src1, op, dest, scratch, OpAdd, s, c);
+}
+
+// Subtract with carry.
+void MacroAssemblerARM::ma_sbc(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpSbc, s, c);
+}
+
+void MacroAssemblerARM::ma_sbc(Register src1, Register dest, SBit s,
+ Condition c) {
+ as_alu(dest, dest, O2Reg(src1), OpSbc, s, c);
+}
+
+void MacroAssemblerARM::ma_sbc(Register src1, Register src2, Register dest,
+ SBit s, Condition c) {
+ as_alu(dest, src1, O2Reg(src2), OpSbc, s, c);
+}
+
+// Subtract.
+void MacroAssemblerARM::ma_sub(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpSub, s, c);
+}
+
+void MacroAssemblerARM::ma_sub(Register src1, Register dest, SBit s,
+ Condition c) {
+ ma_alu(dest, Operand(src1), dest, OpSub, s, c);
+}
+
+void MacroAssemblerARM::ma_sub(Register src1, Register src2, Register dest,
+ SBit s, Condition c) {
+ ma_alu(src1, Operand(src2), dest, OpSub, s, c);
+}
+
+void MacroAssemblerARM::ma_sub(Register src1, Operand op, Register dest, SBit s,
+ Condition c) {
+ ma_alu(src1, op, dest, OpSub, s, c);
+}
+
+void MacroAssemblerARM::ma_sub(Register src1, Imm32 op, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(src1, op, dest, scratch, OpSub, s, c);
+}
+
+// Reverse subtract.
+void MacroAssemblerARM::ma_rsb(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpRsb, s, c);
+}
+
+void MacroAssemblerARM::ma_rsb(Register src1, Register dest, SBit s,
+ Condition c) {
+ as_alu(dest, src1, O2Reg(dest), OpRsb, s, c);
+}
+
+void MacroAssemblerARM::ma_rsb(Register src1, Register src2, Register dest,
+ SBit s, Condition c) {
+ as_alu(dest, src1, O2Reg(src2), OpRsb, s, c);
+}
+
+void MacroAssemblerARM::ma_rsb(Register src1, Imm32 op2, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(src1, op2, dest, scratch, OpRsb, s, c);
+}
+
+// Reverse subtract with carry.
+void MacroAssemblerARM::ma_rsc(Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, SBit s,
+ Condition c) {
+ ma_alu(dest, imm, dest, scratch, OpRsc, s, c);
+}
+
+void MacroAssemblerARM::ma_rsc(Register src1, Register dest, SBit s,
+ Condition c) {
+ as_alu(dest, dest, O2Reg(src1), OpRsc, s, c);
+}
+
+void MacroAssemblerARM::ma_rsc(Register src1, Register src2, Register dest,
+ SBit s, Condition c) {
+ as_alu(dest, src1, O2Reg(src2), OpRsc, s, c);
+}
+
+// Compares/tests.
+// Compare negative (sets condition codes as src1 + src2 would).
+void MacroAssemblerARM::ma_cmn(Register src1, Imm32 imm,
+ AutoRegisterScope& scratch, Condition c) {
+ ma_alu(src1, imm, InvalidReg, scratch, OpCmn, SetCC, c);
+}
+
+void MacroAssemblerARM::ma_cmn(Register src1, Register src2, Condition c) {
+ as_alu(InvalidReg, src2, O2Reg(src1), OpCmn, SetCC, c);
+}
+
+void MacroAssemblerARM::ma_cmn(Register src1, Operand op, Condition c) {
+ MOZ_CRASH("Feature NYI");
+}
+
+// Compare (src - src2).
+void MacroAssemblerARM::ma_cmp(Register src1, Imm32 imm,
+ AutoRegisterScope& scratch, Condition c) {
+ ma_alu(src1, imm, InvalidReg, scratch, OpCmp, SetCC, c);
+}
+
+void MacroAssemblerARM::ma_cmp(Register src1, ImmTag tag, Condition c) {
+ // ImmTag comparisons can always be done without use of a scratch register.
+ Imm8 negtag = Imm8(-tag.value);
+ MOZ_ASSERT(!negtag.invalid());
+ as_cmn(src1, negtag, c);
+}
+
+void MacroAssemblerARM::ma_cmp(Register src1, ImmWord ptr,
+ AutoRegisterScope& scratch, Condition c) {
+ ma_cmp(src1, Imm32(ptr.value), scratch, c);
+}
+
+void MacroAssemblerARM::ma_cmp(Register src1, ImmGCPtr ptr,
+ AutoRegisterScope& scratch, Condition c) {
+ ma_mov(ptr, scratch);
+ ma_cmp(src1, scratch, c);
+}
+
+void MacroAssemblerARM::ma_cmp(Register src1, Operand op,
+ AutoRegisterScope& scratch,
+ AutoRegisterScope& scratch2, Condition c) {
+ switch (op.tag()) {
+ case Operand::Tag::OP2:
+ as_cmp(src1, op.toOp2(), c);
+ break;
+ case Operand::Tag::MEM:
+ ma_ldr(op.toAddress(), scratch, scratch2);
+ as_cmp(src1, O2Reg(scratch), c);
+ break;
+ default:
+ MOZ_CRASH("trying to compare FP and integer registers");
+ }
+}
+
+void MacroAssemblerARM::ma_cmp(Register src1, Register src2, Condition c) {
+ as_cmp(src1, O2Reg(src2), c);
+}
+
+// Test for equality, (src1 ^ src2).
+void MacroAssemblerARM::ma_teq(Register src1, Imm32 imm,
+ AutoRegisterScope& scratch, Condition c) {
+ ma_alu(src1, imm, InvalidReg, scratch, OpTeq, SetCC, c);
+}
+
+void MacroAssemblerARM::ma_teq(Register src1, Register src2, Condition c) {
+ as_tst(src1, O2Reg(src2), c);
+}
+
+void MacroAssemblerARM::ma_teq(Register src1, Operand op, Condition c) {
+ as_teq(src1, op.toOp2(), c);
+}
+
+// Test (src1 & src2).
+void MacroAssemblerARM::ma_tst(Register src1, Imm32 imm,
+ AutoRegisterScope& scratch, Condition c) {
+ ma_alu(src1, imm, InvalidReg, scratch, OpTst, SetCC, c);
+}
+
+void MacroAssemblerARM::ma_tst(Register src1, Register src2, Condition c) {
+ as_tst(src1, O2Reg(src2), c);
+}
+
+void MacroAssemblerARM::ma_tst(Register src1, Operand op, Condition c) {
+ as_tst(src1, op.toOp2(), c);
+}
+
+void MacroAssemblerARM::ma_mul(Register src1, Register src2, Register dest) {
+ as_mul(dest, src1, src2);
+}
+
+void MacroAssemblerARM::ma_mul(Register src1, Imm32 imm, Register dest,
+ AutoRegisterScope& scratch) {
+ ma_mov(imm, scratch);
+ as_mul(dest, src1, scratch);
+}
+
+Assembler::Condition MacroAssemblerARM::ma_check_mul(Register src1,
+ Register src2,
+ Register dest,
+ AutoRegisterScope& scratch,
+ Condition cond) {
+ // TODO: this operation is illegal on armv6 and earlier
+ // if src2 == scratch or src2 == dest.
+ if (cond == Equal || cond == NotEqual) {
+ as_smull(scratch, dest, src1, src2, SetCC);
+ return cond;
+ }
+
+ if (cond == Overflow) {
+ as_smull(scratch, dest, src1, src2);
+ as_cmp(scratch, asr(dest, 31));
+ return NotEqual;
+ }
+
+ MOZ_CRASH("Condition NYI");
+}
+
+Assembler::Condition MacroAssemblerARM::ma_check_mul(Register src1, Imm32 imm,
+ Register dest,
+ AutoRegisterScope& scratch,
+ Condition cond) {
+ ma_mov(imm, scratch);
+
+ if (cond == Equal || cond == NotEqual) {
+ as_smull(scratch, dest, scratch, src1, SetCC);
+ return cond;
+ }
+
+ if (cond == Overflow) {
+ as_smull(scratch, dest, scratch, src1);
+ as_cmp(scratch, asr(dest, 31));
+ return NotEqual;
+ }
+
+ MOZ_CRASH("Condition NYI");
+}
+
+void MacroAssemblerARM::ma_umull(Register src1, Imm32 imm, Register destHigh,
+ Register destLow, AutoRegisterScope& scratch) {
+ ma_mov(imm, scratch);
+ as_umull(destHigh, destLow, src1, scratch);
+}
+
+void MacroAssemblerARM::ma_umull(Register src1, Register src2,
+ Register destHigh, Register destLow) {
+ as_umull(destHigh, destLow, src1, src2);
+}
+
+void MacroAssemblerARM::ma_mod_mask(Register src, Register dest, Register hold,
+ Register tmp, AutoRegisterScope& scratch,
+ AutoRegisterScope& scratch2,
+ int32_t shift) {
+ // We wish to compute x % (1<<y) - 1 for a known constant, y.
+ //
+ // 1. Let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit dividend as
+ // a number in base b, namely c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n
+ //
+ // 2. Since both addition and multiplication commute with modulus:
+ // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C ==
+ // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)...
+ //
+ // 3. Since b == C + 1, b % C == 1, and b^n % C == 1 the whole thing
+ // simplifies to: c_0 + c_1 + c_2 ... c_n % C
+ //
+ // Each c_n can easily be computed by a shift/bitextract, and the modulus
+ // can be maintained by simply subtracting by C whenever the number gets
+ // over C.
+ int32_t mask = (1 << shift) - 1;
+ Label head;
+
+ // Register 'hold' holds -1 if the value was negative, 1 otherwise. The
+ // scratch reg holds the remaining bits that have not been processed lr
+ // serves as a temporary location to store extracted bits into as well as
+ // holding the trial subtraction as a temp value dest is the accumulator
+ // (and holds the final result)
+ //
+ // Move the whole value into tmp, setting the codition codes so we can muck
+ // with them later.
+ as_mov(tmp, O2Reg(src), SetCC);
+ // Zero out the dest.
+ ma_mov(Imm32(0), dest);
+ // Set the hold appropriately.
+ ma_mov(Imm32(1), hold);
+ ma_mov(Imm32(-1), hold, Signed);
+ as_rsb(tmp, tmp, Imm8(0), SetCC, Signed);
+
+ // Begin the main loop.
+ bind(&head);
+ {
+ // Extract the bottom bits.
+ ma_and(Imm32(mask), tmp, scratch, scratch2);
+ // Add those bits to the accumulator.
+ ma_add(scratch, dest, dest);
+ // Do a trial subtraction, this is the same operation as cmp, but we store
+ // the dest.
+ ma_sub(dest, Imm32(mask), scratch, scratch2, SetCC);
+ // If (sum - C) > 0, store sum - C back into sum, thus performing a modulus.
+ ma_mov(scratch, dest, LeaveCC, NotSigned);
+ // Get rid of the bits that we extracted before, and set the condition
+ // codes.
+ as_mov(tmp, lsr(tmp, shift), SetCC);
+ // If the shift produced zero, finish, otherwise, continue in the loop.
+ ma_b(&head, NonZero);
+ }
+
+ // Check the hold to see if we need to negate the result. Hold can only be
+ // 1 or -1, so this will never set the 0 flag.
+ as_cmp(hold, Imm8(0));
+ // If the hold was non-zero, negate the result to be in line with what JS
+ // wants this will set the condition codes if we try to negate.
+ as_rsb(dest, dest, Imm8(0), SetCC, Signed);
+ // Since the Zero flag is not set by the compare, we can *only* set the Zero
+ // flag in the rsb, so Zero is set iff we negated zero (e.g. the result of
+ // the computation was -0.0).
+}
+
+void MacroAssemblerARM::ma_smod(Register num, Register div, Register dest,
+ AutoRegisterScope& scratch) {
+ as_sdiv(scratch, num, div);
+ as_mls(dest, num, scratch, div);
+}
+
+void MacroAssemblerARM::ma_umod(Register num, Register div, Register dest,
+ AutoRegisterScope& scratch) {
+ as_udiv(scratch, num, div);
+ as_mls(dest, num, scratch, div);
+}
+
+// Division
+void MacroAssemblerARM::ma_sdiv(Register num, Register div, Register dest,
+ Condition cond) {
+ as_sdiv(dest, num, div, cond);
+}
+
+void MacroAssemblerARM::ma_udiv(Register num, Register div, Register dest,
+ Condition cond) {
+ as_udiv(dest, num, div, cond);
+}
+
+// Miscellaneous instructions.
+void MacroAssemblerARM::ma_clz(Register src, Register dest, Condition cond) {
+ as_clz(dest, src, cond);
+}
+
+void MacroAssemblerARM::ma_ctz(Register src, Register dest,
+ AutoRegisterScope& scratch) {
+ // int c = __clz(a & -a);
+ // return a ? 31 - c : c;
+ as_rsb(scratch, src, Imm8(0), SetCC);
+ as_and(dest, src, O2Reg(scratch), LeaveCC);
+ as_clz(dest, dest);
+ as_rsb(dest, dest, Imm8(0x1F), LeaveCC, Assembler::NotEqual);
+}
+
+// Memory.
+// Shortcut for when we know we're transferring 32 bits of data.
+void MacroAssemblerARM::ma_dtr(LoadStore ls, Register rn, Imm32 offset,
+ Register rt, AutoRegisterScope& scratch,
+ Index mode, Assembler::Condition cc) {
+ ma_dataTransferN(ls, 32, true, rn, offset, rt, scratch, mode, cc);
+}
+
+void MacroAssemblerARM::ma_dtr(LoadStore ls, Register rt, const Address& addr,
+ AutoRegisterScope& scratch, Index mode,
+ Condition cc) {
+ ma_dataTransferN(ls, 32, true, addr.base, Imm32(addr.offset), rt, scratch,
+ mode, cc);
+}
+
+void MacroAssemblerARM::ma_str(Register rt, DTRAddr addr, Index mode,
+ Condition cc) {
+ as_dtr(IsStore, 32, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_str(Register rt, const Address& addr,
+ AutoRegisterScope& scratch, Index mode,
+ Condition cc) {
+ ma_dtr(IsStore, rt, addr, scratch, mode, cc);
+}
+
+void MacroAssemblerARM::ma_strd(Register rt, DebugOnly<Register> rt2,
+ EDtrAddr addr, Index mode, Condition cc) {
+ MOZ_ASSERT((rt.code() & 1) == 0);
+ MOZ_ASSERT(rt2.value.code() == rt.code() + 1);
+ as_extdtr(IsStore, 64, true, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_ldr(DTRAddr addr, Register rt, Index mode,
+ Condition cc) {
+ as_dtr(IsLoad, 32, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_ldr(const Address& addr, Register rt,
+ AutoRegisterScope& scratch, Index mode,
+ Condition cc) {
+ ma_dtr(IsLoad, rt, addr, scratch, mode, cc);
+}
+
+void MacroAssemblerARM::ma_ldrb(DTRAddr addr, Register rt, Index mode,
+ Condition cc) {
+ as_dtr(IsLoad, 8, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_ldrsh(EDtrAddr addr, Register rt, Index mode,
+ Condition cc) {
+ as_extdtr(IsLoad, 16, true, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_ldrh(EDtrAddr addr, Register rt, Index mode,
+ Condition cc) {
+ as_extdtr(IsLoad, 16, false, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_ldrsb(EDtrAddr addr, Register rt, Index mode,
+ Condition cc) {
+ as_extdtr(IsLoad, 8, true, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_ldrd(EDtrAddr addr, Register rt,
+ DebugOnly<Register> rt2, Index mode,
+ Condition cc) {
+ MOZ_ASSERT((rt.code() & 1) == 0);
+ MOZ_ASSERT(rt2.value.code() == rt.code() + 1);
+ MOZ_ASSERT(addr.maybeOffsetRegister() !=
+ rt); // Undefined behavior if rm == rt/rt2.
+ MOZ_ASSERT(addr.maybeOffsetRegister() != rt2);
+ as_extdtr(IsLoad, 64, true, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_strh(Register rt, EDtrAddr addr, Index mode,
+ Condition cc) {
+ as_extdtr(IsStore, 16, false, mode, rt, addr, cc);
+}
+
+void MacroAssemblerARM::ma_strb(Register rt, DTRAddr addr, Index mode,
+ Condition cc) {
+ as_dtr(IsStore, 8, mode, rt, addr, cc);
+}
+
+// Specialty for moving N bits of data, where n == 8,16,32,64.
+BufferOffset MacroAssemblerARM::ma_dataTransferN(
+ LoadStore ls, int size, bool IsSigned, Register rn, Register rm,
+ Register rt, AutoRegisterScope& scratch, Index mode,
+ Assembler::Condition cc, Scale scale) {
+ MOZ_ASSERT(size == 8 || size == 16 || size == 32 || size == 64);
+
+ if (size == 32 || (size == 8 && !IsSigned)) {
+ return as_dtr(ls, size, mode, rt,
+ DTRAddr(rn, DtrRegImmShift(rm, LSL, scale)), cc);
+ }
+
+ if (scale != TimesOne) {
+ ma_lsl(Imm32(scale), rm, scratch);
+ rm = scratch;
+ }
+
+ return as_extdtr(ls, size, IsSigned, mode, rt, EDtrAddr(rn, EDtrOffReg(rm)),
+ cc);
+}
+
+// No scratch register is required if scale is TimesOne.
+BufferOffset MacroAssemblerARM::ma_dataTransferN(LoadStore ls, int size,
+ bool IsSigned, Register rn,
+ Register rm, Register rt,
+ Index mode,
+ Assembler::Condition cc) {
+ MOZ_ASSERT(size == 8 || size == 16 || size == 32 || size == 64);
+ if (size == 32 || (size == 8 && !IsSigned)) {
+ return as_dtr(ls, size, mode, rt,
+ DTRAddr(rn, DtrRegImmShift(rm, LSL, TimesOne)), cc);
+ }
+ return as_extdtr(ls, size, IsSigned, mode, rt, EDtrAddr(rn, EDtrOffReg(rm)),
+ cc);
+}
+
+BufferOffset MacroAssemblerARM::ma_dataTransferN(LoadStore ls, int size,
+ bool IsSigned, Register rn,
+ Imm32 offset, Register rt,
+ AutoRegisterScope& scratch,
+ Index mode,
+ Assembler::Condition cc) {
+ MOZ_ASSERT(!(ls == IsLoad && mode == PostIndex && rt == pc),
+ "Large-offset PostIndex loading into PC requires special logic: "
+ "see ma_popn_pc().");
+
+ int off = offset.value;
+
+ // We can encode this as a standard ldr.
+ if (size == 32 || (size == 8 && !IsSigned)) {
+ if (off < 4096 && off > -4096) {
+ // This encodes as a single instruction, Emulating mode's behavior
+ // in a multi-instruction sequence is not necessary.
+ return as_dtr(ls, size, mode, rt, DTRAddr(rn, DtrOffImm(off)), cc);
+ }
+
+ // We cannot encode this offset in a single ldr. For mode == index,
+ // try to encode it as |add scratch, base, imm; ldr dest, [scratch,
+ // +offset]|. This does not wark for mode == PreIndex or mode == PostIndex.
+ // PreIndex is simple, just do the add into the base register first,
+ // then do a PreIndex'ed load. PostIndexed loads can be tricky.
+ // Normally, doing the load with an index of 0, then doing an add would
+ // work, but if the destination is the PC, you don't get to execute the
+ // instruction after the branch, which will lead to the base register
+ // not being updated correctly. Explicitly handle this case, without
+ // doing anything fancy, then handle all of the other cases.
+
+ // mode == Offset
+ // add scratch, base, offset_hi
+ // ldr dest, [scratch, +offset_lo]
+ //
+ // mode == PreIndex
+ // add base, base, offset_hi
+ // ldr dest, [base, +offset_lo]!
+
+ int bottom = off & 0xfff;
+ int neg_bottom = 0x1000 - bottom;
+
+ MOZ_ASSERT(rn != scratch);
+ MOZ_ASSERT(mode != PostIndex);
+
+ // At this point, both off - bottom and off + neg_bottom will be
+ // reasonable-ish quantities.
+ //
+ // Note a neg_bottom of 0x1000 can not be encoded as an immediate
+ // negative offset in the instruction and this occurs when bottom is
+ // zero, so this case is guarded against below.
+ if (off < 0) {
+ Operand2 sub_off = Imm8(-(off - bottom)); // sub_off = bottom - off
+ if (!sub_off.invalid()) {
+ // - sub_off = off - bottom
+ as_sub(scratch, rn, sub_off, LeaveCC, cc);
+ return as_dtr(ls, size, Offset, rt, DTRAddr(scratch, DtrOffImm(bottom)),
+ cc);
+ }
+
+ // sub_off = -neg_bottom - off
+ sub_off = Imm8(-(off + neg_bottom));
+ if (!sub_off.invalid() && bottom != 0) {
+ // Guarded against by: bottom != 0
+ MOZ_ASSERT(neg_bottom < 0x1000);
+ // - sub_off = neg_bottom + off
+ as_sub(scratch, rn, sub_off, LeaveCC, cc);
+ return as_dtr(ls, size, Offset, rt,
+ DTRAddr(scratch, DtrOffImm(-neg_bottom)), cc);
+ }
+ } else {
+ // sub_off = off - bottom
+ Operand2 sub_off = Imm8(off - bottom);
+ if (!sub_off.invalid()) {
+ // sub_off = off - bottom
+ as_add(scratch, rn, sub_off, LeaveCC, cc);
+ return as_dtr(ls, size, Offset, rt, DTRAddr(scratch, DtrOffImm(bottom)),
+ cc);
+ }
+
+ // sub_off = neg_bottom + off
+ sub_off = Imm8(off + neg_bottom);
+ if (!sub_off.invalid() && bottom != 0) {
+ // Guarded against by: bottom != 0
+ MOZ_ASSERT(neg_bottom < 0x1000);
+ // sub_off = neg_bottom + off
+ as_add(scratch, rn, sub_off, LeaveCC, cc);
+ return as_dtr(ls, size, Offset, rt,
+ DTRAddr(scratch, DtrOffImm(-neg_bottom)), cc);
+ }
+ }
+
+ ma_mov(offset, scratch);
+ return as_dtr(ls, size, mode, rt,
+ DTRAddr(rn, DtrRegImmShift(scratch, LSL, 0)));
+ } else {
+ // Should attempt to use the extended load/store instructions.
+ if (off < 256 && off > -256) {
+ return as_extdtr(ls, size, IsSigned, mode, rt,
+ EDtrAddr(rn, EDtrOffImm(off)), cc);
+ }
+
+ // We cannot encode this offset in a single extldr. Try to encode it as
+ // an add scratch, base, imm; extldr dest, [scratch, +offset].
+ int bottom = off & 0xff;
+ int neg_bottom = 0x100 - bottom;
+ // At this point, both off - bottom and off + neg_bottom will be
+ // reasonable-ish quantities.
+ //
+ // Note a neg_bottom of 0x100 can not be encoded as an immediate
+ // negative offset in the instruction and this occurs when bottom is
+ // zero, so this case is guarded against below.
+ if (off < 0) {
+ // sub_off = bottom - off
+ Operand2 sub_off = Imm8(-(off - bottom));
+ if (!sub_off.invalid()) {
+ // - sub_off = off - bottom
+ as_sub(scratch, rn, sub_off, LeaveCC, cc);
+ return as_extdtr(ls, size, IsSigned, Offset, rt,
+ EDtrAddr(scratch, EDtrOffImm(bottom)), cc);
+ }
+ // sub_off = -neg_bottom - off
+ sub_off = Imm8(-(off + neg_bottom));
+ if (!sub_off.invalid() && bottom != 0) {
+ // Guarded against by: bottom != 0
+ MOZ_ASSERT(neg_bottom < 0x100);
+ // - sub_off = neg_bottom + off
+ as_sub(scratch, rn, sub_off, LeaveCC, cc);
+ return as_extdtr(ls, size, IsSigned, Offset, rt,
+ EDtrAddr(scratch, EDtrOffImm(-neg_bottom)), cc);
+ }
+ } else {
+ // sub_off = off - bottom
+ Operand2 sub_off = Imm8(off - bottom);
+ if (!sub_off.invalid()) {
+ // sub_off = off - bottom
+ as_add(scratch, rn, sub_off, LeaveCC, cc);
+ return as_extdtr(ls, size, IsSigned, Offset, rt,
+ EDtrAddr(scratch, EDtrOffImm(bottom)), cc);
+ }
+ // sub_off = neg_bottom + off
+ sub_off = Imm8(off + neg_bottom);
+ if (!sub_off.invalid() && bottom != 0) {
+ // Guarded against by: bottom != 0
+ MOZ_ASSERT(neg_bottom < 0x100);
+ // sub_off = neg_bottom + off
+ as_add(scratch, rn, sub_off, LeaveCC, cc);
+ return as_extdtr(ls, size, IsSigned, Offset, rt,
+ EDtrAddr(scratch, EDtrOffImm(-neg_bottom)), cc);
+ }
+ }
+ ma_mov(offset, scratch);
+ return as_extdtr(ls, size, IsSigned, mode, rt,
+ EDtrAddr(rn, EDtrOffReg(scratch)), cc);
+ }
+}
+
+void MacroAssemblerARM::ma_pop(Register r) {
+ as_dtr(IsLoad, 32, PostIndex, r, DTRAddr(sp, DtrOffImm(4)));
+}
+
+void MacroAssemblerARM::ma_popn_pc(Imm32 n, AutoRegisterScope& scratch,
+ AutoRegisterScope& scratch2) {
+ // pc <- [sp]; sp += n
+ int32_t nv = n.value;
+
+ if (nv < 4096 && nv >= -4096) {
+ as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(nv)));
+ } else {
+ ma_mov(sp, scratch);
+ ma_add(Imm32(n), sp, scratch2);
+ as_dtr(IsLoad, 32, Offset, pc, DTRAddr(scratch, DtrOffImm(0)));
+ }
+}
+
+void MacroAssemblerARM::ma_push(Register r) {
+ MOZ_ASSERT(r != sp, "Use ma_push_sp().");
+ as_dtr(IsStore, 32, PreIndex, r, DTRAddr(sp, DtrOffImm(-4)));
+}
+
+void MacroAssemblerARM::ma_push_sp(Register r, AutoRegisterScope& scratch) {
+ // Pushing sp is not well-defined: use two instructions.
+ MOZ_ASSERT(r == sp);
+ ma_mov(sp, scratch);
+ as_dtr(IsStore, 32, PreIndex, scratch, DTRAddr(sp, DtrOffImm(-4)));
+}
+
+void MacroAssemblerARM::ma_vpop(VFPRegister r) {
+ startFloatTransferM(IsLoad, sp, IA, WriteBack);
+ transferFloatReg(r);
+ finishFloatTransfer();
+}
+
+void MacroAssemblerARM::ma_vpush(VFPRegister r) {
+ startFloatTransferM(IsStore, sp, DB, WriteBack);
+ transferFloatReg(r);
+ finishFloatTransfer();
+}
+
+// Barriers
+void MacroAssemblerARM::ma_dmb(BarrierOption option) {
+ if (HasDMBDSBISB()) {
+ as_dmb(option);
+ } else {
+ as_dmb_trap();
+ }
+}
+
+void MacroAssemblerARM::ma_dsb(BarrierOption option) {
+ if (HasDMBDSBISB()) {
+ as_dsb(option);
+ } else {
+ as_dsb_trap();
+ }
+}
+
+// Branches when done from within arm-specific code.
+BufferOffset MacroAssemblerARM::ma_b(Label* dest, Assembler::Condition c) {
+ return as_b(dest, c);
+}
+
+void MacroAssemblerARM::ma_bx(Register dest, Assembler::Condition c) {
+ as_bx(dest, c);
+}
+
+void MacroAssemblerARM::ma_b(void* target, Assembler::Condition c) {
+ // An immediate pool is used for easier patching.
+ as_Imm32Pool(pc, uint32_t(target), c);
+}
+
+// This is almost NEVER necessary: we'll basically never be calling a label,
+// except possibly in the crazy bailout-table case.
+void MacroAssemblerARM::ma_bl(Label* dest, Assembler::Condition c) {
+ as_bl(dest, c);
+}
+
+void MacroAssemblerARM::ma_blx(Register reg, Assembler::Condition c) {
+ as_blx(reg, c);
+}
+
+// VFP/ALU
+void MacroAssemblerARM::ma_vadd(FloatRegister src1, FloatRegister src2,
+ FloatRegister dst) {
+ as_vadd(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
+}
+
+void MacroAssemblerARM::ma_vadd_f32(FloatRegister src1, FloatRegister src2,
+ FloatRegister dst) {
+ as_vadd(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
+ VFPRegister(src2).singleOverlay());
+}
+
+void MacroAssemblerARM::ma_vsub(FloatRegister src1, FloatRegister src2,
+ FloatRegister dst) {
+ as_vsub(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
+}
+
+void MacroAssemblerARM::ma_vsub_f32(FloatRegister src1, FloatRegister src2,
+ FloatRegister dst) {
+ as_vsub(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
+ VFPRegister(src2).singleOverlay());
+}
+
+void MacroAssemblerARM::ma_vmul(FloatRegister src1, FloatRegister src2,
+ FloatRegister dst) {
+ as_vmul(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
+}
+
+void MacroAssemblerARM::ma_vmul_f32(FloatRegister src1, FloatRegister src2,
+ FloatRegister dst) {
+ as_vmul(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
+ VFPRegister(src2).singleOverlay());
+}
+
+void MacroAssemblerARM::ma_vdiv(FloatRegister src1, FloatRegister src2,
+ FloatRegister dst) {
+ as_vdiv(VFPRegister(dst), VFPRegister(src1), VFPRegister(src2));
+}
+
+void MacroAssemblerARM::ma_vdiv_f32(FloatRegister src1, FloatRegister src2,
+ FloatRegister dst) {
+ as_vdiv(VFPRegister(dst).singleOverlay(), VFPRegister(src1).singleOverlay(),
+ VFPRegister(src2).singleOverlay());
+}
+
+void MacroAssemblerARM::ma_vmov(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ as_vmov(dest, src, cc);
+}
+
+void MacroAssemblerARM::ma_vmov_f32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ as_vmov(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
+ cc);
+}
+
+void MacroAssemblerARM::ma_vneg(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ as_vneg(dest, src, cc);
+}
+
+void MacroAssemblerARM::ma_vneg_f32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ as_vneg(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
+ cc);
+}
+
+void MacroAssemblerARM::ma_vabs(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ as_vabs(dest, src, cc);
+}
+
+void MacroAssemblerARM::ma_vabs_f32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ as_vabs(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
+ cc);
+}
+
+void MacroAssemblerARM::ma_vsqrt(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ as_vsqrt(dest, src, cc);
+}
+
+void MacroAssemblerARM::ma_vsqrt_f32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ as_vsqrt(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
+ cc);
+}
+
+static inline uint32_t DoubleHighWord(double d) {
+ return static_cast<uint32_t>(BitwiseCast<uint64_t>(d) >> 32);
+}
+
+static inline uint32_t DoubleLowWord(double d) {
+ return static_cast<uint32_t>(BitwiseCast<uint64_t>(d)) & uint32_t(0xffffffff);
+}
+
+void MacroAssemblerARM::ma_vimm(double value, FloatRegister dest,
+ Condition cc) {
+ if (HasVFPv3()) {
+ if (DoubleLowWord(value) == 0) {
+ if (DoubleHighWord(value) == 0) {
+ // To zero a register, load 1.0, then execute dN <- dN - dN
+ as_vimm(dest, VFPImm::One, cc);
+ as_vsub(dest, dest, dest, cc);
+ return;
+ }
+
+ VFPImm enc(DoubleHighWord(value));
+ if (enc.isValid()) {
+ as_vimm(dest, enc, cc);
+ return;
+ }
+ }
+ }
+ // Fall back to putting the value in a pool.
+ as_FImm64Pool(dest, value, cc);
+}
+
+void MacroAssemblerARM::ma_vimm_f32(float value, FloatRegister dest,
+ Condition cc) {
+ VFPRegister vd = VFPRegister(dest).singleOverlay();
+ if (HasVFPv3()) {
+ if (IsPositiveZero(value)) {
+ // To zero a register, load 1.0, then execute sN <- sN - sN.
+ as_vimm(vd, VFPImm::One, cc);
+ as_vsub(vd, vd, vd, cc);
+ return;
+ }
+
+ // Note that the vimm immediate float32 instruction encoding differs
+ // from the vimm immediate double encoding, but this difference matches
+ // the difference in the floating point formats, so it is possible to
+ // convert the float32 to a double and then use the double encoding
+ // paths. It is still necessary to firstly check that the double low
+ // word is zero because some float32 numbers set these bits and this can
+ // not be ignored.
+ double doubleValue(value);
+ if (DoubleLowWord(doubleValue) == 0) {
+ VFPImm enc(DoubleHighWord(doubleValue));
+ if (enc.isValid()) {
+ as_vimm(vd, enc, cc);
+ return;
+ }
+ }
+ }
+
+ // Fall back to putting the value in a pool.
+ as_FImm32Pool(vd, value, cc);
+}
+
+void MacroAssemblerARM::ma_vcmp(FloatRegister src1, FloatRegister src2,
+ Condition cc) {
+ as_vcmp(VFPRegister(src1), VFPRegister(src2), cc);
+}
+
+void MacroAssemblerARM::ma_vcmp_f32(FloatRegister src1, FloatRegister src2,
+ Condition cc) {
+ as_vcmp(VFPRegister(src1).singleOverlay(), VFPRegister(src2).singleOverlay(),
+ cc);
+}
+
+void MacroAssemblerARM::ma_vcmpz(FloatRegister src1, Condition cc) {
+ as_vcmpz(VFPRegister(src1), cc);
+}
+
+void MacroAssemblerARM::ma_vcmpz_f32(FloatRegister src1, Condition cc) {
+ as_vcmpz(VFPRegister(src1).singleOverlay(), cc);
+}
+
+void MacroAssemblerARM::ma_vcvt_F64_I32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ MOZ_ASSERT(src.isDouble());
+ MOZ_ASSERT(dest.isSInt());
+ as_vcvt(dest, src, false, cc);
+}
+
+void MacroAssemblerARM::ma_vcvt_F64_U32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ MOZ_ASSERT(src.isDouble());
+ MOZ_ASSERT(dest.isUInt());
+ as_vcvt(dest, src, false, cc);
+}
+
+void MacroAssemblerARM::ma_vcvt_I32_F64(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ MOZ_ASSERT(src.isSInt());
+ MOZ_ASSERT(dest.isDouble());
+ as_vcvt(dest, src, false, cc);
+}
+
+void MacroAssemblerARM::ma_vcvt_U32_F64(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ MOZ_ASSERT(src.isUInt());
+ MOZ_ASSERT(dest.isDouble());
+ as_vcvt(dest, src, false, cc);
+}
+
+void MacroAssemblerARM::ma_vcvt_F32_I32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ MOZ_ASSERT(src.isSingle());
+ MOZ_ASSERT(dest.isSInt());
+ as_vcvt(VFPRegister(dest).sintOverlay(), VFPRegister(src).singleOverlay(),
+ false, cc);
+}
+
+void MacroAssemblerARM::ma_vcvt_F32_U32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ MOZ_ASSERT(src.isSingle());
+ MOZ_ASSERT(dest.isUInt());
+ as_vcvt(VFPRegister(dest).uintOverlay(), VFPRegister(src).singleOverlay(),
+ false, cc);
+}
+
+void MacroAssemblerARM::ma_vcvt_I32_F32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ MOZ_ASSERT(src.isSInt());
+ MOZ_ASSERT(dest.isSingle());
+ as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src).sintOverlay(),
+ false, cc);
+}
+
+void MacroAssemblerARM::ma_vcvt_U32_F32(FloatRegister src, FloatRegister dest,
+ Condition cc) {
+ MOZ_ASSERT(src.isUInt());
+ MOZ_ASSERT(dest.isSingle());
+ as_vcvt(VFPRegister(dest).singleOverlay(), VFPRegister(src).uintOverlay(),
+ false, cc);
+}
+
+void MacroAssemblerARM::ma_vxfer(FloatRegister src, Register dest,
+ Condition cc) {
+ as_vxfer(dest, InvalidReg, VFPRegister(src).singleOverlay(), FloatToCore, cc);
+}
+
+void MacroAssemblerARM::ma_vxfer(FloatRegister src, Register dest1,
+ Register dest2, Condition cc) {
+ as_vxfer(dest1, dest2, VFPRegister(src), FloatToCore, cc);
+}
+
+void MacroAssemblerARM::ma_vxfer(Register src, FloatRegister dest,
+ Condition cc) {
+ as_vxfer(src, InvalidReg, VFPRegister(dest).singleOverlay(), CoreToFloat, cc);
+}
+
+void MacroAssemblerARM::ma_vxfer(Register src1, Register src2,
+ FloatRegister dest, Condition cc) {
+ as_vxfer(src1, src2, VFPRegister(dest), CoreToFloat, cc);
+}
+
+BufferOffset MacroAssemblerARM::ma_vdtr(LoadStore ls, const Address& addr,
+ VFPRegister rt,
+ AutoRegisterScope& scratch,
+ Condition cc) {
+ int off = addr.offset;
+ MOZ_ASSERT((off & 3) == 0);
+ Register base = addr.base;
+ if (off > -1024 && off < 1024) {
+ return as_vdtr(ls, rt, Operand(addr).toVFPAddr(), cc);
+ }
+
+ // We cannot encode this offset in a a single ldr. Try to encode it as an
+ // add scratch, base, imm; ldr dest, [scratch, +offset].
+ int bottom = off & (0xff << 2);
+ int neg_bottom = (0x100 << 2) - bottom;
+ // At this point, both off - bottom and off + neg_bottom will be
+ // reasonable-ish quantities.
+ //
+ // Note a neg_bottom of 0x400 can not be encoded as an immediate negative
+ // offset in the instruction and this occurs when bottom is zero, so this
+ // case is guarded against below.
+ if (off < 0) {
+ // sub_off = bottom - off
+ Operand2 sub_off = Imm8(-(off - bottom));
+ if (!sub_off.invalid()) {
+ // - sub_off = off - bottom
+ as_sub(scratch, base, sub_off, LeaveCC, cc);
+ return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(bottom)), cc);
+ }
+ // sub_off = -neg_bottom - off
+ sub_off = Imm8(-(off + neg_bottom));
+ if (!sub_off.invalid() && bottom != 0) {
+ // Guarded against by: bottom != 0
+ MOZ_ASSERT(neg_bottom < 0x400);
+ // - sub_off = neg_bottom + off
+ as_sub(scratch, base, sub_off, LeaveCC, cc);
+ return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(-neg_bottom)), cc);
+ }
+ } else {
+ // sub_off = off - bottom
+ Operand2 sub_off = Imm8(off - bottom);
+ if (!sub_off.invalid()) {
+ // sub_off = off - bottom
+ as_add(scratch, base, sub_off, LeaveCC, cc);
+ return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(bottom)), cc);
+ }
+ // sub_off = neg_bottom + off
+ sub_off = Imm8(off + neg_bottom);
+ if (!sub_off.invalid() && bottom != 0) {
+ // Guarded against by: bottom != 0
+ MOZ_ASSERT(neg_bottom < 0x400);
+ // sub_off = neg_bottom + off
+ as_add(scratch, base, sub_off, LeaveCC, cc);
+ return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(-neg_bottom)), cc);
+ }
+ }
+
+ // Safe to use scratch as dest, since ma_add() overwrites dest at the end
+ // and can't use it as internal scratch since it may also == base.
+ ma_add(base, Imm32(off), scratch, scratch, LeaveCC, cc);
+ return as_vdtr(ls, rt, VFPAddr(scratch, VFPOffImm(0)), cc);
+}
+
+BufferOffset MacroAssemblerARM::ma_vldr(VFPAddr addr, VFPRegister dest,
+ Condition cc) {
+ return as_vdtr(IsLoad, dest, addr, cc);
+}
+
+BufferOffset MacroAssemblerARM::ma_vldr(const Address& addr, VFPRegister dest,
+ AutoRegisterScope& scratch,
+ Condition cc) {
+ return ma_vdtr(IsLoad, addr, dest, scratch, cc);
+}
+
+BufferOffset MacroAssemblerARM::ma_vldr(VFPRegister src, Register base,
+ Register index,
+ AutoRegisterScope& scratch,
+ int32_t shift, Condition cc) {
+ as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
+ return as_vdtr(IsLoad, src, Operand(Address(scratch, 0)).toVFPAddr(), cc);
+}
+
+BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, VFPAddr addr,
+ Condition cc) {
+ return as_vdtr(IsStore, src, addr, cc);
+}
+
+BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, const Address& addr,
+ AutoRegisterScope& scratch,
+ Condition cc) {
+ return ma_vdtr(IsStore, addr, src, scratch, cc);
+}
+
+BufferOffset MacroAssemblerARM::ma_vstr(
+ VFPRegister src, Register base, Register index, AutoRegisterScope& scratch,
+ AutoRegisterScope& scratch2, int32_t shift, int32_t offset, Condition cc) {
+ as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
+ return ma_vstr(src, Address(scratch, offset), scratch2, cc);
+}
+
+// Without an offset, no second scratch register is necessary.
+BufferOffset MacroAssemblerARM::ma_vstr(VFPRegister src, Register base,
+ Register index,
+ AutoRegisterScope& scratch,
+ int32_t shift, Condition cc) {
+ as_add(scratch, base, lsl(index, shift), LeaveCC, cc);
+ return as_vdtr(IsStore, src, Operand(Address(scratch, 0)).toVFPAddr(), cc);
+}
+
+bool MacroAssemblerARMCompat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
+ asMasm().PushFrameDescriptor(FrameType::IonJS); // descriptor_
+ asMasm().Push(ImmPtr(fakeReturnAddr));
+ asMasm().Push(FramePointer);
+ return true;
+}
+
+void MacroAssemblerARMCompat::move32(Imm32 imm, Register dest) {
+ ma_mov(imm, dest);
+}
+
+void MacroAssemblerARMCompat::move32(Register src, Register dest) {
+ ma_mov(src, dest);
+}
+
+void MacroAssemblerARMCompat::movePtr(Register src, Register dest) {
+ ma_mov(src, dest);
+}
+
+void MacroAssemblerARMCompat::movePtr(ImmWord imm, Register dest) {
+ ma_mov(Imm32(imm.value), dest);
+}
+
+void MacroAssemblerARMCompat::movePtr(ImmGCPtr imm, Register dest) {
+ ma_mov(imm, dest);
+}
+
+void MacroAssemblerARMCompat::movePtr(ImmPtr imm, Register dest) {
+ movePtr(ImmWord(uintptr_t(imm.value)), dest);
+}
+
+void MacroAssemblerARMCompat::movePtr(wasm::SymbolicAddress imm,
+ Register dest) {
+ append(wasm::SymbolicAccess(CodeOffset(currentOffset()), imm));
+ ma_movPatchable(Imm32(-1), dest, Always);
+}
+
+void MacroAssemblerARMCompat::load8ZeroExtend(const Address& address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_dataTransferN(IsLoad, 8, false, address.base, Imm32(address.offset), dest,
+ scratch);
+}
+
+void MacroAssemblerARMCompat::load8ZeroExtend(const BaseIndex& src,
+ Register dest) {
+ Register base = src.base;
+ uint32_t scale = Imm32::ShiftOf(src.scale).value;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (src.offset == 0) {
+ ma_ldrb(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), dest);
+ } else {
+ ma_add(base, Imm32(src.offset), scratch, scratch2);
+ ma_ldrb(DTRAddr(scratch, DtrRegImmShift(src.index, LSL, scale)), dest);
+ }
+}
+
+void MacroAssemblerARMCompat::load8SignExtend(const Address& address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_dataTransferN(IsLoad, 8, true, address.base, Imm32(address.offset), dest,
+ scratch);
+}
+
+void MacroAssemblerARMCompat::load8SignExtend(const BaseIndex& src,
+ Register dest) {
+ Register index = src.index;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ // ARMv7 does not have LSL on an index register with an extended load.
+ if (src.scale != TimesOne) {
+ ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
+ index = scratch;
+ }
+
+ if (src.offset != 0) {
+ if (index != scratch) {
+ ma_mov(index, scratch);
+ index = scratch;
+ }
+ ma_add(Imm32(src.offset), index, scratch2);
+ }
+ ma_ldrsb(EDtrAddr(src.base, EDtrOffReg(index)), dest);
+}
+
+void MacroAssemblerARMCompat::load16ZeroExtend(const Address& address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_dataTransferN(IsLoad, 16, false, address.base, Imm32(address.offset), dest,
+ scratch);
+}
+
+void MacroAssemblerARMCompat::load16ZeroExtend(const BaseIndex& src,
+ Register dest) {
+ Register index = src.index;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ // ARMv7 does not have LSL on an index register with an extended load.
+ if (src.scale != TimesOne) {
+ ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
+ index = scratch;
+ }
+
+ if (src.offset != 0) {
+ if (index != scratch) {
+ ma_mov(index, scratch);
+ index = scratch;
+ }
+ ma_add(Imm32(src.offset), index, scratch2);
+ }
+ ma_ldrh(EDtrAddr(src.base, EDtrOffReg(index)), dest);
+}
+
+void MacroAssemblerARMCompat::load16SignExtend(const Address& address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_dataTransferN(IsLoad, 16, true, address.base, Imm32(address.offset), dest,
+ scratch);
+}
+
+void MacroAssemblerARMCompat::load16SignExtend(const BaseIndex& src,
+ Register dest) {
+ Register index = src.index;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ // We don't have LSL on index register yet.
+ if (src.scale != TimesOne) {
+ ma_lsl(Imm32::ShiftOf(src.scale), index, scratch);
+ index = scratch;
+ }
+
+ if (src.offset != 0) {
+ if (index != scratch) {
+ ma_mov(index, scratch);
+ index = scratch;
+ }
+ ma_add(Imm32(src.offset), index, scratch2);
+ }
+ ma_ldrsh(EDtrAddr(src.base, EDtrOffReg(index)), dest);
+}
+
+void MacroAssemblerARMCompat::load32(const Address& address, Register dest) {
+ loadPtr(address, dest);
+}
+
+void MacroAssemblerARMCompat::load32(const BaseIndex& address, Register dest) {
+ loadPtr(address, dest);
+}
+
+void MacroAssemblerARMCompat::load32(AbsoluteAddress address, Register dest) {
+ loadPtr(address, dest);
+}
+
+void MacroAssemblerARMCompat::loadPtr(const Address& address, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(address, dest, scratch);
+}
+
+void MacroAssemblerARMCompat::loadPtr(const BaseIndex& src, Register dest) {
+ Register base = src.base;
+ uint32_t scale = Imm32::ShiftOf(src.scale).value;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (src.offset != 0) {
+ ma_add(base, Imm32(src.offset), scratch, scratch2);
+ ma_ldr(DTRAddr(scratch, DtrRegImmShift(src.index, LSL, scale)), dest);
+ } else {
+ ma_ldr(DTRAddr(base, DtrRegImmShift(src.index, LSL, scale)), dest);
+ }
+}
+
+void MacroAssemblerARMCompat::loadPtr(AbsoluteAddress address, Register dest) {
+ MOZ_ASSERT(dest != pc); // Use dest as a scratch register.
+ movePtr(ImmWord(uintptr_t(address.addr)), dest);
+ loadPtr(Address(dest, 0), dest);
+}
+
+void MacroAssemblerARMCompat::loadPtr(wasm::SymbolicAddress address,
+ Register dest) {
+ MOZ_ASSERT(dest != pc); // Use dest as a scratch register.
+ movePtr(address, dest);
+ loadPtr(Address(dest, 0), dest);
+}
+
+void MacroAssemblerARMCompat::loadPrivate(const Address& address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(ToPayload(address), dest, scratch);
+}
+
+void MacroAssemblerARMCompat::loadDouble(const Address& address,
+ FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_vldr(address, dest, scratch);
+}
+
+void MacroAssemblerARMCompat::loadDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ // VFP instructions don't even support register Base + register Index modes,
+ // so just add the index, then handle the offset like normal.
+ Register base = src.base;
+ Register index = src.index;
+ uint32_t scale = Imm32::ShiftOf(src.scale).value;
+ int32_t offset = src.offset;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ as_add(scratch, base, lsl(index, scale));
+ ma_vldr(Address(scratch, offset), dest, scratch2);
+}
+
+void MacroAssemblerARMCompat::loadFloatAsDouble(const Address& address,
+ FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+
+ VFPRegister rt = dest;
+ ma_vldr(address, rt.singleOverlay(), scratch);
+ as_vcvt(rt, rt.singleOverlay());
+}
+
+void MacroAssemblerARMCompat::loadFloatAsDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ // VFP instructions don't even support register Base + register Index modes,
+ // so just add the index, then handle the offset like normal.
+ Register base = src.base;
+ Register index = src.index;
+ uint32_t scale = Imm32::ShiftOf(src.scale).value;
+ int32_t offset = src.offset;
+ VFPRegister rt = dest;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ as_add(scratch, base, lsl(index, scale));
+ ma_vldr(Address(scratch, offset), rt.singleOverlay(), scratch2);
+ as_vcvt(rt, rt.singleOverlay());
+}
+
+void MacroAssemblerARMCompat::loadFloat32(const Address& address,
+ FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_vldr(address, VFPRegister(dest).singleOverlay(), scratch);
+}
+
+void MacroAssemblerARMCompat::loadFloat32(const BaseIndex& src,
+ FloatRegister dest) {
+ // VFP instructions don't even support register Base + register Index modes,
+ // so just add the index, then handle the offset like normal.
+ Register base = src.base;
+ Register index = src.index;
+ uint32_t scale = Imm32::ShiftOf(src.scale).value;
+ int32_t offset = src.offset;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ as_add(scratch, base, lsl(index, scale));
+ ma_vldr(Address(scratch, offset), VFPRegister(dest).singleOverlay(),
+ scratch2);
+}
+
+void MacroAssemblerARMCompat::store8(Imm32 imm, const Address& address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_mov(imm, scratch2);
+ store8(scratch2, address);
+}
+
+void MacroAssemblerARMCompat::store8(Register src, const Address& address) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_dataTransferN(IsStore, 8, false, address.base, Imm32(address.offset), src,
+ scratch);
+}
+
+void MacroAssemblerARMCompat::store8(Imm32 imm, const BaseIndex& dest) {
+ Register base = dest.base;
+ uint32_t scale = Imm32::ShiftOf(dest.scale).value;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (dest.offset != 0) {
+ ma_add(base, Imm32(dest.offset), scratch, scratch2);
+ ma_mov(imm, scratch2);
+ ma_strb(scratch2, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
+ } else {
+ ma_mov(imm, scratch2);
+ ma_strb(scratch2, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
+ }
+}
+
+void MacroAssemblerARMCompat::store8(Register src, const BaseIndex& dest) {
+ Register base = dest.base;
+ uint32_t scale = Imm32::ShiftOf(dest.scale).value;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (dest.offset != 0) {
+ ma_add(base, Imm32(dest.offset), scratch, scratch2);
+ ma_strb(src, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
+ } else {
+ ma_strb(src, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
+ }
+}
+
+void MacroAssemblerARMCompat::store16(Imm32 imm, const Address& address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_mov(imm, scratch2);
+ store16(scratch2, address);
+}
+
+void MacroAssemblerARMCompat::store16(Register src, const Address& address) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_dataTransferN(IsStore, 16, false, address.base, Imm32(address.offset), src,
+ scratch);
+}
+
+void MacroAssemblerARMCompat::store16(Imm32 imm, const BaseIndex& dest) {
+ Register index = dest.index;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ // We don't have LSL on index register yet.
+ if (dest.scale != TimesOne) {
+ ma_lsl(Imm32::ShiftOf(dest.scale), index, scratch);
+ index = scratch;
+ }
+
+ if (dest.offset != 0) {
+ ma_add(index, Imm32(dest.offset), scratch, scratch2);
+ index = scratch;
+ }
+
+ ma_mov(imm, scratch2);
+ ma_strh(scratch2, EDtrAddr(dest.base, EDtrOffReg(index)));
+}
+
+void MacroAssemblerARMCompat::store16(Register src, const BaseIndex& address) {
+ Register index = address.index;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ // We don't have LSL on index register yet.
+ if (address.scale != TimesOne) {
+ ma_lsl(Imm32::ShiftOf(address.scale), index, scratch);
+ index = scratch;
+ }
+
+ if (address.offset != 0) {
+ ma_add(index, Imm32(address.offset), scratch, scratch2);
+ index = scratch;
+ }
+ ma_strh(src, EDtrAddr(address.base, EDtrOffReg(index)));
+}
+
+void MacroAssemblerARMCompat::store32(Register src, AbsoluteAddress address) {
+ storePtr(src, address);
+}
+
+void MacroAssemblerARMCompat::store32(Register src, const Address& address) {
+ storePtr(src, address);
+}
+
+void MacroAssemblerARMCompat::store32(Imm32 src, const Address& address) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ move32(src, scratch);
+ ma_str(scratch, address, scratch2);
+}
+
+void MacroAssemblerARMCompat::store32(Imm32 imm, const BaseIndex& dest) {
+ Register base = dest.base;
+ uint32_t scale = Imm32::ShiftOf(dest.scale).value;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (dest.offset != 0) {
+ ma_add(base, Imm32(dest.offset), scratch, scratch2);
+ ma_mov(imm, scratch2);
+ ma_str(scratch2, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
+ } else {
+ ma_mov(imm, scratch);
+ ma_str(scratch, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
+ }
+}
+
+void MacroAssemblerARMCompat::store32(Register src, const BaseIndex& dest) {
+ Register base = dest.base;
+ uint32_t scale = Imm32::ShiftOf(dest.scale).value;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (dest.offset != 0) {
+ ma_add(base, Imm32(dest.offset), scratch, scratch2);
+ ma_str(src, DTRAddr(scratch, DtrRegImmShift(dest.index, LSL, scale)));
+ } else {
+ ma_str(src, DTRAddr(base, DtrRegImmShift(dest.index, LSL, scale)));
+ }
+}
+
+void MacroAssemblerARMCompat::storePtr(ImmWord imm, const Address& address) {
+ store32(Imm32(imm.value), address);
+}
+
+void MacroAssemblerARMCompat::storePtr(ImmWord imm, const BaseIndex& address) {
+ store32(Imm32(imm.value), address);
+}
+
+void MacroAssemblerARMCompat::storePtr(ImmPtr imm, const Address& address) {
+ store32(Imm32(uintptr_t(imm.value)), address);
+}
+
+void MacroAssemblerARMCompat::storePtr(ImmPtr imm, const BaseIndex& address) {
+ store32(Imm32(uintptr_t(imm.value)), address);
+}
+
+void MacroAssemblerARMCompat::storePtr(ImmGCPtr imm, const Address& address) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_mov(imm, scratch);
+ ma_str(scratch, address, scratch2);
+}
+
+void MacroAssemblerARMCompat::storePtr(ImmGCPtr imm, const BaseIndex& address) {
+ Register base = address.base;
+ uint32_t scale = Imm32::ShiftOf(address.scale).value;
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (address.offset != 0) {
+ ma_add(base, Imm32(address.offset), scratch, scratch2);
+ ma_mov(imm, scratch2);
+ ma_str(scratch2,
+ DTRAddr(scratch, DtrRegImmShift(address.index, LSL, scale)));
+ } else {
+ ma_mov(imm, scratch);
+ ma_str(scratch, DTRAddr(base, DtrRegImmShift(address.index, LSL, scale)));
+ }
+}
+
+void MacroAssemblerARMCompat::storePtr(Register src, const Address& address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_str(src, address, scratch2);
+}
+
+void MacroAssemblerARMCompat::storePtr(Register src, const BaseIndex& address) {
+ store32(src, address);
+}
+
+void MacroAssemblerARMCompat::storePtr(Register src, AbsoluteAddress dest) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(ImmWord(uintptr_t(dest.addr)), scratch);
+ ma_str(src, DTRAddr(scratch, DtrOffImm(0)));
+}
+
+// Note: this function clobbers the input register.
+void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
+ if (HasVFPv3()) {
+ Label notSplit;
+ {
+ ScratchDoubleScope scratchDouble(*this);
+ MOZ_ASSERT(input != scratchDouble);
+ loadConstantDouble(0.5, scratchDouble);
+
+ ma_vadd(input, scratchDouble, scratchDouble);
+ // Convert the double into an unsigned fixed point value with 24 bits of
+ // precision. The resulting number will look like 0xII.DDDDDD
+ as_vcvtFixed(scratchDouble, false, 24, true);
+ }
+
+ // Move the fixed point value into an integer register.
+ {
+ ScratchFloat32Scope scratchFloat(*this);
+ as_vxfer(output, InvalidReg, scratchFloat.uintOverlay(), FloatToCore);
+ }
+
+ ScratchRegisterScope scratch(*this);
+
+ // See if this value *might* have been an exact integer after adding
+ // 0.5. This tests the 1/2 through 1/16,777,216th places, but 0.5 needs
+ // to be tested out to the 1/140,737,488,355,328th place.
+ ma_tst(output, Imm32(0x00ffffff), scratch);
+ // Convert to a uint8 by shifting out all of the fraction bits.
+ ma_lsr(Imm32(24), output, output);
+ // If any of the bottom 24 bits were non-zero, then we're good, since
+ // this number can't be exactly XX.0
+ ma_b(&notSplit, NonZero);
+ as_vxfer(scratch, InvalidReg, input, FloatToCore);
+ as_cmp(scratch, Imm8(0));
+ // If the lower 32 bits of the double were 0, then this was an exact number,
+ // and it should be even.
+ as_bic(output, output, Imm8(1), LeaveCC, Zero);
+ bind(&notSplit);
+ } else {
+ ScratchDoubleScope scratchDouble(*this);
+ MOZ_ASSERT(input != scratchDouble);
+ loadConstantDouble(0.5, scratchDouble);
+
+ Label outOfRange;
+ ma_vcmpz(input);
+ // Do the add, in place so we can reference it later.
+ ma_vadd(input, scratchDouble, input);
+ // Do the conversion to an integer.
+ as_vcvt(VFPRegister(scratchDouble).uintOverlay(), VFPRegister(input));
+ // Copy the converted value out.
+ as_vxfer(output, InvalidReg, scratchDouble, FloatToCore);
+ as_vmrs(pc);
+ ma_mov(Imm32(0), output, Overflow); // NaN => 0
+ ma_b(&outOfRange, Overflow); // NaN
+ as_cmp(output, Imm8(0xff));
+ ma_mov(Imm32(0xff), output, Above);
+ ma_b(&outOfRange, Above);
+ // Convert it back to see if we got the same value back.
+ as_vcvt(scratchDouble, VFPRegister(scratchDouble).uintOverlay());
+ // Do the check.
+ as_vcmp(scratchDouble, input);
+ as_vmrs(pc);
+ as_bic(output, output, Imm8(1), LeaveCC, Zero);
+ bind(&outOfRange);
+ }
+}
+
+void MacroAssemblerARMCompat::cmp32(Register lhs, Imm32 rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_cmp(lhs, rhs, scratch);
+}
+
+void MacroAssemblerARMCompat::cmp32(Register lhs, Register rhs) {
+ ma_cmp(lhs, rhs);
+}
+
+void MacroAssemblerARMCompat::cmp32(const Address& lhs, Imm32 rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(lhs, scratch, scratch2);
+ ma_cmp(scratch, rhs, scratch2);
+}
+
+void MacroAssemblerARMCompat::cmp32(const Address& lhs, Register rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(lhs, scratch, scratch2);
+ ma_cmp(scratch, rhs);
+}
+
+void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmWord rhs) {
+ cmp32(lhs, Imm32(rhs.value));
+}
+
+void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+}
+
+void MacroAssemblerARMCompat::cmpPtr(Register lhs, Register rhs) {
+ ma_cmp(lhs, rhs);
+}
+
+void MacroAssemblerARMCompat::cmpPtr(Register lhs, ImmGCPtr rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_cmp(lhs, rhs, scratch);
+}
+
+void MacroAssemblerARMCompat::cmpPtr(Register lhs, Imm32 rhs) {
+ cmp32(lhs, rhs);
+}
+
+void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, Register rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(lhs, scratch, scratch2);
+ ma_cmp(scratch, rhs);
+}
+
+void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmWord rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(lhs, scratch, scratch2);
+ ma_cmp(scratch, Imm32(rhs.value), scratch2);
+}
+
+void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+}
+
+void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, ImmGCPtr rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(lhs, scratch, scratch2);
+ ma_cmp(scratch, rhs, scratch2);
+}
+
+void MacroAssemblerARMCompat::cmpPtr(const Address& lhs, Imm32 rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(lhs, scratch, scratch2);
+ ma_cmp(scratch, rhs, scratch2);
+}
+
+void MacroAssemblerARMCompat::setStackArg(Register reg, uint32_t arg) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_dataTransferN(IsStore, 32, true, sp, Imm32(arg * sizeof(intptr_t)), reg,
+ scratch);
+}
+
+void MacroAssemblerARMCompat::minMaxDouble(FloatRegister srcDest,
+ FloatRegister second, bool canBeNaN,
+ bool isMax) {
+ FloatRegister first = srcDest;
+
+ Label nan, equal, returnSecond, done;
+
+ Assembler::Condition cond = isMax ? Assembler::VFP_LessThanOrEqual
+ : Assembler::VFP_GreaterThanOrEqual;
+
+ compareDouble(first, second);
+ // First or second is NaN, result is NaN.
+ ma_b(&nan, Assembler::VFP_Unordered);
+ // Make sure we handle -0 and 0 right.
+ ma_b(&equal, Assembler::VFP_Equal);
+ ma_b(&returnSecond, cond);
+ ma_b(&done);
+
+ // Check for zero.
+ bind(&equal);
+ compareDouble(first, NoVFPRegister);
+ // First wasn't 0 or -0, so just return it.
+ ma_b(&done, Assembler::VFP_NotEqualOrUnordered);
+ // So now both operands are either -0 or 0.
+ if (isMax) {
+ // -0 + -0 = -0 and -0 + 0 = 0.
+ ma_vadd(second, first, first);
+ } else {
+ ma_vneg(first, first);
+ ma_vsub(first, second, first);
+ ma_vneg(first, first);
+ }
+ ma_b(&done);
+
+ bind(&nan);
+ // If the first argument is the NaN, return it; otherwise return the second
+ // operand.
+ compareDouble(first, first);
+ ma_vmov(first, srcDest, Assembler::VFP_Unordered);
+ ma_b(&done, Assembler::VFP_Unordered);
+
+ bind(&returnSecond);
+ ma_vmov(second, srcDest);
+
+ bind(&done);
+}
+
+void MacroAssemblerARMCompat::minMaxFloat32(FloatRegister srcDest,
+ FloatRegister second, bool canBeNaN,
+ bool isMax) {
+ FloatRegister first = srcDest;
+
+ Label nan, equal, returnSecond, done;
+
+ Assembler::Condition cond = isMax ? Assembler::VFP_LessThanOrEqual
+ : Assembler::VFP_GreaterThanOrEqual;
+
+ compareFloat(first, second);
+ // First or second is NaN, result is NaN.
+ ma_b(&nan, Assembler::VFP_Unordered);
+ // Make sure we handle -0 and 0 right.
+ ma_b(&equal, Assembler::VFP_Equal);
+ ma_b(&returnSecond, cond);
+ ma_b(&done);
+
+ // Check for zero.
+ bind(&equal);
+ compareFloat(first, NoVFPRegister);
+ // First wasn't 0 or -0, so just return it.
+ ma_b(&done, Assembler::VFP_NotEqualOrUnordered);
+ // So now both operands are either -0 or 0.
+ if (isMax) {
+ // -0 + -0 = -0 and -0 + 0 = 0.
+ ma_vadd_f32(second, first, first);
+ } else {
+ ma_vneg_f32(first, first);
+ ma_vsub_f32(first, second, first);
+ ma_vneg_f32(first, first);
+ }
+ ma_b(&done);
+
+ bind(&nan);
+ // See comment in minMaxDouble.
+ compareFloat(first, first);
+ ma_vmov_f32(first, srcDest, Assembler::VFP_Unordered);
+ ma_b(&done, Assembler::VFP_Unordered);
+
+ bind(&returnSecond);
+ ma_vmov_f32(second, srcDest);
+
+ bind(&done);
+}
+
+void MacroAssemblerARMCompat::compareDouble(FloatRegister lhs,
+ FloatRegister rhs) {
+ // Compare the doubles, setting vector status flags.
+ if (rhs.isMissing()) {
+ ma_vcmpz(lhs);
+ } else {
+ ma_vcmp(lhs, rhs);
+ }
+
+ // Move vector status bits to normal status flags.
+ as_vmrs(pc);
+}
+
+void MacroAssemblerARMCompat::compareFloat(FloatRegister lhs,
+ FloatRegister rhs) {
+ // Compare the doubles, setting vector status flags.
+ if (rhs.isMissing()) {
+ as_vcmpz(VFPRegister(lhs).singleOverlay());
+ } else {
+ as_vcmp(VFPRegister(lhs).singleOverlay(), VFPRegister(rhs).singleOverlay());
+ }
+
+ // Move vector status bits to normal status flags.
+ as_vmrs(pc);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testInt32(
+ Assembler::Condition cond, const ValueOperand& value) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_INT32));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBoolean(
+ Assembler::Condition cond, const ValueOperand& value) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_BOOLEAN));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testDouble(
+ Assembler::Condition cond, const ValueOperand& value) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ Assembler::Condition actual = (cond == Equal) ? Below : AboveOrEqual;
+ ScratchRegisterScope scratch(asMasm());
+ ma_cmp(value.typeReg(), ImmTag(JSVAL_TAG_CLEAR), scratch);
+ return actual;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testNull(
+ Assembler::Condition cond, const ValueOperand& value) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_NULL));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testUndefined(
+ Assembler::Condition cond, const ValueOperand& value) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ ma_cmp(value.typeReg(), ImmType(JSVAL_TYPE_UNDEFINED));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testString(
+ Assembler::Condition cond, const ValueOperand& value) {
+ return testString(cond, value.typeReg());
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testSymbol(
+ Assembler::Condition cond, const ValueOperand& value) {
+ return testSymbol(cond, value.typeReg());
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBigInt(
+ Assembler::Condition cond, const ValueOperand& value) {
+ return testBigInt(cond, value.typeReg());
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testObject(
+ Assembler::Condition cond, const ValueOperand& value) {
+ return testObject(cond, value.typeReg());
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testNumber(
+ Assembler::Condition cond, const ValueOperand& value) {
+ return testNumber(cond, value.typeReg());
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testMagic(
+ Assembler::Condition cond, const ValueOperand& value) {
+ return testMagic(cond, value.typeReg());
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testPrimitive(
+ Assembler::Condition cond, const ValueOperand& value) {
+ return testPrimitive(cond, value.typeReg());
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testGCThing(
+ Assembler::Condition cond, const ValueOperand& value) {
+ return testGCThing(cond, value.typeReg());
+}
+
+// Register-based tests.
+Assembler::Condition MacroAssemblerARMCompat::testInt32(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_INT32));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBoolean(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testNull(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_NULL));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testUndefined(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testString(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_STRING));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testSymbol(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBigInt(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testObject(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testMagic(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testPrimitive(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
+ return cond == Equal ? Below : AboveOrEqual;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testGCThing(
+ Assembler::Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testGCThing(
+ Assembler::Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ ma_cmp(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testMagic(
+ Assembler::Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testInt32(
+ Assembler::Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_INT32));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testDouble(
+ Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testDouble(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBoolean(
+ Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testBoolean(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testNull(Condition cond,
+ const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testNull(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testUndefined(
+ Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testUndefined(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testString(
+ Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testString(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testSymbol(
+ Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testSymbol(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBigInt(
+ Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testBigInt(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testObject(
+ Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testObject(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testNumber(
+ Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ return testNumber(cond, tag);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testDouble(Condition cond,
+ Register tag) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ Condition actual = (cond == Equal) ? Below : AboveOrEqual;
+ ma_cmp(tag, ImmTag(JSVAL_TAG_CLEAR));
+ return actual;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testNumber(Condition cond,
+ Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp(tag, ImmTag(JS::detail::ValueUpperInclNumberTag));
+ return cond == Equal ? BelowOrEqual : Above;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testUndefined(
+ Condition cond, const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testNull(Condition cond,
+ const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_NULL));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBoolean(
+ Condition cond, const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testString(Condition cond,
+ const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_STRING));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testSymbol(Condition cond,
+ const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBigInt(Condition cond,
+ const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testInt32(Condition cond,
+ const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_INT32));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testObject(Condition cond,
+ const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testDouble(Condition cond,
+ const BaseIndex& src) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Assembler::Condition actual = (cond == Equal) ? Below : AboveOrEqual;
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(src, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_CLEAR));
+ return actual;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testMagic(
+ Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ ma_cmp(tag, ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testGCThing(
+ Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ Register tag = extractTag(address, scratch);
+ ma_cmp(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+}
+
+// Unboxing code.
+void MacroAssemblerARMCompat::unboxNonDouble(const ValueOperand& operand,
+ Register dest, JSValueType type) {
+ auto movPayloadToDest = [&]() {
+ if (operand.payloadReg() != dest) {
+ ma_mov(operand.payloadReg(), dest, LeaveCC);
+ }
+ };
+ if (!JitOptions.spectreValueMasking) {
+ movPayloadToDest();
+ return;
+ }
+
+ // Spectre mitigation: We zero the payload if the tag does not match the
+ // expected type and if this is a pointer type.
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movPayloadToDest();
+ return;
+ }
+
+ // We zero the destination register and move the payload into it if
+ // the tag corresponds to the given type.
+ ma_cmp(operand.typeReg(), ImmType(type));
+ movPayloadToDest();
+ ma_mov(Imm32(0), dest, NotEqual);
+}
+
+void MacroAssemblerARMCompat::unboxNonDouble(const Address& src, Register dest,
+ JSValueType type) {
+ ScratchRegisterScope scratch(asMasm());
+ if (!JitOptions.spectreValueMasking) {
+ ma_ldr(ToPayload(src), dest, scratch);
+ return;
+ }
+
+ // Spectre mitigation: We zero the payload if the tag does not match the
+ // expected type and if this is a pointer type.
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ ma_ldr(ToPayload(src), dest, scratch);
+ return;
+ }
+
+ // We zero the destination register and move the payload into it if
+ // the tag corresponds to the given type.
+ ma_ldr(ToType(src), scratch, scratch);
+ ma_cmp(scratch, ImmType(type));
+ ma_ldr(ToPayload(src), dest, scratch, Offset, Equal);
+ ma_mov(Imm32(0), dest, NotEqual);
+}
+
+void MacroAssemblerARMCompat::unboxNonDouble(const BaseIndex& src,
+ Register dest, JSValueType type) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_alu(src.base, lsl(src.index, src.scale), scratch2, OpAdd);
+ Address value(scratch2, src.offset);
+ unboxNonDouble(value, dest, type);
+}
+
+void MacroAssemblerARMCompat::unboxDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ MOZ_ASSERT(dest.isDouble());
+ as_vxfer(operand.payloadReg(), operand.typeReg(), VFPRegister(dest),
+ CoreToFloat);
+}
+
+void MacroAssemblerARMCompat::unboxDouble(const Address& src,
+ FloatRegister dest) {
+ MOZ_ASSERT(dest.isDouble());
+ loadDouble(src, dest);
+}
+
+void MacroAssemblerARMCompat::unboxDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ MOZ_ASSERT(dest.isDouble());
+ loadDouble(src, dest);
+}
+
+void MacroAssemblerARMCompat::unboxValue(const ValueOperand& src,
+ AnyRegister dest, JSValueType type) {
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.payloadReg(), dest.fpu());
+ ma_b(&end);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else {
+ unboxNonDouble(src, dest.gpr(), type);
+ }
+}
+
+void MacroAssemblerARMCompat::boxDouble(FloatRegister src,
+ const ValueOperand& dest,
+ FloatRegister) {
+ as_vxfer(dest.payloadReg(), dest.typeReg(), VFPRegister(src), FloatToCore);
+}
+
+void MacroAssemblerARMCompat::boxNonDouble(JSValueType type, Register src,
+ const ValueOperand& dest) {
+ if (src != dest.payloadReg()) {
+ ma_mov(src, dest.payloadReg());
+ }
+ ma_mov(ImmType(type), dest.typeReg());
+}
+
+void MacroAssemblerARMCompat::boolValueToDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ VFPRegister d = VFPRegister(dest);
+ loadConstantDouble(1.0, dest);
+ as_cmp(operand.payloadReg(), Imm8(0));
+ // If the source is 0, then subtract the dest from itself, producing 0.
+ as_vsub(d, d, d, Equal);
+}
+
+void MacroAssemblerARMCompat::int32ValueToDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ // Transfer the integral value to a floating point register.
+ VFPRegister vfpdest = VFPRegister(dest);
+ as_vxfer(operand.payloadReg(), InvalidReg, vfpdest.sintOverlay(),
+ CoreToFloat);
+ // Convert the value to a double.
+ as_vcvt(vfpdest, vfpdest.sintOverlay());
+}
+
+void MacroAssemblerARMCompat::boolValueToFloat32(const ValueOperand& operand,
+ FloatRegister dest) {
+ VFPRegister d = VFPRegister(dest).singleOverlay();
+ loadConstantFloat32(1.0, dest);
+ as_cmp(operand.payloadReg(), Imm8(0));
+ // If the source is 0, then subtract the dest from itself, producing 0.
+ as_vsub(d, d, d, Equal);
+}
+
+void MacroAssemblerARMCompat::int32ValueToFloat32(const ValueOperand& operand,
+ FloatRegister dest) {
+ // Transfer the integral value to a floating point register.
+ VFPRegister vfpdest = VFPRegister(dest).singleOverlay();
+ as_vxfer(operand.payloadReg(), InvalidReg, vfpdest.sintOverlay(),
+ CoreToFloat);
+ // Convert the value to a float.
+ as_vcvt(vfpdest, vfpdest.sintOverlay());
+}
+
+void MacroAssemblerARMCompat::loadConstantFloat32(float f, FloatRegister dest) {
+ ma_vimm_f32(f, dest);
+}
+
+void MacroAssemblerARMCompat::loadInt32OrDouble(const Address& src,
+ FloatRegister dest) {
+ Label notInt32, end;
+
+ // If it's an int, convert to a double.
+ {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ ma_ldr(ToType(src), scratch, scratch2);
+ asMasm().branchTestInt32(Assembler::NotEqual, scratch, &notInt32);
+ ma_ldr(ToPayload(src), scratch, scratch2);
+ convertInt32ToDouble(scratch, dest);
+ ma_b(&end);
+ }
+
+ // Not an int, just load as double.
+ bind(&notInt32);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_vldr(src, dest, scratch);
+ }
+ bind(&end);
+}
+
+void MacroAssemblerARMCompat::loadInt32OrDouble(Register base, Register index,
+ FloatRegister dest,
+ int32_t shift) {
+ Label notInt32, end;
+
+ static_assert(NUNBOX32_PAYLOAD_OFFSET == 0);
+
+ ScratchRegisterScope scratch(asMasm());
+
+ // If it's an int, convert it to double.
+ ma_alu(base, lsl(index, shift), scratch, OpAdd);
+
+ // Since we only have one scratch register, we need to stomp over it with
+ // the tag.
+ ma_ldr(DTRAddr(scratch, DtrOffImm(NUNBOX32_TYPE_OFFSET)), scratch);
+ asMasm().branchTestInt32(Assembler::NotEqual, scratch, &notInt32);
+
+ // Implicitly requires NUNBOX32_PAYLOAD_OFFSET == 0: no offset provided
+ ma_ldr(DTRAddr(base, DtrRegImmShift(index, LSL, shift)), scratch);
+ convertInt32ToDouble(scratch, dest);
+ ma_b(&end);
+
+ // Not an int, just load as double.
+ bind(&notInt32);
+ // First, recompute the offset that had been stored in the scratch register
+ // since the scratch register was overwritten loading in the type.
+ ma_alu(base, lsl(index, shift), scratch, OpAdd);
+ ma_vldr(VFPAddr(scratch, VFPOffImm(0)), dest);
+ bind(&end);
+}
+
+void MacroAssemblerARMCompat::loadConstantDouble(double dp,
+ FloatRegister dest) {
+ ma_vimm(dp, dest);
+}
+
+// Treat the value as a boolean, and set condition codes accordingly.
+Assembler::Condition MacroAssemblerARMCompat::testInt32Truthy(
+ bool truthy, const ValueOperand& operand) {
+ ma_tst(operand.payloadReg(), operand.payloadReg());
+ return truthy ? NonZero : Zero;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBooleanTruthy(
+ bool truthy, const ValueOperand& operand) {
+ ma_tst(operand.payloadReg(), operand.payloadReg());
+ return truthy ? NonZero : Zero;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testDoubleTruthy(
+ bool truthy, FloatRegister reg) {
+ as_vcmpz(VFPRegister(reg));
+ as_vmrs(pc);
+ as_cmp(r0, O2Reg(r0), Overflow);
+ return truthy ? NonZero : Zero;
+}
+
+Register MacroAssemblerARMCompat::extractObject(const Address& address,
+ Register scratch) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(ToPayload(address), scratch, scratch2);
+ return scratch;
+}
+
+Register MacroAssemblerARMCompat::extractTag(const Address& address,
+ Register scratch) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(ToType(address), scratch, scratch2);
+ return scratch;
+}
+
+Register MacroAssemblerARMCompat::extractTag(const BaseIndex& address,
+ Register scratch) {
+ ma_alu(address.base, lsl(address.index, address.scale), scratch, OpAdd,
+ LeaveCC);
+ return extractTag(Address(scratch, address.offset), scratch);
+}
+
+/////////////////////////////////////////////////////////////////
+// X86/X64-common (ARM too now) interface.
+/////////////////////////////////////////////////////////////////
+void MacroAssemblerARMCompat::storeValue(ValueOperand val, const Address& dst) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_str(val.payloadReg(), ToPayload(dst), scratch2);
+ ma_str(val.typeReg(), ToType(dst), scratch2);
+}
+
+void MacroAssemblerARMCompat::storeValue(ValueOperand val,
+ const BaseIndex& dest) {
+ ScratchRegisterScope scratch(asMasm());
+
+ if (isValueDTRDCandidate(val) && Abs(dest.offset) <= 255) {
+ Register tmpIdx;
+ if (dest.offset == 0) {
+ if (dest.scale == TimesOne) {
+ tmpIdx = dest.index;
+ } else {
+ ma_lsl(Imm32(dest.scale), dest.index, scratch);
+ tmpIdx = scratch;
+ }
+ ma_strd(val.payloadReg(), val.typeReg(),
+ EDtrAddr(dest.base, EDtrOffReg(tmpIdx)));
+ } else {
+ ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
+ ma_strd(val.payloadReg(), val.typeReg(),
+ EDtrAddr(scratch, EDtrOffImm(dest.offset)));
+ }
+ } else {
+ ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
+ storeValue(val, Address(scratch, dest.offset));
+ }
+}
+
+void MacroAssemblerARMCompat::loadValue(const BaseIndex& addr,
+ ValueOperand val) {
+ ScratchRegisterScope scratch(asMasm());
+
+ if (isValueDTRDCandidate(val) && Abs(addr.offset) <= 255) {
+ Register tmpIdx;
+ if (addr.offset == 0) {
+ if (addr.scale == TimesOne) {
+ // If the offset register is the same as one of the destination
+ // registers, LDRD's behavior is undefined. Use the scratch
+ // register to avoid this.
+ if (val.aliases(addr.index)) {
+ ma_mov(addr.index, scratch);
+ tmpIdx = scratch;
+ } else {
+ tmpIdx = addr.index;
+ }
+ } else {
+ ma_lsl(Imm32(addr.scale), addr.index, scratch);
+ tmpIdx = scratch;
+ }
+ ma_ldrd(EDtrAddr(addr.base, EDtrOffReg(tmpIdx)), val.payloadReg(),
+ val.typeReg());
+ } else {
+ ma_alu(addr.base, lsl(addr.index, addr.scale), scratch, OpAdd);
+ ma_ldrd(EDtrAddr(scratch, EDtrOffImm(addr.offset)), val.payloadReg(),
+ val.typeReg());
+ }
+ } else {
+ ma_alu(addr.base, lsl(addr.index, addr.scale), scratch, OpAdd);
+ loadValue(Address(scratch, addr.offset), val);
+ }
+}
+
+void MacroAssemblerARMCompat::loadValue(Address src, ValueOperand val) {
+ // TODO: copy this code into a generic function that acts on all sequences
+ // of memory accesses
+ if (isValueDTRDCandidate(val)) {
+ // If the value we want is in two consecutive registers starting with an
+ // even register, they can be combined as a single ldrd.
+ int offset = src.offset;
+ if (offset < 256 && offset > -256) {
+ ma_ldrd(EDtrAddr(src.base, EDtrOffImm(src.offset)), val.payloadReg(),
+ val.typeReg());
+ return;
+ }
+ }
+ // If the value is lower than the type, then we may be able to use an ldm
+ // instruction.
+
+ if (val.payloadReg().code() < val.typeReg().code()) {
+ if (src.offset <= 4 && src.offset >= -8 && (src.offset & 3) == 0) {
+ // Turns out each of the 4 value -8, -4, 0, 4 corresponds exactly
+ // with one of LDM{DB, DA, IA, IB}
+ DTMMode mode;
+ switch (src.offset) {
+ case -8:
+ mode = DB;
+ break;
+ case -4:
+ mode = DA;
+ break;
+ case 0:
+ mode = IA;
+ break;
+ case 4:
+ mode = IB;
+ break;
+ default:
+ MOZ_CRASH("Bogus Offset for LoadValue as DTM");
+ }
+ startDataTransferM(IsLoad, src.base, mode);
+ transferReg(val.payloadReg());
+ transferReg(val.typeReg());
+ finishDataTransfer();
+ return;
+ }
+ }
+
+ loadUnalignedValue(src, val);
+}
+
+void MacroAssemblerARMCompat::loadUnalignedValue(const Address& src,
+ ValueOperand dest) {
+ Address payload = ToPayload(src);
+ Address type = ToType(src);
+
+ // Ensure that loading the payload does not erase the pointer to the Value
+ // in memory.
+ if (type.base != dest.payloadReg()) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(payload, dest.payloadReg(), scratch2);
+ ma_ldr(type, dest.typeReg(), scratch2);
+ } else {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(type, dest.typeReg(), scratch2);
+ ma_ldr(payload, dest.payloadReg(), scratch2);
+ }
+}
+
+void MacroAssemblerARMCompat::tagValue(JSValueType type, Register payload,
+ ValueOperand dest) {
+ MOZ_ASSERT(dest.typeReg() != dest.payloadReg());
+ if (payload != dest.payloadReg()) {
+ ma_mov(payload, dest.payloadReg());
+ }
+ ma_mov(ImmType(type), dest.typeReg());
+}
+
+void MacroAssemblerARMCompat::pushValue(ValueOperand val) {
+ ma_push(val.typeReg());
+ ma_push(val.payloadReg());
+}
+
+void MacroAssemblerARMCompat::pushValue(const Address& addr) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ ma_ldr(ToType(addr), scratch, scratch2);
+ ma_push(scratch);
+ ma_ldr(ToPayloadAfterStackPush(addr), scratch, scratch2);
+ ma_push(scratch);
+}
+
+void MacroAssemblerARMCompat::popValue(ValueOperand val) {
+ ma_pop(val.payloadReg());
+ ma_pop(val.typeReg());
+}
+
+void MacroAssemblerARMCompat::storePayload(const Value& val,
+ const Address& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (val.isGCThing()) {
+ ma_mov(ImmGCPtr(val.toGCThing()), scratch);
+ } else {
+ ma_mov(Imm32(val.toNunboxPayload()), scratch);
+ }
+ ma_str(scratch, ToPayload(dest), scratch2);
+}
+
+void MacroAssemblerARMCompat::storePayload(Register src, const Address& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_str(src, ToPayload(dest), scratch);
+}
+
+void MacroAssemblerARMCompat::storePayload(const Value& val,
+ const BaseIndex& dest) {
+ unsigned shift = ScaleToShift(dest.scale);
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (val.isGCThing()) {
+ ma_mov(ImmGCPtr(val.toGCThing()), scratch);
+ } else {
+ ma_mov(Imm32(val.toNunboxPayload()), scratch);
+ }
+
+ // If NUNBOX32_PAYLOAD_OFFSET is not zero, the memory operand [base + index
+ // << shift + imm] cannot be encoded into a single instruction, and cannot
+ // be integrated into the as_dtr call.
+ static_assert(NUNBOX32_PAYLOAD_OFFSET == 0);
+
+ // If an offset is used, modify the base so that a [base + index << shift]
+ // instruction format can be used.
+ if (dest.offset != 0) {
+ ma_add(dest.base, Imm32(dest.offset), dest.base, scratch2);
+ }
+
+ as_dtr(IsStore, 32, Offset, scratch,
+ DTRAddr(dest.base, DtrRegImmShift(dest.index, LSL, shift)));
+
+ // Restore the original value of the base, if necessary.
+ if (dest.offset != 0) {
+ ma_sub(dest.base, Imm32(dest.offset), dest.base, scratch);
+ }
+}
+
+void MacroAssemblerARMCompat::storePayload(Register src,
+ const BaseIndex& dest) {
+ unsigned shift = ScaleToShift(dest.scale);
+ MOZ_ASSERT(shift < 32);
+
+ ScratchRegisterScope scratch(asMasm());
+
+ // If NUNBOX32_PAYLOAD_OFFSET is not zero, the memory operand [base + index
+ // << shift + imm] cannot be encoded into a single instruction, and cannot
+ // be integrated into the as_dtr call.
+ static_assert(NUNBOX32_PAYLOAD_OFFSET == 0);
+
+ // Save/restore the base if the BaseIndex has an offset, as above.
+ if (dest.offset != 0) {
+ ma_add(dest.base, Imm32(dest.offset), dest.base, scratch);
+ }
+
+ // Technically, shift > -32 can be handle by changing LSL to ASR, but should
+ // never come up, and this is one less code path to get wrong.
+ as_dtr(IsStore, 32, Offset, src,
+ DTRAddr(dest.base, DtrRegImmShift(dest.index, LSL, shift)));
+
+ if (dest.offset != 0) {
+ ma_sub(dest.base, Imm32(dest.offset), dest.base, scratch);
+ }
+}
+
+void MacroAssemblerARMCompat::storeTypeTag(ImmTag tag, const Address& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ ma_mov(tag, scratch);
+ ma_str(scratch, ToType(dest), scratch2);
+}
+
+void MacroAssemblerARMCompat::storeTypeTag(ImmTag tag, const BaseIndex& dest) {
+ Register base = dest.base;
+ Register index = dest.index;
+ unsigned shift = ScaleToShift(dest.scale);
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ MOZ_ASSERT(base != scratch && base != scratch2);
+ MOZ_ASSERT(index != scratch && index != scratch2);
+
+ ma_add(base, Imm32(dest.offset + NUNBOX32_TYPE_OFFSET), scratch2, scratch);
+ ma_mov(tag, scratch);
+ ma_str(scratch, DTRAddr(scratch2, DtrRegImmShift(index, LSL, shift)));
+}
+
+void MacroAssemblerARM::ma_call(ImmPtr dest) {
+ ma_movPatchable(dest, CallReg, Always);
+ as_blx(CallReg);
+}
+
+void MacroAssemblerARMCompat::breakpoint() { as_bkpt(); }
+
+void MacroAssemblerARMCompat::simulatorStop(const char* msg) {
+#ifdef JS_SIMULATOR_ARM
+ MOZ_ASSERT(sizeof(char*) == 4);
+ writeInst(0xefffffff);
+ writeInst((int)msg);
+#endif
+}
+
+void MacroAssemblerARMCompat::ensureDouble(const ValueOperand& source,
+ FloatRegister dest, Label* failure) {
+ Label isDouble, done;
+ asMasm().branchTestDouble(Assembler::Equal, source.typeReg(), &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, source.typeReg(), failure);
+
+ convertInt32ToDouble(source.payloadReg(), dest);
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerARMCompat::breakpoint(Condition cc) {
+ ma_ldr(DTRAddr(r12, DtrRegImmShift(r12, LSL, 0, IsDown)), r12, Offset, cc);
+}
+
+void MacroAssemblerARMCompat::checkStackAlignment() {
+ asMasm().assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssemblerARMCompat::handleFailureWithHandlerTail(
+ Label* profilerExitTail, Label* bailoutTail) {
+ // Reserve space for exception information.
+ int size = (sizeof(ResumeFromException) + 7) & ~7;
+
+ Imm8 size8(size);
+ as_sub(sp, sp, size8);
+ ma_mov(sp, r0);
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(r1);
+ asMasm().passABIArg(r0);
+ asMasm().callWithABI<Fn, HandleException>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfKind()), r0, scratch);
+ }
+
+ asMasm().branch32(Assembler::Equal, r0,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, r0,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, r0,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::WasmCatch),
+ &wasmCatch);
+
+ breakpoint(); // Invalid kind.
+
+ // No exception handler. Load the error value, restore state and return from
+ // the entry frame.
+ bind(&entryFrame);
+ asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfFramePointer()), r11,
+ scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
+ scratch);
+ }
+
+ // We're going to be returning by the ion calling convention, which returns
+ // by ??? (for now, I think ldr pc, [sp]!)
+ as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(4)));
+
+ // If we found a catch handler, this must be a baseline frame. Restore state
+ // and jump to the catch block.
+ bind(&catch_);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfTarget()), r0, scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfFramePointer()), r11,
+ scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
+ scratch);
+ }
+ jump(r0);
+
+ // If we found a finally block, this must be a baseline frame. Push two
+ // values expected by the finally block: the exception and BooleanValue(true).
+ bind(&finally);
+ ValueOperand exception = ValueOperand(r1, r2);
+ loadValue(Operand(sp, ResumeFromException::offsetOfException()), exception);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfTarget()), r0, scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfFramePointer()), r11,
+ scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
+ scratch);
+ }
+
+ pushValue(exception);
+ pushValue(BooleanValue(true));
+ jump(r0);
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfFramePointer()), r11,
+ scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
+ scratch);
+ }
+ loadValue(Address(r11, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ jump(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(Address(sp, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfFramePointer()), r11,
+ scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
+ scratch);
+ }
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ // Test if profiler enabled.
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ ma_mov(r11, sp);
+ pop(r11);
+ ret();
+
+ // If we are bailing out to baseline to handle an exception, jump to the
+ // bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
+ bind(&bailout);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfBailoutInfo()), r2,
+ scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
+ scratch);
+ ma_mov(Imm32(1), ReturnReg);
+ }
+ jump(bailoutTail);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfFramePointer()), r11,
+ scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
+ scratch);
+ }
+ as_dtr(IsLoad, 32, PostIndex, pc, DTRAddr(sp, DtrOffImm(4)));
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+ {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(Address(sp, ResumeFromException::offsetOfTarget()), r1, scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfFramePointer()), r11,
+ scratch);
+ ma_ldr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp,
+ scratch);
+ }
+ jump(r1);
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testStringTruthy(
+ bool truthy, const ValueOperand& value) {
+ Register string = value.payloadReg();
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ ma_dtr(IsLoad, string, Imm32(JSString::offsetOfLength()), scratch, scratch2);
+ as_cmp(scratch, Imm8(0));
+ return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+Assembler::Condition MacroAssemblerARMCompat::testBigIntTruthy(
+ bool truthy, const ValueOperand& value) {
+ Register bi = value.payloadReg();
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ ma_dtr(IsLoad, bi, Imm32(BigInt::offsetOfDigitLength()), scratch, scratch2);
+ as_cmp(scratch, Imm8(0));
+ return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+void MacroAssemblerARMCompat::floor(FloatRegister input, Register output,
+ Label* bail) {
+ Label handleZero;
+ Label handleNeg;
+ Label fin;
+
+ ScratchDoubleScope scratchDouble(asMasm());
+
+ compareDouble(input, NoVFPRegister);
+ ma_b(&handleZero, Assembler::Equal);
+ ma_b(&handleNeg, Assembler::Signed);
+ // NaN is always a bail condition, just bail directly.
+ ma_b(bail, Assembler::Overflow);
+
+ // The argument is a positive number, truncation is the path to glory. Since
+ // it is known to be > 0.0, explicitly convert to a larger range, then a
+ // value that rounds to INT_MAX is explicitly different from an argument
+ // that clamps to INT_MAX.
+ ma_vcvt_F64_U32(input, scratchDouble.uintOverlay());
+ ma_vxfer(scratchDouble.uintOverlay(), output);
+ ma_mov(output, output, SetCC);
+ ma_b(bail, Signed);
+ ma_b(&fin);
+
+ bind(&handleZero);
+ // Move the top word of the double into the output reg, if it is non-zero,
+ // then the original value was -0.0.
+ as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
+ as_cmp(output, Imm8(0));
+ ma_b(bail, NonZero);
+ ma_b(&fin);
+
+ bind(&handleNeg);
+ // Negative case, negate, then start dancing.
+ ma_vneg(input, input);
+ ma_vcvt_F64_U32(input, scratchDouble.uintOverlay());
+ ma_vxfer(scratchDouble.uintOverlay(), output);
+ ma_vcvt_U32_F64(scratchDouble.uintOverlay(), scratchDouble);
+ compareDouble(scratchDouble, input);
+ as_add(output, output, Imm8(1), LeaveCC, NotEqual);
+ // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
+ // result will still be a negative number.
+ as_rsb(output, output, Imm8(0), SetCC);
+ // Flip the negated input back to its original value.
+ ma_vneg(input, input);
+ // If the result looks non-negative, then this value didn't actually fit
+ // into the int range, and special handling is required. Zero is also caught
+ // by this case, but floor of a negative number should never be zero.
+ ma_b(bail, NotSigned);
+
+ bind(&fin);
+}
+
+void MacroAssemblerARMCompat::floorf(FloatRegister input, Register output,
+ Label* bail) {
+ Label handleZero;
+ Label handleNeg;
+ Label fin;
+ compareFloat(input, NoVFPRegister);
+ ma_b(&handleZero, Assembler::Equal);
+ ma_b(&handleNeg, Assembler::Signed);
+ // NaN is always a bail condition, just bail directly.
+ ma_b(bail, Assembler::Overflow);
+
+ // The argument is a positive number, truncation is the path to glory; Since
+ // it is known to be > 0.0, explicitly convert to a larger range, then a
+ // value that rounds to INT_MAX is explicitly different from an argument
+ // that clamps to INT_MAX.
+ {
+ ScratchFloat32Scope scratch(asMasm());
+ ma_vcvt_F32_U32(input, scratch.uintOverlay());
+ ma_vxfer(VFPRegister(scratch).uintOverlay(), output);
+ }
+ ma_mov(output, output, SetCC);
+ ma_b(bail, Signed);
+ ma_b(&fin);
+
+ bind(&handleZero);
+ // Move the top word of the double into the output reg, if it is non-zero,
+ // then the original value was -0.0.
+ as_vxfer(output, InvalidReg, VFPRegister(input).singleOverlay(), FloatToCore,
+ Always, 0);
+ as_cmp(output, Imm8(0));
+ ma_b(bail, NonZero);
+ ma_b(&fin);
+
+ bind(&handleNeg);
+ // Negative case, negate, then start dancing.
+ {
+ ScratchFloat32Scope scratch(asMasm());
+ ma_vneg_f32(input, input);
+ ma_vcvt_F32_U32(input, scratch.uintOverlay());
+ ma_vxfer(VFPRegister(scratch).uintOverlay(), output);
+ ma_vcvt_U32_F32(scratch.uintOverlay(), scratch);
+ compareFloat(scratch, input);
+ as_add(output, output, Imm8(1), LeaveCC, NotEqual);
+ }
+ // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
+ // result will still be a negative number.
+ as_rsb(output, output, Imm8(0), SetCC);
+ // Flip the negated input back to its original value.
+ ma_vneg_f32(input, input);
+ // If the result looks non-negative, then this value didn't actually fit
+ // into the int range, and special handling is required. Zero is also caught
+ // by this case, but floor of a negative number should never be zero.
+ ma_b(bail, NotSigned);
+
+ bind(&fin);
+}
+
+void MacroAssemblerARMCompat::ceil(FloatRegister input, Register output,
+ Label* bail) {
+ Label handleZero;
+ Label handlePos;
+ Label fin;
+
+ compareDouble(input, NoVFPRegister);
+ // NaN is always a bail condition, just bail directly.
+ ma_b(bail, Assembler::Overflow);
+ ma_b(&handleZero, Assembler::Equal);
+ ma_b(&handlePos, Assembler::NotSigned);
+
+ ScratchDoubleScope scratchDouble(asMasm());
+
+ // We are in the ]-Inf; 0[ range
+ // If we are in the ]-1; 0[ range => bailout
+ loadConstantDouble(-1.0, scratchDouble);
+ compareDouble(input, scratchDouble);
+ ma_b(bail, Assembler::GreaterThan);
+
+ // We are in the ]-Inf; -1] range: ceil(x) == -floor(-x) and floor can be
+ // computed with direct truncation here (x > 0).
+ ma_vneg(input, scratchDouble);
+ FloatRegister ScratchUIntReg = scratchDouble.uintOverlay();
+ ma_vcvt_F64_U32(scratchDouble, ScratchUIntReg);
+ ma_vxfer(ScratchUIntReg, output);
+ ma_neg(output, output, SetCC);
+ ma_b(bail, NotSigned);
+ ma_b(&fin);
+
+ // Test for 0.0 / -0.0: if the top word of the input double is not zero,
+ // then it was -0 and we need to bail out.
+ bind(&handleZero);
+ as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
+ as_cmp(output, Imm8(0));
+ ma_b(bail, NonZero);
+ ma_b(&fin);
+
+ // We are in the ]0; +inf] range: truncate integer values, maybe add 1 for
+ // non integer values, maybe bail if overflow.
+ bind(&handlePos);
+ ma_vcvt_F64_U32(input, ScratchUIntReg);
+ ma_vxfer(ScratchUIntReg, output);
+ ma_vcvt_U32_F64(ScratchUIntReg, scratchDouble);
+ compareDouble(scratchDouble, input);
+ as_add(output, output, Imm8(1), LeaveCC, NotEqual);
+ // Bail out if the add overflowed or the result is non positive.
+ ma_mov(output, output, SetCC);
+ ma_b(bail, Signed);
+ ma_b(bail, Zero);
+
+ bind(&fin);
+}
+
+void MacroAssemblerARMCompat::ceilf(FloatRegister input, Register output,
+ Label* bail) {
+ Label handleZero;
+ Label handlePos;
+ Label fin;
+
+ compareFloat(input, NoVFPRegister);
+ // NaN is always a bail condition, just bail directly.
+ ma_b(bail, Assembler::Overflow);
+ ma_b(&handleZero, Assembler::Equal);
+ ma_b(&handlePos, Assembler::NotSigned);
+
+ // We are in the ]-Inf; 0[ range
+ // If we are in the ]-1; 0[ range => bailout
+ {
+ ScratchFloat32Scope scratch(asMasm());
+ loadConstantFloat32(-1.f, scratch);
+ compareFloat(input, scratch);
+ ma_b(bail, Assembler::GreaterThan);
+ }
+
+ // We are in the ]-Inf; -1] range: ceil(x) == -floor(-x) and floor can be
+ // computed with direct truncation here (x > 0).
+ {
+ ScratchDoubleScope scratchDouble(asMasm());
+ FloatRegister scratchFloat = scratchDouble.asSingle();
+ FloatRegister scratchUInt = scratchDouble.uintOverlay();
+
+ ma_vneg_f32(input, scratchFloat);
+ ma_vcvt_F32_U32(scratchFloat, scratchUInt);
+ ma_vxfer(scratchUInt, output);
+ ma_neg(output, output, SetCC);
+ ma_b(bail, NotSigned);
+ ma_b(&fin);
+ }
+
+ // Test for 0.0 / -0.0: if the top word of the input double is not zero,
+ // then it was -0 and we need to bail out.
+ bind(&handleZero);
+ as_vxfer(output, InvalidReg, VFPRegister(input).singleOverlay(), FloatToCore,
+ Always, 0);
+ as_cmp(output, Imm8(0));
+ ma_b(bail, NonZero);
+ ma_b(&fin);
+
+ // We are in the ]0; +inf] range: truncate integer values, maybe add 1 for
+ // non integer values, maybe bail if overflow.
+ bind(&handlePos);
+ {
+ ScratchDoubleScope scratchDouble(asMasm());
+ FloatRegister scratchFloat = scratchDouble.asSingle();
+ FloatRegister scratchUInt = scratchDouble.uintOverlay();
+
+ ma_vcvt_F32_U32(input, scratchUInt);
+ ma_vxfer(scratchUInt, output);
+ ma_vcvt_U32_F32(scratchUInt, scratchFloat);
+ compareFloat(scratchFloat, input);
+ as_add(output, output, Imm8(1), LeaveCC, NotEqual);
+
+ // Bail on overflow or non-positive result.
+ ma_mov(output, output, SetCC);
+ ma_b(bail, Signed);
+ ma_b(bail, Zero);
+ }
+
+ bind(&fin);
+}
+
+CodeOffset MacroAssemblerARMCompat::toggledJump(Label* label) {
+ // Emit a B that can be toggled to a CMP. See ToggleToJmp(), ToggleToCmp().
+ BufferOffset b = ma_b(label, Always);
+ CodeOffset ret(b.getOffset());
+ return ret;
+}
+
+CodeOffset MacroAssemblerARMCompat::toggledCall(JitCode* target, bool enabled) {
+ BufferOffset bo = nextOffset();
+ addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ ScratchRegisterScope scratch(asMasm());
+ ma_movPatchable(ImmPtr(target->raw()), scratch, Always);
+ if (enabled) {
+ ma_blx(scratch);
+ } else {
+ ma_nop();
+ }
+ return CodeOffset(bo.getOffset());
+}
+
+void MacroAssemblerARMCompat::round(FloatRegister input, Register output,
+ Label* bail, FloatRegister tmp) {
+ Label handleZero;
+ Label handleNeg;
+ Label fin;
+
+ ScratchDoubleScope scratchDouble(asMasm());
+
+ // Do a compare based on the original value, then do most other things based
+ // on the shifted value.
+ ma_vcmpz(input);
+ // Since we already know the sign bit, flip all numbers to be positive,
+ // stored in tmp.
+ ma_vabs(input, tmp);
+ as_vmrs(pc);
+ ma_b(&handleZero, Assembler::Equal);
+ ma_b(&handleNeg, Assembler::Signed);
+ // NaN is always a bail condition, just bail directly.
+ ma_b(bail, Assembler::Overflow);
+
+ // The argument is a positive number, truncation is the path to glory; Since
+ // it is known to be > 0.0, explicitly convert to a larger range, then a
+ // value that rounds to INT_MAX is explicitly different from an argument
+ // that clamps to INT_MAX.
+
+ // Add the biggest number less than 0.5 (not 0.5, because adding that to
+ // the biggest number less than 0.5 would undesirably round up to 1), and
+ // store the result into tmp.
+ loadConstantDouble(GetBiggestNumberLessThan(0.5), scratchDouble);
+ ma_vadd(scratchDouble, tmp, tmp);
+
+ ma_vcvt_F64_U32(tmp, scratchDouble.uintOverlay());
+ ma_vxfer(VFPRegister(scratchDouble).uintOverlay(), output);
+ ma_mov(output, output, SetCC);
+ ma_b(bail, Signed);
+ ma_b(&fin);
+
+ bind(&handleZero);
+ // Move the top word of the double into the output reg, if it is non-zero,
+ // then the original value was -0.0
+ as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
+ as_cmp(output, Imm8(0));
+ ma_b(bail, NonZero);
+ ma_b(&fin);
+
+ bind(&handleNeg);
+ // Negative case, negate, then start dancing. This number may be positive,
+ // since we added 0.5.
+
+ // Add 0.5 to negative numbers, store the result into tmp
+ loadConstantDouble(0.5, scratchDouble);
+ ma_vadd(scratchDouble, tmp, tmp);
+
+ ma_vcvt_F64_U32(tmp, scratchDouble.uintOverlay());
+ ma_vxfer(VFPRegister(scratchDouble).uintOverlay(), output);
+
+ // -output is now a correctly rounded value, unless the original value was
+ // exactly halfway between two integers, at which point, it has been rounded
+ // away from zero, when it should be rounded towards \infty.
+ ma_vcvt_U32_F64(scratchDouble.uintOverlay(), scratchDouble);
+ compareDouble(scratchDouble, tmp);
+ as_sub(output, output, Imm8(1), LeaveCC, Equal);
+ // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
+ // result will still be a negative number.
+ as_rsb(output, output, Imm8(0), SetCC);
+
+ // If the result looks non-negative, then this value didn't actually fit
+ // into the int range, and special handling is required, or it was zero,
+ // which means the result is actually -0.0 which also requires special
+ // handling.
+ ma_b(bail, NotSigned);
+
+ bind(&fin);
+}
+
+void MacroAssemblerARMCompat::roundf(FloatRegister input, Register output,
+ Label* bail, FloatRegister tmp) {
+ Label handleZero;
+ Label handleNeg;
+ Label fin;
+
+ ScratchFloat32Scope scratchFloat(asMasm());
+
+ // Do a compare based on the original value, then do most other things based
+ // on the shifted value.
+ compareFloat(input, NoVFPRegister);
+ ma_b(&handleZero, Assembler::Equal);
+ ma_b(&handleNeg, Assembler::Signed);
+
+ // NaN is always a bail condition, just bail directly.
+ ma_b(bail, Assembler::Overflow);
+
+ // The argument is a positive number, truncation is the path to glory; Since
+ // it is known to be > 0.0, explicitly convert to a larger range, then a
+ // value that rounds to INT_MAX is explicitly different from an argument
+ // that clamps to INT_MAX.
+
+ // Add the biggest number less than 0.5f (not 0.5f, because adding that to
+ // the biggest number less than 0.5f would undesirably round up to 1), and
+ // store the result into tmp.
+ loadConstantFloat32(GetBiggestNumberLessThan(0.5f), scratchFloat);
+ ma_vadd_f32(scratchFloat, input, tmp);
+
+ // Note: it doesn't matter whether x + .5 === x or not here, as it doesn't
+ // affect the semantics of the float to unsigned conversion (in particular,
+ // we are not applying any fixup after the operation).
+ ma_vcvt_F32_U32(tmp, scratchFloat.uintOverlay());
+ ma_vxfer(VFPRegister(scratchFloat).uintOverlay(), output);
+ ma_mov(output, output, SetCC);
+ ma_b(bail, Signed);
+ ma_b(&fin);
+
+ bind(&handleZero);
+
+ // Move the whole float32 into the output reg, if it is non-zero, then the
+ // original value was -0.0.
+ as_vxfer(output, InvalidReg, input, FloatToCore, Always, 0);
+ as_cmp(output, Imm8(0));
+ ma_b(bail, NonZero);
+ ma_b(&fin);
+
+ bind(&handleNeg);
+
+ // Add 0.5 to negative numbers, storing the result into tmp.
+ ma_vneg_f32(input, tmp);
+ loadConstantFloat32(0.5f, scratchFloat);
+ ma_vadd_f32(tmp, scratchFloat, scratchFloat);
+
+ // Adding 0.5 to a float input has chances to yield the wrong result, if
+ // the input is too large. In this case, skip the -1 adjustment made below.
+ compareFloat(scratchFloat, tmp);
+
+ // Negative case, negate, then start dancing. This number may be positive,
+ // since we added 0.5.
+ // /!\ The conditional jump afterwards depends on these two instructions
+ // *not* setting the status flags. They need to not change after the
+ // comparison above.
+ ma_vcvt_F32_U32(scratchFloat, tmp.uintOverlay());
+ ma_vxfer(VFPRegister(tmp).uintOverlay(), output);
+
+ Label flipSign;
+ ma_b(&flipSign, Equal);
+
+ // -output is now a correctly rounded value, unless the original value was
+ // exactly halfway between two integers, at which point, it has been rounded
+ // away from zero, when it should be rounded towards \infty.
+ ma_vcvt_U32_F32(tmp.uintOverlay(), tmp);
+ compareFloat(tmp, scratchFloat);
+ as_sub(output, output, Imm8(1), LeaveCC, Equal);
+
+ // Negate the output. Since INT_MIN < -INT_MAX, even after adding 1, the
+ // result will still be a negative number.
+ bind(&flipSign);
+ as_rsb(output, output, Imm8(0), SetCC);
+
+ // If the result looks non-negative, then this value didn't actually fit
+ // into the int range, and special handling is required, or it was zero,
+ // which means the result is actually -0.0 which also requires special
+ // handling.
+ ma_b(bail, NotSigned);
+
+ bind(&fin);
+}
+
+void MacroAssemblerARMCompat::trunc(FloatRegister input, Register output,
+ Label* bail) {
+ Label handleZero;
+ Label handlePos;
+ Label fin;
+
+ compareDouble(input, NoVFPRegister);
+ // NaN is always a bail condition, just bail directly.
+ ma_b(bail, Assembler::Overflow);
+ ma_b(&handleZero, Assembler::Equal);
+ ma_b(&handlePos, Assembler::NotSigned);
+
+ ScratchDoubleScope scratchDouble(asMasm());
+
+ // We are in the ]-Inf; 0[ range
+ // If we are in the ]-1; 0[ range => bailout
+ loadConstantDouble(-1.0, scratchDouble);
+ compareDouble(input, scratchDouble);
+ ma_b(bail, Assembler::GreaterThan);
+
+ // We are in the ]-Inf; -1] range: trunc(x) == -floor(-x) and floor can be
+ // computed with direct truncation here (x > 0).
+ ma_vneg(input, scratchDouble);
+ ma_vcvt_F64_U32(scratchDouble, scratchDouble.uintOverlay());
+ ma_vxfer(scratchDouble.uintOverlay(), output);
+ ma_neg(output, output, SetCC);
+ ma_b(bail, NotSigned);
+ ma_b(&fin);
+
+ // Test for 0.0 / -0.0: if the top word of the input double is not zero,
+ // then it was -0 and we need to bail out.
+ bind(&handleZero);
+ as_vxfer(output, InvalidReg, input, FloatToCore, Always, 1);
+ as_cmp(output, Imm8(0));
+ ma_b(bail, NonZero);
+ ma_b(&fin);
+
+ // We are in the ]0; +inf] range: truncation is the path to glory. Since
+ // it is known to be > 0.0, explicitly convert to a larger range, then a
+ // value that rounds to INT_MAX is explicitly different from an argument
+ // that clamps to INT_MAX.
+ bind(&handlePos);
+ ma_vcvt_F64_U32(input, scratchDouble.uintOverlay());
+ ma_vxfer(scratchDouble.uintOverlay(), output);
+ ma_mov(output, output, SetCC);
+ ma_b(bail, Signed);
+
+ bind(&fin);
+}
+
+void MacroAssemblerARMCompat::truncf(FloatRegister input, Register output,
+ Label* bail) {
+ Label handleZero;
+ Label handlePos;
+ Label fin;
+
+ compareFloat(input, NoVFPRegister);
+ // NaN is always a bail condition, just bail directly.
+ ma_b(bail, Assembler::Overflow);
+ ma_b(&handleZero, Assembler::Equal);
+ ma_b(&handlePos, Assembler::NotSigned);
+
+ // We are in the ]-Inf; 0[ range
+ // If we are in the ]-1; 0[ range => bailout
+ {
+ ScratchFloat32Scope scratch(asMasm());
+ loadConstantFloat32(-1.f, scratch);
+ compareFloat(input, scratch);
+ ma_b(bail, Assembler::GreaterThan);
+ }
+
+ // We are in the ]-Inf; -1] range: trunc(x) == -floor(-x) and floor can be
+ // computed with direct truncation here (x > 0).
+ {
+ ScratchDoubleScope scratchDouble(asMasm());
+ FloatRegister scratchFloat = scratchDouble.asSingle();
+ FloatRegister scratchUInt = scratchDouble.uintOverlay();
+
+ ma_vneg_f32(input, scratchFloat);
+ ma_vcvt_F32_U32(scratchFloat, scratchUInt);
+ ma_vxfer(scratchUInt, output);
+ ma_neg(output, output, SetCC);
+ ma_b(bail, NotSigned);
+ ma_b(&fin);
+ }
+
+ // Test for 0.0 / -0.0: if the top word of the input double is not zero,
+ // then it was -0 and we need to bail out.
+ bind(&handleZero);
+ as_vxfer(output, InvalidReg, VFPRegister(input).singleOverlay(), FloatToCore,
+ Always, 0);
+ as_cmp(output, Imm8(0));
+ ma_b(bail, NonZero);
+ ma_b(&fin);
+
+ // We are in the ]0; +inf] range: truncation is the path to glory; Since
+ // it is known to be > 0.0, explicitly convert to a larger range, then a
+ // value that rounds to INT_MAX is explicitly different from an argument
+ bind(&handlePos);
+ {
+ // The argument is a positive number,
+ // that clamps to INT_MAX.
+ {
+ ScratchFloat32Scope scratch(asMasm());
+ ma_vcvt_F32_U32(input, scratch.uintOverlay());
+ ma_vxfer(VFPRegister(scratch).uintOverlay(), output);
+ }
+ ma_mov(output, output, SetCC);
+ ma_b(bail, Signed);
+ }
+
+ bind(&fin);
+}
+
+void MacroAssemblerARMCompat::profilerEnterFrame(Register framePtr,
+ Register scratch) {
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr,
+ Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr),
+ Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerARMCompat::profilerExitFrame() {
+ jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+MacroAssembler& MacroAssemblerARM::asMasm() {
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerARM::asMasm() const {
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+MacroAssembler& MacroAssemblerARMCompat::asMasm() {
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerARMCompat::asMasm() const {
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) {
+ ScratchRegisterScope scratch(*this);
+ if (imm32.value) {
+ ma_sub(imm32, sp, scratch);
+ }
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// MacroAssembler high-level usage.
+
+void MacroAssembler::flush() { Assembler::flush(); }
+
+void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); }
+
+// ===============================================================
+// Stack manipulation functions.
+
+size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
+ return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
+}
+
+void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
+ mozilla::DebugOnly<size_t> framePushedInitial = framePushed();
+
+ int32_t diffF = set.fpus().getPushSizeInBytes();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+
+ if (set.gprs().size() > 1) {
+ adjustFrame(diffG);
+ startDataTransferM(IsStore, StackPointer, DB, WriteBack);
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
+ ++iter) {
+ diffG -= sizeof(intptr_t);
+ transferReg(*iter);
+ }
+ finishDataTransfer();
+ } else {
+ reserveStack(diffG);
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
+ ++iter) {
+ diffG -= sizeof(intptr_t);
+ storePtr(*iter, Address(StackPointer, diffG));
+ }
+ }
+ MOZ_ASSERT(diffG == 0);
+
+ // It's possible that the logic is just fine as it is if the reduced set
+ // maps SIMD pairs to plain doubles and transferMultipleByRuns() stores
+ // and loads doubles.
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ adjustFrame(diffF);
+ diffF += transferMultipleByRuns(set.fpus(), IsStore, StackPointer, DB);
+ MOZ_ASSERT(diffF == 0);
+
+ MOZ_ASSERT(framePushed() - framePushedInitial ==
+ PushRegsInMaskSizeInBytes(set));
+}
+
+void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
+ Register scratch) {
+ mozilla::DebugOnly<size_t> offsetInitial = dest.offset;
+
+ int32_t diffF = set.fpus().getPushSizeInBytes();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+
+ MOZ_ASSERT(dest.offset >= diffF + diffG);
+
+ if (set.gprs().size() > 1) {
+ computeEffectiveAddress(dest, scratch);
+
+ startDataTransferM(IsStore, scratch, DB, WriteBack);
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
+ ++iter) {
+ diffG -= sizeof(intptr_t);
+ dest.offset -= sizeof(intptr_t);
+ transferReg(*iter);
+ }
+ finishDataTransfer();
+ } else {
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
+ ++iter) {
+ diffG -= sizeof(intptr_t);
+ dest.offset -= sizeof(intptr_t);
+ storePtr(*iter, dest);
+ }
+ }
+ MOZ_ASSERT(diffG == 0);
+ (void)diffG;
+
+ // See above.
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ MOZ_ASSERT(diffF >= 0);
+ if (diffF > 0) {
+ computeEffectiveAddress(dest, scratch);
+ diffF += transferMultipleByRuns(set.fpus(), IsStore, scratch, DB);
+ }
+
+ MOZ_ASSERT(diffF == 0);
+
+ // "The amount of space actually used does not exceed what
+ // `PushRegsInMaskSizeInBytes` claims will be used."
+ MOZ_ASSERT(offsetInitial - dest.offset <= PushRegsInMaskSizeInBytes(set));
+}
+
+void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
+ LiveRegisterSet ignore) {
+ mozilla::DebugOnly<size_t> framePushedInitial = framePushed();
+
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+ int32_t diffF = set.fpus().getPushSizeInBytes();
+ const int32_t reservedG = diffG;
+ const int32_t reservedF = diffF;
+
+ // See above.
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ // ARM can load multiple registers at once, but only if we want back all
+ // the registers we previously saved to the stack.
+ if (ignore.emptyFloat()) {
+ diffF -= transferMultipleByRuns(set.fpus(), IsLoad, StackPointer, IA);
+ adjustFrame(-reservedF);
+ } else {
+ LiveFloatRegisterSet fpset(set.fpus().reduceSetForPush());
+ LiveFloatRegisterSet fpignore(ignore.fpus().reduceSetForPush());
+ for (FloatRegisterBackwardIterator iter(fpset); iter.more(); ++iter) {
+ diffF -= (*iter).size();
+ if (!fpignore.has(*iter)) {
+ loadDouble(Address(StackPointer, diffF), *iter);
+ }
+ }
+ freeStack(reservedF);
+ }
+ MOZ_ASSERT(diffF == 0);
+
+ if (set.gprs().size() > 1 && ignore.emptyGeneral()) {
+ startDataTransferM(IsLoad, StackPointer, IA, WriteBack);
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
+ ++iter) {
+ diffG -= sizeof(intptr_t);
+ transferReg(*iter);
+ }
+ finishDataTransfer();
+ adjustFrame(-reservedG);
+ } else {
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
+ ++iter) {
+ diffG -= sizeof(intptr_t);
+ if (!ignore.has(*iter)) {
+ loadPtr(Address(StackPointer, diffG), *iter);
+ }
+ }
+ freeStack(reservedG);
+ }
+ MOZ_ASSERT(diffG == 0);
+
+ MOZ_ASSERT(framePushedInitial - framePushed() ==
+ PushRegsInMaskSizeInBytes(set));
+}
+
+void MacroAssembler::Push(Register reg) {
+ push(reg);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const Imm32 imm) {
+ push(imm);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const ImmWord imm) {
+ push(imm);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const ImmPtr imm) {
+ Push(ImmWord(uintptr_t(imm.value)));
+}
+
+void MacroAssembler::Push(const ImmGCPtr ptr) {
+ push(ptr);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(FloatRegister reg) {
+ VFPRegister r = VFPRegister(reg);
+ ma_vpush(VFPRegister(reg));
+ adjustFrame(r.size());
+}
+
+void MacroAssembler::PushBoxed(FloatRegister reg) {
+ MOZ_ASSERT(reg.isDouble());
+ Push(reg);
+}
+
+void MacroAssembler::Pop(Register reg) {
+ ma_pop(reg);
+ adjustFrame(-sizeof(intptr_t));
+}
+
+void MacroAssembler::Pop(FloatRegister reg) {
+ ma_vpop(reg);
+ adjustFrame(-reg.size());
+}
+
+void MacroAssembler::Pop(const ValueOperand& val) {
+ popValue(val);
+ adjustFrame(-sizeof(Value));
+}
+
+void MacroAssembler::PopStackPtr() {
+ as_dtr(IsLoad, 32, Offset, sp, DTRAddr(sp, DtrOffImm(0)));
+ adjustFrame(-sizeof(intptr_t));
+}
+
+// ===============================================================
+// Simple call functions.
+
+CodeOffset MacroAssembler::call(Register reg) {
+ as_blx(reg);
+ return CodeOffset(currentOffset());
+}
+
+CodeOffset MacroAssembler::call(Label* label) {
+ // For now, assume that it'll be nearby.
+ as_bl(label, Always);
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::call(ImmWord imm) { call(ImmPtr((void*)imm.value)); }
+
+void MacroAssembler::call(ImmPtr imm) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, imm, RelocationKind::HARDCODED);
+ ma_call(imm);
+}
+
+CodeOffset MacroAssembler::call(wasm::SymbolicAddress imm) {
+ movePtr(imm, CallReg);
+ return call(CallReg);
+}
+
+void MacroAssembler::call(const Address& addr) {
+ loadPtr(addr, CallReg);
+ call(CallReg);
+}
+
+void MacroAssembler::call(JitCode* c) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ScratchRegisterScope scratch(*this);
+ ma_movPatchable(ImmPtr(c->raw()), scratch, Always);
+ callJitNoProfiler(scratch);
+}
+
+CodeOffset MacroAssembler::callWithPatch() {
+ // The caller ensures that the call is always in range using thunks (below)
+ // as necessary.
+ as_bl(BOffImm(), Always, /* documentation */ nullptr);
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
+ BufferOffset inst(callerOffset - 4);
+ BOffImm off = BufferOffset(calleeOffset).diffB<BOffImm>(inst);
+ MOZ_RELEASE_ASSERT(!off.isInvalid(),
+ "Failed to insert necessary far jump islands");
+ as_bl(off, Always, inst);
+}
+
+CodeOffset MacroAssembler::farJumpWithPatch() {
+ static_assert(32 * 1024 * 1024 - JumpImmediateRange >
+ wasm::MaxFuncs * 3 * sizeof(Instruction),
+ "always enough space for thunks");
+
+ // The goal of the thunk is to be able to jump to any address without the
+ // usual 32MiB branch range limitation. Additionally, to make the thunk
+ // simple to use, the thunk does not use the constant pool or require
+ // patching an absolute address. Instead, a relative offset is used which
+ // can be patched during compilation.
+
+ // Inhibit pools since these three words must be contiguous so that the offset
+ // calculations below are valid.
+ AutoForbidPoolsAndNops afp(this, 3);
+
+ // When pc is used, the read value is the address of the instruction + 8.
+ // This is exactly the address of the uint32 word we want to load.
+ ScratchRegisterScope scratch(*this);
+ ma_ldr(DTRAddr(pc, DtrOffImm(0)), scratch);
+
+ // Branch by making pc the destination register.
+ ma_add(pc, scratch, pc, LeaveCC, Always);
+
+ // Allocate space which will be patched by patchFarJump().
+ CodeOffset farJump(currentOffset());
+ writeInst(UINT32_MAX);
+
+ return farJump;
+}
+
+void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
+ uint32_t* u32 =
+ reinterpret_cast<uint32_t*>(editSrc(BufferOffset(farJump.offset())));
+ MOZ_ASSERT(*u32 == UINT32_MAX);
+
+ uint32_t addOffset = farJump.offset() - 4;
+ MOZ_ASSERT(editSrc(BufferOffset(addOffset))->is<InstALU>());
+
+ // When pc is read as the operand of the add, its value is the address of
+ // the add instruction + 8.
+ *u32 = (targetOffset - addOffset) - 8;
+}
+
+CodeOffset MacroAssembler::nopPatchableToCall() {
+ AutoForbidPoolsAndNops afp(this,
+ /* max number of instructions in scope = */ 1);
+ ma_nop();
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
+ uint8_t* inst = call - 4;
+ MOZ_ASSERT(reinterpret_cast<Instruction*>(inst)->is<InstBLImm>() ||
+ reinterpret_cast<Instruction*>(inst)->is<InstNOP>());
+
+ new (inst) InstBLImm(BOffImm(target - inst), Assembler::Always);
+}
+
+void MacroAssembler::patchCallToNop(uint8_t* call) {
+ uint8_t* inst = call - 4;
+ MOZ_ASSERT(reinterpret_cast<Instruction*>(inst)->is<InstBLImm>() ||
+ reinterpret_cast<Instruction*>(inst)->is<InstNOP>());
+ new (inst) InstNOP();
+}
+
+void MacroAssembler::pushReturnAddress() { push(lr); }
+
+void MacroAssembler::popReturnAddress() { pop(lr); }
+
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ ma_mov(sp, scratch);
+ // Force sp to be aligned.
+ as_bic(sp, sp, Imm8(ABIStackAlignment - 1));
+ ma_push(scratch);
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ if (dynamicAlignment_) {
+ // sizeof(intptr_t) accounts for the saved stack pointer pushed by
+ // setupUnalignedABICall.
+ stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t),
+ ABIStackAlignment);
+ } else {
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(
+ stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+ }
+
+ *stackAdjust = stackForCall;
+ reserveStack(stackForCall);
+
+ // Position all arguments.
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_) {
+ return;
+ }
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ assertStackAlignment(ABIStackAlignment);
+
+ // Save the lr register if we need to preserve it.
+ if (secondScratchReg_ != lr) {
+ ma_mov(lr, secondScratchReg_);
+ }
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool callFromWasm) {
+ if (secondScratchReg_ != lr) {
+ ma_mov(secondScratchReg_, lr);
+ }
+
+ // Calls to native functions in wasm pass through a thunk which already
+ // fixes up the return value for us.
+ if (!callFromWasm && !UseHardFpABI()) {
+ switch (result) {
+ case MoveOp::DOUBLE:
+ // Move double from r0/r1 to ReturnFloatReg.
+ ma_vxfer(r0, r1, ReturnDoubleReg);
+ break;
+ case MoveOp::FLOAT32:
+ // Move float32 from r0 to ReturnFloatReg.
+ ma_vxfer(r0, ReturnFloat32Reg);
+ break;
+ case MoveOp::GENERAL:
+ break;
+ default:
+ MOZ_CRASH("unexpected callWithABI result");
+ }
+ }
+
+ freeStack(stackAdjust);
+
+ if (dynamicAlignment_) {
+ // While the x86 supports pop esp, on ARM that isn't well defined, so
+ // just do it manually.
+ as_dtr(IsLoad, 32, Offset, sp, DTRAddr(sp, DtrOffImm(0)));
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ // Load the callee in r12, as above.
+ ma_mov(fun, r12);
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(r12);
+ callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ // Load the callee in r12, no instruction between the ldr and call should
+ // clobber it. Note that we can't use fun.base because it may be one of the
+ // IntArg registers clobbered before the call.
+ {
+ ScratchRegisterScope scratch(*this);
+ ma_ldr(fun, r12, scratch);
+ }
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(r12);
+ callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Jit Frames.
+
+uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
+ // On ARM any references to the pc, adds an additional 8 to it, which
+ // correspond to 2 instructions of 4 bytes. Thus we use an additional nop
+ // to pad until we reach the pushed pc.
+ //
+ // Note: In practice this should not be necessary, as this fake return
+ // address is never used for resuming any execution. Thus theoriticaly we
+ // could just do a Push(pc), and ignore the nop as well as the pool.
+ enterNoPool(2);
+ DebugOnly<uint32_t> offsetBeforePush = currentOffset();
+ Push(pc); // actually pushes $pc + 8.
+ ma_nop();
+ uint32_t pseudoReturnOffset = currentOffset();
+ leaveNoPool();
+
+ MOZ_ASSERT_IF(!oom(), pseudoReturnOffset - offsetBeforePush == 8);
+ return pseudoReturnOffset;
+}
+
+void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
+ ExitFrameType type) {
+ enterFakeExitFrame(cxreg, scratch, type);
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ if (reg.gpr() != dest.payloadReg()) {
+ mov(reg.gpr(), dest.payloadReg());
+ }
+ mov(ImmWord(MIRTypeToTag(type)), dest.typeReg());
+ return;
+ }
+
+ ScratchFloat32Scope scratch(*this);
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ ma_vxfer(freg, dest.payloadReg(), dest.typeReg());
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ Register s0 = src.typeReg();
+ Register s1 = src.payloadReg();
+ Register d0 = dest.typeReg();
+ Register d1 = dest.payloadReg();
+
+ // Either one or both of the source registers could be the same as a
+ // destination register.
+ if (s1 == d0) {
+ if (s0 == d1) {
+ // If both are, this is just a swap of two registers.
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(d1 != scratch);
+ MOZ_ASSERT(d0 != scratch);
+ ma_mov(d1, scratch);
+ ma_mov(d0, d1);
+ ma_mov(scratch, d0);
+ return;
+ }
+ // If only one is, copy that source first.
+ std::swap(s0, s1);
+ std::swap(d0, d1);
+ }
+
+ if (s0 != d0) {
+ ma_mov(s0, d0);
+ }
+ if (s1 != d1) {
+ ma_mov(s1, d1);
+ }
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ ma_mov(Imm32(src.toNunboxTag()), dest.typeReg());
+ if (src.isGCThing()) {
+ ma_mov(ImmGCPtr(src.toGCThing()), dest.payloadReg());
+ } else {
+ ma_mov(Imm32(src.toNunboxPayload()), dest.payloadReg());
+ }
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+ ma_lsr(Imm32(gc::ChunkShift), ptr, buffer);
+ ma_lsl(Imm32(gc::ChunkShift), buffer, buffer);
+ load32(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+ Register temp, Label* label) {
+ Maybe<SecondScratchRegisterScope> scratch2;
+ if (temp == Register::Invalid()) {
+ scratch2.emplace(*this);
+ temp = scratch2.ref();
+ }
+
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(ptr != temp);
+
+ ma_lsr(Imm32(gc::ChunkShift), ptr, temp);
+ ma_lsl(Imm32(gc::ChunkShift), temp, temp);
+ loadPtr(Address(temp, gc::ChunkStoreBufferOffset), temp);
+ branchPtr(InvertCondition(cond), temp, ImmWord(0), label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label done;
+
+ branchTestGCThing(Assembler::NotEqual, address,
+ cond == Assembler::Equal ? &done : label);
+
+ loadPtr(ToPayload(address), temp);
+ SecondScratchRegisterScope scratch2(*this);
+ branchPtrInNurseryChunk(cond, temp, scratch2, label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label done;
+
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+ branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ // If cond == NotEqual, branch when a.payload != b.payload || a.tag !=
+ // b.tag. If the payloads are equal, compare the tags. If the payloads are
+ // not equal, short circuit true (NotEqual).
+ //
+ // If cand == Equal, branch when a.payload == b.payload && a.tag == b.tag.
+ // If the payloads are equal, compare the tags. If the payloads are not
+ // equal, short circuit false (NotEqual).
+ ScratchRegisterScope scratch(*this);
+
+ if (rhs.isGCThing()) {
+ ma_cmp(lhs.payloadReg(), ImmGCPtr(rhs.toGCThing()), scratch);
+ } else {
+ ma_cmp(lhs.payloadReg(), Imm32(rhs.toNunboxPayload()), scratch);
+ }
+ ma_cmp(lhs.typeReg(), Imm32(rhs.toNunboxTag()), scratch, Equal);
+ ma_b(label, cond);
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ storeDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ // Store the type tag.
+ storeTypeTag(ImmType(ValueTypeFromMIRType(valueType)), dest);
+
+ // Store the payload.
+ if (value.constant()) {
+ storePayload(value.value(), dest);
+ } else {
+ storePayload(value.reg().typedReg().gpr(), dest);
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+CodeOffset MacroAssembler::wasmTrapInstruction() {
+ return CodeOffset(as_illegal_trap().getOffset());
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* ok) {
+ as_cmp(index, O2Reg(boundsCheckLimit));
+ as_b(ok, cond);
+ if (JitOptions.spectreIndexMasking) {
+ ma_mov(boundsCheckLimit, index, LeaveCC, cond);
+ }
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* ok) {
+ ScratchRegisterScope scratch(*this);
+ ma_ldr(DTRAddr(boundsCheckLimit.base, DtrOffImm(boundsCheckLimit.offset)),
+ scratch);
+ as_cmp(index, O2Reg(scratch));
+ as_b(ok, cond);
+ if (JitOptions.spectreIndexMasking) {
+ ma_mov(scratch, index, LeaveCC, cond);
+ }
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok) {
+ Label notOk;
+ cmp32(index.high, Imm32(0));
+ j(Assembler::NonZero, &notOk);
+ wasmBoundsCheck32(cond, index.low, boundsCheckLimit.low, ok);
+ bind(&notOk);
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok) {
+ Label notOk;
+ cmp32(index.high, Imm32(0));
+ j(Assembler::NonZero, &notOk);
+ wasmBoundsCheck32(cond, index.low, boundsCheckLimit, ok);
+ bind(&notOk);
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ wasmTruncateToInt32(input, output, MIRType::Double, /* isUnsigned= */ true,
+ isSaturating, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ wasmTruncateToInt32(input, output, MIRType::Double, /* isUnsigned= */ false,
+ isSaturating, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ wasmTruncateToInt32(input, output, MIRType::Float32, /* isUnsigned= */ true,
+ isSaturating, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ wasmTruncateToInt32(input, output, MIRType::Float32, /* isUnsigned= */ false,
+ isSaturating, oolEntry);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToIntCheck(input, MIRType::Float32, MIRType::Int32,
+ flags, rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToIntCheck(input, MIRType::Double, MIRType::Int32, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToIntCheck(input, MIRType::Float32, MIRType::Int64,
+ flags, rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToIntCheck(input, MIRType::Double, MIRType::Int64, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, AnyRegister output) {
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output,
+ Register64::Invalid());
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output) {
+ MOZ_ASSERT_IF(access.isAtomic(), access.byteSize() <= 4);
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(), output);
+}
+
+void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Register memoryBase,
+ Register ptr, Register ptrScratch) {
+ wasmStoreImpl(access, value, Register64::Invalid(), memoryBase, ptr,
+ ptrScratch);
+}
+
+void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch) {
+ MOZ_ASSERT(!access.isAtomic());
+ wasmStoreImpl(access, AnyRegister(), value, memoryBase, ptr, ptrScratch);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+static Register ComputePointerForAtomic(MacroAssembler& masm,
+ const BaseIndex& src, Register r) {
+ Register base = src.base;
+ Register index = src.index;
+ uint32_t scale = Imm32::ShiftOf(src.scale).value;
+ int32_t offset = src.offset;
+
+ ScratchRegisterScope scratch(masm);
+
+ masm.as_add(r, base, lsl(index, scale));
+ if (offset != 0) {
+ masm.ma_add(r, Imm32(offset), r, scratch);
+ }
+ return r;
+}
+
+static Register ComputePointerForAtomic(MacroAssembler& masm,
+ const Address& src, Register r) {
+ ScratchRegisterScope scratch(masm);
+ if (src.offset == 0) {
+ return src.base;
+ }
+ masm.ma_add(src.base, Imm32(src.offset), r, scratch);
+ return r;
+}
+
+// General algorithm:
+//
+// ... ptr, <addr> ; compute address of item
+// dmb
+// L0 ldrex* output, [ptr]
+// sxt* output, output, 0 ; sign-extend if applicable
+// *xt* tmp, oldval, 0 ; sign-extend or zero-extend if applicable
+// cmp output, tmp
+// bne L1 ; failed - values are different
+// strex* tmp, newval, [ptr]
+// cmp tmp, 1
+// beq L0 ; failed - location is dirty, retry
+// L1 dmb
+//
+// Discussion here: http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html.
+// However note that that discussion uses 'isb' as the trailing fence.
+// I've not quite figured out why, and I've gone with dmb here which
+// is safe. Also see the LLVM source, which uses 'dmb ish' generally.
+// (Apple's Swift CPU apparently handles ish in a non-default, faster
+// way.)
+
+template <typename T>
+static void CompareExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ const T& mem, Register oldval, Register newval,
+ Register output) {
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ MOZ_ASSERT(nbytes <= 4);
+
+ Label again;
+ Label done;
+
+ SecondScratchRegisterScope scratch2(masm);
+ Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ ScratchRegisterScope scratch(masm);
+
+ // NOTE: the generated code must match the assembly code in gen_cmpxchg in
+ // GenerateAtomicOperations.py
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ BufferOffset firstAccess;
+ switch (nbytes) {
+ case 1:
+ firstAccess = masm.as_ldrexb(output, ptr);
+ if (signExtend) {
+ masm.as_sxtb(output, output, 0);
+ masm.as_sxtb(scratch, oldval, 0);
+ } else {
+ masm.as_uxtb(scratch, oldval, 0);
+ }
+ break;
+ case 2:
+ firstAccess = masm.as_ldrexh(output, ptr);
+ if (signExtend) {
+ masm.as_sxth(output, output, 0);
+ masm.as_sxth(scratch, oldval, 0);
+ } else {
+ masm.as_uxth(scratch, oldval, 0);
+ }
+ break;
+ case 4:
+ firstAccess = masm.as_ldrex(output, ptr);
+ break;
+ }
+ if (access) {
+ masm.append(*access, firstAccess.getOffset());
+ }
+
+ if (nbytes < 4) {
+ masm.as_cmp(output, O2Reg(scratch));
+ } else {
+ masm.as_cmp(output, O2Reg(oldval));
+ }
+ masm.as_b(&done, MacroAssembler::NotEqual);
+ switch (nbytes) {
+ case 1:
+ masm.as_strexb(scratch, newval, ptr);
+ break;
+ case 2:
+ masm.as_strexh(scratch, newval, ptr);
+ break;
+ case 4:
+ masm.as_strex(scratch, newval, ptr);
+ break;
+ }
+ masm.as_cmp(scratch, Imm8(1));
+ masm.as_b(&again, MacroAssembler::Equal);
+ masm.bind(&done);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const Address& address, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, nullptr, type, sync, address, oldval, newval, output);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const BaseIndex& address, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, nullptr, type, sync, address, oldval, newval, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
+ newval, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
+ newval, output);
+}
+
+template <typename T>
+static void AtomicExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ const T& mem, Register value, Register output) {
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ MOZ_ASSERT(nbytes <= 4);
+
+ // Bug 1077321: We may further optimize for ARMv8 (AArch32) here.
+ Label again;
+ Label done;
+
+ SecondScratchRegisterScope scratch2(masm);
+ Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ ScratchRegisterScope scratch(masm);
+
+ // NOTE: the generated code must match the assembly code in gen_exchange in
+ // GenerateAtomicOperations.py
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ BufferOffset firstAccess;
+ switch (nbytes) {
+ case 1:
+ firstAccess = masm.as_ldrexb(output, ptr);
+ if (signExtend) {
+ masm.as_sxtb(output, output, 0);
+ }
+ masm.as_strexb(scratch, value, ptr);
+ break;
+ case 2:
+ firstAccess = masm.as_ldrexh(output, ptr);
+ if (signExtend) {
+ masm.as_sxth(output, output, 0);
+ }
+ masm.as_strexh(scratch, value, ptr);
+ break;
+ case 4:
+ firstAccess = masm.as_ldrex(output, ptr);
+ masm.as_strex(scratch, value, ptr);
+ break;
+ }
+ if (access) {
+ masm.append(*access, firstAccess.getOffset());
+ }
+
+ masm.as_cmp(scratch, Imm8(1));
+ masm.as_b(&again, MacroAssembler::Equal);
+ masm.bind(&done);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const Address& address, Register value,
+ Register output) {
+ AtomicExchange(*this, nullptr, type, sync, address, value, output);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const BaseIndex& address, Register value,
+ Register output) {
+ AtomicExchange(*this, nullptr, type, sync, address, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
+ output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
+ output);
+}
+
+// General algorithm:
+//
+// ... ptr, <addr> ; compute address of item
+// dmb
+// L0 ldrex* output, [ptr]
+// sxt* output, output, 0 ; sign-extend if applicable
+// OP tmp, output, value ; compute value to store
+// strex* tmp2, tmp, [ptr] ; tmp2 required by strex
+// cmp tmp2, 1
+// beq L0 ; failed - location is dirty, retry
+// dmb ; ordering barrier required
+//
+// Also see notes above at compareExchange re the barrier strategy.
+//
+// Observe that the value being operated into the memory element need
+// not be sign-extended because no OP will make use of bits to the
+// left of the bits indicated by the width of the element, and neither
+// output nor the bits stored are affected by OP.
+
+template <typename T>
+static void AtomicFetchOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, const Register& value, const T& mem,
+ Register flagTemp, Register output) {
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ MOZ_ASSERT(nbytes <= 4);
+ MOZ_ASSERT(flagTemp != InvalidReg);
+ MOZ_ASSERT(output != value);
+
+ Label again;
+
+ SecondScratchRegisterScope scratch2(masm);
+ Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ // NOTE: the generated code must match the assembly code in gen_fetchop in
+ // GenerateAtomicOperations.py
+ masm.memoryBarrierBefore(sync);
+
+ ScratchRegisterScope scratch(masm);
+
+ masm.bind(&again);
+
+ BufferOffset firstAccess;
+ switch (nbytes) {
+ case 1:
+ firstAccess = masm.as_ldrexb(output, ptr);
+ if (signExtend) {
+ masm.as_sxtb(output, output, 0);
+ }
+ break;
+ case 2:
+ firstAccess = masm.as_ldrexh(output, ptr);
+ if (signExtend) {
+ masm.as_sxth(output, output, 0);
+ }
+ break;
+ case 4:
+ firstAccess = masm.as_ldrex(output, ptr);
+ break;
+ }
+ if (access) {
+ masm.append(*access, firstAccess.getOffset());
+ }
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_add(scratch, output, O2Reg(value));
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sub(scratch, output, O2Reg(value));
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(scratch, output, O2Reg(value));
+ break;
+ case AtomicFetchOrOp:
+ masm.as_orr(scratch, output, O2Reg(value));
+ break;
+ case AtomicFetchXorOp:
+ masm.as_eor(scratch, output, O2Reg(value));
+ break;
+ }
+ // Rd must differ from the two other arguments to strex.
+ switch (nbytes) {
+ case 1:
+ masm.as_strexb(flagTemp, scratch, ptr);
+ break;
+ case 2:
+ masm.as_strexh(flagTemp, scratch, ptr);
+ break;
+ case 4:
+ masm.as_strex(flagTemp, scratch, ptr);
+ break;
+ }
+ masm.as_cmp(flagTemp, Imm8(1));
+ masm.as_b(&again, MacroAssembler::Equal);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type type,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp, Register output) {
+ AtomicFetchOp(*this, nullptr, type, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type type,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp, Register output) {
+ AtomicFetchOp(*this, nullptr, type, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register temp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), access.sync(), op, value, mem,
+ temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem, Register temp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), access.sync(), op, value, mem,
+ temp, output);
+}
+
+// Uses both scratch registers, one for the address and one for a temp,
+// but needs two temps for strex:
+//
+// ... ptr, <addr> ; compute address of item
+// dmb
+// L0 ldrex* temp, [ptr]
+// OP temp, temp, value ; compute value to store
+// strex* temp2, temp, [ptr]
+// cmp temp2, 1
+// beq L0 ; failed - location is dirty, retry
+// dmb ; ordering barrier required
+
+template <typename T>
+static void AtomicEffectOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, const Register& value, const T& mem,
+ Register flagTemp) {
+ unsigned nbytes = Scalar::byteSize(type);
+
+ MOZ_ASSERT(nbytes <= 4);
+ MOZ_ASSERT(flagTemp != InvalidReg);
+
+ Label again;
+
+ SecondScratchRegisterScope scratch2(masm);
+ Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ masm.memoryBarrierBefore(sync);
+
+ ScratchRegisterScope scratch(masm);
+
+ masm.bind(&again);
+
+ BufferOffset firstAccess;
+ switch (nbytes) {
+ case 1:
+ firstAccess = masm.as_ldrexb(scratch, ptr);
+ break;
+ case 2:
+ firstAccess = masm.as_ldrexh(scratch, ptr);
+ break;
+ case 4:
+ firstAccess = masm.as_ldrex(scratch, ptr);
+ break;
+ }
+ if (access) {
+ masm.append(*access, firstAccess.getOffset());
+ }
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_add(scratch, scratch, O2Reg(value));
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sub(scratch, scratch, O2Reg(value));
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(scratch, scratch, O2Reg(value));
+ break;
+ case AtomicFetchOrOp:
+ masm.as_orr(scratch, scratch, O2Reg(value));
+ break;
+ case AtomicFetchXorOp:
+ masm.as_eor(scratch, scratch, O2Reg(value));
+ break;
+ }
+ // Rd must differ from the two other arguments to strex.
+ switch (nbytes) {
+ case 1:
+ masm.as_strexb(flagTemp, scratch, ptr);
+ break;
+ case 2:
+ masm.as_strexh(flagTemp, scratch, ptr);
+ break;
+ case 4:
+ masm.as_strex(flagTemp, scratch, ptr);
+ break;
+ }
+ masm.as_cmp(flagTemp, Imm8(1));
+ masm.as_b(&again, MacroAssembler::Equal);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register temp) {
+ AtomicEffectOp(*this, &access, access.type(), access.sync(), op, value, mem,
+ temp);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem, Register temp) {
+ AtomicEffectOp(*this, &access, access.type(), access.sync(), op, value, mem,
+ temp);
+}
+
+template <typename T>
+static void AtomicLoad64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, const T& mem,
+ Register64 output) {
+ MOZ_ASSERT((output.low.code() & 1) == 0);
+ MOZ_ASSERT(output.low.code() + 1 == output.high.code());
+
+ masm.memoryBarrierBefore(sync);
+
+ SecondScratchRegisterScope scratch2(masm);
+ Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ BufferOffset load = masm.as_ldrexd(output.low, output.high, ptr);
+ if (access) {
+ masm.append(*access, load.getOffset());
+ }
+ masm.as_clrex();
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <typename T>
+static void WasmAtomicLoad64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access, const T& mem,
+ Register64 temp, Register64 output) {
+ MOZ_ASSERT(temp.low == InvalidReg && temp.high == InvalidReg);
+
+ AtomicLoad64(masm, &access, access.sync(), mem, output);
+}
+
+void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ WasmAtomicLoad64(*this, access, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ WasmAtomicLoad64(*this, access, mem, temp, output);
+}
+
+template <typename T>
+static void CompareExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, const T& mem,
+ Register64 expect, Register64 replace,
+ Register64 output) {
+ MOZ_ASSERT(expect != replace && replace != output && output != expect);
+
+ MOZ_ASSERT((replace.low.code() & 1) == 0);
+ MOZ_ASSERT(replace.low.code() + 1 == replace.high.code());
+
+ MOZ_ASSERT((output.low.code() & 1) == 0);
+ MOZ_ASSERT(output.low.code() + 1 == output.high.code());
+
+ Label again;
+ Label done;
+
+ SecondScratchRegisterScope scratch2(masm);
+ Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ // NOTE: the generated code must match the assembly code in gen_cmpxchg in
+ // GenerateAtomicOperations.py
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+ BufferOffset load = masm.as_ldrexd(output.low, output.high, ptr);
+ if (access) {
+ masm.append(*access, load.getOffset());
+ }
+
+ masm.as_cmp(output.low, O2Reg(expect.low));
+ masm.as_cmp(output.high, O2Reg(expect.high), MacroAssembler::Equal);
+ masm.as_b(&done, MacroAssembler::NotEqual);
+
+ ScratchRegisterScope scratch(masm);
+
+ // Rd (temp) must differ from the two other arguments to strex.
+ masm.as_strexd(scratch, replace.low, replace.high, ptr);
+ masm.as_cmp(scratch, Imm8(1));
+ masm.as_b(&again, MacroAssembler::Equal);
+ masm.bind(&done);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
+ output);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
+ output);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization& sync,
+ const Address& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
+}
+
+template <typename T>
+static void AtomicExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, const T& mem,
+ Register64 value, Register64 output) {
+ MOZ_ASSERT(output != value);
+
+ MOZ_ASSERT((value.low.code() & 1) == 0);
+ MOZ_ASSERT(value.low.code() + 1 == value.high.code());
+
+ MOZ_ASSERT((output.low.code() & 1) == 0);
+ MOZ_ASSERT(output.low.code() + 1 == output.high.code());
+
+ Label again;
+
+ SecondScratchRegisterScope scratch2(masm);
+ Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+ BufferOffset load = masm.as_ldrexd(output.low, output.high, ptr);
+ if (access) {
+ masm.append(*access, load.getOffset());
+ }
+
+ ScratchRegisterScope scratch(masm);
+
+ masm.as_strexd(scratch, value.low, value.high, ptr);
+ masm.as_cmp(scratch, Imm8(1));
+ masm.as_b(&again, MacroAssembler::Equal);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <typename T>
+static void WasmAtomicExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access,
+ const T& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(masm, &access, access.sync(), mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ WasmAtomicExchange64(*this, access, mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 value, Register64 output) {
+ WasmAtomicExchange64(*this, access, mem, value, output);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization& sync,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, nullptr, sync, mem, value, output);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, nullptr, sync, mem, value, output);
+}
+
+template <typename T>
+static void AtomicFetchOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, AtomicOp op,
+ Register64 value, const T& mem, Register64 temp,
+ Register64 output) {
+ MOZ_ASSERT(temp.low != InvalidReg && temp.high != InvalidReg);
+ MOZ_ASSERT(output != value);
+ MOZ_ASSERT(temp != value);
+
+ MOZ_ASSERT((temp.low.code() & 1) == 0);
+ MOZ_ASSERT(temp.low.code() + 1 == temp.high.code());
+
+ // We could avoid this pair requirement but in that case we would end up
+ // with two moves in the loop to preserve the loaded value in output. The
+ // prize would be less register spilling around this op since the pair
+ // requirement will tend to force more spilling.
+
+ MOZ_ASSERT((output.low.code() & 1) == 0);
+ MOZ_ASSERT(output.low.code() + 1 == output.high.code());
+
+ Label again;
+
+ SecondScratchRegisterScope scratch2(masm);
+ Register ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+ BufferOffset load = masm.as_ldrexd(output.low, output.high, ptr);
+ if (access) {
+ masm.append(*access, load.getOffset());
+ }
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_add(temp.low, output.low, O2Reg(value.low), SetCC);
+ masm.as_adc(temp.high, output.high, O2Reg(value.high));
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sub(temp.low, output.low, O2Reg(value.low), SetCC);
+ masm.as_sbc(temp.high, output.high, O2Reg(value.high));
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(temp.low, output.low, O2Reg(value.low));
+ masm.as_and(temp.high, output.high, O2Reg(value.high));
+ break;
+ case AtomicFetchOrOp:
+ masm.as_orr(temp.low, output.low, O2Reg(value.low));
+ masm.as_orr(temp.high, output.high, O2Reg(value.high));
+ break;
+ case AtomicFetchXorOp:
+ masm.as_eor(temp.low, output.low, O2Reg(value.low));
+ masm.as_eor(temp.high, output.high, O2Reg(value.high));
+ break;
+ }
+
+ ScratchRegisterScope scratch(masm);
+
+ // Rd (temp) must differ from the two other arguments to strex.
+ masm.as_strexd(scratch, temp.low, temp.high, ptr);
+ masm.as_cmp(scratch, Imm8(1));
+ masm.as_b(&again, MacroAssembler::Equal);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <typename T>
+static void WasmAtomicFetchOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value, const T& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(masm, &access, access.sync(), op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ WasmAtomicFetchOp64(*this, access, op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ WasmAtomicFetchOp64(*this, access, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
+}
+
+// ========================================================================
+// JS atomic operations.
+
+template <typename T>
+static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register oldval, Register newval, Register temp,
+ AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, output.gpr());
+ }
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register oldval,
+ Register newval, Register temp,
+ AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register temp,
+ AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
+}
+
+template <typename T>
+static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register value, Register temp,
+ AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicExchange(arrayType, sync, mem, value, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicExchange(arrayType, sync, mem, value, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register value,
+ Register temp, AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register temp, AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
+}
+
+template <typename T>
+static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const T& mem, Register temp1,
+ Register temp2, AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, temp2, temp1);
+ masm.convertUInt32ToDouble(temp1, output.fpu());
+ } else {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, temp1, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp1, Register temp2,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp1, Register temp2,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp) {
+ AtomicEffectOp(*this, nullptr, arrayType, sync, op, value, mem, temp);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp) {
+ AtomicEffectOp(*this, nullptr, arrayType, sync, op, value, mem, temp);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+void MacroAssembler::atomicLoad64(const Synchronization& sync,
+ const Address& mem, Register64 output) {
+ AtomicLoad64(*this, nullptr, sync, mem, output);
+}
+
+void MacroAssembler::atomicLoad64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 output) {
+ AtomicLoad64(*this, nullptr, sync, mem, output);
+}
+
+void MacroAssembler::atomicStore64(const Synchronization& sync,
+ const Address& mem, Register64 value,
+ Register64 temp) {
+ AtomicExchange64(*this, nullptr, sync, mem, value, temp);
+}
+
+void MacroAssembler::atomicStore64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 value,
+ Register64 temp) {
+ AtomicExchange64(*this, nullptr, sync, mem, value, temp);
+}
+
+// ========================================================================
+// Convert floating point.
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ ScratchDoubleScope scratchDouble(*this);
+
+ convertUInt32ToDouble(src.high, dest);
+ {
+ ScratchRegisterScope scratch(*this);
+ movePtr(ImmPtr(&TO_DOUBLE_HIGH_SCALE), scratch);
+ ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), scratchDouble);
+ }
+ mulDouble(scratchDouble, dest);
+ convertUInt32ToDouble(src.low, scratchDouble);
+ addDouble(scratchDouble, dest);
+}
+
+void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) {
+ ScratchDoubleScope scratchDouble(*this);
+
+ convertInt32ToDouble(src.high, dest);
+ {
+ ScratchRegisterScope scratch(*this);
+ movePtr(ImmPtr(&TO_DOUBLE_HIGH_SCALE), scratch);
+ ma_vldr(Operand(Address(scratch, 0)).toVFPAddr(), scratchDouble);
+ }
+ mulDouble(scratchDouble, dest);
+ convertUInt32ToDouble(src.low, scratchDouble);
+ addDouble(scratchDouble, dest);
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ convertInt32ToDouble(src, dest);
+}
+
+extern "C" {
+extern MOZ_EXPORT int64_t __aeabi_idivmod(int, int);
+extern MOZ_EXPORT int64_t __aeabi_uidivmod(int, int);
+}
+
+inline void EmitRemainderOrQuotient(bool isRemainder, MacroAssembler& masm,
+ Register rhs, Register lhsOutput,
+ bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs) {
+ // Currently this helper can't handle this situation.
+ MOZ_ASSERT(lhsOutput != rhs);
+
+ if (HasIDIV()) {
+ if (isRemainder) {
+ masm.remainder32(rhs, lhsOutput, isUnsigned);
+ } else {
+ masm.quotient32(rhs, lhsOutput, isUnsigned);
+ }
+ } else {
+ // Ensure that the output registers are saved and restored properly,
+ MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal0));
+ MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal1));
+
+ masm.PushRegsInMask(volatileLiveRegs);
+ using Fn = int64_t (*)(int, int);
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.setupUnalignedABICall(scratch);
+ }
+ masm.passABIArg(lhsOutput);
+ masm.passABIArg(rhs);
+ if (isUnsigned) {
+ masm.callWithABI<Fn, __aeabi_uidivmod>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+ } else {
+ masm.callWithABI<Fn, __aeabi_idivmod>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+ }
+ if (isRemainder) {
+ masm.mov(ReturnRegVal1, lhsOutput);
+ } else {
+ masm.mov(ReturnRegVal0, lhsOutput);
+ }
+
+ LiveRegisterSet ignore;
+ ignore.add(lhsOutput);
+ masm.PopRegsInMaskIgnore(volatileLiveRegs, ignore);
+ }
+}
+
+void MacroAssembler::flexibleQuotient32(
+ Register rhs, Register srcDest, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs) {
+ EmitRemainderOrQuotient(false, *this, rhs, srcDest, isUnsigned,
+ volatileLiveRegs);
+}
+
+void MacroAssembler::flexibleRemainder32(
+ Register rhs, Register srcDest, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs) {
+ EmitRemainderOrQuotient(true, *this, rhs, srcDest, isUnsigned,
+ volatileLiveRegs);
+}
+
+void MacroAssembler::flexibleDivMod32(Register rhs, Register lhsOutput,
+ Register remOutput, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs) {
+ // Currently this helper can't handle this situation.
+ MOZ_ASSERT(lhsOutput != rhs);
+
+ if (HasIDIV()) {
+ mov(lhsOutput, remOutput);
+ remainder32(rhs, remOutput, isUnsigned);
+ quotient32(rhs, lhsOutput, isUnsigned);
+ } else {
+ // Ensure that the output registers are saved and restored properly,
+ MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal0));
+ MOZ_ASSERT(volatileLiveRegs.has(ReturnRegVal1));
+ PushRegsInMask(volatileLiveRegs);
+
+ using Fn = int64_t (*)(int, int);
+ {
+ ScratchRegisterScope scratch(*this);
+ setupUnalignedABICall(scratch);
+ }
+ passABIArg(lhsOutput);
+ passABIArg(rhs);
+ if (isUnsigned) {
+ callWithABI<Fn, __aeabi_uidivmod>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+ } else {
+ callWithABI<Fn, __aeabi_idivmod>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+ }
+ moveRegPair(ReturnRegVal0, ReturnRegVal1, lhsOutput, remOutput);
+
+ LiveRegisterSet ignore;
+ ignore.add(remOutput);
+ ignore.add(lhsOutput);
+ PopRegsInMaskIgnore(volatileLiveRegs, ignore);
+ }
+}
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ return movWithPatch(ImmPtr(nullptr), dest);
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target) {
+ PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr));
+}
+
+// ========================================================================
+// Spectre Mitigations.
+
+void MacroAssembler::speculationBarrier() {
+ // Spectre mitigation recommended by ARM for cases where csel/cmov cannot be
+ // used.
+ as_csdb();
+}
+
+void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ floorf(src, dest, fail);
+}
+
+void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ floor(src, dest, fail);
+}
+
+void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ceilf(src, dest, fail);
+}
+
+void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ceil(src, dest, fail);
+}
+
+void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ roundf(src, dest, fail, temp);
+}
+
+void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ round(src, dest, fail, temp);
+}
+
+void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ truncf(src, dest, fail);
+}
+
+void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ trunc(src, dest, fail);
+}
+
+void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
+ Register pointer) {
+ if (IsShiftInScaleRange(shift)) {
+ computeEffectiveAddress(
+ BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer);
+ return;
+ }
+ lshift32(Imm32(shift), indexTemp32);
+ addPtr(indexTemp32, pointer);
+}
+
+//}}} check_macroassembler_style
+
+void MacroAssemblerARM::wasmTruncateToInt32(FloatRegister input,
+ Register output, MIRType fromType,
+ bool isUnsigned, bool isSaturating,
+ Label* oolEntry) {
+ ScratchDoubleScope scratchScope(asMasm());
+ ScratchRegisterScope scratchReg(asMasm());
+ FloatRegister scratch = scratchScope.uintOverlay();
+
+ // ARM conversion instructions clamp the value to ensure it fits within the
+ // target's type bounds, so every time we see those, we need to check the
+ // input. A NaN check is not necessary because NaN is converted to zero and
+ // on a zero result we branch out of line to do further processing anyway.
+ if (isUnsigned) {
+ if (fromType == MIRType::Double) {
+ ma_vcvt_F64_U32(input, scratch);
+ } else if (fromType == MIRType::Float32) {
+ ma_vcvt_F32_U32(input, scratch);
+ } else {
+ MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
+ }
+
+ ma_vxfer(scratch, output);
+
+ if (!isSaturating) {
+ // int32_t(UINT32_MAX) == -1.
+ ma_cmp(output, Imm32(-1), scratchReg);
+ as_cmp(output, Imm8(0), Assembler::NotEqual);
+ ma_b(oolEntry, Assembler::Equal);
+ }
+
+ return;
+ }
+
+ // vcvt* converts NaN into 0, so check for NaNs here.
+ if (!isSaturating) {
+ if (fromType == MIRType::Double) {
+ asMasm().compareDouble(input, input);
+ } else if (fromType == MIRType::Float32) {
+ asMasm().compareFloat(input, input);
+ } else {
+ MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
+ }
+
+ ma_b(oolEntry, Assembler::VFP_Unordered);
+ }
+
+ scratch = scratchScope.sintOverlay();
+
+ if (fromType == MIRType::Double) {
+ ma_vcvt_F64_I32(input, scratch);
+ } else if (fromType == MIRType::Float32) {
+ ma_vcvt_F32_I32(input, scratch);
+ } else {
+ MOZ_CRASH("unexpected type in visitWasmTruncateToInt32");
+ }
+
+ ma_vxfer(scratch, output);
+
+ if (!isSaturating) {
+ ma_cmp(output, Imm32(INT32_MAX), scratchReg);
+ ma_cmp(output, Imm32(INT32_MIN), scratchReg, Assembler::NotEqual);
+ ma_b(oolEntry, Assembler::Equal);
+ }
+}
+
+void MacroAssemblerARM::outOfLineWasmTruncateToIntCheck(
+ FloatRegister input, MIRType fromType, MIRType toType, TruncFlags flags,
+ Label* rejoin, wasm::BytecodeOffset trapOffset) {
+ // On ARM, saturating truncation codegen handles saturating itself rather
+ // than relying on out-of-line fixup code.
+ if (flags & TRUNC_SATURATING) {
+ return;
+ }
+
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ ScratchDoubleScope scratchScope(asMasm());
+ FloatRegister scratch;
+
+ // Eagerly take care of NaNs.
+ Label inputIsNaN;
+ if (fromType == MIRType::Double) {
+ asMasm().branchDouble(Assembler::DoubleUnordered, input, input,
+ &inputIsNaN);
+ } else if (fromType == MIRType::Float32) {
+ asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN);
+ } else {
+ MOZ_CRASH("unexpected type in visitOutOfLineWasmTruncateCheck");
+ }
+
+ // Handle special values.
+ Label fail;
+
+ // By default test for the following inputs and bail:
+ // signed: ] -Inf, INTXX_MIN - 1.0 ] and [ INTXX_MAX + 1.0 : +Inf [
+ // unsigned: ] -Inf, -1.0 ] and [ UINTXX_MAX + 1.0 : +Inf [
+ // Note: we cannot always represent those exact values. As a result
+ // this changes the actual comparison a bit.
+ double minValue, maxValue;
+ Assembler::DoubleCondition minCond = Assembler::DoubleLessThanOrEqual;
+ Assembler::DoubleCondition maxCond = Assembler::DoubleGreaterThanOrEqual;
+ if (toType == MIRType::Int64) {
+ if (isUnsigned) {
+ minValue = -1;
+ maxValue = double(UINT64_MAX) + 1.0;
+ } else {
+ // In the float32/double range there exists no value between
+ // INT64_MIN and INT64_MIN - 1.0. Making INT64_MIN the lower-bound.
+ minValue = double(INT64_MIN);
+ minCond = Assembler::DoubleLessThan;
+ maxValue = double(INT64_MAX) + 1.0;
+ }
+ } else {
+ if (isUnsigned) {
+ minValue = -1;
+ maxValue = double(UINT32_MAX) + 1.0;
+ } else {
+ if (fromType == MIRType::Float32) {
+ // In the float32 range there exists no value between
+ // INT32_MIN and INT32_MIN - 1.0. Making INT32_MIN the lower-bound.
+ minValue = double(INT32_MIN);
+ minCond = Assembler::DoubleLessThan;
+ } else {
+ minValue = double(INT32_MIN) - 1.0;
+ }
+ maxValue = double(INT32_MAX) + 1.0;
+ }
+ }
+
+ if (fromType == MIRType::Double) {
+ scratch = scratchScope.doubleOverlay();
+ asMasm().loadConstantDouble(minValue, scratch);
+ asMasm().branchDouble(minCond, input, scratch, &fail);
+
+ asMasm().loadConstantDouble(maxValue, scratch);
+ asMasm().branchDouble(maxCond, input, scratch, &fail);
+ } else {
+ MOZ_ASSERT(fromType == MIRType::Float32);
+ scratch = scratchScope.singleOverlay();
+ asMasm().loadConstantFloat32(float(minValue), scratch);
+ asMasm().branchFloat(minCond, input, scratch, &fail);
+
+ asMasm().loadConstantFloat32(float(maxValue), scratch);
+ asMasm().branchFloat(maxCond, input, scratch, &fail);
+ }
+
+ // We had an actual correct value, get back to where we were.
+ ma_b(rejoin);
+
+ // Handle errors.
+ bind(&fail);
+ asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
+
+ bind(&inputIsNaN);
+ asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
+}
+
+void MacroAssemblerARM::wasmLoadImpl(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, AnyRegister output,
+ Register64 out64) {
+ MOZ_ASSERT(ptr == ptrScratch);
+ MOZ_ASSERT(!access.isZeroExtendSimd128Load());
+ MOZ_ASSERT(!access.isSplatSimd128Load());
+ MOZ_ASSERT(!access.isWidenSimd128Load());
+
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+
+ Scalar::Type type = access.type();
+
+ // Maybe add the offset.
+ if (offset || type == Scalar::Int64) {
+ ScratchRegisterScope scratch(asMasm());
+ if (offset) {
+ ma_add(Imm32(offset), ptr, scratch);
+ }
+ }
+
+ bool isSigned = type == Scalar::Int8 || type == Scalar::Int16 ||
+ type == Scalar::Int32 || type == Scalar::Int64;
+ unsigned byteSize = access.byteSize();
+
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ asMasm().memoryBarrierBefore(access.sync());
+
+ BufferOffset load;
+ if (out64 != Register64::Invalid()) {
+ if (type == Scalar::Int64) {
+ static_assert(INT64LOW_OFFSET == 0);
+
+ load = ma_dataTransferN(IsLoad, 32, /* signed = */ false, memoryBase, ptr,
+ out64.low);
+ append(access, load.getOffset());
+
+ as_add(ptr, ptr, Imm8(INT64HIGH_OFFSET));
+
+ load =
+ ma_dataTransferN(IsLoad, 32, isSigned, memoryBase, ptr, out64.high);
+ append(access, load.getOffset());
+ } else {
+ load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, memoryBase, ptr,
+ out64.low);
+ append(access, load.getOffset());
+
+ if (isSigned) {
+ ma_asr(Imm32(31), out64.low, out64.high);
+ } else {
+ ma_mov(Imm32(0), out64.high);
+ }
+ }
+ } else {
+ bool isFloat = output.isFloat();
+ if (isFloat) {
+ MOZ_ASSERT((byteSize == 4) == output.fpu().isSingle());
+ ScratchRegisterScope scratch(asMasm());
+ FloatRegister dest = output.fpu();
+ ma_add(memoryBase, ptr, scratch);
+
+ // FP loads can't use VLDR as that has stringent alignment checks and will
+ // SIGBUS on unaligned accesses. Choose a different strategy depending on
+ // the available hardware. We don't gate Wasm on the presence of NEON.
+ if (HasNEON()) {
+ // NEON available: The VLD1 multiple-single-elements variant will only
+ // trap if SCTRL.A==1, but we already assume (for integer accesses) that
+ // the hardware/OS handles that transparently.
+ //
+ // An additional complication is that if we're targeting the high single
+ // then an unaligned load is not possible, and we may need to go via the
+ // FPR scratch.
+ if (byteSize == 4 && dest.code() & 1) {
+ ScratchFloat32Scope fscratch(asMasm());
+ load = as_vldr_unaligned(fscratch, scratch);
+ as_vmov(dest, fscratch);
+ } else {
+ load = as_vldr_unaligned(dest, scratch);
+ }
+ } else {
+ // NEON not available: Load to GPR scratch, move to FPR destination. We
+ // don't have adjacent scratches for the f64, so use individual LDRs,
+ // not LDRD.
+ SecondScratchRegisterScope scratch2(asMasm());
+ if (byteSize == 4) {
+ load = as_dtr(IsLoad, 32, Offset, scratch2,
+ DTRAddr(scratch, DtrOffImm(0)), Always);
+ as_vxfer(scratch2, InvalidReg, VFPRegister(dest), CoreToFloat,
+ Always);
+ } else {
+ // The trap information is associated with the load of the high word,
+ // which must be done first.
+ load = as_dtr(IsLoad, 32, Offset, scratch2,
+ DTRAddr(scratch, DtrOffImm(4)), Always);
+ as_dtr(IsLoad, 32, Offset, scratch, DTRAddr(scratch, DtrOffImm(0)),
+ Always);
+ as_vxfer(scratch, scratch2, VFPRegister(dest), CoreToFloat, Always);
+ }
+ }
+ append(access, load.getOffset());
+ } else {
+ load = ma_dataTransferN(IsLoad, byteSize * 8, isSigned, memoryBase, ptr,
+ output.gpr());
+ append(access, load.getOffset());
+ }
+ }
+
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerARM::wasmStoreImpl(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Register64 val64,
+ Register memoryBase, Register ptr,
+ Register ptrScratch) {
+ static_assert(INT64LOW_OFFSET == 0);
+ static_assert(INT64HIGH_OFFSET == 4);
+
+ MOZ_ASSERT(ptr == ptrScratch);
+
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+
+ unsigned byteSize = access.byteSize();
+ Scalar::Type type = access.type();
+
+ // Maybe add the offset.
+ if (offset || type == Scalar::Int64) {
+ ScratchRegisterScope scratch(asMasm());
+ // We need to store the high word of an Int64 first, so always adjust the
+ // pointer to point to the high word in this case. The adjustment is always
+ // OK because wasmMaxOffsetGuardLimit is computed so that we can add up to
+ // sizeof(LargestValue)-1 without skipping past the guard page, and we
+ // assert above that offset < wasmMaxOffsetGuardLimit.
+ if (type == Scalar::Int64) {
+ offset += INT64HIGH_OFFSET;
+ }
+ if (offset) {
+ ma_add(Imm32(offset), ptr, scratch);
+ }
+ }
+
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ asMasm().memoryBarrierBefore(access.sync());
+
+ BufferOffset store;
+ if (type == Scalar::Int64) {
+ store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ false,
+ memoryBase, ptr, val64.high);
+ append(access, store.getOffset());
+
+ as_sub(ptr, ptr, Imm8(INT64HIGH_OFFSET));
+
+ store = ma_dataTransferN(IsStore, 32 /* bits */, /* signed */ true,
+ memoryBase, ptr, val64.low);
+ append(access, store.getOffset());
+ } else {
+ if (value.isFloat()) {
+ ScratchRegisterScope scratch(asMasm());
+ FloatRegister val = value.fpu();
+ MOZ_ASSERT((byteSize == 4) == val.isSingle());
+ ma_add(memoryBase, ptr, scratch);
+
+ // See comments above at wasmLoadImpl for more about this logic.
+ if (HasNEON()) {
+ if (byteSize == 4 && (val.code() & 1)) {
+ ScratchFloat32Scope fscratch(asMasm());
+ as_vmov(fscratch, val);
+ store = as_vstr_unaligned(fscratch, scratch);
+ } else {
+ store = as_vstr_unaligned(val, scratch);
+ }
+ } else {
+ // NEON not available: Move FPR to GPR scratch, store GPR. We have only
+ // one scratch to hold the value, so for f64 we must do two separate
+ // moves. That's OK - this is really a corner case. If we really cared
+ // we would pass in a temp to avoid the second move.
+ SecondScratchRegisterScope scratch2(asMasm());
+ if (byteSize == 4) {
+ as_vxfer(scratch2, InvalidReg, VFPRegister(val), FloatToCore, Always);
+ store = as_dtr(IsStore, 32, Offset, scratch2,
+ DTRAddr(scratch, DtrOffImm(0)), Always);
+ } else {
+ // The trap information is associated with the store of the high word,
+ // which must be done first.
+ as_vxfer(scratch2, InvalidReg, VFPRegister(val).singleOverlay(1),
+ FloatToCore, Always);
+ store = as_dtr(IsStore, 32, Offset, scratch2,
+ DTRAddr(scratch, DtrOffImm(4)), Always);
+ as_vxfer(scratch2, InvalidReg, VFPRegister(val).singleOverlay(0),
+ FloatToCore, Always);
+ as_dtr(IsStore, 32, Offset, scratch2, DTRAddr(scratch, DtrOffImm(0)),
+ Always);
+ }
+ }
+ append(access, store.getOffset());
+ } else {
+ bool isSigned = type == Scalar::Uint32 ||
+ type == Scalar::Int32; // see AsmJSStoreHeap;
+ Register val = value.gpr();
+
+ store = ma_dataTransferN(IsStore, 8 * byteSize /* bits */, isSigned,
+ memoryBase, ptr, val);
+ append(access, store.getOffset());
+ }
+ }
+
+ asMasm().memoryBarrierAfter(access.sync());
+}
diff --git a/js/src/jit/arm/MacroAssembler-arm.h b/js/src/jit/arm/MacroAssembler-arm.h
new file mode 100644
index 0000000000..8302dc1ddb
--- /dev/null
+++ b/js/src/jit/arm/MacroAssembler-arm.h
@@ -0,0 +1,1402 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_MacroAssembler_arm_h
+#define jit_arm_MacroAssembler_arm_h
+
+#include "mozilla/DebugOnly.h"
+
+#include "jit/arm/Assembler-arm.h"
+#include "jit/MoveResolver.h"
+#include "vm/BytecodeUtil.h"
+#include "wasm/WasmBuiltins.h"
+#include "wasm/WasmCodegenTypes.h"
+
+namespace js {
+namespace jit {
+
+static Register CallReg = ip;
+static const int defaultShift = 3;
+static_assert(1 << defaultShift == sizeof(JS::Value));
+
+// See documentation for ScratchTagScope and ScratchTagScopeRelease in
+// MacroAssembler-x64.h.
+
+class ScratchTagScope {
+ const ValueOperand& v_;
+
+ public:
+ ScratchTagScope(MacroAssembler&, const ValueOperand& v) : v_(v) {}
+ operator Register() { return v_.typeReg(); }
+ void release() {}
+ void reacquire() {}
+};
+
+class ScratchTagScopeRelease {
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope*) {}
+};
+
+// MacroAssemblerARM is inheriting form Assembler defined in
+// Assembler-arm.{h,cpp}
+class MacroAssemblerARM : public Assembler {
+ private:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ protected:
+ // On ARM, some instructions require a second scratch register. This
+ // register defaults to lr, since it's non-allocatable (as it can be
+ // clobbered by some instructions). Allow the baseline compiler to override
+ // this though, since baseline IC stubs rely on lr holding the return
+ // address.
+ Register secondScratchReg_;
+
+ public:
+ Register getSecondScratchReg() const { return secondScratchReg_; }
+
+ public:
+ // Higher level tag testing code.
+ // TODO: Can probably remove the Operand versions.
+ Operand ToPayload(Operand base) const {
+ return Operand(Register::FromCode(base.base()), base.disp());
+ }
+ Address ToPayload(const Address& base) const { return base; }
+ BaseIndex ToPayload(const BaseIndex& base) const { return base; }
+
+ protected:
+ Operand ToType(Operand base) const {
+ return Operand(Register::FromCode(base.base()),
+ base.disp() + sizeof(void*));
+ }
+ Address ToType(const Address& base) const {
+ return ToType(Operand(base)).toAddress();
+ }
+ BaseIndex ToType(const BaseIndex& base) const {
+ return BaseIndex(base.base, base.index, base.scale,
+ base.offset + sizeof(void*));
+ }
+
+ Address ToPayloadAfterStackPush(const Address& base) const {
+ // If we are based on StackPointer, pass over the type tag just pushed.
+ if (base.base == StackPointer) {
+ return Address(base.base, base.offset + sizeof(void*));
+ }
+ return ToPayload(base);
+ }
+
+ public:
+ MacroAssemblerARM() : secondScratchReg_(lr) {}
+
+ void setSecondScratchReg(Register reg) {
+ MOZ_ASSERT(reg != ScratchRegister);
+ secondScratchReg_ = reg;
+ }
+
+ void convertBoolToInt32(Register source, Register dest);
+ void convertInt32ToDouble(Register src, FloatRegister dest);
+ void convertInt32ToDouble(const Address& src, FloatRegister dest);
+ void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest);
+ void convertUInt32ToFloat32(Register src, FloatRegister dest);
+ void convertUInt32ToDouble(Register src, FloatRegister dest);
+ void convertDoubleToFloat32(FloatRegister src, FloatRegister dest,
+ Condition c = Always);
+ void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true) {
+ convertDoubleToInt32(src, dest, fail, negativeZeroCheck);
+ }
+ void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+
+ void convertFloat32ToDouble(FloatRegister src, FloatRegister dest);
+ void convertInt32ToFloat32(Register src, FloatRegister dest);
+ void convertInt32ToFloat32(const Address& src, FloatRegister dest);
+
+ void wasmTruncateToInt32(FloatRegister input, Register output,
+ MIRType fromType, bool isUnsigned, bool isSaturating,
+ Label* oolEntry);
+ void outOfLineWasmTruncateToIntCheck(FloatRegister input, MIRType fromType,
+ MIRType toType, TruncFlags flags,
+ Label* rejoin,
+ wasm::BytecodeOffset trapOffset);
+
+ // Somewhat direct wrappers for the low-level assembler funcitons
+ // bitops. Attempt to encode a virtual alu instruction using two real
+ // instructions.
+ private:
+ bool alu_dbl(Register src1, Imm32 imm, Register dest, ALUOp op, SBit s,
+ Condition c);
+
+ public:
+ void ma_alu(Register src1, Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, ALUOp op, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_alu(Register src1, Operand2 op2, Register dest, ALUOp op,
+ SBit s = LeaveCC, Condition c = Always);
+ void ma_alu(Register src1, Operand op2, Register dest, ALUOp op,
+ SBit s = LeaveCC, Condition c = Always);
+ void ma_nop();
+
+ BufferOffset ma_movPatchable(Imm32 imm, Register dest,
+ Assembler::Condition c);
+ BufferOffset ma_movPatchable(ImmPtr imm, Register dest,
+ Assembler::Condition c);
+
+ // To be used with Iter := InstructionIterator or BufferInstructionIterator.
+ template <class Iter>
+ static void ma_mov_patch(Imm32 imm, Register dest, Assembler::Condition c,
+ RelocStyle rs, Iter iter);
+
+ // ALU based ops
+ // mov
+ void ma_mov(Register src, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ void ma_mov(Imm32 imm, Register dest, Condition c = Always);
+ void ma_mov(ImmWord imm, Register dest, Condition c = Always);
+
+ void ma_mov(ImmGCPtr ptr, Register dest);
+
+ // Shifts (just a move with a shifting op2)
+ void ma_lsl(Imm32 shift, Register src, Register dst);
+ void ma_lsr(Imm32 shift, Register src, Register dst);
+ void ma_asr(Imm32 shift, Register src, Register dst);
+ void ma_ror(Imm32 shift, Register src, Register dst);
+ void ma_rol(Imm32 shift, Register src, Register dst);
+
+ void ma_lsl(Register shift, Register src, Register dst);
+ void ma_lsr(Register shift, Register src, Register dst);
+ void ma_asr(Register shift, Register src, Register dst);
+ void ma_ror(Register shift, Register src, Register dst);
+ void ma_rol(Register shift, Register src, Register dst,
+ AutoRegisterScope& scratch);
+
+ // Move not (dest <- ~src)
+ void ma_mvn(Register src1, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Negate (dest <- -src) implemented as rsb dest, src, 0
+ void ma_neg(Register src, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ void ma_neg(Register64 src, Register64 dest);
+
+ // And
+ void ma_and(Register src, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ void ma_and(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ void ma_and(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+
+ void ma_and(Imm32 imm, Register src1, Register dest,
+ AutoRegisterScope& scratch, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Bit clear (dest <- dest & ~imm) or (dest <- src1 & ~src2)
+ void ma_bic(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+
+ // Exclusive or
+ void ma_eor(Register src, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ void ma_eor(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ void ma_eor(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+
+ void ma_eor(Imm32 imm, Register src1, Register dest,
+ AutoRegisterScope& scratch, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Or
+ void ma_orr(Register src, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ void ma_orr(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ void ma_orr(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+
+ void ma_orr(Imm32 imm, Register src1, Register dest,
+ AutoRegisterScope& scratch, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Arithmetic based ops.
+ // Add with carry:
+ void ma_adc(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+ void ma_adc(Register src, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_adc(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_adc(Register src1, Imm32 op, Register dest,
+ AutoRegisterScope& scratch, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Add:
+ void ma_add(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+ void ma_add(Register src1, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_add(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_add(Register src1, Operand op, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_add(Register src1, Imm32 op, Register dest,
+ AutoRegisterScope& scratch, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Subtract with carry:
+ void ma_sbc(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+ void ma_sbc(Register src1, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_sbc(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Subtract:
+ void ma_sub(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+ void ma_sub(Register src1, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_sub(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_sub(Register src1, Operand op, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_sub(Register src1, Imm32 op, Register dest,
+ AutoRegisterScope& scratch, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Reverse subtract:
+ void ma_rsb(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+ void ma_rsb(Register src1, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_rsb(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_rsb(Register src1, Imm32 op2, Register dest,
+ AutoRegisterScope& scratch, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Reverse subtract with carry:
+ void ma_rsc(Imm32 imm, Register dest, AutoRegisterScope& scratch,
+ SBit s = LeaveCC, Condition c = Always);
+ void ma_rsc(Register src1, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+ void ma_rsc(Register src1, Register src2, Register dest, SBit s = LeaveCC,
+ Condition c = Always);
+
+ // Compares/tests.
+ // Compare negative (sets condition codes as src1 + src2 would):
+ void ma_cmn(Register src1, Imm32 imm, AutoRegisterScope& scratch,
+ Condition c = Always);
+ void ma_cmn(Register src1, Register src2, Condition c = Always);
+ void ma_cmn(Register src1, Operand op, Condition c = Always);
+
+ // Compare (src - src2):
+ void ma_cmp(Register src1, Imm32 imm, AutoRegisterScope& scratch,
+ Condition c = Always);
+ void ma_cmp(Register src1, ImmTag tag, Condition c = Always);
+ void ma_cmp(Register src1, ImmWord ptr, AutoRegisterScope& scratch,
+ Condition c = Always);
+ void ma_cmp(Register src1, ImmGCPtr ptr, AutoRegisterScope& scratch,
+ Condition c = Always);
+ void ma_cmp(Register src1, Operand op, AutoRegisterScope& scratch,
+ AutoRegisterScope& scratch2, Condition c = Always);
+ void ma_cmp(Register src1, Register src2, Condition c = Always);
+
+ // Test for equality, (src1 ^ src2):
+ void ma_teq(Register src1, Imm32 imm, AutoRegisterScope& scratch,
+ Condition c = Always);
+ void ma_teq(Register src1, Register src2, Condition c = Always);
+ void ma_teq(Register src1, Operand op, Condition c = Always);
+
+ // Test (src1 & src2):
+ void ma_tst(Register src1, Imm32 imm, AutoRegisterScope& scratch,
+ Condition c = Always);
+ void ma_tst(Register src1, Register src2, Condition c = Always);
+ void ma_tst(Register src1, Operand op, Condition c = Always);
+
+ // Multiplies. For now, there are only two that we care about.
+ void ma_mul(Register src1, Register src2, Register dest);
+ void ma_mul(Register src1, Imm32 imm, Register dest,
+ AutoRegisterScope& scratch);
+ Condition ma_check_mul(Register src1, Register src2, Register dest,
+ AutoRegisterScope& scratch, Condition cond);
+ Condition ma_check_mul(Register src1, Imm32 imm, Register dest,
+ AutoRegisterScope& scratch, Condition cond);
+
+ void ma_umull(Register src1, Imm32 imm, Register destHigh, Register destLow,
+ AutoRegisterScope& scratch);
+ void ma_umull(Register src1, Register src2, Register destHigh,
+ Register destLow);
+
+ // Fast mod, uses scratch registers, and thus needs to be in the assembler
+ // implicitly assumes that we can overwrite dest at the beginning of the
+ // sequence.
+ void ma_mod_mask(Register src, Register dest, Register hold, Register tmp,
+ AutoRegisterScope& scratch, AutoRegisterScope& scratch2,
+ int32_t shift);
+
+ // Mod - depends on integer divide instructions being supported.
+ void ma_smod(Register num, Register div, Register dest,
+ AutoRegisterScope& scratch);
+ void ma_umod(Register num, Register div, Register dest,
+ AutoRegisterScope& scratch);
+
+ // Division - depends on integer divide instructions being supported.
+ void ma_sdiv(Register num, Register div, Register dest,
+ Condition cond = Always);
+ void ma_udiv(Register num, Register div, Register dest,
+ Condition cond = Always);
+ // Misc operations
+ void ma_clz(Register src, Register dest, Condition cond = Always);
+ void ma_ctz(Register src, Register dest, AutoRegisterScope& scratch);
+ // Memory:
+ // Shortcut for when we know we're transferring 32 bits of data.
+ void ma_dtr(LoadStore ls, Register rn, Imm32 offset, Register rt,
+ AutoRegisterScope& scratch, Index mode = Offset,
+ Condition cc = Always);
+ void ma_dtr(LoadStore ls, Register rt, const Address& addr,
+ AutoRegisterScope& scratch, Index mode, Condition cc);
+
+ void ma_str(Register rt, DTRAddr addr, Index mode = Offset,
+ Condition cc = Always);
+ void ma_str(Register rt, const Address& addr, AutoRegisterScope& scratch,
+ Index mode = Offset, Condition cc = Always);
+
+ void ma_ldr(DTRAddr addr, Register rt, Index mode = Offset,
+ Condition cc = Always);
+ void ma_ldr(const Address& addr, Register rt, AutoRegisterScope& scratch,
+ Index mode = Offset, Condition cc = Always);
+
+ void ma_ldrb(DTRAddr addr, Register rt, Index mode = Offset,
+ Condition cc = Always);
+ void ma_ldrh(EDtrAddr addr, Register rt, Index mode = Offset,
+ Condition cc = Always);
+ void ma_ldrsh(EDtrAddr addr, Register rt, Index mode = Offset,
+ Condition cc = Always);
+ void ma_ldrsb(EDtrAddr addr, Register rt, Index mode = Offset,
+ Condition cc = Always);
+ void ma_ldrd(EDtrAddr addr, Register rt, mozilla::DebugOnly<Register> rt2,
+ Index mode = Offset, Condition cc = Always);
+ void ma_strb(Register rt, DTRAddr addr, Index mode = Offset,
+ Condition cc = Always);
+ void ma_strh(Register rt, EDtrAddr addr, Index mode = Offset,
+ Condition cc = Always);
+ void ma_strd(Register rt, mozilla::DebugOnly<Register> rt2, EDtrAddr addr,
+ Index mode = Offset, Condition cc = Always);
+
+ // Specialty for moving N bits of data, where n == 8,16,32,64.
+ BufferOffset ma_dataTransferN(LoadStore ls, int size, bool IsSigned,
+ Register rn, Register rm, Register rt,
+ AutoRegisterScope& scratch, Index mode = Offset,
+ Condition cc = Always, Scale scale = TimesOne);
+
+ BufferOffset ma_dataTransferN(LoadStore ls, int size, bool IsSigned,
+ Register rn, Register rm, Register rt,
+ Index mode = Offset, Condition cc = Always);
+
+ BufferOffset ma_dataTransferN(LoadStore ls, int size, bool IsSigned,
+ Register rn, Imm32 offset, Register rt,
+ AutoRegisterScope& scratch, Index mode = Offset,
+ Condition cc = Always);
+
+ void ma_pop(Register r);
+ void ma_popn_pc(Imm32 n, AutoRegisterScope& scratch,
+ AutoRegisterScope& scratch2);
+ void ma_push(Register r);
+ void ma_push_sp(Register r, AutoRegisterScope& scratch);
+
+ void ma_vpop(VFPRegister r);
+ void ma_vpush(VFPRegister r);
+
+ // Barriers.
+ void ma_dmb(BarrierOption option = BarrierSY);
+ void ma_dsb(BarrierOption option = BarrierSY);
+
+ // Branches when done from within arm-specific code.
+ BufferOffset ma_b(Label* dest, Condition c = Always);
+ void ma_b(void* target, Condition c = Always);
+ void ma_bx(Register dest, Condition c = Always);
+
+ // This is almost NEVER necessary, we'll basically never be calling a label
+ // except, possibly in the crazy bailout-table case.
+ void ma_bl(Label* dest, Condition c = Always);
+
+ void ma_blx(Register dest, Condition c = Always);
+
+ // VFP/ALU:
+ void ma_vadd(FloatRegister src1, FloatRegister src2, FloatRegister dst);
+ void ma_vsub(FloatRegister src1, FloatRegister src2, FloatRegister dst);
+
+ void ma_vmul(FloatRegister src1, FloatRegister src2, FloatRegister dst);
+ void ma_vdiv(FloatRegister src1, FloatRegister src2, FloatRegister dst);
+
+ void ma_vneg(FloatRegister src, FloatRegister dest, Condition cc = Always);
+ void ma_vmov(FloatRegister src, FloatRegister dest, Condition cc = Always);
+ void ma_vmov_f32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+ void ma_vabs(FloatRegister src, FloatRegister dest, Condition cc = Always);
+ void ma_vabs_f32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+
+ void ma_vsqrt(FloatRegister src, FloatRegister dest, Condition cc = Always);
+ void ma_vsqrt_f32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+
+ void ma_vimm(double value, FloatRegister dest, Condition cc = Always);
+ void ma_vimm_f32(float value, FloatRegister dest, Condition cc = Always);
+
+ void ma_vcmp(FloatRegister src1, FloatRegister src2, Condition cc = Always);
+ void ma_vcmp_f32(FloatRegister src1, FloatRegister src2,
+ Condition cc = Always);
+ void ma_vcmpz(FloatRegister src1, Condition cc = Always);
+ void ma_vcmpz_f32(FloatRegister src1, Condition cc = Always);
+
+ void ma_vadd_f32(FloatRegister src1, FloatRegister src2, FloatRegister dst);
+ void ma_vsub_f32(FloatRegister src1, FloatRegister src2, FloatRegister dst);
+
+ void ma_vmul_f32(FloatRegister src1, FloatRegister src2, FloatRegister dst);
+ void ma_vdiv_f32(FloatRegister src1, FloatRegister src2, FloatRegister dst);
+
+ void ma_vneg_f32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+
+ // Source is F64, dest is I32:
+ void ma_vcvt_F64_I32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+ void ma_vcvt_F64_U32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+
+ // Source is I32, dest is F64:
+ void ma_vcvt_I32_F64(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+ void ma_vcvt_U32_F64(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+
+ // Source is F32, dest is I32:
+ void ma_vcvt_F32_I32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+ void ma_vcvt_F32_U32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+
+ // Source is I32, dest is F32:
+ void ma_vcvt_I32_F32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+ void ma_vcvt_U32_F32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always);
+
+ // Transfer (do not coerce) a float into a gpr.
+ void ma_vxfer(VFPRegister src, Register dest, Condition cc = Always);
+ // Transfer (do not coerce) a double into a couple of gpr.
+ void ma_vxfer(VFPRegister src, Register dest1, Register dest2,
+ Condition cc = Always);
+
+ // Transfer (do not coerce) a gpr into a float
+ void ma_vxfer(Register src, FloatRegister dest, Condition cc = Always);
+ // Transfer (do not coerce) a couple of gpr into a double
+ void ma_vxfer(Register src1, Register src2, FloatRegister dest,
+ Condition cc = Always);
+
+ BufferOffset ma_vdtr(LoadStore ls, const Address& addr, VFPRegister dest,
+ AutoRegisterScope& scratch, Condition cc = Always);
+
+ BufferOffset ma_vldr(VFPAddr addr, VFPRegister dest, Condition cc = Always);
+ BufferOffset ma_vldr(const Address& addr, VFPRegister dest,
+ AutoRegisterScope& scratch, Condition cc = Always);
+ BufferOffset ma_vldr(VFPRegister src, Register base, Register index,
+ AutoRegisterScope& scratch, int32_t shift = defaultShift,
+ Condition cc = Always);
+
+ BufferOffset ma_vstr(VFPRegister src, VFPAddr addr, Condition cc = Always);
+ BufferOffset ma_vstr(VFPRegister src, const Address& addr,
+ AutoRegisterScope& scratch, Condition cc = Always);
+ BufferOffset ma_vstr(VFPRegister src, Register base, Register index,
+ AutoRegisterScope& scratch, AutoRegisterScope& scratch2,
+ int32_t shift, int32_t offset, Condition cc = Always);
+ BufferOffset ma_vstr(VFPRegister src, Register base, Register index,
+ AutoRegisterScope& scratch, int32_t shift,
+ Condition cc = Always);
+
+ void ma_call(ImmPtr dest);
+
+ // Float registers can only be loaded/stored in continuous runs when using
+ // vstm/vldm. This function breaks set into continuous runs and loads/stores
+ // them at [rm]. rm will be modified and left in a state logically suitable
+ // for the next load/store. Returns the offset from [dm] for the logical
+ // next load/store.
+ int32_t transferMultipleByRuns(FloatRegisterSet set, LoadStore ls,
+ Register rm, DTMMode mode) {
+ if (mode == IA) {
+ return transferMultipleByRunsImpl<FloatRegisterForwardIterator>(
+ set, ls, rm, mode, 1);
+ }
+ if (mode == DB) {
+ return transferMultipleByRunsImpl<FloatRegisterBackwardIterator>(
+ set, ls, rm, mode, -1);
+ }
+ MOZ_CRASH("Invalid data transfer addressing mode");
+ }
+
+ // `outAny` is valid if and only if `out64` == Register64::Invalid().
+ void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, AnyRegister outAny,
+ Register64 out64);
+
+ // `valAny` is valid if and only if `val64` == Register64::Invalid().
+ void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister valAny,
+ Register64 val64, Register memoryBase, Register ptr,
+ Register ptrScratch);
+
+ private:
+ // Implementation for transferMultipleByRuns so we can use different
+ // iterators for forward/backward traversals. The sign argument should be 1
+ // if we traverse forwards, -1 if we traverse backwards.
+ template <typename RegisterIterator>
+ int32_t transferMultipleByRunsImpl(FloatRegisterSet set, LoadStore ls,
+ Register rm, DTMMode mode, int32_t sign) {
+ MOZ_ASSERT(sign == 1 || sign == -1);
+
+ int32_t delta = sign * sizeof(float);
+ int32_t offset = 0;
+ // Build up a new set, which is the sum of all of the single and double
+ // registers. This set can have up to 48 registers in it total
+ // s0-s31 and d16-d31
+ FloatRegisterSet mod = set.reduceSetForPush();
+
+ RegisterIterator iter(mod);
+ while (iter.more()) {
+ startFloatTransferM(ls, rm, mode, WriteBack);
+ int32_t reg = (*iter).code();
+ do {
+ offset += delta;
+ if ((*iter).isDouble()) {
+ offset += delta;
+ }
+ transferFloatReg(*iter);
+ } while ((++iter).more() && int32_t((*iter).code()) == (reg += sign));
+ finishFloatTransfer();
+ }
+ return offset;
+ }
+};
+
+class MacroAssembler;
+
+class MacroAssemblerARMCompat : public MacroAssemblerARM {
+ private:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ public:
+ MacroAssemblerARMCompat() {}
+
+ public:
+ // Jumps + other functions that should be called from non-arm specific
+ // code. Basically, an x86 front end on top of the ARM code.
+ void j(Condition code, Label* dest) { as_b(dest, code); }
+ void j(Label* dest) { as_b(dest, Always); }
+
+ void mov(Register src, Register dest) { ma_mov(src, dest); }
+ void mov(ImmWord imm, Register dest) { ma_mov(Imm32(imm.value), dest); }
+ void mov(ImmPtr imm, Register dest) {
+ mov(ImmWord(uintptr_t(imm.value)), dest);
+ }
+
+ void branch(JitCode* c) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ScratchRegisterScope scratch(asMasm());
+ ma_movPatchable(ImmPtr(c->raw()), scratch, Always);
+ ma_bx(scratch);
+ }
+ void branch(const Register reg) { ma_bx(reg); }
+ void nop() { ma_nop(); }
+ void shortJumpSizedNop() { ma_nop(); }
+ void ret() { ma_pop(pc); }
+ void retn(Imm32 n) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_popn_pc(n, scratch, scratch2);
+ }
+ void push(Imm32 imm) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_mov(imm, scratch);
+ ma_push(scratch);
+ }
+ void push(ImmWord imm) { push(Imm32(imm.value)); }
+ void push(ImmGCPtr imm) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_mov(imm, scratch);
+ ma_push(scratch);
+ }
+ void push(const Address& addr) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(addr, scratch, scratch2);
+ ma_push(scratch);
+ }
+ void push(Register reg) {
+ if (reg == sp) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_push_sp(reg, scratch);
+ } else {
+ ma_push(reg);
+ }
+ }
+ void push(FloatRegister reg) { ma_vpush(VFPRegister(reg)); }
+ void pushWithPadding(Register reg, const Imm32 extraSpace) {
+ ScratchRegisterScope scratch(asMasm());
+ Imm32 totSpace = Imm32(extraSpace.value + 4);
+ ma_dtr(IsStore, sp, totSpace, reg, scratch, PreIndex);
+ }
+ void pushWithPadding(Imm32 imm, const Imm32 extraSpace) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ Imm32 totSpace = Imm32(extraSpace.value + 4);
+ ma_mov(imm, scratch);
+ ma_dtr(IsStore, sp, totSpace, scratch, scratch2, PreIndex);
+ }
+
+ void pop(Register reg) { ma_pop(reg); }
+ void pop(FloatRegister reg) { ma_vpop(VFPRegister(reg)); }
+
+ void popN(Register reg, Imm32 extraSpace) {
+ ScratchRegisterScope scratch(asMasm());
+ Imm32 totSpace = Imm32(extraSpace.value + 4);
+ ma_dtr(IsLoad, sp, totSpace, reg, scratch, PostIndex);
+ }
+
+ CodeOffset toggledJump(Label* label);
+
+ // Emit a BLX or NOP instruction. ToggleCall can be used to patch this
+ // instruction.
+ CodeOffset toggledCall(JitCode* target, bool enabled);
+
+ CodeOffset pushWithPatch(ImmWord imm) {
+ ScratchRegisterScope scratch(asMasm());
+ CodeOffset label = movWithPatch(imm, scratch);
+ ma_push(scratch);
+ return label;
+ }
+
+ CodeOffset movWithPatch(ImmWord imm, Register dest) {
+ CodeOffset label = CodeOffset(currentOffset());
+ ma_movPatchable(Imm32(imm.value), dest, Always);
+ return label;
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ return movWithPatch(ImmWord(uintptr_t(imm.value)), dest);
+ }
+
+ void jump(Label* label) { as_b(label); }
+ void jump(JitCode* code) { branch(code); }
+ void jump(ImmPtr ptr) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(ptr, scratch);
+ ma_bx(scratch);
+ }
+ void jump(TrampolinePtr code) { jump(ImmPtr(code.value)); }
+ void jump(Register reg) { ma_bx(reg); }
+ void jump(const Address& addr) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(addr, scratch, scratch2);
+ ma_bx(scratch);
+ }
+
+ void negl(Register reg) { ma_neg(reg, reg, SetCC); }
+ void test32(Register lhs, Register rhs) { ma_tst(lhs, rhs); }
+ void test32(Register lhs, Imm32 imm) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_tst(lhs, imm, scratch);
+ }
+ void test32(const Address& addr, Imm32 imm) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ldr(addr, scratch, scratch2);
+ ma_tst(scratch, imm, scratch2);
+ }
+ void testPtr(Register lhs, Register rhs) { test32(lhs, rhs); }
+
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ MOZ_ASSERT(value.typeReg() == tag);
+ }
+
+ // Higher level tag testing code.
+ Condition testInt32(Condition cond, const ValueOperand& value);
+ Condition testBoolean(Condition cond, const ValueOperand& value);
+ Condition testDouble(Condition cond, const ValueOperand& value);
+ Condition testNull(Condition cond, const ValueOperand& value);
+ Condition testUndefined(Condition cond, const ValueOperand& value);
+ Condition testString(Condition cond, const ValueOperand& value);
+ Condition testSymbol(Condition cond, const ValueOperand& value);
+ Condition testBigInt(Condition cond, const ValueOperand& value);
+ Condition testObject(Condition cond, const ValueOperand& value);
+ Condition testNumber(Condition cond, const ValueOperand& value);
+ Condition testMagic(Condition cond, const ValueOperand& value);
+
+ Condition testPrimitive(Condition cond, const ValueOperand& value);
+ Condition testGCThing(Condition cond, const ValueOperand& value);
+
+ // Register-based tests.
+ Condition testInt32(Condition cond, Register tag);
+ Condition testBoolean(Condition cond, Register tag);
+ Condition testNull(Condition cond, Register tag);
+ Condition testUndefined(Condition cond, Register tag);
+ Condition testString(Condition cond, Register tag);
+ Condition testSymbol(Condition cond, Register tag);
+ Condition testBigInt(Condition cond, Register tag);
+ Condition testObject(Condition cond, Register tag);
+ Condition testDouble(Condition cond, Register tag);
+ Condition testNumber(Condition cond, Register tag);
+ Condition testMagic(Condition cond, Register tag);
+ Condition testPrimitive(Condition cond, Register tag);
+ Condition testGCThing(Condition cond, Register tag);
+
+ Condition testGCThing(Condition cond, const Address& address);
+ Condition testMagic(Condition cond, const Address& address);
+ Condition testInt32(Condition cond, const Address& address);
+ Condition testDouble(Condition cond, const Address& address);
+ Condition testBoolean(Condition cond, const Address& address);
+ Condition testNull(Condition cond, const Address& address);
+ Condition testUndefined(Condition cond, const Address& address);
+ Condition testString(Condition cond, const Address& address);
+ Condition testSymbol(Condition cond, const Address& address);
+ Condition testBigInt(Condition cond, const Address& address);
+ Condition testObject(Condition cond, const Address& address);
+ Condition testNumber(Condition cond, const Address& address);
+
+ Condition testUndefined(Condition cond, const BaseIndex& src);
+ Condition testNull(Condition cond, const BaseIndex& src);
+ Condition testBoolean(Condition cond, const BaseIndex& src);
+ Condition testString(Condition cond, const BaseIndex& src);
+ Condition testSymbol(Condition cond, const BaseIndex& src);
+ Condition testBigInt(Condition cond, const BaseIndex& src);
+ Condition testInt32(Condition cond, const BaseIndex& src);
+ Condition testObject(Condition cond, const BaseIndex& src);
+ Condition testDouble(Condition cond, const BaseIndex& src);
+ Condition testMagic(Condition cond, const BaseIndex& src);
+ Condition testGCThing(Condition cond, const BaseIndex& src);
+
+ // Unboxing code.
+ void unboxNonDouble(const ValueOperand& operand, Register dest,
+ JSValueType type);
+ void unboxNonDouble(const Address& src, Register dest, JSValueType type);
+ void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type);
+ void unboxInt32(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_INT32);
+ }
+ void unboxInt32(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_INT32);
+ }
+ void unboxInt32(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_INT32);
+ }
+ void unboxBoolean(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN);
+ }
+ void unboxBoolean(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN);
+ }
+ void unboxBoolean(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN);
+ }
+ void unboxString(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxSymbol(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxSymbol(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxBigInt(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxObject(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObjectOrNull(const ValueOperand& src, Register dest) {
+ // Due to Spectre mitigation logic (see Value.h), if the value is an Object
+ // then this yields the object; otherwise it yields zero (null), as desired.
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObjectOrNull(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObjectOrNull(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxDouble(const ValueOperand& src, FloatRegister dest);
+ void unboxDouble(const Address& src, FloatRegister dest);
+ void unboxDouble(const BaseIndex& src, FloatRegister dest);
+
+ void unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType type);
+
+ // See comment in MacroAssembler-x64.h.
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ load32(ToPayload(src), dest);
+ }
+
+ void notBoolean(const ValueOperand& val) {
+ as_eor(val.payloadReg(), val.payloadReg(), Imm8(1));
+ }
+
+ template <typename T>
+ void fallibleUnboxPtrImpl(const T& src, Register dest, JSValueType type,
+ Label* fail);
+
+ // Boxing code.
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister);
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ [[nodiscard]] Register extractObject(const Address& address,
+ Register scratch);
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ unboxNonDouble(value, value.payloadReg(), JSVAL_TYPE_OBJECT);
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ unboxNonDouble(value, value.payloadReg(), JSVAL_TYPE_SYMBOL);
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch);
+ [[nodiscard]] Register extractTag(const BaseIndex& address, Register scratch);
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ return value.typeReg();
+ }
+
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void loadInt32OrDouble(const Address& src, FloatRegister dest);
+ void loadInt32OrDouble(Register base, Register index, FloatRegister dest,
+ int32_t shift = defaultShift);
+ void loadConstantDouble(double dp, FloatRegister dest);
+
+ // Treat the value as a boolean, and set condition codes accordingly.
+ Condition testInt32Truthy(bool truthy, const ValueOperand& operand);
+ Condition testBooleanTruthy(bool truthy, const ValueOperand& operand);
+ Condition testDoubleTruthy(bool truthy, FloatRegister reg);
+ Condition testStringTruthy(bool truthy, const ValueOperand& value);
+ Condition testBigIntTruthy(bool truthy, const ValueOperand& value);
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void loadUnboxedValue(Address address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(address, dest.fpu());
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ ma_ldr(address, dest.gpr(), scratch);
+ }
+ }
+
+ void loadUnboxedValue(BaseIndex address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(address.base, address.index, dest.fpu(), address.scale);
+ } else {
+ load32(address, dest.gpr());
+ }
+ }
+
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes,
+ JSValueType) {
+ switch (nbytes) {
+ case 4:
+ storePtr(value.payloadReg(), address);
+ return;
+ case 1:
+ store8(value.payloadReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void storeValue(ValueOperand val, const Address& dst);
+ void storeValue(ValueOperand val, const BaseIndex& dest);
+ void storeValue(JSValueType type, Register reg, BaseIndex dest) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ int32_t payloadoffset = dest.offset + NUNBOX32_PAYLOAD_OFFSET;
+ int32_t typeoffset = dest.offset + NUNBOX32_TYPE_OFFSET;
+
+ ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
+
+ // Store the payload.
+ if (payloadoffset < 4096 && payloadoffset > -4096) {
+ ma_str(reg, DTRAddr(scratch, DtrOffImm(payloadoffset)));
+ } else {
+ ma_str(reg, Address(scratch, payloadoffset), scratch2);
+ }
+
+ // Store the type.
+ if (typeoffset < 4096 && typeoffset > -4096) {
+ // Encodable as DTRAddr, so only two instructions needed.
+ ma_mov(ImmTag(JSVAL_TYPE_TO_TAG(type)), scratch2);
+ ma_str(scratch2, DTRAddr(scratch, DtrOffImm(typeoffset)));
+ } else {
+ // Since there are only two scratch registers, the offset must be
+ // applied early using a third instruction to be safe.
+ ma_add(Imm32(typeoffset), scratch, scratch2);
+ ma_mov(ImmTag(JSVAL_TYPE_TO_TAG(type)), scratch2);
+ ma_str(scratch2, DTRAddr(scratch, DtrOffImm(0)));
+ }
+ }
+ void storeValue(JSValueType type, Register reg, Address dest) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ ma_str(reg, dest, scratch2);
+ ma_mov(ImmTag(JSVAL_TYPE_TO_TAG(type)), scratch);
+ ma_str(scratch, Address(dest.base, dest.offset + NUNBOX32_TYPE_OFFSET),
+ scratch2);
+ }
+ void storeValue(const Value& val, const Address& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ ma_mov(Imm32(val.toNunboxTag()), scratch);
+ ma_str(scratch, ToType(dest), scratch2);
+ if (val.isGCThing()) {
+ ma_mov(ImmGCPtr(val.toGCThing()), scratch);
+ } else {
+ ma_mov(Imm32(val.toNunboxPayload()), scratch);
+ }
+ ma_str(scratch, ToPayload(dest), scratch2);
+ }
+ void storeValue(const Value& val, BaseIndex dest) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ int32_t typeoffset = dest.offset + NUNBOX32_TYPE_OFFSET;
+ int32_t payloadoffset = dest.offset + NUNBOX32_PAYLOAD_OFFSET;
+
+ ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
+
+ // Store the type.
+ if (typeoffset < 4096 && typeoffset > -4096) {
+ ma_mov(Imm32(val.toNunboxTag()), scratch2);
+ ma_str(scratch2, DTRAddr(scratch, DtrOffImm(typeoffset)));
+ } else {
+ ma_add(Imm32(typeoffset), scratch, scratch2);
+ ma_mov(Imm32(val.toNunboxTag()), scratch2);
+ ma_str(scratch2, DTRAddr(scratch, DtrOffImm(0)));
+ // Restore scratch for the payload store.
+ ma_alu(dest.base, lsl(dest.index, dest.scale), scratch, OpAdd);
+ }
+
+ // Store the payload, marking if necessary.
+ if (payloadoffset < 4096 && payloadoffset > -4096) {
+ if (val.isGCThing()) {
+ ma_mov(ImmGCPtr(val.toGCThing()), scratch2);
+ } else {
+ ma_mov(Imm32(val.toNunboxPayload()), scratch2);
+ }
+ ma_str(scratch2, DTRAddr(scratch, DtrOffImm(payloadoffset)));
+ } else {
+ ma_add(Imm32(payloadoffset), scratch, scratch2);
+ if (val.isGCThing()) {
+ ma_mov(ImmGCPtr(val.toGCThing()), scratch2);
+ } else {
+ ma_mov(Imm32(val.toNunboxPayload()), scratch2);
+ }
+ ma_str(scratch2, DTRAddr(scratch, DtrOffImm(0)));
+ }
+ }
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ load32(ToType(src), temp);
+ store32(temp, ToType(dest));
+
+ load32(ToPayload(src), temp);
+ store32(temp, ToPayload(dest));
+ }
+
+ void storePrivateValue(Register src, const Address& dest) {
+ store32(Imm32(0), ToType(dest));
+ store32(src, ToPayload(dest));
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ store32(Imm32(0), ToType(dest));
+ storePtr(imm, ToPayload(dest));
+ }
+
+ void loadValue(Address src, ValueOperand val);
+ void loadValue(Operand dest, ValueOperand val) {
+ loadValue(dest.toAddress(), val);
+ }
+ void loadValue(const BaseIndex& addr, ValueOperand val);
+
+ // Like loadValue but guaranteed to not use LDRD or LDM instructions (these
+ // don't support unaligned accesses).
+ void loadUnalignedValue(const Address& src, ValueOperand dest);
+
+ void tagValue(JSValueType type, Register payload, ValueOperand dest);
+
+ void pushValue(ValueOperand val);
+ void popValue(ValueOperand val);
+ void pushValue(const Value& val) {
+ push(Imm32(val.toNunboxTag()));
+ if (val.isGCThing()) {
+ push(ImmGCPtr(val.toGCThing()));
+ } else {
+ push(Imm32(val.toNunboxPayload()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ push(ImmTag(JSVAL_TYPE_TO_TAG(type)));
+ ma_push(reg);
+ }
+ void pushValue(const Address& addr);
+
+ void storePayload(const Value& val, const Address& dest);
+ void storePayload(Register src, const Address& dest);
+ void storePayload(const Value& val, const BaseIndex& dest);
+ void storePayload(Register src, const BaseIndex& dest);
+ void storeTypeTag(ImmTag tag, const Address& dest);
+ void storeTypeTag(ImmTag tag, const BaseIndex& dest);
+
+ void handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail);
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+ public:
+ void not32(Register reg);
+
+ void move32(Imm32 imm, Register dest);
+ void move32(Register src, Register dest);
+
+ void movePtr(Register src, Register dest);
+ void movePtr(ImmWord imm, Register dest);
+ void movePtr(ImmPtr imm, Register dest);
+ void movePtr(wasm::SymbolicAddress imm, Register dest);
+ void movePtr(ImmGCPtr imm, Register dest);
+
+ void load8SignExtend(const Address& address, Register dest);
+ void load8SignExtend(const BaseIndex& src, Register dest);
+
+ void load8ZeroExtend(const Address& address, Register dest);
+ void load8ZeroExtend(const BaseIndex& src, Register dest);
+
+ void load16SignExtend(const Address& address, Register dest);
+ void load16SignExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ void load16UnalignedSignExtend(const S& src, Register dest) {
+ // load16SignExtend uses |ldrsh|, which supports unaligned access.
+ load16SignExtend(src, dest);
+ }
+
+ void load16ZeroExtend(const Address& address, Register dest);
+ void load16ZeroExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ void load16UnalignedZeroExtend(const S& src, Register dest) {
+ // load16ZeroExtend uses |ldrh|, which supports unaligned access.
+ load16ZeroExtend(src, dest);
+ }
+
+ void load32(const Address& address, Register dest);
+ void load32(const BaseIndex& address, Register dest);
+ void load32(AbsoluteAddress address, Register dest);
+
+ template <typename S>
+ void load32Unaligned(const S& src, Register dest) {
+ // load32 uses |ldr|, which supports unaligned access.
+ load32(src, dest);
+ }
+
+ void load64(const Address& address, Register64 dest) {
+ bool highBeforeLow = address.base == dest.low;
+ if (highBeforeLow) {
+ load32(HighWord(address), dest.high);
+ load32(LowWord(address), dest.low);
+ } else {
+ load32(LowWord(address), dest.low);
+ load32(HighWord(address), dest.high);
+ }
+ }
+ void load64(const BaseIndex& address, Register64 dest) {
+ // If you run into this, relax your register allocation constraints.
+ MOZ_RELEASE_ASSERT(
+ !((address.base == dest.low || address.base == dest.high) &&
+ (address.index == dest.low || address.index == dest.high)));
+ bool highBeforeLow = address.base == dest.low || address.index == dest.low;
+ if (highBeforeLow) {
+ load32(HighWord(address), dest.high);
+ load32(LowWord(address), dest.low);
+ } else {
+ load32(LowWord(address), dest.low);
+ load32(HighWord(address), dest.high);
+ }
+ }
+
+ template <typename S>
+ void load64Unaligned(const S& src, Register64 dest) {
+ // load64 calls load32, which supports unaligned accesses.
+ load64(src, dest);
+ }
+
+ void loadPtr(const Address& address, Register dest);
+ void loadPtr(const BaseIndex& src, Register dest);
+ void loadPtr(AbsoluteAddress address, Register dest);
+ void loadPtr(wasm::SymbolicAddress address, Register dest);
+
+ void loadPrivate(const Address& address, Register dest);
+
+ void loadDouble(const Address& addr, FloatRegister dest);
+ void loadDouble(const BaseIndex& src, FloatRegister dest);
+
+ // Load a float value into a register, then expand it to a double.
+ void loadFloatAsDouble(const Address& addr, FloatRegister dest);
+ void loadFloatAsDouble(const BaseIndex& src, FloatRegister dest);
+
+ void loadFloat32(const Address& addr, FloatRegister dest);
+ void loadFloat32(const BaseIndex& src, FloatRegister dest);
+
+ void store8(Register src, const Address& address);
+ void store8(Imm32 imm, const Address& address);
+ void store8(Register src, const BaseIndex& address);
+ void store8(Imm32 imm, const BaseIndex& address);
+
+ void store16(Register src, const Address& address);
+ void store16(Imm32 imm, const Address& address);
+ void store16(Register src, const BaseIndex& address);
+ void store16(Imm32 imm, const BaseIndex& address);
+
+ template <typename S, typename T>
+ void store16Unaligned(const S& src, const T& dest) {
+ // store16 uses |strh|, which supports unaligned access.
+ store16(src, dest);
+ }
+
+ void store32(Register src, AbsoluteAddress address);
+ void store32(Register src, const Address& address);
+ void store32(Register src, const BaseIndex& address);
+ void store32(Imm32 src, const Address& address);
+ void store32(Imm32 src, const BaseIndex& address);
+
+ template <typename S, typename T>
+ void store32Unaligned(const S& src, const T& dest) {
+ // store32 uses |str|, which supports unaligned access.
+ store32(src, dest);
+ }
+
+ void store64(Register64 src, Address address) {
+ store32(src.low, LowWord(address));
+ store32(src.high, HighWord(address));
+ }
+
+ void store64(Register64 src, const BaseIndex& address) {
+ store32(src.low, LowWord(address));
+ store32(src.high, HighWord(address));
+ }
+
+ void store64(Imm64 imm, Address address) {
+ store32(imm.low(), LowWord(address));
+ store32(imm.hi(), HighWord(address));
+ }
+
+ void store64(Imm64 imm, const BaseIndex& address) {
+ store32(imm.low(), LowWord(address));
+ store32(imm.hi(), HighWord(address));
+ }
+
+ template <typename S, typename T>
+ void store64Unaligned(const S& src, const T& dest) {
+ // store64 calls store32, which supports unaligned access.
+ store64(src, dest);
+ }
+
+ void storePtr(ImmWord imm, const Address& address);
+ void storePtr(ImmWord imm, const BaseIndex& address);
+ void storePtr(ImmPtr imm, const Address& address);
+ void storePtr(ImmPtr imm, const BaseIndex& address);
+ void storePtr(ImmGCPtr imm, const Address& address);
+ void storePtr(ImmGCPtr imm, const BaseIndex& address);
+ void storePtr(Register src, const Address& address);
+ void storePtr(Register src, const BaseIndex& address);
+ void storePtr(Register src, AbsoluteAddress dest);
+
+ void moveDouble(FloatRegister src, FloatRegister dest,
+ Condition cc = Always) {
+ ma_vmov(src, dest, cc);
+ }
+
+ inline void incrementInt32Value(const Address& addr);
+
+ void cmp32(Register lhs, Imm32 rhs);
+ void cmp32(Register lhs, Register rhs);
+ void cmp32(const Address& lhs, Imm32 rhs);
+ void cmp32(const Address& lhs, Register rhs);
+
+ void cmpPtr(Register lhs, Register rhs);
+ void cmpPtr(Register lhs, ImmWord rhs);
+ void cmpPtr(Register lhs, ImmPtr rhs);
+ void cmpPtr(Register lhs, ImmGCPtr rhs);
+ void cmpPtr(Register lhs, Imm32 rhs);
+ void cmpPtr(const Address& lhs, Register rhs);
+ void cmpPtr(const Address& lhs, ImmWord rhs);
+ void cmpPtr(const Address& lhs, ImmPtr rhs);
+ void cmpPtr(const Address& lhs, ImmGCPtr rhs);
+ void cmpPtr(const Address& lhs, Imm32 rhs);
+
+ void setStackArg(Register reg, uint32_t arg);
+
+ void breakpoint();
+ // Conditional breakpoint.
+ void breakpoint(Condition cc);
+
+ // Trigger the simulator's interactive read-eval-print loop.
+ // The message will be printed at the stopping point.
+ // (On non-simulator builds, does nothing.)
+ void simulatorStop(const char* msg);
+
+ // Evaluate srcDest = minmax<isMax>{Float32,Double}(srcDest, other).
+ // Checks for NaN if canBeNaN is true.
+ void minMaxDouble(FloatRegister srcDest, FloatRegister other, bool canBeNaN,
+ bool isMax);
+ void minMaxFloat32(FloatRegister srcDest, FloatRegister other, bool canBeNaN,
+ bool isMax);
+
+ void compareDouble(FloatRegister lhs, FloatRegister rhs);
+
+ void compareFloat(FloatRegister lhs, FloatRegister rhs);
+
+ void checkStackAlignment();
+
+ // If source is a double, load it into dest. If source is int32, convert it
+ // to double. Else, branch to failure.
+ void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ void emitSet(Assembler::Condition cond, Register dest) {
+ ma_mov(Imm32(0), dest);
+ ma_mov(Imm32(1), dest, cond);
+ }
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testNull(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testObject(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest) {
+ cond = testUndefined(cond, value);
+ emitSet(cond, dest);
+ }
+
+ protected:
+ bool buildOOLFakeExitFrame(void* fakeReturnAddr);
+
+ public:
+ void computeEffectiveAddress(const Address& address, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_add(address.base, Imm32(address.offset), dest, scratch, LeaveCC);
+ }
+ void computeEffectiveAddress(const BaseIndex& address, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_alu(address.base, lsl(address.index, address.scale), dest, OpAdd,
+ LeaveCC);
+ if (address.offset) {
+ ma_add(dest, Imm32(address.offset), dest, scratch, LeaveCC);
+ }
+ }
+ void floor(FloatRegister input, Register output, Label* handleNotAnInt);
+ void floorf(FloatRegister input, Register output, Label* handleNotAnInt);
+ void ceil(FloatRegister input, Register output, Label* handleNotAnInt);
+ void ceilf(FloatRegister input, Register output, Label* handleNotAnInt);
+ void round(FloatRegister input, Register output, Label* handleNotAnInt,
+ FloatRegister tmp);
+ void roundf(FloatRegister input, Register output, Label* handleNotAnInt,
+ FloatRegister tmp);
+ void trunc(FloatRegister input, Register output, Label* handleNotAnInt);
+ void truncf(FloatRegister input, Register output, Label* handleNotAnInt);
+
+ void clampCheck(Register r, Label* handleNotAnInt) {
+ // Check explicitly for r == INT_MIN || r == INT_MAX
+ // This is the instruction sequence that gcc generated for this
+ // operation.
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_sub(r, Imm32(0x80000001), scratch, scratch2);
+ as_cmn(scratch, Imm8(3));
+ ma_b(handleNotAnInt, Above);
+ }
+
+ void lea(Operand addr, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_add(addr.baseReg(), Imm32(addr.disp()), dest, scratch);
+ }
+
+ void abiret() { as_bx(lr); }
+
+ void moveFloat32(FloatRegister src, FloatRegister dest,
+ Condition cc = Always) {
+ as_vmov(VFPRegister(dest).singleOverlay(), VFPRegister(src).singleOverlay(),
+ cc);
+ }
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+typedef MacroAssemblerARMCompat MacroAssemblerSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_MacroAssembler_arm_h */
diff --git a/js/src/jit/arm/MoveEmitter-arm.cpp b/js/src/jit/arm/MoveEmitter-arm.cpp
new file mode 100644
index 0000000000..1807c41b50
--- /dev/null
+++ b/js/src/jit/arm/MoveEmitter-arm.cpp
@@ -0,0 +1,413 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm/MoveEmitter-arm.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+MoveEmitterARM::MoveEmitterARM(MacroAssembler& masm)
+ : inCycle_(0),
+ masm(masm),
+ pushedAtCycle_(-1),
+ pushedAtSpill_(-1),
+ spilledReg_(InvalidReg),
+ spilledFloatReg_(InvalidFloatReg) {
+ pushedAtStart_ = masm.framePushed();
+}
+
+void MoveEmitterARM::emit(const MoveResolver& moves) {
+ if (moves.numCycles()) {
+ // Reserve stack for cycle resolution
+ static_assert(SpillSlotSize == 8);
+ masm.reserveStack(moves.numCycles() * SpillSlotSize);
+ pushedAtCycle_ = masm.framePushed();
+ }
+
+ for (size_t i = 0; i < moves.numMoves(); i++) {
+ emit(moves.getMove(i));
+ }
+}
+
+MoveEmitterARM::~MoveEmitterARM() { assertDone(); }
+
+Address MoveEmitterARM::cycleSlot(uint32_t slot, uint32_t subslot) const {
+ int32_t offset = masm.framePushed() - pushedAtCycle_;
+ MOZ_ASSERT(offset < 4096 && offset > -4096);
+ return Address(StackPointer, offset + slot * sizeof(double) + subslot);
+}
+
+Address MoveEmitterARM::spillSlot() const {
+ int32_t offset = masm.framePushed() - pushedAtSpill_;
+ MOZ_ASSERT(offset < 4096 && offset > -4096);
+ return Address(StackPointer, offset);
+}
+
+Address MoveEmitterARM::toAddress(const MoveOperand& operand) const {
+ MOZ_ASSERT(operand.isMemoryOrEffectiveAddress());
+
+ if (operand.base() != StackPointer) {
+ return Address(operand.base(), operand.disp());
+ }
+
+ MOZ_ASSERT(operand.disp() >= 0);
+
+ // Otherwise, the stack offset may need to be adjusted.
+ return Address(StackPointer,
+ operand.disp() + (masm.framePushed() - pushedAtStart_));
+}
+
+Register MoveEmitterARM::tempReg() {
+ if (spilledReg_ != InvalidReg) {
+ return spilledReg_;
+ }
+
+ // For now, just pick r12/ip as the eviction point. This is totally random,
+ // and if it ends up being bad, we can use actual heuristics later. r12 is
+ // actually a bad choice. It is the scratch register, which is frequently
+ // used for address computations, such as those found when we attempt to
+ // access values more than 4096 off of the stack pointer. Instead, use lr,
+ // the LinkRegister.
+ spilledReg_ = r14;
+ if (pushedAtSpill_ == -1) {
+ masm.Push(spilledReg_);
+ pushedAtSpill_ = masm.framePushed();
+ } else {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_str(spilledReg_, spillSlot(), scratch);
+ }
+ return spilledReg_;
+}
+
+void MoveEmitterARM::breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slotId) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (A -> B), which we reach first. We save B, then allow
+ // the original move to continue.
+
+ ScratchRegisterScope scratch(masm);
+
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ ScratchFloat32Scope scratchFloat32(masm);
+ masm.ma_vldr(toAddress(to), scratchFloat32, scratch);
+ // Since it is uncertain if the load will be aligned or not
+ // just fill both of them with the same value.
+ masm.ma_vstr(scratchFloat32, cycleSlot(slotId, 0), scratch);
+ masm.ma_vstr(scratchFloat32, cycleSlot(slotId, 4), scratch);
+ } else if (to.isGeneralReg()) {
+ // Since it is uncertain if the load will be aligned or not
+ // just fill both of them with the same value.
+ masm.ma_str(to.reg(), cycleSlot(slotId, 0), scratch);
+ masm.ma_str(to.reg(), cycleSlot(slotId, 4), scratch);
+ } else {
+ FloatRegister src = to.floatReg();
+ // Just always store the largest possible size. Currently, this is
+ // a double. When SIMD is added, two doubles will need to be stored.
+ masm.ma_vstr(src.doubleOverlay(), cycleSlot(slotId, 0), scratch);
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ ScratchDoubleScope scratchDouble(masm);
+ masm.ma_vldr(toAddress(to), scratchDouble, scratch);
+ masm.ma_vstr(scratchDouble, cycleSlot(slotId, 0), scratch);
+ } else if (to.isGeneralRegPair()) {
+ ScratchDoubleScope scratchDouble(masm);
+ masm.ma_vxfer(to.evenReg(), to.oddReg(), scratchDouble);
+ masm.ma_vstr(scratchDouble, cycleSlot(slotId, 0), scratch);
+ } else {
+ masm.ma_vstr(to.floatReg().doubleOverlay(), cycleSlot(slotId, 0),
+ scratch);
+ }
+ break;
+ case MoveOp::INT32:
+ case MoveOp::GENERAL:
+ // an non-vfp value
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.ma_ldr(toAddress(to), temp, scratch);
+ masm.ma_str(temp, cycleSlot(0, 0), scratch);
+ } else {
+ if (to.reg() == spilledReg_) {
+ // If the destination was spilled, restore it first.
+ masm.ma_ldr(spillSlot(), spilledReg_, scratch);
+ spilledReg_ = InvalidReg;
+ }
+ masm.ma_str(to.reg(), cycleSlot(0, 0), scratch);
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterARM::completeCycle(const MoveOperand& from,
+ const MoveOperand& to, MoveOp::Type type,
+ uint32_t slotId) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (B -> A), which we reach last. We emit a move from the
+ // saved value of B, to A.
+
+ ScratchRegisterScope scratch(masm);
+
+ switch (type) {
+ case MoveOp::FLOAT32:
+ MOZ_ASSERT(!to.isGeneralRegPair());
+ if (to.isMemory()) {
+ ScratchFloat32Scope scratchFloat32(masm);
+ masm.ma_vldr(cycleSlot(slotId, 0), scratchFloat32, scratch);
+ masm.ma_vstr(scratchFloat32, toAddress(to), scratch);
+ } else if (to.isGeneralReg()) {
+ MOZ_ASSERT(type == MoveOp::FLOAT32);
+ masm.ma_ldr(toAddress(from), to.reg(), scratch);
+ } else {
+ uint32_t offset = 0;
+ if ((!from.isMemory()) && from.floatReg().numAlignedAliased() == 1) {
+ offset = sizeof(float);
+ }
+ masm.ma_vldr(cycleSlot(slotId, offset), to.floatReg(), scratch);
+ }
+ break;
+ case MoveOp::DOUBLE:
+ MOZ_ASSERT(!to.isGeneralReg());
+ if (to.isMemory()) {
+ ScratchDoubleScope scratchDouble(masm);
+ masm.ma_vldr(cycleSlot(slotId, 0), scratchDouble, scratch);
+ masm.ma_vstr(scratchDouble, toAddress(to), scratch);
+ } else if (to.isGeneralRegPair()) {
+ MOZ_ASSERT(type == MoveOp::DOUBLE);
+ ScratchDoubleScope scratchDouble(masm);
+ masm.ma_vldr(toAddress(from), scratchDouble, scratch);
+ masm.ma_vxfer(scratchDouble, to.evenReg(), to.oddReg());
+ } else {
+ uint32_t offset = 0;
+ if ((!from.isMemory()) && from.floatReg().numAlignedAliased() == 1) {
+ offset = sizeof(float);
+ }
+ masm.ma_vldr(cycleSlot(slotId, offset), to.floatReg(), scratch);
+ }
+ break;
+ case MoveOp::INT32:
+ case MoveOp::GENERAL:
+ MOZ_ASSERT(slotId == 0);
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.ma_ldr(cycleSlot(slotId, 0), temp, scratch);
+ masm.ma_str(temp, toAddress(to), scratch);
+ } else {
+ if (to.reg() == spilledReg_) {
+ // Make sure we don't re-clobber the spilled register later.
+ spilledReg_ = InvalidReg;
+ }
+ masm.ma_ldr(cycleSlot(slotId, 0), to.reg(), scratch);
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterARM::emitMove(const MoveOperand& from, const MoveOperand& to) {
+ // Register pairs are used to store Double values during calls.
+ MOZ_ASSERT(!from.isGeneralRegPair());
+ MOZ_ASSERT(!to.isGeneralRegPair());
+
+ ScratchRegisterScope scratch(masm);
+
+ if (to.isGeneralReg() && to.reg() == spilledReg_) {
+ // If the destination is the spilled register, make sure we
+ // don't re-clobber its value.
+ spilledReg_ = InvalidReg;
+ }
+
+ if (from.isGeneralReg()) {
+ if (from.reg() == spilledReg_) {
+ // If the source is a register that has been spilled, make sure
+ // to load the source back into that register.
+ masm.ma_ldr(spillSlot(), spilledReg_, scratch);
+ spilledReg_ = InvalidReg;
+ }
+ if (to.isMemoryOrEffectiveAddress()) {
+ masm.ma_str(from.reg(), toAddress(to), scratch);
+ } else {
+ masm.ma_mov(from.reg(), to.reg());
+ }
+ } else if (to.isGeneralReg()) {
+ MOZ_ASSERT(from.isMemoryOrEffectiveAddress());
+ if (from.isMemory()) {
+ masm.ma_ldr(toAddress(from), to.reg(), scratch);
+ } else {
+ masm.ma_add(from.base(), Imm32(from.disp()), to.reg(), scratch);
+ }
+ } else {
+ // Memory to memory gpr move.
+ Register reg = tempReg();
+
+ MOZ_ASSERT(from.isMemoryOrEffectiveAddress());
+ if (from.isMemory()) {
+ masm.ma_ldr(toAddress(from), reg, scratch);
+ } else {
+ masm.ma_add(from.base(), Imm32(from.disp()), reg, scratch);
+ }
+ MOZ_ASSERT(to.base() != reg);
+ masm.ma_str(reg, toAddress(to), scratch);
+ }
+}
+
+void MoveEmitterARM::emitFloat32Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ // Register pairs are used to store Double values during calls.
+ MOZ_ASSERT(!from.isGeneralRegPair());
+ MOZ_ASSERT(!to.isGeneralRegPair());
+
+ ScratchRegisterScope scratch(masm);
+
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.ma_vmov_f32(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ masm.ma_vxfer(from.floatReg(), to.reg());
+ } else {
+ masm.ma_vstr(VFPRegister(from.floatReg()).singleOverlay(), toAddress(to),
+ scratch);
+ }
+ } else if (from.isGeneralReg()) {
+ if (to.isFloatReg()) {
+ masm.ma_vxfer(from.reg(), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ masm.ma_mov(from.reg(), to.reg());
+ } else {
+ masm.ma_str(from.reg(), toAddress(to), scratch);
+ }
+ } else if (to.isFloatReg()) {
+ masm.ma_vldr(toAddress(from), VFPRegister(to.floatReg()).singleOverlay(),
+ scratch);
+ } else if (to.isGeneralReg()) {
+ masm.ma_ldr(toAddress(from), to.reg(), scratch);
+ } else {
+ // Memory to memory move.
+ MOZ_ASSERT(from.isMemory());
+ ScratchFloat32Scope scratchFloat32(masm);
+ masm.ma_vldr(toAddress(from), scratchFloat32, scratch);
+ masm.ma_vstr(scratchFloat32, toAddress(to), scratch);
+ }
+}
+
+void MoveEmitterARM::emitDoubleMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ // Registers are used to store pointers / int32 / float32 values.
+ MOZ_ASSERT(!from.isGeneralReg());
+ MOZ_ASSERT(!to.isGeneralReg());
+
+ ScratchRegisterScope scratch(masm);
+
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.ma_vmov(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralRegPair()) {
+ masm.ma_vxfer(from.floatReg(), to.evenReg(), to.oddReg());
+ } else {
+ masm.ma_vstr(from.floatReg(), toAddress(to), scratch);
+ }
+ } else if (from.isGeneralRegPair()) {
+ if (to.isFloatReg()) {
+ masm.ma_vxfer(from.evenReg(), from.oddReg(), to.floatReg());
+ } else if (to.isGeneralRegPair()) {
+ MOZ_ASSERT(!from.aliases(to));
+ masm.ma_mov(from.evenReg(), to.evenReg());
+ masm.ma_mov(from.oddReg(), to.oddReg());
+ } else {
+ ScratchDoubleScope scratchDouble(masm);
+ masm.ma_vxfer(from.evenReg(), from.oddReg(), scratchDouble);
+ masm.ma_vstr(scratchDouble, toAddress(to), scratch);
+ }
+ } else if (to.isFloatReg()) {
+ masm.ma_vldr(toAddress(from), to.floatReg(), scratch);
+ } else if (to.isGeneralRegPair()) {
+ MOZ_ASSERT(from.isMemory());
+ Address src = toAddress(from);
+ // Note: We can safely use the MoveOperand's displacement here,
+ // even if the base is SP: MoveEmitter::toOperand adjusts
+ // SP-relative operands by the difference between the current
+ // stack usage and stackAdjust, which emitter.finish() resets to
+ // 0.
+ //
+ // Warning: if the offset isn't within [-255,+255] then this
+ // will assert-fail (or, if non-debug, load the wrong words).
+ // Nothing uses such an offset at the time of this writing.
+ masm.ma_ldrd(EDtrAddr(src.base, EDtrOffImm(src.offset)), to.evenReg(),
+ to.oddReg());
+ } else {
+ // Memory to memory move.
+ MOZ_ASSERT(from.isMemory());
+ ScratchDoubleScope scratchDouble(masm);
+ masm.ma_vldr(toAddress(from), scratchDouble, scratch);
+ masm.ma_vstr(scratchDouble, toAddress(to), scratch);
+ }
+}
+
+void MoveEmitterARM::emit(const MoveOp& move) {
+ const MoveOperand& from = move.from();
+ const MoveOperand& to = move.to();
+
+ if (move.isCycleEnd() && move.isCycleBegin()) {
+ // A fun consequence of aliased registers is you can have multiple
+ // cycles at once, and one can end exactly where another begins.
+ breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot());
+ completeCycle(from, to, move.type(), move.cycleEndSlot());
+ return;
+ }
+
+ if (move.isCycleEnd()) {
+ MOZ_ASSERT(inCycle_);
+ completeCycle(from, to, move.type(), move.cycleEndSlot());
+ MOZ_ASSERT(inCycle_ > 0);
+ inCycle_--;
+ return;
+ }
+
+ if (move.isCycleBegin()) {
+ breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot());
+ inCycle_++;
+ }
+
+ switch (move.type()) {
+ case MoveOp::FLOAT32:
+ emitFloat32Move(from, to);
+ break;
+ case MoveOp::DOUBLE:
+ emitDoubleMove(from, to);
+ break;
+ case MoveOp::INT32:
+ case MoveOp::GENERAL:
+ emitMove(from, to);
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterARM::assertDone() { MOZ_ASSERT(inCycle_ == 0); }
+
+void MoveEmitterARM::finish() {
+ assertDone();
+
+ if (pushedAtSpill_ != -1 && spilledReg_ != InvalidReg) {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_ldr(spillSlot(), spilledReg_, scratch);
+ }
+ masm.freeStack(masm.framePushed() - pushedAtStart_);
+}
diff --git a/js/src/jit/arm/MoveEmitter-arm.h b/js/src/jit/arm/MoveEmitter-arm.h
new file mode 100644
index 0000000000..26a84fdbcc
--- /dev/null
+++ b/js/src/jit/arm/MoveEmitter-arm.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_MoveEmitter_arm_h
+#define jit_arm_MoveEmitter_arm_h
+
+#include <stdint.h>
+
+#include "jit/MoveResolver.h"
+#include "jit/Registers.h"
+
+namespace js {
+namespace jit {
+
+struct Address;
+class MacroAssembler;
+
+class MoveEmitterARM {
+ uint32_t inCycle_;
+ MacroAssembler& masm;
+
+ // Original stack push value.
+ uint32_t pushedAtStart_;
+
+ // These store stack offsets to spill locations, snapshotting
+ // codegen->framePushed_ at the time they were allocated. They are -1 if no
+ // stack space has been allocated for that particular spill.
+ int32_t pushedAtCycle_;
+ int32_t pushedAtSpill_;
+
+ // These are registers that are available for temporary use. They may be
+ // assigned InvalidReg. If no corresponding spill space has been assigned,
+ // then these registers do not need to be spilled.
+ Register spilledReg_;
+ FloatRegister spilledFloatReg_;
+
+ void assertDone();
+ Register tempReg();
+ FloatRegister tempFloatReg();
+ Address cycleSlot(uint32_t slot, uint32_t subslot) const;
+ Address spillSlot() const;
+ Address toAddress(const MoveOperand& operand) const;
+
+ void emitMove(const MoveOperand& from, const MoveOperand& to);
+ void emitFloat32Move(const MoveOperand& from, const MoveOperand& to);
+ void emitDoubleMove(const MoveOperand& from, const MoveOperand& to);
+ void breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+ void completeCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+ void emit(const MoveOp& move);
+
+ public:
+ explicit MoveEmitterARM(MacroAssembler& masm);
+ ~MoveEmitterARM();
+ void emit(const MoveResolver& moves);
+ void finish();
+
+ void setScratchRegister(Register reg) {}
+};
+
+typedef MoveEmitterARM MoveEmitter;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_MoveEmitter_arm_h */
diff --git a/js/src/jit/arm/SharedICHelpers-arm-inl.h b/js/src/jit/arm/SharedICHelpers-arm-inl.h
new file mode 100644
index 0000000000..2943bafbd8
--- /dev/null
+++ b/js/src/jit/arm/SharedICHelpers-arm-inl.h
@@ -0,0 +1,79 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_SharedICHelpers_arm_inl_h
+#define jit_arm_SharedICHelpers_arm_inl_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+ uint32_t argSize) {
+#ifdef DEBUG
+ // We assume during this that R0 and R1 have been pushed, and that R2 is
+ // unused.
+ static_assert(R2 == ValueOperand(r1, r0));
+
+ // Store frame size without VMFunction arguments for debug assertions.
+ masm.movePtr(FramePointer, r0);
+ masm.ma_sub(StackPointer, r0);
+ masm.sub32(Imm32(argSize), r0);
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(r0, frameSizeAddr);
+#endif
+
+ // Push frame descriptor and perform the tail call.
+ // ICTailCallReg (lr) already contains the return address (as we keep
+ // it there through the stub calls), but the VMWrapper code being called
+ // expects the return address to also be pushed on the stack.
+ static_assert(ICTailCallReg == lr);
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(lr);
+ masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) {
+ MOZ_ASSERT(scratch != ICTailCallReg);
+
+#ifdef DEBUG
+ // Compute frame size.
+ masm.mov(FramePointer, scratch);
+ masm.ma_sub(StackPointer, scratch);
+
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Push frame descriptor and return address.
+ masm.PushFrameDescriptor(FrameType::BaselineJS);
+ masm.Push(ICTailCallReg);
+
+ // Save old frame pointer, stack pointer and stub reg.
+ masm.Push(FramePointer);
+ masm.mov(StackPointer, FramePointer);
+
+ masm.Push(ICStubReg);
+
+ // We pushed 4 words, so the stack is still aligned to 8 bytes.
+ masm.checkStackAlignment();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_SharedICHelpers_arm_inl_h */
diff --git a/js/src/jit/arm/SharedICHelpers-arm.h b/js/src/jit/arm/SharedICHelpers-arm.h
new file mode 100644
index 0000000000..93475abc62
--- /dev/null
+++ b/js/src/jit/arm/SharedICHelpers-arm.h
@@ -0,0 +1,80 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_SharedICHelpers_arm_h
+#define jit_arm_SharedICHelpers_arm_h
+
+#include "jit/BaselineIC.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from sp to the top Value inside an IC stub (no return address on the
+// stack on ARM).
+static const size_t ICStackValueOffset = 0;
+
+inline void EmitRestoreTailCallReg(MacroAssembler& masm) {
+ // No-op on ARM because link register is always holding the return address.
+}
+
+inline void EmitRepushTailCallReg(MacroAssembler& masm) {
+ // No-op on ARM because link register is always holding the return address.
+}
+
+inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
+ // The stub pointer must already be in ICStubReg.
+ // Load stubcode pointer from the ICStub.
+ // R2 won't be active when we call ICs, so we can use r0.
+ static_assert(R2 == ValueOperand(r1, r0));
+ masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), r0);
+
+ // Call the stubcode via a direct branch-and-link.
+ masm.ma_blx(r0);
+ *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void EmitReturnFromIC(MacroAssembler& masm) { masm.ma_mov(lr, pc); }
+
+inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
+ Address stubAddr(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP);
+ masm.loadPtr(stubAddr, ICStubReg);
+
+ masm.mov(FramePointer, StackPointer);
+ masm.Pop(FramePointer);
+
+ // Load the return address.
+ masm.Pop(ICTailCallReg);
+
+ // Discard the frame descriptor.
+ ScratchRegisterScope scratch(masm);
+ masm.Pop(scratch);
+}
+
+template <typename AddrType>
+inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr,
+ MIRType type) {
+ // On ARM, lr is clobbered by guardedCallPreBarrier. Save it first.
+ masm.push(lr);
+ masm.guardedCallPreBarrier(addr, type);
+ masm.pop(lr);
+}
+
+inline void EmitStubGuardFailure(MacroAssembler& masm) {
+ // Load next stub into ICStubReg.
+ masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+ // Return address is already loaded, just jump to the next stubcode.
+ static_assert(ICTailCallReg == lr);
+ masm.jump(Address(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_SharedICHelpers_arm_h */
diff --git a/js/src/jit/arm/SharedICRegisters-arm.h b/js/src/jit/arm/SharedICRegisters-arm.h
new file mode 100644
index 0000000000..16aabbf0b3
--- /dev/null
+++ b/js/src/jit/arm/SharedICRegisters-arm.h
@@ -0,0 +1,52 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm_SharedICRegisters_arm_h
+#define jit_arm_SharedICRegisters_arm_h
+
+#include "jit/arm/Assembler-arm.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+// r15 = program-counter
+// r14 = link-register
+// r13 = stack-pointer
+// r11 = frame-pointer
+
+// ValueOperands R0, R1, and R2.
+// R0 == JSReturnReg, and R2 uses registers not preserved across calls. R1 value
+// should be preserved across calls.
+static constexpr ValueOperand R0(r3, r2);
+static constexpr ValueOperand R1(r5, r4);
+static constexpr ValueOperand R2(r1, r0);
+
+// ICTailCallReg and ICStubReg
+// These use registers that are not preserved across calls.
+static constexpr Register ICTailCallReg = r14;
+static constexpr Register ICStubReg = r9;
+
+// Register used internally by MacroAssemblerARM.
+static constexpr Register BaselineSecondScratchReg = r6;
+
+// R7 - R9 are generally available for use within stubcode.
+
+// Note that ICTailCallReg is actually just the link register. In ARM code
+// emission, we do not clobber ICTailCallReg since we keep the return
+// address for calls there.
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = d0;
+static constexpr FloatRegister FloatReg1 = d1;
+static constexpr FloatRegister FloatReg2 = d2;
+static constexpr FloatRegister FloatReg3 = d3;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm_SharedICRegisters_arm_h */
diff --git a/js/src/jit/arm/Simulator-arm.cpp b/js/src/jit/arm/Simulator-arm.cpp
new file mode 100644
index 0000000000..6113792fc1
--- /dev/null
+++ b/js/src/jit/arm/Simulator-arm.cpp
@@ -0,0 +1,5449 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm/Simulator-arm.h"
+
+#include "mozilla/Casting.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/Likely.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/arm/Assembler-arm.h"
+#include "jit/arm/disasm/Constants-arm.h"
+#include "jit/AtomicOperations.h"
+#include "js/UniquePtr.h"
+#include "js/Utility.h"
+#include "threading/LockGuard.h"
+#include "vm/JSContext.h"
+#include "vm/Runtime.h"
+#include "vm/SharedMem.h"
+#include "wasm/WasmInstance.h"
+#include "wasm/WasmSignalHandlers.h"
+
+extern "C" {
+
+MOZ_EXPORT int64_t __aeabi_idivmod(int x, int y) {
+ // Run-time ABI for the ARM architecture specifies that for |INT_MIN / -1|
+ // "an implementation is (sic) may return any convenient value, possibly the
+ // original numerator."
+ //
+ // |INT_MIN / -1| traps on x86, which isn't listed as an allowed behavior in
+ // the ARM docs, so instead follow LLVM and return the numerator. (And zero
+ // for the remainder.)
+
+ if (x == INT32_MIN && y == -1) {
+ return uint32_t(x);
+ }
+
+ uint32_t lo = uint32_t(x / y);
+ uint32_t hi = uint32_t(x % y);
+ return (int64_t(hi) << 32) | lo;
+}
+
+MOZ_EXPORT int64_t __aeabi_uidivmod(int x, int y) {
+ uint32_t lo = uint32_t(x) / uint32_t(y);
+ uint32_t hi = uint32_t(x) % uint32_t(y);
+ return (int64_t(hi) << 32) | lo;
+}
+}
+
+namespace js {
+namespace jit {
+
+// For decoding load-exclusive and store-exclusive instructions.
+namespace excl {
+
+// Bit positions.
+enum {
+ ExclusiveOpHi = 24, // Hi bit of opcode field
+ ExclusiveOpLo = 23, // Lo bit of opcode field
+ ExclusiveSizeHi = 22, // Hi bit of operand size field
+ ExclusiveSizeLo = 21, // Lo bit of operand size field
+ ExclusiveLoad = 20 // Bit indicating load
+};
+
+// Opcode bits for exclusive instructions.
+enum { ExclusiveOpcode = 3 };
+
+// Operand size, Bits(ExclusiveSizeHi,ExclusiveSizeLo).
+enum {
+ ExclusiveWord = 0,
+ ExclusiveDouble = 1,
+ ExclusiveByte = 2,
+ ExclusiveHalf = 3
+};
+
+} // namespace excl
+
+// Load/store multiple addressing mode.
+enum BlockAddrMode {
+ // Alias modes for comparison when writeback does not matter.
+ da_x = (0 | 0 | 0) << 21, // Decrement after.
+ ia_x = (0 | 4 | 0) << 21, // Increment after.
+ db_x = (8 | 0 | 0) << 21, // Decrement before.
+ ib_x = (8 | 4 | 0) << 21, // Increment before.
+};
+
+// Type of VFP register. Determines register encoding.
+enum VFPRegPrecision { kSinglePrecision = 0, kDoublePrecision = 1 };
+
+enum NeonListType { nlt_1 = 0x7, nlt_2 = 0xA, nlt_3 = 0x6, nlt_4 = 0x2 };
+
+// Supervisor Call (svc) specific support.
+
+// Special Software Interrupt codes when used in the presence of the ARM
+// simulator.
+// svc (formerly swi) provides a 24bit immediate value. Use bits 22:0 for
+// standard SoftwareInterrupCode. Bit 23 is reserved for the stop feature.
+enum SoftwareInterruptCodes {
+ kCallRtRedirected = 0x10, // Transition to C code.
+ kBreakpoint = 0x20, // Breakpoint.
+ kStopCode = 1 << 23 // Stop.
+};
+
+const uint32_t kStopCodeMask = kStopCode - 1;
+const uint32_t kMaxStopCode = kStopCode - 1;
+
+// -----------------------------------------------------------------------------
+// Instruction abstraction.
+
+// The class Instruction enables access to individual fields defined in the ARM
+// architecture instruction set encoding as described in figure A3-1.
+// Note that the Assembler uses typedef int32_t Instr.
+//
+// Example: Test whether the instruction at ptr does set the condition code
+// bits.
+//
+// bool InstructionSetsConditionCodes(byte* ptr) {
+// Instruction* instr = Instruction::At(ptr);
+// int type = instr->TypeValue();
+// return ((type == 0) || (type == 1)) && instr->hasS();
+// }
+//
+class SimInstruction {
+ public:
+ enum { kInstrSize = 4, kPCReadOffset = 8 };
+
+ // Get the raw instruction bits.
+ inline Instr instructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void setInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int bit(int nr) const { return (instructionBits() >> nr) & 1; }
+
+ // Read a bit field's value out of the instruction bits.
+ inline int bits(int hi, int lo) const {
+ return (instructionBits() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read a bit field out of the instruction bits.
+ inline int bitField(int hi, int lo) const {
+ return instructionBits() & (((2 << (hi - lo)) - 1) << lo);
+ }
+
+ // Accessors for the different named fields used in the ARM encoding.
+ // The naming of these accessor corresponds to figure A3-1.
+ //
+ // Two kind of accessors are declared:
+ // - <Name>Field() will return the raw field, i.e. the field's bits at their
+ // original place in the instruction encoding.
+ // e.g. if instr is the 'addgt r0, r1, r2' instruction, encoded as
+ // 0xC0810002 conditionField(instr) will return 0xC0000000.
+ // - <Name>Value() will return the field value, shifted back to bit 0.
+ // e.g. if instr is the 'addgt r0, r1, r2' instruction, encoded as
+ // 0xC0810002 conditionField(instr) will return 0xC.
+
+ // Generally applicable fields
+ inline Assembler::ARMCondition conditionField() const {
+ return static_cast<Assembler::ARMCondition>(bitField(31, 28));
+ }
+ inline int typeValue() const { return bits(27, 25); }
+ inline int specialValue() const { return bits(27, 23); }
+
+ inline int rnValue() const { return bits(19, 16); }
+ inline int rdValue() const { return bits(15, 12); }
+
+ inline int coprocessorValue() const { return bits(11, 8); }
+
+ // Support for VFP.
+ // Vn(19-16) | Vd(15-12) | Vm(3-0)
+ inline int vnValue() const { return bits(19, 16); }
+ inline int vmValue() const { return bits(3, 0); }
+ inline int vdValue() const { return bits(15, 12); }
+ inline int nValue() const { return bit(7); }
+ inline int mValue() const { return bit(5); }
+ inline int dValue() const { return bit(22); }
+ inline int rtValue() const { return bits(15, 12); }
+ inline int pValue() const { return bit(24); }
+ inline int uValue() const { return bit(23); }
+ inline int opc1Value() const { return (bit(23) << 2) | bits(21, 20); }
+ inline int opc2Value() const { return bits(19, 16); }
+ inline int opc3Value() const { return bits(7, 6); }
+ inline int szValue() const { return bit(8); }
+ inline int VLValue() const { return bit(20); }
+ inline int VCValue() const { return bit(8); }
+ inline int VAValue() const { return bits(23, 21); }
+ inline int VBValue() const { return bits(6, 5); }
+ inline int VFPNRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 16, 7);
+ }
+ inline int VFPMRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 0, 5);
+ }
+ inline int VFPDRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 12, 22);
+ }
+
+ // Fields used in Data processing instructions.
+ inline int opcodeValue() const { return static_cast<ALUOp>(bits(24, 21)); }
+ inline ALUOp opcodeField() const {
+ return static_cast<ALUOp>(bitField(24, 21));
+ }
+ inline int sValue() const { return bit(20); }
+
+ // With register.
+ inline int rmValue() const { return bits(3, 0); }
+ inline ShiftType shifttypeValue() const {
+ return static_cast<ShiftType>(bits(6, 5));
+ }
+ inline int rsValue() const { return bits(11, 8); }
+ inline int shiftAmountValue() const { return bits(11, 7); }
+
+ // With immediate.
+ inline int rotateValue() const { return bits(11, 8); }
+ inline int immed8Value() const { return bits(7, 0); }
+ inline int immed4Value() const { return bits(19, 16); }
+ inline int immedMovwMovtValue() const {
+ return immed4Value() << 12 | offset12Value();
+ }
+
+ // Fields used in Load/Store instructions.
+ inline int PUValue() const { return bits(24, 23); }
+ inline int PUField() const { return bitField(24, 23); }
+ inline int bValue() const { return bit(22); }
+ inline int wValue() const { return bit(21); }
+ inline int lValue() const { return bit(20); }
+
+ // With register uses same fields as Data processing instructions above with
+ // immediate.
+ inline int offset12Value() const { return bits(11, 0); }
+
+ // Multiple.
+ inline int rlistValue() const { return bits(15, 0); }
+
+ // Extra loads and stores.
+ inline int signValue() const { return bit(6); }
+ inline int hValue() const { return bit(5); }
+ inline int immedHValue() const { return bits(11, 8); }
+ inline int immedLValue() const { return bits(3, 0); }
+
+ // Fields used in Branch instructions.
+ inline int linkValue() const { return bit(24); }
+ inline int sImmed24Value() const { return ((instructionBits() << 8) >> 8); }
+
+ // Fields used in Software interrupt instructions.
+ inline SoftwareInterruptCodes svcValue() const {
+ return static_cast<SoftwareInterruptCodes>(bits(23, 0));
+ }
+
+ // Test for special encodings of type 0 instructions (extra loads and
+ // stores, as well as multiplications).
+ inline bool isSpecialType0() const { return (bit(7) == 1) && (bit(4) == 1); }
+
+ // Test for miscellaneous instructions encodings of type 0 instructions.
+ inline bool isMiscType0() const {
+ return bit(24) == 1 && bit(23) == 0 && bit(20) == 0 && (bit(7) == 0);
+ }
+
+ // Test for a nop instruction, which falls under type 1.
+ inline bool isNopType1() const { return bits(24, 0) == 0x0120F000; }
+
+ // Test for a nop instruction, which falls under type 1.
+ inline bool isCsdbType1() const { return bits(24, 0) == 0x0120F014; }
+
+ // Test for a stop instruction.
+ inline bool isStop() const {
+ return typeValue() == 7 && bit(24) == 1 && svcValue() >= kStopCode;
+ }
+
+ // Test for a udf instruction, which falls under type 3.
+ inline bool isUDF() const {
+ return (instructionBits() & 0xfff000f0) == 0xe7f000f0;
+ }
+
+ // Special accessors that test for existence of a value.
+ inline bool hasS() const { return sValue() == 1; }
+ inline bool hasB() const { return bValue() == 1; }
+ inline bool hasW() const { return wValue() == 1; }
+ inline bool hasL() const { return lValue() == 1; }
+ inline bool hasU() const { return uValue() == 1; }
+ inline bool hasSign() const { return signValue() == 1; }
+ inline bool hasH() const { return hValue() == 1; }
+ inline bool hasLink() const { return linkValue() == 1; }
+
+ // Decoding the double immediate in the vmov instruction.
+ double doubleImmedVmov() const;
+ // Decoding the float32 immediate in the vmov.f32 instruction.
+ float float32ImmedVmov() const;
+
+ private:
+ // Join split register codes, depending on single or double precision.
+ // four_bit is the position of the least-significant bit of the four
+ // bit specifier. one_bit is the position of the additional single bit
+ // specifier.
+ inline int VFPGlueRegValue(VFPRegPrecision pre, int four_bit, int one_bit) {
+ if (pre == kSinglePrecision) {
+ return (bits(four_bit + 3, four_bit) << 1) | bit(one_bit);
+ }
+ return (bit(one_bit) << 4) | bits(four_bit + 3, four_bit);
+ }
+
+ SimInstruction() = delete;
+ SimInstruction(const SimInstruction& other) = delete;
+ void operator=(const SimInstruction& other) = delete;
+};
+
+double SimInstruction::doubleImmedVmov() const {
+ // Reconstruct a double from the immediate encoded in the vmov instruction.
+ //
+ // instruction: [xxxxxxxx,xxxxabcd,xxxxxxxx,xxxxefgh]
+ // double: [aBbbbbbb,bbcdefgh,00000000,00000000,
+ // 00000000,00000000,00000000,00000000]
+ //
+ // where B = ~b. Only the high 16 bits are affected.
+ uint64_t high16;
+ high16 = (bits(17, 16) << 4) | bits(3, 0); // xxxxxxxx,xxcdefgh.
+ high16 |= (0xff * bit(18)) << 6; // xxbbbbbb,bbxxxxxx.
+ high16 |= (bit(18) ^ 1) << 14; // xBxxxxxx,xxxxxxxx.
+ high16 |= bit(19) << 15; // axxxxxxx,xxxxxxxx.
+
+ uint64_t imm = high16 << 48;
+ return mozilla::BitwiseCast<double>(imm);
+}
+
+float SimInstruction::float32ImmedVmov() const {
+ // Reconstruct a float32 from the immediate encoded in the vmov instruction.
+ //
+ // instruction: [xxxxxxxx,xxxxabcd,xxxxxxxx,xxxxefgh]
+ // float32: [aBbbbbbc, defgh000, 00000000, 00000000]
+ //
+ // where B = ~b. Only the high 16 bits are affected.
+ uint32_t imm;
+ imm = (bits(17, 16) << 23) | (bits(3, 0) << 19); // xxxxxxxc,defgh000.0.0
+ imm |= (0x1f * bit(18)) << 25; // xxbbbbbx,xxxxxxxx.0.0
+ imm |= (bit(18) ^ 1) << 30; // xBxxxxxx,xxxxxxxx.0.0
+ imm |= bit(19) << 31; // axxxxxxx,xxxxxxxx.0.0
+
+ return mozilla::BitwiseCast<float>(imm);
+}
+
+class CachePage {
+ public:
+ static const int LINE_VALID = 0;
+ static const int LINE_INVALID = 1;
+ static const int kPageShift = 12;
+ static const int kPageSize = 1 << kPageShift;
+ static const int kPageMask = kPageSize - 1;
+ static const int kLineShift = 2; // The cache line is only 4 bytes right now.
+ static const int kLineLength = 1 << kLineShift;
+ static const int kLineMask = kLineLength - 1;
+
+ CachePage() { memset(&validity_map_, LINE_INVALID, sizeof(validity_map_)); }
+ char* validityByte(int offset) {
+ return &validity_map_[offset >> kLineShift];
+ }
+ char* cachedData(int offset) { return &data_[offset]; }
+
+ private:
+ char data_[kPageSize]; // The cached data.
+ static const int kValidityMapSize = kPageSize >> kLineShift;
+ char validity_map_[kValidityMapSize]; // One byte per line.
+};
+
+// Protects the icache() and redirection() properties of the
+// Simulator.
+class AutoLockSimulatorCache : public LockGuard<Mutex> {
+ using Base = LockGuard<Mutex>;
+
+ public:
+ explicit AutoLockSimulatorCache()
+ : Base(SimulatorProcess::singleton_->cacheLock_) {}
+};
+
+mozilla::Atomic<size_t, mozilla::ReleaseAcquire>
+ SimulatorProcess::ICacheCheckingDisableCount(
+ 1); // Checking is disabled by default.
+SimulatorProcess* SimulatorProcess::singleton_ = nullptr;
+
+int64_t Simulator::StopSimAt = -1L;
+
+Simulator* Simulator::Create() {
+ auto sim = MakeUnique<Simulator>();
+ if (!sim) {
+ return nullptr;
+ }
+
+ if (!sim->init()) {
+ return nullptr;
+ }
+
+ char* stopAtStr = getenv("ARM_SIM_STOP_AT");
+ int64_t stopAt;
+ if (stopAtStr && sscanf(stopAtStr, "%lld", &stopAt) == 1) {
+ fprintf(stderr, "\nStopping simulation at icount %lld\n", stopAt);
+ Simulator::StopSimAt = stopAt;
+ }
+
+ return sim.release();
+}
+
+void Simulator::Destroy(Simulator* sim) { js_delete(sim); }
+
+void Simulator::disassemble(SimInstruction* instr, size_t n) {
+#ifdef JS_DISASM_ARM
+ disasm::NameConverter converter;
+ disasm::Disassembler dasm(converter);
+ disasm::EmbeddedVector<char, disasm::ReasonableBufferSize> buffer;
+ while (n-- > 0) {
+ dasm.InstructionDecode(buffer, reinterpret_cast<uint8_t*>(instr));
+ fprintf(stderr, " 0x%08x %s\n", uint32_t(instr), buffer.start());
+ instr = reinterpret_cast<SimInstruction*>(
+ reinterpret_cast<uint8_t*>(instr) + 4);
+ }
+#endif
+}
+
+void Simulator::disasm(SimInstruction* instr) { disassemble(instr, 1); }
+
+void Simulator::disasm(SimInstruction* instr, size_t n) {
+ disassemble(instr, n);
+}
+
+void Simulator::disasm(SimInstruction* instr, size_t m, size_t n) {
+ disassemble(reinterpret_cast<SimInstruction*>(
+ reinterpret_cast<uint8_t*>(instr) - m * 4),
+ n);
+}
+
+// The ArmDebugger class is used by the simulator while debugging simulated ARM
+// code.
+class ArmDebugger {
+ public:
+ explicit ArmDebugger(Simulator* sim) : sim_(sim) {}
+
+ void stop(SimInstruction* instr);
+ void debug();
+
+ private:
+ static const Instr kBreakpointInstr =
+ (Assembler::AL | (7 * (1 << 25)) | (1 * (1 << 24)) | kBreakpoint);
+ static const Instr kNopInstr = (Assembler::AL | (13 * (1 << 21)));
+
+ Simulator* sim_;
+
+ int32_t getRegisterValue(int regnum);
+ double getRegisterPairDoubleValue(int regnum);
+ void getVFPDoubleRegisterValue(int regnum, double* value);
+ bool getValue(const char* desc, int32_t* value);
+ bool getVFPDoubleValue(const char* desc, double* value);
+
+ // Set or delete a breakpoint. Returns true if successful.
+ bool setBreakpoint(SimInstruction* breakpc);
+ bool deleteBreakpoint(SimInstruction* breakpc);
+
+ // Undo and redo all breakpoints. This is needed to bracket disassembly and
+ // execution to skip past breakpoints when run from the debugger.
+ void undoBreakpoints();
+ void redoBreakpoints();
+};
+
+void ArmDebugger::stop(SimInstruction* instr) {
+ // Get the stop code.
+ uint32_t code = instr->svcValue() & kStopCodeMask;
+ // Retrieve the encoded address, which comes just after this stop.
+ char* msg =
+ *reinterpret_cast<char**>(sim_->get_pc() + SimInstruction::kInstrSize);
+ // Update this stop description.
+ if (sim_->isWatchedStop(code) && !sim_->watched_stops_[code].desc) {
+ sim_->watched_stops_[code].desc = msg;
+ }
+ // Print the stop message and code if it is not the default code.
+ if (code != kMaxStopCode) {
+ printf("Simulator hit stop %u: %s\n", code, msg);
+ } else {
+ printf("Simulator hit %s\n", msg);
+ }
+ sim_->set_pc(sim_->get_pc() + 2 * SimInstruction::kInstrSize);
+ debug();
+}
+
+int32_t ArmDebugger::getRegisterValue(int regnum) {
+ if (regnum == Registers::pc) {
+ return sim_->get_pc();
+ }
+ return sim_->get_register(regnum);
+}
+
+double ArmDebugger::getRegisterPairDoubleValue(int regnum) {
+ return sim_->get_double_from_register_pair(regnum);
+}
+
+void ArmDebugger::getVFPDoubleRegisterValue(int regnum, double* out) {
+ sim_->get_double_from_d_register(regnum, out);
+}
+
+bool ArmDebugger::getValue(const char* desc, int32_t* value) {
+ Register reg = Register::FromName(desc);
+ if (reg != InvalidReg) {
+ *value = getRegisterValue(reg.code());
+ return true;
+ }
+ if (strncmp(desc, "0x", 2) == 0) {
+ return sscanf(desc + 2, "%x", reinterpret_cast<uint32_t*>(value)) == 1;
+ }
+ return sscanf(desc, "%u", reinterpret_cast<uint32_t*>(value)) == 1;
+}
+
+bool ArmDebugger::getVFPDoubleValue(const char* desc, double* value) {
+ FloatRegister reg = FloatRegister::FromCode(FloatRegister::FromName(desc));
+ if (reg.isInvalid()) {
+ return false;
+ }
+
+ if (reg.isSingle()) {
+ float fval;
+ sim_->get_float_from_s_register(reg.id(), &fval);
+ *value = fval;
+ return true;
+ }
+
+ sim_->get_double_from_d_register(reg.id(), value);
+ return true;
+}
+
+bool ArmDebugger::setBreakpoint(SimInstruction* breakpc) {
+ // Check if a breakpoint can be set. If not return without any side-effects.
+ if (sim_->break_pc_) {
+ return false;
+ }
+
+ // Set the breakpoint.
+ sim_->break_pc_ = breakpc;
+ sim_->break_instr_ = breakpc->instructionBits();
+ // Not setting the breakpoint instruction in the code itself. It will be set
+ // when the debugger shell continues.
+ return true;
+}
+
+bool ArmDebugger::deleteBreakpoint(SimInstruction* breakpc) {
+ if (sim_->break_pc_ != nullptr) {
+ sim_->break_pc_->setInstructionBits(sim_->break_instr_);
+ }
+
+ sim_->break_pc_ = nullptr;
+ sim_->break_instr_ = 0;
+ return true;
+}
+
+void ArmDebugger::undoBreakpoints() {
+ if (sim_->break_pc_) {
+ sim_->break_pc_->setInstructionBits(sim_->break_instr_);
+ }
+}
+
+void ArmDebugger::redoBreakpoints() {
+ if (sim_->break_pc_) {
+ sim_->break_pc_->setInstructionBits(kBreakpointInstr);
+ }
+}
+
+static char* ReadLine(const char* prompt) {
+ UniqueChars result;
+ char line_buf[256];
+ int offset = 0;
+ bool keep_going = true;
+ fprintf(stdout, "%s", prompt);
+ fflush(stdout);
+ while (keep_going) {
+ if (fgets(line_buf, sizeof(line_buf), stdin) == nullptr) {
+ // fgets got an error. Just give up.
+ return nullptr;
+ }
+ int len = strlen(line_buf);
+ if (len > 0 && line_buf[len - 1] == '\n') {
+ // Since we read a new line we are done reading the line. This will
+ // exit the loop after copying this buffer into the result.
+ keep_going = false;
+ }
+ if (!result) {
+ // Allocate the initial result and make room for the terminating
+ // '\0'.
+ result.reset(js_pod_malloc<char>(len + 1));
+ if (!result) {
+ return nullptr;
+ }
+ } else {
+ // Allocate a new result with enough room for the new addition.
+ int new_len = offset + len + 1;
+ char* new_result = js_pod_malloc<char>(new_len);
+ if (!new_result) {
+ return nullptr;
+ }
+ // Copy the existing input into the new array and set the new
+ // array as the result.
+ memcpy(new_result, result.get(), offset * sizeof(char));
+ result.reset(new_result);
+ }
+ // Copy the newly read line into the result.
+ memcpy(result.get() + offset, line_buf, len * sizeof(char));
+ offset += len;
+ }
+
+ MOZ_ASSERT(result);
+ result[offset] = '\0';
+ return result.release();
+}
+
+void ArmDebugger::debug() {
+ intptr_t last_pc = -1;
+ bool done = false;
+
+#define COMMAND_SIZE 63
+#define ARG_SIZE 255
+
+#define STR(a) #a
+#define XSTR(a) STR(a)
+
+ char cmd[COMMAND_SIZE + 1];
+ char arg1[ARG_SIZE + 1];
+ char arg2[ARG_SIZE + 1];
+ char* argv[3] = {cmd, arg1, arg2};
+
+ // Make sure to have a proper terminating character if reaching the limit.
+ cmd[COMMAND_SIZE] = 0;
+ arg1[ARG_SIZE] = 0;
+ arg2[ARG_SIZE] = 0;
+
+ // Undo all set breakpoints while running in the debugger shell. This will
+ // make them invisible to all commands.
+ undoBreakpoints();
+
+#ifndef JS_DISASM_ARM
+ static bool disasm_warning_printed = false;
+ if (!disasm_warning_printed) {
+ printf(
+ " No ARM disassembler present. Enable JS_DISASM_ARM in "
+ "configure.in.");
+ disasm_warning_printed = true;
+ }
+#endif
+
+ while (!done && !sim_->has_bad_pc()) {
+ if (last_pc != sim_->get_pc()) {
+#ifdef JS_DISASM_ARM
+ disasm::NameConverter converter;
+ disasm::Disassembler dasm(converter);
+ disasm::EmbeddedVector<char, disasm::ReasonableBufferSize> buffer;
+ dasm.InstructionDecode(buffer,
+ reinterpret_cast<uint8_t*>(sim_->get_pc()));
+ printf(" 0x%08x %s\n", sim_->get_pc(), buffer.start());
+#endif
+ last_pc = sim_->get_pc();
+ }
+ char* line = ReadLine("sim> ");
+ if (line == nullptr) {
+ break;
+ } else {
+ char* last_input = sim_->lastDebuggerInput();
+ if (strcmp(line, "\n") == 0 && last_input != nullptr) {
+ line = last_input;
+ } else {
+ // Ownership is transferred to sim_;
+ sim_->setLastDebuggerInput(line);
+ }
+
+ // Use sscanf to parse the individual parts of the command line. At the
+ // moment no command expects more than two parameters.
+ int argc = sscanf(line,
+ "%" XSTR(COMMAND_SIZE) "s "
+ "%" XSTR(ARG_SIZE) "s "
+ "%" XSTR(ARG_SIZE) "s",
+ cmd, arg1, arg2);
+ if (argc < 0) {
+ continue;
+ } else if ((strcmp(cmd, "si") == 0) || (strcmp(cmd, "stepi") == 0)) {
+ sim_->instructionDecode(
+ reinterpret_cast<SimInstruction*>(sim_->get_pc()));
+ sim_->icount_++;
+ } else if ((strcmp(cmd, "skip") == 0)) {
+ sim_->set_pc(sim_->get_pc() + 4);
+ sim_->icount_++;
+ } else if ((strcmp(cmd, "c") == 0) || (strcmp(cmd, "cont") == 0)) {
+ // Execute the one instruction we broke at with breakpoints
+ // disabled.
+ sim_->instructionDecode(
+ reinterpret_cast<SimInstruction*>(sim_->get_pc()));
+ sim_->icount_++;
+ // Leave the debugger shell.
+ done = true;
+ } else if ((strcmp(cmd, "p") == 0) || (strcmp(cmd, "print") == 0)) {
+ if (argc == 2 || (argc == 3 && strcmp(arg2, "fp") == 0)) {
+ int32_t value;
+ double dvalue;
+ if (strcmp(arg1, "all") == 0) {
+ for (uint32_t i = 0; i < Registers::Total; i++) {
+ value = getRegisterValue(i);
+ printf("%3s: 0x%08x %10d", Registers::GetName(i), value, value);
+ if ((argc == 3 && strcmp(arg2, "fp") == 0) && i < 8 &&
+ (i % 2) == 0) {
+ dvalue = getRegisterPairDoubleValue(i);
+ printf(" (%.16g)\n", dvalue);
+ } else {
+ printf("\n");
+ }
+ }
+ for (uint32_t i = 0; i < FloatRegisters::TotalPhys; i++) {
+ getVFPDoubleRegisterValue(i, &dvalue);
+ uint64_t as_words = mozilla::BitwiseCast<uint64_t>(dvalue);
+ printf("%3s: %.16g 0x%08x %08x\n",
+ FloatRegister::FromCode(i).name(), dvalue,
+ static_cast<uint32_t>(as_words >> 32),
+ static_cast<uint32_t>(as_words & 0xffffffff));
+ }
+ } else {
+ if (getValue(arg1, &value)) {
+ printf("%s: 0x%08x %d \n", arg1, value, value);
+ } else if (getVFPDoubleValue(arg1, &dvalue)) {
+ uint64_t as_words = mozilla::BitwiseCast<uint64_t>(dvalue);
+ printf("%s: %.16g 0x%08x %08x\n", arg1, dvalue,
+ static_cast<uint32_t>(as_words >> 32),
+ static_cast<uint32_t>(as_words & 0xffffffff));
+ } else {
+ printf("%s unrecognized\n", arg1);
+ }
+ }
+ } else {
+ printf("print <register>\n");
+ }
+ } else if (strcmp(cmd, "stack") == 0 || strcmp(cmd, "mem") == 0) {
+ int32_t* cur = nullptr;
+ int32_t* end = nullptr;
+ int next_arg = 1;
+
+ if (strcmp(cmd, "stack") == 0) {
+ cur = reinterpret_cast<int32_t*>(sim_->get_register(Simulator::sp));
+ } else { // "mem"
+ int32_t value;
+ if (!getValue(arg1, &value)) {
+ printf("%s unrecognized\n", arg1);
+ continue;
+ }
+ cur = reinterpret_cast<int32_t*>(value);
+ next_arg++;
+ }
+
+ int32_t words;
+ if (argc == next_arg) {
+ words = 10;
+ } else {
+ if (!getValue(argv[next_arg], &words)) {
+ words = 10;
+ }
+ }
+ end = cur + words;
+
+ while (cur < end) {
+ printf(" %p: 0x%08x %10d", cur, *cur, *cur);
+ printf("\n");
+ cur++;
+ }
+ } else if (strcmp(cmd, "disasm") == 0 || strcmp(cmd, "di") == 0) {
+#ifdef JS_DISASM_ARM
+ uint8_t* prev = nullptr;
+ uint8_t* cur = nullptr;
+ uint8_t* end = nullptr;
+
+ if (argc == 1) {
+ cur = reinterpret_cast<uint8_t*>(sim_->get_pc());
+ end = cur + (10 * SimInstruction::kInstrSize);
+ } else if (argc == 2) {
+ Register reg = Register::FromName(arg1);
+ if (reg != InvalidReg || strncmp(arg1, "0x", 2) == 0) {
+ // The argument is an address or a register name.
+ int32_t value;
+ if (getValue(arg1, &value)) {
+ cur = reinterpret_cast<uint8_t*>(value);
+ // Disassemble 10 instructions at <arg1>.
+ end = cur + (10 * SimInstruction::kInstrSize);
+ }
+ } else {
+ // The argument is the number of instructions.
+ int32_t value;
+ if (getValue(arg1, &value)) {
+ cur = reinterpret_cast<uint8_t*>(sim_->get_pc());
+ // Disassemble <arg1> instructions.
+ end = cur + (value * SimInstruction::kInstrSize);
+ }
+ }
+ } else {
+ int32_t value1;
+ int32_t value2;
+ if (getValue(arg1, &value1) && getValue(arg2, &value2)) {
+ cur = reinterpret_cast<uint8_t*>(value1);
+ end = cur + (value2 * SimInstruction::kInstrSize);
+ }
+ }
+ while (cur < end) {
+ disasm::NameConverter converter;
+ disasm::Disassembler dasm(converter);
+ disasm::EmbeddedVector<char, disasm::ReasonableBufferSize> buffer;
+
+ prev = cur;
+ cur += dasm.InstructionDecode(buffer, cur);
+ printf(" 0x%08x %s\n", reinterpret_cast<uint32_t>(prev),
+ buffer.start());
+ }
+#endif
+ } else if (strcmp(cmd, "gdb") == 0) {
+ printf("relinquishing control to gdb\n");
+#ifdef _MSC_VER
+ __debugbreak();
+#else
+ asm("int $3");
+#endif
+ printf("regaining control from gdb\n");
+ } else if (strcmp(cmd, "break") == 0) {
+ if (argc == 2) {
+ int32_t value;
+ if (getValue(arg1, &value)) {
+ if (!setBreakpoint(reinterpret_cast<SimInstruction*>(value))) {
+ printf("setting breakpoint failed\n");
+ }
+ } else {
+ printf("%s unrecognized\n", arg1);
+ }
+ } else {
+ printf("break <address>\n");
+ }
+ } else if (strcmp(cmd, "del") == 0) {
+ if (!deleteBreakpoint(nullptr)) {
+ printf("deleting breakpoint failed\n");
+ }
+ } else if (strcmp(cmd, "flags") == 0) {
+ printf("N flag: %d; ", sim_->n_flag_);
+ printf("Z flag: %d; ", sim_->z_flag_);
+ printf("C flag: %d; ", sim_->c_flag_);
+ printf("V flag: %d\n", sim_->v_flag_);
+ printf("INVALID OP flag: %d; ", sim_->inv_op_vfp_flag_);
+ printf("DIV BY ZERO flag: %d; ", sim_->div_zero_vfp_flag_);
+ printf("OVERFLOW flag: %d; ", sim_->overflow_vfp_flag_);
+ printf("UNDERFLOW flag: %d; ", sim_->underflow_vfp_flag_);
+ printf("INEXACT flag: %d;\n", sim_->inexact_vfp_flag_);
+ } else if (strcmp(cmd, "stop") == 0) {
+ int32_t value;
+ intptr_t stop_pc = sim_->get_pc() - 2 * SimInstruction::kInstrSize;
+ SimInstruction* stop_instr = reinterpret_cast<SimInstruction*>(stop_pc);
+ SimInstruction* msg_address = reinterpret_cast<SimInstruction*>(
+ stop_pc + SimInstruction::kInstrSize);
+ if ((argc == 2) && (strcmp(arg1, "unstop") == 0)) {
+ // Remove the current stop.
+ if (sim_->isStopInstruction(stop_instr)) {
+ stop_instr->setInstructionBits(kNopInstr);
+ msg_address->setInstructionBits(kNopInstr);
+ } else {
+ printf("Not at debugger stop.\n");
+ }
+ } else if (argc == 3) {
+ // Print information about all/the specified breakpoint(s).
+ if (strcmp(arg1, "info") == 0) {
+ if (strcmp(arg2, "all") == 0) {
+ printf("Stop information:\n");
+ for (uint32_t i = 0; i < sim_->kNumOfWatchedStops; i++) {
+ sim_->printStopInfo(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->printStopInfo(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ } else if (strcmp(arg1, "enable") == 0) {
+ // Enable all/the specified breakpoint(s).
+ if (strcmp(arg2, "all") == 0) {
+ for (uint32_t i = 0; i < sim_->kNumOfWatchedStops; i++) {
+ sim_->enableStop(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->enableStop(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ } else if (strcmp(arg1, "disable") == 0) {
+ // Disable all/the specified breakpoint(s).
+ if (strcmp(arg2, "all") == 0) {
+ for (uint32_t i = 0; i < sim_->kNumOfWatchedStops; i++) {
+ sim_->disableStop(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->disableStop(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ }
+ } else {
+ printf("Wrong usage. Use help command for more information.\n");
+ }
+ } else if ((strcmp(cmd, "h") == 0) || (strcmp(cmd, "help") == 0)) {
+ printf("cont\n");
+ printf(" continue execution (alias 'c')\n");
+ printf("skip\n");
+ printf(" skip one instruction (set pc to next instruction)\n");
+ printf("stepi\n");
+ printf(" step one instruction (alias 'si')\n");
+ printf("print <register>\n");
+ printf(" print register content (alias 'p')\n");
+ printf(" use register name 'all' to print all registers\n");
+ printf(" add argument 'fp' to print register pair double values\n");
+ printf("flags\n");
+ printf(" print flags\n");
+ printf("stack [<words>]\n");
+ printf(" dump stack content, default dump 10 words)\n");
+ printf("mem <address> [<words>]\n");
+ printf(" dump memory content, default dump 10 words)\n");
+ printf("disasm [<instructions>]\n");
+ printf("disasm [<address/register>]\n");
+ printf("disasm [[<address/register>] <instructions>]\n");
+ printf(" disassemble code, default is 10 instructions\n");
+ printf(" from pc (alias 'di')\n");
+ printf("gdb\n");
+ printf(" enter gdb\n");
+ printf("break <address>\n");
+ printf(" set a break point on the address\n");
+ printf("del\n");
+ printf(" delete the breakpoint\n");
+ printf("stop feature:\n");
+ printf(" Description:\n");
+ printf(" Stops are debug instructions inserted by\n");
+ printf(" the Assembler::stop() function.\n");
+ printf(" When hitting a stop, the Simulator will\n");
+ printf(" stop and and give control to the ArmDebugger.\n");
+ printf(" The first %d stop codes are watched:\n",
+ Simulator::kNumOfWatchedStops);
+ printf(" - They can be enabled / disabled: the Simulator\n");
+ printf(" will / won't stop when hitting them.\n");
+ printf(" - The Simulator keeps track of how many times they \n");
+ printf(" are met. (See the info command.) Going over a\n");
+ printf(" disabled stop still increases its counter. \n");
+ printf(" Commands:\n");
+ printf(" stop info all/<code> : print infos about number <code>\n");
+ printf(" or all stop(s).\n");
+ printf(" stop enable/disable all/<code> : enables / disables\n");
+ printf(" all or number <code> stop(s)\n");
+ printf(" stop unstop\n");
+ printf(" ignore the stop instruction at the current location\n");
+ printf(" from now on\n");
+ } else {
+ printf("Unknown command: %s\n", cmd);
+ }
+ }
+ }
+
+ // Add all the breakpoints back to stop execution and enter the debugger
+ // shell when hit.
+ redoBreakpoints();
+
+#undef COMMAND_SIZE
+#undef ARG_SIZE
+
+#undef STR
+#undef XSTR
+}
+
+static bool AllOnOnePage(uintptr_t start, int size) {
+ intptr_t start_page = (start & ~CachePage::kPageMask);
+ intptr_t end_page = ((start + size) & ~CachePage::kPageMask);
+ return start_page == end_page;
+}
+
+static CachePage* GetCachePageLocked(SimulatorProcess::ICacheMap& i_cache,
+ void* page) {
+ SimulatorProcess::ICacheMap::AddPtr p = i_cache.lookupForAdd(page);
+ if (p) {
+ return p->value();
+ }
+
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ CachePage* new_page = js_new<CachePage>();
+ if (!new_page || !i_cache.add(p, page, new_page)) {
+ oomUnsafe.crash("Simulator CachePage");
+ }
+
+ return new_page;
+}
+
+// Flush from start up to and not including start + size.
+static void FlushOnePageLocked(SimulatorProcess::ICacheMap& i_cache,
+ intptr_t start, int size) {
+ MOZ_ASSERT(size <= CachePage::kPageSize);
+ MOZ_ASSERT(AllOnOnePage(start, size - 1));
+ MOZ_ASSERT((start & CachePage::kLineMask) == 0);
+ MOZ_ASSERT((size & CachePage::kLineMask) == 0);
+
+ void* page = reinterpret_cast<void*>(start & (~CachePage::kPageMask));
+ int offset = (start & CachePage::kPageMask);
+ CachePage* cache_page = GetCachePageLocked(i_cache, page);
+ char* valid_bytemap = cache_page->validityByte(offset);
+ memset(valid_bytemap, CachePage::LINE_INVALID, size >> CachePage::kLineShift);
+}
+
+static void FlushICacheLocked(SimulatorProcess::ICacheMap& i_cache,
+ void* start_addr, size_t size) {
+ intptr_t start = reinterpret_cast<intptr_t>(start_addr);
+ int intra_line = (start & CachePage::kLineMask);
+ start -= intra_line;
+ size += intra_line;
+ size = ((size - 1) | CachePage::kLineMask) + 1;
+ int offset = (start & CachePage::kPageMask);
+ while (!AllOnOnePage(start, size - 1)) {
+ int bytes_to_flush = CachePage::kPageSize - offset;
+ FlushOnePageLocked(i_cache, start, bytes_to_flush);
+ start += bytes_to_flush;
+ size -= bytes_to_flush;
+ MOZ_ASSERT((start & CachePage::kPageMask) == 0);
+ offset = 0;
+ }
+ if (size != 0) {
+ FlushOnePageLocked(i_cache, start, size);
+ }
+}
+
+/* static */
+void SimulatorProcess::checkICacheLocked(SimInstruction* instr) {
+ intptr_t address = reinterpret_cast<intptr_t>(instr);
+ void* page = reinterpret_cast<void*>(address & (~CachePage::kPageMask));
+ void* line = reinterpret_cast<void*>(address & (~CachePage::kLineMask));
+ int offset = (address & CachePage::kPageMask);
+ CachePage* cache_page = GetCachePageLocked(icache(), page);
+ char* cache_valid_byte = cache_page->validityByte(offset);
+ bool cache_hit = (*cache_valid_byte == CachePage::LINE_VALID);
+ char* cached_line = cache_page->cachedData(offset & ~CachePage::kLineMask);
+
+ if (cache_hit) {
+ // Check that the data in memory matches the contents of the I-cache.
+ mozilla::DebugOnly<int> cmpret =
+ memcmp(reinterpret_cast<void*>(instr), cache_page->cachedData(offset),
+ SimInstruction::kInstrSize);
+ MOZ_ASSERT(cmpret == 0);
+ } else {
+ // Cache miss. Load memory into the cache.
+ memcpy(cached_line, line, CachePage::kLineLength);
+ *cache_valid_byte = CachePage::LINE_VALID;
+ }
+}
+
+HashNumber SimulatorProcess::ICacheHasher::hash(const Lookup& l) {
+ return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(l)) >> 2;
+}
+
+bool SimulatorProcess::ICacheHasher::match(const Key& k, const Lookup& l) {
+ MOZ_ASSERT((reinterpret_cast<intptr_t>(k) & CachePage::kPageMask) == 0);
+ MOZ_ASSERT((reinterpret_cast<intptr_t>(l) & CachePage::kPageMask) == 0);
+ return k == l;
+}
+
+void Simulator::setLastDebuggerInput(char* input) {
+ js_free(lastDebuggerInput_);
+ lastDebuggerInput_ = input;
+}
+
+/* static */
+void SimulatorProcess::FlushICache(void* start_addr, size_t size) {
+ JitSpewCont(JitSpew_CacheFlush, "[%p %zx]", start_addr, size);
+ if (!ICacheCheckingDisableCount) {
+ AutoLockSimulatorCache als;
+ js::jit::FlushICacheLocked(icache(), start_addr, size);
+ }
+}
+
+Simulator::Simulator() {
+ // Set up simulator support first. Some of this information is needed to
+ // setup the architecture state.
+
+ // Note, allocation and anything that depends on allocated memory is
+ // deferred until init(), in order to handle OOM properly.
+
+ stack_ = nullptr;
+ stackLimit_ = 0;
+ pc_modified_ = false;
+ icount_ = 0L;
+ break_pc_ = nullptr;
+ break_instr_ = 0;
+ single_stepping_ = false;
+ single_step_callback_ = nullptr;
+ single_step_callback_arg_ = nullptr;
+ skipCalleeSavedRegsCheck = false;
+
+ // Set up architecture state.
+ // All registers are initialized to zero to start with.
+ for (int i = 0; i < num_registers; i++) {
+ registers_[i] = 0;
+ }
+
+ n_flag_ = false;
+ z_flag_ = false;
+ c_flag_ = false;
+ v_flag_ = false;
+
+ for (int i = 0; i < num_d_registers * 2; i++) {
+ vfp_registers_[i] = 0;
+ }
+
+ n_flag_FPSCR_ = false;
+ z_flag_FPSCR_ = false;
+ c_flag_FPSCR_ = false;
+ v_flag_FPSCR_ = false;
+ FPSCR_rounding_mode_ = SimRZ;
+ FPSCR_default_NaN_mode_ = true;
+
+ inv_op_vfp_flag_ = false;
+ div_zero_vfp_flag_ = false;
+ overflow_vfp_flag_ = false;
+ underflow_vfp_flag_ = false;
+ inexact_vfp_flag_ = false;
+
+ // The lr and pc are initialized to a known bad value that will cause an
+ // access violation if the simulator ever tries to execute it.
+ registers_[pc] = bad_lr;
+ registers_[lr] = bad_lr;
+
+ lastDebuggerInput_ = nullptr;
+
+ exclusiveMonitorHeld_ = false;
+ exclusiveMonitor_ = 0;
+}
+
+bool Simulator::init() {
+ // Allocate 2MB for the stack. Note that we will only use 1MB, see below.
+ static const size_t stackSize = 2 * 1024 * 1024;
+ stack_ = js_pod_malloc<char>(stackSize);
+ if (!stack_) {
+ return false;
+ }
+
+ // Leave a safety margin of 1MB to prevent overrunning the stack when
+ // pushing values (total stack size is 2MB).
+ stackLimit_ = reinterpret_cast<uintptr_t>(stack_) + 1024 * 1024;
+
+ // The sp is initialized to point to the bottom (high address) of the
+ // allocated stack area. To be safe in potential stack underflows we leave
+ // some buffer below.
+ registers_[sp] = reinterpret_cast<int32_t>(stack_) + stackSize - 64;
+
+ return true;
+}
+
+// When the generated code calls a VM function (masm.callWithABI) we need to
+// call that function instead of trying to execute it with the simulator
+// (because it's x86 code instead of arm code). We do that by redirecting the VM
+// call to a svc (Supervisor Call) instruction that is handled by the
+// simulator. We write the original destination of the jump just at a known
+// offset from the svc instruction so the simulator knows what to call.
+class Redirection {
+ friend class SimulatorProcess;
+
+ // sim's lock must already be held.
+ Redirection(void* nativeFunction, ABIFunctionType type)
+ : nativeFunction_(nativeFunction),
+ swiInstruction_(Assembler::AL | (0xf * (1 << 24)) | kCallRtRedirected),
+ type_(type),
+ next_(nullptr) {
+ next_ = SimulatorProcess::redirection();
+ if (!SimulatorProcess::ICacheCheckingDisableCount) {
+ FlushICacheLocked(SimulatorProcess::icache(), addressOfSwiInstruction(),
+ SimInstruction::kInstrSize);
+ }
+ SimulatorProcess::setRedirection(this);
+ }
+
+ public:
+ void* addressOfSwiInstruction() { return &swiInstruction_; }
+ void* nativeFunction() const { return nativeFunction_; }
+ ABIFunctionType type() const { return type_; }
+
+ static Redirection* Get(void* nativeFunction, ABIFunctionType type) {
+ AutoLockSimulatorCache als;
+
+ Redirection* current = SimulatorProcess::redirection();
+ for (; current != nullptr; current = current->next_) {
+ if (current->nativeFunction_ == nativeFunction) {
+ MOZ_ASSERT(current->type() == type);
+ return current;
+ }
+ }
+
+ // Note: we can't use js_new here because the constructor is private.
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ Redirection* redir = js_pod_malloc<Redirection>(1);
+ if (!redir) {
+ oomUnsafe.crash("Simulator redirection");
+ }
+ new (redir) Redirection(nativeFunction, type);
+ return redir;
+ }
+
+ static Redirection* FromSwiInstruction(SimInstruction* swiInstruction) {
+ uint8_t* addrOfSwi = reinterpret_cast<uint8_t*>(swiInstruction);
+ uint8_t* addrOfRedirection =
+ addrOfSwi - offsetof(Redirection, swiInstruction_);
+ return reinterpret_cast<Redirection*>(addrOfRedirection);
+ }
+
+ private:
+ void* nativeFunction_;
+ uint32_t swiInstruction_;
+ ABIFunctionType type_;
+ Redirection* next_;
+};
+
+Simulator::~Simulator() { js_free(stack_); }
+
+SimulatorProcess::SimulatorProcess()
+ : cacheLock_(mutexid::SimulatorCacheLock), redirection_(nullptr) {
+ if (getenv("ARM_SIM_ICACHE_CHECKS")) {
+ ICacheCheckingDisableCount = 0;
+ }
+}
+
+SimulatorProcess::~SimulatorProcess() {
+ Redirection* r = redirection_;
+ while (r) {
+ Redirection* next = r->next_;
+ js_delete(r);
+ r = next;
+ }
+}
+
+/* static */
+void* Simulator::RedirectNativeFunction(void* nativeFunction,
+ ABIFunctionType type) {
+ Redirection* redirection = Redirection::Get(nativeFunction, type);
+ return redirection->addressOfSwiInstruction();
+}
+
+// Sets the register in the architecture state. It will also deal with updating
+// Simulator internal state for special registers such as PC.
+void Simulator::set_register(int reg, int32_t value) {
+ MOZ_ASSERT(reg >= 0 && reg < num_registers);
+ if (reg == pc) {
+ pc_modified_ = true;
+ }
+ registers_[reg] = value;
+}
+
+// Get the register from the architecture state. This function does handle the
+// special case of accessing the PC register.
+int32_t Simulator::get_register(int reg) const {
+ MOZ_ASSERT(reg >= 0 && reg < num_registers);
+ // Work around GCC bug: http://gcc.gnu.org/bugzilla/show_bug.cgi?id=43949
+ if (reg >= num_registers) return 0;
+ return registers_[reg] + ((reg == pc) ? SimInstruction::kPCReadOffset : 0);
+}
+
+double Simulator::get_double_from_register_pair(int reg) {
+ MOZ_ASSERT(reg >= 0 && reg < num_registers && (reg % 2) == 0);
+
+ // Read the bits from the unsigned integer register_[] array into the double
+ // precision floating point value and return it.
+ double dm_val = 0.0;
+ char buffer[2 * sizeof(vfp_registers_[0])];
+ memcpy(buffer, &registers_[reg], 2 * sizeof(registers_[0]));
+ memcpy(&dm_val, buffer, 2 * sizeof(registers_[0]));
+ return dm_val;
+}
+
+void Simulator::set_register_pair_from_double(int reg, double* value) {
+ MOZ_ASSERT(reg >= 0 && reg < num_registers && (reg % 2) == 0);
+ memcpy(registers_ + reg, value, sizeof(*value));
+}
+
+void Simulator::set_dw_register(int dreg, const int* dbl) {
+ MOZ_ASSERT(dreg >= 0 && dreg < num_d_registers);
+ registers_[dreg] = dbl[0];
+ registers_[dreg + 1] = dbl[1];
+}
+
+void Simulator::get_d_register(int dreg, uint64_t* value) {
+ MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::TotalPhys));
+ memcpy(value, vfp_registers_ + dreg * 2, sizeof(*value));
+}
+
+void Simulator::set_d_register(int dreg, const uint64_t* value) {
+ MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::TotalPhys));
+ memcpy(vfp_registers_ + dreg * 2, value, sizeof(*value));
+}
+
+void Simulator::get_d_register(int dreg, uint32_t* value) {
+ MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::TotalPhys));
+ memcpy(value, vfp_registers_ + dreg * 2, sizeof(*value) * 2);
+}
+
+void Simulator::set_d_register(int dreg, const uint32_t* value) {
+ MOZ_ASSERT(dreg >= 0 && dreg < int(FloatRegisters::TotalPhys));
+ memcpy(vfp_registers_ + dreg * 2, value, sizeof(*value) * 2);
+}
+
+void Simulator::get_q_register(int qreg, uint64_t* value) {
+ MOZ_ASSERT(qreg >= 0 && qreg < num_q_registers);
+ memcpy(value, vfp_registers_ + qreg * 4, sizeof(*value) * 2);
+}
+
+void Simulator::set_q_register(int qreg, const uint64_t* value) {
+ MOZ_ASSERT(qreg >= 0 && qreg < num_q_registers);
+ memcpy(vfp_registers_ + qreg * 4, value, sizeof(*value) * 2);
+}
+
+void Simulator::get_q_register(int qreg, uint32_t* value) {
+ MOZ_ASSERT(qreg >= 0 && qreg < num_q_registers);
+ memcpy(value, vfp_registers_ + qreg * 4, sizeof(*value) * 4);
+}
+
+void Simulator::set_q_register(int qreg, const uint32_t* value) {
+ MOZ_ASSERT((qreg >= 0) && (qreg < num_q_registers));
+ memcpy(vfp_registers_ + qreg * 4, value, sizeof(*value) * 4);
+}
+
+void Simulator::set_pc(int32_t value) {
+ pc_modified_ = true;
+ registers_[pc] = value;
+}
+
+bool Simulator::has_bad_pc() const {
+ return registers_[pc] == bad_lr || registers_[pc] == end_sim_pc;
+}
+
+// Raw access to the PC register without the special adjustment when reading.
+int32_t Simulator::get_pc() const { return registers_[pc]; }
+
+void Simulator::set_s_register(int sreg, unsigned int value) {
+ MOZ_ASSERT(sreg >= 0 && sreg < num_s_registers);
+ vfp_registers_[sreg] = value;
+}
+
+unsigned Simulator::get_s_register(int sreg) const {
+ MOZ_ASSERT(sreg >= 0 && sreg < num_s_registers);
+ return vfp_registers_[sreg];
+}
+
+template <class InputType, int register_size>
+void Simulator::setVFPRegister(int reg_index, const InputType& value) {
+ MOZ_ASSERT(reg_index >= 0);
+ MOZ_ASSERT_IF(register_size == 1, reg_index < num_s_registers);
+ MOZ_ASSERT_IF(register_size == 2, reg_index < int(FloatRegisters::TotalPhys));
+
+ char buffer[register_size * sizeof(vfp_registers_[0])];
+ memcpy(buffer, &value, register_size * sizeof(vfp_registers_[0]));
+ memcpy(&vfp_registers_[reg_index * register_size], buffer,
+ register_size * sizeof(vfp_registers_[0]));
+}
+
+template <class ReturnType, int register_size>
+void Simulator::getFromVFPRegister(int reg_index, ReturnType* out) {
+ MOZ_ASSERT(reg_index >= 0);
+ MOZ_ASSERT_IF(register_size == 1, reg_index < num_s_registers);
+ MOZ_ASSERT_IF(register_size == 2, reg_index < int(FloatRegisters::TotalPhys));
+
+ char buffer[register_size * sizeof(vfp_registers_[0])];
+ memcpy(buffer, &vfp_registers_[register_size * reg_index],
+ register_size * sizeof(vfp_registers_[0]));
+ memcpy(out, buffer, register_size * sizeof(vfp_registers_[0]));
+}
+
+// These forced-instantiations are for jsapi-tests. Evidently, nothing
+// requires these to be instantiated.
+template void Simulator::getFromVFPRegister<double, 2>(int reg_index,
+ double* out);
+template void Simulator::getFromVFPRegister<float, 1>(int reg_index,
+ float* out);
+template void Simulator::setVFPRegister<double, 2>(int reg_index,
+ const double& value);
+template void Simulator::setVFPRegister<float, 1>(int reg_index,
+ const float& value);
+
+void Simulator::getFpArgs(double* x, double* y, int32_t* z) {
+ if (UseHardFpABI()) {
+ get_double_from_d_register(0, x);
+ get_double_from_d_register(1, y);
+ *z = get_register(0);
+ } else {
+ *x = get_double_from_register_pair(0);
+ *y = get_double_from_register_pair(2);
+ *z = get_register(2);
+ }
+}
+
+void Simulator::getFpFromStack(int32_t* stack, double* x) {
+ MOZ_ASSERT(stack && x);
+ char buffer[2 * sizeof(stack[0])];
+ memcpy(buffer, stack, 2 * sizeof(stack[0]));
+ memcpy(x, buffer, 2 * sizeof(stack[0]));
+}
+
+void Simulator::setCallResultDouble(double result) {
+ // The return value is either in r0/r1 or d0.
+ if (UseHardFpABI()) {
+ char buffer[2 * sizeof(vfp_registers_[0])];
+ memcpy(buffer, &result, sizeof(buffer));
+ // Copy result to d0.
+ memcpy(vfp_registers_, buffer, sizeof(buffer));
+ } else {
+ char buffer[2 * sizeof(registers_[0])];
+ memcpy(buffer, &result, sizeof(buffer));
+ // Copy result to r0 and r1.
+ memcpy(registers_, buffer, sizeof(buffer));
+ }
+}
+
+void Simulator::setCallResultFloat(float result) {
+ if (UseHardFpABI()) {
+ char buffer[sizeof(registers_[0])];
+ memcpy(buffer, &result, sizeof(buffer));
+ // Copy result to s0.
+ memcpy(vfp_registers_, buffer, sizeof(buffer));
+ } else {
+ char buffer[sizeof(registers_[0])];
+ memcpy(buffer, &result, sizeof(buffer));
+ // Copy result to r0.
+ memcpy(registers_, buffer, sizeof(buffer));
+ }
+}
+
+void Simulator::setCallResult(int64_t res) {
+ set_register(r0, static_cast<int32_t>(res));
+ set_register(r1, static_cast<int32_t>(res >> 32));
+}
+
+void Simulator::exclusiveMonitorSet(uint64_t value) {
+ exclusiveMonitor_ = value;
+ exclusiveMonitorHeld_ = true;
+}
+
+uint64_t Simulator::exclusiveMonitorGetAndClear(bool* held) {
+ *held = exclusiveMonitorHeld_;
+ exclusiveMonitorHeld_ = false;
+ return *held ? exclusiveMonitor_ : 0;
+}
+
+void Simulator::exclusiveMonitorClear() { exclusiveMonitorHeld_ = false; }
+
+JS::ProfilingFrameIterator::RegisterState Simulator::registerState() {
+ wasm::RegisterState state;
+ state.pc = (void*)get_pc();
+ state.fp = (void*)get_register(fp);
+ state.sp = (void*)get_register(sp);
+ state.lr = (void*)get_register(lr);
+ return state;
+}
+
+uint64_t Simulator::readQ(int32_t addr, SimInstruction* instr,
+ UnalignedPolicy f) {
+ if (handleWasmSegFault(addr, 8)) {
+ return UINT64_MAX;
+ }
+
+ if ((addr & 3) == 0 || (f == AllowUnaligned && !HasAlignmentFault())) {
+ uint64_t* ptr = reinterpret_cast<uint64_t*>(addr);
+ return *ptr;
+ }
+
+ // See the comments below in readW.
+ if (FixupFault() && wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ char* ptr = reinterpret_cast<char*>(addr);
+ uint64_t value;
+ memcpy(&value, ptr, sizeof(value));
+ return value;
+ }
+
+ printf("Unaligned read at 0x%08x, pc=%p\n", addr, instr);
+ MOZ_CRASH();
+}
+
+void Simulator::writeQ(int32_t addr, uint64_t value, SimInstruction* instr,
+ UnalignedPolicy f) {
+ if (handleWasmSegFault(addr, 8)) {
+ return;
+ }
+
+ if ((addr & 3) == 0 || (f == AllowUnaligned && !HasAlignmentFault())) {
+ uint64_t* ptr = reinterpret_cast<uint64_t*>(addr);
+ *ptr = value;
+ return;
+ }
+
+ // See the comments below in readW.
+ if (FixupFault() && wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ char* ptr = reinterpret_cast<char*>(addr);
+ memcpy(ptr, &value, sizeof(value));
+ return;
+ }
+
+ printf("Unaligned write at 0x%08x, pc=%p\n", addr, instr);
+ MOZ_CRASH();
+}
+
+int Simulator::readW(int32_t addr, SimInstruction* instr, UnalignedPolicy f) {
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ if ((addr & 3) == 0 || (f == AllowUnaligned && !HasAlignmentFault())) {
+ intptr_t* ptr = reinterpret_cast<intptr_t*>(addr);
+ return *ptr;
+ }
+
+ // In WebAssembly, we want unaligned accesses to either raise a signal or
+ // do the right thing. Making this simulator properly emulate the behavior
+ // of raising a signal is complex, so as a special-case, when in wasm code,
+ // we just do the right thing.
+ if (FixupFault() && wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ char* ptr = reinterpret_cast<char*>(addr);
+ int value;
+ memcpy(&value, ptr, sizeof(value));
+ return value;
+ }
+
+ printf("Unaligned read at 0x%08x, pc=%p\n", addr, instr);
+ MOZ_CRASH();
+}
+
+void Simulator::writeW(int32_t addr, int value, SimInstruction* instr,
+ UnalignedPolicy f) {
+ if (handleWasmSegFault(addr, 4)) {
+ return;
+ }
+
+ if ((addr & 3) == 0 || (f == AllowUnaligned && !HasAlignmentFault())) {
+ intptr_t* ptr = reinterpret_cast<intptr_t*>(addr);
+ *ptr = value;
+ return;
+ }
+
+ // See the comments above in readW.
+ if (FixupFault() && wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ char* ptr = reinterpret_cast<char*>(addr);
+ memcpy(ptr, &value, sizeof(value));
+ return;
+ }
+
+ printf("Unaligned write at 0x%08x, pc=%p\n", addr, instr);
+ MOZ_CRASH();
+}
+
+// For the time being, define Relaxed operations in terms of SeqCst
+// operations - we don't yet need Relaxed operations anywhere else in
+// the system, and the distinction is not important to the simulation
+// at the level where we're operating.
+
+template <typename T>
+static T loadRelaxed(SharedMem<T*> addr) {
+ return AtomicOperations::loadSeqCst(addr);
+}
+
+template <typename T>
+static T compareExchangeRelaxed(SharedMem<T*> addr, T oldval, T newval) {
+ return AtomicOperations::compareExchangeSeqCst(addr, oldval, newval);
+}
+
+int Simulator::readExW(int32_t addr, SimInstruction* instr) {
+ if (addr & 3) {
+ MOZ_CRASH("Unaligned exclusive read");
+ }
+
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ SharedMem<int32_t*> ptr =
+ SharedMem<int32_t*>::shared(reinterpret_cast<int32_t*>(addr));
+ int32_t value = loadRelaxed(ptr);
+ exclusiveMonitorSet(value);
+ return value;
+}
+
+int32_t Simulator::writeExW(int32_t addr, int value, SimInstruction* instr) {
+ if (addr & 3) {
+ MOZ_CRASH("Unaligned exclusive write");
+ }
+
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ SharedMem<int32_t*> ptr =
+ SharedMem<int32_t*>::shared(reinterpret_cast<int32_t*>(addr));
+ bool held;
+ int32_t expected = int32_t(exclusiveMonitorGetAndClear(&held));
+ if (!held) {
+ return 1;
+ }
+ int32_t old = compareExchangeRelaxed(ptr, expected, int32_t(value));
+ return old != expected;
+}
+
+uint16_t Simulator::readHU(int32_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return UINT16_MAX;
+ }
+
+ // The regexp engine emits unaligned loads, so we don't check for them here
+ // like most of the other methods do.
+ if ((addr & 1) == 0 || !HasAlignmentFault()) {
+ uint16_t* ptr = reinterpret_cast<uint16_t*>(addr);
+ return *ptr;
+ }
+
+ // See comments above in readW.
+ if (FixupFault() && wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ char* ptr = reinterpret_cast<char*>(addr);
+ uint16_t value;
+ memcpy(&value, ptr, sizeof(value));
+ return value;
+ }
+
+ printf("Unaligned unsigned halfword read at 0x%08x, pc=%p\n", addr, instr);
+ MOZ_CRASH();
+ return 0;
+}
+
+int16_t Simulator::readH(int32_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return -1;
+ }
+
+ if ((addr & 1) == 0 || !HasAlignmentFault()) {
+ int16_t* ptr = reinterpret_cast<int16_t*>(addr);
+ return *ptr;
+ }
+
+ // See comments above in readW.
+ if (FixupFault() && wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ char* ptr = reinterpret_cast<char*>(addr);
+ int16_t value;
+ memcpy(&value, ptr, sizeof(value));
+ return value;
+ }
+
+ printf("Unaligned signed halfword read at 0x%08x\n", addr);
+ MOZ_CRASH();
+ return 0;
+}
+
+void Simulator::writeH(int32_t addr, uint16_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return;
+ }
+
+ if ((addr & 1) == 0 || !HasAlignmentFault()) {
+ uint16_t* ptr = reinterpret_cast<uint16_t*>(addr);
+ *ptr = value;
+ return;
+ }
+
+ // See the comments above in readW.
+ if (FixupFault() && wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ char* ptr = reinterpret_cast<char*>(addr);
+ memcpy(ptr, &value, sizeof(value));
+ return;
+ }
+
+ printf("Unaligned unsigned halfword write at 0x%08x, pc=%p\n", addr, instr);
+ MOZ_CRASH();
+}
+
+void Simulator::writeH(int32_t addr, int16_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return;
+ }
+
+ if ((addr & 1) == 0 || !HasAlignmentFault()) {
+ int16_t* ptr = reinterpret_cast<int16_t*>(addr);
+ *ptr = value;
+ return;
+ }
+
+ // See the comments above in readW.
+ if (FixupFault() && wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ char* ptr = reinterpret_cast<char*>(addr);
+ memcpy(ptr, &value, sizeof(value));
+ return;
+ }
+
+ printf("Unaligned halfword write at 0x%08x, pc=%p\n", addr, instr);
+ MOZ_CRASH();
+}
+
+uint16_t Simulator::readExHU(int32_t addr, SimInstruction* instr) {
+ if (addr & 1) {
+ MOZ_CRASH("Unaligned exclusive read");
+ }
+
+ if (handleWasmSegFault(addr, 2)) {
+ return UINT16_MAX;
+ }
+
+ SharedMem<uint16_t*> ptr =
+ SharedMem<uint16_t*>::shared(reinterpret_cast<uint16_t*>(addr));
+ uint16_t value = loadRelaxed(ptr);
+ exclusiveMonitorSet(value);
+ return value;
+}
+
+int32_t Simulator::writeExH(int32_t addr, uint16_t value,
+ SimInstruction* instr) {
+ if (addr & 1) {
+ MOZ_CRASH("Unaligned exclusive write");
+ }
+
+ if (handleWasmSegFault(addr, 2)) {
+ return -1;
+ }
+
+ SharedMem<uint16_t*> ptr =
+ SharedMem<uint16_t*>::shared(reinterpret_cast<uint16_t*>(addr));
+ bool held;
+ uint16_t expected = uint16_t(exclusiveMonitorGetAndClear(&held));
+ if (!held) {
+ return 1;
+ }
+ uint16_t old = compareExchangeRelaxed(ptr, expected, value);
+ return old != expected;
+}
+
+uint8_t Simulator::readBU(int32_t addr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return UINT8_MAX;
+ }
+
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(addr);
+ return *ptr;
+}
+
+uint8_t Simulator::readExBU(int32_t addr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return UINT8_MAX;
+ }
+
+ SharedMem<uint8_t*> ptr =
+ SharedMem<uint8_t*>::shared(reinterpret_cast<uint8_t*>(addr));
+ uint8_t value = loadRelaxed(ptr);
+ exclusiveMonitorSet(value);
+ return value;
+}
+
+int32_t Simulator::writeExB(int32_t addr, uint8_t value) {
+ if (handleWasmSegFault(addr, 1)) {
+ return -1;
+ }
+
+ SharedMem<uint8_t*> ptr =
+ SharedMem<uint8_t*>::shared(reinterpret_cast<uint8_t*>(addr));
+ bool held;
+ uint8_t expected = uint8_t(exclusiveMonitorGetAndClear(&held));
+ if (!held) {
+ return 1;
+ }
+ uint8_t old = compareExchangeRelaxed(ptr, expected, value);
+ return old != expected;
+}
+
+int8_t Simulator::readB(int32_t addr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return -1;
+ }
+
+ int8_t* ptr = reinterpret_cast<int8_t*>(addr);
+ return *ptr;
+}
+
+void Simulator::writeB(int32_t addr, uint8_t value) {
+ if (handleWasmSegFault(addr, 1)) {
+ return;
+ }
+
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(addr);
+ *ptr = value;
+}
+
+void Simulator::writeB(int32_t addr, int8_t value) {
+ if (handleWasmSegFault(addr, 1)) {
+ return;
+ }
+
+ int8_t* ptr = reinterpret_cast<int8_t*>(addr);
+ *ptr = value;
+}
+
+int32_t* Simulator::readDW(int32_t addr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return nullptr;
+ }
+
+ if ((addr & 3) == 0) {
+ int32_t* ptr = reinterpret_cast<int32_t*>(addr);
+ return ptr;
+ }
+
+ printf("Unaligned read at 0x%08x\n", addr);
+ MOZ_CRASH();
+}
+
+void Simulator::writeDW(int32_t addr, int32_t value1, int32_t value2) {
+ if (handleWasmSegFault(addr, 8)) {
+ return;
+ }
+
+ if ((addr & 3) == 0) {
+ int32_t* ptr = reinterpret_cast<int32_t*>(addr);
+ *ptr++ = value1;
+ *ptr = value2;
+ return;
+ }
+
+ printf("Unaligned write at 0x%08x\n", addr);
+ MOZ_CRASH();
+}
+
+int32_t Simulator::readExDW(int32_t addr, int32_t* hibits) {
+ if (addr & 3) {
+ MOZ_CRASH("Unaligned exclusive read");
+ }
+
+ if (handleWasmSegFault(addr, 8)) {
+ return -1;
+ }
+
+ SharedMem<uint64_t*> ptr =
+ SharedMem<uint64_t*>::shared(reinterpret_cast<uint64_t*>(addr));
+ // The spec says that the low part of value shall be read from addr and
+ // the high part shall be read from addr+4. On a little-endian system
+ // where we read a 64-bit quadword the low part of the value will be in
+ // the low part of the quadword, and the high part of the value in the
+ // high part of the quadword.
+ uint64_t value = loadRelaxed(ptr);
+ exclusiveMonitorSet(value);
+ *hibits = int32_t(value >> 32);
+ return int32_t(value);
+}
+
+int32_t Simulator::writeExDW(int32_t addr, int32_t value1, int32_t value2) {
+ if (addr & 3) {
+ MOZ_CRASH("Unaligned exclusive write");
+ }
+
+ if (handleWasmSegFault(addr, 8)) {
+ return -1;
+ }
+
+ SharedMem<uint64_t*> ptr =
+ SharedMem<uint64_t*>::shared(reinterpret_cast<uint64_t*>(addr));
+ // The spec says that value1 shall be stored at addr and value2 at
+ // addr+4. On a little-endian system that means constructing a 64-bit
+ // value where value1 is in the low half of a 64-bit quadword and value2
+ // is in the high half of the quadword.
+ uint64_t value = (uint64_t(value2) << 32) | uint32_t(value1);
+ bool held;
+ uint64_t expected = exclusiveMonitorGetAndClear(&held);
+ if (!held) {
+ return 1;
+ }
+ uint64_t old = compareExchangeRelaxed(ptr, expected, value);
+ return old != expected;
+}
+
+uintptr_t Simulator::stackLimit() const { return stackLimit_; }
+
+uintptr_t* Simulator::addressOfStackLimit() { return &stackLimit_; }
+
+bool Simulator::overRecursed(uintptr_t newsp) const {
+ if (newsp == 0) {
+ newsp = get_register(sp);
+ }
+ return newsp <= stackLimit();
+}
+
+bool Simulator::overRecursedWithExtra(uint32_t extra) const {
+ uintptr_t newsp = get_register(sp) - extra;
+ return newsp <= stackLimit();
+}
+
+// Checks if the current instruction should be executed based on its condition
+// bits.
+bool Simulator::conditionallyExecute(SimInstruction* instr) {
+ switch (instr->conditionField()) {
+ case Assembler::EQ:
+ return z_flag_;
+ case Assembler::NE:
+ return !z_flag_;
+ case Assembler::CS:
+ return c_flag_;
+ case Assembler::CC:
+ return !c_flag_;
+ case Assembler::MI:
+ return n_flag_;
+ case Assembler::PL:
+ return !n_flag_;
+ case Assembler::VS:
+ return v_flag_;
+ case Assembler::VC:
+ return !v_flag_;
+ case Assembler::HI:
+ return c_flag_ && !z_flag_;
+ case Assembler::LS:
+ return !c_flag_ || z_flag_;
+ case Assembler::GE:
+ return n_flag_ == v_flag_;
+ case Assembler::LT:
+ return n_flag_ != v_flag_;
+ case Assembler::GT:
+ return !z_flag_ && (n_flag_ == v_flag_);
+ case Assembler::LE:
+ return z_flag_ || (n_flag_ != v_flag_);
+ case Assembler::AL:
+ return true;
+ default:
+ MOZ_CRASH();
+ }
+ return false;
+}
+
+// Calculate and set the Negative and Zero flags.
+void Simulator::setNZFlags(int32_t val) {
+ n_flag_ = (val < 0);
+ z_flag_ = (val == 0);
+}
+
+// Set the Carry flag.
+void Simulator::setCFlag(bool val) { c_flag_ = val; }
+
+// Set the oVerflow flag.
+void Simulator::setVFlag(bool val) { v_flag_ = val; }
+
+// Calculate C flag value for additions.
+bool Simulator::carryFrom(int32_t left, int32_t right, int32_t carry) {
+ uint32_t uleft = static_cast<uint32_t>(left);
+ uint32_t uright = static_cast<uint32_t>(right);
+ uint32_t urest = 0xffffffffU - uleft;
+ return (uright > urest) ||
+ (carry && (((uright + 1) > urest) || (uright > (urest - 1))));
+}
+
+// Calculate C flag value for subtractions.
+bool Simulator::borrowFrom(int32_t left, int32_t right) {
+ uint32_t uleft = static_cast<uint32_t>(left);
+ uint32_t uright = static_cast<uint32_t>(right);
+ return (uright > uleft);
+}
+
+// Calculate V flag value for additions and subtractions.
+bool Simulator::overflowFrom(int32_t alu_out, int32_t left, int32_t right,
+ bool addition) {
+ bool overflow;
+ if (addition) {
+ // Operands have the same sign.
+ overflow = ((left >= 0 && right >= 0) || (left < 0 && right < 0))
+ // And operands and result have different sign.
+ && ((left < 0 && alu_out >= 0) || (left >= 0 && alu_out < 0));
+ } else {
+ // Operands have different signs.
+ overflow = ((left < 0 && right >= 0) || (left >= 0 && right < 0))
+ // And first operand and result have different signs.
+ && ((left < 0 && alu_out >= 0) || (left >= 0 && alu_out < 0));
+ }
+ return overflow;
+}
+
+// Support for VFP comparisons.
+void Simulator::compute_FPSCR_Flags(double val1, double val2) {
+ if (mozilla::IsNaN(val1) || mozilla::IsNaN(val2)) {
+ n_flag_FPSCR_ = false;
+ z_flag_FPSCR_ = false;
+ c_flag_FPSCR_ = true;
+ v_flag_FPSCR_ = true;
+ // All non-NaN cases.
+ } else if (val1 == val2) {
+ n_flag_FPSCR_ = false;
+ z_flag_FPSCR_ = true;
+ c_flag_FPSCR_ = true;
+ v_flag_FPSCR_ = false;
+ } else if (val1 < val2) {
+ n_flag_FPSCR_ = true;
+ z_flag_FPSCR_ = false;
+ c_flag_FPSCR_ = false;
+ v_flag_FPSCR_ = false;
+ } else {
+ // Case when (val1 > val2).
+ n_flag_FPSCR_ = false;
+ z_flag_FPSCR_ = false;
+ c_flag_FPSCR_ = true;
+ v_flag_FPSCR_ = false;
+ }
+}
+
+void Simulator::copy_FPSCR_to_APSR() {
+ n_flag_ = n_flag_FPSCR_;
+ z_flag_ = z_flag_FPSCR_;
+ c_flag_ = c_flag_FPSCR_;
+ v_flag_ = v_flag_FPSCR_;
+}
+
+// Addressing Mode 1 - Data-processing operands:
+// Get the value based on the shifter_operand with register.
+int32_t Simulator::getShiftRm(SimInstruction* instr, bool* carry_out) {
+ ShiftType shift = instr->shifttypeValue();
+ int shift_amount = instr->shiftAmountValue();
+ int32_t result = get_register(instr->rmValue());
+ if (instr->bit(4) == 0) {
+ // By immediate.
+ if (shift == ROR && shift_amount == 0) {
+ MOZ_CRASH("NYI");
+ return result;
+ }
+ if ((shift == LSR || shift == ASR) && shift_amount == 0) {
+ shift_amount = 32;
+ }
+ switch (shift) {
+ case ASR: {
+ if (shift_amount == 0) {
+ if (result < 0) {
+ result = 0xffffffff;
+ *carry_out = true;
+ } else {
+ result = 0;
+ *carry_out = false;
+ }
+ } else {
+ result >>= (shift_amount - 1);
+ *carry_out = (result & 1) == 1;
+ result >>= 1;
+ }
+ break;
+ }
+
+ case LSL: {
+ if (shift_amount == 0) {
+ *carry_out = c_flag_;
+ } else {
+ result <<= (shift_amount - 1);
+ *carry_out = (result < 0);
+ result <<= 1;
+ }
+ break;
+ }
+
+ case LSR: {
+ if (shift_amount == 0) {
+ result = 0;
+ *carry_out = c_flag_;
+ } else {
+ uint32_t uresult = static_cast<uint32_t>(result);
+ uresult >>= (shift_amount - 1);
+ *carry_out = (uresult & 1) == 1;
+ uresult >>= 1;
+ result = static_cast<int32_t>(uresult);
+ }
+ break;
+ }
+
+ case ROR: {
+ if (shift_amount == 0) {
+ *carry_out = c_flag_;
+ } else {
+ uint32_t left = static_cast<uint32_t>(result) >> shift_amount;
+ uint32_t right = static_cast<uint32_t>(result) << (32 - shift_amount);
+ result = right | left;
+ *carry_out = (static_cast<uint32_t>(result) >> 31) != 0;
+ }
+ break;
+ }
+
+ default:
+ MOZ_CRASH();
+ }
+ } else {
+ // By register.
+ int rs = instr->rsValue();
+ shift_amount = get_register(rs) & 0xff;
+ switch (shift) {
+ case ASR: {
+ if (shift_amount == 0) {
+ *carry_out = c_flag_;
+ } else if (shift_amount < 32) {
+ result >>= (shift_amount - 1);
+ *carry_out = (result & 1) == 1;
+ result >>= 1;
+ } else {
+ MOZ_ASSERT(shift_amount >= 32);
+ if (result < 0) {
+ *carry_out = true;
+ result = 0xffffffff;
+ } else {
+ *carry_out = false;
+ result = 0;
+ }
+ }
+ break;
+ }
+
+ case LSL: {
+ if (shift_amount == 0) {
+ *carry_out = c_flag_;
+ } else if (shift_amount < 32) {
+ result <<= (shift_amount - 1);
+ *carry_out = (result < 0);
+ result <<= 1;
+ } else if (shift_amount == 32) {
+ *carry_out = (result & 1) == 1;
+ result = 0;
+ } else {
+ MOZ_ASSERT(shift_amount > 32);
+ *carry_out = false;
+ result = 0;
+ }
+ break;
+ }
+
+ case LSR: {
+ if (shift_amount == 0) {
+ *carry_out = c_flag_;
+ } else if (shift_amount < 32) {
+ uint32_t uresult = static_cast<uint32_t>(result);
+ uresult >>= (shift_amount - 1);
+ *carry_out = (uresult & 1) == 1;
+ uresult >>= 1;
+ result = static_cast<int32_t>(uresult);
+ } else if (shift_amount == 32) {
+ *carry_out = (result < 0);
+ result = 0;
+ } else {
+ *carry_out = false;
+ result = 0;
+ }
+ break;
+ }
+
+ case ROR: {
+ if (shift_amount == 0) {
+ *carry_out = c_flag_;
+ } else {
+ uint32_t left = static_cast<uint32_t>(result) >> shift_amount;
+ uint32_t right = static_cast<uint32_t>(result) << (32 - shift_amount);
+ result = right | left;
+ *carry_out = (static_cast<uint32_t>(result) >> 31) != 0;
+ }
+ break;
+ }
+
+ default:
+ MOZ_CRASH();
+ }
+ }
+ return result;
+}
+
+// Addressing Mode 1 - Data-processing operands:
+// Get the value based on the shifter_operand with immediate.
+int32_t Simulator::getImm(SimInstruction* instr, bool* carry_out) {
+ int rotate = instr->rotateValue() * 2;
+ int immed8 = instr->immed8Value();
+ int imm = (immed8 >> rotate) | (immed8 << (32 - rotate));
+ *carry_out = (rotate == 0) ? c_flag_ : (imm < 0);
+ return imm;
+}
+
+int32_t Simulator::processPU(SimInstruction* instr, int num_regs, int reg_size,
+ intptr_t* start_address, intptr_t* end_address) {
+ int rn = instr->rnValue();
+ int32_t rn_val = get_register(rn);
+ switch (instr->PUField()) {
+ case da_x:
+ MOZ_CRASH();
+ break;
+ case ia_x:
+ *start_address = rn_val;
+ *end_address = rn_val + (num_regs * reg_size) - reg_size;
+ rn_val = rn_val + (num_regs * reg_size);
+ break;
+ case db_x:
+ *start_address = rn_val - (num_regs * reg_size);
+ *end_address = rn_val - reg_size;
+ rn_val = *start_address;
+ break;
+ case ib_x:
+ *start_address = rn_val + reg_size;
+ *end_address = rn_val + (num_regs * reg_size);
+ rn_val = *end_address;
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ return rn_val;
+}
+
+// Addressing Mode 4 - Load and Store Multiple
+void Simulator::handleRList(SimInstruction* instr, bool load) {
+ int rlist = instr->rlistValue();
+ int num_regs = mozilla::CountPopulation32(rlist);
+
+ intptr_t start_address = 0;
+ intptr_t end_address = 0;
+ int32_t rn_val =
+ processPU(instr, num_regs, sizeof(void*), &start_address, &end_address);
+ intptr_t* address = reinterpret_cast<intptr_t*>(start_address);
+
+ // Catch null pointers a little earlier.
+ MOZ_ASSERT(start_address > 8191 || start_address < 0);
+
+ int reg = 0;
+ while (rlist != 0) {
+ if ((rlist & 1) != 0) {
+ if (load) {
+ set_register(reg, *address);
+ } else {
+ *address = get_register(reg);
+ }
+ address += 1;
+ }
+ reg++;
+ rlist >>= 1;
+ }
+ MOZ_ASSERT(end_address == ((intptr_t)address) - 4);
+ if (instr->hasW()) {
+ set_register(instr->rnValue(), rn_val);
+ }
+}
+
+// Addressing Mode 6 - Load and Store Multiple Coprocessor registers.
+void Simulator::handleVList(SimInstruction* instr) {
+ VFPRegPrecision precision =
+ (instr->szValue() == 0) ? kSinglePrecision : kDoublePrecision;
+ int operand_size = (precision == kSinglePrecision) ? 4 : 8;
+ bool load = (instr->VLValue() == 0x1);
+
+ int vd;
+ int num_regs;
+ vd = instr->VFPDRegValue(precision);
+ if (precision == kSinglePrecision) {
+ num_regs = instr->immed8Value();
+ } else {
+ num_regs = instr->immed8Value() / 2;
+ }
+
+ intptr_t start_address = 0;
+ intptr_t end_address = 0;
+ int32_t rn_val =
+ processPU(instr, num_regs, operand_size, &start_address, &end_address);
+
+ intptr_t* address = reinterpret_cast<intptr_t*>(start_address);
+ for (int reg = vd; reg < vd + num_regs; reg++) {
+ if (precision == kSinglePrecision) {
+ if (load) {
+ set_s_register_from_sinteger(
+ reg, readW(reinterpret_cast<int32_t>(address), instr));
+ } else {
+ writeW(reinterpret_cast<int32_t>(address),
+ get_sinteger_from_s_register(reg), instr);
+ }
+ address += 1;
+ } else {
+ if (load) {
+ int32_t data[] = {readW(reinterpret_cast<int32_t>(address), instr),
+ readW(reinterpret_cast<int32_t>(address + 1), instr)};
+ double d;
+ memcpy(&d, data, 8);
+ set_d_register_from_double(reg, d);
+ } else {
+ int32_t data[2];
+ double d;
+ get_double_from_d_register(reg, &d);
+ memcpy(data, &d, 8);
+ writeW(reinterpret_cast<int32_t>(address), data[0], instr);
+ writeW(reinterpret_cast<int32_t>(address + 1), data[1], instr);
+ }
+ address += 2;
+ }
+ }
+ MOZ_ASSERT(reinterpret_cast<intptr_t>(address) - operand_size == end_address);
+ if (instr->hasW()) {
+ set_register(instr->rnValue(), rn_val);
+ }
+}
+
+// Note: With the code below we assume that all runtime calls return a 64 bits
+// result. If they don't, the r1 result register contains a bogus value, which
+// is fine because it is caller-saved.
+typedef int64_t (*Prototype_General0)();
+typedef int64_t (*Prototype_General1)(int32_t arg0);
+typedef int64_t (*Prototype_General2)(int32_t arg0, int32_t arg1);
+typedef int64_t (*Prototype_General3)(int32_t arg0, int32_t arg1, int32_t arg2);
+typedef int64_t (*Prototype_General4)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3);
+typedef int64_t (*Prototype_General5)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3, int32_t arg4);
+typedef int64_t (*Prototype_General6)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3, int32_t arg4, int32_t arg5);
+typedef int64_t (*Prototype_General7)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3, int32_t arg4, int32_t arg5,
+ int32_t arg6);
+typedef int64_t (*Prototype_General8)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3, int32_t arg4, int32_t arg5,
+ int32_t arg6, int32_t arg7);
+typedef int64_t (*Prototype_GeneralGeneralGeneralInt64)(int32_t arg0,
+ int32_t arg1,
+ int32_t arg2,
+ int64_t arg3);
+typedef int64_t (*Prototype_GeneralGeneralInt64Int64)(int32_t arg0,
+ int32_t arg1,
+ int64_t arg2,
+ int64_t arg3);
+
+typedef double (*Prototype_Double_None)();
+typedef double (*Prototype_Double_Double)(double arg0);
+typedef double (*Prototype_Double_Int)(int32_t arg0);
+typedef double (*Prototype_Double_IntInt)(int32_t arg0, int32_t arg1);
+typedef int32_t (*Prototype_Int_Double)(double arg0);
+typedef int64_t (*Prototype_Int64_Double)(double arg0);
+typedef int32_t (*Prototype_Int_DoubleIntInt)(double arg0, int32_t arg1,
+ int32_t arg2);
+typedef int32_t (*Prototype_Int_IntDoubleIntInt)(int32_t arg0, double arg1,
+ int32_t arg2, int32_t arg3);
+
+typedef int32_t (*Prototype_Int_Float32)(float arg0);
+typedef float (*Prototype_Float32_Float32)(float arg0);
+typedef float (*Prototype_Float32_Float32Float32)(float arg0, float arg1);
+typedef float (*Prototype_Float32_IntInt)(int arg0, int arg1);
+
+typedef double (*Prototype_Double_DoubleInt)(double arg0, int32_t arg1);
+typedef double (*Prototype_Double_IntDouble)(int32_t arg0, double arg1);
+typedef double (*Prototype_Double_DoubleDouble)(double arg0, double arg1);
+typedef int32_t (*Prototype_Int_IntDouble)(int32_t arg0, double arg1);
+typedef int32_t (*Prototype_Int_DoubleInt)(double arg0, int32_t arg1);
+
+typedef double (*Prototype_Double_DoubleDoubleDouble)(double arg0, double arg1,
+ double arg2);
+typedef double (*Prototype_Double_DoubleDoubleDoubleDouble)(double arg0,
+ double arg1,
+ double arg2,
+ double arg3);
+
+typedef int32_t (*Prototype_Int32_General)(int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32)(int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32)(int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32)(int32_t, int32_t,
+ int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32Int32)(
+ int32_t, int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32General)(
+ int32_t, int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32Int32Int32General)(
+ int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (
+ *Prototype_Int32_GeneralInt32Float32Float32Int32Int32Int32General)(
+ int32_t, int32_t, float, float, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (
+ *Prototype_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General)(
+ int32_t, int32_t, float, float, float, float, int32_t, int32_t, int32_t,
+ int32_t, int32_t);
+typedef int32_t (
+ *Prototype_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General)(
+ int32_t, int32_t, float, float, int32_t, float, float, int32_t, float,
+ int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32General)(
+ int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int64)(int32_t, int32_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32General)(int32_t, int32_t,
+ int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int64Int64)(int32_t, int32_t,
+ int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32)(int32_t, int32_t,
+ int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32Int32)(
+ int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneral)(int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralGeneral)(int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralInt32Int32)(int32_t, int32_t,
+ int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int32Int32)(int32_t, int64_t,
+ int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32)(int32_t, int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int64)(int32_t, int64_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int64General)(
+ int32_t, int64_t, int32_t, int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int64Int64)(int32_t, int64_t,
+ int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int64Int64General)(
+ int32_t, int64_t, int64_t, int64_t, int32_t);
+typedef int32_t (*Prototype_General_GeneralInt32)(int32_t, int32_t);
+typedef int32_t (*Prototype_General_GeneralInt32Int32)(int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_General_GeneralInt32General)(int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_General_GeneralInt32Int32GeneralInt32)(
+ int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32)(
+ int32_t, int32_t, int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int64_t (*Prototype_Int64_General)(int32_t);
+typedef int64_t (*Prototype_Int64_GeneralInt64)(int32_t, int64_t);
+
+// Fill the volatile registers with scratch values.
+//
+// Some of the ABI calls assume that the float registers are not scratched,
+// even though the ABI defines them as volatile - a performance
+// optimization. These are all calls passing operands in integer registers,
+// so for now the simulator does not scratch any float registers for these
+// calls. Should try to narrow it further in future.
+//
+void Simulator::scratchVolatileRegisters(bool scratchFloat) {
+ int32_t scratch_value = 0xa5a5a5a5 ^ uint32_t(icount_);
+ set_register(r0, scratch_value);
+ set_register(r1, scratch_value);
+ set_register(r2, scratch_value);
+ set_register(r3, scratch_value);
+ set_register(r12, scratch_value); // Intra-Procedure-call scratch register.
+ set_register(r14, scratch_value); // Link register.
+
+ if (scratchFloat) {
+ uint64_t scratch_value_d =
+ 0x5a5a5a5a5a5a5a5aLU ^ uint64_t(icount_) ^ (uint64_t(icount_) << 30);
+ for (uint32_t i = d0; i < d8; i++) {
+ set_d_register(i, &scratch_value_d);
+ }
+ for (uint32_t i = d16; i < FloatRegisters::TotalPhys; i++) {
+ set_d_register(i, &scratch_value_d);
+ }
+ }
+}
+
+static int64_t MakeInt64(int32_t first, int32_t second) {
+ // Little-endian order.
+ return ((int64_t)second << 32) | (uint32_t)first;
+}
+
+// Software interrupt instructions are used by the simulator to call into C++.
+void Simulator::softwareInterrupt(SimInstruction* instr) {
+ int svc = instr->svcValue();
+ switch (svc) {
+ case kCallRtRedirected: {
+ Redirection* redirection = Redirection::FromSwiInstruction(instr);
+ int32_t arg0 = get_register(r0);
+ int32_t arg1 = get_register(r1);
+ int32_t arg2 = get_register(r2);
+ int32_t arg3 = get_register(r3);
+ int32_t* stack_pointer = reinterpret_cast<int32_t*>(get_register(sp));
+ int32_t arg4 = stack_pointer[0];
+ int32_t arg5 = stack_pointer[1];
+ int32_t arg6 = stack_pointer[2];
+ int32_t arg7 = stack_pointer[3];
+ int32_t arg8 = stack_pointer[4];
+ int32_t arg9 = stack_pointer[5];
+ int32_t arg10 = stack_pointer[6];
+ int32_t arg11 = stack_pointer[7];
+ int32_t arg12 = stack_pointer[8];
+ int32_t arg13 = stack_pointer[9];
+
+ int32_t saved_lr = get_register(lr);
+ intptr_t external =
+ reinterpret_cast<intptr_t>(redirection->nativeFunction());
+
+ bool stack_aligned = (get_register(sp) & (ABIStackAlignment - 1)) == 0;
+ if (!stack_aligned) {
+ fprintf(stderr, "Runtime call with unaligned stack!\n");
+ MOZ_CRASH();
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ switch (redirection->type()) {
+ case Args_General0: {
+ Prototype_General0 target =
+ reinterpret_cast<Prototype_General0>(external);
+ int64_t result = target();
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_General1: {
+ Prototype_General1 target =
+ reinterpret_cast<Prototype_General1>(external);
+ int64_t result = target(arg0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_General2: {
+ Prototype_General2 target =
+ reinterpret_cast<Prototype_General2>(external);
+ int64_t result = target(arg0, arg1);
+ // The ARM backend makes calls to __aeabi_idivmod and
+ // __aeabi_uidivmod assuming that the float registers are
+ // non-volatile as a performance optimization, so the float
+ // registers must not be scratch when calling these.
+ bool scratchFloat =
+ target != __aeabi_idivmod && target != __aeabi_uidivmod;
+ scratchVolatileRegisters(/* scratchFloat = */ scratchFloat);
+ setCallResult(result);
+ break;
+ }
+ case Args_General3: {
+ Prototype_General3 target =
+ reinterpret_cast<Prototype_General3>(external);
+ int64_t result = target(arg0, arg1, arg2);
+ scratchVolatileRegisters(/* scratchFloat = true*/);
+ setCallResult(result);
+ break;
+ }
+ case Args_General4: {
+ Prototype_General4 target =
+ reinterpret_cast<Prototype_General4>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ scratchVolatileRegisters(/* scratchFloat = true*/);
+ setCallResult(result);
+ break;
+ }
+ case Args_General5: {
+ Prototype_General5 target =
+ reinterpret_cast<Prototype_General5>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_General6: {
+ Prototype_General6 target =
+ reinterpret_cast<Prototype_General6>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_General7: {
+ Prototype_General7 target =
+ reinterpret_cast<Prototype_General7>(external);
+ int32_t arg6 = stack_pointer[2];
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_General8: {
+ Prototype_General8 target =
+ reinterpret_cast<Prototype_General8>(external);
+ int32_t arg6 = stack_pointer[2];
+ int32_t arg7 = stack_pointer[3];
+ int64_t result =
+ target(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int_GeneralGeneralGeneralInt64: {
+ Prototype_GeneralGeneralGeneralInt64 target =
+ reinterpret_cast<Prototype_GeneralGeneralGeneralInt64>(external);
+ // The int64 arg is not split across register and stack
+ int64_t result = target(arg0, arg1, arg2, MakeInt64(arg4, arg5));
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int_GeneralGeneralInt64Int64: {
+ Prototype_GeneralGeneralInt64Int64 target =
+ reinterpret_cast<Prototype_GeneralGeneralInt64Int64>(external);
+ int64_t result =
+ target(arg0, arg1, MakeInt64(arg2, arg3), MakeInt64(arg4, arg5));
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int64_Double: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Int64_Double target =
+ reinterpret_cast<Prototype_Int64_Double>(external);
+ int64_t result = target(dval0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Double_None: {
+ Prototype_Double_None target =
+ reinterpret_cast<Prototype_Double_None>(external);
+ double dresult = target();
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int_Double: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Int_Double target =
+ reinterpret_cast<Prototype_Int_Double>(external);
+ int32_t res = target(dval0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ set_register(r0, res);
+ break;
+ }
+ case Args_Int_Float32: {
+ float fval0;
+ if (UseHardFpABI()) {
+ get_float_from_s_register(0, &fval0);
+ } else {
+ fval0 = mozilla::BitwiseCast<float>(arg0);
+ }
+ auto target = reinterpret_cast<Prototype_Int_Float32>(external);
+ int32_t res = target(fval0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ set_register(r0, res);
+ break;
+ }
+ case Args_Double_Double: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Double_Double target =
+ reinterpret_cast<Prototype_Double_Double>(external);
+ double dresult = target(dval0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Float32_Float32: {
+ float fval0;
+ if (UseHardFpABI()) {
+ get_float_from_s_register(0, &fval0);
+ } else {
+ fval0 = mozilla::BitwiseCast<float>(arg0);
+ }
+ Prototype_Float32_Float32 target =
+ reinterpret_cast<Prototype_Float32_Float32>(external);
+ float fresult = target(fval0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Float32_Float32Float32: {
+ float fval0, fval1;
+ if (UseHardFpABI()) {
+ get_float_from_s_register(0, &fval0);
+ get_float_from_s_register(1, &fval1);
+ } else {
+ fval0 = mozilla::BitwiseCast<float>(arg0);
+ fval1 = mozilla::BitwiseCast<float>(arg1);
+ }
+ Prototype_Float32_Float32Float32 target =
+ reinterpret_cast<Prototype_Float32_Float32Float32>(external);
+ float fresult = target(fval0, fval1);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Float32_IntInt: {
+ Prototype_Float32_IntInt target =
+ reinterpret_cast<Prototype_Float32_IntInt>(external);
+ float fresult = target(arg0, arg1);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Double_Int: {
+ Prototype_Double_Int target =
+ reinterpret_cast<Prototype_Double_Int>(external);
+ double dresult = target(arg0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_IntInt: {
+ Prototype_Double_IntInt target =
+ reinterpret_cast<Prototype_Double_IntInt>(external);
+ double dresult = target(arg0, arg1);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleInt: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Double_DoubleInt target =
+ reinterpret_cast<Prototype_Double_DoubleInt>(external);
+ double dresult = target(dval0, ival);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleDouble: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Double_DoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDouble>(external);
+ double dresult = target(dval0, dval1);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_IntDouble: {
+ int32_t ival = get_register(0);
+ double dval0;
+ if (UseHardFpABI()) {
+ get_double_from_d_register(0, &dval0);
+ } else {
+ dval0 = get_double_from_register_pair(2);
+ }
+ Prototype_Double_IntDouble target =
+ reinterpret_cast<Prototype_Double_IntDouble>(external);
+ double dresult = target(ival, dval0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int_IntDouble: {
+ int32_t ival = get_register(0);
+ double dval0;
+ if (UseHardFpABI()) {
+ get_double_from_d_register(0, &dval0);
+ } else {
+ dval0 = get_double_from_register_pair(2);
+ }
+ Prototype_Int_IntDouble target =
+ reinterpret_cast<Prototype_Int_IntDouble>(external);
+ int32_t result = target(ival, dval0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ set_register(r0, result);
+ break;
+ }
+ case Args_Int_DoubleInt: {
+ double dval;
+ int32_t result;
+ Prototype_Int_DoubleInt target =
+ reinterpret_cast<Prototype_Int_DoubleInt>(external);
+ if (UseHardFpABI()) {
+ get_double_from_d_register(0, &dval);
+ result = target(dval, arg0);
+ } else {
+ dval = get_double_from_register_pair(0);
+ result = target(dval, arg2);
+ }
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ set_register(r0, result);
+ break;
+ }
+ case Args_Int_DoubleIntInt: {
+ double dval;
+ int32_t result;
+ Prototype_Int_DoubleIntInt target =
+ reinterpret_cast<Prototype_Int_DoubleIntInt>(external);
+ if (UseHardFpABI()) {
+ get_double_from_d_register(0, &dval);
+ result = target(dval, arg0, arg1);
+ } else {
+ dval = get_double_from_register_pair(0);
+ result = target(dval, arg2, arg3);
+ }
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ set_register(r0, result);
+ break;
+ }
+ case Args_Int_IntDoubleIntInt: {
+ double dval;
+ int32_t result;
+ Prototype_Int_IntDoubleIntInt target =
+ reinterpret_cast<Prototype_Int_IntDoubleIntInt>(external);
+ if (UseHardFpABI()) {
+ get_double_from_d_register(0, &dval);
+ result = target(arg0, dval, arg1, arg2);
+ } else {
+ dval = get_double_from_register_pair(2);
+ result = target(arg0, dval, arg4, arg5);
+ }
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ set_register(r0, result);
+ break;
+ }
+ case Args_Double_DoubleDoubleDouble: {
+ double dval0, dval1, dval2;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ // the last argument is on stack
+ getFpFromStack(stack_pointer, &dval2);
+ Prototype_Double_DoubleDoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDoubleDouble>(external);
+ double dresult = target(dval0, dval1, dval2);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleDoubleDoubleDouble: {
+ double dval0, dval1, dval2, dval3;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ // the two last arguments are on stack
+ getFpFromStack(stack_pointer, &dval2);
+ getFpFromStack(stack_pointer + 2, &dval3);
+ Prototype_Double_DoubleDoubleDoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDoubleDoubleDouble>(
+ external);
+ double dresult = target(dval0, dval1, dval2, dval3);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResultDouble(dresult);
+ break;
+ }
+
+ case Args_Int32_General: {
+ Prototype_Int32_General target =
+ reinterpret_cast<Prototype_Int32_General>(external);
+ int64_t result = target(arg0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32: {
+ Prototype_Int32_GeneralInt32 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32>(external);
+ int64_t result = target(arg0, arg1);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32: {
+ Prototype_Int32_GeneralInt32Int32 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32>(external);
+ int64_t result = target(arg0, arg1, arg2);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32: {
+ Prototype_Int32_GeneralInt32Int32Int32Int32 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32Int32: {
+ Prototype_Int32_GeneralInt32Int32Int32Int32Int32 target =
+ reinterpret_cast<
+ Prototype_Int32_GeneralInt32Int32Int32Int32Int32>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32General: {
+ Prototype_Int32_GeneralInt32Int32Int32Int32General target =
+ reinterpret_cast<
+ Prototype_Int32_GeneralInt32Int32Int32Int32General>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32Int32Int32General: {
+ Prototype_Int32_GeneralInt32Int32Int32Int32Int32Int32General target =
+ reinterpret_cast<
+ Prototype_Int32_GeneralInt32Int32Int32Int32Int32Int32General>(
+ external);
+ int64_t result =
+ target(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Float32Float32Int32Int32Int32General: {
+ float fval0, fval1;
+ if (UseHardFpABI()) {
+ get_float_from_s_register(2, &fval0);
+ get_float_from_s_register(3, &fval1);
+ } else {
+ fval0 = mozilla::BitwiseCast<float>(arg2);
+ fval1 = mozilla::BitwiseCast<float>(arg3);
+ }
+ Prototype_Int32_GeneralInt32Float32Float32Int32Int32Int32General
+ target = reinterpret_cast<
+ Prototype_Int32_GeneralInt32Float32Float32Int32Int32Int32General>(
+ external);
+ int64_t result =
+ target(arg0, arg1, fval0, fval1, arg4, arg5, arg6, arg7);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General: {
+ float fval0, fval1, fval2, fval3;
+ if (UseHardFpABI()) {
+ get_float_from_s_register(2, &fval0);
+ get_float_from_s_register(3, &fval1);
+ get_float_from_s_register(4, &fval2);
+ get_float_from_s_register(5, &fval3);
+ } else {
+ fval0 = mozilla::BitwiseCast<float>(arg2);
+ fval1 = mozilla::BitwiseCast<float>(arg3);
+ fval2 = mozilla::BitwiseCast<float>(arg4);
+ fval3 = mozilla::BitwiseCast<float>(arg5);
+ }
+ Prototype_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General
+ target = reinterpret_cast<
+ Prototype_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General>(
+ external);
+ int64_t result = target(arg0, arg1, fval0, fval1, fval2, fval3, arg6,
+ arg7, arg8, arg9, arg10);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General: {
+ float fval0, fval1, fval2, fval3, fval4;
+ if (UseHardFpABI()) {
+ get_float_from_s_register(2, &fval0);
+ get_float_from_s_register(3, &fval1);
+ get_float_from_s_register(5, &fval2);
+ get_float_from_s_register(6, &fval3);
+ get_float_from_s_register(8, &fval4);
+ } else {
+ fval0 = mozilla::BitwiseCast<float>(arg2);
+ fval1 = mozilla::BitwiseCast<float>(arg3);
+ fval2 = mozilla::BitwiseCast<float>(arg5);
+ fval3 = mozilla::BitwiseCast<float>(arg6);
+ fval4 = mozilla::BitwiseCast<float>(arg8);
+ }
+ Prototype_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General
+ target = reinterpret_cast<
+ Prototype_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General>(
+ external);
+ int64_t result =
+ target(arg0, arg1, fval0, fval1, arg4, fval2, fval3, arg7, fval4,
+ arg9, arg10, arg11, arg12, arg13);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32General: {
+ Prototype_Int32_GeneralInt32Int32Int32General target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32General>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int64: {
+ Prototype_Int32_GeneralInt32Int32Int64 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int64>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, MakeInt64(arg3, arg4));
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32General: {
+ Prototype_Int32_GeneralInt32Int32General target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32General>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32Int64Int64: {
+ Prototype_Int32_GeneralInt32Int64Int64 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int64Int64>(
+ external);
+ int64_t result =
+ target(arg0, arg1, MakeInt64(arg2, arg3), MakeInt64(arg4, arg5));
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32GeneralInt32: {
+ Prototype_Int32_GeneralInt32GeneralInt32 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32GeneralInt32>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt32GeneralInt32Int32: {
+ Prototype_Int32_GeneralInt32GeneralInt32Int32 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt32GeneralInt32Int32>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralGeneral: {
+ Prototype_Int32_GeneralGeneral target =
+ reinterpret_cast<Prototype_Int32_GeneralGeneral>(external);
+ int64_t result = target(arg0, arg1);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralGeneralGeneral: {
+ Prototype_Int32_GeneralGeneralGeneral target =
+ reinterpret_cast<Prototype_Int32_GeneralGeneralGeneral>(external);
+ int64_t result = target(arg0, arg1, arg2);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralGeneralInt32Int32: {
+ Prototype_Int32_GeneralGeneralInt32Int32 target =
+ reinterpret_cast<Prototype_Int32_GeneralGeneralInt32Int32>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt64Int32Int32Int32: {
+ Prototype_Int32_GeneralInt64Int32Int32Int32 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int32Int32>(
+ external);
+ int64_t result =
+ target(arg0, MakeInt64(arg2, arg3), arg4, arg5, arg6);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt64Int32: {
+ Prototype_Int32_GeneralInt64Int32 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int32>(external);
+ int64_t result = target(arg0, MakeInt64(arg2, arg3), arg4);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt64Int32Int64: {
+ Prototype_Int32_GeneralInt64Int32Int64 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int64>(
+ external);
+ int64_t result =
+ target(arg0, MakeInt64(arg2, arg3), arg4, MakeInt64(arg6, arg7));
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt64Int32Int64General: {
+ Prototype_Int32_GeneralInt64Int32Int64General target =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int64General>(
+ external);
+ int64_t result = target(arg0, MakeInt64(arg2, arg3), arg4,
+ MakeInt64(arg6, arg7), arg8);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt64Int64Int64: {
+ Prototype_Int32_GeneralInt64Int64Int64 target =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int64Int64>(
+ external);
+ int64_t result = target(arg0, MakeInt64(arg2, arg3),
+ MakeInt64(arg4, arg5), MakeInt64(arg6, arg7));
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int32_GeneralInt64Int64Int64General: {
+ Prototype_Int32_GeneralInt64Int64Int64General target =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int64Int64General>(
+ external);
+ int64_t result =
+ target(arg0, MakeInt64(arg2, arg3), MakeInt64(arg4, arg5),
+ MakeInt64(arg6, arg7), arg8);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_General_GeneralInt32: {
+ Prototype_General_GeneralInt32 target =
+ reinterpret_cast<Prototype_General_GeneralInt32>(external);
+ int64_t result = target(arg0, arg1);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_General_GeneralInt32Int32: {
+ Prototype_General_GeneralInt32Int32 target =
+ reinterpret_cast<Prototype_General_GeneralInt32Int32>(external);
+ int64_t result = target(arg0, arg1, arg2);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_General_GeneralInt32General: {
+ Prototype_General_GeneralInt32General target =
+ reinterpret_cast<Prototype_General_GeneralInt32General>(external);
+ int64_t result = target(arg0, arg1, arg2);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case js::jit::Args_General_GeneralInt32Int32GeneralInt32: {
+ Prototype_General_GeneralInt32Int32GeneralInt32 target =
+ reinterpret_cast<Prototype_General_GeneralInt32Int32GeneralInt32>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralGeneralInt32GeneralInt32Int32Int32: {
+ Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32 target =
+ reinterpret_cast<
+ Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32>(
+ external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int64_General: {
+ Prototype_Int64_General target =
+ reinterpret_cast<Prototype_Int64_General>(external);
+ int64_t result = target(arg0);
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int64_GeneralInt64: {
+ Prototype_Int64_GeneralInt64 target =
+ reinterpret_cast<Prototype_Int64_GeneralInt64>(external);
+ int64_t result = target(arg0, MakeInt64(arg2, arg3));
+ scratchVolatileRegisters(/* scratchFloat = true */);
+ setCallResult(result);
+ break;
+ }
+
+ default:
+ MOZ_CRASH("call");
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ set_register(lr, saved_lr);
+ set_pc(get_register(lr));
+ break;
+ }
+ case kBreakpoint: {
+ ArmDebugger dbg(this);
+ dbg.debug();
+ break;
+ }
+ default: { // Stop uses all codes greater than 1 << 23.
+ if (svc >= (1 << 23)) {
+ uint32_t code = svc & kStopCodeMask;
+ if (isWatchedStop(code)) {
+ increaseStopCounter(code);
+ }
+
+ // Stop if it is enabled, otherwise go on jumping over the stop and
+ // the message address.
+ if (isEnabledStop(code)) {
+ ArmDebugger dbg(this);
+ dbg.stop(instr);
+ } else {
+ set_pc(get_pc() + 2 * SimInstruction::kInstrSize);
+ }
+ } else {
+ // This is not a valid svc code.
+ MOZ_CRASH();
+ break;
+ }
+ }
+ }
+}
+
+void Simulator::canonicalizeNaN(double* value) {
+ if (!wasm::CodeExists && !wasm::LookupCodeSegment(get_pc_as<void*>()) &&
+ FPSCR_default_NaN_mode_) {
+ *value = JS::CanonicalizeNaN(*value);
+ }
+}
+
+void Simulator::canonicalizeNaN(float* value) {
+ if (!wasm::CodeExists && !wasm::LookupCodeSegment(get_pc_as<void*>()) &&
+ FPSCR_default_NaN_mode_) {
+ *value = JS::CanonicalizeNaN(*value);
+ }
+}
+
+// Stop helper functions.
+bool Simulator::isStopInstruction(SimInstruction* instr) {
+ return (instr->bits(27, 24) == 0xF) && (instr->svcValue() >= kStopCode);
+}
+
+bool Simulator::isWatchedStop(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ return code < kNumOfWatchedStops;
+}
+
+bool Simulator::isEnabledStop(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ // Unwatched stops are always enabled.
+ return !isWatchedStop(code) ||
+ !(watched_stops_[code].count & kStopDisabledBit);
+}
+
+void Simulator::enableStop(uint32_t code) {
+ MOZ_ASSERT(isWatchedStop(code));
+ if (!isEnabledStop(code)) {
+ watched_stops_[code].count &= ~kStopDisabledBit;
+ }
+}
+
+void Simulator::disableStop(uint32_t code) {
+ MOZ_ASSERT(isWatchedStop(code));
+ if (isEnabledStop(code)) {
+ watched_stops_[code].count |= kStopDisabledBit;
+ }
+}
+
+void Simulator::increaseStopCounter(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ MOZ_ASSERT(isWatchedStop(code));
+ if ((watched_stops_[code].count & ~(1 << 31)) == 0x7fffffff) {
+ printf(
+ "Stop counter for code %i has overflowed.\n"
+ "Enabling this code and reseting the counter to 0.\n",
+ code);
+ watched_stops_[code].count = 0;
+ enableStop(code);
+ } else {
+ watched_stops_[code].count++;
+ }
+}
+
+// Print a stop status.
+void Simulator::printStopInfo(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ if (!isWatchedStop(code)) {
+ printf("Stop not watched.");
+ } else {
+ const char* state = isEnabledStop(code) ? "Enabled" : "Disabled";
+ int32_t count = watched_stops_[code].count & ~kStopDisabledBit;
+ // Don't print the state of unused breakpoints.
+ if (count != 0) {
+ if (watched_stops_[code].desc) {
+ printf("stop %i - 0x%x: \t%s, \tcounter = %i, \t%s\n", code, code,
+ state, count, watched_stops_[code].desc);
+ } else {
+ printf("stop %i - 0x%x: \t%s, \tcounter = %i\n", code, code, state,
+ count);
+ }
+ }
+ }
+}
+
+// Instruction types 0 and 1 are both rolled into one function because they only
+// differ in the handling of the shifter_operand.
+void Simulator::decodeType01(SimInstruction* instr) {
+ int type = instr->typeValue();
+ if (type == 0 && instr->isSpecialType0()) {
+ // Multiply instruction or extra loads and stores.
+ if (instr->bits(7, 4) == 9) {
+ if (instr->bit(24) == 0) {
+ // Raw field decoding here. Multiply instructions have their Rd
+ // in funny places.
+ int rn = instr->rnValue();
+ int rm = instr->rmValue();
+ int rs = instr->rsValue();
+ int32_t rs_val = get_register(rs);
+ int32_t rm_val = get_register(rm);
+ if (instr->bit(23) == 0) {
+ if (instr->bit(21) == 0) {
+ // The MUL instruction description (A 4.1.33) refers to
+ // Rd as being the destination for the operation, but it
+ // confusingly uses the Rn field to encode it.
+ int rd = rn; // Remap the rn field to the Rd register.
+ int32_t alu_out = rm_val * rs_val;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ }
+ } else {
+ int rd = instr->rdValue();
+ int32_t acc_value = get_register(rd);
+ if (instr->bit(22) == 0) {
+ // The MLA instruction description (A 4.1.28) refers
+ // to the order of registers as "Rd, Rm, Rs,
+ // Rn". But confusingly it uses the Rn field to
+ // encode the Rd register and the Rd field to encode
+ // the Rn register.
+ int32_t mul_out = rm_val * rs_val;
+ int32_t result = acc_value + mul_out;
+ set_register(rn, result);
+ } else {
+ int32_t mul_out = rm_val * rs_val;
+ int32_t result = acc_value - mul_out;
+ set_register(rn, result);
+ }
+ }
+ } else {
+ // The signed/long multiply instructions use the terms RdHi
+ // and RdLo when referring to the target registers. They are
+ // mapped to the Rn and Rd fields as follows:
+ // RdLo == Rd
+ // RdHi == Rn (This is confusingly stored in variable rd here
+ // because the mul instruction from above uses the
+ // Rn field to encode the Rd register. Good luck figuring
+ // this out without reading the ARM instruction manual
+ // at a very detailed level.)
+ int rd_hi = rn; // Remap the rn field to the RdHi register.
+ int rd_lo = instr->rdValue();
+ int32_t hi_res = 0;
+ int32_t lo_res = 0;
+ if (instr->bit(22) == 1) {
+ int64_t left_op = static_cast<int32_t>(rm_val);
+ int64_t right_op = static_cast<int32_t>(rs_val);
+ uint64_t result = left_op * right_op;
+ hi_res = static_cast<int32_t>(result >> 32);
+ lo_res = static_cast<int32_t>(result & 0xffffffff);
+ } else {
+ // Unsigned multiply.
+ uint64_t left_op = static_cast<uint32_t>(rm_val);
+ uint64_t right_op = static_cast<uint32_t>(rs_val);
+ uint64_t result = left_op * right_op;
+ hi_res = static_cast<int32_t>(result >> 32);
+ lo_res = static_cast<int32_t>(result & 0xffffffff);
+ }
+ set_register(rd_lo, lo_res);
+ set_register(rd_hi, hi_res);
+ if (instr->hasS()) {
+ MOZ_CRASH();
+ }
+ }
+ } else {
+ if (instr->bits(excl::ExclusiveOpHi, excl::ExclusiveOpLo) ==
+ excl::ExclusiveOpcode) {
+ // Load-exclusive / store-exclusive.
+ if (instr->bit(excl::ExclusiveLoad)) {
+ int rn = instr->rnValue();
+ int rt = instr->rtValue();
+ int32_t address = get_register(rn);
+ switch (instr->bits(excl::ExclusiveSizeHi, excl::ExclusiveSizeLo)) {
+ case excl::ExclusiveWord:
+ set_register(rt, readExW(address, instr));
+ break;
+ case excl::ExclusiveDouble: {
+ MOZ_ASSERT((rt % 2) == 0);
+ int32_t hibits;
+ int32_t lobits = readExDW(address, &hibits);
+ set_register(rt, lobits);
+ set_register(rt + 1, hibits);
+ break;
+ }
+ case excl::ExclusiveByte:
+ set_register(rt, readExBU(address));
+ break;
+ case excl::ExclusiveHalf:
+ set_register(rt, readExHU(address, instr));
+ break;
+ }
+ } else {
+ int rn = instr->rnValue();
+ int rd = instr->rdValue();
+ int rt = instr->bits(3, 0);
+ int32_t address = get_register(rn);
+ int32_t value = get_register(rt);
+ int32_t result = 0;
+ switch (instr->bits(excl::ExclusiveSizeHi, excl::ExclusiveSizeLo)) {
+ case excl::ExclusiveWord:
+ result = writeExW(address, value, instr);
+ break;
+ case excl::ExclusiveDouble: {
+ MOZ_ASSERT((rt % 2) == 0);
+ int32_t value2 = get_register(rt + 1);
+ result = writeExDW(address, value, value2);
+ break;
+ }
+ case excl::ExclusiveByte:
+ result = writeExB(address, (uint8_t)value);
+ break;
+ case excl::ExclusiveHalf:
+ result = writeExH(address, (uint16_t)value, instr);
+ break;
+ }
+ set_register(rd, result);
+ }
+ } else {
+ MOZ_CRASH(); // Not used atm
+ }
+ }
+ } else {
+ // Extra load/store instructions.
+ int rd = instr->rdValue();
+ int rn = instr->rnValue();
+ int32_t rn_val = get_register(rn);
+ int32_t addr = 0;
+ if (instr->bit(22) == 0) {
+ int rm = instr->rmValue();
+ int32_t rm_val = get_register(rm);
+ switch (instr->PUField()) {
+ case da_x:
+ MOZ_ASSERT(!instr->hasW());
+ addr = rn_val;
+ rn_val -= rm_val;
+ set_register(rn, rn_val);
+ break;
+ case ia_x:
+ MOZ_ASSERT(!instr->hasW());
+ addr = rn_val;
+ rn_val += rm_val;
+ set_register(rn, rn_val);
+ break;
+ case db_x:
+ rn_val -= rm_val;
+ addr = rn_val;
+ if (instr->hasW()) {
+ set_register(rn, rn_val);
+ }
+ break;
+ case ib_x:
+ rn_val += rm_val;
+ addr = rn_val;
+ if (instr->hasW()) {
+ set_register(rn, rn_val);
+ }
+ break;
+ default:
+ // The PU field is a 2-bit field.
+ MOZ_CRASH();
+ break;
+ }
+ } else {
+ int32_t imm_val = (instr->immedHValue() << 4) | instr->immedLValue();
+ switch (instr->PUField()) {
+ case da_x:
+ MOZ_ASSERT(!instr->hasW());
+ addr = rn_val;
+ rn_val -= imm_val;
+ set_register(rn, rn_val);
+ break;
+ case ia_x:
+ MOZ_ASSERT(!instr->hasW());
+ addr = rn_val;
+ rn_val += imm_val;
+ set_register(rn, rn_val);
+ break;
+ case db_x:
+ rn_val -= imm_val;
+ addr = rn_val;
+ if (instr->hasW()) {
+ set_register(rn, rn_val);
+ }
+ break;
+ case ib_x:
+ rn_val += imm_val;
+ addr = rn_val;
+ if (instr->hasW()) {
+ set_register(rn, rn_val);
+ }
+ break;
+ default:
+ // The PU field is a 2-bit field.
+ MOZ_CRASH();
+ break;
+ }
+ }
+ if ((instr->bits(7, 4) & 0xd) == 0xd && instr->bit(20) == 0) {
+ MOZ_ASSERT((rd % 2) == 0);
+ if (instr->hasH()) {
+ // The strd instruction.
+ int32_t value1 = get_register(rd);
+ int32_t value2 = get_register(rd + 1);
+ writeDW(addr, value1, value2);
+ } else {
+ // The ldrd instruction.
+ int* rn_data = readDW(addr);
+ if (rn_data) {
+ set_dw_register(rd, rn_data);
+ }
+ }
+ } else if (instr->hasH()) {
+ if (instr->hasSign()) {
+ if (instr->hasL()) {
+ int16_t val = readH(addr, instr);
+ set_register(rd, val);
+ } else {
+ int16_t val = get_register(rd);
+ writeH(addr, val, instr);
+ }
+ } else {
+ if (instr->hasL()) {
+ uint16_t val = readHU(addr, instr);
+ set_register(rd, val);
+ } else {
+ uint16_t val = get_register(rd);
+ writeH(addr, val, instr);
+ }
+ }
+ } else {
+ // Signed byte loads.
+ MOZ_ASSERT(instr->hasSign());
+ MOZ_ASSERT(instr->hasL());
+ int8_t val = readB(addr);
+ set_register(rd, val);
+ }
+ return;
+ }
+ } else if ((type == 0) && instr->isMiscType0()) {
+ if (instr->bits(7, 4) == 0) {
+ if (instr->bit(21) == 0) {
+ // mrs
+ int rd = instr->rdValue();
+ uint32_t flags;
+ if (instr->bit(22) == 0) {
+ // CPSR. Note: The Q flag is not yet implemented!
+ flags = (n_flag_ << 31) | (z_flag_ << 30) | (c_flag_ << 29) |
+ (v_flag_ << 28);
+ } else {
+ // SPSR
+ MOZ_CRASH();
+ }
+ set_register(rd, flags);
+ } else {
+ // msr
+ if (instr->bits(27, 23) == 2) {
+ // Register operand. For now we only emit mask 0b1100.
+ int rm = instr->rmValue();
+ mozilla::DebugOnly<uint32_t> mask = instr->bits(19, 16);
+ MOZ_ASSERT(mask == (3 << 2));
+
+ uint32_t flags = get_register(rm);
+ n_flag_ = (flags >> 31) & 1;
+ z_flag_ = (flags >> 30) & 1;
+ c_flag_ = (flags >> 29) & 1;
+ v_flag_ = (flags >> 28) & 1;
+ } else {
+ MOZ_CRASH();
+ }
+ }
+ } else if (instr->bits(22, 21) == 1) {
+ int rm = instr->rmValue();
+ switch (instr->bits(7, 4)) {
+ case 1: // BX
+ set_pc(get_register(rm));
+ break;
+ case 3: { // BLX
+ uint32_t old_pc = get_pc();
+ set_pc(get_register(rm));
+ set_register(lr, old_pc + SimInstruction::kInstrSize);
+ break;
+ }
+ case 7: { // BKPT
+ fprintf(stderr, "Simulator hit BKPT.\n");
+ if (getenv("ARM_SIM_DEBUGGER")) {
+ ArmDebugger dbg(this);
+ dbg.debug();
+ } else {
+ fprintf(stderr,
+ "Use ARM_SIM_DEBUGGER=1 to enter the builtin debugger.\n");
+ MOZ_CRASH("ARM simulator breakpoint");
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH();
+ }
+ } else if (instr->bits(22, 21) == 3) {
+ int rm = instr->rmValue();
+ int rd = instr->rdValue();
+ switch (instr->bits(7, 4)) {
+ case 1: { // CLZ
+ uint32_t bits = get_register(rm);
+ int leading_zeros = 0;
+ if (bits == 0) {
+ leading_zeros = 32;
+ } else {
+ leading_zeros = mozilla::CountLeadingZeroes32(bits);
+ }
+ set_register(rd, leading_zeros);
+ break;
+ }
+ default:
+ MOZ_CRASH();
+ break;
+ }
+ } else {
+ printf("%08x\n", instr->instructionBits());
+ MOZ_CRASH();
+ }
+ } else if ((type == 1) && instr->isNopType1()) {
+ // NOP.
+ } else if ((type == 1) && instr->isCsdbType1()) {
+ // Speculation barrier. (No-op for the simulator)
+ } else {
+ int rd = instr->rdValue();
+ int rn = instr->rnValue();
+ int32_t rn_val = get_register(rn);
+ int32_t shifter_operand = 0;
+ bool shifter_carry_out = 0;
+ if (type == 0) {
+ shifter_operand = getShiftRm(instr, &shifter_carry_out);
+ } else {
+ MOZ_ASSERT(instr->typeValue() == 1);
+ shifter_operand = getImm(instr, &shifter_carry_out);
+ }
+ int32_t alu_out;
+ switch (instr->opcodeField()) {
+ case OpAnd:
+ alu_out = rn_val & shifter_operand;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(shifter_carry_out);
+ }
+ break;
+ case OpEor:
+ alu_out = rn_val ^ shifter_operand;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(shifter_carry_out);
+ }
+ break;
+ case OpSub:
+ alu_out = rn_val - shifter_operand;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(!borrowFrom(rn_val, shifter_operand));
+ setVFlag(overflowFrom(alu_out, rn_val, shifter_operand, false));
+ }
+ break;
+ case OpRsb:
+ alu_out = shifter_operand - rn_val;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(!borrowFrom(shifter_operand, rn_val));
+ setVFlag(overflowFrom(alu_out, shifter_operand, rn_val, false));
+ }
+ break;
+ case OpAdd:
+ alu_out = rn_val + shifter_operand;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(carryFrom(rn_val, shifter_operand));
+ setVFlag(overflowFrom(alu_out, rn_val, shifter_operand, true));
+ }
+ break;
+ case OpAdc:
+ alu_out = rn_val + shifter_operand + getCarry();
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(carryFrom(rn_val, shifter_operand, getCarry()));
+ setVFlag(overflowFrom(alu_out, rn_val, shifter_operand, true));
+ }
+ break;
+ case OpSbc:
+ alu_out = rn_val - shifter_operand - (getCarry() == 0 ? 1 : 0);
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ MOZ_CRASH();
+ }
+ break;
+ case OpRsc:
+ alu_out = shifter_operand - rn_val - (getCarry() == 0 ? 1 : 0);
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ MOZ_CRASH();
+ }
+ break;
+ case OpTst:
+ if (instr->hasS()) {
+ alu_out = rn_val & shifter_operand;
+ setNZFlags(alu_out);
+ setCFlag(shifter_carry_out);
+ } else {
+ alu_out = instr->immedMovwMovtValue();
+ set_register(rd, alu_out);
+ }
+ break;
+ case OpTeq:
+ if (instr->hasS()) {
+ alu_out = rn_val ^ shifter_operand;
+ setNZFlags(alu_out);
+ setCFlag(shifter_carry_out);
+ } else {
+ // Other instructions matching this pattern are handled in the
+ // miscellaneous instructions part above.
+ MOZ_CRASH();
+ }
+ break;
+ case OpCmp:
+ if (instr->hasS()) {
+ alu_out = rn_val - shifter_operand;
+ setNZFlags(alu_out);
+ setCFlag(!borrowFrom(rn_val, shifter_operand));
+ setVFlag(overflowFrom(alu_out, rn_val, shifter_operand, false));
+ } else {
+ alu_out =
+ (get_register(rd) & 0xffff) | (instr->immedMovwMovtValue() << 16);
+ set_register(rd, alu_out);
+ }
+ break;
+ case OpCmn:
+ if (instr->hasS()) {
+ alu_out = rn_val + shifter_operand;
+ setNZFlags(alu_out);
+ setCFlag(carryFrom(rn_val, shifter_operand));
+ setVFlag(overflowFrom(alu_out, rn_val, shifter_operand, true));
+ } else {
+ // Other instructions matching this pattern are handled in the
+ // miscellaneous instructions part above.
+ MOZ_CRASH();
+ }
+ break;
+ case OpOrr:
+ alu_out = rn_val | shifter_operand;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(shifter_carry_out);
+ }
+ break;
+ case OpMov:
+ alu_out = shifter_operand;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(shifter_carry_out);
+ }
+ break;
+ case OpBic:
+ alu_out = rn_val & ~shifter_operand;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(shifter_carry_out);
+ }
+ break;
+ case OpMvn:
+ alu_out = ~shifter_operand;
+ set_register(rd, alu_out);
+ if (instr->hasS()) {
+ setNZFlags(alu_out);
+ setCFlag(shifter_carry_out);
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ break;
+ }
+ }
+}
+
+void Simulator::decodeType2(SimInstruction* instr) {
+ int rd = instr->rdValue();
+ int rn = instr->rnValue();
+ int32_t rn_val = get_register(rn);
+ int32_t im_val = instr->offset12Value();
+ int32_t addr = 0;
+ switch (instr->PUField()) {
+ case da_x:
+ MOZ_ASSERT(!instr->hasW());
+ addr = rn_val;
+ rn_val -= im_val;
+ set_register(rn, rn_val);
+ break;
+ case ia_x:
+ MOZ_ASSERT(!instr->hasW());
+ addr = rn_val;
+ rn_val += im_val;
+ set_register(rn, rn_val);
+ break;
+ case db_x:
+ rn_val -= im_val;
+ addr = rn_val;
+ if (instr->hasW()) {
+ set_register(rn, rn_val);
+ }
+ break;
+ case ib_x:
+ rn_val += im_val;
+ addr = rn_val;
+ if (instr->hasW()) {
+ set_register(rn, rn_val);
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ break;
+ }
+ if (instr->hasB()) {
+ if (instr->hasL()) {
+ uint8_t val = readBU(addr);
+ set_register(rd, val);
+ } else {
+ uint8_t val = get_register(rd);
+ writeB(addr, val);
+ }
+ } else {
+ if (instr->hasL()) {
+ set_register(rd, readW(addr, instr, AllowUnaligned));
+ } else {
+ writeW(addr, get_register(rd), instr, AllowUnaligned);
+ }
+ }
+}
+
+static uint32_t rotateBytes(uint32_t val, int32_t rotate) {
+ switch (rotate) {
+ default:
+ return val;
+ case 1:
+ return (val >> 8) | (val << 24);
+ case 2:
+ return (val >> 16) | (val << 16);
+ case 3:
+ return (val >> 24) | (val << 8);
+ }
+}
+
+void Simulator::decodeType3(SimInstruction* instr) {
+ if (MOZ_UNLIKELY(instr->isUDF())) {
+ uint8_t* newPC;
+ if (wasm::HandleIllegalInstruction(registerState(), &newPC)) {
+ set_pc((int32_t)newPC);
+ return;
+ }
+ MOZ_CRASH("illegal instruction encountered");
+ }
+
+ int rd = instr->rdValue();
+ int rn = instr->rnValue();
+ int32_t rn_val = get_register(rn);
+ bool shifter_carry_out = 0;
+ int32_t shifter_operand = getShiftRm(instr, &shifter_carry_out);
+ int32_t addr = 0;
+ switch (instr->PUField()) {
+ case da_x:
+ MOZ_ASSERT(!instr->hasW());
+ MOZ_CRASH();
+ break;
+ case ia_x: {
+ if (instr->bit(4) == 0) {
+ // Memop.
+ } else {
+ if (instr->bit(5) == 0) {
+ switch (instr->bits(22, 21)) {
+ case 0:
+ if (instr->bit(20) == 0) {
+ if (instr->bit(6) == 0) {
+ // Pkhbt.
+ uint32_t rn_val = get_register(rn);
+ uint32_t rm_val = get_register(instr->rmValue());
+ int32_t shift = instr->bits(11, 7);
+ rm_val <<= shift;
+ set_register(rd, (rn_val & 0xFFFF) | (rm_val & 0xFFFF0000U));
+ } else {
+ // Pkhtb.
+ uint32_t rn_val = get_register(rn);
+ int32_t rm_val = get_register(instr->rmValue());
+ int32_t shift = instr->bits(11, 7);
+ if (shift == 0) {
+ shift = 32;
+ }
+ rm_val >>= shift;
+ set_register(rd, (rn_val & 0xFFFF0000U) | (rm_val & 0xFFFF));
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 1:
+ MOZ_CRASH();
+ break;
+ case 2:
+ MOZ_CRASH();
+ break;
+ case 3: {
+ // Usat.
+ int32_t sat_pos = instr->bits(20, 16);
+ int32_t sat_val = (1 << sat_pos) - 1;
+ int32_t shift = instr->bits(11, 7);
+ int32_t shift_type = instr->bit(6);
+ int32_t rm_val = get_register(instr->rmValue());
+ if (shift_type == 0) { // LSL
+ rm_val <<= shift;
+ } else { // ASR
+ rm_val >>= shift;
+ }
+
+ // If saturation occurs, the Q flag should be set in the
+ // CPSR. There is no Q flag yet, and no instruction (MRS)
+ // to read the CPSR directly.
+ if (rm_val > sat_val) {
+ rm_val = sat_val;
+ } else if (rm_val < 0) {
+ rm_val = 0;
+ }
+ set_register(rd, rm_val);
+ break;
+ }
+ }
+ } else {
+ switch (instr->bits(22, 21)) {
+ case 0:
+ MOZ_CRASH();
+ break;
+ case 1:
+ if (instr->bits(7, 4) == 7 && instr->bits(19, 16) == 15) {
+ uint32_t rm_val = rotateBytes(get_register(instr->rmValue()),
+ instr->bits(11, 10));
+ if (instr->bit(20)) {
+ // Sxth.
+ set_register(rd, (int32_t)(int16_t)(rm_val & 0xFFFF));
+ } else {
+ // Sxtb.
+ set_register(rd, (int32_t)(int8_t)(rm_val & 0xFF));
+ }
+ } else if (instr->bits(20, 16) == 0b1'1111 &&
+ instr->bits(11, 4) == 0b1111'0011) {
+ // Rev
+ uint32_t rm_val = get_register(instr->rmValue());
+
+ static_assert(MOZ_LITTLE_ENDIAN());
+ set_register(rd,
+ mozilla::NativeEndian::swapToBigEndian(rm_val));
+ } else if (instr->bits(20, 16) == 0b1'1111 &&
+ instr->bits(11, 4) == 0b1111'1011) {
+ // Rev16
+ uint32_t rm_val = get_register(instr->rmValue());
+
+ static_assert(MOZ_LITTLE_ENDIAN());
+ uint32_t hi = mozilla::NativeEndian::swapToBigEndian(
+ uint16_t(rm_val >> 16));
+ uint32_t lo =
+ mozilla::NativeEndian::swapToBigEndian(uint16_t(rm_val));
+ set_register(rd, (hi << 16) | lo);
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 2:
+ if ((instr->bit(20) == 0) && (instr->bits(9, 6) == 1)) {
+ if (instr->bits(19, 16) == 0xF) {
+ // Uxtb16.
+ uint32_t rm_val = rotateBytes(get_register(instr->rmValue()),
+ instr->bits(11, 10));
+ set_register(rd, (rm_val & 0xFF) | (rm_val & 0xFF0000));
+ } else {
+ MOZ_CRASH();
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 3:
+ if ((instr->bit(20) == 0) && (instr->bits(9, 6) == 1)) {
+ if (instr->bits(19, 16) == 0xF) {
+ // Uxtb.
+ uint32_t rm_val = rotateBytes(get_register(instr->rmValue()),
+ instr->bits(11, 10));
+ set_register(rd, (rm_val & 0xFF));
+ } else {
+ // Uxtab.
+ uint32_t rn_val = get_register(rn);
+ uint32_t rm_val = rotateBytes(get_register(instr->rmValue()),
+ instr->bits(11, 10));
+ set_register(rd, rn_val + (rm_val & 0xFF));
+ }
+ } else if ((instr->bit(20) == 1) && (instr->bits(9, 6) == 1)) {
+ if (instr->bits(19, 16) == 0xF) {
+ // Uxth.
+ uint32_t rm_val = rotateBytes(get_register(instr->rmValue()),
+ instr->bits(11, 10));
+ set_register(rd, (rm_val & 0xFFFF));
+ } else {
+ // Uxtah.
+ uint32_t rn_val = get_register(rn);
+ uint32_t rm_val = rotateBytes(get_register(instr->rmValue()),
+ instr->bits(11, 10));
+ set_register(rd, rn_val + (rm_val & 0xFFFF));
+ }
+ } else if (instr->bits(20, 16) == 0b1'1111 &&
+ instr->bits(11, 4) == 0b1111'1011) {
+ // Revsh
+ uint32_t rm_val = get_register(instr->rmValue());
+
+ static_assert(MOZ_LITTLE_ENDIAN());
+ set_register(
+ rd, int32_t(int16_t(mozilla::NativeEndian::swapToBigEndian(
+ uint16_t(rm_val)))));
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ }
+ }
+ return;
+ }
+ break;
+ }
+ case db_x: { // sudiv
+ if (instr->bit(22) == 0x0 && instr->bit(20) == 0x1 &&
+ instr->bits(15, 12) == 0x0f && instr->bits(7, 4) == 0x1) {
+ if (!instr->hasW()) {
+ // sdiv (in V8 notation matching ARM ISA format) rn = rm/rs.
+ int rm = instr->rmValue();
+ int32_t rm_val = get_register(rm);
+ int rs = instr->rsValue();
+ int32_t rs_val = get_register(rs);
+ int32_t ret_val = 0;
+ MOZ_ASSERT(rs_val != 0);
+ if ((rm_val == INT32_MIN) && (rs_val == -1)) {
+ ret_val = INT32_MIN;
+ } else {
+ ret_val = rm_val / rs_val;
+ }
+ set_register(rn, ret_val);
+ return;
+ } else {
+ // udiv (in V8 notation matching ARM ISA format) rn = rm/rs.
+ int rm = instr->rmValue();
+ uint32_t rm_val = get_register(rm);
+ int rs = instr->rsValue();
+ uint32_t rs_val = get_register(rs);
+ uint32_t ret_val = 0;
+ MOZ_ASSERT(rs_val != 0);
+ ret_val = rm_val / rs_val;
+ set_register(rn, ret_val);
+ return;
+ }
+ }
+
+ addr = rn_val - shifter_operand;
+ if (instr->hasW()) {
+ set_register(rn, addr);
+ }
+ break;
+ }
+ case ib_x: {
+ if (instr->hasW() && (instr->bits(6, 4) == 0x5)) {
+ uint32_t widthminus1 = static_cast<uint32_t>(instr->bits(20, 16));
+ uint32_t lsbit = static_cast<uint32_t>(instr->bits(11, 7));
+ uint32_t msbit = widthminus1 + lsbit;
+ if (msbit <= 31) {
+ if (instr->bit(22)) {
+ // ubfx - unsigned bitfield extract.
+ uint32_t rm_val =
+ static_cast<uint32_t>(get_register(instr->rmValue()));
+ uint32_t extr_val = rm_val << (31 - msbit);
+ extr_val = extr_val >> (31 - widthminus1);
+ set_register(instr->rdValue(), extr_val);
+ } else {
+ // sbfx - signed bitfield extract.
+ int32_t rm_val = get_register(instr->rmValue());
+ int32_t extr_val = rm_val << (31 - msbit);
+ extr_val = extr_val >> (31 - widthminus1);
+ set_register(instr->rdValue(), extr_val);
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ return;
+ } else if (!instr->hasW() && (instr->bits(6, 4) == 0x1)) {
+ uint32_t lsbit = static_cast<uint32_t>(instr->bits(11, 7));
+ uint32_t msbit = static_cast<uint32_t>(instr->bits(20, 16));
+ if (msbit >= lsbit) {
+ // bfc or bfi - bitfield clear/insert.
+ uint32_t rd_val =
+ static_cast<uint32_t>(get_register(instr->rdValue()));
+ uint32_t bitcount = msbit - lsbit + 1;
+ uint32_t mask = (1 << bitcount) - 1;
+ rd_val &= ~(mask << lsbit);
+ if (instr->rmValue() != 15) {
+ // bfi - bitfield insert.
+ uint32_t rm_val =
+ static_cast<uint32_t>(get_register(instr->rmValue()));
+ rm_val &= mask;
+ rd_val |= rm_val << lsbit;
+ }
+ set_register(instr->rdValue(), rd_val);
+ } else {
+ MOZ_CRASH();
+ }
+ return;
+ } else {
+ addr = rn_val + shifter_operand;
+ if (instr->hasW()) {
+ set_register(rn, addr);
+ }
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH();
+ break;
+ }
+ if (instr->hasB()) {
+ if (instr->hasL()) {
+ uint8_t byte = readB(addr);
+ set_register(rd, byte);
+ } else {
+ uint8_t byte = get_register(rd);
+ writeB(addr, byte);
+ }
+ } else {
+ if (instr->hasL()) {
+ set_register(rd, readW(addr, instr, AllowUnaligned));
+ } else {
+ writeW(addr, get_register(rd), instr, AllowUnaligned);
+ }
+ }
+}
+
+void Simulator::decodeType4(SimInstruction* instr) {
+ // Only allowed to be set in privileged mode.
+ MOZ_ASSERT(instr->bit(22) == 0);
+ bool load = instr->hasL();
+ handleRList(instr, load);
+}
+
+void Simulator::decodeType5(SimInstruction* instr) {
+ int off = instr->sImmed24Value() << 2;
+ intptr_t pc_address = get_pc();
+ if (instr->hasLink()) {
+ set_register(lr, pc_address + SimInstruction::kInstrSize);
+ }
+ int pc_reg = get_register(pc);
+ set_pc(pc_reg + off);
+}
+
+void Simulator::decodeType6(SimInstruction* instr) {
+ decodeType6CoprocessorIns(instr);
+}
+
+void Simulator::decodeType7(SimInstruction* instr) {
+ if (instr->bit(24) == 1) {
+ softwareInterrupt(instr);
+ } else if (instr->bit(4) == 1 && instr->bits(11, 9) != 5) {
+ decodeType7CoprocessorIns(instr);
+ } else {
+ decodeTypeVFP(instr);
+ }
+}
+
+void Simulator::decodeType7CoprocessorIns(SimInstruction* instr) {
+ if (instr->bit(20) == 0) {
+ // MCR, MCR2
+ if (instr->coprocessorValue() == 15) {
+ int opc1 = instr->bits(23, 21);
+ int opc2 = instr->bits(7, 5);
+ int CRn = instr->bits(19, 16);
+ int CRm = instr->bits(3, 0);
+ if (opc1 == 0 && opc2 == 4 && CRn == 7 && CRm == 10) {
+ // ARMv6 DSB instruction. We do not use DSB.
+ MOZ_CRASH("DSB not implemented");
+ } else if (opc1 == 0 && opc2 == 5 && CRn == 7 && CRm == 10) {
+ // ARMv6 DMB instruction.
+ AtomicOperations::fenceSeqCst();
+ } else if (opc1 == 0 && opc2 == 4 && CRn == 7 && CRm == 5) {
+ // ARMv6 ISB instruction. We do not use ISB.
+ MOZ_CRASH("ISB not implemented");
+ } else {
+ MOZ_CRASH();
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ } else {
+ // MRC, MRC2
+ MOZ_CRASH();
+ }
+}
+
+void Simulator::decodeTypeVFP(SimInstruction* instr) {
+ MOZ_ASSERT(instr->typeValue() == 7 && instr->bit(24) == 0);
+ MOZ_ASSERT(instr->bits(11, 9) == 0x5);
+
+ // Obtain double precision register codes.
+ VFPRegPrecision precision =
+ (instr->szValue() == 1) ? kDoublePrecision : kSinglePrecision;
+ int vm = instr->VFPMRegValue(precision);
+ int vd = instr->VFPDRegValue(precision);
+ int vn = instr->VFPNRegValue(precision);
+
+ if (instr->bit(4) == 0) {
+ if (instr->opc1Value() == 0x7) {
+ // Other data processing instructions.
+ if ((instr->opc2Value() == 0x0) && (instr->opc3Value() == 0x1)) {
+ // vmov register to register.
+ if (instr->szValue() == 0x1) {
+ int m = instr->VFPMRegValue(kDoublePrecision);
+ int d = instr->VFPDRegValue(kDoublePrecision);
+ double temp;
+ get_double_from_d_register(m, &temp);
+ set_d_register_from_double(d, temp);
+ } else {
+ int m = instr->VFPMRegValue(kSinglePrecision);
+ int d = instr->VFPDRegValue(kSinglePrecision);
+ float temp;
+ get_float_from_s_register(m, &temp);
+ set_s_register_from_float(d, temp);
+ }
+ } else if ((instr->opc2Value() == 0x0) && (instr->opc3Value() == 0x3)) {
+ // vabs
+ if (instr->szValue() == 0x1) {
+ union {
+ double f64;
+ uint64_t u64;
+ } u;
+ get_double_from_d_register(vm, &u.f64);
+ u.u64 &= 0x7fffffffffffffffu;
+ double dd_value = u.f64;
+ canonicalizeNaN(&dd_value);
+ set_d_register_from_double(vd, dd_value);
+ } else {
+ union {
+ float f32;
+ uint32_t u32;
+ } u;
+ get_float_from_s_register(vm, &u.f32);
+ u.u32 &= 0x7fffffffu;
+ float fd_value = u.f32;
+ canonicalizeNaN(&fd_value);
+ set_s_register_from_float(vd, fd_value);
+ }
+ } else if ((instr->opc2Value() == 0x1) && (instr->opc3Value() == 0x1)) {
+ // vneg
+ if (instr->szValue() == 0x1) {
+ double dm_value;
+ get_double_from_d_register(vm, &dm_value);
+ double dd_value = -dm_value;
+ canonicalizeNaN(&dd_value);
+ set_d_register_from_double(vd, dd_value);
+ } else {
+ float fm_value;
+ get_float_from_s_register(vm, &fm_value);
+ float fd_value = -fm_value;
+ canonicalizeNaN(&fd_value);
+ set_s_register_from_float(vd, fd_value);
+ }
+ } else if ((instr->opc2Value() == 0x7) && (instr->opc3Value() == 0x3)) {
+ decodeVCVTBetweenDoubleAndSingle(instr);
+ } else if ((instr->opc2Value() == 0x8) && (instr->opc3Value() & 0x1)) {
+ decodeVCVTBetweenFloatingPointAndInteger(instr);
+ } else if ((instr->opc2Value() == 0xA) && (instr->opc3Value() == 0x3) &&
+ (instr->bit(8) == 1)) {
+ // vcvt.f64.s32 Dd, Dd, #<fbits>.
+ int fraction_bits = 32 - ((instr->bits(3, 0) << 1) | instr->bit(5));
+ int fixed_value = get_sinteger_from_s_register(vd * 2);
+ double divide = 1 << fraction_bits;
+ set_d_register_from_double(vd, fixed_value / divide);
+ } else if (((instr->opc2Value() >> 1) == 0x6) &&
+ (instr->opc3Value() & 0x1)) {
+ decodeVCVTBetweenFloatingPointAndInteger(instr);
+ } else if (((instr->opc2Value() == 0x4) || (instr->opc2Value() == 0x5)) &&
+ (instr->opc3Value() & 0x1)) {
+ decodeVCMP(instr);
+ } else if (((instr->opc2Value() == 0x1)) && (instr->opc3Value() == 0x3)) {
+ // vsqrt
+ if (instr->szValue() == 0x1) {
+ double dm_value;
+ get_double_from_d_register(vm, &dm_value);
+ double dd_value = std::sqrt(dm_value);
+ canonicalizeNaN(&dd_value);
+ set_d_register_from_double(vd, dd_value);
+ } else {
+ float fm_value;
+ get_float_from_s_register(vm, &fm_value);
+ float fd_value = std::sqrt(fm_value);
+ canonicalizeNaN(&fd_value);
+ set_s_register_from_float(vd, fd_value);
+ }
+ } else if (instr->opc3Value() == 0x0) {
+ // vmov immediate.
+ if (instr->szValue() == 0x1) {
+ set_d_register_from_double(vd, instr->doubleImmedVmov());
+ } else {
+ // vmov.f32 immediate.
+ set_s_register_from_float(vd, instr->float32ImmedVmov());
+ }
+ } else {
+ decodeVCVTBetweenFloatingPointAndIntegerFrac(instr);
+ }
+ } else if (instr->opc1Value() == 0x3) {
+ if (instr->szValue() != 0x1) {
+ if (instr->opc3Value() & 0x1) {
+ // vsub
+ float fn_value;
+ get_float_from_s_register(vn, &fn_value);
+ float fm_value;
+ get_float_from_s_register(vm, &fm_value);
+ float fd_value = fn_value - fm_value;
+ canonicalizeNaN(&fd_value);
+ set_s_register_from_float(vd, fd_value);
+ } else {
+ // vadd
+ float fn_value;
+ get_float_from_s_register(vn, &fn_value);
+ float fm_value;
+ get_float_from_s_register(vm, &fm_value);
+ float fd_value = fn_value + fm_value;
+ canonicalizeNaN(&fd_value);
+ set_s_register_from_float(vd, fd_value);
+ }
+ } else {
+ if (instr->opc3Value() & 0x1) {
+ // vsub
+ double dn_value;
+ get_double_from_d_register(vn, &dn_value);
+ double dm_value;
+ get_double_from_d_register(vm, &dm_value);
+ double dd_value = dn_value - dm_value;
+ canonicalizeNaN(&dd_value);
+ set_d_register_from_double(vd, dd_value);
+ } else {
+ // vadd
+ double dn_value;
+ get_double_from_d_register(vn, &dn_value);
+ double dm_value;
+ get_double_from_d_register(vm, &dm_value);
+ double dd_value = dn_value + dm_value;
+ canonicalizeNaN(&dd_value);
+ set_d_register_from_double(vd, dd_value);
+ }
+ }
+ } else if ((instr->opc1Value() == 0x2) && !(instr->opc3Value() & 0x1)) {
+ // vmul
+ if (instr->szValue() != 0x1) {
+ float fn_value;
+ get_float_from_s_register(vn, &fn_value);
+ float fm_value;
+ get_float_from_s_register(vm, &fm_value);
+ float fd_value = fn_value * fm_value;
+ canonicalizeNaN(&fd_value);
+ set_s_register_from_float(vd, fd_value);
+ } else {
+ double dn_value;
+ get_double_from_d_register(vn, &dn_value);
+ double dm_value;
+ get_double_from_d_register(vm, &dm_value);
+ double dd_value = dn_value * dm_value;
+ canonicalizeNaN(&dd_value);
+ set_d_register_from_double(vd, dd_value);
+ }
+ } else if ((instr->opc1Value() == 0x0)) {
+ // vmla, vmls
+ const bool is_vmls = (instr->opc3Value() & 0x1);
+
+ if (instr->szValue() != 0x1) {
+ MOZ_CRASH("Not used by V8.");
+ }
+
+ double dd_val;
+ get_double_from_d_register(vd, &dd_val);
+ double dn_val;
+ get_double_from_d_register(vn, &dn_val);
+ double dm_val;
+ get_double_from_d_register(vm, &dm_val);
+
+ // Note: we do the mul and add/sub in separate steps to avoid
+ // getting a result with too high precision.
+ set_d_register_from_double(vd, dn_val * dm_val);
+ double temp;
+ get_double_from_d_register(vd, &temp);
+ if (is_vmls) {
+ temp = dd_val - temp;
+ } else {
+ temp = dd_val + temp;
+ }
+ canonicalizeNaN(&temp);
+ set_d_register_from_double(vd, temp);
+ } else if ((instr->opc1Value() == 0x4) && !(instr->opc3Value() & 0x1)) {
+ // vdiv
+ if (instr->szValue() != 0x1) {
+ float fn_value;
+ get_float_from_s_register(vn, &fn_value);
+ float fm_value;
+ get_float_from_s_register(vm, &fm_value);
+ float fd_value = fn_value / fm_value;
+ div_zero_vfp_flag_ = (fm_value == 0);
+ canonicalizeNaN(&fd_value);
+ set_s_register_from_float(vd, fd_value);
+ } else {
+ double dn_value;
+ get_double_from_d_register(vn, &dn_value);
+ double dm_value;
+ get_double_from_d_register(vm, &dm_value);
+ double dd_value = dn_value / dm_value;
+ div_zero_vfp_flag_ = (dm_value == 0);
+ canonicalizeNaN(&dd_value);
+ set_d_register_from_double(vd, dd_value);
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ } else {
+ if (instr->VCValue() == 0x0 && instr->VAValue() == 0x0) {
+ decodeVMOVBetweenCoreAndSinglePrecisionRegisters(instr);
+ } else if ((instr->VLValue() == 0x0) && (instr->VCValue() == 0x1) &&
+ (instr->bit(23) == 0x0)) {
+ // vmov (ARM core register to scalar).
+ int vd = instr->bits(19, 16) | (instr->bit(7) << 4);
+ double dd_value;
+ get_double_from_d_register(vd, &dd_value);
+ int32_t data[2];
+ memcpy(data, &dd_value, 8);
+ data[instr->bit(21)] = get_register(instr->rtValue());
+ memcpy(&dd_value, data, 8);
+ set_d_register_from_double(vd, dd_value);
+ } else if ((instr->VLValue() == 0x1) && (instr->VCValue() == 0x1) &&
+ (instr->bit(23) == 0x0)) {
+ // vmov (scalar to ARM core register).
+ int vn = instr->bits(19, 16) | (instr->bit(7) << 4);
+ double dn_value;
+ get_double_from_d_register(vn, &dn_value);
+ int32_t data[2];
+ memcpy(data, &dn_value, 8);
+ set_register(instr->rtValue(), data[instr->bit(21)]);
+ } else if ((instr->VLValue() == 0x1) && (instr->VCValue() == 0x0) &&
+ (instr->VAValue() == 0x7) && (instr->bits(19, 16) == 0x1)) {
+ // vmrs
+ uint32_t rt = instr->rtValue();
+ if (rt == 0xF) {
+ copy_FPSCR_to_APSR();
+ } else {
+ // Emulate FPSCR from the Simulator flags.
+ uint32_t fpscr = (n_flag_FPSCR_ << 31) | (z_flag_FPSCR_ << 30) |
+ (c_flag_FPSCR_ << 29) | (v_flag_FPSCR_ << 28) |
+ (FPSCR_default_NaN_mode_ << 25) |
+ (inexact_vfp_flag_ << 4) | (underflow_vfp_flag_ << 3) |
+ (overflow_vfp_flag_ << 2) | (div_zero_vfp_flag_ << 1) |
+ (inv_op_vfp_flag_ << 0) | (FPSCR_rounding_mode_);
+ set_register(rt, fpscr);
+ }
+ } else if ((instr->VLValue() == 0x0) && (instr->VCValue() == 0x0) &&
+ (instr->VAValue() == 0x7) && (instr->bits(19, 16) == 0x1)) {
+ // vmsr
+ uint32_t rt = instr->rtValue();
+ if (rt == pc) {
+ MOZ_CRASH();
+ } else {
+ uint32_t rt_value = get_register(rt);
+ n_flag_FPSCR_ = (rt_value >> 31) & 1;
+ z_flag_FPSCR_ = (rt_value >> 30) & 1;
+ c_flag_FPSCR_ = (rt_value >> 29) & 1;
+ v_flag_FPSCR_ = (rt_value >> 28) & 1;
+ FPSCR_default_NaN_mode_ = (rt_value >> 25) & 1;
+ inexact_vfp_flag_ = (rt_value >> 4) & 1;
+ underflow_vfp_flag_ = (rt_value >> 3) & 1;
+ overflow_vfp_flag_ = (rt_value >> 2) & 1;
+ div_zero_vfp_flag_ = (rt_value >> 1) & 1;
+ inv_op_vfp_flag_ = (rt_value >> 0) & 1;
+ FPSCR_rounding_mode_ =
+ static_cast<VFPRoundingMode>((rt_value)&kVFPRoundingModeMask);
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ }
+}
+
+void Simulator::decodeVMOVBetweenCoreAndSinglePrecisionRegisters(
+ SimInstruction* instr) {
+ MOZ_ASSERT(instr->bit(4) == 1 && instr->VCValue() == 0x0 &&
+ instr->VAValue() == 0x0);
+
+ int t = instr->rtValue();
+ int n = instr->VFPNRegValue(kSinglePrecision);
+ bool to_arm_register = (instr->VLValue() == 0x1);
+ if (to_arm_register) {
+ int32_t int_value = get_sinteger_from_s_register(n);
+ set_register(t, int_value);
+ } else {
+ int32_t rs_val = get_register(t);
+ set_s_register_from_sinteger(n, rs_val);
+ }
+}
+
+void Simulator::decodeVCMP(SimInstruction* instr) {
+ MOZ_ASSERT((instr->bit(4) == 0) && (instr->opc1Value() == 0x7));
+ MOZ_ASSERT(((instr->opc2Value() == 0x4) || (instr->opc2Value() == 0x5)) &&
+ (instr->opc3Value() & 0x1));
+ // Comparison.
+
+ VFPRegPrecision precision = kSinglePrecision;
+ if (instr->szValue() == 1) {
+ precision = kDoublePrecision;
+ }
+
+ int d = instr->VFPDRegValue(precision);
+ int m = 0;
+ if (instr->opc2Value() == 0x4) {
+ m = instr->VFPMRegValue(precision);
+ }
+
+ if (precision == kDoublePrecision) {
+ double dd_value;
+ get_double_from_d_register(d, &dd_value);
+ double dm_value = 0.0;
+ if (instr->opc2Value() == 0x4) {
+ get_double_from_d_register(m, &dm_value);
+ }
+
+ // Raise exceptions for quiet NaNs if necessary.
+ if (instr->bit(7) == 1) {
+ if (mozilla::IsNaN(dd_value)) {
+ inv_op_vfp_flag_ = true;
+ }
+ }
+ compute_FPSCR_Flags(dd_value, dm_value);
+ } else {
+ float fd_value;
+ get_float_from_s_register(d, &fd_value);
+ float fm_value = 0.0;
+ if (instr->opc2Value() == 0x4) {
+ get_float_from_s_register(m, &fm_value);
+ }
+
+ // Raise exceptions for quiet NaNs if necessary.
+ if (instr->bit(7) == 1) {
+ if (mozilla::IsNaN(fd_value)) {
+ inv_op_vfp_flag_ = true;
+ }
+ }
+ compute_FPSCR_Flags(fd_value, fm_value);
+ }
+}
+
+void Simulator::decodeVCVTBetweenDoubleAndSingle(SimInstruction* instr) {
+ MOZ_ASSERT(instr->bit(4) == 0 && instr->opc1Value() == 0x7);
+ MOZ_ASSERT(instr->opc2Value() == 0x7 && instr->opc3Value() == 0x3);
+
+ VFPRegPrecision dst_precision = kDoublePrecision;
+ VFPRegPrecision src_precision = kSinglePrecision;
+ if (instr->szValue() == 1) {
+ dst_precision = kSinglePrecision;
+ src_precision = kDoublePrecision;
+ }
+
+ int dst = instr->VFPDRegValue(dst_precision);
+ int src = instr->VFPMRegValue(src_precision);
+
+ if (dst_precision == kSinglePrecision) {
+ double val;
+ get_double_from_d_register(src, &val);
+ set_s_register_from_float(dst, static_cast<float>(val));
+ } else {
+ float val;
+ get_float_from_s_register(src, &val);
+ set_d_register_from_double(dst, static_cast<double>(val));
+ }
+}
+
+static bool get_inv_op_vfp_flag(VFPRoundingMode mode, double val,
+ bool unsigned_) {
+ MOZ_ASSERT(mode == SimRN || mode == SimRM || mode == SimRZ);
+ double max_uint = static_cast<double>(0xffffffffu);
+ double max_int = static_cast<double>(INT32_MAX);
+ double min_int = static_cast<double>(INT32_MIN);
+
+ // Check for NaN.
+ if (val != val) {
+ return true;
+ }
+
+ // Check for overflow. This code works because 32bit integers can be exactly
+ // represented by ieee-754 64bit floating-point values.
+ switch (mode) {
+ case SimRN:
+ return unsigned_ ? (val >= (max_uint + 0.5)) || (val < -0.5)
+ : (val >= (max_int + 0.5)) || (val < (min_int - 0.5));
+ case SimRM:
+ return unsigned_ ? (val >= (max_uint + 1.0)) || (val < 0)
+ : (val >= (max_int + 1.0)) || (val < min_int);
+ case SimRZ:
+ return unsigned_ ? (val >= (max_uint + 1.0)) || (val <= -1)
+ : (val >= (max_int + 1.0)) || (val <= (min_int - 1.0));
+ default:
+ MOZ_CRASH();
+ return true;
+ }
+}
+
+// We call this function only if we had a vfp invalid exception.
+// It returns the correct saturated value.
+static int VFPConversionSaturate(double val, bool unsigned_res) {
+ if (val != val) { // NaN.
+ return 0;
+ }
+ if (unsigned_res) {
+ return (val < 0) ? 0 : 0xffffffffu;
+ }
+ return (val < 0) ? INT32_MIN : INT32_MAX;
+}
+
+void Simulator::decodeVCVTBetweenFloatingPointAndInteger(
+ SimInstruction* instr) {
+ MOZ_ASSERT((instr->bit(4) == 0) && (instr->opc1Value() == 0x7) &&
+ (instr->bits(27, 23) == 0x1D));
+ MOZ_ASSERT(
+ ((instr->opc2Value() == 0x8) && (instr->opc3Value() & 0x1)) ||
+ (((instr->opc2Value() >> 1) == 0x6) && (instr->opc3Value() & 0x1)));
+
+ // Conversion between floating-point and integer.
+ bool to_integer = (instr->bit(18) == 1);
+
+ VFPRegPrecision src_precision =
+ (instr->szValue() == 1) ? kDoublePrecision : kSinglePrecision;
+
+ if (to_integer) {
+ // We are playing with code close to the C++ standard's limits below,
+ // hence the very simple code and heavy checks.
+ //
+ // Note: C++ defines default type casting from floating point to integer
+ // as (close to) rounding toward zero ("fractional part discarded").
+
+ int dst = instr->VFPDRegValue(kSinglePrecision);
+ int src = instr->VFPMRegValue(src_precision);
+
+ // Bit 7 in vcvt instructions indicates if we should use the FPSCR
+ // rounding mode or the default Round to Zero mode.
+ VFPRoundingMode mode = (instr->bit(7) != 1) ? FPSCR_rounding_mode_ : SimRZ;
+ MOZ_ASSERT(mode == SimRM || mode == SimRZ || mode == SimRN);
+
+ bool unsigned_integer = (instr->bit(16) == 0);
+ bool double_precision = (src_precision == kDoublePrecision);
+
+ double val;
+ if (double_precision) {
+ get_double_from_d_register(src, &val);
+ } else {
+ float fval;
+ get_float_from_s_register(src, &fval);
+ val = double(fval);
+ }
+
+ int temp = unsigned_integer ? static_cast<uint32_t>(val)
+ : static_cast<int32_t>(val);
+
+ inv_op_vfp_flag_ = get_inv_op_vfp_flag(mode, val, unsigned_integer);
+
+ double abs_diff = unsigned_integer
+ ? std::fabs(val - static_cast<uint32_t>(temp))
+ : std::fabs(val - temp);
+
+ inexact_vfp_flag_ = (abs_diff != 0);
+
+ if (inv_op_vfp_flag_) {
+ temp = VFPConversionSaturate(val, unsigned_integer);
+ } else {
+ switch (mode) {
+ case SimRN: {
+ int val_sign = (val > 0) ? 1 : -1;
+ if (abs_diff > 0.5) {
+ temp += val_sign;
+ } else if (abs_diff == 0.5) {
+ // Round to even if exactly halfway.
+ temp = ((temp % 2) == 0) ? temp : temp + val_sign;
+ }
+ break;
+ }
+
+ case SimRM:
+ temp = temp > val ? temp - 1 : temp;
+ break;
+
+ case SimRZ:
+ // Nothing to do.
+ break;
+
+ default:
+ MOZ_CRASH();
+ }
+ }
+
+ // Update the destination register.
+ set_s_register_from_sinteger(dst, temp);
+ } else {
+ bool unsigned_integer = (instr->bit(7) == 0);
+ int dst = instr->VFPDRegValue(src_precision);
+ int src = instr->VFPMRegValue(kSinglePrecision);
+
+ int val = get_sinteger_from_s_register(src);
+
+ if (src_precision == kDoublePrecision) {
+ if (unsigned_integer) {
+ set_d_register_from_double(
+ dst, static_cast<double>(static_cast<uint32_t>(val)));
+ } else {
+ set_d_register_from_double(dst, static_cast<double>(val));
+ }
+ } else {
+ if (unsigned_integer) {
+ set_s_register_from_float(
+ dst, static_cast<float>(static_cast<uint32_t>(val)));
+ } else {
+ set_s_register_from_float(dst, static_cast<float>(val));
+ }
+ }
+ }
+}
+
+// A VFPv3 specific instruction.
+void Simulator::decodeVCVTBetweenFloatingPointAndIntegerFrac(
+ SimInstruction* instr) {
+ MOZ_ASSERT(instr->bits(27, 24) == 0xE && instr->opc1Value() == 0x7 &&
+ instr->bit(19) == 1 && instr->bit(17) == 1 &&
+ instr->bits(11, 9) == 0x5 && instr->bit(6) == 1 &&
+ instr->bit(4) == 0);
+
+ int size = (instr->bit(7) == 1) ? 32 : 16;
+
+ int fraction_bits = size - ((instr->bits(3, 0) << 1) | instr->bit(5));
+ double mult = 1 << fraction_bits;
+
+ MOZ_ASSERT(size == 32); // Only handling size == 32 for now.
+
+ // Conversion between floating-point and integer.
+ bool to_fixed = (instr->bit(18) == 1);
+
+ VFPRegPrecision precision =
+ (instr->szValue() == 1) ? kDoublePrecision : kSinglePrecision;
+
+ if (to_fixed) {
+ // We are playing with code close to the C++ standard's limits below,
+ // hence the very simple code and heavy checks.
+ //
+ // Note: C++ defines default type casting from floating point to integer
+ // as (close to) rounding toward zero ("fractional part discarded").
+
+ int dst = instr->VFPDRegValue(precision);
+
+ bool unsigned_integer = (instr->bit(16) == 1);
+ bool double_precision = (precision == kDoublePrecision);
+
+ double val;
+ if (double_precision) {
+ get_double_from_d_register(dst, &val);
+ } else {
+ float fval;
+ get_float_from_s_register(dst, &fval);
+ val = double(fval);
+ }
+
+ // Scale value by specified number of fraction bits.
+ val *= mult;
+
+ // Rounding down towards zero. No need to account for the rounding error
+ // as this instruction always rounds down towards zero. See SimRZ below.
+ int temp = unsigned_integer ? static_cast<uint32_t>(val)
+ : static_cast<int32_t>(val);
+
+ inv_op_vfp_flag_ = get_inv_op_vfp_flag(SimRZ, val, unsigned_integer);
+
+ double abs_diff = unsigned_integer
+ ? std::fabs(val - static_cast<uint32_t>(temp))
+ : std::fabs(val - temp);
+
+ inexact_vfp_flag_ = (abs_diff != 0);
+
+ if (inv_op_vfp_flag_) {
+ temp = VFPConversionSaturate(val, unsigned_integer);
+ }
+
+ // Update the destination register.
+ if (double_precision) {
+ uint32_t dbl[2];
+ dbl[0] = temp;
+ dbl[1] = 0;
+ set_d_register(dst, dbl);
+ } else {
+ set_s_register_from_sinteger(dst, temp);
+ }
+ } else {
+ MOZ_CRASH(); // Not implemented, fixed to float.
+ }
+}
+
+void Simulator::decodeType6CoprocessorIns(SimInstruction* instr) {
+ MOZ_ASSERT(instr->typeValue() == 6);
+
+ if (instr->coprocessorValue() == 0xA) {
+ switch (instr->opcodeValue()) {
+ case 0x8:
+ case 0xA:
+ case 0xC:
+ case 0xE: { // Load and store single precision float to memory.
+ int rn = instr->rnValue();
+ int vd = instr->VFPDRegValue(kSinglePrecision);
+ int offset = instr->immed8Value();
+ if (!instr->hasU()) {
+ offset = -offset;
+ }
+
+ int32_t address = get_register(rn) + 4 * offset;
+ if (instr->hasL()) {
+ // Load double from memory: vldr.
+ set_s_register_from_sinteger(vd, readW(address, instr));
+ } else {
+ // Store double to memory: vstr.
+ writeW(address, get_sinteger_from_s_register(vd), instr);
+ }
+ break;
+ }
+ case 0x4:
+ case 0x5:
+ case 0x6:
+ case 0x7:
+ case 0x9:
+ case 0xB:
+ // Load/store multiple single from memory: vldm/vstm.
+ handleVList(instr);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ } else if (instr->coprocessorValue() == 0xB) {
+ switch (instr->opcodeValue()) {
+ case 0x2:
+ // Load and store double to two GP registers
+ if (instr->bits(7, 6) != 0 || instr->bit(4) != 1) {
+ MOZ_CRASH(); // Not used atm.
+ } else {
+ int rt = instr->rtValue();
+ int rn = instr->rnValue();
+ int vm = instr->VFPMRegValue(kDoublePrecision);
+ if (instr->hasL()) {
+ int32_t data[2];
+ double d;
+ get_double_from_d_register(vm, &d);
+ memcpy(data, &d, 8);
+ set_register(rt, data[0]);
+ set_register(rn, data[1]);
+ } else {
+ int32_t data[] = {get_register(rt), get_register(rn)};
+ double d;
+ memcpy(&d, data, 8);
+ set_d_register_from_double(vm, d);
+ }
+ }
+ break;
+ case 0x8:
+ case 0xA:
+ case 0xC:
+ case 0xE: { // Load and store double to memory.
+ int rn = instr->rnValue();
+ int vd = instr->VFPDRegValue(kDoublePrecision);
+ int offset = instr->immed8Value();
+ if (!instr->hasU()) {
+ offset = -offset;
+ }
+ int32_t address = get_register(rn) + 4 * offset;
+ if (instr->hasL()) {
+ // Load double from memory: vldr.
+ uint64_t data = readQ(address, instr);
+ double val;
+ memcpy(&val, &data, 8);
+ set_d_register_from_double(vd, val);
+ } else {
+ // Store double to memory: vstr.
+ uint64_t data;
+ double val;
+ get_double_from_d_register(vd, &val);
+ memcpy(&data, &val, 8);
+ writeQ(address, data, instr);
+ }
+ break;
+ }
+ case 0x4:
+ case 0x5:
+ case 0x6:
+ case 0x7:
+ case 0x9:
+ case 0xB:
+ // Load/store multiple double from memory: vldm/vstm.
+ handleVList(instr);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ } else {
+ MOZ_CRASH();
+ }
+}
+
+void Simulator::decodeSpecialCondition(SimInstruction* instr) {
+ switch (instr->specialValue()) {
+ case 5:
+ if (instr->bits(18, 16) == 0 && instr->bits(11, 6) == 0x28 &&
+ instr->bit(4) == 1) {
+ // vmovl signed
+ if ((instr->vdValue() & 1) != 0) {
+ MOZ_CRASH("Undefined behavior");
+ }
+ int Vd = (instr->bit(22) << 3) | (instr->vdValue() >> 1);
+ int Vm = (instr->bit(5) << 4) | instr->vmValue();
+ int imm3 = instr->bits(21, 19);
+ if (imm3 != 1 && imm3 != 2 && imm3 != 4) {
+ MOZ_CRASH();
+ }
+ int esize = 8 * imm3;
+ int elements = 64 / esize;
+ int8_t from[8];
+ get_d_register(Vm, reinterpret_cast<uint64_t*>(from));
+ int16_t to[8];
+ int e = 0;
+ while (e < elements) {
+ to[e] = from[e];
+ e++;
+ }
+ set_q_register(Vd, reinterpret_cast<uint64_t*>(to));
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 7:
+ if (instr->bits(18, 16) == 0 && instr->bits(11, 6) == 0x28 &&
+ instr->bit(4) == 1) {
+ // vmovl unsigned.
+ if ((instr->vdValue() & 1) != 0) {
+ MOZ_CRASH("Undefined behavior");
+ }
+ int Vd = (instr->bit(22) << 3) | (instr->vdValue() >> 1);
+ int Vm = (instr->bit(5) << 4) | instr->vmValue();
+ int imm3 = instr->bits(21, 19);
+ if (imm3 != 1 && imm3 != 2 && imm3 != 4) {
+ MOZ_CRASH();
+ }
+ int esize = 8 * imm3;
+ int elements = 64 / esize;
+ uint8_t from[8];
+ get_d_register(Vm, reinterpret_cast<uint64_t*>(from));
+ uint16_t to[8];
+ int e = 0;
+ while (e < elements) {
+ to[e] = from[e];
+ e++;
+ }
+ set_q_register(Vd, reinterpret_cast<uint64_t*>(to));
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 8:
+ if (instr->bits(21, 20) == 0) {
+ // vst1
+ int Vd = (instr->bit(22) << 4) | instr->vdValue();
+ int Rn = instr->vnValue();
+ int type = instr->bits(11, 8);
+ int Rm = instr->vmValue();
+ int32_t address = get_register(Rn);
+ int regs = 0;
+ switch (type) {
+ case nlt_1:
+ regs = 1;
+ break;
+ case nlt_2:
+ regs = 2;
+ break;
+ case nlt_3:
+ regs = 3;
+ break;
+ case nlt_4:
+ regs = 4;
+ break;
+ default:
+ MOZ_CRASH();
+ break;
+ }
+ int r = 0;
+ while (r < regs) {
+ uint32_t data[2];
+ get_d_register(Vd + r, data);
+ // TODO: We should AllowUnaligned here only if the alignment attribute
+ // of the instruction calls for default alignment.
+ //
+ // Use writeQ to get handling of traps right. (The spec says to
+ // perform two individual word writes, but let's not worry about
+ // that.)
+ writeQ(address, (uint64_t(data[1]) << 32) | uint64_t(data[0]), instr,
+ AllowUnaligned);
+ address += 8;
+ r++;
+ }
+ if (Rm != 15) {
+ if (Rm == 13) {
+ set_register(Rn, address);
+ } else {
+ set_register(Rn, get_register(Rn) + get_register(Rm));
+ }
+ }
+ } else if (instr->bits(21, 20) == 2) {
+ // vld1
+ int Vd = (instr->bit(22) << 4) | instr->vdValue();
+ int Rn = instr->vnValue();
+ int type = instr->bits(11, 8);
+ int Rm = instr->vmValue();
+ int32_t address = get_register(Rn);
+ int regs = 0;
+ switch (type) {
+ case nlt_1:
+ regs = 1;
+ break;
+ case nlt_2:
+ regs = 2;
+ break;
+ case nlt_3:
+ regs = 3;
+ break;
+ case nlt_4:
+ regs = 4;
+ break;
+ default:
+ MOZ_CRASH();
+ break;
+ }
+ int r = 0;
+ while (r < regs) {
+ uint32_t data[2];
+ // TODO: We should AllowUnaligned here only if the alignment attribute
+ // of the instruction calls for default alignment.
+ //
+ // Use readQ to get handling of traps right. (The spec says to
+ // perform two individual word reads, but let's not worry about that.)
+ uint64_t tmp = readQ(address, instr, AllowUnaligned);
+ data[0] = tmp;
+ data[1] = tmp >> 32;
+ set_d_register(Vd + r, data);
+ address += 8;
+ r++;
+ }
+ if (Rm != 15) {
+ if (Rm == 13) {
+ set_register(Rn, address);
+ } else {
+ set_register(Rn, get_register(Rn) + get_register(Rm));
+ }
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 9:
+ if (instr->bits(9, 8) == 0) {
+ int Vd = (instr->bit(22) << 4) | instr->vdValue();
+ int Rn = instr->vnValue();
+ int size = instr->bits(11, 10);
+ int Rm = instr->vmValue();
+ int index = instr->bits(7, 5);
+ int align = instr->bit(4);
+ int32_t address = get_register(Rn);
+ if (size != 2 || align) {
+ MOZ_CRASH("NYI");
+ }
+ int a = instr->bits(5, 4);
+ if (a != 0 && a != 3) {
+ MOZ_CRASH("Unspecified");
+ }
+ if (index > 1) {
+ Vd++;
+ index -= 2;
+ }
+ uint32_t data[2];
+ get_d_register(Vd, data);
+ switch (instr->bits(21, 20)) {
+ case 0:
+ // vst1 single element from one lane
+ writeW(address, data[index], instr, AllowUnaligned);
+ break;
+ case 2:
+ // vld1 single element to one lane
+ data[index] = readW(address, instr, AllowUnaligned);
+ set_d_register(Vd, data);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+ address += 4;
+ if (Rm != 15) {
+ if (Rm == 13) {
+ set_register(Rn, address);
+ } else {
+ set_register(Rn, get_register(Rn) + get_register(Rm));
+ }
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 0xA:
+ if (instr->bits(31, 20) == 0xf57) {
+ switch (instr->bits(7, 4)) {
+ case 1: // CLREX
+ exclusiveMonitorClear();
+ break;
+ case 5: // DMB
+ AtomicOperations::fenceSeqCst();
+ break;
+ case 4: // DSB
+ // We do not use DSB.
+ MOZ_CRASH("DSB unimplemented");
+ case 6: // ISB
+ // We do not use ISB.
+ MOZ_CRASH("ISB unimplemented");
+ default:
+ MOZ_CRASH();
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 0xB:
+ if (instr->bits(22, 20) == 5 && instr->bits(15, 12) == 0xf) {
+ // pld: ignore instruction.
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 0x1C:
+ case 0x1D:
+ if (instr->bit(4) == 1 && instr->bits(11, 9) != 5) {
+ // MCR, MCR2, MRC, MRC2 with cond == 15
+ decodeType7CoprocessorIns(instr);
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+// Executes the current instruction.
+void Simulator::instructionDecode(SimInstruction* instr) {
+ if (!SimulatorProcess::ICacheCheckingDisableCount) {
+ AutoLockSimulatorCache als;
+ SimulatorProcess::checkICacheLocked(instr);
+ }
+
+ pc_modified_ = false;
+
+ static const uint32_t kSpecialCondition = 15 << 28;
+ if (instr->conditionField() == kSpecialCondition) {
+ decodeSpecialCondition(instr);
+ } else if (conditionallyExecute(instr)) {
+ switch (instr->typeValue()) {
+ case 0:
+ case 1:
+ decodeType01(instr);
+ break;
+ case 2:
+ decodeType2(instr);
+ break;
+ case 3:
+ decodeType3(instr);
+ break;
+ case 4:
+ decodeType4(instr);
+ break;
+ case 5:
+ decodeType5(instr);
+ break;
+ case 6:
+ decodeType6(instr);
+ break;
+ case 7:
+ decodeType7(instr);
+ break;
+ default:
+ MOZ_CRASH();
+ break;
+ }
+ // If the instruction is a non taken conditional stop, we need to skip
+ // the inlined message address.
+ } else if (instr->isStop()) {
+ set_pc(get_pc() + 2 * SimInstruction::kInstrSize);
+ }
+ if (!pc_modified_) {
+ set_register(pc,
+ reinterpret_cast<int32_t>(instr) + SimInstruction::kInstrSize);
+ }
+}
+
+void Simulator::enable_single_stepping(SingleStepCallback cb, void* arg) {
+ single_stepping_ = true;
+ single_step_callback_ = cb;
+ single_step_callback_arg_ = arg;
+ single_step_callback_(single_step_callback_arg_, this, (void*)get_pc());
+}
+
+void Simulator::disable_single_stepping() {
+ if (!single_stepping_) {
+ return;
+ }
+ single_step_callback_(single_step_callback_arg_, this, (void*)get_pc());
+ single_stepping_ = false;
+ single_step_callback_ = nullptr;
+ single_step_callback_arg_ = nullptr;
+}
+
+template <bool EnableStopSimAt>
+void Simulator::execute() {
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ // Get the PC to simulate. Cannot use the accessor here as we need the raw
+ // PC value and not the one used as input to arithmetic instructions.
+ int program_counter = get_pc();
+
+ while (program_counter != end_sim_pc) {
+ if (EnableStopSimAt && (icount_ == Simulator::StopSimAt)) {
+ fprintf(stderr, "\nStopped simulation at icount %lld\n", icount_);
+ ArmDebugger dbg(this);
+ dbg.debug();
+ } else {
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this,
+ (void*)program_counter);
+ }
+ SimInstruction* instr =
+ reinterpret_cast<SimInstruction*>(program_counter);
+ instructionDecode(instr);
+ icount_++;
+ }
+ program_counter = get_pc();
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+}
+
+void Simulator::callInternal(uint8_t* entry) {
+ // Prepare to execute the code at entry.
+ set_register(pc, reinterpret_cast<int32_t>(entry));
+
+ // Put down marker for end of simulation. The simulator will stop simulation
+ // when the PC reaches this value. By saving the "end simulation" value into
+ // the LR the simulation stops when returning to this call point.
+ set_register(lr, end_sim_pc);
+
+ // Remember the values of callee-saved registers. The code below assumes
+ // that r9 is not used as sb (static base) in simulator code and therefore
+ // is regarded as a callee-saved register.
+ int32_t r4_val = get_register(r4);
+ int32_t r5_val = get_register(r5);
+ int32_t r6_val = get_register(r6);
+ int32_t r7_val = get_register(r7);
+ int32_t r8_val = get_register(r8);
+ int32_t r9_val = get_register(r9);
+ int32_t r10_val = get_register(r10);
+ int32_t r11_val = get_register(r11);
+
+ // Remember d8 to d15 which are callee-saved.
+ uint64_t d8_val;
+ get_d_register(d8, &d8_val);
+ uint64_t d9_val;
+ get_d_register(d9, &d9_val);
+ uint64_t d10_val;
+ get_d_register(d10, &d10_val);
+ uint64_t d11_val;
+ get_d_register(d11, &d11_val);
+ uint64_t d12_val;
+ get_d_register(d12, &d12_val);
+ uint64_t d13_val;
+ get_d_register(d13, &d13_val);
+ uint64_t d14_val;
+ get_d_register(d14, &d14_val);
+ uint64_t d15_val;
+ get_d_register(d15, &d15_val);
+
+ // Set up the callee-saved registers with a known value. To be able to check
+ // that they are preserved properly across JS execution.
+ int32_t callee_saved_value = uint32_t(icount_);
+ uint64_t callee_saved_value_d = uint64_t(icount_);
+
+ if (!skipCalleeSavedRegsCheck) {
+ set_register(r4, callee_saved_value);
+ set_register(r5, callee_saved_value);
+ set_register(r6, callee_saved_value);
+ set_register(r7, callee_saved_value);
+ set_register(r8, callee_saved_value);
+ set_register(r9, callee_saved_value);
+ set_register(r10, callee_saved_value);
+ set_register(r11, callee_saved_value);
+
+ set_d_register(d8, &callee_saved_value_d);
+ set_d_register(d9, &callee_saved_value_d);
+ set_d_register(d10, &callee_saved_value_d);
+ set_d_register(d11, &callee_saved_value_d);
+ set_d_register(d12, &callee_saved_value_d);
+ set_d_register(d13, &callee_saved_value_d);
+ set_d_register(d14, &callee_saved_value_d);
+ set_d_register(d15, &callee_saved_value_d);
+ }
+ // Start the simulation.
+ if (Simulator::StopSimAt != -1L) {
+ execute<true>();
+ } else {
+ execute<false>();
+ }
+
+ if (!skipCalleeSavedRegsCheck) {
+ // Check that the callee-saved registers have been preserved.
+ MOZ_ASSERT(callee_saved_value == get_register(r4));
+ MOZ_ASSERT(callee_saved_value == get_register(r5));
+ MOZ_ASSERT(callee_saved_value == get_register(r6));
+ MOZ_ASSERT(callee_saved_value == get_register(r7));
+ MOZ_ASSERT(callee_saved_value == get_register(r8));
+ MOZ_ASSERT(callee_saved_value == get_register(r9));
+ MOZ_ASSERT(callee_saved_value == get_register(r10));
+ MOZ_ASSERT(callee_saved_value == get_register(r11));
+
+ uint64_t value;
+ get_d_register(d8, &value);
+ MOZ_ASSERT(callee_saved_value_d == value);
+ get_d_register(d9, &value);
+ MOZ_ASSERT(callee_saved_value_d == value);
+ get_d_register(d10, &value);
+ MOZ_ASSERT(callee_saved_value_d == value);
+ get_d_register(d11, &value);
+ MOZ_ASSERT(callee_saved_value_d == value);
+ get_d_register(d12, &value);
+ MOZ_ASSERT(callee_saved_value_d == value);
+ get_d_register(d13, &value);
+ MOZ_ASSERT(callee_saved_value_d == value);
+ get_d_register(d14, &value);
+ MOZ_ASSERT(callee_saved_value_d == value);
+ get_d_register(d15, &value);
+ MOZ_ASSERT(callee_saved_value_d == value);
+
+ // Restore callee-saved registers with the original value.
+ set_register(r4, r4_val);
+ set_register(r5, r5_val);
+ set_register(r6, r6_val);
+ set_register(r7, r7_val);
+ set_register(r8, r8_val);
+ set_register(r9, r9_val);
+ set_register(r10, r10_val);
+ set_register(r11, r11_val);
+
+ set_d_register(d8, &d8_val);
+ set_d_register(d9, &d9_val);
+ set_d_register(d10, &d10_val);
+ set_d_register(d11, &d11_val);
+ set_d_register(d12, &d12_val);
+ set_d_register(d13, &d13_val);
+ set_d_register(d14, &d14_val);
+ set_d_register(d15, &d15_val);
+ }
+}
+
+int32_t Simulator::call(uint8_t* entry, int argument_count, ...) {
+ va_list parameters;
+ va_start(parameters, argument_count);
+
+ // First four arguments passed in registers.
+ if (argument_count >= 1) {
+ set_register(r0, va_arg(parameters, int32_t));
+ }
+ if (argument_count >= 2) {
+ set_register(r1, va_arg(parameters, int32_t));
+ }
+ if (argument_count >= 3) {
+ set_register(r2, va_arg(parameters, int32_t));
+ }
+ if (argument_count >= 4) {
+ set_register(r3, va_arg(parameters, int32_t));
+ }
+
+ // Remaining arguments passed on stack.
+ int original_stack = get_register(sp);
+ int entry_stack = original_stack;
+ if (argument_count >= 4) {
+ entry_stack -= (argument_count - 4) * sizeof(int32_t);
+ }
+
+ entry_stack &= ~ABIStackAlignment;
+
+ // Store remaining arguments on stack, from low to high memory.
+ intptr_t* stack_argument = reinterpret_cast<intptr_t*>(entry_stack);
+ for (int i = 4; i < argument_count; i++) {
+ stack_argument[i - 4] = va_arg(parameters, int32_t);
+ }
+ va_end(parameters);
+ set_register(sp, entry_stack);
+
+ callInternal(entry);
+
+ // Pop stack passed arguments.
+ MOZ_ASSERT(entry_stack == get_register(sp));
+ set_register(sp, original_stack);
+
+ int32_t result = get_register(r0);
+ return result;
+}
+
+Simulator* Simulator::Current() {
+ JSContext* cx = TlsContext.get();
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cx->runtime()));
+ return cx->simulator();
+}
+
+} // namespace jit
+} // namespace js
+
+js::jit::Simulator* JSContext::simulator() const { return simulator_; }
diff --git a/js/src/jit/arm/Simulator-arm.h b/js/src/jit/arm/Simulator-arm.h
new file mode 100644
index 0000000000..fba0f8ce5e
--- /dev/null
+++ b/js/src/jit/arm/Simulator-arm.h
@@ -0,0 +1,632 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+// Copyright 2012 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef jit_arm_Simulator_arm_h
+#define jit_arm_Simulator_arm_h
+
+#ifdef JS_SIMULATOR_ARM
+
+# include "mozilla/Atomics.h"
+
+# include "jit/arm/Architecture-arm.h"
+# include "jit/arm/disasm/Disasm-arm.h"
+# include "jit/IonTypes.h"
+# include "js/AllocPolicy.h"
+# include "js/ProfilingFrameIterator.h"
+# include "threading/Thread.h"
+# include "vm/MutexIDs.h"
+# include "wasm/WasmSignalHandlers.h"
+
+namespace js {
+namespace jit {
+
+class JitActivation;
+class Simulator;
+class Redirection;
+class CachePage;
+class AutoLockSimulator;
+
+// When the SingleStepCallback is called, the simulator is about to execute
+// sim->get_pc() and the current machine state represents the completed
+// execution of the previous pc.
+typedef void (*SingleStepCallback)(void* arg, Simulator* sim, void* pc);
+
+// VFP rounding modes. See ARM DDI 0406B Page A2-29.
+enum VFPRoundingMode {
+ SimRN = 0 << 22, // Round to Nearest.
+ SimRP = 1 << 22, // Round towards Plus Infinity.
+ SimRM = 2 << 22, // Round towards Minus Infinity.
+ SimRZ = 3 << 22, // Round towards zero.
+
+ // Aliases.
+ kRoundToNearest = SimRN,
+ kRoundToPlusInf = SimRP,
+ kRoundToMinusInf = SimRM,
+ kRoundToZero = SimRZ
+};
+
+const uint32_t kVFPRoundingModeMask = 3 << 22;
+
+typedef int32_t Instr;
+class SimInstruction;
+
+// Per thread simulator state.
+class Simulator {
+ public:
+ friend class ArmDebugger;
+ enum Register {
+ no_reg = -1,
+ r0 = 0,
+ r1,
+ r2,
+ r3,
+ r4,
+ r5,
+ r6,
+ r7,
+ r8,
+ r9,
+ r10,
+ r11,
+ r12,
+ r13,
+ r14,
+ r15,
+ num_registers,
+ fp = 11,
+ ip = 12,
+ sp = 13,
+ lr = 14,
+ pc = 15,
+ s0 = 0,
+ s1,
+ s2,
+ s3,
+ s4,
+ s5,
+ s6,
+ s7,
+ s8,
+ s9,
+ s10,
+ s11,
+ s12,
+ s13,
+ s14,
+ s15,
+ s16,
+ s17,
+ s18,
+ s19,
+ s20,
+ s21,
+ s22,
+ s23,
+ s24,
+ s25,
+ s26,
+ s27,
+ s28,
+ s29,
+ s30,
+ s31,
+ num_s_registers = 32,
+ d0 = 0,
+ d1,
+ d2,
+ d3,
+ d4,
+ d5,
+ d6,
+ d7,
+ d8,
+ d9,
+ d10,
+ d11,
+ d12,
+ d13,
+ d14,
+ d15,
+ d16,
+ d17,
+ d18,
+ d19,
+ d20,
+ d21,
+ d22,
+ d23,
+ d24,
+ d25,
+ d26,
+ d27,
+ d28,
+ d29,
+ d30,
+ d31,
+ num_d_registers = 32,
+ q0 = 0,
+ q1,
+ q2,
+ q3,
+ q4,
+ q5,
+ q6,
+ q7,
+ q8,
+ q9,
+ q10,
+ q11,
+ q12,
+ q13,
+ q14,
+ q15,
+ num_q_registers = 16
+ };
+
+ // Returns nullptr on OOM.
+ static Simulator* Create();
+
+ static void Destroy(Simulator* simulator);
+
+ // Constructor/destructor are for internal use only; use the static methods
+ // above.
+ Simulator();
+ ~Simulator();
+
+ // The currently executing Simulator instance. Potentially there can be one
+ // for each native thread.
+ static Simulator* Current();
+
+ static uintptr_t StackLimit() { return Simulator::Current()->stackLimit(); }
+
+ // Disassemble some instructions starting at instr and print them
+ // on stdout. Useful for working within GDB after a MOZ_CRASH(),
+ // among other things.
+ //
+ // Typical use within a crashed instruction decoding method is simply:
+ //
+ // call Simulator::disassemble(instr, 1)
+ //
+ // or use one of the more convenient inline methods below.
+ static void disassemble(SimInstruction* instr, size_t n);
+
+ // Disassemble one instruction.
+ // "call disasm(instr)"
+ void disasm(SimInstruction* instr);
+
+ // Disassemble n instructions starting at instr.
+ // "call disasm(instr, 3)"
+ void disasm(SimInstruction* instr, size_t n);
+
+ // Skip backwards m instructions before starting, then disassemble n
+ // instructions.
+ // "call disasm(instr, 3, 7)"
+ void disasm(SimInstruction* instr, size_t m, size_t n);
+
+ uintptr_t* addressOfStackLimit();
+
+ // Accessors for register state. Reading the pc value adheres to the ARM
+ // architecture specification and is off by a 8 from the currently executing
+ // instruction.
+ void set_register(int reg, int32_t value);
+ int32_t get_register(int reg) const;
+ double get_double_from_register_pair(int reg);
+ void set_register_pair_from_double(int reg, double* value);
+ void set_dw_register(int dreg, const int* dbl);
+
+ // Support for VFP.
+ void get_d_register(int dreg, uint64_t* value);
+ void set_d_register(int dreg, const uint64_t* value);
+ void get_d_register(int dreg, uint32_t* value);
+ void set_d_register(int dreg, const uint32_t* value);
+ void get_q_register(int qreg, uint64_t* value);
+ void set_q_register(int qreg, const uint64_t* value);
+ void get_q_register(int qreg, uint32_t* value);
+ void set_q_register(int qreg, const uint32_t* value);
+ void set_s_register(int reg, unsigned int value);
+ unsigned int get_s_register(int reg) const;
+
+ void set_d_register_from_double(int dreg, const double& dbl) {
+ setVFPRegister<double, 2>(dreg, dbl);
+ }
+ void get_double_from_d_register(int dreg, double* out) {
+ getFromVFPRegister<double, 2>(dreg, out);
+ }
+ void set_s_register_from_float(int sreg, const float flt) {
+ setVFPRegister<float, 1>(sreg, flt);
+ }
+ void get_float_from_s_register(int sreg, float* out) {
+ getFromVFPRegister<float, 1>(sreg, out);
+ }
+ void set_s_register_from_sinteger(int sreg, const int sint) {
+ setVFPRegister<int, 1>(sreg, sint);
+ }
+ int get_sinteger_from_s_register(int sreg) {
+ int ret;
+ getFromVFPRegister<int, 1>(sreg, &ret);
+ return ret;
+ }
+
+ // Special case of set_register and get_register to access the raw PC value.
+ void set_pc(int32_t value);
+ int32_t get_pc() const;
+
+ template <typename T>
+ T get_pc_as() const {
+ return reinterpret_cast<T>(get_pc());
+ }
+
+ void enable_single_stepping(SingleStepCallback cb, void* arg);
+ void disable_single_stepping();
+
+ uintptr_t stackLimit() const;
+ bool overRecursed(uintptr_t newsp = 0) const;
+ bool overRecursedWithExtra(uint32_t extra) const;
+
+ // Executes ARM instructions until the PC reaches end_sim_pc.
+ template <bool EnableStopSimAt>
+ void execute();
+
+ // Sets up the simulator state and grabs the result on return.
+ int32_t call(uint8_t* entry, int argument_count, ...);
+
+ // Debugger input.
+ void setLastDebuggerInput(char* input);
+ char* lastDebuggerInput() { return lastDebuggerInput_; }
+
+ // Returns true if pc register contains one of the 'special_values' defined
+ // below (bad_lr, end_sim_pc).
+ bool has_bad_pc() const;
+
+ private:
+ enum special_values {
+ // Known bad pc value to ensure that the simulator does not execute
+ // without being properly setup.
+ bad_lr = -1,
+ // A pc value used to signal the simulator to stop execution. Generally
+ // the lr is set to this value on transition from native C code to
+ // simulated execution, so that the simulator can "return" to the native
+ // C code.
+ end_sim_pc = -2
+ };
+
+ // ForbidUnaligned means "always fault on unaligned access".
+ //
+ // AllowUnaligned means "allow the unaligned access if other conditions are
+ // met". The "other conditions" vary with the instruction: For all
+ // instructions the base condition is !HasAlignmentFault(), ie, the chip is
+ // configured to allow unaligned accesses. For instructions like VLD1
+ // there is an additional constraint that the alignment attribute in the
+ // instruction must be set to "default alignment".
+
+ enum UnalignedPolicy { ForbidUnaligned, AllowUnaligned };
+
+ bool init();
+
+ // Checks if the current instruction should be executed based on its
+ // condition bits.
+ inline bool conditionallyExecute(SimInstruction* instr);
+
+ // Helper functions to set the conditional flags in the architecture state.
+ void setNZFlags(int32_t val);
+ void setCFlag(bool val);
+ void setVFlag(bool val);
+ bool carryFrom(int32_t left, int32_t right, int32_t carry = 0);
+ bool borrowFrom(int32_t left, int32_t right);
+ bool overflowFrom(int32_t alu_out, int32_t left, int32_t right,
+ bool addition);
+
+ inline int getCarry() { return c_flag_ ? 1 : 0; };
+
+ // Support for VFP.
+ void compute_FPSCR_Flags(double val1, double val2);
+ void copy_FPSCR_to_APSR();
+ inline void canonicalizeNaN(double* value);
+ inline void canonicalizeNaN(float* value);
+
+ // Helper functions to decode common "addressing" modes
+ int32_t getShiftRm(SimInstruction* instr, bool* carry_out);
+ int32_t getImm(SimInstruction* instr, bool* carry_out);
+ int32_t processPU(SimInstruction* instr, int num_regs, int operand_size,
+ intptr_t* start_address, intptr_t* end_address);
+ void handleRList(SimInstruction* instr, bool load);
+ void handleVList(SimInstruction* inst);
+ void softwareInterrupt(SimInstruction* instr);
+
+ // Stop helper functions.
+ inline bool isStopInstruction(SimInstruction* instr);
+ inline bool isWatchedStop(uint32_t bkpt_code);
+ inline bool isEnabledStop(uint32_t bkpt_code);
+ inline void enableStop(uint32_t bkpt_code);
+ inline void disableStop(uint32_t bkpt_code);
+ inline void increaseStopCounter(uint32_t bkpt_code);
+ void printStopInfo(uint32_t code);
+
+ // Handle a wasm interrupt triggered by an async signal handler.
+ JS::ProfilingFrameIterator::RegisterState registerState();
+
+ // Handle any wasm faults, returning true if the fault was handled.
+ // This method is rather hot so inline the normal (no-wasm) case.
+ bool MOZ_ALWAYS_INLINE handleWasmSegFault(int32_t addr, unsigned numBytes) {
+ if (MOZ_LIKELY(!wasm::CodeExists)) {
+ return false;
+ }
+
+ uint8_t* newPC;
+ if (!wasm::MemoryAccessTraps(registerState(), (uint8_t*)addr, numBytes,
+ &newPC)) {
+ return false;
+ }
+
+ set_pc(int32_t(newPC));
+ return true;
+ }
+
+ // Read and write memory.
+ inline uint8_t readBU(int32_t addr);
+ inline int8_t readB(int32_t addr);
+ inline void writeB(int32_t addr, uint8_t value);
+ inline void writeB(int32_t addr, int8_t value);
+
+ inline uint8_t readExBU(int32_t addr);
+ inline int32_t writeExB(int32_t addr, uint8_t value);
+
+ inline uint16_t readHU(int32_t addr, SimInstruction* instr);
+ inline int16_t readH(int32_t addr, SimInstruction* instr);
+ // Note: Overloaded on the sign of the value.
+ inline void writeH(int32_t addr, uint16_t value, SimInstruction* instr);
+ inline void writeH(int32_t addr, int16_t value, SimInstruction* instr);
+
+ inline uint16_t readExHU(int32_t addr, SimInstruction* instr);
+ inline int32_t writeExH(int32_t addr, uint16_t value, SimInstruction* instr);
+
+ inline int readW(int32_t addr, SimInstruction* instr,
+ UnalignedPolicy f = ForbidUnaligned);
+ inline void writeW(int32_t addr, int value, SimInstruction* instr,
+ UnalignedPolicy f = ForbidUnaligned);
+
+ inline uint64_t readQ(int32_t addr, SimInstruction* instr,
+ UnalignedPolicy f = ForbidUnaligned);
+ inline void writeQ(int32_t addr, uint64_t value, SimInstruction* instr,
+ UnalignedPolicy f = ForbidUnaligned);
+
+ inline int readExW(int32_t addr, SimInstruction* instr);
+ inline int writeExW(int32_t addr, int value, SimInstruction* instr);
+
+ int32_t* readDW(int32_t addr);
+ void writeDW(int32_t addr, int32_t value1, int32_t value2);
+
+ int32_t readExDW(int32_t addr, int32_t* hibits);
+ int32_t writeExDW(int32_t addr, int32_t value1, int32_t value2);
+
+ // Executing is handled based on the instruction type.
+ // Both type 0 and type 1 rolled into one.
+ void decodeType01(SimInstruction* instr);
+ void decodeType2(SimInstruction* instr);
+ void decodeType3(SimInstruction* instr);
+ void decodeType4(SimInstruction* instr);
+ void decodeType5(SimInstruction* instr);
+ void decodeType6(SimInstruction* instr);
+ void decodeType7(SimInstruction* instr);
+
+ // Support for VFP.
+ void decodeTypeVFP(SimInstruction* instr);
+ void decodeType6CoprocessorIns(SimInstruction* instr);
+ void decodeSpecialCondition(SimInstruction* instr);
+
+ void decodeVMOVBetweenCoreAndSinglePrecisionRegisters(SimInstruction* instr);
+ void decodeVCMP(SimInstruction* instr);
+ void decodeVCVTBetweenDoubleAndSingle(SimInstruction* instr);
+ void decodeVCVTBetweenFloatingPointAndInteger(SimInstruction* instr);
+ void decodeVCVTBetweenFloatingPointAndIntegerFrac(SimInstruction* instr);
+
+ // Support for some system functions.
+ void decodeType7CoprocessorIns(SimInstruction* instr);
+
+ // Executes one instruction.
+ void instructionDecode(SimInstruction* instr);
+
+ public:
+ static int64_t StopSimAt;
+
+ // For testing the MoveResolver code, a MoveResolver is set up, and
+ // the VFP registers are loaded with pre-determined values,
+ // then the sequence of code is simulated. In order to test this with the
+ // simulator, the callee-saved registers can't be trashed. This flag
+ // disables that feature.
+ bool skipCalleeSavedRegsCheck;
+
+ // Runtime call support.
+ static void* RedirectNativeFunction(void* nativeFunction,
+ ABIFunctionType type);
+
+ private:
+ // Handle arguments and return value for runtime FP functions.
+ void getFpArgs(double* x, double* y, int32_t* z);
+ void getFpFromStack(int32_t* stack, double* x1);
+ void setCallResultDouble(double result);
+ void setCallResultFloat(float result);
+ void setCallResult(int64_t res);
+ void scratchVolatileRegisters(bool scratchFloat = true);
+
+ template <class ReturnType, int register_size>
+ void getFromVFPRegister(int reg_index, ReturnType* out);
+
+ template <class InputType, int register_size>
+ void setVFPRegister(int reg_index, const InputType& value);
+
+ void callInternal(uint8_t* entry);
+
+ // Architecture state.
+ // Saturating instructions require a Q flag to indicate saturation.
+ // There is currently no way to read the CPSR directly, and thus read the Q
+ // flag, so this is left unimplemented.
+ int32_t registers_[16];
+ bool n_flag_;
+ bool z_flag_;
+ bool c_flag_;
+ bool v_flag_;
+
+ // VFP architecture state.
+ uint32_t vfp_registers_[num_d_registers * 2];
+ bool n_flag_FPSCR_;
+ bool z_flag_FPSCR_;
+ bool c_flag_FPSCR_;
+ bool v_flag_FPSCR_;
+
+ // VFP rounding mode. See ARM DDI 0406B Page A2-29.
+ VFPRoundingMode FPSCR_rounding_mode_;
+ bool FPSCR_default_NaN_mode_;
+
+ // VFP FP exception flags architecture state.
+ bool inv_op_vfp_flag_;
+ bool div_zero_vfp_flag_;
+ bool overflow_vfp_flag_;
+ bool underflow_vfp_flag_;
+ bool inexact_vfp_flag_;
+
+ // Simulator support.
+ char* stack_;
+ uintptr_t stackLimit_;
+ bool pc_modified_;
+ int64_t icount_;
+
+ // Debugger input.
+ char* lastDebuggerInput_;
+
+ // Registered breakpoints.
+ SimInstruction* break_pc_;
+ Instr break_instr_;
+
+ // Single-stepping support
+ bool single_stepping_;
+ SingleStepCallback single_step_callback_;
+ void* single_step_callback_arg_;
+
+ // A stop is watched if its code is less than kNumOfWatchedStops.
+ // Only watched stops support enabling/disabling and the counter feature.
+ static const uint32_t kNumOfWatchedStops = 256;
+
+ // Breakpoint is disabled if bit 31 is set.
+ static const uint32_t kStopDisabledBit = 1 << 31;
+
+ // A stop is enabled, meaning the simulator will stop when meeting the
+ // instruction, if bit 31 of watched_stops_[code].count is unset.
+ // The value watched_stops_[code].count & ~(1 << 31) indicates how many times
+ // the breakpoint was hit or gone through.
+ struct StopCountAndDesc {
+ uint32_t count;
+ char* desc;
+ };
+ StopCountAndDesc watched_stops_[kNumOfWatchedStops];
+
+ public:
+ int64_t icount() { return icount_; }
+
+ private:
+ // Exclusive access monitor
+ void exclusiveMonitorSet(uint64_t value);
+ uint64_t exclusiveMonitorGetAndClear(bool* held);
+ void exclusiveMonitorClear();
+
+ bool exclusiveMonitorHeld_;
+ uint64_t exclusiveMonitor_;
+};
+
+// Process wide simulator state.
+class SimulatorProcess {
+ friend class Redirection;
+ friend class AutoLockSimulatorCache;
+
+ private:
+ // ICache checking.
+ struct ICacheHasher {
+ typedef void* Key;
+ typedef void* Lookup;
+ static HashNumber hash(const Lookup& l);
+ static bool match(const Key& k, const Lookup& l);
+ };
+
+ public:
+ typedef HashMap<void*, CachePage*, ICacheHasher, SystemAllocPolicy> ICacheMap;
+
+ static mozilla::Atomic<size_t, mozilla::ReleaseAcquire>
+ ICacheCheckingDisableCount;
+ static void FlushICache(void* start, size_t size);
+
+ static void checkICacheLocked(SimInstruction* instr);
+
+ static bool initialize() {
+ singleton_ = js_new<SimulatorProcess>();
+ return singleton_;
+ }
+ static void destroy() {
+ js_delete(singleton_);
+ singleton_ = nullptr;
+ }
+
+ SimulatorProcess();
+ ~SimulatorProcess();
+
+ private:
+ static SimulatorProcess* singleton_;
+
+ // This lock creates a critical section around 'redirection_' and
+ // 'icache_', which are referenced both by the execution engine
+ // and by the off-thread compiler (see Redirection::Get in the cpp file).
+ Mutex cacheLock_ MOZ_UNANNOTATED;
+
+ Redirection* redirection_;
+ ICacheMap icache_;
+
+ public:
+ static ICacheMap& icache() {
+ // Technically we need the lock to access the innards of the
+ // icache, not to take its address, but the latter condition
+ // serves as a useful complement to the former.
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ return singleton_->icache_;
+ }
+
+ static Redirection* redirection() {
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ return singleton_->redirection_;
+ }
+
+ static void setRedirection(js::jit::Redirection* redirection) {
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ singleton_->redirection_ = redirection;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* JS_SIMULATOR_ARM */
+
+#endif /* jit_arm_Simulator_arm_h */
diff --git a/js/src/jit/arm/Trampoline-arm.cpp b/js/src/jit/arm/Trampoline-arm.cpp
new file mode 100644
index 0000000000..c116a93c64
--- /dev/null
+++ b/js/src/jit/arm/Trampoline-arm.cpp
@@ -0,0 +1,829 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm/SharedICHelpers-arm.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+static const FloatRegisterSet NonVolatileFloatRegs = FloatRegisterSet(
+ (1ULL << FloatRegisters::d8) | (1ULL << FloatRegisters::d9) |
+ (1ULL << FloatRegisters::d10) | (1ULL << FloatRegisters::d11) |
+ (1ULL << FloatRegisters::d12) | (1ULL << FloatRegisters::d13) |
+ (1ULL << FloatRegisters::d14) | (1ULL << FloatRegisters::d15));
+
+static void GenerateReturn(MacroAssembler& masm, int returnCode) {
+ // Restore non-volatile floating point registers.
+ masm.transferMultipleByRuns(NonVolatileFloatRegs, IsLoad, StackPointer, IA);
+
+ // Get rid of padding word.
+ masm.addPtr(Imm32(sizeof(void*)), sp);
+
+ // Set up return value
+ masm.ma_mov(Imm32(returnCode), r0);
+
+ // Pop and return
+ masm.startDataTransferM(IsLoad, sp, IA, WriteBack);
+ masm.transferReg(r4);
+ masm.transferReg(r5);
+ masm.transferReg(r6);
+ masm.transferReg(r7);
+ masm.transferReg(r8);
+ masm.transferReg(r9);
+ masm.transferReg(r10);
+ masm.transferReg(r11);
+ // r12 isn't saved, so it shouldn't be restored.
+ masm.transferReg(pc);
+ masm.finishDataTransfer();
+ masm.flushBuffer();
+}
+
+struct EnterJITStack {
+ double d8;
+ double d9;
+ double d10;
+ double d11;
+ double d12;
+ double d13;
+ double d14;
+ double d15;
+
+ // Padding.
+ void* padding;
+
+ // Non-volatile registers.
+ void* r4;
+ void* r5;
+ void* r6;
+ void* r7;
+ void* r8;
+ void* r9;
+ void* r10;
+ void* r11;
+ // The abi does not expect r12 (ip) to be preserved
+ void* lr;
+
+ // Arguments.
+ // code == r0
+ // argc == r1
+ // argv == r2
+ // frame == r3
+ CalleeToken token;
+ JSObject* scopeChain;
+ size_t numStackValues;
+ Value* vp;
+};
+
+/*
+ * This method generates a trampoline for a c++ function with the following
+ * signature:
+ * void enter(void* code, int argc, Value* argv, InterpreterFrame* fp,
+ * CalleeToken calleeToken, JSObject* scopeChain, Value* vp)
+ * ...using standard EABI calling convention
+ */
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ const Address slot_token(sp, offsetof(EnterJITStack, token));
+ const Address slot_vp(sp, offsetof(EnterJITStack, vp));
+
+ static_assert(OsrFrameReg == r3);
+
+ Assembler* aasm = &masm;
+
+ // Save non-volatile registers. These must be saved by the trampoline,
+ // rather than the JIT'd code, because they are scanned by the conservative
+ // scanner.
+ masm.startDataTransferM(IsStore, sp, DB, WriteBack);
+ masm.transferReg(r4); // [sp,0]
+ masm.transferReg(r5); // [sp,4]
+ masm.transferReg(r6); // [sp,8]
+ masm.transferReg(r7); // [sp,12]
+ masm.transferReg(r8); // [sp,16]
+ masm.transferReg(r9); // [sp,20]
+ masm.transferReg(r10); // [sp,24]
+ masm.transferReg(r11); // [sp,28]
+ // The abi does not expect r12 (ip) to be preserved
+ masm.transferReg(lr); // [sp,32]
+ // The 5th argument is located at [sp, 36]
+ masm.finishDataTransfer();
+
+ // Add padding word.
+ masm.subPtr(Imm32(sizeof(void*)), sp);
+
+ // Push the float registers.
+ masm.transferMultipleByRuns(NonVolatileFloatRegs, IsStore, sp, DB);
+
+ // Load calleeToken into r9.
+ masm.loadPtr(slot_token, r9);
+
+ // Save stack pointer.
+ masm.movePtr(sp, r11);
+
+ // Load the number of actual arguments into r10.
+ masm.loadPtr(slot_vp, r10);
+ masm.unboxInt32(Address(r10, 0), r10);
+
+ {
+ Label noNewTarget;
+ masm.branchTest32(Assembler::Zero, r9,
+ Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+ masm.add32(Imm32(1), r1);
+
+ masm.bind(&noNewTarget);
+ }
+
+ // Guarantee stack alignment of Jit frames.
+ //
+ // This code moves the stack pointer to the location where it should be when
+ // we enter the Jit frame. It moves the stack pointer such that we have
+ // enough space reserved for pushing the arguments, and the JitFrameLayout.
+ // The stack pointer is also aligned on the alignment expected by the Jit
+ // frames.
+ //
+ // At the end the register r4, is a pointer to the stack where the first
+ // argument is expected by the Jit frame.
+ //
+ aasm->as_sub(r4, sp, O2RegImmShift(r1, LSL, 3)); // r4 = sp - argc*8
+ aasm->as_bic(r4, r4, Imm8(JitStackAlignment - 1));
+ // r4 is now the aligned on the bottom of the list of arguments.
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ // sp' = ~(JitStackAlignment - 1) & (sp - argc * sizeof(Value))
+ masm.movePtr(r4, sp);
+
+ // Get a copy of the number of args to use as a decrement counter, also set
+ // the zero condition code.
+ aasm->as_mov(r5, O2Reg(r1), SetCC);
+
+ // Loop over arguments, copying them from an unknown buffer onto the Ion
+ // stack so they can be accessed from JIT'ed code.
+ {
+ Label header, footer;
+ // If there aren't any arguments, don't do anything.
+ aasm->as_b(&footer, Assembler::Zero);
+ // Get the top of the loop.
+ masm.bind(&header);
+ aasm->as_sub(r5, r5, Imm8(1), SetCC);
+ // We could be more awesome, and unroll this, using a loadm
+ // (particularly since the offset is effectively 0) but that seems more
+ // error prone, and complex.
+ // BIG FAT WARNING: this loads both r6 and r7.
+ aasm->as_extdtr(IsLoad, 64, true, PostIndex, r6,
+ EDtrAddr(r2, EDtrOffImm(8)));
+ aasm->as_extdtr(IsStore, 64, true, PostIndex, r6,
+ EDtrAddr(r4, EDtrOffImm(8)));
+ aasm->as_b(&header, Assembler::NonZero);
+ masm.bind(&footer);
+ }
+
+ // Push the callee token.
+ masm.push(r9);
+
+ // Push the frame descriptor.
+ masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, r10, r10);
+
+ Label returnLabel;
+ {
+ // Handle Interpreter -> Baseline OSR.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(r11));
+ regs.take(OsrFrameReg);
+ regs.take(r0); // jitcode
+ MOZ_ASSERT(!regs.has(ReturnReg), "ReturnReg matches r0");
+
+ const Address slot_numStackValues(r11,
+ offsetof(EnterJITStack, numStackValues));
+
+ Label notOsr;
+ masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, &notOsr);
+
+ Register scratch = regs.takeAny();
+
+ Register numStackValues = regs.takeAny();
+ masm.load32(slot_numStackValues, numStackValues);
+
+ // Write return address. On ARM, CodeLabel is only used for tableswitch,
+ // so we can't use it here to get the return address. Instead, we use pc
+ // + a fixed offset to a jump to returnLabel. The pc register holds pc +
+ // 8, so we add the size of 2 instructions to skip the instructions
+ // emitted by push and jump(&skipJump).
+ {
+ AutoForbidPoolsAndNops afp(&masm, 5);
+ Label skipJump;
+ masm.mov(pc, scratch);
+ masm.addPtr(Imm32(2 * sizeof(uint32_t)), scratch);
+ masm.push(scratch);
+ masm.jump(&skipJump);
+ masm.jump(&returnLabel);
+ masm.bind(&skipJump);
+ }
+
+ // Frame prologue.
+ masm.push(FramePointer);
+ masm.mov(sp, FramePointer);
+
+ // Reserve frame.
+ masm.subPtr(Imm32(BaselineFrame::Size()), sp);
+
+ Register framePtrScratch = regs.takeAny();
+ masm.touchFrameValues(numStackValues, scratch, framePtrScratch);
+ masm.mov(sp, framePtrScratch);
+
+ // Reserve space for locals and stack values.
+ masm.ma_lsl(Imm32(3), numStackValues, scratch);
+ masm.ma_sub(sp, scratch, sp);
+
+ // Enter exit frame.
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(Imm32(0)); // Fake return address.
+ masm.push(FramePointer);
+ // No GC things to mark on the stack, push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ masm.push(r0); // jitcode
+
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(framePtrScratch); // BaselineFrame
+ masm.passABIArg(OsrFrameReg); // InterpreterFrame
+ masm.passABIArg(numStackValues);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Register jitcode = regs.takeAny();
+ masm.pop(jitcode);
+
+ MOZ_ASSERT(jitcode != ReturnReg);
+
+ Label error;
+ masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), sp);
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.profilerEnterFrame(FramePointer, scratch);
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+ masm.jump(jitcode);
+
+ // OOM: frame epilogue, load error value, discard return address and return.
+ masm.bind(&error);
+ masm.mov(FramePointer, sp);
+ masm.pop(FramePointer);
+ masm.addPtr(Imm32(sizeof(uintptr_t)), sp); // Return address.
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.jump(&returnLabel);
+
+ masm.bind(&notOsr);
+ // Load the scope chain in R1.
+ MOZ_ASSERT(R1.scratchReg() != r0);
+ masm.loadPtr(Address(r11, offsetof(EnterJITStack, scopeChain)),
+ R1.scratchReg());
+ }
+
+ // The callee will push the return address and frame pointer on the stack,
+ // thus we check that the stack would be aligned once the call is complete.
+ masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+ // Call the function.
+ masm.callJitNoProfiler(r0);
+
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&returnLabel);
+
+ // Discard arguments and padding. Set sp to the address of the EnterJITStack
+ // on the stack.
+ masm.mov(r11, sp);
+
+ // Store the returned value into the slot_vp
+ masm.loadPtr(slot_vp, r5);
+ masm.storeValue(JSReturnOperand, Address(r5, 0));
+
+ // Restore non-volatile registers and return.
+ GenerateReturn(masm, true);
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ // Not supported, or not implemented yet.
+ // TODO: Implement along with the corresponding stack-walker changes, in
+ // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
+ return mozilla::Nothing{};
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+ // See large comment in x86's JitRuntime::generateInvalidator.
+
+ AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // At this point, one of two things has happened:
+ // 1) Execution has just returned from C code, which left the stack aligned
+ // 2) Execution has just returned from Ion code, which left the stack
+ // unaligned. The old return address should not matter, but we still want the
+ // stack to be aligned, and there is no good reason to automatically align it
+ // with a call to setupUnalignedABICall.
+ masm.as_bic(sp, sp, Imm8(7));
+ masm.startDataTransferM(IsStore, sp, DB, WriteBack);
+ // We don't have to push everything, but this is likely easier.
+ // Setting regs_.
+ for (uint32_t i = 0; i < Registers::Total; i++) {
+ masm.transferReg(Register::FromCode(i));
+ }
+ masm.finishDataTransfer();
+
+ // Since our datastructures for stack inspection are compile-time fixed,
+ // if there are only 16 double registers, then we need to reserve
+ // space on the stack for the missing 16.
+ if (FloatRegisters::ActualTotalPhys() != FloatRegisters::TotalPhys) {
+ ScratchRegisterScope scratch(masm);
+ int missingRegs =
+ FloatRegisters::TotalPhys - FloatRegisters::ActualTotalPhys();
+ masm.ma_sub(Imm32(missingRegs * sizeof(double)), sp, scratch);
+ }
+
+ masm.startFloatTransferM(IsStore, sp, DB, WriteBack);
+ for (uint32_t i = 0; i < FloatRegisters::ActualTotalPhys(); i++) {
+ masm.transferFloatReg(FloatRegister(i, FloatRegister::Double));
+ }
+ masm.finishFloatTransfer();
+
+ masm.ma_mov(sp, r0);
+ // Reserve 8 bytes for the outparam to ensure alignment for
+ // setupAlignedABICall.
+ masm.reserveStack(sizeof(void*) * 2);
+ masm.mov(sp, r1);
+ using Fn =
+ bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupAlignedABICall();
+ masm.passABIArg(r0);
+ masm.passABIArg(r1);
+ masm.callWithABI<Fn, InvalidationBailout>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(r2); // Get bailoutInfo outparam.
+
+ // Pop the machine state and the dead frame.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+ masm.pushReturnAddress();
+
+ // Frame prologue.
+ //
+ // NOTE: if this changes, fix the Baseline bailout code too!
+ // See BaselineStackBuilder::calculatePrevFramePtr and
+ // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+ masm.push(FramePointer);
+ masm.mov(StackPointer, FramePointer);
+
+ static_assert(JitStackAlignment == sizeof(Value));
+
+ // Copy number of actual arguments into r0 and r8.
+ masm.loadNumActualArgs(FramePointer, r0);
+ masm.mov(r0, r8);
+
+ // Load the number of |undefined|s to push into r6.
+ masm.loadPtr(
+ Address(FramePointer, RectifierFrameLayout::offsetOfCalleeToken()), r1);
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_and(Imm32(CalleeTokenMask), r1, r6, scratch);
+ }
+ masm.loadFunctionArgCount(r6, r6);
+
+ masm.ma_sub(r6, r8, r2);
+
+ // Get the topmost argument.
+ {
+ ScratchRegisterScope scratch(masm);
+ masm.ma_alu(sp, lsl(r8, 3), r3, OpAdd); // r3 <- sp + nargs * 8
+ masm.ma_add(r3, Imm32(sizeof(RectifierFrameLayout)), r3, scratch);
+ }
+
+ {
+ Label notConstructing;
+
+ masm.branchTest32(Assembler::Zero, r1,
+ Imm32(CalleeToken_FunctionConstructing),
+ &notConstructing);
+
+ // Add sizeof(Value) to overcome |this|
+ masm.as_extdtr(IsLoad, 64, true, Offset, r4, EDtrAddr(r3, EDtrOffImm(8)));
+ masm.as_extdtr(IsStore, 64, true, PreIndex, r4,
+ EDtrAddr(sp, EDtrOffImm(-8)));
+
+ masm.bind(&notConstructing);
+ }
+
+ // Push undefined.
+ masm.moveValue(UndefinedValue(), ValueOperand(r5, r4));
+ {
+ Label undefLoopTop;
+ masm.bind(&undefLoopTop);
+ masm.as_extdtr(IsStore, 64, true, PreIndex, r4,
+ EDtrAddr(sp, EDtrOffImm(-8)));
+ masm.as_sub(r2, r2, Imm8(1), SetCC);
+
+ masm.ma_b(&undefLoopTop, Assembler::NonZero);
+ }
+
+ // Push arguments, |nargs| + 1 times (to include |this|).
+ {
+ Label copyLoopTop;
+ masm.bind(&copyLoopTop);
+ masm.as_extdtr(IsLoad, 64, true, PostIndex, r4,
+ EDtrAddr(r3, EDtrOffImm(-8)));
+ masm.as_extdtr(IsStore, 64, true, PreIndex, r4,
+ EDtrAddr(sp, EDtrOffImm(-8)));
+
+ masm.as_sub(r8, r8, Imm8(1), SetCC);
+ masm.ma_b(&copyLoopTop, Assembler::NotSigned);
+ }
+
+ // Construct JitFrameLayout.
+ masm.ma_push(r1); // callee token
+ masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, r0, r0);
+
+ // Call the target function.
+ masm.andPtr(Imm32(CalleeTokenMask), r1);
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(r1, r3);
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(r3);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(r1, r3, &noBaselineScript);
+ masm.callJitNoProfiler(r3);
+ masm.jump(&done);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(r1, r3);
+ masm.callJitNoProfiler(r3);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.mov(FramePointer, StackPointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ // STEP 1a: Save our register sets to the stack so Bailout() can read
+ // everything.
+ // sp % 8 == 0
+
+ masm.startDataTransferM(IsStore, sp, DB, WriteBack);
+ // We don't have to push everything, but this is likely easier.
+ // Setting regs_.
+ for (uint32_t i = 0; i < Registers::Total; i++) {
+ masm.transferReg(Register::FromCode(i));
+ }
+ masm.finishDataTransfer();
+
+ ScratchRegisterScope scratch(masm);
+
+ // Since our datastructures for stack inspection are compile-time fixed,
+ // if there are only 16 double registers, then we need to reserve
+ // space on the stack for the missing 16.
+ if (FloatRegisters::ActualTotalPhys() != FloatRegisters::TotalPhys) {
+ int missingRegs =
+ FloatRegisters::TotalPhys - FloatRegisters::ActualTotalPhys();
+ masm.ma_sub(Imm32(missingRegs * sizeof(double)), sp, scratch);
+ }
+ masm.startFloatTransferM(IsStore, sp, DB, WriteBack);
+ for (uint32_t i = 0; i < FloatRegisters::ActualTotalPhys(); i++) {
+ masm.transferFloatReg(FloatRegister(i, FloatRegister::Double));
+ }
+ masm.finishFloatTransfer();
+
+ // The current stack pointer is the first argument to jit::Bailout.
+ masm.ma_mov(sp, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+ PushBailoutFrame(masm, r0);
+
+ // Make space for Bailout's bailoutInfo outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.mov(sp, r1);
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupAlignedABICall();
+
+ masm.passABIArg(r0);
+ masm.passABIArg(r1);
+
+ masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+ masm.pop(r2); // Get the bailoutInfo outparam.
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+ *wrapperOffset = startTrampolineCode(masm);
+
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert(
+ (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set must be a superset of Volatile register set.");
+
+ // The context is the first argument; r0 is the first argument register.
+ Register cxreg = r0;
+ regs.take(cxreg);
+
+ // Stack is:
+ // ... frame ...
+ // +8 [args] + argPadding
+ // +0 ExitFrame
+ //
+ // If it isn't a tail call, then the return address needs to be saved.
+ // Push the frame pointer to finish the exit frame, then link it up.
+ if (f.expectTailCall == NonTailCall) {
+ masm.pushReturnAddress();
+ }
+ masm.Push(FramePointer);
+ masm.loadJSContext(cxreg);
+ masm.enterExitFrame(cxreg, regs.getAny(), &f);
+
+ // Save the base of the argument set stored on the stack.
+ Register argsBase = InvalidReg;
+ if (f.explicitArgs) {
+ argsBase = r5;
+ regs.take(argsBase);
+ ScratchRegisterScope scratch(masm);
+ masm.ma_add(sp, Imm32(ExitFrameLayout::SizeWithFooter()), argsBase,
+ scratch);
+ }
+
+ // Reserve space for the outparameter.
+ Register outReg = InvalidReg;
+ switch (f.outParam) {
+ case Type_Value:
+ outReg = r4;
+ regs.take(outReg);
+ masm.reserveStack(sizeof(Value));
+ masm.ma_mov(sp, outReg);
+ break;
+
+ case Type_Handle:
+ outReg = r4;
+ regs.take(outReg);
+ masm.PushEmptyRooted(f.outParamRootType);
+ masm.ma_mov(sp, outReg);
+ break;
+
+ case Type_Int32:
+ case Type_Pointer:
+ case Type_Bool:
+ outReg = r4;
+ regs.take(outReg);
+ masm.reserveStack(sizeof(int32_t));
+ masm.ma_mov(sp, outReg);
+ break;
+
+ case Type_Double:
+ outReg = r4;
+ regs.take(outReg);
+ masm.reserveStack(sizeof(double));
+ masm.ma_mov(sp, outReg);
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ masm.setupUnalignedABICall(regs.getAny());
+ masm.passABIArg(cxreg);
+
+ size_t argDisp = 0;
+
+ // Copy any arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByValue:
+ // Values should be passed by reference, not by value, so we assert
+ // that the argument is a double-precision float.
+ MOZ_ASSERT(f.argPassedInFloatReg(explicitArg));
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
+ argDisp += sizeof(double);
+ break;
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += 2 * sizeof(void*);
+ break;
+ }
+ }
+
+ // Copy the implicit outparam, if any.
+ if (outReg != InvalidReg) {
+ masm.passABIArg(outReg);
+ }
+
+ masm.callWithABI(nativeFun, MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, r0, r0, masm.failureLabel());
+ break;
+ case Type_Bool:
+ masm.branchIfFalseBool(r0, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ // Load the outparam and free any allocated stack.
+ switch (f.outParam) {
+ case Type_Handle:
+ masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+ break;
+
+ case Type_Value:
+ masm.loadValue(Address(sp, 0), JSReturnOperand);
+ masm.freeStack(sizeof(Value));
+ break;
+
+ case Type_Int32:
+ case Type_Pointer:
+ masm.load32(Address(sp, 0), ReturnReg);
+ masm.freeStack(sizeof(int32_t));
+ break;
+
+ case Type_Bool:
+ masm.load8ZeroExtend(Address(sp, 0), ReturnReg);
+ masm.freeStack(sizeof(int32_t));
+ break;
+
+ case Type_Double:
+ masm.loadDouble(Address(sp, 0), ReturnDoubleReg);
+ masm.freeStack(sizeof(double));
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ // Until C++ code is instrumented against Spectre, prevent speculative
+ // execution from returning any private data.
+ if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
+ masm.speculationBarrier();
+ }
+
+ // Pop ExitFooterFrame and the frame pointer.
+ masm.leaveExitFrame(sizeof(void*));
+
+ // Return. Subtract sizeof(void*) for the frame pointer.
+ masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+ f.explicitStackSlots() * sizeof(void*) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type) {
+ AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+ uint32_t offset = startTrampolineCode(masm);
+
+ masm.pushReturnAddress();
+
+ static_assert(PreBarrierReg == r1);
+ Register temp1 = r2;
+ Register temp2 = r3;
+ Register temp3 = r4;
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+ &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ LiveRegisterSet save;
+ save.set() =
+ RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileDoubleMask));
+ masm.PushRegsInMask(save);
+
+ masm.movePtr(ImmPtr(cx->runtime()), r0);
+
+ masm.setupUnalignedABICall(r2);
+ masm.passABIArg(r0);
+ masm.passABIArg(r1);
+ masm.callWithABI(JitPreWriteBarrier(type));
+ masm.PopRegsInMask(save);
+ masm.ret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.ret();
+
+ return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+ masm.bind(bailoutTail);
+ masm.generateBailoutTail(r1, r2);
+}
diff --git a/js/src/jit/arm/disasm/Constants-arm.cpp b/js/src/jit/arm/disasm/Constants-arm.cpp
new file mode 100644
index 0000000000..408e2df686
--- /dev/null
+++ b/js/src/jit/arm/disasm/Constants-arm.cpp
@@ -0,0 +1,117 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ */
+// Copyright 2009 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "jit/arm/disasm/Constants-arm.h"
+
+#ifdef JS_DISASM_ARM
+
+namespace js {
+namespace jit {
+namespace disasm {
+
+double Instruction::DoubleImmedVmov() const {
+ // Reconstruct a double from the immediate encoded in the vmov instruction.
+ //
+ // instruction: [xxxxxxxx,xxxxabcd,xxxxxxxx,xxxxefgh]
+ // double: [aBbbbbbb,bbcdefgh,00000000,00000000,
+ // 00000000,00000000,00000000,00000000]
+ //
+ // where B = ~b. Only the high 16 bits are affected.
+ uint64_t high16;
+ high16 = (Bits(17, 16) << 4) | Bits(3, 0); // xxxxxxxx,xxcdefgh.
+ high16 |= (0xff * Bit(18)) << 6; // xxbbbbbb,bbxxxxxx.
+ high16 |= (Bit(18) ^ 1) << 14; // xBxxxxxx,xxxxxxxx.
+ high16 |= Bit(19) << 15; // axxxxxxx,xxxxxxxx.
+
+ uint64_t imm = high16 << 48;
+ double d;
+ memcpy(&d, &imm, 8);
+ return d;
+}
+
+// These register names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+const char* Registers::names_[kNumRegisters] = {
+ "r0", "r1", "r2", "r3", "r4", "r5", "r6", "r7",
+ "r8", "r9", "r10", "fp", "ip", "sp", "lr", "pc",
+};
+
+// List of alias names which can be used when referring to ARM registers.
+const Registers::RegisterAlias Registers::aliases_[] = {
+ {10, "sl"}, {11, "r11"}, {12, "r12"}, {13, "r13"},
+ {14, "r14"}, {15, "r15"}, {kNoRegister, NULL}};
+
+const char* Registers::Name(int reg) {
+ const char* result;
+ if ((0 <= reg) && (reg < kNumRegisters)) {
+ result = names_[reg];
+ } else {
+ result = "noreg";
+ }
+ return result;
+}
+
+// Support for VFP registers s0 to s31 (d0 to d15) and d16-d31.
+// Note that "sN:sM" is the same as "dN/2" up to d15.
+// These register names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+const char* VFPRegisters::names_[kNumVFPRegisters] = {
+ "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9", "s10",
+ "s11", "s12", "s13", "s14", "s15", "s16", "s17", "s18", "s19", "s20", "s21",
+ "s22", "s23", "s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31", "d0",
+ "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9", "d10", "d11",
+ "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19", "d20", "d21", "d22",
+ "d23", "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"};
+
+const char* VFPRegisters::Name(int reg, bool is_double) {
+ MOZ_ASSERT((0 <= reg) && (reg < kNumVFPRegisters));
+ return names_[reg + (is_double ? kNumVFPSingleRegisters : 0)];
+}
+
+int VFPRegisters::Number(const char* name, bool* is_double) {
+ for (int i = 0; i < kNumVFPRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ if (i < kNumVFPSingleRegisters) {
+ *is_double = false;
+ return i;
+ } else {
+ *is_double = true;
+ return i - kNumVFPSingleRegisters;
+ }
+ }
+ }
+
+ // No register with the requested name found.
+ return kNoRegister;
+}
+
+int Registers::Number(const char* name) {
+ // Look through the canonical names.
+ for (int i = 0; i < kNumRegisters; i++) {
+ if (strcmp(names_[i], name) == 0) {
+ return i;
+ }
+ }
+
+ // Look through the alias names.
+ int i = 0;
+ while (aliases_[i].reg != kNoRegister) {
+ if (strcmp(aliases_[i].name, name) == 0) {
+ return aliases_[i].reg;
+ }
+ i++;
+ }
+
+ // No register with the requested name found.
+ return kNoRegister;
+}
+
+} // namespace disasm
+} // namespace jit
+} // namespace js
+
+#endif // JS_DISASM_ARM
diff --git a/js/src/jit/arm/disasm/Constants-arm.h b/js/src/jit/arm/disasm/Constants-arm.h
new file mode 100644
index 0000000000..0128062b3f
--- /dev/null
+++ b/js/src/jit/arm/disasm/Constants-arm.h
@@ -0,0 +1,684 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ */
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef jit_arm_disasm_Constants_arm_h
+#define jit_arm_disasm_Constants_arm_h
+
+#ifdef JS_DISASM_ARM
+
+# include "mozilla/Assertions.h"
+# include "mozilla/Types.h"
+
+# include <string.h>
+
+namespace js {
+namespace jit {
+namespace disasm {
+
+// Constant pool marker.
+// Use UDF, the permanently undefined instruction.
+const int kConstantPoolMarkerMask = 0xfff000f0;
+const int kConstantPoolMarker = 0xe7f000f0;
+const int kConstantPoolLengthMaxMask = 0xffff;
+
+inline int EncodeConstantPoolLength(int length) {
+ MOZ_ASSERT((length & kConstantPoolLengthMaxMask) == length);
+ return ((length & 0xfff0) << 4) | (length & 0xf);
+}
+
+inline int DecodeConstantPoolLength(int instr) {
+ MOZ_ASSERT((instr & kConstantPoolMarkerMask) == kConstantPoolMarker);
+ return ((instr >> 4) & 0xfff0) | (instr & 0xf);
+}
+
+// Used in code age prologue - ldr(pc, MemOperand(pc, -4))
+const int kCodeAgeJumpInstruction = 0xe51ff004;
+
+// Number of registers in normal ARM mode.
+const int kNumRegisters = 16;
+
+// VFP support.
+const int kNumVFPSingleRegisters = 32;
+const int kNumVFPDoubleRegisters = 32;
+const int kNumVFPRegisters = kNumVFPSingleRegisters + kNumVFPDoubleRegisters;
+
+// PC is register 15.
+const int kPCRegister = 15;
+const int kNoRegister = -1;
+
+// -----------------------------------------------------------------------------
+// Conditions.
+
+// Defines constants and accessor classes to assemble, disassemble and
+// simulate ARM instructions.
+//
+// Section references in the code refer to the "ARM Architecture Reference
+// Manual" from July 2005 (available at http://www.arm.com/miscPDFs/14128.pdf)
+//
+// Constants for specific fields are defined in their respective named enums.
+// General constants are in an anonymous enum in class Instr.
+
+// Values for the condition field as defined in section A3.2
+enum Condition {
+ kNoCondition = -1,
+
+ eq = 0 << 28, // Z set Equal.
+ ne = 1 << 28, // Z clear Not equal.
+ cs = 2 << 28, // C set Unsigned higher or same.
+ cc = 3 << 28, // C clear Unsigned lower.
+ mi = 4 << 28, // N set Negative.
+ pl = 5 << 28, // N clear Positive or zero.
+ vs = 6 << 28, // V set Overflow.
+ vc = 7 << 28, // V clear No overflow.
+ hi = 8 << 28, // C set, Z clear Unsigned higher.
+ ls = 9 << 28, // C clear or Z set Unsigned lower or same.
+ ge = 10 << 28, // N == V Greater or equal.
+ lt = 11 << 28, // N != V Less than.
+ gt = 12 << 28, // Z clear, N == V Greater than.
+ le = 13 << 28, // Z set or N != V Less then or equal
+ al = 14 << 28, // Always.
+
+ kSpecialCondition = 15 << 28, // Special condition (refer to section A3.2.1).
+ kNumberOfConditions = 16,
+
+ // Aliases.
+ hs = cs, // C set Unsigned higher or same.
+ lo = cc // C clear Unsigned lower.
+};
+
+inline Condition NegateCondition(Condition cond) {
+ MOZ_ASSERT(cond != al);
+ return static_cast<Condition>(cond ^ ne);
+}
+
+// Commute a condition such that {a cond b == b cond' a}.
+inline Condition CommuteCondition(Condition cond) {
+ switch (cond) {
+ case lo:
+ return hi;
+ case hi:
+ return lo;
+ case hs:
+ return ls;
+ case ls:
+ return hs;
+ case lt:
+ return gt;
+ case gt:
+ return lt;
+ case ge:
+ return le;
+ case le:
+ return ge;
+ default:
+ return cond;
+ }
+}
+
+// -----------------------------------------------------------------------------
+// Instructions encoding.
+
+// Instr is merely used by the Assembler to distinguish 32bit integers
+// representing instructions from usual 32 bit values.
+// Instruction objects are pointers to 32bit values, and provide methods to
+// access the various ISA fields.
+typedef int32_t Instr;
+
+// Opcodes for Data-processing instructions (instructions with a type 0 and 1)
+// as defined in section A3.4
+enum Opcode {
+ AND = 0 << 21, // Logical AND.
+ EOR = 1 << 21, // Logical Exclusive OR.
+ SUB = 2 << 21, // Subtract.
+ RSB = 3 << 21, // Reverse Subtract.
+ ADD = 4 << 21, // Add.
+ ADC = 5 << 21, // Add with Carry.
+ SBC = 6 << 21, // Subtract with Carry.
+ RSC = 7 << 21, // Reverse Subtract with Carry.
+ TST = 8 << 21, // Test.
+ TEQ = 9 << 21, // Test Equivalence.
+ CMP = 10 << 21, // Compare.
+ CMN = 11 << 21, // Compare Negated.
+ ORR = 12 << 21, // Logical (inclusive) OR.
+ MOV = 13 << 21, // Move.
+ BIC = 14 << 21, // Bit Clear.
+ MVN = 15 << 21 // Move Not.
+};
+
+// The bits for bit 7-4 for some type 0 miscellaneous instructions.
+enum MiscInstructionsBits74 {
+ // With bits 22-21 01.
+ BX = 1 << 4,
+ BXJ = 2 << 4,
+ BLX = 3 << 4,
+ BKPT = 7 << 4,
+
+ // With bits 22-21 11.
+ CLZ = 1 << 4
+};
+
+// Load and store exclusive instructions.
+
+// Bit positions.
+enum {
+ ExclusiveOpHi = 24, // Hi bit of opcode field
+ ExclusiveOpLo = 23, // Lo bit of opcode field
+ ExclusiveSizeHi = 22, // Hi bit of operand size field
+ ExclusiveSizeLo = 21, // Lo bit of operand size field
+ ExclusiveLoad = 20 // Bit indicating load
+};
+
+// Opcode bits for exclusive instructions.
+enum { ExclusiveOpcode = 3 };
+
+// Operand size, Bits(ExclusiveSizeHi,ExclusiveSizeLo).
+enum {
+ ExclusiveWord = 0,
+ ExclusiveDouble = 1,
+ ExclusiveByte = 2,
+ ExclusiveHalf = 3
+};
+
+// Instruction encoding bits and masks.
+enum {
+ H = 1 << 5, // Halfword (or byte).
+ S6 = 1 << 6, // Signed (or unsigned).
+ L = 1 << 20, // Load (or store).
+ S = 1 << 20, // Set condition code (or leave unchanged).
+ W = 1 << 21, // Writeback base register (or leave unchanged).
+ A = 1 << 21, // Accumulate in multiply instruction (or not).
+ B = 1 << 22, // Unsigned byte (or word).
+ N = 1 << 22, // Long (or short).
+ U = 1 << 23, // Positive (or negative) offset/index.
+ P = 1 << 24, // Offset/pre-indexed addressing (or post-indexed addressing).
+ I = 1 << 25, // Immediate shifter operand (or not).
+ B0 = 1 << 0,
+ B4 = 1 << 4,
+ B5 = 1 << 5,
+ B6 = 1 << 6,
+ B7 = 1 << 7,
+ B8 = 1 << 8,
+ B9 = 1 << 9,
+ B12 = 1 << 12,
+ B16 = 1 << 16,
+ B17 = 1 << 17,
+ B18 = 1 << 18,
+ B19 = 1 << 19,
+ B20 = 1 << 20,
+ B21 = 1 << 21,
+ B22 = 1 << 22,
+ B23 = 1 << 23,
+ B24 = 1 << 24,
+ B25 = 1 << 25,
+ B26 = 1 << 26,
+ B27 = 1 << 27,
+ B28 = 1 << 28,
+
+ // Instruction bit masks.
+ kCondMask = 15 << 28,
+ kALUMask = 0x6f << 21,
+ kRdMask = 15 << 12, // In str instruction.
+ kCoprocessorMask = 15 << 8,
+ kOpCodeMask = 15 << 21, // In data-processing instructions.
+ kImm24Mask = (1 << 24) - 1,
+ kImm16Mask = (1 << 16) - 1,
+ kImm8Mask = (1 << 8) - 1,
+ kOff12Mask = (1 << 12) - 1,
+ kOff8Mask = (1 << 8) - 1
+};
+
+// -----------------------------------------------------------------------------
+// Addressing modes and instruction variants.
+
+// Condition code updating mode.
+enum SBit {
+ SetCC = 1 << 20, // Set condition code.
+ LeaveCC = 0 << 20 // Leave condition code unchanged.
+};
+
+// Status register selection.
+enum SRegister { CPSR = 0 << 22, SPSR = 1 << 22 };
+
+// Shifter types for Data-processing operands as defined in section A5.1.2.
+enum ShiftOp {
+ LSL = 0 << 5, // Logical shift left.
+ LSR = 1 << 5, // Logical shift right.
+ ASR = 2 << 5, // Arithmetic shift right.
+ ROR = 3 << 5, // Rotate right.
+
+ // RRX is encoded as ROR with shift_imm == 0.
+ // Use a special code to make the distinction. The RRX ShiftOp is only used
+ // as an argument, and will never actually be encoded. The Assembler will
+ // detect it and emit the correct ROR shift operand with shift_imm == 0.
+ RRX = -1,
+ kNumberOfShifts = 4
+};
+
+// Status register fields.
+enum SRegisterField {
+ CPSR_c = CPSR | 1 << 16,
+ CPSR_x = CPSR | 1 << 17,
+ CPSR_s = CPSR | 1 << 18,
+ CPSR_f = CPSR | 1 << 19,
+ SPSR_c = SPSR | 1 << 16,
+ SPSR_x = SPSR | 1 << 17,
+ SPSR_s = SPSR | 1 << 18,
+ SPSR_f = SPSR | 1 << 19
+};
+
+// Status register field mask (or'ed SRegisterField enum values).
+typedef uint32_t SRegisterFieldMask;
+
+// Memory operand addressing mode.
+enum AddrMode {
+ // Bit encoding P U W.
+ Offset = (8 | 4 | 0) << 21, // Offset (without writeback to base).
+ PreIndex = (8 | 4 | 1) << 21, // Pre-indexed addressing with writeback.
+ PostIndex = (0 | 4 | 0) << 21, // Post-indexed addressing with writeback.
+ NegOffset =
+ (8 | 0 | 0) << 21, // Negative offset (without writeback to base).
+ NegPreIndex = (8 | 0 | 1) << 21, // Negative pre-indexed with writeback.
+ NegPostIndex = (0 | 0 | 0) << 21 // Negative post-indexed with writeback.
+};
+
+// Load/store multiple addressing mode.
+enum BlockAddrMode {
+ // Bit encoding P U W .
+ da = (0 | 0 | 0) << 21, // Decrement after.
+ ia = (0 | 4 | 0) << 21, // Increment after.
+ db = (8 | 0 | 0) << 21, // Decrement before.
+ ib = (8 | 4 | 0) << 21, // Increment before.
+ da_w = (0 | 0 | 1) << 21, // Decrement after with writeback to base.
+ ia_w = (0 | 4 | 1) << 21, // Increment after with writeback to base.
+ db_w = (8 | 0 | 1) << 21, // Decrement before with writeback to base.
+ ib_w = (8 | 4 | 1) << 21, // Increment before with writeback to base.
+
+ // Alias modes for comparison when writeback does not matter.
+ da_x = (0 | 0 | 0) << 21, // Decrement after.
+ ia_x = (0 | 4 | 0) << 21, // Increment after.
+ db_x = (8 | 0 | 0) << 21, // Decrement before.
+ ib_x = (8 | 4 | 0) << 21, // Increment before.
+
+ kBlockAddrModeMask = (8 | 4 | 1) << 21
+};
+
+// Coprocessor load/store operand size.
+enum LFlag {
+ Long = 1 << 22, // Long load/store coprocessor.
+ Short = 0 << 22 // Short load/store coprocessor.
+};
+
+// NEON data type
+enum NeonDataType {
+ NeonS8 = 0x1, // U = 0, imm3 = 0b001
+ NeonS16 = 0x2, // U = 0, imm3 = 0b010
+ NeonS32 = 0x4, // U = 0, imm3 = 0b100
+ NeonU8 = 1 << 24 | 0x1, // U = 1, imm3 = 0b001
+ NeonU16 = 1 << 24 | 0x2, // U = 1, imm3 = 0b010
+ NeonU32 = 1 << 24 | 0x4, // U = 1, imm3 = 0b100
+ NeonDataTypeSizeMask = 0x7,
+ NeonDataTypeUMask = 1 << 24
+};
+
+enum NeonListType { nlt_1 = 0x7, nlt_2 = 0xA, nlt_3 = 0x6, nlt_4 = 0x2 };
+
+enum NeonSize { Neon8 = 0x0, Neon16 = 0x1, Neon32 = 0x2, Neon64 = 0x3 };
+
+// -----------------------------------------------------------------------------
+// Supervisor Call (svc) specific support.
+
+// Special Software Interrupt codes when used in the presence of the ARM
+// simulator.
+// svc (formerly swi) provides a 24bit immediate value. Use bits 22:0 for
+// standard SoftwareInterrupCode. Bit 23 is reserved for the stop feature.
+enum SoftwareInterruptCodes {
+ // transition to C code
+ kCallRtRedirected = 0x10,
+ // break point
+ kBreakpoint = 0x20,
+ // stop
+ kStopCode = 1 << 23
+};
+const uint32_t kStopCodeMask = kStopCode - 1;
+const uint32_t kMaxStopCode = kStopCode - 1;
+const int32_t kDefaultStopCode = -1;
+
+// Type of VFP register. Determines register encoding.
+enum VFPRegPrecision { kSinglePrecision = 0, kDoublePrecision = 1 };
+
+// VFP FPSCR constants.
+enum VFPConversionMode { kFPSCRRounding = 0, kDefaultRoundToZero = 1 };
+
+// This mask does not include the "inexact" or "input denormal" cumulative
+// exceptions flags, because we usually don't want to check for it.
+const uint32_t kVFPExceptionMask = 0xf;
+const uint32_t kVFPInvalidOpExceptionBit = 1 << 0;
+const uint32_t kVFPOverflowExceptionBit = 1 << 2;
+const uint32_t kVFPUnderflowExceptionBit = 1 << 3;
+const uint32_t kVFPInexactExceptionBit = 1 << 4;
+const uint32_t kVFPFlushToZeroMask = 1 << 24;
+const uint32_t kVFPDefaultNaNModeControlBit = 1 << 25;
+
+const uint32_t kVFPNConditionFlagBit = 1 << 31;
+const uint32_t kVFPZConditionFlagBit = 1 << 30;
+const uint32_t kVFPCConditionFlagBit = 1 << 29;
+const uint32_t kVFPVConditionFlagBit = 1 << 28;
+
+// VFP rounding modes. See ARM DDI 0406B Page A2-29.
+enum VFPRoundingMode {
+ RN = 0 << 22, // Round to Nearest.
+ RP = 1 << 22, // Round towards Plus Infinity.
+ RM = 2 << 22, // Round towards Minus Infinity.
+ RZ = 3 << 22, // Round towards zero.
+
+ // Aliases.
+ kRoundToNearest = RN,
+ kRoundToPlusInf = RP,
+ kRoundToMinusInf = RM,
+ kRoundToZero = RZ
+};
+
+const uint32_t kVFPRoundingModeMask = 3 << 22;
+
+enum CheckForInexactConversion {
+ kCheckForInexactConversion,
+ kDontCheckForInexactConversion
+};
+
+// -----------------------------------------------------------------------------
+// Hints.
+
+// Branch hints are not used on the ARM. They are defined so that they can
+// appear in shared function signatures, but will be ignored in ARM
+// implementations.
+enum Hint { no_hint };
+
+// Hints are not used on the arm. Negating is trivial.
+inline Hint NegateHint(Hint ignored) { return no_hint; }
+
+// -----------------------------------------------------------------------------
+// Instruction abstraction.
+
+// The class Instruction enables access to individual fields defined in the ARM
+// architecture instruction set encoding as described in figure A3-1.
+// Note that the Assembler uses typedef int32_t Instr.
+//
+// Example: Test whether the instruction at ptr does set the condition code
+// bits.
+//
+// bool InstructionSetsConditionCodes(byte* ptr) {
+// Instruction* instr = Instruction::At(ptr);
+// int type = instr->TypeValue();
+// return ((type == 0) || (type == 1)) && instr->HasS();
+// }
+//
+class Instruction {
+ public:
+ enum { kInstrSize = 4, kInstrSizeLog2 = 2, kPCReadOffset = 8 };
+
+ // Helper macro to define static accessors.
+ // We use the cast to char* trick to bypass the strict anti-aliasing rules.
+# define DECLARE_STATIC_TYPED_ACCESSOR(return_type, Name) \
+ static inline return_type Name(Instr instr) { \
+ char* temp = reinterpret_cast<char*>(&instr); \
+ return reinterpret_cast<Instruction*>(temp)->Name(); \
+ }
+
+# define DECLARE_STATIC_ACCESSOR(Name) DECLARE_STATIC_TYPED_ACCESSOR(int, Name)
+
+ // Get the raw instruction bits.
+ inline Instr InstructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void SetInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int Bit(int nr) const { return (InstructionBits() >> nr) & 1; }
+
+ // Read a bit field's value out of the instruction bits.
+ inline int Bits(int hi, int lo) const {
+ return (InstructionBits() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read a bit field out of the instruction bits.
+ inline int BitField(int hi, int lo) const {
+ return InstructionBits() & (((2 << (hi - lo)) - 1) << lo);
+ }
+
+ // Static support.
+
+ // Read one particular bit out of the instruction bits.
+ static inline int Bit(Instr instr, int nr) { return (instr >> nr) & 1; }
+
+ // Read the value of a bit field out of the instruction bits.
+ static inline int Bits(Instr instr, int hi, int lo) {
+ return (instr >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Read a bit field out of the instruction bits.
+ static inline int BitField(Instr instr, int hi, int lo) {
+ return instr & (((2 << (hi - lo)) - 1) << lo);
+ }
+
+ // Accessors for the different named fields used in the ARM encoding.
+ // The naming of these accessor corresponds to figure A3-1.
+ //
+ // Two kind of accessors are declared:
+ // - <Name>Field() will return the raw field, i.e. the field's bits at their
+ // original place in the instruction encoding.
+ // e.g. if instr is the 'addgt r0, r1, r2' instruction, encoded as
+ // 0xC0810002 ConditionField(instr) will return 0xC0000000.
+ // - <Name>Value() will return the field value, shifted back to bit 0.
+ // e.g. if instr is the 'addgt r0, r1, r2' instruction, encoded as
+ // 0xC0810002 ConditionField(instr) will return 0xC.
+
+ // Generally applicable fields
+ inline Condition ConditionValue() const {
+ return static_cast<Condition>(Bits(31, 28));
+ }
+ inline Condition ConditionField() const {
+ return static_cast<Condition>(BitField(31, 28));
+ }
+ DECLARE_STATIC_TYPED_ACCESSOR(Condition, ConditionValue);
+ DECLARE_STATIC_TYPED_ACCESSOR(Condition, ConditionField);
+
+ inline int TypeValue() const { return Bits(27, 25); }
+ inline int SpecialValue() const { return Bits(27, 23); }
+
+ inline int RnValue() const { return Bits(19, 16); }
+ DECLARE_STATIC_ACCESSOR(RnValue);
+ inline int RdValue() const { return Bits(15, 12); }
+ DECLARE_STATIC_ACCESSOR(RdValue);
+
+ inline int CoprocessorValue() const { return Bits(11, 8); }
+ // Support for VFP.
+ // Vn(19-16) | Vd(15-12) | Vm(3-0)
+ inline int VnValue() const { return Bits(19, 16); }
+ inline int VmValue() const { return Bits(3, 0); }
+ inline int VdValue() const { return Bits(15, 12); }
+ inline int NValue() const { return Bit(7); }
+ inline int MValue() const { return Bit(5); }
+ inline int DValue() const { return Bit(22); }
+ inline int RtValue() const { return Bits(15, 12); }
+ inline int PValue() const { return Bit(24); }
+ inline int UValue() const { return Bit(23); }
+ inline int Opc1Value() const { return (Bit(23) << 2) | Bits(21, 20); }
+ inline int Opc2Value() const { return Bits(19, 16); }
+ inline int Opc3Value() const { return Bits(7, 6); }
+ inline int SzValue() const { return Bit(8); }
+ inline int VLValue() const { return Bit(20); }
+ inline int VCValue() const { return Bit(8); }
+ inline int VAValue() const { return Bits(23, 21); }
+ inline int VBValue() const { return Bits(6, 5); }
+ inline int VFPNRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 16, 7);
+ }
+ inline int VFPMRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 0, 5);
+ }
+ inline int VFPDRegValue(VFPRegPrecision pre) {
+ return VFPGlueRegValue(pre, 12, 22);
+ }
+
+ // Fields used in Data processing instructions
+ inline int OpcodeValue() const { return static_cast<Opcode>(Bits(24, 21)); }
+ inline Opcode OpcodeField() const {
+ return static_cast<Opcode>(BitField(24, 21));
+ }
+ inline int SValue() const { return Bit(20); }
+ // with register
+ inline int RmValue() const { return Bits(3, 0); }
+ DECLARE_STATIC_ACCESSOR(RmValue);
+ inline int ShiftValue() const { return static_cast<ShiftOp>(Bits(6, 5)); }
+ inline ShiftOp ShiftField() const {
+ return static_cast<ShiftOp>(BitField(6, 5));
+ }
+ inline int RegShiftValue() const { return Bit(4); }
+ inline int RsValue() const { return Bits(11, 8); }
+ inline int ShiftAmountValue() const { return Bits(11, 7); }
+ // with immediate
+ inline int RotateValue() const { return Bits(11, 8); }
+ DECLARE_STATIC_ACCESSOR(RotateValue);
+ inline int Immed8Value() const { return Bits(7, 0); }
+ DECLARE_STATIC_ACCESSOR(Immed8Value);
+ inline int Immed4Value() const { return Bits(19, 16); }
+ inline int ImmedMovwMovtValue() const {
+ return Immed4Value() << 12 | Offset12Value();
+ }
+ DECLARE_STATIC_ACCESSOR(ImmedMovwMovtValue);
+
+ // Fields used in Load/Store instructions
+ inline int PUValue() const { return Bits(24, 23); }
+ inline int PUField() const { return BitField(24, 23); }
+ inline int BValue() const { return Bit(22); }
+ inline int WValue() const { return Bit(21); }
+ inline int LValue() const { return Bit(20); }
+ // with register uses same fields as Data processing instructions above
+ // with immediate
+ inline int Offset12Value() const { return Bits(11, 0); }
+ // multiple
+ inline int RlistValue() const { return Bits(15, 0); }
+ // extra loads and stores
+ inline int SignValue() const { return Bit(6); }
+ inline int HValue() const { return Bit(5); }
+ inline int ImmedHValue() const { return Bits(11, 8); }
+ inline int ImmedLValue() const { return Bits(3, 0); }
+
+ // Fields used in Branch instructions
+ inline int LinkValue() const { return Bit(24); }
+ inline int SImmed24Value() const { return ((InstructionBits() << 8) >> 8); }
+
+ // Fields used in Software interrupt instructions
+ inline SoftwareInterruptCodes SvcValue() const {
+ return static_cast<SoftwareInterruptCodes>(Bits(23, 0));
+ }
+
+ // Test for special encodings of type 0 instructions (extra loads and stores,
+ // as well as multiplications).
+ inline bool IsSpecialType0() const { return (Bit(7) == 1) && (Bit(4) == 1); }
+
+ // Test for miscellaneous instructions encodings of type 0 instructions.
+ inline bool IsMiscType0() const {
+ return (Bit(24) == 1) && (Bit(23) == 0) && (Bit(20) == 0) &&
+ ((Bit(7) == 0));
+ }
+
+ // Test for a nop instruction, which falls under type 1.
+ inline bool IsNopType1() const { return Bits(24, 0) == 0x0120F000; }
+
+ // Test for a nop instruction, which falls under type 1.
+ inline bool IsCsdbType1() const { return Bits(24, 0) == 0x0120F014; }
+
+ // Test for a stop instruction.
+ inline bool IsStop() const {
+ return (TypeValue() == 7) && (Bit(24) == 1) && (SvcValue() >= kStopCode);
+ }
+
+ // Special accessors that test for existence of a value.
+ inline bool HasS() const { return SValue() == 1; }
+ inline bool HasB() const { return BValue() == 1; }
+ inline bool HasW() const { return WValue() == 1; }
+ inline bool HasL() const { return LValue() == 1; }
+ inline bool HasU() const { return UValue() == 1; }
+ inline bool HasSign() const { return SignValue() == 1; }
+ inline bool HasH() const { return HValue() == 1; }
+ inline bool HasLink() const { return LinkValue() == 1; }
+
+ // Decoding the double immediate in the vmov instruction.
+ double DoubleImmedVmov() const;
+
+ // Instructions are read of out a code stream. The only way to get a
+ // reference to an instruction is to convert a pointer. There is no way
+ // to allocate or create instances of class Instruction.
+ // Use the At(pc) function to create references to Instruction.
+ static Instruction* At(uint8_t* pc) {
+ return reinterpret_cast<Instruction*>(pc);
+ }
+
+ private:
+ // Join split register codes, depending on single or double precision.
+ // four_bit is the position of the least-significant bit of the four
+ // bit specifier. one_bit is the position of the additional single bit
+ // specifier.
+ inline int VFPGlueRegValue(VFPRegPrecision pre, int four_bit, int one_bit) {
+ if (pre == kSinglePrecision) {
+ return (Bits(four_bit + 3, four_bit) << 1) | Bit(one_bit);
+ }
+ return (Bit(one_bit) << 4) | Bits(four_bit + 3, four_bit);
+ }
+
+ // We need to prevent the creation of instances of class Instruction.
+ Instruction() = delete;
+ Instruction(const Instruction&) = delete;
+ void operator=(const Instruction&) = delete;
+};
+
+// Helper functions for converting between register numbers and names.
+class Registers {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg);
+
+ // Lookup the register number for the name provided.
+ static int Number(const char* name);
+
+ struct RegisterAlias {
+ int reg;
+ const char* name;
+ };
+
+ private:
+ static const char* names_[kNumRegisters];
+ static const RegisterAlias aliases_[];
+};
+
+// Helper functions for converting between VFP register numbers and names.
+class VFPRegisters {
+ public:
+ // Return the name of the register.
+ static const char* Name(int reg, bool is_double);
+
+ // Lookup the register number for the name provided.
+ // Set flag pointed by is_double to true if register
+ // is double-precision.
+ static int Number(const char* name, bool* is_double);
+
+ private:
+ static const char* names_[kNumVFPRegisters];
+};
+
+} // namespace disasm
+} // namespace jit
+} // namespace js
+
+#endif // JS_DISASM_ARM
+
+#endif // jit_arm_disasm_Constants_arm_h
diff --git a/js/src/jit/arm/disasm/Disasm-arm.cpp b/js/src/jit/arm/disasm/Disasm-arm.cpp
new file mode 100644
index 0000000000..97f39e1331
--- /dev/null
+++ b/js/src/jit/arm/disasm/Disasm-arm.cpp
@@ -0,0 +1,2031 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ */
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// A Disassembler object is used to disassemble a block of code instruction by
+// instruction. The default implementation of the NameConverter object can be
+// overriden to modify register names or to do symbol lookup on addresses.
+//
+// The example below will disassemble a block of code and print it to stdout.
+//
+// disasm::NameConverter converter;
+// disasm::Disassembler d(converter);
+// for (uint8_t* pc = begin; pc < end;) {
+// disasm::EmbeddedVector<char, disasm::ReasonableBufferSize> buffer;
+// uint8_t* prev_pc = pc;
+// pc += d.InstructionDecode(buffer, pc);
+// printf("%p %08x %s\n",
+// prev_pc, *reinterpret_cast<int32_t*>(prev_pc), buffer);
+// }
+//
+// The Disassembler class also has a convenience method to disassemble a block
+// of code into a FILE*, meaning that the above functionality could also be
+// achieved by just calling Disassembler::Disassemble(stdout, begin, end);
+
+#include "jit/arm/disasm/Disasm-arm.h"
+
+#ifdef JS_DISASM_ARM
+
+# include <stdarg.h>
+# include <stdio.h>
+# include <string.h>
+
+# include "jit/arm/disasm/Constants-arm.h"
+
+namespace js {
+namespace jit {
+namespace disasm {
+
+// Helper function for printing to a Vector.
+static int MOZ_FORMAT_PRINTF(2, 3)
+ SNPrintF(V8Vector<char> str, const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ int result = vsnprintf(str.start(), str.length(), format, args);
+ va_end(args);
+ return result;
+}
+
+//------------------------------------------------------------------------------
+
+// Decoder decodes and disassembles instructions into an output buffer.
+// It uses the converter to convert register names and call destinations into
+// more informative description.
+class Decoder {
+ public:
+ Decoder(const disasm::NameConverter& converter, V8Vector<char> out_buffer)
+ : converter_(converter), out_buffer_(out_buffer), out_buffer_pos_(0) {
+ out_buffer_[out_buffer_pos_] = '\0';
+ }
+
+ ~Decoder() {}
+
+ // Writes one disassembled instruction into 'buffer' (0-terminated).
+ // Returns the length of the disassembled machine instruction in bytes.
+ int InstructionDecode(uint8_t* instruction);
+
+ static bool IsConstantPoolAt(uint8_t* instr_ptr);
+ static int ConstantPoolSizeAt(uint8_t* instr_ptr);
+
+ private:
+ // Bottleneck functions to print into the out_buffer.
+ void PrintChar(const char ch);
+ void Print(const char* str);
+
+ // Printing of common values.
+ void PrintRegister(int reg);
+ void PrintSRegister(int reg);
+ void PrintDRegister(int reg);
+ int FormatVFPRegister(Instruction* instr, const char* format);
+ void PrintMovwMovt(Instruction* instr);
+ int FormatVFPinstruction(Instruction* instr, const char* format);
+ void PrintCondition(Instruction* instr);
+ void PrintShiftRm(Instruction* instr);
+ void PrintShiftImm(Instruction* instr);
+ void PrintShiftSat(Instruction* instr);
+ void PrintPU(Instruction* instr);
+ void PrintSoftwareInterrupt(SoftwareInterruptCodes svc);
+
+ // Handle formatting of instructions and their options.
+ int FormatRegister(Instruction* instr, const char* option);
+ void FormatNeonList(int Vd, int type);
+ void FormatNeonMemory(int Rn, int align, int Rm);
+ int FormatOption(Instruction* instr, const char* option);
+ void Format(Instruction* instr, const char* format);
+ void Unknown(Instruction* instr);
+
+ // Each of these functions decodes one particular instruction type, a 3-bit
+ // field in the instruction encoding.
+ // Types 0 and 1 are combined as they are largely the same except for the way
+ // they interpret the shifter operand.
+ void DecodeType01(Instruction* instr);
+ void DecodeType2(Instruction* instr);
+ void DecodeType3(Instruction* instr);
+ void DecodeType4(Instruction* instr);
+ void DecodeType5(Instruction* instr);
+ void DecodeType6(Instruction* instr);
+ // Type 7 includes special Debugger instructions.
+ int DecodeType7(Instruction* instr);
+ // For VFP support.
+ void DecodeTypeVFP(Instruction* instr);
+ void DecodeType6CoprocessorIns(Instruction* instr);
+
+ void DecodeSpecialCondition(Instruction* instr);
+
+ void DecodeVMOVBetweenCoreAndSinglePrecisionRegisters(Instruction* instr);
+ void DecodeVCMP(Instruction* instr);
+ void DecodeVCVTBetweenDoubleAndSingle(Instruction* instr);
+ void DecodeVCVTBetweenFloatingPointAndInteger(Instruction* instr);
+
+ const disasm::NameConverter& converter_;
+ V8Vector<char> out_buffer_;
+ int out_buffer_pos_;
+
+ // Disallow copy and assign.
+ Decoder(const Decoder&) = delete;
+ void operator=(const Decoder&) = delete;
+};
+
+// Support for assertions in the Decoder formatting functions.
+# define STRING_STARTS_WITH(string, compare_string) \
+ (strncmp(string, compare_string, strlen(compare_string)) == 0)
+
+// Append the ch to the output buffer.
+void Decoder::PrintChar(const char ch) { out_buffer_[out_buffer_pos_++] = ch; }
+
+// Append the str to the output buffer.
+void Decoder::Print(const char* str) {
+ char cur = *str++;
+ while (cur != '\0' && (out_buffer_pos_ < int(out_buffer_.length() - 1))) {
+ PrintChar(cur);
+ cur = *str++;
+ }
+ out_buffer_[out_buffer_pos_] = 0;
+}
+
+// These condition names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+static const char* const cond_names[kNumberOfConditions] = {
+ "eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc",
+ "hi", "ls", "ge", "lt", "gt", "le", "", "invalid",
+};
+
+// Print the condition guarding the instruction.
+void Decoder::PrintCondition(Instruction* instr) {
+ Print(cond_names[instr->ConditionValue()]);
+}
+
+// Print the register name according to the active name converter.
+void Decoder::PrintRegister(int reg) {
+ Print(converter_.NameOfCPURegister(reg));
+}
+
+// Print the VFP S register name according to the active name converter.
+void Decoder::PrintSRegister(int reg) { Print(VFPRegisters::Name(reg, false)); }
+
+// Print the VFP D register name according to the active name converter.
+void Decoder::PrintDRegister(int reg) { Print(VFPRegisters::Name(reg, true)); }
+
+// These shift names are defined in a way to match the native disassembler
+// formatting. See for example the command "objdump -d <binary file>".
+static const char* const shift_names[kNumberOfShifts] = {"lsl", "lsr", "asr",
+ "ror"};
+
+// Print the register shift operands for the instruction. Generally used for
+// data processing instructions.
+void Decoder::PrintShiftRm(Instruction* instr) {
+ ShiftOp shift = instr->ShiftField();
+ int shift_index = instr->ShiftValue();
+ int shift_amount = instr->ShiftAmountValue();
+ int rm = instr->RmValue();
+
+ PrintRegister(rm);
+
+ if ((instr->RegShiftValue() == 0) && (shift == LSL) && (shift_amount == 0)) {
+ // Special case for using rm only.
+ return;
+ }
+ if (instr->RegShiftValue() == 0) {
+ // by immediate
+ if ((shift == ROR) && (shift_amount == 0)) {
+ Print(", RRX");
+ return;
+ } else if (((shift == LSR) || (shift == ASR)) && (shift_amount == 0)) {
+ shift_amount = 32;
+ }
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, ", %s #%d",
+ shift_names[shift_index], shift_amount);
+ } else {
+ // by register
+ int rs = instr->RsValue();
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, ", %s ",
+ shift_names[shift_index]);
+ PrintRegister(rs);
+ }
+}
+
+static inline uint32_t RotateRight32(uint32_t value, uint32_t shift) {
+ if (shift == 0) return value;
+ return (value >> shift) | (value << (32 - shift));
+}
+
+// Print the immediate operand for the instruction. Generally used for data
+// processing instructions.
+void Decoder::PrintShiftImm(Instruction* instr) {
+ int rotate = instr->RotateValue() * 2;
+ int immed8 = instr->Immed8Value();
+ int imm = RotateRight32(immed8, rotate);
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "#%d", imm);
+}
+
+// Print the optional shift and immediate used by saturating instructions.
+void Decoder::PrintShiftSat(Instruction* instr) {
+ int shift = instr->Bits(11, 7);
+ if (shift > 0) {
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, ", %s #%d",
+ shift_names[instr->Bit(6) * 2], instr->Bits(11, 7));
+ }
+}
+
+// Print PU formatting to reduce complexity of FormatOption.
+void Decoder::PrintPU(Instruction* instr) {
+ switch (instr->PUField()) {
+ case da_x: {
+ Print("da");
+ break;
+ }
+ case ia_x: {
+ Print("ia");
+ break;
+ }
+ case db_x: {
+ Print("db");
+ break;
+ }
+ case ib_x: {
+ Print("ib");
+ break;
+ }
+ default: {
+ MOZ_CRASH();
+ break;
+ }
+ }
+}
+
+// Print SoftwareInterrupt codes. Factoring this out reduces the complexity of
+// the FormatOption method.
+void Decoder::PrintSoftwareInterrupt(SoftwareInterruptCodes svc) {
+ switch (svc) {
+ case kCallRtRedirected:
+ Print("call rt redirected");
+ return;
+ case kBreakpoint:
+ Print("breakpoint");
+ return;
+ default:
+ if (svc >= kStopCode) {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "%d - 0x%x",
+ svc & kStopCodeMask, svc & kStopCodeMask);
+ } else {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "%d", svc);
+ }
+ return;
+ }
+}
+
+// Handle all register based formatting in this function to reduce the
+// complexity of FormatOption.
+int Decoder::FormatRegister(Instruction* instr, const char* format) {
+ MOZ_ASSERT(format[0] == 'r');
+ if (format[1] == 'n') { // 'rn: Rn register
+ int reg = instr->RnValue();
+ PrintRegister(reg);
+ return 2;
+ } else if (format[1] == 'd') { // 'rd: Rd register
+ int reg = instr->RdValue();
+ PrintRegister(reg);
+ return 2;
+ } else if (format[1] == 's') { // 'rs: Rs register
+ int reg = instr->RsValue();
+ PrintRegister(reg);
+ return 2;
+ } else if (format[1] == 'm') { // 'rm: Rm register
+ int reg = instr->RmValue();
+ PrintRegister(reg);
+ return 2;
+ } else if (format[1] == 't') { // 'rt: Rt register
+ int reg = instr->RtValue();
+ PrintRegister(reg);
+ return 2;
+ } else if (format[1] == 'l') {
+ // 'rlist: register list for load and store multiple instructions
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "rlist"));
+ int rlist = instr->RlistValue();
+ int reg = 0;
+ Print("{");
+ // Print register list in ascending order, by scanning the bit mask.
+ while (rlist != 0) {
+ if ((rlist & 1) != 0) {
+ PrintRegister(reg);
+ if ((rlist >> 1) != 0) {
+ Print(", ");
+ }
+ }
+ reg++;
+ rlist >>= 1;
+ }
+ Print("}");
+ return 5;
+ }
+ MOZ_CRASH();
+ return -1;
+}
+
+// Handle all VFP register based formatting in this function to reduce the
+// complexity of FormatOption.
+int Decoder::FormatVFPRegister(Instruction* instr, const char* format) {
+ MOZ_ASSERT((format[0] == 'S') || (format[0] == 'D'));
+
+ VFPRegPrecision precision =
+ format[0] == 'D' ? kDoublePrecision : kSinglePrecision;
+
+ int retval = 2;
+ int reg = -1;
+ if (format[1] == 'n') {
+ reg = instr->VFPNRegValue(precision);
+ } else if (format[1] == 'm') {
+ reg = instr->VFPMRegValue(precision);
+ } else if (format[1] == 'd') {
+ if ((instr->TypeValue() == 7) && (instr->Bit(24) == 0x0) &&
+ (instr->Bits(11, 9) == 0x5) && (instr->Bit(4) == 0x1)) {
+ // vmov.32 has Vd in a different place.
+ reg = instr->Bits(19, 16) | (instr->Bit(7) << 4);
+ } else {
+ reg = instr->VFPDRegValue(precision);
+ }
+
+ if (format[2] == '+') {
+ int immed8 = instr->Immed8Value();
+ if (format[0] == 'S') reg += immed8 - 1;
+ if (format[0] == 'D') reg += (immed8 / 2 - 1);
+ }
+ if (format[2] == '+') retval = 3;
+ } else {
+ MOZ_CRASH();
+ }
+
+ if (precision == kSinglePrecision) {
+ PrintSRegister(reg);
+ } else {
+ PrintDRegister(reg);
+ }
+
+ return retval;
+}
+
+int Decoder::FormatVFPinstruction(Instruction* instr, const char* format) {
+ Print(format);
+ return 0;
+}
+
+void Decoder::FormatNeonList(int Vd, int type) {
+ if (type == nlt_1) {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "{d%d}", Vd);
+ } else if (type == nlt_2) {
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "{d%d, d%d}", Vd, Vd + 1);
+ } else if (type == nlt_3) {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_,
+ "{d%d, d%d, d%d}", Vd, Vd + 1, Vd + 2);
+ } else if (type == nlt_4) {
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "{d%d, d%d, d%d, d%d}", Vd,
+ Vd + 1, Vd + 2, Vd + 3);
+ }
+}
+
+void Decoder::FormatNeonMemory(int Rn, int align, int Rm) {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "[r%d", Rn);
+ if (align != 0) {
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, ":%d", (1 << align) << 6);
+ }
+ if (Rm == 15) {
+ Print("]");
+ } else if (Rm == 13) {
+ Print("]!");
+ } else {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "], r%d", Rm);
+ }
+}
+
+// Print the movw or movt instruction.
+void Decoder::PrintMovwMovt(Instruction* instr) {
+ int imm = instr->ImmedMovwMovtValue();
+ int rd = instr->RdValue();
+ PrintRegister(rd);
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, ", #%d", imm);
+}
+
+// FormatOption takes a formatting string and interprets it based on
+// the current instructions. The format string points to the first
+// character of the option string (the option escape has already been
+// consumed by the caller.) FormatOption returns the number of
+// characters that were consumed from the formatting string.
+int Decoder::FormatOption(Instruction* instr, const char* format) {
+ switch (format[0]) {
+ case 'a': { // 'a: accumulate multiplies
+ if (instr->Bit(21) == 0) {
+ Print("ul");
+ } else {
+ Print("la");
+ }
+ return 1;
+ }
+ case 'b': { // 'b: byte loads or stores
+ if (instr->HasB()) {
+ Print("b");
+ }
+ return 1;
+ }
+ case 'c': { // 'cond: conditional execution
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "cond"));
+ PrintCondition(instr);
+ return 4;
+ }
+ case 'd': { // 'd: vmov double immediate.
+ double d = instr->DoubleImmedVmov();
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "#%g", d);
+ return 1;
+ }
+ case 'f': { // 'f: bitfield instructions - v7 and above.
+ uint32_t lsbit = instr->Bits(11, 7);
+ uint32_t width = instr->Bits(20, 16) + 1;
+ if (instr->Bit(21) == 0) {
+ // BFC/BFI:
+ // Bits 20-16 represent most-significant bit. Covert to width.
+ width -= lsbit;
+ MOZ_ASSERT(width > 0);
+ }
+ MOZ_ASSERT((width + lsbit) <= 32);
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "#%d, #%d", lsbit, width);
+ return 1;
+ }
+ case 'h': { // 'h: halfword operation for extra loads and stores
+ if (instr->HasH()) {
+ Print("h");
+ } else {
+ Print("b");
+ }
+ return 1;
+ }
+ case 'i': { // 'i: immediate value from adjacent bits.
+ // Expects tokens in the form imm%02d@%02d, i.e. imm05@07, imm10@16
+ int width = (format[3] - '0') * 10 + (format[4] - '0');
+ int lsb = (format[6] - '0') * 10 + (format[7] - '0');
+
+ MOZ_ASSERT((width >= 1) && (width <= 32));
+ MOZ_ASSERT((lsb >= 0) && (lsb <= 31));
+ MOZ_ASSERT((width + lsb) <= 32);
+
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "%d",
+ instr->Bits(width + lsb - 1, lsb));
+ return 8;
+ }
+ case 'l': { // 'l: branch and link
+ if (instr->HasLink()) {
+ Print("l");
+ }
+ return 1;
+ }
+ case 'm': {
+ if (format[1] == 'w') {
+ // 'mw: movt/movw instructions.
+ PrintMovwMovt(instr);
+ return 2;
+ }
+ if (format[1] == 'e') { // 'memop: load/store instructions.
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "memop"));
+ if (instr->HasL()) {
+ Print("ldr");
+ } else {
+ if ((instr->Bits(27, 25) == 0) && (instr->Bit(20) == 0) &&
+ (instr->Bits(7, 6) == 3) && (instr->Bit(4) == 1)) {
+ if (instr->Bit(5) == 1) {
+ Print("strd");
+ } else {
+ Print("ldrd");
+ }
+ return 5;
+ }
+ Print("str");
+ }
+ return 5;
+ }
+ // 'msg: for simulator break instructions
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "msg"));
+ uint8_t* str =
+ reinterpret_cast<uint8_t*>(instr->InstructionBits() & 0x0fffffff);
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "%s",
+ converter_.NameInCode(str));
+ return 3;
+ }
+ case 'o': {
+ if ((format[3] == '1') && (format[4] == '2')) {
+ // 'off12: 12-bit offset for load and store instructions
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "off12"));
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "%d",
+ instr->Offset12Value());
+ return 5;
+ } else if (format[3] == '0') {
+ // 'off0to3and8to19 16-bit immediate encoded in bits 19-8 and 3-0.
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "off0to3and8to19"));
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "%d",
+ (instr->Bits(19, 8) << 4) + instr->Bits(3, 0));
+ return 15;
+ }
+ // 'off8: 8-bit offset for extra load and store instructions
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "off8"));
+ int offs8 = (instr->ImmedHValue() << 4) | instr->ImmedLValue();
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "%d", offs8);
+ return 4;
+ }
+ case 'p': { // 'pu: P and U bits for load and store instructions
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "pu"));
+ PrintPU(instr);
+ return 2;
+ }
+ case 'r': {
+ return FormatRegister(instr, format);
+ }
+ case 's': {
+ if (format[1] == 'h') { // 'shift_op or 'shift_rm or 'shift_sat.
+ if (format[6] == 'o') { // 'shift_op
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "shift_op"));
+ if (instr->TypeValue() == 0) {
+ PrintShiftRm(instr);
+ } else {
+ MOZ_ASSERT(instr->TypeValue() == 1);
+ PrintShiftImm(instr);
+ }
+ return 8;
+ } else if (format[6] == 's') { // 'shift_sat.
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "shift_sat"));
+ PrintShiftSat(instr);
+ return 9;
+ } else { // 'shift_rm
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "shift_rm"));
+ PrintShiftRm(instr);
+ return 8;
+ }
+ } else if (format[1] == 'v') { // 'svc
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "svc"));
+ PrintSoftwareInterrupt(instr->SvcValue());
+ return 3;
+ } else if (format[1] == 'i') { // 'sign: signed extra loads and stores
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "sign"));
+ if (instr->HasSign()) {
+ Print("s");
+ }
+ return 4;
+ }
+ // 's: S field of data processing instructions
+ if (instr->HasS()) {
+ Print("s");
+ }
+ return 1;
+ }
+ case 't': { // 'target: target of branch instructions
+ MOZ_ASSERT(STRING_STARTS_WITH(format, "target"));
+ int off = (instr->SImmed24Value() << 2) + 8;
+ out_buffer_pos_ += SNPrintF(
+ out_buffer_ + out_buffer_pos_, "%+d -> %s", off,
+ converter_.NameOfAddress(reinterpret_cast<uint8_t*>(instr) + off));
+ return 6;
+ }
+ case 'u': { // 'u: signed or unsigned multiplies
+ // The manual gets the meaning of bit 22 backwards in the multiply
+ // instruction overview on page A3.16.2. The instructions that
+ // exist in u and s variants are the following:
+ // smull A4.1.87
+ // umull A4.1.129
+ // umlal A4.1.128
+ // smlal A4.1.76
+ // For these 0 means u and 1 means s. As can be seen on their individual
+ // pages. The other 18 mul instructions have the bit set or unset in
+ // arbitrary ways that are unrelated to the signedness of the instruction.
+ // None of these 18 instructions exist in both a 'u' and an 's' variant.
+
+ if (instr->Bit(22) == 0) {
+ Print("u");
+ } else {
+ Print("s");
+ }
+ return 1;
+ }
+ case 'v': {
+ return FormatVFPinstruction(instr, format);
+ }
+ case 'S':
+ case 'D': {
+ return FormatVFPRegister(instr, format);
+ }
+ case 'w': { // 'w: W field of load and store instructions
+ if (instr->HasW()) {
+ Print("!");
+ }
+ return 1;
+ }
+ default: {
+ MOZ_CRASH();
+ break;
+ }
+ }
+ MOZ_CRASH();
+ return -1;
+}
+
+// Format takes a formatting string for a whole instruction and prints it into
+// the output buffer. All escaped options are handed to FormatOption to be
+// parsed further.
+void Decoder::Format(Instruction* instr, const char* format) {
+ char cur = *format++;
+ while ((cur != 0) && (out_buffer_pos_ < (out_buffer_.length() - 1))) {
+ if (cur == '\'') { // Single quote is used as the formatting escape.
+ format += FormatOption(instr, format);
+ } else {
+ out_buffer_[out_buffer_pos_++] = cur;
+ }
+ cur = *format++;
+ }
+ out_buffer_[out_buffer_pos_] = '\0';
+}
+
+// The disassembler may end up decoding data inlined in the code. We do not want
+// it to crash if the data does not ressemble any known instruction.
+# define VERIFY(condition) \
+ if (!(condition)) { \
+ Unknown(instr); \
+ return; \
+ }
+
+// For currently unimplemented decodings the disassembler calls Unknown(instr)
+// which will just print "unknown" of the instruction bits.
+void Decoder::Unknown(Instruction* instr) { Format(instr, "unknown"); }
+
+void Decoder::DecodeType01(Instruction* instr) {
+ int type = instr->TypeValue();
+ if ((type == 0) && instr->IsSpecialType0()) {
+ // multiply instruction or extra loads and stores
+ if (instr->Bits(7, 4) == 9) {
+ if (instr->Bit(24) == 0) {
+ // multiply instructions
+ if (instr->Bit(23) == 0) {
+ if (instr->Bit(21) == 0) {
+ // The MUL instruction description (A 4.1.33) refers to Rd as being
+ // the destination for the operation, but it confusingly uses the
+ // Rn field to encode it.
+ Format(instr, "mul'cond's 'rn, 'rm, 'rs");
+ } else {
+ if (instr->Bit(22) == 0) {
+ // The MLA instruction description (A 4.1.28) refers to the order
+ // of registers as "Rd, Rm, Rs, Rn". But confusingly it uses the
+ // Rn field to encode the Rd register and the Rd field to encode
+ // the Rn register.
+ Format(instr, "mla'cond's 'rn, 'rm, 'rs, 'rd");
+ } else {
+ // The MLS instruction description (A 4.1.29) refers to the order
+ // of registers as "Rd, Rm, Rs, Rn". But confusingly it uses the
+ // Rn field to encode the Rd register and the Rd field to encode
+ // the Rn register.
+ Format(instr, "mls'cond's 'rn, 'rm, 'rs, 'rd");
+ }
+ }
+ } else {
+ // The signed/long multiply instructions use the terms RdHi and RdLo
+ // when referring to the target registers. They are mapped to the Rn
+ // and Rd fields as follows:
+ // RdLo == Rd field
+ // RdHi == Rn field
+ // The order of registers is: <RdLo>, <RdHi>, <Rm>, <Rs>
+ Format(instr, "'um'al'cond's 'rd, 'rn, 'rm, 'rs");
+ }
+ } else {
+ if (instr->Bits(ExclusiveOpHi, ExclusiveOpLo) == ExclusiveOpcode) {
+ if (instr->Bit(ExclusiveLoad) == 1) {
+ switch (instr->Bits(ExclusiveSizeHi, ExclusiveSizeLo)) {
+ case ExclusiveWord:
+ Format(instr, "ldrex'cond 'rt, ['rn]");
+ break;
+ case ExclusiveDouble:
+ Format(instr, "ldrexd'cond 'rt, ['rn]");
+ break;
+ case ExclusiveByte:
+ Format(instr, "ldrexb'cond 'rt, ['rn]");
+ break;
+ case ExclusiveHalf:
+ Format(instr, "ldrexh'cond 'rt, ['rn]");
+ break;
+ }
+ } else {
+ // The documentation names the low four bits of the
+ // store-exclusive instructions "Rt" but canonically
+ // for disassembly they are really "Rm".
+ switch (instr->Bits(ExclusiveSizeHi, ExclusiveSizeLo)) {
+ case ExclusiveWord:
+ Format(instr, "strex'cond 'rd, 'rm, ['rn]");
+ break;
+ case ExclusiveDouble:
+ Format(instr, "strexd'cond 'rd, 'rm, ['rn]");
+ break;
+ case ExclusiveByte:
+ Format(instr, "strexb'cond 'rd, 'rm, ['rn]");
+ break;
+ case ExclusiveHalf:
+ Format(instr, "strexh'cond 'rd, 'rm, ['rn]");
+ break;
+ }
+ }
+ } else {
+ Unknown(instr);
+ }
+ }
+ } else if ((instr->Bit(20) == 0) && ((instr->Bits(7, 4) & 0xd) == 0xd)) {
+ // ldrd, strd
+ switch (instr->PUField()) {
+ case da_x: {
+ if (instr->Bit(22) == 0) {
+ Format(instr, "'memop'cond's 'rd, ['rn], -'rm");
+ } else {
+ Format(instr, "'memop'cond's 'rd, ['rn], #-'off8");
+ }
+ break;
+ }
+ case ia_x: {
+ if (instr->Bit(22) == 0) {
+ Format(instr, "'memop'cond's 'rd, ['rn], +'rm");
+ } else {
+ Format(instr, "'memop'cond's 'rd, ['rn], #+'off8");
+ }
+ break;
+ }
+ case db_x: {
+ if (instr->Bit(22) == 0) {
+ Format(instr, "'memop'cond's 'rd, ['rn, -'rm]'w");
+ } else {
+ Format(instr, "'memop'cond's 'rd, ['rn, #-'off8]'w");
+ }
+ break;
+ }
+ case ib_x: {
+ if (instr->Bit(22) == 0) {
+ Format(instr, "'memop'cond's 'rd, ['rn, +'rm]'w");
+ } else {
+ Format(instr, "'memop'cond's 'rd, ['rn, #+'off8]'w");
+ }
+ break;
+ }
+ default: {
+ // The PU field is a 2-bit field.
+ MOZ_CRASH();
+ break;
+ }
+ }
+ } else {
+ // extra load/store instructions
+ switch (instr->PUField()) {
+ case da_x: {
+ if (instr->Bit(22) == 0) {
+ Format(instr, "'memop'cond'sign'h 'rd, ['rn], -'rm");
+ } else {
+ Format(instr, "'memop'cond'sign'h 'rd, ['rn], #-'off8");
+ }
+ break;
+ }
+ case ia_x: {
+ if (instr->Bit(22) == 0) {
+ Format(instr, "'memop'cond'sign'h 'rd, ['rn], +'rm");
+ } else {
+ Format(instr, "'memop'cond'sign'h 'rd, ['rn], #+'off8");
+ }
+ break;
+ }
+ case db_x: {
+ if (instr->Bit(22) == 0) {
+ Format(instr, "'memop'cond'sign'h 'rd, ['rn, -'rm]'w");
+ } else {
+ Format(instr, "'memop'cond'sign'h 'rd, ['rn, #-'off8]'w");
+ }
+ break;
+ }
+ case ib_x: {
+ if (instr->Bit(22) == 0) {
+ Format(instr, "'memop'cond'sign'h 'rd, ['rn, +'rm]'w");
+ } else {
+ Format(instr, "'memop'cond'sign'h 'rd, ['rn, #+'off8]'w");
+ }
+ break;
+ }
+ default: {
+ // The PU field is a 2-bit field.
+ MOZ_CRASH();
+ break;
+ }
+ }
+ return;
+ }
+ } else if ((type == 0) && instr->IsMiscType0()) {
+ if (instr->Bits(22, 21) == 1) {
+ switch (instr->BitField(7, 4)) {
+ case BX:
+ Format(instr, "bx'cond 'rm");
+ break;
+ case BLX:
+ Format(instr, "blx'cond 'rm");
+ break;
+ case BKPT:
+ Format(instr, "bkpt 'off0to3and8to19");
+ break;
+ default:
+ Unknown(instr); // not used by V8
+ break;
+ }
+ } else if (instr->Bits(22, 21) == 3) {
+ switch (instr->BitField(7, 4)) {
+ case CLZ:
+ Format(instr, "clz'cond 'rd, 'rm");
+ break;
+ default:
+ Unknown(instr); // not used by V8
+ break;
+ }
+ } else {
+ Unknown(instr); // not used by V8
+ }
+ } else if ((type == 1) && instr->IsNopType1()) {
+ Format(instr, "nop'cond");
+ } else if ((type == 1) && instr->IsCsdbType1()) {
+ Format(instr, "csdb'cond");
+ } else {
+ switch (instr->OpcodeField()) {
+ case AND: {
+ Format(instr, "and'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case EOR: {
+ Format(instr, "eor'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case SUB: {
+ Format(instr, "sub'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case RSB: {
+ Format(instr, "rsb'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case ADD: {
+ Format(instr, "add'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case ADC: {
+ Format(instr, "adc'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case SBC: {
+ Format(instr, "sbc'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case RSC: {
+ Format(instr, "rsc'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case TST: {
+ if (instr->HasS()) {
+ Format(instr, "tst'cond 'rn, 'shift_op");
+ } else {
+ Format(instr, "movw'cond 'mw");
+ }
+ break;
+ }
+ case TEQ: {
+ if (instr->HasS()) {
+ Format(instr, "teq'cond 'rn, 'shift_op");
+ } else {
+ // Other instructions matching this pattern are handled in the
+ // miscellaneous instructions part above.
+ MOZ_CRASH();
+ }
+ break;
+ }
+ case CMP: {
+ if (instr->HasS()) {
+ Format(instr, "cmp'cond 'rn, 'shift_op");
+ } else {
+ Format(instr, "movt'cond 'mw");
+ }
+ break;
+ }
+ case CMN: {
+ if (instr->HasS()) {
+ Format(instr, "cmn'cond 'rn, 'shift_op");
+ } else {
+ // Other instructions matching this pattern are handled in the
+ // miscellaneous instructions part above.
+ MOZ_CRASH();
+ }
+ break;
+ }
+ case ORR: {
+ Format(instr, "orr'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case MOV: {
+ Format(instr, "mov'cond's 'rd, 'shift_op");
+ break;
+ }
+ case BIC: {
+ Format(instr, "bic'cond's 'rd, 'rn, 'shift_op");
+ break;
+ }
+ case MVN: {
+ Format(instr, "mvn'cond's 'rd, 'shift_op");
+ break;
+ }
+ default: {
+ // The Opcode field is a 4-bit field.
+ MOZ_CRASH();
+ break;
+ }
+ }
+ }
+}
+
+void Decoder::DecodeType2(Instruction* instr) {
+ switch (instr->PUField()) {
+ case da_x: {
+ if (instr->HasW()) {
+ Unknown(instr); // not used in V8
+ return;
+ }
+ Format(instr, "'memop'cond'b 'rd, ['rn], #-'off12");
+ break;
+ }
+ case ia_x: {
+ if (instr->HasW()) {
+ Unknown(instr); // not used in V8
+ return;
+ }
+ Format(instr, "'memop'cond'b 'rd, ['rn], #+'off12");
+ break;
+ }
+ case db_x: {
+ Format(instr, "'memop'cond'b 'rd, ['rn, #-'off12]'w");
+ break;
+ }
+ case ib_x: {
+ Format(instr, "'memop'cond'b 'rd, ['rn, #+'off12]'w");
+ break;
+ }
+ default: {
+ // The PU field is a 2-bit field.
+ MOZ_CRASH();
+ break;
+ }
+ }
+}
+
+void Decoder::DecodeType3(Instruction* instr) {
+ switch (instr->PUField()) {
+ case da_x: {
+ VERIFY(!instr->HasW());
+ Format(instr, "'memop'cond'b 'rd, ['rn], -'shift_rm");
+ break;
+ }
+ case ia_x: {
+ if (instr->Bit(4) == 0) {
+ Format(instr, "'memop'cond'b 'rd, ['rn], +'shift_rm");
+ } else {
+ if (instr->Bit(5) == 0) {
+ switch (instr->Bits(22, 21)) {
+ case 0:
+ if (instr->Bit(20) == 0) {
+ if (instr->Bit(6) == 0) {
+ Format(instr, "pkhbt'cond 'rd, 'rn, 'rm, lsl #'imm05@07");
+ } else {
+ if (instr->Bits(11, 7) == 0) {
+ Format(instr, "pkhtb'cond 'rd, 'rn, 'rm, asr #32");
+ } else {
+ Format(instr, "pkhtb'cond 'rd, 'rn, 'rm, asr #'imm05@07");
+ }
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 1:
+ MOZ_CRASH();
+ break;
+ case 2:
+ MOZ_CRASH();
+ break;
+ case 3:
+ Format(instr, "usat 'rd, #'imm05@16, 'rm'shift_sat");
+ break;
+ }
+ } else {
+ switch (instr->Bits(22, 21)) {
+ case 0:
+ MOZ_CRASH();
+ break;
+ case 1:
+ if (instr->Bits(9, 6) == 1) {
+ if (instr->Bit(20) == 0) {
+ if (instr->Bits(19, 16) == 0xF) {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "sxtb'cond 'rd, 'rm");
+ break;
+ case 1:
+ Format(instr, "sxtb'cond 'rd, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "sxtb'cond 'rd, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "sxtb'cond 'rd, 'rm, ror #24");
+ break;
+ }
+ } else {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "sxtab'cond 'rd, 'rn, 'rm");
+ break;
+ case 1:
+ Format(instr, "sxtab'cond 'rd, 'rn, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "sxtab'cond 'rd, 'rn, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "sxtab'cond 'rd, 'rn, 'rm, ror #24");
+ break;
+ }
+ }
+ } else {
+ if (instr->Bits(19, 16) == 0xF) {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "sxth'cond 'rd, 'rm");
+ break;
+ case 1:
+ Format(instr, "sxth'cond 'rd, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "sxth'cond 'rd, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "sxth'cond 'rd, 'rm, ror #24");
+ break;
+ }
+ } else {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "sxtah'cond 'rd, 'rn, 'rm");
+ break;
+ case 1:
+ Format(instr, "sxtah'cond 'rd, 'rn, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "sxtah'cond 'rd, 'rn, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "sxtah'cond 'rd, 'rn, 'rm, ror #24");
+ break;
+ }
+ }
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 2:
+ if ((instr->Bit(20) == 0) && (instr->Bits(9, 6) == 1)) {
+ if (instr->Bits(19, 16) == 0xF) {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "uxtb16'cond 'rd, 'rm");
+ break;
+ case 1:
+ Format(instr, "uxtb16'cond 'rd, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "uxtb16'cond 'rd, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "uxtb16'cond 'rd, 'rm, ror #24");
+ break;
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ case 3:
+ if ((instr->Bits(9, 6) == 1)) {
+ if ((instr->Bit(20) == 0)) {
+ if (instr->Bits(19, 16) == 0xF) {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "uxtb'cond 'rd, 'rm");
+ break;
+ case 1:
+ Format(instr, "uxtb'cond 'rd, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "uxtb'cond 'rd, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "uxtb'cond 'rd, 'rm, ror #24");
+ break;
+ }
+ } else {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "uxtab'cond 'rd, 'rn, 'rm");
+ break;
+ case 1:
+ Format(instr, "uxtab'cond 'rd, 'rn, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "uxtab'cond 'rd, 'rn, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "uxtab'cond 'rd, 'rn, 'rm, ror #24");
+ break;
+ }
+ }
+ } else {
+ if (instr->Bits(19, 16) == 0xF) {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "uxth'cond 'rd, 'rm");
+ break;
+ case 1:
+ Format(instr, "uxth'cond 'rd, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "uxth'cond 'rd, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "uxth'cond 'rd, 'rm, ror #24");
+ break;
+ }
+ } else {
+ switch (instr->Bits(11, 10)) {
+ case 0:
+ Format(instr, "uxtah'cond 'rd, 'rn, 'rm");
+ break;
+ case 1:
+ Format(instr, "uxtah'cond 'rd, 'rn, 'rm, ror #8");
+ break;
+ case 2:
+ Format(instr, "uxtah'cond 'rd, 'rn, 'rm, ror #16");
+ break;
+ case 3:
+ Format(instr, "uxtah'cond 'rd, 'rn, 'rm, ror #24");
+ break;
+ }
+ }
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ }
+ }
+ }
+ break;
+ }
+ case db_x: {
+ if (instr->Bits(22, 20) == 0x5) {
+ if (instr->Bits(7, 4) == 0x1) {
+ if (instr->Bits(15, 12) == 0xF) {
+ Format(instr, "smmul'cond 'rn, 'rm, 'rs");
+ } else {
+ // SMMLA (in V8 notation matching ARM ISA format)
+ Format(instr, "smmla'cond 'rn, 'rm, 'rs, 'rd");
+ }
+ break;
+ }
+ }
+ bool FLAG_enable_sudiv = true; // Flag doesn't exist in our engine.
+ if (FLAG_enable_sudiv) {
+ if (instr->Bits(5, 4) == 0x1) {
+ if ((instr->Bit(22) == 0x0) && (instr->Bit(20) == 0x1)) {
+ if (instr->Bit(21) == 0x1) {
+ // UDIV (in V8 notation matching ARM ISA format) rn = rm/rs
+ Format(instr, "udiv'cond'b 'rn, 'rm, 'rs");
+ } else {
+ // SDIV (in V8 notation matching ARM ISA format) rn = rm/rs
+ Format(instr, "sdiv'cond'b 'rn, 'rm, 'rs");
+ }
+ break;
+ }
+ }
+ }
+ Format(instr, "'memop'cond'b 'rd, ['rn, -'shift_rm]'w");
+ break;
+ }
+ case ib_x: {
+ if (instr->HasW() && (instr->Bits(6, 4) == 0x5)) {
+ uint32_t widthminus1 = static_cast<uint32_t>(instr->Bits(20, 16));
+ uint32_t lsbit = static_cast<uint32_t>(instr->Bits(11, 7));
+ uint32_t msbit = widthminus1 + lsbit;
+ if (msbit <= 31) {
+ if (instr->Bit(22)) {
+ Format(instr, "ubfx'cond 'rd, 'rm, 'f");
+ } else {
+ Format(instr, "sbfx'cond 'rd, 'rm, 'f");
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ } else if (!instr->HasW() && (instr->Bits(6, 4) == 0x1)) {
+ uint32_t lsbit = static_cast<uint32_t>(instr->Bits(11, 7));
+ uint32_t msbit = static_cast<uint32_t>(instr->Bits(20, 16));
+ if (msbit >= lsbit) {
+ if (instr->RmValue() == 15) {
+ Format(instr, "bfc'cond 'rd, 'f");
+ } else {
+ Format(instr, "bfi'cond 'rd, 'rm, 'f");
+ }
+ } else {
+ MOZ_CRASH();
+ }
+ } else {
+ Format(instr, "'memop'cond'b 'rd, ['rn, +'shift_rm]'w");
+ }
+ break;
+ }
+ default: {
+ // The PU field is a 2-bit field.
+ MOZ_CRASH();
+ break;
+ }
+ }
+}
+
+void Decoder::DecodeType4(Instruction* instr) {
+ if (instr->Bit(22) != 0) {
+ // Privileged mode currently not supported.
+ Unknown(instr);
+ } else {
+ if (instr->HasL()) {
+ Format(instr, "ldm'cond'pu 'rn'w, 'rlist");
+ } else {
+ Format(instr, "stm'cond'pu 'rn'w, 'rlist");
+ }
+ }
+}
+
+void Decoder::DecodeType5(Instruction* instr) {
+ Format(instr, "b'l'cond 'target");
+}
+
+void Decoder::DecodeType6(Instruction* instr) {
+ DecodeType6CoprocessorIns(instr);
+}
+
+int Decoder::DecodeType7(Instruction* instr) {
+ if (instr->Bit(24) == 1) {
+ if (instr->SvcValue() >= kStopCode) {
+ Format(instr, "stop'cond 'svc");
+ // Also print the stop message. Its address is encoded
+ // in the following 4 bytes.
+ out_buffer_pos_ += SNPrintF(
+ out_buffer_ + out_buffer_pos_, "\n %p %08x stop message: %s",
+ reinterpret_cast<void*>(instr + Instruction::kInstrSize),
+ *reinterpret_cast<uint32_t*>(instr + Instruction::kInstrSize),
+ *reinterpret_cast<char**>(instr + Instruction::kInstrSize));
+ // We have decoded 2 * Instruction::kInstrSize bytes.
+ return 2 * Instruction::kInstrSize;
+ } else {
+ Format(instr, "svc'cond 'svc");
+ }
+ } else {
+ DecodeTypeVFP(instr);
+ }
+ return Instruction::kInstrSize;
+}
+
+// void Decoder::DecodeTypeVFP(Instruction* instr)
+// vmov: Sn = Rt
+// vmov: Rt = Sn
+// vcvt: Dd = Sm
+// vcvt: Sd = Dm
+// vcvt.f64.s32 Dd, Dd, #<fbits>
+// Dd = vabs(Dm)
+// Sd = vabs(Sm)
+// Dd = vneg(Dm)
+// Sd = vneg(Sm)
+// Dd = vadd(Dn, Dm)
+// Sd = vadd(Sn, Sm)
+// Dd = vsub(Dn, Dm)
+// Sd = vsub(Sn, Sm)
+// Dd = vmul(Dn, Dm)
+// Sd = vmul(Sn, Sm)
+// Dd = vmla(Dn, Dm)
+// Sd = vmla(Sn, Sm)
+// Dd = vmls(Dn, Dm)
+// Sd = vmls(Sn, Sm)
+// Dd = vdiv(Dn, Dm)
+// Sd = vdiv(Sn, Sm)
+// vcmp(Dd, Dm)
+// vcmp(Sd, Sm)
+// Dd = vsqrt(Dm)
+// Sd = vsqrt(Sm)
+// vmrs
+// vmsr
+void Decoder::DecodeTypeVFP(Instruction* instr) {
+ VERIFY((instr->TypeValue() == 7) && (instr->Bit(24) == 0x0));
+ VERIFY(instr->Bits(11, 9) == 0x5);
+
+ if (instr->Bit(4) == 0) {
+ if (instr->Opc1Value() == 0x7) {
+ // Other data processing instructions
+ if ((instr->Opc2Value() == 0x0) && (instr->Opc3Value() == 0x1)) {
+ // vmov register to register.
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vmov'cond.f64 'Dd, 'Dm");
+ } else {
+ Format(instr, "vmov'cond.f32 'Sd, 'Sm");
+ }
+ } else if ((instr->Opc2Value() == 0x0) && (instr->Opc3Value() == 0x3)) {
+ // vabs
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vabs'cond.f64 'Dd, 'Dm");
+ } else {
+ Format(instr, "vabs'cond.f32 'Sd, 'Sm");
+ }
+ } else if ((instr->Opc2Value() == 0x1) && (instr->Opc3Value() == 0x1)) {
+ // vneg
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vneg'cond.f64 'Dd, 'Dm");
+ } else {
+ Format(instr, "vneg'cond.f32 'Sd, 'Sm");
+ }
+ } else if ((instr->Opc2Value() == 0x7) && (instr->Opc3Value() == 0x3)) {
+ DecodeVCVTBetweenDoubleAndSingle(instr);
+ } else if ((instr->Opc2Value() == 0x8) && (instr->Opc3Value() & 0x1)) {
+ DecodeVCVTBetweenFloatingPointAndInteger(instr);
+ } else if ((instr->Opc2Value() == 0xA) && (instr->Opc3Value() == 0x3) &&
+ (instr->Bit(8) == 1)) {
+ // vcvt.f64.s32 Dd, Dd, #<fbits>
+ int fraction_bits = 32 - ((instr->Bits(3, 0) << 1) | instr->Bit(5));
+ Format(instr, "vcvt'cond.f64.s32 'Dd, 'Dd");
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, ", #%d", fraction_bits);
+ } else if (((instr->Opc2Value() >> 1) == 0x6) &&
+ (instr->Opc3Value() & 0x1)) {
+ DecodeVCVTBetweenFloatingPointAndInteger(instr);
+ } else if (((instr->Opc2Value() == 0x4) || (instr->Opc2Value() == 0x5)) &&
+ (instr->Opc3Value() & 0x1)) {
+ DecodeVCMP(instr);
+ } else if (((instr->Opc2Value() == 0x1)) && (instr->Opc3Value() == 0x3)) {
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vsqrt'cond.f64 'Dd, 'Dm");
+ } else {
+ Format(instr, "vsqrt'cond.f32 'Sd, 'Sm");
+ }
+ } else if (instr->Opc3Value() == 0x0) {
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vmov'cond.f64 'Dd, 'd");
+ } else {
+ Unknown(instr); // Not used by V8.
+ }
+ } else if (((instr->Opc2Value() == 0x6)) && instr->Opc3Value() == 0x3) {
+ // vrintz - round towards zero (truncate)
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vrintz'cond.f64.f64 'Dd, 'Dm");
+ } else {
+ Format(instr, "vrintz'cond.f32.f32 'Sd, 'Sm");
+ }
+ } else {
+ Unknown(instr); // Not used by V8.
+ }
+ } else if (instr->Opc1Value() == 0x3) {
+ if (instr->SzValue() == 0x1) {
+ if (instr->Opc3Value() & 0x1) {
+ Format(instr, "vsub'cond.f64 'Dd, 'Dn, 'Dm");
+ } else {
+ Format(instr, "vadd'cond.f64 'Dd, 'Dn, 'Dm");
+ }
+ } else {
+ if (instr->Opc3Value() & 0x1) {
+ Format(instr, "vsub'cond.f32 'Sd, 'Sn, 'Sm");
+ } else {
+ Format(instr, "vadd'cond.f32 'Sd, 'Sn, 'Sm");
+ }
+ }
+ } else if ((instr->Opc1Value() == 0x2) && !(instr->Opc3Value() & 0x1)) {
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vmul'cond.f64 'Dd, 'Dn, 'Dm");
+ } else {
+ Format(instr, "vmul'cond.f32 'Sd, 'Sn, 'Sm");
+ }
+ } else if ((instr->Opc1Value() == 0x0) && !(instr->Opc3Value() & 0x1)) {
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vmla'cond.f64 'Dd, 'Dn, 'Dm");
+ } else {
+ Format(instr, "vmla'cond.f32 'Sd, 'Sn, 'Sm");
+ }
+ } else if ((instr->Opc1Value() == 0x0) && (instr->Opc3Value() & 0x1)) {
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vmls'cond.f64 'Dd, 'Dn, 'Dm");
+ } else {
+ Format(instr, "vmls'cond.f32 'Sd, 'Sn, 'Sm");
+ }
+ } else if ((instr->Opc1Value() == 0x4) && !(instr->Opc3Value() & 0x1)) {
+ if (instr->SzValue() == 0x1) {
+ Format(instr, "vdiv'cond.f64 'Dd, 'Dn, 'Dm");
+ } else {
+ Format(instr, "vdiv'cond.f32 'Sd, 'Sn, 'Sm");
+ }
+ } else {
+ Unknown(instr); // Not used by V8.
+ }
+ } else {
+ if ((instr->VCValue() == 0x0) && (instr->VAValue() == 0x0)) {
+ DecodeVMOVBetweenCoreAndSinglePrecisionRegisters(instr);
+ } else if ((instr->VLValue() == 0x0) && (instr->VCValue() == 0x1) &&
+ (instr->Bit(23) == 0x0)) {
+ if (instr->Bit(21) == 0x0) {
+ Format(instr, "vmov'cond.32 'Dd[0], 'rt");
+ } else {
+ Format(instr, "vmov'cond.32 'Dd[1], 'rt");
+ }
+ } else if ((instr->VLValue() == 0x1) && (instr->VCValue() == 0x1) &&
+ (instr->Bit(23) == 0x0)) {
+ if (instr->Bit(21) == 0x0) {
+ Format(instr, "vmov'cond.32 'rt, 'Dd[0]");
+ } else {
+ Format(instr, "vmov'cond.32 'rt, 'Dd[1]");
+ }
+ } else if ((instr->VCValue() == 0x0) && (instr->VAValue() == 0x7) &&
+ (instr->Bits(19, 16) == 0x1)) {
+ if (instr->VLValue() == 0) {
+ if (instr->Bits(15, 12) == 0xF) {
+ Format(instr, "vmsr'cond FPSCR, APSR");
+ } else {
+ Format(instr, "vmsr'cond FPSCR, 'rt");
+ }
+ } else {
+ if (instr->Bits(15, 12) == 0xF) {
+ Format(instr, "vmrs'cond APSR, FPSCR");
+ } else {
+ Format(instr, "vmrs'cond 'rt, FPSCR");
+ }
+ }
+ }
+ }
+}
+
+void Decoder::DecodeVMOVBetweenCoreAndSinglePrecisionRegisters(
+ Instruction* instr) {
+ VERIFY((instr->Bit(4) == 1) && (instr->VCValue() == 0x0) &&
+ (instr->VAValue() == 0x0));
+
+ bool to_arm_register = (instr->VLValue() == 0x1);
+
+ if (to_arm_register) {
+ Format(instr, "vmov'cond 'rt, 'Sn");
+ } else {
+ Format(instr, "vmov'cond 'Sn, 'rt");
+ }
+}
+
+void Decoder::DecodeVCMP(Instruction* instr) {
+ VERIFY((instr->Bit(4) == 0) && (instr->Opc1Value() == 0x7));
+ VERIFY(((instr->Opc2Value() == 0x4) || (instr->Opc2Value() == 0x5)) &&
+ (instr->Opc3Value() & 0x1));
+
+ // Comparison.
+ bool dp_operation = (instr->SzValue() == 1);
+ bool raise_exception_for_qnan = (instr->Bit(7) == 0x1);
+
+ if (dp_operation && !raise_exception_for_qnan) {
+ if (instr->Opc2Value() == 0x4) {
+ Format(instr, "vcmp'cond.f64 'Dd, 'Dm");
+ } else if (instr->Opc2Value() == 0x5) {
+ Format(instr, "vcmp'cond.f64 'Dd, #0.0");
+ } else {
+ Unknown(instr); // invalid
+ }
+ } else if (!raise_exception_for_qnan) {
+ if (instr->Opc2Value() == 0x4) {
+ Format(instr, "vcmp'cond.f32 'Sd, 'Sm");
+ } else if (instr->Opc2Value() == 0x5) {
+ Format(instr, "vcmp'cond.f32 'Sd, #0.0");
+ } else {
+ Unknown(instr); // invalid
+ }
+ } else {
+ Unknown(instr); // Not used by V8.
+ }
+}
+
+void Decoder::DecodeVCVTBetweenDoubleAndSingle(Instruction* instr) {
+ VERIFY((instr->Bit(4) == 0) && (instr->Opc1Value() == 0x7));
+ VERIFY((instr->Opc2Value() == 0x7) && (instr->Opc3Value() == 0x3));
+
+ bool double_to_single = (instr->SzValue() == 1);
+
+ if (double_to_single) {
+ Format(instr, "vcvt'cond.f32.f64 'Sd, 'Dm");
+ } else {
+ Format(instr, "vcvt'cond.f64.f32 'Dd, 'Sm");
+ }
+}
+
+void Decoder::DecodeVCVTBetweenFloatingPointAndInteger(Instruction* instr) {
+ VERIFY((instr->Bit(4) == 0) && (instr->Opc1Value() == 0x7));
+ VERIFY(((instr->Opc2Value() == 0x8) && (instr->Opc3Value() & 0x1)) ||
+ (((instr->Opc2Value() >> 1) == 0x6) && (instr->Opc3Value() & 0x1)));
+
+ bool to_integer = (instr->Bit(18) == 1);
+ bool dp_operation = (instr->SzValue() == 1);
+ if (to_integer) {
+ bool unsigned_integer = (instr->Bit(16) == 0);
+
+ if (dp_operation) {
+ if (unsigned_integer) {
+ Format(instr, "vcvt'cond.u32.f64 'Sd, 'Dm");
+ } else {
+ Format(instr, "vcvt'cond.s32.f64 'Sd, 'Dm");
+ }
+ } else {
+ if (unsigned_integer) {
+ Format(instr, "vcvt'cond.u32.f32 'Sd, 'Sm");
+ } else {
+ Format(instr, "vcvt'cond.s32.f32 'Sd, 'Sm");
+ }
+ }
+ } else {
+ bool unsigned_integer = (instr->Bit(7) == 0);
+
+ if (dp_operation) {
+ if (unsigned_integer) {
+ Format(instr, "vcvt'cond.f64.u32 'Dd, 'Sm");
+ } else {
+ Format(instr, "vcvt'cond.f64.s32 'Dd, 'Sm");
+ }
+ } else {
+ if (unsigned_integer) {
+ Format(instr, "vcvt'cond.f32.u32 'Sd, 'Sm");
+ } else {
+ Format(instr, "vcvt'cond.f32.s32 'Sd, 'Sm");
+ }
+ }
+ }
+}
+
+// Decode Type 6 coprocessor instructions.
+// Dm = vmov(Rt, Rt2)
+// <Rt, Rt2> = vmov(Dm)
+// Ddst = MEM(Rbase + 4*offset).
+// MEM(Rbase + 4*offset) = Dsrc.
+void Decoder::DecodeType6CoprocessorIns(Instruction* instr) {
+ VERIFY(instr->TypeValue() == 6);
+
+ if (instr->CoprocessorValue() == 0xA) {
+ switch (instr->OpcodeValue()) {
+ case 0x8:
+ case 0xA:
+ if (instr->HasL()) {
+ Format(instr, "vldr'cond 'Sd, ['rn - 4*'imm08@00]");
+ } else {
+ Format(instr, "vstr'cond 'Sd, ['rn - 4*'imm08@00]");
+ }
+ break;
+ case 0xC:
+ case 0xE:
+ if (instr->HasL()) {
+ Format(instr, "vldr'cond 'Sd, ['rn + 4*'imm08@00]");
+ } else {
+ Format(instr, "vstr'cond 'Sd, ['rn + 4*'imm08@00]");
+ }
+ break;
+ case 0x4:
+ case 0x5:
+ case 0x6:
+ case 0x7:
+ case 0x9:
+ case 0xB: {
+ bool to_vfp_register = (instr->VLValue() == 0x1);
+ if (to_vfp_register) {
+ Format(instr, "vldm'cond'pu 'rn'w, {'Sd-'Sd+}");
+ } else {
+ Format(instr, "vstm'cond'pu 'rn'w, {'Sd-'Sd+}");
+ }
+ break;
+ }
+ default:
+ Unknown(instr); // Not used by V8.
+ }
+ } else if (instr->CoprocessorValue() == 0xB) {
+ switch (instr->OpcodeValue()) {
+ case 0x2:
+ // Load and store double to two GP registers
+ if (instr->Bits(7, 6) != 0 || instr->Bit(4) != 1) {
+ Unknown(instr); // Not used by V8.
+ } else if (instr->HasL()) {
+ Format(instr, "vmov'cond 'rt, 'rn, 'Dm");
+ } else {
+ Format(instr, "vmov'cond 'Dm, 'rt, 'rn");
+ }
+ break;
+ case 0x8:
+ case 0xA:
+ if (instr->HasL()) {
+ Format(instr, "vldr'cond 'Dd, ['rn - 4*'imm08@00]");
+ } else {
+ Format(instr, "vstr'cond 'Dd, ['rn - 4*'imm08@00]");
+ }
+ break;
+ case 0xC:
+ case 0xE:
+ if (instr->HasL()) {
+ Format(instr, "vldr'cond 'Dd, ['rn + 4*'imm08@00]");
+ } else {
+ Format(instr, "vstr'cond 'Dd, ['rn + 4*'imm08@00]");
+ }
+ break;
+ case 0x4:
+ case 0x5:
+ case 0x6:
+ case 0x7:
+ case 0x9:
+ case 0xB: {
+ bool to_vfp_register = (instr->VLValue() == 0x1);
+ if (to_vfp_register) {
+ Format(instr, "vldm'cond'pu 'rn'w, {'Dd-'Dd+}");
+ } else {
+ Format(instr, "vstm'cond'pu 'rn'w, {'Dd-'Dd+}");
+ }
+ break;
+ }
+ default:
+ Unknown(instr); // Not used by V8.
+ }
+ } else {
+ Unknown(instr); // Not used by V8.
+ }
+}
+
+void Decoder::DecodeSpecialCondition(Instruction* instr) {
+ switch (instr->SpecialValue()) {
+ case 5:
+ if ((instr->Bits(18, 16) == 0) && (instr->Bits(11, 6) == 0x28) &&
+ (instr->Bit(4) == 1)) {
+ // vmovl signed
+ if ((instr->VdValue() & 1) != 0) Unknown(instr);
+ int Vd = (instr->Bit(22) << 3) | (instr->VdValue() >> 1);
+ int Vm = (instr->Bit(5) << 4) | instr->VmValue();
+ int imm3 = instr->Bits(21, 19);
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_,
+ "vmovl.s%d q%d, d%d", imm3 * 8, Vd, Vm);
+ } else {
+ Unknown(instr);
+ }
+ break;
+ case 7:
+ if ((instr->Bits(18, 16) == 0) && (instr->Bits(11, 6) == 0x28) &&
+ (instr->Bit(4) == 1)) {
+ // vmovl unsigned
+ if ((instr->VdValue() & 1) != 0) Unknown(instr);
+ int Vd = (instr->Bit(22) << 3) | (instr->VdValue() >> 1);
+ int Vm = (instr->Bit(5) << 4) | instr->VmValue();
+ int imm3 = instr->Bits(21, 19);
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_,
+ "vmovl.u%d q%d, d%d", imm3 * 8, Vd, Vm);
+ } else {
+ Unknown(instr);
+ }
+ break;
+ case 8:
+ if (instr->Bits(21, 20) == 0) {
+ // vst1
+ int Vd = (instr->Bit(22) << 4) | instr->VdValue();
+ int Rn = instr->VnValue();
+ int type = instr->Bits(11, 8);
+ int size = instr->Bits(7, 6);
+ int align = instr->Bits(5, 4);
+ int Rm = instr->VmValue();
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "vst1.%d ",
+ (1 << size) << 3);
+ FormatNeonList(Vd, type);
+ Print(", ");
+ FormatNeonMemory(Rn, align, Rm);
+ } else if (instr->Bits(21, 20) == 2) {
+ // vld1
+ int Vd = (instr->Bit(22) << 4) | instr->VdValue();
+ int Rn = instr->VnValue();
+ int type = instr->Bits(11, 8);
+ int size = instr->Bits(7, 6);
+ int align = instr->Bits(5, 4);
+ int Rm = instr->VmValue();
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "vld1.%d ",
+ (1 << size) << 3);
+ FormatNeonList(Vd, type);
+ Print(", ");
+ FormatNeonMemory(Rn, align, Rm);
+ } else {
+ Unknown(instr);
+ }
+ break;
+ case 9:
+ if (instr->Bits(21, 20) == 0 && instr->Bits(9, 8) == 0) {
+ // vst1
+ int Vd = (instr->Bit(22) << 4) | instr->VdValue();
+ int Rn = instr->VnValue();
+ int size = instr->Bits(11, 10);
+ int index = instr->Bits(7, 5);
+ int align = instr->Bit(4);
+ int Rm = instr->VmValue();
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "vst1.%d {d%d[%d]}, ",
+ (1 << size) << 3, Vd, index);
+ FormatNeonMemory(Rn, align, Rm);
+ } else if (instr->Bits(21, 20) == 2 && instr->Bits(9, 8) == 0) {
+ // vld1
+ int Vd = (instr->Bit(22) << 4) | instr->VdValue();
+ int Rn = instr->VnValue();
+ int size = instr->Bits(11, 10);
+ int index = instr->Bits(7, 5);
+ int align = instr->Bit(4);
+ int Rm = instr->VmValue();
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "vld1.%d {d%d[%d]}, ",
+ (1 << size) << 3, Vd, index);
+ FormatNeonMemory(Rn, align, Rm);
+ } else {
+ Unknown(instr);
+ }
+ break;
+ case 0xA:
+ if (instr->Bits(22, 20) == 7) {
+ const char* option = "?";
+ switch (instr->Bits(3, 0)) {
+ case 2:
+ option = "oshst";
+ break;
+ case 3:
+ option = "osh";
+ break;
+ case 6:
+ option = "nshst";
+ break;
+ case 7:
+ option = "nsh";
+ break;
+ case 10:
+ option = "ishst";
+ break;
+ case 11:
+ option = "ish";
+ break;
+ case 14:
+ option = "st";
+ break;
+ case 15:
+ option = "sy";
+ break;
+ }
+ switch (instr->Bits(7, 4)) {
+ case 1:
+ Print("clrex");
+ break;
+ case 4:
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "dsb %s", option);
+ break;
+ case 5:
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "dmb %s", option);
+ break;
+ default:
+ Unknown(instr);
+ }
+ break;
+ }
+ [[fallthrough]];
+ case 0xB:
+ if ((instr->Bits(22, 20) == 5) && (instr->Bits(15, 12) == 0xf)) {
+ int Rn = instr->Bits(19, 16);
+ int offset = instr->Bits(11, 0);
+ if (offset == 0) {
+ out_buffer_pos_ +=
+ SNPrintF(out_buffer_ + out_buffer_pos_, "pld [r%d]", Rn);
+ } else if (instr->Bit(23) == 0) {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_,
+ "pld [r%d, #-%d]", Rn, offset);
+ } else {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_,
+ "pld [r%d, #+%d]", Rn, offset);
+ }
+ } else {
+ Unknown(instr);
+ }
+ break;
+ case 0x1D:
+ if (instr->Opc1Value() == 0x7 && instr->Bits(19, 18) == 0x2 &&
+ instr->Bits(11, 9) == 0x5 && instr->Bits(7, 6) == 0x1 &&
+ instr->Bit(4) == 0x0) {
+ // VRINTA, VRINTN, VRINTP, VRINTM (floating-point)
+ bool dp_operation = (instr->SzValue() == 1);
+ int rounding_mode = instr->Bits(17, 16);
+ switch (rounding_mode) {
+ case 0x0:
+ if (dp_operation) {
+ Format(instr, "vrinta.f64.f64 'Dd, 'Dm");
+ } else {
+ Unknown(instr);
+ }
+ break;
+ case 0x1:
+ if (dp_operation) {
+ Format(instr, "vrintn.f64.f64 'Dd, 'Dm");
+ } else {
+ Unknown(instr);
+ }
+ break;
+ case 0x2:
+ if (dp_operation) {
+ Format(instr, "vrintp.f64.f64 'Dd, 'Dm");
+ } else {
+ Unknown(instr);
+ }
+ break;
+ case 0x3:
+ if (dp_operation) {
+ Format(instr, "vrintm.f64.f64 'Dd, 'Dm");
+ } else {
+ Unknown(instr);
+ }
+ break;
+ default:
+ MOZ_CRASH(); // Case analysis is exhaustive.
+ break;
+ }
+ } else {
+ Unknown(instr);
+ }
+ break;
+ default:
+ Unknown(instr);
+ break;
+ }
+}
+
+# undef VERIFIY
+
+bool Decoder::IsConstantPoolAt(uint8_t* instr_ptr) {
+ int instruction_bits = *(reinterpret_cast<int*>(instr_ptr));
+ return (instruction_bits & kConstantPoolMarkerMask) == kConstantPoolMarker;
+}
+
+int Decoder::ConstantPoolSizeAt(uint8_t* instr_ptr) {
+ if (IsConstantPoolAt(instr_ptr)) {
+ int instruction_bits = *(reinterpret_cast<int*>(instr_ptr));
+ return DecodeConstantPoolLength(instruction_bits);
+ } else {
+ return -1;
+ }
+}
+
+// Disassemble the instruction at *instr_ptr into the output buffer.
+int Decoder::InstructionDecode(uint8_t* instr_ptr) {
+ Instruction* instr = Instruction::At(instr_ptr);
+ // Print raw instruction bytes.
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_, "%08x ",
+ instr->InstructionBits());
+ if (instr->ConditionField() == kSpecialCondition) {
+ DecodeSpecialCondition(instr);
+ return Instruction::kInstrSize;
+ }
+ int instruction_bits = *(reinterpret_cast<int*>(instr_ptr));
+ if ((instruction_bits & kConstantPoolMarkerMask) == kConstantPoolMarker) {
+ out_buffer_pos_ += SNPrintF(out_buffer_ + out_buffer_pos_,
+ "constant pool begin (length %d)",
+ DecodeConstantPoolLength(instruction_bits));
+ return Instruction::kInstrSize;
+ } else if (instruction_bits == kCodeAgeJumpInstruction) {
+ // The code age prologue has a constant immediatly following the jump
+ // instruction.
+ Instruction* target = Instruction::At(instr_ptr + Instruction::kInstrSize);
+ DecodeType2(instr);
+ SNPrintF(out_buffer_ + out_buffer_pos_, " (0x%08x)",
+ target->InstructionBits());
+ return 2 * Instruction::kInstrSize;
+ }
+ switch (instr->TypeValue()) {
+ case 0:
+ case 1: {
+ DecodeType01(instr);
+ break;
+ }
+ case 2: {
+ DecodeType2(instr);
+ break;
+ }
+ case 3: {
+ DecodeType3(instr);
+ break;
+ }
+ case 4: {
+ DecodeType4(instr);
+ break;
+ }
+ case 5: {
+ DecodeType5(instr);
+ break;
+ }
+ case 6: {
+ DecodeType6(instr);
+ break;
+ }
+ case 7: {
+ return DecodeType7(instr);
+ }
+ default: {
+ // The type field is 3-bits in the ARM encoding.
+ MOZ_CRASH();
+ break;
+ }
+ }
+ return Instruction::kInstrSize;
+}
+
+} // namespace disasm
+
+# undef STRING_STARTS_WITH
+# undef VERIFY
+
+//------------------------------------------------------------------------------
+
+namespace disasm {
+
+const char* NameConverter::NameOfAddress(uint8_t* addr) const {
+ SNPrintF(tmp_buffer_, "%p", addr);
+ return tmp_buffer_.start();
+}
+
+const char* NameConverter::NameOfConstant(uint8_t* addr) const {
+ return NameOfAddress(addr);
+}
+
+const char* NameConverter::NameOfCPURegister(int reg) const {
+ return disasm::Registers::Name(reg);
+}
+
+const char* NameConverter::NameOfByteCPURegister(int reg) const {
+ MOZ_CRASH(); // ARM does not have the concept of a byte register
+ return "nobytereg";
+}
+
+const char* NameConverter::NameOfXMMRegister(int reg) const {
+ MOZ_CRASH(); // ARM does not have any XMM registers
+ return "noxmmreg";
+}
+
+const char* NameConverter::NameInCode(uint8_t* addr) const {
+ // The default name converter is called for unknown code. So we will not try
+ // to access any memory.
+ return "";
+}
+
+//------------------------------------------------------------------------------
+
+Disassembler::Disassembler(const NameConverter& converter)
+ : converter_(converter) {}
+
+Disassembler::~Disassembler() {}
+
+int Disassembler::InstructionDecode(V8Vector<char> buffer,
+ uint8_t* instruction) {
+ Decoder d(converter_, buffer);
+ return d.InstructionDecode(instruction);
+}
+
+int Disassembler::ConstantPoolSizeAt(uint8_t* instruction) {
+ return Decoder::ConstantPoolSizeAt(instruction);
+}
+
+void Disassembler::Disassemble(FILE* f, uint8_t* begin, uint8_t* end) {
+ NameConverter converter;
+ Disassembler d(converter);
+ for (uint8_t* pc = begin; pc < end;) {
+ EmbeddedVector<char, ReasonableBufferSize> buffer;
+ buffer[0] = '\0';
+ uint8_t* prev_pc = pc;
+ pc += d.InstructionDecode(buffer, pc);
+ fprintf(f, "%p %08x %s\n", prev_pc,
+ *reinterpret_cast<int32_t*>(prev_pc), buffer.start());
+ }
+}
+
+} // namespace disasm
+} // namespace jit
+} // namespace js
+
+#endif // JS_DISASM_ARM
diff --git a/js/src/jit/arm/disasm/Disasm-arm.h b/js/src/jit/arm/disasm/Disasm-arm.h
new file mode 100644
index 0000000000..3415965645
--- /dev/null
+++ b/js/src/jit/arm/disasm/Disasm-arm.h
@@ -0,0 +1,141 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ */
+// Copyright 2007-2008 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef jit_arm_disasm_Disasm_arm_h
+#define jit_arm_disasm_Disasm_arm_h
+
+#ifdef JS_DISASM_ARM
+
+# include "mozilla/Assertions.h"
+# include "mozilla/Types.h"
+
+# include <stdio.h>
+
+namespace js {
+namespace jit {
+namespace disasm {
+
+typedef unsigned char byte;
+
+// A reasonable (ie, safe) buffer size for the disassembly of a single
+// instruction.
+const int ReasonableBufferSize = 256;
+
+// Vector as used by the original code to allow for minimal modification.
+// Functions exactly like a character array with helper methods.
+template <typename T>
+class V8Vector {
+ public:
+ V8Vector() : start_(nullptr), length_(0) {}
+ V8Vector(T* data, int length) : start_(data), length_(length) {
+ MOZ_ASSERT(length == 0 || (length > 0 && data != nullptr));
+ }
+
+ // Returns the length of the vector.
+ int length() const { return length_; }
+
+ // Returns the pointer to the start of the data in the vector.
+ T* start() const { return start_; }
+
+ // Access individual vector elements - checks bounds in debug mode.
+ T& operator[](int index) const {
+ MOZ_ASSERT(0 <= index && index < length_);
+ return start_[index];
+ }
+
+ inline V8Vector<T> operator+(int offset) {
+ MOZ_ASSERT(offset < length_);
+ return V8Vector<T>(start_ + offset, length_ - offset);
+ }
+
+ private:
+ T* start_;
+ int length_;
+};
+
+template <typename T, int kSize>
+class EmbeddedVector : public V8Vector<T> {
+ public:
+ EmbeddedVector() : V8Vector<T>(buffer_, kSize) {}
+
+ explicit EmbeddedVector(T initial_value) : V8Vector<T>(buffer_, kSize) {
+ for (int i = 0; i < kSize; ++i) {
+ buffer_[i] = initial_value;
+ }
+ }
+
+ // When copying, make underlying Vector to reference our buffer.
+ EmbeddedVector(const EmbeddedVector& rhs) : V8Vector<T>(rhs) {
+ MemCopy(buffer_, rhs.buffer_, sizeof(T) * kSize);
+ this->set_start(buffer_);
+ }
+
+ EmbeddedVector& operator=(const EmbeddedVector& rhs) {
+ if (this == &rhs) return *this;
+ V8Vector<T>::operator=(rhs);
+ MemCopy(buffer_, rhs.buffer_, sizeof(T) * kSize);
+ this->set_start(buffer_);
+ return *this;
+ }
+
+ private:
+ T buffer_[kSize];
+};
+
+// Interface and default implementation for converting addresses and
+// register-numbers to text. The default implementation is machine
+// specific.
+class NameConverter {
+ public:
+ virtual ~NameConverter() {}
+ virtual const char* NameOfCPURegister(int reg) const;
+ virtual const char* NameOfByteCPURegister(int reg) const;
+ virtual const char* NameOfXMMRegister(int reg) const;
+ virtual const char* NameOfAddress(byte* addr) const;
+ virtual const char* NameOfConstant(byte* addr) const;
+ virtual const char* NameInCode(byte* addr) const;
+
+ protected:
+ EmbeddedVector<char, 128> tmp_buffer_;
+};
+
+// A generic Disassembler interface
+class Disassembler {
+ public:
+ // Caller deallocates converter.
+ explicit Disassembler(const NameConverter& converter);
+
+ virtual ~Disassembler();
+
+ // Writes one disassembled instruction into 'buffer' (0-terminated).
+ // Returns the length of the disassembled machine instruction in bytes.
+ int InstructionDecode(V8Vector<char> buffer, uint8_t* instruction);
+
+ // Returns -1 if instruction does not mark the beginning of a constant pool,
+ // or the number of entries in the constant pool beginning here.
+ int ConstantPoolSizeAt(byte* instruction);
+
+ // Write disassembly into specified file 'f' using specified NameConverter
+ // (see constructor).
+ static void Disassemble(FILE* f, uint8_t* begin, uint8_t* end);
+
+ private:
+ const NameConverter& converter_;
+
+ // Disallow implicit constructors.
+ Disassembler() = delete;
+ Disassembler(const Disassembler&) = delete;
+ void operator=(const Disassembler&) = delete;
+};
+
+} // namespace disasm
+} // namespace jit
+} // namespace js
+
+#endif // JS_DISASM_ARM
+
+#endif // jit_arm_disasm_Disasm_arm_h
diff --git a/js/src/jit/arm/gen-double-encoder-table.py b/js/src/jit/arm/gen-double-encoder-table.py
new file mode 100644
index 0000000000..fd622da82e
--- /dev/null
+++ b/js/src/jit/arm/gen-double-encoder-table.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""Generate tables of immediately-encodable VFP doubles.
+
+DOES NOT get automatically run during the build process. If you need to
+modify this file (which is unlikely), you must re-run this script:
+
+python gen-double-encode-table.py > $(topsrcdir)/path/to/DoubleEntryTable.tbl
+"""
+
+import operator
+
+
+def rep(bit, count):
+ return reduce(operator.ior, [bit << c for c in range(count)])
+
+
+def encodeDouble(value):
+ """Generate an ARM ARM 'VFP modified immediate constant' with format:
+ aBbbbbbb bbcdefgh 000...
+
+ We will return the top 32 bits of the double; the rest are 0."""
+ assert (0 <= value) and (value <= 255)
+ a = value >> 7
+ b = (value >> 6) & 1
+ B = int(b == 0)
+ cdefgh = value & 0x3F
+ return (a << 31) | (B << 30) | (rep(b, 8) << 22) | cdefgh << 16
+
+
+print("/* THIS FILE IS AUTOMATICALLY GENERATED BY gen-double-encode-table.py. */")
+for i in range(256):
+ print(" { 0x%08x, { %d, %d, 0 } }," % (encodeDouble(i), i & 0xF, i >> 4))
diff --git a/js/src/jit/arm/llvm-compiler-rt/arm/aeabi_idivmod.S b/js/src/jit/arm/llvm-compiler-rt/arm/aeabi_idivmod.S
new file mode 100644
index 0000000000..0237f2221d
--- /dev/null
+++ b/js/src/jit/arm/llvm-compiler-rt/arm/aeabi_idivmod.S
@@ -0,0 +1,27 @@
+//===-- aeabi_idivmod.S - EABI idivmod implementation ---------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../assembly.h"
+
+// struct { int quot, int rem} __aeabi_idivmod(int numerator, int denominator) {
+// int rem, quot;
+// quot = __divmodsi4(numerator, denominator, &rem);
+// return {quot, rem};
+// }
+
+ .syntax unified
+ .align 2
+DEFINE_COMPILERRT_FUNCTION(__aeabi_idivmod)
+ push { lr }
+ sub sp, sp, #4
+ mov r2, sp
+ bl SYMBOL_NAME(__divmodsi4)
+ ldr r1, [sp]
+ add sp, sp, #4
+ pop { pc }
diff --git a/js/src/jit/arm/llvm-compiler-rt/arm/aeabi_uidivmod.S b/js/src/jit/arm/llvm-compiler-rt/arm/aeabi_uidivmod.S
new file mode 100644
index 0000000000..f7e1d2ebed
--- /dev/null
+++ b/js/src/jit/arm/llvm-compiler-rt/arm/aeabi_uidivmod.S
@@ -0,0 +1,28 @@
+//===-- aeabi_uidivmod.S - EABI uidivmod implementation -------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is dual licensed under the MIT and the University of Illinois Open
+// Source Licenses. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "../assembly.h"
+
+// struct { unsigned quot, unsigned rem}
+// __aeabi_uidivmod(unsigned numerator, unsigned denominator) {
+// unsigned rem, quot;
+// quot = __udivmodsi4(numerator, denominator, &rem);
+// return {quot, rem};
+// }
+
+ .syntax unified
+ .align 2
+DEFINE_COMPILERRT_FUNCTION(__aeabi_uidivmod)
+ push { lr }
+ sub sp, sp, #4
+ mov r2, sp
+ bl SYMBOL_NAME(__udivmodsi4)
+ ldr r1, [sp]
+ add sp, sp, #4
+ pop { pc }
diff --git a/js/src/jit/arm/llvm-compiler-rt/assembly.h b/js/src/jit/arm/llvm-compiler-rt/assembly.h
new file mode 100644
index 0000000000..802d1e2870
--- /dev/null
+++ b/js/src/jit/arm/llvm-compiler-rt/assembly.h
@@ -0,0 +1,67 @@
+/* ===-- assembly.h - compiler-rt assembler support macros -----------------===
+ *
+ * The LLVM Compiler Infrastructure
+ *
+ * This file is dual licensed under the MIT and the University of Illinois Open
+ * Source Licenses. See LICENSE.TXT for details.
+ *
+ * ===----------------------------------------------------------------------===
+ *
+ * This file defines macros for use in compiler-rt assembler source.
+ * This file is not part of the interface of this library.
+ *
+ * ===----------------------------------------------------------------------===
+ */
+
+#ifndef COMPILERRT_ASSEMBLY_H
+#define COMPILERRT_ASSEMBLY_H
+
+#if defined(__POWERPC__) || defined(__powerpc__) || defined(__ppc__)
+# define SEPARATOR @
+#else
+# define SEPARATOR ;
+#endif
+
+#if defined(__APPLE__)
+# define HIDDEN_DIRECTIVE .private_extern
+# define LOCAL_LABEL(name) L_##name
+#else
+# define HIDDEN_DIRECTIVE .hidden
+# define LOCAL_LABEL(name) .L_##name
+#endif
+
+#define GLUE2(a, b) a##b
+#define GLUE(a, b) GLUE2(a, b)
+#define SYMBOL_NAME(name) GLUE(__USER_LABEL_PREFIX__, name)
+
+#ifdef VISIBILITY_HIDDEN
+# define DECLARE_SYMBOL_VISIBILITY(name) \
+ HIDDEN_DIRECTIVE SYMBOL_NAME(name) SEPARATOR
+#else
+# define DECLARE_SYMBOL_VISIBILITY(name)
+#endif
+
+#define DEFINE_COMPILERRT_FUNCTION(name) \
+ .globl SYMBOL_NAME(name) \
+ SEPARATOR DECLARE_SYMBOL_VISIBILITY(name) SYMBOL_NAME(name) :
+
+#define DEFINE_COMPILERRT_PRIVATE_FUNCTION(name) \
+ .globl SYMBOL_NAME(name) \
+ SEPARATOR HIDDEN_DIRECTIVE SYMBOL_NAME(name) \
+ SEPARATOR SYMBOL_NAME(name) :
+
+#define DEFINE_COMPILERRT_PRIVATE_FUNCTION_UNMANGLED(name) \
+ .globl name SEPARATOR HIDDEN_DIRECTIVE name SEPARATOR name:
+
+#define DEFINE_COMPILERRT_FUNCTION_ALIAS(name, target) \
+ .globl SYMBOL_NAME(name) SEPARATOR.set SYMBOL_NAME(name), \
+ SYMBOL_NAME(target) SEPARATOR
+
+#if defined(__ARM_EABI__)
+# define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name) \
+ DEFINE_COMPILERRT_FUNCTION_ALIAS(aeabi_name, name)
+#else
+# define DEFINE_AEABI_FUNCTION_ALIAS(aeabi_name, name)
+#endif
+
+#endif /* COMPILERRT_ASSEMBLY_H */
diff --git a/js/src/jit/arm64/Architecture-arm64.cpp b/js/src/jit/arm64/Architecture-arm64.cpp
new file mode 100644
index 0000000000..eb3dd67b1a
--- /dev/null
+++ b/js/src/jit/arm64/Architecture-arm64.cpp
@@ -0,0 +1,129 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm64/Architecture-arm64.h"
+
+#include <cstring>
+
+#include "jit/arm64/vixl/Cpu-vixl.h"
+#include "jit/FlushICache.h" // js::jit::FlushICache
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+Registers::Code Registers::FromName(const char* name) {
+ // Check for some register aliases first.
+ if (strcmp(name, "ip0") == 0) {
+ return ip0;
+ }
+ if (strcmp(name, "ip1") == 0) {
+ return ip1;
+ }
+ if (strcmp(name, "fp") == 0) {
+ return fp;
+ }
+
+ for (uint32_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(i), name) == 0) {
+ return Code(i);
+ }
+ }
+
+ return Invalid;
+}
+
+FloatRegisters::Code FloatRegisters::FromName(const char* name) {
+ for (size_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(i), name) == 0) {
+ return Code(i);
+ }
+ }
+
+ return Invalid;
+}
+
+// This must sync with GetPushSizeInBytes just below and also with
+// MacroAssembler::PushRegsInMask.
+FloatRegisterSet FloatRegister::ReduceSetForPush(const FloatRegisterSet& s) {
+ SetType all = s.bits();
+ SetType set128b =
+ (all & FloatRegisters::AllSimd128Mask) >> FloatRegisters::ShiftSimd128;
+ SetType doubleSet =
+ (all & FloatRegisters::AllDoubleMask) >> FloatRegisters::ShiftDouble;
+ SetType singleSet =
+ (all & FloatRegisters::AllSingleMask) >> FloatRegisters::ShiftSingle;
+
+ // See GetPushSizeInBytes.
+ SetType set64b = (singleSet | doubleSet) & ~set128b;
+
+ SetType reduced = (set128b << FloatRegisters::ShiftSimd128) |
+ (set64b << FloatRegisters::ShiftDouble);
+ return FloatRegisterSet(reduced);
+}
+
+// Compute the size of the dump area for |s.ReduceSetForPush()|, as defined by
+// MacroAssembler::PushRegsInMask for this target.
+uint32_t FloatRegister::GetPushSizeInBytes(const FloatRegisterSet& s) {
+ SetType all = s.bits();
+ SetType set128b =
+ (all & FloatRegisters::AllSimd128Mask) >> FloatRegisters::ShiftSimd128;
+ SetType doubleSet =
+ (all & FloatRegisters::AllDoubleMask) >> FloatRegisters::ShiftDouble;
+ SetType singleSet =
+ (all & FloatRegisters::AllSingleMask) >> FloatRegisters::ShiftSingle;
+
+ // PushRegsInMask pushes singles as if they were doubles. Also we need to
+ // remove singles or doubles which are also pushed as part of a vector
+ // register.
+ SetType set64b = (singleSet | doubleSet) & ~set128b;
+
+ // The "+ 1) & ~1" is to take into account the alignment hole below the
+ // double-reg dump area. See MacroAssembler::PushRegsInMaskSizeInBytes.
+ return ((set64b.size() + 1) & ~1) * sizeof(double) +
+ set128b.size() * SizeOfSimd128;
+}
+
+uint32_t FloatRegister::getRegisterDumpOffsetInBytes() {
+ // See block comment in MacroAssembler.h for further required invariants.
+ static_assert(sizeof(jit::FloatRegisters::RegisterContent) == 16);
+ return encoding() * sizeof(jit::FloatRegisters::RegisterContent);
+}
+
+// For N in 0..31, if any of sN, dN or qN is a member of `s`, the returned set
+// will contain all of sN, dN and qN.
+FloatRegisterSet FloatRegister::BroadcastToAllSizes(const FloatRegisterSet& s) {
+ SetType all = s.bits();
+ SetType set128b =
+ (all & FloatRegisters::AllSimd128Mask) >> FloatRegisters::ShiftSimd128;
+ SetType doubleSet =
+ (all & FloatRegisters::AllDoubleMask) >> FloatRegisters::ShiftDouble;
+ SetType singleSet =
+ (all & FloatRegisters::AllSingleMask) >> FloatRegisters::ShiftSingle;
+
+ SetType merged = set128b | doubleSet | singleSet;
+ SetType broadcasted = (merged << FloatRegisters::ShiftSimd128) |
+ (merged << FloatRegisters::ShiftDouble) |
+ (merged << FloatRegisters::ShiftSingle);
+
+ return FloatRegisterSet(broadcasted);
+}
+
+uint32_t GetARM64Flags() { return 0; }
+
+// CPU flags handling on ARM64 is currently different from other platforms:
+// the flags are computed and stored per-assembler and are thus "always
+// computed".
+bool CPUFlagsHaveBeenComputed() { return true; }
+
+void FlushICache(void* code, size_t size) {
+ vixl::CPU::EnsureIAndDCacheCoherency(code, size);
+}
+
+void FlushExecutionContext() { vixl::CPU::FlushExecutionContext(); }
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/arm64/Architecture-arm64.h b/js/src/jit/arm64/Architecture-arm64.h
new file mode 100644
index 0000000000..387c2255b3
--- /dev/null
+++ b/js/src/jit/arm64/Architecture-arm64.h
@@ -0,0 +1,775 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_Architecture_arm64_h
+#define jit_arm64_Architecture_arm64_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+#include <iterator>
+
+#include "jit/arm64/vixl/Instructions-vixl.h"
+#include "jit/shared/Architecture-shared.h"
+
+#include "js/Utility.h"
+
+#define JS_HAS_HIDDEN_SP
+static const uint32_t HiddenSPEncoding = vixl::kSPRegInternalCode;
+
+namespace js {
+namespace jit {
+
+// AArch64 has 32 64-bit integer registers, x0 though x31.
+//
+// x31 (or, more accurately, the integer register with encoding 31, since
+// there is no x31 per se) is special and functions as both the stack pointer
+// and a zero register.
+//
+// The bottom 32 bits of each of the X registers is accessible as w0 through
+// w31. The program counter is not accessible as a register.
+//
+// SIMD and scalar floating-point registers share a register bank.
+// 32 bit float registers are s0 through s31.
+// 64 bit double registers are d0 through d31.
+// 128 bit SIMD registers are v0 through v31.
+// e.g., s0 is the bottom 32 bits of d0, which is the bottom 64 bits of v0.
+
+// AArch64 Calling Convention:
+// x0 - x7: arguments and return value
+// x8: indirect result (struct) location
+// x9 - x15: temporary registers
+// x16 - x17: intra-call-use registers (PLT, linker)
+// x18: platform specific use (TLS)
+// x19 - x28: callee-saved registers
+// x29: frame pointer
+// x30: link register
+
+// AArch64 Calling Convention for Floats:
+// d0 - d7: arguments and return value
+// d8 - d15: callee-saved registers
+// Bits 64:128 are not saved for v8-v15.
+// d16 - d31: temporary registers
+
+// AArch64 does not have soft float.
+
+class Registers {
+ public:
+ enum RegisterID {
+ w0 = 0,
+ x0 = 0,
+ w1 = 1,
+ x1 = 1,
+ w2 = 2,
+ x2 = 2,
+ w3 = 3,
+ x3 = 3,
+ w4 = 4,
+ x4 = 4,
+ w5 = 5,
+ x5 = 5,
+ w6 = 6,
+ x6 = 6,
+ w7 = 7,
+ x7 = 7,
+ w8 = 8,
+ x8 = 8,
+ w9 = 9,
+ x9 = 9,
+ w10 = 10,
+ x10 = 10,
+ w11 = 11,
+ x11 = 11,
+ w12 = 12,
+ x12 = 12,
+ w13 = 13,
+ x13 = 13,
+ w14 = 14,
+ x14 = 14,
+ w15 = 15,
+ x15 = 15,
+ w16 = 16,
+ x16 = 16,
+ ip0 = 16, // MacroAssembler scratch register 1.
+ w17 = 17,
+ x17 = 17,
+ ip1 = 17, // MacroAssembler scratch register 2.
+ w18 = 18,
+ x18 = 18,
+ tls = 18, // Platform-specific use (TLS).
+ w19 = 19,
+ x19 = 19,
+ w20 = 20,
+ x20 = 20,
+ w21 = 21,
+ x21 = 21,
+ w22 = 22,
+ x22 = 22,
+ w23 = 23,
+ x23 = 23,
+ w24 = 24,
+ x24 = 24,
+ w25 = 25,
+ x25 = 25,
+ w26 = 26,
+ x26 = 26,
+ w27 = 27,
+ x27 = 27,
+ w28 = 28,
+ x28 = 28,
+ w29 = 29,
+ x29 = 29,
+ fp = 29,
+ w30 = 30,
+ x30 = 30,
+ lr = 30,
+ w31 = 31,
+ x31 = 31,
+ wzr = 31,
+ xzr = 31,
+ sp = 31, // Special: both stack pointer and a zero register.
+ };
+ typedef uint8_t Code;
+ typedef uint32_t Encoding;
+ typedef uint32_t SetType;
+
+ static const Code Invalid = 0xFF;
+
+ union RegisterContent {
+ uintptr_t r;
+ };
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return mozilla::CountPopulation32(x);
+ }
+ static uint32_t FirstBit(SetType x) {
+ return mozilla::CountTrailingZeroes32(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ return 31 - mozilla::CountLeadingZeroes32(x);
+ }
+
+ static const char* GetName(uint32_t code) {
+ static const char* const Names[] = {
+ "x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
+ "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
+ "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
+ "x24", "x25", "x26", "x27", "x28", "x29", "lr", "sp"};
+ static_assert(Total == std::size(Names), "Table is the correct size");
+ if (code >= Total) {
+ return "invalid";
+ }
+ return Names[code];
+ }
+
+ static Code FromName(const char* name);
+
+ static const uint32_t Total = 32;
+ static const uint32_t TotalPhys = 32;
+ static const uint32_t Allocatable =
+ 27; // No named special-function registers.
+
+ static const SetType AllMask = 0xFFFFFFFF;
+ static const SetType NoneMask = 0x0;
+
+ static const SetType ArgRegMask =
+ (1 << Registers::x0) | (1 << Registers::x1) | (1 << Registers::x2) |
+ (1 << Registers::x3) | (1 << Registers::x4) | (1 << Registers::x5) |
+ (1 << Registers::x6) | (1 << Registers::x7) | (1 << Registers::x8);
+
+ static const SetType VolatileMask =
+ (1 << Registers::x0) | (1 << Registers::x1) | (1 << Registers::x2) |
+ (1 << Registers::x3) | (1 << Registers::x4) | (1 << Registers::x5) |
+ (1 << Registers::x6) | (1 << Registers::x7) | (1 << Registers::x8) |
+ (1 << Registers::x9) | (1 << Registers::x10) | (1 << Registers::x11) |
+ (1 << Registers::x12) | (1 << Registers::x13) | (1 << Registers::x14) |
+ (1 << Registers::x15) | (1 << Registers::x16) | (1 << Registers::x17) |
+ (1 << Registers::x18);
+
+ static const SetType NonVolatileMask =
+ (1 << Registers::x19) | (1 << Registers::x20) | (1 << Registers::x21) |
+ (1 << Registers::x22) | (1 << Registers::x23) | (1 << Registers::x24) |
+ (1 << Registers::x25) | (1 << Registers::x26) | (1 << Registers::x27) |
+ (1 << Registers::x28) | (1 << Registers::x29) | (1 << Registers::x30);
+
+ static const SetType SingleByteRegs = VolatileMask | NonVolatileMask;
+
+ static const SetType NonAllocatableMask =
+ (1 << Registers::x28) | // PseudoStackPointer.
+ (1 << Registers::ip0) | // First scratch register.
+ (1 << Registers::ip1) | // Second scratch register.
+ (1 << Registers::tls) | (1 << Registers::lr) | (1 << Registers::sp) |
+ (1 << Registers::fp);
+
+ static const SetType WrapperMask = VolatileMask;
+
+ // Registers returned from a JS -> JS call.
+ static const SetType JSCallMask = (1 << Registers::x2);
+
+ // Registers returned from a JS -> C call.
+ static const SetType CallMask = (1 << Registers::x0);
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+};
+
+// Smallest integer type that can hold a register bitmask.
+typedef uint32_t PackedRegisterMask;
+
+template <typename T>
+class TypedRegisterSet;
+
+// 128-bit bitset for FloatRegisters::SetType.
+
+class Bitset128 {
+ // The order (hi, lo) looks best in the debugger.
+ uint64_t hi, lo;
+
+ public:
+ MOZ_IMPLICIT constexpr Bitset128(uint64_t initial) : hi(0), lo(initial) {}
+ MOZ_IMPLICIT constexpr Bitset128(const Bitset128& that)
+ : hi(that.hi), lo(that.lo) {}
+
+ constexpr Bitset128(uint64_t hi, uint64_t lo) : hi(hi), lo(lo) {}
+
+ constexpr uint64_t high() const { return hi; }
+
+ constexpr uint64_t low() const { return lo; }
+
+ constexpr Bitset128 operator|(Bitset128 that) const {
+ return Bitset128(hi | that.hi, lo | that.lo);
+ }
+
+ constexpr Bitset128 operator&(Bitset128 that) const {
+ return Bitset128(hi & that.hi, lo & that.lo);
+ }
+
+ constexpr Bitset128 operator^(Bitset128 that) const {
+ return Bitset128(hi ^ that.hi, lo ^ that.lo);
+ }
+
+ constexpr Bitset128 operator~() const { return Bitset128(~hi, ~lo); }
+
+ // We must avoid shifting by the word width, which is complex. Inlining plus
+ // shift-by-constant will remove a lot of code in the normal case.
+
+ constexpr Bitset128 operator<<(size_t shift) const {
+ if (shift == 0) {
+ return *this;
+ }
+ if (shift < 64) {
+ return Bitset128((hi << shift) | (lo >> (64 - shift)), lo << shift);
+ }
+ if (shift == 64) {
+ return Bitset128(lo, 0);
+ }
+ return Bitset128(lo << (shift - 64), 0);
+ }
+
+ constexpr Bitset128 operator>>(size_t shift) const {
+ if (shift == 0) {
+ return *this;
+ }
+ if (shift < 64) {
+ return Bitset128(hi >> shift, (lo >> shift) | (hi << (64 - shift)));
+ }
+ if (shift == 64) {
+ return Bitset128(0, hi);
+ }
+ return Bitset128(0, hi >> (shift - 64));
+ }
+
+ constexpr bool operator==(Bitset128 that) const {
+ return lo == that.lo && hi == that.hi;
+ }
+
+ constexpr bool operator!=(Bitset128 that) const {
+ return lo != that.lo || hi != that.hi;
+ }
+
+ constexpr bool operator!() const { return (hi | lo) == 0; }
+
+ Bitset128& operator|=(const Bitset128& that) {
+ hi |= that.hi;
+ lo |= that.lo;
+ return *this;
+ }
+
+ Bitset128& operator&=(const Bitset128& that) {
+ hi &= that.hi;
+ lo &= that.lo;
+ return *this;
+ }
+
+ uint32_t size() const {
+ return mozilla::CountPopulation64(hi) + mozilla::CountPopulation64(lo);
+ }
+
+ uint32_t countTrailingZeroes() const {
+ if (lo) {
+ return mozilla::CountTrailingZeroes64(lo);
+ }
+ return mozilla::CountTrailingZeroes64(hi) + 64;
+ }
+
+ uint32_t countLeadingZeroes() const {
+ if (hi) {
+ return mozilla::CountLeadingZeroes64(hi);
+ }
+ return mozilla::CountLeadingZeroes64(lo) + 64;
+ }
+};
+
+class FloatRegisters {
+ public:
+ enum FPRegisterID {
+ s0 = 0,
+ d0 = 0,
+ v0 = 0,
+ s1 = 1,
+ d1 = 1,
+ v1 = 1,
+ s2 = 2,
+ d2 = 2,
+ v2 = 2,
+ s3 = 3,
+ d3 = 3,
+ v3 = 3,
+ s4 = 4,
+ d4 = 4,
+ v4 = 4,
+ s5 = 5,
+ d5 = 5,
+ v5 = 5,
+ s6 = 6,
+ d6 = 6,
+ v6 = 6,
+ s7 = 7,
+ d7 = 7,
+ v7 = 7,
+ s8 = 8,
+ d8 = 8,
+ v8 = 8,
+ s9 = 9,
+ d9 = 9,
+ v9 = 9,
+ s10 = 10,
+ d10 = 10,
+ v10 = 10,
+ s11 = 11,
+ d11 = 11,
+ v11 = 11,
+ s12 = 12,
+ d12 = 12,
+ v12 = 12,
+ s13 = 13,
+ d13 = 13,
+ v13 = 13,
+ s14 = 14,
+ d14 = 14,
+ v14 = 14,
+ s15 = 15,
+ d15 = 15,
+ v15 = 15,
+ s16 = 16,
+ d16 = 16,
+ v16 = 16,
+ s17 = 17,
+ d17 = 17,
+ v17 = 17,
+ s18 = 18,
+ d18 = 18,
+ v18 = 18,
+ s19 = 19,
+ d19 = 19,
+ v19 = 19,
+ s20 = 20,
+ d20 = 20,
+ v20 = 20,
+ s21 = 21,
+ d21 = 21,
+ v21 = 21,
+ s22 = 22,
+ d22 = 22,
+ v22 = 22,
+ s23 = 23,
+ d23 = 23,
+ v23 = 23,
+ s24 = 24,
+ d24 = 24,
+ v24 = 24,
+ s25 = 25,
+ d25 = 25,
+ v25 = 25,
+ s26 = 26,
+ d26 = 26,
+ v26 = 26,
+ s27 = 27,
+ d27 = 27,
+ v27 = 27,
+ s28 = 28,
+ d28 = 28,
+ v28 = 28,
+ s29 = 29,
+ d29 = 29,
+ v29 = 29,
+ s30 = 30,
+ d30 = 30,
+ v30 = 30,
+ s31 = 31,
+ d31 = 31,
+ v31 = 31, // Scratch register.
+ };
+
+ // Eight bits: (invalid << 7) | (kind << 5) | encoding
+ typedef uint8_t Code;
+ typedef FPRegisterID Encoding;
+ typedef Bitset128 SetType;
+
+ enum Kind : uint8_t { Single, Double, Simd128, NumTypes };
+
+ static constexpr Code Invalid = 0x80;
+
+ static const char* GetName(uint32_t code) {
+ // clang-format off
+ static const char* const Names[] = {
+ "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8", "s9",
+ "s10", "s11", "s12", "s13", "s14", "s15", "s16", "s17", "s18", "s19",
+ "s20", "s21", "s22", "s23", "s24", "s25", "s26", "s27", "s28", "s29",
+ "s30", "s31",
+
+ "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", "d8", "d9",
+ "d10", "d11", "d12", "d13", "d14", "d15", "d16", "d17", "d18", "d19",
+ "d20", "d21", "d22", "d23", "d24", "d25", "d26", "d27", "d28", "d29",
+ "d30", "d31",
+
+ "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9",
+ "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19",
+ "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29",
+ "v30", "v31",
+ };
+ // clang-format on
+ static_assert(Total == std::size(Names), "Table is the correct size");
+ if (code >= Total) {
+ return "invalid";
+ }
+ return Names[code];
+ }
+
+ static Code FromName(const char* name);
+
+ static const uint32_t TotalPhys = 32;
+ static const uint32_t Total = TotalPhys * NumTypes;
+ static const uint32_t Allocatable = 31; // Without d31, the scratch register.
+
+ static_assert(sizeof(SetType) * 8 >= Total,
+ "SetType should be large enough to enumerate all registers.");
+
+ static constexpr unsigned ShiftSingle = uint32_t(Single) * TotalPhys;
+ static constexpr unsigned ShiftDouble = uint32_t(Double) * TotalPhys;
+ static constexpr unsigned ShiftSimd128 = uint32_t(Simd128) * TotalPhys;
+
+ static constexpr SetType NoneMask = SetType(0);
+ static constexpr SetType AllPhysMask = ~(~SetType(0) << TotalPhys);
+ static constexpr SetType AllSingleMask = AllPhysMask << ShiftSingle;
+ static constexpr SetType AllDoubleMask = AllPhysMask << ShiftDouble;
+ static constexpr SetType AllSimd128Mask = AllPhysMask << ShiftSimd128;
+ static constexpr SetType AllMask =
+ AllDoubleMask | AllSingleMask | AllSimd128Mask;
+ static constexpr SetType AliasMask = (SetType(1) << ShiftSingle) |
+ (SetType(1) << ShiftDouble) |
+ (SetType(1) << ShiftSimd128);
+
+ static_assert(ShiftSingle == 0,
+ "Or the NonVolatileMask must be computed differently");
+
+ // s31 is the ScratchFloatReg.
+ static constexpr SetType NonVolatileSingleMask =
+ SetType((1 << FloatRegisters::s8) | (1 << FloatRegisters::s9) |
+ (1 << FloatRegisters::s10) | (1 << FloatRegisters::s11) |
+ (1 << FloatRegisters::s12) | (1 << FloatRegisters::s13) |
+ (1 << FloatRegisters::s14) | (1 << FloatRegisters::s15) |
+ (1 << FloatRegisters::s16) | (1 << FloatRegisters::s17) |
+ (1 << FloatRegisters::s18) | (1 << FloatRegisters::s19) |
+ (1 << FloatRegisters::s20) | (1 << FloatRegisters::s21) |
+ (1 << FloatRegisters::s22) | (1 << FloatRegisters::s23) |
+ (1 << FloatRegisters::s24) | (1 << FloatRegisters::s25) |
+ (1 << FloatRegisters::s26) | (1 << FloatRegisters::s27) |
+ (1 << FloatRegisters::s28) | (1 << FloatRegisters::s29) |
+ (1 << FloatRegisters::s30));
+
+ static constexpr SetType NonVolatileMask =
+ (NonVolatileSingleMask << ShiftSingle) |
+ (NonVolatileSingleMask << ShiftDouble) |
+ (NonVolatileSingleMask << ShiftSimd128);
+
+ static constexpr SetType VolatileMask = AllMask & ~NonVolatileMask;
+
+ static constexpr SetType WrapperMask = VolatileMask;
+
+ static_assert(ShiftSingle == 0,
+ "Or the NonAllocatableMask must be computed differently");
+
+ // d31 is the ScratchFloatReg.
+ static constexpr SetType NonAllocatableSingleMask =
+ (SetType(1) << FloatRegisters::s31);
+
+ static constexpr SetType NonAllocatableMask =
+ NonAllocatableSingleMask | (NonAllocatableSingleMask << ShiftDouble) |
+ (NonAllocatableSingleMask << ShiftSimd128);
+
+ static constexpr SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ float s;
+ double d;
+ uint8_t v128[16];
+ };
+
+ static constexpr Encoding encoding(Code c) {
+ // assert() not available in constexpr function.
+ // assert(c < Total);
+ return Encoding(c & 31);
+ }
+
+ static constexpr Kind kind(Code c) {
+ // assert() not available in constexpr function.
+ // assert(c < Total && ((c >> 5) & 3) < NumTypes);
+ return Kind((c >> 5) & 3);
+ }
+
+ static constexpr Code fromParts(uint32_t encoding, uint32_t kind,
+ uint32_t invalid) {
+ return Code((invalid << 7) | (kind << 5) | encoding);
+ }
+};
+
+static const uint32_t SpillSlotSize =
+ std::max(sizeof(Registers::RegisterContent),
+ sizeof(FloatRegisters::RegisterContent));
+
+static const uint32_t ShadowStackSpace = 0;
+
+// When our only strategy for far jumps is to encode the offset directly, and
+// not insert any jump islands during assembly for even further jumps, then the
+// architecture restricts us to -2^27 .. 2^27-4, to fit into a signed 28-bit
+// value. We further reduce this range to allow the far-jump inserting code to
+// have some breathing room.
+static const uint32_t JumpImmediateRange = ((1 << 27) - (20 * 1024 * 1024));
+
+static const uint32_t ABIStackAlignment = 16;
+static const uint32_t CodeAlignment = 16;
+static const bool StackKeptAligned = false;
+
+// Although sp is only usable if 16-byte alignment is kept,
+// the Pseudo-StackPointer enables use of 8-byte alignment.
+static const uint32_t StackAlignment = 8;
+static const uint32_t NativeFrameSize = 8;
+
+struct FloatRegister {
+ typedef FloatRegisters Codes;
+ typedef Codes::Code Code;
+ typedef Codes::Encoding Encoding;
+ typedef Codes::SetType SetType;
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 16, "SetType must be 128 bits");
+ x |= x >> FloatRegisters::TotalPhys;
+ x |= x >> FloatRegisters::TotalPhys;
+ x &= FloatRegisters::AllPhysMask;
+ MOZ_ASSERT(x.high() == 0);
+ MOZ_ASSERT((x.low() >> 32) == 0);
+ return mozilla::CountPopulation32(x.low());
+ }
+
+ static uint32_t FirstBit(SetType x) {
+ static_assert(sizeof(SetType) == 16, "SetType");
+ return x.countTrailingZeroes();
+ }
+ static uint32_t LastBit(SetType x) {
+ static_assert(sizeof(SetType) == 16, "SetType");
+ return 127 - x.countLeadingZeroes();
+ }
+
+ static constexpr size_t SizeOfSimd128 = 16;
+
+ private:
+ // These fields only hold valid values: an invalid register is always
+ // represented as a valid encoding and kind with the invalid_ bit set.
+ uint8_t encoding_; // 32 encodings
+ uint8_t kind_; // Double, Single, Simd128
+ bool invalid_;
+
+ typedef Codes::Kind Kind;
+
+ public:
+ constexpr FloatRegister(Encoding encoding, Kind kind)
+ : encoding_(encoding), kind_(kind), invalid_(false) {
+ // assert(uint32_t(encoding) < Codes::TotalPhys);
+ }
+
+ constexpr FloatRegister()
+ : encoding_(0), kind_(FloatRegisters::Double), invalid_(true) {}
+
+ static FloatRegister FromCode(uint32_t i) {
+ MOZ_ASSERT(i < Codes::Total);
+ return FloatRegister(FloatRegisters::encoding(i), FloatRegisters::kind(i));
+ }
+
+ bool isSingle() const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == FloatRegisters::Single;
+ }
+ bool isDouble() const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == FloatRegisters::Double;
+ }
+ bool isSimd128() const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == FloatRegisters::Simd128;
+ }
+ bool isInvalid() const { return invalid_; }
+
+ FloatRegister asSingle() const {
+ MOZ_ASSERT(!invalid_);
+ return FloatRegister(Encoding(encoding_), FloatRegisters::Single);
+ }
+ FloatRegister asDouble() const {
+ MOZ_ASSERT(!invalid_);
+ return FloatRegister(Encoding(encoding_), FloatRegisters::Double);
+ }
+ FloatRegister asSimd128() const {
+ MOZ_ASSERT(!invalid_);
+ return FloatRegister(Encoding(encoding_), FloatRegisters::Simd128);
+ }
+
+ constexpr uint32_t size() const {
+ MOZ_ASSERT(!invalid_);
+ if (kind_ == FloatRegisters::Double) {
+ return sizeof(double);
+ }
+ if (kind_ == FloatRegisters::Single) {
+ return sizeof(float);
+ }
+ MOZ_ASSERT(kind_ == FloatRegisters::Simd128);
+ return SizeOfSimd128;
+ }
+
+ constexpr Code code() const {
+ // assert(!invalid_);
+ return Codes::fromParts(encoding_, kind_, invalid_);
+ }
+
+ constexpr Encoding encoding() const {
+ MOZ_ASSERT(!invalid_);
+ return Encoding(encoding_);
+ }
+
+ const char* name() const { return FloatRegisters::GetName(code()); }
+ bool volatile_() const {
+ MOZ_ASSERT(!invalid_);
+ return !!((SetType(1) << code()) & FloatRegisters::VolatileMask);
+ }
+ constexpr bool operator!=(FloatRegister other) const {
+ return code() != other.code();
+ }
+ constexpr bool operator==(FloatRegister other) const {
+ return code() == other.code();
+ }
+
+ bool aliases(FloatRegister other) const {
+ return other.encoding_ == encoding_;
+ }
+ // This function mostly exists for the ARM backend. It is to ensure that two
+ // floating point registers' types are equivalent. e.g. S0 is not equivalent
+ // to D16, since S0 holds a float32, and D16 holds a Double.
+ // Since all floating point registers on x86 and x64 are equivalent, it is
+ // reasonable for this function to do the same.
+ bool equiv(FloatRegister other) const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == other.kind_;
+ }
+
+ uint32_t numAliased() const { return Codes::NumTypes; }
+ uint32_t numAlignedAliased() { return numAliased(); }
+
+ FloatRegister aliased(uint32_t aliasIdx) {
+ MOZ_ASSERT(!invalid_);
+ MOZ_ASSERT(aliasIdx < numAliased());
+ return FloatRegister(Encoding(encoding_),
+ Kind((aliasIdx + kind_) % Codes::NumTypes));
+ }
+ FloatRegister alignedAliased(uint32_t aliasIdx) { return aliased(aliasIdx); }
+ SetType alignedOrDominatedAliasedSet() const {
+ return Codes::AliasMask << encoding_;
+ }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName Name = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return LiveAsIndexableSet<Name>(s);
+ }
+
+ static TypedRegisterSet<FloatRegister> ReduceSetForPush(
+ const TypedRegisterSet<FloatRegister>& s);
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>& s);
+ uint32_t getRegisterDumpOffsetInBytes();
+
+ // For N in 0..31, if any of sN, dN or qN is a member of `s`, the
+ // returned set will contain all of sN, dN and qN.
+ static TypedRegisterSet<FloatRegister> BroadcastToAllSizes(
+ const TypedRegisterSet<FloatRegister>& s);
+};
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ return set & FloatRegisters::AllDoubleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Vector128>(SetType set) {
+ return set & FloatRegisters::AllSimd128Mask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Any>(SetType set) {
+ return set;
+}
+
+// ARM/D32 has double registers that cannot be treated as float32.
+// Luckily, ARMv8 doesn't have the same misfortune.
+inline bool hasUnaliasedDouble() { return false; }
+
+// ARM prior to ARMv8 also has doubles that alias multiple floats.
+// Again, ARMv8 is in the clear.
+inline bool hasMultiAlias() { return false; }
+
+uint32_t GetARM64Flags();
+
+bool CanFlushICacheFromBackgroundThreads();
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_arm64_Architecture_arm64_h
diff --git a/js/src/jit/arm64/Assembler-arm64.cpp b/js/src/jit/arm64/Assembler-arm64.cpp
new file mode 100644
index 0000000000..1e441ae635
--- /dev/null
+++ b/js/src/jit/arm64/Assembler-arm64.cpp
@@ -0,0 +1,609 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm64/Assembler-arm64.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+
+#include "gc/Marking.h"
+#include "jit/arm64/Architecture-arm64.h"
+#include "jit/arm64/MacroAssembler-arm64.h"
+#include "jit/arm64/vixl/Disasm-vixl.h"
+#include "jit/AutoWritableJitCode.h"
+#include "jit/ExecutableAllocator.h"
+#include "vm/Realm.h"
+
+#include "gc/StoreBuffer-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::CountLeadingZeroes32;
+using mozilla::DebugOnly;
+
+// Note this is used for inter-wasm calls and may pass arguments and results
+// in floating point registers even if the system ABI does not.
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ if (intRegIndex_ == NumIntArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uintptr_t);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_));
+ intRegIndex_++;
+ break;
+
+ case MIRType::Float32:
+ case MIRType::Double:
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(double);
+ break;
+ }
+ current_ = ABIArg(FloatRegister(FloatRegisters::Encoding(floatRegIndex_),
+ type == MIRType::Double
+ ? FloatRegisters::Double
+ : FloatRegisters::Single));
+ floatRegIndex_++;
+ break;
+
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment);
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += FloatRegister::SizeOfSimd128;
+ break;
+ }
+ current_ = ABIArg(FloatRegister(FloatRegisters::Encoding(floatRegIndex_),
+ FloatRegisters::Simd128));
+ floatRegIndex_++;
+ break;
+#endif
+
+ default:
+ // Note that in Assembler-x64.cpp there's a special case for Win64 which
+ // does not allow passing SIMD by value. Since there's Win64 on ARM64 we
+ // may need to duplicate that logic here.
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+}
+
+namespace js {
+namespace jit {
+
+void Assembler::finish() {
+ armbuffer_.flushPool();
+
+ // The extended jump table is part of the code buffer.
+ ExtendedJumpTable_ = emitExtendedJumpTable();
+ Assembler::FinalizeCode();
+}
+
+bool Assembler::appendRawCode(const uint8_t* code, size_t numBytes) {
+ flush();
+ return armbuffer_.appendRawCode(code, numBytes);
+}
+
+bool Assembler::reserve(size_t size) {
+ // This buffer uses fixed-size chunks so there's no point in reserving
+ // now vs. on-demand.
+ return !oom();
+}
+
+bool Assembler::swapBuffer(wasm::Bytes& bytes) {
+ // For now, specialize to the one use case. As long as wasm::Bytes is a
+ // Vector, not a linked-list of chunks, there's not much we can do other
+ // than copy.
+ MOZ_ASSERT(bytes.empty());
+ if (!bytes.resize(bytesNeeded())) {
+ return false;
+ }
+ armbuffer_.executableCopy(bytes.begin());
+ return true;
+}
+
+BufferOffset Assembler::emitExtendedJumpTable() {
+ if (!pendingJumps_.length() || oom()) {
+ return BufferOffset();
+ }
+
+ armbuffer_.flushPool();
+ armbuffer_.align(SizeOfJumpTableEntry);
+
+ BufferOffset tableOffset = armbuffer_.nextOffset();
+
+ for (size_t i = 0; i < pendingJumps_.length(); i++) {
+ // Each JumpTableEntry is of the form:
+ // LDR ip0 [PC, 8]
+ // BR ip0
+ // [Patchable 8-byte constant low bits]
+ // [Patchable 8-byte constant high bits]
+ DebugOnly<size_t> preOffset = size_t(armbuffer_.nextOffset().getOffset());
+
+ // The unguarded use of ScratchReg64 here is OK:
+ //
+ // - The present function is called from code that does not claim any
+ // scratch registers, we're done compiling user code and are emitting jump
+ // tables. Hence the scratch registers are available when we enter.
+ //
+ // - The pendingJumps_ represent jumps to other code sections that are not
+ // known to this MacroAssembler instance, and we're generating code to
+ // jump there. It is safe to assume that any code using such a generated
+ // branch to an unknown location did not store any valuable value in any
+ // scratch register. Hence the scratch registers can definitely be
+ // clobbered here.
+ //
+ // - Scratch register usage is restricted to sequential control flow within
+ // MacroAssembler functions. Hence the scratch registers will not be
+ // clobbered by ldr and br as they are Assembler primitives, not
+ // MacroAssembler functions.
+
+ ldr(ScratchReg64, ptrdiff_t(8 / vixl::kInstructionSize));
+ br(ScratchReg64);
+
+ DebugOnly<size_t> prePointer = size_t(armbuffer_.nextOffset().getOffset());
+ MOZ_ASSERT_IF(!oom(),
+ prePointer - preOffset == OffsetOfJumpTableEntryPointer);
+
+ brk(0x0);
+ brk(0x0);
+
+ DebugOnly<size_t> postOffset = size_t(armbuffer_.nextOffset().getOffset());
+
+ MOZ_ASSERT_IF(!oom(), postOffset - preOffset == SizeOfJumpTableEntry);
+ }
+
+ if (oom()) {
+ return BufferOffset();
+ }
+
+ return tableOffset;
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ // Copy the code and all constant pools into the output buffer.
+ armbuffer_.executableCopy(buffer);
+
+ // Patch any relative jumps that target code outside the buffer.
+ // The extended jump table may be used for distant jumps.
+ for (size_t i = 0; i < pendingJumps_.length(); i++) {
+ RelativePatch& rp = pendingJumps_[i];
+ MOZ_ASSERT(rp.target);
+
+ Instruction* target = (Instruction*)rp.target;
+ Instruction* branch = (Instruction*)(buffer + rp.offset.getOffset());
+ JumpTableEntry* extendedJumpTable = reinterpret_cast<JumpTableEntry*>(
+ buffer + ExtendedJumpTable_.getOffset());
+ if (branch->BranchType() != vixl::UnknownBranchType) {
+ if (branch->IsTargetReachable(target)) {
+ branch->SetImmPCOffsetTarget(target);
+ } else {
+ JumpTableEntry* entry = &extendedJumpTable[i];
+ branch->SetImmPCOffsetTarget(entry->getLdr());
+ entry->data = target;
+ }
+ } else {
+ // Currently a two-instruction call, it should be possible to optimize
+ // this into a single instruction call + nop in some instances, but this
+ // will work.
+ }
+ }
+}
+
+BufferOffset Assembler::immPool(ARMRegister dest, uint8_t* value,
+ vixl::LoadLiteralOp op, const LiteralDoc& doc,
+ ARMBuffer::PoolEntry* pe) {
+ uint32_t inst = op | Rt(dest);
+ const size_t numInst = 1;
+ const unsigned sizeOfPoolEntryInBytes = 4;
+ const unsigned numPoolEntries = sizeof(value) / sizeOfPoolEntryInBytes;
+ return allocLiteralLoadEntry(numInst, numPoolEntries, (uint8_t*)&inst, value,
+ doc, pe);
+}
+
+BufferOffset Assembler::immPool64(ARMRegister dest, uint64_t value,
+ ARMBuffer::PoolEntry* pe) {
+ return immPool(dest, (uint8_t*)&value, vixl::LDR_x_lit, LiteralDoc(value),
+ pe);
+}
+
+BufferOffset Assembler::fImmPool(ARMFPRegister dest, uint8_t* value,
+ vixl::LoadLiteralOp op,
+ const LiteralDoc& doc) {
+ uint32_t inst = op | Rt(dest);
+ const size_t numInst = 1;
+ const unsigned sizeOfPoolEntryInBits = 32;
+ const unsigned numPoolEntries = dest.size() / sizeOfPoolEntryInBits;
+ return allocLiteralLoadEntry(numInst, numPoolEntries, (uint8_t*)&inst, value,
+ doc);
+}
+
+BufferOffset Assembler::fImmPool64(ARMFPRegister dest, double value) {
+ return fImmPool(dest, (uint8_t*)&value, vixl::LDR_d_lit, LiteralDoc(value));
+}
+
+BufferOffset Assembler::fImmPool32(ARMFPRegister dest, float value) {
+ return fImmPool(dest, (uint8_t*)&value, vixl::LDR_s_lit, LiteralDoc(value));
+}
+
+void Assembler::bind(Label* label, BufferOffset targetOffset) {
+#ifdef JS_DISASM_ARM64
+ spew_.spewBind(label);
+#endif
+ // Nothing has seen the label yet: just mark the location.
+ // If we've run out of memory, don't attempt to modify the buffer which may
+ // not be there. Just mark the label as bound to the (possibly bogus)
+ // targetOffset.
+ if (!label->used() || oom()) {
+ label->bind(targetOffset.getOffset());
+ return;
+ }
+
+ // Get the most recent instruction that used the label, as stored in the
+ // label. This instruction is the head of an implicit linked list of label
+ // uses.
+ BufferOffset branchOffset(label);
+
+ while (branchOffset.assigned()) {
+ // Before overwriting the offset in this instruction, get the offset of
+ // the next link in the implicit branch list.
+ BufferOffset nextOffset = NextLink(branchOffset);
+
+ // Linking against the actual (Instruction*) would be invalid,
+ // since that Instruction could be anywhere in memory.
+ // Instead, just link against the correct relative offset, assuming
+ // no constant pools, which will be taken into consideration
+ // during finalization.
+ ptrdiff_t relativeByteOffset =
+ targetOffset.getOffset() - branchOffset.getOffset();
+ Instruction* link = getInstructionAt(branchOffset);
+
+ // This branch may still be registered for callbacks. Stop tracking it.
+ vixl::ImmBranchType branchType = link->BranchType();
+ vixl::ImmBranchRangeType branchRange =
+ Instruction::ImmBranchTypeToRange(branchType);
+ if (branchRange < vixl::NumShortBranchRangeTypes) {
+ BufferOffset deadline(
+ branchOffset.getOffset() +
+ Instruction::ImmBranchMaxForwardOffset(branchRange));
+ armbuffer_.unregisterBranchDeadline(branchRange, deadline);
+ }
+
+ // Is link able to reach the label?
+ if (link->IsPCRelAddressing() ||
+ link->IsTargetReachable(link + relativeByteOffset)) {
+ // Write a new relative offset into the instruction.
+ link->SetImmPCOffsetTarget(link + relativeByteOffset);
+ } else {
+ // This is a short-range branch, and it can't reach the label directly.
+ // Verify that it branches to a veneer: an unconditional branch.
+ MOZ_ASSERT(getInstructionAt(nextOffset)->BranchType() ==
+ vixl::UncondBranchType);
+ }
+
+ branchOffset = nextOffset;
+ }
+
+ // Bind the label, so that future uses may encode the offset immediately.
+ label->bind(targetOffset.getOffset());
+}
+
+void Assembler::addPendingJump(BufferOffset src, ImmPtr target,
+ RelocationKind reloc) {
+ MOZ_ASSERT(target.value != nullptr);
+
+ if (reloc == RelocationKind::JITCODE) {
+ jumpRelocations_.writeUnsigned(src.getOffset());
+ }
+
+ // This jump is not patchable at runtime. Extended jump table entry
+ // requirements cannot be known until finalization, so to be safe, give each
+ // jump and entry. This also causes GC tracing of the target.
+ enoughMemory_ &=
+ pendingJumps_.append(RelativePatch(src, target.value, reloc));
+}
+
+void Assembler::PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall) {
+ Instruction* dest = (Instruction*)start.raw();
+ ptrdiff_t relTarget = (Instruction*)toCall.raw() - dest;
+ ptrdiff_t relTarget00 = relTarget >> 2;
+ MOZ_RELEASE_ASSERT((relTarget & 0x3) == 0);
+ MOZ_RELEASE_ASSERT(vixl::IsInt26(relTarget00));
+
+ bl(dest, relTarget00);
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expected) {
+ Instruction* i = (Instruction*)label.raw();
+ void** pValue = i->LiteralAddress<void**>();
+ MOZ_ASSERT(*pValue == expected.value);
+ *pValue = newValue.value;
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ ImmPtr newValue, ImmPtr expected) {
+ PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value),
+ PatchedImmPtr(expected.value));
+}
+
+void Assembler::ToggleToJmp(CodeLocationLabel inst_) {
+ Instruction* i = (Instruction*)inst_.raw();
+ MOZ_ASSERT(i->IsAddSubImmediate());
+
+ // Refer to instruction layout in ToggleToCmp().
+ int imm19 = (int)i->Bits(23, 5);
+ MOZ_ASSERT(vixl::IsInt19(imm19));
+
+ b(i, imm19, Always);
+}
+
+void Assembler::ToggleToCmp(CodeLocationLabel inst_) {
+ Instruction* i = (Instruction*)inst_.raw();
+ MOZ_ASSERT(i->IsCondB());
+
+ int imm19 = i->ImmCondBranch();
+ // bit 23 is reserved, and the simulator throws an assertion when this happens
+ // It'll be messy to decode, but we can steal bit 30 or bit 31.
+ MOZ_ASSERT(vixl::IsInt18(imm19));
+
+ // 31 - 64-bit if set, 32-bit if unset. (OK!)
+ // 30 - sub if set, add if unset. (OK!)
+ // 29 - SetFlagsBit. Must be set.
+ // 22:23 - ShiftAddSub. (OK!)
+ // 10:21 - ImmAddSub. (OK!)
+ // 5:9 - First source register (Rn). (OK!)
+ // 0:4 - Destination Register. Must be xzr.
+
+ // From the above, there is a safe 19-bit contiguous region from 5:23.
+ Emit(i, vixl::ThirtyTwoBits | vixl::AddSubImmediateFixed | vixl::SUB |
+ Flags(vixl::SetFlags) | Rd(vixl::xzr) |
+ (imm19 << vixl::Rn_offset));
+}
+
+void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) {
+ const Instruction* first = reinterpret_cast<Instruction*>(inst_.raw());
+ Instruction* load;
+ Instruction* call;
+
+ // There might be a constant pool at the very first instruction.
+ first = first->skipPool();
+
+ // Skip the stack pointer restore instruction.
+ if (first->IsStackPtrSync()) {
+ first = first->InstructionAtOffset(vixl::kInstructionSize)->skipPool();
+ }
+
+ load = const_cast<Instruction*>(first);
+
+ // The call instruction follows the load, but there may be an injected
+ // constant pool.
+ call = const_cast<Instruction*>(
+ load->InstructionAtOffset(vixl::kInstructionSize)->skipPool());
+
+ if (call->IsBLR() == enabled) {
+ return;
+ }
+
+ if (call->IsBLR()) {
+ // If the second instruction is blr(), then we have:
+ // ldr x17, [pc, offset]
+ // blr x17
+ MOZ_ASSERT(load->IsLDR());
+ // We want to transform this to:
+ // adr xzr, [pc, offset]
+ // nop
+ int32_t offset = load->ImmLLiteral();
+ adr(load, xzr, int32_t(offset));
+ nop(call);
+ } else {
+ // We have:
+ // adr xzr, [pc, offset] (or ldr x17, [pc, offset])
+ // nop
+ MOZ_ASSERT(load->IsADR() || load->IsLDR());
+ MOZ_ASSERT(call->IsNOP());
+ // Transform this to:
+ // ldr x17, [pc, offset]
+ // blr x17
+ int32_t offset = (int)load->ImmPCRawOffset();
+ MOZ_ASSERT(vixl::IsInt19(offset));
+ ldr(load, ScratchReg2_64, int32_t(offset));
+ blr(call, ScratchReg2_64);
+ }
+}
+
+// Patches loads generated by MacroAssemblerCompat::mov(CodeLabel*, Register).
+// The loading code is implemented in movePatchablePtr().
+void Assembler::UpdateLoad64Value(Instruction* inst0, uint64_t value) {
+ MOZ_ASSERT(inst0->IsLDR());
+ uint64_t* literal = inst0->LiteralAddress<uint64_t*>();
+ *literal = value;
+}
+
+class RelocationIterator {
+ CompactBufferReader reader_;
+ uint32_t offset_ = 0;
+
+ public:
+ explicit RelocationIterator(CompactBufferReader& reader) : reader_(reader) {}
+
+ bool read() {
+ if (!reader_.more()) {
+ return false;
+ }
+ offset_ = reader_.readUnsigned();
+ return true;
+ }
+
+ uint32_t offset() const { return offset_; }
+};
+
+static JitCode* CodeFromJump(JitCode* code, uint8_t* jump) {
+ const Instruction* inst = (const Instruction*)jump;
+ uint8_t* target;
+
+ // We're expecting a call created by MacroAssembler::call(JitCode*).
+ // It looks like:
+ //
+ // ldr scratch, [pc, offset]
+ // blr scratch
+ //
+ // If the call has been toggled by ToggleCall(), it looks like:
+ //
+ // adr xzr, [pc, offset]
+ // nop
+ //
+ // There might be a constant pool at the very first instruction.
+ // See also ToggleCall().
+ inst = inst->skipPool();
+
+ // Skip the stack pointer restore instruction.
+ if (inst->IsStackPtrSync()) {
+ inst = inst->InstructionAtOffset(vixl::kInstructionSize)->skipPool();
+ }
+
+ if (inst->BranchType() != vixl::UnknownBranchType) {
+ // This is an immediate branch.
+ target = (uint8_t*)inst->ImmPCOffsetTarget();
+ } else if (inst->IsLDR()) {
+ // This is an ldr+blr call that is enabled. See ToggleCall().
+ mozilla::DebugOnly<const Instruction*> nextInst =
+ inst->InstructionAtOffset(vixl::kInstructionSize)->skipPool();
+ MOZ_ASSERT(nextInst->IsNOP() || nextInst->IsBLR());
+ target = (uint8_t*)inst->Literal64();
+ } else if (inst->IsADR()) {
+ // This is a disabled call: adr+nop. See ToggleCall().
+ mozilla::DebugOnly<const Instruction*> nextInst =
+ inst->InstructionAtOffset(vixl::kInstructionSize)->skipPool();
+ MOZ_ASSERT(nextInst->IsNOP());
+ ptrdiff_t offset = inst->ImmPCRawOffset() << vixl::kLiteralEntrySizeLog2;
+ // This is what Literal64 would do with the corresponding ldr.
+ memcpy(&target, inst + offset, sizeof(target));
+ } else {
+ MOZ_CRASH("Unrecognized jump instruction.");
+ }
+
+ // If the jump is within the code buffer, it uses the extended jump table.
+ if (target >= code->raw() &&
+ target < code->raw() + code->instructionsSize()) {
+ MOZ_ASSERT(target + Assembler::SizeOfJumpTableEntry <=
+ code->raw() + code->instructionsSize());
+
+ uint8_t** patchablePtr =
+ (uint8_t**)(target + Assembler::OffsetOfJumpTableEntryPointer);
+ target = *patchablePtr;
+ }
+
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ RelocationIterator iter(reader);
+ while (iter.read()) {
+ JitCode* child = CodeFromJump(code, code->raw() + iter.offset());
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ MOZ_ASSERT(child == CodeFromJump(code, code->raw() + iter.offset()));
+ }
+}
+
+/* static */
+void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+
+ uint8_t* buffer = code->raw();
+
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ Instruction* load = (Instruction*)&buffer[offset];
+
+ // The only valid traceable operation is a 64-bit load to an ARMRegister.
+ // Refer to movePatchablePtr() for generation.
+ MOZ_ASSERT(load->Mask(vixl::LoadLiteralMask) == vixl::LDR_x_lit);
+
+ uintptr_t* literalAddr = load->LiteralAddress<uintptr_t*>();
+ uintptr_t literal = *literalAddr;
+
+ // Data relocations can be for Values or for raw pointers. If a Value is
+ // zero-tagged, we can trace it as if it were a raw pointer. If a Value
+ // is not zero-tagged, we have to interpret it as a Value to ensure that the
+ // tag bits are masked off to recover the actual pointer.
+
+ if (literal >> JSVAL_TAG_SHIFT) {
+ // This relocation is a Value with a non-zero tag.
+ Value v = Value::fromRawBits(literal);
+ TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value");
+ if (*literalAddr != v.asRawBits()) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ *literalAddr = v.asRawBits();
+ }
+ continue;
+ }
+
+ // This relocation is a raw pointer or a Value with a zero tag.
+ // No barriers needed since the pointers are constants.
+ gc::Cell* cell = reinterpret_cast<gc::Cell*>(literal);
+ MOZ_ASSERT(gc::IsCellPointerValid(cell));
+ TraceManuallyBarrieredGenericPointerEdge(trc, &cell, "jit-masm-ptr");
+ if (uintptr_t(cell) != literal) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ *literalAddr = uintptr_t(cell);
+ }
+ }
+}
+
+void Assembler::retarget(Label* label, Label* target) {
+#ifdef JS_DISASM_ARM64
+ spew_.spewRetarget(label, target);
+#endif
+ if (label->used()) {
+ if (target->bound()) {
+ bind(label, BufferOffset(target));
+ } else if (target->used()) {
+ // The target is not bound but used. Prepend label's branch list
+ // onto target's.
+ BufferOffset labelBranchOffset(label);
+
+ // Find the head of the use chain for label.
+ BufferOffset next = NextLink(labelBranchOffset);
+ while (next.assigned()) {
+ labelBranchOffset = next;
+ next = NextLink(next);
+ }
+
+ // Then patch the head of label's use chain to the tail of target's
+ // use chain, prepending the entire use chain of target.
+ SetNextLink(labelBranchOffset, BufferOffset(target));
+ target->use(label->offset());
+ } else {
+ // The target is unbound and unused. We can just take the head of
+ // the list hanging off of label, and dump that into target.
+ target->use(label->offset());
+ }
+ }
+ label->reset();
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/arm64/Assembler-arm64.h b/js/src/jit/arm64/Assembler-arm64.h
new file mode 100644
index 0000000000..adaa45c0b4
--- /dev/null
+++ b/js/src/jit/arm64/Assembler-arm64.h
@@ -0,0 +1,788 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef A64_ASSEMBLER_A64_H_
+#define A64_ASSEMBLER_A64_H_
+
+#include <iterator>
+
+#include "jit/arm64/vixl/Assembler-vixl.h"
+
+#include "jit/CompactBuffer.h"
+#include "jit/shared/Disassembler-shared.h"
+#include "wasm/WasmTypeDecls.h"
+
+namespace js {
+namespace jit {
+
+// VIXL imports.
+typedef vixl::Register ARMRegister;
+typedef vixl::FPRegister ARMFPRegister;
+using vixl::ARMBuffer;
+using vixl::Instruction;
+
+using LabelDoc = DisassemblerSpew::LabelDoc;
+using LiteralDoc = DisassemblerSpew::LiteralDoc;
+
+static const uint32_t AlignmentAtPrologue = 0;
+static const uint32_t AlignmentMidPrologue = 8;
+static const Scale ScalePointer = TimesEight;
+
+// The MacroAssembler uses scratch registers extensively and unexpectedly.
+// For safety, scratch registers should always be acquired using
+// vixl::UseScratchRegisterScope.
+static constexpr Register ScratchReg{Registers::ip0};
+static constexpr ARMRegister ScratchReg64 = {ScratchReg, 64};
+
+static constexpr Register ScratchReg2{Registers::ip1};
+static constexpr ARMRegister ScratchReg2_64 = {ScratchReg2, 64};
+
+static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::d0,
+ FloatRegisters::Double};
+static constexpr FloatRegister ScratchDoubleReg_ = {FloatRegisters::d31,
+ FloatRegisters::Double};
+struct ScratchDoubleScope : public AutoFloatRegisterScope {
+ explicit ScratchDoubleScope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {}
+};
+
+static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::s0,
+ FloatRegisters::Single};
+static constexpr FloatRegister ScratchFloat32Reg_ = {FloatRegisters::s31,
+ FloatRegisters::Single};
+struct ScratchFloat32Scope : public AutoFloatRegisterScope {
+ explicit ScratchFloat32Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {}
+};
+
+#ifdef ENABLE_WASM_SIMD
+static constexpr FloatRegister ReturnSimd128Reg = {FloatRegisters::v0,
+ FloatRegisters::Simd128};
+static constexpr FloatRegister ScratchSimd128Reg = {FloatRegisters::v31,
+ FloatRegisters::Simd128};
+struct ScratchSimd128Scope : public AutoFloatRegisterScope {
+ explicit ScratchSimd128Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchSimd128Reg) {}
+};
+#else
+struct ScratchSimd128Scope : public AutoFloatRegisterScope {
+ explicit ScratchSimd128Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {
+ MOZ_CRASH("SIMD not enabled");
+ }
+};
+#endif
+
+static constexpr Register InvalidReg{Registers::Invalid};
+static constexpr FloatRegister InvalidFloatReg = {};
+
+static constexpr Register OsrFrameReg{Registers::x3};
+static constexpr Register CallTempReg0{Registers::x9};
+static constexpr Register CallTempReg1{Registers::x10};
+static constexpr Register CallTempReg2{Registers::x11};
+static constexpr Register CallTempReg3{Registers::x12};
+static constexpr Register CallTempReg4{Registers::x13};
+static constexpr Register CallTempReg5{Registers::x14};
+
+static constexpr Register PreBarrierReg{Registers::x1};
+
+static constexpr Register InterpreterPCReg{Registers::x9};
+
+static constexpr Register ReturnReg{Registers::x0};
+static constexpr Register64 ReturnReg64(ReturnReg);
+static constexpr Register JSReturnReg{Registers::x2};
+static constexpr Register FramePointer{Registers::fp};
+static constexpr ARMRegister FramePointer64{FramePointer, 64};
+static constexpr Register ZeroRegister{Registers::sp};
+static constexpr ARMRegister ZeroRegister64{Registers::sp, 64};
+static constexpr ARMRegister ZeroRegister32{Registers::sp, 32};
+
+// [SMDOC] AArch64 Stack Pointer and Pseudo Stack Pointer conventions
+//
+// ================
+//
+// Stack pointer (SP), PseudoStackPointer (PSP), and RealStackPointer:
+//
+// The ARM64 real SP has a constraint: it must be 16-byte aligned whenever it
+// is used as the base pointer for a memory access. (SP+offset need not be
+// 16-byte aligned, but the SP value itself must be.) The SP register may
+// take on unaligned values but may not be used for a memory access while it
+// is unaligned.
+//
+// Stack-alignment checking can be enabled or disabled by a control register;
+// however that register cannot be modified by user space. We have to assume
+// stack alignment checking is enabled, and that does usually appear to be the
+// case. See the ARM Architecture Reference Manual, "D1.8.2 SP alignment
+// checking", for further details.
+//
+// A second constraint is forced upon us by the ARM64 ABI. This requires that
+// all accesses to the stack must be at or above SP. Accesses below SP are
+// strictly forbidden, presumably because the kernel might use that area of
+// memory for its own purposes -- in particular, signal delivery -- and hence
+// it may get trashed at any time.
+//
+// Note this doesn't mean that accesses to the stack must be based off
+// register SP. Only that the effective addresses must be >= SP, regardless
+// of how the address is formed.
+//
+// In order to allow word-wise pushes and pops, some of our ARM64 jits
+// (JS-Baseline, JS-Ion, and Wasm-Ion, but not Wasm-Baseline) dedicate x28 to
+// be used as a PseudoStackPointer (PSP).
+//
+// Initially the PSP will have the same value as the SP. Code can, if it
+// wants, push a single word by subtracting 8 from the PSP, doing SP := PSP,
+// then storing the value at PSP+0. Given other constraints on the alignment
+// of the SP at function call boundaries, this works out OK, at the cost of
+// the two extra instructions per push / pop.
+//
+// This is all a bit messy, and is probably not robustly adhered to. However,
+// the following appear to be the intended, and mostly implemented, current
+// invariants:
+//
+// (1) PSP is "primary", SP is "secondary". Most stack refs are
+// PSP-relative. SP-relative is rare and (obviously) only done when we
+// know that SP is aligned.
+//
+// (2) At all times, the relationship SP <= PSP is maintained. The fact that
+// SP may validly be less than PSP means that pushes on the stack force
+// the two values to become equal, by copying PSP into SP. However, pops
+// behave differently: PSP moves back up and SP stays the same, since that
+// doesn't break the SP <= PSP invariant.
+//
+// (3) However, immediately before a call instruction, SP and PSP must be the
+// same. To enforce this, PSP is copied into SP by the arm64-specific
+// MacroAssembler::call routines.
+//
+// (4) Also, after a function has returned, it is expected that SP holds the
+// "primary" value. How exactly this is implemented remains not entirely
+// clear and merits further investigation. The following points are
+// believed to be relevant:
+//
+// - For calls to functions observing the system AArch64 ABI, PSP (x28) is
+// callee-saved. That, combined with (3) above, implies SP == PSP
+// immediately after the call returns.
+//
+// - JIT-generated routines return using MacroAssemblerCompat::retn, and
+// that copies PSP into SP (bizarrely; this would make more sense if it
+// copied SP into PSP); but in any case, the point is that they are the
+// same at the point that the return instruction executes.
+//
+// - MacroAssembler::callWithABIPost copies PSP into SP after the return
+// of a call requiring dynamic alignment.
+//
+// Given the above, it is unclear exactly where in the return sequence it
+// is expected that SP == PSP, and also whether it is the callee or caller
+// that is expected to enforce it.
+//
+// In general it would be nice to be able to move (at some time in the future,
+// not now) to a world where *every* assignment to PSP or SP is followed
+// immediately by a copy into the other register. That would make all
+// required correctness proofs trivial in the sense that it would require only
+// local inspection of code immediately following (dominated by) any such
+// assignment. For the moment, however, this is a guideline, not a hard
+// requirement.
+//
+// ================
+//
+// Mechanics of keeping the stack pointers in sync:
+//
+// The following two methods require that the masm's SP has been set to the PSP
+// with MacroAssembler::SetStackPointer64(PseudoStackPointer64), or they will be
+// no-ops. The setup is performed manually by the jits after creating the masm.
+//
+// * MacroAssembler::syncStackPtr() performs SP := PSP, presumably after PSP has
+// been updated, so SP needs to move too. This is used pretty liberally
+// throughout the code base.
+//
+// * MacroAssembler::initPseudoStackPtr() performs PSP := SP. This can be used
+// after calls to non-ABI compliant code; it's not used much.
+//
+// In the ARM64 assembler there is a function Instruction::IsStackPtrSync() that
+// recognizes the instruction emitted by syncStackPtr(), and this is used to
+// skip that instruction a few places, should it be present, in the JS JIT where
+// code is generated to deal with toggled calls.
+//
+// In various places there are calls to MacroAssembler::syncStackPtr() which
+// appear to be redundant. Investigation shows that they often are redundant,
+// but not always. Finding and removing such redundancies would be quite some
+// work, so we live for now with the occasional redundant update. Perusal of
+// the Cortex-A55 and -A72 optimization guides shows no evidence that such
+// assignments are any more expensive than assignments between vanilla integer
+// registers, so the costs of such redundant updates are assumed to be small.
+//
+// Invariants on the PSP at function call boundaries:
+//
+// It *appears* that the following invariants exist:
+//
+// * On entry to JIT code, PSP == SP, ie the stack pointer is transmitted via
+// both registers.
+//
+// * On entry to C++ code, PSP == SP. Certainly it appears that all calls
+// created by the MacroAssembler::call(..) routines perform 'syncStackPtr'
+// immediately before the call, and all ABI calls are routed through the
+// MacroAssembler::call layer.
+//
+// * The stubs generated by WasmStubs.cpp assume that, on entry, SP is the
+// active stack pointer and that PSP is dead.
+//
+// * The PSP is non-volatile (callee-saved). Along a normal return path from
+// JIT code, simply having PSP == SP on exit is correct, since the exit SP is
+// the same as the entry SP by the JIT ABI.
+//
+// * Call-outs to non-JIT C++ code do not need to set up the PSP (it won't be
+// used), and will not need to restore the PSP on return because x28 is
+// non-volatile in the ARM64 ABI.
+//
+// ================
+//
+// Future cleanups to the SP-vs-PSP machinery:
+//
+// Currently we have somewhat unclear invariants, which are not obviously
+// always enforced, and which may require complex non-local reasoning.
+// Auditing the code to ensure that the invariants always hold, whilst not
+// generating duplicate syncs, is close to impossible. A future rework to
+// tidy this might be as follows. (This suggestion pertains the the entire
+// JIT complex: all of the JS compilers, wasm compilers, stub generators,
+// regexp compilers, etc).
+//
+// Currently we have that, in JIT-generated code, PSP is "primary" and SP is
+// "secondary", meaning that PSP has the "real" stack pointer value and SP is
+// updated whenever PSP acquires a lower value, so as to ensure that SP <= PSP.
+// An exception to this scheme is the stubs code generated by WasmStubs.cpp,
+// which assumes that SP is "primary" and PSP is dead.
+//
+// It might give us an easier incremental path to eventually removing PSP
+// entirely if we switched to having SP always be the primary. That is:
+//
+// (1) SP is primary, PSP is secondary
+// (2) After any assignment to SP, it is copied into PSP
+// (3) All (non-frame-pointer-based) stack accesses are PSP-relative
+// (as at present)
+//
+// This would have the effect that:
+//
+// * It would reinstate the invariant that on all targets, the "real" SP value
+// is in the ABI-and-or-hardware-mandated stack pointer register.
+//
+// * It would give us a simple story about calls and returns:
+// - for calls to non-JIT generated code (viz, C++ etc), we need no extra
+// copies, because PSP (x28) is callee-saved
+// - for calls to JIT-generated code, we need no extra copies, because of (2)
+// above
+//
+// * We could incrementally migrate those parts of the code generator where we
+// know that SP is 16-aligned, to use SP- rather than PSP-relative accesses
+//
+// * The consistent use of (2) would remove the requirement to have to perform
+// path-dependent reasoning (for paths in the generated code, not in the
+// compiler) when reading/understanding the code.
+//
+// * x28 would become free for use by stubs and the baseline compiler without
+// having to worry about interoperating with code that expects x28 to hold a
+// valid PSP.
+//
+// One might ask what mechanical checks we can add to ensure correctness, rather
+// than having to verify these invariants by hand indefinitely. Maybe some
+// combination of:
+//
+// * In debug builds, compiling-in assert(SP == PSP) at critical places. This
+// can be done using the existing `assertStackPtrsSynced` function.
+//
+// * In debug builds, scanning sections of generated code to ensure no
+// SP-relative stack accesses have been created -- for some sections, at
+// least every assignment to SP is immediately followed by a copy to x28.
+// This would also facilitate detection of duplicate syncs.
+//
+// ================
+//
+// Other investigative notes, for the code base at present:
+//
+// * Some disassembly dumps suggest that we sync the stack pointer too often.
+// This could be the result of various pieces of code working at cross
+// purposes when syncing the stack pointer, or of not paying attention to the
+// precise invariants.
+//
+// * As documented in RegExpNativeMacroAssembler.cpp, function
+// SMRegExpMacroAssembler::createStackFrame:
+//
+// // ARM64 communicates stack address via SP, but uses a pseudo-sp (PSP) for
+// // addressing. The register we use for PSP may however also be used by
+// // calling code, and it is nonvolatile, so save it. Do this as a special
+// // case first because the generic save/restore code needs the PSP to be
+// // initialized already.
+//
+// and also in function SMRegExpMacroAssembler::exitHandler:
+//
+// // Restore the saved value of the PSP register, this value is whatever the
+// // caller had saved in it, not any actual SP value, and it must not be
+// // overwritten subsequently.
+//
+// The original source for these comments was a patch for bug 1445907.
+//
+// * MacroAssembler-arm64.h has an interesting comment in the retn()
+// function:
+//
+// syncStackPtr(); // SP is always used to transmit the stack between calls.
+//
+// Same comment at abiret() in that file, and in MacroAssembler-arm64.cpp,
+// at callWithABIPre and callWithABIPost.
+//
+// * In Trampoline-arm64.cpp function JitRuntime::generateVMWrapper we find
+//
+// // SP is used to transfer stack across call boundaries.
+// masm.initPseudoStackPtr();
+//
+// after the return point of a callWithVMWrapper. The only reasonable
+// conclusion from all those (assuming they are right) is that SP == PSP.
+//
+// * Wasm-Baseline does not use the PSP, but as Wasm-Ion code requires SP==PSP
+// and tiered code can have Baseline->Ion calls, Baseline will set PSP=SP
+// before a call to wasm code.
+//
+// ================
+
+// StackPointer is intentionally undefined on ARM64 to prevent misuse: using
+// sp as a base register is only valid if sp % 16 == 0.
+static constexpr Register RealStackPointer{Registers::sp};
+
+static constexpr Register PseudoStackPointer{Registers::x28};
+static constexpr ARMRegister PseudoStackPointer64 = {Registers::x28, 64};
+static constexpr ARMRegister PseudoStackPointer32 = {Registers::x28, 32};
+
+static constexpr Register IntArgReg0{Registers::x0};
+static constexpr Register IntArgReg1{Registers::x1};
+static constexpr Register IntArgReg2{Registers::x2};
+static constexpr Register IntArgReg3{Registers::x3};
+static constexpr Register IntArgReg4{Registers::x4};
+static constexpr Register IntArgReg5{Registers::x5};
+static constexpr Register IntArgReg6{Registers::x6};
+static constexpr Register IntArgReg7{Registers::x7};
+static constexpr Register HeapReg{Registers::x21};
+
+// Define unsized Registers.
+#define DEFINE_UNSIZED_REGISTERS(N) \
+ static constexpr Register r##N{Registers::x##N};
+REGISTER_CODE_LIST(DEFINE_UNSIZED_REGISTERS)
+#undef DEFINE_UNSIZED_REGISTERS
+static constexpr Register ip0{Registers::x16};
+static constexpr Register ip1{Registers::x17};
+static constexpr Register fp{Registers::x29};
+static constexpr Register lr{Registers::x30};
+static constexpr Register rzr{Registers::xzr};
+
+// Import VIXL registers into the js::jit namespace.
+#define IMPORT_VIXL_REGISTERS(N) \
+ static constexpr ARMRegister w##N = vixl::w##N; \
+ static constexpr ARMRegister x##N = vixl::x##N;
+REGISTER_CODE_LIST(IMPORT_VIXL_REGISTERS)
+#undef IMPORT_VIXL_REGISTERS
+static constexpr ARMRegister wzr = vixl::wzr;
+static constexpr ARMRegister xzr = vixl::xzr;
+static constexpr ARMRegister wsp = vixl::wsp;
+static constexpr ARMRegister sp = vixl::sp;
+
+// Import VIXL VRegisters into the js::jit namespace.
+#define IMPORT_VIXL_VREGISTERS(N) \
+ static constexpr ARMFPRegister s##N = vixl::s##N; \
+ static constexpr ARMFPRegister d##N = vixl::d##N;
+REGISTER_CODE_LIST(IMPORT_VIXL_VREGISTERS)
+#undef IMPORT_VIXL_VREGISTERS
+
+static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg);
+
+// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registerd used in RegExpTester instruction (do not use ReturnReg).
+static constexpr Register RegExpTesterRegExpReg = CallTempReg0;
+static constexpr Register RegExpTesterStringReg = CallTempReg1;
+static constexpr Register RegExpTesterLastIndexReg = CallTempReg2;
+
+static constexpr Register JSReturnReg_Type = r3;
+static constexpr Register JSReturnReg_Data = r2;
+
+static constexpr FloatRegister NANReg = {FloatRegisters::d14,
+ FloatRegisters::Single};
+// N.B. r8 isn't listed as an aapcs temp register, but we can use it as such
+// because we never use return-structs.
+static constexpr Register CallTempNonArgRegs[] = {r8, r9, r10, r11,
+ r12, r13, r14, r15};
+static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr uint32_t JitStackAlignment = 16;
+
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+ JitStackValueAlignment >= 1,
+ "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+static constexpr uint32_t SimdMemoryAlignment = 16;
+
+static_assert(CodeAlignment % SimdMemoryAlignment == 0,
+ "Code alignment should be larger than any of the alignments "
+ "which are used for "
+ "the constant sections of the code buffer. Thus it should be "
+ "larger than the "
+ "alignment for SIMD constants.");
+
+static const uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static const uint32_t WasmTrapInstructionLength = 4;
+
+// See comments in wasm::GenerateFunctionPrologue. The difference between these
+// is the size of the largest callable prologue on the platform.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+class Assembler : public vixl::Assembler {
+ public:
+ Assembler() : vixl::Assembler() {}
+
+ typedef vixl::Condition Condition;
+
+ void finish();
+ bool appendRawCode(const uint8_t* code, size_t numBytes);
+ bool reserve(size_t size);
+ bool swapBuffer(wasm::Bytes& bytes);
+
+ // Emit the jump table, returning the BufferOffset to the first entry in the
+ // table.
+ BufferOffset emitExtendedJumpTable();
+ BufferOffset ExtendedJumpTable_;
+ void executableCopy(uint8_t* buffer);
+
+ BufferOffset immPool(ARMRegister dest, uint8_t* value, vixl::LoadLiteralOp op,
+ const LiteralDoc& doc,
+ ARMBuffer::PoolEntry* pe = nullptr);
+ BufferOffset immPool64(ARMRegister dest, uint64_t value,
+ ARMBuffer::PoolEntry* pe = nullptr);
+ BufferOffset fImmPool(ARMFPRegister dest, uint8_t* value,
+ vixl::LoadLiteralOp op, const LiteralDoc& doc);
+ BufferOffset fImmPool64(ARMFPRegister dest, double value);
+ BufferOffset fImmPool32(ARMFPRegister dest, float value);
+
+ uint32_t currentOffset() const { return nextOffset().getOffset(); }
+
+ void bind(Label* label) { bind(label, nextOffset()); }
+ void bind(Label* label, BufferOffset boff);
+ void bind(CodeLabel* label) { label->target()->bind(currentOffset()); }
+
+ void setUnlimitedBuffer() { armbuffer_.setUnlimited(); }
+ bool oom() const {
+ return AssemblerShared::oom() || armbuffer_.oom() ||
+ jumpRelocations_.oom() || dataRelocations_.oom();
+ }
+
+ void copyJumpRelocationTable(uint8_t* dest) const {
+ if (jumpRelocations_.length()) {
+ memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
+ }
+ }
+ void copyDataRelocationTable(uint8_t* dest) const {
+ if (dataRelocations_.length()) {
+ memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
+ }
+ }
+
+ size_t jumpRelocationTableBytes() const { return jumpRelocations_.length(); }
+ size_t dataRelocationTableBytes() const { return dataRelocations_.length(); }
+ size_t bytesNeeded() const {
+ return SizeOfCodeGenerated() + jumpRelocationTableBytes() +
+ dataRelocationTableBytes();
+ }
+
+ void processCodeLabels(uint8_t* rawCode) {
+ for (const CodeLabel& label : codeLabels_) {
+ Bind(rawCode, label);
+ }
+ }
+
+ static void UpdateLoad64Value(Instruction* inst0, uint64_t value);
+
+ static void Bind(uint8_t* rawCode, const CodeLabel& label) {
+ auto mode = label.linkMode();
+ size_t patchAtOffset = label.patchAt().offset();
+ size_t targetOffset = label.target().offset();
+
+ if (mode == CodeLabel::MoveImmediate) {
+ Instruction* inst = (Instruction*)(rawCode + patchAtOffset);
+ Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + targetOffset));
+ } else {
+ *reinterpret_cast<const void**>(rawCode + patchAtOffset) =
+ rawCode + targetOffset;
+ }
+ }
+
+ void retarget(Label* cur, Label* next);
+
+ // The buffer is about to be linked. Ensure any constant pools or
+ // excess bookkeeping has been flushed to the instruction stream.
+ void flush() { armbuffer_.flushPool(); }
+
+ void comment(const char* msg) {
+#ifdef JS_DISASM_ARM64
+ spew_.spew("; %s", msg);
+#endif
+ }
+
+ void setPrinter(Sprinter* sp) {
+#ifdef JS_DISASM_ARM64
+ spew_.setPrinter(sp);
+#endif
+ }
+
+ static bool SupportsFloatingPoint() { return true; }
+ static bool SupportsUnalignedAccesses() { return true; }
+ static bool SupportsFastUnalignedFPAccesses() { return true; }
+ static bool SupportsWasmSimd() { return true; }
+
+ static bool HasRoundInstruction(RoundingMode mode) {
+ switch (mode) {
+ case RoundingMode::Up:
+ case RoundingMode::Down:
+ case RoundingMode::NearestTiesToEven:
+ case RoundingMode::TowardsZero:
+ return true;
+ }
+ MOZ_CRASH("unexpected mode");
+ }
+
+ protected:
+ // Add a jump whose target is unknown until finalization.
+ // The jump may not be patched at runtime.
+ void addPendingJump(BufferOffset src, ImmPtr target, RelocationKind kind);
+
+ public:
+ static uint32_t PatchWrite_NearCallSize() { return 4; }
+
+ static uint32_t NopSize() { return 4; }
+
+ static void PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall);
+ static void PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expected);
+
+ static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
+ ImmPtr expected);
+
+ static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) {
+ // Raw is going to be the return address.
+ uint32_t* raw = (uint32_t*)label.raw();
+ // Overwrite the 4 bytes before the return address, which will end up being
+ // the call instruction.
+ *(raw - 1) = imm.value;
+ }
+ static uint32_t AlignDoubleArg(uint32_t offset) {
+ MOZ_CRASH("AlignDoubleArg()");
+ }
+ static uintptr_t GetPointer(uint8_t* ptr) {
+ Instruction* i = reinterpret_cast<Instruction*>(ptr);
+ uint64_t ret = i->Literal64();
+ return ret;
+ }
+
+ // Toggle a jmp or cmp emitted by toggledJump().
+ static void ToggleToJmp(CodeLocationLabel inst_);
+ static void ToggleToCmp(CodeLocationLabel inst_);
+ static void ToggleCall(CodeLocationLabel inst_, bool enabled);
+
+ static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+ static void TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ void assertNoGCThings() const {
+#ifdef DEBUG
+ MOZ_ASSERT(dataRelocations_.length() == 0);
+ for (auto& j : pendingJumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+#endif
+ }
+
+ public:
+ // A Jump table entry is 2 instructions, with 8 bytes of raw data
+ static const size_t SizeOfJumpTableEntry = 16;
+
+ struct JumpTableEntry {
+ uint32_t ldr;
+ uint32_t br;
+ void* data;
+
+ Instruction* getLdr() { return reinterpret_cast<Instruction*>(&ldr); }
+ };
+
+ // Offset of the patchable target for the given entry.
+ static const size_t OffsetOfJumpTableEntryPointer = 8;
+
+ public:
+ void writeCodePointer(CodeLabel* label) {
+ armbuffer_.assertNoPoolAndNoNops();
+ uintptr_t x = uintptr_t(-1);
+ BufferOffset off = EmitData(&x, sizeof(uintptr_t));
+ label->patchAt()->bind(off.getOffset());
+ }
+
+ void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
+ const Disassembler::HeapAccess& heapAccess) {
+ MOZ_CRASH("verifyHeapAccessDisassembly");
+ }
+
+ protected:
+ // Structure for fixing up pc-relative loads/jumps when the machine
+ // code gets moved (executable copy, gc, etc.).
+ struct RelativePatch {
+ BufferOffset offset;
+ void* target;
+ RelocationKind kind;
+
+ RelativePatch(BufferOffset offset, void* target, RelocationKind kind)
+ : offset(offset), target(target), kind(kind) {}
+ };
+
+ // List of jumps for which the target is either unknown until finalization,
+ // or cannot be known due to GC. Each entry here requires a unique entry
+ // in the extended jump table, and is patched at finalization.
+ js::Vector<RelativePatch, 8, SystemAllocPolicy> pendingJumps_;
+
+ // Final output formatters.
+ CompactBufferWriter jumpRelocations_;
+ CompactBufferWriter dataRelocations_;
+};
+
+static const uint32_t NumIntArgRegs = 8;
+static const uint32_t NumFloatArgRegs = 8;
+
+class ABIArgGenerator {
+ public:
+ ABIArgGenerator()
+ : intRegIndex_(0), floatRegIndex_(0), stackOffset_(0), current_() {}
+
+ ABIArg next(MIRType argType);
+ ABIArg& current() { return current_; }
+ uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+ void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+
+ protected:
+ unsigned intRegIndex_;
+ unsigned floatRegIndex_;
+ uint32_t stackOffset_;
+ ABIArg current_;
+};
+
+// These registers may be volatile or nonvolatile.
+static constexpr Register ABINonArgReg0 = r8;
+static constexpr Register ABINonArgReg1 = r9;
+static constexpr Register ABINonArgReg2 = r10;
+static constexpr Register ABINonArgReg3 = r11;
+
+// This register may be volatile or nonvolatile. Avoid d31 which is the
+// ScratchDoubleReg_.
+static constexpr FloatRegister ABINonArgDoubleReg = {FloatRegisters::s16,
+ FloatRegisters::Single};
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = r8;
+static constexpr Register ABINonArgReturnReg1 = r9;
+static constexpr Register ABINonVolatileReg{Registers::x19};
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = lr;
+
+// Instance pointer argument register for WebAssembly functions. This must not
+// alias any other register used for passing function arguments or return
+// values. Preserved by WebAssembly functions. Must be nonvolatile.
+static constexpr Register InstanceReg{Registers::x23};
+
+// Registers used for wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = r9;
+
+static inline bool GetIntArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs,
+ Register* out) {
+ if (usedIntArgs >= NumIntArgRegs) {
+ return false;
+ }
+ *out = Register::FromCode(usedIntArgs);
+ return true;
+}
+
+static inline bool GetFloatArgReg(uint32_t usedIntArgs, uint32_t usedFloatArgs,
+ FloatRegister* out) {
+ if (usedFloatArgs >= NumFloatArgRegs) {
+ return false;
+ }
+ *out = FloatRegister::FromCode(usedFloatArgs);
+ return true;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs, Register* out) {
+ if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) {
+ return true;
+ }
+ // Unfortunately, we have to assume things about the point at which
+ // GetIntArgReg returns false, because we need to know how many registers it
+ // can allocate.
+ usedIntArgs -= NumIntArgRegs;
+ if (usedIntArgs >= NumCallTempNonArgRegs) {
+ return false;
+ }
+ *out = CallTempNonArgRegs[usedIntArgs];
+ return true;
+}
+
+inline Imm32 Imm64::firstHalf() const { return low(); }
+
+inline Imm32 Imm64::secondHalf() const { return hi(); }
+
+// Forbids nop filling for testing purposes. Not nestable.
+class AutoForbidNops {
+ protected:
+ Assembler* asm_;
+
+ public:
+ explicit AutoForbidNops(Assembler* asm_) : asm_(asm_) { asm_->enterNoNops(); }
+ ~AutoForbidNops() { asm_->leaveNoNops(); }
+};
+
+// Forbids pool generation during a specified interval. Not nestable.
+class AutoForbidPoolsAndNops : public AutoForbidNops {
+ public:
+ AutoForbidPoolsAndNops(Assembler* asm_, size_t maxInst)
+ : AutoForbidNops(asm_) {
+ asm_->enterNoPool(maxInst);
+ }
+ ~AutoForbidPoolsAndNops() { asm_->leaveNoPool(); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif // A64_ASSEMBLER_A64_H_
diff --git a/js/src/jit/arm64/CodeGenerator-arm64.cpp b/js/src/jit/arm64/CodeGenerator-arm64.cpp
new file mode 100644
index 0000000000..13696a7374
--- /dev/null
+++ b/js/src/jit/arm64/CodeGenerator-arm64.cpp
@@ -0,0 +1,4248 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm64/CodeGenerator-arm64.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jsnum.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitRuntime.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "jit/ReciprocalMulConstants.h"
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+#include "vm/Shape.h"
+
+#include "jit/shared/CodeGenerator-shared-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::GenericNaN;
+using mozilla::FloorLog2;
+using mozilla::Maybe;
+using mozilla::NegativeInfinity;
+using mozilla::Nothing;
+using mozilla::Some;
+
+// shared
+CodeGeneratorARM64::CodeGeneratorARM64(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorShared(gen, graph, masm) {}
+
+bool CodeGeneratorARM64::generateOutOfLineCode() {
+ AutoCreatedBy acb(masm, "CodeGeneratorARM64::generateOutOfLineCode");
+
+ if (!CodeGeneratorShared::generateOutOfLineCode()) {
+ return false;
+ }
+
+ if (deoptLabel_.used()) {
+ // All non-table-based bailouts will go here.
+ masm.bind(&deoptLabel_);
+
+ // Store the frame size, so the handler can recover the IonScript.
+ masm.push(Imm32(frameSize()));
+
+ TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
+ masm.jump(handler);
+ }
+
+ return !masm.oom();
+}
+
+void CodeGeneratorARM64::emitBranch(Assembler::Condition cond,
+ MBasicBlock* mirTrue,
+ MBasicBlock* mirFalse) {
+ if (isNextBlock(mirFalse->lir())) {
+ jumpToBlock(mirTrue, cond);
+ } else {
+ jumpToBlock(mirFalse, Assembler::InvertCondition(cond));
+ jumpToBlock(mirTrue);
+ }
+}
+
+void OutOfLineBailout::accept(CodeGeneratorARM64* codegen) {
+ codegen->visitOutOfLineBailout(this);
+}
+
+void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
+ Register input = ToRegister(test->input());
+ MBasicBlock* mirTrue = test->ifTrue();
+ MBasicBlock* mirFalse = test->ifFalse();
+
+ // Jump to the True block if NonZero.
+ // Jump to the False block if Zero.
+ if (isNextBlock(mirFalse->lir())) {
+ masm.branch32(Assembler::NonZero, input, Imm32(0),
+ getJumpLabelForBranch(mirTrue));
+ } else {
+ masm.branch32(Assembler::Zero, input, Imm32(0),
+ getJumpLabelForBranch(mirFalse));
+ if (!isNextBlock(mirTrue->lir())) {
+ jumpToBlock(mirTrue);
+ }
+ }
+}
+
+void CodeGenerator::visitCompare(LCompare* comp) {
+ const MCompare* mir = comp->mir();
+ const MCompare::CompareType type = mir->compareType();
+ const Assembler::Condition cond = JSOpToCondition(type, comp->jsop());
+ const Register leftreg = ToRegister(comp->getOperand(0));
+ const LAllocation* right = comp->getOperand(1);
+ const Register defreg = ToRegister(comp->getDef(0));
+
+ if (type == MCompare::Compare_Object || type == MCompare::Compare_Symbol ||
+ type == MCompare::Compare_UIntPtr ||
+ type == MCompare::Compare_RefOrNull) {
+ if (right->isConstant()) {
+ MOZ_ASSERT(type == MCompare::Compare_UIntPtr);
+ masm.cmpPtrSet(cond, leftreg, Imm32(ToInt32(right)), defreg);
+ } else {
+ masm.cmpPtrSet(cond, leftreg, ToRegister(right), defreg);
+ }
+ return;
+ }
+
+ if (right->isConstant()) {
+ masm.cmp32Set(cond, leftreg, Imm32(ToInt32(right)), defreg);
+ } else {
+ masm.cmp32Set(cond, leftreg, ToRegister(right), defreg);
+ }
+}
+
+void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
+ const MCompare* mir = comp->cmpMir();
+ const MCompare::CompareType type = mir->compareType();
+ const LAllocation* left = comp->left();
+ const LAllocation* right = comp->right();
+
+ if (type == MCompare::Compare_Object || type == MCompare::Compare_Symbol ||
+ type == MCompare::Compare_UIntPtr ||
+ type == MCompare::Compare_RefOrNull) {
+ if (right->isConstant()) {
+ MOZ_ASSERT(type == MCompare::Compare_UIntPtr);
+ masm.cmpPtr(ToRegister(left), Imm32(ToInt32(right)));
+ } else {
+ masm.cmpPtr(ToRegister(left), ToRegister(right));
+ }
+ } else if (right->isConstant()) {
+ masm.cmp32(ToRegister(left), Imm32(ToInt32(right)));
+ } else {
+ masm.cmp32(ToRegister(left), ToRegister(right));
+ }
+
+ Assembler::Condition cond = JSOpToCondition(type, comp->jsop());
+ emitBranch(cond, comp->ifTrue(), comp->ifFalse());
+}
+
+void CodeGeneratorARM64::bailoutIf(Assembler::Condition condition,
+ LSnapshot* snapshot) {
+ encode(snapshot);
+
+ InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
+ OutOfLineBailout* ool = new (alloc()) OutOfLineBailout(snapshot);
+ addOutOfLineCode(ool,
+ new (alloc()) BytecodeSite(tree, tree->script()->code()));
+
+ masm.B(ool->entry(), condition);
+}
+
+void CodeGeneratorARM64::bailoutFrom(Label* label, LSnapshot* snapshot) {
+ MOZ_ASSERT_IF(!masm.oom(), label->used());
+ MOZ_ASSERT_IF(!masm.oom(), !label->bound());
+
+ encode(snapshot);
+
+ InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
+ OutOfLineBailout* ool = new (alloc()) OutOfLineBailout(snapshot);
+ addOutOfLineCode(ool,
+ new (alloc()) BytecodeSite(tree, tree->script()->code()));
+
+ masm.retarget(label, ool->entry());
+}
+
+void CodeGeneratorARM64::bailout(LSnapshot* snapshot) {
+ Label label;
+ masm.b(&label);
+ bailoutFrom(&label, snapshot);
+}
+
+void CodeGeneratorARM64::visitOutOfLineBailout(OutOfLineBailout* ool) {
+ masm.push(Imm32(ool->snapshot()->snapshotOffset()));
+ masm.B(&deoptLabel_);
+}
+
+void CodeGenerator::visitMinMaxD(LMinMaxD* ins) {
+ ARMFPRegister lhs(ToFloatRegister(ins->first()), 64);
+ ARMFPRegister rhs(ToFloatRegister(ins->second()), 64);
+ ARMFPRegister output(ToFloatRegister(ins->output()), 64);
+ if (ins->mir()->isMax()) {
+ masm.Fmax(output, lhs, rhs);
+ } else {
+ masm.Fmin(output, lhs, rhs);
+ }
+}
+
+void CodeGenerator::visitMinMaxF(LMinMaxF* ins) {
+ ARMFPRegister lhs(ToFloatRegister(ins->first()), 32);
+ ARMFPRegister rhs(ToFloatRegister(ins->second()), 32);
+ ARMFPRegister output(ToFloatRegister(ins->output()), 32);
+ if (ins->mir()->isMax()) {
+ masm.Fmax(output, lhs, rhs);
+ } else {
+ masm.Fmin(output, lhs, rhs);
+ }
+}
+
+// FIXME: Uh, is this a static function? It looks like it is...
+template <typename T>
+ARMRegister toWRegister(const T* a) {
+ return ARMRegister(ToRegister(a), 32);
+}
+
+// FIXME: Uh, is this a static function? It looks like it is...
+template <typename T>
+ARMRegister toXRegister(const T* a) {
+ return ARMRegister(ToRegister(a), 64);
+}
+
+Operand toWOperand(const LAllocation* a) {
+ if (a->isConstant()) {
+ return Operand(ToInt32(a));
+ }
+ return Operand(toWRegister(a));
+}
+
+vixl::CPURegister ToCPURegister(const LAllocation* a, Scalar::Type type) {
+ if (a->isFloatReg() && type == Scalar::Float64) {
+ return ARMFPRegister(ToFloatRegister(a), 64);
+ }
+ if (a->isFloatReg() && type == Scalar::Float32) {
+ return ARMFPRegister(ToFloatRegister(a), 32);
+ }
+ if (a->isGeneralReg()) {
+ return ARMRegister(ToRegister(a), 32);
+ }
+ MOZ_CRASH("Unknown LAllocation");
+}
+
+vixl::CPURegister ToCPURegister(const LDefinition* d, Scalar::Type type) {
+ return ToCPURegister(d->output(), type);
+}
+
+// Let |cond| be an ARM64 condition code that we could reasonably use in a
+// conditional branch or select following a comparison instruction. This
+// function returns the condition to use in the case where we swap the two
+// operands of the comparison instruction.
+Assembler::Condition GetCondForSwappedOperands(Assembler::Condition cond) {
+ // EQ and NE map to themselves
+ // Of the remaining 14 cases, 4 other pairings can meaningfully swap:
+ // HS -- LS
+ // LO -- HI
+ // GE -- LE
+ // GT -- LT
+ switch (cond) {
+ case vixl::eq:
+ case vixl::ne:
+ return cond;
+ case vixl::hs:
+ return vixl::ls;
+ case vixl::ls:
+ return vixl::hs;
+ case vixl::lo:
+ return vixl::hi;
+ case vixl::hi:
+ return vixl::lo;
+ case vixl::ge:
+ return vixl::le;
+ case vixl::le:
+ return vixl::ge;
+ case vixl::gt:
+ return vixl::lt;
+ case vixl::lt:
+ return vixl::gt;
+ default:
+ MOZ_CRASH("no meaningful swapped-operand condition");
+ }
+}
+
+void CodeGenerator::visitAddI(LAddI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ // Platforms with three-operand arithmetic ops don't need recovery.
+ MOZ_ASSERT(!ins->recoversInput());
+
+ if (ins->snapshot()) {
+ masm.Adds(toWRegister(dest), toWRegister(lhs), toWOperand(rhs));
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ } else {
+ masm.Add(toWRegister(dest), toWRegister(lhs), toWOperand(rhs));
+ }
+}
+
+void CodeGenerator::visitSubI(LSubI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ // Platforms with three-operand arithmetic ops don't need recovery.
+ MOZ_ASSERT(!ins->recoversInput());
+
+ if (ins->snapshot()) {
+ masm.Subs(toWRegister(dest), toWRegister(lhs), toWOperand(rhs));
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ } else {
+ masm.Sub(toWRegister(dest), toWRegister(lhs), toWOperand(rhs));
+ }
+}
+
+void CodeGenerator::visitMulI(LMulI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+ MMul* mul = ins->mir();
+ MOZ_ASSERT_IF(mul->mode() == MMul::Integer,
+ !mul->canBeNegativeZero() && !mul->canOverflow());
+
+ Register lhsreg = ToRegister(lhs);
+ const ARMRegister lhsreg32 = ARMRegister(lhsreg, 32);
+ Register destreg = ToRegister(dest);
+ const ARMRegister destreg32 = ARMRegister(destreg, 32);
+
+ if (rhs->isConstant()) {
+ // Bailout on -0.0.
+ int32_t constant = ToInt32(rhs);
+ if (mul->canBeNegativeZero() && constant <= 0) {
+ Assembler::Condition bailoutCond =
+ (constant == 0) ? Assembler::LessThan : Assembler::Equal;
+ masm.Cmp(toWRegister(lhs), Operand(0));
+ bailoutIf(bailoutCond, ins->snapshot());
+ }
+
+ switch (constant) {
+ case -1:
+ masm.Negs(destreg32, Operand(lhsreg32));
+ break; // Go to overflow check.
+ case 0:
+ masm.Mov(destreg32, wzr);
+ return; // Avoid overflow check.
+ case 1:
+ if (destreg != lhsreg) {
+ masm.Mov(destreg32, lhsreg32);
+ }
+ return; // Avoid overflow check.
+ case 2:
+ masm.Adds(destreg32, lhsreg32, Operand(lhsreg32));
+ break; // Go to overflow check.
+ default:
+ // Use shift if cannot overflow and constant is a power of 2
+ if (!mul->canOverflow() && constant > 0) {
+ int32_t shift = FloorLog2(constant);
+ if ((1 << shift) == constant) {
+ masm.Lsl(destreg32, lhsreg32, shift);
+ return;
+ }
+ }
+
+ // Otherwise, just multiply. We have to check for overflow.
+ // Negative zero was handled above.
+ Label bailout;
+ Label* onOverflow = mul->canOverflow() ? &bailout : nullptr;
+
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const Register scratch = temps.AcquireW().asUnsized();
+
+ masm.move32(Imm32(constant), scratch);
+ masm.mul32(lhsreg, scratch, destreg, onOverflow);
+
+ if (onOverflow) {
+ MOZ_ASSERT(lhsreg != destreg);
+ bailoutFrom(&bailout, ins->snapshot());
+ }
+ return;
+ }
+
+ // Overflow check.
+ if (mul->canOverflow()) {
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ }
+ } else {
+ Register rhsreg = ToRegister(rhs);
+ const ARMRegister rhsreg32 = ARMRegister(rhsreg, 32);
+
+ Label bailout;
+ Label* onOverflow = mul->canOverflow() ? &bailout : nullptr;
+
+ if (mul->canBeNegativeZero()) {
+ // The product of two integer operands is negative zero iff one
+ // operand is zero, and the other is negative. Therefore, the
+ // sum of the two operands will also be negative (specifically,
+ // it will be the non-zero operand). If the result of the
+ // multiplication is 0, we can check the sign of the sum to
+ // determine whether we should bail out.
+
+ // This code can bailout, so lowering guarantees that the input
+ // operands are not overwritten.
+ MOZ_ASSERT(destreg != lhsreg);
+ MOZ_ASSERT(destreg != rhsreg);
+
+ // Do the multiplication.
+ masm.mul32(lhsreg, rhsreg, destreg, onOverflow);
+
+ // Set Zero flag if destreg is 0.
+ masm.test32(destreg, destreg);
+
+ // ccmn is 'conditional compare negative'.
+ // If the Zero flag is set:
+ // perform a compare negative (compute lhs+rhs and set flags)
+ // else:
+ // clear flags
+ masm.Ccmn(lhsreg32, rhsreg32, vixl::NoFlag, Assembler::Zero);
+
+ // Bails out if (lhs * rhs == 0) && (lhs + rhs < 0):
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+
+ } else {
+ masm.mul32(lhsreg, rhsreg, destreg, onOverflow);
+ }
+ if (onOverflow) {
+ bailoutFrom(&bailout, ins->snapshot());
+ }
+ }
+}
+
+void CodeGenerator::visitDivI(LDivI* ins) {
+ const Register lhs = ToRegister(ins->lhs());
+ const Register rhs = ToRegister(ins->rhs());
+ const Register output = ToRegister(ins->output());
+
+ const ARMRegister lhs32 = toWRegister(ins->lhs());
+ const ARMRegister rhs32 = toWRegister(ins->rhs());
+ const ARMRegister temp32 = toWRegister(ins->getTemp(0));
+ const ARMRegister output32 = toWRegister(ins->output());
+
+ MDiv* mir = ins->mir();
+
+ Label done;
+
+ // Handle division by zero.
+ if (mir->canBeDivideByZero()) {
+ masm.test32(rhs, rhs);
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.j(Assembler::NonZero, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else if (mir->canTruncateInfinities()) {
+ // Truncated division by zero is zero: (Infinity|0 = 0).
+ Label nonZero;
+ masm.j(Assembler::NonZero, &nonZero);
+ masm.Mov(output32, wzr);
+ masm.jump(&done);
+ masm.bind(&nonZero);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ }
+ }
+
+ // Handle an integer overflow from (INT32_MIN / -1).
+ // The integer division gives INT32_MIN, but should be -(double)INT32_MIN.
+ if (mir->canBeNegativeOverflow()) {
+ Label notOverflow;
+
+ // Branch to handle the non-overflow cases.
+ masm.branch32(Assembler::NotEqual, lhs, Imm32(INT32_MIN), &notOverflow);
+ masm.branch32(Assembler::NotEqual, rhs, Imm32(-1), &notOverflow);
+
+ // Handle overflow.
+ if (mir->trapOnError()) {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
+ } else if (mir->canTruncateOverflow()) {
+ // (-INT32_MIN)|0 == INT32_MIN, which is already in lhs.
+ masm.move32(lhs, output);
+ masm.jump(&done);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailout(ins->snapshot());
+ }
+ masm.bind(&notOverflow);
+ }
+
+ // Handle negative zero: lhs == 0 && rhs < 0.
+ if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
+ Label nonZero;
+ masm.branch32(Assembler::NotEqual, lhs, Imm32(0), &nonZero);
+ masm.cmp32(rhs, Imm32(0));
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+ masm.bind(&nonZero);
+ }
+
+ // Perform integer division.
+ if (mir->canTruncateRemainder()) {
+ masm.Sdiv(output32, lhs32, rhs32);
+ } else {
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ ARMRegister scratch32 = temps.AcquireW();
+
+ // ARM does not automatically calculate the remainder.
+ // The ISR suggests multiplication to determine whether a remainder exists.
+ masm.Sdiv(scratch32, lhs32, rhs32);
+ masm.Mul(temp32, scratch32, rhs32);
+ masm.Cmp(lhs32, temp32);
+ bailoutIf(Assembler::NotEqual, ins->snapshot());
+ masm.Mov(output32, scratch32);
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) {
+ const Register numerator = ToRegister(ins->numerator());
+ const ARMRegister numerator32 = toWRegister(ins->numerator());
+ const ARMRegister output32 = toWRegister(ins->output());
+
+ int32_t shift = ins->shift();
+ bool negativeDivisor = ins->negativeDivisor();
+ MDiv* mir = ins->mir();
+
+ if (!mir->isTruncated() && negativeDivisor) {
+ // 0 divided by a negative number returns a -0 double.
+ bailoutTest32(Assembler::Zero, numerator, numerator, ins->snapshot());
+ }
+
+ if (shift) {
+ if (!mir->isTruncated()) {
+ // If the remainder is != 0, bailout since this must be a double.
+ bailoutTest32(Assembler::NonZero, numerator,
+ Imm32(UINT32_MAX >> (32 - shift)), ins->snapshot());
+ }
+
+ if (mir->isUnsigned()) {
+ // shift right
+ masm.Lsr(output32, numerator32, shift);
+ } else {
+ ARMRegister temp32 = numerator32;
+ // Adjust the value so that shifting produces a correctly
+ // rounded result when the numerator is negative. See 10-1
+ // "Signed Division by a Known Power of 2" in Henry
+ // S. Warren, Jr.'s Hacker's Delight.
+ if (mir->canBeNegativeDividend() && mir->isTruncated()) {
+ if (shift > 1) {
+ // Copy the sign bit of the numerator. (= (2^32 - 1) or 0)
+ masm.Asr(output32, numerator32, 31);
+ temp32 = output32;
+ }
+ // Divide by 2^(32 - shift)
+ // i.e. (= (2^32 - 1) / 2^(32 - shift) or 0)
+ // i.e. (= (2^shift - 1) or 0)
+ masm.Lsr(output32, temp32, 32 - shift);
+ // If signed, make any 1 bit below the shifted bits to bubble up, such
+ // that once shifted the value would be rounded towards 0.
+ masm.Add(output32, output32, numerator32);
+ temp32 = output32;
+ }
+ masm.Asr(output32, temp32, shift);
+
+ if (negativeDivisor) {
+ masm.Neg(output32, output32);
+ }
+ }
+ return;
+ }
+
+ if (negativeDivisor) {
+ // INT32_MIN / -1 overflows.
+ if (!mir->isTruncated()) {
+ masm.Negs(output32, numerator32);
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ } else if (mir->trapOnError()) {
+ Label ok;
+ masm.Negs(output32, numerator32);
+ masm.branch(Assembler::NoOverflow, &ok);
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
+ masm.bind(&ok);
+ } else {
+ // Do not set condition flags.
+ masm.Neg(output32, numerator32);
+ }
+ } else {
+ if (mir->isUnsigned() && !mir->isTruncated()) {
+ // Copy and set flags.
+ masm.Adds(output32, numerator32, 0);
+ // Unsigned division by 1 can overflow if output is not truncated, as we
+ // do not have an Unsigned type for MIR instructions.
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ } else {
+ // Copy the result.
+ masm.Mov(output32, numerator32);
+ }
+ }
+}
+
+void CodeGenerator::visitDivConstantI(LDivConstantI* ins) {
+ const ARMRegister lhs32 = toWRegister(ins->numerator());
+ const ARMRegister lhs64 = toXRegister(ins->numerator());
+ const ARMRegister const32 = toWRegister(ins->temp());
+ const ARMRegister output32 = toWRegister(ins->output());
+ const ARMRegister output64 = toXRegister(ins->output());
+ int32_t d = ins->denominator();
+
+ // The absolute value of the denominator isn't a power of 2.
+ using mozilla::Abs;
+ MOZ_ASSERT((Abs(d) & (Abs(d) - 1)) != 0);
+
+ // We will first divide by Abs(d), and negate the answer if d is negative.
+ // If desired, this can be avoided by generalizing computeDivisionConstants.
+ auto rmc = ReciprocalMulConstants::computeSignedDivisionConstants(Abs(d));
+
+ // We first compute (M * n) >> 32, where M = rmc.multiplier.
+ masm.Mov(const32, int32_t(rmc.multiplier));
+ if (rmc.multiplier > INT32_MAX) {
+ MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 32));
+
+ // We actually compute (int32_t(M) * n) instead, without the upper bit.
+ // Thus, (M * n) = (int32_t(M) * n) + n << 32.
+ //
+ // ((int32_t(M) * n) + n << 32) can't overflow, as both operands have
+ // opposite signs because int32_t(M) is negative.
+ masm.Lsl(output64, lhs64, 32);
+
+ // Store (M * n) in output64.
+ masm.Smaddl(output64, const32, lhs32, output64);
+ } else {
+ // Store (M * n) in output64.
+ masm.Smull(output64, const32, lhs32);
+ }
+
+ // (M * n) >> (32 + shift) is the truncated division answer if n is
+ // non-negative, as proved in the comments of computeDivisionConstants. We
+ // must add 1 later if n is negative to get the right answer in all cases.
+ masm.Asr(output64, output64, 32 + rmc.shiftAmount);
+
+ // We'll subtract -1 instead of adding 1, because (n < 0 ? -1 : 0) can be
+ // computed with just a sign-extending shift of 31 bits.
+ if (ins->canBeNegativeDividend()) {
+ masm.Asr(const32, lhs32, 31);
+ masm.Sub(output32, output32, const32);
+ }
+
+ // After this, output32 contains the correct truncated division result.
+ if (d < 0) {
+ masm.Neg(output32, output32);
+ }
+
+ if (!ins->mir()->isTruncated()) {
+ // This is a division op. Multiply the obtained value by d to check if
+ // the correct answer is an integer. This cannot overflow, since |d| > 1.
+ masm.Mov(const32, d);
+ masm.Msub(const32, output32, const32, lhs32);
+ // bailout if (lhs - output * d != 0)
+ masm.Cmp(const32, wzr);
+ auto bailoutCond = Assembler::NonZero;
+
+ // If lhs is zero and the divisor is negative, the answer should have
+ // been -0.
+ if (d < 0) {
+ // or bailout if (lhs == 0).
+ // ^ ^
+ // | '-- masm.Ccmp(lhs32, lhs32, .., ..)
+ // '-- masm.Ccmp(.., .., vixl::ZFlag, ! bailoutCond)
+ masm.Ccmp(lhs32, wzr, vixl::ZFlag, Assembler::Zero);
+ bailoutCond = Assembler::Zero;
+ }
+
+ // bailout if (lhs - output * d != 0) or (d < 0 && lhs == 0)
+ bailoutIf(bailoutCond, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitUDivConstantI(LUDivConstantI* ins) {
+ const ARMRegister lhs32 = toWRegister(ins->numerator());
+ const ARMRegister lhs64 = toXRegister(ins->numerator());
+ const ARMRegister const32 = toWRegister(ins->temp());
+ const ARMRegister output32 = toWRegister(ins->output());
+ const ARMRegister output64 = toXRegister(ins->output());
+ uint32_t d = ins->denominator();
+
+ if (d == 0) {
+ if (ins->mir()->isTruncated()) {
+ if (ins->mir()->trapOnError()) {
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero,
+ ins->mir()->bytecodeOffset());
+ } else {
+ masm.Mov(output32, wzr);
+ }
+ } else {
+ bailout(ins->snapshot());
+ }
+ return;
+ }
+
+ // The denominator isn't a power of 2 (see LDivPowTwoI).
+ MOZ_ASSERT((d & (d - 1)) != 0);
+
+ auto rmc = ReciprocalMulConstants::computeUnsignedDivisionConstants(d);
+
+ // We first compute (M * n), where M = rmc.multiplier.
+ masm.Mov(const32, int32_t(rmc.multiplier));
+ masm.Umull(output64, const32, lhs32);
+ if (rmc.multiplier > UINT32_MAX) {
+ // M >= 2^32 and shift == 0 is impossible, as d >= 2 implies that
+ // ((M * n) >> (32 + shift)) >= n > floor(n/d) whenever n >= d,
+ // contradicting the proof of correctness in computeDivisionConstants.
+ MOZ_ASSERT(rmc.shiftAmount > 0);
+ MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 33));
+
+ // We actually compute (uint32_t(M) * n) instead, without the upper bit.
+ // Thus, (M * n) = (uint32_t(M) * n) + n << 32.
+ //
+ // ((uint32_t(M) * n) + n << 32) can overflow. Hacker's Delight explains a
+ // trick to avoid this overflow case, but we can avoid it by computing the
+ // addition on 64 bits registers.
+ //
+ // Compute ((uint32_t(M) * n) >> 32 + n)
+ masm.Add(output64, lhs64, Operand(output64, vixl::LSR, 32));
+
+ // (M * n) >> (32 + shift) is the truncated division answer.
+ masm.Lsr(output64, output64, rmc.shiftAmount);
+ } else {
+ // (M * n) >> (32 + shift) is the truncated division answer.
+ masm.Lsr(output64, output64, 32 + rmc.shiftAmount);
+ }
+
+ // We now have the truncated division value. We are checking whether the
+ // division resulted in an integer, we multiply the obtained value by d and
+ // check the remainder of the division.
+ if (!ins->mir()->isTruncated()) {
+ masm.Mov(const32, d);
+ masm.Msub(const32, output32, const32, lhs32);
+ // bailout if (lhs - output * d != 0)
+ masm.Cmp(const32, const32);
+ bailoutIf(Assembler::NonZero, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitModI(LModI* ins) {
+ ARMRegister lhs = toWRegister(ins->lhs());
+ ARMRegister rhs = toWRegister(ins->rhs());
+ ARMRegister output = toWRegister(ins->output());
+ Label done;
+
+ MMod* mir = ins->mir();
+
+ // Prevent divide by zero.
+ if (mir->canBeDivideByZero()) {
+ if (mir->isTruncated()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.Cbnz(rhs, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ // Truncated division by zero yields integer zero.
+ masm.Mov(output, rhs);
+ masm.Cbz(rhs, &done);
+ }
+ } else {
+ // Non-truncated division by zero produces a non-integer.
+ MOZ_ASSERT(!gen->compilingWasm());
+ masm.Cmp(rhs, Operand(0));
+ bailoutIf(Assembler::Equal, ins->snapshot());
+ }
+ }
+
+ // Signed division.
+ masm.Sdiv(output, lhs, rhs);
+
+ // Compute the remainder: output = lhs - (output * rhs).
+ masm.Msub(output, output, rhs, lhs);
+
+ if (mir->canBeNegativeDividend() && !mir->isTruncated()) {
+ // If output == 0 and lhs < 0, then the result should be double -0.0.
+ // Note that this guard handles lhs == INT_MIN and rhs == -1:
+ // output = INT_MIN - (INT_MIN / -1) * -1
+ // = INT_MIN - INT_MIN
+ // = 0
+ masm.Cbnz(output, &done);
+ bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot());
+ }
+
+ if (done.used()) {
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) {
+ Register lhs = ToRegister(ins->getOperand(0));
+ ARMRegister lhsw = toWRegister(ins->getOperand(0));
+ ARMRegister outw = toWRegister(ins->output());
+
+ int32_t shift = ins->shift();
+ bool canBeNegative =
+ !ins->mir()->isUnsigned() && ins->mir()->canBeNegativeDividend();
+
+ Label negative;
+ if (canBeNegative) {
+ // Switch based on sign of the lhs.
+ // Positive numbers are just a bitmask.
+ masm.branchTest32(Assembler::Signed, lhs, lhs, &negative);
+ }
+
+ masm.And(outw, lhsw, Operand((uint32_t(1) << shift) - 1));
+
+ if (canBeNegative) {
+ Label done;
+ masm.jump(&done);
+
+ // Negative numbers need a negate, bitmask, negate.
+ masm.bind(&negative);
+ masm.Neg(outw, Operand(lhsw));
+ masm.And(outw, outw, Operand((uint32_t(1) << shift) - 1));
+
+ // Since a%b has the same sign as b, and a is negative in this branch,
+ // an answer of 0 means the correct result is actually -0. Bail out.
+ if (!ins->mir()->isTruncated()) {
+ masm.Negs(outw, Operand(outw));
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ } else {
+ masm.Neg(outw, Operand(outw));
+ }
+
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitModMaskI(LModMaskI* ins) {
+ MMod* mir = ins->mir();
+ int32_t shift = ins->shift();
+
+ const Register src = ToRegister(ins->getOperand(0));
+ const Register dest = ToRegister(ins->getDef(0));
+ const Register hold = ToRegister(ins->getTemp(0));
+ const Register remain = ToRegister(ins->getTemp(1));
+
+ const ARMRegister src32 = ARMRegister(src, 32);
+ const ARMRegister dest32 = ARMRegister(dest, 32);
+ const ARMRegister remain32 = ARMRegister(remain, 32);
+
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMRegister scratch32 = temps.AcquireW();
+ const Register scratch = scratch32.asUnsized();
+
+ // We wish to compute x % (1<<y) - 1 for a known constant, y.
+ //
+ // 1. Let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit dividend as
+ // a number in base b, namely c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n
+ //
+ // 2. Since both addition and multiplication commute with modulus:
+ // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C ==
+ // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)...
+ //
+ // 3. Since b == C + 1, b % C == 1, and b^n % C == 1 the whole thing
+ // simplifies to: c_0 + c_1 + c_2 ... c_n % C
+ //
+ // Each c_n can easily be computed by a shift/bitextract, and the modulus
+ // can be maintained by simply subtracting by C whenever the number gets
+ // over C.
+ int32_t mask = (1 << shift) - 1;
+ Label loop;
+
+ // Register 'hold' holds -1 if the value was negative, 1 otherwise.
+ // The remain reg holds the remaining bits that have not been processed.
+ // The scratch reg serves as a temporary location to store extracted bits.
+ // The dest reg is the accumulator, becoming final result.
+ //
+ // Move the whole value into the remain.
+ masm.Mov(remain32, src32);
+ // Zero out the dest.
+ masm.Mov(dest32, wzr);
+ // Set the hold appropriately.
+ {
+ Label negative;
+ masm.branch32(Assembler::Signed, remain, Imm32(0), &negative);
+ masm.move32(Imm32(1), hold);
+ masm.jump(&loop);
+
+ masm.bind(&negative);
+ masm.move32(Imm32(-1), hold);
+ masm.neg32(remain);
+ }
+
+ // Begin the main loop.
+ masm.bind(&loop);
+ {
+ // Extract the bottom bits into scratch.
+ masm.And(scratch32, remain32, Operand(mask));
+ // Add those bits to the accumulator.
+ masm.Add(dest32, dest32, scratch32);
+ // Do a trial subtraction. This functions as a cmp but remembers the result.
+ masm.Subs(scratch32, dest32, Operand(mask));
+ // If (sum - C) > 0, store sum - C back into sum, thus performing a modulus.
+ {
+ Label sumSigned;
+ masm.branch32(Assembler::Signed, scratch, scratch, &sumSigned);
+ masm.Mov(dest32, scratch32);
+ masm.bind(&sumSigned);
+ }
+ // Get rid of the bits that we extracted before.
+ masm.Lsr(remain32, remain32, shift);
+ // If the shift produced zero, finish, otherwise, continue in the loop.
+ masm.branchTest32(Assembler::NonZero, remain, remain, &loop);
+ }
+
+ // Check the hold to see if we need to negate the result.
+ {
+ Label done;
+
+ // If the hold was non-zero, negate the result to match JS expectations.
+ masm.branchTest32(Assembler::NotSigned, hold, hold, &done);
+ if (mir->canBeNegativeDividend() && !mir->isTruncated()) {
+ // Bail in case of negative zero hold.
+ bailoutTest32(Assembler::Zero, hold, hold, ins->snapshot());
+ }
+
+ masm.neg32(dest);
+ masm.bind(&done);
+ }
+}
+
+void CodeGeneratorARM64::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ const ARMRegister dividend64(dividend, 64);
+ const ARMRegister divisor64(divisor, 64);
+
+ masm.Sdiv(/* result= */ dividend64, dividend64, divisor64);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorARM64::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ const ARMRegister dividend64(dividend, 64);
+ const ARMRegister divisor64(divisor, 64);
+ const ARMRegister output64(output, 64);
+
+ // Signed division.
+ masm.Sdiv(output64, dividend64, divisor64);
+
+ // Compute the remainder: output = dividend - (output * divisor).
+ masm.Msub(/* result= */ dividend64, output64, divisor64, dividend64);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGenerator::visitBitNotI(LBitNotI* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ const LDefinition* output = ins->getDef(0);
+ masm.Mvn(toWRegister(output), toWOperand(input));
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+ masm.Mvn(vixl::Register(output, 64), vixl::Register(input, 64));
+}
+
+void CodeGenerator::visitBitOpI(LBitOpI* ins) {
+ const ARMRegister lhs = toWRegister(ins->getOperand(0));
+ const Operand rhs = toWOperand(ins->getOperand(1));
+ const ARMRegister dest = toWRegister(ins->getDef(0));
+
+ switch (ins->bitop()) {
+ case JSOp::BitOr:
+ masm.Orr(dest, lhs, rhs);
+ break;
+ case JSOp::BitXor:
+ masm.Eor(dest, lhs, rhs);
+ break;
+ case JSOp::BitAnd:
+ masm.And(dest, lhs, rhs);
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitShiftI(LShiftI* ins) {
+ const ARMRegister lhs = toWRegister(ins->lhs());
+ const LAllocation* rhs = ins->rhs();
+ const ARMRegister dest = toWRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ masm.Lsl(dest, lhs, shift);
+ break;
+ case JSOp::Rsh:
+ masm.Asr(dest, lhs, shift);
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.Lsr(dest, lhs, shift);
+ } else if (ins->mir()->toUrsh()->fallible()) {
+ // x >>> 0 can overflow.
+ masm.Ands(dest, lhs, Operand(0xFFFFFFFF));
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ } else {
+ masm.Mov(dest, lhs);
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ } else {
+ const ARMRegister rhsreg = toWRegister(rhs);
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ masm.Lsl(dest, lhs, rhsreg);
+ break;
+ case JSOp::Rsh:
+ masm.Asr(dest, lhs, rhsreg);
+ break;
+ case JSOp::Ursh:
+ masm.Lsr(dest, lhs, rhsreg);
+ if (ins->mir()->toUrsh()->fallible()) {
+ /// x >>> 0 can overflow.
+ masm.Cmp(dest, Operand(0));
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ }
+}
+
+void CodeGenerator::visitUrshD(LUrshD* ins) {
+ const ARMRegister lhs = toWRegister(ins->lhs());
+ const LAllocation* rhs = ins->rhs();
+ const FloatRegister out = ToFloatRegister(ins->output());
+
+ const Register temp = ToRegister(ins->temp());
+ const ARMRegister temp32 = toWRegister(ins->temp());
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ if (shift) {
+ masm.Lsr(temp32, lhs, shift);
+ masm.convertUInt32ToDouble(temp, out);
+ } else {
+ masm.convertUInt32ToDouble(ToRegister(ins->lhs()), out);
+ }
+ } else {
+ masm.And(temp32, toWRegister(rhs), Operand(0x1F));
+ masm.Lsr(temp32, lhs, temp32);
+ masm.convertUInt32ToDouble(temp, out);
+ }
+}
+
+void CodeGenerator::visitPowHalfD(LPowHalfD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ ScratchDoubleScope scratch(masm);
+
+ Label done, sqrt;
+
+ if (!ins->mir()->operandIsNeverNegativeInfinity()) {
+ // Branch if not -Infinity.
+ masm.loadConstantDouble(NegativeInfinity<double>(), scratch);
+
+ Assembler::DoubleCondition cond = Assembler::DoubleNotEqualOrUnordered;
+ if (ins->mir()->operandIsNeverNaN()) {
+ cond = Assembler::DoubleNotEqual;
+ }
+ masm.branchDouble(cond, input, scratch, &sqrt);
+
+ // Math.pow(-Infinity, 0.5) == Infinity.
+ masm.zeroDouble(output);
+ masm.subDouble(scratch, output);
+ masm.jump(&done);
+
+ masm.bind(&sqrt);
+ }
+
+ if (!ins->mir()->operandIsNeverNegativeZero()) {
+ // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
+ // Adding 0 converts any -0 to 0.
+ masm.zeroDouble(scratch);
+ masm.addDouble(input, scratch);
+ masm.sqrtDouble(scratch, output);
+ } else {
+ masm.sqrtDouble(input, output);
+ }
+
+ masm.bind(&done);
+}
+
+MoveOperand CodeGeneratorARM64::toMoveOperand(const LAllocation a) const {
+ if (a.isGeneralReg()) {
+ return MoveOperand(ToRegister(a));
+ }
+ if (a.isFloatReg()) {
+ return MoveOperand(ToFloatRegister(a));
+ }
+ MoveOperand::Kind kind =
+ a.isStackArea() ? MoveOperand::EFFECTIVE_ADDRESS : MoveOperand::MEMORY;
+ return MoveOperand(ToAddress(a), kind);
+}
+
+class js::jit::OutOfLineTableSwitch
+ : public OutOfLineCodeBase<CodeGeneratorARM64> {
+ MTableSwitch* mir_;
+ CodeLabel jumpLabel_;
+
+ void accept(CodeGeneratorARM64* codegen) override {
+ codegen->visitOutOfLineTableSwitch(this);
+ }
+
+ public:
+ explicit OutOfLineTableSwitch(MTableSwitch* mir) : mir_(mir) {}
+
+ MTableSwitch* mir() const { return mir_; }
+
+ CodeLabel* jumpLabel() { return &jumpLabel_; }
+};
+
+void CodeGeneratorARM64::visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool) {
+ MTableSwitch* mir = ool->mir();
+
+ // Prevent nop and pools sequences to appear in the jump table.
+ AutoForbidPoolsAndNops afp(
+ &masm, (mir->numCases() + 1) * (sizeof(void*) / vixl::kInstructionSize));
+ masm.haltingAlign(sizeof(void*));
+ masm.bind(ool->jumpLabel());
+ masm.addCodeLabel(*ool->jumpLabel());
+
+ for (size_t i = 0; i < mir->numCases(); i++) {
+ LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir();
+ Label* caseheader = caseblock->label();
+ uint32_t caseoffset = caseheader->offset();
+
+ // The entries of the jump table need to be absolute addresses,
+ // and thus must be patched after codegen is finished.
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ cl.target()->bind(caseoffset);
+ masm.addCodeLabel(cl);
+ }
+}
+
+void CodeGeneratorARM64::emitTableSwitchDispatch(MTableSwitch* mir,
+ Register index,
+ Register base) {
+ Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
+
+ // Let the lowest table entry be indexed at 0.
+ if (mir->low() != 0) {
+ masm.sub32(Imm32(mir->low()), index);
+ }
+
+ // Jump to the default case if input is out of range.
+ int32_t cases = mir->numCases();
+ masm.branch32(Assembler::AboveOrEqual, index, Imm32(cases), defaultcase);
+
+ // Because the target code has not yet been generated, we cannot know the
+ // instruction offsets for use as jump targets. Therefore we construct
+ // an OutOfLineTableSwitch that winds up holding the jump table.
+ //
+ // Because the jump table is generated as part of out-of-line code,
+ // it is generated after all the regular codegen, so the jump targets
+ // are guaranteed to exist when generating the jump table.
+ OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(mir);
+ addOutOfLineCode(ool, mir);
+
+ // Use the index to get the address of the jump target from the table.
+ masm.mov(ool->jumpLabel(), base);
+ BaseIndex pointer(base, index, ScalePointer);
+
+ // Load the target from the jump table and branch to it.
+ masm.branchToComputedAddress(pointer);
+}
+
+void CodeGenerator::visitMathD(LMathD* math) {
+ ARMFPRegister lhs(ToFloatRegister(math->lhs()), 64);
+ ARMFPRegister rhs(ToFloatRegister(math->rhs()), 64);
+ ARMFPRegister output(ToFloatRegister(math->output()), 64);
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.Fadd(output, lhs, rhs);
+ break;
+ case JSOp::Sub:
+ masm.Fsub(output, lhs, rhs);
+ break;
+ case JSOp::Mul:
+ masm.Fmul(output, lhs, rhs);
+ break;
+ case JSOp::Div:
+ masm.Fdiv(output, lhs, rhs);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitMathF(LMathF* math) {
+ ARMFPRegister lhs(ToFloatRegister(math->lhs()), 32);
+ ARMFPRegister rhs(ToFloatRegister(math->rhs()), 32);
+ ARMFPRegister output(ToFloatRegister(math->output()), 32);
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.Fadd(output, lhs, rhs);
+ break;
+ case JSOp::Sub:
+ masm.Fsub(output, lhs, rhs);
+ break;
+ case JSOp::Mul:
+ masm.Fmul(output, lhs, rhs);
+ break;
+ case JSOp::Div:
+ masm.Fdiv(output, lhs, rhs);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitClzI(LClzI* lir) {
+ ARMRegister input = toWRegister(lir->input());
+ ARMRegister output = toWRegister(lir->output());
+ masm.Clz(output, input);
+}
+
+void CodeGenerator::visitCtzI(LCtzI* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ masm.ctz32(input, output, /* knownNotZero = */ false);
+}
+
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+ emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void CodeGenerator::visitNearbyInt(LNearbyInt* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ RoundingMode roundingMode = lir->mir()->roundingMode();
+ masm.nearbyIntDouble(roundingMode, input, output);
+}
+
+void CodeGenerator::visitNearbyIntF(LNearbyIntF* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ RoundingMode roundingMode = lir->mir()->roundingMode();
+ masm.nearbyIntFloat32(roundingMode, input, output);
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* lir) {
+ emitTruncateDouble(ToFloatRegister(lir->getOperand(0)),
+ ToRegister(lir->getDef(0)), lir->mir());
+}
+
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+ emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+ LWasmBuiltinTruncateFToInt32* lir) {
+ emitTruncateFloat32(ToFloatRegister(lir->getOperand(0)),
+ ToRegister(lir->getDef(0)), lir->mir());
+}
+
+ValueOperand CodeGeneratorARM64::ToValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getOperand(pos)));
+}
+
+ValueOperand CodeGeneratorARM64::ToTempValue(LInstruction* ins, size_t pos) {
+ MOZ_CRASH("CodeGeneratorARM64::ToTempValue");
+}
+
+void CodeGenerator::visitValue(LValue* value) {
+ ValueOperand result = ToOutValue(value);
+ masm.moveValue(value->value(), result);
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+ const LAllocation* in = box->getOperand(0);
+ ValueOperand result = ToOutValue(box);
+
+ masm.moveValue(TypedOrValueRegister(box->type(), ToAnyRegister(in)), result);
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+ MUnbox* mir = unbox->mir();
+
+ Register result = ToRegister(unbox->output());
+
+ if (mir->fallible()) {
+ const ValueOperand value = ToValue(unbox, LUnbox::Input);
+ Label bail;
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.fallibleUnboxInt32(value, result, &bail);
+ break;
+ case MIRType::Boolean:
+ masm.fallibleUnboxBoolean(value, result, &bail);
+ break;
+ case MIRType::Object:
+ masm.fallibleUnboxObject(value, result, &bail);
+ break;
+ case MIRType::String:
+ masm.fallibleUnboxString(value, result, &bail);
+ break;
+ case MIRType::Symbol:
+ masm.fallibleUnboxSymbol(value, result, &bail);
+ break;
+ case MIRType::BigInt:
+ masm.fallibleUnboxBigInt(value, result, &bail);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ bailoutFrom(&bail, unbox->snapshot());
+ return;
+ }
+
+ // Infallible unbox.
+
+ ValueOperand input = ToValue(unbox, LUnbox::Input);
+
+#ifdef DEBUG
+ // Assert the types match.
+ JSValueTag tag = MIRTypeToTag(mir->type());
+ Label ok;
+ {
+ ScratchTagScope scratch(masm, input);
+ masm.splitTagForTest(input, scratch);
+ masm.cmpTag(scratch, ImmTag(tag));
+ }
+ masm.B(&ok, Assembler::Condition::Equal);
+ masm.assumeUnreachable("Infallible unbox type mismatch");
+ masm.bind(&ok);
+#endif
+
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(input, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(input, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(input, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(input, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(input, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(input, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+}
+
+void CodeGenerator::visitDouble(LDouble* ins) {
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantDouble(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitFloat32(LFloat32* ins) {
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantFloat32(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) {
+ const LAllocation* opd = test->input();
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ masm.Fcmp(ARMFPRegister(ToFloatRegister(opd), 64), 0.0);
+
+ // If the compare set the 0 bit, then the result is definitely false.
+ jumpToBlock(ifFalse, Assembler::Zero);
+
+ // Overflow means one of the operands was NaN, which is also false.
+ jumpToBlock(ifFalse, Assembler::Overflow);
+ jumpToBlock(ifTrue);
+}
+
+void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) {
+ const LAllocation* opd = test->input();
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ masm.Fcmp(ARMFPRegister(ToFloatRegister(opd), 32), 0.0);
+
+ // If the compare set the 0 bit, then the result is definitely false.
+ jumpToBlock(ifFalse, Assembler::Zero);
+
+ // Overflow means one of the operands was NaN, which is also false.
+ jumpToBlock(ifFalse, Assembler::Overflow);
+ jumpToBlock(ifTrue);
+}
+
+void CodeGenerator::visitCompareD(LCompareD* comp) {
+ const FloatRegister left = ToFloatRegister(comp->left());
+ const FloatRegister right = ToFloatRegister(comp->right());
+ ARMRegister output = toWRegister(comp->output());
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+
+ masm.compareDouble(cond, left, right);
+ masm.cset(output, Assembler::ConditionFromDoubleCondition(cond));
+}
+
+void CodeGenerator::visitCompareF(LCompareF* comp) {
+ const FloatRegister left = ToFloatRegister(comp->left());
+ const FloatRegister right = ToFloatRegister(comp->right());
+ ARMRegister output = toWRegister(comp->output());
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+
+ masm.compareFloat(cond, left, right);
+ masm.cset(output, Assembler::ConditionFromDoubleCondition(cond));
+}
+
+void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
+ const FloatRegister left = ToFloatRegister(comp->left());
+ const FloatRegister right = ToFloatRegister(comp->right());
+ Assembler::DoubleCondition doubleCond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ Assembler::Condition cond =
+ Assembler::ConditionFromDoubleCondition(doubleCond);
+
+ masm.compareDouble(doubleCond, left, right);
+ emitBranch(cond, comp->ifTrue(), comp->ifFalse());
+}
+
+void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
+ const FloatRegister left = ToFloatRegister(comp->left());
+ const FloatRegister right = ToFloatRegister(comp->right());
+ Assembler::DoubleCondition doubleCond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ Assembler::Condition cond =
+ Assembler::ConditionFromDoubleCondition(doubleCond);
+
+ masm.compareFloat(doubleCond, left, right);
+ emitBranch(cond, comp->ifTrue(), comp->ifFalse());
+}
+
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
+ if (baab->is64()) {
+ ARMRegister regL = toXRegister(baab->left());
+ if (baab->right()->isConstant()) {
+ masm.Tst(regL, Operand(ToInt64(baab->right())));
+ } else {
+ masm.Tst(regL, toXRegister(baab->right()));
+ }
+ } else {
+ ARMRegister regL = toWRegister(baab->left());
+ if (baab->right()->isConstant()) {
+ masm.Tst(regL, Operand(ToInt32(baab->right())));
+ } else {
+ masm.Tst(regL, toWRegister(baab->right()));
+ }
+ }
+ emitBranch(baab->cond(), baab->ifTrue(), baab->ifFalse());
+}
+
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+ masm.convertUInt32ToDouble(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+ masm.convertUInt32ToFloat32(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitNotI(LNotI* ins) {
+ ARMRegister input = toWRegister(ins->input());
+ ARMRegister output = toWRegister(ins->output());
+
+ masm.Cmp(input, ZeroRegister32);
+ masm.Cset(output, Assembler::Zero);
+}
+
+// NZCV
+// NAN -> 0011
+// == -> 0110
+// < -> 1000
+// > -> 0010
+void CodeGenerator::visitNotD(LNotD* ins) {
+ ARMFPRegister input(ToFloatRegister(ins->input()), 64);
+ ARMRegister output = toWRegister(ins->output());
+
+ // Set output to 1 if input compares equal to 0.0, else 0.
+ masm.Fcmp(input, 0.0);
+ masm.Cset(output, Assembler::Equal);
+
+ // Comparison with NaN sets V in the NZCV register.
+ // If the input was NaN, output must now be zero, so it can be incremented.
+ // The instruction is read: "output = if NoOverflow then output else 0+1".
+ masm.Csinc(output, output, ZeroRegister32, Assembler::NoOverflow);
+}
+
+void CodeGenerator::visitNotF(LNotF* ins) {
+ ARMFPRegister input(ToFloatRegister(ins->input()), 32);
+ ARMRegister output = toWRegister(ins->output());
+
+ // Set output to 1 input compares equal to 0.0, else 0.
+ masm.Fcmp(input, 0.0);
+ masm.Cset(output, Assembler::Equal);
+
+ // Comparison with NaN sets V in the NZCV register.
+ // If the input was NaN, output must now be zero, so it can be incremented.
+ // The instruction is read: "output = if NoOverflow then output else 0+1".
+ masm.Csinc(output, output, ZeroRegister32, Assembler::NoOverflow);
+}
+
+void CodeGeneratorARM64::generateInvalidateEpilogue() {
+ // Ensure that there is enough space in the buffer for the OsiPoint patching
+ // to occur. Otherwise, we could overwrite the invalidation epilogue.
+ for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) {
+ masm.nop();
+ }
+
+ masm.bind(&invalidate_);
+
+ // Push the return address of the point that we bailout out onto the stack.
+ masm.push(lr);
+
+ // Push the Ion script onto the stack (when we determine what that pointer
+ // is).
+ invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
+
+ // Jump to the invalidator which will replace the current frame.
+ TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
+ masm.jump(thunk);
+}
+
+template <class U>
+Register getBase(U* mir) {
+ switch (mir->base()) {
+ case U::Heap:
+ return HeapReg;
+ }
+ return InvalidReg;
+}
+
+void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) {
+ const MAsmJSLoadHeap* mir = ins->mir();
+ MOZ_ASSERT(!mir->hasMemoryBase());
+
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ Register ptrReg = ToRegister(ptr);
+ Scalar::Type accessType = mir->accessType();
+ bool isFloat = accessType == Scalar::Float32 || accessType == Scalar::Float64;
+ Label done;
+
+ if (mir->needsBoundsCheck()) {
+ Label boundsCheckPassed;
+ Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
+ masm.wasmBoundsCheck32(Assembler::Below, ptrReg, boundsCheckLimitReg,
+ &boundsCheckPassed);
+ // Return a default value in case of a bounds-check failure.
+ if (isFloat) {
+ if (accessType == Scalar::Float32) {
+ masm.loadConstantFloat32(GenericNaN(), ToFloatRegister(ins->output()));
+ } else {
+ masm.loadConstantDouble(GenericNaN(), ToFloatRegister(ins->output()));
+ }
+ } else {
+ masm.Mov(ARMRegister(ToRegister(ins->output()), 64), 0);
+ }
+ masm.jump(&done);
+ masm.bind(&boundsCheckPassed);
+ }
+
+ MemOperand addr(ARMRegister(HeapReg, 64), ARMRegister(ptrReg, 64));
+ switch (accessType) {
+ case Scalar::Int8:
+ masm.Ldrb(toWRegister(ins->output()), addr);
+ masm.Sxtb(toWRegister(ins->output()), toWRegister(ins->output()));
+ break;
+ case Scalar::Uint8:
+ masm.Ldrb(toWRegister(ins->output()), addr);
+ break;
+ case Scalar::Int16:
+ masm.Ldrh(toWRegister(ins->output()), addr);
+ masm.Sxth(toWRegister(ins->output()), toWRegister(ins->output()));
+ break;
+ case Scalar::Uint16:
+ masm.Ldrh(toWRegister(ins->output()), addr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ masm.Ldr(toWRegister(ins->output()), addr);
+ break;
+ case Scalar::Float64:
+ masm.Ldr(ARMFPRegister(ToFloatRegister(ins->output()), 64), addr);
+ break;
+ case Scalar::Float32:
+ masm.Ldr(ARMFPRegister(ToFloatRegister(ins->output()), 32), addr);
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+ if (done.used()) {
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) {
+ const MAsmJSStoreHeap* mir = ins->mir();
+ MOZ_ASSERT(!mir->hasMemoryBase());
+
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ Register ptrReg = ToRegister(ptr);
+
+ Label done;
+ if (mir->needsBoundsCheck()) {
+ Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg, boundsCheckLimitReg,
+ &done);
+ }
+
+ MemOperand addr(ARMRegister(HeapReg, 64), ARMRegister(ptrReg, 64));
+ switch (mir->accessType()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ masm.Strb(toWRegister(ins->value()), addr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ masm.Strh(toWRegister(ins->value()), addr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ masm.Str(toWRegister(ins->value()), addr);
+ break;
+ case Scalar::Float64:
+ masm.Str(ARMFPRegister(ToFloatRegister(ins->value()), 64), addr);
+ break;
+ case Scalar::Float32:
+ masm.Str(ARMFPRegister(ToFloatRegister(ins->value()), 32), addr);
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+ if (done.used()) {
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitWasmCompareExchangeHeap(
+ LWasmCompareExchangeHeap* ins) {
+ MWasmCompareExchangeHeap* mir = ins->mir();
+
+ Register ptr = ToRegister(ins->ptr());
+ Register oldval = ToRegister(ins->oldValue());
+ Register newval = ToRegister(ins->newValue());
+ Register out = ToRegister(ins->output());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (mir->access().type() == Scalar::Int64) {
+ masm.wasmCompareExchange64(mir->access(), srcAddr, Register64(oldval),
+ Register64(newval), Register64(out));
+ } else {
+ masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval, out);
+ }
+}
+
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+ MWasmAtomicExchangeHeap* mir = ins->mir();
+
+ Register ptr = ToRegister(ins->ptr());
+ Register oldval = ToRegister(ins->value());
+ Register out = ToRegister(ins->output());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (mir->access().type() == Scalar::Int64) {
+ masm.wasmAtomicExchange64(mir->access(), srcAddr, Register64(oldval),
+ Register64(out));
+ } else {
+ masm.wasmAtomicExchange(mir->access(), srcAddr, oldval, out);
+ }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+
+ MOZ_ASSERT(mir->hasUses());
+
+ Register ptr = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ Register flagTemp = ToRegister(ins->flagTemp());
+ Register out = ToRegister(ins->output());
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+ AtomicOp op = mir->operation();
+
+ if (mir->access().type() == Scalar::Int64) {
+ masm.wasmAtomicFetchOp64(mir->access(), op, Register64(value), srcAddr,
+ Register64(flagTemp), Register64(out));
+ } else {
+ masm.wasmAtomicFetchOp(mir->access(), op, value, srcAddr, flagTemp, out);
+ }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+ LWasmAtomicBinopHeapForEffect* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+
+ MOZ_ASSERT(!mir->hasUses());
+
+ Register ptr = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ Register flagTemp = ToRegister(ins->flagTemp());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+ AtomicOp op = mir->operation();
+
+ if (mir->access().type() == Scalar::Int64) {
+ masm.wasmAtomicEffectOp64(mir->access(), op, Register64(value), srcAddr,
+ Register64(flagTemp));
+ } else {
+ masm.wasmAtomicEffectOp(mir->access(), op, value, srcAddr, flagTemp);
+ }
+}
+
+void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(masm.getStackPointer(), mir->spOffset());
+ if (ins->arg()->isConstant()) {
+ masm.storePtr(ImmWord(ToInt32(ins->arg())), dst);
+ } else if (ins->arg()->isGeneralReg()) {
+ masm.storePtr(ToRegister(ins->arg()), dst);
+ } else {
+ switch (mir->input()->type()) {
+ case MIRType::Double:
+ masm.storeDouble(ToFloatRegister(ins->arg()), dst);
+ return;
+ case MIRType::Float32:
+ masm.storeFloat32(ToFloatRegister(ins->arg()), dst);
+ return;
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ masm.storeUnalignedSimd128(ToFloatRegister(ins->arg()), dst);
+ return;
+#endif
+ default:
+ break;
+ }
+ MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE(
+ "unexpected mir type in WasmStackArg");
+ }
+}
+
+void CodeGenerator::visitUDiv(LUDiv* ins) {
+ MDiv* mir = ins->mir();
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+ ARMRegister lhs32 = ARMRegister(lhs, 32);
+ ARMRegister rhs32 = ARMRegister(rhs, 32);
+ ARMRegister output32 = ARMRegister(output, 32);
+
+ // Prevent divide by zero.
+ if (mir->canBeDivideByZero()) {
+ if (mir->isTruncated()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.branchTest32(Assembler::NonZero, rhs, rhs, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ // ARM64 UDIV instruction will return 0 when divided by 0.
+ // No need for extra tests.
+ }
+ } else {
+ bailoutTest32(Assembler::Zero, rhs, rhs, ins->snapshot());
+ }
+ }
+
+ // Unsigned division.
+ masm.Udiv(output32, lhs32, rhs32);
+
+ // If the remainder is > 0, bailout since this must be a double.
+ if (!mir->canTruncateRemainder()) {
+ Register remainder = ToRegister(ins->remainder());
+ ARMRegister remainder32 = ARMRegister(remainder, 32);
+
+ // Compute the remainder: remainder = lhs - (output * rhs).
+ masm.Msub(remainder32, output32, rhs32, lhs32);
+
+ bailoutTest32(Assembler::NonZero, remainder, remainder, ins->snapshot());
+ }
+
+ // Unsigned div can return a value that's not a signed int32.
+ // If our users aren't expecting that, bail.
+ if (!mir->isTruncated()) {
+ bailoutTest32(Assembler::Signed, output, output, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitUMod(LUMod* ins) {
+ MMod* mir = ins->mir();
+ ARMRegister lhs = toWRegister(ins->lhs());
+ ARMRegister rhs = toWRegister(ins->rhs());
+ ARMRegister output = toWRegister(ins->output());
+ Label done;
+
+ if (mir->canBeDivideByZero()) {
+ if (mir->isTruncated()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.Cbnz(rhs, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ // Truncated division by zero yields integer zero.
+ masm.Mov(output, rhs);
+ masm.Cbz(rhs, &done);
+ }
+ } else {
+ // Non-truncated division by zero produces a non-integer.
+ masm.Cmp(rhs, Operand(0));
+ bailoutIf(Assembler::Equal, ins->snapshot());
+ }
+ }
+
+ // Unsigned division.
+ masm.Udiv(output, lhs, rhs);
+
+ // Compute the remainder: output = lhs - (output * rhs).
+ masm.Msub(output, output, rhs, lhs);
+
+ if (!mir->isTruncated()) {
+ // Bail if the output would be negative.
+ //
+ // LUMod inputs may be Uint32, so care is taken to ensure the result
+ // is not unexpectedly signed.
+ bailoutCmp32(Assembler::LessThan, output, Imm32(0), ins->snapshot());
+ }
+
+ if (done.used()) {
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins) {
+ const MEffectiveAddress* mir = ins->mir();
+ const ARMRegister base = toWRegister(ins->base());
+ const ARMRegister index = toWRegister(ins->index());
+ const ARMRegister output = toWRegister(ins->output());
+
+ masm.Add(output, base, Operand(index, vixl::LSL, mir->scale()));
+ masm.Add(output, output, Operand(mir->displacement()));
+}
+
+void CodeGenerator::visitNegI(LNegI* ins) {
+ const ARMRegister input = toWRegister(ins->input());
+ const ARMRegister output = toWRegister(ins->output());
+ masm.Neg(output, input);
+}
+
+void CodeGenerator::visitNegI64(LNegI64* ins) {
+ const ARMRegister input = toXRegister(ins->input());
+ const ARMRegister output = toXRegister(ins->output());
+ masm.Neg(output, input);
+}
+
+void CodeGenerator::visitNegD(LNegD* ins) {
+ const ARMFPRegister input(ToFloatRegister(ins->input()), 64);
+ const ARMFPRegister output(ToFloatRegister(ins->output()), 64);
+ masm.Fneg(output, input);
+}
+
+void CodeGenerator::visitNegF(LNegF* ins) {
+ const ARMFPRegister input(ToFloatRegister(ins->input()), 32);
+ const ARMFPRegister output(ToFloatRegister(ins->output()), 32);
+ masm.Fneg(output, input);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement(
+ LCompareExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register temp =
+ lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
+
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, temp, output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, temp, output);
+ }
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement(
+ LAtomicExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register temp =
+ lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
+
+ Register value = ToRegister(lir->value());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
+ output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
+ output);
+ }
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ auto sync = Synchronization::Load();
+
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ }
+ masm.memoryBarrierAfter(sync);
+
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp1);
+
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ auto sync = Synchronization::Store();
+
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.store64(temp1, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.store64(temp1, dest);
+ }
+ masm.memoryBarrierAfter(sync);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 tempOut(out);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(oldval, temp1);
+ masm.loadBigInt64(newval, tempOut);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut,
+ temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = Register64(ToRegister(lir->temp2()));
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 tempOut = Register64(out);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ tempOut, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ tempOut, temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ }
+}
+
+void CodeGeneratorARM64::emitSimpleBinaryI64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* lir, JSOp op) {
+ const ARMRegister dest = ARMRegister(ToOutRegister64(lir).reg, 64);
+ const ARMRegister lhs =
+ ARMRegister(ToRegister64(lir->getInt64Operand(0)).reg, 64);
+ const LInt64Allocation rhsAlloc = lir->getInt64Operand(INT64_PIECES);
+ Operand rhs;
+
+ if (IsConstant(rhsAlloc)) {
+ rhs = Operand(ToInt64(rhsAlloc));
+ } else {
+ rhs = Operand(ARMRegister(ToRegister64(rhsAlloc).reg, 64));
+ }
+ switch (op) {
+ case JSOp::Add:
+ masm.Add(dest, lhs, rhs);
+ break;
+ case JSOp::Sub:
+ masm.Sub(dest, lhs, rhs);
+ break;
+ case JSOp::BitOr:
+ masm.Orr(dest, lhs, rhs);
+ break;
+ case JSOp::BitXor:
+ masm.Eor(dest, lhs, rhs);
+ break;
+ case JSOp::BitAnd:
+ masm.And(dest, lhs, rhs);
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitAddI64(LAddI64* lir) {
+ emitSimpleBinaryI64(lir, JSOp::Add);
+}
+
+void CodeGenerator::visitClzI64(LClzI64* ins) {
+ masm.clz64(ToRegister64(ins->getInt64Operand(0)), ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* ins) {
+ masm.ctz64(ToRegister64(ins->getInt64Operand(0)), ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitMulI64(LMulI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
+ const Register64 output = ToOutRegister64(lir);
+
+ if (IsConstant(rhs)) {
+ int64_t constant = ToInt64(rhs);
+ // Ad-hoc strength reduction, cf the x64 code as well as the 32-bit code
+ // higher up in this file. Bug 1712298 will lift this code to the MIR
+ // constant folding pass, or to lowering.
+ //
+ // This is for wasm integers only, so no input guards or overflow checking
+ // are needed.
+ switch (constant) {
+ case -1:
+ masm.Neg(ARMRegister(output.reg, 64),
+ ARMRegister(ToRegister64(lhs).reg, 64));
+ break;
+ case 0:
+ masm.Mov(ARMRegister(output.reg, 64), xzr);
+ break;
+ case 1:
+ if (ToRegister64(lhs) != output) {
+ masm.move64(ToRegister64(lhs), output);
+ }
+ break;
+ case 2:
+ masm.Add(ARMRegister(output.reg, 64),
+ ARMRegister(ToRegister64(lhs).reg, 64),
+ ARMRegister(ToRegister64(lhs).reg, 64));
+ break;
+ default:
+ // Use shift if constant is nonnegative power of 2.
+ if (constant > 0) {
+ int32_t shift = mozilla::FloorLog2(constant);
+ if (int64_t(1) << shift == constant) {
+ masm.Lsl(ARMRegister(output.reg, 64),
+ ARMRegister(ToRegister64(lhs).reg, 64), shift);
+ break;
+ }
+ }
+ masm.mul64(Imm64(constant), ToRegister64(lhs), output);
+ break;
+ }
+ } else {
+ masm.mul64(ToRegister64(lhs), ToRegister64(rhs), output);
+ }
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+ const Register64 input = ToRegister64(lir->getInt64Operand(0));
+ const Register64 output = ToOutRegister64(lir);
+ masm.Cmp(ARMRegister(input.reg, 64), ZeroRegister64);
+ masm.Cset(ARMRegister(output.reg, 64), Assembler::Zero);
+}
+
+void CodeGenerator::visitSubI64(LSubI64* lir) {
+ emitSimpleBinaryI64(lir, JSOp::Sub);
+}
+
+void CodeGenerator::visitPopcntI(LPopcntI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+ Register temp = ToRegister(ins->temp0());
+ masm.popcnt32(input, output, temp);
+}
+
+void CodeGenerator::visitBitOpI64(LBitOpI64* lir) {
+ emitSimpleBinaryI64(lir, lir->bitop());
+}
+
+void CodeGenerator::visitShiftI64(LShiftI64* lir) {
+ ARMRegister lhs(ToRegister64(lir->getInt64Operand(LShiftI64::Lhs)).reg, 64);
+ LAllocation* rhsAlloc = lir->getOperand(LShiftI64::Rhs);
+ ARMRegister dest(ToOutRegister64(lir).reg, 64);
+
+ if (rhsAlloc->isConstant()) {
+ int32_t shift = int32_t(rhsAlloc->toConstant()->toInt64() & 0x3F);
+ if (shift == 0) {
+ if (lhs.code() != dest.code()) {
+ masm.Mov(dest, lhs);
+ }
+ } else {
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ masm.Lsl(dest, lhs, shift);
+ break;
+ case JSOp::Rsh:
+ masm.Asr(dest, lhs, shift);
+ break;
+ case JSOp::Ursh:
+ masm.Lsr(dest, lhs, shift);
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ }
+ } else {
+ ARMRegister rhs(ToRegister(rhsAlloc), 64);
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ masm.Lsl(dest, lhs, rhs);
+ break;
+ case JSOp::Rsh:
+ masm.Asr(dest, lhs, rhs);
+ break;
+ case JSOp::Ursh:
+ masm.Lsr(dest, lhs, rhs);
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ }
+}
+
+void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) {
+ MOZ_ASSERT(ins->instance()->isBogus());
+ masm.movePtr(HeapReg, ToRegister(ins->output()));
+}
+
+// If we have a constant base ptr, try to add the offset to it, to generate
+// better code when the full address is known. The addition may overflow past
+// 32 bits because the front end does nothing special if the base is a large
+// constant and base+offset overflows; sidestep this by performing the addition
+// anyway, overflowing to 64-bit.
+
+static Maybe<uint64_t> IsAbsoluteAddress(const LAllocation* ptr,
+ const wasm::MemoryAccessDesc& access) {
+ if (ptr->isConstantValue()) {
+ const MConstant* c = ptr->toConstant();
+ uint64_t base_address = c->type() == MIRType::Int32
+ ? uint64_t(uint32_t(c->toInt32()))
+ : uint64_t(c->toInt64());
+ uint64_t offset = access.offset();
+ return Some(base_address + offset);
+ }
+ return Nothing();
+}
+
+void CodeGenerator::visitWasmLoad(LWasmLoad* lir) {
+ const MWasmLoad* mir = lir->mir();
+
+ if (Maybe<uint64_t> absAddr = IsAbsoluteAddress(lir->ptr(), mir->access())) {
+ masm.wasmLoadAbsolute(mir->access(), HeapReg, absAddr.value(),
+ ToAnyRegister(lir->output()), Register64::Invalid());
+ return;
+ }
+
+ // ptr is a GPR and is either a 32-bit value zero-extended to 64-bit, or a
+ // true 64-bit value.
+ masm.wasmLoad(mir->access(), HeapReg, ToRegister(lir->ptr()),
+ ToAnyRegister(lir->output()));
+}
+
+void CodeGenerator::visitCopySignD(LCopySignD* ins) {
+ MOZ_ASSERT(ins->getTemp(0)->isBogusTemp());
+ MOZ_ASSERT(ins->getTemp(1)->isBogusTemp());
+ masm.copySignDouble(ToFloatRegister(ins->getOperand(0)),
+ ToFloatRegister(ins->getOperand(1)),
+ ToFloatRegister(ins->getDef(0)));
+}
+
+void CodeGenerator::visitCopySignF(LCopySignF* ins) {
+ MOZ_ASSERT(ins->getTemp(0)->isBogusTemp());
+ MOZ_ASSERT(ins->getTemp(1)->isBogusTemp());
+ masm.copySignFloat32(ToFloatRegister(ins->getOperand(0)),
+ ToFloatRegister(ins->getOperand(1)),
+ ToFloatRegister(ins->getDef(0)));
+}
+
+void CodeGenerator::visitPopcntI64(LPopcntI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ Register temp = ToRegister(lir->getTemp(0));
+ masm.popcnt64(input, output, temp);
+}
+
+void CodeGenerator::visitRotateI64(LRotateI64* lir) {
+ bool rotateLeft = lir->mir()->isLeftRotate();
+ Register64 input = ToRegister64(lir->input());
+ Register64 output = ToOutRegister64(lir);
+ const LAllocation* count = lir->count();
+
+ if (count->isConstant()) {
+ int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F);
+ if (c == 0) {
+ if (input != output) {
+ masm.move64(input, output);
+ return;
+ }
+ }
+ if (rotateLeft) {
+ masm.rotateLeft64(Imm32(c), input, output, InvalidReg);
+ } else {
+ masm.rotateRight64(Imm32(c), input, output, InvalidReg);
+ }
+ } else {
+ Register c = ToRegister(count);
+ if (rotateLeft) {
+ masm.rotateLeft64(c, input, output, InvalidReg);
+ } else {
+ masm.rotateRight64(c, input, output, InvalidReg);
+ }
+ }
+}
+
+void CodeGenerator::visitWasmStore(LWasmStore* lir) {
+ const MWasmStore* mir = lir->mir();
+
+ if (Maybe<uint64_t> absAddr = IsAbsoluteAddress(lir->ptr(), mir->access())) {
+ masm.wasmStoreAbsolute(mir->access(), ToAnyRegister(lir->value()),
+ Register64::Invalid(), HeapReg, absAddr.value());
+ return;
+ }
+
+ masm.wasmStore(mir->access(), ToAnyRegister(lir->value()), HeapReg,
+ ToRegister(lir->ptr()));
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+ MCompare* mir = lir->mir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ Register output = ToRegister(lir->output());
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+
+ if (IsConstant(rhs)) {
+ masm.cmpPtrSet(JSOpToCondition(lir->jsop(), isSigned), lhsReg,
+ ImmWord(ToInt64(rhs)), output);
+ } else if (rhs.value().isGeneralReg()) {
+ masm.cmpPtrSet(JSOpToCondition(lir->jsop(), isSigned), lhsReg,
+ ToRegister64(rhs).reg, output);
+ } else {
+ masm.cmpPtrSet(
+ GetCondForSwappedOperands(JSOpToCondition(lir->jsop(), isSigned)),
+ ToAddress(rhs.value()), lhsReg, output);
+ }
+}
+
+void CodeGenerator::visitWasmSelect(LWasmSelect* lir) {
+ MIRType mirType = lir->mir()->type();
+ Register condReg = ToRegister(lir->condExpr());
+
+ masm.test32(condReg, condReg);
+
+ switch (mirType) {
+ case MIRType::Int32:
+ case MIRType::RefOrNull: {
+ Register outReg = ToRegister(lir->output());
+ Register trueReg = ToRegister(lir->trueExpr());
+ Register falseReg = ToRegister(lir->falseExpr());
+
+ if (mirType == MIRType::Int32) {
+ masm.Csel(ARMRegister(outReg, 32), ARMRegister(trueReg, 32),
+ ARMRegister(falseReg, 32), Assembler::NonZero);
+ } else {
+ masm.Csel(ARMRegister(outReg, 64), ARMRegister(trueReg, 64),
+ ARMRegister(falseReg, 64), Assembler::NonZero);
+ }
+ break;
+ }
+
+ case MIRType::Float32:
+ case MIRType::Double:
+ case MIRType::Simd128: {
+ FloatRegister outReg = ToFloatRegister(lir->output());
+ FloatRegister trueReg = ToFloatRegister(lir->trueExpr());
+ FloatRegister falseReg = ToFloatRegister(lir->falseExpr());
+
+ switch (mirType) {
+ case MIRType::Float32:
+ masm.Fcsel(ARMFPRegister(outReg, 32), ARMFPRegister(trueReg, 32),
+ ARMFPRegister(falseReg, 32), Assembler::NonZero);
+ break;
+ case MIRType::Double:
+ masm.Fcsel(ARMFPRegister(outReg, 64), ARMFPRegister(trueReg, 64),
+ ARMFPRegister(falseReg, 64), Assembler::NonZero);
+ break;
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128: {
+ MOZ_ASSERT(outReg == trueReg);
+ Label done;
+ masm.j(Assembler::NonZero, &done);
+ masm.moveSimd128(falseReg, outReg);
+ masm.bind(&done);
+ break;
+ }
+#endif
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ }
+
+ default: {
+ MOZ_CRASH("unhandled type in visitWasmSelect!");
+ }
+ }
+}
+
+// We expect to handle the cases: compare is {{U,}Int32, {U,}Int64}, Float32,
+// Double}, and select is {{U,}Int32, {U,}Int64}, Float32, Double},
+// independently.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ MCompare::CompareType compTy = ins->compareType();
+
+ // Set flag.
+ if (compTy == MCompare::Compare_Int32 || compTy == MCompare::Compare_UInt32) {
+ Register lhs = ToRegister(ins->leftExpr());
+ if (ins->rightExpr()->isConstant()) {
+ masm.cmp32(lhs, Imm32(ins->rightExpr()->toConstant()->toInt32()));
+ } else {
+ masm.cmp32(lhs, ToRegister(ins->rightExpr()));
+ }
+ } else if (compTy == MCompare::Compare_Int64 ||
+ compTy == MCompare::Compare_UInt64) {
+ Register lhs = ToRegister(ins->leftExpr());
+ if (ins->rightExpr()->isConstant()) {
+ masm.cmpPtr(lhs, Imm64(ins->rightExpr()->toConstant()->toInt64()));
+ } else {
+ masm.cmpPtr(lhs, ToRegister(ins->rightExpr()));
+ }
+ } else if (compTy == MCompare::Compare_Float32) {
+ masm.compareFloat(JSOpToDoubleCondition(ins->jsop()),
+ ToFloatRegister(ins->leftExpr()),
+ ToFloatRegister(ins->rightExpr()));
+ } else if (compTy == MCompare::Compare_Double) {
+ masm.compareDouble(JSOpToDoubleCondition(ins->jsop()),
+ ToFloatRegister(ins->leftExpr()),
+ ToFloatRegister(ins->rightExpr()));
+ } else {
+ // Ref types not supported yet; v128 is not yet observed to be worth
+ // optimizing.
+ MOZ_CRASH("CodeGenerator::visitWasmCompareAndSelect: unexpected type (1)");
+ }
+
+ // Act on flag.
+ Assembler::Condition cond;
+ if (compTy == MCompare::Compare_Float32 ||
+ compTy == MCompare::Compare_Double) {
+ cond = Assembler::ConditionFromDoubleCondition(
+ JSOpToDoubleCondition(ins->jsop()));
+ } else {
+ cond = JSOpToCondition(compTy, ins->jsop());
+ }
+ MIRType insTy = ins->mir()->type();
+ if (insTy == MIRType::Int32 || insTy == MIRType::Int64) {
+ Register destReg = ToRegister(ins->output());
+ Register trueReg = ToRegister(ins->ifTrueExpr());
+ Register falseReg = ToRegister(ins->ifFalseExpr());
+ size_t size = insTy == MIRType::Int32 ? 32 : 64;
+ masm.Csel(ARMRegister(destReg, size), ARMRegister(trueReg, size),
+ ARMRegister(falseReg, size), cond);
+ } else if (insTy == MIRType::Float32 || insTy == MIRType::Double) {
+ FloatRegister destReg = ToFloatRegister(ins->output());
+ FloatRegister trueReg = ToFloatRegister(ins->ifTrueExpr());
+ FloatRegister falseReg = ToFloatRegister(ins->ifFalseExpr());
+ size_t size = MIRTypeToSize(insTy) * 8;
+ masm.Fcsel(ARMFPRegister(destReg, size), ARMFPRegister(trueReg, size),
+ ARMFPRegister(falseReg, size), cond);
+ } else {
+ // See above.
+ MOZ_CRASH("CodeGenerator::visitWasmCompareAndSelect: unexpected type (2)");
+ }
+}
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir) {
+ const MWasmLoad* mir = lir->mir();
+
+ if (Maybe<uint64_t> absAddr = IsAbsoluteAddress(lir->ptr(), mir->access())) {
+ masm.wasmLoadAbsolute(mir->access(), HeapReg, absAddr.value(),
+ AnyRegister(), ToOutRegister64(lir));
+ return;
+ }
+
+ masm.wasmLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()),
+ ToOutRegister64(lir));
+}
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir) {
+ const MWasmStore* mir = lir->mir();
+
+ if (Maybe<uint64_t> absAddr = IsAbsoluteAddress(lir->ptr(), mir->access())) {
+ masm.wasmStoreAbsolute(mir->access(), AnyRegister(),
+ ToRegister64(lir->value()), HeapReg,
+ absAddr.value());
+ return;
+ }
+
+ masm.wasmStoreI64(mir->access(), ToRegister64(lir->value()), HeapReg,
+ ToRegister(lir->ptr()));
+}
+
+void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) {
+ masm.memoryBarrier(ins->type());
+}
+
+void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register base = ToRegister(lir->base());
+ Register out = ToRegister(lir->output());
+
+ masm.Adds(ARMRegister(out, 32), ARMRegister(base, 32),
+ Operand(mir->offset()));
+ OutOfLineAbortingWasmTrap* ool = new (alloc())
+ OutOfLineAbortingWasmTrap(mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
+ addOutOfLineCode(ool, mir);
+ masm.j(Assembler::CarrySet, ool->entry());
+}
+
+void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register64 base = ToRegister64(lir->base());
+ Register64 out = ToOutRegister64(lir);
+
+ masm.Adds(ARMRegister(out.reg, 64), ARMRegister(base.reg, 64),
+ Operand(mir->offset()));
+ OutOfLineAbortingWasmTrap* ool = new (alloc())
+ OutOfLineAbortingWasmTrap(mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
+ addOutOfLineCode(ool, mir);
+ masm.j(Assembler::CarrySet, ool->entry());
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ Register condReg = ToRegister(lir->condExpr());
+ Register64 trueReg = ToRegister64(lir->trueExpr());
+ Register64 falseReg = ToRegister64(lir->falseExpr());
+ Register64 outReg = ToOutRegister64(lir);
+
+ masm.test32(condReg, condReg);
+ masm.Csel(ARMRegister(outReg.reg, 64), ARMRegister(trueReg.reg, 64),
+ ARMRegister(falseReg.reg, 64), Assembler::NonZero);
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ Register64 output = ToOutRegister64(ins);
+ switch (ins->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.move8To64SignExtend(input.reg, output);
+ break;
+ case MSignExtendInt64::Half:
+ masm.move16To64SignExtend(input.reg, output);
+ break;
+ case MSignExtendInt64::Word:
+ masm.move32To64SignExtend(input.reg, output);
+ break;
+ }
+}
+
+void CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MWasmReinterpret* ins = lir->mir();
+
+ MIRType to = ins->type();
+ mozilla::DebugOnly<MIRType> from = ins->input()->type();
+
+ switch (to) {
+ case MIRType::Int32:
+ MOZ_ASSERT(from == MIRType::Float32);
+ masm.moveFloat32ToGPR(ToFloatRegister(lir->input()),
+ ToRegister(lir->output()));
+ break;
+ case MIRType::Float32:
+ MOZ_ASSERT(from == MIRType::Int32);
+ masm.moveGPRToFloat32(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+ break;
+ case MIRType::Double:
+ case MIRType::Int64:
+ MOZ_CRASH("not handled by this LIR opcode");
+ default:
+ MOZ_CRASH("unexpected WasmReinterpret");
+ }
+}
+
+void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(masm.getStackPointer(), mir->spOffset());
+ if (IsConstant(ins->arg())) {
+ masm.store64(Imm64(ToInt64(ins->arg())), dst);
+ } else {
+ masm.store64(ToRegister64(ins->arg()), dst);
+ }
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ MBasicBlock* mirTrue = lir->ifTrue();
+ MBasicBlock* mirFalse = lir->ifFalse();
+
+ // Jump to the True block if NonZero.
+ // Jump to the False block if Zero.
+ if (isNextBlock(mirFalse->lir())) {
+ masm.Cbnz(ARMRegister(input.reg, 64), getJumpLabelForBranch(mirTrue));
+ } else {
+ masm.Cbz(ARMRegister(input.reg, 64), getJumpLabelForBranch(mirFalse));
+ if (!isNextBlock(mirTrue->lir())) {
+ jumpToBlock(mirTrue);
+ }
+ }
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ if (input->isMemory()) {
+ masm.load32(ToAddress(input), output);
+ } else {
+ // Really this is a 64-bit input register and we could use move64To32.
+ masm.Mov(ARMRegister(output, 32), ARMRegister(ToRegister(input), 32));
+ }
+ } else {
+ MOZ_CRASH("Not implemented.");
+ }
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ Register input = ToRegister(lir->getOperand(0));
+ Register64 output = ToOutRegister64(lir);
+
+ if (lir->mir()->isUnsigned()) {
+ masm.move32To64ZeroExtend(input, output);
+ } else {
+ masm.move32To64SignExtend(input, output);
+ }
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
+ // Generates no code on this platform because the input is assumed to have
+ // canonical form.
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+ masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ // Generates no code on this platform because the input is assumed to have
+ // canonical form.
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+ masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* comp) {
+ const MCompare* mir = comp->cmpMir();
+ const mozilla::DebugOnly<MCompare::CompareType> type = mir->compareType();
+ const LInt64Allocation left =
+ comp->getInt64Operand(LCompareI64AndBranch::Lhs);
+ const LInt64Allocation right =
+ comp->getInt64Operand(LCompareI64AndBranch::Rhs);
+
+ MOZ_ASSERT(type == MCompare::Compare_Int64 ||
+ type == MCompare::Compare_UInt64);
+ if (IsConstant(right)) {
+ masm.Cmp(ARMRegister(ToRegister64(left).reg, 64), ToInt64(right));
+ } else {
+ masm.Cmp(ARMRegister(ToRegister64(left).reg, 64),
+ ARMRegister(ToRegister64(right).reg, 64));
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition cond = JSOpToCondition(comp->jsop(), isSigned);
+ emitBranch(cond, comp->ifTrue(), comp->ifFalse());
+}
+
+void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) {
+ auto input = ToFloatRegister(lir->input());
+ auto output = ToRegister(lir->output());
+
+ MWasmTruncateToInt32* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ if (mir->isUnsigned()) {
+ if (fromType == MIRType::Double) {
+ masm.wasmTruncateDoubleToUInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else if (fromType == MIRType::Float32) {
+ masm.wasmTruncateFloat32ToUInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+
+ masm.bind(ool->rejoin());
+ return;
+ }
+
+ if (fromType == MIRType::Double) {
+ masm.wasmTruncateDoubleToInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else if (fromType == MIRType::Float32) {
+ masm.wasmTruncateFloat32ToInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ MWasmTruncateToInt64* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ Label* oolRejoin = ool->rejoin();
+ bool isSaturating = mir->isSaturating();
+
+ if (fromType == MIRType::Double) {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ } else {
+ masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ }
+ } else {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ } else {
+ masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ }
+ }
+}
+
+void CodeGeneratorARM64::visitOutOfLineWasmTruncateCheck(
+ OutOfLineWasmTruncateCheck* ool) {
+ FloatRegister input = ool->input();
+ Register output = ool->output();
+ Register64 output64 = ool->output64();
+ MIRType fromType = ool->fromType();
+ MIRType toType = ool->toType();
+ Label* oolRejoin = ool->rejoin();
+ TruncFlags flags = ool->flags();
+ wasm::BytecodeOffset off = ool->bytecodeOffset();
+
+ if (fromType == MIRType::Float32) {
+ if (toType == MIRType::Int32) {
+ masm.oolWasmTruncateCheckF32ToI32(input, output, flags, off, oolRejoin);
+ } else if (toType == MIRType::Int64) {
+ masm.oolWasmTruncateCheckF32ToI64(input, output64, flags, off, oolRejoin);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+ } else if (fromType == MIRType::Double) {
+ if (toType == MIRType::Int32) {
+ masm.oolWasmTruncateCheckF64ToI32(input, output, flags, off, oolRejoin);
+ } else if (toType == MIRType::Int64) {
+ masm.oolWasmTruncateCheckF64ToI64(input, output64, flags, off, oolRejoin);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+ masm.moveDoubleToGPR64(ToFloatRegister(lir->input()), ToOutRegister64(lir));
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+ masm.moveGPR64ToDouble(
+ ToRegister64(lir->getInt64Operand(LWasmReinterpretFromI64::Input)),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop(
+ LAtomicTypedArrayElementBinop* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register elements = ToRegister(lir->elements());
+ Register flagTemp = ToRegister(lir->temp1());
+ Register outTemp =
+ lir->temp2()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp2());
+ Register value = ToRegister(lir->value());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, flagTemp, outTemp,
+ output);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, flagTemp, outTemp,
+ output);
+ }
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(
+ LAtomicTypedArrayElementBinopForEffect* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register flagTemp = ToRegister(lir->flagTemp());
+ Register value = ToRegister(lir->value());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, flagTemp);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, flagTemp);
+ }
+}
+
+void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ MIRType outputType = lir->mir()->type();
+ MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
+
+ if (outputType == MIRType::Double) {
+ if (lir->mir()->isUnsigned()) {
+ masm.convertUInt64ToDouble(input, output, Register::Invalid());
+ } else {
+ masm.convertInt64ToDouble(input, output);
+ }
+ } else {
+ if (lir->mir()->isUnsigned()) {
+ masm.convertUInt64ToFloat32(input, output, Register::Invalid());
+ } else {
+ masm.convertInt64ToFloat32(input, output);
+ }
+ }
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ Label done;
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label isNotDivByZero;
+ masm.Cbnz(ARMRegister(rhs, 64), &isNotDivByZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&isNotDivByZero);
+ }
+
+ // Handle an integer overflow exception from INT64_MIN / -1.
+ if (lir->canBeNegativeOverflow()) {
+ Label noOverflow;
+ masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &noOverflow);
+ masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &noOverflow);
+ if (lir->mir()->isMod()) {
+ masm.movePtr(ImmWord(0), output);
+ } else {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ masm.jump(&done);
+ masm.bind(&noOverflow);
+ }
+
+ masm.Sdiv(ARMRegister(output, 64), ARMRegister(lhs, 64),
+ ARMRegister(rhs, 64));
+ if (lir->mir()->isMod()) {
+ masm.Msub(ARMRegister(output, 64), ARMRegister(output, 64),
+ ARMRegister(rhs, 64), ARMRegister(lhs, 64));
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ Label done;
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label isNotDivByZero;
+ masm.Cbnz(ARMRegister(rhs, 64), &isNotDivByZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&isNotDivByZero);
+ }
+
+ masm.Udiv(ARMRegister(output, 64), ARMRegister(lhs, 64),
+ ARMRegister(rhs, 64));
+ if (lir->mir()->isMod()) {
+ masm.Msub(ARMRegister(output, 64), ARMRegister(output, 64),
+ ARMRegister(rhs, 64), ARMRegister(lhs, 64));
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitSimd128(LSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantSimd128(ins->simd128(), ToFloatRegister(out));
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128Bitselect: {
+ FloatRegister lhs = ToFloatRegister(ins->v0());
+ FloatRegister rhs = ToFloatRegister(ins->v1());
+ FloatRegister controlDest = ToFloatRegister(ins->v2());
+ masm.bitwiseSelectSimd128(lhs, rhs, controlDest);
+ break;
+ }
+ case wasm::SimdOp::F32x4RelaxedFma:
+ masm.fmaFloat32x4(ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::F32x4RelaxedFnma:
+ masm.fnmaFloat32x4(ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::F64x2RelaxedFma:
+ masm.fmaFloat64x2(ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::F64x2RelaxedFnma:
+ masm.fnmaFloat64x2(ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::I8x16RelaxedLaneSelect:
+ case wasm::SimdOp::I16x8RelaxedLaneSelect:
+ case wasm::SimdOp::I32x4RelaxedLaneSelect:
+ case wasm::SimdOp::I64x2RelaxedLaneSelect: {
+ FloatRegister lhs = ToFloatRegister(ins->v0());
+ FloatRegister rhs = ToFloatRegister(ins->v1());
+ FloatRegister maskDest = ToFloatRegister(ins->v2());
+ masm.laneSelectSimd128(maskDest, lhs, rhs, maskDest);
+ break;
+ }
+ case wasm::SimdOp::I32x4DotI8x16I7x16AddS:
+ masm.dotInt8x16Int7x16ThenAdd(
+ ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()), ToFloatRegister(ins->temp()));
+ break;
+ case wasm::SimdOp::F32x4RelaxedDotBF16x8AddF32x4:
+ masm.dotBFloat16x8ThenAdd(
+ ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()), ToFloatRegister(ins->temp()));
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister lhs = ToFloatRegister(ins->lhs());
+ FloatRegister rhs = ToFloatRegister(ins->rhs());
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128And:
+ masm.bitwiseAndSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128Or:
+ masm.bitwiseOrSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128Xor:
+ masm.bitwiseXorSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128AndNot:
+ masm.bitwiseAndNotSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16AvgrU:
+ masm.unsignedAverageInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8AvgrU:
+ masm.unsignedAverageInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Add:
+ masm.addInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16AddSatS:
+ masm.addSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16AddSatU:
+ masm.unsignedAddSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Sub:
+ masm.subInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16SubSatS:
+ masm.subSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16SubSatU:
+ masm.unsignedSubSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MinS:
+ masm.minInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MinU:
+ masm.unsignedMinInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MaxS:
+ masm.maxInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MaxU:
+ masm.unsignedMaxInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Add:
+ masm.addInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8AddSatS:
+ masm.addSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8AddSatU:
+ masm.unsignedAddSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Sub:
+ masm.subInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8SubSatS:
+ masm.subSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8SubSatU:
+ masm.unsignedSubSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Mul:
+ masm.mulInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MinS:
+ masm.minInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MinU:
+ masm.unsignedMinInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MaxS:
+ masm.maxInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MaxU:
+ masm.unsignedMaxInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Add:
+ masm.addInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Sub:
+ masm.subInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Mul:
+ masm.mulInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MinS:
+ masm.minInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MinU:
+ masm.unsignedMinInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MaxS:
+ masm.maxInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MaxU:
+ masm.unsignedMaxInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Add:
+ masm.addInt64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Sub:
+ masm.subInt64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Mul: {
+ auto temp1 = ToFloatRegister(ins->getTemp(0));
+ auto temp2 = ToFloatRegister(ins->getTemp(1));
+ masm.mulInt64x2(lhs, rhs, dest, temp1, temp2);
+ break;
+ }
+ case wasm::SimdOp::F32x4Add:
+ masm.addFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Sub:
+ masm.subFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Mul:
+ masm.mulFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Div:
+ masm.divFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Min:
+ masm.minFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Max:
+ masm.maxFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Add:
+ masm.addFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Sub:
+ masm.subFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Mul:
+ masm.mulFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Div:
+ masm.divFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Min:
+ masm.minFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Max:
+ masm.maxFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Swizzle:
+ masm.swizzleInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16RelaxedSwizzle:
+ masm.swizzleInt8x16Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16NarrowI16x8S:
+ masm.narrowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16NarrowI16x8U:
+ masm.unsignedNarrowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8NarrowI32x4S:
+ masm.narrowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8NarrowI32x4U:
+ masm.unsignedNarrowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Eq:
+ masm.compareInt8x16(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Ne:
+ masm.compareInt8x16(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LtS:
+ masm.compareInt8x16(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GtS:
+ masm.compareInt8x16(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LeS:
+ masm.compareInt8x16(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GeS:
+ masm.compareInt8x16(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LtU:
+ masm.compareInt8x16(Assembler::Below, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GtU:
+ masm.compareInt8x16(Assembler::Above, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LeU:
+ masm.compareInt8x16(Assembler::BelowOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GeU:
+ masm.compareInt8x16(Assembler::AboveOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Eq:
+ masm.compareInt16x8(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Ne:
+ masm.compareInt16x8(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LtS:
+ masm.compareInt16x8(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GtS:
+ masm.compareInt16x8(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LeS:
+ masm.compareInt16x8(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GeS:
+ masm.compareInt16x8(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LtU:
+ masm.compareInt16x8(Assembler::Below, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GtU:
+ masm.compareInt16x8(Assembler::Above, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LeU:
+ masm.compareInt16x8(Assembler::BelowOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GeU:
+ masm.compareInt16x8(Assembler::AboveOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Eq:
+ masm.compareInt32x4(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Ne:
+ masm.compareInt32x4(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LtS:
+ masm.compareInt32x4(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GtS:
+ masm.compareInt32x4(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LeS:
+ masm.compareInt32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GeS:
+ masm.compareInt32x4(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LtU:
+ masm.compareInt32x4(Assembler::Below, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GtU:
+ masm.compareInt32x4(Assembler::Above, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LeU:
+ masm.compareInt32x4(Assembler::BelowOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GeU:
+ masm.compareInt32x4(Assembler::AboveOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Eq:
+ masm.compareInt64x2(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2LtS:
+ masm.compareInt64x2(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2GtS:
+ masm.compareInt64x2(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2LeS:
+ masm.compareInt64x2(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2GeS:
+ masm.compareInt64x2(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Ne:
+ masm.compareInt64x2(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Eq:
+ masm.compareFloat32x4(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Ne:
+ masm.compareFloat32x4(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Lt:
+ masm.compareFloat32x4(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Gt:
+ masm.compareFloat32x4(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Le:
+ masm.compareFloat32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Ge:
+ masm.compareFloat32x4(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Eq:
+ masm.compareFloat64x2(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Ne:
+ masm.compareFloat64x2(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Lt:
+ masm.compareFloat64x2(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Gt:
+ masm.compareFloat64x2(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Le:
+ masm.compareFloat64x2(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Ge:
+ masm.compareFloat64x2(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4PMax:
+ masm.pseudoMaxFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4PMin:
+ masm.pseudoMinFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2PMax:
+ masm.pseudoMaxFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2PMin:
+ masm.pseudoMinFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4DotI16x8S:
+ masm.widenDotInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtmulLowI8x16S:
+ masm.extMulLowInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtmulHighI8x16S:
+ masm.extMulHighInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtmulLowI8x16U:
+ masm.unsignedExtMulLowInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtmulHighI8x16U:
+ masm.unsignedExtMulHighInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtmulLowI16x8S:
+ masm.extMulLowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtmulHighI16x8S:
+ masm.extMulHighInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtmulLowI16x8U:
+ masm.unsignedExtMulLowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtmulHighI16x8U:
+ masm.unsignedExtMulHighInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtmulLowI32x4S:
+ masm.extMulLowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtmulHighI32x4S:
+ masm.extMulHighInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtmulLowI32x4U:
+ masm.unsignedExtMulLowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtmulHighI32x4U:
+ masm.unsignedExtMulHighInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Q15MulrSatS:
+ masm.q15MulrSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4RelaxedMin:
+ masm.minFloat32x4Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4RelaxedMax:
+ masm.maxFloat32x4Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2RelaxedMin:
+ masm.minFloat64x2Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2RelaxedMax:
+ masm.maxFloat64x2Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8RelaxedQ15MulrS:
+ masm.q15MulrInt16x8Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8DotI8x16I7x16S:
+ masm.dotInt8x16Int7x16(lhs, rhs, dest);
+ break;
+ default:
+ MOZ_CRASH("Binary SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmBinarySimd128WithConstant(
+ LWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmVariableShiftSimd128(
+ LWasmVariableShiftSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister lhs = ToFloatRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Shl:
+ masm.leftShiftInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16ShrS:
+ masm.rightShiftInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16ShrU:
+ masm.unsignedRightShiftInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Shl:
+ masm.leftShiftInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ShrS:
+ masm.rightShiftInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ShrU:
+ masm.unsignedRightShiftInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Shl:
+ masm.leftShiftInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ShrS:
+ masm.rightShiftInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ShrU:
+ masm.unsignedRightShiftInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Shl:
+ masm.leftShiftInt64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ShrS:
+ masm.rightShiftInt64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ShrU:
+ masm.unsignedRightShiftInt64x2(lhs, rhs, dest);
+ break;
+ default:
+ MOZ_CRASH("Shift SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmConstantShiftSimd128(
+ LWasmConstantShiftSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+ int32_t shift = ins->shift();
+
+ if (shift == 0) {
+ if (src != dest) {
+ masm.moveSimd128(src, dest);
+ }
+ return;
+ }
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Shl:
+ masm.leftShiftInt8x16(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I8x16ShrS:
+ masm.rightShiftInt8x16(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I8x16ShrU:
+ masm.unsignedRightShiftInt8x16(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I16x8Shl:
+ masm.leftShiftInt16x8(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I16x8ShrS:
+ masm.rightShiftInt16x8(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I16x8ShrU:
+ masm.unsignedRightShiftInt16x8(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I32x4Shl:
+ masm.leftShiftInt32x4(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I32x4ShrS:
+ masm.rightShiftInt32x4(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I32x4ShrU:
+ masm.unsignedRightShiftInt32x4(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I64x2Shl:
+ masm.leftShiftInt64x2(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I64x2ShrS:
+ masm.rightShiftInt64x2(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I64x2ShrU:
+ masm.unsignedRightShiftInt64x2(Imm32(shift), src, dest);
+ break;
+ default:
+ MOZ_CRASH("Shift SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmSignReplicationSimd128(
+ LWasmSignReplicationSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister lhs = ToFloatRegister(ins->lhs());
+ FloatRegister rhs = ToFloatRegister(ins->rhs());
+ FloatRegister dest = ToFloatRegister(ins->output());
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ SimdConstant control = ins->control();
+ switch (ins->op()) {
+ case SimdShuffleOp::BLEND_8x16: {
+ masm.blendInt8x16(reinterpret_cast<const uint8_t*>(control.asInt8x16()),
+ lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::BLEND_16x8: {
+ masm.blendInt16x8(reinterpret_cast<const uint16_t*>(control.asInt16x8()),
+ lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::CONCAT_RIGHT_SHIFT_8x16: {
+ int8_t count = 16 - control.asInt8x16()[0];
+ MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
+ masm.concatAndRightShiftSimd128(lhs, rhs, dest, count);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_HIGH_8x16: {
+ masm.interleaveHighInt8x16(lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_HIGH_16x8: {
+ masm.interleaveHighInt16x8(lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_HIGH_32x4: {
+ masm.interleaveHighInt32x4(lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_HIGH_64x2: {
+ masm.interleaveHighInt64x2(lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_LOW_8x16: {
+ masm.interleaveLowInt8x16(lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_LOW_16x8: {
+ masm.interleaveLowInt16x8(lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_LOW_32x4: {
+ masm.interleaveLowInt32x4(lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_LOW_64x2: {
+ masm.interleaveLowInt64x2(lhs, rhs, dest);
+ break;
+ }
+ case SimdShuffleOp::SHUFFLE_BLEND_8x16: {
+ masm.shuffleInt8x16(reinterpret_cast<const uint8_t*>(control.asInt8x16()),
+ lhs, rhs, dest);
+ break;
+ }
+ default: {
+ MOZ_CRASH("Unsupported SIMD shuffle operation");
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+ SimdConstant control = ins->control();
+ switch (ins->op()) {
+ case SimdPermuteOp::BROADCAST_8x16: {
+ const SimdConstant::I8x16& mask = control.asInt8x16();
+ int8_t source = mask[0];
+ masm.splatX16(source, src, dest);
+ break;
+ }
+ case SimdPermuteOp::BROADCAST_16x8: {
+ const SimdConstant::I16x8& mask = control.asInt16x8();
+ int16_t source = mask[0];
+ masm.splatX8(source, src, dest);
+ break;
+ }
+ case SimdPermuteOp::MOVE: {
+ masm.moveSimd128(src, dest);
+ break;
+ }
+ case SimdPermuteOp::PERMUTE_8x16: {
+ const SimdConstant::I8x16& mask = control.asInt8x16();
+# ifdef DEBUG
+ mozilla::DebugOnly<int> i;
+ for (i = 0; i < 16 && mask[i] == i; i++) {
+ }
+ MOZ_ASSERT(i < 16, "Should have been a MOVE operation");
+# endif
+ masm.permuteInt8x16(reinterpret_cast<const uint8_t*>(mask), src, dest);
+ break;
+ }
+ case SimdPermuteOp::PERMUTE_16x8: {
+ const SimdConstant::I16x8& mask = control.asInt16x8();
+# ifdef DEBUG
+ mozilla::DebugOnly<int> i;
+ for (i = 0; i < 8 && mask[i] == i; i++) {
+ }
+ MOZ_ASSERT(i < 8, "Should have been a MOVE operation");
+# endif
+ masm.permuteInt16x8(reinterpret_cast<const uint16_t*>(mask), src, dest);
+ break;
+ }
+ case SimdPermuteOp::PERMUTE_32x4: {
+ const SimdConstant::I32x4& mask = control.asInt32x4();
+# ifdef DEBUG
+ mozilla::DebugOnly<int> i;
+ for (i = 0; i < 4 && mask[i] == i; i++) {
+ }
+ MOZ_ASSERT(i < 4, "Should have been a MOVE operation");
+# endif
+ masm.permuteInt32x4(reinterpret_cast<const uint32_t*>(mask), src, dest);
+ break;
+ }
+ case SimdPermuteOp::ROTATE_RIGHT_8x16: {
+ int8_t count = control.asInt8x16()[0];
+ MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
+ masm.rotateRightSimd128(src, dest, count);
+ break;
+ }
+ case SimdPermuteOp::SHIFT_LEFT_8x16: {
+ int8_t count = control.asInt8x16()[0];
+ MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
+ masm.leftShiftSimd128(Imm32(count), src, dest);
+ break;
+ }
+ case SimdPermuteOp::SHIFT_RIGHT_8x16: {
+ int8_t count = control.asInt8x16()[0];
+ MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
+ masm.rightShiftSimd128(Imm32(count), src, dest);
+ break;
+ }
+ case SimdPermuteOp::REVERSE_16x8:
+ masm.reverseInt16x8(src, dest);
+ break;
+ case SimdPermuteOp::REVERSE_32x4:
+ masm.reverseInt32x4(src, dest);
+ break;
+ case SimdPermuteOp::REVERSE_64x2:
+ masm.reverseInt64x2(src, dest);
+ break;
+ default: {
+ MOZ_CRASH("Unsupported SIMD permutation operation");
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ToFloatRegister(ins->lhs()) == ToFloatRegister(ins->output()));
+ FloatRegister lhsDest = ToFloatRegister(ins->lhs());
+ const LAllocation* rhs = ins->rhs();
+ uint32_t laneIndex = ins->laneIndex();
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16ReplaceLane:
+ masm.replaceLaneInt8x16(laneIndex, ToRegister(rhs), lhsDest);
+ break;
+ case wasm::SimdOp::I16x8ReplaceLane:
+ masm.replaceLaneInt16x8(laneIndex, ToRegister(rhs), lhsDest);
+ break;
+ case wasm::SimdOp::I32x4ReplaceLane:
+ masm.replaceLaneInt32x4(laneIndex, ToRegister(rhs), lhsDest);
+ break;
+ case wasm::SimdOp::F32x4ReplaceLane:
+ masm.replaceLaneFloat32x4(laneIndex, ToFloatRegister(rhs), lhsDest);
+ break;
+ case wasm::SimdOp::F64x2ReplaceLane:
+ masm.replaceLaneFloat64x2(laneIndex, ToFloatRegister(rhs), lhsDest);
+ break;
+ default:
+ MOZ_CRASH("ReplaceLane SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
+ LWasmReplaceInt64LaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_RELEASE_ASSERT(ins->simdOp() == wasm::SimdOp::I64x2ReplaceLane);
+ MOZ_ASSERT(ToFloatRegister(ins->lhs()) == ToFloatRegister(ins->output()));
+ masm.replaceLaneInt64x2(ins->laneIndex(), ToRegister64(ins->rhs()),
+ ToFloatRegister(ins->lhs()));
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Splat:
+ masm.splatX16(ToRegister(ins->src()), dest);
+ break;
+ case wasm::SimdOp::I16x8Splat:
+ masm.splatX8(ToRegister(ins->src()), dest);
+ break;
+ case wasm::SimdOp::I32x4Splat:
+ masm.splatX4(ToRegister(ins->src()), dest);
+ break;
+ case wasm::SimdOp::F32x4Splat:
+ masm.splatX4(ToFloatRegister(ins->src()), dest);
+ break;
+ case wasm::SimdOp::F64x2Splat:
+ masm.splatX2(ToFloatRegister(ins->src()), dest);
+ break;
+ default:
+ MOZ_CRASH("ScalarToSimd128 SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ Register64 src = ToRegister64(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I64x2Splat:
+ masm.splatX2(src, dest);
+ break;
+ case wasm::SimdOp::V128Load8x8S:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.widenLowInt8x16(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load8x8U:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.unsignedWidenLowInt8x16(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load16x4S:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.widenLowInt16x8(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load16x4U:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.unsignedWidenLowInt16x8(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load32x2S:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.widenLowInt32x4(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load32x2U:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.unsignedWidenLowInt32x4(dest, dest);
+ break;
+ default:
+ MOZ_CRASH("Int64ToSimd128 SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Neg:
+ masm.negInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8Neg:
+ masm.negInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtendLowI8x16S:
+ masm.widenLowInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtendHighI8x16S:
+ masm.widenHighInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtendLowI8x16U:
+ masm.unsignedWidenLowInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtendHighI8x16U:
+ masm.unsignedWidenHighInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I32x4Neg:
+ masm.negInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtendLowI16x8S:
+ masm.widenLowInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtendHighI16x8S:
+ masm.widenHighInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtendLowI16x8U:
+ masm.unsignedWidenLowInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtendHighI16x8U:
+ masm.unsignedWidenHighInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4TruncSatF32x4S:
+ masm.truncSatFloat32x4ToInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I32x4TruncSatF32x4U:
+ masm.unsignedTruncSatFloat32x4ToInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2Neg:
+ masm.negInt64x2(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtendLowI32x4S:
+ masm.widenLowInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtendHighI32x4S:
+ masm.widenHighInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtendLowI32x4U:
+ masm.unsignedWidenLowInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtendHighI32x4U:
+ masm.unsignedWidenHighInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Abs:
+ masm.absFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Neg:
+ masm.negFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Sqrt:
+ masm.sqrtFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4ConvertI32x4S:
+ masm.convertInt32x4ToFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4ConvertI32x4U:
+ masm.unsignedConvertInt32x4ToFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Abs:
+ masm.absFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Neg:
+ masm.negFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Sqrt:
+ masm.sqrtFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::V128Not:
+ masm.bitwiseNotSimd128(src, dest);
+ break;
+ case wasm::SimdOp::I8x16Abs:
+ masm.absInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8Abs:
+ masm.absInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4Abs:
+ masm.absInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2Abs:
+ masm.absInt64x2(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Ceil:
+ masm.ceilFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Floor:
+ masm.floorFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Trunc:
+ masm.truncFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Nearest:
+ masm.nearestFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Ceil:
+ masm.ceilFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Floor:
+ masm.floorFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Trunc:
+ masm.truncFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Nearest:
+ masm.nearestFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F32x4DemoteF64x2Zero:
+ masm.convertFloat64x2ToFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F64x2PromoteLowF32x4:
+ masm.convertFloat32x4ToFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2ConvertLowI32x4S:
+ masm.convertInt32x4ToFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2ConvertLowI32x4U:
+ masm.unsignedConvertInt32x4ToFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::I32x4TruncSatF64x2SZero:
+ masm.truncSatFloat64x2ToInt32x4(src, dest, ToFloatRegister(ins->temp()));
+ break;
+ case wasm::SimdOp::I32x4TruncSatF64x2UZero:
+ masm.unsignedTruncSatFloat64x2ToInt32x4(src, dest,
+ ToFloatRegister(ins->temp()));
+ break;
+ case wasm::SimdOp::I16x8ExtaddPairwiseI8x16S:
+ masm.extAddPairwiseInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtaddPairwiseI8x16U:
+ masm.unsignedExtAddPairwiseInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtaddPairwiseI16x8S:
+ masm.extAddPairwiseInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtaddPairwiseI16x8U:
+ masm.unsignedExtAddPairwiseInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I8x16Popcnt:
+ masm.popcntInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I32x4RelaxedTruncSSatF32x4:
+ masm.truncSatFloat32x4ToInt32x4Relaxed(src, dest);
+ break;
+ case wasm::SimdOp::I32x4RelaxedTruncUSatF32x4:
+ masm.unsignedTruncSatFloat32x4ToInt32x4Relaxed(src, dest);
+ break;
+ case wasm::SimdOp::I32x4RelaxedTruncSatF64x2SZero:
+ masm.truncSatFloat64x2ToInt32x4Relaxed(src, dest);
+ break;
+ case wasm::SimdOp::I32x4RelaxedTruncSatF64x2UZero:
+ masm.unsignedTruncSatFloat64x2ToInt32x4Relaxed(src, dest);
+ break;
+ default:
+ MOZ_CRASH("Unary SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ const LDefinition* dest = ins->output();
+ uint32_t imm = ins->imm();
+ FloatRegister temp = ToTempFloatRegisterOrInvalid(ins->getTemp(0));
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128AnyTrue:
+ masm.anyTrueSimd128(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I8x16AllTrue:
+ masm.allTrueInt8x16(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I16x8AllTrue:
+ masm.allTrueInt16x8(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I32x4AllTrue:
+ masm.allTrueInt32x4(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I64x2AllTrue:
+ masm.allTrueInt64x2(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I8x16Bitmask:
+ masm.bitmaskInt8x16(src, ToRegister(dest), temp);
+ break;
+ case wasm::SimdOp::I16x8Bitmask:
+ masm.bitmaskInt16x8(src, ToRegister(dest), temp);
+ break;
+ case wasm::SimdOp::I32x4Bitmask:
+ masm.bitmaskInt32x4(src, ToRegister(dest), temp);
+ break;
+ case wasm::SimdOp::I64x2Bitmask:
+ masm.bitmaskInt64x2(src, ToRegister(dest), temp);
+ break;
+ case wasm::SimdOp::I8x16ExtractLaneS:
+ masm.extractLaneInt8x16(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I8x16ExtractLaneU:
+ masm.unsignedExtractLaneInt8x16(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I16x8ExtractLaneS:
+ masm.extractLaneInt16x8(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I16x8ExtractLaneU:
+ masm.unsignedExtractLaneInt16x8(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I32x4ExtractLane:
+ masm.extractLaneInt32x4(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::F32x4ExtractLane:
+ masm.extractLaneFloat32x4(imm, src, ToFloatRegister(dest));
+ break;
+ case wasm::SimdOp::F64x2ExtractLane:
+ masm.extractLaneFloat64x2(imm, src, ToFloatRegister(dest));
+ break;
+ default:
+ MOZ_CRASH("Reduce SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReduceAndBranchSimd128(
+ LWasmReduceAndBranchSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+
+ ScratchSimd128Scope scratch(masm);
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const Register test = temps.AcquireX().asUnsized();
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128AnyTrue:
+ masm.Addp(Simd1D(scratch), Simd2D(src));
+ masm.Umov(ARMRegister(test, 64), Simd1D(scratch), 0);
+ masm.branch64(Assembler::Equal, Register64(test), Imm64(0),
+ getJumpLabelForBranch(ins->ifFalse()));
+ jumpToBlock(ins->ifTrue());
+ break;
+ case wasm::SimdOp::I8x16AllTrue:
+ case wasm::SimdOp::I16x8AllTrue:
+ case wasm::SimdOp::I32x4AllTrue:
+ case wasm::SimdOp::I64x2AllTrue: {
+ // Compare all lanes to zero.
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16AllTrue:
+ masm.Cmeq(Simd16B(scratch), Simd16B(src), 0);
+ break;
+ case wasm::SimdOp::I16x8AllTrue:
+ masm.Cmeq(Simd8H(scratch), Simd8H(src), 0);
+ break;
+ case wasm::SimdOp::I32x4AllTrue:
+ masm.Cmeq(Simd4S(scratch), Simd4S(src), 0);
+ break;
+ case wasm::SimdOp::I64x2AllTrue:
+ masm.Cmeq(Simd2D(scratch), Simd2D(src), 0);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ masm.Addp(Simd1D(scratch), Simd2D(scratch));
+ masm.Umov(ARMRegister(test, 64), Simd1D(scratch), 0);
+ masm.branch64(Assembler::NotEqual, Register64(test), Imm64(0),
+ getJumpLabelForBranch(ins->ifFalse()));
+ jumpToBlock(ins->ifTrue());
+ break;
+ }
+ default:
+ MOZ_CRASH("Reduce-and-branch SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReduceSimd128ToInt64(
+ LWasmReduceSimd128ToInt64* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ Register64 dest = ToOutRegister64(ins);
+ uint32_t imm = ins->imm();
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I64x2ExtractLane:
+ masm.extractLaneInt64x2(imm, src, dest);
+ break;
+ default:
+ MOZ_CRASH("Reduce SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+static inline wasm::MemoryAccessDesc DeriveMemoryAccessDesc(
+ const wasm::MemoryAccessDesc& access, Scalar::Type type) {
+ return wasm::MemoryAccessDesc(type, access.align(), access.offset(),
+ access.trapOffset());
+}
+
+void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ // Forward loading to wasmLoad, and use replaceLane after that.
+ const MWasmLoadLaneSimd128* mir = ins->mir();
+ Register temp = ToRegister(ins->temp());
+ FloatRegister src = ToFloatRegister(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+ // replaceLane takes an lhsDest argument.
+ masm.moveSimd128(src, dest);
+ switch (ins->laneSize()) {
+ case 1: {
+ masm.wasmLoad(DeriveMemoryAccessDesc(mir->access(), Scalar::Int8),
+ HeapReg, ToRegister(ins->ptr()), AnyRegister(temp));
+ masm.replaceLaneInt8x16(ins->laneIndex(), temp, dest);
+ break;
+ }
+ case 2: {
+ masm.wasmLoad(DeriveMemoryAccessDesc(mir->access(), Scalar::Int16),
+ HeapReg, ToRegister(ins->ptr()), AnyRegister(temp));
+ masm.replaceLaneInt16x8(ins->laneIndex(), temp, dest);
+ break;
+ }
+ case 4: {
+ masm.wasmLoad(DeriveMemoryAccessDesc(mir->access(), Scalar::Int32),
+ HeapReg, ToRegister(ins->ptr()), AnyRegister(temp));
+ masm.replaceLaneInt32x4(ins->laneIndex(), temp, dest);
+ break;
+ }
+ case 8: {
+ masm.wasmLoadI64(DeriveMemoryAccessDesc(mir->access(), Scalar::Int64),
+ HeapReg, ToRegister(ins->ptr()), Register64(temp));
+ masm.replaceLaneInt64x2(ins->laneIndex(), Register64(temp), dest);
+ break;
+ }
+ default:
+ MOZ_CRASH("Unsupported load lane size");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ // Forward storing to wasmStore for the result of extractLane.
+ const MWasmStoreLaneSimd128* mir = ins->mir();
+ Register temp = ToRegister(ins->temp());
+ FloatRegister src = ToFloatRegister(ins->src());
+ switch (ins->laneSize()) {
+ case 1: {
+ masm.extractLaneInt8x16(ins->laneIndex(), src, temp);
+ masm.wasmStore(DeriveMemoryAccessDesc(mir->access(), Scalar::Int8),
+ AnyRegister(temp), HeapReg, ToRegister(ins->ptr()));
+ break;
+ }
+ case 2: {
+ masm.extractLaneInt16x8(ins->laneIndex(), src, temp);
+ masm.wasmStore(DeriveMemoryAccessDesc(mir->access(), Scalar::Int16),
+ AnyRegister(temp), HeapReg, ToRegister(ins->ptr()));
+ break;
+ }
+ case 4: {
+ masm.extractLaneInt32x4(ins->laneIndex(), src, temp);
+ masm.wasmStore(DeriveMemoryAccessDesc(mir->access(), Scalar::Int32),
+ AnyRegister(temp), HeapReg, ToRegister(ins->ptr()));
+ break;
+ }
+ case 8: {
+ masm.extractLaneInt64x2(ins->laneIndex(), src, Register64(temp));
+ masm.wasmStoreI64(DeriveMemoryAccessDesc(mir->access(), Scalar::Int64),
+ Register64(temp), HeapReg, ToRegister(ins->ptr()));
+ break;
+ }
+ default:
+ MOZ_CRASH("Unsupported store lane size");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
diff --git a/js/src/jit/arm64/CodeGenerator-arm64.h b/js/src/jit/arm64/CodeGenerator-arm64.h
new file mode 100644
index 0000000000..43cd24fddf
--- /dev/null
+++ b/js/src/jit/arm64/CodeGenerator-arm64.h
@@ -0,0 +1,135 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_CodeGenerator_arm64_h
+#define jit_arm64_CodeGenerator_arm64_h
+
+#include "jit/arm64/Assembler-arm64.h"
+#include "jit/shared/CodeGenerator-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorARM64;
+class OutOfLineBailout;
+class OutOfLineTableSwitch;
+
+using OutOfLineWasmTruncateCheck =
+ OutOfLineWasmTruncateCheckBase<CodeGeneratorARM64>;
+
+class CodeGeneratorARM64 : public CodeGeneratorShared {
+ friend class MoveResolverARM64;
+
+ protected:
+ CodeGeneratorARM64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
+
+ NonAssertingLabel deoptLabel_;
+
+ MoveOperand toMoveOperand(const LAllocation a) const;
+
+ void bailoutIf(Assembler::Condition condition, LSnapshot* snapshot);
+ void bailoutFrom(Label* label, LSnapshot* snapshot);
+ void bailout(LSnapshot* snapshot);
+
+ template <typename T1, typename T2>
+ void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.cmpPtr(lhs, rhs);
+ return bailoutIf(c, snapshot);
+ }
+ void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs,
+ LSnapshot* snapshot) {
+ masm.testPtr(lhs, rhs);
+ return bailoutIf(c, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.cmp32(lhs, rhs);
+ return bailoutIf(c, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutTest32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.test32(lhs, rhs);
+ return bailoutIf(c, snapshot);
+ }
+ void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) {
+ masm.test32(reg, Imm32(0xFF));
+ return bailoutIf(Assembler::Zero, snapshot);
+ }
+
+ bool generateOutOfLineCode();
+
+ // Emits a branch that directs control flow to the true block if |cond| is
+ // true, and the false block if |cond| is false.
+ void emitBranch(Assembler::Condition cond, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse);
+
+ void testNullEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ cond = masm.testNull(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+ void testUndefinedEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ cond = masm.testUndefined(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+ void testObjectEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ cond = masm.testObject(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+ void testZeroEmitBranch(Assembler::Condition cond, Register reg,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ masm.cmpPtr(reg, ImmWord(0));
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+
+ void emitTableSwitchDispatch(MTableSwitch* mir, Register index,
+ Register base);
+
+ void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitSimpleBinaryI64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* lir, JSOp op);
+
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ void generateInvalidateEpilogue();
+
+ public:
+ void visitOutOfLineBailout(OutOfLineBailout* ool);
+ void visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool);
+ void visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool);
+};
+
+typedef CodeGeneratorARM64 CodeGeneratorSpecific;
+
+// An out-of-line bailout thunk.
+class OutOfLineBailout : public OutOfLineCodeBase<CodeGeneratorARM64> {
+ protected: // Silence Clang warning.
+ LSnapshot* snapshot_;
+
+ public:
+ explicit OutOfLineBailout(LSnapshot* snapshot) : snapshot_(snapshot) {}
+
+ void accept(CodeGeneratorARM64* codegen) override;
+
+ LSnapshot* snapshot() const { return snapshot_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm64_CodeGenerator_arm64_h */
diff --git a/js/src/jit/arm64/LIR-arm64.h b/js/src/jit/arm64/LIR-arm64.h
new file mode 100644
index 0000000000..d825209b1e
--- /dev/null
+++ b/js/src/jit/arm64/LIR-arm64.h
@@ -0,0 +1,373 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_LIR_arm64_h
+#define jit_arm64_LIR_arm64_h
+
+namespace js {
+namespace jit {
+
+class LUnboxBase : public LInstructionHelper<1, 1, 0> {
+ public:
+ LUnboxBase(LNode::Opcode opcode, const LAllocation& input)
+ : LInstructionHelper(opcode) {
+ setOperand(0, input);
+ }
+
+ static const size_t Input = 0;
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+};
+
+class LUnbox : public LUnboxBase {
+ public:
+ LIR_HEADER(Unbox);
+
+ explicit LUnbox(const LAllocation& input) : LUnboxBase(classOpcode, input) {}
+
+ const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LUnboxBase {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint);
+
+ LUnboxFloatingPoint(const LAllocation& input, MIRType type)
+ : LUnboxBase(classOpcode, input), type_(type) {}
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToDouble)
+
+ explicit LWasmUint32ToDouble(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToFloat32)
+
+ explicit LWasmUint32ToFloat32(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+class LDivI : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivI);
+
+ LDivI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LDivPowTwoI : public LInstructionHelper<1, 1, 0> {
+ const int32_t shift_;
+ const bool negativeDivisor_;
+
+ public:
+ LIR_HEADER(DivPowTwoI)
+
+ LDivPowTwoI(const LAllocation& lhs, int32_t shift, bool negativeDivisor)
+ : LInstructionHelper(classOpcode),
+ shift_(shift),
+ negativeDivisor_(negativeDivisor) {
+ setOperand(0, lhs);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+
+ int32_t shift() { return shift_; }
+ bool negativeDivisor() { return negativeDivisor_; }
+
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LDivConstantI : public LInstructionHelper<1, 1, 1> {
+ const int32_t denominator_;
+
+ public:
+ LIR_HEADER(DivConstantI)
+
+ LDivConstantI(const LAllocation& lhs, int32_t denominator,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode), denominator_(denominator) {
+ setOperand(0, lhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+ const LDefinition* temp() { return getTemp(0); }
+ int32_t denominator() const { return denominator_; }
+ MDiv* mir() const { return mir_->toDiv(); }
+ bool canBeNegativeDividend() const { return mir()->canBeNegativeDividend(); }
+};
+
+class LUDivConstantI : public LInstructionHelper<1, 1, 1> {
+ const int32_t denominator_;
+
+ public:
+ LIR_HEADER(UDivConstantI)
+
+ LUDivConstantI(const LAllocation& lhs, int32_t denominator,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode), denominator_(denominator) {
+ setOperand(0, lhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+ const LDefinition* temp() { return getTemp(0); }
+ int32_t denominator() const { return denominator_; }
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LModI : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(ModI);
+
+ LModI(const LAllocation& lhs, const LAllocation& rhs)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ }
+
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModPowTwoI : public LInstructionHelper<1, 1, 0> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModPowTwoI);
+ int32_t shift() { return shift_; }
+
+ LModPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ }
+
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModMaskI : public LInstructionHelper<1, 1, 2> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModMaskI);
+
+ LModMaskI(const LAllocation& lhs, const LDefinition& temp1,
+ const LDefinition& temp2, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ int32_t shift() const { return shift_; }
+
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitch : public LInstructionHelper<0, 1, 2> {
+ public:
+ LIR_HEADER(TableSwitch);
+
+ LTableSwitch(const LAllocation& in, const LDefinition& inputCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setTemp(0, inputCopy);
+ setTemp(1, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+
+ const LAllocation* index() { return getOperand(0); }
+ const LDefinition* tempInt() { return getTemp(0); }
+ // This is added to share the same CodeGenerator prefixes.
+ const LDefinition* tempPointer() { return getTemp(1); }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitchV : public LInstructionHelper<0, BOX_PIECES, 3> {
+ public:
+ LIR_HEADER(TableSwitchV);
+
+ LTableSwitchV(const LBoxAllocation& input, const LDefinition& inputCopy,
+ const LDefinition& floatCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setBoxOperand(InputValue, input);
+ setTemp(0, inputCopy);
+ setTemp(1, floatCopy);
+ setTemp(2, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+
+ static const size_t InputValue = 0;
+
+ const LDefinition* tempInt() { return getTemp(0); }
+ const LDefinition* tempFloat() { return getTemp(1); }
+ const LDefinition* tempPointer() { return getTemp(2); }
+};
+
+class LMulI : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(MulI);
+
+ LMulI() : LBinaryMath(classOpcode) {}
+
+ MMul* mir() { return mir_->toMul(); }
+};
+
+class LUDiv : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(UDiv);
+
+ LUDiv(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& remainder)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, remainder);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+
+ MDiv* mir() { return mir_->toDiv(); }
+};
+
+class LUMod : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(UMod);
+
+ LUMod(const LAllocation& lhs, const LAllocation& rhs)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ }
+
+ MMod* mir() { return mir_->toMod(); }
+};
+
+class LInt64ToFloatingPoint : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(Int64ToFloatingPoint);
+
+ explicit LInt64ToFloatingPoint(const LInt64Allocation& in)
+ : LInstructionHelper(classOpcode) {
+ setInt64Operand(0, in);
+ }
+
+ MInt64ToFloatingPoint* mir() const { return mir_->toInt64ToFloatingPoint(); }
+};
+
+class LWasmTruncateToInt64 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ explicit LWasmTruncateToInt64(const LAllocation& in)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ }
+
+ MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); }
+};
+
+class LDivOrModI64 : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LUDivOrModI64 : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(UDivOrModI64);
+
+ LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ }
+
+ const char* extraName() const {
+ return mir()->isTruncated() ? "Truncated" : nullptr;
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm64_LIR_arm64_h */
diff --git a/js/src/jit/arm64/Lowering-arm64.cpp b/js/src/jit/arm64/Lowering-arm64.cpp
new file mode 100644
index 0000000000..9b8f0a1c71
--- /dev/null
+++ b/js/src/jit/arm64/Lowering-arm64.cpp
@@ -0,0 +1,1444 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm64/Lowering-arm64.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/arm64/Assembler-arm64.h"
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::FloorLog2;
+
+LBoxAllocation LIRGeneratorARM64::useBoxFixed(MDefinition* mir, Register reg1,
+ Register, bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart));
+}
+
+LAllocation LIRGeneratorARM64::useByteOpRegister(MDefinition* mir) {
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorARM64::useByteOpRegisterAtStart(MDefinition* mir) {
+ return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorARM64::useByteOpRegisterOrNonDoubleConstant(
+ MDefinition* mir) {
+ return useRegisterOrNonDoubleConstant(mir);
+}
+
+LDefinition LIRGeneratorARM64::tempByteOpRegister() { return temp(); }
+
+LDefinition LIRGeneratorARM64::tempToUnbox() { return temp(); }
+
+void LIRGenerator::visitBox(MBox* box) {
+ MDefinition* opd = box->getOperand(0);
+
+ // If the operand is a constant, emit near its uses.
+ if (opd->isConstant() && box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (opd->isConstant()) {
+ define(new (alloc()) LValue(opd->toConstant()->toJSValue()), box,
+ LDefinition(LDefinition::BOX));
+ } else {
+ LBox* ins = new (alloc()) LBox(useRegister(opd), opd->type());
+ define(ins, box, LDefinition(LDefinition::BOX));
+ }
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+ MDefinition* box = unbox->getOperand(0);
+ MOZ_ASSERT(box->type() == MIRType::Value);
+
+ LUnboxBase* lir;
+ if (IsFloatingPointType(unbox->type())) {
+ lir = new (alloc())
+ LUnboxFloatingPoint(useRegisterAtStart(box), unbox->type());
+ } else if (unbox->fallible()) {
+ // If the unbox is fallible, load the Value in a register first to
+ // avoid multiple loads.
+ lir = new (alloc()) LUnbox(useRegisterAtStart(box));
+ } else {
+ // FIXME: It should be possible to useAtStart() here, but the DEBUG
+ // code in CodeGenerator::visitUnbox() needs to handle non-Register
+ // cases. ARM64 doesn't have an Operand type.
+ lir = new (alloc()) LUnbox(useRegisterAtStart(box));
+ }
+
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+
+ define(lir, unbox);
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new (alloc()) LReturn(isGenerator);
+ ins->setOperand(0, useFixed(opd, JSReturnReg));
+ add(ins);
+}
+
+// x = !y
+void LIRGeneratorARM64::lowerForALU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(
+ 0, ins->snapshot() ? useRegister(input) : useRegisterAtStart(input));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+// z = x+y
+void LIRGeneratorARM64::lowerForALU(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0,
+ ins->snapshot() ? useRegister(lhs) : useRegisterAtStart(lhs));
+ ins->setOperand(1, ins->snapshot() ? useRegisterOrConstant(rhs)
+ : useRegisterOrConstantAtStart(rhs));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+void LIRGeneratorARM64::lowerForFPU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(0, useRegisterAtStart(input));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template <size_t Temps>
+void LIRGeneratorARM64::lowerForFPU(LInstructionHelper<1, 2, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegisterAtStart(lhs));
+ ins->setOperand(1, useRegisterAtStart(rhs));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template void LIRGeneratorARM64::lowerForFPU(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+template void LIRGeneratorARM64::lowerForFPU(LInstructionHelper<1, 2, 1>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+
+void LIRGeneratorARM64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+ MDefinition* input) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+ defineInt64(ins, mir);
+}
+
+// These all currently have codegen that depends on reuse but only because the
+// masm API depends on that. We need new three-address masm APIs, for both
+// constant and variable rhs.
+//
+// MAdd => LAddI64
+// MSub => LSubI64
+// MBitAnd, MBitOr, MBitXor => LBitOpI64
+void LIRGeneratorARM64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, useInt64RegisterOrConstantAtStart(rhs));
+ defineInt64(ins, mir);
+}
+
+void LIRGeneratorARM64::lowerForMulInt64(LMulI64* ins, MMul* mir,
+ MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(LMulI64::Lhs, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(LMulI64::Rhs, useInt64RegisterOrConstantAtStart(rhs));
+ defineInt64(ins, mir);
+}
+
+template <size_t Temps>
+void LIRGeneratorARM64::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+
+ static_assert(LShiftI64::Rhs == INT64_PIECES,
+ "Assume Rhs is located at INT64_PIECES.");
+ static_assert(LRotateI64::Count == INT64_PIECES,
+ "Assume Count is located at INT64_PIECES.");
+
+ ins->setOperand(INT64_PIECES, useRegisterOrConstantAtStart(rhs));
+ defineInt64(ins, mir);
+}
+
+template void LIRGeneratorARM64::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+template void LIRGeneratorARM64::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 1>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+void LIRGeneratorARM64::lowerForCompareI64AndBranch(MTest* mir, MCompare* comp,
+ JSOp op, MDefinition* left,
+ MDefinition* right,
+ MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ auto* lir = new (alloc())
+ LCompareI64AndBranch(comp, op, useInt64Register(left),
+ useInt64RegisterOrConstant(right), ifTrue, ifFalse);
+ add(lir, mir);
+}
+
+void LIRGeneratorARM64::lowerForBitAndAndBranch(LBitAndAndBranch* baab,
+ MInstruction* mir,
+ MDefinition* lhs,
+ MDefinition* rhs) {
+ baab->setOperand(0, useRegisterAtStart(lhs));
+ baab->setOperand(1, useRegisterOrConstantAtStart(rhs));
+ add(baab, mir);
+}
+
+void LIRGeneratorARM64::lowerWasmBuiltinTruncateToInt32(
+ MWasmBuiltinTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ if (opd->type() == MIRType::Double) {
+ define(new (alloc()) LWasmBuiltinTruncateDToInt32(
+ useRegister(opd), useFixed(ins->instance(), InstanceReg),
+ LDefinition::BogusTemp()),
+ ins);
+ return;
+ }
+
+ define(new (alloc()) LWasmBuiltinTruncateFToInt32(
+ useRegister(opd), useFixed(ins->instance(), InstanceReg),
+ LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGeneratorARM64::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void LIRGeneratorARM64::lowerForShift(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegisterOrConstant(rhs));
+ define(ins, mir);
+}
+
+void LIRGeneratorARM64::lowerDivI(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDiv(div);
+ return;
+ }
+
+ if (div->rhs()->isConstant()) {
+ LAllocation lhs = useRegister(div->lhs());
+ int32_t rhs = div->rhs()->toConstant()->toInt32();
+ int32_t shift = mozilla::FloorLog2(mozilla::Abs(rhs));
+
+ if (rhs != 0 && uint32_t(1) << shift == mozilla::Abs(rhs)) {
+ LDivPowTwoI* lir = new (alloc()) LDivPowTwoI(lhs, shift, rhs < 0);
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+ if (rhs != 0) {
+ LDivConstantI* lir = new (alloc()) LDivConstantI(lhs, rhs, temp());
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+ }
+
+ LDivI* lir = new (alloc())
+ LDivI(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+}
+
+void LIRGeneratorARM64::lowerNegI(MInstruction* ins, MDefinition* input) {
+ define(new (alloc()) LNegI(useRegisterAtStart(input)), ins);
+}
+
+void LIRGeneratorARM64::lowerNegI64(MInstruction* ins, MDefinition* input) {
+ defineInt64(new (alloc()) LNegI64(useInt64RegisterAtStart(input)), ins);
+}
+
+void LIRGeneratorARM64::lowerMulI(MMul* mul, MDefinition* lhs,
+ MDefinition* rhs) {
+ LMulI* lir = new (alloc()) LMulI;
+ if (mul->fallible()) {
+ assignSnapshot(lir, mul->bailoutKind());
+ }
+ lowerForALU(lir, mul, lhs, rhs);
+}
+
+void LIRGeneratorARM64::lowerModI(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUMod(mod);
+ return;
+ }
+
+ if (mod->rhs()->isConstant()) {
+ int32_t rhs = mod->rhs()->toConstant()->toInt32();
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LModPowTwoI* lir =
+ new (alloc()) LModPowTwoI(useRegister(mod->lhs()), shift);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ } else if (shift < 31 && (1 << (shift + 1)) - 1 == rhs) {
+ LModMaskI* lir = new (alloc())
+ LModMaskI(useRegister(mod->lhs()), temp(), temp(), shift + 1);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ }
+ }
+
+ LModI* lir =
+ new (alloc()) LModI(useRegister(mod->lhs()), useRegister(mod->rhs()));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+}
+
+void LIRGeneratorARM64::lowerDivI64(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDivI64(div);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc())
+ LDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()));
+ defineInt64(lir, div);
+}
+
+void LIRGeneratorARM64::lowerUDivI64(MDiv* div) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()));
+ defineInt64(lir, div);
+}
+
+void LIRGeneratorARM64::lowerUModI64(MMod* mod) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()));
+ defineInt64(lir, mod);
+}
+
+void LIRGeneratorARM64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ MOZ_CRASH("We don't use runtime div for this architecture");
+}
+
+void LIRGeneratorARM64::lowerModI64(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUModI64(mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc())
+ LDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()));
+ defineInt64(lir, mod);
+}
+
+void LIRGeneratorARM64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ MOZ_CRASH("We don't use runtime mod for this architecture");
+}
+
+void LIRGenerator::visitPowHalf(MPowHalf* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Double);
+ LPowHalfD* lir = new (alloc()) LPowHalfD(useRegister(input));
+ define(lir, ins);
+}
+
+void LIRGeneratorARM64::lowerWasmSelectI(MWasmSelect* select) {
+ if (select->type() == MIRType::Simd128) {
+ LAllocation t = useRegisterAtStart(select->trueExpr());
+ LAllocation f = useRegister(select->falseExpr());
+ LAllocation c = useRegister(select->condExpr());
+ auto* lir = new (alloc()) LWasmSelect(t, f, c);
+ defineReuseInput(lir, select, LWasmSelect::TrueExprIndex);
+ } else {
+ LAllocation t = useRegisterAtStart(select->trueExpr());
+ LAllocation f = useRegisterAtStart(select->falseExpr());
+ LAllocation c = useRegisterAtStart(select->condExpr());
+ define(new (alloc()) LWasmSelect(t, f, c), select);
+ }
+}
+
+void LIRGeneratorARM64::lowerWasmSelectI64(MWasmSelect* select) {
+ LInt64Allocation t = useInt64RegisterAtStart(select->trueExpr());
+ LInt64Allocation f = useInt64RegisterAtStart(select->falseExpr());
+ LAllocation c = useRegisterAtStart(select->condExpr());
+ defineInt64(new (alloc()) LWasmSelectI64(t, f, c), select);
+}
+
+// On arm64 we specialize the cases: compare is {{U,}Int32, {U,}Int64},
+// Float32, Double}, and select is {{U,}Int32, {U,}Int64}, Float32, Double},
+// independently.
+bool LIRGeneratorARM64::canSpecializeWasmCompareAndSelect(
+ MCompare::CompareType compTy, MIRType insTy) {
+ return (insTy == MIRType::Int32 || insTy == MIRType::Int64 ||
+ insTy == MIRType::Float32 || insTy == MIRType::Double) &&
+ (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32 ||
+ compTy == MCompare::Compare_Int64 ||
+ compTy == MCompare::Compare_UInt64 ||
+ compTy == MCompare::Compare_Float32 ||
+ compTy == MCompare::Compare_Double);
+}
+
+void LIRGeneratorARM64::lowerWasmCompareAndSelect(MWasmSelect* ins,
+ MDefinition* lhs,
+ MDefinition* rhs,
+ MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+ LAllocation rhsAlloc;
+ if (compTy == MCompare::Compare_Float32 ||
+ compTy == MCompare::Compare_Double) {
+ rhsAlloc = useRegisterAtStart(rhs);
+ } else if (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32 ||
+ compTy == MCompare::Compare_Int64 ||
+ compTy == MCompare::Compare_UInt64) {
+ rhsAlloc = useRegisterOrConstantAtStart(rhs);
+ } else {
+ MOZ_CRASH("Unexpected type");
+ }
+ auto* lir = new (alloc())
+ LWasmCompareAndSelect(useRegisterAtStart(lhs), rhsAlloc, compTy, jsop,
+ useRegisterAtStart(ins->trueExpr()),
+ useRegisterAtStart(ins->falseExpr()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAbs(MAbs* ins) {
+ define(allocateAbs(ins, useRegisterAtStart(ins->input())), ins);
+}
+
+LTableSwitch* LIRGeneratorARM64::newLTableSwitch(const LAllocation& in,
+ const LDefinition& inputCopy,
+ MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitch(in, inputCopy, temp(), tableswitch);
+}
+
+LTableSwitchV* LIRGeneratorARM64::newLTableSwitchV(MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitchV(useBox(tableswitch->getOperand(0)), temp(),
+ tempDouble(), temp(), tableswitch);
+}
+
+void LIRGeneratorARM64::lowerUrshD(MUrsh* mir) {
+ MDefinition* lhs = mir->lhs();
+ MDefinition* rhs = mir->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+
+ LUrshD* lir = new (alloc())
+ LUrshD(useRegister(lhs), useRegisterOrConstant(rhs), temp());
+ define(lir, mir);
+}
+
+void LIRGeneratorARM64::lowerPowOfTwoI(MPow* mir) {
+ int32_t base = mir->input()->toConstant()->toInt32();
+ MDefinition* power = mir->power();
+
+ auto* lir = new (alloc()) LPowOfTwoI(useRegister(power), base);
+ assignSnapshot(lir, mir->bailoutKind());
+ define(lir, mir);
+}
+
+void LIRGeneratorARM64::lowerBigIntLsh(MBigIntLsh* ins) {
+ auto* lir = new (alloc()) LBigIntLsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorARM64::lowerBigIntRsh(MBigIntRsh* ins) {
+ auto* lir = new (alloc()) LBigIntRsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorARM64::lowerBigIntDiv(MBigIntDiv* ins) {
+ auto* lir = new (alloc()) LBigIntDiv(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorARM64::lowerBigIntMod(MBigIntMod* ins) {
+ auto* lir = new (alloc()) LBigIntMod(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+#ifdef ENABLE_WASM_SIMD
+
+bool LIRGeneratorARM64::canFoldReduceSimd128AndBranch(wasm::SimdOp op) {
+ switch (op) {
+ case wasm::SimdOp::V128AnyTrue:
+ case wasm::SimdOp::I8x16AllTrue:
+ case wasm::SimdOp::I16x8AllTrue:
+ case wasm::SimdOp::I32x4AllTrue:
+ case wasm::SimdOp::I64x2AllTrue:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool LIRGeneratorARM64::canEmitWasmReduceSimd128AtUses(
+ MWasmReduceSimd128* ins) {
+ if (!ins->canEmitAtUses()) {
+ return false;
+ }
+ // Only specific ops generating int32.
+ if (ins->type() != MIRType::Int32) {
+ return false;
+ }
+ if (!canFoldReduceSimd128AndBranch(ins->simdOp())) {
+ return false;
+ }
+ // If never used then defer (it will be removed).
+ MUseIterator iter(ins->usesBegin());
+ if (iter == ins->usesEnd()) {
+ return true;
+ }
+ // We require an MTest consumer.
+ MNode* node = iter->consumer();
+ if (!node->isDefinition() || !node->toDefinition()->isTest()) {
+ return false;
+ }
+ // Defer only if there's only one use.
+ iter++;
+ return iter == ins->usesEnd();
+}
+
+#endif
+
+void LIRGenerator::visitWasmNeg(MWasmNeg* ins) {
+ switch (ins->type()) {
+ case MIRType::Int32:
+ define(new (alloc()) LNegI(useRegisterAtStart(ins->input())), ins);
+ break;
+ case MIRType::Float32:
+ define(new (alloc()) LNegF(useRegisterAtStart(ins->input())), ins);
+ break;
+ case MIRType::Double:
+ define(new (alloc()) LNegD(useRegisterAtStart(ins->input())), ins);
+ break;
+ default:
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void LIRGeneratorARM64::lowerUDiv(MDiv* div) {
+ LAllocation lhs = useRegister(div->lhs());
+ if (div->rhs()->isConstant()) {
+ int32_t rhs = div->rhs()->toConstant()->toInt32();
+ int32_t shift = mozilla::FloorLog2(mozilla::Abs(rhs));
+
+ if (rhs != 0 && uint32_t(1) << shift == mozilla::Abs(rhs)) {
+ LDivPowTwoI* lir = new (alloc()) LDivPowTwoI(lhs, shift, false);
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+
+ LUDivConstantI* lir = new (alloc()) LUDivConstantI(lhs, rhs, temp());
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+
+ // Generate UDiv
+ LAllocation rhs = useRegister(div->rhs());
+ LDefinition remainder = LDefinition::BogusTemp();
+ if (!div->canTruncateRemainder()) {
+ remainder = temp();
+ }
+
+ LUDiv* lir = new (alloc()) LUDiv(lhs, rhs, remainder);
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+}
+
+void LIRGeneratorARM64::lowerUMod(MMod* mod) {
+ LUMod* lir = new (alloc())
+ LUMod(useRegister(mod->getOperand(0)), useRegister(mod->getOperand(1)));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+}
+
+void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToDouble* lir =
+ new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToFloat32* lir =
+ new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT_IF(ins->needsBoundsCheck(),
+ boundsCheckLimit->type() == MIRType::Int32);
+
+ LAllocation baseAlloc = useRegisterAtStart(base);
+
+ LAllocation limitAlloc = ins->needsBoundsCheck()
+ ? useRegisterAtStart(boundsCheckLimit)
+ : LAllocation();
+
+ // We have no memory-base value, meaning that HeapReg is to be used as the
+ // memory base. This follows from the definition of
+ // FunctionCompiler::maybeLoadMemoryBase() in WasmIonCompile.cpp.
+ MOZ_ASSERT(!ins->hasMemoryBase());
+ auto* lir =
+ new (alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc, LAllocation());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT_IF(ins->needsBoundsCheck(),
+ boundsCheckLimit->type() == MIRType::Int32);
+
+ LAllocation baseAlloc = useRegisterAtStart(base);
+
+ LAllocation limitAlloc = ins->needsBoundsCheck()
+ ? useRegisterAtStart(boundsCheckLimit)
+ : LAllocation();
+
+ // See comment in LIRGenerator::visitAsmJSStoreHeap just above.
+ MOZ_ASSERT(!ins->hasMemoryBase());
+ add(new (alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()),
+ limitAlloc, LAllocation()),
+ ins);
+}
+
+void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ // Note, the access type may be Int64 here.
+
+ LWasmCompareExchangeHeap* lir = new (alloc())
+ LWasmCompareExchangeHeap(useRegister(base), useRegister(ins->oldValue()),
+ useRegister(ins->newValue()));
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ // Note, the access type may be Int64 here.
+
+ LWasmAtomicExchangeHeap* lir = new (alloc())
+ LWasmAtomicExchangeHeap(useRegister(base), useRegister(ins->value()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ // Note, the access type may be Int64 here.
+
+ if (!ins->hasUses()) {
+ LWasmAtomicBinopHeapForEffect* lir =
+ new (alloc()) LWasmAtomicBinopHeapForEffect(useRegister(base),
+ useRegister(ins->value()),
+ /* flagTemp= */ temp());
+ add(lir, ins);
+ return;
+ }
+
+ LWasmAtomicBinopHeap* lir = new (alloc())
+ LWasmAtomicBinopHeap(useRegister(base), useRegister(ins->value()),
+ /* temp= */ LDefinition::BogusTemp(),
+ /* flagTemp= */ temp());
+ define(lir, ins);
+}
+
+void LIRGeneratorARM64::lowerTruncateDToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double);
+ define(new (alloc())
+ LTruncateDToInt32(useRegister(opd), LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGeneratorARM64::lowerTruncateFToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Float32);
+ define(new (alloc())
+ LTruncateFToInt32(useRegister(opd), LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64();
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We can omit allocating the result BigInt.
+
+ if (ins->isForEffect()) {
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+ elements, index, value, temp1, temp2);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+
+ auto* lir = new (alloc())
+ LAtomicTypedArrayElementBinop64(elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ if (ins->isForEffect()) {
+ auto* lir = new (alloc())
+ LAtomicTypedArrayElementBinopForEffect(elements, index, value, temp());
+ add(lir, ins);
+ return;
+ }
+
+ LDefinition tempDef1 = temp();
+ LDefinition tempDef2 = LDefinition::BogusTemp();
+ if (ins->arrayType() == Scalar::Uint32) {
+ tempDef2 = temp();
+ }
+
+ LAtomicTypedArrayElementBinop* lir = new (alloc())
+ LAtomicTypedArrayElementBinop(elements, index, value, tempDef1, tempDef2);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation newval = useRegister(ins->newval());
+ const LAllocation oldval = useRegister(ins->oldval());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64();
+
+ auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+ elements, index, oldval, newval, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ // If the target is an FPReg then we need a temporary at the CodeGenerator
+ // level for creating the result.
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+ if (ins->arrayType() == Scalar::Uint32) {
+ outTemp = temp();
+ }
+
+ LCompareExchangeTypedArrayElement* lir =
+ new (alloc()) LCompareExchangeTypedArrayElement(elements, index, oldval,
+ newval, outTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LDefinition temp2 = temp();
+
+ auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64(
+ elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32);
+
+ LDefinition tempDef = LDefinition::BogusTemp();
+ if (ins->arrayType() == Scalar::Uint32) {
+ tempDef = temp();
+ }
+
+ LAtomicExchangeTypedArrayElement* lir = new (alloc())
+ LAtomicExchangeTypedArrayElement(elements, index, value, tempDef);
+
+ define(lir, ins);
+}
+
+void LIRGeneratorARM64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+ auto* lir = new (alloc()) LAtomicLoad64(elements, index, temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorARM64::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value = useRegister(ins->value());
+
+ add(new (alloc()) LAtomicStore64(elements, index, value, tempInt64()), ins);
+}
+
+void LIRGenerator::visitSubstr(MSubstr* ins) {
+ LSubstr* lir = new (alloc())
+ LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
+ useRegister(ins->length()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd)), ins);
+}
+
+void LIRGeneratorARM64::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use WasmBuiltinTruncateToInt64 for arm64");
+}
+
+void LIRGeneratorARM64::lowerBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) {
+ auto* lir = new (alloc()) LWasmHeapBase(LAllocation());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
+ MDefinition* base = ins->base();
+ // 'base' is a GPR but may be of either type. If it is 32-bit it is
+ // zero-extended and can act as 64-bit.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ LAllocation ptr = useRegisterOrConstantAtStart(base);
+
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new (alloc()) LWasmLoadI64(ptr);
+ defineInt64(lir, ins);
+ } else {
+ auto* lir = new (alloc()) LWasmLoad(ptr);
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitWasmStore(MWasmStore* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ MDefinition* value = ins->value();
+
+ if (ins->access().type() == Scalar::Int64) {
+ LAllocation baseAlloc = useRegisterOrConstantAtStart(base);
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+ auto* lir = new (alloc()) LWasmStoreI64(baseAlloc, valueAlloc);
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation baseAlloc = useRegisterOrConstantAtStart(base);
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Int64);
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+ define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd)), ins);
+}
+
+void LIRGenerator::visitCopySign(MCopySign* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(IsFloatingPointType(lhs->type()));
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(lhs->type() == ins->type());
+
+ LInstructionHelper<1, 2, 2>* lir;
+ if (lhs->type() == MIRType::Double) {
+ lir = new (alloc()) LCopySignD();
+ } else {
+ lir = new (alloc()) LCopySignF();
+ }
+
+ lir->setOperand(0, useRegisterAtStart(lhs));
+ lir->setOperand(1, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useRegister(rhs)
+ : useRegisterAtStart(rhs));
+ // The copySignDouble and copySignFloat32 are optimized for lhs == output.
+ // It also prevents rhs == output when lhs != output, avoids clobbering.
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
+ defineInt64(
+ new (alloc()) LExtendInt32ToInt64(useRegisterAtStart(ins->input())), ins);
+}
+
+void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
+ defineInt64(new (alloc())
+ LSignExtendInt64(useInt64RegisterAtStart(ins->input())),
+ ins);
+}
+
+void LIRGenerator::visitWasmTernarySimd128(MWasmTernarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->v0()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->v1()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->v2()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128Bitselect: {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()), useRegister(ins->v1()),
+ useRegisterAtStart(ins->v2()));
+ // On ARM64, control register is used as output at machine instruction.
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V2);
+ break;
+ }
+ case wasm::SimdOp::F32x4RelaxedFma:
+ case wasm::SimdOp::F32x4RelaxedFnma:
+ case wasm::SimdOp::F64x2RelaxedFma:
+ case wasm::SimdOp::F64x2RelaxedFnma: {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()), useRegister(ins->v1()),
+ useRegisterAtStart(ins->v2()));
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V2);
+ break;
+ }
+ case wasm::SimdOp::I32x4DotI8x16I7x16AddS: {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()), useRegister(ins->v1()),
+ useRegisterAtStart(ins->v2()), tempSimd128());
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V2);
+ break;
+ }
+ case wasm::SimdOp::I8x16RelaxedLaneSelect:
+ case wasm::SimdOp::I16x8RelaxedLaneSelect:
+ case wasm::SimdOp::I32x4RelaxedLaneSelect:
+ case wasm::SimdOp::I64x2RelaxedLaneSelect: {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()), useRegister(ins->v1()),
+ useRegisterAtStart(ins->v2()));
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V2);
+ break;
+ }
+ case wasm::SimdOp::F32x4RelaxedDotBF16x8AddF32x4: {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()), useRegister(ins->v1()),
+ useRegisterAtStart(ins->v2()), tempSimd128());
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V2);
+ break;
+ }
+ default:
+ MOZ_CRASH("NYI");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+ wasm::SimdOp op = ins->simdOp();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Simd128);
+ MOZ_ASSERT(rhs->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ LAllocation lhsAlloc = useRegisterAtStart(lhs);
+ LAllocation rhsAlloc = useRegisterAtStart(rhs);
+ LDefinition tempReg0 = LDefinition::BogusTemp();
+ LDefinition tempReg1 = LDefinition::BogusTemp();
+ if (op == wasm::SimdOp::I64x2Mul) {
+ tempReg0 = tempSimd128();
+ tempReg1 = tempSimd128();
+ }
+ auto* lir = new (alloc())
+ LWasmBinarySimd128(op, lhsAlloc, rhsAlloc, tempReg0, tempReg1);
+ define(lir, ins);
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+#ifdef ENABLE_WASM_SIMD
+bool MWasmTernarySimd128::specializeBitselectConstantMaskAsShuffle(
+ int8_t shuffle[16]) {
+ return false;
+}
+bool MWasmTernarySimd128::canRelaxBitselect() { return false; }
+
+bool MWasmBinarySimd128::canPmaddubsw() { return false; }
+#endif
+
+bool MWasmBinarySimd128::specializeForConstantRhs() {
+ // Probably many we want to do here
+ return false;
+}
+
+void LIRGenerator::visitWasmBinarySimd128WithConstant(
+ MWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("binary SIMD with constant NYI");
+}
+
+void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Simd128);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ if (rhs->isConstant()) {
+ int32_t shiftCount = rhs->toConstant()->toInt32();
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Shl:
+ case wasm::SimdOp::I8x16ShrU:
+ case wasm::SimdOp::I8x16ShrS:
+ shiftCount &= 7;
+ break;
+ case wasm::SimdOp::I16x8Shl:
+ case wasm::SimdOp::I16x8ShrU:
+ case wasm::SimdOp::I16x8ShrS:
+ shiftCount &= 15;
+ break;
+ case wasm::SimdOp::I32x4Shl:
+ case wasm::SimdOp::I32x4ShrU:
+ case wasm::SimdOp::I32x4ShrS:
+ shiftCount &= 31;
+ break;
+ case wasm::SimdOp::I64x2Shl:
+ case wasm::SimdOp::I64x2ShrU:
+ case wasm::SimdOp::I64x2ShrS:
+ shiftCount &= 63;
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift operation");
+ }
+# ifdef DEBUG
+ js::wasm::ReportSimdAnalysis("shift -> constant shift");
+# endif
+ auto* lir = new (alloc())
+ LWasmConstantShiftSimd128(useRegisterAtStart(lhs), shiftCount);
+ define(lir, ins);
+ return;
+ }
+
+# ifdef DEBUG
+ js::wasm::ReportSimdAnalysis("shift -> variable shift");
+# endif
+
+ LAllocation lhsDestAlloc = useRegisterAtStart(lhs);
+ LAllocation rhsAlloc = useRegisterAtStart(rhs);
+ auto* lir = new (alloc()) LWasmVariableShiftSimd128(lhsDestAlloc, rhsAlloc,
+ LDefinition::BogusTemp());
+ define(lir, ins);
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->lhs()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->rhs()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ SimdShuffle s = ins->shuffle();
+ switch (s.opd) {
+ case SimdShuffle::Operand::LEFT:
+ case SimdShuffle::Operand::RIGHT: {
+ LAllocation src;
+ switch (*s.permuteOp) {
+ case SimdPermuteOp::MOVE:
+ case SimdPermuteOp::BROADCAST_8x16:
+ case SimdPermuteOp::BROADCAST_16x8:
+ case SimdPermuteOp::PERMUTE_8x16:
+ case SimdPermuteOp::PERMUTE_16x8:
+ case SimdPermuteOp::PERMUTE_32x4:
+ case SimdPermuteOp::ROTATE_RIGHT_8x16:
+ case SimdPermuteOp::SHIFT_LEFT_8x16:
+ case SimdPermuteOp::SHIFT_RIGHT_8x16:
+ case SimdPermuteOp::REVERSE_16x8:
+ case SimdPermuteOp::REVERSE_32x4:
+ case SimdPermuteOp::REVERSE_64x2:
+ break;
+ default:
+ MOZ_CRASH("Unexpected operator");
+ }
+ if (s.opd == SimdShuffle::Operand::LEFT) {
+ src = useRegisterAtStart(ins->lhs());
+ } else {
+ src = useRegisterAtStart(ins->rhs());
+ }
+ auto* lir =
+ new (alloc()) LWasmPermuteSimd128(src, *s.permuteOp, s.control);
+ define(lir, ins);
+ break;
+ }
+ case SimdShuffle::Operand::BOTH:
+ case SimdShuffle::Operand::BOTH_SWAPPED: {
+ LDefinition temp = LDefinition::BogusTemp();
+ LAllocation lhs;
+ LAllocation rhs;
+ if (s.opd == SimdShuffle::Operand::BOTH) {
+ lhs = useRegisterAtStart(ins->lhs());
+ rhs = useRegisterAtStart(ins->rhs());
+ } else {
+ lhs = useRegisterAtStart(ins->rhs());
+ rhs = useRegisterAtStart(ins->lhs());
+ }
+ auto* lir = new (alloc())
+ LWasmShuffleSimd128(lhs, rhs, temp, *s.shuffleOp, s.control);
+ define(lir, ins);
+ break;
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->lhs()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ // Optimal code generation reuses the lhs register because the rhs scalar is
+ // merged into a vector lhs.
+ LAllocation lhs = useRegisterAtStart(ins->lhs());
+ if (ins->rhs()->type() == MIRType::Int64) {
+ auto* lir = new (alloc())
+ LWasmReplaceInt64LaneSimd128(lhs, useInt64Register(ins->rhs()));
+ defineReuseInput(lir, ins, 0);
+ } else {
+ auto* lir =
+ new (alloc()) LWasmReplaceLaneSimd128(lhs, useRegister(ins->rhs()));
+ defineReuseInput(lir, ins, 0);
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ switch (ins->input()->type()) {
+ case MIRType::Int64: {
+ // 64-bit integer splats.
+ // Load-and-(sign|zero)extend.
+ auto* lir = new (alloc())
+ LWasmInt64ToSimd128(useInt64RegisterAtStart(ins->input()));
+ define(lir, ins);
+ break;
+ }
+ case MIRType::Float32:
+ case MIRType::Double: {
+ // Floating-point splats.
+ auto* lir =
+ new (alloc()) LWasmScalarToSimd128(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+ break;
+ }
+ default: {
+ // 32-bit integer splats.
+ auto* lir =
+ new (alloc()) LWasmScalarToSimd128(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+ break;
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->input()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ LDefinition tempReg = LDefinition::BogusTemp();
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Neg:
+ case wasm::SimdOp::I16x8Neg:
+ case wasm::SimdOp::I32x4Neg:
+ case wasm::SimdOp::I64x2Neg:
+ case wasm::SimdOp::F32x4Neg:
+ case wasm::SimdOp::F64x2Neg:
+ case wasm::SimdOp::F32x4Abs:
+ case wasm::SimdOp::F64x2Abs:
+ case wasm::SimdOp::V128Not:
+ case wasm::SimdOp::F32x4Sqrt:
+ case wasm::SimdOp::F64x2Sqrt:
+ case wasm::SimdOp::I8x16Abs:
+ case wasm::SimdOp::I16x8Abs:
+ case wasm::SimdOp::I32x4Abs:
+ case wasm::SimdOp::I64x2Abs:
+ case wasm::SimdOp::I32x4TruncSatF32x4S:
+ case wasm::SimdOp::F32x4ConvertI32x4U:
+ case wasm::SimdOp::I32x4TruncSatF32x4U:
+ case wasm::SimdOp::I16x8ExtendLowI8x16S:
+ case wasm::SimdOp::I16x8ExtendHighI8x16S:
+ case wasm::SimdOp::I16x8ExtendLowI8x16U:
+ case wasm::SimdOp::I16x8ExtendHighI8x16U:
+ case wasm::SimdOp::I32x4ExtendLowI16x8S:
+ case wasm::SimdOp::I32x4ExtendHighI16x8S:
+ case wasm::SimdOp::I32x4ExtendLowI16x8U:
+ case wasm::SimdOp::I32x4ExtendHighI16x8U:
+ case wasm::SimdOp::I64x2ExtendLowI32x4S:
+ case wasm::SimdOp::I64x2ExtendHighI32x4S:
+ case wasm::SimdOp::I64x2ExtendLowI32x4U:
+ case wasm::SimdOp::I64x2ExtendHighI32x4U:
+ case wasm::SimdOp::F32x4ConvertI32x4S:
+ case wasm::SimdOp::F32x4Ceil:
+ case wasm::SimdOp::F32x4Floor:
+ case wasm::SimdOp::F32x4Trunc:
+ case wasm::SimdOp::F32x4Nearest:
+ case wasm::SimdOp::F64x2Ceil:
+ case wasm::SimdOp::F64x2Floor:
+ case wasm::SimdOp::F64x2Trunc:
+ case wasm::SimdOp::F64x2Nearest:
+ case wasm::SimdOp::F32x4DemoteF64x2Zero:
+ case wasm::SimdOp::F64x2PromoteLowF32x4:
+ case wasm::SimdOp::F64x2ConvertLowI32x4S:
+ case wasm::SimdOp::F64x2ConvertLowI32x4U:
+ case wasm::SimdOp::I16x8ExtaddPairwiseI8x16S:
+ case wasm::SimdOp::I16x8ExtaddPairwiseI8x16U:
+ case wasm::SimdOp::I32x4ExtaddPairwiseI16x8S:
+ case wasm::SimdOp::I32x4ExtaddPairwiseI16x8U:
+ case wasm::SimdOp::I8x16Popcnt:
+ case wasm::SimdOp::I32x4RelaxedTruncSSatF32x4:
+ case wasm::SimdOp::I32x4RelaxedTruncUSatF32x4:
+ case wasm::SimdOp::I32x4RelaxedTruncSatF64x2SZero:
+ case wasm::SimdOp::I32x4RelaxedTruncSatF64x2UZero:
+ break;
+ case wasm::SimdOp::I32x4TruncSatF64x2SZero:
+ case wasm::SimdOp::I32x4TruncSatF64x2UZero:
+ tempReg = tempSimd128();
+ break;
+ default:
+ MOZ_CRASH("Unary SimdOp not implemented");
+ }
+
+ LUse input = useRegisterAtStart(ins->input());
+ LWasmUnarySimd128* lir = new (alloc()) LWasmUnarySimd128(input, tempReg);
+ define(lir, ins);
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ if (canEmitWasmReduceSimd128AtUses(ins)) {
+ emitAtUses(ins);
+ return;
+ }
+
+ // Reductions (any_true, all_true, bitmask, extract_lane) uniformly prefer
+ // useRegisterAtStart:
+ //
+ // - In most cases, the input type differs from the output type, so there's no
+ // conflict and it doesn't really matter.
+ //
+ // - For extract_lane(0) on F32x4 and F64x2, input == output results in zero
+ // code being generated.
+ //
+ // - For extract_lane(k > 0) on F32x4 and F64x2, allowing the input register
+ // to be targeted lowers register pressure if it's the last use of the
+ // input.
+
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new (alloc())
+ LWasmReduceSimd128ToInt64(useRegisterAtStart(ins->input()));
+ defineInt64(lir, ins);
+ } else {
+ LDefinition tempReg = LDefinition::BogusTemp();
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Bitmask:
+ case wasm::SimdOp::I16x8Bitmask:
+ case wasm::SimdOp::I32x4Bitmask:
+ case wasm::SimdOp::I64x2Bitmask:
+ tempReg = tempSimd128();
+ break;
+ default:
+ break;
+ }
+
+ // Ideally we would reuse the input register for floating extract_lane if
+ // the lane is zero, but constraints in the register allocator require the
+ // input and output register types to be the same.
+ auto* lir = new (alloc())
+ LWasmReduceSimd128(useRegisterAtStart(ins->input()), tempReg);
+ define(lir, ins);
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ // On 64-bit systems, the base pointer can be 32 bits or 64 bits. Either way,
+ // it fits in a GPR so we can ignore the Register/Register64 distinction here.
+
+ // Optimal allocation here reuses the value input for the output register
+ // because codegen otherwise has to copy the input to the output; this is
+ // because load-lane is implemented as load + replace-lane. Bug 1706106 may
+ // change all of that, so leave it alone for now.
+ LUse base = useRegisterAtStart(ins->base());
+ LUse inputUse = useRegisterAtStart(ins->value());
+ MOZ_ASSERT(!ins->hasMemoryBase());
+ LWasmLoadLaneSimd128* lir =
+ new (alloc()) LWasmLoadLaneSimd128(base, inputUse, temp(), LAllocation());
+ define(lir, ins);
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ // See comment above about the base pointer.
+
+ LUse base = useRegisterAtStart(ins->base());
+ LUse input = useRegisterAtStart(ins->value());
+ MOZ_ASSERT(!ins->hasMemoryBase());
+ LWasmStoreLaneSimd128* lir =
+ new (alloc()) LWasmStoreLaneSimd128(base, input, temp(), LAllocation());
+ add(lir, ins);
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
diff --git a/js/src/jit/arm64/Lowering-arm64.h b/js/src/jit/arm64/Lowering-arm64.h
new file mode 100644
index 0000000000..4ab52dd464
--- /dev/null
+++ b/js/src/jit/arm64/Lowering-arm64.h
@@ -0,0 +1,135 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_Lowering_arm64_h
+#define jit_arm64_Lowering_arm64_h
+
+#include "jit/shared/Lowering-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorARM64 : public LIRGeneratorShared {
+ protected:
+ LIRGeneratorARM64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorShared(gen, graph, lirGraph) {}
+
+ // Returns a box allocation. reg2 is ignored on 64-bit platforms.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2,
+ bool useAtStart = false);
+
+ LAllocation useByteOpRegister(MDefinition* mir);
+ LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+ LDefinition tempByteOpRegister();
+
+ LDefinition tempToUnbox();
+
+ bool needTempForPostBarrier() { return true; }
+
+ // ARM64 has a scratch register, so no need for another temp for dispatch ICs.
+ LDefinition tempForDispatchCache(MIRType outputType = MIRType::None) {
+ return LDefinition::BogusTemp();
+ }
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+ }
+ void defineInt64Phi(MPhi* phi, size_t lirIndex) {
+ defineTypedPhi(phi, lirIndex);
+ }
+ void lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerUrshD(MUrsh* mir);
+
+ void lowerPowOfTwoI(MPow* mir);
+
+ void lowerForALU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* input);
+ void lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* input);
+ void lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+ void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs,
+ MDefinition* rhs);
+ template <size_t Temps>
+ void lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForCompareI64AndBranch(MTest* mir, MCompare* comp, JSOp op,
+ MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse);
+
+ void lowerForFPU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* input);
+
+ template <size_t Temps>
+ void lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+ void lowerForBitAndAndBranch(LBitAndAndBranch* baab, MInstruction* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins);
+ void lowerTruncateDToInt32(MTruncateToInt32* ins);
+ void lowerTruncateFToInt32(MTruncateToInt32* ins);
+ void lowerDivI(MDiv* div);
+ void lowerModI(MMod* mod);
+ void lowerDivI64(MDiv* div);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+ void lowerModI64(MMod* mod);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+ void lowerUDivI64(MDiv* div);
+ void lowerUModI64(MMod* mod);
+ void lowerNegI(MInstruction* ins, MDefinition* input);
+ void lowerNegI64(MInstruction* ins, MDefinition* input);
+ void lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs);
+ void lowerUDiv(MDiv* div);
+ void lowerUMod(MMod* mod);
+ void lowerWasmSelectI(MWasmSelect* select);
+ void lowerWasmSelectI64(MWasmSelect* select);
+ bool canSpecializeWasmCompareAndSelect(MCompare::CompareType compTy,
+ MIRType insTy);
+ void lowerWasmCompareAndSelect(MWasmSelect* ins, MDefinition* lhs,
+ MDefinition* rhs, MCompare::CompareType compTy,
+ JSOp jsop);
+
+ void lowerBigIntLsh(MBigIntLsh* ins);
+ void lowerBigIntRsh(MBigIntRsh* ins);
+ void lowerBigIntDiv(MBigIntDiv* ins);
+ void lowerBigIntMod(MBigIntMod* ins);
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+ void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+
+#ifdef ENABLE_WASM_SIMD
+ bool canFoldReduceSimd128AndBranch(wasm::SimdOp op);
+ bool canEmitWasmReduceSimd128AtUses(MWasmReduceSimd128* ins);
+#endif
+
+ LTableSwitchV* newLTableSwitchV(MTableSwitch* ins);
+ LTableSwitch* newLTableSwitch(const LAllocation& in,
+ const LDefinition& inputCopy,
+ MTableSwitch* ins);
+
+ void lowerPhi(MPhi* phi);
+};
+
+typedef LIRGeneratorARM64 LIRGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm64_Lowering_arm64_h */
diff --git a/js/src/jit/arm64/MacroAssembler-arm64-inl.h b/js/src/jit/arm64/MacroAssembler-arm64-inl.h
new file mode 100644
index 0000000000..154e5c418a
--- /dev/null
+++ b/js/src/jit/arm64/MacroAssembler-arm64-inl.h
@@ -0,0 +1,4052 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_MacroAssembler_arm64_inl_h
+#define jit_arm64_MacroAssembler_arm64_inl_h
+
+#include "jit/arm64/MacroAssembler-arm64.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+ Mov(ARMRegister(dest.reg, 64), ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+ Mov(ARMRegister(dest.reg, 64), imm.value);
+}
+
+void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
+ Fmov(ARMRegister(dest, 32), ARMFPRegister(src, 32));
+}
+
+void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
+ Fmov(ARMFPRegister(dest, 32), ARMRegister(src, 32));
+}
+
+void MacroAssembler::move8SignExtend(Register src, Register dest) {
+ Sxtb(ARMRegister(dest, 32), ARMRegister(src, 32));
+}
+
+void MacroAssembler::move16SignExtend(Register src, Register dest) {
+ Sxth(ARMRegister(dest, 32), ARMRegister(src, 32));
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ Fmov(ARMRegister(dest.reg, 64), ARMFPRegister(src, 64));
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ Fmov(ARMFPRegister(dest, 64), ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+ Mov(ARMRegister(dest, 32), ARMRegister(src.reg, 32));
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ Mov(ARMRegister(dest.reg, 32), ARMRegister(src, 32));
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ Sxtb(ARMRegister(dest.reg, 64), ARMRegister(src, 32));
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ Sxth(ARMRegister(dest.reg, 64), ARMRegister(src, 32));
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ Sxtw(ARMRegister(dest.reg, 64), ARMRegister(src, 32));
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ Sxtw(ARMRegister(dest, 64), ARMRegister(src, 32));
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ Mov(ARMRegister(dest, 32), ARMRegister(src, 32));
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ load32(src, dest);
+ move32To64SignExtend(dest, Register64(dest));
+}
+
+void MacroAssembler::loadAbiReturnAddress(Register dest) { movePtr(lr, dest); }
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::not32(Register reg) {
+ Orn(ARMRegister(reg, 32), vixl::wzr, ARMRegister(reg, 32));
+}
+
+void MacroAssembler::notPtr(Register reg) {
+ Orn(ARMRegister(reg, 64), vixl::xzr, ARMRegister(reg, 64));
+}
+
+void MacroAssembler::and32(Register src, Register dest) {
+ And(ARMRegister(dest, 32), ARMRegister(dest, 32),
+ Operand(ARMRegister(src, 32)));
+}
+
+void MacroAssembler::and32(Imm32 imm, Register dest) {
+ And(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
+}
+
+void MacroAssembler::and32(Imm32 imm, Register src, Register dest) {
+ And(ARMRegister(dest, 32), ARMRegister(src, 32), Operand(imm.value));
+}
+
+void MacroAssembler::and32(Imm32 imm, const Address& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != dest.base);
+ load32(dest, scratch32.asUnsized());
+ And(scratch32, scratch32, Operand(imm.value));
+ store32(scratch32.asUnsized(), dest);
+}
+
+void MacroAssembler::and32(const Address& src, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != src.base);
+ load32(src, scratch32.asUnsized());
+ And(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(scratch32));
+}
+
+void MacroAssembler::andPtr(Register src, Register dest) {
+ And(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(ARMRegister(src, 64)));
+}
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) {
+ And(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
+}
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+ And(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
+}
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+ And(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64),
+ ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+ Orr(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
+}
+
+void MacroAssembler::or32(Imm32 imm, Register dest) {
+ Orr(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
+}
+
+void MacroAssembler::or32(Register src, Register dest) {
+ Orr(ARMRegister(dest, 32), ARMRegister(dest, 32),
+ Operand(ARMRegister(src, 32)));
+}
+
+void MacroAssembler::or32(Imm32 imm, const Address& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != dest.base);
+ load32(dest, scratch32.asUnsized());
+ Orr(scratch32, scratch32, Operand(imm.value));
+ store32(scratch32.asUnsized(), dest);
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) {
+ Orr(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(ARMRegister(src, 64)));
+}
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) {
+ Orr(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
+}
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+ orPtr(src.reg, dest.reg);
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+ xorPtr(src.reg, dest.reg);
+}
+
+void MacroAssembler::xor32(Register src, Register dest) {
+ Eor(ARMRegister(dest, 32), ARMRegister(dest, 32),
+ Operand(ARMRegister(src, 32)));
+}
+
+void MacroAssembler::xor32(Imm32 imm, Register dest) {
+ Eor(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
+}
+
+void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != dest.base);
+ load32(dest, scratch32.asUnsized());
+ Eor(scratch32, scratch32, Operand(imm.value));
+ store32(scratch32.asUnsized(), dest);
+}
+
+void MacroAssembler::xor32(const Address& src, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != src.base);
+ load32(src, scratch32.asUnsized());
+ Eor(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(scratch32));
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) {
+ Eor(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(ARMRegister(src, 64)));
+}
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) {
+ Eor(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+ Eor(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
+}
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap16SignExtend(Register reg) {
+ rev16(ARMRegister(reg, 32), ARMRegister(reg, 32));
+ sxth(ARMRegister(reg, 32), ARMRegister(reg, 32));
+}
+
+void MacroAssembler::byteSwap16ZeroExtend(Register reg) {
+ rev16(ARMRegister(reg, 32), ARMRegister(reg, 32));
+ uxth(ARMRegister(reg, 32), ARMRegister(reg, 32));
+}
+
+void MacroAssembler::byteSwap32(Register reg) {
+ rev(ARMRegister(reg, 32), ARMRegister(reg, 32));
+}
+
+void MacroAssembler::byteSwap64(Register64 reg) {
+ rev(ARMRegister(reg.reg, 64), ARMRegister(reg.reg, 64));
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::add32(Register src, Register dest) {
+ Add(ARMRegister(dest, 32), ARMRegister(dest, 32),
+ Operand(ARMRegister(src, 32)));
+}
+
+void MacroAssembler::add32(Imm32 imm, Register dest) {
+ Add(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
+}
+
+void MacroAssembler::add32(Imm32 imm, const Address& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != dest.base);
+
+ Ldr(scratch32, toMemOperand(dest));
+ Add(scratch32, scratch32, Operand(imm.value));
+ Str(scratch32, toMemOperand(dest));
+}
+
+void MacroAssembler::addPtr(Register src, Register dest) {
+ addPtr(src, dest, dest);
+}
+
+void MacroAssembler::addPtr(Register src1, Register src2, Register dest) {
+ Add(ARMRegister(dest, 64), ARMRegister(src1, 64),
+ Operand(ARMRegister(src2, 64)));
+}
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) {
+ addPtr(imm, dest, dest);
+}
+
+void MacroAssembler::addPtr(Imm32 imm, Register src, Register dest) {
+ Add(ARMRegister(dest, 64), ARMRegister(src, 64), Operand(imm.value));
+}
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+ Add(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != dest.base);
+
+ Ldr(scratch64, toMemOperand(dest));
+ Add(scratch64, scratch64, Operand(imm.value));
+ Str(scratch64, toMemOperand(dest));
+}
+
+void MacroAssembler::addPtr(const Address& src, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != src.base);
+
+ Ldr(scratch64, toMemOperand(src));
+ Add(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(scratch64));
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+ addPtr(src.reg, dest.reg);
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) {
+ Add(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
+}
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+ Add(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ AutoForbidPoolsAndNops afp(this,
+ /* max number of instructions in scope = */ 3);
+ CodeOffset offs = CodeOffset(currentOffset());
+ movz(scratch, 0, 0);
+ movk(scratch, 0, 16);
+ Sub(ARMRegister(dest, 64), sp, scratch);
+ return offs;
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ Instruction* i1 = getInstructionAt(BufferOffset(offset.offset()));
+ MOZ_ASSERT(i1->IsMovz());
+ i1->SetInstructionBits(i1->InstructionBits() |
+ ImmMoveWide(uint16_t(imm.value)));
+
+ Instruction* i2 = getInstructionAt(BufferOffset(offset.offset() + 4));
+ MOZ_ASSERT(i2->IsMovk());
+ i2->SetInstructionBits(i2->InstructionBits() |
+ ImmMoveWide(uint16_t(imm.value >> 16)));
+}
+
+void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
+ fadd(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64),
+ ARMFPRegister(src, 64));
+}
+
+void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
+ fadd(ARMFPRegister(dest, 32), ARMFPRegister(dest, 32),
+ ARMFPRegister(src, 32));
+}
+
+void MacroAssembler::sub32(Imm32 imm, Register dest) {
+ Sub(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
+}
+
+void MacroAssembler::sub32(Register src, Register dest) {
+ Sub(ARMRegister(dest, 32), ARMRegister(dest, 32),
+ Operand(ARMRegister(src, 32)));
+}
+
+void MacroAssembler::sub32(const Address& src, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != src.base);
+ load32(src, scratch32.asUnsized());
+ Sub(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(scratch32));
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) {
+ Sub(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(ARMRegister(src, 64)));
+}
+
+void MacroAssembler::subPtr(Register src, const Address& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != dest.base);
+
+ Ldr(scratch64, toMemOperand(dest));
+ Sub(scratch64, scratch64, Operand(ARMRegister(src, 64)));
+ Str(scratch64, toMemOperand(dest));
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) {
+ Sub(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
+}
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != addr.base);
+
+ Ldr(scratch64, toMemOperand(addr));
+ Sub(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(scratch64));
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+ Sub(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64),
+ ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+ Sub(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), Operand(imm.value));
+}
+
+void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
+ fsub(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64),
+ ARMFPRegister(src, 64));
+}
+
+void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
+ fsub(ARMFPRegister(dest, 32), ARMFPRegister(dest, 32),
+ ARMFPRegister(src, 32));
+}
+
+void MacroAssembler::mul32(Register rhs, Register srcDest) {
+ mul32(srcDest, rhs, srcDest, nullptr);
+}
+
+void MacroAssembler::mul32(Imm32 imm, Register srcDest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+
+ move32(imm, scratch32.asUnsized());
+ mul32(scratch32.asUnsized(), srcDest);
+}
+
+void MacroAssembler::mul32(Register src1, Register src2, Register dest,
+ Label* onOver) {
+ if (onOver) {
+ Smull(ARMRegister(dest, 64), ARMRegister(src1, 32), ARMRegister(src2, 32));
+ Cmp(ARMRegister(dest, 64), Operand(ARMRegister(dest, 32), vixl::SXTW));
+ B(onOver, NotEqual);
+
+ // Clear upper 32 bits.
+ Mov(ARMRegister(dest, 32), ARMRegister(dest, 32));
+ } else {
+ Mul(ARMRegister(dest, 32), ARMRegister(src1, 32), ARMRegister(src2, 32));
+ }
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+
+ Mov(scratch32, int32_t(imm.value));
+ Umull(ARMRegister(dest, 64), scratch32, ARMRegister(src, 32));
+
+ Lsr(ARMRegister(dest, 64), ARMRegister(dest, 64), 32);
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+ Mul(ARMRegister(srcDest, 64), ARMRegister(srcDest, 64), ARMRegister(rhs, 64));
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(dest.reg != scratch64.asUnsized());
+ mov(ImmWord(imm.value), scratch64.asUnsized());
+ Mul(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), scratch64);
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ Mul(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64),
+ ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::mul64(const Register64& src1, const Register64& src2,
+ const Register64& dest) {
+ Mul(ARMRegister(dest.reg, 64), ARMRegister(src1.reg, 64),
+ ARMRegister(src2.reg, 64));
+}
+
+void MacroAssembler::mul64(Imm64 src1, const Register64& src2,
+ const Register64& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(dest.reg != scratch64.asUnsized());
+ mov(ImmWord(src1.value), scratch64.asUnsized());
+ Mul(ARMRegister(dest.reg, 64), ARMRegister(src2.reg, 64), scratch64);
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+ ARMRegister xdest(dest, 64);
+ ARMRegister xsrc(src, 64);
+ Add(xdest, xsrc, Operand(xsrc, vixl::LSL, 1));
+}
+
+void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
+ fmul(ARMFPRegister(dest, 32), ARMFPRegister(dest, 32),
+ ARMFPRegister(src, 32));
+}
+
+void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
+ fmul(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64),
+ ARMFPRegister(src, 64));
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+ FloatRegister dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(temp != scratch);
+ movePtr(imm, scratch);
+ const ARMFPRegister scratchDouble = temps.AcquireD();
+ Ldr(scratchDouble, MemOperand(Address(scratch, 0)));
+ fmul(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64), scratchDouble);
+}
+
+void MacroAssembler::quotient32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ if (isUnsigned) {
+ Udiv(ARMRegister(srcDest, 32), ARMRegister(srcDest, 32),
+ ARMRegister(rhs, 32));
+ } else {
+ Sdiv(ARMRegister(srcDest, 32), ARMRegister(srcDest, 32),
+ ARMRegister(rhs, 32));
+ }
+}
+
+// This does not deal with x % 0 or INT_MIN % -1, the caller needs to filter
+// those cases when they may occur.
+
+void MacroAssembler::remainder32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ vixl::UseScratchRegisterScope temps(this);
+ ARMRegister scratch = temps.AcquireW();
+ if (isUnsigned) {
+ Udiv(scratch, ARMRegister(srcDest, 32), ARMRegister(rhs, 32));
+ } else {
+ Sdiv(scratch, ARMRegister(srcDest, 32), ARMRegister(rhs, 32));
+ }
+ Mul(scratch, scratch, ARMRegister(rhs, 32));
+ Sub(ARMRegister(srcDest, 32), ARMRegister(srcDest, 32), scratch);
+}
+
+void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
+ fdiv(ARMFPRegister(dest, 32), ARMFPRegister(dest, 32),
+ ARMFPRegister(src, 32));
+}
+
+void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
+ fdiv(ARMFPRegister(dest, 64), ARMFPRegister(dest, 64),
+ ARMFPRegister(src, 64));
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratchAddr64 = temps.AcquireX();
+ const ARMRegister scratch64 = temps.AcquireX();
+
+ Mov(scratchAddr64, uint64_t(dest.addr));
+ Ldr(scratch64, MemOperand(scratchAddr64, 0));
+ Add(scratch64, scratch64, Operand(1));
+ Str(scratch64, MemOperand(scratchAddr64, 0));
+}
+
+void MacroAssembler::neg32(Register reg) {
+ Negs(ARMRegister(reg, 32), Operand(ARMRegister(reg, 32)));
+}
+
+void MacroAssembler::neg64(Register64 reg) { negPtr(reg.reg); }
+
+void MacroAssembler::negPtr(Register reg) {
+ Negs(ARMRegister(reg, 64), Operand(ARMRegister(reg, 64)));
+}
+
+void MacroAssembler::negateFloat(FloatRegister reg) {
+ fneg(ARMFPRegister(reg, 32), ARMFPRegister(reg, 32));
+}
+
+void MacroAssembler::negateDouble(FloatRegister reg) {
+ fneg(ARMFPRegister(reg, 64), ARMFPRegister(reg, 64));
+}
+
+void MacroAssembler::abs32(Register src, Register dest) {
+ Cmp(ARMRegister(src, 32), wzr);
+ Cneg(ARMRegister(dest, 32), ARMRegister(src, 32), Assembler::LessThan);
+}
+
+void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
+ fabs(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
+}
+
+void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
+ fabs(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
+}
+
+void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
+ fsqrt(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
+}
+
+void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
+ fsqrt(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
+}
+
+void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ MOZ_ASSERT(handleNaN); // Always true for wasm
+ fmin(ARMFPRegister(srcDest, 32), ARMFPRegister(srcDest, 32),
+ ARMFPRegister(other, 32));
+}
+
+void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ MOZ_ASSERT(handleNaN); // Always true for wasm
+ fmin(ARMFPRegister(srcDest, 64), ARMFPRegister(srcDest, 64),
+ ARMFPRegister(other, 64));
+}
+
+void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ MOZ_ASSERT(handleNaN); // Always true for wasm
+ fmax(ARMFPRegister(srcDest, 32), ARMFPRegister(srcDest, 32),
+ ARMFPRegister(other, 32));
+}
+
+void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ MOZ_ASSERT(handleNaN); // Always true for wasm
+ fmax(ARMFPRegister(srcDest, 64), ARMFPRegister(srcDest, 64),
+ ARMFPRegister(other, 64));
+}
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ Lsl(ARMRegister(dest, 64), ARMRegister(dest, 64), imm.value);
+}
+
+void MacroAssembler::lshiftPtr(Register shift, Register dest) {
+ Lsl(ARMRegister(dest, 64), ARMRegister(dest, 64), ARMRegister(shift, 64));
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ lshiftPtr(imm, dest.reg);
+}
+
+void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
+ Lsl(ARMRegister(srcDest.reg, 64), ARMRegister(srcDest.reg, 64),
+ ARMRegister(shift, 64));
+}
+
+void MacroAssembler::lshift32(Register shift, Register dest) {
+ Lsl(ARMRegister(dest, 32), ARMRegister(dest, 32), ARMRegister(shift, 32));
+}
+
+void MacroAssembler::flexibleLshift32(Register src, Register dest) {
+ lshift32(src, dest);
+}
+
+void MacroAssembler::lshift32(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ Lsl(ARMRegister(dest, 32), ARMRegister(dest, 32), imm.value);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ Lsr(ARMRegister(dest, 64), ARMRegister(dest, 64), imm.value);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register src, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ Lsr(ARMRegister(dest, 64), ARMRegister(src, 64), imm.value);
+}
+
+void MacroAssembler::rshiftPtr(Register shift, Register dest) {
+ Lsr(ARMRegister(dest, 64), ARMRegister(dest, 64), ARMRegister(shift, 64));
+}
+
+void MacroAssembler::rshift32(Register shift, Register dest) {
+ Lsr(ARMRegister(dest, 32), ARMRegister(dest, 32), ARMRegister(shift, 32));
+}
+
+void MacroAssembler::flexibleRshift32(Register src, Register dest) {
+ rshift32(src, dest);
+}
+
+void MacroAssembler::rshift32(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ Lsr(ARMRegister(dest, 32), ARMRegister(dest, 32), imm.value);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ Asr(ARMRegister(dest, 64), ARMRegister(dest, 64), imm.value);
+}
+
+void MacroAssembler::rshift32Arithmetic(Register shift, Register dest) {
+ Asr(ARMRegister(dest, 32), ARMRegister(dest, 32), ARMRegister(shift, 32));
+}
+
+void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ Asr(ARMRegister(dest, 32), ARMRegister(dest, 32), imm.value);
+}
+
+void MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest) {
+ rshift32Arithmetic(src, dest);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ rshiftPtr(imm, dest.reg);
+}
+
+void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
+ Lsr(ARMRegister(srcDest.reg, 64), ARMRegister(srcDest.reg, 64),
+ ARMRegister(shift, 64));
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ Asr(ARMRegister(dest.reg, 64), ARMRegister(dest.reg, 64), imm.value);
+}
+
+void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
+ Asr(ARMRegister(srcDest.reg, 64), ARMRegister(srcDest.reg, 64),
+ ARMRegister(shift, 64));
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, scratch);
+ cmp32Set(cond, scratch, Imm32(uint8_t(rhs.value)), dest);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, scratch);
+ cmp32Set(cond, scratch, Imm32(int8_t(rhs.value)), dest);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load16ZeroExtend(lhs, scratch);
+ cmp32Set(cond, scratch, Imm32(uint16_t(rhs.value)), dest);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load16SignExtend(lhs, scratch);
+ cmp32Set(cond, scratch, Imm32(int16_t(rhs.value)), dest);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ cmp32(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ cmpPtrSet(cond, lhs, ImmWord(static_cast<uintptr_t>(rhs.value)), dest);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ cmpPtr(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+// ===============================================================
+// Rotation functions
+
+void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
+ Ror(ARMRegister(dest, 32), ARMRegister(input, 32), (32 - count.value) & 31);
+}
+
+void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireW();
+ // Really 32 - count, but the upper bits of the result are ignored.
+ Neg(scratch, ARMRegister(count, 32));
+ Ror(ARMRegister(dest, 32), ARMRegister(input, 32), scratch);
+}
+
+void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
+ Ror(ARMRegister(dest, 32), ARMRegister(input, 32), count.value & 31);
+}
+
+void MacroAssembler::rotateRight(Register count, Register input,
+ Register dest) {
+ Ror(ARMRegister(dest, 32), ARMRegister(input, 32), ARMRegister(count, 32));
+}
+
+void MacroAssembler::rotateLeft64(Register count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ // Really 64 - count, but the upper bits of the result are ignored.
+ Neg(scratch, ARMRegister(count, 64));
+ Ror(ARMRegister(dest.reg, 64), ARMRegister(input.reg, 64), scratch);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+
+ Ror(ARMRegister(dest.reg, 64), ARMRegister(input.reg, 64),
+ (64 - count.value) & 63);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+
+ Ror(ARMRegister(dest.reg, 64), ARMRegister(input.reg, 64),
+ ARMRegister(count, 64));
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+
+ Ror(ARMRegister(dest.reg, 64), ARMRegister(input.reg, 64), count.value & 63);
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
+ Clz(ARMRegister(dest, 32), ARMRegister(src, 32));
+}
+
+void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
+ Rbit(ARMRegister(dest, 32), ARMRegister(src, 32));
+ Clz(ARMRegister(dest, 32), ARMRegister(dest, 32));
+}
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+ Clz(ARMRegister(dest, 64), ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+ Rbit(ARMRegister(dest, 64), ARMRegister(src.reg, 64));
+ Clz(ARMRegister(dest, 64), ARMRegister(dest, 64));
+}
+
+void MacroAssembler::popcnt32(Register src_, Register dest_, Register tmp_) {
+ MOZ_ASSERT(tmp_ != Register::Invalid());
+
+ // Equivalent to mozilla::CountPopulation32().
+
+ ARMRegister src(src_, 32);
+ ARMRegister dest(dest_, 32);
+ ARMRegister tmp(tmp_, 32);
+
+ Mov(tmp, src);
+ if (src_ != dest_) {
+ Mov(dest, src);
+ }
+ Lsr(dest, dest, 1);
+ And(dest, dest, 0x55555555);
+ Sub(dest, tmp, dest);
+ Lsr(tmp, dest, 2);
+ And(tmp, tmp, 0x33333333);
+ And(dest, dest, 0x33333333);
+ Add(dest, tmp, dest);
+ Add(dest, dest, Operand(dest, vixl::LSR, 4));
+ And(dest, dest, 0x0F0F0F0F);
+ Add(dest, dest, Operand(dest, vixl::LSL, 8));
+ Add(dest, dest, Operand(dest, vixl::LSL, 16));
+ Lsr(dest, dest, 24);
+}
+
+void MacroAssembler::popcnt64(Register64 src_, Register64 dest_,
+ Register tmp_) {
+ MOZ_ASSERT(tmp_ != Register::Invalid());
+
+ // Equivalent to mozilla::CountPopulation64(), though likely more efficient.
+
+ ARMRegister src(src_.reg, 64);
+ ARMRegister dest(dest_.reg, 64);
+ ARMRegister tmp(tmp_, 64);
+
+ Mov(tmp, src);
+ if (src_ != dest_) {
+ Mov(dest, src);
+ }
+ Lsr(dest, dest, 1);
+ And(dest, dest, 0x5555555555555555);
+ Sub(dest, tmp, dest);
+ Lsr(tmp, dest, 2);
+ And(tmp, tmp, 0x3333333333333333);
+ And(dest, dest, 0x3333333333333333);
+ Add(dest, tmp, dest);
+ Add(dest, dest, Operand(dest, vixl::LSR, 4));
+ And(dest, dest, 0x0F0F0F0F0F0F0F0F);
+ Add(dest, dest, Operand(dest, vixl::LSL, 8));
+ Add(dest, dest, Operand(dest, vixl::LSL, 16));
+ Add(dest, dest, Operand(dest, vixl::LSL, 32));
+ Lsr(dest, dest, 56);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, scratch);
+ branch32(cond, scratch, Imm32(uint8_t(rhs.value)), label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, scratch);
+ branch32(cond, scratch, Imm32(int8_t(rhs.value)), label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, scratch);
+ branch32(cond, scratch, rhs, label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, scratch);
+ branch32(cond, scratch, rhs, label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load16ZeroExtend(lhs, scratch);
+ branch32(cond, scratch, Imm32(uint16_t(rhs.value)), label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load16SignExtend(lhs, scratch);
+ branch32(cond, scratch, Imm32(int16_t(rhs.value)), label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ cmp32(lhs, rhs);
+ B(label, cond);
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm,
+ L label) {
+ if (imm.value == 0 && cond == Assembler::Equal) {
+ Cbz(ARMRegister(lhs, 32), label);
+ } else if (imm.value == 0 && cond == Assembler::NotEqual) {
+ Cbnz(ARMRegister(lhs, 32), label);
+ } else {
+ cmp32(lhs, imm);
+ B(label, cond);
+ }
+}
+
+void MacroAssembler::branch32(Condition cond, Register lhs, const Address& rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs);
+ MOZ_ASSERT(scratch != rhs.base);
+ load32(rhs, scratch);
+ branch32(cond, lhs, scratch, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ MOZ_ASSERT(scratch != rhs);
+ load32(lhs, scratch);
+ branch32(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 imm,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ load32(lhs, scratch);
+ branch32(cond, scratch, imm, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ movePtr(ImmPtr(lhs.addr), scratch);
+ branch32(cond, Address(scratch, 0), rhs, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ load32(lhs, scratch);
+ branch32(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != lhs.base);
+ MOZ_ASSERT(scratch32.asUnsized() != lhs.index);
+ doBaseIndex(scratch32, lhs, vixl::LDR_w);
+ branch32(cond, scratch32.asUnsized(), rhs, label);
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
+ Imm32 rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ movePtr(lhs, scratch);
+ branch32(cond, Address(scratch, 0), rhs, label);
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ if (val.value == 0 && cond == Assembler::Equal) {
+ Cbz(ARMRegister(lhs.reg, 64), success);
+ } else if (val.value == 0 && cond == Assembler::NotEqual) {
+ Cbnz(ARMRegister(lhs.reg, 64), success);
+ } else {
+ Cmp(ARMRegister(lhs.reg, 64), val.value);
+ B(success, cond);
+ }
+ if (fail) {
+ B(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ Cmp(ARMRegister(lhs.reg, 64), ARMRegister(rhs.reg, 64));
+ B(success, cond);
+ if (fail) {
+ B(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, ImmWord(val.value), label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, rhs.reg, label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ loadPtr(rhs, scratch);
+ branchPtr(cond, lhs, scratch, label);
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ Cmp(ARMRegister(lhs, 64), ARMRegister(rhs, 64));
+ B(label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ if (rhs.value == 0 && cond == Assembler::Equal) {
+ Cbz(ARMRegister(lhs, 64), label);
+ } else if (rhs.value == 0 && cond == Assembler::NotEqual) {
+ Cbnz(ARMRegister(lhs, 64), label);
+ } else {
+ cmpPtr(lhs, rhs);
+ B(label, cond);
+ }
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
+ Label* label) {
+ if (rhs.value == 0 && cond == Assembler::Equal) {
+ Cbz(ARMRegister(lhs, 64), label);
+ } else if (rhs.value == 0 && cond == Assembler::NotEqual) {
+ Cbnz(ARMRegister(lhs, 64), label);
+ } else {
+ cmpPtr(lhs, rhs);
+ B(label, cond);
+ }
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs);
+ movePtr(rhs, scratch);
+ branchPtr(cond, lhs, scratch, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
+ Label* label) {
+ if (rhs.value == 0 && cond == Assembler::Equal) {
+ Cbz(ARMRegister(lhs, 64), label);
+ } else if (rhs.value == 0 && cond == Assembler::NotEqual) {
+ Cbnz(ARMRegister(lhs, 64), label);
+ } else {
+ cmpPtr(lhs, rhs);
+ B(label, cond);
+ }
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
+ L label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ MOZ_ASSERT(scratch != rhs);
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch1_64 = temps.AcquireX();
+ const ARMRegister scratch2_64 = temps.AcquireX();
+ MOZ_ASSERT(scratch1_64.asUnsized() != lhs.base);
+ MOZ_ASSERT(scratch2_64.asUnsized() != lhs.base);
+
+ movePtr(rhs, scratch1_64.asUnsized());
+ loadPtr(lhs, scratch2_64.asUnsized());
+ branchPtr(cond, scratch2_64.asUnsized(), scratch1_64.asUnsized(), label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != rhs);
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ ImmWord rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+ Register rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != rhs);
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ ImmWord rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ MOZ_ASSERT(scratch != lhs.index);
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ Register rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ MOZ_ASSERT(scratch != lhs.index);
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ branchPtr(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ compareFloat(cond, lhs, rhs);
+ switch (cond) {
+ case DoubleNotEqual: {
+ Label unordered;
+ // not equal *and* ordered
+ branch(Overflow, &unordered);
+ branch(NotEqual, label);
+ bind(&unordered);
+ break;
+ }
+ case DoubleEqualOrUnordered:
+ branch(Overflow, label);
+ branch(Equal, label);
+ break;
+ default:
+ branch(Condition(cond), label);
+ }
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+
+ ARMFPRegister src32(src, 32);
+ ARMRegister dest64(dest, 64);
+
+ MOZ_ASSERT(!scratch64.Is(dest64));
+
+ Fcvtzs(dest64, src32);
+ Add(scratch64, dest64, Operand(0x7fffffffffffffff));
+ Cmn(scratch64, 3);
+ B(fail, Assembler::Above);
+ And(dest64, dest64, Operand(0xffffffff));
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ convertFloat32ToInt32(src, dest, fail, false);
+}
+
+void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ compareDouble(cond, lhs, rhs);
+ switch (cond) {
+ case DoubleNotEqual: {
+ Label unordered;
+ // not equal *and* ordered
+ branch(Overflow, &unordered);
+ branch(NotEqual, label);
+ bind(&unordered);
+ break;
+ }
+ case DoubleEqualOrUnordered:
+ branch(Overflow, label);
+ branch(Equal, label);
+ break;
+ default:
+ branch(Condition(cond), label);
+ }
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+
+ // An out of range integer will be saturated to the destination size.
+ ARMFPRegister src64(src, 64);
+ ARMRegister dest64(dest, 64);
+
+ MOZ_ASSERT(!scratch64.Is(dest64));
+
+ Fcvtzs(dest64, src64);
+ Add(scratch64, dest64, Operand(0x7fffffffffffffff));
+ Cmn(scratch64, 3);
+ B(fail, Assembler::Above);
+ And(dest64, dest64, Operand(0xffffffff));
+}
+
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ convertDoubleToInt32(src, dest, fail, false);
+}
+
+template <typename T>
+void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
+ Label* label) {
+ adds32(src, dest);
+ B(label, cond);
+}
+
+template <typename T>
+void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
+ Label* label) {
+ subs32(src, dest);
+ branch(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Overflow);
+ vixl::UseScratchRegisterScope temps(this);
+ mul32(src, dest, dest, label);
+}
+
+template <typename T>
+void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ rshift32(src, dest);
+ branch32(cond == Zero ? Equal : NotEqual, dest, Imm32(0), label);
+}
+
+void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
+ MOZ_ASSERT(cond == Overflow);
+ neg32(reg);
+ B(label, cond);
+}
+
+template <typename T>
+void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ adds64(src, dest);
+ B(label, cond);
+}
+
+template <typename T>
+void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ subs64(src, dest);
+ B(label, cond);
+}
+
+void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Overflow);
+
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ const ARMRegister src64(src, 64);
+ const ARMRegister dest64(dest, 64);
+
+ Smulh(scratch64, dest64, src64);
+ Mul(dest64, dest64, src64);
+ Cmp(scratch64, Operand(dest64, vixl::ASR, 63));
+ B(label, NotEqual);
+}
+
+void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ Subs(ARMRegister(lhs, 64), ARMRegister(lhs, 64), Operand(rhs.value));
+ B(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ // The x86-biased front end prefers |test foo, foo| to |cmp foo, #0|. We look
+ // for the former pattern and expand as Cbz/Cbnz when possible.
+ if (lhs == rhs && cond == Zero) {
+ Cbz(ARMRegister(lhs, 32), label);
+ } else if (lhs == rhs && cond == NonZero) {
+ Cbnz(ARMRegister(lhs, 32), label);
+ } else {
+ test32(lhs, rhs);
+ B(label, cond);
+ }
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ test32(lhs, rhs);
+ B(label, cond);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ load32(lhs, scratch);
+ branchTest32(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ load32(lhs, scratch);
+ branchTest32(cond, scratch, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ // See branchTest32.
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ if (lhs == rhs && cond == Zero) {
+ Cbz(ARMRegister(lhs, 64), label);
+ } else if (lhs == rhs && cond == NonZero) {
+ Cbnz(ARMRegister(lhs, 64), label);
+ } else {
+ Tst(ARMRegister(lhs, 64), Operand(ARMRegister(rhs, 64)));
+ B(label, cond);
+ }
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ Tst(ARMRegister(lhs, 64), Operand(rhs.value));
+ B(label, cond);
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
+ Imm32 rhs, Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ loadPtr(lhs, scratch);
+ branchTestPtr(cond, scratch, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ branchTestPtr(cond, lhs.reg, rhs.reg, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
+ Label* label) {
+ branchTestUndefinedImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
+ Label* label) {
+ branchTestUndefinedImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const BaseIndex& address,
+ Label* label) {
+ branchTestUndefinedImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestUndefinedImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestUndefinedImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testUndefined(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, Register tag,
+ Label* label) {
+ branchTestInt32Impl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
+ Label* label) {
+ branchTestInt32Impl(cond, address, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestInt32Impl(cond, address, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestInt32Impl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestInt32Impl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testInt32(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestInt32Truthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition c = testInt32Truthy(truthy, value);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, Register tag,
+ Label* label) {
+ branchTestDoubleImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
+ Label* label) {
+ branchTestDoubleImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestDoubleImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestDoubleImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestDoubleImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testDouble(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestDoubleTruthy(bool truthy, FloatRegister reg,
+ Label* label) {
+ Fcmp(ARMFPRegister(reg, 64), 0.0);
+ if (!truthy) {
+ // falsy values are zero, and NaN.
+ branch(Zero, label);
+ branch(Overflow, label);
+ } else {
+ // truthy values are non-zero and not nan.
+ // If it is overflow
+ Label onFalse;
+ branch(Zero, &onFalse);
+ branch(Overflow, &onFalse);
+ B(label);
+ bind(&onFalse);
+ }
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, Register tag,
+ Label* label) {
+ branchTestNumberImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestNumberImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestNumberImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testNumber(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
+ Label* label) {
+ branchTestBooleanImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
+ Label* label) {
+ branchTestBooleanImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestBooleanImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestBooleanImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestBooleanImpl(Condition cond, const T& tag,
+ Label* label) {
+ Condition c = testBoolean(cond, tag);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition c = testBooleanTruthy(truthy, value);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestString(Condition cond, Register tag,
+ Label* label) {
+ branchTestStringImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const Address& address,
+ Label* label) {
+ branchTestStringImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestStringImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestStringImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestStringImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testString(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestStringTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition c = testStringTruthy(truthy, value);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
+ Label* label) {
+ branchTestSymbolImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
+ Label* label) {
+ branchTestSymbolImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestSymbolImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestSymbolImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestSymbolImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testSymbol(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
+ Label* label) {
+ branchTestBigIntImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
+ Label* label) {
+ branchTestBigIntImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestBigIntImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestBigIntImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestBigIntImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testBigInt(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestBigIntTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition c = testBigIntTruthy(truthy, value);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, Register tag,
+ Label* label) {
+ branchTestNullImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const Address& address,
+ Label* label) {
+ branchTestNullImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestNullImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestNullImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestNullImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testNull(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, Register tag,
+ Label* label) {
+ branchTestObjectImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const Address& address,
+ Label* label) {
+ branchTestObjectImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestObjectImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestObjectImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestObjectImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testObject(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestGCThingImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& src,
+ Label* label) {
+ Condition c = testGCThing(cond, src);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
+ Label* label) {
+ branchTestPrimitiveImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestPrimitiveImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestPrimitiveImpl(Condition cond, const T& t,
+ Label* label) {
+ Condition c = testPrimitive(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, Register tag,
+ Label* label) {
+ branchTestMagicImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
+ Label* label) {
+ branchTestMagicImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestMagicImpl(cond, address, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
+ L label) {
+ branchTestMagicImpl(cond, value, label);
+}
+
+template <typename T, class L>
+void MacroAssembler::branchTestMagicImpl(Condition cond, const T& t, L label) {
+ Condition c = testMagic(cond, t);
+ B(label, c);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ uint64_t magic = MagicValue(why).asRawBits();
+ cmpPtr(valaddr, ImmWord(magic));
+ B(label, cond);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ branchPtr(cond, lhs, rhs.valueReg(), label);
+}
+
+template <typename T>
+void MacroAssembler::testNumberSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testNumber(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testBooleanSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testBoolean(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testStringSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testString(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testSymbolSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testSymbol(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testBigIntSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testBigInt(cond, src);
+ emitSet(cond, dest);
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
+ vixl::UseScratchRegisterScope temps(&this->asVIXL());
+ const ARMRegister scratch64 = temps.AcquireX();
+ loadPtr(addr, scratch64.asUnsized());
+ Br(scratch64);
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmp32(lhs, rhs);
+ Csel(ARMRegister(dest, 32), ARMRegister(src, 32), ARMRegister(dest, 32),
+ cond);
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ MOZ_CRASH("NYI");
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmpPtr(lhs, rhs);
+ Csel(ARMRegister(dest, 64), ARMRegister(src, 64), ARMRegister(dest, 64),
+ cond);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ MOZ_CRASH("NYI");
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
+ const Address& rhs, const Address& src,
+ Register dest) {
+ MOZ_CRASH("NYI");
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
+ const Address& src, Register dest) {
+ MOZ_CRASH("NYI");
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ cmp32(lhs, rhs);
+ Csel(ARMRegister(dest, 64), ARMRegister(src, 64), ARMRegister(dest, 64),
+ cond);
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ // ARM64 does not support conditional loads, so we use a branch with a CSel
+ // (to prevent Spectre attacks).
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+
+ // Can't use branch32() here, because it may select Cbz/Cbnz which don't
+ // affect condition flags.
+ Label done;
+ cmp32(lhs, rhs);
+ B(&done, Assembler::InvertCondition(cond));
+
+ loadPtr(src, scratch64.asUnsized());
+ Csel(ARMRegister(dest, 64), scratch64, ARMRegister(dest, 64), cond);
+ bind(&done);
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+ Imm32 mask, const Address& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+
+ // ARM64 does not support conditional loads, so we use a branch with a CSel
+ // (to prevent Spectre attacks).
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ Label done;
+ branchTest32(Assembler::InvertCondition(cond), addr, mask, &done);
+ loadPtr(src, scratch64.asUnsized());
+ Csel(ARMRegister(dest, 64), scratch64, ARMRegister(dest, 64), cond);
+ bind(&done);
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+ Imm32 mask, Register src, Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ Csel(ARMRegister(dest, 64), ARMRegister(src, 64), ARMRegister(dest, 64),
+ cond);
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+ Register dest) {
+ Csel(ARMRegister(dest, 64), ARMRegister(src, 64), ARMRegister(dest, 64),
+ cond);
+}
+
+void MacroAssembler::spectreZeroRegister(Condition cond, Register,
+ Register dest) {
+ Csel(ARMRegister(dest, 64), ARMRegister(dest, 64), vixl::xzr,
+ Assembler::InvertCondition(cond));
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(length != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ Csel(ARMRegister(index, 32), ARMRegister(index, 32), vixl::wzr,
+ Assembler::Above);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ Csel(ARMRegister(index, 32), ARMRegister(index, 32), vixl::wzr,
+ Assembler::Above);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(length != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ branchPtr(Assembler::BelowOrEqual, length, index, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ Csel(ARMRegister(index, 64), ARMRegister(index, 64), vixl::xzr,
+ Assembler::Above);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+ const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ branchPtr(Assembler::BelowOrEqual, length, index, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ Csel(ARMRegister(index, 64), ARMRegister(index, 64), vixl::xzr,
+ Assembler::Above);
+ }
+}
+
+// ========================================================================
+// Memory access primitives.
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const Address& dest) {
+ Str(ARMFPRegister(src, 64), toMemOperand(dest));
+}
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const BaseIndex& dest) {
+ doBaseIndex(ARMFPRegister(src, 64), dest, vixl::STR_d);
+}
+
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const Address& addr) {
+ Str(ARMFPRegister(src, 32), toMemOperand(addr));
+}
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const BaseIndex& addr) {
+ doBaseIndex(ARMFPRegister(src, 32), addr, vixl::STR_s);
+}
+
+void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) {
+ // Bug 1715494: Discriminating barriers such as StoreStore are hard to reason
+ // about. Execute the full barrier for everything that requires a barrier.
+ if (barrier) {
+ Dmb(vixl::InnerShareable, vixl::BarrierAll);
+ }
+}
+
+// ===============================================================
+// Clamping functions.
+
+void MacroAssembler::clampIntToUint8(Register reg) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ const ARMRegister reg32(reg, 32);
+ MOZ_ASSERT(!scratch32.Is(reg32));
+
+ Cmp(reg32, Operand(reg32, vixl::UXTB));
+ Csel(reg32, reg32, vixl::wzr, Assembler::GreaterThanOrEqual);
+ Mov(scratch32, Operand(0xff));
+ Csel(reg32, reg32, scratch32, Assembler::LessThanOrEqual);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) {
+ MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
+ type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
+ // dest := src XOR mask
+ // fail if dest >> JSVAL_TAG_SHIFT != 0
+ const ARMRegister src64(src.valueReg(), 64);
+ const ARMRegister dest64(dest, 64);
+ Eor(dest64, src64, Operand(JSVAL_TYPE_TO_SHIFTED_TAG(type)));
+ Cmp(vixl::xzr, Operand(dest64, vixl::LSR, JSVAL_TAG_SHIFT));
+ j(Assembler::NotEqual, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) {
+ loadValue(src, ValueOperand(dest));
+ fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) {
+ loadValue(src, ValueOperand(dest));
+ fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
+}
+
+//}}} check_macroassembler_style
+
+// Wasm SIMD
+
+static inline ARMFPRegister SimdReg(FloatRegister r) {
+ MOZ_ASSERT(r.isSimd128());
+ return ARMFPRegister(r, 128);
+}
+
+static inline ARMFPRegister Simd16B(FloatRegister r) {
+ return SimdReg(r).V16B();
+}
+
+static inline ARMFPRegister Simd8B(FloatRegister r) { return SimdReg(r).V8B(); }
+
+static inline ARMFPRegister Simd8H(FloatRegister r) { return SimdReg(r).V8H(); }
+
+static inline ARMFPRegister Simd4H(FloatRegister r) { return SimdReg(r).V4H(); }
+
+static inline ARMFPRegister Simd4S(FloatRegister r) { return SimdReg(r).V4S(); }
+
+static inline ARMFPRegister Simd2S(FloatRegister r) { return SimdReg(r).V2S(); }
+
+static inline ARMFPRegister Simd2D(FloatRegister r) { return SimdReg(r).V2D(); }
+
+static inline ARMFPRegister Simd1D(FloatRegister r) { return SimdReg(r).V1D(); }
+
+static inline ARMFPRegister SimdQ(FloatRegister r) { return SimdReg(r).Q(); }
+
+//{{{ check_macroassembler_style
+
+// Moves
+
+void MacroAssembler::moveSimd128(FloatRegister src, FloatRegister dest) {
+ if (src != dest) {
+ Mov(SimdReg(dest), SimdReg(src));
+ }
+}
+
+void MacroAssembler::loadConstantSimd128(const SimdConstant& v,
+ FloatRegister dest) {
+ // Movi does not yet generate good code for many cases, bug 1664397.
+ SimdConstant c = SimdConstant::CreateX2((const int64_t*)v.bytes());
+ Movi(SimdReg(dest), c.asInt64x2()[1], c.asInt64x2()[0]);
+}
+
+// Splat
+
+void MacroAssembler::splatX16(Register src, FloatRegister dest) {
+ Dup(Simd16B(dest), ARMRegister(src, 32));
+}
+
+void MacroAssembler::splatX16(uint32_t srcLane, FloatRegister src,
+ FloatRegister dest) {
+ Dup(Simd16B(dest), Simd16B(src), srcLane);
+}
+
+void MacroAssembler::splatX8(Register src, FloatRegister dest) {
+ Dup(Simd8H(dest), ARMRegister(src, 32));
+}
+
+void MacroAssembler::splatX8(uint32_t srcLane, FloatRegister src,
+ FloatRegister dest) {
+ Dup(Simd8H(dest), Simd8H(src), srcLane);
+}
+
+void MacroAssembler::splatX4(Register src, FloatRegister dest) {
+ Dup(Simd4S(dest), ARMRegister(src, 32));
+}
+
+void MacroAssembler::splatX4(FloatRegister src, FloatRegister dest) {
+ Dup(Simd4S(dest), ARMFPRegister(src), 0);
+}
+
+void MacroAssembler::splatX2(Register64 src, FloatRegister dest) {
+ Dup(Simd2D(dest), ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::splatX2(FloatRegister src, FloatRegister dest) {
+ Dup(Simd2D(dest), ARMFPRegister(src), 0);
+}
+
+// Extract lane as scalar. Float extraction does not canonicalize the value.
+
+void MacroAssembler::extractLaneInt8x16(uint32_t lane, FloatRegister src,
+ Register dest_) {
+ MOZ_ASSERT(lane < 16);
+ ARMRegister dest(dest_, 32);
+ Umov(dest, Simd4S(src), lane / 4);
+ Sbfx(dest, dest, (lane % 4) * 8, 8);
+}
+
+void MacroAssembler::unsignedExtractLaneInt8x16(uint32_t lane,
+ FloatRegister src,
+ Register dest_) {
+ MOZ_ASSERT(lane < 16);
+ ARMRegister dest(dest_, 32);
+ Umov(dest, Simd4S(src), lane / 4);
+ Ubfx(dest, dest, (lane % 4) * 8, 8);
+}
+
+void MacroAssembler::extractLaneInt16x8(uint32_t lane, FloatRegister src,
+ Register dest_) {
+ MOZ_ASSERT(lane < 8);
+ ARMRegister dest(dest_, 32);
+ Umov(dest, Simd4S(src), lane / 2);
+ Sbfx(dest, dest, (lane % 2) * 16, 16);
+}
+
+void MacroAssembler::unsignedExtractLaneInt16x8(uint32_t lane,
+ FloatRegister src,
+ Register dest_) {
+ MOZ_ASSERT(lane < 8);
+ ARMRegister dest(dest_, 32);
+ Umov(dest, Simd4S(src), lane / 2);
+ Ubfx(dest, dest, (lane % 2) * 16, 16);
+}
+
+void MacroAssembler::extractLaneInt32x4(uint32_t lane, FloatRegister src,
+ Register dest_) {
+ MOZ_ASSERT(lane < 4);
+ ARMRegister dest(dest_, 32);
+ Umov(dest, Simd4S(src), lane);
+}
+
+void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src,
+ Register64 dest_) {
+ MOZ_ASSERT(lane < 2);
+ ARMRegister dest(dest_.reg, 64);
+ Umov(dest, Simd2D(src), lane);
+}
+
+void MacroAssembler::extractLaneFloat32x4(uint32_t lane, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(lane < 4);
+ Mov(ARMFPRegister(dest).V4S(), 0, Simd4S(src), lane);
+}
+
+void MacroAssembler::extractLaneFloat64x2(uint32_t lane, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(lane < 2);
+ Mov(ARMFPRegister(dest).V2D(), 0, Simd2D(src), lane);
+}
+
+// Replace lane value
+
+void MacroAssembler::replaceLaneInt8x16(unsigned lane, Register rhs,
+ FloatRegister lhsDest) {
+ MOZ_ASSERT(lane < 16);
+ Mov(Simd16B(lhsDest), lane, ARMRegister(rhs, 32));
+}
+
+void MacroAssembler::replaceLaneInt16x8(unsigned lane, Register rhs,
+ FloatRegister lhsDest) {
+ MOZ_ASSERT(lane < 8);
+ Mov(Simd8H(lhsDest), lane, ARMRegister(rhs, 32));
+}
+
+void MacroAssembler::replaceLaneInt32x4(unsigned lane, Register rhs,
+ FloatRegister lhsDest) {
+ MOZ_ASSERT(lane < 4);
+ Mov(Simd4S(lhsDest), lane, ARMRegister(rhs, 32));
+}
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs,
+ FloatRegister lhsDest) {
+ MOZ_ASSERT(lane < 2);
+ Mov(Simd2D(lhsDest), lane, ARMRegister(rhs.reg, 64));
+}
+
+void MacroAssembler::replaceLaneFloat32x4(unsigned lane, FloatRegister rhs,
+ FloatRegister lhsDest) {
+ MOZ_ASSERT(lane < 4);
+ Mov(Simd4S(lhsDest), lane, ARMFPRegister(rhs).V4S(), 0);
+}
+
+void MacroAssembler::replaceLaneFloat64x2(unsigned lane, FloatRegister rhs,
+ FloatRegister lhsDest) {
+ MOZ_ASSERT(lane < 2);
+ Mov(Simd2D(lhsDest), lane, ARMFPRegister(rhs).V2D(), 0);
+}
+
+// Shuffle - blend and permute with immediate indices, and its many
+// specializations. Lane values other than those mentioned are illegal.
+
+// lane values 0..31
+void MacroAssembler::shuffleInt8x16(const uint8_t lanes[16], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest) {
+ // The general solution generates ho-hum code. Realistic programs will use
+ // patterns that can be specialized, and this will be much better. That will
+ // be handled by bug 1656834, so don't worry about it here.
+
+ // Set scratch to the lanevalue when it selects from lhs or ~lanevalue when it
+ // selects from rhs.
+ ScratchSimd128Scope scratch(*this);
+ int8_t idx[16];
+
+ if (lhs == rhs) {
+ for (unsigned i = 0; i < 16; i++) {
+ idx[i] = lanes[i] < 16 ? lanes[i] : (lanes[i] - 16);
+ }
+ loadConstantSimd128(SimdConstant::CreateX16(idx), scratch);
+ Tbl(Simd16B(dest), Simd16B(lhs), Simd16B(scratch));
+ return;
+ }
+
+ if (rhs != dest) {
+ for (unsigned i = 0; i < 16; i++) {
+ idx[i] = lanes[i] < 16 ? lanes[i] : ~(lanes[i] - 16);
+ }
+ } else {
+ MOZ_ASSERT(lhs != dest);
+ for (unsigned i = 0; i < 16; i++) {
+ idx[i] = lanes[i] < 16 ? ~lanes[i] : (lanes[i] - 16);
+ }
+ std::swap(lhs, rhs);
+ }
+ loadConstantSimd128(SimdConstant::CreateX16(idx), scratch);
+ Tbl(Simd16B(dest), Simd16B(lhs), Simd16B(scratch));
+ Not(Simd16B(scratch), Simd16B(scratch));
+ Tbx(Simd16B(dest), Simd16B(rhs), Simd16B(scratch));
+}
+
+void MacroAssembler::shuffleInt8x16(const uint8_t lanes[16], FloatRegister rhs,
+ FloatRegister lhsDest) {
+ shuffleInt8x16(lanes, lhsDest, rhs, lhsDest);
+}
+
+void MacroAssembler::blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ int8_t lanes_[16];
+
+ if (rhs == dest) {
+ for (unsigned i = 0; i < 16; i++) {
+ lanes_[i] = lanes[i] == 0 ? i : 16 + i;
+ }
+ loadConstantSimd128(SimdConstant::CreateX16(lanes_), scratch);
+ Tbx(Simd16B(dest), Simd16B(lhs), Simd16B(scratch));
+ return;
+ }
+
+ moveSimd128(lhs, dest);
+ for (unsigned i = 0; i < 16; i++) {
+ lanes_[i] = lanes[i] != 0 ? i : 16 + i;
+ }
+ loadConstantSimd128(SimdConstant::CreateX16(lanes_), scratch);
+ Tbx(Simd16B(dest), Simd16B(rhs), Simd16B(scratch));
+}
+
+void MacroAssembler::blendInt16x8(const uint16_t lanes[8], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest) {
+ static_assert(sizeof(const uint16_t /*lanes*/[8]) == sizeof(uint8_t[16]));
+ blendInt8x16(reinterpret_cast<const uint8_t*>(lanes), lhs, rhs, dest);
+}
+
+void MacroAssembler::laneSelectSimd128(FloatRegister mask, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest) {
+ MOZ_ASSERT(mask == dest);
+ Bsl(Simd16B(mask), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::interleaveHighInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Zip2(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::interleaveHighInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Zip2(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::interleaveHighInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Zip2(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+void MacroAssembler::interleaveHighInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Zip2(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::interleaveLowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Zip1(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::interleaveLowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Zip1(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::interleaveLowInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Zip1(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+void MacroAssembler::interleaveLowInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Zip1(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::permuteInt8x16(const uint8_t lanes[16], FloatRegister src,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ loadConstantSimd128(SimdConstant::CreateX16((const int8_t*)lanes), scratch);
+ Tbl(Simd16B(dest), Simd16B(src), Simd16B(scratch));
+}
+
+void MacroAssembler::permuteInt16x8(const uint16_t lanes[8], FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(lanes[0] < 8 && lanes[1] < 8 && lanes[2] < 8 && lanes[3] < 8 &&
+ lanes[4] < 8 && lanes[5] < 8 && lanes[6] < 8 && lanes[7] < 8);
+ const int8_t lanes_[16] = {
+ (int8_t)(lanes[0] << 1), (int8_t)((lanes[0] << 1) + 1),
+ (int8_t)(lanes[1] << 1), (int8_t)((lanes[1] << 1) + 1),
+ (int8_t)(lanes[2] << 1), (int8_t)((lanes[2] << 1) + 1),
+ (int8_t)(lanes[3] << 1), (int8_t)((lanes[3] << 1) + 1),
+ (int8_t)(lanes[4] << 1), (int8_t)((lanes[4] << 1) + 1),
+ (int8_t)(lanes[5] << 1), (int8_t)((lanes[5] << 1) + 1),
+ (int8_t)(lanes[6] << 1), (int8_t)((lanes[6] << 1) + 1),
+ (int8_t)(lanes[7] << 1), (int8_t)((lanes[7] << 1) + 1),
+ };
+ ScratchSimd128Scope scratch(*this);
+ loadConstantSimd128(SimdConstant::CreateX16(lanes_), scratch);
+ Tbl(Simd16B(dest), Simd16B(src), Simd16B(scratch));
+}
+
+void MacroAssembler::permuteInt32x4(const uint32_t lanes[4], FloatRegister src,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ const int8_t lanes_[16] = {
+ (int8_t)(lanes[0] << 2), (int8_t)((lanes[0] << 2) + 1),
+ (int8_t)((lanes[0] << 2) + 2), (int8_t)((lanes[0] << 2) + 3),
+ (int8_t)(lanes[1] << 2), (int8_t)((lanes[1] << 2) + 1),
+ (int8_t)((lanes[1] << 2) + 2), (int8_t)((lanes[1] << 2) + 3),
+ (int8_t)(lanes[2] << 2), (int8_t)((lanes[2] << 2) + 1),
+ (int8_t)((lanes[2] << 2) + 2), (int8_t)((lanes[2] << 2) + 3),
+ (int8_t)(lanes[3] << 2), (int8_t)((lanes[3] << 2) + 1),
+ (int8_t)((lanes[3] << 2) + 2), (int8_t)((lanes[3] << 2) + 3),
+ };
+ loadConstantSimd128(SimdConstant::CreateX16(lanes_), scratch);
+ Tbl(Simd16B(dest), Simd16B(src), Simd16B(scratch));
+}
+
+void MacroAssembler::rotateRightSimd128(FloatRegister src, FloatRegister dest,
+ uint32_t shift) {
+ Ext(Simd16B(dest), Simd16B(src), Simd16B(src), shift);
+}
+
+void MacroAssembler::leftShiftSimd128(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(count.value < 16);
+ ScratchSimd128Scope scratch(*this);
+ Movi(Simd16B(scratch), 0);
+ Ext(Simd16B(dest), Simd16B(scratch), Simd16B(src), 16 - count.value);
+}
+
+void MacroAssembler::rightShiftSimd128(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(count.value < 16);
+ ScratchSimd128Scope scratch(*this);
+ Movi(Simd16B(scratch), 0);
+ Ext(Simd16B(dest), Simd16B(src), Simd16B(scratch), count.value);
+}
+
+void MacroAssembler::concatAndRightShiftSimd128(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest,
+ uint32_t shift) {
+ MOZ_ASSERT(shift < 16);
+ Ext(Simd16B(dest), Simd16B(rhs), Simd16B(lhs), shift);
+}
+
+// Reverse bytes in lanes.
+
+void MacroAssembler::reverseInt16x8(FloatRegister src, FloatRegister dest) {
+ Rev16(Simd16B(dest), Simd16B(src));
+}
+
+void MacroAssembler::reverseInt32x4(FloatRegister src, FloatRegister dest) {
+ Rev32(Simd16B(dest), Simd16B(src));
+}
+
+void MacroAssembler::reverseInt64x2(FloatRegister src, FloatRegister dest) {
+ Rev64(Simd16B(dest), Simd16B(src));
+}
+
+// Swizzle - permute with variable indices. `rhs` holds the lanes parameter.
+
+void MacroAssembler::swizzleInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Tbl(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::swizzleInt8x16Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Tbl(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+// Integer Add
+
+void MacroAssembler::addInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Add(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::addInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Add(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::addInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Add(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::addInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Add(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+// Integer Subtract
+
+void MacroAssembler::subInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sub(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::subInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sub(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::subInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sub(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::subInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sub(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+// Integer Multiply
+
+void MacroAssembler::mulInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Mul(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::mulInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Mul(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::mulInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) {
+ // As documented at https://chromium-review.googlesource.com/c/v8/v8/+/1781696
+ // lhs = <D C> <B A>
+ // rhs = <H G> <F E>
+ // result = <(DG+CH)_low+CG_high CG_low> <(BE+AF)_low+AE_high AE_low>
+ ScratchSimd128Scope scratch(*this);
+ Rev64(Simd4S(temp2), Simd4S(lhs)); // temp2 = <C D> <A B>
+ Mul(Simd4S(temp2), Simd4S(temp2), Simd4S(rhs)); // temp2 = <CH DG> <AF BE>
+ Xtn(Simd2S(temp1), Simd2D(rhs)); // temp1 = <0 0> <G E>
+ Addp(Simd4S(temp2), Simd4S(temp2), Simd4S(temp2)); // temp2 = <CH+DG AF+BE>..
+ Xtn(Simd2S(scratch), Simd2D(lhs)); // scratch = <0 0> <C A>
+ Shll(Simd2D(dest), Simd2S(temp2), 32); // dest = <(DG+CH)_low 0>
+ // <(BE+AF)_low 0>
+ Umlal(Simd2D(dest), Simd2S(scratch), Simd2S(temp1));
+}
+
+void MacroAssembler::extMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smull(Simd8H(dest), Simd8B(lhs), Simd8B(rhs));
+}
+
+void MacroAssembler::extMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smull2(Simd8H(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::unsignedExtMulLowInt8x16(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ Umull(Simd8H(dest), Simd8B(lhs), Simd8B(rhs));
+}
+
+void MacroAssembler::unsignedExtMulHighInt8x16(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ Umull2(Simd8H(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::extMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smull(Simd4S(dest), Simd4H(lhs), Simd4H(rhs));
+}
+
+void MacroAssembler::extMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smull2(Simd4S(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::unsignedExtMulLowInt16x8(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ Umull(Simd4S(dest), Simd4H(lhs), Simd4H(rhs));
+}
+
+void MacroAssembler::unsignedExtMulHighInt16x8(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ Umull2(Simd4S(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::extMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smull(Simd2D(dest), Simd2S(lhs), Simd2S(rhs));
+}
+
+void MacroAssembler::extMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smull2(Simd2D(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::unsignedExtMulLowInt32x4(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ Umull(Simd2D(dest), Simd2S(lhs), Simd2S(rhs));
+}
+
+void MacroAssembler::unsignedExtMulHighInt32x4(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ Umull2(Simd2D(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::q15MulrSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sqrdmulh(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::q15MulrInt16x8Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sqrdmulh(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+// Integer Negate
+
+void MacroAssembler::negInt8x16(FloatRegister src, FloatRegister dest) {
+ Neg(Simd16B(dest), Simd16B(src));
+}
+
+void MacroAssembler::negInt16x8(FloatRegister src, FloatRegister dest) {
+ Neg(Simd8H(dest), Simd8H(src));
+}
+
+void MacroAssembler::negInt32x4(FloatRegister src, FloatRegister dest) {
+ Neg(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::negInt64x2(FloatRegister src, FloatRegister dest) {
+ Neg(Simd2D(dest), Simd2D(src));
+}
+
+// Saturating integer add
+
+void MacroAssembler::addSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sqadd(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::unsignedAddSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Uqadd(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::addSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sqadd(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::unsignedAddSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Uqadd(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+// Saturating integer subtract
+
+void MacroAssembler::subSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sqsub(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::unsignedSubSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Uqsub(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::subSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Sqsub(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::unsignedSubSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Uqsub(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+// Lane-wise integer minimum
+
+void MacroAssembler::minInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smin(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::unsignedMinInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Umin(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::minInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smin(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::unsignedMinInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Umin(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::minInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smin(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::unsignedMinInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Umin(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+// Lane-wise integer maximum
+
+void MacroAssembler::maxInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smax(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::unsignedMaxInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Umax(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::maxInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smax(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::unsignedMaxInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Umax(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::maxInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Smax(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::unsignedMaxInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Umax(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+// Lane-wise integer rounding average
+
+void MacroAssembler::unsignedAverageInt8x16(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ Urhadd(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::unsignedAverageInt16x8(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ Urhadd(Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+// Lane-wise integer absolute value
+
+void MacroAssembler::absInt8x16(FloatRegister src, FloatRegister dest) {
+ Abs(Simd16B(dest), Simd16B(src));
+}
+
+void MacroAssembler::absInt16x8(FloatRegister src, FloatRegister dest) {
+ Abs(Simd8H(dest), Simd8H(src));
+}
+
+void MacroAssembler::absInt32x4(FloatRegister src, FloatRegister dest) {
+ Abs(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::absInt64x2(FloatRegister src, FloatRegister dest) {
+ Abs(Simd2D(dest), Simd2D(src));
+}
+
+// Left shift by variable scalar
+
+void MacroAssembler::leftShiftInt8x16(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope vscratch(*this);
+ Dup(Simd16B(vscratch), ARMRegister(rhs, 32));
+ Sshl(Simd16B(dest), Simd16B(lhs), Simd16B(vscratch));
+}
+
+void MacroAssembler::leftShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Shl(Simd16B(dest), Simd16B(src), count.value);
+}
+
+void MacroAssembler::leftShiftInt16x8(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope vscratch(*this);
+ Dup(Simd8H(vscratch), ARMRegister(rhs, 32));
+ Sshl(Simd8H(dest), Simd8H(lhs), Simd8H(vscratch));
+}
+
+void MacroAssembler::leftShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Shl(Simd8H(dest), Simd8H(src), count.value);
+}
+
+void MacroAssembler::leftShiftInt32x4(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope vscratch(*this);
+ Dup(Simd4S(vscratch), ARMRegister(rhs, 32));
+ Sshl(Simd4S(dest), Simd4S(lhs), Simd4S(vscratch));
+}
+
+void MacroAssembler::leftShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Shl(Simd4S(dest), Simd4S(src), count.value);
+}
+
+void MacroAssembler::leftShiftInt64x2(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope vscratch(*this);
+ Dup(Simd2D(vscratch), ARMRegister(rhs, 64));
+ Sshl(Simd2D(dest), Simd2D(lhs), Simd2D(vscratch));
+}
+
+void MacroAssembler::leftShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Shl(Simd2D(dest), Simd2D(src), count.value);
+}
+
+// Right shift by variable scalar
+
+void MacroAssembler::rightShiftInt8x16(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ MacroAssemblerCompat::rightShiftInt8x16(lhs, rhs, dest,
+ /* isUnsigned */ false);
+}
+
+void MacroAssembler::rightShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Sshr(Simd16B(dest), Simd16B(src), count.value);
+}
+
+void MacroAssembler::unsignedRightShiftInt8x16(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ MacroAssemblerCompat::rightShiftInt8x16(lhs, rhs, dest,
+ /* isUnsigned */ true);
+}
+
+void MacroAssembler::unsignedRightShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Ushr(Simd16B(dest), Simd16B(src), count.value);
+}
+
+void MacroAssembler::rightShiftInt16x8(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ MacroAssemblerCompat::rightShiftInt16x8(lhs, rhs, dest,
+ /* isUnsigned */ false);
+}
+
+void MacroAssembler::rightShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Sshr(Simd8H(dest), Simd8H(src), count.value);
+}
+
+void MacroAssembler::unsignedRightShiftInt16x8(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ MacroAssemblerCompat::rightShiftInt16x8(lhs, rhs, dest,
+ /* isUnsigned */ true);
+}
+
+void MacroAssembler::unsignedRightShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Ushr(Simd8H(dest), Simd8H(src), count.value);
+}
+
+void MacroAssembler::rightShiftInt32x4(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ MacroAssemblerCompat::rightShiftInt32x4(lhs, rhs, dest,
+ /* isUnsigned */ false);
+}
+
+void MacroAssembler::rightShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Sshr(Simd4S(dest), Simd4S(src), count.value);
+}
+
+void MacroAssembler::unsignedRightShiftInt32x4(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ MacroAssemblerCompat::rightShiftInt32x4(lhs, rhs, dest,
+ /* isUnsigned */ true);
+}
+
+void MacroAssembler::unsignedRightShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Ushr(Simd4S(dest), Simd4S(src), count.value);
+}
+
+void MacroAssembler::rightShiftInt64x2(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ MacroAssemblerCompat::rightShiftInt64x2(lhs, rhs, dest,
+ /* isUnsigned */ false);
+}
+
+void MacroAssembler::rightShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Sshr(Simd2D(dest), Simd2D(src), count.value);
+}
+
+void MacroAssembler::unsignedRightShiftInt64x2(FloatRegister lhs, Register rhs,
+ FloatRegister dest) {
+ MacroAssemblerCompat::rightShiftInt64x2(lhs, rhs, dest,
+ /* isUnsigned */ true);
+}
+
+void MacroAssembler::unsignedRightShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ Ushr(Simd2D(dest), Simd2D(src), count.value);
+}
+
+// Bitwise and, or, xor, not
+
+void MacroAssembler::bitwiseAndSimd128(FloatRegister rhs,
+ FloatRegister lhsDest) {
+ And(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
+}
+
+void MacroAssembler::bitwiseAndSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ And(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::bitwiseOrSimd128(FloatRegister rhs,
+ FloatRegister lhsDest) {
+ Orr(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
+}
+
+void MacroAssembler::bitwiseOrSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Orr(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::bitwiseXorSimd128(FloatRegister rhs,
+ FloatRegister lhsDest) {
+ Eor(Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
+}
+
+void MacroAssembler::bitwiseXorSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Eor(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::bitwiseNotSimd128(FloatRegister src, FloatRegister dest) {
+ Not(Simd16B(dest), Simd16B(src));
+}
+
+void MacroAssembler::bitwiseAndNotSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Bic(Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+// Bitwise AND with complement: dest = ~lhs & rhs, note this is not what Wasm
+// wants but what the x86 hardware offers. Hence the name. Since arm64 has
+// dest = lhs & ~rhs we just swap operands.
+
+void MacroAssembler::bitwiseNotAndSimd128(FloatRegister rhs,
+ FloatRegister lhsDest) {
+ Bic(Simd16B(lhsDest), Simd16B(rhs), Simd16B(lhsDest));
+}
+
+// Bitwise select
+
+void MacroAssembler::bitwiseSelectSimd128(FloatRegister onTrue,
+ FloatRegister onFalse,
+ FloatRegister maskDest) {
+ Bsl(Simd16B(maskDest), Simd16B(onTrue), Simd16B(onFalse));
+}
+
+// Population count
+
+void MacroAssembler::popcntInt8x16(FloatRegister src, FloatRegister dest) {
+ Cnt(Simd16B(dest), Simd16B(src));
+}
+
+// Any lane true, ie, any bit set
+
+void MacroAssembler::anyTrueSimd128(FloatRegister src, Register dest_) {
+ ScratchSimd128Scope scratch_(*this);
+ ARMFPRegister scratch(Simd1D(scratch_));
+ ARMRegister dest(dest_, 64);
+ Addp(scratch, Simd2D(src));
+ Umov(dest, scratch, 0);
+ Cmp(dest, Operand(0));
+ Cset(dest, Assembler::NonZero);
+}
+
+// All lanes true
+
+void MacroAssembler::allTrueInt8x16(FloatRegister src, Register dest_) {
+ ScratchSimd128Scope scratch(*this);
+ ARMRegister dest(dest_, 64);
+ Cmeq(Simd16B(scratch), Simd16B(src), 0);
+ Addp(Simd1D(scratch), Simd2D(scratch));
+ Umov(dest, Simd1D(scratch), 0);
+ Cmp(dest, Operand(0));
+ Cset(dest, Assembler::Zero);
+}
+
+void MacroAssembler::allTrueInt16x8(FloatRegister src, Register dest_) {
+ ScratchSimd128Scope scratch(*this);
+ ARMRegister dest(dest_, 64);
+ Cmeq(Simd8H(scratch), Simd8H(src), 0);
+ Addp(Simd1D(scratch), Simd2D(scratch));
+ Umov(dest, Simd1D(scratch), 0);
+ Cmp(dest, Operand(0));
+ Cset(dest, Assembler::Zero);
+}
+
+void MacroAssembler::allTrueInt32x4(FloatRegister src, Register dest_) {
+ ScratchSimd128Scope scratch(*this);
+ ARMRegister dest(dest_, 64);
+ Cmeq(Simd4S(scratch), Simd4S(src), 0);
+ Addp(Simd1D(scratch), Simd2D(scratch));
+ Umov(dest, Simd1D(scratch), 0);
+ Cmp(dest, Operand(0));
+ Cset(dest, Assembler::Zero);
+}
+
+void MacroAssembler::allTrueInt64x2(FloatRegister src, Register dest_) {
+ ScratchSimd128Scope scratch(*this);
+ ARMRegister dest(dest_, 64);
+ Cmeq(Simd2D(scratch), Simd2D(src), 0);
+ Addp(Simd1D(scratch), Simd2D(scratch));
+ Umov(dest, Simd1D(scratch), 0);
+ Cmp(dest, Operand(0));
+ Cset(dest, Assembler::Zero);
+}
+
+// Bitmask, ie extract and compress high bits of all lanes
+//
+// There's no direct support for this on the chip. These implementations come
+// from the writeup that added the instruction to the SIMD instruction set.
+// Generally, shifting and masking is used to isolate the sign bit of each
+// element in the right position, then a horizontal add creates the result. For
+// 8-bit elements an intermediate step is needed to assemble the bits of the
+// upper and lower 8 bytes into 8 halfwords.
+
+void MacroAssembler::bitmaskInt8x16(FloatRegister src, Register dest,
+ FloatRegister temp) {
+ ScratchSimd128Scope scratch(*this);
+ int8_t values[] = {1, 2, 4, 8, 16, 32, 64, -128,
+ 1, 2, 4, 8, 16, 32, 64, -128};
+ loadConstantSimd128(SimdConstant::CreateX16(values), temp);
+ Sshr(Simd16B(scratch), Simd16B(src), 7);
+ And(Simd16B(scratch), Simd16B(scratch), Simd16B(temp));
+ Ext(Simd16B(temp), Simd16B(scratch), Simd16B(scratch), 8);
+ Zip1(Simd16B(temp), Simd16B(scratch), Simd16B(temp));
+ Addv(ARMFPRegister(temp, 16), Simd8H(temp));
+ Mov(ARMRegister(dest, 32), Simd8H(temp), 0);
+}
+
+void MacroAssembler::bitmaskInt16x8(FloatRegister src, Register dest,
+ FloatRegister temp) {
+ ScratchSimd128Scope scratch(*this);
+ int16_t values[] = {1, 2, 4, 8, 16, 32, 64, 128};
+ loadConstantSimd128(SimdConstant::CreateX8(values), temp);
+ Sshr(Simd8H(scratch), Simd8H(src), 15);
+ And(Simd16B(scratch), Simd16B(scratch), Simd16B(temp));
+ Addv(ARMFPRegister(scratch, 16), Simd8H(scratch));
+ Mov(ARMRegister(dest, 32), Simd8H(scratch), 0);
+}
+
+void MacroAssembler::bitmaskInt32x4(FloatRegister src, Register dest,
+ FloatRegister temp) {
+ ScratchSimd128Scope scratch(*this);
+ int32_t values[] = {1, 2, 4, 8};
+ loadConstantSimd128(SimdConstant::CreateX4(values), temp);
+ Sshr(Simd4S(scratch), Simd4S(src), 31);
+ And(Simd16B(scratch), Simd16B(scratch), Simd16B(temp));
+ Addv(ARMFPRegister(scratch, 32), Simd4S(scratch));
+ Mov(ARMRegister(dest, 32), Simd4S(scratch), 0);
+}
+
+void MacroAssembler::bitmaskInt64x2(FloatRegister src, Register dest,
+ FloatRegister temp) {
+ Sqxtn(Simd2S(temp), Simd2D(src));
+ Ushr(Simd2S(temp), Simd2S(temp), 31);
+ Usra(ARMFPRegister(temp, 64), ARMFPRegister(temp, 64), 31);
+ Fmov(ARMRegister(dest, 32), ARMFPRegister(temp, 32));
+}
+
+// Comparisons (integer and floating-point)
+
+void MacroAssembler::compareInt8x16(Assembler::Condition cond,
+ FloatRegister rhs, FloatRegister lhsDest) {
+ compareSimd128Int(cond, Simd16B(lhsDest), Simd16B(lhsDest), Simd16B(rhs));
+}
+
+void MacroAssembler::compareInt8x16(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ compareSimd128Int(cond, Simd16B(dest), Simd16B(lhs), Simd16B(rhs));
+}
+
+void MacroAssembler::compareInt16x8(Assembler::Condition cond,
+ FloatRegister rhs, FloatRegister lhsDest) {
+ compareSimd128Int(cond, Simd8H(lhsDest), Simd8H(lhsDest), Simd8H(rhs));
+}
+
+void MacroAssembler::compareInt16x8(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ compareSimd128Int(cond, Simd8H(dest), Simd8H(lhs), Simd8H(rhs));
+}
+
+void MacroAssembler::compareInt32x4(Assembler::Condition cond,
+ FloatRegister rhs, FloatRegister lhsDest) {
+ compareSimd128Int(cond, Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
+}
+
+void MacroAssembler::compareInt32x4(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ compareSimd128Int(cond, Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::compareInt64x2(Assembler::Condition cond,
+ FloatRegister rhs, FloatRegister lhsDest) {
+ compareSimd128Int(cond, Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
+}
+
+void MacroAssembler::compareInt64x2(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ compareSimd128Int(cond, Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+void MacroAssembler::compareFloat32x4(Assembler::Condition cond,
+ FloatRegister rhs,
+ FloatRegister lhsDest) {
+ compareSimd128Float(cond, Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
+}
+
+void MacroAssembler::compareFloat32x4(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ compareSimd128Float(cond, Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::compareFloat64x2(Assembler::Condition cond,
+ FloatRegister rhs,
+ FloatRegister lhsDest) {
+ compareSimd128Float(cond, Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
+}
+
+void MacroAssembler::compareFloat64x2(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ compareSimd128Float(cond, Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+// Load
+
+void MacroAssembler::loadUnalignedSimd128(const Address& src,
+ FloatRegister dest) {
+ Ldr(ARMFPRegister(dest, 128), toMemOperand(src));
+}
+
+void MacroAssembler::loadUnalignedSimd128(const BaseIndex& address,
+ FloatRegister dest) {
+ doBaseIndex(ARMFPRegister(dest, 128), address, vixl::LDR_q);
+}
+
+// Store
+
+void MacroAssembler::storeUnalignedSimd128(FloatRegister src,
+ const Address& dest) {
+ Str(ARMFPRegister(src, 128), toMemOperand(dest));
+}
+
+void MacroAssembler::storeUnalignedSimd128(FloatRegister src,
+ const BaseIndex& dest) {
+ doBaseIndex(ARMFPRegister(src, 128), dest, vixl::STR_q);
+}
+
+// Floating point negation
+
+void MacroAssembler::negFloat32x4(FloatRegister src, FloatRegister dest) {
+ Fneg(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::negFloat64x2(FloatRegister src, FloatRegister dest) {
+ Fneg(Simd2D(dest), Simd2D(src));
+}
+
+// Floating point absolute value
+
+void MacroAssembler::absFloat32x4(FloatRegister src, FloatRegister dest) {
+ Fabs(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::absFloat64x2(FloatRegister src, FloatRegister dest) {
+ Fabs(Simd2D(dest), Simd2D(src));
+}
+
+// NaN-propagating minimum
+
+void MacroAssembler::minFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmin(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::minFloat32x4(FloatRegister rhs, FloatRegister lhsDest) {
+ Fmin(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
+}
+
+void MacroAssembler::minFloat64x2(FloatRegister rhs, FloatRegister lhsDest) {
+ Fmin(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
+}
+
+void MacroAssembler::minFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmin(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+// NaN-propagating maximum
+
+void MacroAssembler::maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmax(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::maxFloat32x4(FloatRegister rhs, FloatRegister lhsDest) {
+ Fmax(Simd4S(lhsDest), Simd4S(lhsDest), Simd4S(rhs));
+}
+
+void MacroAssembler::maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmax(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+void MacroAssembler::maxFloat64x2(FloatRegister rhs, FloatRegister lhsDest) {
+ Fmax(Simd2D(lhsDest), Simd2D(lhsDest), Simd2D(rhs));
+}
+
+// Floating add
+
+void MacroAssembler::addFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fadd(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::addFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fadd(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+// Floating subtract
+
+void MacroAssembler::subFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fsub(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::subFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fsub(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+// Floating division
+
+void MacroAssembler::divFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fdiv(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::divFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fdiv(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+// Floating Multiply
+
+void MacroAssembler::mulFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmul(Simd4S(dest), Simd4S(lhs), Simd4S(rhs));
+}
+
+void MacroAssembler::mulFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmul(Simd2D(dest), Simd2D(lhs), Simd2D(rhs));
+}
+
+// Pairwise add
+
+void MacroAssembler::extAddPairwiseInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ Saddlp(Simd8H(dest), Simd16B(src));
+}
+
+void MacroAssembler::unsignedExtAddPairwiseInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ Uaddlp(Simd8H(dest), Simd16B(src));
+}
+
+void MacroAssembler::extAddPairwiseInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ Saddlp(Simd4S(dest), Simd8H(src));
+}
+
+void MacroAssembler::unsignedExtAddPairwiseInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ Uaddlp(Simd4S(dest), Simd8H(src));
+}
+
+// Floating square root
+
+void MacroAssembler::sqrtFloat32x4(FloatRegister src, FloatRegister dest) {
+ Fsqrt(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::sqrtFloat64x2(FloatRegister src, FloatRegister dest) {
+ Fsqrt(Simd2D(dest), Simd2D(src));
+}
+
+// Integer to floating point with rounding
+
+void MacroAssembler::convertInt32x4ToFloat32x4(FloatRegister src,
+ FloatRegister dest) {
+ Scvtf(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::unsignedConvertInt32x4ToFloat32x4(FloatRegister src,
+ FloatRegister dest) {
+ Ucvtf(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::convertInt32x4ToFloat64x2(FloatRegister src,
+ FloatRegister dest) {
+ Sshll(Simd2D(dest), Simd2S(src), 0);
+ Scvtf(Simd2D(dest), Simd2D(dest));
+}
+
+void MacroAssembler::unsignedConvertInt32x4ToFloat64x2(FloatRegister src,
+ FloatRegister dest) {
+ Ushll(Simd2D(dest), Simd2S(src), 0);
+ Ucvtf(Simd2D(dest), Simd2D(dest));
+}
+
+// Floating point to integer with saturation
+
+void MacroAssembler::truncSatFloat32x4ToInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ Fcvtzs(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ Fcvtzu(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::truncSatFloat64x2ToInt32x4(FloatRegister src,
+ FloatRegister dest,
+ FloatRegister temp) {
+ Fcvtzs(Simd2D(dest), Simd2D(src));
+ Sqxtn(Simd2S(dest), Simd2D(dest));
+}
+
+void MacroAssembler::unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src,
+ FloatRegister dest,
+ FloatRegister temp) {
+ Fcvtzu(Simd2D(dest), Simd2D(src));
+ Uqxtn(Simd2S(dest), Simd2D(dest));
+}
+
+void MacroAssembler::truncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest) {
+ Fcvtzs(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::unsignedTruncSatFloat32x4ToInt32x4Relaxed(
+ FloatRegister src, FloatRegister dest) {
+ Fcvtzu(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::truncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest) {
+ Fcvtzs(Simd2D(dest), Simd2D(src));
+ Sqxtn(Simd2S(dest), Simd2D(dest));
+}
+
+void MacroAssembler::unsignedTruncSatFloat64x2ToInt32x4Relaxed(
+ FloatRegister src, FloatRegister dest) {
+ Fcvtzu(Simd2D(dest), Simd2D(src));
+ Uqxtn(Simd2S(dest), Simd2D(dest));
+}
+
+// Floating point narrowing
+
+void MacroAssembler::convertFloat64x2ToFloat32x4(FloatRegister src,
+ FloatRegister dest) {
+ Fcvtn(Simd2S(dest), Simd2D(src));
+}
+
+// Floating point widening
+
+void MacroAssembler::convertFloat32x4ToFloat64x2(FloatRegister src,
+ FloatRegister dest) {
+ Fcvtl(Simd2D(dest), Simd2S(src));
+}
+
+// Integer to integer narrowing
+
+void MacroAssembler::narrowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (rhs == dest) {
+ Mov(scratch, SimdReg(rhs));
+ rhs = scratch;
+ }
+ Sqxtn(Simd8B(dest), Simd8H(lhs));
+ Sqxtn2(Simd16B(dest), Simd8H(rhs));
+}
+
+void MacroAssembler::unsignedNarrowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (rhs == dest) {
+ Mov(scratch, SimdReg(rhs));
+ rhs = scratch;
+ }
+ Sqxtun(Simd8B(dest), Simd8H(lhs));
+ Sqxtun2(Simd16B(dest), Simd8H(rhs));
+}
+
+void MacroAssembler::narrowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (rhs == dest) {
+ Mov(scratch, SimdReg(rhs));
+ rhs = scratch;
+ }
+ Sqxtn(Simd4H(dest), Simd4S(lhs));
+ Sqxtn2(Simd8H(dest), Simd4S(rhs));
+}
+
+void MacroAssembler::unsignedNarrowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (rhs == dest) {
+ Mov(scratch, SimdReg(rhs));
+ rhs = scratch;
+ }
+ Sqxtun(Simd4H(dest), Simd4S(lhs));
+ Sqxtun2(Simd8H(dest), Simd4S(rhs));
+}
+
+// Integer to integer widening
+
+void MacroAssembler::widenLowInt8x16(FloatRegister src, FloatRegister dest) {
+ Sshll(Simd8H(dest), Simd8B(src), 0);
+}
+
+void MacroAssembler::widenHighInt8x16(FloatRegister src, FloatRegister dest) {
+ Sshll2(Simd8H(dest), Simd16B(src), 0);
+}
+
+void MacroAssembler::unsignedWidenLowInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ Ushll(Simd8H(dest), Simd8B(src), 0);
+}
+
+void MacroAssembler::unsignedWidenHighInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ Ushll2(Simd8H(dest), Simd16B(src), 0);
+}
+
+void MacroAssembler::widenLowInt16x8(FloatRegister src, FloatRegister dest) {
+ Sshll(Simd4S(dest), Simd4H(src), 0);
+}
+
+void MacroAssembler::widenHighInt16x8(FloatRegister src, FloatRegister dest) {
+ Sshll2(Simd4S(dest), Simd8H(src), 0);
+}
+
+void MacroAssembler::unsignedWidenLowInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ Ushll(Simd4S(dest), Simd4H(src), 0);
+}
+
+void MacroAssembler::unsignedWidenHighInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ Ushll2(Simd4S(dest), Simd8H(src), 0);
+}
+
+void MacroAssembler::widenLowInt32x4(FloatRegister src, FloatRegister dest) {
+ Sshll(Simd2D(dest), Simd2S(src), 0);
+}
+
+void MacroAssembler::unsignedWidenLowInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ Ushll(Simd2D(dest), Simd2S(src), 0);
+}
+
+void MacroAssembler::widenHighInt32x4(FloatRegister src, FloatRegister dest) {
+ Sshll2(Simd2D(dest), Simd4S(src), 0);
+}
+
+void MacroAssembler::unsignedWidenHighInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ Ushll2(Simd2D(dest), Simd4S(src), 0);
+}
+
+// Compare-based minimum/maximum (experimental as of August, 2020)
+// https://github.com/WebAssembly/simd/pull/122
+
+void MacroAssembler::pseudoMinFloat32x4(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest) {
+ // Shut up the linter by using the same names as in the declaration, then
+ // aliasing here.
+ FloatRegister rhs = rhsOrRhsDest;
+ FloatRegister lhsDest = lhsOrLhsDest;
+ ScratchSimd128Scope scratch(*this);
+ Fcmgt(Simd4S(scratch), Simd4S(lhsDest), Simd4S(rhs));
+ Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhsDest));
+ Mov(SimdReg(lhsDest), scratch);
+}
+
+void MacroAssembler::pseudoMinFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ Fcmgt(Simd4S(scratch), Simd4S(lhs), Simd4S(rhs));
+ Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhs));
+ Mov(SimdReg(dest), scratch);
+}
+
+void MacroAssembler::pseudoMinFloat64x2(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest) {
+ FloatRegister rhs = rhsOrRhsDest;
+ FloatRegister lhsDest = lhsOrLhsDest;
+ ScratchSimd128Scope scratch(*this);
+ Fcmgt(Simd2D(scratch), Simd2D(lhsDest), Simd2D(rhs));
+ Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhsDest));
+ Mov(SimdReg(lhsDest), scratch);
+}
+
+void MacroAssembler::pseudoMinFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ Fcmgt(Simd2D(scratch), Simd2D(lhs), Simd2D(rhs));
+ Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhs));
+ Mov(SimdReg(dest), scratch);
+}
+
+void MacroAssembler::pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest) {
+ FloatRegister rhs = rhsOrRhsDest;
+ FloatRegister lhsDest = lhsOrLhsDest;
+ ScratchSimd128Scope scratch(*this);
+ Fcmgt(Simd4S(scratch), Simd4S(rhs), Simd4S(lhsDest));
+ Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhsDest));
+ Mov(SimdReg(lhsDest), scratch);
+}
+
+void MacroAssembler::pseudoMaxFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ Fcmgt(Simd4S(scratch), Simd4S(rhs), Simd4S(lhs));
+ Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhs));
+ Mov(SimdReg(dest), scratch);
+}
+
+void MacroAssembler::pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest) {
+ FloatRegister rhs = rhsOrRhsDest;
+ FloatRegister lhsDest = lhsOrLhsDest;
+ ScratchSimd128Scope scratch(*this);
+ Fcmgt(Simd2D(scratch), Simd2D(rhs), Simd2D(lhsDest));
+ Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhsDest));
+ Mov(SimdReg(lhsDest), scratch);
+}
+
+void MacroAssembler::pseudoMaxFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ Fcmgt(Simd2D(scratch), Simd2D(rhs), Simd2D(lhs));
+ Bsl(Simd16B(scratch), Simd16B(rhs), Simd16B(lhs));
+ Mov(SimdReg(dest), scratch);
+}
+
+// Widening/pairwise integer dot product (experimental as of August, 2020)
+// https://github.com/WebAssembly/simd/pull/127
+
+void MacroAssembler::widenDotInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ Smull(Simd4S(scratch), Simd4H(lhs), Simd4H(rhs));
+ Smull2(Simd4S(dest), Simd8H(lhs), Simd8H(rhs));
+ Addp(Simd4S(dest), Simd4S(scratch), Simd4S(dest));
+}
+
+void MacroAssembler::dotInt8x16Int7x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ Smull(Simd8H(scratch), Simd8B(lhs), Simd8B(rhs));
+ Smull2(Simd8H(dest), Simd16B(lhs), Simd16B(rhs));
+ Addp(Simd8H(dest), Simd8H(scratch), Simd8H(dest));
+}
+
+void MacroAssembler::dotInt8x16Int7x16ThenAdd(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest,
+ FloatRegister temp) {
+ ScratchSimd128Scope scratch(*this);
+ Smull(Simd8H(scratch), Simd8B(lhs), Simd8B(rhs));
+ Smull2(Simd8H(temp), Simd16B(lhs), Simd16B(rhs));
+ Addp(Simd8H(temp), Simd8H(scratch), Simd8H(temp));
+ Sadalp(Simd4S(dest), Simd8H(temp));
+}
+
+void MacroAssembler::dotBFloat16x8ThenAdd(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest,
+ FloatRegister temp) {
+ MOZ_ASSERT(lhs != dest && rhs != dest);
+ ScratchSimd128Scope scratch(*this);
+ Shl(Simd4S(scratch), Simd4S(lhs), 16);
+ Shl(Simd4S(temp), Simd4S(rhs), 16);
+ Fmla(Simd4S(dest), Simd4S(scratch), Simd4S(temp));
+ loadConstantSimd128(SimdConstant::SplatX4(int32_t(0xFFFF0000)), temp);
+ And(Simd16B(scratch), Simd16B(lhs), Simd16B(temp));
+ And(Simd16B(temp), Simd16B(rhs), Simd16B(temp));
+ Fmla(Simd4S(dest), Simd4S(scratch), Simd4S(temp));
+}
+
+// Floating point rounding (experimental as of August, 2020)
+// https://github.com/WebAssembly/simd/pull/232
+
+void MacroAssembler::ceilFloat32x4(FloatRegister src, FloatRegister dest) {
+ Frintp(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::ceilFloat64x2(FloatRegister src, FloatRegister dest) {
+ Frintp(Simd2D(dest), Simd2D(src));
+}
+
+void MacroAssembler::floorFloat32x4(FloatRegister src, FloatRegister dest) {
+ Frintm(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::floorFloat64x2(FloatRegister src, FloatRegister dest) {
+ Frintm(Simd2D(dest), Simd2D(src));
+}
+
+void MacroAssembler::truncFloat32x4(FloatRegister src, FloatRegister dest) {
+ Frintz(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::truncFloat64x2(FloatRegister src, FloatRegister dest) {
+ Frintz(Simd2D(dest), Simd2D(src));
+}
+
+void MacroAssembler::nearestFloat32x4(FloatRegister src, FloatRegister dest) {
+ Frintn(Simd4S(dest), Simd4S(src));
+}
+
+void MacroAssembler::nearestFloat64x2(FloatRegister src, FloatRegister dest) {
+ Frintn(Simd2D(dest), Simd2D(src));
+}
+
+// Floating multiply-accumulate: srcDest [+-]= src1 * src2
+
+void MacroAssembler::fmaFloat32x4(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) {
+ Fmla(Simd4S(srcDest), Simd4S(src1), Simd4S(src2));
+}
+
+void MacroAssembler::fnmaFloat32x4(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) {
+ Fmls(Simd4S(srcDest), Simd4S(src1), Simd4S(src2));
+}
+
+void MacroAssembler::fmaFloat64x2(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) {
+ Fmla(Simd2D(srcDest), Simd2D(src1), Simd2D(src2));
+}
+
+void MacroAssembler::fnmaFloat64x2(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) {
+ Fmls(Simd2D(srcDest), Simd2D(src1), Simd2D(src2));
+}
+
+void MacroAssembler::minFloat32x4Relaxed(FloatRegister src,
+ FloatRegister srcDest) {
+ Fmin(Simd4S(srcDest), Simd4S(src), Simd4S(srcDest));
+}
+
+void MacroAssembler::minFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmin(Simd4S(dest), Simd4S(rhs), Simd4S(lhs));
+}
+
+void MacroAssembler::maxFloat32x4Relaxed(FloatRegister src,
+ FloatRegister srcDest) {
+ Fmax(Simd4S(srcDest), Simd4S(src), Simd4S(srcDest));
+}
+
+void MacroAssembler::maxFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmax(Simd4S(dest), Simd4S(rhs), Simd4S(lhs));
+}
+
+void MacroAssembler::minFloat64x2Relaxed(FloatRegister src,
+ FloatRegister srcDest) {
+ Fmin(Simd2D(srcDest), Simd2D(src), Simd2D(srcDest));
+}
+
+void MacroAssembler::minFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmin(Simd2D(dest), Simd2D(rhs), Simd2D(lhs));
+}
+
+void MacroAssembler::maxFloat64x2Relaxed(FloatRegister src,
+ FloatRegister srcDest) {
+ Fmax(Simd2D(srcDest), Simd2D(src), Simd2D(srcDest));
+}
+
+void MacroAssembler::maxFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ Fmax(Simd2D(dest), Simd2D(rhs), Simd2D(lhs));
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+void MacroAssemblerCompat::addToStackPtr(Register src) {
+ Add(GetStackPointer64(), GetStackPointer64(), ARMRegister(src, 64));
+ // Given that required invariant SP <= PSP, this is probably pointless,
+ // since it gives PSP a larger value.
+ syncStackPtr();
+}
+
+void MacroAssemblerCompat::addToStackPtr(Imm32 imm) {
+ Add(GetStackPointer64(), GetStackPointer64(), Operand(imm.value));
+ // As above, probably pointless.
+ syncStackPtr();
+}
+
+void MacroAssemblerCompat::addToStackPtr(const Address& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ Ldr(scratch, toMemOperand(src));
+ Add(GetStackPointer64(), GetStackPointer64(), scratch);
+ // As above, probably pointless.
+ syncStackPtr();
+}
+
+void MacroAssemblerCompat::addStackPtrTo(Register dest) {
+ Add(ARMRegister(dest, 64), ARMRegister(dest, 64), GetStackPointer64());
+}
+
+void MacroAssemblerCompat::subFromStackPtr(Register src) {
+ Sub(GetStackPointer64(), GetStackPointer64(), ARMRegister(src, 64));
+ syncStackPtr();
+}
+
+void MacroAssemblerCompat::subFromStackPtr(Imm32 imm) {
+ Sub(GetStackPointer64(), GetStackPointer64(), Operand(imm.value));
+ syncStackPtr();
+}
+
+void MacroAssemblerCompat::subStackPtrFrom(Register dest) {
+ Sub(ARMRegister(dest, 64), ARMRegister(dest, 64), GetStackPointer64());
+}
+
+void MacroAssemblerCompat::andToStackPtr(Imm32 imm) {
+ if (sp.Is(GetStackPointer64())) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ Mov(scratch, sp);
+ And(sp, scratch, Operand(imm.value));
+ // syncStackPtr() not needed since our SP is the real SP.
+ } else {
+ And(GetStackPointer64(), GetStackPointer64(), Operand(imm.value));
+ syncStackPtr();
+ }
+}
+
+void MacroAssemblerCompat::moveToStackPtr(Register src) {
+ Mov(GetStackPointer64(), ARMRegister(src, 64));
+ syncStackPtr();
+}
+
+void MacroAssemblerCompat::moveStackPtrTo(Register dest) {
+ Mov(ARMRegister(dest, 64), GetStackPointer64());
+}
+
+void MacroAssemblerCompat::loadStackPtr(const Address& src) {
+ if (sp.Is(GetStackPointer64())) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ Ldr(scratch, toMemOperand(src));
+ Mov(sp, scratch);
+ // syncStackPtr() not needed since our SP is the real SP.
+ } else {
+ Ldr(GetStackPointer64(), toMemOperand(src));
+ syncStackPtr();
+ }
+}
+
+void MacroAssemblerCompat::storeStackPtr(const Address& dest) {
+ if (sp.Is(GetStackPointer64())) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ Mov(scratch, sp);
+ Str(scratch, toMemOperand(dest));
+ } else {
+ Str(GetStackPointer64(), toMemOperand(dest));
+ }
+}
+
+void MacroAssemblerCompat::branchTestStackPtr(Condition cond, Imm32 rhs,
+ Label* label) {
+ if (sp.Is(GetStackPointer64())) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ Mov(scratch, sp);
+ Tst(scratch, Operand(rhs.value));
+ } else {
+ Tst(GetStackPointer64(), Operand(rhs.value));
+ }
+ B(label, cond);
+}
+
+void MacroAssemblerCompat::branchStackPtr(Condition cond, Register rhs_,
+ Label* label) {
+ ARMRegister rhs(rhs_, 64);
+ if (sp.Is(GetStackPointer64())) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ Mov(scratch, sp);
+ Cmp(scratch, rhs);
+ } else {
+ Cmp(GetStackPointer64(), rhs);
+ }
+ B(label, cond);
+}
+
+void MacroAssemblerCompat::branchStackPtrRhs(Condition cond, Address lhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ Ldr(scratch, toMemOperand(lhs));
+ // Cmp disallows SP as the rhs, so flip the operands and invert the
+ // condition.
+ Cmp(GetStackPointer64(), scratch);
+ B(label, Assembler::InvertCondition(cond));
+}
+
+void MacroAssemblerCompat::branchStackPtrRhs(Condition cond,
+ AbsoluteAddress lhs,
+ Label* label) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ loadPtr(lhs, scratch.asUnsized());
+ // Cmp disallows SP as the rhs, so flip the operands and invert the
+ // condition.
+ Cmp(GetStackPointer64(), scratch);
+ B(label, Assembler::InvertCondition(cond));
+}
+
+// If source is a double, load into dest.
+// If source is int32, convert to double and store in dest.
+// Else, branch to failure.
+void MacroAssemblerCompat::ensureDouble(const ValueOperand& source,
+ FloatRegister dest, Label* failure) {
+ Label isDouble, done;
+
+ {
+ ScratchTagScope tag(asMasm(), source);
+ splitTagForTest(source, tag);
+ asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, tag, failure);
+ }
+
+ convertInt32ToDouble(source.valueReg(), dest);
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerCompat::unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType type) {
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.valueReg(), dest.fpu());
+ jump(&end);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else {
+ unboxNonDouble(src, dest.gpr(), type);
+ }
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm64_MacroAssembler_arm64_inl_h */
diff --git a/js/src/jit/arm64/MacroAssembler-arm64.cpp b/js/src/jit/arm64/MacroAssembler-arm64.cpp
new file mode 100644
index 0000000000..a176a2974f
--- /dev/null
+++ b/js/src/jit/arm64/MacroAssembler-arm64.cpp
@@ -0,0 +1,3140 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm64/MacroAssembler-arm64.h"
+
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Maybe.h"
+
+#include "jsmath.h"
+
+#include "jit/arm64/MoveEmitter-arm64.h"
+#include "jit/arm64/SharedICRegisters-arm64.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "util/Memory.h"
+#include "vm/BigIntType.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/StringType.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+enum class Width { _32 = 32, _64 = 64 };
+
+static inline ARMRegister X(Register r) { return ARMRegister(r, 64); }
+
+static inline ARMRegister X(MacroAssembler& masm, RegisterOrSP r) {
+ return masm.toARMRegister(r, 64);
+}
+
+static inline ARMRegister W(Register r) { return ARMRegister(r, 32); }
+
+static inline ARMRegister R(Register r, Width w) {
+ return ARMRegister(r, unsigned(w));
+}
+
+void MacroAssemblerCompat::boxValue(JSValueType type, Register src,
+ Register dest) {
+#ifdef DEBUG
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ Label upper32BitsZeroed;
+ movePtr(ImmWord(UINT32_MAX), dest);
+ asMasm().branchPtr(Assembler::BelowOrEqual, src, dest, &upper32BitsZeroed);
+ breakpoint();
+ bind(&upper32BitsZeroed);
+ }
+#endif
+ Orr(ARMRegister(dest, 64), ARMRegister(src, 64),
+ Operand(ImmShiftedTag(type).value));
+}
+
+#ifdef ENABLE_WASM_SIMD
+bool MacroAssembler::MustMaskShiftCountSimd128(wasm::SimdOp op, int32_t* mask) {
+ switch (op) {
+ case wasm::SimdOp::I8x16Shl:
+ case wasm::SimdOp::I8x16ShrU:
+ case wasm::SimdOp::I8x16ShrS:
+ *mask = 7;
+ break;
+ case wasm::SimdOp::I16x8Shl:
+ case wasm::SimdOp::I16x8ShrU:
+ case wasm::SimdOp::I16x8ShrS:
+ *mask = 15;
+ break;
+ case wasm::SimdOp::I32x4Shl:
+ case wasm::SimdOp::I32x4ShrU:
+ case wasm::SimdOp::I32x4ShrS:
+ *mask = 31;
+ break;
+ case wasm::SimdOp::I64x2Shl:
+ case wasm::SimdOp::I64x2ShrU:
+ case wasm::SimdOp::I64x2ShrS:
+ *mask = 63;
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift operation");
+ }
+ return true;
+}
+#endif
+
+void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
+ ARMRegister dest(output, 32);
+ Fcvtns(dest, ARMFPRegister(input, 64));
+
+ {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+
+ Mov(scratch32, Operand(0xff));
+ Cmp(dest, scratch32);
+ Csel(dest, dest, scratch32, LessThan);
+ }
+
+ Cmp(dest, Operand(0));
+ Csel(dest, dest, wzr, GreaterThan);
+}
+
+js::jit::MacroAssembler& MacroAssemblerCompat::asMasm() {
+ return *static_cast<js::jit::MacroAssembler*>(this);
+}
+
+const js::jit::MacroAssembler& MacroAssemblerCompat::asMasm() const {
+ return *static_cast<const js::jit::MacroAssembler*>(this);
+}
+
+vixl::MacroAssembler& MacroAssemblerCompat::asVIXL() {
+ return *static_cast<vixl::MacroAssembler*>(this);
+}
+
+const vixl::MacroAssembler& MacroAssemblerCompat::asVIXL() const {
+ return *static_cast<const vixl::MacroAssembler*>(this);
+}
+
+void MacroAssemblerCompat::mov(CodeLabel* label, Register dest) {
+ BufferOffset bo = movePatchablePtr(ImmWord(/* placeholder */ 0), dest);
+ label->patchAt()->bind(bo.getOffset());
+ label->setLinkMode(CodeLabel::MoveImmediate);
+}
+
+BufferOffset MacroAssemblerCompat::movePatchablePtr(ImmPtr ptr, Register dest) {
+ const size_t numInst = 1; // Inserting one load instruction.
+ const unsigned numPoolEntries = 2; // Every pool entry is 4 bytes.
+ uint8_t* literalAddr = (uint8_t*)(&ptr.value); // TODO: Should be const.
+
+ // Scratch space for generating the load instruction.
+ //
+ // allocLiteralLoadEntry() will use InsertIndexIntoTag() to store a temporary
+ // index to the corresponding PoolEntry in the instruction itself.
+ //
+ // That index will be fixed up later when finishPool()
+ // walks over all marked loads and calls PatchConstantPoolLoad().
+ uint32_t instructionScratch = 0;
+
+ // Emit the instruction mask in the scratch space.
+ // The offset doesn't matter: it will be fixed up later.
+ vixl::Assembler::ldr((Instruction*)&instructionScratch, ARMRegister(dest, 64),
+ 0);
+
+ // Add the entry to the pool, fix up the LDR imm19 offset,
+ // and add the completed instruction to the buffer.
+ return allocLiteralLoadEntry(numInst, numPoolEntries,
+ (uint8_t*)&instructionScratch, literalAddr);
+}
+
+BufferOffset MacroAssemblerCompat::movePatchablePtr(ImmWord ptr,
+ Register dest) {
+ const size_t numInst = 1; // Inserting one load instruction.
+ const unsigned numPoolEntries = 2; // Every pool entry is 4 bytes.
+ uint8_t* literalAddr = (uint8_t*)(&ptr.value);
+
+ // Scratch space for generating the load instruction.
+ //
+ // allocLiteralLoadEntry() will use InsertIndexIntoTag() to store a temporary
+ // index to the corresponding PoolEntry in the instruction itself.
+ //
+ // That index will be fixed up later when finishPool()
+ // walks over all marked loads and calls PatchConstantPoolLoad().
+ uint32_t instructionScratch = 0;
+
+ // Emit the instruction mask in the scratch space.
+ // The offset doesn't matter: it will be fixed up later.
+ vixl::Assembler::ldr((Instruction*)&instructionScratch, ARMRegister(dest, 64),
+ 0);
+
+ // Add the entry to the pool, fix up the LDR imm19 offset,
+ // and add the completed instruction to the buffer.
+ return allocLiteralLoadEntry(numInst, numPoolEntries,
+ (uint8_t*)&instructionScratch, literalAddr);
+}
+
+void MacroAssemblerCompat::loadPrivate(const Address& src, Register dest) {
+ loadPtr(src, dest);
+}
+
+void MacroAssemblerCompat::handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail) {
+ // Fail rather than silently create wrong code.
+ MOZ_RELEASE_ASSERT(GetStackPointer64().Is(PseudoStackPointer64));
+
+ // Reserve space for exception information.
+ int64_t size = (sizeof(ResumeFromException) + 7) & ~7;
+ Sub(PseudoStackPointer64, PseudoStackPointer64, Operand(size));
+ syncStackPtr();
+
+ MOZ_ASSERT(!x0.Is(PseudoStackPointer64));
+ Mov(x0, PseudoStackPointer64);
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(r1);
+ asMasm().passABIArg(r0);
+ asMasm().callWithABI<Fn, HandleException>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ // Check the `asMasm` calls above didn't mess with the StackPointer identity.
+ MOZ_ASSERT(GetStackPointer64().Is(PseudoStackPointer64));
+
+ loadPtr(Address(PseudoStackPointer, ResumeFromException::offsetOfKind()), r0);
+ asMasm().branch32(Assembler::Equal, r0,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, r0,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, r0,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, r0, Imm32(ExceptionResumeKind::WasmCatch),
+ &wasmCatch);
+
+ breakpoint(); // Invalid kind.
+
+ // No exception handler. Load the error value, restore state and return from
+ // the entry frame.
+ bind(&entryFrame);
+ moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ loadPtr(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfStackPointer()),
+ PseudoStackPointer);
+
+ // `retn` does indeed sync the stack pointer, but before doing that it reads
+ // from the stack. Consequently, if we remove this call to syncStackPointer
+ // then we take on the requirement to prove that the immediately preceding
+ // loadPtr produces a value for PSP which maintains the SP <= PSP invariant.
+ // That's a proof burden we don't want to take on. In general it would be
+ // good to move (at some time in the future, not now) to a world where
+ // *every* assignment to PSP or SP is followed immediately by a copy into
+ // the other register. That would make all required correctness proofs
+ // trivial in the sense that it requires only local inspection of code
+ // immediately following (dominated by) any such assignment.
+ syncStackPtr();
+ retn(Imm32(1 * sizeof(void*))); // Pop from stack and return.
+
+ // If we found a catch handler, this must be a baseline frame. Restore state
+ // and jump to the catch block.
+ bind(&catch_);
+ loadPtr(Address(PseudoStackPointer, ResumeFromException::offsetOfTarget()),
+ r0);
+ loadPtr(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfStackPointer()),
+ PseudoStackPointer);
+ syncStackPtr();
+ Br(x0);
+
+ // If we found a finally block, this must be a baseline frame. Push two
+ // values expected by the finally block: the exception and BooleanValue(true).
+ bind(&finally);
+ ARMRegister exception = x1;
+ Ldr(exception, MemOperand(PseudoStackPointer64,
+ ResumeFromException::offsetOfException()));
+ Ldr(x0,
+ MemOperand(PseudoStackPointer64, ResumeFromException::offsetOfTarget()));
+ Ldr(ARMRegister(FramePointer, 64),
+ MemOperand(PseudoStackPointer64,
+ ResumeFromException::offsetOfFramePointer()));
+ Ldr(PseudoStackPointer64,
+ MemOperand(PseudoStackPointer64,
+ ResumeFromException::offsetOfStackPointer()));
+ syncStackPtr();
+ push(exception);
+ pushValue(BooleanValue(true));
+ Br(x0);
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ loadPtr(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfStackPointer()),
+ PseudoStackPointer);
+ // See comment further up beginning "`retn` does indeed sync the stack
+ // pointer". That comment applies here too.
+ syncStackPtr();
+ loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ jump(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ loadPtr(
+ Address(PseudoStackPointer, offsetof(ResumeFromException, framePointer)),
+ FramePointer);
+ loadPtr(
+ Address(PseudoStackPointer, offsetof(ResumeFromException, stackPointer)),
+ PseudoStackPointer);
+ syncStackPtr();
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ movePtr(FramePointer, PseudoStackPointer);
+ syncStackPtr();
+ vixl::MacroAssembler::Pop(ARMRegister(FramePointer, 64));
+
+ vixl::MacroAssembler::Pop(vixl::lr);
+ syncStackPtr();
+ vixl::MacroAssembler::Ret(vixl::lr);
+
+ // If we are bailing out to baseline to handle an exception, jump to the
+ // bailout tail stub. Load 1 (true) in x0 (ReturnReg) to indicate success.
+ bind(&bailout);
+ Ldr(x2, MemOperand(PseudoStackPointer64,
+ ResumeFromException::offsetOfBailoutInfo()));
+ Ldr(PseudoStackPointer64,
+ MemOperand(PseudoStackPointer64,
+ ResumeFromException::offsetOfStackPointer()));
+ syncStackPtr();
+ Mov(x0, 1);
+ jump(bailoutTail);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ Ldr(x29, MemOperand(PseudoStackPointer64,
+ ResumeFromException::offsetOfFramePointer()));
+ Ldr(PseudoStackPointer64,
+ MemOperand(PseudoStackPointer64,
+ ResumeFromException::offsetOfStackPointer()));
+ syncStackPtr();
+ ret();
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+ loadPtr(Address(PseudoStackPointer, ResumeFromException::offsetOfTarget()),
+ r0);
+ loadPtr(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfFramePointer()),
+ r29);
+ loadPtr(
+ Address(PseudoStackPointer, ResumeFromException::offsetOfStackPointer()),
+ PseudoStackPointer);
+ syncStackPtr();
+ Br(x0);
+
+ MOZ_ASSERT(GetStackPointer64().Is(PseudoStackPointer64));
+}
+
+void MacroAssemblerCompat::profilerEnterFrame(Register framePtr,
+ Register scratch) {
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr,
+ Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr),
+ Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerCompat::profilerExitFrame() {
+ jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+Assembler::Condition MacroAssemblerCompat::testStringTruthy(
+ bool truthy, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ const ARMRegister scratch32(scratch, 32);
+ const ARMRegister scratch64(scratch, 64);
+
+ MOZ_ASSERT(value.valueReg() != scratch);
+
+ unboxString(value, scratch);
+ Ldr(scratch32, MemOperand(scratch64, JSString::offsetOfLength()));
+ Cmp(scratch32, Operand(0));
+ return truthy ? Condition::NonZero : Condition::Zero;
+}
+
+Assembler::Condition MacroAssemblerCompat::testBigIntTruthy(
+ bool truthy, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+
+ MOZ_ASSERT(value.valueReg() != scratch);
+
+ unboxBigInt(value, scratch);
+ load32(Address(scratch, BigInt::offsetOfDigitLength()), scratch);
+ cmp32(scratch, Imm32(0));
+ return truthy ? Condition::NonZero : Condition::Zero;
+}
+
+void MacroAssemblerCompat::breakpoint() {
+ // Note, other payloads are possible, but GDB is known to misinterpret them
+ // sometimes and iloop on the breakpoint instead of stopping properly.
+ Brk(0);
+}
+
+// Either `any` is valid or `sixtyfour` is valid. Return a 32-bit ARMRegister
+// in the first case and an ARMRegister of the desired size in the latter case.
+
+static inline ARMRegister SelectGPReg(AnyRegister any, Register64 sixtyfour,
+ unsigned size = 64) {
+ MOZ_ASSERT(any.isValid() != (sixtyfour != Register64::Invalid()));
+
+ if (sixtyfour == Register64::Invalid()) {
+ return ARMRegister(any.gpr(), 32);
+ }
+
+ return ARMRegister(sixtyfour.reg, size);
+}
+
+// Assert that `sixtyfour` is invalid and then return an FP register from `any`
+// of the desired size.
+
+static inline ARMFPRegister SelectFPReg(AnyRegister any, Register64 sixtyfour,
+ unsigned size) {
+ MOZ_ASSERT(sixtyfour == Register64::Invalid());
+ return ARMFPRegister(any.fpu(), size);
+}
+
+void MacroAssemblerCompat::wasmLoadImpl(const wasm::MemoryAccessDesc& access,
+ Register memoryBase_, Register ptr_,
+ AnyRegister outany, Register64 out64) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+
+ ARMRegister memoryBase(memoryBase_, 64);
+ ARMRegister ptr(ptr_, 64);
+ if (offset) {
+ vixl::UseScratchRegisterScope temps(this);
+ ARMRegister scratch = temps.AcquireX();
+ Add(scratch, ptr, Operand(offset));
+ MemOperand srcAddr(memoryBase, scratch);
+ wasmLoadImpl(access, srcAddr, outany, out64);
+ } else {
+ MemOperand srcAddr(memoryBase, ptr);
+ wasmLoadImpl(access, srcAddr, outany, out64);
+ }
+}
+
+void MacroAssemblerCompat::wasmLoadImpl(const wasm::MemoryAccessDesc& access,
+ MemOperand srcAddr, AnyRegister outany,
+ Register64 out64) {
+ // Reg+Reg and Reg+SmallImm addressing is directly encodable in one Load
+ // instruction, hence we expect exactly one instruction to be emitted in the
+ // window.
+ int32_t instructionsExpected = 1;
+
+ // Splat and widen however require an additional instruction to be emitted
+ // after the load, so allow one more instruction in the window.
+ if (access.isSplatSimd128Load() || access.isWidenSimd128Load()) {
+ MOZ_ASSERT(access.type() == Scalar::Float64);
+ instructionsExpected++;
+ }
+
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ asMasm().memoryBarrierBefore(access.sync());
+
+ {
+ // The AutoForbidPoolsAndNops asserts if we emit more than the expected
+ // number of instructions and thus ensures that the access metadata is
+ // emitted at the address of the Load.
+ AutoForbidPoolsAndNops afp(this, instructionsExpected);
+
+ append(access, asMasm().currentOffset());
+ switch (access.type()) {
+ case Scalar::Int8:
+ Ldrsb(SelectGPReg(outany, out64), srcAddr);
+ break;
+ case Scalar::Uint8:
+ Ldrb(SelectGPReg(outany, out64), srcAddr);
+ break;
+ case Scalar::Int16:
+ Ldrsh(SelectGPReg(outany, out64), srcAddr);
+ break;
+ case Scalar::Uint16:
+ Ldrh(SelectGPReg(outany, out64), srcAddr);
+ break;
+ case Scalar::Int32:
+ if (out64 != Register64::Invalid()) {
+ Ldrsw(SelectGPReg(outany, out64), srcAddr);
+ } else {
+ Ldr(SelectGPReg(outany, out64, 32), srcAddr);
+ }
+ break;
+ case Scalar::Uint32:
+ Ldr(SelectGPReg(outany, out64, 32), srcAddr);
+ break;
+ case Scalar::Int64:
+ Ldr(SelectGPReg(outany, out64), srcAddr);
+ break;
+ case Scalar::Float32:
+ // LDR does the right thing also for access.isZeroExtendSimd128Load()
+ Ldr(SelectFPReg(outany, out64, 32), srcAddr);
+ break;
+ case Scalar::Float64:
+ if (access.isSplatSimd128Load() || access.isWidenSimd128Load()) {
+ ScratchSimd128Scope scratch_(asMasm());
+ ARMFPRegister scratch = Simd1D(scratch_);
+ Ldr(scratch, srcAddr);
+ if (access.isSplatSimd128Load()) {
+ Dup(SelectFPReg(outany, out64, 128).V2D(), scratch, 0);
+ } else {
+ MOZ_ASSERT(access.isWidenSimd128Load());
+ switch (access.widenSimdOp()) {
+ case wasm::SimdOp::V128Load8x8S:
+ Sshll(SelectFPReg(outany, out64, 128).V8H(), scratch.V8B(), 0);
+ break;
+ case wasm::SimdOp::V128Load8x8U:
+ Ushll(SelectFPReg(outany, out64, 128).V8H(), scratch.V8B(), 0);
+ break;
+ case wasm::SimdOp::V128Load16x4S:
+ Sshll(SelectFPReg(outany, out64, 128).V4S(), scratch.V4H(), 0);
+ break;
+ case wasm::SimdOp::V128Load16x4U:
+ Ushll(SelectFPReg(outany, out64, 128).V4S(), scratch.V4H(), 0);
+ break;
+ case wasm::SimdOp::V128Load32x2S:
+ Sshll(SelectFPReg(outany, out64, 128).V2D(), scratch.V2S(), 0);
+ break;
+ case wasm::SimdOp::V128Load32x2U:
+ Ushll(SelectFPReg(outany, out64, 128).V2D(), scratch.V2S(), 0);
+ break;
+ default:
+ MOZ_CRASH("Unexpected widening op for wasmLoad");
+ }
+ }
+ } else {
+ // LDR does the right thing also for access.isZeroExtendSimd128Load()
+ Ldr(SelectFPReg(outany, out64, 64), srcAddr);
+ }
+ break;
+ case Scalar::Simd128:
+ Ldr(SelectFPReg(outany, out64, 128), srcAddr);
+ break;
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+ }
+
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+// Return true if `address` can be represented as an immediate (possibly scaled
+// by the access size) in an LDR/STR type instruction.
+//
+// For more about the logic here, see vixl::MacroAssembler::LoadStoreMacro().
+static bool IsLSImmediateOffset(uint64_t address, size_t accessByteSize) {
+ // The predicates below operate on signed values only.
+ if (address > INT64_MAX) {
+ return false;
+ }
+
+ // The access size is always a power of 2, so computing the log amounts to
+ // counting trailing zeroes.
+ unsigned logAccessSize = mozilla::CountTrailingZeroes32(accessByteSize);
+ return (MacroAssemblerCompat::IsImmLSUnscaled(int64_t(address)) ||
+ MacroAssemblerCompat::IsImmLSScaled(int64_t(address), logAccessSize));
+}
+
+void MacroAssemblerCompat::wasmLoadAbsolute(
+ const wasm::MemoryAccessDesc& access, Register memoryBase, uint64_t address,
+ AnyRegister output, Register64 out64) {
+ if (!IsLSImmediateOffset(address, access.byteSize())) {
+ // The access will require the constant to be loaded into a temp register.
+ // Do so here, to keep the logic in wasmLoadImpl() tractable wrt emitting
+ // trap information.
+ //
+ // Almost all constant addresses will in practice be handled by a single MOV
+ // so do not worry about additional optimizations here.
+ vixl::UseScratchRegisterScope temps(this);
+ ARMRegister scratch = temps.AcquireX();
+ Mov(scratch, address);
+ MemOperand srcAddr(X(memoryBase), scratch);
+ wasmLoadImpl(access, srcAddr, output, out64);
+ } else {
+ MemOperand srcAddr(X(memoryBase), address);
+ wasmLoadImpl(access, srcAddr, output, out64);
+ }
+}
+
+void MacroAssemblerCompat::wasmStoreImpl(const wasm::MemoryAccessDesc& access,
+ AnyRegister valany, Register64 val64,
+ Register memoryBase_, Register ptr_) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+
+ ARMRegister memoryBase(memoryBase_, 64);
+ ARMRegister ptr(ptr_, 64);
+ if (offset) {
+ vixl::UseScratchRegisterScope temps(this);
+ ARMRegister scratch = temps.AcquireX();
+ Add(scratch, ptr, Operand(offset));
+ MemOperand destAddr(memoryBase, scratch);
+ wasmStoreImpl(access, destAddr, valany, val64);
+ } else {
+ MemOperand destAddr(memoryBase, ptr);
+ wasmStoreImpl(access, destAddr, valany, val64);
+ }
+}
+
+void MacroAssemblerCompat::wasmStoreImpl(const wasm::MemoryAccessDesc& access,
+ MemOperand dstAddr, AnyRegister valany,
+ Register64 val64) {
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ asMasm().memoryBarrierBefore(access.sync());
+
+ {
+ // Reg+Reg addressing is directly encodable in one Store instruction, hence
+ // the AutoForbidPoolsAndNops will ensure that the access metadata is
+ // emitted at the address of the Store. The AutoForbidPoolsAndNops will
+ // assert if we emit more than one instruction.
+
+ AutoForbidPoolsAndNops afp(this,
+ /* max number of instructions in scope = */ 1);
+
+ append(access, asMasm().currentOffset());
+ switch (access.type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ Strb(SelectGPReg(valany, val64), dstAddr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ Strh(SelectGPReg(valany, val64), dstAddr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ Str(SelectGPReg(valany, val64), dstAddr);
+ break;
+ case Scalar::Int64:
+ Str(SelectGPReg(valany, val64), dstAddr);
+ break;
+ case Scalar::Float32:
+ Str(SelectFPReg(valany, val64, 32), dstAddr);
+ break;
+ case Scalar::Float64:
+ Str(SelectFPReg(valany, val64, 64), dstAddr);
+ break;
+ case Scalar::Simd128:
+ Str(SelectFPReg(valany, val64, 128), dstAddr);
+ break;
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+ }
+
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerCompat::wasmStoreAbsolute(
+ const wasm::MemoryAccessDesc& access, AnyRegister value, Register64 value64,
+ Register memoryBase, uint64_t address) {
+ // See comments in wasmLoadAbsolute.
+ unsigned logAccessSize = mozilla::CountTrailingZeroes32(access.byteSize());
+ if (address > INT64_MAX || !(IsImmLSScaled(int64_t(address), logAccessSize) ||
+ IsImmLSUnscaled(int64_t(address)))) {
+ vixl::UseScratchRegisterScope temps(this);
+ ARMRegister scratch = temps.AcquireX();
+ Mov(scratch, address);
+ MemOperand destAddr(X(memoryBase), scratch);
+ wasmStoreImpl(access, destAddr, value, value64);
+ } else {
+ MemOperand destAddr(X(memoryBase), address);
+ wasmStoreImpl(access, destAddr, value, value64);
+ }
+}
+
+void MacroAssemblerCompat::compareSimd128Int(Assembler::Condition cond,
+ ARMFPRegister dest,
+ ARMFPRegister lhs,
+ ARMFPRegister rhs) {
+ switch (cond) {
+ case Assembler::Equal:
+ Cmeq(dest, lhs, rhs);
+ break;
+ case Assembler::NotEqual:
+ Cmeq(dest, lhs, rhs);
+ Mvn(dest, dest);
+ break;
+ case Assembler::GreaterThan:
+ Cmgt(dest, lhs, rhs);
+ break;
+ case Assembler::GreaterThanOrEqual:
+ Cmge(dest, lhs, rhs);
+ break;
+ case Assembler::LessThan:
+ Cmgt(dest, rhs, lhs);
+ break;
+ case Assembler::LessThanOrEqual:
+ Cmge(dest, rhs, lhs);
+ break;
+ case Assembler::Above:
+ Cmhi(dest, lhs, rhs);
+ break;
+ case Assembler::AboveOrEqual:
+ Cmhs(dest, lhs, rhs);
+ break;
+ case Assembler::Below:
+ Cmhi(dest, rhs, lhs);
+ break;
+ case Assembler::BelowOrEqual:
+ Cmhs(dest, rhs, lhs);
+ break;
+ default:
+ MOZ_CRASH("Unexpected SIMD integer condition");
+ }
+}
+
+void MacroAssemblerCompat::compareSimd128Float(Assembler::Condition cond,
+ ARMFPRegister dest,
+ ARMFPRegister lhs,
+ ARMFPRegister rhs) {
+ switch (cond) {
+ case Assembler::Equal:
+ Fcmeq(dest, lhs, rhs);
+ break;
+ case Assembler::NotEqual:
+ Fcmeq(dest, lhs, rhs);
+ Mvn(dest, dest);
+ break;
+ case Assembler::GreaterThan:
+ Fcmgt(dest, lhs, rhs);
+ break;
+ case Assembler::GreaterThanOrEqual:
+ Fcmge(dest, lhs, rhs);
+ break;
+ case Assembler::LessThan:
+ Fcmgt(dest, rhs, lhs);
+ break;
+ case Assembler::LessThanOrEqual:
+ Fcmge(dest, rhs, lhs);
+ break;
+ default:
+ MOZ_CRASH("Unexpected SIMD integer condition");
+ }
+}
+
+void MacroAssemblerCompat::rightShiftInt8x16(FloatRegister lhs, Register rhs,
+ FloatRegister dest,
+ bool isUnsigned) {
+ ScratchSimd128Scope scratch_(asMasm());
+ ARMFPRegister shift = Simd16B(scratch_);
+
+ Dup(shift, ARMRegister(rhs, 32));
+ Neg(shift, shift);
+
+ if (isUnsigned) {
+ Ushl(Simd16B(dest), Simd16B(lhs), shift);
+ } else {
+ Sshl(Simd16B(dest), Simd16B(lhs), shift);
+ }
+}
+
+void MacroAssemblerCompat::rightShiftInt16x8(FloatRegister lhs, Register rhs,
+ FloatRegister dest,
+ bool isUnsigned) {
+ ScratchSimd128Scope scratch_(asMasm());
+ ARMFPRegister shift = Simd8H(scratch_);
+
+ Dup(shift, ARMRegister(rhs, 32));
+ Neg(shift, shift);
+
+ if (isUnsigned) {
+ Ushl(Simd8H(dest), Simd8H(lhs), shift);
+ } else {
+ Sshl(Simd8H(dest), Simd8H(lhs), shift);
+ }
+}
+
+void MacroAssemblerCompat::rightShiftInt32x4(FloatRegister lhs, Register rhs,
+ FloatRegister dest,
+ bool isUnsigned) {
+ ScratchSimd128Scope scratch_(asMasm());
+ ARMFPRegister shift = Simd4S(scratch_);
+
+ Dup(shift, ARMRegister(rhs, 32));
+ Neg(shift, shift);
+
+ if (isUnsigned) {
+ Ushl(Simd4S(dest), Simd4S(lhs), shift);
+ } else {
+ Sshl(Simd4S(dest), Simd4S(lhs), shift);
+ }
+}
+
+void MacroAssemblerCompat::rightShiftInt64x2(FloatRegister lhs, Register rhs,
+ FloatRegister dest,
+ bool isUnsigned) {
+ ScratchSimd128Scope scratch_(asMasm());
+ ARMFPRegister shift = Simd2D(scratch_);
+
+ Dup(shift, ARMRegister(rhs, 64));
+ Neg(shift, shift);
+
+ if (isUnsigned) {
+ Ushl(Simd2D(dest), Simd2D(lhs), shift);
+ } else {
+ Sshl(Simd2D(dest), Simd2D(lhs), shift);
+ }
+}
+
+void MacroAssembler::reserveStack(uint32_t amount) {
+ // TODO: This bumps |sp| every time we reserve using a second register.
+ // It would save some instructions if we had a fixed frame size.
+ vixl::MacroAssembler::Claim(Operand(amount));
+ adjustFrame(amount);
+}
+
+void MacroAssembler::Push(RegisterOrSP reg) {
+ if (IsHiddenSP(reg)) {
+ push(sp);
+ } else {
+ push(AsRegister(reg));
+ }
+ adjustFrame(sizeof(intptr_t));
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// MacroAssembler high-level usage.
+
+void MacroAssembler::flush() { Assembler::flush(); }
+
+// ===============================================================
+// Stack manipulation functions.
+
+// Routines for saving/restoring registers on the stack. The format is:
+//
+// (highest address)
+//
+// integer (X) regs in any order size: 8 * # int regs
+//
+// if # int regs is odd,
+// then an 8 byte alignment hole size: 0 or 8
+//
+// double (D) regs in any order size: 8 * # double regs
+//
+// if # double regs is odd,
+// then an 8 byte alignment hole size: 0 or 8
+//
+// vector (Q) regs in any order size: 16 * # vector regs
+//
+// (lowest address)
+//
+// Hence the size of the save area is 0 % 16. And, provided that the base
+// (highest) address is 16-aligned, then the vector reg save/restore accesses
+// will also be 16-aligned, as will pairwise operations for the double regs.
+//
+// Implied by this is that the format of the double and vector dump area
+// corresponds with what FloatRegister::GetPushSizeInBytes computes.
+// See block comment in MacroAssembler.h for more details.
+
+size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
+ size_t numIntRegs = set.gprs().size();
+ return ((numIntRegs + 1) & ~1) * sizeof(intptr_t) +
+ FloatRegister::GetPushSizeInBytes(set.fpus());
+}
+
+// Generate code to dump the values in `set`, either on the stack if `dest` is
+// `Nothing` or working backwards from the address denoted by `dest` if it is
+// `Some`. These two cases are combined so as to minimise the chance of
+// mistakenly generating different formats for the same `set`, given that the
+// `Some` `dest` case is used extremely rarely.
+static void PushOrStoreRegsInMask(MacroAssembler* masm, LiveRegisterSet set,
+ mozilla::Maybe<Address> dest) {
+ static_assert(sizeof(FloatRegisters::RegisterContent) == 16);
+
+ // If we're saving to arbitrary memory, check the destination is big enough.
+ if (dest) {
+ mozilla::DebugOnly<size_t> bytesRequired =
+ masm->PushRegsInMaskSizeInBytes(set);
+ MOZ_ASSERT(dest->offset >= 0);
+ MOZ_ASSERT(((size_t)dest->offset) >= bytesRequired);
+ }
+
+ // Note the high limit point; we'll check it again later.
+ mozilla::DebugOnly<size_t> maxExtentInitial =
+ dest ? dest->offset : masm->framePushed();
+
+ // Gather up the integer registers in groups of four, and either push each
+ // group as a single transfer so as to minimise the number of stack pointer
+ // changes, or write them individually to memory. Take care to ensure the
+ // space used remains 16-aligned.
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();) {
+ vixl::CPURegister src[4] = {vixl::NoCPUReg, vixl::NoCPUReg, vixl::NoCPUReg,
+ vixl::NoCPUReg};
+ size_t i;
+ for (i = 0; i < 4 && iter.more(); i++) {
+ src[i] = ARMRegister(*iter, 64);
+ ++iter;
+ }
+ MOZ_ASSERT(i > 0);
+
+ if (i == 1 || i == 3) {
+ // Ensure the stack remains 16-aligned
+ MOZ_ASSERT(!iter.more());
+ src[i] = vixl::xzr;
+ i++;
+ }
+ MOZ_ASSERT(i == 2 || i == 4);
+
+ if (dest) {
+ for (size_t j = 0; j < i; j++) {
+ Register ireg = Register::FromCode(src[j].IsZero() ? Registers::xzr
+ : src[j].code());
+ dest->offset -= sizeof(intptr_t);
+ masm->storePtr(ireg, *dest);
+ }
+ } else {
+ masm->adjustFrame(i * 8);
+ masm->vixl::MacroAssembler::Push(src[0], src[1], src[2], src[3]);
+ }
+ }
+
+ // Now the same for the FP double registers. Note that because of how
+ // ReduceSetForPush works, an underlying AArch64 SIMD/FP register can either
+ // be present as a double register, or as a V128 register, but not both.
+ // Firstly, round up the registers to be pushed.
+
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+ vixl::CPURegister allSrcs[FloatRegisters::TotalPhys];
+ size_t numAllSrcs = 0;
+
+ for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ if (reg.isDouble()) {
+ MOZ_RELEASE_ASSERT(numAllSrcs < FloatRegisters::TotalPhys);
+ allSrcs[numAllSrcs] = ARMFPRegister(reg, 64);
+ numAllSrcs++;
+ } else {
+ MOZ_ASSERT(reg.isSimd128());
+ }
+ }
+ MOZ_RELEASE_ASSERT(numAllSrcs <= FloatRegisters::TotalPhys);
+
+ if ((numAllSrcs & 1) == 1) {
+ // We've got an odd number of doubles. In order to maintain 16-alignment,
+ // push the last register twice. We'll skip over the duplicate in
+ // PopRegsInMaskIgnore.
+ allSrcs[numAllSrcs] = allSrcs[numAllSrcs - 1];
+ numAllSrcs++;
+ }
+ MOZ_RELEASE_ASSERT(numAllSrcs <= FloatRegisters::TotalPhys);
+ MOZ_RELEASE_ASSERT((numAllSrcs & 1) == 0);
+
+ // And now generate the transfers.
+ size_t i;
+ if (dest) {
+ for (i = 0; i < numAllSrcs; i++) {
+ FloatRegister freg =
+ FloatRegister(FloatRegisters::FPRegisterID(allSrcs[i].code()),
+ FloatRegisters::Kind::Double);
+ dest->offset -= sizeof(double);
+ masm->storeDouble(freg, *dest);
+ }
+ } else {
+ i = 0;
+ while (i < numAllSrcs) {
+ vixl::CPURegister src[4] = {vixl::NoCPUReg, vixl::NoCPUReg,
+ vixl::NoCPUReg, vixl::NoCPUReg};
+ size_t j;
+ for (j = 0; j < 4 && j + i < numAllSrcs; j++) {
+ src[j] = allSrcs[j + i];
+ }
+ masm->adjustFrame(8 * j);
+ masm->vixl::MacroAssembler::Push(src[0], src[1], src[2], src[3]);
+ i += j;
+ }
+ }
+ MOZ_ASSERT(i == numAllSrcs);
+
+ // Finally, deal with the SIMD (V128) registers. This is a bit simpler
+ // as there's no need for special-casing to maintain 16-alignment.
+
+ numAllSrcs = 0;
+ for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ if (reg.isSimd128()) {
+ MOZ_RELEASE_ASSERT(numAllSrcs < FloatRegisters::TotalPhys);
+ allSrcs[numAllSrcs] = ARMFPRegister(reg, 128);
+ numAllSrcs++;
+ }
+ }
+ MOZ_RELEASE_ASSERT(numAllSrcs <= FloatRegisters::TotalPhys);
+
+ // Generate the transfers.
+ if (dest) {
+ for (i = 0; i < numAllSrcs; i++) {
+ FloatRegister freg =
+ FloatRegister(FloatRegisters::FPRegisterID(allSrcs[i].code()),
+ FloatRegisters::Kind::Simd128);
+ dest->offset -= FloatRegister::SizeOfSimd128;
+ masm->storeUnalignedSimd128(freg, *dest);
+ }
+ } else {
+ i = 0;
+ while (i < numAllSrcs) {
+ vixl::CPURegister src[4] = {vixl::NoCPUReg, vixl::NoCPUReg,
+ vixl::NoCPUReg, vixl::NoCPUReg};
+ size_t j;
+ for (j = 0; j < 4 && j + i < numAllSrcs; j++) {
+ src[j] = allSrcs[j + i];
+ }
+ masm->adjustFrame(16 * j);
+ masm->vixl::MacroAssembler::Push(src[0], src[1], src[2], src[3]);
+ i += j;
+ }
+ }
+ MOZ_ASSERT(i == numAllSrcs);
+
+ // Final overrun check.
+ if (dest) {
+ MOZ_ASSERT(maxExtentInitial - dest->offset ==
+ masm->PushRegsInMaskSizeInBytes(set));
+ } else {
+ MOZ_ASSERT(masm->framePushed() - maxExtentInitial ==
+ masm->PushRegsInMaskSizeInBytes(set));
+ }
+}
+
+void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
+ PushOrStoreRegsInMask(this, set, mozilla::Nothing());
+}
+
+void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
+ Register scratch) {
+ PushOrStoreRegsInMask(this, set, mozilla::Some(dest));
+}
+
+// This is a helper function for PopRegsInMaskIgnore below. It emits the
+// loads described by dests[0] and [1] and offsets[0] and [1], generating a
+// load-pair if it can.
+static void GeneratePendingLoadsThenFlush(MacroAssembler* masm,
+ vixl::CPURegister* dests,
+ uint32_t* offsets,
+ uint32_t transactionSize) {
+ // Generate the loads ..
+ if (!dests[0].IsNone()) {
+ if (!dests[1].IsNone()) {
+ // [0] and [1] both present.
+ if (offsets[0] + transactionSize == offsets[1]) {
+ masm->Ldp(dests[0], dests[1],
+ MemOperand(masm->GetStackPointer64(), offsets[0]));
+ } else {
+ // Theoretically we could check for a load-pair with the destinations
+ // switched, but our callers will never generate that. Hence there's
+ // no loss in giving up at this point and generating two loads.
+ masm->Ldr(dests[0], MemOperand(masm->GetStackPointer64(), offsets[0]));
+ masm->Ldr(dests[1], MemOperand(masm->GetStackPointer64(), offsets[1]));
+ }
+ } else {
+ // [0] only.
+ masm->Ldr(dests[0], MemOperand(masm->GetStackPointer64(), offsets[0]));
+ }
+ } else {
+ if (!dests[1].IsNone()) {
+ // [1] only. Can't happen because callers always fill [0] before [1].
+ MOZ_CRASH("GenerateLoadsThenFlush");
+ } else {
+ // Neither entry valid. This can happen.
+ }
+ }
+
+ // .. and flush.
+ dests[0] = dests[1] = vixl::NoCPUReg;
+ offsets[0] = offsets[1] = 0;
+}
+
+void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
+ LiveRegisterSet ignore) {
+ mozilla::DebugOnly<size_t> framePushedInitial = framePushed();
+
+ // The offset of the data from the stack pointer.
+ uint32_t offset = 0;
+
+ // The set of FP/SIMD registers we need to restore.
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+
+ // The set of registers to ignore. BroadcastToAllSizes() is used to avoid
+ // any ambiguities arising from (eg) `fpuSet` containing q17 but `ignore`
+ // containing d17.
+ FloatRegisterSet ignoreFpusBroadcasted(
+ FloatRegister::BroadcastToAllSizes(ignore.fpus()));
+
+ // First recover the SIMD (V128) registers. This is straightforward in that
+ // we don't need to think about alignment holes.
+
+ // These three form a two-entry queue that holds loads that we know we
+ // need, but which we haven't yet emitted.
+ vixl::CPURegister pendingDests[2] = {vixl::NoCPUReg, vixl::NoCPUReg};
+ uint32_t pendingOffsets[2] = {0, 0};
+ size_t nPending = 0;
+
+ for (FloatRegisterIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ if (reg.isDouble()) {
+ continue;
+ }
+ MOZ_RELEASE_ASSERT(reg.isSimd128());
+
+ uint32_t offsetForReg = offset;
+ offset += FloatRegister::SizeOfSimd128;
+
+ if (ignoreFpusBroadcasted.hasRegisterIndex(reg)) {
+ continue;
+ }
+
+ MOZ_ASSERT(nPending <= 2);
+ if (nPending == 2) {
+ GeneratePendingLoadsThenFlush(this, pendingDests, pendingOffsets, 16);
+ nPending = 0;
+ }
+ pendingDests[nPending] = ARMFPRegister(reg, 128);
+ pendingOffsets[nPending] = offsetForReg;
+ nPending++;
+ }
+ GeneratePendingLoadsThenFlush(this, pendingDests, pendingOffsets, 16);
+ nPending = 0;
+
+ MOZ_ASSERT((offset % 16) == 0);
+
+ // Now recover the FP double registers. This is more tricky in that we need
+ // to skip over the lowest-addressed of them if the number of them was odd.
+
+ if ((((fpuSet.bits() & FloatRegisters::AllDoubleMask).size()) & 1) == 1) {
+ offset += sizeof(double);
+ }
+
+ for (FloatRegisterIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ if (reg.isSimd128()) {
+ continue;
+ }
+ /* true but redundant, per loop above: MOZ_RELEASE_ASSERT(reg.isDouble()) */
+
+ uint32_t offsetForReg = offset;
+ offset += sizeof(double);
+
+ if (ignoreFpusBroadcasted.hasRegisterIndex(reg)) {
+ continue;
+ }
+
+ MOZ_ASSERT(nPending <= 2);
+ if (nPending == 2) {
+ GeneratePendingLoadsThenFlush(this, pendingDests, pendingOffsets, 8);
+ nPending = 0;
+ }
+ pendingDests[nPending] = ARMFPRegister(reg, 64);
+ pendingOffsets[nPending] = offsetForReg;
+ nPending++;
+ }
+ GeneratePendingLoadsThenFlush(this, pendingDests, pendingOffsets, 8);
+ nPending = 0;
+
+ MOZ_ASSERT((offset % 16) == 0);
+ MOZ_ASSERT(offset == set.fpus().getPushSizeInBytes());
+
+ // And finally recover the integer registers, again skipping an alignment
+ // hole if it exists.
+
+ if ((set.gprs().size() & 1) == 1) {
+ offset += sizeof(uint64_t);
+ }
+
+ for (GeneralRegisterIterator iter(set.gprs()); iter.more(); ++iter) {
+ Register reg = *iter;
+
+ uint32_t offsetForReg = offset;
+ offset += sizeof(uint64_t);
+
+ if (ignore.has(reg)) {
+ continue;
+ }
+
+ MOZ_ASSERT(nPending <= 2);
+ if (nPending == 2) {
+ GeneratePendingLoadsThenFlush(this, pendingDests, pendingOffsets, 8);
+ nPending = 0;
+ }
+ pendingDests[nPending] = ARMRegister(reg, 64);
+ pendingOffsets[nPending] = offsetForReg;
+ nPending++;
+ }
+ GeneratePendingLoadsThenFlush(this, pendingDests, pendingOffsets, 8);
+
+ MOZ_ASSERT((offset % 16) == 0);
+
+ size_t bytesPushed = PushRegsInMaskSizeInBytes(set);
+ MOZ_ASSERT(offset == bytesPushed);
+ freeStack(bytesPushed);
+}
+
+void MacroAssembler::Push(Register reg) {
+ push(reg);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(Register reg1, Register reg2, Register reg3,
+ Register reg4) {
+ push(reg1, reg2, reg3, reg4);
+ adjustFrame(4 * sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const Imm32 imm) {
+ push(imm);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const ImmWord imm) {
+ push(imm);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const ImmPtr imm) {
+ push(imm);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const ImmGCPtr ptr) {
+ push(ptr);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(FloatRegister f) {
+ push(f);
+ adjustFrame(sizeof(double));
+}
+
+void MacroAssembler::PushBoxed(FloatRegister reg) {
+ subFromStackPtr(Imm32(sizeof(double)));
+ boxDouble(reg, Address(getStackPointer(), 0));
+ adjustFrame(sizeof(double));
+}
+
+void MacroAssembler::Pop(Register reg) {
+ pop(reg);
+ adjustFrame(-1 * int64_t(sizeof(int64_t)));
+}
+
+void MacroAssembler::Pop(FloatRegister f) {
+ loadDouble(Address(getStackPointer(), 0), f);
+ freeStack(sizeof(double));
+}
+
+void MacroAssembler::Pop(const ValueOperand& val) {
+ pop(val);
+ adjustFrame(-1 * int64_t(sizeof(int64_t)));
+}
+
+// ===============================================================
+// Simple call functions.
+
+CodeOffset MacroAssembler::call(Register reg) {
+ // This sync has been observed (and is expected) to be necessary.
+ // eg testcase: tests/debug/bug1107525.js
+ syncStackPtr();
+ Blr(ARMRegister(reg, 64));
+ return CodeOffset(currentOffset());
+}
+
+CodeOffset MacroAssembler::call(Label* label) {
+ // This sync has been observed (and is expected) to be necessary.
+ // eg testcase: tests/basic/testBug504520Harder.js
+ syncStackPtr();
+ Bl(label);
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::call(ImmPtr imm) {
+ // This sync has been observed (and is expected) to be necessary.
+ // eg testcase: asm.js/testTimeout5.js
+ syncStackPtr();
+ vixl::UseScratchRegisterScope temps(this);
+ MOZ_ASSERT(temps.IsAvailable(ScratchReg64)); // ip0
+ temps.Exclude(ScratchReg64);
+ movePtr(imm, ScratchReg64.asUnsized());
+ Blr(ScratchReg64);
+}
+
+void MacroAssembler::call(ImmWord imm) { call(ImmPtr((void*)imm.value)); }
+
+CodeOffset MacroAssembler::call(wasm::SymbolicAddress imm) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ // This sync is believed to be necessary, although no case in jit-test/tests
+ // has been observed to cause SP != PSP here.
+ syncStackPtr();
+ movePtr(imm, scratch);
+ Blr(ARMRegister(scratch, 64));
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::call(const Address& addr) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ // This sync has been observed (and is expected) to be necessary.
+ // eg testcase: tests/backup-point-bug1315634.js
+ syncStackPtr();
+ loadPtr(addr, scratch);
+ Blr(ARMRegister(scratch, 64));
+}
+
+void MacroAssembler::call(JitCode* c) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ // This sync has been observed (and is expected) to be necessary.
+ // eg testcase: arrays/new-array-undefined-undefined-more-args-2.js
+ syncStackPtr();
+ BufferOffset off = immPool64(scratch64, uint64_t(c->raw()));
+ addPendingJump(off, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ blr(scratch64);
+}
+
+CodeOffset MacroAssembler::callWithPatch() {
+ // This needs to sync. Wasm goes through this one for intramodule calls.
+ //
+ // In other cases, wasm goes through masm.wasmCallImport(),
+ // masm.wasmCallBuiltinInstanceMethod, masm.wasmCallIndirect, all of which
+ // sync.
+ //
+ // This sync is believed to be necessary, although no case in jit-test/tests
+ // has been observed to cause SP != PSP here.
+ syncStackPtr();
+ bl(0, LabelDoc());
+ return CodeOffset(currentOffset());
+}
+void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
+ Instruction* inst = getInstructionAt(BufferOffset(callerOffset - 4));
+ MOZ_ASSERT(inst->IsBL());
+ ptrdiff_t relTarget = (int)calleeOffset - ((int)callerOffset - 4);
+ ptrdiff_t relTarget00 = relTarget >> 2;
+ MOZ_RELEASE_ASSERT((relTarget & 0x3) == 0);
+ MOZ_RELEASE_ASSERT(vixl::IsInt26(relTarget00));
+ bl(inst, relTarget00);
+}
+
+CodeOffset MacroAssembler::farJumpWithPatch() {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ const ARMRegister scratch2 = temps.AcquireX();
+
+ AutoForbidPoolsAndNops afp(this,
+ /* max number of instructions in scope = */ 7);
+
+ mozilla::DebugOnly<uint32_t> before = currentOffset();
+
+ align(8); // At most one nop
+
+ Label branch;
+ adr(scratch2, &branch);
+ ldr(scratch, vixl::MemOperand(scratch2, 4));
+ add(scratch2, scratch2, scratch);
+ CodeOffset offs(currentOffset());
+ bind(&branch);
+ br(scratch2);
+ Emit(UINT32_MAX);
+ Emit(UINT32_MAX);
+
+ mozilla::DebugOnly<uint32_t> after = currentOffset();
+
+ MOZ_ASSERT(after - before == 24 || after - before == 28);
+
+ return offs;
+}
+
+void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
+ Instruction* inst1 = getInstructionAt(BufferOffset(farJump.offset() + 4));
+ Instruction* inst2 = getInstructionAt(BufferOffset(farJump.offset() + 8));
+
+ int64_t distance = (int64_t)targetOffset - (int64_t)farJump.offset();
+
+ MOZ_ASSERT(inst1->InstructionBits() == UINT32_MAX);
+ MOZ_ASSERT(inst2->InstructionBits() == UINT32_MAX);
+
+ inst1->SetInstructionBits((uint32_t)distance);
+ inst2->SetInstructionBits((uint32_t)(distance >> 32));
+}
+
+CodeOffset MacroAssembler::nopPatchableToCall() {
+ AutoForbidPoolsAndNops afp(this,
+ /* max number of instructions in scope = */ 1);
+ Nop();
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
+ uint8_t* inst = call - 4;
+ Instruction* instr = reinterpret_cast<Instruction*>(inst);
+ MOZ_ASSERT(instr->IsBL() || instr->IsNOP());
+ bl(instr, (target - inst) >> 2);
+}
+
+void MacroAssembler::patchCallToNop(uint8_t* call) {
+ uint8_t* inst = call - 4;
+ Instruction* instr = reinterpret_cast<Instruction*>(inst);
+ MOZ_ASSERT(instr->IsBL() || instr->IsNOP());
+ nop(instr);
+}
+
+void MacroAssembler::pushReturnAddress() {
+ MOZ_RELEASE_ASSERT(!sp.Is(GetStackPointer64()), "Not valid");
+ push(lr);
+}
+
+void MacroAssembler::popReturnAddress() {
+ MOZ_RELEASE_ASSERT(!sp.Is(GetStackPointer64()), "Not valid");
+ pop(lr);
+}
+
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+ // Because wasm operates without the need for dynamic alignment of SP, it is
+ // implied that this routine should never be called when generating wasm.
+ MOZ_ASSERT(!IsCompilingWasm());
+
+ // The following won't work for SP -- needs slightly different logic.
+ MOZ_RELEASE_ASSERT(GetStackPointer64().Is(PseudoStackPointer64));
+
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ int64_t alignment = ~(int64_t(ABIStackAlignment) - 1);
+ ARMRegister scratch64(scratch, 64);
+ MOZ_ASSERT(!scratch64.Is(PseudoStackPointer64));
+
+ // Always save LR -- Baseline ICs assume that LR isn't modified.
+ push(lr);
+
+ // Remember the stack address on entry. This is reloaded in callWithABIPost
+ // below.
+ Mov(scratch64, PseudoStackPointer64);
+
+ // Make alignment, including the effective push of the previous sp.
+ Sub(PseudoStackPointer64, PseudoStackPointer64, Operand(8));
+ And(PseudoStackPointer64, PseudoStackPointer64, Operand(alignment));
+ syncStackPtr();
+
+ // Store previous sp to the top of the stack, aligned. This is also
+ // reloaded in callWithABIPost.
+ Str(scratch64, MemOperand(PseudoStackPointer64, 0));
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ // wasm operates without the need for dynamic alignment of SP.
+ MOZ_ASSERT(!(dynamicAlignment_ && callFromWasm));
+
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ // ARM64 *really* wants SP to always be 16-aligned, so ensure this now.
+ if (dynamicAlignment_) {
+ stackForCall += ComputeByteAlignment(stackForCall, StackAlignment);
+ } else {
+ // This can happen when we attach out-of-line stubs for rare cases. For
+ // example CodeGenerator::visitWasmTruncateToInt32 adds an out-of-line
+ // chunk.
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(
+ stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+ }
+
+ *stackAdjust = stackForCall;
+ reserveStack(*stackAdjust);
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_) {
+ return;
+ }
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ // Call boundaries communicate stack via SP.
+ // (jseward, 2021Mar03) This sync may well be redundant, given that all of
+ // the MacroAssembler::call methods generate a sync before the call.
+ // Removing it does not cause any failures for all of jit-tests.
+ syncStackPtr();
+
+ assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool callFromWasm) {
+ // wasm operates without the need for dynamic alignment of SP.
+ MOZ_ASSERT(!(dynamicAlignment_ && callFromWasm));
+
+ // Call boundaries communicate stack via SP, so we must resync PSP now.
+ initPseudoStackPtr();
+
+ freeStack(stackAdjust);
+
+ if (dynamicAlignment_) {
+ // This then-clause makes more sense if you first read
+ // setupUnalignedABICall above.
+ //
+ // Restore the stack pointer from entry. The stack pointer will have been
+ // saved by setupUnalignedABICall. This is fragile in that it assumes
+ // that uses of this routine (callWithABIPost) with `dynamicAlignment_ ==
+ // true` are preceded by matching calls to setupUnalignedABICall. But
+ // there's nothing that enforce that mechanically. If we really want to
+ // enforce this, we could add a debug-only CallWithABIState enum to the
+ // MacroAssembler and assert that setupUnalignedABICall updates it before
+ // we get here, then reset it to its initial state.
+ Ldr(GetStackPointer64(), MemOperand(GetStackPointer64(), 0));
+ syncStackPtr();
+
+ // Restore LR. This restores LR to the value stored by
+ // setupUnalignedABICall, which should have been called just before
+ // callWithABIPre. This is, per the above comment, also fragile.
+ pop(lr);
+
+ // SP may be < PSP now. That is expected from the behaviour of `pop`. It
+ // is not clear why the following `syncStackPtr` is necessary, but it is:
+ // without it, the following test segfaults:
+ // tests/backup-point-bug1315634.js
+ syncStackPtr();
+ }
+
+ // If the ABI's return regs are where ION is expecting them, then
+ // no other work needs to be done.
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ movePtr(fun, scratch);
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(scratch);
+ callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ loadPtr(fun, scratch);
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(scratch);
+ callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Jit Frames.
+
+uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
+ enterNoPool(3);
+ Label fakeCallsite;
+
+ Adr(ARMRegister(scratch, 64), &fakeCallsite);
+ Push(scratch);
+ bind(&fakeCallsite);
+ uint32_t pseudoReturnOffset = currentOffset();
+
+ leaveNoPool();
+ return pseudoReturnOffset;
+}
+
+bool MacroAssemblerCompat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
+ asMasm().PushFrameDescriptor(FrameType::IonJS);
+ asMasm().Push(ImmPtr(fakeReturnAddr));
+ asMasm().Push(FramePointer);
+ return true;
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ boxNonDouble(ValueTypeFromMIRType(type), reg.gpr(), dest);
+ return;
+ }
+
+ ScratchDoubleScope scratch(*this);
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ boxDouble(freg, dest, scratch);
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ if (src == dest) {
+ return;
+ }
+ movePtr(src.valueReg(), dest.valueReg());
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ if (!src.isGCThing()) {
+ movePtr(ImmWord(src.asRawBits()), dest.valueReg());
+ return;
+ }
+
+ BufferOffset load =
+ movePatchablePtr(ImmPtr(src.bitsAsPunboxPointer()), dest.valueReg());
+ writeDataRelocation(src, load);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+ And(ARMRegister(buffer, 64), ARMRegister(ptr, 64),
+ Operand(int32_t(~gc::ChunkMask)));
+ loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+ Register temp, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(ptr != temp);
+ MOZ_ASSERT(ptr != ScratchReg &&
+ ptr != ScratchReg2); // Both may be used internally.
+ MOZ_ASSERT(temp != ScratchReg && temp != ScratchReg2);
+
+ And(ARMRegister(temp, 64), ARMRegister(ptr, 64),
+ Operand(int32_t(~gc::ChunkMask)));
+ branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
+ ImmWord(0), label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ branchValueIsNurseryCellImpl(cond, address, temp, label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ branchValueIsNurseryCellImpl(cond, value, temp, label);
+}
+
+template <typename T>
+void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
+ const T& value, Register temp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(temp != ScratchReg &&
+ temp != ScratchReg2); // Both may be used internally.
+
+ Label done;
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+
+ getGCThingValueChunk(value, temp);
+ branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
+ ImmWord(0), label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != lhs.valueReg());
+ moveValue(rhs, ValueOperand(scratch64.asUnsized()));
+ Cmp(ARMRegister(lhs.valueReg(), 64), scratch64);
+ B(label, cond);
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ boxDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ if (value.constant()) {
+ storeValue(value.value(), dest);
+ } else {
+ storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
+ dest);
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); }
+
+// ========================================================================
+// wasm support
+
+CodeOffset MacroAssembler::wasmTrapInstruction() {
+ AutoForbidPoolsAndNops afp(this,
+ /* max number of instructions in scope = */ 1);
+ CodeOffset offs(currentOffset());
+ Unreachable();
+ return offs;
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* ok) {
+ branch32(cond, index, boundsCheckLimit, ok);
+ if (JitOptions.spectreIndexMasking) {
+ csel(ARMRegister(index, 32), vixl::wzr, ARMRegister(index, 32), cond);
+ }
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* ok) {
+ branch32(cond, index, boundsCheckLimit, ok);
+ if (JitOptions.spectreIndexMasking) {
+ csel(ARMRegister(index, 32), vixl::wzr, ARMRegister(index, 32), cond);
+ }
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok) {
+ branchPtr(cond, index.reg, boundsCheckLimit.reg, ok);
+ if (JitOptions.spectreIndexMasking) {
+ csel(ARMRegister(index.reg, 64), vixl::xzr, ARMRegister(index.reg, 64),
+ cond);
+ }
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok) {
+ branchPtr(InvertCondition(cond), boundsCheckLimit, index.reg, ok);
+ if (JitOptions.spectreIndexMasking) {
+ csel(ARMRegister(index.reg, 64), vixl::xzr, ARMRegister(index.reg, 64),
+ cond);
+ }
+}
+
+// FCVTZU behaves as follows:
+//
+// on NaN it produces zero
+// on too large it produces UINT_MAX (for appropriate type)
+// on too small it produces zero
+//
+// FCVTZS behaves as follows:
+//
+// on NaN it produces zero
+// on too large it produces INT_MAX (for appropriate type)
+// on too small it produces INT_MIN (ditto)
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input_,
+ Register output_,
+ bool isSaturating,
+ Label* oolEntry) {
+ ARMRegister output(output_, 32);
+ ARMFPRegister input(input_, 64);
+ Fcvtzu(output, input);
+ if (!isSaturating) {
+ Cmp(output, 0);
+ Ccmp(output, -1, vixl::ZFlag, Assembler::NotEqual);
+ B(oolEntry, Assembler::Equal);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input_,
+ Register output_,
+ bool isSaturating,
+ Label* oolEntry) {
+ ARMRegister output(output_, 32);
+ ARMFPRegister input(input_, 32);
+ Fcvtzu(output, input);
+ if (!isSaturating) {
+ Cmp(output, 0);
+ Ccmp(output, -1, vixl::ZFlag, Assembler::NotEqual);
+ B(oolEntry, Assembler::Equal);
+ }
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input_,
+ Register output_,
+ bool isSaturating,
+ Label* oolEntry) {
+ ARMRegister output(output_, 32);
+ ARMFPRegister input(input_, 64);
+ Fcvtzs(output, input);
+ if (!isSaturating) {
+ Cmp(output, 0);
+ Ccmp(output, INT32_MAX, vixl::ZFlag, Assembler::NotEqual);
+ Ccmp(output, INT32_MIN, vixl::ZFlag, Assembler::NotEqual);
+ B(oolEntry, Assembler::Equal);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input_,
+ Register output_,
+ bool isSaturating,
+ Label* oolEntry) {
+ ARMRegister output(output_, 32);
+ ARMFPRegister input(input_, 32);
+ Fcvtzs(output, input);
+ if (!isSaturating) {
+ Cmp(output, 0);
+ Ccmp(output, INT32_MAX, vixl::ZFlag, Assembler::NotEqual);
+ Ccmp(output, INT32_MIN, vixl::ZFlag, Assembler::NotEqual);
+ B(oolEntry, Assembler::Equal);
+ }
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt64(
+ FloatRegister input_, Register64 output_, bool isSaturating,
+ Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_ASSERT(tempDouble.isInvalid());
+
+ ARMRegister output(output_.reg, 64);
+ ARMFPRegister input(input_, 64);
+ Fcvtzu(output, input);
+ if (!isSaturating) {
+ Cmp(output, 0);
+ Ccmp(output, -1, vixl::ZFlag, Assembler::NotEqual);
+ B(oolEntry, Assembler::Equal);
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt64(
+ FloatRegister input_, Register64 output_, bool isSaturating,
+ Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_ASSERT(tempDouble.isInvalid());
+
+ ARMRegister output(output_.reg, 64);
+ ARMFPRegister input(input_, 32);
+ Fcvtzu(output, input);
+ if (!isSaturating) {
+ Cmp(output, 0);
+ Ccmp(output, -1, vixl::ZFlag, Assembler::NotEqual);
+ B(oolEntry, Assembler::Equal);
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt64(
+ FloatRegister input_, Register64 output_, bool isSaturating,
+ Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_ASSERT(tempDouble.isInvalid());
+
+ ARMRegister output(output_.reg, 64);
+ ARMFPRegister input(input_, 64);
+ Fcvtzs(output, input);
+ if (!isSaturating) {
+ Cmp(output, 0);
+ Ccmp(output, INT64_MAX, vixl::ZFlag, Assembler::NotEqual);
+ Ccmp(output, INT64_MIN, vixl::ZFlag, Assembler::NotEqual);
+ B(oolEntry, Assembler::Equal);
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt64(
+ FloatRegister input_, Register64 output_, bool isSaturating,
+ Label* oolEntry, Label* oolRejoin, FloatRegister tempDouble) {
+ ARMRegister output(output_.reg, 64);
+ ARMFPRegister input(input_, 32);
+ Fcvtzs(output, input);
+ if (!isSaturating) {
+ Cmp(output, 0);
+ Ccmp(output, INT64_MAX, vixl::ZFlag, Assembler::NotEqual);
+ Ccmp(output, INT64_MIN, vixl::ZFlag, Assembler::NotEqual);
+ B(oolEntry, Assembler::Equal);
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ Label notNaN;
+ branchFloat(Assembler::DoubleOrdered, input, input, &notNaN);
+ wasmTrap(wasm::Trap::InvalidConversionToInteger, off);
+ bind(&notNaN);
+
+ Label isOverflow;
+ const float two_31 = -float(INT32_MIN);
+ ScratchFloat32Scope fpscratch(*this);
+ if (flags & TRUNC_UNSIGNED) {
+ loadConstantFloat32(two_31 * 2, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &isOverflow);
+ loadConstantFloat32(-1.0f, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThan, input, fpscratch, rejoin);
+ } else {
+ loadConstantFloat32(two_31, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &isOverflow);
+ loadConstantFloat32(-two_31, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, fpscratch, rejoin);
+ }
+ bind(&isOverflow);
+ wasmTrap(wasm::Trap::IntegerOverflow, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ Label notNaN;
+ branchDouble(Assembler::DoubleOrdered, input, input, &notNaN);
+ wasmTrap(wasm::Trap::InvalidConversionToInteger, off);
+ bind(&notNaN);
+
+ Label isOverflow;
+ const double two_31 = -double(INT32_MIN);
+ ScratchDoubleScope fpscratch(*this);
+ if (flags & TRUNC_UNSIGNED) {
+ loadConstantDouble(two_31 * 2, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &isOverflow);
+ loadConstantDouble(-1.0, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThan, input, fpscratch, rejoin);
+ } else {
+ loadConstantDouble(two_31, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &isOverflow);
+ loadConstantDouble(-two_31 - 1, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThan, input, fpscratch, rejoin);
+ }
+ bind(&isOverflow);
+ wasmTrap(wasm::Trap::IntegerOverflow, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ Label notNaN;
+ branchFloat(Assembler::DoubleOrdered, input, input, &notNaN);
+ wasmTrap(wasm::Trap::InvalidConversionToInteger, off);
+ bind(&notNaN);
+
+ Label isOverflow;
+ const float two_63 = -float(INT64_MIN);
+ ScratchFloat32Scope fpscratch(*this);
+ if (flags & TRUNC_UNSIGNED) {
+ loadConstantFloat32(two_63 * 2, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &isOverflow);
+ loadConstantFloat32(-1.0f, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThan, input, fpscratch, rejoin);
+ } else {
+ loadConstantFloat32(two_63, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &isOverflow);
+ loadConstantFloat32(-two_63, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, fpscratch, rejoin);
+ }
+ bind(&isOverflow);
+ wasmTrap(wasm::Trap::IntegerOverflow, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ Label notNaN;
+ branchDouble(Assembler::DoubleOrdered, input, input, &notNaN);
+ wasmTrap(wasm::Trap::InvalidConversionToInteger, off);
+ bind(&notNaN);
+
+ Label isOverflow;
+ const double two_63 = -double(INT64_MIN);
+ ScratchDoubleScope fpscratch(*this);
+ if (flags & TRUNC_UNSIGNED) {
+ loadConstantDouble(two_63 * 2, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &isOverflow);
+ loadConstantDouble(-1.0, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThan, input, fpscratch, rejoin);
+ } else {
+ loadConstantDouble(two_63, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &isOverflow);
+ loadConstantDouble(-two_63, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, input, fpscratch, rejoin);
+ }
+ bind(&isOverflow);
+ wasmTrap(wasm::Trap::IntegerOverflow, off);
+}
+
+void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ AnyRegister output) {
+ wasmLoadImpl(access, memoryBase, ptr, output, Register64::Invalid());
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register64 output) {
+ wasmLoadImpl(access, memoryBase, ptr, AnyRegister(), output);
+}
+
+void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Register memoryBase,
+ Register ptr) {
+ wasmStoreImpl(access, value, Register64::Invalid(), memoryBase, ptr);
+}
+
+void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value, Register memoryBase,
+ Register ptr) {
+ wasmStoreImpl(access, AnyRegister(), value, memoryBase, ptr);
+}
+
+void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
+ ExitFrameType type) {
+ // Wasm stubs use the native SP, not the PSP.
+
+ linkExitFrame(cxreg, scratch);
+
+ MOZ_RELEASE_ASSERT(sp.Is(GetStackPointer64()));
+
+ // SP has to be 16-byte aligned when we do a load/store, so push |type| twice
+ // and then add 8 bytes to SP. This leaves SP unaligned.
+ move32(Imm32(int32_t(type)), scratch);
+ push(scratch, scratch);
+ Add(sp, sp, 8);
+
+ // Despite the above assertion, it is possible for control to flow from here
+ // to the code generated by
+ // MacroAssemblerCompat::handleFailureWithHandlerTail without any
+ // intervening assignment to PSP. But handleFailureWithHandlerTail assumes
+ // that PSP is the active stack pointer. Hence the following is necessary
+ // for safety. Note we can't use initPseudoStackPtr here as that would
+ // generate no instructions.
+ Mov(PseudoStackPointer64, sp);
+}
+
+void MacroAssembler::widenInt32(Register r) {
+ move32To64ZeroExtend(r, Register64(r));
+}
+
+// ========================================================================
+// Convert floating point.
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ Ucvtf(ARMFPRegister(dest, 64), ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) {
+ Scvtf(ARMFPRegister(dest, 64), ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::convertUInt64ToFloat32(Register64 src, FloatRegister dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ Ucvtf(ARMFPRegister(dest, 32), ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) {
+ Scvtf(ARMFPRegister(dest, 32), ARMRegister(src.reg, 64));
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ convertInt64ToDouble(Register64(src), dest);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+// The computed MemOperand must be Reg+0 because the load/store exclusive
+// instructions only take a single pointer register.
+
+static MemOperand ComputePointerForAtomic(MacroAssembler& masm,
+ const Address& address,
+ Register scratch) {
+ if (address.offset == 0) {
+ return MemOperand(X(masm, address.base), 0);
+ }
+
+ masm.Add(X(scratch), X(masm, address.base), address.offset);
+ return MemOperand(X(scratch), 0);
+}
+
+static MemOperand ComputePointerForAtomic(MacroAssembler& masm,
+ const BaseIndex& address,
+ Register scratch) {
+ masm.Add(X(scratch), X(masm, address.base),
+ Operand(X(address.index), vixl::LSL, address.scale));
+ if (address.offset) {
+ masm.Add(X(scratch), X(scratch), address.offset);
+ }
+ return MemOperand(X(scratch), 0);
+}
+
+// This sign extends to targetWidth and leaves any higher bits zero.
+
+static void SignOrZeroExtend(MacroAssembler& masm, Scalar::Type srcType,
+ Width targetWidth, Register src, Register dest) {
+ bool signExtend = Scalar::isSignedIntType(srcType);
+
+ switch (Scalar::byteSize(srcType)) {
+ case 1:
+ if (signExtend) {
+ masm.Sbfm(R(dest, targetWidth), R(src, targetWidth), 0, 7);
+ } else {
+ masm.Ubfm(R(dest, targetWidth), R(src, targetWidth), 0, 7);
+ }
+ break;
+ case 2:
+ if (signExtend) {
+ masm.Sbfm(R(dest, targetWidth), R(src, targetWidth), 0, 15);
+ } else {
+ masm.Ubfm(R(dest, targetWidth), R(src, targetWidth), 0, 15);
+ }
+ break;
+ case 4:
+ if (targetWidth == Width::_64) {
+ if (signExtend) {
+ masm.Sbfm(X(dest), X(src), 0, 31);
+ } else {
+ masm.Ubfm(X(dest), X(src), 0, 31);
+ }
+ } else if (src != dest) {
+ masm.Mov(R(dest, targetWidth), R(src, targetWidth));
+ }
+ break;
+ case 8:
+ if (src != dest) {
+ masm.Mov(R(dest, targetWidth), R(src, targetWidth));
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+// Exclusive-loads zero-extend their values to the full width of the X register.
+//
+// Note, we've promised to leave the high bits of the 64-bit register clear if
+// the targetWidth is 32.
+
+static void LoadExclusive(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type srcType, Width targetWidth,
+ MemOperand ptr, Register dest) {
+ bool signExtend = Scalar::isSignedIntType(srcType);
+
+ // With this address form, a single native ldxr* will be emitted, and the
+ // AutoForbidPoolsAndNops ensures that the metadata is emitted at the address
+ // of the ldxr*.
+ MOZ_ASSERT(ptr.IsImmediateOffset() && ptr.offset() == 0);
+
+ switch (Scalar::byteSize(srcType)) {
+ case 1: {
+ {
+ AutoForbidPoolsAndNops afp(
+ &masm,
+ /* max number of instructions in scope = */ 1);
+ if (access) {
+ masm.append(*access, masm.currentOffset());
+ }
+ masm.Ldxrb(W(dest), ptr);
+ }
+ if (signExtend) {
+ masm.Sbfm(R(dest, targetWidth), R(dest, targetWidth), 0, 7);
+ }
+ break;
+ }
+ case 2: {
+ {
+ AutoForbidPoolsAndNops afp(
+ &masm,
+ /* max number of instructions in scope = */ 1);
+ if (access) {
+ masm.append(*access, masm.currentOffset());
+ }
+ masm.Ldxrh(W(dest), ptr);
+ }
+ if (signExtend) {
+ masm.Sbfm(R(dest, targetWidth), R(dest, targetWidth), 0, 15);
+ }
+ break;
+ }
+ case 4: {
+ {
+ AutoForbidPoolsAndNops afp(
+ &masm,
+ /* max number of instructions in scope = */ 1);
+ if (access) {
+ masm.append(*access, masm.currentOffset());
+ }
+ masm.Ldxr(W(dest), ptr);
+ }
+ if (targetWidth == Width::_64 && signExtend) {
+ masm.Sbfm(X(dest), X(dest), 0, 31);
+ }
+ break;
+ }
+ case 8: {
+ {
+ AutoForbidPoolsAndNops afp(
+ &masm,
+ /* max number of instructions in scope = */ 1);
+ if (access) {
+ masm.append(*access, masm.currentOffset());
+ }
+ masm.Ldxr(X(dest), ptr);
+ }
+ break;
+ }
+ default: {
+ MOZ_CRASH();
+ }
+ }
+}
+
+static void StoreExclusive(MacroAssembler& masm, Scalar::Type type,
+ Register status, Register src, MemOperand ptr) {
+ switch (Scalar::byteSize(type)) {
+ case 1:
+ masm.Stxrb(W(status), W(src), ptr);
+ break;
+ case 2:
+ masm.Stxrh(W(status), W(src), ptr);
+ break;
+ case 4:
+ masm.Stxr(W(status), W(src), ptr);
+ break;
+ case 8:
+ masm.Stxr(W(status), X(src), ptr);
+ break;
+ }
+}
+
+template <typename T>
+static void CompareExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, Width targetWidth,
+ const Synchronization& sync, const T& mem,
+ Register oldval, Register newval, Register output) {
+ MOZ_ASSERT(oldval != output && newval != output);
+
+ Label again;
+ Label done;
+
+ vixl::UseScratchRegisterScope temps(&masm);
+
+ Register scratch2 = temps.AcquireX().asUnsized();
+ MemOperand ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ MOZ_ASSERT(ptr.base().asUnsized() != output);
+
+ // NOTE: the generated code must match the assembly code in gen_cmpxchg in
+ // GenerateAtomicOperations.py
+ masm.memoryBarrierBefore(sync);
+
+ Register scratch = temps.AcquireX().asUnsized();
+
+ masm.bind(&again);
+ SignOrZeroExtend(masm, type, targetWidth, oldval, scratch);
+ LoadExclusive(masm, access, type, targetWidth, ptr, output);
+ masm.Cmp(R(output, targetWidth), R(scratch, targetWidth));
+ masm.B(&done, MacroAssembler::NotEqual);
+ StoreExclusive(masm, type, scratch, newval, ptr);
+ masm.Cbnz(W(scratch), &again);
+ masm.bind(&done);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <typename T>
+static void AtomicExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, Width targetWidth,
+ const Synchronization& sync, const T& mem,
+ Register value, Register output) {
+ MOZ_ASSERT(value != output);
+
+ Label again;
+
+ vixl::UseScratchRegisterScope temps(&masm);
+
+ Register scratch2 = temps.AcquireX().asUnsized();
+ MemOperand ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ // NOTE: the generated code must match the assembly code in gen_exchange in
+ // GenerateAtomicOperations.py
+ masm.memoryBarrierBefore(sync);
+
+ Register scratch = temps.AcquireX().asUnsized();
+
+ masm.bind(&again);
+ LoadExclusive(masm, access, type, targetWidth, ptr, output);
+ StoreExclusive(masm, type, scratch, value, ptr);
+ masm.Cbnz(W(scratch), &again);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <bool wantResult, typename T>
+static void AtomicFetchOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, Width targetWidth,
+ const Synchronization& sync, AtomicOp op,
+ const T& mem, Register value, Register temp,
+ Register output) {
+ MOZ_ASSERT(value != output);
+ MOZ_ASSERT(value != temp);
+ MOZ_ASSERT_IF(wantResult, output != temp);
+
+ Label again;
+
+ vixl::UseScratchRegisterScope temps(&masm);
+
+ Register scratch2 = temps.AcquireX().asUnsized();
+ MemOperand ptr = ComputePointerForAtomic(masm, mem, scratch2);
+
+ // NOTE: the generated code must match the assembly code in gen_fetchop in
+ // GenerateAtomicOperations.py
+ masm.memoryBarrierBefore(sync);
+
+ Register scratch = temps.AcquireX().asUnsized();
+
+ masm.bind(&again);
+ LoadExclusive(masm, access, type, targetWidth, ptr, output);
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.Add(X(temp), X(output), X(value));
+ break;
+ case AtomicFetchSubOp:
+ masm.Sub(X(temp), X(output), X(value));
+ break;
+ case AtomicFetchAndOp:
+ masm.And(X(temp), X(output), X(value));
+ break;
+ case AtomicFetchOrOp:
+ masm.Orr(X(temp), X(output), X(value));
+ break;
+ case AtomicFetchXorOp:
+ masm.Eor(X(temp), X(output), X(value));
+ break;
+ }
+ StoreExclusive(masm, type, scratch, temp, ptr);
+ masm.Cbnz(W(scratch), &again);
+ if (wantResult) {
+ SignOrZeroExtend(masm, type, targetWidth, output, output);
+ }
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const Address& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, nullptr, type, Width::_32, sync, mem, oldval, newval,
+ output);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, nullptr, type, Width::_32, sync, mem, oldval, newval,
+ output);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization& sync,
+ const Address& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange(*this, nullptr, Scalar::Int64, Width::_64, sync, mem,
+ expect.reg, replace.reg, output.reg);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange(*this, nullptr, Scalar::Int64, Width::_64, sync, mem,
+ expect.reg, replace.reg, output.reg);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization& sync,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange(*this, nullptr, Scalar::Int64, Width::_64, sync, mem,
+ value.reg, output.reg);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange(*this, nullptr, Scalar::Int64, Width::_64, sync, mem,
+ value.reg, output.reg);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp<true>(*this, nullptr, Scalar::Int64, Width::_64, sync, op, mem,
+ value.reg, temp.reg, output.reg);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp<true>(*this, nullptr, Scalar::Int64, Width::_64, sync, op, mem,
+ value.reg, temp.reg, output.reg);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp) {
+ AtomicFetchOp<false>(*this, nullptr, Scalar::Int64, Width::_64, sync, op, mem,
+ value.reg, temp.reg, temp.reg);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp) {
+ AtomicFetchOp<false>(*this, nullptr, Scalar::Int64, Width::_64, sync, op, mem,
+ value.reg, temp.reg, temp.reg);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, &access, access.type(), Width::_32, access.sync(), mem,
+ oldval, newval, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, &access, access.type(), Width::_32, access.sync(), mem,
+ oldval, newval, output);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const Address& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, nullptr, type, Width::_32, sync, mem, value, output);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, nullptr, type, Width::_32, sync, mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, &access, access.type(), Width::_32, access.sync(), mem,
+ value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, &access, access.type(), Width::_32, access.sync(), mem,
+ value, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type type,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp, Register output) {
+ AtomicFetchOp<true>(*this, nullptr, type, Width::_32, sync, op, mem, value,
+ temp, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type type,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp, Register output) {
+ AtomicFetchOp<true>(*this, nullptr, type, Width::_32, sync, op, mem, value,
+ temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register temp,
+ Register output) {
+ AtomicFetchOp<true>(*this, &access, access.type(), Width::_32, access.sync(),
+ op, mem, value, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem, Register temp,
+ Register output) {
+ AtomicFetchOp<true>(*this, &access, access.type(), Width::_32, access.sync(),
+ op, mem, value, temp, output);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register temp) {
+ AtomicFetchOp<false>(*this, &access, access.type(), Width::_32, access.sync(),
+ op, mem, value, temp, temp);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem, Register temp) {
+ AtomicFetchOp<false>(*this, &access, access.type(), Width::_32, access.sync(),
+ op, mem, value, temp, temp);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange(*this, &access, Scalar::Int64, Width::_64, access.sync(), mem,
+ expect.reg, replace.reg, output.reg);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange(*this, &access, Scalar::Int64, Width::_64, access.sync(), mem,
+ expect.reg, replace.reg, output.reg);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange(*this, &access, Scalar::Int64, Width::_64, access.sync(), mem,
+ value.reg, output.reg);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 value, Register64 output) {
+ AtomicExchange(*this, &access, Scalar::Int64, Width::_64, access.sync(), mem,
+ value.reg, output.reg);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp<true>(*this, &access, Scalar::Int64, Width::_64, access.sync(),
+ op, mem, value.reg, temp.reg, output.reg);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp<true>(*this, &access, Scalar::Int64, Width::_64, access.sync(),
+ op, mem, value.reg, temp.reg, output.reg);
+}
+
+void MacroAssembler::wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem,
+ Register64 temp) {
+ AtomicFetchOp<false>(*this, &access, Scalar::Int64, Width::_64, access.sync(),
+ op, mem, value.reg, temp.reg, temp.reg);
+}
+
+// ========================================================================
+// JS atomic operations.
+
+template <typename T>
+static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register oldval, Register newval, Register temp,
+ AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, output.gpr());
+ }
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register oldval,
+ Register newval, Register temp,
+ AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register temp,
+ AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
+}
+
+template <typename T>
+static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register value, Register temp,
+ AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicExchange(arrayType, sync, mem, value, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicExchange(arrayType, sync, mem, value, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register value,
+ Register temp, AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register temp, AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
+}
+
+template <typename T>
+static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const T& mem, Register temp1,
+ Register temp2, AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, temp2, temp1);
+ masm.convertUInt32ToDouble(temp1, output.fpu());
+ } else {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, temp1, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp1, Register temp2,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp1, Register temp2,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp) {
+ AtomicFetchOp<false>(*this, nullptr, arrayType, Width::_32, sync, op, mem,
+ value, temp, temp);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp) {
+ AtomicFetchOp<false>(*this, nullptr, arrayType, Width::_32, sync, op, mem,
+ value, temp, temp);
+}
+
+void MacroAssembler::flexibleQuotient32(Register rhs, Register srcDest,
+ bool isUnsigned,
+ const LiveRegisterSet&) {
+ quotient32(rhs, srcDest, isUnsigned);
+}
+
+void MacroAssembler::flexibleRemainder32(Register rhs, Register srcDest,
+ bool isUnsigned,
+ const LiveRegisterSet&) {
+ remainder32(rhs, srcDest, isUnsigned);
+}
+
+void MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest,
+ Register remOutput, bool isUnsigned,
+ const LiveRegisterSet&) {
+ vixl::UseScratchRegisterScope temps(this);
+ ARMRegister scratch = temps.AcquireW();
+ ARMRegister src = temps.AcquireW();
+
+ // Preserve src for remainder computation
+ Mov(src, ARMRegister(srcDest, 32));
+
+ if (isUnsigned) {
+ Udiv(ARMRegister(srcDest, 32), src, ARMRegister(rhs, 32));
+ } else {
+ Sdiv(ARMRegister(srcDest, 32), src, ARMRegister(rhs, 32));
+ }
+ // Compute remainder
+ Mul(scratch, ARMRegister(srcDest, 32), ARMRegister(rhs, 32));
+ Sub(ARMRegister(remOutput, 32), src, scratch);
+}
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ AutoForbidPoolsAndNops afp(this,
+ /* max number of instructions in scope = */ 1);
+ CodeOffset offset(currentOffset());
+ adr(ARMRegister(dest, 64), 0, LabelDoc());
+ return offset;
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target) {
+ ptrdiff_t off = target - loc;
+ MOZ_RELEASE_ASSERT(vixl::IsInt21(off));
+
+ Instruction* cur = reinterpret_cast<Instruction*>(loc.raw());
+ MOZ_ASSERT(cur->IsADR());
+
+ vixl::Register rd = vixl::Register::XRegFromCode(cur->Rd());
+ adr(cur, rd, off);
+}
+
+// ========================================================================
+// Spectre Mitigations.
+
+void MacroAssembler::speculationBarrier() {
+ // Conditional speculation barrier.
+ csdb();
+}
+
+void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ floorf(src, dest, fail);
+}
+
+void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ floor(src, dest, fail);
+}
+
+void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ceilf(src, dest, fail);
+}
+
+void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ceil(src, dest, fail);
+}
+
+void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ const ARMFPRegister src32(src, 32);
+
+ Label done, zeroCase;
+
+ // Convert scalar to signed 32-bit fixed-point, rounding toward zero.
+ // In the case of overflow, the output is saturated.
+ // In the case of NaN and -0, the output is zero.
+ Fcvtzs(ARMRegister(dest, 32), src32);
+
+ // If the output was zero, worry about special cases.
+ branch32(Assembler::Equal, dest, Imm32(0), &zeroCase);
+
+ // Fail on overflow cases.
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+
+ // If the output was non-zero and wasn't saturated, just return it.
+ jump(&done);
+
+ // Handle the case of a zero output:
+ // 1. The input may have been NaN, requiring a failure.
+ // 2. The input may have been in (-1,-0], requiring a failure.
+ {
+ bind(&zeroCase);
+
+ // If input is a negative number that truncated to zero, the real
+ // output should be the non-integer -0.
+ // The use of "lt" instead of "lo" also catches unordered NaN input.
+ Fcmp(src32, 0.0f);
+ B(fail, vixl::lt);
+
+ // Check explicitly for -0, bitwise.
+ Fmov(ARMRegister(dest, 32), src32);
+ branchTest32(Assembler::Signed, dest, dest, fail);
+ move32(Imm32(0), dest);
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ const ARMFPRegister src64(src, 64);
+
+ Label done, zeroCase;
+
+ // Convert scalar to signed 32-bit fixed-point, rounding toward zero.
+ // In the case of overflow, the output is saturated.
+ // In the case of NaN and -0, the output is zero.
+ Fcvtzs(ARMRegister(dest, 32), src64);
+
+ // If the output was zero, worry about special cases.
+ branch32(Assembler::Equal, dest, Imm32(0), &zeroCase);
+
+ // Fail on overflow cases.
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+
+ // If the output was non-zero and wasn't saturated, just return it.
+ jump(&done);
+
+ // Handle the case of a zero output:
+ // 1. The input may have been NaN, requiring a failure.
+ // 2. The input may have been in (-1,-0], requiring a failure.
+ {
+ bind(&zeroCase);
+
+ // If input is a negative number that truncated to zero, the real
+ // output should be the non-integer -0.
+ // The use of "lt" instead of "lo" also catches unordered NaN input.
+ Fcmp(src64, 0.0);
+ B(fail, vixl::lt);
+
+ // Check explicitly for -0, bitwise.
+ Fmov(ARMRegister(dest, 64), src64);
+ branchTestPtr(Assembler::Signed, dest, dest, fail);
+ movePtr(ImmPtr(0), dest);
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ const ARMFPRegister src32(src, 32);
+ ScratchFloat32Scope scratch(*this);
+
+ Label negative, done;
+
+ // Branch to a slow path if input < 0.0 due to complicated rounding rules.
+ // Note that Fcmp with NaN unsets the negative flag.
+ Fcmp(src32, 0.0);
+ B(&negative, Assembler::Condition::lo);
+
+ // Handle the simple case of a positive input, and also -0 and NaN.
+ // Rounding proceeds with consideration of the fractional part of the input:
+ // 1. If > 0.5, round to integer with higher absolute value (so, up).
+ // 2. If < 0.5, round to integer with lower absolute value (so, down).
+ // 3. If = 0.5, round to +Infinity (so, up).
+ {
+ // Convert to signed 32-bit integer, rounding halfway cases away from zero.
+ // In the case of overflow, the output is saturated.
+ // In the case of NaN and -0, the output is zero.
+ Fcvtas(ARMRegister(dest, 32), src32);
+ // If the output potentially saturated, fail.
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ // If the result of the rounding was non-zero, return the output.
+ // In the case of zero, the input may have been NaN or -0, which must bail.
+ branch32(Assembler::NotEqual, dest, Imm32(0), &done);
+ {
+ // If input is NaN, comparisons set the C and V bits of the NZCV flags.
+ Fcmp(src32, 0.0f);
+ B(fail, Assembler::Overflow);
+
+ // Move all 32 bits of the input into a scratch register to check for -0.
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratchGPR32 = temps.AcquireW();
+ Fmov(scratchGPR32, src32);
+ Cmp(scratchGPR32, vixl::Operand(uint32_t(0x80000000)));
+ B(fail, Assembler::Equal);
+ }
+
+ jump(&done);
+ }
+
+ // Handle the complicated case of a negative input.
+ // Rounding proceeds with consideration of the fractional part of the input:
+ // 1. If > 0.5, round to integer with higher absolute value (so, down).
+ // 2. If < 0.5, round to integer with lower absolute value (so, up).
+ // 3. If = 0.5, round to +Infinity (so, up).
+ bind(&negative);
+ {
+ // Inputs in [-0.5, 0) need 0.5 added; other negative inputs need
+ // the biggest double less than 0.5.
+ Label join;
+ loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp);
+ loadConstantFloat32(-0.5f, scratch);
+ branchFloat(Assembler::DoubleLessThan, src, scratch, &join);
+ loadConstantFloat32(0.5f, temp);
+ bind(&join);
+
+ addFloat32(src, temp);
+ // Round all values toward -Infinity.
+ // In the case of overflow, the output is saturated.
+ // NaN and -0 are already handled by the "positive number" path above.
+ Fcvtms(ARMRegister(dest, 32), temp);
+ // If the output potentially saturated, fail.
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+
+ // If output is zero, then the actual result is -0. Fail.
+ branchTest32(Assembler::Zero, dest, dest, fail);
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ const ARMFPRegister src64(src, 64);
+ ScratchDoubleScope scratch(*this);
+
+ Label negative, done;
+
+ // Branch to a slow path if input < 0.0 due to complicated rounding rules.
+ // Note that Fcmp with NaN unsets the negative flag.
+ Fcmp(src64, 0.0);
+ B(&negative, Assembler::Condition::lo);
+
+ // Handle the simple case of a positive input, and also -0 and NaN.
+ // Rounding proceeds with consideration of the fractional part of the input:
+ // 1. If > 0.5, round to integer with higher absolute value (so, up).
+ // 2. If < 0.5, round to integer with lower absolute value (so, down).
+ // 3. If = 0.5, round to +Infinity (so, up).
+ {
+ // Convert to signed 32-bit integer, rounding halfway cases away from zero.
+ // In the case of overflow, the output is saturated.
+ // In the case of NaN and -0, the output is zero.
+ Fcvtas(ARMRegister(dest, 32), src64);
+ // If the output potentially saturated, fail.
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ // If the result of the rounding was non-zero, return the output.
+ // In the case of zero, the input may have been NaN or -0, which must bail.
+ branch32(Assembler::NotEqual, dest, Imm32(0), &done);
+ {
+ // If input is NaN, comparisons set the C and V bits of the NZCV flags.
+ Fcmp(src64, 0.0);
+ B(fail, Assembler::Overflow);
+
+ // Move all 64 bits of the input into a scratch register to check for -0.
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratchGPR64 = temps.AcquireX();
+ Fmov(scratchGPR64, src64);
+ Cmp(scratchGPR64, vixl::Operand(uint64_t(0x8000000000000000)));
+ B(fail, Assembler::Equal);
+ }
+
+ jump(&done);
+ }
+
+ // Handle the complicated case of a negative input.
+ // Rounding proceeds with consideration of the fractional part of the input:
+ // 1. If > 0.5, round to integer with higher absolute value (so, down).
+ // 2. If < 0.5, round to integer with lower absolute value (so, up).
+ // 3. If = 0.5, round to +Infinity (so, up).
+ bind(&negative);
+ {
+ // Inputs in [-0.5, 0) need 0.5 added; other negative inputs need
+ // the biggest double less than 0.5.
+ Label join;
+ loadConstantDouble(GetBiggestNumberLessThan(0.5), temp);
+ loadConstantDouble(-0.5, scratch);
+ branchDouble(Assembler::DoubleLessThan, src, scratch, &join);
+ loadConstantDouble(0.5, temp);
+ bind(&join);
+
+ addDouble(src, temp);
+ // Round all values toward -Infinity.
+ // In the case of overflow, the output is saturated.
+ // NaN and -0 are already handled by the "positive number" path above.
+ Fcvtms(ARMRegister(dest, 32), temp);
+ // If the output potentially saturated, fail.
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+
+ // If output is zero, then the actual result is -0. Fail.
+ branchTest32(Assembler::Zero, dest, dest, fail);
+ }
+
+ bind(&done);
+}
+
+void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ switch (mode) {
+ case RoundingMode::Up:
+ frintp(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
+ return;
+ case RoundingMode::Down:
+ frintm(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
+ return;
+ case RoundingMode::NearestTiesToEven:
+ frintn(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
+ return;
+ case RoundingMode::TowardsZero:
+ frintz(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
+ return;
+ }
+ MOZ_CRASH("unexpected mode");
+}
+
+void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ switch (mode) {
+ case RoundingMode::Up:
+ frintp(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
+ return;
+ case RoundingMode::Down:
+ frintm(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
+ return;
+ case RoundingMode::NearestTiesToEven:
+ frintn(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
+ return;
+ case RoundingMode::TowardsZero:
+ frintz(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
+ return;
+ }
+ MOZ_CRASH("unexpected mode");
+}
+
+void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) {
+ ScratchDoubleScope scratch(*this);
+
+ // Double with only the sign bit set
+ loadConstantDouble(-0.0, scratch);
+
+ if (lhs != output) {
+ moveDouble(lhs, output);
+ }
+
+ bit(ARMFPRegister(output.encoding(), vixl::VectorFormat::kFormat8B),
+ ARMFPRegister(rhs.encoding(), vixl::VectorFormat::kFormat8B),
+ ARMFPRegister(scratch.encoding(), vixl::VectorFormat::kFormat8B));
+}
+
+void MacroAssembler::copySignFloat32(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) {
+ ScratchFloat32Scope scratch(*this);
+
+ // Float with only the sign bit set
+ loadConstantFloat32(-0.0f, scratch);
+
+ if (lhs != output) {
+ moveFloat32(lhs, output);
+ }
+
+ bit(ARMFPRegister(output.encoding(), vixl::VectorFormat::kFormat8B),
+ ARMFPRegister(rhs.encoding(), vixl::VectorFormat::kFormat8B),
+ ARMFPRegister(scratch.encoding(), vixl::VectorFormat::kFormat8B));
+}
+
+void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
+ Register pointer) {
+ Add(ARMRegister(pointer, 64), ARMRegister(pointer, 64),
+ Operand(ARMRegister(indexTemp32, 64), vixl::LSL, shift));
+}
+
+//}}} check_macroassembler_style
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/arm64/MacroAssembler-arm64.h b/js/src/jit/arm64/MacroAssembler-arm64.h
new file mode 100644
index 0000000000..5b918fa0e2
--- /dev/null
+++ b/js/src/jit/arm64/MacroAssembler-arm64.h
@@ -0,0 +1,2335 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_MacroAssembler_arm64_h
+#define jit_arm64_MacroAssembler_arm64_h
+
+#include "jit/arm64/Assembler-arm64.h"
+#include "jit/arm64/vixl/Debugger-vixl.h"
+#include "jit/arm64/vixl/MacroAssembler-vixl.h"
+#include "jit/AtomicOp.h"
+#include "jit/MoveResolver.h"
+#include "vm/BigIntType.h" // JS::BigInt
+#include "wasm/WasmBuiltins.h"
+
+#ifdef _M_ARM64
+# ifdef move32
+# undef move32
+# endif
+# ifdef move64
+# undef move64
+# endif
+#endif
+
+namespace js {
+namespace jit {
+
+// Import VIXL operands directly into the jit namespace for shared code.
+using vixl::MemOperand;
+using vixl::Operand;
+
+struct ImmShiftedTag : public ImmWord {
+ explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {}
+
+ explicit ImmShiftedTag(JSValueType type)
+ : ImmWord(uintptr_t(JSValueShiftedTag(JSVAL_TYPE_TO_SHIFTED_TAG(type)))) {
+ }
+};
+
+struct ImmTag : public Imm32 {
+ explicit ImmTag(JSValueTag tag) : Imm32(tag) {}
+};
+
+class ScratchTagScope;
+
+class MacroAssemblerCompat : public vixl::MacroAssembler {
+ public:
+ typedef vixl::Condition Condition;
+
+ private:
+ // Perform a downcast. Should be removed by Bug 996602.
+ js::jit::MacroAssembler& asMasm();
+ const js::jit::MacroAssembler& asMasm() const;
+
+ public:
+ // Restrict to only VIXL-internal functions.
+ vixl::MacroAssembler& asVIXL();
+ const MacroAssembler& asVIXL() const;
+
+ protected:
+ bool enoughMemory_;
+ uint32_t framePushed_;
+
+ MacroAssemblerCompat()
+ : vixl::MacroAssembler(), enoughMemory_(true), framePushed_(0) {}
+
+ protected:
+ MoveResolver moveResolver_;
+
+ public:
+ bool oom() const { return Assembler::oom() || !enoughMemory_; }
+ static ARMRegister toARMRegister(RegisterOrSP r, size_t size) {
+ if (IsHiddenSP(r)) {
+ MOZ_ASSERT(size == 64);
+ return sp;
+ }
+ return ARMRegister(AsRegister(r), size);
+ }
+ static MemOperand toMemOperand(const Address& a) {
+ return MemOperand(toARMRegister(a.base, 64), a.offset);
+ }
+ void doBaseIndex(const vixl::CPURegister& rt, const BaseIndex& addr,
+ vixl::LoadStoreOp op) {
+ const ARMRegister base = toARMRegister(addr.base, 64);
+ const ARMRegister index = ARMRegister(addr.index, 64);
+ const unsigned scale = addr.scale;
+
+ if (!addr.offset &&
+ (!scale || scale == static_cast<unsigned>(CalcLSDataSize(op)))) {
+ LoadStoreMacro(rt, MemOperand(base, index, vixl::LSL, scale), op);
+ return;
+ }
+
+ vixl::UseScratchRegisterScope temps(this);
+ ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(!scratch64.Is(rt));
+ MOZ_ASSERT(!scratch64.Is(base));
+ MOZ_ASSERT(!scratch64.Is(index));
+
+ Add(scratch64, base, Operand(index, vixl::LSL, scale));
+ LoadStoreMacro(rt, MemOperand(scratch64, addr.offset), op);
+ }
+ void Push(ARMRegister reg) {
+ push(reg);
+ adjustFrame(reg.size() / 8);
+ }
+ void Push(Register reg) {
+ vixl::MacroAssembler::Push(ARMRegister(reg, 64));
+ adjustFrame(8);
+ }
+ void Push(Imm32 imm) {
+ push(imm);
+ adjustFrame(8);
+ }
+ void Push(FloatRegister f) {
+ push(ARMFPRegister(f, 64));
+ adjustFrame(8);
+ }
+ void Push(ImmPtr imm) {
+ push(imm);
+ adjustFrame(sizeof(void*));
+ }
+ void push(FloatRegister f) {
+ vixl::MacroAssembler::Push(ARMFPRegister(f, 64));
+ }
+ void push(ARMFPRegister f) { vixl::MacroAssembler::Push(f); }
+ void push(Imm32 imm) {
+ if (imm.value == 0) {
+ vixl::MacroAssembler::Push(vixl::xzr);
+ } else {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ move32(imm, scratch64.asUnsized());
+ vixl::MacroAssembler::Push(scratch64);
+ }
+ }
+ void push(ImmWord imm) {
+ if (imm.value == 0) {
+ vixl::MacroAssembler::Push(vixl::xzr);
+ } else {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ Mov(scratch64, imm.value);
+ vixl::MacroAssembler::Push(scratch64);
+ }
+ }
+ void push(ImmPtr imm) {
+ if (imm.value == nullptr) {
+ vixl::MacroAssembler::Push(vixl::xzr);
+ } else {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ movePtr(imm, scratch64.asUnsized());
+ vixl::MacroAssembler::Push(scratch64);
+ }
+ }
+ void push(ImmGCPtr imm) {
+ if (imm.value == nullptr) {
+ vixl::MacroAssembler::Push(vixl::xzr);
+ } else {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ movePtr(imm, scratch64.asUnsized());
+ vixl::MacroAssembler::Push(scratch64);
+ }
+ }
+ void push(ARMRegister reg) { vixl::MacroAssembler::Push(reg); }
+ void push(Address a) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(a.base != scratch64.asUnsized());
+ loadPtr(a, scratch64.asUnsized());
+ vixl::MacroAssembler::Push(scratch64);
+ }
+
+ // Push registers.
+ void push(Register reg) { vixl::MacroAssembler::Push(ARMRegister(reg, 64)); }
+ void push(RegisterOrSP reg) {
+ if (IsHiddenSP(reg)) {
+ vixl::MacroAssembler::Push(sp);
+ }
+ vixl::MacroAssembler::Push(toARMRegister(reg, 64));
+ }
+ void push(Register r0, Register r1) {
+ vixl::MacroAssembler::Push(ARMRegister(r0, 64), ARMRegister(r1, 64));
+ }
+ void push(Register r0, Register r1, Register r2) {
+ vixl::MacroAssembler::Push(ARMRegister(r0, 64), ARMRegister(r1, 64),
+ ARMRegister(r2, 64));
+ }
+ void push(Register r0, Register r1, Register r2, Register r3) {
+ vixl::MacroAssembler::Push(ARMRegister(r0, 64), ARMRegister(r1, 64),
+ ARMRegister(r2, 64), ARMRegister(r3, 64));
+ }
+ void push(ARMFPRegister r0, ARMFPRegister r1, ARMFPRegister r2,
+ ARMFPRegister r3) {
+ vixl::MacroAssembler::Push(r0, r1, r2, r3);
+ }
+
+ // Pop registers.
+ void pop(Register reg) { vixl::MacroAssembler::Pop(ARMRegister(reg, 64)); }
+ void pop(Register r0, Register r1) {
+ vixl::MacroAssembler::Pop(ARMRegister(r0, 64), ARMRegister(r1, 64));
+ }
+ void pop(Register r0, Register r1, Register r2) {
+ vixl::MacroAssembler::Pop(ARMRegister(r0, 64), ARMRegister(r1, 64),
+ ARMRegister(r2, 64));
+ }
+ void pop(Register r0, Register r1, Register r2, Register r3) {
+ vixl::MacroAssembler::Pop(ARMRegister(r0, 64), ARMRegister(r1, 64),
+ ARMRegister(r2, 64), ARMRegister(r3, 64));
+ }
+ void pop(ARMFPRegister r0, ARMFPRegister r1, ARMFPRegister r2,
+ ARMFPRegister r3) {
+ vixl::MacroAssembler::Pop(r0, r1, r2, r3);
+ }
+
+ void pop(const ValueOperand& v) { pop(v.valueReg()); }
+ void pop(const FloatRegister& f) {
+ vixl::MacroAssembler::Pop(ARMFPRegister(f, 64));
+ }
+
+ void implicitPop(uint32_t args) {
+ MOZ_ASSERT(args % sizeof(intptr_t) == 0);
+ adjustFrame(0 - args);
+ }
+ void Pop(ARMRegister r) {
+ vixl::MacroAssembler::Pop(r);
+ adjustFrame(0 - r.size() / 8);
+ }
+ // FIXME: This is the same on every arch.
+ // FIXME: If we can share framePushed_, we can share this.
+ // FIXME: Or just make it at the highest level.
+ CodeOffset PushWithPatch(ImmWord word) {
+ framePushed_ += sizeof(word.value);
+ return pushWithPatch(word);
+ }
+ CodeOffset PushWithPatch(ImmPtr ptr) {
+ return PushWithPatch(ImmWord(uintptr_t(ptr.value)));
+ }
+
+ uint32_t framePushed() const { return framePushed_; }
+ void adjustFrame(int32_t diff) { setFramePushed(framePushed_ + diff); }
+
+ void setFramePushed(uint32_t framePushed) { framePushed_ = framePushed; }
+
+ void freeStack(Register amount) {
+ vixl::MacroAssembler::Drop(Operand(ARMRegister(amount, 64)));
+ }
+
+ // Update sp with the value of the current active stack pointer, if necessary.
+ void syncStackPtr() {
+ if (!GetStackPointer64().Is(vixl::sp)) {
+ Mov(vixl::sp, GetStackPointer64());
+ }
+ }
+ void initPseudoStackPtr() {
+ if (!GetStackPointer64().Is(vixl::sp)) {
+ Mov(GetStackPointer64(), vixl::sp);
+ }
+ }
+ // In debug builds only, cause a trap if PSP is active and PSP != SP
+ void assertStackPtrsSynced(uint32_t id) {
+#ifdef DEBUG
+ // The add and sub instructions below will only take a 12-bit immediate.
+ MOZ_ASSERT(id <= 0xFFF);
+ if (!GetStackPointer64().Is(vixl::sp)) {
+ Label ok;
+ // Add a marker, so we can figure out who requested the check when
+ // inspecting the generated code. Note, a more concise way to encode
+ // the marker would be to use it as an immediate for the `brk`
+ // instruction as generated by `Unreachable()`, and removing the add/sub.
+ Add(GetStackPointer64(), GetStackPointer64(), Operand(id));
+ Sub(GetStackPointer64(), GetStackPointer64(), Operand(id));
+ Cmp(vixl::sp, GetStackPointer64());
+ B(Equal, &ok);
+ Unreachable();
+ bind(&ok);
+ }
+#endif
+ }
+ // In debug builds only, add a marker that doesn't change the machine's
+ // state. Note these markers are x16-based, as opposed to the x28-based
+ // ones made by `assertStackPtrsSynced`.
+ void addMarker(uint32_t id) {
+#ifdef DEBUG
+ // Only 12 bits of immediate are allowed.
+ MOZ_ASSERT(id <= 0xFFF);
+ ARMRegister x16 = ARMRegister(r16, 64);
+ Add(x16, x16, Operand(id));
+ Sub(x16, x16, Operand(id));
+#endif
+ }
+
+ void storeValue(ValueOperand val, const Address& dest) {
+ storePtr(val.valueReg(), dest);
+ }
+
+ template <typename T>
+ void storeValue(JSValueType type, Register reg, const T& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != reg);
+ tagValue(type, reg, ValueOperand(scratch));
+ storeValue(ValueOperand(scratch), dest);
+ }
+ template <typename T>
+ void storeValue(const Value& val, const T& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ moveValue(val, ValueOperand(scratch));
+ storeValue(ValueOperand(scratch), dest);
+ }
+ void storeValue(ValueOperand val, BaseIndex dest) {
+ storePtr(val.valueReg(), dest);
+ }
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ loadPtr(src, temp);
+ storePtr(temp, dest);
+ }
+
+ void storePrivateValue(Register src, const Address& dest) {
+ storePtr(src, dest);
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ storePtr(imm, dest);
+ }
+
+ void loadValue(Address src, Register val) {
+ Ldr(ARMRegister(val, 64), MemOperand(src));
+ }
+ void loadValue(Address src, ValueOperand val) {
+ Ldr(ARMRegister(val.valueReg(), 64), MemOperand(src));
+ }
+ void loadValue(const BaseIndex& src, ValueOperand val) {
+ doBaseIndex(ARMRegister(val.valueReg(), 64), src, vixl::LDR_x);
+ }
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+ void tagValue(JSValueType type, Register payload, ValueOperand dest) {
+ // This could be cleverer, but the first attempt had bugs.
+ Orr(ARMRegister(dest.valueReg(), 64), ARMRegister(payload, 64),
+ Operand(ImmShiftedTag(type).value));
+ }
+ void pushValue(ValueOperand val) {
+ vixl::MacroAssembler::Push(ARMRegister(val.valueReg(), 64));
+ }
+ void popValue(ValueOperand val) {
+ vixl::MacroAssembler::Pop(ARMRegister(val.valueReg(), 64));
+ // SP may be < PSP now (that's OK).
+ // eg testcase: tests/backup-point-bug1315634.js
+ }
+ void pushValue(const Value& val) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ if (val.isGCThing()) {
+ BufferOffset load =
+ movePatchablePtr(ImmPtr(val.bitsAsPunboxPointer()), scratch);
+ writeDataRelocation(val, load);
+ push(scratch);
+ } else {
+ moveValue(val, scratch);
+ push(scratch);
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != reg);
+ tagValue(type, reg, ValueOperand(scratch));
+ push(scratch);
+ }
+ void pushValue(const Address& addr) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != addr.base);
+ loadValue(addr, scratch);
+ push(scratch);
+ }
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes,
+ JSValueType type) {
+ switch (nbytes) {
+ case 8: {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ if (type == JSVAL_TYPE_OBJECT) {
+ unboxObjectOrNull(value, scratch);
+ } else {
+ unboxNonDouble(value, scratch, type);
+ }
+ storePtr(scratch, address);
+ return;
+ }
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+ void moveValue(const Value& val, Register dest) {
+ if (val.isGCThing()) {
+ BufferOffset load =
+ movePatchablePtr(ImmPtr(val.bitsAsPunboxPointer()), dest);
+ writeDataRelocation(val, load);
+ } else {
+ movePtr(ImmWord(val.asRawBits()), dest);
+ }
+ }
+ void moveValue(const Value& src, const ValueOperand& dest) {
+ moveValue(src, dest.valueReg());
+ }
+
+ CodeOffset pushWithPatch(ImmWord imm) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ CodeOffset label = movWithPatch(imm, scratch);
+ push(scratch);
+ return label;
+ }
+
+ CodeOffset movWithPatch(ImmWord imm, Register dest) {
+ BufferOffset off = immPool64(ARMRegister(dest, 64), imm.value);
+ return CodeOffset(off.getOffset());
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ BufferOffset off = immPool64(ARMRegister(dest, 64), uint64_t(imm.value));
+ return CodeOffset(off.getOffset());
+ }
+
+ void boxValue(JSValueType type, Register src, Register dest);
+
+ void splitSignExtTag(Register src, Register dest) {
+ sbfx(ARMRegister(dest, 64), ARMRegister(src, 64), JSVAL_TAG_SHIFT,
+ (64 - JSVAL_TAG_SHIFT));
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch) {
+ loadPtr(address, scratch);
+ splitSignExtTag(scratch, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ splitSignExtTag(value.valueReg(), scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractObject(const Address& address,
+ Register scratch) {
+ loadPtr(address, scratch);
+ unboxObject(scratch, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ unboxObject(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ unboxSymbol(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ unboxInt32(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ unboxBoolean(value, scratch);
+ return scratch;
+ }
+
+ inline void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ void emitSet(Condition cond, Register dest) {
+ Cset(ARMRegister(dest, 64), cond);
+ }
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testNull(cond, value);
+ emitSet(cond, dest);
+ }
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testObject(cond, value);
+ emitSet(cond, dest);
+ }
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest) {
+ cond = testUndefined(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void convertBoolToInt32(Register source, Register dest) {
+ Uxtb(ARMRegister(dest, 64), ARMRegister(source, 64));
+ }
+
+ void convertInt32ToDouble(Register src, FloatRegister dest) {
+ Scvtf(ARMFPRegister(dest, 64),
+ ARMRegister(src, 32)); // Uses FPCR rounding mode.
+ }
+ void convertInt32ToDouble(const Address& src, FloatRegister dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != src.base);
+ load32(src, scratch);
+ convertInt32ToDouble(scratch, dest);
+ }
+ void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != src.base);
+ MOZ_ASSERT(scratch != src.index);
+ load32(src, scratch);
+ convertInt32ToDouble(scratch, dest);
+ }
+
+ void convertInt32ToFloat32(Register src, FloatRegister dest) {
+ Scvtf(ARMFPRegister(dest, 32),
+ ARMRegister(src, 32)); // Uses FPCR rounding mode.
+ }
+ void convertInt32ToFloat32(const Address& src, FloatRegister dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != src.base);
+ load32(src, scratch);
+ convertInt32ToFloat32(scratch, dest);
+ }
+
+ void convertUInt32ToDouble(Register src, FloatRegister dest) {
+ Ucvtf(ARMFPRegister(dest, 64),
+ ARMRegister(src, 32)); // Uses FPCR rounding mode.
+ }
+ void convertUInt32ToDouble(const Address& src, FloatRegister dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != src.base);
+ load32(src, scratch);
+ convertUInt32ToDouble(scratch, dest);
+ }
+
+ void convertUInt32ToFloat32(Register src, FloatRegister dest) {
+ Ucvtf(ARMFPRegister(dest, 32),
+ ARMRegister(src, 32)); // Uses FPCR rounding mode.
+ }
+ void convertUInt32ToFloat32(const Address& src, FloatRegister dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != src.base);
+ load32(src, scratch);
+ convertUInt32ToFloat32(scratch, dest);
+ }
+
+ void convertFloat32ToDouble(FloatRegister src, FloatRegister dest) {
+ Fcvt(ARMFPRegister(dest, 64), ARMFPRegister(src, 32));
+ }
+ void convertDoubleToFloat32(FloatRegister src, FloatRegister dest) {
+ Fcvt(ARMFPRegister(dest, 32), ARMFPRegister(src, 64));
+ }
+
+ using vixl::MacroAssembler::B;
+
+ void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true) {
+ ARMFPRegister fsrc64(src, 64);
+ ARMRegister dest32(dest, 32);
+
+ // ARMv8.3 chips support the FJCVTZS instruction, which handles exactly this
+ // logic. But the simulator does not implement it, and when the simulator
+ // runs on ARM64 hardware we want to override vixl's detection of it.
+#if defined(JS_SIMULATOR_ARM64) && (defined(__aarch64__) || defined(_M_ARM64))
+ const bool fjscvt = false;
+#else
+ const bool fjscvt =
+ CPUHas(vixl::CPUFeatures::kFP, vixl::CPUFeatures::kJSCVT);
+#endif
+ if (fjscvt) {
+ // Convert double to integer, rounding toward zero.
+ // The Z-flag is set iff the conversion is exact. -0 unsets the Z-flag.
+ Fjcvtzs(dest32, fsrc64);
+
+ if (negativeZeroCheck) {
+ B(fail, Assembler::NonZero);
+ } else {
+ Label done;
+ B(&done, Assembler::Zero); // If conversion was exact, go to end.
+
+ // The conversion was inexact, but the caller intends to allow -0.
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMFPRegister scratch64 = temps.AcquireD();
+ MOZ_ASSERT(!scratch64.Is(fsrc64));
+
+ // Compare fsrc64 to 0.
+ // If fsrc64 == 0 and FJCVTZS conversion was inexact, then fsrc64 is -0.
+ Fmov(scratch64, xzr);
+ Fcmp(scratch64, fsrc64);
+ B(fail, Assembler::NotEqual); // Pass through -0; fail otherwise.
+
+ bind(&done);
+ }
+ } else {
+ // Older processors use a significantly slower path.
+ ARMRegister dest64(dest, 64);
+
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMFPRegister scratch64 = temps.AcquireD();
+ MOZ_ASSERT(!scratch64.Is(fsrc64));
+
+ Fcvtzs(dest32, fsrc64); // Convert, rounding toward zero.
+ Scvtf(scratch64, dest32); // Convert back, using FPCR rounding mode.
+ Fcmp(scratch64, fsrc64);
+ B(fail, Assembler::NotEqual);
+
+ if (negativeZeroCheck) {
+ Label nonzero;
+ Cbnz(dest32, &nonzero);
+ Fmov(dest64, fsrc64);
+ Cbnz(dest64, fail);
+ bind(&nonzero);
+ }
+ }
+ }
+ void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMFPRegister scratch32 = temps.AcquireS();
+
+ ARMFPRegister fsrc(src, 32);
+ ARMRegister dest32(dest, 32);
+ ARMRegister dest64(dest, 64);
+
+ MOZ_ASSERT(!scratch32.Is(fsrc));
+
+ Fcvtzs(dest64, fsrc); // Convert, rounding toward zero.
+ Scvtf(scratch32, dest32); // Convert back, using FPCR rounding mode.
+ Fcmp(scratch32, fsrc);
+ B(fail, Assembler::NotEqual);
+
+ if (negativeZeroCheck) {
+ Label nonzero;
+ Cbnz(dest32, &nonzero);
+ Fmov(dest32, fsrc);
+ Cbnz(dest32, fail);
+ bind(&nonzero);
+ }
+ And(dest64, dest64, Operand(0xffffffff));
+ }
+
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true) {
+ ARMFPRegister fsrc64(src, 64);
+ ARMRegister dest64(dest, 64);
+
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMFPRegister scratch64 = temps.AcquireD();
+ MOZ_ASSERT(!scratch64.Is(fsrc64));
+
+ // Note: we can't use the FJCVTZS instruction here because that only works
+ // for 32-bit values.
+
+ Fcvtzs(dest64, fsrc64); // Convert, rounding toward zero.
+ Scvtf(scratch64, dest64); // Convert back, using FPCR rounding mode.
+ Fcmp(scratch64, fsrc64);
+ B(fail, Assembler::NotEqual);
+
+ if (negativeZeroCheck) {
+ Label nonzero;
+ Cbnz(dest64, &nonzero);
+ Fmov(dest64, fsrc64);
+ Cbnz(dest64, fail);
+ bind(&nonzero);
+ }
+ }
+
+ void floor(FloatRegister input, Register output, Label* bail) {
+ Label handleZero;
+ // Label handleNeg;
+ Label fin;
+ ARMFPRegister iDbl(input, 64);
+ ARMRegister o64(output, 64);
+ ARMRegister o32(output, 32);
+ Fcmp(iDbl, 0.0);
+ B(Assembler::Equal, &handleZero);
+ // B(Assembler::Signed, &handleNeg);
+ // NaN is always a bail condition, just bail directly.
+ B(Assembler::Overflow, bail);
+ Fcvtms(o64, iDbl);
+ Cmp(o64, Operand(o64, vixl::SXTW));
+ B(NotEqual, bail);
+ Mov(o32, o32);
+ B(&fin);
+
+ bind(&handleZero);
+ // Move the top word of the double into the output reg, if it is non-zero,
+ // then the original value was -0.0.
+ Fmov(o64, iDbl);
+ Cbnz(o64, bail);
+ bind(&fin);
+ }
+
+ void floorf(FloatRegister input, Register output, Label* bail) {
+ Label handleZero;
+ // Label handleNeg;
+ Label fin;
+ ARMFPRegister iFlt(input, 32);
+ ARMRegister o64(output, 64);
+ ARMRegister o32(output, 32);
+ Fcmp(iFlt, 0.0);
+ B(Assembler::Equal, &handleZero);
+ // B(Assembler::Signed, &handleNeg);
+ // NaN is always a bail condition, just bail directly.
+ B(Assembler::Overflow, bail);
+ Fcvtms(o64, iFlt);
+ Cmp(o64, Operand(o64, vixl::SXTW));
+ B(NotEqual, bail);
+ Mov(o32, o32);
+ B(&fin);
+
+ bind(&handleZero);
+ // Move the top word of the double into the output reg, if it is non-zero,
+ // then the original value was -0.0.
+ Fmov(o32, iFlt);
+ Cbnz(o32, bail);
+ bind(&fin);
+ }
+
+ void ceil(FloatRegister input, Register output, Label* bail) {
+ Label handleZero;
+ Label fin;
+ ARMFPRegister iDbl(input, 64);
+ ARMRegister o64(output, 64);
+ ARMRegister o32(output, 32);
+ Fcmp(iDbl, 0.0);
+ B(Assembler::Overflow, bail);
+ Fcvtps(o64, iDbl);
+ Cmp(o64, Operand(o64, vixl::SXTW));
+ B(NotEqual, bail);
+ Cbz(o64, &handleZero);
+ Mov(o32, o32);
+ B(&fin);
+
+ bind(&handleZero);
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ Fmov(scratch, iDbl);
+ Cbnz(scratch, bail);
+ bind(&fin);
+ }
+
+ void ceilf(FloatRegister input, Register output, Label* bail) {
+ Label handleZero;
+ Label fin;
+ ARMFPRegister iFlt(input, 32);
+ ARMRegister o64(output, 64);
+ ARMRegister o32(output, 32);
+ Fcmp(iFlt, 0.0);
+
+ // NaN is always a bail condition, just bail directly.
+ B(Assembler::Overflow, bail);
+ Fcvtps(o64, iFlt);
+ Cmp(o64, Operand(o64, vixl::SXTW));
+ B(NotEqual, bail);
+ Cbz(o64, &handleZero);
+ Mov(o32, o32);
+ B(&fin);
+
+ bind(&handleZero);
+ // Move the top word of the double into the output reg, if it is non-zero,
+ // then the original value was -0.0.
+ Fmov(o32, iFlt);
+ Cbnz(o32, bail);
+ bind(&fin);
+ }
+
+ void jump(Label* label) { B(label); }
+ void jump(JitCode* code) { branch(code); }
+ void jump(ImmPtr ptr) {
+ // It is unclear why this sync is necessary:
+ // * PSP and SP have been observed to be different in testcase
+ // tests/asm.js/testBug1046688.js.
+ // * Removing the sync causes no failures in all of jit-tests.
+ //
+ // Also see branch(JitCode*) below. This version of jump() is called only
+ // from jump(TrampolinePtr) which is called on various very slow paths,
+ // probably only in JS.
+ syncStackPtr();
+ BufferOffset loc =
+ b(-1,
+ LabelDoc()); // The jump target will be patched by executableCopy().
+ addPendingJump(loc, ptr, RelocationKind::HARDCODED);
+ }
+ void jump(TrampolinePtr code) { jump(ImmPtr(code.value)); }
+ void jump(Register reg) { Br(ARMRegister(reg, 64)); }
+ void jump(const Address& addr) {
+ vixl::UseScratchRegisterScope temps(this);
+ MOZ_ASSERT(temps.IsAvailable(ScratchReg64)); // ip0
+ temps.Exclude(ScratchReg64);
+ MOZ_ASSERT(addr.base != ScratchReg64.asUnsized());
+ loadPtr(addr, ScratchReg64.asUnsized());
+ br(ScratchReg64);
+ }
+
+ void align(int alignment) { armbuffer_.align(alignment); }
+
+ void haltingAlign(int alignment) {
+ armbuffer_.align(alignment, vixl::HLT | ImmException(0xBAAD));
+ }
+ void nopAlign(int alignment) { armbuffer_.align(alignment); }
+
+ void movePtr(Register src, Register dest) {
+ Mov(ARMRegister(dest, 64), ARMRegister(src, 64));
+ }
+ void movePtr(ImmWord imm, Register dest) {
+ Mov(ARMRegister(dest, 64), int64_t(imm.value));
+ }
+ void movePtr(ImmPtr imm, Register dest) {
+ Mov(ARMRegister(dest, 64), int64_t(imm.value));
+ }
+ void movePtr(wasm::SymbolicAddress imm, Register dest) {
+ BufferOffset off = movePatchablePtr(ImmWord(0xffffffffffffffffULL), dest);
+ append(wasm::SymbolicAccess(CodeOffset(off.getOffset()), imm));
+ }
+ void movePtr(ImmGCPtr imm, Register dest) {
+ BufferOffset load = movePatchablePtr(ImmPtr(imm.value), dest);
+ writeDataRelocation(imm, load);
+ }
+
+ void mov(ImmWord imm, Register dest) { movePtr(imm, dest); }
+ void mov(ImmPtr imm, Register dest) { movePtr(imm, dest); }
+ void mov(wasm::SymbolicAddress imm, Register dest) { movePtr(imm, dest); }
+ void mov(Register src, Register dest) { movePtr(src, dest); }
+ void mov(CodeLabel* label, Register dest);
+
+ void move32(Imm32 imm, Register dest) {
+ Mov(ARMRegister(dest, 32), (int64_t)imm.value);
+ }
+ void move32(Register src, Register dest) {
+ Mov(ARMRegister(dest, 32), ARMRegister(src, 32));
+ }
+
+ // Move a pointer using a literal pool, so that the pointer
+ // may be easily patched or traced.
+ // Returns the BufferOffset of the load instruction emitted.
+ BufferOffset movePatchablePtr(ImmWord ptr, Register dest);
+ BufferOffset movePatchablePtr(ImmPtr ptr, Register dest);
+
+ void loadPtr(wasm::SymbolicAddress address, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ movePtr(address, scratch.asUnsized());
+ Ldr(ARMRegister(dest, 64), MemOperand(scratch));
+ }
+ void loadPtr(AbsoluteAddress address, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ movePtr(ImmWord((uintptr_t)address.addr), scratch.asUnsized());
+ Ldr(ARMRegister(dest, 64), MemOperand(scratch));
+ }
+ void loadPtr(const Address& address, Register dest) {
+ Ldr(ARMRegister(dest, 64), MemOperand(address));
+ }
+ void loadPtr(const BaseIndex& src, Register dest) {
+ ARMRegister base = toARMRegister(src.base, 64);
+ uint32_t scale = Imm32::ShiftOf(src.scale).value;
+ ARMRegister dest64(dest, 64);
+ ARMRegister index64(src.index, 64);
+
+ if (src.offset) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch = temps.AcquireX();
+ MOZ_ASSERT(!scratch.Is(base));
+ MOZ_ASSERT(!scratch.Is(dest64));
+ MOZ_ASSERT(!scratch.Is(index64));
+
+ Add(scratch, base, Operand(int64_t(src.offset)));
+ Ldr(dest64, MemOperand(scratch, index64, vixl::LSL, scale));
+ return;
+ }
+
+ Ldr(dest64, MemOperand(base, index64, vixl::LSL, scale));
+ }
+ void loadPrivate(const Address& src, Register dest);
+
+ void store8(Register src, const Address& address) {
+ Strb(ARMRegister(src, 32), toMemOperand(address));
+ }
+ void store8(Imm32 imm, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != address.base);
+ move32(imm, scratch32.asUnsized());
+ Strb(scratch32, toMemOperand(address));
+ }
+ void store8(Register src, const BaseIndex& address) {
+ doBaseIndex(ARMRegister(src, 32), address, vixl::STRB_w);
+ }
+ void store8(Imm32 imm, const BaseIndex& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != address.base);
+ MOZ_ASSERT(scratch32.asUnsized() != address.index);
+ Mov(scratch32, Operand(imm.value));
+ doBaseIndex(scratch32, address, vixl::STRB_w);
+ }
+
+ void store16(Register src, const Address& address) {
+ Strh(ARMRegister(src, 32), toMemOperand(address));
+ }
+ void store16(Imm32 imm, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != address.base);
+ move32(imm, scratch32.asUnsized());
+ Strh(scratch32, toMemOperand(address));
+ }
+ void store16(Register src, const BaseIndex& address) {
+ doBaseIndex(ARMRegister(src, 32), address, vixl::STRH_w);
+ }
+ void store16(Imm32 imm, const BaseIndex& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != address.base);
+ MOZ_ASSERT(scratch32.asUnsized() != address.index);
+ Mov(scratch32, Operand(imm.value));
+ doBaseIndex(scratch32, address, vixl::STRH_w);
+ }
+ template <typename S, typename T>
+ void store16Unaligned(const S& src, const T& dest) {
+ store16(src, dest);
+ }
+
+ void storePtr(ImmWord imm, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != address.base);
+ movePtr(imm, scratch);
+ storePtr(scratch, address);
+ }
+ void storePtr(ImmPtr imm, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != address.base);
+ Mov(scratch64, uint64_t(imm.value));
+ Str(scratch64, toMemOperand(address));
+ }
+ void storePtr(ImmGCPtr imm, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != address.base);
+ movePtr(imm, scratch);
+ storePtr(scratch, address);
+ }
+ void storePtr(Register src, const Address& address) {
+ Str(ARMRegister(src, 64), toMemOperand(address));
+ }
+
+ void storePtr(ImmWord imm, const BaseIndex& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != address.base);
+ MOZ_ASSERT(scratch64.asUnsized() != address.index);
+ Mov(scratch64, Operand(imm.value));
+ doBaseIndex(scratch64, address, vixl::STR_x);
+ }
+ void storePtr(ImmGCPtr imm, const BaseIndex& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != address.base);
+ MOZ_ASSERT(scratch != address.index);
+ movePtr(imm, scratch);
+ doBaseIndex(ARMRegister(scratch, 64), address, vixl::STR_x);
+ }
+ void storePtr(Register src, const BaseIndex& address) {
+ doBaseIndex(ARMRegister(src, 64), address, vixl::STR_x);
+ }
+
+ void storePtr(Register src, AbsoluteAddress address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ Mov(scratch64, uint64_t(address.addr));
+ Str(ARMRegister(src, 64), MemOperand(scratch64));
+ }
+
+ void store32(Register src, AbsoluteAddress address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ Mov(scratch64, uint64_t(address.addr));
+ Str(ARMRegister(src, 32), MemOperand(scratch64));
+ }
+ void store32(Imm32 imm, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != address.base);
+ Mov(scratch32, uint64_t(imm.value));
+ Str(scratch32, toMemOperand(address));
+ }
+ void store32(Register r, const Address& address) {
+ Str(ARMRegister(r, 32), toMemOperand(address));
+ }
+ void store32(Imm32 imm, const BaseIndex& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != address.base);
+ MOZ_ASSERT(scratch32.asUnsized() != address.index);
+ Mov(scratch32, imm.value);
+ doBaseIndex(scratch32, address, vixl::STR_w);
+ }
+ void store32(Register r, const BaseIndex& address) {
+ doBaseIndex(ARMRegister(r, 32), address, vixl::STR_w);
+ }
+
+ void store32_NoSecondScratch(Imm32 imm, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ temps.Exclude(ARMRegister(ScratchReg2, 32)); // Disallow ScratchReg2.
+ const ARMRegister scratch32 = temps.AcquireW();
+
+ MOZ_ASSERT(scratch32.asUnsized() != address.base);
+ Mov(scratch32, uint64_t(imm.value));
+ Str(scratch32, toMemOperand(address));
+ }
+
+ template <typename S, typename T>
+ void store32Unaligned(const S& src, const T& dest) {
+ store32(src, dest);
+ }
+
+ void store64(Register64 src, Address address) { storePtr(src.reg, address); }
+
+ void store64(Register64 src, const BaseIndex& address) {
+ storePtr(src.reg, address);
+ }
+
+ void store64(Imm64 imm, const BaseIndex& address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+
+ void store64(Imm64 imm, const Address& address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+
+ template <typename S, typename T>
+ void store64Unaligned(const S& src, const T& dest) {
+ store64(src, dest);
+ }
+
+ // StackPointer manipulation.
+ inline void addToStackPtr(Register src);
+ inline void addToStackPtr(Imm32 imm);
+ inline void addToStackPtr(const Address& src);
+ inline void addStackPtrTo(Register dest);
+
+ inline void subFromStackPtr(Register src);
+ inline void subFromStackPtr(Imm32 imm);
+ inline void subStackPtrFrom(Register dest);
+
+ inline void andToStackPtr(Imm32 t);
+
+ inline void moveToStackPtr(Register src);
+ inline void moveStackPtrTo(Register dest);
+
+ inline void loadStackPtr(const Address& src);
+ inline void storeStackPtr(const Address& dest);
+
+ // StackPointer testing functions.
+ inline void branchTestStackPtr(Condition cond, Imm32 rhs, Label* label);
+ inline void branchStackPtr(Condition cond, Register rhs, Label* label);
+ inline void branchStackPtrRhs(Condition cond, Address lhs, Label* label);
+ inline void branchStackPtrRhs(Condition cond, AbsoluteAddress lhs,
+ Label* label);
+
+ void testPtr(Register lhs, Register rhs) {
+ Tst(ARMRegister(lhs, 64), Operand(ARMRegister(rhs, 64)));
+ }
+ void test32(Register lhs, Register rhs) {
+ Tst(ARMRegister(lhs, 32), Operand(ARMRegister(rhs, 32)));
+ }
+ void test32(const Address& addr, Imm32 imm) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != addr.base);
+ load32(addr, scratch32.asUnsized());
+ Tst(scratch32, Operand(imm.value));
+ }
+ void test32(Register lhs, Imm32 rhs) {
+ Tst(ARMRegister(lhs, 32), Operand(rhs.value));
+ }
+ void cmp32(Register lhs, Imm32 rhs) {
+ Cmp(ARMRegister(lhs, 32), Operand(rhs.value));
+ }
+ void cmp32(Register a, Register b) {
+ Cmp(ARMRegister(a, 32), Operand(ARMRegister(b, 32)));
+ }
+ void cmp32(const Address& lhs, Imm32 rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != lhs.base);
+ Ldr(scratch32, toMemOperand(lhs));
+ Cmp(scratch32, Operand(rhs.value));
+ }
+ void cmp32(const Address& lhs, Register rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != lhs.base);
+ MOZ_ASSERT(scratch32.asUnsized() != rhs);
+ Ldr(scratch32, toMemOperand(lhs));
+ Cmp(scratch32, Operand(ARMRegister(rhs, 32)));
+ }
+ void cmp32(const vixl::Operand& lhs, Imm32 rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ Mov(scratch32, lhs);
+ Cmp(scratch32, Operand(rhs.value));
+ }
+ void cmp32(const vixl::Operand& lhs, Register rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ Mov(scratch32, lhs);
+ Cmp(scratch32, Operand(ARMRegister(rhs, 32)));
+ }
+
+ void cmn32(Register lhs, Imm32 rhs) {
+ Cmn(ARMRegister(lhs, 32), Operand(rhs.value));
+ }
+
+ void cmpPtr(Register lhs, Imm32 rhs) {
+ Cmp(ARMRegister(lhs, 64), Operand(rhs.value));
+ }
+ void cmpPtr(Register lhs, ImmWord rhs) {
+ Cmp(ARMRegister(lhs, 64), Operand(rhs.value));
+ }
+ void cmpPtr(Register lhs, ImmPtr rhs) {
+ Cmp(ARMRegister(lhs, 64), Operand(uint64_t(rhs.value)));
+ }
+ void cmpPtr(Register lhs, Imm64 rhs) {
+ Cmp(ARMRegister(lhs, 64), Operand(uint64_t(rhs.value)));
+ }
+ void cmpPtr(Register lhs, Register rhs) {
+ Cmp(ARMRegister(lhs, 64), ARMRegister(rhs, 64));
+ }
+ void cmpPtr(Register lhs, ImmGCPtr rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs);
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+
+ void cmpPtr(const Address& lhs, Register rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != lhs.base);
+ MOZ_ASSERT(scratch64.asUnsized() != rhs);
+ Ldr(scratch64, toMemOperand(lhs));
+ Cmp(scratch64, Operand(ARMRegister(rhs, 64)));
+ }
+ void cmpPtr(const Address& lhs, ImmWord rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != lhs.base);
+ Ldr(scratch64, toMemOperand(lhs));
+ Cmp(scratch64, Operand(rhs.value));
+ }
+ void cmpPtr(const Address& lhs, ImmPtr rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != lhs.base);
+ Ldr(scratch64, toMemOperand(lhs));
+ Cmp(scratch64, Operand(uint64_t(rhs.value)));
+ }
+ void cmpPtr(const Address& lhs, ImmGCPtr rhs) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != lhs.base);
+ loadPtr(lhs, scratch);
+ cmpPtr(scratch, rhs);
+ }
+
+ void loadDouble(const Address& src, FloatRegister dest) {
+ Ldr(ARMFPRegister(dest, 64), MemOperand(src));
+ }
+ void loadDouble(const BaseIndex& src, FloatRegister dest) {
+ ARMRegister base = toARMRegister(src.base, 64);
+ ARMRegister index(src.index, 64);
+
+ if (src.offset == 0) {
+ Ldr(ARMFPRegister(dest, 64),
+ MemOperand(base, index, vixl::LSL, unsigned(src.scale)));
+ return;
+ }
+
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != src.base);
+ MOZ_ASSERT(scratch64.asUnsized() != src.index);
+
+ Add(scratch64, base, Operand(index, vixl::LSL, unsigned(src.scale)));
+ Ldr(ARMFPRegister(dest, 64), MemOperand(scratch64, src.offset));
+ }
+ void loadFloatAsDouble(const Address& addr, FloatRegister dest) {
+ Ldr(ARMFPRegister(dest, 32), toMemOperand(addr));
+ fcvt(ARMFPRegister(dest, 64), ARMFPRegister(dest, 32));
+ }
+ void loadFloatAsDouble(const BaseIndex& src, FloatRegister dest) {
+ ARMRegister base = toARMRegister(src.base, 64);
+ ARMRegister index(src.index, 64);
+ if (src.offset == 0) {
+ Ldr(ARMFPRegister(dest, 32),
+ MemOperand(base, index, vixl::LSL, unsigned(src.scale)));
+ } else {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != src.base);
+ MOZ_ASSERT(scratch64.asUnsized() != src.index);
+
+ Add(scratch64, base, Operand(index, vixl::LSL, unsigned(src.scale)));
+ Ldr(ARMFPRegister(dest, 32), MemOperand(scratch64, src.offset));
+ }
+ fcvt(ARMFPRegister(dest, 64), ARMFPRegister(dest, 32));
+ }
+
+ void loadFloat32(const Address& addr, FloatRegister dest) {
+ Ldr(ARMFPRegister(dest, 32), toMemOperand(addr));
+ }
+ void loadFloat32(const BaseIndex& src, FloatRegister dest) {
+ ARMRegister base = toARMRegister(src.base, 64);
+ ARMRegister index(src.index, 64);
+ if (src.offset == 0) {
+ Ldr(ARMFPRegister(dest, 32),
+ MemOperand(base, index, vixl::LSL, unsigned(src.scale)));
+ } else {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != src.base);
+ MOZ_ASSERT(scratch64.asUnsized() != src.index);
+
+ Add(scratch64, base, Operand(index, vixl::LSL, unsigned(src.scale)));
+ Ldr(ARMFPRegister(dest, 32), MemOperand(scratch64, src.offset));
+ }
+ }
+
+ void moveDouble(FloatRegister src, FloatRegister dest) {
+ fmov(ARMFPRegister(dest, 64), ARMFPRegister(src, 64));
+ }
+ void zeroDouble(FloatRegister reg) {
+ fmov(ARMFPRegister(reg, 64), vixl::xzr);
+ }
+ void zeroFloat32(FloatRegister reg) {
+ fmov(ARMFPRegister(reg, 32), vixl::wzr);
+ }
+
+ void moveFloat32(FloatRegister src, FloatRegister dest) {
+ fmov(ARMFPRegister(dest, 32), ARMFPRegister(src, 32));
+ }
+ void moveFloatAsDouble(Register src, FloatRegister dest) {
+ MOZ_CRASH("moveFloatAsDouble");
+ }
+
+ void moveSimd128(FloatRegister src, FloatRegister dest) {
+ fmov(ARMFPRegister(dest, 128), ARMFPRegister(src, 128));
+ }
+
+ void splitSignExtTag(const ValueOperand& operand, Register dest) {
+ splitSignExtTag(operand.valueReg(), dest);
+ }
+ void splitSignExtTag(const Address& operand, Register dest) {
+ loadPtr(operand, dest);
+ splitSignExtTag(dest, dest);
+ }
+ void splitSignExtTag(const BaseIndex& operand, Register dest) {
+ loadPtr(operand, dest);
+ splitSignExtTag(dest, dest);
+ }
+
+ // Extracts the tag of a value and places it in tag
+ inline void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag);
+ void cmpTag(const ValueOperand& operand, ImmTag tag) { MOZ_CRASH("cmpTag"); }
+
+ void load32(const Address& address, Register dest) {
+ Ldr(ARMRegister(dest, 32), toMemOperand(address));
+ }
+ void load32(const BaseIndex& src, Register dest) {
+ doBaseIndex(ARMRegister(dest, 32), src, vixl::LDR_w);
+ }
+ void load32(AbsoluteAddress address, Register dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ movePtr(ImmWord((uintptr_t)address.addr), scratch64.asUnsized());
+ ldr(ARMRegister(dest, 32), MemOperand(scratch64));
+ }
+ template <typename S>
+ void load32Unaligned(const S& src, Register dest) {
+ load32(src, dest);
+ }
+ void load64(const Address& address, Register64 dest) {
+ loadPtr(address, dest.reg);
+ }
+ void load64(const BaseIndex& address, Register64 dest) {
+ loadPtr(address, dest.reg);
+ }
+ template <typename S>
+ void load64Unaligned(const S& src, Register64 dest) {
+ load64(src, dest);
+ }
+
+ void load8SignExtend(const Address& address, Register dest) {
+ Ldrsb(ARMRegister(dest, 32), toMemOperand(address));
+ }
+ void load8SignExtend(const BaseIndex& src, Register dest) {
+ doBaseIndex(ARMRegister(dest, 32), src, vixl::LDRSB_w);
+ }
+
+ void load8ZeroExtend(const Address& address, Register dest) {
+ Ldrb(ARMRegister(dest, 32), toMemOperand(address));
+ }
+ void load8ZeroExtend(const BaseIndex& src, Register dest) {
+ doBaseIndex(ARMRegister(dest, 32), src, vixl::LDRB_w);
+ }
+
+ void load16SignExtend(const Address& address, Register dest) {
+ Ldrsh(ARMRegister(dest, 32), toMemOperand(address));
+ }
+ void load16SignExtend(const BaseIndex& src, Register dest) {
+ doBaseIndex(ARMRegister(dest, 32), src, vixl::LDRSH_w);
+ }
+ template <typename S>
+ void load16UnalignedSignExtend(const S& src, Register dest) {
+ load16SignExtend(src, dest);
+ }
+
+ void load16ZeroExtend(const Address& address, Register dest) {
+ Ldrh(ARMRegister(dest, 32), toMemOperand(address));
+ }
+ void load16ZeroExtend(const BaseIndex& src, Register dest) {
+ doBaseIndex(ARMRegister(dest, 32), src, vixl::LDRH_w);
+ }
+ template <typename S>
+ void load16UnalignedZeroExtend(const S& src, Register dest) {
+ load16ZeroExtend(src, dest);
+ }
+
+ void adds32(Register src, Register dest) {
+ Adds(ARMRegister(dest, 32), ARMRegister(dest, 32),
+ Operand(ARMRegister(src, 32)));
+ }
+ void adds32(Imm32 imm, Register dest) {
+ Adds(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
+ }
+ void adds32(Imm32 imm, const Address& dest) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != dest.base);
+
+ Ldr(scratch32, toMemOperand(dest));
+ Adds(scratch32, scratch32, Operand(imm.value));
+ Str(scratch32, toMemOperand(dest));
+ }
+ void adds64(Imm32 imm, Register dest) {
+ Adds(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
+ }
+ void adds64(ImmWord imm, Register dest) {
+ Adds(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
+ }
+ void adds64(Register src, Register dest) {
+ Adds(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(ARMRegister(src, 64)));
+ }
+
+ void subs32(Imm32 imm, Register dest) {
+ Subs(ARMRegister(dest, 32), ARMRegister(dest, 32), Operand(imm.value));
+ }
+ void subs32(Register src, Register dest) {
+ Subs(ARMRegister(dest, 32), ARMRegister(dest, 32),
+ Operand(ARMRegister(src, 32)));
+ }
+ void subs64(Imm32 imm, Register dest) {
+ Subs(ARMRegister(dest, 64), ARMRegister(dest, 64), Operand(imm.value));
+ }
+ void subs64(Register src, Register dest) {
+ Subs(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(ARMRegister(src, 64)));
+ }
+
+ void ret() {
+ pop(lr);
+ abiret();
+ }
+
+ void retn(Imm32 n) {
+ vixl::UseScratchRegisterScope temps(this);
+ MOZ_ASSERT(temps.IsAvailable(ScratchReg64)); // ip0
+ temps.Exclude(ScratchReg64);
+ // ip0 <- [sp]; sp += n; ret ip0
+ Ldr(ScratchReg64,
+ MemOperand(GetStackPointer64(), ptrdiff_t(n.value), vixl::PostIndex));
+ syncStackPtr(); // SP is always used to transmit the stack between calls.
+ Ret(ScratchReg64);
+ }
+
+ void j(Condition cond, Label* dest) { B(dest, cond); }
+
+ void branch(Condition cond, Label* label) { B(label, cond); }
+ void branch(JitCode* target) {
+ // It is unclear why this sync is necessary:
+ // * PSP and SP have been observed to be different in testcase
+ // tests/async/debugger-reject-after-fulfill.js
+ // * Removing the sync causes no failures in all of jit-tests.
+ //
+ // Also see jump() above. This is used only to implement jump(JitCode*)
+ // and only for JS, it appears.
+ syncStackPtr();
+ BufferOffset loc =
+ b(-1,
+ LabelDoc()); // The jump target will be patched by executableCopy().
+ addPendingJump(loc, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+
+ void compareDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs) {
+ Fcmp(ARMFPRegister(lhs, 64), ARMFPRegister(rhs, 64));
+ }
+
+ void compareFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs) {
+ Fcmp(ARMFPRegister(lhs, 32), ARMFPRegister(rhs, 32));
+ }
+
+ void compareSimd128Int(Assembler::Condition cond, ARMFPRegister dest,
+ ARMFPRegister lhs, ARMFPRegister rhs);
+ void compareSimd128Float(Assembler::Condition cond, ARMFPRegister dest,
+ ARMFPRegister lhs, ARMFPRegister rhs);
+ void rightShiftInt8x16(FloatRegister lhs, Register rhs, FloatRegister dest,
+ bool isUnsigned);
+ void rightShiftInt16x8(FloatRegister lhs, Register rhs, FloatRegister dest,
+ bool isUnsigned);
+ void rightShiftInt32x4(FloatRegister lhs, Register rhs, FloatRegister dest,
+ bool isUnsigned);
+ void rightShiftInt64x2(FloatRegister lhs, Register rhs, FloatRegister dest,
+ bool isUnsigned);
+
+ void branchNegativeZero(FloatRegister reg, Register scratch, Label* label) {
+ MOZ_CRASH("branchNegativeZero");
+ }
+ void branchNegativeZeroFloat32(FloatRegister reg, Register scratch,
+ Label* label) {
+ MOZ_CRASH("branchNegativeZeroFloat32");
+ }
+
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister) {
+ Fmov(ARMRegister(dest.valueReg(), 64), ARMFPRegister(src, 64));
+ }
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) {
+ boxValue(type, src, dest.valueReg());
+ }
+
+ // Note that the |dest| register here may be ScratchReg, so we shouldn't use
+ // it.
+ void unboxInt32(const ValueOperand& src, Register dest) {
+ move32(src.valueReg(), dest);
+ }
+ void unboxInt32(const Address& src, Register dest) { load32(src, dest); }
+ void unboxInt32(const BaseIndex& src, Register dest) { load32(src, dest); }
+
+ template <typename T>
+ void unboxDouble(const T& src, FloatRegister dest) {
+ loadDouble(src, dest);
+ }
+ void unboxDouble(const ValueOperand& src, FloatRegister dest) {
+ Fmov(ARMFPRegister(dest, 64), ARMRegister(src.valueReg(), 64));
+ }
+
+ void unboxArgObjMagic(const ValueOperand& src, Register dest) {
+ MOZ_CRASH("unboxArgObjMagic");
+ }
+ void unboxArgObjMagic(const Address& src, Register dest) {
+ MOZ_CRASH("unboxArgObjMagic");
+ }
+
+ void unboxBoolean(const ValueOperand& src, Register dest) {
+ move32(src.valueReg(), dest);
+ }
+ void unboxBoolean(const Address& src, Register dest) { load32(src, dest); }
+ void unboxBoolean(const BaseIndex& src, Register dest) { load32(src, dest); }
+
+ void unboxMagic(const ValueOperand& src, Register dest) {
+ move32(src.valueReg(), dest);
+ }
+ void unboxNonDouble(const ValueOperand& src, Register dest,
+ JSValueType type) {
+ unboxNonDouble(src.valueReg(), dest, type);
+ }
+
+ template <typename T>
+ void unboxNonDouble(T src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ load32(src, dest);
+ return;
+ }
+ loadPtr(src, dest);
+ unboxNonDouble(dest, dest, type);
+ }
+
+ void unboxNonDouble(Register src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ move32(src, dest);
+ return;
+ }
+ Eor(ARMRegister(dest, 64), ARMRegister(src, 64),
+ Operand(JSVAL_TYPE_TO_SHIFTED_TAG(type)));
+ }
+
+ void notBoolean(const ValueOperand& val) {
+ ARMRegister r(val.valueReg(), 64);
+ eor(r, r, Operand(1));
+ }
+ void unboxObject(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src.valueReg(), dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Address& src, Register dest) {
+ loadPtr(src, dest);
+ unboxNonDouble(dest, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const BaseIndex& src, Register dest) {
+ doBaseIndex(ARMRegister(dest, 64), src, vixl::LDR_x);
+ unboxNonDouble(dest, dest, JSVAL_TYPE_OBJECT);
+ }
+
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ And(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(~JS::detail::ValueObjectOrNullBit));
+ }
+
+ // See comment in MacroAssembler-x64.h.
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ loadPtr(src, dest);
+ And(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(JS::detail::ValueGCThingPayloadMask));
+ }
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ And(ARMRegister(dest, 64), ARMRegister(src.valueReg(), 64),
+ Operand(JS::detail::ValueGCThingPayloadMask));
+ }
+
+ // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
+ void getGCThingValueChunk(const Address& src, Register dest) {
+ loadPtr(src, dest);
+ And(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(JS::detail::ValueGCThingPayloadChunkMask));
+ }
+ void getGCThingValueChunk(const ValueOperand& src, Register dest) {
+ And(ARMRegister(dest, 64), ARMRegister(src.valueReg(), 64),
+ Operand(JS::detail::ValueGCThingPayloadChunkMask));
+ }
+
+ inline void unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType type);
+
+ void unboxString(const ValueOperand& operand, Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxSymbol(const ValueOperand& operand, Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxSymbol(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxBigInt(const ValueOperand& operand, Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ // These two functions use the low 32-bits of the full value register.
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+ }
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+ }
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+ }
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+ }
+
+ void loadConstantDouble(double d, FloatRegister dest) {
+ ARMFPRegister r(dest, 64);
+ if (d == 0.0) {
+ // Clang11 does movi for 0 and movi+fneg for -0, and this seems like a
+ // good implementation-independent strategy as it avoids any gpr->fpr
+ // moves or memory traffic.
+ Movi(r, 0);
+ if (std::signbit(d)) {
+ Fneg(r, r);
+ }
+ } else {
+ Fmov(r, d);
+ }
+ }
+ void loadConstantFloat32(float f, FloatRegister dest) {
+ ARMFPRegister r(dest, 32);
+ if (f == 0.0) {
+ // See comments above. There's not a movi variant for a single register,
+ // so clear the double.
+ Movi(ARMFPRegister(dest, 64), 0);
+ if (std::signbit(f)) {
+ Fneg(r, r);
+ }
+ } else {
+ Fmov(r, f);
+ }
+ }
+
+ void cmpTag(Register tag, ImmTag ref) {
+ // As opposed to other architecture, splitTag is replaced by splitSignExtTag
+ // which extract the tag with a sign extension. The reason being that cmp32
+ // with a tag value would be too large to fit as a 12 bits immediate value,
+ // and would require the VIXL macro assembler to add an extra instruction
+ // and require extra scratch register to load the Tag value.
+ //
+ // Instead, we compare with the negative value of the sign extended tag with
+ // the CMN instruction. The sign extended tag is expected to be a negative
+ // value. Therefore the negative of the sign extended tag is expected to be
+ // near 0 and fit on 12 bits.
+ //
+ // Ignoring the sign extension, the logic is the following:
+ //
+ // CMP32(Reg, Tag) = Reg - Tag
+ // = Reg + (-Tag)
+ // = CMN32(Reg, -Tag)
+ //
+ // Note: testGCThing, testPrimitive and testNumber which are checking for
+ // inequalities should use unsigned comparisons (as done by default) in
+ // order to keep the same relation order after the sign extension, i.e.
+ // using Above or Below which are based on the carry flag.
+ uint32_t hiShift = JSVAL_TAG_SHIFT - 32;
+ int32_t seTag = int32_t(ref.value);
+ seTag = (seTag << hiShift) >> hiShift;
+ MOZ_ASSERT(seTag < 0);
+ int32_t negTag = -seTag;
+ // Check thest negTag is encoded on a 12 bits immediate value.
+ MOZ_ASSERT((negTag & ~0xFFF) == 0);
+ cmn32(tag, Imm32(negTag));
+ }
+
+ // Register-based tests.
+ Condition testUndefined(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+ }
+ Condition testInt32(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_INT32));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+ }
+ Condition testNull(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_NULL));
+ return cond;
+ }
+ Condition testString(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_STRING));
+ return cond;
+ }
+ Condition testSymbol(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+ }
+ Condition testBigInt(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+ }
+ Condition testObject(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+ }
+ Condition testDouble(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_MAX_DOUBLE));
+ // Requires unsigned comparison due to cmpTag internals.
+ return (cond == Equal) ? BelowOrEqual : Above;
+ }
+ Condition testNumber(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JS::detail::ValueUpperInclNumberTag));
+ // Requires unsigned comparison due to cmpTag internals.
+ return (cond == Equal) ? BelowOrEqual : Above;
+ }
+ Condition testGCThing(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag));
+ // Requires unsigned comparison due to cmpTag internals.
+ return (cond == Equal) ? AboveOrEqual : Below;
+ }
+ Condition testMagic(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+ }
+ Condition testPrimitive(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmpTag(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
+ // Requires unsigned comparison due to cmpTag internals.
+ return (cond == Equal) ? Below : AboveOrEqual;
+ }
+ Condition testError(Condition cond, Register tag) {
+ return testMagic(cond, tag);
+ }
+
+ // ValueOperand-based tests.
+ Condition testInt32(Condition cond, const ValueOperand& value) {
+ // The incoming ValueOperand may use scratch registers.
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(scratch != value.valueReg());
+
+ splitSignExtTag(value, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testNull(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testUndefined(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testString(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testObject(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testNumber(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testNumber(cond, scratch);
+ }
+ Condition testPrimitive(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testPrimitive(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const ValueOperand& value) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(value.valueReg() != scratch);
+ splitSignExtTag(value, scratch);
+ return testGCThing(cond, scratch);
+ }
+ Condition testError(Condition cond, const ValueOperand& value) {
+ return testMagic(cond, value);
+ }
+
+ // Address-based tests.
+ Condition testGCThing(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testGCThing(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testInt32(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testNull(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testUndefined(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testString(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testObject(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testNumber(Condition cond, const Address& address) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(address.base != scratch);
+ splitSignExtTag(address, scratch);
+ return testNumber(cond, scratch);
+ }
+
+ // BaseIndex-based tests.
+ Condition testUndefined(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testNull(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testString(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testInt32(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testObject(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const BaseIndex& src) {
+ vixl::UseScratchRegisterScope temps(this);
+ const Register scratch = temps.AcquireX().asUnsized();
+ MOZ_ASSERT(src.base != scratch);
+ MOZ_ASSERT(src.index != scratch);
+ splitSignExtTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+
+ Condition testInt32Truthy(bool truthy, const ValueOperand& operand) {
+ ARMRegister payload32(operand.valueReg(), 32);
+ Tst(payload32, payload32);
+ return truthy ? NonZero : Zero;
+ }
+
+ Condition testBooleanTruthy(bool truthy, const ValueOperand& operand) {
+ ARMRegister payload32(operand.valueReg(), 32);
+ Tst(payload32, payload32);
+ return truthy ? NonZero : Zero;
+ }
+
+ Condition testBigIntTruthy(bool truthy, const ValueOperand& value);
+ Condition testStringTruthy(bool truthy, const ValueOperand& value);
+
+ void int32OrDouble(Register src, ARMFPRegister dest) {
+ Label isInt32;
+ Label join;
+ testInt32(Equal, ValueOperand(src));
+ B(&isInt32, Equal);
+ // is double, move the bits as is
+ Fmov(dest, ARMRegister(src, 64));
+ B(&join);
+ bind(&isInt32);
+ // is int32, do a conversion while moving
+ Scvtf(dest, ARMRegister(src, 32));
+ bind(&join);
+ }
+ void loadUnboxedValue(Address address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != address.base);
+ Ldr(scratch64, toMemOperand(address));
+ int32OrDouble(scratch64.asUnsized(), ARMFPRegister(dest.fpu(), 64));
+ } else {
+ unboxNonDouble(address, dest.gpr(), ValueTypeFromMIRType(type));
+ }
+ }
+
+ void loadUnboxedValue(BaseIndex address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT(scratch64.asUnsized() != address.base);
+ MOZ_ASSERT(scratch64.asUnsized() != address.index);
+ doBaseIndex(scratch64, address, vixl::LDR_x);
+ int32OrDouble(scratch64.asUnsized(), ARMFPRegister(dest.fpu(), 64));
+ } else {
+ unboxNonDouble(address, dest.gpr(), ValueTypeFromMIRType(type));
+ }
+ }
+
+ void loadInstructionPointerAfterCall(Register dest) {
+ MOZ_CRASH("loadInstructionPointerAfterCall");
+ }
+
+ // Emit a B that can be toggled to a CMP. See ToggleToJmp(), ToggleToCmp().
+ CodeOffset toggledJump(Label* label) {
+ BufferOffset offset = b(label, Always);
+ CodeOffset ret(offset.getOffset());
+ return ret;
+ }
+
+ // load: offset to the load instruction obtained by movePatchablePtr().
+ void writeDataRelocation(ImmGCPtr ptr, BufferOffset load) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (ptr.value) {
+ if (gc::IsInsideNursery(ptr.value)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(load.getOffset());
+ }
+ }
+ void writeDataRelocation(const Value& val, BufferOffset load) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (val.isGCThing()) {
+ gc::Cell* cell = val.toGCThing();
+ if (cell && gc::IsInsideNursery(cell)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(load.getOffset());
+ }
+ }
+
+ void computeEffectiveAddress(const Address& address, Register dest) {
+ Add(ARMRegister(dest, 64), toARMRegister(address.base, 64),
+ Operand(address.offset));
+ }
+ void computeEffectiveAddress(const Address& address, RegisterOrSP dest) {
+ Add(toARMRegister(dest, 64), toARMRegister(address.base, 64),
+ Operand(address.offset));
+ }
+ void computeEffectiveAddress(const BaseIndex& address, Register dest) {
+ ARMRegister dest64(dest, 64);
+ ARMRegister base64 = toARMRegister(address.base, 64);
+ ARMRegister index64(address.index, 64);
+
+ Add(dest64, base64, Operand(index64, vixl::LSL, address.scale));
+ if (address.offset) {
+ Add(dest64, dest64, Operand(address.offset));
+ }
+ }
+
+ public:
+ void handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail);
+
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+ Address ToPayload(Address value) { return value; }
+ Address ToType(Address value) { return value; }
+
+ void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, AnyRegister outany, Register64 out64);
+ void wasmLoadImpl(const wasm::MemoryAccessDesc& access, MemOperand srcAddr,
+ AnyRegister outany, Register64 out64);
+ void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister valany,
+ Register64 val64, Register memoryBase, Register ptr);
+ void wasmStoreImpl(const wasm::MemoryAccessDesc& access, MemOperand destAddr,
+ AnyRegister valany, Register64 val64);
+ // The complete address is in `address`, and `access` is used for its type
+ // attributes only; its `offset` is ignored.
+ void wasmLoadAbsolute(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, uint64_t address, AnyRegister out,
+ Register64 out64);
+ void wasmStoreAbsolute(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Register64 value64,
+ Register memoryBase, uint64_t address);
+
+ // Emit a BLR or NOP instruction. ToggleCall can be used to patch
+ // this instruction.
+ CodeOffset toggledCall(JitCode* target, bool enabled) {
+ // The returned offset must be to the first instruction generated,
+ // for the debugger to match offset with Baseline's pcMappingEntries_.
+ BufferOffset offset = nextOffset();
+
+ // It is unclear why this sync is necessary:
+ // * PSP and SP have been observed to be different in testcase
+ // tests/cacheir/bug1448136.js
+ // * Removing the sync causes no failures in all of jit-tests.
+ syncStackPtr();
+
+ BufferOffset loadOffset;
+ {
+ vixl::UseScratchRegisterScope temps(this);
+
+ // The register used for the load is hardcoded, so that ToggleCall
+ // can patch in the branch instruction easily. This could be changed,
+ // but then ToggleCall must read the target register from the load.
+ MOZ_ASSERT(temps.IsAvailable(ScratchReg2_64));
+ temps.Exclude(ScratchReg2_64);
+
+ loadOffset = immPool64(ScratchReg2_64, uint64_t(target->raw()));
+
+ if (enabled) {
+ blr(ScratchReg2_64);
+ } else {
+ nop();
+ }
+ }
+
+ addPendingJump(loadOffset, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ CodeOffset ret(offset.getOffset());
+ return ret;
+ }
+
+ static size_t ToggledCallSize(uint8_t* code) {
+ // The call site is a sequence of two or three instructions:
+ //
+ // syncStack (optional)
+ // ldr/adr
+ // nop/blr
+ //
+ // Flushed constant pools can appear before any of the instructions.
+
+ const Instruction* cur = (const Instruction*)code;
+ cur = cur->skipPool();
+ if (cur->IsStackPtrSync()) cur = cur->NextInstruction();
+ cur = cur->skipPool();
+ cur = cur->NextInstruction(); // LDR/ADR
+ cur = cur->skipPool();
+ cur = cur->NextInstruction(); // NOP/BLR
+ return (uint8_t*)cur - code;
+ }
+
+ void checkARMRegAlignment(const ARMRegister& reg) {
+#ifdef DEBUG
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch64 = temps.AcquireX();
+ MOZ_ASSERT_IF(!reg.IsSP(), scratch64.asUnsized() != reg.asUnsized());
+ Label aligned;
+ Mov(scratch64, reg);
+ Tst(scratch64, Operand(StackAlignment - 1));
+ B(Zero, &aligned);
+ breakpoint();
+ bind(&aligned);
+ Mov(scratch64, vixl::xzr); // Clear the scratch register for sanity.
+#endif
+ }
+
+ void checkStackAlignment() {
+#ifdef DEBUG
+ checkARMRegAlignment(GetStackPointer64());
+
+ // If another register is being used to track pushes, check sp explicitly.
+ if (!GetStackPointer64().Is(vixl::sp)) {
+ checkARMRegAlignment(vixl::sp);
+ }
+#endif
+ }
+
+ void abiret() {
+ syncStackPtr(); // SP is always used to transmit the stack between calls.
+ vixl::MacroAssembler::Ret(vixl::lr);
+ }
+
+ void clampCheck(Register r, Label* handleNotAnInt) {
+ MOZ_CRASH("clampCheck");
+ }
+
+ void stackCheck(ImmWord limitAddr, Label* label) { MOZ_CRASH("stackCheck"); }
+
+ void incrementInt32Value(const Address& addr) {
+ vixl::UseScratchRegisterScope temps(this);
+ const ARMRegister scratch32 = temps.AcquireW();
+ MOZ_ASSERT(scratch32.asUnsized() != addr.base);
+
+ load32(addr, scratch32.asUnsized());
+ Add(scratch32, scratch32, Operand(1));
+ store32(scratch32.asUnsized(), addr);
+ }
+
+ void breakpoint();
+
+ // Emits a simulator directive to save the current sp on an internal stack.
+ void simulatorMarkSP() {
+#ifdef JS_SIMULATOR_ARM64
+ svc(vixl::kMarkStackPointer);
+#endif
+ }
+
+ // Emits a simulator directive to pop from its internal stack
+ // and assert that the value is equal to the current sp.
+ void simulatorCheckSP() {
+#ifdef JS_SIMULATOR_ARM64
+ svc(vixl::kCheckStackPointer);
+#endif
+ }
+
+ // Overwrites the payload bits of a dest register containing a Value.
+ void movePayload(Register src, Register dest) {
+ // Bfxil cannot be used with the zero register as a source.
+ if (src == rzr) {
+ And(ARMRegister(dest, 64), ARMRegister(dest, 64),
+ Operand(JS::detail::ValueTagMask));
+ } else {
+ Bfxil(ARMRegister(dest, 64), ARMRegister(src, 64), 0, JSVAL_TAG_SHIFT);
+ }
+ }
+
+ protected:
+ bool buildOOLFakeExitFrame(void* fakeReturnAddr);
+};
+
+// See documentation for ScratchTagScope and ScratchTagScopeRelease in
+// MacroAssembler-x64.h.
+
+class ScratchTagScope {
+ vixl::UseScratchRegisterScope temps_;
+ ARMRegister scratch64_;
+ bool owned_;
+ mozilla::DebugOnly<bool> released_;
+
+ public:
+ ScratchTagScope(MacroAssemblerCompat& masm, const ValueOperand&)
+ : temps_(&masm), owned_(true), released_(false) {
+ scratch64_ = temps_.AcquireX();
+ }
+
+ operator Register() {
+ MOZ_ASSERT(!released_);
+ return scratch64_.asUnsized();
+ }
+
+ void release() {
+ MOZ_ASSERT(!released_);
+ released_ = true;
+ if (owned_) {
+ temps_.Release(scratch64_);
+ owned_ = false;
+ }
+ }
+
+ void reacquire() {
+ MOZ_ASSERT(released_);
+ released_ = false;
+ }
+};
+
+class ScratchTagScopeRelease {
+ ScratchTagScope* ts_;
+
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
+ ts_->release();
+ }
+ ~ScratchTagScopeRelease() { ts_->reacquire(); }
+};
+
+inline void MacroAssemblerCompat::splitTagForTest(const ValueOperand& value,
+ ScratchTagScope& tag) {
+ splitSignExtTag(value, tag);
+}
+
+typedef MacroAssemblerCompat MacroAssemblerSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_arm64_MacroAssembler_arm64_h
diff --git a/js/src/jit/arm64/MoveEmitter-arm64.cpp b/js/src/jit/arm64/MoveEmitter-arm64.cpp
new file mode 100644
index 0000000000..fa1bb1209e
--- /dev/null
+++ b/js/src/jit/arm64/MoveEmitter-arm64.cpp
@@ -0,0 +1,329 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm64/MoveEmitter-arm64.h"
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+MemOperand MoveEmitterARM64::toMemOperand(const MoveOperand& operand) const {
+ MOZ_ASSERT(operand.isMemory());
+ ARMRegister base(operand.base(), 64);
+ if (operand.base() == masm.getStackPointer()) {
+ return MemOperand(base,
+ operand.disp() + (masm.framePushed() - pushedAtStart_));
+ }
+ return MemOperand(base, operand.disp());
+}
+
+void MoveEmitterARM64::emit(const MoveResolver& moves) {
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ // We have two scratch general registers, so use one as temporary storage for
+ // breaking cycles and leave the other available for memory to memory moves.
+ //
+ // This register is used when breaking GENERAL, INT32, FLOAT32, and DOUBLE
+ // move cycles. For FLOAT32/DOUBLE, this involves a fmov between float and
+ // general registers. We could avoid this if we had an extra scratch float
+ // register, otherwise we need the scratch float register for memory to
+ // memory moves that may happen in the cycle. We cannot use the scratch
+ // general register for SIMD128 cycles as it is not large enough.
+ cycleGeneralReg_ = temps.AcquireX();
+
+ for (size_t i = 0; i < moves.numMoves(); i++) {
+ emitMove(moves.getMove(i));
+ }
+
+ cycleGeneralReg_ = ARMRegister();
+}
+
+void MoveEmitterARM64::finish() {
+ assertDone();
+ masm.freeStack(masm.framePushed() - pushedAtStart_);
+ MOZ_ASSERT(masm.framePushed() == pushedAtStart_);
+}
+
+void MoveEmitterARM64::emitMove(const MoveOp& move) {
+ const MoveOperand& from = move.from();
+ const MoveOperand& to = move.to();
+
+ if (move.isCycleBegin()) {
+ MOZ_ASSERT(!inCycle_ && !move.isCycleEnd());
+ breakCycle(from, to, move.endCycleType());
+ inCycle_ = true;
+ } else if (move.isCycleEnd()) {
+ MOZ_ASSERT(inCycle_);
+ completeCycle(from, to, move.type());
+ inCycle_ = false;
+ return;
+ }
+
+ switch (move.type()) {
+ case MoveOp::FLOAT32:
+ emitFloat32Move(from, to);
+ break;
+ case MoveOp::DOUBLE:
+ emitDoubleMove(from, to);
+ break;
+ case MoveOp::SIMD128:
+ emitSimd128Move(from, to);
+ break;
+ case MoveOp::INT32:
+ emitInt32Move(from, to);
+ break;
+ case MoveOp::GENERAL:
+ emitGeneralMove(from, to);
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterARM64::emitFloat32Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.Fmov(toFPReg(to, MoveOp::FLOAT32), toFPReg(from, MoveOp::FLOAT32));
+ } else {
+ masm.Str(toFPReg(from, MoveOp::FLOAT32), toMemOperand(to));
+ }
+ return;
+ }
+
+ if (to.isFloatReg()) {
+ masm.Ldr(toFPReg(to, MoveOp::FLOAT32), toMemOperand(from));
+ return;
+ }
+
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMFPRegister scratch32 = temps.AcquireS();
+ masm.Ldr(scratch32, toMemOperand(from));
+ masm.Str(scratch32, toMemOperand(to));
+}
+
+void MoveEmitterARM64::emitDoubleMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.Fmov(toFPReg(to, MoveOp::DOUBLE), toFPReg(from, MoveOp::DOUBLE));
+ } else {
+ masm.Str(toFPReg(from, MoveOp::DOUBLE), toMemOperand(to));
+ }
+ return;
+ }
+
+ if (to.isFloatReg()) {
+ masm.Ldr(toFPReg(to, MoveOp::DOUBLE), toMemOperand(from));
+ return;
+ }
+
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMFPRegister scratch = temps.AcquireD();
+ masm.Ldr(scratch, toMemOperand(from));
+ masm.Str(scratch, toMemOperand(to));
+}
+
+void MoveEmitterARM64::emitSimd128Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.Mov(toFPReg(to, MoveOp::SIMD128), toFPReg(from, MoveOp::SIMD128));
+ } else {
+ masm.Str(toFPReg(from, MoveOp::SIMD128), toMemOperand(to));
+ }
+ return;
+ }
+
+ if (to.isFloatReg()) {
+ masm.Ldr(toFPReg(to, MoveOp::SIMD128), toMemOperand(from));
+ return;
+ }
+
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMFPRegister scratch = temps.AcquireQ();
+ masm.Ldr(scratch, toMemOperand(from));
+ masm.Str(scratch, toMemOperand(to));
+}
+
+void MoveEmitterARM64::emitInt32Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isGeneralReg()) {
+ if (to.isGeneralReg()) {
+ masm.Mov(toARMReg32(to), toARMReg32(from));
+ } else {
+ masm.Str(toARMReg32(from), toMemOperand(to));
+ }
+ return;
+ }
+
+ if (to.isGeneralReg()) {
+ masm.Ldr(toARMReg32(to), toMemOperand(from));
+ return;
+ }
+
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMRegister scratch32 = temps.AcquireW();
+ masm.Ldr(scratch32, toMemOperand(from));
+ masm.Str(scratch32, toMemOperand(to));
+}
+
+void MoveEmitterARM64::emitGeneralMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isGeneralReg()) {
+ MOZ_ASSERT(to.isGeneralReg() || to.isMemory());
+ if (to.isGeneralReg()) {
+ masm.Mov(toARMReg64(to), toARMReg64(from));
+ } else {
+ masm.Str(toARMReg64(from), toMemOperand(to));
+ }
+ return;
+ }
+
+ // {Memory OR EffectiveAddress} -> Register move.
+ if (to.isGeneralReg()) {
+ MOZ_ASSERT(from.isMemoryOrEffectiveAddress());
+ if (from.isMemory()) {
+ masm.Ldr(toARMReg64(to), toMemOperand(from));
+ } else {
+ masm.Add(toARMReg64(to), toARMReg64(from), Operand(from.disp()));
+ }
+ return;
+ }
+
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMRegister scratch64 = temps.AcquireX();
+
+ // Memory -> Memory move.
+ if (from.isMemory()) {
+ MOZ_ASSERT(to.isMemory());
+ masm.Ldr(scratch64, toMemOperand(from));
+ masm.Str(scratch64, toMemOperand(to));
+ return;
+ }
+
+ // EffectiveAddress -> Memory move.
+ MOZ_ASSERT(from.isEffectiveAddress());
+ MOZ_ASSERT(to.isMemory());
+ masm.Add(scratch64, toARMReg64(from), Operand(from.disp()));
+ masm.Str(scratch64, toMemOperand(to));
+}
+
+MemOperand MoveEmitterARM64::cycleSlot() {
+ // Using SP as stack pointer requires alignment preservation below.
+ MOZ_ASSERT(!masm.GetStackPointer64().Is(sp));
+
+ // Allocate a slot for breaking cycles if we have not already
+ if (pushedAtCycle_ == -1) {
+ static_assert(SpillSlotSize == 16);
+ masm.reserveStack(SpillSlotSize);
+ pushedAtCycle_ = masm.framePushed();
+ }
+
+ return MemOperand(masm.GetStackPointer64(),
+ masm.framePushed() - pushedAtCycle_);
+}
+
+void MoveEmitterARM64::breakCycle(const MoveOperand& from,
+ const MoveOperand& to, MoveOp::Type type) {
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ masm.Ldr(cycleGeneralReg_.W(), toMemOperand(to));
+ } else {
+ masm.Fmov(cycleGeneralReg_.W(), toFPReg(to, type));
+ }
+ break;
+
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ masm.Ldr(cycleGeneralReg_.X(), toMemOperand(to));
+ } else {
+ masm.Fmov(cycleGeneralReg_.X(), toFPReg(to, type));
+ }
+ break;
+
+ case MoveOp::SIMD128:
+ if (to.isMemory()) {
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMFPRegister scratch128 = temps.AcquireQ();
+ masm.Ldr(scratch128, toMemOperand(to));
+ masm.Str(scratch128, cycleSlot());
+ } else {
+ masm.Str(toFPReg(to, type), cycleSlot());
+ }
+ break;
+
+ case MoveOp::INT32:
+ if (to.isMemory()) {
+ masm.Ldr(cycleGeneralReg_.W(), toMemOperand(to));
+ } else {
+ masm.Mov(cycleGeneralReg_.W(), toARMReg32(to));
+ }
+ break;
+
+ case MoveOp::GENERAL:
+ if (to.isMemory()) {
+ masm.Ldr(cycleGeneralReg_.X(), toMemOperand(to));
+ } else {
+ masm.Mov(cycleGeneralReg_.X(), toARMReg64(to));
+ }
+ break;
+
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterARM64::completeCycle(const MoveOperand& from,
+ const MoveOperand& to, MoveOp::Type type) {
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ masm.Str(cycleGeneralReg_.W(), toMemOperand(to));
+ } else {
+ masm.Fmov(toFPReg(to, type), cycleGeneralReg_.W());
+ }
+ break;
+
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ masm.Str(cycleGeneralReg_.X(), toMemOperand(to));
+ } else {
+ masm.Fmov(toFPReg(to, type), cycleGeneralReg_.X());
+ }
+ break;
+
+ case MoveOp::SIMD128:
+ if (to.isMemory()) {
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMFPRegister scratch = temps.AcquireQ();
+ masm.Ldr(scratch, cycleSlot());
+ masm.Str(scratch, toMemOperand(to));
+ } else {
+ masm.Ldr(toFPReg(to, type), cycleSlot());
+ }
+ break;
+
+ case MoveOp::INT32:
+ if (to.isMemory()) {
+ masm.Str(cycleGeneralReg_.W(), toMemOperand(to));
+ } else {
+ masm.Mov(toARMReg32(to), cycleGeneralReg_.W());
+ }
+ break;
+
+ case MoveOp::GENERAL:
+ if (to.isMemory()) {
+ masm.Str(cycleGeneralReg_.X(), toMemOperand(to));
+ } else {
+ masm.Mov(toARMReg64(to), cycleGeneralReg_.X());
+ }
+ break;
+
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
diff --git a/js/src/jit/arm64/MoveEmitter-arm64.h b/js/src/jit/arm64/MoveEmitter-arm64.h
new file mode 100644
index 0000000000..fec2e3e012
--- /dev/null
+++ b/js/src/jit/arm64/MoveEmitter-arm64.h
@@ -0,0 +1,99 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_MoveEmitter_arm64_h
+#define jit_arm64_MoveEmitter_arm64_h
+
+#include "mozilla/Assertions.h"
+
+#include <stdint.h>
+
+#include "jit/arm64/Assembler-arm64.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MoveResolver.h"
+#include "jit/Registers.h"
+
+namespace js {
+namespace jit {
+
+class CodeGenerator;
+
+class MoveEmitterARM64 {
+ bool inCycle_;
+ MacroAssembler& masm;
+
+ // A scratch general register used to break cycles.
+ ARMRegister cycleGeneralReg_;
+
+ // Original stack push value.
+ uint32_t pushedAtStart_;
+
+ // This stores a stack offset to a spill location, snapshotting
+ // codegen->framePushed_ at the time it was allocated. It is -1 if no
+ // stack space has been allocated for that particular spill.
+ int32_t pushedAtCycle_;
+
+ void assertDone() { MOZ_ASSERT(!inCycle_); }
+
+ MemOperand cycleSlot();
+ MemOperand toMemOperand(const MoveOperand& operand) const;
+ ARMRegister toARMReg32(const MoveOperand& operand) const {
+ MOZ_ASSERT(operand.isGeneralReg());
+ return ARMRegister(operand.reg(), 32);
+ }
+ ARMRegister toARMReg64(const MoveOperand& operand) const {
+ if (operand.isGeneralReg()) {
+ return ARMRegister(operand.reg(), 64);
+ } else {
+ return ARMRegister(operand.base(), 64);
+ }
+ }
+ ARMFPRegister toFPReg(const MoveOperand& operand, MoveOp::Type t) const {
+ MOZ_ASSERT(operand.isFloatReg());
+ switch (t) {
+ case MoveOp::FLOAT32:
+ return ARMFPRegister(operand.floatReg().encoding(), 32);
+ case MoveOp::DOUBLE:
+ return ARMFPRegister(operand.floatReg().encoding(), 64);
+ case MoveOp::SIMD128:
+ return ARMFPRegister(operand.floatReg().encoding(), 128);
+ default:
+ MOZ_MAKE_COMPILER_ASSUME_IS_UNREACHABLE("Bad register type");
+ }
+ }
+
+ void emitFloat32Move(const MoveOperand& from, const MoveOperand& to);
+ void emitDoubleMove(const MoveOperand& from, const MoveOperand& to);
+ void emitSimd128Move(const MoveOperand& from, const MoveOperand& to);
+ void emitInt32Move(const MoveOperand& from, const MoveOperand& to);
+ void emitGeneralMove(const MoveOperand& from, const MoveOperand& to);
+
+ void emitMove(const MoveOp& move);
+ void breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type);
+ void completeCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type);
+
+ public:
+ explicit MoveEmitterARM64(MacroAssembler& masm)
+ : inCycle_(false),
+ masm(masm),
+ pushedAtStart_(masm.framePushed()),
+ pushedAtCycle_(-1) {}
+
+ ~MoveEmitterARM64() { assertDone(); }
+
+ void emit(const MoveResolver& moves);
+ void finish();
+ void setScratchRegister(Register reg) {}
+};
+
+typedef MoveEmitterARM64 MoveEmitter;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_arm64_MoveEmitter_arm64_h */
diff --git a/js/src/jit/arm64/SharedICHelpers-arm64-inl.h b/js/src/jit/arm64/SharedICHelpers-arm64-inl.h
new file mode 100644
index 0000000000..8261a8b94f
--- /dev/null
+++ b/js/src/jit/arm64/SharedICHelpers-arm64-inl.h
@@ -0,0 +1,79 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_SharedICHelpers_arm64_inl_h
+#define jit_arm64_SharedICHelpers_arm64_inl_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+ uint32_t argSize) {
+#ifdef DEBUG
+ // We assume that R0 has been pushed, and R2 is unused.
+ static_assert(R2 == ValueOperand(r0));
+
+ // Store frame size without VMFunction arguments for debug assertions.
+ masm.Sub(x0, FramePointer64, masm.GetStackPointer64());
+ masm.Sub(w0, w0, Operand(argSize));
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(w0.asUnsized(), frameSizeAddr);
+#endif
+
+ // Push frame descriptor (minus the return address) and perform the tail call.
+ static_assert(ICTailCallReg == lr);
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+
+ // The return address will be pushed by the VM wrapper, for compatibility
+ // with direct calls. Refer to the top of generateVMWrapper().
+ // ICTailCallReg (lr) already contains the return address (as we keep
+ // it there through the stub calls).
+
+ masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) {
+ MOZ_ASSERT(scratch != ICTailCallReg);
+
+#ifdef DEBUG
+ // Compute frame size.
+ masm.Sub(ARMRegister(scratch, 64), FramePointer64, masm.GetStackPointer64());
+
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Push frame descriptor and return address.
+ // Save old frame pointer, stack pointer, and stub reg.
+ masm.PushFrameDescriptor(FrameType::BaselineJS);
+ masm.Push(ICTailCallReg);
+ masm.Push(FramePointer);
+
+ // Update the frame register.
+ masm.Mov(FramePointer64, masm.GetStackPointer64());
+
+ masm.Push(ICStubReg);
+
+ // Stack should remain 16-byte aligned.
+ masm.checkStackAlignment();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_arm64_SharedICHelpers_arm64_inl_h
diff --git a/js/src/jit/arm64/SharedICHelpers-arm64.h b/js/src/jit/arm64/SharedICHelpers-arm64.h
new file mode 100644
index 0000000000..2ea45c80fb
--- /dev/null
+++ b/js/src/jit/arm64/SharedICHelpers-arm64.h
@@ -0,0 +1,82 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_SharedICHelpers_arm64_h
+#define jit_arm64_SharedICHelpers_arm64_h
+
+#include "jit/BaselineIC.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from sp to the top Value inside an IC stub (no return address on the
+// stack on ARM).
+static const size_t ICStackValueOffset = 0;
+
+inline void EmitRestoreTailCallReg(MacroAssembler& masm) {
+ // No-op on ARM because link register is always holding the return address.
+}
+
+inline void EmitRepushTailCallReg(MacroAssembler& masm) {
+ // No-op on ARM because link register is always holding the return address.
+}
+
+inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
+ // The stub pointer must already be in ICStubReg.
+ // Load stubcode pointer from the ICStub.
+ // R2 won't be active when we call ICs, so we can use r0.
+ static_assert(R2 == ValueOperand(r0));
+ masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), r0);
+
+ // Call the stubcode via a direct branch-and-link.
+ masm.Blr(x0);
+ *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void EmitReturnFromIC(MacroAssembler& masm) {
+ masm.abiret(); // Defaults to lr.
+}
+
+inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ const ARMRegister scratch64 = temps.AcquireX();
+
+ Address stubAddr(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP);
+ masm.loadPtr(stubAddr, ICStubReg);
+
+ masm.moveToStackPtr(FramePointer);
+
+ // Pop values, discarding the frame descriptor.
+ masm.pop(FramePointer, ICTailCallReg, scratch64.asUnsized());
+
+ // Stack should remain 16-byte aligned.
+ masm.checkStackAlignment();
+}
+
+template <typename AddrType>
+inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr,
+ MIRType type) {
+ // On AArch64, lr is clobbered by guardedCallPreBarrier. Save it first.
+ masm.push(lr);
+ masm.guardedCallPreBarrier(addr, type);
+ masm.pop(lr);
+}
+
+inline void EmitStubGuardFailure(MacroAssembler& masm) {
+ // Load next stub into ICStubReg.
+ masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+ // Return address is already loaded, just jump to the next stubcode.
+ masm.jump(Address(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_arm64_SharedICHelpers_arm64_h
diff --git a/js/src/jit/arm64/SharedICRegisters-arm64.h b/js/src/jit/arm64/SharedICRegisters-arm64.h
new file mode 100644
index 0000000000..1aa49d651c
--- /dev/null
+++ b/js/src/jit/arm64/SharedICRegisters-arm64.h
@@ -0,0 +1,51 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_arm64_SharedICRegisters_arm64_h
+#define jit_arm64_SharedICRegisters_arm64_h
+
+#include "jit/arm64/Assembler-arm64.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+// ValueOperands R0, R1, and R2.
+// R0 == JSReturnReg, and R2 uses registers not preserved across calls.
+// R1 value should be preserved across calls.
+static constexpr Register R0_ = r2;
+static constexpr Register R1_ = r19;
+static constexpr Register R2_ = r0;
+
+static constexpr ValueOperand R0(R0_);
+static constexpr ValueOperand R1(R1_);
+static constexpr ValueOperand R2(R2_);
+
+// ICTailCallReg and ICStubReg use registers that are not preserved across
+// calls.
+static constexpr Register ICTailCallReg = r30;
+static constexpr Register ICStubReg = r9;
+
+// R7 - R9 are generally available for use within stubcode.
+
+// Note that BaselineTailCallReg is actually just the link
+// register. In ARM code emission, we do not clobber BaselineTailCallReg
+// since we keep the return address for calls there.
+
+static constexpr FloatRegister FloatReg0 = {FloatRegisters::d0,
+ FloatRegisters::Double};
+static constexpr FloatRegister FloatReg1 = {FloatRegisters::d1,
+ FloatRegisters::Double};
+static constexpr FloatRegister FloatReg2 = {FloatRegisters::d2,
+ FloatRegisters::Double};
+static constexpr FloatRegister FloatReg3 = {FloatRegisters::d3,
+ FloatRegisters::Double};
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_arm64_SharedICRegisters_arm64_h
diff --git a/js/src/jit/arm64/Trampoline-arm64.cpp b/js/src/jit/arm64/Trampoline-arm64.cpp
new file mode 100644
index 0000000000..99cecdcc1b
--- /dev/null
+++ b/js/src/jit/arm64/Trampoline-arm64.cpp
@@ -0,0 +1,838 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/arm64/SharedICHelpers-arm64.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+/* This method generates a trampoline on ARM64 for a c++ function with
+ * the following signature:
+ * bool blah(void* code, int argc, Value* argv,
+ * JSObject* scopeChain, Value* vp)
+ * ...using standard AArch64 calling convention
+ */
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ const Register reg_code = IntArgReg0; // EnterJitData::jitcode.
+ const Register reg_argc = IntArgReg1; // EnterJitData::maxArgc.
+ const Register reg_argv = IntArgReg2; // EnterJitData::maxArgv.
+ const Register reg_osrFrame = IntArgReg3; // EnterJitData::osrFrame.
+ const Register reg_callee = IntArgReg4; // EnterJitData::calleeToken.
+ const Register reg_scope = IntArgReg5; // EnterJitData::scopeChain.
+ const Register reg_osrNStack =
+ IntArgReg6; // EnterJitData::osrNumStackValues.
+ const Register reg_vp = IntArgReg7; // Address of EnterJitData::result.
+
+ static_assert(OsrFrameReg == IntArgReg3);
+
+ // During the pushes below, use the normal stack pointer.
+ masm.SetStackPointer64(sp);
+
+ // Save return address and old frame pointer; set new frame pointer.
+ masm.push(r30, r29);
+ masm.moveStackPtrTo(r29);
+
+ // Save callee-save integer registers.
+ // Also save x7 (reg_vp) and x30 (lr), for use later.
+ masm.push(r19, r20, r21, r22);
+ masm.push(r23, r24, r25, r26);
+ masm.push(r27, r28, r7, r30);
+
+ // Save callee-save floating-point registers.
+ // AArch64 ABI specifies that only the lower 64 bits must be saved.
+ masm.push(d8, d9, d10, d11);
+ masm.push(d12, d13, d14, d15);
+
+#ifdef DEBUG
+ // Emit stack canaries.
+ masm.movePtr(ImmWord(0xdeadd00d), r23);
+ masm.movePtr(ImmWord(0xdeadd11d), r24);
+ masm.push(r23, r24);
+#endif
+
+ // Common code below attempts to push single registers at a time,
+ // which breaks the stack pointer's 16-byte alignment requirement.
+ // Note that movePtr() is invalid because StackPointer is treated as xzr.
+ //
+ // FIXME: After testing, this entire function should be rewritten to not
+ // use the PseudoStackPointer: since the amount of data pushed is
+ // precalculated, we can just allocate the whole frame header at once and
+ // index off sp. This will save a significant number of instructions where
+ // Push() updates sp.
+ masm.Mov(PseudoStackPointer64, sp);
+ masm.SetStackPointer64(PseudoStackPointer64);
+
+ // Remember stack depth without padding and arguments.
+ masm.moveStackPtrTo(r19);
+
+ // If constructing, include newTarget in argument vector.
+ {
+ Label noNewTarget;
+ Imm32 constructingToken(CalleeToken_FunctionConstructing);
+ masm.branchTest32(Assembler::Zero, reg_callee, constructingToken,
+ &noNewTarget);
+ masm.add32(Imm32(1), reg_argc);
+ masm.bind(&noNewTarget);
+ }
+
+ // JitFrameLayout is as follows (higher is higher in memory):
+ // N*8 - [ JS argument vector ] (base 16-byte aligned)
+ // 8 - calleeToken
+ // 8 - frameDescriptor (16-byte aligned)
+ // 8 - returnAddress
+ // 8 - frame pointer (16-byte aligned, pushed by callee)
+
+ // Touch frame incrementally (a requirement for Windows).
+ //
+ // Use already saved callee-save registers r20 and r21 as temps.
+ //
+ // This has to be done outside the ScratchRegisterScope, as the temps are
+ // under demand inside the touchFrameValues call.
+
+ // Give sp 16-byte alignment and sync stack pointers.
+ masm.andToStackPtr(Imm32(~0xf));
+ // We needn't worry about the Gecko Profiler mark because touchFrameValues
+ // touches in large increments.
+ masm.touchFrameValues(reg_argc, r20, r21);
+ // Restore stack pointer, preserved above.
+ masm.moveToStackPtr(r19);
+
+ // Push the argument vector onto the stack.
+ // WARNING: destructively modifies reg_argv
+ {
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+
+ const ARMRegister tmp_argc = temps.AcquireX();
+ const ARMRegister tmp_sp = temps.AcquireX();
+
+ Label noArguments;
+ Label loopHead;
+
+ masm.movePtr(reg_argc, tmp_argc.asUnsized());
+
+ // sp -= 8
+ // Since we're using PostIndex Str below, this is necessary to avoid
+ // overwriting the Gecko Profiler mark pushed above.
+ masm.subFromStackPtr(Imm32(8));
+
+ // sp -= 8 * argc
+ masm.Sub(PseudoStackPointer64, PseudoStackPointer64,
+ Operand(tmp_argc, vixl::SXTX, 3));
+
+ // Give sp 16-byte alignment and sync stack pointers.
+ masm.andToStackPtr(Imm32(~0xf));
+ masm.moveStackPtrTo(tmp_sp.asUnsized());
+
+ masm.branchTestPtr(Assembler::Zero, reg_argc, reg_argc, &noArguments);
+
+ // Begin argument-pushing loop.
+ // This could be optimized using Ldp and Stp.
+ {
+ masm.bind(&loopHead);
+
+ // Load an argument from argv, then increment argv by 8.
+ masm.Ldr(x24, MemOperand(ARMRegister(reg_argv, 64), Operand(8),
+ vixl::PostIndex));
+
+ // Store the argument to tmp_sp, then increment tmp_sp by 8.
+ masm.Str(x24, MemOperand(tmp_sp, Operand(8), vixl::PostIndex));
+
+ // Decrement tmp_argc and set the condition codes for the new value.
+ masm.Subs(tmp_argc, tmp_argc, Operand(1));
+
+ // Branch if arguments remain.
+ masm.B(&loopHead, vixl::Condition::NonZero);
+ }
+
+ masm.bind(&noArguments);
+ }
+ masm.checkStackAlignment();
+
+ // Push the calleeToken and the frame descriptor.
+ // The result address is used to store the actual number of arguments
+ // without adding an argument to EnterJIT.
+ {
+ vixl::UseScratchRegisterScope temps(&masm.asVIXL());
+ MOZ_ASSERT(temps.IsAvailable(ScratchReg64)); // ip0
+ temps.Exclude(ScratchReg64);
+ Register scratch = ScratchReg64.asUnsized();
+ masm.push(reg_callee);
+
+ // Push the descriptor.
+ masm.unboxInt32(Address(reg_vp, 0x0), scratch);
+ masm.PushFrameDescriptorForJitCall(FrameType::CppToJSJit, scratch, scratch);
+ }
+ masm.checkStackAlignment();
+
+ Label osrReturnPoint;
+ {
+ // Check for Interpreter -> Baseline OSR.
+
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.take(OsrFrameReg);
+ regs.take(reg_code);
+ regs.take(reg_osrNStack);
+ MOZ_ASSERT(!regs.has(ReturnReg), "ReturnReg matches reg_code");
+
+ Label notOsr;
+ masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, &notOsr);
+
+ Register scratch = regs.takeAny();
+
+ // Frame prologue.
+ masm.Adr(ARMRegister(scratch, 64), &osrReturnPoint);
+ masm.push(scratch, FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ // Reserve frame.
+ masm.subFromStackPtr(Imm32(BaselineFrame::Size()));
+
+ Register framePtrScratch = regs.takeAny();
+ masm.touchFrameValues(reg_osrNStack, scratch, framePtrScratch);
+ masm.moveStackPtrTo(framePtrScratch);
+
+ // Reserve space for locals and stack values.
+ // scratch = num_stack_values * sizeof(Value).
+ masm.Lsl(ARMRegister(scratch, 32), ARMRegister(reg_osrNStack, 32), 3);
+ masm.subFromStackPtr(scratch);
+
+ // Enter exit frame.
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(xzr); // Push xzr for a fake return address.
+ masm.push(FramePointer);
+ // No GC things to mark: push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ masm.push(reg_code);
+
+ // Initialize the frame, including filling in the slots.
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(r19);
+ masm.passABIArg(framePtrScratch); // BaselineFrame.
+ masm.passABIArg(reg_osrFrame); // InterpreterFrame.
+ masm.passABIArg(reg_osrNStack);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ masm.pop(scratch);
+ MOZ_ASSERT(scratch != ReturnReg);
+
+ masm.addToStackPtr(Imm32(ExitFrameLayout::SizeWithFooter()));
+
+ Label error;
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.profilerEnterFrame(FramePointer, regs.getAny());
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+ masm.jump(scratch);
+
+ // OOM: frame epilogue, load error value, discard return address and return.
+ masm.bind(&error);
+ masm.moveToStackPtr(FramePointer);
+ masm.pop(FramePointer);
+ masm.addToStackPtr(Imm32(sizeof(uintptr_t))); // Return address.
+ masm.syncStackPtr();
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.B(&osrReturnPoint);
+
+ masm.bind(&notOsr);
+ masm.movePtr(reg_scope, R1_);
+ }
+
+ // The callee will push the return address and frame pointer on the stack,
+ // thus we check that the stack would be aligned once the call is complete.
+ masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+ // Call function.
+ // Since AArch64 doesn't have the pc register available, the callee must push
+ // lr.
+ masm.callJitNoProfiler(reg_code);
+
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&osrReturnPoint);
+
+ // Discard arguments and padding. Set sp to the address of the saved
+ // registers. In debug builds we have to include the two stack canaries
+ // checked below.
+#ifdef DEBUG
+ static constexpr size_t SavedRegSize = 22 * sizeof(void*);
+#else
+ static constexpr size_t SavedRegSize = 20 * sizeof(void*);
+#endif
+ masm.computeEffectiveAddress(Address(FramePointer, -int32_t(SavedRegSize)),
+ masm.getStackPointer());
+
+ masm.syncStackPtr();
+ masm.SetStackPointer64(sp);
+
+#ifdef DEBUG
+ // Check that canaries placed on function entry are still present.
+ masm.pop(r24, r23);
+ Label x23OK, x24OK;
+
+ masm.branchPtr(Assembler::Equal, r23, ImmWord(0xdeadd00d), &x23OK);
+ masm.breakpoint();
+ masm.bind(&x23OK);
+
+ masm.branchPtr(Assembler::Equal, r24, ImmWord(0xdeadd11d), &x24OK);
+ masm.breakpoint();
+ masm.bind(&x24OK);
+#endif
+
+ // Restore callee-save floating-point registers.
+ masm.pop(d15, d14, d13, d12);
+ masm.pop(d11, d10, d9, d8);
+
+ // Restore callee-save integer registers.
+ // Also restore x7 (reg_vp) and x30 (lr).
+ masm.pop(r30, r7, r28, r27);
+ masm.pop(r26, r25, r24, r23);
+ masm.pop(r22, r21, r20, r19);
+
+ // Store return value (in JSReturnReg = x2 to just-popped reg_vp).
+ masm.storeValue(JSReturnOperand, Address(reg_vp, 0));
+
+ // Restore old frame pointer.
+ masm.pop(r29, r30);
+
+ // Return using the value popped into x30.
+ masm.abiret();
+
+ // Reset stack pointer.
+ masm.SetStackPointer64(PseudoStackPointer64);
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ // Not supported, or not implemented yet.
+ // TODO: Implement along with the corresponding stack-walker changes, in
+ // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
+ return mozilla::Nothing{};
+}
+
+static void PushRegisterDump(MacroAssembler& masm) {
+ const LiveRegisterSet First28GeneralRegisters = LiveRegisterSet(
+ GeneralRegisterSet(Registers::AllMask &
+ ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28)),
+ FloatRegisterSet(FloatRegisters::NoneMask));
+
+ const LiveRegisterSet AllFloatRegisters =
+ LiveRegisterSet(GeneralRegisterSet(Registers::NoneMask),
+ FloatRegisterSet(FloatRegisters::AllMask));
+
+ // Push all general-purpose registers.
+ //
+ // The ARM64 ABI does not treat SP as a normal register that can
+ // be pushed. So pushing happens in two phases.
+ //
+ // Registers are pushed in reverse order of code.
+ //
+ // See block comment in MacroAssembler.h for further required invariants.
+
+ // First, push the last four registers, passing zero for sp.
+ // Zero is pushed for x28 and x31: the pseudo-SP and SP, respectively.
+ masm.asVIXL().Push(xzr, x30, x29, xzr);
+
+ // Second, push the first 28 registers that serve no special purpose.
+ masm.PushRegsInMask(First28GeneralRegisters);
+
+ // Finally, push all floating-point registers, completing the RegisterDump.
+ masm.PushRegsInMask(AllFloatRegisters);
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // The InvalidationBailoutStack saved in r0 must be:
+ // - osiPointReturnAddress_
+ // - ionScript_ (pushed by CodeGeneratorARM64::generateInvalidateEpilogue())
+ // - regs_ (pushed here)
+ // - fpregs_ (pushed here) [=r0]
+ PushRegisterDump(masm);
+ masm.moveStackPtrTo(r0);
+
+ // Reserve space for InvalidationBailout's bailoutInfo outparam.
+ masm.Sub(x1, masm.GetStackPointer64(), Operand(sizeof(void*)));
+ masm.moveToStackPtr(r1);
+
+ using Fn =
+ bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(r10);
+ masm.passABIArg(r0);
+ masm.passABIArg(r1);
+
+ masm.callWithABI<Fn, InvalidationBailout>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(r2); // Get the bailoutInfo outparam.
+
+ // Pop the machine state and the dead frame.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+
+ // Save the return address for later.
+ masm.push(lr);
+
+ // Frame prologue.
+ //
+ // NOTE: if this changes, fix the Baseline bailout code too!
+ // See BaselineStackBuilder::calculatePrevFramePtr and
+ // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+ masm.push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ // Load the information that the rectifier needs from the stack.
+ masm.loadNumActualArgs(FramePointer, r0);
+ masm.loadPtr(
+ Address(FramePointer, RectifierFrameLayout::offsetOfCalleeToken()), r1);
+
+ // Extract a JSFunction pointer from the callee token and keep the
+ // intermediary to avoid later recalculation.
+ masm.And(x5, x1, Operand(CalleeTokenMask));
+
+ // Get the arguments from the function object.
+ masm.loadFunctionArgCount(x5.asUnsized(), x6.asUnsized());
+
+ static_assert(CalleeToken_FunctionConstructing == 0x1,
+ "Constructing must be low-order bit");
+ masm.And(x4, x1, Operand(CalleeToken_FunctionConstructing));
+ masm.Add(x7, x6, x4);
+
+ // Copy the number of actual arguments into r8.
+ masm.mov(r0, r8);
+
+ // Calculate the position that our arguments are at before sp gets modified.
+ masm.Add(x3, masm.GetStackPointer64(), Operand(x8, vixl::LSL, 3));
+ masm.Add(x3, x3, Operand(sizeof(RectifierFrameLayout)));
+
+ // If the number of Values without |this| is even, push 8 padding bytes to
+ // ensure the stack is 16-byte aligned.
+ Label noPadding;
+ masm.Tbnz(x7, 0, &noPadding);
+ masm.asVIXL().Push(xzr);
+ masm.bind(&noPadding);
+
+ {
+ Label notConstructing;
+ masm.Cbz(x4, &notConstructing);
+
+ // new.target lives at the end of the pushed args
+ // NB: The arg vector holder starts at the beginning of the last arg,
+ // add a value to get to argv[argc]
+ masm.loadPtr(Address(r3, sizeof(Value)), r4);
+ masm.Push(r4);
+
+ masm.bind(&notConstructing);
+ }
+
+ // Calculate the number of undefineds that need to be pushed.
+ masm.Sub(w2, w6, w8);
+
+ // Put an undefined in a register so it can be pushed.
+ masm.moveValue(UndefinedValue(), ValueOperand(r4));
+
+ // Push undefined N times.
+ {
+ Label undefLoopTop;
+ masm.bind(&undefLoopTop);
+ masm.Push(r4);
+ masm.Subs(w2, w2, Operand(1));
+ masm.B(&undefLoopTop, Assembler::NonZero);
+ }
+
+ // Arguments copy loop. Copy for x8 >= 0 to include |this|.
+ {
+ Label copyLoopTop;
+ masm.bind(&copyLoopTop);
+ masm.Ldr(x4, MemOperand(x3, -sizeof(Value), vixl::PostIndex));
+ masm.Push(r4);
+ masm.Subs(x8, x8, Operand(1));
+ masm.B(&copyLoopTop, Assembler::NotSigned);
+ }
+
+ masm.push(r1); // Callee token.
+ masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, r0, r0);
+
+ // Call the target function.
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(r5, r3);
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(r3);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(r5, r3, &noBaselineScript);
+ masm.callJitNoProfiler(r3);
+ masm.jump(&done);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(r5, r3);
+ masm.callJitNoProfiler(r3);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.moveToStackPtr(FramePointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+ // This assumes no SIMD registers, as JS does not support SIMD.
+
+ // The stack saved in spArg must be (higher entries have higher memory
+ // addresses):
+ // - snapshotOffset_
+ // - frameSize_
+ // - regs_
+ // - fpregs_ (spArg + 0)
+ PushRegisterDump(masm);
+ masm.moveStackPtrTo(spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+ PushBailoutFrame(masm, r0);
+
+ // SP % 8 == 4
+ // STEP 1c: Call the bailout function, giving a pointer to the
+ // structure we just blitted onto the stack.
+ // Make space for the BaselineBailoutInfo* outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.moveStackPtrTo(r1);
+
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(r2);
+ masm.passABIArg(r0);
+ masm.passABIArg(r1);
+ masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ // Get the bailoutInfo outparam.
+ masm.pop(r2);
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+ *wrapperOffset = startTrampolineCode(masm);
+
+ // Avoid conflicts with argument registers while discarding the result after
+ // the function call.
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert(
+ (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set must be a superset of the Volatile register set.");
+
+ // Unlike on other platforms, it is the responsibility of the VM *callee* to
+ // push the return address, while the caller must ensure that the address
+ // is stored in lr on entry. This allows the VM wrapper to work with both
+ // direct calls and tail calls.
+ masm.push(lr);
+
+ // First argument is the JSContext.
+ Register reg_cx = IntArgReg0;
+ regs.take(reg_cx);
+
+ // Stack is:
+ // ... frame ...
+ // +12 [args]
+ // +8 descriptor
+ // +0 returnAddress (pushed by this function, caller sets as lr)
+ //
+ // Push the frame pointer to finish the exit frame, then link it up.
+ masm.Push(FramePointer);
+ masm.loadJSContext(reg_cx);
+ masm.enterExitFrame(reg_cx, regs.getAny(), &f);
+
+ // Save the current stack pointer as the base for copying arguments.
+ Register argsBase = InvalidReg;
+ if (f.explicitArgs) {
+ // argsBase can't be an argument register. Bad things would happen if
+ // the MoveResolver didn't throw an assertion failure first.
+ argsBase = r8;
+ regs.take(argsBase);
+ masm.Add(ARMRegister(argsBase, 64), masm.GetStackPointer64(),
+ Operand(ExitFrameLayout::SizeWithFooter()));
+ }
+
+ // Reserve space for any outparameter.
+ Register outReg = InvalidReg;
+ switch (f.outParam) {
+ case Type_Value:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(Value));
+ masm.moveStackPtrTo(outReg);
+ break;
+
+ case Type_Handle:
+ outReg = regs.takeAny();
+ masm.PushEmptyRooted(f.outParamRootType);
+ masm.moveStackPtrTo(outReg);
+ break;
+
+ case Type_Int32:
+ case Type_Bool:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(int64_t));
+ masm.moveStackPtrTo(outReg);
+ break;
+
+ case Type_Double:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(double));
+ masm.moveStackPtrTo(outReg);
+ break;
+
+ case Type_Pointer:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(uintptr_t));
+ masm.moveStackPtrTo(outReg);
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ masm.setupUnalignedABICall(regs.getAny());
+ masm.passABIArg(reg_cx);
+
+ size_t argDisp = 0;
+
+ // Copy arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ masm.passABIArg(MoveOperand(argsBase, argDisp),
+ (f.argPassedInFloatReg(explicitArg) ? MoveOp::DOUBLE
+ : MoveOp::GENERAL));
+ argDisp += sizeof(void*);
+ break;
+
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+
+ case VMFunctionData::DoubleByValue:
+ case VMFunctionData::DoubleByRef:
+ MOZ_CRASH("NYI: AArch64 callVM should not be used with 128bit values.");
+ }
+ }
+
+ // Copy the semi-implicit outparam, if any.
+ // It is not a C++-abi outparam, which would get passed in the
+ // outparam register, but a real parameter to the function, which
+ // was stack-allocated above.
+ if (outReg != InvalidReg) {
+ masm.passABIArg(outReg);
+ }
+
+ masm.callWithABI(nativeFun, MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // SP is used to transfer stack across call boundaries.
+ masm.initPseudoStackPtr();
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, r0, r0, masm.failureLabel());
+ break;
+ case Type_Bool:
+ masm.branchIfFalseBool(r0, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ // Load the outparam and free any allocated stack.
+ switch (f.outParam) {
+ case Type_Value:
+ masm.Ldr(ARMRegister(JSReturnReg, 64),
+ MemOperand(masm.GetStackPointer64()));
+ masm.freeStack(sizeof(Value));
+ break;
+
+ case Type_Handle:
+ masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+ break;
+
+ case Type_Int32:
+ masm.Ldr(ARMRegister(ReturnReg, 32),
+ MemOperand(masm.GetStackPointer64()));
+ masm.freeStack(sizeof(int64_t));
+ break;
+
+ case Type_Bool:
+ masm.Ldrb(ARMRegister(ReturnReg, 32),
+ MemOperand(masm.GetStackPointer64()));
+ masm.freeStack(sizeof(int64_t));
+ break;
+
+ case Type_Double:
+ masm.Ldr(ARMFPRegister(ReturnDoubleReg, 64),
+ MemOperand(masm.GetStackPointer64()));
+ masm.freeStack(sizeof(double));
+ break;
+
+ case Type_Pointer:
+ masm.Ldr(ARMRegister(ReturnReg, 64),
+ MemOperand(masm.GetStackPointer64()));
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ // Until C++ code is instrumented against Spectre, prevent speculative
+ // execution from returning any private data.
+ if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
+ masm.speculationBarrier();
+ }
+
+ // Pop ExitFooterFrame and the frame pointer.
+ masm.leaveExitFrame(sizeof(void*));
+
+ // Return. Subtract sizeof(void*) for the frame pointer.
+ masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+ f.explicitStackSlots() * sizeof(void*) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type) {
+ AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+ uint32_t offset = startTrampolineCode(masm);
+
+ static_assert(PreBarrierReg == r1);
+ Register temp1 = r2;
+ Register temp2 = r3;
+ Register temp3 = r4;
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+ &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ LiveRegisterSet regs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileMask));
+
+ // Also preserve the return address.
+ regs.add(lr);
+
+ masm.PushRegsInMask(regs);
+
+ masm.movePtr(ImmPtr(cx->runtime()), r3);
+
+ masm.setupUnalignedABICall(r0);
+ masm.passABIArg(r3);
+ masm.passABIArg(PreBarrierReg);
+ masm.callWithABI(JitPreWriteBarrier(type));
+
+ // Pop the volatile regs and restore LR.
+ masm.PopRegsInMask(regs);
+ masm.abiret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.abiret();
+
+ return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+ masm.bind(bailoutTail);
+ masm.generateBailoutTail(r1, r2);
+}
diff --git a/js/src/jit/arm64/vixl/.clang-format b/js/src/jit/arm64/vixl/.clang-format
new file mode 100644
index 0000000000..122a79540d
--- /dev/null
+++ b/js/src/jit/arm64/vixl/.clang-format
@@ -0,0 +1,4 @@
+BasedOnStyle: Chromium
+
+# Ignore all comments because they aren't reflowed properly.
+CommentPragmas: "^"
diff --git a/js/src/jit/arm64/vixl/Assembler-vixl.cpp b/js/src/jit/arm64/vixl/Assembler-vixl.cpp
new file mode 100644
index 0000000000..7f6cd0f2da
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Assembler-vixl.cpp
@@ -0,0 +1,5145 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Assembler-vixl.h"
+
+#include <cmath>
+
+#include "jit/arm64/vixl/MacroAssembler-vixl.h"
+
+namespace vixl {
+
+// CPURegList utilities.
+CPURegister CPURegList::PopLowestIndex() {
+ if (IsEmpty()) {
+ return NoCPUReg;
+ }
+ int index = CountTrailingZeros(list_);
+ VIXL_ASSERT((1ULL << index) & list_);
+ Remove(index);
+ return CPURegister(index, size_, type_);
+}
+
+
+CPURegister CPURegList::PopHighestIndex() {
+ VIXL_ASSERT(IsValid());
+ if (IsEmpty()) {
+ return NoCPUReg;
+ }
+ int index = CountLeadingZeros(list_);
+ index = kRegListSizeInBits - 1 - index;
+ VIXL_ASSERT((1ULL << index) & list_);
+ Remove(index);
+ return CPURegister(index, size_, type_);
+}
+
+
+bool CPURegList::IsValid() const {
+ if ((type_ == CPURegister::kRegister) ||
+ (type_ == CPURegister::kVRegister)) {
+ bool is_valid = true;
+ // Try to create a CPURegister for each element in the list.
+ for (int i = 0; i < kRegListSizeInBits; i++) {
+ if (((list_ >> i) & 1) != 0) {
+ is_valid &= CPURegister(i, size_, type_).IsValid();
+ }
+ }
+ return is_valid;
+ } else if (type_ == CPURegister::kNoRegister) {
+ // We can't use IsEmpty here because that asserts IsValid().
+ return list_ == 0;
+ } else {
+ return false;
+ }
+}
+
+
+void CPURegList::RemoveCalleeSaved() {
+ if (type() == CPURegister::kRegister) {
+ Remove(GetCalleeSaved(RegisterSizeInBits()));
+ } else if (type() == CPURegister::kVRegister) {
+ Remove(GetCalleeSavedV(RegisterSizeInBits()));
+ } else {
+ VIXL_ASSERT(type() == CPURegister::kNoRegister);
+ VIXL_ASSERT(IsEmpty());
+ // The list must already be empty, so do nothing.
+ }
+}
+
+
+CPURegList CPURegList::Union(const CPURegList& list_1,
+ const CPURegList& list_2,
+ const CPURegList& list_3) {
+ return Union(list_1, Union(list_2, list_3));
+}
+
+
+CPURegList CPURegList::Union(const CPURegList& list_1,
+ const CPURegList& list_2,
+ const CPURegList& list_3,
+ const CPURegList& list_4) {
+ return Union(Union(list_1, list_2), Union(list_3, list_4));
+}
+
+
+CPURegList CPURegList::Intersection(const CPURegList& list_1,
+ const CPURegList& list_2,
+ const CPURegList& list_3) {
+ return Intersection(list_1, Intersection(list_2, list_3));
+}
+
+
+CPURegList CPURegList::Intersection(const CPURegList& list_1,
+ const CPURegList& list_2,
+ const CPURegList& list_3,
+ const CPURegList& list_4) {
+ return Intersection(Intersection(list_1, list_2),
+ Intersection(list_3, list_4));
+}
+
+
+CPURegList CPURegList::GetCalleeSaved(unsigned size) {
+ return CPURegList(CPURegister::kRegister, size, 19, 29);
+}
+
+
+CPURegList CPURegList::GetCalleeSavedV(unsigned size) {
+ return CPURegList(CPURegister::kVRegister, size, 8, 15);
+}
+
+
+CPURegList CPURegList::GetCallerSaved(unsigned size) {
+ // Registers x0-x18 and lr (x30) are caller-saved.
+ CPURegList list = CPURegList(CPURegister::kRegister, size, 0, 18);
+ // Do not use lr directly to avoid initialisation order fiasco bugs for users.
+ list.Combine(Register(30, kXRegSize));
+ return list;
+}
+
+
+CPURegList CPURegList::GetCallerSavedV(unsigned size) {
+ // Registers d0-d7 and d16-d31 are caller-saved.
+ CPURegList list = CPURegList(CPURegister::kVRegister, size, 0, 7);
+ list.Combine(CPURegList(CPURegister::kVRegister, size, 16, 31));
+ return list;
+}
+
+
+const CPURegList kCalleeSaved = CPURegList::GetCalleeSaved();
+const CPURegList kCalleeSavedV = CPURegList::GetCalleeSavedV();
+const CPURegList kCallerSaved = CPURegList::GetCallerSaved();
+const CPURegList kCallerSavedV = CPURegList::GetCallerSavedV();
+
+
+// Registers.
+#define WREG(n) w##n,
+const Register Register::wregisters[] = {
+REGISTER_CODE_LIST(WREG)
+};
+#undef WREG
+
+#define XREG(n) x##n,
+const Register Register::xregisters[] = {
+REGISTER_CODE_LIST(XREG)
+};
+#undef XREG
+
+#define BREG(n) b##n,
+const VRegister VRegister::bregisters[] = {
+REGISTER_CODE_LIST(BREG)
+};
+#undef BREG
+
+#define HREG(n) h##n,
+const VRegister VRegister::hregisters[] = {
+REGISTER_CODE_LIST(HREG)
+};
+#undef HREG
+
+#define SREG(n) s##n,
+const VRegister VRegister::sregisters[] = {
+REGISTER_CODE_LIST(SREG)
+};
+#undef SREG
+
+#define DREG(n) d##n,
+const VRegister VRegister::dregisters[] = {
+REGISTER_CODE_LIST(DREG)
+};
+#undef DREG
+
+#define QREG(n) q##n,
+const VRegister VRegister::qregisters[] = {
+REGISTER_CODE_LIST(QREG)
+};
+#undef QREG
+
+#define VREG(n) v##n,
+const VRegister VRegister::vregisters[] = {
+REGISTER_CODE_LIST(VREG)
+};
+#undef VREG
+
+
+const Register& Register::WRegFromCode(unsigned code) {
+ if (code == kSPRegInternalCode) {
+ return wsp;
+ } else {
+ VIXL_ASSERT(code < kNumberOfRegisters);
+ return wregisters[code];
+ }
+}
+
+
+const Register& Register::XRegFromCode(unsigned code) {
+ if (code == kSPRegInternalCode) {
+ return sp;
+ } else {
+ VIXL_ASSERT(code < kNumberOfRegisters);
+ return xregisters[code];
+ }
+}
+
+
+const VRegister& VRegister::BRegFromCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+ return bregisters[code];
+}
+
+
+const VRegister& VRegister::HRegFromCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+ return hregisters[code];
+}
+
+
+const VRegister& VRegister::SRegFromCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+ return sregisters[code];
+}
+
+
+const VRegister& VRegister::DRegFromCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+ return dregisters[code];
+}
+
+
+const VRegister& VRegister::QRegFromCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+ return qregisters[code];
+}
+
+
+const VRegister& VRegister::VRegFromCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+ return vregisters[code];
+}
+
+
+const Register& CPURegister::W() const {
+ VIXL_ASSERT(IsValidRegister());
+ return Register::WRegFromCode(code_);
+}
+
+
+const Register& CPURegister::X() const {
+ VIXL_ASSERT(IsValidRegister());
+ return Register::XRegFromCode(code_);
+}
+
+
+const VRegister& CPURegister::B() const {
+ VIXL_ASSERT(IsValidVRegister());
+ return VRegister::BRegFromCode(code_);
+}
+
+
+const VRegister& CPURegister::H() const {
+ VIXL_ASSERT(IsValidVRegister());
+ return VRegister::HRegFromCode(code_);
+}
+
+
+const VRegister& CPURegister::S() const {
+ VIXL_ASSERT(IsValidVRegister());
+ return VRegister::SRegFromCode(code_);
+}
+
+
+const VRegister& CPURegister::D() const {
+ VIXL_ASSERT(IsValidVRegister());
+ return VRegister::DRegFromCode(code_);
+}
+
+
+const VRegister& CPURegister::Q() const {
+ VIXL_ASSERT(IsValidVRegister());
+ return VRegister::QRegFromCode(code_);
+}
+
+
+const VRegister& CPURegister::V() const {
+ VIXL_ASSERT(IsValidVRegister());
+ return VRegister::VRegFromCode(code_);
+}
+
+
+// Operand.
+Operand::Operand(int64_t immediate)
+ : immediate_(immediate),
+ reg_(NoReg),
+ shift_(NO_SHIFT),
+ extend_(NO_EXTEND),
+ shift_amount_(0) {}
+
+
+Operand::Operand(Register reg, Shift shift, unsigned shift_amount)
+ : reg_(reg),
+ shift_(shift),
+ extend_(NO_EXTEND),
+ shift_amount_(shift_amount) {
+ VIXL_ASSERT(shift != MSL);
+ VIXL_ASSERT(reg.Is64Bits() || (shift_amount < kWRegSize));
+ VIXL_ASSERT(reg.Is32Bits() || (shift_amount < kXRegSize));
+ VIXL_ASSERT(!reg.IsSP());
+}
+
+
+Operand::Operand(Register reg, Extend extend, unsigned shift_amount)
+ : reg_(reg),
+ shift_(NO_SHIFT),
+ extend_(extend),
+ shift_amount_(shift_amount) {
+ VIXL_ASSERT(reg.IsValid());
+ VIXL_ASSERT(shift_amount <= 4);
+ VIXL_ASSERT(!reg.IsSP());
+
+ // Extend modes SXTX and UXTX require a 64-bit register.
+ VIXL_ASSERT(reg.Is64Bits() || ((extend != SXTX) && (extend != UXTX)));
+}
+
+
+bool Operand::IsImmediate() const {
+ return reg_.Is(NoReg);
+}
+
+
+bool Operand::IsShiftedRegister() const {
+ return reg_.IsValid() && (shift_ != NO_SHIFT);
+}
+
+
+bool Operand::IsExtendedRegister() const {
+ return reg_.IsValid() && (extend_ != NO_EXTEND);
+}
+
+
+bool Operand::IsZero() const {
+ if (IsImmediate()) {
+ return immediate() == 0;
+ } else {
+ return reg().IsZero();
+ }
+}
+
+
+Operand Operand::ToExtendedRegister() const {
+ VIXL_ASSERT(IsShiftedRegister());
+ VIXL_ASSERT((shift_ == LSL) && (shift_amount_ <= 4));
+ return Operand(reg_, reg_.Is64Bits() ? UXTX : UXTW, shift_amount_);
+}
+
+
+// MemOperand
+MemOperand::MemOperand(Register base, int64_t offset, AddrMode addrmode)
+ : base_(base), regoffset_(NoReg), offset_(offset), addrmode_(addrmode) {
+ VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
+}
+
+
+MemOperand::MemOperand(Register base,
+ Register regoffset,
+ Extend extend,
+ unsigned shift_amount)
+ : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
+ shift_(NO_SHIFT), extend_(extend), shift_amount_(shift_amount) {
+ VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
+ VIXL_ASSERT(!regoffset.IsSP());
+ VIXL_ASSERT((extend == UXTW) || (extend == SXTW) || (extend == SXTX));
+
+ // SXTX extend mode requires a 64-bit offset register.
+ VIXL_ASSERT(regoffset.Is64Bits() || (extend != SXTX));
+}
+
+
+MemOperand::MemOperand(Register base,
+ Register regoffset,
+ Shift shift,
+ unsigned shift_amount)
+ : base_(base), regoffset_(regoffset), offset_(0), addrmode_(Offset),
+ shift_(shift), extend_(NO_EXTEND), shift_amount_(shift_amount) {
+ VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
+ VIXL_ASSERT(regoffset.Is64Bits() && !regoffset.IsSP());
+ VIXL_ASSERT(shift == LSL);
+}
+
+
+MemOperand::MemOperand(Register base, const Operand& offset, AddrMode addrmode)
+ : base_(base), regoffset_(NoReg), addrmode_(addrmode) {
+ VIXL_ASSERT(base.Is64Bits() && !base.IsZero());
+
+ if (offset.IsImmediate()) {
+ offset_ = offset.immediate();
+ } else if (offset.IsShiftedRegister()) {
+ VIXL_ASSERT((addrmode == Offset) || (addrmode == PostIndex));
+
+ regoffset_ = offset.reg();
+ shift_ = offset.shift();
+ shift_amount_ = offset.shift_amount();
+
+ extend_ = NO_EXTEND;
+ offset_ = 0;
+
+ // These assertions match those in the shifted-register constructor.
+ VIXL_ASSERT(regoffset_.Is64Bits() && !regoffset_.IsSP());
+ VIXL_ASSERT(shift_ == LSL);
+ } else {
+ VIXL_ASSERT(offset.IsExtendedRegister());
+ VIXL_ASSERT(addrmode == Offset);
+
+ regoffset_ = offset.reg();
+ extend_ = offset.extend();
+ shift_amount_ = offset.shift_amount();
+
+ shift_ = NO_SHIFT;
+ offset_ = 0;
+
+ // These assertions match those in the extended-register constructor.
+ VIXL_ASSERT(!regoffset_.IsSP());
+ VIXL_ASSERT((extend_ == UXTW) || (extend_ == SXTW) || (extend_ == SXTX));
+ VIXL_ASSERT((regoffset_.Is64Bits() || (extend_ != SXTX)));
+ }
+}
+
+
+bool MemOperand::IsImmediateOffset() const {
+ return (addrmode_ == Offset) && regoffset_.Is(NoReg);
+}
+
+
+bool MemOperand::IsRegisterOffset() const {
+ return (addrmode_ == Offset) && !regoffset_.Is(NoReg);
+}
+
+
+bool MemOperand::IsPreIndex() const {
+ return addrmode_ == PreIndex;
+}
+
+
+bool MemOperand::IsPostIndex() const {
+ return addrmode_ == PostIndex;
+}
+
+
+void MemOperand::AddOffset(int64_t offset) {
+ VIXL_ASSERT(IsImmediateOffset());
+ offset_ += offset;
+}
+
+
+// Assembler
+Assembler::Assembler(PositionIndependentCodeOption pic)
+ : pic_(pic),
+ cpu_features_(CPUFeatures::AArch64LegacyBaseline())
+{
+ // Mozilla change: always use maximally-present features.
+ cpu_features_.Combine(CPUFeatures::InferFromOS());
+
+ // Mozilla change: Compile time hard-coded value from js-config.mozbuild.
+#ifndef MOZ_AARCH64_JSCVT
+# error "MOZ_AARCH64_JSCVT must be defined."
+#elif MOZ_AARCH64_JSCVT >= 1
+ // Note, vixl backend implements the JSCVT flag as a boolean despite having 3
+ // extra bits reserved for forward compatibility in the ARMv8 documentation.
+ cpu_features_.Combine(CPUFeatures::kJSCVT);
+#endif
+}
+
+
+// Code generation.
+void Assembler::br(const Register& xn) {
+ VIXL_ASSERT(xn.Is64Bits());
+ Emit(BR | Rn(xn));
+}
+
+
+void Assembler::blr(const Register& xn) {
+ VIXL_ASSERT(xn.Is64Bits());
+ Emit(BLR | Rn(xn));
+}
+
+
+void Assembler::ret(const Register& xn) {
+ VIXL_ASSERT(xn.Is64Bits());
+ Emit(RET | Rn(xn));
+}
+
+
+void Assembler::NEONTable(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEONTableOp op) {
+ VIXL_ASSERT(vd.Is16B() || vd.Is8B());
+ VIXL_ASSERT(vn.Is16B());
+ VIXL_ASSERT(AreSameFormat(vd, vm));
+ Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ NEONTable(vd, vn, vm, NEON_TBL_1v);
+}
+
+
+void Assembler::tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vm) {
+ USE(vn2);
+ VIXL_ASSERT(AreSameFormat(vn, vn2));
+ VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
+
+ NEONTable(vd, vn, vm, NEON_TBL_2v);
+}
+
+
+void Assembler::tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vm) {
+ USE(vn2, vn3);
+ VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
+ VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
+ VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
+
+ NEONTable(vd, vn, vm, NEON_TBL_3v);
+}
+
+
+void Assembler::tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vn4,
+ const VRegister& vm) {
+ USE(vn2, vn3, vn4);
+ VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
+ VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
+ VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
+ VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters));
+
+ NEONTable(vd, vn, vm, NEON_TBL_4v);
+}
+
+
+void Assembler::tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ NEONTable(vd, vn, vm, NEON_TBX_1v);
+}
+
+
+void Assembler::tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vm) {
+ USE(vn2);
+ VIXL_ASSERT(AreSameFormat(vn, vn2));
+ VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
+
+ NEONTable(vd, vn, vm, NEON_TBX_2v);
+}
+
+
+void Assembler::tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vm) {
+ USE(vn2, vn3);
+ VIXL_ASSERT(AreSameFormat(vn, vn2, vn3));
+ VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
+ VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
+
+ NEONTable(vd, vn, vm, NEON_TBX_3v);
+}
+
+
+void Assembler::tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vn4,
+ const VRegister& vm) {
+ USE(vn2, vn3, vn4);
+ VIXL_ASSERT(AreSameFormat(vn, vn2, vn3, vn4));
+ VIXL_ASSERT(vn2.code() == ((vn.code() + 1) % kNumberOfVRegisters));
+ VIXL_ASSERT(vn3.code() == ((vn.code() + 2) % kNumberOfVRegisters));
+ VIXL_ASSERT(vn4.code() == ((vn.code() + 3) % kNumberOfVRegisters));
+
+ NEONTable(vd, vn, vm, NEON_TBX_4v);
+}
+
+
+void Assembler::add(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSub(rd, rn, operand, LeaveFlags, ADD);
+}
+
+
+void Assembler::adds(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSub(rd, rn, operand, SetFlags, ADD);
+}
+
+
+void Assembler::cmn(const Register& rn,
+ const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rn);
+ adds(zr, rn, operand);
+}
+
+
+void Assembler::sub(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSub(rd, rn, operand, LeaveFlags, SUB);
+}
+
+
+void Assembler::subs(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSub(rd, rn, operand, SetFlags, SUB);
+}
+
+
+void Assembler::cmp(const Register& rn, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rn);
+ subs(zr, rn, operand);
+}
+
+
+void Assembler::neg(const Register& rd, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ sub(rd, zr, operand);
+}
+
+
+void Assembler::negs(const Register& rd, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ subs(rd, zr, operand);
+}
+
+
+void Assembler::adc(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarry(rd, rn, operand, LeaveFlags, ADC);
+}
+
+
+void Assembler::adcs(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarry(rd, rn, operand, SetFlags, ADC);
+}
+
+
+void Assembler::sbc(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarry(rd, rn, operand, LeaveFlags, SBC);
+}
+
+
+void Assembler::sbcs(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarry(rd, rn, operand, SetFlags, SBC);
+}
+
+
+void Assembler::ngc(const Register& rd, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ sbc(rd, zr, operand);
+}
+
+
+void Assembler::ngcs(const Register& rd, const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ sbcs(rd, zr, operand);
+}
+
+
+// Logical instructions.
+void Assembler::and_(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, AND);
+}
+
+
+void Assembler::bic(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, BIC);
+}
+
+
+void Assembler::bics(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, BICS);
+}
+
+
+void Assembler::orr(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, ORR);
+}
+
+
+void Assembler::orn(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, ORN);
+}
+
+
+void Assembler::eor(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, EOR);
+}
+
+
+void Assembler::eon(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Logical(rd, rn, operand, EON);
+}
+
+
+void Assembler::lslv(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == rm.size());
+ Emit(SF(rd) | LSLV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::lsrv(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == rm.size());
+ Emit(SF(rd) | LSRV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::asrv(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == rm.size());
+ Emit(SF(rd) | ASRV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::rorv(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == rm.size());
+ Emit(SF(rd) | RORV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+
+// Bitfield operations.
+void Assembler::bfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
+ Emit(SF(rd) | BFM | N |
+ ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::sbfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms) {
+ VIXL_ASSERT(rd.Is64Bits() || rn.Is32Bits());
+ Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
+ Emit(SF(rd) | SBFM | N |
+ ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::ubfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
+ Emit(SF(rd) | UBFM | N |
+ ImmR(immr, rd.size()) | ImmS(imms, rn.size()) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::extr(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ unsigned lsb) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == rm.size());
+ Instr N = SF(rd) >> (kSFOffset - kBitfieldNOffset);
+ Emit(SF(rd) | EXTR | N | Rm(rm) | ImmS(lsb, rn.size()) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::csel(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond) {
+ ConditionalSelect(rd, rn, rm, cond, CSEL);
+}
+
+
+void Assembler::csinc(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond) {
+ ConditionalSelect(rd, rn, rm, cond, CSINC);
+}
+
+
+void Assembler::csinv(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond) {
+ ConditionalSelect(rd, rn, rm, cond, CSINV);
+}
+
+
+void Assembler::csneg(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond) {
+ ConditionalSelect(rd, rn, rm, cond, CSNEG);
+}
+
+
+void Assembler::cset(const Register &rd, Condition cond) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ Register zr = AppropriateZeroRegFor(rd);
+ csinc(rd, zr, zr, InvertCondition(cond));
+}
+
+
+void Assembler::csetm(const Register &rd, Condition cond) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ Register zr = AppropriateZeroRegFor(rd);
+ csinv(rd, zr, zr, InvertCondition(cond));
+}
+
+
+void Assembler::cinc(const Register &rd, const Register &rn, Condition cond) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ csinc(rd, rn, rn, InvertCondition(cond));
+}
+
+
+void Assembler::cinv(const Register &rd, const Register &rn, Condition cond) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ csinv(rd, rn, rn, InvertCondition(cond));
+}
+
+
+void Assembler::cneg(const Register &rd, const Register &rn, Condition cond) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ csneg(rd, rn, rn, InvertCondition(cond));
+}
+
+
+void Assembler::ConditionalSelect(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond,
+ ConditionalSelectOp op) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == rm.size());
+ Emit(SF(rd) | op | Rm(rm) | Cond(cond) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::ccmn(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond) {
+ ConditionalCompare(rn, operand, nzcv, cond, CCMN);
+}
+
+
+void Assembler::ccmp(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond) {
+ ConditionalCompare(rn, operand, nzcv, cond, CCMP);
+}
+
+
+void Assembler::DataProcessing3Source(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra,
+ DataProcessing3SourceOp op) {
+ Emit(SF(rd) | op | Rm(rm) | Ra(ra) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::crc32b(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
+ Emit(SF(rm) | Rm(rm) | CRC32B | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::crc32h(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
+ Emit(SF(rm) | Rm(rm) | CRC32H | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::crc32w(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
+ Emit(SF(rm) | Rm(rm) | CRC32W | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::crc32x(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits());
+ Emit(SF(rm) | Rm(rm) | CRC32X | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::crc32cb(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
+ Emit(SF(rm) | Rm(rm) | CRC32CB | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::crc32ch(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
+ Emit(SF(rm) | Rm(rm) | CRC32CH | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::crc32cw(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is32Bits());
+ Emit(SF(rm) | Rm(rm) | CRC32CW | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::crc32cx(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is32Bits() && rn.Is32Bits() && rm.Is64Bits());
+ Emit(SF(rm) | Rm(rm) | CRC32CX | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::mul(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
+ DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MADD);
+}
+
+
+void Assembler::madd(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ DataProcessing3Source(rd, rn, rm, ra, MADD);
+}
+
+
+void Assembler::mneg(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(AreSameSizeAndType(rd, rn, rm));
+ DataProcessing3Source(rd, rn, rm, AppropriateZeroRegFor(rd), MSUB);
+}
+
+
+void Assembler::msub(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ DataProcessing3Source(rd, rn, rm, ra, MSUB);
+}
+
+
+void Assembler::umaddl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
+ VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, ra, UMADDL_x);
+}
+
+
+void Assembler::smaddl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
+ VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, ra, SMADDL_x);
+}
+
+
+void Assembler::umsubl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
+ VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, ra, UMSUBL_x);
+}
+
+
+void Assembler::smsubl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(rd.Is64Bits() && ra.Is64Bits());
+ VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, ra, SMSUBL_x);
+}
+
+
+void Assembler::smull(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.Is64Bits());
+ VIXL_ASSERT(rn.Is32Bits() && rm.Is32Bits());
+ DataProcessing3Source(rd, rn, rm, xzr, SMADDL_x);
+}
+
+
+void Assembler::sdiv(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == rm.size());
+ Emit(SF(rd) | SDIV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::smulh(const Register& xd,
+ const Register& xn,
+ const Register& xm) {
+ VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
+ DataProcessing3Source(xd, xn, xm, xzr, SMULH_x);
+}
+
+
+void Assembler::umulh(const Register& xd,
+ const Register& xn,
+ const Register& xm) {
+ VIXL_ASSERT(xd.Is64Bits() && xn.Is64Bits() && xm.Is64Bits());
+ DataProcessing3Source(xd, xn, xm, xzr, UMULH_x);
+}
+
+
+void Assembler::udiv(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == rm.size());
+ Emit(SF(rd) | UDIV | Rm(rm) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::rbit(const Register& rd,
+ const Register& rn) {
+ DataProcessing1Source(rd, rn, RBIT);
+}
+
+
+void Assembler::rev16(const Register& rd,
+ const Register& rn) {
+ DataProcessing1Source(rd, rn, REV16);
+}
+
+
+void Assembler::rev32(const Register& rd,
+ const Register& rn) {
+ VIXL_ASSERT(rd.Is64Bits());
+ DataProcessing1Source(rd, rn, REV);
+}
+
+
+void Assembler::rev(const Register& rd,
+ const Register& rn) {
+ DataProcessing1Source(rd, rn, rd.Is64Bits() ? REV_x : REV_w);
+}
+
+
+void Assembler::clz(const Register& rd,
+ const Register& rn) {
+ DataProcessing1Source(rd, rn, CLZ);
+}
+
+
+void Assembler::cls(const Register& rd,
+ const Register& rn) {
+ DataProcessing1Source(rd, rn, CLS);
+}
+
+
+void Assembler::ldp(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& src) {
+ LoadStorePair(rt, rt2, src, LoadPairOpFor(rt, rt2));
+}
+
+
+void Assembler::stp(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& dst) {
+ LoadStorePair(rt, rt2, dst, StorePairOpFor(rt, rt2));
+}
+
+
+void Assembler::ldpsw(const Register& rt,
+ const Register& rt2,
+ const MemOperand& src) {
+ VIXL_ASSERT(rt.Is64Bits());
+ LoadStorePair(rt, rt2, src, LDPSW_x);
+}
+
+
+void Assembler::LoadStorePair(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& addr,
+ LoadStorePairOp op) {
+ // 'rt' and 'rt2' can only be aliased for stores.
+ VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || !rt.Is(rt2));
+ VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
+ VIXL_ASSERT(IsImmLSPair(addr.offset(), CalcLSPairDataSize(op)));
+
+ int offset = static_cast<int>(addr.offset());
+ Instr memop = op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) |
+ ImmLSPair(offset, CalcLSPairDataSize(op));
+
+ Instr addrmodeop;
+ if (addr.IsImmediateOffset()) {
+ addrmodeop = LoadStorePairOffsetFixed;
+ } else {
+ VIXL_ASSERT(addr.offset() != 0);
+ if (addr.IsPreIndex()) {
+ addrmodeop = LoadStorePairPreIndexFixed;
+ } else {
+ VIXL_ASSERT(addr.IsPostIndex());
+ addrmodeop = LoadStorePairPostIndexFixed;
+ }
+ }
+ Emit(addrmodeop | memop);
+}
+
+
+void Assembler::ldnp(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& src) {
+ LoadStorePairNonTemporal(rt, rt2, src,
+ LoadPairNonTemporalOpFor(rt, rt2));
+}
+
+
+void Assembler::stnp(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& dst) {
+ LoadStorePairNonTemporal(rt, rt2, dst,
+ StorePairNonTemporalOpFor(rt, rt2));
+}
+
+
+void Assembler::LoadStorePairNonTemporal(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& addr,
+ LoadStorePairNonTemporalOp op) {
+ VIXL_ASSERT(!rt.Is(rt2));
+ VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
+ VIXL_ASSERT(addr.IsImmediateOffset());
+
+ unsigned size = CalcLSPairDataSize(
+ static_cast<LoadStorePairOp>(op & LoadStorePairMask));
+ VIXL_ASSERT(IsImmLSPair(addr.offset(), size));
+ int offset = static_cast<int>(addr.offset());
+ Emit(op | Rt(rt) | Rt2(rt2) | RnSP(addr.base()) | ImmLSPair(offset, size));
+}
+
+
+// Memory instructions.
+void Assembler::ldrb(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, src, LDRB_w, option);
+}
+
+
+void Assembler::strb(const Register& rt, const MemOperand& dst,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, dst, STRB_w, option);
+}
+
+
+void Assembler::ldrsb(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
+}
+
+
+void Assembler::ldrh(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, src, LDRH_w, option);
+}
+
+
+void Assembler::strh(const Register& rt, const MemOperand& dst,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, dst, STRH_w, option);
+}
+
+
+void Assembler::ldrsh(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
+}
+
+
+void Assembler::ldr(const CPURegister& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, src, LoadOpFor(rt), option);
+}
+
+
+void Assembler::str(const CPURegister& rt, const MemOperand& dst,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, dst, StoreOpFor(rt), option);
+}
+
+
+void Assembler::ldrsw(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(rt.Is64Bits());
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ LoadStore(rt, src, LDRSW_x, option);
+}
+
+
+void Assembler::ldurb(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, src, LDRB_w, option);
+}
+
+
+void Assembler::sturb(const Register& rt, const MemOperand& dst,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, dst, STRB_w, option);
+}
+
+
+void Assembler::ldursb(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, src, rt.Is64Bits() ? LDRSB_x : LDRSB_w, option);
+}
+
+
+void Assembler::ldurh(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, src, LDRH_w, option);
+}
+
+
+void Assembler::sturh(const Register& rt, const MemOperand& dst,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, dst, STRH_w, option);
+}
+
+
+void Assembler::ldursh(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, src, rt.Is64Bits() ? LDRSH_x : LDRSH_w, option);
+}
+
+
+void Assembler::ldur(const CPURegister& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, src, LoadOpFor(rt), option);
+}
+
+
+void Assembler::stur(const CPURegister& rt, const MemOperand& dst,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, dst, StoreOpFor(rt), option);
+}
+
+
+void Assembler::ldursw(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(rt.Is64Bits());
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ LoadStore(rt, src, LDRSW_x, option);
+}
+
+
+void Assembler::ldrsw(const Register& rt, int imm19) {
+ Emit(LDRSW_x_lit | ImmLLiteral(imm19) | Rt(rt));
+}
+
+
+void Assembler::ldr(const CPURegister& rt, int imm19) {
+ LoadLiteralOp op = LoadLiteralOpFor(rt);
+ Emit(op | ImmLLiteral(imm19) | Rt(rt));
+}
+
+
+void Assembler::prfm(PrefetchOperation op, int imm19) {
+ Emit(PRFM_lit | ImmPrefetchOperation(op) | ImmLLiteral(imm19));
+}
+
+
+// Exclusive-access instructions.
+void Assembler::stxrb(const Register& rs,
+ const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ Emit(STXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::stxrh(const Register& rs,
+ const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ Emit(STXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::stxr(const Register& rs,
+ const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? STXR_x : STXR_w;
+ Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::ldxrb(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ Emit(LDXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::ldxrh(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ Emit(LDXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::ldxr(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? LDXR_x : LDXR_w;
+ Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::stxp(const Register& rs,
+ const Register& rt,
+ const Register& rt2,
+ const MemOperand& dst) {
+ VIXL_ASSERT(rt.size() == rt2.size());
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? STXP_x : STXP_w;
+ Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base()));
+}
+
+
+void Assembler::ldxp(const Register& rt,
+ const Register& rt2,
+ const MemOperand& src) {
+ VIXL_ASSERT(rt.size() == rt2.size());
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? LDXP_x : LDXP_w;
+ Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base()));
+}
+
+
+void Assembler::stlxrb(const Register& rs,
+ const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ Emit(STLXRB_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::stlxrh(const Register& rs,
+ const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ Emit(STLXRH_w | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::stlxr(const Register& rs,
+ const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? STLXR_x : STLXR_w;
+ Emit(op | Rs(rs) | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::ldaxrb(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ Emit(LDAXRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::ldaxrh(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ Emit(LDAXRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::ldaxr(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? LDAXR_x : LDAXR_w;
+ Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::stlxp(const Register& rs,
+ const Register& rt,
+ const Register& rt2,
+ const MemOperand& dst) {
+ VIXL_ASSERT(rt.size() == rt2.size());
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? STLXP_x : STLXP_w;
+ Emit(op | Rs(rs) | Rt(rt) | Rt2(rt2) | RnSP(dst.base()));
+}
+
+
+void Assembler::ldaxp(const Register& rt,
+ const Register& rt2,
+ const MemOperand& src) {
+ VIXL_ASSERT(rt.size() == rt2.size());
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? LDAXP_x : LDAXP_w;
+ Emit(op | Rs_mask | Rt(rt) | Rt2(rt2) | RnSP(src.base()));
+}
+
+
+void Assembler::stlrb(const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ Emit(STLRB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::stlrh(const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ Emit(STLRH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::stlr(const Register& rt,
+ const MemOperand& dst) {
+ VIXL_ASSERT(dst.IsImmediateOffset() && (dst.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? STLR_x : STLR_w;
+ Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(dst.base()));
+}
+
+
+void Assembler::ldarb(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ Emit(LDARB_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::ldarh(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ Emit(LDARH_w | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::ldar(const Register& rt,
+ const MemOperand& src) {
+ VIXL_ASSERT(src.IsImmediateOffset() && (src.offset() == 0));
+ LoadStoreExclusive op = rt.Is64Bits() ? LDAR_x : LDAR_w;
+ Emit(op | Rs_mask | Rt(rt) | Rt2_mask | RnSP(src.base()));
+}
+
+
+void Assembler::prfm(PrefetchOperation op, const MemOperand& address,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireUnscaledOffset);
+ VIXL_ASSERT(option != PreferUnscaledOffset);
+ Prefetch(op, address, option);
+}
+
+
+void Assembler::prfum(PrefetchOperation op, const MemOperand& address,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(option != RequireScaledOffset);
+ VIXL_ASSERT(option != PreferScaledOffset);
+ Prefetch(op, address, option);
+}
+
+
+void Assembler::sys(int op1, int crn, int crm, int op2, const Register& rt) {
+ Emit(SYS | ImmSysOp1(op1) | CRn(crn) | CRm(crm) | ImmSysOp2(op2) | Rt(rt));
+}
+
+
+void Assembler::sys(int op, const Register& rt) {
+ Emit(SYS | SysOp(op) | Rt(rt));
+}
+
+
+void Assembler::dc(DataCacheOp op, const Register& rt) {
+ VIXL_ASSERT((op == CVAC) || (op == CVAU) || (op == CIVAC) || (op == ZVA));
+ sys(op, rt);
+}
+
+
+void Assembler::ic(InstructionCacheOp op, const Register& rt) {
+ VIXL_ASSERT(op == IVAU);
+ sys(op, rt);
+}
+
+
+// NEON structure loads and stores.
+Instr Assembler::LoadStoreStructAddrModeField(const MemOperand& addr) {
+ Instr addr_field = RnSP(addr.base());
+
+ if (addr.IsPostIndex()) {
+ VIXL_STATIC_ASSERT(NEONLoadStoreMultiStructPostIndex ==
+ static_cast<NEONLoadStoreMultiStructPostIndexOp>(
+ NEONLoadStoreSingleStructPostIndex));
+
+ addr_field |= NEONLoadStoreMultiStructPostIndex;
+ if (addr.offset() == 0) {
+ addr_field |= RmNot31(addr.regoffset());
+ } else {
+ // The immediate post index addressing mode is indicated by rm = 31.
+ // The immediate is implied by the number of vector registers used.
+ addr_field |= (0x1f << Rm_offset);
+ }
+ } else {
+ VIXL_ASSERT(addr.IsImmediateOffset() && (addr.offset() == 0));
+ }
+ return addr_field;
+}
+
+void Assembler::LoadStoreStructVerify(const VRegister& vt,
+ const MemOperand& addr,
+ Instr op) {
+#ifdef DEBUG
+ // Assert that addressing mode is either offset (with immediate 0), post
+ // index by immediate of the size of the register list, or post index by a
+ // value in a core register.
+ if (addr.IsImmediateOffset()) {
+ VIXL_ASSERT(addr.offset() == 0);
+ } else {
+ int offset = vt.SizeInBytes();
+ switch (op) {
+ case NEON_LD1_1v:
+ case NEON_ST1_1v:
+ offset *= 1; break;
+ case NEONLoadStoreSingleStructLoad1:
+ case NEONLoadStoreSingleStructStore1:
+ case NEON_LD1R:
+ offset = (offset / vt.lanes()) * 1; break;
+
+ case NEON_LD1_2v:
+ case NEON_ST1_2v:
+ case NEON_LD2:
+ case NEON_ST2:
+ offset *= 2;
+ break;
+ case NEONLoadStoreSingleStructLoad2:
+ case NEONLoadStoreSingleStructStore2:
+ case NEON_LD2R:
+ offset = (offset / vt.lanes()) * 2; break;
+
+ case NEON_LD1_3v:
+ case NEON_ST1_3v:
+ case NEON_LD3:
+ case NEON_ST3:
+ offset *= 3; break;
+ case NEONLoadStoreSingleStructLoad3:
+ case NEONLoadStoreSingleStructStore3:
+ case NEON_LD3R:
+ offset = (offset / vt.lanes()) * 3; break;
+
+ case NEON_LD1_4v:
+ case NEON_ST1_4v:
+ case NEON_LD4:
+ case NEON_ST4:
+ offset *= 4; break;
+ case NEONLoadStoreSingleStructLoad4:
+ case NEONLoadStoreSingleStructStore4:
+ case NEON_LD4R:
+ offset = (offset / vt.lanes()) * 4; break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+ VIXL_ASSERT(!addr.regoffset().Is(NoReg) ||
+ addr.offset() == offset);
+ }
+#else
+ USE(vt, addr, op);
+#endif
+}
+
+void Assembler::LoadStoreStruct(const VRegister& vt,
+ const MemOperand& addr,
+ NEONLoadStoreMultiStructOp op) {
+ LoadStoreStructVerify(vt, addr, op);
+ VIXL_ASSERT(vt.IsVector() || vt.Is1D());
+ Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
+}
+
+
+void Assembler::LoadStoreStructSingleAllLanes(const VRegister& vt,
+ const MemOperand& addr,
+ NEONLoadStoreSingleStructOp op) {
+ LoadStoreStructVerify(vt, addr, op);
+ Emit(op | LoadStoreStructAddrModeField(addr) | LSVFormat(vt) | Rt(vt));
+}
+
+
+void Assembler::ld1(const VRegister& vt,
+ const MemOperand& src) {
+ LoadStoreStruct(vt, src, NEON_LD1_1v);
+}
+
+
+void Assembler::ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src) {
+ USE(vt2);
+ VIXL_ASSERT(AreSameFormat(vt, vt2));
+ VIXL_ASSERT(AreConsecutive(vt, vt2));
+ LoadStoreStruct(vt, src, NEON_LD1_2v);
+}
+
+
+void Assembler::ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src) {
+ USE(vt2, vt3);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStruct(vt, src, NEON_LD1_3v);
+}
+
+
+void Assembler::ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src) {
+ USE(vt2, vt3, vt4);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStruct(vt, src, NEON_LD1_4v);
+}
+
+
+void Assembler::ld2(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src) {
+ USE(vt2);
+ VIXL_ASSERT(AreSameFormat(vt, vt2));
+ VIXL_ASSERT(AreConsecutive(vt, vt2));
+ LoadStoreStruct(vt, src, NEON_LD2);
+}
+
+
+void Assembler::ld2(const VRegister& vt,
+ const VRegister& vt2,
+ int lane,
+ const MemOperand& src) {
+ USE(vt2);
+ VIXL_ASSERT(AreSameFormat(vt, vt2));
+ VIXL_ASSERT(AreConsecutive(vt, vt2));
+ LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad2);
+}
+
+
+void Assembler::ld2r(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src) {
+ USE(vt2);
+ VIXL_ASSERT(AreSameFormat(vt, vt2));
+ VIXL_ASSERT(AreConsecutive(vt, vt2));
+ LoadStoreStructSingleAllLanes(vt, src, NEON_LD2R);
+}
+
+
+void Assembler::ld3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src) {
+ USE(vt2, vt3);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStruct(vt, src, NEON_LD3);
+}
+
+
+void Assembler::ld3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ int lane,
+ const MemOperand& src) {
+ USE(vt2, vt3);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad3);
+}
+
+
+void Assembler::ld3r(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src) {
+ USE(vt2, vt3);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStructSingleAllLanes(vt, src, NEON_LD3R);
+}
+
+
+void Assembler::ld4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src) {
+ USE(vt2, vt3, vt4);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStruct(vt, src, NEON_LD4);
+}
+
+
+void Assembler::ld4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ int lane,
+ const MemOperand& src) {
+ USE(vt2, vt3, vt4);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad4);
+}
+
+
+void Assembler::ld4r(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src) {
+ USE(vt2, vt3, vt4);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStructSingleAllLanes(vt, src, NEON_LD4R);
+}
+
+
+void Assembler::st1(const VRegister& vt,
+ const MemOperand& src) {
+ LoadStoreStruct(vt, src, NEON_ST1_1v);
+}
+
+
+void Assembler::st1(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src) {
+ USE(vt2);
+ VIXL_ASSERT(AreSameFormat(vt, vt2));
+ VIXL_ASSERT(AreConsecutive(vt, vt2));
+ LoadStoreStruct(vt, src, NEON_ST1_2v);
+}
+
+
+void Assembler::st1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src) {
+ USE(vt2, vt3);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStruct(vt, src, NEON_ST1_3v);
+}
+
+
+void Assembler::st1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src) {
+ USE(vt2, vt3, vt4);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStruct(vt, src, NEON_ST1_4v);
+}
+
+
+void Assembler::st2(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& dst) {
+ USE(vt2);
+ VIXL_ASSERT(AreSameFormat(vt, vt2));
+ VIXL_ASSERT(AreConsecutive(vt, vt2));
+ LoadStoreStruct(vt, dst, NEON_ST2);
+}
+
+
+void Assembler::st2(const VRegister& vt,
+ const VRegister& vt2,
+ int lane,
+ const MemOperand& dst) {
+ USE(vt2);
+ VIXL_ASSERT(AreSameFormat(vt, vt2));
+ VIXL_ASSERT(AreConsecutive(vt, vt2));
+ LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore2);
+}
+
+
+void Assembler::st3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& dst) {
+ USE(vt2, vt3);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStruct(vt, dst, NEON_ST3);
+}
+
+
+void Assembler::st3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ int lane,
+ const MemOperand& dst) {
+ USE(vt2, vt3);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3));
+ LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore3);
+}
+
+
+void Assembler::st4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& dst) {
+ USE(vt2, vt3, vt4);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStruct(vt, dst, NEON_ST4);
+}
+
+
+void Assembler::st4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ int lane,
+ const MemOperand& dst) {
+ USE(vt2, vt3, vt4);
+ VIXL_ASSERT(AreSameFormat(vt, vt2, vt3, vt4));
+ VIXL_ASSERT(AreConsecutive(vt, vt2, vt3, vt4));
+ LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore4);
+}
+
+
+void Assembler::LoadStoreStructSingle(const VRegister& vt,
+ uint32_t lane,
+ const MemOperand& addr,
+ NEONLoadStoreSingleStructOp op) {
+ LoadStoreStructVerify(vt, addr, op);
+
+ // We support vt arguments of the form vt.VxT() or vt.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ unsigned lane_size = vt.LaneSizeInBytes();
+ VIXL_ASSERT(lane < (kQRegSizeInBytes / lane_size));
+
+ // Lane size is encoded in the opcode field. Lane index is encoded in the Q,
+ // S and size fields.
+ lane *= lane_size;
+ if (lane_size == 8) lane++;
+
+ Instr size = (lane << NEONLSSize_offset) & NEONLSSize_mask;
+ Instr s = (lane << (NEONS_offset - 2)) & NEONS_mask;
+ Instr q = (lane << (NEONQ_offset - 3)) & NEONQ_mask;
+
+ Instr instr = op;
+ switch (lane_size) {
+ case 1: instr |= NEONLoadStoreSingle_b; break;
+ case 2: instr |= NEONLoadStoreSingle_h; break;
+ case 4: instr |= NEONLoadStoreSingle_s; break;
+ default:
+ VIXL_ASSERT(lane_size == 8);
+ instr |= NEONLoadStoreSingle_d;
+ }
+
+ Emit(instr | LoadStoreStructAddrModeField(addr) | q | size | s | Rt(vt));
+}
+
+
+void Assembler::ld1(const VRegister& vt,
+ int lane,
+ const MemOperand& src) {
+ LoadStoreStructSingle(vt, lane, src, NEONLoadStoreSingleStructLoad1);
+}
+
+
+void Assembler::ld1r(const VRegister& vt,
+ const MemOperand& src) {
+ LoadStoreStructSingleAllLanes(vt, src, NEON_LD1R);
+}
+
+
+void Assembler::st1(const VRegister& vt,
+ int lane,
+ const MemOperand& dst) {
+ LoadStoreStructSingle(vt, lane, dst, NEONLoadStoreSingleStructStore1);
+}
+
+
+void Assembler::NEON3DifferentL(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEON3DifferentOp vop) {
+ VIXL_ASSERT(AreSameFormat(vn, vm));
+ VIXL_ASSERT((vn.Is1H() && vd.Is1S()) ||
+ (vn.Is1S() && vd.Is1D()) ||
+ (vn.Is8B() && vd.Is8H()) ||
+ (vn.Is4H() && vd.Is4S()) ||
+ (vn.Is2S() && vd.Is2D()) ||
+ (vn.Is16B() && vd.Is8H())||
+ (vn.Is8H() && vd.Is4S()) ||
+ (vn.Is4S() && vd.Is2D()));
+ Instr format, op = vop;
+ if (vd.IsScalar()) {
+ op |= NEON_Q | NEONScalar;
+ format = SFormat(vn);
+ } else {
+ format = VFormat(vn);
+ }
+ Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEON3DifferentW(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEON3DifferentOp vop) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT((vm.Is8B() && vd.Is8H()) ||
+ (vm.Is4H() && vd.Is4S()) ||
+ (vm.Is2S() && vd.Is2D()) ||
+ (vm.Is16B() && vd.Is8H())||
+ (vm.Is8H() && vd.Is4S()) ||
+ (vm.Is4S() && vd.Is2D()));
+ Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEON3DifferentHN(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEON3DifferentOp vop) {
+ VIXL_ASSERT(AreSameFormat(vm, vn));
+ VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
+ (vd.Is4H() && vn.Is4S()) ||
+ (vd.Is2S() && vn.Is2D()) ||
+ (vd.Is16B() && vn.Is8H())||
+ (vd.Is8H() && vn.Is4S()) ||
+ (vd.Is4S() && vn.Is2D()));
+ Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+#define NEON_3DIFF_LONG_LIST(V) \
+ V(pmull, NEON_PMULL, vn.IsVector() && vn.Is8B()) \
+ V(pmull2, NEON_PMULL2, vn.IsVector() && vn.Is16B()) \
+ V(saddl, NEON_SADDL, vn.IsVector() && vn.IsD()) \
+ V(saddl2, NEON_SADDL2, vn.IsVector() && vn.IsQ()) \
+ V(sabal, NEON_SABAL, vn.IsVector() && vn.IsD()) \
+ V(sabal2, NEON_SABAL2, vn.IsVector() && vn.IsQ()) \
+ V(uabal, NEON_UABAL, vn.IsVector() && vn.IsD()) \
+ V(uabal2, NEON_UABAL2, vn.IsVector() && vn.IsQ()) \
+ V(sabdl, NEON_SABDL, vn.IsVector() && vn.IsD()) \
+ V(sabdl2, NEON_SABDL2, vn.IsVector() && vn.IsQ()) \
+ V(uabdl, NEON_UABDL, vn.IsVector() && vn.IsD()) \
+ V(uabdl2, NEON_UABDL2, vn.IsVector() && vn.IsQ()) \
+ V(smlal, NEON_SMLAL, vn.IsVector() && vn.IsD()) \
+ V(smlal2, NEON_SMLAL2, vn.IsVector() && vn.IsQ()) \
+ V(umlal, NEON_UMLAL, vn.IsVector() && vn.IsD()) \
+ V(umlal2, NEON_UMLAL2, vn.IsVector() && vn.IsQ()) \
+ V(smlsl, NEON_SMLSL, vn.IsVector() && vn.IsD()) \
+ V(smlsl2, NEON_SMLSL2, vn.IsVector() && vn.IsQ()) \
+ V(umlsl, NEON_UMLSL, vn.IsVector() && vn.IsD()) \
+ V(umlsl2, NEON_UMLSL2, vn.IsVector() && vn.IsQ()) \
+ V(smull, NEON_SMULL, vn.IsVector() && vn.IsD()) \
+ V(smull2, NEON_SMULL2, vn.IsVector() && vn.IsQ()) \
+ V(umull, NEON_UMULL, vn.IsVector() && vn.IsD()) \
+ V(umull2, NEON_UMULL2, vn.IsVector() && vn.IsQ()) \
+ V(ssubl, NEON_SSUBL, vn.IsVector() && vn.IsD()) \
+ V(ssubl2, NEON_SSUBL2, vn.IsVector() && vn.IsQ()) \
+ V(uaddl, NEON_UADDL, vn.IsVector() && vn.IsD()) \
+ V(uaddl2, NEON_UADDL2, vn.IsVector() && vn.IsQ()) \
+ V(usubl, NEON_USUBL, vn.IsVector() && vn.IsD()) \
+ V(usubl2, NEON_USUBL2, vn.IsVector() && vn.IsQ()) \
+ V(sqdmlal, NEON_SQDMLAL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
+ V(sqdmlal2, NEON_SQDMLAL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
+ V(sqdmlsl, NEON_SQDMLSL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
+ V(sqdmlsl2, NEON_SQDMLSL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
+ V(sqdmull, NEON_SQDMULL, vn.Is1H() || vn.Is1S() || vn.Is4H() || vn.Is2S()) \
+ V(sqdmull2, NEON_SQDMULL2, vn.Is1H() || vn.Is1S() || vn.Is8H() || vn.Is4S()) \
+
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm) { \
+ VIXL_ASSERT(AS); \
+ NEON3DifferentL(vd, vn, vm, OP); \
+}
+NEON_3DIFF_LONG_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+#define NEON_3DIFF_HN_LIST(V) \
+ V(addhn, NEON_ADDHN, vd.IsD()) \
+ V(addhn2, NEON_ADDHN2, vd.IsQ()) \
+ V(raddhn, NEON_RADDHN, vd.IsD()) \
+ V(raddhn2, NEON_RADDHN2, vd.IsQ()) \
+ V(subhn, NEON_SUBHN, vd.IsD()) \
+ V(subhn2, NEON_SUBHN2, vd.IsQ()) \
+ V(rsubhn, NEON_RSUBHN, vd.IsD()) \
+ V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm) { \
+ VIXL_ASSERT(AS); \
+ NEON3DifferentHN(vd, vn, vm, OP); \
+}
+NEON_3DIFF_HN_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+void Assembler::uaddw(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(vm.IsD());
+ NEON3DifferentW(vd, vn, vm, NEON_UADDW);
+}
+
+
+void Assembler::uaddw2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(vm.IsQ());
+ NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
+}
+
+
+void Assembler::saddw(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(vm.IsD());
+ NEON3DifferentW(vd, vn, vm, NEON_SADDW);
+}
+
+
+void Assembler::saddw2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(vm.IsQ());
+ NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
+}
+
+
+void Assembler::usubw(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(vm.IsD());
+ NEON3DifferentW(vd, vn, vm, NEON_USUBW);
+}
+
+
+void Assembler::usubw2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(vm.IsQ());
+ NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
+}
+
+
+void Assembler::ssubw(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(vm.IsD());
+ NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
+}
+
+
+void Assembler::ssubw2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(vm.IsQ());
+ NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
+}
+
+
+void Assembler::mov(const Register& rd, const Register& rm) {
+ // Moves involving the stack pointer are encoded as add immediate with
+ // second operand of zero. Otherwise, orr with first operand zr is
+ // used.
+ if (rd.IsSP() || rm.IsSP()) {
+ add(rd, rm, 0);
+ } else {
+ orr(rd, AppropriateZeroRegFor(rd), rm);
+ }
+}
+
+
+void Assembler::mvn(const Register& rd, const Operand& operand) {
+ orn(rd, AppropriateZeroRegFor(rd), operand);
+}
+
+
+void Assembler::mrs(const Register& rt, SystemRegister sysreg) {
+ VIXL_ASSERT(rt.Is64Bits());
+ Emit(MRS | ImmSystemRegister(sysreg) | Rt(rt));
+}
+
+
+void Assembler::msr(SystemRegister sysreg, const Register& rt) {
+ VIXL_ASSERT(rt.Is64Bits());
+ Emit(MSR | Rt(rt) | ImmSystemRegister(sysreg));
+}
+
+
+void Assembler::clrex(int imm4) {
+ Emit(CLREX | CRm(imm4));
+}
+
+
+void Assembler::dmb(BarrierDomain domain, BarrierType type) {
+ Emit(DMB | ImmBarrierDomain(domain) | ImmBarrierType(type));
+}
+
+
+void Assembler::dsb(BarrierDomain domain, BarrierType type) {
+ Emit(DSB | ImmBarrierDomain(domain) | ImmBarrierType(type));
+}
+
+
+void Assembler::isb() {
+ Emit(ISB | ImmBarrierDomain(FullSystem) | ImmBarrierType(BarrierAll));
+}
+
+
+void Assembler::fmov(const VRegister& vd, double imm) {
+ if (vd.IsScalar()) {
+ VIXL_ASSERT(vd.Is1D());
+ Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
+ } else {
+ VIXL_ASSERT(vd.Is2D());
+ Instr op = NEONModifiedImmediate_MOVI | NEONModifiedImmediateOpBit;
+ Instr q = NEON_Q;
+ uint32_t encoded_imm = FP64ToImm8(imm);
+ Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
+ }
+}
+
+
+void Assembler::fmov(const VRegister& vd, float imm) {
+ if (vd.IsScalar()) {
+ VIXL_ASSERT(vd.Is1S());
+ Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
+ } else {
+ VIXL_ASSERT(vd.Is2S() || vd.Is4S());
+ Instr op = NEONModifiedImmediate_MOVI;
+ Instr q = vd.Is4S() ? NEON_Q : 0;
+ uint32_t encoded_imm = FP32ToImm8(imm);
+ Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
+ }
+}
+
+
+void Assembler::fmov(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(vn.Is1S() || vn.Is1D());
+ VIXL_ASSERT(rd.size() == vn.size());
+ FPIntegerConvertOp op = rd.Is32Bits() ? FMOV_ws : FMOV_xd;
+ Emit(op | Rd(rd) | Rn(vn));
+}
+
+
+void Assembler::fmov(const VRegister& vd, const Register& rn) {
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D());
+ VIXL_ASSERT(vd.size() == rn.size());
+ FPIntegerConvertOp op = vd.Is32Bits() ? FMOV_sw : FMOV_dx;
+ Emit(op | Rd(vd) | Rn(rn));
+}
+
+
+void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D());
+ VIXL_ASSERT(vd.IsSameFormat(vn));
+ Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
+}
+
+
+void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
+ VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
+ USE(index);
+ Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
+}
+
+
+void Assembler::fmov(const Register& rd, const VRegister& vn, int index) {
+ VIXL_ASSERT((index == 1) && vn.Is1D() && rd.IsX());
+ USE(index);
+ Emit(FMOV_x_d1 | Rd(rd) | Rn(vn));
+}
+
+
+void Assembler::fmadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va) {
+ FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMADD_s : FMADD_d);
+}
+
+
+void Assembler::fmsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va) {
+ FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FMSUB_s : FMSUB_d);
+}
+
+
+void Assembler::fnmadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va) {
+ FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMADD_s : FNMADD_d);
+}
+
+
+void Assembler::fnmsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va) {
+ FPDataProcessing3Source(vd, vn, vm, va, vd.Is1S() ? FNMSUB_s : FNMSUB_d);
+}
+
+
+void Assembler::fnmul(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
+ Instr op = vd.Is1S() ? FNMUL_s : FNMUL_d;
+ Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::FPCompareMacro(const VRegister& vn,
+ double value,
+ FPTrapFlags trap) {
+ USE(value);
+ // Although the fcmp{e} instructions can strictly only take an immediate
+ // value of +0.0, we don't need to check for -0.0 because the sign of 0.0
+ // doesn't affect the result of the comparison.
+ VIXL_ASSERT(value == 0.0);
+ VIXL_ASSERT(vn.Is1S() || vn.Is1D());
+ Instr op = (trap == EnableTrap) ? FCMPE_zero : FCMP_zero;
+ Emit(FPType(vn) | op | Rn(vn));
+}
+
+
+void Assembler::FPCompareMacro(const VRegister& vn,
+ const VRegister& vm,
+ FPTrapFlags trap) {
+ VIXL_ASSERT(vn.Is1S() || vn.Is1D());
+ VIXL_ASSERT(vn.IsSameSizeAndType(vm));
+ Instr op = (trap == EnableTrap) ? FCMPE : FCMP;
+ Emit(FPType(vn) | op | Rm(vm) | Rn(vn));
+}
+
+
+void Assembler::fcmp(const VRegister& vn,
+ const VRegister& vm) {
+ FPCompareMacro(vn, vm, DisableTrap);
+}
+
+
+void Assembler::fcmpe(const VRegister& vn,
+ const VRegister& vm) {
+ FPCompareMacro(vn, vm, EnableTrap);
+}
+
+
+void Assembler::fcmp(const VRegister& vn,
+ double value) {
+ FPCompareMacro(vn, value, DisableTrap);
+}
+
+
+void Assembler::fcmpe(const VRegister& vn,
+ double value) {
+ FPCompareMacro(vn, value, EnableTrap);
+}
+
+
+void Assembler::FPCCompareMacro(const VRegister& vn,
+ const VRegister& vm,
+ StatusFlags nzcv,
+ Condition cond,
+ FPTrapFlags trap) {
+ VIXL_ASSERT(vn.Is1S() || vn.Is1D());
+ VIXL_ASSERT(vn.IsSameSizeAndType(vm));
+ Instr op = (trap == EnableTrap) ? FCCMPE : FCCMP;
+ Emit(FPType(vn) | op | Rm(vm) | Cond(cond) | Rn(vn) | Nzcv(nzcv));
+}
+
+void Assembler::fccmp(const VRegister& vn,
+ const VRegister& vm,
+ StatusFlags nzcv,
+ Condition cond) {
+ FPCCompareMacro(vn, vm, nzcv, cond, DisableTrap);
+}
+
+
+void Assembler::fccmpe(const VRegister& vn,
+ const VRegister& vm,
+ StatusFlags nzcv,
+ Condition cond) {
+ FPCCompareMacro(vn, vm, nzcv, cond, EnableTrap);
+}
+
+
+void Assembler::fcsel(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ Condition cond) {
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D());
+ VIXL_ASSERT(AreSameFormat(vd, vn, vm));
+ Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
+}
+
+void Assembler::fjcvtzs(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(CPUHas(CPUFeatures::kFP, CPUFeatures::kJSCVT));
+ VIXL_ASSERT(rd.IsW() && vn.Is1D());
+ Emit(FJCVTZS | Rn(vn) | Rd(rd));
+}
+
+
+void Assembler::NEONFPConvertToInt(const Register& rd,
+ const VRegister& vn,
+ Instr op) {
+ Emit(SF(rd) | FPType(vn) | op | Rn(vn) | Rd(rd));
+}
+
+
+void Assembler::NEONFPConvertToInt(const VRegister& vd,
+ const VRegister& vn,
+ Instr op) {
+ if (vn.IsScalar()) {
+ VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
+ op |= NEON_Q | NEONScalar;
+ }
+ Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fcvt(const VRegister& vd,
+ const VRegister& vn) {
+ FPDataProcessing1SourceOp op;
+ if (vd.Is1D()) {
+ VIXL_ASSERT(vn.Is1S() || vn.Is1H());
+ op = vn.Is1S() ? FCVT_ds : FCVT_dh;
+ } else if (vd.Is1S()) {
+ VIXL_ASSERT(vn.Is1D() || vn.Is1H());
+ op = vn.Is1D() ? FCVT_sd : FCVT_sh;
+ } else {
+ VIXL_ASSERT(vd.Is1H());
+ VIXL_ASSERT(vn.Is1D() || vn.Is1S());
+ op = vn.Is1D() ? FCVT_hd : FCVT_hs;
+ }
+ FPDataProcessing1Source(vd, vn, op);
+}
+
+
+void Assembler::fcvtl(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vd.Is4S() && vn.Is4H()) ||
+ (vd.Is2D() && vn.Is2S()));
+ Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
+ Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fcvtl2(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vd.Is4S() && vn.Is8H()) ||
+ (vd.Is2D() && vn.Is4S()));
+ Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
+ Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fcvtn(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vn.Is4S() && vd.Is4H()) ||
+ (vn.Is2D() && vd.Is2S()));
+ Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
+ Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fcvtn2(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vn.Is4S() && vd.Is8H()) ||
+ (vn.Is2D() && vd.Is4S()));
+ Instr format = vn.Is2D() ? (1 << NEONSize_offset) : 0;
+ Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fcvtxn(const VRegister& vd,
+ const VRegister& vn) {
+ Instr format = 1 << NEONSize_offset;
+ if (vd.IsScalar()) {
+ VIXL_ASSERT(vd.Is1S() && vn.Is1D());
+ Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
+ } else {
+ VIXL_ASSERT(vd.Is2S() && vn.Is2D());
+ Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
+ }
+}
+
+
+void Assembler::fcvtxn2(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.Is4S() && vn.Is2D());
+ Instr format = 1 << NEONSize_offset;
+ Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
+}
+
+
+#define NEON_FP2REGMISC_FCVT_LIST(V) \
+ V(fcvtnu, NEON_FCVTNU, FCVTNU) \
+ V(fcvtns, NEON_FCVTNS, FCVTNS) \
+ V(fcvtpu, NEON_FCVTPU, FCVTPU) \
+ V(fcvtps, NEON_FCVTPS, FCVTPS) \
+ V(fcvtmu, NEON_FCVTMU, FCVTMU) \
+ V(fcvtms, NEON_FCVTMS, FCVTMS) \
+ V(fcvtau, NEON_FCVTAU, FCVTAU) \
+ V(fcvtas, NEON_FCVTAS, FCVTAS)
+
+#define DEFINE_ASM_FUNCS(FN, VEC_OP, SCA_OP) \
+void Assembler::FN(const Register& rd, \
+ const VRegister& vn) { \
+ NEONFPConvertToInt(rd, vn, SCA_OP); \
+} \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn) { \
+ NEONFPConvertToInt(vd, vn, VEC_OP); \
+}
+NEON_FP2REGMISC_FCVT_LIST(DEFINE_ASM_FUNCS)
+#undef DEFINE_ASM_FUNCS
+
+
+void Assembler::fcvtzs(const Register& rd,
+ const VRegister& vn,
+ int fbits) {
+ VIXL_ASSERT(vn.Is1S() || vn.Is1D());
+ VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits()));
+ if (fbits == 0) {
+ Emit(SF(rd) | FPType(vn) | FCVTZS | Rn(vn) | Rd(rd));
+ } else {
+ Emit(SF(rd) | FPType(vn) | FCVTZS_fixed | FPScale(64 - fbits) | Rn(vn) |
+ Rd(rd));
+ }
+}
+
+
+void Assembler::fcvtzs(const VRegister& vd,
+ const VRegister& vn,
+ int fbits) {
+ VIXL_ASSERT(fbits >= 0);
+ if (fbits == 0) {
+ NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
+ } else {
+ VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
+ NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
+ }
+}
+
+
+void Assembler::fcvtzu(const Register& rd,
+ const VRegister& vn,
+ int fbits) {
+ VIXL_ASSERT(vn.Is1S() || vn.Is1D());
+ VIXL_ASSERT((fbits >= 0) && (fbits <= rd.SizeInBits()));
+ if (fbits == 0) {
+ Emit(SF(rd) | FPType(vn) | FCVTZU | Rn(vn) | Rd(rd));
+ } else {
+ Emit(SF(rd) | FPType(vn) | FCVTZU_fixed | FPScale(64 - fbits) | Rn(vn) |
+ Rd(rd));
+ }
+}
+
+
+void Assembler::fcvtzu(const VRegister& vd,
+ const VRegister& vn,
+ int fbits) {
+ VIXL_ASSERT(fbits >= 0);
+ if (fbits == 0) {
+ NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
+ } else {
+ VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
+ NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
+ }
+}
+
+void Assembler::ucvtf(const VRegister& vd,
+ const VRegister& vn,
+ int fbits) {
+ VIXL_ASSERT(fbits >= 0);
+ if (fbits == 0) {
+ NEONFP2RegMisc(vd, vn, NEON_UCVTF);
+ } else {
+ VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
+ NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
+ }
+}
+
+void Assembler::scvtf(const VRegister& vd,
+ const VRegister& vn,
+ int fbits) {
+ VIXL_ASSERT(fbits >= 0);
+ if (fbits == 0) {
+ NEONFP2RegMisc(vd, vn, NEON_SCVTF);
+ } else {
+ VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S());
+ NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
+ }
+}
+
+
+void Assembler::scvtf(const VRegister& vd,
+ const Register& rn,
+ int fbits) {
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D());
+ VIXL_ASSERT(fbits >= 0);
+ if (fbits == 0) {
+ Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
+ } else {
+ Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
+ Rd(vd));
+ }
+}
+
+
+void Assembler::ucvtf(const VRegister& vd,
+ const Register& rn,
+ int fbits) {
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D());
+ VIXL_ASSERT(fbits >= 0);
+ if (fbits == 0) {
+ Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
+ } else {
+ Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
+ Rd(vd));
+ }
+}
+
+
+void Assembler::NEON3Same(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEON3SameOp vop) {
+ VIXL_ASSERT(AreSameFormat(vd, vn, vm));
+ VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
+
+ Instr format, op = vop;
+ if (vd.IsScalar()) {
+ op |= NEON_Q | NEONScalar;
+ format = SFormat(vd);
+ } else {
+ format = VFormat(vd);
+ }
+
+ Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEONFP3Same(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ Instr op) {
+ VIXL_ASSERT(AreSameFormat(vd, vn, vm));
+ Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+#define NEON_FP2REGMISC_LIST(V) \
+ V(fabs, NEON_FABS, FABS) \
+ V(fneg, NEON_FNEG, FNEG) \
+ V(fsqrt, NEON_FSQRT, FSQRT) \
+ V(frintn, NEON_FRINTN, FRINTN) \
+ V(frinta, NEON_FRINTA, FRINTA) \
+ V(frintp, NEON_FRINTP, FRINTP) \
+ V(frintm, NEON_FRINTM, FRINTM) \
+ V(frintx, NEON_FRINTX, FRINTX) \
+ V(frintz, NEON_FRINTZ, FRINTZ) \
+ V(frinti, NEON_FRINTI, FRINTI) \
+ V(frsqrte, NEON_FRSQRTE, NEON_FRSQRTE_scalar) \
+ V(frecpe, NEON_FRECPE, NEON_FRECPE_scalar )
+
+
+#define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn) { \
+ Instr op; \
+ if (vd.IsScalar()) { \
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
+ op = SCA_OP; \
+ } else { \
+ VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
+ op = VEC_OP; \
+ } \
+ NEONFP2RegMisc(vd, vn, op); \
+}
+NEON_FP2REGMISC_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+
+void Assembler::NEONFP2RegMisc(const VRegister& vd,
+ const VRegister& vn,
+ Instr op) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEON2RegMisc(const VRegister& vd,
+ const VRegister& vn,
+ NEON2RegMiscOp vop,
+ int value) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(value == 0);
+ USE(value);
+
+ Instr format, op = vop;
+ if (vd.IsScalar()) {
+ op |= NEON_Q | NEONScalar;
+ format = SFormat(vd);
+ } else {
+ format = VFormat(vd);
+ }
+
+ Emit(format | op | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::cmeq(const VRegister& vd,
+ const VRegister& vn,
+ int value) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
+}
+
+
+void Assembler::cmge(const VRegister& vd,
+ const VRegister& vn,
+ int value) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
+}
+
+
+void Assembler::cmgt(const VRegister& vd,
+ const VRegister& vn,
+ int value) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
+}
+
+
+void Assembler::cmle(const VRegister& vd,
+ const VRegister& vn,
+ int value) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
+}
+
+
+void Assembler::cmlt(const VRegister& vd,
+ const VRegister& vn,
+ int value) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
+}
+
+
+void Assembler::shll(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
+ (vd.Is4S() && vn.Is4H() && shift == 16) ||
+ (vd.Is2D() && vn.Is2S() && shift == 32));
+ USE(shift);
+ Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::shll2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ USE(shift);
+ VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
+ (vd.Is4S() && vn.Is8H() && shift == 16) ||
+ (vd.Is2D() && vn.Is4S() && shift == 32));
+ Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEONFP2RegMisc(const VRegister& vd,
+ const VRegister& vn,
+ NEON2RegMiscOp vop,
+ double value) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(value == 0.0);
+ USE(value);
+
+ Instr op = vop;
+ if (vd.IsScalar()) {
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D());
+ op |= NEON_Q | NEONScalar;
+ } else {
+ VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
+ }
+
+ Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fcmeq(const VRegister& vd,
+ const VRegister& vn,
+ double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
+}
+
+
+void Assembler::fcmge(const VRegister& vd,
+ const VRegister& vn,
+ double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
+}
+
+
+void Assembler::fcmgt(const VRegister& vd,
+ const VRegister& vn,
+ double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
+}
+
+
+void Assembler::fcmle(const VRegister& vd,
+ const VRegister& vn,
+ double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
+}
+
+
+void Assembler::fcmlt(const VRegister& vd,
+ const VRegister& vn,
+ double value) {
+ NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
+}
+
+
+void Assembler::frecpx(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsScalar());
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D());
+ Emit(FPFormat(vd) | NEON_FRECPX_scalar | Rn(vn) | Rd(vd));
+}
+
+
+#define NEON_3SAME_LIST(V) \
+ V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \
+ V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \
+ V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \
+ V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \
+ V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \
+ V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \
+ V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \
+ V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \
+ V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \
+ V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \
+ V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \
+ V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \
+ V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \
+ V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
+ V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
+ V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \
+ V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \
+ V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \
+ V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \
+ V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \
+ V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \
+ V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \
+ V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \
+ V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \
+ V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \
+ V(uqadd, NEON_UQADD, true) \
+ V(sqadd, NEON_SQADD, true) \
+ V(uqsub, NEON_UQSUB, true) \
+ V(sqsub, NEON_SQSUB, true) \
+ V(sqshl, NEON_SQSHL, true) \
+ V(uqshl, NEON_UQSHL, true) \
+ V(sqrshl, NEON_SQRSHL, true) \
+ V(uqrshl, NEON_UQRSHL, true)
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm) { \
+ VIXL_ASSERT(AS); \
+ NEON3Same(vd, vn, vm, OP); \
+}
+NEON_3SAME_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+
+#define NEON_FP3SAME_OP_LIST(V) \
+ V(fadd, NEON_FADD, FADD) \
+ V(fsub, NEON_FSUB, FSUB) \
+ V(fmul, NEON_FMUL, FMUL) \
+ V(fdiv, NEON_FDIV, FDIV) \
+ V(fmax, NEON_FMAX, FMAX) \
+ V(fmaxnm, NEON_FMAXNM, FMAXNM) \
+ V(fmin, NEON_FMIN, FMIN) \
+ V(fminnm, NEON_FMINNM, FMINNM) \
+ V(fmulx, NEON_FMULX, NEON_FMULX_scalar) \
+ V(frecps, NEON_FRECPS, NEON_FRECPS_scalar) \
+ V(frsqrts, NEON_FRSQRTS, NEON_FRSQRTS_scalar) \
+ V(fabd, NEON_FABD, NEON_FABD_scalar) \
+ V(fmla, NEON_FMLA, 0) \
+ V(fmls, NEON_FMLS, 0) \
+ V(facge, NEON_FACGE, NEON_FACGE_scalar) \
+ V(facgt, NEON_FACGT, NEON_FACGT_scalar) \
+ V(fcmeq, NEON_FCMEQ, NEON_FCMEQ_scalar) \
+ V(fcmge, NEON_FCMGE, NEON_FCMGE_scalar) \
+ V(fcmgt, NEON_FCMGT, NEON_FCMGT_scalar) \
+ V(faddp, NEON_FADDP, 0) \
+ V(fmaxp, NEON_FMAXP, 0) \
+ V(fminp, NEON_FMINP, 0) \
+ V(fmaxnmp, NEON_FMAXNMP, 0) \
+ V(fminnmp, NEON_FMINNMP, 0)
+
+#define DEFINE_ASM_FUNC(FN, VEC_OP, SCA_OP) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm) { \
+ Instr op; \
+ if ((SCA_OP != 0) && vd.IsScalar()) { \
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
+ op = SCA_OP; \
+ } else { \
+ VIXL_ASSERT(vd.IsVector()); \
+ VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
+ op = VEC_OP; \
+ } \
+ NEONFP3Same(vd, vn, vm, op); \
+}
+NEON_FP3SAME_OP_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+
+void Assembler::addp(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
+ Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::faddp(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
+ (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fmaxp(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
+ (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fminp(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
+ (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fmaxnmp(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
+ (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::fminnmp(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT((vd.Is1S() && vn.Is2S()) ||
+ (vd.Is1D() && vn.Is2D()));
+ Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::orr(const VRegister& vd,
+ const int imm8,
+ const int left_shift) {
+ NEONModifiedImmShiftLsl(vd, imm8, left_shift,
+ NEONModifiedImmediate_ORR);
+}
+
+
+void Assembler::mov(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ if (vd.IsD()) {
+ orr(vd.V8B(), vn.V8B(), vn.V8B());
+ } else {
+ VIXL_ASSERT(vd.IsQ());
+ orr(vd.V16B(), vn.V16B(), vn.V16B());
+ }
+}
+
+
+void Assembler::bic(const VRegister& vd,
+ const int imm8,
+ const int left_shift) {
+ NEONModifiedImmShiftLsl(vd, imm8, left_shift,
+ NEONModifiedImmediate_BIC);
+}
+
+
+void Assembler::movi(const VRegister& vd,
+ const uint64_t imm,
+ Shift shift,
+ const int shift_amount) {
+ VIXL_ASSERT((shift == LSL) || (shift == MSL));
+ if (vd.Is2D() || vd.Is1D()) {
+ VIXL_ASSERT(shift_amount == 0);
+ int imm8 = 0;
+ for (int i = 0; i < 8; ++i) {
+ int byte = (imm >> (i * 8)) & 0xff;
+ VIXL_ASSERT((byte == 0) || (byte == 0xff));
+ if (byte == 0xff) {
+ imm8 |= (1 << i);
+ }
+ }
+ int q = vd.Is2D() ? NEON_Q : 0;
+ Emit(q | NEONModImmOp(1) | NEONModifiedImmediate_MOVI |
+ ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
+ } else if (shift == LSL) {
+ VIXL_ASSERT(IsUint8(imm));
+ NEONModifiedImmShiftLsl(vd, static_cast<int>(imm), shift_amount,
+ NEONModifiedImmediate_MOVI);
+ } else {
+ VIXL_ASSERT(IsUint8(imm));
+ NEONModifiedImmShiftMsl(vd, static_cast<int>(imm), shift_amount,
+ NEONModifiedImmediate_MOVI);
+ }
+}
+
+
+void Assembler::mvn(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ if (vd.IsD()) {
+ not_(vd.V8B(), vn.V8B());
+ } else {
+ VIXL_ASSERT(vd.IsQ());
+ not_(vd.V16B(), vn.V16B());
+ }
+}
+
+
+void Assembler::mvni(const VRegister& vd,
+ const int imm8,
+ Shift shift,
+ const int shift_amount) {
+ VIXL_ASSERT((shift == LSL) || (shift == MSL));
+ if (shift == LSL) {
+ NEONModifiedImmShiftLsl(vd, imm8, shift_amount,
+ NEONModifiedImmediate_MVNI);
+ } else {
+ NEONModifiedImmShiftMsl(vd, imm8, shift_amount,
+ NEONModifiedImmediate_MVNI);
+ }
+}
+
+
+void Assembler::NEONFPByElement(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index,
+ NEONByIndexedElementOp vop) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT((vd.Is2S() && vm.Is1S()) ||
+ (vd.Is4S() && vm.Is1S()) ||
+ (vd.Is1S() && vm.Is1S()) ||
+ (vd.Is2D() && vm.Is1D()) ||
+ (vd.Is1D() && vm.Is1D()));
+ VIXL_ASSERT((vm.Is1S() && (vm_index < 4)) ||
+ (vm.Is1D() && (vm_index < 2)));
+
+ Instr op = vop;
+ int index_num_bits = vm.Is1S() ? 2 : 1;
+ if (vd.IsScalar()) {
+ op |= NEON_Q | NEONScalar;
+ }
+
+ Emit(FPFormat(vd) | op | ImmNEONHLM(vm_index, index_num_bits) |
+ Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEONByElement(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index,
+ NEONByIndexedElementOp vop) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT((vd.Is4H() && vm.Is1H()) ||
+ (vd.Is8H() && vm.Is1H()) ||
+ (vd.Is1H() && vm.Is1H()) ||
+ (vd.Is2S() && vm.Is1S()) ||
+ (vd.Is4S() && vm.Is1S()) ||
+ (vd.Is1S() && vm.Is1S()));
+ VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
+ (vm.Is1S() && (vm_index < 4)));
+
+ Instr format, op = vop;
+ int index_num_bits = vm.Is1H() ? 3 : 2;
+ if (vd.IsScalar()) {
+ op |= NEONScalar | NEON_Q;
+ format = SFormat(vn);
+ } else {
+ format = VFormat(vn);
+ }
+ Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) |
+ Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEONByElementL(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index,
+ NEONByIndexedElementOp vop) {
+ VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
+ (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
+ (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
+ (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
+ (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
+ (vd.Is1D() && vn.Is1S() && vm.Is1S()));
+
+ VIXL_ASSERT((vm.Is1H() && (vm.code() < 16) && (vm_index < 8)) ||
+ (vm.Is1S() && (vm_index < 4)));
+
+ Instr format, op = vop;
+ int index_num_bits = vm.Is1H() ? 3 : 2;
+ if (vd.IsScalar()) {
+ op |= NEONScalar | NEON_Q;
+ format = SFormat(vn);
+ } else {
+ format = VFormat(vn);
+ }
+ Emit(format | op | ImmNEONHLM(vm_index, index_num_bits) |
+ Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+#define NEON_BYELEMENT_LIST(V) \
+ V(mul, NEON_MUL_byelement, vn.IsVector()) \
+ V(mla, NEON_MLA_byelement, vn.IsVector()) \
+ V(mls, NEON_MLS_byelement, vn.IsVector()) \
+ V(sqdmulh, NEON_SQDMULH_byelement, true) \
+ V(sqrdmulh, NEON_SQRDMULH_byelement, true)
+
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm, \
+ int vm_index) { \
+ VIXL_ASSERT(AS); \
+ NEONByElement(vd, vn, vm, vm_index, OP); \
+}
+NEON_BYELEMENT_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+
+#define NEON_FPBYELEMENT_LIST(V) \
+ V(fmul, NEON_FMUL_byelement) \
+ V(fmla, NEON_FMLA_byelement) \
+ V(fmls, NEON_FMLS_byelement) \
+ V(fmulx, NEON_FMULX_byelement)
+
+
+#define DEFINE_ASM_FUNC(FN, OP) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm, \
+ int vm_index) { \
+ NEONFPByElement(vd, vn, vm, vm_index, OP); \
+}
+NEON_FPBYELEMENT_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+
+#define NEON_BYELEMENT_LONG_LIST(V) \
+ V(sqdmull, NEON_SQDMULL_byelement, vn.IsScalar() || vn.IsD()) \
+ V(sqdmull2, NEON_SQDMULL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(sqdmlal, NEON_SQDMLAL_byelement, vn.IsScalar() || vn.IsD()) \
+ V(sqdmlal2, NEON_SQDMLAL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(sqdmlsl, NEON_SQDMLSL_byelement, vn.IsScalar() || vn.IsD()) \
+ V(sqdmlsl2, NEON_SQDMLSL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(smull, NEON_SMULL_byelement, vn.IsVector() && vn.IsD()) \
+ V(smull2, NEON_SMULL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(umull, NEON_UMULL_byelement, vn.IsVector() && vn.IsD()) \
+ V(umull2, NEON_UMULL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(smlal, NEON_SMLAL_byelement, vn.IsVector() && vn.IsD()) \
+ V(smlal2, NEON_SMLAL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(umlal, NEON_UMLAL_byelement, vn.IsVector() && vn.IsD()) \
+ V(umlal2, NEON_UMLAL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(smlsl, NEON_SMLSL_byelement, vn.IsVector() && vn.IsD()) \
+ V(smlsl2, NEON_SMLSL_byelement, vn.IsVector() && vn.IsQ()) \
+ V(umlsl, NEON_UMLSL_byelement, vn.IsVector() && vn.IsD()) \
+ V(umlsl2, NEON_UMLSL_byelement, vn.IsVector() && vn.IsQ())
+
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm, \
+ int vm_index) { \
+ VIXL_ASSERT(AS); \
+ NEONByElementL(vd, vn, vm, vm_index, OP); \
+}
+NEON_BYELEMENT_LONG_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+
+void Assembler::suqadd(const VRegister& vd,
+ const VRegister& vn) {
+ NEON2RegMisc(vd, vn, NEON_SUQADD);
+}
+
+
+void Assembler::usqadd(const VRegister& vd,
+ const VRegister& vn) {
+ NEON2RegMisc(vd, vn, NEON_USQADD);
+}
+
+
+void Assembler::abs(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_ABS);
+}
+
+
+void Assembler::sqabs(const VRegister& vd,
+ const VRegister& vn) {
+ NEON2RegMisc(vd, vn, NEON_SQABS);
+}
+
+
+void Assembler::neg(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEON2RegMisc(vd, vn, NEON_NEG);
+}
+
+
+void Assembler::sqneg(const VRegister& vd,
+ const VRegister& vn) {
+ NEON2RegMisc(vd, vn, NEON_SQNEG);
+}
+
+
+void Assembler::NEONXtn(const VRegister& vd,
+ const VRegister& vn,
+ NEON2RegMiscOp vop) {
+ Instr format, op = vop;
+ if (vd.IsScalar()) {
+ VIXL_ASSERT((vd.Is1B() && vn.Is1H()) ||
+ (vd.Is1H() && vn.Is1S()) ||
+ (vd.Is1S() && vn.Is1D()));
+ op |= NEON_Q | NEONScalar;
+ format = SFormat(vd);
+ } else {
+ VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
+ (vd.Is4H() && vn.Is4S()) ||
+ (vd.Is2S() && vn.Is2D()) ||
+ (vd.Is16B() && vn.Is8H()) ||
+ (vd.Is8H() && vn.Is4S()) ||
+ (vd.Is4S() && vn.Is2D()));
+ format = VFormat(vd);
+ }
+ Emit(format | op | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::xtn(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsVector() && vd.IsD());
+ NEONXtn(vd, vn, NEON_XTN);
+}
+
+
+void Assembler::xtn2(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsVector() && vd.IsQ());
+ NEONXtn(vd, vn, NEON_XTN);
+}
+
+
+void Assembler::sqxtn(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsScalar() || vd.IsD());
+ NEONXtn(vd, vn, NEON_SQXTN);
+}
+
+
+void Assembler::sqxtn2(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsVector() && vd.IsQ());
+ NEONXtn(vd, vn, NEON_SQXTN);
+}
+
+
+void Assembler::sqxtun(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsScalar() || vd.IsD());
+ NEONXtn(vd, vn, NEON_SQXTUN);
+}
+
+
+void Assembler::sqxtun2(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsVector() && vd.IsQ());
+ NEONXtn(vd, vn, NEON_SQXTUN);
+}
+
+
+void Assembler::uqxtn(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsScalar() || vd.IsD());
+ NEONXtn(vd, vn, NEON_UQXTN);
+}
+
+
+void Assembler::uqxtn2(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(vd.IsVector() && vd.IsQ());
+ NEONXtn(vd, vn, NEON_UQXTN);
+}
+
+
+// NEON NOT and RBIT are distinguised by bit 22, the bottom bit of "size".
+void Assembler::not_(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(vd.Is8B() || vd.Is16B());
+ Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::rbit(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(vd.Is8B() || vd.Is16B());
+ Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::ext(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int index) {
+ VIXL_ASSERT(AreSameFormat(vd, vn, vm));
+ VIXL_ASSERT(vd.Is8B() || vd.Is16B());
+ VIXL_ASSERT((0 <= index) && (index < vd.lanes()));
+ Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::dup(const VRegister& vd,
+ const VRegister& vn,
+ int vn_index) {
+ Instr q, scalar;
+
+ // We support vn arguments of the form vn.VxT() or vn.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ int lane_size = vn.LaneSizeInBytes();
+ NEONFormatField format;
+ switch (lane_size) {
+ case 1: format = NEON_16B; break;
+ case 2: format = NEON_8H; break;
+ case 4: format = NEON_4S; break;
+ default:
+ VIXL_ASSERT(lane_size == 8);
+ format = NEON_2D;
+ break;
+ }
+
+ if (vd.IsScalar()) {
+ q = NEON_Q;
+ scalar = NEONScalar;
+ } else {
+ VIXL_ASSERT(!vd.Is1D());
+ q = vd.IsD() ? 0 : NEON_Q;
+ scalar = 0;
+ }
+ Emit(q | scalar | NEON_DUP_ELEMENT |
+ ImmNEON5(format, vn_index) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::mov(const VRegister& vd,
+ const VRegister& vn,
+ int vn_index) {
+ VIXL_ASSERT(vn.IsScalar());
+ dup(vd, vn, vn_index);
+}
+
+
+void Assembler::dup(const VRegister& vd, const Register& rn) {
+ VIXL_ASSERT(!vd.Is1D());
+ VIXL_ASSERT(vd.Is2D() == rn.IsX());
+ int q = vd.IsD() ? 0 : NEON_Q;
+ Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
+}
+
+
+void Assembler::ins(const VRegister& vd,
+ int vd_index,
+ const VRegister& vn,
+ int vn_index) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ int lane_size = vd.LaneSizeInBytes();
+ NEONFormatField format;
+ switch (lane_size) {
+ case 1: format = NEON_16B; break;
+ case 2: format = NEON_8H; break;
+ case 4: format = NEON_4S; break;
+ default:
+ VIXL_ASSERT(lane_size == 8);
+ format = NEON_2D;
+ break;
+ }
+
+ VIXL_ASSERT((0 <= vd_index) &&
+ (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ VIXL_ASSERT((0 <= vn_index) &&
+ (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ Emit(NEON_INS_ELEMENT | ImmNEON5(format, vd_index) |
+ ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::mov(const VRegister& vd,
+ int vd_index,
+ const VRegister& vn,
+ int vn_index) {
+ ins(vd, vd_index, vn, vn_index);
+}
+
+
+void Assembler::ins(const VRegister& vd,
+ int vd_index,
+ const Register& rn) {
+ // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ int lane_size = vd.LaneSizeInBytes();
+ NEONFormatField format;
+ switch (lane_size) {
+ case 1: format = NEON_16B; VIXL_ASSERT(rn.IsW()); break;
+ case 2: format = NEON_8H; VIXL_ASSERT(rn.IsW()); break;
+ case 4: format = NEON_4S; VIXL_ASSERT(rn.IsW()); break;
+ default:
+ VIXL_ASSERT(lane_size == 8);
+ VIXL_ASSERT(rn.IsX());
+ format = NEON_2D;
+ break;
+ }
+
+ VIXL_ASSERT((0 <= vd_index) &&
+ (vd_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
+}
+
+
+void Assembler::mov(const VRegister& vd,
+ int vd_index,
+ const Register& rn) {
+ ins(vd, vd_index, rn);
+}
+
+
+void Assembler::umov(const Register& rd,
+ const VRegister& vn,
+ int vn_index) {
+ // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
+ // number of lanes, and T is b, h, s or d.
+ int lane_size = vn.LaneSizeInBytes();
+ NEONFormatField format;
+ Instr q = 0;
+ switch (lane_size) {
+ case 1: format = NEON_16B; VIXL_ASSERT(rd.IsW()); break;
+ case 2: format = NEON_8H; VIXL_ASSERT(rd.IsW()); break;
+ case 4: format = NEON_4S; VIXL_ASSERT(rd.IsW()); break;
+ default:
+ VIXL_ASSERT(lane_size == 8);
+ VIXL_ASSERT(rd.IsX());
+ format = NEON_2D;
+ q = NEON_Q;
+ break;
+ }
+
+ VIXL_ASSERT((0 <= vn_index) &&
+ (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ Emit(q | NEON_UMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
+}
+
+
+void Assembler::mov(const Register& rd,
+ const VRegister& vn,
+ int vn_index) {
+ VIXL_ASSERT(vn.SizeInBytes() >= 4);
+ umov(rd, vn, vn_index);
+}
+
+
+void Assembler::smov(const Register& rd,
+ const VRegister& vn,
+ int vn_index) {
+ // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
+ // number of lanes, and T is b, h, s.
+ int lane_size = vn.LaneSizeInBytes();
+ NEONFormatField format;
+ Instr q = 0;
+ VIXL_ASSERT(lane_size != 8);
+ switch (lane_size) {
+ case 1: format = NEON_16B; break;
+ case 2: format = NEON_8H; break;
+ default:
+ VIXL_ASSERT(lane_size == 4);
+ VIXL_ASSERT(rd.IsX());
+ format = NEON_4S;
+ break;
+ }
+ q = rd.IsW() ? 0 : NEON_Q;
+ VIXL_ASSERT((0 <= vn_index) &&
+ (vn_index < LaneCountFromFormat(static_cast<VectorFormat>(format))));
+ Emit(q | NEON_SMOV | ImmNEON5(format, vn_index) | Rn(vn) | Rd(rd));
+}
+
+
+void Assembler::cls(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
+ Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::clz(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
+ Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::cnt(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(vd.Is8B() || vd.Is16B());
+ Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::rev16(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(vd.Is8B() || vd.Is16B());
+ Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::rev32(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
+ Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::rev64(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
+ Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::ursqrte(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(vd.Is2S() || vd.Is4S());
+ Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::urecpe(const VRegister& vd,
+ const VRegister& vn) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ VIXL_ASSERT(vd.Is2S() || vd.Is4S());
+ Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEONAddlp(const VRegister& vd,
+ const VRegister& vn,
+ NEON2RegMiscOp op) {
+ VIXL_ASSERT((op == NEON_SADDLP) ||
+ (op == NEON_UADDLP) ||
+ (op == NEON_SADALP) ||
+ (op == NEON_UADALP));
+
+ VIXL_ASSERT((vn.Is8B() && vd.Is4H()) ||
+ (vn.Is4H() && vd.Is2S()) ||
+ (vn.Is2S() && vd.Is1D()) ||
+ (vn.Is16B() && vd.Is8H())||
+ (vn.Is8H() && vd.Is4S()) ||
+ (vn.Is4S() && vd.Is2D()));
+ Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::saddlp(const VRegister& vd,
+ const VRegister& vn) {
+ NEONAddlp(vd, vn, NEON_SADDLP);
+}
+
+
+void Assembler::uaddlp(const VRegister& vd,
+ const VRegister& vn) {
+ NEONAddlp(vd, vn, NEON_UADDLP);
+}
+
+
+void Assembler::sadalp(const VRegister& vd,
+ const VRegister& vn) {
+ NEONAddlp(vd, vn, NEON_SADALP);
+}
+
+
+void Assembler::uadalp(const VRegister& vd,
+ const VRegister& vn) {
+ NEONAddlp(vd, vn, NEON_UADALP);
+}
+
+
+void Assembler::NEONAcrossLanesL(const VRegister& vd,
+ const VRegister& vn,
+ NEONAcrossLanesOp op) {
+ VIXL_ASSERT((vn.Is8B() && vd.Is1H()) ||
+ (vn.Is16B() && vd.Is1H()) ||
+ (vn.Is4H() && vd.Is1S()) ||
+ (vn.Is8H() && vd.Is1S()) ||
+ (vn.Is4S() && vd.Is1D()));
+ Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::saddlv(const VRegister& vd,
+ const VRegister& vn) {
+ NEONAcrossLanesL(vd, vn, NEON_SADDLV);
+}
+
+
+void Assembler::uaddlv(const VRegister& vd,
+ const VRegister& vn) {
+ NEONAcrossLanesL(vd, vn, NEON_UADDLV);
+}
+
+
+void Assembler::NEONAcrossLanes(const VRegister& vd,
+ const VRegister& vn,
+ NEONAcrossLanesOp op) {
+ VIXL_ASSERT((vn.Is8B() && vd.Is1B()) ||
+ (vn.Is16B() && vd.Is1B()) ||
+ (vn.Is4H() && vd.Is1H()) ||
+ (vn.Is8H() && vd.Is1H()) ||
+ (vn.Is4S() && vd.Is1S()));
+ if ((op & NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
+ Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
+ } else {
+ Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
+ }
+}
+
+
+#define NEON_ACROSSLANES_LIST(V) \
+ V(fmaxv, NEON_FMAXV, vd.Is1S()) \
+ V(fminv, NEON_FMINV, vd.Is1S()) \
+ V(fmaxnmv, NEON_FMAXNMV, vd.Is1S()) \
+ V(fminnmv, NEON_FMINNMV, vd.Is1S()) \
+ V(addv, NEON_ADDV, true) \
+ V(smaxv, NEON_SMAXV, true) \
+ V(sminv, NEON_SMINV, true) \
+ V(umaxv, NEON_UMAXV, true) \
+ V(uminv, NEON_UMINV, true)
+
+
+#define DEFINE_ASM_FUNC(FN, OP, AS) \
+void Assembler::FN(const VRegister& vd, \
+ const VRegister& vn) { \
+ VIXL_ASSERT(AS); \
+ NEONAcrossLanes(vd, vn, OP); \
+}
+NEON_ACROSSLANES_LIST(DEFINE_ASM_FUNC)
+#undef DEFINE_ASM_FUNC
+
+
+void Assembler::NEONPerm(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEONPermOp op) {
+ VIXL_ASSERT(AreSameFormat(vd, vn, vm));
+ VIXL_ASSERT(!vd.Is1D());
+ Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::trn1(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_TRN1);
+}
+
+
+void Assembler::trn2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_TRN2);
+}
+
+
+void Assembler::uzp1(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_UZP1);
+}
+
+
+void Assembler::uzp2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_UZP2);
+}
+
+
+void Assembler::zip1(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_ZIP1);
+}
+
+
+void Assembler::zip2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ NEONPerm(vd, vn, vm, NEON_ZIP2);
+}
+
+
+void Assembler::NEONShiftImmediate(const VRegister& vd,
+ const VRegister& vn,
+ NEONShiftImmediateOp op,
+ int immh_immb) {
+ VIXL_ASSERT(AreSameFormat(vd, vn));
+ Instr q, scalar;
+ if (vn.IsScalar()) {
+ q = NEON_Q;
+ scalar = NEONScalar;
+ } else {
+ q = vd.IsD() ? 0 : NEON_Q;
+ scalar = 0;
+ }
+ Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
+ const VRegister& vn,
+ int shift,
+ NEONShiftImmediateOp op) {
+ int laneSizeInBits = vn.LaneSizeInBits();
+ VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
+ NEONShiftImmediate(vd, vn, op, (laneSizeInBits + shift) << 16);
+}
+
+
+void Assembler::NEONShiftRightImmediate(const VRegister& vd,
+ const VRegister& vn,
+ int shift,
+ NEONShiftImmediateOp op) {
+ int laneSizeInBits = vn.LaneSizeInBits();
+ VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
+ NEONShiftImmediate(vd, vn, op, ((2 * laneSizeInBits) - shift) << 16);
+}
+
+
+void Assembler::NEONShiftImmediateL(const VRegister& vd,
+ const VRegister& vn,
+ int shift,
+ NEONShiftImmediateOp op) {
+ int laneSizeInBits = vn.LaneSizeInBits();
+ VIXL_ASSERT((shift >= 0) && (shift < laneSizeInBits));
+ int immh_immb = (laneSizeInBits + shift) << 16;
+
+ VIXL_ASSERT((vn.Is8B() && vd.Is8H()) ||
+ (vn.Is4H() && vd.Is4S()) ||
+ (vn.Is2S() && vd.Is2D()) ||
+ (vn.Is16B() && vd.Is8H())||
+ (vn.Is8H() && vd.Is4S()) ||
+ (vn.Is4S() && vd.Is2D()));
+ Instr q;
+ q = vn.IsD() ? 0 : NEON_Q;
+ Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::NEONShiftImmediateN(const VRegister& vd,
+ const VRegister& vn,
+ int shift,
+ NEONShiftImmediateOp op) {
+ Instr q, scalar;
+ int laneSizeInBits = vd.LaneSizeInBits();
+ VIXL_ASSERT((shift >= 1) && (shift <= laneSizeInBits));
+ int immh_immb = (2 * laneSizeInBits - shift) << 16;
+
+ if (vn.IsScalar()) {
+ VIXL_ASSERT((vd.Is1B() && vn.Is1H()) ||
+ (vd.Is1H() && vn.Is1S()) ||
+ (vd.Is1S() && vn.Is1D()));
+ q = NEON_Q;
+ scalar = NEONScalar;
+ } else {
+ VIXL_ASSERT((vd.Is8B() && vn.Is8H()) ||
+ (vd.Is4H() && vn.Is4S()) ||
+ (vd.Is2S() && vn.Is2D()) ||
+ (vd.Is16B() && vn.Is8H())||
+ (vd.Is8H() && vn.Is4S()) ||
+ (vd.Is4S() && vn.Is2D()));
+ scalar = 0;
+ q = vd.IsD() ? 0 : NEON_Q;
+ }
+ Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::shl(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
+}
+
+
+void Assembler::sli(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
+}
+
+
+void Assembler::sqshl(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
+}
+
+
+void Assembler::sqshlu(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
+}
+
+
+void Assembler::uqshl(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
+}
+
+
+void Assembler::sshll(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsD());
+ NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
+}
+
+
+void Assembler::sshll2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsQ());
+ NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
+}
+
+
+void Assembler::sxtl(const VRegister& vd,
+ const VRegister& vn) {
+ sshll(vd, vn, 0);
+}
+
+
+void Assembler::sxtl2(const VRegister& vd,
+ const VRegister& vn) {
+ sshll2(vd, vn, 0);
+}
+
+
+void Assembler::ushll(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsD());
+ NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
+}
+
+
+void Assembler::ushll2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsQ());
+ NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
+}
+
+
+void Assembler::uxtl(const VRegister& vd,
+ const VRegister& vn) {
+ ushll(vd, vn, 0);
+}
+
+
+void Assembler::uxtl2(const VRegister& vd,
+ const VRegister& vn) {
+ ushll2(vd, vn, 0);
+}
+
+
+void Assembler::sri(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
+}
+
+
+void Assembler::sshr(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
+}
+
+
+void Assembler::ushr(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
+}
+
+
+void Assembler::srshr(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
+}
+
+
+void Assembler::urshr(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
+}
+
+
+void Assembler::ssra(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
+}
+
+
+void Assembler::usra(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
+}
+
+
+void Assembler::srsra(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
+}
+
+
+void Assembler::ursra(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsVector() || vd.Is1D());
+ NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
+}
+
+
+void Assembler::shrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsD());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
+}
+
+
+void Assembler::shrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
+}
+
+
+void Assembler::rshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsD());
+ NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
+}
+
+
+void Assembler::rshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
+}
+
+
+void Assembler::sqshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
+}
+
+
+void Assembler::sqshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
+}
+
+
+void Assembler::sqrshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
+}
+
+
+void Assembler::sqrshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
+}
+
+
+void Assembler::sqshrun(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
+}
+
+
+void Assembler::sqshrun2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
+}
+
+
+void Assembler::sqrshrun(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
+}
+
+
+void Assembler::sqrshrun2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
+}
+
+
+void Assembler::uqshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
+}
+
+
+void Assembler::uqshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
+}
+
+
+void Assembler::uqrshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
+ NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
+}
+
+
+void Assembler::uqrshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift) {
+ VIXL_ASSERT(vn.IsVector() && vd.IsQ());
+ NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
+}
+
+
+// Note:
+// Below, a difference in case for the same letter indicates a
+// negated bit.
+// If b is 1, then B is 0.
+uint32_t Assembler::FP32ToImm8(float imm) {
+ VIXL_ASSERT(IsImmFP32(imm));
+ // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
+ uint32_t bits = FloatToRawbits(imm);
+ // bit7: a000.0000
+ uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
+ // bit6: 0b00.0000
+ uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
+ // bit5_to_0: 00cd.efgh
+ uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
+
+ return bit7 | bit6 | bit5_to_0;
+}
+
+
+Instr Assembler::ImmFP32(float imm) {
+ return FP32ToImm8(imm) << ImmFP_offset;
+}
+
+
+uint32_t Assembler::FP64ToImm8(double imm) {
+ VIXL_ASSERT(IsImmFP64(imm));
+ // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000
+ uint64_t bits = DoubleToRawbits(imm);
+ // bit7: a000.0000
+ uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
+ // bit6: 0b00.0000
+ uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
+ // bit5_to_0: 00cd.efgh
+ uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
+
+ return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
+}
+
+
+Instr Assembler::ImmFP64(double imm) {
+ return FP64ToImm8(imm) << ImmFP_offset;
+}
+
+
+// Code generation helpers.
+void Assembler::MoveWide(const Register& rd,
+ uint64_t imm,
+ int shift,
+ MoveWideImmediateOp mov_op) {
+ // Ignore the top 32 bits of an immediate if we're moving to a W register.
+ if (rd.Is32Bits()) {
+ // Check that the top 32 bits are zero (a positive 32-bit number) or top
+ // 33 bits are one (a negative 32-bit number, sign extended to 64 bits).
+ VIXL_ASSERT(((imm >> kWRegSize) == 0) ||
+ ((imm >> (kWRegSize - 1)) == 0x1ffffffff));
+ imm &= kWRegMask;
+ }
+
+ if (shift >= 0) {
+ // Explicit shift specified.
+ VIXL_ASSERT((shift == 0) || (shift == 16) ||
+ (shift == 32) || (shift == 48));
+ VIXL_ASSERT(rd.Is64Bits() || (shift == 0) || (shift == 16));
+ shift /= 16;
+ } else {
+ // Calculate a new immediate and shift combination to encode the immediate
+ // argument.
+ shift = 0;
+ if ((imm & 0xffffffffffff0000) == 0) {
+ // Nothing to do.
+ } else if ((imm & 0xffffffff0000ffff) == 0) {
+ imm >>= 16;
+ shift = 1;
+ } else if ((imm & 0xffff0000ffffffff) == 0) {
+ VIXL_ASSERT(rd.Is64Bits());
+ imm >>= 32;
+ shift = 2;
+ } else if ((imm & 0x0000ffffffffffff) == 0) {
+ VIXL_ASSERT(rd.Is64Bits());
+ imm >>= 48;
+ shift = 3;
+ }
+ }
+
+ VIXL_ASSERT(IsUint16(imm));
+
+ Emit(SF(rd) | MoveWideImmediateFixed | mov_op |
+ Rd(rd) | ImmMoveWide(imm) | ShiftMoveWide(shift));
+}
+
+
+void Assembler::AddSub(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ AddSubOp op) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ if (operand.IsImmediate()) {
+ int64_t immediate = operand.immediate();
+ VIXL_ASSERT(IsImmAddSub(immediate));
+ Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
+ Emit(SF(rd) | AddSubImmediateFixed | op | Flags(S) |
+ ImmAddSub(static_cast<int>(immediate)) | dest_reg | RnSP(rn));
+ } else if (operand.IsShiftedRegister()) {
+ VIXL_ASSERT(operand.reg().size() == rd.size());
+ VIXL_ASSERT(operand.shift() != ROR);
+
+ // For instructions of the form:
+ // add/sub wsp, <Wn>, <Wm> [, LSL #0-3 ]
+ // add/sub <Wd>, wsp, <Wm> [, LSL #0-3 ]
+ // add/sub wsp, wsp, <Wm> [, LSL #0-3 ]
+ // adds/subs <Wd>, wsp, <Wm> [, LSL #0-3 ]
+ // or their 64-bit register equivalents, convert the operand from shifted to
+ // extended register mode, and emit an add/sub extended instruction.
+ if (rn.IsSP() || rd.IsSP()) {
+ VIXL_ASSERT(!(rd.IsSP() && (S == SetFlags)));
+ DataProcExtendedRegister(rd, rn, operand.ToExtendedRegister(), S,
+ AddSubExtendedFixed | op);
+ } else {
+ DataProcShiftedRegister(rd, rn, operand, S, AddSubShiftedFixed | op);
+ }
+ } else {
+ VIXL_ASSERT(operand.IsExtendedRegister());
+ DataProcExtendedRegister(rd, rn, operand, S, AddSubExtendedFixed | op);
+ }
+}
+
+
+void Assembler::AddSubWithCarry(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ AddSubWithCarryOp op) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ VIXL_ASSERT(rd.size() == operand.reg().size());
+ VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
+ Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::hlt(int code) {
+ VIXL_ASSERT(IsUint16(code));
+ Emit(HLT | ImmException(code));
+}
+
+
+void Assembler::brk(int code) {
+ VIXL_ASSERT(IsUint16(code));
+ Emit(BRK | ImmException(code));
+}
+
+
+void Assembler::svc(int code) {
+ Emit(SVC | ImmException(code));
+}
+
+
+void Assembler::ConditionalCompare(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond,
+ ConditionalCompareOp op) {
+ Instr ccmpop;
+ if (operand.IsImmediate()) {
+ int64_t immediate = operand.immediate();
+ VIXL_ASSERT(IsImmConditionalCompare(immediate));
+ ccmpop = ConditionalCompareImmediateFixed | op |
+ ImmCondCmp(static_cast<unsigned>(immediate));
+ } else {
+ VIXL_ASSERT(operand.IsShiftedRegister() && (operand.shift_amount() == 0));
+ ccmpop = ConditionalCompareRegisterFixed | op | Rm(operand.reg());
+ }
+ Emit(SF(rn) | ccmpop | Cond(cond) | Rn(rn) | Nzcv(nzcv));
+}
+
+
+void Assembler::DataProcessing1Source(const Register& rd,
+ const Register& rn,
+ DataProcessing1SourceOp op) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ Emit(SF(rn) | op | Rn(rn) | Rd(rd));
+}
+
+
+void Assembler::FPDataProcessing1Source(const VRegister& vd,
+ const VRegister& vn,
+ FPDataProcessing1SourceOp op) {
+ VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
+ Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
+}
+
+
+void Assembler::FPDataProcessing3Source(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va,
+ FPDataProcessing3SourceOp op) {
+ VIXL_ASSERT(vd.Is1S() || vd.Is1D());
+ VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
+ Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
+}
+
+
+void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
+ const int imm8,
+ const int left_shift,
+ NEONModifiedImmediateOp op) {
+ VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() ||
+ vd.Is2S() || vd.Is4S());
+ VIXL_ASSERT((left_shift == 0) || (left_shift == 8) ||
+ (left_shift == 16) || (left_shift == 24));
+ VIXL_ASSERT(IsUint8(imm8));
+
+ int cmode_1, cmode_2, cmode_3;
+ if (vd.Is8B() || vd.Is16B()) {
+ VIXL_ASSERT(op == NEONModifiedImmediate_MOVI);
+ cmode_1 = 1;
+ cmode_2 = 1;
+ cmode_3 = 1;
+ } else {
+ cmode_1 = (left_shift >> 3) & 1;
+ cmode_2 = left_shift >> 4;
+ cmode_3 = 0;
+ if (vd.Is4H() || vd.Is8H()) {
+ VIXL_ASSERT((left_shift == 0) || (left_shift == 8));
+ cmode_3 = 1;
+ }
+ }
+ int cmode = (cmode_3 << 3) | (cmode_2 << 2) | (cmode_1 << 1);
+
+ int q = vd.IsQ() ? NEON_Q : 0;
+
+ Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
+}
+
+
+void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
+ const int imm8,
+ const int shift_amount,
+ NEONModifiedImmediateOp op) {
+ VIXL_ASSERT(vd.Is2S() || vd.Is4S());
+ VIXL_ASSERT((shift_amount == 8) || (shift_amount == 16));
+ VIXL_ASSERT(IsUint8(imm8));
+
+ int cmode_0 = (shift_amount >> 4) & 1;
+ int cmode = 0xc | cmode_0;
+
+ int q = vd.IsQ() ? NEON_Q : 0;
+
+ Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
+}
+
+
+void Assembler::EmitShift(const Register& rd,
+ const Register& rn,
+ Shift shift,
+ unsigned shift_amount) {
+ switch (shift) {
+ case LSL:
+ lsl(rd, rn, shift_amount);
+ break;
+ case LSR:
+ lsr(rd, rn, shift_amount);
+ break;
+ case ASR:
+ asr(rd, rn, shift_amount);
+ break;
+ case ROR:
+ ror(rd, rn, shift_amount);
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+}
+
+
+void Assembler::EmitExtendShift(const Register& rd,
+ const Register& rn,
+ Extend extend,
+ unsigned left_shift) {
+ VIXL_ASSERT(rd.size() >= rn.size());
+ unsigned reg_size = rd.size();
+ // Use the correct size of register.
+ Register rn_ = Register(rn.code(), rd.size());
+ // Bits extracted are high_bit:0.
+ unsigned high_bit = (8 << (extend & 0x3)) - 1;
+ // Number of bits left in the result that are not introduced by the shift.
+ unsigned non_shift_bits = (reg_size - left_shift) & (reg_size - 1);
+
+ if ((non_shift_bits > high_bit) || (non_shift_bits == 0)) {
+ switch (extend) {
+ case UXTB:
+ case UXTH:
+ case UXTW: ubfm(rd, rn_, non_shift_bits, high_bit); break;
+ case SXTB:
+ case SXTH:
+ case SXTW: sbfm(rd, rn_, non_shift_bits, high_bit); break;
+ case UXTX:
+ case SXTX: {
+ VIXL_ASSERT(rn.size() == kXRegSize);
+ // Nothing to extend. Just shift.
+ lsl(rd, rn_, left_shift);
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ } else {
+ // No need to extend as the extended bits would be shifted away.
+ lsl(rd, rn_, left_shift);
+ }
+}
+
+
+void Assembler::DataProcExtendedRegister(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ Instr op) {
+ Instr dest_reg = (S == SetFlags) ? Rd(rd) : RdSP(rd);
+ Emit(SF(rd) | op | Flags(S) | Rm(operand.reg()) |
+ ExtendMode(operand.extend()) | ImmExtendShift(operand.shift_amount()) |
+ dest_reg | RnSP(rn));
+}
+
+
+Instr Assembler::LoadStoreMemOperand(const MemOperand& addr,
+ unsigned access_size,
+ LoadStoreScalingOption option) {
+ Instr base = RnSP(addr.base());
+ int64_t offset = addr.offset();
+
+ if (addr.IsImmediateOffset()) {
+ bool prefer_unscaled = (option == PreferUnscaledOffset) ||
+ (option == RequireUnscaledOffset);
+ if (prefer_unscaled && IsImmLSUnscaled(offset)) {
+ // Use the unscaled addressing mode.
+ return base | LoadStoreUnscaledOffsetFixed |
+ ImmLS(static_cast<int>(offset));
+ }
+
+ if ((option != RequireUnscaledOffset) &&
+ IsImmLSScaled(offset, access_size)) {
+ // Use the scaled addressing mode.
+ return base | LoadStoreUnsignedOffsetFixed |
+ ImmLSUnsigned(static_cast<int>(offset) >> access_size);
+ }
+
+ if ((option != RequireScaledOffset) && IsImmLSUnscaled(offset)) {
+ // Use the unscaled addressing mode.
+ return base | LoadStoreUnscaledOffsetFixed |
+ ImmLS(static_cast<int>(offset));
+ }
+ }
+
+ // All remaining addressing modes are register-offset, pre-indexed or
+ // post-indexed modes.
+ VIXL_ASSERT((option != RequireUnscaledOffset) &&
+ (option != RequireScaledOffset));
+
+ if (addr.IsRegisterOffset()) {
+ Extend ext = addr.extend();
+ Shift shift = addr.shift();
+ unsigned shift_amount = addr.shift_amount();
+
+ // LSL is encoded in the option field as UXTX.
+ if (shift == LSL) {
+ ext = UXTX;
+ }
+
+ // Shifts are encoded in one bit, indicating a left shift by the memory
+ // access size.
+ VIXL_ASSERT((shift_amount == 0) || (shift_amount == access_size));
+ return base | LoadStoreRegisterOffsetFixed | Rm(addr.regoffset()) |
+ ExtendMode(ext) | ImmShiftLS((shift_amount > 0) ? 1 : 0);
+ }
+
+ if (addr.IsPreIndex() && IsImmLSUnscaled(offset)) {
+ return base | LoadStorePreIndexFixed | ImmLS(static_cast<int>(offset));
+ }
+
+ if (addr.IsPostIndex() && IsImmLSUnscaled(offset)) {
+ return base | LoadStorePostIndexFixed | ImmLS(static_cast<int>(offset));
+ }
+
+ // If this point is reached, the MemOperand (addr) cannot be encoded.
+ VIXL_UNREACHABLE();
+ return 0;
+}
+
+
+void Assembler::LoadStore(const CPURegister& rt,
+ const MemOperand& addr,
+ LoadStoreOp op,
+ LoadStoreScalingOption option) {
+ Emit(op | Rt(rt) | LoadStoreMemOperand(addr, CalcLSDataSize(op), option));
+}
+
+
+void Assembler::Prefetch(PrefetchOperation op,
+ const MemOperand& addr,
+ LoadStoreScalingOption option) {
+ VIXL_ASSERT(addr.IsRegisterOffset() || addr.IsImmediateOffset());
+
+ Instr prfop = ImmPrefetchOperation(op);
+ Emit(PRFM | prfop | LoadStoreMemOperand(addr, kXRegSizeInBytesLog2, option));
+}
+
+
+bool Assembler::IsImmAddSub(int64_t immediate) {
+ return IsUint12(immediate) ||
+ (IsUint12(immediate >> 12) && ((immediate & 0xfff) == 0));
+}
+
+
+bool Assembler::IsImmConditionalCompare(int64_t immediate) {
+ return IsUint5(immediate);
+}
+
+
+bool Assembler::IsImmFP32(float imm) {
+ // Valid values will have the form:
+ // aBbb.bbbc.defg.h000.0000.0000.0000.0000
+ uint32_t bits = FloatToRawbits(imm);
+ // bits[19..0] are cleared.
+ if ((bits & 0x7ffff) != 0) {
+ return false;
+ }
+
+ // bits[29..25] are all set or all cleared.
+ uint32_t b_pattern = (bits >> 16) & 0x3e00;
+ if (b_pattern != 0 && b_pattern != 0x3e00) {
+ return false;
+ }
+
+ // bit[30] and bit[29] are opposite.
+ if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
+ return false;
+ }
+
+ return true;
+}
+
+
+bool Assembler::IsImmFP64(double imm) {
+ // Valid values will have the form:
+ // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000
+ uint64_t bits = DoubleToRawbits(imm);
+ // bits[47..0] are cleared.
+ if ((bits & 0x0000ffffffffffff) != 0) {
+ return false;
+ }
+
+ // bits[61..54] are all set or all cleared.
+ uint32_t b_pattern = (bits >> 48) & 0x3fc0;
+ if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
+ return false;
+ }
+
+ // bit[62] and bit[61] are opposite.
+ if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
+ return false;
+ }
+
+ return true;
+}
+
+
+bool Assembler::IsImmLSPair(int64_t offset, unsigned access_size) {
+ VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
+ bool offset_is_size_multiple =
+ (((offset >> access_size) << access_size) == offset);
+ return offset_is_size_multiple && IsInt7(offset >> access_size);
+}
+
+
+bool Assembler::IsImmLSScaled(int64_t offset, unsigned access_size) {
+ VIXL_ASSERT(access_size <= kQRegSizeInBytesLog2);
+ bool offset_is_size_multiple =
+ (((offset >> access_size) << access_size) == offset);
+ return offset_is_size_multiple && IsUint12(offset >> access_size);
+}
+
+
+bool Assembler::IsImmLSUnscaled(int64_t offset) {
+ return IsInt9(offset);
+}
+
+
+// The movn instruction can generate immediates containing an arbitrary 16-bit
+// value, with remaining bits set, eg. 0xffff1234, 0xffff1234ffffffff.
+bool Assembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
+ return IsImmMovz(~imm, reg_size);
+}
+
+
+// The movz instruction can generate immediates containing an arbitrary 16-bit
+// value, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
+bool Assembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
+ VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
+ return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
+}
+
+
+// Test if a given value can be encoded in the immediate field of a logical
+// instruction.
+// If it can be encoded, the function returns true, and values pointed to by n,
+// imm_s and imm_r are updated with immediates encoded in the format required
+// by the corresponding fields in the logical instruction.
+// If it can not be encoded, the function returns false, and the values pointed
+// to by n, imm_s and imm_r are undefined.
+bool Assembler::IsImmLogical(uint64_t value,
+ unsigned width,
+ unsigned* n,
+ unsigned* imm_s,
+ unsigned* imm_r) {
+ VIXL_ASSERT((width == kWRegSize) || (width == kXRegSize));
+
+ bool negate = false;
+
+ // Logical immediates are encoded using parameters n, imm_s and imm_r using
+ // the following table:
+ //
+ // N imms immr size S R
+ // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
+ // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
+ // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
+ // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
+ // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
+ // 0 11110s xxxxxr 2 UInt(s) UInt(r)
+ // (s bits must not be all set)
+ //
+ // A pattern is constructed of size bits, where the least significant S+1 bits
+ // are set. The pattern is rotated right by R, and repeated across a 32 or
+ // 64-bit value, depending on destination register width.
+ //
+ // Put another way: the basic format of a logical immediate is a single
+ // contiguous stretch of 1 bits, repeated across the whole word at intervals
+ // given by a power of 2. To identify them quickly, we first locate the
+ // lowest stretch of 1 bits, then the next 1 bit above that; that combination
+ // is different for every logical immediate, so it gives us all the
+ // information we need to identify the only logical immediate that our input
+ // could be, and then we simply check if that's the value we actually have.
+ //
+ // (The rotation parameter does give the possibility of the stretch of 1 bits
+ // going 'round the end' of the word. To deal with that, we observe that in
+ // any situation where that happens the bitwise NOT of the value is also a
+ // valid logical immediate. So we simply invert the input whenever its low bit
+ // is set, and then we know that the rotated case can't arise.)
+
+ if (value & 1) {
+ // If the low bit is 1, negate the value, and set a flag to remember that we
+ // did (so that we can adjust the return values appropriately).
+ negate = true;
+ value = ~value;
+ }
+
+ if (width == kWRegSize) {
+ // To handle 32-bit logical immediates, the very easiest thing is to repeat
+ // the input value twice to make a 64-bit word. The correct encoding of that
+ // as a logical immediate will also be the correct encoding of the 32-bit
+ // value.
+
+ // Avoid making the assumption that the most-significant 32 bits are zero by
+ // shifting the value left and duplicating it.
+ value <<= kWRegSize;
+ value |= value >> kWRegSize;
+ }
+
+ // The basic analysis idea: imagine our input word looks like this.
+ //
+ // 0011111000111110001111100011111000111110001111100011111000111110
+ // c b a
+ // |<--d-->|
+ //
+ // We find the lowest set bit (as an actual power-of-2 value, not its index)
+ // and call it a. Then we add a to our original number, which wipes out the
+ // bottommost stretch of set bits and replaces it with a 1 carried into the
+ // next zero bit. Then we look for the new lowest set bit, which is in
+ // position b, and subtract it, so now our number is just like the original
+ // but with the lowest stretch of set bits completely gone. Now we find the
+ // lowest set bit again, which is position c in the diagram above. Then we'll
+ // measure the distance d between bit positions a and c (using CLZ), and that
+ // tells us that the only valid logical immediate that could possibly be equal
+ // to this number is the one in which a stretch of bits running from a to just
+ // below b is replicated every d bits.
+ uint64_t a = LowestSetBit(value);
+ uint64_t value_plus_a = value + a;
+ uint64_t b = LowestSetBit(value_plus_a);
+ uint64_t value_plus_a_minus_b = value_plus_a - b;
+ uint64_t c = LowestSetBit(value_plus_a_minus_b);
+
+ int d, clz_a, out_n;
+ uint64_t mask;
+
+ if (c != 0) {
+ // The general case, in which there is more than one stretch of set bits.
+ // Compute the repeat distance d, and set up a bitmask covering the basic
+ // unit of repetition (i.e. a word with the bottom d bits set). Also, in all
+ // of these cases the N bit of the output will be zero.
+ clz_a = CountLeadingZeros(a, kXRegSize);
+ int clz_c = CountLeadingZeros(c, kXRegSize);
+ d = clz_a - clz_c;
+ mask = ((UINT64_C(1) << d) - 1);
+ out_n = 0;
+ } else {
+ // Handle degenerate cases.
+ //
+ // If any of those 'find lowest set bit' operations didn't find a set bit at
+ // all, then the word will have been zero thereafter, so in particular the
+ // last lowest_set_bit operation will have returned zero. So we can test for
+ // all the special case conditions in one go by seeing if c is zero.
+ if (a == 0) {
+ // The input was zero (or all 1 bits, which will come to here too after we
+ // inverted it at the start of the function), for which we just return
+ // false.
+ return false;
+ } else {
+ // Otherwise, if c was zero but a was not, then there's just one stretch
+ // of set bits in our word, meaning that we have the trivial case of
+ // d == 64 and only one 'repetition'. Set up all the same variables as in
+ // the general case above, and set the N bit in the output.
+ clz_a = CountLeadingZeros(a, kXRegSize);
+ d = 64;
+ mask = ~UINT64_C(0);
+ out_n = 1;
+ }
+ }
+
+ // If the repeat period d is not a power of two, it can't be encoded.
+ if (!IsPowerOf2(d)) {
+ return false;
+ }
+
+ if (((b - a) & ~mask) != 0) {
+ // If the bit stretch (b - a) does not fit within the mask derived from the
+ // repeat period, then fail.
+ return false;
+ }
+
+ // The only possible option is b - a repeated every d bits. Now we're going to
+ // actually construct the valid logical immediate derived from that
+ // specification, and see if it equals our original input.
+ //
+ // To repeat a value every d bits, we multiply it by a number of the form
+ // (1 + 2^d + 2^(2d) + ...), i.e. 0x0001000100010001 or similar. These can
+ // be derived using a table lookup on CLZ(d).
+ static const uint64_t multipliers[] = {
+ 0x0000000000000001UL,
+ 0x0000000100000001UL,
+ 0x0001000100010001UL,
+ 0x0101010101010101UL,
+ 0x1111111111111111UL,
+ 0x5555555555555555UL,
+ };
+ uint64_t multiplier = multipliers[CountLeadingZeros(d, kXRegSize) - 57];
+ uint64_t candidate = (b - a) * multiplier;
+
+ if (value != candidate) {
+ // The candidate pattern doesn't match our input value, so fail.
+ return false;
+ }
+
+ // We have a match! This is a valid logical immediate, so now we have to
+ // construct the bits and pieces of the instruction encoding that generates
+ // it.
+
+ // Count the set bits in our basic stretch. The special case of clz(0) == -1
+ // makes the answer come out right for stretches that reach the very top of
+ // the word (e.g. numbers like 0xffffc00000000000).
+ int clz_b = (b == 0) ? -1 : CountLeadingZeros(b, kXRegSize);
+ int s = clz_a - clz_b;
+
+ // Decide how many bits to rotate right by, to put the low bit of that basic
+ // stretch in position a.
+ int r;
+ if (negate) {
+ // If we inverted the input right at the start of this function, here's
+ // where we compensate: the number of set bits becomes the number of clear
+ // bits, and the rotation count is based on position b rather than position
+ // a (since b is the location of the 'lowest' 1 bit after inversion).
+ s = d - s;
+ r = (clz_b + 1) & (d - 1);
+ } else {
+ r = (clz_a + 1) & (d - 1);
+ }
+
+ // Now we're done, except for having to encode the S output in such a way that
+ // it gives both the number of set bits and the length of the repeated
+ // segment. The s field is encoded like this:
+ //
+ // imms size S
+ // ssssss 64 UInt(ssssss)
+ // 0sssss 32 UInt(sssss)
+ // 10ssss 16 UInt(ssss)
+ // 110sss 8 UInt(sss)
+ // 1110ss 4 UInt(ss)
+ // 11110s 2 UInt(s)
+ //
+ // So we 'or' (-d << 1) with our computed s to form imms.
+ if ((n != NULL) || (imm_s != NULL) || (imm_r != NULL)) {
+ *n = out_n;
+ *imm_s = ((-d << 1) | (s - 1)) & 0x3f;
+ *imm_r = r;
+ }
+
+ return true;
+}
+
+
+LoadStoreOp Assembler::LoadOpFor(const CPURegister& rt) {
+ VIXL_ASSERT(rt.IsValid());
+ if (rt.IsRegister()) {
+ return rt.Is64Bits() ? LDR_x : LDR_w;
+ } else {
+ VIXL_ASSERT(rt.IsVRegister());
+ switch (rt.SizeInBits()) {
+ case kBRegSize: return LDR_b;
+ case kHRegSize: return LDR_h;
+ case kSRegSize: return LDR_s;
+ case kDRegSize: return LDR_d;
+ default:
+ VIXL_ASSERT(rt.IsQ());
+ return LDR_q;
+ }
+ }
+}
+
+
+LoadStoreOp Assembler::StoreOpFor(const CPURegister& rt) {
+ VIXL_ASSERT(rt.IsValid());
+ if (rt.IsRegister()) {
+ return rt.Is64Bits() ? STR_x : STR_w;
+ } else {
+ VIXL_ASSERT(rt.IsVRegister());
+ switch (rt.SizeInBits()) {
+ case kBRegSize: return STR_b;
+ case kHRegSize: return STR_h;
+ case kSRegSize: return STR_s;
+ case kDRegSize: return STR_d;
+ default:
+ VIXL_ASSERT(rt.IsQ());
+ return STR_q;
+ }
+ }
+}
+
+
+LoadStorePairOp Assembler::StorePairOpFor(const CPURegister& rt,
+ const CPURegister& rt2) {
+ VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
+ USE(rt2);
+ if (rt.IsRegister()) {
+ return rt.Is64Bits() ? STP_x : STP_w;
+ } else {
+ VIXL_ASSERT(rt.IsVRegister());
+ switch (rt.SizeInBytes()) {
+ case kSRegSizeInBytes: return STP_s;
+ case kDRegSizeInBytes: return STP_d;
+ default:
+ VIXL_ASSERT(rt.IsQ());
+ return STP_q;
+ }
+ }
+}
+
+
+LoadStorePairOp Assembler::LoadPairOpFor(const CPURegister& rt,
+ const CPURegister& rt2) {
+ VIXL_ASSERT((STP_w | LoadStorePairLBit) == LDP_w);
+ return static_cast<LoadStorePairOp>(StorePairOpFor(rt, rt2) |
+ LoadStorePairLBit);
+}
+
+
+LoadStorePairNonTemporalOp Assembler::StorePairNonTemporalOpFor(
+ const CPURegister& rt, const CPURegister& rt2) {
+ VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
+ USE(rt2);
+ if (rt.IsRegister()) {
+ return rt.Is64Bits() ? STNP_x : STNP_w;
+ } else {
+ VIXL_ASSERT(rt.IsVRegister());
+ switch (rt.SizeInBytes()) {
+ case kSRegSizeInBytes: return STNP_s;
+ case kDRegSizeInBytes: return STNP_d;
+ default:
+ VIXL_ASSERT(rt.IsQ());
+ return STNP_q;
+ }
+ }
+}
+
+
+LoadStorePairNonTemporalOp Assembler::LoadPairNonTemporalOpFor(
+ const CPURegister& rt, const CPURegister& rt2) {
+ VIXL_ASSERT((STNP_w | LoadStorePairNonTemporalLBit) == LDNP_w);
+ return static_cast<LoadStorePairNonTemporalOp>(
+ StorePairNonTemporalOpFor(rt, rt2) | LoadStorePairNonTemporalLBit);
+}
+
+
+LoadLiteralOp Assembler::LoadLiteralOpFor(const CPURegister& rt) {
+ if (rt.IsRegister()) {
+ return rt.IsX() ? LDR_x_lit : LDR_w_lit;
+ } else {
+ VIXL_ASSERT(rt.IsVRegister());
+ switch (rt.SizeInBytes()) {
+ case kSRegSizeInBytes: return LDR_s_lit;
+ case kDRegSizeInBytes: return LDR_d_lit;
+ default:
+ VIXL_ASSERT(rt.IsQ());
+ return LDR_q_lit;
+ }
+ }
+}
+
+
+bool Assembler::CPUHas(const CPURegister& rt) const {
+ // Core registers are available without any particular CPU features.
+ if (rt.IsRegister()) return true;
+ VIXL_ASSERT(rt.IsVRegister());
+ // The architecture does not allow FP and NEON to be implemented separately,
+ // but we can crudely categorise them based on register size, since FP only
+ // uses D, S and (occasionally) H registers.
+ if (rt.IsH() || rt.IsS() || rt.IsD()) {
+ return CPUHas(CPUFeatures::kFP) || CPUHas(CPUFeatures::kNEON);
+ }
+ VIXL_ASSERT(rt.IsB() || rt.IsQ());
+ return CPUHas(CPUFeatures::kNEON);
+}
+
+
+bool Assembler::CPUHas(const CPURegister& rt, const CPURegister& rt2) const {
+ // This is currently only used for loads and stores, where rt and rt2 must
+ // have the same size and type. We could extend this to cover other cases if
+ // necessary, but for now we can avoid checking both registers.
+ VIXL_ASSERT(AreSameSizeAndType(rt, rt2));
+ USE(rt2);
+ return CPUHas(rt);
+}
+
+
+bool Assembler::CPUHas(SystemRegister sysreg) const {
+ switch (sysreg) {
+ case RNDR:
+ case RNDRRS:
+ return CPUHas(CPUFeatures::kRNG);
+ case FPCR:
+ case NZCV:
+ break;
+ }
+ return true;
+}
+
+
+bool AreAliased(const CPURegister& reg1, const CPURegister& reg2,
+ const CPURegister& reg3, const CPURegister& reg4,
+ const CPURegister& reg5, const CPURegister& reg6,
+ const CPURegister& reg7, const CPURegister& reg8) {
+ int number_of_valid_regs = 0;
+ int number_of_valid_fpregs = 0;
+
+ RegList unique_regs = 0;
+ RegList unique_fpregs = 0;
+
+ const CPURegister regs[] = {reg1, reg2, reg3, reg4, reg5, reg6, reg7, reg8};
+
+ for (unsigned i = 0; i < sizeof(regs) / sizeof(regs[0]); i++) {
+ if (regs[i].IsRegister()) {
+ number_of_valid_regs++;
+ unique_regs |= regs[i].Bit();
+ } else if (regs[i].IsVRegister()) {
+ number_of_valid_fpregs++;
+ unique_fpregs |= regs[i].Bit();
+ } else {
+ VIXL_ASSERT(!regs[i].IsValid());
+ }
+ }
+
+ int number_of_unique_regs = CountSetBits(unique_regs);
+ int number_of_unique_fpregs = CountSetBits(unique_fpregs);
+
+ VIXL_ASSERT(number_of_valid_regs >= number_of_unique_regs);
+ VIXL_ASSERT(number_of_valid_fpregs >= number_of_unique_fpregs);
+
+ return (number_of_valid_regs != number_of_unique_regs) ||
+ (number_of_valid_fpregs != number_of_unique_fpregs);
+}
+
+
+bool AreSameSizeAndType(const CPURegister& reg1, const CPURegister& reg2,
+ const CPURegister& reg3, const CPURegister& reg4,
+ const CPURegister& reg5, const CPURegister& reg6,
+ const CPURegister& reg7, const CPURegister& reg8) {
+ VIXL_ASSERT(reg1.IsValid());
+ bool match = true;
+ match &= !reg2.IsValid() || reg2.IsSameSizeAndType(reg1);
+ match &= !reg3.IsValid() || reg3.IsSameSizeAndType(reg1);
+ match &= !reg4.IsValid() || reg4.IsSameSizeAndType(reg1);
+ match &= !reg5.IsValid() || reg5.IsSameSizeAndType(reg1);
+ match &= !reg6.IsValid() || reg6.IsSameSizeAndType(reg1);
+ match &= !reg7.IsValid() || reg7.IsSameSizeAndType(reg1);
+ match &= !reg8.IsValid() || reg8.IsSameSizeAndType(reg1);
+ return match;
+}
+
+
+bool AreSameFormat(const VRegister& reg1, const VRegister& reg2,
+ const VRegister& reg3, const VRegister& reg4) {
+ VIXL_ASSERT(reg1.IsValid());
+ bool match = true;
+ match &= !reg2.IsValid() || reg2.IsSameFormat(reg1);
+ match &= !reg3.IsValid() || reg3.IsSameFormat(reg1);
+ match &= !reg4.IsValid() || reg4.IsSameFormat(reg1);
+ return match;
+}
+
+
+bool AreConsecutive(const VRegister& reg1, const VRegister& reg2,
+ const VRegister& reg3, const VRegister& reg4) {
+ VIXL_ASSERT(reg1.IsValid());
+ bool match = true;
+ match &= !reg2.IsValid() ||
+ (reg2.code() == ((reg1.code() + 1) % kNumberOfVRegisters));
+ match &= !reg3.IsValid() ||
+ (reg3.code() == ((reg1.code() + 2) % kNumberOfVRegisters));
+ match &= !reg4.IsValid() ||
+ (reg4.code() == ((reg1.code() + 3) % kNumberOfVRegisters));
+ return match;
+}
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/Assembler-vixl.h b/js/src/jit/arm64/vixl/Assembler-vixl.h
new file mode 100644
index 0000000000..3ae415f977
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Assembler-vixl.h
@@ -0,0 +1,4327 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_ASSEMBLER_A64_H_
+#define VIXL_A64_ASSEMBLER_A64_H_
+
+#include "jit/arm64/vixl/Cpu-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Instructions-vixl.h"
+#include "jit/arm64/vixl/MozBaseAssembler-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+
+#include "jit/JitSpewer.h"
+
+#include "jit/shared/Assembler-shared.h"
+#include "jit/shared/Disassembler-shared.h"
+#include "jit/shared/IonAssemblerBufferWithConstantPools.h"
+
+#if defined(_M_ARM64)
+#ifdef mvn
+#undef mvn
+#endif
+#endif
+
+namespace vixl {
+
+using js::jit::BufferOffset;
+using js::jit::Label;
+using js::jit::Address;
+using js::jit::BaseIndex;
+using js::jit::DisassemblerSpew;
+
+using LabelDoc = DisassemblerSpew::LabelDoc;
+
+typedef uint64_t RegList;
+static const int kRegListSizeInBits = sizeof(RegList) * 8;
+
+
+// Registers.
+
+// Some CPURegister methods can return Register or VRegister types, so we need
+// to declare them in advance.
+class Register;
+class VRegister;
+
+class CPURegister {
+ public:
+ enum RegisterType {
+ // The kInvalid value is used to detect uninitialized static instances,
+ // which are always zero-initialized before any constructors are called.
+ kInvalid = 0,
+ kRegister,
+ kVRegister,
+ kFPRegister = kVRegister,
+ kNoRegister
+ };
+
+ constexpr CPURegister() : code_(0), size_(0), type_(kNoRegister) {
+ }
+
+ constexpr CPURegister(unsigned code, unsigned size, RegisterType type)
+ : code_(code), size_(size), type_(type) {
+ }
+
+ unsigned code() const {
+ VIXL_ASSERT(IsValid());
+ return code_;
+ }
+
+ RegisterType type() const {
+ VIXL_ASSERT(IsValidOrNone());
+ return type_;
+ }
+
+ RegList Bit() const {
+ VIXL_ASSERT(code_ < (sizeof(RegList) * 8));
+ return IsValid() ? (static_cast<RegList>(1) << code_) : 0;
+ }
+
+ unsigned size() const {
+ VIXL_ASSERT(IsValid());
+ return size_;
+ }
+
+ int SizeInBytes() const {
+ VIXL_ASSERT(IsValid());
+ VIXL_ASSERT(size() % 8 == 0);
+ return size_ / 8;
+ }
+
+ int SizeInBits() const {
+ VIXL_ASSERT(IsValid());
+ return size_;
+ }
+
+ bool Is8Bits() const {
+ VIXL_ASSERT(IsValid());
+ return size_ == 8;
+ }
+
+ bool Is16Bits() const {
+ VIXL_ASSERT(IsValid());
+ return size_ == 16;
+ }
+
+ bool Is32Bits() const {
+ VIXL_ASSERT(IsValid());
+ return size_ == 32;
+ }
+
+ bool Is64Bits() const {
+ VIXL_ASSERT(IsValid());
+ return size_ == 64;
+ }
+
+ bool Is128Bits() const {
+ VIXL_ASSERT(IsValid());
+ return size_ == 128;
+ }
+
+ bool IsValid() const {
+ if (IsValidRegister() || IsValidVRegister()) {
+ VIXL_ASSERT(!IsNone());
+ return true;
+ } else {
+ // This assert is hit when the register has not been properly initialized.
+ // One cause for this can be an initialisation order fiasco. See
+ // https://isocpp.org/wiki/faq/ctors#static-init-order for some details.
+ VIXL_ASSERT(IsNone());
+ return false;
+ }
+ }
+
+ bool IsValidRegister() const {
+ return IsRegister() &&
+ ((size_ == kWRegSize) || (size_ == kXRegSize)) &&
+ ((code_ < kNumberOfRegisters) || (code_ == kSPRegInternalCode));
+ }
+
+ bool IsValidVRegister() const {
+ return IsVRegister() &&
+ ((size_ == kBRegSize) || (size_ == kHRegSize) ||
+ (size_ == kSRegSize) || (size_ == kDRegSize) ||
+ (size_ == kQRegSize)) &&
+ (code_ < kNumberOfVRegisters);
+ }
+
+ bool IsValidFPRegister() const {
+ return IsFPRegister() && (code_ < kNumberOfVRegisters);
+ }
+
+ bool IsNone() const {
+ // kNoRegister types should always have size 0 and code 0.
+ VIXL_ASSERT((type_ != kNoRegister) || (code_ == 0));
+ VIXL_ASSERT((type_ != kNoRegister) || (size_ == 0));
+
+ return type_ == kNoRegister;
+ }
+
+ bool Aliases(const CPURegister& other) const {
+ VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
+ return (code_ == other.code_) && (type_ == other.type_);
+ }
+
+ bool Is(const CPURegister& other) const {
+ VIXL_ASSERT(IsValidOrNone() && other.IsValidOrNone());
+ return Aliases(other) && (size_ == other.size_);
+ }
+
+ bool IsZero() const {
+ VIXL_ASSERT(IsValid());
+ return IsRegister() && (code_ == kZeroRegCode);
+ }
+
+ bool IsSP() const {
+ VIXL_ASSERT(IsValid());
+ return IsRegister() && (code_ == kSPRegInternalCode);
+ }
+
+ bool IsRegister() const {
+ return type_ == kRegister;
+ }
+
+ bool IsVRegister() const {
+ return type_ == kVRegister;
+ }
+
+ bool IsFPRegister() const {
+ return IsS() || IsD();
+ }
+
+ bool IsW() const { return IsValidRegister() && Is32Bits(); }
+ bool IsX() const { return IsValidRegister() && Is64Bits(); }
+
+ // These assertions ensure that the size and type of the register are as
+ // described. They do not consider the number of lanes that make up a vector.
+ // So, for example, Is8B() implies IsD(), and Is1D() implies IsD, but IsD()
+ // does not imply Is1D() or Is8B().
+ // Check the number of lanes, ie. the format of the vector, using methods such
+ // as Is8B(), Is1D(), etc. in the VRegister class.
+ bool IsV() const { return IsVRegister(); }
+ bool IsB() const { return IsV() && Is8Bits(); }
+ bool IsH() const { return IsV() && Is16Bits(); }
+ bool IsS() const { return IsV() && Is32Bits(); }
+ bool IsD() const { return IsV() && Is64Bits(); }
+ bool IsQ() const { return IsV() && Is128Bits(); }
+
+ const Register& W() const;
+ const Register& X() const;
+ const VRegister& V() const;
+ const VRegister& B() const;
+ const VRegister& H() const;
+ const VRegister& S() const;
+ const VRegister& D() const;
+ const VRegister& Q() const;
+
+ bool IsSameSizeAndType(const CPURegister& other) const {
+ return (size_ == other.size_) && (type_ == other.type_);
+ }
+
+ protected:
+ unsigned code_;
+ unsigned size_;
+ RegisterType type_;
+
+ private:
+ bool IsValidOrNone() const {
+ return IsValid() || IsNone();
+ }
+};
+
+
+class Register : public CPURegister {
+ public:
+ Register() : CPURegister() {}
+ explicit Register(const CPURegister& other)
+ : CPURegister(other.code(), other.size(), other.type()) {
+ VIXL_ASSERT(IsValidRegister());
+ }
+ constexpr Register(unsigned code, unsigned size)
+ : CPURegister(code, size, kRegister) {}
+
+ constexpr Register(js::jit::Register r, unsigned size)
+ : CPURegister(r.code(), size, kRegister) {}
+
+ bool IsValid() const {
+ VIXL_ASSERT(IsRegister() || IsNone());
+ return IsValidRegister();
+ }
+
+ js::jit::Register asUnsized() const {
+ // asUnsized() is only ever used on temp registers or on registers that
+ // are known not to be SP, and there should be no risk of it being
+ // applied to SP. Check anyway.
+ VIXL_ASSERT(code_ != kSPRegInternalCode);
+ return js::jit::Register::FromCode((js::jit::Register::Code)code_);
+ }
+
+
+ static const Register& WRegFromCode(unsigned code);
+ static const Register& XRegFromCode(unsigned code);
+
+ private:
+ static const Register wregisters[];
+ static const Register xregisters[];
+};
+
+
+class VRegister : public CPURegister {
+ public:
+ VRegister() : CPURegister(), lanes_(1) {}
+ explicit VRegister(const CPURegister& other)
+ : CPURegister(other.code(), other.size(), other.type()), lanes_(1) {
+ VIXL_ASSERT(IsValidVRegister());
+ VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+ }
+ constexpr VRegister(unsigned code, unsigned size, unsigned lanes = 1)
+ : CPURegister(code, size, kVRegister), lanes_(lanes) {
+ // VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+ }
+ constexpr VRegister(js::jit::FloatRegister r)
+ : CPURegister(r.encoding(), r.size() * 8, kVRegister), lanes_(1) {
+ }
+ constexpr VRegister(js::jit::FloatRegister r, unsigned size)
+ : CPURegister(r.encoding(), size, kVRegister), lanes_(1) {
+ }
+ VRegister(unsigned code, VectorFormat format)
+ : CPURegister(code, RegisterSizeInBitsFromFormat(format), kVRegister),
+ lanes_(IsVectorFormat(format) ? LaneCountFromFormat(format) : 1) {
+ VIXL_ASSERT(IsPowerOf2(lanes_) && (lanes_ <= 16));
+ }
+
+ bool IsValid() const {
+ VIXL_ASSERT(IsVRegister() || IsNone());
+ return IsValidVRegister();
+ }
+
+ static const VRegister& BRegFromCode(unsigned code);
+ static const VRegister& HRegFromCode(unsigned code);
+ static const VRegister& SRegFromCode(unsigned code);
+ static const VRegister& DRegFromCode(unsigned code);
+ static const VRegister& QRegFromCode(unsigned code);
+ static const VRegister& VRegFromCode(unsigned code);
+
+ VRegister V8B() const { return VRegister(code_, kDRegSize, 8); }
+ VRegister V16B() const { return VRegister(code_, kQRegSize, 16); }
+ VRegister V4H() const { return VRegister(code_, kDRegSize, 4); }
+ VRegister V8H() const { return VRegister(code_, kQRegSize, 8); }
+ VRegister V2S() const { return VRegister(code_, kDRegSize, 2); }
+ VRegister V4S() const { return VRegister(code_, kQRegSize, 4); }
+ VRegister V2D() const { return VRegister(code_, kQRegSize, 2); }
+ VRegister V1D() const { return VRegister(code_, kDRegSize, 1); }
+
+ bool Is8B() const { return (Is64Bits() && (lanes_ == 8)); }
+ bool Is16B() const { return (Is128Bits() && (lanes_ == 16)); }
+ bool Is4H() const { return (Is64Bits() && (lanes_ == 4)); }
+ bool Is8H() const { return (Is128Bits() && (lanes_ == 8)); }
+ bool Is2S() const { return (Is64Bits() && (lanes_ == 2)); }
+ bool Is4S() const { return (Is128Bits() && (lanes_ == 4)); }
+ bool Is1D() const { return (Is64Bits() && (lanes_ == 1)); }
+ bool Is2D() const { return (Is128Bits() && (lanes_ == 2)); }
+
+ // For consistency, we assert the number of lanes of these scalar registers,
+ // even though there are no vectors of equivalent total size with which they
+ // could alias.
+ bool Is1B() const {
+ VIXL_ASSERT(!(Is8Bits() && IsVector()));
+ return Is8Bits();
+ }
+ bool Is1H() const {
+ VIXL_ASSERT(!(Is16Bits() && IsVector()));
+ return Is16Bits();
+ }
+ bool Is1S() const {
+ VIXL_ASSERT(!(Is32Bits() && IsVector()));
+ return Is32Bits();
+ }
+
+ bool IsLaneSizeB() const { return LaneSizeInBits() == kBRegSize; }
+ bool IsLaneSizeH() const { return LaneSizeInBits() == kHRegSize; }
+ bool IsLaneSizeS() const { return LaneSizeInBits() == kSRegSize; }
+ bool IsLaneSizeD() const { return LaneSizeInBits() == kDRegSize; }
+
+ int lanes() const {
+ return lanes_;
+ }
+
+ bool IsScalar() const {
+ return lanes_ == 1;
+ }
+
+ bool IsVector() const {
+ return lanes_ > 1;
+ }
+
+ bool IsSameFormat(const VRegister& other) const {
+ return (size_ == other.size_) && (lanes_ == other.lanes_);
+ }
+
+ unsigned LaneSizeInBytes() const {
+ return SizeInBytes() / lanes_;
+ }
+
+ unsigned LaneSizeInBits() const {
+ return LaneSizeInBytes() * 8;
+ }
+
+ private:
+ static const VRegister bregisters[];
+ static const VRegister hregisters[];
+ static const VRegister sregisters[];
+ static const VRegister dregisters[];
+ static const VRegister qregisters[];
+ static const VRegister vregisters[];
+ int lanes_;
+};
+
+
+// Backward compatibility for FPRegisters.
+typedef VRegister FPRegister;
+
+// No*Reg is used to indicate an unused argument, or an error case. Note that
+// these all compare equal (using the Is() method). The Register and VRegister
+// variants are provided for convenience.
+const Register NoReg;
+const VRegister NoVReg;
+const FPRegister NoFPReg; // For backward compatibility.
+const CPURegister NoCPUReg;
+
+
+#define DEFINE_REGISTERS(N) \
+constexpr Register w##N(N, kWRegSize); \
+constexpr Register x##N(N, kXRegSize);
+REGISTER_CODE_LIST(DEFINE_REGISTERS)
+#undef DEFINE_REGISTERS
+constexpr Register wsp(kSPRegInternalCode, kWRegSize);
+constexpr Register sp(kSPRegInternalCode, kXRegSize);
+
+
+#define DEFINE_VREGISTERS(N) \
+constexpr VRegister b##N(N, kBRegSize); \
+constexpr VRegister h##N(N, kHRegSize); \
+constexpr VRegister s##N(N, kSRegSize); \
+constexpr VRegister d##N(N, kDRegSize); \
+constexpr VRegister q##N(N, kQRegSize); \
+constexpr VRegister v##N(N, kQRegSize);
+REGISTER_CODE_LIST(DEFINE_VREGISTERS)
+#undef DEFINE_VREGISTERS
+
+
+// Registers aliases.
+constexpr Register ip0 = x16;
+constexpr Register ip1 = x17;
+constexpr Register lr = x30;
+constexpr Register xzr = x31;
+constexpr Register wzr = w31;
+
+
+// AreAliased returns true if any of the named registers overlap. Arguments
+// set to NoReg are ignored. The system stack pointer may be specified.
+bool AreAliased(const CPURegister& reg1,
+ const CPURegister& reg2,
+ const CPURegister& reg3 = NoReg,
+ const CPURegister& reg4 = NoReg,
+ const CPURegister& reg5 = NoReg,
+ const CPURegister& reg6 = NoReg,
+ const CPURegister& reg7 = NoReg,
+ const CPURegister& reg8 = NoReg);
+
+
+// AreSameSizeAndType returns true if all of the specified registers have the
+// same size, and are of the same type. The system stack pointer may be
+// specified. Arguments set to NoReg are ignored, as are any subsequent
+// arguments. At least one argument (reg1) must be valid (not NoCPUReg).
+bool AreSameSizeAndType(const CPURegister& reg1,
+ const CPURegister& reg2,
+ const CPURegister& reg3 = NoCPUReg,
+ const CPURegister& reg4 = NoCPUReg,
+ const CPURegister& reg5 = NoCPUReg,
+ const CPURegister& reg6 = NoCPUReg,
+ const CPURegister& reg7 = NoCPUReg,
+ const CPURegister& reg8 = NoCPUReg);
+
+
+// AreSameFormat returns true if all of the specified VRegisters have the same
+// vector format. Arguments set to NoReg are ignored, as are any subsequent
+// arguments. At least one argument (reg1) must be valid (not NoVReg).
+bool AreSameFormat(const VRegister& reg1,
+ const VRegister& reg2,
+ const VRegister& reg3 = NoVReg,
+ const VRegister& reg4 = NoVReg);
+
+
+// AreConsecutive returns true if all of the specified VRegisters are
+// consecutive in the register file. Arguments set to NoReg are ignored, as are
+// any subsequent arguments. At least one argument (reg1) must be valid
+// (not NoVReg).
+bool AreConsecutive(const VRegister& reg1,
+ const VRegister& reg2,
+ const VRegister& reg3 = NoVReg,
+ const VRegister& reg4 = NoVReg);
+
+
+// Lists of registers.
+class CPURegList {
+ public:
+ explicit CPURegList(CPURegister reg1,
+ CPURegister reg2 = NoCPUReg,
+ CPURegister reg3 = NoCPUReg,
+ CPURegister reg4 = NoCPUReg)
+ : list_(reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit()),
+ size_(reg1.size()), type_(reg1.type()) {
+ VIXL_ASSERT(AreSameSizeAndType(reg1, reg2, reg3, reg4));
+ VIXL_ASSERT(IsValid());
+ }
+
+ CPURegList(CPURegister::RegisterType type, unsigned size, RegList list)
+ : list_(list), size_(size), type_(type) {
+ VIXL_ASSERT(IsValid());
+ }
+
+ CPURegList(CPURegister::RegisterType type, unsigned size,
+ unsigned first_reg, unsigned last_reg)
+ : size_(size), type_(type) {
+ VIXL_ASSERT(((type == CPURegister::kRegister) &&
+ (last_reg < kNumberOfRegisters)) ||
+ ((type == CPURegister::kVRegister) &&
+ (last_reg < kNumberOfVRegisters)));
+ VIXL_ASSERT(last_reg >= first_reg);
+ list_ = (UINT64_C(1) << (last_reg + 1)) - 1;
+ list_ &= ~((UINT64_C(1) << first_reg) - 1);
+ VIXL_ASSERT(IsValid());
+ }
+
+ CPURegister::RegisterType type() const {
+ VIXL_ASSERT(IsValid());
+ return type_;
+ }
+
+ // Combine another CPURegList into this one. Registers that already exist in
+ // this list are left unchanged. The type and size of the registers in the
+ // 'other' list must match those in this list.
+ void Combine(const CPURegList& other) {
+ VIXL_ASSERT(IsValid());
+ VIXL_ASSERT(other.type() == type_);
+ VIXL_ASSERT(other.RegisterSizeInBits() == size_);
+ list_ |= other.list();
+ }
+
+ // Remove every register in the other CPURegList from this one. Registers that
+ // do not exist in this list are ignored. The type and size of the registers
+ // in the 'other' list must match those in this list.
+ void Remove(const CPURegList& other) {
+ VIXL_ASSERT(IsValid());
+ VIXL_ASSERT(other.type() == type_);
+ VIXL_ASSERT(other.RegisterSizeInBits() == size_);
+ list_ &= ~other.list();
+ }
+
+ // Variants of Combine and Remove which take a single register.
+ void Combine(const CPURegister& other) {
+ VIXL_ASSERT(other.type() == type_);
+ VIXL_ASSERT(other.size() == size_);
+ Combine(other.code());
+ }
+
+ void Remove(const CPURegister& other) {
+ VIXL_ASSERT(other.type() == type_);
+ VIXL_ASSERT(other.size() == size_);
+ Remove(other.code());
+ }
+
+ // Variants of Combine and Remove which take a single register by its code;
+ // the type and size of the register is inferred from this list.
+ void Combine(int code) {
+ VIXL_ASSERT(IsValid());
+ VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
+ list_ |= (UINT64_C(1) << code);
+ }
+
+ void Remove(int code) {
+ VIXL_ASSERT(IsValid());
+ VIXL_ASSERT(CPURegister(code, size_, type_).IsValid());
+ list_ &= ~(UINT64_C(1) << code);
+ }
+
+ static CPURegList Union(const CPURegList& list_1, const CPURegList& list_2) {
+ VIXL_ASSERT(list_1.type_ == list_2.type_);
+ VIXL_ASSERT(list_1.size_ == list_2.size_);
+ return CPURegList(list_1.type_, list_1.size_, list_1.list_ | list_2.list_);
+ }
+ static CPURegList Union(const CPURegList& list_1,
+ const CPURegList& list_2,
+ const CPURegList& list_3);
+ static CPURegList Union(const CPURegList& list_1,
+ const CPURegList& list_2,
+ const CPURegList& list_3,
+ const CPURegList& list_4);
+
+ static CPURegList Intersection(const CPURegList& list_1,
+ const CPURegList& list_2) {
+ VIXL_ASSERT(list_1.type_ == list_2.type_);
+ VIXL_ASSERT(list_1.size_ == list_2.size_);
+ return CPURegList(list_1.type_, list_1.size_, list_1.list_ & list_2.list_);
+ }
+ static CPURegList Intersection(const CPURegList& list_1,
+ const CPURegList& list_2,
+ const CPURegList& list_3);
+ static CPURegList Intersection(const CPURegList& list_1,
+ const CPURegList& list_2,
+ const CPURegList& list_3,
+ const CPURegList& list_4);
+
+ bool Overlaps(const CPURegList& other) const {
+ return (type_ == other.type_) && ((list_ & other.list_) != 0);
+ }
+
+ RegList list() const {
+ VIXL_ASSERT(IsValid());
+ return list_;
+ }
+
+ void set_list(RegList new_list) {
+ VIXL_ASSERT(IsValid());
+ list_ = new_list;
+ }
+
+ // Remove all callee-saved registers from the list. This can be useful when
+ // preparing registers for an AAPCS64 function call, for example.
+ void RemoveCalleeSaved();
+
+ CPURegister PopLowestIndex();
+ CPURegister PopHighestIndex();
+
+ // AAPCS64 callee-saved registers.
+ static CPURegList GetCalleeSaved(unsigned size = kXRegSize);
+ static CPURegList GetCalleeSavedV(unsigned size = kDRegSize);
+
+ // AAPCS64 caller-saved registers. Note that this includes lr.
+ // TODO(all): Determine how we handle d8-d15 being callee-saved, but the top
+ // 64-bits being caller-saved.
+ static CPURegList GetCallerSaved(unsigned size = kXRegSize);
+ static CPURegList GetCallerSavedV(unsigned size = kDRegSize);
+
+ bool IsEmpty() const {
+ VIXL_ASSERT(IsValid());
+ return list_ == 0;
+ }
+
+ bool IncludesAliasOf(const CPURegister& other) const {
+ VIXL_ASSERT(IsValid());
+ return (type_ == other.type()) && ((other.Bit() & list_) != 0);
+ }
+
+ bool IncludesAliasOf(int code) const {
+ VIXL_ASSERT(IsValid());
+ return ((code & list_) != 0);
+ }
+
+ int Count() const {
+ VIXL_ASSERT(IsValid());
+ return CountSetBits(list_);
+ }
+
+ unsigned RegisterSizeInBits() const {
+ VIXL_ASSERT(IsValid());
+ return size_;
+ }
+
+ unsigned RegisterSizeInBytes() const {
+ int size_in_bits = RegisterSizeInBits();
+ VIXL_ASSERT((size_in_bits % 8) == 0);
+ return size_in_bits / 8;
+ }
+
+ unsigned TotalSizeInBytes() const {
+ VIXL_ASSERT(IsValid());
+ return RegisterSizeInBytes() * Count();
+ }
+
+ private:
+ RegList list_;
+ unsigned size_;
+ CPURegister::RegisterType type_;
+
+ bool IsValid() const;
+};
+
+
+// AAPCS64 callee-saved registers.
+extern const CPURegList kCalleeSaved;
+extern const CPURegList kCalleeSavedV;
+
+
+// AAPCS64 caller-saved registers. Note that this includes lr.
+extern const CPURegList kCallerSaved;
+extern const CPURegList kCallerSavedV;
+
+
+// Operand.
+class Operand {
+ public:
+ // #<immediate>
+ // where <immediate> is int64_t.
+ // This is allowed to be an implicit constructor because Operand is
+ // a wrapper class that doesn't normally perform any type conversion.
+ Operand(int64_t immediate = 0); // NOLINT(runtime/explicit)
+
+ // rm, {<shift> #<shift_amount>}
+ // where <shift> is one of {LSL, LSR, ASR, ROR}.
+ // <shift_amount> is uint6_t.
+ // This is allowed to be an implicit constructor because Operand is
+ // a wrapper class that doesn't normally perform any type conversion.
+ Operand(Register reg,
+ Shift shift = LSL,
+ unsigned shift_amount = 0); // NOLINT(runtime/explicit)
+
+ // rm, {<extend> {#<shift_amount>}}
+ // where <extend> is one of {UXTB, UXTH, UXTW, UXTX, SXTB, SXTH, SXTW, SXTX}.
+ // <shift_amount> is uint2_t.
+ explicit Operand(Register reg, Extend extend, unsigned shift_amount = 0);
+
+ bool IsImmediate() const;
+ bool IsShiftedRegister() const;
+ bool IsExtendedRegister() const;
+ bool IsZero() const;
+
+ // This returns an LSL shift (<= 4) operand as an equivalent extend operand,
+ // which helps in the encoding of instructions that use the stack pointer.
+ Operand ToExtendedRegister() const;
+
+ int64_t immediate() const {
+ VIXL_ASSERT(IsImmediate());
+ return immediate_;
+ }
+
+ Register reg() const {
+ VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
+ return reg_;
+ }
+
+ CPURegister maybeReg() const {
+ if (IsShiftedRegister() || IsExtendedRegister())
+ return reg_;
+ return NoCPUReg;
+ }
+
+ Shift shift() const {
+ VIXL_ASSERT(IsShiftedRegister());
+ return shift_;
+ }
+
+ Extend extend() const {
+ VIXL_ASSERT(IsExtendedRegister());
+ return extend_;
+ }
+
+ unsigned shift_amount() const {
+ VIXL_ASSERT(IsShiftedRegister() || IsExtendedRegister());
+ return shift_amount_;
+ }
+
+ private:
+ int64_t immediate_;
+ Register reg_;
+ Shift shift_;
+ Extend extend_;
+ unsigned shift_amount_;
+};
+
+
+// MemOperand represents the addressing mode of a load or store instruction.
+class MemOperand {
+ public:
+ explicit MemOperand(Register base,
+ int64_t offset = 0,
+ AddrMode addrmode = Offset);
+ MemOperand(Register base,
+ Register regoffset,
+ Shift shift = LSL,
+ unsigned shift_amount = 0);
+ MemOperand(Register base,
+ Register regoffset,
+ Extend extend,
+ unsigned shift_amount = 0);
+ MemOperand(Register base,
+ const Operand& offset,
+ AddrMode addrmode = Offset);
+
+ // Adapter constructors using C++11 delegating.
+ // TODO: If sp == kSPRegInternalCode, the xzr check isn't necessary.
+ explicit MemOperand(js::jit::Address addr)
+ : MemOperand(IsHiddenSP(addr.base) ? sp : Register(AsRegister(addr.base), 64),
+ (ptrdiff_t)addr.offset) {
+ }
+
+ const Register& base() const { return base_; }
+ const Register& regoffset() const { return regoffset_; }
+ int64_t offset() const { return offset_; }
+ AddrMode addrmode() const { return addrmode_; }
+ Shift shift() const { return shift_; }
+ Extend extend() const { return extend_; }
+ unsigned shift_amount() const { return shift_amount_; }
+ bool IsImmediateOffset() const;
+ bool IsRegisterOffset() const;
+ bool IsPreIndex() const;
+ bool IsPostIndex() const;
+
+ void AddOffset(int64_t offset);
+
+ private:
+ Register base_;
+ Register regoffset_;
+ int64_t offset_;
+ AddrMode addrmode_;
+ Shift shift_;
+ Extend extend_;
+ unsigned shift_amount_;
+};
+
+
+// Control whether or not position-independent code should be emitted.
+enum PositionIndependentCodeOption {
+ // All code generated will be position-independent; all branches and
+ // references to labels generated with the Label class will use PC-relative
+ // addressing.
+ PositionIndependentCode,
+
+ // Allow VIXL to generate code that refers to absolute addresses. With this
+ // option, it will not be possible to copy the code buffer and run it from a
+ // different address; code must be generated in its final location.
+ PositionDependentCode,
+
+ // Allow VIXL to assume that the bottom 12 bits of the address will be
+ // constant, but that the top 48 bits may change. This allows `adrp` to
+ // function in systems which copy code between pages, but otherwise maintain
+ // 4KB page alignment.
+ PageOffsetDependentCode
+};
+
+
+// Control how scaled- and unscaled-offset loads and stores are generated.
+enum LoadStoreScalingOption {
+ // Prefer scaled-immediate-offset instructions, but emit unscaled-offset,
+ // register-offset, pre-index or post-index instructions if necessary.
+ PreferScaledOffset,
+
+ // Prefer unscaled-immediate-offset instructions, but emit scaled-offset,
+ // register-offset, pre-index or post-index instructions if necessary.
+ PreferUnscaledOffset,
+
+ // Require scaled-immediate-offset instructions.
+ RequireScaledOffset,
+
+ // Require unscaled-immediate-offset instructions.
+ RequireUnscaledOffset
+};
+
+
+// Assembler.
+class Assembler : public MozBaseAssembler {
+ public:
+ Assembler(PositionIndependentCodeOption pic = PositionIndependentCode);
+
+ // System functions.
+
+ // Finalize a code buffer of generated instructions. This function must be
+ // called before executing or copying code from the buffer.
+ void FinalizeCode();
+
+#define COPYENUM(v) static const Condition v = vixl::v
+#define COPYENUM_(v) static const Condition v = vixl::v##_
+ COPYENUM(Equal);
+ COPYENUM(Zero);
+ COPYENUM(NotEqual);
+ COPYENUM(NonZero);
+ COPYENUM(AboveOrEqual);
+ COPYENUM(CarrySet);
+ COPYENUM(Below);
+ COPYENUM(CarryClear);
+ COPYENUM(Signed);
+ COPYENUM(NotSigned);
+ COPYENUM(Overflow);
+ COPYENUM(NoOverflow);
+ COPYENUM(Above);
+ COPYENUM(BelowOrEqual);
+ COPYENUM_(GreaterThanOrEqual);
+ COPYENUM_(LessThan);
+ COPYENUM_(GreaterThan);
+ COPYENUM_(LessThanOrEqual);
+ COPYENUM(Always);
+ COPYENUM(Never);
+#undef COPYENUM
+#undef COPYENUM_
+
+ // Bit set when a DoubleCondition does not map to a single ARM condition.
+ // The MacroAssembler must special-case these conditions, or else
+ // ConditionFromDoubleCondition will complain.
+ static const int DoubleConditionBitSpecial = 0x100;
+
+ enum DoubleCondition {
+ DoubleOrdered = Condition::vc,
+ DoubleEqual = Condition::eq,
+ DoubleNotEqual = Condition::ne | DoubleConditionBitSpecial,
+ DoubleGreaterThan = Condition::gt,
+ DoubleGreaterThanOrEqual = Condition::ge,
+ DoubleLessThan = Condition::lo, // Could also use Condition::mi.
+ DoubleLessThanOrEqual = Condition::ls,
+
+ // If either operand is NaN, these conditions always evaluate to true.
+ DoubleUnordered = Condition::vs,
+ DoubleEqualOrUnordered = Condition::eq | DoubleConditionBitSpecial,
+ DoubleNotEqualOrUnordered = Condition::ne,
+ DoubleGreaterThanOrUnordered = Condition::hi,
+ DoubleGreaterThanOrEqualOrUnordered = Condition::hs,
+ DoubleLessThanOrUnordered = Condition::lt,
+ DoubleLessThanOrEqualOrUnordered = Condition::le
+ };
+
+ static inline Condition InvertCondition(Condition cond) {
+ // Conditions al and nv behave identically, as "always true". They can't be
+ // inverted, because there is no "always false" condition.
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ return static_cast<Condition>(cond ^ 1);
+ }
+
+ // This is chaging the condition codes for cmp a, b to the same codes for cmp b, a.
+ static inline Condition InvertCmpCondition(Condition cond) {
+ // Conditions al and nv behave identically, as "always true". They can't be
+ // inverted, because there is no "always false" condition.
+ switch (cond) {
+ case eq:
+ case ne:
+ return cond;
+ case gt:
+ return le;
+ case le:
+ return gt;
+ case ge:
+ return lt;
+ case lt:
+ return ge;
+ case hi:
+ return lo;
+ case lo:
+ return hi;
+ case hs:
+ return ls;
+ case ls:
+ return hs;
+ case mi:
+ return pl;
+ case pl:
+ return mi;
+ default:
+ MOZ_CRASH("TODO: figure this case out.");
+ }
+ return static_cast<Condition>(cond ^ 1);
+ }
+
+ static inline DoubleCondition InvertCondition(DoubleCondition cond) {
+ switch (cond) {
+ case DoubleOrdered:
+ return DoubleUnordered;
+ case DoubleEqual:
+ return DoubleNotEqualOrUnordered;
+ case DoubleNotEqual:
+ return DoubleEqualOrUnordered;
+ case DoubleGreaterThan:
+ return DoubleLessThanOrEqualOrUnordered;
+ case DoubleGreaterThanOrEqual:
+ return DoubleLessThanOrUnordered;
+ case DoubleLessThan:
+ return DoubleGreaterThanOrEqualOrUnordered;
+ case DoubleLessThanOrEqual:
+ return DoubleGreaterThanOrUnordered;
+ case DoubleUnordered:
+ return DoubleOrdered;
+ case DoubleEqualOrUnordered:
+ return DoubleNotEqual;
+ case DoubleNotEqualOrUnordered:
+ return DoubleEqual;
+ case DoubleGreaterThanOrUnordered:
+ return DoubleLessThanOrEqual;
+ case DoubleGreaterThanOrEqualOrUnordered:
+ return DoubleLessThan;
+ case DoubleLessThanOrUnordered:
+ return DoubleGreaterThanOrEqual;
+ case DoubleLessThanOrEqualOrUnordered:
+ return DoubleGreaterThan;
+ default:
+ MOZ_CRASH("Bad condition");
+ }
+ }
+
+ static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
+ VIXL_ASSERT(!(cond & DoubleConditionBitSpecial));
+ return static_cast<Condition>(cond);
+ }
+
+ // Instruction set functions.
+
+ // Branch / Jump instructions.
+ // Branch to register.
+ void br(const Register& xn);
+ static void br(Instruction* at, const Register& xn);
+
+ // Branch with link to register.
+ void blr(const Register& xn);
+ static void blr(Instruction* at, const Register& blr);
+
+ // Branch to register with return hint.
+ void ret(const Register& xn = lr);
+
+ // Unconditional branch to label.
+ BufferOffset b(Label* label);
+
+ // Conditional branch to label.
+ BufferOffset b(Label* label, Condition cond);
+
+ // Unconditional branch to PC offset.
+ BufferOffset b(int imm26, const LabelDoc& doc);
+ static void b(Instruction* at, int imm26);
+
+ // Conditional branch to PC offset.
+ BufferOffset b(int imm19, Condition cond, const LabelDoc& doc);
+ static void b(Instruction*at, int imm19, Condition cond);
+
+ // Branch with link to label.
+ void bl(Label* label);
+
+ // Branch with link to PC offset.
+ void bl(int imm26, const LabelDoc& doc);
+ static void bl(Instruction* at, int imm26);
+
+ // Compare and branch to label if zero.
+ void cbz(const Register& rt, Label* label);
+
+ // Compare and branch to PC offset if zero.
+ void cbz(const Register& rt, int imm19, const LabelDoc& doc);
+ static void cbz(Instruction* at, const Register& rt, int imm19);
+
+ // Compare and branch to label if not zero.
+ void cbnz(const Register& rt, Label* label);
+
+ // Compare and branch to PC offset if not zero.
+ void cbnz(const Register& rt, int imm19, const LabelDoc& doc);
+ static void cbnz(Instruction* at, const Register& rt, int imm19);
+
+ // Table lookup from one register.
+ void tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Table lookup from two registers.
+ void tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vm);
+
+ // Table lookup from three registers.
+ void tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vm);
+
+ // Table lookup from four registers.
+ void tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vn4,
+ const VRegister& vm);
+
+ // Table lookup extension from one register.
+ void tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Table lookup extension from two registers.
+ void tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vm);
+
+ // Table lookup extension from three registers.
+ void tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vm);
+
+ // Table lookup extension from four registers.
+ void tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vn4,
+ const VRegister& vm);
+
+ // Test bit and branch to label if zero.
+ void tbz(const Register& rt, unsigned bit_pos, Label* label);
+
+ // Test bit and branch to PC offset if zero.
+ void tbz(const Register& rt, unsigned bit_pos, int imm14, const LabelDoc& doc);
+ static void tbz(Instruction* at, const Register& rt, unsigned bit_pos, int imm14);
+
+ // Test bit and branch to label if not zero.
+ void tbnz(const Register& rt, unsigned bit_pos, Label* label);
+
+ // Test bit and branch to PC offset if not zero.
+ void tbnz(const Register& rt, unsigned bit_pos, int imm14, const LabelDoc& doc);
+ static void tbnz(Instruction* at, const Register& rt, unsigned bit_pos, int imm14);
+
+ // Address calculation instructions.
+ // Calculate a PC-relative address. Unlike for branches the offset in adr is
+ // unscaled (i.e. the result can be unaligned).
+
+ // Calculate the address of a label.
+ void adr(const Register& rd, Label* label);
+
+ // Calculate the address of a PC offset.
+ void adr(const Register& rd, int imm21, const LabelDoc& doc);
+ static void adr(Instruction* at, const Register& rd, int imm21);
+
+ // Calculate the page address of a label.
+ void adrp(const Register& rd, Label* label);
+
+ // Calculate the page address of a PC offset.
+ void adrp(const Register& rd, int imm21, const LabelDoc& doc);
+ static void adrp(Instruction* at, const Register& rd, int imm21);
+
+ // Data Processing instructions.
+ // Add.
+ void add(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Add and update status flags.
+ void adds(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Compare negative.
+ void cmn(const Register& rn, const Operand& operand);
+
+ // Subtract.
+ void sub(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Subtract and update status flags.
+ void subs(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Compare.
+ void cmp(const Register& rn, const Operand& operand);
+
+ // Negate.
+ void neg(const Register& rd,
+ const Operand& operand);
+
+ // Negate and update status flags.
+ void negs(const Register& rd,
+ const Operand& operand);
+
+ // Add with carry bit.
+ void adc(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Add with carry bit and update status flags.
+ void adcs(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Subtract with carry bit.
+ void sbc(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Subtract with carry bit and update status flags.
+ void sbcs(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Negate with carry bit.
+ void ngc(const Register& rd,
+ const Operand& operand);
+
+ // Negate with carry bit and update status flags.
+ void ngcs(const Register& rd,
+ const Operand& operand);
+
+ // Logical instructions.
+ // Bitwise and (A & B).
+ void and_(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Bitwise and (A & B) and update status flags.
+ BufferOffset ands(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Bit test and set flags.
+ BufferOffset tst(const Register& rn, const Operand& operand);
+
+ // Bit clear (A & ~B).
+ void bic(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Bit clear (A & ~B) and update status flags.
+ void bics(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+
+ // Bitwise or (A | B).
+ void orr(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bitwise nor (A | ~B).
+ void orn(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bitwise eor/xor (A ^ B).
+ void eor(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Bitwise enor/xnor (A ^ ~B).
+ void eon(const Register& rd, const Register& rn, const Operand& operand);
+
+ // Logical shift left by variable.
+ void lslv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Logical shift right by variable.
+ void lsrv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Arithmetic shift right by variable.
+ void asrv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Rotate right by variable.
+ void rorv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Bitfield instructions.
+ // Bitfield move.
+ void bfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms);
+
+ // Signed bitfield move.
+ void sbfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms);
+
+ // Unsigned bitfield move.
+ void ubfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms);
+
+ // Bfm aliases.
+ // Bitfield insert.
+ void bfi(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(width >= 1);
+ VIXL_ASSERT(lsb + width <= rn.size());
+ bfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+ }
+
+ // Bitfield extract and insert low.
+ void bfxil(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(width >= 1);
+ VIXL_ASSERT(lsb + width <= rn.size());
+ bfm(rd, rn, lsb, lsb + width - 1);
+ }
+
+ // Sbfm aliases.
+ // Arithmetic shift right.
+ void asr(const Register& rd, const Register& rn, unsigned shift) {
+ VIXL_ASSERT(shift < rd.size());
+ sbfm(rd, rn, shift, rd.size() - 1);
+ }
+
+ // Signed bitfield insert with zero at right.
+ void sbfiz(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(width >= 1);
+ VIXL_ASSERT(lsb + width <= rn.size());
+ sbfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+ }
+
+ // Signed bitfield extract.
+ void sbfx(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(width >= 1);
+ VIXL_ASSERT(lsb + width <= rn.size());
+ sbfm(rd, rn, lsb, lsb + width - 1);
+ }
+
+ // Signed extend byte.
+ void sxtb(const Register& rd, const Register& rn) {
+ sbfm(rd, rn, 0, 7);
+ }
+
+ // Signed extend halfword.
+ void sxth(const Register& rd, const Register& rn) {
+ sbfm(rd, rn, 0, 15);
+ }
+
+ // Signed extend word.
+ void sxtw(const Register& rd, const Register& rn) {
+ sbfm(rd, rn, 0, 31);
+ }
+
+ // Ubfm aliases.
+ // Logical shift left.
+ void lsl(const Register& rd, const Register& rn, unsigned shift) {
+ unsigned reg_size = rd.size();
+ VIXL_ASSERT(shift < reg_size);
+ ubfm(rd, rn, (reg_size - shift) % reg_size, reg_size - shift - 1);
+ }
+
+ // Logical shift right.
+ void lsr(const Register& rd, const Register& rn, unsigned shift) {
+ VIXL_ASSERT(shift < rd.size());
+ ubfm(rd, rn, shift, rd.size() - 1);
+ }
+
+ // Unsigned bitfield insert with zero at right.
+ void ubfiz(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(width >= 1);
+ VIXL_ASSERT(lsb + width <= rn.size());
+ ubfm(rd, rn, (rd.size() - lsb) & (rd.size() - 1), width - 1);
+ }
+
+ // Unsigned bitfield extract.
+ void ubfx(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(width >= 1);
+ VIXL_ASSERT(lsb + width <= rn.size());
+ ubfm(rd, rn, lsb, lsb + width - 1);
+ }
+
+ // Unsigned extend byte.
+ void uxtb(const Register& rd, const Register& rn) {
+ ubfm(rd, rn, 0, 7);
+ }
+
+ // Unsigned extend halfword.
+ void uxth(const Register& rd, const Register& rn) {
+ ubfm(rd, rn, 0, 15);
+ }
+
+ // Unsigned extend word.
+ void uxtw(const Register& rd, const Register& rn) {
+ ubfm(rd, rn, 0, 31);
+ }
+
+ // Extract.
+ void extr(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ unsigned lsb);
+
+ // Conditional select: rd = cond ? rn : rm.
+ void csel(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond);
+
+ // Conditional select increment: rd = cond ? rn : rm + 1.
+ void csinc(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond);
+
+ // Conditional select inversion: rd = cond ? rn : ~rm.
+ void csinv(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond);
+
+ // Conditional select negation: rd = cond ? rn : -rm.
+ void csneg(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond);
+
+ // Conditional set: rd = cond ? 1 : 0.
+ void cset(const Register& rd, Condition cond);
+
+ // Conditional set mask: rd = cond ? -1 : 0.
+ void csetm(const Register& rd, Condition cond);
+
+ // Conditional increment: rd = cond ? rn + 1 : rn.
+ void cinc(const Register& rd, const Register& rn, Condition cond);
+
+ // Conditional invert: rd = cond ? ~rn : rn.
+ void cinv(const Register& rd, const Register& rn, Condition cond);
+
+ // Conditional negate: rd = cond ? -rn : rn.
+ void cneg(const Register& rd, const Register& rn, Condition cond);
+
+ // Rotate right.
+ void ror(const Register& rd, const Register& rs, unsigned shift) {
+ extr(rd, rs, rs, shift);
+ }
+
+ // Conditional comparison.
+ // Conditional compare negative.
+ void ccmn(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond);
+
+ // Conditional compare.
+ void ccmp(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond);
+
+ // CRC-32 checksum from byte.
+ void crc32b(const Register& rd,
+ const Register& rn,
+ const Register& rm);
+
+ // CRC-32 checksum from half-word.
+ void crc32h(const Register& rd,
+ const Register& rn,
+ const Register& rm);
+
+ // CRC-32 checksum from word.
+ void crc32w(const Register& rd,
+ const Register& rn,
+ const Register& rm);
+
+ // CRC-32 checksum from double word.
+ void crc32x(const Register& rd,
+ const Register& rn,
+ const Register& rm);
+
+ // CRC-32 C checksum from byte.
+ void crc32cb(const Register& rd,
+ const Register& rn,
+ const Register& rm);
+
+ // CRC-32 C checksum from half-word.
+ void crc32ch(const Register& rd,
+ const Register& rn,
+ const Register& rm);
+
+ // CRC-32 C checksum from word.
+ void crc32cw(const Register& rd,
+ const Register& rn,
+ const Register& rm);
+
+ // CRC-32C checksum from double word.
+ void crc32cx(const Register& rd,
+ const Register& rn,
+ const Register& rm);
+
+ // Multiply.
+ void mul(const Register& rd, const Register& rn, const Register& rm);
+
+ // Negated multiply.
+ void mneg(const Register& rd, const Register& rn, const Register& rm);
+
+ // Signed long multiply: 32 x 32 -> 64-bit.
+ void smull(const Register& rd, const Register& rn, const Register& rm);
+
+ // Signed multiply high: 64 x 64 -> 64-bit <127:64>.
+ void smulh(const Register& xd, const Register& xn, const Register& xm);
+
+ // Multiply and accumulate.
+ void madd(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra);
+
+ // Multiply and subtract.
+ void msub(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra);
+
+ // Signed long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
+ void smaddl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra);
+
+ // Unsigned long multiply and accumulate: 32 x 32 + 64 -> 64-bit.
+ void umaddl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra);
+
+ // Unsigned long multiply: 32 x 32 -> 64-bit.
+ void umull(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ umaddl(rd, rn, rm, xzr);
+ }
+
+ // Unsigned multiply high: 64 x 64 -> 64-bit <127:64>.
+ void umulh(const Register& xd,
+ const Register& xn,
+ const Register& xm);
+
+ // Signed long multiply and subtract: 64 - (32 x 32) -> 64-bit.
+ void smsubl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra);
+
+ // Unsigned long multiply and subtract: 64 - (32 x 32) -> 64-bit.
+ void umsubl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra);
+
+ // Signed integer divide.
+ void sdiv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Unsigned integer divide.
+ void udiv(const Register& rd, const Register& rn, const Register& rm);
+
+ // Bit reverse.
+ void rbit(const Register& rd, const Register& rn);
+
+ // Reverse bytes in 16-bit half words.
+ void rev16(const Register& rd, const Register& rn);
+
+ // Reverse bytes in 32-bit words.
+ void rev32(const Register& rd, const Register& rn);
+
+ // Reverse bytes.
+ void rev(const Register& rd, const Register& rn);
+
+ // Count leading zeroes.
+ void clz(const Register& rd, const Register& rn);
+
+ // Count leading sign bits.
+ void cls(const Register& rd, const Register& rn);
+
+ // Memory instructions.
+ // Load integer or FP register.
+ void ldr(const CPURegister& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Store integer or FP register.
+ void str(const CPURegister& rt, const MemOperand& dst,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Load word with sign extension.
+ void ldrsw(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Load byte.
+ void ldrb(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Store byte.
+ void strb(const Register& rt, const MemOperand& dst,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Load byte with sign extension.
+ void ldrsb(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Load half-word.
+ void ldrh(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Store half-word.
+ void strh(const Register& rt, const MemOperand& dst,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Load half-word with sign extension.
+ void ldrsh(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Load integer or FP register (with unscaled offset).
+ void ldur(const CPURegister& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Store integer or FP register (with unscaled offset).
+ void stur(const CPURegister& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Load word with sign extension.
+ void ldursw(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Load byte (with unscaled offset).
+ void ldurb(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Store byte (with unscaled offset).
+ void sturb(const Register& rt, const MemOperand& dst,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Load byte with sign extension (and unscaled offset).
+ void ldursb(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Load half-word (with unscaled offset).
+ void ldurh(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Store half-word (with unscaled offset).
+ void sturh(const Register& rt, const MemOperand& dst,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Load half-word with sign extension (and unscaled offset).
+ void ldursh(const Register& rt, const MemOperand& src,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Load integer or FP register pair.
+ void ldp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& src);
+
+ // Store integer or FP register pair.
+ void stp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& dst);
+
+ // Load word pair with sign extension.
+ void ldpsw(const Register& rt, const Register& rt2, const MemOperand& src);
+
+ // Load integer or FP register pair, non-temporal.
+ void ldnp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& src);
+
+ // Store integer or FP register pair, non-temporal.
+ void stnp(const CPURegister& rt, const CPURegister& rt2,
+ const MemOperand& dst);
+
+ // Load integer or FP register from pc + imm19 << 2.
+ void ldr(const CPURegister& rt, int imm19);
+ static void ldr(Instruction* at, const CPURegister& rt, int imm19);
+
+ // Load word with sign extension from pc + imm19 << 2.
+ void ldrsw(const Register& rt, int imm19);
+
+ // Store exclusive byte.
+ void stxrb(const Register& rs, const Register& rt, const MemOperand& dst);
+
+ // Store exclusive half-word.
+ void stxrh(const Register& rs, const Register& rt, const MemOperand& dst);
+
+ // Store exclusive register.
+ void stxr(const Register& rs, const Register& rt, const MemOperand& dst);
+
+ // Load exclusive byte.
+ void ldxrb(const Register& rt, const MemOperand& src);
+
+ // Load exclusive half-word.
+ void ldxrh(const Register& rt, const MemOperand& src);
+
+ // Load exclusive register.
+ void ldxr(const Register& rt, const MemOperand& src);
+
+ // Store exclusive register pair.
+ void stxp(const Register& rs,
+ const Register& rt,
+ const Register& rt2,
+ const MemOperand& dst);
+
+ // Load exclusive register pair.
+ void ldxp(const Register& rt, const Register& rt2, const MemOperand& src);
+
+ // Store-release exclusive byte.
+ void stlxrb(const Register& rs, const Register& rt, const MemOperand& dst);
+
+ // Store-release exclusive half-word.
+ void stlxrh(const Register& rs, const Register& rt, const MemOperand& dst);
+
+ // Store-release exclusive register.
+ void stlxr(const Register& rs, const Register& rt, const MemOperand& dst);
+
+ // Load-acquire exclusive byte.
+ void ldaxrb(const Register& rt, const MemOperand& src);
+
+ // Load-acquire exclusive half-word.
+ void ldaxrh(const Register& rt, const MemOperand& src);
+
+ // Load-acquire exclusive register.
+ void ldaxr(const Register& rt, const MemOperand& src);
+
+ // Store-release exclusive register pair.
+ void stlxp(const Register& rs,
+ const Register& rt,
+ const Register& rt2,
+ const MemOperand& dst);
+
+ // Load-acquire exclusive register pair.
+ void ldaxp(const Register& rt, const Register& rt2, const MemOperand& src);
+
+ // Store-release byte.
+ void stlrb(const Register& rt, const MemOperand& dst);
+
+ // Store-release half-word.
+ void stlrh(const Register& rt, const MemOperand& dst);
+
+ // Store-release register.
+ void stlr(const Register& rt, const MemOperand& dst);
+
+ // Load-acquire byte.
+ void ldarb(const Register& rt, const MemOperand& src);
+
+ // Load-acquire half-word.
+ void ldarh(const Register& rt, const MemOperand& src);
+
+ // Load-acquire register.
+ void ldar(const Register& rt, const MemOperand& src);
+
+ // Prefetch memory.
+ void prfm(PrefetchOperation op, const MemOperand& addr,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ // Prefetch memory (with unscaled offset).
+ void prfum(PrefetchOperation op, const MemOperand& addr,
+ LoadStoreScalingOption option = PreferUnscaledOffset);
+
+ // Prefetch from pc + imm19 << 2.
+ void prfm(PrefetchOperation op, int imm19);
+
+ // Move instructions. The default shift of -1 indicates that the move
+ // instruction will calculate an appropriate 16-bit immediate and left shift
+ // that is equal to the 64-bit immediate argument. If an explicit left shift
+ // is specified (0, 16, 32 or 48), the immediate must be a 16-bit value.
+ //
+ // For movk, an explicit shift can be used to indicate which half word should
+ // be overwritten, eg. movk(x0, 0, 0) will overwrite the least-significant
+ // half word with zero, whereas movk(x0, 0, 48) will overwrite the
+ // most-significant.
+
+ // Move immediate and keep.
+ void movk(const Register& rd, uint64_t imm, int shift = -1) {
+ MoveWide(rd, imm, shift, MOVK);
+ }
+
+ // Move inverted immediate.
+ void movn(const Register& rd, uint64_t imm, int shift = -1) {
+ MoveWide(rd, imm, shift, MOVN);
+ }
+
+ // Move immediate.
+ void movz(const Register& rd, uint64_t imm, int shift = -1) {
+ MoveWide(rd, imm, shift, MOVZ);
+ }
+
+ // Misc instructions.
+ // Monitor debug-mode breakpoint.
+ void brk(int code);
+
+ // Halting debug-mode breakpoint.
+ void hlt(int code);
+
+ // Generate exception targeting EL1.
+ void svc(int code);
+ static void svc(Instruction* at, int code);
+
+ // Move register to register.
+ void mov(const Register& rd, const Register& rn);
+
+ // Move inverted operand to register.
+ void mvn(const Register& rd, const Operand& operand);
+
+ // System instructions.
+ // Move to register from system register.
+ void mrs(const Register& rt, SystemRegister sysreg);
+
+ // Move from register to system register.
+ void msr(SystemRegister sysreg, const Register& rt);
+
+ // System instruction.
+ void sys(int op1, int crn, int crm, int op2, const Register& rt = xzr);
+
+ // System instruction with pre-encoded op (op1:crn:crm:op2).
+ void sys(int op, const Register& rt = xzr);
+
+ // System data cache operation.
+ void dc(DataCacheOp op, const Register& rt);
+
+ // System instruction cache operation.
+ void ic(InstructionCacheOp op, const Register& rt);
+
+ // System hint.
+ BufferOffset hint(SystemHint code);
+ static void hint(Instruction* at, SystemHint code);
+
+ // Clear exclusive monitor.
+ void clrex(int imm4 = 0xf);
+
+ // Data memory barrier.
+ void dmb(BarrierDomain domain, BarrierType type);
+
+ // Data synchronization barrier.
+ void dsb(BarrierDomain domain, BarrierType type);
+
+ // Instruction synchronization barrier.
+ void isb();
+
+ // Alias for system instructions.
+ // No-op.
+ BufferOffset nop() {
+ return hint(NOP);
+ }
+ static void nop(Instruction* at);
+
+ // Alias for system instructions.
+ // Conditional speculation barrier.
+ BufferOffset csdb() {
+ return hint(CSDB);
+ }
+ static void csdb(Instruction* at);
+
+ // FP and NEON instructions.
+ // Move double precision immediate to FP register.
+ void fmov(const VRegister& vd, double imm);
+
+ // Move single precision immediate to FP register.
+ void fmov(const VRegister& vd, float imm);
+
+ // Move FP register to register.
+ void fmov(const Register& rd, const VRegister& fn);
+
+ // Move register to FP register.
+ void fmov(const VRegister& vd, const Register& rn);
+
+ // Move FP register to FP register.
+ void fmov(const VRegister& vd, const VRegister& fn);
+
+ // Move 64-bit register to top half of 128-bit FP register.
+ void fmov(const VRegister& vd, int index, const Register& rn);
+
+ // Move top half of 128-bit FP register to 64-bit register.
+ void fmov(const Register& rd, const VRegister& vn, int index);
+
+ // FP add.
+ void fadd(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP subtract.
+ void fsub(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP multiply.
+ void fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm);
+
+ // FP fused multiply-add.
+ void fmadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va);
+
+ // FP fused multiply-subtract.
+ void fmsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va);
+
+ // FP fused multiply-add and negate.
+ void fnmadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va);
+
+ // FP fused multiply-subtract and negate.
+ void fnmsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va);
+
+ // FP multiply-negate scalar.
+ void fnmul(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP reciprocal exponent scalar.
+ void frecpx(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP divide.
+ void fdiv(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+ // FP maximum.
+ void fmax(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+ // FP minimum.
+ void fmin(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+ // FP maximum number.
+ void fmaxnm(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+ // FP minimum number.
+ void fminnm(const VRegister& vd, const VRegister& fn, const VRegister& vm);
+
+ // FP absolute.
+ void fabs(const VRegister& vd, const VRegister& vn);
+
+ // FP negate.
+ void fneg(const VRegister& vd, const VRegister& vn);
+
+ // FP square root.
+ void fsqrt(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, nearest with ties to away.
+ void frinta(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, implicit rounding.
+ void frinti(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, toward minus infinity.
+ void frintm(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, nearest with ties to even.
+ void frintn(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, toward plus infinity.
+ void frintp(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, exact, implicit rounding.
+ void frintx(const VRegister& vd, const VRegister& vn);
+
+ // FP round to integer, towards zero.
+ void frintz(const VRegister& vd, const VRegister& vn);
+
+ void FPCompareMacro(const VRegister& vn,
+ double value,
+ FPTrapFlags trap);
+
+ void FPCompareMacro(const VRegister& vn,
+ const VRegister& vm,
+ FPTrapFlags trap);
+
+ // FP compare registers.
+ void fcmp(const VRegister& vn, const VRegister& vm);
+
+ // FP compare immediate.
+ void fcmp(const VRegister& vn, double value);
+
+ void FPCCompareMacro(const VRegister& vn,
+ const VRegister& vm,
+ StatusFlags nzcv,
+ Condition cond,
+ FPTrapFlags trap);
+
+ // FP conditional compare.
+ void fccmp(const VRegister& vn,
+ const VRegister& vm,
+ StatusFlags nzcv,
+ Condition cond);
+
+ // FP signaling compare registers.
+ void fcmpe(const VRegister& vn, const VRegister& vm);
+
+ // FP signaling compare immediate.
+ void fcmpe(const VRegister& vn, double value);
+
+ // FP conditional signaling compare.
+ void fccmpe(const VRegister& vn,
+ const VRegister& vm,
+ StatusFlags nzcv,
+ Condition cond);
+
+ // FP conditional select.
+ void fcsel(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ Condition cond);
+
+ // Common FP Convert functions.
+ void NEONFPConvertToInt(const Register& rd,
+ const VRegister& vn,
+ Instr op);
+ void NEONFPConvertToInt(const VRegister& vd,
+ const VRegister& vn,
+ Instr op);
+
+ // FP convert between precisions.
+ void fcvt(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to higher precision.
+ void fcvtl(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to higher precision (second part).
+ void fcvtl2(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to lower precision.
+ void fcvtn(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to lower prevision (second part).
+ void fcvtn2(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to lower precision, rounding to odd.
+ void fcvtxn(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to lower precision, rounding to odd (second part).
+ void fcvtxn2(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to signed integer, nearest with ties to away.
+ void fcvtas(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, nearest with ties to away.
+ void fcvtau(const Register& rd, const VRegister& vn);
+
+ // FP convert to signed integer, nearest with ties to away.
+ void fcvtas(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to unsigned integer, nearest with ties to away.
+ void fcvtau(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to signed integer, round towards -infinity.
+ void fcvtms(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, round towards -infinity.
+ void fcvtmu(const Register& rd, const VRegister& vn);
+
+ // FP convert to signed integer, round towards -infinity.
+ void fcvtms(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to unsigned integer, round towards -infinity.
+ void fcvtmu(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to signed integer, nearest with ties to even.
+ void fcvtns(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, nearest with ties to even.
+ void fcvtnu(const Register& rd, const VRegister& vn);
+
+ // FP convert to signed integer, nearest with ties to even.
+ void fcvtns(const VRegister& rd, const VRegister& vn);
+
+ // FP JavaScript convert to signed integer, rounding toward zero [Armv8.3].
+ void fjcvtzs(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, nearest with ties to even.
+ void fcvtnu(const VRegister& rd, const VRegister& vn);
+
+ // FP convert to signed integer or fixed-point, round towards zero.
+ void fcvtzs(const Register& rd, const VRegister& vn, int fbits = 0);
+
+ // FP convert to unsigned integer or fixed-point, round towards zero.
+ void fcvtzu(const Register& rd, const VRegister& vn, int fbits = 0);
+
+ // FP convert to signed integer or fixed-point, round towards zero.
+ void fcvtzs(const VRegister& vd, const VRegister& vn, int fbits = 0);
+
+ // FP convert to unsigned integer or fixed-point, round towards zero.
+ void fcvtzu(const VRegister& vd, const VRegister& vn, int fbits = 0);
+
+ // FP convert to signed integer, round towards +infinity.
+ void fcvtps(const Register& rd, const VRegister& vn);
+
+ // FP convert to unsigned integer, round towards +infinity.
+ void fcvtpu(const Register& rd, const VRegister& vn);
+
+ // FP convert to signed integer, round towards +infinity.
+ void fcvtps(const VRegister& vd, const VRegister& vn);
+
+ // FP convert to unsigned integer, round towards +infinity.
+ void fcvtpu(const VRegister& vd, const VRegister& vn);
+
+ // Convert signed integer or fixed point to FP.
+ void scvtf(const VRegister& fd, const Register& rn, int fbits = 0);
+
+ // Convert unsigned integer or fixed point to FP.
+ void ucvtf(const VRegister& fd, const Register& rn, int fbits = 0);
+
+ // Convert signed integer or fixed-point to FP.
+ void scvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
+
+ // Convert unsigned integer or fixed-point to FP.
+ void ucvtf(const VRegister& fd, const VRegister& vn, int fbits = 0);
+
+ // Unsigned absolute difference.
+ void uabd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed absolute difference.
+ void sabd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned absolute difference and accumulate.
+ void uaba(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed absolute difference and accumulate.
+ void saba(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Add.
+ void add(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Subtract.
+ void sub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned halving add.
+ void uhadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed halving add.
+ void shadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned rounding halving add.
+ void urhadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed rounding halving add.
+ void srhadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned halving sub.
+ void uhsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed halving sub.
+ void shsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned saturating add.
+ void uqadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating add.
+ void sqadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned saturating subtract.
+ void uqsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating subtract.
+ void sqsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Add pairwise.
+ void addp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Add pair of elements scalar.
+ void addp(const VRegister& vd,
+ const VRegister& vn);
+
+ // Multiply-add to accumulator.
+ void mla(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Multiply-subtract to accumulator.
+ void mls(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Multiply.
+ void mul(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Multiply by scalar element.
+ void mul(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Multiply-add by scalar element.
+ void mla(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Multiply-subtract by scalar element.
+ void mls(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply-add by scalar element.
+ void smlal(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply-add by scalar element (second part).
+ void smlal2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply-add by scalar element.
+ void umlal(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply-add by scalar element (second part).
+ void umlal2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply-sub by scalar element.
+ void smlsl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply-sub by scalar element (second part).
+ void smlsl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply-sub by scalar element.
+ void umlsl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply-sub by scalar element (second part).
+ void umlsl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply by scalar element.
+ void smull(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed long multiply by scalar element (second part).
+ void smull2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply by scalar element.
+ void umull(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply by scalar element (second part).
+ void umull2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating double long multiply by element.
+ void sqdmull(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating double long multiply by element (second part).
+ void sqdmull2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating doubling long multiply-add by element.
+ void sqdmlal(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating doubling long multiply-add by element (second part).
+ void sqdmlal2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating doubling long multiply-sub by element.
+ void sqdmlsl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating doubling long multiply-sub by element (second part).
+ void sqdmlsl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Compare equal.
+ void cmeq(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Compare signed greater than or equal.
+ void cmge(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Compare signed greater than.
+ void cmgt(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Compare unsigned higher.
+ void cmhi(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Compare unsigned higher or same.
+ void cmhs(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Compare bitwise test bits nonzero.
+ void cmtst(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Compare bitwise to zero.
+ void cmeq(const VRegister& vd,
+ const VRegister& vn,
+ int value);
+
+ // Compare signed greater than or equal to zero.
+ void cmge(const VRegister& vd,
+ const VRegister& vn,
+ int value);
+
+ // Compare signed greater than zero.
+ void cmgt(const VRegister& vd,
+ const VRegister& vn,
+ int value);
+
+ // Compare signed less than or equal to zero.
+ void cmle(const VRegister& vd,
+ const VRegister& vn,
+ int value);
+
+ // Compare signed less than zero.
+ void cmlt(const VRegister& vd,
+ const VRegister& vn,
+ int value);
+
+ // Signed shift left by register.
+ void sshl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned shift left by register.
+ void ushl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating shift left by register.
+ void sqshl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned saturating shift left by register.
+ void uqshl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed rounding shift left by register.
+ void srshl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned rounding shift left by register.
+ void urshl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating rounding shift left by register.
+ void sqrshl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned saturating rounding shift left by register.
+ void uqrshl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Bitwise and.
+ void and_(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Bitwise or.
+ void orr(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Bitwise or immediate.
+ void orr(const VRegister& vd,
+ const int imm8,
+ const int left_shift = 0);
+
+ // Move register to register.
+ void mov(const VRegister& vd,
+ const VRegister& vn);
+
+ // Bitwise orn.
+ void orn(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Bitwise eor.
+ void eor(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Bit clear immediate.
+ void bic(const VRegister& vd,
+ const int imm8,
+ const int left_shift = 0);
+
+ // Bit clear.
+ void bic(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Bitwise insert if false.
+ void bif(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Bitwise insert if true.
+ void bit(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Bitwise select.
+ void bsl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Polynomial multiply.
+ void pmul(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Vector move immediate.
+ void movi(const VRegister& vd,
+ const uint64_t imm,
+ Shift shift = LSL,
+ const int shift_amount = 0);
+
+ // Bitwise not.
+ void mvn(const VRegister& vd,
+ const VRegister& vn);
+
+ // Vector move inverted immediate.
+ void mvni(const VRegister& vd,
+ const int imm8,
+ Shift shift = LSL,
+ const int shift_amount = 0);
+
+ // Signed saturating accumulate of unsigned value.
+ void suqadd(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned saturating accumulate of signed value.
+ void usqadd(const VRegister& vd,
+ const VRegister& vn);
+
+ // Absolute value.
+ void abs(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed saturating absolute value.
+ void sqabs(const VRegister& vd,
+ const VRegister& vn);
+
+ // Negate.
+ void neg(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed saturating negate.
+ void sqneg(const VRegister& vd,
+ const VRegister& vn);
+
+ // Bitwise not.
+ void not_(const VRegister& vd,
+ const VRegister& vn);
+
+ // Extract narrow.
+ void xtn(const VRegister& vd,
+ const VRegister& vn);
+
+ // Extract narrow (second part).
+ void xtn2(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed saturating extract narrow.
+ void sqxtn(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed saturating extract narrow (second part).
+ void sqxtn2(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned saturating extract narrow.
+ void uqxtn(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned saturating extract narrow (second part).
+ void uqxtn2(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed saturating extract unsigned narrow.
+ void sqxtun(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed saturating extract unsigned narrow (second part).
+ void sqxtun2(const VRegister& vd,
+ const VRegister& vn);
+
+ // Extract vector from pair of vectors.
+ void ext(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int index);
+
+ // Duplicate vector element to vector or scalar.
+ void dup(const VRegister& vd,
+ const VRegister& vn,
+ int vn_index);
+
+ // Move vector element to scalar.
+ void mov(const VRegister& vd,
+ const VRegister& vn,
+ int vn_index);
+
+ // Duplicate general-purpose register to vector.
+ void dup(const VRegister& vd,
+ const Register& rn);
+
+ // Insert vector element from another vector element.
+ void ins(const VRegister& vd,
+ int vd_index,
+ const VRegister& vn,
+ int vn_index);
+
+ // Move vector element to another vector element.
+ void mov(const VRegister& vd,
+ int vd_index,
+ const VRegister& vn,
+ int vn_index);
+
+ // Insert vector element from general-purpose register.
+ void ins(const VRegister& vd,
+ int vd_index,
+ const Register& rn);
+
+ // Move general-purpose register to a vector element.
+ void mov(const VRegister& vd,
+ int vd_index,
+ const Register& rn);
+
+ // Unsigned move vector element to general-purpose register.
+ void umov(const Register& rd,
+ const VRegister& vn,
+ int vn_index);
+
+ // Move vector element to general-purpose register.
+ void mov(const Register& rd,
+ const VRegister& vn,
+ int vn_index);
+
+ // Signed move vector element to general-purpose register.
+ void smov(const Register& rd,
+ const VRegister& vn,
+ int vn_index);
+
+ // One-element structure load to one register.
+ void ld1(const VRegister& vt,
+ const MemOperand& src);
+
+ // One-element structure load to two registers.
+ void ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src);
+
+ // One-element structure load to three registers.
+ void ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src);
+
+ // One-element structure load to four registers.
+ void ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src);
+
+ // One-element single structure load to one lane.
+ void ld1(const VRegister& vt,
+ int lane,
+ const MemOperand& src);
+
+ // One-element single structure load to all lanes.
+ void ld1r(const VRegister& vt,
+ const MemOperand& src);
+
+ // Two-element structure load.
+ void ld2(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src);
+
+ // Two-element single structure load to one lane.
+ void ld2(const VRegister& vt,
+ const VRegister& vt2,
+ int lane,
+ const MemOperand& src);
+
+ // Two-element single structure load to all lanes.
+ void ld2r(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src);
+
+ // Three-element structure load.
+ void ld3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src);
+
+ // Three-element single structure load to one lane.
+ void ld3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ int lane,
+ const MemOperand& src);
+
+ // Three-element single structure load to all lanes.
+ void ld3r(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src);
+
+ // Four-element structure load.
+ void ld4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src);
+
+ // Four-element single structure load to one lane.
+ void ld4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ int lane,
+ const MemOperand& src);
+
+ // Four-element single structure load to all lanes.
+ void ld4r(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src);
+
+ // Count leading sign bits.
+ void cls(const VRegister& vd,
+ const VRegister& vn);
+
+ // Count leading zero bits (vector).
+ void clz(const VRegister& vd,
+ const VRegister& vn);
+
+ // Population count per byte.
+ void cnt(const VRegister& vd,
+ const VRegister& vn);
+
+ // Reverse bit order.
+ void rbit(const VRegister& vd,
+ const VRegister& vn);
+
+ // Reverse elements in 16-bit halfwords.
+ void rev16(const VRegister& vd,
+ const VRegister& vn);
+
+ // Reverse elements in 32-bit words.
+ void rev32(const VRegister& vd,
+ const VRegister& vn);
+
+ // Reverse elements in 64-bit doublewords.
+ void rev64(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned reciprocal square root estimate.
+ void ursqrte(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned reciprocal estimate.
+ void urecpe(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed pairwise long add.
+ void saddlp(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned pairwise long add.
+ void uaddlp(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed pairwise long add and accumulate.
+ void sadalp(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned pairwise long add and accumulate.
+ void uadalp(const VRegister& vd,
+ const VRegister& vn);
+
+ // Shift left by immediate.
+ void shl(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed saturating shift left by immediate.
+ void sqshl(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed saturating shift left unsigned by immediate.
+ void sqshlu(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned saturating shift left by immediate.
+ void uqshl(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed shift left long by immediate.
+ void sshll(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed shift left long by immediate (second part).
+ void sshll2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed extend long.
+ void sxtl(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed extend long (second part).
+ void sxtl2(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned shift left long by immediate.
+ void ushll(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned shift left long by immediate (second part).
+ void ushll2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Shift left long by element size.
+ void shll(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Shift left long by element size (second part).
+ void shll2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned extend long.
+ void uxtl(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned extend long (second part).
+ void uxtl2(const VRegister& vd,
+ const VRegister& vn);
+
+ // Shift left by immediate and insert.
+ void sli(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Shift right by immediate and insert.
+ void sri(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed maximum.
+ void smax(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed pairwise maximum.
+ void smaxp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Add across vector.
+ void addv(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed add long across vector.
+ void saddlv(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned add long across vector.
+ void uaddlv(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP maximum number across vector.
+ void fmaxnmv(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP maximum across vector.
+ void fmaxv(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP minimum number across vector.
+ void fminnmv(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP minimum across vector.
+ void fminv(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed maximum across vector.
+ void smaxv(const VRegister& vd,
+ const VRegister& vn);
+
+ // Signed minimum.
+ void smin(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed minimum pairwise.
+ void sminp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed minimum across vector.
+ void sminv(const VRegister& vd,
+ const VRegister& vn);
+
+ // One-element structure store from one register.
+ void st1(const VRegister& vt,
+ const MemOperand& src);
+
+ // One-element structure store from two registers.
+ void st1(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src);
+
+ // One-element structure store from three registers.
+ void st1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src);
+
+ // One-element structure store from four registers.
+ void st1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src);
+
+ // One-element single structure store from one lane.
+ void st1(const VRegister& vt,
+ int lane,
+ const MemOperand& src);
+
+ // Two-element structure store from two registers.
+ void st2(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src);
+
+ // Two-element single structure store from two lanes.
+ void st2(const VRegister& vt,
+ const VRegister& vt2,
+ int lane,
+ const MemOperand& src);
+
+ // Three-element structure store from three registers.
+ void st3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src);
+
+ // Three-element single structure store from three lanes.
+ void st3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ int lane,
+ const MemOperand& src);
+
+ // Four-element structure store from four registers.
+ void st4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src);
+
+ // Four-element single structure store from four lanes.
+ void st4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ int lane,
+ const MemOperand& src);
+
+ // Unsigned add long.
+ void uaddl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned add long (second part).
+ void uaddl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned add wide.
+ void uaddw(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned add wide (second part).
+ void uaddw2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed add long.
+ void saddl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed add long (second part).
+ void saddl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed add wide.
+ void saddw(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed add wide (second part).
+ void saddw2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned subtract long.
+ void usubl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned subtract long (second part).
+ void usubl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned subtract wide.
+ void usubw(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned subtract wide (second part).
+ void usubw2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed subtract long.
+ void ssubl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed subtract long (second part).
+ void ssubl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed integer subtract wide.
+ void ssubw(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed integer subtract wide (second part).
+ void ssubw2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned maximum.
+ void umax(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned pairwise maximum.
+ void umaxp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned maximum across vector.
+ void umaxv(const VRegister& vd,
+ const VRegister& vn);
+
+ // Unsigned minimum.
+ void umin(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned pairwise minimum.
+ void uminp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned minimum across vector.
+ void uminv(const VRegister& vd,
+ const VRegister& vn);
+
+ // Transpose vectors (primary).
+ void trn1(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Transpose vectors (secondary).
+ void trn2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unzip vectors (primary).
+ void uzp1(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unzip vectors (secondary).
+ void uzp2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Zip vectors (primary).
+ void zip1(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Zip vectors (secondary).
+ void zip2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed shift right by immediate.
+ void sshr(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned shift right by immediate.
+ void ushr(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed rounding shift right by immediate.
+ void srshr(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned rounding shift right by immediate.
+ void urshr(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed shift right by immediate and accumulate.
+ void ssra(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned shift right by immediate and accumulate.
+ void usra(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed rounding shift right by immediate and accumulate.
+ void srsra(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned rounding shift right by immediate and accumulate.
+ void ursra(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Shift right narrow by immediate.
+ void shrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Shift right narrow by immediate (second part).
+ void shrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Rounding shift right narrow by immediate.
+ void rshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Rounding shift right narrow by immediate (second part).
+ void rshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned saturating shift right narrow by immediate.
+ void uqshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned saturating shift right narrow by immediate (second part).
+ void uqshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned saturating rounding shift right narrow by immediate.
+ void uqrshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Unsigned saturating rounding shift right narrow by immediate (second part).
+ void uqrshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed saturating shift right narrow by immediate.
+ void sqshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed saturating shift right narrow by immediate (second part).
+ void sqshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed saturating rounded shift right narrow by immediate.
+ void sqrshrn(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed saturating rounded shift right narrow by immediate (second part).
+ void sqrshrn2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed saturating shift right unsigned narrow by immediate.
+ void sqshrun(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed saturating shift right unsigned narrow by immediate (second part).
+ void sqshrun2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed sat rounded shift right unsigned narrow by immediate.
+ void sqrshrun(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // Signed sat rounded shift right unsigned narrow by immediate (second part).
+ void sqrshrun2(const VRegister& vd,
+ const VRegister& vn,
+ int shift);
+
+ // FP reciprocal step.
+ void frecps(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP reciprocal estimate.
+ void frecpe(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP reciprocal square root estimate.
+ void frsqrte(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP reciprocal square root step.
+ void frsqrts(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed absolute difference and accumulate long.
+ void sabal(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed absolute difference and accumulate long (second part).
+ void sabal2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned absolute difference and accumulate long.
+ void uabal(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned absolute difference and accumulate long (second part).
+ void uabal2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed absolute difference long.
+ void sabdl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed absolute difference long (second part).
+ void sabdl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned absolute difference long.
+ void uabdl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned absolute difference long (second part).
+ void uabdl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Polynomial multiply long.
+ void pmull(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Polynomial multiply long (second part).
+ void pmull2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed long multiply-add.
+ void smlal(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed long multiply-add (second part).
+ void smlal2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned long multiply-add.
+ void umlal(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned long multiply-add (second part).
+ void umlal2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed long multiply-sub.
+ void smlsl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed long multiply-sub (second part).
+ void smlsl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned long multiply-sub.
+ void umlsl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned long multiply-sub (second part).
+ void umlsl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed long multiply.
+ void smull(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed long multiply (second part).
+ void smull2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating doubling long multiply-add.
+ void sqdmlal(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating doubling long multiply-add (second part).
+ void sqdmlal2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating doubling long multiply-subtract.
+ void sqdmlsl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating doubling long multiply-subtract (second part).
+ void sqdmlsl2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating doubling long multiply.
+ void sqdmull(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating doubling long multiply (second part).
+ void sqdmull2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating doubling multiply returning high half.
+ void sqdmulh(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating rounding doubling multiply returning high half.
+ void sqrdmulh(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Signed saturating doubling multiply element returning high half.
+ void sqdmulh(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Signed saturating rounding doubling multiply element returning high half.
+ void sqrdmulh(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // Unsigned long multiply long.
+ void umull(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Unsigned long multiply (second part).
+ void umull2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Add narrow returning high half.
+ void addhn(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Add narrow returning high half (second part).
+ void addhn2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Rounding add narrow returning high half.
+ void raddhn(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Rounding add narrow returning high half (second part).
+ void raddhn2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Subtract narrow returning high half.
+ void subhn(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Subtract narrow returning high half (second part).
+ void subhn2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Rounding subtract narrow returning high half.
+ void rsubhn(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // Rounding subtract narrow returning high half (second part).
+ void rsubhn2(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP vector multiply accumulate.
+ void fmla(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP vector multiply subtract.
+ void fmls(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP vector multiply extended.
+ void fmulx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP absolute greater than or equal.
+ void facge(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP absolute greater than.
+ void facgt(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP multiply by element.
+ void fmul(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // FP fused multiply-add to accumulator by element.
+ void fmla(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // FP fused multiply-sub from accumulator by element.
+ void fmls(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // FP multiply extended by element.
+ void fmulx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index);
+
+ // FP compare equal.
+ void fcmeq(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP greater than.
+ void fcmgt(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP greater than or equal.
+ void fcmge(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP compare equal to zero.
+ void fcmeq(const VRegister& vd,
+ const VRegister& vn,
+ double imm);
+
+ // FP greater than zero.
+ void fcmgt(const VRegister& vd,
+ const VRegister& vn,
+ double imm);
+
+ // FP greater than or equal to zero.
+ void fcmge(const VRegister& vd,
+ const VRegister& vn,
+ double imm);
+
+ // FP less than or equal to zero.
+ void fcmle(const VRegister& vd,
+ const VRegister& vn,
+ double imm);
+
+ // FP less than to zero.
+ void fcmlt(const VRegister& vd,
+ const VRegister& vn,
+ double imm);
+
+ // FP absolute difference.
+ void fabd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP pairwise add vector.
+ void faddp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP pairwise add scalar.
+ void faddp(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP pairwise maximum vector.
+ void fmaxp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP pairwise maximum scalar.
+ void fmaxp(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP pairwise minimum vector.
+ void fminp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP pairwise minimum scalar.
+ void fminp(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP pairwise maximum number vector.
+ void fmaxnmp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP pairwise maximum number scalar.
+ void fmaxnmp(const VRegister& vd,
+ const VRegister& vn);
+
+ // FP pairwise minimum number vector.
+ void fminnmp(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm);
+
+ // FP pairwise minimum number scalar.
+ void fminnmp(const VRegister& vd,
+ const VRegister& vn);
+
+ // Emit generic instructions.
+ // Emit raw instructions into the instruction stream.
+ void dci(Instr raw_inst) { Emit(raw_inst); }
+
+ // Emit 32 bits of data into the instruction stream.
+ void dc32(uint32_t data) {
+ EmitData(&data, sizeof(data));
+ }
+
+ // Emit 64 bits of data into the instruction stream.
+ void dc64(uint64_t data) {
+ EmitData(&data, sizeof(data));
+ }
+
+ // Code generation helpers.
+
+ // Register encoding.
+ static Instr Rd(CPURegister rd) {
+ VIXL_ASSERT(rd.code() != kSPRegInternalCode);
+ return rd.code() << Rd_offset;
+ }
+
+ static Instr Rn(CPURegister rn) {
+ VIXL_ASSERT(rn.code() != kSPRegInternalCode);
+ return rn.code() << Rn_offset;
+ }
+
+ static Instr Rm(CPURegister rm) {
+ VIXL_ASSERT(rm.code() != kSPRegInternalCode);
+ return rm.code() << Rm_offset;
+ }
+
+ static Instr RmNot31(CPURegister rm) {
+ VIXL_ASSERT(rm.code() != kSPRegInternalCode);
+ VIXL_ASSERT(!rm.IsZero());
+ return Rm(rm);
+ }
+
+ static Instr Ra(CPURegister ra) {
+ VIXL_ASSERT(ra.code() != kSPRegInternalCode);
+ return ra.code() << Ra_offset;
+ }
+
+ static Instr Rt(CPURegister rt) {
+ VIXL_ASSERT(rt.code() != kSPRegInternalCode);
+ return rt.code() << Rt_offset;
+ }
+
+ static Instr Rt2(CPURegister rt2) {
+ VIXL_ASSERT(rt2.code() != kSPRegInternalCode);
+ return rt2.code() << Rt2_offset;
+ }
+
+ static Instr Rs(CPURegister rs) {
+ VIXL_ASSERT(rs.code() != kSPRegInternalCode);
+ return rs.code() << Rs_offset;
+ }
+
+ // These encoding functions allow the stack pointer to be encoded, and
+ // disallow the zero register.
+ static Instr RdSP(Register rd) {
+ VIXL_ASSERT(!rd.IsZero());
+ return (rd.code() & kRegCodeMask) << Rd_offset;
+ }
+
+ static Instr RnSP(Register rn) {
+ VIXL_ASSERT(!rn.IsZero());
+ return (rn.code() & kRegCodeMask) << Rn_offset;
+ }
+
+ // Flags encoding.
+ static Instr Flags(FlagsUpdate S) {
+ if (S == SetFlags) {
+ return 1 << FlagsUpdate_offset;
+ } else if (S == LeaveFlags) {
+ return 0 << FlagsUpdate_offset;
+ }
+ VIXL_UNREACHABLE();
+ return 0;
+ }
+
+ static Instr Cond(Condition cond) {
+ return cond << Condition_offset;
+ }
+
+ // PC-relative address encoding.
+ static Instr ImmPCRelAddress(int imm21) {
+ VIXL_ASSERT(IsInt21(imm21));
+ Instr imm = static_cast<Instr>(TruncateToUint21(imm21));
+ Instr immhi = (imm >> ImmPCRelLo_width) << ImmPCRelHi_offset;
+ Instr immlo = imm << ImmPCRelLo_offset;
+ return (immhi & ImmPCRelHi_mask) | (immlo & ImmPCRelLo_mask);
+ }
+
+ // Branch encoding.
+ static Instr ImmUncondBranch(int imm26) {
+ VIXL_ASSERT(IsInt26(imm26));
+ return TruncateToUint26(imm26) << ImmUncondBranch_offset;
+ }
+
+ static Instr ImmCondBranch(int imm19) {
+ VIXL_ASSERT(IsInt19(imm19));
+ return TruncateToUint19(imm19) << ImmCondBranch_offset;
+ }
+
+ static Instr ImmCmpBranch(int imm19) {
+ VIXL_ASSERT(IsInt19(imm19));
+ return TruncateToUint19(imm19) << ImmCmpBranch_offset;
+ }
+
+ static Instr ImmTestBranch(int imm14) {
+ VIXL_ASSERT(IsInt14(imm14));
+ return TruncateToUint14(imm14) << ImmTestBranch_offset;
+ }
+
+ static Instr ImmTestBranchBit(unsigned bit_pos) {
+ VIXL_ASSERT(IsUint6(bit_pos));
+ // Subtract five from the shift offset, as we need bit 5 from bit_pos.
+ unsigned b5 = bit_pos << (ImmTestBranchBit5_offset - 5);
+ unsigned b40 = bit_pos << ImmTestBranchBit40_offset;
+ b5 &= ImmTestBranchBit5_mask;
+ b40 &= ImmTestBranchBit40_mask;
+ return b5 | b40;
+ }
+
+ // Data Processing encoding.
+ static Instr SF(Register rd) {
+ return rd.Is64Bits() ? SixtyFourBits : ThirtyTwoBits;
+ }
+
+ static Instr ImmAddSub(int imm) {
+ VIXL_ASSERT(IsImmAddSub(imm));
+ if (IsUint12(imm)) { // No shift required.
+ imm <<= ImmAddSub_offset;
+ } else {
+ imm = ((imm >> 12) << ImmAddSub_offset) | (1 << ShiftAddSub_offset);
+ }
+ return imm;
+ }
+
+ static Instr ImmS(unsigned imms, unsigned reg_size) {
+ VIXL_ASSERT(((reg_size == kXRegSize) && IsUint6(imms)) ||
+ ((reg_size == kWRegSize) && IsUint5(imms)));
+ USE(reg_size);
+ return imms << ImmS_offset;
+ }
+
+ static Instr ImmR(unsigned immr, unsigned reg_size) {
+ VIXL_ASSERT(((reg_size == kXRegSize) && IsUint6(immr)) ||
+ ((reg_size == kWRegSize) && IsUint5(immr)));
+ USE(reg_size);
+ VIXL_ASSERT(IsUint6(immr));
+ return immr << ImmR_offset;
+ }
+
+ static Instr ImmSetBits(unsigned imms, unsigned reg_size) {
+ VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+ VIXL_ASSERT(IsUint6(imms));
+ VIXL_ASSERT((reg_size == kXRegSize) || IsUint6(imms + 3));
+ USE(reg_size);
+ return imms << ImmSetBits_offset;
+ }
+
+ static Instr ImmRotate(unsigned immr, unsigned reg_size) {
+ VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+ VIXL_ASSERT(((reg_size == kXRegSize) && IsUint6(immr)) ||
+ ((reg_size == kWRegSize) && IsUint5(immr)));
+ USE(reg_size);
+ return immr << ImmRotate_offset;
+ }
+
+ static Instr ImmLLiteral(int imm19) {
+ VIXL_ASSERT(IsInt19(imm19));
+ return TruncateToUint19(imm19) << ImmLLiteral_offset;
+ }
+
+ static Instr BitN(unsigned bitn, unsigned reg_size) {
+ VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+ VIXL_ASSERT((reg_size == kXRegSize) || (bitn == 0));
+ USE(reg_size);
+ return bitn << BitN_offset;
+ }
+
+ static Instr ShiftDP(Shift shift) {
+ VIXL_ASSERT(shift == LSL || shift == LSR || shift == ASR || shift == ROR);
+ return shift << ShiftDP_offset;
+ }
+
+ static Instr ImmDPShift(unsigned amount) {
+ VIXL_ASSERT(IsUint6(amount));
+ return amount << ImmDPShift_offset;
+ }
+
+ static Instr ExtendMode(Extend extend) {
+ return extend << ExtendMode_offset;
+ }
+
+ static Instr ImmExtendShift(unsigned left_shift) {
+ VIXL_ASSERT(left_shift <= 4);
+ return left_shift << ImmExtendShift_offset;
+ }
+
+ static Instr ImmCondCmp(unsigned imm) {
+ VIXL_ASSERT(IsUint5(imm));
+ return imm << ImmCondCmp_offset;
+ }
+
+ static Instr Nzcv(StatusFlags nzcv) {
+ return ((nzcv >> Flags_offset) & 0xf) << Nzcv_offset;
+ }
+
+ // MemOperand offset encoding.
+ static Instr ImmLSUnsigned(int imm12) {
+ VIXL_ASSERT(IsUint12(imm12));
+ return imm12 << ImmLSUnsigned_offset;
+ }
+
+ static Instr ImmLS(int imm9) {
+ VIXL_ASSERT(IsInt9(imm9));
+ return TruncateToUint9(imm9) << ImmLS_offset;
+ }
+
+ static Instr ImmLSPair(int imm7, unsigned access_size) {
+ VIXL_ASSERT(((imm7 >> access_size) << access_size) == imm7);
+ int scaled_imm7 = imm7 >> access_size;
+ VIXL_ASSERT(IsInt7(scaled_imm7));
+ return TruncateToUint7(scaled_imm7) << ImmLSPair_offset;
+ }
+
+ static Instr ImmShiftLS(unsigned shift_amount) {
+ VIXL_ASSERT(IsUint1(shift_amount));
+ return shift_amount << ImmShiftLS_offset;
+ }
+
+ static Instr ImmPrefetchOperation(int imm5) {
+ VIXL_ASSERT(IsUint5(imm5));
+ return imm5 << ImmPrefetchOperation_offset;
+ }
+
+ static Instr ImmException(int imm16) {
+ VIXL_ASSERT(IsUint16(imm16));
+ return imm16 << ImmException_offset;
+ }
+
+ static Instr ImmSystemRegister(int imm15) {
+ VIXL_ASSERT(IsUint15(imm15));
+ return imm15 << ImmSystemRegister_offset;
+ }
+
+ static Instr ImmHint(int imm7) {
+ VIXL_ASSERT(IsUint7(imm7));
+ return imm7 << ImmHint_offset;
+ }
+
+ static Instr CRm(int imm4) {
+ VIXL_ASSERT(IsUint4(imm4));
+ return imm4 << CRm_offset;
+ }
+
+ static Instr CRn(int imm4) {
+ VIXL_ASSERT(IsUint4(imm4));
+ return imm4 << CRn_offset;
+ }
+
+ static Instr SysOp(int imm14) {
+ VIXL_ASSERT(IsUint14(imm14));
+ return imm14 << SysOp_offset;
+ }
+
+ static Instr ImmSysOp1(int imm3) {
+ VIXL_ASSERT(IsUint3(imm3));
+ return imm3 << SysOp1_offset;
+ }
+
+ static Instr ImmSysOp2(int imm3) {
+ VIXL_ASSERT(IsUint3(imm3));
+ return imm3 << SysOp2_offset;
+ }
+
+ static Instr ImmBarrierDomain(int imm2) {
+ VIXL_ASSERT(IsUint2(imm2));
+ return imm2 << ImmBarrierDomain_offset;
+ }
+
+ static Instr ImmBarrierType(int imm2) {
+ VIXL_ASSERT(IsUint2(imm2));
+ return imm2 << ImmBarrierType_offset;
+ }
+
+ // Move immediates encoding.
+ static Instr ImmMoveWide(uint64_t imm) {
+ VIXL_ASSERT(IsUint16(imm));
+ return static_cast<Instr>(imm << ImmMoveWide_offset);
+ }
+
+ static Instr ShiftMoveWide(int64_t shift) {
+ VIXL_ASSERT(IsUint2(shift));
+ return static_cast<Instr>(shift << ShiftMoveWide_offset);
+ }
+
+ // FP Immediates.
+ static Instr ImmFP32(float imm);
+ static Instr ImmFP64(double imm);
+
+ // FP register type.
+ static Instr FPType(FPRegister fd) {
+ return fd.Is64Bits() ? FP64 : FP32;
+ }
+
+ static Instr FPScale(unsigned scale) {
+ VIXL_ASSERT(IsUint6(scale));
+ return scale << FPScale_offset;
+ }
+
+ // Immediate field checking helpers.
+ static bool IsImmAddSub(int64_t immediate);
+ static bool IsImmConditionalCompare(int64_t immediate);
+ static bool IsImmFP32(float imm);
+ static bool IsImmFP64(double imm);
+ static bool IsImmLogical(uint64_t value,
+ unsigned width,
+ unsigned* n = NULL,
+ unsigned* imm_s = NULL,
+ unsigned* imm_r = NULL);
+ static bool IsImmLSPair(int64_t offset, unsigned access_size);
+ static bool IsImmLSScaled(int64_t offset, unsigned access_size);
+ static bool IsImmLSUnscaled(int64_t offset);
+ static bool IsImmMovn(uint64_t imm, unsigned reg_size);
+ static bool IsImmMovz(uint64_t imm, unsigned reg_size);
+
+ // Instruction bits for vector format in data processing operations.
+ static Instr VFormat(VRegister vd) {
+ if (vd.Is64Bits()) {
+ switch (vd.lanes()) {
+ case 2: return NEON_2S;
+ case 4: return NEON_4H;
+ case 8: return NEON_8B;
+ default: return 0xffffffff;
+ }
+ } else {
+ VIXL_ASSERT(vd.Is128Bits());
+ switch (vd.lanes()) {
+ case 2: return NEON_2D;
+ case 4: return NEON_4S;
+ case 8: return NEON_8H;
+ case 16: return NEON_16B;
+ default: return 0xffffffff;
+ }
+ }
+ }
+
+ // Instruction bits for vector format in floating point data processing
+ // operations.
+ static Instr FPFormat(VRegister vd) {
+ if (vd.lanes() == 1) {
+ // Floating point scalar formats.
+ VIXL_ASSERT(vd.Is32Bits() || vd.Is64Bits());
+ return vd.Is64Bits() ? FP64 : FP32;
+ }
+
+ // Two lane floating point vector formats.
+ if (vd.lanes() == 2) {
+ VIXL_ASSERT(vd.Is64Bits() || vd.Is128Bits());
+ return vd.Is128Bits() ? NEON_FP_2D : NEON_FP_2S;
+ }
+
+ // Four lane floating point vector format.
+ VIXL_ASSERT((vd.lanes() == 4) && vd.Is128Bits());
+ return NEON_FP_4S;
+ }
+
+ // Instruction bits for vector format in load and store operations.
+ static Instr LSVFormat(VRegister vd) {
+ if (vd.Is64Bits()) {
+ switch (vd.lanes()) {
+ case 1: return LS_NEON_1D;
+ case 2: return LS_NEON_2S;
+ case 4: return LS_NEON_4H;
+ case 8: return LS_NEON_8B;
+ default: return 0xffffffff;
+ }
+ } else {
+ VIXL_ASSERT(vd.Is128Bits());
+ switch (vd.lanes()) {
+ case 2: return LS_NEON_2D;
+ case 4: return LS_NEON_4S;
+ case 8: return LS_NEON_8H;
+ case 16: return LS_NEON_16B;
+ default: return 0xffffffff;
+ }
+ }
+ }
+
+ // Instruction bits for scalar format in data processing operations.
+ static Instr SFormat(VRegister vd) {
+ VIXL_ASSERT(vd.lanes() == 1);
+ switch (vd.SizeInBytes()) {
+ case 1: return NEON_B;
+ case 2: return NEON_H;
+ case 4: return NEON_S;
+ case 8: return NEON_D;
+ default: return 0xffffffff;
+ }
+ }
+
+ static Instr ImmNEONHLM(int index, int num_bits) {
+ int h, l, m;
+ if (num_bits == 3) {
+ VIXL_ASSERT(IsUint3(index));
+ h = (index >> 2) & 1;
+ l = (index >> 1) & 1;
+ m = (index >> 0) & 1;
+ } else if (num_bits == 2) {
+ VIXL_ASSERT(IsUint2(index));
+ h = (index >> 1) & 1;
+ l = (index >> 0) & 1;
+ m = 0;
+ } else {
+ VIXL_ASSERT(IsUint1(index) && (num_bits == 1));
+ h = (index >> 0) & 1;
+ l = 0;
+ m = 0;
+ }
+ return (h << NEONH_offset) | (l << NEONL_offset) | (m << NEONM_offset);
+ }
+
+ static Instr ImmNEONExt(int imm4) {
+ VIXL_ASSERT(IsUint4(imm4));
+ return imm4 << ImmNEONExt_offset;
+ }
+
+ static Instr ImmNEON5(Instr format, int index) {
+ VIXL_ASSERT(IsUint4(index));
+ int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
+ int imm5 = (index << (s + 1)) | (1 << s);
+ return imm5 << ImmNEON5_offset;
+ }
+
+ static Instr ImmNEON4(Instr format, int index) {
+ VIXL_ASSERT(IsUint4(index));
+ int s = LaneSizeInBytesLog2FromFormat(static_cast<VectorFormat>(format));
+ int imm4 = index << s;
+ return imm4 << ImmNEON4_offset;
+ }
+
+ static Instr ImmNEONabcdefgh(int imm8) {
+ VIXL_ASSERT(IsUint8(imm8));
+ Instr instr;
+ instr = ((imm8 >> 5) & 7) << ImmNEONabc_offset;
+ instr |= (imm8 & 0x1f) << ImmNEONdefgh_offset;
+ return instr;
+ }
+
+ static Instr NEONCmode(int cmode) {
+ VIXL_ASSERT(IsUint4(cmode));
+ return cmode << NEONCmode_offset;
+ }
+
+ static Instr NEONModImmOp(int op) {
+ VIXL_ASSERT(IsUint1(op));
+ return op << NEONModImmOp_offset;
+ }
+
+ size_t size() const {
+ return SizeOfCodeGenerated();
+ }
+
+ size_t SizeOfCodeGenerated() const {
+ return armbuffer_.size();
+ }
+
+ PositionIndependentCodeOption pic() const {
+ return pic_;
+ }
+
+ CPUFeatures* GetCPUFeatures() { return &cpu_features_; }
+
+ void SetCPUFeatures(const CPUFeatures& cpu_features) {
+ cpu_features_ = cpu_features;
+ }
+
+ bool AllowPageOffsetDependentCode() const {
+ return (pic() == PageOffsetDependentCode) ||
+ (pic() == PositionDependentCode);
+ }
+
+ static const Register& AppropriateZeroRegFor(const CPURegister& reg) {
+ return reg.Is64Bits() ? xzr : wzr;
+ }
+
+
+ protected:
+ void LoadStore(const CPURegister& rt,
+ const MemOperand& addr,
+ LoadStoreOp op,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ void LoadStorePair(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& addr,
+ LoadStorePairOp op);
+ void LoadStoreStruct(const VRegister& vt,
+ const MemOperand& addr,
+ NEONLoadStoreMultiStructOp op);
+ void LoadStoreStruct1(const VRegister& vt,
+ int reg_count,
+ const MemOperand& addr);
+ void LoadStoreStructSingle(const VRegister& vt,
+ uint32_t lane,
+ const MemOperand& addr,
+ NEONLoadStoreSingleStructOp op);
+ void LoadStoreStructSingleAllLanes(const VRegister& vt,
+ const MemOperand& addr,
+ NEONLoadStoreSingleStructOp op);
+ void LoadStoreStructVerify(const VRegister& vt,
+ const MemOperand& addr,
+ Instr op);
+
+ void Prefetch(PrefetchOperation op,
+ const MemOperand& addr,
+ LoadStoreScalingOption option = PreferScaledOffset);
+
+ BufferOffset Logical(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ LogicalOp op);
+ BufferOffset LogicalImmediate(const Register& rd,
+ const Register& rn,
+ unsigned n,
+ unsigned imm_s,
+ unsigned imm_r,
+ LogicalOp op);
+
+ void ConditionalCompare(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond,
+ ConditionalCompareOp op);
+
+ void AddSubWithCarry(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ AddSubWithCarryOp op);
+
+
+ // Functions for emulating operands not directly supported by the instruction
+ // set.
+ void EmitShift(const Register& rd,
+ const Register& rn,
+ Shift shift,
+ unsigned amount);
+ void EmitExtendShift(const Register& rd,
+ const Register& rn,
+ Extend extend,
+ unsigned left_shift);
+
+ void AddSub(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ AddSubOp op);
+
+ void NEONTable(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEONTableOp op);
+
+ // Find an appropriate LoadStoreOp or LoadStorePairOp for the specified
+ // registers. Only simple loads are supported; sign- and zero-extension (such
+ // as in LDPSW_x or LDRB_w) are not supported.
+ static LoadStoreOp LoadOpFor(const CPURegister& rt);
+ static LoadStorePairOp LoadPairOpFor(const CPURegister& rt,
+ const CPURegister& rt2);
+ static LoadStoreOp StoreOpFor(const CPURegister& rt);
+ static LoadStorePairOp StorePairOpFor(const CPURegister& rt,
+ const CPURegister& rt2);
+ static LoadStorePairNonTemporalOp LoadPairNonTemporalOpFor(
+ const CPURegister& rt, const CPURegister& rt2);
+ static LoadStorePairNonTemporalOp StorePairNonTemporalOpFor(
+ const CPURegister& rt, const CPURegister& rt2);
+ static LoadLiteralOp LoadLiteralOpFor(const CPURegister& rt);
+
+ // Convenience pass-through for CPU feature checks.
+ bool CPUHas(CPUFeatures::Feature feature0,
+ CPUFeatures::Feature feature1 = CPUFeatures::kNone,
+ CPUFeatures::Feature feature2 = CPUFeatures::kNone,
+ CPUFeatures::Feature feature3 = CPUFeatures::kNone) const {
+ return cpu_features_.Has(feature0, feature1, feature2, feature3);
+ }
+
+ // Determine whether the target CPU has the specified registers, based on the
+ // currently-enabled CPU features. Presence of a register does not imply
+ // support for arbitrary operations on it. For example, CPUs with FP have H
+ // registers, but most half-precision operations require the FPHalf feature.
+ //
+ // These are used to check CPU features in loads and stores that have the same
+ // entry point for both integer and FP registers.
+ bool CPUHas(const CPURegister& rt) const;
+ bool CPUHas(const CPURegister& rt, const CPURegister& rt2) const;
+
+ bool CPUHas(SystemRegister sysreg) const;
+
+ private:
+ static uint32_t FP32ToImm8(float imm);
+ static uint32_t FP64ToImm8(double imm);
+
+ // Instruction helpers.
+ void MoveWide(const Register& rd,
+ uint64_t imm,
+ int shift,
+ MoveWideImmediateOp mov_op);
+ BufferOffset DataProcShiftedRegister(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ Instr op);
+ void DataProcExtendedRegister(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ Instr op);
+ void LoadStorePairNonTemporal(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& addr,
+ LoadStorePairNonTemporalOp op);
+ void LoadLiteral(const CPURegister& rt, uint64_t imm, LoadLiteralOp op);
+ void ConditionalSelect(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond,
+ ConditionalSelectOp op);
+ void DataProcessing1Source(const Register& rd,
+ const Register& rn,
+ DataProcessing1SourceOp op);
+ void DataProcessing3Source(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra,
+ DataProcessing3SourceOp op);
+ void FPDataProcessing1Source(const VRegister& fd,
+ const VRegister& fn,
+ FPDataProcessing1SourceOp op);
+ void FPDataProcessing3Source(const VRegister& fd,
+ const VRegister& fn,
+ const VRegister& fm,
+ const VRegister& fa,
+ FPDataProcessing3SourceOp op);
+ void NEONAcrossLanesL(const VRegister& vd,
+ const VRegister& vn,
+ NEONAcrossLanesOp op);
+ void NEONAcrossLanes(const VRegister& vd,
+ const VRegister& vn,
+ NEONAcrossLanesOp op);
+ void NEONModifiedImmShiftLsl(const VRegister& vd,
+ const int imm8,
+ const int left_shift,
+ NEONModifiedImmediateOp op);
+ void NEONModifiedImmShiftMsl(const VRegister& vd,
+ const int imm8,
+ const int shift_amount,
+ NEONModifiedImmediateOp op);
+ void NEONFP2Same(const VRegister& vd,
+ const VRegister& vn,
+ Instr vop);
+ void NEON3Same(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEON3SameOp vop);
+ void NEONFP3Same(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ Instr op);
+ void NEON3DifferentL(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEON3DifferentOp vop);
+ void NEON3DifferentW(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEON3DifferentOp vop);
+ void NEON3DifferentHN(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEON3DifferentOp vop);
+ void NEONFP2RegMisc(const VRegister& vd,
+ const VRegister& vn,
+ NEON2RegMiscOp vop,
+ double value = 0.0);
+ void NEON2RegMisc(const VRegister& vd,
+ const VRegister& vn,
+ NEON2RegMiscOp vop,
+ int value = 0);
+ void NEONFP2RegMisc(const VRegister& vd,
+ const VRegister& vn,
+ Instr op);
+ void NEONAddlp(const VRegister& vd,
+ const VRegister& vn,
+ NEON2RegMiscOp op);
+ void NEONPerm(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ NEONPermOp op);
+ void NEONFPByElement(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index,
+ NEONByIndexedElementOp op);
+ void NEONByElement(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index,
+ NEONByIndexedElementOp op);
+ void NEONByElementL(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int vm_index,
+ NEONByIndexedElementOp op);
+ void NEONShiftImmediate(const VRegister& vd,
+ const VRegister& vn,
+ NEONShiftImmediateOp op,
+ int immh_immb);
+ void NEONShiftLeftImmediate(const VRegister& vd,
+ const VRegister& vn,
+ int shift,
+ NEONShiftImmediateOp op);
+ void NEONShiftRightImmediate(const VRegister& vd,
+ const VRegister& vn,
+ int shift,
+ NEONShiftImmediateOp op);
+ void NEONShiftImmediateL(const VRegister& vd,
+ const VRegister& vn,
+ int shift,
+ NEONShiftImmediateOp op);
+ void NEONShiftImmediateN(const VRegister& vd,
+ const VRegister& vn,
+ int shift,
+ NEONShiftImmediateOp op);
+ void NEONXtn(const VRegister& vd,
+ const VRegister& vn,
+ NEON2RegMiscOp vop);
+
+ Instr LoadStoreStructAddrModeField(const MemOperand& addr);
+
+ // Encode the specified MemOperand for the specified access size and scaling
+ // preference.
+ Instr LoadStoreMemOperand(const MemOperand& addr,
+ unsigned access_size,
+ LoadStoreScalingOption option);
+
+ protected:
+ // Prevent generation of a literal pool for the next |maxInst| instructions.
+ // Guarantees instruction linearity.
+ class AutoBlockLiteralPool {
+ ARMBuffer* armbuffer_;
+
+ public:
+ AutoBlockLiteralPool(Assembler* assembler, size_t maxInst)
+ : armbuffer_(&assembler->armbuffer_) {
+ armbuffer_->enterNoPool(maxInst);
+ }
+ ~AutoBlockLiteralPool() {
+ armbuffer_->leaveNoPool();
+ }
+ };
+
+ protected:
+ // Buffer where the code is emitted.
+ PositionIndependentCodeOption pic_;
+
+ CPUFeatures cpu_features_;
+
+#ifdef DEBUG
+ bool finalized_;
+#endif
+};
+
+} // namespace vixl
+
+#endif // VIXL_A64_ASSEMBLER_A64_H_
diff --git a/js/src/jit/arm64/vixl/CompilerIntrinsics-vixl.h b/js/src/jit/arm64/vixl/CompilerIntrinsics-vixl.h
new file mode 100644
index 0000000000..e13eef6135
--- /dev/null
+++ b/js/src/jit/arm64/vixl/CompilerIntrinsics-vixl.h
@@ -0,0 +1,179 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+#ifndef VIXL_COMPILER_INTRINSICS_H
+#define VIXL_COMPILER_INTRINSICS_H
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/arm64/vixl/Globals-vixl.h"
+
+namespace vixl {
+
+// Helper to check whether the version of GCC used is greater than the specified
+// requirement.
+#define MAJOR 1000000
+#define MINOR 1000
+#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__)
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel) \
+ ((__GNUC__ * MAJOR + __GNUC_MINOR__ * MINOR + __GNUC_PATCHLEVEL__) >= \
+ ((major) * MAJOR + (minor) * MINOR + (patchlevel)))
+#elif defined(__GNUC__) && defined(__GNUC_MINOR__)
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel) \
+ ((__GNUC__ * MAJOR + __GNUC_MINOR__ * MINOR) >= \
+ ((major) * MAJOR + (minor) * MINOR + (patchlevel)))
+#else
+#define GCC_VERSION_OR_NEWER(major, minor, patchlevel) 0
+#endif
+
+
+#if defined(__clang__) && !defined(VIXL_NO_COMPILER_BUILTINS)
+
+#define COMPILER_HAS_BUILTIN_CLRSB (__has_builtin(__builtin_clrsb))
+#define COMPILER_HAS_BUILTIN_CLZ (__has_builtin(__builtin_clz))
+#define COMPILER_HAS_BUILTIN_CTZ (__has_builtin(__builtin_ctz))
+#define COMPILER_HAS_BUILTIN_FFS (__has_builtin(__builtin_ffs))
+#define COMPILER_HAS_BUILTIN_POPCOUNT (__has_builtin(__builtin_popcount))
+
+#elif defined(__GNUC__) && !defined(VIXL_NO_COMPILER_BUILTINS)
+// The documentation for these builtins is available at:
+// https://gcc.gnu.org/onlinedocs/gcc-$MAJOR.$MINOR.$PATCHLEVEL/gcc//Other-Builtins.html
+
+# define COMPILER_HAS_BUILTIN_CLRSB (GCC_VERSION_OR_NEWER(4, 7, 0))
+# define COMPILER_HAS_BUILTIN_CLZ (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_CTZ (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_FFS (GCC_VERSION_OR_NEWER(3, 4, 0))
+# define COMPILER_HAS_BUILTIN_POPCOUNT (GCC_VERSION_OR_NEWER(3, 4, 0))
+
+#else
+// One can define VIXL_NO_COMPILER_BUILTINS to force using the manually
+// implemented C++ methods.
+
+#define COMPILER_HAS_BUILTIN_BSWAP false
+#define COMPILER_HAS_BUILTIN_CLRSB false
+#define COMPILER_HAS_BUILTIN_CLZ false
+#define COMPILER_HAS_BUILTIN_CTZ false
+#define COMPILER_HAS_BUILTIN_FFS false
+#define COMPILER_HAS_BUILTIN_POPCOUNT false
+
+#endif
+
+
+template<typename V>
+inline bool IsPowerOf2(V value) {
+ return (value != 0) && ((value & (value - 1)) == 0);
+}
+
+
+// Implementation of intrinsics functions.
+// TODO: The implementations could be improved for sizes different from 32bit
+// and 64bit: we could mask the values and call the appropriate builtin.
+
+
+template<typename V>
+inline int CountLeadingZeros(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CLZ
+ if (width == 32) {
+ return (value == 0) ? 32 : __builtin_clz(static_cast<unsigned>(value));
+ } else if (width == 64) {
+ return (value == 0) ? 64 : __builtin_clzll(value);
+ }
+ MOZ_CRASH("Unhandled width.");
+#else
+ if (width == 32) {
+ return mozilla::CountLeadingZeroes32(value);
+ } else if (width == 64) {
+ return mozilla::CountLeadingZeroes64(value);
+ }
+ MOZ_CRASH("Unhandled width.");
+#endif
+}
+
+
+template<typename V>
+inline int CountLeadingSignBits(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CLRSB
+ if (width == 32) {
+ return __builtin_clrsb(value);
+ } else if (width == 64) {
+ return __builtin_clrsbll(value);
+ }
+ MOZ_CRASH("Unhandled width.");
+#else
+ VIXL_ASSERT(IsPowerOf2(width) && (width <= 64));
+ if (value >= 0) {
+ return CountLeadingZeros(value, width) - 1;
+ } else {
+ return CountLeadingZeros(~value, width) - 1;
+ }
+#endif
+}
+
+
+template<typename V>
+inline int CountSetBits(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_POPCOUNT
+ if (width == 32) {
+ return __builtin_popcount(static_cast<unsigned>(value));
+ } else if (width == 64) {
+ return __builtin_popcountll(value);
+ }
+ MOZ_CRASH("Unhandled width.");
+#else
+ if (width == 32) {
+ return mozilla::CountPopulation32(value);
+ } else if (width == 64) {
+ return mozilla::CountPopulation64(value);
+ }
+ MOZ_CRASH("Unhandled width.");
+#endif
+}
+
+
+template<typename V>
+inline int CountTrailingZeros(V value, int width = (sizeof(V) * 8)) {
+#if COMPILER_HAS_BUILTIN_CTZ
+ if (width == 32) {
+ return (value == 0) ? 32 : __builtin_ctz(static_cast<unsigned>(value));
+ } else if (width == 64) {
+ return (value == 0) ? 64 : __builtin_ctzll(value);
+ }
+ MOZ_CRASH("Unhandled width.");
+#else
+ if (width == 32) {
+ return mozilla::CountTrailingZeroes32(value);
+ } else if (width == 64) {
+ return mozilla::CountTrailingZeroes64(value);
+ }
+ MOZ_CRASH("Unhandled width.");
+#endif
+}
+
+} // namespace vixl
+
+#endif // VIXL_COMPILER_INTRINSICS_H
+
diff --git a/js/src/jit/arm64/vixl/Constants-vixl.h b/js/src/jit/arm64/vixl/Constants-vixl.h
new file mode 100644
index 0000000000..2c174e61a5
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Constants-vixl.h
@@ -0,0 +1,2694 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_CONSTANTS_A64_H_
+#define VIXL_A64_CONSTANTS_A64_H_
+
+#include <stdint.h>
+
+#include "jit/arm64/vixl/Globals-vixl.h"
+
+namespace vixl {
+
+// Supervisor Call (svc) specific support.
+//
+// The SVC instruction encodes an optional 16-bit immediate value.
+// The simulator understands the codes below.
+enum SVCSimulatorCodes {
+ kCallRtRedirected = 0x10, // Transition to x86_64 C code.
+ kMarkStackPointer = 0x11, // Push the current SP on a special Simulator stack.
+ kCheckStackPointer = 0x12 // Pop from the special Simulator stack and compare to SP.
+};
+
+const unsigned kNumberOfRegisters = 32;
+const unsigned kNumberOfVRegisters = 32;
+const unsigned kNumberOfFPRegisters = kNumberOfVRegisters;
+// Callee saved registers are x21-x30(lr).
+const int kNumberOfCalleeSavedRegisters = 10;
+const int kFirstCalleeSavedRegisterIndex = 21;
+// Callee saved FP registers are d8-d15. Note that the high parts of v8-v15 are
+// still caller-saved.
+const int kNumberOfCalleeSavedFPRegisters = 8;
+const int kFirstCalleeSavedFPRegisterIndex = 8;
+
+#define REGISTER_CODE_LIST(R) \
+R(0) R(1) R(2) R(3) R(4) R(5) R(6) R(7) \
+R(8) R(9) R(10) R(11) R(12) R(13) R(14) R(15) \
+R(16) R(17) R(18) R(19) R(20) R(21) R(22) R(23) \
+R(24) R(25) R(26) R(27) R(28) R(29) R(30) R(31)
+
+#define INSTRUCTION_FIELDS_LIST(V_) \
+/* Register fields */ \
+V_(Rd, 4, 0, Bits) /* Destination register. */ \
+V_(Rn, 9, 5, Bits) /* First source register. */ \
+V_(Rm, 20, 16, Bits) /* Second source register. */ \
+V_(Ra, 14, 10, Bits) /* Third source register. */ \
+V_(Rt, 4, 0, Bits) /* Load/store register. */ \
+V_(Rt2, 14, 10, Bits) /* Load/store second register. */ \
+V_(Rs, 20, 16, Bits) /* Exclusive access status. */ \
+ \
+/* Common bits */ \
+V_(SixtyFourBits, 31, 31, Bits) \
+V_(FlagsUpdate, 29, 29, Bits) \
+ \
+/* PC relative addressing */ \
+V_(ImmPCRelHi, 23, 5, SignedBits) \
+V_(ImmPCRelLo, 30, 29, Bits) \
+ \
+/* Add/subtract/logical shift register */ \
+V_(ShiftDP, 23, 22, Bits) \
+V_(ImmDPShift, 15, 10, Bits) \
+ \
+/* Add/subtract immediate */ \
+V_(ImmAddSub, 21, 10, Bits) \
+V_(ShiftAddSub, 23, 22, Bits) \
+ \
+/* Add/substract extend */ \
+V_(ImmExtendShift, 12, 10, Bits) \
+V_(ExtendMode, 15, 13, Bits) \
+ \
+/* Move wide */ \
+V_(ImmMoveWide, 20, 5, Bits) \
+V_(ShiftMoveWide, 22, 21, Bits) \
+ \
+/* Logical immediate, bitfield and extract */ \
+V_(BitN, 22, 22, Bits) \
+V_(ImmRotate, 21, 16, Bits) \
+V_(ImmSetBits, 15, 10, Bits) \
+V_(ImmR, 21, 16, Bits) \
+V_(ImmS, 15, 10, Bits) \
+ \
+/* Test and branch immediate */ \
+V_(ImmTestBranch, 18, 5, SignedBits) \
+V_(ImmTestBranchBit40, 23, 19, Bits) \
+V_(ImmTestBranchBit5, 31, 31, Bits) \
+ \
+/* Conditionals */ \
+V_(Condition, 15, 12, Bits) \
+V_(ConditionBranch, 3, 0, Bits) \
+V_(Nzcv, 3, 0, Bits) \
+V_(ImmCondCmp, 20, 16, Bits) \
+V_(ImmCondBranch, 23, 5, SignedBits) \
+ \
+/* Floating point */ \
+V_(FPType, 23, 22, Bits) \
+V_(ImmFP, 20, 13, Bits) \
+V_(FPScale, 15, 10, Bits) \
+ \
+/* Load Store */ \
+V_(ImmLS, 20, 12, SignedBits) \
+V_(ImmLSUnsigned, 21, 10, Bits) \
+V_(ImmLSPair, 21, 15, SignedBits) \
+V_(ImmShiftLS, 12, 12, Bits) \
+V_(LSOpc, 23, 22, Bits) \
+V_(LSVector, 26, 26, Bits) \
+V_(LSSize, 31, 30, Bits) \
+V_(ImmPrefetchOperation, 4, 0, Bits) \
+V_(PrefetchHint, 4, 3, Bits) \
+V_(PrefetchTarget, 2, 1, Bits) \
+V_(PrefetchStream, 0, 0, Bits) \
+ \
+/* Other immediates */ \
+V_(ImmUncondBranch, 25, 0, SignedBits) \
+V_(ImmCmpBranch, 23, 5, SignedBits) \
+V_(ImmLLiteral, 23, 5, SignedBits) \
+V_(ImmException, 20, 5, Bits) \
+V_(ImmHint, 11, 5, Bits) \
+V_(ImmBarrierDomain, 11, 10, Bits) \
+V_(ImmBarrierType, 9, 8, Bits) \
+ \
+/* System (MRS, MSR, SYS) */ \
+V_(ImmSystemRegister, 19, 5, Bits) \
+V_(SysO0, 19, 19, Bits) \
+V_(SysOp, 18, 5, Bits) \
+V_(SysOp1, 18, 16, Bits) \
+V_(SysOp2, 7, 5, Bits) \
+V_(CRn, 15, 12, Bits) \
+V_(CRm, 11, 8, Bits) \
+ \
+/* Load-/store-exclusive */ \
+V_(LdStXLoad, 22, 22, Bits) \
+V_(LdStXNotExclusive, 23, 23, Bits) \
+V_(LdStXAcquireRelease, 15, 15, Bits) \
+V_(LdStXSizeLog2, 31, 30, Bits) \
+V_(LdStXPair, 21, 21, Bits) \
+ \
+/* NEON generic fields */ \
+V_(NEONQ, 30, 30, Bits) \
+V_(NEONSize, 23, 22, Bits) \
+V_(NEONLSSize, 11, 10, Bits) \
+V_(NEONS, 12, 12, Bits) \
+V_(NEONL, 21, 21, Bits) \
+V_(NEONM, 20, 20, Bits) \
+V_(NEONH, 11, 11, Bits) \
+V_(ImmNEONExt, 14, 11, Bits) \
+V_(ImmNEON5, 20, 16, Bits) \
+V_(ImmNEON4, 14, 11, Bits) \
+ \
+/* NEON Modified Immediate fields */ \
+V_(ImmNEONabc, 18, 16, Bits) \
+V_(ImmNEONdefgh, 9, 5, Bits) \
+V_(NEONModImmOp, 29, 29, Bits) \
+V_(NEONCmode, 15, 12, Bits) \
+ \
+/* NEON Shift Immediate fields */ \
+V_(ImmNEONImmhImmb, 22, 16, Bits) \
+V_(ImmNEONImmh, 22, 19, Bits) \
+V_(ImmNEONImmb, 18, 16, Bits)
+
+#define SYSTEM_REGISTER_FIELDS_LIST(V_, M_) \
+/* NZCV */ \
+V_(Flags, 31, 28, Bits) \
+V_(N, 31, 31, Bits) \
+V_(Z, 30, 30, Bits) \
+V_(C, 29, 29, Bits) \
+V_(V, 28, 28, Bits) \
+M_(NZCV, Flags_mask) \
+/* FPCR */ \
+V_(AHP, 26, 26, Bits) \
+V_(DN, 25, 25, Bits) \
+V_(FZ, 24, 24, Bits) \
+V_(RMode, 23, 22, Bits) \
+M_(FPCR, AHP_mask | DN_mask | FZ_mask | RMode_mask)
+
+// Fields offsets.
+#define DECLARE_FIELDS_OFFSETS(Name, HighBit, LowBit, X) \
+const int Name##_offset = LowBit; \
+const int Name##_width = HighBit - LowBit + 1; \
+const uint32_t Name##_mask = ((1 << Name##_width) - 1) << LowBit;
+#define NOTHING(A, B)
+INSTRUCTION_FIELDS_LIST(DECLARE_FIELDS_OFFSETS)
+SYSTEM_REGISTER_FIELDS_LIST(DECLARE_FIELDS_OFFSETS, NOTHING)
+#undef NOTHING
+#undef DECLARE_FIELDS_BITS
+
+// ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST), formed
+// from ImmPCRelLo and ImmPCRelHi.
+const int ImmPCRel_mask = ImmPCRelLo_mask | ImmPCRelHi_mask;
+
+// Condition codes.
+enum Condition {
+ eq = 0, // Z set Equal.
+ ne = 1, // Z clear Not equal.
+ cs = 2, // C set Carry set.
+ cc = 3, // C clear Carry clear.
+ mi = 4, // N set Negative.
+ pl = 5, // N clear Positive or zero.
+ vs = 6, // V set Overflow.
+ vc = 7, // V clear No overflow.
+ hi = 8, // C set, Z clear Unsigned higher.
+ ls = 9, // C clear or Z set Unsigned lower or same.
+ ge = 10, // N == V Greater or equal.
+ lt = 11, // N != V Less than.
+ gt = 12, // Z clear, N == V Greater than.
+ le = 13, // Z set or N != V Less then or equal
+ al = 14, // Always.
+ nv = 15, // Behaves as always/al.
+
+ // Aliases.
+ hs = cs, // C set Unsigned higher or same.
+ lo = cc, // C clear Unsigned lower.
+
+ // Mozilla expanded aliases.
+ Equal = 0, Zero = 0,
+ NotEqual = 1, NonZero = 1,
+ AboveOrEqual = 2, CarrySet = 2,
+ Below = 3, CarryClear = 3,
+ Signed = 4,
+ NotSigned = 5,
+ Overflow = 6,
+ NoOverflow = 7,
+ Above = 8,
+ BelowOrEqual = 9,
+ GreaterThanOrEqual_ = 10,
+ LessThan_ = 11,
+ GreaterThan_ = 12,
+ LessThanOrEqual_ = 13,
+ Always = 14,
+ Never = 15
+};
+
+inline Condition InvertCondition(Condition cond) {
+ // Conditions al and nv behave identically, as "always true". They can't be
+ // inverted, because there is no "always false" condition.
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ return static_cast<Condition>(cond ^ 1);
+}
+
+enum FPTrapFlags {
+ EnableTrap = 1,
+ DisableTrap = 0
+};
+
+enum FlagsUpdate {
+ SetFlags = 1,
+ LeaveFlags = 0
+};
+
+enum StatusFlags {
+ NoFlag = 0,
+
+ // Derive the flag combinations from the system register bit descriptions.
+ NFlag = N_mask,
+ ZFlag = Z_mask,
+ CFlag = C_mask,
+ VFlag = V_mask,
+ NZFlag = NFlag | ZFlag,
+ NCFlag = NFlag | CFlag,
+ NVFlag = NFlag | VFlag,
+ ZCFlag = ZFlag | CFlag,
+ ZVFlag = ZFlag | VFlag,
+ CVFlag = CFlag | VFlag,
+ NZCFlag = NFlag | ZFlag | CFlag,
+ NZVFlag = NFlag | ZFlag | VFlag,
+ NCVFlag = NFlag | CFlag | VFlag,
+ ZCVFlag = ZFlag | CFlag | VFlag,
+ NZCVFlag = NFlag | ZFlag | CFlag | VFlag,
+
+ // Floating-point comparison results.
+ FPEqualFlag = ZCFlag,
+ FPLessThanFlag = NFlag,
+ FPGreaterThanFlag = CFlag,
+ FPUnorderedFlag = CVFlag
+};
+
+enum Shift {
+ NO_SHIFT = -1,
+ LSL = 0x0,
+ LSR = 0x1,
+ ASR = 0x2,
+ ROR = 0x3,
+ MSL = 0x4
+};
+
+enum Extend {
+ NO_EXTEND = -1,
+ UXTB = 0,
+ UXTH = 1,
+ UXTW = 2,
+ UXTX = 3,
+ SXTB = 4,
+ SXTH = 5,
+ SXTW = 6,
+ SXTX = 7
+};
+
+enum SystemHint {
+ NOP = 0,
+ YIELD = 1,
+ WFE = 2,
+ WFI = 3,
+ SEV = 4,
+ SEVL = 5,
+ ESB = 16,
+ CSDB = 20,
+ BTI = 32,
+ BTI_c = 34,
+ BTI_j = 36,
+ BTI_jc = 38
+};
+
+enum BranchTargetIdentifier {
+ EmitBTI_none = NOP,
+ EmitBTI = BTI,
+ EmitBTI_c = BTI_c,
+ EmitBTI_j = BTI_j,
+ EmitBTI_jc = BTI_jc,
+
+ // These correspond to the values of the CRm:op2 fields in the equivalent HINT
+ // instruction.
+ EmitPACIASP = 25,
+ EmitPACIBSP = 27
+};
+
+enum BarrierDomain {
+ OuterShareable = 0,
+ NonShareable = 1,
+ InnerShareable = 2,
+ FullSystem = 3
+};
+
+enum BarrierType {
+ BarrierOther = 0,
+ BarrierReads = 1,
+ BarrierWrites = 2,
+ BarrierAll = 3
+};
+
+enum PrefetchOperation {
+ PLDL1KEEP = 0x00,
+ PLDL1STRM = 0x01,
+ PLDL2KEEP = 0x02,
+ PLDL2STRM = 0x03,
+ PLDL3KEEP = 0x04,
+ PLDL3STRM = 0x05,
+
+ PLIL1KEEP = 0x08,
+ PLIL1STRM = 0x09,
+ PLIL2KEEP = 0x0a,
+ PLIL2STRM = 0x0b,
+ PLIL3KEEP = 0x0c,
+ PLIL3STRM = 0x0d,
+
+ PSTL1KEEP = 0x10,
+ PSTL1STRM = 0x11,
+ PSTL2KEEP = 0x12,
+ PSTL2STRM = 0x13,
+ PSTL3KEEP = 0x14,
+ PSTL3STRM = 0x15
+};
+
+enum BType {
+ // Set when executing any instruction on a guarded page, except those cases
+ // listed below.
+ DefaultBType = 0,
+
+ // Set when an indirect branch is taken from an unguarded page to a guarded
+ // page, or from a guarded page to ip0 or ip1 (x16 or x17), eg "br ip0".
+ BranchFromUnguardedOrToIP = 1,
+
+ // Set when an indirect branch and link (call) is taken, eg. "blr x0".
+ BranchAndLink = 2,
+
+ // Set when an indirect branch is taken from a guarded page to a register
+ // that is not ip0 or ip1 (x16 or x17), eg, "br x0".
+ BranchFromGuardedNotToIP = 3
+};
+
+template<int op0, int op1, int crn, int crm, int op2>
+class SystemRegisterEncoder {
+ public:
+ static const uint32_t value =
+ ((op0 << SysO0_offset) |
+ (op1 << SysOp1_offset) |
+ (crn << CRn_offset) |
+ (crm << CRm_offset) |
+ (op2 << SysOp2_offset)) >> ImmSystemRegister_offset;
+};
+
+// System/special register names.
+// This information is not encoded as one field but as the concatenation of
+// multiple fields (Op0, Op1, Crn, Crm, Op2).
+enum SystemRegister {
+ NZCV = SystemRegisterEncoder<3, 3, 4, 2, 0>::value,
+ FPCR = SystemRegisterEncoder<3, 3, 4, 4, 0>::value,
+ RNDR = SystemRegisterEncoder<3, 3, 2, 4, 0>::value, // Random number.
+ RNDRRS = SystemRegisterEncoder<3, 3, 2, 4, 1>::value // Reseeded random number.
+};
+
+template<int op1, int crn, int crm, int op2>
+class CacheOpEncoder {
+ public:
+ static const uint32_t value =
+ ((op1 << SysOp1_offset) |
+ (crn << CRn_offset) |
+ (crm << CRm_offset) |
+ (op2 << SysOp2_offset)) >> SysOp_offset;
+};
+
+enum InstructionCacheOp : uint32_t {
+ IVAU = CacheOpEncoder<3, 7, 5, 1>::value
+};
+
+enum DataCacheOp : uint32_t {
+ CVAC = CacheOpEncoder<3, 7, 10, 1>::value,
+ CVAU = CacheOpEncoder<3, 7, 11, 1>::value,
+ CVAP = CacheOpEncoder<3, 7, 12, 1>::value,
+ CVADP = CacheOpEncoder<3, 7, 13, 1>::value,
+ CIVAC = CacheOpEncoder<3, 7, 14, 1>::value,
+ ZVA = CacheOpEncoder<3, 7, 4, 1>::value
+};
+
+// Instruction enumerations.
+//
+// These are the masks that define a class of instructions, and the list of
+// instructions within each class. Each enumeration has a Fixed, FMask and
+// Mask value.
+//
+// Fixed: The fixed bits in this instruction class.
+// FMask: The mask used to extract the fixed bits in the class.
+// Mask: The mask used to identify the instructions within a class.
+//
+// The enumerations can be used like this:
+//
+// VIXL_ASSERT(instr->Mask(PCRelAddressingFMask) == PCRelAddressingFixed);
+// switch(instr->Mask(PCRelAddressingMask)) {
+// case ADR: Format("adr 'Xd, 'AddrPCRelByte"); break;
+// case ADRP: Format("adrp 'Xd, 'AddrPCRelPage"); break;
+// default: printf("Unknown instruction\n");
+// }
+
+
+// Generic fields.
+enum GenericInstrField : uint32_t {
+ SixtyFourBits = 0x80000000,
+ ThirtyTwoBits = 0x00000000,
+
+ FPTypeMask = 0x00C00000,
+ FP16 = 0x00C00000,
+ FP32 = 0x00000000,
+ FP64 = 0x00400000
+};
+
+enum NEONFormatField : uint32_t {
+ NEONFormatFieldMask = 0x40C00000,
+ NEON_Q = 0x40000000,
+ NEON_8B = 0x00000000,
+ NEON_16B = NEON_8B | NEON_Q,
+ NEON_4H = 0x00400000,
+ NEON_8H = NEON_4H | NEON_Q,
+ NEON_2S = 0x00800000,
+ NEON_4S = NEON_2S | NEON_Q,
+ NEON_1D = 0x00C00000,
+ NEON_2D = 0x00C00000 | NEON_Q
+};
+
+enum NEONFPFormatField : uint32_t {
+ NEONFPFormatFieldMask = 0x40400000,
+ NEON_FP_4H = FP16,
+ NEON_FP_2S = FP32,
+ NEON_FP_8H = FP16 | NEON_Q,
+ NEON_FP_4S = FP32 | NEON_Q,
+ NEON_FP_2D = FP64 | NEON_Q
+};
+
+enum NEONLSFormatField : uint32_t {
+ NEONLSFormatFieldMask = 0x40000C00,
+ LS_NEON_8B = 0x00000000,
+ LS_NEON_16B = LS_NEON_8B | NEON_Q,
+ LS_NEON_4H = 0x00000400,
+ LS_NEON_8H = LS_NEON_4H | NEON_Q,
+ LS_NEON_2S = 0x00000800,
+ LS_NEON_4S = LS_NEON_2S | NEON_Q,
+ LS_NEON_1D = 0x00000C00,
+ LS_NEON_2D = LS_NEON_1D | NEON_Q
+};
+
+enum NEONScalarFormatField : uint32_t {
+ NEONScalarFormatFieldMask = 0x00C00000,
+ NEONScalar = 0x10000000,
+ NEON_B = 0x00000000,
+ NEON_H = 0x00400000,
+ NEON_S = 0x00800000,
+ NEON_D = 0x00C00000
+};
+
+// PC relative addressing.
+enum PCRelAddressingOp : uint32_t {
+ PCRelAddressingFixed = 0x10000000,
+ PCRelAddressingFMask = 0x1F000000,
+ PCRelAddressingMask = 0x9F000000,
+ ADR = PCRelAddressingFixed | 0x00000000,
+ ADRP = PCRelAddressingFixed | 0x80000000
+};
+
+// Add/sub (immediate, shifted and extended.)
+const int kSFOffset = 31;
+enum AddSubOp : uint32_t {
+ AddSubOpMask = 0x60000000,
+ AddSubSetFlagsBit = 0x20000000,
+ ADD = 0x00000000,
+ ADDS = ADD | AddSubSetFlagsBit,
+ SUB = 0x40000000,
+ SUBS = SUB | AddSubSetFlagsBit
+};
+
+#define ADD_SUB_OP_LIST(V) \
+ V(ADD), \
+ V(ADDS), \
+ V(SUB), \
+ V(SUBS)
+
+enum AddSubImmediateOp : uint32_t {
+ AddSubImmediateFixed = 0x11000000,
+ AddSubImmediateFMask = 0x1F000000,
+ AddSubImmediateMask = 0xFF000000,
+ #define ADD_SUB_IMMEDIATE(A) \
+ A##_w_imm = AddSubImmediateFixed | A, \
+ A##_x_imm = AddSubImmediateFixed | A | SixtyFourBits
+ ADD_SUB_OP_LIST(ADD_SUB_IMMEDIATE)
+ #undef ADD_SUB_IMMEDIATE
+};
+
+enum AddSubShiftedOp : uint32_t {
+ AddSubShiftedFixed = 0x0B000000,
+ AddSubShiftedFMask = 0x1F200000,
+ AddSubShiftedMask = 0xFF200000,
+ #define ADD_SUB_SHIFTED(A) \
+ A##_w_shift = AddSubShiftedFixed | A, \
+ A##_x_shift = AddSubShiftedFixed | A | SixtyFourBits
+ ADD_SUB_OP_LIST(ADD_SUB_SHIFTED)
+ #undef ADD_SUB_SHIFTED
+};
+
+enum AddSubExtendedOp : uint32_t {
+ AddSubExtendedFixed = 0x0B200000,
+ AddSubExtendedFMask = 0x1F200000,
+ AddSubExtendedMask = 0xFFE00000,
+ #define ADD_SUB_EXTENDED(A) \
+ A##_w_ext = AddSubExtendedFixed | A, \
+ A##_x_ext = AddSubExtendedFixed | A | SixtyFourBits
+ ADD_SUB_OP_LIST(ADD_SUB_EXTENDED)
+ #undef ADD_SUB_EXTENDED
+};
+
+// Add/sub with carry.
+enum AddSubWithCarryOp : uint32_t {
+ AddSubWithCarryFixed = 0x1A000000,
+ AddSubWithCarryFMask = 0x1FE00000,
+ AddSubWithCarryMask = 0xFFE0FC00,
+ ADC_w = AddSubWithCarryFixed | ADD,
+ ADC_x = AddSubWithCarryFixed | ADD | SixtyFourBits,
+ ADC = ADC_w,
+ ADCS_w = AddSubWithCarryFixed | ADDS,
+ ADCS_x = AddSubWithCarryFixed | ADDS | SixtyFourBits,
+ SBC_w = AddSubWithCarryFixed | SUB,
+ SBC_x = AddSubWithCarryFixed | SUB | SixtyFourBits,
+ SBC = SBC_w,
+ SBCS_w = AddSubWithCarryFixed | SUBS,
+ SBCS_x = AddSubWithCarryFixed | SUBS | SixtyFourBits
+};
+
+// Rotate right into flags.
+enum RotateRightIntoFlagsOp : uint32_t {
+ RotateRightIntoFlagsFixed = 0x1A000400,
+ RotateRightIntoFlagsFMask = 0x1FE07C00,
+ RotateRightIntoFlagsMask = 0xFFE07C10,
+ RMIF = RotateRightIntoFlagsFixed | 0xA0000000
+};
+
+// Evaluate into flags.
+enum EvaluateIntoFlagsOp : uint32_t {
+ EvaluateIntoFlagsFixed = 0x1A000800,
+ EvaluateIntoFlagsFMask = 0x1FE03C00,
+ EvaluateIntoFlagsMask = 0xFFE07C1F,
+ SETF8 = EvaluateIntoFlagsFixed | 0x2000000D,
+ SETF16 = EvaluateIntoFlagsFixed | 0x2000400D
+};
+
+// Logical (immediate and shifted register).
+enum LogicalOp : uint32_t {
+ LogicalOpMask = 0x60200000,
+ NOT = 0x00200000,
+ AND = 0x00000000,
+ BIC = AND | NOT,
+ ORR = 0x20000000,
+ ORN = ORR | NOT,
+ EOR = 0x40000000,
+ EON = EOR | NOT,
+ ANDS = 0x60000000,
+ BICS = ANDS | NOT
+};
+
+// Logical immediate.
+enum LogicalImmediateOp : uint32_t {
+ LogicalImmediateFixed = 0x12000000,
+ LogicalImmediateFMask = 0x1F800000,
+ LogicalImmediateMask = 0xFF800000,
+ AND_w_imm = LogicalImmediateFixed | AND,
+ AND_x_imm = LogicalImmediateFixed | AND | SixtyFourBits,
+ ORR_w_imm = LogicalImmediateFixed | ORR,
+ ORR_x_imm = LogicalImmediateFixed | ORR | SixtyFourBits,
+ EOR_w_imm = LogicalImmediateFixed | EOR,
+ EOR_x_imm = LogicalImmediateFixed | EOR | SixtyFourBits,
+ ANDS_w_imm = LogicalImmediateFixed | ANDS,
+ ANDS_x_imm = LogicalImmediateFixed | ANDS | SixtyFourBits
+};
+
+// Logical shifted register.
+enum LogicalShiftedOp : uint32_t {
+ LogicalShiftedFixed = 0x0A000000,
+ LogicalShiftedFMask = 0x1F000000,
+ LogicalShiftedMask = 0xFF200000,
+ AND_w = LogicalShiftedFixed | AND,
+ AND_x = LogicalShiftedFixed | AND | SixtyFourBits,
+ AND_shift = AND_w,
+ BIC_w = LogicalShiftedFixed | BIC,
+ BIC_x = LogicalShiftedFixed | BIC | SixtyFourBits,
+ BIC_shift = BIC_w,
+ ORR_w = LogicalShiftedFixed | ORR,
+ ORR_x = LogicalShiftedFixed | ORR | SixtyFourBits,
+ ORR_shift = ORR_w,
+ ORN_w = LogicalShiftedFixed | ORN,
+ ORN_x = LogicalShiftedFixed | ORN | SixtyFourBits,
+ ORN_shift = ORN_w,
+ EOR_w = LogicalShiftedFixed | EOR,
+ EOR_x = LogicalShiftedFixed | EOR | SixtyFourBits,
+ EOR_shift = EOR_w,
+ EON_w = LogicalShiftedFixed | EON,
+ EON_x = LogicalShiftedFixed | EON | SixtyFourBits,
+ EON_shift = EON_w,
+ ANDS_w = LogicalShiftedFixed | ANDS,
+ ANDS_x = LogicalShiftedFixed | ANDS | SixtyFourBits,
+ ANDS_shift = ANDS_w,
+ BICS_w = LogicalShiftedFixed | BICS,
+ BICS_x = LogicalShiftedFixed | BICS | SixtyFourBits,
+ BICS_shift = BICS_w
+};
+
+// Move wide immediate.
+enum MoveWideImmediateOp : uint32_t {
+ MoveWideImmediateFixed = 0x12800000,
+ MoveWideImmediateFMask = 0x1F800000,
+ MoveWideImmediateMask = 0xFF800000,
+ MOVN = 0x00000000,
+ MOVZ = 0x40000000,
+ MOVK = 0x60000000,
+ MOVN_w = MoveWideImmediateFixed | MOVN,
+ MOVN_x = MoveWideImmediateFixed | MOVN | SixtyFourBits,
+ MOVZ_w = MoveWideImmediateFixed | MOVZ,
+ MOVZ_x = MoveWideImmediateFixed | MOVZ | SixtyFourBits,
+ MOVK_w = MoveWideImmediateFixed | MOVK,
+ MOVK_x = MoveWideImmediateFixed | MOVK | SixtyFourBits
+};
+
+// Bitfield.
+const int kBitfieldNOffset = 22;
+enum BitfieldOp : uint32_t {
+ BitfieldFixed = 0x13000000,
+ BitfieldFMask = 0x1F800000,
+ BitfieldMask = 0xFF800000,
+ SBFM_w = BitfieldFixed | 0x00000000,
+ SBFM_x = BitfieldFixed | 0x80000000,
+ SBFM = SBFM_w,
+ BFM_w = BitfieldFixed | 0x20000000,
+ BFM_x = BitfieldFixed | 0xA0000000,
+ BFM = BFM_w,
+ UBFM_w = BitfieldFixed | 0x40000000,
+ UBFM_x = BitfieldFixed | 0xC0000000,
+ UBFM = UBFM_w
+ // Bitfield N field.
+};
+
+// Extract.
+enum ExtractOp : uint32_t {
+ ExtractFixed = 0x13800000,
+ ExtractFMask = 0x1F800000,
+ ExtractMask = 0xFFA00000,
+ EXTR_w = ExtractFixed | 0x00000000,
+ EXTR_x = ExtractFixed | 0x80000000,
+ EXTR = EXTR_w
+};
+
+// Unconditional branch.
+enum UnconditionalBranchOp : uint32_t {
+ UnconditionalBranchFixed = 0x14000000,
+ UnconditionalBranchFMask = 0x7C000000,
+ UnconditionalBranchMask = 0xFC000000,
+ B = UnconditionalBranchFixed | 0x00000000,
+ BL = UnconditionalBranchFixed | 0x80000000
+};
+
+// Unconditional branch to register.
+enum UnconditionalBranchToRegisterOp : uint32_t {
+ UnconditionalBranchToRegisterFixed = 0xD6000000,
+ UnconditionalBranchToRegisterFMask = 0xFE000000,
+ UnconditionalBranchToRegisterMask = 0xFFFFFC00,
+ BR = UnconditionalBranchToRegisterFixed | 0x001F0000,
+ BLR = UnconditionalBranchToRegisterFixed | 0x003F0000,
+ RET = UnconditionalBranchToRegisterFixed | 0x005F0000,
+
+ BRAAZ = UnconditionalBranchToRegisterFixed | 0x001F0800,
+ BRABZ = UnconditionalBranchToRegisterFixed | 0x001F0C00,
+ BLRAAZ = UnconditionalBranchToRegisterFixed | 0x003F0800,
+ BLRABZ = UnconditionalBranchToRegisterFixed | 0x003F0C00,
+ RETAA = UnconditionalBranchToRegisterFixed | 0x005F0800,
+ RETAB = UnconditionalBranchToRegisterFixed | 0x005F0C00,
+ BRAA = UnconditionalBranchToRegisterFixed | 0x011F0800,
+ BRAB = UnconditionalBranchToRegisterFixed | 0x011F0C00,
+ BLRAA = UnconditionalBranchToRegisterFixed | 0x013F0800,
+ BLRAB = UnconditionalBranchToRegisterFixed | 0x013F0C00
+};
+
+// Compare and branch.
+enum CompareBranchOp : uint32_t {
+ CompareBranchFixed = 0x34000000,
+ CompareBranchFMask = 0x7E000000,
+ CompareBranchMask = 0xFF000000,
+ CBZ_w = CompareBranchFixed | 0x00000000,
+ CBZ_x = CompareBranchFixed | 0x80000000,
+ CBZ = CBZ_w,
+ CBNZ_w = CompareBranchFixed | 0x01000000,
+ CBNZ_x = CompareBranchFixed | 0x81000000,
+ CBNZ = CBNZ_w
+};
+
+// Test and branch.
+enum TestBranchOp : uint32_t {
+ TestBranchFixed = 0x36000000,
+ TestBranchFMask = 0x7E000000,
+ TestBranchMask = 0x7F000000,
+ TBZ = TestBranchFixed | 0x00000000,
+ TBNZ = TestBranchFixed | 0x01000000
+};
+
+// Conditional branch.
+enum ConditionalBranchOp : uint32_t {
+ ConditionalBranchFixed = 0x54000000,
+ ConditionalBranchFMask = 0xFE000000,
+ ConditionalBranchMask = 0xFF000010,
+ B_cond = ConditionalBranchFixed | 0x00000000
+};
+
+// System.
+// System instruction encoding is complicated because some instructions use op
+// and CR fields to encode parameters. To handle this cleanly, the system
+// instructions are split into more than one enum.
+
+enum SystemOp : uint32_t {
+ SystemFixed = 0xD5000000,
+ SystemFMask = 0xFFC00000
+};
+
+enum SystemSysRegOp : uint32_t {
+ SystemSysRegFixed = 0xD5100000,
+ SystemSysRegFMask = 0xFFD00000,
+ SystemSysRegMask = 0xFFF00000,
+ MRS = SystemSysRegFixed | 0x00200000,
+ MSR = SystemSysRegFixed | 0x00000000
+};
+
+enum SystemPStateOp : uint32_t {
+ SystemPStateFixed = 0xD5004000,
+ SystemPStateFMask = 0xFFF8F000,
+ SystemPStateMask = 0xFFFFF0FF,
+ CFINV = SystemPStateFixed | 0x0000001F,
+ XAFLAG = SystemPStateFixed | 0x0000003F,
+ AXFLAG = SystemPStateFixed | 0x0000005F
+};
+
+enum SystemHintOp : uint32_t {
+ SystemHintFixed = 0xD503201F,
+ SystemHintFMask = 0xFFFFF01F,
+ SystemHintMask = 0xFFFFF01F,
+ HINT = SystemHintFixed | 0x00000000
+};
+
+enum SystemSysOp : uint32_t {
+ SystemSysFixed = 0xD5080000,
+ SystemSysFMask = 0xFFF80000,
+ SystemSysMask = 0xFFF80000,
+ SYS = SystemSysFixed | 0x00000000
+};
+
+// Exception.
+enum ExceptionOp : uint32_t {
+ ExceptionFixed = 0xD4000000,
+ ExceptionFMask = 0xFF000000,
+ ExceptionMask = 0xFFE0001F,
+ HLT = ExceptionFixed | 0x00400000,
+ BRK = ExceptionFixed | 0x00200000,
+ SVC = ExceptionFixed | 0x00000001,
+ HVC = ExceptionFixed | 0x00000002,
+ SMC = ExceptionFixed | 0x00000003,
+ DCPS1 = ExceptionFixed | 0x00A00001,
+ DCPS2 = ExceptionFixed | 0x00A00002,
+ DCPS3 = ExceptionFixed | 0x00A00003
+};
+
+enum MemBarrierOp : uint32_t {
+ MemBarrierFixed = 0xD503309F,
+ MemBarrierFMask = 0xFFFFF09F,
+ MemBarrierMask = 0xFFFFF0FF,
+ DSB = MemBarrierFixed | 0x00000000,
+ DMB = MemBarrierFixed | 0x00000020,
+ ISB = MemBarrierFixed | 0x00000040
+};
+
+enum SystemExclusiveMonitorOp : uint32_t {
+ SystemExclusiveMonitorFixed = 0xD503305F,
+ SystemExclusiveMonitorFMask = 0xFFFFF0FF,
+ SystemExclusiveMonitorMask = 0xFFFFF0FF,
+ CLREX = SystemExclusiveMonitorFixed
+};
+
+enum SystemPAuthOp : uint32_t {
+ SystemPAuthFixed = 0xD503211F,
+ SystemPAuthFMask = 0xFFFFFD1F,
+ SystemPAuthMask = 0xFFFFFFFF,
+ PACIA1716 = SystemPAuthFixed | 0x00000100,
+ PACIB1716 = SystemPAuthFixed | 0x00000140,
+ AUTIA1716 = SystemPAuthFixed | 0x00000180,
+ AUTIB1716 = SystemPAuthFixed | 0x000001C0,
+ PACIAZ = SystemPAuthFixed | 0x00000300,
+ PACIASP = SystemPAuthFixed | 0x00000320,
+ PACIBZ = SystemPAuthFixed | 0x00000340,
+ PACIBSP = SystemPAuthFixed | 0x00000360,
+ AUTIAZ = SystemPAuthFixed | 0x00000380,
+ AUTIASP = SystemPAuthFixed | 0x000003A0,
+ AUTIBZ = SystemPAuthFixed | 0x000003C0,
+ AUTIBSP = SystemPAuthFixed | 0x000003E0,
+
+ // XPACLRI has the same fixed mask as System Hints and needs to be handled
+ // differently.
+ XPACLRI = 0xD50320FF
+};
+
+// Any load or store.
+enum LoadStoreAnyOp : uint32_t {
+ LoadStoreAnyFMask = 0x0a000000,
+ LoadStoreAnyFixed = 0x08000000
+};
+
+// Any load pair or store pair.
+enum LoadStorePairAnyOp : uint32_t {
+ LoadStorePairAnyFMask = 0x3a000000,
+ LoadStorePairAnyFixed = 0x28000000
+};
+
+#define LOAD_STORE_PAIR_OP_LIST(V) \
+ V(STP, w, 0x00000000), \
+ V(LDP, w, 0x00400000), \
+ V(LDPSW, x, 0x40400000), \
+ V(STP, x, 0x80000000), \
+ V(LDP, x, 0x80400000), \
+ V(STP, s, 0x04000000), \
+ V(LDP, s, 0x04400000), \
+ V(STP, d, 0x44000000), \
+ V(LDP, d, 0x44400000), \
+ V(STP, q, 0x84000000), \
+ V(LDP, q, 0x84400000)
+
+// Load/store pair (post, pre and offset.)
+enum LoadStorePairOp : uint32_t {
+ LoadStorePairMask = 0xC4400000,
+ LoadStorePairLBit = 1 << 22,
+ #define LOAD_STORE_PAIR(A, B, C) \
+ A##_##B = C
+ LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR)
+ #undef LOAD_STORE_PAIR
+};
+
+enum LoadStorePairPostIndexOp : uint32_t {
+ LoadStorePairPostIndexFixed = 0x28800000,
+ LoadStorePairPostIndexFMask = 0x3B800000,
+ LoadStorePairPostIndexMask = 0xFFC00000,
+ #define LOAD_STORE_PAIR_POST_INDEX(A, B, C) \
+ A##_##B##_post = LoadStorePairPostIndexFixed | A##_##B
+ LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_POST_INDEX)
+ #undef LOAD_STORE_PAIR_POST_INDEX
+};
+
+enum LoadStorePairPreIndexOp : uint32_t {
+ LoadStorePairPreIndexFixed = 0x29800000,
+ LoadStorePairPreIndexFMask = 0x3B800000,
+ LoadStorePairPreIndexMask = 0xFFC00000,
+ #define LOAD_STORE_PAIR_PRE_INDEX(A, B, C) \
+ A##_##B##_pre = LoadStorePairPreIndexFixed | A##_##B
+ LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_PRE_INDEX)
+ #undef LOAD_STORE_PAIR_PRE_INDEX
+};
+
+enum LoadStorePairOffsetOp : uint32_t {
+ LoadStorePairOffsetFixed = 0x29000000,
+ LoadStorePairOffsetFMask = 0x3B800000,
+ LoadStorePairOffsetMask = 0xFFC00000,
+ #define LOAD_STORE_PAIR_OFFSET(A, B, C) \
+ A##_##B##_off = LoadStorePairOffsetFixed | A##_##B
+ LOAD_STORE_PAIR_OP_LIST(LOAD_STORE_PAIR_OFFSET)
+ #undef LOAD_STORE_PAIR_OFFSET
+};
+
+enum LoadStorePairNonTemporalOp : uint32_t {
+ LoadStorePairNonTemporalFixed = 0x28000000,
+ LoadStorePairNonTemporalFMask = 0x3B800000,
+ LoadStorePairNonTemporalMask = 0xFFC00000,
+ LoadStorePairNonTemporalLBit = 1 << 22,
+ STNP_w = LoadStorePairNonTemporalFixed | STP_w,
+ LDNP_w = LoadStorePairNonTemporalFixed | LDP_w,
+ STNP_x = LoadStorePairNonTemporalFixed | STP_x,
+ LDNP_x = LoadStorePairNonTemporalFixed | LDP_x,
+ STNP_s = LoadStorePairNonTemporalFixed | STP_s,
+ LDNP_s = LoadStorePairNonTemporalFixed | LDP_s,
+ STNP_d = LoadStorePairNonTemporalFixed | STP_d,
+ LDNP_d = LoadStorePairNonTemporalFixed | LDP_d,
+ STNP_q = LoadStorePairNonTemporalFixed | STP_q,
+ LDNP_q = LoadStorePairNonTemporalFixed | LDP_q
+};
+
+// Load with pointer authentication.
+enum LoadStorePACOp : uint32_t {
+ LoadStorePACFixed = 0xF8200400,
+ LoadStorePACFMask = 0xFF200400,
+ LoadStorePACMask = 0xFFA00C00,
+ LoadStorePACPreBit = 0x00000800,
+ LDRAA = LoadStorePACFixed | 0x00000000,
+ LDRAA_pre = LoadStorePACPreBit | LDRAA,
+ LDRAB = LoadStorePACFixed | 0x00800000,
+ LDRAB_pre = LoadStorePACPreBit | LDRAB
+};
+
+// Load literal.
+enum LoadLiteralOp : uint32_t {
+ LoadLiteralFixed = 0x18000000,
+ LoadLiteralFMask = 0x3B000000,
+ LoadLiteralMask = 0xFF000000,
+ LDR_w_lit = LoadLiteralFixed | 0x00000000,
+ LDR_x_lit = LoadLiteralFixed | 0x40000000,
+ LDRSW_x_lit = LoadLiteralFixed | 0x80000000,
+ PRFM_lit = LoadLiteralFixed | 0xC0000000,
+ LDR_s_lit = LoadLiteralFixed | 0x04000000,
+ LDR_d_lit = LoadLiteralFixed | 0x44000000,
+ LDR_q_lit = LoadLiteralFixed | 0x84000000
+};
+
+#define LOAD_STORE_OP_LIST(V) \
+ V(ST, RB, w, 0x00000000), \
+ V(ST, RH, w, 0x40000000), \
+ V(ST, R, w, 0x80000000), \
+ V(ST, R, x, 0xC0000000), \
+ V(LD, RB, w, 0x00400000), \
+ V(LD, RH, w, 0x40400000), \
+ V(LD, R, w, 0x80400000), \
+ V(LD, R, x, 0xC0400000), \
+ V(LD, RSB, x, 0x00800000), \
+ V(LD, RSH, x, 0x40800000), \
+ V(LD, RSW, x, 0x80800000), \
+ V(LD, RSB, w, 0x00C00000), \
+ V(LD, RSH, w, 0x40C00000), \
+ V(ST, R, b, 0x04000000), \
+ V(ST, R, h, 0x44000000), \
+ V(ST, R, s, 0x84000000), \
+ V(ST, R, d, 0xC4000000), \
+ V(ST, R, q, 0x04800000), \
+ V(LD, R, b, 0x04400000), \
+ V(LD, R, h, 0x44400000), \
+ V(LD, R, s, 0x84400000), \
+ V(LD, R, d, 0xC4400000), \
+ V(LD, R, q, 0x04C00000)
+
+// Load/store (post, pre, offset and unsigned.)
+enum LoadStoreOp : uint32_t {
+ LoadStoreMask = 0xC4C00000,
+ LoadStoreVMask = 0x04000000,
+ #define LOAD_STORE(A, B, C, D) \
+ A##B##_##C = D
+ LOAD_STORE_OP_LIST(LOAD_STORE),
+ #undef LOAD_STORE
+ PRFM = 0xC0800000
+};
+
+// Load/store unscaled offset.
+enum LoadStoreUnscaledOffsetOp : uint32_t {
+ LoadStoreUnscaledOffsetFixed = 0x38000000,
+ LoadStoreUnscaledOffsetFMask = 0x3B200C00,
+ LoadStoreUnscaledOffsetMask = 0xFFE00C00,
+ PRFUM = LoadStoreUnscaledOffsetFixed | PRFM,
+ #define LOAD_STORE_UNSCALED(A, B, C, D) \
+ A##U##B##_##C = LoadStoreUnscaledOffsetFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_UNSCALED)
+ #undef LOAD_STORE_UNSCALED
+};
+
+// Load/store post index.
+enum LoadStorePostIndex : uint32_t {
+ LoadStorePostIndexFixed = 0x38000400,
+ LoadStorePostIndexFMask = 0x3B200C00,
+ LoadStorePostIndexMask = 0xFFE00C00,
+ #define LOAD_STORE_POST_INDEX(A, B, C, D) \
+ A##B##_##C##_post = LoadStorePostIndexFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_POST_INDEX)
+ #undef LOAD_STORE_POST_INDEX
+};
+
+// Load/store pre index.
+enum LoadStorePreIndex : uint32_t {
+ LoadStorePreIndexFixed = 0x38000C00,
+ LoadStorePreIndexFMask = 0x3B200C00,
+ LoadStorePreIndexMask = 0xFFE00C00,
+ #define LOAD_STORE_PRE_INDEX(A, B, C, D) \
+ A##B##_##C##_pre = LoadStorePreIndexFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_PRE_INDEX)
+ #undef LOAD_STORE_PRE_INDEX
+};
+
+// Load/store unsigned offset.
+enum LoadStoreUnsignedOffset : uint32_t {
+ LoadStoreUnsignedOffsetFixed = 0x39000000,
+ LoadStoreUnsignedOffsetFMask = 0x3B000000,
+ LoadStoreUnsignedOffsetMask = 0xFFC00000,
+ PRFM_unsigned = LoadStoreUnsignedOffsetFixed | PRFM,
+ #define LOAD_STORE_UNSIGNED_OFFSET(A, B, C, D) \
+ A##B##_##C##_unsigned = LoadStoreUnsignedOffsetFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_UNSIGNED_OFFSET)
+ #undef LOAD_STORE_UNSIGNED_OFFSET
+};
+
+// Load/store register offset.
+enum LoadStoreRegisterOffset : uint32_t {
+ LoadStoreRegisterOffsetFixed = 0x38200800,
+ LoadStoreRegisterOffsetFMask = 0x3B200C00,
+ LoadStoreRegisterOffsetMask = 0xFFE00C00,
+ PRFM_reg = LoadStoreRegisterOffsetFixed | PRFM,
+ #define LOAD_STORE_REGISTER_OFFSET(A, B, C, D) \
+ A##B##_##C##_reg = LoadStoreRegisterOffsetFixed | D
+ LOAD_STORE_OP_LIST(LOAD_STORE_REGISTER_OFFSET)
+ #undef LOAD_STORE_REGISTER_OFFSET
+};
+
+enum LoadStoreExclusive : uint32_t {
+ LoadStoreExclusiveFixed = 0x08000000,
+ LoadStoreExclusiveFMask = 0x3F000000,
+ LoadStoreExclusiveMask = 0xFFE08000,
+ STXRB_w = LoadStoreExclusiveFixed | 0x00000000,
+ STXRH_w = LoadStoreExclusiveFixed | 0x40000000,
+ STXR_w = LoadStoreExclusiveFixed | 0x80000000,
+ STXR_x = LoadStoreExclusiveFixed | 0xC0000000,
+ LDXRB_w = LoadStoreExclusiveFixed | 0x00400000,
+ LDXRH_w = LoadStoreExclusiveFixed | 0x40400000,
+ LDXR_w = LoadStoreExclusiveFixed | 0x80400000,
+ LDXR_x = LoadStoreExclusiveFixed | 0xC0400000,
+ STXP_w = LoadStoreExclusiveFixed | 0x80200000,
+ STXP_x = LoadStoreExclusiveFixed | 0xC0200000,
+ LDXP_w = LoadStoreExclusiveFixed | 0x80600000,
+ LDXP_x = LoadStoreExclusiveFixed | 0xC0600000,
+ STLXRB_w = LoadStoreExclusiveFixed | 0x00008000,
+ STLXRH_w = LoadStoreExclusiveFixed | 0x40008000,
+ STLXR_w = LoadStoreExclusiveFixed | 0x80008000,
+ STLXR_x = LoadStoreExclusiveFixed | 0xC0008000,
+ LDAXRB_w = LoadStoreExclusiveFixed | 0x00408000,
+ LDAXRH_w = LoadStoreExclusiveFixed | 0x40408000,
+ LDAXR_w = LoadStoreExclusiveFixed | 0x80408000,
+ LDAXR_x = LoadStoreExclusiveFixed | 0xC0408000,
+ STLXP_w = LoadStoreExclusiveFixed | 0x80208000,
+ STLXP_x = LoadStoreExclusiveFixed | 0xC0208000,
+ LDAXP_w = LoadStoreExclusiveFixed | 0x80608000,
+ LDAXP_x = LoadStoreExclusiveFixed | 0xC0608000,
+ STLRB_w = LoadStoreExclusiveFixed | 0x00808000,
+ STLRH_w = LoadStoreExclusiveFixed | 0x40808000,
+ STLR_w = LoadStoreExclusiveFixed | 0x80808000,
+ STLR_x = LoadStoreExclusiveFixed | 0xC0808000,
+ LDARB_w = LoadStoreExclusiveFixed | 0x00C08000,
+ LDARH_w = LoadStoreExclusiveFixed | 0x40C08000,
+ LDAR_w = LoadStoreExclusiveFixed | 0x80C08000,
+ LDAR_x = LoadStoreExclusiveFixed | 0xC0C08000,
+
+ // v8.1 Load/store LORegion ops
+ STLLRB = LoadStoreExclusiveFixed | 0x00800000,
+ LDLARB = LoadStoreExclusiveFixed | 0x00C00000,
+ STLLRH = LoadStoreExclusiveFixed | 0x40800000,
+ LDLARH = LoadStoreExclusiveFixed | 0x40C00000,
+ STLLR_w = LoadStoreExclusiveFixed | 0x80800000,
+ LDLAR_w = LoadStoreExclusiveFixed | 0x80C00000,
+ STLLR_x = LoadStoreExclusiveFixed | 0xC0800000,
+ LDLAR_x = LoadStoreExclusiveFixed | 0xC0C00000,
+
+ // v8.1 Load/store exclusive ops
+ LSEBit_l = 0x00400000,
+ LSEBit_o0 = 0x00008000,
+ LSEBit_sz = 0x40000000,
+ CASFixed = LoadStoreExclusiveFixed | 0x80A00000,
+ CASBFixed = LoadStoreExclusiveFixed | 0x00A00000,
+ CASHFixed = LoadStoreExclusiveFixed | 0x40A00000,
+ CASPFixed = LoadStoreExclusiveFixed | 0x00200000,
+ CAS_w = CASFixed,
+ CAS_x = CASFixed | LSEBit_sz,
+ CASA_w = CASFixed | LSEBit_l,
+ CASA_x = CASFixed | LSEBit_l | LSEBit_sz,
+ CASL_w = CASFixed | LSEBit_o0,
+ CASL_x = CASFixed | LSEBit_o0 | LSEBit_sz,
+ CASAL_w = CASFixed | LSEBit_l | LSEBit_o0,
+ CASAL_x = CASFixed | LSEBit_l | LSEBit_o0 | LSEBit_sz,
+ CASB = CASBFixed,
+ CASAB = CASBFixed | LSEBit_l,
+ CASLB = CASBFixed | LSEBit_o0,
+ CASALB = CASBFixed | LSEBit_l | LSEBit_o0,
+ CASH = CASHFixed,
+ CASAH = CASHFixed | LSEBit_l,
+ CASLH = CASHFixed | LSEBit_o0,
+ CASALH = CASHFixed | LSEBit_l | LSEBit_o0,
+ CASP_w = CASPFixed,
+ CASP_x = CASPFixed | LSEBit_sz,
+ CASPA_w = CASPFixed | LSEBit_l,
+ CASPA_x = CASPFixed | LSEBit_l | LSEBit_sz,
+ CASPL_w = CASPFixed | LSEBit_o0,
+ CASPL_x = CASPFixed | LSEBit_o0 | LSEBit_sz,
+ CASPAL_w = CASPFixed | LSEBit_l | LSEBit_o0,
+ CASPAL_x = CASPFixed | LSEBit_l | LSEBit_o0 | LSEBit_sz
+};
+
+// Load/store RCpc unscaled offset.
+enum LoadStoreRCpcUnscaledOffsetOp : uint32_t {
+ LoadStoreRCpcUnscaledOffsetFixed = 0x19000000,
+ LoadStoreRCpcUnscaledOffsetFMask = 0x3F200C00,
+ LoadStoreRCpcUnscaledOffsetMask = 0xFFE00C00,
+ STLURB = LoadStoreRCpcUnscaledOffsetFixed | 0x00000000,
+ LDAPURB = LoadStoreRCpcUnscaledOffsetFixed | 0x00400000,
+ LDAPURSB_x = LoadStoreRCpcUnscaledOffsetFixed | 0x00800000,
+ LDAPURSB_w = LoadStoreRCpcUnscaledOffsetFixed | 0x00C00000,
+ STLURH = LoadStoreRCpcUnscaledOffsetFixed | 0x40000000,
+ LDAPURH = LoadStoreRCpcUnscaledOffsetFixed | 0x40400000,
+ LDAPURSH_x = LoadStoreRCpcUnscaledOffsetFixed | 0x40800000,
+ LDAPURSH_w = LoadStoreRCpcUnscaledOffsetFixed | 0x40C00000,
+ STLUR_w = LoadStoreRCpcUnscaledOffsetFixed | 0x80000000,
+ LDAPUR_w = LoadStoreRCpcUnscaledOffsetFixed | 0x80400000,
+ LDAPURSW = LoadStoreRCpcUnscaledOffsetFixed | 0x80800000,
+ STLUR_x = LoadStoreRCpcUnscaledOffsetFixed | 0xC0000000,
+ LDAPUR_x = LoadStoreRCpcUnscaledOffsetFixed | 0xC0400000
+};
+
+#define ATOMIC_MEMORY_SIMPLE_OPC_LIST(V) \
+ V(LDADD, 0x00000000), \
+ V(LDCLR, 0x00001000), \
+ V(LDEOR, 0x00002000), \
+ V(LDSET, 0x00003000), \
+ V(LDSMAX, 0x00004000), \
+ V(LDSMIN, 0x00005000), \
+ V(LDUMAX, 0x00006000), \
+ V(LDUMIN, 0x00007000)
+
+// Atomic memory.
+enum AtomicMemoryOp : uint32_t {
+ AtomicMemoryFixed = 0x38200000,
+ AtomicMemoryFMask = 0x3B200C00,
+ AtomicMemoryMask = 0xFFE0FC00,
+ SWPB = AtomicMemoryFixed | 0x00008000,
+ SWPAB = AtomicMemoryFixed | 0x00808000,
+ SWPLB = AtomicMemoryFixed | 0x00408000,
+ SWPALB = AtomicMemoryFixed | 0x00C08000,
+ SWPH = AtomicMemoryFixed | 0x40008000,
+ SWPAH = AtomicMemoryFixed | 0x40808000,
+ SWPLH = AtomicMemoryFixed | 0x40408000,
+ SWPALH = AtomicMemoryFixed | 0x40C08000,
+ SWP_w = AtomicMemoryFixed | 0x80008000,
+ SWPA_w = AtomicMemoryFixed | 0x80808000,
+ SWPL_w = AtomicMemoryFixed | 0x80408000,
+ SWPAL_w = AtomicMemoryFixed | 0x80C08000,
+ SWP_x = AtomicMemoryFixed | 0xC0008000,
+ SWPA_x = AtomicMemoryFixed | 0xC0808000,
+ SWPL_x = AtomicMemoryFixed | 0xC0408000,
+ SWPAL_x = AtomicMemoryFixed | 0xC0C08000,
+ LDAPRB = AtomicMemoryFixed | 0x0080C000,
+ LDAPRH = AtomicMemoryFixed | 0x4080C000,
+ LDAPR_w = AtomicMemoryFixed | 0x8080C000,
+ LDAPR_x = AtomicMemoryFixed | 0xC080C000,
+
+ AtomicMemorySimpleFMask = 0x3B208C00,
+ AtomicMemorySimpleOpMask = 0x00007000,
+#define ATOMIC_MEMORY_SIMPLE(N, OP) \
+ N##Op = OP, \
+ N##B = AtomicMemoryFixed | OP, \
+ N##AB = AtomicMemoryFixed | OP | 0x00800000, \
+ N##LB = AtomicMemoryFixed | OP | 0x00400000, \
+ N##ALB = AtomicMemoryFixed | OP | 0x00C00000, \
+ N##H = AtomicMemoryFixed | OP | 0x40000000, \
+ N##AH = AtomicMemoryFixed | OP | 0x40800000, \
+ N##LH = AtomicMemoryFixed | OP | 0x40400000, \
+ N##ALH = AtomicMemoryFixed | OP | 0x40C00000, \
+ N##_w = AtomicMemoryFixed | OP | 0x80000000, \
+ N##A_w = AtomicMemoryFixed | OP | 0x80800000, \
+ N##L_w = AtomicMemoryFixed | OP | 0x80400000, \
+ N##AL_w = AtomicMemoryFixed | OP | 0x80C00000, \
+ N##_x = AtomicMemoryFixed | OP | 0xC0000000, \
+ N##A_x = AtomicMemoryFixed | OP | 0xC0800000, \
+ N##L_x = AtomicMemoryFixed | OP | 0xC0400000, \
+ N##AL_x = AtomicMemoryFixed | OP | 0xC0C00000
+
+ ATOMIC_MEMORY_SIMPLE_OPC_LIST(ATOMIC_MEMORY_SIMPLE)
+#undef ATOMIC_MEMORY_SIMPLE
+};
+
+// Conditional compare.
+enum ConditionalCompareOp : uint32_t {
+ ConditionalCompareMask = 0x60000000,
+ CCMN = 0x20000000,
+ CCMP = 0x60000000
+};
+
+// Conditional compare register.
+enum ConditionalCompareRegisterOp : uint32_t {
+ ConditionalCompareRegisterFixed = 0x1A400000,
+ ConditionalCompareRegisterFMask = 0x1FE00800,
+ ConditionalCompareRegisterMask = 0xFFE00C10,
+ CCMN_w = ConditionalCompareRegisterFixed | CCMN,
+ CCMN_x = ConditionalCompareRegisterFixed | SixtyFourBits | CCMN,
+ CCMP_w = ConditionalCompareRegisterFixed | CCMP,
+ CCMP_x = ConditionalCompareRegisterFixed | SixtyFourBits | CCMP
+};
+
+// Conditional compare immediate.
+enum ConditionalCompareImmediateOp : uint32_t {
+ ConditionalCompareImmediateFixed = 0x1A400800,
+ ConditionalCompareImmediateFMask = 0x1FE00800,
+ ConditionalCompareImmediateMask = 0xFFE00C10,
+ CCMN_w_imm = ConditionalCompareImmediateFixed | CCMN,
+ CCMN_x_imm = ConditionalCompareImmediateFixed | SixtyFourBits | CCMN,
+ CCMP_w_imm = ConditionalCompareImmediateFixed | CCMP,
+ CCMP_x_imm = ConditionalCompareImmediateFixed | SixtyFourBits | CCMP
+};
+
+// Conditional select.
+enum ConditionalSelectOp : uint32_t {
+ ConditionalSelectFixed = 0x1A800000,
+ ConditionalSelectFMask = 0x1FE00000,
+ ConditionalSelectMask = 0xFFE00C00,
+ CSEL_w = ConditionalSelectFixed | 0x00000000,
+ CSEL_x = ConditionalSelectFixed | 0x80000000,
+ CSEL = CSEL_w,
+ CSINC_w = ConditionalSelectFixed | 0x00000400,
+ CSINC_x = ConditionalSelectFixed | 0x80000400,
+ CSINC = CSINC_w,
+ CSINV_w = ConditionalSelectFixed | 0x40000000,
+ CSINV_x = ConditionalSelectFixed | 0xC0000000,
+ CSINV = CSINV_w,
+ CSNEG_w = ConditionalSelectFixed | 0x40000400,
+ CSNEG_x = ConditionalSelectFixed | 0xC0000400,
+ CSNEG = CSNEG_w
+};
+
+// Data processing 1 source.
+enum DataProcessing1SourceOp : uint32_t {
+ DataProcessing1SourceFixed = 0x5AC00000,
+ DataProcessing1SourceFMask = 0x5FE00000,
+ DataProcessing1SourceMask = 0xFFFFFC00,
+ RBIT = DataProcessing1SourceFixed | 0x00000000,
+ RBIT_w = RBIT,
+ RBIT_x = RBIT | SixtyFourBits,
+ REV16 = DataProcessing1SourceFixed | 0x00000400,
+ REV16_w = REV16,
+ REV16_x = REV16 | SixtyFourBits,
+ REV = DataProcessing1SourceFixed | 0x00000800,
+ REV_w = REV,
+ REV32_x = REV | SixtyFourBits,
+ REV_x = DataProcessing1SourceFixed | SixtyFourBits | 0x00000C00,
+ CLZ = DataProcessing1SourceFixed | 0x00001000,
+ CLZ_w = CLZ,
+ CLZ_x = CLZ | SixtyFourBits,
+ CLS = DataProcessing1SourceFixed | 0x00001400,
+ CLS_w = CLS,
+ CLS_x = CLS | SixtyFourBits,
+
+ // Pointer authentication instructions in Armv8.3.
+ PACIA = DataProcessing1SourceFixed | 0x80010000,
+ PACIB = DataProcessing1SourceFixed | 0x80010400,
+ PACDA = DataProcessing1SourceFixed | 0x80010800,
+ PACDB = DataProcessing1SourceFixed | 0x80010C00,
+ AUTIA = DataProcessing1SourceFixed | 0x80011000,
+ AUTIB = DataProcessing1SourceFixed | 0x80011400,
+ AUTDA = DataProcessing1SourceFixed | 0x80011800,
+ AUTDB = DataProcessing1SourceFixed | 0x80011C00,
+ PACIZA = DataProcessing1SourceFixed | 0x80012000,
+ PACIZB = DataProcessing1SourceFixed | 0x80012400,
+ PACDZA = DataProcessing1SourceFixed | 0x80012800,
+ PACDZB = DataProcessing1SourceFixed | 0x80012C00,
+ AUTIZA = DataProcessing1SourceFixed | 0x80013000,
+ AUTIZB = DataProcessing1SourceFixed | 0x80013400,
+ AUTDZA = DataProcessing1SourceFixed | 0x80013800,
+ AUTDZB = DataProcessing1SourceFixed | 0x80013C00,
+ XPACI = DataProcessing1SourceFixed | 0x80014000,
+ XPACD = DataProcessing1SourceFixed | 0x80014400
+};
+
+// Data processing 2 source.
+enum DataProcessing2SourceOp : uint32_t {
+ DataProcessing2SourceFixed = 0x1AC00000,
+ DataProcessing2SourceFMask = 0x5FE00000,
+ DataProcessing2SourceMask = 0xFFE0FC00,
+ UDIV_w = DataProcessing2SourceFixed | 0x00000800,
+ UDIV_x = DataProcessing2SourceFixed | 0x80000800,
+ UDIV = UDIV_w,
+ SDIV_w = DataProcessing2SourceFixed | 0x00000C00,
+ SDIV_x = DataProcessing2SourceFixed | 0x80000C00,
+ SDIV = SDIV_w,
+ LSLV_w = DataProcessing2SourceFixed | 0x00002000,
+ LSLV_x = DataProcessing2SourceFixed | 0x80002000,
+ LSLV = LSLV_w,
+ LSRV_w = DataProcessing2SourceFixed | 0x00002400,
+ LSRV_x = DataProcessing2SourceFixed | 0x80002400,
+ LSRV = LSRV_w,
+ ASRV_w = DataProcessing2SourceFixed | 0x00002800,
+ ASRV_x = DataProcessing2SourceFixed | 0x80002800,
+ ASRV = ASRV_w,
+ RORV_w = DataProcessing2SourceFixed | 0x00002C00,
+ RORV_x = DataProcessing2SourceFixed | 0x80002C00,
+ RORV = RORV_w,
+ PACGA = DataProcessing2SourceFixed | SixtyFourBits | 0x00003000,
+ CRC32B = DataProcessing2SourceFixed | 0x00004000,
+ CRC32H = DataProcessing2SourceFixed | 0x00004400,
+ CRC32W = DataProcessing2SourceFixed | 0x00004800,
+ CRC32X = DataProcessing2SourceFixed | SixtyFourBits | 0x00004C00,
+ CRC32CB = DataProcessing2SourceFixed | 0x00005000,
+ CRC32CH = DataProcessing2SourceFixed | 0x00005400,
+ CRC32CW = DataProcessing2SourceFixed | 0x00005800,
+ CRC32CX = DataProcessing2SourceFixed | SixtyFourBits | 0x00005C00
+};
+
+// Data processing 3 source.
+enum DataProcessing3SourceOp : uint32_t {
+ DataProcessing3SourceFixed = 0x1B000000,
+ DataProcessing3SourceFMask = 0x1F000000,
+ DataProcessing3SourceMask = 0xFFE08000,
+ MADD_w = DataProcessing3SourceFixed | 0x00000000,
+ MADD_x = DataProcessing3SourceFixed | 0x80000000,
+ MADD = MADD_w,
+ MSUB_w = DataProcessing3SourceFixed | 0x00008000,
+ MSUB_x = DataProcessing3SourceFixed | 0x80008000,
+ MSUB = MSUB_w,
+ SMADDL_x = DataProcessing3SourceFixed | 0x80200000,
+ SMSUBL_x = DataProcessing3SourceFixed | 0x80208000,
+ SMULH_x = DataProcessing3SourceFixed | 0x80400000,
+ UMADDL_x = DataProcessing3SourceFixed | 0x80A00000,
+ UMSUBL_x = DataProcessing3SourceFixed | 0x80A08000,
+ UMULH_x = DataProcessing3SourceFixed | 0x80C00000
+};
+
+// Floating point compare.
+enum FPCompareOp : uint32_t {
+ FPCompareFixed = 0x1E202000,
+ FPCompareFMask = 0x5F203C00,
+ FPCompareMask = 0xFFE0FC1F,
+ FCMP_h = FPCompareFixed | FP16 | 0x00000000,
+ FCMP_s = FPCompareFixed | 0x00000000,
+ FCMP_d = FPCompareFixed | FP64 | 0x00000000,
+ FCMP = FCMP_s,
+ FCMP_h_zero = FPCompareFixed | FP16 | 0x00000008,
+ FCMP_s_zero = FPCompareFixed | 0x00000008,
+ FCMP_d_zero = FPCompareFixed | FP64 | 0x00000008,
+ FCMP_zero = FCMP_s_zero,
+ FCMPE_h = FPCompareFixed | FP16 | 0x00000010,
+ FCMPE_s = FPCompareFixed | 0x00000010,
+ FCMPE_d = FPCompareFixed | FP64 | 0x00000010,
+ FCMPE = FCMPE_s,
+ FCMPE_h_zero = FPCompareFixed | FP16 | 0x00000018,
+ FCMPE_s_zero = FPCompareFixed | 0x00000018,
+ FCMPE_d_zero = FPCompareFixed | FP64 | 0x00000018,
+ FCMPE_zero = FCMPE_s_zero
+};
+
+// Floating point conditional compare.
+enum FPConditionalCompareOp : uint32_t {
+ FPConditionalCompareFixed = 0x1E200400,
+ FPConditionalCompareFMask = 0x5F200C00,
+ FPConditionalCompareMask = 0xFFE00C10,
+ FCCMP_h = FPConditionalCompareFixed | FP16 | 0x00000000,
+ FCCMP_s = FPConditionalCompareFixed | 0x00000000,
+ FCCMP_d = FPConditionalCompareFixed | FP64 | 0x00000000,
+ FCCMP = FCCMP_s,
+ FCCMPE_h = FPConditionalCompareFixed | FP16 | 0x00000010,
+ FCCMPE_s = FPConditionalCompareFixed | 0x00000010,
+ FCCMPE_d = FPConditionalCompareFixed | FP64 | 0x00000010,
+ FCCMPE = FCCMPE_s
+};
+
+// Floating point conditional select.
+enum FPConditionalSelectOp : uint32_t {
+ FPConditionalSelectFixed = 0x1E200C00,
+ FPConditionalSelectFMask = 0x5F200C00,
+ FPConditionalSelectMask = 0xFFE00C00,
+ FCSEL_h = FPConditionalSelectFixed | FP16 | 0x00000000,
+ FCSEL_s = FPConditionalSelectFixed | 0x00000000,
+ FCSEL_d = FPConditionalSelectFixed | FP64 | 0x00000000,
+ FCSEL = FCSEL_s
+};
+
+// Floating point immediate.
+enum FPImmediateOp : uint32_t {
+ FPImmediateFixed = 0x1E201000,
+ FPImmediateFMask = 0x5F201C00,
+ FPImmediateMask = 0xFFE01C00,
+ FMOV_h_imm = FPImmediateFixed | FP16 | 0x00000000,
+ FMOV_s_imm = FPImmediateFixed | 0x00000000,
+ FMOV_d_imm = FPImmediateFixed | FP64 | 0x00000000
+};
+
+// Floating point data processing 1 source.
+enum FPDataProcessing1SourceOp : uint32_t {
+ FPDataProcessing1SourceFixed = 0x1E204000,
+ FPDataProcessing1SourceFMask = 0x5F207C00,
+ FPDataProcessing1SourceMask = 0xFFFFFC00,
+ FMOV_h = FPDataProcessing1SourceFixed | FP16 | 0x00000000,
+ FMOV_s = FPDataProcessing1SourceFixed | 0x00000000,
+ FMOV_d = FPDataProcessing1SourceFixed | FP64 | 0x00000000,
+ FMOV = FMOV_s,
+ FABS_h = FPDataProcessing1SourceFixed | FP16 | 0x00008000,
+ FABS_s = FPDataProcessing1SourceFixed | 0x00008000,
+ FABS_d = FPDataProcessing1SourceFixed | FP64 | 0x00008000,
+ FABS = FABS_s,
+ FNEG_h = FPDataProcessing1SourceFixed | FP16 | 0x00010000,
+ FNEG_s = FPDataProcessing1SourceFixed | 0x00010000,
+ FNEG_d = FPDataProcessing1SourceFixed | FP64 | 0x00010000,
+ FNEG = FNEG_s,
+ FSQRT_h = FPDataProcessing1SourceFixed | FP16 | 0x00018000,
+ FSQRT_s = FPDataProcessing1SourceFixed | 0x00018000,
+ FSQRT_d = FPDataProcessing1SourceFixed | FP64 | 0x00018000,
+ FSQRT = FSQRT_s,
+ FCVT_ds = FPDataProcessing1SourceFixed | 0x00028000,
+ FCVT_sd = FPDataProcessing1SourceFixed | FP64 | 0x00020000,
+ FCVT_hs = FPDataProcessing1SourceFixed | 0x00038000,
+ FCVT_hd = FPDataProcessing1SourceFixed | FP64 | 0x00038000,
+ FCVT_sh = FPDataProcessing1SourceFixed | 0x00C20000,
+ FCVT_dh = FPDataProcessing1SourceFixed | 0x00C28000,
+ FRINT32X_s = FPDataProcessing1SourceFixed | 0x00088000,
+ FRINT32X_d = FPDataProcessing1SourceFixed | FP64 | 0x00088000,
+ FRINT32X = FRINT32X_s,
+ FRINT32Z_s = FPDataProcessing1SourceFixed | 0x00080000,
+ FRINT32Z_d = FPDataProcessing1SourceFixed | FP64 | 0x00080000,
+ FRINT32Z = FRINT32Z_s,
+ FRINT64X_s = FPDataProcessing1SourceFixed | 0x00098000,
+ FRINT64X_d = FPDataProcessing1SourceFixed | FP64 | 0x00098000,
+ FRINT64X = FRINT64X_s,
+ FRINT64Z_s = FPDataProcessing1SourceFixed | 0x00090000,
+ FRINT64Z_d = FPDataProcessing1SourceFixed | FP64 | 0x00090000,
+ FRINT64Z = FRINT64Z_s,
+ FRINTN_h = FPDataProcessing1SourceFixed | FP16 | 0x00040000,
+ FRINTN_s = FPDataProcessing1SourceFixed | 0x00040000,
+ FRINTN_d = FPDataProcessing1SourceFixed | FP64 | 0x00040000,
+ FRINTN = FRINTN_s,
+ FRINTP_h = FPDataProcessing1SourceFixed | FP16 | 0x00048000,
+ FRINTP_s = FPDataProcessing1SourceFixed | 0x00048000,
+ FRINTP_d = FPDataProcessing1SourceFixed | FP64 | 0x00048000,
+ FRINTP = FRINTP_s,
+ FRINTM_h = FPDataProcessing1SourceFixed | FP16 | 0x00050000,
+ FRINTM_s = FPDataProcessing1SourceFixed | 0x00050000,
+ FRINTM_d = FPDataProcessing1SourceFixed | FP64 | 0x00050000,
+ FRINTM = FRINTM_s,
+ FRINTZ_h = FPDataProcessing1SourceFixed | FP16 | 0x00058000,
+ FRINTZ_s = FPDataProcessing1SourceFixed | 0x00058000,
+ FRINTZ_d = FPDataProcessing1SourceFixed | FP64 | 0x00058000,
+ FRINTZ = FRINTZ_s,
+ FRINTA_h = FPDataProcessing1SourceFixed | FP16 | 0x00060000,
+ FRINTA_s = FPDataProcessing1SourceFixed | 0x00060000,
+ FRINTA_d = FPDataProcessing1SourceFixed | FP64 | 0x00060000,
+ FRINTA = FRINTA_s,
+ FRINTX_h = FPDataProcessing1SourceFixed | FP16 | 0x00070000,
+ FRINTX_s = FPDataProcessing1SourceFixed | 0x00070000,
+ FRINTX_d = FPDataProcessing1SourceFixed | FP64 | 0x00070000,
+ FRINTX = FRINTX_s,
+ FRINTI_h = FPDataProcessing1SourceFixed | FP16 | 0x00078000,
+ FRINTI_s = FPDataProcessing1SourceFixed | 0x00078000,
+ FRINTI_d = FPDataProcessing1SourceFixed | FP64 | 0x00078000,
+ FRINTI = FRINTI_s
+};
+
+// Floating point data processing 2 source.
+enum FPDataProcessing2SourceOp : uint32_t {
+ FPDataProcessing2SourceFixed = 0x1E200800,
+ FPDataProcessing2SourceFMask = 0x5F200C00,
+ FPDataProcessing2SourceMask = 0xFFE0FC00,
+ FMUL = FPDataProcessing2SourceFixed | 0x00000000,
+ FMUL_h = FMUL | FP16,
+ FMUL_s = FMUL,
+ FMUL_d = FMUL | FP64,
+ FDIV = FPDataProcessing2SourceFixed | 0x00001000,
+ FDIV_h = FDIV | FP16,
+ FDIV_s = FDIV,
+ FDIV_d = FDIV | FP64,
+ FADD = FPDataProcessing2SourceFixed | 0x00002000,
+ FADD_h = FADD | FP16,
+ FADD_s = FADD,
+ FADD_d = FADD | FP64,
+ FSUB = FPDataProcessing2SourceFixed | 0x00003000,
+ FSUB_h = FSUB | FP16,
+ FSUB_s = FSUB,
+ FSUB_d = FSUB | FP64,
+ FMAX = FPDataProcessing2SourceFixed | 0x00004000,
+ FMAX_h = FMAX | FP16,
+ FMAX_s = FMAX,
+ FMAX_d = FMAX | FP64,
+ FMIN = FPDataProcessing2SourceFixed | 0x00005000,
+ FMIN_h = FMIN | FP16,
+ FMIN_s = FMIN,
+ FMIN_d = FMIN | FP64,
+ FMAXNM = FPDataProcessing2SourceFixed | 0x00006000,
+ FMAXNM_h = FMAXNM | FP16,
+ FMAXNM_s = FMAXNM,
+ FMAXNM_d = FMAXNM | FP64,
+ FMINNM = FPDataProcessing2SourceFixed | 0x00007000,
+ FMINNM_h = FMINNM | FP16,
+ FMINNM_s = FMINNM,
+ FMINNM_d = FMINNM | FP64,
+ FNMUL = FPDataProcessing2SourceFixed | 0x00008000,
+ FNMUL_h = FNMUL | FP16,
+ FNMUL_s = FNMUL,
+ FNMUL_d = FNMUL | FP64
+};
+
+// Floating point data processing 3 source.
+enum FPDataProcessing3SourceOp : uint32_t {
+ FPDataProcessing3SourceFixed = 0x1F000000,
+ FPDataProcessing3SourceFMask = 0x5F000000,
+ FPDataProcessing3SourceMask = 0xFFE08000,
+ FMADD_h = FPDataProcessing3SourceFixed | 0x00C00000,
+ FMSUB_h = FPDataProcessing3SourceFixed | 0x00C08000,
+ FNMADD_h = FPDataProcessing3SourceFixed | 0x00E00000,
+ FNMSUB_h = FPDataProcessing3SourceFixed | 0x00E08000,
+ FMADD_s = FPDataProcessing3SourceFixed | 0x00000000,
+ FMSUB_s = FPDataProcessing3SourceFixed | 0x00008000,
+ FNMADD_s = FPDataProcessing3SourceFixed | 0x00200000,
+ FNMSUB_s = FPDataProcessing3SourceFixed | 0x00208000,
+ FMADD_d = FPDataProcessing3SourceFixed | 0x00400000,
+ FMSUB_d = FPDataProcessing3SourceFixed | 0x00408000,
+ FNMADD_d = FPDataProcessing3SourceFixed | 0x00600000,
+ FNMSUB_d = FPDataProcessing3SourceFixed | 0x00608000
+};
+
+// Conversion between floating point and integer.
+enum FPIntegerConvertOp : uint32_t {
+ FPIntegerConvertFixed = 0x1E200000,
+ FPIntegerConvertFMask = 0x5F20FC00,
+ FPIntegerConvertMask = 0xFFFFFC00,
+ FCVTNS = FPIntegerConvertFixed | 0x00000000,
+ FCVTNS_wh = FCVTNS | FP16,
+ FCVTNS_xh = FCVTNS | SixtyFourBits | FP16,
+ FCVTNS_ws = FCVTNS,
+ FCVTNS_xs = FCVTNS | SixtyFourBits,
+ FCVTNS_wd = FCVTNS | FP64,
+ FCVTNS_xd = FCVTNS | SixtyFourBits | FP64,
+ FCVTNU = FPIntegerConvertFixed | 0x00010000,
+ FCVTNU_wh = FCVTNU | FP16,
+ FCVTNU_xh = FCVTNU | SixtyFourBits | FP16,
+ FCVTNU_ws = FCVTNU,
+ FCVTNU_xs = FCVTNU | SixtyFourBits,
+ FCVTNU_wd = FCVTNU | FP64,
+ FCVTNU_xd = FCVTNU | SixtyFourBits | FP64,
+ FCVTPS = FPIntegerConvertFixed | 0x00080000,
+ FCVTPS_wh = FCVTPS | FP16,
+ FCVTPS_xh = FCVTPS | SixtyFourBits | FP16,
+ FCVTPS_ws = FCVTPS,
+ FCVTPS_xs = FCVTPS | SixtyFourBits,
+ FCVTPS_wd = FCVTPS | FP64,
+ FCVTPS_xd = FCVTPS | SixtyFourBits | FP64,
+ FCVTPU = FPIntegerConvertFixed | 0x00090000,
+ FCVTPU_wh = FCVTPU | FP16,
+ FCVTPU_xh = FCVTPU | SixtyFourBits | FP16,
+ FCVTPU_ws = FCVTPU,
+ FCVTPU_xs = FCVTPU | SixtyFourBits,
+ FCVTPU_wd = FCVTPU | FP64,
+ FCVTPU_xd = FCVTPU | SixtyFourBits | FP64,
+ FCVTMS = FPIntegerConvertFixed | 0x00100000,
+ FCVTMS_wh = FCVTMS | FP16,
+ FCVTMS_xh = FCVTMS | SixtyFourBits | FP16,
+ FCVTMS_ws = FCVTMS,
+ FCVTMS_xs = FCVTMS | SixtyFourBits,
+ FCVTMS_wd = FCVTMS | FP64,
+ FCVTMS_xd = FCVTMS | SixtyFourBits | FP64,
+ FCVTMU = FPIntegerConvertFixed | 0x00110000,
+ FCVTMU_wh = FCVTMU | FP16,
+ FCVTMU_xh = FCVTMU | SixtyFourBits | FP16,
+ FCVTMU_ws = FCVTMU,
+ FCVTMU_xs = FCVTMU | SixtyFourBits,
+ FCVTMU_wd = FCVTMU | FP64,
+ FCVTMU_xd = FCVTMU | SixtyFourBits | FP64,
+ FCVTZS = FPIntegerConvertFixed | 0x00180000,
+ FCVTZS_wh = FCVTZS | FP16,
+ FCVTZS_xh = FCVTZS | SixtyFourBits | FP16,
+ FCVTZS_ws = FCVTZS,
+ FCVTZS_xs = FCVTZS | SixtyFourBits,
+ FCVTZS_wd = FCVTZS | FP64,
+ FCVTZS_xd = FCVTZS | SixtyFourBits | FP64,
+ FCVTZU = FPIntegerConvertFixed | 0x00190000,
+ FCVTZU_wh = FCVTZU | FP16,
+ FCVTZU_xh = FCVTZU | SixtyFourBits | FP16,
+ FCVTZU_ws = FCVTZU,
+ FCVTZU_xs = FCVTZU | SixtyFourBits,
+ FCVTZU_wd = FCVTZU | FP64,
+ FCVTZU_xd = FCVTZU | SixtyFourBits | FP64,
+ SCVTF = FPIntegerConvertFixed | 0x00020000,
+ SCVTF_hw = SCVTF | FP16,
+ SCVTF_hx = SCVTF | SixtyFourBits | FP16,
+ SCVTF_sw = SCVTF,
+ SCVTF_sx = SCVTF | SixtyFourBits,
+ SCVTF_dw = SCVTF | FP64,
+ SCVTF_dx = SCVTF | SixtyFourBits | FP64,
+ UCVTF = FPIntegerConvertFixed | 0x00030000,
+ UCVTF_hw = UCVTF | FP16,
+ UCVTF_hx = UCVTF | SixtyFourBits | FP16,
+ UCVTF_sw = UCVTF,
+ UCVTF_sx = UCVTF | SixtyFourBits,
+ UCVTF_dw = UCVTF | FP64,
+ UCVTF_dx = UCVTF | SixtyFourBits | FP64,
+ FCVTAS = FPIntegerConvertFixed | 0x00040000,
+ FCVTAS_wh = FCVTAS | FP16,
+ FCVTAS_xh = FCVTAS | SixtyFourBits | FP16,
+ FCVTAS_ws = FCVTAS,
+ FCVTAS_xs = FCVTAS | SixtyFourBits,
+ FCVTAS_wd = FCVTAS | FP64,
+ FCVTAS_xd = FCVTAS | SixtyFourBits | FP64,
+ FCVTAU = FPIntegerConvertFixed | 0x00050000,
+ FCVTAU_wh = FCVTAU | FP16,
+ FCVTAU_xh = FCVTAU | SixtyFourBits | FP16,
+ FCVTAU_ws = FCVTAU,
+ FCVTAU_xs = FCVTAU | SixtyFourBits,
+ FCVTAU_wd = FCVTAU | FP64,
+ FCVTAU_xd = FCVTAU | SixtyFourBits | FP64,
+ FMOV_wh = FPIntegerConvertFixed | 0x00060000 | FP16,
+ FMOV_hw = FPIntegerConvertFixed | 0x00070000 | FP16,
+ FMOV_xh = FMOV_wh | SixtyFourBits,
+ FMOV_hx = FMOV_hw | SixtyFourBits,
+ FMOV_ws = FPIntegerConvertFixed | 0x00060000,
+ FMOV_sw = FPIntegerConvertFixed | 0x00070000,
+ FMOV_xd = FMOV_ws | SixtyFourBits | FP64,
+ FMOV_dx = FMOV_sw | SixtyFourBits | FP64,
+ FMOV_d1_x = FPIntegerConvertFixed | SixtyFourBits | 0x008F0000,
+ FMOV_x_d1 = FPIntegerConvertFixed | SixtyFourBits | 0x008E0000,
+ FJCVTZS = FPIntegerConvertFixed | FP64 | 0x001E0000
+};
+
+// Conversion between fixed point and floating point.
+enum FPFixedPointConvertOp : uint32_t {
+ FPFixedPointConvertFixed = 0x1E000000,
+ FPFixedPointConvertFMask = 0x5F200000,
+ FPFixedPointConvertMask = 0xFFFF0000,
+ FCVTZS_fixed = FPFixedPointConvertFixed | 0x00180000,
+ FCVTZS_wh_fixed = FCVTZS_fixed | FP16,
+ FCVTZS_xh_fixed = FCVTZS_fixed | SixtyFourBits | FP16,
+ FCVTZS_ws_fixed = FCVTZS_fixed,
+ FCVTZS_xs_fixed = FCVTZS_fixed | SixtyFourBits,
+ FCVTZS_wd_fixed = FCVTZS_fixed | FP64,
+ FCVTZS_xd_fixed = FCVTZS_fixed | SixtyFourBits | FP64,
+ FCVTZU_fixed = FPFixedPointConvertFixed | 0x00190000,
+ FCVTZU_wh_fixed = FCVTZU_fixed | FP16,
+ FCVTZU_xh_fixed = FCVTZU_fixed | SixtyFourBits | FP16,
+ FCVTZU_ws_fixed = FCVTZU_fixed,
+ FCVTZU_xs_fixed = FCVTZU_fixed | SixtyFourBits,
+ FCVTZU_wd_fixed = FCVTZU_fixed | FP64,
+ FCVTZU_xd_fixed = FCVTZU_fixed | SixtyFourBits | FP64,
+ SCVTF_fixed = FPFixedPointConvertFixed | 0x00020000,
+ SCVTF_hw_fixed = SCVTF_fixed | FP16,
+ SCVTF_hx_fixed = SCVTF_fixed | SixtyFourBits | FP16,
+ SCVTF_sw_fixed = SCVTF_fixed,
+ SCVTF_sx_fixed = SCVTF_fixed | SixtyFourBits,
+ SCVTF_dw_fixed = SCVTF_fixed | FP64,
+ SCVTF_dx_fixed = SCVTF_fixed | SixtyFourBits | FP64,
+ UCVTF_fixed = FPFixedPointConvertFixed | 0x00030000,
+ UCVTF_hw_fixed = UCVTF_fixed | FP16,
+ UCVTF_hx_fixed = UCVTF_fixed | SixtyFourBits | FP16,
+ UCVTF_sw_fixed = UCVTF_fixed,
+ UCVTF_sx_fixed = UCVTF_fixed | SixtyFourBits,
+ UCVTF_dw_fixed = UCVTF_fixed | FP64,
+ UCVTF_dx_fixed = UCVTF_fixed | SixtyFourBits | FP64
+};
+
+// Crypto - two register SHA.
+enum Crypto2RegSHAOp : uint32_t {
+ Crypto2RegSHAFixed = 0x5E280800,
+ Crypto2RegSHAFMask = 0xFF3E0C00
+};
+
+// Crypto - three register SHA.
+enum Crypto3RegSHAOp : uint32_t {
+ Crypto3RegSHAFixed = 0x5E000000,
+ Crypto3RegSHAFMask = 0xFF208C00
+};
+
+// Crypto - AES.
+enum CryptoAESOp : uint32_t {
+ CryptoAESFixed = 0x4E280800,
+ CryptoAESFMask = 0xFF3E0C00
+};
+
+// NEON instructions with two register operands.
+enum NEON2RegMiscOp : uint32_t {
+ NEON2RegMiscFixed = 0x0E200800,
+ NEON2RegMiscFMask = 0x9F3E0C00,
+ NEON2RegMiscMask = 0xBF3FFC00,
+ NEON2RegMiscUBit = 0x20000000,
+ NEON_REV64 = NEON2RegMiscFixed | 0x00000000,
+ NEON_REV32 = NEON2RegMiscFixed | 0x20000000,
+ NEON_REV16 = NEON2RegMiscFixed | 0x00001000,
+ NEON_SADDLP = NEON2RegMiscFixed | 0x00002000,
+ NEON_UADDLP = NEON_SADDLP | NEON2RegMiscUBit,
+ NEON_SUQADD = NEON2RegMiscFixed | 0x00003000,
+ NEON_USQADD = NEON_SUQADD | NEON2RegMiscUBit,
+ NEON_CLS = NEON2RegMiscFixed | 0x00004000,
+ NEON_CLZ = NEON2RegMiscFixed | 0x20004000,
+ NEON_CNT = NEON2RegMiscFixed | 0x00005000,
+ NEON_RBIT_NOT = NEON2RegMiscFixed | 0x20005000,
+ NEON_SADALP = NEON2RegMiscFixed | 0x00006000,
+ NEON_UADALP = NEON_SADALP | NEON2RegMiscUBit,
+ NEON_SQABS = NEON2RegMiscFixed | 0x00007000,
+ NEON_SQNEG = NEON2RegMiscFixed | 0x20007000,
+ NEON_CMGT_zero = NEON2RegMiscFixed | 0x00008000,
+ NEON_CMGE_zero = NEON2RegMiscFixed | 0x20008000,
+ NEON_CMEQ_zero = NEON2RegMiscFixed | 0x00009000,
+ NEON_CMLE_zero = NEON2RegMiscFixed | 0x20009000,
+ NEON_CMLT_zero = NEON2RegMiscFixed | 0x0000A000,
+ NEON_ABS = NEON2RegMiscFixed | 0x0000B000,
+ NEON_NEG = NEON2RegMiscFixed | 0x2000B000,
+ NEON_XTN = NEON2RegMiscFixed | 0x00012000,
+ NEON_SQXTUN = NEON2RegMiscFixed | 0x20012000,
+ NEON_SHLL = NEON2RegMiscFixed | 0x20013000,
+ NEON_SQXTN = NEON2RegMiscFixed | 0x00014000,
+ NEON_UQXTN = NEON_SQXTN | NEON2RegMiscUBit,
+
+ NEON2RegMiscOpcode = 0x0001F000,
+ NEON_RBIT_NOT_opcode = NEON_RBIT_NOT & NEON2RegMiscOpcode,
+ NEON_NEG_opcode = NEON_NEG & NEON2RegMiscOpcode,
+ NEON_XTN_opcode = NEON_XTN & NEON2RegMiscOpcode,
+ NEON_UQXTN_opcode = NEON_UQXTN & NEON2RegMiscOpcode,
+
+ // These instructions use only one bit of the size field. The other bit is
+ // used to distinguish between instructions.
+ NEON2RegMiscFPMask = NEON2RegMiscMask | 0x00800000,
+ NEON_FABS = NEON2RegMiscFixed | 0x0080F000,
+ NEON_FNEG = NEON2RegMiscFixed | 0x2080F000,
+ NEON_FCVTN = NEON2RegMiscFixed | 0x00016000,
+ NEON_FCVTXN = NEON2RegMiscFixed | 0x20016000,
+ NEON_FCVTL = NEON2RegMiscFixed | 0x00017000,
+ NEON_FRINT32X = NEON2RegMiscFixed | 0x2001E000,
+ NEON_FRINT32Z = NEON2RegMiscFixed | 0x0001E000,
+ NEON_FRINT64X = NEON2RegMiscFixed | 0x2001F000,
+ NEON_FRINT64Z = NEON2RegMiscFixed | 0x0001F000,
+ NEON_FRINTN = NEON2RegMiscFixed | 0x00018000,
+ NEON_FRINTA = NEON2RegMiscFixed | 0x20018000,
+ NEON_FRINTP = NEON2RegMiscFixed | 0x00818000,
+ NEON_FRINTM = NEON2RegMiscFixed | 0x00019000,
+ NEON_FRINTX = NEON2RegMiscFixed | 0x20019000,
+ NEON_FRINTZ = NEON2RegMiscFixed | 0x00819000,
+ NEON_FRINTI = NEON2RegMiscFixed | 0x20819000,
+ NEON_FCVTNS = NEON2RegMiscFixed | 0x0001A000,
+ NEON_FCVTNU = NEON_FCVTNS | NEON2RegMiscUBit,
+ NEON_FCVTPS = NEON2RegMiscFixed | 0x0081A000,
+ NEON_FCVTPU = NEON_FCVTPS | NEON2RegMiscUBit,
+ NEON_FCVTMS = NEON2RegMiscFixed | 0x0001B000,
+ NEON_FCVTMU = NEON_FCVTMS | NEON2RegMiscUBit,
+ NEON_FCVTZS = NEON2RegMiscFixed | 0x0081B000,
+ NEON_FCVTZU = NEON_FCVTZS | NEON2RegMiscUBit,
+ NEON_FCVTAS = NEON2RegMiscFixed | 0x0001C000,
+ NEON_FCVTAU = NEON_FCVTAS | NEON2RegMiscUBit,
+ NEON_FSQRT = NEON2RegMiscFixed | 0x2081F000,
+ NEON_SCVTF = NEON2RegMiscFixed | 0x0001D000,
+ NEON_UCVTF = NEON_SCVTF | NEON2RegMiscUBit,
+ NEON_URSQRTE = NEON2RegMiscFixed | 0x2081C000,
+ NEON_URECPE = NEON2RegMiscFixed | 0x0081C000,
+ NEON_FRSQRTE = NEON2RegMiscFixed | 0x2081D000,
+ NEON_FRECPE = NEON2RegMiscFixed | 0x0081D000,
+ NEON_FCMGT_zero = NEON2RegMiscFixed | 0x0080C000,
+ NEON_FCMGE_zero = NEON2RegMiscFixed | 0x2080C000,
+ NEON_FCMEQ_zero = NEON2RegMiscFixed | 0x0080D000,
+ NEON_FCMLE_zero = NEON2RegMiscFixed | 0x2080D000,
+ NEON_FCMLT_zero = NEON2RegMiscFixed | 0x0080E000,
+
+ NEON_FCVTL_opcode = NEON_FCVTL & NEON2RegMiscOpcode,
+ NEON_FCVTN_opcode = NEON_FCVTN & NEON2RegMiscOpcode
+};
+
+// NEON instructions with two register operands (FP16).
+enum NEON2RegMiscFP16Op : uint32_t {
+ NEON2RegMiscFP16Fixed = 0x0E780800,
+ NEON2RegMiscFP16FMask = 0x9F7E0C00,
+ NEON2RegMiscFP16Mask = 0xBFFFFC00,
+ NEON_FRINTN_H = NEON2RegMiscFP16Fixed | 0x00018000,
+ NEON_FRINTM_H = NEON2RegMiscFP16Fixed | 0x00019000,
+ NEON_FCVTNS_H = NEON2RegMiscFP16Fixed | 0x0001A000,
+ NEON_FCVTMS_H = NEON2RegMiscFP16Fixed | 0x0001B000,
+ NEON_FCVTAS_H = NEON2RegMiscFP16Fixed | 0x0001C000,
+ NEON_SCVTF_H = NEON2RegMiscFP16Fixed | 0x0001D000,
+ NEON_FCMGT_H_zero = NEON2RegMiscFP16Fixed | 0x0080C000,
+ NEON_FCMEQ_H_zero = NEON2RegMiscFP16Fixed | 0x0080D000,
+ NEON_FCMLT_H_zero = NEON2RegMiscFP16Fixed | 0x0080E000,
+ NEON_FABS_H = NEON2RegMiscFP16Fixed | 0x0080F000,
+ NEON_FRINTP_H = NEON2RegMiscFP16Fixed | 0x00818000,
+ NEON_FRINTZ_H = NEON2RegMiscFP16Fixed | 0x00819000,
+ NEON_FCVTPS_H = NEON2RegMiscFP16Fixed | 0x0081A000,
+ NEON_FCVTZS_H = NEON2RegMiscFP16Fixed | 0x0081B000,
+ NEON_FRECPE_H = NEON2RegMiscFP16Fixed | 0x0081D000,
+ NEON_FRINTA_H = NEON2RegMiscFP16Fixed | 0x20018000,
+ NEON_FRINTX_H = NEON2RegMiscFP16Fixed | 0x20019000,
+ NEON_FCVTNU_H = NEON2RegMiscFP16Fixed | 0x2001A000,
+ NEON_FCVTMU_H = NEON2RegMiscFP16Fixed | 0x2001B000,
+ NEON_FCVTAU_H = NEON2RegMiscFP16Fixed | 0x2001C000,
+ NEON_UCVTF_H = NEON2RegMiscFP16Fixed | 0x2001D000,
+ NEON_FCMGE_H_zero = NEON2RegMiscFP16Fixed | 0x2080C000,
+ NEON_FCMLE_H_zero = NEON2RegMiscFP16Fixed | 0x2080D000,
+ NEON_FNEG_H = NEON2RegMiscFP16Fixed | 0x2080F000,
+ NEON_FRINTI_H = NEON2RegMiscFP16Fixed | 0x20819000,
+ NEON_FCVTPU_H = NEON2RegMiscFP16Fixed | 0x2081A000,
+ NEON_FCVTZU_H = NEON2RegMiscFP16Fixed | 0x2081B000,
+ NEON_FRSQRTE_H = NEON2RegMiscFP16Fixed | 0x2081D000,
+ NEON_FSQRT_H = NEON2RegMiscFP16Fixed | 0x2081F000
+};
+
+// NEON instructions with three same-type operands.
+enum NEON3SameOp : uint32_t {
+ NEON3SameFixed = 0x0E200400,
+ NEON3SameFMask = 0x9F200400,
+ NEON3SameMask = 0xBF20FC00,
+ NEON3SameUBit = 0x20000000,
+ NEON_ADD = NEON3SameFixed | 0x00008000,
+ NEON_ADDP = NEON3SameFixed | 0x0000B800,
+ NEON_SHADD = NEON3SameFixed | 0x00000000,
+ NEON_SHSUB = NEON3SameFixed | 0x00002000,
+ NEON_SRHADD = NEON3SameFixed | 0x00001000,
+ NEON_CMEQ = NEON3SameFixed | NEON3SameUBit | 0x00008800,
+ NEON_CMGE = NEON3SameFixed | 0x00003800,
+ NEON_CMGT = NEON3SameFixed | 0x00003000,
+ NEON_CMHI = NEON3SameFixed | NEON3SameUBit | NEON_CMGT,
+ NEON_CMHS = NEON3SameFixed | NEON3SameUBit | NEON_CMGE,
+ NEON_CMTST = NEON3SameFixed | 0x00008800,
+ NEON_MLA = NEON3SameFixed | 0x00009000,
+ NEON_MLS = NEON3SameFixed | 0x20009000,
+ NEON_MUL = NEON3SameFixed | 0x00009800,
+ NEON_PMUL = NEON3SameFixed | 0x20009800,
+ NEON_SRSHL = NEON3SameFixed | 0x00005000,
+ NEON_SQSHL = NEON3SameFixed | 0x00004800,
+ NEON_SQRSHL = NEON3SameFixed | 0x00005800,
+ NEON_SSHL = NEON3SameFixed | 0x00004000,
+ NEON_SMAX = NEON3SameFixed | 0x00006000,
+ NEON_SMAXP = NEON3SameFixed | 0x0000A000,
+ NEON_SMIN = NEON3SameFixed | 0x00006800,
+ NEON_SMINP = NEON3SameFixed | 0x0000A800,
+ NEON_SABD = NEON3SameFixed | 0x00007000,
+ NEON_SABA = NEON3SameFixed | 0x00007800,
+ NEON_UABD = NEON3SameFixed | NEON3SameUBit | NEON_SABD,
+ NEON_UABA = NEON3SameFixed | NEON3SameUBit | NEON_SABA,
+ NEON_SQADD = NEON3SameFixed | 0x00000800,
+ NEON_SQSUB = NEON3SameFixed | 0x00002800,
+ NEON_SUB = NEON3SameFixed | NEON3SameUBit | 0x00008000,
+ NEON_UHADD = NEON3SameFixed | NEON3SameUBit | NEON_SHADD,
+ NEON_UHSUB = NEON3SameFixed | NEON3SameUBit | NEON_SHSUB,
+ NEON_URHADD = NEON3SameFixed | NEON3SameUBit | NEON_SRHADD,
+ NEON_UMAX = NEON3SameFixed | NEON3SameUBit | NEON_SMAX,
+ NEON_UMAXP = NEON3SameFixed | NEON3SameUBit | NEON_SMAXP,
+ NEON_UMIN = NEON3SameFixed | NEON3SameUBit | NEON_SMIN,
+ NEON_UMINP = NEON3SameFixed | NEON3SameUBit | NEON_SMINP,
+ NEON_URSHL = NEON3SameFixed | NEON3SameUBit | NEON_SRSHL,
+ NEON_UQADD = NEON3SameFixed | NEON3SameUBit | NEON_SQADD,
+ NEON_UQRSHL = NEON3SameFixed | NEON3SameUBit | NEON_SQRSHL,
+ NEON_UQSHL = NEON3SameFixed | NEON3SameUBit | NEON_SQSHL,
+ NEON_UQSUB = NEON3SameFixed | NEON3SameUBit | NEON_SQSUB,
+ NEON_USHL = NEON3SameFixed | NEON3SameUBit | NEON_SSHL,
+ NEON_SQDMULH = NEON3SameFixed | 0x0000B000,
+ NEON_SQRDMULH = NEON3SameFixed | 0x2000B000,
+
+ // NEON floating point instructions with three same-type operands.
+ NEON3SameFPFixed = NEON3SameFixed | 0x0000C000,
+ NEON3SameFPFMask = NEON3SameFMask | 0x0000C000,
+ NEON3SameFPMask = NEON3SameMask | 0x00800000,
+ NEON_FADD = NEON3SameFixed | 0x0000D000,
+ NEON_FSUB = NEON3SameFixed | 0x0080D000,
+ NEON_FMUL = NEON3SameFixed | 0x2000D800,
+ NEON_FDIV = NEON3SameFixed | 0x2000F800,
+ NEON_FMAX = NEON3SameFixed | 0x0000F000,
+ NEON_FMAXNM = NEON3SameFixed | 0x0000C000,
+ NEON_FMAXP = NEON3SameFixed | 0x2000F000,
+ NEON_FMAXNMP = NEON3SameFixed | 0x2000C000,
+ NEON_FMIN = NEON3SameFixed | 0x0080F000,
+ NEON_FMINNM = NEON3SameFixed | 0x0080C000,
+ NEON_FMINP = NEON3SameFixed | 0x2080F000,
+ NEON_FMINNMP = NEON3SameFixed | 0x2080C000,
+ NEON_FMLA = NEON3SameFixed | 0x0000C800,
+ NEON_FMLS = NEON3SameFixed | 0x0080C800,
+ NEON_FMULX = NEON3SameFixed | 0x0000D800,
+ NEON_FRECPS = NEON3SameFixed | 0x0000F800,
+ NEON_FRSQRTS = NEON3SameFixed | 0x0080F800,
+ NEON_FABD = NEON3SameFixed | 0x2080D000,
+ NEON_FADDP = NEON3SameFixed | 0x2000D000,
+ NEON_FCMEQ = NEON3SameFixed | 0x0000E000,
+ NEON_FCMGE = NEON3SameFixed | 0x2000E000,
+ NEON_FCMGT = NEON3SameFixed | 0x2080E000,
+ NEON_FACGE = NEON3SameFixed | 0x2000E800,
+ NEON_FACGT = NEON3SameFixed | 0x2080E800,
+
+ // NEON logical instructions with three same-type operands.
+ NEON3SameLogicalFixed = NEON3SameFixed | 0x00001800,
+ NEON3SameLogicalFMask = NEON3SameFMask | 0x0000F800,
+ NEON3SameLogicalMask = 0xBFE0FC00,
+ NEON3SameLogicalFormatMask = NEON_Q,
+ NEON_AND = NEON3SameLogicalFixed | 0x00000000,
+ NEON_ORR = NEON3SameLogicalFixed | 0x00A00000,
+ NEON_ORN = NEON3SameLogicalFixed | 0x00C00000,
+ NEON_EOR = NEON3SameLogicalFixed | 0x20000000,
+ NEON_BIC = NEON3SameLogicalFixed | 0x00400000,
+ NEON_BIF = NEON3SameLogicalFixed | 0x20C00000,
+ NEON_BIT = NEON3SameLogicalFixed | 0x20800000,
+ NEON_BSL = NEON3SameLogicalFixed | 0x20400000,
+
+ // FHM (FMLAL-like) instructions have an oddball encoding scheme under 3Same.
+ NEON3SameFHMMask = 0xBFE0FC00, // U size opcode
+ NEON_FMLAL = NEON3SameFixed | 0x0000E800, // 0 00 11101
+ NEON_FMLAL2 = NEON3SameFixed | 0x2000C800, // 1 00 11001
+ NEON_FMLSL = NEON3SameFixed | 0x0080E800, // 0 10 11101
+ NEON_FMLSL2 = NEON3SameFixed | 0x2080C800 // 1 10 11001
+};
+
+enum NEON3SameFP16 : uint32_t {
+ NEON3SameFP16Fixed = 0x0E400400,
+ NEON3SameFP16FMask = 0x9F60C400,
+ NEON3SameFP16Mask = 0xBFE0FC00,
+ NEON_FMAXNM_H = NEON3SameFP16Fixed | 0x00000000,
+ NEON_FMLA_H = NEON3SameFP16Fixed | 0x00000800,
+ NEON_FADD_H = NEON3SameFP16Fixed | 0x00001000,
+ NEON_FMULX_H = NEON3SameFP16Fixed | 0x00001800,
+ NEON_FCMEQ_H = NEON3SameFP16Fixed | 0x00002000,
+ NEON_FMAX_H = NEON3SameFP16Fixed | 0x00003000,
+ NEON_FRECPS_H = NEON3SameFP16Fixed | 0x00003800,
+ NEON_FMINNM_H = NEON3SameFP16Fixed | 0x00800000,
+ NEON_FMLS_H = NEON3SameFP16Fixed | 0x00800800,
+ NEON_FSUB_H = NEON3SameFP16Fixed | 0x00801000,
+ NEON_FMIN_H = NEON3SameFP16Fixed | 0x00803000,
+ NEON_FRSQRTS_H = NEON3SameFP16Fixed | 0x00803800,
+ NEON_FMAXNMP_H = NEON3SameFP16Fixed | 0x20000000,
+ NEON_FADDP_H = NEON3SameFP16Fixed | 0x20001000,
+ NEON_FMUL_H = NEON3SameFP16Fixed | 0x20001800,
+ NEON_FCMGE_H = NEON3SameFP16Fixed | 0x20002000,
+ NEON_FACGE_H = NEON3SameFP16Fixed | 0x20002800,
+ NEON_FMAXP_H = NEON3SameFP16Fixed | 0x20003000,
+ NEON_FDIV_H = NEON3SameFP16Fixed | 0x20003800,
+ NEON_FMINNMP_H = NEON3SameFP16Fixed | 0x20800000,
+ NEON_FABD_H = NEON3SameFP16Fixed | 0x20801000,
+ NEON_FCMGT_H = NEON3SameFP16Fixed | 0x20802000,
+ NEON_FACGT_H = NEON3SameFP16Fixed | 0x20802800,
+ NEON_FMINP_H = NEON3SameFP16Fixed | 0x20803000
+};
+
+// 'Extra' NEON instructions with three same-type operands.
+enum NEON3SameExtraOp : uint32_t {
+ NEON3SameExtraFixed = 0x0E008400,
+ NEON3SameExtraUBit = 0x20000000,
+ NEON3SameExtraFMask = 0x9E208400,
+ NEON3SameExtraMask = 0xBE20FC00,
+ NEON_SQRDMLAH = NEON3SameExtraFixed | NEON3SameExtraUBit,
+ NEON_SQRDMLSH = NEON3SameExtraFixed | NEON3SameExtraUBit | 0x00000800,
+ NEON_SDOT = NEON3SameExtraFixed | 0x00001000,
+ NEON_UDOT = NEON3SameExtraFixed | NEON3SameExtraUBit | 0x00001000,
+
+ /* v8.3 Complex Numbers */
+ NEON3SameExtraFCFixed = 0x2E00C400,
+ NEON3SameExtraFCFMask = 0xBF20C400,
+ // FCMLA fixes opcode<3:2>, and uses opcode<1:0> to encode <rotate>.
+ NEON3SameExtraFCMLAMask = NEON3SameExtraFCFMask | 0x00006000,
+ NEON_FCMLA = NEON3SameExtraFCFixed,
+ // FCADD fixes opcode<3:2, 0>, and uses opcode<1> to encode <rotate>.
+ NEON3SameExtraFCADDMask = NEON3SameExtraFCFMask | 0x00006800,
+ NEON_FCADD = NEON3SameExtraFCFixed | 0x00002000
+ // Other encodings under NEON3SameExtraFCFMask are UNALLOCATED.
+};
+
+// NEON instructions with three different-type operands.
+enum NEON3DifferentOp : uint32_t {
+ NEON3DifferentFixed = 0x0E200000,
+ NEON3DifferentFMask = 0x9F200C00,
+ NEON3DifferentMask = 0xFF20FC00,
+ NEON_ADDHN = NEON3DifferentFixed | 0x00004000,
+ NEON_ADDHN2 = NEON_ADDHN | NEON_Q,
+ NEON_PMULL = NEON3DifferentFixed | 0x0000E000,
+ NEON_PMULL2 = NEON_PMULL | NEON_Q,
+ NEON_RADDHN = NEON3DifferentFixed | 0x20004000,
+ NEON_RADDHN2 = NEON_RADDHN | NEON_Q,
+ NEON_RSUBHN = NEON3DifferentFixed | 0x20006000,
+ NEON_RSUBHN2 = NEON_RSUBHN | NEON_Q,
+ NEON_SABAL = NEON3DifferentFixed | 0x00005000,
+ NEON_SABAL2 = NEON_SABAL | NEON_Q,
+ NEON_SABDL = NEON3DifferentFixed | 0x00007000,
+ NEON_SABDL2 = NEON_SABDL | NEON_Q,
+ NEON_SADDL = NEON3DifferentFixed | 0x00000000,
+ NEON_SADDL2 = NEON_SADDL | NEON_Q,
+ NEON_SADDW = NEON3DifferentFixed | 0x00001000,
+ NEON_SADDW2 = NEON_SADDW | NEON_Q,
+ NEON_SMLAL = NEON3DifferentFixed | 0x00008000,
+ NEON_SMLAL2 = NEON_SMLAL | NEON_Q,
+ NEON_SMLSL = NEON3DifferentFixed | 0x0000A000,
+ NEON_SMLSL2 = NEON_SMLSL | NEON_Q,
+ NEON_SMULL = NEON3DifferentFixed | 0x0000C000,
+ NEON_SMULL2 = NEON_SMULL | NEON_Q,
+ NEON_SSUBL = NEON3DifferentFixed | 0x00002000,
+ NEON_SSUBL2 = NEON_SSUBL | NEON_Q,
+ NEON_SSUBW = NEON3DifferentFixed | 0x00003000,
+ NEON_SSUBW2 = NEON_SSUBW | NEON_Q,
+ NEON_SQDMLAL = NEON3DifferentFixed | 0x00009000,
+ NEON_SQDMLAL2 = NEON_SQDMLAL | NEON_Q,
+ NEON_SQDMLSL = NEON3DifferentFixed | 0x0000B000,
+ NEON_SQDMLSL2 = NEON_SQDMLSL | NEON_Q,
+ NEON_SQDMULL = NEON3DifferentFixed | 0x0000D000,
+ NEON_SQDMULL2 = NEON_SQDMULL | NEON_Q,
+ NEON_SUBHN = NEON3DifferentFixed | 0x00006000,
+ NEON_SUBHN2 = NEON_SUBHN | NEON_Q,
+ NEON_UABAL = NEON_SABAL | NEON3SameUBit,
+ NEON_UABAL2 = NEON_UABAL | NEON_Q,
+ NEON_UABDL = NEON_SABDL | NEON3SameUBit,
+ NEON_UABDL2 = NEON_UABDL | NEON_Q,
+ NEON_UADDL = NEON_SADDL | NEON3SameUBit,
+ NEON_UADDL2 = NEON_UADDL | NEON_Q,
+ NEON_UADDW = NEON_SADDW | NEON3SameUBit,
+ NEON_UADDW2 = NEON_UADDW | NEON_Q,
+ NEON_UMLAL = NEON_SMLAL | NEON3SameUBit,
+ NEON_UMLAL2 = NEON_UMLAL | NEON_Q,
+ NEON_UMLSL = NEON_SMLSL | NEON3SameUBit,
+ NEON_UMLSL2 = NEON_UMLSL | NEON_Q,
+ NEON_UMULL = NEON_SMULL | NEON3SameUBit,
+ NEON_UMULL2 = NEON_UMULL | NEON_Q,
+ NEON_USUBL = NEON_SSUBL | NEON3SameUBit,
+ NEON_USUBL2 = NEON_USUBL | NEON_Q,
+ NEON_USUBW = NEON_SSUBW | NEON3SameUBit,
+ NEON_USUBW2 = NEON_USUBW | NEON_Q
+};
+
+// NEON instructions operating across vectors.
+enum NEONAcrossLanesOp : uint32_t {
+ NEONAcrossLanesFixed = 0x0E300800,
+ NEONAcrossLanesFMask = 0x9F3E0C00,
+ NEONAcrossLanesMask = 0xBF3FFC00,
+ NEON_ADDV = NEONAcrossLanesFixed | 0x0001B000,
+ NEON_SADDLV = NEONAcrossLanesFixed | 0x00003000,
+ NEON_UADDLV = NEONAcrossLanesFixed | 0x20003000,
+ NEON_SMAXV = NEONAcrossLanesFixed | 0x0000A000,
+ NEON_SMINV = NEONAcrossLanesFixed | 0x0001A000,
+ NEON_UMAXV = NEONAcrossLanesFixed | 0x2000A000,
+ NEON_UMINV = NEONAcrossLanesFixed | 0x2001A000,
+
+ NEONAcrossLanesFP16Fixed = NEONAcrossLanesFixed | 0x0000C000,
+ NEONAcrossLanesFP16FMask = NEONAcrossLanesFMask | 0x2000C000,
+ NEONAcrossLanesFP16Mask = NEONAcrossLanesMask | 0x20800000,
+ NEON_FMAXNMV_H = NEONAcrossLanesFP16Fixed | 0x00000000,
+ NEON_FMAXV_H = NEONAcrossLanesFP16Fixed | 0x00003000,
+ NEON_FMINNMV_H = NEONAcrossLanesFP16Fixed | 0x00800000,
+ NEON_FMINV_H = NEONAcrossLanesFP16Fixed | 0x00803000,
+
+ // NEON floating point across instructions.
+ NEONAcrossLanesFPFixed = NEONAcrossLanesFixed | 0x2000C000,
+ NEONAcrossLanesFPFMask = NEONAcrossLanesFMask | 0x2000C000,
+ NEONAcrossLanesFPMask = NEONAcrossLanesMask | 0x20800000,
+
+ NEON_FMAXV = NEONAcrossLanesFPFixed | 0x2000F000,
+ NEON_FMINV = NEONAcrossLanesFPFixed | 0x2080F000,
+ NEON_FMAXNMV = NEONAcrossLanesFPFixed | 0x2000C000,
+ NEON_FMINNMV = NEONAcrossLanesFPFixed | 0x2080C000
+};
+
+// NEON instructions with indexed element operand.
+enum NEONByIndexedElementOp : uint32_t {
+ NEONByIndexedElementFixed = 0x0F000000,
+ NEONByIndexedElementFMask = 0x9F000400,
+ NEONByIndexedElementMask = 0xBF00F400,
+ NEON_MUL_byelement = NEONByIndexedElementFixed | 0x00008000,
+ NEON_MLA_byelement = NEONByIndexedElementFixed | 0x20000000,
+ NEON_MLS_byelement = NEONByIndexedElementFixed | 0x20004000,
+ NEON_SMULL_byelement = NEONByIndexedElementFixed | 0x0000A000,
+ NEON_SMLAL_byelement = NEONByIndexedElementFixed | 0x00002000,
+ NEON_SMLSL_byelement = NEONByIndexedElementFixed | 0x00006000,
+ NEON_UMULL_byelement = NEONByIndexedElementFixed | 0x2000A000,
+ NEON_UMLAL_byelement = NEONByIndexedElementFixed | 0x20002000,
+ NEON_UMLSL_byelement = NEONByIndexedElementFixed | 0x20006000,
+ NEON_SQDMULL_byelement = NEONByIndexedElementFixed | 0x0000B000,
+ NEON_SQDMLAL_byelement = NEONByIndexedElementFixed | 0x00003000,
+ NEON_SQDMLSL_byelement = NEONByIndexedElementFixed | 0x00007000,
+ NEON_SQDMULH_byelement = NEONByIndexedElementFixed | 0x0000C000,
+ NEON_SQRDMULH_byelement = NEONByIndexedElementFixed | 0x0000D000,
+ NEON_SDOT_byelement = NEONByIndexedElementFixed | 0x0000E000,
+ NEON_SQRDMLAH_byelement = NEONByIndexedElementFixed | 0x2000D000,
+ NEON_UDOT_byelement = NEONByIndexedElementFixed | 0x2000E000,
+ NEON_SQRDMLSH_byelement = NEONByIndexedElementFixed | 0x2000F000,
+
+ NEON_FMLA_H_byelement = NEONByIndexedElementFixed | 0x00001000,
+ NEON_FMLS_H_byelement = NEONByIndexedElementFixed | 0x00005000,
+ NEON_FMUL_H_byelement = NEONByIndexedElementFixed | 0x00009000,
+ NEON_FMULX_H_byelement = NEONByIndexedElementFixed | 0x20009000,
+
+ // Floating point instructions.
+ NEONByIndexedElementFPFixed = NEONByIndexedElementFixed | 0x00800000,
+ NEONByIndexedElementFPMask = NEONByIndexedElementMask | 0x00800000,
+ NEON_FMLA_byelement = NEONByIndexedElementFPFixed | 0x00001000,
+ NEON_FMLS_byelement = NEONByIndexedElementFPFixed | 0x00005000,
+ NEON_FMUL_byelement = NEONByIndexedElementFPFixed | 0x00009000,
+ NEON_FMULX_byelement = NEONByIndexedElementFPFixed | 0x20009000,
+
+ // FMLAL-like instructions.
+ // For all cases: U = x, size = 10, opcode = xx00
+ NEONByIndexedElementFPLongFixed = NEONByIndexedElementFixed | 0x00800000,
+ NEONByIndexedElementFPLongFMask = NEONByIndexedElementFMask | 0x00C03000,
+ NEONByIndexedElementFPLongMask = 0xBFC0F400,
+ NEON_FMLAL_H_byelement = NEONByIndexedElementFixed | 0x00800000,
+ NEON_FMLAL2_H_byelement = NEONByIndexedElementFixed | 0x20808000,
+ NEON_FMLSL_H_byelement = NEONByIndexedElementFixed | 0x00804000,
+ NEON_FMLSL2_H_byelement = NEONByIndexedElementFixed | 0x2080C000,
+
+ // Complex instruction(s).
+ // This is necessary because the 'rot' encoding moves into the
+ // NEONByIndex..Mask space.
+ NEONByIndexedElementFPComplexMask = 0xBF009400,
+ NEON_FCMLA_byelement = NEONByIndexedElementFixed | 0x20001000
+};
+
+// NEON register copy.
+enum NEONCopyOp : uint32_t {
+ NEONCopyFixed = 0x0E000400,
+ NEONCopyFMask = 0x9FE08400,
+ NEONCopyMask = 0x3FE08400,
+ NEONCopyInsElementMask = NEONCopyMask | 0x40000000,
+ NEONCopyInsGeneralMask = NEONCopyMask | 0x40007800,
+ NEONCopyDupElementMask = NEONCopyMask | 0x20007800,
+ NEONCopyDupGeneralMask = NEONCopyDupElementMask,
+ NEONCopyUmovMask = NEONCopyMask | 0x20007800,
+ NEONCopySmovMask = NEONCopyMask | 0x20007800,
+ NEON_INS_ELEMENT = NEONCopyFixed | 0x60000000,
+ NEON_INS_GENERAL = NEONCopyFixed | 0x40001800,
+ NEON_DUP_ELEMENT = NEONCopyFixed | 0x00000000,
+ NEON_DUP_GENERAL = NEONCopyFixed | 0x00000800,
+ NEON_SMOV = NEONCopyFixed | 0x00002800,
+ NEON_UMOV = NEONCopyFixed | 0x00003800
+};
+
+// NEON extract.
+enum NEONExtractOp : uint32_t {
+ NEONExtractFixed = 0x2E000000,
+ NEONExtractFMask = 0xBF208400,
+ NEONExtractMask = 0xBFE08400,
+ NEON_EXT = NEONExtractFixed | 0x00000000
+};
+
+enum NEONLoadStoreMultiOp : uint32_t {
+ NEONLoadStoreMultiL = 0x00400000,
+ NEONLoadStoreMulti1_1v = 0x00007000,
+ NEONLoadStoreMulti1_2v = 0x0000A000,
+ NEONLoadStoreMulti1_3v = 0x00006000,
+ NEONLoadStoreMulti1_4v = 0x00002000,
+ NEONLoadStoreMulti2 = 0x00008000,
+ NEONLoadStoreMulti3 = 0x00004000,
+ NEONLoadStoreMulti4 = 0x00000000
+};
+
+// NEON load/store multiple structures.
+enum NEONLoadStoreMultiStructOp : uint32_t {
+ NEONLoadStoreMultiStructFixed = 0x0C000000,
+ NEONLoadStoreMultiStructFMask = 0xBFBF0000,
+ NEONLoadStoreMultiStructMask = 0xBFFFF000,
+ NEONLoadStoreMultiStructStore = NEONLoadStoreMultiStructFixed,
+ NEONLoadStoreMultiStructLoad = NEONLoadStoreMultiStructFixed |
+ NEONLoadStoreMultiL,
+ NEON_LD1_1v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_1v,
+ NEON_LD1_2v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_2v,
+ NEON_LD1_3v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_3v,
+ NEON_LD1_4v = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti1_4v,
+ NEON_LD2 = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti2,
+ NEON_LD3 = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti3,
+ NEON_LD4 = NEONLoadStoreMultiStructLoad | NEONLoadStoreMulti4,
+ NEON_ST1_1v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_1v,
+ NEON_ST1_2v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_2v,
+ NEON_ST1_3v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_3v,
+ NEON_ST1_4v = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti1_4v,
+ NEON_ST2 = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti2,
+ NEON_ST3 = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti3,
+ NEON_ST4 = NEONLoadStoreMultiStructStore | NEONLoadStoreMulti4
+};
+
+// NEON load/store multiple structures with post-index addressing.
+enum NEONLoadStoreMultiStructPostIndexOp : uint32_t {
+ NEONLoadStoreMultiStructPostIndexFixed = 0x0C800000,
+ NEONLoadStoreMultiStructPostIndexFMask = 0xBFA00000,
+ NEONLoadStoreMultiStructPostIndexMask = 0xBFE0F000,
+ NEONLoadStoreMultiStructPostIndex = 0x00800000,
+ NEON_LD1_1v_post = NEON_LD1_1v | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD1_2v_post = NEON_LD1_2v | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD1_3v_post = NEON_LD1_3v | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD1_4v_post = NEON_LD1_4v | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD2_post = NEON_LD2 | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD3_post = NEON_LD3 | NEONLoadStoreMultiStructPostIndex,
+ NEON_LD4_post = NEON_LD4 | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST1_1v_post = NEON_ST1_1v | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST1_2v_post = NEON_ST1_2v | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST1_3v_post = NEON_ST1_3v | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST1_4v_post = NEON_ST1_4v | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST2_post = NEON_ST2 | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST3_post = NEON_ST3 | NEONLoadStoreMultiStructPostIndex,
+ NEON_ST4_post = NEON_ST4 | NEONLoadStoreMultiStructPostIndex
+};
+
+enum NEONLoadStoreSingleOp : uint32_t {
+ NEONLoadStoreSingle1 = 0x00000000,
+ NEONLoadStoreSingle2 = 0x00200000,
+ NEONLoadStoreSingle3 = 0x00002000,
+ NEONLoadStoreSingle4 = 0x00202000,
+ NEONLoadStoreSingleL = 0x00400000,
+ NEONLoadStoreSingle_b = 0x00000000,
+ NEONLoadStoreSingle_h = 0x00004000,
+ NEONLoadStoreSingle_s = 0x00008000,
+ NEONLoadStoreSingle_d = 0x00008400,
+ NEONLoadStoreSingleAllLanes = 0x0000C000,
+ NEONLoadStoreSingleLenMask = 0x00202000
+};
+
+// NEON load/store single structure.
+enum NEONLoadStoreSingleStructOp : uint32_t {
+ NEONLoadStoreSingleStructFixed = 0x0D000000,
+ NEONLoadStoreSingleStructFMask = 0xBF9F0000,
+ NEONLoadStoreSingleStructMask = 0xBFFFE000,
+ NEONLoadStoreSingleStructStore = NEONLoadStoreSingleStructFixed,
+ NEONLoadStoreSingleStructLoad = NEONLoadStoreSingleStructFixed |
+ NEONLoadStoreSingleL,
+ NEONLoadStoreSingleStructLoad1 = NEONLoadStoreSingle1 |
+ NEONLoadStoreSingleStructLoad,
+ NEONLoadStoreSingleStructLoad2 = NEONLoadStoreSingle2 |
+ NEONLoadStoreSingleStructLoad,
+ NEONLoadStoreSingleStructLoad3 = NEONLoadStoreSingle3 |
+ NEONLoadStoreSingleStructLoad,
+ NEONLoadStoreSingleStructLoad4 = NEONLoadStoreSingle4 |
+ NEONLoadStoreSingleStructLoad,
+ NEONLoadStoreSingleStructStore1 = NEONLoadStoreSingle1 |
+ NEONLoadStoreSingleStructFixed,
+ NEONLoadStoreSingleStructStore2 = NEONLoadStoreSingle2 |
+ NEONLoadStoreSingleStructFixed,
+ NEONLoadStoreSingleStructStore3 = NEONLoadStoreSingle3 |
+ NEONLoadStoreSingleStructFixed,
+ NEONLoadStoreSingleStructStore4 = NEONLoadStoreSingle4 |
+ NEONLoadStoreSingleStructFixed,
+ NEON_LD1_b = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_b,
+ NEON_LD1_h = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_h,
+ NEON_LD1_s = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_s,
+ NEON_LD1_d = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingle_d,
+ NEON_LD1R = NEONLoadStoreSingleStructLoad1 | NEONLoadStoreSingleAllLanes,
+ NEON_ST1_b = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_b,
+ NEON_ST1_h = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_h,
+ NEON_ST1_s = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_s,
+ NEON_ST1_d = NEONLoadStoreSingleStructStore1 | NEONLoadStoreSingle_d,
+
+ NEON_LD2_b = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_b,
+ NEON_LD2_h = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_h,
+ NEON_LD2_s = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_s,
+ NEON_LD2_d = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingle_d,
+ NEON_LD2R = NEONLoadStoreSingleStructLoad2 | NEONLoadStoreSingleAllLanes,
+ NEON_ST2_b = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_b,
+ NEON_ST2_h = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_h,
+ NEON_ST2_s = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_s,
+ NEON_ST2_d = NEONLoadStoreSingleStructStore2 | NEONLoadStoreSingle_d,
+
+ NEON_LD3_b = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_b,
+ NEON_LD3_h = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_h,
+ NEON_LD3_s = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_s,
+ NEON_LD3_d = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingle_d,
+ NEON_LD3R = NEONLoadStoreSingleStructLoad3 | NEONLoadStoreSingleAllLanes,
+ NEON_ST3_b = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_b,
+ NEON_ST3_h = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_h,
+ NEON_ST3_s = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_s,
+ NEON_ST3_d = NEONLoadStoreSingleStructStore3 | NEONLoadStoreSingle_d,
+
+ NEON_LD4_b = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_b,
+ NEON_LD4_h = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_h,
+ NEON_LD4_s = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_s,
+ NEON_LD4_d = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingle_d,
+ NEON_LD4R = NEONLoadStoreSingleStructLoad4 | NEONLoadStoreSingleAllLanes,
+ NEON_ST4_b = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_b,
+ NEON_ST4_h = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_h,
+ NEON_ST4_s = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_s,
+ NEON_ST4_d = NEONLoadStoreSingleStructStore4 | NEONLoadStoreSingle_d
+};
+
+// NEON load/store single structure with post-index addressing.
+enum NEONLoadStoreSingleStructPostIndexOp : uint32_t {
+ NEONLoadStoreSingleStructPostIndexFixed = 0x0D800000,
+ NEONLoadStoreSingleStructPostIndexFMask = 0xBF800000,
+ NEONLoadStoreSingleStructPostIndexMask = 0xBFE0E000,
+ NEONLoadStoreSingleStructPostIndex = 0x00800000,
+ NEON_LD1_b_post = NEON_LD1_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD1_h_post = NEON_LD1_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD1_s_post = NEON_LD1_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD1_d_post = NEON_LD1_d | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD1R_post = NEON_LD1R | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST1_b_post = NEON_ST1_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST1_h_post = NEON_ST1_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST1_s_post = NEON_ST1_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST1_d_post = NEON_ST1_d | NEONLoadStoreSingleStructPostIndex,
+
+ NEON_LD2_b_post = NEON_LD2_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD2_h_post = NEON_LD2_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD2_s_post = NEON_LD2_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD2_d_post = NEON_LD2_d | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD2R_post = NEON_LD2R | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST2_b_post = NEON_ST2_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST2_h_post = NEON_ST2_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST2_s_post = NEON_ST2_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST2_d_post = NEON_ST2_d | NEONLoadStoreSingleStructPostIndex,
+
+ NEON_LD3_b_post = NEON_LD3_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD3_h_post = NEON_LD3_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD3_s_post = NEON_LD3_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD3_d_post = NEON_LD3_d | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD3R_post = NEON_LD3R | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST3_b_post = NEON_ST3_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST3_h_post = NEON_ST3_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST3_s_post = NEON_ST3_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST3_d_post = NEON_ST3_d | NEONLoadStoreSingleStructPostIndex,
+
+ NEON_LD4_b_post = NEON_LD4_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD4_h_post = NEON_LD4_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD4_s_post = NEON_LD4_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD4_d_post = NEON_LD4_d | NEONLoadStoreSingleStructPostIndex,
+ NEON_LD4R_post = NEON_LD4R | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST4_b_post = NEON_ST4_b | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST4_h_post = NEON_ST4_h | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST4_s_post = NEON_ST4_s | NEONLoadStoreSingleStructPostIndex,
+ NEON_ST4_d_post = NEON_ST4_d | NEONLoadStoreSingleStructPostIndex
+};
+
+// NEON modified immediate.
+enum NEONModifiedImmediateOp : uint32_t {
+ NEONModifiedImmediateFixed = 0x0F000400,
+ NEONModifiedImmediateFMask = 0x9FF80400,
+ NEONModifiedImmediateOpBit = 0x20000000,
+ NEONModifiedImmediate_FMOV = NEONModifiedImmediateFixed | 0x00000800,
+ NEONModifiedImmediate_MOVI = NEONModifiedImmediateFixed | 0x00000000,
+ NEONModifiedImmediate_MVNI = NEONModifiedImmediateFixed | 0x20000000,
+ NEONModifiedImmediate_ORR = NEONModifiedImmediateFixed | 0x00001000,
+ NEONModifiedImmediate_BIC = NEONModifiedImmediateFixed | 0x20001000
+};
+
+// NEON shift immediate.
+enum NEONShiftImmediateOp : uint32_t {
+ NEONShiftImmediateFixed = 0x0F000400,
+ NEONShiftImmediateFMask = 0x9F800400,
+ NEONShiftImmediateMask = 0xBF80FC00,
+ NEONShiftImmediateUBit = 0x20000000,
+ NEON_SHL = NEONShiftImmediateFixed | 0x00005000,
+ NEON_SSHLL = NEONShiftImmediateFixed | 0x0000A000,
+ NEON_USHLL = NEONShiftImmediateFixed | 0x2000A000,
+ NEON_SLI = NEONShiftImmediateFixed | 0x20005000,
+ NEON_SRI = NEONShiftImmediateFixed | 0x20004000,
+ NEON_SHRN = NEONShiftImmediateFixed | 0x00008000,
+ NEON_RSHRN = NEONShiftImmediateFixed | 0x00008800,
+ NEON_UQSHRN = NEONShiftImmediateFixed | 0x20009000,
+ NEON_UQRSHRN = NEONShiftImmediateFixed | 0x20009800,
+ NEON_SQSHRN = NEONShiftImmediateFixed | 0x00009000,
+ NEON_SQRSHRN = NEONShiftImmediateFixed | 0x00009800,
+ NEON_SQSHRUN = NEONShiftImmediateFixed | 0x20008000,
+ NEON_SQRSHRUN = NEONShiftImmediateFixed | 0x20008800,
+ NEON_SSHR = NEONShiftImmediateFixed | 0x00000000,
+ NEON_SRSHR = NEONShiftImmediateFixed | 0x00002000,
+ NEON_USHR = NEONShiftImmediateFixed | 0x20000000,
+ NEON_URSHR = NEONShiftImmediateFixed | 0x20002000,
+ NEON_SSRA = NEONShiftImmediateFixed | 0x00001000,
+ NEON_SRSRA = NEONShiftImmediateFixed | 0x00003000,
+ NEON_USRA = NEONShiftImmediateFixed | 0x20001000,
+ NEON_URSRA = NEONShiftImmediateFixed | 0x20003000,
+ NEON_SQSHLU = NEONShiftImmediateFixed | 0x20006000,
+ NEON_SCVTF_imm = NEONShiftImmediateFixed | 0x0000E000,
+ NEON_UCVTF_imm = NEONShiftImmediateFixed | 0x2000E000,
+ NEON_FCVTZS_imm = NEONShiftImmediateFixed | 0x0000F800,
+ NEON_FCVTZU_imm = NEONShiftImmediateFixed | 0x2000F800,
+ NEON_SQSHL_imm = NEONShiftImmediateFixed | 0x00007000,
+ NEON_UQSHL_imm = NEONShiftImmediateFixed | 0x20007000
+};
+
+// NEON table.
+enum NEONTableOp : uint32_t {
+ NEONTableFixed = 0x0E000000,
+ NEONTableFMask = 0xBF208C00,
+ NEONTableExt = 0x00001000,
+ NEONTableMask = 0xBF20FC00,
+ NEON_TBL_1v = NEONTableFixed | 0x00000000,
+ NEON_TBL_2v = NEONTableFixed | 0x00002000,
+ NEON_TBL_3v = NEONTableFixed | 0x00004000,
+ NEON_TBL_4v = NEONTableFixed | 0x00006000,
+ NEON_TBX_1v = NEON_TBL_1v | NEONTableExt,
+ NEON_TBX_2v = NEON_TBL_2v | NEONTableExt,
+ NEON_TBX_3v = NEON_TBL_3v | NEONTableExt,
+ NEON_TBX_4v = NEON_TBL_4v | NEONTableExt
+};
+
+// NEON perm.
+enum NEONPermOp : uint32_t {
+ NEONPermFixed = 0x0E000800,
+ NEONPermFMask = 0xBF208C00,
+ NEONPermMask = 0x3F20FC00,
+ NEON_UZP1 = NEONPermFixed | 0x00001000,
+ NEON_TRN1 = NEONPermFixed | 0x00002000,
+ NEON_ZIP1 = NEONPermFixed | 0x00003000,
+ NEON_UZP2 = NEONPermFixed | 0x00005000,
+ NEON_TRN2 = NEONPermFixed | 0x00006000,
+ NEON_ZIP2 = NEONPermFixed | 0x00007000
+};
+
+// NEON scalar instructions with two register operands.
+enum NEONScalar2RegMiscOp : uint32_t {
+ NEONScalar2RegMiscFixed = 0x5E200800,
+ NEONScalar2RegMiscFMask = 0xDF3E0C00,
+ NEONScalar2RegMiscMask = NEON_Q | NEONScalar | NEON2RegMiscMask,
+ NEON_CMGT_zero_scalar = NEON_Q | NEONScalar | NEON_CMGT_zero,
+ NEON_CMEQ_zero_scalar = NEON_Q | NEONScalar | NEON_CMEQ_zero,
+ NEON_CMLT_zero_scalar = NEON_Q | NEONScalar | NEON_CMLT_zero,
+ NEON_CMGE_zero_scalar = NEON_Q | NEONScalar | NEON_CMGE_zero,
+ NEON_CMLE_zero_scalar = NEON_Q | NEONScalar | NEON_CMLE_zero,
+ NEON_ABS_scalar = NEON_Q | NEONScalar | NEON_ABS,
+ NEON_SQABS_scalar = NEON_Q | NEONScalar | NEON_SQABS,
+ NEON_NEG_scalar = NEON_Q | NEONScalar | NEON_NEG,
+ NEON_SQNEG_scalar = NEON_Q | NEONScalar | NEON_SQNEG,
+ NEON_SQXTN_scalar = NEON_Q | NEONScalar | NEON_SQXTN,
+ NEON_UQXTN_scalar = NEON_Q | NEONScalar | NEON_UQXTN,
+ NEON_SQXTUN_scalar = NEON_Q | NEONScalar | NEON_SQXTUN,
+ NEON_SUQADD_scalar = NEON_Q | NEONScalar | NEON_SUQADD,
+ NEON_USQADD_scalar = NEON_Q | NEONScalar | NEON_USQADD,
+
+ NEONScalar2RegMiscOpcode = NEON2RegMiscOpcode,
+ NEON_NEG_scalar_opcode = NEON_NEG_scalar & NEONScalar2RegMiscOpcode,
+
+ NEONScalar2RegMiscFPMask = NEONScalar2RegMiscMask | 0x00800000,
+ NEON_FRSQRTE_scalar = NEON_Q | NEONScalar | NEON_FRSQRTE,
+ NEON_FRECPE_scalar = NEON_Q | NEONScalar | NEON_FRECPE,
+ NEON_SCVTF_scalar = NEON_Q | NEONScalar | NEON_SCVTF,
+ NEON_UCVTF_scalar = NEON_Q | NEONScalar | NEON_UCVTF,
+ NEON_FCMGT_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGT_zero,
+ NEON_FCMEQ_zero_scalar = NEON_Q | NEONScalar | NEON_FCMEQ_zero,
+ NEON_FCMLT_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLT_zero,
+ NEON_FCMGE_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGE_zero,
+ NEON_FCMLE_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLE_zero,
+ NEON_FRECPX_scalar = NEONScalar2RegMiscFixed | 0x0081F000,
+ NEON_FCVTNS_scalar = NEON_Q | NEONScalar | NEON_FCVTNS,
+ NEON_FCVTNU_scalar = NEON_Q | NEONScalar | NEON_FCVTNU,
+ NEON_FCVTPS_scalar = NEON_Q | NEONScalar | NEON_FCVTPS,
+ NEON_FCVTPU_scalar = NEON_Q | NEONScalar | NEON_FCVTPU,
+ NEON_FCVTMS_scalar = NEON_Q | NEONScalar | NEON_FCVTMS,
+ NEON_FCVTMU_scalar = NEON_Q | NEONScalar | NEON_FCVTMU,
+ NEON_FCVTZS_scalar = NEON_Q | NEONScalar | NEON_FCVTZS,
+ NEON_FCVTZU_scalar = NEON_Q | NEONScalar | NEON_FCVTZU,
+ NEON_FCVTAS_scalar = NEON_Q | NEONScalar | NEON_FCVTAS,
+ NEON_FCVTAU_scalar = NEON_Q | NEONScalar | NEON_FCVTAU,
+ NEON_FCVTXN_scalar = NEON_Q | NEONScalar | NEON_FCVTXN
+};
+
+// NEON instructions with two register operands (FP16).
+enum NEONScalar2RegMiscFP16Op : uint32_t {
+ NEONScalar2RegMiscFP16Fixed = 0x5E780800,
+ NEONScalar2RegMiscFP16FMask = 0xDF7E0C00,
+ NEONScalar2RegMiscFP16Mask = 0xFFFFFC00,
+ NEON_FCVTNS_H_scalar = NEON_Q | NEONScalar | NEON_FCVTNS_H,
+ NEON_FCVTMS_H_scalar = NEON_Q | NEONScalar | NEON_FCVTMS_H,
+ NEON_FCVTAS_H_scalar = NEON_Q | NEONScalar | NEON_FCVTAS_H,
+ NEON_SCVTF_H_scalar = NEON_Q | NEONScalar | NEON_SCVTF_H,
+ NEON_FCMGT_H_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGT_H_zero,
+ NEON_FCMEQ_H_zero_scalar = NEON_Q | NEONScalar | NEON_FCMEQ_H_zero,
+ NEON_FCMLT_H_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLT_H_zero,
+ NEON_FCVTPS_H_scalar = NEON_Q | NEONScalar | NEON_FCVTPS_H,
+ NEON_FCVTZS_H_scalar = NEON_Q | NEONScalar | NEON_FCVTZS_H,
+ NEON_FRECPE_H_scalar = NEON_Q | NEONScalar | NEON_FRECPE_H,
+ NEON_FRECPX_H_scalar = NEONScalar2RegMiscFP16Fixed | 0x0081F000,
+ NEON_FCVTNU_H_scalar = NEON_Q | NEONScalar | NEON_FCVTNU_H,
+ NEON_FCVTMU_H_scalar = NEON_Q | NEONScalar | NEON_FCVTMU_H,
+ NEON_FCVTAU_H_scalar = NEON_Q | NEONScalar | NEON_FCVTAU_H,
+ NEON_UCVTF_H_scalar = NEON_Q | NEONScalar | NEON_UCVTF_H,
+ NEON_FCMGE_H_zero_scalar = NEON_Q | NEONScalar | NEON_FCMGE_H_zero,
+ NEON_FCMLE_H_zero_scalar = NEON_Q | NEONScalar | NEON_FCMLE_H_zero,
+ NEON_FCVTPU_H_scalar = NEON_Q | NEONScalar | NEON_FCVTPU_H,
+ NEON_FCVTZU_H_scalar = NEON_Q | NEONScalar | NEON_FCVTZU_H,
+ NEON_FRSQRTE_H_scalar = NEON_Q | NEONScalar | NEON_FRSQRTE_H
+};
+
+// NEON scalar instructions with three same-type operands.
+enum NEONScalar3SameOp : uint32_t {
+ NEONScalar3SameFixed = 0x5E200400,
+ NEONScalar3SameFMask = 0xDF200400,
+ NEONScalar3SameMask = 0xFF20FC00,
+ NEON_ADD_scalar = NEON_Q | NEONScalar | NEON_ADD,
+ NEON_CMEQ_scalar = NEON_Q | NEONScalar | NEON_CMEQ,
+ NEON_CMGE_scalar = NEON_Q | NEONScalar | NEON_CMGE,
+ NEON_CMGT_scalar = NEON_Q | NEONScalar | NEON_CMGT,
+ NEON_CMHI_scalar = NEON_Q | NEONScalar | NEON_CMHI,
+ NEON_CMHS_scalar = NEON_Q | NEONScalar | NEON_CMHS,
+ NEON_CMTST_scalar = NEON_Q | NEONScalar | NEON_CMTST,
+ NEON_SUB_scalar = NEON_Q | NEONScalar | NEON_SUB,
+ NEON_UQADD_scalar = NEON_Q | NEONScalar | NEON_UQADD,
+ NEON_SQADD_scalar = NEON_Q | NEONScalar | NEON_SQADD,
+ NEON_UQSUB_scalar = NEON_Q | NEONScalar | NEON_UQSUB,
+ NEON_SQSUB_scalar = NEON_Q | NEONScalar | NEON_SQSUB,
+ NEON_USHL_scalar = NEON_Q | NEONScalar | NEON_USHL,
+ NEON_SSHL_scalar = NEON_Q | NEONScalar | NEON_SSHL,
+ NEON_UQSHL_scalar = NEON_Q | NEONScalar | NEON_UQSHL,
+ NEON_SQSHL_scalar = NEON_Q | NEONScalar | NEON_SQSHL,
+ NEON_URSHL_scalar = NEON_Q | NEONScalar | NEON_URSHL,
+ NEON_SRSHL_scalar = NEON_Q | NEONScalar | NEON_SRSHL,
+ NEON_UQRSHL_scalar = NEON_Q | NEONScalar | NEON_UQRSHL,
+ NEON_SQRSHL_scalar = NEON_Q | NEONScalar | NEON_SQRSHL,
+ NEON_SQDMULH_scalar = NEON_Q | NEONScalar | NEON_SQDMULH,
+ NEON_SQRDMULH_scalar = NEON_Q | NEONScalar | NEON_SQRDMULH,
+
+ // NEON floating point scalar instructions with three same-type operands.
+ NEONScalar3SameFPFixed = NEONScalar3SameFixed | 0x0000C000,
+ NEONScalar3SameFPFMask = NEONScalar3SameFMask | 0x0000C000,
+ NEONScalar3SameFPMask = NEONScalar3SameMask | 0x00800000,
+ NEON_FACGE_scalar = NEON_Q | NEONScalar | NEON_FACGE,
+ NEON_FACGT_scalar = NEON_Q | NEONScalar | NEON_FACGT,
+ NEON_FCMEQ_scalar = NEON_Q | NEONScalar | NEON_FCMEQ,
+ NEON_FCMGE_scalar = NEON_Q | NEONScalar | NEON_FCMGE,
+ NEON_FCMGT_scalar = NEON_Q | NEONScalar | NEON_FCMGT,
+ NEON_FMULX_scalar = NEON_Q | NEONScalar | NEON_FMULX,
+ NEON_FRECPS_scalar = NEON_Q | NEONScalar | NEON_FRECPS,
+ NEON_FRSQRTS_scalar = NEON_Q | NEONScalar | NEON_FRSQRTS,
+ NEON_FABD_scalar = NEON_Q | NEONScalar | NEON_FABD
+};
+
+// NEON scalar instructions with three different-type operands.
+enum NEONScalar3DiffOp : uint32_t {
+ NEONScalar3DiffFixed = 0x5E200000,
+ NEONScalar3DiffFMask = 0xDF200C00,
+ NEONScalar3DiffMask = NEON_Q | NEONScalar | NEON3DifferentMask,
+ NEON_SQDMLAL_scalar = NEON_Q | NEONScalar | NEON_SQDMLAL,
+ NEON_SQDMLSL_scalar = NEON_Q | NEONScalar | NEON_SQDMLSL,
+ NEON_SQDMULL_scalar = NEON_Q | NEONScalar | NEON_SQDMULL
+};
+
+// NEON scalar instructions with indexed element operand.
+enum NEONScalarByIndexedElementOp : uint32_t {
+ NEONScalarByIndexedElementFixed = 0x5F000000,
+ NEONScalarByIndexedElementFMask = 0xDF000400,
+ NEONScalarByIndexedElementMask = 0xFF00F400,
+ NEON_SQDMLAL_byelement_scalar = NEON_Q | NEONScalar | NEON_SQDMLAL_byelement,
+ NEON_SQDMLSL_byelement_scalar = NEON_Q | NEONScalar | NEON_SQDMLSL_byelement,
+ NEON_SQDMULL_byelement_scalar = NEON_Q | NEONScalar | NEON_SQDMULL_byelement,
+ NEON_SQDMULH_byelement_scalar = NEON_Q | NEONScalar | NEON_SQDMULH_byelement,
+ NEON_SQRDMULH_byelement_scalar
+ = NEON_Q | NEONScalar | NEON_SQRDMULH_byelement,
+ NEON_SQRDMLAH_byelement_scalar
+ = NEON_Q | NEONScalar | NEON_SQRDMLAH_byelement,
+ NEON_SQRDMLSH_byelement_scalar
+ = NEON_Q | NEONScalar | NEON_SQRDMLSH_byelement,
+ NEON_FMLA_H_byelement_scalar = NEON_Q | NEONScalar | NEON_FMLA_H_byelement,
+ NEON_FMLS_H_byelement_scalar = NEON_Q | NEONScalar | NEON_FMLS_H_byelement,
+ NEON_FMUL_H_byelement_scalar = NEON_Q | NEONScalar | NEON_FMUL_H_byelement,
+ NEON_FMULX_H_byelement_scalar = NEON_Q | NEONScalar | NEON_FMULX_H_byelement,
+
+ // Floating point instructions.
+ NEONScalarByIndexedElementFPFixed
+ = NEONScalarByIndexedElementFixed | 0x00800000,
+ NEONScalarByIndexedElementFPMask
+ = NEONScalarByIndexedElementMask | 0x00800000,
+ NEON_FMLA_byelement_scalar = NEON_Q | NEONScalar | NEON_FMLA_byelement,
+ NEON_FMLS_byelement_scalar = NEON_Q | NEONScalar | NEON_FMLS_byelement,
+ NEON_FMUL_byelement_scalar = NEON_Q | NEONScalar | NEON_FMUL_byelement,
+ NEON_FMULX_byelement_scalar = NEON_Q | NEONScalar | NEON_FMULX_byelement
+};
+
+// NEON scalar register copy.
+enum NEONScalarCopyOp : uint32_t {
+ NEONScalarCopyFixed = 0x5E000400,
+ NEONScalarCopyFMask = 0xDFE08400,
+ NEONScalarCopyMask = 0xFFE0FC00,
+ NEON_DUP_ELEMENT_scalar = NEON_Q | NEONScalar | NEON_DUP_ELEMENT
+};
+
+// NEON scalar pairwise instructions.
+enum NEONScalarPairwiseOp : uint32_t {
+ NEONScalarPairwiseFixed = 0x5E300800,
+ NEONScalarPairwiseFMask = 0xDF3E0C00,
+ NEONScalarPairwiseMask = 0xFFB1F800,
+ NEON_ADDP_scalar = NEONScalarPairwiseFixed | 0x0081B000,
+ NEON_FMAXNMP_h_scalar = NEONScalarPairwiseFixed | 0x0000C000,
+ NEON_FADDP_h_scalar = NEONScalarPairwiseFixed | 0x0000D000,
+ NEON_FMAXP_h_scalar = NEONScalarPairwiseFixed | 0x0000F000,
+ NEON_FMINNMP_h_scalar = NEONScalarPairwiseFixed | 0x0080C000,
+ NEON_FMINP_h_scalar = NEONScalarPairwiseFixed | 0x0080F000,
+ NEON_FMAXNMP_scalar = NEONScalarPairwiseFixed | 0x2000C000,
+ NEON_FMINNMP_scalar = NEONScalarPairwiseFixed | 0x2080C000,
+ NEON_FADDP_scalar = NEONScalarPairwiseFixed | 0x2000D000,
+ NEON_FMAXP_scalar = NEONScalarPairwiseFixed | 0x2000F000,
+ NEON_FMINP_scalar = NEONScalarPairwiseFixed | 0x2080F000
+};
+
+// NEON scalar shift immediate.
+enum NEONScalarShiftImmediateOp : uint32_t {
+ NEONScalarShiftImmediateFixed = 0x5F000400,
+ NEONScalarShiftImmediateFMask = 0xDF800400,
+ NEONScalarShiftImmediateMask = 0xFF80FC00,
+ NEON_SHL_scalar = NEON_Q | NEONScalar | NEON_SHL,
+ NEON_SLI_scalar = NEON_Q | NEONScalar | NEON_SLI,
+ NEON_SRI_scalar = NEON_Q | NEONScalar | NEON_SRI,
+ NEON_SSHR_scalar = NEON_Q | NEONScalar | NEON_SSHR,
+ NEON_USHR_scalar = NEON_Q | NEONScalar | NEON_USHR,
+ NEON_SRSHR_scalar = NEON_Q | NEONScalar | NEON_SRSHR,
+ NEON_URSHR_scalar = NEON_Q | NEONScalar | NEON_URSHR,
+ NEON_SSRA_scalar = NEON_Q | NEONScalar | NEON_SSRA,
+ NEON_USRA_scalar = NEON_Q | NEONScalar | NEON_USRA,
+ NEON_SRSRA_scalar = NEON_Q | NEONScalar | NEON_SRSRA,
+ NEON_URSRA_scalar = NEON_Q | NEONScalar | NEON_URSRA,
+ NEON_UQSHRN_scalar = NEON_Q | NEONScalar | NEON_UQSHRN,
+ NEON_UQRSHRN_scalar = NEON_Q | NEONScalar | NEON_UQRSHRN,
+ NEON_SQSHRN_scalar = NEON_Q | NEONScalar | NEON_SQSHRN,
+ NEON_SQRSHRN_scalar = NEON_Q | NEONScalar | NEON_SQRSHRN,
+ NEON_SQSHRUN_scalar = NEON_Q | NEONScalar | NEON_SQSHRUN,
+ NEON_SQRSHRUN_scalar = NEON_Q | NEONScalar | NEON_SQRSHRUN,
+ NEON_SQSHLU_scalar = NEON_Q | NEONScalar | NEON_SQSHLU,
+ NEON_SQSHL_imm_scalar = NEON_Q | NEONScalar | NEON_SQSHL_imm,
+ NEON_UQSHL_imm_scalar = NEON_Q | NEONScalar | NEON_UQSHL_imm,
+ NEON_SCVTF_imm_scalar = NEON_Q | NEONScalar | NEON_SCVTF_imm,
+ NEON_UCVTF_imm_scalar = NEON_Q | NEONScalar | NEON_UCVTF_imm,
+ NEON_FCVTZS_imm_scalar = NEON_Q | NEONScalar | NEON_FCVTZS_imm,
+ NEON_FCVTZU_imm_scalar = NEON_Q | NEONScalar | NEON_FCVTZU_imm
+};
+
+enum ReservedOp : uint32_t {
+ ReservedFixed = 0x00000000,
+ ReservedFMask = 0x1E000000,
+ ReservedMask = 0xFFFF0000,
+
+ UDF = ReservedFixed | 0x00000000
+};
+
+// Unimplemented and unallocated instructions. These are defined to make fixed
+// bit assertion easier.
+enum UnimplementedOp : uint32_t {
+ UnimplementedFixed = 0x00000000,
+ UnimplementedFMask = 0x00000000
+};
+
+enum UnallocatedOp : uint32_t {
+ UnallocatedFixed = 0x00000000,
+ UnallocatedFMask = 0x00000000
+};
+
+// Instruction bit pattern for an undefined instruction, that will trigger a
+// SIGILL at runtime.
+//
+// A couple of strategies we can use here. There are no unencoded
+// instructions in the instruction set that are guaranteed to remain that
+// way. However there are some currently (as of 2018) unencoded
+// instructions that are good candidates.
+//
+// Ideally, unencoded instructions should be non-destructive to the register
+// state, and should be unencoded at all exception levels.
+//
+// At the trap the pc will hold the address of the offending instruction.
+//
+// Some candidates for unencoded instructions:
+//
+// 0xd4a00000 (essentially dcps0, a good one since it is nonsensical and may
+// remain unencoded in the future for that reason)
+// 0x33000000 (bfm variant)
+// 0xd67f0000 (br variant)
+// 0x5ac00c00 (rbit variant)
+//
+// This instruction is "dcps0", also has 16-bit payload if needed.
+static constexpr uint32_t UNDEFINED_INST_PATTERN = 0xd4a00000;
+
+} // namespace vixl
+
+#endif // VIXL_A64_CONSTANTS_A64_H_
diff --git a/js/src/jit/arm64/vixl/Cpu-Features-vixl.cpp b/js/src/jit/arm64/vixl/Cpu-Features-vixl.cpp
new file mode 100644
index 0000000000..f31c22fbf5
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Cpu-Features-vixl.cpp
@@ -0,0 +1,231 @@
+// Copyright 2018, VIXL authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+#include "jit/arm64/vixl/Cpu-Features-vixl.h"
+
+#include <ostream>
+
+#include "jit/arm64/vixl/Cpu-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+
+#define VIXL_USE_AARCH64_CPU_HELPERS
+
+namespace vixl {
+
+static uint64_t MakeFeatureMask(CPUFeatures::Feature feature) {
+ if (feature == CPUFeatures::kNone) {
+ return 0;
+ } else {
+ // Check that the shift is well-defined, and that the feature is valid.
+ VIXL_STATIC_ASSERT(CPUFeatures::kNumberOfFeatures <=
+ (sizeof(uint64_t) * 8));
+ VIXL_ASSERT(feature < CPUFeatures::kNumberOfFeatures);
+ return UINT64_C(1) << feature;
+ }
+}
+
+CPUFeatures::CPUFeatures(Feature feature0,
+ Feature feature1,
+ Feature feature2,
+ Feature feature3)
+ : features_(0) {
+ Combine(feature0, feature1, feature2, feature3);
+}
+
+CPUFeatures CPUFeatures::All() {
+ CPUFeatures all;
+ // Check that the shift is well-defined.
+ VIXL_STATIC_ASSERT(CPUFeatures::kNumberOfFeatures < (sizeof(uint64_t) * 8));
+ all.features_ = (UINT64_C(1) << kNumberOfFeatures) - 1;
+ return all;
+}
+
+CPUFeatures CPUFeatures::InferFromIDRegisters() {
+ // This function assumes that kIDRegisterEmulation is available.
+ CPUFeatures features(CPUFeatures::kIDRegisterEmulation);
+#ifdef VIXL_USE_AARCH64_CPU_HELPERS
+ // Note that the Linux kernel filters these values during emulation, so the
+ // results may not exactly match the expected hardware support.
+ features.Combine(CPU::InferCPUFeaturesFromIDRegisters());
+#endif
+ return features;
+}
+
+CPUFeatures CPUFeatures::InferFromOS(QueryIDRegistersOption option) {
+#ifdef VIXL_USE_AARCH64_CPU_HELPERS
+ return CPU::InferCPUFeaturesFromOS(option);
+#else
+ USE(option);
+ return CPUFeatures();
+#endif
+}
+
+void CPUFeatures::Combine(const CPUFeatures& other) {
+ features_ |= other.features_;
+}
+
+void CPUFeatures::Combine(Feature feature0,
+ Feature feature1,
+ Feature feature2,
+ Feature feature3) {
+ features_ |= MakeFeatureMask(feature0);
+ features_ |= MakeFeatureMask(feature1);
+ features_ |= MakeFeatureMask(feature2);
+ features_ |= MakeFeatureMask(feature3);
+}
+
+void CPUFeatures::Remove(const CPUFeatures& other) {
+ features_ &= ~other.features_;
+}
+
+void CPUFeatures::Remove(Feature feature0,
+ Feature feature1,
+ Feature feature2,
+ Feature feature3) {
+ features_ &= ~MakeFeatureMask(feature0);
+ features_ &= ~MakeFeatureMask(feature1);
+ features_ &= ~MakeFeatureMask(feature2);
+ features_ &= ~MakeFeatureMask(feature3);
+}
+
+CPUFeatures CPUFeatures::With(const CPUFeatures& other) const {
+ CPUFeatures f(*this);
+ f.Combine(other);
+ return f;
+}
+
+CPUFeatures CPUFeatures::With(Feature feature0,
+ Feature feature1,
+ Feature feature2,
+ Feature feature3) const {
+ CPUFeatures f(*this);
+ f.Combine(feature0, feature1, feature2, feature3);
+ return f;
+}
+
+CPUFeatures CPUFeatures::Without(const CPUFeatures& other) const {
+ CPUFeatures f(*this);
+ f.Remove(other);
+ return f;
+}
+
+CPUFeatures CPUFeatures::Without(Feature feature0,
+ Feature feature1,
+ Feature feature2,
+ Feature feature3) const {
+ CPUFeatures f(*this);
+ f.Remove(feature0, feature1, feature2, feature3);
+ return f;
+}
+
+bool CPUFeatures::Has(const CPUFeatures& other) const {
+ return (features_ & other.features_) == other.features_;
+}
+
+bool CPUFeatures::Has(Feature feature0,
+ Feature feature1,
+ Feature feature2,
+ Feature feature3) const {
+ uint64_t mask = MakeFeatureMask(feature0) | MakeFeatureMask(feature1) |
+ MakeFeatureMask(feature2) | MakeFeatureMask(feature3);
+ return (features_ & mask) == mask;
+}
+
+size_t CPUFeatures::Count() const { return CountSetBits(features_); }
+
+std::ostream& operator<<(std::ostream& os, CPUFeatures::Feature feature) {
+ // clang-format off
+ switch (feature) {
+#define VIXL_FORMAT_FEATURE(SYMBOL, NAME, CPUINFO) \
+ case CPUFeatures::SYMBOL: \
+ return os << NAME;
+VIXL_CPU_FEATURE_LIST(VIXL_FORMAT_FEATURE)
+#undef VIXL_FORMAT_FEATURE
+ case CPUFeatures::kNone:
+ return os << "none";
+ case CPUFeatures::kNumberOfFeatures:
+ VIXL_UNREACHABLE();
+ }
+ // clang-format on
+ VIXL_UNREACHABLE();
+ return os;
+}
+
+CPUFeatures::const_iterator CPUFeatures::begin() const {
+ if (features_ == 0) return const_iterator(this, kNone);
+
+ int feature_number = CountTrailingZeros(features_);
+ vixl::CPUFeatures::Feature feature =
+ static_cast<CPUFeatures::Feature>(feature_number);
+ return const_iterator(this, feature);
+}
+
+CPUFeatures::const_iterator CPUFeatures::end() const {
+ return const_iterator(this, kNone);
+}
+
+std::ostream& operator<<(std::ostream& os, const CPUFeatures& features) {
+ CPUFeatures::const_iterator it = features.begin();
+ while (it != features.end()) {
+ os << *it;
+ ++it;
+ if (it != features.end()) os << ", ";
+ }
+ return os;
+}
+
+bool CPUFeaturesConstIterator::operator==(
+ const CPUFeaturesConstIterator& other) const {
+ VIXL_ASSERT(IsValid());
+ return (cpu_features_ == other.cpu_features_) && (feature_ == other.feature_);
+}
+
+CPUFeatures::Feature CPUFeaturesConstIterator::operator++() { // Prefix
+ VIXL_ASSERT(IsValid());
+ do {
+ // Find the next feature. The order is unspecified.
+ feature_ = static_cast<CPUFeatures::Feature>(feature_ + 1);
+ if (feature_ == CPUFeatures::kNumberOfFeatures) {
+ feature_ = CPUFeatures::kNone;
+ VIXL_STATIC_ASSERT(CPUFeatures::kNone == -1);
+ }
+ VIXL_ASSERT(CPUFeatures::kNone <= feature_);
+ VIXL_ASSERT(feature_ < CPUFeatures::kNumberOfFeatures);
+ // cpu_features_->Has(kNone) is always true, so this will terminate even if
+ // the features list is empty.
+ } while (!cpu_features_->Has(feature_));
+ return feature_;
+}
+
+CPUFeatures::Feature CPUFeaturesConstIterator::operator++(int) { // Postfix
+ CPUFeatures::Feature result = feature_;
+ ++(*this);
+ return result;
+}
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/Cpu-Features-vixl.h b/js/src/jit/arm64/vixl/Cpu-Features-vixl.h
new file mode 100644
index 0000000000..b980233bf2
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Cpu-Features-vixl.h
@@ -0,0 +1,397 @@
+// Copyright 2018, VIXL authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_CPU_FEATURES_H
+#define VIXL_CPU_FEATURES_H
+
+#include <ostream>
+
+#include "jit/arm64/vixl/Globals-vixl.h"
+
+
+namespace vixl {
+
+
+// clang-format off
+#define VIXL_CPU_FEATURE_LIST(V) \
+ /* If set, the OS traps and emulates MRS accesses to relevant (EL1) ID_* */ \
+ /* registers, so that the detailed feature registers can be read */ \
+ /* directly. */ \
+ V(kIDRegisterEmulation, "ID register emulation", "cpuid") \
+ \
+ V(kFP, "FP", "fp") \
+ V(kNEON, "NEON", "asimd") \
+ V(kCRC32, "CRC32", "crc32") \
+ /* Cryptographic support instructions. */ \
+ V(kAES, "AES", "aes") \
+ V(kSHA1, "SHA1", "sha1") \
+ V(kSHA2, "SHA2", "sha2") \
+ /* A form of PMULL{2} with a 128-bit (1Q) result. */ \
+ V(kPmull1Q, "Pmull1Q", "pmull") \
+ /* Atomic operations on memory: CAS, LDADD, STADD, SWP, etc. */ \
+ V(kAtomics, "Atomics", "atomics") \
+ /* Limited ordering regions: LDLAR, STLLR and their variants. */ \
+ V(kLORegions, "LORegions", NULL) \
+ /* Rounding doubling multiply add/subtract: SQRDMLAH and SQRDMLSH. */ \
+ V(kRDM, "RDM", "asimdrdm") \
+ /* Scalable Vector Extension. */ \
+ V(kSVE, "SVE", "sve") \
+ /* SDOT and UDOT support (in NEON). */ \
+ V(kDotProduct, "DotProduct", "asimddp") \
+ /* Half-precision (FP16) support for FP and NEON, respectively. */ \
+ V(kFPHalf, "FPHalf", "fphp") \
+ V(kNEONHalf, "NEONHalf", "asimdhp") \
+ /* The RAS extension, including the ESB instruction. */ \
+ V(kRAS, "RAS", NULL) \
+ /* Data cache clean to the point of persistence: DC CVAP. */ \
+ V(kDCPoP, "DCPoP", "dcpop") \
+ /* Data cache clean to the point of deep persistence: DC CVADP. */ \
+ V(kDCCVADP, "DCCVADP", NULL) \
+ /* Cryptographic support instructions. */ \
+ V(kSHA3, "SHA3", "sha3") \
+ V(kSHA512, "SHA512", "sha512") \
+ V(kSM3, "SM3", "sm3") \
+ V(kSM4, "SM4", "sm4") \
+ /* Pointer authentication for addresses. */ \
+ V(kPAuth, "PAuth", NULL) \
+ /* Pointer authentication for addresses uses QARMA. */ \
+ V(kPAuthQARMA, "PAuthQARMA", NULL) \
+ /* Generic authentication (using the PACGA instruction). */ \
+ V(kPAuthGeneric, "PAuthGeneric", NULL) \
+ /* Generic authentication uses QARMA. */ \
+ V(kPAuthGenericQARMA, "PAuthGenericQARMA", NULL) \
+ /* JavaScript-style FP -> integer conversion instruction: FJCVTZS. */ \
+ V(kJSCVT, "JSCVT", "jscvt") \
+ /* Complex number support for NEON: FCMLA and FCADD. */ \
+ V(kFcma, "Fcma", "fcma") \
+ /* RCpc-based model (for weaker release consistency): LDAPR and variants. */ \
+ V(kRCpc, "RCpc", "lrcpc") \
+ V(kRCpcImm, "RCpc (imm)", "ilrcpc") \
+ /* Flag manipulation instructions: SETF{8,16}, CFINV, RMIF. */ \
+ V(kFlagM, "FlagM", "flagm") \
+ /* Unaligned single-copy atomicity. */ \
+ V(kUSCAT, "USCAT", "uscat") \
+ /* FP16 fused multiply-add or -subtract long: FMLAL{2}, FMLSL{2}. */ \
+ V(kFHM, "FHM", "asimdfhm") \
+ /* Data-independent timing (for selected instructions). */ \
+ V(kDIT, "DIT", "dit") \
+ /* Branch target identification. */ \
+ V(kBTI, "BTI", NULL) \
+ /* Flag manipulation instructions: {AX,XA}FLAG */ \
+ V(kAXFlag, "AXFlag", NULL) \
+ /* Random number generation extension, */ \
+ V(kRNG, "RNG", NULL) \
+ /* Floating-point round to {32,64}-bit integer. */ \
+ V(kFrintToFixedSizedInt,"Frint (bounded)", NULL)
+// clang-format on
+
+
+class CPUFeaturesConstIterator;
+
+// A representation of the set of features known to be supported by the target
+// device. Each feature is represented by a simple boolean flag.
+//
+// - When the Assembler is asked to assemble an instruction, it asserts (in
+// debug mode) that the necessary features are available.
+//
+// - TODO: The MacroAssembler relies on the Assembler's assertions, but in
+// some cases it may be useful for macros to generate a fall-back sequence
+// in case features are not available.
+//
+// - The Simulator assumes by default that all features are available, but it
+// is possible to configure it to fail if the simulated code uses features
+// that are not enabled.
+//
+// The Simulator also offers pseudo-instructions to allow features to be
+// enabled and disabled dynamically. This is useful when you want to ensure
+// that some features are constrained to certain areas of code.
+//
+// - The base Disassembler knows nothing about CPU features, but the
+// PrintDisassembler can be configured to annotate its output with warnings
+// about unavailable features. The Simulator uses this feature when
+// instruction trace is enabled.
+//
+// - The Decoder-based components -- the Simulator and PrintDisassembler --
+// rely on a CPUFeaturesAuditor visitor. This visitor keeps a list of
+// features actually encountered so that a large block of code can be
+// examined (either directly or through simulation), and the required
+// features analysed later.
+//
+// Expected usage:
+//
+// // By default, VIXL uses CPUFeatures::AArch64LegacyBaseline(), for
+// // compatibility with older version of VIXL.
+// MacroAssembler masm;
+//
+// // Generate code only for the current CPU.
+// masm.SetCPUFeatures(CPUFeatures::InferFromOS());
+//
+// // Turn off feature checking entirely.
+// masm.SetCPUFeatures(CPUFeatures::All());
+//
+// Feature set manipulation:
+//
+// CPUFeatures f; // The default constructor gives an empty set.
+// // Individual features can be added (or removed).
+// f.Combine(CPUFeatures::kFP, CPUFeatures::kNEON, CPUFeatures::AES);
+// f.Remove(CPUFeatures::kNEON);
+//
+// // Some helpers exist for extensions that provide several features.
+// f.Remove(CPUFeatures::All());
+// f.Combine(CPUFeatures::AArch64LegacyBaseline());
+//
+// // Chained construction is also possible.
+// CPUFeatures g =
+// f.With(CPUFeatures::kPmull1Q).Without(CPUFeatures::kCRC32);
+//
+// // Features can be queried. Where multiple features are given, they are
+// // combined with logical AND.
+// if (h.Has(CPUFeatures::kNEON)) { ... }
+// if (h.Has(CPUFeatures::kFP, CPUFeatures::kNEON)) { ... }
+// if (h.Has(g)) { ... }
+// // If the empty set is requested, the result is always 'true'.
+// VIXL_ASSERT(h.Has(CPUFeatures()));
+//
+// // For debug and reporting purposes, features can be enumerated (or
+// // printed directly):
+// std::cout << CPUFeatures::kNEON; // Prints something like "NEON".
+// std::cout << f; // Prints something like "FP, NEON, CRC32".
+class CPUFeatures {
+ public:
+ // clang-format off
+ // Individual features.
+ // These should be treated as opaque tokens. User code should not rely on
+ // specific numeric values or ordering.
+ enum Feature {
+ // Refer to VIXL_CPU_FEATURE_LIST (above) for the list of feature names that
+ // this class supports.
+
+ kNone = -1,
+#define VIXL_DECLARE_FEATURE(SYMBOL, NAME, CPUINFO) SYMBOL,
+ VIXL_CPU_FEATURE_LIST(VIXL_DECLARE_FEATURE)
+#undef VIXL_DECLARE_FEATURE
+ kNumberOfFeatures
+ };
+ // clang-format on
+
+ // By default, construct with no features enabled.
+ CPUFeatures() : features_(0) {}
+
+ // Construct with some features already enabled.
+ CPUFeatures(Feature feature0,
+ Feature feature1 = kNone,
+ Feature feature2 = kNone,
+ Feature feature3 = kNone);
+
+ // Construct with all features enabled. This can be used to disable feature
+ // checking: `Has(...)` returns true regardless of the argument.
+ static CPUFeatures All();
+
+ // Construct an empty CPUFeatures. This is equivalent to the default
+ // constructor, but is provided for symmetry and convenience.
+ static CPUFeatures None() { return CPUFeatures(); }
+
+ // The presence of these features was assumed by version of VIXL before this
+ // API was added, so using this set by default ensures API compatibility.
+ static CPUFeatures AArch64LegacyBaseline() {
+ return CPUFeatures(kFP, kNEON, kCRC32);
+ }
+
+ // Construct a new CPUFeatures object using ID registers. This assumes that
+ // kIDRegisterEmulation is present.
+ static CPUFeatures InferFromIDRegisters();
+
+ enum QueryIDRegistersOption {
+ kDontQueryIDRegisters,
+ kQueryIDRegistersIfAvailable
+ };
+
+ // Construct a new CPUFeatures object based on what the OS reports.
+ static CPUFeatures InferFromOS(
+ QueryIDRegistersOption option = kQueryIDRegistersIfAvailable);
+
+ // Combine another CPUFeatures object into this one. Features that already
+ // exist in this set are left unchanged.
+ void Combine(const CPUFeatures& other);
+
+ // Combine specific features into this set. Features that already exist in
+ // this set are left unchanged.
+ void Combine(Feature feature0,
+ Feature feature1 = kNone,
+ Feature feature2 = kNone,
+ Feature feature3 = kNone);
+
+ // Remove features in another CPUFeatures object from this one.
+ void Remove(const CPUFeatures& other);
+
+ // Remove specific features from this set.
+ void Remove(Feature feature0,
+ Feature feature1 = kNone,
+ Feature feature2 = kNone,
+ Feature feature3 = kNone);
+
+ // Chaining helpers for convenient construction.
+ CPUFeatures With(const CPUFeatures& other) const;
+ CPUFeatures With(Feature feature0,
+ Feature feature1 = kNone,
+ Feature feature2 = kNone,
+ Feature feature3 = kNone) const;
+ CPUFeatures Without(const CPUFeatures& other) const;
+ CPUFeatures Without(Feature feature0,
+ Feature feature1 = kNone,
+ Feature feature2 = kNone,
+ Feature feature3 = kNone) const;
+
+ // Query features.
+ // Note that an empty query (like `Has(kNone)`) always returns true.
+ bool Has(const CPUFeatures& other) const;
+ bool Has(Feature feature0,
+ Feature feature1 = kNone,
+ Feature feature2 = kNone,
+ Feature feature3 = kNone) const;
+
+ // Return the number of enabled features.
+ size_t Count() const;
+ bool HasNoFeatures() const { return Count() == 0; }
+
+ // Check for equivalence.
+ bool operator==(const CPUFeatures& other) const {
+ return Has(other) && other.Has(*this);
+ }
+ bool operator!=(const CPUFeatures& other) const { return !(*this == other); }
+
+ typedef CPUFeaturesConstIterator const_iterator;
+
+ const_iterator begin() const;
+ const_iterator end() const;
+
+ private:
+ // Each bit represents a feature. This field will be replaced as needed if
+ // features are added.
+ uint64_t features_;
+
+ friend std::ostream& operator<<(std::ostream& os,
+ const vixl::CPUFeatures& features);
+};
+
+std::ostream& operator<<(std::ostream& os, vixl::CPUFeatures::Feature feature);
+std::ostream& operator<<(std::ostream& os, const vixl::CPUFeatures& features);
+
+// This is not a proper C++ iterator type, but it simulates enough of
+// ForwardIterator that simple loops can be written.
+class CPUFeaturesConstIterator {
+ public:
+ CPUFeaturesConstIterator(const CPUFeatures* cpu_features = NULL,
+ CPUFeatures::Feature start = CPUFeatures::kNone)
+ : cpu_features_(cpu_features), feature_(start) {
+ VIXL_ASSERT(IsValid());
+ }
+
+ bool operator==(const CPUFeaturesConstIterator& other) const;
+ bool operator!=(const CPUFeaturesConstIterator& other) const {
+ return !(*this == other);
+ }
+ CPUFeatures::Feature operator++();
+ CPUFeatures::Feature operator++(int);
+
+ CPUFeatures::Feature operator*() const {
+ VIXL_ASSERT(IsValid());
+ return feature_;
+ }
+
+ // For proper support of C++'s simplest "Iterator" concept, this class would
+ // have to define member types (such as CPUFeaturesIterator::pointer) to make
+ // it appear as if it iterates over Feature objects in memory. That is, we'd
+ // need CPUFeatures::iterator to behave like std::vector<Feature>::iterator.
+ // This is at least partially possible -- the std::vector<bool> specialisation
+ // does something similar -- but it doesn't seem worthwhile for a
+ // special-purpose debug helper, so they are omitted here.
+ private:
+ const CPUFeatures* cpu_features_;
+ CPUFeatures::Feature feature_;
+
+ bool IsValid() const {
+ return ((cpu_features_ == NULL) && (feature_ == CPUFeatures::kNone)) ||
+ cpu_features_->Has(feature_);
+ }
+};
+
+// A convenience scope for temporarily modifying a CPU features object. This
+// allows features to be enabled for short sequences.
+//
+// Expected usage:
+//
+// {
+// CPUFeaturesScope cpu(&masm, CPUFeatures::kCRC32);
+// // This scope can now use CRC32, as well as anything else that was enabled
+// // before the scope.
+//
+// ...
+//
+// // At the end of the scope, the original CPU features are restored.
+// }
+class CPUFeaturesScope {
+ public:
+ // Start a CPUFeaturesScope on any object that implements
+ // `CPUFeatures* GetCPUFeatures()`.
+ template <typename T>
+ explicit CPUFeaturesScope(T* cpu_features_wrapper,
+ CPUFeatures::Feature feature0 = CPUFeatures::kNone,
+ CPUFeatures::Feature feature1 = CPUFeatures::kNone,
+ CPUFeatures::Feature feature2 = CPUFeatures::kNone,
+ CPUFeatures::Feature feature3 = CPUFeatures::kNone)
+ : cpu_features_(cpu_features_wrapper->GetCPUFeatures()),
+ old_features_(*cpu_features_) {
+ cpu_features_->Combine(feature0, feature1, feature2, feature3);
+ }
+
+ template <typename T>
+ CPUFeaturesScope(T* cpu_features_wrapper, const CPUFeatures& other)
+ : cpu_features_(cpu_features_wrapper->GetCPUFeatures()),
+ old_features_(*cpu_features_) {
+ cpu_features_->Combine(other);
+ }
+
+ ~CPUFeaturesScope() { *cpu_features_ = old_features_; }
+
+ // For advanced usage, the CPUFeatures object can be accessed directly.
+ // The scope will restore the original state when it ends.
+
+ CPUFeatures* GetCPUFeatures() const { return cpu_features_; }
+
+ void SetCPUFeatures(const CPUFeatures& cpu_features) {
+ *cpu_features_ = cpu_features;
+ }
+
+ private:
+ CPUFeatures* const cpu_features_;
+ const CPUFeatures old_features_;
+};
+
+
+} // namespace vixl
+
+#endif // VIXL_CPU_FEATURES_H
diff --git a/js/src/jit/arm64/vixl/Cpu-vixl.cpp b/js/src/jit/arm64/vixl/Cpu-vixl.cpp
new file mode 100644
index 0000000000..13b232d5b7
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Cpu-vixl.cpp
@@ -0,0 +1,252 @@
+// Copyright 2015, VIXL authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Cpu-vixl.h"
+
+#include "jstypes.h"
+
+#if defined(__aarch64__) && (defined(__ANDROID__) || defined(__linux__))
+#include <sys/auxv.h>
+#define VIXL_USE_LINUX_HWCAP 1
+#endif
+
+#include "jit/arm64/vixl/Utils-vixl.h"
+
+
+namespace vixl {
+
+
+const IDRegister::Field AA64PFR0::kFP(16, Field::kSigned);
+const IDRegister::Field AA64PFR0::kAdvSIMD(20, Field::kSigned);
+const IDRegister::Field AA64PFR0::kSVE(32);
+const IDRegister::Field AA64PFR0::kDIT(48);
+
+const IDRegister::Field AA64PFR1::kBT(0);
+
+const IDRegister::Field AA64ISAR0::kAES(4);
+const IDRegister::Field AA64ISAR0::kSHA1(8);
+const IDRegister::Field AA64ISAR0::kSHA2(12);
+const IDRegister::Field AA64ISAR0::kCRC32(16);
+const IDRegister::Field AA64ISAR0::kAtomic(20);
+const IDRegister::Field AA64ISAR0::kRDM(28);
+const IDRegister::Field AA64ISAR0::kSHA3(32);
+const IDRegister::Field AA64ISAR0::kSM3(36);
+const IDRegister::Field AA64ISAR0::kSM4(40);
+const IDRegister::Field AA64ISAR0::kDP(44);
+const IDRegister::Field AA64ISAR0::kFHM(48);
+const IDRegister::Field AA64ISAR0::kTS(52);
+
+const IDRegister::Field AA64ISAR1::kDPB(0);
+const IDRegister::Field AA64ISAR1::kAPA(4);
+const IDRegister::Field AA64ISAR1::kAPI(8);
+const IDRegister::Field AA64ISAR1::kJSCVT(12);
+const IDRegister::Field AA64ISAR1::kFCMA(16);
+const IDRegister::Field AA64ISAR1::kLRCPC(20);
+const IDRegister::Field AA64ISAR1::kGPA(24);
+const IDRegister::Field AA64ISAR1::kGPI(28);
+const IDRegister::Field AA64ISAR1::kFRINTTS(32);
+const IDRegister::Field AA64ISAR1::kSB(36);
+const IDRegister::Field AA64ISAR1::kSPECRES(40);
+
+const IDRegister::Field AA64MMFR1::kLO(16);
+
+CPUFeatures AA64PFR0::GetCPUFeatures() const {
+ CPUFeatures f;
+ if (Get(kFP) >= 0) f.Combine(CPUFeatures::kFP);
+ if (Get(kFP) >= 1) f.Combine(CPUFeatures::kFPHalf);
+ if (Get(kAdvSIMD) >= 0) f.Combine(CPUFeatures::kNEON);
+ if (Get(kAdvSIMD) >= 1) f.Combine(CPUFeatures::kNEONHalf);
+ if (Get(kSVE) >= 1) f.Combine(CPUFeatures::kSVE);
+ if (Get(kDIT) >= 1) f.Combine(CPUFeatures::kDIT);
+ return f;
+}
+
+CPUFeatures AA64PFR1::GetCPUFeatures() const {
+ CPUFeatures f;
+ if (Get(kBT) >= 1) f.Combine(CPUFeatures::kBTI);
+ return f;
+}
+
+CPUFeatures AA64ISAR0::GetCPUFeatures() const {
+ CPUFeatures f;
+ if (Get(kAES) >= 1) f.Combine(CPUFeatures::kAES);
+ if (Get(kAES) >= 2) f.Combine(CPUFeatures::kPmull1Q);
+ if (Get(kSHA1) >= 1) f.Combine(CPUFeatures::kSHA1);
+ if (Get(kSHA2) >= 1) f.Combine(CPUFeatures::kSHA2);
+ if (Get(kSHA2) >= 2) f.Combine(CPUFeatures::kSHA512);
+ if (Get(kCRC32) >= 1) f.Combine(CPUFeatures::kCRC32);
+ if (Get(kAtomic) >= 1) f.Combine(CPUFeatures::kAtomics);
+ if (Get(kRDM) >= 1) f.Combine(CPUFeatures::kRDM);
+ if (Get(kSHA3) >= 1) f.Combine(CPUFeatures::kSHA3);
+ if (Get(kSM3) >= 1) f.Combine(CPUFeatures::kSM3);
+ if (Get(kSM4) >= 1) f.Combine(CPUFeatures::kSM4);
+ if (Get(kDP) >= 1) f.Combine(CPUFeatures::kDotProduct);
+ if (Get(kFHM) >= 1) f.Combine(CPUFeatures::kFHM);
+ if (Get(kTS) >= 1) f.Combine(CPUFeatures::kFlagM);
+ if (Get(kTS) >= 2) f.Combine(CPUFeatures::kAXFlag);
+ return f;
+}
+
+CPUFeatures AA64ISAR1::GetCPUFeatures() const {
+ CPUFeatures f;
+ if (Get(kDPB) >= 1) f.Combine(CPUFeatures::kDCPoP);
+ if (Get(kJSCVT) >= 1) f.Combine(CPUFeatures::kJSCVT);
+ if (Get(kFCMA) >= 1) f.Combine(CPUFeatures::kFcma);
+ if (Get(kLRCPC) >= 1) f.Combine(CPUFeatures::kRCpc);
+ if (Get(kLRCPC) >= 2) f.Combine(CPUFeatures::kRCpcImm);
+ if (Get(kFRINTTS) >= 1) f.Combine(CPUFeatures::kFrintToFixedSizedInt);
+
+ if (Get(kAPI) >= 1) f.Combine(CPUFeatures::kPAuth);
+ if (Get(kAPA) >= 1) f.Combine(CPUFeatures::kPAuth, CPUFeatures::kPAuthQARMA);
+ if (Get(kGPI) >= 1) f.Combine(CPUFeatures::kPAuthGeneric);
+ if (Get(kGPA) >= 1) {
+ f.Combine(CPUFeatures::kPAuthGeneric, CPUFeatures::kPAuthGenericQARMA);
+ }
+ return f;
+}
+
+CPUFeatures AA64MMFR1::GetCPUFeatures() const {
+ CPUFeatures f;
+ if (Get(kLO) >= 1) f.Combine(CPUFeatures::kLORegions);
+ return f;
+}
+
+int IDRegister::Get(IDRegister::Field field) const {
+ int msb = field.GetMsb();
+ int lsb = field.GetLsb();
+ VIXL_STATIC_ASSERT(static_cast<size_t>(Field::kMaxWidthInBits) <
+ (sizeof(int) * kBitsPerByte));
+ switch (field.GetType()) {
+ case Field::kSigned:
+ return static_cast<int>(ExtractSignedBitfield64(msb, lsb, value_));
+ case Field::kUnsigned:
+ return static_cast<int>(ExtractUnsignedBitfield64(msb, lsb, value_));
+ }
+ VIXL_UNREACHABLE();
+ return 0;
+}
+
+CPUFeatures CPU::InferCPUFeaturesFromIDRegisters() {
+ CPUFeatures f;
+#define VIXL_COMBINE_ID_REG(NAME) f.Combine(Read##NAME().GetCPUFeatures());
+ VIXL_AARCH64_ID_REG_LIST(VIXL_COMBINE_ID_REG)
+#undef VIXL_COMBINE_ID_REG
+ return f;
+}
+
+CPUFeatures CPU::InferCPUFeaturesFromOS(
+ CPUFeatures::QueryIDRegistersOption option) {
+ CPUFeatures features;
+
+#if VIXL_USE_LINUX_HWCAP
+ // Map each set bit onto a feature. Ideally, we'd use HWCAP_* macros rather
+ // than explicit bits, but explicit bits allow us to identify features that
+ // the toolchain doesn't know about.
+ static const CPUFeatures::Feature kFeatureBits[] = {
+ // Bits 0-7
+ CPUFeatures::kFP,
+ CPUFeatures::kNEON,
+ CPUFeatures::kNone, // "EVTSTRM", which VIXL doesn't track.
+ CPUFeatures::kAES,
+ CPUFeatures::kPmull1Q,
+ CPUFeatures::kSHA1,
+ CPUFeatures::kSHA2,
+ CPUFeatures::kCRC32,
+ // Bits 8-15
+ CPUFeatures::kAtomics,
+ CPUFeatures::kFPHalf,
+ CPUFeatures::kNEONHalf,
+ CPUFeatures::kIDRegisterEmulation,
+ CPUFeatures::kRDM,
+ CPUFeatures::kJSCVT,
+ CPUFeatures::kFcma,
+ CPUFeatures::kRCpc,
+ // Bits 16-23
+ CPUFeatures::kDCPoP,
+ CPUFeatures::kSHA3,
+ CPUFeatures::kSM3,
+ CPUFeatures::kSM4,
+ CPUFeatures::kDotProduct,
+ CPUFeatures::kSHA512,
+ CPUFeatures::kSVE,
+ CPUFeatures::kFHM,
+ // Bits 24-27
+ CPUFeatures::kDIT,
+ CPUFeatures::kUSCAT,
+ CPUFeatures::kRCpcImm,
+ CPUFeatures::kFlagM
+ // Bits 28-31 are unassigned.
+ };
+ static const size_t kFeatureBitCount =
+ sizeof(kFeatureBits) / sizeof(kFeatureBits[0]);
+
+ // Mozilla change: Set the default for the simulator.
+#ifdef JS_SIMULATOR_ARM64
+ unsigned long auxv = ~(0UL); // Enable all features for the Simulator.
+#else
+ unsigned long auxv = getauxval(AT_HWCAP); // NOLINT(runtime/int)
+#endif
+
+ VIXL_STATIC_ASSERT(kFeatureBitCount < (sizeof(auxv) * kBitsPerByte));
+ for (size_t i = 0; i < kFeatureBitCount; i++) {
+ if (auxv & (1UL << i)) features.Combine(kFeatureBits[i]);
+ }
+#endif // VIXL_USE_LINUX_HWCAP
+
+ if ((option == CPUFeatures::kQueryIDRegistersIfAvailable) &&
+ (features.Has(CPUFeatures::kIDRegisterEmulation))) {
+ features.Combine(InferCPUFeaturesFromIDRegisters());
+ }
+ return features;
+}
+
+
+#ifdef __aarch64__
+#define VIXL_READ_ID_REG(NAME) \
+ NAME CPU::Read##NAME() { \
+ uint64_t value = 0; \
+ __asm__("mrs %0, ID_" #NAME "_EL1" : "=r"(value)); \
+ return NAME(value); \
+ }
+#else // __aarch64__
+#define VIXL_READ_ID_REG(NAME) \
+ NAME CPU::Read##NAME() { \
+ /* TODO: Use VIXL_UNREACHABLE once it works in release builds. */ \
+ VIXL_ABORT(); \
+ }
+#endif // __aarch64__
+
+VIXL_AARCH64_ID_REG_LIST(VIXL_READ_ID_REG)
+
+#undef VIXL_READ_ID_REG
+
+
+// Initialise to smallest possible cache size.
+unsigned CPU::dcache_line_size_ = 1;
+unsigned CPU::icache_line_size_ = 1;
+
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/Cpu-vixl.h b/js/src/jit/arm64/vixl/Cpu-vixl.h
new file mode 100644
index 0000000000..4db51aad6b
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Cpu-vixl.h
@@ -0,0 +1,241 @@
+// Copyright 2014, VIXL authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_CPU_AARCH64_H
+#define VIXL_CPU_AARCH64_H
+
+#include "jit/arm64/vixl/Cpu-Features-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+
+#include "jit/arm64/vixl/Instructions-vixl.h"
+
+#ifndef VIXL_INCLUDE_TARGET_AARCH64
+// The supporting .cc file is only compiled when the A64 target is selected.
+// Throw an explicit error now to avoid a harder-to-debug linker error later.
+//
+// These helpers _could_ work on any AArch64 host, even when generating AArch32
+// code, but we don't support this because the available features may differ
+// between AArch32 and AArch64 on the same platform, so basing AArch32 code
+// generation on aarch64::CPU features is probably broken.
+#error cpu-aarch64.h requires VIXL_INCLUDE_TARGET_AARCH64 (scons target=a64).
+#endif
+
+namespace vixl {
+
+// A CPU ID register, for use with CPUFeatures::kIDRegisterEmulation. Fields
+// specific to each register are described in relevant subclasses.
+class IDRegister {
+ protected:
+ explicit IDRegister(uint64_t value = 0) : value_(value) {}
+
+ class Field {
+ public:
+ enum Type { kUnsigned, kSigned };
+
+ explicit Field(int lsb, Type type = kUnsigned) : lsb_(lsb), type_(type) {}
+
+ static const int kMaxWidthInBits = 4;
+
+ int GetWidthInBits() const {
+ // All current ID fields have four bits.
+ return kMaxWidthInBits;
+ }
+ int GetLsb() const { return lsb_; }
+ int GetMsb() const { return lsb_ + GetWidthInBits() - 1; }
+ Type GetType() const { return type_; }
+
+ private:
+ int lsb_;
+ Type type_;
+ };
+
+ public:
+ // Extract the specified field, performing sign-extension for signed fields.
+ // This allows us to implement the 'value >= number' detection mechanism
+ // recommended by the Arm ARM, for both signed and unsigned fields.
+ int Get(Field field) const;
+
+ private:
+ uint64_t value_;
+};
+
+class AA64PFR0 : public IDRegister {
+ public:
+ explicit AA64PFR0(uint64_t value) : IDRegister(value) {}
+
+ CPUFeatures GetCPUFeatures() const;
+
+ private:
+ static const Field kFP;
+ static const Field kAdvSIMD;
+ static const Field kSVE;
+ static const Field kDIT;
+};
+
+class AA64PFR1 : public IDRegister {
+ public:
+ explicit AA64PFR1(uint64_t value) : IDRegister(value) {}
+
+ CPUFeatures GetCPUFeatures() const;
+
+ private:
+ static const Field kBT;
+};
+
+class AA64ISAR0 : public IDRegister {
+ public:
+ explicit AA64ISAR0(uint64_t value) : IDRegister(value) {}
+
+ CPUFeatures GetCPUFeatures() const;
+
+ private:
+ static const Field kAES;
+ static const Field kSHA1;
+ static const Field kSHA2;
+ static const Field kCRC32;
+ static const Field kAtomic;
+ static const Field kRDM;
+ static const Field kSHA3;
+ static const Field kSM3;
+ static const Field kSM4;
+ static const Field kDP;
+ static const Field kFHM;
+ static const Field kTS;
+};
+
+class AA64ISAR1 : public IDRegister {
+ public:
+ explicit AA64ISAR1(uint64_t value) : IDRegister(value) {}
+
+ CPUFeatures GetCPUFeatures() const;
+
+ private:
+ static const Field kDPB;
+ static const Field kAPA;
+ static const Field kAPI;
+ static const Field kJSCVT;
+ static const Field kFCMA;
+ static const Field kLRCPC;
+ static const Field kGPA;
+ static const Field kGPI;
+ static const Field kFRINTTS;
+ static const Field kSB;
+ static const Field kSPECRES;
+};
+
+class AA64MMFR1 : public IDRegister {
+ public:
+ explicit AA64MMFR1(uint64_t value) : IDRegister(value) {}
+
+ CPUFeatures GetCPUFeatures() const;
+
+ private:
+ static const Field kLO;
+};
+
+class CPU {
+ public:
+ // Initialise CPU support.
+ static void SetUp();
+
+ // Ensures the data at a given address and with a given size is the same for
+ // the I and D caches. I and D caches are not automatically coherent on ARM
+ // so this operation is required before any dynamically generated code can
+ // safely run.
+ static void EnsureIAndDCacheCoherency(void* address, size_t length);
+
+ // Flush the local instruction pipeline, forcing a reload of any instructions
+ // beyond this barrier from the icache.
+ static void FlushExecutionContext();
+
+ // Read and interpret the ID registers. This requires
+ // CPUFeatures::kIDRegisterEmulation, and therefore cannot be called on
+ // non-AArch64 platforms.
+ static CPUFeatures InferCPUFeaturesFromIDRegisters();
+
+ // Read and interpret CPUFeatures reported by the OS. Failed queries (or
+ // unsupported platforms) return an empty list. Note that this is
+ // indistinguishable from a successful query on a platform that advertises no
+ // features.
+ //
+ // Non-AArch64 hosts are considered to be unsupported platforms, and this
+ // function returns an empty list.
+ static CPUFeatures InferCPUFeaturesFromOS(
+ CPUFeatures::QueryIDRegistersOption option =
+ CPUFeatures::kQueryIDRegistersIfAvailable);
+
+ // Handle tagged pointers.
+ template <typename T>
+ static T SetPointerTag(T pointer, uint64_t tag) {
+ VIXL_ASSERT(IsUintN(kAddressTagWidth, tag));
+
+ // Use C-style casts to get static_cast behaviour for integral types (T),
+ // and reinterpret_cast behaviour for other types.
+
+ uint64_t raw = (uint64_t)pointer;
+ VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(raw));
+
+ raw = (raw & ~kAddressTagMask) | (tag << kAddressTagOffset);
+ return (T)raw;
+ }
+
+ template <typename T>
+ static uint64_t GetPointerTag(T pointer) {
+ // Use C-style casts to get static_cast behaviour for integral types (T),
+ // and reinterpret_cast behaviour for other types.
+
+ uint64_t raw = (uint64_t)pointer;
+ VIXL_STATIC_ASSERT(sizeof(pointer) == sizeof(raw));
+
+ return (raw & kAddressTagMask) >> kAddressTagOffset;
+ }
+
+ private:
+#define VIXL_AARCH64_ID_REG_LIST(V) \
+ V(AA64PFR0) \
+ V(AA64PFR1) \
+ V(AA64ISAR0) \
+ V(AA64ISAR1) \
+ V(AA64MMFR1)
+
+#define VIXL_READ_ID_REG(NAME) static NAME Read##NAME();
+ // On native AArch64 platforms, read the named CPU ID registers. These require
+ // CPUFeatures::kIDRegisterEmulation, and should not be called on non-AArch64
+ // platforms.
+ VIXL_AARCH64_ID_REG_LIST(VIXL_READ_ID_REG)
+#undef VIXL_READ_ID_REG
+
+ // Return the content of the cache type register.
+ static uint32_t GetCacheType();
+
+ // I and D cache line size in bytes.
+ static unsigned icache_line_size_;
+ static unsigned dcache_line_size_;
+};
+
+} // namespace vixl
+
+#endif // VIXL_CPU_AARCH64_H
diff --git a/js/src/jit/arm64/vixl/Debugger-vixl.cpp b/js/src/jit/arm64/vixl/Debugger-vixl.cpp
new file mode 100644
index 0000000000..fa3e15601e
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Debugger-vixl.cpp
@@ -0,0 +1,1535 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY ARM LIMITED AND CONTRIBUTORS "AS IS" AND ANY
+// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL ARM LIMITED BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA,
+// OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+// EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jstypes.h"
+
+#ifdef JS_SIMULATOR_ARM64
+
+#include "jit/arm64/vixl/Debugger-vixl.h"
+
+#include "mozilla/Vector.h"
+
+#include "js/AllocPolicy.h"
+
+namespace vixl {
+
+// List of commands supported by the debugger.
+#define DEBUG_COMMAND_LIST(C) \
+C(HelpCommand) \
+C(ContinueCommand) \
+C(StepCommand) \
+C(DisasmCommand) \
+C(PrintCommand) \
+C(ExamineCommand)
+
+// Debugger command lines are broken up in token of different type to make
+// processing easier later on.
+class Token {
+ public:
+ virtual ~Token() {}
+
+ // Token type.
+ virtual bool IsRegister() const { return false; }
+ virtual bool IsFPRegister() const { return false; }
+ virtual bool IsIdentifier() const { return false; }
+ virtual bool IsAddress() const { return false; }
+ virtual bool IsInteger() const { return false; }
+ virtual bool IsFormat() const { return false; }
+ virtual bool IsUnknown() const { return false; }
+ // Token properties.
+ virtual bool CanAddressMemory() const { return false; }
+ virtual uint8_t* ToAddress(Debugger* debugger) const = 0;
+ virtual void Print(FILE* out = stdout) const = 0;
+
+ static Token* Tokenize(const char* arg);
+};
+
+typedef mozilla::Vector<Token*, 0, js::SystemAllocPolicy> TokenVector;
+
+// Tokens often hold one value.
+template<typename T> class ValueToken : public Token {
+ public:
+ explicit ValueToken(T value) : value_(value) {}
+ ValueToken() {}
+
+ T value() const { return value_; }
+
+ virtual uint8_t* ToAddress(Debugger* debugger) const override {
+ USE(debugger);
+ VIXL_ABORT();
+ }
+
+ protected:
+ T value_;
+};
+
+// Integer registers (X or W) and their aliases.
+// Format: wn or xn with 0 <= n < 32 or a name in the aliases list.
+class RegisterToken : public ValueToken<const Register> {
+ public:
+ explicit RegisterToken(const Register reg)
+ : ValueToken<const Register>(reg) {}
+
+ virtual bool IsRegister() const override { return true; }
+ virtual bool CanAddressMemory() const override { return value().Is64Bits(); }
+ virtual uint8_t* ToAddress(Debugger* debugger) const override;
+ virtual void Print(FILE* out = stdout) const override;
+ const char* Name() const;
+
+ static Token* Tokenize(const char* arg);
+ static RegisterToken* Cast(Token* tok) {
+ VIXL_ASSERT(tok->IsRegister());
+ return reinterpret_cast<RegisterToken*>(tok);
+ }
+
+ private:
+ static const int kMaxAliasNumber = 4;
+ static const char* kXAliases[kNumberOfRegisters][kMaxAliasNumber];
+ static const char* kWAliases[kNumberOfRegisters][kMaxAliasNumber];
+};
+
+// Floating point registers (D or S).
+// Format: sn or dn with 0 <= n < 32.
+class FPRegisterToken : public ValueToken<const FPRegister> {
+ public:
+ explicit FPRegisterToken(const FPRegister fpreg)
+ : ValueToken<const FPRegister>(fpreg) {}
+
+ virtual bool IsFPRegister() const override { return true; }
+ virtual void Print(FILE* out = stdout) const override;
+
+ static Token* Tokenize(const char* arg);
+ static FPRegisterToken* Cast(Token* tok) {
+ VIXL_ASSERT(tok->IsFPRegister());
+ return reinterpret_cast<FPRegisterToken*>(tok);
+ }
+};
+
+
+// Non-register identifiers.
+// Format: Alphanumeric string starting with a letter.
+class IdentifierToken : public ValueToken<char*> {
+ public:
+ explicit IdentifierToken(const char* name) {
+ size_t size = strlen(name) + 1;
+ value_ = js_pod_malloc<char>(size);
+ strncpy(value_, name, size);
+ }
+ virtual ~IdentifierToken() { js_free(value_); }
+
+ virtual bool IsIdentifier() const override { return true; }
+ virtual bool CanAddressMemory() const override { return strcmp(value(), "pc") == 0; }
+ virtual uint8_t* ToAddress(Debugger* debugger) const override;
+ virtual void Print(FILE* out = stdout) const override;
+
+ static Token* Tokenize(const char* arg);
+ static IdentifierToken* Cast(Token* tok) {
+ VIXL_ASSERT(tok->IsIdentifier());
+ return reinterpret_cast<IdentifierToken*>(tok);
+ }
+};
+
+// 64-bit address literal.
+// Format: 0x... with up to 16 hexadecimal digits.
+class AddressToken : public ValueToken<uint8_t*> {
+ public:
+ explicit AddressToken(uint8_t* address) : ValueToken<uint8_t*>(address) {}
+
+ virtual bool IsAddress() const override { return true; }
+ virtual bool CanAddressMemory() const override { return true; }
+ virtual uint8_t* ToAddress(Debugger* debugger) const override;
+ virtual void Print(FILE* out = stdout) const override;
+
+ static Token* Tokenize(const char* arg);
+ static AddressToken* Cast(Token* tok) {
+ VIXL_ASSERT(tok->IsAddress());
+ return reinterpret_cast<AddressToken*>(tok);
+ }
+};
+
+
+// 64-bit decimal integer literal.
+// Format: n.
+class IntegerToken : public ValueToken<int64_t> {
+ public:
+ explicit IntegerToken(int64_t value) : ValueToken<int64_t>(value) {}
+
+ virtual bool IsInteger() const override { return true; }
+ virtual void Print(FILE* out = stdout) const override;
+
+ static Token* Tokenize(const char* arg);
+ static IntegerToken* Cast(Token* tok) {
+ VIXL_ASSERT(tok->IsInteger());
+ return reinterpret_cast<IntegerToken*>(tok);
+ }
+};
+
+// Literal describing how to print a chunk of data (up to 64 bits).
+// Format: .ln
+// where l (letter) is one of
+// * x: hexadecimal
+// * s: signed integer
+// * u: unsigned integer
+// * f: floating point
+// * i: instruction
+// and n (size) is one of 8, 16, 32 and 64. n should be omitted for
+// instructions.
+class FormatToken : public Token {
+ public:
+ FormatToken() {}
+
+ virtual bool IsFormat() const override { return true; }
+ virtual int SizeOf() const = 0;
+ virtual char type_code() const = 0;
+ virtual void PrintData(void* data, FILE* out = stdout) const = 0;
+ virtual void Print(FILE* out = stdout) const override = 0;
+
+ virtual uint8_t* ToAddress(Debugger* debugger) const override {
+ USE(debugger);
+ VIXL_ABORT();
+ }
+
+ static Token* Tokenize(const char* arg);
+ static FormatToken* Cast(Token* tok) {
+ VIXL_ASSERT(tok->IsFormat());
+ return reinterpret_cast<FormatToken*>(tok);
+ }
+};
+
+
+template<typename T> class Format : public FormatToken {
+ public:
+ Format(const char* fmt, char type_code) : fmt_(fmt), type_code_(type_code) {}
+
+ virtual int SizeOf() const override { return sizeof(T); }
+ virtual char type_code() const override { return type_code_; }
+ virtual void PrintData(void* data, FILE* out = stdout) const override {
+ T value;
+ memcpy(&value, data, sizeof(value));
+ fprintf(out, fmt_, value);
+ }
+ virtual void Print(FILE* out = stdout) const override;
+
+ private:
+ const char* fmt_;
+ char type_code_;
+};
+
+// Tokens which don't fit any of the above.
+class UnknownToken : public Token {
+ public:
+ explicit UnknownToken(const char* arg) {
+ size_t size = strlen(arg) + 1;
+ unknown_ = js_pod_malloc<char>(size);
+ strncpy(unknown_, arg, size);
+ }
+ virtual ~UnknownToken() { js_free(unknown_); }
+ virtual uint8_t* ToAddress(Debugger* debugger) const override {
+ USE(debugger);
+ VIXL_ABORT();
+ }
+
+ virtual bool IsUnknown() const override { return true; }
+ virtual void Print(FILE* out = stdout) const override;
+
+ private:
+ char* unknown_;
+};
+
+
+// All debugger commands must subclass DebugCommand and implement Run, Print
+// and Build. Commands must also define kHelp and kAliases.
+class DebugCommand {
+ public:
+ explicit DebugCommand(Token* name) : name_(IdentifierToken::Cast(name)) {}
+ DebugCommand() : name_(NULL) {}
+ virtual ~DebugCommand() { js_delete(name_); }
+
+ const char* name() { return name_->value(); }
+ // Run the command on the given debugger. The command returns true if
+ // execution should move to the next instruction.
+ virtual bool Run(Debugger * debugger) = 0;
+ virtual void Print(FILE* out = stdout);
+
+ static bool Match(const char* name, const char** aliases);
+ static DebugCommand* Parse(char* line);
+ static void PrintHelp(const char** aliases,
+ const char* args,
+ const char* help);
+
+ private:
+ IdentifierToken* name_;
+};
+
+// For all commands below see their respective kHelp and kAliases in
+// debugger-a64.cc
+class HelpCommand : public DebugCommand {
+ public:
+ explicit HelpCommand(Token* name) : DebugCommand(name) {}
+
+ virtual bool Run(Debugger* debugger) override;
+
+ static DebugCommand* Build(TokenVector&& args);
+
+ static const char* kHelp;
+ static const char* kAliases[];
+ static const char* kArguments;
+};
+
+
+class ContinueCommand : public DebugCommand {
+ public:
+ explicit ContinueCommand(Token* name) : DebugCommand(name) {}
+
+ virtual bool Run(Debugger* debugger) override;
+
+ static DebugCommand* Build(TokenVector&& args);
+
+ static const char* kHelp;
+ static const char* kAliases[];
+ static const char* kArguments;
+};
+
+
+class StepCommand : public DebugCommand {
+ public:
+ StepCommand(Token* name, IntegerToken* count)
+ : DebugCommand(name), count_(count) {}
+ virtual ~StepCommand() { js_delete(count_); }
+
+ int64_t count() { return count_->value(); }
+ virtual bool Run(Debugger* debugger) override;
+ virtual void Print(FILE* out = stdout) override;
+
+ static DebugCommand* Build(TokenVector&& args);
+
+ static const char* kHelp;
+ static const char* kAliases[];
+ static const char* kArguments;
+
+ private:
+ IntegerToken* count_;
+};
+
+class DisasmCommand : public DebugCommand {
+ public:
+ static DebugCommand* Build(TokenVector&& args);
+
+ static const char* kHelp;
+ static const char* kAliases[];
+ static const char* kArguments;
+};
+
+
+class PrintCommand : public DebugCommand {
+ public:
+ PrintCommand(Token* name, Token* target, FormatToken* format)
+ : DebugCommand(name), target_(target), format_(format) {}
+ virtual ~PrintCommand() {
+ js_delete(target_);
+ js_delete(format_);
+ }
+
+ Token* target() { return target_; }
+ FormatToken* format() { return format_; }
+ virtual bool Run(Debugger* debugger) override;
+ virtual void Print(FILE* out = stdout) override;
+
+ static DebugCommand* Build(TokenVector&& args);
+
+ static const char* kHelp;
+ static const char* kAliases[];
+ static const char* kArguments;
+
+ private:
+ Token* target_;
+ FormatToken* format_;
+};
+
+class ExamineCommand : public DebugCommand {
+ public:
+ ExamineCommand(Token* name,
+ Token* target,
+ FormatToken* format,
+ IntegerToken* count)
+ : DebugCommand(name), target_(target), format_(format), count_(count) {}
+ virtual ~ExamineCommand() {
+ js_delete(target_);
+ js_delete(format_);
+ js_delete(count_);
+ }
+
+ Token* target() { return target_; }
+ FormatToken* format() { return format_; }
+ IntegerToken* count() { return count_; }
+ virtual bool Run(Debugger* debugger) override;
+ virtual void Print(FILE* out = stdout) override;
+
+ static DebugCommand* Build(TokenVector&& args);
+
+ static const char* kHelp;
+ static const char* kAliases[];
+ static const char* kArguments;
+
+ private:
+ Token* target_;
+ FormatToken* format_;
+ IntegerToken* count_;
+};
+
+// Commands which name does not match any of the known commnand.
+class UnknownCommand : public DebugCommand {
+ public:
+ explicit UnknownCommand(TokenVector&& args) : args_(std::move(args)) {}
+ virtual ~UnknownCommand();
+
+ virtual bool Run(Debugger* debugger) override;
+
+ private:
+ TokenVector args_;
+};
+
+// Commands which name match a known command but the syntax is invalid.
+class InvalidCommand : public DebugCommand {
+ public:
+ InvalidCommand(TokenVector&& args, int index, const char* cause)
+ : args_(std::move(args)), index_(index), cause_(cause) {}
+ virtual ~InvalidCommand();
+
+ virtual bool Run(Debugger* debugger) override;
+
+ private:
+ TokenVector args_;
+ int index_;
+ const char* cause_;
+};
+
+const char* HelpCommand::kAliases[] = { "help", NULL };
+const char* HelpCommand::kArguments = NULL;
+const char* HelpCommand::kHelp = " Print this help.";
+
+const char* ContinueCommand::kAliases[] = { "continue", "c", NULL };
+const char* ContinueCommand::kArguments = NULL;
+const char* ContinueCommand::kHelp = " Resume execution.";
+
+const char* StepCommand::kAliases[] = { "stepi", "si", NULL };
+const char* StepCommand::kArguments = "[n = 1]";
+const char* StepCommand::kHelp = " Execute n next instruction(s).";
+
+const char* DisasmCommand::kAliases[] = { "disasm", "di", NULL };
+const char* DisasmCommand::kArguments = "[n = 10]";
+const char* DisasmCommand::kHelp =
+ " Disassemble n instruction(s) at pc.\n"
+ " This command is equivalent to x pc.i [n = 10]."
+;
+
+const char* PrintCommand::kAliases[] = { "print", "p", NULL };
+const char* PrintCommand::kArguments = "<entity>[.format]";
+const char* PrintCommand::kHelp =
+ " Print the given entity according to the given format.\n"
+ " The format parameter only affects individual registers; it is ignored\n"
+ " for other entities.\n"
+ " <entity> can be one of the following:\n"
+ " * A register name (such as x0, s1, ...).\n"
+ " * 'regs', to print all integer (W and X) registers.\n"
+ " * 'fpregs' to print all floating-point (S and D) registers.\n"
+ " * 'sysregs' to print all system registers (including NZCV).\n"
+ " * 'pc' to print the current program counter.\n"
+;
+
+const char* ExamineCommand::kAliases[] = { "m", "mem", "x", NULL };
+const char* ExamineCommand::kArguments = "<addr>[.format] [n = 10]";
+const char* ExamineCommand::kHelp =
+ " Examine memory. Print n items of memory at address <addr> according to\n"
+ " the given [.format].\n"
+ " Addr can be an immediate address, a register name or pc.\n"
+ " Format is made of a type letter: 'x' (hexadecimal), 's' (signed), 'u'\n"
+ " (unsigned), 'f' (floating point), i (instruction) and a size in bits\n"
+ " when appropriate (8, 16, 32, 64)\n"
+ " E.g 'x sp.x64' will print 10 64-bit words from the stack in\n"
+ " hexadecimal format."
+;
+
+const char* RegisterToken::kXAliases[kNumberOfRegisters][kMaxAliasNumber] = {
+ { "x0", NULL },
+ { "x1", NULL },
+ { "x2", NULL },
+ { "x3", NULL },
+ { "x4", NULL },
+ { "x5", NULL },
+ { "x6", NULL },
+ { "x7", NULL },
+ { "x8", NULL },
+ { "x9", NULL },
+ { "x10", NULL },
+ { "x11", NULL },
+ { "x12", NULL },
+ { "x13", NULL },
+ { "x14", NULL },
+ { "x15", NULL },
+ { "ip0", "x16", NULL },
+ { "ip1", "x17", NULL },
+ { "x18", "pr", NULL },
+ { "x19", NULL },
+ { "x20", NULL },
+ { "x21", NULL },
+ { "x22", NULL },
+ { "x23", NULL },
+ { "x24", NULL },
+ { "x25", NULL },
+ { "x26", NULL },
+ { "x27", NULL },
+ { "x28", NULL },
+ { "fp", "x29", NULL },
+ { "lr", "x30", NULL },
+ { "sp", NULL}
+};
+
+const char* RegisterToken::kWAliases[kNumberOfRegisters][kMaxAliasNumber] = {
+ { "w0", NULL },
+ { "w1", NULL },
+ { "w2", NULL },
+ { "w3", NULL },
+ { "w4", NULL },
+ { "w5", NULL },
+ { "w6", NULL },
+ { "w7", NULL },
+ { "w8", NULL },
+ { "w9", NULL },
+ { "w10", NULL },
+ { "w11", NULL },
+ { "w12", NULL },
+ { "w13", NULL },
+ { "w14", NULL },
+ { "w15", NULL },
+ { "w16", NULL },
+ { "w17", NULL },
+ { "w18", NULL },
+ { "w19", NULL },
+ { "w20", NULL },
+ { "w21", NULL },
+ { "w22", NULL },
+ { "w23", NULL },
+ { "w24", NULL },
+ { "w25", NULL },
+ { "w26", NULL },
+ { "w27", NULL },
+ { "w28", NULL },
+ { "w29", NULL },
+ { "w30", NULL },
+ { "wsp", NULL }
+};
+
+
+Debugger::Debugger(Decoder* decoder, FILE* stream)
+ : Simulator(decoder, stream),
+ debug_parameters_(DBG_INACTIVE),
+ pending_request_(false),
+ steps_(0),
+ last_command_(NULL) {
+ disasm_ = js_new<PrintDisassembler>(stdout);
+ printer_ = js_new<Decoder>();
+ printer_->AppendVisitor(disasm_);
+}
+
+
+Debugger::~Debugger() {
+ js_delete(disasm_);
+ js_delete(printer_);
+}
+
+
+void Debugger::Run() {
+ pc_modified_ = false;
+ while (pc_ != kEndOfSimAddress) {
+ if (pending_request()) RunDebuggerShell();
+ ExecuteInstruction();
+ LogAllWrittenRegisters();
+ }
+}
+
+
+void Debugger::PrintInstructions(const void* address, int64_t count) {
+ if (count == 0) {
+ return;
+ }
+
+ const Instruction* from = Instruction::CastConst(address);
+ if (count < 0) {
+ count = -count;
+ from -= (count - 1) * kInstructionSize;
+ }
+ const Instruction* to = from + count * kInstructionSize;
+
+ for (const Instruction* current = from;
+ current < to;
+ current = current->NextInstruction()) {
+ printer_->Decode(current);
+ }
+}
+
+
+void Debugger::PrintMemory(const uint8_t* address,
+ const FormatToken* format,
+ int64_t count) {
+ if (count == 0) {
+ return;
+ }
+
+ const uint8_t* from = address;
+ int size = format->SizeOf();
+ if (count < 0) {
+ count = -count;
+ from -= (count - 1) * size;
+ }
+ const uint8_t* to = from + count * size;
+
+ for (const uint8_t* current = from; current < to; current += size) {
+ if (((current - from) % 8) == 0) {
+ printf("\n%p: ", current);
+ }
+
+ uint64_t data = Memory::Read<uint64_t>(current);
+ format->PrintData(&data);
+ printf(" ");
+ }
+ printf("\n\n");
+}
+
+
+void Debugger::PrintRegister(const Register& target_reg,
+ const char* name,
+ const FormatToken* format) {
+ const uint64_t reg_size = target_reg.size();
+ const uint64_t format_size = format->SizeOf() * 8;
+ const uint64_t count = reg_size / format_size;
+ const uint64_t mask = 0xffffffffffffffff >> (64 - format_size);
+ const uint64_t reg_value = reg<uint64_t>(target_reg.code(),
+ Reg31IsStackPointer);
+ VIXL_ASSERT(count > 0);
+
+ printf("%s = ", name);
+ for (uint64_t i = 1; i <= count; i++) {
+ uint64_t data = reg_value >> (reg_size - (i * format_size));
+ data &= mask;
+ format->PrintData(&data);
+ printf(" ");
+ }
+ printf("\n");
+}
+
+
+// TODO(all): fix this for vector registers.
+void Debugger::PrintFPRegister(const FPRegister& target_fpreg,
+ const FormatToken* format) {
+ const unsigned fpreg_size = target_fpreg.size();
+ const uint64_t format_size = format->SizeOf() * 8;
+ const uint64_t count = fpreg_size / format_size;
+ const uint64_t mask = 0xffffffffffffffff >> (64 - format_size);
+ const uint64_t fpreg_value = vreg<uint64_t>(fpreg_size, target_fpreg.code());
+ VIXL_ASSERT(count > 0);
+
+ if (target_fpreg.Is32Bits()) {
+ printf("s%u = ", target_fpreg.code());
+ } else {
+ printf("d%u = ", target_fpreg.code());
+ }
+ for (uint64_t i = 1; i <= count; i++) {
+ uint64_t data = fpreg_value >> (fpreg_size - (i * format_size));
+ data &= mask;
+ format->PrintData(&data);
+ printf(" ");
+ }
+ printf("\n");
+}
+
+
+void Debugger::VisitException(const Instruction* instr) {
+ switch (instr->Mask(ExceptionMask)) {
+ case BRK:
+ DoBreakpoint(instr);
+ return;
+ case HLT:
+ VIXL_FALLTHROUGH();
+ default: Simulator::VisitException(instr);
+ }
+}
+
+
+// Read a command. A command will be at most kMaxDebugShellLine char long and
+// ends with '\n\0'.
+// TODO: Should this be a utility function?
+char* Debugger::ReadCommandLine(const char* prompt, char* buffer, int length) {
+ int fgets_calls = 0;
+ char* end = NULL;
+
+ printf("%s", prompt);
+ fflush(stdout);
+
+ do {
+ if (fgets(buffer, length, stdin) == NULL) {
+ printf(" ** Error while reading command. **\n");
+ return NULL;
+ }
+
+ fgets_calls++;
+ end = strchr(buffer, '\n');
+ } while (end == NULL);
+
+ if (fgets_calls != 1) {
+ printf(" ** Command too long. **\n");
+ return NULL;
+ }
+
+ // Remove the newline from the end of the command.
+ VIXL_ASSERT(end[1] == '\0');
+ VIXL_ASSERT((end - buffer) < (length - 1));
+ end[0] = '\0';
+
+ return buffer;
+}
+
+
+void Debugger::RunDebuggerShell() {
+ if (IsDebuggerRunning()) {
+ if (steps_ > 0) {
+ // Finish stepping first.
+ --steps_;
+ return;
+ }
+
+ printf("Next: ");
+ PrintInstructions(pc());
+ bool done = false;
+ while (!done) {
+ char buffer[kMaxDebugShellLine];
+ char* line = ReadCommandLine("vixl> ", buffer, kMaxDebugShellLine);
+
+ if (line == NULL) continue; // An error occurred.
+
+ DebugCommand* command = DebugCommand::Parse(line);
+ if (command != NULL) {
+ last_command_ = command;
+ }
+
+ if (last_command_ != NULL) {
+ done = last_command_->Run(this);
+ } else {
+ printf("No previous command to run!\n");
+ }
+ }
+
+ if ((debug_parameters_ & DBG_BREAK) != 0) {
+ // The break request has now been handled, move to next instruction.
+ debug_parameters_ &= ~DBG_BREAK;
+ increment_pc();
+ }
+ }
+}
+
+
+void Debugger::DoBreakpoint(const Instruction* instr) {
+ VIXL_ASSERT(instr->Mask(ExceptionMask) == BRK);
+
+ printf("Hit breakpoint at pc=%p.\n", reinterpret_cast<const void*>(instr));
+ set_debug_parameters(debug_parameters() | DBG_BREAK | DBG_ACTIVE);
+ // Make the shell point to the brk instruction.
+ set_pc(instr);
+}
+
+
+static bool StringToUInt64(uint64_t* value, const char* line, int base = 10) {
+ char* endptr = NULL;
+ errno = 0; // Reset errors.
+ uint64_t parsed = strtoul(line, &endptr, base);
+
+ if (errno == ERANGE) {
+ // Overflow.
+ return false;
+ }
+
+ if (endptr == line) {
+ // No digits were parsed.
+ return false;
+ }
+
+ if (*endptr != '\0') {
+ // Non-digit characters present at the end.
+ return false;
+ }
+
+ *value = parsed;
+ return true;
+}
+
+
+static bool StringToInt64(int64_t* value, const char* line, int base = 10) {
+ char* endptr = NULL;
+ errno = 0; // Reset errors.
+ int64_t parsed = strtol(line, &endptr, base);
+
+ if (errno == ERANGE) {
+ // Overflow, undeflow.
+ return false;
+ }
+
+ if (endptr == line) {
+ // No digits were parsed.
+ return false;
+ }
+
+ if (*endptr != '\0') {
+ // Non-digit characters present at the end.
+ return false;
+ }
+
+ *value = parsed;
+ return true;
+}
+
+
+Token* Token::Tokenize(const char* arg) {
+ if ((arg == NULL) || (*arg == '\0')) {
+ return NULL;
+ }
+
+ // The order is important. For example Identifier::Tokenize would consider
+ // any register to be a valid identifier.
+
+ Token* token = RegisterToken::Tokenize(arg);
+ if (token != NULL) {
+ return token;
+ }
+
+ token = FPRegisterToken::Tokenize(arg);
+ if (token != NULL) {
+ return token;
+ }
+
+ token = IdentifierToken::Tokenize(arg);
+ if (token != NULL) {
+ return token;
+ }
+
+ token = AddressToken::Tokenize(arg);
+ if (token != NULL) {
+ return token;
+ }
+
+ token = IntegerToken::Tokenize(arg);
+ if (token != NULL) {
+ return token;
+ }
+
+ return js_new<UnknownToken>(arg);
+}
+
+
+uint8_t* RegisterToken::ToAddress(Debugger* debugger) const {
+ VIXL_ASSERT(CanAddressMemory());
+ uint64_t reg_value = debugger->xreg(value().code(), Reg31IsStackPointer);
+ uint8_t* address = NULL;
+ memcpy(&address, &reg_value, sizeof(address));
+ return address;
+}
+
+
+void RegisterToken::Print(FILE* out) const {
+ VIXL_ASSERT(value().IsValid());
+ fprintf(out, "[Register %s]", Name());
+}
+
+
+const char* RegisterToken::Name() const {
+ if (value().Is32Bits()) {
+ return kWAliases[value().code()][0];
+ } else {
+ return kXAliases[value().code()][0];
+ }
+}
+
+
+Token* RegisterToken::Tokenize(const char* arg) {
+ for (unsigned i = 0; i < kNumberOfRegisters; i++) {
+ // Is it a X register or alias?
+ for (const char** current = kXAliases[i]; *current != NULL; current++) {
+ if (strcmp(arg, *current) == 0) {
+ return js_new<RegisterToken>(Register::XRegFromCode(i));
+ }
+ }
+
+ // Is it a W register or alias?
+ for (const char** current = kWAliases[i]; *current != NULL; current++) {
+ if (strcmp(arg, *current) == 0) {
+ return js_new<RegisterToken>(Register::WRegFromCode(i));
+ }
+ }
+ }
+
+ return NULL;
+}
+
+
+void FPRegisterToken::Print(FILE* out) const {
+ VIXL_ASSERT(value().IsValid());
+ char prefix = value().Is32Bits() ? 's' : 'd';
+ fprintf(out, "[FPRegister %c%" PRIu32 "]", prefix, value().code());
+}
+
+
+Token* FPRegisterToken::Tokenize(const char* arg) {
+ if (strlen(arg) < 2) {
+ return NULL;
+ }
+
+ switch (*arg) {
+ case 's':
+ case 'd':
+ const char* cursor = arg + 1;
+ uint64_t code = 0;
+ if (!StringToUInt64(&code, cursor)) {
+ return NULL;
+ }
+
+ if (code > kNumberOfFPRegisters) {
+ return NULL;
+ }
+
+ VRegister fpreg = NoVReg;
+ switch (*arg) {
+ case 's':
+ fpreg = VRegister::SRegFromCode(static_cast<unsigned>(code));
+ break;
+ case 'd':
+ fpreg = VRegister::DRegFromCode(static_cast<unsigned>(code));
+ break;
+ default: VIXL_UNREACHABLE();
+ }
+
+ return js_new<FPRegisterToken>(fpreg);
+ }
+
+ return NULL;
+}
+
+
+uint8_t* IdentifierToken::ToAddress(Debugger* debugger) const {
+ VIXL_ASSERT(CanAddressMemory());
+ const Instruction* pc_value = debugger->pc();
+ uint8_t* address = NULL;
+ memcpy(&address, &pc_value, sizeof(address));
+ return address;
+}
+
+void IdentifierToken::Print(FILE* out) const {
+ fprintf(out, "[Identifier %s]", value());
+}
+
+
+Token* IdentifierToken::Tokenize(const char* arg) {
+ if (!isalpha(arg[0])) {
+ return NULL;
+ }
+
+ const char* cursor = arg + 1;
+ while ((*cursor != '\0') && isalnum(*cursor)) {
+ ++cursor;
+ }
+
+ if (*cursor == '\0') {
+ return js_new<IdentifierToken>(arg);
+ }
+
+ return NULL;
+}
+
+
+uint8_t* AddressToken::ToAddress(Debugger* debugger) const {
+ USE(debugger);
+ return value();
+}
+
+
+void AddressToken::Print(FILE* out) const {
+ fprintf(out, "[Address %p]", value());
+}
+
+
+Token* AddressToken::Tokenize(const char* arg) {
+ if ((strlen(arg) < 3) || (arg[0] != '0') || (arg[1] != 'x')) {
+ return NULL;
+ }
+
+ uint64_t ptr = 0;
+ if (!StringToUInt64(&ptr, arg, 16)) {
+ return NULL;
+ }
+
+ uint8_t* address = reinterpret_cast<uint8_t*>(ptr);
+ return js_new<AddressToken>(address);
+}
+
+
+void IntegerToken::Print(FILE* out) const {
+ fprintf(out, "[Integer %" PRId64 "]", value());
+}
+
+
+Token* IntegerToken::Tokenize(const char* arg) {
+ int64_t value = 0;
+ if (!StringToInt64(&value, arg)) {
+ return NULL;
+ }
+
+ return js_new<IntegerToken>(value);
+}
+
+
+Token* FormatToken::Tokenize(const char* arg) {
+ size_t length = strlen(arg);
+ switch (arg[0]) {
+ case 'x':
+ case 's':
+ case 'u':
+ case 'f':
+ if (length == 1) return NULL;
+ break;
+ case 'i':
+ if (length == 1) return js_new<Format<uint32_t>>("%08" PRIx32, 'i');
+ VIXL_FALLTHROUGH();
+ default: return NULL;
+ }
+
+ char* endptr = NULL;
+ errno = 0; // Reset errors.
+ uint64_t count = strtoul(arg + 1, &endptr, 10);
+
+ if (errno != 0) {
+ // Overflow, etc.
+ return NULL;
+ }
+
+ if (endptr == arg) {
+ // No digits were parsed.
+ return NULL;
+ }
+
+ if (*endptr != '\0') {
+ // There are unexpected (non-digit) characters after the number.
+ return NULL;
+ }
+
+ switch (arg[0]) {
+ case 'x':
+ switch (count) {
+ case 8: return js_new<Format<uint8_t>>("%02" PRIx8, 'x');
+ case 16: return js_new<Format<uint16_t>>("%04" PRIx16, 'x');
+ case 32: return js_new<Format<uint32_t>>("%08" PRIx32, 'x');
+ case 64: return js_new<Format<uint64_t>>("%016" PRIx64, 'x');
+ default: return NULL;
+ }
+ case 's':
+ switch (count) {
+ case 8: return js_new<Format<int8_t>>("%4" PRId8, 's');
+ case 16: return js_new<Format<int16_t>>("%6" PRId16, 's');
+ case 32: return js_new<Format<int32_t>>("%11" PRId32, 's');
+ case 64: return js_new<Format<int64_t>>("%20" PRId64, 's');
+ default: return NULL;
+ }
+ case 'u':
+ switch (count) {
+ case 8: return js_new<Format<uint8_t>>("%3" PRIu8, 'u');
+ case 16: return js_new<Format<uint16_t>>("%5" PRIu16, 'u');
+ case 32: return js_new<Format<uint32_t>>("%10" PRIu32, 'u');
+ case 64: return js_new<Format<uint64_t>>("%20" PRIu64, 'u');
+ default: return NULL;
+ }
+ case 'f':
+ switch (count) {
+ case 32: return js_new<Format<float>>("%13g", 'f');
+ case 64: return js_new<Format<double>>("%13g", 'f');
+ default: return NULL;
+ }
+ default:
+ VIXL_UNREACHABLE();
+ return NULL;
+ }
+}
+
+
+template<typename T>
+void Format<T>::Print(FILE* out) const {
+ unsigned size = sizeof(T) * 8;
+ fprintf(out, "[Format %c%u - %s]", type_code_, size, fmt_);
+}
+
+
+void UnknownToken::Print(FILE* out) const {
+ fprintf(out, "[Unknown %s]", unknown_);
+}
+
+
+void DebugCommand::Print(FILE* out) {
+ fprintf(out, "%s", name());
+}
+
+
+bool DebugCommand::Match(const char* name, const char** aliases) {
+ for (const char** current = aliases; *current != NULL; current++) {
+ if (strcmp(name, *current) == 0) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+
+DebugCommand* DebugCommand::Parse(char* line) {
+ TokenVector args;
+
+ for (char* chunk = strtok(line, " \t");
+ chunk != NULL;
+ chunk = strtok(NULL, " \t")) {
+ char* dot = strchr(chunk, '.');
+ if (dot != NULL) {
+ // 'Token.format'.
+ Token* format = FormatToken::Tokenize(dot + 1);
+ if (format != NULL) {
+ *dot = '\0';
+ (void)args.append(Token::Tokenize(chunk));
+ (void)args.append(format);
+ } else {
+ // Error while parsing the format, push the UnknownToken so an error
+ // can be accurately reported.
+ (void)args.append(Token::Tokenize(chunk));
+ }
+ } else {
+ (void)args.append(Token::Tokenize(chunk));
+ }
+ }
+
+ if (args.empty()) {
+ return NULL;
+ }
+
+ if (!args[0]->IsIdentifier()) {
+ return js_new<InvalidCommand>(std::move(args), 0, "command name is not valid");
+ }
+
+ const char* name = IdentifierToken::Cast(args[0])->value();
+ #define RETURN_IF_MATCH(Command) \
+ if (Match(name, Command::kAliases)) { \
+ return Command::Build(std::move(args)); \
+ }
+ DEBUG_COMMAND_LIST(RETURN_IF_MATCH);
+ #undef RETURN_IF_MATCH
+
+ return js_new<UnknownCommand>(std::move(args));
+}
+
+
+void DebugCommand::PrintHelp(const char** aliases,
+ const char* args,
+ const char* help) {
+ VIXL_ASSERT(aliases[0] != NULL);
+ VIXL_ASSERT(help != NULL);
+
+ printf("\n----\n\n");
+ for (const char** current = aliases; *current != NULL; current++) {
+ if (args != NULL) {
+ printf("%s %s\n", *current, args);
+ } else {
+ printf("%s\n", *current);
+ }
+ }
+ printf("\n%s\n", help);
+}
+
+
+bool HelpCommand::Run(Debugger* debugger) {
+ VIXL_ASSERT(debugger->IsDebuggerRunning());
+ USE(debugger);
+
+ #define PRINT_HELP(Command) \
+ DebugCommand::PrintHelp(Command::kAliases, \
+ Command::kArguments, \
+ Command::kHelp);
+ DEBUG_COMMAND_LIST(PRINT_HELP);
+ #undef PRINT_HELP
+ printf("\n----\n\n");
+
+ return false;
+}
+
+
+DebugCommand* HelpCommand::Build(TokenVector&& args) {
+ if (args.length() != 1) {
+ return js_new<InvalidCommand>(std::move(args), -1, "too many arguments");
+ }
+
+ return js_new<HelpCommand>(args[0]);
+}
+
+
+bool ContinueCommand::Run(Debugger* debugger) {
+ VIXL_ASSERT(debugger->IsDebuggerRunning());
+
+ debugger->set_debug_parameters(debugger->debug_parameters() & ~DBG_ACTIVE);
+ return true;
+}
+
+
+DebugCommand* ContinueCommand::Build(TokenVector&& args) {
+ if (args.length() != 1) {
+ return js_new<InvalidCommand>(std::move(args), -1, "too many arguments");
+ }
+
+ return js_new<ContinueCommand>(args[0]);
+}
+
+
+bool StepCommand::Run(Debugger* debugger) {
+ VIXL_ASSERT(debugger->IsDebuggerRunning());
+
+ int64_t steps = count();
+ if (steps < 0) {
+ printf(" ** invalid value for steps: %" PRId64 " (<0) **\n", steps);
+ } else if (steps > 1) {
+ debugger->set_steps(steps - 1);
+ }
+
+ return true;
+}
+
+
+void StepCommand::Print(FILE* out) {
+ fprintf(out, "%s %" PRId64 "", name(), count());
+}
+
+
+DebugCommand* StepCommand::Build(TokenVector&& args) {
+ IntegerToken* count = NULL;
+ switch (args.length()) {
+ case 1: { // step [1]
+ count = js_new<IntegerToken>(1);
+ break;
+ }
+ case 2: { // step n
+ Token* first = args[1];
+ if (!first->IsInteger()) {
+ return js_new<InvalidCommand>(std::move(args), 1, "expects int");
+ }
+ count = IntegerToken::Cast(first);
+ break;
+ }
+ default:
+ return js_new<InvalidCommand>(std::move(args), -1, "too many arguments");
+ }
+
+ return js_new<StepCommand>(args[0], count);
+}
+
+
+DebugCommand* DisasmCommand::Build(TokenVector&& args) {
+ IntegerToken* count = NULL;
+ switch (args.length()) {
+ case 1: { // disasm [10]
+ count = js_new<IntegerToken>(10);
+ break;
+ }
+ case 2: { // disasm n
+ Token* first = args[1];
+ if (!first->IsInteger()) {
+ return js_new<InvalidCommand>(std::move(args), 1, "expects int");
+ }
+
+ count = IntegerToken::Cast(first);
+ break;
+ }
+ default:
+ return js_new<InvalidCommand>(std::move(args), -1, "too many arguments");
+ }
+
+ Token* target = js_new<IdentifierToken>("pc");
+ FormatToken* format = js_new<Format<uint32_t>>("%08" PRIx32, 'i');
+ return js_new<ExamineCommand>(args[0], target, format, count);
+}
+
+
+void PrintCommand::Print(FILE* out) {
+ fprintf(out, "%s ", name());
+ target()->Print(out);
+ if (format() != NULL) format()->Print(out);
+}
+
+
+bool PrintCommand::Run(Debugger* debugger) {
+ VIXL_ASSERT(debugger->IsDebuggerRunning());
+
+ Token* tok = target();
+ if (tok->IsIdentifier()) {
+ char* identifier = IdentifierToken::Cast(tok)->value();
+ if (strcmp(identifier, "regs") == 0) {
+ debugger->PrintRegisters();
+ } else if (strcmp(identifier, "fpregs") == 0) {
+ debugger->PrintVRegisters();
+ } else if (strcmp(identifier, "sysregs") == 0) {
+ debugger->PrintSystemRegisters();
+ } else if (strcmp(identifier, "pc") == 0) {
+ printf("pc = %16p\n", reinterpret_cast<const void*>(debugger->pc()));
+ } else {
+ printf(" ** Unknown identifier to print: %s **\n", identifier);
+ }
+
+ return false;
+ }
+
+ FormatToken* format_tok = format();
+ VIXL_ASSERT(format_tok != NULL);
+ if (format_tok->type_code() == 'i') {
+ // TODO(all): Add support for instruction disassembly.
+ printf(" ** unsupported format: instructions **\n");
+ return false;
+ }
+
+ if (tok->IsRegister()) {
+ RegisterToken* reg_tok = RegisterToken::Cast(tok);
+ Register reg = reg_tok->value();
+ debugger->PrintRegister(reg, reg_tok->Name(), format_tok);
+ return false;
+ }
+
+ if (tok->IsFPRegister()) {
+ FPRegister fpreg = FPRegisterToken::Cast(tok)->value();
+ debugger->PrintFPRegister(fpreg, format_tok);
+ return false;
+ }
+
+ VIXL_UNREACHABLE();
+ return false;
+}
+
+
+DebugCommand* PrintCommand::Build(TokenVector&& args) {
+ if (args.length() < 2) {
+ return js_new<InvalidCommand>(std::move(args), -1, "too few arguments");
+ }
+
+ Token* target = args[1];
+ if (!target->IsRegister() &&
+ !target->IsFPRegister() &&
+ !target->IsIdentifier()) {
+ return js_new<InvalidCommand>(std::move(args), 1, "expects reg or identifier");
+ }
+
+ FormatToken* format = NULL;
+ int target_size = 0;
+ if (target->IsRegister()) {
+ Register reg = RegisterToken::Cast(target)->value();
+ target_size = reg.SizeInBytes();
+ } else if (target->IsFPRegister()) {
+ FPRegister fpreg = FPRegisterToken::Cast(target)->value();
+ target_size = fpreg.SizeInBytes();
+ }
+ // If the target is an identifier there must be no format. This is checked
+ // in the switch statement below.
+
+ switch (args.length()) {
+ case 2: {
+ if (target->IsRegister()) {
+ switch (target_size) {
+ case 4: format = js_new<Format<uint32_t>>("%08" PRIx32, 'x'); break;
+ case 8: format = js_new<Format<uint64_t>>("%016" PRIx64, 'x'); break;
+ default: VIXL_UNREACHABLE();
+ }
+ } else if (target->IsFPRegister()) {
+ switch (target_size) {
+ case 4: format = js_new<Format<float>>("%8g", 'f'); break;
+ case 8: format = js_new<Format<double>>("%8g", 'f'); break;
+ default: VIXL_UNREACHABLE();
+ }
+ }
+ break;
+ }
+ case 3: {
+ if (target->IsIdentifier()) {
+ return js_new<InvalidCommand>(std::move(args), 2,
+ "format is only allowed with registers");
+ }
+
+ Token* second = args[2];
+ if (!second->IsFormat()) {
+ return js_new<InvalidCommand>(std::move(args), 2, "expects format");
+ }
+ format = FormatToken::Cast(second);
+
+ if (format->SizeOf() > target_size) {
+ return js_new<InvalidCommand>(std::move(args), 2, "format too wide");
+ }
+
+ break;
+ }
+ default:
+ return js_new<InvalidCommand>(std::move(args), -1, "too many arguments");
+ }
+
+ return js_new<PrintCommand>(args[0], target, format);
+}
+
+
+bool ExamineCommand::Run(Debugger* debugger) {
+ VIXL_ASSERT(debugger->IsDebuggerRunning());
+
+ uint8_t* address = target()->ToAddress(debugger);
+ int64_t amount = count()->value();
+ if (format()->type_code() == 'i') {
+ debugger->PrintInstructions(address, amount);
+ } else {
+ debugger->PrintMemory(address, format(), amount);
+ }
+
+ return false;
+}
+
+
+void ExamineCommand::Print(FILE* out) {
+ fprintf(out, "%s ", name());
+ format()->Print(out);
+ target()->Print(out);
+}
+
+
+DebugCommand* ExamineCommand::Build(TokenVector&& args) {
+ if (args.length() < 2) {
+ return js_new<InvalidCommand>(std::move(args), -1, "too few arguments");
+ }
+
+ Token* target = args[1];
+ if (!target->CanAddressMemory()) {
+ return js_new<InvalidCommand>(std::move(args), 1, "expects address");
+ }
+
+ FormatToken* format = NULL;
+ IntegerToken* count = NULL;
+
+ switch (args.length()) {
+ case 2: { // mem addr[.x64] [10]
+ format = js_new<Format<uint64_t>>("%016" PRIx64, 'x');
+ count = js_new<IntegerToken>(10);
+ break;
+ }
+ case 3: { // mem addr.format [10]
+ // mem addr[.x64] n
+ Token* second = args[2];
+ if (second->IsFormat()) {
+ format = FormatToken::Cast(second);
+ count = js_new<IntegerToken>(10);
+ break;
+ } else if (second->IsInteger()) {
+ format = js_new<Format<uint64_t>>("%016" PRIx64, 'x');
+ count = IntegerToken::Cast(second);
+ } else {
+ return js_new<InvalidCommand>(std::move(args), 2, "expects format or integer");
+ }
+ VIXL_UNREACHABLE();
+ break;
+ }
+ case 4: { // mem addr.format n
+ Token* second = args[2];
+ Token* third = args[3];
+ if (!second->IsFormat() || !third->IsInteger()) {
+ return js_new<InvalidCommand>(std::move(args), -1, "expects addr[.format] [n]");
+ }
+ format = FormatToken::Cast(second);
+ count = IntegerToken::Cast(third);
+ break;
+ }
+ default:
+ return js_new<InvalidCommand>(std::move(args), -1, "too many arguments");
+ }
+
+ return js_new<ExamineCommand>(args[0], target, format, count);
+}
+
+
+UnknownCommand::~UnknownCommand() {
+ const size_t size = args_.length();
+ for (size_t i = 0; i < size; ++i) {
+ js_delete(args_[i]);
+ }
+}
+
+
+bool UnknownCommand::Run(Debugger* debugger) {
+ VIXL_ASSERT(debugger->IsDebuggerRunning());
+ USE(debugger);
+
+ printf(" ** Unknown Command:");
+ const size_t size = args_.length();
+ for (size_t i = 0; i < size; ++i) {
+ printf(" ");
+ args_[i]->Print(stdout);
+ }
+ printf(" **\n");
+
+ return false;
+}
+
+
+InvalidCommand::~InvalidCommand() {
+ const size_t size = args_.length();
+ for (size_t i = 0; i < size; ++i) {
+ js_delete(args_[i]);
+ }
+}
+
+
+bool InvalidCommand::Run(Debugger* debugger) {
+ VIXL_ASSERT(debugger->IsDebuggerRunning());
+ USE(debugger);
+
+ printf(" ** Invalid Command:");
+ const size_t size = args_.length();
+ for (size_t i = 0; i < size; ++i) {
+ printf(" ");
+ if (i == static_cast<size_t>(index_)) {
+ printf(">>");
+ args_[i]->Print(stdout);
+ printf("<<");
+ } else {
+ args_[i]->Print(stdout);
+ }
+ }
+ printf(" **\n");
+ printf(" ** %s\n", cause_);
+
+ return false;
+}
+
+} // namespace vixl
+
+#endif // JS_SIMULATOR_ARM64
diff --git a/js/src/jit/arm64/vixl/Debugger-vixl.h b/js/src/jit/arm64/vixl/Debugger-vixl.h
new file mode 100644
index 0000000000..7236bf1e5e
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Debugger-vixl.h
@@ -0,0 +1,117 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifdef JS_SIMULATOR_ARM64
+
+#ifndef VIXL_A64_DEBUGGER_A64_H_
+#define VIXL_A64_DEBUGGER_A64_H_
+
+#include <ctype.h>
+#include <errno.h>
+#include <limits.h>
+
+#include "jit/arm64/vixl/Constants-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Simulator-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+
+namespace vixl {
+
+// Flags that represent the debugger state.
+enum DebugParameters {
+ DBG_INACTIVE = 0,
+ DBG_ACTIVE = 1 << 0, // The debugger is active.
+ DBG_BREAK = 1 << 1 // The debugger is at a breakpoint.
+};
+
+// Forward declarations.
+class DebugCommand;
+class Token;
+class FormatToken;
+
+class Debugger : public Simulator {
+ public:
+ explicit Debugger(Decoder* decoder, FILE* stream = stdout);
+ ~Debugger();
+
+ virtual void Run() override;
+ virtual void VisitException(const Instruction* instr) override;
+
+ int debug_parameters() const { return debug_parameters_; }
+ void set_debug_parameters(int parameters) {
+ debug_parameters_ = parameters;
+
+ update_pending_request();
+ }
+
+ // Numbers of instructions to execute before the debugger shell is given
+ // back control.
+ int64_t steps() const { return steps_; }
+ void set_steps(int64_t value) {
+ VIXL_ASSERT(value > 1);
+ steps_ = value;
+ }
+
+ bool IsDebuggerRunning() const {
+ return (debug_parameters_ & DBG_ACTIVE) != 0;
+ }
+
+ bool pending_request() const { return pending_request_; }
+ void update_pending_request() {
+ pending_request_ = IsDebuggerRunning();
+ }
+
+ void PrintInstructions(const void* address, int64_t count = 1);
+ void PrintMemory(const uint8_t* address,
+ const FormatToken* format,
+ int64_t count = 1);
+ void PrintRegister(const Register& target_reg,
+ const char* name,
+ const FormatToken* format);
+ void PrintFPRegister(const FPRegister& target_fpreg,
+ const FormatToken* format);
+
+ private:
+ char* ReadCommandLine(const char* prompt, char* buffer, int length);
+ void RunDebuggerShell();
+ void DoBreakpoint(const Instruction* instr);
+
+ int debug_parameters_;
+ bool pending_request_;
+ int64_t steps_;
+ DebugCommand* last_command_;
+ PrintDisassembler* disasm_;
+ Decoder* printer_;
+
+ // Length of the biggest command line accepted by the debugger shell.
+ static const int kMaxDebugShellLine = 256;
+};
+
+} // namespace vixl
+
+#endif // VIXL_A64_DEBUGGER_A64_H_
+
+#endif // JS_SIMULATOR_ARM64
diff --git a/js/src/jit/arm64/vixl/Decoder-vixl.cpp b/js/src/jit/arm64/vixl/Decoder-vixl.cpp
new file mode 100644
index 0000000000..a306e66b3c
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Decoder-vixl.cpp
@@ -0,0 +1,878 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Decoder-vixl.h"
+
+#include <algorithm>
+
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+
+namespace vixl {
+
+void Decoder::DecodeInstruction(const Instruction *instr) {
+ if (instr->Bits(28, 27) == 0) {
+ VisitUnallocated(instr);
+ } else {
+ switch (instr->Bits(27, 24)) {
+ // 0: PC relative addressing.
+ case 0x0: DecodePCRelAddressing(instr); break;
+
+ // 1: Add/sub immediate.
+ case 0x1: DecodeAddSubImmediate(instr); break;
+
+ // A: Logical shifted register.
+ // Add/sub with carry.
+ // Conditional compare register.
+ // Conditional compare immediate.
+ // Conditional select.
+ // Data processing 1 source.
+ // Data processing 2 source.
+ // B: Add/sub shifted register.
+ // Add/sub extended register.
+ // Data processing 3 source.
+ case 0xA:
+ case 0xB: DecodeDataProcessing(instr); break;
+
+ // 2: Logical immediate.
+ // Move wide immediate.
+ case 0x2: DecodeLogical(instr); break;
+
+ // 3: Bitfield.
+ // Extract.
+ case 0x3: DecodeBitfieldExtract(instr); break;
+
+ // 4: Unconditional branch immediate.
+ // Exception generation.
+ // Compare and branch immediate.
+ // 5: Compare and branch immediate.
+ // Conditional branch.
+ // System.
+ // 6,7: Unconditional branch.
+ // Test and branch immediate.
+ case 0x4:
+ case 0x5:
+ case 0x6:
+ case 0x7: DecodeBranchSystemException(instr); break;
+
+ // 8,9: Load/store register pair post-index.
+ // Load register literal.
+ // Load/store register unscaled immediate.
+ // Load/store register immediate post-index.
+ // Load/store register immediate pre-index.
+ // Load/store register offset.
+ // Load/store exclusive.
+ // C,D: Load/store register pair offset.
+ // Load/store register pair pre-index.
+ // Load/store register unsigned immediate.
+ // Advanced SIMD.
+ case 0x8:
+ case 0x9:
+ case 0xC:
+ case 0xD: DecodeLoadStore(instr); break;
+
+ // E: FP fixed point conversion.
+ // FP integer conversion.
+ // FP data processing 1 source.
+ // FP compare.
+ // FP immediate.
+ // FP data processing 2 source.
+ // FP conditional compare.
+ // FP conditional select.
+ // Advanced SIMD.
+ // F: FP data processing 3 source.
+ // Advanced SIMD.
+ case 0xE:
+ case 0xF: DecodeFP(instr); break;
+ }
+ }
+}
+
+void Decoder::AppendVisitor(DecoderVisitor* new_visitor) {
+ MOZ_ALWAYS_TRUE(visitors_.append(new_visitor));
+}
+
+
+void Decoder::PrependVisitor(DecoderVisitor* new_visitor) {
+ MOZ_ALWAYS_TRUE(visitors_.insert(visitors_.begin(), new_visitor));
+}
+
+
+void Decoder::InsertVisitorBefore(DecoderVisitor* new_visitor,
+ DecoderVisitor* registered_visitor) {
+ for (auto it = visitors_.begin(); it != visitors_.end(); it++) {
+ if (*it == registered_visitor) {
+ MOZ_ALWAYS_TRUE(visitors_.insert(it, new_visitor));
+ return;
+ }
+ }
+ // We reached the end of the list without finding registered_visitor.
+ MOZ_ALWAYS_TRUE(visitors_.append(new_visitor));
+}
+
+
+void Decoder::InsertVisitorAfter(DecoderVisitor* new_visitor,
+ DecoderVisitor* registered_visitor) {
+ for (auto it = visitors_.begin(); it != visitors_.end(); it++) {
+ if (*it == registered_visitor) {
+ it++;
+ MOZ_ALWAYS_TRUE(visitors_.insert(it, new_visitor));
+ return;
+ }
+ }
+ // We reached the end of the list without finding registered_visitor.
+ MOZ_ALWAYS_TRUE(visitors_.append(new_visitor));
+}
+
+
+void Decoder::RemoveVisitor(DecoderVisitor* visitor) {
+ visitors_.erase(std::remove(visitors_.begin(), visitors_.end(), visitor),
+ visitors_.end());
+}
+
+
+void Decoder::DecodePCRelAddressing(const Instruction* instr) {
+ VIXL_ASSERT(instr->Bits(27, 24) == 0x0);
+ // We know bit 28 is set, as <b28:b27> = 0 is filtered out at the top level
+ // decode.
+ VIXL_ASSERT(instr->Bit(28) == 0x1);
+ VisitPCRelAddressing(instr);
+}
+
+
+void Decoder::DecodeBranchSystemException(const Instruction* instr) {
+ VIXL_ASSERT((instr->Bits(27, 24) == 0x4) ||
+ (instr->Bits(27, 24) == 0x5) ||
+ (instr->Bits(27, 24) == 0x6) ||
+ (instr->Bits(27, 24) == 0x7) );
+
+ switch (instr->Bits(31, 29)) {
+ case 0:
+ case 4: {
+ VisitUnconditionalBranch(instr);
+ break;
+ }
+ case 1:
+ case 5: {
+ if (instr->Bit(25) == 0) {
+ VisitCompareBranch(instr);
+ } else {
+ VisitTestBranch(instr);
+ }
+ break;
+ }
+ case 2: {
+ if (instr->Bit(25) == 0) {
+ if ((instr->Bit(24) == 0x1) ||
+ (instr->Mask(0x01000010) == 0x00000010)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitConditionalBranch(instr);
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ break;
+ }
+ case 6: {
+ if (instr->Bit(25) == 0) {
+ if (instr->Bit(24) == 0) {
+ if ((instr->Bits(4, 2) != 0) ||
+ (instr->Mask(0x00E0001D) == 0x00200001) ||
+ (instr->Mask(0x00E0001D) == 0x00400001) ||
+ (instr->Mask(0x00E0001E) == 0x00200002) ||
+ (instr->Mask(0x00E0001E) == 0x00400002) ||
+ (instr->Mask(0x00E0001C) == 0x00600000) ||
+ (instr->Mask(0x00E0001C) == 0x00800000) ||
+ (instr->Mask(0x00E0001F) == 0x00A00000) ||
+ (instr->Mask(0x00C0001C) == 0x00C00000)) {
+ if (instr->InstructionBits() == UNDEFINED_INST_PATTERN) {
+ VisitException(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ } else {
+ VisitException(instr);
+ }
+ } else {
+ if (instr->Bits(23, 22) == 0) {
+ const Instr masked_003FF0E0 = instr->Mask(0x003FF0E0);
+ if ((instr->Bits(21, 19) == 0x4) ||
+ (masked_003FF0E0 == 0x00033000) ||
+ (masked_003FF0E0 == 0x003FF020) ||
+ (masked_003FF0E0 == 0x003FF060) ||
+ (masked_003FF0E0 == 0x003FF0E0) ||
+ (instr->Mask(0x00388000) == 0x00008000) ||
+ (instr->Mask(0x0038E000) == 0x00000000) ||
+ (instr->Mask(0x0039E000) == 0x00002000) ||
+ (instr->Mask(0x003AE000) == 0x00002000) ||
+ (instr->Mask(0x003CE000) == 0x00042000) ||
+ (instr->Mask(0x003FFFC0) == 0x000320C0) ||
+ (instr->Mask(0x003FF100) == 0x00032100) ||
+ // (instr->Mask(0x003FF200) == 0x00032200) || // match CSDB
+ (instr->Mask(0x003FF400) == 0x00032400) ||
+ (instr->Mask(0x003FF800) == 0x00032800) ||
+ (instr->Mask(0x0038F000) == 0x00005000) ||
+ (instr->Mask(0x0038E000) == 0x00006000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitSystem(instr);
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ if ((instr->Bit(24) == 0x1) ||
+ (instr->Bits(20, 16) != 0x1F) ||
+ (instr->Bits(15, 10) != 0) ||
+ (instr->Bits(4, 0) != 0) ||
+ (instr->Bits(24, 21) == 0x3) ||
+ (instr->Bits(24, 22) == 0x3)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitUnconditionalBranchToRegister(instr);
+ }
+ }
+ break;
+ }
+ case 3:
+ case 7: {
+ VisitUnallocated(instr);
+ break;
+ }
+ }
+}
+
+
+void Decoder::DecodeLoadStore(const Instruction* instr) {
+ VIXL_ASSERT((instr->Bits(27, 24) == 0x8) ||
+ (instr->Bits(27, 24) == 0x9) ||
+ (instr->Bits(27, 24) == 0xC) ||
+ (instr->Bits(27, 24) == 0xD) );
+ // TODO(all): rearrange the tree to integrate this branch.
+ if ((instr->Bit(28) == 0) && (instr->Bit(29) == 0) && (instr->Bit(26) == 1)) {
+ DecodeNEONLoadStore(instr);
+ return;
+ }
+
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(28) == 0) {
+ if (instr->Bit(29) == 0) {
+ if (instr->Bit(26) == 0) {
+ VisitLoadStoreExclusive(instr);
+ } else {
+ VIXL_UNREACHABLE();
+ }
+ } else {
+ if ((instr->Bits(31, 30) == 0x3) ||
+ (instr->Mask(0xC4400000) == 0x40000000)) {
+ VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ if (instr->Mask(0xC4400000) == 0xC0400000) {
+ VisitUnallocated(instr);
+ } else {
+ VisitLoadStorePairNonTemporal(instr);
+ }
+ } else {
+ VisitLoadStorePairPostIndex(instr);
+ }
+ }
+ }
+ } else {
+ if (instr->Bit(29) == 0) {
+ if (instr->Mask(0xC4000000) == 0xC4000000) {
+ VisitUnallocated(instr);
+ } else {
+ VisitLoadLiteral(instr);
+ }
+ } else {
+ if ((instr->Mask(0x84C00000) == 0x80C00000) ||
+ (instr->Mask(0x44800000) == 0x44800000) ||
+ (instr->Mask(0x84800000) == 0x84800000)) {
+ VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(21) == 0) {
+ switch (instr->Bits(11, 10)) {
+ case 0: {
+ VisitLoadStoreUnscaledOffset(instr);
+ break;
+ }
+ case 1: {
+ if (instr->Mask(0xC4C00000) == 0xC0800000) {
+ VisitUnallocated(instr);
+ } else {
+ VisitLoadStorePostIndex(instr);
+ }
+ break;
+ }
+ case 2: {
+ // TODO: VisitLoadStoreRegisterOffsetUnpriv.
+ VisitUnimplemented(instr);
+ break;
+ }
+ case 3: {
+ if (instr->Mask(0xC4C00000) == 0xC0800000) {
+ VisitUnallocated(instr);
+ } else {
+ VisitLoadStorePreIndex(instr);
+ }
+ break;
+ }
+ }
+ } else {
+ if (instr->Bits(11, 10) == 0x2) {
+ if (instr->Bit(14) == 0) {
+ VisitUnallocated(instr);
+ } else {
+ VisitLoadStoreRegisterOffset(instr);
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ }
+ }
+ }
+ } else {
+ if (instr->Bit(28) == 0) {
+ if (instr->Bit(29) == 0) {
+ VisitUnallocated(instr);
+ } else {
+ if ((instr->Bits(31, 30) == 0x3) ||
+ (instr->Mask(0xC4400000) == 0x40000000)) {
+ VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ VisitLoadStorePairOffset(instr);
+ } else {
+ VisitLoadStorePairPreIndex(instr);
+ }
+ }
+ }
+ } else {
+ if (instr->Bit(29) == 0) {
+ VisitUnallocated(instr);
+ } else {
+ if ((instr->Mask(0x84C00000) == 0x80C00000) ||
+ (instr->Mask(0x44800000) == 0x44800000) ||
+ (instr->Mask(0x84800000) == 0x84800000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitLoadStoreUnsignedOffset(instr);
+ }
+ }
+ }
+ }
+}
+
+
+void Decoder::DecodeLogical(const Instruction* instr) {
+ VIXL_ASSERT(instr->Bits(27, 24) == 0x2);
+
+ if (instr->Mask(0x80400000) == 0x00400000) {
+ VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ VisitLogicalImmediate(instr);
+ } else {
+ if (instr->Bits(30, 29) == 0x1) {
+ VisitUnallocated(instr);
+ } else {
+ VisitMoveWideImmediate(instr);
+ }
+ }
+ }
+}
+
+
+void Decoder::DecodeBitfieldExtract(const Instruction* instr) {
+ VIXL_ASSERT(instr->Bits(27, 24) == 0x3);
+
+ if ((instr->Mask(0x80400000) == 0x80000000) ||
+ (instr->Mask(0x80400000) == 0x00400000) ||
+ (instr->Mask(0x80008000) == 0x00008000)) {
+ VisitUnallocated(instr);
+ } else if (instr->Bit(23) == 0) {
+ if ((instr->Mask(0x80200000) == 0x00200000) ||
+ (instr->Mask(0x60000000) == 0x60000000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitBitfield(instr);
+ }
+ } else {
+ if ((instr->Mask(0x60200000) == 0x00200000) ||
+ (instr->Mask(0x60000000) != 0x00000000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitExtract(instr);
+ }
+ }
+}
+
+
+void Decoder::DecodeAddSubImmediate(const Instruction* instr) {
+ VIXL_ASSERT(instr->Bits(27, 24) == 0x1);
+ if (instr->Bit(23) == 1) {
+ VisitUnallocated(instr);
+ } else {
+ VisitAddSubImmediate(instr);
+ }
+}
+
+
+void Decoder::DecodeDataProcessing(const Instruction* instr) {
+ VIXL_ASSERT((instr->Bits(27, 24) == 0xA) ||
+ (instr->Bits(27, 24) == 0xB));
+
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(28) == 0) {
+ if (instr->Mask(0x80008000) == 0x00008000) {
+ VisitUnallocated(instr);
+ } else {
+ VisitLogicalShifted(instr);
+ }
+ } else {
+ switch (instr->Bits(23, 21)) {
+ case 0: {
+ if (instr->Mask(0x0000FC00) != 0) {
+ VisitUnallocated(instr);
+ } else {
+ VisitAddSubWithCarry(instr);
+ }
+ break;
+ }
+ case 2: {
+ if ((instr->Bit(29) == 0) ||
+ (instr->Mask(0x00000410) != 0)) {
+ VisitUnallocated(instr);
+ } else {
+ if (instr->Bit(11) == 0) {
+ VisitConditionalCompareRegister(instr);
+ } else {
+ VisitConditionalCompareImmediate(instr);
+ }
+ }
+ break;
+ }
+ case 4: {
+ if (instr->Mask(0x20000800) != 0x00000000) {
+ VisitUnallocated(instr);
+ } else {
+ VisitConditionalSelect(instr);
+ }
+ break;
+ }
+ case 6: {
+ if (instr->Bit(29) == 0x1) {
+ VisitUnallocated(instr);
+ VIXL_FALLTHROUGH();
+ } else {
+ if (instr->Bit(30) == 0) {
+ if ((instr->Bit(15) == 0x1) ||
+ (instr->Bits(15, 11) == 0) ||
+ (instr->Bits(15, 12) == 0x1) ||
+ (instr->Bits(15, 12) == 0x3) ||
+ (instr->Bits(15, 13) == 0x3) ||
+ (instr->Mask(0x8000EC00) == 0x00004C00) ||
+ (instr->Mask(0x8000E800) == 0x80004000) ||
+ (instr->Mask(0x8000E400) == 0x80004000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitDataProcessing2Source(instr);
+ }
+ } else {
+ if ((instr->Bit(13) == 1) ||
+ (instr->Bits(20, 16) != 0) ||
+ (instr->Bits(15, 14) != 0) ||
+ (instr->Mask(0xA01FFC00) == 0x00000C00) ||
+ (instr->Mask(0x201FF800) == 0x00001800)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitDataProcessing1Source(instr);
+ }
+ }
+ break;
+ }
+ }
+ case 1:
+ case 3:
+ case 5:
+ case 7: VisitUnallocated(instr); break;
+ }
+ }
+ } else {
+ if (instr->Bit(28) == 0) {
+ if (instr->Bit(21) == 0) {
+ if ((instr->Bits(23, 22) == 0x3) ||
+ (instr->Mask(0x80008000) == 0x00008000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitAddSubShifted(instr);
+ }
+ } else {
+ if ((instr->Mask(0x00C00000) != 0x00000000) ||
+ (instr->Mask(0x00001400) == 0x00001400) ||
+ (instr->Mask(0x00001800) == 0x00001800)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitAddSubExtended(instr);
+ }
+ }
+ } else {
+ if ((instr->Bit(30) == 0x1) ||
+ (instr->Bits(30, 29) == 0x1) ||
+ (instr->Mask(0xE0600000) == 0x00200000) ||
+ (instr->Mask(0xE0608000) == 0x00400000) ||
+ (instr->Mask(0x60608000) == 0x00408000) ||
+ (instr->Mask(0x60E00000) == 0x00E00000) ||
+ (instr->Mask(0x60E00000) == 0x00800000) ||
+ (instr->Mask(0x60E00000) == 0x00600000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitDataProcessing3Source(instr);
+ }
+ }
+ }
+}
+
+
+void Decoder::DecodeFP(const Instruction* instr) {
+ VIXL_ASSERT((instr->Bits(27, 24) == 0xE) ||
+ (instr->Bits(27, 24) == 0xF));
+ if (instr->Bit(28) == 0) {
+ DecodeNEONVectorDataProcessing(instr);
+ } else {
+ if (instr->Bits(31, 30) == 0x3) {
+ VisitUnallocated(instr);
+ } else if (instr->Bits(31, 30) == 0x1) {
+ DecodeNEONScalarDataProcessing(instr);
+ } else {
+ if (instr->Bit(29) == 0) {
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(21) == 0) {
+ if ((instr->Bit(23) == 1) ||
+ (instr->Bit(18) == 1) ||
+ (instr->Mask(0x80008000) == 0x00000000) ||
+ (instr->Mask(0x000E0000) == 0x00000000) ||
+ (instr->Mask(0x000E0000) == 0x000A0000) ||
+ (instr->Mask(0x00160000) == 0x00000000) ||
+ (instr->Mask(0x00160000) == 0x00120000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitFPFixedPointConvert(instr);
+ }
+ } else {
+ if (instr->Bits(15, 10) == 32) {
+ VisitUnallocated(instr);
+ } else if (instr->Bits(15, 10) == 0) {
+ if ((instr->Bits(23, 22) == 0x3) ||
+ (instr->Mask(0x000E0000) == 0x000A0000) ||
+ (instr->Mask(0x000E0000) == 0x000C0000) ||
+ (instr->Mask(0x00160000) == 0x00120000) ||
+ (instr->Mask(0x00160000) == 0x00140000) ||
+ (instr->Mask(0x20C40000) == 0x00800000) ||
+ (instr->Mask(0x20C60000) == 0x00840000) ||
+ (instr->Mask(0xA0C60000) == 0x80060000) ||
+ (instr->Mask(0xA0C60000) == 0x00860000) ||
+ (instr->Mask(0xA0C60000) == 0x00460000) ||
+ (instr->Mask(0xA0CE0000) == 0x80860000) ||
+ (instr->Mask(0xA0CE0000) == 0x804E0000) ||
+ (instr->Mask(0xA0CE0000) == 0x000E0000) ||
+ (instr->Mask(0xA0D60000) == 0x00160000) ||
+ (instr->Mask(0xA0D60000) == 0x80560000) ||
+ (instr->Mask(0xA0D60000) == 0x80960000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitFPIntegerConvert(instr);
+ }
+ } else if (instr->Bits(14, 10) == 16) {
+ const Instr masked_A0DF8000 = instr->Mask(0xA0DF8000);
+ if ((instr->Mask(0x80180000) != 0) ||
+ (masked_A0DF8000 == 0x00020000) ||
+ (masked_A0DF8000 == 0x00030000) ||
+ (masked_A0DF8000 == 0x00068000) ||
+ (masked_A0DF8000 == 0x00428000) ||
+ (masked_A0DF8000 == 0x00430000) ||
+ (masked_A0DF8000 == 0x00468000) ||
+ (instr->Mask(0xA0D80000) == 0x00800000) ||
+ (instr->Mask(0xA0DE0000) == 0x00C00000) ||
+ (instr->Mask(0xA0DF0000) == 0x00C30000) ||
+ (instr->Mask(0xA0DC0000) == 0x00C40000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitFPDataProcessing1Source(instr);
+ }
+ } else if (instr->Bits(13, 10) == 8) {
+ if ((instr->Bits(15, 14) != 0) ||
+ (instr->Bits(2, 0) != 0) ||
+ (instr->Mask(0x80800000) != 0x00000000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitFPCompare(instr);
+ }
+ } else if (instr->Bits(12, 10) == 4) {
+ if ((instr->Bits(9, 5) != 0) ||
+ (instr->Mask(0x80800000) != 0x00000000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitFPImmediate(instr);
+ }
+ } else {
+ if (instr->Mask(0x80800000) != 0x00000000) {
+ VisitUnallocated(instr);
+ } else {
+ switch (instr->Bits(11, 10)) {
+ case 1: {
+ VisitFPConditionalCompare(instr);
+ break;
+ }
+ case 2: {
+ if ((instr->Bits(15, 14) == 0x3) ||
+ (instr->Mask(0x00009000) == 0x00009000) ||
+ (instr->Mask(0x0000A000) == 0x0000A000)) {
+ VisitUnallocated(instr);
+ } else {
+ VisitFPDataProcessing2Source(instr);
+ }
+ break;
+ }
+ case 3: {
+ VisitFPConditionalSelect(instr);
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ }
+ }
+ }
+ } else {
+ // Bit 30 == 1 has been handled earlier.
+ VIXL_ASSERT(instr->Bit(30) == 0);
+ if (instr->Mask(0xA0800000) != 0) {
+ VisitUnallocated(instr);
+ } else {
+ VisitFPDataProcessing3Source(instr);
+ }
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ }
+}
+
+
+void Decoder::DecodeNEONLoadStore(const Instruction* instr) {
+ VIXL_ASSERT(instr->Bits(29, 25) == 0x6);
+ if (instr->Bit(31) == 0) {
+ if ((instr->Bit(24) == 0) && (instr->Bit(21) == 1)) {
+ VisitUnallocated(instr);
+ return;
+ }
+
+ if (instr->Bit(23) == 0) {
+ if (instr->Bits(20, 16) == 0) {
+ if (instr->Bit(24) == 0) {
+ VisitNEONLoadStoreMultiStruct(instr);
+ } else {
+ VisitNEONLoadStoreSingleStruct(instr);
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ } else {
+ if (instr->Bit(24) == 0) {
+ VisitNEONLoadStoreMultiStructPostIndex(instr);
+ } else {
+ VisitNEONLoadStoreSingleStructPostIndex(instr);
+ }
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+}
+
+
+void Decoder::DecodeNEONVectorDataProcessing(const Instruction* instr) {
+ VIXL_ASSERT(instr->Bits(28, 25) == 0x7);
+ if (instr->Bit(31) == 0) {
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(21) == 0) {
+ if (instr->Bit(15) == 0) {
+ if (instr->Bit(10) == 0) {
+ if (instr->Bit(29) == 0) {
+ if (instr->Bit(11) == 0) {
+ VisitNEONTable(instr);
+ } else {
+ VisitNEONPerm(instr);
+ }
+ } else {
+ VisitNEONExtract(instr);
+ }
+ } else {
+ if (instr->Bits(23, 22) == 0) {
+ VisitNEONCopy(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ } else {
+ if (instr->Bit(10) == 0) {
+ if (instr->Bit(11) == 0) {
+ VisitNEON3Different(instr);
+ } else {
+ if (instr->Bits(18, 17) == 0) {
+ if (instr->Bit(20) == 0) {
+ if (instr->Bit(19) == 0) {
+ VisitNEON2RegMisc(instr);
+ } else {
+ if (instr->Bits(30, 29) == 0x2) {
+ VisitCryptoAES(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ if (instr->Bit(19) == 0) {
+ VisitNEONAcrossLanes(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ VisitNEON3Same(instr);
+ }
+ }
+ } else {
+ if (instr->Bit(10) == 0) {
+ VisitNEONByIndexedElement(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ if (instr->Bits(22, 19) == 0) {
+ VisitNEONModifiedImmediate(instr);
+ } else {
+ VisitNEONShiftImmediate(instr);
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+}
+
+
+void Decoder::DecodeNEONScalarDataProcessing(const Instruction* instr) {
+ VIXL_ASSERT(instr->Bits(28, 25) == 0xF);
+ if (instr->Bit(24) == 0) {
+ if (instr->Bit(21) == 0) {
+ if (instr->Bit(15) == 0) {
+ if (instr->Bit(10) == 0) {
+ if (instr->Bit(29) == 0) {
+ if (instr->Bit(11) == 0) {
+ VisitCrypto3RegSHA(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ } else {
+ if (instr->Bits(23, 22) == 0) {
+ VisitNEONScalarCopy(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ } else {
+ if (instr->Bit(10) == 0) {
+ if (instr->Bit(11) == 0) {
+ VisitNEONScalar3Diff(instr);
+ } else {
+ if (instr->Bits(18, 17) == 0) {
+ if (instr->Bit(20) == 0) {
+ if (instr->Bit(19) == 0) {
+ VisitNEONScalar2RegMisc(instr);
+ } else {
+ if (instr->Bit(29) == 0) {
+ VisitCrypto2RegSHA(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ if (instr->Bit(19) == 0) {
+ VisitNEONScalarPairwise(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ } else {
+ VisitNEONScalar3Same(instr);
+ }
+ }
+ } else {
+ if (instr->Bit(10) == 0) {
+ VisitNEONScalarByIndexedElement(instr);
+ } else {
+ if (instr->Bit(23) == 0) {
+ VisitNEONScalarShiftImmediate(instr);
+ } else {
+ VisitUnallocated(instr);
+ }
+ }
+ }
+}
+
+
+#define DEFINE_VISITOR_CALLERS(A) \
+ void Decoder::Visit##A(const Instruction *instr) { \
+ VIXL_ASSERT(instr->Mask(A##FMask) == A##Fixed); \
+ for (auto visitor : visitors_) { \
+ visitor->Visit##A(instr); \
+ } \
+ }
+VISITOR_LIST(DEFINE_VISITOR_CALLERS)
+#undef DEFINE_VISITOR_CALLERS
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/Decoder-vixl.h b/js/src/jit/arm64/vixl/Decoder-vixl.h
new file mode 100644
index 0000000000..4aca78618b
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Decoder-vixl.h
@@ -0,0 +1,275 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_DECODER_A64_H_
+#define VIXL_A64_DECODER_A64_H_
+
+#include "mozilla/Vector.h"
+
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Instructions-vixl.h"
+#include "js/AllocPolicy.h"
+
+
+// List macro containing all visitors needed by the decoder class.
+
+#define VISITOR_LIST_THAT_RETURN(V) \
+ V(PCRelAddressing) \
+ V(AddSubImmediate) \
+ V(LogicalImmediate) \
+ V(MoveWideImmediate) \
+ V(Bitfield) \
+ V(Extract) \
+ V(UnconditionalBranch) \
+ V(UnconditionalBranchToRegister) \
+ V(CompareBranch) \
+ V(TestBranch) \
+ V(ConditionalBranch) \
+ V(System) \
+ V(Exception) \
+ V(LoadStorePairPostIndex) \
+ V(LoadStorePairOffset) \
+ V(LoadStorePairPreIndex) \
+ V(LoadStorePairNonTemporal) \
+ V(LoadLiteral) \
+ V(LoadStoreUnscaledOffset) \
+ V(LoadStorePostIndex) \
+ V(LoadStorePreIndex) \
+ V(LoadStoreRegisterOffset) \
+ V(LoadStoreUnsignedOffset) \
+ V(LoadStoreExclusive) \
+ V(LogicalShifted) \
+ V(AddSubShifted) \
+ V(AddSubExtended) \
+ V(AddSubWithCarry) \
+ V(ConditionalCompareRegister) \
+ V(ConditionalCompareImmediate) \
+ V(ConditionalSelect) \
+ V(DataProcessing1Source) \
+ V(DataProcessing2Source) \
+ V(DataProcessing3Source) \
+ V(FPCompare) \
+ V(FPConditionalCompare) \
+ V(FPConditionalSelect) \
+ V(FPImmediate) \
+ V(FPDataProcessing1Source) \
+ V(FPDataProcessing2Source) \
+ V(FPDataProcessing3Source) \
+ V(FPIntegerConvert) \
+ V(FPFixedPointConvert) \
+ V(Crypto2RegSHA) \
+ V(Crypto3RegSHA) \
+ V(CryptoAES) \
+ V(NEON2RegMisc) \
+ V(NEON3Different) \
+ V(NEON3Same) \
+ V(NEONAcrossLanes) \
+ V(NEONByIndexedElement) \
+ V(NEONCopy) \
+ V(NEONExtract) \
+ V(NEONLoadStoreMultiStruct) \
+ V(NEONLoadStoreMultiStructPostIndex) \
+ V(NEONLoadStoreSingleStruct) \
+ V(NEONLoadStoreSingleStructPostIndex) \
+ V(NEONModifiedImmediate) \
+ V(NEONScalar2RegMisc) \
+ V(NEONScalar3Diff) \
+ V(NEONScalar3Same) \
+ V(NEONScalarByIndexedElement) \
+ V(NEONScalarCopy) \
+ V(NEONScalarPairwise) \
+ V(NEONScalarShiftImmediate) \
+ V(NEONShiftImmediate) \
+ V(NEONTable) \
+ V(NEONPerm) \
+
+#define VISITOR_LIST_THAT_DONT_RETURN(V) \
+ V(Unallocated) \
+ V(Unimplemented) \
+
+#define VISITOR_LIST(V) \
+ VISITOR_LIST_THAT_RETURN(V) \
+ VISITOR_LIST_THAT_DONT_RETURN(V) \
+
+namespace vixl {
+
+// The Visitor interface. Disassembler and simulator (and other tools)
+// must provide implementations for all of these functions.
+class DecoderVisitor {
+ public:
+ enum VisitorConstness {
+ kConstVisitor,
+ kNonConstVisitor
+ };
+ explicit DecoderVisitor(VisitorConstness constness = kConstVisitor)
+ : constness_(constness) {}
+
+ virtual ~DecoderVisitor() {}
+
+ #define DECLARE(A) virtual void Visit##A(const Instruction* instr) = 0;
+ VISITOR_LIST(DECLARE)
+ #undef DECLARE
+
+ bool IsConstVisitor() const { return constness_ == kConstVisitor; }
+ Instruction* MutableInstruction(const Instruction* instr) {
+ VIXL_ASSERT(!IsConstVisitor());
+ return const_cast<Instruction*>(instr);
+ }
+
+ private:
+ const VisitorConstness constness_;
+};
+
+
+class Decoder {
+ public:
+ Decoder() {}
+
+ // Top-level wrappers around the actual decoding function.
+ void Decode(const Instruction* instr) {
+#ifdef DEBUG
+ for (auto visitor : visitors_) {
+ VIXL_ASSERT(visitor->IsConstVisitor());
+ }
+#endif
+ DecodeInstruction(instr);
+ }
+ void Decode(Instruction* instr) {
+ DecodeInstruction(const_cast<const Instruction*>(instr));
+ }
+
+ // Register a new visitor class with the decoder.
+ // Decode() will call the corresponding visitor method from all registered
+ // visitor classes when decoding reaches the leaf node of the instruction
+ // decode tree.
+ // Visitors are called in order.
+ // A visitor can be registered multiple times.
+ //
+ // d.AppendVisitor(V1);
+ // d.AppendVisitor(V2);
+ // d.PrependVisitor(V2);
+ // d.AppendVisitor(V3);
+ //
+ // d.Decode(i);
+ //
+ // will call in order visitor methods in V2, V1, V2, V3.
+ void AppendVisitor(DecoderVisitor* visitor);
+ void PrependVisitor(DecoderVisitor* visitor);
+ // These helpers register `new_visitor` before or after the first instance of
+ // `registered_visiter` in the list.
+ // So if
+ // V1, V2, V1, V2
+ // are registered in this order in the decoder, calls to
+ // d.InsertVisitorAfter(V3, V1);
+ // d.InsertVisitorBefore(V4, V2);
+ // will yield the order
+ // V1, V3, V4, V2, V1, V2
+ //
+ // For more complex modifications of the order of registered visitors, one can
+ // directly access and modify the list of visitors via the `visitors()'
+ // accessor.
+ void InsertVisitorBefore(DecoderVisitor* new_visitor,
+ DecoderVisitor* registered_visitor);
+ void InsertVisitorAfter(DecoderVisitor* new_visitor,
+ DecoderVisitor* registered_visitor);
+
+ // Remove all instances of a previously registered visitor class from the list
+ // of visitors stored by the decoder.
+ void RemoveVisitor(DecoderVisitor* visitor);
+
+ #define DECLARE(A) void Visit##A(const Instruction* instr);
+ VISITOR_LIST(DECLARE)
+ #undef DECLARE
+
+
+ private:
+ // Decodes an instruction and calls the visitor functions registered with the
+ // Decoder class.
+ void DecodeInstruction(const Instruction* instr);
+
+ // Decode the PC relative addressing instruction, and call the corresponding
+ // visitors.
+ // On entry, instruction bits 27:24 = 0x0.
+ void DecodePCRelAddressing(const Instruction* instr);
+
+ // Decode the add/subtract immediate instruction, and call the correspoding
+ // visitors.
+ // On entry, instruction bits 27:24 = 0x1.
+ void DecodeAddSubImmediate(const Instruction* instr);
+
+ // Decode the branch, system command, and exception generation parts of
+ // the instruction tree, and call the corresponding visitors.
+ // On entry, instruction bits 27:24 = {0x4, 0x5, 0x6, 0x7}.
+ void DecodeBranchSystemException(const Instruction* instr);
+
+ // Decode the load and store parts of the instruction tree, and call
+ // the corresponding visitors.
+ // On entry, instruction bits 27:24 = {0x8, 0x9, 0xC, 0xD}.
+ void DecodeLoadStore(const Instruction* instr);
+
+ // Decode the logical immediate and move wide immediate parts of the
+ // instruction tree, and call the corresponding visitors.
+ // On entry, instruction bits 27:24 = 0x2.
+ void DecodeLogical(const Instruction* instr);
+
+ // Decode the bitfield and extraction parts of the instruction tree,
+ // and call the corresponding visitors.
+ // On entry, instruction bits 27:24 = 0x3.
+ void DecodeBitfieldExtract(const Instruction* instr);
+
+ // Decode the data processing parts of the instruction tree, and call the
+ // corresponding visitors.
+ // On entry, instruction bits 27:24 = {0x1, 0xA, 0xB}.
+ void DecodeDataProcessing(const Instruction* instr);
+
+ // Decode the floating point parts of the instruction tree, and call the
+ // corresponding visitors.
+ // On entry, instruction bits 27:24 = {0xE, 0xF}.
+ void DecodeFP(const Instruction* instr);
+
+ // Decode the Advanced SIMD (NEON) load/store part of the instruction tree,
+ // and call the corresponding visitors.
+ // On entry, instruction bits 29:25 = 0x6.
+ void DecodeNEONLoadStore(const Instruction* instr);
+
+ // Decode the Advanced SIMD (NEON) vector data processing part of the
+ // instruction tree, and call the corresponding visitors.
+ // On entry, instruction bits 28:25 = 0x7.
+ void DecodeNEONVectorDataProcessing(const Instruction* instr);
+
+ // Decode the Advanced SIMD (NEON) scalar data processing part of the
+ // instruction tree, and call the corresponding visitors.
+ // On entry, instruction bits 28:25 = 0xF.
+ void DecodeNEONScalarDataProcessing(const Instruction* instr);
+
+ private:
+ // Visitors are registered in a list.
+ mozilla::Vector<DecoderVisitor*, 8, js::SystemAllocPolicy> visitors_;
+};
+
+} // namespace vixl
+
+#endif // VIXL_A64_DECODER_A64_H_
diff --git a/js/src/jit/arm64/vixl/Disasm-vixl.cpp b/js/src/jit/arm64/vixl/Disasm-vixl.cpp
new file mode 100644
index 0000000000..97448654b1
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Disasm-vixl.cpp
@@ -0,0 +1,3519 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Disasm-vixl.h"
+
+#include "mozilla/Sprintf.h"
+#include <cstdlib>
+
+namespace vixl {
+
+Disassembler::Disassembler() {
+ buffer_size_ = 256;
+ buffer_ = reinterpret_cast<char*>(malloc(buffer_size_));
+ buffer_pos_ = 0;
+ own_buffer_ = true;
+ code_address_offset_ = 0;
+}
+
+
+Disassembler::Disassembler(char* text_buffer, int buffer_size) {
+ buffer_size_ = buffer_size;
+ buffer_ = text_buffer;
+ buffer_pos_ = 0;
+ own_buffer_ = false;
+ code_address_offset_ = 0;
+}
+
+
+Disassembler::~Disassembler() {
+ if (own_buffer_) {
+ free(buffer_);
+ }
+}
+
+
+char* Disassembler::GetOutput() {
+ return buffer_;
+}
+
+
+void Disassembler::VisitAddSubImmediate(const Instruction* instr) {
+ bool rd_is_zr = RdIsZROrSP(instr);
+ bool stack_op = (rd_is_zr || RnIsZROrSP(instr)) &&
+ (instr->ImmAddSub() == 0) ? true : false;
+ const char *mnemonic = "";
+ const char *form = "'Rds, 'Rns, 'IAddSub";
+ const char *form_cmp = "'Rns, 'IAddSub";
+ const char *form_mov = "'Rds, 'Rns";
+
+ switch (instr->Mask(AddSubImmediateMask)) {
+ case ADD_w_imm:
+ case ADD_x_imm: {
+ mnemonic = "add";
+ if (stack_op) {
+ mnemonic = "mov";
+ form = form_mov;
+ }
+ break;
+ }
+ case ADDS_w_imm:
+ case ADDS_x_imm: {
+ mnemonic = "adds";
+ if (rd_is_zr) {
+ mnemonic = "cmn";
+ form = form_cmp;
+ }
+ break;
+ }
+ case SUB_w_imm:
+ case SUB_x_imm: mnemonic = "sub"; break;
+ case SUBS_w_imm:
+ case SUBS_x_imm: {
+ mnemonic = "subs";
+ if (rd_is_zr) {
+ mnemonic = "cmp";
+ form = form_cmp;
+ }
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubShifted(const Instruction* instr) {
+ bool rd_is_zr = RdIsZROrSP(instr);
+ bool rn_is_zr = RnIsZROrSP(instr);
+ const char *mnemonic = "";
+ const char *form = "'Rd, 'Rn, 'Rm'NDP";
+ const char *form_cmp = "'Rn, 'Rm'NDP";
+ const char *form_neg = "'Rd, 'Rm'NDP";
+
+ switch (instr->Mask(AddSubShiftedMask)) {
+ case ADD_w_shift:
+ case ADD_x_shift: mnemonic = "add"; break;
+ case ADDS_w_shift:
+ case ADDS_x_shift: {
+ mnemonic = "adds";
+ if (rd_is_zr) {
+ mnemonic = "cmn";
+ form = form_cmp;
+ }
+ break;
+ }
+ case SUB_w_shift:
+ case SUB_x_shift: {
+ mnemonic = "sub";
+ if (rn_is_zr) {
+ mnemonic = "neg";
+ form = form_neg;
+ }
+ break;
+ }
+ case SUBS_w_shift:
+ case SUBS_x_shift: {
+ mnemonic = "subs";
+ if (rd_is_zr) {
+ mnemonic = "cmp";
+ form = form_cmp;
+ } else if (rn_is_zr) {
+ mnemonic = "negs";
+ form = form_neg;
+ }
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubExtended(const Instruction* instr) {
+ bool rd_is_zr = RdIsZROrSP(instr);
+ const char *mnemonic = "";
+ Extend mode = static_cast<Extend>(instr->ExtendMode());
+ const char *form = ((mode == UXTX) || (mode == SXTX)) ?
+ "'Rds, 'Rns, 'Xm'Ext" : "'Rds, 'Rns, 'Wm'Ext";
+ const char *form_cmp = ((mode == UXTX) || (mode == SXTX)) ?
+ "'Rns, 'Xm'Ext" : "'Rns, 'Wm'Ext";
+
+ switch (instr->Mask(AddSubExtendedMask)) {
+ case ADD_w_ext:
+ case ADD_x_ext: mnemonic = "add"; break;
+ case ADDS_w_ext:
+ case ADDS_x_ext: {
+ mnemonic = "adds";
+ if (rd_is_zr) {
+ mnemonic = "cmn";
+ form = form_cmp;
+ }
+ break;
+ }
+ case SUB_w_ext:
+ case SUB_x_ext: mnemonic = "sub"; break;
+ case SUBS_w_ext:
+ case SUBS_x_ext: {
+ mnemonic = "subs";
+ if (rd_is_zr) {
+ mnemonic = "cmp";
+ form = form_cmp;
+ }
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitAddSubWithCarry(const Instruction* instr) {
+ bool rn_is_zr = RnIsZROrSP(instr);
+ const char *mnemonic = "";
+ const char *form = "'Rd, 'Rn, 'Rm";
+ const char *form_neg = "'Rd, 'Rm";
+
+ switch (instr->Mask(AddSubWithCarryMask)) {
+ case ADC_w:
+ case ADC_x: mnemonic = "adc"; break;
+ case ADCS_w:
+ case ADCS_x: mnemonic = "adcs"; break;
+ case SBC_w:
+ case SBC_x: {
+ mnemonic = "sbc";
+ if (rn_is_zr) {
+ mnemonic = "ngc";
+ form = form_neg;
+ }
+ break;
+ }
+ case SBCS_w:
+ case SBCS_x: {
+ mnemonic = "sbcs";
+ if (rn_is_zr) {
+ mnemonic = "ngcs";
+ form = form_neg;
+ }
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLogicalImmediate(const Instruction* instr) {
+ bool rd_is_zr = RdIsZROrSP(instr);
+ bool rn_is_zr = RnIsZROrSP(instr);
+ const char *mnemonic = "";
+ const char *form = "'Rds, 'Rn, 'ITri";
+
+ if (instr->ImmLogical() == 0) {
+ // The immediate encoded in the instruction is not in the expected format.
+ Format(instr, "unallocated", "(LogicalImmediate)");
+ return;
+ }
+
+ switch (instr->Mask(LogicalImmediateMask)) {
+ case AND_w_imm:
+ case AND_x_imm: mnemonic = "and"; break;
+ case ORR_w_imm:
+ case ORR_x_imm: {
+ mnemonic = "orr";
+ unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize
+ : kWRegSize;
+ if (rn_is_zr && !IsMovzMovnImm(reg_size, instr->ImmLogical())) {
+ mnemonic = "mov";
+ form = "'Rds, 'ITri";
+ }
+ break;
+ }
+ case EOR_w_imm:
+ case EOR_x_imm: mnemonic = "eor"; break;
+ case ANDS_w_imm:
+ case ANDS_x_imm: {
+ mnemonic = "ands";
+ if (rd_is_zr) {
+ mnemonic = "tst";
+ form = "'Rn, 'ITri";
+ }
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+bool Disassembler::IsMovzMovnImm(unsigned reg_size, uint64_t value) {
+ VIXL_ASSERT((reg_size == kXRegSize) ||
+ ((reg_size == kWRegSize) && (value <= 0xffffffff)));
+
+ // Test for movz: 16 bits set at positions 0, 16, 32 or 48.
+ if (((value & UINT64_C(0xffffffffffff0000)) == 0) ||
+ ((value & UINT64_C(0xffffffff0000ffff)) == 0) ||
+ ((value & UINT64_C(0xffff0000ffffffff)) == 0) ||
+ ((value & UINT64_C(0x0000ffffffffffff)) == 0)) {
+ return true;
+ }
+
+ // Test for movn: NOT(16 bits set at positions 0, 16, 32 or 48).
+ if ((reg_size == kXRegSize) &&
+ (((~value & UINT64_C(0xffffffffffff0000)) == 0) ||
+ ((~value & UINT64_C(0xffffffff0000ffff)) == 0) ||
+ ((~value & UINT64_C(0xffff0000ffffffff)) == 0) ||
+ ((~value & UINT64_C(0x0000ffffffffffff)) == 0))) {
+ return true;
+ }
+ if ((reg_size == kWRegSize) &&
+ (((value & 0xffff0000) == 0xffff0000) ||
+ ((value & 0x0000ffff) == 0x0000ffff))) {
+ return true;
+ }
+ return false;
+}
+
+
+void Disassembler::VisitLogicalShifted(const Instruction* instr) {
+ bool rd_is_zr = RdIsZROrSP(instr);
+ bool rn_is_zr = RnIsZROrSP(instr);
+ const char *mnemonic = "";
+ const char *form = "'Rd, 'Rn, 'Rm'NLo";
+
+ switch (instr->Mask(LogicalShiftedMask)) {
+ case AND_w:
+ case AND_x: mnemonic = "and"; break;
+ case BIC_w:
+ case BIC_x: mnemonic = "bic"; break;
+ case EOR_w:
+ case EOR_x: mnemonic = "eor"; break;
+ case EON_w:
+ case EON_x: mnemonic = "eon"; break;
+ case BICS_w:
+ case BICS_x: mnemonic = "bics"; break;
+ case ANDS_w:
+ case ANDS_x: {
+ mnemonic = "ands";
+ if (rd_is_zr) {
+ mnemonic = "tst";
+ form = "'Rn, 'Rm'NLo";
+ }
+ break;
+ }
+ case ORR_w:
+ case ORR_x: {
+ mnemonic = "orr";
+ if (rn_is_zr && (instr->ImmDPShift() == 0) && (instr->ShiftDP() == LSL)) {
+ mnemonic = "mov";
+ form = "'Rd, 'Rm";
+ }
+ break;
+ }
+ case ORN_w:
+ case ORN_x: {
+ mnemonic = "orn";
+ if (rn_is_zr) {
+ mnemonic = "mvn";
+ form = "'Rd, 'Rm'NLo";
+ }
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalCompareRegister(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Rn, 'Rm, 'INzcv, 'Cond";
+
+ switch (instr->Mask(ConditionalCompareRegisterMask)) {
+ case CCMN_w:
+ case CCMN_x: mnemonic = "ccmn"; break;
+ case CCMP_w:
+ case CCMP_x: mnemonic = "ccmp"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalCompareImmediate(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Rn, 'IP, 'INzcv, 'Cond";
+
+ switch (instr->Mask(ConditionalCompareImmediateMask)) {
+ case CCMN_w_imm:
+ case CCMN_x_imm: mnemonic = "ccmn"; break;
+ case CCMP_w_imm:
+ case CCMP_x_imm: mnemonic = "ccmp"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitConditionalSelect(const Instruction* instr) {
+ bool rnm_is_zr = (RnIsZROrSP(instr) && RmIsZROrSP(instr));
+ bool rn_is_rm = (instr->Rn() == instr->Rm());
+ const char *mnemonic = "";
+ const char *form = "'Rd, 'Rn, 'Rm, 'Cond";
+ const char *form_test = "'Rd, 'CInv";
+ const char *form_update = "'Rd, 'Rn, 'CInv";
+
+ Condition cond = static_cast<Condition>(instr->Condition());
+ bool invertible_cond = (cond != al) && (cond != nv);
+
+ switch (instr->Mask(ConditionalSelectMask)) {
+ case CSEL_w:
+ case CSEL_x: mnemonic = "csel"; break;
+ case CSINC_w:
+ case CSINC_x: {
+ mnemonic = "csinc";
+ if (rnm_is_zr && invertible_cond) {
+ mnemonic = "cset";
+ form = form_test;
+ } else if (rn_is_rm && invertible_cond) {
+ mnemonic = "cinc";
+ form = form_update;
+ }
+ break;
+ }
+ case CSINV_w:
+ case CSINV_x: {
+ mnemonic = "csinv";
+ if (rnm_is_zr && invertible_cond) {
+ mnemonic = "csetm";
+ form = form_test;
+ } else if (rn_is_rm && invertible_cond) {
+ mnemonic = "cinv";
+ form = form_update;
+ }
+ break;
+ }
+ case CSNEG_w:
+ case CSNEG_x: {
+ mnemonic = "csneg";
+ if (rn_is_rm && invertible_cond) {
+ mnemonic = "cneg";
+ form = form_update;
+ }
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitBitfield(const Instruction* instr) {
+ unsigned s = instr->ImmS();
+ unsigned r = instr->ImmR();
+ unsigned rd_size_minus_1 =
+ ((instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize) - 1;
+ const char *mnemonic = "";
+ const char *form = "";
+ const char *form_shift_right = "'Rd, 'Rn, 'IBr";
+ const char *form_extend = "'Rd, 'Wn";
+ const char *form_bfiz = "'Rd, 'Rn, 'IBZ-r, 'IBs+1";
+ const char *form_bfx = "'Rd, 'Rn, 'IBr, 'IBs-r+1";
+ const char *form_lsl = "'Rd, 'Rn, 'IBZ-r";
+
+ switch (instr->Mask(BitfieldMask)) {
+ case SBFM_w:
+ case SBFM_x: {
+ mnemonic = "sbfx";
+ form = form_bfx;
+ if (r == 0) {
+ form = form_extend;
+ if (s == 7) {
+ mnemonic = "sxtb";
+ } else if (s == 15) {
+ mnemonic = "sxth";
+ } else if ((s == 31) && (instr->SixtyFourBits() == 1)) {
+ mnemonic = "sxtw";
+ } else {
+ form = form_bfx;
+ }
+ } else if (s == rd_size_minus_1) {
+ mnemonic = "asr";
+ form = form_shift_right;
+ } else if (s < r) {
+ mnemonic = "sbfiz";
+ form = form_bfiz;
+ }
+ break;
+ }
+ case UBFM_w:
+ case UBFM_x: {
+ mnemonic = "ubfx";
+ form = form_bfx;
+ if (r == 0) {
+ form = form_extend;
+ if (s == 7) {
+ mnemonic = "uxtb";
+ } else if (s == 15) {
+ mnemonic = "uxth";
+ } else {
+ form = form_bfx;
+ }
+ }
+ if (s == rd_size_minus_1) {
+ mnemonic = "lsr";
+ form = form_shift_right;
+ } else if (r == s + 1) {
+ mnemonic = "lsl";
+ form = form_lsl;
+ } else if (s < r) {
+ mnemonic = "ubfiz";
+ form = form_bfiz;
+ }
+ break;
+ }
+ case BFM_w:
+ case BFM_x: {
+ mnemonic = "bfxil";
+ form = form_bfx;
+ if (s < r) {
+ mnemonic = "bfi";
+ form = form_bfiz;
+ }
+ }
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitExtract(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Rd, 'Rn, 'Rm, 'IExtract";
+
+ switch (instr->Mask(ExtractMask)) {
+ case EXTR_w:
+ case EXTR_x: {
+ if (instr->Rn() == instr->Rm()) {
+ mnemonic = "ror";
+ form = "'Rd, 'Rn, 'IExtract";
+ } else {
+ mnemonic = "extr";
+ }
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitPCRelAddressing(const Instruction* instr) {
+ switch (instr->Mask(PCRelAddressingMask)) {
+ case ADR: Format(instr, "adr", "'Xd, 'AddrPCRelByte"); break;
+ case ADRP: Format(instr, "adrp", "'Xd, 'AddrPCRelPage"); break;
+ default: Format(instr, "unimplemented", "(PCRelAddressing)");
+ }
+}
+
+
+void Disassembler::VisitConditionalBranch(const Instruction* instr) {
+ switch (instr->Mask(ConditionalBranchMask)) {
+ case B_cond: Format(instr, "b.'CBrn", "'TImmCond"); break;
+ default: VIXL_UNREACHABLE();
+ }
+}
+
+
+void Disassembler::VisitUnconditionalBranchToRegister(
+ const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Xn";
+
+ switch (instr->Mask(UnconditionalBranchToRegisterMask)) {
+ case BR: mnemonic = "br"; break;
+ case BLR: mnemonic = "blr"; break;
+ case RET: {
+ mnemonic = "ret";
+ if (instr->Rn() == kLinkRegCode) {
+ form = NULL;
+ }
+ break;
+ }
+ default: form = "(UnconditionalBranchToRegister)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitUnconditionalBranch(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'TImmUncn";
+
+ switch (instr->Mask(UnconditionalBranchMask)) {
+ case B: mnemonic = "b"; break;
+ case BL: mnemonic = "bl"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing1Source(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Rd, 'Rn";
+
+ switch (instr->Mask(DataProcessing1SourceMask)) {
+ #define FORMAT(A, B) \
+ case A##_w: \
+ case A##_x: mnemonic = B; break;
+ FORMAT(RBIT, "rbit");
+ FORMAT(REV16, "rev16");
+ FORMAT(REV, "rev");
+ FORMAT(CLZ, "clz");
+ FORMAT(CLS, "cls");
+ #undef FORMAT
+ case REV32_x: mnemonic = "rev32"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing2Source(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Rd, 'Rn, 'Rm";
+ const char *form_wwx = "'Wd, 'Wn, 'Xm";
+
+ switch (instr->Mask(DataProcessing2SourceMask)) {
+ #define FORMAT(A, B) \
+ case A##_w: \
+ case A##_x: mnemonic = B; break;
+ FORMAT(UDIV, "udiv");
+ FORMAT(SDIV, "sdiv");
+ FORMAT(LSLV, "lsl");
+ FORMAT(LSRV, "lsr");
+ FORMAT(ASRV, "asr");
+ FORMAT(RORV, "ror");
+ #undef FORMAT
+ case CRC32B: mnemonic = "crc32b"; break;
+ case CRC32H: mnemonic = "crc32h"; break;
+ case CRC32W: mnemonic = "crc32w"; break;
+ case CRC32X: mnemonic = "crc32x"; form = form_wwx; break;
+ case CRC32CB: mnemonic = "crc32cb"; break;
+ case CRC32CH: mnemonic = "crc32ch"; break;
+ case CRC32CW: mnemonic = "crc32cw"; break;
+ case CRC32CX: mnemonic = "crc32cx"; form = form_wwx; break;
+ default: form = "(DataProcessing2Source)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitDataProcessing3Source(const Instruction* instr) {
+ bool ra_is_zr = RaIsZROrSP(instr);
+ const char *mnemonic = "";
+ const char *form = "'Xd, 'Wn, 'Wm, 'Xa";
+ const char *form_rrr = "'Rd, 'Rn, 'Rm";
+ const char *form_rrrr = "'Rd, 'Rn, 'Rm, 'Ra";
+ const char *form_xww = "'Xd, 'Wn, 'Wm";
+ const char *form_xxx = "'Xd, 'Xn, 'Xm";
+
+ switch (instr->Mask(DataProcessing3SourceMask)) {
+ case MADD_w:
+ case MADD_x: {
+ mnemonic = "madd";
+ form = form_rrrr;
+ if (ra_is_zr) {
+ mnemonic = "mul";
+ form = form_rrr;
+ }
+ break;
+ }
+ case MSUB_w:
+ case MSUB_x: {
+ mnemonic = "msub";
+ form = form_rrrr;
+ if (ra_is_zr) {
+ mnemonic = "mneg";
+ form = form_rrr;
+ }
+ break;
+ }
+ case SMADDL_x: {
+ mnemonic = "smaddl";
+ if (ra_is_zr) {
+ mnemonic = "smull";
+ form = form_xww;
+ }
+ break;
+ }
+ case SMSUBL_x: {
+ mnemonic = "smsubl";
+ if (ra_is_zr) {
+ mnemonic = "smnegl";
+ form = form_xww;
+ }
+ break;
+ }
+ case UMADDL_x: {
+ mnemonic = "umaddl";
+ if (ra_is_zr) {
+ mnemonic = "umull";
+ form = form_xww;
+ }
+ break;
+ }
+ case UMSUBL_x: {
+ mnemonic = "umsubl";
+ if (ra_is_zr) {
+ mnemonic = "umnegl";
+ form = form_xww;
+ }
+ break;
+ }
+ case SMULH_x: {
+ mnemonic = "smulh";
+ form = form_xxx;
+ break;
+ }
+ case UMULH_x: {
+ mnemonic = "umulh";
+ form = form_xxx;
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitCompareBranch(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Rt, 'TImmCmpa";
+
+ switch (instr->Mask(CompareBranchMask)) {
+ case CBZ_w:
+ case CBZ_x: mnemonic = "cbz"; break;
+ case CBNZ_w:
+ case CBNZ_x: mnemonic = "cbnz"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitTestBranch(const Instruction* instr) {
+ const char *mnemonic = "";
+ // If the top bit of the immediate is clear, the tested register is
+ // disassembled as Wt, otherwise Xt. As the top bit of the immediate is
+ // encoded in bit 31 of the instruction, we can reuse the Rt form, which
+ // uses bit 31 (normally "sf") to choose the register size.
+ const char *form = "'Rt, 'IS, 'TImmTest";
+
+ switch (instr->Mask(TestBranchMask)) {
+ case TBZ: mnemonic = "tbz"; break;
+ case TBNZ: mnemonic = "tbnz"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitMoveWideImmediate(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Rd, 'IMoveImm";
+
+ // Print the shift separately for movk, to make it clear which half word will
+ // be overwritten. Movn and movz print the computed immediate, which includes
+ // shift calculation.
+ switch (instr->Mask(MoveWideImmediateMask)) {
+ case MOVN_w:
+ case MOVN_x:
+ if ((instr->ImmMoveWide()) || (instr->ShiftMoveWide() == 0)) {
+ if ((instr->SixtyFourBits() == 0) && (instr->ImmMoveWide() == 0xffff)) {
+ mnemonic = "movn";
+ } else {
+ mnemonic = "mov";
+ form = "'Rd, 'IMoveNeg";
+ }
+ } else {
+ mnemonic = "movn";
+ }
+ break;
+ case MOVZ_w:
+ case MOVZ_x:
+ if ((instr->ImmMoveWide()) || (instr->ShiftMoveWide() == 0))
+ mnemonic = "mov";
+ else
+ mnemonic = "movz";
+ break;
+ case MOVK_w:
+ case MOVK_x: mnemonic = "movk"; form = "'Rd, 'IMoveLSL"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+#define LOAD_STORE_LIST(V) \
+ V(STRB_w, "strb", "'Wt") \
+ V(STRH_w, "strh", "'Wt") \
+ V(STR_w, "str", "'Wt") \
+ V(STR_x, "str", "'Xt") \
+ V(LDRB_w, "ldrb", "'Wt") \
+ V(LDRH_w, "ldrh", "'Wt") \
+ V(LDR_w, "ldr", "'Wt") \
+ V(LDR_x, "ldr", "'Xt") \
+ V(LDRSB_x, "ldrsb", "'Xt") \
+ V(LDRSH_x, "ldrsh", "'Xt") \
+ V(LDRSW_x, "ldrsw", "'Xt") \
+ V(LDRSB_w, "ldrsb", "'Wt") \
+ V(LDRSH_w, "ldrsh", "'Wt") \
+ V(STR_b, "str", "'Bt") \
+ V(STR_h, "str", "'Ht") \
+ V(STR_s, "str", "'St") \
+ V(STR_d, "str", "'Dt") \
+ V(LDR_b, "ldr", "'Bt") \
+ V(LDR_h, "ldr", "'Ht") \
+ V(LDR_s, "ldr", "'St") \
+ V(LDR_d, "ldr", "'Dt") \
+ V(STR_q, "str", "'Qt") \
+ V(LDR_q, "ldr", "'Qt")
+
+void Disassembler::VisitLoadStorePreIndex(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(LoadStorePreIndex)";
+
+ switch (instr->Mask(LoadStorePreIndexMask)) {
+ #define LS_PREINDEX(A, B, C) \
+ case A##_pre: mnemonic = B; form = C ", ['Xns'ILS]!"; break;
+ LOAD_STORE_LIST(LS_PREINDEX)
+ #undef LS_PREINDEX
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePostIndex(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(LoadStorePostIndex)";
+
+ switch (instr->Mask(LoadStorePostIndexMask)) {
+ #define LS_POSTINDEX(A, B, C) \
+ case A##_post: mnemonic = B; form = C ", ['Xns]'ILS"; break;
+ LOAD_STORE_LIST(LS_POSTINDEX)
+ #undef LS_POSTINDEX
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreUnsignedOffset(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(LoadStoreUnsignedOffset)";
+
+ switch (instr->Mask(LoadStoreUnsignedOffsetMask)) {
+ #define LS_UNSIGNEDOFFSET(A, B, C) \
+ case A##_unsigned: mnemonic = B; form = C ", ['Xns'ILU]"; break;
+ LOAD_STORE_LIST(LS_UNSIGNEDOFFSET)
+ #undef LS_UNSIGNEDOFFSET
+ case PRFM_unsigned: mnemonic = "prfm"; form = "'PrefOp, ['Xns'ILU]";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreRegisterOffset(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(LoadStoreRegisterOffset)";
+
+ switch (instr->Mask(LoadStoreRegisterOffsetMask)) {
+ #define LS_REGISTEROFFSET(A, B, C) \
+ case A##_reg: mnemonic = B; form = C ", ['Xns, 'Offsetreg]"; break;
+ LOAD_STORE_LIST(LS_REGISTEROFFSET)
+ #undef LS_REGISTEROFFSET
+ case PRFM_reg: mnemonic = "prfm"; form = "'PrefOp, ['Xns, 'Offsetreg]";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreUnscaledOffset(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Wt, ['Xns'ILS]";
+ const char *form_x = "'Xt, ['Xns'ILS]";
+ const char *form_b = "'Bt, ['Xns'ILS]";
+ const char *form_h = "'Ht, ['Xns'ILS]";
+ const char *form_s = "'St, ['Xns'ILS]";
+ const char *form_d = "'Dt, ['Xns'ILS]";
+ const char *form_q = "'Qt, ['Xns'ILS]";
+ const char *form_prefetch = "'PrefOp, ['Xns'ILS]";
+
+ switch (instr->Mask(LoadStoreUnscaledOffsetMask)) {
+ case STURB_w: mnemonic = "sturb"; break;
+ case STURH_w: mnemonic = "sturh"; break;
+ case STUR_w: mnemonic = "stur"; break;
+ case STUR_x: mnemonic = "stur"; form = form_x; break;
+ case STUR_b: mnemonic = "stur"; form = form_b; break;
+ case STUR_h: mnemonic = "stur"; form = form_h; break;
+ case STUR_s: mnemonic = "stur"; form = form_s; break;
+ case STUR_d: mnemonic = "stur"; form = form_d; break;
+ case STUR_q: mnemonic = "stur"; form = form_q; break;
+ case LDURB_w: mnemonic = "ldurb"; break;
+ case LDURH_w: mnemonic = "ldurh"; break;
+ case LDUR_w: mnemonic = "ldur"; break;
+ case LDUR_x: mnemonic = "ldur"; form = form_x; break;
+ case LDUR_b: mnemonic = "ldur"; form = form_b; break;
+ case LDUR_h: mnemonic = "ldur"; form = form_h; break;
+ case LDUR_s: mnemonic = "ldur"; form = form_s; break;
+ case LDUR_d: mnemonic = "ldur"; form = form_d; break;
+ case LDUR_q: mnemonic = "ldur"; form = form_q; break;
+ case LDURSB_x: form = form_x; VIXL_FALLTHROUGH();
+ case LDURSB_w: mnemonic = "ldursb"; break;
+ case LDURSH_x: form = form_x; VIXL_FALLTHROUGH();
+ case LDURSH_w: mnemonic = "ldursh"; break;
+ case LDURSW_x: mnemonic = "ldursw"; form = form_x; break;
+ case PRFUM: mnemonic = "prfum"; form = form_prefetch; break;
+ default: form = "(LoadStoreUnscaledOffset)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadLiteral(const Instruction* instr) {
+ const char *mnemonic = "ldr";
+ const char *form = "(LoadLiteral)";
+
+ switch (instr->Mask(LoadLiteralMask)) {
+ case LDR_w_lit: form = "'Wt, 'ILLiteral 'LValue"; break;
+ case LDR_x_lit: form = "'Xt, 'ILLiteral 'LValue"; break;
+ case LDR_s_lit: form = "'St, 'ILLiteral 'LValue"; break;
+ case LDR_d_lit: form = "'Dt, 'ILLiteral 'LValue"; break;
+ case LDR_q_lit: form = "'Qt, 'ILLiteral 'LValue"; break;
+ case LDRSW_x_lit: {
+ mnemonic = "ldrsw";
+ form = "'Xt, 'ILLiteral 'LValue";
+ break;
+ }
+ case PRFM_lit: {
+ mnemonic = "prfm";
+ form = "'PrefOp, 'ILLiteral 'LValue";
+ break;
+ }
+ default: mnemonic = "unimplemented";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+#define LOAD_STORE_PAIR_LIST(V) \
+ V(STP_w, "stp", "'Wt, 'Wt2", "2") \
+ V(LDP_w, "ldp", "'Wt, 'Wt2", "2") \
+ V(LDPSW_x, "ldpsw", "'Xt, 'Xt2", "2") \
+ V(STP_x, "stp", "'Xt, 'Xt2", "3") \
+ V(LDP_x, "ldp", "'Xt, 'Xt2", "3") \
+ V(STP_s, "stp", "'St, 'St2", "2") \
+ V(LDP_s, "ldp", "'St, 'St2", "2") \
+ V(STP_d, "stp", "'Dt, 'Dt2", "3") \
+ V(LDP_d, "ldp", "'Dt, 'Dt2", "3") \
+ V(LDP_q, "ldp", "'Qt, 'Qt2", "4") \
+ V(STP_q, "stp", "'Qt, 'Qt2", "4")
+
+void Disassembler::VisitLoadStorePairPostIndex(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(LoadStorePairPostIndex)";
+
+ switch (instr->Mask(LoadStorePairPostIndexMask)) {
+ #define LSP_POSTINDEX(A, B, C, D) \
+ case A##_post: mnemonic = B; form = C ", ['Xns]'ILP" D; break;
+ LOAD_STORE_PAIR_LIST(LSP_POSTINDEX)
+ #undef LSP_POSTINDEX
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairPreIndex(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(LoadStorePairPreIndex)";
+
+ switch (instr->Mask(LoadStorePairPreIndexMask)) {
+ #define LSP_PREINDEX(A, B, C, D) \
+ case A##_pre: mnemonic = B; form = C ", ['Xns'ILP" D "]!"; break;
+ LOAD_STORE_PAIR_LIST(LSP_PREINDEX)
+ #undef LSP_PREINDEX
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairOffset(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(LoadStorePairOffset)";
+
+ switch (instr->Mask(LoadStorePairOffsetMask)) {
+ #define LSP_OFFSET(A, B, C, D) \
+ case A##_off: mnemonic = B; form = C ", ['Xns'ILP" D "]"; break;
+ LOAD_STORE_PAIR_LIST(LSP_OFFSET)
+ #undef LSP_OFFSET
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStorePairNonTemporal(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form;
+
+ switch (instr->Mask(LoadStorePairNonTemporalMask)) {
+ case STNP_w: mnemonic = "stnp"; form = "'Wt, 'Wt2, ['Xns'ILP2]"; break;
+ case LDNP_w: mnemonic = "ldnp"; form = "'Wt, 'Wt2, ['Xns'ILP2]"; break;
+ case STNP_x: mnemonic = "stnp"; form = "'Xt, 'Xt2, ['Xns'ILP3]"; break;
+ case LDNP_x: mnemonic = "ldnp"; form = "'Xt, 'Xt2, ['Xns'ILP3]"; break;
+ case STNP_s: mnemonic = "stnp"; form = "'St, 'St2, ['Xns'ILP2]"; break;
+ case LDNP_s: mnemonic = "ldnp"; form = "'St, 'St2, ['Xns'ILP2]"; break;
+ case STNP_d: mnemonic = "stnp"; form = "'Dt, 'Dt2, ['Xns'ILP3]"; break;
+ case LDNP_d: mnemonic = "ldnp"; form = "'Dt, 'Dt2, ['Xns'ILP3]"; break;
+ case STNP_q: mnemonic = "stnp"; form = "'Qt, 'Qt2, ['Xns'ILP4]"; break;
+ case LDNP_q: mnemonic = "ldnp"; form = "'Qt, 'Qt2, ['Xns'ILP4]"; break;
+ default: form = "(LoadStorePairNonTemporal)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitLoadStoreExclusive(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form;
+
+ switch (instr->Mask(LoadStoreExclusiveMask)) {
+ case STXRB_w: mnemonic = "stxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
+ case STXRH_w: mnemonic = "stxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
+ case STXR_w: mnemonic = "stxr"; form = "'Ws, 'Wt, ['Xns]"; break;
+ case STXR_x: mnemonic = "stxr"; form = "'Ws, 'Xt, ['Xns]"; break;
+ case LDXRB_w: mnemonic = "ldxrb"; form = "'Wt, ['Xns]"; break;
+ case LDXRH_w: mnemonic = "ldxrh"; form = "'Wt, ['Xns]"; break;
+ case LDXR_w: mnemonic = "ldxr"; form = "'Wt, ['Xns]"; break;
+ case LDXR_x: mnemonic = "ldxr"; form = "'Xt, ['Xns]"; break;
+ case STXP_w: mnemonic = "stxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
+ case STXP_x: mnemonic = "stxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
+ case LDXP_w: mnemonic = "ldxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
+ case LDXP_x: mnemonic = "ldxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
+ case STLXRB_w: mnemonic = "stlxrb"; form = "'Ws, 'Wt, ['Xns]"; break;
+ case STLXRH_w: mnemonic = "stlxrh"; form = "'Ws, 'Wt, ['Xns]"; break;
+ case STLXR_w: mnemonic = "stlxr"; form = "'Ws, 'Wt, ['Xns]"; break;
+ case STLXR_x: mnemonic = "stlxr"; form = "'Ws, 'Xt, ['Xns]"; break;
+ case LDAXRB_w: mnemonic = "ldaxrb"; form = "'Wt, ['Xns]"; break;
+ case LDAXRH_w: mnemonic = "ldaxrh"; form = "'Wt, ['Xns]"; break;
+ case LDAXR_w: mnemonic = "ldaxr"; form = "'Wt, ['Xns]"; break;
+ case LDAXR_x: mnemonic = "ldaxr"; form = "'Xt, ['Xns]"; break;
+ case STLXP_w: mnemonic = "stlxp"; form = "'Ws, 'Wt, 'Wt2, ['Xns]"; break;
+ case STLXP_x: mnemonic = "stlxp"; form = "'Ws, 'Xt, 'Xt2, ['Xns]"; break;
+ case LDAXP_w: mnemonic = "ldaxp"; form = "'Wt, 'Wt2, ['Xns]"; break;
+ case LDAXP_x: mnemonic = "ldaxp"; form = "'Xt, 'Xt2, ['Xns]"; break;
+ case STLRB_w: mnemonic = "stlrb"; form = "'Wt, ['Xns]"; break;
+ case STLRH_w: mnemonic = "stlrh"; form = "'Wt, ['Xns]"; break;
+ case STLR_w: mnemonic = "stlr"; form = "'Wt, ['Xns]"; break;
+ case STLR_x: mnemonic = "stlr"; form = "'Xt, ['Xns]"; break;
+ case LDARB_w: mnemonic = "ldarb"; form = "'Wt, ['Xns]"; break;
+ case LDARH_w: mnemonic = "ldarh"; form = "'Wt, ['Xns]"; break;
+ case LDAR_w: mnemonic = "ldar"; form = "'Wt, ['Xns]"; break;
+ case LDAR_x: mnemonic = "ldar"; form = "'Xt, ['Xns]"; break;
+ default: form = "(LoadStoreExclusive)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPCompare(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Fn, 'Fm";
+ const char *form_zero = "'Fn, #0.0";
+
+ switch (instr->Mask(FPCompareMask)) {
+ case FCMP_s_zero:
+ case FCMP_d_zero: form = form_zero; VIXL_FALLTHROUGH();
+ case FCMP_s:
+ case FCMP_d: mnemonic = "fcmp"; break;
+ case FCMPE_s_zero:
+ case FCMPE_d_zero: form = form_zero; VIXL_FALLTHROUGH();
+ case FCMPE_s:
+ case FCMPE_d: mnemonic = "fcmpe"; break;
+ default: form = "(FPCompare)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPConditionalCompare(const Instruction* instr) {
+ const char *mnemonic = "unmplemented";
+ const char *form = "'Fn, 'Fm, 'INzcv, 'Cond";
+
+ switch (instr->Mask(FPConditionalCompareMask)) {
+ case FCCMP_s:
+ case FCCMP_d: mnemonic = "fccmp"; break;
+ case FCCMPE_s:
+ case FCCMPE_d: mnemonic = "fccmpe"; break;
+ default: form = "(FPConditionalCompare)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPConditionalSelect(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Fd, 'Fn, 'Fm, 'Cond";
+
+ switch (instr->Mask(FPConditionalSelectMask)) {
+ case FCSEL_s:
+ case FCSEL_d: mnemonic = "fcsel"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing1Source(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Fd, 'Fn";
+
+ switch (instr->Mask(FPDataProcessing1SourceMask)) {
+ #define FORMAT(A, B) \
+ case A##_s: \
+ case A##_d: mnemonic = B; break;
+ FORMAT(FMOV, "fmov");
+ FORMAT(FABS, "fabs");
+ FORMAT(FNEG, "fneg");
+ FORMAT(FSQRT, "fsqrt");
+ FORMAT(FRINTN, "frintn");
+ FORMAT(FRINTP, "frintp");
+ FORMAT(FRINTM, "frintm");
+ FORMAT(FRINTZ, "frintz");
+ FORMAT(FRINTA, "frinta");
+ FORMAT(FRINTX, "frintx");
+ FORMAT(FRINTI, "frinti");
+ #undef FORMAT
+ case FCVT_ds: mnemonic = "fcvt"; form = "'Dd, 'Sn"; break;
+ case FCVT_sd: mnemonic = "fcvt"; form = "'Sd, 'Dn"; break;
+ case FCVT_hs: mnemonic = "fcvt"; form = "'Hd, 'Sn"; break;
+ case FCVT_sh: mnemonic = "fcvt"; form = "'Sd, 'Hn"; break;
+ case FCVT_dh: mnemonic = "fcvt"; form = "'Dd, 'Hn"; break;
+ case FCVT_hd: mnemonic = "fcvt"; form = "'Hd, 'Dn"; break;
+ default: form = "(FPDataProcessing1Source)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing2Source(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Fd, 'Fn, 'Fm";
+
+ switch (instr->Mask(FPDataProcessing2SourceMask)) {
+ #define FORMAT(A, B) \
+ case A##_s: \
+ case A##_d: mnemonic = B; break;
+ FORMAT(FMUL, "fmul");
+ FORMAT(FDIV, "fdiv");
+ FORMAT(FADD, "fadd");
+ FORMAT(FSUB, "fsub");
+ FORMAT(FMAX, "fmax");
+ FORMAT(FMIN, "fmin");
+ FORMAT(FMAXNM, "fmaxnm");
+ FORMAT(FMINNM, "fminnm");
+ FORMAT(FNMUL, "fnmul");
+ #undef FORMAT
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPDataProcessing3Source(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Fd, 'Fn, 'Fm, 'Fa";
+
+ switch (instr->Mask(FPDataProcessing3SourceMask)) {
+ #define FORMAT(A, B) \
+ case A##_s: \
+ case A##_d: mnemonic = B; break;
+ FORMAT(FMADD, "fmadd");
+ FORMAT(FMSUB, "fmsub");
+ FORMAT(FNMADD, "fnmadd");
+ FORMAT(FNMSUB, "fnmsub");
+ #undef FORMAT
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPImmediate(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "(FPImmediate)";
+
+ switch (instr->Mask(FPImmediateMask)) {
+ case FMOV_s_imm: mnemonic = "fmov"; form = "'Sd, 'IFPSingle"; break;
+ case FMOV_d_imm: mnemonic = "fmov"; form = "'Dd, 'IFPDouble"; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPIntegerConvert(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(FPIntegerConvert)";
+ const char *form_rf = "'Rd, 'Fn";
+ const char *form_fr = "'Fd, 'Rn";
+
+ switch (instr->Mask(FPIntegerConvertMask)) {
+ case FMOV_ws:
+ case FMOV_xd: mnemonic = "fmov"; form = form_rf; break;
+ case FMOV_sw:
+ case FMOV_dx: mnemonic = "fmov"; form = form_fr; break;
+ case FMOV_d1_x: mnemonic = "fmov"; form = "'Vd.D[1], 'Rn"; break;
+ case FMOV_x_d1: mnemonic = "fmov"; form = "'Rd, 'Vn.D[1]"; break;
+ case FCVTAS_ws:
+ case FCVTAS_xs:
+ case FCVTAS_wd:
+ case FCVTAS_xd: mnemonic = "fcvtas"; form = form_rf; break;
+ case FCVTAU_ws:
+ case FCVTAU_xs:
+ case FCVTAU_wd:
+ case FCVTAU_xd: mnemonic = "fcvtau"; form = form_rf; break;
+ case FCVTMS_ws:
+ case FCVTMS_xs:
+ case FCVTMS_wd:
+ case FCVTMS_xd: mnemonic = "fcvtms"; form = form_rf; break;
+ case FCVTMU_ws:
+ case FCVTMU_xs:
+ case FCVTMU_wd:
+ case FCVTMU_xd: mnemonic = "fcvtmu"; form = form_rf; break;
+ case FCVTNS_ws:
+ case FCVTNS_xs:
+ case FCVTNS_wd:
+ case FCVTNS_xd: mnemonic = "fcvtns"; form = form_rf; break;
+ case FCVTNU_ws:
+ case FCVTNU_xs:
+ case FCVTNU_wd:
+ case FCVTNU_xd: mnemonic = "fcvtnu"; form = form_rf; break;
+ case FCVTZU_xd:
+ case FCVTZU_ws:
+ case FCVTZU_wd:
+ case FCVTZU_xs: mnemonic = "fcvtzu"; form = form_rf; break;
+ case FCVTZS_xd:
+ case FCVTZS_wd:
+ case FCVTZS_xs:
+ case FCVTZS_ws: mnemonic = "fcvtzs"; form = form_rf; break;
+ case FCVTPU_xd:
+ case FCVTPU_ws:
+ case FCVTPU_wd:
+ case FCVTPU_xs: mnemonic = "fcvtpu"; form = form_rf; break;
+ case FCVTPS_xd:
+ case FCVTPS_wd:
+ case FCVTPS_xs:
+ case FCVTPS_ws: mnemonic = "fcvtps"; form = form_rf; break;
+ case SCVTF_sw:
+ case SCVTF_sx:
+ case SCVTF_dw:
+ case SCVTF_dx: mnemonic = "scvtf"; form = form_fr; break;
+ case UCVTF_sw:
+ case UCVTF_sx:
+ case UCVTF_dw:
+ case UCVTF_dx: mnemonic = "ucvtf"; form = form_fr; break;
+ case FJCVTZS: mnemonic = "fjcvtzs"; form = form_rf; break;
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitFPFixedPointConvert(const Instruction* instr) {
+ const char *mnemonic = "";
+ const char *form = "'Rd, 'Fn, 'IFPFBits";
+ const char *form_fr = "'Fd, 'Rn, 'IFPFBits";
+
+ switch (instr->Mask(FPFixedPointConvertMask)) {
+ case FCVTZS_ws_fixed:
+ case FCVTZS_xs_fixed:
+ case FCVTZS_wd_fixed:
+ case FCVTZS_xd_fixed: mnemonic = "fcvtzs"; break;
+ case FCVTZU_ws_fixed:
+ case FCVTZU_xs_fixed:
+ case FCVTZU_wd_fixed:
+ case FCVTZU_xd_fixed: mnemonic = "fcvtzu"; break;
+ case SCVTF_sw_fixed:
+ case SCVTF_sx_fixed:
+ case SCVTF_dw_fixed:
+ case SCVTF_dx_fixed: mnemonic = "scvtf"; form = form_fr; break;
+ case UCVTF_sw_fixed:
+ case UCVTF_sx_fixed:
+ case UCVTF_dw_fixed:
+ case UCVTF_dx_fixed: mnemonic = "ucvtf"; form = form_fr; break;
+ default: VIXL_UNREACHABLE();
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitSystem(const Instruction* instr) {
+ // Some system instructions hijack their Op and Cp fields to represent a
+ // range of immediates instead of indicating a different instruction. This
+ // makes the decoding tricky.
+ const char *mnemonic = "unimplemented";
+ const char *form = "(System)";
+
+ if (instr->Mask(SystemExclusiveMonitorFMask) == SystemExclusiveMonitorFixed) {
+ switch (instr->Mask(SystemExclusiveMonitorMask)) {
+ case CLREX: {
+ mnemonic = "clrex";
+ form = (instr->CRm() == 0xf) ? NULL : "'IX";
+ break;
+ }
+ }
+ } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
+ switch (instr->Mask(SystemSysRegMask)) {
+ case MRS: {
+ mnemonic = "mrs";
+ switch (instr->ImmSystemRegister()) {
+ case NZCV: form = "'Xt, nzcv"; break;
+ case FPCR: form = "'Xt, fpcr"; break;
+ default: form = "'Xt, (unknown)"; break;
+ }
+ break;
+ }
+ case MSR: {
+ mnemonic = "msr";
+ switch (instr->ImmSystemRegister()) {
+ case NZCV: form = "nzcv, 'Xt"; break;
+ case FPCR: form = "fpcr, 'Xt"; break;
+ default: form = "(unknown), 'Xt"; break;
+ }
+ break;
+ }
+ }
+ } else if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
+ switch (instr->ImmHint()) {
+ case NOP: {
+ mnemonic = "nop";
+ form = NULL;
+ break;
+ }
+ case CSDB: {
+ mnemonic = "csdb";
+ form = NULL;
+ break;
+ }
+ }
+ } else if (instr->Mask(MemBarrierFMask) == MemBarrierFixed) {
+ switch (instr->Mask(MemBarrierMask)) {
+ case DMB: {
+ mnemonic = "dmb";
+ form = "'M";
+ break;
+ }
+ case DSB: {
+ mnemonic = "dsb";
+ form = "'M";
+ break;
+ }
+ case ISB: {
+ mnemonic = "isb";
+ form = NULL;
+ break;
+ }
+ }
+ } else if (instr->Mask(SystemSysFMask) == SystemSysFixed) {
+ switch (instr->SysOp()) {
+ case IVAU:
+ mnemonic = "ic";
+ form = "ivau, 'Xt";
+ break;
+ case CVAC:
+ mnemonic = "dc";
+ form = "cvac, 'Xt";
+ break;
+ case CVAU:
+ mnemonic = "dc";
+ form = "cvau, 'Xt";
+ break;
+ case CIVAC:
+ mnemonic = "dc";
+ form = "civac, 'Xt";
+ break;
+ case ZVA:
+ mnemonic = "dc";
+ form = "zva, 'Xt";
+ break;
+ default:
+ mnemonic = "sys";
+ if (instr->Rt() == 31) {
+ form = "'G1, 'Kn, 'Km, 'G2";
+ } else {
+ form = "'G1, 'Kn, 'Km, 'G2, 'Xt";
+ }
+ break;
+ }
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitException(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'IDebug";
+
+ switch (instr->Mask(ExceptionMask)) {
+ case HLT: mnemonic = "hlt"; break;
+ case BRK: mnemonic = "brk"; break;
+ case SVC: mnemonic = "svc"; break;
+ case HVC: mnemonic = "hvc"; break;
+ case SMC: mnemonic = "smc"; break;
+ case DCPS1: mnemonic = "dcps1"; form = "{'IDebug}"; break;
+ case DCPS2: mnemonic = "dcps2"; form = "{'IDebug}"; break;
+ case DCPS3: mnemonic = "dcps3"; form = "{'IDebug}"; break;
+ default: form = "(Exception)";
+ }
+ Format(instr, mnemonic, form);
+}
+
+
+void Disassembler::VisitCrypto2RegSHA(const Instruction* instr) {
+ VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitCrypto3RegSHA(const Instruction* instr) {
+ VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitCryptoAES(const Instruction* instr) {
+ VisitUnimplemented(instr);
+}
+
+
+void Disassembler::VisitNEON2RegMisc(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Vd.%s, 'Vn.%s";
+ const char *form_cmp_zero = "'Vd.%s, 'Vn.%s, #0";
+ const char *form_fcmp_zero = "'Vd.%s, 'Vn.%s, #0.0";
+ NEONFormatDecoder nfd(instr);
+
+ static const NEONFormatMap map_lp_ta = {
+ {23, 22, 30}, {NF_4H, NF_8H, NF_2S, NF_4S, NF_1D, NF_2D}
+ };
+
+ static const NEONFormatMap map_cvt_ta = {
+ {22}, {NF_4S, NF_2D}
+ };
+
+ static const NEONFormatMap map_cvt_tb = {
+ {22, 30}, {NF_4H, NF_8H, NF_2S, NF_4S}
+ };
+
+ if (instr->Mask(NEON2RegMiscOpcode) <= NEON_NEG_opcode) {
+ // These instructions all use a two bit size field, except NOT and RBIT,
+ // which use the field to encode the operation.
+ switch (instr->Mask(NEON2RegMiscMask)) {
+ case NEON_REV64: mnemonic = "rev64"; break;
+ case NEON_REV32: mnemonic = "rev32"; break;
+ case NEON_REV16: mnemonic = "rev16"; break;
+ case NEON_SADDLP:
+ mnemonic = "saddlp";
+ nfd.SetFormatMap(0, &map_lp_ta);
+ break;
+ case NEON_UADDLP:
+ mnemonic = "uaddlp";
+ nfd.SetFormatMap(0, &map_lp_ta);
+ break;
+ case NEON_SUQADD: mnemonic = "suqadd"; break;
+ case NEON_USQADD: mnemonic = "usqadd"; break;
+ case NEON_CLS: mnemonic = "cls"; break;
+ case NEON_CLZ: mnemonic = "clz"; break;
+ case NEON_CNT: mnemonic = "cnt"; break;
+ case NEON_SADALP:
+ mnemonic = "sadalp";
+ nfd.SetFormatMap(0, &map_lp_ta);
+ break;
+ case NEON_UADALP:
+ mnemonic = "uadalp";
+ nfd.SetFormatMap(0, &map_lp_ta);
+ break;
+ case NEON_SQABS: mnemonic = "sqabs"; break;
+ case NEON_SQNEG: mnemonic = "sqneg"; break;
+ case NEON_CMGT_zero: mnemonic = "cmgt"; form = form_cmp_zero; break;
+ case NEON_CMGE_zero: mnemonic = "cmge"; form = form_cmp_zero; break;
+ case NEON_CMEQ_zero: mnemonic = "cmeq"; form = form_cmp_zero; break;
+ case NEON_CMLE_zero: mnemonic = "cmle"; form = form_cmp_zero; break;
+ case NEON_CMLT_zero: mnemonic = "cmlt"; form = form_cmp_zero; break;
+ case NEON_ABS: mnemonic = "abs"; break;
+ case NEON_NEG: mnemonic = "neg"; break;
+ case NEON_RBIT_NOT:
+ switch (instr->FPType()) {
+ case 0: mnemonic = "mvn"; break;
+ case 1: mnemonic = "rbit"; break;
+ default: form = "(NEON2RegMisc)";
+ }
+ nfd.SetFormatMaps(nfd.LogicalFormatMap());
+ break;
+ }
+ } else {
+ // These instructions all use a one bit size field, except XTN, SQXTUN,
+ // SHLL, SQXTN and UQXTN, which use a two bit size field.
+ nfd.SetFormatMaps(nfd.FPFormatMap());
+ switch (instr->Mask(NEON2RegMiscFPMask)) {
+ case NEON_FABS: mnemonic = "fabs"; break;
+ case NEON_FNEG: mnemonic = "fneg"; break;
+ case NEON_FCVTN:
+ mnemonic = instr->Mask(NEON_Q) ? "fcvtn2" : "fcvtn";
+ nfd.SetFormatMap(0, &map_cvt_tb);
+ nfd.SetFormatMap(1, &map_cvt_ta);
+ break;
+ case NEON_FCVTXN:
+ mnemonic = instr->Mask(NEON_Q) ? "fcvtxn2" : "fcvtxn";
+ nfd.SetFormatMap(0, &map_cvt_tb);
+ nfd.SetFormatMap(1, &map_cvt_ta);
+ break;
+ case NEON_FCVTL:
+ mnemonic = instr->Mask(NEON_Q) ? "fcvtl2" : "fcvtl";
+ nfd.SetFormatMap(0, &map_cvt_ta);
+ nfd.SetFormatMap(1, &map_cvt_tb);
+ break;
+ case NEON_FRINTN: mnemonic = "frintn"; break;
+ case NEON_FRINTA: mnemonic = "frinta"; break;
+ case NEON_FRINTP: mnemonic = "frintp"; break;
+ case NEON_FRINTM: mnemonic = "frintm"; break;
+ case NEON_FRINTX: mnemonic = "frintx"; break;
+ case NEON_FRINTZ: mnemonic = "frintz"; break;
+ case NEON_FRINTI: mnemonic = "frinti"; break;
+ case NEON_FCVTNS: mnemonic = "fcvtns"; break;
+ case NEON_FCVTNU: mnemonic = "fcvtnu"; break;
+ case NEON_FCVTPS: mnemonic = "fcvtps"; break;
+ case NEON_FCVTPU: mnemonic = "fcvtpu"; break;
+ case NEON_FCVTMS: mnemonic = "fcvtms"; break;
+ case NEON_FCVTMU: mnemonic = "fcvtmu"; break;
+ case NEON_FCVTZS: mnemonic = "fcvtzs"; break;
+ case NEON_FCVTZU: mnemonic = "fcvtzu"; break;
+ case NEON_FCVTAS: mnemonic = "fcvtas"; break;
+ case NEON_FCVTAU: mnemonic = "fcvtau"; break;
+ case NEON_FSQRT: mnemonic = "fsqrt"; break;
+ case NEON_SCVTF: mnemonic = "scvtf"; break;
+ case NEON_UCVTF: mnemonic = "ucvtf"; break;
+ case NEON_URSQRTE: mnemonic = "ursqrte"; break;
+ case NEON_URECPE: mnemonic = "urecpe"; break;
+ case NEON_FRSQRTE: mnemonic = "frsqrte"; break;
+ case NEON_FRECPE: mnemonic = "frecpe"; break;
+ case NEON_FCMGT_zero: mnemonic = "fcmgt"; form = form_fcmp_zero; break;
+ case NEON_FCMGE_zero: mnemonic = "fcmge"; form = form_fcmp_zero; break;
+ case NEON_FCMEQ_zero: mnemonic = "fcmeq"; form = form_fcmp_zero; break;
+ case NEON_FCMLE_zero: mnemonic = "fcmle"; form = form_fcmp_zero; break;
+ case NEON_FCMLT_zero: mnemonic = "fcmlt"; form = form_fcmp_zero; break;
+ default:
+ if ((NEON_XTN_opcode <= instr->Mask(NEON2RegMiscOpcode)) &&
+ (instr->Mask(NEON2RegMiscOpcode) <= NEON_UQXTN_opcode)) {
+ nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+ nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+
+ switch (instr->Mask(NEON2RegMiscMask)) {
+ case NEON_XTN: mnemonic = "xtn"; break;
+ case NEON_SQXTN: mnemonic = "sqxtn"; break;
+ case NEON_UQXTN: mnemonic = "uqxtn"; break;
+ case NEON_SQXTUN: mnemonic = "sqxtun"; break;
+ case NEON_SHLL:
+ mnemonic = "shll";
+ nfd.SetFormatMap(0, nfd.LongIntegerFormatMap());
+ nfd.SetFormatMap(1, nfd.IntegerFormatMap());
+ switch (instr->NEONSize()) {
+ case 0: form = "'Vd.%s, 'Vn.%s, #8"; break;
+ case 1: form = "'Vd.%s, 'Vn.%s, #16"; break;
+ case 2: form = "'Vd.%s, 'Vn.%s, #32"; break;
+ default: form = "(NEON2RegMisc)";
+ }
+ }
+ Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+ return;
+ } else {
+ form = "(NEON2RegMisc)";
+ }
+ }
+ }
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEON3Same(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+ NEONFormatDecoder nfd(instr);
+
+ if (instr->Mask(NEON3SameLogicalFMask) == NEON3SameLogicalFixed) {
+ switch (instr->Mask(NEON3SameLogicalMask)) {
+ case NEON_AND: mnemonic = "and"; break;
+ case NEON_ORR:
+ mnemonic = "orr";
+ if (instr->Rm() == instr->Rn()) {
+ mnemonic = "mov";
+ form = "'Vd.%s, 'Vn.%s";
+ }
+ break;
+ case NEON_ORN: mnemonic = "orn"; break;
+ case NEON_EOR: mnemonic = "eor"; break;
+ case NEON_BIC: mnemonic = "bic"; break;
+ case NEON_BIF: mnemonic = "bif"; break;
+ case NEON_BIT: mnemonic = "bit"; break;
+ case NEON_BSL: mnemonic = "bsl"; break;
+ default: form = "(NEON3Same)";
+ }
+ nfd.SetFormatMaps(nfd.LogicalFormatMap());
+ } else {
+ static const char *mnemonics[] = {
+ "shadd", "uhadd", "shadd", "uhadd",
+ "sqadd", "uqadd", "sqadd", "uqadd",
+ "srhadd", "urhadd", "srhadd", "urhadd",
+ NULL, NULL, NULL, NULL, // Handled by logical cases above.
+ "shsub", "uhsub", "shsub", "uhsub",
+ "sqsub", "uqsub", "sqsub", "uqsub",
+ "cmgt", "cmhi", "cmgt", "cmhi",
+ "cmge", "cmhs", "cmge", "cmhs",
+ "sshl", "ushl", "sshl", "ushl",
+ "sqshl", "uqshl", "sqshl", "uqshl",
+ "srshl", "urshl", "srshl", "urshl",
+ "sqrshl", "uqrshl", "sqrshl", "uqrshl",
+ "smax", "umax", "smax", "umax",
+ "smin", "umin", "smin", "umin",
+ "sabd", "uabd", "sabd", "uabd",
+ "saba", "uaba", "saba", "uaba",
+ "add", "sub", "add", "sub",
+ "cmtst", "cmeq", "cmtst", "cmeq",
+ "mla", "mls", "mla", "mls",
+ "mul", "pmul", "mul", "pmul",
+ "smaxp", "umaxp", "smaxp", "umaxp",
+ "sminp", "uminp", "sminp", "uminp",
+ "sqdmulh", "sqrdmulh", "sqdmulh", "sqrdmulh",
+ "addp", "unallocated", "addp", "unallocated",
+ "fmaxnm", "fmaxnmp", "fminnm", "fminnmp",
+ "fmla", "unallocated", "fmls", "unallocated",
+ "fadd", "faddp", "fsub", "fabd",
+ "fmulx", "fmul", "unallocated", "unallocated",
+ "fcmeq", "fcmge", "unallocated", "fcmgt",
+ "unallocated", "facge", "unallocated", "facgt",
+ "fmax", "fmaxp", "fmin", "fminp",
+ "frecps", "fdiv", "frsqrts", "unallocated"};
+
+ // Operation is determined by the opcode bits (15-11), the top bit of
+ // size (23) and the U bit (29).
+ unsigned index = (instr->Bits(15, 11) << 2) | (instr->Bit(23) << 1) |
+ instr->Bit(29);
+ VIXL_ASSERT(index < (sizeof(mnemonics) / sizeof(mnemonics[0])));
+ mnemonic = mnemonics[index];
+ // Assert that index is not one of the previously handled logical
+ // instructions.
+ VIXL_ASSERT(mnemonic != NULL);
+
+ if (instr->Mask(NEON3SameFPFMask) == NEON3SameFPFixed) {
+ nfd.SetFormatMaps(nfd.FPFormatMap());
+ }
+ }
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEON3Different(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+
+ NEONFormatDecoder nfd(instr);
+ nfd.SetFormatMap(0, nfd.LongIntegerFormatMap());
+
+ // Ignore the Q bit. Appending a "2" suffix is handled later.
+ switch (instr->Mask(NEON3DifferentMask) & ~NEON_Q) {
+ case NEON_PMULL: mnemonic = "pmull"; break;
+ case NEON_SABAL: mnemonic = "sabal"; break;
+ case NEON_SABDL: mnemonic = "sabdl"; break;
+ case NEON_SADDL: mnemonic = "saddl"; break;
+ case NEON_SMLAL: mnemonic = "smlal"; break;
+ case NEON_SMLSL: mnemonic = "smlsl"; break;
+ case NEON_SMULL: mnemonic = "smull"; break;
+ case NEON_SSUBL: mnemonic = "ssubl"; break;
+ case NEON_SQDMLAL: mnemonic = "sqdmlal"; break;
+ case NEON_SQDMLSL: mnemonic = "sqdmlsl"; break;
+ case NEON_SQDMULL: mnemonic = "sqdmull"; break;
+ case NEON_UABAL: mnemonic = "uabal"; break;
+ case NEON_UABDL: mnemonic = "uabdl"; break;
+ case NEON_UADDL: mnemonic = "uaddl"; break;
+ case NEON_UMLAL: mnemonic = "umlal"; break;
+ case NEON_UMLSL: mnemonic = "umlsl"; break;
+ case NEON_UMULL: mnemonic = "umull"; break;
+ case NEON_USUBL: mnemonic = "usubl"; break;
+ case NEON_SADDW:
+ mnemonic = "saddw";
+ nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+ break;
+ case NEON_SSUBW:
+ mnemonic = "ssubw";
+ nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+ break;
+ case NEON_UADDW:
+ mnemonic = "uaddw";
+ nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+ break;
+ case NEON_USUBW:
+ mnemonic = "usubw";
+ nfd.SetFormatMap(1, nfd.LongIntegerFormatMap());
+ break;
+ case NEON_ADDHN:
+ mnemonic = "addhn";
+ nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+ nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+ break;
+ case NEON_RADDHN:
+ mnemonic = "raddhn";
+ nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+ nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+ break;
+ case NEON_RSUBHN:
+ mnemonic = "rsubhn";
+ nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+ nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+ break;
+ case NEON_SUBHN:
+ mnemonic = "subhn";
+ nfd.SetFormatMaps(nfd.LongIntegerFormatMap());
+ nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+ break;
+ default: form = "(NEON3Different)";
+ }
+ Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONAcrossLanes(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "%sd, 'Vn.%s";
+
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap(),
+ NEONFormatDecoder::IntegerFormatMap());
+
+ if (instr->Mask(NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
+ nfd.SetFormatMap(0, nfd.FPScalarFormatMap());
+ nfd.SetFormatMap(1, nfd.FPFormatMap());
+ switch (instr->Mask(NEONAcrossLanesFPMask)) {
+ case NEON_FMAXV: mnemonic = "fmaxv"; break;
+ case NEON_FMINV: mnemonic = "fminv"; break;
+ case NEON_FMAXNMV: mnemonic = "fmaxnmv"; break;
+ case NEON_FMINNMV: mnemonic = "fminnmv"; break;
+ default: form = "(NEONAcrossLanes)"; break;
+ }
+ } else if (instr->Mask(NEONAcrossLanesFMask) == NEONAcrossLanesFixed) {
+ switch (instr->Mask(NEONAcrossLanesMask)) {
+ case NEON_ADDV: mnemonic = "addv"; break;
+ case NEON_SMAXV: mnemonic = "smaxv"; break;
+ case NEON_SMINV: mnemonic = "sminv"; break;
+ case NEON_UMAXV: mnemonic = "umaxv"; break;
+ case NEON_UMINV: mnemonic = "uminv"; break;
+ case NEON_SADDLV:
+ mnemonic = "saddlv";
+ nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+ break;
+ case NEON_UADDLV:
+ mnemonic = "uaddlv";
+ nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+ break;
+ default: form = "(NEONAcrossLanes)"; break;
+ }
+ }
+ Format(instr, mnemonic, nfd.Substitute(form,
+ NEONFormatDecoder::kPlaceholder, NEONFormatDecoder::kFormat));
+}
+
+
+void Disassembler::VisitNEONByIndexedElement(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ bool l_instr = false;
+ bool fp_instr = false;
+
+ const char *form = "'Vd.%s, 'Vn.%s, 'Ve.%s['IVByElemIndex]";
+
+ static const NEONFormatMap map_ta = {
+ {23, 22}, {NF_UNDEF, NF_4S, NF_2D}
+ };
+ NEONFormatDecoder nfd(instr, &map_ta,
+ NEONFormatDecoder::IntegerFormatMap(),
+ NEONFormatDecoder::ScalarFormatMap());
+
+ switch (instr->Mask(NEONByIndexedElementMask)) {
+ case NEON_SMULL_byelement: mnemonic = "smull"; l_instr = true; break;
+ case NEON_UMULL_byelement: mnemonic = "umull"; l_instr = true; break;
+ case NEON_SMLAL_byelement: mnemonic = "smlal"; l_instr = true; break;
+ case NEON_UMLAL_byelement: mnemonic = "umlal"; l_instr = true; break;
+ case NEON_SMLSL_byelement: mnemonic = "smlsl"; l_instr = true; break;
+ case NEON_UMLSL_byelement: mnemonic = "umlsl"; l_instr = true; break;
+ case NEON_SQDMULL_byelement: mnemonic = "sqdmull"; l_instr = true; break;
+ case NEON_SQDMLAL_byelement: mnemonic = "sqdmlal"; l_instr = true; break;
+ case NEON_SQDMLSL_byelement: mnemonic = "sqdmlsl"; l_instr = true; break;
+ case NEON_MUL_byelement: mnemonic = "mul"; break;
+ case NEON_MLA_byelement: mnemonic = "mla"; break;
+ case NEON_MLS_byelement: mnemonic = "mls"; break;
+ case NEON_SQDMULH_byelement: mnemonic = "sqdmulh"; break;
+ case NEON_SQRDMULH_byelement: mnemonic = "sqrdmulh"; break;
+ default:
+ switch (instr->Mask(NEONByIndexedElementFPMask)) {
+ case NEON_FMUL_byelement: mnemonic = "fmul"; fp_instr = true; break;
+ case NEON_FMLA_byelement: mnemonic = "fmla"; fp_instr = true; break;
+ case NEON_FMLS_byelement: mnemonic = "fmls"; fp_instr = true; break;
+ case NEON_FMULX_byelement: mnemonic = "fmulx"; fp_instr = true; break;
+ }
+ }
+
+ if (l_instr) {
+ Format(instr, nfd.Mnemonic(mnemonic), nfd.Substitute(form));
+ } else if (fp_instr) {
+ nfd.SetFormatMap(0, nfd.FPFormatMap());
+ Format(instr, mnemonic, nfd.Substitute(form));
+ } else {
+ nfd.SetFormatMap(0, nfd.IntegerFormatMap());
+ Format(instr, mnemonic, nfd.Substitute(form));
+ }
+}
+
+
+void Disassembler::VisitNEONCopy(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(NEONCopy)";
+
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::TriangularFormatMap(),
+ NEONFormatDecoder::TriangularScalarFormatMap());
+
+ if (instr->Mask(NEONCopyInsElementMask) == NEON_INS_ELEMENT) {
+ mnemonic = "mov";
+ nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+ form = "'Vd.%s['IVInsIndex1], 'Vn.%s['IVInsIndex2]";
+ } else if (instr->Mask(NEONCopyInsGeneralMask) == NEON_INS_GENERAL) {
+ mnemonic = "mov";
+ nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+ if (nfd.GetVectorFormat() == kFormatD) {
+ form = "'Vd.%s['IVInsIndex1], 'Xn";
+ } else {
+ form = "'Vd.%s['IVInsIndex1], 'Wn";
+ }
+ } else if (instr->Mask(NEONCopyUmovMask) == NEON_UMOV) {
+ if (instr->Mask(NEON_Q) || ((instr->ImmNEON5() & 7) == 4)) {
+ mnemonic = "mov";
+ } else {
+ mnemonic = "umov";
+ }
+ nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+ if (nfd.GetVectorFormat() == kFormatD) {
+ form = "'Xd, 'Vn.%s['IVInsIndex1]";
+ } else {
+ form = "'Wd, 'Vn.%s['IVInsIndex1]";
+ }
+ } else if (instr->Mask(NEONCopySmovMask) == NEON_SMOV) {
+ mnemonic = "smov";
+ nfd.SetFormatMap(0, nfd.TriangularScalarFormatMap());
+ form = "'Rdq, 'Vn.%s['IVInsIndex1]";
+ } else if (instr->Mask(NEONCopyDupElementMask) == NEON_DUP_ELEMENT) {
+ mnemonic = "dup";
+ form = "'Vd.%s, 'Vn.%s['IVInsIndex1]";
+ } else if (instr->Mask(NEONCopyDupGeneralMask) == NEON_DUP_GENERAL) {
+ mnemonic = "dup";
+ if (nfd.GetVectorFormat() == kFormat2D) {
+ form = "'Vd.%s, 'Xn";
+ } else {
+ form = "'Vd.%s, 'Wn";
+ }
+ }
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONExtract(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(NEONExtract)";
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LogicalFormatMap());
+ if (instr->Mask(NEONExtractMask) == NEON_EXT) {
+ mnemonic = "ext";
+ form = "'Vd.%s, 'Vn.%s, 'Vm.%s, 'IVExtract";
+ }
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreMultiStruct(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(NEONLoadStoreMultiStruct)";
+ const char *form_1v = "{'Vt.%1$s}, ['Xns]";
+ const char *form_2v = "{'Vt.%1$s, 'Vt2.%1$s}, ['Xns]";
+ const char *form_3v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s}, ['Xns]";
+ const char *form_4v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns]";
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+ switch (instr->Mask(NEONLoadStoreMultiStructMask)) {
+ case NEON_LD1_1v: mnemonic = "ld1"; form = form_1v; break;
+ case NEON_LD1_2v: mnemonic = "ld1"; form = form_2v; break;
+ case NEON_LD1_3v: mnemonic = "ld1"; form = form_3v; break;
+ case NEON_LD1_4v: mnemonic = "ld1"; form = form_4v; break;
+ case NEON_LD2: mnemonic = "ld2"; form = form_2v; break;
+ case NEON_LD3: mnemonic = "ld3"; form = form_3v; break;
+ case NEON_LD4: mnemonic = "ld4"; form = form_4v; break;
+ case NEON_ST1_1v: mnemonic = "st1"; form = form_1v; break;
+ case NEON_ST1_2v: mnemonic = "st1"; form = form_2v; break;
+ case NEON_ST1_3v: mnemonic = "st1"; form = form_3v; break;
+ case NEON_ST1_4v: mnemonic = "st1"; form = form_4v; break;
+ case NEON_ST2: mnemonic = "st2"; form = form_2v; break;
+ case NEON_ST3: mnemonic = "st3"; form = form_3v; break;
+ case NEON_ST4: mnemonic = "st4"; form = form_4v; break;
+ default: break;
+ }
+
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreMultiStructPostIndex(
+ const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(NEONLoadStoreMultiStructPostIndex)";
+ const char *form_1v = "{'Vt.%1$s}, ['Xns], 'Xmr1";
+ const char *form_2v = "{'Vt.%1$s, 'Vt2.%1$s}, ['Xns], 'Xmr2";
+ const char *form_3v = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s}, ['Xns], 'Xmr3";
+ const char *form_4v =
+ "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns], 'Xmr4";
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+ switch (instr->Mask(NEONLoadStoreMultiStructPostIndexMask)) {
+ case NEON_LD1_1v_post: mnemonic = "ld1"; form = form_1v; break;
+ case NEON_LD1_2v_post: mnemonic = "ld1"; form = form_2v; break;
+ case NEON_LD1_3v_post: mnemonic = "ld1"; form = form_3v; break;
+ case NEON_LD1_4v_post: mnemonic = "ld1"; form = form_4v; break;
+ case NEON_LD2_post: mnemonic = "ld2"; form = form_2v; break;
+ case NEON_LD3_post: mnemonic = "ld3"; form = form_3v; break;
+ case NEON_LD4_post: mnemonic = "ld4"; form = form_4v; break;
+ case NEON_ST1_1v_post: mnemonic = "st1"; form = form_1v; break;
+ case NEON_ST1_2v_post: mnemonic = "st1"; form = form_2v; break;
+ case NEON_ST1_3v_post: mnemonic = "st1"; form = form_3v; break;
+ case NEON_ST1_4v_post: mnemonic = "st1"; form = form_4v; break;
+ case NEON_ST2_post: mnemonic = "st2"; form = form_2v; break;
+ case NEON_ST3_post: mnemonic = "st3"; form = form_3v; break;
+ case NEON_ST4_post: mnemonic = "st4"; form = form_4v; break;
+ default: break;
+ }
+
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreSingleStruct(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(NEONLoadStoreSingleStruct)";
+
+ const char *form_1b = "{'Vt.b}['IVLSLane0], ['Xns]";
+ const char *form_1h = "{'Vt.h}['IVLSLane1], ['Xns]";
+ const char *form_1s = "{'Vt.s}['IVLSLane2], ['Xns]";
+ const char *form_1d = "{'Vt.d}['IVLSLane3], ['Xns]";
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+ switch (instr->Mask(NEONLoadStoreSingleStructMask)) {
+ case NEON_LD1_b: mnemonic = "ld1"; form = form_1b; break;
+ case NEON_LD1_h: mnemonic = "ld1"; form = form_1h; break;
+ case NEON_LD1_s:
+ mnemonic = "ld1";
+ VIXL_STATIC_ASSERT((NEON_LD1_s | (1 << NEONLSSize_offset)) == NEON_LD1_d);
+ form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+ break;
+ case NEON_ST1_b: mnemonic = "st1"; form = form_1b; break;
+ case NEON_ST1_h: mnemonic = "st1"; form = form_1h; break;
+ case NEON_ST1_s:
+ mnemonic = "st1";
+ VIXL_STATIC_ASSERT((NEON_ST1_s | (1 << NEONLSSize_offset)) == NEON_ST1_d);
+ form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+ break;
+ case NEON_LD1R:
+ mnemonic = "ld1r";
+ form = "{'Vt.%s}, ['Xns]";
+ break;
+ case NEON_LD2_b:
+ case NEON_ST2_b:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+ form = "{'Vt.b, 'Vt2.b}['IVLSLane0], ['Xns]";
+ break;
+ case NEON_LD2_h:
+ case NEON_ST2_h:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+ form = "{'Vt.h, 'Vt2.h}['IVLSLane1], ['Xns]";
+ break;
+ case NEON_LD2_s:
+ case NEON_ST2_s:
+ VIXL_STATIC_ASSERT((NEON_ST2_s | (1 << NEONLSSize_offset)) == NEON_ST2_d);
+ VIXL_STATIC_ASSERT((NEON_LD2_s | (1 << NEONLSSize_offset)) == NEON_LD2_d);
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+ if ((instr->NEONLSSize() & 1) == 0)
+ form = "{'Vt.s, 'Vt2.s}['IVLSLane2], ['Xns]";
+ else
+ form = "{'Vt.d, 'Vt2.d}['IVLSLane3], ['Xns]";
+ break;
+ case NEON_LD2R:
+ mnemonic = "ld2r";
+ form = "{'Vt.%s, 'Vt2.%s}, ['Xns]";
+ break;
+ case NEON_LD3_b:
+ case NEON_ST3_b:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+ form = "{'Vt.b, 'Vt2.b, 'Vt3.b}['IVLSLane0], ['Xns]";
+ break;
+ case NEON_LD3_h:
+ case NEON_ST3_h:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+ form = "{'Vt.h, 'Vt2.h, 'Vt3.h}['IVLSLane1], ['Xns]";
+ break;
+ case NEON_LD3_s:
+ case NEON_ST3_s:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+ if ((instr->NEONLSSize() & 1) == 0)
+ form = "{'Vt.s, 'Vt2.s, 'Vt3.s}['IVLSLane2], ['Xns]";
+ else
+ form = "{'Vt.d, 'Vt2.d, 'Vt3.d}['IVLSLane3], ['Xns]";
+ break;
+ case NEON_LD3R:
+ mnemonic = "ld3r";
+ form = "{'Vt.%s, 'Vt2.%s, 'Vt3.%s}, ['Xns]";
+ break;
+ case NEON_LD4_b:
+ case NEON_ST4_b:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+ form = "{'Vt.b, 'Vt2.b, 'Vt3.b, 'Vt4.b}['IVLSLane0], ['Xns]";
+ break;
+ case NEON_LD4_h:
+ case NEON_ST4_h:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+ form = "{'Vt.h, 'Vt2.h, 'Vt3.h, 'Vt4.h}['IVLSLane1], ['Xns]";
+ break;
+ case NEON_LD4_s:
+ case NEON_ST4_s:
+ VIXL_STATIC_ASSERT((NEON_LD4_s | (1 << NEONLSSize_offset)) == NEON_LD4_d);
+ VIXL_STATIC_ASSERT((NEON_ST4_s | (1 << NEONLSSize_offset)) == NEON_ST4_d);
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+ if ((instr->NEONLSSize() & 1) == 0)
+ form = "{'Vt.s, 'Vt2.s, 'Vt3.s, 'Vt4.s}['IVLSLane2], ['Xns]";
+ else
+ form = "{'Vt.d, 'Vt2.d, 'Vt3.d, 'Vt4.d}['IVLSLane3], ['Xns]";
+ break;
+ case NEON_LD4R:
+ mnemonic = "ld4r";
+ form = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns]";
+ break;
+ default: break;
+ }
+
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONLoadStoreSingleStructPostIndex(
+ const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(NEONLoadStoreSingleStructPostIndex)";
+
+ const char *form_1b = "{'Vt.b}['IVLSLane0], ['Xns], 'Xmb1";
+ const char *form_1h = "{'Vt.h}['IVLSLane1], ['Xns], 'Xmb2";
+ const char *form_1s = "{'Vt.s}['IVLSLane2], ['Xns], 'Xmb4";
+ const char *form_1d = "{'Vt.d}['IVLSLane3], ['Xns], 'Xmb8";
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+
+ switch (instr->Mask(NEONLoadStoreSingleStructPostIndexMask)) {
+ case NEON_LD1_b_post: mnemonic = "ld1"; form = form_1b; break;
+ case NEON_LD1_h_post: mnemonic = "ld1"; form = form_1h; break;
+ case NEON_LD1_s_post:
+ mnemonic = "ld1";
+ VIXL_STATIC_ASSERT((NEON_LD1_s | (1 << NEONLSSize_offset)) == NEON_LD1_d);
+ form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+ break;
+ case NEON_ST1_b_post: mnemonic = "st1"; form = form_1b; break;
+ case NEON_ST1_h_post: mnemonic = "st1"; form = form_1h; break;
+ case NEON_ST1_s_post:
+ mnemonic = "st1";
+ VIXL_STATIC_ASSERT((NEON_ST1_s | (1 << NEONLSSize_offset)) == NEON_ST1_d);
+ form = ((instr->NEONLSSize() & 1) == 0) ? form_1s : form_1d;
+ break;
+ case NEON_LD1R_post:
+ mnemonic = "ld1r";
+ form = "{'Vt.%s}, ['Xns], 'Xmz1";
+ break;
+ case NEON_LD2_b_post:
+ case NEON_ST2_b_post:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+ form = "{'Vt.b, 'Vt2.b}['IVLSLane0], ['Xns], 'Xmb2";
+ break;
+ case NEON_ST2_h_post:
+ case NEON_LD2_h_post:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+ form = "{'Vt.h, 'Vt2.h}['IVLSLane1], ['Xns], 'Xmb4";
+ break;
+ case NEON_LD2_s_post:
+ case NEON_ST2_s_post:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld2" : "st2";
+ if ((instr->NEONLSSize() & 1) == 0)
+ form = "{'Vt.s, 'Vt2.s}['IVLSLane2], ['Xns], 'Xmb8";
+ else
+ form = "{'Vt.d, 'Vt2.d}['IVLSLane3], ['Xns], 'Xmb16";
+ break;
+ case NEON_LD2R_post:
+ mnemonic = "ld2r";
+ form = "{'Vt.%s, 'Vt2.%s}, ['Xns], 'Xmz2";
+ break;
+ case NEON_LD3_b_post:
+ case NEON_ST3_b_post:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+ form = "{'Vt.b, 'Vt2.b, 'Vt3.b}['IVLSLane0], ['Xns], 'Xmb3";
+ break;
+ case NEON_LD3_h_post:
+ case NEON_ST3_h_post:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+ form = "{'Vt.h, 'Vt2.h, 'Vt3.h}['IVLSLane1], ['Xns], 'Xmb6";
+ break;
+ case NEON_LD3_s_post:
+ case NEON_ST3_s_post:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld3" : "st3";
+ if ((instr->NEONLSSize() & 1) == 0)
+ form = "{'Vt.s, 'Vt2.s, 'Vt3.s}['IVLSLane2], ['Xns], 'Xmb12";
+ else
+ form = "{'Vt.d, 'Vt2.d, 'Vt3.d}['IVLSLane3], ['Xns], 'Xmr3";
+ break;
+ case NEON_LD3R_post:
+ mnemonic = "ld3r";
+ form = "{'Vt.%s, 'Vt2.%s, 'Vt3.%s}, ['Xns], 'Xmz3";
+ break;
+ case NEON_LD4_b_post:
+ case NEON_ST4_b_post:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+ form = "{'Vt.b, 'Vt2.b, 'Vt3.b, 'Vt4.b}['IVLSLane0], ['Xns], 'Xmb4";
+ break;
+ case NEON_LD4_h_post:
+ case NEON_ST4_h_post:
+ mnemonic = (instr->LdStXLoad()) == 1 ? "ld4" : "st4";
+ form = "{'Vt.h, 'Vt2.h, 'Vt3.h, 'Vt4.h}['IVLSLane1], ['Xns], 'Xmb8";
+ break;
+ case NEON_LD4_s_post:
+ case NEON_ST4_s_post:
+ mnemonic = (instr->LdStXLoad() == 1) ? "ld4" : "st4";
+ if ((instr->NEONLSSize() & 1) == 0)
+ form = "{'Vt.s, 'Vt2.s, 'Vt3.s, 'Vt4.s}['IVLSLane2], ['Xns], 'Xmb16";
+ else
+ form = "{'Vt.d, 'Vt2.d, 'Vt3.d, 'Vt4.d}['IVLSLane3], ['Xns], 'Xmb32";
+ break;
+ case NEON_LD4R_post:
+ mnemonic = "ld4r";
+ form = "{'Vt.%1$s, 'Vt2.%1$s, 'Vt3.%1$s, 'Vt4.%1$s}, ['Xns], 'Xmz4";
+ break;
+ default: break;
+ }
+
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONModifiedImmediate(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Vt.%s, 'IVMIImm8, lsl 'IVMIShiftAmt1";
+
+ int cmode = instr->NEONCmode();
+ int cmode_3 = (cmode >> 3) & 1;
+ int cmode_2 = (cmode >> 2) & 1;
+ int cmode_1 = (cmode >> 1) & 1;
+ int cmode_0 = cmode & 1;
+ int q = instr->NEONQ();
+ int op = instr->NEONModImmOp();
+
+ static const NEONFormatMap map_b = { {30}, {NF_8B, NF_16B} };
+ static const NEONFormatMap map_h = { {30}, {NF_4H, NF_8H} };
+ static const NEONFormatMap map_s = { {30}, {NF_2S, NF_4S} };
+ NEONFormatDecoder nfd(instr, &map_b);
+
+ if (cmode_3 == 0) {
+ if (cmode_0 == 0) {
+ mnemonic = (op == 1) ? "mvni" : "movi";
+ } else { // cmode<0> == '1'.
+ mnemonic = (op == 1) ? "bic" : "orr";
+ }
+ nfd.SetFormatMap(0, &map_s);
+ } else { // cmode<3> == '1'.
+ if (cmode_2 == 0) {
+ if (cmode_0 == 0) {
+ mnemonic = (op == 1) ? "mvni" : "movi";
+ } else { // cmode<0> == '1'.
+ mnemonic = (op == 1) ? "bic" : "orr";
+ }
+ nfd.SetFormatMap(0, &map_h);
+ } else { // cmode<2> == '1'.
+ if (cmode_1 == 0) {
+ mnemonic = (op == 1) ? "mvni" : "movi";
+ form = "'Vt.%s, 'IVMIImm8, msl 'IVMIShiftAmt2";
+ nfd.SetFormatMap(0, &map_s);
+ } else { // cmode<1> == '1'.
+ if (cmode_0 == 0) {
+ mnemonic = "movi";
+ if (op == 0) {
+ form = "'Vt.%s, 'IVMIImm8";
+ } else {
+ form = (q == 0) ? "'Dd, 'IVMIImm" : "'Vt.2d, 'IVMIImm";
+ }
+ } else { // cmode<0> == '1'
+ mnemonic = "fmov";
+ if (op == 0) {
+ form = "'Vt.%s, 'IVMIImmFPSingle";
+ nfd.SetFormatMap(0, &map_s);
+ } else {
+ if (q == 1) {
+ form = "'Vt.2d, 'IVMIImmFPDouble";
+ }
+ }
+ }
+ }
+ }
+ }
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONScalar2RegMisc(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "%sd, %sn";
+ const char *form_0 = "%sd, %sn, #0";
+ const char *form_fp0 = "%sd, %sn, #0.0";
+
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+
+ if (instr->Mask(NEON2RegMiscOpcode) <= NEON_NEG_scalar_opcode) {
+ // These instructions all use a two bit size field, except NOT and RBIT,
+ // which use the field to encode the operation.
+ switch (instr->Mask(NEONScalar2RegMiscMask)) {
+ case NEON_CMGT_zero_scalar: mnemonic = "cmgt"; form = form_0; break;
+ case NEON_CMGE_zero_scalar: mnemonic = "cmge"; form = form_0; break;
+ case NEON_CMLE_zero_scalar: mnemonic = "cmle"; form = form_0; break;
+ case NEON_CMLT_zero_scalar: mnemonic = "cmlt"; form = form_0; break;
+ case NEON_CMEQ_zero_scalar: mnemonic = "cmeq"; form = form_0; break;
+ case NEON_NEG_scalar: mnemonic = "neg"; break;
+ case NEON_SQNEG_scalar: mnemonic = "sqneg"; break;
+ case NEON_ABS_scalar: mnemonic = "abs"; break;
+ case NEON_SQABS_scalar: mnemonic = "sqabs"; break;
+ case NEON_SUQADD_scalar: mnemonic = "suqadd"; break;
+ case NEON_USQADD_scalar: mnemonic = "usqadd"; break;
+ default: form = "(NEONScalar2RegMisc)";
+ }
+ } else {
+ // These instructions all use a one bit size field, except SQXTUN, SQXTN
+ // and UQXTN, which use a two bit size field.
+ nfd.SetFormatMaps(nfd.FPScalarFormatMap());
+ switch (instr->Mask(NEONScalar2RegMiscFPMask)) {
+ case NEON_FRSQRTE_scalar: mnemonic = "frsqrte"; break;
+ case NEON_FRECPE_scalar: mnemonic = "frecpe"; break;
+ case NEON_SCVTF_scalar: mnemonic = "scvtf"; break;
+ case NEON_UCVTF_scalar: mnemonic = "ucvtf"; break;
+ case NEON_FCMGT_zero_scalar: mnemonic = "fcmgt"; form = form_fp0; break;
+ case NEON_FCMGE_zero_scalar: mnemonic = "fcmge"; form = form_fp0; break;
+ case NEON_FCMLE_zero_scalar: mnemonic = "fcmle"; form = form_fp0; break;
+ case NEON_FCMLT_zero_scalar: mnemonic = "fcmlt"; form = form_fp0; break;
+ case NEON_FCMEQ_zero_scalar: mnemonic = "fcmeq"; form = form_fp0; break;
+ case NEON_FRECPX_scalar: mnemonic = "frecpx"; break;
+ case NEON_FCVTNS_scalar: mnemonic = "fcvtns"; break;
+ case NEON_FCVTNU_scalar: mnemonic = "fcvtnu"; break;
+ case NEON_FCVTPS_scalar: mnemonic = "fcvtps"; break;
+ case NEON_FCVTPU_scalar: mnemonic = "fcvtpu"; break;
+ case NEON_FCVTMS_scalar: mnemonic = "fcvtms"; break;
+ case NEON_FCVTMU_scalar: mnemonic = "fcvtmu"; break;
+ case NEON_FCVTZS_scalar: mnemonic = "fcvtzs"; break;
+ case NEON_FCVTZU_scalar: mnemonic = "fcvtzu"; break;
+ case NEON_FCVTAS_scalar: mnemonic = "fcvtas"; break;
+ case NEON_FCVTAU_scalar: mnemonic = "fcvtau"; break;
+ case NEON_FCVTXN_scalar:
+ nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+ mnemonic = "fcvtxn";
+ break;
+ default:
+ nfd.SetFormatMap(0, nfd.ScalarFormatMap());
+ nfd.SetFormatMap(1, nfd.LongScalarFormatMap());
+ switch (instr->Mask(NEONScalar2RegMiscMask)) {
+ case NEON_SQXTN_scalar: mnemonic = "sqxtn"; break;
+ case NEON_UQXTN_scalar: mnemonic = "uqxtn"; break;
+ case NEON_SQXTUN_scalar: mnemonic = "sqxtun"; break;
+ default: form = "(NEONScalar2RegMisc)";
+ }
+ }
+ }
+ Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalar3Diff(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "%sd, %sn, %sm";
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LongScalarFormatMap(),
+ NEONFormatDecoder::ScalarFormatMap());
+
+ switch (instr->Mask(NEONScalar3DiffMask)) {
+ case NEON_SQDMLAL_scalar : mnemonic = "sqdmlal"; break;
+ case NEON_SQDMLSL_scalar : mnemonic = "sqdmlsl"; break;
+ case NEON_SQDMULL_scalar : mnemonic = "sqdmull"; break;
+ default: form = "(NEONScalar3Diff)";
+ }
+ Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalar3Same(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "%sd, %sn, %sm";
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+
+ if (instr->Mask(NEONScalar3SameFPFMask) == NEONScalar3SameFPFixed) {
+ nfd.SetFormatMaps(nfd.FPScalarFormatMap());
+ switch (instr->Mask(NEONScalar3SameFPMask)) {
+ case NEON_FACGE_scalar: mnemonic = "facge"; break;
+ case NEON_FACGT_scalar: mnemonic = "facgt"; break;
+ case NEON_FCMEQ_scalar: mnemonic = "fcmeq"; break;
+ case NEON_FCMGE_scalar: mnemonic = "fcmge"; break;
+ case NEON_FCMGT_scalar: mnemonic = "fcmgt"; break;
+ case NEON_FMULX_scalar: mnemonic = "fmulx"; break;
+ case NEON_FRECPS_scalar: mnemonic = "frecps"; break;
+ case NEON_FRSQRTS_scalar: mnemonic = "frsqrts"; break;
+ case NEON_FABD_scalar: mnemonic = "fabd"; break;
+ default: form = "(NEONScalar3Same)";
+ }
+ } else {
+ switch (instr->Mask(NEONScalar3SameMask)) {
+ case NEON_ADD_scalar: mnemonic = "add"; break;
+ case NEON_SUB_scalar: mnemonic = "sub"; break;
+ case NEON_CMEQ_scalar: mnemonic = "cmeq"; break;
+ case NEON_CMGE_scalar: mnemonic = "cmge"; break;
+ case NEON_CMGT_scalar: mnemonic = "cmgt"; break;
+ case NEON_CMHI_scalar: mnemonic = "cmhi"; break;
+ case NEON_CMHS_scalar: mnemonic = "cmhs"; break;
+ case NEON_CMTST_scalar: mnemonic = "cmtst"; break;
+ case NEON_UQADD_scalar: mnemonic = "uqadd"; break;
+ case NEON_SQADD_scalar: mnemonic = "sqadd"; break;
+ case NEON_UQSUB_scalar: mnemonic = "uqsub"; break;
+ case NEON_SQSUB_scalar: mnemonic = "sqsub"; break;
+ case NEON_USHL_scalar: mnemonic = "ushl"; break;
+ case NEON_SSHL_scalar: mnemonic = "sshl"; break;
+ case NEON_UQSHL_scalar: mnemonic = "uqshl"; break;
+ case NEON_SQSHL_scalar: mnemonic = "sqshl"; break;
+ case NEON_URSHL_scalar: mnemonic = "urshl"; break;
+ case NEON_SRSHL_scalar: mnemonic = "srshl"; break;
+ case NEON_UQRSHL_scalar: mnemonic = "uqrshl"; break;
+ case NEON_SQRSHL_scalar: mnemonic = "sqrshl"; break;
+ case NEON_SQDMULH_scalar: mnemonic = "sqdmulh"; break;
+ case NEON_SQRDMULH_scalar: mnemonic = "sqrdmulh"; break;
+ default: form = "(NEONScalar3Same)";
+ }
+ }
+ Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONScalarByIndexedElement(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "%sd, %sn, 'Ve.%s['IVByElemIndex]";
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+ bool long_instr = false;
+
+ switch (instr->Mask(NEONScalarByIndexedElementMask)) {
+ case NEON_SQDMULL_byelement_scalar:
+ mnemonic = "sqdmull";
+ long_instr = true;
+ break;
+ case NEON_SQDMLAL_byelement_scalar:
+ mnemonic = "sqdmlal";
+ long_instr = true;
+ break;
+ case NEON_SQDMLSL_byelement_scalar:
+ mnemonic = "sqdmlsl";
+ long_instr = true;
+ break;
+ case NEON_SQDMULH_byelement_scalar:
+ mnemonic = "sqdmulh";
+ break;
+ case NEON_SQRDMULH_byelement_scalar:
+ mnemonic = "sqrdmulh";
+ break;
+ default:
+ nfd.SetFormatMap(0, nfd.FPScalarFormatMap());
+ switch (instr->Mask(NEONScalarByIndexedElementFPMask)) {
+ case NEON_FMUL_byelement_scalar: mnemonic = "fmul"; break;
+ case NEON_FMLA_byelement_scalar: mnemonic = "fmla"; break;
+ case NEON_FMLS_byelement_scalar: mnemonic = "fmls"; break;
+ case NEON_FMULX_byelement_scalar: mnemonic = "fmulx"; break;
+ default: form = "(NEONScalarByIndexedElement)";
+ }
+ }
+
+ if (long_instr) {
+ nfd.SetFormatMap(0, nfd.LongScalarFormatMap());
+ }
+
+ Format(instr, mnemonic, nfd.Substitute(
+ form, nfd.kPlaceholder, nfd.kPlaceholder, nfd.kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarCopy(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(NEONScalarCopy)";
+
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::TriangularScalarFormatMap());
+
+ if (instr->Mask(NEONScalarCopyMask) == NEON_DUP_ELEMENT_scalar) {
+ mnemonic = "mov";
+ form = "%sd, 'Vn.%s['IVInsIndex1]";
+ }
+
+ Format(instr, mnemonic, nfd.Substitute(form, nfd.kPlaceholder, nfd.kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarPairwise(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "%sd, 'Vn.%s";
+ NEONFormatMap map = { {22}, {NF_2S, NF_2D} };
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::FPScalarFormatMap(), &map);
+
+ switch (instr->Mask(NEONScalarPairwiseMask)) {
+ case NEON_ADDP_scalar: mnemonic = "addp"; break;
+ case NEON_FADDP_scalar: mnemonic = "faddp"; break;
+ case NEON_FMAXP_scalar: mnemonic = "fmaxp"; break;
+ case NEON_FMAXNMP_scalar: mnemonic = "fmaxnmp"; break;
+ case NEON_FMINP_scalar: mnemonic = "fminp"; break;
+ case NEON_FMINNMP_scalar: mnemonic = "fminnmp"; break;
+ default: form = "(NEONScalarPairwise)";
+ }
+ Format(instr, mnemonic, nfd.Substitute(form,
+ NEONFormatDecoder::kPlaceholder, NEONFormatDecoder::kFormat));
+}
+
+
+void Disassembler::VisitNEONScalarShiftImmediate(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "%sd, %sn, 'Is1";
+ const char *form_2 = "%sd, %sn, 'Is2";
+
+ static const NEONFormatMap map_shift = {
+ {22, 21, 20, 19},
+ {NF_UNDEF, NF_B, NF_H, NF_H, NF_S, NF_S, NF_S, NF_S,
+ NF_D, NF_D, NF_D, NF_D, NF_D, NF_D, NF_D, NF_D}
+ };
+ static const NEONFormatMap map_shift_narrow = {
+ {21, 20, 19},
+ {NF_UNDEF, NF_H, NF_S, NF_S, NF_D, NF_D, NF_D, NF_D}
+ };
+ NEONFormatDecoder nfd(instr, &map_shift);
+
+ if (instr->ImmNEONImmh()) { // immh has to be non-zero.
+ switch (instr->Mask(NEONScalarShiftImmediateMask)) {
+ case NEON_FCVTZU_imm_scalar: mnemonic = "fcvtzu"; break;
+ case NEON_FCVTZS_imm_scalar: mnemonic = "fcvtzs"; break;
+ case NEON_SCVTF_imm_scalar: mnemonic = "scvtf"; break;
+ case NEON_UCVTF_imm_scalar: mnemonic = "ucvtf"; break;
+ case NEON_SRI_scalar: mnemonic = "sri"; break;
+ case NEON_SSHR_scalar: mnemonic = "sshr"; break;
+ case NEON_USHR_scalar: mnemonic = "ushr"; break;
+ case NEON_SRSHR_scalar: mnemonic = "srshr"; break;
+ case NEON_URSHR_scalar: mnemonic = "urshr"; break;
+ case NEON_SSRA_scalar: mnemonic = "ssra"; break;
+ case NEON_USRA_scalar: mnemonic = "usra"; break;
+ case NEON_SRSRA_scalar: mnemonic = "srsra"; break;
+ case NEON_URSRA_scalar: mnemonic = "ursra"; break;
+ case NEON_SHL_scalar: mnemonic = "shl"; form = form_2; break;
+ case NEON_SLI_scalar: mnemonic = "sli"; form = form_2; break;
+ case NEON_SQSHLU_scalar: mnemonic = "sqshlu"; form = form_2; break;
+ case NEON_SQSHL_imm_scalar: mnemonic = "sqshl"; form = form_2; break;
+ case NEON_UQSHL_imm_scalar: mnemonic = "uqshl"; form = form_2; break;
+ case NEON_UQSHRN_scalar:
+ mnemonic = "uqshrn";
+ nfd.SetFormatMap(1, &map_shift_narrow);
+ break;
+ case NEON_UQRSHRN_scalar:
+ mnemonic = "uqrshrn";
+ nfd.SetFormatMap(1, &map_shift_narrow);
+ break;
+ case NEON_SQSHRN_scalar:
+ mnemonic = "sqshrn";
+ nfd.SetFormatMap(1, &map_shift_narrow);
+ break;
+ case NEON_SQRSHRN_scalar:
+ mnemonic = "sqrshrn";
+ nfd.SetFormatMap(1, &map_shift_narrow);
+ break;
+ case NEON_SQSHRUN_scalar:
+ mnemonic = "sqshrun";
+ nfd.SetFormatMap(1, &map_shift_narrow);
+ break;
+ case NEON_SQRSHRUN_scalar:
+ mnemonic = "sqrshrun";
+ nfd.SetFormatMap(1, &map_shift_narrow);
+ break;
+ default:
+ form = "(NEONScalarShiftImmediate)";
+ }
+ } else {
+ form = "(NEONScalarShiftImmediate)";
+ }
+ Format(instr, mnemonic, nfd.SubstitutePlaceholders(form));
+}
+
+
+void Disassembler::VisitNEONShiftImmediate(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Vd.%s, 'Vn.%s, 'Is1";
+ const char *form_shift_2 = "'Vd.%s, 'Vn.%s, 'Is2";
+ const char *form_xtl = "'Vd.%s, 'Vn.%s";
+
+ // 0001->8H, 001x->4S, 01xx->2D, all others undefined.
+ static const NEONFormatMap map_shift_ta = {
+ {22, 21, 20, 19},
+ {NF_UNDEF, NF_8H, NF_4S, NF_4S, NF_2D, NF_2D, NF_2D, NF_2D}
+ };
+
+ // 00010->8B, 00011->16B, 001x0->4H, 001x1->8H,
+ // 01xx0->2S, 01xx1->4S, 1xxx1->2D, all others undefined.
+ static const NEONFormatMap map_shift_tb = {
+ {22, 21, 20, 19, 30},
+ {NF_UNDEF, NF_UNDEF, NF_8B, NF_16B, NF_4H, NF_8H, NF_4H, NF_8H,
+ NF_2S, NF_4S, NF_2S, NF_4S, NF_2S, NF_4S, NF_2S, NF_4S,
+ NF_UNDEF, NF_2D, NF_UNDEF, NF_2D, NF_UNDEF, NF_2D, NF_UNDEF, NF_2D,
+ NF_UNDEF, NF_2D, NF_UNDEF, NF_2D, NF_UNDEF, NF_2D, NF_UNDEF, NF_2D}
+ };
+
+ NEONFormatDecoder nfd(instr, &map_shift_tb);
+
+ if (instr->ImmNEONImmh()) { // immh has to be non-zero.
+ switch (instr->Mask(NEONShiftImmediateMask)) {
+ case NEON_SQSHLU: mnemonic = "sqshlu"; form = form_shift_2; break;
+ case NEON_SQSHL_imm: mnemonic = "sqshl"; form = form_shift_2; break;
+ case NEON_UQSHL_imm: mnemonic = "uqshl"; form = form_shift_2; break;
+ case NEON_SHL: mnemonic = "shl"; form = form_shift_2; break;
+ case NEON_SLI: mnemonic = "sli"; form = form_shift_2; break;
+ case NEON_SCVTF_imm: mnemonic = "scvtf"; break;
+ case NEON_UCVTF_imm: mnemonic = "ucvtf"; break;
+ case NEON_FCVTZU_imm: mnemonic = "fcvtzu"; break;
+ case NEON_FCVTZS_imm: mnemonic = "fcvtzs"; break;
+ case NEON_SRI: mnemonic = "sri"; break;
+ case NEON_SSHR: mnemonic = "sshr"; break;
+ case NEON_USHR: mnemonic = "ushr"; break;
+ case NEON_SRSHR: mnemonic = "srshr"; break;
+ case NEON_URSHR: mnemonic = "urshr"; break;
+ case NEON_SSRA: mnemonic = "ssra"; break;
+ case NEON_USRA: mnemonic = "usra"; break;
+ case NEON_SRSRA: mnemonic = "srsra"; break;
+ case NEON_URSRA: mnemonic = "ursra"; break;
+ case NEON_SHRN:
+ mnemonic = instr->Mask(NEON_Q) ? "shrn2" : "shrn";
+ nfd.SetFormatMap(1, &map_shift_ta);
+ break;
+ case NEON_RSHRN:
+ mnemonic = instr->Mask(NEON_Q) ? "rshrn2" : "rshrn";
+ nfd.SetFormatMap(1, &map_shift_ta);
+ break;
+ case NEON_UQSHRN:
+ mnemonic = instr->Mask(NEON_Q) ? "uqshrn2" : "uqshrn";
+ nfd.SetFormatMap(1, &map_shift_ta);
+ break;
+ case NEON_UQRSHRN:
+ mnemonic = instr->Mask(NEON_Q) ? "uqrshrn2" : "uqrshrn";
+ nfd.SetFormatMap(1, &map_shift_ta);
+ break;
+ case NEON_SQSHRN:
+ mnemonic = instr->Mask(NEON_Q) ? "sqshrn2" : "sqshrn";
+ nfd.SetFormatMap(1, &map_shift_ta);
+ break;
+ case NEON_SQRSHRN:
+ mnemonic = instr->Mask(NEON_Q) ? "sqrshrn2" : "sqrshrn";
+ nfd.SetFormatMap(1, &map_shift_ta);
+ break;
+ case NEON_SQSHRUN:
+ mnemonic = instr->Mask(NEON_Q) ? "sqshrun2" : "sqshrun";
+ nfd.SetFormatMap(1, &map_shift_ta);
+ break;
+ case NEON_SQRSHRUN:
+ mnemonic = instr->Mask(NEON_Q) ? "sqrshrun2" : "sqrshrun";
+ nfd.SetFormatMap(1, &map_shift_ta);
+ break;
+ case NEON_SSHLL:
+ nfd.SetFormatMap(0, &map_shift_ta);
+ if (instr->ImmNEONImmb() == 0 &&
+ CountSetBits(instr->ImmNEONImmh(), 32) == 1) { // sxtl variant.
+ form = form_xtl;
+ mnemonic = instr->Mask(NEON_Q) ? "sxtl2" : "sxtl";
+ } else { // sshll variant.
+ form = form_shift_2;
+ mnemonic = instr->Mask(NEON_Q) ? "sshll2" : "sshll";
+ }
+ break;
+ case NEON_USHLL:
+ nfd.SetFormatMap(0, &map_shift_ta);
+ if (instr->ImmNEONImmb() == 0 &&
+ CountSetBits(instr->ImmNEONImmh(), 32) == 1) { // uxtl variant.
+ form = form_xtl;
+ mnemonic = instr->Mask(NEON_Q) ? "uxtl2" : "uxtl";
+ } else { // ushll variant.
+ form = form_shift_2;
+ mnemonic = instr->Mask(NEON_Q) ? "ushll2" : "ushll";
+ }
+ break;
+ default: form = "(NEONShiftImmediate)";
+ }
+ } else {
+ form = "(NEONShiftImmediate)";
+ }
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitNEONTable(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "(NEONTable)";
+ const char form_1v[] = "'Vd.%%s, {'Vn.16b}, 'Vm.%%s";
+ const char form_2v[] = "'Vd.%%s, {'Vn.16b, v%d.16b}, 'Vm.%%s";
+ const char form_3v[] = "'Vd.%%s, {'Vn.16b, v%d.16b, v%d.16b}, 'Vm.%%s";
+ const char form_4v[] =
+ "'Vd.%%s, {'Vn.16b, v%d.16b, v%d.16b, v%d.16b}, 'Vm.%%s";
+ static const NEONFormatMap map_b = { {30}, {NF_8B, NF_16B} };
+ NEONFormatDecoder nfd(instr, &map_b);
+
+ switch (instr->Mask(NEONTableMask)) {
+ case NEON_TBL_1v: mnemonic = "tbl"; form = form_1v; break;
+ case NEON_TBL_2v: mnemonic = "tbl"; form = form_2v; break;
+ case NEON_TBL_3v: mnemonic = "tbl"; form = form_3v; break;
+ case NEON_TBL_4v: mnemonic = "tbl"; form = form_4v; break;
+ case NEON_TBX_1v: mnemonic = "tbx"; form = form_1v; break;
+ case NEON_TBX_2v: mnemonic = "tbx"; form = form_2v; break;
+ case NEON_TBX_3v: mnemonic = "tbx"; form = form_3v; break;
+ case NEON_TBX_4v: mnemonic = "tbx"; form = form_4v; break;
+ default: break;
+ }
+
+ char re_form[sizeof(form_4v) + 6];
+ int reg_num = instr->Rn();
+ SprintfLiteral(re_form, form,
+ (reg_num + 1) % kNumberOfVRegisters,
+ (reg_num + 2) % kNumberOfVRegisters,
+ (reg_num + 3) % kNumberOfVRegisters);
+
+ Format(instr, mnemonic, nfd.Substitute(re_form));
+}
+
+
+void Disassembler::VisitNEONPerm(const Instruction* instr) {
+ const char *mnemonic = "unimplemented";
+ const char *form = "'Vd.%s, 'Vn.%s, 'Vm.%s";
+ NEONFormatDecoder nfd(instr);
+
+ switch (instr->Mask(NEONPermMask)) {
+ case NEON_TRN1: mnemonic = "trn1"; break;
+ case NEON_TRN2: mnemonic = "trn2"; break;
+ case NEON_UZP1: mnemonic = "uzp1"; break;
+ case NEON_UZP2: mnemonic = "uzp2"; break;
+ case NEON_ZIP1: mnemonic = "zip1"; break;
+ case NEON_ZIP2: mnemonic = "zip2"; break;
+ default: form = "(NEONPerm)";
+ }
+ Format(instr, mnemonic, nfd.Substitute(form));
+}
+
+
+void Disassembler::VisitUnimplemented(const Instruction* instr) {
+ Format(instr, "unimplemented", "(Unimplemented)");
+}
+
+
+void Disassembler::VisitUnallocated(const Instruction* instr) {
+ Format(instr, "unallocated", "(Unallocated)");
+}
+
+
+void Disassembler::ProcessOutput(const Instruction* /*instr*/) {
+ // The base disasm does nothing more than disassembling into a buffer.
+}
+
+
+void Disassembler::AppendRegisterNameToOutput(const Instruction* instr,
+ const CPURegister& reg) {
+ USE(instr);
+ VIXL_ASSERT(reg.IsValid());
+ char reg_char;
+
+ if (reg.IsRegister()) {
+ reg_char = reg.Is64Bits() ? 'x' : 'w';
+ } else {
+ VIXL_ASSERT(reg.IsVRegister());
+ switch (reg.SizeInBits()) {
+ case kBRegSize: reg_char = 'b'; break;
+ case kHRegSize: reg_char = 'h'; break;
+ case kSRegSize: reg_char = 's'; break;
+ case kDRegSize: reg_char = 'd'; break;
+ default:
+ VIXL_ASSERT(reg.Is128Bits());
+ reg_char = 'q';
+ }
+ }
+
+ if (reg.IsVRegister() || !(reg.Aliases(sp) || reg.Aliases(xzr))) {
+ // A core or scalar/vector register: [wx]0 - 30, [bhsdq]0 - 31.
+ AppendToOutput("%c%d", reg_char, reg.code());
+ } else if (reg.Aliases(sp)) {
+ // Disassemble w31/x31 as stack pointer wsp/sp.
+ AppendToOutput("%s", reg.Is64Bits() ? "sp" : "wsp");
+ } else {
+ // Disassemble w31/x31 as zero register wzr/xzr.
+ AppendToOutput("%czr", reg_char);
+ }
+}
+
+
+void Disassembler::AppendPCRelativeOffsetToOutput(const Instruction* instr,
+ int64_t offset) {
+ USE(instr);
+ char sign = (offset < 0) ? '-' : '+';
+ AppendToOutput("#%c0x%" PRIx64, sign, std::abs(offset));
+}
+
+
+void Disassembler::AppendAddressToOutput(const Instruction* instr,
+ const void* addr) {
+ USE(instr);
+ AppendToOutput("(addr 0x%" PRIxPTR ")", reinterpret_cast<uintptr_t>(addr));
+}
+
+
+void Disassembler::AppendCodeAddressToOutput(const Instruction* instr,
+ const void* addr) {
+ AppendAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendDataAddressToOutput(const Instruction* instr,
+ const void* addr) {
+ AppendAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendCodeRelativeAddressToOutput(const Instruction* instr,
+ const void* addr) {
+ USE(instr);
+ int64_t rel_addr = CodeRelativeAddress(addr);
+ if (rel_addr >= 0) {
+ AppendToOutput("(addr 0x%" PRIx64 ")", rel_addr);
+ } else {
+ AppendToOutput("(addr -0x%" PRIx64 ")", -rel_addr);
+ }
+}
+
+
+void Disassembler::AppendCodeRelativeCodeAddressToOutput(
+ const Instruction* instr, const void* addr) {
+ AppendCodeRelativeAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::AppendCodeRelativeDataAddressToOutput(
+ const Instruction* instr, const void* addr) {
+ AppendCodeRelativeAddressToOutput(instr, addr);
+}
+
+
+void Disassembler::MapCodeAddress(int64_t base_address,
+ const Instruction* instr_address) {
+ set_code_address_offset(
+ base_address - reinterpret_cast<intptr_t>(instr_address));
+}
+int64_t Disassembler::CodeRelativeAddress(const void* addr) {
+ return reinterpret_cast<intptr_t>(addr) + code_address_offset();
+}
+
+
+void Disassembler::Format(const Instruction* instr, const char* mnemonic,
+ const char* format) {
+ VIXL_ASSERT(mnemonic != NULL);
+ ResetOutput();
+ uint32_t pos = buffer_pos_;
+ Substitute(instr, mnemonic);
+ if (format != NULL) {
+ uint32_t spaces = buffer_pos_ - pos < 8 ? 8 - (buffer_pos_ - pos) : 1;
+ while (spaces--) {
+ VIXL_ASSERT(buffer_pos_ < buffer_size_);
+ buffer_[buffer_pos_++] = ' ';
+ }
+ Substitute(instr, format);
+ }
+ VIXL_ASSERT(buffer_pos_ < buffer_size_);
+ buffer_[buffer_pos_] = 0;
+ ProcessOutput(instr);
+}
+
+
+void Disassembler::Substitute(const Instruction* instr, const char* string) {
+ char chr = *string++;
+ while (chr != '\0') {
+ if (chr == '\'') {
+ string += SubstituteField(instr, string);
+ } else {
+ VIXL_ASSERT(buffer_pos_ < buffer_size_);
+ buffer_[buffer_pos_++] = chr;
+ }
+ chr = *string++;
+ }
+}
+
+
+int Disassembler::SubstituteField(const Instruction* instr,
+ const char* format) {
+ switch (format[0]) {
+ // NB. The remaining substitution prefix characters are: GJKUZ.
+ case 'R': // Register. X or W, selected by sf bit.
+ case 'F': // FP register. S or D, selected by type field.
+ case 'V': // Vector register, V, vector format.
+ case 'W':
+ case 'X':
+ case 'B':
+ case 'H':
+ case 'S':
+ case 'D':
+ case 'Q': return SubstituteRegisterField(instr, format);
+ case 'I': return SubstituteImmediateField(instr, format);
+ case 'L': return SubstituteLiteralField(instr, format);
+ case 'N': return SubstituteShiftField(instr, format);
+ case 'P': return SubstitutePrefetchField(instr, format);
+ case 'C': return SubstituteConditionField(instr, format);
+ case 'E': return SubstituteExtendField(instr, format);
+ case 'A': return SubstitutePCRelAddressField(instr, format);
+ case 'T': return SubstituteBranchTargetField(instr, format);
+ case 'O': return SubstituteLSRegOffsetField(instr, format);
+ case 'M': return SubstituteBarrierField(instr, format);
+ case 'K': return SubstituteCrField(instr, format);
+ case 'G': return SubstituteSysOpField(instr, format);
+ default: {
+ VIXL_UNREACHABLE();
+ return 1;
+ }
+ }
+}
+
+
+int Disassembler::SubstituteRegisterField(const Instruction* instr,
+ const char* format) {
+ char reg_prefix = format[0];
+ unsigned reg_num = 0;
+ unsigned field_len = 2;
+
+ switch (format[1]) {
+ case 'd':
+ reg_num = instr->Rd();
+ if (format[2] == 'q') {
+ reg_prefix = instr->NEONQ() ? 'X' : 'W';
+ field_len = 3;
+ }
+ break;
+ case 'n': reg_num = instr->Rn(); break;
+ case 'm':
+ reg_num = instr->Rm();
+ switch (format[2]) {
+ // Handle registers tagged with b (bytes), z (instruction), or
+ // r (registers), used for address updates in
+ // NEON load/store instructions.
+ case 'r':
+ case 'b':
+ case 'z': {
+ field_len = 3;
+ char* eimm;
+ int imm = static_cast<int>(strtol(&format[3], &eimm, 10));
+ field_len += eimm - &format[3];
+ if (reg_num == 31) {
+ switch (format[2]) {
+ case 'z':
+ imm *= (1 << instr->NEONLSSize());
+ break;
+ case 'r':
+ imm *= (instr->NEONQ() == 0) ? kDRegSizeInBytes
+ : kQRegSizeInBytes;
+ break;
+ case 'b':
+ break;
+ }
+ AppendToOutput("#%d", imm);
+ return field_len;
+ }
+ break;
+ }
+ }
+ break;
+ case 'e':
+ // This is register Rm, but using a 4-bit specifier. Used in NEON
+ // by-element instructions.
+ reg_num = (instr->Rm() & 0xf);
+ break;
+ case 'a': reg_num = instr->Ra(); break;
+ case 's': reg_num = instr->Rs(); break;
+ case 't':
+ reg_num = instr->Rt();
+ if (format[0] == 'V') {
+ if ((format[2] >= '2') && (format[2] <= '4')) {
+ // Handle consecutive vector register specifiers Vt2, Vt3 and Vt4.
+ reg_num = (reg_num + format[2] - '1') % 32;
+ field_len = 3;
+ }
+ } else {
+ if (format[2] == '2') {
+ // Handle register specifier Rt2.
+ reg_num = instr->Rt2();
+ field_len = 3;
+ }
+ }
+ break;
+ default: VIXL_UNREACHABLE();
+ }
+
+ // Increase field length for registers tagged as stack.
+ if (format[2] == 's') {
+ field_len = 3;
+ }
+
+ CPURegister::RegisterType reg_type = CPURegister::kRegister;
+ unsigned reg_size = kXRegSize;
+
+ if (reg_prefix == 'R') {
+ reg_prefix = instr->SixtyFourBits() ? 'X' : 'W';
+ } else if (reg_prefix == 'F') {
+ reg_prefix = ((instr->FPType() & 1) == 0) ? 'S' : 'D';
+ }
+
+ switch (reg_prefix) {
+ case 'W':
+ reg_type = CPURegister::kRegister; reg_size = kWRegSize; break;
+ case 'X':
+ reg_type = CPURegister::kRegister; reg_size = kXRegSize; break;
+ case 'B':
+ reg_type = CPURegister::kVRegister; reg_size = kBRegSize; break;
+ case 'H':
+ reg_type = CPURegister::kVRegister; reg_size = kHRegSize; break;
+ case 'S':
+ reg_type = CPURegister::kVRegister; reg_size = kSRegSize; break;
+ case 'D':
+ reg_type = CPURegister::kVRegister; reg_size = kDRegSize; break;
+ case 'Q':
+ reg_type = CPURegister::kVRegister; reg_size = kQRegSize; break;
+ case 'V':
+ AppendToOutput("v%d", reg_num);
+ return field_len;
+ default:
+ VIXL_UNREACHABLE();
+ }
+
+ if ((reg_type == CPURegister::kRegister) &&
+ (reg_num == kZeroRegCode) && (format[2] == 's')) {
+ reg_num = kSPRegInternalCode;
+ }
+
+ AppendRegisterNameToOutput(instr, CPURegister(reg_num, reg_size, reg_type));
+
+ return field_len;
+}
+
+
+int Disassembler::SubstituteImmediateField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(format[0] == 'I');
+
+ switch (format[1]) {
+ case 'M': { // IMoveImm, IMoveNeg or IMoveLSL.
+ if (format[5] == 'L') {
+ AppendToOutput("#0x%" PRIx32, instr->ImmMoveWide());
+ if (instr->ShiftMoveWide() > 0) {
+ AppendToOutput(", lsl #%" PRId32, 16 * instr->ShiftMoveWide());
+ }
+ } else {
+ VIXL_ASSERT((format[5] == 'I') || (format[5] == 'N'));
+ uint64_t imm = static_cast<uint64_t>(instr->ImmMoveWide()) <<
+ (16 * instr->ShiftMoveWide());
+ if (format[5] == 'N')
+ imm = ~imm;
+ if (!instr->SixtyFourBits())
+ imm &= UINT64_C(0xffffffff);
+ AppendToOutput("#0x%" PRIx64, imm);
+ }
+ return 8;
+ }
+ case 'L': {
+ switch (format[2]) {
+ case 'L': { // ILLiteral - Immediate Load Literal.
+ AppendToOutput("pc%+" PRId32,
+ instr->ImmLLiteral() << kLiteralEntrySizeLog2);
+ return 9;
+ }
+ case 'S': { // ILS - Immediate Load/Store.
+ if (instr->ImmLS() != 0) {
+ AppendToOutput(", #%" PRId32, instr->ImmLS());
+ }
+ return 3;
+ }
+ case 'P': { // ILPx - Immediate Load/Store Pair, x = access size.
+ if (instr->ImmLSPair() != 0) {
+ // format[3] is the scale value. Convert to a number.
+ int scale = 1 << (format[3] - '0');
+ AppendToOutput(", #%" PRId32, instr->ImmLSPair() * scale);
+ }
+ return 4;
+ }
+ case 'U': { // ILU - Immediate Load/Store Unsigned.
+ if (instr->ImmLSUnsigned() != 0) {
+ int shift = instr->SizeLS();
+ AppendToOutput(", #%" PRId32, instr->ImmLSUnsigned() << shift);
+ }
+ return 3;
+ }
+ default: {
+ VIXL_UNIMPLEMENTED();
+ return 0;
+ }
+ }
+ }
+ case 'C': { // ICondB - Immediate Conditional Branch.
+ int64_t offset = instr->ImmCondBranch() << 2;
+ AppendPCRelativeOffsetToOutput(instr, offset);
+ return 6;
+ }
+ case 'A': { // IAddSub.
+ VIXL_ASSERT(instr->ShiftAddSub() <= 1);
+ int64_t imm = instr->ImmAddSub() << (12 * instr->ShiftAddSub());
+ AppendToOutput("#0x%" PRIx64 " (%" PRId64 ")", imm, imm);
+ return 7;
+ }
+ case 'F': { // IFPSingle, IFPDouble or IFPFBits.
+ if (format[3] == 'F') { // IFPFbits.
+ AppendToOutput("#%" PRId32, 64 - instr->FPScale());
+ return 8;
+ } else {
+ AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmFP(),
+ format[3] == 'S' ? instr->ImmFP32() : instr->ImmFP64());
+ return 9;
+ }
+ }
+ case 'T': { // ITri - Immediate Triangular Encoded.
+ AppendToOutput("#0x%" PRIx64, instr->ImmLogical());
+ return 4;
+ }
+ case 'N': { // INzcv.
+ int nzcv = (instr->Nzcv() << Flags_offset);
+ AppendToOutput("#%c%c%c%c", ((nzcv & NFlag) == 0) ? 'n' : 'N',
+ ((nzcv & ZFlag) == 0) ? 'z' : 'Z',
+ ((nzcv & CFlag) == 0) ? 'c' : 'C',
+ ((nzcv & VFlag) == 0) ? 'v' : 'V');
+ return 5;
+ }
+ case 'P': { // IP - Conditional compare.
+ AppendToOutput("#%" PRId32, instr->ImmCondCmp());
+ return 2;
+ }
+ case 'B': { // Bitfields.
+ return SubstituteBitfieldImmediateField(instr, format);
+ }
+ case 'E': { // IExtract.
+ AppendToOutput("#%" PRId32, instr->ImmS());
+ return 8;
+ }
+ case 'S': { // IS - Test and branch bit.
+ AppendToOutput("#%" PRId32, (instr->ImmTestBranchBit5() << 5) |
+ instr->ImmTestBranchBit40());
+ return 2;
+ }
+ case 's': { // Is - Shift (immediate).
+ switch (format[2]) {
+ case '1': { // Is1 - SSHR.
+ int shift = 16 << HighestSetBitPosition(instr->ImmNEONImmh());
+ shift -= instr->ImmNEONImmhImmb();
+ AppendToOutput("#%d", shift);
+ return 3;
+ }
+ case '2': { // Is2 - SLI.
+ int shift = instr->ImmNEONImmhImmb();
+ shift -= 8 << HighestSetBitPosition(instr->ImmNEONImmh());
+ AppendToOutput("#%d", shift);
+ return 3;
+ }
+ default: {
+ VIXL_UNIMPLEMENTED();
+ return 0;
+ }
+ }
+ }
+ case 'D': { // IDebug - HLT and BRK instructions.
+ AppendToOutput("#0x%" PRIx32, instr->ImmException());
+ return 6;
+ }
+ case 'V': { // Immediate Vector.
+ switch (format[2]) {
+ case 'E': { // IVExtract.
+ AppendToOutput("#%" PRId32, instr->ImmNEONExt());
+ return 9;
+ }
+ case 'B': { // IVByElemIndex.
+ int vm_index = (instr->NEONH() << 1) | instr->NEONL();
+ if (instr->NEONSize() == 1) {
+ vm_index = (vm_index << 1) | instr->NEONM();
+ }
+ AppendToOutput("%d", vm_index);
+ return strlen("IVByElemIndex");
+ }
+ case 'I': { // INS element.
+ if (strncmp(format, "IVInsIndex", strlen("IVInsIndex")) == 0) {
+ int rd_index, rn_index;
+ int imm5 = instr->ImmNEON5();
+ int imm4 = instr->ImmNEON4();
+ int tz = CountTrailingZeros(imm5, 32);
+ rd_index = imm5 >> (tz + 1);
+ rn_index = imm4 >> tz;
+ if (strncmp(format, "IVInsIndex1", strlen("IVInsIndex1")) == 0) {
+ AppendToOutput("%d", rd_index);
+ return strlen("IVInsIndex1");
+ } else if (strncmp(format, "IVInsIndex2",
+ strlen("IVInsIndex2")) == 0) {
+ AppendToOutput("%d", rn_index);
+ return strlen("IVInsIndex2");
+ } else {
+ VIXL_UNIMPLEMENTED();
+ return 0;
+ }
+ }
+ VIXL_FALLTHROUGH();
+ }
+ case 'L': { // IVLSLane[0123] - suffix indicates access size shift.
+ AppendToOutput("%d", instr->NEONLSIndex(format[8] - '0'));
+ return 9;
+ }
+ case 'M': { // Modified Immediate cases.
+ if (strncmp(format,
+ "IVMIImmFPSingle",
+ strlen("IVMIImmFPSingle")) == 0) {
+ AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmNEONabcdefgh(),
+ instr->ImmNEONFP32());
+ return strlen("IVMIImmFPSingle");
+ } else if (strncmp(format,
+ "IVMIImmFPDouble",
+ strlen("IVMIImmFPDouble")) == 0) {
+ AppendToOutput("#0x%" PRIx32 " (%.4f)", instr->ImmNEONabcdefgh(),
+ instr->ImmNEONFP64());
+ return strlen("IVMIImmFPDouble");
+ } else if (strncmp(format, "IVMIImm8", strlen("IVMIImm8")) == 0) {
+ uint64_t imm8 = instr->ImmNEONabcdefgh();
+ AppendToOutput("#0x%" PRIx64, imm8);
+ return strlen("IVMIImm8");
+ } else if (strncmp(format, "IVMIImm", strlen("IVMIImm")) == 0) {
+ uint64_t imm8 = instr->ImmNEONabcdefgh();
+ uint64_t imm = 0;
+ for (int i = 0; i < 8; ++i) {
+ if (imm8 & (1ULL << i)) {
+ imm |= (UINT64_C(0xff) << (8 * i));
+ }
+ }
+ AppendToOutput("#0x%" PRIx64, imm);
+ return strlen("IVMIImm");
+ } else if (strncmp(format, "IVMIShiftAmt1",
+ strlen("IVMIShiftAmt1")) == 0) {
+ int cmode = instr->NEONCmode();
+ int shift_amount = 8 * ((cmode >> 1) & 3);
+ AppendToOutput("#%d", shift_amount);
+ return strlen("IVMIShiftAmt1");
+ } else if (strncmp(format, "IVMIShiftAmt2",
+ strlen("IVMIShiftAmt2")) == 0) {
+ int cmode = instr->NEONCmode();
+ int shift_amount = 8 << (cmode & 1);
+ AppendToOutput("#%d", shift_amount);
+ return strlen("IVMIShiftAmt2");
+ } else {
+ VIXL_UNIMPLEMENTED();
+ return 0;
+ }
+ }
+ default: {
+ VIXL_UNIMPLEMENTED();
+ return 0;
+ }
+ }
+ }
+ case 'X': { // IX - CLREX instruction.
+ AppendToOutput("#0x%" PRIx32, instr->CRm());
+ return 2;
+ }
+ default: {
+ VIXL_UNIMPLEMENTED();
+ return 0;
+ }
+ }
+}
+
+
+int Disassembler::SubstituteBitfieldImmediateField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT((format[0] == 'I') && (format[1] == 'B'));
+ unsigned r = instr->ImmR();
+ unsigned s = instr->ImmS();
+
+ switch (format[2]) {
+ case 'r': { // IBr.
+ AppendToOutput("#%d", r);
+ return 3;
+ }
+ case 's': { // IBs+1 or IBs-r+1.
+ if (format[3] == '+') {
+ AppendToOutput("#%d", s + 1);
+ return 5;
+ } else {
+ VIXL_ASSERT(format[3] == '-');
+ AppendToOutput("#%d", s - r + 1);
+ return 7;
+ }
+ }
+ case 'Z': { // IBZ-r.
+ VIXL_ASSERT((format[3] == '-') && (format[4] == 'r'));
+ unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize : kWRegSize;
+ AppendToOutput("#%d", reg_size - r);
+ return 5;
+ }
+ default: {
+ VIXL_UNREACHABLE();
+ return 0;
+ }
+ }
+}
+
+
+int Disassembler::SubstituteLiteralField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(strncmp(format, "LValue", 6) == 0);
+ USE(format);
+
+ const void * address = instr->LiteralAddress<const void *>();
+ switch (instr->Mask(LoadLiteralMask)) {
+ case LDR_w_lit:
+ case LDR_x_lit:
+ case LDRSW_x_lit:
+ case LDR_s_lit:
+ case LDR_d_lit:
+ case LDR_q_lit:
+ AppendCodeRelativeDataAddressToOutput(instr, address);
+ break;
+ case PRFM_lit: {
+ // Use the prefetch hint to decide how to print the address.
+ switch (instr->PrefetchHint()) {
+ case 0x0: // PLD: prefetch for load.
+ case 0x2: // PST: prepare for store.
+ AppendCodeRelativeDataAddressToOutput(instr, address);
+ break;
+ case 0x1: // PLI: preload instructions.
+ AppendCodeRelativeCodeAddressToOutput(instr, address);
+ break;
+ case 0x3: // Unallocated hint.
+ AppendCodeRelativeAddressToOutput(instr, address);
+ break;
+ }
+ break;
+ }
+ default:
+ VIXL_UNREACHABLE();
+ }
+
+ return 6;
+}
+
+
+int Disassembler::SubstituteShiftField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(format[0] == 'N');
+ VIXL_ASSERT(instr->ShiftDP() <= 0x3);
+
+ switch (format[1]) {
+ case 'D': { // HDP.
+ VIXL_ASSERT(instr->ShiftDP() != ROR);
+ VIXL_FALLTHROUGH();
+ }
+ case 'L': { // HLo.
+ if (instr->ImmDPShift() != 0) {
+ const char* shift_type[] = {"lsl", "lsr", "asr", "ror"};
+ AppendToOutput(", %s #%" PRId32, shift_type[instr->ShiftDP()],
+ instr->ImmDPShift());
+ }
+ return 3;
+ }
+ default:
+ VIXL_UNIMPLEMENTED();
+ return 0;
+ }
+}
+
+
+int Disassembler::SubstituteConditionField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(format[0] == 'C');
+ const char* condition_code[] = { "eq", "ne", "hs", "lo",
+ "mi", "pl", "vs", "vc",
+ "hi", "ls", "ge", "lt",
+ "gt", "le", "al", "nv" };
+ int cond;
+ switch (format[1]) {
+ case 'B': cond = instr->ConditionBranch(); break;
+ case 'I': {
+ cond = InvertCondition(static_cast<Condition>(instr->Condition()));
+ break;
+ }
+ default: cond = instr->Condition();
+ }
+ AppendToOutput("%s", condition_code[cond]);
+ return 4;
+}
+
+
+int Disassembler::SubstitutePCRelAddressField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT((strcmp(format, "AddrPCRelByte") == 0) || // Used by `adr`.
+ (strcmp(format, "AddrPCRelPage") == 0)); // Used by `adrp`.
+
+ int64_t offset = instr->ImmPCRel();
+
+ // Compute the target address based on the effective address (after applying
+ // code_address_offset). This is required for correct behaviour of adrp.
+ const Instruction* base = instr + code_address_offset();
+ if (format[9] == 'P') {
+ offset *= kPageSize;
+ base = AlignDown(base, kPageSize);
+ }
+ // Strip code_address_offset before printing, so we can use the
+ // semantically-correct AppendCodeRelativeAddressToOutput.
+ const void* target =
+ reinterpret_cast<const void*>(base + offset - code_address_offset());
+
+ AppendPCRelativeOffsetToOutput(instr, offset);
+ AppendToOutput(" ");
+ AppendCodeRelativeAddressToOutput(instr, target);
+ return 13;
+}
+
+
+int Disassembler::SubstituteBranchTargetField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(strncmp(format, "TImm", 4) == 0);
+
+ int64_t offset = 0;
+ switch (format[5]) {
+ // BImmUncn - unconditional branch immediate.
+ case 'n': offset = instr->ImmUncondBranch(); break;
+ // BImmCond - conditional branch immediate.
+ case 'o': offset = instr->ImmCondBranch(); break;
+ // BImmCmpa - compare and branch immediate.
+ case 'm': offset = instr->ImmCmpBranch(); break;
+ // BImmTest - test and branch immediate.
+ case 'e': offset = instr->ImmTestBranch(); break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ offset <<= kInstructionSizeLog2;
+ const void* target_address = reinterpret_cast<const void*>(instr + offset);
+ VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
+
+ AppendPCRelativeOffsetToOutput(instr, offset);
+ AppendToOutput(" ");
+ AppendCodeRelativeCodeAddressToOutput(instr, target_address);
+
+ return 8;
+}
+
+
+int Disassembler::SubstituteExtendField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(strncmp(format, "Ext", 3) == 0);
+ VIXL_ASSERT(instr->ExtendMode() <= 7);
+ USE(format);
+
+ const char* extend_mode[] = { "uxtb", "uxth", "uxtw", "uxtx",
+ "sxtb", "sxth", "sxtw", "sxtx" };
+
+ // If rd or rn is SP, uxtw on 32-bit registers and uxtx on 64-bit
+ // registers becomes lsl.
+ if (((instr->Rd() == kZeroRegCode) || (instr->Rn() == kZeroRegCode)) &&
+ (((instr->ExtendMode() == UXTW) && (instr->SixtyFourBits() == 0)) ||
+ (instr->ExtendMode() == UXTX))) {
+ if (instr->ImmExtendShift() > 0) {
+ AppendToOutput(", lsl #%" PRId32, instr->ImmExtendShift());
+ }
+ } else {
+ AppendToOutput(", %s", extend_mode[instr->ExtendMode()]);
+ if (instr->ImmExtendShift() > 0) {
+ AppendToOutput(" #%" PRId32, instr->ImmExtendShift());
+ }
+ }
+ return 3;
+}
+
+
+int Disassembler::SubstituteLSRegOffsetField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(strncmp(format, "Offsetreg", 9) == 0);
+ const char* extend_mode[] = { "undefined", "undefined", "uxtw", "lsl",
+ "undefined", "undefined", "sxtw", "sxtx" };
+ USE(format);
+
+ unsigned shift = instr->ImmShiftLS();
+ Extend ext = static_cast<Extend>(instr->ExtendMode());
+ char reg_type = ((ext == UXTW) || (ext == SXTW)) ? 'w' : 'x';
+
+ unsigned rm = instr->Rm();
+ if (rm == kZeroRegCode) {
+ AppendToOutput("%czr", reg_type);
+ } else {
+ AppendToOutput("%c%d", reg_type, rm);
+ }
+
+ // Extend mode UXTX is an alias for shift mode LSL here.
+ if (!((ext == UXTX) && (shift == 0))) {
+ AppendToOutput(", %s", extend_mode[ext]);
+ if (shift != 0) {
+ AppendToOutput(" #%d", instr->SizeLS());
+ }
+ }
+ return 9;
+}
+
+
+int Disassembler::SubstitutePrefetchField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(format[0] == 'P');
+ USE(format);
+
+ static const char* hints[] = {"ld", "li", "st"};
+ static const char* stream_options[] = {"keep", "strm"};
+
+ unsigned hint = instr->PrefetchHint();
+ unsigned target = instr->PrefetchTarget() + 1;
+ unsigned stream = instr->PrefetchStream();
+
+ if ((hint >= (sizeof(hints) / sizeof(hints[0]))) || (target > 3)) {
+ // Unallocated prefetch operations.
+ int prefetch_mode = instr->ImmPrefetchOperation();
+ AppendToOutput("#0b%c%c%c%c%c",
+ (prefetch_mode & (1 << 4)) ? '1' : '0',
+ (prefetch_mode & (1 << 3)) ? '1' : '0',
+ (prefetch_mode & (1 << 2)) ? '1' : '0',
+ (prefetch_mode & (1 << 1)) ? '1' : '0',
+ (prefetch_mode & (1 << 0)) ? '1' : '0');
+ } else {
+ VIXL_ASSERT(stream < (sizeof(stream_options) / sizeof(stream_options[0])));
+ AppendToOutput("p%sl%d%s", hints[hint], target, stream_options[stream]);
+ }
+ return 6;
+}
+
+int Disassembler::SubstituteBarrierField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(format[0] == 'M');
+ USE(format);
+
+ static const char* options[4][4] = {
+ { "sy (0b0000)", "oshld", "oshst", "osh" },
+ { "sy (0b0100)", "nshld", "nshst", "nsh" },
+ { "sy (0b1000)", "ishld", "ishst", "ish" },
+ { "sy (0b1100)", "ld", "st", "sy" }
+ };
+ int domain = instr->ImmBarrierDomain();
+ int type = instr->ImmBarrierType();
+
+ AppendToOutput("%s", options[domain][type]);
+ return 1;
+}
+
+int Disassembler::SubstituteSysOpField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(format[0] == 'G');
+ int op = -1;
+ switch (format[1]) {
+ case '1': op = instr->SysOp1(); break;
+ case '2': op = instr->SysOp2(); break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+ AppendToOutput("#%d", op);
+ return 2;
+}
+
+int Disassembler::SubstituteCrField(const Instruction* instr,
+ const char* format) {
+ VIXL_ASSERT(format[0] == 'K');
+ int cr = -1;
+ switch (format[1]) {
+ case 'n': cr = instr->CRn(); break;
+ case 'm': cr = instr->CRm(); break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+ AppendToOutput("C%d", cr);
+ return 2;
+}
+
+void Disassembler::ResetOutput() {
+ buffer_pos_ = 0;
+ buffer_[buffer_pos_] = 0;
+}
+
+
+void Disassembler::AppendToOutput(const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ buffer_pos_ += vsnprintf(&buffer_[buffer_pos_], buffer_size_ - buffer_pos_,
+ format, args);
+ va_end(args);
+}
+
+
+void PrintDisassembler::ProcessOutput(const Instruction* instr) {
+ fprintf(stream_, "0x%016" PRIx64 " %08" PRIx32 "\t\t%s\n",
+ reinterpret_cast<uint64_t>(instr),
+ instr->InstructionBits(),
+ GetOutput());
+}
+
+void DisassembleInstruction(char* buffer, size_t bufsize, const Instruction* instr)
+{
+ vixl::Disassembler disasm(buffer, bufsize-1);
+ vixl::Decoder decoder;
+ decoder.AppendVisitor(&disasm);
+ decoder.Decode(instr);
+ buffer[bufsize-1] = 0; // Just to be safe
+}
+
+char* GdbDisassembleInstruction(const Instruction* instr)
+{
+ static char buffer[1024];
+ DisassembleInstruction(buffer, sizeof(buffer), instr);
+ return buffer;
+}
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/Disasm-vixl.h b/js/src/jit/arm64/vixl/Disasm-vixl.h
new file mode 100644
index 0000000000..e04730da83
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Disasm-vixl.h
@@ -0,0 +1,181 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_DISASM_A64_H
+#define VIXL_A64_DISASM_A64_H
+
+#include "jit/arm64/vixl/Assembler-vixl.h"
+#include "jit/arm64/vixl/Decoder-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Instructions-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+
+namespace vixl {
+
+class Disassembler: public DecoderVisitor {
+ public:
+ Disassembler();
+ Disassembler(char* text_buffer, int buffer_size);
+ virtual ~Disassembler();
+ char* GetOutput();
+
+ // Declare all Visitor functions.
+ #define DECLARE(A) virtual void Visit##A(const Instruction* instr) override;
+ VISITOR_LIST(DECLARE)
+ #undef DECLARE
+
+ protected:
+ virtual void ProcessOutput(const Instruction* instr);
+
+ // Default output functions. The functions below implement a default way of
+ // printing elements in the disassembly. A sub-class can override these to
+ // customize the disassembly output.
+
+ // Prints the name of a register.
+ // TODO: This currently doesn't allow renaming of V registers.
+ virtual void AppendRegisterNameToOutput(const Instruction* instr,
+ const CPURegister& reg);
+
+ // Prints a PC-relative offset. This is used for example when disassembling
+ // branches to immediate offsets.
+ virtual void AppendPCRelativeOffsetToOutput(const Instruction* instr,
+ int64_t offset);
+
+ // Prints an address, in the general case. It can be code or data. This is
+ // used for example to print the target address of an ADR instruction.
+ virtual void AppendCodeRelativeAddressToOutput(const Instruction* instr,
+ const void* addr);
+
+ // Prints the address of some code.
+ // This is used for example to print the target address of a branch to an
+ // immediate offset.
+ // A sub-class can for example override this method to lookup the address and
+ // print an appropriate name.
+ virtual void AppendCodeRelativeCodeAddressToOutput(const Instruction* instr,
+ const void* addr);
+
+ // Prints the address of some data.
+ // This is used for example to print the source address of a load literal
+ // instruction.
+ virtual void AppendCodeRelativeDataAddressToOutput(const Instruction* instr,
+ const void* addr);
+
+ // Same as the above, but for addresses that are not relative to the code
+ // buffer. They are currently not used by VIXL.
+ virtual void AppendAddressToOutput(const Instruction* instr,
+ const void* addr);
+ virtual void AppendCodeAddressToOutput(const Instruction* instr,
+ const void* addr);
+ virtual void AppendDataAddressToOutput(const Instruction* instr,
+ const void* addr);
+
+ public:
+ // Get/Set the offset that should be added to code addresses when printing
+ // code-relative addresses in the AppendCodeRelative<Type>AddressToOutput()
+ // helpers.
+ // Below is an example of how a branch immediate instruction in memory at
+ // address 0xb010200 would disassemble with different offsets.
+ // Base address | Disassembly
+ // 0x0 | 0xb010200: b #+0xcc (addr 0xb0102cc)
+ // 0x10000 | 0xb000200: b #+0xcc (addr 0xb0002cc)
+ // 0xb010200 | 0x0: b #+0xcc (addr 0xcc)
+ void MapCodeAddress(int64_t base_address, const Instruction* instr_address);
+ int64_t CodeRelativeAddress(const void* instr);
+
+ private:
+ void Format(
+ const Instruction* instr, const char* mnemonic, const char* format);
+ void Substitute(const Instruction* instr, const char* string);
+ int SubstituteField(const Instruction* instr, const char* format);
+ int SubstituteRegisterField(const Instruction* instr, const char* format);
+ int SubstituteImmediateField(const Instruction* instr, const char* format);
+ int SubstituteLiteralField(const Instruction* instr, const char* format);
+ int SubstituteBitfieldImmediateField(
+ const Instruction* instr, const char* format);
+ int SubstituteShiftField(const Instruction* instr, const char* format);
+ int SubstituteExtendField(const Instruction* instr, const char* format);
+ int SubstituteConditionField(const Instruction* instr, const char* format);
+ int SubstitutePCRelAddressField(const Instruction* instr, const char* format);
+ int SubstituteBranchTargetField(const Instruction* instr, const char* format);
+ int SubstituteLSRegOffsetField(const Instruction* instr, const char* format);
+ int SubstitutePrefetchField(const Instruction* instr, const char* format);
+ int SubstituteBarrierField(const Instruction* instr, const char* format);
+ int SubstituteSysOpField(const Instruction* instr, const char* format);
+ int SubstituteCrField(const Instruction* instr, const char* format);
+ bool RdIsZROrSP(const Instruction* instr) const {
+ return (instr->Rd() == kZeroRegCode);
+ }
+
+ bool RnIsZROrSP(const Instruction* instr) const {
+ return (instr->Rn() == kZeroRegCode);
+ }
+
+ bool RmIsZROrSP(const Instruction* instr) const {
+ return (instr->Rm() == kZeroRegCode);
+ }
+
+ bool RaIsZROrSP(const Instruction* instr) const {
+ return (instr->Ra() == kZeroRegCode);
+ }
+
+ bool IsMovzMovnImm(unsigned reg_size, uint64_t value);
+
+ int64_t code_address_offset() const { return code_address_offset_; }
+
+ protected:
+ void ResetOutput();
+ void AppendToOutput(const char* string, ...) PRINTF_CHECK(2, 3);
+
+ void set_code_address_offset(int64_t code_address_offset) {
+ code_address_offset_ = code_address_offset;
+ }
+
+ char* buffer_;
+ uint32_t buffer_pos_;
+ uint32_t buffer_size_;
+ bool own_buffer_;
+
+ int64_t code_address_offset_;
+};
+
+
+class PrintDisassembler: public Disassembler {
+ public:
+ explicit PrintDisassembler(FILE* stream) : stream_(stream) { }
+
+ protected:
+ virtual void ProcessOutput(const Instruction* instr) override;
+
+ private:
+ FILE *stream_;
+};
+
+void DisassembleInstruction(char* buffer, size_t bufsize, const Instruction* instr);
+char* GdbDisassembleInstruction(const Instruction* instr);
+
+} // namespace vixl
+
+#endif // VIXL_A64_DISASM_A64_H
diff --git a/js/src/jit/arm64/vixl/Globals-vixl.h b/js/src/jit/arm64/vixl/Globals-vixl.h
new file mode 100644
index 0000000000..2c7d5703f1
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Globals-vixl.h
@@ -0,0 +1,272 @@
+// Copyright 2015, VIXL authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_GLOBALS_H
+#define VIXL_GLOBALS_H
+
+// Get standard C99 macros for integer types.
+#ifndef __STDC_CONSTANT_MACROS
+#define __STDC_CONSTANT_MACROS
+#endif
+
+#ifndef __STDC_LIMIT_MACROS
+#define __STDC_LIMIT_MACROS
+#endif
+
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
+#include "mozilla/Assertions.h"
+
+#include <cstdarg>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+
+extern "C" {
+#include <inttypes.h>
+#include <stdint.h>
+}
+
+#include "jstypes.h"
+
+#include "jit/arm64/vixl/Platform-vixl.h"
+#include "js/Utility.h"
+
+#ifdef VIXL_NEGATIVE_TESTING
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#endif
+
+namespace vixl {
+
+typedef uint8_t byte;
+
+const int KBytes = 1024;
+const int MBytes = 1024 * KBytes;
+
+const int kBitsPerByte = 8;
+
+template <int SizeInBits>
+struct Unsigned;
+
+template <>
+struct Unsigned<32> {
+ typedef uint32_t type;
+};
+
+template <>
+struct Unsigned<64> {
+ typedef uint64_t type;
+};
+
+} // namespace vixl
+
+// Detect the host's pointer size.
+#if (UINTPTR_MAX == UINT32_MAX)
+#define VIXL_HOST_POINTER_32
+#elif (UINTPTR_MAX == UINT64_MAX)
+#define VIXL_HOST_POINTER_64
+#else
+#error "Unsupported host pointer size."
+#endif
+
+#ifdef VIXL_NEGATIVE_TESTING
+#define VIXL_ABORT() \
+ do { \
+ std::ostringstream oss; \
+ oss << "Aborting in " << __FILE__ << ", line " << __LINE__ << std::endl; \
+ throw std::runtime_error(oss.str()); \
+ } while (false)
+#define VIXL_ABORT_WITH_MSG(msg) \
+ do { \
+ std::ostringstream oss; \
+ oss << (msg) << "in " << __FILE__ << ", line " << __LINE__ << std::endl; \
+ throw std::runtime_error(oss.str()); \
+ } while (false)
+#define VIXL_CHECK(condition) \
+ do { \
+ if (!(condition)) { \
+ std::ostringstream oss; \
+ oss << "Assertion failed (" #condition ")\nin "; \
+ oss << __FILE__ << ", line " << __LINE__ << std::endl; \
+ throw std::runtime_error(oss.str()); \
+ } \
+ } while (false)
+#else
+#define VIXL_ABORT() \
+ do { \
+ MOZ_CRASH(); \
+ } while (false)
+#define VIXL_ABORT_WITH_MSG(msg) \
+ do { \
+ MOZ_CRASH(msg); \
+ } while (false)
+#define VIXL_CHECK(condition) \
+ do { \
+ if (!(condition)) { \
+ MOZ_CRASH(); \
+ } \
+ } while (false)
+#endif
+#ifdef DEBUG
+#define VIXL_ASSERT(condition) MOZ_ASSERT(condition)
+#define VIXL_UNIMPLEMENTED() \
+ do { \
+ VIXL_ABORT_WITH_MSG("UNIMPLEMENTED "); \
+ } while (false)
+#define VIXL_UNREACHABLE() \
+ do { \
+ VIXL_ABORT_WITH_MSG("UNREACHABLE "); \
+ } while (false)
+#else
+#define VIXL_ASSERT(condition) ((void)0)
+#define VIXL_UNIMPLEMENTED() ((void)0)
+#define VIXL_UNREACHABLE() MOZ_CRASH("vixl unreachable")
+#endif
+// This is not as powerful as template based assertions, but it is simple.
+// It assumes that the descriptions are unique. If this starts being a problem,
+// we can switch to a different implemention.
+#define VIXL_CONCAT(a, b) a##b
+#if __cplusplus >= 201103L
+#define VIXL_STATIC_ASSERT_LINE(line_unused, condition, message) \
+ static_assert(condition, message)
+#else
+#define VIXL_STATIC_ASSERT_LINE(line, condition, message_unused) \
+ typedef char VIXL_CONCAT(STATIC_ASSERT_LINE_, line)[(condition) ? 1 : -1] \
+ __attribute__((unused))
+#endif
+#define VIXL_STATIC_ASSERT(condition) \
+ VIXL_STATIC_ASSERT_LINE(__LINE__, condition, "")
+#define VIXL_STATIC_ASSERT_MESSAGE(condition, message) \
+ VIXL_STATIC_ASSERT_LINE(__LINE__, condition, message)
+
+#define VIXL_WARNING(message) \
+ do { \
+ printf("WARNING in %s, line %i: %s", __FILE__, __LINE__, message); \
+ } while (false)
+
+template <typename T1>
+inline void USE(const T1&) {}
+
+template <typename T1, typename T2>
+inline void USE(const T1&, const T2&) {}
+
+template <typename T1, typename T2, typename T3>
+inline void USE(const T1&, const T2&, const T3&) {}
+
+template <typename T1, typename T2, typename T3, typename T4>
+inline void USE(const T1&, const T2&, const T3&, const T4&) {}
+
+#define VIXL_ALIGNMENT_EXCEPTION() \
+ do { \
+ VIXL_ABORT_WITH_MSG("ALIGNMENT EXCEPTION\t"); \
+ } while (0)
+
+// The clang::fallthrough attribute is used along with the Wimplicit-fallthrough
+// argument to annotate intentional fall-through between switch labels.
+// For more information please refer to:
+// http://clang.llvm.org/docs/AttributeReference.html#fallthrough-clang-fallthrough
+#ifndef __has_warning
+#define __has_warning(x) 0
+#endif
+
+// Fallthrough annotation for Clang and C++11(201103L).
+#if __has_warning("-Wimplicit-fallthrough") && __cplusplus >= 201103L
+#define VIXL_FALLTHROUGH() [[clang::fallthrough]]
+// Fallthrough annotation for GCC >= 7.
+#elif __GNUC__ >= 7
+#define VIXL_FALLTHROUGH() __attribute__((fallthrough))
+#else
+#define VIXL_FALLTHROUGH() \
+ do { \
+ } while (0)
+#endif
+
+#if __cplusplus >= 201103L
+#define VIXL_NO_RETURN [[noreturn]]
+#else
+#define VIXL_NO_RETURN __attribute__((noreturn))
+#endif
+#ifdef VIXL_DEBUG
+#define VIXL_NO_RETURN_IN_DEBUG_MODE VIXL_NO_RETURN
+#else
+#define VIXL_NO_RETURN_IN_DEBUG_MODE
+#endif
+
+#if __cplusplus >= 201103L
+#define VIXL_OVERRIDE override
+#else
+#define VIXL_OVERRIDE
+#endif
+
+#ifdef VIXL_INCLUDE_SIMULATOR_AARCH64
+#ifndef VIXL_AARCH64_GENERATE_SIMULATOR_CODE
+#define VIXL_AARCH64_GENERATE_SIMULATOR_CODE 1
+#endif
+#else
+#ifndef VIXL_AARCH64_GENERATE_SIMULATOR_CODE
+#define VIXL_AARCH64_GENERATE_SIMULATOR_CODE 0
+#endif
+#if VIXL_AARCH64_GENERATE_SIMULATOR_CODE
+#warning "Generating Simulator instructions without Simulator support."
+#endif
+#endif
+
+// We do not have a simulator for AArch32, although we can pretend we do so that
+// tests that require running natively can be skipped.
+#ifndef __arm__
+#define VIXL_INCLUDE_SIMULATOR_AARCH32
+#ifndef VIXL_AARCH32_GENERATE_SIMULATOR_CODE
+#define VIXL_AARCH32_GENERATE_SIMULATOR_CODE 1
+#endif
+#else
+#ifndef VIXL_AARCH32_GENERATE_SIMULATOR_CODE
+#define VIXL_AARCH32_GENERATE_SIMULATOR_CODE 0
+#endif
+#endif
+
+// Target Architecture/ISA
+
+// Hack: always include AArch64.
+#define VIXL_INCLUDE_TARGET_A64
+
+#ifdef VIXL_INCLUDE_TARGET_A64
+#define VIXL_INCLUDE_TARGET_AARCH64
+#endif
+
+#if defined(VIXL_INCLUDE_TARGET_A32) && defined(VIXL_INCLUDE_TARGET_T32)
+#define VIXL_INCLUDE_TARGET_AARCH32
+#elif defined(VIXL_INCLUDE_TARGET_A32)
+#define VIXL_INCLUDE_TARGET_A32_ONLY
+#else
+#define VIXL_INCLUDE_TARGET_T32_ONLY
+#endif
+
+
+#endif // VIXL_GLOBALS_H
diff --git a/js/src/jit/arm64/vixl/Instructions-vixl.cpp b/js/src/jit/arm64/vixl/Instructions-vixl.cpp
new file mode 100644
index 0000000000..887f12d0f2
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Instructions-vixl.cpp
@@ -0,0 +1,642 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Instructions-vixl.h"
+
+#include "jit/arm64/vixl/Assembler-vixl.h"
+
+namespace vixl {
+
+static uint64_t RepeatBitsAcrossReg(unsigned reg_size,
+ uint64_t value,
+ unsigned width) {
+ VIXL_ASSERT((width == 2) || (width == 4) || (width == 8) || (width == 16) ||
+ (width == 32));
+ VIXL_ASSERT((reg_size == kWRegSize) || (reg_size == kXRegSize));
+ uint64_t result = value & ((UINT64_C(1) << width) - 1);
+ for (unsigned i = width; i < reg_size; i *= 2) {
+ result |= (result << i);
+ }
+ return result;
+}
+
+
+bool Instruction::IsLoad() const {
+ if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
+ return false;
+ }
+
+ if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
+ return Mask(LoadStorePairLBit) != 0;
+ } else {
+ LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreMask));
+ switch (op) {
+ case LDRB_w:
+ case LDRH_w:
+ case LDR_w:
+ case LDR_x:
+ case LDRSB_w:
+ case LDRSB_x:
+ case LDRSH_w:
+ case LDRSH_x:
+ case LDRSW_x:
+ case LDR_b:
+ case LDR_h:
+ case LDR_s:
+ case LDR_d:
+ case LDR_q: return true;
+ default: return false;
+ }
+ }
+}
+
+
+bool Instruction::IsStore() const {
+ if (Mask(LoadStoreAnyFMask) != LoadStoreAnyFixed) {
+ return false;
+ }
+
+ if (Mask(LoadStorePairAnyFMask) == LoadStorePairAnyFixed) {
+ return Mask(LoadStorePairLBit) == 0;
+ } else {
+ LoadStoreOp op = static_cast<LoadStoreOp>(Mask(LoadStoreMask));
+ switch (op) {
+ case STRB_w:
+ case STRH_w:
+ case STR_w:
+ case STR_x:
+ case STR_b:
+ case STR_h:
+ case STR_s:
+ case STR_d:
+ case STR_q: return true;
+ default: return false;
+ }
+ }
+}
+
+
+// Logical immediates can't encode zero, so a return value of zero is used to
+// indicate a failure case. Specifically, where the constraints on imm_s are
+// not met.
+uint64_t Instruction::ImmLogical() const {
+ unsigned reg_size = SixtyFourBits() ? kXRegSize : kWRegSize;
+ int32_t n = BitN();
+ int32_t imm_s = ImmSetBits();
+ int32_t imm_r = ImmRotate();
+
+ // An integer is constructed from the n, imm_s and imm_r bits according to
+ // the following table:
+ //
+ // N imms immr size S R
+ // 1 ssssss rrrrrr 64 UInt(ssssss) UInt(rrrrrr)
+ // 0 0sssss xrrrrr 32 UInt(sssss) UInt(rrrrr)
+ // 0 10ssss xxrrrr 16 UInt(ssss) UInt(rrrr)
+ // 0 110sss xxxrrr 8 UInt(sss) UInt(rrr)
+ // 0 1110ss xxxxrr 4 UInt(ss) UInt(rr)
+ // 0 11110s xxxxxr 2 UInt(s) UInt(r)
+ // (s bits must not be all set)
+ //
+ // A pattern is constructed of size bits, where the least significant S+1
+ // bits are set. The pattern is rotated right by R, and repeated across a
+ // 32 or 64-bit value, depending on destination register width.
+ //
+
+ if (n == 1) {
+ if (imm_s == 0x3f) {
+ return 0;
+ }
+ uint64_t bits = (UINT64_C(1) << (imm_s + 1)) - 1;
+ return RotateRight(bits, imm_r, 64);
+ } else {
+ if ((imm_s >> 1) == 0x1f) {
+ return 0;
+ }
+ for (int width = 0x20; width >= 0x2; width >>= 1) {
+ if ((imm_s & width) == 0) {
+ int mask = width - 1;
+ if ((imm_s & mask) == mask) {
+ return 0;
+ }
+ uint64_t bits = (UINT64_C(1) << ((imm_s & mask) + 1)) - 1;
+ return RepeatBitsAcrossReg(reg_size,
+ RotateRight(bits, imm_r & mask, width),
+ width);
+ }
+ }
+ }
+ VIXL_UNREACHABLE();
+ return 0;
+}
+
+
+uint32_t Instruction::ImmNEONabcdefgh() const {
+ return ImmNEONabc() << 5 | ImmNEONdefgh();
+}
+
+
+float Instruction::Imm8ToFP32(uint32_t imm8) {
+ // Imm8: abcdefgh (8 bits)
+ // Single: aBbb.bbbc.defg.h000.0000.0000.0000.0000 (32 bits)
+ // where B is b ^ 1
+ uint32_t bits = imm8;
+ uint32_t bit7 = (bits >> 7) & 0x1;
+ uint32_t bit6 = (bits >> 6) & 0x1;
+ uint32_t bit5_to_0 = bits & 0x3f;
+ uint32_t result = (bit7 << 31) | ((32 - bit6) << 25) | (bit5_to_0 << 19);
+
+ return RawbitsToFloat(result);
+}
+
+
+float Instruction::ImmFP32() const {
+ return Imm8ToFP32(ImmFP());
+}
+
+
+double Instruction::Imm8ToFP64(uint32_t imm8) {
+ // Imm8: abcdefgh (8 bits)
+ // Double: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000 (64 bits)
+ // where B is b ^ 1
+ uint32_t bits = imm8;
+ uint64_t bit7 = (bits >> 7) & 0x1;
+ uint64_t bit6 = (bits >> 6) & 0x1;
+ uint64_t bit5_to_0 = bits & 0x3f;
+ uint64_t result = (bit7 << 63) | ((256 - bit6) << 54) | (bit5_to_0 << 48);
+
+ return RawbitsToDouble(result);
+}
+
+
+double Instruction::ImmFP64() const {
+ return Imm8ToFP64(ImmFP());
+}
+
+
+float Instruction::ImmNEONFP32() const {
+ return Imm8ToFP32(ImmNEONabcdefgh());
+}
+
+
+double Instruction::ImmNEONFP64() const {
+ return Imm8ToFP64(ImmNEONabcdefgh());
+}
+
+
+unsigned CalcLSDataSize(LoadStoreOp op) {
+ VIXL_ASSERT((LSSize_offset + LSSize_width) == (kInstructionSize * 8));
+ unsigned size = static_cast<Instr>(op) >> LSSize_offset;
+ if ((op & LSVector_mask) != 0) {
+ // Vector register memory operations encode the access size in the "size"
+ // and "opc" fields.
+ if ((size == 0) && ((op & LSOpc_mask) >> LSOpc_offset) >= 2) {
+ size = kQRegSizeInBytesLog2;
+ }
+ }
+ return size;
+}
+
+
+unsigned CalcLSPairDataSize(LoadStorePairOp op) {
+ VIXL_STATIC_ASSERT(kXRegSizeInBytes == kDRegSizeInBytes);
+ VIXL_STATIC_ASSERT(kWRegSizeInBytes == kSRegSizeInBytes);
+ switch (op) {
+ case STP_q:
+ case LDP_q: return kQRegSizeInBytesLog2;
+ case STP_x:
+ case LDP_x:
+ case STP_d:
+ case LDP_d: return kXRegSizeInBytesLog2;
+ default: return kWRegSizeInBytesLog2;
+ }
+}
+
+
+int Instruction::ImmBranchRangeBitwidth(ImmBranchType branch_type) {
+ switch (branch_type) {
+ case UncondBranchType:
+ return ImmUncondBranch_width;
+ case CondBranchType:
+ return ImmCondBranch_width;
+ case CompareBranchType:
+ return ImmCmpBranch_width;
+ case TestBranchType:
+ return ImmTestBranch_width;
+ default:
+ VIXL_UNREACHABLE();
+ return 0;
+ }
+}
+
+
+int32_t Instruction::ImmBranchForwardRange(ImmBranchType branch_type) {
+ int32_t encoded_max = 1 << (ImmBranchRangeBitwidth(branch_type) - 1);
+ return encoded_max * kInstructionSize;
+}
+
+
+bool Instruction::IsValidImmPCOffset(ImmBranchType branch_type,
+ int64_t offset) {
+ return IsIntN(ImmBranchRangeBitwidth(branch_type), offset);
+}
+
+ImmBranchRangeType Instruction::ImmBranchTypeToRange(ImmBranchType branch_type)
+{
+ switch (branch_type) {
+ case UncondBranchType:
+ return UncondBranchRangeType;
+ case CondBranchType:
+ case CompareBranchType:
+ return CondBranchRangeType;
+ case TestBranchType:
+ return TestBranchRangeType;
+ default:
+ return UnknownBranchRangeType;
+ }
+}
+
+int32_t Instruction::ImmBranchMaxForwardOffset(ImmBranchRangeType range_type)
+{
+ // Branches encode a pc-relative two's complement number of 32-bit
+ // instructions. Compute the number of bytes corresponding to the largest
+ // positive number of instructions that can be encoded.
+ switch(range_type) {
+ case TestBranchRangeType:
+ return ((1 << ImmTestBranch_width) - 1) / 2 * kInstructionSize;
+ case CondBranchRangeType:
+ return ((1 << ImmCondBranch_width) - 1) / 2 * kInstructionSize;
+ case UncondBranchRangeType:
+ return ((1 << ImmUncondBranch_width) - 1) / 2 * kInstructionSize;
+ default:
+ VIXL_UNREACHABLE();
+ return 0;
+ }
+}
+
+int32_t Instruction::ImmBranchMinBackwardOffset(ImmBranchRangeType range_type)
+{
+ switch(range_type) {
+ case TestBranchRangeType:
+ return -int32_t(1 << ImmTestBranch_width) / int32_t(2 * kInstructionSize);
+ case CondBranchRangeType:
+ return -int32_t(1 << ImmCondBranch_width) / int32_t(2 * kInstructionSize);
+ case UncondBranchRangeType:
+ return -int32_t(1 << ImmUncondBranch_width) / int32_t(2 * kInstructionSize);
+ default:
+ VIXL_UNREACHABLE();
+ return 0;
+ }
+}
+
+const Instruction* Instruction::ImmPCOffsetTarget() const {
+ const Instruction * base = this;
+ ptrdiff_t offset;
+ if (IsPCRelAddressing()) {
+ // ADR and ADRP.
+ offset = ImmPCRel();
+ if (Mask(PCRelAddressingMask) == ADRP) {
+ base = AlignDown(base, kPageSize);
+ offset *= kPageSize;
+ } else {
+ VIXL_ASSERT(Mask(PCRelAddressingMask) == ADR);
+ }
+ } else {
+ // All PC-relative branches.
+ VIXL_ASSERT(BranchType() != UnknownBranchType);
+ // Relative branch offsets are instruction-size-aligned.
+ offset = ImmBranch() << kInstructionSizeLog2;
+ }
+ return base + offset;
+}
+
+
+int Instruction::ImmBranch() const {
+ switch (BranchType()) {
+ case CondBranchType: return ImmCondBranch();
+ case UncondBranchType: return ImmUncondBranch();
+ case CompareBranchType: return ImmCmpBranch();
+ case TestBranchType: return ImmTestBranch();
+ default: VIXL_UNREACHABLE();
+ }
+ return 0;
+}
+
+
+void Instruction::SetImmPCOffsetTarget(const Instruction* target) {
+ if (IsPCRelAddressing()) {
+ SetPCRelImmTarget(target);
+ } else {
+ SetBranchImmTarget(target);
+ }
+}
+
+
+void Instruction::SetPCRelImmTarget(const Instruction* target) {
+ ptrdiff_t imm21;
+ if ((Mask(PCRelAddressingMask) == ADR)) {
+ imm21 = target - this;
+ } else {
+ VIXL_ASSERT(Mask(PCRelAddressingMask) == ADRP);
+ uintptr_t this_page = reinterpret_cast<uintptr_t>(this) / kPageSize;
+ uintptr_t target_page = reinterpret_cast<uintptr_t>(target) / kPageSize;
+ imm21 = target_page - this_page;
+ }
+ Instr imm = Assembler::ImmPCRelAddress(static_cast<int32_t>(imm21));
+
+ SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
+}
+
+
+void Instruction::SetBranchImmTarget(const Instruction* target) {
+ VIXL_ASSERT(((target - this) & 3) == 0);
+ Instr branch_imm = 0;
+ uint32_t imm_mask = 0;
+ int offset = static_cast<int>((target - this) >> kInstructionSizeLog2);
+ switch (BranchType()) {
+ case CondBranchType: {
+ branch_imm = Assembler::ImmCondBranch(offset);
+ imm_mask = ImmCondBranch_mask;
+ break;
+ }
+ case UncondBranchType: {
+ branch_imm = Assembler::ImmUncondBranch(offset);
+ imm_mask = ImmUncondBranch_mask;
+ break;
+ }
+ case CompareBranchType: {
+ branch_imm = Assembler::ImmCmpBranch(offset);
+ imm_mask = ImmCmpBranch_mask;
+ break;
+ }
+ case TestBranchType: {
+ branch_imm = Assembler::ImmTestBranch(offset);
+ imm_mask = ImmTestBranch_mask;
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+ SetInstructionBits(Mask(~imm_mask) | branch_imm);
+}
+
+
+void Instruction::SetImmLLiteral(const Instruction* source) {
+ VIXL_ASSERT(IsWordAligned(source));
+ ptrdiff_t offset = (source - this) >> kLiteralEntrySizeLog2;
+ Instr imm = Assembler::ImmLLiteral(static_cast<int>(offset));
+ Instr mask = ImmLLiteral_mask;
+
+ SetInstructionBits(Mask(~mask) | imm);
+}
+
+
+VectorFormat VectorFormatHalfWidth(const VectorFormat vform) {
+ VIXL_ASSERT(vform == kFormat8H || vform == kFormat4S || vform == kFormat2D ||
+ vform == kFormatH || vform == kFormatS || vform == kFormatD);
+ switch (vform) {
+ case kFormat8H: return kFormat8B;
+ case kFormat4S: return kFormat4H;
+ case kFormat2D: return kFormat2S;
+ case kFormatH: return kFormatB;
+ case kFormatS: return kFormatH;
+ case kFormatD: return kFormatS;
+ default: VIXL_UNREACHABLE(); return kFormatUndefined;
+ }
+}
+
+
+VectorFormat VectorFormatDoubleWidth(const VectorFormat vform) {
+ VIXL_ASSERT(vform == kFormat8B || vform == kFormat4H || vform == kFormat2S ||
+ vform == kFormatB || vform == kFormatH || vform == kFormatS);
+ switch (vform) {
+ case kFormat8B: return kFormat8H;
+ case kFormat4H: return kFormat4S;
+ case kFormat2S: return kFormat2D;
+ case kFormatB: return kFormatH;
+ case kFormatH: return kFormatS;
+ case kFormatS: return kFormatD;
+ default: VIXL_UNREACHABLE(); return kFormatUndefined;
+ }
+}
+
+
+VectorFormat VectorFormatFillQ(const VectorFormat vform) {
+ switch (vform) {
+ case kFormatB:
+ case kFormat8B:
+ case kFormat16B: return kFormat16B;
+ case kFormatH:
+ case kFormat4H:
+ case kFormat8H: return kFormat8H;
+ case kFormatS:
+ case kFormat2S:
+ case kFormat4S: return kFormat4S;
+ case kFormatD:
+ case kFormat1D:
+ case kFormat2D: return kFormat2D;
+ default: VIXL_UNREACHABLE(); return kFormatUndefined;
+ }
+}
+
+VectorFormat VectorFormatHalfWidthDoubleLanes(const VectorFormat vform) {
+ switch (vform) {
+ case kFormat4H: return kFormat8B;
+ case kFormat8H: return kFormat16B;
+ case kFormat2S: return kFormat4H;
+ case kFormat4S: return kFormat8H;
+ case kFormat1D: return kFormat2S;
+ case kFormat2D: return kFormat4S;
+ default: VIXL_UNREACHABLE(); return kFormatUndefined;
+ }
+}
+
+VectorFormat VectorFormatDoubleLanes(const VectorFormat vform) {
+ VIXL_ASSERT(vform == kFormat8B || vform == kFormat4H || vform == kFormat2S);
+ switch (vform) {
+ case kFormat8B: return kFormat16B;
+ case kFormat4H: return kFormat8H;
+ case kFormat2S: return kFormat4S;
+ default: VIXL_UNREACHABLE(); return kFormatUndefined;
+ }
+}
+
+
+VectorFormat VectorFormatHalfLanes(const VectorFormat vform) {
+ VIXL_ASSERT(vform == kFormat16B || vform == kFormat8H || vform == kFormat4S);
+ switch (vform) {
+ case kFormat16B: return kFormat8B;
+ case kFormat8H: return kFormat4H;
+ case kFormat4S: return kFormat2S;
+ default: VIXL_UNREACHABLE(); return kFormatUndefined;
+ }
+}
+
+
+VectorFormat ScalarFormatFromLaneSize(int laneSize) {
+ switch (laneSize) {
+ case 8: return kFormatB;
+ case 16: return kFormatH;
+ case 32: return kFormatS;
+ case 64: return kFormatD;
+ default: VIXL_UNREACHABLE(); return kFormatUndefined;
+ }
+}
+
+
+unsigned RegisterSizeInBitsFromFormat(VectorFormat vform) {
+ VIXL_ASSERT(vform != kFormatUndefined);
+ switch (vform) {
+ case kFormatB: return kBRegSize;
+ case kFormatH: return kHRegSize;
+ case kFormatS: return kSRegSize;
+ case kFormatD: return kDRegSize;
+ case kFormat8B:
+ case kFormat4H:
+ case kFormat2S:
+ case kFormat1D: return kDRegSize;
+ default: return kQRegSize;
+ }
+}
+
+
+unsigned RegisterSizeInBytesFromFormat(VectorFormat vform) {
+ return RegisterSizeInBitsFromFormat(vform) / 8;
+}
+
+
+unsigned LaneSizeInBitsFromFormat(VectorFormat vform) {
+ VIXL_ASSERT(vform != kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ case kFormat8B:
+ case kFormat16B: return 8;
+ case kFormatH:
+ case kFormat4H:
+ case kFormat8H: return 16;
+ case kFormatS:
+ case kFormat2S:
+ case kFormat4S: return 32;
+ case kFormatD:
+ case kFormat1D:
+ case kFormat2D: return 64;
+ default: VIXL_UNREACHABLE(); return 0;
+ }
+}
+
+
+int LaneSizeInBytesFromFormat(VectorFormat vform) {
+ return LaneSizeInBitsFromFormat(vform) / 8;
+}
+
+
+int LaneSizeInBytesLog2FromFormat(VectorFormat vform) {
+ VIXL_ASSERT(vform != kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ case kFormat8B:
+ case kFormat16B: return 0;
+ case kFormatH:
+ case kFormat4H:
+ case kFormat8H: return 1;
+ case kFormatS:
+ case kFormat2S:
+ case kFormat4S: return 2;
+ case kFormatD:
+ case kFormat1D:
+ case kFormat2D: return 3;
+ default: VIXL_UNREACHABLE(); return 0;
+ }
+}
+
+
+int LaneCountFromFormat(VectorFormat vform) {
+ VIXL_ASSERT(vform != kFormatUndefined);
+ switch (vform) {
+ case kFormat16B: return 16;
+ case kFormat8B:
+ case kFormat8H: return 8;
+ case kFormat4H:
+ case kFormat4S: return 4;
+ case kFormat2S:
+ case kFormat2D: return 2;
+ case kFormat1D:
+ case kFormatB:
+ case kFormatH:
+ case kFormatS:
+ case kFormatD: return 1;
+ default: VIXL_UNREACHABLE(); return 0;
+ }
+}
+
+
+int MaxLaneCountFromFormat(VectorFormat vform) {
+ VIXL_ASSERT(vform != kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ case kFormat8B:
+ case kFormat16B: return 16;
+ case kFormatH:
+ case kFormat4H:
+ case kFormat8H: return 8;
+ case kFormatS:
+ case kFormat2S:
+ case kFormat4S: return 4;
+ case kFormatD:
+ case kFormat1D:
+ case kFormat2D: return 2;
+ default: VIXL_UNREACHABLE(); return 0;
+ }
+}
+
+
+// Does 'vform' indicate a vector format or a scalar format?
+bool IsVectorFormat(VectorFormat vform) {
+ VIXL_ASSERT(vform != kFormatUndefined);
+ switch (vform) {
+ case kFormatB:
+ case kFormatH:
+ case kFormatS:
+ case kFormatD: return false;
+ default: return true;
+ }
+}
+
+
+int64_t MaxIntFromFormat(VectorFormat vform) {
+ return INT64_MAX >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+
+int64_t MinIntFromFormat(VectorFormat vform) {
+ return INT64_MIN >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+
+
+uint64_t MaxUintFromFormat(VectorFormat vform) {
+ return UINT64_MAX >> (64 - LaneSizeInBitsFromFormat(vform));
+}
+} // namespace vixl
+
diff --git a/js/src/jit/arm64/vixl/Instructions-vixl.h b/js/src/jit/arm64/vixl/Instructions-vixl.h
new file mode 100644
index 0000000000..80a75599f9
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Instructions-vixl.h
@@ -0,0 +1,805 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_INSTRUCTIONS_A64_H_
+#define VIXL_A64_INSTRUCTIONS_A64_H_
+
+#include "jit/arm64/vixl/Constants-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+
+namespace vixl {
+// ISA constants. --------------------------------------------------------------
+
+typedef uint32_t Instr;
+const unsigned kInstructionSize = 4;
+const unsigned kInstructionSizeLog2 = 2;
+const unsigned kLiteralEntrySize = 4;
+const unsigned kLiteralEntrySizeLog2 = 2;
+const unsigned kMaxLoadLiteralRange = 1 * MBytes;
+
+// This is the nominal page size (as used by the adrp instruction); the actual
+// size of the memory pages allocated by the kernel is likely to differ.
+const unsigned kPageSize = 4 * KBytes;
+const unsigned kPageSizeLog2 = 12;
+
+const unsigned kBRegSize = 8;
+const unsigned kBRegSizeLog2 = 3;
+const unsigned kBRegSizeInBytes = kBRegSize / 8;
+const unsigned kBRegSizeInBytesLog2 = kBRegSizeLog2 - 3;
+const unsigned kHRegSize = 16;
+const unsigned kHRegSizeLog2 = 4;
+const unsigned kHRegSizeInBytes = kHRegSize / 8;
+const unsigned kHRegSizeInBytesLog2 = kHRegSizeLog2 - 3;
+const unsigned kWRegSize = 32;
+const unsigned kWRegSizeLog2 = 5;
+const unsigned kWRegSizeInBytes = kWRegSize / 8;
+const unsigned kWRegSizeInBytesLog2 = kWRegSizeLog2 - 3;
+const unsigned kXRegSize = 64;
+const unsigned kXRegSizeLog2 = 6;
+const unsigned kXRegSizeInBytes = kXRegSize / 8;
+const unsigned kXRegSizeInBytesLog2 = kXRegSizeLog2 - 3;
+const unsigned kSRegSize = 32;
+const unsigned kSRegSizeLog2 = 5;
+const unsigned kSRegSizeInBytes = kSRegSize / 8;
+const unsigned kSRegSizeInBytesLog2 = kSRegSizeLog2 - 3;
+const unsigned kDRegSize = 64;
+const unsigned kDRegSizeLog2 = 6;
+const unsigned kDRegSizeInBytes = kDRegSize / 8;
+const unsigned kDRegSizeInBytesLog2 = kDRegSizeLog2 - 3;
+const unsigned kQRegSize = 128;
+const unsigned kQRegSizeLog2 = 7;
+const unsigned kQRegSizeInBytes = kQRegSize / 8;
+const unsigned kQRegSizeInBytesLog2 = kQRegSizeLog2 - 3;
+const uint64_t kWRegMask = UINT64_C(0xffffffff);
+const uint64_t kXRegMask = UINT64_C(0xffffffffffffffff);
+const uint64_t kSRegMask = UINT64_C(0xffffffff);
+const uint64_t kDRegMask = UINT64_C(0xffffffffffffffff);
+const uint64_t kSSignMask = UINT64_C(0x80000000);
+const uint64_t kDSignMask = UINT64_C(0x8000000000000000);
+const uint64_t kWSignMask = UINT64_C(0x80000000);
+const uint64_t kXSignMask = UINT64_C(0x8000000000000000);
+const uint64_t kByteMask = UINT64_C(0xff);
+const uint64_t kHalfWordMask = UINT64_C(0xffff);
+const uint64_t kWordMask = UINT64_C(0xffffffff);
+const uint64_t kXMaxUInt = UINT64_C(0xffffffffffffffff);
+const uint64_t kXMaxExactUInt = UINT64_C(0xfffffffffffff800);
+const uint64_t kWMaxUInt = UINT64_C(0xffffffff);
+const int64_t kXMaxInt = INT64_C(0x7fffffffffffffff);
+const int64_t kXMaxExactInt = UINT64_C(0x7ffffffffffffc00);
+const int64_t kXMinInt = INT64_C(0x8000000000000000);
+const int32_t kWMaxInt = INT32_C(0x7fffffff);
+const int32_t kWMinInt = INT32_C(0x80000000);
+const unsigned kLinkRegCode = 30;
+const unsigned kZeroRegCode = 31;
+const unsigned kSPRegInternalCode = 63;
+const unsigned kRegCodeMask = 0x1f;
+
+const unsigned kAddressTagOffset = 56;
+const unsigned kAddressTagWidth = 8;
+const uint64_t kAddressTagMask =
+ ((UINT64_C(1) << kAddressTagWidth) - 1) << kAddressTagOffset;
+VIXL_STATIC_ASSERT(kAddressTagMask == UINT64_C(0xff00000000000000));
+
+unsigned CalcLSDataSize(LoadStoreOp op);
+unsigned CalcLSPairDataSize(LoadStorePairOp op);
+
+enum ImmBranchType {
+ UnknownBranchType = 0,
+ CondBranchType = 1,
+ UncondBranchType = 2,
+ CompareBranchType = 3,
+ TestBranchType = 4
+};
+
+// The classes of immediate branch ranges, in order of increasing range.
+// Note that CondBranchType and CompareBranchType have the same range.
+enum ImmBranchRangeType {
+ TestBranchRangeType, // tbz/tbnz: imm14 = +/- 32KB.
+ CondBranchRangeType, // b.cond/cbz/cbnz: imm19 = +/- 1MB.
+ UncondBranchRangeType, // b/bl: imm26 = +/- 128MB.
+ UnknownBranchRangeType,
+
+ // Number of 'short-range' branch range types.
+ // We don't consider unconditional branches 'short-range'.
+ NumShortBranchRangeTypes = UncondBranchRangeType
+};
+
+enum AddrMode {
+ Offset,
+ PreIndex,
+ PostIndex
+};
+
+enum Reg31Mode {
+ Reg31IsStackPointer,
+ Reg31IsZeroRegister
+};
+
+// Instructions. ---------------------------------------------------------------
+
+class Instruction {
+ public:
+ Instr InstructionBits() const {
+ return *(reinterpret_cast<const Instr*>(this));
+ }
+
+ void SetInstructionBits(Instr new_instr) {
+ *(reinterpret_cast<Instr*>(this)) = new_instr;
+ }
+
+ int Bit(int pos) const {
+ return (InstructionBits() >> pos) & 1;
+ }
+
+ uint32_t Bits(int msb, int lsb) const {
+ return ExtractUnsignedBitfield32(msb, lsb, InstructionBits());
+ }
+
+ int32_t SignedBits(int msb, int lsb) const {
+ int32_t bits = *(reinterpret_cast<const int32_t*>(this));
+ return ExtractSignedBitfield32(msb, lsb, bits);
+ }
+
+ Instr Mask(uint32_t mask) const {
+ return InstructionBits() & mask;
+ }
+
+ #define DEFINE_GETTER(Name, HighBit, LowBit, Func) \
+ int32_t Name() const { return Func(HighBit, LowBit); }
+ INSTRUCTION_FIELDS_LIST(DEFINE_GETTER)
+ #undef DEFINE_GETTER
+
+ #define DEFINE_SETTER(Name, HighBit, LowBit, Func) \
+ inline void Set##Name(unsigned n) { SetBits32(HighBit, LowBit, n); }
+ INSTRUCTION_FIELDS_LIST(DEFINE_SETTER)
+ #undef DEFINE_SETTER
+
+ // ImmPCRel is a compound field (not present in INSTRUCTION_FIELDS_LIST),
+ // formed from ImmPCRelLo and ImmPCRelHi.
+ int ImmPCRel() const {
+ int offset =
+ static_cast<int>((ImmPCRelHi() << ImmPCRelLo_width) | ImmPCRelLo());
+ int width = ImmPCRelLo_width + ImmPCRelHi_width;
+ return ExtractSignedBitfield32(width - 1, 0, offset);
+ }
+
+ uint64_t ImmLogical() const;
+ unsigned ImmNEONabcdefgh() const;
+ float ImmFP32() const;
+ double ImmFP64() const;
+ float ImmNEONFP32() const;
+ double ImmNEONFP64() const;
+
+ unsigned SizeLS() const {
+ return CalcLSDataSize(static_cast<LoadStoreOp>(Mask(LoadStoreMask)));
+ }
+
+ unsigned SizeLSPair() const {
+ return CalcLSPairDataSize(
+ static_cast<LoadStorePairOp>(Mask(LoadStorePairMask)));
+ }
+
+ int NEONLSIndex(int access_size_shift) const {
+ int64_t q = NEONQ();
+ int64_t s = NEONS();
+ int64_t size = NEONLSSize();
+ int64_t index = (q << 3) | (s << 2) | size;
+ return static_cast<int>(index >> access_size_shift);
+ }
+
+ // Helpers.
+ bool IsCondBranchImm() const {
+ return Mask(ConditionalBranchFMask) == ConditionalBranchFixed;
+ }
+
+ bool IsUncondBranchImm() const {
+ return Mask(UnconditionalBranchFMask) == UnconditionalBranchFixed;
+ }
+
+ bool IsCompareBranch() const {
+ return Mask(CompareBranchFMask) == CompareBranchFixed;
+ }
+
+ bool IsTestBranch() const {
+ return Mask(TestBranchFMask) == TestBranchFixed;
+ }
+
+ bool IsImmBranch() const {
+ return BranchType() != UnknownBranchType;
+ }
+
+ bool IsPCRelAddressing() const {
+ return Mask(PCRelAddressingFMask) == PCRelAddressingFixed;
+ }
+
+ bool IsLogicalImmediate() const {
+ return Mask(LogicalImmediateFMask) == LogicalImmediateFixed;
+ }
+
+ bool IsAddSubImmediate() const {
+ return Mask(AddSubImmediateFMask) == AddSubImmediateFixed;
+ }
+
+ bool IsAddSubExtended() const {
+ return Mask(AddSubExtendedFMask) == AddSubExtendedFixed;
+ }
+
+ bool IsLoadOrStore() const {
+ return Mask(LoadStoreAnyFMask) == LoadStoreAnyFixed;
+ }
+
+ bool IsLoad() const;
+ bool IsStore() const;
+
+ bool IsLoadLiteral() const {
+ // This includes PRFM_lit.
+ return Mask(LoadLiteralFMask) == LoadLiteralFixed;
+ }
+
+ bool IsMovn() const {
+ return (Mask(MoveWideImmediateMask) == MOVN_x) ||
+ (Mask(MoveWideImmediateMask) == MOVN_w);
+ }
+
+ // Mozilla modifications.
+ bool IsUncondB() const;
+ bool IsCondB() const;
+ bool IsBL() const;
+ bool IsBR() const;
+ bool IsBLR() const;
+ bool IsTBZ() const;
+ bool IsTBNZ() const;
+ bool IsCBZ() const;
+ bool IsCBNZ() const;
+ bool IsLDR() const;
+ bool IsNOP() const;
+ bool IsCSDB() const;
+ bool IsADR() const;
+ bool IsADRP() const;
+ bool IsMovz() const;
+ bool IsMovk() const;
+ bool IsBranchLinkImm() const;
+ bool IsTargetReachable(const Instruction* target) const;
+ ptrdiff_t ImmPCRawOffset() const;
+ void SetImmPCRawOffset(ptrdiff_t offset);
+ void SetBits32(int msb, int lsb, unsigned value);
+
+ // Is this a stack pointer synchronization instruction as inserted by
+ // MacroAssembler::syncStackPtr()?
+ bool IsStackPtrSync() const;
+
+ static int ImmBranchRangeBitwidth(ImmBranchType branch_type);
+ static int32_t ImmBranchForwardRange(ImmBranchType branch_type);
+
+ // Check if offset can be encoded as a RAW offset in a branch_type
+ // instruction. The offset must be encodeable directly as the immediate field
+ // in the instruction, it is not scaled by kInstructionSize first.
+ static bool IsValidImmPCOffset(ImmBranchType branch_type, int64_t offset);
+
+ // Get the range type corresponding to a branch type.
+ static ImmBranchRangeType ImmBranchTypeToRange(ImmBranchType);
+
+ // Get the maximum realizable forward PC offset (in bytes) for an immediate
+ // branch of the given range type.
+ // This is the largest positive multiple of kInstructionSize, offset, such
+ // that:
+ //
+ // IsValidImmPCOffset(xxx, offset / kInstructionSize)
+ //
+ // returns true for the same branch type.
+ static int32_t ImmBranchMaxForwardOffset(ImmBranchRangeType range_type);
+
+ // Get the minimuum realizable backward PC offset (in bytes) for an immediate
+ // branch of the given range type.
+ // This is the smallest (i.e., largest in magnitude) negative multiple of
+ // kInstructionSize, offset, such that:
+ //
+ // IsValidImmPCOffset(xxx, offset / kInstructionSize)
+ //
+ // returns true for the same branch type.
+ static int32_t ImmBranchMinBackwardOffset(ImmBranchRangeType range_type);
+
+ // Indicate whether Rd can be the stack pointer or the zero register. This
+ // does not check that the instruction actually has an Rd field.
+ Reg31Mode RdMode() const {
+ // The following instructions use sp or wsp as Rd:
+ // Add/sub (immediate) when not setting the flags.
+ // Add/sub (extended) when not setting the flags.
+ // Logical (immediate) when not setting the flags.
+ // Otherwise, r31 is the zero register.
+ if (IsAddSubImmediate() || IsAddSubExtended()) {
+ if (Mask(AddSubSetFlagsBit)) {
+ return Reg31IsZeroRegister;
+ } else {
+ return Reg31IsStackPointer;
+ }
+ }
+ if (IsLogicalImmediate()) {
+ // Of the logical (immediate) instructions, only ANDS (and its aliases)
+ // can set the flags. The others can all write into sp.
+ // Note that some logical operations are not available to
+ // immediate-operand instructions, so we have to combine two masks here.
+ if (Mask(LogicalImmediateMask & LogicalOpMask) == ANDS) {
+ return Reg31IsZeroRegister;
+ } else {
+ return Reg31IsStackPointer;
+ }
+ }
+ return Reg31IsZeroRegister;
+ }
+
+ // Indicate whether Rn can be the stack pointer or the zero register. This
+ // does not check that the instruction actually has an Rn field.
+ Reg31Mode RnMode() const {
+ // The following instructions use sp or wsp as Rn:
+ // All loads and stores.
+ // Add/sub (immediate).
+ // Add/sub (extended).
+ // Otherwise, r31 is the zero register.
+ if (IsLoadOrStore() || IsAddSubImmediate() || IsAddSubExtended()) {
+ return Reg31IsStackPointer;
+ }
+ return Reg31IsZeroRegister;
+ }
+
+ ImmBranchType BranchType() const {
+ if (IsCondBranchImm()) {
+ return CondBranchType;
+ } else if (IsUncondBranchImm()) {
+ return UncondBranchType;
+ } else if (IsCompareBranch()) {
+ return CompareBranchType;
+ } else if (IsTestBranch()) {
+ return TestBranchType;
+ } else {
+ return UnknownBranchType;
+ }
+ }
+
+ // Find the target of this instruction. 'this' may be a branch or a
+ // PC-relative addressing instruction.
+ const Instruction* ImmPCOffsetTarget() const;
+
+ // Patch a PC-relative offset to refer to 'target'. 'this' may be a branch or
+ // a PC-relative addressing instruction.
+ void SetImmPCOffsetTarget(const Instruction* target);
+ // Patch a literal load instruction to load from 'source'.
+ void SetImmLLiteral(const Instruction* source);
+
+ // The range of a load literal instruction, expressed as 'instr +- range'.
+ // The range is actually the 'positive' range; the branch instruction can
+ // target [instr - range - kInstructionSize, instr + range].
+ static const int kLoadLiteralImmBitwidth = 19;
+ static const int kLoadLiteralRange =
+ (1 << kLoadLiteralImmBitwidth) / 2 - kInstructionSize;
+
+ // Calculate the address of a literal referred to by a load-literal
+ // instruction, and return it as the specified type.
+ //
+ // The literal itself is safely mutable only if the backing buffer is safely
+ // mutable.
+ template <typename T>
+ T LiteralAddress() const {
+ uint64_t base_raw = reinterpret_cast<uint64_t>(this);
+ int64_t offset = ImmLLiteral() << kLiteralEntrySizeLog2;
+ uint64_t address_raw = base_raw + offset;
+
+ // Cast the address using a C-style cast. A reinterpret_cast would be
+ // appropriate, but it can't cast one integral type to another.
+ T address = (T)(address_raw);
+
+ // Assert that the address can be represented by the specified type.
+ VIXL_ASSERT((uint64_t)(address) == address_raw);
+
+ return address;
+ }
+
+ uint32_t Literal32() const {
+ uint32_t literal;
+ memcpy(&literal, LiteralAddress<const void*>(), sizeof(literal));
+ return literal;
+ }
+
+ uint64_t Literal64() const {
+ uint64_t literal;
+ memcpy(&literal, LiteralAddress<const void*>(), sizeof(literal));
+ return literal;
+ }
+
+ void SetLiteral64(uint64_t literal) const {
+ memcpy(LiteralAddress<void*>(), &literal, sizeof(literal));
+ }
+
+ float LiteralFP32() const {
+ return RawbitsToFloat(Literal32());
+ }
+
+ double LiteralFP64() const {
+ return RawbitsToDouble(Literal64());
+ }
+
+ const Instruction* NextInstruction() const {
+ return this + kInstructionSize;
+ }
+
+ // Skip any constant pools with artificial guards at this point.
+ // Return either |this| or the first instruction after the pool.
+ const Instruction* skipPool() const;
+
+ const Instruction* InstructionAtOffset(int64_t offset) const {
+ VIXL_ASSERT(IsWordAligned(this + offset));
+ return this + offset;
+ }
+
+ template<typename T> static Instruction* Cast(T src) {
+ return reinterpret_cast<Instruction*>(src);
+ }
+
+ template<typename T> static const Instruction* CastConst(T src) {
+ return reinterpret_cast<const Instruction*>(src);
+ }
+
+ private:
+ int ImmBranch() const;
+
+ static float Imm8ToFP32(uint32_t imm8);
+ static double Imm8ToFP64(uint32_t imm8);
+
+ void SetPCRelImmTarget(const Instruction* target);
+ void SetBranchImmTarget(const Instruction* target);
+};
+
+
+// Functions for handling NEON vector format information.
+enum VectorFormat {
+ kFormatUndefined = 0xffffffff,
+ kFormat8B = NEON_8B,
+ kFormat16B = NEON_16B,
+ kFormat4H = NEON_4H,
+ kFormat8H = NEON_8H,
+ kFormat2S = NEON_2S,
+ kFormat4S = NEON_4S,
+ kFormat1D = NEON_1D,
+ kFormat2D = NEON_2D,
+
+ // Scalar formats. We add the scalar bit to distinguish between scalar and
+ // vector enumerations; the bit is always set in the encoding of scalar ops
+ // and always clear for vector ops. Although kFormatD and kFormat1D appear
+ // to be the same, their meaning is subtly different. The first is a scalar
+ // operation, the second a vector operation that only affects one lane.
+ kFormatB = NEON_B | NEONScalar,
+ kFormatH = NEON_H | NEONScalar,
+ kFormatS = NEON_S | NEONScalar,
+ kFormatD = NEON_D | NEONScalar
+};
+
+VectorFormat VectorFormatHalfWidth(const VectorFormat vform);
+VectorFormat VectorFormatDoubleWidth(const VectorFormat vform);
+VectorFormat VectorFormatDoubleLanes(const VectorFormat vform);
+VectorFormat VectorFormatHalfLanes(const VectorFormat vform);
+VectorFormat ScalarFormatFromLaneSize(int lanesize);
+VectorFormat VectorFormatHalfWidthDoubleLanes(const VectorFormat vform);
+VectorFormat VectorFormatFillQ(const VectorFormat vform);
+unsigned RegisterSizeInBitsFromFormat(VectorFormat vform);
+unsigned RegisterSizeInBytesFromFormat(VectorFormat vform);
+// TODO: Make the return types of these functions consistent.
+unsigned LaneSizeInBitsFromFormat(VectorFormat vform);
+int LaneSizeInBytesFromFormat(VectorFormat vform);
+int LaneSizeInBytesLog2FromFormat(VectorFormat vform);
+int LaneCountFromFormat(VectorFormat vform);
+int MaxLaneCountFromFormat(VectorFormat vform);
+bool IsVectorFormat(VectorFormat vform);
+int64_t MaxIntFromFormat(VectorFormat vform);
+int64_t MinIntFromFormat(VectorFormat vform);
+uint64_t MaxUintFromFormat(VectorFormat vform);
+
+
+enum NEONFormat {
+ NF_UNDEF = 0,
+ NF_8B = 1,
+ NF_16B = 2,
+ NF_4H = 3,
+ NF_8H = 4,
+ NF_2S = 5,
+ NF_4S = 6,
+ NF_1D = 7,
+ NF_2D = 8,
+ NF_B = 9,
+ NF_H = 10,
+ NF_S = 11,
+ NF_D = 12
+};
+
+static const unsigned kNEONFormatMaxBits = 6;
+
+struct NEONFormatMap {
+ // The bit positions in the instruction to consider.
+ uint8_t bits[kNEONFormatMaxBits];
+
+ // Mapping from concatenated bits to format.
+ NEONFormat map[1 << kNEONFormatMaxBits];
+};
+
+class NEONFormatDecoder {
+ public:
+ enum SubstitutionMode {
+ kPlaceholder,
+ kFormat
+ };
+
+ // Construct a format decoder with increasingly specific format maps for each
+ // subsitution. If no format map is specified, the default is the integer
+ // format map.
+ explicit NEONFormatDecoder(const Instruction* instr) {
+ instrbits_ = instr->InstructionBits();
+ SetFormatMaps(IntegerFormatMap());
+ }
+ NEONFormatDecoder(const Instruction* instr,
+ const NEONFormatMap* format) {
+ instrbits_ = instr->InstructionBits();
+ SetFormatMaps(format);
+ }
+ NEONFormatDecoder(const Instruction* instr,
+ const NEONFormatMap* format0,
+ const NEONFormatMap* format1) {
+ instrbits_ = instr->InstructionBits();
+ SetFormatMaps(format0, format1);
+ }
+ NEONFormatDecoder(const Instruction* instr,
+ const NEONFormatMap* format0,
+ const NEONFormatMap* format1,
+ const NEONFormatMap* format2) {
+ instrbits_ = instr->InstructionBits();
+ SetFormatMaps(format0, format1, format2);
+ }
+
+ // Set the format mapping for all or individual substitutions.
+ void SetFormatMaps(const NEONFormatMap* format0,
+ const NEONFormatMap* format1 = NULL,
+ const NEONFormatMap* format2 = NULL) {
+ VIXL_ASSERT(format0 != NULL);
+ formats_[0] = format0;
+ formats_[1] = (format1 == NULL) ? formats_[0] : format1;
+ formats_[2] = (format2 == NULL) ? formats_[1] : format2;
+ }
+ void SetFormatMap(unsigned index, const NEONFormatMap* format) {
+ VIXL_ASSERT(index <= (sizeof(formats_) / sizeof(formats_[0])));
+ VIXL_ASSERT(format != NULL);
+ formats_[index] = format;
+ }
+
+ // Substitute %s in the input string with the placeholder string for each
+ // register, ie. "'B", "'H", etc.
+ const char* SubstitutePlaceholders(const char* string) {
+ return Substitute(string, kPlaceholder, kPlaceholder, kPlaceholder);
+ }
+
+ // Substitute %s in the input string with a new string based on the
+ // substitution mode.
+ const char* Substitute(const char* string,
+ SubstitutionMode mode0 = kFormat,
+ SubstitutionMode mode1 = kFormat,
+ SubstitutionMode mode2 = kFormat) {
+ snprintf(form_buffer_, sizeof(form_buffer_), string,
+ GetSubstitute(0, mode0),
+ GetSubstitute(1, mode1),
+ GetSubstitute(2, mode2));
+ return form_buffer_;
+ }
+
+ // Append a "2" to a mnemonic string based of the state of the Q bit.
+ const char* Mnemonic(const char* mnemonic) {
+ if ((instrbits_ & NEON_Q) != 0) {
+ snprintf(mne_buffer_, sizeof(mne_buffer_), "%s2", mnemonic);
+ return mne_buffer_;
+ }
+ return mnemonic;
+ }
+
+ VectorFormat GetVectorFormat(int format_index = 0) {
+ return GetVectorFormat(formats_[format_index]);
+ }
+
+ VectorFormat GetVectorFormat(const NEONFormatMap* format_map) {
+ static const VectorFormat vform[] = {
+ kFormatUndefined,
+ kFormat8B, kFormat16B, kFormat4H, kFormat8H,
+ kFormat2S, kFormat4S, kFormat1D, kFormat2D,
+ kFormatB, kFormatH, kFormatS, kFormatD
+ };
+ VIXL_ASSERT(GetNEONFormat(format_map) < (sizeof(vform) / sizeof(vform[0])));
+ return vform[GetNEONFormat(format_map)];
+ }
+
+ // Built in mappings for common cases.
+
+ // The integer format map uses three bits (Q, size<1:0>) to encode the
+ // "standard" set of NEON integer vector formats.
+ static const NEONFormatMap* IntegerFormatMap() {
+ static const NEONFormatMap map = {
+ {23, 22, 30},
+ {NF_8B, NF_16B, NF_4H, NF_8H, NF_2S, NF_4S, NF_UNDEF, NF_2D}
+ };
+ return &map;
+ }
+
+ // The long integer format map uses two bits (size<1:0>) to encode the
+ // long set of NEON integer vector formats. These are used in narrow, wide
+ // and long operations.
+ static const NEONFormatMap* LongIntegerFormatMap() {
+ static const NEONFormatMap map = {
+ {23, 22}, {NF_8H, NF_4S, NF_2D}
+ };
+ return &map;
+ }
+
+ // The FP format map uses two bits (Q, size<0>) to encode the NEON FP vector
+ // formats: NF_2S, NF_4S, NF_2D.
+ static const NEONFormatMap* FPFormatMap() {
+ // The FP format map assumes two bits (Q, size<0>) are used to encode the
+ // NEON FP vector formats: NF_2S, NF_4S, NF_2D.
+ static const NEONFormatMap map = {
+ {22, 30}, {NF_2S, NF_4S, NF_UNDEF, NF_2D}
+ };
+ return &map;
+ }
+
+ // The load/store format map uses three bits (Q, 11, 10) to encode the
+ // set of NEON vector formats.
+ static const NEONFormatMap* LoadStoreFormatMap() {
+ static const NEONFormatMap map = {
+ {11, 10, 30},
+ {NF_8B, NF_16B, NF_4H, NF_8H, NF_2S, NF_4S, NF_1D, NF_2D}
+ };
+ return &map;
+ }
+
+ // The logical format map uses one bit (Q) to encode the NEON vector format:
+ // NF_8B, NF_16B.
+ static const NEONFormatMap* LogicalFormatMap() {
+ static const NEONFormatMap map = {
+ {30}, {NF_8B, NF_16B}
+ };
+ return &map;
+ }
+
+ // The triangular format map uses between two and five bits to encode the NEON
+ // vector format:
+ // xxx10->8B, xxx11->16B, xx100->4H, xx101->8H
+ // x1000->2S, x1001->4S, 10001->2D, all others undefined.
+ static const NEONFormatMap* TriangularFormatMap() {
+ static const NEONFormatMap map = {
+ {19, 18, 17, 16, 30},
+ {NF_UNDEF, NF_UNDEF, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_2S,
+ NF_4S, NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_UNDEF, NF_2D,
+ NF_8B, NF_16B, NF_4H, NF_8H, NF_8B, NF_16B, NF_2S, NF_4S, NF_8B, NF_16B,
+ NF_4H, NF_8H, NF_8B, NF_16B}
+ };
+ return &map;
+ }
+
+ // The scalar format map uses two bits (size<1:0>) to encode the NEON scalar
+ // formats: NF_B, NF_H, NF_S, NF_D.
+ static const NEONFormatMap* ScalarFormatMap() {
+ static const NEONFormatMap map = {
+ {23, 22}, {NF_B, NF_H, NF_S, NF_D}
+ };
+ return &map;
+ }
+
+ // The long scalar format map uses two bits (size<1:0>) to encode the longer
+ // NEON scalar formats: NF_H, NF_S, NF_D.
+ static const NEONFormatMap* LongScalarFormatMap() {
+ static const NEONFormatMap map = {
+ {23, 22}, {NF_H, NF_S, NF_D}
+ };
+ return &map;
+ }
+
+ // The FP scalar format map assumes one bit (size<0>) is used to encode the
+ // NEON FP scalar formats: NF_S, NF_D.
+ static const NEONFormatMap* FPScalarFormatMap() {
+ static const NEONFormatMap map = {
+ {22}, {NF_S, NF_D}
+ };
+ return &map;
+ }
+
+ // The triangular scalar format map uses between one and four bits to encode
+ // the NEON FP scalar formats:
+ // xxx1->B, xx10->H, x100->S, 1000->D, all others undefined.
+ static const NEONFormatMap* TriangularScalarFormatMap() {
+ static const NEONFormatMap map = {
+ {19, 18, 17, 16},
+ {NF_UNDEF, NF_B, NF_H, NF_B, NF_S, NF_B, NF_H, NF_B,
+ NF_D, NF_B, NF_H, NF_B, NF_S, NF_B, NF_H, NF_B}
+ };
+ return &map;
+ }
+
+ private:
+ // Get a pointer to a string that represents the format or placeholder for
+ // the specified substitution index, based on the format map and instruction.
+ const char* GetSubstitute(int index, SubstitutionMode mode) {
+ if (mode == kFormat) {
+ return NEONFormatAsString(GetNEONFormat(formats_[index]));
+ }
+ VIXL_ASSERT(mode == kPlaceholder);
+ return NEONFormatAsPlaceholder(GetNEONFormat(formats_[index]));
+ }
+
+ // Get the NEONFormat enumerated value for bits obtained from the
+ // instruction based on the specified format mapping.
+ NEONFormat GetNEONFormat(const NEONFormatMap* format_map) {
+ return format_map->map[PickBits(format_map->bits)];
+ }
+
+ // Convert a NEONFormat into a string.
+ static const char* NEONFormatAsString(NEONFormat format) {
+ static const char* formats[] = {
+ "undefined",
+ "8b", "16b", "4h", "8h", "2s", "4s", "1d", "2d",
+ "b", "h", "s", "d"
+ };
+ VIXL_ASSERT(format < (sizeof(formats) / sizeof(formats[0])));
+ return formats[format];
+ }
+
+ // Convert a NEONFormat into a register placeholder string.
+ static const char* NEONFormatAsPlaceholder(NEONFormat format) {
+ VIXL_ASSERT((format == NF_B) || (format == NF_H) ||
+ (format == NF_S) || (format == NF_D) ||
+ (format == NF_UNDEF));
+ static const char* formats[] = {
+ "undefined",
+ "undefined", "undefined", "undefined", "undefined",
+ "undefined", "undefined", "undefined", "undefined",
+ "'B", "'H", "'S", "'D"
+ };
+ return formats[format];
+ }
+
+ // Select bits from instrbits_ defined by the bits array, concatenate them,
+ // and return the value.
+ uint8_t PickBits(const uint8_t bits[]) {
+ uint8_t result = 0;
+ for (unsigned b = 0; b < kNEONFormatMaxBits; b++) {
+ if (bits[b] == 0) break;
+ result <<= 1;
+ result |= ((instrbits_ & (1 << bits[b])) == 0) ? 0 : 1;
+ }
+ return result;
+ }
+
+ Instr instrbits_;
+ const NEONFormatMap* formats_[3];
+ char form_buffer_[64];
+ char mne_buffer_[16];
+};
+} // namespace vixl
+
+#endif // VIXL_A64_INSTRUCTIONS_A64_H_
diff --git a/js/src/jit/arm64/vixl/Instrument-vixl.cpp b/js/src/jit/arm64/vixl/Instrument-vixl.cpp
new file mode 100644
index 0000000000..5b245b6cf9
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Instrument-vixl.cpp
@@ -0,0 +1,844 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Instrument-vixl.h"
+
+namespace vixl {
+
+Counter::Counter(const char* name, CounterType type)
+ : count_(0), enabled_(false), type_(type) {
+ VIXL_ASSERT(name != NULL);
+ strncpy(name_, name, kCounterNameMaxLength);
+}
+
+
+void Counter::Enable() {
+ enabled_ = true;
+}
+
+
+void Counter::Disable() {
+ enabled_ = false;
+}
+
+
+bool Counter::IsEnabled() {
+ return enabled_;
+}
+
+
+void Counter::Increment() {
+ if (enabled_) {
+ count_++;
+ }
+}
+
+
+uint64_t Counter::count() {
+ uint64_t result = count_;
+ if (type_ == Gauge) {
+ // If the counter is a Gauge, reset the count after reading.
+ count_ = 0;
+ }
+ return result;
+}
+
+
+const char* Counter::name() {
+ return name_;
+}
+
+
+CounterType Counter::type() {
+ return type_;
+}
+
+
+struct CounterDescriptor {
+ const char* name;
+ CounterType type;
+};
+
+
+static const CounterDescriptor kCounterList[] = {
+ {"Instruction", Cumulative},
+
+ {"Move Immediate", Gauge},
+ {"Add/Sub DP", Gauge},
+ {"Logical DP", Gauge},
+ {"Other Int DP", Gauge},
+ {"FP DP", Gauge},
+
+ {"Conditional Select", Gauge},
+ {"Conditional Compare", Gauge},
+
+ {"Unconditional Branch", Gauge},
+ {"Compare and Branch", Gauge},
+ {"Test and Branch", Gauge},
+ {"Conditional Branch", Gauge},
+
+ {"Load Integer", Gauge},
+ {"Load FP", Gauge},
+ {"Load Pair", Gauge},
+ {"Load Literal", Gauge},
+
+ {"Store Integer", Gauge},
+ {"Store FP", Gauge},
+ {"Store Pair", Gauge},
+
+ {"PC Addressing", Gauge},
+ {"Other", Gauge},
+ {"NEON", Gauge},
+ {"Crypto", Gauge}
+};
+
+
+Instrument::Instrument(const char* datafile, uint64_t sample_period)
+ : output_stream_(stdout), sample_period_(sample_period) {
+
+ // Set up the output stream. If datafile is non-NULL, use that file. If it
+ // can't be opened, or datafile is NULL, use stdout.
+ if (datafile != NULL) {
+ output_stream_ = fopen(datafile, "w");
+ if (output_stream_ == NULL) {
+ printf("Can't open output file %s. Using stdout.\n", datafile);
+ output_stream_ = stdout;
+ }
+ }
+
+ static const int num_counters =
+ sizeof(kCounterList) / sizeof(CounterDescriptor);
+
+ // Dump an instrumentation description comment at the top of the file.
+ fprintf(output_stream_, "# counters=%d\n", num_counters);
+ fprintf(output_stream_, "# sample_period=%" PRIu64 "\n", sample_period_);
+
+ // Construct Counter objects from counter description array.
+ for (int i = 0; i < num_counters; i++) {
+ if (Counter* counter = js_new<Counter>(kCounterList[i].name, kCounterList[i].type))
+ (void)counters_.append(counter);
+ }
+
+ DumpCounterNames();
+}
+
+
+Instrument::~Instrument() {
+ // Dump any remaining instruction data to the output file.
+ DumpCounters();
+
+ // Free all the counter objects.
+ for (auto counter : counters_) {
+ js_delete(counter);
+ }
+
+ if (output_stream_ != stdout) {
+ fclose(output_stream_);
+ }
+}
+
+
+void Instrument::Update() {
+ // Increment the instruction counter, and dump all counters if a sample period
+ // has elapsed.
+ static Counter* counter = GetCounter("Instruction");
+ VIXL_ASSERT(counter->type() == Cumulative);
+ counter->Increment();
+
+ if (counter->IsEnabled() && (counter->count() % sample_period_) == 0) {
+ DumpCounters();
+ }
+}
+
+
+void Instrument::DumpCounters() {
+ // Iterate through the counter objects, dumping their values to the output
+ // stream.
+ for (auto counter : counters_) {
+ fprintf(output_stream_, "%" PRIu64 ",", counter->count());
+ }
+ fprintf(output_stream_, "\n");
+ fflush(output_stream_);
+}
+
+
+void Instrument::DumpCounterNames() {
+ // Iterate through the counter objects, dumping the counter names to the
+ // output stream.
+ for (auto counter : counters_) {
+ fprintf(output_stream_, "%s,", counter->name());
+ }
+ fprintf(output_stream_, "\n");
+ fflush(output_stream_);
+}
+
+
+void Instrument::HandleInstrumentationEvent(unsigned event) {
+ switch (event) {
+ case InstrumentStateEnable: Enable(); break;
+ case InstrumentStateDisable: Disable(); break;
+ default: DumpEventMarker(event);
+ }
+}
+
+
+void Instrument::DumpEventMarker(unsigned marker) {
+ // Dumpan event marker to the output stream as a specially formatted comment
+ // line.
+ static Counter* counter = GetCounter("Instruction");
+
+ fprintf(output_stream_, "# %c%c @ %" PRId64 "\n", marker & 0xff,
+ (marker >> 8) & 0xff, counter->count());
+}
+
+
+Counter* Instrument::GetCounter(const char* name) {
+ // Get a Counter object by name from the counter list.
+ for (auto counter : counters_) {
+ if (strcmp(counter->name(), name) == 0) {
+ return counter;
+ }
+ }
+
+ // A Counter by that name does not exist: print an error message to stderr
+ // and the output file, and exit.
+ static const char* error_message =
+ "# Error: Unknown counter \"%s\". Exiting.\n";
+ fprintf(stderr, error_message, name);
+ fprintf(output_stream_, error_message, name);
+ exit(1);
+}
+
+
+void Instrument::Enable() {
+ for (auto counter : counters_) {
+ counter->Enable();
+ }
+}
+
+
+void Instrument::Disable() {
+ for (auto counter : counters_) {
+ counter->Disable();
+ }
+}
+
+
+void Instrument::VisitPCRelAddressing(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("PC Addressing");
+ counter->Increment();
+}
+
+
+void Instrument::VisitAddSubImmediate(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Add/Sub DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitLogicalImmediate(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Logical DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitMoveWideImmediate(const Instruction* instr) {
+ Update();
+ static Counter* counter = GetCounter("Move Immediate");
+
+ if (instr->IsMovn() && (instr->Rd() == kZeroRegCode)) {
+ unsigned imm = instr->ImmMoveWide();
+ HandleInstrumentationEvent(imm);
+ } else {
+ counter->Increment();
+ }
+}
+
+
+void Instrument::VisitBitfield(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other Int DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitExtract(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other Int DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitUnconditionalBranch(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Unconditional Branch");
+ counter->Increment();
+}
+
+
+void Instrument::VisitUnconditionalBranchToRegister(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Unconditional Branch");
+ counter->Increment();
+}
+
+
+void Instrument::VisitCompareBranch(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Compare and Branch");
+ counter->Increment();
+}
+
+
+void Instrument::VisitTestBranch(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Test and Branch");
+ counter->Increment();
+}
+
+
+void Instrument::VisitConditionalBranch(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Conditional Branch");
+ counter->Increment();
+}
+
+
+void Instrument::VisitSystem(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other");
+ counter->Increment();
+}
+
+
+void Instrument::VisitException(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other");
+ counter->Increment();
+}
+
+
+void Instrument::InstrumentLoadStorePair(const Instruction* instr) {
+ static Counter* load_pair_counter = GetCounter("Load Pair");
+ static Counter* store_pair_counter = GetCounter("Store Pair");
+
+ if (instr->Mask(LoadStorePairLBit) != 0) {
+ load_pair_counter->Increment();
+ } else {
+ store_pair_counter->Increment();
+ }
+}
+
+
+void Instrument::VisitLoadStorePairPostIndex(const Instruction* instr) {
+ Update();
+ InstrumentLoadStorePair(instr);
+}
+
+
+void Instrument::VisitLoadStorePairOffset(const Instruction* instr) {
+ Update();
+ InstrumentLoadStorePair(instr);
+}
+
+
+void Instrument::VisitLoadStorePairPreIndex(const Instruction* instr) {
+ Update();
+ InstrumentLoadStorePair(instr);
+}
+
+
+void Instrument::VisitLoadStorePairNonTemporal(const Instruction* instr) {
+ Update();
+ InstrumentLoadStorePair(instr);
+}
+
+
+void Instrument::VisitLoadStoreExclusive(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other");
+ counter->Increment();
+}
+
+
+void Instrument::VisitLoadLiteral(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Load Literal");
+ counter->Increment();
+}
+
+
+void Instrument::InstrumentLoadStore(const Instruction* instr) {
+ static Counter* load_int_counter = GetCounter("Load Integer");
+ static Counter* store_int_counter = GetCounter("Store Integer");
+ static Counter* load_fp_counter = GetCounter("Load FP");
+ static Counter* store_fp_counter = GetCounter("Store FP");
+
+ switch (instr->Mask(LoadStoreMask)) {
+ case STRB_w:
+ case STRH_w:
+ case STR_w:
+ VIXL_FALLTHROUGH();
+ case STR_x: store_int_counter->Increment(); break;
+ case STR_s:
+ VIXL_FALLTHROUGH();
+ case STR_d: store_fp_counter->Increment(); break;
+ case LDRB_w:
+ case LDRH_w:
+ case LDR_w:
+ case LDR_x:
+ case LDRSB_x:
+ case LDRSH_x:
+ case LDRSW_x:
+ case LDRSB_w:
+ VIXL_FALLTHROUGH();
+ case LDRSH_w: load_int_counter->Increment(); break;
+ case LDR_s:
+ VIXL_FALLTHROUGH();
+ case LDR_d: load_fp_counter->Increment(); break;
+ }
+}
+
+
+void Instrument::VisitLoadStoreUnscaledOffset(const Instruction* instr) {
+ Update();
+ InstrumentLoadStore(instr);
+}
+
+
+void Instrument::VisitLoadStorePostIndex(const Instruction* instr) {
+ USE(instr);
+ Update();
+ InstrumentLoadStore(instr);
+}
+
+
+void Instrument::VisitLoadStorePreIndex(const Instruction* instr) {
+ Update();
+ InstrumentLoadStore(instr);
+}
+
+
+void Instrument::VisitLoadStoreRegisterOffset(const Instruction* instr) {
+ Update();
+ InstrumentLoadStore(instr);
+}
+
+
+void Instrument::VisitLoadStoreUnsignedOffset(const Instruction* instr) {
+ Update();
+ InstrumentLoadStore(instr);
+}
+
+
+void Instrument::VisitLogicalShifted(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Logical DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitAddSubShifted(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Add/Sub DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitAddSubExtended(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Add/Sub DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitAddSubWithCarry(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Add/Sub DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitConditionalCompareRegister(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Conditional Compare");
+ counter->Increment();
+}
+
+
+void Instrument::VisitConditionalCompareImmediate(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Conditional Compare");
+ counter->Increment();
+}
+
+
+void Instrument::VisitConditionalSelect(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Conditional Select");
+ counter->Increment();
+}
+
+
+void Instrument::VisitDataProcessing1Source(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other Int DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitDataProcessing2Source(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other Int DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitDataProcessing3Source(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other Int DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPCompare(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("FP DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPConditionalCompare(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Conditional Compare");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPConditionalSelect(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Conditional Select");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPImmediate(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("FP DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPDataProcessing1Source(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("FP DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPDataProcessing2Source(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("FP DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPDataProcessing3Source(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("FP DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPIntegerConvert(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("FP DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitFPFixedPointConvert(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("FP DP");
+ counter->Increment();
+}
+
+
+void Instrument::VisitCrypto2RegSHA(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Crypto");
+ counter->Increment();
+}
+
+
+void Instrument::VisitCrypto3RegSHA(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Crypto");
+ counter->Increment();
+}
+
+
+void Instrument::VisitCryptoAES(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Crypto");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEON2RegMisc(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEON3Same(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEON3Different(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONAcrossLanes(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONByIndexedElement(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONCopy(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONExtract(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONLoadStoreMultiStruct(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONLoadStoreMultiStructPostIndex(
+ const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONLoadStoreSingleStruct(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONLoadStoreSingleStructPostIndex(
+ const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONModifiedImmediate(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONScalar2RegMisc(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONScalar3Diff(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONScalar3Same(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONScalarByIndexedElement(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONScalarCopy(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONScalarPairwise(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONScalarShiftImmediate(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONShiftImmediate(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONTable(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitNEONPerm(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("NEON");
+ counter->Increment();
+}
+
+
+void Instrument::VisitUnallocated(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other");
+ counter->Increment();
+}
+
+
+void Instrument::VisitUnimplemented(const Instruction* instr) {
+ USE(instr);
+ Update();
+ static Counter* counter = GetCounter("Other");
+ counter->Increment();
+}
+
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/Instrument-vixl.h b/js/src/jit/arm64/vixl/Instrument-vixl.h
new file mode 100644
index 0000000000..eca076d234
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Instrument-vixl.h
@@ -0,0 +1,109 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_INSTRUMENT_A64_H_
+#define VIXL_A64_INSTRUMENT_A64_H_
+
+#include "mozilla/Vector.h"
+
+#include "jit/arm64/vixl/Constants-vixl.h"
+#include "jit/arm64/vixl/Decoder-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+#include "js/AllocPolicy.h"
+
+namespace vixl {
+
+const int kCounterNameMaxLength = 256;
+const uint64_t kDefaultInstrumentationSamplingPeriod = 1 << 22;
+
+
+enum InstrumentState {
+ InstrumentStateDisable = 0,
+ InstrumentStateEnable = 1
+};
+
+
+enum CounterType {
+ Gauge = 0, // Gauge counters reset themselves after reading.
+ Cumulative = 1 // Cumulative counters keep their value after reading.
+};
+
+
+class Counter {
+ public:
+ explicit Counter(const char* name, CounterType type = Gauge);
+
+ void Increment();
+ void Enable();
+ void Disable();
+ bool IsEnabled();
+ uint64_t count();
+ const char* name();
+ CounterType type();
+
+ private:
+ char name_[kCounterNameMaxLength];
+ uint64_t count_;
+ bool enabled_;
+ CounterType type_;
+};
+
+
+class Instrument: public DecoderVisitor {
+ public:
+ explicit Instrument(const char* datafile = NULL,
+ uint64_t sample_period = kDefaultInstrumentationSamplingPeriod);
+ ~Instrument();
+
+ void Enable();
+ void Disable();
+
+ // Declare all Visitor functions.
+ #define DECLARE(A) void Visit##A(const Instruction* instr) override;
+ VISITOR_LIST(DECLARE)
+ #undef DECLARE
+
+ private:
+ void Update();
+ void DumpCounters();
+ void DumpCounterNames();
+ void DumpEventMarker(unsigned marker);
+ void HandleInstrumentationEvent(unsigned event);
+ Counter* GetCounter(const char* name);
+
+ void InstrumentLoadStore(const Instruction* instr);
+ void InstrumentLoadStorePair(const Instruction* instr);
+
+ mozilla::Vector<Counter*, 8, js::SystemAllocPolicy> counters_;
+
+ FILE *output_stream_;
+ uint64_t sample_period_;
+};
+
+} // namespace vixl
+
+#endif // VIXL_A64_INSTRUMENT_A64_H_
diff --git a/js/src/jit/arm64/vixl/Logic-vixl.cpp b/js/src/jit/arm64/vixl/Logic-vixl.cpp
new file mode 100644
index 0000000000..71821a333f
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Logic-vixl.cpp
@@ -0,0 +1,4738 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifdef JS_SIMULATOR_ARM64
+
+#include <cmath>
+
+#include "jit/arm64/vixl/Simulator-vixl.h"
+
+namespace vixl {
+
+template<> double Simulator::FPDefaultNaN<double>() {
+ return kFP64DefaultNaN;
+}
+
+
+template<> float Simulator::FPDefaultNaN<float>() {
+ return kFP32DefaultNaN;
+}
+
+
+double Simulator::FixedToDouble(int64_t src, int fbits, FPRounding round) {
+ if (src >= 0) {
+ return UFixedToDouble(src, fbits, round);
+ } else {
+ // This works for all negative values, including INT64_MIN.
+ return -UFixedToDouble(-src, fbits, round);
+ }
+}
+
+
+double Simulator::UFixedToDouble(uint64_t src, int fbits, FPRounding round) {
+ // An input of 0 is a special case because the result is effectively
+ // subnormal: The exponent is encoded as 0 and there is no implicit 1 bit.
+ if (src == 0) {
+ return 0.0;
+ }
+
+ // Calculate the exponent. The highest significant bit will have the value
+ // 2^exponent.
+ const int highest_significant_bit = 63 - CountLeadingZeros(src);
+ const int64_t exponent = highest_significant_bit - fbits;
+
+ return FPRoundToDouble(0, exponent, src, round);
+}
+
+
+float Simulator::FixedToFloat(int64_t src, int fbits, FPRounding round) {
+ if (src >= 0) {
+ return UFixedToFloat(src, fbits, round);
+ } else {
+ // This works for all negative values, including INT64_MIN.
+ return -UFixedToFloat(-src, fbits, round);
+ }
+}
+
+
+float Simulator::UFixedToFloat(uint64_t src, int fbits, FPRounding round) {
+ // An input of 0 is a special case because the result is effectively
+ // subnormal: The exponent is encoded as 0 and there is no implicit 1 bit.
+ if (src == 0) {
+ return 0.0f;
+ }
+
+ // Calculate the exponent. The highest significant bit will have the value
+ // 2^exponent.
+ const int highest_significant_bit = 63 - CountLeadingZeros(src);
+ const int32_t exponent = highest_significant_bit - fbits;
+
+ return FPRoundToFloat(0, exponent, src, round);
+}
+
+
+void Simulator::ld1(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, 16))
+ return;
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.ReadUintFromMem(vform, i, addr);
+ addr += LaneSizeInBytesFromFormat(vform);
+ }
+}
+
+
+void Simulator::ld1(VectorFormat vform,
+ LogicVRegister dst,
+ int index,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)))
+ return;
+ dst.ReadUintFromMem(vform, index, addr);
+}
+
+
+void Simulator::ld1r(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)))
+ return;
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.ReadUintFromMem(vform, i, addr);
+ }
+}
+
+
+void Simulator::ld2(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ uint64_t addr1) {
+ if (handle_wasm_seg_fault(addr1, 16*2))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ int esize = LaneSizeInBytesFromFormat(vform);
+ uint64_t addr2 = addr1 + esize;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst1.ReadUintFromMem(vform, i, addr1);
+ dst2.ReadUintFromMem(vform, i, addr2);
+ addr1 += 2 * esize;
+ addr2 += 2 * esize;
+ }
+}
+
+
+void Simulator::ld2(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ int index,
+ uint64_t addr1) {
+ if (handle_wasm_seg_fault(addr1, LaneSizeInBytesFromFormat(vform)*2))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
+ dst1.ReadUintFromMem(vform, index, addr1);
+ dst2.ReadUintFromMem(vform, index, addr2);
+}
+
+
+void Simulator::ld2r(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)*2))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst1.ReadUintFromMem(vform, i, addr);
+ dst2.ReadUintFromMem(vform, i, addr2);
+ }
+}
+
+
+void Simulator::ld3(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ uint64_t addr1) {
+ if (handle_wasm_seg_fault(addr1, 16*3))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ dst3.ClearForWrite(vform);
+ int esize = LaneSizeInBytesFromFormat(vform);
+ uint64_t addr2 = addr1 + esize;
+ uint64_t addr3 = addr2 + esize;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst1.ReadUintFromMem(vform, i, addr1);
+ dst2.ReadUintFromMem(vform, i, addr2);
+ dst3.ReadUintFromMem(vform, i, addr3);
+ addr1 += 3 * esize;
+ addr2 += 3 * esize;
+ addr3 += 3 * esize;
+ }
+}
+
+
+void Simulator::ld3(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ int index,
+ uint64_t addr1) {
+ if (handle_wasm_seg_fault(addr1, LaneSizeInBytesFromFormat(vform)*3))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ dst3.ClearForWrite(vform);
+ uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
+ uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
+ dst1.ReadUintFromMem(vform, index, addr1);
+ dst2.ReadUintFromMem(vform, index, addr2);
+ dst3.ReadUintFromMem(vform, index, addr3);
+}
+
+
+void Simulator::ld3r(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)*3))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ dst3.ClearForWrite(vform);
+ uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
+ uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst1.ReadUintFromMem(vform, i, addr);
+ dst2.ReadUintFromMem(vform, i, addr2);
+ dst3.ReadUintFromMem(vform, i, addr3);
+ }
+}
+
+
+void Simulator::ld4(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ LogicVRegister dst4,
+ uint64_t addr1) {
+ if (handle_wasm_seg_fault(addr1, 16*4))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ dst3.ClearForWrite(vform);
+ dst4.ClearForWrite(vform);
+ int esize = LaneSizeInBytesFromFormat(vform);
+ uint64_t addr2 = addr1 + esize;
+ uint64_t addr3 = addr2 + esize;
+ uint64_t addr4 = addr3 + esize;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst1.ReadUintFromMem(vform, i, addr1);
+ dst2.ReadUintFromMem(vform, i, addr2);
+ dst3.ReadUintFromMem(vform, i, addr3);
+ dst4.ReadUintFromMem(vform, i, addr4);
+ addr1 += 4 * esize;
+ addr2 += 4 * esize;
+ addr3 += 4 * esize;
+ addr4 += 4 * esize;
+ }
+}
+
+
+void Simulator::ld4(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ LogicVRegister dst4,
+ int index,
+ uint64_t addr1) {
+ if (handle_wasm_seg_fault(addr1, LaneSizeInBytesFromFormat(vform)*4))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ dst3.ClearForWrite(vform);
+ dst4.ClearForWrite(vform);
+ uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
+ uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
+ uint64_t addr4 = addr3 + LaneSizeInBytesFromFormat(vform);
+ dst1.ReadUintFromMem(vform, index, addr1);
+ dst2.ReadUintFromMem(vform, index, addr2);
+ dst3.ReadUintFromMem(vform, index, addr3);
+ dst4.ReadUintFromMem(vform, index, addr4);
+}
+
+
+void Simulator::ld4r(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ LogicVRegister dst4,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)*4))
+ return;
+ dst1.ClearForWrite(vform);
+ dst2.ClearForWrite(vform);
+ dst3.ClearForWrite(vform);
+ dst4.ClearForWrite(vform);
+ uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
+ uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
+ uint64_t addr4 = addr3 + LaneSizeInBytesFromFormat(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst1.ReadUintFromMem(vform, i, addr);
+ dst2.ReadUintFromMem(vform, i, addr2);
+ dst3.ReadUintFromMem(vform, i, addr3);
+ dst4.ReadUintFromMem(vform, i, addr4);
+ }
+}
+
+
+void Simulator::st1(VectorFormat vform,
+ LogicVRegister src,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, 16))
+ return;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ src.WriteUintToMem(vform, i, addr);
+ addr += LaneSizeInBytesFromFormat(vform);
+ }
+}
+
+
+void Simulator::st1(VectorFormat vform,
+ LogicVRegister src,
+ int index,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)))
+ return;
+ src.WriteUintToMem(vform, index, addr);
+}
+
+
+void Simulator::st2(VectorFormat vform,
+ LogicVRegister dst,
+ LogicVRegister dst2,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, 16*2))
+ return;
+ int esize = LaneSizeInBytesFromFormat(vform);
+ uint64_t addr2 = addr + esize;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.WriteUintToMem(vform, i, addr);
+ dst2.WriteUintToMem(vform, i, addr2);
+ addr += 2 * esize;
+ addr2 += 2 * esize;
+ }
+}
+
+
+void Simulator::st2(VectorFormat vform,
+ LogicVRegister dst,
+ LogicVRegister dst2,
+ int index,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)*2))
+ return;
+ int esize = LaneSizeInBytesFromFormat(vform);
+ dst.WriteUintToMem(vform, index, addr);
+ dst2.WriteUintToMem(vform, index, addr + 1 * esize);
+}
+
+
+void Simulator::st3(VectorFormat vform,
+ LogicVRegister dst,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, 16*3))
+ return;
+ int esize = LaneSizeInBytesFromFormat(vform);
+ uint64_t addr2 = addr + esize;
+ uint64_t addr3 = addr2 + esize;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.WriteUintToMem(vform, i, addr);
+ dst2.WriteUintToMem(vform, i, addr2);
+ dst3.WriteUintToMem(vform, i, addr3);
+ addr += 3 * esize;
+ addr2 += 3 * esize;
+ addr3 += 3 * esize;
+ }
+}
+
+
+void Simulator::st3(VectorFormat vform,
+ LogicVRegister dst,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ int index,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)*3))
+ return;
+ int esize = LaneSizeInBytesFromFormat(vform);
+ dst.WriteUintToMem(vform, index, addr);
+ dst2.WriteUintToMem(vform, index, addr + 1 * esize);
+ dst3.WriteUintToMem(vform, index, addr + 2 * esize);
+}
+
+
+void Simulator::st4(VectorFormat vform,
+ LogicVRegister dst,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ LogicVRegister dst4,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, 16*4))
+ return;
+ int esize = LaneSizeInBytesFromFormat(vform);
+ uint64_t addr2 = addr + esize;
+ uint64_t addr3 = addr2 + esize;
+ uint64_t addr4 = addr3 + esize;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.WriteUintToMem(vform, i, addr);
+ dst2.WriteUintToMem(vform, i, addr2);
+ dst3.WriteUintToMem(vform, i, addr3);
+ dst4.WriteUintToMem(vform, i, addr4);
+ addr += 4 * esize;
+ addr2 += 4 * esize;
+ addr3 += 4 * esize;
+ addr4 += 4 * esize;
+ }
+}
+
+
+void Simulator::st4(VectorFormat vform,
+ LogicVRegister dst,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ LogicVRegister dst4,
+ int index,
+ uint64_t addr) {
+ if (handle_wasm_seg_fault(addr, LaneSizeInBytesFromFormat(vform)*4))
+ return;
+ int esize = LaneSizeInBytesFromFormat(vform);
+ dst.WriteUintToMem(vform, index, addr);
+ dst2.WriteUintToMem(vform, index, addr + 1 * esize);
+ dst3.WriteUintToMem(vform, index, addr + 2 * esize);
+ dst4.WriteUintToMem(vform, index, addr + 3 * esize);
+}
+
+
+LogicVRegister Simulator::cmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ Condition cond) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ int64_t sa = src1.Int(vform, i);
+ int64_t sb = src2.Int(vform, i);
+ uint64_t ua = src1.Uint(vform, i);
+ uint64_t ub = src2.Uint(vform, i);
+ bool result = false;
+ switch (cond) {
+ case eq: result = (ua == ub); break;
+ case ge: result = (sa >= sb); break;
+ case gt: result = (sa > sb) ; break;
+ case hi: result = (ua > ub) ; break;
+ case hs: result = (ua >= ub); break;
+ case lt: result = (sa < sb) ; break;
+ case le: result = (sa <= sb); break;
+ default: VIXL_UNREACHABLE(); break;
+ }
+ dst.SetUint(vform, i, result ? MaxUintFromFormat(vform) : 0);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::cmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ int imm,
+ Condition cond) {
+ SimVRegister temp;
+ LogicVRegister imm_reg = dup_immediate(vform, temp, imm);
+ return cmp(vform, dst, src1, imm_reg, cond);
+}
+
+
+LogicVRegister Simulator::cmptst(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t ua = src1.Uint(vform, i);
+ uint64_t ub = src2.Uint(vform, i);
+ dst.SetUint(vform, i, ((ua & ub) != 0) ? MaxUintFromFormat(vform) : 0);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::add(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ // TODO(all): consider assigning the result of LaneCountFromFormat to a local.
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ // Test for unsigned saturation.
+ uint64_t ua = src1.UintLeftJustified(vform, i);
+ uint64_t ub = src2.UintLeftJustified(vform, i);
+ uint64_t ur = ua + ub;
+ if (ur < ua) {
+ dst.SetUnsignedSat(i, true);
+ }
+
+ // Test for signed saturation.
+ int64_t sa = src1.IntLeftJustified(vform, i);
+ int64_t sb = src2.IntLeftJustified(vform, i);
+ int64_t sr = sa + sb;
+ // If the signs of the operands are the same, but different from the result,
+ // there was an overflow.
+ if (((sa >= 0) == (sb >= 0)) && ((sa >= 0) != (sr >= 0))) {
+ dst.SetSignedSat(i, sa >= 0);
+ }
+
+ dst.SetInt(vform, i, src1.Int(vform, i) + src2.Int(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::addp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uzp1(vform, temp1, src1, src2);
+ uzp2(vform, temp2, src1, src2);
+ add(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::mla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ mul(vform, temp, src1, src2);
+ add(vform, dst, dst, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::mls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ mul(vform, temp, src1, src2);
+ sub(vform, dst, dst, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::mul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, src1.Uint(vform, i) * src2.Uint(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::mul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform = VectorFormatFillQ(vform);
+ return mul(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::mla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform = VectorFormatFillQ(vform);
+ return mla(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::mls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform = VectorFormatFillQ(vform);
+ return mls(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::smull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return smull(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::smull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return smull2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::umull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return umull(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::umull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return umull2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::smlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return smlal(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::smlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return smlal2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::umlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return umlal(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::umlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return umlal2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::smlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return smlsl(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::smlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return smlsl2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::umlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return umlsl(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::umlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return umlsl2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::sqdmull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return sqdmull(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::sqdmull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return sqdmull2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::sqdmlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return sqdmlal(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::sqdmlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return sqdmlal2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::sqdmlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return sqdmlsl(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::sqdmlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform =
+ VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
+ return sqdmlsl2(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::sqdmulh(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform = VectorFormatFillQ(vform);
+ return sqdmulh(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+LogicVRegister Simulator::sqrdmulh(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ SimVRegister temp;
+ VectorFormat indexform = VectorFormatFillQ(vform);
+ return sqrdmulh(vform, dst, src1, dup_element(indexform, temp, src2, index));
+}
+
+
+uint16_t Simulator::PolynomialMult(uint8_t op1, uint8_t op2) {
+ uint16_t result = 0;
+ uint16_t extended_op2 = op2;
+ for (int i = 0; i < 8; ++i) {
+ if ((op1 >> i) & 1) {
+ result = result ^ (extended_op2 << i);
+ }
+ }
+ return result;
+}
+
+
+LogicVRegister Simulator::pmul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i,
+ PolynomialMult(src1.Uint(vform, i), src2.Uint(vform, i)));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::pmull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ VectorFormat vform_src = VectorFormatHalfWidth(vform);
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, PolynomialMult(src1.Uint(vform_src, i),
+ src2.Uint(vform_src, i)));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::pmull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ VectorFormat vform_src = VectorFormatHalfWidthDoubleLanes(vform);
+ dst.ClearForWrite(vform);
+ int lane_count = LaneCountFromFormat(vform);
+ for (int i = 0; i < lane_count; i++) {
+ dst.SetUint(vform, i, PolynomialMult(src1.Uint(vform_src, lane_count + i),
+ src2.Uint(vform_src, lane_count + i)));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::sub(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ // Test for unsigned saturation.
+ if (src2.Uint(vform, i) > src1.Uint(vform, i)) {
+ dst.SetUnsignedSat(i, false);
+ }
+
+ // Test for signed saturation.
+ int64_t sa = src1.IntLeftJustified(vform, i);
+ int64_t sb = src2.IntLeftJustified(vform, i);
+ int64_t sr = sa - sb;
+ // If the signs of the operands are different, and the sign of the first
+ // operand doesn't match the result, there was an overflow.
+ if (((sa >= 0) != (sb >= 0)) && ((sa >= 0) != (sr >= 0))) {
+ dst.SetSignedSat(i, sr < 0);
+ }
+
+ dst.SetInt(vform, i, src1.Int(vform, i) - src2.Int(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::and_(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, src1.Uint(vform, i) & src2.Uint(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::orr(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, src1.Uint(vform, i) | src2.Uint(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::orn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, src1.Uint(vform, i) | ~src2.Uint(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::eor(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, src1.Uint(vform, i) ^ src2.Uint(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::bic(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, src1.Uint(vform, i) & ~src2.Uint(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::bic(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ uint64_t imm) {
+ uint64_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ result[i] = src.Uint(vform, i) & ~imm;
+ }
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::bif(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t operand1 = dst.Uint(vform, i);
+ uint64_t operand2 = ~src2.Uint(vform, i);
+ uint64_t operand3 = src1.Uint(vform, i);
+ uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
+ dst.SetUint(vform, i, result);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::bit(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t operand1 = dst.Uint(vform, i);
+ uint64_t operand2 = src2.Uint(vform, i);
+ uint64_t operand3 = src1.Uint(vform, i);
+ uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
+ dst.SetUint(vform, i, result);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::bsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t operand1 = src2.Uint(vform, i);
+ uint64_t operand2 = dst.Uint(vform, i);
+ uint64_t operand3 = src1.Uint(vform, i);
+ uint64_t result = operand1 ^ ((operand1 ^ operand3) & operand2);
+ dst.SetUint(vform, i, result);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::sminmax(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ bool max) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ int64_t src1_val = src1.Int(vform, i);
+ int64_t src2_val = src2.Int(vform, i);
+ int64_t dst_val;
+ if (max == true) {
+ dst_val = (src1_val > src2_val) ? src1_val : src2_val;
+ } else {
+ dst_val = (src1_val < src2_val) ? src1_val : src2_val;
+ }
+ dst.SetInt(vform, i, dst_val);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::smax(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ return sminmax(vform, dst, src1, src2, true);
+}
+
+
+LogicVRegister Simulator::smin(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ return sminmax(vform, dst, src1, src2, false);
+}
+
+
+LogicVRegister Simulator::sminmaxp(VectorFormat vform,
+ LogicVRegister dst,
+ int dst_index,
+ const LogicVRegister& src,
+ bool max) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i += 2) {
+ int64_t src1_val = src.Int(vform, i);
+ int64_t src2_val = src.Int(vform, i + 1);
+ int64_t dst_val;
+ if (max == true) {
+ dst_val = (src1_val > src2_val) ? src1_val : src2_val;
+ } else {
+ dst_val = (src1_val < src2_val) ? src1_val : src2_val;
+ }
+ dst.SetInt(vform, dst_index + (i >> 1), dst_val);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::smaxp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ sminmaxp(vform, dst, 0, src1, true);
+ sminmaxp(vform, dst, LaneCountFromFormat(vform) >> 1, src2, true);
+ return dst;
+}
+
+
+LogicVRegister Simulator::sminp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ sminmaxp(vform, dst, 0, src1, false);
+ sminmaxp(vform, dst, LaneCountFromFormat(vform) >> 1, src2, false);
+ return dst;
+}
+
+
+LogicVRegister Simulator::addp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VIXL_ASSERT(vform == kFormatD);
+
+ int64_t dst_val = src.Int(kFormat2D, 0) + src.Int(kFormat2D, 1);
+ dst.ClearForWrite(vform);
+ dst.SetInt(vform, 0, dst_val);
+ return dst;
+}
+
+
+LogicVRegister Simulator::addv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VectorFormat vform_dst
+ = ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
+
+
+ int64_t dst_val = 0;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst_val += src.Int(vform, i);
+ }
+
+ dst.ClearForWrite(vform_dst);
+ dst.SetInt(vform_dst, 0, dst_val);
+ return dst;
+}
+
+
+LogicVRegister Simulator::saddlv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VectorFormat vform_dst
+ = ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform) * 2);
+
+ int64_t dst_val = 0;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst_val += src.Int(vform, i);
+ }
+
+ dst.ClearForWrite(vform_dst);
+ dst.SetInt(vform_dst, 0, dst_val);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uaddlv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VectorFormat vform_dst
+ = ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform) * 2);
+
+ uint64_t dst_val = 0;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst_val += src.Uint(vform, i);
+ }
+
+ dst.ClearForWrite(vform_dst);
+ dst.SetUint(vform_dst, 0, dst_val);
+ return dst;
+}
+
+
+LogicVRegister Simulator::sminmaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ bool max) {
+ dst.ClearForWrite(vform);
+ int64_t dst_val = max ? INT64_MIN : INT64_MAX;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ int64_t src_val = src.Int(vform, i);
+ if (max == true) {
+ dst_val = (src_val > dst_val) ? src_val : dst_val;
+ } else {
+ dst_val = (src_val < dst_val) ? src_val : dst_val;
+ }
+ }
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetInt(vform, i, 0);
+ }
+ dst.SetInt(vform, 0, dst_val);
+ return dst;
+}
+
+
+LogicVRegister Simulator::smaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ sminmaxv(vform, dst, src, true);
+ return dst;
+}
+
+
+LogicVRegister Simulator::sminv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ sminmaxv(vform, dst, src, false);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uminmax(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ bool max) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t src1_val = src1.Uint(vform, i);
+ uint64_t src2_val = src2.Uint(vform, i);
+ uint64_t dst_val;
+ if (max == true) {
+ dst_val = (src1_val > src2_val) ? src1_val : src2_val;
+ } else {
+ dst_val = (src1_val < src2_val) ? src1_val : src2_val;
+ }
+ dst.SetUint(vform, i, dst_val);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::umax(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ return uminmax(vform, dst, src1, src2, true);
+}
+
+
+LogicVRegister Simulator::umin(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ return uminmax(vform, dst, src1, src2, false);
+}
+
+
+LogicVRegister Simulator::uminmaxp(VectorFormat vform,
+ LogicVRegister dst,
+ int dst_index,
+ const LogicVRegister& src,
+ bool max) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i += 2) {
+ uint64_t src1_val = src.Uint(vform, i);
+ uint64_t src2_val = src.Uint(vform, i + 1);
+ uint64_t dst_val;
+ if (max == true) {
+ dst_val = (src1_val > src2_val) ? src1_val : src2_val;
+ } else {
+ dst_val = (src1_val < src2_val) ? src1_val : src2_val;
+ }
+ dst.SetUint(vform, dst_index + (i >> 1), dst_val);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::umaxp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ uminmaxp(vform, dst, 0, src1, true);
+ uminmaxp(vform, dst, LaneCountFromFormat(vform) >> 1, src2, true);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uminp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ uminmaxp(vform, dst, 0, src1, false);
+ uminmaxp(vform, dst, LaneCountFromFormat(vform) >> 1, src2, false);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uminmaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ bool max) {
+ dst.ClearForWrite(vform);
+ uint64_t dst_val = max ? 0 : UINT64_MAX;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t src_val = src.Uint(vform, i);
+ if (max == true) {
+ dst_val = (src_val > dst_val) ? src_val : dst_val;
+ } else {
+ dst_val = (src_val < dst_val) ? src_val : dst_val;
+ }
+ }
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, 0);
+ }
+ dst.SetUint(vform, 0, dst_val);
+ return dst;
+}
+
+
+LogicVRegister Simulator::umaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ uminmaxv(vform, dst, src, true);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uminv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ uminmaxv(vform, dst, src, false);
+ return dst;
+}
+
+
+LogicVRegister Simulator::shl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp;
+ LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
+ return ushl(vform, dst, src, shiftreg);
+}
+
+
+LogicVRegister Simulator::sshll(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp1, temp2;
+ LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
+ LogicVRegister extendedreg = sxtl(vform, temp2, src);
+ return sshl(vform, dst, extendedreg, shiftreg);
+}
+
+
+LogicVRegister Simulator::sshll2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp1, temp2;
+ LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
+ LogicVRegister extendedreg = sxtl2(vform, temp2, src);
+ return sshl(vform, dst, extendedreg, shiftreg);
+}
+
+
+LogicVRegister Simulator::shll(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ int shift = LaneSizeInBitsFromFormat(vform) / 2;
+ return sshll(vform, dst, src, shift);
+}
+
+
+LogicVRegister Simulator::shll2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ int shift = LaneSizeInBitsFromFormat(vform) / 2;
+ return sshll2(vform, dst, src, shift);
+}
+
+
+LogicVRegister Simulator::ushll(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp1, temp2;
+ LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
+ LogicVRegister extendedreg = uxtl(vform, temp2, src);
+ return ushl(vform, dst, extendedreg, shiftreg);
+}
+
+
+LogicVRegister Simulator::ushll2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp1, temp2;
+ LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
+ LogicVRegister extendedreg = uxtl2(vform, temp2, src);
+ return ushl(vform, dst, extendedreg, shiftreg);
+}
+
+
+LogicVRegister Simulator::sli(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ dst.ClearForWrite(vform);
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; i++) {
+ uint64_t src_lane = src.Uint(vform, i);
+ uint64_t dst_lane = dst.Uint(vform, i);
+ uint64_t shifted = src_lane << shift;
+ uint64_t mask = MaxUintFromFormat(vform) << shift;
+ dst.SetUint(vform, i, (dst_lane & ~mask) | shifted);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::sqshl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp;
+ LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
+ return sshl(vform, dst, src, shiftreg).SignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::uqshl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp;
+ LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
+ return ushl(vform, dst, src, shiftreg).UnsignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqshlu(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp;
+ LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
+ return sshl(vform, dst, src, shiftreg).UnsignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sri(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ dst.ClearForWrite(vform);
+ int laneCount = LaneCountFromFormat(vform);
+ VIXL_ASSERT((shift > 0) &&
+ (shift <= static_cast<int>(LaneSizeInBitsFromFormat(vform))));
+ for (int i = 0; i < laneCount; i++) {
+ uint64_t src_lane = src.Uint(vform, i);
+ uint64_t dst_lane = dst.Uint(vform, i);
+ uint64_t shifted;
+ uint64_t mask;
+ if (shift == 64) {
+ shifted = 0;
+ mask = 0;
+ } else {
+ shifted = src_lane >> shift;
+ mask = MaxUintFromFormat(vform) >> shift;
+ }
+ dst.SetUint(vform, i, (dst_lane & ~mask) | shifted);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::ushr(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp;
+ LogicVRegister shiftreg = dup_immediate(vform, temp, -shift);
+ return ushl(vform, dst, src, shiftreg);
+}
+
+
+LogicVRegister Simulator::sshr(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ VIXL_ASSERT(shift >= 0);
+ SimVRegister temp;
+ LogicVRegister shiftreg = dup_immediate(vform, temp, -shift);
+ return sshl(vform, dst, src, shiftreg);
+}
+
+
+LogicVRegister Simulator::ssra(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ LogicVRegister shifted_reg = sshr(vform, temp, src, shift);
+ return add(vform, dst, dst, shifted_reg);
+}
+
+
+LogicVRegister Simulator::usra(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ LogicVRegister shifted_reg = ushr(vform, temp, src, shift);
+ return add(vform, dst, dst, shifted_reg);
+}
+
+
+LogicVRegister Simulator::srsra(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ LogicVRegister shifted_reg = sshr(vform, temp, src, shift).Round(vform);
+ return add(vform, dst, dst, shifted_reg);
+}
+
+
+LogicVRegister Simulator::ursra(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ LogicVRegister shifted_reg = ushr(vform, temp, src, shift).Round(vform);
+ return add(vform, dst, dst, shifted_reg);
+}
+
+
+LogicVRegister Simulator::cls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ uint64_t result[16];
+ int laneSizeInBits = LaneSizeInBitsFromFormat(vform);
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; i++) {
+ result[i] = CountLeadingSignBits(src.Int(vform, i), laneSizeInBits);
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::clz(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ uint64_t result[16];
+ int laneSizeInBits = LaneSizeInBitsFromFormat(vform);
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; i++) {
+ result[i] = CountLeadingZeros(src.Uint(vform, i), laneSizeInBits);
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::cnt(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ uint64_t result[16];
+ int laneSizeInBits = LaneSizeInBitsFromFormat(vform);
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; i++) {
+ uint64_t value = src.Uint(vform, i);
+ result[i] = 0;
+ for (int j = 0; j < laneSizeInBits; j++) {
+ result[i] += (value & 1);
+ value >>= 1;
+ }
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::sshl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ int8_t shift_val = src2.Int(vform, i);
+ int64_t lj_src_val = src1.IntLeftJustified(vform, i);
+
+ // Set signed saturation state.
+ if ((shift_val > CountLeadingSignBits(lj_src_val)) &&
+ (lj_src_val != 0)) {
+ dst.SetSignedSat(i, lj_src_val >= 0);
+ }
+
+ // Set unsigned saturation state.
+ if (lj_src_val < 0) {
+ dst.SetUnsignedSat(i, false);
+ } else if ((shift_val > CountLeadingZeros(lj_src_val)) &&
+ (lj_src_val != 0)) {
+ dst.SetUnsignedSat(i, true);
+ }
+
+ int64_t src_val = src1.Int(vform, i);
+ if (shift_val > 63) {
+ dst.SetInt(vform, i, 0);
+ } else if (shift_val < -63) {
+ dst.SetRounding(i, src_val < 0);
+ dst.SetInt(vform, i, (src_val < 0) ? -1 : 0);
+ } else {
+ if (shift_val < 0) {
+ // Set rounding state. Rounding only needed on right shifts.
+ if (((src_val >> (-shift_val - 1)) & 1) == 1) {
+ dst.SetRounding(i, true);
+ }
+ src_val >>= -shift_val;
+ } else {
+ src_val <<= shift_val;
+ }
+ dst.SetInt(vform, i, src_val);
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::ushl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ int8_t shift_val = src2.Int(vform, i);
+ uint64_t lj_src_val = src1.UintLeftJustified(vform, i);
+
+ // Set saturation state.
+ if ((shift_val > CountLeadingZeros(lj_src_val)) && (lj_src_val != 0)) {
+ dst.SetUnsignedSat(i, true);
+ }
+
+ uint64_t src_val = src1.Uint(vform, i);
+ if ((shift_val > 63) || (shift_val < -64)) {
+ dst.SetUint(vform, i, 0);
+ } else {
+ if (shift_val < 0) {
+ // Set rounding state. Rounding only needed on right shifts.
+ if (((src_val >> (-shift_val - 1)) & 1) == 1) {
+ dst.SetRounding(i, true);
+ }
+
+ if (shift_val == -64) {
+ src_val = 0;
+ } else {
+ src_val >>= -shift_val;
+ }
+ } else {
+ src_val <<= shift_val;
+ }
+ dst.SetUint(vform, i, src_val);
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::neg(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ // Test for signed saturation.
+ int64_t sa = src.Int(vform, i);
+ if (sa == MinIntFromFormat(vform)) {
+ dst.SetSignedSat(i, true);
+ }
+ dst.SetInt(vform, i, -sa);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::suqadd(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ int64_t sa = dst.IntLeftJustified(vform, i);
+ uint64_t ub = src.UintLeftJustified(vform, i);
+ int64_t sr = sa + ub;
+
+ if (sr < sa) { // Test for signed positive saturation.
+ dst.SetInt(vform, i, MaxIntFromFormat(vform));
+ } else {
+ dst.SetInt(vform, i, dst.Int(vform, i) + src.Int(vform, i));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::usqadd(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t ua = dst.UintLeftJustified(vform, i);
+ int64_t sb = src.IntLeftJustified(vform, i);
+ uint64_t ur = ua + sb;
+
+ if ((sb > 0) && (ur <= ua)) {
+ dst.SetUint(vform, i, MaxUintFromFormat(vform)); // Positive saturation.
+ } else if ((sb < 0) && (ur >= ua)) {
+ dst.SetUint(vform, i, 0); // Negative saturation.
+ } else {
+ dst.SetUint(vform, i, dst.Uint(vform, i) + src.Int(vform, i));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::abs(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ // Test for signed saturation.
+ int64_t sa = src.Int(vform, i);
+ if (sa == MinIntFromFormat(vform)) {
+ dst.SetSignedSat(i, true);
+ }
+ if (sa < 0) {
+ dst.SetInt(vform, i, -sa);
+ } else {
+ dst.SetInt(vform, i, sa);
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::extractnarrow(VectorFormat dstform,
+ LogicVRegister dst,
+ bool dstIsSigned,
+ const LogicVRegister& src,
+ bool srcIsSigned) {
+ bool upperhalf = false;
+ VectorFormat srcform = kFormatUndefined;
+ int64_t ssrc[8];
+ uint64_t usrc[8];
+
+ switch (dstform) {
+ case kFormat8B : upperhalf = false; srcform = kFormat8H; break;
+ case kFormat16B: upperhalf = true; srcform = kFormat8H; break;
+ case kFormat4H : upperhalf = false; srcform = kFormat4S; break;
+ case kFormat8H : upperhalf = true; srcform = kFormat4S; break;
+ case kFormat2S : upperhalf = false; srcform = kFormat2D; break;
+ case kFormat4S : upperhalf = true; srcform = kFormat2D; break;
+ case kFormatB : upperhalf = false; srcform = kFormatH; break;
+ case kFormatH : upperhalf = false; srcform = kFormatS; break;
+ case kFormatS : upperhalf = false; srcform = kFormatD; break;
+ default:VIXL_UNIMPLEMENTED();
+ }
+
+ for (int i = 0; i < LaneCountFromFormat(srcform); i++) {
+ ssrc[i] = src.Int(srcform, i);
+ usrc[i] = src.Uint(srcform, i);
+ }
+
+ int offset;
+ if (upperhalf) {
+ offset = LaneCountFromFormat(dstform) / 2;
+ } else {
+ offset = 0;
+ dst.ClearForWrite(dstform);
+ }
+
+ for (int i = 0; i < LaneCountFromFormat(srcform); i++) {
+ // Test for signed saturation
+ if (ssrc[i] > MaxIntFromFormat(dstform)) {
+ dst.SetSignedSat(offset + i, true);
+ } else if (ssrc[i] < MinIntFromFormat(dstform)) {
+ dst.SetSignedSat(offset + i, false);
+ }
+
+ // Test for unsigned saturation
+ if (srcIsSigned) {
+ if (ssrc[i] > static_cast<int64_t>(MaxUintFromFormat(dstform))) {
+ dst.SetUnsignedSat(offset + i, true);
+ } else if (ssrc[i] < 0) {
+ dst.SetUnsignedSat(offset + i, false);
+ }
+ } else {
+ if (usrc[i] > MaxUintFromFormat(dstform)) {
+ dst.SetUnsignedSat(offset + i, true);
+ }
+ }
+
+ int64_t result;
+ if (srcIsSigned) {
+ result = ssrc[i] & MaxUintFromFormat(dstform);
+ } else {
+ result = usrc[i] & MaxUintFromFormat(dstform);
+ }
+
+ if (dstIsSigned) {
+ dst.SetInt(dstform, offset + i, result);
+ } else {
+ dst.SetUint(dstform, offset + i, result);
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::xtn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return extractnarrow(vform, dst, true, src, true);
+}
+
+
+LogicVRegister Simulator::sqxtn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return extractnarrow(vform, dst, true, src, true).SignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqxtun(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return extractnarrow(vform, dst, false, src, true).UnsignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::uqxtn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return extractnarrow(vform, dst, false, src, false).UnsignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::absdiff(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ bool issigned) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ if (issigned) {
+ int64_t sr = src1.Int(vform, i) - src2.Int(vform, i);
+ sr = sr > 0 ? sr : -sr;
+ dst.SetInt(vform, i, sr);
+ } else {
+ int64_t sr = src1.Uint(vform, i) - src2.Uint(vform, i);
+ sr = sr > 0 ? sr : -sr;
+ dst.SetUint(vform, i, sr);
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::saba(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ dst.ClearForWrite(vform);
+ absdiff(vform, temp, src1, src2, true);
+ add(vform, dst, dst, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uaba(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ dst.ClearForWrite(vform);
+ absdiff(vform, temp, src1, src2, false);
+ add(vform, dst, dst, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::not_(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, ~src.Uint(vform, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::rbit(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ uint64_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ int laneSizeInBits = LaneSizeInBitsFromFormat(vform);
+ uint64_t reversed_value;
+ uint64_t value;
+ for (int i = 0; i < laneCount; i++) {
+ value = src.Uint(vform, i);
+ reversed_value = 0;
+ for (int j = 0; j < laneSizeInBits; j++) {
+ reversed_value = (reversed_value << 1) | (value & 1);
+ value >>= 1;
+ }
+ result[i] = reversed_value;
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::rev(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int revSize) {
+ uint64_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ int laneSize = LaneSizeInBytesFromFormat(vform);
+ int lanesPerLoop = revSize / laneSize;
+ for (int i = 0; i < laneCount; i += lanesPerLoop) {
+ for (int j = 0; j < lanesPerLoop; j++) {
+ result[i + lanesPerLoop - 1 - j] = src.Uint(vform, i + j);
+ }
+ }
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::rev16(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return rev(vform, dst, src, 2);
+}
+
+
+LogicVRegister Simulator::rev32(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return rev(vform, dst, src, 4);
+}
+
+
+LogicVRegister Simulator::rev64(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return rev(vform, dst, src, 8);
+}
+
+
+LogicVRegister Simulator::addlp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ bool is_signed,
+ bool do_accumulate) {
+ VectorFormat vformsrc = VectorFormatHalfWidthDoubleLanes(vform);
+
+ int64_t sr[16];
+ uint64_t ur[16];
+
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ if (is_signed) {
+ sr[i] = src.Int(vformsrc, 2 * i) + src.Int(vformsrc, 2 * i + 1);
+ } else {
+ ur[i] = src.Uint(vformsrc, 2 * i) + src.Uint(vformsrc, 2 * i + 1);
+ }
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ if (do_accumulate) {
+ if (is_signed) {
+ dst.SetInt(vform, i, dst.Int(vform, i) + sr[i]);
+ } else {
+ dst.SetUint(vform, i, dst.Uint(vform, i) + ur[i]);
+ }
+ } else {
+ if (is_signed) {
+ dst.SetInt(vform, i, sr[i]);
+ } else {
+ dst.SetUint(vform, i, ur[i]);
+ }
+ }
+ }
+
+ return dst;
+}
+
+
+LogicVRegister Simulator::saddlp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return addlp(vform, dst, src, true, false);
+}
+
+
+LogicVRegister Simulator::uaddlp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return addlp(vform, dst, src, false, false);
+}
+
+
+LogicVRegister Simulator::sadalp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return addlp(vform, dst, src, true, true);
+}
+
+
+LogicVRegister Simulator::uadalp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return addlp(vform, dst, src, false, true);
+}
+
+
+LogicVRegister Simulator::ext(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ uint8_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount - index; ++i) {
+ result[i] = src1.Uint(vform, i + index);
+ }
+ for (int i = 0; i < index; ++i) {
+ result[laneCount - index + i] = src2.Uint(vform, i);
+ }
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::dup_element(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int src_index) {
+ int laneCount = LaneCountFromFormat(vform);
+ uint64_t value = src.Uint(vform, src_index);
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, value);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::dup_immediate(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t imm) {
+ int laneCount = LaneCountFromFormat(vform);
+ uint64_t value = imm & MaxUintFromFormat(vform);
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, value);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::ins_element(VectorFormat vform,
+ LogicVRegister dst,
+ int dst_index,
+ const LogicVRegister& src,
+ int src_index) {
+ dst.SetUint(vform, dst_index, src.Uint(vform, src_index));
+ return dst;
+}
+
+
+LogicVRegister Simulator::ins_immediate(VectorFormat vform,
+ LogicVRegister dst,
+ int dst_index,
+ uint64_t imm) {
+ uint64_t value = imm & MaxUintFromFormat(vform);
+ dst.SetUint(vform, dst_index, value);
+ return dst;
+}
+
+
+LogicVRegister Simulator::mov(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
+ dst.SetUint(vform, lane, src.Uint(vform, lane));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::movi(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t imm) {
+ int laneCount = LaneCountFromFormat(vform);
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, imm);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::mvni(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t imm) {
+ int laneCount = LaneCountFromFormat(vform);
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, ~imm);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::orr(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ uint64_t imm) {
+ uint64_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ result[i] = src.Uint(vform, i) | imm;
+ }
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::uxtl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VectorFormat vform_half = VectorFormatHalfWidth(vform);
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetUint(vform, i, src.Uint(vform_half, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::sxtl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VectorFormat vform_half = VectorFormatHalfWidth(vform);
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetInt(vform, i, src.Int(vform_half, i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::uxtl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VectorFormat vform_half = VectorFormatHalfWidth(vform);
+ int lane_count = LaneCountFromFormat(vform);
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < lane_count; i++) {
+ dst.SetUint(vform, i, src.Uint(vform_half, lane_count + i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::sxtl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VectorFormat vform_half = VectorFormatHalfWidth(vform);
+ int lane_count = LaneCountFromFormat(vform);
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < lane_count; i++) {
+ dst.SetInt(vform, i, src.Int(vform_half, lane_count + i));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::shrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vform_src = VectorFormatDoubleWidth(vform);
+ VectorFormat vform_dst = vform;
+ LogicVRegister shifted_src = ushr(vform_src, temp, src, shift);
+ return extractnarrow(vform_dst, dst, false, shifted_src, false);
+}
+
+
+LogicVRegister Simulator::shrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift);
+ return extractnarrow(vformdst, dst, false, shifted_src, false);
+}
+
+
+LogicVRegister Simulator::rshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift).Round(vformsrc);
+ return extractnarrow(vformdst, dst, false, shifted_src, false);
+}
+
+
+LogicVRegister Simulator::rshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = ushr(vformsrc, temp, src, shift).Round(vformsrc);
+ return extractnarrow(vformdst, dst, false, shifted_src, false);
+}
+
+
+LogicVRegister Simulator::tbl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& ind) {
+ SimVRegister result;
+ movi(vform, result, 0);
+ tbx(vform, result, tab, ind);
+ return orr(vform, dst, result, result);
+}
+
+
+LogicVRegister Simulator::tbl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& ind) {
+ SimVRegister result;
+ movi(vform, result, 0);
+ tbx(vform, result, tab, tab2, ind);
+ return orr(vform, dst, result, result);
+}
+
+
+LogicVRegister Simulator::tbl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& tab3,
+ const LogicVRegister& ind) {
+ SimVRegister result;
+ movi(vform, result, 0);
+ tbx(vform, result, tab, tab2, tab3, ind);
+ return orr(vform, dst, result, result);
+}
+
+
+LogicVRegister Simulator::tbl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& tab3,
+ const LogicVRegister& tab4,
+ const LogicVRegister& ind) {
+ SimVRegister result;
+ movi(vform, result, 0);
+ tbx(vform, result, tab, tab2, tab3, tab4, ind);
+ return orr(vform, dst, result, result);
+}
+
+
+LogicVRegister Simulator::tbx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& ind) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t j = ind.Uint(vform, i);
+ switch (j >> 4) {
+ case 0: dst.SetUint(vform, i, tab.Uint(kFormat16B, j & 15)); break;
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::tbx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& ind) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t j = ind.Uint(vform, i);
+ switch (j >> 4) {
+ case 0: dst.SetUint(vform, i, tab.Uint(kFormat16B, j & 15)); break;
+ case 1: dst.SetUint(vform, i, tab2.Uint(kFormat16B, j & 15)); break;
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::tbx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& tab3,
+ const LogicVRegister& ind) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t j = ind.Uint(vform, i);
+ switch (j >> 4) {
+ case 0: dst.SetUint(vform, i, tab.Uint(kFormat16B, j & 15)); break;
+ case 1: dst.SetUint(vform, i, tab2.Uint(kFormat16B, j & 15)); break;
+ case 2: dst.SetUint(vform, i, tab3.Uint(kFormat16B, j & 15)); break;
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::tbx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& tab3,
+ const LogicVRegister& tab4,
+ const LogicVRegister& ind) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t j = ind.Uint(vform, i);
+ switch (j >> 4) {
+ case 0: dst.SetUint(vform, i, tab.Uint(kFormat16B, j & 15)); break;
+ case 1: dst.SetUint(vform, i, tab2.Uint(kFormat16B, j & 15)); break;
+ case 2: dst.SetUint(vform, i, tab3.Uint(kFormat16B, j & 15)); break;
+ case 3: dst.SetUint(vform, i, tab4.Uint(kFormat16B, j & 15)); break;
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::uqshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ return shrn(vform, dst, src, shift).UnsignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::uqshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ return shrn2(vform, dst, src, shift).UnsignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::uqrshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ return rshrn(vform, dst, src, shift).UnsignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::uqrshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ return rshrn2(vform, dst, src, shift).UnsignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
+ return sqxtn(vformdst, dst, shifted_src);
+}
+
+
+LogicVRegister Simulator::sqshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
+ return sqxtn(vformdst, dst, shifted_src);
+}
+
+
+LogicVRegister Simulator::sqrshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
+ return sqxtn(vformdst, dst, shifted_src);
+}
+
+
+LogicVRegister Simulator::sqrshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
+ return sqxtn(vformdst, dst, shifted_src);
+}
+
+
+LogicVRegister Simulator::sqshrun(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
+ return sqxtun(vformdst, dst, shifted_src);
+}
+
+
+LogicVRegister Simulator::sqshrun2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift);
+ return sqxtun(vformdst, dst, shifted_src);
+}
+
+
+LogicVRegister Simulator::sqrshrun(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
+ return sqxtun(vformdst, dst, shifted_src);
+}
+
+
+LogicVRegister Simulator::sqrshrun2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift) {
+ SimVRegister temp;
+ VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
+ VectorFormat vformdst = vform;
+ LogicVRegister shifted_src = sshr(vformsrc, temp, src, shift).Round(vformsrc);
+ return sqxtun(vformdst, dst, shifted_src);
+}
+
+
+LogicVRegister Simulator::uaddl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl(vform, temp1, src1);
+ uxtl(vform, temp2, src2);
+ add(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uaddl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl2(vform, temp1, src1);
+ uxtl2(vform, temp2, src2);
+ add(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uaddw(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ uxtl(vform, temp, src2);
+ add(vform, dst, src1, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uaddw2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ uxtl2(vform, temp, src2);
+ add(vform, dst, src1, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::saddl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl(vform, temp1, src1);
+ sxtl(vform, temp2, src2);
+ add(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::saddl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl2(vform, temp1, src1);
+ sxtl2(vform, temp2, src2);
+ add(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::saddw(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ sxtl(vform, temp, src2);
+ add(vform, dst, src1, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::saddw2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ sxtl2(vform, temp, src2);
+ add(vform, dst, src1, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::usubl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl(vform, temp1, src1);
+ uxtl(vform, temp2, src2);
+ sub(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::usubl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl2(vform, temp1, src1);
+ uxtl2(vform, temp2, src2);
+ sub(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::usubw(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ uxtl(vform, temp, src2);
+ sub(vform, dst, src1, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::usubw2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ uxtl2(vform, temp, src2);
+ sub(vform, dst, src1, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::ssubl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl(vform, temp1, src1);
+ sxtl(vform, temp2, src2);
+ sub(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::ssubl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl2(vform, temp1, src1);
+ sxtl2(vform, temp2, src2);
+ sub(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::ssubw(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ sxtl(vform, temp, src2);
+ sub(vform, dst, src1, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::ssubw2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ sxtl2(vform, temp, src2);
+ sub(vform, dst, src1, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uabal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl(vform, temp1, src1);
+ uxtl(vform, temp2, src2);
+ uaba(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uabal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl2(vform, temp1, src1);
+ uxtl2(vform, temp2, src2);
+ uaba(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::sabal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl(vform, temp1, src1);
+ sxtl(vform, temp2, src2);
+ saba(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::sabal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl2(vform, temp1, src1);
+ sxtl2(vform, temp2, src2);
+ saba(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uabdl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl(vform, temp1, src1);
+ uxtl(vform, temp2, src2);
+ absdiff(vform, dst, temp1, temp2, false);
+ return dst;
+}
+
+
+LogicVRegister Simulator::uabdl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl2(vform, temp1, src1);
+ uxtl2(vform, temp2, src2);
+ absdiff(vform, dst, temp1, temp2, false);
+ return dst;
+}
+
+
+LogicVRegister Simulator::sabdl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl(vform, temp1, src1);
+ sxtl(vform, temp2, src2);
+ absdiff(vform, dst, temp1, temp2, true);
+ return dst;
+}
+
+
+LogicVRegister Simulator::sabdl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl2(vform, temp1, src1);
+ sxtl2(vform, temp2, src2);
+ absdiff(vform, dst, temp1, temp2, true);
+ return dst;
+}
+
+
+LogicVRegister Simulator::umull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl(vform, temp1, src1);
+ uxtl(vform, temp2, src2);
+ mul(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::umull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl2(vform, temp1, src1);
+ uxtl2(vform, temp2, src2);
+ mul(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::smull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl(vform, temp1, src1);
+ sxtl(vform, temp2, src2);
+ mul(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::smull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl2(vform, temp1, src1);
+ sxtl2(vform, temp2, src2);
+ mul(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::umlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl(vform, temp1, src1);
+ uxtl(vform, temp2, src2);
+ mls(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::umlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl2(vform, temp1, src1);
+ uxtl2(vform, temp2, src2);
+ mls(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::smlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl(vform, temp1, src1);
+ sxtl(vform, temp2, src2);
+ mls(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::smlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl2(vform, temp1, src1);
+ sxtl2(vform, temp2, src2);
+ mls(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::umlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl(vform, temp1, src1);
+ uxtl(vform, temp2, src2);
+ mla(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::umlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ uxtl2(vform, temp1, src1);
+ uxtl2(vform, temp2, src2);
+ mla(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::smlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl(vform, temp1, src1);
+ sxtl(vform, temp2, src2);
+ mla(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::smlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp1, temp2;
+ sxtl2(vform, temp1, src1);
+ sxtl2(vform, temp2, src2);
+ mla(vform, dst, temp1, temp2);
+ return dst;
+}
+
+
+LogicVRegister Simulator::sqdmlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ LogicVRegister product = sqdmull(vform, temp, src1, src2);
+ return add(vform, dst, dst, product).SignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqdmlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ LogicVRegister product = sqdmull2(vform, temp, src1, src2);
+ return add(vform, dst, dst, product).SignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqdmlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ LogicVRegister product = sqdmull(vform, temp, src1, src2);
+ return sub(vform, dst, dst, product).SignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqdmlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ LogicVRegister product = sqdmull2(vform, temp, src1, src2);
+ return sub(vform, dst, dst, product).SignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqdmull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ LogicVRegister product = smull(vform, temp, src1, src2);
+ return add(vform, dst, product, product).SignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqdmull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ LogicVRegister product = smull2(vform, temp, src1, src2);
+ return add(vform, dst, product, product).SignedSaturate(vform);
+}
+
+
+LogicVRegister Simulator::sqrdmulh(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ bool round) {
+ // 2 * INT_32_MIN * INT_32_MIN causes int64_t to overflow.
+ // To avoid this, we use (src1 * src2 + 1 << (esize - 2)) >> (esize - 1)
+ // which is same as (2 * src1 * src2 + 1 << (esize - 1)) >> esize.
+
+ int esize = LaneSizeInBitsFromFormat(vform);
+ int round_const = round ? (1 << (esize - 2)) : 0;
+ int64_t product;
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ product = src1.Int(vform, i) * src2.Int(vform, i);
+ product += round_const;
+ product = product >> (esize - 1);
+
+ if (product > MaxIntFromFormat(vform)) {
+ product = MaxIntFromFormat(vform);
+ } else if (product < MinIntFromFormat(vform)) {
+ product = MinIntFromFormat(vform);
+ }
+ dst.SetInt(vform, i, product);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::sqdmulh(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ return sqrdmulh(vform, dst, src1, src2, false);
+}
+
+
+LogicVRegister Simulator::addhn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ add(VectorFormatDoubleWidth(vform), temp, src1, src2);
+ shrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
+ return dst;
+}
+
+
+LogicVRegister Simulator::addhn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ add(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
+ shrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
+ return dst;
+}
+
+
+LogicVRegister Simulator::raddhn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ add(VectorFormatDoubleWidth(vform), temp, src1, src2);
+ rshrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
+ return dst;
+}
+
+
+LogicVRegister Simulator::raddhn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ add(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
+ rshrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
+ return dst;
+}
+
+
+LogicVRegister Simulator::subhn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ sub(VectorFormatDoubleWidth(vform), temp, src1, src2);
+ shrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
+ return dst;
+}
+
+
+LogicVRegister Simulator::subhn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ sub(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
+ shrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
+ return dst;
+}
+
+
+LogicVRegister Simulator::rsubhn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ sub(VectorFormatDoubleWidth(vform), temp, src1, src2);
+ rshrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
+ return dst;
+}
+
+
+LogicVRegister Simulator::rsubhn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ sub(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
+ rshrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
+ return dst;
+}
+
+
+LogicVRegister Simulator::trn1(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ uint64_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ int pairs = laneCount / 2;
+ for (int i = 0; i < pairs; ++i) {
+ result[2 * i] = src1.Uint(vform, 2 * i);
+ result[(2 * i) + 1] = src2.Uint(vform, 2 * i);
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::trn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ uint64_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ int pairs = laneCount / 2;
+ for (int i = 0; i < pairs; ++i) {
+ result[2 * i] = src1.Uint(vform, (2 * i) + 1);
+ result[(2 * i) + 1] = src2.Uint(vform, (2 * i) + 1);
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::zip1(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ uint64_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ int pairs = laneCount / 2;
+ for (int i = 0; i < pairs; ++i) {
+ result[2 * i] = src1.Uint(vform, i);
+ result[(2 * i) + 1] = src2.Uint(vform, i);
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::zip2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ uint64_t result[16];
+ int laneCount = LaneCountFromFormat(vform);
+ int pairs = laneCount / 2;
+ for (int i = 0; i < pairs; ++i) {
+ result[2 * i] = src1.Uint(vform, pairs + i);
+ result[(2 * i) + 1] = src2.Uint(vform, pairs + i);
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::uzp1(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ uint64_t result[32];
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ result[i] = src1.Uint(vform, i);
+ result[laneCount + i] = src2.Uint(vform, i);
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[2 * i]);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::uzp2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ uint64_t result[32];
+ int laneCount = LaneCountFromFormat(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ result[i] = src1.Uint(vform, i);
+ result[laneCount + i] = src2.Uint(vform, i);
+ }
+
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < laneCount; ++i) {
+ dst.SetUint(vform, i, result[ (2 * i) + 1]);
+ }
+ return dst;
+}
+
+
+template <typename T>
+T Simulator::FPAdd(T op1, T op2) {
+ T result = FPProcessNaNs(op1, op2);
+ if (std::isnan(result)) return result;
+
+ if (std::isinf(op1) && std::isinf(op2) && (op1 != op2)) {
+ // inf + -inf returns the default NaN.
+ FPProcessException();
+ return FPDefaultNaN<T>();
+ } else {
+ // Other cases should be handled by standard arithmetic.
+ return op1 + op2;
+ }
+}
+
+
+template <typename T>
+T Simulator::FPSub(T op1, T op2) {
+ // NaNs should be handled elsewhere.
+ VIXL_ASSERT(!std::isnan(op1) && !std::isnan(op2));
+
+ if (std::isinf(op1) && std::isinf(op2) && (op1 == op2)) {
+ // inf - inf returns the default NaN.
+ FPProcessException();
+ return FPDefaultNaN<T>();
+ } else {
+ // Other cases should be handled by standard arithmetic.
+ return op1 - op2;
+ }
+}
+
+
+template <typename T>
+T Simulator::FPMul(T op1, T op2) {
+ // NaNs should be handled elsewhere.
+ VIXL_ASSERT(!std::isnan(op1) && !std::isnan(op2));
+
+ if ((std::isinf(op1) && (op2 == 0.0)) || (std::isinf(op2) && (op1 == 0.0))) {
+ // inf * 0.0 returns the default NaN.
+ FPProcessException();
+ return FPDefaultNaN<T>();
+ } else {
+ // Other cases should be handled by standard arithmetic.
+ return op1 * op2;
+ }
+}
+
+
+template<typename T>
+T Simulator::FPMulx(T op1, T op2) {
+ if ((std::isinf(op1) && (op2 == 0.0)) || (std::isinf(op2) && (op1 == 0.0))) {
+ // inf * 0.0 returns +/-2.0.
+ T two = 2.0;
+ return copysign(1.0, op1) * copysign(1.0, op2) * two;
+ }
+ return FPMul(op1, op2);
+}
+
+
+template<typename T>
+T Simulator::FPMulAdd(T a, T op1, T op2) {
+ T result = FPProcessNaNs3(a, op1, op2);
+
+ T sign_a = copysign(1.0, a);
+ T sign_prod = copysign(1.0, op1) * copysign(1.0, op2);
+ bool isinf_prod = std::isinf(op1) || std::isinf(op2);
+ bool operation_generates_nan =
+ (std::isinf(op1) && (op2 == 0.0)) || // inf * 0.0
+ (std::isinf(op2) && (op1 == 0.0)) || // 0.0 * inf
+ (std::isinf(a) && isinf_prod && (sign_a != sign_prod)); // inf - inf
+
+ if (std::isnan(result)) {
+ // Generated NaNs override quiet NaNs propagated from a.
+ if (operation_generates_nan && IsQuietNaN(a)) {
+ FPProcessException();
+ return FPDefaultNaN<T>();
+ } else {
+ return result;
+ }
+ }
+
+ // If the operation would produce a NaN, return the default NaN.
+ if (operation_generates_nan) {
+ FPProcessException();
+ return FPDefaultNaN<T>();
+ }
+
+ // Work around broken fma implementations for exact zero results: The sign of
+ // exact 0.0 results is positive unless both a and op1 * op2 are negative.
+ if (((op1 == 0.0) || (op2 == 0.0)) && (a == 0.0)) {
+ return ((sign_a < 0) && (sign_prod < 0)) ? -0.0 : 0.0;
+ }
+
+ result = FusedMultiplyAdd(op1, op2, a);
+ VIXL_ASSERT(!std::isnan(result));
+
+ // Work around broken fma implementations for rounded zero results: If a is
+ // 0.0, the sign of the result is the sign of op1 * op2 before rounding.
+ if ((a == 0.0) && (result == 0.0)) {
+ return copysign(0.0, sign_prod);
+ }
+
+ return result;
+}
+
+
+template <typename T>
+T Simulator::FPDiv(T op1, T op2) {
+ // NaNs should be handled elsewhere.
+ VIXL_ASSERT(!std::isnan(op1) && !std::isnan(op2));
+
+ if ((std::isinf(op1) && std::isinf(op2)) || ((op1 == 0.0) && (op2 == 0.0))) {
+ // inf / inf and 0.0 / 0.0 return the default NaN.
+ FPProcessException();
+ return FPDefaultNaN<T>();
+ } else {
+ if (op2 == 0.0) FPProcessException();
+
+ // Other cases should be handled by standard arithmetic.
+ return op1 / op2;
+ }
+}
+
+
+template <typename T>
+T Simulator::FPSqrt(T op) {
+ if (std::isnan(op)) {
+ return FPProcessNaN(op);
+ } else if (op < 0.0) {
+ FPProcessException();
+ return FPDefaultNaN<T>();
+ } else {
+ return sqrt(op);
+ }
+}
+
+
+template <typename T>
+T Simulator::FPMax(T a, T b) {
+ T result = FPProcessNaNs(a, b);
+ if (std::isnan(result)) return result;
+
+ if ((a == 0.0) && (b == 0.0) &&
+ (copysign(1.0, a) != copysign(1.0, b))) {
+ // a and b are zero, and the sign differs: return +0.0.
+ return 0.0;
+ } else {
+ return (a > b) ? a : b;
+ }
+}
+
+
+template <typename T>
+T Simulator::FPMaxNM(T a, T b) {
+ if (IsQuietNaN(a) && !IsQuietNaN(b)) {
+ a = kFP64NegativeInfinity;
+ } else if (!IsQuietNaN(a) && IsQuietNaN(b)) {
+ b = kFP64NegativeInfinity;
+ }
+
+ T result = FPProcessNaNs(a, b);
+ return std::isnan(result) ? result : FPMax(a, b);
+}
+
+
+template <typename T>
+T Simulator::FPMin(T a, T b) {
+ T result = FPProcessNaNs(a, b);
+ if (std::isnan(result)) return result;
+
+ if ((a == 0.0) && (b == 0.0) &&
+ (copysign(1.0, a) != copysign(1.0, b))) {
+ // a and b are zero, and the sign differs: return -0.0.
+ return -0.0;
+ } else {
+ return (a < b) ? a : b;
+ }
+}
+
+
+template <typename T>
+T Simulator::FPMinNM(T a, T b) {
+ if (IsQuietNaN(a) && !IsQuietNaN(b)) {
+ a = kFP64PositiveInfinity;
+ } else if (!IsQuietNaN(a) && IsQuietNaN(b)) {
+ b = kFP64PositiveInfinity;
+ }
+
+ T result = FPProcessNaNs(a, b);
+ return std::isnan(result) ? result : FPMin(a, b);
+}
+
+
+template <typename T>
+T Simulator::FPRecipStepFused(T op1, T op2) {
+ const T two = 2.0;
+ if ((std::isinf(op1) && (op2 == 0.0))
+ || ((op1 == 0.0) && (std::isinf(op2)))) {
+ return two;
+ } else if (std::isinf(op1) || std::isinf(op2)) {
+ // Return +inf if signs match, otherwise -inf.
+ return ((op1 >= 0.0) == (op2 >= 0.0)) ? kFP64PositiveInfinity
+ : kFP64NegativeInfinity;
+ } else {
+ return FusedMultiplyAdd(op1, op2, two);
+ }
+}
+
+
+template <typename T>
+T Simulator::FPRSqrtStepFused(T op1, T op2) {
+ const T one_point_five = 1.5;
+ const T two = 2.0;
+
+ if ((std::isinf(op1) && (op2 == 0.0))
+ || ((op1 == 0.0) && (std::isinf(op2)))) {
+ return one_point_five;
+ } else if (std::isinf(op1) || std::isinf(op2)) {
+ // Return +inf if signs match, otherwise -inf.
+ return ((op1 >= 0.0) == (op2 >= 0.0)) ? kFP64PositiveInfinity
+ : kFP64NegativeInfinity;
+ } else {
+ // The multiply-add-halve operation must be fully fused, so avoid interim
+ // rounding by checking which operand can be losslessly divided by two
+ // before doing the multiply-add.
+ if (std::isnormal(op1 / two)) {
+ return FusedMultiplyAdd(op1 / two, op2, one_point_five);
+ } else if (std::isnormal(op2 / two)) {
+ return FusedMultiplyAdd(op1, op2 / two, one_point_five);
+ } else {
+ // Neither operand is normal after halving: the result is dominated by
+ // the addition term, so just return that.
+ return one_point_five;
+ }
+ }
+}
+
+int32_t Simulator::FPToFixedJS(double value) {
+ // The Z-flag is set when the conversion from double precision floating-point
+ // to 32-bit integer is exact. If the source value is +/-Infinity, -0.0, NaN,
+ // outside the bounds of a 32-bit integer, or isn't an exact integer then the
+ // Z-flag is unset.
+ int Z = 1;
+ int32_t result;
+
+ if ((value == 0.0) || (value == kFP64PositiveInfinity) ||
+ (value == kFP64NegativeInfinity)) {
+ // +/- zero and infinity all return zero, however -0 and +/- Infinity also
+ // unset the Z-flag.
+ result = 0.0;
+ if ((value != 0.0) || std::signbit(value)) {
+ Z = 0;
+ }
+ } else if (std::isnan(value)) {
+ // NaN values unset the Z-flag and set the result to 0.
+ FPProcessNaN(value);
+ result = 0;
+ Z = 0;
+ } else {
+ // All other values are converted to an integer representation, rounded
+ // toward zero.
+ double int_result = std::floor(value);
+ double error = value - int_result;
+
+ if ((error != 0.0) && (int_result < 0.0)) {
+ int_result++;
+ }
+
+ // Constrain the value into the range [INT32_MIN, INT32_MAX]. We can almost
+ // write a one-liner with std::round, but the behaviour on ties is incorrect
+ // for our purposes.
+ double mod_const = static_cast<double>(UINT64_C(1) << 32);
+ double mod_error =
+ (int_result / mod_const) - std::floor(int_result / mod_const);
+ double constrained;
+ if (mod_error == 0.5) {
+ constrained = INT32_MIN;
+ } else {
+ constrained = int_result - mod_const * round(int_result / mod_const);
+ }
+
+ VIXL_ASSERT(std::floor(constrained) == constrained);
+ VIXL_ASSERT(constrained >= INT32_MIN);
+ VIXL_ASSERT(constrained <= INT32_MAX);
+
+ // Take the bottom 32 bits of the result as a 32-bit integer.
+ result = static_cast<int32_t>(constrained);
+
+ if ((int_result < INT32_MIN) || (int_result > INT32_MAX) ||
+ (error != 0.0)) {
+ // If the integer result is out of range or the conversion isn't exact,
+ // take exception and unset the Z-flag.
+ FPProcessException();
+ Z = 0;
+ }
+ }
+
+ ReadNzcv().SetN(0);
+ ReadNzcv().SetZ(Z);
+ ReadNzcv().SetC(0);
+ ReadNzcv().SetV(0);
+
+ return result;
+}
+
+
+double Simulator::FPRoundInt(double value, FPRounding round_mode) {
+ if ((value == 0.0) || (value == kFP64PositiveInfinity) ||
+ (value == kFP64NegativeInfinity)) {
+ return value;
+ } else if (std::isnan(value)) {
+ return FPProcessNaN(value);
+ }
+
+ double int_result = std::floor(value);
+ double error = value - int_result;
+ switch (round_mode) {
+ case FPTieAway: {
+ // Take care of correctly handling the range ]-0.5, -0.0], which must
+ // yield -0.0.
+ if ((-0.5 < value) && (value < 0.0)) {
+ int_result = -0.0;
+
+ } else if ((error > 0.5) || ((error == 0.5) && (int_result >= 0.0))) {
+ // If the error is greater than 0.5, or is equal to 0.5 and the integer
+ // result is positive, round up.
+ int_result++;
+ }
+ break;
+ }
+ case FPTieEven: {
+ // Take care of correctly handling the range [-0.5, -0.0], which must
+ // yield -0.0.
+ if ((-0.5 <= value) && (value < 0.0)) {
+ int_result = -0.0;
+
+ // If the error is greater than 0.5, or is equal to 0.5 and the integer
+ // result is odd, round up.
+ } else if ((error > 0.5) ||
+ ((error == 0.5) && (std::fmod(int_result, 2) != 0))) {
+ int_result++;
+ }
+ break;
+ }
+ case FPZero: {
+ // If value>0 then we take floor(value)
+ // otherwise, ceil(value).
+ if (value < 0) {
+ int_result = ceil(value);
+ }
+ break;
+ }
+ case FPNegativeInfinity: {
+ // We always use floor(value).
+ break;
+ }
+ case FPPositiveInfinity: {
+ // Take care of correctly handling the range ]-1.0, -0.0], which must
+ // yield -0.0.
+ if ((-1.0 < value) && (value < 0.0)) {
+ int_result = -0.0;
+
+ // If the error is non-zero, round up.
+ } else if (error > 0.0) {
+ int_result++;
+ }
+ break;
+ }
+ default: VIXL_UNIMPLEMENTED();
+ }
+ return int_result;
+}
+
+
+int32_t Simulator::FPToInt32(double value, FPRounding rmode) {
+ value = FPRoundInt(value, rmode);
+ if (value >= kWMaxInt) {
+ return kWMaxInt;
+ } else if (value < kWMinInt) {
+ return kWMinInt;
+ }
+ return std::isnan(value) ? 0 : static_cast<int32_t>(value);
+}
+
+
+int64_t Simulator::FPToInt64(double value, FPRounding rmode) {
+ value = FPRoundInt(value, rmode);
+ // The compiler would have to round kXMaxInt, triggering a warning. Compare
+ // against the largest int64_t that is exactly representable as a double.
+ if (value > kXMaxExactInt) {
+ return kXMaxInt;
+ } else if (value < kXMinInt) {
+ return kXMinInt;
+ }
+ return std::isnan(value) ? 0 : static_cast<int64_t>(value);
+}
+
+
+uint32_t Simulator::FPToUInt32(double value, FPRounding rmode) {
+ value = FPRoundInt(value, rmode);
+ if (value >= kWMaxUInt) {
+ return kWMaxUInt;
+ } else if (value < 0.0) {
+ return 0;
+ }
+ return std::isnan(value) ? 0 : static_cast<uint32_t>(value);
+}
+
+
+uint64_t Simulator::FPToUInt64(double value, FPRounding rmode) {
+ value = FPRoundInt(value, rmode);
+ // The compiler would have to round kXMaxUInt, triggering a warning. Compare
+ // against the largest uint64_t that is exactly representable as a double.
+ if (value > kXMaxExactUInt) {
+ return kXMaxUInt;
+ } else if (value < 0.0) {
+ return 0;
+ }
+ return std::isnan(value) ? 0 : static_cast<uint64_t>(value);
+}
+
+
+#define DEFINE_NEON_FP_VECTOR_OP(FN, OP, PROCNAN) \
+template <typename T> \
+LogicVRegister Simulator::FN(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src1, \
+ const LogicVRegister& src2) { \
+ dst.ClearForWrite(vform); \
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) { \
+ T op1 = src1.Float<T>(i); \
+ T op2 = src2.Float<T>(i); \
+ T result; \
+ if (PROCNAN) { \
+ result = FPProcessNaNs(op1, op2); \
+ if (!std::isnan(result)) { \
+ result = OP(op1, op2); \
+ } \
+ } else { \
+ result = OP(op1, op2); \
+ } \
+ dst.SetFloat(i, result); \
+ } \
+ return dst; \
+} \
+ \
+LogicVRegister Simulator::FN(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src1, \
+ const LogicVRegister& src2) { \
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) { \
+ FN<float>(vform, dst, src1, src2); \
+ } else { \
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize); \
+ FN<double>(vform, dst, src1, src2); \
+ } \
+ return dst; \
+}
+NEON_FP3SAME_LIST(DEFINE_NEON_FP_VECTOR_OP)
+#undef DEFINE_NEON_FP_VECTOR_OP
+
+
+LogicVRegister Simulator::fnmul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ LogicVRegister product = fmul(vform, temp, src1, src2);
+ return fneg(vform, dst, product);
+}
+
+
+template <typename T>
+LogicVRegister Simulator::frecps(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ T op1 = -src1.Float<T>(i);
+ T op2 = src2.Float<T>(i);
+ T result = FPProcessNaNs(op1, op2);
+ dst.SetFloat(i, std::isnan(result) ? result : FPRecipStepFused(op1, op2));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::frecps(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ frecps<float>(vform, dst, src1, src2);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ frecps<double>(vform, dst, src1, src2);
+ }
+ return dst;
+}
+
+
+template <typename T>
+LogicVRegister Simulator::frsqrts(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ T op1 = -src1.Float<T>(i);
+ T op2 = src2.Float<T>(i);
+ T result = FPProcessNaNs(op1, op2);
+ dst.SetFloat(i, std::isnan(result) ? result : FPRSqrtStepFused(op1, op2));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::frsqrts(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ frsqrts<float>(vform, dst, src1, src2);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ frsqrts<double>(vform, dst, src1, src2);
+ }
+ return dst;
+}
+
+
+template <typename T>
+LogicVRegister Simulator::fcmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ Condition cond) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ bool result = false;
+ T op1 = src1.Float<T>(i);
+ T op2 = src2.Float<T>(i);
+ T nan_result = FPProcessNaNs(op1, op2);
+ if (!std::isnan(nan_result)) {
+ switch (cond) {
+ case eq: result = (op1 == op2); break;
+ case ge: result = (op1 >= op2); break;
+ case gt: result = (op1 > op2) ; break;
+ case le: result = (op1 <= op2); break;
+ case lt: result = (op1 < op2) ; break;
+ default: VIXL_UNREACHABLE(); break;
+ }
+ }
+ dst.SetUint(vform, i, result ? MaxUintFromFormat(vform) : 0);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ Condition cond) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ fcmp<float>(vform, dst, src1, src2, cond);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ fcmp<double>(vform, dst, src1, src2, cond);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcmp_zero(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ Condition cond) {
+ SimVRegister temp;
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ LogicVRegister zero_reg = dup_immediate(vform, temp, FloatToRawbits(0.0));
+ fcmp<float>(vform, dst, src, zero_reg, cond);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ LogicVRegister zero_reg = dup_immediate(vform, temp,
+ DoubleToRawbits(0.0));
+ fcmp<double>(vform, dst, src, zero_reg, cond);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fabscmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ Condition cond) {
+ SimVRegister temp1, temp2;
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ LogicVRegister abs_src1 = fabs_<float>(vform, temp1, src1);
+ LogicVRegister abs_src2 = fabs_<float>(vform, temp2, src2);
+ fcmp<float>(vform, dst, abs_src1, abs_src2, cond);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ LogicVRegister abs_src1 = fabs_<double>(vform, temp1, src1);
+ LogicVRegister abs_src2 = fabs_<double>(vform, temp2, src2);
+ fcmp<double>(vform, dst, abs_src1, abs_src2, cond);
+ }
+ return dst;
+}
+
+
+template <typename T>
+LogicVRegister Simulator::fmla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ T op1 = src1.Float<T>(i);
+ T op2 = src2.Float<T>(i);
+ T acc = dst.Float<T>(i);
+ T result = FPMulAdd(acc, op1, op2);
+ dst.SetFloat(i, result);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fmla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ fmla<float>(vform, dst, src1, src2);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ fmla<double>(vform, dst, src1, src2);
+ }
+ return dst;
+}
+
+
+template <typename T>
+LogicVRegister Simulator::fmls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ T op1 = -src1.Float<T>(i);
+ T op2 = src2.Float<T>(i);
+ T acc = dst.Float<T>(i);
+ T result = FPMulAdd(acc, op1, op2);
+ dst.SetFloat(i, result);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fmls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ fmls<float>(vform, dst, src1, src2);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ fmls<double>(vform, dst, src1, src2);
+ }
+ return dst;
+}
+
+
+template <typename T>
+LogicVRegister Simulator::fneg(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ T op = src.Float<T>(i);
+ op = -op;
+ dst.SetFloat(i, op);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fneg(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ fneg<float>(vform, dst, src);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ fneg<double>(vform, dst, src);
+ }
+ return dst;
+}
+
+
+template <typename T>
+LogicVRegister Simulator::fabs_(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ T op = src.Float<T>(i);
+ if (copysign(1.0, op) < 0.0) {
+ op = -op;
+ }
+ dst.SetFloat(i, op);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fabs_(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ fabs_<float>(vform, dst, src);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ fabs_<double>(vform, dst, src);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fabd(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2) {
+ SimVRegister temp;
+ fsub(vform, temp, src1, src2);
+ fabs_(vform, dst, temp);
+ return dst;
+}
+
+
+LogicVRegister Simulator::fsqrt(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ float result = FPSqrt(src.Float<float>(i));
+ dst.SetFloat(i, result);
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ double result = FPSqrt(src.Float<double>(i));
+ dst.SetFloat(i, result);
+ }
+ }
+ return dst;
+}
+
+
+#define DEFINE_NEON_FP_PAIR_OP(FNP, FN, OP) \
+LogicVRegister Simulator::FNP(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src1, \
+ const LogicVRegister& src2) { \
+ SimVRegister temp1, temp2; \
+ uzp1(vform, temp1, src1, src2); \
+ uzp2(vform, temp2, src1, src2); \
+ FN(vform, dst, temp1, temp2); \
+ return dst; \
+} \
+ \
+LogicVRegister Simulator::FNP(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src) { \
+ if (vform == kFormatS) { \
+ float result = OP(src.Float<float>(0), src.Float<float>(1)); \
+ dst.SetFloat(0, result); \
+ } else { \
+ VIXL_ASSERT(vform == kFormatD); \
+ double result = OP(src.Float<double>(0), src.Float<double>(1)); \
+ dst.SetFloat(0, result); \
+ } \
+ dst.ClearForWrite(vform); \
+ return dst; \
+}
+NEON_FPPAIRWISE_LIST(DEFINE_NEON_FP_PAIR_OP)
+#undef DEFINE_NEON_FP_PAIR_OP
+
+
+LogicVRegister Simulator::fminmaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPMinMaxOp Op) {
+ VIXL_ASSERT(vform == kFormat4S);
+ USE(vform);
+ float result1 = (this->*Op)(src.Float<float>(0), src.Float<float>(1));
+ float result2 = (this->*Op)(src.Float<float>(2), src.Float<float>(3));
+ float result = (this->*Op)(result1, result2);
+ dst.ClearForWrite(kFormatS);
+ dst.SetFloat<float>(0, result);
+ return dst;
+}
+
+
+LogicVRegister Simulator::fmaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return fminmaxv(vform, dst, src, &Simulator::FPMax);
+}
+
+
+LogicVRegister Simulator::fminv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return fminmaxv(vform, dst, src, &Simulator::FPMin);
+}
+
+
+LogicVRegister Simulator::fmaxnmv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return fminmaxv(vform, dst, src, &Simulator::FPMaxNM);
+}
+
+
+LogicVRegister Simulator::fminnmv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ return fminmaxv(vform, dst, src, &Simulator::FPMinNM);
+}
+
+
+LogicVRegister Simulator::fmul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ dst.ClearForWrite(vform);
+ SimVRegister temp;
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ LogicVRegister index_reg = dup_element(kFormat4S, temp, src2, index);
+ fmul<float>(vform, dst, src1, index_reg);
+
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ LogicVRegister index_reg = dup_element(kFormat2D, temp, src2, index);
+ fmul<double>(vform, dst, src1, index_reg);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fmla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ dst.ClearForWrite(vform);
+ SimVRegister temp;
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ LogicVRegister index_reg = dup_element(kFormat4S, temp, src2, index);
+ fmla<float>(vform, dst, src1, index_reg);
+
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ LogicVRegister index_reg = dup_element(kFormat2D, temp, src2, index);
+ fmla<double>(vform, dst, src1, index_reg);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fmls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ dst.ClearForWrite(vform);
+ SimVRegister temp;
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ LogicVRegister index_reg = dup_element(kFormat4S, temp, src2, index);
+ fmls<float>(vform, dst, src1, index_reg);
+
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ LogicVRegister index_reg = dup_element(kFormat2D, temp, src2, index);
+ fmls<double>(vform, dst, src1, index_reg);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fmulx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index) {
+ dst.ClearForWrite(vform);
+ SimVRegister temp;
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ LogicVRegister index_reg = dup_element(kFormat4S, temp, src2, index);
+ fmulx<float>(vform, dst, src1, index_reg);
+
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ LogicVRegister index_reg = dup_element(kFormat2D, temp, src2, index);
+ fmulx<double>(vform, dst, src1, index_reg);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::frint(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPRounding rounding_mode,
+ bool inexact_exception) {
+ dst.ClearForWrite(vform);
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ float input = src.Float<float>(i);
+ float rounded = FPRoundInt(input, rounding_mode);
+ if (inexact_exception && !std::isnan(input) && (input != rounded)) {
+ FPProcessException();
+ }
+ dst.SetFloat<float>(i, rounded);
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ double input = src.Float<double>(i);
+ double rounded = FPRoundInt(input, rounding_mode);
+ if (inexact_exception && !std::isnan(input) && (input != rounded)) {
+ FPProcessException();
+ }
+ dst.SetFloat<double>(i, rounded);
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcvts(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPRounding rounding_mode,
+ int fbits) {
+ dst.ClearForWrite(vform);
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ float op = src.Float<float>(i) * std::pow(2.0f, fbits);
+ dst.SetInt(vform, i, FPToInt32(op, rounding_mode));
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ double op = src.Float<double>(i) * std::pow(2.0, fbits);
+ dst.SetInt(vform, i, FPToInt64(op, rounding_mode));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcvtu(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPRounding rounding_mode,
+ int fbits) {
+ dst.ClearForWrite(vform);
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ float op = src.Float<float>(i) * std::pow(2.0f, fbits);
+ dst.SetUint(vform, i, FPToUInt32(op, rounding_mode));
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ double op = src.Float<double>(i) * std::pow(2.0, fbits);
+ dst.SetUint(vform, i, FPToUInt64(op, rounding_mode));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcvtl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
+ // TODO: Full support for SimFloat16 in SimRegister(s).
+ dst.SetFloat(i,
+ FPToFloat(RawbitsToFloat16(src.Float<uint16_t>(i)),
+ ReadDN()));
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
+ dst.SetFloat(i, FPToDouble(src.Float<float>(i), ReadDN()));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcvtl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ int lane_count = LaneCountFromFormat(vform);
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ for (int i = 0; i < lane_count; i++) {
+ // TODO: Full support for SimFloat16 in SimRegister(s).
+ dst.SetFloat(i,
+ FPToFloat(RawbitsToFloat16(
+ src.Float<uint16_t>(i + lane_count)),
+ ReadDN()));
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ for (int i = 0; i < lane_count; i++) {
+ dst.SetFloat(i, FPToDouble(src.Float<float>(i + lane_count), ReadDN()));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcvtn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ SimVRegister tmp;
+ LogicVRegister srctmp = mov(kFormat2D, tmp, src);
+ dst.ClearForWrite(vform);
+ if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetFloat(i,
+ Float16ToRawbits(FPToFloat16(srctmp.Float<float>(i),
+ FPTieEven,
+ ReadDN())));
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetFloat(i, FPToFloat(srctmp.Float<double>(i), FPTieEven, ReadDN()));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcvtn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ int lane_count = LaneCountFromFormat(vform) / 2;
+ if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
+ for (int i = lane_count - 1; i >= 0; i--) {
+ dst.SetFloat(i + lane_count,
+ Float16ToRawbits(
+ FPToFloat16(src.Float<float>(i), FPTieEven, ReadDN())));
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
+ for (int i = lane_count - 1; i >= 0; i--) {
+ dst.SetFloat(i + lane_count,
+ FPToFloat(src.Float<double>(i), FPTieEven, ReadDN()));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcvtxn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ SimVRegister tmp;
+ LogicVRegister srctmp = mov(kFormat2D, tmp, src);
+ dst.ClearForWrite(vform);
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ dst.SetFloat(i, FPToFloat(srctmp.Float<double>(i), FPRoundOdd, ReadDN()));
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::fcvtxn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
+ int lane_count = LaneCountFromFormat(vform) / 2;
+ for (int i = lane_count - 1; i >= 0; i--) {
+ dst.SetFloat(i + lane_count,
+ FPToFloat(src.Float<double>(i), FPRoundOdd, ReadDN()));
+ }
+ return dst;
+}
+
+
+// Based on reference C function recip_sqrt_estimate from ARM ARM.
+double Simulator::recip_sqrt_estimate(double a) {
+ int q0, q1, s;
+ double r;
+ if (a < 0.5) {
+ q0 = static_cast<int>(a * 512.0);
+ r = 1.0 / sqrt((static_cast<double>(q0) + 0.5) / 512.0);
+ } else {
+ q1 = static_cast<int>(a * 256.0);
+ r = 1.0 / sqrt((static_cast<double>(q1) + 0.5) / 256.0);
+ }
+ s = static_cast<int>(256.0 * r + 0.5);
+ return static_cast<double>(s) / 256.0;
+}
+
+
+static inline uint64_t Bits(uint64_t val, int start_bit, int end_bit) {
+ return ExtractUnsignedBitfield64(start_bit, end_bit, val);
+}
+
+
+template <typename T>
+T Simulator::FPRecipSqrtEstimate(T op) {
+ if (std::isnan(op)) {
+ return FPProcessNaN(op);
+ } else if (op == 0.0) {
+ if (copysign(1.0, op) < 0.0) {
+ return kFP64NegativeInfinity;
+ } else {
+ return kFP64PositiveInfinity;
+ }
+ } else if (copysign(1.0, op) < 0.0) {
+ FPProcessException();
+ return FPDefaultNaN<T>();
+ } else if (std::isinf(op)) {
+ return 0.0;
+ } else {
+ uint64_t fraction;
+ int exp, result_exp;
+
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ exp = FloatExp(op);
+ fraction = FloatMantissa(op);
+ fraction <<= 29;
+ } else {
+ exp = DoubleExp(op);
+ fraction = DoubleMantissa(op);
+ }
+
+ if (exp == 0) {
+ while (Bits(fraction, 51, 51) == 0) {
+ fraction = Bits(fraction, 50, 0) << 1;
+ exp -= 1;
+ }
+ fraction = Bits(fraction, 50, 0) << 1;
+ }
+
+ double scaled;
+ if (Bits(exp, 0, 0) == 0) {
+ scaled = DoublePack(0, 1022, Bits(fraction, 51, 44) << 44);
+ } else {
+ scaled = DoublePack(0, 1021, Bits(fraction, 51, 44) << 44);
+ }
+
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ result_exp = (380 - exp) / 2;
+ } else {
+ result_exp = (3068 - exp) / 2;
+ }
+
+ uint64_t estimate = DoubleToRawbits(recip_sqrt_estimate(scaled));
+
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ uint32_t exp_bits = static_cast<uint32_t>(Bits(result_exp, 7, 0));
+ uint32_t est_bits = static_cast<uint32_t>(Bits(estimate, 51, 29));
+ return FloatPack(0, exp_bits, est_bits);
+ } else {
+ return DoublePack(0, Bits(result_exp, 10, 0), Bits(estimate, 51, 0));
+ }
+ }
+}
+
+
+LogicVRegister Simulator::frsqrte(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ float input = src.Float<float>(i);
+ dst.SetFloat(i, FPRecipSqrtEstimate<float>(input));
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ double input = src.Float<double>(i);
+ dst.SetFloat(i, FPRecipSqrtEstimate<double>(input));
+ }
+ }
+ return dst;
+}
+
+template <typename T>
+T Simulator::FPRecipEstimate(T op, FPRounding rounding) {
+ uint32_t sign;
+
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ sign = FloatSign(op);
+ } else {
+ sign = DoubleSign(op);
+ }
+
+ if (std::isnan(op)) {
+ return FPProcessNaN(op);
+ } else if (std::isinf(op)) {
+ return (sign == 1) ? -0.0 : 0.0;
+ } else if (op == 0.0) {
+ FPProcessException(); // FPExc_DivideByZero exception.
+ return (sign == 1) ? kFP64NegativeInfinity : kFP64PositiveInfinity;
+ } else if (((sizeof(T) == sizeof(float)) && // NOLINT(runtime/sizeof)
+ (std::fabs(op) < std::pow(2.0, -128.0))) ||
+ ((sizeof(T) == sizeof(double)) && // NOLINT(runtime/sizeof)
+ (std::fabs(op) < std::pow(2.0, -1024.0)))) {
+ bool overflow_to_inf = false;
+ switch (rounding) {
+ case FPTieEven: overflow_to_inf = true; break;
+ case FPPositiveInfinity: overflow_to_inf = (sign == 0); break;
+ case FPNegativeInfinity: overflow_to_inf = (sign == 1); break;
+ case FPZero: overflow_to_inf = false; break;
+ default: break;
+ }
+ FPProcessException(); // FPExc_Overflow and FPExc_Inexact.
+ if (overflow_to_inf) {
+ return (sign == 1) ? kFP64NegativeInfinity : kFP64PositiveInfinity;
+ } else {
+ // Return FPMaxNormal(sign).
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ return FloatPack(sign, 0xfe, 0x07fffff);
+ } else {
+ return DoublePack(sign, 0x7fe, 0x0fffffffffffffl);
+ }
+ }
+ } else {
+ uint64_t fraction;
+ int exp, result_exp;
+ uint32_t sign;
+
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ sign = FloatSign(op);
+ exp = FloatExp(op);
+ fraction = FloatMantissa(op);
+ fraction <<= 29;
+ } else {
+ sign = DoubleSign(op);
+ exp = DoubleExp(op);
+ fraction = DoubleMantissa(op);
+ }
+
+ if (exp == 0) {
+ if (Bits(fraction, 51, 51) == 0) {
+ exp -= 1;
+ fraction = Bits(fraction, 49, 0) << 2;
+ } else {
+ fraction = Bits(fraction, 50, 0) << 1;
+ }
+ }
+
+ double scaled = DoublePack(0, 1022, Bits(fraction, 51, 44) << 44);
+
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ result_exp = (253 - exp); // In range 253-254 = -1 to 253+1 = 254.
+ } else {
+ result_exp = (2045 - exp); // In range 2045-2046 = -1 to 2045+1 = 2046.
+ }
+
+ double estimate = recip_estimate(scaled);
+
+ fraction = DoubleMantissa(estimate);
+ if (result_exp == 0) {
+ fraction = (UINT64_C(1) << 51) | Bits(fraction, 51, 1);
+ } else if (result_exp == -1) {
+ fraction = (UINT64_C(1) << 50) | Bits(fraction, 51, 2);
+ result_exp = 0;
+ }
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ uint32_t exp_bits = static_cast<uint32_t>(Bits(result_exp, 7, 0));
+ uint32_t frac_bits = static_cast<uint32_t>(Bits(fraction, 51, 29));
+ return FloatPack(sign, exp_bits, frac_bits);
+ } else {
+ return DoublePack(sign, Bits(result_exp, 10, 0), Bits(fraction, 51, 0));
+ }
+ }
+}
+
+
+LogicVRegister Simulator::frecpe(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPRounding round) {
+ dst.ClearForWrite(vform);
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ float input = src.Float<float>(i);
+ dst.SetFloat(i, FPRecipEstimate<float>(input, round));
+ }
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ double input = src.Float<double>(i);
+ dst.SetFloat(i, FPRecipEstimate<double>(input, round));
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::ursqrte(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ uint64_t operand;
+ uint32_t result;
+ double dp_operand, dp_result;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ operand = src.Uint(vform, i);
+ if (operand <= 0x3FFFFFFF) {
+ result = 0xFFFFFFFF;
+ } else {
+ dp_operand = operand * std::pow(2.0, -32);
+ dp_result = recip_sqrt_estimate(dp_operand) * std::pow(2.0, 31);
+ result = static_cast<uint32_t>(dp_result);
+ }
+ dst.SetUint(vform, i, result);
+ }
+ return dst;
+}
+
+
+// Based on reference C function recip_estimate from ARM ARM.
+double Simulator::recip_estimate(double a) {
+ int q, s;
+ double r;
+ q = static_cast<int>(a * 512.0);
+ r = 1.0 / ((static_cast<double>(q) + 0.5) / 512.0);
+ s = static_cast<int>(256.0 * r + 0.5);
+ return static_cast<double>(s) / 256.0;
+}
+
+
+LogicVRegister Simulator::urecpe(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ uint64_t operand;
+ uint32_t result;
+ double dp_operand, dp_result;
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ operand = src.Uint(vform, i);
+ if (operand <= 0x7FFFFFFF) {
+ result = 0xFFFFFFFF;
+ } else {
+ dp_operand = operand * std::pow(2.0, -32);
+ dp_result = recip_estimate(dp_operand) * std::pow(2.0, 31);
+ result = static_cast<uint32_t>(dp_result);
+ }
+ dst.SetUint(vform, i, result);
+ }
+ return dst;
+}
+
+template <typename T>
+LogicVRegister Simulator::frecpx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ dst.ClearForWrite(vform);
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ T op = src.Float<T>(i);
+ T result;
+ if (std::isnan(op)) {
+ result = FPProcessNaN(op);
+ } else {
+ int exp;
+ uint32_t sign;
+ if (sizeof(T) == sizeof(float)) { // NOLINT(runtime/sizeof)
+ sign = FloatSign(op);
+ exp = FloatExp(op);
+ exp = (exp == 0) ? (0xFF - 1) : static_cast<int>(Bits(~exp, 7, 0));
+ result = FloatPack(sign, exp, 0);
+ } else {
+ sign = DoubleSign(op);
+ exp = DoubleExp(op);
+ exp = (exp == 0) ? (0x7FF - 1) : static_cast<int>(Bits(~exp, 10, 0));
+ result = DoublePack(sign, exp, 0);
+ }
+ }
+ dst.SetFloat(i, result);
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::frecpx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ frecpx<float>(vform, dst, src);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ frecpx<double>(vform, dst, src);
+ }
+ return dst;
+}
+
+LogicVRegister Simulator::scvtf(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int fbits,
+ FPRounding round) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ float result = FixedToFloat(src.Int(kFormatS, i), fbits, round);
+ dst.SetFloat<float>(i, result);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ double result = FixedToDouble(src.Int(kFormatD, i), fbits, round);
+ dst.SetFloat<double>(i, result);
+ }
+ }
+ return dst;
+}
+
+
+LogicVRegister Simulator::ucvtf(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int fbits,
+ FPRounding round) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
+ float result = UFixedToFloat(src.Uint(kFormatS, i), fbits, round);
+ dst.SetFloat<float>(i, result);
+ } else {
+ VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
+ double result = UFixedToDouble(src.Uint(kFormatD, i), fbits, round);
+ dst.SetFloat<double>(i, result);
+ }
+ }
+ return dst;
+}
+
+
+} // namespace vixl
+
+#endif // JS_SIMULATOR_ARM64
diff --git a/js/src/jit/arm64/vixl/MacroAssembler-vixl.cpp b/js/src/jit/arm64/vixl/MacroAssembler-vixl.cpp
new file mode 100644
index 0000000000..5c4a5ce145
--- /dev/null
+++ b/js/src/jit/arm64/vixl/MacroAssembler-vixl.cpp
@@ -0,0 +1,2027 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/MacroAssembler-vixl.h"
+
+#include <ctype.h>
+
+namespace vixl {
+
+MacroAssembler::MacroAssembler()
+ : js::jit::Assembler(),
+ sp_(x28),
+ tmp_list_(ip0, ip1),
+ fptmp_list_(d31)
+{
+}
+
+
+void MacroAssembler::FinalizeCode() {
+ Assembler::FinalizeCode();
+}
+
+
+int MacroAssembler::MoveImmediateHelper(MacroAssembler* masm,
+ const Register &rd,
+ uint64_t imm) {
+ bool emit_code = (masm != NULL);
+ VIXL_ASSERT(IsUint32(imm) || IsInt32(imm) || rd.Is64Bits());
+ // The worst case for size is mov 64-bit immediate to sp:
+ // * up to 4 instructions to materialise the constant
+ // * 1 instruction to move to sp
+ MacroEmissionCheckScope guard(masm);
+
+ // Immediates on Aarch64 can be produced using an initial value, and zero to
+ // three move keep operations.
+ //
+ // Initial values can be generated with:
+ // 1. 64-bit move zero (movz).
+ // 2. 32-bit move inverted (movn).
+ // 3. 64-bit move inverted.
+ // 4. 32-bit orr immediate.
+ // 5. 64-bit orr immediate.
+ // Move-keep may then be used to modify each of the 16-bit half words.
+ //
+ // The code below supports all five initial value generators, and
+ // applying move-keep operations to move-zero and move-inverted initial
+ // values.
+
+ // Try to move the immediate in one instruction, and if that fails, switch to
+ // using multiple instructions.
+ if (OneInstrMoveImmediateHelper(masm, rd, imm)) {
+ return 1;
+ } else {
+ int instruction_count = 0;
+ unsigned reg_size = rd.size();
+
+ // Generic immediate case. Imm will be represented by
+ // [imm3, imm2, imm1, imm0], where each imm is 16 bits.
+ // A move-zero or move-inverted is generated for the first non-zero or
+ // non-0xffff immX, and a move-keep for subsequent non-zero immX.
+
+ uint64_t ignored_halfword = 0;
+ bool invert_move = false;
+ // If the number of 0xffff halfwords is greater than the number of 0x0000
+ // halfwords, it's more efficient to use move-inverted.
+ if (CountClearHalfWords(~imm, reg_size) >
+ CountClearHalfWords(imm, reg_size)) {
+ ignored_halfword = 0xffff;
+ invert_move = true;
+ }
+
+ // Mov instructions can't move values into the stack pointer, so set up a
+ // temporary register, if needed.
+ UseScratchRegisterScope temps;
+ Register temp;
+ if (emit_code) {
+ temps.Open(masm);
+ temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
+ }
+
+ // Iterate through the halfwords. Use movn/movz for the first non-ignored
+ // halfword, and movk for subsequent halfwords.
+ VIXL_ASSERT((reg_size % 16) == 0);
+ bool first_mov_done = false;
+ for (unsigned i = 0; i < (temp.size() / 16); i++) {
+ uint64_t imm16 = (imm >> (16 * i)) & 0xffff;
+ if (imm16 != ignored_halfword) {
+ if (!first_mov_done) {
+ if (invert_move) {
+ if (emit_code) masm->movn(temp, ~imm16 & 0xffff, 16 * i);
+ instruction_count++;
+ } else {
+ if (emit_code) masm->movz(temp, imm16, 16 * i);
+ instruction_count++;
+ }
+ first_mov_done = true;
+ } else {
+ // Construct a wider constant.
+ if (emit_code) masm->movk(temp, imm16, 16 * i);
+ instruction_count++;
+ }
+ }
+ }
+
+ VIXL_ASSERT(first_mov_done);
+
+ // Move the temporary if the original destination register was the stack
+ // pointer.
+ if (rd.IsSP()) {
+ if (emit_code) masm->mov(rd, temp);
+ instruction_count++;
+ }
+ return instruction_count;
+ }
+}
+
+
+bool MacroAssembler::OneInstrMoveImmediateHelper(MacroAssembler* masm,
+ const Register& dst,
+ int64_t imm) {
+ bool emit_code = masm != NULL;
+ unsigned n, imm_s, imm_r;
+ int reg_size = dst.size();
+
+ if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
+ // Immediate can be represented in a move zero instruction. Movz can't write
+ // to the stack pointer.
+ if (emit_code) {
+ masm->movz(dst, imm);
+ }
+ return true;
+ } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
+ // Immediate can be represented in a move negative instruction. Movn can't
+ // write to the stack pointer.
+ if (emit_code) {
+ masm->movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
+ }
+ return true;
+ } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
+ // Immediate can be represented in a logical orr instruction.
+ VIXL_ASSERT(!dst.IsZero());
+ if (emit_code) {
+ masm->LogicalImmediate(
+ dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR);
+ }
+ return true;
+ }
+ return false;
+}
+
+
+void MacroAssembler::B(Label* label, BranchType type, Register reg, int bit) {
+ VIXL_ASSERT((reg.Is(NoReg) || (type >= kBranchTypeFirstUsingReg)) &&
+ ((bit == -1) || (type >= kBranchTypeFirstUsingBit)));
+ if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
+ B(static_cast<Condition>(type), label);
+ } else {
+ switch (type) {
+ case always: B(label); break;
+ case never: break;
+ case reg_zero: Cbz(reg, label); break;
+ case reg_not_zero: Cbnz(reg, label); break;
+ case reg_bit_clear: Tbz(reg, bit, label); break;
+ case reg_bit_set: Tbnz(reg, bit, label); break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+ }
+}
+
+
+void MacroAssembler::B(Label* label) {
+ SingleEmissionCheckScope guard(this);
+ b(label);
+}
+
+
+void MacroAssembler::B(Label* label, Condition cond) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ EmissionCheckScope guard(this, 2 * kInstructionSize);
+
+ if (label->bound() && LabelIsOutOfRange(label, CondBranchType)) {
+ Label done;
+ b(&done, InvertCondition(cond));
+ b(label);
+ bind(&done);
+ } else {
+ b(label, cond);
+ }
+}
+
+
+void MacroAssembler::Cbnz(const Register& rt, Label* label) {
+ VIXL_ASSERT(!rt.IsZero());
+ EmissionCheckScope guard(this, 2 * kInstructionSize);
+
+ if (label->bound() && LabelIsOutOfRange(label, CondBranchType)) {
+ Label done;
+ cbz(rt, &done);
+ b(label);
+ bind(&done);
+ } else {
+ cbnz(rt, label);
+ }
+}
+
+
+void MacroAssembler::Cbz(const Register& rt, Label* label) {
+ VIXL_ASSERT(!rt.IsZero());
+ EmissionCheckScope guard(this, 2 * kInstructionSize);
+
+ if (label->bound() && LabelIsOutOfRange(label, CondBranchType)) {
+ Label done;
+ cbnz(rt, &done);
+ b(label);
+ bind(&done);
+ } else {
+ cbz(rt, label);
+ }
+}
+
+
+void MacroAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
+ VIXL_ASSERT(!rt.IsZero());
+ EmissionCheckScope guard(this, 2 * kInstructionSize);
+
+ if (label->bound() && LabelIsOutOfRange(label, TestBranchType)) {
+ Label done;
+ tbz(rt, bit_pos, &done);
+ b(label);
+ bind(&done);
+ } else {
+ tbnz(rt, bit_pos, label);
+ }
+}
+
+
+void MacroAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
+ VIXL_ASSERT(!rt.IsZero());
+ EmissionCheckScope guard(this, 2 * kInstructionSize);
+
+ if (label->bound() && LabelIsOutOfRange(label, TestBranchType)) {
+ Label done;
+ tbnz(rt, bit_pos, &done);
+ b(label);
+ bind(&done);
+ } else {
+ tbz(rt, bit_pos, label);
+ }
+}
+
+
+void MacroAssembler::And(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ LogicalMacro(rd, rn, operand, AND);
+}
+
+
+void MacroAssembler::Ands(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ LogicalMacro(rd, rn, operand, ANDS);
+}
+
+
+void MacroAssembler::Tst(const Register& rn,
+ const Operand& operand) {
+ Ands(AppropriateZeroRegFor(rn), rn, operand);
+}
+
+
+void MacroAssembler::Bic(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ LogicalMacro(rd, rn, operand, BIC);
+}
+
+
+void MacroAssembler::Bics(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ LogicalMacro(rd, rn, operand, BICS);
+}
+
+
+void MacroAssembler::Orr(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ LogicalMacro(rd, rn, operand, ORR);
+}
+
+
+void MacroAssembler::Orn(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ LogicalMacro(rd, rn, operand, ORN);
+}
+
+
+void MacroAssembler::Eor(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ LogicalMacro(rd, rn, operand, EOR);
+}
+
+
+void MacroAssembler::Eon(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ LogicalMacro(rd, rn, operand, EON);
+}
+
+
+void MacroAssembler::LogicalMacro(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ LogicalOp op) {
+ // The worst case for size is logical immediate to sp:
+ // * up to 4 instructions to materialise the constant
+ // * 1 instruction to do the operation
+ // * 1 instruction to move to sp
+ MacroEmissionCheckScope guard(this);
+ UseScratchRegisterScope temps(this);
+
+ if (operand.IsImmediate()) {
+ int64_t immediate = operand.immediate();
+ unsigned reg_size = rd.size();
+
+ // If the operation is NOT, invert the operation and immediate.
+ if ((op & NOT) == NOT) {
+ op = static_cast<LogicalOp>(op & ~NOT);
+ immediate = ~immediate;
+ }
+
+ // Ignore the top 32 bits of an immediate if we're moving to a W register.
+ if (rd.Is32Bits()) {
+ // Check that the top 32 bits are consistent.
+ VIXL_ASSERT(((immediate >> kWRegSize) == 0) ||
+ ((immediate >> kWRegSize) == -1));
+ immediate &= kWRegMask;
+ }
+
+ VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));
+
+ // Special cases for all set or all clear immediates.
+ if (immediate == 0) {
+ switch (op) {
+ case AND:
+ Mov(rd, 0);
+ return;
+ case ORR:
+ VIXL_FALLTHROUGH();
+ case EOR:
+ Mov(rd, rn);
+ return;
+ case ANDS:
+ VIXL_FALLTHROUGH();
+ case BICS:
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+ } else if ((rd.Is64Bits() && (immediate == -1)) ||
+ (rd.Is32Bits() && (immediate == 0xffffffff))) {
+ switch (op) {
+ case AND:
+ Mov(rd, rn);
+ return;
+ case ORR:
+ Mov(rd, immediate);
+ return;
+ case EOR:
+ Mvn(rd, rn);
+ return;
+ case ANDS:
+ VIXL_FALLTHROUGH();
+ case BICS:
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+ }
+
+ unsigned n, imm_s, imm_r;
+ if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
+ // Immediate can be encoded in the instruction.
+ LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
+ } else {
+ // Immediate can't be encoded: synthesize using move immediate.
+ Register temp = temps.AcquireSameSizeAs(rn);
+
+ // If the left-hand input is the stack pointer, we can't pre-shift the
+ // immediate, as the encoding won't allow the subsequent post shift.
+ PreShiftImmMode mode = rn.IsSP() ? kNoShift : kAnyShift;
+ Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode);
+
+ // VIXL can acquire temp registers. Assert that the caller is aware.
+ VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn));
+ VIXL_ASSERT(!temp.Is(operand.maybeReg()));
+
+ if (rd.Is(sp)) {
+ // If rd is the stack pointer we cannot use it as the destination
+ // register so we use the temp register as an intermediate again.
+ Logical(temp, rn, imm_operand, op);
+ Mov(sp, temp);
+ } else {
+ Logical(rd, rn, imm_operand, op);
+ }
+ }
+ } else if (operand.IsExtendedRegister()) {
+ VIXL_ASSERT(operand.reg().size() <= rd.size());
+ // Add/sub extended supports shift <= 4. We want to support exactly the
+ // same modes here.
+ VIXL_ASSERT(operand.shift_amount() <= 4);
+ VIXL_ASSERT(operand.reg().Is64Bits() ||
+ ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
+
+ temps.Exclude(operand.reg());
+ Register temp = temps.AcquireSameSizeAs(rn);
+
+ // VIXL can acquire temp registers. Assert that the caller is aware.
+ VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn));
+ VIXL_ASSERT(!temp.Is(operand.maybeReg()));
+
+ EmitExtendShift(temp, operand.reg(), operand.extend(),
+ operand.shift_amount());
+ Logical(rd, rn, Operand(temp), op);
+ } else {
+ // The operand can be encoded in the instruction.
+ VIXL_ASSERT(operand.IsShiftedRegister());
+ Logical(rd, rn, operand, op);
+ }
+}
+
+
+void MacroAssembler::Mov(const Register& rd,
+ const Operand& operand,
+ DiscardMoveMode discard_mode) {
+ // The worst case for size is mov immediate with up to 4 instructions.
+ MacroEmissionCheckScope guard(this);
+
+ if (operand.IsImmediate()) {
+ // Call the macro assembler for generic immediates.
+ Mov(rd, operand.immediate());
+ } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
+ // Emit a shift instruction if moving a shifted register. This operation
+ // could also be achieved using an orr instruction (like orn used by Mvn),
+ // but using a shift instruction makes the disassembly clearer.
+ EmitShift(rd, operand.reg(), operand.shift(), operand.shift_amount());
+ } else if (operand.IsExtendedRegister()) {
+ // Emit an extend instruction if moving an extended register. This handles
+ // extend with post-shift operations, too.
+ EmitExtendShift(rd, operand.reg(), operand.extend(),
+ operand.shift_amount());
+ } else {
+ // Otherwise, emit a register move only if the registers are distinct, or
+ // if they are not X registers.
+ //
+ // Note that mov(w0, w0) is not a no-op because it clears the top word of
+ // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W
+ // registers is not required to clear the top word of the X register. In
+ // this case, the instruction is discarded.
+ //
+ // If the sp is an operand, add #0 is emitted, otherwise, orr #0.
+ if (!rd.Is(operand.reg()) || (rd.Is32Bits() &&
+ (discard_mode == kDontDiscardForSameWReg))) {
+ mov(rd, operand.reg());
+ }
+ }
+}
+
+
+void MacroAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
+ VIXL_ASSERT(IsUint16(imm));
+ int byte1 = (imm & 0xff);
+ int byte2 = ((imm >> 8) & 0xff);
+ if (byte1 == byte2) {
+ movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
+ } else if (byte1 == 0) {
+ movi(vd, byte2, LSL, 8);
+ } else if (byte2 == 0) {
+ movi(vd, byte1);
+ } else if (byte1 == 0xff) {
+ mvni(vd, ~byte2 & 0xff, LSL, 8);
+ } else if (byte2 == 0xff) {
+ mvni(vd, ~byte1 & 0xff);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireW();
+ movz(temp, imm);
+ dup(vd, temp);
+ }
+}
+
+
+void MacroAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
+ VIXL_ASSERT(IsUint32(imm));
+
+ uint8_t bytes[sizeof(imm)];
+ memcpy(bytes, &imm, sizeof(imm));
+
+ // All bytes are either 0x00 or 0xff.
+ {
+ bool all0orff = true;
+ for (int i = 0; i < 4; ++i) {
+ if ((bytes[i] != 0) && (bytes[i] != 0xff)) {
+ all0orff = false;
+ break;
+ }
+ }
+
+ if (all0orff == true) {
+ movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
+ return;
+ }
+ }
+
+ // Of the 4 bytes, only one byte is non-zero.
+ for (int i = 0; i < 4; i++) {
+ if ((imm & (0xff << (i * 8))) == imm) {
+ movi(vd, bytes[i], LSL, i * 8);
+ return;
+ }
+ }
+
+ // Of the 4 bytes, only one byte is not 0xff.
+ for (int i = 0; i < 4; i++) {
+ uint32_t mask = ~(0xff << (i * 8));
+ if ((imm & mask) == mask) {
+ mvni(vd, ~bytes[i] & 0xff, LSL, i * 8);
+ return;
+ }
+ }
+
+ // Immediate is of the form 0x00MMFFFF.
+ if ((imm & 0xff00ffff) == 0x0000ffff) {
+ movi(vd, bytes[2], MSL, 16);
+ return;
+ }
+
+ // Immediate is of the form 0x0000MMFF.
+ if ((imm & 0xffff00ff) == 0x000000ff) {
+ movi(vd, bytes[1], MSL, 8);
+ return;
+ }
+
+ // Immediate is of the form 0xFFMM0000.
+ if ((imm & 0xff00ffff) == 0xff000000) {
+ mvni(vd, ~bytes[2] & 0xff, MSL, 16);
+ return;
+ }
+ // Immediate is of the form 0xFFFFMM00.
+ if ((imm & 0xffff00ff) == 0xffff0000) {
+ mvni(vd, ~bytes[1] & 0xff, MSL, 8);
+ return;
+ }
+
+ // Top and bottom 16-bits are equal.
+ if (((imm >> 16) & 0xffff) == (imm & 0xffff)) {
+ Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xffff);
+ return;
+ }
+
+ // Default case.
+ {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireW();
+ Mov(temp, imm);
+ dup(vd, temp);
+ }
+}
+
+
+void MacroAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
+ // All bytes are either 0x00 or 0xff.
+ {
+ bool all0orff = true;
+ for (int i = 0; i < 8; ++i) {
+ int byteval = (imm >> (i * 8)) & 0xff;
+ if (byteval != 0 && byteval != 0xff) {
+ all0orff = false;
+ break;
+ }
+ }
+ if (all0orff == true) {
+ movi(vd, imm);
+ return;
+ }
+ }
+
+ // Top and bottom 32-bits are equal.
+ if (((imm >> 32) & 0xffffffff) == (imm & 0xffffffff)) {
+ Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xffffffff);
+ return;
+ }
+
+ // Default case.
+ {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireX();
+ Mov(temp, imm);
+ if (vd.Is1D()) {
+ mov(vd.D(), 0, temp);
+ } else {
+ dup(vd.V2D(), temp);
+ }
+ }
+}
+
+
+void MacroAssembler::Movi(const VRegister& vd,
+ uint64_t imm,
+ Shift shift,
+ int shift_amount) {
+ MacroEmissionCheckScope guard(this);
+ if (shift_amount != 0 || shift != LSL) {
+ movi(vd, imm, shift, shift_amount);
+ } else if (vd.Is8B() || vd.Is16B()) {
+ // 8-bit immediate.
+ VIXL_ASSERT(IsUint8(imm));
+ movi(vd, imm);
+ } else if (vd.Is4H() || vd.Is8H()) {
+ // 16-bit immediate.
+ Movi16bitHelper(vd, imm);
+ } else if (vd.Is2S() || vd.Is4S()) {
+ // 32-bit immediate.
+ Movi32bitHelper(vd, imm);
+ } else {
+ // 64-bit immediate.
+ Movi64bitHelper(vd, imm);
+ }
+}
+
+
+void MacroAssembler::Movi(const VRegister& vd,
+ uint64_t hi,
+ uint64_t lo) {
+ VIXL_ASSERT(vd.Is128Bits());
+ UseScratchRegisterScope temps(this);
+
+ // When hi == lo, the following generates good code.
+ //
+ // In situations where the constants are complex and hi != lo, the following
+ // can turn into up to 10 instructions: 2*(mov + 3*movk + dup/insert). To do
+ // any better, we could try to estimate whether splatting the high value and
+ // updating the low value would generate fewer instructions than vice versa
+ // (what we do now).
+ //
+ // (A PC-relative load from memory to the vector register (ADR + LD2) is going
+ // to have fairly high latency but is fairly compact; not clear what the best
+ // tradeoff is.)
+
+ Movi(vd.V2D(), lo);
+ if (hi != lo) {
+ Register temp = temps.AcquireX();
+ Mov(temp, hi);
+ Ins(vd.V2D(), 1, temp);
+ }
+}
+
+
+void MacroAssembler::Mvn(const Register& rd, const Operand& operand) {
+ // The worst case for size is mvn immediate with up to 4 instructions.
+ MacroEmissionCheckScope guard(this);
+
+ if (operand.IsImmediate()) {
+ // Call the macro assembler for generic immediates.
+ Mvn(rd, operand.immediate());
+ } else if (operand.IsExtendedRegister()) {
+ UseScratchRegisterScope temps(this);
+ temps.Exclude(operand.reg());
+
+ // Emit two instructions for the extend case. This differs from Mov, as
+ // the extend and invert can't be achieved in one instruction.
+ Register temp = temps.AcquireSameSizeAs(rd);
+
+ // VIXL can acquire temp registers. Assert that the caller is aware.
+ VIXL_ASSERT(!temp.Is(rd) && !temp.Is(operand.maybeReg()));
+
+ EmitExtendShift(temp, operand.reg(), operand.extend(),
+ operand.shift_amount());
+ mvn(rd, Operand(temp));
+ } else {
+ // Otherwise, register and shifted register cases can be handled by the
+ // assembler directly, using orn.
+ mvn(rd, operand);
+ }
+}
+
+
+void MacroAssembler::Mov(const Register& rd, uint64_t imm) {
+ MoveImmediateHelper(this, rd, imm);
+}
+
+
+void MacroAssembler::Ccmp(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond) {
+ if (operand.IsImmediate() && (operand.immediate() < 0)) {
+ ConditionalCompareMacro(rn, -operand.immediate(), nzcv, cond, CCMN);
+ } else {
+ ConditionalCompareMacro(rn, operand, nzcv, cond, CCMP);
+ }
+}
+
+
+void MacroAssembler::Ccmn(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond) {
+ if (operand.IsImmediate() && (operand.immediate() < 0)) {
+ ConditionalCompareMacro(rn, -operand.immediate(), nzcv, cond, CCMP);
+ } else {
+ ConditionalCompareMacro(rn, operand, nzcv, cond, CCMN);
+ }
+}
+
+
+void MacroAssembler::ConditionalCompareMacro(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond,
+ ConditionalCompareOp op) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ // The worst case for size is ccmp immediate:
+ // * up to 4 instructions to materialise the constant
+ // * 1 instruction for ccmp
+ MacroEmissionCheckScope guard(this);
+
+ if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) ||
+ (operand.IsImmediate() && IsImmConditionalCompare(operand.immediate()))) {
+ // The immediate can be encoded in the instruction, or the operand is an
+ // unshifted register: call the assembler.
+ ConditionalCompare(rn, operand, nzcv, cond, op);
+ } else {
+ UseScratchRegisterScope temps(this);
+ // The operand isn't directly supported by the instruction: perform the
+ // operation on a temporary register.
+ Register temp = temps.AcquireSameSizeAs(rn);
+ VIXL_ASSERT(!temp.Is(rn) && !temp.Is(operand.maybeReg()));
+ Mov(temp, operand);
+ ConditionalCompare(rn, temp, nzcv, cond, op);
+ }
+}
+
+
+void MacroAssembler::Csel(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ // The worst case for size is csel immediate:
+ // * up to 4 instructions to materialise the constant
+ // * 1 instruction for csel
+ MacroEmissionCheckScope guard(this);
+
+ if (operand.IsImmediate()) {
+ // Immediate argument. Handle special cases of 0, 1 and -1 using zero
+ // register.
+ int64_t imm = operand.immediate();
+ Register zr = AppropriateZeroRegFor(rn);
+ if (imm == 0) {
+ csel(rd, rn, zr, cond);
+ } else if (imm == 1) {
+ csinc(rd, rn, zr, cond);
+ } else if (imm == -1) {
+ csinv(rd, rn, zr, cond);
+ } else {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(rn);
+ VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn));
+ VIXL_ASSERT(!temp.Is(operand.maybeReg()));
+ Mov(temp, operand.immediate());
+ csel(rd, rn, temp, cond);
+ }
+ } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) {
+ // Unshifted register argument.
+ csel(rd, rn, operand.reg(), cond);
+ } else {
+ // All other arguments.
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(rn);
+ VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn));
+ VIXL_ASSERT(!temp.Is(operand.maybeReg()));
+ Mov(temp, operand);
+ csel(rd, rn, temp, cond);
+ }
+}
+
+
+void MacroAssembler::Add(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S) {
+ if (operand.IsImmediate() && (operand.immediate() < 0) &&
+ IsImmAddSub(-operand.immediate())) {
+ AddSubMacro(rd, rn, -operand.immediate(), S, SUB);
+ } else {
+ AddSubMacro(rd, rn, operand, S, ADD);
+ }
+}
+
+
+void MacroAssembler::Adds(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Add(rd, rn, operand, SetFlags);
+}
+
+
+void MacroAssembler::Sub(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S) {
+ if (operand.IsImmediate() && (operand.immediate() < 0) &&
+ IsImmAddSub(-operand.immediate())) {
+ AddSubMacro(rd, rn, -operand.immediate(), S, ADD);
+ } else {
+ AddSubMacro(rd, rn, operand, S, SUB);
+ }
+}
+
+
+void MacroAssembler::Subs(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ Sub(rd, rn, operand, SetFlags);
+}
+
+
+void MacroAssembler::Cmn(const Register& rn, const Operand& operand) {
+ Adds(AppropriateZeroRegFor(rn), rn, operand);
+}
+
+
+void MacroAssembler::Cmp(const Register& rn, const Operand& operand) {
+ Subs(AppropriateZeroRegFor(rn), rn, operand);
+}
+
+
+void MacroAssembler::Fcmp(const FPRegister& fn, double value,
+ FPTrapFlags trap) {
+ // The worst case for size is:
+ // * 1 to materialise the constant, using literal pool if necessary
+ // * 1 instruction for fcmp{e}
+ MacroEmissionCheckScope guard(this);
+ if (value != 0.0) {
+ UseScratchRegisterScope temps(this);
+ FPRegister tmp = temps.AcquireSameSizeAs(fn);
+ VIXL_ASSERT(!tmp.Is(fn));
+ Fmov(tmp, value);
+ FPCompareMacro(fn, tmp, trap);
+ } else {
+ FPCompareMacro(fn, value, trap);
+ }
+}
+
+
+void MacroAssembler::Fcmpe(const FPRegister& fn, double value) {
+ Fcmp(fn, value, EnableTrap);
+}
+
+
+void MacroAssembler::Fmov(VRegister vd, double imm) {
+ // Floating point immediates are loaded through the literal pool.
+ MacroEmissionCheckScope guard(this);
+
+ if (vd.Is1S() || vd.Is2S() || vd.Is4S()) {
+ Fmov(vd, static_cast<float>(imm));
+ return;
+ }
+
+ VIXL_ASSERT(vd.Is1D() || vd.Is2D());
+ if (IsImmFP64(imm)) {
+ fmov(vd, imm);
+ } else {
+ uint64_t rawbits = DoubleToRawbits(imm);
+ if (vd.IsScalar()) {
+ if (rawbits == 0) {
+ fmov(vd, xzr);
+ } else {
+ Assembler::fImmPool64(vd, imm);
+ }
+ } else {
+ // TODO: consider NEON support for load literal.
+ Movi(vd, rawbits);
+ }
+ }
+}
+
+
+void MacroAssembler::Fmov(VRegister vd, float imm) {
+ // Floating point immediates are loaded through the literal pool.
+ MacroEmissionCheckScope guard(this);
+
+ if (vd.Is1D() || vd.Is2D()) {
+ Fmov(vd, static_cast<double>(imm));
+ return;
+ }
+
+ VIXL_ASSERT(vd.Is1S() || vd.Is2S() || vd.Is4S());
+ if (IsImmFP32(imm)) {
+ fmov(vd, imm);
+ } else {
+ uint32_t rawbits = FloatToRawbits(imm);
+ if (vd.IsScalar()) {
+ if (rawbits == 0) {
+ fmov(vd, wzr);
+ } else {
+ Assembler::fImmPool32(vd, imm);
+ }
+ } else {
+ // TODO: consider NEON support for load literal.
+ Movi(vd, rawbits);
+ }
+ }
+}
+
+
+
+void MacroAssembler::Neg(const Register& rd,
+ const Operand& operand) {
+ if (operand.IsImmediate()) {
+ Mov(rd, -operand.immediate());
+ } else {
+ Sub(rd, AppropriateZeroRegFor(rd), operand);
+ }
+}
+
+
+void MacroAssembler::Negs(const Register& rd,
+ const Operand& operand) {
+ Subs(rd, AppropriateZeroRegFor(rd), operand);
+}
+
+
+bool MacroAssembler::TryOneInstrMoveImmediate(const Register& dst,
+ int64_t imm) {
+ return OneInstrMoveImmediateHelper(this, dst, imm);
+}
+
+
+Operand MacroAssembler::MoveImmediateForShiftedOp(const Register& dst,
+ int64_t imm,
+ PreShiftImmMode mode) {
+ int reg_size = dst.size();
+
+ // Encode the immediate in a single move instruction, if possible.
+ if (TryOneInstrMoveImmediate(dst, imm)) {
+ // The move was successful; nothing to do here.
+ } else {
+ // Pre-shift the immediate to the least-significant bits of the register.
+ int shift_low = CountTrailingZeros(imm, reg_size);
+ if (mode == kLimitShiftForSP) {
+ // When applied to the stack pointer, the subsequent arithmetic operation
+ // can use the extend form to shift left by a maximum of four bits. Right
+ // shifts are not allowed, so we filter them out later before the new
+ // immediate is tested.
+ shift_low = std::min(shift_low, 4);
+ }
+
+ int64_t imm_low = imm >> shift_low;
+
+ // Pre-shift the immediate to the most-significant bits of the register,
+ // inserting set bits in the least-significant bits.
+ int shift_high = CountLeadingZeros(imm, reg_size);
+ int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1);
+
+ if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) {
+ // The new immediate has been moved into the destination's low bits:
+ // return a new leftward-shifting operand.
+ return Operand(dst, LSL, shift_low);
+ } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) {
+ // The new immediate has been moved into the destination's high bits:
+ // return a new rightward-shifting operand.
+ return Operand(dst, LSR, shift_high);
+ } else {
+ Mov(dst, imm);
+ }
+ }
+ return Operand(dst);
+}
+
+
+void MacroAssembler::ComputeAddress(const Register& dst,
+ const MemOperand& mem_op) {
+ // We cannot handle pre-indexing or post-indexing.
+ VIXL_ASSERT(mem_op.addrmode() == Offset);
+ Register base = mem_op.base();
+ if (mem_op.IsImmediateOffset()) {
+ Add(dst, base, mem_op.offset());
+ } else {
+ VIXL_ASSERT(mem_op.IsRegisterOffset());
+ Register reg_offset = mem_op.regoffset();
+ Shift shift = mem_op.shift();
+ Extend extend = mem_op.extend();
+ if (shift == NO_SHIFT) {
+ VIXL_ASSERT(extend != NO_EXTEND);
+ Add(dst, base, Operand(reg_offset, extend, mem_op.shift_amount()));
+ } else {
+ VIXL_ASSERT(extend == NO_EXTEND);
+ Add(dst, base, Operand(reg_offset, shift, mem_op.shift_amount()));
+ }
+ }
+}
+
+
+void MacroAssembler::AddSubMacro(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ AddSubOp op) {
+ // Worst case is add/sub immediate:
+ // * up to 4 instructions to materialise the constant
+ // * 1 instruction for add/sub
+ MacroEmissionCheckScope guard(this);
+
+ if (operand.IsZero() && rd.Is(rn) && rd.Is64Bits() && rn.Is64Bits() &&
+ (S == LeaveFlags)) {
+ // The instruction would be a nop. Avoid generating useless code.
+ return;
+ }
+
+ if ((operand.IsImmediate() && !IsImmAddSub(operand.immediate())) ||
+ (rn.IsZero() && !operand.IsShiftedRegister()) ||
+ (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(rn);
+ if (operand.IsImmediate()) {
+ PreShiftImmMode mode = kAnyShift;
+
+ // If the destination or source register is the stack pointer, we can
+ // only pre-shift the immediate right by values supported in the add/sub
+ // extend encoding.
+ if (rd.IsSP()) {
+ // If the destination is SP and flags will be set, we can't pre-shift
+ // the immediate at all.
+ mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP;
+ } else if (rn.IsSP()) {
+ mode = kLimitShiftForSP;
+ }
+
+ Operand imm_operand =
+ MoveImmediateForShiftedOp(temp, operand.immediate(), mode);
+ AddSub(rd, rn, imm_operand, S, op);
+ } else {
+ Mov(temp, operand);
+ AddSub(rd, rn, temp, S, op);
+ }
+ } else {
+ AddSub(rd, rn, operand, S, op);
+ }
+}
+
+
+void MacroAssembler::Adc(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, ADC);
+}
+
+
+void MacroAssembler::Adcs(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarryMacro(rd, rn, operand, SetFlags, ADC);
+}
+
+
+void MacroAssembler::Sbc(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarryMacro(rd, rn, operand, LeaveFlags, SBC);
+}
+
+
+void MacroAssembler::Sbcs(const Register& rd,
+ const Register& rn,
+ const Operand& operand) {
+ AddSubWithCarryMacro(rd, rn, operand, SetFlags, SBC);
+}
+
+
+void MacroAssembler::Ngc(const Register& rd,
+ const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ Sbc(rd, zr, operand);
+}
+
+
+void MacroAssembler::Ngcs(const Register& rd,
+ const Operand& operand) {
+ Register zr = AppropriateZeroRegFor(rd);
+ Sbcs(rd, zr, operand);
+}
+
+
+void MacroAssembler::AddSubWithCarryMacro(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ AddSubWithCarryOp op) {
+ VIXL_ASSERT(rd.size() == rn.size());
+ // Worst case is addc/subc immediate:
+ // * up to 4 instructions to materialise the constant
+ // * 1 instruction for add/sub
+ MacroEmissionCheckScope guard(this);
+ UseScratchRegisterScope temps(this);
+
+ if (operand.IsImmediate() ||
+ (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
+ // Add/sub with carry (immediate or ROR shifted register.)
+ Register temp = temps.AcquireSameSizeAs(rn);
+ VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn) && !temp.Is(operand.maybeReg()));
+ Mov(temp, operand);
+ AddSubWithCarry(rd, rn, Operand(temp), S, op);
+ } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
+ // Add/sub with carry (shifted register).
+ VIXL_ASSERT(operand.reg().size() == rd.size());
+ VIXL_ASSERT(operand.shift() != ROR);
+ VIXL_ASSERT(IsUintN(rd.size() == kXRegSize ? kXRegSizeLog2 : kWRegSizeLog2,
+ operand.shift_amount()));
+ temps.Exclude(operand.reg());
+ Register temp = temps.AcquireSameSizeAs(rn);
+ VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn) && !temp.Is(operand.maybeReg()));
+ EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount());
+ AddSubWithCarry(rd, rn, Operand(temp), S, op);
+ } else if (operand.IsExtendedRegister()) {
+ // Add/sub with carry (extended register).
+ VIXL_ASSERT(operand.reg().size() <= rd.size());
+ // Add/sub extended supports a shift <= 4. We want to support exactly the
+ // same modes.
+ VIXL_ASSERT(operand.shift_amount() <= 4);
+ VIXL_ASSERT(operand.reg().Is64Bits() ||
+ ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
+ temps.Exclude(operand.reg());
+ Register temp = temps.AcquireSameSizeAs(rn);
+ VIXL_ASSERT(!temp.Is(rd) && !temp.Is(rn) && !temp.Is(operand.maybeReg()));
+ EmitExtendShift(temp, operand.reg(), operand.extend(),
+ operand.shift_amount());
+ AddSubWithCarry(rd, rn, Operand(temp), S, op);
+ } else {
+ // The addressing mode is directly supported by the instruction.
+ AddSubWithCarry(rd, rn, operand, S, op);
+ }
+}
+
+
+#define DEFINE_FUNCTION(FN, REGTYPE, REG, OP) \
+void MacroAssembler::FN(const REGTYPE REG, const MemOperand& addr) { \
+ LoadStoreMacro(REG, addr, OP); \
+}
+LS_MACRO_LIST(DEFINE_FUNCTION)
+#undef DEFINE_FUNCTION
+
+
+void MacroAssembler::LoadStoreMacro(const CPURegister& rt,
+ const MemOperand& addr,
+ LoadStoreOp op) {
+ // Worst case is ldr/str pre/post index:
+ // * 1 instruction for ldr/str
+ // * up to 4 instructions to materialise the constant
+ // * 1 instruction to update the base
+ MacroEmissionCheckScope guard(this);
+
+ int64_t offset = addr.offset();
+ unsigned access_size = CalcLSDataSize(op);
+
+ // Check if an immediate offset fits in the immediate field of the
+ // appropriate instruction. If not, emit two instructions to perform
+ // the operation.
+ if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, access_size) &&
+ !IsImmLSUnscaled(offset)) {
+ // Immediate offset that can't be encoded using unsigned or unscaled
+ // addressing modes.
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(addr.base());
+ VIXL_ASSERT(!temp.Is(rt));
+ VIXL_ASSERT(!temp.Is(addr.base()) && !temp.Is(addr.regoffset()));
+ Mov(temp, addr.offset());
+ LoadStore(rt, MemOperand(addr.base(), temp), op);
+ } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
+ // Post-index beyond unscaled addressing range.
+ LoadStore(rt, MemOperand(addr.base()), op);
+ Add(addr.base(), addr.base(), Operand(offset));
+ } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
+ // Pre-index beyond unscaled addressing range.
+ Add(addr.base(), addr.base(), Operand(offset));
+ LoadStore(rt, MemOperand(addr.base()), op);
+ } else {
+ // Encodable in one load/store instruction.
+ LoadStore(rt, addr, op);
+ }
+}
+
+
+#define DEFINE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \
+void MacroAssembler::FN(const REGTYPE REG, \
+ const REGTYPE REG2, \
+ const MemOperand& addr) { \
+ LoadStorePairMacro(REG, REG2, addr, OP); \
+}
+LSPAIR_MACRO_LIST(DEFINE_FUNCTION)
+#undef DEFINE_FUNCTION
+
+void MacroAssembler::LoadStorePairMacro(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& addr,
+ LoadStorePairOp op) {
+ // TODO(all): Should we support register offset for load-store-pair?
+ VIXL_ASSERT(!addr.IsRegisterOffset());
+ // Worst case is ldp/stp immediate:
+ // * 1 instruction for ldp/stp
+ // * up to 4 instructions to materialise the constant
+ // * 1 instruction to update the base
+ MacroEmissionCheckScope guard(this);
+
+ int64_t offset = addr.offset();
+ unsigned access_size = CalcLSPairDataSize(op);
+
+ // Check if the offset fits in the immediate field of the appropriate
+ // instruction. If not, emit two instructions to perform the operation.
+ if (IsImmLSPair(offset, access_size)) {
+ // Encodable in one load/store pair instruction.
+ LoadStorePair(rt, rt2, addr, op);
+ } else {
+ Register base = addr.base();
+ if (addr.IsImmediateOffset()) {
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(base);
+ Add(temp, base, offset);
+ LoadStorePair(rt, rt2, MemOperand(temp), op);
+ } else if (addr.IsPostIndex()) {
+ LoadStorePair(rt, rt2, MemOperand(base), op);
+ Add(base, base, offset);
+ } else {
+ VIXL_ASSERT(addr.IsPreIndex());
+ Add(base, base, offset);
+ LoadStorePair(rt, rt2, MemOperand(base), op);
+ }
+ }
+}
+
+
+void MacroAssembler::Prfm(PrefetchOperation op, const MemOperand& addr) {
+ MacroEmissionCheckScope guard(this);
+
+ // There are no pre- or post-index modes for prfm.
+ VIXL_ASSERT(addr.IsImmediateOffset() || addr.IsRegisterOffset());
+
+ // The access size is implicitly 8 bytes for all prefetch operations.
+ unsigned size = kXRegSizeInBytesLog2;
+
+ // Check if an immediate offset fits in the immediate field of the
+ // appropriate instruction. If not, emit two instructions to perform
+ // the operation.
+ if (addr.IsImmediateOffset() && !IsImmLSScaled(addr.offset(), size) &&
+ !IsImmLSUnscaled(addr.offset())) {
+ // Immediate offset that can't be encoded using unsigned or unscaled
+ // addressing modes.
+ UseScratchRegisterScope temps(this);
+ Register temp = temps.AcquireSameSizeAs(addr.base());
+ Mov(temp, addr.offset());
+ Prefetch(op, MemOperand(addr.base(), temp));
+ } else {
+ // Simple register-offsets are encodable in one instruction.
+ Prefetch(op, addr);
+ }
+}
+
+
+void MacroAssembler::PushStackPointer() {
+ PrepareForPush(1, 8);
+
+ // Pushing a stack pointer leads to implementation-defined
+ // behavior, which may be surprising. In particular,
+ // str x28, [x28, #-8]!
+ // pre-decrements the stack pointer, storing the decremented value.
+ // Additionally, sp is read as xzr in this context, so it cannot be pushed.
+ // So we must use a scratch register.
+ UseScratchRegisterScope temps(this);
+ Register scratch = temps.AcquireX();
+
+ Mov(scratch, GetStackPointer64());
+ str(scratch, MemOperand(GetStackPointer64(), -8, PreIndex));
+}
+
+
+void MacroAssembler::Push(const CPURegister& src0, const CPURegister& src1,
+ const CPURegister& src2, const CPURegister& src3) {
+ VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3));
+ VIXL_ASSERT(src0.IsValid());
+
+ int count = 1 + src1.IsValid() + src2.IsValid() + src3.IsValid();
+ int size = src0.SizeInBytes();
+
+ if (src0.Is(GetStackPointer64())) {
+ VIXL_ASSERT(count == 1);
+ VIXL_ASSERT(size == 8);
+ PushStackPointer();
+ return;
+ }
+
+ PrepareForPush(count, size);
+ PushHelper(count, size, src0, src1, src2, src3);
+}
+
+
+void MacroAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
+ const CPURegister& dst2, const CPURegister& dst3) {
+ // It is not valid to pop into the same register more than once in one
+ // instruction, not even into the zero register.
+ VIXL_ASSERT(!AreAliased(dst0, dst1, dst2, dst3));
+ VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3));
+ VIXL_ASSERT(dst0.IsValid());
+
+ int count = 1 + dst1.IsValid() + dst2.IsValid() + dst3.IsValid();
+ int size = dst0.SizeInBytes();
+
+ PrepareForPop(count, size);
+ PopHelper(count, size, dst0, dst1, dst2, dst3);
+}
+
+
+void MacroAssembler::PushCPURegList(CPURegList registers) {
+ VIXL_ASSERT(!registers.Overlaps(*TmpList()));
+ VIXL_ASSERT(!registers.Overlaps(*FPTmpList()));
+
+ int reg_size = registers.RegisterSizeInBytes();
+ PrepareForPush(registers.Count(), reg_size);
+
+ // Bump the stack pointer and store two registers at the bottom.
+ int size = registers.TotalSizeInBytes();
+ const CPURegister& bottom_0 = registers.PopLowestIndex();
+ const CPURegister& bottom_1 = registers.PopLowestIndex();
+ if (bottom_0.IsValid() && bottom_1.IsValid()) {
+ Stp(bottom_0, bottom_1, MemOperand(GetStackPointer64(), -size, PreIndex));
+ } else if (bottom_0.IsValid()) {
+ Str(bottom_0, MemOperand(GetStackPointer64(), -size, PreIndex));
+ }
+
+ int offset = 2 * reg_size;
+ while (!registers.IsEmpty()) {
+ const CPURegister& src0 = registers.PopLowestIndex();
+ const CPURegister& src1 = registers.PopLowestIndex();
+ if (src1.IsValid()) {
+ Stp(src0, src1, MemOperand(GetStackPointer64(), offset));
+ } else {
+ Str(src0, MemOperand(GetStackPointer64(), offset));
+ }
+ offset += 2 * reg_size;
+ }
+}
+
+
+void MacroAssembler::PopCPURegList(CPURegList registers) {
+ VIXL_ASSERT(!registers.Overlaps(*TmpList()));
+ VIXL_ASSERT(!registers.Overlaps(*FPTmpList()));
+
+ int reg_size = registers.RegisterSizeInBytes();
+ PrepareForPop(registers.Count(), reg_size);
+
+
+ int size = registers.TotalSizeInBytes();
+ const CPURegister& bottom_0 = registers.PopLowestIndex();
+ const CPURegister& bottom_1 = registers.PopLowestIndex();
+
+ int offset = 2 * reg_size;
+ while (!registers.IsEmpty()) {
+ const CPURegister& dst0 = registers.PopLowestIndex();
+ const CPURegister& dst1 = registers.PopLowestIndex();
+ if (dst1.IsValid()) {
+ Ldp(dst0, dst1, MemOperand(GetStackPointer64(), offset));
+ } else {
+ Ldr(dst0, MemOperand(GetStackPointer64(), offset));
+ }
+ offset += 2 * reg_size;
+ }
+
+ // Load the two registers at the bottom and drop the stack pointer.
+ if (bottom_0.IsValid() && bottom_1.IsValid()) {
+ Ldp(bottom_0, bottom_1, MemOperand(GetStackPointer64(), size, PostIndex));
+ } else if (bottom_0.IsValid()) {
+ Ldr(bottom_0, MemOperand(GetStackPointer64(), size, PostIndex));
+ }
+}
+
+
+void MacroAssembler::PushMultipleTimes(int count, Register src) {
+ int size = src.SizeInBytes();
+
+ PrepareForPush(count, size);
+ // Push up to four registers at a time if possible because if the current
+ // stack pointer is sp and the register size is 32, registers must be pushed
+ // in blocks of four in order to maintain the 16-byte alignment for sp.
+ while (count >= 4) {
+ PushHelper(4, size, src, src, src, src);
+ count -= 4;
+ }
+ if (count >= 2) {
+ PushHelper(2, size, src, src, NoReg, NoReg);
+ count -= 2;
+ }
+ if (count == 1) {
+ PushHelper(1, size, src, NoReg, NoReg, NoReg);
+ count -= 1;
+ }
+ VIXL_ASSERT(count == 0);
+}
+
+
+void MacroAssembler::PushHelper(int count, int size,
+ const CPURegister& src0,
+ const CPURegister& src1,
+ const CPURegister& src2,
+ const CPURegister& src3) {
+ // Ensure that we don't unintentionally modify scratch or debug registers.
+ // Worst case for size is 2 stp.
+ InstructionAccurateScope scope(this, 2,
+ InstructionAccurateScope::kMaximumSize);
+
+ VIXL_ASSERT(AreSameSizeAndType(src0, src1, src2, src3));
+ VIXL_ASSERT(size == src0.SizeInBytes());
+
+ // Pushing the stack pointer has unexpected behavior. See PushStackPointer().
+ VIXL_ASSERT(!src0.Is(GetStackPointer64()) && !src0.Is(sp));
+ VIXL_ASSERT(!src1.Is(GetStackPointer64()) && !src1.Is(sp));
+ VIXL_ASSERT(!src2.Is(GetStackPointer64()) && !src2.Is(sp));
+ VIXL_ASSERT(!src3.Is(GetStackPointer64()) && !src3.Is(sp));
+
+ // The JS engine should never push 4 bytes.
+ VIXL_ASSERT(size >= 8);
+
+ // When pushing multiple registers, the store order is chosen such that
+ // Push(a, b) is equivalent to Push(a) followed by Push(b).
+ switch (count) {
+ case 1:
+ VIXL_ASSERT(src1.IsNone() && src2.IsNone() && src3.IsNone());
+ str(src0, MemOperand(GetStackPointer64(), -1 * size, PreIndex));
+ break;
+ case 2:
+ VIXL_ASSERT(src2.IsNone() && src3.IsNone());
+ stp(src1, src0, MemOperand(GetStackPointer64(), -2 * size, PreIndex));
+ break;
+ case 3:
+ VIXL_ASSERT(src3.IsNone());
+ stp(src2, src1, MemOperand(GetStackPointer64(), -3 * size, PreIndex));
+ str(src0, MemOperand(GetStackPointer64(), 2 * size));
+ break;
+ case 4:
+ // Skip over 4 * size, then fill in the gap. This allows four W registers
+ // to be pushed using sp, whilst maintaining 16-byte alignment for sp at
+ // all times.
+ stp(src3, src2, MemOperand(GetStackPointer64(), -4 * size, PreIndex));
+ stp(src1, src0, MemOperand(GetStackPointer64(), 2 * size));
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+}
+
+
+void MacroAssembler::PopHelper(int count, int size,
+ const CPURegister& dst0,
+ const CPURegister& dst1,
+ const CPURegister& dst2,
+ const CPURegister& dst3) {
+ // Ensure that we don't unintentionally modify scratch or debug registers.
+ // Worst case for size is 2 ldp.
+ InstructionAccurateScope scope(this, 2,
+ InstructionAccurateScope::kMaximumSize);
+
+ VIXL_ASSERT(AreSameSizeAndType(dst0, dst1, dst2, dst3));
+ VIXL_ASSERT(size == dst0.SizeInBytes());
+
+ // When popping multiple registers, the load order is chosen such that
+ // Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
+ switch (count) {
+ case 1:
+ VIXL_ASSERT(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
+ ldr(dst0, MemOperand(GetStackPointer64(), 1 * size, PostIndex));
+ break;
+ case 2:
+ VIXL_ASSERT(dst2.IsNone() && dst3.IsNone());
+ ldp(dst0, dst1, MemOperand(GetStackPointer64(), 2 * size, PostIndex));
+ break;
+ case 3:
+ VIXL_ASSERT(dst3.IsNone());
+ ldr(dst2, MemOperand(GetStackPointer64(), 2 * size));
+ ldp(dst0, dst1, MemOperand(GetStackPointer64(), 3 * size, PostIndex));
+ break;
+ case 4:
+ // Load the higher addresses first, then load the lower addresses and skip
+ // the whole block in the second instruction. This allows four W registers
+ // to be popped using sp, whilst maintaining 16-byte alignment for sp at
+ // all times.
+ ldp(dst2, dst3, MemOperand(GetStackPointer64(), 2 * size));
+ ldp(dst0, dst1, MemOperand(GetStackPointer64(), 4 * size, PostIndex));
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+}
+
+
+void MacroAssembler::PrepareForPush(int count, int size) {
+ if (sp.Is(GetStackPointer64())) {
+ // If the current stack pointer is sp, then it must be aligned to 16 bytes
+ // on entry and the total size of the specified registers must also be a
+ // multiple of 16 bytes.
+ VIXL_ASSERT((count * size) % 16 == 0);
+ } else {
+ // Even if the current stack pointer is not the system stack pointer (sp),
+ // the system stack pointer will still be modified in order to comply with
+ // ABI rules about accessing memory below the system stack pointer.
+ BumpSystemStackPointer(count * size);
+ }
+}
+
+
+void MacroAssembler::PrepareForPop(int count, int size) {
+ USE(count, size);
+ if (sp.Is(GetStackPointer64())) {
+ // If the current stack pointer is sp, then it must be aligned to 16 bytes
+ // on entry and the total size of the specified registers must also be a
+ // multiple of 16 bytes.
+ VIXL_ASSERT((count * size) % 16 == 0);
+ }
+}
+
+void MacroAssembler::Poke(const Register& src, const Operand& offset) {
+ if (offset.IsImmediate()) {
+ VIXL_ASSERT(offset.immediate() >= 0);
+ }
+
+ Str(src, MemOperand(GetStackPointer64(), offset));
+}
+
+
+void MacroAssembler::Peek(const Register& dst, const Operand& offset) {
+ if (offset.IsImmediate()) {
+ VIXL_ASSERT(offset.immediate() >= 0);
+ }
+
+ Ldr(dst, MemOperand(GetStackPointer64(), offset));
+}
+
+
+void MacroAssembler::Claim(const Operand& size) {
+
+ if (size.IsZero()) {
+ return;
+ }
+
+ if (size.IsImmediate()) {
+ VIXL_ASSERT(size.immediate() > 0);
+ if (sp.Is(GetStackPointer64())) {
+ VIXL_ASSERT((size.immediate() % 16) == 0);
+ }
+ }
+
+ Sub(GetStackPointer64(), GetStackPointer64(), size);
+
+ // Make sure the real stack pointer reflects the claimed stack space.
+ // We can't use stack memory below the stack pointer, it could be clobbered by
+ // interupts and signal handlers.
+ if (!sp.Is(GetStackPointer64())) {
+ Mov(sp, GetStackPointer64());
+ }
+}
+
+
+void MacroAssembler::Drop(const Operand& size) {
+
+ if (size.IsZero()) {
+ return;
+ }
+
+ if (size.IsImmediate()) {
+ VIXL_ASSERT(size.immediate() > 0);
+ if (sp.Is(GetStackPointer64())) {
+ VIXL_ASSERT((size.immediate() % 16) == 0);
+ }
+ }
+
+ Add(GetStackPointer64(), GetStackPointer64(), size);
+}
+
+
+void MacroAssembler::PushCalleeSavedRegisters() {
+ // Ensure that the macro-assembler doesn't use any scratch registers.
+ // 10 stp will be emitted.
+ // TODO(all): Should we use GetCalleeSaved and SavedFP.
+ InstructionAccurateScope scope(this, 10);
+
+ // This method must not be called unless the current stack pointer is sp.
+ VIXL_ASSERT(sp.Is(GetStackPointer64()));
+
+ MemOperand tos(sp, -2 * static_cast<int>(kXRegSizeInBytes), PreIndex);
+
+ stp(x29, x30, tos);
+ stp(x27, x28, tos);
+ stp(x25, x26, tos);
+ stp(x23, x24, tos);
+ stp(x21, x22, tos);
+ stp(x19, x20, tos);
+
+ stp(d14, d15, tos);
+ stp(d12, d13, tos);
+ stp(d10, d11, tos);
+ stp(d8, d9, tos);
+}
+
+
+void MacroAssembler::PopCalleeSavedRegisters() {
+ // Ensure that the macro-assembler doesn't use any scratch registers.
+ // 10 ldp will be emitted.
+ // TODO(all): Should we use GetCalleeSaved and SavedFP.
+ InstructionAccurateScope scope(this, 10);
+
+ // This method must not be called unless the current stack pointer is sp.
+ VIXL_ASSERT(sp.Is(GetStackPointer64()));
+
+ MemOperand tos(sp, 2 * kXRegSizeInBytes, PostIndex);
+
+ ldp(d8, d9, tos);
+ ldp(d10, d11, tos);
+ ldp(d12, d13, tos);
+ ldp(d14, d15, tos);
+
+ ldp(x19, x20, tos);
+ ldp(x21, x22, tos);
+ ldp(x23, x24, tos);
+ ldp(x25, x26, tos);
+ ldp(x27, x28, tos);
+ ldp(x29, x30, tos);
+}
+
+void MacroAssembler::LoadCPURegList(CPURegList registers,
+ const MemOperand& src) {
+ LoadStoreCPURegListHelper(kLoad, registers, src);
+}
+
+void MacroAssembler::StoreCPURegList(CPURegList registers,
+ const MemOperand& dst) {
+ LoadStoreCPURegListHelper(kStore, registers, dst);
+}
+
+
+void MacroAssembler::LoadStoreCPURegListHelper(LoadStoreCPURegListAction op,
+ CPURegList registers,
+ const MemOperand& mem) {
+ // We do not handle pre-indexing or post-indexing.
+ VIXL_ASSERT(!(mem.IsPreIndex() || mem.IsPostIndex()));
+ VIXL_ASSERT(!registers.Overlaps(tmp_list_));
+ VIXL_ASSERT(!registers.Overlaps(fptmp_list_));
+ VIXL_ASSERT(!registers.IncludesAliasOf(sp));
+
+ UseScratchRegisterScope temps(this);
+
+ MemOperand loc = BaseMemOperandForLoadStoreCPURegList(registers,
+ mem,
+ &temps);
+
+ while (registers.Count() >= 2) {
+ const CPURegister& dst0 = registers.PopLowestIndex();
+ const CPURegister& dst1 = registers.PopLowestIndex();
+ if (op == kStore) {
+ Stp(dst0, dst1, loc);
+ } else {
+ VIXL_ASSERT(op == kLoad);
+ Ldp(dst0, dst1, loc);
+ }
+ loc.AddOffset(2 * registers.RegisterSizeInBytes());
+ }
+ if (!registers.IsEmpty()) {
+ if (op == kStore) {
+ Str(registers.PopLowestIndex(), loc);
+ } else {
+ VIXL_ASSERT(op == kLoad);
+ Ldr(registers.PopLowestIndex(), loc);
+ }
+ }
+}
+
+MemOperand MacroAssembler::BaseMemOperandForLoadStoreCPURegList(
+ const CPURegList& registers,
+ const MemOperand& mem,
+ UseScratchRegisterScope* scratch_scope) {
+ // If necessary, pre-compute the base address for the accesses.
+ if (mem.IsRegisterOffset()) {
+ Register reg_base = scratch_scope->AcquireX();
+ ComputeAddress(reg_base, mem);
+ return MemOperand(reg_base);
+
+ } else if (mem.IsImmediateOffset()) {
+ int reg_size = registers.RegisterSizeInBytes();
+ int total_size = registers.TotalSizeInBytes();
+ int64_t min_offset = mem.offset();
+ int64_t max_offset = mem.offset() + std::max(0, total_size - 2 * reg_size);
+ if ((registers.Count() >= 2) &&
+ (!Assembler::IsImmLSPair(min_offset, WhichPowerOf2(reg_size)) ||
+ !Assembler::IsImmLSPair(max_offset, WhichPowerOf2(reg_size)))) {
+ Register reg_base = scratch_scope->AcquireX();
+ ComputeAddress(reg_base, mem);
+ return MemOperand(reg_base);
+ }
+ }
+
+ return mem;
+}
+
+void MacroAssembler::BumpSystemStackPointer(const Operand& space) {
+ VIXL_ASSERT(!sp.Is(GetStackPointer64()));
+ // TODO: Several callers rely on this not using scratch registers, so we use
+ // the assembler directly here. However, this means that large immediate
+ // values of 'space' cannot be handled.
+ InstructionAccurateScope scope(this, 1);
+ sub(sp, GetStackPointer64(), space);
+}
+
+
+void MacroAssembler::Trace(TraceParameters parameters, TraceCommand command) {
+
+#ifdef JS_SIMULATOR_ARM64
+ // The arguments to the trace pseudo instruction need to be contiguous in
+ // memory, so make sure we don't try to emit a literal pool.
+ InstructionAccurateScope scope(this, kTraceLength / kInstructionSize);
+
+ Label start;
+ bind(&start);
+
+ // Refer to simulator-a64.h for a description of the marker and its
+ // arguments.
+ hlt(kTraceOpcode);
+
+ // VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kTraceParamsOffset);
+ dc32(parameters);
+
+ // VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kTraceCommandOffset);
+ dc32(command);
+#else
+ // Emit nothing on real hardware.
+ USE(parameters, command);
+#endif
+}
+
+
+void MacroAssembler::Log(TraceParameters parameters) {
+
+#ifdef JS_SIMULATOR_ARM64
+ // The arguments to the log pseudo instruction need to be contiguous in
+ // memory, so make sure we don't try to emit a literal pool.
+ InstructionAccurateScope scope(this, kLogLength / kInstructionSize);
+
+ Label start;
+ bind(&start);
+
+ // Refer to simulator-a64.h for a description of the marker and its
+ // arguments.
+ hlt(kLogOpcode);
+
+ // VIXL_ASSERT(SizeOfCodeGeneratedSince(&start) == kLogParamsOffset);
+ dc32(parameters);
+#else
+ // Emit nothing on real hardware.
+ USE(parameters);
+#endif
+}
+
+
+void MacroAssembler::EnableInstrumentation() {
+ VIXL_ASSERT(!isprint(InstrumentStateEnable));
+ InstructionAccurateScope scope(this, 1);
+ movn(xzr, InstrumentStateEnable);
+}
+
+
+void MacroAssembler::DisableInstrumentation() {
+ VIXL_ASSERT(!isprint(InstrumentStateDisable));
+ InstructionAccurateScope scope(this, 1);
+ movn(xzr, InstrumentStateDisable);
+}
+
+
+void MacroAssembler::AnnotateInstrumentation(const char* marker_name) {
+ VIXL_ASSERT(strlen(marker_name) == 2);
+
+ // We allow only printable characters in the marker names. Unprintable
+ // characters are reserved for controlling features of the instrumentation.
+ VIXL_ASSERT(isprint(marker_name[0]) && isprint(marker_name[1]));
+
+ InstructionAccurateScope scope(this, 1);
+ movn(xzr, (marker_name[1] << 8) | marker_name[0]);
+}
+
+
+void UseScratchRegisterScope::Open(MacroAssembler* masm) {
+ VIXL_ASSERT(!initialised_);
+ available_ = masm->TmpList();
+ availablefp_ = masm->FPTmpList();
+ old_available_ = available_->list();
+ old_availablefp_ = availablefp_->list();
+ VIXL_ASSERT(available_->type() == CPURegister::kRegister);
+ VIXL_ASSERT(availablefp_->type() == CPURegister::kVRegister);
+#ifdef DEBUG
+ initialised_ = true;
+#endif
+}
+
+
+void UseScratchRegisterScope::Close() {
+ if (available_) {
+ available_->set_list(old_available_);
+ available_ = NULL;
+ }
+ if (availablefp_) {
+ availablefp_->set_list(old_availablefp_);
+ availablefp_ = NULL;
+ }
+#ifdef DEBUG
+ initialised_ = false;
+#endif
+}
+
+
+UseScratchRegisterScope::UseScratchRegisterScope(MacroAssembler* masm) {
+#ifdef DEBUG
+ initialised_ = false;
+#endif
+ Open(masm);
+}
+
+// This allows deferred (and optional) initialisation of the scope.
+UseScratchRegisterScope::UseScratchRegisterScope()
+ : available_(NULL), availablefp_(NULL),
+ old_available_(0), old_availablefp_(0) {
+#ifdef DEBUG
+ initialised_ = false;
+#endif
+}
+
+UseScratchRegisterScope::~UseScratchRegisterScope() {
+ Close();
+}
+
+
+bool UseScratchRegisterScope::IsAvailable(const CPURegister& reg) const {
+ return available_->IncludesAliasOf(reg) || availablefp_->IncludesAliasOf(reg);
+}
+
+
+Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) {
+ int code = AcquireNextAvailable(available_).code();
+ return Register(code, reg.size());
+}
+
+
+FPRegister UseScratchRegisterScope::AcquireSameSizeAs(const FPRegister& reg) {
+ int code = AcquireNextAvailable(availablefp_).code();
+ return FPRegister(code, reg.size());
+}
+
+
+void UseScratchRegisterScope::Release(const CPURegister& reg) {
+ VIXL_ASSERT(initialised_);
+ if (reg.IsRegister()) {
+ ReleaseByCode(available_, reg.code());
+ } else if (reg.IsFPRegister()) {
+ ReleaseByCode(availablefp_, reg.code());
+ } else {
+ VIXL_ASSERT(reg.IsNone());
+ }
+}
+
+
+void UseScratchRegisterScope::Include(const CPURegList& list) {
+ VIXL_ASSERT(initialised_);
+ if (list.type() == CPURegister::kRegister) {
+ // Make sure that neither sp nor xzr are included the list.
+ IncludeByRegList(available_, list.list() & ~(xzr.Bit() | sp.Bit()));
+ } else {
+ VIXL_ASSERT(list.type() == CPURegister::kVRegister);
+ IncludeByRegList(availablefp_, list.list());
+ }
+}
+
+
+void UseScratchRegisterScope::Include(const Register& reg1,
+ const Register& reg2,
+ const Register& reg3,
+ const Register& reg4) {
+ VIXL_ASSERT(initialised_);
+ RegList include = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit();
+ // Make sure that neither sp nor xzr are included the list.
+ include &= ~(xzr.Bit() | sp.Bit());
+
+ IncludeByRegList(available_, include);
+}
+
+
+void UseScratchRegisterScope::Include(const FPRegister& reg1,
+ const FPRegister& reg2,
+ const FPRegister& reg3,
+ const FPRegister& reg4) {
+ RegList include = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit();
+ IncludeByRegList(availablefp_, include);
+}
+
+
+void UseScratchRegisterScope::Exclude(const CPURegList& list) {
+ if (list.type() == CPURegister::kRegister) {
+ ExcludeByRegList(available_, list.list());
+ } else {
+ VIXL_ASSERT(list.type() == CPURegister::kVRegister);
+ ExcludeByRegList(availablefp_, list.list());
+ }
+}
+
+
+void UseScratchRegisterScope::Exclude(const Register& reg1,
+ const Register& reg2,
+ const Register& reg3,
+ const Register& reg4) {
+ RegList exclude = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit();
+ ExcludeByRegList(available_, exclude);
+}
+
+
+void UseScratchRegisterScope::Exclude(const FPRegister& reg1,
+ const FPRegister& reg2,
+ const FPRegister& reg3,
+ const FPRegister& reg4) {
+ RegList excludefp = reg1.Bit() | reg2.Bit() | reg3.Bit() | reg4.Bit();
+ ExcludeByRegList(availablefp_, excludefp);
+}
+
+
+void UseScratchRegisterScope::Exclude(const CPURegister& reg1,
+ const CPURegister& reg2,
+ const CPURegister& reg3,
+ const CPURegister& reg4) {
+ RegList exclude = 0;
+ RegList excludefp = 0;
+
+ const CPURegister regs[] = {reg1, reg2, reg3, reg4};
+
+ for (unsigned i = 0; i < (sizeof(regs) / sizeof(regs[0])); i++) {
+ if (regs[i].IsRegister()) {
+ exclude |= regs[i].Bit();
+ } else if (regs[i].IsFPRegister()) {
+ excludefp |= regs[i].Bit();
+ } else {
+ VIXL_ASSERT(regs[i].IsNone());
+ }
+ }
+
+ ExcludeByRegList(available_, exclude);
+ ExcludeByRegList(availablefp_, excludefp);
+}
+
+
+void UseScratchRegisterScope::ExcludeAll() {
+ ExcludeByRegList(available_, available_->list());
+ ExcludeByRegList(availablefp_, availablefp_->list());
+}
+
+
+CPURegister UseScratchRegisterScope::AcquireNextAvailable(
+ CPURegList* available) {
+ VIXL_CHECK(!available->IsEmpty());
+ CPURegister result = available->PopLowestIndex();
+ VIXL_ASSERT(!AreAliased(result, xzr, sp));
+ return result;
+}
+
+
+void UseScratchRegisterScope::ReleaseByCode(CPURegList* available, int code) {
+ ReleaseByRegList(available, static_cast<RegList>(1) << code);
+}
+
+
+void UseScratchRegisterScope::ReleaseByRegList(CPURegList* available,
+ RegList regs) {
+ available->set_list(available->list() | regs);
+}
+
+
+void UseScratchRegisterScope::IncludeByRegList(CPURegList* available,
+ RegList regs) {
+ available->set_list(available->list() | regs);
+}
+
+
+void UseScratchRegisterScope::ExcludeByRegList(CPURegList* available,
+ RegList exclude) {
+ available->set_list(available->list() & ~exclude);
+}
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/MacroAssembler-vixl.h b/js/src/jit/arm64/vixl/MacroAssembler-vixl.h
new file mode 100644
index 0000000000..232643be66
--- /dev/null
+++ b/js/src/jit/arm64/vixl/MacroAssembler-vixl.h
@@ -0,0 +1,2516 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_MACRO_ASSEMBLER_A64_H_
+#define VIXL_A64_MACRO_ASSEMBLER_A64_H_
+
+#include <algorithm>
+#include <limits>
+
+#include "jit/arm64/Assembler-arm64.h"
+#include "jit/arm64/vixl/Debugger-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Instrument-vixl.h"
+#include "jit/arm64/vixl/Simulator-Constants-vixl.h"
+
+#define LS_MACRO_LIST(V) \
+ V(Ldrb, Register&, rt, LDRB_w) \
+ V(Strb, Register&, rt, STRB_w) \
+ V(Ldrsb, Register&, rt, rt.Is64Bits() ? LDRSB_x : LDRSB_w) \
+ V(Ldrh, Register&, rt, LDRH_w) \
+ V(Strh, Register&, rt, STRH_w) \
+ V(Ldrsh, Register&, rt, rt.Is64Bits() ? LDRSH_x : LDRSH_w) \
+ V(Ldr, CPURegister&, rt, LoadOpFor(rt)) \
+ V(Str, CPURegister&, rt, StoreOpFor(rt)) \
+ V(Ldrsw, Register&, rt, LDRSW_x)
+
+
+#define LSPAIR_MACRO_LIST(V) \
+ V(Ldp, CPURegister&, rt, rt2, LoadPairOpFor(rt, rt2)) \
+ V(Stp, CPURegister&, rt, rt2, StorePairOpFor(rt, rt2)) \
+ V(Ldpsw, CPURegister&, rt, rt2, LDPSW_x)
+
+namespace vixl {
+
+// Forward declaration
+class MacroAssembler;
+class UseScratchRegisterScope;
+
+// This scope has the following purposes:
+// * Acquire/Release the underlying assembler's code buffer.
+// * This is mandatory before emitting.
+// * Emit the literal or veneer pools if necessary before emitting the
+// macro-instruction.
+// * Ensure there is enough space to emit the macro-instruction.
+class EmissionCheckScope {
+ public:
+ EmissionCheckScope(MacroAssembler* masm, size_t size)
+ : masm_(masm)
+ { }
+
+ protected:
+ MacroAssembler* masm_;
+#ifdef DEBUG
+ Label start_;
+ size_t size_;
+#endif
+};
+
+
+// Helper for common Emission checks.
+// The macro-instruction maps to a single instruction.
+class SingleEmissionCheckScope : public EmissionCheckScope {
+ public:
+ explicit SingleEmissionCheckScope(MacroAssembler* masm)
+ : EmissionCheckScope(masm, kInstructionSize) {}
+};
+
+
+// The macro instruction is a "typical" macro-instruction. Typical macro-
+// instruction only emit a few instructions, a few being defined as 8 here.
+class MacroEmissionCheckScope : public EmissionCheckScope {
+ public:
+ explicit MacroEmissionCheckScope(MacroAssembler* masm)
+ : EmissionCheckScope(masm, kTypicalMacroInstructionMaxSize) {}
+
+ private:
+ static const size_t kTypicalMacroInstructionMaxSize = 8 * kInstructionSize;
+};
+
+
+enum BranchType {
+ // Copies of architectural conditions.
+ // The associated conditions can be used in place of those, the code will
+ // take care of reinterpreting them with the correct type.
+ integer_eq = eq,
+ integer_ne = ne,
+ integer_hs = hs,
+ integer_lo = lo,
+ integer_mi = mi,
+ integer_pl = pl,
+ integer_vs = vs,
+ integer_vc = vc,
+ integer_hi = hi,
+ integer_ls = ls,
+ integer_ge = ge,
+ integer_lt = lt,
+ integer_gt = gt,
+ integer_le = le,
+ integer_al = al,
+ integer_nv = nv,
+
+ // These two are *different* from the architectural codes al and nv.
+ // 'always' is used to generate unconditional branches.
+ // 'never' is used to not generate a branch (generally as the inverse
+ // branch type of 'always).
+ always, never,
+ // cbz and cbnz
+ reg_zero, reg_not_zero,
+ // tbz and tbnz
+ reg_bit_clear, reg_bit_set,
+
+ // Aliases.
+ kBranchTypeFirstCondition = eq,
+ kBranchTypeLastCondition = nv,
+ kBranchTypeFirstUsingReg = reg_zero,
+ kBranchTypeFirstUsingBit = reg_bit_clear
+};
+
+
+enum DiscardMoveMode { kDontDiscardForSameWReg, kDiscardForSameWReg };
+
+// The macro assembler supports moving automatically pre-shifted immediates for
+// arithmetic and logical instructions, and then applying a post shift in the
+// instruction to undo the modification, in order to reduce the code emitted for
+// an operation. For example:
+//
+// Add(x0, x0, 0x1f7de) => movz x16, 0xfbef; add x0, x0, x16, lsl #1.
+//
+// This optimisation can be only partially applied when the stack pointer is an
+// operand or destination, so this enumeration is used to control the shift.
+enum PreShiftImmMode {
+ kNoShift, // Don't pre-shift.
+ kLimitShiftForSP, // Limit pre-shift for add/sub extend use.
+ kAnyShift // Allow any pre-shift.
+};
+
+
+class MacroAssembler : public js::jit::Assembler {
+ public:
+ MacroAssembler();
+
+ // Finalize a code buffer of generated instructions. This function must be
+ // called before executing or copying code from the buffer.
+ void FinalizeCode();
+
+
+ // Constant generation helpers.
+ // These functions return the number of instructions required to move the
+ // immediate into the destination register. Also, if the masm pointer is
+ // non-null, it generates the code to do so.
+ // The two features are implemented using one function to avoid duplication of
+ // the logic.
+ // The function can be used to evaluate the cost of synthesizing an
+ // instruction using 'mov immediate' instructions. A user might prefer loading
+ // a constant using the literal pool instead of using multiple 'mov immediate'
+ // instructions.
+ static int MoveImmediateHelper(MacroAssembler* masm,
+ const Register &rd,
+ uint64_t imm);
+ static bool OneInstrMoveImmediateHelper(MacroAssembler* masm,
+ const Register& dst,
+ int64_t imm);
+
+
+ // Logical macros.
+ void And(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Ands(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Bic(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Bics(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Orr(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Orn(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Eor(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Eon(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Tst(const Register& rn, const Operand& operand);
+ void LogicalMacro(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ LogicalOp op);
+
+ // Add and sub macros.
+ void Add(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S = LeaveFlags);
+ void Adds(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Sub(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S = LeaveFlags);
+ void Subs(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Cmn(const Register& rn, const Operand& operand);
+ void Cmp(const Register& rn, const Operand& operand);
+ void Neg(const Register& rd,
+ const Operand& operand);
+ void Negs(const Register& rd,
+ const Operand& operand);
+
+ void AddSubMacro(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ AddSubOp op);
+
+ // Add/sub with carry macros.
+ void Adc(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Adcs(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Sbc(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Sbcs(const Register& rd,
+ const Register& rn,
+ const Operand& operand);
+ void Ngc(const Register& rd,
+ const Operand& operand);
+ void Ngcs(const Register& rd,
+ const Operand& operand);
+ void AddSubWithCarryMacro(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ FlagsUpdate S,
+ AddSubWithCarryOp op);
+
+ // Move macros.
+ void Mov(const Register& rd, uint64_t imm);
+ void Mov(const Register& rd,
+ const Operand& operand,
+ DiscardMoveMode discard_mode = kDontDiscardForSameWReg);
+ void Mvn(const Register& rd, uint64_t imm) {
+ Mov(rd, (rd.size() == kXRegSize) ? ~imm : (~imm & kWRegMask));
+ }
+ void Mvn(const Register& rd, const Operand& operand);
+
+ // Try to move an immediate into the destination register in a single
+ // instruction. Returns true for success, and updates the contents of dst.
+ // Returns false, otherwise.
+ bool TryOneInstrMoveImmediate(const Register& dst, int64_t imm);
+
+ // Move an immediate into register dst, and return an Operand object for
+ // use with a subsequent instruction that accepts a shift. The value moved
+ // into dst is not necessarily equal to imm; it may have had a shifting
+ // operation applied to it that will be subsequently undone by the shift
+ // applied in the Operand.
+ Operand MoveImmediateForShiftedOp(const Register& dst,
+ int64_t imm,
+ PreShiftImmMode mode);
+
+ // Synthesises the address represented by a MemOperand into a register.
+ void ComputeAddress(const Register& dst, const MemOperand& mem_op);
+
+ // Conditional macros.
+ void Ccmp(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond);
+ void Ccmn(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond);
+ void ConditionalCompareMacro(const Register& rn,
+ const Operand& operand,
+ StatusFlags nzcv,
+ Condition cond,
+ ConditionalCompareOp op);
+ void Csel(const Register& rd,
+ const Register& rn,
+ const Operand& operand,
+ Condition cond);
+
+ // Load/store macros.
+#define DECLARE_FUNCTION(FN, REGTYPE, REG, OP) \
+ void FN(const REGTYPE REG, const MemOperand& addr);
+ LS_MACRO_LIST(DECLARE_FUNCTION)
+#undef DECLARE_FUNCTION
+
+ void LoadStoreMacro(const CPURegister& rt,
+ const MemOperand& addr,
+ LoadStoreOp op);
+
+#define DECLARE_FUNCTION(FN, REGTYPE, REG, REG2, OP) \
+ void FN(const REGTYPE REG, const REGTYPE REG2, const MemOperand& addr);
+ LSPAIR_MACRO_LIST(DECLARE_FUNCTION)
+#undef DECLARE_FUNCTION
+
+ void LoadStorePairMacro(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& addr,
+ LoadStorePairOp op);
+
+ void Prfm(PrefetchOperation op, const MemOperand& addr);
+
+ // Push or pop up to 4 registers of the same width to or from the stack,
+ // using the current stack pointer as set by SetStackPointer.
+ //
+ // If an argument register is 'NoReg', all further arguments are also assumed
+ // to be 'NoReg', and are thus not pushed or popped.
+ //
+ // Arguments are ordered such that "Push(a, b);" is functionally equivalent
+ // to "Push(a); Push(b);".
+ //
+ // It is valid to push the same register more than once, and there is no
+ // restriction on the order in which registers are specified.
+ //
+ // It is not valid to pop into the same register more than once in one
+ // operation, not even into the zero register.
+ //
+ // If the current stack pointer (as set by SetStackPointer) is sp, then it
+ // must be aligned to 16 bytes on entry and the total size of the specified
+ // registers must also be a multiple of 16 bytes.
+ //
+ // Even if the current stack pointer is not the system stack pointer (sp),
+ // Push (and derived methods) will still modify the system stack pointer in
+ // order to comply with ABI rules about accessing memory below the system
+ // stack pointer.
+ //
+ // Other than the registers passed into Pop, the stack pointer and (possibly)
+ // the system stack pointer, these methods do not modify any other registers.
+ void Push(const CPURegister& src0, const CPURegister& src1 = NoReg,
+ const CPURegister& src2 = NoReg, const CPURegister& src3 = NoReg);
+ void Pop(const CPURegister& dst0, const CPURegister& dst1 = NoReg,
+ const CPURegister& dst2 = NoReg, const CPURegister& dst3 = NoReg);
+ void PushStackPointer();
+
+ // Alternative forms of Push and Pop, taking a RegList or CPURegList that
+ // specifies the registers that are to be pushed or popped. Higher-numbered
+ // registers are associated with higher memory addresses (as in the A32 push
+ // and pop instructions).
+ //
+ // (Push|Pop)SizeRegList allow you to specify the register size as a
+ // parameter. Only kXRegSize, kWRegSize, kDRegSize and kSRegSize are
+ // supported.
+ //
+ // Otherwise, (Push|Pop)(CPU|X|W|D|S)RegList is preferred.
+ void PushCPURegList(CPURegList registers);
+ void PopCPURegList(CPURegList registers);
+
+ void PushSizeRegList(RegList registers, unsigned reg_size,
+ CPURegister::RegisterType type = CPURegister::kRegister) {
+ PushCPURegList(CPURegList(type, reg_size, registers));
+ }
+ void PopSizeRegList(RegList registers, unsigned reg_size,
+ CPURegister::RegisterType type = CPURegister::kRegister) {
+ PopCPURegList(CPURegList(type, reg_size, registers));
+ }
+ void PushXRegList(RegList regs) {
+ PushSizeRegList(regs, kXRegSize);
+ }
+ void PopXRegList(RegList regs) {
+ PopSizeRegList(regs, kXRegSize);
+ }
+ void PushWRegList(RegList regs) {
+ PushSizeRegList(regs, kWRegSize);
+ }
+ void PopWRegList(RegList regs) {
+ PopSizeRegList(regs, kWRegSize);
+ }
+ void PushDRegList(RegList regs) {
+ PushSizeRegList(regs, kDRegSize, CPURegister::kVRegister);
+ }
+ void PopDRegList(RegList regs) {
+ PopSizeRegList(regs, kDRegSize, CPURegister::kVRegister);
+ }
+ void PushSRegList(RegList regs) {
+ PushSizeRegList(regs, kSRegSize, CPURegister::kVRegister);
+ }
+ void PopSRegList(RegList regs) {
+ PopSizeRegList(regs, kSRegSize, CPURegister::kVRegister);
+ }
+
+ // Push the specified register 'count' times.
+ void PushMultipleTimes(int count, Register src);
+
+ // Poke 'src' onto the stack. The offset is in bytes.
+ //
+ // If the current stack pointer (as set by SetStackPointer) is sp, then sp
+ // must be aligned to 16 bytes.
+ void Poke(const Register& src, const Operand& offset);
+
+ // Peek at a value on the stack, and put it in 'dst'. The offset is in bytes.
+ //
+ // If the current stack pointer (as set by SetStackPointer) is sp, then sp
+ // must be aligned to 16 bytes.
+ void Peek(const Register& dst, const Operand& offset);
+
+ // Alternative forms of Peek and Poke, taking a RegList or CPURegList that
+ // specifies the registers that are to be pushed or popped. Higher-numbered
+ // registers are associated with higher memory addresses.
+ //
+ // (Peek|Poke)SizeRegList allow you to specify the register size as a
+ // parameter. Only kXRegSize, kWRegSize, kDRegSize and kSRegSize are
+ // supported.
+ //
+ // Otherwise, (Peek|Poke)(CPU|X|W|D|S)RegList is preferred.
+ void PeekCPURegList(CPURegList registers, int64_t offset) {
+ LoadCPURegList(registers, MemOperand(StackPointer(), offset));
+ }
+ void PokeCPURegList(CPURegList registers, int64_t offset) {
+ StoreCPURegList(registers, MemOperand(StackPointer(), offset));
+ }
+
+ void PeekSizeRegList(RegList registers, int64_t offset, unsigned reg_size,
+ CPURegister::RegisterType type = CPURegister::kRegister) {
+ PeekCPURegList(CPURegList(type, reg_size, registers), offset);
+ }
+ void PokeSizeRegList(RegList registers, int64_t offset, unsigned reg_size,
+ CPURegister::RegisterType type = CPURegister::kRegister) {
+ PokeCPURegList(CPURegList(type, reg_size, registers), offset);
+ }
+ void PeekXRegList(RegList regs, int64_t offset) {
+ PeekSizeRegList(regs, offset, kXRegSize);
+ }
+ void PokeXRegList(RegList regs, int64_t offset) {
+ PokeSizeRegList(regs, offset, kXRegSize);
+ }
+ void PeekWRegList(RegList regs, int64_t offset) {
+ PeekSizeRegList(regs, offset, kWRegSize);
+ }
+ void PokeWRegList(RegList regs, int64_t offset) {
+ PokeSizeRegList(regs, offset, kWRegSize);
+ }
+ void PeekDRegList(RegList regs, int64_t offset) {
+ PeekSizeRegList(regs, offset, kDRegSize, CPURegister::kVRegister);
+ }
+ void PokeDRegList(RegList regs, int64_t offset) {
+ PokeSizeRegList(regs, offset, kDRegSize, CPURegister::kVRegister);
+ }
+ void PeekSRegList(RegList regs, int64_t offset) {
+ PeekSizeRegList(regs, offset, kSRegSize, CPURegister::kVRegister);
+ }
+ void PokeSRegList(RegList regs, int64_t offset) {
+ PokeSizeRegList(regs, offset, kSRegSize, CPURegister::kVRegister);
+ }
+
+
+ // Claim or drop stack space without actually accessing memory.
+ //
+ // If the current stack pointer (as set by SetStackPointer) is sp, then it
+ // must be aligned to 16 bytes and the size claimed or dropped must be a
+ // multiple of 16 bytes.
+ void Claim(const Operand& size);
+ void Drop(const Operand& size);
+
+ // Preserve the callee-saved registers (as defined by AAPCS64).
+ //
+ // Higher-numbered registers are pushed before lower-numbered registers, and
+ // thus get higher addresses.
+ // Floating-point registers are pushed before general-purpose registers, and
+ // thus get higher addresses.
+ //
+ // This method must not be called unless StackPointer() is sp, and it is
+ // aligned to 16 bytes.
+ void PushCalleeSavedRegisters();
+
+ // Restore the callee-saved registers (as defined by AAPCS64).
+ //
+ // Higher-numbered registers are popped after lower-numbered registers, and
+ // thus come from higher addresses.
+ // Floating-point registers are popped after general-purpose registers, and
+ // thus come from higher addresses.
+ //
+ // This method must not be called unless StackPointer() is sp, and it is
+ // aligned to 16 bytes.
+ void PopCalleeSavedRegisters();
+
+ void LoadCPURegList(CPURegList registers, const MemOperand& src);
+ void StoreCPURegList(CPURegList registers, const MemOperand& dst);
+
+ // Remaining instructions are simple pass-through calls to the assembler.
+ void Adr(const Register& rd, Label* label) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ adr(rd, label);
+ }
+ void Adrp(const Register& rd, Label* label) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ adrp(rd, label);
+ }
+ void Asr(const Register& rd, const Register& rn, unsigned shift) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ asr(rd, rn, shift);
+ }
+ void Asr(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ asrv(rd, rn, rm);
+ }
+
+ // Branch type inversion relies on these relations.
+ VIXL_STATIC_ASSERT((reg_zero == (reg_not_zero ^ 1)) &&
+ (reg_bit_clear == (reg_bit_set ^ 1)) &&
+ (always == (never ^ 1)));
+
+ BranchType InvertBranchType(BranchType type) {
+ if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
+ return static_cast<BranchType>(
+ InvertCondition(static_cast<Condition>(type)));
+ } else {
+ return static_cast<BranchType>(type ^ 1);
+ }
+ }
+
+ void B(Label* label, BranchType type, Register reg = NoReg, int bit = -1);
+
+ void B(Label* label);
+ void B(Label* label, Condition cond);
+ void B(Condition cond, Label* label) {
+ B(label, cond);
+ }
+ void Bfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ bfm(rd, rn, immr, imms);
+ }
+ void Bfi(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ bfi(rd, rn, lsb, width);
+ }
+ void Bfxil(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ bfxil(rd, rn, lsb, width);
+ }
+ void Bind(Label* label);
+ // Bind a label to a specified offset from the start of the buffer.
+ void BindToOffset(Label* label, ptrdiff_t offset);
+ void Bl(Label* label) {
+ SingleEmissionCheckScope guard(this);
+ bl(label);
+ }
+ void Blr(const Register& xn) {
+ VIXL_ASSERT(!xn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ blr(xn);
+ }
+ void Br(const Register& xn) {
+ VIXL_ASSERT(!xn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ br(xn);
+ }
+ void Brk(int code = 0) {
+ SingleEmissionCheckScope guard(this);
+ brk(code);
+ }
+ void Cbnz(const Register& rt, Label* label);
+ void Cbz(const Register& rt, Label* label);
+ void Cinc(const Register& rd, const Register& rn, Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ cinc(rd, rn, cond);
+ }
+ void Cinv(const Register& rd, const Register& rn, Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ cinv(rd, rn, cond);
+ }
+ void Clrex() {
+ SingleEmissionCheckScope guard(this);
+ clrex();
+ }
+ void Cls(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ cls(rd, rn);
+ }
+ void Clz(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ clz(rd, rn);
+ }
+ void Cneg(const Register& rd, const Register& rn, Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ cneg(rd, rn, cond);
+ }
+ void Cset(const Register& rd, Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ cset(rd, cond);
+ }
+ void Csetm(const Register& rd, Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ csetm(rd, cond);
+ }
+ void Csinc(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ // The VIXL source code contains these assertions, but the AArch64 ISR
+ // explicitly permits the use of zero registers. CSET itself is defined
+ // in terms of CSINC with WZR/XZR.
+ //
+ // VIXL_ASSERT(!rn.IsZero());
+ // VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ SingleEmissionCheckScope guard(this);
+ csinc(rd, rn, rm, cond);
+ }
+ void Csinv(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ SingleEmissionCheckScope guard(this);
+ csinv(rd, rn, rm, cond);
+ }
+ void Csneg(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ Condition cond) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ SingleEmissionCheckScope guard(this);
+ csneg(rd, rn, rm, cond);
+ }
+ void Dmb(BarrierDomain domain, BarrierType type) {
+ SingleEmissionCheckScope guard(this);
+ dmb(domain, type);
+ }
+ void Dsb(BarrierDomain domain, BarrierType type) {
+ SingleEmissionCheckScope guard(this);
+ dsb(domain, type);
+ }
+ void Extr(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ unsigned lsb) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ extr(rd, rn, rm, lsb);
+ }
+ void Fadd(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fadd(vd, vn, vm);
+ }
+ void Fccmp(const VRegister& vn,
+ const VRegister& vm,
+ StatusFlags nzcv,
+ Condition cond,
+ FPTrapFlags trap = DisableTrap) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ SingleEmissionCheckScope guard(this);
+ FPCCompareMacro(vn, vm, nzcv, cond, trap);
+ }
+ void Fccmpe(const VRegister& vn,
+ const VRegister& vm,
+ StatusFlags nzcv,
+ Condition cond) {
+ Fccmp(vn, vm, nzcv, cond, EnableTrap);
+ }
+ void Fcmp(const VRegister& vn, const VRegister& vm,
+ FPTrapFlags trap = DisableTrap) {
+ SingleEmissionCheckScope guard(this);
+ FPCompareMacro(vn, vm, trap);
+ }
+ void Fcmp(const VRegister& vn, double value,
+ FPTrapFlags trap = DisableTrap);
+ void Fcmpe(const VRegister& vn, double value);
+ void Fcmpe(const VRegister& vn, const VRegister& vm) {
+ Fcmp(vn, vm, EnableTrap);
+ }
+ void Fcsel(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ Condition cond) {
+ VIXL_ASSERT((cond != al) && (cond != nv));
+ SingleEmissionCheckScope guard(this);
+ fcsel(vd, vn, vm, cond);
+ }
+ void Fcvt(const VRegister& vd, const VRegister& vn) {
+ SingleEmissionCheckScope guard(this);
+ fcvt(vd, vn);
+ }
+ void Fcvtl(const VRegister& vd, const VRegister& vn) {
+ SingleEmissionCheckScope guard(this);
+ fcvtl(vd, vn);
+ }
+ void Fcvtl2(const VRegister& vd, const VRegister& vn) {
+ SingleEmissionCheckScope guard(this);
+ fcvtl2(vd, vn);
+ }
+ void Fcvtn(const VRegister& vd, const VRegister& vn) {
+ SingleEmissionCheckScope guard(this);
+ fcvtn(vd, vn);
+ }
+ void Fcvtn2(const VRegister& vd, const VRegister& vn) {
+ SingleEmissionCheckScope guard(this);
+ fcvtn2(vd, vn);
+ }
+ void Fcvtxn(const VRegister& vd, const VRegister& vn) {
+ SingleEmissionCheckScope guard(this);
+ fcvtxn(vd, vn);
+ }
+ void Fcvtxn2(const VRegister& vd, const VRegister& vn) {
+ SingleEmissionCheckScope guard(this);
+ fcvtxn2(vd, vn);
+ }
+ void Fcvtas(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtas(rd, vn);
+ }
+ void Fcvtau(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtau(rd, vn);
+ }
+ void Fcvtms(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtms(rd, vn);
+ }
+ void Fcvtmu(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtmu(rd, vn);
+ }
+ void Fcvtns(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtns(rd, vn);
+ }
+ void Fcvtnu(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtnu(rd, vn);
+ }
+ void Fcvtps(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtps(rd, vn);
+ }
+ void Fcvtpu(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtpu(rd, vn);
+ }
+ void Fcvtzs(const Register& rd, const VRegister& vn, int fbits = 0) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtzs(rd, vn, fbits);
+ }
+ void Fjcvtzs(const Register& rd, const VRegister& vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fjcvtzs(rd, vn);
+ }
+ void Fcvtzu(const Register& rd, const VRegister& vn, int fbits = 0) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fcvtzu(rd, vn, fbits);
+ }
+ void Fdiv(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fdiv(vd, vn, vm);
+ }
+ void Fmax(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fmax(vd, vn, vm);
+ }
+ void Fmaxnm(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fmaxnm(vd, vn, vm);
+ }
+ void Fmin(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fmin(vd, vn, vm);
+ }
+ void Fminnm(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fminnm(vd, vn, vm);
+ }
+ void Fmov(VRegister vd, VRegister vn) {
+ SingleEmissionCheckScope guard(this);
+ // Only emit an instruction if vd and vn are different, and they are both D
+ // registers. fmov(s0, s0) is not a no-op because it clears the top word of
+ // d0. Technically, fmov(d0, d0) is not a no-op either because it clears
+ // the top of q0, but VRegister does not currently support Q registers.
+ if (!vd.Is(vn) || !vd.Is64Bits()) {
+ fmov(vd, vn);
+ }
+ }
+ void Fmov(VRegister vd, Register rn) {
+ SingleEmissionCheckScope guard(this);
+ fmov(vd, rn);
+ }
+ void Fmov(const VRegister& vd, int index, const Register& rn) {
+ SingleEmissionCheckScope guard(this);
+ fmov(vd, index, rn);
+ }
+ void Fmov(const Register& rd, const VRegister& vn, int index) {
+ SingleEmissionCheckScope guard(this);
+ fmov(rd, vn, index);
+ }
+
+ // Provide explicit double and float interfaces for FP immediate moves, rather
+ // than relying on implicit C++ casts. This allows signalling NaNs to be
+ // preserved when the immediate matches the format of vd. Most systems convert
+ // signalling NaNs to quiet NaNs when converting between float and double.
+ void Fmov(VRegister vd, double imm);
+ void Fmov(VRegister vd, float imm);
+ // Provide a template to allow other types to be converted automatically.
+ template<typename T>
+ void Fmov(VRegister vd, T imm) {
+ Fmov(vd, static_cast<double>(imm));
+ }
+ void Fmov(Register rd, VRegister vn) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ fmov(rd, vn);
+ }
+ void Fmul(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fmul(vd, vn, vm);
+ }
+ void Fnmul(const VRegister& vd, const VRegister& vn,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fnmul(vd, vn, vm);
+ }
+ void Fmadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va) {
+ SingleEmissionCheckScope guard(this);
+ fmadd(vd, vn, vm, va);
+ }
+ void Fmsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va) {
+ SingleEmissionCheckScope guard(this);
+ fmsub(vd, vn, vm, va);
+ }
+ void Fnmadd(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va) {
+ SingleEmissionCheckScope guard(this);
+ fnmadd(vd, vn, vm, va);
+ }
+ void Fnmsub(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ const VRegister& va) {
+ SingleEmissionCheckScope guard(this);
+ fnmsub(vd, vn, vm, va);
+ }
+ void Fsub(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ fsub(vd, vn, vm);
+ }
+ void Hint(SystemHint code) {
+ SingleEmissionCheckScope guard(this);
+ hint(code);
+ }
+ void Hlt(int code) {
+ SingleEmissionCheckScope guard(this);
+ hlt(code);
+ }
+ void Isb() {
+ SingleEmissionCheckScope guard(this);
+ isb();
+ }
+ void Ldar(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldar(rt, src);
+ }
+ void Ldarb(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldarb(rt, src);
+ }
+ void Ldarh(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldarh(rt, src);
+ }
+ void Ldaxp(const Register& rt, const Register& rt2, const MemOperand& src) {
+ VIXL_ASSERT(!rt.Aliases(rt2));
+ SingleEmissionCheckScope guard(this);
+ ldaxp(rt, rt2, src);
+ }
+ void Ldaxr(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldaxr(rt, src);
+ }
+ void Ldaxrb(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldaxrb(rt, src);
+ }
+ void Ldaxrh(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldaxrh(rt, src);
+ }
+ void Ldnp(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldnp(rt, rt2, src);
+ }
+ // Provide both double and float interfaces for FP immediate loads, rather
+ // than relying on implicit C++ casts. This allows signalling NaNs to be
+ // preserved when the immediate matches the format of fd. Most systems convert
+ // signalling NaNs to quiet NaNs when converting between float and double.
+ void Ldr(const VRegister& vt, double imm) {
+ SingleEmissionCheckScope guard(this);
+ if (vt.Is64Bits()) {
+ ldr(vt, imm);
+ } else {
+ ldr(vt, static_cast<float>(imm));
+ }
+ }
+ void Ldr(const VRegister& vt, float imm) {
+ SingleEmissionCheckScope guard(this);
+ if (vt.Is32Bits()) {
+ ldr(vt, imm);
+ } else {
+ ldr(vt, static_cast<double>(imm));
+ }
+ }
+ /*
+ void Ldr(const VRegister& vt, uint64_t high64, uint64_t low64) {
+ VIXL_ASSERT(vt.IsQ());
+ SingleEmissionCheckScope guard(this);
+ ldr(vt, new Literal<uint64_t>(high64, low64,
+ &literal_pool_,
+ RawLiteral::kDeletedOnPlacementByPool));
+ }
+ */
+ void Ldr(const Register& rt, uint64_t imm) {
+ VIXL_ASSERT(!rt.IsZero());
+ SingleEmissionCheckScope guard(this);
+ ldr(rt, imm);
+ }
+ void Ldrsw(const Register& rt, uint32_t imm) {
+ VIXL_ASSERT(!rt.IsZero());
+ SingleEmissionCheckScope guard(this);
+ ldrsw(rt, imm);
+ }
+ void Ldxp(const Register& rt, const Register& rt2, const MemOperand& src) {
+ VIXL_ASSERT(!rt.Aliases(rt2));
+ SingleEmissionCheckScope guard(this);
+ ldxp(rt, rt2, src);
+ }
+ void Ldxr(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldxr(rt, src);
+ }
+ void Ldxrb(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldxrb(rt, src);
+ }
+ void Ldxrh(const Register& rt, const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ldxrh(rt, src);
+ }
+ void Lsl(const Register& rd, const Register& rn, unsigned shift) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ lsl(rd, rn, shift);
+ }
+ void Lsl(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ lslv(rd, rn, rm);
+ }
+ void Lsr(const Register& rd, const Register& rn, unsigned shift) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ lsr(rd, rn, shift);
+ }
+ void Lsr(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ lsrv(rd, rn, rm);
+ }
+ void Madd(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT(!ra.IsZero());
+ SingleEmissionCheckScope guard(this);
+ madd(rd, rn, rm, ra);
+ }
+ void Mneg(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ mneg(rd, rn, rm);
+ }
+ void Mov(const Register& rd, const Register& rn) {
+ SingleEmissionCheckScope guard(this);
+ mov(rd, rn);
+ }
+ void Movk(const Register& rd, uint64_t imm, int shift = -1) {
+ VIXL_ASSERT(!rd.IsZero());
+ SingleEmissionCheckScope guard(this);
+ movk(rd, imm, shift);
+ }
+ void Mrs(const Register& rt, SystemRegister sysreg) {
+ VIXL_ASSERT(!rt.IsZero());
+ SingleEmissionCheckScope guard(this);
+ mrs(rt, sysreg);
+ }
+ void Msr(SystemRegister sysreg, const Register& rt) {
+ VIXL_ASSERT(!rt.IsZero());
+ SingleEmissionCheckScope guard(this);
+ msr(sysreg, rt);
+ }
+ void Sys(int op1, int crn, int crm, int op2, const Register& rt = xzr) {
+ SingleEmissionCheckScope guard(this);
+ sys(op1, crn, crm, op2, rt);
+ }
+ void Dc(DataCacheOp op, const Register& rt) {
+ SingleEmissionCheckScope guard(this);
+ dc(op, rt);
+ }
+ void Ic(InstructionCacheOp op, const Register& rt) {
+ SingleEmissionCheckScope guard(this);
+ ic(op, rt);
+ }
+ void Msub(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT(!ra.IsZero());
+ SingleEmissionCheckScope guard(this);
+ msub(rd, rn, rm, ra);
+ }
+ void Mul(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ mul(rd, rn, rm);
+ }
+ void Nop() {
+ SingleEmissionCheckScope guard(this);
+ nop();
+ }
+ void Csdb() {
+ SingleEmissionCheckScope guard(this);
+ csdb();
+ }
+ void Rbit(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ rbit(rd, rn);
+ }
+ void Ret(const Register& xn = lr) {
+ VIXL_ASSERT(!xn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ ret(xn);
+ }
+ void Rev(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ rev(rd, rn);
+ }
+ void Rev16(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ rev16(rd, rn);
+ }
+ void Rev32(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ rev32(rd, rn);
+ }
+ void Ror(const Register& rd, const Register& rs, unsigned shift) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rs.IsZero());
+ SingleEmissionCheckScope guard(this);
+ ror(rd, rs, shift);
+ }
+ void Ror(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ rorv(rd, rn, rm);
+ }
+ void Sbfiz(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ sbfiz(rd, rn, lsb, width);
+ }
+ void Sbfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ sbfm(rd, rn, immr, imms);
+ }
+ void Sbfx(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ sbfx(rd, rn, lsb, width);
+ }
+ void Scvtf(const VRegister& vd, const Register& rn, int fbits = 0) {
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ scvtf(vd, rn, fbits);
+ }
+ void Sdiv(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ sdiv(rd, rn, rm);
+ }
+ void Smaddl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT(!ra.IsZero());
+ SingleEmissionCheckScope guard(this);
+ smaddl(rd, rn, rm, ra);
+ }
+ void Smsubl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT(!ra.IsZero());
+ SingleEmissionCheckScope guard(this);
+ smsubl(rd, rn, rm, ra);
+ }
+ void Smull(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ smull(rd, rn, rm);
+ }
+ void Smulh(const Register& xd, const Register& xn, const Register& xm) {
+ VIXL_ASSERT(!xd.IsZero());
+ VIXL_ASSERT(!xn.IsZero());
+ VIXL_ASSERT(!xm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ smulh(xd, xn, xm);
+ }
+ void Stlr(const Register& rt, const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ stlr(rt, dst);
+ }
+ void Stlrb(const Register& rt, const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ stlrb(rt, dst);
+ }
+ void Stlrh(const Register& rt, const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ stlrh(rt, dst);
+ }
+ void Stlxp(const Register& rs,
+ const Register& rt,
+ const Register& rt2,
+ const MemOperand& dst) {
+ VIXL_ASSERT(!rs.Aliases(dst.base()));
+ VIXL_ASSERT(!rs.Aliases(rt));
+ VIXL_ASSERT(!rs.Aliases(rt2));
+ SingleEmissionCheckScope guard(this);
+ stlxp(rs, rt, rt2, dst);
+ }
+ void Stlxr(const Register& rs, const Register& rt, const MemOperand& dst) {
+ VIXL_ASSERT(!rs.Aliases(dst.base()));
+ VIXL_ASSERT(!rs.Aliases(rt));
+ SingleEmissionCheckScope guard(this);
+ stlxr(rs, rt, dst);
+ }
+ void Stlxrb(const Register& rs, const Register& rt, const MemOperand& dst) {
+ VIXL_ASSERT(!rs.Aliases(dst.base()));
+ VIXL_ASSERT(!rs.Aliases(rt));
+ SingleEmissionCheckScope guard(this);
+ stlxrb(rs, rt, dst);
+ }
+ void Stlxrh(const Register& rs, const Register& rt, const MemOperand& dst) {
+ VIXL_ASSERT(!rs.Aliases(dst.base()));
+ VIXL_ASSERT(!rs.Aliases(rt));
+ SingleEmissionCheckScope guard(this);
+ stlxrh(rs, rt, dst);
+ }
+ void Stnp(const CPURegister& rt,
+ const CPURegister& rt2,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ stnp(rt, rt2, dst);
+ }
+ void Stxp(const Register& rs,
+ const Register& rt,
+ const Register& rt2,
+ const MemOperand& dst) {
+ VIXL_ASSERT(!rs.Aliases(dst.base()));
+ VIXL_ASSERT(!rs.Aliases(rt));
+ VIXL_ASSERT(!rs.Aliases(rt2));
+ SingleEmissionCheckScope guard(this);
+ stxp(rs, rt, rt2, dst);
+ }
+ void Stxr(const Register& rs, const Register& rt, const MemOperand& dst) {
+ VIXL_ASSERT(!rs.Aliases(dst.base()));
+ VIXL_ASSERT(!rs.Aliases(rt));
+ SingleEmissionCheckScope guard(this);
+ stxr(rs, rt, dst);
+ }
+ void Stxrb(const Register& rs, const Register& rt, const MemOperand& dst) {
+ VIXL_ASSERT(!rs.Aliases(dst.base()));
+ VIXL_ASSERT(!rs.Aliases(rt));
+ SingleEmissionCheckScope guard(this);
+ stxrb(rs, rt, dst);
+ }
+ void Stxrh(const Register& rs, const Register& rt, const MemOperand& dst) {
+ VIXL_ASSERT(!rs.Aliases(dst.base()));
+ VIXL_ASSERT(!rs.Aliases(rt));
+ SingleEmissionCheckScope guard(this);
+ stxrh(rs, rt, dst);
+ }
+ void Svc(int code) {
+ SingleEmissionCheckScope guard(this);
+ svc(code);
+ }
+ void Sxtb(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ sxtb(rd, rn);
+ }
+ void Sxth(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ sxth(rd, rn);
+ }
+ void Sxtw(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ sxtw(rd, rn);
+ }
+ void Tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ tbl(vd, vn, vm);
+ }
+ void Tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ tbl(vd, vn, vn2, vm);
+ }
+ void Tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ tbl(vd, vn, vn2, vn3, vm);
+ }
+ void Tbl(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vn4,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ tbl(vd, vn, vn2, vn3, vn4, vm);
+ }
+ void Tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ tbx(vd, vn, vm);
+ }
+ void Tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ tbx(vd, vn, vn2, vm);
+ }
+ void Tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ tbx(vd, vn, vn2, vn3, vm);
+ }
+ void Tbx(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vn2,
+ const VRegister& vn3,
+ const VRegister& vn4,
+ const VRegister& vm) {
+ SingleEmissionCheckScope guard(this);
+ tbx(vd, vn, vn2, vn3, vn4, vm);
+ }
+ void Tbnz(const Register& rt, unsigned bit_pos, Label* label);
+ void Tbz(const Register& rt, unsigned bit_pos, Label* label);
+ void Ubfiz(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ ubfiz(rd, rn, lsb, width);
+ }
+ void Ubfm(const Register& rd,
+ const Register& rn,
+ unsigned immr,
+ unsigned imms) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ ubfm(rd, rn, immr, imms);
+ }
+ void Ubfx(const Register& rd,
+ const Register& rn,
+ unsigned lsb,
+ unsigned width) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ ubfx(rd, rn, lsb, width);
+ }
+ void Ucvtf(const VRegister& vd, const Register& rn, int fbits = 0) {
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ ucvtf(vd, rn, fbits);
+ }
+ void Udiv(const Register& rd, const Register& rn, const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ udiv(rd, rn, rm);
+ }
+ void Umaddl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT(!ra.IsZero());
+ SingleEmissionCheckScope guard(this);
+ umaddl(rd, rn, rm, ra);
+ }
+ void Umull(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ umull(rd, rn, rm);
+ }
+ void Umulh(const Register& xd, const Register& xn, const Register& xm) {
+ VIXL_ASSERT(!xd.IsZero());
+ VIXL_ASSERT(!xn.IsZero());
+ VIXL_ASSERT(!xm.IsZero());
+ SingleEmissionCheckScope guard(this);
+ umulh(xd, xn, xm);
+ }
+ void Umsubl(const Register& rd,
+ const Register& rn,
+ const Register& rm,
+ const Register& ra) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ VIXL_ASSERT(!rm.IsZero());
+ VIXL_ASSERT(!ra.IsZero());
+ SingleEmissionCheckScope guard(this);
+ umsubl(rd, rn, rm, ra);
+ }
+
+ void Unreachable() {
+ SingleEmissionCheckScope guard(this);
+ Emit(UNDEFINED_INST_PATTERN);
+ }
+
+ void Uxtb(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ uxtb(rd, rn);
+ }
+ void Uxth(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ uxth(rd, rn);
+ }
+ void Uxtw(const Register& rd, const Register& rn) {
+ VIXL_ASSERT(!rd.IsZero());
+ VIXL_ASSERT(!rn.IsZero());
+ SingleEmissionCheckScope guard(this);
+ uxtw(rd, rn);
+ }
+
+ // NEON 3 vector register instructions.
+ #define NEON_3VREG_MACRO_LIST(V) \
+ V(add, Add) \
+ V(addhn, Addhn) \
+ V(addhn2, Addhn2) \
+ V(addp, Addp) \
+ V(and_, And) \
+ V(bic, Bic) \
+ V(bif, Bif) \
+ V(bit, Bit) \
+ V(bsl, Bsl) \
+ V(cmeq, Cmeq) \
+ V(cmge, Cmge) \
+ V(cmgt, Cmgt) \
+ V(cmhi, Cmhi) \
+ V(cmhs, Cmhs) \
+ V(cmtst, Cmtst) \
+ V(eor, Eor) \
+ V(fabd, Fabd) \
+ V(facge, Facge) \
+ V(facgt, Facgt) \
+ V(faddp, Faddp) \
+ V(fcmeq, Fcmeq) \
+ V(fcmge, Fcmge) \
+ V(fcmgt, Fcmgt) \
+ V(fmaxnmp, Fmaxnmp) \
+ V(fmaxp, Fmaxp) \
+ V(fminnmp, Fminnmp) \
+ V(fminp, Fminp) \
+ V(fmla, Fmla) \
+ V(fmls, Fmls) \
+ V(fmulx, Fmulx) \
+ V(frecps, Frecps) \
+ V(frsqrts, Frsqrts) \
+ V(mla, Mla) \
+ V(mls, Mls) \
+ V(mul, Mul) \
+ V(orn, Orn) \
+ V(orr, Orr) \
+ V(pmul, Pmul) \
+ V(pmull, Pmull) \
+ V(pmull2, Pmull2) \
+ V(raddhn, Raddhn) \
+ V(raddhn2, Raddhn2) \
+ V(rsubhn, Rsubhn) \
+ V(rsubhn2, Rsubhn2) \
+ V(saba, Saba) \
+ V(sabal, Sabal) \
+ V(sabal2, Sabal2) \
+ V(sabd, Sabd) \
+ V(sabdl, Sabdl) \
+ V(sabdl2, Sabdl2) \
+ V(saddl, Saddl) \
+ V(saddl2, Saddl2) \
+ V(saddw, Saddw) \
+ V(saddw2, Saddw2) \
+ V(shadd, Shadd) \
+ V(shsub, Shsub) \
+ V(smax, Smax) \
+ V(smaxp, Smaxp) \
+ V(smin, Smin) \
+ V(sminp, Sminp) \
+ V(smlal, Smlal) \
+ V(smlal2, Smlal2) \
+ V(smlsl, Smlsl) \
+ V(smlsl2, Smlsl2) \
+ V(smull, Smull) \
+ V(smull2, Smull2) \
+ V(sqadd, Sqadd) \
+ V(sqdmlal, Sqdmlal) \
+ V(sqdmlal2, Sqdmlal2) \
+ V(sqdmlsl, Sqdmlsl) \
+ V(sqdmlsl2, Sqdmlsl2) \
+ V(sqdmulh, Sqdmulh) \
+ V(sqdmull, Sqdmull) \
+ V(sqdmull2, Sqdmull2) \
+ V(sqrdmulh, Sqrdmulh) \
+ V(sqrshl, Sqrshl) \
+ V(sqshl, Sqshl) \
+ V(sqsub, Sqsub) \
+ V(srhadd, Srhadd) \
+ V(srshl, Srshl) \
+ V(sshl, Sshl) \
+ V(ssubl, Ssubl) \
+ V(ssubl2, Ssubl2) \
+ V(ssubw, Ssubw) \
+ V(ssubw2, Ssubw2) \
+ V(sub, Sub) \
+ V(subhn, Subhn) \
+ V(subhn2, Subhn2) \
+ V(trn1, Trn1) \
+ V(trn2, Trn2) \
+ V(uaba, Uaba) \
+ V(uabal, Uabal) \
+ V(uabal2, Uabal2) \
+ V(uabd, Uabd) \
+ V(uabdl, Uabdl) \
+ V(uabdl2, Uabdl2) \
+ V(uaddl, Uaddl) \
+ V(uaddl2, Uaddl2) \
+ V(uaddw, Uaddw) \
+ V(uaddw2, Uaddw2) \
+ V(uhadd, Uhadd) \
+ V(uhsub, Uhsub) \
+ V(umax, Umax) \
+ V(umaxp, Umaxp) \
+ V(umin, Umin) \
+ V(uminp, Uminp) \
+ V(umlal, Umlal) \
+ V(umlal2, Umlal2) \
+ V(umlsl, Umlsl) \
+ V(umlsl2, Umlsl2) \
+ V(umull, Umull) \
+ V(umull2, Umull2) \
+ V(uqadd, Uqadd) \
+ V(uqrshl, Uqrshl) \
+ V(uqshl, Uqshl) \
+ V(uqsub, Uqsub) \
+ V(urhadd, Urhadd) \
+ V(urshl, Urshl) \
+ V(ushl, Ushl) \
+ V(usubl, Usubl) \
+ V(usubl2, Usubl2) \
+ V(usubw, Usubw) \
+ V(usubw2, Usubw2) \
+ V(uzp1, Uzp1) \
+ V(uzp2, Uzp2) \
+ V(zip1, Zip1) \
+ V(zip2, Zip2)
+
+ #define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm) { \
+ SingleEmissionCheckScope guard(this); \
+ ASM(vd, vn, vm); \
+ }
+ NEON_3VREG_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+ #undef DEFINE_MACRO_ASM_FUNC
+
+ // NEON 2 vector register instructions.
+ #define NEON_2VREG_MACRO_LIST(V) \
+ V(abs, Abs) \
+ V(addp, Addp) \
+ V(addv, Addv) \
+ V(cls, Cls) \
+ V(clz, Clz) \
+ V(cnt, Cnt) \
+ V(fabs, Fabs) \
+ V(faddp, Faddp) \
+ V(fcvtas, Fcvtas) \
+ V(fcvtau, Fcvtau) \
+ V(fcvtms, Fcvtms) \
+ V(fcvtmu, Fcvtmu) \
+ V(fcvtns, Fcvtns) \
+ V(fcvtnu, Fcvtnu) \
+ V(fcvtps, Fcvtps) \
+ V(fcvtpu, Fcvtpu) \
+ V(fmaxnmp, Fmaxnmp) \
+ V(fmaxnmv, Fmaxnmv) \
+ V(fmaxp, Fmaxp) \
+ V(fmaxv, Fmaxv) \
+ V(fminnmp, Fminnmp) \
+ V(fminnmv, Fminnmv) \
+ V(fminp, Fminp) \
+ V(fminv, Fminv) \
+ V(fneg, Fneg) \
+ V(frecpe, Frecpe) \
+ V(frecpx, Frecpx) \
+ V(frinta, Frinta) \
+ V(frinti, Frinti) \
+ V(frintm, Frintm) \
+ V(frintn, Frintn) \
+ V(frintp, Frintp) \
+ V(frintx, Frintx) \
+ V(frintz, Frintz) \
+ V(frsqrte, Frsqrte) \
+ V(fsqrt, Fsqrt) \
+ V(mov, Mov) \
+ V(mvn, Mvn) \
+ V(neg, Neg) \
+ V(not_, Not) \
+ V(rbit, Rbit) \
+ V(rev16, Rev16) \
+ V(rev32, Rev32) \
+ V(rev64, Rev64) \
+ V(sadalp, Sadalp) \
+ V(saddlp, Saddlp) \
+ V(saddlv, Saddlv) \
+ V(smaxv, Smaxv) \
+ V(sminv, Sminv) \
+ V(sqabs, Sqabs) \
+ V(sqneg, Sqneg) \
+ V(sqxtn, Sqxtn) \
+ V(sqxtn2, Sqxtn2) \
+ V(sqxtun, Sqxtun) \
+ V(sqxtun2, Sqxtun2) \
+ V(suqadd, Suqadd) \
+ V(sxtl, Sxtl) \
+ V(sxtl2, Sxtl2) \
+ V(uadalp, Uadalp) \
+ V(uaddlp, Uaddlp) \
+ V(uaddlv, Uaddlv) \
+ V(umaxv, Umaxv) \
+ V(uminv, Uminv) \
+ V(uqxtn, Uqxtn) \
+ V(uqxtn2, Uqxtn2) \
+ V(urecpe, Urecpe) \
+ V(ursqrte, Ursqrte) \
+ V(usqadd, Usqadd) \
+ V(uxtl, Uxtl) \
+ V(uxtl2, Uxtl2) \
+ V(xtn, Xtn) \
+ V(xtn2, Xtn2)
+
+ #define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, \
+ const VRegister& vn) { \
+ SingleEmissionCheckScope guard(this); \
+ ASM(vd, vn); \
+ }
+ NEON_2VREG_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+ #undef DEFINE_MACRO_ASM_FUNC
+
+ // NEON 2 vector register with immediate instructions.
+ #define NEON_2VREG_FPIMM_MACRO_LIST(V) \
+ V(fcmeq, Fcmeq) \
+ V(fcmge, Fcmge) \
+ V(fcmgt, Fcmgt) \
+ V(fcmle, Fcmle) \
+ V(fcmlt, Fcmlt)
+
+ #define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, \
+ const VRegister& vn, \
+ double imm) { \
+ SingleEmissionCheckScope guard(this); \
+ ASM(vd, vn, imm); \
+ }
+ NEON_2VREG_FPIMM_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+ #undef DEFINE_MACRO_ASM_FUNC
+
+ // NEON by element instructions.
+ #define NEON_BYELEMENT_MACRO_LIST(V) \
+ V(fmul, Fmul) \
+ V(fmla, Fmla) \
+ V(fmls, Fmls) \
+ V(fmulx, Fmulx) \
+ V(mul, Mul) \
+ V(mla, Mla) \
+ V(mls, Mls) \
+ V(sqdmulh, Sqdmulh) \
+ V(sqrdmulh, Sqrdmulh) \
+ V(sqdmull, Sqdmull) \
+ V(sqdmull2, Sqdmull2) \
+ V(sqdmlal, Sqdmlal) \
+ V(sqdmlal2, Sqdmlal2) \
+ V(sqdmlsl, Sqdmlsl) \
+ V(sqdmlsl2, Sqdmlsl2) \
+ V(smull, Smull) \
+ V(smull2, Smull2) \
+ V(smlal, Smlal) \
+ V(smlal2, Smlal2) \
+ V(smlsl, Smlsl) \
+ V(smlsl2, Smlsl2) \
+ V(umull, Umull) \
+ V(umull2, Umull2) \
+ V(umlal, Umlal) \
+ V(umlal2, Umlal2) \
+ V(umlsl, Umlsl) \
+ V(umlsl2, Umlsl2)
+
+ #define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, \
+ const VRegister& vn, \
+ const VRegister& vm, \
+ int vm_index \
+ ) { \
+ SingleEmissionCheckScope guard(this); \
+ ASM(vd, vn, vm, vm_index); \
+ }
+ NEON_BYELEMENT_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+ #undef DEFINE_MACRO_ASM_FUNC
+
+ #define NEON_2VREG_SHIFT_MACRO_LIST(V) \
+ V(rshrn, Rshrn) \
+ V(rshrn2, Rshrn2) \
+ V(shl, Shl) \
+ V(shll, Shll) \
+ V(shll2, Shll2) \
+ V(shrn, Shrn) \
+ V(shrn2, Shrn2) \
+ V(sli, Sli) \
+ V(sqrshrn, Sqrshrn) \
+ V(sqrshrn2, Sqrshrn2) \
+ V(sqrshrun, Sqrshrun) \
+ V(sqrshrun2, Sqrshrun2) \
+ V(sqshl, Sqshl) \
+ V(sqshlu, Sqshlu) \
+ V(sqshrn, Sqshrn) \
+ V(sqshrn2, Sqshrn2) \
+ V(sqshrun, Sqshrun) \
+ V(sqshrun2, Sqshrun2) \
+ V(sri, Sri) \
+ V(srshr, Srshr) \
+ V(srsra, Srsra) \
+ V(sshll, Sshll) \
+ V(sshll2, Sshll2) \
+ V(sshr, Sshr) \
+ V(ssra, Ssra) \
+ V(uqrshrn, Uqrshrn) \
+ V(uqrshrn2, Uqrshrn2) \
+ V(uqshl, Uqshl) \
+ V(uqshrn, Uqshrn) \
+ V(uqshrn2, Uqshrn2) \
+ V(urshr, Urshr) \
+ V(ursra, Ursra) \
+ V(ushll, Ushll) \
+ V(ushll2, Ushll2) \
+ V(ushr, Ushr) \
+ V(usra, Usra) \
+
+ #define DEFINE_MACRO_ASM_FUNC(ASM, MASM) \
+ void MASM(const VRegister& vd, \
+ const VRegister& vn, \
+ int shift) { \
+ SingleEmissionCheckScope guard(this); \
+ ASM(vd, vn, shift); \
+ }
+ NEON_2VREG_SHIFT_MACRO_LIST(DEFINE_MACRO_ASM_FUNC)
+ #undef DEFINE_MACRO_ASM_FUNC
+
+ void Bic(const VRegister& vd,
+ const int imm8,
+ const int left_shift = 0) {
+ SingleEmissionCheckScope guard(this);
+ bic(vd, imm8, left_shift);
+ }
+ void Cmeq(const VRegister& vd,
+ const VRegister& vn,
+ int imm) {
+ SingleEmissionCheckScope guard(this);
+ cmeq(vd, vn, imm);
+ }
+ void Cmge(const VRegister& vd,
+ const VRegister& vn,
+ int imm) {
+ SingleEmissionCheckScope guard(this);
+ cmge(vd, vn, imm);
+ }
+ void Cmgt(const VRegister& vd,
+ const VRegister& vn,
+ int imm) {
+ SingleEmissionCheckScope guard(this);
+ cmgt(vd, vn, imm);
+ }
+ void Cmle(const VRegister& vd,
+ const VRegister& vn,
+ int imm) {
+ SingleEmissionCheckScope guard(this);
+ cmle(vd, vn, imm);
+ }
+ void Cmlt(const VRegister& vd,
+ const VRegister& vn,
+ int imm) {
+ SingleEmissionCheckScope guard(this);
+ cmlt(vd, vn, imm);
+ }
+ void Dup(const VRegister& vd,
+ const VRegister& vn,
+ int index) {
+ SingleEmissionCheckScope guard(this);
+ dup(vd, vn, index);
+ }
+ void Dup(const VRegister& vd,
+ const Register& rn) {
+ SingleEmissionCheckScope guard(this);
+ dup(vd, rn);
+ }
+ void Ext(const VRegister& vd,
+ const VRegister& vn,
+ const VRegister& vm,
+ int index) {
+ SingleEmissionCheckScope guard(this);
+ ext(vd, vn, vm, index);
+ }
+ void Ins(const VRegister& vd,
+ int vd_index,
+ const VRegister& vn,
+ int vn_index) {
+ SingleEmissionCheckScope guard(this);
+ ins(vd, vd_index, vn, vn_index);
+ }
+ void Ins(const VRegister& vd,
+ int vd_index,
+ const Register& rn) {
+ SingleEmissionCheckScope guard(this);
+ ins(vd, vd_index, rn);
+ }
+ void Ld1(const VRegister& vt,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld1(vt, src);
+ }
+ void Ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld1(vt, vt2, src);
+ }
+ void Ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld1(vt, vt2, vt3, src);
+ }
+ void Ld1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld1(vt, vt2, vt3, vt4, src);
+ }
+ void Ld1(const VRegister& vt,
+ int lane,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld1(vt, lane, src);
+ }
+ void Ld1r(const VRegister& vt,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld1r(vt, src);
+ }
+ void Ld2(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld2(vt, vt2, src);
+ }
+ void Ld2(const VRegister& vt,
+ const VRegister& vt2,
+ int lane,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld2(vt, vt2, lane, src);
+ }
+ void Ld2r(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld2r(vt, vt2, src);
+ }
+ void Ld3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld3(vt, vt2, vt3, src);
+ }
+ void Ld3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ int lane,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld3(vt, vt2, vt3, lane, src);
+ }
+ void Ld3r(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld3r(vt, vt2, vt3, src);
+ }
+ void Ld4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld4(vt, vt2, vt3, vt4, src);
+ }
+ void Ld4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ int lane,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld4(vt, vt2, vt3, vt4, lane, src);
+ }
+ void Ld4r(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& src) {
+ SingleEmissionCheckScope guard(this);
+ ld4r(vt, vt2, vt3, vt4, src);
+ }
+ void Mov(const VRegister& vd,
+ int vd_index,
+ const VRegister& vn,
+ int vn_index) {
+ SingleEmissionCheckScope guard(this);
+ mov(vd, vd_index, vn, vn_index);
+ }
+ void Mov(const VRegister& vd,
+ const VRegister& vn,
+ int index) {
+ SingleEmissionCheckScope guard(this);
+ mov(vd, vn, index);
+ }
+ void Mov(const VRegister& vd,
+ int vd_index,
+ const Register& rn) {
+ SingleEmissionCheckScope guard(this);
+ mov(vd, vd_index, rn);
+ }
+ void Mov(const Register& rd,
+ const VRegister& vn,
+ int vn_index) {
+ SingleEmissionCheckScope guard(this);
+ mov(rd, vn, vn_index);
+ }
+ void Movi(const VRegister& vd,
+ uint64_t imm,
+ Shift shift = LSL,
+ int shift_amount = 0);
+ void Movi(const VRegister& vd, uint64_t hi, uint64_t lo);
+ void Mvni(const VRegister& vd,
+ const int imm8,
+ Shift shift = LSL,
+ const int shift_amount = 0) {
+ SingleEmissionCheckScope guard(this);
+ mvni(vd, imm8, shift, shift_amount);
+ }
+ void Orr(const VRegister& vd,
+ const int imm8,
+ const int left_shift = 0) {
+ SingleEmissionCheckScope guard(this);
+ orr(vd, imm8, left_shift);
+ }
+ void Scvtf(const VRegister& vd,
+ const VRegister& vn,
+ int fbits = 0) {
+ SingleEmissionCheckScope guard(this);
+ scvtf(vd, vn, fbits);
+ }
+ void Ucvtf(const VRegister& vd,
+ const VRegister& vn,
+ int fbits = 0) {
+ SingleEmissionCheckScope guard(this);
+ ucvtf(vd, vn, fbits);
+ }
+ void Fcvtzs(const VRegister& vd,
+ const VRegister& vn,
+ int fbits = 0) {
+ SingleEmissionCheckScope guard(this);
+ fcvtzs(vd, vn, fbits);
+ }
+ void Fcvtzu(const VRegister& vd,
+ const VRegister& vn,
+ int fbits = 0) {
+ SingleEmissionCheckScope guard(this);
+ fcvtzu(vd, vn, fbits);
+ }
+ void St1(const VRegister& vt,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st1(vt, dst);
+ }
+ void St1(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st1(vt, vt2, dst);
+ }
+ void St1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st1(vt, vt2, vt3, dst);
+ }
+ void St1(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st1(vt, vt2, vt3, vt4, dst);
+ }
+ void St1(const VRegister& vt,
+ int lane,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st1(vt, lane, dst);
+ }
+ void St2(const VRegister& vt,
+ const VRegister& vt2,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st2(vt, vt2, dst);
+ }
+ void St3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st3(vt, vt2, vt3, dst);
+ }
+ void St4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st4(vt, vt2, vt3, vt4, dst);
+ }
+ void St2(const VRegister& vt,
+ const VRegister& vt2,
+ int lane,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st2(vt, vt2, lane, dst);
+ }
+ void St3(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ int lane,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st3(vt, vt2, vt3, lane, dst);
+ }
+ void St4(const VRegister& vt,
+ const VRegister& vt2,
+ const VRegister& vt3,
+ const VRegister& vt4,
+ int lane,
+ const MemOperand& dst) {
+ SingleEmissionCheckScope guard(this);
+ st4(vt, vt2, vt3, vt4, lane, dst);
+ }
+ void Smov(const Register& rd,
+ const VRegister& vn,
+ int vn_index) {
+ SingleEmissionCheckScope guard(this);
+ smov(rd, vn, vn_index);
+ }
+ void Umov(const Register& rd,
+ const VRegister& vn,
+ int vn_index) {
+ SingleEmissionCheckScope guard(this);
+ umov(rd, vn, vn_index);
+ }
+ void Crc32b(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ SingleEmissionCheckScope guard(this);
+ crc32b(rd, rn, rm);
+ }
+ void Crc32h(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ SingleEmissionCheckScope guard(this);
+ crc32h(rd, rn, rm);
+ }
+ void Crc32w(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ SingleEmissionCheckScope guard(this);
+ crc32w(rd, rn, rm);
+ }
+ void Crc32x(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ SingleEmissionCheckScope guard(this);
+ crc32x(rd, rn, rm);
+ }
+ void Crc32cb(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ SingleEmissionCheckScope guard(this);
+ crc32cb(rd, rn, rm);
+ }
+ void Crc32ch(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ SingleEmissionCheckScope guard(this);
+ crc32ch(rd, rn, rm);
+ }
+ void Crc32cw(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ SingleEmissionCheckScope guard(this);
+ crc32cw(rd, rn, rm);
+ }
+ void Crc32cx(const Register& rd,
+ const Register& rn,
+ const Register& rm) {
+ SingleEmissionCheckScope guard(this);
+ crc32cx(rd, rn, rm);
+ }
+
+ // Push the system stack pointer (sp) down to allow the same to be done to
+ // the current stack pointer (according to StackPointer()). This must be
+ // called _before_ accessing the memory.
+ //
+ // This is necessary when pushing or otherwise adding things to the stack, to
+ // satisfy the AAPCS64 constraint that the memory below the system stack
+ // pointer is not accessed.
+ //
+ // This method asserts that StackPointer() is not sp, since the call does
+ // not make sense in that context.
+ //
+ // TODO: This method can only accept values of 'space' that can be encoded in
+ // one instruction. Refer to the implementation for details.
+ void BumpSystemStackPointer(const Operand& space);
+
+ // Set the current stack pointer, but don't generate any code.
+ void SetStackPointer64(const Register& stack_pointer) {
+ VIXL_ASSERT(!TmpList()->IncludesAliasOf(stack_pointer));
+ sp_ = stack_pointer;
+ }
+
+ // Return the current stack pointer, as set by SetStackPointer.
+ const Register& StackPointer() const {
+ return sp_;
+ }
+
+ const Register& GetStackPointer64() const {
+ return sp_;
+ }
+
+ js::jit::RegisterOrSP getStackPointer() const {
+ return js::jit::RegisterOrSP(sp_.code());
+ }
+
+ CPURegList* TmpList() { return &tmp_list_; }
+ CPURegList* FPTmpList() { return &fptmp_list_; }
+
+ // Trace control when running the debug simulator.
+ //
+ // For example:
+ //
+ // __ Trace(LOG_REGS, TRACE_ENABLE);
+ // Will add registers to the trace if it wasn't already the case.
+ //
+ // __ Trace(LOG_DISASM, TRACE_DISABLE);
+ // Will stop logging disassembly. It has no effect if the disassembly wasn't
+ // already being logged.
+ void Trace(TraceParameters parameters, TraceCommand command);
+
+ // Log the requested data independently of what is being traced.
+ //
+ // For example:
+ //
+ // __ Log(LOG_FLAGS)
+ // Will output the flags.
+ void Log(TraceParameters parameters);
+
+ // Enable or disable instrumentation when an Instrument visitor is attached to
+ // the simulator.
+ void EnableInstrumentation();
+ void DisableInstrumentation();
+
+ // Add a marker to the instrumentation data produced by an Instrument visitor.
+ // The name is a two character string that will be attached to the marker in
+ // the output data.
+ void AnnotateInstrumentation(const char* marker_name);
+
+ private:
+ // The actual Push and Pop implementations. These don't generate any code
+ // other than that required for the push or pop. This allows
+ // (Push|Pop)CPURegList to bundle together setup code for a large block of
+ // registers.
+ //
+ // Note that size is per register, and is specified in bytes.
+ void PushHelper(int count, int size,
+ const CPURegister& src0, const CPURegister& src1,
+ const CPURegister& src2, const CPURegister& src3);
+ void PopHelper(int count, int size,
+ const CPURegister& dst0, const CPURegister& dst1,
+ const CPURegister& dst2, const CPURegister& dst3);
+
+ void Movi16bitHelper(const VRegister& vd, uint64_t imm);
+ void Movi32bitHelper(const VRegister& vd, uint64_t imm);
+ void Movi64bitHelper(const VRegister& vd, uint64_t imm);
+
+ // Perform necessary maintenance operations before a push or pop.
+ //
+ // Note that size is per register, and is specified in bytes.
+ void PrepareForPush(int count, int size);
+ void PrepareForPop(int count, int size);
+
+ // The actual implementation of load and store operations for CPURegList.
+ enum LoadStoreCPURegListAction {
+ kLoad,
+ kStore
+ };
+ void LoadStoreCPURegListHelper(LoadStoreCPURegListAction operation,
+ CPURegList registers,
+ const MemOperand& mem);
+ // Returns a MemOperand suitable for loading or storing a CPURegList at `dst`.
+ // This helper may allocate registers from `scratch_scope` and generate code
+ // to compute an intermediate address. The resulting MemOperand is only valid
+ // as long as `scratch_scope` remains valid.
+ MemOperand BaseMemOperandForLoadStoreCPURegList(
+ const CPURegList& registers,
+ const MemOperand& mem,
+ UseScratchRegisterScope* scratch_scope);
+
+ bool LabelIsOutOfRange(Label* label, ImmBranchType branch_type) {
+ return !Instruction::IsValidImmPCOffset(branch_type, nextOffset().getOffset() - label->offset());
+ }
+
+ // The register to use as a stack pointer for stack operations.
+ Register sp_;
+
+ // Scratch registers available for use by the MacroAssembler.
+ CPURegList tmp_list_;
+ CPURegList fptmp_list_;
+
+ ptrdiff_t checkpoint_;
+ ptrdiff_t recommended_checkpoint_;
+};
+
+
+// All Assembler emits MUST acquire/release the underlying code buffer. The
+// helper scope below will do so and optionally ensure the buffer is big enough
+// to receive the emit. It is possible to request the scope not to perform any
+// checks (kNoCheck) if for example it is known in advance the buffer size is
+// adequate or there is some other size checking mechanism in place.
+class CodeBufferCheckScope {
+ public:
+ // Tell whether or not the scope needs to ensure the associated CodeBuffer
+ // has enough space for the requested size.
+ enum CheckPolicy {
+ kNoCheck,
+ kCheck
+ };
+
+ // Tell whether or not the scope should assert the amount of code emitted
+ // within the scope is consistent with the requested amount.
+ enum AssertPolicy {
+ kNoAssert, // No assert required.
+ kExactSize, // The code emitted must be exactly size bytes.
+ kMaximumSize // The code emitted must be at most size bytes.
+ };
+
+ CodeBufferCheckScope(Assembler* assm,
+ size_t size,
+ CheckPolicy check_policy = kCheck,
+ AssertPolicy assert_policy = kMaximumSize)
+ { }
+
+ // This is a shortcut for CodeBufferCheckScope(assm, 0, kNoCheck, kNoAssert).
+ explicit CodeBufferCheckScope(Assembler* assm) {}
+};
+
+
+// Use this scope when you need a one-to-one mapping between methods and
+// instructions. This scope prevents the MacroAssembler from being called and
+// literal pools from being emitted. It also asserts the number of instructions
+// emitted is what you specified when creating the scope.
+// FIXME: Because of the disabled calls below, this class asserts nothing.
+class InstructionAccurateScope : public CodeBufferCheckScope {
+ public:
+ InstructionAccurateScope(MacroAssembler* masm,
+ int64_t count,
+ AssertPolicy policy = kExactSize)
+ : CodeBufferCheckScope(masm,
+ (count * kInstructionSize),
+ kCheck,
+ policy) {
+ }
+};
+
+
+// This scope utility allows scratch registers to be managed safely. The
+// MacroAssembler's TmpList() (and FPTmpList()) is used as a pool of scratch
+// registers. These registers can be allocated on demand, and will be returned
+// at the end of the scope.
+//
+// When the scope ends, the MacroAssembler's lists will be restored to their
+// original state, even if the lists were modified by some other means.
+class UseScratchRegisterScope {
+ public:
+ // This constructor implicitly calls the `Open` function to initialise the
+ // scope, so it is ready to use immediately after it has been constructed.
+ explicit UseScratchRegisterScope(MacroAssembler* masm);
+ // This constructor allows deferred and optional initialisation of the scope.
+ // The user is required to explicitly call the `Open` function before using
+ // the scope.
+ UseScratchRegisterScope();
+ // This function performs the actual initialisation work.
+ void Open(MacroAssembler* masm);
+
+ // The destructor always implicitly calls the `Close` function.
+ ~UseScratchRegisterScope();
+ // This function performs the cleaning-up work. It must succeed even if the
+ // scope has not been opened. It is safe to call multiple times.
+ void Close();
+
+
+ bool IsAvailable(const CPURegister& reg) const;
+
+
+ // Take a register from the appropriate temps list. It will be returned
+ // automatically when the scope ends.
+ Register AcquireW() { return AcquireNextAvailable(available_).W(); }
+ Register AcquireX() { return AcquireNextAvailable(available_).X(); }
+ VRegister AcquireS() { return AcquireNextAvailable(availablefp_).S(); }
+ VRegister AcquireD() { return AcquireNextAvailable(availablefp_).D(); }
+ VRegister AcquireQ() { return AcquireNextAvailable(availablefp_).Q(); }
+
+
+ Register AcquireSameSizeAs(const Register& reg);
+ VRegister AcquireSameSizeAs(const VRegister& reg);
+
+
+ // Explicitly release an acquired (or excluded) register, putting it back in
+ // the appropriate temps list.
+ void Release(const CPURegister& reg);
+
+
+ // Make the specified registers available as scratch registers for the
+ // duration of this scope.
+ void Include(const CPURegList& list);
+ void Include(const Register& reg1,
+ const Register& reg2 = NoReg,
+ const Register& reg3 = NoReg,
+ const Register& reg4 = NoReg);
+ void Include(const VRegister& reg1,
+ const VRegister& reg2 = NoVReg,
+ const VRegister& reg3 = NoVReg,
+ const VRegister& reg4 = NoVReg);
+
+
+ // Make sure that the specified registers are not available in this scope.
+ // This can be used to prevent helper functions from using sensitive
+ // registers, for example.
+ void Exclude(const CPURegList& list);
+ void Exclude(const Register& reg1,
+ const Register& reg2 = NoReg,
+ const Register& reg3 = NoReg,
+ const Register& reg4 = NoReg);
+ void Exclude(const VRegister& reg1,
+ const VRegister& reg2 = NoVReg,
+ const VRegister& reg3 = NoVReg,
+ const VRegister& reg4 = NoVReg);
+ void Exclude(const CPURegister& reg1,
+ const CPURegister& reg2 = NoCPUReg,
+ const CPURegister& reg3 = NoCPUReg,
+ const CPURegister& reg4 = NoCPUReg);
+
+
+ // Prevent any scratch registers from being used in this scope.
+ void ExcludeAll();
+
+
+ private:
+ static CPURegister AcquireNextAvailable(CPURegList* available);
+
+ static void ReleaseByCode(CPURegList* available, int code);
+
+ static void ReleaseByRegList(CPURegList* available,
+ RegList regs);
+
+ static void IncludeByRegList(CPURegList* available,
+ RegList exclude);
+
+ static void ExcludeByRegList(CPURegList* available,
+ RegList exclude);
+
+ // Available scratch registers.
+ CPURegList* available_; // kRegister
+ CPURegList* availablefp_; // kVRegister
+
+ // The state of the available lists at the start of this scope.
+ RegList old_available_; // kRegister
+ RegList old_availablefp_; // kVRegister
+#ifdef DEBUG
+ bool initialised_;
+#endif
+
+ // Disallow copy constructor and operator=.
+ UseScratchRegisterScope(const UseScratchRegisterScope&) {
+ VIXL_UNREACHABLE();
+ }
+ void operator=(const UseScratchRegisterScope&) {
+ VIXL_UNREACHABLE();
+ }
+};
+
+
+} // namespace vixl
+
+#endif // VIXL_A64_MACRO_ASSEMBLER_A64_H_
diff --git a/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp b/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp
new file mode 100644
index 0000000000..b9189cc23b
--- /dev/null
+++ b/js/src/jit/arm64/vixl/MozAssembler-vixl.cpp
@@ -0,0 +1,610 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Assembler-vixl.h"
+#include "jit/Label.h"
+
+namespace vixl {
+
+using LabelDoc = js::jit::DisassemblerSpew::LabelDoc;
+
+// Assembler
+void Assembler::FinalizeCode() {
+#ifdef DEBUG
+ finalized_ = true;
+#endif
+}
+
+// Unbound Label Representation.
+//
+// We can have multiple branches using the same label before it is bound.
+// Assembler::bind() must then be able to enumerate all the branches and patch
+// them to target the final label location.
+//
+// When a Label is unbound with uses, its offset is pointing to the tip of a
+// linked list of uses. The uses can be branches or adr/adrp instructions. In
+// the case of branches, the next member in the linked list is simply encoded
+// as the branch target. For adr/adrp, the relative pc offset is encoded in the
+// immediate field as a signed instruction offset.
+//
+// In both cases, the end of the list is encoded as a 0 pc offset, i.e. the
+// tail is pointing to itself.
+
+static const ptrdiff_t kEndOfLabelUseList = 0;
+
+BufferOffset
+MozBaseAssembler::NextLink(BufferOffset cur)
+{
+ Instruction* link = getInstructionAt(cur);
+ // Raw encoded offset.
+ ptrdiff_t offset = link->ImmPCRawOffset();
+ // End of the list is encoded as 0.
+ if (offset == kEndOfLabelUseList)
+ return BufferOffset();
+ // The encoded offset is the number of instructions to move.
+ return BufferOffset(cur.getOffset() + offset * kInstructionSize);
+}
+
+static ptrdiff_t
+EncodeOffset(BufferOffset cur, BufferOffset next)
+{
+ MOZ_ASSERT(next.assigned() && cur.assigned());
+ ptrdiff_t offset = next.getOffset() - cur.getOffset();
+ MOZ_ASSERT(offset % kInstructionSize == 0);
+ return offset / kInstructionSize;
+}
+
+void
+MozBaseAssembler::SetNextLink(BufferOffset cur, BufferOffset next)
+{
+ Instruction* link = getInstructionAt(cur);
+ link->SetImmPCRawOffset(EncodeOffset(cur, next));
+}
+
+// A common implementation for the LinkAndGet<Type>OffsetTo helpers.
+//
+// If the label is bound, returns the offset as a multiple of 1 << elementShift.
+// Otherwise, links the instruction to the label and returns the raw offset to
+// encode. (This will be an instruction count.)
+//
+// The offset is calculated by aligning the PC and label addresses down to a
+// multiple of 1 << elementShift, then calculating the (scaled) offset between
+// them. This matches the semantics of adrp, for example. (Assuming that the
+// assembler buffer is page-aligned, which it probably isn't.)
+//
+// For an unbound label, the returned offset will be encodable in the provided
+// branch range. If the label is already bound, the caller is expected to make
+// sure that it is in range, and emit the necessary branch instrutions if it
+// isn't.
+//
+ptrdiff_t
+MozBaseAssembler::LinkAndGetOffsetTo(BufferOffset branch, ImmBranchRangeType branchRange,
+ unsigned elementShift, Label* label)
+{
+ if (armbuffer_.oom())
+ return kEndOfLabelUseList;
+
+ if (label->bound()) {
+ // The label is bound: all uses are already linked.
+ ptrdiff_t branch_offset = ptrdiff_t(branch.getOffset() >> elementShift);
+ ptrdiff_t label_offset = ptrdiff_t(label->offset() >> elementShift);
+ return label_offset - branch_offset;
+ }
+
+ // Keep track of short-range branches targeting unbound labels. We may need
+ // to insert veneers in PatchShortRangeBranchToVeneer() below.
+ if (branchRange < NumShortBranchRangeTypes) {
+ // This is the last possible branch target.
+ BufferOffset deadline(branch.getOffset() +
+ Instruction::ImmBranchMaxForwardOffset(branchRange));
+ armbuffer_.registerBranchDeadline(branchRange, deadline);
+ }
+
+ // The label is unbound and previously unused: Store the offset in the label
+ // itself for patching by bind().
+ if (!label->used()) {
+ label->use(branch.getOffset());
+ return kEndOfLabelUseList;
+ }
+
+ // The label is unbound and has multiple users. Create a linked list between
+ // the branches, and update the linked list head in the label struct. This is
+ // not always trivial since the branches in the linked list have limited
+ // ranges.
+
+ // What is the earliest buffer offset that would be reachable by the branch
+ // we're about to add?
+ ptrdiff_t earliestReachable =
+ branch.getOffset() + Instruction::ImmBranchMinBackwardOffset(branchRange);
+
+ // If the existing instruction at the head of the list is within reach of the
+ // new branch, we can simply insert the new branch at the front of the list.
+ if (label->offset() >= earliestReachable) {
+ ptrdiff_t offset = EncodeOffset(branch, BufferOffset(label));
+ label->use(branch.getOffset());
+ MOZ_ASSERT(offset != kEndOfLabelUseList);
+ return offset;
+ }
+
+ // The label already has a linked list of uses, but we can't reach the head
+ // of the list with the allowed branch range. Insert this branch at a
+ // different position in the list.
+ //
+ // Find an existing branch, exbr, such that:
+ //
+ // 1. The new branch can be reached by exbr, and either
+ // 2a. The new branch can reach exbr's target, or
+ // 2b. The exbr branch is at the end of the list.
+ //
+ // Then the new branch can be inserted after exbr in the linked list.
+ //
+ // We know that it is always possible to find an exbr branch satisfying these
+ // conditions because of the PatchShortRangeBranchToVeneer() mechanism. All
+ // branches are guaranteed to either be able to reach the end of the
+ // assembler buffer, or they will be pointing to an unconditional branch that
+ // can.
+ //
+ // In particular, the end of the list is always a viable candidate, so we'll
+ // just get that.
+ BufferOffset next(label);
+ BufferOffset exbr;
+ do {
+ exbr = next;
+ next = NextLink(next);
+ } while (next.assigned());
+ SetNextLink(exbr, branch);
+
+ // This branch becomes the new end of the list.
+ return kEndOfLabelUseList;
+}
+
+ptrdiff_t MozBaseAssembler::LinkAndGetByteOffsetTo(BufferOffset branch, Label* label) {
+ return LinkAndGetOffsetTo(branch, UncondBranchRangeType, 0, label);
+}
+
+ptrdiff_t MozBaseAssembler::LinkAndGetInstructionOffsetTo(BufferOffset branch,
+ ImmBranchRangeType branchRange,
+ Label* label) {
+ return LinkAndGetOffsetTo(branch, branchRange, kInstructionSizeLog2, label);
+}
+
+ptrdiff_t MozBaseAssembler::LinkAndGetPageOffsetTo(BufferOffset branch, Label* label) {
+ return LinkAndGetOffsetTo(branch, UncondBranchRangeType, kPageSizeLog2, label);
+}
+
+BufferOffset Assembler::b(int imm26, const LabelDoc& doc) {
+ return EmitBranch(B | ImmUncondBranch(imm26), doc);
+}
+
+
+void Assembler::b(Instruction* at, int imm26) {
+ return EmitBranch(at, B | ImmUncondBranch(imm26));
+}
+
+
+BufferOffset Assembler::b(int imm19, Condition cond, const LabelDoc& doc) {
+ return EmitBranch(B_cond | ImmCondBranch(imm19) | cond, doc);
+}
+
+
+void Assembler::b(Instruction* at, int imm19, Condition cond) {
+ EmitBranch(at, B_cond | ImmCondBranch(imm19) | cond);
+}
+
+
+BufferOffset Assembler::b(Label* label) {
+ // Encode the relative offset from the inserted branch to the label.
+ LabelDoc doc = refLabel(label);
+ return b(LinkAndGetInstructionOffsetTo(nextInstrOffset(), UncondBranchRangeType, label), doc);
+}
+
+
+BufferOffset Assembler::b(Label* label, Condition cond) {
+ // Encode the relative offset from the inserted branch to the label.
+ LabelDoc doc = refLabel(label);
+ return b(LinkAndGetInstructionOffsetTo(nextInstrOffset(), CondBranchRangeType, label), cond, doc);
+}
+
+void Assembler::br(Instruction* at, const Register& xn) {
+ VIXL_ASSERT(xn.Is64Bits());
+ // No need for EmitBranch(): no immediate offset needs fixing.
+ Emit(at, BR | Rn(xn));
+}
+
+
+void Assembler::blr(Instruction* at, const Register& xn) {
+ VIXL_ASSERT(xn.Is64Bits());
+ // No need for EmitBranch(): no immediate offset needs fixing.
+ Emit(at, BLR | Rn(xn));
+}
+
+
+void Assembler::bl(int imm26, const LabelDoc& doc) {
+ EmitBranch(BL | ImmUncondBranch(imm26), doc);
+}
+
+
+void Assembler::bl(Instruction* at, int imm26) {
+ EmitBranch(at, BL | ImmUncondBranch(imm26));
+}
+
+
+void Assembler::bl(Label* label) {
+ // Encode the relative offset from the inserted branch to the label.
+ LabelDoc doc = refLabel(label);
+ return bl(LinkAndGetInstructionOffsetTo(nextInstrOffset(), UncondBranchRangeType, label), doc);
+}
+
+
+void Assembler::cbz(const Register& rt, int imm19, const LabelDoc& doc) {
+ EmitBranch(SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt), doc);
+}
+
+
+void Assembler::cbz(Instruction* at, const Register& rt, int imm19) {
+ EmitBranch(at, SF(rt) | CBZ | ImmCmpBranch(imm19) | Rt(rt));
+}
+
+
+void Assembler::cbz(const Register& rt, Label* label) {
+ // Encode the relative offset from the inserted branch to the label.
+ LabelDoc doc = refLabel(label);
+ return cbz(rt, LinkAndGetInstructionOffsetTo(nextInstrOffset(), CondBranchRangeType, label), doc);
+}
+
+
+void Assembler::cbnz(const Register& rt, int imm19, const LabelDoc& doc) {
+ EmitBranch(SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt), doc);
+}
+
+
+void Assembler::cbnz(Instruction* at, const Register& rt, int imm19) {
+ EmitBranch(at, SF(rt) | CBNZ | ImmCmpBranch(imm19) | Rt(rt));
+}
+
+
+void Assembler::cbnz(const Register& rt, Label* label) {
+ // Encode the relative offset from the inserted branch to the label.
+ LabelDoc doc = refLabel(label);
+ return cbnz(rt, LinkAndGetInstructionOffsetTo(nextInstrOffset(), CondBranchRangeType, label), doc);
+}
+
+
+void Assembler::tbz(const Register& rt, unsigned bit_pos, int imm14, const LabelDoc& doc) {
+ VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
+ EmitBranch(TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt), doc);
+}
+
+
+void Assembler::tbz(Instruction* at, const Register& rt, unsigned bit_pos, int imm14) {
+ VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
+ EmitBranch(at, TBZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
+}
+
+
+void Assembler::tbz(const Register& rt, unsigned bit_pos, Label* label) {
+ // Encode the relative offset from the inserted branch to the label.
+ LabelDoc doc = refLabel(label);
+ return tbz(rt, bit_pos, LinkAndGetInstructionOffsetTo(nextInstrOffset(), TestBranchRangeType, label), doc);
+}
+
+
+void Assembler::tbnz(const Register& rt, unsigned bit_pos, int imm14, const LabelDoc& doc) {
+ VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
+ EmitBranch(TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt), doc);
+}
+
+
+void Assembler::tbnz(Instruction* at, const Register& rt, unsigned bit_pos, int imm14) {
+ VIXL_ASSERT(rt.Is64Bits() || (rt.Is32Bits() && (bit_pos < kWRegSize)));
+ EmitBranch(at, TBNZ | ImmTestBranchBit(bit_pos) | ImmTestBranch(imm14) | Rt(rt));
+}
+
+
+void Assembler::tbnz(const Register& rt, unsigned bit_pos, Label* label) {
+ // Encode the relative offset from the inserted branch to the label.
+ LabelDoc doc = refLabel(label);
+ return tbnz(rt, bit_pos, LinkAndGetInstructionOffsetTo(nextInstrOffset(), TestBranchRangeType, label), doc);
+}
+
+
+void Assembler::adr(const Register& rd, int imm21, const LabelDoc& doc) {
+ VIXL_ASSERT(rd.Is64Bits());
+ EmitBranch(ADR | ImmPCRelAddress(imm21) | Rd(rd), doc);
+}
+
+
+void Assembler::adr(Instruction* at, const Register& rd, int imm21) {
+ VIXL_ASSERT(rd.Is64Bits());
+ EmitBranch(at, ADR | ImmPCRelAddress(imm21) | Rd(rd));
+}
+
+
+void Assembler::adr(const Register& rd, Label* label) {
+ // Encode the relative offset from the inserted adr to the label.
+ LabelDoc doc = refLabel(label);
+ return adr(rd, LinkAndGetByteOffsetTo(nextInstrOffset(), label), doc);
+}
+
+
+void Assembler::adrp(const Register& rd, int imm21, const LabelDoc& doc) {
+ VIXL_ASSERT(rd.Is64Bits());
+ EmitBranch(ADRP | ImmPCRelAddress(imm21) | Rd(rd), doc);
+}
+
+
+void Assembler::adrp(Instruction* at, const Register& rd, int imm21) {
+ VIXL_ASSERT(rd.Is64Bits());
+ EmitBranch(at, ADRP | ImmPCRelAddress(imm21) | Rd(rd));
+}
+
+
+void Assembler::adrp(const Register& rd, Label* label) {
+ VIXL_ASSERT(AllowPageOffsetDependentCode());
+ // Encode the relative offset from the inserted adr to the label.
+ LabelDoc doc = refLabel(label);
+ return adrp(rd, LinkAndGetPageOffsetTo(nextInstrOffset(), label), doc);
+}
+
+
+BufferOffset Assembler::ands(const Register& rd, const Register& rn, const Operand& operand) {
+ return Logical(rd, rn, operand, ANDS);
+}
+
+
+BufferOffset Assembler::tst(const Register& rn, const Operand& operand) {
+ return ands(AppropriateZeroRegFor(rn), rn, operand);
+}
+
+
+void Assembler::ldr(Instruction* at, const CPURegister& rt, int imm19) {
+ LoadLiteralOp op = LoadLiteralOpFor(rt);
+ Emit(at, op | ImmLLiteral(imm19) | Rt(rt));
+}
+
+
+BufferOffset Assembler::hint(SystemHint code) {
+ return Emit(HINT | ImmHint(code));
+}
+
+
+void Assembler::hint(Instruction* at, SystemHint code) {
+ Emit(at, HINT | ImmHint(code));
+}
+
+
+void Assembler::svc(Instruction* at, int code) {
+ VIXL_ASSERT(IsUint16(code));
+ Emit(at, SVC | ImmException(code));
+}
+
+
+void Assembler::nop(Instruction* at) {
+ hint(at, NOP);
+}
+
+
+void Assembler::csdb(Instruction* at) {
+ hint(at, CSDB);
+}
+
+
+BufferOffset Assembler::Logical(const Register& rd, const Register& rn,
+ const Operand& operand, LogicalOp op)
+{
+ VIXL_ASSERT(rd.size() == rn.size());
+ if (operand.IsImmediate()) {
+ int64_t immediate = operand.immediate();
+ unsigned reg_size = rd.size();
+
+ VIXL_ASSERT(immediate != 0);
+ VIXL_ASSERT(immediate != -1);
+ VIXL_ASSERT(rd.Is64Bits() || IsUint32(immediate));
+
+ // If the operation is NOT, invert the operation and immediate.
+ if ((op & NOT) == NOT) {
+ op = static_cast<LogicalOp>(op & ~NOT);
+ immediate = rd.Is64Bits() ? ~immediate : (~immediate & kWRegMask);
+ }
+
+ unsigned n, imm_s, imm_r;
+ if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
+ // Immediate can be encoded in the instruction.
+ return LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
+ } else {
+ // This case is handled in the macro assembler.
+ VIXL_UNREACHABLE();
+ }
+ } else {
+ VIXL_ASSERT(operand.IsShiftedRegister());
+ VIXL_ASSERT(operand.reg().size() == rd.size());
+ Instr dp_op = static_cast<Instr>(op | LogicalShiftedFixed);
+ return DataProcShiftedRegister(rd, rn, operand, LeaveFlags, dp_op);
+ }
+}
+
+
+BufferOffset Assembler::LogicalImmediate(const Register& rd, const Register& rn,
+ unsigned n, unsigned imm_s, unsigned imm_r, LogicalOp op)
+{
+ unsigned reg_size = rd.size();
+ Instr dest_reg = (op == ANDS) ? Rd(rd) : RdSP(rd);
+ return Emit(SF(rd) | LogicalImmediateFixed | op | BitN(n, reg_size) |
+ ImmSetBits(imm_s, reg_size) | ImmRotate(imm_r, reg_size) | dest_reg | Rn(rn));
+}
+
+
+BufferOffset Assembler::DataProcShiftedRegister(const Register& rd, const Register& rn,
+ const Operand& operand, FlagsUpdate S, Instr op)
+{
+ VIXL_ASSERT(operand.IsShiftedRegister());
+ VIXL_ASSERT(rn.Is64Bits() || (rn.Is32Bits() && IsUint5(operand.shift_amount())));
+ return Emit(SF(rd) | op | Flags(S) |
+ ShiftDP(operand.shift()) | ImmDPShift(operand.shift_amount()) |
+ Rm(operand.reg()) | Rn(rn) | Rd(rd));
+}
+
+
+void MozBaseAssembler::InsertIndexIntoTag(uint8_t* load, uint32_t index) {
+ // Store the js::jit::PoolEntry index into the instruction.
+ // finishPool() will walk over all literal load instructions
+ // and use PatchConstantPoolLoad() to patch to the final relative offset.
+ *((uint32_t*)load) |= Assembler::ImmLLiteral(index);
+}
+
+
+bool MozBaseAssembler::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr) {
+ Instruction* load = reinterpret_cast<Instruction*>(loadAddr);
+
+ // The load currently contains the js::jit::PoolEntry's index,
+ // as written by InsertIndexIntoTag().
+ uint32_t index = load->ImmLLiteral();
+
+ // Each entry in the literal pool is uint32_t-sized,
+ // but literals may use multiple entries.
+ uint32_t* constPool = reinterpret_cast<uint32_t*>(constPoolAddr);
+ Instruction* source = reinterpret_cast<Instruction*>(&constPool[index]);
+
+ load->SetImmLLiteral(source);
+ return false; // Nothing uses the return value.
+}
+
+void
+MozBaseAssembler::PatchShortRangeBranchToVeneer(ARMBuffer* buffer, unsigned rangeIdx,
+ BufferOffset deadline, BufferOffset veneer)
+{
+ // Reconstruct the position of the branch from (rangeIdx, deadline).
+ vixl::ImmBranchRangeType branchRange = static_cast<vixl::ImmBranchRangeType>(rangeIdx);
+ BufferOffset branch(deadline.getOffset() - Instruction::ImmBranchMaxForwardOffset(branchRange));
+ Instruction *branchInst = buffer->getInst(branch);
+ Instruction *veneerInst = buffer->getInst(veneer);
+
+ // Verify that the branch range matches what's encoded.
+ MOZ_ASSERT(Instruction::ImmBranchTypeToRange(branchInst->BranchType()) == branchRange);
+
+ // We want to insert veneer after branch in the linked list of instructions
+ // that use the same unbound label.
+ // The veneer should be an unconditional branch.
+ ptrdiff_t nextElemOffset = branchInst->ImmPCRawOffset();
+
+ // If offset is 0, this is the end of the linked list.
+ if (nextElemOffset != kEndOfLabelUseList) {
+ // Make the offset relative to veneer so it targets the same instruction
+ // as branchInst.
+ nextElemOffset *= kInstructionSize;
+ nextElemOffset += branch.getOffset() - veneer.getOffset();
+ nextElemOffset /= kInstructionSize;
+ }
+ Assembler::b(veneerInst, nextElemOffset);
+
+ // Now point branchInst at veneer. See also SetNextLink() above.
+ branchInst->SetImmPCRawOffset(EncodeOffset(branch, veneer));
+}
+
+struct PoolHeader {
+ uint32_t data;
+
+ struct Header {
+ // The size should take into account the pool header.
+ // The size is in units of Instruction (4bytes), not byte.
+ union {
+ struct {
+ uint32_t size : 15;
+
+ // "Natural" guards are part of the normal instruction stream,
+ // while "non-natural" guards are inserted for the sole purpose
+ // of skipping around a pool.
+ uint32_t isNatural : 1;
+ uint32_t ONES : 16;
+ };
+ uint32_t data;
+ };
+
+ Header(int size_, bool isNatural_)
+ : size(size_),
+ isNatural(isNatural_),
+ ONES(0xffff)
+ { }
+
+ Header(uint32_t data)
+ : data(data)
+ {
+ VIXL_STATIC_ASSERT(sizeof(Header) == sizeof(uint32_t));
+ VIXL_ASSERT(ONES == 0xffff);
+ }
+
+ uint32_t raw() const {
+ VIXL_STATIC_ASSERT(sizeof(Header) == sizeof(uint32_t));
+ return data;
+ }
+ };
+
+ PoolHeader(int size_, bool isNatural_)
+ : data(Header(size_, isNatural_).raw())
+ { }
+
+ uint32_t size() const {
+ Header tmp(data);
+ return tmp.size;
+ }
+
+ uint32_t isNatural() const {
+ Header tmp(data);
+ return tmp.isNatural;
+ }
+};
+
+
+void MozBaseAssembler::WritePoolHeader(uint8_t* start, js::jit::Pool* p, bool isNatural) {
+ static_assert(sizeof(PoolHeader) == 4);
+
+ // Get the total size of the pool.
+ const uintptr_t totalPoolSize = sizeof(PoolHeader) + p->getPoolSize();
+ const uintptr_t totalPoolInstructions = totalPoolSize / kInstructionSize;
+
+ VIXL_ASSERT((totalPoolSize & 0x3) == 0);
+ VIXL_ASSERT(totalPoolInstructions < (1 << 15));
+
+ PoolHeader header(totalPoolInstructions, isNatural);
+ *(PoolHeader*)start = header;
+}
+
+
+void MozBaseAssembler::WritePoolFooter(uint8_t* start, js::jit::Pool* p, bool isNatural) {
+ return;
+}
+
+
+void MozBaseAssembler::WritePoolGuard(BufferOffset branch, Instruction* inst, BufferOffset dest) {
+ int byteOffset = dest.getOffset() - branch.getOffset();
+ VIXL_ASSERT(byteOffset % kInstructionSize == 0);
+
+ int instOffset = byteOffset >> kInstructionSizeLog2;
+ Assembler::b(inst, instOffset);
+}
+
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h b/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h
new file mode 100644
index 0000000000..5d12f81bb1
--- /dev/null
+++ b/js/src/jit/arm64/vixl/MozBaseAssembler-vixl.h
@@ -0,0 +1,356 @@
+// Copyright 2013, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef jit_arm64_vixl_MozBaseAssembler_vixl_h
+#define jit_arm64_vixl_MozBaseAssembler_vixl_h
+
+
+#include "mozilla/Assertions.h" // MOZ_ASSERT
+#include "mozilla/Sprintf.h" // SprintfLiteral
+
+#include <stddef.h> // size_t
+#include <stdint.h> // uint8_t, uint32_t
+#include <string.h> // strstr
+
+#include "jit/arm64/vixl/Constants-vixl.h" // vixl::{HINT, NOP, ImmHint_offset}
+#include "jit/arm64/vixl/Globals-vixl.h" // VIXL_ASSERT
+#include "jit/arm64/vixl/Instructions-vixl.h" // vixl::{Instruction, NumShortBranchRangeTypes, Instr, ImmBranchRangeType}
+
+#include "jit/Label.h" // jit::Label
+#include "jit/shared/Assembler-shared.h" // jit::AssemblerShared
+#include "jit/shared/Disassembler-shared.h" // jit::DisassemblerSpew
+#include "jit/shared/IonAssemblerBuffer.h" // jit::BufferOffset
+#include "jit/shared/IonAssemblerBufferWithConstantPools.h" // jit::AssemblerBufferWithConstantPools
+
+namespace vixl {
+
+
+using js::jit::BufferOffset;
+using js::jit::DisassemblerSpew;
+using js::jit::Label;
+
+using LabelDoc = DisassemblerSpew::LabelDoc;
+using LiteralDoc = DisassemblerSpew::LiteralDoc;
+
+#ifdef JS_DISASM_ARM64
+void DisassembleInstruction(char* buffer, size_t bufsize, const Instruction* instr);
+#endif
+
+class MozBaseAssembler;
+typedef js::jit::AssemblerBufferWithConstantPools<1024, 4, Instruction, MozBaseAssembler,
+ NumShortBranchRangeTypes> ARMBuffer;
+
+// Base class for vixl::Assembler, for isolating Moz-specific changes to VIXL.
+class MozBaseAssembler : public js::jit::AssemblerShared {
+ // Buffer initialization constants.
+ static const unsigned BufferGuardSize = 1;
+ static const unsigned BufferHeaderSize = 1;
+ static const size_t BufferCodeAlignment = 8;
+ static const size_t BufferMaxPoolOffset = 1024;
+ static const unsigned BufferPCBias = 0;
+ static const uint32_t BufferAlignmentFillInstruction = HINT | (NOP << ImmHint_offset);
+ static const uint32_t BufferNopFillInstruction = HINT | (NOP << ImmHint_offset);
+ static const unsigned BufferNumDebugNopsToInsert = 0;
+
+#ifdef JS_DISASM_ARM64
+ static constexpr const char* const InstrIndent = " ";
+ static constexpr const char* const LabelIndent = " ";
+ static constexpr const char* const TargetIndent = " ";
+#endif
+
+ public:
+ MozBaseAssembler()
+ : armbuffer_(BufferGuardSize,
+ BufferHeaderSize,
+ BufferCodeAlignment,
+ BufferMaxPoolOffset,
+ BufferPCBias,
+ BufferAlignmentFillInstruction,
+ BufferNopFillInstruction,
+ BufferNumDebugNopsToInsert)
+ {
+#ifdef JS_DISASM_ARM64
+ spew_.setLabelIndent(LabelIndent);
+ spew_.setTargetIndent(TargetIndent);
+#endif
+}
+ ~MozBaseAssembler()
+ {
+#ifdef JS_DISASM_ARM64
+ spew_.spewOrphans();
+#endif
+ }
+
+ public:
+ // Return the Instruction at a given byte offset.
+ Instruction* getInstructionAt(BufferOffset offset) {
+ return armbuffer_.getInst(offset);
+ }
+
+ // Return the byte offset of a bound label.
+ template <typename T>
+ inline T GetLabelByteOffset(const js::jit::Label* label) {
+ VIXL_ASSERT(label->bound());
+ static_assert(sizeof(T) >= sizeof(uint32_t));
+ return reinterpret_cast<T>(label->offset());
+ }
+
+ protected:
+ // Get the buffer offset of the next inserted instruction. This may flush
+ // constant pools.
+ BufferOffset nextInstrOffset() {
+ return armbuffer_.nextInstrOffset();
+ }
+
+ // Get the next usable buffer offset. Note that a constant pool may be placed
+ // here before the next instruction is emitted.
+ BufferOffset nextOffset() const {
+ return armbuffer_.nextOffset();
+ }
+
+ // Allocate memory in the buffer by forwarding to armbuffer_.
+ // Propagate OOM errors.
+ BufferOffset allocLiteralLoadEntry(size_t numInst, unsigned numPoolEntries,
+ uint8_t* inst, uint8_t* data,
+ const LiteralDoc& doc = LiteralDoc(),
+ ARMBuffer::PoolEntry* pe = nullptr)
+ {
+ MOZ_ASSERT(inst);
+ MOZ_ASSERT(numInst == 1); /* If not, then fix disassembly */
+ BufferOffset offset = armbuffer_.allocEntry(numInst, numPoolEntries, inst,
+ data, pe);
+ propagateOOM(offset.assigned());
+#ifdef JS_DISASM_ARM64
+ Instruction* instruction = armbuffer_.getInstOrNull(offset);
+ if (instruction)
+ spewLiteralLoad(offset,
+ reinterpret_cast<vixl::Instruction*>(instruction), doc);
+#endif
+ return offset;
+ }
+
+#ifdef JS_DISASM_ARM64
+ DisassemblerSpew spew_;
+
+ void spew(BufferOffset offs, const vixl::Instruction* instr) {
+ if (spew_.isDisabled() || !instr)
+ return;
+
+ char buffer[2048];
+ DisassembleInstruction(buffer, sizeof(buffer), instr);
+ spew_.spew("%06" PRIx32 " %08" PRIx32 "%s%s",
+ (uint32_t)offs.getOffset(),
+ instr->InstructionBits(), InstrIndent, buffer);
+ }
+
+ void spewBranch(BufferOffset offs,
+ const vixl::Instruction* instr, const LabelDoc& target) {
+ if (spew_.isDisabled() || !instr)
+ return;
+
+ char buffer[2048];
+ DisassembleInstruction(buffer, sizeof(buffer), instr);
+
+ char labelBuf[128];
+ labelBuf[0] = 0;
+
+ bool hasTarget = target.valid;
+ if (!hasTarget)
+ SprintfLiteral(labelBuf, "-> (link-time target)");
+
+ if (instr->IsImmBranch() && hasTarget) {
+ // The target information in the instruction is likely garbage, so remove it.
+ // The target label will in any case be printed if we have it.
+ //
+ // The format of the instruction disassembly is /.*#.*/. Strip the # and later.
+ size_t i;
+ const size_t BUFLEN = sizeof(buffer)-1;
+ for ( i=0 ; i < BUFLEN && buffer[i] && buffer[i] != '#' ; i++ )
+ ;
+ buffer[i] = 0;
+
+ SprintfLiteral(labelBuf, "-> %d%s", target.doc, !target.bound ? "f" : "");
+ hasTarget = false;
+ }
+
+ spew_.spew("%06" PRIx32 " %08" PRIx32 "%s%s%s",
+ (uint32_t)offs.getOffset(),
+ instr->InstructionBits(), InstrIndent, buffer, labelBuf);
+
+ if (hasTarget)
+ spew_.spewRef(target);
+ }
+
+ void spewLiteralLoad(BufferOffset offs,
+ const vixl::Instruction* instr, const LiteralDoc& doc) {
+ if (spew_.isDisabled() || !instr)
+ return;
+
+ char buffer[2048];
+ DisassembleInstruction(buffer, sizeof(buffer), instr);
+
+ char litbuf[2048];
+ spew_.formatLiteral(doc, litbuf, sizeof(litbuf));
+
+ // The instruction will have the form /^.*pc\+0/ followed by junk that we
+ // don't need; try to strip it.
+
+ char *probe = strstr(buffer, "pc+0");
+ if (probe)
+ *(probe + 4) = 0;
+ spew_.spew("%06" PRIx32 " %08" PRIx32 "%s%s ; .const %s",
+ (uint32_t)offs.getOffset(),
+ instr->InstructionBits(), InstrIndent, buffer, litbuf);
+ }
+
+ LabelDoc refLabel(Label* label) {
+ if (spew_.isDisabled())
+ return LabelDoc();
+
+ return spew_.refLabel(label);
+ }
+#else
+ LabelDoc refLabel(js::jit::Label*) {
+ return LabelDoc();
+ }
+#endif
+
+ // Emit the instruction, returning its offset.
+ BufferOffset Emit(Instr instruction, bool isBranch = false) {
+ static_assert(sizeof(instruction) == kInstructionSize);
+ // TODO: isBranch is obsolete and should be removed.
+ (void)isBranch;
+ MOZ_ASSERT(hasCreator());
+ BufferOffset offs = armbuffer_.putInt(*(uint32_t*)(&instruction));
+#ifdef JS_DISASM_ARM64
+ if (!isBranch)
+ spew(offs, armbuffer_.getInstOrNull(offs));
+#endif
+ return offs;
+ }
+
+ BufferOffset EmitBranch(Instr instruction, const LabelDoc& doc) {
+ BufferOffset offs = Emit(instruction, true);
+#ifdef JS_DISASM_ARM64
+ spewBranch(offs, armbuffer_.getInstOrNull(offs), doc);
+#endif
+ return offs;
+ }
+
+ public:
+ // Emit the instruction at |at|.
+ static void Emit(Instruction* at, Instr instruction) {
+ static_assert(sizeof(instruction) == kInstructionSize);
+ memcpy(at, &instruction, sizeof(instruction));
+ }
+
+ static void EmitBranch(Instruction* at, Instr instruction) {
+ // TODO: Assert that the buffer already has the instruction marked as a branch.
+ Emit(at, instruction);
+ }
+
+ // Emit data inline in the instruction stream.
+ BufferOffset EmitData(void const * data, unsigned size) {
+ VIXL_ASSERT(size % 4 == 0);
+ MOZ_ASSERT(hasCreator());
+ return armbuffer_.allocEntry(size / sizeof(uint32_t), 0, (uint8_t*)(data), nullptr);
+ }
+
+ public:
+ // Size of the code generated in bytes, including pools.
+ size_t SizeOfCodeGenerated() const {
+ return armbuffer_.size();
+ }
+
+ // Move the pool into the instruction stream.
+ void flushBuffer() {
+ armbuffer_.flushPool();
+ }
+
+ // Inhibit pool flushing for the given number of instructions.
+ // Generating more than |maxInst| instructions in a no-pool region
+ // triggers an assertion within the ARMBuffer.
+ // Does not nest.
+ void enterNoPool(size_t maxInst) {
+ armbuffer_.enterNoPool(maxInst);
+ }
+
+ // Marks the end of a no-pool region.
+ void leaveNoPool() {
+ armbuffer_.leaveNoPool();
+ }
+
+ void enterNoNops() {
+ armbuffer_.enterNoNops();
+ }
+ void leaveNoNops() {
+ armbuffer_.leaveNoNops();
+ }
+
+ public:
+ // Static interface used by IonAssemblerBufferWithConstantPools.
+ static void InsertIndexIntoTag(uint8_t* load, uint32_t index);
+ static bool PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr);
+ static void PatchShortRangeBranchToVeneer(ARMBuffer*, unsigned rangeIdx, BufferOffset deadline,
+ BufferOffset veneer);
+ static uint32_t PlaceConstantPoolBarrier(int offset);
+
+ static void WritePoolHeader(uint8_t* start, js::jit::Pool* p, bool isNatural);
+ static void WritePoolFooter(uint8_t* start, js::jit::Pool* p, bool isNatural);
+ static void WritePoolGuard(BufferOffset branch, Instruction* inst, BufferOffset dest);
+
+ protected:
+ // Functions for managing Labels and linked lists of Label uses.
+
+ // Get the next Label user in the linked list of Label uses.
+ // Return an unassigned BufferOffset when the end of the list is reached.
+ BufferOffset NextLink(BufferOffset cur);
+
+ // Patch the instruction at cur to link to the instruction at next.
+ void SetNextLink(BufferOffset cur, BufferOffset next);
+
+ // Link the current (not-yet-emitted) instruction to the specified label,
+ // then return a raw offset to be encoded in the instruction.
+ ptrdiff_t LinkAndGetByteOffsetTo(BufferOffset branch, js::jit::Label* label);
+ ptrdiff_t LinkAndGetInstructionOffsetTo(BufferOffset branch, ImmBranchRangeType branchRange,
+ js::jit::Label* label);
+ ptrdiff_t LinkAndGetPageOffsetTo(BufferOffset branch, js::jit::Label* label);
+
+ // A common implementation for the LinkAndGet<Type>OffsetTo helpers.
+ ptrdiff_t LinkAndGetOffsetTo(BufferOffset branch, ImmBranchRangeType branchRange,
+ unsigned elementSizeBits, js::jit::Label* label);
+
+ protected:
+ // The buffer into which code and relocation info are generated.
+ ARMBuffer armbuffer_;
+};
+
+
+} // namespace vixl
+
+
+#endif // jit_arm64_vixl_MozBaseAssembler_vixl_h
+
diff --git a/js/src/jit/arm64/vixl/MozCachingDecoder.h b/js/src/jit/arm64/vixl/MozCachingDecoder.h
new file mode 100644
index 0000000000..5b4cfc17d5
--- /dev/null
+++ b/js/src/jit/arm64/vixl/MozCachingDecoder.h
@@ -0,0 +1,179 @@
+#ifndef VIXL_A64_MOZ_CACHING_DECODER_A64_H_
+#define VIXL_A64_MOZ_CACHING_DECODER_A64_H_
+
+#include "mozilla/HashTable.h"
+
+#include "jit/arm64/vixl/Decoder-vixl.h"
+#include "js/AllocPolicy.h"
+
+#ifdef DEBUG
+#define JS_CACHE_SIMULATOR_ARM64 1
+#endif
+
+#ifdef JS_CACHE_SIMULATOR_ARM64
+namespace vixl {
+
+// This enumeration list the different kind of instructions which can be
+// decoded. These kind correspond to the set of visitor defined by the default
+// Decoder.
+enum class InstDecodedKind : uint8_t {
+ NotDecodedYet,
+#define DECLARE(E) E,
+ VISITOR_LIST(DECLARE)
+#undef DECLARE
+};
+
+// A SinglePageDecodeCache is used to store the decoded kind of all instructions
+// in an executable page of code. Each time an instruction is decoded, its
+// decoded kind is recorded in this structure. The previous instruction value is
+// also recorded in this structure when using a debug build.
+//
+// The next time the same offset is visited, the instruction would be decoded
+// using the previously recorded decode kind. It is also compared against the
+// previously recorded bits of the instruction to check for potential missing
+// cache invalidations, in debug builds.
+//
+// This structure stores the equivalent of a single page of code to have better
+// memory locality when using the simulator. As opposed to having a hash-table
+// for all instructions. However a hash-table is used by the CachingDecoder to
+// map the prefixes of page addresses to these SinglePageDecodeCaches.
+class SinglePageDecodeCache {
+ public:
+ static const uintptr_t PageSize = 1 << 12;
+ static const uintptr_t PageMask = PageSize - 1;
+ static const uintptr_t InstSize = vixl::kInstructionSize;
+ static const uintptr_t InstMask = InstSize - 1;
+ static const uintptr_t InstPerPage = PageSize / InstSize;
+
+ SinglePageDecodeCache(const Instruction* inst)
+ : pageStart_(PageStart(inst))
+ {
+ memset(&decodeCache_, int(InstDecodedKind::NotDecodedYet), sizeof(decodeCache_));
+ }
+ // Compute the start address of the page which contains this instruction.
+ static uintptr_t PageStart(const Instruction* inst) {
+ return uintptr_t(inst) & ~PageMask;
+ }
+ // Returns whether the instruction decoded kind is stored in this
+ // SinglePageDecodeCache.
+ bool contains(const Instruction* inst) {
+ return pageStart_ == PageStart(inst);
+ }
+ void clearDecode(const Instruction* inst) {
+ uintptr_t offset = (uintptr_t(inst) & PageMask) / InstSize;
+ decodeCache_[offset] = InstDecodedKind::NotDecodedYet;
+ }
+ InstDecodedKind* decodePtr(const Instruction* inst) {
+ uintptr_t offset = (uintptr_t(inst) & PageMask) / InstSize;
+ uint32_t instValue = *reinterpret_cast<const uint32_t*>(inst);
+ instCache_[offset] = instValue;
+ return &decodeCache_[offset];
+ }
+ InstDecodedKind decode(const Instruction* inst) const {
+ uintptr_t offset = (uintptr_t(inst) & PageMask) / InstSize;
+ InstDecodedKind val = decodeCache_[offset];
+ uint32_t instValue = *reinterpret_cast<const uint32_t*>(inst);
+ MOZ_ASSERT_IF(val != InstDecodedKind::NotDecodedYet,
+ instCache_[offset] == instValue);
+ return val;
+ }
+
+ private:
+ // Record the address at which the corresponding code page starts.
+ const uintptr_t pageStart_;
+
+ // Cache what instruction got decoded previously, in order to assert if we see
+ // any stale instructions after.
+ uint32_t instCache_[InstPerPage];
+
+ // Cache the decoding of the instruction such that we can skip the decoding
+ // part.
+ InstDecodedKind decodeCache_[InstPerPage];
+};
+
+// A DecoderVisitor which will record which visitor function should be called
+// the next time we want to decode the same instruction.
+class CachingDecoderVisitor : public DecoderVisitor {
+ public:
+ CachingDecoderVisitor() = default;
+ virtual ~CachingDecoderVisitor() {}
+
+#define DECLARE(A) virtual void Visit##A(const Instruction* instr) { \
+ if (last_) { \
+ MOZ_ASSERT(*last_ == InstDecodedKind::NotDecodedYet); \
+ *last_ = InstDecodedKind::A; \
+ last_ = nullptr; \
+ } \
+ };
+
+ VISITOR_LIST(DECLARE)
+#undef DECLARE
+
+ void setDecodePtr(InstDecodedKind* ptr) {
+ last_ = ptr;
+ }
+
+ private:
+ InstDecodedKind* last_;
+};
+
+// The Caching decoder works by extending the default vixl Decoder class. It
+// extends it by overloading the Decode function.
+//
+// The overloaded Decode function checks whether the instruction given as
+// argument got decoded before or since it got invalidated. If it was not
+// previously decoded, the value of the instruction is recorded as well as the
+// kind of instruction. Otherwise, the value of the instruction is checked
+// against the previously recorded value and the instruction kind is used to
+// skip the decoding visitor and resume the execution of instruction.
+//
+// The caching decoder stores the equivalent of a page of executable code in a
+// hash-table. Each SinglePageDecodeCache stores an array of decoded kind as
+// well as the value of the previously decoded instruction.
+//
+// When testing if an instruction was decoded before, we check if the address of
+// the instruction is contained in the last SinglePageDecodeCache. If it is not,
+// then the hash-table entry is queried and created if necessary, and the last
+// SinglePageDecodeCache is updated. Then, the last SinglePageDecodeCache
+// necessary contains the decoded kind of the instruction given as argument.
+//
+// The caching decoder add an extra function for flushing the cache, which is in
+// charge of clearing the decoded kind of instruction in the range of addresses
+// given as argument. This is indirectly called by
+// CPU::EnsureIAndDCacheCoherency.
+class CachingDecoder : public Decoder {
+ using ICacheMap = mozilla::HashMap<uintptr_t, SinglePageDecodeCache*>;
+ public:
+ CachingDecoder()
+ : lastPage_(nullptr)
+ {
+ PrependVisitor(&cachingDecoder_);
+ }
+ ~CachingDecoder() {
+ RemoveVisitor(&cachingDecoder_);
+ }
+
+ void Decode(const Instruction* instr);
+ void Decode(Instruction* instr) {
+ Decode(const_cast<const Instruction*>(instr));
+ }
+
+ void FlushICache(void* start, size_t size);
+
+ private:
+ // Record the type of the decoded instruction, to avoid decoding it a second
+ // time the next time we execute it.
+ CachingDecoderVisitor cachingDecoder_;
+
+ // Store the mapping of Instruction pointer to the corresponding
+ // SinglePageDecodeCache.
+ ICacheMap iCache_;
+
+ // Record the last SinglePageDecodeCache seen, such that we can quickly access
+ // it for the next instruction.
+ SinglePageDecodeCache* lastPage_;
+};
+
+}
+#endif // !JS_CACHE_SIMULATOR_ARM64
+#endif // !VIXL_A64_MOZ_CACHING_DECODER_A64_H_
diff --git a/js/src/jit/arm64/vixl/MozCpu-vixl.cpp b/js/src/jit/arm64/vixl/MozCpu-vixl.cpp
new file mode 100644
index 0000000000..909cc590ae
--- /dev/null
+++ b/js/src/jit/arm64/vixl/MozCpu-vixl.cpp
@@ -0,0 +1,226 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Cpu-vixl.h"
+#include "jit/arm64/vixl/Simulator-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+#include "util/WindowsWrapper.h"
+
+#if defined(XP_DARWIN)
+# include <libkern/OSCacheControl.h>
+#endif
+
+namespace vixl {
+
+// Currently computes I and D cache line size.
+void CPU::SetUp() {
+ uint32_t cache_type_register = GetCacheType();
+
+ // The cache type register holds information about the caches, including I
+ // D caches line size.
+ static const int kDCacheLineSizeShift = 16;
+ static const int kICacheLineSizeShift = 0;
+ static const uint32_t kDCacheLineSizeMask = 0xf << kDCacheLineSizeShift;
+ static const uint32_t kICacheLineSizeMask = 0xf << kICacheLineSizeShift;
+
+ // The cache type register holds the size of the I and D caches in words as
+ // a power of two.
+ uint32_t dcache_line_size_power_of_two =
+ (cache_type_register & kDCacheLineSizeMask) >> kDCacheLineSizeShift;
+ uint32_t icache_line_size_power_of_two =
+ (cache_type_register & kICacheLineSizeMask) >> kICacheLineSizeShift;
+
+ dcache_line_size_ = 4 << dcache_line_size_power_of_two;
+ icache_line_size_ = 4 << icache_line_size_power_of_two;
+
+ // Bug 1521158 suggests that having CPU with different cache line sizes could
+ // cause issues as we would only invalidate half of the cache line of we
+ // invalidate every 128 bytes, but other little cores have a different stride
+ // such as 64 bytes. To be conservative, we will try reducing the stride to 32
+ // bytes, which should be smaller than any known cache line.
+ const uint32_t conservative_line_size = 32;
+ dcache_line_size_ = std::min(dcache_line_size_, conservative_line_size);
+ icache_line_size_ = std::min(icache_line_size_, conservative_line_size);
+}
+
+
+uint32_t CPU::GetCacheType() {
+#if defined(__aarch64__) && (defined(__linux__) || defined(__android__))
+ uint64_t cache_type_register;
+ // Copy the content of the cache type register to a core register.
+ __asm__ __volatile__ ("mrs %[ctr], ctr_el0" // NOLINT
+ : [ctr] "=r" (cache_type_register));
+ VIXL_ASSERT(IsUint32(cache_type_register));
+ return static_cast<uint32_t>(cache_type_register);
+#else
+ // This will lead to a cache with 1 byte long lines, which is fine since
+ // neither EnsureIAndDCacheCoherency nor the simulator will need this
+ // information.
+ return 0;
+#endif
+}
+
+void CPU::EnsureIAndDCacheCoherency(void* address, size_t length) {
+#if defined(JS_SIMULATOR_ARM64) && defined(JS_CACHE_SIMULATOR_ARM64)
+ // This code attempts to emulate what the following assembly sequence is
+ // doing, which is sending the information to all cores that some cache line
+ // have to be invalidated and invalidating them only on the current core.
+ //
+ // This is done by recording the current range to be flushed to all
+ // simulators, then if there is a simulator associated with the current
+ // thread, applying all flushed ranges as the "isb" instruction would do.
+ //
+ // As we have no control over the CPU cores used by the code generator and the
+ // execution threads, this code assumes that each thread runs on its own core.
+ //
+ // See Bug 1529933 for more detailed explanation of this issue.
+ using js::jit::SimulatorProcess;
+ js::jit::AutoLockSimulatorCache alsc;
+ if (length > 0) {
+ SimulatorProcess::recordICacheFlush(address, length);
+ }
+ Simulator* sim = vixl::Simulator::Current();
+ if (sim) {
+ sim->FlushICache();
+ }
+#elif defined(_MSC_VER) && defined(_M_ARM64)
+ FlushInstructionCache(GetCurrentProcess(), address, length);
+#elif defined(XP_DARWIN)
+ sys_icache_invalidate(address, length);
+#elif defined(__aarch64__) && (defined(__linux__) || defined(__android__))
+ // Implement the cache synchronisation for all targets where AArch64 is the
+ // host, even if we're building the simulator for an AAarch64 host. This
+ // allows for cases where the user wants to simulate code as well as run it
+ // natively.
+
+ if (length == 0) {
+ return;
+ }
+
+ // The code below assumes user space cache operations are allowed.
+
+ // Work out the line sizes for each cache, and use them to determine the
+ // start addresses.
+ uintptr_t start = reinterpret_cast<uintptr_t>(address);
+ uintptr_t dsize = static_cast<uintptr_t>(dcache_line_size_);
+ uintptr_t isize = static_cast<uintptr_t>(icache_line_size_);
+ uintptr_t dline = start & ~(dsize - 1);
+ uintptr_t iline = start & ~(isize - 1);
+
+ // Cache line sizes are always a power of 2.
+ VIXL_ASSERT(IsPowerOf2(dsize));
+ VIXL_ASSERT(IsPowerOf2(isize));
+ uintptr_t end = start + length;
+
+ do {
+ __asm__ __volatile__ (
+ // Clean each line of the D cache containing the target data.
+ //
+ // dc : Data Cache maintenance
+ // c : Clean
+ // i : Invalidate
+ // va : by (Virtual) Address
+ // c : to the point of Coherency
+ // Original implementation used cvau, but changed to civac due to
+ // errata on Cortex-A53 819472, 826319, 827319 and 824069.
+ // See ARM DDI 0406B page B2-12 for more information.
+ //
+ " dc civac, %[dline]\n"
+ :
+ : [dline] "r" (dline)
+ // This code does not write to memory, but the "memory" dependency
+ // prevents GCC from reordering the code.
+ : "memory");
+ dline += dsize;
+ } while (dline < end);
+
+ __asm__ __volatile__ (
+ // Make sure that the data cache operations (above) complete before the
+ // instruction cache operations (below).
+ //
+ // dsb : Data Synchronisation Barrier
+ // ish : Inner SHareable domain
+ //
+ // The point of unification for an Inner Shareable shareability domain is
+ // the point by which the instruction and data caches of all the processors
+ // in that Inner Shareable shareability domain are guaranteed to see the
+ // same copy of a memory location. See ARM DDI 0406B page B2-12 for more
+ // information.
+ " dsb ish\n"
+ : : : "memory");
+
+ do {
+ __asm__ __volatile__ (
+ // Invalidate each line of the I cache containing the target data.
+ //
+ // ic : Instruction Cache maintenance
+ // i : Invalidate
+ // va : by Address
+ // u : to the point of Unification
+ " ic ivau, %[iline]\n"
+ :
+ : [iline] "r" (iline)
+ : "memory");
+ iline += isize;
+ } while (iline < end);
+
+ __asm__ __volatile__(
+ // Make sure that the instruction cache operations (above) take effect
+ // before the isb (below).
+ " dsb ish\n"
+
+ // Ensure that any instructions already in the pipeline are discarded and
+ // reloaded from the new data.
+ // isb : Instruction Synchronisation Barrier
+ " isb\n"
+ :
+ :
+ : "memory");
+#else
+ // If the host isn't AArch64, we must be using the simulator, so this function
+ // doesn't have to do anything.
+ USE(address, length);
+#endif
+}
+
+void CPU::FlushExecutionContext() {
+#if defined(JS_SIMULATOR_ARM64) && defined(JS_CACHE_SIMULATOR_ARM64)
+ // Performing an 'isb' will ensure the current core instruction pipeline is
+ // synchronized with an icache flush executed by another core.
+ using js::jit::SimulatorProcess;
+ js::jit::AutoLockSimulatorCache alsc;
+ Simulator* sim = vixl::Simulator::Current();
+ if (sim) {
+ sim->FlushICache();
+ }
+#elif defined(__aarch64__)
+ // Ensure that any instructions already in the pipeline are discarded and
+ // reloaded from the icache.
+ __asm__ __volatile__("isb\n" : : : "memory");
+#endif
+}
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/MozInstructions-vixl.cpp b/js/src/jit/arm64/vixl/MozInstructions-vixl.cpp
new file mode 100644
index 0000000000..398f864493
--- /dev/null
+++ b/js/src/jit/arm64/vixl/MozInstructions-vixl.cpp
@@ -0,0 +1,211 @@
+// Copyright 2013, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/Architecture-arm64.h"
+#include "jit/arm64/vixl/Assembler-vixl.h"
+#include "jit/arm64/vixl/Instructions-vixl.h"
+
+namespace vixl {
+
+bool Instruction::IsUncondB() const {
+ return Mask(UnconditionalBranchMask) == (UnconditionalBranchFixed | B);
+}
+
+
+bool Instruction::IsCondB() const {
+ return Mask(ConditionalBranchMask) == (ConditionalBranchFixed | B_cond);
+}
+
+
+bool Instruction::IsBL() const {
+ return Mask(UnconditionalBranchMask) == (UnconditionalBranchFixed | BL);
+}
+
+
+bool Instruction::IsBR() const {
+ return Mask(UnconditionalBranchToRegisterMask) == (UnconditionalBranchToRegisterFixed | BR);
+}
+
+
+bool Instruction::IsBLR() const {
+ return Mask(UnconditionalBranchToRegisterMask) == (UnconditionalBranchToRegisterFixed | BLR);
+}
+
+
+bool Instruction::IsTBZ() const {
+ return Mask(TestBranchMask) == TBZ;
+}
+
+
+bool Instruction::IsTBNZ() const {
+ return Mask(TestBranchMask) == TBNZ;
+}
+
+
+bool Instruction::IsCBZ() const {
+ return Mask(CompareBranchMask) == CBZ_w || Mask(CompareBranchMask) == CBZ_x;
+}
+
+
+bool Instruction::IsCBNZ() const {
+ return Mask(CompareBranchMask) == CBNZ_w || Mask(CompareBranchMask) == CBNZ_x;
+}
+
+
+bool Instruction::IsLDR() const {
+ return Mask(LoadLiteralMask) == LDR_x_lit;
+}
+
+
+bool Instruction::IsNOP() const {
+ return Mask(SystemHintMask) == HINT && ImmHint() == NOP;
+}
+
+
+bool Instruction::IsCSDB() const {
+ return Mask(SystemHintMask) == HINT && ImmHint() == CSDB;
+}
+
+
+bool Instruction::IsADR() const {
+ return Mask(PCRelAddressingMask) == ADR;
+}
+
+
+bool Instruction::IsADRP() const {
+ return Mask(PCRelAddressingMask) == ADRP;
+}
+
+
+bool Instruction::IsMovz() const {
+ return (Mask(MoveWideImmediateMask) == MOVZ_x) ||
+ (Mask(MoveWideImmediateMask) == MOVZ_w);
+}
+
+
+bool Instruction::IsMovk() const {
+ return (Mask(MoveWideImmediateMask) == MOVK_x) ||
+ (Mask(MoveWideImmediateMask) == MOVK_w);
+}
+
+bool Instruction::IsBranchLinkImm() const {
+ return Mask(UnconditionalBranchFMask) == (UnconditionalBranchFixed | BL);
+}
+
+
+bool Instruction::IsTargetReachable(const Instruction* target) const {
+ VIXL_ASSERT(((target - this) & 3) == 0);
+ int offset = (target - this) >> kInstructionSizeLog2;
+ switch (BranchType()) {
+ case CondBranchType:
+ return IsInt19(offset);
+ case UncondBranchType:
+ return IsInt26(offset);
+ case CompareBranchType:
+ return IsInt19(offset);
+ case TestBranchType:
+ return IsInt14(offset);
+ default:
+ VIXL_UNREACHABLE();
+ }
+}
+
+
+ptrdiff_t Instruction::ImmPCRawOffset() const {
+ ptrdiff_t offset;
+ if (IsPCRelAddressing()) {
+ // ADR and ADRP.
+ offset = ImmPCRel();
+ } else if (BranchType() == UnknownBranchType) {
+ offset = ImmLLiteral();
+ } else {
+ offset = ImmBranch();
+ }
+ return offset;
+}
+
+void
+Instruction::SetImmPCRawOffset(ptrdiff_t offset)
+{
+ if (IsPCRelAddressing()) {
+ // ADR and ADRP. We're encoding a raw offset here.
+ // See also SetPCRelImmTarget().
+ Instr imm = vixl::Assembler::ImmPCRelAddress(offset);
+ SetInstructionBits(Mask(~ImmPCRel_mask) | imm);
+ } else {
+ SetBranchImmTarget(this + (offset << kInstructionSizeLog2));
+ }
+}
+
+// Is this a stack pointer synchronization instruction as inserted by
+// MacroAssembler::syncStackPtr()?
+bool
+Instruction::IsStackPtrSync() const
+{
+ // The stack pointer sync is a move to the stack pointer.
+ // This is encoded as 'add sp, Rs, #0'.
+ return IsAddSubImmediate() && Rd() == js::jit::Registers::sp && ImmAddSub() == 0;
+}
+
+// Skip over a constant pool at |this| if there is one.
+//
+// If |this| is pointing to the artifical guard branch around a constant pool,
+// return the instruction after the pool. Otherwise return |this| itself.
+//
+// This function does not skip constant pools with a natural guard branch. It
+// is assumed that anyone inspecting the instruction stream understands about
+// branches that were inserted naturally.
+const Instruction*
+Instruction::skipPool() const
+{
+ // Artificial pool guards can only be B (rather than BR), and they must be
+ // forward branches.
+ if (!IsUncondB() || ImmUncondBranch() <= 0)
+ return this;
+
+ // Check for a constant pool header which has the high 16 bits set. See
+ // struct PoolHeader. Bit 15 indicates a natural pool guard when set. It
+ // must be clear which indicates an artificial pool guard.
+ const Instruction *header = InstructionAtOffset(kInstructionSize);
+ if (header->Mask(0xffff8000) != 0xffff0000)
+ return this;
+
+ // OK, this is an artificial jump around a constant pool.
+ return ImmPCOffsetTarget();
+}
+
+
+void Instruction::SetBits32(int msb, int lsb, unsigned value) {
+ uint32_t me;
+ memcpy(&me, this, sizeof(me));
+ uint32_t new_mask = (1 << (msb+1)) - (1 << lsb);
+ uint32_t keep_mask = ~new_mask;
+ me = (me & keep_mask) | ((value << lsb) & new_mask);
+ memcpy(this, &me, sizeof(me));
+}
+
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/MozSimulator-vixl.cpp b/js/src/jit/arm64/vixl/MozSimulator-vixl.cpp
new file mode 100644
index 0000000000..b6ac3952c6
--- /dev/null
+++ b/js/src/jit/arm64/vixl/MozSimulator-vixl.cpp
@@ -0,0 +1,1239 @@
+// Copyright 2013, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "mozilla/DebugOnly.h"
+
+#include "jit/arm64/vixl/Debugger-vixl.h"
+#include "jit/arm64/vixl/MozCachingDecoder.h"
+#include "jit/arm64/vixl/Simulator-vixl.h"
+#include "jit/IonTypes.h"
+#include "js/UniquePtr.h"
+#include "js/Utility.h"
+#include "threading/LockGuard.h"
+#include "vm/JSContext.h"
+#include "vm/Runtime.h"
+
+js::jit::SimulatorProcess* js::jit::SimulatorProcess::singleton_ = nullptr;
+
+namespace vixl {
+
+using mozilla::DebugOnly;
+using js::jit::ABIFunctionType;
+using js::jit::JitActivation;
+using js::jit::SimulatorProcess;
+
+Simulator::Simulator(Decoder* decoder, FILE* stream)
+ : stream_(nullptr)
+ , print_disasm_(nullptr)
+ , instrumentation_(nullptr)
+ , stack_(nullptr)
+ , stack_limit_(nullptr)
+ , decoder_(nullptr)
+ , oom_(false)
+{
+ this->init(decoder, stream);
+
+ // If this environment variable is present, trace the executed instructions.
+ // (Very helpful for debugging code generation crashes.)
+ if (getenv("VIXL_TRACE")) {
+ set_trace_parameters(LOG_DISASM);
+ }
+}
+
+
+Simulator::~Simulator() {
+ js_free(stack_);
+ stack_ = nullptr;
+
+ // The decoder may outlive the simulator.
+ if (print_disasm_) {
+ decoder_->RemoveVisitor(print_disasm_);
+ js_delete(print_disasm_);
+ print_disasm_ = nullptr;
+ }
+
+ if (instrumentation_) {
+ decoder_->RemoveVisitor(instrumentation_);
+ js_delete(instrumentation_);
+ instrumentation_ = nullptr;
+ }
+}
+
+
+void Simulator::ResetState() {
+ // Reset the system registers.
+ nzcv_ = SimSystemRegister::DefaultValueFor(NZCV);
+ fpcr_ = SimSystemRegister::DefaultValueFor(FPCR);
+
+ // Reset registers to 0.
+ pc_ = nullptr;
+ pc_modified_ = false;
+ for (unsigned i = 0; i < kNumberOfRegisters; i++) {
+ set_xreg(i, 0xbadbeef);
+ }
+ // Set FP registers to a value that is a NaN in both 32-bit and 64-bit FP.
+ uint64_t nan_bits = UINT64_C(0x7ff0dead7f8beef1);
+ VIXL_ASSERT(IsSignallingNaN(RawbitsToDouble(nan_bits & kDRegMask)));
+ VIXL_ASSERT(IsSignallingNaN(RawbitsToFloat(nan_bits & kSRegMask)));
+ for (unsigned i = 0; i < kNumberOfFPRegisters; i++) {
+ set_dreg_bits(i, nan_bits);
+ }
+ // Returning to address 0 exits the Simulator.
+ set_lr(kEndOfSimAddress);
+}
+
+
+void Simulator::init(Decoder* decoder, FILE* stream) {
+ // Ensure that shift operations act as the simulator expects.
+ VIXL_ASSERT((static_cast<int32_t>(-1) >> 1) == -1);
+ VIXL_ASSERT((static_cast<uint32_t>(-1) >> 1) == 0x7FFFFFFF);
+
+ instruction_stats_ = false;
+
+ // Set up the decoder.
+ decoder_ = decoder;
+ decoder_->AppendVisitor(this);
+
+ stream_ = stream;
+ print_disasm_ = js_new<PrintDisassembler>(stream_);
+ if (!print_disasm_) {
+ oom_ = true;
+ return;
+ }
+ set_coloured_trace(false);
+ trace_parameters_ = LOG_NONE;
+
+ ResetState();
+
+ // Allocate and set up the simulator stack.
+ stack_ = js_pod_malloc<byte>(stack_size_);
+ if (!stack_) {
+ oom_ = true;
+ return;
+ }
+ stack_limit_ = stack_ + stack_protection_size_;
+ // Configure the starting stack pointer.
+ // - Find the top of the stack.
+ byte * tos = stack_ + stack_size_;
+ // - There's a protection region at both ends of the stack.
+ tos -= stack_protection_size_;
+ // - The stack pointer must be 16-byte aligned.
+ tos = AlignDown(tos, 16);
+ set_sp(tos);
+
+ // Set the sample period to 10, as the VIXL examples and tests are short.
+ if (getenv("VIXL_STATS")) {
+ instrumentation_ = js_new<Instrument>("vixl_stats.csv", 10);
+ if (!instrumentation_) {
+ oom_ = true;
+ return;
+ }
+ }
+
+ // Print a warning about exclusive-access instructions, but only the first
+ // time they are encountered. This warning can be silenced using
+ // SilenceExclusiveAccessWarning().
+ print_exclusive_access_warning_ = true;
+}
+
+
+Simulator* Simulator::Current() {
+ JSContext* cx = js::TlsContext.get();
+ if (!cx) {
+ return nullptr;
+ }
+ JSRuntime* rt = cx->runtime();
+ if (!rt) {
+ return nullptr;
+ }
+ if (!js::CurrentThreadCanAccessRuntime(rt)) {
+ return nullptr;
+ }
+ return cx->simulator();
+}
+
+
+Simulator* Simulator::Create() {
+ Decoder *decoder = js_new<Decoder>();
+ if (!decoder)
+ return nullptr;
+
+ // FIXME: This just leaks the Decoder object for now, which is probably OK.
+ // FIXME: We should free it at some point.
+ // FIXME: Note that it can't be stored in the SimulatorRuntime due to lifetime conflicts.
+ js::UniquePtr<Simulator> sim;
+ if (getenv("USE_DEBUGGER") != nullptr) {
+ sim.reset(js_new<Debugger>(decoder, stdout));
+ } else {
+ sim.reset(js_new<Simulator>(decoder, stdout));
+ }
+
+ // Check if Simulator:init ran out of memory.
+ if (sim && sim->oom()) {
+ return nullptr;
+ }
+
+#ifdef JS_CACHE_SIMULATOR_ARM64
+ // Register the simulator in the Simulator process to handle cache flushes
+ // across threads.
+ js::jit::AutoLockSimulatorCache alsc;
+ if (!SimulatorProcess::registerSimulator(sim.get())) {
+ return nullptr;
+ }
+#endif
+
+ return sim.release();
+}
+
+
+void Simulator::Destroy(Simulator* sim) {
+#ifdef JS_CACHE_SIMULATOR_ARM64
+ if (sim) {
+ js::jit::AutoLockSimulatorCache alsc;
+ SimulatorProcess::unregisterSimulator(sim);
+ }
+#endif
+
+ js_delete(sim);
+}
+
+
+void Simulator::ExecuteInstruction() {
+ // The program counter should always be aligned.
+ VIXL_ASSERT(IsWordAligned(pc_));
+#ifdef JS_CACHE_SIMULATOR_ARM64
+ if (pendingCacheRequests) {
+ // We're here emulating the behavior of the membarrier carried over on
+ // real hardware does; see syscalls to membarrier in MozCpu-vixl.cpp.
+ // There's a slight difference that the simulator is not being
+ // interrupted: instead, we effectively run the icache flush request
+ // before executing the next instruction, which is close enough and
+ // sufficient for our use case.
+ js::jit::AutoLockSimulatorCache alsc;
+ FlushICache();
+ }
+#endif
+ decoder_->Decode(pc_);
+ increment_pc();
+}
+
+
+uintptr_t Simulator::stackLimit() const {
+ return reinterpret_cast<uintptr_t>(stack_limit_);
+}
+
+
+uintptr_t* Simulator::addressOfStackLimit() {
+ return (uintptr_t*)&stack_limit_;
+}
+
+
+bool Simulator::overRecursed(uintptr_t newsp) const {
+ if (newsp == 0) {
+ newsp = get_sp();
+ }
+ return newsp <= stackLimit();
+}
+
+
+bool Simulator::overRecursedWithExtra(uint32_t extra) const {
+ uintptr_t newsp = get_sp() - extra;
+ return newsp <= stackLimit();
+}
+
+
+JS::ProfilingFrameIterator::RegisterState
+Simulator::registerState()
+{
+ JS::ProfilingFrameIterator::RegisterState state;
+ state.pc = (uint8_t*) get_pc();
+ state.fp = (uint8_t*) get_fp();
+ state.lr = (uint8_t*) get_lr();
+ state.sp = (uint8_t*) get_sp();
+ return state;
+}
+
+int64_t Simulator::call(uint8_t* entry, int argument_count, ...) {
+ va_list parameters;
+ va_start(parameters, argument_count);
+
+ // First eight arguments passed in registers.
+ VIXL_ASSERT(argument_count <= 8);
+ // This code should use the type of the called function
+ // (with templates, like the callVM machinery), but since the
+ // number of called functions is miniscule, their types have been
+ // divined from the number of arguments.
+ if (argument_count == 8) {
+ // EnterJitData::jitcode.
+ set_xreg(0, va_arg(parameters, int64_t));
+ // EnterJitData::maxArgc.
+ set_xreg(1, va_arg(parameters, unsigned));
+ // EnterJitData::maxArgv.
+ set_xreg(2, va_arg(parameters, int64_t));
+ // EnterJitData::osrFrame.
+ set_xreg(3, va_arg(parameters, int64_t));
+ // EnterJitData::calleeToken.
+ set_xreg(4, va_arg(parameters, int64_t));
+ // EnterJitData::scopeChain.
+ set_xreg(5, va_arg(parameters, int64_t));
+ // EnterJitData::osrNumStackValues.
+ set_xreg(6, va_arg(parameters, unsigned));
+ // Address of EnterJitData::result.
+ set_xreg(7, va_arg(parameters, int64_t));
+ } else if (argument_count == 2) {
+ // EntryArg* args
+ set_xreg(0, va_arg(parameters, int64_t));
+ // uint8_t* GlobalData
+ set_xreg(1, va_arg(parameters, int64_t));
+ } else if (argument_count == 1) { // irregexp
+ // InputOutputData& data
+ set_xreg(0, va_arg(parameters, int64_t));
+ } else if (argument_count == 0) { // testsJit.cpp
+ // accept.
+ } else {
+ MOZ_CRASH("Unknown number of arguments");
+ }
+
+ va_end(parameters);
+
+ // Call must transition back to native code on exit.
+ VIXL_ASSERT(get_lr() == int64_t(kEndOfSimAddress));
+
+ // Execute the simulation.
+ DebugOnly<int64_t> entryStack = get_sp();
+ RunFrom((Instruction*)entry);
+ DebugOnly<int64_t> exitStack = get_sp();
+ VIXL_ASSERT(entryStack == exitStack);
+
+ int64_t result = xreg(0);
+ if (getenv("USE_DEBUGGER")) {
+ printf("LEAVE\n");
+ }
+ return result;
+}
+
+
+// When the generated code calls a VM function (masm.callWithABI) we need to
+// call that function instead of trying to execute it with the simulator
+// (because it's x64 code instead of AArch64 code). We do that by redirecting the VM
+// call to a svc (Supervisor Call) instruction that is handled by the
+// simulator. We write the original destination of the jump just at a known
+// offset from the svc instruction so the simulator knows what to call.
+class Redirection
+{
+ friend class Simulator;
+
+ Redirection(void* nativeFunction, ABIFunctionType type)
+ : nativeFunction_(nativeFunction),
+ type_(type),
+ next_(nullptr)
+ {
+ next_ = SimulatorProcess::redirection();
+ SimulatorProcess::setRedirection(this);
+
+ Instruction* instr = (Instruction*)(&svcInstruction_);
+ vixl::Assembler::svc(instr, kCallRtRedirected);
+ }
+
+ public:
+ void* addressOfSvcInstruction() { return &svcInstruction_; }
+ void* nativeFunction() const { return nativeFunction_; }
+ ABIFunctionType type() const { return type_; }
+
+ static Redirection* Get(void* nativeFunction, ABIFunctionType type) {
+ js::jit::AutoLockSimulatorCache alsr;
+
+ // TODO: Store srt_ in the simulator for this assertion.
+ // VIXL_ASSERT_IF(pt->simulator(), pt->simulator()->srt_ == srt);
+
+ Redirection* current = SimulatorProcess::redirection();
+ for (; current != nullptr; current = current->next_) {
+ if (current->nativeFunction_ == nativeFunction) {
+ VIXL_ASSERT(current->type() == type);
+ return current;
+ }
+ }
+
+ // Note: we can't use js_new here because the constructor is private.
+ js::AutoEnterOOMUnsafeRegion oomUnsafe;
+ Redirection* redir = js_pod_malloc<Redirection>(1);
+ if (!redir)
+ oomUnsafe.crash("Simulator redirection");
+ new(redir) Redirection(nativeFunction, type);
+ return redir;
+ }
+
+ static const Redirection* FromSvcInstruction(const Instruction* svcInstruction) {
+ const uint8_t* addrOfSvc = reinterpret_cast<const uint8_t*>(svcInstruction);
+ const uint8_t* addrOfRedirection = addrOfSvc - offsetof(Redirection, svcInstruction_);
+ return reinterpret_cast<const Redirection*>(addrOfRedirection);
+ }
+
+ private:
+ void* nativeFunction_;
+ uint32_t svcInstruction_;
+ ABIFunctionType type_;
+ Redirection* next_;
+};
+
+
+
+
+void* Simulator::RedirectNativeFunction(void* nativeFunction, ABIFunctionType type) {
+ Redirection* redirection = Redirection::Get(nativeFunction, type);
+ return redirection->addressOfSvcInstruction();
+}
+
+void Simulator::VisitException(const Instruction* instr) {
+ if (instr->InstructionBits() == UNDEFINED_INST_PATTERN) {
+ uint8_t* newPC;
+ if (js::wasm::HandleIllegalInstruction(registerState(), &newPC)) {
+ set_pc((Instruction*)newPC);
+ return;
+ }
+ DoUnreachable(instr);
+ }
+
+ switch (instr->Mask(ExceptionMask)) {
+ case BRK: {
+ int lowbit = ImmException_offset;
+ int highbit = ImmException_offset + ImmException_width - 1;
+ HostBreakpoint(instr->Bits(highbit, lowbit));
+ break;
+ }
+ case HLT:
+ switch (instr->ImmException()) {
+ case kTraceOpcode:
+ DoTrace(instr);
+ return;
+ case kLogOpcode:
+ DoLog(instr);
+ return;
+ case kPrintfOpcode:
+ DoPrintf(instr);
+ return;
+ default:
+ HostBreakpoint();
+ return;
+ }
+ case SVC:
+ // The SVC instruction is hijacked by the JIT as a pseudo-instruction
+ // causing the Simulator to execute host-native code for callWithABI.
+ switch (instr->ImmException()) {
+ case kCallRtRedirected:
+ VisitCallRedirection(instr);
+ return;
+ case kMarkStackPointer: {
+ js::AutoEnterOOMUnsafeRegion oomUnsafe;
+ if (!spStack_.append(get_sp()))
+ oomUnsafe.crash("tracking stack for ARM64 simulator");
+ return;
+ }
+ case kCheckStackPointer: {
+ DebugOnly<int64_t> current = get_sp();
+ DebugOnly<int64_t> expected = spStack_.popCopy();
+ VIXL_ASSERT(current == expected);
+ return;
+ }
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::setGPR32Result(int32_t result) {
+ set_wreg(0, result);
+}
+
+
+void Simulator::setGPR64Result(int64_t result) {
+ set_xreg(0, result);
+}
+
+
+void Simulator::setFP32Result(float result) {
+ set_sreg(0, result);
+}
+
+
+void Simulator::setFP64Result(double result) {
+ set_dreg(0, result);
+}
+
+
+typedef int64_t (*Prototype_General0)();
+typedef int64_t (*Prototype_General1)(int64_t arg0);
+typedef int64_t (*Prototype_General2)(int64_t arg0, int64_t arg1);
+typedef int64_t (*Prototype_General3)(int64_t arg0, int64_t arg1, int64_t arg2);
+typedef int64_t (*Prototype_General4)(int64_t arg0, int64_t arg1, int64_t arg2, int64_t arg3);
+typedef int64_t (*Prototype_General5)(int64_t arg0, int64_t arg1, int64_t arg2, int64_t arg3,
+ int64_t arg4);
+typedef int64_t (*Prototype_General6)(int64_t arg0, int64_t arg1, int64_t arg2, int64_t arg3,
+ int64_t arg4, int64_t arg5);
+typedef int64_t (*Prototype_General7)(int64_t arg0, int64_t arg1, int64_t arg2, int64_t arg3,
+ int64_t arg4, int64_t arg5, int64_t arg6);
+typedef int64_t (*Prototype_General8)(int64_t arg0, int64_t arg1, int64_t arg2, int64_t arg3,
+ int64_t arg4, int64_t arg5, int64_t arg6, int64_t arg7);
+typedef int64_t (*Prototype_GeneralGeneralGeneralInt64)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3);
+typedef int64_t (*Prototype_GeneralGeneralInt64Int64)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3);
+
+typedef int64_t (*Prototype_Int_Double)(double arg0);
+typedef int64_t (*Prototype_Int_IntDouble)(int64_t arg0, double arg1);
+typedef int64_t (*Prototype_Int_DoubleInt)(double arg0, int64_t arg1);
+typedef int64_t (*Prototype_Int_DoubleIntInt)(double arg0, uint64_t arg1, uint64_t arg2);
+typedef int64_t (*Prototype_Int_IntDoubleIntInt)(uint64_t arg0, double arg1,
+ uint64_t arg2, uint64_t arg3);
+
+typedef float (*Prototype_Float32_Float32)(float arg0);
+typedef int64_t (*Prototype_Int_Float32)(float arg0);
+typedef float (*Prototype_Float32_Float32Float32)(float arg0, float arg1);
+
+typedef double (*Prototype_Double_None)();
+typedef double (*Prototype_Double_Double)(double arg0);
+typedef double (*Prototype_Double_Int)(int64_t arg0);
+typedef double (*Prototype_Double_DoubleInt)(double arg0, int64_t arg1);
+typedef double (*Prototype_Double_IntDouble)(int64_t arg0, double arg1);
+typedef double (*Prototype_Double_DoubleDouble)(double arg0, double arg1);
+typedef double (*Prototype_Double_DoubleDoubleDouble)(double arg0, double arg1, double arg2);
+typedef double (*Prototype_Double_DoubleDoubleDoubleDouble)(double arg0, double arg1,
+ double arg2, double arg3);
+
+typedef int32_t (*Prototype_Int32_General)(int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32)(int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32)(int64_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32)(int64_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32Int32)(int64_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32General)(int64_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32Int32Int32General)(int64_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Float32Float32Int32Int32Int32General)(int64_t,
+ int32_t,
+ float,
+ float,
+ int32_t,
+ int32_t,
+ int32_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General)(int64_t,
+ int32_t,
+ float,
+ float,
+ float,
+ float,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General)(int64_t,
+ int32_t,
+ float,
+ float,
+ int32_t,
+ float,
+ float,
+ int32_t,
+ float,
+ int32_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32General)(int64_t,
+ int32_t,
+ int32_t,
+ int32_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int64)(int64_t,
+ int32_t,
+ int32_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32General)(int64_t,
+ int32_t,
+ int32_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int64Int64)(int64_t,
+ int32_t,
+ int64_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32)(int64_t,
+ int32_t,
+ int64_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32Int32)(int64_t,
+ int32_t,
+ int64_t,
+ int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneral)(int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralGeneral)(int64_t,
+ int64_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralInt32Int32)(int64_t,
+ int64_t,
+ int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int32Int32)(int64_t, int64_t,
+ int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32)(int64_t, int64_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int64)(int64_t, int64_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int64General)(int64_t, int64_t,
+ int32_t, int64_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int64Int64)(int64_t, int64_t,
+ int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int64Int64General)(int64_t, int64_t,
+ int64_t, int64_t,
+ int64_t);
+typedef int64_t (*Prototype_General_GeneralInt32)(int64_t, int32_t);
+typedef int64_t (*Prototype_General_GeneralInt32Int32)(int64_t,
+ int32_t,
+ int32_t);
+typedef int64_t (*Prototype_General_GeneralInt32General)(int64_t,
+ int32_t,
+ int64_t);
+typedef int64_t (*Prototype_General_GeneralInt32Int32GeneralInt32)(int64_t,
+ int32_t,
+ int32_t,
+ int64_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32)(
+ int64_t,
+ int64_t,
+ int32_t,
+ int64_t,
+ int32_t,
+ int32_t,
+ int32_t);
+typedef int64_t (*Prototype_Int64_General)(int64_t);
+typedef int64_t (*Prototype_Int64_GeneralInt64)(int64_t, int64_t);
+
+// Simulator support for callWithABI().
+void
+Simulator::VisitCallRedirection(const Instruction* instr)
+{
+ VIXL_ASSERT(instr->Mask(ExceptionMask) == SVC);
+ VIXL_ASSERT(instr->ImmException() == kCallRtRedirected);
+
+ const Redirection* redir = Redirection::FromSvcInstruction(instr);
+ uintptr_t nativeFn = reinterpret_cast<uintptr_t>(redir->nativeFunction());
+
+ // Stack must be aligned prior to the call.
+ // FIXME: It's actually our job to perform the alignment...
+ //VIXL_ASSERT((xreg(31, Reg31IsStackPointer) & (StackAlignment - 1)) == 0);
+
+ // Used to assert that callee-saved registers are preserved.
+ DebugOnly<int64_t> x19 = xreg(19);
+ DebugOnly<int64_t> x20 = xreg(20);
+ DebugOnly<int64_t> x21 = xreg(21);
+ DebugOnly<int64_t> x22 = xreg(22);
+ DebugOnly<int64_t> x23 = xreg(23);
+ DebugOnly<int64_t> x24 = xreg(24);
+ DebugOnly<int64_t> x25 = xreg(25);
+ DebugOnly<int64_t> x26 = xreg(26);
+ DebugOnly<int64_t> x27 = xreg(27);
+ DebugOnly<int64_t> x28 = xreg(28);
+ DebugOnly<int64_t> x29 = xreg(29);
+ DebugOnly<int64_t> savedSP = get_sp();
+
+ // Remember LR for returning from the "call".
+ int64_t savedLR = xreg(30);
+
+ // Allow recursive Simulator calls: returning from the call must stop
+ // the simulation and transition back to native Simulator code.
+ set_xreg(30, int64_t(kEndOfSimAddress));
+
+ // Store argument register values in local variables for ease of use below.
+ int64_t x0 = xreg(0);
+ int64_t x1 = xreg(1);
+ int64_t x2 = xreg(2);
+ int64_t x3 = xreg(3);
+ int64_t x4 = xreg(4);
+ int64_t x5 = xreg(5);
+ int64_t x6 = xreg(6);
+ int64_t x7 = xreg(7);
+ int64_t x8 = xreg(8);
+ double d0 = dreg(0);
+ double d1 = dreg(1);
+ double d2 = dreg(2);
+ double d3 = dreg(3);
+ float s0 = sreg(0);
+ float s1 = sreg(1);
+ float s2 = sreg(2);
+ float s3 = sreg(3);
+ float s4 = sreg(4);
+
+ // Dispatch the call and set the return value.
+ switch (redir->type()) {
+ // Cases with int64_t return type.
+ case js::jit::Args_General0: {
+ int64_t ret = reinterpret_cast<Prototype_General0>(nativeFn)();
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General1: {
+ int64_t ret = reinterpret_cast<Prototype_General1>(nativeFn)(x0);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General2: {
+ int64_t ret = reinterpret_cast<Prototype_General2>(nativeFn)(x0, x1);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General3: {
+ int64_t ret = reinterpret_cast<Prototype_General3>(nativeFn)(x0, x1, x2);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General4: {
+ int64_t ret = reinterpret_cast<Prototype_General4>(nativeFn)(x0, x1, x2, x3);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General5: {
+ int64_t ret = reinterpret_cast<Prototype_General5>(nativeFn)(x0, x1, x2, x3, x4);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General6: {
+ int64_t ret = reinterpret_cast<Prototype_General6>(nativeFn)(x0, x1, x2, x3, x4, x5);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General7: {
+ int64_t ret = reinterpret_cast<Prototype_General7>(nativeFn)(x0, x1, x2, x3, x4, x5, x6);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General8: {
+ int64_t ret = reinterpret_cast<Prototype_General8>(nativeFn)(x0, x1, x2, x3, x4, x5, x6, x7);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_Int_GeneralGeneralGeneralInt64: {
+ int64_t ret = reinterpret_cast<Prototype_GeneralGeneralGeneralInt64>(nativeFn)(x0, x1, x2, x3);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_Int_GeneralGeneralInt64Int64: {
+ int64_t ret = reinterpret_cast<Prototype_GeneralGeneralInt64Int64>(nativeFn)(x0, x1, x2, x3);
+ setGPR64Result(ret);
+ break;
+ }
+
+ // Cases with GPR return type. This can be int32 or int64, but int64 is a safer assumption.
+ case js::jit::Args_Int_Double: {
+ int64_t ret = reinterpret_cast<Prototype_Int_Double>(nativeFn)(d0);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_Int_IntDouble: {
+ int64_t ret = reinterpret_cast<Prototype_Int_IntDouble>(nativeFn)(x0, d0);
+ setGPR64Result(ret);
+ break;
+ }
+
+ case js::jit::Args_Int_DoubleInt: {
+ int64_t ret = reinterpret_cast<Prototype_Int_DoubleInt>(nativeFn)(d0, x0);
+ setGPR64Result(ret);
+ break;
+ }
+
+ case js::jit::Args_Int_IntDoubleIntInt: {
+ int64_t ret = reinterpret_cast<Prototype_Int_IntDoubleIntInt>(nativeFn)(x0, d0, x1, x2);
+ setGPR64Result(ret);
+ break;
+ }
+
+ case js::jit::Args_Int_DoubleIntInt: {
+ int64_t ret = reinterpret_cast<Prototype_Int_DoubleIntInt>(nativeFn)(d0, x0, x1);
+ setGPR64Result(ret);
+ break;
+ }
+
+ // Cases with float return type.
+ case js::jit::Args_Float32_Float32: {
+ float ret = reinterpret_cast<Prototype_Float32_Float32>(nativeFn)(s0);
+ setFP32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int_Float32: {
+ int64_t ret = reinterpret_cast<Prototype_Int_Float32>(nativeFn)(s0);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_Float32_Float32Float32: {
+ float ret = reinterpret_cast<Prototype_Float32_Float32Float32>(nativeFn)(s0, s1);
+ setFP32Result(ret);
+ break;
+ }
+
+ // Cases with double return type.
+ case js::jit::Args_Double_None: {
+ double ret = reinterpret_cast<Prototype_Double_None>(nativeFn)();
+ setFP64Result(ret);
+ break;
+ }
+ case js::jit::Args_Double_Double: {
+ double ret = reinterpret_cast<Prototype_Double_Double>(nativeFn)(d0);
+ setFP64Result(ret);
+ break;
+ }
+ case js::jit::Args_Double_Int: {
+ double ret = reinterpret_cast<Prototype_Double_Int>(nativeFn)(x0);
+ setFP64Result(ret);
+ break;
+ }
+ case js::jit::Args_Double_DoubleInt: {
+ double ret = reinterpret_cast<Prototype_Double_DoubleInt>(nativeFn)(d0, x0);
+ setFP64Result(ret);
+ break;
+ }
+ case js::jit::Args_Double_DoubleDouble: {
+ double ret = reinterpret_cast<Prototype_Double_DoubleDouble>(nativeFn)(d0, d1);
+ setFP64Result(ret);
+ break;
+ }
+ case js::jit::Args_Double_DoubleDoubleDouble: {
+ double ret = reinterpret_cast<Prototype_Double_DoubleDoubleDouble>(nativeFn)(d0, d1, d2);
+ setFP64Result(ret);
+ break;
+ }
+ case js::jit::Args_Double_DoubleDoubleDoubleDouble: {
+ double ret = reinterpret_cast<Prototype_Double_DoubleDoubleDoubleDouble>(nativeFn)(d0, d1, d2, d3);
+ setFP64Result(ret);
+ break;
+ }
+
+ case js::jit::Args_Double_IntDouble: {
+ double ret = reinterpret_cast<Prototype_Double_IntDouble>(nativeFn)(x0, d0);
+ setFP64Result(ret);
+ break;
+ }
+
+ case js::jit::Args_Int32_General: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_General>(nativeFn)(x0);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32>(nativeFn)(x0, x1);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int32>(
+ nativeFn)(x0, x1, x2);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int32Int32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32>(
+ nativeFn)(x0, x1, x2, x3, x4);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int32Int32Int32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32Int32>(
+ nativeFn)(x0, x1, x2, x3, x4, x5);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int32Int32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32General>(
+ nativeFn)(x0, x1, x2, x3, x4, x5);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int32Int32Int32Int32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32Int32Int32General>(
+ nativeFn)(x0, x1, x2, x3, x4, x5, x6, x7);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Float32Float32Int32Int32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Float32Float32Int32Int32Int32General>(
+ nativeFn)(x0, x1, s0, s1, x2, x3, x4, x5);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Float32Float32Float32Float32Int32Int32Int32Int32General>(
+ nativeFn)(x0, x1, s0, s1, s2, s3, x2, x3, x4, x5, x6);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Float32Float32Int32Float32Float32Int32Float32Int32Int32Int32Int32General>(
+ nativeFn)(x0, x1, s0, s1, x2, s2, s3, x3, s4, x4, x5, x6, x7, x8);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32General>(
+ nativeFn)(x0, x1, x2, x3, x4);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int32Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int64>(
+ nativeFn)(x0, x1, x2, x3);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int32General: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int32General>(
+ nativeFn)(x0, x1, x2, x3);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32Int64Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int64Int64>(
+ nativeFn)(x0, x1, x2, x3);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32GeneralInt32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32GeneralInt32>(
+ nativeFn)(x0, x1, x2, x3);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt32GeneralInt32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32GeneralInt32Int32>(
+ nativeFn)(x0, x1, x2, x3, x4);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralGeneral: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralGeneral>(nativeFn)(x0, x1);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralGeneralGeneral: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralGeneralGeneral>(
+ nativeFn)(x0, x1, x2);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralGeneralInt32Int32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralGeneralInt32Int32>(
+ nativeFn)(x0, x1, x2, x3);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int32Int32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int32Int32>(
+ nativeFn)(x0, x1, x2, x3, x4);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int32>(
+ nativeFn)(x0, x1, x2);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int64>(
+ nativeFn)(x0, x1, x2, x3);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int64General: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int64General>(
+ nativeFn)(x0, x1, x2, x3, x4);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int64Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int64Int64>(
+ nativeFn)(x0, x1, x2, x3);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int64Int64General: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int64Int64General>(
+ nativeFn)(x0, x1, x2, x3, x4);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_General_GeneralInt32: {
+ int64_t ret =
+ reinterpret_cast<Prototype_General_GeneralInt32>(nativeFn)(x0, x1);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General_GeneralInt32Int32: {
+ int64_t ret = reinterpret_cast<Prototype_General_GeneralInt32Int32>(
+ nativeFn)(x0, x1, x2);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General_GeneralInt32General: {
+ int64_t ret =
+ reinterpret_cast<Prototype_General_GeneralInt32General>(
+ nativeFn)(x0, x1, x2);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_General_GeneralInt32Int32GeneralInt32: {
+ int64_t ret =
+ reinterpret_cast<Prototype_General_GeneralInt32Int32GeneralInt32>(
+ nativeFn)(x0, x1, x2, x3, x4);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralGeneralInt32GeneralInt32Int32Int32: {
+ int32_t ret = reinterpret_cast<
+ Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32>(nativeFn)(
+ x0, x1, x2, x3, x4, x5, x6);
+ setGPR32Result(ret);
+ break;
+ }
+ case js::jit::Args_Int64_General: {
+ int64_t ret =
+ reinterpret_cast<Prototype_Int64_General>(
+ nativeFn)(x0);
+ setGPR64Result(ret);
+ break;
+ }
+ case js::jit::Args_Int64_GeneralInt64: {
+ int64_t ret =
+ reinterpret_cast<Prototype_Int64_GeneralInt64>(
+ nativeFn)(x0, x1);
+ setGPR64Result(ret);
+ break;
+ }
+
+ default:
+ MOZ_CRASH("Unknown function type.");
+ }
+
+ // Nuke the volatile registers. x0-x7 are used as result registers, but except
+ // for x0, none are used in the above signatures.
+ for (int i = 1; i <= 18; i++) {
+ // Code feed 1 bad data
+ set_xreg(i, int64_t(0xc0defeed1badda7a));
+ }
+
+ // Assert that callee-saved registers are unchanged.
+ VIXL_ASSERT(xreg(19) == x19);
+ VIXL_ASSERT(xreg(20) == x20);
+ VIXL_ASSERT(xreg(21) == x21);
+ VIXL_ASSERT(xreg(22) == x22);
+ VIXL_ASSERT(xreg(23) == x23);
+ VIXL_ASSERT(xreg(24) == x24);
+ VIXL_ASSERT(xreg(25) == x25);
+ VIXL_ASSERT(xreg(26) == x26);
+ VIXL_ASSERT(xreg(27) == x27);
+ VIXL_ASSERT(xreg(28) == x28);
+ VIXL_ASSERT(xreg(29) == x29);
+
+ // Assert that the stack is unchanged.
+ VIXL_ASSERT(savedSP == get_sp());
+
+ // Simulate a return.
+ set_lr(savedLR);
+ set_pc((Instruction*)savedLR);
+ if (getenv("USE_DEBUGGER"))
+ printf("SVCRET\n");
+}
+
+#ifdef JS_CACHE_SIMULATOR_ARM64
+void
+Simulator::FlushICache()
+{
+ // Flush the caches recorded by the current thread as well as what got
+ // recorded from other threads before this call.
+ auto& vec = SimulatorProcess::getICacheFlushes(this);
+ for (auto& flush : vec) {
+ decoder_->FlushICache(flush.start, flush.length);
+ }
+ vec.clear();
+ pendingCacheRequests = false;
+}
+
+void CachingDecoder::Decode(const Instruction* instr) {
+ InstDecodedKind state;
+ if (lastPage_ && lastPage_->contains(instr)) {
+ state = lastPage_->decode(instr);
+ } else {
+ uintptr_t key = SinglePageDecodeCache::PageStart(instr);
+ ICacheMap::AddPtr p = iCache_.lookupForAdd(key);
+ if (p) {
+ lastPage_ = p->value();
+ state = lastPage_->decode(instr);
+ } else {
+ js::AutoEnterOOMUnsafeRegion oomUnsafe;
+ SinglePageDecodeCache* newPage = js_new<SinglePageDecodeCache>(instr);
+ if (!newPage || !iCache_.add(p, key, newPage)) {
+ oomUnsafe.crash("Simulator SinglePageDecodeCache");
+ }
+ lastPage_ = newPage;
+ state = InstDecodedKind::NotDecodedYet;
+ }
+ }
+
+ switch (state) {
+ case InstDecodedKind::NotDecodedYet: {
+ cachingDecoder_.setDecodePtr(lastPage_->decodePtr(instr));
+ this->Decoder::Decode(instr);
+ break;
+ }
+#define CASE(A) \
+ case InstDecodedKind::A: { \
+ Visit##A(instr); \
+ break; \
+ }
+
+ VISITOR_LIST(CASE)
+#undef CASE
+ }
+}
+
+void CachingDecoder::FlushICache(void* start, size_t size) {
+ MOZ_ASSERT(uintptr_t(start) % vixl::kInstructionSize == 0);
+ MOZ_ASSERT(size % vixl::kInstructionSize == 0);
+ const uint8_t* it = reinterpret_cast<const uint8_t*>(start);
+ const uint8_t* end = it + size;
+ SinglePageDecodeCache* last = nullptr;
+ for (; it < end; it += vixl::kInstructionSize) {
+ auto instr = reinterpret_cast<const Instruction*>(it);
+ if (last && last->contains(instr)) {
+ last->clearDecode(instr);
+ } else {
+ uintptr_t key = SinglePageDecodeCache::PageStart(instr);
+ ICacheMap::Ptr p = iCache_.lookup(key);
+ if (p) {
+ last = p->value();
+ last->clearDecode(instr);
+ }
+ }
+ }
+}
+#endif
+
+} // namespace vixl
+
+namespace js {
+namespace jit {
+
+#ifdef JS_CACHE_SIMULATOR_ARM64
+void SimulatorProcess::recordICacheFlush(void* start, size_t length) {
+ singleton_->lock_.assertOwnedByCurrentThread();
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ ICacheFlush range{start, length};
+ for (auto& s : singleton_->pendingFlushes_) {
+ if (!s.records.append(range)) {
+ oomUnsafe.crash("Simulator recordFlushICache");
+ }
+ }
+}
+
+void SimulatorProcess::membarrier() {
+ singleton_->lock_.assertOwnedByCurrentThread();
+ for (auto& s : singleton_->pendingFlushes_) {
+ s.thread->pendingCacheRequests = true;
+ }
+}
+
+SimulatorProcess::ICacheFlushes& SimulatorProcess::getICacheFlushes(Simulator* sim) {
+ singleton_->lock_.assertOwnedByCurrentThread();
+ for (auto& s : singleton_->pendingFlushes_) {
+ if (s.thread == sim) {
+ return s.records;
+ }
+ }
+ MOZ_CRASH("Simulator is not registered in the SimulatorProcess");
+}
+
+bool SimulatorProcess::registerSimulator(Simulator* sim) {
+ singleton_->lock_.assertOwnedByCurrentThread();
+ ICacheFlushes empty;
+ SimFlushes simFlushes{sim, std::move(empty)};
+ return singleton_->pendingFlushes_.append(std::move(simFlushes));
+}
+
+void SimulatorProcess::unregisterSimulator(Simulator* sim) {
+ singleton_->lock_.assertOwnedByCurrentThread();
+ for (auto& s : singleton_->pendingFlushes_) {
+ if (s.thread == sim) {
+ singleton_->pendingFlushes_.erase(&s);
+ return;
+ }
+ }
+ MOZ_CRASH("Simulator is not registered in the SimulatorProcess");
+}
+#endif // !JS_CACHE_SIMULATOR_ARM64
+
+} // namespace jit
+} // namespace js
+
+vixl::Simulator* JSContext::simulator() const {
+ return simulator_;
+}
diff --git a/js/src/jit/arm64/vixl/Platform-vixl.h b/js/src/jit/arm64/vixl/Platform-vixl.h
new file mode 100644
index 0000000000..a4de54c785
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Platform-vixl.h
@@ -0,0 +1,39 @@
+// Copyright 2014, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_PLATFORM_H
+#define VIXL_PLATFORM_H
+
+// Define platform specific functionalities.
+#include <signal.h>
+
+#include "jstypes.h"
+
+namespace vixl {
+inline void HostBreakpoint(int64_t code = 0) { raise(SIGINT); }
+} // namespace vixl
+
+#endif
diff --git a/js/src/jit/arm64/vixl/README.md b/js/src/jit/arm64/vixl/README.md
new file mode 100644
index 0000000000..7111753279
--- /dev/null
+++ b/js/src/jit/arm64/vixl/README.md
@@ -0,0 +1,7 @@
+This directory is a mix of VIXL files for ARM64, and files added to integrate
+VIXL within SpiderMonkey MacroAssembler. Many of SpiderMonkey extension would be
+in files prefixed with Moz*, but some might be spread across imported files when
+convenient.
+
+VIXL upstream sources can be found at:
+https://git.linaro.org/arm/vixl.git/about/
diff --git a/js/src/jit/arm64/vixl/Simulator-Constants-vixl.h b/js/src/jit/arm64/vixl/Simulator-Constants-vixl.h
new file mode 100644
index 0000000000..4b9064a89b
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Simulator-Constants-vixl.h
@@ -0,0 +1,140 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_SIMULATOR_CONSTANTS_A64_H_
+#define VIXL_A64_SIMULATOR_CONSTANTS_A64_H_
+
+namespace vixl {
+
+// Debug instructions.
+//
+// VIXL's macro-assembler and simulator support a few pseudo instructions to
+// make debugging easier. These pseudo instructions do not exist on real
+// hardware.
+//
+// TODO: Also consider allowing these pseudo-instructions to be disabled in the
+// simulator, so that users can check that the input is a valid native code.
+// (This isn't possible in all cases. Printf won't work, for example.)
+//
+// Each debug pseudo instruction is represented by a HLT instruction. The HLT
+// immediate field is used to identify the type of debug pseudo instruction.
+
+enum DebugHltOpcodes {
+ kPrintfOpcode,
+ kTraceOpcode,
+ kLogOpcode,
+ // Aliases.
+ kDebugHltFirstOpcode = kPrintfOpcode,
+ kDebugHltLastOpcode = kLogOpcode
+};
+
+// Each pseudo instruction uses a custom encoding for additional arguments, as
+// described below.
+
+// Unreachable - kUnreachableOpcode
+//
+// Instruction which should never be executed. This is used as a guard in parts
+// of the code that should not be reachable, such as in data encoded inline in
+// the instructions.
+
+// Printf - kPrintfOpcode
+// - arg_count: The number of arguments.
+// - arg_pattern: A set of PrintfArgPattern values, packed into two-bit fields.
+//
+// Simulate a call to printf.
+//
+// Floating-point and integer arguments are passed in separate sets of registers
+// in AAPCS64 (even for varargs functions), so it is not possible to determine
+// the type of each argument without some information about the values that were
+// passed in. This information could be retrieved from the printf format string,
+// but the format string is not trivial to parse so we encode the relevant
+// information with the HLT instruction.
+//
+// Also, the following registers are populated (as if for a native A64 call):
+// x0: The format string
+// x1-x7: Optional arguments, if type == CPURegister::kRegister
+// d0-d7: Optional arguments, if type == CPURegister::kFPRegister
+const unsigned kPrintfArgCountOffset = 1 * kInstructionSize;
+const unsigned kPrintfArgPatternListOffset = 2 * kInstructionSize;
+const unsigned kPrintfLength = 3 * kInstructionSize;
+
+const unsigned kPrintfMaxArgCount = 4;
+
+// The argument pattern is a set of two-bit-fields, each with one of the
+// following values:
+enum PrintfArgPattern {
+ kPrintfArgW = 1,
+ kPrintfArgX = 2,
+ // There is no kPrintfArgS because floats are always converted to doubles in C
+ // varargs calls.
+ kPrintfArgD = 3
+};
+static const unsigned kPrintfArgPatternBits = 2;
+
+// Trace - kTraceOpcode
+// - parameter: TraceParameter stored as a uint32_t
+// - command: TraceCommand stored as a uint32_t
+//
+// Allow for trace management in the generated code. This enables or disables
+// automatic tracing of the specified information for every simulated
+// instruction.
+const unsigned kTraceParamsOffset = 1 * kInstructionSize;
+const unsigned kTraceCommandOffset = 2 * kInstructionSize;
+const unsigned kTraceLength = 3 * kInstructionSize;
+
+// Trace parameters.
+enum TraceParameters {
+ LOG_DISASM = 1 << 0, // Log disassembly.
+ LOG_REGS = 1 << 1, // Log general purpose registers.
+ LOG_VREGS = 1 << 2, // Log NEON and floating-point registers.
+ LOG_SYSREGS = 1 << 3, // Log the flags and system registers.
+ LOG_WRITE = 1 << 4, // Log writes to memory.
+
+ LOG_NONE = 0,
+ LOG_STATE = LOG_REGS | LOG_VREGS | LOG_SYSREGS,
+ LOG_ALL = LOG_DISASM | LOG_STATE | LOG_WRITE
+};
+
+// Trace commands.
+enum TraceCommand {
+ TRACE_ENABLE = 1,
+ TRACE_DISABLE = 2
+};
+
+// Log - kLogOpcode
+// - parameter: TraceParameter stored as a uint32_t
+//
+// Print the specified information once. This mechanism is separate from Trace.
+// In particular, _all_ of the specified registers are printed, rather than just
+// the registers that the instruction writes.
+//
+// Any combination of the TraceParameters values can be used, except that
+// LOG_DISASM is not supported for Log.
+const unsigned kLogParamsOffset = 1 * kInstructionSize;
+const unsigned kLogLength = 2 * kInstructionSize;
+} // namespace vixl
+
+#endif // VIXL_A64_SIMULATOR_CONSTANTS_A64_H_
diff --git a/js/src/jit/arm64/vixl/Simulator-vixl.cpp b/js/src/jit/arm64/vixl/Simulator-vixl.cpp
new file mode 100644
index 0000000000..539d610b6a
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Simulator-vixl.cpp
@@ -0,0 +1,3993 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jstypes.h"
+
+#ifdef JS_SIMULATOR_ARM64
+
+#include "jit/arm64/vixl/Simulator-vixl.h"
+
+#include <cmath>
+#include <string.h>
+
+#include "jit/AtomicOperations.h"
+
+namespace vixl {
+
+const Instruction* Simulator::kEndOfSimAddress = NULL;
+
+void SimSystemRegister::SetBits(int msb, int lsb, uint32_t bits) {
+ int width = msb - lsb + 1;
+ VIXL_ASSERT(IsUintN(width, bits) || IsIntN(width, bits));
+
+ bits <<= lsb;
+ uint32_t mask = ((1 << width) - 1) << lsb;
+ VIXL_ASSERT((mask & write_ignore_mask_) == 0);
+
+ value_ = (value_ & ~mask) | (bits & mask);
+}
+
+
+SimSystemRegister SimSystemRegister::DefaultValueFor(SystemRegister id) {
+ switch (id) {
+ case NZCV:
+ return SimSystemRegister(0x00000000, NZCVWriteIgnoreMask);
+ case FPCR:
+ return SimSystemRegister(0x00000000, FPCRWriteIgnoreMask);
+ default:
+ VIXL_UNREACHABLE();
+ return SimSystemRegister();
+ }
+}
+
+
+void Simulator::Run() {
+ pc_modified_ = false;
+ while (pc_ != kEndOfSimAddress) {
+ ExecuteInstruction();
+ LogAllWrittenRegisters();
+ }
+}
+
+
+void Simulator::RunFrom(const Instruction* first) {
+ set_pc(first);
+ Run();
+}
+
+
+const char* Simulator::xreg_names[] = {
+"x0", "x1", "x2", "x3", "x4", "x5", "x6", "x7",
+"x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15",
+"x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23",
+"x24", "x25", "x26", "x27", "x28", "x29", "lr", "xzr", "sp"};
+
+const char* Simulator::wreg_names[] = {
+"w0", "w1", "w2", "w3", "w4", "w5", "w6", "w7",
+"w8", "w9", "w10", "w11", "w12", "w13", "w14", "w15",
+"w16", "w17", "w18", "w19", "w20", "w21", "w22", "w23",
+"w24", "w25", "w26", "w27", "w28", "w29", "w30", "wzr", "wsp"};
+
+const char* Simulator::sreg_names[] = {
+"s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
+"s8", "s9", "s10", "s11", "s12", "s13", "s14", "s15",
+"s16", "s17", "s18", "s19", "s20", "s21", "s22", "s23",
+"s24", "s25", "s26", "s27", "s28", "s29", "s30", "s31"};
+
+const char* Simulator::dreg_names[] = {
+"d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7",
+"d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15",
+"d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23",
+"d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31"};
+
+const char* Simulator::vreg_names[] = {
+"v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7",
+"v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15",
+"v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23",
+"v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31"};
+
+
+
+const char* Simulator::WRegNameForCode(unsigned code, Reg31Mode mode) {
+ VIXL_ASSERT(code < kNumberOfRegisters);
+ // If the code represents the stack pointer, index the name after zr.
+ if ((code == kZeroRegCode) && (mode == Reg31IsStackPointer)) {
+ code = kZeroRegCode + 1;
+ }
+ return wreg_names[code];
+}
+
+
+const char* Simulator::XRegNameForCode(unsigned code, Reg31Mode mode) {
+ VIXL_ASSERT(code < kNumberOfRegisters);
+ // If the code represents the stack pointer, index the name after zr.
+ if ((code == kZeroRegCode) && (mode == Reg31IsStackPointer)) {
+ code = kZeroRegCode + 1;
+ }
+ return xreg_names[code];
+}
+
+
+const char* Simulator::SRegNameForCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfFPRegisters);
+ return sreg_names[code];
+}
+
+
+const char* Simulator::DRegNameForCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfFPRegisters);
+ return dreg_names[code];
+}
+
+
+const char* Simulator::VRegNameForCode(unsigned code) {
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+ return vreg_names[code];
+}
+
+
+#define COLOUR(colour_code) "\033[0;" colour_code "m"
+#define COLOUR_BOLD(colour_code) "\033[1;" colour_code "m"
+#define NORMAL ""
+#define GREY "30"
+#define RED "31"
+#define GREEN "32"
+#define YELLOW "33"
+#define BLUE "34"
+#define MAGENTA "35"
+#define CYAN "36"
+#define WHITE "37"
+void Simulator::set_coloured_trace(bool value) {
+ coloured_trace_ = value;
+
+ clr_normal = value ? COLOUR(NORMAL) : "";
+ clr_flag_name = value ? COLOUR_BOLD(WHITE) : "";
+ clr_flag_value = value ? COLOUR(NORMAL) : "";
+ clr_reg_name = value ? COLOUR_BOLD(CYAN) : "";
+ clr_reg_value = value ? COLOUR(CYAN) : "";
+ clr_vreg_name = value ? COLOUR_BOLD(MAGENTA) : "";
+ clr_vreg_value = value ? COLOUR(MAGENTA) : "";
+ clr_memory_address = value ? COLOUR_BOLD(BLUE) : "";
+ clr_warning = value ? COLOUR_BOLD(YELLOW) : "";
+ clr_warning_message = value ? COLOUR(YELLOW) : "";
+ clr_printf = value ? COLOUR(GREEN) : "";
+}
+#undef COLOUR
+#undef COLOUR_BOLD
+#undef NORMAL
+#undef GREY
+#undef RED
+#undef GREEN
+#undef YELLOW
+#undef BLUE
+#undef MAGENTA
+#undef CYAN
+#undef WHITE
+
+
+void Simulator::set_trace_parameters(int parameters) {
+ bool disasm_before = trace_parameters_ & LOG_DISASM;
+ trace_parameters_ = parameters;
+ bool disasm_after = trace_parameters_ & LOG_DISASM;
+
+ if (disasm_before != disasm_after) {
+ if (disasm_after) {
+ decoder_->InsertVisitorBefore(print_disasm_, this);
+ } else {
+ decoder_->RemoveVisitor(print_disasm_);
+ }
+ }
+}
+
+
+void Simulator::set_instruction_stats(bool value) {
+ if (instrumentation_ == nullptr) {
+ return;
+ }
+
+ if (value != instruction_stats_) {
+ if (value) {
+ decoder_->AppendVisitor(instrumentation_);
+ } else {
+ decoder_->RemoveVisitor(instrumentation_);
+ }
+ instruction_stats_ = value;
+ }
+}
+
+// Helpers ---------------------------------------------------------------------
+uint64_t Simulator::AddWithCarry(unsigned reg_size,
+ bool set_flags,
+ uint64_t left,
+ uint64_t right,
+ int carry_in) {
+ VIXL_ASSERT((carry_in == 0) || (carry_in == 1));
+ VIXL_ASSERT((reg_size == kXRegSize) || (reg_size == kWRegSize));
+
+ uint64_t max_uint = (reg_size == kWRegSize) ? kWMaxUInt : kXMaxUInt;
+ uint64_t reg_mask = (reg_size == kWRegSize) ? kWRegMask : kXRegMask;
+ uint64_t sign_mask = (reg_size == kWRegSize) ? kWSignMask : kXSignMask;
+
+ left &= reg_mask;
+ right &= reg_mask;
+ uint64_t result = (left + right + carry_in) & reg_mask;
+
+ if (set_flags) {
+ nzcv().SetN(CalcNFlag(result, reg_size));
+ nzcv().SetZ(CalcZFlag(result));
+
+ // Compute the C flag by comparing the result to the max unsigned integer.
+ uint64_t max_uint_2op = max_uint - carry_in;
+ bool C = (left > max_uint_2op) || ((max_uint_2op - left) < right);
+ nzcv().SetC(C ? 1 : 0);
+
+ // Overflow iff the sign bit is the same for the two inputs and different
+ // for the result.
+ uint64_t left_sign = left & sign_mask;
+ uint64_t right_sign = right & sign_mask;
+ uint64_t result_sign = result & sign_mask;
+ bool V = (left_sign == right_sign) && (left_sign != result_sign);
+ nzcv().SetV(V ? 1 : 0);
+
+ LogSystemRegister(NZCV);
+ }
+ return result;
+}
+
+
+int64_t Simulator::ShiftOperand(unsigned reg_size,
+ int64_t value,
+ Shift shift_type,
+ unsigned amount) {
+ if (amount == 0) {
+ return value;
+ }
+ int64_t mask = reg_size == kXRegSize ? kXRegMask : kWRegMask;
+ switch (shift_type) {
+ case LSL:
+ return (value << amount) & mask;
+ case LSR:
+ return static_cast<uint64_t>(value) >> amount;
+ case ASR: {
+ // Shift used to restore the sign.
+ unsigned s_shift = kXRegSize - reg_size;
+ // Value with its sign restored.
+ int64_t s_value = (value << s_shift) >> s_shift;
+ return (s_value >> amount) & mask;
+ }
+ case ROR: {
+ if (reg_size == kWRegSize) {
+ value &= kWRegMask;
+ }
+ return (static_cast<uint64_t>(value) >> amount) |
+ ((value & ((INT64_C(1) << amount) - 1)) <<
+ (reg_size - amount));
+ }
+ default:
+ VIXL_UNIMPLEMENTED();
+ return 0;
+ }
+}
+
+
+int64_t Simulator::ExtendValue(unsigned reg_size,
+ int64_t value,
+ Extend extend_type,
+ unsigned left_shift) {
+ switch (extend_type) {
+ case UXTB:
+ value &= kByteMask;
+ break;
+ case UXTH:
+ value &= kHalfWordMask;
+ break;
+ case UXTW:
+ value &= kWordMask;
+ break;
+ case SXTB:
+ value = (value << 56) >> 56;
+ break;
+ case SXTH:
+ value = (value << 48) >> 48;
+ break;
+ case SXTW:
+ value = (value << 32) >> 32;
+ break;
+ case UXTX:
+ case SXTX:
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+ int64_t mask = (reg_size == kXRegSize) ? kXRegMask : kWRegMask;
+ return (value << left_shift) & mask;
+}
+
+
+void Simulator::FPCompare(double val0, double val1, FPTrapFlags trap) {
+ AssertSupportedFPCR();
+
+ // TODO: This assumes that the C++ implementation handles comparisons in the
+ // way that we expect (as per AssertSupportedFPCR()).
+ bool process_exception = false;
+ if ((std::isnan(val0) != 0) || (std::isnan(val1) != 0)) {
+ nzcv().SetRawValue(FPUnorderedFlag);
+ if (IsSignallingNaN(val0) || IsSignallingNaN(val1) ||
+ (trap == EnableTrap)) {
+ process_exception = true;
+ }
+ } else if (val0 < val1) {
+ nzcv().SetRawValue(FPLessThanFlag);
+ } else if (val0 > val1) {
+ nzcv().SetRawValue(FPGreaterThanFlag);
+ } else if (val0 == val1) {
+ nzcv().SetRawValue(FPEqualFlag);
+ } else {
+ VIXL_UNREACHABLE();
+ }
+ LogSystemRegister(NZCV);
+ if (process_exception) FPProcessException();
+}
+
+
+Simulator::PrintRegisterFormat Simulator::GetPrintRegisterFormatForSize(
+ unsigned reg_size, unsigned lane_size) {
+ VIXL_ASSERT(reg_size >= lane_size);
+
+ uint32_t format = 0;
+ if (reg_size != lane_size) {
+ switch (reg_size) {
+ default: VIXL_UNREACHABLE(); break;
+ case kQRegSizeInBytes: format = kPrintRegAsQVector; break;
+ case kDRegSizeInBytes: format = kPrintRegAsDVector; break;
+ }
+ }
+
+ switch (lane_size) {
+ default: VIXL_UNREACHABLE(); break;
+ case kQRegSizeInBytes: format |= kPrintReg1Q; break;
+ case kDRegSizeInBytes: format |= kPrintReg1D; break;
+ case kSRegSizeInBytes: format |= kPrintReg1S; break;
+ case kHRegSizeInBytes: format |= kPrintReg1H; break;
+ case kBRegSizeInBytes: format |= kPrintReg1B; break;
+ }
+ // These sizes would be duplicate case labels.
+ VIXL_STATIC_ASSERT(kXRegSizeInBytes == kDRegSizeInBytes);
+ VIXL_STATIC_ASSERT(kWRegSizeInBytes == kSRegSizeInBytes);
+ VIXL_STATIC_ASSERT(kPrintXReg == kPrintReg1D);
+ VIXL_STATIC_ASSERT(kPrintWReg == kPrintReg1S);
+
+ return static_cast<PrintRegisterFormat>(format);
+}
+
+
+Simulator::PrintRegisterFormat Simulator::GetPrintRegisterFormat(
+ VectorFormat vform) {
+ switch (vform) {
+ default: VIXL_UNREACHABLE(); return kPrintReg16B;
+ case kFormat16B: return kPrintReg16B;
+ case kFormat8B: return kPrintReg8B;
+ case kFormat8H: return kPrintReg8H;
+ case kFormat4H: return kPrintReg4H;
+ case kFormat4S: return kPrintReg4S;
+ case kFormat2S: return kPrintReg2S;
+ case kFormat2D: return kPrintReg2D;
+ case kFormat1D: return kPrintReg1D;
+ }
+}
+
+
+void Simulator::PrintWrittenRegisters() {
+ for (unsigned i = 0; i < kNumberOfRegisters; i++) {
+ if (registers_[i].WrittenSinceLastLog()) PrintRegister(i);
+ }
+}
+
+
+void Simulator::PrintWrittenVRegisters() {
+ for (unsigned i = 0; i < kNumberOfVRegisters; i++) {
+ // At this point there is no type information, so print as a raw 1Q.
+ if (vregisters_[i].WrittenSinceLastLog()) PrintVRegister(i, kPrintReg1Q);
+ }
+}
+
+
+void Simulator::PrintSystemRegisters() {
+ PrintSystemRegister(NZCV);
+ PrintSystemRegister(FPCR);
+}
+
+
+void Simulator::PrintRegisters() {
+ for (unsigned i = 0; i < kNumberOfRegisters; i++) {
+ PrintRegister(i);
+ }
+}
+
+
+void Simulator::PrintVRegisters() {
+ for (unsigned i = 0; i < kNumberOfVRegisters; i++) {
+ // At this point there is no type information, so print as a raw 1Q.
+ PrintVRegister(i, kPrintReg1Q);
+ }
+}
+
+
+// Print a register's name and raw value.
+//
+// Only the least-significant `size_in_bytes` bytes of the register are printed,
+// but the value is aligned as if the whole register had been printed.
+//
+// For typical register updates, size_in_bytes should be set to kXRegSizeInBytes
+// -- the default -- so that the whole register is printed. Other values of
+// size_in_bytes are intended for use when the register hasn't actually been
+// updated (such as in PrintWrite).
+//
+// No newline is printed. This allows the caller to print more details (such as
+// a memory access annotation).
+void Simulator::PrintRegisterRawHelper(unsigned code, Reg31Mode r31mode,
+ int size_in_bytes) {
+ // The template for all supported sizes.
+ // "# x{code}: 0xffeeddccbbaa9988"
+ // "# w{code}: 0xbbaa9988"
+ // "# w{code}<15:0>: 0x9988"
+ // "# w{code}<7:0>: 0x88"
+ unsigned padding_chars = (kXRegSizeInBytes - size_in_bytes) * 2;
+
+ const char * name = "";
+ const char * suffix = "";
+ switch (size_in_bytes) {
+ case kXRegSizeInBytes: name = XRegNameForCode(code, r31mode); break;
+ case kWRegSizeInBytes: name = WRegNameForCode(code, r31mode); break;
+ case 2:
+ name = WRegNameForCode(code, r31mode);
+ suffix = "<15:0>";
+ padding_chars -= strlen(suffix);
+ break;
+ case 1:
+ name = WRegNameForCode(code, r31mode);
+ suffix = "<7:0>";
+ padding_chars -= strlen(suffix);
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+ fprintf(stream_, "# %s%5s%s: ", clr_reg_name, name, suffix);
+
+ // Print leading padding spaces.
+ VIXL_ASSERT(padding_chars < (kXRegSizeInBytes * 2));
+ for (unsigned i = 0; i < padding_chars; i++) {
+ putc(' ', stream_);
+ }
+
+ // Print the specified bits in hexadecimal format.
+ uint64_t bits = reg<uint64_t>(code, r31mode);
+ bits &= kXRegMask >> ((kXRegSizeInBytes - size_in_bytes) * 8);
+ VIXL_STATIC_ASSERT(sizeof(bits) == kXRegSizeInBytes);
+
+ int chars = size_in_bytes * 2;
+ fprintf(stream_, "%s0x%0*" PRIx64 "%s",
+ clr_reg_value, chars, bits, clr_normal);
+}
+
+
+void Simulator::PrintRegister(unsigned code, Reg31Mode r31mode) {
+ registers_[code].NotifyRegisterLogged();
+
+ // Don't print writes into xzr.
+ if ((code == kZeroRegCode) && (r31mode == Reg31IsZeroRegister)) {
+ return;
+ }
+
+ // The template for all x and w registers:
+ // "# x{code}: 0x{value}"
+ // "# w{code}: 0x{value}"
+
+ PrintRegisterRawHelper(code, r31mode);
+ fprintf(stream_, "\n");
+}
+
+
+// Print a register's name and raw value.
+//
+// The `bytes` and `lsb` arguments can be used to limit the bytes that are
+// printed. These arguments are intended for use in cases where register hasn't
+// actually been updated (such as in PrintVWrite).
+//
+// No newline is printed. This allows the caller to print more details (such as
+// a floating-point interpretation or a memory access annotation).
+void Simulator::PrintVRegisterRawHelper(unsigned code, int bytes, int lsb) {
+ // The template for vector types:
+ // "# v{code}: 0xffeeddccbbaa99887766554433221100".
+ // An example with bytes=4 and lsb=8:
+ // "# v{code}: 0xbbaa9988 ".
+ fprintf(stream_, "# %s%5s: %s",
+ clr_vreg_name, VRegNameForCode(code), clr_vreg_value);
+
+ int msb = lsb + bytes - 1;
+ int byte = kQRegSizeInBytes - 1;
+
+ // Print leading padding spaces. (Two spaces per byte.)
+ while (byte > msb) {
+ fprintf(stream_, " ");
+ byte--;
+ }
+
+ // Print the specified part of the value, byte by byte.
+ qreg_t rawbits = qreg(code);
+ fprintf(stream_, "0x");
+ while (byte >= lsb) {
+ fprintf(stream_, "%02x", rawbits.val[byte]);
+ byte--;
+ }
+
+ // Print trailing padding spaces.
+ while (byte >= 0) {
+ fprintf(stream_, " ");
+ byte--;
+ }
+ fprintf(stream_, "%s", clr_normal);
+}
+
+
+// Print each of the specified lanes of a register as a float or double value.
+//
+// The `lane_count` and `lslane` arguments can be used to limit the lanes that
+// are printed. These arguments are intended for use in cases where register
+// hasn't actually been updated (such as in PrintVWrite).
+//
+// No newline is printed. This allows the caller to print more details (such as
+// a memory access annotation).
+void Simulator::PrintVRegisterFPHelper(unsigned code,
+ unsigned lane_size_in_bytes,
+ int lane_count,
+ int rightmost_lane) {
+ VIXL_ASSERT((lane_size_in_bytes == kSRegSizeInBytes) ||
+ (lane_size_in_bytes == kDRegSizeInBytes));
+
+ unsigned msb = ((lane_count + rightmost_lane) * lane_size_in_bytes);
+ VIXL_ASSERT(msb <= kQRegSizeInBytes);
+
+ // For scalar types ((lane_count == 1) && (rightmost_lane == 0)), a register
+ // name is used:
+ // " (s{code}: {value})"
+ // " (d{code}: {value})"
+ // For vector types, "..." is used to represent one or more omitted lanes.
+ // " (..., {value}, {value}, ...)"
+ if ((lane_count == 1) && (rightmost_lane == 0)) {
+ const char * name =
+ (lane_size_in_bytes == kSRegSizeInBytes) ? SRegNameForCode(code)
+ : DRegNameForCode(code);
+ fprintf(stream_, " (%s%s: ", clr_vreg_name, name);
+ } else {
+ if (msb < (kQRegSizeInBytes - 1)) {
+ fprintf(stream_, " (..., ");
+ } else {
+ fprintf(stream_, " (");
+ }
+ }
+
+ // Print the list of values.
+ const char * separator = "";
+ int leftmost_lane = rightmost_lane + lane_count - 1;
+ for (int lane = leftmost_lane; lane >= rightmost_lane; lane--) {
+ double value =
+ (lane_size_in_bytes == kSRegSizeInBytes) ? vreg(code).Get<float>(lane)
+ : vreg(code).Get<double>(lane);
+ fprintf(stream_, "%s%s%#g%s", separator, clr_vreg_value, value, clr_normal);
+ separator = ", ";
+ }
+
+ if (rightmost_lane > 0) {
+ fprintf(stream_, ", ...");
+ }
+ fprintf(stream_, ")");
+}
+
+
+void Simulator::PrintVRegister(unsigned code, PrintRegisterFormat format) {
+ vregisters_[code].NotifyRegisterLogged();
+
+ int lane_size_log2 = format & kPrintRegLaneSizeMask;
+
+ int reg_size_log2;
+ if (format & kPrintRegAsQVector) {
+ reg_size_log2 = kQRegSizeInBytesLog2;
+ } else if (format & kPrintRegAsDVector) {
+ reg_size_log2 = kDRegSizeInBytesLog2;
+ } else {
+ // Scalar types.
+ reg_size_log2 = lane_size_log2;
+ }
+
+ int lane_count = 1 << (reg_size_log2 - lane_size_log2);
+ int lane_size = 1 << lane_size_log2;
+
+ // The template for vector types:
+ // "# v{code}: 0x{rawbits} (..., {value}, ...)".
+ // The template for scalar types:
+ // "# v{code}: 0x{rawbits} ({reg}:{value})".
+ // The values in parentheses after the bit representations are floating-point
+ // interpretations. They are displayed only if the kPrintVRegAsFP bit is set.
+
+ PrintVRegisterRawHelper(code);
+ if (format & kPrintRegAsFP) {
+ PrintVRegisterFPHelper(code, lane_size, lane_count);
+ }
+
+ fprintf(stream_, "\n");
+}
+
+
+void Simulator::PrintSystemRegister(SystemRegister id) {
+ switch (id) {
+ case NZCV:
+ fprintf(stream_, "# %sNZCV: %sN:%d Z:%d C:%d V:%d%s\n",
+ clr_flag_name, clr_flag_value,
+ nzcv().N(), nzcv().Z(), nzcv().C(), nzcv().V(),
+ clr_normal);
+ break;
+ case FPCR: {
+ static const char * rmode[] = {
+ "0b00 (Round to Nearest)",
+ "0b01 (Round towards Plus Infinity)",
+ "0b10 (Round towards Minus Infinity)",
+ "0b11 (Round towards Zero)"
+ };
+ VIXL_ASSERT(fpcr().RMode() < (sizeof(rmode) / sizeof(rmode[0])));
+ fprintf(stream_,
+ "# %sFPCR: %sAHP:%d DN:%d FZ:%d RMode:%s%s\n",
+ clr_flag_name, clr_flag_value,
+ fpcr().AHP(), fpcr().DN(), fpcr().FZ(), rmode[fpcr().RMode()],
+ clr_normal);
+ break;
+ }
+ default:
+ VIXL_UNREACHABLE();
+ }
+}
+
+
+void Simulator::PrintRead(uintptr_t address,
+ unsigned reg_code,
+ PrintRegisterFormat format) {
+ registers_[reg_code].NotifyRegisterLogged();
+
+ USE(format);
+
+ // The template is "# {reg}: 0x{value} <- {address}".
+ PrintRegisterRawHelper(reg_code, Reg31IsZeroRegister);
+ fprintf(stream_, " <- %s0x%016" PRIxPTR "%s\n",
+ clr_memory_address, address, clr_normal);
+}
+
+
+void Simulator::PrintVRead(uintptr_t address,
+ unsigned reg_code,
+ PrintRegisterFormat format,
+ unsigned lane) {
+ vregisters_[reg_code].NotifyRegisterLogged();
+
+ // The template is "# v{code}: 0x{rawbits} <- address".
+ PrintVRegisterRawHelper(reg_code);
+ if (format & kPrintRegAsFP) {
+ PrintVRegisterFPHelper(reg_code, GetPrintRegLaneSizeInBytes(format),
+ GetPrintRegLaneCount(format), lane);
+ }
+ fprintf(stream_, " <- %s0x%016" PRIxPTR "%s\n",
+ clr_memory_address, address, clr_normal);
+}
+
+
+void Simulator::PrintWrite(uintptr_t address,
+ unsigned reg_code,
+ PrintRegisterFormat format) {
+ VIXL_ASSERT(GetPrintRegLaneCount(format) == 1);
+
+ // The template is "# v{code}: 0x{value} -> {address}". To keep the trace tidy
+ // and readable, the value is aligned with the values in the register trace.
+ PrintRegisterRawHelper(reg_code, Reg31IsZeroRegister,
+ GetPrintRegSizeInBytes(format));
+ fprintf(stream_, " -> %s0x%016" PRIxPTR "%s\n",
+ clr_memory_address, address, clr_normal);
+}
+
+
+void Simulator::PrintVWrite(uintptr_t address,
+ unsigned reg_code,
+ PrintRegisterFormat format,
+ unsigned lane) {
+ // The templates:
+ // "# v{code}: 0x{rawbits} -> {address}"
+ // "# v{code}: 0x{rawbits} (..., {value}, ...) -> {address}".
+ // "# v{code}: 0x{rawbits} ({reg}:{value}) -> {address}"
+ // Because this trace doesn't represent a change to the source register's
+ // value, only the relevant part of the value is printed. To keep the trace
+ // tidy and readable, the raw value is aligned with the other values in the
+ // register trace.
+ int lane_count = GetPrintRegLaneCount(format);
+ int lane_size = GetPrintRegLaneSizeInBytes(format);
+ int reg_size = GetPrintRegSizeInBytes(format);
+ PrintVRegisterRawHelper(reg_code, reg_size, lane_size * lane);
+ if (format & kPrintRegAsFP) {
+ PrintVRegisterFPHelper(reg_code, lane_size, lane_count, lane);
+ }
+ fprintf(stream_, " -> %s0x%016" PRIxPTR "%s\n",
+ clr_memory_address, address, clr_normal);
+}
+
+
+// Visitors---------------------------------------------------------------------
+
+void Simulator::VisitUnimplemented(const Instruction* instr) {
+ printf("Unimplemented instruction at %p: 0x%08" PRIx32 "\n",
+ reinterpret_cast<const void*>(instr), instr->InstructionBits());
+ VIXL_UNIMPLEMENTED();
+}
+
+
+void Simulator::VisitUnallocated(const Instruction* instr) {
+ printf("Unallocated instruction at %p: 0x%08" PRIx32 "\n",
+ reinterpret_cast<const void*>(instr), instr->InstructionBits());
+ VIXL_UNIMPLEMENTED();
+}
+
+
+void Simulator::VisitPCRelAddressing(const Instruction* instr) {
+ VIXL_ASSERT((instr->Mask(PCRelAddressingMask) == ADR) ||
+ (instr->Mask(PCRelAddressingMask) == ADRP));
+
+ set_reg(instr->Rd(), instr->ImmPCOffsetTarget());
+}
+
+
+void Simulator::VisitUnconditionalBranch(const Instruction* instr) {
+ switch (instr->Mask(UnconditionalBranchMask)) {
+ case BL:
+ set_lr(instr->NextInstruction());
+ VIXL_FALLTHROUGH();
+ case B:
+ set_pc(instr->ImmPCOffsetTarget());
+ break;
+ default: VIXL_UNREACHABLE();
+ }
+}
+
+
+void Simulator::VisitConditionalBranch(const Instruction* instr) {
+ VIXL_ASSERT(instr->Mask(ConditionalBranchMask) == B_cond);
+ if (ConditionPassed(instr->ConditionBranch())) {
+ set_pc(instr->ImmPCOffsetTarget());
+ }
+}
+
+
+void Simulator::VisitUnconditionalBranchToRegister(const Instruction* instr) {
+ const Instruction* target = Instruction::Cast(xreg(instr->Rn()));
+
+ switch (instr->Mask(UnconditionalBranchToRegisterMask)) {
+ case BLR:
+ set_lr(instr->NextInstruction());
+ VIXL_FALLTHROUGH();
+ case BR:
+ case RET: set_pc(target); break;
+ default: VIXL_UNREACHABLE();
+ }
+}
+
+
+void Simulator::VisitTestBranch(const Instruction* instr) {
+ unsigned bit_pos = (instr->ImmTestBranchBit5() << 5) |
+ instr->ImmTestBranchBit40();
+ bool bit_zero = ((xreg(instr->Rt()) >> bit_pos) & 1) == 0;
+ bool take_branch = false;
+ switch (instr->Mask(TestBranchMask)) {
+ case TBZ: take_branch = bit_zero; break;
+ case TBNZ: take_branch = !bit_zero; break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ if (take_branch) {
+ set_pc(instr->ImmPCOffsetTarget());
+ }
+}
+
+
+void Simulator::VisitCompareBranch(const Instruction* instr) {
+ unsigned rt = instr->Rt();
+ bool take_branch = false;
+ switch (instr->Mask(CompareBranchMask)) {
+ case CBZ_w: take_branch = (wreg(rt) == 0); break;
+ case CBZ_x: take_branch = (xreg(rt) == 0); break;
+ case CBNZ_w: take_branch = (wreg(rt) != 0); break;
+ case CBNZ_x: take_branch = (xreg(rt) != 0); break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ if (take_branch) {
+ set_pc(instr->ImmPCOffsetTarget());
+ }
+}
+
+
+void Simulator::AddSubHelper(const Instruction* instr, int64_t op2) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ bool set_flags = instr->FlagsUpdate();
+ int64_t new_val = 0;
+ Instr operation = instr->Mask(AddSubOpMask);
+
+ switch (operation) {
+ case ADD:
+ case ADDS: {
+ new_val = AddWithCarry(reg_size,
+ set_flags,
+ reg(reg_size, instr->Rn(), instr->RnMode()),
+ op2);
+ break;
+ }
+ case SUB:
+ case SUBS: {
+ new_val = AddWithCarry(reg_size,
+ set_flags,
+ reg(reg_size, instr->Rn(), instr->RnMode()),
+ ~op2,
+ 1);
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+
+ set_reg(reg_size, instr->Rd(), new_val, LogRegWrites, instr->RdMode());
+}
+
+
+void Simulator::VisitAddSubShifted(const Instruction* instr) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ int64_t op2 = ShiftOperand(reg_size,
+ reg(reg_size, instr->Rm()),
+ static_cast<Shift>(instr->ShiftDP()),
+ instr->ImmDPShift());
+ AddSubHelper(instr, op2);
+}
+
+
+void Simulator::VisitAddSubImmediate(const Instruction* instr) {
+ int64_t op2 = instr->ImmAddSub() << ((instr->ShiftAddSub() == 1) ? 12 : 0);
+ AddSubHelper(instr, op2);
+}
+
+
+void Simulator::VisitAddSubExtended(const Instruction* instr) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ int64_t op2 = ExtendValue(reg_size,
+ reg(reg_size, instr->Rm()),
+ static_cast<Extend>(instr->ExtendMode()),
+ instr->ImmExtendShift());
+ AddSubHelper(instr, op2);
+}
+
+
+void Simulator::VisitAddSubWithCarry(const Instruction* instr) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ int64_t op2 = reg(reg_size, instr->Rm());
+ int64_t new_val;
+
+ if ((instr->Mask(AddSubOpMask) == SUB) || instr->Mask(AddSubOpMask) == SUBS) {
+ op2 = ~op2;
+ }
+
+ new_val = AddWithCarry(reg_size,
+ instr->FlagsUpdate(),
+ reg(reg_size, instr->Rn()),
+ op2,
+ C());
+
+ set_reg(reg_size, instr->Rd(), new_val);
+}
+
+
+void Simulator::VisitLogicalShifted(const Instruction* instr) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ Shift shift_type = static_cast<Shift>(instr->ShiftDP());
+ unsigned shift_amount = instr->ImmDPShift();
+ int64_t op2 = ShiftOperand(reg_size, reg(reg_size, instr->Rm()), shift_type,
+ shift_amount);
+ if (instr->Mask(NOT) == NOT) {
+ op2 = ~op2;
+ }
+ LogicalHelper(instr, op2);
+}
+
+
+void Simulator::VisitLogicalImmediate(const Instruction* instr) {
+ LogicalHelper(instr, instr->ImmLogical());
+}
+
+
+void Simulator::LogicalHelper(const Instruction* instr, int64_t op2) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ int64_t op1 = reg(reg_size, instr->Rn());
+ int64_t result = 0;
+ bool update_flags = false;
+
+ // Switch on the logical operation, stripping out the NOT bit, as it has a
+ // different meaning for logical immediate instructions.
+ switch (instr->Mask(LogicalOpMask & ~NOT)) {
+ case ANDS: update_flags = true; VIXL_FALLTHROUGH();
+ case AND: result = op1 & op2; break;
+ case ORR: result = op1 | op2; break;
+ case EOR: result = op1 ^ op2; break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+
+ if (update_flags) {
+ nzcv().SetN(CalcNFlag(result, reg_size));
+ nzcv().SetZ(CalcZFlag(result));
+ nzcv().SetC(0);
+ nzcv().SetV(0);
+ LogSystemRegister(NZCV);
+ }
+
+ set_reg(reg_size, instr->Rd(), result, LogRegWrites, instr->RdMode());
+}
+
+
+void Simulator::VisitConditionalCompareRegister(const Instruction* instr) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ ConditionalCompareHelper(instr, reg(reg_size, instr->Rm()));
+}
+
+
+void Simulator::VisitConditionalCompareImmediate(const Instruction* instr) {
+ ConditionalCompareHelper(instr, instr->ImmCondCmp());
+}
+
+
+void Simulator::ConditionalCompareHelper(const Instruction* instr,
+ int64_t op2) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ int64_t op1 = reg(reg_size, instr->Rn());
+
+ if (ConditionPassed(instr->Condition())) {
+ // If the condition passes, set the status flags to the result of comparing
+ // the operands.
+ if (instr->Mask(ConditionalCompareMask) == CCMP) {
+ AddWithCarry(reg_size, true, op1, ~op2, 1);
+ } else {
+ VIXL_ASSERT(instr->Mask(ConditionalCompareMask) == CCMN);
+ AddWithCarry(reg_size, true, op1, op2, 0);
+ }
+ } else {
+ // If the condition fails, set the status flags to the nzcv immediate.
+ nzcv().SetFlags(instr->Nzcv());
+ LogSystemRegister(NZCV);
+ }
+}
+
+
+void Simulator::VisitLoadStoreUnsignedOffset(const Instruction* instr) {
+ int offset = instr->ImmLSUnsigned() << instr->SizeLS();
+ LoadStoreHelper(instr, offset, Offset);
+}
+
+
+void Simulator::VisitLoadStoreUnscaledOffset(const Instruction* instr) {
+ LoadStoreHelper(instr, instr->ImmLS(), Offset);
+}
+
+
+void Simulator::VisitLoadStorePreIndex(const Instruction* instr) {
+ LoadStoreHelper(instr, instr->ImmLS(), PreIndex);
+}
+
+
+void Simulator::VisitLoadStorePostIndex(const Instruction* instr) {
+ LoadStoreHelper(instr, instr->ImmLS(), PostIndex);
+}
+
+
+void Simulator::VisitLoadStoreRegisterOffset(const Instruction* instr) {
+ Extend ext = static_cast<Extend>(instr->ExtendMode());
+ VIXL_ASSERT((ext == UXTW) || (ext == UXTX) || (ext == SXTW) || (ext == SXTX));
+ unsigned shift_amount = instr->ImmShiftLS() * instr->SizeLS();
+
+ int64_t offset = ExtendValue(kXRegSize, xreg(instr->Rm()), ext,
+ shift_amount);
+ LoadStoreHelper(instr, offset, Offset);
+}
+
+template<typename T>
+static T Faulted() {
+ return ~0;
+}
+
+template<>
+Simulator::qreg_t Faulted() {
+ static_assert(kQRegSizeInBytes == 16, "Known constraint");
+ static Simulator::qreg_t dummy = { {
+ 255, 255, 255, 255, 255, 255, 255, 255,
+ 255, 255, 255, 255, 255, 255, 255, 255
+ } };
+ return dummy;
+}
+
+template<typename T> T
+Simulator::Read(uintptr_t address)
+{
+ address = Memory::AddressUntag(address);
+ if (handle_wasm_seg_fault(address, sizeof(T)))
+ return Faulted<T>();
+ return Memory::Read<T>(address);
+}
+
+template <typename T> void
+Simulator::Write(uintptr_t address, T value)
+{
+ address = Memory::AddressUntag(address);
+ if (handle_wasm_seg_fault(address, sizeof(T)))
+ return;
+ Memory::Write<T>(address, value);
+}
+
+void Simulator::LoadStoreHelper(const Instruction* instr,
+ int64_t offset,
+ AddrMode addrmode) {
+ unsigned srcdst = instr->Rt();
+ uintptr_t address = AddressModeHelper(instr->Rn(), offset, addrmode);
+
+ LoadStoreOp op = static_cast<LoadStoreOp>(instr->Mask(LoadStoreMask));
+ switch (op) {
+ case LDRB_w:
+ set_wreg(srcdst, Read<uint8_t>(address), NoRegLog); break;
+ case LDRH_w:
+ set_wreg(srcdst, Read<uint16_t>(address), NoRegLog); break;
+ case LDR_w:
+ set_wreg(srcdst, Read<uint32_t>(address), NoRegLog); break;
+ case LDR_x:
+ set_xreg(srcdst, Read<uint64_t>(address), NoRegLog); break;
+ case LDRSB_w:
+ set_wreg(srcdst, Read<int8_t>(address), NoRegLog); break;
+ case LDRSH_w:
+ set_wreg(srcdst, Read<int16_t>(address), NoRegLog); break;
+ case LDRSB_x:
+ set_xreg(srcdst, Read<int8_t>(address), NoRegLog); break;
+ case LDRSH_x:
+ set_xreg(srcdst, Read<int16_t>(address), NoRegLog); break;
+ case LDRSW_x:
+ set_xreg(srcdst, Read<int32_t>(address), NoRegLog); break;
+ case LDR_b:
+ set_breg(srcdst, Read<uint8_t>(address), NoRegLog); break;
+ case LDR_h:
+ set_hreg(srcdst, Read<uint16_t>(address), NoRegLog); break;
+ case LDR_s:
+ set_sreg(srcdst, Read<float>(address), NoRegLog); break;
+ case LDR_d:
+ set_dreg(srcdst, Read<double>(address), NoRegLog); break;
+ case LDR_q:
+ set_qreg(srcdst, Read<qreg_t>(address), NoRegLog); break;
+
+ case STRB_w: Write<uint8_t>(address, wreg(srcdst)); break;
+ case STRH_w: Write<uint16_t>(address, wreg(srcdst)); break;
+ case STR_w: Write<uint32_t>(address, wreg(srcdst)); break;
+ case STR_x: Write<uint64_t>(address, xreg(srcdst)); break;
+ case STR_b: Write<uint8_t>(address, breg(srcdst)); break;
+ case STR_h: Write<uint16_t>(address, hreg(srcdst)); break;
+ case STR_s: Write<float>(address, sreg(srcdst)); break;
+ case STR_d: Write<double>(address, dreg(srcdst)); break;
+ case STR_q: Write<qreg_t>(address, qreg(srcdst)); break;
+
+ // Ignore prfm hint instructions.
+ case PRFM: break;
+
+ default: VIXL_UNIMPLEMENTED();
+ }
+
+ unsigned access_size = 1 << instr->SizeLS();
+ if (instr->IsLoad()) {
+ if ((op == LDR_s) || (op == LDR_d)) {
+ LogVRead(address, srcdst, GetPrintRegisterFormatForSizeFP(access_size));
+ } else if ((op == LDR_b) || (op == LDR_h) || (op == LDR_q)) {
+ LogVRead(address, srcdst, GetPrintRegisterFormatForSize(access_size));
+ } else {
+ LogRead(address, srcdst, GetPrintRegisterFormatForSize(access_size));
+ }
+ } else {
+ if ((op == STR_s) || (op == STR_d)) {
+ LogVWrite(address, srcdst, GetPrintRegisterFormatForSizeFP(access_size));
+ } else if ((op == STR_b) || (op == STR_h) || (op == STR_q)) {
+ LogVWrite(address, srcdst, GetPrintRegisterFormatForSize(access_size));
+ } else {
+ LogWrite(address, srcdst, GetPrintRegisterFormatForSize(access_size));
+ }
+ }
+
+ local_monitor_.MaybeClear();
+}
+
+
+void Simulator::VisitLoadStorePairOffset(const Instruction* instr) {
+ LoadStorePairHelper(instr, Offset);
+}
+
+
+void Simulator::VisitLoadStorePairPreIndex(const Instruction* instr) {
+ LoadStorePairHelper(instr, PreIndex);
+}
+
+
+void Simulator::VisitLoadStorePairPostIndex(const Instruction* instr) {
+ LoadStorePairHelper(instr, PostIndex);
+}
+
+
+void Simulator::VisitLoadStorePairNonTemporal(const Instruction* instr) {
+ LoadStorePairHelper(instr, Offset);
+}
+
+
+void Simulator::LoadStorePairHelper(const Instruction* instr,
+ AddrMode addrmode) {
+ unsigned rt = instr->Rt();
+ unsigned rt2 = instr->Rt2();
+ int element_size = 1 << instr->SizeLSPair();
+ int64_t offset = instr->ImmLSPair() * element_size;
+ uintptr_t address = AddressModeHelper(instr->Rn(), offset, addrmode);
+ uintptr_t address2 = address + element_size;
+
+ LoadStorePairOp op =
+ static_cast<LoadStorePairOp>(instr->Mask(LoadStorePairMask));
+
+ // 'rt' and 'rt2' can only be aliased for stores.
+ VIXL_ASSERT(((op & LoadStorePairLBit) == 0) || (rt != rt2));
+
+ switch (op) {
+ // Use NoRegLog to suppress the register trace (LOG_REGS, LOG_FP_REGS). We
+ // will print a more detailed log.
+ case LDP_w: {
+ set_wreg(rt, Read<uint32_t>(address), NoRegLog);
+ set_wreg(rt2, Read<uint32_t>(address2), NoRegLog);
+ break;
+ }
+ case LDP_s: {
+ set_sreg(rt, Read<float>(address), NoRegLog);
+ set_sreg(rt2, Read<float>(address2), NoRegLog);
+ break;
+ }
+ case LDP_x: {
+ set_xreg(rt, Read<uint64_t>(address), NoRegLog);
+ set_xreg(rt2, Read<uint64_t>(address2), NoRegLog);
+ break;
+ }
+ case LDP_d: {
+ set_dreg(rt, Read<double>(address), NoRegLog);
+ set_dreg(rt2, Read<double>(address2), NoRegLog);
+ break;
+ }
+ case LDP_q: {
+ set_qreg(rt, Read<qreg_t>(address), NoRegLog);
+ set_qreg(rt2, Read<qreg_t>(address2), NoRegLog);
+ break;
+ }
+ case LDPSW_x: {
+ set_xreg(rt, Read<int32_t>(address), NoRegLog);
+ set_xreg(rt2, Read<int32_t>(address2), NoRegLog);
+ break;
+ }
+ case STP_w: {
+ Write<uint32_t>(address, wreg(rt));
+ Write<uint32_t>(address2, wreg(rt2));
+ break;
+ }
+ case STP_s: {
+ Write<float>(address, sreg(rt));
+ Write<float>(address2, sreg(rt2));
+ break;
+ }
+ case STP_x: {
+ Write<uint64_t>(address, xreg(rt));
+ Write<uint64_t>(address2, xreg(rt2));
+ break;
+ }
+ case STP_d: {
+ Write<double>(address, dreg(rt));
+ Write<double>(address2, dreg(rt2));
+ break;
+ }
+ case STP_q: {
+ Write<qreg_t>(address, qreg(rt));
+ Write<qreg_t>(address2, qreg(rt2));
+ break;
+ }
+ default: VIXL_UNREACHABLE();
+ }
+
+ // Print a detailed trace (including the memory address) instead of the basic
+ // register:value trace generated by set_*reg().
+ if (instr->IsLoad()) {
+ if ((op == LDP_s) || (op == LDP_d)) {
+ LogVRead(address, rt, GetPrintRegisterFormatForSizeFP(element_size));
+ LogVRead(address2, rt2, GetPrintRegisterFormatForSizeFP(element_size));
+ } else if (op == LDP_q) {
+ LogVRead(address, rt, GetPrintRegisterFormatForSize(element_size));
+ LogVRead(address2, rt2, GetPrintRegisterFormatForSize(element_size));
+ } else {
+ LogRead(address, rt, GetPrintRegisterFormatForSize(element_size));
+ LogRead(address2, rt2, GetPrintRegisterFormatForSize(element_size));
+ }
+ } else {
+ if ((op == STP_s) || (op == STP_d)) {
+ LogVWrite(address, rt, GetPrintRegisterFormatForSizeFP(element_size));
+ LogVWrite(address2, rt2, GetPrintRegisterFormatForSizeFP(element_size));
+ } else if (op == STP_q) {
+ LogVWrite(address, rt, GetPrintRegisterFormatForSize(element_size));
+ LogVWrite(address2, rt2, GetPrintRegisterFormatForSize(element_size));
+ } else {
+ LogWrite(address, rt, GetPrintRegisterFormatForSize(element_size));
+ LogWrite(address2, rt2, GetPrintRegisterFormatForSize(element_size));
+ }
+ }
+
+ local_monitor_.MaybeClear();
+}
+
+
+void Simulator::PrintExclusiveAccessWarning() {
+ if (print_exclusive_access_warning_) {
+ fprintf(
+ stderr,
+ "%sWARNING:%s VIXL simulator support for load-/store-/clear-exclusive "
+ "instructions is limited. Refer to the README for details.%s\n",
+ clr_warning, clr_warning_message, clr_normal);
+ print_exclusive_access_warning_ = false;
+ }
+}
+
+
+void Simulator::VisitLoadStoreExclusive(const Instruction* instr) {
+ PrintExclusiveAccessWarning();
+
+ unsigned rs = instr->Rs();
+ unsigned rt = instr->Rt();
+ unsigned rt2 = instr->Rt2();
+ unsigned rn = instr->Rn();
+
+ LoadStoreExclusive op =
+ static_cast<LoadStoreExclusive>(instr->Mask(LoadStoreExclusiveMask));
+
+ bool is_acquire_release = instr->LdStXAcquireRelease();
+ bool is_exclusive = !instr->LdStXNotExclusive();
+ bool is_load = instr->LdStXLoad();
+ bool is_pair = instr->LdStXPair();
+
+ unsigned element_size = 1 << instr->LdStXSizeLog2();
+ unsigned access_size = is_pair ? element_size * 2 : element_size;
+ uint64_t address = reg<uint64_t>(rn, Reg31IsStackPointer);
+
+ // Verify that the address is available to the host.
+ VIXL_ASSERT(address == static_cast<uintptr_t>(address));
+
+ // Check the alignment of `address`.
+ if (AlignDown(address, access_size) != address) {
+ VIXL_ALIGNMENT_EXCEPTION();
+ }
+
+ // The sp must be aligned to 16 bytes when it is accessed.
+ if ((rn == 31) && (AlignDown(address, 16) != address)) {
+ VIXL_ALIGNMENT_EXCEPTION();
+ }
+
+ if (is_load) {
+ if (is_exclusive) {
+ local_monitor_.MarkExclusive(address, access_size);
+ } else {
+ // Any non-exclusive load can clear the local monitor as a side effect. We
+ // don't need to do this, but it is useful to stress the simulated code.
+ local_monitor_.Clear();
+ }
+
+ // Use NoRegLog to suppress the register trace (LOG_REGS, LOG_FP_REGS). We
+ // will print a more detailed log.
+ switch (op) {
+ case LDXRB_w:
+ case LDAXRB_w:
+ case LDARB_w:
+ set_wreg(rt, Read<uint8_t>(address), NoRegLog);
+ break;
+ case LDXRH_w:
+ case LDAXRH_w:
+ case LDARH_w:
+ set_wreg(rt, Read<uint16_t>(address), NoRegLog);
+ break;
+ case LDXR_w:
+ case LDAXR_w:
+ case LDAR_w:
+ set_wreg(rt, Read<uint32_t>(address), NoRegLog);
+ break;
+ case LDXR_x:
+ case LDAXR_x:
+ case LDAR_x:
+ set_xreg(rt, Read<uint64_t>(address), NoRegLog);
+ break;
+ case LDXP_w:
+ case LDAXP_w:
+ set_wreg(rt, Read<uint32_t>(address), NoRegLog);
+ set_wreg(rt2, Read<uint32_t>(address + element_size), NoRegLog);
+ break;
+ case LDXP_x:
+ case LDAXP_x:
+ set_xreg(rt, Read<uint64_t>(address), NoRegLog);
+ set_xreg(rt2, Read<uint64_t>(address + element_size), NoRegLog);
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+
+ if (is_acquire_release) {
+ // Approximate load-acquire by issuing a full barrier after the load.
+ js::jit::AtomicOperations::fenceSeqCst();
+ }
+
+ LogRead(address, rt, GetPrintRegisterFormatForSize(element_size));
+ if (is_pair) {
+ LogRead(address + element_size, rt2,
+ GetPrintRegisterFormatForSize(element_size));
+ }
+ } else {
+ if (is_acquire_release) {
+ // Approximate store-release by issuing a full barrier before the store.
+ js::jit::AtomicOperations::fenceSeqCst();
+ }
+
+ bool do_store = true;
+ if (is_exclusive) {
+ do_store = local_monitor_.IsExclusive(address, access_size) &&
+ global_monitor_.IsExclusive(address, access_size);
+ set_wreg(rs, do_store ? 0 : 1);
+
+ // - All exclusive stores explicitly clear the local monitor.
+ local_monitor_.Clear();
+ } else {
+ // - Any other store can clear the local monitor as a side effect.
+ local_monitor_.MaybeClear();
+ }
+
+ if (do_store) {
+ switch (op) {
+ case STXRB_w:
+ case STLXRB_w:
+ case STLRB_w:
+ Write<uint8_t>(address, wreg(rt));
+ break;
+ case STXRH_w:
+ case STLXRH_w:
+ case STLRH_w:
+ Write<uint16_t>(address, wreg(rt));
+ break;
+ case STXR_w:
+ case STLXR_w:
+ case STLR_w:
+ Write<uint32_t>(address, wreg(rt));
+ break;
+ case STXR_x:
+ case STLXR_x:
+ case STLR_x:
+ Write<uint64_t>(address, xreg(rt));
+ break;
+ case STXP_w:
+ case STLXP_w:
+ Write<uint32_t>(address, wreg(rt));
+ Write<uint32_t>(address + element_size, wreg(rt2));
+ break;
+ case STXP_x:
+ case STLXP_x:
+ Write<uint64_t>(address, xreg(rt));
+ Write<uint64_t>(address + element_size, xreg(rt2));
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+
+ LogWrite(address, rt, GetPrintRegisterFormatForSize(element_size));
+ if (is_pair) {
+ LogWrite(address + element_size, rt2,
+ GetPrintRegisterFormatForSize(element_size));
+ }
+ }
+ }
+}
+
+
+void Simulator::VisitLoadLiteral(const Instruction* instr) {
+ unsigned rt = instr->Rt();
+ uint64_t address = instr->LiteralAddress<uint64_t>();
+
+ // Verify that the calculated address is available to the host.
+ VIXL_ASSERT(address == static_cast<uintptr_t>(address));
+
+ switch (instr->Mask(LoadLiteralMask)) {
+ // Use NoRegLog to suppress the register trace (LOG_REGS, LOG_VREGS), then
+ // print a more detailed log.
+ case LDR_w_lit:
+ set_wreg(rt, Read<uint32_t>(address), NoRegLog);
+ LogRead(address, rt, kPrintWReg);
+ break;
+ case LDR_x_lit:
+ set_xreg(rt, Read<uint64_t>(address), NoRegLog);
+ LogRead(address, rt, kPrintXReg);
+ break;
+ case LDR_s_lit:
+ set_sreg(rt, Read<float>(address), NoRegLog);
+ LogVRead(address, rt, kPrintSReg);
+ break;
+ case LDR_d_lit:
+ set_dreg(rt, Read<double>(address), NoRegLog);
+ LogVRead(address, rt, kPrintDReg);
+ break;
+ case LDR_q_lit:
+ set_qreg(rt, Read<qreg_t>(address), NoRegLog);
+ LogVRead(address, rt, kPrintReg1Q);
+ break;
+ case LDRSW_x_lit:
+ set_xreg(rt, Read<int32_t>(address), NoRegLog);
+ LogRead(address, rt, kPrintWReg);
+ break;
+
+ // Ignore prfm hint instructions.
+ case PRFM_lit: break;
+
+ default: VIXL_UNREACHABLE();
+ }
+
+ local_monitor_.MaybeClear();
+}
+
+
+uintptr_t Simulator::AddressModeHelper(unsigned addr_reg,
+ int64_t offset,
+ AddrMode addrmode) {
+ uint64_t address = xreg(addr_reg, Reg31IsStackPointer);
+
+ if ((addr_reg == 31) && ((address % 16) != 0)) {
+ // When the base register is SP the stack pointer is required to be
+ // quadword aligned prior to the address calculation and write-backs.
+ // Misalignment will cause a stack alignment fault.
+ VIXL_ALIGNMENT_EXCEPTION();
+ }
+
+ if ((addrmode == PreIndex) || (addrmode == PostIndex)) {
+ VIXL_ASSERT(offset != 0);
+ // Only preindex should log the register update here. For Postindex, the
+ // update will be printed automatically by LogWrittenRegisters _after_ the
+ // memory access itself is logged.
+ RegLogMode log_mode = (addrmode == PreIndex) ? LogRegWrites : NoRegLog;
+ set_xreg(addr_reg, address + offset, log_mode, Reg31IsStackPointer);
+ }
+
+ if ((addrmode == Offset) || (addrmode == PreIndex)) {
+ address += offset;
+ }
+
+ // Verify that the calculated address is available to the host.
+ VIXL_ASSERT(address == static_cast<uintptr_t>(address));
+
+ return static_cast<uintptr_t>(address);
+}
+
+
+void Simulator::VisitMoveWideImmediate(const Instruction* instr) {
+ MoveWideImmediateOp mov_op =
+ static_cast<MoveWideImmediateOp>(instr->Mask(MoveWideImmediateMask));
+ int64_t new_xn_val = 0;
+
+ bool is_64_bits = instr->SixtyFourBits() == 1;
+ // Shift is limited for W operations.
+ VIXL_ASSERT(is_64_bits || (instr->ShiftMoveWide() < 2));
+
+ // Get the shifted immediate.
+ int64_t shift = instr->ShiftMoveWide() * 16;
+ int64_t shifted_imm16 = static_cast<int64_t>(instr->ImmMoveWide()) << shift;
+
+ // Compute the new value.
+ switch (mov_op) {
+ case MOVN_w:
+ case MOVN_x: {
+ new_xn_val = ~shifted_imm16;
+ if (!is_64_bits) new_xn_val &= kWRegMask;
+ break;
+ }
+ case MOVK_w:
+ case MOVK_x: {
+ unsigned reg_code = instr->Rd();
+ int64_t prev_xn_val = is_64_bits ? xreg(reg_code)
+ : wreg(reg_code);
+ new_xn_val =
+ (prev_xn_val & ~(INT64_C(0xffff) << shift)) | shifted_imm16;
+ break;
+ }
+ case MOVZ_w:
+ case MOVZ_x: {
+ new_xn_val = shifted_imm16;
+ break;
+ }
+ default:
+ VIXL_UNREACHABLE();
+ }
+
+ // Update the destination register.
+ set_xreg(instr->Rd(), new_xn_val);
+}
+
+
+void Simulator::VisitConditionalSelect(const Instruction* instr) {
+ uint64_t new_val = xreg(instr->Rn());
+
+ if (ConditionFailed(static_cast<Condition>(instr->Condition()))) {
+ new_val = xreg(instr->Rm());
+ switch (instr->Mask(ConditionalSelectMask)) {
+ case CSEL_w:
+ case CSEL_x: break;
+ case CSINC_w:
+ case CSINC_x: new_val++; break;
+ case CSINV_w:
+ case CSINV_x: new_val = ~new_val; break;
+ case CSNEG_w:
+ case CSNEG_x: new_val = -new_val; break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ }
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ set_reg(reg_size, instr->Rd(), new_val);
+}
+
+
+void Simulator::VisitDataProcessing1Source(const Instruction* instr) {
+ unsigned dst = instr->Rd();
+ unsigned src = instr->Rn();
+
+ switch (instr->Mask(DataProcessing1SourceMask)) {
+ case RBIT_w: set_wreg(dst, ReverseBits(wreg(src))); break;
+ case RBIT_x: set_xreg(dst, ReverseBits(xreg(src))); break;
+ case REV16_w: set_wreg(dst, ReverseBytes(wreg(src), 1)); break;
+ case REV16_x: set_xreg(dst, ReverseBytes(xreg(src), 1)); break;
+ case REV_w: set_wreg(dst, ReverseBytes(wreg(src), 2)); break;
+ case REV32_x: set_xreg(dst, ReverseBytes(xreg(src), 2)); break;
+ case REV_x: set_xreg(dst, ReverseBytes(xreg(src), 3)); break;
+ case CLZ_w: set_wreg(dst, CountLeadingZeros(wreg(src))); break;
+ case CLZ_x: set_xreg(dst, CountLeadingZeros(xreg(src))); break;
+ case CLS_w: {
+ set_wreg(dst, CountLeadingSignBits(wreg(src)));
+ break;
+ }
+ case CLS_x: {
+ set_xreg(dst, CountLeadingSignBits(xreg(src)));
+ break;
+ }
+ default: VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+uint32_t Simulator::Poly32Mod2(unsigned n, uint64_t data, uint32_t poly) {
+ VIXL_ASSERT((n > 32) && (n <= 64));
+ for (unsigned i = (n - 1); i >= 32; i--) {
+ if (((data >> i) & 1) != 0) {
+ uint64_t polysh32 = (uint64_t)poly << (i - 32);
+ uint64_t mask = (UINT64_C(1) << i) - 1;
+ data = ((data & mask) ^ polysh32);
+ }
+ }
+ return data & 0xffffffff;
+}
+
+
+template <typename T>
+uint32_t Simulator::Crc32Checksum(uint32_t acc, T val, uint32_t poly) {
+ unsigned size = sizeof(val) * 8; // Number of bits in type T.
+ VIXL_ASSERT((size == 8) || (size == 16) || (size == 32));
+ uint64_t tempacc = static_cast<uint64_t>(ReverseBits(acc)) << size;
+ uint64_t tempval = static_cast<uint64_t>(ReverseBits(val)) << 32;
+ return ReverseBits(Poly32Mod2(32 + size, tempacc ^ tempval, poly));
+}
+
+
+uint32_t Simulator::Crc32Checksum(uint32_t acc, uint64_t val, uint32_t poly) {
+ // Poly32Mod2 cannot handle inputs with more than 32 bits, so compute
+ // the CRC of each 32-bit word sequentially.
+ acc = Crc32Checksum(acc, (uint32_t)(val & 0xffffffff), poly);
+ return Crc32Checksum(acc, (uint32_t)(val >> 32), poly);
+}
+
+
+void Simulator::VisitDataProcessing2Source(const Instruction* instr) {
+ Shift shift_op = NO_SHIFT;
+ int64_t result = 0;
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+
+ switch (instr->Mask(DataProcessing2SourceMask)) {
+ case SDIV_w: {
+ int32_t rn = wreg(instr->Rn());
+ int32_t rm = wreg(instr->Rm());
+ if ((rn == kWMinInt) && (rm == -1)) {
+ result = kWMinInt;
+ } else if (rm == 0) {
+ // Division by zero can be trapped, but not on A-class processors.
+ result = 0;
+ } else {
+ result = rn / rm;
+ }
+ break;
+ }
+ case SDIV_x: {
+ int64_t rn = xreg(instr->Rn());
+ int64_t rm = xreg(instr->Rm());
+ if ((rn == kXMinInt) && (rm == -1)) {
+ result = kXMinInt;
+ } else if (rm == 0) {
+ // Division by zero can be trapped, but not on A-class processors.
+ result = 0;
+ } else {
+ result = rn / rm;
+ }
+ break;
+ }
+ case UDIV_w: {
+ uint32_t rn = static_cast<uint32_t>(wreg(instr->Rn()));
+ uint32_t rm = static_cast<uint32_t>(wreg(instr->Rm()));
+ if (rm == 0) {
+ // Division by zero can be trapped, but not on A-class processors.
+ result = 0;
+ } else {
+ result = rn / rm;
+ }
+ break;
+ }
+ case UDIV_x: {
+ uint64_t rn = static_cast<uint64_t>(xreg(instr->Rn()));
+ uint64_t rm = static_cast<uint64_t>(xreg(instr->Rm()));
+ if (rm == 0) {
+ // Division by zero can be trapped, but not on A-class processors.
+ result = 0;
+ } else {
+ result = rn / rm;
+ }
+ break;
+ }
+ case LSLV_w:
+ case LSLV_x: shift_op = LSL; break;
+ case LSRV_w:
+ case LSRV_x: shift_op = LSR; break;
+ case ASRV_w:
+ case ASRV_x: shift_op = ASR; break;
+ case RORV_w:
+ case RORV_x: shift_op = ROR; break;
+ case CRC32B: {
+ uint32_t acc = reg<uint32_t>(instr->Rn());
+ uint8_t val = reg<uint8_t>(instr->Rm());
+ result = Crc32Checksum(acc, val, CRC32_POLY);
+ break;
+ }
+ case CRC32H: {
+ uint32_t acc = reg<uint32_t>(instr->Rn());
+ uint16_t val = reg<uint16_t>(instr->Rm());
+ result = Crc32Checksum(acc, val, CRC32_POLY);
+ break;
+ }
+ case CRC32W: {
+ uint32_t acc = reg<uint32_t>(instr->Rn());
+ uint32_t val = reg<uint32_t>(instr->Rm());
+ result = Crc32Checksum(acc, val, CRC32_POLY);
+ break;
+ }
+ case CRC32X: {
+ uint32_t acc = reg<uint32_t>(instr->Rn());
+ uint64_t val = reg<uint64_t>(instr->Rm());
+ result = Crc32Checksum(acc, val, CRC32_POLY);
+ reg_size = kWRegSize;
+ break;
+ }
+ case CRC32CB: {
+ uint32_t acc = reg<uint32_t>(instr->Rn());
+ uint8_t val = reg<uint8_t>(instr->Rm());
+ result = Crc32Checksum(acc, val, CRC32C_POLY);
+ break;
+ }
+ case CRC32CH: {
+ uint32_t acc = reg<uint32_t>(instr->Rn());
+ uint16_t val = reg<uint16_t>(instr->Rm());
+ result = Crc32Checksum(acc, val, CRC32C_POLY);
+ break;
+ }
+ case CRC32CW: {
+ uint32_t acc = reg<uint32_t>(instr->Rn());
+ uint32_t val = reg<uint32_t>(instr->Rm());
+ result = Crc32Checksum(acc, val, CRC32C_POLY);
+ break;
+ }
+ case CRC32CX: {
+ uint32_t acc = reg<uint32_t>(instr->Rn());
+ uint64_t val = reg<uint64_t>(instr->Rm());
+ result = Crc32Checksum(acc, val, CRC32C_POLY);
+ reg_size = kWRegSize;
+ break;
+ }
+ default: VIXL_UNIMPLEMENTED();
+ }
+
+ if (shift_op != NO_SHIFT) {
+ // Shift distance encoded in the least-significant five/six bits of the
+ // register.
+ int mask = (instr->SixtyFourBits() == 1) ? 0x3f : 0x1f;
+ unsigned shift = wreg(instr->Rm()) & mask;
+ result = ShiftOperand(reg_size, reg(reg_size, instr->Rn()), shift_op,
+ shift);
+ }
+ set_reg(reg_size, instr->Rd(), result);
+}
+
+
+// The algorithm used is adapted from the one described in section 8.2 of
+// Hacker's Delight, by Henry S. Warren, Jr.
+// It assumes that a right shift on a signed integer is an arithmetic shift.
+// Type T must be either uint64_t or int64_t.
+template <typename T>
+static T MultiplyHigh(T u, T v) {
+ uint64_t u0, v0, w0;
+ T u1, v1, w1, w2, t;
+
+ VIXL_ASSERT(sizeof(u) == sizeof(u0));
+
+ u0 = u & 0xffffffff;
+ u1 = u >> 32;
+ v0 = v & 0xffffffff;
+ v1 = v >> 32;
+
+ w0 = u0 * v0;
+ t = u1 * v0 + (w0 >> 32);
+ w1 = t & 0xffffffff;
+ w2 = t >> 32;
+ w1 = u0 * v1 + w1;
+
+ return u1 * v1 + w2 + (w1 >> 32);
+}
+
+
+void Simulator::VisitDataProcessing3Source(const Instruction* instr) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+
+ int64_t result = 0;
+ // Extract and sign- or zero-extend 32-bit arguments for widening operations.
+ uint64_t rn_u32 = reg<uint32_t>(instr->Rn());
+ uint64_t rm_u32 = reg<uint32_t>(instr->Rm());
+ int64_t rn_s32 = reg<int32_t>(instr->Rn());
+ int64_t rm_s32 = reg<int32_t>(instr->Rm());
+ switch (instr->Mask(DataProcessing3SourceMask)) {
+ case MADD_w:
+ case MADD_x:
+ result = xreg(instr->Ra()) + (xreg(instr->Rn()) * xreg(instr->Rm()));
+ break;
+ case MSUB_w:
+ case MSUB_x:
+ result = xreg(instr->Ra()) - (xreg(instr->Rn()) * xreg(instr->Rm()));
+ break;
+ case SMADDL_x: result = xreg(instr->Ra()) + (rn_s32 * rm_s32); break;
+ case SMSUBL_x: result = xreg(instr->Ra()) - (rn_s32 * rm_s32); break;
+ case UMADDL_x: result = xreg(instr->Ra()) + (rn_u32 * rm_u32); break;
+ case UMSUBL_x: result = xreg(instr->Ra()) - (rn_u32 * rm_u32); break;
+ case UMULH_x:
+ result = MultiplyHigh(reg<uint64_t>(instr->Rn()),
+ reg<uint64_t>(instr->Rm()));
+ break;
+ case SMULH_x:
+ result = MultiplyHigh(xreg(instr->Rn()), xreg(instr->Rm()));
+ break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ set_reg(reg_size, instr->Rd(), result);
+}
+
+
+void Simulator::VisitBitfield(const Instruction* instr) {
+ unsigned reg_size = instr->SixtyFourBits() ? kXRegSize : kWRegSize;
+ int64_t reg_mask = instr->SixtyFourBits() ? kXRegMask : kWRegMask;
+ int64_t R = instr->ImmR();
+ int64_t S = instr->ImmS();
+ int64_t diff = S - R;
+ int64_t mask;
+ if (diff >= 0) {
+ mask = (diff < (reg_size - 1)) ? (INT64_C(1) << (diff + 1)) - 1
+ : reg_mask;
+ } else {
+ mask = (INT64_C(1) << (S + 1)) - 1;
+ mask = (static_cast<uint64_t>(mask) >> R) | (mask << (reg_size - R));
+ diff += reg_size;
+ }
+
+ // inzero indicates if the extracted bitfield is inserted into the
+ // destination register value or in zero.
+ // If extend is true, extend the sign of the extracted bitfield.
+ bool inzero = false;
+ bool extend = false;
+ switch (instr->Mask(BitfieldMask)) {
+ case BFM_x:
+ case BFM_w:
+ break;
+ case SBFM_x:
+ case SBFM_w:
+ inzero = true;
+ extend = true;
+ break;
+ case UBFM_x:
+ case UBFM_w:
+ inzero = true;
+ break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+
+ int64_t dst = inzero ? 0 : reg(reg_size, instr->Rd());
+ int64_t src = reg(reg_size, instr->Rn());
+ // Rotate source bitfield into place.
+ int64_t result = (static_cast<uint64_t>(src) >> R) | (src << (reg_size - R));
+ // Determine the sign extension.
+ int64_t topbits = ((INT64_C(1) << (reg_size - diff - 1)) - 1) << (diff + 1);
+ int64_t signbits = extend && ((src >> S) & 1) ? topbits : 0;
+
+ // Merge sign extension, dest/zero and bitfield.
+ result = signbits | (result & mask) | (dst & ~mask);
+
+ set_reg(reg_size, instr->Rd(), result);
+}
+
+
+void Simulator::VisitExtract(const Instruction* instr) {
+ unsigned lsb = instr->ImmS();
+ unsigned reg_size = (instr->SixtyFourBits() == 1) ? kXRegSize
+ : kWRegSize;
+ uint64_t low_res = static_cast<uint64_t>(reg(reg_size, instr->Rm())) >> lsb;
+ uint64_t high_res =
+ (lsb == 0) ? 0 : reg(reg_size, instr->Rn()) << (reg_size - lsb);
+ set_reg(reg_size, instr->Rd(), low_res | high_res);
+}
+
+
+void Simulator::VisitFPImmediate(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ unsigned dest = instr->Rd();
+ switch (instr->Mask(FPImmediateMask)) {
+ case FMOV_s_imm: set_sreg(dest, instr->ImmFP32()); break;
+ case FMOV_d_imm: set_dreg(dest, instr->ImmFP64()); break;
+ default: VIXL_UNREACHABLE();
+ }
+}
+
+
+void Simulator::VisitFPIntegerConvert(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ unsigned dst = instr->Rd();
+ unsigned src = instr->Rn();
+
+ FPRounding round = RMode();
+
+ switch (instr->Mask(FPIntegerConvertMask)) {
+ case FCVTAS_ws: set_wreg(dst, FPToInt32(sreg(src), FPTieAway)); break;
+ case FCVTAS_xs: set_xreg(dst, FPToInt64(sreg(src), FPTieAway)); break;
+ case FCVTAS_wd: set_wreg(dst, FPToInt32(dreg(src), FPTieAway)); break;
+ case FCVTAS_xd: set_xreg(dst, FPToInt64(dreg(src), FPTieAway)); break;
+ case FCVTAU_ws: set_wreg(dst, FPToUInt32(sreg(src), FPTieAway)); break;
+ case FCVTAU_xs: set_xreg(dst, FPToUInt64(sreg(src), FPTieAway)); break;
+ case FCVTAU_wd: set_wreg(dst, FPToUInt32(dreg(src), FPTieAway)); break;
+ case FCVTAU_xd: set_xreg(dst, FPToUInt64(dreg(src), FPTieAway)); break;
+ case FCVTMS_ws:
+ set_wreg(dst, FPToInt32(sreg(src), FPNegativeInfinity));
+ break;
+ case FCVTMS_xs:
+ set_xreg(dst, FPToInt64(sreg(src), FPNegativeInfinity));
+ break;
+ case FCVTMS_wd:
+ set_wreg(dst, FPToInt32(dreg(src), FPNegativeInfinity));
+ break;
+ case FCVTMS_xd:
+ set_xreg(dst, FPToInt64(dreg(src), FPNegativeInfinity));
+ break;
+ case FCVTMU_ws:
+ set_wreg(dst, FPToUInt32(sreg(src), FPNegativeInfinity));
+ break;
+ case FCVTMU_xs:
+ set_xreg(dst, FPToUInt64(sreg(src), FPNegativeInfinity));
+ break;
+ case FCVTMU_wd:
+ set_wreg(dst, FPToUInt32(dreg(src), FPNegativeInfinity));
+ break;
+ case FCVTMU_xd:
+ set_xreg(dst, FPToUInt64(dreg(src), FPNegativeInfinity));
+ break;
+ case FCVTPS_ws:
+ set_wreg(dst, FPToInt32(sreg(src), FPPositiveInfinity));
+ break;
+ case FCVTPS_xs:
+ set_xreg(dst, FPToInt64(sreg(src), FPPositiveInfinity));
+ break;
+ case FCVTPS_wd:
+ set_wreg(dst, FPToInt32(dreg(src), FPPositiveInfinity));
+ break;
+ case FCVTPS_xd:
+ set_xreg(dst, FPToInt64(dreg(src), FPPositiveInfinity));
+ break;
+ case FCVTPU_ws:
+ set_wreg(dst, FPToUInt32(sreg(src), FPPositiveInfinity));
+ break;
+ case FCVTPU_xs:
+ set_xreg(dst, FPToUInt64(sreg(src), FPPositiveInfinity));
+ break;
+ case FCVTPU_wd:
+ set_wreg(dst, FPToUInt32(dreg(src), FPPositiveInfinity));
+ break;
+ case FCVTPU_xd:
+ set_xreg(dst, FPToUInt64(dreg(src), FPPositiveInfinity));
+ break;
+ case FCVTNS_ws: set_wreg(dst, FPToInt32(sreg(src), FPTieEven)); break;
+ case FCVTNS_xs: set_xreg(dst, FPToInt64(sreg(src), FPTieEven)); break;
+ case FCVTNS_wd: set_wreg(dst, FPToInt32(dreg(src), FPTieEven)); break;
+ case FCVTNS_xd: set_xreg(dst, FPToInt64(dreg(src), FPTieEven)); break;
+ case FCVTNU_ws: set_wreg(dst, FPToUInt32(sreg(src), FPTieEven)); break;
+ case FCVTNU_xs: set_xreg(dst, FPToUInt64(sreg(src), FPTieEven)); break;
+ case FCVTNU_wd: set_wreg(dst, FPToUInt32(dreg(src), FPTieEven)); break;
+ case FCVTNU_xd: set_xreg(dst, FPToUInt64(dreg(src), FPTieEven)); break;
+ case FCVTZS_ws: set_wreg(dst, FPToInt32(sreg(src), FPZero)); break;
+ case FCVTZS_xs: set_xreg(dst, FPToInt64(sreg(src), FPZero)); break;
+ case FCVTZS_wd: set_wreg(dst, FPToInt32(dreg(src), FPZero)); break;
+ case FCVTZS_xd: set_xreg(dst, FPToInt64(dreg(src), FPZero)); break;
+ case FCVTZU_ws: set_wreg(dst, FPToUInt32(sreg(src), FPZero)); break;
+ case FCVTZU_xs: set_xreg(dst, FPToUInt64(sreg(src), FPZero)); break;
+ case FCVTZU_wd: set_wreg(dst, FPToUInt32(dreg(src), FPZero)); break;
+ case FCVTZU_xd: set_xreg(dst, FPToUInt64(dreg(src), FPZero)); break;
+ case FJCVTZS: set_wreg(dst, FPToFixedJS(dreg(src))); break;
+ case FMOV_ws: set_wreg(dst, sreg_bits(src)); break;
+ case FMOV_xd: set_xreg(dst, dreg_bits(src)); break;
+ case FMOV_sw: set_sreg_bits(dst, wreg(src)); break;
+ case FMOV_dx: set_dreg_bits(dst, xreg(src)); break;
+ case FMOV_d1_x:
+ LogicVRegister(vreg(dst)).SetUint(kFormatD, 1, xreg(src));
+ break;
+ case FMOV_x_d1:
+ set_xreg(dst, LogicVRegister(vreg(src)).Uint(kFormatD, 1));
+ break;
+
+ // A 32-bit input can be handled in the same way as a 64-bit input, since
+ // the sign- or zero-extension will not affect the conversion.
+ case SCVTF_dx: set_dreg(dst, FixedToDouble(xreg(src), 0, round)); break;
+ case SCVTF_dw: set_dreg(dst, FixedToDouble(wreg(src), 0, round)); break;
+ case UCVTF_dx: set_dreg(dst, UFixedToDouble(xreg(src), 0, round)); break;
+ case UCVTF_dw: {
+ set_dreg(dst, UFixedToDouble(static_cast<uint32_t>(wreg(src)), 0, round));
+ break;
+ }
+ case SCVTF_sx: set_sreg(dst, FixedToFloat(xreg(src), 0, round)); break;
+ case SCVTF_sw: set_sreg(dst, FixedToFloat(wreg(src), 0, round)); break;
+ case UCVTF_sx: set_sreg(dst, UFixedToFloat(xreg(src), 0, round)); break;
+ case UCVTF_sw: {
+ set_sreg(dst, UFixedToFloat(static_cast<uint32_t>(wreg(src)), 0, round));
+ break;
+ }
+
+ default: VIXL_UNREACHABLE();
+ }
+}
+
+
+void Simulator::VisitFPFixedPointConvert(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ unsigned dst = instr->Rd();
+ unsigned src = instr->Rn();
+ int fbits = 64 - instr->FPScale();
+
+ FPRounding round = RMode();
+
+ switch (instr->Mask(FPFixedPointConvertMask)) {
+ // A 32-bit input can be handled in the same way as a 64-bit input, since
+ // the sign- or zero-extension will not affect the conversion.
+ case SCVTF_dx_fixed:
+ set_dreg(dst, FixedToDouble(xreg(src), fbits, round));
+ break;
+ case SCVTF_dw_fixed:
+ set_dreg(dst, FixedToDouble(wreg(src), fbits, round));
+ break;
+ case UCVTF_dx_fixed:
+ set_dreg(dst, UFixedToDouble(xreg(src), fbits, round));
+ break;
+ case UCVTF_dw_fixed: {
+ set_dreg(dst,
+ UFixedToDouble(static_cast<uint32_t>(wreg(src)), fbits, round));
+ break;
+ }
+ case SCVTF_sx_fixed:
+ set_sreg(dst, FixedToFloat(xreg(src), fbits, round));
+ break;
+ case SCVTF_sw_fixed:
+ set_sreg(dst, FixedToFloat(wreg(src), fbits, round));
+ break;
+ case UCVTF_sx_fixed:
+ set_sreg(dst, UFixedToFloat(xreg(src), fbits, round));
+ break;
+ case UCVTF_sw_fixed: {
+ set_sreg(dst,
+ UFixedToFloat(static_cast<uint32_t>(wreg(src)), fbits, round));
+ break;
+ }
+ case FCVTZS_xd_fixed:
+ set_xreg(dst, FPToInt64(dreg(src) * std::pow(2.0, fbits), FPZero));
+ break;
+ case FCVTZS_wd_fixed:
+ set_wreg(dst, FPToInt32(dreg(src) * std::pow(2.0, fbits), FPZero));
+ break;
+ case FCVTZU_xd_fixed:
+ set_xreg(dst, FPToUInt64(dreg(src) * std::pow(2.0, fbits), FPZero));
+ break;
+ case FCVTZU_wd_fixed:
+ set_wreg(dst, FPToUInt32(dreg(src) * std::pow(2.0, fbits), FPZero));
+ break;
+ case FCVTZS_xs_fixed:
+ set_xreg(dst, FPToInt64(sreg(src) * std::pow(2.0f, fbits), FPZero));
+ break;
+ case FCVTZS_ws_fixed:
+ set_wreg(dst, FPToInt32(sreg(src) * std::pow(2.0f, fbits), FPZero));
+ break;
+ case FCVTZU_xs_fixed:
+ set_xreg(dst, FPToUInt64(sreg(src) * std::pow(2.0f, fbits), FPZero));
+ break;
+ case FCVTZU_ws_fixed:
+ set_wreg(dst, FPToUInt32(sreg(src) * std::pow(2.0f, fbits), FPZero));
+ break;
+ default: VIXL_UNREACHABLE();
+ }
+}
+
+
+void Simulator::VisitFPCompare(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ FPTrapFlags trap = DisableTrap;
+ switch (instr->Mask(FPCompareMask)) {
+ case FCMPE_s: trap = EnableTrap; VIXL_FALLTHROUGH();
+ case FCMP_s: FPCompare(sreg(instr->Rn()), sreg(instr->Rm()), trap); break;
+ case FCMPE_d: trap = EnableTrap; VIXL_FALLTHROUGH();
+ case FCMP_d: FPCompare(dreg(instr->Rn()), dreg(instr->Rm()), trap); break;
+ case FCMPE_s_zero: trap = EnableTrap; VIXL_FALLTHROUGH();
+ case FCMP_s_zero: FPCompare(sreg(instr->Rn()), 0.0f, trap); break;
+ case FCMPE_d_zero: trap = EnableTrap; VIXL_FALLTHROUGH();
+ case FCMP_d_zero: FPCompare(dreg(instr->Rn()), 0.0, trap); break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitFPConditionalCompare(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ FPTrapFlags trap = DisableTrap;
+ switch (instr->Mask(FPConditionalCompareMask)) {
+ case FCCMPE_s: trap = EnableTrap;
+ VIXL_FALLTHROUGH();
+ case FCCMP_s:
+ if (ConditionPassed(instr->Condition())) {
+ FPCompare(sreg(instr->Rn()), sreg(instr->Rm()), trap);
+ } else {
+ nzcv().SetFlags(instr->Nzcv());
+ LogSystemRegister(NZCV);
+ }
+ break;
+ case FCCMPE_d: trap = EnableTrap;
+ VIXL_FALLTHROUGH();
+ case FCCMP_d:
+ if (ConditionPassed(instr->Condition())) {
+ FPCompare(dreg(instr->Rn()), dreg(instr->Rm()), trap);
+ } else {
+ nzcv().SetFlags(instr->Nzcv());
+ LogSystemRegister(NZCV);
+ }
+ break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitFPConditionalSelect(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ Instr selected;
+ if (ConditionPassed(instr->Condition())) {
+ selected = instr->Rn();
+ } else {
+ selected = instr->Rm();
+ }
+
+ switch (instr->Mask(FPConditionalSelectMask)) {
+ case FCSEL_s: set_sreg(instr->Rd(), sreg(selected)); break;
+ case FCSEL_d: set_dreg(instr->Rd(), dreg(selected)); break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitFPDataProcessing1Source(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ FPRounding fpcr_rounding = static_cast<FPRounding>(fpcr().RMode());
+ VectorFormat vform = (instr->Mask(FP64) == FP64) ? kFormatD : kFormatS;
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ bool inexact_exception = false;
+
+ unsigned fd = instr->Rd();
+ unsigned fn = instr->Rn();
+
+ switch (instr->Mask(FPDataProcessing1SourceMask)) {
+ case FMOV_s: set_sreg(fd, sreg(fn)); return;
+ case FMOV_d: set_dreg(fd, dreg(fn)); return;
+ case FABS_s: fabs_(kFormatS, vreg(fd), vreg(fn)); return;
+ case FABS_d: fabs_(kFormatD, vreg(fd), vreg(fn)); return;
+ case FNEG_s: fneg(kFormatS, vreg(fd), vreg(fn)); return;
+ case FNEG_d: fneg(kFormatD, vreg(fd), vreg(fn)); return;
+ case FCVT_ds:
+ set_dreg(fd, FPToDouble(sreg(fn), ReadDN()));
+ return;
+ case FCVT_sd:
+ set_sreg(fd, FPToFloat(dreg(fn), FPTieEven, ReadDN()));
+ return;
+ case FCVT_hs:
+ set_hreg(fd, Float16ToRawbits(FPToFloat16(sreg(fn), FPTieEven, ReadDN())));
+ return;
+ case FCVT_sh:
+ set_sreg(fd, FPToFloat(RawbitsToFloat16(hreg(fn)), ReadDN()));
+ return;
+ case FCVT_dh:
+ set_dreg(fd, FPToDouble(hreg(fn), ReadDN()));
+ return;
+ case FCVT_hd:
+ set_hreg(fd, Float16ToRawbits(FPToFloat16(dreg(fn), FPTieEven, ReadDN())));
+ return;
+ case FSQRT_s:
+ case FSQRT_d: fsqrt(vform, rd, rn); return;
+ case FRINTI_s:
+ case FRINTI_d: break; // Use FPCR rounding mode.
+ case FRINTX_s:
+ case FRINTX_d: inexact_exception = true; break;
+ case FRINTA_s:
+ case FRINTA_d: fpcr_rounding = FPTieAway; break;
+ case FRINTM_s:
+ case FRINTM_d: fpcr_rounding = FPNegativeInfinity; break;
+ case FRINTN_s:
+ case FRINTN_d: fpcr_rounding = FPTieEven; break;
+ case FRINTP_s:
+ case FRINTP_d: fpcr_rounding = FPPositiveInfinity; break;
+ case FRINTZ_s:
+ case FRINTZ_d: fpcr_rounding = FPZero; break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+
+ // Only FRINT* instructions fall through the switch above.
+ frint(vform, rd, rn, fpcr_rounding, inexact_exception);
+}
+
+
+void Simulator::VisitFPDataProcessing2Source(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ VectorFormat vform = (instr->Mask(FP64) == FP64) ? kFormatD : kFormatS;
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ SimVRegister& rm = vreg(instr->Rm());
+
+ switch (instr->Mask(FPDataProcessing2SourceMask)) {
+ case FADD_s:
+ case FADD_d: fadd(vform, rd, rn, rm); break;
+ case FSUB_s:
+ case FSUB_d: fsub(vform, rd, rn, rm); break;
+ case FMUL_s:
+ case FMUL_d: fmul(vform, rd, rn, rm); break;
+ case FNMUL_s:
+ case FNMUL_d: fnmul(vform, rd, rn, rm); break;
+ case FDIV_s:
+ case FDIV_d: fdiv(vform, rd, rn, rm); break;
+ case FMAX_s:
+ case FMAX_d: fmax(vform, rd, rn, rm); break;
+ case FMIN_s:
+ case FMIN_d: fmin(vform, rd, rn, rm); break;
+ case FMAXNM_s:
+ case FMAXNM_d: fmaxnm(vform, rd, rn, rm); break;
+ case FMINNM_s:
+ case FMINNM_d: fminnm(vform, rd, rn, rm); break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+}
+
+
+void Simulator::VisitFPDataProcessing3Source(const Instruction* instr) {
+ AssertSupportedFPCR();
+
+ unsigned fd = instr->Rd();
+ unsigned fn = instr->Rn();
+ unsigned fm = instr->Rm();
+ unsigned fa = instr->Ra();
+
+ switch (instr->Mask(FPDataProcessing3SourceMask)) {
+ // fd = fa +/- (fn * fm)
+ case FMADD_s: set_sreg(fd, FPMulAdd(sreg(fa), sreg(fn), sreg(fm))); break;
+ case FMSUB_s: set_sreg(fd, FPMulAdd(sreg(fa), -sreg(fn), sreg(fm))); break;
+ case FMADD_d: set_dreg(fd, FPMulAdd(dreg(fa), dreg(fn), dreg(fm))); break;
+ case FMSUB_d: set_dreg(fd, FPMulAdd(dreg(fa), -dreg(fn), dreg(fm))); break;
+ // Negated variants of the above.
+ case FNMADD_s:
+ set_sreg(fd, FPMulAdd(-sreg(fa), -sreg(fn), sreg(fm)));
+ break;
+ case FNMSUB_s:
+ set_sreg(fd, FPMulAdd(-sreg(fa), sreg(fn), sreg(fm)));
+ break;
+ case FNMADD_d:
+ set_dreg(fd, FPMulAdd(-dreg(fa), -dreg(fn), dreg(fm)));
+ break;
+ case FNMSUB_d:
+ set_dreg(fd, FPMulAdd(-dreg(fa), dreg(fn), dreg(fm)));
+ break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+bool Simulator::FPProcessNaNs(const Instruction* instr) {
+ unsigned fd = instr->Rd();
+ unsigned fn = instr->Rn();
+ unsigned fm = instr->Rm();
+ bool done = false;
+
+ if (instr->Mask(FP64) == FP64) {
+ double result = FPProcessNaNs(dreg(fn), dreg(fm));
+ if (std::isnan(result)) {
+ set_dreg(fd, result);
+ done = true;
+ }
+ } else {
+ float result = FPProcessNaNs(sreg(fn), sreg(fm));
+ if (std::isnan(result)) {
+ set_sreg(fd, result);
+ done = true;
+ }
+ }
+
+ return done;
+}
+
+
+void Simulator::SysOp_W(int op, int64_t val) {
+ switch (op) {
+ case IVAU:
+ case CVAC:
+ case CVAU:
+ case CIVAC: {
+ // Perform a dummy memory access to ensure that we have read access
+ // to the specified address.
+ volatile uint8_t y = Read<uint8_t>(val);
+ USE(y);
+ // TODO: Implement "case ZVA:".
+ break;
+ }
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitSystem(const Instruction* instr) {
+ // Some system instructions hijack their Op and Cp fields to represent a
+ // range of immediates instead of indicating a different instruction. This
+ // makes the decoding tricky.
+ if (instr->Mask(SystemExclusiveMonitorFMask) == SystemExclusiveMonitorFixed) {
+ VIXL_ASSERT(instr->Mask(SystemExclusiveMonitorMask) == CLREX);
+ switch (instr->Mask(SystemExclusiveMonitorMask)) {
+ case CLREX: {
+ PrintExclusiveAccessWarning();
+ ClearLocalMonitor();
+ break;
+ }
+ }
+ } else if (instr->Mask(SystemSysRegFMask) == SystemSysRegFixed) {
+ switch (instr->Mask(SystemSysRegMask)) {
+ case MRS: {
+ switch (instr->ImmSystemRegister()) {
+ case NZCV: set_xreg(instr->Rt(), nzcv().RawValue()); break;
+ case FPCR: set_xreg(instr->Rt(), fpcr().RawValue()); break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ break;
+ }
+ case MSR: {
+ switch (instr->ImmSystemRegister()) {
+ case NZCV:
+ nzcv().SetRawValue(wreg(instr->Rt()));
+ LogSystemRegister(NZCV);
+ break;
+ case FPCR:
+ fpcr().SetRawValue(wreg(instr->Rt()));
+ LogSystemRegister(FPCR);
+ break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ break;
+ }
+ }
+ } else if (instr->Mask(SystemHintFMask) == SystemHintFixed) {
+ VIXL_ASSERT(instr->Mask(SystemHintMask) == HINT);
+ switch (instr->ImmHint()) {
+ case NOP: break;
+ case CSDB: break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ } else if (instr->Mask(MemBarrierFMask) == MemBarrierFixed) {
+ js::jit::AtomicOperations::fenceSeqCst();
+ } else if ((instr->Mask(SystemSysFMask) == SystemSysFixed)) {
+ switch (instr->Mask(SystemSysMask)) {
+ case SYS: SysOp_W(instr->SysOp(), xreg(instr->Rt())); break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ } else {
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitCrypto2RegSHA(const Instruction* instr) {
+ VisitUnimplemented(instr);
+}
+
+
+void Simulator::VisitCrypto3RegSHA(const Instruction* instr) {
+ VisitUnimplemented(instr);
+}
+
+
+void Simulator::VisitCryptoAES(const Instruction* instr) {
+ VisitUnimplemented(instr);
+}
+
+
+void Simulator::VisitNEON2RegMisc(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr);
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ static const NEONFormatMap map_lp = {
+ {23, 22, 30}, {NF_4H, NF_8H, NF_2S, NF_4S, NF_1D, NF_2D}
+ };
+ VectorFormat vf_lp = nfd.GetVectorFormat(&map_lp);
+
+ static const NEONFormatMap map_fcvtl = {
+ {22}, {NF_4S, NF_2D}
+ };
+ VectorFormat vf_fcvtl = nfd.GetVectorFormat(&map_fcvtl);
+
+ static const NEONFormatMap map_fcvtn = {
+ {22, 30}, {NF_4H, NF_8H, NF_2S, NF_4S}
+ };
+ VectorFormat vf_fcvtn = nfd.GetVectorFormat(&map_fcvtn);
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+
+ if (instr->Mask(NEON2RegMiscOpcode) <= NEON_NEG_opcode) {
+ // These instructions all use a two bit size field, except NOT and RBIT,
+ // which use the field to encode the operation.
+ switch (instr->Mask(NEON2RegMiscMask)) {
+ case NEON_REV64: rev64(vf, rd, rn); break;
+ case NEON_REV32: rev32(vf, rd, rn); break;
+ case NEON_REV16: rev16(vf, rd, rn); break;
+ case NEON_SUQADD: suqadd(vf, rd, rn); break;
+ case NEON_USQADD: usqadd(vf, rd, rn); break;
+ case NEON_CLS: cls(vf, rd, rn); break;
+ case NEON_CLZ: clz(vf, rd, rn); break;
+ case NEON_CNT: cnt(vf, rd, rn); break;
+ case NEON_SQABS: abs(vf, rd, rn).SignedSaturate(vf); break;
+ case NEON_SQNEG: neg(vf, rd, rn).SignedSaturate(vf); break;
+ case NEON_CMGT_zero: cmp(vf, rd, rn, 0, gt); break;
+ case NEON_CMGE_zero: cmp(vf, rd, rn, 0, ge); break;
+ case NEON_CMEQ_zero: cmp(vf, rd, rn, 0, eq); break;
+ case NEON_CMLE_zero: cmp(vf, rd, rn, 0, le); break;
+ case NEON_CMLT_zero: cmp(vf, rd, rn, 0, lt); break;
+ case NEON_ABS: abs(vf, rd, rn); break;
+ case NEON_NEG: neg(vf, rd, rn); break;
+ case NEON_SADDLP: saddlp(vf_lp, rd, rn); break;
+ case NEON_UADDLP: uaddlp(vf_lp, rd, rn); break;
+ case NEON_SADALP: sadalp(vf_lp, rd, rn); break;
+ case NEON_UADALP: uadalp(vf_lp, rd, rn); break;
+ case NEON_RBIT_NOT:
+ vf = nfd.GetVectorFormat(nfd.LogicalFormatMap());
+ switch (instr->FPType()) {
+ case 0: not_(vf, rd, rn); break;
+ case 1: rbit(vf, rd, rn);; break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ break;
+ }
+ } else {
+ VectorFormat fpf = nfd.GetVectorFormat(nfd.FPFormatMap());
+ FPRounding fpcr_rounding = static_cast<FPRounding>(fpcr().RMode());
+ bool inexact_exception = false;
+
+ // These instructions all use a one bit size field, except XTN, SQXTUN,
+ // SHLL, SQXTN and UQXTN, which use a two bit size field.
+ switch (instr->Mask(NEON2RegMiscFPMask)) {
+ case NEON_FABS: fabs_(fpf, rd, rn); return;
+ case NEON_FNEG: fneg(fpf, rd, rn); return;
+ case NEON_FSQRT: fsqrt(fpf, rd, rn); return;
+ case NEON_FCVTL:
+ if (instr->Mask(NEON_Q)) {
+ fcvtl2(vf_fcvtl, rd, rn);
+ } else {
+ fcvtl(vf_fcvtl, rd, rn);
+ }
+ return;
+ case NEON_FCVTN:
+ if (instr->Mask(NEON_Q)) {
+ fcvtn2(vf_fcvtn, rd, rn);
+ } else {
+ fcvtn(vf_fcvtn, rd, rn);
+ }
+ return;
+ case NEON_FCVTXN:
+ if (instr->Mask(NEON_Q)) {
+ fcvtxn2(vf_fcvtn, rd, rn);
+ } else {
+ fcvtxn(vf_fcvtn, rd, rn);
+ }
+ return;
+
+ // The following instructions break from the switch statement, rather
+ // than return.
+ case NEON_FRINTI: break; // Use FPCR rounding mode.
+ case NEON_FRINTX: inexact_exception = true; break;
+ case NEON_FRINTA: fpcr_rounding = FPTieAway; break;
+ case NEON_FRINTM: fpcr_rounding = FPNegativeInfinity; break;
+ case NEON_FRINTN: fpcr_rounding = FPTieEven; break;
+ case NEON_FRINTP: fpcr_rounding = FPPositiveInfinity; break;
+ case NEON_FRINTZ: fpcr_rounding = FPZero; break;
+
+ case NEON_FCVTNS: fcvts(fpf, rd, rn, FPTieEven); return;
+ case NEON_FCVTNU: fcvtu(fpf, rd, rn, FPTieEven); return;
+ case NEON_FCVTPS: fcvts(fpf, rd, rn, FPPositiveInfinity); return;
+ case NEON_FCVTPU: fcvtu(fpf, rd, rn, FPPositiveInfinity); return;
+ case NEON_FCVTMS: fcvts(fpf, rd, rn, FPNegativeInfinity); return;
+ case NEON_FCVTMU: fcvtu(fpf, rd, rn, FPNegativeInfinity); return;
+ case NEON_FCVTZS: fcvts(fpf, rd, rn, FPZero); return;
+ case NEON_FCVTZU: fcvtu(fpf, rd, rn, FPZero); return;
+ case NEON_FCVTAS: fcvts(fpf, rd, rn, FPTieAway); return;
+ case NEON_FCVTAU: fcvtu(fpf, rd, rn, FPTieAway); return;
+ case NEON_SCVTF: scvtf(fpf, rd, rn, 0, fpcr_rounding); return;
+ case NEON_UCVTF: ucvtf(fpf, rd, rn, 0, fpcr_rounding); return;
+ case NEON_URSQRTE: ursqrte(fpf, rd, rn); return;
+ case NEON_URECPE: urecpe(fpf, rd, rn); return;
+ case NEON_FRSQRTE: frsqrte(fpf, rd, rn); return;
+ case NEON_FRECPE: frecpe(fpf, rd, rn, fpcr_rounding); return;
+ case NEON_FCMGT_zero: fcmp_zero(fpf, rd, rn, gt); return;
+ case NEON_FCMGE_zero: fcmp_zero(fpf, rd, rn, ge); return;
+ case NEON_FCMEQ_zero: fcmp_zero(fpf, rd, rn, eq); return;
+ case NEON_FCMLE_zero: fcmp_zero(fpf, rd, rn, le); return;
+ case NEON_FCMLT_zero: fcmp_zero(fpf, rd, rn, lt); return;
+ default:
+ if ((NEON_XTN_opcode <= instr->Mask(NEON2RegMiscOpcode)) &&
+ (instr->Mask(NEON2RegMiscOpcode) <= NEON_UQXTN_opcode)) {
+ switch (instr->Mask(NEON2RegMiscMask)) {
+ case NEON_XTN: xtn(vf, rd, rn); return;
+ case NEON_SQXTN: sqxtn(vf, rd, rn); return;
+ case NEON_UQXTN: uqxtn(vf, rd, rn); return;
+ case NEON_SQXTUN: sqxtun(vf, rd, rn); return;
+ case NEON_SHLL:
+ vf = nfd.GetVectorFormat(nfd.LongIntegerFormatMap());
+ if (instr->Mask(NEON_Q)) {
+ shll2(vf, rd, rn);
+ } else {
+ shll(vf, rd, rn);
+ }
+ return;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ } else {
+ VIXL_UNIMPLEMENTED();
+ }
+ }
+
+ // Only FRINT* instructions fall through the switch above.
+ frint(fpf, rd, rn, fpcr_rounding, inexact_exception);
+ }
+}
+
+
+void Simulator::VisitNEON3Same(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr);
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ SimVRegister& rm = vreg(instr->Rm());
+
+ if (instr->Mask(NEON3SameLogicalFMask) == NEON3SameLogicalFixed) {
+ VectorFormat vf = nfd.GetVectorFormat(nfd.LogicalFormatMap());
+ switch (instr->Mask(NEON3SameLogicalMask)) {
+ case NEON_AND: and_(vf, rd, rn, rm); break;
+ case NEON_ORR: orr(vf, rd, rn, rm); break;
+ case NEON_ORN: orn(vf, rd, rn, rm); break;
+ case NEON_EOR: eor(vf, rd, rn, rm); break;
+ case NEON_BIC: bic(vf, rd, rn, rm); break;
+ case NEON_BIF: bif(vf, rd, rn, rm); break;
+ case NEON_BIT: bit(vf, rd, rn, rm); break;
+ case NEON_BSL: bsl(vf, rd, rn, rm); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ } else if (instr->Mask(NEON3SameFPFMask) == NEON3SameFPFixed) {
+ VectorFormat vf = nfd.GetVectorFormat(nfd.FPFormatMap());
+ switch (instr->Mask(NEON3SameFPMask)) {
+ case NEON_FADD: fadd(vf, rd, rn, rm); break;
+ case NEON_FSUB: fsub(vf, rd, rn, rm); break;
+ case NEON_FMUL: fmul(vf, rd, rn, rm); break;
+ case NEON_FDIV: fdiv(vf, rd, rn, rm); break;
+ case NEON_FMAX: fmax(vf, rd, rn, rm); break;
+ case NEON_FMIN: fmin(vf, rd, rn, rm); break;
+ case NEON_FMAXNM: fmaxnm(vf, rd, rn, rm); break;
+ case NEON_FMINNM: fminnm(vf, rd, rn, rm); break;
+ case NEON_FMLA: fmla(vf, rd, rn, rm); break;
+ case NEON_FMLS: fmls(vf, rd, rn, rm); break;
+ case NEON_FMULX: fmulx(vf, rd, rn, rm); break;
+ case NEON_FACGE: fabscmp(vf, rd, rn, rm, ge); break;
+ case NEON_FACGT: fabscmp(vf, rd, rn, rm, gt); break;
+ case NEON_FCMEQ: fcmp(vf, rd, rn, rm, eq); break;
+ case NEON_FCMGE: fcmp(vf, rd, rn, rm, ge); break;
+ case NEON_FCMGT: fcmp(vf, rd, rn, rm, gt); break;
+ case NEON_FRECPS: frecps(vf, rd, rn, rm); break;
+ case NEON_FRSQRTS: frsqrts(vf, rd, rn, rm); break;
+ case NEON_FABD: fabd(vf, rd, rn, rm); break;
+ case NEON_FADDP: faddp(vf, rd, rn, rm); break;
+ case NEON_FMAXP: fmaxp(vf, rd, rn, rm); break;
+ case NEON_FMAXNMP: fmaxnmp(vf, rd, rn, rm); break;
+ case NEON_FMINP: fminp(vf, rd, rn, rm); break;
+ case NEON_FMINNMP: fminnmp(vf, rd, rn, rm); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ } else {
+ VectorFormat vf = nfd.GetVectorFormat();
+ switch (instr->Mask(NEON3SameMask)) {
+ case NEON_ADD: add(vf, rd, rn, rm); break;
+ case NEON_ADDP: addp(vf, rd, rn, rm); break;
+ case NEON_CMEQ: cmp(vf, rd, rn, rm, eq); break;
+ case NEON_CMGE: cmp(vf, rd, rn, rm, ge); break;
+ case NEON_CMGT: cmp(vf, rd, rn, rm, gt); break;
+ case NEON_CMHI: cmp(vf, rd, rn, rm, hi); break;
+ case NEON_CMHS: cmp(vf, rd, rn, rm, hs); break;
+ case NEON_CMTST: cmptst(vf, rd, rn, rm); break;
+ case NEON_MLS: mls(vf, rd, rn, rm); break;
+ case NEON_MLA: mla(vf, rd, rn, rm); break;
+ case NEON_MUL: mul(vf, rd, rn, rm); break;
+ case NEON_PMUL: pmul(vf, rd, rn, rm); break;
+ case NEON_SMAX: smax(vf, rd, rn, rm); break;
+ case NEON_SMAXP: smaxp(vf, rd, rn, rm); break;
+ case NEON_SMIN: smin(vf, rd, rn, rm); break;
+ case NEON_SMINP: sminp(vf, rd, rn, rm); break;
+ case NEON_SUB: sub(vf, rd, rn, rm); break;
+ case NEON_UMAX: umax(vf, rd, rn, rm); break;
+ case NEON_UMAXP: umaxp(vf, rd, rn, rm); break;
+ case NEON_UMIN: umin(vf, rd, rn, rm); break;
+ case NEON_UMINP: uminp(vf, rd, rn, rm); break;
+ case NEON_SSHL: sshl(vf, rd, rn, rm); break;
+ case NEON_USHL: ushl(vf, rd, rn, rm); break;
+ case NEON_SABD: absdiff(vf, rd, rn, rm, true); break;
+ case NEON_UABD: absdiff(vf, rd, rn, rm, false); break;
+ case NEON_SABA: saba(vf, rd, rn, rm); break;
+ case NEON_UABA: uaba(vf, rd, rn, rm); break;
+ case NEON_UQADD: add(vf, rd, rn, rm).UnsignedSaturate(vf); break;
+ case NEON_SQADD: add(vf, rd, rn, rm).SignedSaturate(vf); break;
+ case NEON_UQSUB: sub(vf, rd, rn, rm).UnsignedSaturate(vf); break;
+ case NEON_SQSUB: sub(vf, rd, rn, rm).SignedSaturate(vf); break;
+ case NEON_SQDMULH: sqdmulh(vf, rd, rn, rm); break;
+ case NEON_SQRDMULH: sqrdmulh(vf, rd, rn, rm); break;
+ case NEON_UQSHL: ushl(vf, rd, rn, rm).UnsignedSaturate(vf); break;
+ case NEON_SQSHL: sshl(vf, rd, rn, rm).SignedSaturate(vf); break;
+ case NEON_URSHL: ushl(vf, rd, rn, rm).Round(vf); break;
+ case NEON_SRSHL: sshl(vf, rd, rn, rm).Round(vf); break;
+ case NEON_UQRSHL:
+ ushl(vf, rd, rn, rm).Round(vf).UnsignedSaturate(vf);
+ break;
+ case NEON_SQRSHL:
+ sshl(vf, rd, rn, rm).Round(vf).SignedSaturate(vf);
+ break;
+ case NEON_UHADD:
+ add(vf, rd, rn, rm).Uhalve(vf);
+ break;
+ case NEON_URHADD:
+ add(vf, rd, rn, rm).Uhalve(vf).Round(vf);
+ break;
+ case NEON_SHADD:
+ add(vf, rd, rn, rm).Halve(vf);
+ break;
+ case NEON_SRHADD:
+ add(vf, rd, rn, rm).Halve(vf).Round(vf);
+ break;
+ case NEON_UHSUB:
+ sub(vf, rd, rn, rm).Uhalve(vf);
+ break;
+ case NEON_SHSUB:
+ sub(vf, rd, rn, rm).Halve(vf);
+ break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ }
+}
+
+
+void Simulator::VisitNEON3Different(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr);
+ VectorFormat vf = nfd.GetVectorFormat();
+ VectorFormat vf_l = nfd.GetVectorFormat(nfd.LongIntegerFormatMap());
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ SimVRegister& rm = vreg(instr->Rm());
+
+ switch (instr->Mask(NEON3DifferentMask)) {
+ case NEON_PMULL: pmull(vf_l, rd, rn, rm); break;
+ case NEON_PMULL2: pmull2(vf_l, rd, rn, rm); break;
+ case NEON_UADDL: uaddl(vf_l, rd, rn, rm); break;
+ case NEON_UADDL2: uaddl2(vf_l, rd, rn, rm); break;
+ case NEON_SADDL: saddl(vf_l, rd, rn, rm); break;
+ case NEON_SADDL2: saddl2(vf_l, rd, rn, rm); break;
+ case NEON_USUBL: usubl(vf_l, rd, rn, rm); break;
+ case NEON_USUBL2: usubl2(vf_l, rd, rn, rm); break;
+ case NEON_SSUBL: ssubl(vf_l, rd, rn, rm); break;
+ case NEON_SSUBL2: ssubl2(vf_l, rd, rn, rm); break;
+ case NEON_SABAL: sabal(vf_l, rd, rn, rm); break;
+ case NEON_SABAL2: sabal2(vf_l, rd, rn, rm); break;
+ case NEON_UABAL: uabal(vf_l, rd, rn, rm); break;
+ case NEON_UABAL2: uabal2(vf_l, rd, rn, rm); break;
+ case NEON_SABDL: sabdl(vf_l, rd, rn, rm); break;
+ case NEON_SABDL2: sabdl2(vf_l, rd, rn, rm); break;
+ case NEON_UABDL: uabdl(vf_l, rd, rn, rm); break;
+ case NEON_UABDL2: uabdl2(vf_l, rd, rn, rm); break;
+ case NEON_SMLAL: smlal(vf_l, rd, rn, rm); break;
+ case NEON_SMLAL2: smlal2(vf_l, rd, rn, rm); break;
+ case NEON_UMLAL: umlal(vf_l, rd, rn, rm); break;
+ case NEON_UMLAL2: umlal2(vf_l, rd, rn, rm); break;
+ case NEON_SMLSL: smlsl(vf_l, rd, rn, rm); break;
+ case NEON_SMLSL2: smlsl2(vf_l, rd, rn, rm); break;
+ case NEON_UMLSL: umlsl(vf_l, rd, rn, rm); break;
+ case NEON_UMLSL2: umlsl2(vf_l, rd, rn, rm); break;
+ case NEON_SMULL: smull(vf_l, rd, rn, rm); break;
+ case NEON_SMULL2: smull2(vf_l, rd, rn, rm); break;
+ case NEON_UMULL: umull(vf_l, rd, rn, rm); break;
+ case NEON_UMULL2: umull2(vf_l, rd, rn, rm); break;
+ case NEON_SQDMLAL: sqdmlal(vf_l, rd, rn, rm); break;
+ case NEON_SQDMLAL2: sqdmlal2(vf_l, rd, rn, rm); break;
+ case NEON_SQDMLSL: sqdmlsl(vf_l, rd, rn, rm); break;
+ case NEON_SQDMLSL2: sqdmlsl2(vf_l, rd, rn, rm); break;
+ case NEON_SQDMULL: sqdmull(vf_l, rd, rn, rm); break;
+ case NEON_SQDMULL2: sqdmull2(vf_l, rd, rn, rm); break;
+ case NEON_UADDW: uaddw(vf_l, rd, rn, rm); break;
+ case NEON_UADDW2: uaddw2(vf_l, rd, rn, rm); break;
+ case NEON_SADDW: saddw(vf_l, rd, rn, rm); break;
+ case NEON_SADDW2: saddw2(vf_l, rd, rn, rm); break;
+ case NEON_USUBW: usubw(vf_l, rd, rn, rm); break;
+ case NEON_USUBW2: usubw2(vf_l, rd, rn, rm); break;
+ case NEON_SSUBW: ssubw(vf_l, rd, rn, rm); break;
+ case NEON_SSUBW2: ssubw2(vf_l, rd, rn, rm); break;
+ case NEON_ADDHN: addhn(vf, rd, rn, rm); break;
+ case NEON_ADDHN2: addhn2(vf, rd, rn, rm); break;
+ case NEON_RADDHN: raddhn(vf, rd, rn, rm); break;
+ case NEON_RADDHN2: raddhn2(vf, rd, rn, rm); break;
+ case NEON_SUBHN: subhn(vf, rd, rn, rm); break;
+ case NEON_SUBHN2: subhn2(vf, rd, rn, rm); break;
+ case NEON_RSUBHN: rsubhn(vf, rd, rn, rm); break;
+ case NEON_RSUBHN2: rsubhn2(vf, rd, rn, rm); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitNEONAcrossLanes(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr);
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+
+ // The input operand's VectorFormat is passed for these instructions.
+ if (instr->Mask(NEONAcrossLanesFPFMask) == NEONAcrossLanesFPFixed) {
+ VectorFormat vf = nfd.GetVectorFormat(nfd.FPFormatMap());
+
+ switch (instr->Mask(NEONAcrossLanesFPMask)) {
+ case NEON_FMAXV: fmaxv(vf, rd, rn); break;
+ case NEON_FMINV: fminv(vf, rd, rn); break;
+ case NEON_FMAXNMV: fmaxnmv(vf, rd, rn); break;
+ case NEON_FMINNMV: fminnmv(vf, rd, rn); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ } else {
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ switch (instr->Mask(NEONAcrossLanesMask)) {
+ case NEON_ADDV: addv(vf, rd, rn); break;
+ case NEON_SMAXV: smaxv(vf, rd, rn); break;
+ case NEON_SMINV: sminv(vf, rd, rn); break;
+ case NEON_UMAXV: umaxv(vf, rd, rn); break;
+ case NEON_UMINV: uminv(vf, rd, rn); break;
+ case NEON_SADDLV: saddlv(vf, rd, rn); break;
+ case NEON_UADDLV: uaddlv(vf, rd, rn); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ }
+}
+
+
+void Simulator::VisitNEONByIndexedElement(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr);
+ VectorFormat vf_r = nfd.GetVectorFormat();
+ VectorFormat vf = nfd.GetVectorFormat(nfd.LongIntegerFormatMap());
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+
+ ByElementOp Op = NULL;
+
+ int rm_reg = instr->Rm();
+ int index = (instr->NEONH() << 1) | instr->NEONL();
+ if (instr->NEONSize() == 1) {
+ rm_reg &= 0xf;
+ index = (index << 1) | instr->NEONM();
+ }
+
+ switch (instr->Mask(NEONByIndexedElementMask)) {
+ case NEON_MUL_byelement: Op = &Simulator::mul; vf = vf_r; break;
+ case NEON_MLA_byelement: Op = &Simulator::mla; vf = vf_r; break;
+ case NEON_MLS_byelement: Op = &Simulator::mls; vf = vf_r; break;
+ case NEON_SQDMULH_byelement: Op = &Simulator::sqdmulh; vf = vf_r; break;
+ case NEON_SQRDMULH_byelement: Op = &Simulator::sqrdmulh; vf = vf_r; break;
+ case NEON_SMULL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::smull2;
+ } else {
+ Op = &Simulator::smull;
+ }
+ break;
+ case NEON_UMULL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::umull2;
+ } else {
+ Op = &Simulator::umull;
+ }
+ break;
+ case NEON_SMLAL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::smlal2;
+ } else {
+ Op = &Simulator::smlal;
+ }
+ break;
+ case NEON_UMLAL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::umlal2;
+ } else {
+ Op = &Simulator::umlal;
+ }
+ break;
+ case NEON_SMLSL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::smlsl2;
+ } else {
+ Op = &Simulator::smlsl;
+ }
+ break;
+ case NEON_UMLSL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::umlsl2;
+ } else {
+ Op = &Simulator::umlsl;
+ }
+ break;
+ case NEON_SQDMULL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::sqdmull2;
+ } else {
+ Op = &Simulator::sqdmull;
+ }
+ break;
+ case NEON_SQDMLAL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::sqdmlal2;
+ } else {
+ Op = &Simulator::sqdmlal;
+ }
+ break;
+ case NEON_SQDMLSL_byelement:
+ if (instr->Mask(NEON_Q)) {
+ Op = &Simulator::sqdmlsl2;
+ } else {
+ Op = &Simulator::sqdmlsl;
+ }
+ break;
+ default:
+ index = instr->NEONH();
+ if ((instr->FPType() & 1) == 0) {
+ index = (index << 1) | instr->NEONL();
+ }
+
+ vf = nfd.GetVectorFormat(nfd.FPFormatMap());
+
+ switch (instr->Mask(NEONByIndexedElementFPMask)) {
+ case NEON_FMUL_byelement: Op = &Simulator::fmul; break;
+ case NEON_FMLA_byelement: Op = &Simulator::fmla; break;
+ case NEON_FMLS_byelement: Op = &Simulator::fmls; break;
+ case NEON_FMULX_byelement: Op = &Simulator::fmulx; break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ }
+
+ (this->*Op)(vf, rd, rn, vreg(rm_reg), index);
+}
+
+
+void Simulator::VisitNEONCopy(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::TriangularFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ int imm5 = instr->ImmNEON5();
+ int tz = CountTrailingZeros(imm5, 32);
+ int reg_index = imm5 >> (tz + 1);
+
+ if (instr->Mask(NEONCopyInsElementMask) == NEON_INS_ELEMENT) {
+ int imm4 = instr->ImmNEON4();
+ int rn_index = imm4 >> tz;
+ ins_element(vf, rd, reg_index, rn, rn_index);
+ } else if (instr->Mask(NEONCopyInsGeneralMask) == NEON_INS_GENERAL) {
+ ins_immediate(vf, rd, reg_index, xreg(instr->Rn()));
+ } else if (instr->Mask(NEONCopyUmovMask) == NEON_UMOV) {
+ uint64_t value = LogicVRegister(rn).Uint(vf, reg_index);
+ value &= MaxUintFromFormat(vf);
+ set_xreg(instr->Rd(), value);
+ } else if (instr->Mask(NEONCopyUmovMask) == NEON_SMOV) {
+ int64_t value = LogicVRegister(rn).Int(vf, reg_index);
+ if (instr->NEONQ()) {
+ set_xreg(instr->Rd(), value);
+ } else {
+ set_wreg(instr->Rd(), (int32_t)value);
+ }
+ } else if (instr->Mask(NEONCopyDupElementMask) == NEON_DUP_ELEMENT) {
+ dup_element(vf, rd, rn, reg_index);
+ } else if (instr->Mask(NEONCopyDupGeneralMask) == NEON_DUP_GENERAL) {
+ dup_immediate(vf, rd, xreg(instr->Rn()));
+ } else {
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitNEONExtract(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LogicalFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ SimVRegister& rm = vreg(instr->Rm());
+ if (instr->Mask(NEONExtractMask) == NEON_EXT) {
+ int index = instr->ImmNEONExt();
+ ext(vf, rd, rn, rm, index);
+ } else {
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::NEONLoadStoreMultiStructHelper(const Instruction* instr,
+ AddrMode addr_mode) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ uint64_t addr_base = xreg(instr->Rn(), Reg31IsStackPointer);
+ int reg_size = RegisterSizeInBytesFromFormat(vf);
+
+ int reg[4];
+ uint64_t addr[4];
+ for (int i = 0; i < 4; i++) {
+ reg[i] = (instr->Rt() + i) % kNumberOfVRegisters;
+ addr[i] = addr_base + (i * reg_size);
+ }
+ int count = 1;
+ bool log_read = true;
+
+ Instr itype = instr->Mask(NEONLoadStoreMultiStructMask);
+ if (((itype == NEON_LD1_1v) || (itype == NEON_LD1_2v) ||
+ (itype == NEON_LD1_3v) || (itype == NEON_LD1_4v) ||
+ (itype == NEON_ST1_1v) || (itype == NEON_ST1_2v) ||
+ (itype == NEON_ST1_3v) || (itype == NEON_ST1_4v)) &&
+ (instr->Bits(20, 16) != 0)) {
+ VIXL_UNREACHABLE();
+ }
+
+ // We use the PostIndex mask here, as it works in this case for both Offset
+ // and PostIndex addressing.
+ switch (instr->Mask(NEONLoadStoreMultiStructPostIndexMask)) {
+ case NEON_LD1_4v:
+ case NEON_LD1_4v_post: ld1(vf, vreg(reg[3]), addr[3]); count++;
+ VIXL_FALLTHROUGH();
+ case NEON_LD1_3v:
+ case NEON_LD1_3v_post: ld1(vf, vreg(reg[2]), addr[2]); count++;
+ VIXL_FALLTHROUGH();
+ case NEON_LD1_2v:
+ case NEON_LD1_2v_post: ld1(vf, vreg(reg[1]), addr[1]); count++;
+ VIXL_FALLTHROUGH();
+ case NEON_LD1_1v:
+ case NEON_LD1_1v_post:
+ ld1(vf, vreg(reg[0]), addr[0]);
+ log_read = true;
+ break;
+ case NEON_ST1_4v:
+ case NEON_ST1_4v_post: st1(vf, vreg(reg[3]), addr[3]); count++;
+ VIXL_FALLTHROUGH();
+ case NEON_ST1_3v:
+ case NEON_ST1_3v_post: st1(vf, vreg(reg[2]), addr[2]); count++;
+ VIXL_FALLTHROUGH();
+ case NEON_ST1_2v:
+ case NEON_ST1_2v_post: st1(vf, vreg(reg[1]), addr[1]); count++;
+ VIXL_FALLTHROUGH();
+ case NEON_ST1_1v:
+ case NEON_ST1_1v_post:
+ st1(vf, vreg(reg[0]), addr[0]);
+ log_read = false;
+ break;
+ case NEON_LD2_post:
+ case NEON_LD2:
+ ld2(vf, vreg(reg[0]), vreg(reg[1]), addr[0]);
+ count = 2;
+ break;
+ case NEON_ST2:
+ case NEON_ST2_post:
+ st2(vf, vreg(reg[0]), vreg(reg[1]), addr[0]);
+ count = 2;
+ break;
+ case NEON_LD3_post:
+ case NEON_LD3:
+ ld3(vf, vreg(reg[0]), vreg(reg[1]), vreg(reg[2]), addr[0]);
+ count = 3;
+ break;
+ case NEON_ST3:
+ case NEON_ST3_post:
+ st3(vf, vreg(reg[0]), vreg(reg[1]), vreg(reg[2]), addr[0]);
+ count = 3;
+ break;
+ case NEON_ST4:
+ case NEON_ST4_post:
+ st4(vf, vreg(reg[0]), vreg(reg[1]), vreg(reg[2]), vreg(reg[3]),
+ addr[0]);
+ count = 4;
+ break;
+ case NEON_LD4_post:
+ case NEON_LD4:
+ ld4(vf, vreg(reg[0]), vreg(reg[1]), vreg(reg[2]), vreg(reg[3]),
+ addr[0]);
+ count = 4;
+ break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+
+ // Explicitly log the register update whilst we have type information.
+ for (int i = 0; i < count; i++) {
+ // For de-interleaving loads, only print the base address.
+ int lane_size = LaneSizeInBytesFromFormat(vf);
+ PrintRegisterFormat format = GetPrintRegisterFormatTryFP(
+ GetPrintRegisterFormatForSize(reg_size, lane_size));
+ if (log_read) {
+ LogVRead(addr_base, reg[i], format);
+ } else {
+ LogVWrite(addr_base, reg[i], format);
+ }
+ }
+
+ if (addr_mode == PostIndex) {
+ int rm = instr->Rm();
+ // The immediate post index addressing mode is indicated by rm = 31.
+ // The immediate is implied by the number of vector registers used.
+ addr_base += (rm == 31) ? RegisterSizeInBytesFromFormat(vf) * count
+ : xreg(rm);
+ set_xreg(instr->Rn(), addr_base);
+ } else {
+ VIXL_ASSERT(addr_mode == Offset);
+ }
+}
+
+
+void Simulator::VisitNEONLoadStoreMultiStruct(const Instruction* instr) {
+ NEONLoadStoreMultiStructHelper(instr, Offset);
+}
+
+
+void Simulator::VisitNEONLoadStoreMultiStructPostIndex(
+ const Instruction* instr) {
+ NEONLoadStoreMultiStructHelper(instr, PostIndex);
+}
+
+
+void Simulator::NEONLoadStoreSingleStructHelper(const Instruction* instr,
+ AddrMode addr_mode) {
+ uint64_t addr = xreg(instr->Rn(), Reg31IsStackPointer);
+ int rt = instr->Rt();
+
+ Instr itype = instr->Mask(NEONLoadStoreSingleStructMask);
+ if (((itype == NEON_LD1_b) || (itype == NEON_LD1_h) ||
+ (itype == NEON_LD1_s) || (itype == NEON_LD1_d)) &&
+ (instr->Bits(20, 16) != 0)) {
+ VIXL_UNREACHABLE();
+ }
+
+ // We use the PostIndex mask here, as it works in this case for both Offset
+ // and PostIndex addressing.
+ bool do_load = false;
+
+ bool replicating = false;
+
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LoadStoreFormatMap());
+ VectorFormat vf_t = nfd.GetVectorFormat();
+
+ VectorFormat vf = kFormat16B;
+ switch (instr->Mask(NEONLoadStoreSingleStructPostIndexMask)) {
+ case NEON_LD1_b:
+ case NEON_LD1_b_post:
+ case NEON_LD2_b:
+ case NEON_LD2_b_post:
+ case NEON_LD3_b:
+ case NEON_LD3_b_post:
+ case NEON_LD4_b:
+ case NEON_LD4_b_post: do_load = true;
+ VIXL_FALLTHROUGH();
+ case NEON_ST1_b:
+ case NEON_ST1_b_post:
+ case NEON_ST2_b:
+ case NEON_ST2_b_post:
+ case NEON_ST3_b:
+ case NEON_ST3_b_post:
+ case NEON_ST4_b:
+ case NEON_ST4_b_post: break;
+
+ case NEON_LD1_h:
+ case NEON_LD1_h_post:
+ case NEON_LD2_h:
+ case NEON_LD2_h_post:
+ case NEON_LD3_h:
+ case NEON_LD3_h_post:
+ case NEON_LD4_h:
+ case NEON_LD4_h_post: do_load = true;
+ VIXL_FALLTHROUGH();
+ case NEON_ST1_h:
+ case NEON_ST1_h_post:
+ case NEON_ST2_h:
+ case NEON_ST2_h_post:
+ case NEON_ST3_h:
+ case NEON_ST3_h_post:
+ case NEON_ST4_h:
+ case NEON_ST4_h_post: vf = kFormat8H; break;
+ case NEON_LD1_s:
+ case NEON_LD1_s_post:
+ case NEON_LD2_s:
+ case NEON_LD2_s_post:
+ case NEON_LD3_s:
+ case NEON_LD3_s_post:
+ case NEON_LD4_s:
+ case NEON_LD4_s_post: do_load = true;
+ VIXL_FALLTHROUGH();
+ case NEON_ST1_s:
+ case NEON_ST1_s_post:
+ case NEON_ST2_s:
+ case NEON_ST2_s_post:
+ case NEON_ST3_s:
+ case NEON_ST3_s_post:
+ case NEON_ST4_s:
+ case NEON_ST4_s_post: {
+ VIXL_STATIC_ASSERT((NEON_LD1_s | (1 << NEONLSSize_offset)) == NEON_LD1_d);
+ VIXL_STATIC_ASSERT(
+ (NEON_LD1_s_post | (1 << NEONLSSize_offset)) == NEON_LD1_d_post);
+ VIXL_STATIC_ASSERT((NEON_ST1_s | (1 << NEONLSSize_offset)) == NEON_ST1_d);
+ VIXL_STATIC_ASSERT(
+ (NEON_ST1_s_post | (1 << NEONLSSize_offset)) == NEON_ST1_d_post);
+ vf = ((instr->NEONLSSize() & 1) == 0) ? kFormat4S : kFormat2D;
+ break;
+ }
+
+ case NEON_LD1R:
+ case NEON_LD1R_post:
+ case NEON_LD2R:
+ case NEON_LD2R_post:
+ case NEON_LD3R:
+ case NEON_LD3R_post:
+ case NEON_LD4R:
+ case NEON_LD4R_post: {
+ vf = vf_t;
+ do_load = true;
+ replicating = true;
+ break;
+ }
+ default: VIXL_UNIMPLEMENTED();
+ }
+
+ PrintRegisterFormat print_format =
+ GetPrintRegisterFormatTryFP(GetPrintRegisterFormat(vf));
+ // Make sure that the print_format only includes a single lane.
+ print_format =
+ static_cast<PrintRegisterFormat>(print_format & ~kPrintRegAsVectorMask);
+
+ int esize = LaneSizeInBytesFromFormat(vf);
+ int index_shift = LaneSizeInBytesLog2FromFormat(vf);
+ int lane = instr->NEONLSIndex(index_shift);
+ int scale = 0;
+ int rt2 = (rt + 1) % kNumberOfVRegisters;
+ int rt3 = (rt2 + 1) % kNumberOfVRegisters;
+ int rt4 = (rt3 + 1) % kNumberOfVRegisters;
+ switch (instr->Mask(NEONLoadStoreSingleLenMask)) {
+ case NEONLoadStoreSingle1:
+ scale = 1;
+ if (do_load) {
+ if (replicating) {
+ ld1r(vf, vreg(rt), addr);
+ } else {
+ ld1(vf, vreg(rt), lane, addr);
+ }
+ LogVRead(addr, rt, print_format, lane);
+ } else {
+ st1(vf, vreg(rt), lane, addr);
+ LogVWrite(addr, rt, print_format, lane);
+ }
+ break;
+ case NEONLoadStoreSingle2:
+ scale = 2;
+ if (do_load) {
+ if (replicating) {
+ ld2r(vf, vreg(rt), vreg(rt2), addr);
+ } else {
+ ld2(vf, vreg(rt), vreg(rt2), lane, addr);
+ }
+ LogVRead(addr, rt, print_format, lane);
+ LogVRead(addr + esize, rt2, print_format, lane);
+ } else {
+ st2(vf, vreg(rt), vreg(rt2), lane, addr);
+ LogVWrite(addr, rt, print_format, lane);
+ LogVWrite(addr + esize, rt2, print_format, lane);
+ }
+ break;
+ case NEONLoadStoreSingle3:
+ scale = 3;
+ if (do_load) {
+ if (replicating) {
+ ld3r(vf, vreg(rt), vreg(rt2), vreg(rt3), addr);
+ } else {
+ ld3(vf, vreg(rt), vreg(rt2), vreg(rt3), lane, addr);
+ }
+ LogVRead(addr, rt, print_format, lane);
+ LogVRead(addr + esize, rt2, print_format, lane);
+ LogVRead(addr + (2 * esize), rt3, print_format, lane);
+ } else {
+ st3(vf, vreg(rt), vreg(rt2), vreg(rt3), lane, addr);
+ LogVWrite(addr, rt, print_format, lane);
+ LogVWrite(addr + esize, rt2, print_format, lane);
+ LogVWrite(addr + (2 * esize), rt3, print_format, lane);
+ }
+ break;
+ case NEONLoadStoreSingle4:
+ scale = 4;
+ if (do_load) {
+ if (replicating) {
+ ld4r(vf, vreg(rt), vreg(rt2), vreg(rt3), vreg(rt4), addr);
+ } else {
+ ld4(vf, vreg(rt), vreg(rt2), vreg(rt3), vreg(rt4), lane, addr);
+ }
+ LogVRead(addr, rt, print_format, lane);
+ LogVRead(addr + esize, rt2, print_format, lane);
+ LogVRead(addr + (2 * esize), rt3, print_format, lane);
+ LogVRead(addr + (3 * esize), rt4, print_format, lane);
+ } else {
+ st4(vf, vreg(rt), vreg(rt2), vreg(rt3), vreg(rt4), lane, addr);
+ LogVWrite(addr, rt, print_format, lane);
+ LogVWrite(addr + esize, rt2, print_format, lane);
+ LogVWrite(addr + (2 * esize), rt3, print_format, lane);
+ LogVWrite(addr + (3 * esize), rt4, print_format, lane);
+ }
+ break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+
+ if (addr_mode == PostIndex) {
+ int rm = instr->Rm();
+ int lane_size = LaneSizeInBytesFromFormat(vf);
+ set_xreg(instr->Rn(), addr + ((rm == 31) ? (scale * lane_size) : xreg(rm)));
+ }
+}
+
+
+void Simulator::VisitNEONLoadStoreSingleStruct(const Instruction* instr) {
+ NEONLoadStoreSingleStructHelper(instr, Offset);
+}
+
+
+void Simulator::VisitNEONLoadStoreSingleStructPostIndex(
+ const Instruction* instr) {
+ NEONLoadStoreSingleStructHelper(instr, PostIndex);
+}
+
+
+void Simulator::VisitNEONModifiedImmediate(const Instruction* instr) {
+ SimVRegister& rd = vreg(instr->Rd());
+ int cmode = instr->NEONCmode();
+ int cmode_3_1 = (cmode >> 1) & 7;
+ int cmode_3 = (cmode >> 3) & 1;
+ int cmode_2 = (cmode >> 2) & 1;
+ int cmode_1 = (cmode >> 1) & 1;
+ int cmode_0 = cmode & 1;
+ int q = instr->NEONQ();
+ int op_bit = instr->NEONModImmOp();
+ uint64_t imm8 = instr->ImmNEONabcdefgh();
+
+ // Find the format and immediate value
+ uint64_t imm = 0;
+ VectorFormat vform = kFormatUndefined;
+ switch (cmode_3_1) {
+ case 0x0:
+ case 0x1:
+ case 0x2:
+ case 0x3:
+ vform = (q == 1) ? kFormat4S : kFormat2S;
+ imm = imm8 << (8 * cmode_3_1);
+ break;
+ case 0x4:
+ case 0x5:
+ vform = (q == 1) ? kFormat8H : kFormat4H;
+ imm = imm8 << (8 * cmode_1);
+ break;
+ case 0x6:
+ vform = (q == 1) ? kFormat4S : kFormat2S;
+ if (cmode_0 == 0) {
+ imm = imm8 << 8 | 0x000000ff;
+ } else {
+ imm = imm8 << 16 | 0x0000ffff;
+ }
+ break;
+ case 0x7:
+ if (cmode_0 == 0 && op_bit == 0) {
+ vform = q ? kFormat16B : kFormat8B;
+ imm = imm8;
+ } else if (cmode_0 == 0 && op_bit == 1) {
+ vform = q ? kFormat2D : kFormat1D;
+ imm = 0;
+ for (int i = 0; i < 8; ++i) {
+ if (imm8 & (1ULL << i)) {
+ imm |= (UINT64_C(0xff) << (8 * i));
+ }
+ }
+ } else { // cmode_0 == 1, cmode == 0xf.
+ if (op_bit == 0) {
+ vform = q ? kFormat4S : kFormat2S;
+ imm = FloatToRawbits(instr->ImmNEONFP32());
+ } else if (q == 1) {
+ vform = kFormat2D;
+ imm = DoubleToRawbits(instr->ImmNEONFP64());
+ } else {
+ VIXL_ASSERT((q == 0) && (op_bit == 1) && (cmode == 0xf));
+ VisitUnallocated(instr);
+ }
+ }
+ break;
+ default: VIXL_UNREACHABLE(); break;
+ }
+
+ // Find the operation
+ NEONModifiedImmediateOp op;
+ if (cmode_3 == 0) {
+ if (cmode_0 == 0) {
+ op = op_bit ? NEONModifiedImmediate_MVNI : NEONModifiedImmediate_MOVI;
+ } else { // cmode<0> == '1'
+ op = op_bit ? NEONModifiedImmediate_BIC : NEONModifiedImmediate_ORR;
+ }
+ } else { // cmode<3> == '1'
+ if (cmode_2 == 0) {
+ if (cmode_0 == 0) {
+ op = op_bit ? NEONModifiedImmediate_MVNI : NEONModifiedImmediate_MOVI;
+ } else { // cmode<0> == '1'
+ op = op_bit ? NEONModifiedImmediate_BIC : NEONModifiedImmediate_ORR;
+ }
+ } else { // cmode<2> == '1'
+ if (cmode_1 == 0) {
+ op = op_bit ? NEONModifiedImmediate_MVNI : NEONModifiedImmediate_MOVI;
+ } else { // cmode<1> == '1'
+ if (cmode_0 == 0) {
+ op = NEONModifiedImmediate_MOVI;
+ } else { // cmode<0> == '1'
+ op = NEONModifiedImmediate_MOVI;
+ }
+ }
+ }
+ }
+
+ // Call the logic function
+ if (op == NEONModifiedImmediate_ORR) {
+ orr(vform, rd, rd, imm);
+ } else if (op == NEONModifiedImmediate_BIC) {
+ bic(vform, rd, rd, imm);
+ } else if (op == NEONModifiedImmediate_MOVI) {
+ movi(vform, rd, imm);
+ } else if (op == NEONModifiedImmediate_MVNI) {
+ mvni(vform, rd, imm);
+ } else {
+ VisitUnimplemented(instr);
+ }
+}
+
+
+void Simulator::VisitNEONScalar2RegMisc(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+
+ if (instr->Mask(NEON2RegMiscOpcode) <= NEON_NEG_scalar_opcode) {
+ // These instructions all use a two bit size field, except NOT and RBIT,
+ // which use the field to encode the operation.
+ switch (instr->Mask(NEONScalar2RegMiscMask)) {
+ case NEON_CMEQ_zero_scalar: cmp(vf, rd, rn, 0, eq); break;
+ case NEON_CMGE_zero_scalar: cmp(vf, rd, rn, 0, ge); break;
+ case NEON_CMGT_zero_scalar: cmp(vf, rd, rn, 0, gt); break;
+ case NEON_CMLT_zero_scalar: cmp(vf, rd, rn, 0, lt); break;
+ case NEON_CMLE_zero_scalar: cmp(vf, rd, rn, 0, le); break;
+ case NEON_ABS_scalar: abs(vf, rd, rn); break;
+ case NEON_SQABS_scalar: abs(vf, rd, rn).SignedSaturate(vf); break;
+ case NEON_NEG_scalar: neg(vf, rd, rn); break;
+ case NEON_SQNEG_scalar: neg(vf, rd, rn).SignedSaturate(vf); break;
+ case NEON_SUQADD_scalar: suqadd(vf, rd, rn); break;
+ case NEON_USQADD_scalar: usqadd(vf, rd, rn); break;
+ default: VIXL_UNIMPLEMENTED(); break;
+ }
+ } else {
+ VectorFormat fpf = nfd.GetVectorFormat(nfd.FPScalarFormatMap());
+ FPRounding fpcr_rounding = static_cast<FPRounding>(fpcr().RMode());
+
+ // These instructions all use a one bit size field, except SQXTUN, SQXTN
+ // and UQXTN, which use a two bit size field.
+ switch (instr->Mask(NEONScalar2RegMiscFPMask)) {
+ case NEON_FRECPE_scalar: frecpe(fpf, rd, rn, fpcr_rounding); break;
+ case NEON_FRECPX_scalar: frecpx(fpf, rd, rn); break;
+ case NEON_FRSQRTE_scalar: frsqrte(fpf, rd, rn); break;
+ case NEON_FCMGT_zero_scalar: fcmp_zero(fpf, rd, rn, gt); break;
+ case NEON_FCMGE_zero_scalar: fcmp_zero(fpf, rd, rn, ge); break;
+ case NEON_FCMEQ_zero_scalar: fcmp_zero(fpf, rd, rn, eq); break;
+ case NEON_FCMLE_zero_scalar: fcmp_zero(fpf, rd, rn, le); break;
+ case NEON_FCMLT_zero_scalar: fcmp_zero(fpf, rd, rn, lt); break;
+ case NEON_SCVTF_scalar: scvtf(fpf, rd, rn, 0, fpcr_rounding); break;
+ case NEON_UCVTF_scalar: ucvtf(fpf, rd, rn, 0, fpcr_rounding); break;
+ case NEON_FCVTNS_scalar: fcvts(fpf, rd, rn, FPTieEven); break;
+ case NEON_FCVTNU_scalar: fcvtu(fpf, rd, rn, FPTieEven); break;
+ case NEON_FCVTPS_scalar: fcvts(fpf, rd, rn, FPPositiveInfinity); break;
+ case NEON_FCVTPU_scalar: fcvtu(fpf, rd, rn, FPPositiveInfinity); break;
+ case NEON_FCVTMS_scalar: fcvts(fpf, rd, rn, FPNegativeInfinity); break;
+ case NEON_FCVTMU_scalar: fcvtu(fpf, rd, rn, FPNegativeInfinity); break;
+ case NEON_FCVTZS_scalar: fcvts(fpf, rd, rn, FPZero); break;
+ case NEON_FCVTZU_scalar: fcvtu(fpf, rd, rn, FPZero); break;
+ case NEON_FCVTAS_scalar: fcvts(fpf, rd, rn, FPTieAway); break;
+ case NEON_FCVTAU_scalar: fcvtu(fpf, rd, rn, FPTieAway); break;
+ case NEON_FCVTXN_scalar:
+ // Unlike all of the other FP instructions above, fcvtxn encodes dest
+ // size S as size<0>=1. There's only one case, so we ignore the form.
+ VIXL_ASSERT(instr->Bit(22) == 1);
+ fcvtxn(kFormatS, rd, rn);
+ break;
+ default:
+ switch (instr->Mask(NEONScalar2RegMiscMask)) {
+ case NEON_SQXTN_scalar: sqxtn(vf, rd, rn); break;
+ case NEON_UQXTN_scalar: uqxtn(vf, rd, rn); break;
+ case NEON_SQXTUN_scalar: sqxtun(vf, rd, rn); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ }
+ }
+}
+
+
+void Simulator::VisitNEONScalar3Diff(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LongScalarFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ SimVRegister& rm = vreg(instr->Rm());
+ switch (instr->Mask(NEONScalar3DiffMask)) {
+ case NEON_SQDMLAL_scalar: sqdmlal(vf, rd, rn, rm); break;
+ case NEON_SQDMLSL_scalar: sqdmlsl(vf, rd, rn, rm); break;
+ case NEON_SQDMULL_scalar: sqdmull(vf, rd, rn, rm); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitNEONScalar3Same(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::ScalarFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ SimVRegister& rm = vreg(instr->Rm());
+
+ if (instr->Mask(NEONScalar3SameFPFMask) == NEONScalar3SameFPFixed) {
+ vf = nfd.GetVectorFormat(nfd.FPScalarFormatMap());
+ switch (instr->Mask(NEONScalar3SameFPMask)) {
+ case NEON_FMULX_scalar: fmulx(vf, rd, rn, rm); break;
+ case NEON_FACGE_scalar: fabscmp(vf, rd, rn, rm, ge); break;
+ case NEON_FACGT_scalar: fabscmp(vf, rd, rn, rm, gt); break;
+ case NEON_FCMEQ_scalar: fcmp(vf, rd, rn, rm, eq); break;
+ case NEON_FCMGE_scalar: fcmp(vf, rd, rn, rm, ge); break;
+ case NEON_FCMGT_scalar: fcmp(vf, rd, rn, rm, gt); break;
+ case NEON_FRECPS_scalar: frecps(vf, rd, rn, rm); break;
+ case NEON_FRSQRTS_scalar: frsqrts(vf, rd, rn, rm); break;
+ case NEON_FABD_scalar: fabd(vf, rd, rn, rm); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ } else {
+ switch (instr->Mask(NEONScalar3SameMask)) {
+ case NEON_ADD_scalar: add(vf, rd, rn, rm); break;
+ case NEON_SUB_scalar: sub(vf, rd, rn, rm); break;
+ case NEON_CMEQ_scalar: cmp(vf, rd, rn, rm, eq); break;
+ case NEON_CMGE_scalar: cmp(vf, rd, rn, rm, ge); break;
+ case NEON_CMGT_scalar: cmp(vf, rd, rn, rm, gt); break;
+ case NEON_CMHI_scalar: cmp(vf, rd, rn, rm, hi); break;
+ case NEON_CMHS_scalar: cmp(vf, rd, rn, rm, hs); break;
+ case NEON_CMTST_scalar: cmptst(vf, rd, rn, rm); break;
+ case NEON_USHL_scalar: ushl(vf, rd, rn, rm); break;
+ case NEON_SSHL_scalar: sshl(vf, rd, rn, rm); break;
+ case NEON_SQDMULH_scalar: sqdmulh(vf, rd, rn, rm); break;
+ case NEON_SQRDMULH_scalar: sqrdmulh(vf, rd, rn, rm); break;
+ case NEON_UQADD_scalar:
+ add(vf, rd, rn, rm).UnsignedSaturate(vf);
+ break;
+ case NEON_SQADD_scalar:
+ add(vf, rd, rn, rm).SignedSaturate(vf);
+ break;
+ case NEON_UQSUB_scalar:
+ sub(vf, rd, rn, rm).UnsignedSaturate(vf);
+ break;
+ case NEON_SQSUB_scalar:
+ sub(vf, rd, rn, rm).SignedSaturate(vf);
+ break;
+ case NEON_UQSHL_scalar:
+ ushl(vf, rd, rn, rm).UnsignedSaturate(vf);
+ break;
+ case NEON_SQSHL_scalar:
+ sshl(vf, rd, rn, rm).SignedSaturate(vf);
+ break;
+ case NEON_URSHL_scalar:
+ ushl(vf, rd, rn, rm).Round(vf);
+ break;
+ case NEON_SRSHL_scalar:
+ sshl(vf, rd, rn, rm).Round(vf);
+ break;
+ case NEON_UQRSHL_scalar:
+ ushl(vf, rd, rn, rm).Round(vf).UnsignedSaturate(vf);
+ break;
+ case NEON_SQRSHL_scalar:
+ sshl(vf, rd, rn, rm).Round(vf).SignedSaturate(vf);
+ break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+ }
+}
+
+
+void Simulator::VisitNEONScalarByIndexedElement(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LongScalarFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+ VectorFormat vf_r = nfd.GetVectorFormat(nfd.ScalarFormatMap());
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ ByElementOp Op = NULL;
+
+ int rm_reg = instr->Rm();
+ int index = (instr->NEONH() << 1) | instr->NEONL();
+ if (instr->NEONSize() == 1) {
+ rm_reg &= 0xf;
+ index = (index << 1) | instr->NEONM();
+ }
+
+ switch (instr->Mask(NEONScalarByIndexedElementMask)) {
+ case NEON_SQDMULL_byelement_scalar: Op = &Simulator::sqdmull; break;
+ case NEON_SQDMLAL_byelement_scalar: Op = &Simulator::sqdmlal; break;
+ case NEON_SQDMLSL_byelement_scalar: Op = &Simulator::sqdmlsl; break;
+ case NEON_SQDMULH_byelement_scalar:
+ Op = &Simulator::sqdmulh;
+ vf = vf_r;
+ break;
+ case NEON_SQRDMULH_byelement_scalar:
+ Op = &Simulator::sqrdmulh;
+ vf = vf_r;
+ break;
+ default:
+ vf = nfd.GetVectorFormat(nfd.FPScalarFormatMap());
+ index = instr->NEONH();
+ if ((instr->FPType() & 1) == 0) {
+ index = (index << 1) | instr->NEONL();
+ }
+ switch (instr->Mask(NEONScalarByIndexedElementFPMask)) {
+ case NEON_FMUL_byelement_scalar: Op = &Simulator::fmul; break;
+ case NEON_FMLA_byelement_scalar: Op = &Simulator::fmla; break;
+ case NEON_FMLS_byelement_scalar: Op = &Simulator::fmls; break;
+ case NEON_FMULX_byelement_scalar: Op = &Simulator::fmulx; break;
+ default: VIXL_UNIMPLEMENTED();
+ }
+ }
+
+ (this->*Op)(vf, rd, rn, vreg(rm_reg), index);
+}
+
+
+void Simulator::VisitNEONScalarCopy(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::TriangularScalarFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+
+ if (instr->Mask(NEONScalarCopyMask) == NEON_DUP_ELEMENT_scalar) {
+ int imm5 = instr->ImmNEON5();
+ int tz = CountTrailingZeros(imm5, 32);
+ int rn_index = imm5 >> (tz + 1);
+ dup_element(vf, rd, rn, rn_index);
+ } else {
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitNEONScalarPairwise(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::FPScalarFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ switch (instr->Mask(NEONScalarPairwiseMask)) {
+ case NEON_ADDP_scalar: addp(vf, rd, rn); break;
+ case NEON_FADDP_scalar: faddp(vf, rd, rn); break;
+ case NEON_FMAXP_scalar: fmaxp(vf, rd, rn); break;
+ case NEON_FMAXNMP_scalar: fmaxnmp(vf, rd, rn); break;
+ case NEON_FMINP_scalar: fminp(vf, rd, rn); break;
+ case NEON_FMINNMP_scalar: fminnmp(vf, rd, rn); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitNEONScalarShiftImmediate(const Instruction* instr) {
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ FPRounding fpcr_rounding = static_cast<FPRounding>(fpcr().RMode());
+
+ static const NEONFormatMap map = {
+ {22, 21, 20, 19},
+ {NF_UNDEF, NF_B, NF_H, NF_H, NF_S, NF_S, NF_S, NF_S,
+ NF_D, NF_D, NF_D, NF_D, NF_D, NF_D, NF_D, NF_D}
+ };
+ NEONFormatDecoder nfd(instr, &map);
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ int highestSetBit = HighestSetBitPosition(instr->ImmNEONImmh());
+ int immhimmb = instr->ImmNEONImmhImmb();
+ int right_shift = (16 << highestSetBit) - immhimmb;
+ int left_shift = immhimmb - (8 << highestSetBit);
+ switch (instr->Mask(NEONScalarShiftImmediateMask)) {
+ case NEON_SHL_scalar: shl(vf, rd, rn, left_shift); break;
+ case NEON_SLI_scalar: sli(vf, rd, rn, left_shift); break;
+ case NEON_SQSHL_imm_scalar: sqshl(vf, rd, rn, left_shift); break;
+ case NEON_UQSHL_imm_scalar: uqshl(vf, rd, rn, left_shift); break;
+ case NEON_SQSHLU_scalar: sqshlu(vf, rd, rn, left_shift); break;
+ case NEON_SRI_scalar: sri(vf, rd, rn, right_shift); break;
+ case NEON_SSHR_scalar: sshr(vf, rd, rn, right_shift); break;
+ case NEON_USHR_scalar: ushr(vf, rd, rn, right_shift); break;
+ case NEON_SRSHR_scalar: sshr(vf, rd, rn, right_shift).Round(vf); break;
+ case NEON_URSHR_scalar: ushr(vf, rd, rn, right_shift).Round(vf); break;
+ case NEON_SSRA_scalar: ssra(vf, rd, rn, right_shift); break;
+ case NEON_USRA_scalar: usra(vf, rd, rn, right_shift); break;
+ case NEON_SRSRA_scalar: srsra(vf, rd, rn, right_shift); break;
+ case NEON_URSRA_scalar: ursra(vf, rd, rn, right_shift); break;
+ case NEON_UQSHRN_scalar: uqshrn(vf, rd, rn, right_shift); break;
+ case NEON_UQRSHRN_scalar: uqrshrn(vf, rd, rn, right_shift); break;
+ case NEON_SQSHRN_scalar: sqshrn(vf, rd, rn, right_shift); break;
+ case NEON_SQRSHRN_scalar: sqrshrn(vf, rd, rn, right_shift); break;
+ case NEON_SQSHRUN_scalar: sqshrun(vf, rd, rn, right_shift); break;
+ case NEON_SQRSHRUN_scalar: sqrshrun(vf, rd, rn, right_shift); break;
+ case NEON_FCVTZS_imm_scalar: fcvts(vf, rd, rn, FPZero, right_shift); break;
+ case NEON_FCVTZU_imm_scalar: fcvtu(vf, rd, rn, FPZero, right_shift); break;
+ case NEON_SCVTF_imm_scalar:
+ scvtf(vf, rd, rn, right_shift, fpcr_rounding);
+ break;
+ case NEON_UCVTF_imm_scalar:
+ ucvtf(vf, rd, rn, right_shift, fpcr_rounding);
+ break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitNEONShiftImmediate(const Instruction* instr) {
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ FPRounding fpcr_rounding = static_cast<FPRounding>(fpcr().RMode());
+
+ // 00010->8B, 00011->16B, 001x0->4H, 001x1->8H,
+ // 01xx0->2S, 01xx1->4S, 1xxx1->2D, all others undefined.
+ static const NEONFormatMap map = {
+ {22, 21, 20, 19, 30},
+ {NF_UNDEF, NF_UNDEF, NF_8B, NF_16B, NF_4H, NF_8H, NF_4H, NF_8H,
+ NF_2S, NF_4S, NF_2S, NF_4S, NF_2S, NF_4S, NF_2S, NF_4S,
+ NF_UNDEF, NF_2D, NF_UNDEF, NF_2D, NF_UNDEF, NF_2D, NF_UNDEF, NF_2D,
+ NF_UNDEF, NF_2D, NF_UNDEF, NF_2D, NF_UNDEF, NF_2D, NF_UNDEF, NF_2D}
+ };
+ NEONFormatDecoder nfd(instr, &map);
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ // 0001->8H, 001x->4S, 01xx->2D, all others undefined.
+ static const NEONFormatMap map_l = {
+ {22, 21, 20, 19},
+ {NF_UNDEF, NF_8H, NF_4S, NF_4S, NF_2D, NF_2D, NF_2D, NF_2D}
+ };
+ VectorFormat vf_l = nfd.GetVectorFormat(&map_l);
+
+ int highestSetBit = HighestSetBitPosition(instr->ImmNEONImmh());
+ int immhimmb = instr->ImmNEONImmhImmb();
+ int right_shift = (16 << highestSetBit) - immhimmb;
+ int left_shift = immhimmb - (8 << highestSetBit);
+
+ switch (instr->Mask(NEONShiftImmediateMask)) {
+ case NEON_SHL: shl(vf, rd, rn, left_shift); break;
+ case NEON_SLI: sli(vf, rd, rn, left_shift); break;
+ case NEON_SQSHLU: sqshlu(vf, rd, rn, left_shift); break;
+ case NEON_SRI: sri(vf, rd, rn, right_shift); break;
+ case NEON_SSHR: sshr(vf, rd, rn, right_shift); break;
+ case NEON_USHR: ushr(vf, rd, rn, right_shift); break;
+ case NEON_SRSHR: sshr(vf, rd, rn, right_shift).Round(vf); break;
+ case NEON_URSHR: ushr(vf, rd, rn, right_shift).Round(vf); break;
+ case NEON_SSRA: ssra(vf, rd, rn, right_shift); break;
+ case NEON_USRA: usra(vf, rd, rn, right_shift); break;
+ case NEON_SRSRA: srsra(vf, rd, rn, right_shift); break;
+ case NEON_URSRA: ursra(vf, rd, rn, right_shift); break;
+ case NEON_SQSHL_imm: sqshl(vf, rd, rn, left_shift); break;
+ case NEON_UQSHL_imm: uqshl(vf, rd, rn, left_shift); break;
+ case NEON_SCVTF_imm: scvtf(vf, rd, rn, right_shift, fpcr_rounding); break;
+ case NEON_UCVTF_imm: ucvtf(vf, rd, rn, right_shift, fpcr_rounding); break;
+ case NEON_FCVTZS_imm: fcvts(vf, rd, rn, FPZero, right_shift); break;
+ case NEON_FCVTZU_imm: fcvtu(vf, rd, rn, FPZero, right_shift); break;
+ case NEON_SSHLL:
+ vf = vf_l;
+ if (instr->Mask(NEON_Q)) {
+ sshll2(vf, rd, rn, left_shift);
+ } else {
+ sshll(vf, rd, rn, left_shift);
+ }
+ break;
+ case NEON_USHLL:
+ vf = vf_l;
+ if (instr->Mask(NEON_Q)) {
+ ushll2(vf, rd, rn, left_shift);
+ } else {
+ ushll(vf, rd, rn, left_shift);
+ }
+ break;
+ case NEON_SHRN:
+ if (instr->Mask(NEON_Q)) {
+ shrn2(vf, rd, rn, right_shift);
+ } else {
+ shrn(vf, rd, rn, right_shift);
+ }
+ break;
+ case NEON_RSHRN:
+ if (instr->Mask(NEON_Q)) {
+ rshrn2(vf, rd, rn, right_shift);
+ } else {
+ rshrn(vf, rd, rn, right_shift);
+ }
+ break;
+ case NEON_UQSHRN:
+ if (instr->Mask(NEON_Q)) {
+ uqshrn2(vf, rd, rn, right_shift);
+ } else {
+ uqshrn(vf, rd, rn, right_shift);
+ }
+ break;
+ case NEON_UQRSHRN:
+ if (instr->Mask(NEON_Q)) {
+ uqrshrn2(vf, rd, rn, right_shift);
+ } else {
+ uqrshrn(vf, rd, rn, right_shift);
+ }
+ break;
+ case NEON_SQSHRN:
+ if (instr->Mask(NEON_Q)) {
+ sqshrn2(vf, rd, rn, right_shift);
+ } else {
+ sqshrn(vf, rd, rn, right_shift);
+ }
+ break;
+ case NEON_SQRSHRN:
+ if (instr->Mask(NEON_Q)) {
+ sqrshrn2(vf, rd, rn, right_shift);
+ } else {
+ sqrshrn(vf, rd, rn, right_shift);
+ }
+ break;
+ case NEON_SQSHRUN:
+ if (instr->Mask(NEON_Q)) {
+ sqshrun2(vf, rd, rn, right_shift);
+ } else {
+ sqshrun(vf, rd, rn, right_shift);
+ }
+ break;
+ case NEON_SQRSHRUN:
+ if (instr->Mask(NEON_Q)) {
+ sqrshrun2(vf, rd, rn, right_shift);
+ } else {
+ sqrshrun(vf, rd, rn, right_shift);
+ }
+ break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitNEONTable(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr, NEONFormatDecoder::LogicalFormatMap());
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ SimVRegister& rn2 = vreg((instr->Rn() + 1) % kNumberOfVRegisters);
+ SimVRegister& rn3 = vreg((instr->Rn() + 2) % kNumberOfVRegisters);
+ SimVRegister& rn4 = vreg((instr->Rn() + 3) % kNumberOfVRegisters);
+ SimVRegister& rm = vreg(instr->Rm());
+
+ switch (instr->Mask(NEONTableMask)) {
+ case NEON_TBL_1v: tbl(vf, rd, rn, rm); break;
+ case NEON_TBL_2v: tbl(vf, rd, rn, rn2, rm); break;
+ case NEON_TBL_3v: tbl(vf, rd, rn, rn2, rn3, rm); break;
+ case NEON_TBL_4v: tbl(vf, rd, rn, rn2, rn3, rn4, rm); break;
+ case NEON_TBX_1v: tbx(vf, rd, rn, rm); break;
+ case NEON_TBX_2v: tbx(vf, rd, rn, rn2, rm); break;
+ case NEON_TBX_3v: tbx(vf, rd, rn, rn2, rn3, rm); break;
+ case NEON_TBX_4v: tbx(vf, rd, rn, rn2, rn3, rn4, rm); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::VisitNEONPerm(const Instruction* instr) {
+ NEONFormatDecoder nfd(instr);
+ VectorFormat vf = nfd.GetVectorFormat();
+
+ SimVRegister& rd = vreg(instr->Rd());
+ SimVRegister& rn = vreg(instr->Rn());
+ SimVRegister& rm = vreg(instr->Rm());
+
+ switch (instr->Mask(NEONPermMask)) {
+ case NEON_TRN1: trn1(vf, rd, rn, rm); break;
+ case NEON_TRN2: trn2(vf, rd, rn, rm); break;
+ case NEON_UZP1: uzp1(vf, rd, rn, rm); break;
+ case NEON_UZP2: uzp2(vf, rd, rn, rm); break;
+ case NEON_ZIP1: zip1(vf, rd, rn, rm); break;
+ case NEON_ZIP2: zip2(vf, rd, rn, rm); break;
+ default:
+ VIXL_UNIMPLEMENTED();
+ }
+}
+
+
+void Simulator::DoUnreachable(const Instruction* instr) {
+ VIXL_ASSERT(instr->InstructionBits() == UNDEFINED_INST_PATTERN);
+
+ fprintf(stream_, "Hit UNREACHABLE marker at pc=%p.\n",
+ reinterpret_cast<const void*>(instr));
+ abort();
+}
+
+
+void Simulator::DoTrace(const Instruction* instr) {
+ VIXL_ASSERT((instr->Mask(ExceptionMask) == HLT) &&
+ (instr->ImmException() == kTraceOpcode));
+
+ // Read the arguments encoded inline in the instruction stream.
+ uint32_t parameters;
+ uint32_t command;
+
+ VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
+ memcpy(&parameters, instr + kTraceParamsOffset, sizeof(parameters));
+ memcpy(&command, instr + kTraceCommandOffset, sizeof(command));
+
+ switch (command) {
+ case TRACE_ENABLE:
+ set_trace_parameters(trace_parameters() | parameters);
+ break;
+ case TRACE_DISABLE:
+ set_trace_parameters(trace_parameters() & ~parameters);
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ }
+
+ set_pc(instr->InstructionAtOffset(kTraceLength));
+}
+
+
+void Simulator::DoLog(const Instruction* instr) {
+ VIXL_ASSERT((instr->Mask(ExceptionMask) == HLT) &&
+ (instr->ImmException() == kLogOpcode));
+
+ // Read the arguments encoded inline in the instruction stream.
+ uint32_t parameters;
+
+ VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
+ memcpy(&parameters, instr + kTraceParamsOffset, sizeof(parameters));
+
+ // We don't support a one-shot LOG_DISASM.
+ VIXL_ASSERT((parameters & LOG_DISASM) == 0);
+ // Print the requested information.
+ if (parameters & LOG_SYSREGS) PrintSystemRegisters();
+ if (parameters & LOG_REGS) PrintRegisters();
+ if (parameters & LOG_VREGS) PrintVRegisters();
+
+ set_pc(instr->InstructionAtOffset(kLogLength));
+}
+
+
+void Simulator::DoPrintf(const Instruction* instr) {
+ VIXL_ASSERT((instr->Mask(ExceptionMask) == HLT) &&
+ (instr->ImmException() == kPrintfOpcode));
+
+ // Read the arguments encoded inline in the instruction stream.
+ uint32_t arg_count;
+ uint32_t arg_pattern_list;
+ VIXL_STATIC_ASSERT(sizeof(*instr) == 1);
+ memcpy(&arg_count,
+ instr + kPrintfArgCountOffset,
+ sizeof(arg_count));
+ memcpy(&arg_pattern_list,
+ instr + kPrintfArgPatternListOffset,
+ sizeof(arg_pattern_list));
+
+ VIXL_ASSERT(arg_count <= kPrintfMaxArgCount);
+ VIXL_ASSERT((arg_pattern_list >> (kPrintfArgPatternBits * arg_count)) == 0);
+
+ // We need to call the host printf function with a set of arguments defined by
+ // arg_pattern_list. Because we don't know the types and sizes of the
+ // arguments, this is very difficult to do in a robust and portable way. To
+ // work around the problem, we pick apart the format string, and print one
+ // format placeholder at a time.
+
+ // Allocate space for the format string. We take a copy, so we can modify it.
+ // Leave enough space for one extra character per expected argument (plus the
+ // '\0' termination).
+ const char * format_base = reg<const char *>(0);
+ VIXL_ASSERT(format_base != NULL);
+ size_t length = strlen(format_base) + 1;
+ char * const format = (char *)js_calloc(length + arg_count);
+
+ // A list of chunks, each with exactly one format placeholder.
+ const char * chunks[kPrintfMaxArgCount];
+
+ // Copy the format string and search for format placeholders.
+ uint32_t placeholder_count = 0;
+ char * format_scratch = format;
+ for (size_t i = 0; i < length; i++) {
+ if (format_base[i] != '%') {
+ *format_scratch++ = format_base[i];
+ } else {
+ if (format_base[i + 1] == '%') {
+ // Ignore explicit "%%" sequences.
+ *format_scratch++ = format_base[i];
+ i++;
+ // Chunks after the first are passed as format strings to printf, so we
+ // need to escape '%' characters in those chunks.
+ if (placeholder_count > 0) *format_scratch++ = format_base[i];
+ } else {
+ VIXL_CHECK(placeholder_count < arg_count);
+ // Insert '\0' before placeholders, and store their locations.
+ *format_scratch++ = '\0';
+ chunks[placeholder_count++] = format_scratch;
+ *format_scratch++ = format_base[i];
+ }
+ }
+ }
+ VIXL_CHECK(placeholder_count == arg_count);
+
+ // Finally, call printf with each chunk, passing the appropriate register
+ // argument. Normally, printf returns the number of bytes transmitted, so we
+ // can emulate a single printf call by adding the result from each chunk. If
+ // any call returns a negative (error) value, though, just return that value.
+
+ printf("%s", clr_printf);
+
+ // Because '\0' is inserted before each placeholder, the first string in
+ // 'format' contains no format placeholders and should be printed literally.
+ int result = printf("%s", format);
+ int pcs_r = 1; // Start at x1. x0 holds the format string.
+ int pcs_f = 0; // Start at d0.
+ if (result >= 0) {
+ for (uint32_t i = 0; i < placeholder_count; i++) {
+ int part_result = -1;
+
+ uint32_t arg_pattern = arg_pattern_list >> (i * kPrintfArgPatternBits);
+ arg_pattern &= (1 << kPrintfArgPatternBits) - 1;
+ switch (arg_pattern) {
+ case kPrintfArgW: part_result = printf(chunks[i], wreg(pcs_r++)); break;
+ case kPrintfArgX: part_result = printf(chunks[i], xreg(pcs_r++)); break;
+ case kPrintfArgD: part_result = printf(chunks[i], dreg(pcs_f++)); break;
+ default: VIXL_UNREACHABLE();
+ }
+
+ if (part_result < 0) {
+ // Handle error values.
+ result = part_result;
+ break;
+ }
+
+ result += part_result;
+ }
+ }
+
+ printf("%s", clr_normal);
+
+ // Printf returns its result in x0 (just like the C library's printf).
+ set_xreg(0, result);
+
+ // The printf parameters are inlined in the code, so skip them.
+ set_pc(instr->InstructionAtOffset(kPrintfLength));
+
+ // Set LR as if we'd just called a native printf function.
+ set_lr(pc());
+
+ js_free(format);
+}
+
+} // namespace vixl
+
+#endif // JS_SIMULATOR_ARM64
diff --git a/js/src/jit/arm64/vixl/Simulator-vixl.h b/js/src/jit/arm64/vixl/Simulator-vixl.h
new file mode 100644
index 0000000000..a0515bee06
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Simulator-vixl.h
@@ -0,0 +1,2571 @@
+// Copyright 2015, ARM Limited
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_A64_SIMULATOR_A64_H_
+#define VIXL_A64_SIMULATOR_A64_H_
+
+#include "jstypes.h"
+
+#ifdef JS_SIMULATOR_ARM64
+
+#include "mozilla/Vector.h"
+
+#include "jit/arm64/vixl/Assembler-vixl.h"
+#include "jit/arm64/vixl/Disasm-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+#include "jit/arm64/vixl/Instructions-vixl.h"
+#include "jit/arm64/vixl/Instrument-vixl.h"
+#include "jit/arm64/vixl/MozCachingDecoder.h"
+#include "jit/arm64/vixl/Simulator-Constants-vixl.h"
+#include "jit/arm64/vixl/Utils-vixl.h"
+#include "jit/IonTypes.h"
+#include "js/AllocPolicy.h"
+#include "vm/MutexIDs.h"
+#include "wasm/WasmSignalHandlers.h"
+
+namespace vixl {
+
+// Representation of memory, with typed getters and setters for access.
+class Memory {
+ public:
+ template <typename T>
+ static T AddressUntag(T address) {
+ // Cast the address using a C-style cast. A reinterpret_cast would be
+ // appropriate, but it can't cast one integral type to another.
+ uint64_t bits = (uint64_t)address;
+ return (T)(bits & ~kAddressTagMask);
+ }
+
+ template <typename T, typename A>
+ static T Read(A address) {
+ T value;
+ address = AddressUntag(address);
+ VIXL_ASSERT((sizeof(value) == 1) || (sizeof(value) == 2) ||
+ (sizeof(value) == 4) || (sizeof(value) == 8) ||
+ (sizeof(value) == 16));
+ memcpy(&value, reinterpret_cast<const char *>(address), sizeof(value));
+ return value;
+ }
+
+ template <typename T, typename A>
+ static void Write(A address, T value) {
+ address = AddressUntag(address);
+ VIXL_ASSERT((sizeof(value) == 1) || (sizeof(value) == 2) ||
+ (sizeof(value) == 4) || (sizeof(value) == 8) ||
+ (sizeof(value) == 16));
+ memcpy(reinterpret_cast<char *>(address), &value, sizeof(value));
+ }
+};
+
+// Represent a register (r0-r31, v0-v31).
+template<int kSizeInBytes>
+class SimRegisterBase {
+ public:
+ SimRegisterBase() : written_since_last_log_(false) {}
+
+ // Write the specified value. The value is zero-extended if necessary.
+ template<typename T>
+ void Set(T new_value) {
+ VIXL_STATIC_ASSERT(sizeof(new_value) <= kSizeInBytes);
+ if (sizeof(new_value) < kSizeInBytes) {
+ // All AArch64 registers are zero-extending.
+ memset(value_ + sizeof(new_value), 0, kSizeInBytes - sizeof(new_value));
+ }
+ memcpy(value_, &new_value, sizeof(new_value));
+ NotifyRegisterWrite();
+ }
+
+ // Insert a typed value into a register, leaving the rest of the register
+ // unchanged. The lane parameter indicates where in the register the value
+ // should be inserted, in the range [ 0, sizeof(value_) / sizeof(T) ), where
+ // 0 represents the least significant bits.
+ template<typename T>
+ void Insert(int lane, T new_value) {
+ VIXL_ASSERT(lane >= 0);
+ VIXL_ASSERT((sizeof(new_value) +
+ (lane * sizeof(new_value))) <= kSizeInBytes);
+ memcpy(&value_[lane * sizeof(new_value)], &new_value, sizeof(new_value));
+ NotifyRegisterWrite();
+ }
+
+ // Read the value as the specified type. The value is truncated if necessary.
+ template<typename T>
+ T Get(int lane = 0) const {
+ T result;
+ VIXL_ASSERT(lane >= 0);
+ VIXL_ASSERT((sizeof(result) + (lane * sizeof(result))) <= kSizeInBytes);
+ memcpy(&result, &value_[lane * sizeof(result)], sizeof(result));
+ return result;
+ }
+
+ // TODO: Make this return a map of updated bytes, so that we can highlight
+ // updated lanes for load-and-insert. (That never happens for scalar code, but
+ // NEON has some instructions that can update individual lanes.)
+ bool WrittenSinceLastLog() const {
+ return written_since_last_log_;
+ }
+
+ void NotifyRegisterLogged() {
+ written_since_last_log_ = false;
+ }
+
+ protected:
+ uint8_t value_[kSizeInBytes];
+
+ // Helpers to aid with register tracing.
+ bool written_since_last_log_;
+
+ void NotifyRegisterWrite() {
+ written_since_last_log_ = true;
+ }
+};
+typedef SimRegisterBase<kXRegSizeInBytes> SimRegister; // r0-r31
+typedef SimRegisterBase<kQRegSizeInBytes> SimVRegister; // v0-v31
+
+// Representation of a vector register, with typed getters and setters for lanes
+// and additional information to represent lane state.
+class LogicVRegister {
+ public:
+ inline LogicVRegister(SimVRegister& other) // NOLINT
+ : register_(other) {
+ for (unsigned i = 0; i < sizeof(saturated_) / sizeof(saturated_[0]); i++) {
+ saturated_[i] = kNotSaturated;
+ }
+ for (unsigned i = 0; i < sizeof(round_) / sizeof(round_[0]); i++) {
+ round_[i] = 0;
+ }
+ }
+
+ int64_t Int(VectorFormat vform, int index) const {
+ int64_t element;
+ switch (LaneSizeInBitsFromFormat(vform)) {
+ case 8: element = register_.Get<int8_t>(index); break;
+ case 16: element = register_.Get<int16_t>(index); break;
+ case 32: element = register_.Get<int32_t>(index); break;
+ case 64: element = register_.Get<int64_t>(index); break;
+ default: VIXL_UNREACHABLE(); return 0;
+ }
+ return element;
+ }
+
+ uint64_t Uint(VectorFormat vform, int index) const {
+ uint64_t element;
+ switch (LaneSizeInBitsFromFormat(vform)) {
+ case 8: element = register_.Get<uint8_t>(index); break;
+ case 16: element = register_.Get<uint16_t>(index); break;
+ case 32: element = register_.Get<uint32_t>(index); break;
+ case 64: element = register_.Get<uint64_t>(index); break;
+ default: VIXL_UNREACHABLE(); return 0;
+ }
+ return element;
+ }
+
+ int64_t IntLeftJustified(VectorFormat vform, int index) const {
+ return Int(vform, index) << (64 - LaneSizeInBitsFromFormat(vform));
+ }
+
+ uint64_t UintLeftJustified(VectorFormat vform, int index) const {
+ return Uint(vform, index) << (64 - LaneSizeInBitsFromFormat(vform));
+ }
+
+ void SetInt(VectorFormat vform, int index, int64_t value) const {
+ switch (LaneSizeInBitsFromFormat(vform)) {
+ case 8: register_.Insert(index, static_cast<int8_t>(value)); break;
+ case 16: register_.Insert(index, static_cast<int16_t>(value)); break;
+ case 32: register_.Insert(index, static_cast<int32_t>(value)); break;
+ case 64: register_.Insert(index, static_cast<int64_t>(value)); break;
+ default: VIXL_UNREACHABLE(); return;
+ }
+ }
+
+ void SetUint(VectorFormat vform, int index, uint64_t value) const {
+ switch (LaneSizeInBitsFromFormat(vform)) {
+ case 8: register_.Insert(index, static_cast<uint8_t>(value)); break;
+ case 16: register_.Insert(index, static_cast<uint16_t>(value)); break;
+ case 32: register_.Insert(index, static_cast<uint32_t>(value)); break;
+ case 64: register_.Insert(index, static_cast<uint64_t>(value)); break;
+ default: VIXL_UNREACHABLE(); return;
+ }
+ }
+
+ void ReadUintFromMem(VectorFormat vform, int index, uint64_t addr) const {
+ switch (LaneSizeInBitsFromFormat(vform)) {
+ case 8: register_.Insert(index, Memory::Read<uint8_t>(addr)); break;
+ case 16: register_.Insert(index, Memory::Read<uint16_t>(addr)); break;
+ case 32: register_.Insert(index, Memory::Read<uint32_t>(addr)); break;
+ case 64: register_.Insert(index, Memory::Read<uint64_t>(addr)); break;
+ default: VIXL_UNREACHABLE(); return;
+ }
+ }
+
+ void WriteUintToMem(VectorFormat vform, int index, uint64_t addr) const {
+ uint64_t value = Uint(vform, index);
+ switch (LaneSizeInBitsFromFormat(vform)) {
+ case 8: Memory::Write(addr, static_cast<uint8_t>(value)); break;
+ case 16: Memory::Write(addr, static_cast<uint16_t>(value)); break;
+ case 32: Memory::Write(addr, static_cast<uint32_t>(value)); break;
+ case 64: Memory::Write(addr, value); break;
+ }
+ }
+
+ template <typename T>
+ T Float(int index) const {
+ return register_.Get<T>(index);
+ }
+
+ template <typename T>
+ void SetFloat(int index, T value) const {
+ register_.Insert(index, value);
+ }
+
+ // When setting a result in a register of size less than Q, the top bits of
+ // the Q register must be cleared.
+ void ClearForWrite(VectorFormat vform) const {
+ unsigned size = RegisterSizeInBytesFromFormat(vform);
+ for (unsigned i = size; i < kQRegSizeInBytes; i++) {
+ SetUint(kFormat16B, i, 0);
+ }
+ }
+
+ // Saturation state for each lane of a vector.
+ enum Saturation {
+ kNotSaturated = 0,
+ kSignedSatPositive = 1 << 0,
+ kSignedSatNegative = 1 << 1,
+ kSignedSatMask = kSignedSatPositive | kSignedSatNegative,
+ kSignedSatUndefined = kSignedSatMask,
+ kUnsignedSatPositive = 1 << 2,
+ kUnsignedSatNegative = 1 << 3,
+ kUnsignedSatMask = kUnsignedSatPositive | kUnsignedSatNegative,
+ kUnsignedSatUndefined = kUnsignedSatMask
+ };
+
+ // Getters for saturation state.
+ Saturation GetSignedSaturation(int index) {
+ return static_cast<Saturation>(saturated_[index] & kSignedSatMask);
+ }
+
+ Saturation GetUnsignedSaturation(int index) {
+ return static_cast<Saturation>(saturated_[index] & kUnsignedSatMask);
+ }
+
+ // Setters for saturation state.
+ void ClearSat(int index) {
+ saturated_[index] = kNotSaturated;
+ }
+
+ void SetSignedSat(int index, bool positive) {
+ SetSatFlag(index, positive ? kSignedSatPositive : kSignedSatNegative);
+ }
+
+ void SetUnsignedSat(int index, bool positive) {
+ SetSatFlag(index, positive ? kUnsignedSatPositive : kUnsignedSatNegative);
+ }
+
+ void SetSatFlag(int index, Saturation sat) {
+ saturated_[index] = static_cast<Saturation>(saturated_[index] | sat);
+ VIXL_ASSERT((sat & kUnsignedSatMask) != kUnsignedSatUndefined);
+ VIXL_ASSERT((sat & kSignedSatMask) != kSignedSatUndefined);
+ }
+
+ // Saturate lanes of a vector based on saturation state.
+ LogicVRegister& SignedSaturate(VectorFormat vform) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ Saturation sat = GetSignedSaturation(i);
+ if (sat == kSignedSatPositive) {
+ SetInt(vform, i, MaxIntFromFormat(vform));
+ } else if (sat == kSignedSatNegative) {
+ SetInt(vform, i, MinIntFromFormat(vform));
+ }
+ }
+ return *this;
+ }
+
+ LogicVRegister& UnsignedSaturate(VectorFormat vform) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ Saturation sat = GetUnsignedSaturation(i);
+ if (sat == kUnsignedSatPositive) {
+ SetUint(vform, i, MaxUintFromFormat(vform));
+ } else if (sat == kUnsignedSatNegative) {
+ SetUint(vform, i, 0);
+ }
+ }
+ return *this;
+ }
+
+ // Getter for rounding state.
+ bool GetRounding(int index) {
+ return round_[index];
+ }
+
+ // Setter for rounding state.
+ void SetRounding(int index, bool round) {
+ round_[index] = round;
+ }
+
+ // Round lanes of a vector based on rounding state.
+ LogicVRegister& Round(VectorFormat vform) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ SetInt(vform, i, Int(vform, i) + (GetRounding(i) ? 1 : 0));
+ }
+ return *this;
+ }
+
+ // Unsigned halve lanes of a vector, and use the saturation state to set the
+ // top bit.
+ LogicVRegister& Uhalve(VectorFormat vform) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ uint64_t val = Uint(vform, i);
+ SetRounding(i, (val & 1) == 1);
+ val >>= 1;
+ if (GetUnsignedSaturation(i) != kNotSaturated) {
+ // If the operation causes unsigned saturation, the bit shifted into the
+ // most significant bit must be set.
+ val |= (MaxUintFromFormat(vform) >> 1) + 1;
+ }
+ SetInt(vform, i, val);
+ }
+ return *this;
+ }
+
+ // Signed halve lanes of a vector, and use the carry state to set the top bit.
+ LogicVRegister& Halve(VectorFormat vform) {
+ for (int i = 0; i < LaneCountFromFormat(vform); i++) {
+ int64_t val = Int(vform, i);
+ SetRounding(i, (val & 1) == 1);
+ val >>= 1;
+ if (GetSignedSaturation(i) != kNotSaturated) {
+ // If the operation causes signed saturation, the sign bit must be
+ // inverted.
+ val ^= (MaxUintFromFormat(vform) >> 1) + 1;
+ }
+ SetInt(vform, i, val);
+ }
+ return *this;
+ }
+
+ private:
+ SimVRegister& register_;
+
+ // Allocate one saturation state entry per lane; largest register is type Q,
+ // and lanes can be a minimum of one byte wide.
+ Saturation saturated_[kQRegSizeInBytes];
+
+ // Allocate one rounding state entry per lane.
+ bool round_[kQRegSizeInBytes];
+};
+
+// The proper way to initialize a simulated system register (such as NZCV) is as
+// follows:
+// SimSystemRegister nzcv = SimSystemRegister::DefaultValueFor(NZCV);
+class SimSystemRegister {
+ public:
+ // The default constructor represents a register which has no writable bits.
+ // It is not possible to set its value to anything other than 0.
+ SimSystemRegister() : value_(0), write_ignore_mask_(0xffffffff) { }
+
+ uint32_t RawValue() const {
+ return value_;
+ }
+
+ void SetRawValue(uint32_t new_value) {
+ value_ = (value_ & write_ignore_mask_) | (new_value & ~write_ignore_mask_);
+ }
+
+ uint32_t Bits(int msb, int lsb) const {
+ return ExtractUnsignedBitfield32(msb, lsb, value_);
+ }
+
+ int32_t SignedBits(int msb, int lsb) const {
+ return ExtractSignedBitfield32(msb, lsb, value_);
+ }
+
+ void SetBits(int msb, int lsb, uint32_t bits);
+
+ // Default system register values.
+ static SimSystemRegister DefaultValueFor(SystemRegister id);
+
+#define DEFINE_GETTER(Name, HighBit, LowBit, Func) \
+ uint32_t Name() const { return Func(HighBit, LowBit); } \
+ void Set##Name(uint32_t bits) { SetBits(HighBit, LowBit, bits); }
+#define DEFINE_WRITE_IGNORE_MASK(Name, Mask) \
+ static const uint32_t Name##WriteIgnoreMask = ~static_cast<uint32_t>(Mask);
+
+ SYSTEM_REGISTER_FIELDS_LIST(DEFINE_GETTER, DEFINE_WRITE_IGNORE_MASK)
+
+#undef DEFINE_ZERO_BITS
+#undef DEFINE_GETTER
+
+ protected:
+ // Most system registers only implement a few of the bits in the word. Other
+ // bits are "read-as-zero, write-ignored". The write_ignore_mask argument
+ // describes the bits which are not modifiable.
+ SimSystemRegister(uint32_t value, uint32_t write_ignore_mask)
+ : value_(value), write_ignore_mask_(write_ignore_mask) { }
+
+ uint32_t value_;
+ uint32_t write_ignore_mask_;
+};
+
+
+class SimExclusiveLocalMonitor {
+ public:
+ SimExclusiveLocalMonitor() : kSkipClearProbability(8), seed_(0x87654321) {
+ Clear();
+ }
+
+ // Clear the exclusive monitor (like clrex).
+ void Clear() {
+ address_ = 0;
+ size_ = 0;
+ }
+
+ // Clear the exclusive monitor most of the time.
+ void MaybeClear() {
+ if ((seed_ % kSkipClearProbability) != 0) {
+ Clear();
+ }
+
+ // Advance seed_ using a simple linear congruential generator.
+ seed_ = (seed_ * 48271) % 2147483647;
+ }
+
+ // Mark the address range for exclusive access (like load-exclusive).
+ void MarkExclusive(uint64_t address, size_t size) {
+ address_ = address;
+ size_ = size;
+ }
+
+ // Return true if the address range is marked (like store-exclusive).
+ // This helper doesn't implicitly clear the monitor.
+ bool IsExclusive(uint64_t address, size_t size) {
+ VIXL_ASSERT(size > 0);
+ // Be pedantic: Require both the address and the size to match.
+ return (size == size_) && (address == address_);
+ }
+
+ private:
+ uint64_t address_;
+ size_t size_;
+
+ const int kSkipClearProbability;
+ uint32_t seed_;
+};
+
+
+// We can't accurate simulate the global monitor since it depends on external
+// influences. Instead, this implementation occasionally causes accesses to
+// fail, according to kPassProbability.
+class SimExclusiveGlobalMonitor {
+ public:
+ SimExclusiveGlobalMonitor() : kPassProbability(8), seed_(0x87654321) {}
+
+ bool IsExclusive(uint64_t address, size_t size) {
+ USE(address, size);
+
+ bool pass = (seed_ % kPassProbability) != 0;
+ // Advance seed_ using a simple linear congruential generator.
+ seed_ = (seed_ * 48271) % 2147483647;
+ return pass;
+ }
+
+ private:
+ const int kPassProbability;
+ uint32_t seed_;
+};
+
+class Redirection;
+
+class Simulator : public DecoderVisitor {
+ public:
+#ifdef JS_CACHE_SIMULATOR_ARM64
+ using Decoder = CachingDecoder;
+ mozilla::Atomic<bool> pendingCacheRequests = mozilla::Atomic<bool>{ false };
+#endif
+ explicit Simulator(Decoder* decoder, FILE* stream = stdout);
+ ~Simulator();
+
+ // Moz changes.
+ void init(Decoder* decoder, FILE* stream);
+ static Simulator* Current();
+ static Simulator* Create();
+ static void Destroy(Simulator* sim);
+ uintptr_t stackLimit() const;
+ uintptr_t* addressOfStackLimit();
+ bool overRecursed(uintptr_t newsp = 0) const;
+ bool overRecursedWithExtra(uint32_t extra) const;
+ int64_t call(uint8_t* entry, int argument_count, ...);
+ static void* RedirectNativeFunction(void* nativeFunction, js::jit::ABIFunctionType type);
+ void setGPR32Result(int32_t result);
+ void setGPR64Result(int64_t result);
+ void setFP32Result(float result);
+ void setFP64Result(double result);
+#ifdef JS_CACHE_SIMULATOR_ARM64
+ void FlushICache();
+#endif
+ void VisitCallRedirection(const Instruction* instr);
+ static uintptr_t StackLimit() {
+ return Simulator::Current()->stackLimit();
+ }
+ template<typename T> T Read(uintptr_t address);
+ template <typename T> void Write(uintptr_t address_, T value);
+ JS::ProfilingFrameIterator::RegisterState registerState();
+
+ void ResetState();
+
+ // Run the simulator.
+ virtual void Run();
+ void RunFrom(const Instruction* first);
+
+ // Simulation helpers.
+ const Instruction* pc() const { return pc_; }
+ const Instruction* get_pc() const { return pc_; }
+ int64_t get_sp() const { return xreg(31, Reg31IsStackPointer); }
+ int64_t get_lr() const { return xreg(30); }
+ int64_t get_fp() const { return xreg(29); }
+
+ template <typename T>
+ T get_pc_as() const { return reinterpret_cast<T>(const_cast<Instruction*>(pc())); }
+
+ void set_pc(const Instruction* new_pc) {
+ pc_ = Memory::AddressUntag(new_pc);
+ pc_modified_ = true;
+ }
+
+ // Handle any wasm faults, returning true if the fault was handled.
+ // This method is rather hot so inline the normal (no-wasm) case.
+ bool MOZ_ALWAYS_INLINE handle_wasm_seg_fault(uintptr_t addr, unsigned numBytes) {
+ if (MOZ_LIKELY(!js::wasm::CodeExists)) {
+ return false;
+ }
+
+ uint8_t* newPC;
+ if (!js::wasm::MemoryAccessTraps(registerState(), (uint8_t*)addr, numBytes, &newPC)) {
+ return false;
+ }
+
+ set_pc((Instruction*)newPC);
+ return true;
+ }
+
+ void increment_pc() {
+ if (!pc_modified_) {
+ pc_ = pc_->NextInstruction();
+ }
+
+ pc_modified_ = false;
+ }
+
+ void ExecuteInstruction();
+
+ // Declare all Visitor functions.
+ #define DECLARE(A) virtual void Visit##A(const Instruction* instr) override;
+ VISITOR_LIST_THAT_RETURN(DECLARE)
+ VISITOR_LIST_THAT_DONT_RETURN(DECLARE)
+ #undef DECLARE
+
+
+ // Integer register accessors.
+
+ // Basic accessor: Read the register as the specified type.
+ template<typename T>
+ T reg(unsigned code, Reg31Mode r31mode = Reg31IsZeroRegister) const {
+ VIXL_ASSERT(code < kNumberOfRegisters);
+ if ((code == 31) && (r31mode == Reg31IsZeroRegister)) {
+ T result;
+ memset(&result, 0, sizeof(result));
+ return result;
+ }
+ return registers_[code].Get<T>();
+ }
+
+ // Common specialized accessors for the reg() template.
+ int32_t wreg(unsigned code,
+ Reg31Mode r31mode = Reg31IsZeroRegister) const {
+ return reg<int32_t>(code, r31mode);
+ }
+
+ int64_t xreg(unsigned code,
+ Reg31Mode r31mode = Reg31IsZeroRegister) const {
+ return reg<int64_t>(code, r31mode);
+ }
+
+ // As above, with parameterized size and return type. The value is
+ // either zero-extended or truncated to fit, as required.
+ template<typename T>
+ T reg(unsigned size, unsigned code,
+ Reg31Mode r31mode = Reg31IsZeroRegister) const {
+ uint64_t raw;
+ switch (size) {
+ case kWRegSize: raw = reg<uint32_t>(code, r31mode); break;
+ case kXRegSize: raw = reg<uint64_t>(code, r31mode); break;
+ default:
+ VIXL_UNREACHABLE();
+ return 0;
+ }
+
+ T result;
+ VIXL_STATIC_ASSERT(sizeof(result) <= sizeof(raw));
+ // Copy the result and truncate to fit. This assumes a little-endian host.
+ memcpy(&result, &raw, sizeof(result));
+ return result;
+ }
+
+ // Use int64_t by default if T is not specified.
+ int64_t reg(unsigned size, unsigned code,
+ Reg31Mode r31mode = Reg31IsZeroRegister) const {
+ return reg<int64_t>(size, code, r31mode);
+ }
+
+ enum RegLogMode {
+ LogRegWrites,
+ NoRegLog
+ };
+
+ // Write 'value' into an integer register. The value is zero-extended. This
+ // behaviour matches AArch64 register writes.
+ template<typename T>
+ void set_reg(unsigned code, T value,
+ RegLogMode log_mode = LogRegWrites,
+ Reg31Mode r31mode = Reg31IsZeroRegister) {
+ VIXL_STATIC_ASSERT((sizeof(T) == kWRegSizeInBytes) ||
+ (sizeof(T) == kXRegSizeInBytes));
+ VIXL_ASSERT(code < kNumberOfRegisters);
+
+ if ((code == 31) && (r31mode == Reg31IsZeroRegister)) {
+ return;
+ }
+
+ registers_[code].Set(value);
+
+ if (log_mode == LogRegWrites) LogRegister(code, r31mode);
+ }
+
+ // Common specialized accessors for the set_reg() template.
+ void set_wreg(unsigned code, int32_t value,
+ RegLogMode log_mode = LogRegWrites,
+ Reg31Mode r31mode = Reg31IsZeroRegister) {
+ set_reg(code, value, log_mode, r31mode);
+ }
+
+ void set_xreg(unsigned code, int64_t value,
+ RegLogMode log_mode = LogRegWrites,
+ Reg31Mode r31mode = Reg31IsZeroRegister) {
+ set_reg(code, value, log_mode, r31mode);
+ }
+
+ // As above, with parameterized size and type. The value is either
+ // zero-extended or truncated to fit, as required.
+ template<typename T>
+ void set_reg(unsigned size, unsigned code, T value,
+ RegLogMode log_mode = LogRegWrites,
+ Reg31Mode r31mode = Reg31IsZeroRegister) {
+ // Zero-extend the input.
+ uint64_t raw = 0;
+ VIXL_STATIC_ASSERT(sizeof(value) <= sizeof(raw));
+ memcpy(&raw, &value, sizeof(value));
+
+ // Write (and possibly truncate) the value.
+ switch (size) {
+ case kWRegSize:
+ set_reg(code, static_cast<uint32_t>(raw), log_mode, r31mode);
+ break;
+ case kXRegSize:
+ set_reg(code, raw, log_mode, r31mode);
+ break;
+ default:
+ VIXL_UNREACHABLE();
+ return;
+ }
+ }
+
+ // Common specialized accessors for the set_reg() template.
+
+ // Commonly-used special cases.
+ template<typename T>
+ void set_lr(T value) {
+ set_reg(kLinkRegCode, value);
+ }
+
+ template<typename T>
+ void set_sp(T value) {
+ set_reg(31, value, LogRegWrites, Reg31IsStackPointer);
+ }
+
+ // Vector register accessors.
+ // These are equivalent to the integer register accessors, but for vector
+ // registers.
+
+ // A structure for representing a 128-bit Q register.
+ struct qreg_t { uint8_t val[kQRegSizeInBytes]; };
+
+ // Basic accessor: read the register as the specified type.
+ template<typename T>
+ T vreg(unsigned code) const {
+ VIXL_STATIC_ASSERT((sizeof(T) == kBRegSizeInBytes) ||
+ (sizeof(T) == kHRegSizeInBytes) ||
+ (sizeof(T) == kSRegSizeInBytes) ||
+ (sizeof(T) == kDRegSizeInBytes) ||
+ (sizeof(T) == kQRegSizeInBytes));
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+
+ return vregisters_[code].Get<T>();
+ }
+
+ // Common specialized accessors for the vreg() template.
+ int8_t breg(unsigned code) const {
+ return vreg<int8_t>(code);
+ }
+
+ int16_t hreg(unsigned code) const {
+ return vreg<int16_t>(code);
+ }
+
+ float sreg(unsigned code) const {
+ return vreg<float>(code);
+ }
+
+ uint32_t sreg_bits(unsigned code) const {
+ return vreg<uint32_t>(code);
+ }
+
+ double dreg(unsigned code) const {
+ return vreg<double>(code);
+ }
+
+ uint64_t dreg_bits(unsigned code) const {
+ return vreg<uint64_t>(code);
+ }
+
+ qreg_t qreg(unsigned code) const {
+ return vreg<qreg_t>(code);
+ }
+
+ // As above, with parameterized size and return type. The value is
+ // either zero-extended or truncated to fit, as required.
+ template<typename T>
+ T vreg(unsigned size, unsigned code) const {
+ uint64_t raw = 0;
+ T result;
+
+ switch (size) {
+ case kSRegSize: raw = vreg<uint32_t>(code); break;
+ case kDRegSize: raw = vreg<uint64_t>(code); break;
+ default:
+ VIXL_UNREACHABLE();
+ break;
+ }
+
+ VIXL_STATIC_ASSERT(sizeof(result) <= sizeof(raw));
+ // Copy the result and truncate to fit. This assumes a little-endian host.
+ memcpy(&result, &raw, sizeof(result));
+ return result;
+ }
+
+ inline SimVRegister& vreg(unsigned code) {
+ return vregisters_[code];
+ }
+
+ // Basic accessor: Write the specified value.
+ template<typename T>
+ void set_vreg(unsigned code, T value,
+ RegLogMode log_mode = LogRegWrites) {
+ VIXL_STATIC_ASSERT((sizeof(value) == kBRegSizeInBytes) ||
+ (sizeof(value) == kHRegSizeInBytes) ||
+ (sizeof(value) == kSRegSizeInBytes) ||
+ (sizeof(value) == kDRegSizeInBytes) ||
+ (sizeof(value) == kQRegSizeInBytes));
+ VIXL_ASSERT(code < kNumberOfVRegisters);
+ vregisters_[code].Set(value);
+
+ if (log_mode == LogRegWrites) {
+ LogVRegister(code, GetPrintRegisterFormat(value));
+ }
+ }
+
+ // Common specialized accessors for the set_vreg() template.
+ void set_breg(unsigned code, int8_t value,
+ RegLogMode log_mode = LogRegWrites) {
+ set_vreg(code, value, log_mode);
+ }
+
+ void set_hreg(unsigned code, int16_t value,
+ RegLogMode log_mode = LogRegWrites) {
+ set_vreg(code, value, log_mode);
+ }
+
+ void set_sreg(unsigned code, float value,
+ RegLogMode log_mode = LogRegWrites) {
+ set_vreg(code, value, log_mode);
+ }
+
+ void set_sreg_bits(unsigned code, uint32_t value,
+ RegLogMode log_mode = LogRegWrites) {
+ set_vreg(code, value, log_mode);
+ }
+
+ void set_dreg(unsigned code, double value,
+ RegLogMode log_mode = LogRegWrites) {
+ set_vreg(code, value, log_mode);
+ }
+
+ void set_dreg_bits(unsigned code, uint64_t value,
+ RegLogMode log_mode = LogRegWrites) {
+ set_vreg(code, value, log_mode);
+ }
+
+ void set_qreg(unsigned code, qreg_t value,
+ RegLogMode log_mode = LogRegWrites) {
+ set_vreg(code, value, log_mode);
+ }
+
+ bool N() const { return nzcv_.N() != 0; }
+ bool Z() const { return nzcv_.Z() != 0; }
+ bool C() const { return nzcv_.C() != 0; }
+ bool V() const { return nzcv_.V() != 0; }
+
+ SimSystemRegister& ReadNzcv() { return nzcv_; }
+ SimSystemRegister& nzcv() { return nzcv_; }
+
+ // TODO: Find a way to make the fpcr_ members return the proper types, so
+ // these accessors are not necessary.
+ FPRounding RMode() { return static_cast<FPRounding>(fpcr_.RMode()); }
+ bool DN() { return fpcr_.DN() != 0; }
+ SimSystemRegister& fpcr() { return fpcr_; }
+
+ UseDefaultNaN ReadDN() const {
+ return fpcr_.DN() != 0 ? kUseDefaultNaN : kIgnoreDefaultNaN;
+ }
+
+ // Specify relevant register formats for Print(V)Register and related helpers.
+ enum PrintRegisterFormat {
+ // The lane size.
+ kPrintRegLaneSizeB = 0 << 0,
+ kPrintRegLaneSizeH = 1 << 0,
+ kPrintRegLaneSizeS = 2 << 0,
+ kPrintRegLaneSizeW = kPrintRegLaneSizeS,
+ kPrintRegLaneSizeD = 3 << 0,
+ kPrintRegLaneSizeX = kPrintRegLaneSizeD,
+ kPrintRegLaneSizeQ = 4 << 0,
+
+ kPrintRegLaneSizeOffset = 0,
+ kPrintRegLaneSizeMask = 7 << 0,
+
+ // The lane count.
+ kPrintRegAsScalar = 0,
+ kPrintRegAsDVector = 1 << 3,
+ kPrintRegAsQVector = 2 << 3,
+
+ kPrintRegAsVectorMask = 3 << 3,
+
+ // Indicate floating-point format lanes. (This flag is only supported for S-
+ // and D-sized lanes.)
+ kPrintRegAsFP = 1 << 5,
+
+ // Supported combinations.
+
+ kPrintXReg = kPrintRegLaneSizeX | kPrintRegAsScalar,
+ kPrintWReg = kPrintRegLaneSizeW | kPrintRegAsScalar,
+ kPrintSReg = kPrintRegLaneSizeS | kPrintRegAsScalar | kPrintRegAsFP,
+ kPrintDReg = kPrintRegLaneSizeD | kPrintRegAsScalar | kPrintRegAsFP,
+
+ kPrintReg1B = kPrintRegLaneSizeB | kPrintRegAsScalar,
+ kPrintReg8B = kPrintRegLaneSizeB | kPrintRegAsDVector,
+ kPrintReg16B = kPrintRegLaneSizeB | kPrintRegAsQVector,
+ kPrintReg1H = kPrintRegLaneSizeH | kPrintRegAsScalar,
+ kPrintReg4H = kPrintRegLaneSizeH | kPrintRegAsDVector,
+ kPrintReg8H = kPrintRegLaneSizeH | kPrintRegAsQVector,
+ kPrintReg1S = kPrintRegLaneSizeS | kPrintRegAsScalar,
+ kPrintReg2S = kPrintRegLaneSizeS | kPrintRegAsDVector,
+ kPrintReg4S = kPrintRegLaneSizeS | kPrintRegAsQVector,
+ kPrintReg1SFP = kPrintRegLaneSizeS | kPrintRegAsScalar | kPrintRegAsFP,
+ kPrintReg2SFP = kPrintRegLaneSizeS | kPrintRegAsDVector | kPrintRegAsFP,
+ kPrintReg4SFP = kPrintRegLaneSizeS | kPrintRegAsQVector | kPrintRegAsFP,
+ kPrintReg1D = kPrintRegLaneSizeD | kPrintRegAsScalar,
+ kPrintReg2D = kPrintRegLaneSizeD | kPrintRegAsQVector,
+ kPrintReg1DFP = kPrintRegLaneSizeD | kPrintRegAsScalar | kPrintRegAsFP,
+ kPrintReg2DFP = kPrintRegLaneSizeD | kPrintRegAsQVector | kPrintRegAsFP,
+ kPrintReg1Q = kPrintRegLaneSizeQ | kPrintRegAsScalar
+ };
+
+ unsigned GetPrintRegLaneSizeInBytesLog2(PrintRegisterFormat format) {
+ return (format & kPrintRegLaneSizeMask) >> kPrintRegLaneSizeOffset;
+ }
+
+ unsigned GetPrintRegLaneSizeInBytes(PrintRegisterFormat format) {
+ return 1 << GetPrintRegLaneSizeInBytesLog2(format);
+ }
+
+ unsigned GetPrintRegSizeInBytesLog2(PrintRegisterFormat format) {
+ if (format & kPrintRegAsDVector) return kDRegSizeInBytesLog2;
+ if (format & kPrintRegAsQVector) return kQRegSizeInBytesLog2;
+
+ // Scalar types.
+ return GetPrintRegLaneSizeInBytesLog2(format);
+ }
+
+ unsigned GetPrintRegSizeInBytes(PrintRegisterFormat format) {
+ return 1 << GetPrintRegSizeInBytesLog2(format);
+ }
+
+ unsigned GetPrintRegLaneCount(PrintRegisterFormat format) {
+ unsigned reg_size_log2 = GetPrintRegSizeInBytesLog2(format);
+ unsigned lane_size_log2 = GetPrintRegLaneSizeInBytesLog2(format);
+ VIXL_ASSERT(reg_size_log2 >= lane_size_log2);
+ return 1 << (reg_size_log2 - lane_size_log2);
+ }
+
+ PrintRegisterFormat GetPrintRegisterFormatForSize(unsigned reg_size,
+ unsigned lane_size);
+
+ PrintRegisterFormat GetPrintRegisterFormatForSize(unsigned size) {
+ return GetPrintRegisterFormatForSize(size, size);
+ }
+
+ PrintRegisterFormat GetPrintRegisterFormatForSizeFP(unsigned size) {
+ switch (size) {
+ default: VIXL_UNREACHABLE(); return kPrintDReg;
+ case kDRegSizeInBytes: return kPrintDReg;
+ case kSRegSizeInBytes: return kPrintSReg;
+ }
+ }
+
+ PrintRegisterFormat GetPrintRegisterFormatTryFP(PrintRegisterFormat format) {
+ if ((GetPrintRegLaneSizeInBytes(format) == kSRegSizeInBytes) ||
+ (GetPrintRegLaneSizeInBytes(format) == kDRegSizeInBytes)) {
+ return static_cast<PrintRegisterFormat>(format | kPrintRegAsFP);
+ }
+ return format;
+ }
+
+ template<typename T>
+ PrintRegisterFormat GetPrintRegisterFormat(T value) {
+ return GetPrintRegisterFormatForSize(sizeof(value));
+ }
+
+ PrintRegisterFormat GetPrintRegisterFormat(double value) {
+ VIXL_STATIC_ASSERT(sizeof(value) == kDRegSizeInBytes);
+ return GetPrintRegisterFormatForSizeFP(sizeof(value));
+ }
+
+ PrintRegisterFormat GetPrintRegisterFormat(float value) {
+ VIXL_STATIC_ASSERT(sizeof(value) == kSRegSizeInBytes);
+ return GetPrintRegisterFormatForSizeFP(sizeof(value));
+ }
+
+ PrintRegisterFormat GetPrintRegisterFormat(VectorFormat vform);
+
+ // Print all registers of the specified types.
+ void PrintRegisters();
+ void PrintVRegisters();
+ void PrintSystemRegisters();
+
+ // As above, but only print the registers that have been updated.
+ void PrintWrittenRegisters();
+ void PrintWrittenVRegisters();
+
+ // As above, but respect LOG_REG and LOG_VREG.
+ void LogWrittenRegisters() {
+ if (trace_parameters() & LOG_REGS) PrintWrittenRegisters();
+ }
+ void LogWrittenVRegisters() {
+ if (trace_parameters() & LOG_VREGS) PrintWrittenVRegisters();
+ }
+ void LogAllWrittenRegisters() {
+ LogWrittenRegisters();
+ LogWrittenVRegisters();
+ }
+
+ // Print individual register values (after update).
+ void PrintRegister(unsigned code, Reg31Mode r31mode = Reg31IsStackPointer);
+ void PrintVRegister(unsigned code, PrintRegisterFormat format);
+ void PrintSystemRegister(SystemRegister id);
+
+ // Like Print* (above), but respect trace_parameters().
+ void LogRegister(unsigned code, Reg31Mode r31mode = Reg31IsStackPointer) {
+ if (trace_parameters() & LOG_REGS) PrintRegister(code, r31mode);
+ }
+ void LogVRegister(unsigned code, PrintRegisterFormat format) {
+ if (trace_parameters() & LOG_VREGS) PrintVRegister(code, format);
+ }
+ void LogSystemRegister(SystemRegister id) {
+ if (trace_parameters() & LOG_SYSREGS) PrintSystemRegister(id);
+ }
+
+ // Print memory accesses.
+ void PrintRead(uintptr_t address, unsigned reg_code,
+ PrintRegisterFormat format);
+ void PrintWrite(uintptr_t address, unsigned reg_code,
+ PrintRegisterFormat format);
+ void PrintVRead(uintptr_t address, unsigned reg_code,
+ PrintRegisterFormat format, unsigned lane);
+ void PrintVWrite(uintptr_t address, unsigned reg_code,
+ PrintRegisterFormat format, unsigned lane);
+
+ // Like Print* (above), but respect trace_parameters().
+ void LogRead(uintptr_t address, unsigned reg_code,
+ PrintRegisterFormat format) {
+ if (trace_parameters() & LOG_REGS) PrintRead(address, reg_code, format);
+ }
+ void LogWrite(uintptr_t address, unsigned reg_code,
+ PrintRegisterFormat format) {
+ if (trace_parameters() & LOG_WRITE) PrintWrite(address, reg_code, format);
+ }
+ void LogVRead(uintptr_t address, unsigned reg_code,
+ PrintRegisterFormat format, unsigned lane = 0) {
+ if (trace_parameters() & LOG_VREGS) {
+ PrintVRead(address, reg_code, format, lane);
+ }
+ }
+ void LogVWrite(uintptr_t address, unsigned reg_code,
+ PrintRegisterFormat format, unsigned lane = 0) {
+ if (trace_parameters() & LOG_WRITE) {
+ PrintVWrite(address, reg_code, format, lane);
+ }
+ }
+
+ // Helper functions for register tracing.
+ void PrintRegisterRawHelper(unsigned code, Reg31Mode r31mode,
+ int size_in_bytes = kXRegSizeInBytes);
+ void PrintVRegisterRawHelper(unsigned code, int bytes = kQRegSizeInBytes,
+ int lsb = 0);
+ void PrintVRegisterFPHelper(unsigned code, unsigned lane_size_in_bytes,
+ int lane_count = 1, int rightmost_lane = 0);
+
+ void DoUnreachable(const Instruction* instr);
+ void DoTrace(const Instruction* instr);
+ void DoLog(const Instruction* instr);
+
+ static const char* WRegNameForCode(unsigned code,
+ Reg31Mode mode = Reg31IsZeroRegister);
+ static const char* XRegNameForCode(unsigned code,
+ Reg31Mode mode = Reg31IsZeroRegister);
+ static const char* SRegNameForCode(unsigned code);
+ static const char* DRegNameForCode(unsigned code);
+ static const char* VRegNameForCode(unsigned code);
+
+ bool coloured_trace() const { return coloured_trace_; }
+ void set_coloured_trace(bool value);
+
+ int trace_parameters() const { return trace_parameters_; }
+ void set_trace_parameters(int parameters);
+
+ void set_instruction_stats(bool value);
+
+ // Clear the simulated local monitor to force the next store-exclusive
+ // instruction to fail.
+ void ClearLocalMonitor() {
+ local_monitor_.Clear();
+ }
+
+ void SilenceExclusiveAccessWarning() {
+ print_exclusive_access_warning_ = false;
+ }
+
+ protected:
+ const char* clr_normal;
+ const char* clr_flag_name;
+ const char* clr_flag_value;
+ const char* clr_reg_name;
+ const char* clr_reg_value;
+ const char* clr_vreg_name;
+ const char* clr_vreg_value;
+ const char* clr_memory_address;
+ const char* clr_warning;
+ const char* clr_warning_message;
+ const char* clr_printf;
+
+ // Simulation helpers ------------------------------------
+ bool ConditionPassed(Condition cond) {
+ switch (cond) {
+ case eq:
+ return Z();
+ case ne:
+ return !Z();
+ case hs:
+ return C();
+ case lo:
+ return !C();
+ case mi:
+ return N();
+ case pl:
+ return !N();
+ case vs:
+ return V();
+ case vc:
+ return !V();
+ case hi:
+ return C() && !Z();
+ case ls:
+ return !(C() && !Z());
+ case ge:
+ return N() == V();
+ case lt:
+ return N() != V();
+ case gt:
+ return !Z() && (N() == V());
+ case le:
+ return !(!Z() && (N() == V()));
+ case nv:
+ VIXL_FALLTHROUGH();
+ case al:
+ return true;
+ default:
+ VIXL_UNREACHABLE();
+ return false;
+ }
+ }
+
+ bool ConditionPassed(Instr cond) {
+ return ConditionPassed(static_cast<Condition>(cond));
+ }
+
+ bool ConditionFailed(Condition cond) {
+ return !ConditionPassed(cond);
+ }
+
+ void AddSubHelper(const Instruction* instr, int64_t op2);
+ uint64_t AddWithCarry(unsigned reg_size,
+ bool set_flags,
+ uint64_t left,
+ uint64_t right,
+ int carry_in = 0);
+ void LogicalHelper(const Instruction* instr, int64_t op2);
+ void ConditionalCompareHelper(const Instruction* instr, int64_t op2);
+ void LoadStoreHelper(const Instruction* instr,
+ int64_t offset,
+ AddrMode addrmode);
+ void LoadStorePairHelper(const Instruction* instr, AddrMode addrmode);
+ uintptr_t AddressModeHelper(unsigned addr_reg,
+ int64_t offset,
+ AddrMode addrmode);
+ void NEONLoadStoreMultiStructHelper(const Instruction* instr,
+ AddrMode addr_mode);
+ void NEONLoadStoreSingleStructHelper(const Instruction* instr,
+ AddrMode addr_mode);
+
+ uint64_t AddressUntag(uint64_t address) {
+ return address & ~kAddressTagMask;
+ }
+
+ template <typename T>
+ T* AddressUntag(T* address) {
+ uintptr_t address_raw = reinterpret_cast<uintptr_t>(address);
+ return reinterpret_cast<T*>(AddressUntag(address_raw));
+ }
+
+ int64_t ShiftOperand(unsigned reg_size,
+ int64_t value,
+ Shift shift_type,
+ unsigned amount);
+ int64_t Rotate(unsigned reg_width,
+ int64_t value,
+ Shift shift_type,
+ unsigned amount);
+ int64_t ExtendValue(unsigned reg_width,
+ int64_t value,
+ Extend extend_type,
+ unsigned left_shift = 0);
+ uint16_t PolynomialMult(uint8_t op1, uint8_t op2);
+
+ void ld1(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t addr);
+ void ld1(VectorFormat vform,
+ LogicVRegister dst,
+ int index,
+ uint64_t addr);
+ void ld1r(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t addr);
+ void ld2(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ uint64_t addr);
+ void ld2(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ int index,
+ uint64_t addr);
+ void ld2r(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ uint64_t addr);
+ void ld3(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ uint64_t addr);
+ void ld3(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ int index,
+ uint64_t addr);
+ void ld3r(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ uint64_t addr);
+ void ld4(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ LogicVRegister dst4,
+ uint64_t addr);
+ void ld4(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ LogicVRegister dst4,
+ int index,
+ uint64_t addr);
+ void ld4r(VectorFormat vform,
+ LogicVRegister dst1,
+ LogicVRegister dst2,
+ LogicVRegister dst3,
+ LogicVRegister dst4,
+ uint64_t addr);
+ void st1(VectorFormat vform,
+ LogicVRegister src,
+ uint64_t addr);
+ void st1(VectorFormat vform,
+ LogicVRegister src,
+ int index,
+ uint64_t addr);
+ void st2(VectorFormat vform,
+ LogicVRegister src,
+ LogicVRegister src2,
+ uint64_t addr);
+ void st2(VectorFormat vform,
+ LogicVRegister src,
+ LogicVRegister src2,
+ int index,
+ uint64_t addr);
+ void st3(VectorFormat vform,
+ LogicVRegister src,
+ LogicVRegister src2,
+ LogicVRegister src3,
+ uint64_t addr);
+ void st3(VectorFormat vform,
+ LogicVRegister src,
+ LogicVRegister src2,
+ LogicVRegister src3,
+ int index,
+ uint64_t addr);
+ void st4(VectorFormat vform,
+ LogicVRegister src,
+ LogicVRegister src2,
+ LogicVRegister src3,
+ LogicVRegister src4,
+ uint64_t addr);
+ void st4(VectorFormat vform,
+ LogicVRegister src,
+ LogicVRegister src2,
+ LogicVRegister src3,
+ LogicVRegister src4,
+ int index,
+ uint64_t addr);
+ LogicVRegister cmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ Condition cond);
+ LogicVRegister cmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ int imm,
+ Condition cond);
+ LogicVRegister cmptst(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister add(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister addp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister mla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister mls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister mul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister mul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister mla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister mls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister pmul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+
+ typedef LogicVRegister (Simulator::*ByElementOp)(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister fmul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister fmla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister fmls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister fmulx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister smull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister smull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister umull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister umull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister smlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister smlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister umlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister umlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister smlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister smlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister umlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister umlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sqdmull(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sqdmull2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sqdmlal(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sqdmlal2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sqdmlsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sqdmlsl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sqdmulh(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sqrdmulh(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister sub(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister and_(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister orr(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister orn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister eor(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister bic(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister bic(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ uint64_t imm);
+ LogicVRegister bif(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister bit(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister bsl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister cls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister clz(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister cnt(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister not_(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister rbit(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister rev(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int revSize);
+ LogicVRegister rev16(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister rev32(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister rev64(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister addlp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ bool is_signed,
+ bool do_accumulate);
+ LogicVRegister saddlp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister uaddlp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister sadalp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister uadalp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister ext(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ int index);
+ LogicVRegister ins_element(VectorFormat vform,
+ LogicVRegister dst,
+ int dst_index,
+ const LogicVRegister& src,
+ int src_index);
+ LogicVRegister ins_immediate(VectorFormat vform,
+ LogicVRegister dst,
+ int dst_index,
+ uint64_t imm);
+ LogicVRegister dup_element(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int src_index);
+ LogicVRegister dup_immediate(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t imm);
+ LogicVRegister mov(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister movi(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t imm);
+ LogicVRegister mvni(VectorFormat vform,
+ LogicVRegister dst,
+ uint64_t imm);
+ LogicVRegister orr(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ uint64_t imm);
+ LogicVRegister sshl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister ushl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister sminmax(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ bool max);
+ LogicVRegister smax(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister smin(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister sminmaxp(VectorFormat vform,
+ LogicVRegister dst,
+ int dst_index,
+ const LogicVRegister& src,
+ bool max);
+ LogicVRegister smaxp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister sminp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister addp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister addv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister uaddlv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister saddlv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister sminmaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ bool max);
+ LogicVRegister smaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister sminv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister uxtl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister uxtl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister sxtl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister sxtl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister tbl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& ind);
+ LogicVRegister tbl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& ind);
+ LogicVRegister tbl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& tab3,
+ const LogicVRegister& ind);
+ LogicVRegister tbl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& tab3,
+ const LogicVRegister& tab4,
+ const LogicVRegister& ind);
+ LogicVRegister tbx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& ind);
+ LogicVRegister tbx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& ind);
+ LogicVRegister tbx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& tab3,
+ const LogicVRegister& ind);
+ LogicVRegister tbx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& tab,
+ const LogicVRegister& tab2,
+ const LogicVRegister& tab3,
+ const LogicVRegister& tab4,
+ const LogicVRegister& ind);
+ LogicVRegister uaddl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uaddl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uaddw(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uaddw2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister saddl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister saddl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister saddw(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister saddw2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister usubl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister usubl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister usubw(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister usubw2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister ssubl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister ssubl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister ssubw(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister ssubw2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uminmax(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ bool max);
+ LogicVRegister umax(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister umin(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uminmaxp(VectorFormat vform,
+ LogicVRegister dst,
+ int dst_index,
+ const LogicVRegister& src,
+ bool max);
+ LogicVRegister umaxp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uminp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uminmaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ bool max);
+ LogicVRegister umaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister uminv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister trn1(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister trn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister zip1(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister zip2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uzp1(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uzp2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister shl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister scvtf(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int fbits,
+ FPRounding rounding_mode);
+ LogicVRegister ucvtf(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int fbits,
+ FPRounding rounding_mode);
+ LogicVRegister sshll(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sshll2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister shll(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister shll2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister ushll(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister ushll2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sli(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sri(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sshr(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister ushr(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister ssra(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister usra(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister srsra(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister ursra(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister suqadd(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister usqadd(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister sqshl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister uqshl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqshlu(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister abs(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister neg(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister extractnarrow(VectorFormat vform,
+ LogicVRegister dst,
+ bool dstIsSigned,
+ const LogicVRegister& src,
+ bool srcIsSigned);
+ LogicVRegister xtn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister sqxtn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister uqxtn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister sqxtun(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister absdiff(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ bool issigned);
+ LogicVRegister saba(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister uaba(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister shrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister shrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister rshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister rshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister uqshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister uqshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister uqrshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister uqrshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqrshrn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqrshrn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqshrun(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqshrun2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqrshrun(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqrshrun2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ int shift);
+ LogicVRegister sqrdmulh(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ bool round = true);
+ LogicVRegister sqdmulh(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ #define NEON_3VREG_LOGIC_LIST(V) \
+ V(addhn) \
+ V(addhn2) \
+ V(raddhn) \
+ V(raddhn2) \
+ V(subhn) \
+ V(subhn2) \
+ V(rsubhn) \
+ V(rsubhn2) \
+ V(pmull) \
+ V(pmull2) \
+ V(sabal) \
+ V(sabal2) \
+ V(uabal) \
+ V(uabal2) \
+ V(sabdl) \
+ V(sabdl2) \
+ V(uabdl) \
+ V(uabdl2) \
+ V(smull) \
+ V(smull2) \
+ V(umull) \
+ V(umull2) \
+ V(smlal) \
+ V(smlal2) \
+ V(umlal) \
+ V(umlal2) \
+ V(smlsl) \
+ V(smlsl2) \
+ V(umlsl) \
+ V(umlsl2) \
+ V(sqdmlal) \
+ V(sqdmlal2) \
+ V(sqdmlsl) \
+ V(sqdmlsl2) \
+ V(sqdmull) \
+ V(sqdmull2)
+
+ #define DEFINE_LOGIC_FUNC(FXN) \
+ LogicVRegister FXN(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src1, \
+ const LogicVRegister& src2);
+ NEON_3VREG_LOGIC_LIST(DEFINE_LOGIC_FUNC)
+ #undef DEFINE_LOGIC_FUNC
+
+ #define NEON_FP3SAME_LIST(V) \
+ V(fadd, FPAdd, false) \
+ V(fsub, FPSub, true) \
+ V(fmul, FPMul, true) \
+ V(fmulx, FPMulx, true) \
+ V(fdiv, FPDiv, true) \
+ V(fmax, FPMax, false) \
+ V(fmin, FPMin, false) \
+ V(fmaxnm, FPMaxNM, false) \
+ V(fminnm, FPMinNM, false)
+
+ #define DECLARE_NEON_FP_VECTOR_OP(FN, OP, PROCNAN) \
+ template <typename T> \
+ LogicVRegister FN(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src1, \
+ const LogicVRegister& src2); \
+ LogicVRegister FN(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src1, \
+ const LogicVRegister& src2);
+ NEON_FP3SAME_LIST(DECLARE_NEON_FP_VECTOR_OP)
+ #undef DECLARE_NEON_FP_VECTOR_OP
+
+ #define NEON_FPPAIRWISE_LIST(V) \
+ V(faddp, fadd, FPAdd) \
+ V(fmaxp, fmax, FPMax) \
+ V(fmaxnmp, fmaxnm, FPMaxNM) \
+ V(fminp, fmin, FPMin) \
+ V(fminnmp, fminnm, FPMinNM)
+
+ #define DECLARE_NEON_FP_PAIR_OP(FNP, FN, OP) \
+ LogicVRegister FNP(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src1, \
+ const LogicVRegister& src2); \
+ LogicVRegister FNP(VectorFormat vform, \
+ LogicVRegister dst, \
+ const LogicVRegister& src);
+ NEON_FPPAIRWISE_LIST(DECLARE_NEON_FP_PAIR_OP)
+ #undef DECLARE_NEON_FP_PAIR_OP
+
+ template <typename T>
+ LogicVRegister frecps(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister frecps(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ template <typename T>
+ LogicVRegister frsqrts(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister frsqrts(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ template <typename T>
+ LogicVRegister fmla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister fmla(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ template <typename T>
+ LogicVRegister fmls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister fmls(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister fnmul(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+
+ template <typename T>
+ LogicVRegister fcmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ Condition cond);
+ LogicVRegister fcmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ Condition cond);
+ LogicVRegister fabscmp(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2,
+ Condition cond);
+ LogicVRegister fcmp_zero(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ Condition cond);
+
+ template <typename T>
+ LogicVRegister fneg(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fneg(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ template <typename T>
+ LogicVRegister frecpx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister frecpx(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ template <typename T>
+ LogicVRegister fabs_(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fabs_(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fabd(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src1,
+ const LogicVRegister& src2);
+ LogicVRegister frint(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPRounding rounding_mode,
+ bool inexact_exception = false);
+ LogicVRegister fcvts(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPRounding rounding_mode,
+ int fbits = 0);
+ LogicVRegister fcvtu(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPRounding rounding_mode,
+ int fbits = 0);
+ LogicVRegister fcvtl(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fcvtl2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fcvtn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fcvtn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fcvtxn(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fcvtxn2(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fsqrt(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister frsqrte(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister frecpe(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPRounding rounding);
+ LogicVRegister ursqrte(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister urecpe(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+
+ typedef float (Simulator::*FPMinMaxOp)(float a, float b);
+
+ LogicVRegister fminmaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src,
+ FPMinMaxOp Op);
+
+ LogicVRegister fminv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fmaxv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fminnmv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+ LogicVRegister fmaxnmv(VectorFormat vform,
+ LogicVRegister dst,
+ const LogicVRegister& src);
+
+ static const uint32_t CRC32_POLY = 0x04C11DB7;
+ static const uint32_t CRC32C_POLY = 0x1EDC6F41;
+ uint32_t Poly32Mod2(unsigned n, uint64_t data, uint32_t poly);
+ template <typename T>
+ uint32_t Crc32Checksum(uint32_t acc, T val, uint32_t poly);
+ uint32_t Crc32Checksum(uint32_t acc, uint64_t val, uint32_t poly);
+
+ void SysOp_W(int op, int64_t val);
+
+ template <typename T>
+ T FPRecipSqrtEstimate(T op);
+ template <typename T>
+ T FPRecipEstimate(T op, FPRounding rounding);
+ template <typename T, typename R>
+ R FPToFixed(T op, int fbits, bool is_signed, FPRounding rounding);
+
+ void FPCompare(double val0, double val1, FPTrapFlags trap);
+ double FPRoundInt(double value, FPRounding round_mode);
+ double recip_sqrt_estimate(double a);
+ double recip_estimate(double a);
+ double FPRecipSqrtEstimate(double a);
+ double FPRecipEstimate(double a);
+ double FixedToDouble(int64_t src, int fbits, FPRounding round_mode);
+ double UFixedToDouble(uint64_t src, int fbits, FPRounding round_mode);
+ float FixedToFloat(int64_t src, int fbits, FPRounding round_mode);
+ float UFixedToFloat(uint64_t src, int fbits, FPRounding round_mode);
+ int32_t FPToInt32(double value, FPRounding rmode);
+ int64_t FPToInt64(double value, FPRounding rmode);
+ uint32_t FPToUInt32(double value, FPRounding rmode);
+ uint64_t FPToUInt64(double value, FPRounding rmode);
+ int32_t FPToFixedJS(double value);
+
+ template <typename T>
+ T FPAdd(T op1, T op2);
+
+ template <typename T>
+ T FPDiv(T op1, T op2);
+
+ template <typename T>
+ T FPMax(T a, T b);
+
+ template <typename T>
+ T FPMaxNM(T a, T b);
+
+ template <typename T>
+ T FPMin(T a, T b);
+
+ template <typename T>
+ T FPMinNM(T a, T b);
+
+ template <typename T>
+ T FPMul(T op1, T op2);
+
+ template <typename T>
+ T FPMulx(T op1, T op2);
+
+ template <typename T>
+ T FPMulAdd(T a, T op1, T op2);
+
+ template <typename T>
+ T FPSqrt(T op);
+
+ template <typename T>
+ T FPSub(T op1, T op2);
+
+ template <typename T>
+ T FPRecipStepFused(T op1, T op2);
+
+ template <typename T>
+ T FPRSqrtStepFused(T op1, T op2);
+
+ // This doesn't do anything at the moment. We'll need it if we want support
+ // for cumulative exception bits or floating-point exceptions.
+ void FPProcessException() { }
+
+ bool FPProcessNaNs(const Instruction* instr);
+
+ // Pseudo Printf instruction
+ void DoPrintf(const Instruction* instr);
+
+ // Processor state ---------------------------------------
+
+ // Simulated monitors for exclusive access instructions.
+ SimExclusiveLocalMonitor local_monitor_;
+ SimExclusiveGlobalMonitor global_monitor_;
+
+ // Output stream.
+ FILE* stream_;
+ PrintDisassembler* print_disasm_;
+
+ // Instruction statistics instrumentation.
+ Instrument* instrumentation_;
+
+ // General purpose registers. Register 31 is the stack pointer.
+ SimRegister registers_[kNumberOfRegisters];
+
+ // Vector registers
+ SimVRegister vregisters_[kNumberOfVRegisters];
+
+ // Program Status Register.
+ // bits[31, 27]: Condition flags N, Z, C, and V.
+ // (Negative, Zero, Carry, Overflow)
+ SimSystemRegister nzcv_;
+
+ // Floating-Point Control Register
+ SimSystemRegister fpcr_;
+
+ // Only a subset of FPCR features are supported by the simulator. This helper
+ // checks that the FPCR settings are supported.
+ //
+ // This is checked when floating-point instructions are executed, not when
+ // FPCR is set. This allows generated code to modify FPCR for external
+ // functions, or to save and restore it when entering and leaving generated
+ // code.
+ void AssertSupportedFPCR() {
+ VIXL_ASSERT(fpcr().FZ() == 0); // No flush-to-zero support.
+ VIXL_ASSERT(fpcr().RMode() == FPTieEven); // Ties-to-even rounding only.
+
+ // The simulator does not support half-precision operations so fpcr().AHP()
+ // is irrelevant, and is not checked here.
+ }
+
+ static int CalcNFlag(uint64_t result, unsigned reg_size) {
+ return (result >> (reg_size - 1)) & 1;
+ }
+
+ static int CalcZFlag(uint64_t result) {
+ return (result == 0) ? 1 : 0;
+ }
+
+ static const uint32_t kConditionFlagsMask = 0xf0000000;
+
+ // Stack
+ byte* stack_;
+ static const int stack_protection_size_ = 512 * KBytes;
+ static const int stack_size_ = (2 * MBytes) + (2 * stack_protection_size_);
+ byte* stack_limit_;
+
+ Decoder* decoder_;
+ // Indicates if the pc has been modified by the instruction and should not be
+ // automatically incremented.
+ bool pc_modified_;
+ const Instruction* pc_;
+
+ static const char* xreg_names[];
+ static const char* wreg_names[];
+ static const char* sreg_names[];
+ static const char* dreg_names[];
+ static const char* vreg_names[];
+
+ static const Instruction* kEndOfSimAddress;
+
+ private:
+ template <typename T>
+ static T FPDefaultNaN();
+
+ // Standard NaN processing.
+ template <typename T>
+ T FPProcessNaN(T op) {
+ VIXL_ASSERT(std::isnan(op));
+ if (IsSignallingNaN(op)) {
+ FPProcessException();
+ }
+ return DN() ? FPDefaultNaN<T>() : ToQuietNaN(op);
+ }
+
+ template <typename T>
+ T FPProcessNaNs(T op1, T op2) {
+ if (IsSignallingNaN(op1)) {
+ return FPProcessNaN(op1);
+ } else if (IsSignallingNaN(op2)) {
+ return FPProcessNaN(op2);
+ } else if (std::isnan(op1)) {
+ VIXL_ASSERT(IsQuietNaN(op1));
+ return FPProcessNaN(op1);
+ } else if (std::isnan(op2)) {
+ VIXL_ASSERT(IsQuietNaN(op2));
+ return FPProcessNaN(op2);
+ } else {
+ return 0.0;
+ }
+ }
+
+ template <typename T>
+ T FPProcessNaNs3(T op1, T op2, T op3) {
+ if (IsSignallingNaN(op1)) {
+ return FPProcessNaN(op1);
+ } else if (IsSignallingNaN(op2)) {
+ return FPProcessNaN(op2);
+ } else if (IsSignallingNaN(op3)) {
+ return FPProcessNaN(op3);
+ } else if (std::isnan(op1)) {
+ VIXL_ASSERT(IsQuietNaN(op1));
+ return FPProcessNaN(op1);
+ } else if (std::isnan(op2)) {
+ VIXL_ASSERT(IsQuietNaN(op2));
+ return FPProcessNaN(op2);
+ } else if (std::isnan(op3)) {
+ VIXL_ASSERT(IsQuietNaN(op3));
+ return FPProcessNaN(op3);
+ } else {
+ return 0.0;
+ }
+ }
+
+ bool coloured_trace_;
+
+ // A set of TraceParameters flags.
+ int trace_parameters_;
+
+ // Indicates whether the instruction instrumentation is active.
+ bool instruction_stats_;
+
+ // Indicates whether the exclusive-access warning has been printed.
+ bool print_exclusive_access_warning_;
+ void PrintExclusiveAccessWarning();
+
+ // Indicates that the simulator ran out of memory at some point.
+ // Data structures may not be fully allocated.
+ bool oom_;
+
+ public:
+ // True if the simulator ran out of memory during or after construction.
+ bool oom() const { return oom_; }
+
+ protected:
+ mozilla::Vector<int64_t, 0, js::SystemAllocPolicy> spStack_;
+};
+
+} // namespace vixl
+
+namespace js {
+namespace jit {
+
+class SimulatorProcess
+{
+ public:
+ static SimulatorProcess* singleton_;
+
+ SimulatorProcess()
+ : lock_(mutexid::Arm64SimulatorLock)
+ , redirection_(nullptr)
+ {}
+
+ // Synchronizes access between main thread and compilation threads.
+ js::Mutex lock_ MOZ_UNANNOTATED;
+ vixl::Redirection* redirection_;
+
+#ifdef JS_CACHE_SIMULATOR_ARM64
+ // For each simulator, record what other thread registered as instruction
+ // being invalidated.
+ struct ICacheFlush {
+ void* start;
+ size_t length;
+ };
+ using ICacheFlushes = mozilla::Vector<ICacheFlush, 2>;
+ struct SimFlushes {
+ vixl::Simulator* thread;
+ ICacheFlushes records;
+ };
+ mozilla::Vector<SimFlushes, 1> pendingFlushes_;
+
+ static void recordICacheFlush(void* start, size_t length);
+ static void membarrier();
+ static ICacheFlushes& getICacheFlushes(vixl::Simulator* sim);
+ [[nodiscard]] static bool registerSimulator(vixl::Simulator* sim);
+ static void unregisterSimulator(vixl::Simulator* sim);
+#endif
+
+ static void setRedirection(vixl::Redirection* redirection) {
+ singleton_->lock_.assertOwnedByCurrentThread();
+ singleton_->redirection_ = redirection;
+ }
+
+ static vixl::Redirection* redirection() {
+ singleton_->lock_.assertOwnedByCurrentThread();
+ return singleton_->redirection_;
+ }
+
+ static bool initialize() {
+ singleton_ = js_new<SimulatorProcess>();
+ return !!singleton_;
+ }
+ static void destroy() {
+ js_delete(singleton_);
+ singleton_ = nullptr;
+ }
+};
+
+// Protects the icache and redirection properties of the simulator.
+class AutoLockSimulatorCache : public js::LockGuard<js::Mutex>
+{
+ using Base = js::LockGuard<js::Mutex>;
+
+ public:
+ explicit AutoLockSimulatorCache()
+ : Base(SimulatorProcess::singleton_->lock_)
+ {
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif // JS_SIMULATOR_ARM64
+#endif // VIXL_A64_SIMULATOR_A64_H_
diff --git a/js/src/jit/arm64/vixl/Utils-vixl.cpp b/js/src/jit/arm64/vixl/Utils-vixl.cpp
new file mode 100644
index 0000000000..381c3501d1
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Utils-vixl.cpp
@@ -0,0 +1,555 @@
+// Copyright 2015, VIXL authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/arm64/vixl/Utils-vixl.h"
+
+#include <cstdio>
+
+namespace vixl {
+
+// The default NaN values (for FPCR.DN=1).
+const double kFP64DefaultNaN = RawbitsToDouble(UINT64_C(0x7ff8000000000000));
+const float kFP32DefaultNaN = RawbitsToFloat(0x7fc00000);
+const Float16 kFP16DefaultNaN = RawbitsToFloat16(0x7e00);
+
+// Floating-point zero values.
+const Float16 kFP16PositiveZero = RawbitsToFloat16(0x0);
+const Float16 kFP16NegativeZero = RawbitsToFloat16(0x8000);
+
+// Floating-point infinity values.
+const Float16 kFP16PositiveInfinity = RawbitsToFloat16(0x7c00);
+const Float16 kFP16NegativeInfinity = RawbitsToFloat16(0xfc00);
+const float kFP32PositiveInfinity = RawbitsToFloat(0x7f800000);
+const float kFP32NegativeInfinity = RawbitsToFloat(0xff800000);
+const double kFP64PositiveInfinity =
+ RawbitsToDouble(UINT64_C(0x7ff0000000000000));
+const double kFP64NegativeInfinity =
+ RawbitsToDouble(UINT64_C(0xfff0000000000000));
+
+bool IsZero(Float16 value) {
+ uint16_t bits = Float16ToRawbits(value);
+ return (bits == Float16ToRawbits(kFP16PositiveZero) ||
+ bits == Float16ToRawbits(kFP16NegativeZero));
+}
+
+uint16_t Float16ToRawbits(Float16 value) { return value.rawbits_; }
+
+uint32_t FloatToRawbits(float value) {
+ uint32_t bits = 0;
+ memcpy(&bits, &value, 4);
+ return bits;
+}
+
+
+uint64_t DoubleToRawbits(double value) {
+ uint64_t bits = 0;
+ memcpy(&bits, &value, 8);
+ return bits;
+}
+
+
+Float16 RawbitsToFloat16(uint16_t bits) {
+ Float16 f;
+ f.rawbits_ = bits;
+ return f;
+}
+
+
+float RawbitsToFloat(uint32_t bits) {
+ float value = 0.0;
+ memcpy(&value, &bits, 4);
+ return value;
+}
+
+
+double RawbitsToDouble(uint64_t bits) {
+ double value = 0.0;
+ memcpy(&value, &bits, 8);
+ return value;
+}
+
+
+uint32_t Float16Sign(internal::SimFloat16 val) {
+ uint16_t rawbits = Float16ToRawbits(val);
+ return ExtractUnsignedBitfield32(15, 15, rawbits);
+}
+
+
+uint32_t Float16Exp(internal::SimFloat16 val) {
+ uint16_t rawbits = Float16ToRawbits(val);
+ return ExtractUnsignedBitfield32(14, 10, rawbits);
+}
+
+uint32_t Float16Mantissa(internal::SimFloat16 val) {
+ uint16_t rawbits = Float16ToRawbits(val);
+ return ExtractUnsignedBitfield32(9, 0, rawbits);
+}
+
+
+uint32_t FloatSign(float val) {
+ uint32_t rawbits = FloatToRawbits(val);
+ return ExtractUnsignedBitfield32(31, 31, rawbits);
+}
+
+
+uint32_t FloatExp(float val) {
+ uint32_t rawbits = FloatToRawbits(val);
+ return ExtractUnsignedBitfield32(30, 23, rawbits);
+}
+
+
+uint32_t FloatMantissa(float val) {
+ uint32_t rawbits = FloatToRawbits(val);
+ return ExtractUnsignedBitfield32(22, 0, rawbits);
+}
+
+
+uint32_t DoubleSign(double val) {
+ uint64_t rawbits = DoubleToRawbits(val);
+ return static_cast<uint32_t>(ExtractUnsignedBitfield64(63, 63, rawbits));
+}
+
+
+uint32_t DoubleExp(double val) {
+ uint64_t rawbits = DoubleToRawbits(val);
+ return static_cast<uint32_t>(ExtractUnsignedBitfield64(62, 52, rawbits));
+}
+
+
+uint64_t DoubleMantissa(double val) {
+ uint64_t rawbits = DoubleToRawbits(val);
+ return ExtractUnsignedBitfield64(51, 0, rawbits);
+}
+
+
+internal::SimFloat16 Float16Pack(uint16_t sign,
+ uint16_t exp,
+ uint16_t mantissa) {
+ uint16_t bits = (sign << 15) | (exp << 10) | mantissa;
+ return RawbitsToFloat16(bits);
+}
+
+
+float FloatPack(uint32_t sign, uint32_t exp, uint32_t mantissa) {
+ uint32_t bits = (sign << 31) | (exp << 23) | mantissa;
+ return RawbitsToFloat(bits);
+}
+
+
+double DoublePack(uint64_t sign, uint64_t exp, uint64_t mantissa) {
+ uint64_t bits = (sign << 63) | (exp << 52) | mantissa;
+ return RawbitsToDouble(bits);
+}
+
+
+int Float16Classify(Float16 value) {
+ uint16_t bits = Float16ToRawbits(value);
+ uint16_t exponent_max = (1 << 5) - 1;
+ uint16_t exponent_mask = exponent_max << 10;
+ uint16_t mantissa_mask = (1 << 10) - 1;
+
+ uint16_t exponent = (bits & exponent_mask) >> 10;
+ uint16_t mantissa = bits & mantissa_mask;
+ if (exponent == 0) {
+ if (mantissa == 0) {
+ return FP_ZERO;
+ }
+ return FP_SUBNORMAL;
+ } else if (exponent == exponent_max) {
+ if (mantissa == 0) {
+ return FP_INFINITE;
+ }
+ return FP_NAN;
+ }
+ return FP_NORMAL;
+}
+
+
+unsigned CountClearHalfWords(uint64_t imm, unsigned reg_size) {
+ VIXL_ASSERT((reg_size % 8) == 0);
+ int count = 0;
+ for (unsigned i = 0; i < (reg_size / 16); i++) {
+ if ((imm & 0xffff) == 0) {
+ count++;
+ }
+ imm >>= 16;
+ }
+ return count;
+}
+
+
+int BitCount(uint64_t value) { return CountSetBits(value); }
+
+// Float16 definitions.
+
+Float16::Float16(double dvalue) {
+ rawbits_ =
+ Float16ToRawbits(FPToFloat16(dvalue, FPTieEven, kIgnoreDefaultNaN));
+}
+
+namespace internal {
+
+SimFloat16 SimFloat16::operator-() const {
+ return RawbitsToFloat16(rawbits_ ^ 0x8000);
+}
+
+// SimFloat16 definitions.
+SimFloat16 SimFloat16::operator+(SimFloat16 rhs) const {
+ return static_cast<double>(*this) + static_cast<double>(rhs);
+}
+
+SimFloat16 SimFloat16::operator-(SimFloat16 rhs) const {
+ return static_cast<double>(*this) - static_cast<double>(rhs);
+}
+
+SimFloat16 SimFloat16::operator*(SimFloat16 rhs) const {
+ return static_cast<double>(*this) * static_cast<double>(rhs);
+}
+
+SimFloat16 SimFloat16::operator/(SimFloat16 rhs) const {
+ return static_cast<double>(*this) / static_cast<double>(rhs);
+}
+
+bool SimFloat16::operator<(SimFloat16 rhs) const {
+ return static_cast<double>(*this) < static_cast<double>(rhs);
+}
+
+bool SimFloat16::operator>(SimFloat16 rhs) const {
+ return static_cast<double>(*this) > static_cast<double>(rhs);
+}
+
+bool SimFloat16::operator==(SimFloat16 rhs) const {
+ if (IsNaN(*this) || IsNaN(rhs)) {
+ return false;
+ } else if (IsZero(rhs) && IsZero(*this)) {
+ // +0 and -0 should be treated as equal.
+ return true;
+ }
+ return this->rawbits_ == rhs.rawbits_;
+}
+
+bool SimFloat16::operator!=(SimFloat16 rhs) const { return !(*this == rhs); }
+
+bool SimFloat16::operator==(double rhs) const {
+ return static_cast<double>(*this) == static_cast<double>(rhs);
+}
+
+SimFloat16::operator double() const {
+ return FPToDouble(*this, kIgnoreDefaultNaN);
+}
+
+Int64 BitCount(Uint32 value) { return CountSetBits(value.Get()); }
+
+} // namespace internal
+
+float FPToFloat(Float16 value, UseDefaultNaN DN, bool* exception) {
+ uint16_t bits = Float16ToRawbits(value);
+ uint32_t sign = bits >> 15;
+ uint32_t exponent =
+ ExtractUnsignedBitfield32(kFloat16MantissaBits + kFloat16ExponentBits - 1,
+ kFloat16MantissaBits,
+ bits);
+ uint32_t mantissa =
+ ExtractUnsignedBitfield32(kFloat16MantissaBits - 1, 0, bits);
+
+ switch (Float16Classify(value)) {
+ case FP_ZERO:
+ return (sign == 0) ? 0.0f : -0.0f;
+
+ case FP_INFINITE:
+ return (sign == 0) ? kFP32PositiveInfinity : kFP32NegativeInfinity;
+
+ case FP_SUBNORMAL: {
+ // Calculate shift required to put mantissa into the most-significant bits
+ // of the destination mantissa.
+ int shift = CountLeadingZeros(mantissa << (32 - 10));
+
+ // Shift mantissa and discard implicit '1'.
+ mantissa <<= (kFloatMantissaBits - kFloat16MantissaBits) + shift + 1;
+ mantissa &= (1 << kFloatMantissaBits) - 1;
+
+ // Adjust the exponent for the shift applied, and rebias.
+ exponent = exponent - shift + (-15 + 127);
+ break;
+ }
+
+ case FP_NAN:
+ if (IsSignallingNaN(value)) {
+ if (exception != NULL) {
+ *exception = true;
+ }
+ }
+ if (DN == kUseDefaultNaN) return kFP32DefaultNaN;
+
+ // Convert NaNs as the processor would:
+ // - The sign is propagated.
+ // - The payload (mantissa) is transferred entirely, except that the top
+ // bit is forced to '1', making the result a quiet NaN. The unused
+ // (low-order) payload bits are set to 0.
+ exponent = (1 << kFloatExponentBits) - 1;
+
+ // Increase bits in mantissa, making low-order bits 0.
+ mantissa <<= (kFloatMantissaBits - kFloat16MantissaBits);
+ mantissa |= 1 << 22; // Force a quiet NaN.
+ break;
+
+ case FP_NORMAL:
+ // Increase bits in mantissa, making low-order bits 0.
+ mantissa <<= (kFloatMantissaBits - kFloat16MantissaBits);
+
+ // Change exponent bias.
+ exponent += (-15 + 127);
+ break;
+
+ default:
+ VIXL_UNREACHABLE();
+ }
+ return RawbitsToFloat((sign << 31) | (exponent << kFloatMantissaBits) |
+ mantissa);
+}
+
+
+float FPToFloat(double value,
+ FPRounding round_mode,
+ UseDefaultNaN DN,
+ bool* exception) {
+ // Only the FPTieEven rounding mode is implemented.
+ VIXL_ASSERT((round_mode == FPTieEven) || (round_mode == FPRoundOdd));
+ USE(round_mode);
+
+ switch (std::fpclassify(value)) {
+ case FP_NAN: {
+ if (IsSignallingNaN(value)) {
+ if (exception != NULL) {
+ *exception = true;
+ }
+ }
+ if (DN == kUseDefaultNaN) return kFP32DefaultNaN;
+
+ // Convert NaNs as the processor would:
+ // - The sign is propagated.
+ // - The payload (mantissa) is transferred as much as possible, except
+ // that the top bit is forced to '1', making the result a quiet NaN.
+ uint64_t raw = DoubleToRawbits(value);
+
+ uint32_t sign = raw >> 63;
+ uint32_t exponent = (1 << 8) - 1;
+ uint32_t payload =
+ static_cast<uint32_t>(ExtractUnsignedBitfield64(50, 52 - 23, raw));
+ payload |= (1 << 22); // Force a quiet NaN.
+
+ return RawbitsToFloat((sign << 31) | (exponent << 23) | payload);
+ }
+
+ case FP_ZERO:
+ case FP_INFINITE: {
+ // In a C++ cast, any value representable in the target type will be
+ // unchanged. This is always the case for +/-0.0 and infinities.
+ return static_cast<float>(value);
+ }
+
+ case FP_NORMAL:
+ case FP_SUBNORMAL: {
+ // Convert double-to-float as the processor would, assuming that FPCR.FZ
+ // (flush-to-zero) is not set.
+ uint64_t raw = DoubleToRawbits(value);
+ // Extract the IEEE-754 double components.
+ uint32_t sign = raw >> 63;
+ // Extract the exponent and remove the IEEE-754 encoding bias.
+ int32_t exponent =
+ static_cast<int32_t>(ExtractUnsignedBitfield64(62, 52, raw)) - 1023;
+ // Extract the mantissa and add the implicit '1' bit.
+ uint64_t mantissa = ExtractUnsignedBitfield64(51, 0, raw);
+ if (std::fpclassify(value) == FP_NORMAL) {
+ mantissa |= (UINT64_C(1) << 52);
+ }
+ return FPRoundToFloat(sign, exponent, mantissa, round_mode);
+ }
+ }
+
+ VIXL_UNREACHABLE();
+ return value;
+}
+
+// TODO: We should consider implementing a full FPToDouble(Float16)
+// conversion function (for performance reasons).
+double FPToDouble(Float16 value, UseDefaultNaN DN, bool* exception) {
+ // We can rely on implicit float to double conversion here.
+ return FPToFloat(value, DN, exception);
+}
+
+
+double FPToDouble(float value, UseDefaultNaN DN, bool* exception) {
+ switch (std::fpclassify(value)) {
+ case FP_NAN: {
+ if (IsSignallingNaN(value)) {
+ if (exception != NULL) {
+ *exception = true;
+ }
+ }
+ if (DN == kUseDefaultNaN) return kFP64DefaultNaN;
+
+ // Convert NaNs as the processor would:
+ // - The sign is propagated.
+ // - The payload (mantissa) is transferred entirely, except that the top
+ // bit is forced to '1', making the result a quiet NaN. The unused
+ // (low-order) payload bits are set to 0.
+ uint32_t raw = FloatToRawbits(value);
+
+ uint64_t sign = raw >> 31;
+ uint64_t exponent = (1 << 11) - 1;
+ uint64_t payload = ExtractUnsignedBitfield64(21, 0, raw);
+ payload <<= (52 - 23); // The unused low-order bits should be 0.
+ payload |= (UINT64_C(1) << 51); // Force a quiet NaN.
+
+ return RawbitsToDouble((sign << 63) | (exponent << 52) | payload);
+ }
+
+ case FP_ZERO:
+ case FP_NORMAL:
+ case FP_SUBNORMAL:
+ case FP_INFINITE: {
+ // All other inputs are preserved in a standard cast, because every value
+ // representable using an IEEE-754 float is also representable using an
+ // IEEE-754 double.
+ return static_cast<double>(value);
+ }
+ }
+
+ VIXL_UNREACHABLE();
+ return static_cast<double>(value);
+}
+
+
+Float16 FPToFloat16(float value,
+ FPRounding round_mode,
+ UseDefaultNaN DN,
+ bool* exception) {
+ // Only the FPTieEven rounding mode is implemented.
+ VIXL_ASSERT(round_mode == FPTieEven);
+ USE(round_mode);
+
+ uint32_t raw = FloatToRawbits(value);
+ int32_t sign = raw >> 31;
+ int32_t exponent = ExtractUnsignedBitfield32(30, 23, raw) - 127;
+ uint32_t mantissa = ExtractUnsignedBitfield32(22, 0, raw);
+
+ switch (std::fpclassify(value)) {
+ case FP_NAN: {
+ if (IsSignallingNaN(value)) {
+ if (exception != NULL) {
+ *exception = true;
+ }
+ }
+ if (DN == kUseDefaultNaN) return kFP16DefaultNaN;
+
+ // Convert NaNs as the processor would:
+ // - The sign is propagated.
+ // - The payload (mantissa) is transferred as much as possible, except
+ // that the top bit is forced to '1', making the result a quiet NaN.
+ uint16_t result = (sign == 0) ? Float16ToRawbits(kFP16PositiveInfinity)
+ : Float16ToRawbits(kFP16NegativeInfinity);
+ result |= mantissa >> (kFloatMantissaBits - kFloat16MantissaBits);
+ result |= (1 << 9); // Force a quiet NaN;
+ return RawbitsToFloat16(result);
+ }
+
+ case FP_ZERO:
+ return (sign == 0) ? kFP16PositiveZero : kFP16NegativeZero;
+
+ case FP_INFINITE:
+ return (sign == 0) ? kFP16PositiveInfinity : kFP16NegativeInfinity;
+
+ case FP_NORMAL:
+ case FP_SUBNORMAL: {
+ // Convert float-to-half as the processor would, assuming that FPCR.FZ
+ // (flush-to-zero) is not set.
+
+ // Add the implicit '1' bit to the mantissa.
+ mantissa += (1 << 23);
+ return FPRoundToFloat16(sign, exponent, mantissa, round_mode);
+ }
+ }
+
+ VIXL_UNREACHABLE();
+ return kFP16PositiveZero;
+}
+
+
+Float16 FPToFloat16(double value,
+ FPRounding round_mode,
+ UseDefaultNaN DN,
+ bool* exception) {
+ // Only the FPTieEven rounding mode is implemented.
+ VIXL_ASSERT(round_mode == FPTieEven);
+ USE(round_mode);
+
+ uint64_t raw = DoubleToRawbits(value);
+ int32_t sign = raw >> 63;
+ int64_t exponent = ExtractUnsignedBitfield64(62, 52, raw) - 1023;
+ uint64_t mantissa = ExtractUnsignedBitfield64(51, 0, raw);
+
+ switch (std::fpclassify(value)) {
+ case FP_NAN: {
+ if (IsSignallingNaN(value)) {
+ if (exception != NULL) {
+ *exception = true;
+ }
+ }
+ if (DN == kUseDefaultNaN) return kFP16DefaultNaN;
+
+ // Convert NaNs as the processor would:
+ // - The sign is propagated.
+ // - The payload (mantissa) is transferred as much as possible, except
+ // that the top bit is forced to '1', making the result a quiet NaN.
+ uint16_t result = (sign == 0) ? Float16ToRawbits(kFP16PositiveInfinity)
+ : Float16ToRawbits(kFP16NegativeInfinity);
+ result |= mantissa >> (kDoubleMantissaBits - kFloat16MantissaBits);
+ result |= (1 << 9); // Force a quiet NaN;
+ return RawbitsToFloat16(result);
+ }
+
+ case FP_ZERO:
+ return (sign == 0) ? kFP16PositiveZero : kFP16NegativeZero;
+
+ case FP_INFINITE:
+ return (sign == 0) ? kFP16PositiveInfinity : kFP16NegativeInfinity;
+ case FP_NORMAL:
+ case FP_SUBNORMAL: {
+ // Convert double-to-half as the processor would, assuming that FPCR.FZ
+ // (flush-to-zero) is not set.
+
+ // Add the implicit '1' bit to the mantissa.
+ mantissa += (UINT64_C(1) << 52);
+ return FPRoundToFloat16(sign, exponent, mantissa, round_mode);
+ }
+ }
+
+ VIXL_UNREACHABLE();
+ return kFP16PositiveZero;
+}
+
+} // namespace vixl
diff --git a/js/src/jit/arm64/vixl/Utils-vixl.h b/js/src/jit/arm64/vixl/Utils-vixl.h
new file mode 100644
index 0000000000..d1f6a835f8
--- /dev/null
+++ b/js/src/jit/arm64/vixl/Utils-vixl.h
@@ -0,0 +1,1283 @@
+// Copyright 2015, VIXL authors
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+// this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above copyright notice,
+// this list of conditions and the following disclaimer in the documentation
+// and/or other materials provided with the distribution.
+// * Neither the name of ARM Limited nor the names of its contributors may be
+// used to endorse or promote products derived from this software without
+// specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef VIXL_UTILS_H
+#define VIXL_UTILS_H
+
+#include "mozilla/FloatingPoint.h"
+
+#include <cmath>
+#include <cstring>
+#include <limits>
+#include <vector>
+
+#include "jit/arm64/vixl/CompilerIntrinsics-vixl.h"
+#include "jit/arm64/vixl/Globals-vixl.h"
+
+namespace vixl {
+
+// Macros for compile-time format checking.
+#if GCC_VERSION_OR_NEWER(4, 4, 0)
+#define PRINTF_CHECK(format_index, varargs_index) \
+ __attribute__((format(gnu_printf, format_index, varargs_index)))
+#else
+#define PRINTF_CHECK(format_index, varargs_index)
+#endif
+
+#ifdef __GNUC__
+#define VIXL_HAS_DEPRECATED_WITH_MSG
+#elif defined(__clang__)
+#ifdef __has_extension
+#define VIXL_HAS_DEPRECATED_WITH_MSG
+#endif
+#endif
+
+#ifdef VIXL_HAS_DEPRECATED_WITH_MSG
+#define VIXL_DEPRECATED(replaced_by, declarator) \
+ __attribute__((deprecated("Use \"" replaced_by "\" instead"))) declarator
+#else
+#define VIXL_DEPRECATED(replaced_by, declarator) declarator
+#endif
+
+#ifdef VIXL_DEBUG
+#define VIXL_UNREACHABLE_OR_FALLTHROUGH() VIXL_UNREACHABLE()
+#else
+#define VIXL_UNREACHABLE_OR_FALLTHROUGH() VIXL_FALLTHROUGH()
+#endif
+
+template <typename T, size_t n>
+size_t ArrayLength(const T (&)[n]) {
+ return n;
+}
+
+// Check number width.
+// TODO: Refactor these using templates.
+inline bool IsIntN(unsigned n, uint32_t x) {
+ VIXL_ASSERT((0 < n) && (n < 32));
+ uint32_t limit = UINT32_C(1) << (n - 1);
+ return x < limit;
+}
+inline bool IsIntN(unsigned n, int32_t x) {
+ VIXL_ASSERT((0 < n) && (n < 32));
+ int32_t limit = INT32_C(1) << (n - 1);
+ return (-limit <= x) && (x < limit);
+}
+inline bool IsIntN(unsigned n, uint64_t x) {
+ VIXL_ASSERT((0 < n) && (n < 64));
+ uint64_t limit = UINT64_C(1) << (n - 1);
+ return x < limit;
+}
+inline bool IsIntN(unsigned n, int64_t x) {
+ VIXL_ASSERT((0 < n) && (n < 64));
+ int64_t limit = INT64_C(1) << (n - 1);
+ return (-limit <= x) && (x < limit);
+}
+VIXL_DEPRECATED("IsIntN", inline bool is_intn(unsigned n, int64_t x)) {
+ return IsIntN(n, x);
+}
+
+inline bool IsUintN(unsigned n, uint32_t x) {
+ VIXL_ASSERT((0 < n) && (n < 32));
+ return !(x >> n);
+}
+inline bool IsUintN(unsigned n, int32_t x) {
+ VIXL_ASSERT((0 < n) && (n < 32));
+ // Convert to an unsigned integer to avoid implementation-defined behavior.
+ return !(static_cast<uint32_t>(x) >> n);
+}
+inline bool IsUintN(unsigned n, uint64_t x) {
+ VIXL_ASSERT((0 < n) && (n < 64));
+ return !(x >> n);
+}
+inline bool IsUintN(unsigned n, int64_t x) {
+ VIXL_ASSERT((0 < n) && (n < 64));
+ // Convert to an unsigned integer to avoid implementation-defined behavior.
+ return !(static_cast<uint64_t>(x) >> n);
+}
+VIXL_DEPRECATED("IsUintN", inline bool is_uintn(unsigned n, int64_t x)) {
+ return IsUintN(n, x);
+}
+
+inline uint64_t TruncateToUintN(unsigned n, uint64_t x) {
+ VIXL_ASSERT((0 < n) && (n < 64));
+ return static_cast<uint64_t>(x) & ((UINT64_C(1) << n) - 1);
+}
+VIXL_DEPRECATED("TruncateToUintN",
+ inline uint64_t truncate_to_intn(unsigned n, int64_t x)) {
+ return TruncateToUintN(n, x);
+}
+
+// clang-format off
+#define INT_1_TO_32_LIST(V) \
+V(1) V(2) V(3) V(4) V(5) V(6) V(7) V(8) \
+V(9) V(10) V(11) V(12) V(13) V(14) V(15) V(16) \
+V(17) V(18) V(19) V(20) V(21) V(22) V(23) V(24) \
+V(25) V(26) V(27) V(28) V(29) V(30) V(31) V(32)
+
+#define INT_33_TO_63_LIST(V) \
+V(33) V(34) V(35) V(36) V(37) V(38) V(39) V(40) \
+V(41) V(42) V(43) V(44) V(45) V(46) V(47) V(48) \
+V(49) V(50) V(51) V(52) V(53) V(54) V(55) V(56) \
+V(57) V(58) V(59) V(60) V(61) V(62) V(63)
+
+#define INT_1_TO_63_LIST(V) INT_1_TO_32_LIST(V) INT_33_TO_63_LIST(V)
+
+// clang-format on
+
+#define DECLARE_IS_INT_N(N) \
+ inline bool IsInt##N(int64_t x) { return IsIntN(N, x); } \
+ VIXL_DEPRECATED("IsInt" #N, inline bool is_int##N(int64_t x)) { \
+ return IsIntN(N, x); \
+ }
+
+#define DECLARE_IS_UINT_N(N) \
+ inline bool IsUint##N(int64_t x) { return IsUintN(N, x); } \
+ VIXL_DEPRECATED("IsUint" #N, inline bool is_uint##N(int64_t x)) { \
+ return IsUintN(N, x); \
+ }
+
+#define DECLARE_TRUNCATE_TO_UINT_32(N) \
+ inline uint32_t TruncateToUint##N(uint64_t x) { \
+ return static_cast<uint32_t>(TruncateToUintN(N, x)); \
+ } \
+ VIXL_DEPRECATED("TruncateToUint" #N, \
+ inline uint32_t truncate_to_int##N(int64_t x)) { \
+ return TruncateToUint##N(x); \
+ }
+
+INT_1_TO_63_LIST(DECLARE_IS_INT_N)
+INT_1_TO_63_LIST(DECLARE_IS_UINT_N)
+INT_1_TO_32_LIST(DECLARE_TRUNCATE_TO_UINT_32)
+
+#undef DECLARE_IS_INT_N
+#undef DECLARE_IS_UINT_N
+#undef DECLARE_TRUNCATE_TO_INT_N
+
+// Bit field extraction.
+inline uint64_t ExtractUnsignedBitfield64(int msb, int lsb, uint64_t x) {
+ VIXL_ASSERT((static_cast<size_t>(msb) < sizeof(x) * 8) && (lsb >= 0) &&
+ (msb >= lsb));
+ if ((msb == 63) && (lsb == 0)) return x;
+ return (x >> lsb) & ((static_cast<uint64_t>(1) << (1 + msb - lsb)) - 1);
+}
+
+
+inline uint32_t ExtractUnsignedBitfield32(int msb, int lsb, uint32_t x) {
+ VIXL_ASSERT((static_cast<size_t>(msb) < sizeof(x) * 8) && (lsb >= 0) &&
+ (msb >= lsb));
+ return TruncateToUint32(ExtractUnsignedBitfield64(msb, lsb, x));
+}
+
+
+inline int64_t ExtractSignedBitfield64(int msb, int lsb, int64_t x) {
+ VIXL_ASSERT((static_cast<size_t>(msb) < sizeof(x) * 8) && (lsb >= 0) &&
+ (msb >= lsb));
+ uint64_t temp = ExtractUnsignedBitfield64(msb, lsb, x);
+ // If the highest extracted bit is set, sign extend.
+ if ((temp >> (msb - lsb)) == 1) {
+ temp |= ~UINT64_C(0) << (msb - lsb);
+ }
+ int64_t result;
+ memcpy(&result, &temp, sizeof(result));
+ return result;
+}
+
+
+inline int32_t ExtractSignedBitfield32(int msb, int lsb, int32_t x) {
+ VIXL_ASSERT((static_cast<size_t>(msb) < sizeof(x) * 8) && (lsb >= 0) &&
+ (msb >= lsb));
+ uint32_t temp = TruncateToUint32(ExtractSignedBitfield64(msb, lsb, x));
+ int32_t result;
+ memcpy(&result, &temp, sizeof(result));
+ return result;
+}
+
+
+inline uint64_t RotateRight(uint64_t value,
+ unsigned int rotate,
+ unsigned int width) {
+ VIXL_ASSERT((width > 0) && (width <= 64));
+ uint64_t width_mask = ~UINT64_C(0) >> (64 - width);
+ rotate &= 63;
+ if (rotate > 0) {
+ value &= width_mask;
+ value = (value << (width - rotate)) | (value >> rotate);
+ }
+ return value & width_mask;
+}
+
+
+// Wrapper class for passing FP16 values through the assembler.
+// This is purely to aid with type checking/casting.
+class Float16 {
+ public:
+ explicit Float16(double dvalue);
+ Float16() : rawbits_(0x0) {}
+ friend uint16_t Float16ToRawbits(Float16 value);
+ friend Float16 RawbitsToFloat16(uint16_t bits);
+
+ protected:
+ uint16_t rawbits_;
+};
+
+// Floating point representation.
+uint16_t Float16ToRawbits(Float16 value);
+
+
+uint32_t FloatToRawbits(float value);
+VIXL_DEPRECATED("FloatToRawbits",
+ inline uint32_t float_to_rawbits(float value)) {
+ return FloatToRawbits(value);
+}
+
+uint64_t DoubleToRawbits(double value);
+VIXL_DEPRECATED("DoubleToRawbits",
+ inline uint64_t double_to_rawbits(double value)) {
+ return DoubleToRawbits(value);
+}
+
+Float16 RawbitsToFloat16(uint16_t bits);
+
+float RawbitsToFloat(uint32_t bits);
+VIXL_DEPRECATED("RawbitsToFloat",
+ inline float rawbits_to_float(uint32_t bits)) {
+ return RawbitsToFloat(bits);
+}
+
+double RawbitsToDouble(uint64_t bits);
+VIXL_DEPRECATED("RawbitsToDouble",
+ inline double rawbits_to_double(uint64_t bits)) {
+ return RawbitsToDouble(bits);
+}
+
+namespace internal {
+
+// Internal simulation class used solely by the simulator to
+// provide an abstraction layer for any half-precision arithmetic.
+class SimFloat16 : public Float16 {
+ public:
+ // TODO: We should investigate making this constructor explicit.
+ // This is currently difficult to do due to a number of templated
+ // functions in the simulator which rely on returning double values.
+ SimFloat16(double dvalue) : Float16(dvalue) {} // NOLINT(runtime/explicit)
+ SimFloat16(Float16 f) { // NOLINT(runtime/explicit)
+ this->rawbits_ = Float16ToRawbits(f);
+ }
+ SimFloat16() : Float16() {}
+ SimFloat16 operator-() const;
+ SimFloat16 operator+(SimFloat16 rhs) const;
+ SimFloat16 operator-(SimFloat16 rhs) const;
+ SimFloat16 operator*(SimFloat16 rhs) const;
+ SimFloat16 operator/(SimFloat16 rhs) const;
+ bool operator<(SimFloat16 rhs) const;
+ bool operator>(SimFloat16 rhs) const;
+ bool operator==(SimFloat16 rhs) const;
+ bool operator!=(SimFloat16 rhs) const;
+ // This is necessary for conversions peformed in (macro asm) Fmov.
+ bool operator==(double rhs) const;
+ operator double() const;
+};
+} // namespace internal
+
+uint32_t Float16Sign(internal::SimFloat16 value);
+
+uint32_t Float16Exp(internal::SimFloat16 value);
+
+uint32_t Float16Mantissa(internal::SimFloat16 value);
+
+uint32_t FloatSign(float value);
+VIXL_DEPRECATED("FloatSign", inline uint32_t float_sign(float value)) {
+ return FloatSign(value);
+}
+
+uint32_t FloatExp(float value);
+VIXL_DEPRECATED("FloatExp", inline uint32_t float_exp(float value)) {
+ return FloatExp(value);
+}
+
+uint32_t FloatMantissa(float value);
+VIXL_DEPRECATED("FloatMantissa", inline uint32_t float_mantissa(float value)) {
+ return FloatMantissa(value);
+}
+
+uint32_t DoubleSign(double value);
+VIXL_DEPRECATED("DoubleSign", inline uint32_t double_sign(double value)) {
+ return DoubleSign(value);
+}
+
+uint32_t DoubleExp(double value);
+VIXL_DEPRECATED("DoubleExp", inline uint32_t double_exp(double value)) {
+ return DoubleExp(value);
+}
+
+uint64_t DoubleMantissa(double value);
+VIXL_DEPRECATED("DoubleMantissa",
+ inline uint64_t double_mantissa(double value)) {
+ return DoubleMantissa(value);
+}
+
+internal::SimFloat16 Float16Pack(uint16_t sign,
+ uint16_t exp,
+ uint16_t mantissa);
+
+float FloatPack(uint32_t sign, uint32_t exp, uint32_t mantissa);
+VIXL_DEPRECATED("FloatPack",
+ inline float float_pack(uint32_t sign,
+ uint32_t exp,
+ uint32_t mantissa)) {
+ return FloatPack(sign, exp, mantissa);
+}
+
+double DoublePack(uint64_t sign, uint64_t exp, uint64_t mantissa);
+VIXL_DEPRECATED("DoublePack",
+ inline double double_pack(uint32_t sign,
+ uint32_t exp,
+ uint64_t mantissa)) {
+ return DoublePack(sign, exp, mantissa);
+}
+
+// An fpclassify() function for 16-bit half-precision floats.
+int Float16Classify(Float16 value);
+VIXL_DEPRECATED("Float16Classify", inline int float16classify(uint16_t value)) {
+ return Float16Classify(RawbitsToFloat16(value));
+}
+
+bool IsZero(Float16 value);
+
+inline bool IsNaN(float value) { return std::isnan(value); }
+
+inline bool IsNaN(double value) { return std::isnan(value); }
+
+inline bool IsNaN(Float16 value) { return Float16Classify(value) == FP_NAN; }
+
+inline bool IsInf(float value) { return std::isinf(value); }
+
+inline bool IsInf(double value) { return std::isinf(value); }
+
+inline bool IsInf(Float16 value) {
+ return Float16Classify(value) == FP_INFINITE;
+}
+
+
+// NaN tests.
+inline bool IsSignallingNaN(double num) {
+ const uint64_t kFP64QuietNaNMask = UINT64_C(0x0008000000000000);
+ uint64_t raw = DoubleToRawbits(num);
+ if (IsNaN(num) && ((raw & kFP64QuietNaNMask) == 0)) {
+ return true;
+ }
+ return false;
+}
+
+
+inline bool IsSignallingNaN(float num) {
+ const uint32_t kFP32QuietNaNMask = 0x00400000;
+ uint32_t raw = FloatToRawbits(num);
+ if (IsNaN(num) && ((raw & kFP32QuietNaNMask) == 0)) {
+ return true;
+ }
+ return false;
+}
+
+
+inline bool IsSignallingNaN(Float16 num) {
+ const uint16_t kFP16QuietNaNMask = 0x0200;
+ return IsNaN(num) && ((Float16ToRawbits(num) & kFP16QuietNaNMask) == 0);
+}
+
+
+template <typename T>
+inline bool IsQuietNaN(T num) {
+ return IsNaN(num) && !IsSignallingNaN(num);
+}
+
+
+// Convert the NaN in 'num' to a quiet NaN.
+inline double ToQuietNaN(double num) {
+ const uint64_t kFP64QuietNaNMask = UINT64_C(0x0008000000000000);
+ VIXL_ASSERT(IsNaN(num));
+ return RawbitsToDouble(DoubleToRawbits(num) | kFP64QuietNaNMask);
+}
+
+
+inline float ToQuietNaN(float num) {
+ const uint32_t kFP32QuietNaNMask = 0x00400000;
+ VIXL_ASSERT(IsNaN(num));
+ return RawbitsToFloat(FloatToRawbits(num) | kFP32QuietNaNMask);
+}
+
+
+inline internal::SimFloat16 ToQuietNaN(internal::SimFloat16 num) {
+ const uint16_t kFP16QuietNaNMask = 0x0200;
+ VIXL_ASSERT(IsNaN(num));
+ return internal::SimFloat16(
+ RawbitsToFloat16(Float16ToRawbits(num) | kFP16QuietNaNMask));
+}
+
+
+// Fused multiply-add.
+inline double FusedMultiplyAdd(double op1, double op2, double a) {
+ return fma(op1, op2, a);
+}
+
+
+inline float FusedMultiplyAdd(float op1, float op2, float a) {
+ return fmaf(op1, op2, a);
+}
+
+
+inline uint64_t LowestSetBit(uint64_t value) { return value & -value; }
+
+
+template <typename T>
+inline int HighestSetBitPosition(T value) {
+ VIXL_ASSERT(value != 0);
+ return (sizeof(value) * 8 - 1) - CountLeadingZeros(value);
+}
+
+
+template <typename V>
+inline int WhichPowerOf2(V value) {
+ VIXL_ASSERT(IsPowerOf2(value));
+ return CountTrailingZeros(value);
+}
+
+
+unsigned CountClearHalfWords(uint64_t imm, unsigned reg_size);
+
+
+int BitCount(uint64_t value);
+
+
+template <typename T>
+T ReverseBits(T value) {
+ VIXL_ASSERT((sizeof(value) == 1) || (sizeof(value) == 2) ||
+ (sizeof(value) == 4) || (sizeof(value) == 8));
+ T result = 0;
+ for (unsigned i = 0; i < (sizeof(value) * 8); i++) {
+ result = (result << 1) | (value & 1);
+ value >>= 1;
+ }
+ return result;
+}
+
+
+template <typename T>
+inline T SignExtend(T val, int bitSize) {
+ VIXL_ASSERT(bitSize > 0);
+ T mask = (T(2) << (bitSize - 1)) - T(1);
+ val &= mask;
+ T sign_bits = -((val >> (bitSize - 1)) << bitSize);
+ val |= sign_bits;
+ return val;
+}
+
+
+template <typename T>
+T ReverseBytes(T value, int block_bytes_log2) {
+ VIXL_ASSERT((sizeof(value) == 4) || (sizeof(value) == 8));
+ VIXL_ASSERT((1U << block_bytes_log2) <= sizeof(value));
+ // Split the 64-bit value into an 8-bit array, where b[0] is the least
+ // significant byte, and b[7] is the most significant.
+ uint8_t bytes[8];
+ uint64_t mask = UINT64_C(0xff00000000000000);
+ for (int i = 7; i >= 0; i--) {
+ bytes[i] = (static_cast<uint64_t>(value) & mask) >> (i * 8);
+ mask >>= 8;
+ }
+
+ // Permutation tables for REV instructions.
+ // permute_table[0] is used by REV16_x, REV16_w
+ // permute_table[1] is used by REV32_x, REV_w
+ // permute_table[2] is used by REV_x
+ VIXL_ASSERT((0 < block_bytes_log2) && (block_bytes_log2 < 4));
+ static const uint8_t permute_table[3][8] = {{6, 7, 4, 5, 2, 3, 0, 1},
+ {4, 5, 6, 7, 0, 1, 2, 3},
+ {0, 1, 2, 3, 4, 5, 6, 7}};
+ uint64_t temp = 0;
+ for (int i = 0; i < 8; i++) {
+ temp <<= 8;
+ temp |= bytes[permute_table[block_bytes_log2 - 1][i]];
+ }
+
+ T result;
+ VIXL_STATIC_ASSERT(sizeof(result) <= sizeof(temp));
+ memcpy(&result, &temp, sizeof(result));
+ return result;
+}
+
+template <unsigned MULTIPLE, typename T>
+inline bool IsMultiple(T value) {
+ VIXL_ASSERT(IsPowerOf2(MULTIPLE));
+ return (value & (MULTIPLE - 1)) == 0;
+}
+
+template <typename T>
+inline bool IsMultiple(T value, unsigned multiple) {
+ VIXL_ASSERT(IsPowerOf2(multiple));
+ return (value & (multiple - 1)) == 0;
+}
+
+template <typename T>
+inline bool IsAligned(T pointer, int alignment) {
+ VIXL_ASSERT(IsPowerOf2(alignment));
+ return (pointer & (alignment - 1)) == 0;
+}
+
+// Pointer alignment
+// TODO: rename/refactor to make it specific to instructions.
+template <unsigned ALIGN, typename T>
+inline bool IsAligned(T pointer) {
+ VIXL_ASSERT(sizeof(pointer) == sizeof(intptr_t)); // NOLINT(runtime/sizeof)
+ // Use C-style casts to get static_cast behaviour for integral types (T), and
+ // reinterpret_cast behaviour for other types.
+ return IsAligned((intptr_t)(pointer), ALIGN);
+}
+
+template <typename T>
+bool IsWordAligned(T pointer) {
+ return IsAligned<4>(pointer);
+}
+
+// Increment a pointer until it has the specified alignment. The alignment must
+// be a power of two.
+template <class T>
+T AlignUp(T pointer,
+ typename Unsigned<sizeof(T) * kBitsPerByte>::type alignment) {
+ VIXL_ASSERT(IsPowerOf2(alignment));
+ // Use C-style casts to get static_cast behaviour for integral types (T), and
+ // reinterpret_cast behaviour for other types.
+
+ typename Unsigned<sizeof(T)* kBitsPerByte>::type pointer_raw =
+ (typename Unsigned<sizeof(T) * kBitsPerByte>::type)pointer;
+ VIXL_STATIC_ASSERT(sizeof(pointer) <= sizeof(pointer_raw));
+
+ size_t mask = alignment - 1;
+ T result = (T)((pointer_raw + mask) & ~mask);
+ VIXL_ASSERT(result >= pointer);
+
+ return result;
+}
+
+// Decrement a pointer until it has the specified alignment. The alignment must
+// be a power of two.
+template <class T>
+T AlignDown(T pointer,
+ typename Unsigned<sizeof(T) * kBitsPerByte>::type alignment) {
+ VIXL_ASSERT(IsPowerOf2(alignment));
+ // Use C-style casts to get static_cast behaviour for integral types (T), and
+ // reinterpret_cast behaviour for other types.
+
+ typename Unsigned<sizeof(T)* kBitsPerByte>::type pointer_raw =
+ (typename Unsigned<sizeof(T) * kBitsPerByte>::type)pointer;
+ VIXL_STATIC_ASSERT(sizeof(pointer) <= sizeof(pointer_raw));
+
+ size_t mask = alignment - 1;
+ return (T)(pointer_raw & ~mask);
+}
+
+
+template <typename T>
+inline T ExtractBit(T value, unsigned bit) {
+ return (value >> bit) & T(1);
+}
+
+template <typename Ts, typename Td>
+inline Td ExtractBits(Ts value, int least_significant_bit, Td mask) {
+ return Td((value >> least_significant_bit) & Ts(mask));
+}
+
+template <typename Ts, typename Td>
+inline void AssignBit(Td& dst, // NOLINT(runtime/references)
+ int bit,
+ Ts value) {
+ VIXL_ASSERT((value == Ts(0)) || (value == Ts(1)));
+ VIXL_ASSERT(bit >= 0);
+ VIXL_ASSERT(bit < static_cast<int>(sizeof(Td) * 8));
+ Td mask(1);
+ dst &= ~(mask << bit);
+ dst |= Td(value) << bit;
+}
+
+template <typename Td, typename Ts>
+inline void AssignBits(Td& dst, // NOLINT(runtime/references)
+ int least_significant_bit,
+ Ts mask,
+ Ts value) {
+ VIXL_ASSERT(least_significant_bit >= 0);
+ VIXL_ASSERT(least_significant_bit < static_cast<int>(sizeof(Td) * 8));
+ VIXL_ASSERT(((Td(mask) << least_significant_bit) >> least_significant_bit) ==
+ Td(mask));
+ VIXL_ASSERT((value & mask) == value);
+ dst &= ~(Td(mask) << least_significant_bit);
+ dst |= Td(value) << least_significant_bit;
+}
+
+class VFP {
+ public:
+ static uint32_t FP32ToImm8(float imm) {
+ // bits: aBbb.bbbc.defg.h000.0000.0000.0000.0000
+ uint32_t bits = FloatToRawbits(imm);
+ // bit7: a000.0000
+ uint32_t bit7 = ((bits >> 31) & 0x1) << 7;
+ // bit6: 0b00.0000
+ uint32_t bit6 = ((bits >> 29) & 0x1) << 6;
+ // bit5_to_0: 00cd.efgh
+ uint32_t bit5_to_0 = (bits >> 19) & 0x3f;
+ return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
+ }
+ static uint32_t FP64ToImm8(double imm) {
+ // bits: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000
+ uint64_t bits = DoubleToRawbits(imm);
+ // bit7: a000.0000
+ uint64_t bit7 = ((bits >> 63) & 0x1) << 7;
+ // bit6: 0b00.0000
+ uint64_t bit6 = ((bits >> 61) & 0x1) << 6;
+ // bit5_to_0: 00cd.efgh
+ uint64_t bit5_to_0 = (bits >> 48) & 0x3f;
+
+ return static_cast<uint32_t>(bit7 | bit6 | bit5_to_0);
+ }
+ static float Imm8ToFP32(uint32_t imm8) {
+ // Imm8: abcdefgh (8 bits)
+ // Single: aBbb.bbbc.defg.h000.0000.0000.0000.0000 (32 bits)
+ // where B is b ^ 1
+ uint32_t bits = imm8;
+ uint32_t bit7 = (bits >> 7) & 0x1;
+ uint32_t bit6 = (bits >> 6) & 0x1;
+ uint32_t bit5_to_0 = bits & 0x3f;
+ uint32_t result = (bit7 << 31) | ((32 - bit6) << 25) | (bit5_to_0 << 19);
+
+ return RawbitsToFloat(result);
+ }
+ static double Imm8ToFP64(uint32_t imm8) {
+ // Imm8: abcdefgh (8 bits)
+ // Double: aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000 (64 bits)
+ // where B is b ^ 1
+ uint32_t bits = imm8;
+ uint64_t bit7 = (bits >> 7) & 0x1;
+ uint64_t bit6 = (bits >> 6) & 0x1;
+ uint64_t bit5_to_0 = bits & 0x3f;
+ uint64_t result = (bit7 << 63) | ((256 - bit6) << 54) | (bit5_to_0 << 48);
+ return RawbitsToDouble(result);
+ }
+ static bool IsImmFP32(float imm) {
+ // Valid values will have the form:
+ // aBbb.bbbc.defg.h000.0000.0000.0000.0000
+ uint32_t bits = FloatToRawbits(imm);
+ // bits[19..0] are cleared.
+ if ((bits & 0x7ffff) != 0) {
+ return false;
+ }
+
+
+ // bits[29..25] are all set or all cleared.
+ uint32_t b_pattern = (bits >> 16) & 0x3e00;
+ if (b_pattern != 0 && b_pattern != 0x3e00) {
+ return false;
+ }
+ // bit[30] and bit[29] are opposite.
+ if (((bits ^ (bits << 1)) & 0x40000000) == 0) {
+ return false;
+ }
+ return true;
+ }
+ static bool IsImmFP64(double imm) {
+ // Valid values will have the form:
+ // aBbb.bbbb.bbcd.efgh.0000.0000.0000.0000
+ // 0000.0000.0000.0000.0000.0000.0000.0000
+ uint64_t bits = DoubleToRawbits(imm);
+ // bits[47..0] are cleared.
+ if ((bits & 0x0000ffffffffffff) != 0) {
+ return false;
+ }
+ // bits[61..54] are all set or all cleared.
+ uint32_t b_pattern = (bits >> 48) & 0x3fc0;
+ if ((b_pattern != 0) && (b_pattern != 0x3fc0)) {
+ return false;
+ }
+ // bit[62] and bit[61] are opposite.
+ if (((bits ^ (bits << 1)) & (UINT64_C(1) << 62)) == 0) {
+ return false;
+ }
+ return true;
+ }
+};
+
+class BitField {
+ // ForEachBitHelper is a functor that will call
+ // bool ForEachBitHelper::execute(ElementType id) const
+ // and expects a boolean in return whether to continue (if true)
+ // or stop (if false)
+ // check_set will check if the bits are on (true) or off(false)
+ template <typename ForEachBitHelper, bool check_set>
+ bool ForEachBit(const ForEachBitHelper& helper) {
+ for (int i = 0; static_cast<size_t>(i) < bitfield_.size(); i++) {
+ if (bitfield_[i] == check_set)
+ if (!helper.execute(i)) return false;
+ }
+ return true;
+ }
+
+ public:
+ explicit BitField(unsigned size) : bitfield_(size, 0) {}
+
+ void Set(int i) {
+ VIXL_ASSERT((i >= 0) && (static_cast<size_t>(i) < bitfield_.size()));
+ bitfield_[i] = true;
+ }
+
+ void Unset(int i) {
+ VIXL_ASSERT((i >= 0) && (static_cast<size_t>(i) < bitfield_.size()));
+ bitfield_[i] = true;
+ }
+
+ bool IsSet(int i) const { return bitfield_[i]; }
+
+ // For each bit not set in the bitfield call the execute functor
+ // execute.
+ // ForEachBitSetHelper::execute returns true if the iteration through
+ // the bits can continue, otherwise it will stop.
+ // struct ForEachBitSetHelper {
+ // bool execute(int /*id*/) { return false; }
+ // };
+ template <typename ForEachBitNotSetHelper>
+ bool ForEachBitNotSet(const ForEachBitNotSetHelper& helper) {
+ return ForEachBit<ForEachBitNotSetHelper, false>(helper);
+ }
+
+ // For each bit set in the bitfield call the execute functor
+ // execute.
+ template <typename ForEachBitSetHelper>
+ bool ForEachBitSet(const ForEachBitSetHelper& helper) {
+ return ForEachBit<ForEachBitSetHelper, true>(helper);
+ }
+
+ private:
+ std::vector<bool> bitfield_;
+};
+
+namespace internal {
+
+typedef int64_t Int64;
+class Uint64;
+class Uint128;
+
+class Uint32 {
+ uint32_t data_;
+
+ public:
+ // Unlike uint32_t, Uint32 has a default constructor.
+ Uint32() { data_ = 0; }
+ explicit Uint32(uint32_t data) : data_(data) {}
+ inline explicit Uint32(Uint64 data);
+ uint32_t Get() const { return data_; }
+ template <int N>
+ int32_t GetSigned() const {
+ return ExtractSignedBitfield32(N - 1, 0, data_);
+ }
+ int32_t GetSigned() const { return data_; }
+ Uint32 operator~() const { return Uint32(~data_); }
+ Uint32 operator-() const { return Uint32(-data_); }
+ bool operator==(Uint32 value) const { return data_ == value.data_; }
+ bool operator!=(Uint32 value) const { return data_ != value.data_; }
+ bool operator>(Uint32 value) const { return data_ > value.data_; }
+ Uint32 operator+(Uint32 value) const { return Uint32(data_ + value.data_); }
+ Uint32 operator-(Uint32 value) const { return Uint32(data_ - value.data_); }
+ Uint32 operator&(Uint32 value) const { return Uint32(data_ & value.data_); }
+ Uint32 operator&=(Uint32 value) {
+ data_ &= value.data_;
+ return *this;
+ }
+ Uint32 operator^(Uint32 value) const { return Uint32(data_ ^ value.data_); }
+ Uint32 operator^=(Uint32 value) {
+ data_ ^= value.data_;
+ return *this;
+ }
+ Uint32 operator|(Uint32 value) const { return Uint32(data_ | value.data_); }
+ Uint32 operator|=(Uint32 value) {
+ data_ |= value.data_;
+ return *this;
+ }
+ // Unlike uint32_t, the shift functions can accept negative shift and
+ // return 0 when the shift is too big.
+ Uint32 operator>>(int shift) const {
+ if (shift == 0) return *this;
+ if (shift < 0) {
+ int tmp = -shift;
+ if (tmp >= 32) return Uint32(0);
+ return Uint32(data_ << tmp);
+ }
+ int tmp = shift;
+ if (tmp >= 32) return Uint32(0);
+ return Uint32(data_ >> tmp);
+ }
+ Uint32 operator<<(int shift) const {
+ if (shift == 0) return *this;
+ if (shift < 0) {
+ int tmp = -shift;
+ if (tmp >= 32) return Uint32(0);
+ return Uint32(data_ >> tmp);
+ }
+ int tmp = shift;
+ if (tmp >= 32) return Uint32(0);
+ return Uint32(data_ << tmp);
+ }
+};
+
+class Uint64 {
+ uint64_t data_;
+
+ public:
+ // Unlike uint64_t, Uint64 has a default constructor.
+ Uint64() { data_ = 0; }
+ explicit Uint64(uint64_t data) : data_(data) {}
+ explicit Uint64(Uint32 data) : data_(data.Get()) {}
+ inline explicit Uint64(Uint128 data);
+ uint64_t Get() const { return data_; }
+ int64_t GetSigned(int N) const {
+ return ExtractSignedBitfield64(N - 1, 0, data_);
+ }
+ int64_t GetSigned() const { return data_; }
+ Uint32 ToUint32() const {
+ VIXL_ASSERT((data_ >> 32) == 0);
+ return Uint32(static_cast<uint32_t>(data_));
+ }
+ Uint32 GetHigh32() const { return Uint32(data_ >> 32); }
+ Uint32 GetLow32() const { return Uint32(data_ & 0xffffffff); }
+ Uint64 operator~() const { return Uint64(~data_); }
+ Uint64 operator-() const { return Uint64(-data_); }
+ bool operator==(Uint64 value) const { return data_ == value.data_; }
+ bool operator!=(Uint64 value) const { return data_ != value.data_; }
+ Uint64 operator+(Uint64 value) const { return Uint64(data_ + value.data_); }
+ Uint64 operator-(Uint64 value) const { return Uint64(data_ - value.data_); }
+ Uint64 operator&(Uint64 value) const { return Uint64(data_ & value.data_); }
+ Uint64 operator&=(Uint64 value) {
+ data_ &= value.data_;
+ return *this;
+ }
+ Uint64 operator^(Uint64 value) const { return Uint64(data_ ^ value.data_); }
+ Uint64 operator^=(Uint64 value) {
+ data_ ^= value.data_;
+ return *this;
+ }
+ Uint64 operator|(Uint64 value) const { return Uint64(data_ | value.data_); }
+ Uint64 operator|=(Uint64 value) {
+ data_ |= value.data_;
+ return *this;
+ }
+ // Unlike uint64_t, the shift functions can accept negative shift and
+ // return 0 when the shift is too big.
+ Uint64 operator>>(int shift) const {
+ if (shift == 0) return *this;
+ if (shift < 0) {
+ int tmp = -shift;
+ if (tmp >= 64) return Uint64(0);
+ return Uint64(data_ << tmp);
+ }
+ int tmp = shift;
+ if (tmp >= 64) return Uint64(0);
+ return Uint64(data_ >> tmp);
+ }
+ Uint64 operator<<(int shift) const {
+ if (shift == 0) return *this;
+ if (shift < 0) {
+ int tmp = -shift;
+ if (tmp >= 64) return Uint64(0);
+ return Uint64(data_ >> tmp);
+ }
+ int tmp = shift;
+ if (tmp >= 64) return Uint64(0);
+ return Uint64(data_ << tmp);
+ }
+};
+
+class Uint128 {
+ uint64_t data_high_;
+ uint64_t data_low_;
+
+ public:
+ Uint128() : data_high_(0), data_low_(0) {}
+ explicit Uint128(uint64_t data_low) : data_high_(0), data_low_(data_low) {}
+ explicit Uint128(Uint64 data_low)
+ : data_high_(0), data_low_(data_low.Get()) {}
+ Uint128(uint64_t data_high, uint64_t data_low)
+ : data_high_(data_high), data_low_(data_low) {}
+ Uint64 ToUint64() const {
+ VIXL_ASSERT(data_high_ == 0);
+ return Uint64(data_low_);
+ }
+ Uint64 GetHigh64() const { return Uint64(data_high_); }
+ Uint64 GetLow64() const { return Uint64(data_low_); }
+ Uint128 operator~() const { return Uint128(~data_high_, ~data_low_); }
+ bool operator==(Uint128 value) const {
+ return (data_high_ == value.data_high_) && (data_low_ == value.data_low_);
+ }
+ Uint128 operator&(Uint128 value) const {
+ return Uint128(data_high_ & value.data_high_, data_low_ & value.data_low_);
+ }
+ Uint128 operator&=(Uint128 value) {
+ data_high_ &= value.data_high_;
+ data_low_ &= value.data_low_;
+ return *this;
+ }
+ Uint128 operator|=(Uint128 value) {
+ data_high_ |= value.data_high_;
+ data_low_ |= value.data_low_;
+ return *this;
+ }
+ Uint128 operator>>(int shift) const {
+ VIXL_ASSERT((shift >= 0) && (shift < 128));
+ if (shift == 0) return *this;
+ if (shift >= 64) {
+ return Uint128(0, data_high_ >> (shift - 64));
+ }
+ uint64_t tmp = (data_high_ << (64 - shift)) | (data_low_ >> shift);
+ return Uint128(data_high_ >> shift, tmp);
+ }
+ Uint128 operator<<(int shift) const {
+ VIXL_ASSERT((shift >= 0) && (shift < 128));
+ if (shift == 0) return *this;
+ if (shift >= 64) {
+ return Uint128(data_low_ << (shift - 64), 0);
+ }
+ uint64_t tmp = (data_high_ << shift) | (data_low_ >> (64 - shift));
+ return Uint128(tmp, data_low_ << shift);
+ }
+};
+
+Uint32::Uint32(Uint64 data) : data_(data.ToUint32().Get()) {}
+Uint64::Uint64(Uint128 data) : data_(data.ToUint64().Get()) {}
+
+Int64 BitCount(Uint32 value);
+
+} // namespace internal
+
+// The default NaN values (for FPCR.DN=1).
+extern const double kFP64DefaultNaN;
+extern const float kFP32DefaultNaN;
+extern const Float16 kFP16DefaultNaN;
+
+// Floating-point infinity values.
+extern const Float16 kFP16PositiveInfinity;
+extern const Float16 kFP16NegativeInfinity;
+extern const float kFP32PositiveInfinity;
+extern const float kFP32NegativeInfinity;
+extern const double kFP64PositiveInfinity;
+extern const double kFP64NegativeInfinity;
+
+// Floating-point zero values.
+extern const Float16 kFP16PositiveZero;
+extern const Float16 kFP16NegativeZero;
+
+// AArch64 floating-point specifics. These match IEEE-754.
+const unsigned kDoubleMantissaBits = 52;
+const unsigned kDoubleExponentBits = 11;
+const unsigned kFloatMantissaBits = 23;
+const unsigned kFloatExponentBits = 8;
+const unsigned kFloat16MantissaBits = 10;
+const unsigned kFloat16ExponentBits = 5;
+
+enum FPRounding {
+ // The first four values are encodable directly by FPCR<RMode>.
+ FPTieEven = 0x0,
+ FPPositiveInfinity = 0x1,
+ FPNegativeInfinity = 0x2,
+ FPZero = 0x3,
+
+ // The final rounding modes are only available when explicitly specified by
+ // the instruction (such as with fcvta). It cannot be set in FPCR.
+ FPTieAway,
+ FPRoundOdd
+};
+
+enum UseDefaultNaN { kUseDefaultNaN, kIgnoreDefaultNaN };
+
+// Assemble the specified IEEE-754 components into the target type and apply
+// appropriate rounding.
+// sign: 0 = positive, 1 = negative
+// exponent: Unbiased IEEE-754 exponent.
+// mantissa: The mantissa of the input. The top bit (which is not encoded for
+// normal IEEE-754 values) must not be omitted. This bit has the
+// value 'pow(2, exponent)'.
+//
+// The input value is assumed to be a normalized value. That is, the input may
+// not be infinity or NaN. If the source value is subnormal, it must be
+// normalized before calling this function such that the highest set bit in the
+// mantissa has the value 'pow(2, exponent)'.
+//
+// Callers should use FPRoundToFloat or FPRoundToDouble directly, rather than
+// calling a templated FPRound.
+template <class T, int ebits, int mbits>
+T FPRound(int64_t sign,
+ int64_t exponent,
+ uint64_t mantissa,
+ FPRounding round_mode) {
+ VIXL_ASSERT((sign == 0) || (sign == 1));
+
+ // Only FPTieEven and FPRoundOdd rounding modes are implemented.
+ VIXL_ASSERT((round_mode == FPTieEven) || (round_mode == FPRoundOdd));
+
+ // Rounding can promote subnormals to normals, and normals to infinities. For
+ // example, a double with exponent 127 (FLT_MAX_EXP) would appear to be
+ // encodable as a float, but rounding based on the low-order mantissa bits
+ // could make it overflow. With ties-to-even rounding, this value would become
+ // an infinity.
+
+ // ---- Rounding Method ----
+ //
+ // The exponent is irrelevant in the rounding operation, so we treat the
+ // lowest-order bit that will fit into the result ('onebit') as having
+ // the value '1'. Similarly, the highest-order bit that won't fit into
+ // the result ('halfbit') has the value '0.5'. The 'point' sits between
+ // 'onebit' and 'halfbit':
+ //
+ // These bits fit into the result.
+ // |---------------------|
+ // mantissa = 0bxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
+ // ||
+ // / |
+ // / halfbit
+ // onebit
+ //
+ // For subnormal outputs, the range of representable bits is smaller and
+ // the position of onebit and halfbit depends on the exponent of the
+ // input, but the method is otherwise similar.
+ //
+ // onebit(frac)
+ // |
+ // | halfbit(frac) halfbit(adjusted)
+ // | / /
+ // | | |
+ // 0b00.0 (exact) -> 0b00.0 (exact) -> 0b00
+ // 0b00.0... -> 0b00.0... -> 0b00
+ // 0b00.1 (exact) -> 0b00.0111..111 -> 0b00
+ // 0b00.1... -> 0b00.1... -> 0b01
+ // 0b01.0 (exact) -> 0b01.0 (exact) -> 0b01
+ // 0b01.0... -> 0b01.0... -> 0b01
+ // 0b01.1 (exact) -> 0b01.1 (exact) -> 0b10
+ // 0b01.1... -> 0b01.1... -> 0b10
+ // 0b10.0 (exact) -> 0b10.0 (exact) -> 0b10
+ // 0b10.0... -> 0b10.0... -> 0b10
+ // 0b10.1 (exact) -> 0b10.0111..111 -> 0b10
+ // 0b10.1... -> 0b10.1... -> 0b11
+ // 0b11.0 (exact) -> 0b11.0 (exact) -> 0b11
+ // ... / | / |
+ // / | / |
+ // / |
+ // adjusted = frac - (halfbit(mantissa) & ~onebit(frac)); / |
+ //
+ // mantissa = (mantissa >> shift) + halfbit(adjusted);
+
+ static const int mantissa_offset = 0;
+ static const int exponent_offset = mantissa_offset + mbits;
+ static const int sign_offset = exponent_offset + ebits;
+ VIXL_ASSERT(sign_offset == (sizeof(T) * 8 - 1));
+
+ // Bail out early for zero inputs.
+ if (mantissa == 0) {
+ return static_cast<T>(sign << sign_offset);
+ }
+
+ // If all bits in the exponent are set, the value is infinite or NaN.
+ // This is true for all binary IEEE-754 formats.
+ static const int infinite_exponent = (1 << ebits) - 1;
+ static const int max_normal_exponent = infinite_exponent - 1;
+
+ // Apply the exponent bias to encode it for the result. Doing this early makes
+ // it easy to detect values that will be infinite or subnormal.
+ exponent += max_normal_exponent >> 1;
+
+ if (exponent > max_normal_exponent) {
+ // Overflow: the input is too large for the result type to represent.
+ if (round_mode == FPTieEven) {
+ // FPTieEven rounding mode handles overflows using infinities.
+ exponent = infinite_exponent;
+ mantissa = 0;
+ } else {
+ VIXL_ASSERT(round_mode == FPRoundOdd);
+ // FPRoundOdd rounding mode handles overflows using the largest magnitude
+ // normal number.
+ exponent = max_normal_exponent;
+ mantissa = (UINT64_C(1) << exponent_offset) - 1;
+ }
+ return static_cast<T>((sign << sign_offset) |
+ (exponent << exponent_offset) |
+ (mantissa << mantissa_offset));
+ }
+
+ // Calculate the shift required to move the top mantissa bit to the proper
+ // place in the destination type.
+ const int highest_significant_bit = 63 - CountLeadingZeros(mantissa);
+ int shift = highest_significant_bit - mbits;
+
+ if (exponent <= 0) {
+ // The output will be subnormal (before rounding).
+ // For subnormal outputs, the shift must be adjusted by the exponent. The +1
+ // is necessary because the exponent of a subnormal value (encoded as 0) is
+ // the same as the exponent of the smallest normal value (encoded as 1).
+ shift += -exponent + 1;
+
+ // Handle inputs that would produce a zero output.
+ //
+ // Shifts higher than highest_significant_bit+1 will always produce a zero
+ // result. A shift of exactly highest_significant_bit+1 might produce a
+ // non-zero result after rounding.
+ if (shift > (highest_significant_bit + 1)) {
+ if (round_mode == FPTieEven) {
+ // The result will always be +/-0.0.
+ return static_cast<T>(sign << sign_offset);
+ } else {
+ VIXL_ASSERT(round_mode == FPRoundOdd);
+ VIXL_ASSERT(mantissa != 0);
+ // For FPRoundOdd, if the mantissa is too small to represent and
+ // non-zero return the next "odd" value.
+ return static_cast<T>((sign << sign_offset) | 1);
+ }
+ }
+
+ // Properly encode the exponent for a subnormal output.
+ exponent = 0;
+ } else {
+ // Clear the topmost mantissa bit, since this is not encoded in IEEE-754
+ // normal values.
+ mantissa &= ~(UINT64_C(1) << highest_significant_bit);
+ }
+
+ // The casts below are only well-defined for unsigned integers.
+ VIXL_STATIC_ASSERT(std::numeric_limits<T>::is_integer);
+ VIXL_STATIC_ASSERT(!std::numeric_limits<T>::is_signed);
+
+ if (shift > 0) {
+ if (round_mode == FPTieEven) {
+ // We have to shift the mantissa to the right. Some precision is lost, so
+ // we need to apply rounding.
+ uint64_t onebit_mantissa = (mantissa >> (shift)) & 1;
+ uint64_t halfbit_mantissa = (mantissa >> (shift - 1)) & 1;
+ uint64_t adjustment = (halfbit_mantissa & ~onebit_mantissa);
+ uint64_t adjusted = mantissa - adjustment;
+ T halfbit_adjusted = (adjusted >> (shift - 1)) & 1;
+
+ T result =
+ static_cast<T>((sign << sign_offset) | (exponent << exponent_offset) |
+ ((mantissa >> shift) << mantissa_offset));
+
+ // A very large mantissa can overflow during rounding. If this happens,
+ // the exponent should be incremented and the mantissa set to 1.0
+ // (encoded as 0). Applying halfbit_adjusted after assembling the float
+ // has the nice side-effect that this case is handled for free.
+ //
+ // This also handles cases where a very large finite value overflows to
+ // infinity, or where a very large subnormal value overflows to become
+ // normal.
+ return result + halfbit_adjusted;
+ } else {
+ VIXL_ASSERT(round_mode == FPRoundOdd);
+ // If any bits at position halfbit or below are set, onebit (ie. the
+ // bottom bit of the resulting mantissa) must be set.
+ uint64_t fractional_bits = mantissa & ((UINT64_C(1) << shift) - 1);
+ if (fractional_bits != 0) {
+ mantissa |= UINT64_C(1) << shift;
+ }
+
+ return static_cast<T>((sign << sign_offset) |
+ (exponent << exponent_offset) |
+ ((mantissa >> shift) << mantissa_offset));
+ }
+ } else {
+ // We have to shift the mantissa to the left (or not at all). The input
+ // mantissa is exactly representable in the output mantissa, so apply no
+ // rounding correction.
+ return static_cast<T>((sign << sign_offset) |
+ (exponent << exponent_offset) |
+ ((mantissa << -shift) << mantissa_offset));
+ }
+}
+
+
+// See FPRound for a description of this function.
+inline double FPRoundToDouble(int64_t sign,
+ int64_t exponent,
+ uint64_t mantissa,
+ FPRounding round_mode) {
+ uint64_t bits =
+ FPRound<uint64_t, kDoubleExponentBits, kDoubleMantissaBits>(sign,
+ exponent,
+ mantissa,
+ round_mode);
+ return RawbitsToDouble(bits);
+}
+
+
+// See FPRound for a description of this function.
+inline Float16 FPRoundToFloat16(int64_t sign,
+ int64_t exponent,
+ uint64_t mantissa,
+ FPRounding round_mode) {
+ return RawbitsToFloat16(
+ FPRound<uint16_t,
+ kFloat16ExponentBits,
+ kFloat16MantissaBits>(sign, exponent, mantissa, round_mode));
+}
+
+
+// See FPRound for a description of this function.
+static inline float FPRoundToFloat(int64_t sign,
+ int64_t exponent,
+ uint64_t mantissa,
+ FPRounding round_mode) {
+ uint32_t bits =
+ FPRound<uint32_t, kFloatExponentBits, kFloatMantissaBits>(sign,
+ exponent,
+ mantissa,
+ round_mode);
+ return RawbitsToFloat(bits);
+}
+
+
+float FPToFloat(Float16 value, UseDefaultNaN DN, bool* exception = NULL);
+float FPToFloat(double value,
+ FPRounding round_mode,
+ UseDefaultNaN DN,
+ bool* exception = NULL);
+
+double FPToDouble(Float16 value, UseDefaultNaN DN, bool* exception = NULL);
+double FPToDouble(float value, UseDefaultNaN DN, bool* exception = NULL);
+
+Float16 FPToFloat16(float value,
+ FPRounding round_mode,
+ UseDefaultNaN DN,
+ bool* exception = NULL);
+
+Float16 FPToFloat16(double value,
+ FPRounding round_mode,
+ UseDefaultNaN DN,
+ bool* exception = NULL);
+} // namespace vixl
+
+#endif // VIXL_UTILS_H
diff --git a/js/src/jit/loong64/Architecture-loong64.cpp b/js/src/jit/loong64/Architecture-loong64.cpp
new file mode 100644
index 0000000000..6b1069a592
--- /dev/null
+++ b/js/src/jit/loong64/Architecture-loong64.cpp
@@ -0,0 +1,87 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/loong64/Architecture-loong64.h"
+
+#include "jit/FlushICache.h" // js::jit::FlushICache
+#include "jit/loong64/Simulator-loong64.h"
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+Registers::Code Registers::FromName(const char* name) {
+ for (size_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(i), name) == 0) {
+ return Code(i);
+ }
+ }
+
+ return Invalid;
+}
+
+FloatRegisters::Code FloatRegisters::FromName(const char* name) {
+ for (size_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(i), name) == 0) {
+ return Code(i);
+ }
+ }
+
+ return Invalid;
+}
+
+FloatRegisterSet FloatRegister::ReduceSetForPush(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ LiveFloatRegisterSet ret;
+ for (FloatRegisterIterator iter(s); iter.more(); ++iter) {
+ ret.addUnchecked(FromCode((*iter).encoding()));
+ }
+ return ret.set();
+}
+
+uint32_t FloatRegister::GetPushSizeInBytes(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ return s.size() * sizeof(double);
+}
+
+uint32_t FloatRegister::getRegisterDumpOffsetInBytes() {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ return encoding() * sizeof(double);
+}
+
+bool CPUFlagsHaveBeenComputed() {
+ // TODO(loong64): Add CPU flags support.
+ return true;
+}
+
+uint32_t GetLOONG64Flags() { return 0; }
+
+void FlushICache(void* code, size_t size) {
+#if defined(JS_SIMULATOR)
+ js::jit::SimulatorProcess::FlushICache(code, size);
+
+#elif defined(__GNUC__)
+ intptr_t end = reinterpret_cast<intptr_t>(code) + size;
+ __builtin___clear_cache(reinterpret_cast<char*>(code),
+ reinterpret_cast<char*>(end));
+
+#else
+ _flush_cache(reinterpret_cast<char*>(code), size, BCACHE);
+
+#endif
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/loong64/Architecture-loong64.h b/js/src/jit/loong64/Architecture-loong64.h
new file mode 100644
index 0000000000..9812c1d44f
--- /dev/null
+++ b/js/src/jit/loong64/Architecture-loong64.h
@@ -0,0 +1,524 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_Architecture_loong64_h
+#define jit_loong64_Architecture_loong64_h
+
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+#include <iterator>
+
+#include "jit/shared/Architecture-shared.h"
+
+#include "js/Utility.h"
+
+namespace js {
+namespace jit {
+
+// LoongArch64 has 32 64-bit integer registers, r0 though r31.
+// The program counter is not accessible as a register.
+//
+// SIMD and scalar floating-point registers share a register bank.
+// Floating-point registers are f0 through f31.
+// 128 bit SIMD registers are vr0 through vr31.
+// e.g., f0 is the bottom 64 bits of vr0.
+
+// LoongArch64 INT Register Convention:
+// Name Alias Usage
+// $r0 $zero Constant zero
+// $r1 $ra Return address
+// $r2 $tp TLS
+// $r3 $sp Stack pointer
+// $r4-$r11 $a0-$a7 Argument registers
+// $r4-$r5 $v0-$v1 Return values
+// $r12-$r20 $t0-$t8 Temporary registers
+// $r21 $x Reserved
+// $r22 $fp Frame pointer
+// $r23-$r31 $s0-$s8 Callee-saved registers
+
+// LoongArch64 FP Register Convention:
+// Name Alias Usage
+// $f0-$f7 $fa0-$fa7 Argument registers
+// $f0-$f1 $fv0-$fv1 Return values
+// $f8-f23 $ft0-$ft15 Temporary registers
+// $f24-$f31 $fs0-$fs7 Callee-saved registers
+
+class Registers {
+ public:
+ enum RegisterID {
+ r0 = 0,
+ r1,
+ r2,
+ r3,
+ r4,
+ r5,
+ r6,
+ r7,
+ r8,
+ r9,
+ r10,
+ r11,
+ r12,
+ r13,
+ r14,
+ r15,
+ r16,
+ r17,
+ r18,
+ r19,
+ r20,
+ r21,
+ r22,
+ r23,
+ r24,
+ r25,
+ r26,
+ r27,
+ r28,
+ r29,
+ r30,
+ r31,
+ zero = r0,
+ ra = r1,
+ tp = r2,
+ sp = r3,
+ a0 = r4,
+ a1 = r5,
+ a2 = r6,
+ a3 = r7,
+ a4 = r8,
+ a5 = r9,
+ a6 = r10,
+ a7 = r11,
+ t0 = r12,
+ t1 = r13,
+ t2 = r14,
+ t3 = r15,
+ t4 = r16,
+ t5 = r17,
+ t6 = r18,
+ t7 = r19,
+ t8 = r20,
+ rx = r21,
+ fp = r22,
+ s0 = r23,
+ s1 = r24,
+ s2 = r25,
+ s3 = r26,
+ s4 = r27,
+ s5 = r28,
+ s6 = r29,
+ s7 = r30,
+ s8 = r31,
+ invalid_reg,
+ };
+ typedef uint8_t Code;
+ typedef RegisterID Encoding;
+ typedef uint32_t SetType;
+
+ static const Encoding StackPointer = sp;
+ static const Encoding Invalid = invalid_reg;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ uintptr_t r;
+ };
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return mozilla::CountPopulation32(x);
+ }
+ static uint32_t FirstBit(SetType x) {
+ return mozilla::CountTrailingZeroes32(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ return 31 - mozilla::CountLeadingZeroes32(x);
+ }
+
+ static const char* GetName(uint32_t code) {
+ static const char* const Names[] = {
+ "zero", "ra", "tp", "sp", "a0", "a1", "a2", "a3", "a4", "a5", "a6",
+ "a7", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "rx",
+ "fp", "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7", "s8"};
+ static_assert(Total == std::size(Names), "Table is the correct size");
+ if (code >= Total) {
+ return "invalid";
+ }
+ return Names[code];
+ }
+
+ static Code FromName(const char* name);
+
+ static const uint32_t Total = 32;
+ static const uint32_t TotalPhys = 32;
+ static const uint32_t Allocatable =
+ 23; // No named special-function registers.
+
+ static const SetType AllMask = 0xFFFFFFFF;
+ static const SetType NoneMask = 0x0;
+
+ static const SetType ArgRegMask =
+ (1 << Registers::a0) | (1 << Registers::a1) | (1 << Registers::a2) |
+ (1 << Registers::a3) | (1 << Registers::a4) | (1 << Registers::a5) |
+ (1 << Registers::a6) | (1 << Registers::a7);
+
+ static const SetType VolatileMask =
+ (1 << Registers::a0) | (1 << Registers::a1) | (1 << Registers::a2) |
+ (1 << Registers::a3) | (1 << Registers::a4) | (1 << Registers::a5) |
+ (1 << Registers::a6) | (1 << Registers::a7) | (1 << Registers::t0) |
+ (1 << Registers::t1) | (1 << Registers::t2) | (1 << Registers::t3) |
+ (1 << Registers::t4) | (1 << Registers::t5) | (1 << Registers::t6);
+
+ // We use this constant to save registers when entering functions. This
+ // is why $ra is added here even though it is not "Non Volatile".
+ static const SetType NonVolatileMask =
+ (1 << Registers::ra) | (1 << Registers::fp) | (1 << Registers::s0) |
+ (1 << Registers::s1) | (1 << Registers::s2) | (1 << Registers::s3) |
+ (1 << Registers::s4) | (1 << Registers::s5) | (1 << Registers::s6) |
+ (1 << Registers::s7) | (1 << Registers::s8);
+
+ static const SetType SingleByteRegs = VolatileMask | NonVolatileMask;
+
+ static const SetType NonAllocatableMask =
+ (1 << Registers::zero) | // Always be zero.
+ (1 << Registers::t7) | // First scratch register.
+ (1 << Registers::t8) | // Second scratch register.
+ (1 << Registers::rx) | // Reserved Register.
+ (1 << Registers::ra) | (1 << Registers::tp) | (1 << Registers::sp) |
+ (1 << Registers::fp);
+
+ static const SetType WrapperMask = VolatileMask;
+
+ // Registers returned from a JS -> JS call.
+ static const SetType JSCallMask = (1 << Registers::a2);
+
+ // Registers returned from a JS -> C call.
+ static const SetType CallMask = (1 << Registers::a0);
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+};
+
+// Smallest integer type that can hold a register bitmask.
+typedef uint32_t PackedRegisterMask;
+
+template <typename T>
+class TypedRegisterSet;
+
+class FloatRegisters {
+ public:
+ enum FPRegisterID {
+ f0 = 0,
+ f1,
+ f2,
+ f3,
+ f4,
+ f5,
+ f6,
+ f7,
+ f8,
+ f9,
+ f10,
+ f11,
+ f12,
+ f13,
+ f14,
+ f15,
+ f16,
+ f17,
+ f18,
+ f19,
+ f20,
+ f21,
+ f22,
+ f23, // Scratch register.
+ f24,
+ f25,
+ f26,
+ f27,
+ f28,
+ f29,
+ f30,
+ f31,
+ };
+
+ // Eight bits: (invalid << 7) | (kind << 5) | encoding
+ typedef uint8_t Code;
+ typedef FPRegisterID Encoding;
+ typedef uint64_t SetType;
+
+ enum Kind : uint8_t { Double, Single, NumTypes };
+
+ static constexpr Code Invalid = 0x80;
+
+ static const char* GetName(uint32_t code) {
+ static const char* const Names[] = {
+ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
+ "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",
+ "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
+ "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"};
+ static_assert(TotalPhys == std::size(Names), "Table is the correct size");
+ if (code >= Total) {
+ return "invalid";
+ }
+ return Names[code];
+ }
+
+ static Code FromName(const char* name);
+
+ static const uint32_t TotalPhys = 32;
+ static const uint32_t Total = TotalPhys * NumTypes;
+ static const uint32_t Allocatable = 31; // Without f23, the scratch register.
+
+ static_assert(sizeof(SetType) * 8 >= Total,
+ "SetType should be large enough to enumerate all registers.");
+
+ // Magic values which are used to duplicate a mask of physical register for
+ // a specific type of register. A multiplication is used to copy and shift
+ // the bits of the physical register mask.
+ static const SetType SpreadSingle = SetType(1)
+ << (uint32_t(Single) * TotalPhys);
+ static const SetType SpreadDouble = SetType(1)
+ << (uint32_t(Double) * TotalPhys);
+ static const SetType Spread = SpreadSingle | SpreadDouble;
+
+ static const SetType AllPhysMask = ((SetType(1) << TotalPhys) - 1);
+ static const SetType AllMask = AllPhysMask * Spread;
+ static const SetType AllSingleMask = AllPhysMask * SpreadSingle;
+ static const SetType AllDoubleMask = AllPhysMask * SpreadDouble;
+ static const SetType NoneMask = SetType(0);
+
+ // TODO(loong64): Much less than ARM64 here.
+ static const SetType NonVolatileMask =
+ SetType((1 << FloatRegisters::f24) | (1 << FloatRegisters::f25) |
+ (1 << FloatRegisters::f26) | (1 << FloatRegisters::f27) |
+ (1 << FloatRegisters::f28) | (1 << FloatRegisters::f29) |
+ (1 << FloatRegisters::f30) | (1 << FloatRegisters::f31)) *
+ Spread;
+
+ static const SetType VolatileMask = AllMask & ~NonVolatileMask;
+
+ static const SetType WrapperMask = VolatileMask;
+
+ // f23 is the scratch register.
+ static const SetType NonAllocatableMask =
+ (SetType(1) << FloatRegisters::f23) * Spread;
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ float s;
+ double d;
+ };
+
+ static constexpr Encoding encoding(Code c) {
+ // assert() not available in constexpr function.
+ // assert(c < Total);
+ return Encoding(c & 31);
+ }
+
+ static constexpr Kind kind(Code c) {
+ // assert() not available in constexpr function.
+ // assert(c < Total && ((c >> 5) & 3) < NumTypes);
+ return Kind((c >> 5) & 3);
+ }
+
+ static constexpr Code fromParts(uint32_t encoding, uint32_t kind,
+ uint32_t invalid) {
+ return Code((invalid << 7) | (kind << 5) | encoding);
+ }
+};
+
+static const uint32_t SpillSlotSize =
+ std::max(sizeof(Registers::RegisterContent),
+ sizeof(FloatRegisters::RegisterContent));
+
+static const uint32_t ShadowStackSpace = 0;
+static const uint32_t SizeOfReturnAddressAfterCall = 0;
+
+// When our only strategy for far jumps is to encode the offset directly, and
+// not insert any jump islands during assembly for even further jumps, then the
+// architecture restricts us to -2^27 .. 2^27-4, to fit into a signed 28-bit
+// value. We further reduce this range to allow the far-jump inserting code to
+// have some breathing room.
+static const uint32_t JumpImmediateRange = ((1 << 27) - (20 * 1024 * 1024));
+
+struct FloatRegister {
+ typedef FloatRegisters Codes;
+ typedef size_t Code;
+ typedef Codes::Encoding Encoding;
+ typedef Codes::SetType SetType;
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
+ x |= x >> FloatRegisters::TotalPhys;
+ x &= FloatRegisters::AllPhysMask;
+ return mozilla::CountPopulation32(x);
+ }
+
+ static uint32_t FirstBit(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType");
+ return mozilla::CountTrailingZeroes64(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType");
+ return 63 - mozilla::CountLeadingZeroes64(x);
+ }
+
+ private:
+ // These fields only hold valid values: an invalid register is always
+ // represented as a valid encoding and kind with the invalid_ bit set.
+ uint8_t encoding_; // 32 encodings
+ uint8_t kind_; // Double, Single; more later
+ bool invalid_;
+
+ typedef Codes::Kind Kind;
+
+ public:
+ constexpr FloatRegister(Encoding encoding, Kind kind)
+ : encoding_(encoding), kind_(kind), invalid_(false) {
+ // assert(uint32_t(encoding) < Codes::TotalPhys);
+ }
+
+ constexpr FloatRegister()
+ : encoding_(0), kind_(FloatRegisters::Double), invalid_(true) {}
+
+ static FloatRegister FromCode(uint32_t i) {
+ MOZ_ASSERT(i < Codes::Total);
+ return FloatRegister(FloatRegisters::encoding(i), FloatRegisters::kind(i));
+ }
+
+ bool isSingle() const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == FloatRegisters::Single;
+ }
+ bool isDouble() const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == FloatRegisters::Double;
+ }
+ bool isSimd128() const {
+ MOZ_ASSERT(!invalid_);
+ return false;
+ }
+ bool isInvalid() const { return invalid_; }
+
+ FloatRegister asSingle() const {
+ MOZ_ASSERT(!invalid_);
+ return FloatRegister(Encoding(encoding_), FloatRegisters::Single);
+ }
+ FloatRegister asDouble() const {
+ MOZ_ASSERT(!invalid_);
+ return FloatRegister(Encoding(encoding_), FloatRegisters::Double);
+ }
+ FloatRegister asSimd128() const { MOZ_CRASH(); }
+
+ constexpr uint32_t size() const {
+ MOZ_ASSERT(!invalid_);
+ if (kind_ == FloatRegisters::Double) {
+ return sizeof(double);
+ }
+ MOZ_ASSERT(kind_ == FloatRegisters::Single);
+ return sizeof(float);
+ }
+
+ constexpr Code code() const {
+ // assert(!invalid_);
+ return Codes::fromParts(encoding_, kind_, invalid_);
+ }
+
+ constexpr Encoding encoding() const {
+ MOZ_ASSERT(!invalid_);
+ return Encoding(encoding_);
+ }
+
+ const char* name() const { return FloatRegisters::GetName(code()); }
+ bool volatile_() const {
+ MOZ_ASSERT(!invalid_);
+ return !!((SetType(1) << code()) & FloatRegisters::VolatileMask);
+ }
+ constexpr bool operator!=(FloatRegister other) const {
+ return code() != other.code();
+ }
+ constexpr bool operator==(FloatRegister other) const {
+ return code() == other.code();
+ }
+
+ bool aliases(FloatRegister other) const {
+ return other.encoding_ == encoding_;
+ }
+ // Ensure that two floating point registers' types are equivalent.
+ bool equiv(FloatRegister other) const {
+ MOZ_ASSERT(!invalid_);
+ return kind_ == other.kind_;
+ }
+
+ uint32_t numAliased() const { return Codes::NumTypes; }
+ uint32_t numAlignedAliased() { return numAliased(); }
+
+ FloatRegister aliased(uint32_t aliasIdx) {
+ MOZ_ASSERT(!invalid_);
+ MOZ_ASSERT(aliasIdx < numAliased());
+ return FloatRegister(Encoding(encoding_),
+ Kind((aliasIdx + kind_) % numAliased()));
+ }
+ FloatRegister alignedAliased(uint32_t aliasIdx) {
+ MOZ_ASSERT(aliasIdx < numAliased());
+ return aliased(aliasIdx);
+ }
+ SetType alignedOrDominatedAliasedSet() const {
+ return Codes::Spread << encoding_;
+ }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName Name = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return LiveAsIndexableSet<Name>(s);
+ }
+
+ static TypedRegisterSet<FloatRegister> ReduceSetForPush(
+ const TypedRegisterSet<FloatRegister>& s);
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>& s);
+ uint32_t getRegisterDumpOffsetInBytes();
+};
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ return set & FloatRegisters::AllDoubleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Any>(SetType set) {
+ return set;
+}
+
+// LoongArch doesn't have double registers that cannot be treated as float32.
+inline bool hasUnaliasedDouble() { return false; }
+
+// LoongArch doesn't have double registers that alias multiple floats.
+inline bool hasMultiAlias() { return false; }
+
+uint32_t GetLOONG64Flags();
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_Architecture_loong64_h */
diff --git a/js/src/jit/loong64/Assembler-loong64.cpp b/js/src/jit/loong64/Assembler-loong64.cpp
new file mode 100644
index 0000000000..1a4976d07a
--- /dev/null
+++ b/js/src/jit/loong64/Assembler-loong64.cpp
@@ -0,0 +1,2478 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/loong64/Assembler-loong64.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Maybe.h"
+
+#include "gc/Marking.h"
+#include "jit/AutoWritableJitCode.h"
+#include "jit/ExecutableAllocator.h"
+#include "vm/Realm.h"
+
+using mozilla::DebugOnly;
+
+using namespace js;
+using namespace js::jit;
+
+// Note this is used for inter-wasm calls and may pass arguments and results
+// in floating point registers even if the system ABI does not.
+
+// TODO(loong64): Inconsistent with LoongArch's calling convention.
+// LoongArch floating-point parameters calling convention:
+// The first eight floating-point parameters should be passed in f0-f7, and
+// the other floating point parameters will be passed like integer parameters.
+// But we just pass the other floating-point parameters on stack here.
+ABIArg ABIArgGenerator::next(MIRType type) {
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults: {
+ if (intRegIndex_ == NumIntArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uintptr_t);
+ break;
+ }
+ current_ = ABIArg(Register::FromCode(intRegIndex_ + a0.encoding()));
+ intRegIndex_++;
+ break;
+ }
+ case MIRType::Float32:
+ case MIRType::Double: {
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(double);
+ break;
+ }
+ current_ = ABIArg(FloatRegister(
+ FloatRegisters::Encoding(floatRegIndex_ + f0.encoding()),
+ type == MIRType::Double ? FloatRegisters::Double
+ : FloatRegisters::Single));
+ floatRegIndex_++;
+ break;
+ }
+ case MIRType::Simd128: {
+ MOZ_CRASH("LoongArch does not support simd yet.");
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+}
+
+// Encode a standard register when it is being used as rd, the rj, and
+// an extra register(rk). These should never be called with an InvalidReg.
+uint32_t js::jit::RJ(Register r) {
+ MOZ_ASSERT(r != InvalidReg);
+ return r.encoding() << RJShift;
+}
+
+uint32_t js::jit::RK(Register r) {
+ MOZ_ASSERT(r != InvalidReg);
+ return r.encoding() << RKShift;
+}
+
+uint32_t js::jit::RD(Register r) {
+ MOZ_ASSERT(r != InvalidReg);
+ return r.encoding() << RDShift;
+}
+
+uint32_t js::jit::FJ(FloatRegister r) { return r.encoding() << RJShift; }
+
+uint32_t js::jit::FK(FloatRegister r) { return r.encoding() << RKShift; }
+
+uint32_t js::jit::FD(FloatRegister r) { return r.encoding() << RDShift; }
+
+uint32_t js::jit::FA(FloatRegister r) { return r.encoding() << FAShift; }
+
+uint32_t js::jit::SA2(uint32_t value) {
+ MOZ_ASSERT(value < 4);
+ return (value & SA2Mask) << SAShift;
+}
+
+uint32_t js::jit::SA3(uint32_t value) {
+ MOZ_ASSERT(value < 8);
+ return (value & SA3Mask) << SAShift;
+}
+
+Register js::jit::toRK(Instruction& i) {
+ return Register::FromCode((i.encode() & RKMask) >> RKShift);
+}
+
+Register js::jit::toRJ(Instruction& i) {
+ return Register::FromCode((i.encode() & RJMask) >> RJShift);
+}
+
+Register js::jit::toRD(Instruction& i) {
+ return Register::FromCode((i.encode() & RDMask) >> RDShift);
+}
+
+Register js::jit::toR(Instruction& i) {
+ return Register::FromCode(i.encode() & RegMask);
+}
+
+void InstImm::extractImm16(BOffImm16* dest) { *dest = BOffImm16(*this); }
+
+void AssemblerLOONG64::finish() {
+ MOZ_ASSERT(!isFinished);
+ isFinished = true;
+}
+
+bool AssemblerLOONG64::appendRawCode(const uint8_t* code, size_t numBytes) {
+ return m_buffer.appendRawCode(code, numBytes);
+}
+
+bool AssemblerLOONG64::reserve(size_t size) {
+ // This buffer uses fixed-size chunks so there's no point in reserving
+ // now vs. on-demand.
+ return !oom();
+}
+
+bool AssemblerLOONG64::swapBuffer(wasm::Bytes& bytes) {
+ // For now, specialize to the one use case. As long as wasm::Bytes is a
+ // Vector, not a linked-list of chunks, there's not much we can do other
+ // than copy.
+ MOZ_ASSERT(bytes.empty());
+ if (!bytes.resize(bytesNeeded())) {
+ return false;
+ }
+ m_buffer.executableCopy(bytes.begin());
+ return true;
+}
+
+void AssemblerLOONG64::copyJumpRelocationTable(uint8_t* dest) {
+ if (jumpRelocations_.length()) {
+ memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
+ }
+}
+
+void AssemblerLOONG64::copyDataRelocationTable(uint8_t* dest) {
+ if (dataRelocations_.length()) {
+ memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
+ }
+}
+
+AssemblerLOONG64::Condition AssemblerLOONG64::InvertCondition(Condition cond) {
+ switch (cond) {
+ case Equal:
+ return NotEqual;
+ case NotEqual:
+ return Equal;
+ case Zero:
+ return NonZero;
+ case NonZero:
+ return Zero;
+ case LessThan:
+ return GreaterThanOrEqual;
+ case LessThanOrEqual:
+ return GreaterThan;
+ case GreaterThan:
+ return LessThanOrEqual;
+ case GreaterThanOrEqual:
+ return LessThan;
+ case Above:
+ return BelowOrEqual;
+ case AboveOrEqual:
+ return Below;
+ case Below:
+ return AboveOrEqual;
+ case BelowOrEqual:
+ return Above;
+ case Signed:
+ return NotSigned;
+ case NotSigned:
+ return Signed;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+AssemblerLOONG64::DoubleCondition AssemblerLOONG64::InvertCondition(
+ DoubleCondition cond) {
+ switch (cond) {
+ case DoubleOrdered:
+ return DoubleUnordered;
+ case DoubleEqual:
+ return DoubleNotEqualOrUnordered;
+ case DoubleNotEqual:
+ return DoubleEqualOrUnordered;
+ case DoubleGreaterThan:
+ return DoubleLessThanOrEqualOrUnordered;
+ case DoubleGreaterThanOrEqual:
+ return DoubleLessThanOrUnordered;
+ case DoubleLessThan:
+ return DoubleGreaterThanOrEqualOrUnordered;
+ case DoubleLessThanOrEqual:
+ return DoubleGreaterThanOrUnordered;
+ case DoubleUnordered:
+ return DoubleOrdered;
+ case DoubleEqualOrUnordered:
+ return DoubleNotEqual;
+ case DoubleNotEqualOrUnordered:
+ return DoubleEqual;
+ case DoubleGreaterThanOrUnordered:
+ return DoubleLessThanOrEqual;
+ case DoubleGreaterThanOrEqualOrUnordered:
+ return DoubleLessThan;
+ case DoubleLessThanOrUnordered:
+ return DoubleGreaterThanOrEqual;
+ case DoubleLessThanOrEqualOrUnordered:
+ return DoubleGreaterThan;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+AssemblerLOONG64::Condition AssemblerLOONG64::InvertCmpCondition(
+ Condition cond) {
+ switch (cond) {
+ case Equal:
+ case NotEqual:
+ return cond;
+ case LessThan:
+ return GreaterThan;
+ case LessThanOrEqual:
+ return GreaterThanOrEqual;
+ case GreaterThan:
+ return LessThanOrEqual;
+ case GreaterThanOrEqual:
+ return LessThan;
+ case Above:
+ return Below;
+ case AboveOrEqual:
+ return BelowOrEqual;
+ case Below:
+ return Above;
+ case BelowOrEqual:
+ return AboveOrEqual;
+ default:
+ MOZ_CRASH("no meaningful swapped-operand condition");
+ }
+}
+
+BOffImm16::BOffImm16(InstImm inst)
+ : data((inst.encode() >> Imm16Shift) & Imm16Mask) {}
+
+Instruction* BOffImm16::getDest(Instruction* src) const {
+ return &src[(((int32_t)data << 16) >> 16) + 1];
+}
+
+bool AssemblerLOONG64::oom() const {
+ return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() ||
+ dataRelocations_.oom();
+}
+
+// Size of the instruction stream, in bytes.
+size_t AssemblerLOONG64::size() const { return m_buffer.size(); }
+
+// Size of the relocation table, in bytes.
+size_t AssemblerLOONG64::jumpRelocationTableBytes() const {
+ return jumpRelocations_.length();
+}
+
+size_t AssemblerLOONG64::dataRelocationTableBytes() const {
+ return dataRelocations_.length();
+}
+
+// Size of the data table, in bytes.
+size_t AssemblerLOONG64::bytesNeeded() const {
+ return size() + jumpRelocationTableBytes() + dataRelocationTableBytes();
+}
+
+// write a blob of binary into the instruction stream
+BufferOffset AssemblerLOONG64::writeInst(uint32_t x, uint32_t* dest) {
+ MOZ_ASSERT(hasCreator());
+ if (dest == nullptr) {
+ return m_buffer.putInt(x);
+ }
+
+ WriteInstStatic(x, dest);
+ return BufferOffset();
+}
+
+void AssemblerLOONG64::WriteInstStatic(uint32_t x, uint32_t* dest) {
+ MOZ_ASSERT(dest != nullptr);
+ *dest = x;
+}
+
+BufferOffset AssemblerLOONG64::haltingAlign(int alignment) {
+ // TODO(loong64): Implement a proper halting align.
+ return nopAlign(alignment);
+}
+
+BufferOffset AssemblerLOONG64::nopAlign(int alignment) {
+ BufferOffset ret;
+ MOZ_ASSERT(m_buffer.isAligned(4));
+ if (alignment == 8) {
+ if (!m_buffer.isAligned(alignment)) {
+ BufferOffset tmp = as_nop();
+ if (!ret.assigned()) {
+ ret = tmp;
+ }
+ }
+ } else {
+ MOZ_ASSERT((alignment & (alignment - 1)) == 0);
+ while (size() & (alignment - 1)) {
+ BufferOffset tmp = as_nop();
+ if (!ret.assigned()) {
+ ret = tmp;
+ }
+ }
+ }
+ return ret;
+}
+
+// Logical operations.
+BufferOffset AssemblerLOONG64::as_and(Register rd, Register rj, Register rk) {
+ spew("and %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_and, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_or(Register rd, Register rj, Register rk) {
+ spew("or %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_or, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_xor(Register rd, Register rj, Register rk) {
+ spew("xor %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_xor, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_nor(Register rd, Register rj, Register rk) {
+ spew("nor %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_nor, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_andn(Register rd, Register rj, Register rk) {
+ spew("andn %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_andn, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_orn(Register rd, Register rj, Register rk) {
+ spew("orn %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_orn, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_andi(Register rd, Register rj, int32_t ui12) {
+ MOZ_ASSERT(is_uintN(ui12, 12));
+ spew("andi %3s,%3s,0x%x", rd.name(), rj.name(), ui12);
+ return writeInst(InstImm(op_andi, ui12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ori(Register rd, Register rj, int32_t ui12) {
+ MOZ_ASSERT(is_uintN(ui12, 12));
+ spew("ori %3s,%3s,0x%x", rd.name(), rj.name(), ui12);
+ return writeInst(InstImm(op_ori, ui12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_xori(Register rd, Register rj, int32_t ui12) {
+ MOZ_ASSERT(is_uintN(ui12, 12));
+ spew("xori %3s,%3s,0x%x", rd.name(), rj.name(), ui12);
+ return writeInst(InstImm(op_xori, ui12, rj, rd, 12).encode());
+}
+
+// Branch and jump instructions
+BufferOffset AssemblerLOONG64::as_b(JOffImm26 off) {
+ spew("b %d", off.decode());
+ return writeInst(InstJump(op_b, off).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bl(JOffImm26 off) {
+ spew("bl %d", off.decode());
+ return writeInst(InstJump(op_bl, off).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_jirl(Register rd, Register rj,
+ BOffImm16 off) {
+ spew("jirl %3s, %3s, %d", rd.name(), rj.name(), off.decode());
+ return writeInst(InstImm(op_jirl, off, rj, rd).encode());
+}
+
+InstImm AssemblerLOONG64::getBranchCode(JumpOrCall jumpOrCall) {
+ // jirl or beq
+ if (jumpOrCall == BranchIsCall) {
+ return InstImm(op_jirl, BOffImm16(0), zero, ra);
+ }
+
+ return InstImm(op_beq, BOffImm16(0), zero, zero);
+}
+
+InstImm AssemblerLOONG64::getBranchCode(Register rj, Register rd, Condition c) {
+ // beq, bne
+ MOZ_ASSERT(c == AssemblerLOONG64::Equal || c == AssemblerLOONG64::NotEqual);
+ return InstImm(c == AssemblerLOONG64::Equal ? op_beq : op_bne, BOffImm16(0),
+ rj, rd);
+}
+
+InstImm AssemblerLOONG64::getBranchCode(Register rj, Condition c) {
+ // beq, bne, blt, bge
+ switch (c) {
+ case AssemblerLOONG64::Equal:
+ case AssemblerLOONG64::Zero:
+ case AssemblerLOONG64::BelowOrEqual:
+ return InstImm(op_beq, BOffImm16(0), rj, zero);
+ case AssemblerLOONG64::NotEqual:
+ case AssemblerLOONG64::NonZero:
+ case AssemblerLOONG64::Above:
+ return InstImm(op_bne, BOffImm16(0), rj, zero);
+ case AssemblerLOONG64::GreaterThan:
+ return InstImm(op_blt, BOffImm16(0), zero, rj);
+ case AssemblerLOONG64::GreaterThanOrEqual:
+ case AssemblerLOONG64::NotSigned:
+ return InstImm(op_bge, BOffImm16(0), rj, zero);
+ case AssemblerLOONG64::LessThan:
+ case AssemblerLOONG64::Signed:
+ return InstImm(op_blt, BOffImm16(0), rj, zero);
+ case AssemblerLOONG64::LessThanOrEqual:
+ return InstImm(op_bge, BOffImm16(0), zero, rj);
+ default:
+ MOZ_CRASH("Condition not supported.");
+ }
+}
+
+// Code semantics must conform to compareFloatingpoint
+InstImm AssemblerLOONG64::getBranchCode(FPConditionBit cj) {
+ return InstImm(op_bcz, 0, cj, true); // bcnez
+}
+
+// Arithmetic instructions
+BufferOffset AssemblerLOONG64::as_add_w(Register rd, Register rj, Register rk) {
+ spew("add_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_add_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_add_d(Register rd, Register rj, Register rk) {
+ spew("add_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_add_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sub_w(Register rd, Register rj, Register rk) {
+ spew("sub_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_sub_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sub_d(Register rd, Register rj, Register rk) {
+ spew("sub_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_sub_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_addi_w(Register rd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("addi_w %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_addi_w, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_addi_d(Register rd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("addi_d %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_addi_d, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_addu16i_d(Register rd, Register rj,
+ int32_t si16) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(si16));
+ spew("addu16i_d %3s,%3s,0x%x", rd.name(), rj.name(), si16);
+ return writeInst(InstImm(op_addu16i_d, Imm16(si16), rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_alsl_w(Register rd, Register rj, Register rk,
+ uint32_t sa2) {
+ MOZ_ASSERT(sa2 < 4);
+ spew("alsl_w %3s,%3s,0x%x", rd.name(), rj.name(), sa2);
+ return writeInst(InstReg(op_alsl_w, sa2, rk, rj, rd, 2).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_alsl_wu(Register rd, Register rj, Register rk,
+ uint32_t sa2) {
+ MOZ_ASSERT(sa2 < 4);
+ spew("alsl_wu %3s,%3s,0x%x", rd.name(), rj.name(), sa2);
+ return writeInst(InstReg(op_alsl_wu, sa2, rk, rj, rd, 2).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_alsl_d(Register rd, Register rj, Register rk,
+ uint32_t sa2) {
+ MOZ_ASSERT(sa2 < 4);
+ spew("alsl_d %3s,%3s,%3s,0x%x", rd.name(), rj.name(), rk.name(), sa2);
+ return writeInst(InstReg(op_alsl_d, sa2, rk, rj, rd, 2).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_lu12i_w(Register rd, int32_t si20) {
+ spew("lu12i_w %3s,0x%x", rd.name(), si20);
+ return writeInst(InstImm(op_lu12i_w, si20, rd, false).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_lu32i_d(Register rd, int32_t si20) {
+ spew("lu32i_d %3s,0x%x", rd.name(), si20);
+ return writeInst(InstImm(op_lu32i_d, si20, rd, false).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_lu52i_d(Register rd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_uintN(si12, 12));
+ spew("lu52i_d %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_lu52i_d, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_slt(Register rd, Register rj, Register rk) {
+ spew("slt %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_slt, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sltu(Register rd, Register rj, Register rk) {
+ spew("sltu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_sltu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_slti(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("slti %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_slti, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sltui(Register rd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("sltui %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_sltui, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_pcaddi(Register rd, int32_t si20) {
+ spew("pcaddi %3s,0x%x", rd.name(), si20);
+ return writeInst(InstImm(op_pcaddi, si20, rd, false).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_pcaddu12i(Register rd, int32_t si20) {
+ spew("pcaddu12i %3s,0x%x", rd.name(), si20);
+ return writeInst(InstImm(op_pcaddu12i, si20, rd, false).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_pcaddu18i(Register rd, int32_t si20) {
+ spew("pcaddu18i %3s,0x%x", rd.name(), si20);
+ return writeInst(InstImm(op_pcaddu18i, si20, rd, false).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_pcalau12i(Register rd, int32_t si20) {
+ spew("pcalau12i %3s,0x%x", rd.name(), si20);
+ return writeInst(InstImm(op_pcalau12i, si20, rd, false).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mul_w(Register rd, Register rj, Register rk) {
+ spew("mul_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mul_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mulh_w(Register rd, Register rj,
+ Register rk) {
+ spew("mulh_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mulh_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mulh_wu(Register rd, Register rj,
+ Register rk) {
+ spew("mulh_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mulh_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mul_d(Register rd, Register rj, Register rk) {
+ spew("mul_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mul_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mulh_d(Register rd, Register rj,
+ Register rk) {
+ spew("mulh_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mulh_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mulh_du(Register rd, Register rj,
+ Register rk) {
+ spew("mulh_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mulh_du, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mulw_d_w(Register rd, Register rj,
+ Register rk) {
+ spew("mulw_d_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mulw_d_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mulw_d_wu(Register rd, Register rj,
+ Register rk) {
+ spew("mulw_d_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mulw_d_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_div_w(Register rd, Register rj, Register rk) {
+ spew("div_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_div_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mod_w(Register rd, Register rj, Register rk) {
+ spew("mod_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mod_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_div_wu(Register rd, Register rj,
+ Register rk) {
+ spew("div_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_div_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mod_wu(Register rd, Register rj,
+ Register rk) {
+ spew("mod_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mod_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_div_d(Register rd, Register rj, Register rk) {
+ spew("div_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_div_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mod_d(Register rd, Register rj, Register rk) {
+ spew("mod_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mod_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_div_du(Register rd, Register rj,
+ Register rk) {
+ spew("div_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_div_du, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_mod_du(Register rd, Register rj,
+ Register rk) {
+ spew("mod_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_mod_du, rk, rj, rd).encode());
+}
+
+// Shift instructions
+BufferOffset AssemblerLOONG64::as_sll_w(Register rd, Register rj, Register rk) {
+ spew("sll_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_sll_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_srl_w(Register rd, Register rj, Register rk) {
+ spew("srl_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_srl_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sra_w(Register rd, Register rj, Register rk) {
+ spew("sra_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_sra_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_rotr_w(Register rd, Register rj,
+ Register rk) {
+ spew("rotr_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_rotr_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_slli_w(Register rd, Register rj,
+ int32_t ui5) {
+ MOZ_ASSERT(is_uintN(ui5, 5));
+ spew("slli_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5);
+ return writeInst(InstImm(op_slli_w, ui5, rj, rd, 5).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_srli_w(Register rd, Register rj,
+ int32_t ui5) {
+ MOZ_ASSERT(is_uintN(ui5, 5));
+ spew("srli_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5);
+ return writeInst(InstImm(op_srli_w, ui5, rj, rd, 5).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_srai_w(Register rd, Register rj,
+ int32_t ui5) {
+ MOZ_ASSERT(is_uintN(ui5, 5));
+ spew("srai_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5);
+ return writeInst(InstImm(op_srai_w, ui5, rj, rd, 5).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_rotri_w(Register rd, Register rj,
+ int32_t ui5) {
+ MOZ_ASSERT(is_uintN(ui5, 5));
+ spew("rotri_w %3s,%3s,0x%x", rd.name(), rj.name(), ui5);
+ return writeInst(InstImm(op_rotri_w, ui5, rj, rd, 5).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sll_d(Register rd, Register rj, Register rk) {
+ spew("sll_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_sll_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_srl_d(Register rd, Register rj, Register rk) {
+ spew("srl_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_srl_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sra_d(Register rd, Register rj, Register rk) {
+ spew("sra_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_sra_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_rotr_d(Register rd, Register rj,
+ Register rk) {
+ spew("rotr_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_rotr_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_slli_d(Register rd, Register rj,
+ int32_t ui6) {
+ MOZ_ASSERT(is_uintN(ui6, 6));
+ spew("slli_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6);
+ return writeInst(InstImm(op_slli_d, ui6, rj, rd, 6).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_srli_d(Register rd, Register rj,
+ int32_t ui6) {
+ MOZ_ASSERT(is_uintN(ui6, 6));
+ spew("srli_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6);
+ return writeInst(InstImm(op_srli_d, ui6, rj, rd, 6).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_srai_d(Register rd, Register rj,
+ int32_t ui6) {
+ MOZ_ASSERT(is_uintN(ui6, 6));
+ spew("srai_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6);
+ return writeInst(InstImm(op_srai_d, ui6, rj, rd, 6).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_rotri_d(Register rd, Register rj,
+ int32_t ui6) {
+ MOZ_ASSERT(is_uintN(ui6, 6));
+ spew("rotri_d %3s,%3s,0x%x", rd.name(), rj.name(), ui6);
+ return writeInst(InstImm(op_rotri_d, ui6, rj, rd, 6).encode());
+}
+
+// Bit operation instrucitons
+BufferOffset AssemblerLOONG64::as_ext_w_b(Register rd, Register rj) {
+ spew("ext_w_b %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_ext_w_b, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ext_w_h(Register rd, Register rj) {
+ spew("ext_w_h %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_ext_w_h, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_clo_w(Register rd, Register rj) {
+ spew("clo_w %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_clo_w, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_clz_w(Register rd, Register rj) {
+ spew("clz_w %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_clz_w, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_cto_w(Register rd, Register rj) {
+ spew("cto_w %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_cto_w, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ctz_w(Register rd, Register rj) {
+ spew("ctz_w %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_ctz_w, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_clo_d(Register rd, Register rj) {
+ spew("clo_d %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_clo_d, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_clz_d(Register rd, Register rj) {
+ spew("clz_d %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_clz_d, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_cto_d(Register rd, Register rj) {
+ spew("cto_d %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_cto_d, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ctz_d(Register rd, Register rj) {
+ spew("ctz_d %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_ctz_d, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bytepick_w(Register rd, Register rj,
+ Register rk, int32_t sa2) {
+ MOZ_ASSERT(sa2 < 4);
+ spew("bytepick_w %3s,%3s,%3s, 0x%x", rd.name(), rj.name(), rk.name(), sa2);
+ return writeInst(InstReg(op_bytepick_w, sa2, rk, rj, rd, 2).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bytepick_d(Register rd, Register rj,
+ Register rk, int32_t sa3) {
+ MOZ_ASSERT(sa3 < 8);
+ spew("bytepick_d %3s,%3s,%3s, 0x%x", rd.name(), rj.name(), rk.name(), sa3);
+ return writeInst(InstReg(op_bytepick_d, sa3, rk, rj, rd, 3).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_revb_2h(Register rd, Register rj) {
+ spew("revb_2h %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_revb_2h, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_revb_4h(Register rd, Register rj) {
+ spew("revb_4h %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_revb_4h, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_revb_2w(Register rd, Register rj) {
+ spew("revb_2w %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_revb_2w, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_revb_d(Register rd, Register rj) {
+ spew("revb_d %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_revb_d, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_revh_2w(Register rd, Register rj) {
+ spew("revh_2w %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_revh_2w, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_revh_d(Register rd, Register rj) {
+ spew("revh_d %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_revh_d, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bitrev_4b(Register rd, Register rj) {
+ spew("bitrev_4b %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_bitrev_4b, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bitrev_8b(Register rd, Register rj) {
+ spew("bitrev_8b %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_bitrev_8b, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bitrev_w(Register rd, Register rj) {
+ spew("bitrev_w %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_bitrev_w, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bitrev_d(Register rd, Register rj) {
+ spew("bitrev_d %3s,%3s", rd.name(), rj.name());
+ return writeInst(InstReg(op_bitrev_d, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bstrins_w(Register rd, Register rj,
+ int32_t msbw, int32_t lsbw) {
+ MOZ_ASSERT(lsbw <= msbw);
+ spew("bstrins_w %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbw, lsbw);
+ return writeInst(InstImm(op_bstr_w, msbw, lsbw, rj, rd, 5).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bstrins_d(Register rd, Register rj,
+ int32_t msbd, int32_t lsbd) {
+ MOZ_ASSERT(lsbd <= msbd);
+ spew("bstrins_d %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbd, lsbd);
+ return writeInst(InstImm(op_bstrins_d, msbd, lsbd, rj, rd, 6).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bstrpick_w(Register rd, Register rj,
+ int32_t msbw, int32_t lsbw) {
+ MOZ_ASSERT(lsbw <= msbw);
+ spew("bstrpick_w %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbw, lsbw);
+ return writeInst(InstImm(op_bstr_w, msbw, lsbw, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_bstrpick_d(Register rd, Register rj,
+ int32_t msbd, int32_t lsbd) {
+ MOZ_ASSERT(lsbd <= msbd);
+ spew("bstrpick_d %3s,%3s,0x%x,0x%x", rd.name(), rj.name(), msbd, lsbd);
+ return writeInst(InstImm(op_bstrpick_d, msbd, lsbd, rj, rd, 6).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_maskeqz(Register rd, Register rj,
+ Register rk) {
+ spew("maskeqz %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_maskeqz, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_masknez(Register rd, Register rj,
+ Register rk) {
+ spew("masknez %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_masknez, rk, rj, rd).encode());
+}
+
+// Load and store instructions
+BufferOffset AssemblerLOONG64::as_ld_b(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("ld_b %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_ld_b, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ld_h(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("ld_h %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_ld_h, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ld_w(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("ld_w %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_ld_w, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ld_d(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("ld_d %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_ld_d, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ld_bu(Register rd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("ld_bu %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_ld_bu, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ld_hu(Register rd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("ld_hu %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_ld_hu, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ld_wu(Register rd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("ld_wu %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_ld_wu, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_st_b(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("st_b %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_st_b, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_st_h(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("st_h %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_st_h, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_st_w(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("st_w %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_st_w, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_st_d(Register rd, Register rj, int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("st_d %3s,%3s,0x%x", rd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_st_d, si12, rj, rd, 12).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldx_b(Register rd, Register rj, Register rk) {
+ spew("ldx_b %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ldx_b, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldx_h(Register rd, Register rj, Register rk) {
+ spew("ldx_h %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ldx_h, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldx_w(Register rd, Register rj, Register rk) {
+ spew("ldx_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ldx_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldx_d(Register rd, Register rj, Register rk) {
+ spew("ldx_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ldx_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldx_bu(Register rd, Register rj,
+ Register rk) {
+ spew("ldx_bu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ldx_bu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldx_hu(Register rd, Register rj,
+ Register rk) {
+ spew("ldx_hu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ldx_hu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldx_wu(Register rd, Register rj,
+ Register rk) {
+ spew("ldx_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ldx_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_stx_b(Register rd, Register rj, Register rk) {
+ spew("stx_b %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_stx_b, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_stx_h(Register rd, Register rj, Register rk) {
+ spew("stx_h %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_stx_h, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_stx_w(Register rd, Register rj, Register rk) {
+ spew("stx_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_stx_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_stx_d(Register rd, Register rj, Register rk) {
+ spew("stx_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_stx_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldptr_w(Register rd, Register rj,
+ int32_t si14) {
+ MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0));
+ spew("ldptr_w %3s,%3s,0x%x", rd.name(), rj.name(), si14);
+ return writeInst(InstImm(op_ldptr_w, si14 >> 2, rj, rd, 14).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ldptr_d(Register rd, Register rj,
+ int32_t si14) {
+ MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0));
+ spew("ldptr_d %3s,%3s,0x%x", rd.name(), rj.name(), si14);
+ return writeInst(InstImm(op_ldptr_d, si14 >> 2, rj, rd, 14).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_stptr_w(Register rd, Register rj,
+ int32_t si14) {
+ MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0));
+ spew("stptr_w %3s,%3s,0x%x", rd.name(), rj.name(), si14);
+ return writeInst(InstImm(op_stptr_w, si14 >> 2, rj, rd, 14).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_stptr_d(Register rd, Register rj,
+ int32_t si14) {
+ MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0));
+ spew("stptr_d %3s,%3s,0x%x", rd.name(), rj.name(), si14);
+ return writeInst(InstImm(op_stptr_d, si14 >> 2, rj, rd, 14).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_preld(int32_t hint, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("preld 0x%x,%3s,0x%x", hint, rj.name(), si12);
+ return writeInst(InstImm(op_preld, si12, rj, hint).encode());
+}
+
+// Atomic instructions
+BufferOffset AssemblerLOONG64::as_amswap_w(Register rd, Register rj,
+ Register rk) {
+ spew("amswap_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amswap_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amswap_d(Register rd, Register rj,
+ Register rk) {
+ spew("amswap_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amswap_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amadd_w(Register rd, Register rj,
+ Register rk) {
+ spew("amadd_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amadd_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amadd_d(Register rd, Register rj,
+ Register rk) {
+ spew("amadd_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amadd_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amand_w(Register rd, Register rj,
+ Register rk) {
+ spew("amand_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amand_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amand_d(Register rd, Register rj,
+ Register rk) {
+ spew("amand_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amand_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amor_w(Register rd, Register rj,
+ Register rk) {
+ spew("amor_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amor_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amor_d(Register rd, Register rj,
+ Register rk) {
+ spew("amor_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amor_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amxor_w(Register rd, Register rj,
+ Register rk) {
+ spew("amxor_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amxor_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amxor_d(Register rd, Register rj,
+ Register rk) {
+ spew("amxor_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amxor_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammax_w(Register rd, Register rj,
+ Register rk) {
+ spew("ammax_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammax_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammax_d(Register rd, Register rj,
+ Register rk) {
+ spew("ammax_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammax_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammin_w(Register rd, Register rj,
+ Register rk) {
+ spew("ammin_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammin_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammin_d(Register rd, Register rj,
+ Register rk) {
+ spew("ammin_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammin_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammax_wu(Register rd, Register rj,
+ Register rk) {
+ spew("ammax_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammax_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammax_du(Register rd, Register rj,
+ Register rk) {
+ spew("ammax_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammax_du, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammin_wu(Register rd, Register rj,
+ Register rk) {
+ spew("ammin_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammin_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammin_du(Register rd, Register rj,
+ Register rk) {
+ spew("ammin_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammin_du, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amswap_db_w(Register rd, Register rj,
+ Register rk) {
+ spew("amswap_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amswap_db_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amswap_db_d(Register rd, Register rj,
+ Register rk) {
+ spew("amswap_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amswap_db_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amadd_db_w(Register rd, Register rj,
+ Register rk) {
+ spew("amadd_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amadd_db_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amadd_db_d(Register rd, Register rj,
+ Register rk) {
+ spew("amadd_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amadd_db_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amand_db_w(Register rd, Register rj,
+ Register rk) {
+ spew("amand_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amand_db_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amand_db_d(Register rd, Register rj,
+ Register rk) {
+ spew("amand_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amand_db_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amor_db_w(Register rd, Register rj,
+ Register rk) {
+ spew("amor_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amor_db_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amor_db_d(Register rd, Register rj,
+ Register rk) {
+ spew("amor_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amor_db_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amxor_db_w(Register rd, Register rj,
+ Register rk) {
+ spew("amxor_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amxor_db_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_amxor_db_d(Register rd, Register rj,
+ Register rk) {
+ spew("amxor_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_amxor_db_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammax_db_w(Register rd, Register rj,
+ Register rk) {
+ spew("ammax_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammax_db_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammax_db_d(Register rd, Register rj,
+ Register rk) {
+ spew("ammax_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammax_db_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammin_db_w(Register rd, Register rj,
+ Register rk) {
+ spew("ammin_db_w %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammin_db_w, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammin_db_d(Register rd, Register rj,
+ Register rk) {
+ spew("ammin_db_d %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammin_db_d, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammax_db_wu(Register rd, Register rj,
+ Register rk) {
+ spew("ammax_db_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammax_db_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammax_db_du(Register rd, Register rj,
+ Register rk) {
+ spew("ammax_db_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammax_db_du, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammin_db_wu(Register rd, Register rj,
+ Register rk) {
+ spew("ammin_db_wu %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammin_db_wu, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ammin_db_du(Register rd, Register rj,
+ Register rk) {
+ spew("ammin_db_du %3s,%3s,%3s", rd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_ammin_db_du, rk, rj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ll_w(Register rd, Register rj, int32_t si14) {
+ spew("ll_w %3s,%3s,0x%x", rd.name(), rj.name(), si14);
+ MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0));
+ return writeInst(InstImm(op_ll_w, si14 >> 2, rj, rd, 14).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ll_d(Register rd, Register rj, int32_t si14) {
+ spew("ll_d %3s,%3s,0x%x", rd.name(), rj.name(), si14);
+ MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0));
+ return writeInst(InstImm(op_ll_d, si14 >> 2, rj, rd, 14).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sc_w(Register rd, Register rj, int32_t si14) {
+ spew("sc_w %3s,%3s,0x%x", rd.name(), rj.name(), si14);
+ MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0));
+ return writeInst(InstImm(op_sc_w, si14 >> 2, rj, rd, 14).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_sc_d(Register rd, Register rj, int32_t si14) {
+ spew("sc_d %3s,%3s,0x%x", rd.name(), rj.name(), si14);
+ MOZ_ASSERT(is_intN(si14, 16) && ((si14 & 0x3) == 0));
+ return writeInst(InstImm(op_sc_d, si14 >> 2, rj, rd, 14).encode());
+}
+
+// Barrier instructions
+BufferOffset AssemblerLOONG64::as_dbar(int32_t hint) {
+ MOZ_ASSERT(is_uintN(hint, 15));
+ spew("dbar 0x%x", hint);
+ return writeInst(InstImm(op_dbar, hint).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ibar(int32_t hint) {
+ MOZ_ASSERT(is_uintN(hint, 15));
+ spew("ibar 0x%x", hint);
+ return writeInst(InstImm(op_ibar, hint).encode());
+}
+
+/* =============================================================== */
+
+// FP Arithmetic instructions
+BufferOffset AssemblerLOONG64::as_fadd_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fadd_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fadd_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fadd_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fadd_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fadd_d, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fsub_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fsub_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fsub_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fsub_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fsub_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fsub_d, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmul_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmul_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmul_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmul_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmul_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmul_d, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fdiv_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fdiv_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fdiv_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fdiv_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fdiv_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fdiv_d, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmadd_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FloatRegister fa) {
+ spew("fmadd_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(),
+ fa.name());
+ return writeInst(InstReg(op_fmadd_s, fa, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmadd_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FloatRegister fa) {
+ spew("fmadd_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(),
+ fa.name());
+ return writeInst(InstReg(op_fmadd_d, fa, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmsub_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FloatRegister fa) {
+ spew("fmsub_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(),
+ fa.name());
+ return writeInst(InstReg(op_fmsub_s, fa, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmsub_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FloatRegister fa) {
+ spew("fmsub_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(),
+ fa.name());
+ return writeInst(InstReg(op_fmsub_d, fa, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fnmadd_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FloatRegister fa) {
+ spew("fnmadd_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(),
+ fa.name());
+ return writeInst(InstReg(op_fnmadd_s, fa, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fnmadd_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FloatRegister fa) {
+ spew("fnmadd_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(),
+ fa.name());
+ return writeInst(InstReg(op_fnmadd_d, fa, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fnmsub_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FloatRegister fa) {
+ spew("fnmsub_s %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(),
+ fa.name());
+ return writeInst(InstReg(op_fnmsub_s, fa, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fnmsub_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FloatRegister fa) {
+ spew("fnmsub_d %3s,%3s,%3s,%3s", fd.name(), fj.name(), fk.name(),
+ fa.name());
+ return writeInst(InstReg(op_fnmsub_d, fa, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmax_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmax_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmax_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmax_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmax_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmax_d, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmin_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmin_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmin_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmin_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmin_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmin_d, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmaxa_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmaxa_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmaxa_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmaxa_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmaxa_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmaxa_d, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmina_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmina_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmina_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmina_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk) {
+ spew("fmina_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fmina_d, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fabs_s(FloatRegister fd, FloatRegister fj) {
+ spew("fabs_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fabs_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fabs_d(FloatRegister fd, FloatRegister fj) {
+ spew("fabs_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fabs_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fneg_s(FloatRegister fd, FloatRegister fj) {
+ spew("fneg_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fneg_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fneg_d(FloatRegister fd, FloatRegister fj) {
+ spew("fneg_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fneg_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fsqrt_s(FloatRegister fd, FloatRegister fj) {
+ spew("fsqrt_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fsqrt_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fsqrt_d(FloatRegister fd, FloatRegister fj) {
+ spew("fsqrt_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fsqrt_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fcopysign_s(FloatRegister fd,
+ FloatRegister fj,
+ FloatRegister fk) {
+ spew("fcopysign_s %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fcopysign_s, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fcopysign_d(FloatRegister fd,
+ FloatRegister fj,
+ FloatRegister fk) {
+ spew("fcopysign_d %3s,%3s,%3s", fd.name(), fj.name(), fk.name());
+ return writeInst(InstReg(op_fcopysign_d, fk, fj, fd).encode());
+}
+
+// FP compare instructions
+// fcmp.cond.s and fcmp.cond.d instructions
+BufferOffset AssemblerLOONG64::as_fcmp_cor(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_cor_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, COR, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_cor_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, COR, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_ceq(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_ceq_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CEQ, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_ceq_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CEQ, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_cne(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_cne_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CNE, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_cne_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CNE, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_cle(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_cle_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CLE, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_cle_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CLE, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_clt(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_clt_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CLT, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_clt_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CLT, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_cun(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_cun_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CUN, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_cun_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CUN, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_cueq(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_cueq_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CUEQ, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_cueq_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CUEQ, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_cune(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_cune_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CUNE, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_cune_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CUNE, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_cule(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_cule_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CULE, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_cule_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CULE, fk, fj, cd).encode());
+ }
+}
+
+BufferOffset AssemblerLOONG64::as_fcmp_cult(FloatFormat fmt, FloatRegister fj,
+ FloatRegister fk,
+ FPConditionBit cd) {
+ if (fmt == DoubleFloat) {
+ spew("fcmp_cult_d FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_d, CULT, fk, fj, cd).encode());
+ } else {
+ spew("fcmp_cult_s FCC%d,%3s,%3s", cd, fj.name(), fk.name());
+ return writeInst(InstReg(op_fcmp_cond_s, CULT, fk, fj, cd).encode());
+ }
+}
+
+// FP conversion instructions
+BufferOffset AssemblerLOONG64::as_fcvt_s_d(FloatRegister fd, FloatRegister fj) {
+ spew("fcvt_s_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fcvt_s_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fcvt_d_s(FloatRegister fd, FloatRegister fj) {
+ spew("fcvt_d_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fcvt_d_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ffint_s_w(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ffint_s_w %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ffint_s_w, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ffint_s_l(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ffint_s_l %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ffint_s_l, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ffint_d_w(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ffint_d_w %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ffint_d_w, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ffint_d_l(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ffint_d_l %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ffint_d_l, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftint_w_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftint_w_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftint_w_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftint_w_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftint_w_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftint_w_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftint_l_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftint_l_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftint_l_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftint_l_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftint_l_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftint_l_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrm_w_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrm_w_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrm_w_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrm_w_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrm_w_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrm_w_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrm_l_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrm_l_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrm_l_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrm_l_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrm_l_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrm_l_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrp_w_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrp_w_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrp_w_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrp_w_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrp_w_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrp_w_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrp_l_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrp_l_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrp_l_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrp_l_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrp_l_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrp_l_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrz_w_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrz_w_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrz_w_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrz_w_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrz_w_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrz_w_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrz_l_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrz_l_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrz_l_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrz_l_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrz_l_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrz_l_d, fj, fd).encode());
+}
+BufferOffset AssemblerLOONG64::as_ftintrne_w_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrne_w_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrne_w_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrne_w_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrne_w_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrne_w_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrne_l_s(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrne_l_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrne_l_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_ftintrne_l_d(FloatRegister fd,
+ FloatRegister fj) {
+ spew("ftintrne_l_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_ftintrne_l_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_frint_s(FloatRegister fd, FloatRegister fj) {
+ spew("frint_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_frint_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_frint_d(FloatRegister fd, FloatRegister fj) {
+ spew("frint_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_frint_d, fj, fd).encode());
+}
+
+// FP mov instructions
+BufferOffset AssemblerLOONG64::as_fmov_s(FloatRegister fd, FloatRegister fj) {
+ spew("fmov_s %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fmov_s, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fmov_d(FloatRegister fd, FloatRegister fj) {
+ spew("fmov_d %3s,%3s", fd.name(), fj.name());
+ return writeInst(InstReg(op_fmov_d, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fsel(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk, FPConditionBit ca) {
+ spew("fsel %3s,%3s,%3s,%d", fd.name(), fj.name(), fk.name(), ca);
+ return writeInst(InstReg(op_fsel, ca, fk, fj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movgr2fr_w(FloatRegister fd, Register rj) {
+ spew("movgr2fr_w %3s,%3s", fd.name(), rj.name());
+ return writeInst(InstReg(op_movgr2fr_w, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movgr2fr_d(FloatRegister fd, Register rj) {
+ spew("movgr2fr_d %3s,%3s", fd.name(), rj.name());
+ return writeInst(InstReg(op_movgr2fr_d, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movgr2frh_w(FloatRegister fd, Register rj) {
+ spew("movgr2frh_w %3s,%3s", fd.name(), rj.name());
+ return writeInst(InstReg(op_movgr2frh_w, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movfr2gr_s(Register rd, FloatRegister fj) {
+ spew("movfr2gr_s %3s,%3s", rd.name(), fj.name());
+ return writeInst(InstReg(op_movfr2gr_s, fj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movfr2gr_d(Register rd, FloatRegister fj) {
+ spew("movfr2gr_d %3s,%3s", rd.name(), fj.name());
+ return writeInst(InstReg(op_movfr2gr_d, fj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movfrh2gr_s(Register rd, FloatRegister fj) {
+ spew("movfrh2gr_s %3s,%3s", rd.name(), fj.name());
+ return writeInst(InstReg(op_movfrh2gr_s, fj, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movgr2fcsr(Register rj) {
+ spew("movgr2fcsr %3s", rj.name());
+ return writeInst(InstReg(op_movgr2fcsr, rj, FCSR).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movfcsr2gr(Register rd) {
+ spew("movfcsr2gr %3s", rd.name());
+ return writeInst(InstReg(op_movfcsr2gr, FCSR, rd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movfr2cf(FPConditionBit cd,
+ FloatRegister fj) {
+ spew("movfr2cf %d,%3s", cd, fj.name());
+ return writeInst(InstReg(op_movfr2cf, fj, cd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movcf2fr(FloatRegister fd,
+ FPConditionBit cj) {
+ spew("movcf2fr %3s,%d", fd.name(), cj);
+ return writeInst(InstReg(op_movcf2fr, cj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movgr2cf(FPConditionBit cd, Register rj) {
+ spew("movgr2cf %d,%3s", cd, rj.name());
+ return writeInst(InstReg(op_movgr2cf, rj, cd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_movcf2gr(Register rd, FPConditionBit cj) {
+ spew("movcf2gr %3s,%d", rd.name(), cj);
+ return writeInst(InstReg(op_movcf2gr, cj, rd).encode());
+}
+
+// FP load/store instructions
+BufferOffset AssemblerLOONG64::as_fld_s(FloatRegister fd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("fld_s %3s,%3s,0x%x", fd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_fld_s, si12, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fld_d(FloatRegister fd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("fld_d %3s,%3s,0x%x", fd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_fld_d, si12, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fst_s(FloatRegister fd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("fst_s %3s,%3s,0x%x", fd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_fst_s, si12, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fst_d(FloatRegister fd, Register rj,
+ int32_t si12) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ spew("fst_d %3s,%3s,0x%x", fd.name(), rj.name(), si12);
+ return writeInst(InstImm(op_fst_d, si12, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fldx_s(FloatRegister fd, Register rj,
+ Register rk) {
+ spew("fldx_s %3s,%3s,%3s", fd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_fldx_s, rk, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fldx_d(FloatRegister fd, Register rj,
+ Register rk) {
+ spew("fldx_d %3s,%3s,%3s", fd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_fldx_d, rk, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fstx_s(FloatRegister fd, Register rj,
+ Register rk) {
+ spew("fstx_s %3s,%3s,%3s", fd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_fstx_s, rk, rj, fd).encode());
+}
+
+BufferOffset AssemblerLOONG64::as_fstx_d(FloatRegister fd, Register rj,
+ Register rk) {
+ spew("fstx_d %3s,%3s,%3s", fd.name(), rj.name(), rk.name());
+ return writeInst(InstReg(op_fstx_d, rk, rj, fd).encode());
+}
+
+/* ========================================================================= */
+
+void AssemblerLOONG64::bind(Label* label, BufferOffset boff) {
+ spew(".set Llabel %p", label);
+ // If our caller didn't give us an explicit target to bind to
+ // then we want to bind to the location of the next instruction
+ BufferOffset dest = boff.assigned() ? boff : nextOffset();
+ if (label->used()) {
+ int32_t next;
+
+ // A used label holds a link to branch that uses it.
+ BufferOffset b(label);
+ do {
+ // Even a 0 offset may be invalid if we're out of memory.
+ if (oom()) {
+ return;
+ }
+
+ Instruction* inst = editSrc(b);
+
+ // Second word holds a pointer to the next branch in label's chain.
+ next = inst[1].encode();
+ bind(reinterpret_cast<InstImm*>(inst), b.getOffset(), dest.getOffset());
+
+ b = BufferOffset(next);
+ } while (next != LabelBase::INVALID_OFFSET);
+ }
+ label->bind(dest.getOffset());
+}
+
+void AssemblerLOONG64::retarget(Label* label, Label* target) {
+ spew("retarget %p -> %p", label, target);
+ if (label->used() && !oom()) {
+ if (target->bound()) {
+ bind(label, BufferOffset(target));
+ } else if (target->used()) {
+ // The target is not bound but used. Prepend label's branch list
+ // onto target's.
+ int32_t next;
+ BufferOffset labelBranchOffset(label);
+
+ // Find the head of the use chain for label.
+ do {
+ Instruction* inst = editSrc(labelBranchOffset);
+
+ // Second word holds a pointer to the next branch in chain.
+ next = inst[1].encode();
+ labelBranchOffset = BufferOffset(next);
+ } while (next != LabelBase::INVALID_OFFSET);
+
+ // Then patch the head of label's use chain to the tail of
+ // target's use chain, prepending the entire use chain of target.
+ Instruction* inst = editSrc(labelBranchOffset);
+ int32_t prev = target->offset();
+ target->use(label->offset());
+ inst[1].setData(prev);
+ } else {
+ // The target is unbound and unused. We can just take the head of
+ // the list hanging off of label, and dump that into target.
+ target->use(label->offset());
+ }
+ }
+ label->reset();
+}
+
+void dbg_break() {}
+
+void AssemblerLOONG64::as_break(uint32_t code) {
+ MOZ_ASSERT(code <= MAX_BREAK_CODE);
+ spew("break %d", code);
+ writeInst(InstImm(op_break, code).encode());
+}
+
+// This just stomps over memory with 32 bits of raw data. Its purpose is to
+// overwrite the call of JITed code with 32 bits worth of an offset. This will
+// is only meant to function on code that has been invalidated, so it should
+// be totally safe. Since that instruction will never be executed again, a
+// ICache flush should not be necessary
+void AssemblerLOONG64::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) {
+ // Raw is going to be the return address.
+ uint32_t* raw = (uint32_t*)label.raw();
+ // Overwrite the 4 bytes before the return address, which will
+ // end up being the call instruction.
+ *(raw - 1) = imm.value;
+}
+
+uint8_t* AssemblerLOONG64::NextInstruction(uint8_t* inst_, uint32_t* count) {
+ Instruction* inst = reinterpret_cast<Instruction*>(inst_);
+ if (count != nullptr) {
+ *count += sizeof(Instruction);
+ }
+ return reinterpret_cast<uint8_t*>(inst->next());
+}
+
+void AssemblerLOONG64::ToggleToJmp(CodeLocationLabel inst_) {
+ InstImm* inst = (InstImm*)inst_.raw();
+
+ MOZ_ASSERT(inst->extractBitField(31, 26) == (uint32_t)op_addu16i_d >> 26);
+ // We converted beq to addu16i_d, so now we restore it.
+ inst->setOpcode(op_beq, 6);
+}
+
+void AssemblerLOONG64::ToggleToCmp(CodeLocationLabel inst_) {
+ InstImm* inst = (InstImm*)inst_.raw();
+
+ // toggledJump is allways used for short jumps.
+ MOZ_ASSERT(inst->extractBitField(31, 26) == (uint32_t)op_beq >> 26);
+ // Replace "beq $zero, $zero, offset" with "addu16i_d $zero, $zero, offset"
+ inst->setOpcode(op_addu16i_d, 6);
+}
+
+// Since there are no pools in LoongArch64 implementation, this should be
+// simple.
+Instruction* Instruction::next() { return this + 1; }
+
+InstImm AssemblerLOONG64::invertBranch(InstImm branch, BOffImm16 skipOffset) {
+ uint32_t rj = 0;
+ OpcodeField opcode = (OpcodeField)((branch.extractBitField(31, 26)) << 26);
+ switch (opcode) {
+ case op_beq:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_bne, 6);
+ return branch;
+ case op_bne:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_beq, 6);
+ return branch;
+ case op_bge:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_blt, 6);
+ return branch;
+ case op_bgeu:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_bltu, 6);
+ return branch;
+ case op_blt:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_bge, 6);
+ return branch;
+ case op_bltu:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_bgeu, 6);
+ return branch;
+ case op_beqz:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_bnez, 6);
+ return branch;
+ case op_bnez:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_beqz, 6);
+ return branch;
+ case op_bcz:
+ branch.setBOffImm16(skipOffset);
+ rj = branch.extractRJ();
+ if (rj & 0x8) {
+ branch.setRJ(rj & 0x17);
+ } else {
+ branch.setRJ(rj | 0x8);
+ }
+ return branch;
+ default:
+ MOZ_CRASH("Error creating long branch.");
+ }
+}
+
+#ifdef JS_JITSPEW
+void AssemblerLOONG64::decodeBranchInstAndSpew(InstImm branch) {
+ OpcodeField opcode = (OpcodeField)((branch.extractBitField(31, 26)) << 26);
+ uint32_t rd_id;
+ uint32_t rj_id;
+ uint32_t cj_id;
+ uint32_t immi = branch.extractImm16Value();
+ switch (opcode) {
+ case op_beq:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("beq 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), Registers::GetName(rd_id));
+ break;
+ case op_bne:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("bne 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), Registers::GetName(rd_id));
+ break;
+ case op_bge:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("bge 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), Registers::GetName(rd_id));
+ break;
+ case op_bgeu:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("bgeu 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), Registers::GetName(rd_id));
+ break;
+ case op_blt:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("blt 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), Registers::GetName(rd_id));
+ break;
+ case op_bltu:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("bltu 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), Registers::GetName(rd_id));
+ break;
+ case op_beqz:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("beqz 0x%x,%3s,0x%x", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), rd_id);
+ break;
+ case op_bnez:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("bnez 0x%x,%3s,0x%x", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), rd_id);
+ break;
+ case op_bcz:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ cj_id = branch.extractBitField(CJShift + CJBits - 1, CJShift);
+ if (rj_id & 0x8) {
+ spew("bcnez 0x%x,FCC%d,0x%x", (int32_t(immi << 18) >> 16) + 4, cj_id,
+ rd_id);
+ } else {
+ spew("bceqz 0x%x,FCC%d,0x%x", (int32_t(immi << 18) >> 16) + 4, cj_id,
+ rd_id);
+ }
+ break;
+ case op_jirl:
+ rd_id = branch.extractRD();
+ rj_id = branch.extractRJ();
+ spew("beqz 0x%x,%3s,%3s", (int32_t(immi << 18) >> 16) + 4,
+ Registers::GetName(rj_id), Registers::GetName(rd_id));
+ break;
+ default:
+ MOZ_CRASH("Error disassemble branch.");
+ }
+}
+#endif
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ MOZ_ASSERT(isFinished);
+ m_buffer.executableCopy(buffer);
+}
+
+uintptr_t Assembler::GetPointer(uint8_t* instPtr) {
+ Instruction* inst = (Instruction*)instPtr;
+ return Assembler::ExtractLoad64Value(inst);
+}
+
+static JitCode* CodeFromJump(Instruction* jump) {
+ uint8_t* target = (uint8_t*)Assembler::ExtractLoad64Value(jump);
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ while (reader.more()) {
+ JitCode* child =
+ CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned()));
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ }
+}
+
+static void TraceOneDataRelocation(JSTracer* trc,
+ mozilla::Maybe<AutoWritableJitCode>& awjc,
+ JitCode* code, Instruction* inst) {
+ void* ptr = (void*)Assembler::ExtractLoad64Value(inst);
+ void* prior = ptr;
+
+ // Data relocations can be for Values or for raw pointers. If a Value is
+ // zero-tagged, we can trace it as if it were a raw pointer. If a Value
+ // is not zero-tagged, we have to interpret it as a Value to ensure that the
+ // tag bits are masked off to recover the actual pointer.
+ uintptr_t word = reinterpret_cast<uintptr_t>(ptr);
+ if (word >> JSVAL_TAG_SHIFT) {
+ // This relocation is a Value with a non-zero tag.
+ Value v = Value::fromRawBits(word);
+ TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value");
+ ptr = (void*)v.bitsAsPunboxPointer();
+ } else {
+ // This relocation is a raw pointer or a Value with a zero tag.
+ // No barrier needed since these are constants.
+ TraceManuallyBarrieredGenericPointerEdge(
+ trc, reinterpret_cast<gc::Cell**>(&ptr), "jit-masm-ptr");
+ }
+
+ if (ptr != prior) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ Assembler::UpdateLoad64Value(inst, uint64_t(ptr));
+ }
+}
+
+/* static */
+void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ Instruction* inst = (Instruction*)(code->raw() + offset);
+ TraceOneDataRelocation(trc, awjc, code, inst);
+ }
+}
+
+void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) {
+ if (label.patchAt().bound()) {
+ auto mode = label.linkMode();
+ intptr_t offset = label.patchAt().offset();
+ intptr_t target = label.target().offset();
+
+ if (mode == CodeLabel::RawPointer) {
+ *reinterpret_cast<const void**>(rawCode + offset) = rawCode + target;
+ } else {
+ MOZ_ASSERT(mode == CodeLabel::MoveImmediate ||
+ mode == CodeLabel::JumpImmediate);
+ Instruction* inst = (Instruction*)(rawCode + offset);
+ Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + target));
+ }
+ }
+}
+
+void Assembler::bind(InstImm* inst, uintptr_t branch, uintptr_t target) {
+ int64_t offset = target - branch;
+ InstImm inst_jirl = InstImm(op_jirl, BOffImm16(0), zero, ra);
+ InstImm inst_beq = InstImm(op_beq, BOffImm16(0), zero, zero);
+
+ // If encoded offset is 4, then the jump must be short
+ if (BOffImm16(inst[0]).decode() == 4) {
+ MOZ_ASSERT(BOffImm16::IsInRange(offset));
+ inst[0].setBOffImm16(BOffImm16(offset));
+ inst[1].makeNop(); // because before set INVALID_OFFSET
+ return;
+ }
+
+ // Generate the long jump for calls because return address has to be the
+ // address after the reserved block.
+ if (inst[0].encode() == inst_jirl.encode()) {
+ addLongJump(BufferOffset(branch), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+ inst[3].makeNop(); // There are 1 nop.
+ inst[4] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra);
+ return;
+ }
+
+ if (BOffImm16::IsInRange(offset)) {
+ // Skip trailing nops .
+ bool skipNops = (inst[0].encode() != inst_jirl.encode() &&
+ inst[0].encode() != inst_beq.encode());
+
+ inst[0].setBOffImm16(BOffImm16(offset));
+ inst[1].makeNop();
+
+ if (skipNops) {
+ inst[2] = InstImm(op_bge, BOffImm16(3 * sizeof(uint32_t)), zero, zero);
+ // There are 2 nops after this
+ }
+ return;
+ }
+
+ if (inst[0].encode() == inst_beq.encode()) {
+ // Handle long unconditional jump. Only four 4 instruction.
+ addLongJump(BufferOffset(branch), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+ inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, zero);
+ } else {
+ // Handle long conditional jump.
+ inst[0] = invertBranch(inst[0], BOffImm16(5 * sizeof(uint32_t)));
+ // No need for a "nop" here because we can clobber scratch.
+ addLongJump(BufferOffset(branch + sizeof(uint32_t)), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(&inst[1], ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+ inst[4] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, zero);
+ }
+}
+
+void Assembler::processCodeLabels(uint8_t* rawCode) {
+ for (const CodeLabel& label : codeLabels_) {
+ Bind(rawCode, label);
+ }
+}
+
+uint32_t Assembler::PatchWrite_NearCallSize() {
+ // Load an address needs 3 instructions, and a jump.
+ return (3 + 1) * sizeof(uint32_t);
+}
+
+void Assembler::PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall) {
+ Instruction* inst = (Instruction*)start.raw();
+ uint8_t* dest = toCall.raw();
+
+ // Overwrite whatever instruction used to be here with a call.
+ // Always use long jump for two reasons:
+ // - Jump has to be the same size because of PatchWrite_NearCallSize.
+ // - Return address has to be at the end of replaced block.
+ // Short jump wouldn't be more efficient.
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)dest);
+ inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra);
+}
+
+uint64_t Assembler::ExtractLoad64Value(Instruction* inst0) {
+ InstImm* i0 = (InstImm*)inst0;
+ InstImm* i1 = (InstImm*)i0->next();
+ InstImm* i2 = (InstImm*)i1->next();
+ InstImm* i3 = (InstImm*)i2->next();
+
+ MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25));
+ MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22));
+ MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25));
+
+ if ((i3->extractBitField(31, 22)) == ((uint32_t)op_lu52i_d >> 22)) {
+ // Li64
+ uint64_t value =
+ (uint64_t(i0->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift))
+ << 12) |
+ (uint64_t(
+ i1->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift))) |
+ (uint64_t(i2->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift))
+ << 32) |
+ (uint64_t(i3->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift))
+ << 52);
+ return value;
+ } else {
+ // Li48
+ uint64_t value =
+ (uint64_t(i0->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift))
+ << 12) |
+ (uint64_t(
+ i1->extractBitField(Imm12Bits + Imm12Shift - 1, Imm12Shift))) |
+ (uint64_t(i2->extractBitField(Imm20Bits + Imm20Shift - 1, Imm20Shift))
+ << 32);
+
+ return uint64_t((int64_t(value) << 16) >> 16);
+ }
+}
+
+void Assembler::UpdateLoad64Value(Instruction* inst0, uint64_t value) {
+ // Todo: with ma_liPatchable
+ InstImm* i0 = (InstImm*)inst0;
+ InstImm* i1 = (InstImm*)i0->next();
+ InstImm* i2 = (InstImm*)i1->next();
+ InstImm* i3 = (InstImm*)i2->next();
+
+ MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25));
+ MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22));
+ MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25));
+
+ if ((i3->extractBitField(31, 22)) == ((uint32_t)op_lu52i_d >> 22)) {
+ // Li64
+ *i0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff),
+ Register::FromCode(i0->extractRD()), false);
+ *i1 = InstImm(op_ori, (int32_t)(value & 0xfff),
+ Register::FromCode(i1->extractRJ()),
+ Register::FromCode(i1->extractRD()), 12);
+ *i2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff),
+ Register::FromCode(i2->extractRD()), false);
+ *i3 = InstImm(op_lu52i_d, (int32_t)((value >> 52) & 0xfff),
+ Register::FromCode(i3->extractRJ()),
+ Register::FromCode(i3->extractRD()), 12);
+ } else {
+ // Li48
+ *i0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff),
+ Register::FromCode(i0->extractRD()), false);
+ *i1 = InstImm(op_ori, (int32_t)(value & 0xfff),
+ Register::FromCode(i1->extractRJ()),
+ Register::FromCode(i1->extractRD()), 12);
+ *i2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff),
+ Register::FromCode(i2->extractRD()), false);
+ }
+}
+
+void Assembler::WriteLoad64Instructions(Instruction* inst0, Register reg,
+ uint64_t value) {
+ Instruction* inst1 = inst0->next();
+ Instruction* inst2 = inst1->next();
+ *inst0 = InstImm(op_lu12i_w, (int32_t)((value >> 12) & 0xfffff), reg, false);
+ *inst1 = InstImm(op_ori, (int32_t)(value & 0xfff), reg, reg, 12);
+ *inst2 = InstImm(op_lu32i_d, (int32_t)((value >> 32) & 0xfffff), reg, false);
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ ImmPtr newValue, ImmPtr expectedValue) {
+ PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value),
+ PatchedImmPtr(expectedValue.value));
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue) {
+ Instruction* inst = (Instruction*)label.raw();
+
+ // Extract old Value
+ DebugOnly<uint64_t> value = Assembler::ExtractLoad64Value(inst);
+ MOZ_ASSERT(value == uint64_t(expectedValue.value));
+
+ // Replace with new value
+ Assembler::UpdateLoad64Value(inst, uint64_t(newValue.value));
+}
+
+uint64_t Assembler::ExtractInstructionImmediate(uint8_t* code) {
+ InstImm* inst = (InstImm*)code;
+ return Assembler::ExtractLoad64Value(inst);
+}
+
+void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) {
+ Instruction* inst = (Instruction*)inst_.raw();
+ InstImm* i0 = (InstImm*)inst;
+ InstImm* i1 = (InstImm*)i0->next();
+ InstImm* i2 = (InstImm*)i1->next();
+ Instruction* i3 = (Instruction*)i2->next();
+
+ MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25));
+ MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22));
+ MOZ_ASSERT((i2->extractBitField(31, 25)) == ((uint32_t)op_lu32i_d >> 25));
+
+ if (enabled) {
+ MOZ_ASSERT((i3->extractBitField(31, 25)) != ((uint32_t)op_lu12i_w >> 25));
+ InstImm jirl = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra);
+ *i3 = jirl;
+ } else {
+ InstNOP nop;
+ *i3 = nop;
+ }
+}
diff --git a/js/src/jit/loong64/Assembler-loong64.h b/js/src/jit/loong64/Assembler-loong64.h
new file mode 100644
index 0000000000..65c28d558e
--- /dev/null
+++ b/js/src/jit/loong64/Assembler-loong64.h
@@ -0,0 +1,1879 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_Assembler_loong64_h
+#define jit_loong64_Assembler_loong64_h
+
+#include "mozilla/Sprintf.h"
+#include <iterator>
+
+#include "jit/CompactBuffer.h"
+#include "jit/JitCode.h"
+#include "jit/JitSpewer.h"
+#include "jit/loong64/Architecture-loong64.h"
+#include "jit/shared/Assembler-shared.h"
+#include "jit/shared/Disassembler-shared.h"
+#include "jit/shared/IonAssemblerBuffer.h"
+#include "wasm/WasmTypeDecls.h"
+
+namespace js {
+namespace jit {
+
+static constexpr Register zero{Registers::zero};
+static constexpr Register ra{Registers::ra};
+static constexpr Register tp{Registers::tp};
+static constexpr Register sp{Registers::sp};
+static constexpr Register a0{Registers::a0};
+static constexpr Register a1{Registers::a1};
+static constexpr Register a2{Registers::a2};
+static constexpr Register a3{Registers::a3};
+static constexpr Register a4{Registers::a4};
+static constexpr Register a5{Registers::a5};
+static constexpr Register a6{Registers::a6};
+static constexpr Register a7{Registers::a7};
+static constexpr Register t0{Registers::t0};
+static constexpr Register t1{Registers::t1};
+static constexpr Register t2{Registers::t2};
+static constexpr Register t3{Registers::t3};
+static constexpr Register t4{Registers::t4};
+static constexpr Register t5{Registers::t5};
+static constexpr Register t6{Registers::t6};
+static constexpr Register t7{Registers::t7};
+static constexpr Register t8{Registers::t8};
+static constexpr Register rx{Registers::rx};
+static constexpr Register fp{Registers::fp};
+static constexpr Register s0{Registers::s0};
+static constexpr Register s1{Registers::s1};
+static constexpr Register s2{Registers::s2};
+static constexpr Register s3{Registers::s3};
+static constexpr Register s4{Registers::s4};
+static constexpr Register s5{Registers::s5};
+static constexpr Register s6{Registers::s6};
+static constexpr Register s7{Registers::s7};
+static constexpr Register s8{Registers::s8};
+
+static constexpr FloatRegister f0{FloatRegisters::f0, FloatRegisters::Double};
+static constexpr FloatRegister f1{FloatRegisters::f1, FloatRegisters::Double};
+static constexpr FloatRegister f2{FloatRegisters::f2, FloatRegisters::Double};
+static constexpr FloatRegister f3{FloatRegisters::f3, FloatRegisters::Double};
+static constexpr FloatRegister f4{FloatRegisters::f4, FloatRegisters::Double};
+static constexpr FloatRegister f5{FloatRegisters::f5, FloatRegisters::Double};
+static constexpr FloatRegister f6{FloatRegisters::f6, FloatRegisters::Double};
+static constexpr FloatRegister f7{FloatRegisters::f7, FloatRegisters::Double};
+static constexpr FloatRegister f8{FloatRegisters::f8, FloatRegisters::Double};
+static constexpr FloatRegister f9{FloatRegisters::f9, FloatRegisters::Double};
+static constexpr FloatRegister f10{FloatRegisters::f10, FloatRegisters::Double};
+static constexpr FloatRegister f11{FloatRegisters::f11, FloatRegisters::Double};
+static constexpr FloatRegister f12{FloatRegisters::f12, FloatRegisters::Double};
+static constexpr FloatRegister f13{FloatRegisters::f13, FloatRegisters::Double};
+static constexpr FloatRegister f14{FloatRegisters::f14, FloatRegisters::Double};
+static constexpr FloatRegister f15{FloatRegisters::f15, FloatRegisters::Double};
+static constexpr FloatRegister f16{FloatRegisters::f16, FloatRegisters::Double};
+static constexpr FloatRegister f17{FloatRegisters::f17, FloatRegisters::Double};
+static constexpr FloatRegister f18{FloatRegisters::f18, FloatRegisters::Double};
+static constexpr FloatRegister f19{FloatRegisters::f19, FloatRegisters::Double};
+static constexpr FloatRegister f20{FloatRegisters::f20, FloatRegisters::Double};
+static constexpr FloatRegister f21{FloatRegisters::f21, FloatRegisters::Double};
+static constexpr FloatRegister f22{FloatRegisters::f22, FloatRegisters::Double};
+static constexpr FloatRegister f23{FloatRegisters::f23, FloatRegisters::Double};
+static constexpr FloatRegister f24{FloatRegisters::f24, FloatRegisters::Double};
+static constexpr FloatRegister f25{FloatRegisters::f25, FloatRegisters::Double};
+static constexpr FloatRegister f26{FloatRegisters::f26, FloatRegisters::Double};
+static constexpr FloatRegister f27{FloatRegisters::f27, FloatRegisters::Double};
+static constexpr FloatRegister f28{FloatRegisters::f28, FloatRegisters::Double};
+static constexpr FloatRegister f29{FloatRegisters::f29, FloatRegisters::Double};
+static constexpr FloatRegister f30{FloatRegisters::f30, FloatRegisters::Double};
+static constexpr FloatRegister f31{FloatRegisters::f31, FloatRegisters::Double};
+
+static constexpr Register InvalidReg{Registers::Invalid};
+static constexpr FloatRegister InvalidFloatReg;
+
+static constexpr Register StackPointer = sp;
+static constexpr Register FramePointer = fp;
+static constexpr Register ReturnReg = a0;
+static constexpr Register64 ReturnReg64(ReturnReg);
+static constexpr FloatRegister ReturnFloat32Reg{FloatRegisters::f0,
+ FloatRegisters::Single};
+static constexpr FloatRegister ReturnDoubleReg = f0;
+static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg;
+
+static constexpr Register ScratchRegister = t7;
+static constexpr Register SecondScratchReg = t8;
+
+// Helper classes for ScratchRegister usage. Asserts that only one piece
+// of code thinks it has exclusive ownership of each scratch register.
+struct ScratchRegisterScope : public AutoRegisterScope {
+ explicit ScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, ScratchRegister) {}
+};
+
+struct SecondScratchRegisterScope : public AutoRegisterScope {
+ explicit SecondScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, SecondScratchReg) {}
+};
+
+static constexpr FloatRegister ScratchFloat32Reg{FloatRegisters::f23,
+ FloatRegisters::Single};
+static constexpr FloatRegister ScratchDoubleReg = f23;
+static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg;
+
+struct ScratchFloat32Scope : public AutoFloatRegisterScope {
+ explicit ScratchFloat32Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchFloat32Reg) {}
+};
+
+struct ScratchDoubleScope : public AutoFloatRegisterScope {
+ explicit ScratchDoubleScope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchDoubleReg) {}
+};
+
+// Use arg reg from EnterJIT function as OsrFrameReg.
+static constexpr Register OsrFrameReg = a3;
+static constexpr Register PreBarrierReg = a1;
+static constexpr Register InterpreterPCReg = t0;
+static constexpr Register CallTempReg0 = t0;
+static constexpr Register CallTempReg1 = t1;
+static constexpr Register CallTempReg2 = t2;
+static constexpr Register CallTempReg3 = t3;
+static constexpr Register CallTempReg4 = t4;
+static constexpr Register CallTempReg5 = t5;
+static constexpr Register CallTempNonArgRegs[] = {t0, t1, t2, t3};
+static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr Register IntArgReg0 = a0;
+static constexpr Register IntArgReg1 = a1;
+static constexpr Register IntArgReg2 = a2;
+static constexpr Register IntArgReg3 = a3;
+static constexpr Register IntArgReg4 = a4;
+static constexpr Register IntArgReg5 = a5;
+static constexpr Register IntArgReg6 = a6;
+static constexpr Register IntArgReg7 = a7;
+static constexpr Register HeapReg = s7;
+
+// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registerd used in RegExpTester instruction (do not use ReturnReg).
+static constexpr Register RegExpTesterRegExpReg = CallTempReg0;
+static constexpr Register RegExpTesterStringReg = CallTempReg1;
+static constexpr Register RegExpTesterLastIndexReg = CallTempReg2;
+
+static constexpr Register JSReturnReg_Type = a3;
+static constexpr Register JSReturnReg_Data = a2;
+static constexpr Register JSReturnReg = a2;
+static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg);
+
+// These registers may be volatile or nonvolatile.
+static constexpr Register ABINonArgReg0 = t0;
+static constexpr Register ABINonArgReg1 = t1;
+static constexpr Register ABINonArgReg2 = t2;
+static constexpr Register ABINonArgReg3 = t3;
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = t0;
+static constexpr Register ABINonArgReturnReg1 = t1;
+static constexpr Register ABINonVolatileReg = s0;
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = ra;
+
+// This register may be volatile or nonvolatile.
+// Avoid f23 which is the scratch register.
+static constexpr FloatRegister ABINonArgDoubleReg{FloatRegisters::f21,
+ FloatRegisters::Double};
+
+// Instance pointer argument register for WebAssembly functions. This must not
+// alias any other register used for passing function arguments or return
+// values. Preserved by WebAssembly functions. Must be nonvolatile.
+static constexpr Register InstanceReg = s4;
+
+// Registers used for wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = t1;
+
+static constexpr uint32_t ABIStackAlignment = 16;
+static constexpr uint32_t CodeAlignment = 16;
+static constexpr uint32_t JitStackAlignment = 16;
+
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+ JitStackValueAlignment >= 1,
+ "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+// TODO(loong64): this is just a filler to prevent a build failure. The
+// LoongArch SIMD alignment requirements still need to be explored.
+static constexpr uint32_t SimdMemoryAlignment = 16;
+
+static_assert(CodeAlignment % SimdMemoryAlignment == 0,
+ "Code alignment should be larger than any of the alignments "
+ "which are used for "
+ "the constant sections of the code buffer. Thus it should be "
+ "larger than the "
+ "alignment for SIMD constants.");
+
+static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static const uint32_t WasmTrapInstructionLength = 4;
+
+// See comments in wasm::GenerateFunctionPrologue. The difference between these
+// is the size of the largest callable prologue on the platform.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+static constexpr Scale ScalePointer = TimesEight;
+
+// TODO(loong64): Add LoongArch instruction types description.
+
+// LoongArch instruction encoding constants.
+static const uint32_t RJShift = 5;
+static const uint32_t RJBits = 5;
+static const uint32_t RKShift = 10;
+static const uint32_t RKBits = 5;
+static const uint32_t RDShift = 0;
+static const uint32_t RDBits = 5;
+static const uint32_t FJShift = 5;
+static const uint32_t FJBits = 5;
+static const uint32_t FKShift = 10;
+static const uint32_t FKBits = 5;
+static const uint32_t FDShift = 0;
+static const uint32_t FDBits = 5;
+static const uint32_t FAShift = 15;
+static const uint32_t FABits = 5;
+static const uint32_t CJShift = 5;
+static const uint32_t CJBits = 3;
+static const uint32_t CDShift = 0;
+static const uint32_t CDBits = 3;
+static const uint32_t CAShift = 15;
+static const uint32_t CABits = 3;
+static const uint32_t CONDShift = 15;
+static const uint32_t CONDBits = 5;
+
+static const uint32_t SAShift = 15;
+static const uint32_t SA2Bits = 2;
+static const uint32_t SA3Bits = 3;
+static const uint32_t LSBWShift = 10;
+static const uint32_t LSBWBits = 5;
+static const uint32_t LSBDShift = 10;
+static const uint32_t LSBDBits = 6;
+static const uint32_t MSBWShift = 16;
+static const uint32_t MSBWBits = 5;
+static const uint32_t MSBDShift = 16;
+static const uint32_t MSBDBits = 6;
+static const uint32_t Imm5Shift = 10;
+static const uint32_t Imm5Bits = 5;
+static const uint32_t Imm6Shift = 10;
+static const uint32_t Imm6Bits = 6;
+static const uint32_t Imm12Shift = 10;
+static const uint32_t Imm12Bits = 12;
+static const uint32_t Imm14Shift = 10;
+static const uint32_t Imm14Bits = 14;
+static const uint32_t Imm15Shift = 0;
+static const uint32_t Imm15Bits = 15;
+static const uint32_t Imm16Shift = 10;
+static const uint32_t Imm16Bits = 16;
+static const uint32_t Imm20Shift = 5;
+static const uint32_t Imm20Bits = 20;
+static const uint32_t Imm21Shift = 0;
+static const uint32_t Imm21Bits = 21;
+static const uint32_t Imm26Shift = 0;
+static const uint32_t Imm26Bits = 26;
+static const uint32_t CODEShift = 0;
+static const uint32_t CODEBits = 15;
+
+// LoongArch instruction field bit masks.
+static const uint32_t RJMask = (1 << RJBits) - 1;
+static const uint32_t RKMask = (1 << RKBits) - 1;
+static const uint32_t RDMask = (1 << RDBits) - 1;
+static const uint32_t SA2Mask = (1 << SA2Bits) - 1;
+static const uint32_t SA3Mask = (1 << SA3Bits) - 1;
+static const uint32_t CONDMask = (1 << CONDBits) - 1;
+static const uint32_t LSBWMask = (1 << LSBWBits) - 1;
+static const uint32_t LSBDMask = (1 << LSBDBits) - 1;
+static const uint32_t MSBWMask = (1 << MSBWBits) - 1;
+static const uint32_t MSBDMask = (1 << MSBDBits) - 1;
+static const uint32_t CODEMask = (1 << CODEBits) - 1;
+static const uint32_t Imm5Mask = (1 << Imm5Bits) - 1;
+static const uint32_t Imm6Mask = (1 << Imm6Bits) - 1;
+static const uint32_t Imm12Mask = (1 << Imm12Bits) - 1;
+static const uint32_t Imm14Mask = (1 << Imm14Bits) - 1;
+static const uint32_t Imm15Mask = (1 << Imm15Bits) - 1;
+static const uint32_t Imm16Mask = (1 << Imm16Bits) - 1;
+static const uint32_t Imm20Mask = (1 << Imm20Bits) - 1;
+static const uint32_t Imm21Mask = (1 << Imm21Bits) - 1;
+static const uint32_t Imm26Mask = (1 << Imm26Bits) - 1;
+static const uint32_t BOffImm16Mask = ((1 << Imm16Bits) - 1) << Imm16Shift;
+static const uint32_t BOffImm21Mask = ((1 << Imm21Bits) - 1) << Imm21Shift;
+static const uint32_t BOffImm26Mask = ((1 << Imm26Bits) - 1) << Imm26Shift;
+static const uint32_t RegMask = Registers::Total - 1;
+
+// TODO(loong64) Change to syscall?
+static const uint32_t MAX_BREAK_CODE = 1024 - 1;
+static const uint32_t WASM_TRAP = 6; // BRK_OVERFLOW
+
+// TODO(loong64) Change to LoongArch instruction type.
+class Instruction;
+class InstReg;
+class InstImm;
+class InstJump;
+
+uint32_t RJ(Register r);
+uint32_t RK(Register r);
+uint32_t RD(Register r);
+uint32_t FJ(FloatRegister r);
+uint32_t FK(FloatRegister r);
+uint32_t FD(FloatRegister r);
+uint32_t FA(FloatRegister r);
+uint32_t SA2(uint32_t value);
+uint32_t SA2(FloatRegister r);
+uint32_t SA3(uint32_t value);
+uint32_t SA3(FloatRegister r);
+
+Register toRK(Instruction& i);
+Register toRJ(Instruction& i);
+Register toRD(Instruction& i);
+Register toR(Instruction& i);
+
+// LoongArch enums for instruction fields
+enum OpcodeField {
+ op_beqz = 0x10U << 26,
+ op_bnez = 0x11U << 26,
+ op_bcz = 0x12U << 26, // bceqz & bcnez
+ op_jirl = 0x13U << 26,
+ op_b = 0x14U << 26,
+ op_bl = 0x15U << 26,
+ op_beq = 0x16U << 26,
+ op_bne = 0x17U << 26,
+ op_blt = 0x18U << 26,
+ op_bge = 0x19U << 26,
+ op_bltu = 0x1aU << 26,
+ op_bgeu = 0x1bU << 26,
+
+ op_addu16i_d = 0x4U << 26,
+
+ op_lu12i_w = 0xaU << 25,
+ op_lu32i_d = 0xbU << 25,
+ op_pcaddi = 0xcU << 25,
+ op_pcalau12i = 0xdU << 25,
+ op_pcaddu12i = 0xeU << 25,
+ op_pcaddu18i = 0xfU << 25,
+ op_ll_w = 0x20U << 24,
+ op_sc_w = 0x21U << 24,
+ op_ll_d = 0x22U << 24,
+ op_sc_d = 0x23U << 24,
+ op_ldptr_w = 0x24U << 24,
+ op_stptr_w = 0x25U << 24,
+ op_ldptr_d = 0x26U << 24,
+ op_stptr_d = 0x27U << 24,
+ op_bstrins_d = 0x2U << 22,
+ op_bstrpick_d = 0x3U << 22,
+ op_slti = 0x8U << 22,
+ op_sltui = 0x9U << 22,
+ op_addi_w = 0xaU << 22,
+ op_addi_d = 0xbU << 22,
+ op_lu52i_d = 0xcU << 22,
+ op_andi = 0xdU << 22,
+ op_ori = 0xeU << 22,
+ op_xori = 0xfU << 22,
+ op_ld_b = 0xa0U << 22,
+ op_ld_h = 0xa1U << 22,
+ op_ld_w = 0xa2U << 22,
+ op_ld_d = 0xa3U << 22,
+ op_st_b = 0xa4U << 22,
+ op_st_h = 0xa5U << 22,
+ op_st_w = 0xa6U << 22,
+ op_st_d = 0xa7U << 22,
+ op_ld_bu = 0xa8U << 22,
+ op_ld_hu = 0xa9U << 22,
+ op_ld_wu = 0xaaU << 22,
+ op_preld = 0xabU << 22,
+ op_fld_s = 0xacU << 22,
+ op_fst_s = 0xadU << 22,
+ op_fld_d = 0xaeU << 22,
+ op_fst_d = 0xafU << 22,
+ op_bstr_w = 0x3U << 21, // BSTRINS_W & BSTRPICK_W
+ op_fmadd_s = 0x81U << 20,
+ op_fmadd_d = 0x82U << 20,
+ op_fmsub_s = 0x85U << 20,
+ op_fmsub_d = 0x86U << 20,
+ op_fnmadd_s = 0x89U << 20,
+ op_fnmadd_d = 0x8aU << 20,
+ op_fnmsub_s = 0x8dU << 20,
+ op_fnmsub_d = 0x8eU << 20,
+ op_fcmp_cond_s = 0xc1U << 20,
+ op_fcmp_cond_d = 0xc2U << 20,
+
+ op_bytepick_d = 0x3U << 18,
+ op_fsel = 0x340U << 18,
+
+ op_bytepick_w = 0x4U << 17,
+ op_alsl_w = 0x2U << 17,
+ op_alsl_wu = 0x3U << 17,
+ op_alsl_d = 0x16U << 17,
+
+ op_slli_d = 0x41U << 16,
+ op_srli_d = 0x45U << 16,
+ op_srai_d = 0x49U << 16,
+
+ op_slli_w = 0x81U << 15,
+ op_srli_w = 0x89U << 15,
+ op_srai_w = 0x91U << 15,
+ op_add_w = 0x20U << 15,
+ op_add_d = 0x21U << 15,
+ op_sub_w = 0x22U << 15,
+ op_sub_d = 0x23U << 15,
+ op_slt = 0x24U << 15,
+ op_sltu = 0x25U << 15,
+ op_maskeqz = 0x26U << 15,
+ op_masknez = 0x27U << 15,
+ op_nor = 0x28U << 15,
+ op_and = 0x29U << 15,
+ op_or = 0x2aU << 15,
+ op_xor = 0x2bU << 15,
+ op_orn = 0x2cU << 15,
+ op_andn = 0x2dU << 15,
+ op_sll_w = 0x2eU << 15,
+ op_srl_w = 0x2fU << 15,
+ op_sra_w = 0x30U << 15,
+ op_sll_d = 0x31U << 15,
+ op_srl_d = 0x32U << 15,
+ op_sra_d = 0x33U << 15,
+ op_rotr_w = 0x36U << 15,
+ op_rotr_d = 0x37U << 15,
+ op_rotri_w = 0x99U << 15,
+ op_rotri_d = 0x4DU << 16,
+ op_mul_w = 0x38U << 15,
+ op_mulh_w = 0x39U << 15,
+ op_mulh_wu = 0x3aU << 15,
+ op_mul_d = 0x3bU << 15,
+ op_mulh_d = 0x3cU << 15,
+ op_mulh_du = 0x3dU << 15,
+ op_mulw_d_w = 0x3eU << 15,
+ op_mulw_d_wu = 0x3fU << 15,
+ op_div_w = 0x40U << 15,
+ op_mod_w = 0x41U << 15,
+ op_div_wu = 0x42U << 15,
+ op_mod_wu = 0x43U << 15,
+ op_div_d = 0x44U << 15,
+ op_mod_d = 0x45U << 15,
+ op_div_du = 0x46U << 15,
+ op_mod_du = 0x47U << 15,
+ op_break = 0x54U << 15,
+ op_syscall = 0x56U << 15,
+ op_fadd_s = 0x201U << 15,
+ op_fadd_d = 0x202U << 15,
+ op_fsub_s = 0x205U << 15,
+ op_fsub_d = 0x206U << 15,
+ op_fmul_s = 0x209U << 15,
+ op_fmul_d = 0x20aU << 15,
+ op_fdiv_s = 0x20dU << 15,
+ op_fdiv_d = 0x20eU << 15,
+ op_fmax_s = 0x211U << 15,
+ op_fmax_d = 0x212U << 15,
+ op_fmin_s = 0x215U << 15,
+ op_fmin_d = 0x216U << 15,
+ op_fmaxa_s = 0x219U << 15,
+ op_fmaxa_d = 0x21aU << 15,
+ op_fmina_s = 0x21dU << 15,
+ op_fmina_d = 0x21eU << 15,
+ op_fcopysign_s = 0x225U << 15,
+ op_fcopysign_d = 0x226U << 15,
+ op_ldx_b = 0x7000U << 15,
+ op_ldx_h = 0x7008U << 15,
+ op_ldx_w = 0x7010U << 15,
+ op_ldx_d = 0x7018U << 15,
+ op_stx_b = 0x7020U << 15,
+ op_stx_h = 0x7028U << 15,
+ op_stx_w = 0x7030U << 15,
+ op_stx_d = 0x7038U << 15,
+ op_ldx_bu = 0x7040U << 15,
+ op_ldx_hu = 0x7048U << 15,
+ op_ldx_wu = 0x7050U << 15,
+ op_fldx_s = 0x7060U << 15,
+ op_fldx_d = 0x7068U << 15,
+ op_fstx_s = 0x7070U << 15,
+ op_fstx_d = 0x7078U << 15,
+ op_amswap_w = 0x70c0U << 15,
+ op_amswap_d = 0x70c1U << 15,
+ op_amadd_w = 0x70c2U << 15,
+ op_amadd_d = 0x70c3U << 15,
+ op_amand_w = 0x70c4U << 15,
+ op_amand_d = 0x70c5U << 15,
+ op_amor_w = 0x70c6U << 15,
+ op_amor_d = 0x70c7U << 15,
+ op_amxor_w = 0x70c8U << 15,
+ op_amxor_d = 0x70c9U << 15,
+ op_ammax_w = 0x70caU << 15,
+ op_ammax_d = 0x70cbU << 15,
+ op_ammin_w = 0x70ccU << 15,
+ op_ammin_d = 0x70cdU << 15,
+ op_ammax_wu = 0x70ceU << 15,
+ op_ammax_du = 0x70cfU << 15,
+ op_ammin_wu = 0x70d0U << 15,
+ op_ammin_du = 0x70d1U << 15,
+ op_amswap_db_w = 0x70d2U << 15,
+ op_amswap_db_d = 0x70d3U << 15,
+ op_amadd_db_w = 0x70d4U << 15,
+ op_amadd_db_d = 0x70d5U << 15,
+ op_amand_db_w = 0x70d6U << 15,
+ op_amand_db_d = 0x70d7U << 15,
+ op_amor_db_w = 0x70d8U << 15,
+ op_amor_db_d = 0x70d9U << 15,
+ op_amxor_db_w = 0x70daU << 15,
+ op_amxor_db_d = 0x70dbU << 15,
+ op_ammax_db_w = 0x70dcU << 15,
+ op_ammax_db_d = 0x70ddU << 15,
+ op_ammin_db_w = 0x70deU << 15,
+ op_ammin_db_d = 0x70dfU << 15,
+ op_ammax_db_wu = 0x70e0U << 15,
+ op_ammax_db_du = 0x70e1U << 15,
+ op_ammin_db_wu = 0x70e2U << 15,
+ op_ammin_db_du = 0x70e3U << 15,
+ op_dbar = 0x70e4U << 15,
+ op_ibar = 0x70e5U << 15,
+ op_clo_w = 0x4U << 10,
+ op_clz_w = 0x5U << 10,
+ op_cto_w = 0x6U << 10,
+ op_ctz_w = 0x7U << 10,
+ op_clo_d = 0x8U << 10,
+ op_clz_d = 0x9U << 10,
+ op_cto_d = 0xaU << 10,
+ op_ctz_d = 0xbU << 10,
+ op_revb_2h = 0xcU << 10,
+ op_revb_4h = 0xdU << 10,
+ op_revb_2w = 0xeU << 10,
+ op_revb_d = 0xfU << 10,
+ op_revh_2w = 0x10U << 10,
+ op_revh_d = 0x11U << 10,
+ op_bitrev_4b = 0x12U << 10,
+ op_bitrev_8b = 0x13U << 10,
+ op_bitrev_w = 0x14U << 10,
+ op_bitrev_d = 0x15U << 10,
+ op_ext_w_h = 0x16U << 10,
+ op_ext_w_b = 0x17U << 10,
+ op_fabs_s = 0x4501U << 10,
+ op_fabs_d = 0x4502U << 10,
+ op_fneg_s = 0x4505U << 10,
+ op_fneg_d = 0x4506U << 10,
+ op_fsqrt_s = 0x4511U << 10,
+ op_fsqrt_d = 0x4512U << 10,
+ op_fmov_s = 0x4525U << 10,
+ op_fmov_d = 0x4526U << 10,
+ op_movgr2fr_w = 0x4529U << 10,
+ op_movgr2fr_d = 0x452aU << 10,
+ op_movgr2frh_w = 0x452bU << 10,
+ op_movfr2gr_s = 0x452dU << 10,
+ op_movfr2gr_d = 0x452eU << 10,
+ op_movfrh2gr_s = 0x452fU << 10,
+ op_movgr2fcsr = 0x4530U << 10,
+ op_movfcsr2gr = 0x4532U << 10,
+ op_movfr2cf = 0x4534U << 10,
+ op_movgr2cf = 0x4536U << 10,
+ op_fcvt_s_d = 0x4646U << 10,
+ op_fcvt_d_s = 0x4649U << 10,
+ op_ftintrm_w_s = 0x4681U << 10,
+ op_ftintrm_w_d = 0x4682U << 10,
+ op_ftintrm_l_s = 0x4689U << 10,
+ op_ftintrm_l_d = 0x468aU << 10,
+ op_ftintrp_w_s = 0x4691U << 10,
+ op_ftintrp_w_d = 0x4692U << 10,
+ op_ftintrp_l_s = 0x4699U << 10,
+ op_ftintrp_l_d = 0x469aU << 10,
+ op_ftintrz_w_s = 0x46a1U << 10,
+ op_ftintrz_w_d = 0x46a2U << 10,
+ op_ftintrz_l_s = 0x46a9U << 10,
+ op_ftintrz_l_d = 0x46aaU << 10,
+ op_ftintrne_w_s = 0x46b1U << 10,
+ op_ftintrne_w_d = 0x46b2U << 10,
+ op_ftintrne_l_s = 0x46b9U << 10,
+ op_ftintrne_l_d = 0x46baU << 10,
+ op_ftint_w_s = 0x46c1U << 10,
+ op_ftint_w_d = 0x46c2U << 10,
+ op_ftint_l_s = 0x46c9U << 10,
+ op_ftint_l_d = 0x46caU << 10,
+ op_ffint_s_w = 0x4744U << 10,
+ op_ffint_s_l = 0x4746U << 10,
+ op_ffint_d_w = 0x4748U << 10,
+ op_ffint_d_l = 0x474aU << 10,
+ op_frint_s = 0x4791U << 10,
+ op_frint_d = 0x4792U << 10,
+ op_movcf2fr = 0x114d4U << 8,
+ op_movcf2gr = 0x114dcU << 8,
+};
+
+class Operand;
+
+// A BOffImm16 is a 16 bit immediate that is used for branches.
+class BOffImm16 {
+ uint32_t data;
+
+ public:
+ uint32_t encode() {
+ MOZ_ASSERT(!isInvalid());
+ return data;
+ }
+ int32_t decode() {
+ MOZ_ASSERT(!isInvalid());
+ return (int32_t(data << 18) >> 16);
+ }
+
+ explicit BOffImm16(int offset) : data((offset) >> 2 & Imm16Mask) {
+ MOZ_ASSERT((offset & 0x3) == 0);
+ MOZ_ASSERT(IsInRange(offset));
+ }
+ static bool IsInRange(int offset) {
+ if ((offset) < int(unsigned(INT16_MIN) << 2)) {
+ return false;
+ }
+ if ((offset) > (INT16_MAX << 2)) {
+ return false;
+ }
+ return true;
+ }
+ static const uint32_t INVALID = 0x00020000;
+ BOffImm16() : data(INVALID) {}
+
+ bool isInvalid() { return data == INVALID; }
+ Instruction* getDest(Instruction* src) const;
+
+ BOffImm16(InstImm inst);
+};
+
+// A JOffImm26 is a 26 bit immediate that is used for unconditional jumps.
+class JOffImm26 {
+ uint32_t data;
+
+ public:
+ uint32_t encode() {
+ MOZ_ASSERT(!isInvalid());
+ return data;
+ }
+ int32_t decode() {
+ MOZ_ASSERT(!isInvalid());
+ return (int32_t(data << 8) >> 6);
+ }
+
+ explicit JOffImm26(int offset) : data((offset) >> 2 & Imm26Mask) {
+ MOZ_ASSERT((offset & 0x3) == 0);
+ MOZ_ASSERT(IsInRange(offset));
+ }
+ static bool IsInRange(int offset) {
+ if ((offset) < -536870912) {
+ return false;
+ }
+ if ((offset) > 536870908) {
+ return false;
+ }
+ return true;
+ }
+ static const uint32_t INVALID = 0x20000000;
+ JOffImm26() : data(INVALID) {}
+
+ bool isInvalid() { return data == INVALID; }
+ Instruction* getDest(Instruction* src);
+};
+
+class Imm16 {
+ uint16_t value;
+
+ public:
+ Imm16();
+ Imm16(uint32_t imm) : value(imm) {}
+ uint32_t encode() { return value; }
+ int32_t decodeSigned() { return value; }
+ uint32_t decodeUnsigned() { return value; }
+
+ static bool IsInSignedRange(int32_t imm) {
+ return imm >= INT16_MIN && imm <= INT16_MAX;
+ }
+
+ static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT16_MAX; }
+};
+
+class Imm8 {
+ uint8_t value;
+
+ public:
+ Imm8();
+ Imm8(uint32_t imm) : value(imm) {}
+ uint32_t encode(uint32_t shift) { return value << shift; }
+ int32_t decodeSigned() { return value; }
+ uint32_t decodeUnsigned() { return value; }
+ static bool IsInSignedRange(int32_t imm) {
+ return imm >= INT8_MIN && imm <= INT8_MAX;
+ }
+ static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT8_MAX; }
+ static Imm8 Lower(Imm16 imm) { return Imm8(imm.decodeSigned() & 0xff); }
+ static Imm8 Upper(Imm16 imm) {
+ return Imm8((imm.decodeSigned() >> 8) & 0xff);
+ }
+};
+
+class Operand {
+ public:
+ enum Tag { REG, FREG, MEM };
+
+ private:
+ Tag tag : 3;
+ uint32_t reg : 5;
+ int32_t offset;
+
+ public:
+ Operand(Register reg_) : tag(REG), reg(reg_.code()) {}
+
+ Operand(FloatRegister freg) : tag(FREG), reg(freg.code()) {}
+
+ Operand(Register base, Imm32 off)
+ : tag(MEM), reg(base.code()), offset(off.value) {}
+
+ Operand(Register base, int32_t off)
+ : tag(MEM), reg(base.code()), offset(off) {}
+
+ Operand(const Address& addr)
+ : tag(MEM), reg(addr.base.code()), offset(addr.offset) {}
+
+ Tag getTag() const { return tag; }
+
+ Register toReg() const {
+ MOZ_ASSERT(tag == REG);
+ return Register::FromCode(reg);
+ }
+
+ FloatRegister toFReg() const {
+ MOZ_ASSERT(tag == FREG);
+ return FloatRegister::FromCode(reg);
+ }
+
+ void toAddr(Register* r, Imm32* dest) const {
+ MOZ_ASSERT(tag == MEM);
+ *r = Register::FromCode(reg);
+ *dest = Imm32(offset);
+ }
+ Address toAddress() const {
+ MOZ_ASSERT(tag == MEM);
+ return Address(Register::FromCode(reg), offset);
+ }
+ int32_t disp() const {
+ MOZ_ASSERT(tag == MEM);
+ return offset;
+ }
+
+ int32_t base() const {
+ MOZ_ASSERT(tag == MEM);
+ return reg;
+ }
+ Register baseReg() const {
+ MOZ_ASSERT(tag == MEM);
+ return Register::FromCode(reg);
+ }
+};
+
+// int check.
+inline bool is_intN(int32_t x, unsigned n) {
+ MOZ_ASSERT((0 < n) && (n < 64));
+ int32_t limit = static_cast<int32_t>(1) << (n - 1);
+ return (-limit <= x) && (x < limit);
+}
+
+inline bool is_uintN(int32_t x, unsigned n) {
+ MOZ_ASSERT((0 < n) && (n < (sizeof(x) * 8)));
+ return !(x >> n);
+}
+
+inline Imm32 Imm64::firstHalf() const { return low(); }
+
+inline Imm32 Imm64::secondHalf() const { return hi(); }
+
+static constexpr int32_t SliceSize = 1024;
+typedef js::jit::AssemblerBuffer<SliceSize, Instruction> LOONGBuffer;
+
+class LOONGBufferWithExecutableCopy : public LOONGBuffer {
+ public:
+ void executableCopy(uint8_t* buffer) {
+ if (this->oom()) {
+ return;
+ }
+
+ for (Slice* cur = head; cur != nullptr; cur = cur->getNext()) {
+ memcpy(buffer, &cur->instructions, cur->length());
+ buffer += cur->length();
+ }
+ }
+
+ bool appendRawCode(const uint8_t* code, size_t numBytes) {
+ if (this->oom()) {
+ return false;
+ }
+ while (numBytes > SliceSize) {
+ this->putBytes(SliceSize, code);
+ numBytes -= SliceSize;
+ code += SliceSize;
+ }
+ this->putBytes(numBytes, code);
+ return !this->oom();
+ }
+};
+
+class AssemblerLOONG64 : public AssemblerShared {
+ public:
+ // TODO(loong64): Should we remove these conditions here?
+ enum Condition {
+ Equal,
+ NotEqual,
+ Above,
+ AboveOrEqual,
+ Below,
+ BelowOrEqual,
+ GreaterThan,
+ GreaterThanOrEqual,
+ GreaterThanOrEqual_Signed,
+ GreaterThanOrEqual_NotSigned,
+ LessThan,
+ LessThan_Signed,
+ LessThan_NotSigned,
+ LessThanOrEqual,
+ Overflow,
+ CarrySet,
+ CarryClear,
+ Signed,
+ NotSigned,
+ Zero,
+ NonZero,
+ Always,
+ };
+
+ enum DoubleCondition {
+ DoubleOrdered,
+ DoubleEqual,
+ DoubleNotEqual,
+ DoubleGreaterThan,
+ DoubleGreaterThanOrEqual,
+ DoubleLessThan,
+ DoubleLessThanOrEqual,
+ DoubleUnordered,
+ DoubleEqualOrUnordered,
+ DoubleNotEqualOrUnordered,
+ DoubleGreaterThanOrUnordered,
+ DoubleGreaterThanOrEqualOrUnordered,
+ DoubleLessThanOrUnordered,
+ DoubleLessThanOrEqualOrUnordered
+ };
+
+ enum FPUCondition {
+ kNoFPUCondition = -1,
+
+ CAF = 0x00,
+ SAF = 0x01,
+ CLT = 0x02,
+ SLT = 0x03,
+ CEQ = 0x04,
+ SEQ = 0x05,
+ CLE = 0x06,
+ SLE = 0x07,
+ CUN = 0x08,
+ SUN = 0x09,
+ CULT = 0x0a,
+ SULT = 0x0b,
+ CUEQ = 0x0c,
+ SUEQ = 0x0d,
+ CULE = 0x0e,
+ SULE = 0x0f,
+ CNE = 0x10,
+ SNE = 0x11,
+ COR = 0x14,
+ SOR = 0x15,
+ CUNE = 0x18,
+ SUNE = 0x19,
+ };
+
+ enum FPConditionBit { FCC0 = 0, FCC1, FFC2, FCC3, FCC4, FCC5, FCC6, FCC7 };
+
+ enum FPControl { FCSR = 0 };
+
+ enum FCSRBit { CauseI = 24, CauseU, CauseO, CauseZ, CauseV };
+
+ enum FloatFormat { SingleFloat, DoubleFloat };
+
+ enum JumpOrCall { BranchIsJump, BranchIsCall };
+
+ enum FloatTestKind { TestForTrue, TestForFalse };
+
+ // :( this should be protected, but since CodeGenerator
+ // wants to use it, It needs to go out here :(
+
+ BufferOffset nextOffset() { return m_buffer.nextOffset(); }
+
+ protected:
+ Instruction* editSrc(BufferOffset bo) { return m_buffer.getInst(bo); }
+
+ // structure for fixing up pc-relative loads/jumps when a the machine code
+ // gets moved (executable copy, gc, etc.)
+ struct RelativePatch {
+ // the offset within the code buffer where the value is loaded that
+ // we want to fix-up
+ BufferOffset offset;
+ void* target;
+ RelocationKind kind;
+
+ RelativePatch(BufferOffset offset, void* target, RelocationKind kind)
+ : offset(offset), target(target), kind(kind) {}
+ };
+
+ js::Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;
+
+ CompactBufferWriter jumpRelocations_;
+ CompactBufferWriter dataRelocations_;
+
+ LOONGBufferWithExecutableCopy m_buffer;
+
+#ifdef JS_JITSPEW
+ Sprinter* printer;
+#endif
+
+ public:
+ AssemblerLOONG64()
+ : m_buffer(),
+#ifdef JS_JITSPEW
+ printer(nullptr),
+#endif
+ isFinished(false) {
+ }
+
+ static Condition InvertCondition(Condition cond);
+ static DoubleCondition InvertCondition(DoubleCondition cond);
+ // This is changing the condition codes for cmp a, b to the same codes for cmp
+ // b, a.
+ static Condition InvertCmpCondition(Condition cond);
+
+ // As opposed to x86/x64 version, the data relocation has to be executed
+ // before to recover the pointer, and not after.
+ void writeDataRelocation(ImmGCPtr ptr) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // TraceOneDataRelocation.
+ if (ptr.value) {
+ if (gc::IsInsideNursery(ptr.value)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(nextOffset().getOffset());
+ }
+ }
+
+ void assertNoGCThings() const {
+#ifdef DEBUG
+ MOZ_ASSERT(dataRelocations_.length() == 0);
+ for (auto& j : jumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+#endif
+ }
+
+ public:
+ void setUnlimitedBuffer() { m_buffer.setUnlimited(); }
+ bool oom() const;
+
+ void setPrinter(Sprinter* sp) {
+#ifdef JS_JITSPEW
+ printer = sp;
+#endif
+ }
+
+#ifdef JS_JITSPEW
+ inline void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) {
+ if (MOZ_UNLIKELY(printer || JitSpewEnabled(JitSpew_Codegen))) {
+ va_list va;
+ va_start(va, fmt);
+ spew(fmt, va);
+ va_end(va);
+ }
+ }
+
+ void decodeBranchInstAndSpew(InstImm branch);
+#else
+ MOZ_ALWAYS_INLINE void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) {}
+#endif
+
+#ifdef JS_JITSPEW
+ MOZ_COLD void spew(const char* fmt, va_list va) MOZ_FORMAT_PRINTF(2, 0) {
+ // Buffer to hold the formatted string. Note that this may contain
+ // '%' characters, so do not pass it directly to printf functions.
+ char buf[200];
+
+ int i = VsprintfLiteral(buf, fmt, va);
+ if (i > -1) {
+ if (printer) {
+ printer->printf("%s\n", buf);
+ }
+ js::jit::JitSpew(js::jit::JitSpew_Codegen, "%s", buf);
+ }
+ }
+#endif
+
+ Register getStackPointer() const { return StackPointer; }
+
+ protected:
+ bool isFinished;
+
+ public:
+ void finish();
+ bool appendRawCode(const uint8_t* code, size_t numBytes);
+ bool reserve(size_t size);
+ bool swapBuffer(wasm::Bytes& bytes);
+ void executableCopy(void* buffer);
+ void copyJumpRelocationTable(uint8_t* dest);
+ void copyDataRelocationTable(uint8_t* dest);
+
+ // Size of the instruction stream, in bytes.
+ size_t size() const;
+ // Size of the jump relocation table, in bytes.
+ size_t jumpRelocationTableBytes() const;
+ size_t dataRelocationTableBytes() const;
+
+ // Size of the data table, in bytes.
+ size_t bytesNeeded() const;
+
+ // Write a blob of binary into the instruction stream *OR*
+ // into a destination address. If dest is nullptr (the default), then the
+ // instruction gets written into the instruction stream. If dest is not null
+ // it is interpreted as a pointer to the location that we want the
+ // instruction to be written.
+ BufferOffset writeInst(uint32_t x, uint32_t* dest = nullptr);
+ // A static variant for the cases where we don't want to have an assembler
+ // object at all. Normally, you would use the dummy (nullptr) object.
+ static void WriteInstStatic(uint32_t x, uint32_t* dest);
+
+ public:
+ BufferOffset haltingAlign(int alignment);
+ BufferOffset nopAlign(int alignment);
+ BufferOffset as_nop() { return as_andi(zero, zero, 0); }
+
+ // Branch and jump instructions
+ BufferOffset as_b(JOffImm26 off);
+ BufferOffset as_bl(JOffImm26 off);
+ BufferOffset as_jirl(Register rd, Register rj, BOffImm16 off);
+
+ InstImm getBranchCode(JumpOrCall jumpOrCall); // b, bl
+ InstImm getBranchCode(Register rd, Register rj,
+ Condition c); // beq, bne, bge, bgeu, blt, bltu
+ InstImm getBranchCode(Register rj, Condition c); // beqz, bnez
+ InstImm getBranchCode(FPConditionBit cj); // bceqz, bcnez
+
+ // Arithmetic instructions
+ BufferOffset as_add_w(Register rd, Register rj, Register rk);
+ BufferOffset as_add_d(Register rd, Register rj, Register rk);
+ BufferOffset as_sub_w(Register rd, Register rj, Register rk);
+ BufferOffset as_sub_d(Register rd, Register rj, Register rk);
+
+ BufferOffset as_addi_w(Register rd, Register rj, int32_t si12);
+ BufferOffset as_addi_d(Register rd, Register rj, int32_t si12);
+ BufferOffset as_addu16i_d(Register rd, Register rj, int32_t si16);
+
+ BufferOffset as_alsl_w(Register rd, Register rj, Register rk, uint32_t sa2);
+ BufferOffset as_alsl_wu(Register rd, Register rj, Register rk, uint32_t sa2);
+ BufferOffset as_alsl_d(Register rd, Register rj, Register rk, uint32_t sa2);
+
+ BufferOffset as_lu12i_w(Register rd, int32_t si20);
+ BufferOffset as_lu32i_d(Register rd, int32_t si20);
+ BufferOffset as_lu52i_d(Register rd, Register rj, int32_t si12);
+
+ BufferOffset as_slt(Register rd, Register rj, Register rk);
+ BufferOffset as_sltu(Register rd, Register rj, Register rk);
+ BufferOffset as_slti(Register rd, Register rj, int32_t si12);
+ BufferOffset as_sltui(Register rd, Register rj, int32_t si12);
+
+ BufferOffset as_pcaddi(Register rd, int32_t si20);
+ BufferOffset as_pcaddu12i(Register rd, int32_t si20);
+ BufferOffset as_pcaddu18i(Register rd, int32_t si20);
+ BufferOffset as_pcalau12i(Register rd, int32_t si20);
+
+ BufferOffset as_mul_w(Register rd, Register rj, Register rk);
+ BufferOffset as_mulh_w(Register rd, Register rj, Register rk);
+ BufferOffset as_mulh_wu(Register rd, Register rj, Register rk);
+ BufferOffset as_mul_d(Register rd, Register rj, Register rk);
+ BufferOffset as_mulh_d(Register rd, Register rj, Register rk);
+ BufferOffset as_mulh_du(Register rd, Register rj, Register rk);
+
+ BufferOffset as_mulw_d_w(Register rd, Register rj, Register rk);
+ BufferOffset as_mulw_d_wu(Register rd, Register rj, Register rk);
+
+ BufferOffset as_div_w(Register rd, Register rj, Register rk);
+ BufferOffset as_mod_w(Register rd, Register rj, Register rk);
+ BufferOffset as_div_wu(Register rd, Register rj, Register rk);
+ BufferOffset as_mod_wu(Register rd, Register rj, Register rk);
+ BufferOffset as_div_d(Register rd, Register rj, Register rk);
+ BufferOffset as_mod_d(Register rd, Register rj, Register rk);
+ BufferOffset as_div_du(Register rd, Register rj, Register rk);
+ BufferOffset as_mod_du(Register rd, Register rj, Register rk);
+
+ // Logical instructions
+ BufferOffset as_and(Register rd, Register rj, Register rk);
+ BufferOffset as_or(Register rd, Register rj, Register rk);
+ BufferOffset as_xor(Register rd, Register rj, Register rk);
+ BufferOffset as_nor(Register rd, Register rj, Register rk);
+ BufferOffset as_andn(Register rd, Register rj, Register rk);
+ BufferOffset as_orn(Register rd, Register rj, Register rk);
+
+ BufferOffset as_andi(Register rd, Register rj, int32_t ui12);
+ BufferOffset as_ori(Register rd, Register rj, int32_t ui12);
+ BufferOffset as_xori(Register rd, Register rj, int32_t ui12);
+
+ // Shift instructions
+ BufferOffset as_sll_w(Register rd, Register rj, Register rk);
+ BufferOffset as_srl_w(Register rd, Register rj, Register rk);
+ BufferOffset as_sra_w(Register rd, Register rj, Register rk);
+ BufferOffset as_rotr_w(Register rd, Register rj, Register rk);
+
+ BufferOffset as_slli_w(Register rd, Register rj, int32_t ui5);
+ BufferOffset as_srli_w(Register rd, Register rj, int32_t ui5);
+ BufferOffset as_srai_w(Register rd, Register rj, int32_t ui5);
+ BufferOffset as_rotri_w(Register rd, Register rj, int32_t ui5);
+
+ BufferOffset as_sll_d(Register rd, Register rj, Register rk);
+ BufferOffset as_srl_d(Register rd, Register rj, Register rk);
+ BufferOffset as_sra_d(Register rd, Register rj, Register rk);
+ BufferOffset as_rotr_d(Register rd, Register rj, Register rk);
+
+ BufferOffset as_slli_d(Register rd, Register rj, int32_t ui6);
+ BufferOffset as_srli_d(Register rd, Register rj, int32_t ui6);
+ BufferOffset as_srai_d(Register rd, Register rj, int32_t ui6);
+ BufferOffset as_rotri_d(Register rd, Register rj, int32_t ui6);
+
+ // Bit operation instrucitons
+ BufferOffset as_ext_w_b(Register rd, Register rj);
+ BufferOffset as_ext_w_h(Register rd, Register rj);
+
+ BufferOffset as_clo_w(Register rd, Register rj);
+ BufferOffset as_clz_w(Register rd, Register rj);
+ BufferOffset as_cto_w(Register rd, Register rj);
+ BufferOffset as_ctz_w(Register rd, Register rj);
+ BufferOffset as_clo_d(Register rd, Register rj);
+ BufferOffset as_clz_d(Register rd, Register rj);
+ BufferOffset as_cto_d(Register rd, Register rj);
+ BufferOffset as_ctz_d(Register rd, Register rj);
+
+ BufferOffset as_bytepick_w(Register rd, Register rj, Register rk,
+ int32_t sa2);
+ BufferOffset as_bytepick_d(Register rd, Register rj, Register rk,
+ int32_t sa3);
+
+ BufferOffset as_revb_2h(Register rd, Register rj);
+ BufferOffset as_revb_4h(Register rd, Register rj);
+ BufferOffset as_revb_2w(Register rd, Register rj);
+ BufferOffset as_revb_d(Register rd, Register rj);
+
+ BufferOffset as_revh_2w(Register rd, Register rj);
+ BufferOffset as_revh_d(Register rd, Register rj);
+
+ BufferOffset as_bitrev_4b(Register rd, Register rj);
+ BufferOffset as_bitrev_8b(Register rd, Register rj);
+
+ BufferOffset as_bitrev_w(Register rd, Register rj);
+ BufferOffset as_bitrev_d(Register rd, Register rj);
+
+ BufferOffset as_bstrins_w(Register rd, Register rj, int32_t msbw,
+ int32_t lsbw);
+ BufferOffset as_bstrins_d(Register rd, Register rj, int32_t msbd,
+ int32_t lsbd);
+ BufferOffset as_bstrpick_w(Register rd, Register rj, int32_t msbw,
+ int32_t lsbw);
+ BufferOffset as_bstrpick_d(Register rd, Register rj, int32_t msbd,
+ int32_t lsbd);
+
+ BufferOffset as_maskeqz(Register rd, Register rj, Register rk);
+ BufferOffset as_masknez(Register rd, Register rj, Register rk);
+
+ // Load and store instructions
+ BufferOffset as_ld_b(Register rd, Register rj, int32_t si12);
+ BufferOffset as_ld_h(Register rd, Register rj, int32_t si12);
+ BufferOffset as_ld_w(Register rd, Register rj, int32_t si12);
+ BufferOffset as_ld_d(Register rd, Register rj, int32_t si12);
+ BufferOffset as_ld_bu(Register rd, Register rj, int32_t si12);
+ BufferOffset as_ld_hu(Register rd, Register rj, int32_t si12);
+ BufferOffset as_ld_wu(Register rd, Register rj, int32_t si12);
+ BufferOffset as_st_b(Register rd, Register rj, int32_t si12);
+ BufferOffset as_st_h(Register rd, Register rj, int32_t si12);
+ BufferOffset as_st_w(Register rd, Register rj, int32_t si12);
+ BufferOffset as_st_d(Register rd, Register rj, int32_t si12);
+
+ BufferOffset as_ldx_b(Register rd, Register rj, Register rk);
+ BufferOffset as_ldx_h(Register rd, Register rj, Register rk);
+ BufferOffset as_ldx_w(Register rd, Register rj, Register rk);
+ BufferOffset as_ldx_d(Register rd, Register rj, Register rk);
+ BufferOffset as_ldx_bu(Register rd, Register rj, Register rk);
+ BufferOffset as_ldx_hu(Register rd, Register rj, Register rk);
+ BufferOffset as_ldx_wu(Register rd, Register rj, Register rk);
+ BufferOffset as_stx_b(Register rd, Register rj, Register rk);
+ BufferOffset as_stx_h(Register rd, Register rj, Register rk);
+ BufferOffset as_stx_w(Register rd, Register rj, Register rk);
+ BufferOffset as_stx_d(Register rd, Register rj, Register rk);
+
+ BufferOffset as_ldptr_w(Register rd, Register rj, int32_t si14);
+ BufferOffset as_ldptr_d(Register rd, Register rj, int32_t si14);
+ BufferOffset as_stptr_w(Register rd, Register rj, int32_t si14);
+ BufferOffset as_stptr_d(Register rd, Register rj, int32_t si14);
+
+ BufferOffset as_preld(int32_t hint, Register rj, int32_t si12);
+
+ // Atomic instructions
+ BufferOffset as_amswap_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amswap_d(Register rd, Register rj, Register rk);
+ BufferOffset as_amadd_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amadd_d(Register rd, Register rj, Register rk);
+ BufferOffset as_amand_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amand_d(Register rd, Register rj, Register rk);
+ BufferOffset as_amor_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amor_d(Register rd, Register rj, Register rk);
+ BufferOffset as_amxor_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amxor_d(Register rd, Register rj, Register rk);
+ BufferOffset as_ammax_w(Register rd, Register rj, Register rk);
+ BufferOffset as_ammax_d(Register rd, Register rj, Register rk);
+ BufferOffset as_ammin_w(Register rd, Register rj, Register rk);
+ BufferOffset as_ammin_d(Register rd, Register rj, Register rk);
+ BufferOffset as_ammax_wu(Register rd, Register rj, Register rk);
+ BufferOffset as_ammax_du(Register rd, Register rj, Register rk);
+ BufferOffset as_ammin_wu(Register rd, Register rj, Register rk);
+ BufferOffset as_ammin_du(Register rd, Register rj, Register rk);
+
+ BufferOffset as_amswap_db_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amswap_db_d(Register rd, Register rj, Register rk);
+ BufferOffset as_amadd_db_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amadd_db_d(Register rd, Register rj, Register rk);
+ BufferOffset as_amand_db_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amand_db_d(Register rd, Register rj, Register rk);
+ BufferOffset as_amor_db_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amor_db_d(Register rd, Register rj, Register rk);
+ BufferOffset as_amxor_db_w(Register rd, Register rj, Register rk);
+ BufferOffset as_amxor_db_d(Register rd, Register rj, Register rk);
+ BufferOffset as_ammax_db_w(Register rd, Register rj, Register rk);
+ BufferOffset as_ammax_db_d(Register rd, Register rj, Register rk);
+ BufferOffset as_ammin_db_w(Register rd, Register rj, Register rk);
+ BufferOffset as_ammin_db_d(Register rd, Register rj, Register rk);
+ BufferOffset as_ammax_db_wu(Register rd, Register rj, Register rk);
+ BufferOffset as_ammax_db_du(Register rd, Register rj, Register rk);
+ BufferOffset as_ammin_db_wu(Register rd, Register rj, Register rk);
+ BufferOffset as_ammin_db_du(Register rd, Register rj, Register rk);
+
+ BufferOffset as_ll_w(Register rd, Register rj, int32_t si14);
+ BufferOffset as_ll_d(Register rd, Register rj, int32_t si14);
+ BufferOffset as_sc_w(Register rd, Register rj, int32_t si14);
+ BufferOffset as_sc_d(Register rd, Register rj, int32_t si14);
+
+ // Barrier instructions
+ BufferOffset as_dbar(int32_t hint);
+ BufferOffset as_ibar(int32_t hint);
+
+ // FP Arithmetic instructions
+ BufferOffset as_fadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fmul_s(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fmul_d(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fdiv_s(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fdiv_d(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+
+ BufferOffset as_fmadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FloatRegister fa);
+ BufferOffset as_fmadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FloatRegister fa);
+ BufferOffset as_fmsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FloatRegister fa);
+ BufferOffset as_fmsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FloatRegister fa);
+ BufferOffset as_fnmadd_s(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FloatRegister fa);
+ BufferOffset as_fnmadd_d(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FloatRegister fa);
+ BufferOffset as_fnmsub_s(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FloatRegister fa);
+ BufferOffset as_fnmsub_d(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FloatRegister fa);
+
+ BufferOffset as_fmax_s(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fmax_d(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fmin_s(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fmin_d(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+
+ BufferOffset as_fmaxa_s(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fmaxa_d(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fmina_s(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+ BufferOffset as_fmina_d(FloatRegister fd, FloatRegister fj, FloatRegister fk);
+
+ BufferOffset as_fabs_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_fabs_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_fneg_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_fneg_d(FloatRegister fd, FloatRegister fj);
+
+ BufferOffset as_fsqrt_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_fsqrt_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_fcopysign_s(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk);
+ BufferOffset as_fcopysign_d(FloatRegister fd, FloatRegister fj,
+ FloatRegister fk);
+
+ // FP compare instructions (fcmp.cond.s fcmp.cond.d)
+ BufferOffset as_fcmp_cor(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_ceq(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_cne(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_cle(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_clt(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_cun(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_cueq(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_cune(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_cule(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+ BufferOffset as_fcmp_cult(FloatFormat fmt, FloatRegister fj, FloatRegister fk,
+ FPConditionBit cd);
+
+ // FP conversion instructions
+ BufferOffset as_fcvt_s_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_fcvt_d_s(FloatRegister fd, FloatRegister fj);
+
+ BufferOffset as_ffint_s_w(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ffint_s_l(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ffint_d_w(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ffint_d_l(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftint_w_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftint_w_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftint_l_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftint_l_d(FloatRegister fd, FloatRegister fj);
+
+ BufferOffset as_ftintrm_w_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrm_w_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrm_l_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrm_l_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrp_w_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrp_w_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrp_l_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrp_l_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrz_w_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrz_w_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrz_l_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrz_l_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrne_w_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrne_w_d(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrne_l_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_ftintrne_l_d(FloatRegister fd, FloatRegister fj);
+
+ BufferOffset as_frint_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_frint_d(FloatRegister fd, FloatRegister fj);
+
+ // FP mov instructions
+ BufferOffset as_fmov_s(FloatRegister fd, FloatRegister fj);
+ BufferOffset as_fmov_d(FloatRegister fd, FloatRegister fj);
+
+ BufferOffset as_fsel(FloatRegister fd, FloatRegister fj, FloatRegister fk,
+ FPConditionBit ca);
+
+ BufferOffset as_movgr2fr_w(FloatRegister fd, Register rj);
+ BufferOffset as_movgr2fr_d(FloatRegister fd, Register rj);
+ BufferOffset as_movgr2frh_w(FloatRegister fd, Register rj);
+
+ BufferOffset as_movfr2gr_s(Register rd, FloatRegister fj);
+ BufferOffset as_movfr2gr_d(Register rd, FloatRegister fj);
+ BufferOffset as_movfrh2gr_s(Register rd, FloatRegister fj);
+
+ BufferOffset as_movgr2fcsr(Register rj);
+ BufferOffset as_movfcsr2gr(Register rd);
+
+ BufferOffset as_movfr2cf(FPConditionBit cd, FloatRegister fj);
+ BufferOffset as_movcf2fr(FloatRegister fd, FPConditionBit cj);
+
+ BufferOffset as_movgr2cf(FPConditionBit cd, Register rj);
+ BufferOffset as_movcf2gr(Register rd, FPConditionBit cj);
+
+ // FP load/store instructions
+ BufferOffset as_fld_s(FloatRegister fd, Register rj, int32_t si12);
+ BufferOffset as_fld_d(FloatRegister fd, Register rj, int32_t si12);
+ BufferOffset as_fst_s(FloatRegister fd, Register rj, int32_t si12);
+ BufferOffset as_fst_d(FloatRegister fd, Register rj, int32_t si12);
+
+ BufferOffset as_fldx_s(FloatRegister fd, Register rj, Register rk);
+ BufferOffset as_fldx_d(FloatRegister fd, Register rj, Register rk);
+ BufferOffset as_fstx_s(FloatRegister fd, Register rj, Register rk);
+ BufferOffset as_fstx_d(FloatRegister fd, Register rj, Register rk);
+
+ // label operations
+ void bind(Label* label, BufferOffset boff = BufferOffset());
+ virtual void bind(InstImm* inst, uintptr_t branch, uintptr_t target) = 0;
+ void bind(CodeLabel* label) { label->target()->bind(currentOffset()); }
+ uint32_t currentOffset() { return nextOffset().getOffset(); }
+ void retarget(Label* label, Label* target);
+
+ void call(Label* label);
+ void call(void* target);
+
+ void as_break(uint32_t code);
+
+ public:
+ static bool SupportsFloatingPoint() {
+#if defined(__loongarch_hard_float) || defined(JS_SIMULATOR_LOONG64)
+ return true;
+#else
+ return false;
+#endif
+ }
+ static bool SupportsUnalignedAccesses() { return true; }
+ static bool SupportsFastUnalignedFPAccesses() { return true; }
+
+ static bool HasRoundInstruction(RoundingMode mode) { return false; }
+
+ protected:
+ InstImm invertBranch(InstImm branch, BOffImm16 skipOffset);
+ void addPendingJump(BufferOffset src, ImmPtr target, RelocationKind kind) {
+ enoughMemory_ &= jumps_.append(RelativePatch(src, target.value, kind));
+ if (kind == RelocationKind::JITCODE) {
+ jumpRelocations_.writeUnsigned(src.getOffset());
+ }
+ }
+
+ void addLongJump(BufferOffset src, BufferOffset dst) {
+ CodeLabel cl;
+ cl.patchAt()->bind(src.getOffset());
+ cl.target()->bind(dst.getOffset());
+ cl.setLinkMode(CodeLabel::JumpImmediate);
+ addCodeLabel(std::move(cl));
+ }
+
+ public:
+ void flushBuffer() {}
+
+ void comment(const char* msg) { spew("; %s", msg); }
+
+ static uint32_t NopSize() { return 4; }
+
+ static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm);
+
+ static uint8_t* NextInstruction(uint8_t* instruction,
+ uint32_t* count = nullptr);
+
+ static void ToggleToJmp(CodeLocationLabel inst_);
+ static void ToggleToCmp(CodeLocationLabel inst_);
+
+ void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
+ const Disassembler::HeapAccess& heapAccess) {
+ // Implement this if we implement a disassembler.
+ }
+}; // AssemblerLOONG64
+
+// andi r0, r0, 0
+const uint32_t NopInst = 0x03400000;
+
+// An Instruction is a structure for both encoding and decoding any and all
+// LoongArch instructions.
+class Instruction {
+ public:
+ uint32_t data;
+
+ protected:
+ // Standard constructor
+ Instruction(uint32_t data_) : data(data_) {}
+ // You should never create an instruction directly. You should create a
+ // more specific instruction which will eventually call one of these
+ // constructors for you.
+
+ public:
+ uint32_t encode() const { return data; }
+
+ void makeNop() { data = NopInst; }
+
+ void setData(uint32_t data) { this->data = data; }
+
+ const Instruction& operator=(const Instruction& src) {
+ data = src.data;
+ return *this;
+ }
+
+ // Extract the one particular bit.
+ uint32_t extractBit(uint32_t bit) { return (encode() >> bit) & 1; }
+ // Extract a bit field out of the instruction
+ uint32_t extractBitField(uint32_t hi, uint32_t lo) {
+ return (encode() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Get the next instruction in the instruction stream.
+ // This does neat things like ignoreconstant pools and their guards.
+ Instruction* next();
+
+ // Sometimes, an api wants a uint32_t (or a pointer to it) rather than
+ // an instruction. raw() just coerces this into a pointer to a uint32_t
+ const uint32_t* raw() const { return &data; }
+ uint32_t size() const { return 4; }
+}; // Instruction
+
+// make sure that it is the right size
+static_assert(sizeof(Instruction) == 4,
+ "Size of Instruction class has to be 4 bytes.");
+
+class InstNOP : public Instruction {
+ public:
+ InstNOP() : Instruction(NopInst) {}
+};
+
+// Class for register type instructions.
+class InstReg : public Instruction {
+ public:
+ InstReg(OpcodeField op, Register rj, Register rd)
+ : Instruction(op | RJ(rj) | RD(rd)) {}
+ InstReg(OpcodeField op, Register rk, Register rj, Register rd)
+ : Instruction(op | RK(rk) | RJ(rj) | RD(rd)) {}
+ InstReg(OpcodeField op, uint32_t sa, Register rk, Register rj, Register rd,
+ uint32_t sa_bit)
+ : Instruction(sa_bit == 2 ? op | SA2(sa) | RK(rk) | RJ(rj) | RD(rd)
+ : op | SA3(sa) | RK(rk) | RJ(rj) | RD(rd)) {
+ MOZ_ASSERT(sa_bit == 2 || sa_bit == 3);
+ }
+ InstReg(OpcodeField op, Register rj, Register rd, bool HasRd)
+ : Instruction(HasRd ? op | RJ(rj) | RD(rd) : op | RK(rj) | RJ(rd)) {}
+
+ // For floating-point
+ InstReg(OpcodeField op, Register rj, FloatRegister fd)
+ : Instruction(op | RJ(rj) | FD(fd)) {}
+ InstReg(OpcodeField op, FloatRegister fj, FloatRegister fd)
+ : Instruction(op | FJ(fj) | FD(fd)) {}
+ InstReg(OpcodeField op, FloatRegister fk, FloatRegister fj, FloatRegister fd)
+ : Instruction(op | FK(fk) | FJ(fj) | FD(fd)) {}
+ InstReg(OpcodeField op, Register rk, Register rj, FloatRegister fd)
+ : Instruction(op | RK(rk) | RJ(rj) | FD(fd)) {}
+ InstReg(OpcodeField op, FloatRegister fa, FloatRegister fk, FloatRegister fj,
+ FloatRegister fd)
+ : Instruction(op | FA(fa) | FK(fk) | FJ(fj) | FD(fd)) {}
+ InstReg(OpcodeField op, AssemblerLOONG64::FPConditionBit ca, FloatRegister fk,
+ FloatRegister fj, FloatRegister fd)
+ : Instruction(op | ca << CAShift | FK(fk) | FJ(fj) | FD(fd)) {
+ MOZ_ASSERT(op == op_fsel);
+ }
+ InstReg(OpcodeField op, FloatRegister fj, Register rd)
+ : Instruction(op | FJ(fj) | RD(rd)) {
+ MOZ_ASSERT((op == op_movfr2gr_s) || (op == op_movfr2gr_d) ||
+ (op == op_movfrh2gr_s));
+ }
+ InstReg(OpcodeField op, Register rj, uint32_t fd)
+ : Instruction(op | RJ(rj) | fd) {
+ MOZ_ASSERT(op == op_movgr2fcsr);
+ }
+ InstReg(OpcodeField op, uint32_t fj, Register rd)
+ : Instruction(op | (fj << FJShift) | RD(rd)) {
+ MOZ_ASSERT(op == op_movfcsr2gr);
+ }
+ InstReg(OpcodeField op, FloatRegister fj, AssemblerLOONG64::FPConditionBit cd)
+ : Instruction(op | FJ(fj) | cd) {
+ MOZ_ASSERT(op == op_movfr2cf);
+ }
+ InstReg(OpcodeField op, AssemblerLOONG64::FPConditionBit cj, FloatRegister fd)
+ : Instruction(op | (cj << CJShift) | FD(fd)) {
+ MOZ_ASSERT(op == op_movcf2fr);
+ }
+ InstReg(OpcodeField op, Register rj, AssemblerLOONG64::FPConditionBit cd)
+ : Instruction(op | RJ(rj) | cd) {
+ MOZ_ASSERT(op == op_movgr2cf);
+ }
+ InstReg(OpcodeField op, AssemblerLOONG64::FPConditionBit cj, Register rd)
+ : Instruction(op | (cj << CJShift) | RD(rd)) {
+ MOZ_ASSERT(op == op_movcf2gr);
+ }
+ InstReg(OpcodeField op, int32_t cond, FloatRegister fk, FloatRegister fj,
+ AssemblerLOONG64::FPConditionBit cd)
+ : Instruction(op | (cond & CONDMask) << CONDShift | FK(fk) | FJ(fj) |
+ (cd & RDMask)) {
+ MOZ_ASSERT(is_uintN(cond, 5));
+ }
+
+ uint32_t extractRK() {
+ return extractBitField(RKShift + RKBits - 1, RKShift);
+ }
+ uint32_t extractRJ() {
+ return extractBitField(RJShift + RJBits - 1, RJShift);
+ }
+ uint32_t extractRD() {
+ return extractBitField(RDShift + RDBits - 1, RDShift);
+ }
+ uint32_t extractSA2() {
+ return extractBitField(SAShift + SA2Bits - 1, SAShift);
+ }
+ uint32_t extractSA3() {
+ return extractBitField(SAShift + SA3Bits - 1, SAShift);
+ }
+};
+
+// Class for branch, load and store instructions with immediate offset.
+class InstImm : public Instruction {
+ public:
+ void extractImm16(BOffImm16* dest);
+ uint32_t genImm(int32_t value, uint32_t value_bits) {
+ uint32_t imm = value & Imm5Mask;
+ if (value_bits == 6) {
+ imm = value & Imm6Mask;
+ } else if (value_bits == 12) {
+ imm = value & Imm12Mask;
+ } else if (value_bits == 14) {
+ imm = value & Imm14Mask;
+ }
+
+ return imm;
+ }
+
+ InstImm(OpcodeField op, int32_t value, Register rj, Register rd,
+ uint32_t value_bits)
+ : Instruction(op | genImm(value, value_bits) << RKShift | RJ(rj) |
+ RD(rd)) {
+ MOZ_ASSERT(value_bits == 5 || value_bits == 6 || value_bits == 12 ||
+ value_bits == 14);
+ }
+ InstImm(OpcodeField op, BOffImm16 off, Register rj, Register rd)
+ : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift | RJ(rj) |
+ RD(rd)) {}
+ InstImm(OpcodeField op, int32_t si21, Register rj, bool NotHasRd)
+ : Instruction(NotHasRd ? op | (si21 & Imm16Mask) << RKShift | RJ(rj) |
+ (si21 & Imm21Mask) >> 16
+ : op | (si21 & Imm20Mask) << Imm20Shift | RD(rj)) {
+ if (NotHasRd) {
+ MOZ_ASSERT(op == op_beqz || op == op_bnez);
+ MOZ_ASSERT(is_intN(si21, 21));
+ } else {
+ MOZ_ASSERT(op == op_lu12i_w || op == op_lu32i_d || op == op_pcaddi ||
+ op == op_pcaddu12i || op == op_pcaddu18i ||
+ op == op_pcalau12i);
+ // si20
+ MOZ_ASSERT(is_intN(si21, 20) || is_uintN(si21, 20));
+ }
+ }
+ InstImm(OpcodeField op, int32_t si21, AssemblerLOONG64::FPConditionBit cj,
+ bool isNotEqual)
+ : Instruction(isNotEqual
+ ? op | (si21 & Imm16Mask) << RKShift |
+ (cj + 8) << CJShift | (si21 & Imm21Mask) >> 16
+ : op | (si21 & Imm16Mask) << RKShift | cj << CJShift |
+ (si21 & Imm21Mask) >> 16) {
+ MOZ_ASSERT(is_intN(si21, 21));
+ MOZ_ASSERT(op == op_bcz);
+ MOZ_ASSERT(cj >= 0 && cj <= 7);
+ }
+ InstImm(OpcodeField op, Imm16 off, Register rj, Register rd)
+ : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift | RJ(rj) |
+ RD(rd)) {}
+ InstImm(OpcodeField op, int32_t bit15)
+ : Instruction(op | (bit15 & Imm15Mask)) {
+ MOZ_ASSERT(is_uintN(bit15, 15));
+ }
+
+ InstImm(OpcodeField op, int32_t bit26, bool jump)
+ : Instruction(op | (bit26 & Imm16Mask) << Imm16Shift |
+ (bit26 & Imm26Mask) >> 16) {
+ MOZ_ASSERT(is_intN(bit26, 26));
+ }
+ InstImm(OpcodeField op, int32_t si12, Register rj, int32_t hint)
+ : Instruction(op | (si12 & Imm12Mask) << Imm12Shift | RJ(rj) |
+ (hint & RDMask)) {
+ MOZ_ASSERT(op == op_preld);
+ }
+ InstImm(OpcodeField op, int32_t msb, int32_t lsb, Register rj, Register rd,
+ uint32_t sb_bits)
+ : Instruction((sb_bits == 5)
+ ? op | (msb & MSBWMask) << MSBWShift |
+ (lsb & LSBWMask) << LSBWShift | RJ(rj) | RD(rd)
+ : op | (msb & MSBDMask) << MSBDShift |
+ (lsb & LSBDMask) << LSBDShift | RJ(rj) | RD(rd)) {
+ MOZ_ASSERT(sb_bits == 5 || sb_bits == 6);
+ MOZ_ASSERT(op == op_bstr_w || op == op_bstrins_d || op == op_bstrpick_d);
+ }
+ InstImm(OpcodeField op, int32_t msb, int32_t lsb, Register rj, Register rd)
+ : Instruction(op | (msb & MSBWMask) << MSBWShift |
+ ((lsb + 0x20) & LSBDMask) << LSBWShift | RJ(rj) | RD(rd)) {
+ MOZ_ASSERT(op == op_bstr_w);
+ }
+
+ // For floating-point loads and stores.
+ InstImm(OpcodeField op, int32_t si12, Register rj, FloatRegister fd)
+ : Instruction(op | (si12 & Imm12Mask) << Imm12Shift | RJ(rj) | FD(fd)) {
+ MOZ_ASSERT(is_intN(si12, 12));
+ }
+
+ void setOpcode(OpcodeField op, uint32_t opBits) {
+ // opBits not greater than 24.
+ MOZ_ASSERT(opBits < 25);
+ uint32_t OpcodeShift = 32 - opBits;
+ uint32_t OpcodeMask = ((1 << opBits) - 1) << OpcodeShift;
+ data = (data & ~OpcodeMask) | op;
+ }
+ uint32_t extractRK() {
+ return extractBitField(RKShift + RKBits - 1, RKShift);
+ }
+ uint32_t extractRJ() {
+ return extractBitField(RJShift + RJBits - 1, RJShift);
+ }
+ void setRJ(uint32_t rj) { data = (data & ~RJMask) | (rj << RJShift); }
+ uint32_t extractRD() {
+ return extractBitField(RDShift + RDBits - 1, RDShift);
+ }
+ uint32_t extractImm16Value() {
+ return extractBitField(Imm16Shift + Imm16Bits - 1, Imm16Shift);
+ }
+ void setBOffImm16(BOffImm16 off) {
+ // Reset immediate field and replace it
+ data = (data & ~BOffImm16Mask) | (off.encode() << Imm16Shift);
+ }
+ void setImm21(int32_t off) {
+ // Reset immediate field and replace it
+ uint32_t low16 = (off >> 2) & Imm16Mask;
+ int32_t high5 = (off >> 18) & Imm5Mask;
+ uint32_t fcc_info = (data >> 5) & 0x1F;
+ data = (data & ~BOffImm26Mask) | (low16 << Imm16Shift) | high5 |
+ (fcc_info << 5);
+ }
+};
+
+// Class for Jump type instructions.
+class InstJump : public Instruction {
+ public:
+ InstJump(OpcodeField op, JOffImm26 off)
+ : Instruction(op | (off.encode() & Imm16Mask) << Imm16Shift |
+ (off.encode() & Imm26Mask) >> 16) {
+ MOZ_ASSERT(op == op_b || op == op_bl);
+ }
+
+ void setJOffImm26(JOffImm26 off) {
+ // Reset immediate field and replace it
+ data = (data & ~BOffImm26Mask) |
+ ((off.encode() & Imm16Mask) << Imm16Shift) |
+ ((off.encode() >> 16) & 0x3ff);
+ }
+ uint32_t extractImm26Value() {
+ return extractBitField(Imm26Shift + Imm26Bits - 1, Imm26Shift);
+ }
+};
+
+class ABIArgGenerator {
+ public:
+ ABIArgGenerator()
+ : intRegIndex_(0), floatRegIndex_(0), stackOffset_(0), current_() {}
+
+ ABIArg next(MIRType argType);
+ ABIArg& current() { return current_; }
+ uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+ void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+
+ protected:
+ unsigned intRegIndex_;
+ unsigned floatRegIndex_;
+ uint32_t stackOffset_;
+ ABIArg current_;
+};
+
+class Assembler : public AssemblerLOONG64 {
+ public:
+ Assembler() : AssemblerLOONG64() {}
+
+ static uintptr_t GetPointer(uint8_t*);
+
+ using AssemblerLOONG64::bind;
+
+ static void Bind(uint8_t* rawCode, const CodeLabel& label);
+
+ void processCodeLabels(uint8_t* rawCode);
+
+ static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+ static void TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ void bind(InstImm* inst, uintptr_t branch, uintptr_t target);
+
+ // Copy the assembly code to the given buffer, and perform any pending
+ // relocations relying on the target address.
+ void executableCopy(uint8_t* buffer);
+
+ static uint32_t PatchWrite_NearCallSize();
+
+ static uint64_t ExtractLoad64Value(Instruction* inst0);
+ static void UpdateLoad64Value(Instruction* inst0, uint64_t value);
+ static void WriteLoad64Instructions(Instruction* inst0, Register reg,
+ uint64_t value);
+
+ static void PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall);
+ static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
+ ImmPtr expectedValue);
+ static void PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue);
+
+ static uint64_t ExtractInstructionImmediate(uint8_t* code);
+
+ static void ToggleCall(CodeLocationLabel inst_, bool enabled);
+}; // Assembler
+
+static const uint32_t NumIntArgRegs = 8;
+static const uint32_t NumFloatArgRegs = 8;
+
+static inline bool GetIntArgReg(uint32_t usedIntArgs, Register* out) {
+ if (usedIntArgs < NumIntArgRegs) {
+ *out = Register::FromCode(a0.code() + usedIntArgs);
+ return true;
+ }
+ return false;
+}
+
+static inline bool GetFloatArgReg(uint32_t usedFloatArgs, FloatRegister* out) {
+ if (usedFloatArgs < NumFloatArgRegs) {
+ *out = FloatRegister::FromCode(f0.code() + usedFloatArgs);
+ return true;
+ }
+ return false;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs, Register* out) {
+ // NOTE: We can't properly determine which regs are used if there are
+ // float arguments. If this is needed, we will have to guess.
+ MOZ_ASSERT(usedFloatArgs == 0);
+
+ if (GetIntArgReg(usedIntArgs, out)) {
+ return true;
+ }
+ // Unfortunately, we have to assume things about the point at which
+ // GetIntArgReg returns false, because we need to know how many registers it
+ // can allocate.
+ usedIntArgs -= NumIntArgRegs;
+ if (usedIntArgs >= NumCallTempNonArgRegs) {
+ return false;
+ }
+ *out = CallTempNonArgRegs[usedIntArgs];
+ return true;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_Assembler_loong64_h */
diff --git a/js/src/jit/loong64/CodeGenerator-loong64.cpp b/js/src/jit/loong64/CodeGenerator-loong64.cpp
new file mode 100644
index 0000000000..7fc8534e7b
--- /dev/null
+++ b/js/src/jit/loong64/CodeGenerator-loong64.cpp
@@ -0,0 +1,2765 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/loong64/CodeGenerator-loong64.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jsnum.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitRuntime.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+#include "vm/Shape.h"
+
+#include "jit/shared/CodeGenerator-shared-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::GenericNaN;
+using mozilla::FloorLog2;
+using mozilla::NegativeInfinity;
+
+// shared
+CodeGeneratorLOONG64::CodeGeneratorLOONG64(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorShared(gen, graph, masm) {}
+
+Operand CodeGeneratorLOONG64::ToOperand(const LAllocation& a) {
+ if (a.isGeneralReg()) {
+ return Operand(a.toGeneralReg()->reg());
+ }
+ if (a.isFloatReg()) {
+ return Operand(a.toFloatReg()->reg());
+ }
+ return Operand(ToAddress(a));
+}
+
+Operand CodeGeneratorLOONG64::ToOperand(const LAllocation* a) {
+ return ToOperand(*a);
+}
+
+Operand CodeGeneratorLOONG64::ToOperand(const LDefinition* def) {
+ return ToOperand(def->output());
+}
+
+#ifdef JS_PUNBOX64
+Operand CodeGeneratorLOONG64::ToOperandOrRegister64(
+ const LInt64Allocation input) {
+ return ToOperand(input.value());
+}
+#else
+Register64 CodeGeneratorLOONG64::ToOperandOrRegister64(
+ const LInt64Allocation input) {
+ return ToRegister64(input);
+}
+#endif
+
+void CodeGeneratorLOONG64::branchToBlock(Assembler::FloatFormat fmt,
+ FloatRegister lhs, FloatRegister rhs,
+ MBasicBlock* mir,
+ Assembler::DoubleCondition cond) {
+ // Skip past trivial blocks.
+ Label* label = skipTrivialBlocks(mir)->lir()->label();
+ if (fmt == Assembler::DoubleFloat) {
+ masm.branchDouble(cond, lhs, rhs, label);
+ } else {
+ masm.branchFloat(cond, lhs, rhs, label);
+ }
+}
+
+void OutOfLineBailout::accept(CodeGeneratorLOONG64* codegen) {
+ codegen->visitOutOfLineBailout(this);
+}
+
+MoveOperand CodeGeneratorLOONG64::toMoveOperand(LAllocation a) const {
+ if (a.isGeneralReg()) {
+ return MoveOperand(ToRegister(a));
+ }
+ if (a.isFloatReg()) {
+ return MoveOperand(ToFloatRegister(a));
+ }
+ MoveOperand::Kind kind =
+ a.isStackArea() ? MoveOperand::EFFECTIVE_ADDRESS : MoveOperand::MEMORY;
+ Address address = ToAddress(a);
+ MOZ_ASSERT((address.offset & 3) == 0);
+
+ return MoveOperand(address, kind);
+}
+
+void CodeGeneratorLOONG64::bailoutFrom(Label* label, LSnapshot* snapshot) {
+ MOZ_ASSERT_IF(!masm.oom(), label->used());
+ MOZ_ASSERT_IF(!masm.oom(), !label->bound());
+
+ encode(snapshot);
+
+ InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
+ OutOfLineBailout* ool = new (alloc()) OutOfLineBailout(snapshot);
+ addOutOfLineCode(ool,
+ new (alloc()) BytecodeSite(tree, tree->script()->code()));
+
+ masm.retarget(label, ool->entry());
+}
+
+void CodeGeneratorLOONG64::bailout(LSnapshot* snapshot) {
+ Label label;
+ masm.jump(&label);
+ bailoutFrom(&label, snapshot);
+}
+
+bool CodeGeneratorLOONG64::generateOutOfLineCode() {
+ if (!CodeGeneratorShared::generateOutOfLineCode()) {
+ return false;
+ }
+
+ if (deoptLabel_.used()) {
+ // All non-table-based bailouts will go here.
+ masm.bind(&deoptLabel_);
+
+ // Push the frame size, so the handler can recover the IonScript.
+ // Frame size is stored in 'ra' and pushed by GenerateBailoutThunk
+ // We have to use 'ra' because generateBailoutTable will implicitly do
+ // the same.
+ masm.move32(Imm32(frameSize()), ra);
+
+ TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
+ masm.jump(handler);
+ }
+
+ return !masm.oom();
+}
+
+class js::jit::OutOfLineTableSwitch
+ : public OutOfLineCodeBase<CodeGeneratorLOONG64> {
+ MTableSwitch* mir_;
+ CodeLabel jumpLabel_;
+
+ void accept(CodeGeneratorLOONG64* codegen) {
+ codegen->visitOutOfLineTableSwitch(this);
+ }
+
+ public:
+ OutOfLineTableSwitch(MTableSwitch* mir) : mir_(mir) {}
+
+ MTableSwitch* mir() const { return mir_; }
+
+ CodeLabel* jumpLabel() { return &jumpLabel_; }
+};
+
+void CodeGeneratorLOONG64::emitTableSwitchDispatch(MTableSwitch* mir,
+ Register index,
+ Register base) {
+ Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
+
+ // Lower value with low value
+ if (mir->low() != 0) {
+ masm.subPtr(Imm32(mir->low()), index);
+ }
+
+ // Jump to default case if input is out of range
+ int32_t cases = mir->numCases();
+ masm.branchPtr(Assembler::AboveOrEqual, index, ImmWord(cases), defaultcase);
+
+ // To fill in the CodeLabels for the case entries, we need to first
+ // generate the case entries (we don't yet know their offsets in the
+ // instruction stream).
+ OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(mir);
+ addOutOfLineCode(ool, mir);
+
+ // Compute the position where a pointer to the right case stands.
+ masm.ma_li(base, ool->jumpLabel());
+
+ BaseIndex pointer(base, index, ScalePointer);
+
+ // Jump to the right case
+ masm.branchToComputedAddress(pointer);
+}
+
+void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) {
+ MOZ_ASSERT(ins->instance()->isBogus());
+ masm.movePtr(HeapReg, ToRegister(ins->output()));
+}
+
+template <typename T>
+void CodeGeneratorLOONG64::emitWasmLoad(T* lir) {
+ const MWasmLoad* mir = lir->mir();
+ SecondScratchRegisterScope scratch2(masm);
+
+ Register ptr = ToRegister(lir->ptr());
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (mir->base()->type() == MIRType::Int32) {
+ masm.move32To64ZeroExtend(ptr, Register64(scratch2));
+ ptr = scratch2;
+ ptrScratch = ptrScratch != InvalidReg ? scratch2 : InvalidReg;
+ }
+
+ // ptr is a GPR and is either a 32-bit value zero-extended to 64-bit, or a
+ // true 64-bit value.
+ masm.wasmLoad(mir->access(), HeapReg, ptr, ptrScratch,
+ ToAnyRegister(lir->output()));
+}
+
+template <typename T>
+void CodeGeneratorLOONG64::emitWasmStore(T* lir) {
+ const MWasmStore* mir = lir->mir();
+ SecondScratchRegisterScope scratch2(masm);
+
+ Register ptr = ToRegister(lir->ptr());
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (mir->base()->type() == MIRType::Int32) {
+ masm.move32To64ZeroExtend(ptr, Register64(scratch2));
+ ptr = scratch2;
+ ptrScratch = ptrScratch != InvalidReg ? scratch2 : InvalidReg;
+ }
+
+ // ptr is a GPR and is either a 32-bit value zero-extended to 64-bit, or a
+ // true 64-bit value.
+ masm.wasmStore(mir->access(), ToAnyRegister(lir->value()), HeapReg, ptr,
+ ptrScratch);
+}
+
+void CodeGeneratorLOONG64::generateInvalidateEpilogue() {
+ // Ensure that there is enough space in the buffer for the OsiPoint
+ // patching to occur. Otherwise, we could overwrite the invalidation
+ // epilogue
+ for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) {
+ masm.nop();
+ }
+
+ masm.bind(&invalidate_);
+
+ // Push the return address of the point that we bailed out at to the stack
+ masm.Push(ra);
+
+ // Push the Ion script onto the stack (when we determine what that
+ // pointer is).
+ invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
+
+ // Jump to the invalidator which will replace the current frame.
+ TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
+
+ masm.jump(thunk);
+}
+
+void CodeGeneratorLOONG64::visitOutOfLineBailout(OutOfLineBailout* ool) {
+ // Push snapshotOffset and make sure stack is aligned.
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.storePtr(ImmWord(ool->snapshot()->snapshotOffset()),
+ Address(StackPointer, 0));
+
+ masm.jump(&deoptLabel_);
+}
+
+void CodeGeneratorLOONG64::visitOutOfLineTableSwitch(
+ OutOfLineTableSwitch* ool) {
+ MTableSwitch* mir = ool->mir();
+
+ masm.haltingAlign(sizeof(void*));
+ masm.bind(ool->jumpLabel());
+ masm.addCodeLabel(*ool->jumpLabel());
+
+ for (size_t i = 0; i < mir->numCases(); i++) {
+ LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir();
+ Label* caseheader = caseblock->label();
+ uint32_t caseoffset = caseheader->offset();
+
+ // The entries of the jump table need to be absolute addresses and thus
+ // must be patched after codegen is finished.
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ cl.target()->bind(caseoffset);
+ masm.addCodeLabel(cl);
+ }
+}
+
+void CodeGeneratorLOONG64::visitOutOfLineWasmTruncateCheck(
+ OutOfLineWasmTruncateCheck* ool) {
+ if (ool->toType() == MIRType::Int32) {
+ masm.outOfLineWasmTruncateToInt32Check(
+ ool->input(), ool->output(), ool->fromType(), ool->flags(),
+ ool->rejoin(), ool->bytecodeOffset());
+ } else {
+ MOZ_ASSERT(ool->toType() == MIRType::Int64);
+ masm.outOfLineWasmTruncateToInt64Check(
+ ool->input(), ool->output64(), ool->fromType(), ool->flags(),
+ ool->rejoin(), ool->bytecodeOffset());
+ }
+}
+
+ValueOperand CodeGeneratorLOONG64::ToValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getOperand(pos)));
+}
+
+ValueOperand CodeGeneratorLOONG64::ToTempValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getTemp(pos)));
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+ const LAllocation* in = box->getOperand(0);
+ ValueOperand result = ToOutValue(box);
+
+ masm.moveValue(TypedOrValueRegister(box->type(), ToAnyRegister(in)), result);
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+ MUnbox* mir = unbox->mir();
+
+ Register result = ToRegister(unbox->output());
+
+ if (mir->fallible()) {
+ const ValueOperand value = ToValue(unbox, LUnbox::Input);
+ Label bail;
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.fallibleUnboxInt32(value, result, &bail);
+ break;
+ case MIRType::Boolean:
+ masm.fallibleUnboxBoolean(value, result, &bail);
+ break;
+ case MIRType::Object:
+ masm.fallibleUnboxObject(value, result, &bail);
+ break;
+ case MIRType::String:
+ masm.fallibleUnboxString(value, result, &bail);
+ break;
+ case MIRType::Symbol:
+ masm.fallibleUnboxSymbol(value, result, &bail);
+ break;
+ case MIRType::BigInt:
+ masm.fallibleUnboxBigInt(value, result, &bail);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ bailoutFrom(&bail, unbox->snapshot());
+ return;
+ }
+
+ LAllocation* input = unbox->getOperand(LUnbox::Input);
+ if (input->isRegister()) {
+ Register inputReg = ToRegister(input);
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(inputReg, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(inputReg, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(inputReg, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(inputReg, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(inputReg, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(inputReg, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ return;
+ }
+
+ Address inputAddr = ToAddress(input);
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(inputAddr, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(inputAddr, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(inputAddr, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(inputAddr, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(inputAddr, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(inputAddr, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+}
+
+void CodeGeneratorLOONG64::splitTagForTest(const ValueOperand& value,
+ ScratchTagScope& tag) {
+ masm.splitTag(value.valueReg(), tag);
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+ MCompare* mir = lir->mir();
+ const mozilla::DebugOnly<MCompare::CompareType> type = mir->compareType();
+ MOZ_ASSERT(type == MCompare::Compare_Int64 ||
+ type == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ Register output = ToRegister(lir->output());
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned);
+
+ if (IsConstant(rhs)) {
+ masm.cmpPtrSet(cond, lhsReg, ImmWord(ToInt64(rhs)), output);
+ } else if (rhs.value().isGeneralReg()) {
+ masm.cmpPtrSet(cond, lhsReg, ToRegister64(rhs).reg, output);
+ } else {
+ masm.cmpPtrSet(cond, lhsReg, ToAddress(rhs.value()), output);
+ }
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+ MCompare* mir = lir->cmpMir();
+ const mozilla::DebugOnly<MCompare::CompareType> type = mir->compareType();
+ MOZ_ASSERT(type == MCompare::Compare_Int64 ||
+ type == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned);
+
+ if (IsConstant(rhs)) {
+ emitBranch(lhsReg, ImmWord(ToInt64(rhs)), cond, lir->ifTrue(),
+ lir->ifFalse());
+ } else if (rhs.value().isGeneralReg()) {
+ emitBranch(lhsReg, ToRegister64(rhs).reg, cond, lir->ifTrue(),
+ lir->ifFalse());
+ } else {
+ emitBranch(lhsReg, ToAddress(rhs.value()), cond, lir->ifTrue(),
+ lir->ifFalse());
+ }
+}
+
+void CodeGenerator::visitCompare(LCompare* comp) {
+ MCompare* mir = comp->mir();
+ Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop());
+ const LAllocation* left = comp->getOperand(0);
+ const LAllocation* right = comp->getOperand(1);
+ const LDefinition* def = comp->getDef(0);
+
+ if (mir->compareType() == MCompare::Compare_Object ||
+ mir->compareType() == MCompare::Compare_Symbol ||
+ mir->compareType() == MCompare::Compare_UIntPtr ||
+ mir->compareType() == MCompare::Compare_RefOrNull) {
+ if (right->isConstant()) {
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_UIntPtr);
+ masm.cmpPtrSet(cond, ToRegister(left), Imm32(ToInt32(right)),
+ ToRegister(def));
+ } else if (right->isGeneralReg()) {
+ masm.cmpPtrSet(cond, ToRegister(left), ToRegister(right),
+ ToRegister(def));
+ } else {
+ masm.cmpPtrSet(cond, ToRegister(left), ToAddress(right), ToRegister(def));
+ }
+ return;
+ }
+
+ if (right->isConstant()) {
+ masm.cmp32Set(cond, ToRegister(left), Imm32(ToInt32(right)),
+ ToRegister(def));
+ } else if (right->isGeneralReg()) {
+ masm.cmp32Set(cond, ToRegister(left), ToRegister(right), ToRegister(def));
+ } else {
+ masm.cmp32Set(cond, ToRegister(left), ToAddress(right), ToRegister(def));
+ }
+}
+
+void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
+ const MCompare* mir = comp->cmpMir();
+ const MCompare::CompareType type = mir->compareType();
+ const LAllocation* lhs = comp->left();
+ const LAllocation* rhs = comp->right();
+ MBasicBlock* ifTrue = comp->ifTrue();
+ MBasicBlock* ifFalse = comp->ifFalse();
+ Register lhsReg = ToRegister(lhs);
+ const Assembler::Condition cond = JSOpToCondition(type, comp->jsop());
+
+ if (type == MCompare::Compare_Object || type == MCompare::Compare_Symbol ||
+ type == MCompare::Compare_UIntPtr ||
+ type == MCompare::Compare_RefOrNull) {
+ if (rhs->isConstant()) {
+ emitBranch(ToRegister(lhs), Imm32(ToInt32(rhs)), cond, ifTrue, ifFalse);
+ } else if (rhs->isGeneralReg()) {
+ emitBranch(lhsReg, ToRegister(rhs), cond, ifTrue, ifFalse);
+ } else {
+ MOZ_CRASH("NYI");
+ }
+ return;
+ }
+
+ if (rhs->isConstant()) {
+ emitBranch(lhsReg, Imm32(ToInt32(comp->right())), cond, ifTrue, ifFalse);
+ } else if (comp->right()->isGeneralReg()) {
+ emitBranch(lhsReg, ToRegister(rhs), cond, ifTrue, ifFalse);
+ } else {
+ // TODO(loong64): emitBranch with 32-bit comparision
+ ScratchRegisterScope scratch(masm);
+ masm.load32(ToAddress(rhs), scratch);
+ emitBranch(lhsReg, Register(scratch), cond, ifTrue, ifFalse);
+ }
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ Label done;
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ // Handle an integer overflow exception from INT64_MIN / -1.
+ if (lir->canBeNegativeOverflow()) {
+ Label notOverflow;
+ masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &notOverflow);
+ masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &notOverflow);
+ if (lir->mir()->isMod()) {
+ masm.as_xor(output, output, output);
+ } else {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ masm.jump(&done);
+ masm.bind(&notOverflow);
+ }
+
+ if (lir->mir()->isMod()) {
+ masm.as_mod_d(output, lhs, rhs);
+ } else {
+ masm.as_div_d(output, lhs, rhs);
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ Label done;
+
+ // Prevent divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ if (lir->mir()->isMod()) {
+ masm.as_mod_du(output, lhs, rhs);
+ } else {
+ masm.as_div_du(output, lhs, rhs);
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGeneratorLOONG64::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+ masm.as_div_d(/* result= */ dividend, dividend, divisor);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorLOONG64::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+ masm.as_mod_d(/* result= */ dividend, dividend, divisor);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir) {
+ const MWasmLoad* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ masm.wasmLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToOutRegister64(lir));
+}
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir) {
+ const MWasmStore* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ masm.wasmStoreI64(mir->access(), ToRegister64(lir->value()), HeapReg,
+ ToRegister(lir->ptr()), ptrScratch);
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+
+ Register cond = ToRegister(lir->condExpr());
+ const LInt64Allocation falseExpr = lir->falseExpr();
+
+ Register64 out = ToOutRegister64(lir);
+ MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
+ "true expr is reused for input");
+
+ if (falseExpr.value().isRegister()) {
+ masm.moveIfZero(out.reg, ToRegister(falseExpr.value()), cond);
+ } else {
+ Label done;
+ masm.ma_b(cond, cond, &done, Assembler::NonZero, ShortJump);
+ masm.loadPtr(ToAddress(falseExpr.value()), out.reg);
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+ masm.as_movgr2fr_d(ToFloatRegister(lir->output()), ToRegister(lir->input()));
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+ masm.as_movfr2gr_d(ToRegister(lir->output()), ToFloatRegister(lir->input()));
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->isUnsigned()) {
+ masm.as_bstrpick_d(output, ToRegister(input), 31, 0);
+ } else {
+ masm.as_slli_w(output, ToRegister(input), 0);
+ }
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ if (input->isMemory()) {
+ masm.load32(ToAddress(input), output);
+ } else {
+ masm.as_slli_w(output, ToRegister(input), 0);
+ }
+ } else {
+ MOZ_CRASH("Not implemented.");
+ }
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ switch (lir->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.move32To64SignExtend(input.reg, output);
+ masm.move8SignExtend(output.reg, output.reg);
+ break;
+ case MSignExtendInt64::Half:
+ masm.move32To64SignExtend(input.reg, output);
+ masm.move16SignExtend(output.reg, output.reg);
+ break;
+ case MSignExtendInt64::Word:
+ masm.move32To64SignExtend(input.reg, output);
+ break;
+ }
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(input == output);
+ masm.move32To64ZeroExtend(input, Register64(output));
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(input == output);
+ masm.move64To32(Register64(input), output);
+}
+
+void CodeGenerator::visitClzI64(LClzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.clz64(input, output.reg);
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.ctz64(input, output.reg);
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register output = ToRegister(lir->output());
+
+ masm.ma_cmp_set(output, input.reg, zero, Assembler::Equal);
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ MWasmTruncateToInt64* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ Label* oolRejoin = ool->rejoin();
+ bool isSaturating = mir->isSaturating();
+
+ if (fromType == MIRType::Double) {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ } else {
+ masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ }
+ } else {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ } else {
+ masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ }
+ }
+}
+
+void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ MIRType outputType = lir->mir()->type();
+ MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
+
+ if (outputType == MIRType::Double) {
+ if (lir->mir()->isUnsigned()) {
+ masm.convertUInt64ToDouble(input, output, Register::Invalid());
+ } else {
+ masm.convertInt64ToDouble(input, output);
+ }
+ } else {
+ if (lir->mir()->isUnsigned()) {
+ masm.convertUInt64ToFloat32(input, output, Register::Invalid());
+ } else {
+ masm.convertInt64ToFloat32(input, output);
+ }
+ }
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ MBasicBlock* ifTrue = lir->ifTrue();
+ MBasicBlock* ifFalse = lir->ifFalse();
+
+ emitBranch(input.reg, Imm32(0), Assembler::NonZero, ifTrue, ifFalse);
+}
+
+void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
+ const LAllocation* opd = test->getOperand(0);
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ emitBranch(ToRegister(opd), Imm32(0), Assembler::NonZero, ifTrue, ifFalse);
+}
+
+void CodeGenerator::visitMinMaxD(LMinMaxD* ins) {
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+
+ MOZ_ASSERT(first == ToFloatRegister(ins->output()));
+
+ if (ins->mir()->isMax()) {
+ masm.maxDouble(second, first, true);
+ } else {
+ masm.minDouble(second, first, true);
+ }
+}
+
+void CodeGenerator::visitMinMaxF(LMinMaxF* ins) {
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+
+ MOZ_ASSERT(first == ToFloatRegister(ins->output()));
+
+ if (ins->mir()->isMax()) {
+ masm.maxFloat32(second, first, true);
+ } else {
+ masm.minFloat32(second, first, true);
+ }
+}
+
+void CodeGenerator::visitAddI(LAddI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ MOZ_ASSERT(rhs->isConstant() || rhs->isGeneralReg());
+
+ // If there is no snapshot, we don't need to check for overflow
+ if (!ins->snapshot()) {
+ if (rhs->isConstant()) {
+ masm.ma_add_w(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ } else {
+ masm.as_add_w(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ }
+ return;
+ }
+
+ Label overflow;
+ if (rhs->isConstant()) {
+ masm.ma_add32TestOverflow(ToRegister(dest), ToRegister(lhs),
+ Imm32(ToInt32(rhs)), &overflow);
+ } else {
+ masm.ma_add32TestOverflow(ToRegister(dest), ToRegister(lhs),
+ ToRegister(rhs), &overflow);
+ }
+
+ bailoutFrom(&overflow, ins->snapshot());
+}
+
+void CodeGenerator::visitAddI64(LAddI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void CodeGenerator::visitSubI(LSubI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ MOZ_ASSERT(rhs->isConstant() || rhs->isGeneralReg());
+
+ // If there is no snapshot, we don't need to check for overflow
+
+ if (!ins->snapshot()) {
+ if (rhs->isConstant()) {
+ masm.ma_sub_w(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ } else {
+ masm.as_sub_w(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ }
+ return;
+ }
+
+ Label overflow;
+ if (rhs->isConstant()) {
+ masm.ma_sub32TestOverflow(ToRegister(dest), ToRegister(lhs),
+ Imm32(ToInt32(rhs)), &overflow);
+ } else {
+ masm.ma_sub32TestOverflow(ToRegister(dest), ToRegister(lhs),
+ ToRegister(rhs), &overflow);
+ }
+
+ bailoutFrom(&overflow, ins->snapshot());
+}
+
+void CodeGenerator::visitSubI64(LSubI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void CodeGenerator::visitMulI(LMulI* ins) {
+ const LAllocation* lhs = ins->lhs();
+ const LAllocation* rhs = ins->rhs();
+ Register dest = ToRegister(ins->output());
+ MMul* mul = ins->mir();
+
+ MOZ_ASSERT_IF(mul->mode() == MMul::Integer,
+ !mul->canBeNegativeZero() && !mul->canOverflow());
+
+ if (rhs->isConstant()) {
+ int32_t constant = ToInt32(rhs);
+ Register src = ToRegister(lhs);
+
+ // Bailout on -0.0
+ if (mul->canBeNegativeZero() && constant <= 0) {
+ Assembler::Condition cond =
+ (constant == 0) ? Assembler::LessThan : Assembler::Equal;
+ bailoutCmp32(cond, src, Imm32(0), ins->snapshot());
+ }
+
+ switch (constant) {
+ case -1:
+ if (mul->canOverflow()) {
+ bailoutCmp32(Assembler::Equal, src, Imm32(INT32_MIN),
+ ins->snapshot());
+ }
+
+ masm.as_sub_w(dest, zero, src);
+ break;
+ case 0:
+ masm.move32(zero, dest);
+ break;
+ case 1:
+ masm.move32(src, dest);
+ break;
+ case 2:
+ if (mul->canOverflow()) {
+ Label mulTwoOverflow;
+ masm.ma_add32TestOverflow(dest, src, src, &mulTwoOverflow);
+
+ bailoutFrom(&mulTwoOverflow, ins->snapshot());
+ } else {
+ masm.as_add_w(dest, src, src);
+ }
+ break;
+ default:
+ uint32_t shift = FloorLog2(constant);
+
+ if (!mul->canOverflow() && (constant > 0)) {
+ // If it cannot overflow, we can do lots of optimizations.
+ uint32_t rest = constant - (1 << shift);
+
+ // See if the constant has one bit set, meaning it can be
+ // encoded as a bitshift.
+ if ((1 << shift) == constant) {
+ masm.as_slli_w(dest, src, shift % 32);
+ return;
+ }
+
+ // If the constant cannot be encoded as (1<<C1), see if it can
+ // be encoded as (1<<C1) | (1<<C2), which can be computed
+ // using an add and a shift.
+ uint32_t shift_rest = FloorLog2(rest);
+ if (src != dest && (1u << shift_rest) == rest) {
+ masm.as_slli_w(dest, src, (shift - shift_rest) % 32);
+ masm.add32(src, dest);
+ if (shift_rest != 0) {
+ masm.as_slli_w(dest, dest, shift_rest % 32);
+ }
+ return;
+ }
+ }
+
+ if (mul->canOverflow() && (constant > 0) && (src != dest)) {
+ // To stay on the safe side, only optimize things that are a
+ // power of 2.
+
+ if ((1 << shift) == constant) {
+ ScratchRegisterScope scratch(masm);
+ // dest = lhs * pow(2, shift)
+ masm.as_slli_w(dest, src, shift % 32);
+ // At runtime, check (lhs == dest >> shift), if this does
+ // not hold, some bits were lost due to overflow, and the
+ // computation should be resumed as a double.
+ masm.as_srai_w(scratch, dest, shift % 32);
+ bailoutCmp32(Assembler::NotEqual, src, Register(scratch),
+ ins->snapshot());
+ return;
+ }
+ }
+
+ if (mul->canOverflow()) {
+ Label mulConstOverflow;
+ masm.ma_mul32TestOverflow(dest, ToRegister(lhs), Imm32(ToInt32(rhs)),
+ &mulConstOverflow);
+
+ bailoutFrom(&mulConstOverflow, ins->snapshot());
+ } else {
+ masm.ma_mul(dest, src, Imm32(ToInt32(rhs)));
+ }
+ break;
+ }
+ } else {
+ Label multRegOverflow;
+
+ if (mul->canOverflow()) {
+ masm.ma_mul32TestOverflow(dest, ToRegister(lhs), ToRegister(rhs),
+ &multRegOverflow);
+ bailoutFrom(&multRegOverflow, ins->snapshot());
+ } else {
+ masm.as_mul_w(dest, ToRegister(lhs), ToRegister(rhs));
+ }
+
+ if (mul->canBeNegativeZero()) {
+ Label done;
+ masm.ma_b(dest, dest, &done, Assembler::NonZero, ShortJump);
+
+ // Result is -0 if lhs or rhs is negative.
+ // In that case result must be double value so bailout
+ Register scratch = SecondScratchReg;
+ masm.as_or(scratch, ToRegister(lhs), ToRegister(rhs));
+ bailoutCmp32(Assembler::Signed, scratch, scratch, ins->snapshot());
+
+ masm.bind(&done);
+ }
+ }
+}
+
+void CodeGenerator::visitMulI64(LMulI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
+ const Register64 output = ToOutRegister64(lir);
+
+ if (IsConstant(rhs)) {
+ int64_t constant = ToInt64(rhs);
+ switch (constant) {
+ case -1:
+ masm.neg64(ToRegister64(lhs));
+ return;
+ case 0:
+ masm.xor64(ToRegister64(lhs), ToRegister64(lhs));
+ return;
+ case 1:
+ // nop
+ return;
+ default:
+ if (constant > 0) {
+ if (mozilla::IsPowerOfTwo(static_cast<uint32_t>(constant + 1))) {
+ masm.move64(ToRegister64(lhs), output);
+ masm.lshift64(Imm32(FloorLog2(constant + 1)), output);
+ masm.sub64(ToRegister64(lhs), output);
+ return;
+ } else if (mozilla::IsPowerOfTwo(
+ static_cast<uint32_t>(constant - 1))) {
+ masm.move64(ToRegister64(lhs), output);
+ masm.lshift64(Imm32(FloorLog2(constant - 1u)), output);
+ masm.add64(ToRegister64(lhs), output);
+ return;
+ }
+ // Use shift if constant is power of 2.
+ int32_t shift = mozilla::FloorLog2(constant);
+ if (int64_t(1) << shift == constant) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ return;
+ }
+ }
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
+ }
+ } else {
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp);
+ }
+}
+
+void CodeGenerator::visitDivI(LDivI* ins) {
+ // Extract the registers from this instruction
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register dest = ToRegister(ins->output());
+ Register temp = ToRegister(ins->getTemp(0));
+ MDiv* mir = ins->mir();
+
+ Label done;
+
+ // Handle divide by zero.
+ if (mir->canBeDivideByZero()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else if (mir->canTruncateInfinities()) {
+ // Truncated division by zero is zero (Infinity|0 == 0)
+ Label notzero;
+ masm.ma_b(rhs, rhs, &notzero, Assembler::NonZero, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&notzero);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Zero, rhs, rhs, ins->snapshot());
+ }
+ }
+
+ // Handle an integer overflow exception from -2147483648 / -1.
+ if (mir->canBeNegativeOverflow()) {
+ Label notMinInt;
+ masm.move32(Imm32(INT32_MIN), temp);
+ masm.ma_b(lhs, temp, &notMinInt, Assembler::NotEqual, ShortJump);
+
+ masm.move32(Imm32(-1), temp);
+ if (mir->trapOnError()) {
+ Label ok;
+ masm.ma_b(rhs, temp, &ok, Assembler::NotEqual);
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
+ masm.bind(&ok);
+ } else if (mir->canTruncateOverflow()) {
+ // (-INT32_MIN)|0 == INT32_MIN
+ Label skip;
+ masm.ma_b(rhs, temp, &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(INT32_MIN), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, rhs, temp, ins->snapshot());
+ }
+ masm.bind(&notMinInt);
+ }
+
+ // Handle negative 0. (0/-Y)
+ if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
+ Label nonzero;
+ masm.ma_b(lhs, lhs, &nonzero, Assembler::NonZero, ShortJump);
+ bailoutCmp32(Assembler::LessThan, rhs, Imm32(0), ins->snapshot());
+ masm.bind(&nonzero);
+ }
+ // Note: above safety checks could not be verified as Ion seems to be
+ // smarter and requires double arithmetic in such cases.
+
+ // All regular. Lets call div.
+ if (mir->canTruncateRemainder()) {
+ masm.as_div_w(dest, lhs, rhs);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+
+ Label remainderNonZero;
+ masm.ma_div_branch_overflow(dest, lhs, rhs, &remainderNonZero);
+ bailoutFrom(&remainderNonZero, ins->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) {
+ Register lhs = ToRegister(ins->numerator());
+ Register dest = ToRegister(ins->output());
+ Register tmp = ToRegister(ins->getTemp(0));
+ int32_t shift = ins->shift();
+
+ if (shift != 0) {
+ MDiv* mir = ins->mir();
+ if (!mir->isTruncated()) {
+ // If the remainder is going to be != 0, bailout since this must
+ // be a double.
+ masm.as_slli_w(tmp, lhs, (32 - shift) % 32);
+ bailoutCmp32(Assembler::NonZero, tmp, tmp, ins->snapshot());
+ }
+
+ if (!mir->canBeNegativeDividend()) {
+ // Numerator is unsigned, so needs no adjusting. Do the shift.
+ masm.as_srai_w(dest, lhs, shift % 32);
+ return;
+ }
+
+ // Adjust the value so that shifting produces a correctly rounded result
+ // when the numerator is negative. See 10-1 "Signed Division by a Known
+ // Power of 2" in Henry S. Warren, Jr.'s Hacker's Delight.
+ if (shift > 1) {
+ masm.as_srai_w(tmp, lhs, 31);
+ masm.as_srli_w(tmp, tmp, (32 - shift) % 32);
+ masm.add32(lhs, tmp);
+ } else {
+ masm.as_srli_w(tmp, lhs, (32 - shift) % 32);
+ masm.add32(lhs, tmp);
+ }
+
+ // Do the shift.
+ masm.as_srai_w(dest, tmp, shift % 32);
+ } else {
+ masm.move32(lhs, dest);
+ }
+}
+
+void CodeGenerator::visitModI(LModI* ins) {
+ // Extract the registers from this instruction
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register dest = ToRegister(ins->output());
+ Register callTemp = ToRegister(ins->callTemp());
+ MMod* mir = ins->mir();
+ Label done, prevent;
+
+ masm.move32(lhs, callTemp);
+
+ // Prevent INT_MIN % -1;
+ // The integer division will give INT_MIN, but we want -(double)INT_MIN.
+ if (mir->canBeNegativeDividend()) {
+ masm.ma_b(lhs, Imm32(INT_MIN), &prevent, Assembler::NotEqual, ShortJump);
+ if (mir->isTruncated()) {
+ // (INT_MIN % -1)|0 == 0
+ Label skip;
+ masm.ma_b(rhs, Imm32(-1), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(-1), ins->snapshot());
+ }
+ masm.bind(&prevent);
+ }
+
+ // 0/X (with X < 0) is bad because both of these values *should* be
+ // doubles, and the result should be -0.0, which cannot be represented in
+ // integers. X/0 is bad because it will give garbage (or abort), when it
+ // should give either \infty, -\infty or NAN.
+
+ // Prevent 0 / X (with X < 0) and X / 0
+ // testing X / Y. Compare Y with 0.
+ // There are three cases: (Y < 0), (Y == 0) and (Y > 0)
+ // If (Y < 0), then we compare X with 0, and bail if X == 0
+ // If (Y == 0), then we simply want to bail.
+ // if (Y > 0), we don't bail.
+
+ if (mir->canBeDivideByZero()) {
+ if (mir->isTruncated()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ Label skip;
+ masm.ma_b(rhs, Imm32(0), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ }
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot());
+ }
+ }
+
+ if (mir->canBeNegativeDividend()) {
+ Label notNegative;
+ masm.ma_b(rhs, Imm32(0), &notNegative, Assembler::GreaterThan, ShortJump);
+ if (mir->isTruncated()) {
+ // NaN|0 == 0 and (0 % -X)|0 == 0
+ Label skip;
+ masm.ma_b(lhs, Imm32(0), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, lhs, Imm32(0), ins->snapshot());
+ }
+ masm.bind(&notNegative);
+ }
+
+ masm.as_mod_w(dest, lhs, rhs);
+
+ // If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0
+ if (mir->canBeNegativeDividend()) {
+ if (mir->isTruncated()) {
+ // -0.0|0 == 0
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ // See if X < 0
+ masm.ma_b(dest, Imm32(0), &done, Assembler::NotEqual, ShortJump);
+ bailoutCmp32(Assembler::Signed, callTemp, Imm32(0), ins->snapshot());
+ }
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) {
+ Register in = ToRegister(ins->getOperand(0));
+ Register out = ToRegister(ins->getDef(0));
+ MMod* mir = ins->mir();
+ Label negative, done;
+
+ masm.move32(in, out);
+ masm.ma_b(in, in, &done, Assembler::Zero, ShortJump);
+ // Switch based on sign of the lhs.
+ // Positive numbers are just a bitmask
+ masm.ma_b(in, in, &negative, Assembler::Signed, ShortJump);
+ {
+ masm.and32(Imm32((1 << ins->shift()) - 1), out);
+ masm.ma_b(&done, ShortJump);
+ }
+
+ // Negative numbers need a negate, bitmask, negate
+ {
+ masm.bind(&negative);
+ masm.neg32(out);
+ masm.and32(Imm32((1 << ins->shift()) - 1), out);
+ masm.neg32(out);
+ }
+ if (mir->canBeNegativeDividend()) {
+ if (!mir->isTruncated()) {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, out, zero, ins->snapshot());
+ } else {
+ // -0|0 == 0
+ }
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitModMaskI(LModMaskI* ins) {
+ Register src = ToRegister(ins->getOperand(0));
+ Register dest = ToRegister(ins->getDef(0));
+ Register tmp0 = ToRegister(ins->getTemp(0));
+ Register tmp1 = ToRegister(ins->getTemp(1));
+ MMod* mir = ins->mir();
+
+ if (!mir->isTruncated() && mir->canBeNegativeDividend()) {
+ MOZ_ASSERT(mir->fallible());
+
+ Label bail;
+ masm.ma_mod_mask(src, dest, tmp0, tmp1, ins->shift(), &bail);
+ bailoutFrom(&bail, ins->snapshot());
+ } else {
+ masm.ma_mod_mask(src, dest, tmp0, tmp1, ins->shift(), nullptr);
+ }
+}
+
+void CodeGenerator::visitBitNotI(LBitNotI* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ const LDefinition* dest = ins->getDef(0);
+ MOZ_ASSERT(!input->isConstant());
+
+ masm.as_nor(ToRegister(dest), ToRegister(input), zero);
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ MOZ_ASSERT(!input->isConstant());
+ Register inputReg = ToRegister(input);
+ MOZ_ASSERT(inputReg == ToRegister(ins->output()));
+ masm.as_nor(inputReg, inputReg, zero);
+}
+
+void CodeGenerator::visitBitOpI(LBitOpI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+ // all of these bitops should be either imm32's, or integer registers.
+ switch (ins->bitop()) {
+ case JSOp::BitOr:
+ if (rhs->isConstant()) {
+ masm.ma_or(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)),
+ true);
+ } else {
+ masm.as_or(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ masm.as_slli_w(ToRegister(dest), ToRegister(dest), 0);
+ }
+ break;
+ case JSOp::BitXor:
+ if (rhs->isConstant()) {
+ masm.ma_xor(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)),
+ true);
+ } else {
+ masm.as_xor(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ masm.as_slli_w(ToRegister(dest), ToRegister(dest), 0);
+ }
+ break;
+ case JSOp::BitAnd:
+ if (rhs->isConstant()) {
+ masm.ma_and(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)),
+ true);
+ } else {
+ masm.as_and(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ masm.as_slli_w(ToRegister(dest), ToRegister(dest), 0);
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitBitOpI64(LBitOpI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LBitOpI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LBitOpI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ switch (lir->bitop()) {
+ case JSOp::BitOr:
+ if (IsConstant(rhs)) {
+ masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.or64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::BitXor:
+ if (IsConstant(rhs)) {
+ masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.xor64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::BitAnd:
+ if (IsConstant(rhs)) {
+ masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.and64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitShiftI(LShiftI* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ const LAllocation* rhs = ins->rhs();
+ Register dest = ToRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ if (shift) {
+ masm.as_slli_w(dest, lhs, shift % 32);
+ } else {
+ masm.move32(lhs, dest);
+ }
+ break;
+ case JSOp::Rsh:
+ if (shift) {
+ masm.as_srai_w(dest, lhs, shift % 32);
+ } else {
+ masm.move32(lhs, dest);
+ }
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.as_srli_w(dest, lhs, shift % 32);
+ } else {
+ // x >>> 0 can overflow.
+ if (ins->mir()->toUrsh()->fallible()) {
+ bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot());
+ }
+ masm.move32(lhs, dest);
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ } else {
+ // The shift amounts should be AND'ed into the 0-31 range
+ masm.ma_and(dest, ToRegister(rhs), Imm32(0x1F));
+
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ masm.as_sll_w(dest, lhs, dest);
+ break;
+ case JSOp::Rsh:
+ masm.as_sra_w(dest, lhs, dest);
+ break;
+ case JSOp::Ursh:
+ masm.as_srl_w(dest, lhs, dest);
+ if (ins->mir()->toUrsh()->fallible()) {
+ // x >>> 0 can overflow.
+ bailoutCmp32(Assembler::LessThan, dest, Imm32(0), ins->snapshot());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ }
+}
+
+void CodeGenerator::visitShiftI64(LShiftI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LShiftI64::Lhs);
+ LAllocation* rhs = lir->getOperand(LShiftI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (rhs->isConstant()) {
+ int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ if (shift) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::Rsh:
+ if (shift) {
+ masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.rshift64(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ return;
+ }
+
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ masm.lshift64(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::Rsh:
+ masm.rshift64Arithmetic(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::Ursh:
+ masm.rshift64(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+}
+
+void CodeGenerator::visitRotateI64(LRotateI64* lir) {
+ MRotate* mir = lir->mir();
+ LAllocation* count = lir->count();
+
+ Register64 input = ToRegister64(lir->input());
+ Register64 output = ToOutRegister64(lir);
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+
+ MOZ_ASSERT(input == output);
+
+ if (count->isConstant()) {
+ int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F);
+ if (!c) {
+ return;
+ }
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft64(Imm32(c), input, output, temp);
+ } else {
+ masm.rotateRight64(Imm32(c), input, output, temp);
+ }
+ } else {
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft64(ToRegister(count), input, output, temp);
+ } else {
+ masm.rotateRight64(ToRegister(count), input, output, temp);
+ }
+ }
+}
+
+void CodeGenerator::visitUrshD(LUrshD* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register temp = ToRegister(ins->temp());
+
+ const LAllocation* rhs = ins->rhs();
+ FloatRegister out = ToFloatRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ masm.as_srli_w(temp, lhs, ToInt32(rhs) % 32);
+ } else {
+ masm.as_srl_w(temp, lhs, ToRegister(rhs));
+ }
+
+ masm.convertUInt32ToDouble(temp, out);
+}
+
+void CodeGenerator::visitClzI(LClzI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.as_clz_w(output, input);
+}
+
+void CodeGenerator::visitCtzI(LCtzI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.as_ctz_w(output, input);
+}
+
+void CodeGenerator::visitPopcntI(LPopcntI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+ Register tmp = ToRegister(ins->temp0());
+
+ masm.popcnt32(input, output, tmp);
+}
+
+void CodeGenerator::visitPopcntI64(LPopcntI64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ Register64 output = ToOutRegister64(ins);
+ Register tmp = ToRegister(ins->getTemp(0));
+
+ masm.popcnt64(input, output, tmp);
+}
+
+void CodeGenerator::visitPowHalfD(LPowHalfD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+ ScratchDoubleScope fpscratch(masm);
+
+ Label done, skip;
+
+ // Masm.pow(-Infinity, 0.5) == Infinity.
+ masm.loadConstantDouble(NegativeInfinity<double>(), fpscratch);
+ masm.ma_bc_d(input, fpscratch, &skip, Assembler::DoubleNotEqualOrUnordered,
+ ShortJump);
+ masm.as_fneg_d(output, fpscratch);
+ masm.ma_b(&done, ShortJump);
+
+ masm.bind(&skip);
+ // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
+ // Adding 0 converts any -0 to 0.
+ masm.loadConstantDouble(0.0, fpscratch);
+ masm.as_fadd_d(output, input, fpscratch);
+ masm.as_fsqrt_d(output, output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitMathD(LMathD* math) {
+ FloatRegister src1 = ToFloatRegister(math->getOperand(0));
+ FloatRegister src2 = ToFloatRegister(math->getOperand(1));
+ FloatRegister output = ToFloatRegister(math->getDef(0));
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.as_fadd_d(output, src1, src2);
+ break;
+ case JSOp::Sub:
+ masm.as_fsub_d(output, src1, src2);
+ break;
+ case JSOp::Mul:
+ masm.as_fmul_d(output, src1, src2);
+ break;
+ case JSOp::Div:
+ masm.as_fdiv_d(output, src1, src2);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitMathF(LMathF* math) {
+ FloatRegister src1 = ToFloatRegister(math->getOperand(0));
+ FloatRegister src2 = ToFloatRegister(math->getOperand(1));
+ FloatRegister output = ToFloatRegister(math->getDef(0));
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.as_fadd_s(output, src1, src2);
+ break;
+ case JSOp::Sub:
+ masm.as_fsub_s(output, src1, src2);
+ break;
+ case JSOp::Mul:
+ masm.as_fmul_s(output, src1, src2);
+ break;
+ case JSOp::Div:
+ masm.as_fdiv_s(output, src1, src2);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+ emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+ emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* lir) {
+ emitTruncateDouble(ToFloatRegister(lir->getOperand(0)),
+ ToRegister(lir->getDef(0)), lir->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+ LWasmBuiltinTruncateFToInt32* lir) {
+ emitTruncateFloat32(ToFloatRegister(lir->getOperand(0)),
+ ToRegister(lir->getDef(0)), lir->mir());
+}
+
+void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) {
+ auto input = ToFloatRegister(lir->input());
+ auto output = ToRegister(lir->output());
+
+ MWasmTruncateToInt32* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ if (mir->isUnsigned()) {
+ if (fromType == MIRType::Double) {
+ masm.wasmTruncateDoubleToUInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else if (fromType == MIRType::Float32) {
+ masm.wasmTruncateFloat32ToUInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+
+ masm.bind(ool->rejoin());
+ return;
+ }
+
+ if (fromType == MIRType::Double) {
+ masm.wasmTruncateDoubleToInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else if (fromType == MIRType::Float32) {
+ masm.wasmTruncateFloat32ToInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitCopySignF(LCopySignF* ins) {
+ FloatRegister lhs = ToFloatRegister(ins->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(ins->getOperand(1));
+ FloatRegister output = ToFloatRegister(ins->getDef(0));
+
+ Register lhsi = ToRegister(ins->getTemp(0));
+ Register rhsi = ToRegister(ins->getTemp(1));
+
+ masm.moveFromFloat32(lhs, lhsi);
+ masm.moveFromFloat32(rhs, rhsi);
+
+ // Combine.
+ masm.as_bstrins_w(rhsi, lhsi, 30, 0);
+
+ masm.moveToFloat32(rhsi, output);
+}
+
+void CodeGenerator::visitCopySignD(LCopySignD* ins) {
+ FloatRegister lhs = ToFloatRegister(ins->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(ins->getOperand(1));
+ FloatRegister output = ToFloatRegister(ins->getDef(0));
+
+ Register lhsi = ToRegister(ins->getTemp(0));
+ Register rhsi = ToRegister(ins->getTemp(1));
+
+ // Manipulate high words of double inputs.
+ masm.moveFromDoubleHi(lhs, lhsi);
+ masm.moveFromDoubleHi(rhs, rhsi);
+
+ // Combine.
+ masm.as_bstrins_w(rhsi, lhsi, 30, 0);
+
+ masm.moveToDoubleHi(rhsi, output);
+}
+
+void CodeGenerator::visitValue(LValue* value) {
+ const ValueOperand out = ToOutValue(value);
+
+ masm.moveValue(value->value(), out);
+}
+
+void CodeGenerator::visitDouble(LDouble* ins) {
+ const LDefinition* out = ins->getDef(0);
+
+ masm.loadConstantDouble(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitFloat32(LFloat32* ins) {
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantFloat32(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) {
+ FloatRegister input = ToFloatRegister(test->input());
+ ScratchDoubleScope fpscratch(masm);
+
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ masm.loadConstantDouble(0.0, fpscratch);
+ // If 0, or NaN, the result is false.
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::DoubleFloat, input, fpscratch, ifTrue,
+ Assembler::DoubleNotEqual);
+ } else {
+ branchToBlock(Assembler::DoubleFloat, input, fpscratch, ifFalse,
+ Assembler::DoubleEqualOrUnordered);
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) {
+ FloatRegister input = ToFloatRegister(test->input());
+ ScratchFloat32Scope fpscratch(masm);
+
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ masm.loadConstantFloat32(0.0f, fpscratch);
+ // If 0, or NaN, the result is false.
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::SingleFloat, input, fpscratch, ifTrue,
+ Assembler::DoubleNotEqual);
+ } else {
+ branchToBlock(Assembler::SingleFloat, input, fpscratch, ifFalse,
+ Assembler::DoubleEqualOrUnordered);
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitCompareD(LCompareD* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+ Register dest = ToRegister(comp->output());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+ masm.ma_cmp_set_double(dest, lhs, rhs, cond);
+}
+
+void CodeGenerator::visitCompareF(LCompareF* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+ Register dest = ToRegister(comp->output());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+ masm.ma_cmp_set_float32(dest, lhs, rhs, cond);
+}
+
+void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ MBasicBlock* ifTrue = comp->ifTrue();
+ MBasicBlock* ifFalse = comp->ifFalse();
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::DoubleFloat, lhs, rhs, ifTrue, cond);
+ } else {
+ branchToBlock(Assembler::DoubleFloat, lhs, rhs, ifFalse,
+ Assembler::InvertCondition(cond));
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ MBasicBlock* ifTrue = comp->ifTrue();
+ MBasicBlock* ifFalse = comp->ifFalse();
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::SingleFloat, lhs, rhs, ifTrue, cond);
+ } else {
+ branchToBlock(Assembler::SingleFloat, lhs, rhs, ifFalse,
+ Assembler::InvertCondition(cond));
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* lir) {
+ ScratchRegisterScope scratch(masm);
+ if (lir->right()->isConstant()) {
+ masm.ma_and(scratch, ToRegister(lir->left()), Imm32(ToInt32(lir->right())));
+ } else {
+ masm.as_and(scratch, ToRegister(lir->left()), ToRegister(lir->right()));
+ }
+ emitBranch(scratch, Register(scratch), lir->cond(), lir->ifTrue(),
+ lir->ifFalse());
+}
+
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+ masm.convertUInt32ToDouble(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+ masm.convertUInt32ToFloat32(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitNotI(LNotI* ins) {
+ masm.cmp32Set(Assembler::Equal, ToRegister(ins->input()), Imm32(0),
+ ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitNotD(LNotD* ins) {
+ // Since this operation is not, we want to set a bit if
+ // the double is falsey, which means 0.0, -0.0 or NaN.
+ FloatRegister in = ToFloatRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+ ScratchDoubleScope fpscratch(masm);
+
+ masm.loadConstantDouble(0.0, fpscratch);
+ masm.ma_cmp_set_double(dest, in, fpscratch,
+ Assembler::DoubleEqualOrUnordered);
+}
+
+void CodeGenerator::visitNotF(LNotF* ins) {
+ // Since this operation is not, we want to set a bit if
+ // the float32 is falsey, which means 0.0, -0.0 or NaN.
+ FloatRegister in = ToFloatRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+ ScratchFloat32Scope fpscratch(masm);
+
+ masm.loadConstantFloat32(0.0f, fpscratch);
+ masm.ma_cmp_set_float32(dest, in, fpscratch,
+ Assembler::DoubleEqualOrUnordered);
+}
+
+void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) {
+ masm.memoryBarrier(ins->type());
+}
+
+void CodeGenerator::visitWasmLoad(LWasmLoad* lir) { emitWasmLoad(lir); }
+
+void CodeGenerator::visitWasmStore(LWasmStore* lir) { emitWasmStore(lir); }
+
+void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) {
+ const MAsmJSLoadHeap* mir = ins->mir();
+ MOZ_ASSERT(!mir->hasMemoryBase());
+
+ const LAllocation* ptr = ins->ptr();
+ const LDefinition* output = ins->output();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ Register ptrReg = ToRegister(ptr);
+ Scalar::Type accessType = mir->accessType();
+ bool isFloat = accessType == Scalar::Float32 || accessType == Scalar::Float64;
+ Label done;
+
+ if (mir->needsBoundsCheck()) {
+ Label boundsCheckPassed;
+ Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
+ masm.wasmBoundsCheck32(Assembler::Below, ptrReg, boundsCheckLimitReg,
+ &boundsCheckPassed);
+ // Return a default value in case of a bounds-check failure.
+ if (isFloat) {
+ if (accessType == Scalar::Float32) {
+ masm.loadConstantFloat32(GenericNaN(), ToFloatRegister(output));
+ } else {
+ masm.loadConstantDouble(GenericNaN(), ToFloatRegister(output));
+ }
+ } else {
+ masm.mov(zero, ToRegister(output));
+ }
+ masm.jump(&done);
+ masm.bind(&boundsCheckPassed);
+ }
+
+ // TODO(loong64): zero-extend index in asm.js?
+ SecondScratchRegisterScope scratch2(masm);
+ masm.move32To64ZeroExtend(ptrReg, Register64(scratch2));
+
+ switch (accessType) {
+ case Scalar::Int8:
+ masm.as_ldx_b(ToRegister(output), HeapReg, scratch2);
+ break;
+ case Scalar::Uint8:
+ masm.as_ldx_bu(ToRegister(output), HeapReg, scratch2);
+ break;
+ case Scalar::Int16:
+ masm.as_ldx_h(ToRegister(output), HeapReg, scratch2);
+ break;
+ case Scalar::Uint16:
+ masm.as_ldx_hu(ToRegister(output), HeapReg, scratch2);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ masm.as_ldx_w(ToRegister(output), HeapReg, scratch2);
+ break;
+ case Scalar::Float64:
+ masm.as_fldx_d(ToFloatRegister(output), HeapReg, scratch2);
+ break;
+ case Scalar::Float32:
+ masm.as_fldx_s(ToFloatRegister(output), HeapReg, scratch2);
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ if (done.used()) {
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) {
+ const MAsmJSStoreHeap* mir = ins->mir();
+ MOZ_ASSERT(!mir->hasMemoryBase());
+
+ const LAllocation* value = ins->value();
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ Register ptrReg = ToRegister(ptr);
+
+ Label done;
+ if (mir->needsBoundsCheck()) {
+ Register boundsCheckLimitReg = ToRegister(boundsCheckLimit);
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg, boundsCheckLimitReg,
+ &done);
+ }
+
+ // TODO(loong64): zero-extend index in asm.js?
+ SecondScratchRegisterScope scratch2(masm);
+ masm.move32To64ZeroExtend(ptrReg, Register64(scratch2));
+
+ switch (mir->accessType()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ masm.as_stx_b(ToRegister(value), HeapReg, scratch2);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ masm.as_stx_h(ToRegister(value), HeapReg, scratch2);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ masm.as_stx_w(ToRegister(value), HeapReg, scratch2);
+ break;
+ case Scalar::Float64:
+ masm.as_fstx_d(ToFloatRegister(value), HeapReg, scratch2);
+ break;
+ case Scalar::Float32:
+ masm.as_fstx_s(ToFloatRegister(value), HeapReg, scratch2);
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ if (done.used()) {
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitWasmCompareExchangeHeap(
+ LWasmCompareExchangeHeap* ins) {
+ MWasmCompareExchangeHeap* mir = ins->mir();
+ Register ptrReg = ToRegister(ins->ptr());
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Register oldval = ToRegister(ins->oldValue());
+ Register newval = ToRegister(ins->newValue());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+ MWasmAtomicExchangeHeap* mir = ins->mir();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ masm.wasmAtomicExchange(mir->access(), srcAddr, value, valueTemp, offsetTemp,
+ maskTemp, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+ MOZ_ASSERT(ins->mir()->hasUses());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+
+ masm.wasmAtomicFetchOp(mir->access(), mir->operation(),
+ ToRegister(ins->value()), srcAddr, valueTemp,
+ offsetTemp, maskTemp, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+ LWasmAtomicBinopHeapForEffect* ins) {
+ MOZ_ASSERT(!ins->mir()->hasUses());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ masm.wasmAtomicEffectOp(mir->access(), mir->operation(),
+ ToRegister(ins->value()), srcAddr, valueTemp,
+ offsetTemp, maskTemp);
+}
+
+void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ if (ins->arg()->isConstant()) {
+ masm.storePtr(ImmWord(ToInt32(ins->arg())),
+ Address(StackPointer, mir->spOffset()));
+ } else {
+ if (ins->arg()->isGeneralReg()) {
+ masm.storePtr(ToRegister(ins->arg()),
+ Address(StackPointer, mir->spOffset()));
+ } else if (mir->input()->type() == MIRType::Double) {
+ masm.storeDouble(ToFloatRegister(ins->arg()),
+ Address(StackPointer, mir->spOffset()));
+ } else {
+ masm.storeFloat32(ToFloatRegister(ins->arg()),
+ Address(StackPointer, mir->spOffset()));
+ }
+ }
+}
+
+void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(StackPointer, mir->spOffset());
+ if (IsConstant(ins->arg())) {
+ masm.store64(Imm64(ToInt64(ins->arg())), dst);
+ } else {
+ masm.store64(ToRegister64(ins->arg()), dst);
+ }
+}
+
+void CodeGenerator::visitWasmSelect(LWasmSelect* ins) {
+ MIRType mirType = ins->mir()->type();
+
+ Register cond = ToRegister(ins->condExpr());
+ const LAllocation* falseExpr = ins->falseExpr();
+
+ if (mirType == MIRType::Int32 || mirType == MIRType::RefOrNull) {
+ Register out = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->trueExpr()) == out,
+ "true expr input is reused for output");
+ if (falseExpr->isRegister()) {
+ masm.moveIfZero(out, ToRegister(falseExpr), cond);
+ } else {
+ masm.cmp32Load32(Assembler::Zero, cond, cond, ToAddress(falseExpr), out);
+ }
+ return;
+ }
+
+ FloatRegister out = ToFloatRegister(ins->output());
+ MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out,
+ "true expr input is reused for output");
+
+ if (falseExpr->isFloatReg()) {
+ if (mirType == MIRType::Float32) {
+ masm.ma_fmovz(Assembler::SingleFloat, out, ToFloatRegister(falseExpr),
+ cond);
+ } else if (mirType == MIRType::Double) {
+ masm.ma_fmovz(Assembler::DoubleFloat, out, ToFloatRegister(falseExpr),
+ cond);
+ } else {
+ MOZ_CRASH("unhandled type in visitWasmSelect!");
+ }
+ } else {
+ Label done;
+ masm.ma_b(cond, cond, &done, Assembler::NonZero, ShortJump);
+
+ if (mirType == MIRType::Float32) {
+ masm.loadFloat32(ToAddress(falseExpr), out);
+ } else if (mirType == MIRType::Double) {
+ masm.loadDouble(ToAddress(falseExpr), out);
+ } else {
+ MOZ_CRASH("unhandled type in visitWasmSelect!");
+ }
+
+ masm.bind(&done);
+ }
+}
+
+// We expect to handle only the case where compare is {U,}Int32 and select is
+// {U,}Int32, and the "true" input is reused for the output.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
+ ins->compareType() == MCompare::Compare_UInt32;
+ bool selIs32bit = ins->mir()->type() == MIRType::Int32;
+
+ MOZ_RELEASE_ASSERT(
+ cmpIs32bit && selIs32bit,
+ "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
+
+ Register trueExprAndDest = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
+ "true expr input is reused for output");
+
+ Assembler::Condition cond = Assembler::InvertCondition(
+ JSOpToCondition(ins->compareType(), ins->jsop()));
+ const LAllocation* rhs = ins->rightExpr();
+ const LAllocation* falseExpr = ins->ifFalseExpr();
+ Register lhs = ToRegister(ins->leftExpr());
+
+ masm.cmp32Move32(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+}
+
+void CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MWasmReinterpret* ins = lir->mir();
+
+ MIRType to = ins->type();
+ mozilla::DebugOnly<MIRType> from = ins->input()->type();
+
+ switch (to) {
+ case MIRType::Int32:
+ MOZ_ASSERT(from == MIRType::Float32);
+ masm.as_movfr2gr_s(ToRegister(lir->output()),
+ ToFloatRegister(lir->input()));
+ break;
+ case MIRType::Float32:
+ MOZ_ASSERT(from == MIRType::Int32);
+ masm.as_movgr2fr_w(ToFloatRegister(lir->output()),
+ ToRegister(lir->input()));
+ break;
+ case MIRType::Double:
+ case MIRType::Int64:
+ MOZ_CRASH("not handled by this LIR opcode");
+ default:
+ MOZ_CRASH("unexpected WasmReinterpret");
+ }
+}
+
+void CodeGenerator::visitUDivOrMod(LUDivOrMod* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+ Label done;
+
+ // Prevent divide by zero.
+ if (ins->canBeDivideByZero()) {
+ if (ins->mir()->isTruncated()) {
+ if (ins->trapOnError()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ // Infinity|0 == 0
+ Label notzero;
+ masm.ma_b(rhs, rhs, &notzero, Assembler::NonZero, ShortJump);
+ masm.move32(Imm32(0), output);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&notzero);
+ }
+ } else {
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot());
+ }
+ }
+
+ masm.as_mod_wu(output, lhs, rhs);
+
+ // If the remainder is > 0, bailout since this must be a double.
+ if (ins->mir()->isDiv()) {
+ if (!ins->mir()->toDiv()->canTruncateRemainder()) {
+ bailoutCmp32(Assembler::NonZero, output, output, ins->snapshot());
+ }
+ // Get quotient
+ masm.as_div_wu(output, lhs, rhs);
+ }
+
+ if (!ins->mir()->isTruncated()) {
+ bailoutCmp32(Assembler::LessThan, output, Imm32(0), ins->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins) {
+ const MEffectiveAddress* mir = ins->mir();
+ Register base = ToRegister(ins->base());
+ Register index = ToRegister(ins->index());
+ Register output = ToRegister(ins->output());
+
+ BaseIndex address(base, index, mir->scale(), mir->displacement());
+ masm.computeEffectiveAddress(address, output);
+}
+
+void CodeGenerator::visitNegI(LNegI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.as_sub_w(output, zero, input);
+}
+
+void CodeGenerator::visitNegI64(LNegI64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ MOZ_ASSERT(input == ToOutRegister64(ins));
+ masm.neg64(input);
+}
+
+void CodeGenerator::visitNegD(LNegD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ masm.as_fneg_d(output, input);
+}
+
+void CodeGenerator::visitNegF(LNegF* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ masm.as_fneg_s(output, input);
+}
+
+void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register base = ToRegister(lir->base());
+ Register out = ToRegister(lir->output());
+
+ Label ok;
+ masm.ma_add32TestCarry(Assembler::CarryClear, out, base, Imm32(mir->offset()),
+ &ok);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
+ masm.bind(&ok);
+}
+
+void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register64 base = ToRegister64(lir->base());
+ Register64 out = ToOutRegister64(lir);
+
+ Label ok;
+ masm.ma_addPtrTestCarry(Assembler::CarryClear, out.reg, base.reg,
+ ImmWord(mir->offset()), &ok);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
+ masm.bind(&ok);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop(
+ LAtomicTypedArrayElementBinop* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register elements = ToRegister(lir->elements());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp2());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+ Register value = ToRegister(lir->value());
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, valueTemp,
+ offsetTemp, maskTemp, outTemp, output);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, valueTemp,
+ offsetTemp, maskTemp, outTemp, output);
+ }
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(
+ LAtomicTypedArrayElementBinopForEffect* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+ Register value = ToRegister(lir->value());
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, valueTemp,
+ offsetTemp, maskTemp);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, valueTemp,
+ offsetTemp, maskTemp);
+ }
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement(
+ LCompareExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp());
+
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, valueTemp, offsetTemp, maskTemp, outTemp,
+ output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, valueTemp, offsetTemp, maskTemp, outTemp,
+ output);
+ }
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement(
+ LAtomicExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp());
+
+ Register value = ToRegister(lir->value());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value,
+ valueTemp, offsetTemp, maskTemp, outTemp, output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value,
+ valueTemp, offsetTemp, maskTemp, outTemp, output);
+ }
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 tempOut(out);
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(oldval, temp1);
+ masm.loadBigInt64(newval, tempOut);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut,
+ temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = Register64(ToRegister(lir->temp2()));
+ Register out = ToRegister(lir->output());
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_ASSERT(lir->mir()->hasUses());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 tempOut = Register64(out);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ tempOut, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ tempOut, temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_ASSERT(!lir->mir()->hasUses());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ }
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ auto sync = Synchronization::Load();
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ }
+ masm.memoryBarrierAfter(sync);
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp1);
+ auto sync = Synchronization::Store();
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.store64(temp1, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.store64(temp1, dest);
+ }
+ masm.memoryBarrierAfter(sync);
+}
+
+void CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 oldValue = ToRegister64(lir->oldValue());
+ Register64 newValue = ToRegister64(lir->newValue());
+ Register64 output = ToOutRegister64(lir);
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+ masm.wasmCompareExchange64(lir->mir()->access(), addr, oldValue, newValue,
+ output);
+}
+
+void CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 output = ToOutRegister64(lir);
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+ masm.wasmAtomicExchange64(lir->mir()->access(), addr, value, output);
+}
+
+void CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 output = ToOutRegister64(lir);
+ Register64 temp(ToRegister(lir->getTemp(0)));
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+
+ masm.wasmAtomicFetchOp64(lir->mir()->access(), lir->mir()->operation(), value,
+ addr, temp, output);
+}
+
+void CodeGenerator::visitNearbyInt(LNearbyInt*) { MOZ_CRASH("NYI"); }
+
+void CodeGenerator::visitNearbyIntF(LNearbyIntF*) { MOZ_CRASH("NYI"); }
+
+void CodeGenerator::visitSimd128(LSimd128* ins) { MOZ_CRASH("No SIMD"); }
+
+void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmBinarySimd128WithConstant(
+ LWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmVariableShiftSimd128(
+ LWasmVariableShiftSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmConstantShiftSimd128(
+ LWasmConstantShiftSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmSignReplicationSimd128(
+ LWasmSignReplicationSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
+ LWasmReplaceInt64LaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceAndBranchSimd128(
+ LWasmReduceAndBranchSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceSimd128ToInt64(
+ LWasmReduceSimd128ToInt64* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
diff --git a/js/src/jit/loong64/CodeGenerator-loong64.h b/js/src/jit/loong64/CodeGenerator-loong64.h
new file mode 100644
index 0000000000..5c75f65cd2
--- /dev/null
+++ b/js/src/jit/loong64/CodeGenerator-loong64.h
@@ -0,0 +1,209 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_CodeGenerator_loong64_h
+#define jit_loong64_CodeGenerator_loong64_h
+
+#include "jit/loong64/Assembler-loong64.h"
+#include "jit/shared/CodeGenerator-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorLOONG64;
+class OutOfLineBailout;
+class OutOfLineTableSwitch;
+
+using OutOfLineWasmTruncateCheck =
+ OutOfLineWasmTruncateCheckBase<CodeGeneratorLOONG64>;
+
+class CodeGeneratorLOONG64 : public CodeGeneratorShared {
+ friend class MoveResolverLA;
+
+ protected:
+ CodeGeneratorLOONG64(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm);
+
+ NonAssertingLabel deoptLabel_;
+
+ Operand ToOperand(const LAllocation& a);
+ Operand ToOperand(const LAllocation* a);
+ Operand ToOperand(const LDefinition* def);
+
+#ifdef JS_PUNBOX64
+ Operand ToOperandOrRegister64(const LInt64Allocation input);
+#else
+ Register64 ToOperandOrRegister64(const LInt64Allocation input);
+#endif
+
+ MoveOperand toMoveOperand(LAllocation a) const;
+
+ template <typename T1, typename T2>
+ void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ Label bail;
+ masm.branch32(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutTest32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ Label bail;
+ masm.branchTest32(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ Label bail;
+ masm.branchPtr(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs,
+ LSnapshot* snapshot) {
+ // TODO(loong64) Didn't use branchTestPtr due to '-Wundefined-inline'.
+ MOZ_ASSERT(c == Assembler::Zero || c == Assembler::NonZero ||
+ c == Assembler::Signed || c == Assembler::NotSigned);
+ Label bail;
+ if (lhs == rhs) {
+ masm.ma_b(lhs, rhs, &bail, c);
+ } else {
+ ScratchRegisterScope scratch(masm);
+ masm.as_and(scratch, lhs, rhs);
+ masm.ma_b(scratch, scratch, &bail, c);
+ }
+ bailoutFrom(&bail, snapshot);
+ }
+ void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) {
+ Label bail;
+ ScratchRegisterScope scratch(masm);
+ masm.ma_and(scratch, reg, Imm32(0xFF));
+ masm.ma_b(scratch, scratch, &bail, Assembler::Zero);
+ bailoutFrom(&bail, snapshot);
+ }
+
+ void bailoutFrom(Label* label, LSnapshot* snapshot);
+ void bailout(LSnapshot* snapshot);
+
+ bool generateOutOfLineCode();
+
+ template <typename T>
+ void branchToBlock(Register lhs, T rhs, MBasicBlock* mir,
+ Assembler::Condition cond) {
+ masm.ma_b(lhs, rhs, skipTrivialBlocks(mir)->lir()->label(), cond);
+ }
+ void branchToBlock(Assembler::FloatFormat fmt, FloatRegister lhs,
+ FloatRegister rhs, MBasicBlock* mir,
+ Assembler::DoubleCondition cond);
+
+ // Emits a branch that directs control flow to the true block if |cond| is
+ // true, and the false block if |cond| is false.
+ template <typename T>
+ void emitBranch(Register lhs, T rhs, Assembler::Condition cond,
+ MBasicBlock* mirTrue, MBasicBlock* mirFalse) {
+ if (isNextBlock(mirFalse->lir())) {
+ branchToBlock(lhs, rhs, mirTrue, cond);
+ } else {
+ branchToBlock(lhs, rhs, mirFalse, Assembler::InvertCondition(cond));
+ jumpToBlock(mirTrue);
+ }
+ }
+ void testZeroEmitBranch(Assembler::Condition cond, Register reg,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ emitBranch(reg, Imm32(0), cond, ifTrue, ifFalse);
+ }
+
+ void emitTableSwitchDispatch(MTableSwitch* mir, Register index,
+ Register base);
+
+ template <typename T>
+ void emitWasmLoad(T* ins);
+ template <typename T>
+ void emitWasmStore(T* ins);
+
+ void generateInvalidateEpilogue();
+
+ // Generating a result.
+ template <typename S, typename T>
+ void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
+ const S& value, const T& mem,
+ Register flagTemp, Register outTemp,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, AnyRegister output);
+
+ // Generating no result.
+ template <typename S, typename T>
+ void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
+ const S& value, const T& mem,
+ Register flagTemp, Register valueTemp,
+ Register offsetTemp, Register maskTemp);
+
+ public:
+ // Out of line visitors.
+ void visitOutOfLineBailout(OutOfLineBailout* ool);
+ void visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool);
+ void visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool);
+
+ protected:
+ void testNullEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_NULL), cond, ifTrue, ifFalse);
+ }
+ void testUndefinedEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_UNDEFINED), cond, ifTrue,
+ ifFalse);
+ }
+ void testObjectEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_OBJECT), cond, ifTrue,
+ ifFalse);
+ }
+
+ void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+
+ template <typename T>
+ void emitWasmLoadI64(T* ins);
+ template <typename T>
+ void emitWasmStoreI64(T* ins);
+
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ // Functions for LTestVAndBranch.
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag);
+};
+
+typedef CodeGeneratorLOONG64 CodeGeneratorSpecific;
+
+// An out-of-line bailout thunk.
+class OutOfLineBailout : public OutOfLineCodeBase<CodeGeneratorLOONG64> {
+ protected:
+ LSnapshot* snapshot_;
+
+ public:
+ OutOfLineBailout(LSnapshot* snapshot) : snapshot_(snapshot) {}
+
+ void accept(CodeGeneratorLOONG64* codegen) override;
+
+ LSnapshot* snapshot() const { return snapshot_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_CodeGenerator_loong64_h */
diff --git a/js/src/jit/loong64/LIR-loong64.h b/js/src/jit/loong64/LIR-loong64.h
new file mode 100644
index 0000000000..20fde694a1
--- /dev/null
+++ b/js/src/jit/loong64/LIR-loong64.h
@@ -0,0 +1,399 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_LIR_loong64_h
+#define jit_loong64_LIR_loong64_h
+
+namespace js {
+namespace jit {
+
+class LUnbox : public LInstructionHelper<1, 1, 0> {
+ protected:
+ LUnbox(LNode::Opcode opcode, const LAllocation& input)
+ : LInstructionHelper(opcode) {
+ setOperand(0, input);
+ }
+
+ public:
+ LIR_HEADER(Unbox);
+
+ explicit LUnbox(const LAllocation& input) : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+
+ static const size_t Input = 0;
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+ const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LUnbox {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint);
+
+ LUnboxFloatingPoint(const LAllocation& input, MIRType type)
+ : LUnbox(classOpcode, input), type_(type) {}
+
+ MIRType type() const { return type_; }
+};
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToDouble)
+
+ explicit LWasmUint32ToDouble(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToFloat32)
+
+ explicit LWasmUint32ToFloat32(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+class LDivI : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivI);
+
+ LDivI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LDivPowTwoI : public LInstructionHelper<1, 1, 1> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(DivPowTwoI)
+
+ LDivPowTwoI(const LAllocation& lhs, int32_t shift, const LDefinition& temp)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+ int32_t shift() const { return shift_; }
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LModI : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(ModI);
+
+ LModI(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& callTemp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, callTemp);
+ }
+
+ const LDefinition* callTemp() { return getTemp(0); }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModPowTwoI : public LInstructionHelper<1, 1, 0> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModPowTwoI);
+
+ LModPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ }
+
+ int32_t shift() const { return shift_; }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModMaskI : public LInstructionHelper<1, 1, 2> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModMaskI);
+
+ LModMaskI(const LAllocation& lhs, const LDefinition& temp0,
+ const LDefinition& temp1, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ }
+
+ int32_t shift() const { return shift_; }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitch : public LInstructionHelper<0, 1, 2> {
+ public:
+ LIR_HEADER(TableSwitch);
+
+ LTableSwitch(const LAllocation& in, const LDefinition& inputCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setTemp(0, inputCopy);
+ setTemp(1, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+ const LAllocation* index() { return getOperand(0); }
+ const LDefinition* tempInt() { return getTemp(0); }
+ // This is added to share the same CodeGenerator prefixes.
+ const LDefinition* tempPointer() { return getTemp(1); }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitchV : public LInstructionHelper<0, BOX_PIECES, 3> {
+ public:
+ LIR_HEADER(TableSwitchV);
+
+ LTableSwitchV(const LBoxAllocation& input, const LDefinition& inputCopy,
+ const LDefinition& floatCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setBoxOperand(InputValue, input);
+ setTemp(0, inputCopy);
+ setTemp(1, floatCopy);
+ setTemp(2, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+
+ static const size_t InputValue = 0;
+
+ const LDefinition* tempInt() { return getTemp(0); }
+ const LDefinition* tempFloat() { return getTemp(1); }
+ const LDefinition* tempPointer() { return getTemp(2); }
+};
+
+class LMulI : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(MulI);
+
+ LMulI() : LBinaryMath(classOpcode) {}
+
+ MMul* mir() { return mir_->toMul(); }
+};
+
+class LUDivOrMod : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(UDivOrMod);
+
+ LUDivOrMod() : LBinaryMath(classOpcode) {}
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+
+ bool trapOnError() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->trapOnError();
+ }
+ return mir_->toDiv()->trapOnError();
+ }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LWasmCompareExchangeI64
+ : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES + INT64_PIECES,
+ 0> {
+ public:
+ LIR_HEADER(WasmCompareExchangeI64);
+
+ LWasmCompareExchangeI64(const LAllocation& ptr,
+ const LInt64Allocation& oldValue,
+ const LInt64Allocation& newValue)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, oldValue);
+ setInt64Operand(1 + INT64_PIECES, newValue);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation oldValue() { return getInt64Operand(1); }
+ const LInt64Allocation newValue() {
+ return getInt64Operand(1 + INT64_PIECES);
+ }
+ const MWasmCompareExchangeHeap* mir() const {
+ return mir_->toWasmCompareExchangeHeap();
+ }
+};
+
+class LWasmAtomicExchangeI64
+ : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(WasmAtomicExchangeI64);
+
+ LWasmAtomicExchangeI64(const LAllocation& ptr, const LInt64Allocation& value)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation value() { return getInt64Operand(1); }
+ const MWasmAtomicExchangeHeap* mir() const {
+ return mir_->toWasmAtomicExchangeHeap();
+ }
+};
+
+class LWasmAtomicBinopI64
+ : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES, 2> {
+ public:
+ LIR_HEADER(WasmAtomicBinopI64);
+
+ LWasmAtomicBinopI64(const LAllocation& ptr, const LInt64Allocation& value)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation value() { return getInt64Operand(1); }
+ const MWasmAtomicBinopHeap* mir() const {
+ return mir_->toWasmAtomicBinopHeap();
+ }
+};
+
+class LDivOrModI64 : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LUDivOrModI64 : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(UDivOrModI64);
+
+ LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+ const char* extraName() const {
+ return mir()->isTruncated() ? "Truncated" : nullptr;
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LWasmTruncateToInt64 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ explicit LWasmTruncateToInt64(const LAllocation& in)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ }
+
+ MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); }
+};
+
+class LInt64ToFloatingPoint : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(Int64ToFloatingPoint);
+
+ explicit LInt64ToFloatingPoint(const LInt64Allocation& in)
+ : LInstructionHelper(classOpcode) {
+ setInt64Operand(0, in);
+ }
+
+ MInt64ToFloatingPoint* mir() const { return mir_->toInt64ToFloatingPoint(); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_LIR_loong64_h */
diff --git a/js/src/jit/loong64/Lowering-loong64.cpp b/js/src/jit/loong64/Lowering-loong64.cpp
new file mode 100644
index 0000000000..525a37fcde
--- /dev/null
+++ b/js/src/jit/loong64/Lowering-loong64.cpp
@@ -0,0 +1,1087 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/loong64/Lowering-loong64.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/loong64/Assembler-loong64.h"
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::FloorLog2;
+
+LTableSwitch* LIRGeneratorLOONG64::newLTableSwitch(const LAllocation& in,
+ const LDefinition& inputCopy,
+ MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitch(in, inputCopy, temp(), tableswitch);
+}
+
+LTableSwitchV* LIRGeneratorLOONG64::newLTableSwitchV(
+ MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitchV(useBox(tableswitch->getOperand(0)), temp(),
+ tempDouble(), temp(), tableswitch);
+}
+
+void LIRGeneratorLOONG64::lowerForShift(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegisterOrConstant(rhs));
+ define(ins, mir);
+}
+
+template <size_t Temps>
+void LIRGeneratorLOONG64::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+
+ static_assert(LShiftI64::Rhs == INT64_PIECES,
+ "Assume Rhs is located at INT64_PIECES.");
+ static_assert(LRotateI64::Count == INT64_PIECES,
+ "Assume Count is located at INT64_PIECES.");
+
+ ins->setOperand(INT64_PIECES, useRegisterOrConstant(rhs));
+
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+template void LIRGeneratorLOONG64::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+template void LIRGeneratorLOONG64::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 1>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+// x = !y
+void LIRGeneratorLOONG64::lowerForALU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(0, useRegister(input));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+// z = x + y
+void LIRGeneratorLOONG64::lowerForALU(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegisterOrConstant(rhs));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+void LIRGeneratorLOONG64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+ MDefinition* input) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorLOONG64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorLOONG64::lowerForMulInt64(LMulI64* ins, MMul* mir,
+ MDefinition* lhs, MDefinition* rhs) {
+ bool needsTemp = false;
+ bool cannotAliasRhs = false;
+ bool reuseInput = true;
+
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES,
+ (willHaveDifferentLIRNodes(lhs, rhs) || cannotAliasRhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+
+ if (needsTemp) {
+ ins->setTemp(0, temp());
+ }
+ if (reuseInput) {
+ defineInt64ReuseInput(ins, mir, 0);
+ } else {
+ defineInt64(ins, mir);
+ }
+}
+
+void LIRGeneratorLOONG64::lowerForFPU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(0, useRegister(input));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template <size_t Temps>
+void LIRGeneratorLOONG64::lowerForFPU(LInstructionHelper<1, 2, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegister(rhs));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template void LIRGeneratorLOONG64::lowerForFPU(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir,
+ MDefinition* lhs,
+ MDefinition* rhs);
+template void LIRGeneratorLOONG64::lowerForFPU(LInstructionHelper<1, 2, 1>* ins,
+ MDefinition* mir,
+ MDefinition* lhs,
+ MDefinition* rhs);
+
+void LIRGeneratorLOONG64::lowerForCompareI64AndBranch(
+ MTest* mir, MCompare* comp, JSOp op, MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ LCompareI64AndBranch* lir = new (alloc())
+ LCompareI64AndBranch(comp, op, useInt64Register(left),
+ useInt64OrConstant(right), ifTrue, ifFalse);
+ add(lir, mir);
+}
+
+void LIRGeneratorLOONG64::lowerForBitAndAndBranch(LBitAndAndBranch* baab,
+ MInstruction* mir,
+ MDefinition* lhs,
+ MDefinition* rhs) {
+ baab->setOperand(0, useRegisterAtStart(lhs));
+ baab->setOperand(1, useRegisterOrConstantAtStart(rhs));
+ add(baab, mir);
+}
+
+LBoxAllocation LIRGeneratorLOONG64::useBoxFixed(MDefinition* mir, Register reg1,
+ Register reg2,
+ bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart));
+}
+
+LAllocation LIRGeneratorLOONG64::useByteOpRegister(MDefinition* mir) {
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorLOONG64::useByteOpRegisterAtStart(MDefinition* mir) {
+ return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorLOONG64::useByteOpRegisterOrNonDoubleConstant(
+ MDefinition* mir) {
+ return useRegisterOrNonDoubleConstant(mir);
+}
+
+LDefinition LIRGeneratorLOONG64::tempByteOpRegister() { return temp(); }
+
+LDefinition LIRGeneratorLOONG64::tempToUnbox() { return temp(); }
+
+void LIRGeneratorLOONG64::lowerUntypedPhiInput(MPhi* phi,
+ uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+void LIRGeneratorLOONG64::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+void LIRGeneratorLOONG64::defineInt64Phi(MPhi* phi, size_t lirIndex) {
+ defineTypedPhi(phi, lirIndex);
+}
+
+void LIRGeneratorLOONG64::lowerNegI(MInstruction* ins, MDefinition* input) {
+ define(new (alloc()) LNegI(useRegisterAtStart(input)), ins);
+}
+void LIRGeneratorLOONG64::lowerNegI64(MInstruction* ins, MDefinition* input) {
+ defineInt64ReuseInput(new (alloc()) LNegI64(useInt64RegisterAtStart(input)),
+ ins, 0);
+}
+
+void LIRGeneratorLOONG64::lowerMulI(MMul* mul, MDefinition* lhs,
+ MDefinition* rhs) {
+ LMulI* lir = new (alloc()) LMulI;
+ if (mul->fallible()) {
+ assignSnapshot(lir, mul->bailoutKind());
+ }
+
+ lowerForALU(lir, mul, lhs, rhs);
+}
+
+void LIRGeneratorLOONG64::lowerDivI(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDiv(div);
+ return;
+ }
+
+ // Division instructions are slow. Division by constant denominators can be
+ // rewritten to use other instructions.
+ if (div->rhs()->isConstant()) {
+ int32_t rhs = div->rhs()->toConstant()->toInt32();
+ // Check for division by a positive power of two, which is an easy and
+ // important case to optimize. Note that other optimizations are also
+ // possible; division by negative powers of two can be optimized in a
+ // similar manner as positive powers of two, and division by other
+ // constants can be optimized by a reciprocal multiplication technique.
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LDivPowTwoI* lir =
+ new (alloc()) LDivPowTwoI(useRegister(div->lhs()), shift, temp());
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+ }
+
+ LDivI* lir = new (alloc())
+ LDivI(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+}
+
+void LIRGeneratorLOONG64::lowerDivI64(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDivI64(div);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc())
+ LDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ defineInt64(lir, div);
+}
+
+void LIRGeneratorLOONG64::lowerModI(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUMod(mod);
+ return;
+ }
+
+ if (mod->rhs()->isConstant()) {
+ int32_t rhs = mod->rhs()->toConstant()->toInt32();
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LModPowTwoI* lir =
+ new (alloc()) LModPowTwoI(useRegister(mod->lhs()), shift);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ } else if (shift < 31 && (1 << (shift + 1)) - 1 == rhs) {
+ LModMaskI* lir = new (alloc())
+ LModMaskI(useRegister(mod->lhs()), temp(LDefinition::GENERAL),
+ temp(LDefinition::GENERAL), shift + 1);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ }
+ }
+ LModI* lir =
+ new (alloc()) LModI(useRegister(mod->lhs()), useRegister(mod->rhs()),
+ temp(LDefinition::GENERAL));
+
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+}
+
+void LIRGeneratorLOONG64::lowerModI64(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUModI64(mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc())
+ LDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()), temp());
+ defineInt64(lir, mod);
+}
+
+void LIRGeneratorLOONG64::lowerUDiv(MDiv* div) {
+ MDefinition* lhs = div->getOperand(0);
+ MDefinition* rhs = div->getOperand(1);
+
+ LUDivOrMod* lir = new (alloc()) LUDivOrMod;
+ lir->setOperand(0, useRegister(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+
+ define(lir, div);
+}
+
+void LIRGeneratorLOONG64::lowerUDivI64(MDiv* div) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ defineInt64(lir, div);
+}
+
+void LIRGeneratorLOONG64::lowerUMod(MMod* mod) {
+ MDefinition* lhs = mod->getOperand(0);
+ MDefinition* rhs = mod->getOperand(1);
+
+ LUDivOrMod* lir = new (alloc()) LUDivOrMod;
+ lir->setOperand(0, useRegister(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+
+ define(lir, mod);
+}
+
+void LIRGeneratorLOONG64::lowerUModI64(MMod* mod) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()), temp());
+ defineInt64(lir, mod);
+}
+
+void LIRGeneratorLOONG64::lowerUrshD(MUrsh* mir) {
+ MDefinition* lhs = mir->lhs();
+ MDefinition* rhs = mir->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+
+ LUrshD* lir = new (alloc())
+ LUrshD(useRegister(lhs), useRegisterOrConstant(rhs), temp());
+ define(lir, mir);
+}
+
+void LIRGeneratorLOONG64::lowerPowOfTwoI(MPow* mir) {
+ int32_t base = mir->input()->toConstant()->toInt32();
+ MDefinition* power = mir->power();
+
+ auto* lir = new (alloc()) LPowOfTwoI(useRegister(power), base);
+ assignSnapshot(lir, mir->bailoutKind());
+ define(lir, mir);
+}
+
+void LIRGeneratorLOONG64::lowerBigIntDiv(MBigIntDiv* ins) {
+ auto* lir = new (alloc()) LBigIntDiv(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorLOONG64::lowerBigIntMod(MBigIntMod* ins) {
+ auto* lir = new (alloc()) LBigIntMod(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorLOONG64::lowerBigIntLsh(MBigIntLsh* ins) {
+ auto* lir = new (alloc()) LBigIntLsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorLOONG64::lowerBigIntRsh(MBigIntRsh* ins) {
+ auto* lir = new (alloc()) LBigIntRsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorLOONG64::lowerTruncateDToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double);
+
+ define(new (alloc()) LTruncateDToInt32(useRegister(opd), tempDouble()), ins);
+}
+
+void LIRGeneratorLOONG64::lowerTruncateFToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Float32);
+
+ define(new (alloc()) LTruncateFToInt32(useRegister(opd), tempFloat32()), ins);
+}
+
+void LIRGeneratorLOONG64::lowerBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGeneratorLOONG64::lowerWasmSelectI(MWasmSelect* select) {
+ auto* lir = new (alloc())
+ LWasmSelect(useRegisterAtStart(select->trueExpr()),
+ useAny(select->falseExpr()), useRegister(select->condExpr()));
+ defineReuseInput(lir, select, LWasmSelect::TrueExprIndex);
+}
+
+void LIRGeneratorLOONG64::lowerWasmSelectI64(MWasmSelect* select) {
+ auto* lir = new (alloc()) LWasmSelectI64(
+ useInt64RegisterAtStart(select->trueExpr()),
+ useInt64(select->falseExpr()), useRegister(select->condExpr()));
+ defineInt64ReuseInput(lir, select, LWasmSelectI64::TrueExprIndex);
+}
+
+// On loong64 we specialize the only cases where compare is {U,}Int32 and select
+// is {U,}Int32.
+bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
+ MCompare::CompareType compTy, MIRType insTy) {
+ return insTy == MIRType::Int32 && (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32);
+}
+
+void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
+ MDefinition* lhs,
+ MDefinition* rhs,
+ MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+ auto* lir = new (alloc()) LWasmCompareAndSelect(
+ useRegister(lhs), useRegister(rhs), compTy, jsop,
+ useRegisterAtStart(ins->trueExpr()), useRegister(ins->falseExpr()));
+ defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
+}
+
+void LIRGeneratorLOONG64::lowerWasmBuiltinTruncateToInt32(
+ MWasmBuiltinTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ if (opd->type() == MIRType::Double) {
+ define(new (alloc()) LWasmBuiltinTruncateDToInt32(
+ useRegister(opd), useFixed(ins->instance(), InstanceReg),
+ LDefinition::BogusTemp()),
+ ins);
+ return;
+ }
+
+ define(new (alloc()) LWasmBuiltinTruncateFToInt32(
+ useRegister(opd), useFixed(ins->instance(), InstanceReg),
+ LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGeneratorLOONG64::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGeneratorLOONG64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ MOZ_CRASH("We don't use runtime div for this architecture");
+}
+
+void LIRGeneratorLOONG64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ MOZ_CRASH("We don't use runtime mod for this architecture");
+}
+
+void LIRGeneratorLOONG64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+ auto* lir = new (alloc()) LAtomicLoad64(elements, index, temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorLOONG64::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value = useRegister(ins->value());
+
+ add(new (alloc()) LAtomicStore64(elements, index, value, tempInt64()), ins);
+}
+
+void LIRGenerator::visitBox(MBox* box) {
+ MDefinition* opd = box->getOperand(0);
+
+ // If the operand is a constant, emit near its uses.
+ if (opd->isConstant() && box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (opd->isConstant()) {
+ define(new (alloc()) LValue(opd->toConstant()->toJSValue()), box,
+ LDefinition(LDefinition::BOX));
+ } else {
+ LBox* ins = new (alloc()) LBox(useRegister(opd), opd->type());
+ define(ins, box, LDefinition(LDefinition::BOX));
+ }
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+ MDefinition* box = unbox->getOperand(0);
+ MOZ_ASSERT(box->type() == MIRType::Value);
+
+ LUnbox* lir;
+ if (IsFloatingPointType(unbox->type())) {
+ lir = new (alloc())
+ LUnboxFloatingPoint(useRegisterAtStart(box), unbox->type());
+ } else if (unbox->fallible()) {
+ // If the unbox is fallible, load the Value in a register first to
+ // avoid multiple loads.
+ lir = new (alloc()) LUnbox(useRegisterAtStart(box));
+ } else {
+ lir = new (alloc()) LUnbox(useAtStart(box));
+ }
+
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+
+ define(lir, unbox);
+}
+
+void LIRGenerator::visitAbs(MAbs* ins) {
+ define(allocateAbs(ins, useRegisterAtStart(ins->input())), ins);
+}
+
+void LIRGenerator::visitCopySign(MCopySign* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(IsFloatingPointType(lhs->type()));
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(lhs->type() == ins->type());
+
+ LInstructionHelper<1, 2, 2>* lir;
+ if (lhs->type() == MIRType::Double) {
+ lir = new (alloc()) LCopySignD();
+ } else {
+ lir = new (alloc()) LCopySignF();
+ }
+
+ lir->setTemp(0, temp());
+ lir->setTemp(1, temp());
+
+ lir->setOperand(0, useRegisterAtStart(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitPowHalf(MPowHalf* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Double);
+ LPowHalfD* lir = new (alloc()) LPowHalfD(useRegisterAtStart(input));
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
+ defineInt64(
+ new (alloc()) LExtendInt32ToInt64(useRegisterAtStart(ins->input())), ins);
+}
+
+void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
+ defineInt64(new (alloc())
+ LSignExtendInt64(useInt64RegisterAtStart(ins->input())),
+ ins);
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Int64);
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+ define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd)), ins);
+}
+
+void LIRGenerator::visitSubstr(MSubstr* ins) {
+ LSubstr* lir = new (alloc())
+ LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
+ useRegister(ins->length()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation newval = useRegister(ins->newval());
+ const LAllocation oldval = useRegister(ins->oldval());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64();
+
+ auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+ elements, index, oldval, newval, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ // If the target is a floating register then we need a temp at the
+ // CodeGenerator level for creating the result.
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ outTemp = temp();
+ }
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LCompareExchangeTypedArrayElement* lir = new (alloc())
+ LCompareExchangeTypedArrayElement(elements, index, oldval, newval,
+ outTemp, valueTemp, offsetTemp,
+ maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LDefinition temp2 = temp();
+
+ auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64(
+ elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ // If the target is a floating register then we need a temp at the
+ // CodeGenerator level for creating the result.
+
+ MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32);
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32) {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ outTemp = temp();
+ }
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LAtomicExchangeTypedArrayElement* lir =
+ new (alloc()) LAtomicExchangeTypedArrayElement(
+ elements, index, value, outTemp, valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64();
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We can omit allocating the result BigInt.
+
+ if (ins->isForEffect()) {
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+ elements, index, value, temp1, temp2);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+
+ auto* lir = new (alloc())
+ LAtomicTypedArrayElementBinop64(elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ if (ins->isForEffect()) {
+ LAtomicTypedArrayElementBinopForEffect* lir =
+ new (alloc()) LAtomicTypedArrayElementBinopForEffect(
+ elements, index, value, valueTemp, offsetTemp, maskTemp);
+ add(lir, ins);
+ return;
+ }
+
+ // For a Uint32Array with a known double result we need a temp for
+ // the intermediate output.
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ outTemp = temp();
+ }
+
+ LAtomicTypedArrayElementBinop* lir =
+ new (alloc()) LAtomicTypedArrayElementBinop(
+ elements, index, value, outTemp, valueTemp, offsetTemp, maskTemp);
+ define(lir, ins);
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new (alloc()) LReturn(isGenerator);
+ ins->setOperand(0, useFixed(opd, JSReturnReg));
+ add(ins);
+}
+
+void LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT_IF(ins->needsBoundsCheck(),
+ boundsCheckLimit->type() == MIRType::Int32);
+
+ LAllocation baseAlloc = useRegisterAtStart(base);
+
+ LAllocation limitAlloc = ins->needsBoundsCheck()
+ ? useRegisterAtStart(boundsCheckLimit)
+ : LAllocation();
+
+ // We have no memory-base value, meaning that HeapReg is to be used as the
+ // memory base. This follows from the definition of
+ // FunctionCompiler::maybeLoadMemoryBase() in WasmIonCompile.cpp.
+ MOZ_ASSERT(!ins->hasMemoryBase());
+ auto* lir =
+ new (alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc, LAllocation());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT_IF(ins->needsBoundsCheck(),
+ boundsCheckLimit->type() == MIRType::Int32);
+
+ LAllocation baseAlloc = useRegisterAtStart(base);
+
+ LAllocation limitAlloc = ins->needsBoundsCheck()
+ ? useRegisterAtStart(boundsCheckLimit)
+ : LAllocation();
+
+ // See comment in LIRGenerator::visitAsmJSStoreHeap just above.
+ MOZ_ASSERT(!ins->hasMemoryBase());
+ add(new (alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()),
+ limitAlloc, LAllocation()),
+ ins);
+}
+
+void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) {
+ auto* lir = new (alloc()) LWasmHeapBase(LAllocation());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
+ MDefinition* base = ins->base();
+ // 'base' is a GPR but may be of either type. If it is 32-bit, it is
+ // sign-extended on loongarch64 platform and we should explicitly promote it
+ // to 64-bit when use it as an index register in memory accesses.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ LAllocation ptr;
+ ptr = useRegisterAtStart(base);
+
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new (alloc()) LWasmLoadI64(ptr);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ defineInt64(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LWasmLoad(ptr);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmStore(MWasmStore* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ MDefinition* value = ins->value();
+
+ if (ins->access().type() == Scalar::Int64) {
+ LAllocation baseAlloc = useRegisterAtStart(base);
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+ auto* lir = new (alloc()) LWasmStoreI64(baseAlloc, valueAlloc);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation baseAlloc = useRegisterAtStart(base);
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmNeg(MWasmNeg* ins) {
+ if (ins->type() == MIRType::Int32) {
+ define(new (alloc()) LNegI(useRegisterAtStart(ins->input())), ins);
+ } else if (ins->type() == MIRType::Float32) {
+ define(new (alloc()) LNegF(useRegisterAtStart(ins->input())), ins);
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ define(new (alloc()) LNegD(useRegisterAtStart(ins->input())), ins);
+ }
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd)), ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToDouble* lir =
+ new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToFloat32* lir =
+ new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmCompareExchangeI64(
+ useRegister(base), useInt64Register(ins->oldValue()),
+ useInt64Register(ins->newValue()));
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LWasmCompareExchangeHeap* lir = new (alloc()) LWasmCompareExchangeHeap(
+ useRegister(base), useRegister(ins->oldValue()),
+ useRegister(ins->newValue()), valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmAtomicExchangeI64(
+ useRegister(base), useInt64Register(ins->value()));
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LWasmAtomicExchangeHeap* lir = new (alloc())
+ LWasmAtomicExchangeHeap(useRegister(base), useRegister(ins->value()),
+ valueTemp, offsetTemp, maskTemp);
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc())
+ LWasmAtomicBinopI64(useRegister(base), useInt64Register(ins->value()));
+ lir->setTemp(0, temp());
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ if (!ins->hasUses()) {
+ LWasmAtomicBinopHeapForEffect* lir = new (alloc())
+ LWasmAtomicBinopHeapForEffect(useRegister(base),
+ useRegister(ins->value()), valueTemp,
+ offsetTemp, maskTemp);
+ add(lir, ins);
+ return;
+ }
+
+ LWasmAtomicBinopHeap* lir = new (alloc())
+ LWasmAtomicBinopHeap(useRegister(base), useRegister(ins->value()),
+ valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmTernarySimd128(MWasmTernarySimd128* ins) {
+ MOZ_CRASH("ternary SIMD NYI");
+}
+
+void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) {
+ MOZ_CRASH("binary SIMD NYI");
+}
+
+#ifdef ENABLE_WASM_SIMD
+bool MWasmTernarySimd128::specializeBitselectConstantMaskAsShuffle(
+ int8_t shuffle[16]) {
+ return false;
+}
+#endif
+
+bool MWasmBinarySimd128::specializeForConstantRhs() {
+ // Probably many we want to do here
+ return false;
+}
+
+void LIRGenerator::visitWasmBinarySimd128WithConstant(
+ MWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("binary SIMD with constant NYI");
+}
+
+void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) {
+ MOZ_CRASH("shift SIMD NYI");
+}
+
+void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) {
+ MOZ_CRASH("shuffle SIMD NYI");
+}
+
+void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH("replace-lane SIMD NYI");
+}
+
+void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) {
+ MOZ_CRASH("scalar-to-SIMD NYI");
+}
+
+void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) {
+ MOZ_CRASH("unary SIMD NYI");
+}
+
+void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) {
+ MOZ_CRASH("reduce-SIMD NYI");
+}
+
+void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH("load-lane SIMD NYI");
+}
+
+void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH("store-lane SIMD NYI");
+}
diff --git a/js/src/jit/loong64/Lowering-loong64.h b/js/src/jit/loong64/Lowering-loong64.h
new file mode 100644
index 0000000000..6285b13291
--- /dev/null
+++ b/js/src/jit/loong64/Lowering-loong64.h
@@ -0,0 +1,110 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_Lowering_loong64_h
+#define jit_loong64_Lowering_loong64_h
+
+#include "jit/shared/Lowering-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorLOONG64 : public LIRGeneratorShared {
+ protected:
+ LIRGeneratorLOONG64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorShared(gen, graph, lirGraph) {}
+
+ LTableSwitch* newLTableSwitch(const LAllocation& in,
+ const LDefinition& inputCopy,
+ MTableSwitch* ins);
+ LTableSwitchV* newLTableSwitchV(MTableSwitch* ins);
+
+ void lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ template <size_t Temps>
+ void lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForALU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* input);
+ void lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* input);
+ void lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+ void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs,
+ MDefinition* rhs);
+
+ void lowerForFPU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* src);
+ template <size_t Temps>
+ void lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForCompareI64AndBranch(MTest* mir, MCompare* comp, JSOp op,
+ MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse);
+ void lowerForBitAndAndBranch(LBitAndAndBranch* baab, MInstruction* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ // Returns a box allocation. reg2 is ignored on 64-bit platforms.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2,
+ bool useAtStart = false);
+
+ LAllocation useByteOpRegister(MDefinition* mir);
+ LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+ LDefinition tempByteOpRegister();
+
+ LDefinition tempToUnbox();
+
+ bool needTempForPostBarrier() { return true; }
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void lowerInt64PhiInput(MPhi*, uint32_t, LBlock*, size_t);
+ void defineInt64Phi(MPhi*, size_t);
+
+ void lowerNegI(MInstruction* ins, MDefinition* input);
+ void lowerNegI64(MInstruction* ins, MDefinition* input);
+ void lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs);
+ void lowerDivI(MDiv* div);
+ void lowerDivI64(MDiv* div);
+ void lowerModI(MMod* mod);
+ void lowerModI64(MMod* mod);
+ void lowerUDiv(MDiv* div);
+ void lowerUDivI64(MDiv* div);
+ void lowerUMod(MMod* mod);
+ void lowerUModI64(MMod* mod);
+ void lowerUrshD(MUrsh* mir);
+ void lowerPowOfTwoI(MPow* mir);
+ void lowerBigIntDiv(MBigIntDiv* ins);
+ void lowerBigIntMod(MBigIntMod* ins);
+ void lowerBigIntLsh(MBigIntLsh* ins);
+ void lowerBigIntRsh(MBigIntRsh* ins);
+ void lowerTruncateDToInt32(MTruncateToInt32* ins);
+ void lowerTruncateFToInt32(MTruncateToInt32* ins);
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+ void lowerWasmSelectI(MWasmSelect* select);
+ void lowerWasmSelectI64(MWasmSelect* select);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+ void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+ void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+};
+
+typedef LIRGeneratorLOONG64 LIRGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_Lowering_loong64_h */
diff --git a/js/src/jit/loong64/MacroAssembler-loong64-inl.h b/js/src/jit/loong64/MacroAssembler-loong64-inl.h
new file mode 100644
index 0000000000..3eb98d166e
--- /dev/null
+++ b/js/src/jit/loong64/MacroAssembler-loong64-inl.h
@@ -0,0 +1,2120 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_MacroAssembler_loong64_inl_h
+#define jit_loong64_MacroAssembler_loong64_inl_h
+
+#include "jit/loong64/MacroAssembler-loong64.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+ movePtr(src.reg, dest.reg);
+}
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+ movePtr(ImmWord(imm.value), dest.reg);
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ moveFromDouble(src, dest.reg);
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ moveToDouble(src.reg, dest);
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+ as_slli_w(dest, src.reg, 0);
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ as_bstrpick_d(dest.reg, src, 31, 0);
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ move32To64SignExtend(src, dest);
+ move8SignExtend(dest.reg, dest.reg);
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ move32To64SignExtend(src, dest);
+ move16SignExtend(dest.reg, dest.reg);
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ as_slli_w(dest.reg, src, 0);
+}
+
+void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
+ moveFromFloat32(src, dest);
+}
+
+void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
+ moveToFloat32(src, dest);
+}
+
+void MacroAssembler::move8SignExtend(Register src, Register dest) {
+ as_ext_w_b(dest, src);
+}
+
+void MacroAssembler::move16SignExtend(Register src, Register dest) {
+ as_ext_w_h(dest, src);
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ as_slli_w(dest, src, 0);
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ as_bstrpick_d(dest, src, 31, 0);
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ load32(src, dest);
+}
+
+void MacroAssembler::loadAbiReturnAddress(Register dest) { movePtr(ra, dest); }
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::not32(Register reg) { as_nor(reg, reg, zero); }
+
+void MacroAssembler::notPtr(Register reg) { as_nor(reg, reg, zero); }
+
+void MacroAssembler::andPtr(Register src, Register dest) {
+ as_and(dest, dest, src);
+}
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) {
+ ma_and(dest, dest, imm);
+}
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, ImmWord(imm.value));
+ as_and(dest.reg, dest.reg, scratch);
+}
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+ as_and(dest.reg, dest.reg, src.reg);
+}
+
+void MacroAssembler::and64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ ScratchRegisterScope scratch(*this);
+ Register64 scratch64(scratch);
+
+ load64(src.toAddress(), scratch64);
+ and64(scratch64, dest);
+ } else {
+ and64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::and32(Register src, Register dest) {
+ as_and(dest, dest, src);
+}
+
+void MacroAssembler::and32(Imm32 imm, Register dest) {
+ ma_and(dest, dest, imm);
+}
+
+void MacroAssembler::and32(Imm32 imm, const Address& dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(dest, scratch2);
+ and32(imm, scratch2);
+ store32(scratch2, dest);
+}
+
+void MacroAssembler::and32(const Address& src, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(src, scratch2);
+ as_and(dest, dest, scratch2);
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, ImmWord(imm.value));
+ as_or(dest.reg, dest.reg, scratch);
+}
+
+void MacroAssembler::or32(Register src, Register dest) {
+ as_or(dest, dest, src);
+}
+
+void MacroAssembler::or32(Imm32 imm, Register dest) { ma_or(dest, dest, imm); }
+
+void MacroAssembler::or32(Imm32 imm, const Address& dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(dest, scratch2);
+ or32(imm, scratch2);
+ store32(scratch2, dest);
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, ImmWord(imm.value));
+ as_xor(dest.reg, dest.reg, scratch);
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) {
+ as_or(dest, dest, src);
+}
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) { ma_or(dest, dest, imm); }
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+ as_or(dest.reg, dest.reg, src.reg);
+}
+
+void MacroAssembler::or64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ ScratchRegisterScope scratch(asMasm());
+ Register64 scratch64(scratch);
+
+ load64(src.toAddress(), scratch64);
+ or64(scratch64, dest);
+ } else {
+ or64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+ as_xor(dest.reg, dest.reg, src.reg);
+}
+
+void MacroAssembler::xor64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ ScratchRegisterScope scratch(asMasm());
+ Register64 scratch64(scratch);
+
+ load64(src.toAddress(), scratch64);
+ xor64(scratch64, dest);
+ } else {
+ xor64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) {
+ as_xor(dest, dest, src);
+}
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) {
+ ma_xor(dest, dest, imm);
+}
+
+void MacroAssembler::xor32(Register src, Register dest) {
+ as_xor(dest, dest, src);
+}
+
+void MacroAssembler::xor32(Imm32 imm, Register dest) {
+ ma_xor(dest, dest, imm);
+}
+
+void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(dest, scratch2);
+ xor32(imm, scratch2);
+ store32(scratch2, dest);
+}
+
+void MacroAssembler::xor32(const Address& src, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(src, scratch2);
+ xor32(scratch2, dest);
+}
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap16SignExtend(Register reg) {
+ as_revb_2h(reg, reg);
+ as_ext_w_h(reg, reg);
+}
+
+void MacroAssembler::byteSwap16ZeroExtend(Register reg) {
+ as_revb_2h(reg, reg);
+ as_bstrpick_d(reg, reg, 15, 0);
+}
+
+void MacroAssembler::byteSwap32(Register reg) {
+ as_revb_2w(reg, reg);
+ as_slli_w(reg, reg, 0);
+}
+
+void MacroAssembler::byteSwap64(Register64 reg64) {
+ as_revb_d(reg64.reg, reg64.reg);
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(Register src, Register dest) {
+ as_add_d(dest, dest, src);
+}
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) {
+ ma_add_d(dest, dest, imm);
+}
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(imm, scratch);
+ addPtr(scratch, dest);
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+ addPtr(src.reg, dest.reg);
+}
+
+void MacroAssembler::add64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ ScratchRegisterScope scratch(asMasm());
+ Register64 scratch64(scratch);
+
+ load64(src.toAddress(), scratch64);
+ add64(scratch64, dest);
+ } else {
+ add64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) {
+ ma_add_d(dest.reg, dest.reg, imm);
+}
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest.reg != scratch);
+ mov(ImmWord(imm.value), scratch);
+ as_add_d(dest.reg, dest.reg, scratch);
+}
+
+void MacroAssembler::add32(Register src, Register dest) {
+ as_add_w(dest, dest, src);
+}
+
+void MacroAssembler::add32(Imm32 imm, Register dest) {
+ ma_add_w(dest, dest, imm);
+}
+
+void MacroAssembler::add32(Imm32 imm, const Address& dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(dest, scratch2);
+ ma_add_w(scratch2, scratch2, imm);
+ store32(scratch2, dest);
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ loadPtr(dest, scratch);
+ addPtr(imm, scratch);
+ storePtr(scratch, dest);
+}
+
+void MacroAssembler::addPtr(const Address& src, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ loadPtr(src, scratch);
+ addPtr(scratch, dest);
+}
+
+void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
+ as_fadd_d(dest, dest, src);
+}
+
+void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
+ as_fadd_s(dest, dest, src);
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ MacroAssemblerLOONG64::ma_liPatchable(dest, Imm32(0));
+ as_sub_d(dest, StackPointer, dest);
+ return offset;
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ // TODO: by wangqing
+ Instruction* inst0 =
+ (Instruction*)m_buffer.getInst(BufferOffset(offset.offset()));
+
+ InstImm* i0 = (InstImm*)inst0;
+ InstImm* i1 = (InstImm*)i0->next();
+
+ MOZ_ASSERT((i0->extractBitField(31, 25)) == ((uint32_t)op_lu12i_w >> 25));
+ MOZ_ASSERT((i1->extractBitField(31, 22)) == ((uint32_t)op_ori >> 22));
+
+ *i0 = InstImm(op_lu12i_w, (int32_t)((imm.value >> 12) & 0xfffff),
+ Register::FromCode(i0->extractRD()), false);
+ *i1 = InstImm(op_ori, (int32_t)(imm.value & 0xfff),
+ Register::FromCode(i1->extractRJ()),
+ Register::FromCode(i1->extractRD()), 12);
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) {
+ as_sub_d(dest, dest, src);
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) {
+ ma_sub_d(dest, dest, imm);
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+ as_sub_d(dest.reg, dest.reg, src.reg);
+}
+
+void MacroAssembler::sub64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ ScratchRegisterScope scratch(asMasm());
+ Register64 scratch64(scratch);
+
+ load64(src.toAddress(), scratch64);
+ sub64(scratch64, dest);
+ } else {
+ sub64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest.reg != scratch);
+ mov(ImmWord(imm.value), scratch);
+ as_sub_d(dest.reg, dest.reg, scratch);
+}
+
+void MacroAssembler::sub32(Register src, Register dest) {
+ as_sub_w(dest, dest, src);
+}
+
+void MacroAssembler::sub32(Imm32 imm, Register dest) {
+ ma_sub_w(dest, dest, imm);
+}
+
+void MacroAssembler::sub32(const Address& src, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(src, scratch2);
+ as_sub_w(dest, dest, scratch2);
+}
+
+void MacroAssembler::subPtr(Register src, const Address& dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(dest, scratch2);
+ subPtr(src, scratch2);
+ storePtr(scratch2, dest);
+}
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(addr, scratch2);
+ subPtr(scratch2, dest);
+}
+
+void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
+ as_fsub_d(dest, dest, src);
+}
+
+void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
+ as_fsub_s(dest, dest, src);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest.reg != scratch);
+ mov(ImmWord(imm.value), scratch);
+ as_mul_d(dest.reg, dest.reg, scratch);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ mul64(imm, dest);
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ as_mul_d(dest.reg, dest.reg, src.reg);
+}
+
+void MacroAssembler::mul64(const Operand& src, const Register64& dest,
+ const Register temp) {
+ if (src.getTag() == Operand::MEM) {
+ ScratchRegisterScope scratch(asMasm());
+ Register64 scratch64(scratch);
+
+ load64(src.toAddress(), scratch64);
+ mul64(scratch64, dest, temp);
+ } else {
+ mul64(Register64(src.toReg()), dest, temp);
+ }
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+ as_mul_d(srcDest, srcDest, rhs);
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(src != scratch);
+ as_add_d(scratch, src, src);
+ as_add_d(dest, scratch, src);
+}
+
+void MacroAssembler::mul32(Register rhs, Register srcDest) {
+ as_mul_w(srcDest, srcDest, rhs);
+}
+
+void MacroAssembler::mul32(Imm32 imm, Register srcDest) {
+ ScratchRegisterScope scratch(asMasm());
+ move32(imm, scratch);
+ mul32(scratch, srcDest);
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(src != scratch);
+ move32(imm, scratch);
+ as_mulh_wu(dest, src, scratch);
+}
+
+void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
+ as_fmul_s(dest, dest, src);
+}
+
+void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
+ as_fmul_d(dest, dest, src);
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+ FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+ movePtr(imm, scratch);
+ loadDouble(Address(scratch, 0), fpscratch);
+ mulDouble(fpscratch, dest);
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch, ImmWord(uintptr_t(dest.addr)));
+ as_ld_d(scratch2, scratch, 0);
+ as_addi_d(scratch2, scratch2, 1);
+ as_st_d(scratch2, scratch, 0);
+}
+
+void MacroAssembler::quotient32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ if (isUnsigned) {
+ as_div_wu(srcDest, srcDest, rhs);
+ } else {
+ as_div_w(srcDest, srcDest, rhs);
+ }
+}
+
+void MacroAssembler::remainder32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ if (isUnsigned) {
+ as_mod_wu(srcDest, srcDest, rhs);
+ } else {
+ as_mod_w(srcDest, srcDest, rhs);
+ }
+}
+
+void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
+ as_fdiv_s(dest, dest, src);
+}
+
+void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
+ as_fdiv_d(dest, dest, src);
+}
+
+void MacroAssembler::neg64(Register64 reg) { as_sub_d(reg.reg, zero, reg.reg); }
+
+void MacroAssembler::negPtr(Register reg) { as_sub_d(reg, zero, reg); }
+
+void MacroAssembler::neg32(Register reg) { as_sub_w(reg, zero, reg); }
+
+void MacroAssembler::negateDouble(FloatRegister reg) { as_fneg_d(reg, reg); }
+
+void MacroAssembler::negateFloat(FloatRegister reg) { as_fneg_s(reg, reg); }
+
+void MacroAssembler::abs32(Register src, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ as_srai_w(scratch, src, 31);
+ as_xor(dest, src, scratch);
+ as_sub_w(dest, dest, scratch);
+}
+
+void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
+ as_fabs_s(dest, src);
+}
+
+void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
+ as_fabs_d(dest, src);
+}
+
+void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
+ as_fsqrt_s(dest, src);
+}
+
+void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
+ as_fsqrt_d(dest, src);
+}
+
+void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxFloat32(srcDest, other, handleNaN, false);
+}
+
+void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxDouble(srcDest, other, handleNaN, false);
+}
+
+void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxFloat32(srcDest, other, handleNaN, true);
+}
+
+void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxDouble(srcDest, other, handleNaN, true);
+}
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshift32(Register src, Register dest) {
+ as_sll_w(dest, dest, src);
+}
+
+void MacroAssembler::lshift32(Imm32 imm, Register dest) {
+ as_slli_w(dest, dest, imm.value % 32);
+}
+
+void MacroAssembler::flexibleLshift32(Register src, Register dest) {
+ lshift32(src, dest);
+}
+
+void MacroAssembler::lshift64(Register shift, Register64 dest) {
+ as_sll_d(dest.reg, dest.reg, shift);
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ as_slli_d(dest.reg, dest.reg, imm.value);
+}
+
+void MacroAssembler::lshiftPtr(Register shift, Register dest) {
+ as_sll_d(dest, dest, shift);
+}
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ as_slli_d(dest, dest, imm.value);
+}
+
+void MacroAssembler::rshift32(Register src, Register dest) {
+ as_srl_w(dest, dest, src);
+}
+
+void MacroAssembler::rshift32(Imm32 imm, Register dest) {
+ as_srli_w(dest, dest, imm.value % 32);
+}
+
+void MacroAssembler::flexibleRshift32(Register src, Register dest) {
+ rshift32(src, dest);
+}
+
+void MacroAssembler::rshift32Arithmetic(Register src, Register dest) {
+ as_sra_w(dest, dest, src);
+}
+
+void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) {
+ as_srai_w(dest, dest, imm.value % 32);
+}
+
+void MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest) {
+ rshift32Arithmetic(src, dest);
+}
+
+void MacroAssembler::rshift64(Register shift, Register64 dest) {
+ as_srl_d(dest.reg, dest.reg, shift);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ as_srli_d(dest.reg, dest.reg, imm.value);
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ as_srai_d(dest.reg, dest.reg, imm.value);
+}
+
+void MacroAssembler::rshift64Arithmetic(Register shift, Register64 dest) {
+ as_sra_d(dest.reg, dest.reg, shift);
+}
+
+void MacroAssembler::rshiftPtr(Register shift, Register dest) {
+ as_srl_d(dest, dest, shift);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ as_srli_d(dest, dest, imm.value);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ as_srai_d(dest, dest, imm.value);
+}
+
+// ===============================================================
+// Rotation functions
+
+void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ as_sub_w(scratch, zero, count);
+ as_rotr_w(dest, input, scratch);
+}
+
+void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
+ as_rotri_w(dest, input, (32 - count.value) & 31);
+}
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ ScratchRegisterScope scratch(asMasm());
+ as_sub_d(scratch, zero, count);
+ as_rotr_d(dest.reg, src.reg, scratch);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ as_rotri_d(dest.reg, src.reg, (64 - count.value) & 63);
+}
+
+void MacroAssembler::rotateRight(Register count, Register input,
+ Register dest) {
+ as_rotr_w(dest, input, count);
+}
+
+void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
+ as_rotri_w(dest, input, count.value & 31);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ as_rotr_d(dest.reg, src.reg, count);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ as_rotri_d(dest.reg, src.reg, count.value & 63);
+}
+
+// Bit counting functions
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+ as_clz_d(dest, src.reg);
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+ as_ctz_d(dest, src.reg);
+}
+
+void MacroAssembler::popcnt64(Register64 input, Register64 output,
+ Register tmp) {
+ ScratchRegisterScope scratch(asMasm());
+ as_or(output.reg, input.reg, zero);
+ as_srai_d(tmp, input.reg, 1);
+ ma_li(scratch, Imm32(0x55555555));
+ as_bstrins_d(scratch, scratch, 63, 32);
+ as_and(tmp, tmp, scratch);
+ as_sub_d(output.reg, output.reg, tmp);
+ as_srai_d(tmp, output.reg, 2);
+ ma_li(scratch, Imm32(0x33333333));
+ as_bstrins_d(scratch, scratch, 63, 32);
+ as_and(output.reg, output.reg, scratch);
+ as_and(tmp, tmp, scratch);
+ as_add_d(output.reg, output.reg, tmp);
+ as_srli_d(tmp, output.reg, 4);
+ as_add_d(output.reg, output.reg, tmp);
+ ma_li(scratch, Imm32(0xF0F0F0F));
+ as_bstrins_d(scratch, scratch, 63, 32);
+ as_and(output.reg, output.reg, scratch);
+ ma_li(tmp, Imm32(0x1010101));
+ as_bstrins_d(tmp, tmp, 63, 32);
+ as_mul_d(output.reg, output.reg, tmp);
+ as_srai_d(output.reg, output.reg, 56);
+}
+
+void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
+ as_clz_w(dest, src);
+}
+
+void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
+ as_ctz_w(dest, src);
+}
+
+void MacroAssembler::popcnt32(Register input, Register output, Register tmp) {
+ // Equivalent to GCC output of mozilla::CountPopulation32()
+ as_or(output, input, zero);
+ as_srai_w(tmp, input, 1);
+ ma_and(tmp, tmp, Imm32(0x55555555));
+ as_sub_w(output, output, tmp);
+ as_srai_w(tmp, output, 2);
+ ma_and(output, output, Imm32(0x33333333));
+ ma_and(tmp, tmp, Imm32(0x33333333));
+ as_add_w(output, output, tmp);
+ as_srli_w(tmp, output, 4);
+ as_add_w(output, output, tmp);
+ ma_and(output, output, Imm32(0xF0F0F0F));
+ as_slli_w(tmp, output, 8);
+ as_add_w(output, output, tmp);
+ as_slli_w(tmp, output, 16);
+ as_add_w(output, output, tmp);
+ as_srai_w(output, output, 24);
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, scratch2);
+ ma_cmp_set(dest, scratch2, Imm32(uint8_t(rhs.value)), cond);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, scratch2);
+ ma_cmp_set(dest, scratch2, Imm32(int8_t(rhs.value)), cond);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load16ZeroExtend(lhs, scratch2);
+ ma_cmp_set(dest, scratch2, Imm32(uint16_t(rhs.value)), cond);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load16SignExtend(lhs, scratch2);
+ ma_cmp_set(dest, scratch2, Imm32(int16_t(rhs.value)), cond);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+// Also see below for specializations of cmpPtrSet.
+template <typename T1, typename T2>
+void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ ma_cmp_set(dest, lhs, rhs, cond);
+}
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ ma_cmp_set(dest, lhs, ImmWord(uint64_t(rhs.value)), cond);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ ma_cmp_set(dest, lhs, rhs, cond);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, scratch2);
+ branch32(cond, scratch2, Imm32(uint8_t(rhs.value)), label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, scratch2);
+ branch32(cond, scratch2, Imm32(int8_t(rhs.value)), label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ computeScaledAddress(lhs, scratch2);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(Address(scratch2, lhs.offset), scratch2);
+ branch32(cond, scratch2, rhs, label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(Address(scratch2, lhs.offset), scratch2);
+ branch32(cond, scratch2, rhs, label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load16ZeroExtend(lhs, scratch2);
+ branch32(cond, scratch2, Imm32(uint16_t(rhs.value)), label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load16SignExtend(lhs, scratch2);
+ branch32(cond, scratch2, Imm32(int16_t(rhs.value)), label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm,
+ L label) {
+ ma_b(lhs, imm, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(lhs, scratch2);
+ ma_b(scratch2, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(lhs, scratch2);
+ ma_b(scratch2, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(lhs, scratch2);
+ ma_b(scratch2, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(lhs, scratch2);
+ ma_b(scratch2, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(lhs, scratch2);
+ ma_b(scratch2, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress addr,
+ Imm32 imm, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(addr, scratch2);
+ ma_b(scratch2, imm, label, cond);
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan ||
+ cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan ||
+ cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, ImmWord(val.value), success);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan ||
+ cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan ||
+ cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, rhs.reg, success);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, ImmWord(val.value), label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, rhs.reg, label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ loadPtr(rhs, scratch);
+ branchPtr(cond, lhs, scratch, label);
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
+ Label* label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
+ Label* label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
+ Label* label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
+ L label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ ImmWord rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+ Register rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ Register rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ ImmWord rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchPtr(cond, scratch2, rhs, label);
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ branchPtr(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ ma_bc_s(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+ as_ftintrz_l_s(fpscratch, src);
+ as_movfcsr2gr(scratch);
+ moveFromDouble(fpscratch, dest);
+ MOZ_ASSERT(Assembler::CauseV < 32);
+ as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ ma_b(scratch, Imm32(0), fail, Assembler::NotEqual);
+
+ as_slli_w(dest, dest, 0);
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ convertFloat32ToInt32(src, dest, fail, false);
+}
+
+void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ ma_bc_d(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+ as_ftintrz_l_d(fpscratch, src);
+ as_movfcsr2gr(scratch);
+ moveFromDouble(fpscratch, dest);
+ MOZ_ASSERT(Assembler::CauseV < 32);
+ as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ ma_b(scratch, Imm32(0), fail, Assembler::NotEqual);
+
+ as_slli_w(dest, dest, 0);
+}
+
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ convertDoubleToInt32(src, dest, fail, false);
+}
+
+template <typename T>
+void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
+ Label* overflow) {
+ switch (cond) {
+ case Overflow:
+ ma_add32TestOverflow(dest, dest, src, overflow);
+ break;
+ case CarryClear:
+ case CarrySet:
+ ma_add32TestCarry(cond, dest, dest, src, overflow);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+// the type of 'T src' maybe a Register, maybe a Imm32,depends on who call it.
+template <typename T>
+void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
+ Label* overflow) {
+ switch (cond) {
+ case Overflow:
+ ma_sub32TestOverflow(dest, dest, src, overflow);
+ break;
+ case NonZero:
+ case Zero:
+ case Signed:
+ case NotSigned:
+ ma_sub_w(dest, dest, src);
+ ma_b(dest, dest, overflow, cond);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <typename T>
+void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
+ Label* overflow) {
+ MOZ_ASSERT(cond == Assembler::Overflow);
+ ma_mul32TestOverflow(dest, dest, src, overflow);
+}
+
+template <typename T>
+void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ rshift32(src, dest);
+ branch32(cond == Zero ? Equal : NotEqual, dest, Imm32(0), label);
+}
+
+void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
+ MOZ_ASSERT(cond == Overflow);
+ neg32(reg);
+ branch32(Assembler::Equal, reg, Imm32(INT32_MIN), label);
+}
+
+template <typename T>
+void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ switch (cond) {
+ case Overflow:
+ ma_addPtrTestOverflow(dest, dest, src, label);
+ break;
+ case CarryClear:
+ case CarrySet:
+ ma_addPtrTestCarry(cond, dest, dest, src, label);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <typename T>
+void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ switch (cond) {
+ case Overflow:
+ ma_subPtrTestOverflow(dest, dest, src, label);
+ break;
+ case NonZero:
+ case Zero:
+ case Signed:
+ case NotSigned:
+ subPtr(src, dest);
+ ma_b(dest, dest, label, cond);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Overflow);
+ ma_mulPtrTestOverflow(dest, dest, src, label);
+}
+
+void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ subPtr(rhs, lhs);
+ branchPtr(cond, lhs, Imm32(0), label);
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ if (lhs == rhs) {
+ ma_b(lhs, rhs, label, cond);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ as_and(scratch, lhs, rhs);
+ ma_b(scratch, scratch, label, cond);
+ }
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_and(scratch2, lhs, rhs);
+ ma_b(scratch2, scratch2, label, cond);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(lhs, scratch2);
+ and32(rhs, scratch2);
+ ma_b(scratch2, scratch2, label, cond);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(lhs, scratch2);
+ and32(rhs, scratch2);
+ ma_b(scratch2, scratch2, label, cond);
+}
+
+template <class L>
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ if (lhs == rhs) {
+ ma_b(lhs, rhs, label, cond);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ as_and(scratch, lhs, rhs);
+ ma_b(scratch, scratch, label, cond);
+ }
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ ScratchRegisterScope scratch(asMasm());
+ ma_and(scratch, lhs, rhs);
+ ma_b(scratch, scratch, label, cond);
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
+ Imm32 rhs, Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ branchTestPtr(cond, scratch2, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ branchTestPtr(cond, lhs.reg, rhs.reg, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_UNDEFINED), label, cond);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestUndefined(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestUndefined(cond, tag, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestUndefined(cond, tag, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_INT32), label, cond);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestInt32(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestInt32(cond, tag, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestInt32(cond, tag, label);
+}
+
+void MacroAssembler::branchTestInt32Truthy(bool b, const ValueOperand& value,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ as_bstrpick_d(scratch, value.valueReg(), 31, 0);
+ ma_b(scratch, scratch, label, b ? NonZero : Zero);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = (cond == Equal) ? BelowOrEqual : Above;
+ ma_b(tag, ImmTag(JSVAL_TAG_MAX_DOUBLE), label, actual);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestDouble(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestDouble(cond, tag, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestDouble(cond, tag, label);
+}
+
+void MacroAssembler::branchTestDoubleTruthy(bool b, FloatRegister value,
+ Label* label) {
+ ScratchDoubleScope fpscratch(*this);
+ ma_lid(fpscratch, 0.0);
+ DoubleCondition cond = b ? DoubleNotEqual : DoubleEqualOrUnordered;
+ ma_bc_d(value, fpscratch, label, cond);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = cond == Equal ? BelowOrEqual : Above;
+ ma_b(tag, ImmTag(JS::detail::ValueUpperInclNumberTag), label, actual);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestNumber(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_BOOLEAN), label, cond);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestBoolean(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestBoolean(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestBoolean(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ unboxBoolean(value, scratch2);
+ ma_b(scratch2, scratch2, label, b ? NonZero : Zero);
+}
+
+void MacroAssembler::branchTestString(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_STRING), label, cond);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestString(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestString(cond, tag, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestString(cond, tag, label);
+}
+
+void MacroAssembler::branchTestStringTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ unboxString(value, scratch2);
+ load32(Address(scratch2, JSString::offsetOfLength()), scratch2);
+ ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_SYMBOL), label, cond);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestSymbol(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestSymbol(cond, tag, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestSymbol(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_BIGINT), label, cond);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestBigInt(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestBigInt(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ computeEffectiveAddress(address, scratch2);
+ splitTag(scratch2, scratch2);
+ branchTestBigInt(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBigIntTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ unboxBigInt(value, scratch2);
+ load32(Address(scratch2, BigInt::offsetOfDigitLength()), scratch2);
+ ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_NULL), label, cond);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestNull(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestNull(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestNull(cond, tag, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_OBJECT), label, cond);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestObject(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestObject(cond, tag, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestObject(cond, tag, label);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestPrimitive(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond,
+ const ValueOperand& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& address,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ ma_b(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag), label,
+ (cond == Equal) ? AboveOrEqual : Below);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag), label,
+ (cond == Equal) ? Below : AboveOrEqual);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_MAGIC), label, cond);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestMagic(cond, tag, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestMagic(cond, tag, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
+ L label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ ma_b(scratch2, ImmTag(JSVAL_TAG_MAGIC), label, cond);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ uint64_t magic = MagicValue(why).asRawBits();
+ SecondScratchRegisterScope scratch(*this);
+ loadPtr(valaddr, scratch);
+ ma_b(scratch, ImmWord(magic), label, cond);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ branchPtr(cond, lhs, rhs.valueReg(), label);
+}
+
+template <typename T>
+void MacroAssembler::testNumberSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JS::detail::ValueUpperInclNumberTag),
+ cond == Equal ? BelowOrEqual : Above);
+}
+
+template <typename T>
+void MacroAssembler::testBooleanSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_BOOLEAN), cond);
+}
+
+template <typename T>
+void MacroAssembler::testStringSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_STRING), cond);
+}
+
+template <typename T>
+void MacroAssembler::testSymbolSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_SYMBOL), cond);
+}
+
+template <typename T>
+void MacroAssembler::testBigIntSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_BIGINT), cond);
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(addr, scratch2);
+ branch(scratch2);
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ cmp32Set(cond, lhs, rhs, scratch2);
+ moveIfNotZero(dest, src, scratch2);
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(lhs != scratch2 && src != scratch2 && dest != scratch2);
+ load32(rhs, scratch2);
+ cmp32Move32(cond, lhs, scratch2, src, dest);
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ cmp32Set(cond, lhs, rhs, scratch2);
+ moveIfNotZero(dest, src, scratch2);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ cmpPtrSet(cond, lhs, rhs, scratch2);
+ moveIfNotZero(dest, src, scratch2);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ MOZ_CRASH("NYI");
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
+ const Address& rhs, const Address& src,
+ Register dest) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(lhs != scratch && dest != scratch);
+ load32(rhs, scratch);
+ cmp32Load32(cond, lhs, scratch, src, dest);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
+ const Address& src, Register dest) {
+ Label skip;
+ branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
+ load32(src, dest);
+ bind(&skip);
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ Label skip;
+ branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
+ loadPtr(src, dest);
+ bind(&skip);
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+ Imm32 mask, const Address& src,
+ Register dest) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreStringMitigations);
+ Label skip;
+ branchTest32(Assembler::InvertCondition(cond), addr, mask, &skip);
+ loadPtr(src, dest);
+ bind(&skip);
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+ Imm32 mask, Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
+ branchPtr(Assembler::BelowOrEqual, length, index, failure);
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+ const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
+ branchPtr(Assembler::BelowOrEqual, length, index, failure);
+}
+
+// ========================================================================
+// Memory access primitives.
+
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const Address& addr) {
+ ma_fst_s(src, addr);
+}
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const BaseIndex& addr) {
+ ma_fst_s(src, addr);
+}
+
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const Address& addr) {
+ ma_fst_d(src, addr);
+}
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const BaseIndex& addr) {
+ ma_fst_d(src, addr);
+}
+
+void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) {
+ if (barrier) {
+ as_dbar(0);
+ }
+}
+
+// ===============================================================
+// Clamping functions.
+
+void MacroAssembler::clampIntToUint8(Register reg) {
+ ScratchRegisterScope scratch(*this);
+ // If reg is < 0, then we want to clamp to 0.
+ as_slti(scratch, reg, 0);
+ as_masknez(reg, reg, scratch);
+
+ // If reg is >= 255, then we want to clamp to 255.
+ as_addi_d(reg, reg, -255);
+ as_slt(scratch, reg, zero);
+ as_maskeqz(reg, reg, scratch);
+ as_addi_d(reg, reg, 255);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) {
+ MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
+ type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
+ // dest := src XOR mask
+ // scratch := dest >> JSVAL_TAG_SHIFT
+ // fail if scratch != 0
+ //
+ // Note: src and dest can be the same register
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(src.valueReg() != scratch);
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ as_xor(dest, src.valueReg(), scratch);
+ as_srli_d(scratch, dest, JSVAL_TAG_SHIFT);
+ ma_b(scratch, Imm32(0), fail, Assembler::NotEqual);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) {
+ loadValue(src, ValueOperand(dest));
+ fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) {
+ loadValue(src, ValueOperand(dest));
+ fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+// The specializations for cmpPtrSet are outside the braces because
+// check_macroassembler_style can't yet deal with specializations.
+
+template <>
+inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs,
+ ImmPtr rhs, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(lhs, scratch2);
+ cmpPtrSet(cond, Register(scratch2), rhs, dest);
+}
+
+template <>
+inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Register lhs,
+ Address rhs, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ loadPtr(rhs, scratch);
+ cmpPtrSet(cond, lhs, Register(scratch), dest);
+}
+
+template <>
+inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs,
+ Register rhs, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(rhs != scratch2);
+ loadPtr(lhs, scratch2);
+ cmpPtrSet(cond, Register(scratch2), rhs, dest);
+}
+
+template <>
+inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Register lhs,
+ Address rhs, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ load32(rhs, scratch);
+ cmp32Set(cond, lhs, Register(scratch), dest);
+}
+
+template <>
+inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Address lhs,
+ Register rhs, Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(rhs != scratch2);
+ load32(lhs, scratch2);
+ cmp32Set(cond, Register(scratch2), rhs, dest);
+}
+
+void MacroAssemblerLOONG64Compat::incrementInt32Value(const Address& addr) {
+ asMasm().add32(Imm32(1), addr);
+}
+
+void MacroAssemblerLOONG64Compat::retn(Imm32 n) {
+ // pc <- [sp]; sp += n
+ loadPtr(Address(StackPointer, 0), ra);
+ asMasm().addPtr(n, StackPointer);
+ as_jirl(zero, ra, BOffImm16(0));
+}
+
+// If source is a double, load into dest.
+// If source is int32, convert to double and store in dest.
+// Else, branch to failure.
+void MacroAssemblerLOONG64Compat::ensureDouble(const ValueOperand& source,
+ FloatRegister dest,
+ Label* failure) {
+ Label isDouble, done;
+ {
+ ScratchTagScope tag(asMasm(), source);
+ splitTagForTest(source, tag);
+ asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, tag, failure);
+ }
+
+ convertInt32ToDouble(source.valueReg(), dest);
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_MacroAssembler_loong64_inl_h */
diff --git a/js/src/jit/loong64/MacroAssembler-loong64.cpp b/js/src/jit/loong64/MacroAssembler-loong64.cpp
new file mode 100644
index 0000000000..41367a2e76
--- /dev/null
+++ b/js/src/jit/loong64/MacroAssembler-loong64.cpp
@@ -0,0 +1,5388 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/loong64/MacroAssembler-loong64.h"
+
+#include "jsmath.h"
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/loong64/SharedICRegisters-loong64.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MoveEmitter.h"
+#include "util/Memory.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+ as_ftintrne_l_d(fpscratch, input);
+ as_movfr2gr_d(output, fpscratch);
+ // if (res < 0); res = 0;
+ as_slt(scratch, output, zero);
+ as_masknez(output, output, scratch);
+ // if res > 255; res = 255;
+ as_sltui(scratch, output, 255);
+ as_addi_d(output, output, -255);
+ as_maskeqz(output, output, scratch);
+ as_addi_d(output, output, 255);
+}
+
+bool MacroAssemblerLOONG64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
+ asMasm().PushFrameDescriptor(FrameType::IonJS); // descriptor_
+ asMasm().Push(ImmPtr(fakeReturnAddr));
+ asMasm().Push(FramePointer);
+ return true;
+}
+
+void MacroAssemblerLOONG64Compat::convertUInt32ToDouble(Register src,
+ FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+ as_bstrpick_d(scratch, src, 31, 0);
+ asMasm().convertInt64ToDouble(Register64(scratch), dest);
+}
+
+void MacroAssemblerLOONG64Compat::convertUInt64ToDouble(Register src,
+ FloatRegister dest) {
+ Label positive, done;
+ ma_b(src, src, &positive, NotSigned, ShortJump);
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ MOZ_ASSERT(src != scratch);
+ MOZ_ASSERT(src != scratch2);
+
+ ma_and(scratch, src, Imm32(1));
+ as_srli_d(scratch2, src, 1);
+ as_or(scratch, scratch, scratch2);
+ as_movgr2fr_d(dest, scratch);
+ as_ffint_d_l(dest, dest);
+ asMasm().addDouble(dest, dest);
+ ma_b(&done, ShortJump);
+
+ bind(&positive);
+ as_movgr2fr_d(dest, src);
+ as_ffint_d_l(dest, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerLOONG64Compat::convertUInt32ToFloat32(Register src,
+ FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+ as_bstrpick_d(scratch, src, 31, 0);
+ asMasm().convertInt64ToFloat32(Register64(scratch), dest);
+}
+
+void MacroAssemblerLOONG64Compat::convertDoubleToFloat32(FloatRegister src,
+ FloatRegister dest) {
+ as_fcvt_s_d(dest, src);
+}
+
+const int CauseBitPos = int(Assembler::CauseI);
+const int CauseBitCount = 1 + int(Assembler::CauseV) - int(Assembler::CauseI);
+const int CauseIOrVMask = ((1 << int(Assembler::CauseI)) |
+ (1 << int(Assembler::CauseV))) >>
+ int(Assembler::CauseI);
+
+// Checks whether a double is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void MacroAssemblerLOONG64Compat::convertDoubleToInt32(FloatRegister src,
+ Register dest,
+ Label* fail,
+ bool negativeZeroCheck) {
+ if (negativeZeroCheck) {
+ moveFromDouble(src, dest);
+ as_rotri_d(dest, dest, 63);
+ ma_b(dest, Imm32(1), fail, Assembler::Equal);
+ }
+
+ ScratchRegisterScope scratch(asMasm());
+ ScratchFloat32Scope fpscratch(asMasm());
+ // Truncate double to int ; if result is inexact or invalid fail.
+ as_ftintrz_w_d(fpscratch, src);
+ as_movfcsr2gr(scratch);
+ moveFromFloat32(fpscratch, dest);
+ as_bstrpick_d(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos);
+ as_andi(scratch, scratch,
+ CauseIOrVMask); // masking for Inexact and Invalid flag.
+ ma_b(scratch, zero, fail, Assembler::NotEqual);
+}
+
+void MacroAssemblerLOONG64Compat::convertDoubleToPtr(FloatRegister src,
+ Register dest, Label* fail,
+ bool negativeZeroCheck) {
+ if (negativeZeroCheck) {
+ moveFromDouble(src, dest);
+ as_rotri_d(dest, dest, 63);
+ ma_b(dest, Imm32(1), fail, Assembler::Equal);
+ }
+
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+ // Truncate double to int64 ; if result is inexact or invalid fail.
+ as_ftintrz_l_d(fpscratch, src);
+ as_movfcsr2gr(scratch);
+ moveFromDouble(fpscratch, dest);
+ as_bstrpick_d(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos);
+ as_andi(scratch, scratch,
+ CauseIOrVMask); // masking for Inexact and Invalid flag.
+ ma_b(scratch, zero, fail, Assembler::NotEqual);
+}
+
+// Checks whether a float32 is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void MacroAssemblerLOONG64Compat::convertFloat32ToInt32(
+ FloatRegister src, Register dest, Label* fail, bool negativeZeroCheck) {
+ if (negativeZeroCheck) {
+ moveFromFloat32(src, dest);
+ ma_b(dest, Imm32(INT32_MIN), fail, Assembler::Equal);
+ }
+
+ ScratchRegisterScope scratch(asMasm());
+ ScratchFloat32Scope fpscratch(asMasm());
+ as_ftintrz_w_s(fpscratch, src);
+ as_movfcsr2gr(scratch);
+ moveFromFloat32(fpscratch, dest);
+ MOZ_ASSERT(CauseBitPos + CauseBitCount < 33);
+ MOZ_ASSERT(CauseBitPos < 32);
+ as_bstrpick_w(scratch, scratch, CauseBitPos + CauseBitCount - 1, CauseBitPos);
+ as_andi(scratch, scratch, CauseIOrVMask);
+ ma_b(scratch, zero, fail, Assembler::NotEqual);
+}
+
+void MacroAssemblerLOONG64Compat::convertFloat32ToDouble(FloatRegister src,
+ FloatRegister dest) {
+ as_fcvt_d_s(dest, src);
+}
+
+void MacroAssemblerLOONG64Compat::convertInt32ToFloat32(Register src,
+ FloatRegister dest) {
+ as_movgr2fr_w(dest, src);
+ as_ffint_s_w(dest, dest);
+}
+
+void MacroAssemblerLOONG64Compat::convertInt32ToFloat32(const Address& src,
+ FloatRegister dest) {
+ ma_fld_s(dest, src);
+ as_ffint_s_w(dest, dest);
+}
+
+void MacroAssemblerLOONG64Compat::movq(Register rj, Register rd) {
+ as_or(rd, rj, zero);
+}
+
+void MacroAssemblerLOONG64::ma_li(Register dest, CodeLabel* label) {
+ BufferOffset bo = m_buffer.nextOffset();
+ ma_liPatchable(dest, ImmWord(/* placeholder */ 0));
+ label->patchAt()->bind(bo.getOffset());
+ label->setLinkMode(CodeLabel::MoveImmediate);
+}
+
+void MacroAssemblerLOONG64::ma_li(Register dest, ImmWord imm) {
+ int64_t value = imm.value;
+
+ if (-1 == (value >> 11) || 0 == (value >> 11)) {
+ as_addi_w(dest, zero, value);
+ return;
+ }
+
+ if (0 == (value >> 12)) {
+ as_ori(dest, zero, value);
+ return;
+ }
+
+ if (-1 == (value >> 31) || 0 == (value >> 31)) {
+ as_lu12i_w(dest, (value >> 12) & 0xfffff);
+ } else if (0 == (value >> 32)) {
+ as_lu12i_w(dest, (value >> 12) & 0xfffff);
+ as_bstrins_d(dest, zero, 63, 32);
+ } else if (-1 == (value >> 51) || 0 == (value >> 51)) {
+ if (is_uintN((value >> 12) & 0xfffff, 20)) {
+ as_lu12i_w(dest, (value >> 12) & 0xfffff);
+ }
+ as_lu32i_d(dest, (value >> 32) & 0xfffff);
+ } else if (0 == (value >> 52)) {
+ if (is_uintN((value >> 12) & 0xfffff, 20)) {
+ as_lu12i_w(dest, (value >> 12) & 0xfffff);
+ }
+ as_lu32i_d(dest, (value >> 32) & 0xfffff);
+ as_bstrins_d(dest, zero, 63, 52);
+ } else {
+ if (is_uintN((value >> 12) & 0xfffff, 20)) {
+ as_lu12i_w(dest, (value >> 12) & 0xfffff);
+ }
+ if (is_uintN((value >> 32) & 0xfffff, 20)) {
+ as_lu32i_d(dest, (value >> 32) & 0xfffff);
+ }
+ as_lu52i_d(dest, dest, (value >> 52) & 0xfff);
+ }
+
+ if (is_uintN(value & 0xfff, 12)) {
+ as_ori(dest, dest, value & 0xfff);
+ }
+}
+
+// This method generates lu32i_d, lu12i_w and ori instruction block that can be
+// modified by UpdateLoad64Value, either during compilation (eg.
+// Assembler::bind), or during execution (eg. jit::PatchJump).
+void MacroAssemblerLOONG64::ma_liPatchable(Register dest, ImmPtr imm) {
+ return ma_liPatchable(dest, ImmWord(uintptr_t(imm.value)));
+}
+
+void MacroAssemblerLOONG64::ma_liPatchable(Register dest, ImmWord imm,
+ LiFlags flags) {
+ // hi12, hi20, low20, low12
+ if (Li64 == flags) { // Li64: Imm data
+ m_buffer.ensureSpace(4 * sizeof(uint32_t));
+ as_lu12i_w(dest, imm.value >> 12 & 0xfffff); // low20
+ as_ori(dest, dest, imm.value & 0xfff); // low12
+ as_lu32i_d(dest, imm.value >> 32 & 0xfffff); // hi20
+ as_lu52i_d(dest, dest, imm.value >> 52 & 0xfff); // hi12
+ } else { // Li48 address
+ m_buffer.ensureSpace(3 * sizeof(uint32_t));
+ as_lu12i_w(dest, imm.value >> 12 & 0xfffff); // low20
+ as_ori(dest, dest, imm.value & 0xfff); // low12
+ as_lu32i_d(dest, imm.value >> 32 & 0xfffff); // hi20
+ }
+}
+
+// Memory access ops.
+
+void MacroAssemblerLOONG64::ma_ld_b(Register dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_ld_b(dest, base, offset);
+ } else if (base != dest) {
+ ma_li(dest, Imm32(offset));
+ as_ldx_b(dest, base, dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_ldx_b(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_ld_bu(Register dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_ld_bu(dest, base, offset);
+ } else if (base != dest) {
+ ma_li(dest, Imm32(offset));
+ as_ldx_bu(dest, base, dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_ldx_bu(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_ld_h(Register dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_ld_h(dest, base, offset);
+ } else if (base != dest) {
+ ma_li(dest, Imm32(offset));
+ as_ldx_h(dest, base, dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_ldx_h(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_ld_hu(Register dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_ld_hu(dest, base, offset);
+ } else if (base != dest) {
+ ma_li(dest, Imm32(offset));
+ as_ldx_hu(dest, base, dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_ldx_hu(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_ld_w(Register dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_ld_w(dest, base, offset);
+ } else if (base != dest) {
+ ma_li(dest, Imm32(offset));
+ as_ldx_w(dest, base, dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_ldx_w(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_ld_wu(Register dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_ld_wu(dest, base, offset);
+ } else if (base != dest) {
+ ma_li(dest, Imm32(offset));
+ as_ldx_wu(dest, base, dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_ldx_wu(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_ld_d(Register dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_ld_d(dest, base, offset);
+ } else if (base != dest) {
+ ma_li(dest, Imm32(offset));
+ as_ldx_d(dest, base, dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_ldx_d(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_st_b(Register src, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_st_b(src, base, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(src != scratch);
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_stx_b(src, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_st_h(Register src, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_st_h(src, base, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(src != scratch);
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_stx_h(src, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_st_w(Register src, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_st_w(src, base, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(src != scratch);
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_stx_w(src, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_st_d(Register src, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_st_d(src, base, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(src != scratch);
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_stx_d(src, base, scratch);
+ }
+}
+
+// Arithmetic-based ops.
+
+// Add.
+void MacroAssemblerLOONG64::ma_add_d(Register rd, Register rj, Imm32 imm) {
+ if (is_intN(imm.value, 12)) {
+ as_addi_d(rd, rj, imm.value);
+ } else if (rd != rj) {
+ ma_li(rd, imm);
+ as_add_d(rd, rj, rd);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_add_d(rd, rj, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_add32TestOverflow(Register rd, Register rj,
+ Register rk, Label* overflow) {
+ ScratchRegisterScope scratch(asMasm());
+ as_add_d(scratch, rj, rk);
+ as_add_w(rd, rj, rk);
+ ma_b(rd, Register(scratch), overflow, Assembler::NotEqual);
+}
+
+void MacroAssemblerLOONG64::ma_add32TestOverflow(Register rd, Register rj,
+ Imm32 imm, Label* overflow) {
+ // Check for signed range because of as_addi_d
+ if (is_intN(imm.value, 12)) {
+ ScratchRegisterScope scratch(asMasm());
+ as_addi_d(scratch, rj, imm.value);
+ as_addi_w(rd, rj, imm.value);
+ ma_b(rd, scratch, overflow, Assembler::NotEqual);
+ } else {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch2, imm);
+ ma_add32TestOverflow(rd, rj, scratch2, overflow);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_addPtrTestOverflow(Register rd, Register rj,
+ Register rk,
+ Label* overflow) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rd != scratch);
+
+ if (rj == rk) {
+ if (rj == rd) {
+ as_or(scratch, rj, zero);
+ rj = scratch;
+ }
+
+ as_add_d(rd, rj, rj);
+ as_xor(scratch, rj, rd);
+ ma_b(scratch, zero, overflow, Assembler::LessThan);
+ } else {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ MOZ_ASSERT(rd != scratch2);
+
+ if (rj == rd) {
+ as_or(scratch2, rj, zero);
+ rj = scratch2;
+ }
+
+ as_add_d(rd, rj, rk);
+ as_slti(scratch, rj, 0);
+ as_slt(scratch2, rd, rj);
+ ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_addPtrTestOverflow(Register rd, Register rj,
+ Imm32 imm, Label* overflow) {
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (imm.value == 0) {
+ as_ori(rd, rj, 0);
+ return;
+ }
+
+ if (rj == rd) {
+ as_ori(scratch2, rj, 0);
+ rj = scratch2;
+ }
+
+ ma_add_d(rd, rj, imm);
+
+ if (imm.value > 0) {
+ ma_b(rd, rj, overflow, Assembler::LessThan);
+ } else {
+ MOZ_ASSERT(imm.value < 0);
+ ma_b(rd, rj, overflow, Assembler::GreaterThan);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_addPtrTestOverflow(Register rd, Register rj,
+ ImmWord imm,
+ Label* overflow) {
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ if (imm.value == 0) {
+ as_ori(rd, rj, 0);
+ return;
+ }
+
+ if (rj == rd) {
+ MOZ_ASSERT(rj != scratch2);
+ as_ori(scratch2, rj, 0);
+ rj = scratch2;
+ }
+
+ ma_li(rd, imm);
+ as_add_d(rd, rj, rd);
+
+ if (imm.value > 0) {
+ ma_b(rd, rj, overflow, Assembler::LessThan);
+ } else {
+ MOZ_ASSERT(imm.value < 0);
+ ma_b(rd, rj, overflow, Assembler::GreaterThan);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_addPtrTestCarry(Condition cond, Register rd,
+ Register rj, Register rk,
+ Label* label) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rd != rk);
+ MOZ_ASSERT(rd != scratch);
+ as_add_d(rd, rj, rk);
+ as_sltu(scratch, rd, rk);
+ ma_b(scratch, Register(scratch), label,
+ cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
+}
+
+void MacroAssemblerLOONG64::ma_addPtrTestCarry(Condition cond, Register rd,
+ Register rj, Imm32 imm,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ // Check for signed range because of as_addi_d
+ if (is_intN(imm.value, 12)) {
+ as_addi_d(rd, rj, imm.value);
+ as_sltui(scratch2, rd, imm.value);
+ ma_b(scratch2, scratch2, label,
+ cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
+ } else {
+ ma_li(scratch2, imm);
+ ma_addPtrTestCarry(cond, rd, rj, scratch2, label);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_addPtrTestCarry(Condition cond, Register rd,
+ Register rj, ImmWord imm,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ // Check for signed range because of as_addi_d
+ if (is_intN(imm.value, 12)) {
+ as_addi_d(rd, rj, imm.value);
+ as_sltui(scratch2, rd, imm.value);
+ ma_b(scratch2, scratch2, label,
+ cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
+ } else {
+ ma_li(scratch2, imm);
+ ma_addPtrTestCarry(cond, rd, rj, scratch2, label);
+ }
+}
+
+// Subtract.
+void MacroAssemblerLOONG64::ma_sub_d(Register rd, Register rj, Imm32 imm) {
+ if (is_intN(-imm.value, 12)) {
+ as_addi_d(rd, rj, -imm.value);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ as_sub_d(rd, rj, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_sub32TestOverflow(Register rd, Register rj,
+ Register rk, Label* overflow) {
+ ScratchRegisterScope scratch(asMasm());
+ as_sub_d(scratch, rj, rk);
+ as_sub_w(rd, rj, rk);
+ ma_b(rd, Register(scratch), overflow, Assembler::NotEqual);
+}
+
+void MacroAssemblerLOONG64::ma_subPtrTestOverflow(Register rd, Register rj,
+ Register rk,
+ Label* overflow) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT_IF(rj == rd, rj != rk);
+ MOZ_ASSERT(rj != scratch2);
+ MOZ_ASSERT(rk != scratch2);
+ MOZ_ASSERT(rd != scratch2);
+
+ Register rj_copy = rj;
+
+ if (rj == rd) {
+ as_or(scratch2, rj, zero);
+ rj_copy = scratch2;
+ }
+
+ {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rd != scratch);
+
+ as_sub_d(rd, rj, rk);
+ // If the sign of rj and rk are the same, no overflow
+ as_xor(scratch, rj_copy, rk);
+ // Check if the sign of rd and rj are the same
+ as_xor(scratch2, rd, rj_copy);
+ as_and(scratch2, scratch2, scratch);
+ }
+
+ ma_b(scratch2, zero, overflow, Assembler::LessThan);
+}
+
+void MacroAssemblerLOONG64::ma_subPtrTestOverflow(Register rd, Register rj,
+ Imm32 imm, Label* overflow) {
+ // TODO(loong64): Check subPtrTestOverflow
+ MOZ_ASSERT(imm.value != INT32_MIN);
+ ma_addPtrTestOverflow(rd, rj, Imm32(-imm.value), overflow);
+}
+
+void MacroAssemblerLOONG64::ma_mul_d(Register rd, Register rj, Imm32 imm) {
+ // li handles the relocation.
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_mul_d(rd, rj, scratch);
+}
+
+void MacroAssemblerLOONG64::ma_mulh_d(Register rd, Register rj, Imm32 imm) {
+ // li handles the relocation.
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_mulh_d(rd, rj, scratch);
+}
+
+void MacroAssemblerLOONG64::ma_mulPtrTestOverflow(Register rd, Register rj,
+ Register rk,
+ Label* overflow) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(rd != scratch);
+
+ if (rd == rj) {
+ as_or(scratch, rj, zero);
+ rj = scratch;
+ rk = (rd == rk) ? rj : rk;
+ } else if (rd == rk) {
+ as_or(scratch, rk, zero);
+ rk = scratch;
+ }
+
+ as_mul_d(rd, rj, rk);
+ as_mulh_d(scratch, rj, rk);
+ as_srai_d(scratch2, rd, 63);
+ ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual);
+}
+
+// Memory.
+
+void MacroAssemblerLOONG64::ma_load(Register dest, Address address,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ int32_t encodedOffset;
+ Register base;
+
+ // TODO: use as_ldx_b/h/w/d, could decrease as_add_d instr.
+ switch (size) {
+ case SizeByte:
+ case SizeHalfWord:
+ if (!is_intN(address.offset, 12)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_add_d(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = 0;
+ } else {
+ encodedOffset = address.offset;
+ base = address.base;
+ }
+
+ if (size == SizeByte) {
+ if (ZeroExtend == extension) {
+ as_ld_bu(dest, base, encodedOffset);
+ } else {
+ as_ld_b(dest, base, encodedOffset);
+ }
+ } else {
+ if (ZeroExtend == extension) {
+ as_ld_hu(dest, base, encodedOffset);
+ } else {
+ as_ld_h(dest, base, encodedOffset);
+ }
+ }
+ break;
+ case SizeWord:
+ case SizeDouble:
+ if ((address.offset & 0x3) == 0 &&
+ (size == SizeDouble ||
+ (size == SizeWord && SignExtend == extension))) {
+ if (!Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_add_d(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = 0;
+ } else {
+ encodedOffset = address.offset;
+ base = address.base;
+ }
+
+ if (size == SizeWord) {
+ as_ldptr_w(dest, base, encodedOffset);
+ } else {
+ as_ldptr_d(dest, base, encodedOffset);
+ }
+ } else {
+ if (!is_intN(address.offset, 12)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_add_d(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = 0;
+ } else {
+ encodedOffset = address.offset;
+ base = address.base;
+ }
+
+ if (size == SizeWord) {
+ if (ZeroExtend == extension) {
+ as_ld_wu(dest, base, encodedOffset);
+ } else {
+ as_ld_w(dest, base, encodedOffset);
+ }
+ } else {
+ as_ld_d(dest, base, encodedOffset);
+ }
+ }
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+}
+
+void MacroAssemblerLOONG64::ma_store(Register data, Address address,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ int32_t encodedOffset;
+ Register base;
+
+ // TODO: use as_stx_b/h/w/d, could decrease as_add_d instr.
+ switch (size) {
+ case SizeByte:
+ case SizeHalfWord:
+ if (!is_intN(address.offset, 12)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_add_d(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = 0;
+ } else {
+ encodedOffset = address.offset;
+ base = address.base;
+ }
+
+ if (size == SizeByte) {
+ as_st_b(data, base, encodedOffset);
+ } else {
+ as_st_h(data, base, encodedOffset);
+ }
+ break;
+ case SizeWord:
+ case SizeDouble:
+ if ((address.offset & 0x3) == 0) {
+ if (!Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_add_d(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = 0;
+ } else {
+ encodedOffset = address.offset;
+ base = address.base;
+ }
+
+ if (size == SizeWord) {
+ as_stptr_w(data, base, encodedOffset);
+ } else {
+ as_stptr_d(data, base, encodedOffset);
+ }
+ } else {
+ if (!is_intN(address.offset, 12)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_add_d(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = 0;
+ } else {
+ encodedOffset = address.offset;
+ base = address.base;
+ }
+
+ if (size == SizeWord) {
+ as_st_w(data, base, encodedOffset);
+ } else {
+ as_st_d(data, base, encodedOffset);
+ }
+ }
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+}
+
+void MacroAssemblerLOONG64Compat::computeScaledAddress(const BaseIndex& address,
+ Register dest) {
+ Register base = address.base;
+ Register index = address.index;
+ int32_t shift = Imm32::ShiftOf(address.scale).value;
+
+ if (shift) {
+ MOZ_ASSERT(shift <= 4);
+ as_alsl_d(dest, index, base, shift - 1);
+ } else {
+ as_add_d(dest, base, index);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_pop(Register r) {
+ MOZ_ASSERT(r != StackPointer);
+ as_ld_d(r, StackPointer, 0);
+ as_addi_d(StackPointer, StackPointer, sizeof(intptr_t));
+}
+
+void MacroAssemblerLOONG64::ma_push(Register r) {
+ if (r == StackPointer) {
+ ScratchRegisterScope scratch(asMasm());
+ as_or(scratch, r, zero);
+ as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(intptr_t));
+ as_st_d(scratch, StackPointer, 0);
+ } else {
+ as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(intptr_t));
+ as_st_d(r, StackPointer, 0);
+ }
+}
+
+// Branches when done from within loongarch-specific code.
+void MacroAssemblerLOONG64::ma_b(Register lhs, ImmWord imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ if (imm.value <= INT32_MAX) {
+ ma_b(lhs, Imm32(uint32_t(imm.value)), label, c, jumpKind);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ ma_li(scratch, imm);
+ ma_b(lhs, Register(scratch), label, c, jumpKind);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_b(Register lhs, Address addr, Label* label,
+ Condition c, JumpKind jumpKind) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ ma_ld_d(scratch, addr);
+ ma_b(lhs, Register(scratch), label, c, jumpKind);
+}
+
+void MacroAssemblerLOONG64::ma_b(Address addr, Imm32 imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ld_d(scratch2, addr);
+ ma_b(Register(scratch2), imm, label, c, jumpKind);
+}
+
+void MacroAssemblerLOONG64::ma_b(Address addr, ImmGCPtr imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ld_d(scratch2, addr);
+ ma_b(Register(scratch2), imm, label, c, jumpKind);
+}
+
+void MacroAssemblerLOONG64::ma_bl(Label* label) {
+ spew("branch .Llabel %p\n", label);
+ if (label->bound()) {
+ // Generate the long jump for calls because return address has to be
+ // the address after the reserved block.
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ScratchRegisterScope scratch(asMasm());
+ ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET));
+ as_jirl(ra, scratch, BOffImm16(0));
+ return;
+ }
+
+ // Second word holds a pointer to the next branch in label's chain.
+ uint32_t nextInChain =
+ label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+
+ // Make the whole branch continous in the buffer. The '5'
+ // instructions are writing at below.
+ m_buffer.ensureSpace(5 * sizeof(uint32_t));
+
+ spew("bal .Llabel %p\n", label);
+ BufferOffset bo = writeInst(getBranchCode(BranchIsCall).encode());
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ // Leave space for long jump.
+ as_nop();
+ as_nop();
+ as_nop();
+}
+
+void MacroAssemblerLOONG64::branchWithCode(InstImm code, Label* label,
+ JumpKind jumpKind) {
+ // simply output the pointer of one label as its id,
+ // notice that after one label destructor, the pointer will be reused.
+ spew("branch .Llabel %p", label);
+ MOZ_ASSERT(code.encode() !=
+ InstImm(op_jirl, BOffImm16(0), zero, ra).encode());
+ InstImm inst_beq = InstImm(op_beq, BOffImm16(0), zero, zero);
+
+ if (label->bound()) {
+ int32_t offset = label->offset() - m_buffer.nextOffset().getOffset();
+
+ if (BOffImm16::IsInRange(offset)) {
+ jumpKind = ShortJump;
+ }
+
+ // ShortJump
+ if (jumpKind == ShortJump) {
+ MOZ_ASSERT(BOffImm16::IsInRange(offset));
+
+ if (code.extractBitField(31, 26) == ((uint32_t)op_bcz >> 26)) {
+ code.setImm21(offset);
+ } else {
+ code.setBOffImm16(BOffImm16(offset));
+ }
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ writeInst(code.encode());
+ return;
+ }
+
+ // LongJump
+ if (code.encode() == inst_beq.encode()) {
+ // Handle long jump
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ScratchRegisterScope scratch(asMasm());
+ ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET));
+ as_jirl(zero, scratch, BOffImm16(0)); // jr scratch
+ as_nop();
+ return;
+ }
+
+ // OpenLongJump
+ // Handle long conditional branch, the target offset is based on self,
+ // point to next instruction of nop at below.
+ spew("invert branch .Llabel %p", label);
+ InstImm code_r = invertBranch(code, BOffImm16(5 * sizeof(uint32_t)));
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code_r);
+#endif
+ writeInst(code_r.encode());
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ScratchRegisterScope scratch(asMasm());
+ ma_liPatchable(scratch, ImmWord(LabelBase::INVALID_OFFSET));
+ as_jirl(zero, scratch, BOffImm16(0));
+ as_nop();
+ return;
+ }
+
+ // Generate open jump and link it to a label.
+
+ // Second word holds a pointer to the next branch in label's chain.
+ uint32_t nextInChain =
+ label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+
+ if (jumpKind == ShortJump) {
+ // Make the whole branch continous in the buffer.
+ m_buffer.ensureSpace(2 * sizeof(uint32_t));
+
+ // Indicate that this is short jump with offset 4.
+ code.setBOffImm16(BOffImm16(4));
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ BufferOffset bo = writeInst(code.encode());
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ return;
+ }
+
+ bool conditional = code.encode() != inst_beq.encode();
+
+ // Make the whole branch continous in the buffer. The '5'
+ // instructions are writing at below (contain conditional nop).
+ m_buffer.ensureSpace(5 * sizeof(uint32_t));
+
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ BufferOffset bo = writeInst(code.encode()); // invert
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ // Leave space for potential long jump.
+ as_nop();
+ as_nop();
+ if (conditional) {
+ as_nop();
+ }
+}
+
+void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, ImmWord imm,
+ Condition c) {
+ if (imm.value <= INT32_MAX) {
+ ma_cmp_set(rd, rj, Imm32(uint32_t(imm.value)), c);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ ma_cmp_set(rd, rj, scratch, c);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, ImmPtr imm,
+ Condition c) {
+ ma_cmp_set(rd, rj, ImmWord(uintptr_t(imm.value)), c);
+}
+
+void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Address address, Imm32 imm,
+ Condition c) {
+ // TODO(loong64): 32-bit ma_cmp_set?
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ld_w(scratch2, address);
+ ma_cmp_set(rd, Register(scratch2), imm, c);
+}
+
+void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Address address,
+ ImmWord imm, Condition c) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_ld_d(scratch2, address);
+ ma_cmp_set(rd, Register(scratch2), imm, c);
+}
+
+// fp instructions
+void MacroAssemblerLOONG64::ma_lid(FloatRegister dest, double value) {
+ ImmWord imm(mozilla::BitwiseCast<uint64_t>(value));
+
+ if (imm.value != 0) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ moveToDouble(scratch, dest);
+ } else {
+ moveToDouble(zero, dest);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_mv(FloatRegister src, ValueOperand dest) {
+ as_movfr2gr_d(dest.valueReg(), src);
+}
+
+void MacroAssemblerLOONG64::ma_mv(ValueOperand src, FloatRegister dest) {
+ as_movgr2fr_d(dest, src.valueReg());
+}
+
+void MacroAssemblerLOONG64::ma_fld_s(FloatRegister dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_fld_s(dest, base, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_fldx_s(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_fld_d(FloatRegister dest, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_fld_d(dest, base, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_fldx_d(dest, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_fst_s(FloatRegister src, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_fst_s(src, base, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_fstx_s(src, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_fst_d(FloatRegister src, Address address) {
+ int32_t offset = address.offset;
+ Register base = address.base;
+
+ if (is_intN(offset, 12)) {
+ as_fst_d(src, base, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(base != scratch);
+ ma_li(scratch, Imm32(offset));
+ as_fstx_d(src, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_pop(FloatRegister f) {
+ as_fld_d(f, StackPointer, 0);
+ as_addi_d(StackPointer, StackPointer, sizeof(double));
+}
+
+void MacroAssemblerLOONG64::ma_push(FloatRegister f) {
+ as_addi_d(StackPointer, StackPointer, (int32_t) - sizeof(double));
+ as_fst_d(f, StackPointer, 0);
+}
+
+void MacroAssemblerLOONG64::ma_li(Register dest, ImmGCPtr ptr) {
+ writeDataRelocation(ptr);
+ asMasm().ma_liPatchable(dest, ImmPtr(ptr.value));
+}
+
+void MacroAssemblerLOONG64::ma_li(Register dest, Imm32 imm) {
+ if (is_intN(imm.value, 12)) {
+ as_addi_w(dest, zero, imm.value);
+ } else if (is_uintN(imm.value, 12)) {
+ as_ori(dest, zero, imm.value & 0xfff);
+ } else {
+ as_lu12i_w(dest, imm.value >> 12 & 0xfffff);
+ if (imm.value & 0xfff) {
+ as_ori(dest, dest, imm.value & 0xfff);
+ }
+ }
+}
+
+// This method generates lu12i_w and ori instruction pair that can be modified
+// by UpdateLuiOriValue, either during compilation (eg. Assembler::bind), or
+// during execution (eg. jit::PatchJump).
+void MacroAssemblerLOONG64::ma_liPatchable(Register dest, Imm32 imm) {
+ m_buffer.ensureSpace(2 * sizeof(uint32_t));
+ as_lu12i_w(dest, imm.value >> 12 & 0xfffff);
+ as_ori(dest, dest, imm.value & 0xfff);
+}
+
+void MacroAssemblerLOONG64::ma_fmovz(FloatFormat fmt, FloatRegister fd,
+ FloatRegister fj, Register rk) {
+ Label done;
+ ma_b(rk, zero, &done, Assembler::NotEqual);
+ if (fmt == SingleFloat) {
+ as_fmov_s(fd, fj);
+ } else {
+ as_fmov_d(fd, fj);
+ }
+ bind(&done);
+}
+
+void MacroAssemblerLOONG64::ma_fmovn(FloatFormat fmt, FloatRegister fd,
+ FloatRegister fj, Register rk) {
+ Label done;
+ ma_b(rk, zero, &done, Assembler::Equal);
+ if (fmt == SingleFloat) {
+ as_fmov_s(fd, fj);
+ } else {
+ as_fmov_d(fd, fj);
+ }
+ bind(&done);
+}
+
+void MacroAssemblerLOONG64::ma_and(Register rd, Register rj, Imm32 imm,
+ bool bit32) {
+ if (is_uintN(imm.value, 12)) {
+ as_andi(rd, rj, imm.value);
+ } else if (rd != rj) {
+ ma_li(rd, imm);
+ as_and(rd, rj, rd);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_and(rd, rj, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_or(Register rd, Register rj, Imm32 imm,
+ bool bit32) {
+ if (is_uintN(imm.value, 12)) {
+ as_ori(rd, rj, imm.value);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_or(rd, rj, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_xor(Register rd, Register rj, Imm32 imm,
+ bool bit32) {
+ if (is_uintN(imm.value, 12)) {
+ as_xori(rd, rj, imm.value);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_xor(rd, rj, scratch);
+ }
+}
+
+// Arithmetic-based ops.
+
+// Add.
+void MacroAssemblerLOONG64::ma_add_w(Register rd, Register rj, Imm32 imm) {
+ if (is_intN(imm.value, 12)) {
+ as_addi_w(rd, rj, imm.value);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_add_w(rd, rj, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_add32TestCarry(Condition cond, Register rd,
+ Register rj, Register rk,
+ Label* overflow) {
+ MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear);
+ MOZ_ASSERT_IF(rd == rj, rk != rd);
+ ScratchRegisterScope scratch(asMasm());
+ as_add_w(rd, rj, rk);
+ as_sltu(scratch, rd, rd == rj ? rk : rj);
+ ma_b(Register(scratch), Register(scratch), overflow,
+ cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
+}
+
+void MacroAssemblerLOONG64::ma_add32TestCarry(Condition cond, Register rd,
+ Register rj, Imm32 imm,
+ Label* overflow) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(rj != scratch2);
+ ma_li(scratch2, imm);
+ ma_add32TestCarry(cond, rd, rj, scratch2, overflow);
+}
+
+// Subtract.
+void MacroAssemblerLOONG64::ma_sub_w(Register rd, Register rj, Imm32 imm) {
+ if (is_intN(-imm.value, 12)) {
+ as_addi_w(rd, rj, -imm.value);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_sub_w(rd, rj, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_sub_w(Register rd, Register rj, Register rk) {
+ as_sub_w(rd, rj, rk);
+}
+
+void MacroAssemblerLOONG64::ma_sub32TestOverflow(Register rd, Register rj,
+ Imm32 imm, Label* overflow) {
+ if (imm.value != INT32_MIN) {
+ asMasm().ma_add32TestOverflow(rd, rj, Imm32(-imm.value), overflow);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, Imm32(imm.value));
+ asMasm().ma_sub32TestOverflow(rd, rj, scratch, overflow);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_mul(Register rd, Register rj, Imm32 imm) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rj != scratch);
+ ma_li(scratch, imm);
+ as_mul_w(rd, rj, scratch);
+}
+
+void MacroAssemblerLOONG64::ma_mul32TestOverflow(Register rd, Register rj,
+ Register rk, Label* overflow) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ as_mulh_w(scratch, rj, rk);
+ as_mul_w(rd, rj, rk);
+ as_srai_w(scratch2, rd, 31);
+ ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual);
+}
+
+void MacroAssemblerLOONG64::ma_mul32TestOverflow(Register rd, Register rj,
+ Imm32 imm, Label* overflow) {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch, imm);
+ as_mulh_w(scratch2, rj, scratch);
+ as_mul_w(rd, rj, scratch);
+ as_srai_w(scratch, rd, 31);
+ ma_b(scratch, Register(scratch2), overflow, Assembler::NotEqual);
+}
+
+void MacroAssemblerLOONG64::ma_div_branch_overflow(Register rd, Register rj,
+ Register rk,
+ Label* overflow) {
+ ScratchRegisterScope scratch(asMasm());
+ as_mod_w(scratch, rj, rk);
+ ma_b(scratch, scratch, overflow, Assembler::NonZero);
+ as_div_w(rd, rj, rk);
+}
+
+void MacroAssemblerLOONG64::ma_div_branch_overflow(Register rd, Register rj,
+ Imm32 imm, Label* overflow) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch2, imm);
+ ma_div_branch_overflow(rd, rj, scratch2, overflow);
+}
+
+void MacroAssemblerLOONG64::ma_mod_mask(Register src, Register dest,
+ Register hold, Register remain,
+ int32_t shift, Label* negZero) {
+ // MATH:
+ // We wish to compute x % (1<<y) - 1 for a known constant, y.
+ // First, let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit
+ // dividend as a number in base b, namely
+ // c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n
+ // now, since both addition and multiplication commute with modulus,
+ // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C ==
+ // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)...
+ // now, since b == C + 1, b % C == 1, and b^n % C == 1
+ // this means that the whole thing simplifies to:
+ // c_0 + c_1 + c_2 ... c_n % C
+ // each c_n can easily be computed by a shift/bitextract, and the modulus
+ // can be maintained by simply subtracting by C whenever the number gets
+ // over C.
+ int32_t mask = (1 << shift) - 1;
+ Label head, negative, sumSigned, done;
+
+ // hold holds -1 if the value was negative, 1 otherwise.
+ // remain holds the remaining bits that have not been processed
+ // SecondScratchReg serves as a temporary location to store extracted bits
+ // into as well as holding the trial subtraction as a temp value dest is
+ // the accumulator (and holds the final result)
+
+ // move the whole value into the remain.
+ as_or(remain, src, zero);
+ // Zero out the dest.
+ ma_li(dest, Imm32(0));
+ // Set the hold appropriately.
+ ma_b(remain, remain, &negative, Signed, ShortJump);
+ ma_li(hold, Imm32(1));
+ ma_b(&head, ShortJump);
+
+ bind(&negative);
+ ma_li(hold, Imm32(-1));
+ as_sub_w(remain, zero, remain);
+
+ // Begin the main loop.
+ bind(&head);
+
+ SecondScratchRegisterScope scratch2(asMasm());
+ // Extract the bottom bits into SecondScratchReg.
+ ma_and(scratch2, remain, Imm32(mask));
+ // Add those bits to the accumulator.
+ as_add_w(dest, dest, scratch2);
+ // Do a trial subtraction
+ ma_sub_w(scratch2, dest, Imm32(mask));
+ // If (sum - C) > 0, store sum - C back into sum, thus performing a
+ // modulus.
+ ma_b(scratch2, Register(scratch2), &sumSigned, Signed, ShortJump);
+ as_or(dest, scratch2, zero);
+ bind(&sumSigned);
+ // Get rid of the bits that we extracted before.
+ as_srli_w(remain, remain, shift);
+ // If the shift produced zero, finish, otherwise, continue in the loop.
+ ma_b(remain, remain, &head, NonZero, ShortJump);
+ // Check the hold to see if we need to negate the result.
+ ma_b(hold, hold, &done, NotSigned, ShortJump);
+
+ // If the hold was non-zero, negate the result to be in line with
+ // what JS wants
+ if (negZero != nullptr) {
+ // Jump out in case of negative zero.
+ ma_b(hold, hold, negZero, Zero);
+ as_sub_w(dest, zero, dest);
+ } else {
+ as_sub_w(dest, zero, dest);
+ }
+
+ bind(&done);
+}
+
+// Memory.
+
+void MacroAssemblerLOONG64::ma_load(Register dest, const BaseIndex& src,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ asMasm().computeScaledAddress(src, scratch2);
+ asMasm().ma_load(dest, Address(scratch2, src.offset), size, extension);
+}
+
+void MacroAssemblerLOONG64::ma_store(Register data, const BaseIndex& dest,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ asMasm().computeScaledAddress(dest, scratch2);
+ asMasm().ma_store(data, Address(scratch2, dest.offset), size, extension);
+}
+
+void MacroAssemblerLOONG64::ma_store(Imm32 imm, const BaseIndex& dest,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ // Make sure that scratch2 contains absolute address so that offset is 0.
+ asMasm().computeEffectiveAddress(dest, scratch2);
+
+ ScratchRegisterScope scratch(asMasm());
+ // Scrach register is free now, use it for loading imm value
+ ma_li(scratch, imm);
+
+ // with offset=0 ScratchRegister will not be used in ma_store()
+ // so we can use it as a parameter here
+ asMasm().ma_store(scratch, Address(scratch2, 0), size, extension);
+}
+
+// Branches when done from within loongarch-specific code.
+// TODO(loong64) Optimize ma_b
+void MacroAssemblerLOONG64::ma_b(Register lhs, Register rhs, Label* label,
+ Condition c, JumpKind jumpKind) {
+ switch (c) {
+ case Equal:
+ case NotEqual:
+ asMasm().branchWithCode(getBranchCode(lhs, rhs, c), label, jumpKind);
+ break;
+ case Always:
+ ma_b(label, jumpKind);
+ break;
+ case Zero:
+ case NonZero:
+ case Signed:
+ case NotSigned:
+ MOZ_ASSERT(lhs == rhs);
+ asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind);
+ break;
+ default: {
+ Condition cond = ma_cmp(ScratchRegister, lhs, rhs, c);
+ asMasm().branchWithCode(getBranchCode(ScratchRegister, cond), label,
+ jumpKind);
+ break;
+ }
+ }
+}
+
+void MacroAssemblerLOONG64::ma_b(Register lhs, Imm32 imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ MOZ_ASSERT(c != Overflow);
+ if (imm.value == 0) {
+ if (c == Always || c == AboveOrEqual) {
+ ma_b(label, jumpKind);
+ } else if (c == Below) {
+ ; // This condition is always false. No branch required.
+ } else {
+ asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind);
+ }
+ } else {
+ switch (c) {
+ case Equal:
+ case NotEqual:
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_b(lhs, ScratchRegister, label, c, jumpKind);
+ break;
+ default:
+ Condition cond = ma_cmp(ScratchRegister, lhs, imm, c);
+ asMasm().branchWithCode(getBranchCode(ScratchRegister, cond), label,
+ jumpKind);
+ }
+ }
+}
+
+void MacroAssemblerLOONG64::ma_b(Register lhs, ImmPtr imm, Label* l,
+ Condition c, JumpKind jumpKind) {
+ asMasm().ma_b(lhs, ImmWord(uintptr_t(imm.value)), l, c, jumpKind);
+}
+
+void MacroAssemblerLOONG64::ma_b(Label* label, JumpKind jumpKind) {
+ asMasm().branchWithCode(getBranchCode(BranchIsJump), label, jumpKind);
+}
+
+Assembler::Condition MacroAssemblerLOONG64::ma_cmp(Register dest, Register lhs,
+ Register rhs, Condition c) {
+ switch (c) {
+ case Above:
+ // bgtu s,t,label =>
+ // sltu at,t,s
+ // bne at,$zero,offs
+ as_sltu(dest, rhs, lhs);
+ return NotEqual;
+ case AboveOrEqual:
+ // bgeu s,t,label =>
+ // sltu at,s,t
+ // beq at,$zero,offs
+ as_sltu(dest, lhs, rhs);
+ return Equal;
+ case Below:
+ // bltu s,t,label =>
+ // sltu at,s,t
+ // bne at,$zero,offs
+ as_sltu(dest, lhs, rhs);
+ return NotEqual;
+ case BelowOrEqual:
+ // bleu s,t,label =>
+ // sltu at,t,s
+ // beq at,$zero,offs
+ as_sltu(dest, rhs, lhs);
+ return Equal;
+ case GreaterThan:
+ // bgt s,t,label =>
+ // slt at,t,s
+ // bne at,$zero,offs
+ as_slt(dest, rhs, lhs);
+ return NotEqual;
+ case GreaterThanOrEqual:
+ // bge s,t,label =>
+ // slt at,s,t
+ // beq at,$zero,offs
+ as_slt(dest, lhs, rhs);
+ return Equal;
+ case LessThan:
+ // blt s,t,label =>
+ // slt at,s,t
+ // bne at,$zero,offs
+ as_slt(dest, lhs, rhs);
+ return NotEqual;
+ case LessThanOrEqual:
+ // ble s,t,label =>
+ // slt at,t,s
+ // beq at,$zero,offs
+ as_slt(dest, rhs, lhs);
+ return Equal;
+ default:
+ MOZ_CRASH("Invalid condition.");
+ }
+ return Always;
+}
+
+Assembler::Condition MacroAssemblerLOONG64::ma_cmp(Register dest, Register lhs,
+ Imm32 imm, Condition c) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_RELEASE_ASSERT(lhs != scratch);
+
+ switch (c) {
+ case Above:
+ case BelowOrEqual:
+ if (imm.value != 0x7fffffff && is_intN(imm.value + 1, 12) &&
+ imm.value != -1) {
+ // lhs <= rhs via lhs < rhs + 1 if rhs + 1 does not overflow
+ as_sltui(dest, lhs, imm.value + 1);
+
+ return (c == BelowOrEqual ? NotEqual : Equal);
+ } else {
+ ma_li(scratch, imm);
+ as_sltu(dest, scratch, lhs);
+ return (c == BelowOrEqual ? Equal : NotEqual);
+ }
+ case AboveOrEqual:
+ case Below:
+ if (is_intN(imm.value, 12)) {
+ as_sltui(dest, lhs, imm.value);
+ } else {
+ ma_li(scratch, imm);
+ as_sltu(dest, lhs, scratch);
+ }
+ return (c == AboveOrEqual ? Equal : NotEqual);
+ case GreaterThan:
+ case LessThanOrEqual:
+ if (imm.value != 0x7fffffff && is_intN(imm.value + 1, 12)) {
+ // lhs <= rhs via lhs < rhs + 1.
+ as_slti(dest, lhs, imm.value + 1);
+ return (c == LessThanOrEqual ? NotEqual : Equal);
+ } else {
+ ma_li(scratch, imm);
+ as_slt(dest, scratch, lhs);
+ return (c == LessThanOrEqual ? Equal : NotEqual);
+ }
+ case GreaterThanOrEqual:
+ case LessThan:
+ if (is_intN(imm.value, 12)) {
+ as_slti(dest, lhs, imm.value);
+ } else {
+ ma_li(scratch, imm);
+ as_slt(dest, lhs, scratch);
+ }
+ return (c == GreaterThanOrEqual ? Equal : NotEqual);
+ default:
+ MOZ_CRASH("Invalid condition.");
+ }
+ return Always;
+}
+
+// fp instructions
+void MacroAssemblerLOONG64::ma_lis(FloatRegister dest, float value) {
+ Imm32 imm(mozilla::BitwiseCast<uint32_t>(value));
+
+ if (imm.value != 0) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ moveToFloat32(scratch, dest);
+ } else {
+ moveToFloat32(zero, dest);
+ }
+}
+
+void MacroAssemblerLOONG64::ma_fst_d(FloatRegister ft, BaseIndex address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ asMasm().computeScaledAddress(address, scratch2);
+ asMasm().ma_fst_d(ft, Address(scratch2, address.offset));
+}
+
+void MacroAssemblerLOONG64::ma_fst_s(FloatRegister ft, BaseIndex address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ asMasm().computeScaledAddress(address, scratch2);
+ asMasm().ma_fst_s(ft, Address(scratch2, address.offset));
+}
+
+void MacroAssemblerLOONG64::ma_fld_d(FloatRegister ft, const BaseIndex& src) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ asMasm().computeScaledAddress(src, scratch2);
+ asMasm().ma_fld_d(ft, Address(scratch2, src.offset));
+}
+
+void MacroAssemblerLOONG64::ma_fld_s(FloatRegister ft, const BaseIndex& src) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ asMasm().computeScaledAddress(src, scratch2);
+ asMasm().ma_fld_s(ft, Address(scratch2, src.offset));
+}
+
+void MacroAssemblerLOONG64::ma_bc_s(FloatRegister lhs, FloatRegister rhs,
+ Label* label, DoubleCondition c,
+ JumpKind jumpKind, FPConditionBit fcc) {
+ compareFloatingPoint(SingleFloat, lhs, rhs, c, fcc);
+ asMasm().branchWithCode(getBranchCode(fcc), label, jumpKind);
+}
+
+void MacroAssemblerLOONG64::ma_bc_d(FloatRegister lhs, FloatRegister rhs,
+ Label* label, DoubleCondition c,
+ JumpKind jumpKind, FPConditionBit fcc) {
+ compareFloatingPoint(DoubleFloat, lhs, rhs, c, fcc);
+ asMasm().branchWithCode(getBranchCode(fcc), label, jumpKind);
+}
+
+void MacroAssemblerLOONG64::ma_call(ImmPtr dest) {
+ asMasm().ma_liPatchable(CallReg, dest);
+ as_jirl(ra, CallReg, BOffImm16(0));
+}
+
+void MacroAssemblerLOONG64::ma_jump(ImmPtr dest) {
+ ScratchRegisterScope scratch(asMasm());
+ asMasm().ma_liPatchable(scratch, dest);
+ as_jirl(zero, scratch, BOffImm16(0));
+}
+
+void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, Register rk,
+ Condition c) {
+ switch (c) {
+ case Equal:
+ // seq d,s,t =>
+ // xor d,s,t
+ // sltiu d,d,1
+ as_xor(rd, rj, rk);
+ as_sltui(rd, rd, 1);
+ break;
+ case NotEqual:
+ // sne d,s,t =>
+ // xor d,s,t
+ // sltu d,$zero,d
+ as_xor(rd, rj, rk);
+ as_sltu(rd, zero, rd);
+ break;
+ case Above:
+ // sgtu d,s,t =>
+ // sltu d,t,s
+ as_sltu(rd, rk, rj);
+ break;
+ case AboveOrEqual:
+ // sgeu d,s,t =>
+ // sltu d,s,t
+ // xori d,d,1
+ as_sltu(rd, rj, rk);
+ as_xori(rd, rd, 1);
+ break;
+ case Below:
+ // sltu d,s,t
+ as_sltu(rd, rj, rk);
+ break;
+ case BelowOrEqual:
+ // sleu d,s,t =>
+ // sltu d,t,s
+ // xori d,d,1
+ as_sltu(rd, rk, rj);
+ as_xori(rd, rd, 1);
+ break;
+ case GreaterThan:
+ // sgt d,s,t =>
+ // slt d,t,s
+ as_slt(rd, rk, rj);
+ break;
+ case GreaterThanOrEqual:
+ // sge d,s,t =>
+ // slt d,s,t
+ // xori d,d,1
+ as_slt(rd, rj, rk);
+ as_xori(rd, rd, 1);
+ break;
+ case LessThan:
+ // slt d,s,t
+ as_slt(rd, rj, rk);
+ break;
+ case LessThanOrEqual:
+ // sle d,s,t =>
+ // slt d,t,s
+ // xori d,d,1
+ as_slt(rd, rk, rj);
+ as_xori(rd, rd, 1);
+ break;
+ case Zero:
+ MOZ_ASSERT(rj == rk);
+ // seq d,s,$zero =>
+ // sltiu d,s,1
+ as_sltui(rd, rj, 1);
+ break;
+ case NonZero:
+ MOZ_ASSERT(rj == rk);
+ // sne d,s,$zero =>
+ // sltu d,$zero,s
+ as_sltu(rd, zero, rj);
+ break;
+ case Signed:
+ MOZ_ASSERT(rj == rk);
+ as_slt(rd, rj, zero);
+ break;
+ case NotSigned:
+ MOZ_ASSERT(rj == rk);
+ // sge d,s,$zero =>
+ // slt d,s,$zero
+ // xori d,d,1
+ as_slt(rd, rj, zero);
+ as_xori(rd, rd, 1);
+ break;
+ default:
+ MOZ_CRASH("Invalid condition.");
+ }
+}
+
+void MacroAssemblerLOONG64::ma_cmp_set_double(Register dest, FloatRegister lhs,
+ FloatRegister rhs,
+ DoubleCondition c) {
+ compareFloatingPoint(DoubleFloat, lhs, rhs, c);
+ as_movcf2gr(dest, FCC0);
+}
+
+void MacroAssemblerLOONG64::ma_cmp_set_float32(Register dest, FloatRegister lhs,
+ FloatRegister rhs,
+ DoubleCondition c) {
+ compareFloatingPoint(SingleFloat, lhs, rhs, c);
+ as_movcf2gr(dest, FCC0);
+}
+
+void MacroAssemblerLOONG64::ma_cmp_set(Register rd, Register rj, Imm32 imm,
+ Condition c) {
+ if (imm.value == 0) {
+ switch (c) {
+ case Equal:
+ case BelowOrEqual:
+ as_sltui(rd, rj, 1);
+ break;
+ case NotEqual:
+ case Above:
+ as_sltu(rd, zero, rj);
+ break;
+ case AboveOrEqual:
+ case Below:
+ as_ori(rd, zero, c == AboveOrEqual ? 1 : 0);
+ break;
+ case GreaterThan:
+ case LessThanOrEqual:
+ as_slt(rd, zero, rj);
+ if (c == LessThanOrEqual) {
+ as_xori(rd, rd, 1);
+ }
+ break;
+ case LessThan:
+ case GreaterThanOrEqual:
+ as_slt(rd, rj, zero);
+ if (c == GreaterThanOrEqual) {
+ as_xori(rd, rd, 1);
+ }
+ break;
+ case Zero:
+ as_sltui(rd, rj, 1);
+ break;
+ case NonZero:
+ as_sltu(rd, zero, rj);
+ break;
+ case Signed:
+ as_slt(rd, rj, zero);
+ break;
+ case NotSigned:
+ as_slt(rd, rj, zero);
+ as_xori(rd, rd, 1);
+ break;
+ default:
+ MOZ_CRASH("Invalid condition.");
+ }
+ return;
+ }
+
+ switch (c) {
+ case Equal:
+ case NotEqual:
+ ma_xor(rd, rj, imm);
+ if (c == Equal) {
+ as_sltui(rd, rd, 1);
+ } else {
+ as_sltu(rd, zero, rd);
+ }
+ break;
+ case Zero:
+ case NonZero:
+ case Signed:
+ case NotSigned:
+ MOZ_CRASH("Invalid condition.");
+ default:
+ Condition cond = ma_cmp(rd, rj, imm, c);
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+
+ if (cond == Equal) as_xori(rd, rd, 1);
+ }
+}
+
+void MacroAssemblerLOONG64::compareFloatingPoint(FloatFormat fmt,
+ FloatRegister lhs,
+ FloatRegister rhs,
+ DoubleCondition c,
+ FPConditionBit fcc) {
+ switch (c) {
+ case DoubleOrdered:
+ as_fcmp_cor(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleEqual:
+ as_fcmp_ceq(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleNotEqual:
+ as_fcmp_cne(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleGreaterThan:
+ as_fcmp_clt(fmt, rhs, lhs, fcc);
+ break;
+ case DoubleGreaterThanOrEqual:
+ as_fcmp_cle(fmt, rhs, lhs, fcc);
+ break;
+ case DoubleLessThan:
+ as_fcmp_clt(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleLessThanOrEqual:
+ as_fcmp_cle(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleUnordered:
+ as_fcmp_cun(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleEqualOrUnordered:
+ as_fcmp_cueq(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleNotEqualOrUnordered:
+ as_fcmp_cune(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleGreaterThanOrUnordered:
+ as_fcmp_cult(fmt, rhs, lhs, fcc);
+ break;
+ case DoubleGreaterThanOrEqualOrUnordered:
+ as_fcmp_cule(fmt, rhs, lhs, fcc);
+ break;
+ case DoubleLessThanOrUnordered:
+ as_fcmp_cult(fmt, lhs, rhs, fcc);
+ break;
+ case DoubleLessThanOrEqualOrUnordered:
+ as_fcmp_cule(fmt, lhs, rhs, fcc);
+ break;
+ default:
+ MOZ_CRASH("Invalid DoubleCondition.");
+ }
+}
+
+void MacroAssemblerLOONG64::minMaxDouble(FloatRegister srcDest,
+ FloatRegister second, bool handleNaN,
+ bool isMax) {
+ if (srcDest == second) return;
+
+ Label nan, done;
+
+ // First or second is NaN, result is NaN.
+ ma_bc_d(srcDest, second, &nan, Assembler::DoubleUnordered, ShortJump);
+ if (isMax) {
+ as_fmax_d(srcDest, srcDest, second);
+ } else {
+ as_fmin_d(srcDest, srcDest, second);
+ }
+ ma_b(&done, ShortJump);
+
+ bind(&nan);
+ as_fadd_d(srcDest, srcDest, second);
+
+ bind(&done);
+}
+
+void MacroAssemblerLOONG64::minMaxFloat32(FloatRegister srcDest,
+ FloatRegister second, bool handleNaN,
+ bool isMax) {
+ if (srcDest == second) return;
+
+ Label nan, done;
+
+ // First or second is NaN, result is NaN.
+ ma_bc_s(srcDest, second, &nan, Assembler::DoubleUnordered, ShortJump);
+ if (isMax) {
+ as_fmax_s(srcDest, srcDest, second);
+ } else {
+ as_fmin_s(srcDest, srcDest, second);
+ }
+ ma_b(&done, ShortJump);
+
+ bind(&nan);
+ as_fadd_s(srcDest, srcDest, second);
+
+ bind(&done);
+}
+
+void MacroAssemblerLOONG64::loadDouble(const Address& address,
+ FloatRegister dest) {
+ asMasm().ma_fld_d(dest, address);
+}
+
+void MacroAssemblerLOONG64::loadDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ asMasm().ma_fld_d(dest, src);
+}
+
+void MacroAssemblerLOONG64::loadFloatAsDouble(const Address& address,
+ FloatRegister dest) {
+ asMasm().ma_fld_s(dest, address);
+ as_fcvt_d_s(dest, dest);
+}
+
+void MacroAssemblerLOONG64::loadFloatAsDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ asMasm().loadFloat32(src, dest);
+ as_fcvt_d_s(dest, dest);
+}
+
+void MacroAssemblerLOONG64::loadFloat32(const Address& address,
+ FloatRegister dest) {
+ asMasm().ma_fld_s(dest, address);
+}
+
+void MacroAssemblerLOONG64::loadFloat32(const BaseIndex& src,
+ FloatRegister dest) {
+ asMasm().ma_fld_s(dest, src);
+}
+
+void MacroAssemblerLOONG64::wasmLoadImpl(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch,
+ AnyRegister output, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(ImmWord(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+
+ switch (access.type()) {
+ case Scalar::Int8:
+ as_ldx_b(output.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Uint8:
+ as_ldx_bu(output.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Int16:
+ as_ldx_h(output.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Uint16:
+ as_ldx_hu(output.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ as_ldx_w(output.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Float64:
+ as_fldx_d(output.fpu(), memoryBase, ptr);
+ break;
+ case Scalar::Float32:
+ as_fldx_s(output.fpu(), memoryBase, ptr);
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerLOONG64::wasmStoreImpl(const wasm::MemoryAccessDesc& access,
+ AnyRegister value,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(ImmWord(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+
+ switch (access.type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ as_stx_b(value.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ as_stx_h(value.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ as_stx_w(value.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Int64:
+ as_stx_d(value.gpr(), memoryBase, ptr);
+ break;
+ case Scalar::Float64:
+ as_fstx_d(value.fpu(), memoryBase, ptr);
+ break;
+ case Scalar::Float32:
+ as_fstx_s(value.fpu(), memoryBase, ptr);
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ // Only the last emitted instruction is a memory access.
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerLOONG64Compat::wasmLoadI64Impl(
+ const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(ImmWord(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+
+ switch (access.type()) {
+ case Scalar::Int8:
+ as_ldx_b(output.reg, memoryBase, ptr);
+ break;
+ case Scalar::Uint8:
+ as_ldx_bu(output.reg, memoryBase, ptr);
+ break;
+ case Scalar::Int16:
+ as_ldx_h(output.reg, memoryBase, ptr);
+ break;
+ case Scalar::Uint16:
+ as_ldx_hu(output.reg, memoryBase, ptr);
+ break;
+ case Scalar::Int32:
+ as_ldx_w(output.reg, memoryBase, ptr);
+ break;
+ case Scalar::Uint32:
+ // TODO(loong64): Why need zero-extension here?
+ as_ldx_wu(output.reg, memoryBase, ptr);
+ break;
+ case Scalar::Int64:
+ as_ldx_d(output.reg, memoryBase, ptr);
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerLOONG64Compat::wasmStoreI64Impl(
+ const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(ImmWord(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+
+ switch (access.type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ as_stx_b(value.reg, memoryBase, ptr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ as_stx_h(value.reg, memoryBase, ptr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ as_stx_w(value.reg, memoryBase, ptr);
+ break;
+ case Scalar::Int64:
+ as_stx_d(value.reg, memoryBase, ptr);
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerLOONG64::outOfLineWasmTruncateToInt32Check(
+ FloatRegister input, Register output, MIRType fromType, TruncFlags flags,
+ Label* rejoin, wasm::BytecodeOffset trapOffset) {
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ if (isSaturating) {
+ ScratchDoubleScope fpscratch(asMasm());
+ if (fromType == MIRType::Double) {
+ asMasm().loadConstantDouble(0.0, fpscratch);
+ } else {
+ asMasm().loadConstantFloat32(0.0f, fpscratch);
+ }
+
+ if (isUnsigned) {
+ ma_li(output, Imm32(UINT32_MAX));
+
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input,
+ fpscratch, Assembler::DoubleLessThanOrUnordered);
+
+ ScratchRegisterScope scratch(asMasm());
+ as_movcf2gr(scratch, FCC0);
+ // FCC0 = 1, output = zero; else not change.
+ as_masknez(output, output, scratch);
+ } else {
+ // Positive overflow is already saturated to INT32_MAX, so we only have
+ // to handle NaN and negative overflow here.
+
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, input,
+ Assembler::DoubleLessThanOrUnordered);
+
+ ScratchRegisterScope scratch(asMasm());
+ as_movcf2gr(scratch, FCC0);
+ // FCC0 = 1, output = zero; else not change.
+ as_masknez(output, output, scratch);
+
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input,
+ fpscratch, Assembler::DoubleLessThan);
+
+ as_movcf2gr(scratch, FCC0);
+ // FCC0 == 1, move INT32_MIN to output; else not change.
+ as_slli_w(scratch, scratch, 31);
+ as_or(output, output, scratch);
+ }
+
+ MOZ_ASSERT(rejoin->bound());
+ asMasm().jump(rejoin);
+ return;
+ }
+
+ Label inputIsNaN;
+
+ if (fromType == MIRType::Double) {
+ asMasm().branchDouble(Assembler::DoubleUnordered, input, input,
+ &inputIsNaN);
+ } else if (fromType == MIRType::Float32) {
+ asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN);
+ }
+
+ asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
+ asMasm().bind(&inputIsNaN);
+ asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
+}
+
+void MacroAssemblerLOONG64::outOfLineWasmTruncateToInt64Check(
+ FloatRegister input, Register64 output_, MIRType fromType, TruncFlags flags,
+ Label* rejoin, wasm::BytecodeOffset trapOffset) {
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ if (isSaturating) {
+ ScratchDoubleScope fpscratch(asMasm());
+ Register output = output_.reg;
+
+ if (fromType == MIRType::Double) {
+ asMasm().loadConstantDouble(0.0, fpscratch);
+ } else {
+ asMasm().loadConstantFloat32(0.0f, fpscratch);
+ }
+
+ if (isUnsigned) {
+ asMasm().ma_li(output, ImmWord(UINT64_MAX));
+
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input,
+ fpscratch, Assembler::DoubleLessThanOrUnordered);
+
+ ScratchRegisterScope scratch(asMasm());
+ as_movcf2gr(scratch, FCC0);
+ // FCC0 = 1, output = zero; else not change.
+ as_masknez(output, output, scratch);
+ } else {
+ // Positive overflow is already saturated to INT64_MAX, so we only have
+ // to handle NaN and negative overflow here.
+
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, input,
+ Assembler::DoubleLessThanOrUnordered);
+
+ ScratchRegisterScope scratch(asMasm());
+ as_movcf2gr(scratch, FCC0);
+ // FCC0 = 1, output = zero; else not change.
+ as_masknez(output, output, scratch);
+
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input,
+ fpscratch, Assembler::DoubleLessThan);
+
+ as_movcf2gr(scratch, FCC0);
+ // FCC0 == 1, move INT64_MIN to output; else not change.
+ as_slli_d(scratch, scratch, 63);
+ as_or(output, output, scratch);
+ }
+
+ MOZ_ASSERT(rejoin->bound());
+ asMasm().jump(rejoin);
+ return;
+ }
+
+ Label inputIsNaN;
+
+ if (fromType == MIRType::Double) {
+ asMasm().branchDouble(Assembler::DoubleUnordered, input, input,
+ &inputIsNaN);
+ } else if (fromType == MIRType::Float32) {
+ asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN);
+ }
+
+ asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
+ asMasm().bind(&inputIsNaN);
+ asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
+}
+
+void MacroAssemblerLOONG64Compat::profilerEnterFrame(Register framePtr,
+ Register scratch) {
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr,
+ Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr),
+ Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerLOONG64Compat::profilerExitFrame() {
+ jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+MacroAssembler& MacroAssemblerLOONG64::asMasm() {
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerLOONG64::asMasm() const {
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) {
+ if (imm32.value) {
+ asMasm().subPtr(imm32, StackPointer);
+ }
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// MacroAssembler high-level usage.
+
+void MacroAssembler::flush() {}
+
+// ===============================================================
+// Stack manipulation functions.
+
+size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
+ return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
+}
+
+void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
+ int32_t diff =
+ set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
+ const int32_t reserved = diff;
+
+ reserveStack(reserved);
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diff -= sizeof(intptr_t);
+ storePtr(*iter, Address(StackPointer, diff));
+ }
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush());
+ iter.more(); ++iter) {
+ diff -= sizeof(double);
+ storeDouble(*iter, Address(StackPointer, diff));
+ }
+ MOZ_ASSERT(diff == 0);
+}
+
+void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
+ LiveRegisterSet ignore) {
+ int32_t diff =
+ set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
+ const int32_t reserved = diff;
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diff -= sizeof(intptr_t);
+ if (!ignore.has(*iter)) {
+ loadPtr(Address(StackPointer, diff), *iter);
+ }
+ }
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush());
+ iter.more(); ++iter) {
+ diff -= sizeof(double);
+ if (!ignore.has(*iter)) {
+ loadDouble(Address(StackPointer, diff), *iter);
+ }
+ }
+ MOZ_ASSERT(diff == 0);
+ freeStack(reserved);
+}
+
+void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
+ Register) {
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+ mozilla::DebugOnly<unsigned> numFpu = fpuSet.size();
+ int32_t diffF = fpuSet.getPushSizeInBytes();
+ mozilla::DebugOnly<int32_t> diffG = set.gprs().size() * sizeof(intptr_t);
+
+ MOZ_ASSERT(dest.offset >= diffG + diffF);
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ dest.offset -= sizeof(intptr_t);
+ storePtr(*iter, dest);
+ }
+ MOZ_ASSERT(diffG == 0);
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ diffF -= reg.size();
+ numFpu -= 1;
+ dest.offset -= reg.size();
+ if (reg.isDouble()) {
+ storeDouble(reg, dest);
+ } else if (reg.isSingle()) {
+ storeFloat32(reg, dest);
+ } else {
+ MOZ_CRASH("Unknown register type.");
+ }
+ }
+ MOZ_ASSERT(numFpu == 0);
+ diffF -= diffF % sizeof(uintptr_t);
+ MOZ_ASSERT(diffF == 0);
+}
+
+void MacroAssembler::Push(Register reg) {
+ ma_push(reg);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Push(const Imm32 imm) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ ma_push(scratch);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Push(const ImmWord imm) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ ma_push(scratch);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Push(const ImmPtr imm) {
+ Push(ImmWord(uintptr_t(imm.value)));
+}
+
+void MacroAssembler::Push(const ImmGCPtr ptr) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, ptr);
+ ma_push(scratch);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Push(FloatRegister f) {
+ ma_push(f);
+ adjustFrame(int32_t(sizeof(double)));
+}
+
+void MacroAssembler::PushBoxed(FloatRegister reg) {
+ subFromStackPtr(Imm32(sizeof(double)));
+ boxDouble(reg, Address(getStackPointer(), 0));
+ adjustFrame(sizeof(double));
+}
+
+void MacroAssembler::Pop(Register reg) {
+ ma_pop(reg);
+ adjustFrame(-int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Pop(FloatRegister f) {
+ ma_pop(f);
+ adjustFrame(-int32_t(sizeof(double)));
+}
+
+void MacroAssembler::Pop(const ValueOperand& val) {
+ popValue(val);
+ adjustFrame(-int32_t(sizeof(Value)));
+}
+
+void MacroAssembler::PopStackPtr() {
+ loadPtr(Address(StackPointer, 0), StackPointer);
+ adjustFrame(-int32_t(sizeof(intptr_t)));
+}
+
+// ===============================================================
+// Simple call functions.
+
+CodeOffset MacroAssembler::call(Register reg) {
+ as_jirl(ra, reg, BOffImm16(0));
+ return CodeOffset(currentOffset());
+}
+
+CodeOffset MacroAssembler::call(Label* label) {
+ ma_bl(label);
+ return CodeOffset(currentOffset());
+}
+
+CodeOffset MacroAssembler::callWithPatch() {
+ as_bl(JOffImm26(1 * sizeof(uint32_t)));
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
+ BufferOffset call(callerOffset - 1 * sizeof(uint32_t));
+
+ JOffImm26 offset = BufferOffset(calleeOffset).diffB<JOffImm26>(call);
+ if (!offset.isInvalid()) {
+ InstJump* bal = (InstJump*)editSrc(call);
+ bal->setJOffImm26(offset);
+ } else {
+ uint32_t u32Offset = callerOffset - 4 * sizeof(uint32_t);
+ uint32_t* u32 =
+ reinterpret_cast<uint32_t*>(editSrc(BufferOffset(u32Offset)));
+ *u32 = calleeOffset - callerOffset;
+ }
+}
+
+CodeOffset MacroAssembler::farJumpWithPatch() {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ as_pcaddi(scratch, 4);
+ as_ld_w(scratch2, scratch, 0);
+ as_add_d(scratch, scratch, scratch2);
+ as_jirl(zero, scratch, BOffImm16(0));
+ // Allocate space which will be patched by patchFarJump().
+ CodeOffset farJump(currentOffset());
+ spew(".space 32bit initValue 0xffff ffff");
+ writeInst(UINT32_MAX);
+ return farJump;
+}
+
+void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
+ uint32_t* u32 =
+ reinterpret_cast<uint32_t*>(editSrc(BufferOffset(farJump.offset())));
+ MOZ_ASSERT(*u32 == UINT32_MAX);
+ *u32 = targetOffset - farJump.offset();
+}
+
+CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) {
+ movePtr(target, CallReg);
+ return call(CallReg);
+}
+
+void MacroAssembler::call(const Address& addr) {
+ loadPtr(addr, CallReg);
+ call(CallReg);
+}
+
+void MacroAssembler::call(ImmWord target) { call(ImmPtr((void*)target.value)); }
+
+void MacroAssembler::call(ImmPtr target) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, target, RelocationKind::HARDCODED);
+ ma_call(target);
+}
+
+void MacroAssembler::call(JitCode* c) {
+ ScratchRegisterScope scratch(asMasm());
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(scratch, ImmPtr(c->raw()));
+ callJitNoProfiler(scratch);
+}
+
+CodeOffset MacroAssembler::nopPatchableToCall() {
+ // LOONG64
+ as_nop(); // lu12i_w
+ as_nop(); // ori
+ as_nop(); // lu32i_d
+ as_nop(); // jirl
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
+ Instruction* inst = (Instruction*)call - 4 /* four nops */;
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)target);
+ inst[3] = InstImm(op_jirl, BOffImm16(0), ScratchRegister, ra);
+}
+
+void MacroAssembler::patchCallToNop(uint8_t* call) {
+ Instruction* inst = (Instruction*)call - 4 /* four nops */;
+ inst[0].makeNop(); // lu12i_w
+ inst[1].makeNop(); // ori
+ inst[2].makeNop(); // lu32i_d
+ inst[3].makeNop(); // jirl
+}
+
+void MacroAssembler::pushReturnAddress() { push(ra); }
+
+void MacroAssembler::popReturnAddress() { pop(ra); }
+
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+ MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls");
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ as_or(scratch, StackPointer, zero);
+
+ // Force sp to be aligned
+ asMasm().subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1)));
+ storePtr(scratch, Address(StackPointer, 0));
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ // Reserve place for $ra.
+ stackForCall += sizeof(intptr_t);
+
+ if (dynamicAlignment_) {
+ stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment);
+ } else {
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(
+ stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+ }
+
+ *stackAdjust = stackForCall;
+ reserveStack(stackForCall);
+
+ // Save $ra because call is going to clobber it. Restore it in
+ // callWithABIPost. NOTE: This is needed for calls from SharedIC.
+ // Maybe we can do this differently.
+ storePtr(ra, Address(StackPointer, stackForCall - sizeof(intptr_t)));
+
+ // Position all arguments.
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_) {
+ return;
+ }
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool callFromWasm) {
+ // Restore ra value (as stored in callWithABIPre()).
+ loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra);
+
+ if (dynamicAlignment_) {
+ // Restore sp value from stack (as stored in setupUnalignedABICall()).
+ loadPtr(Address(StackPointer, stackAdjust), StackPointer);
+ // Use adjustFrame instead of freeStack because we already restored sp.
+ adjustFrame(-stackAdjust);
+ } else {
+ freeStack(stackAdjust);
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ // Load the callee in scratch2, no instruction between the movePtr and
+ // call should clobber it. Note that we can't use fun because it may be
+ // one of the IntArg registers clobbered before the call.
+ movePtr(fun, scratch2);
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(scratch2);
+ callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ // Load the callee in scratch2, as above.
+ loadPtr(fun, scratch2);
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(scratch2);
+ callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Jit Frames.
+
+uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
+ CodeLabel cl;
+
+ ma_li(scratch, &cl);
+ Push(scratch);
+ bind(&cl);
+ uint32_t retAddr = currentOffset();
+
+ addCodeLabel(cl);
+ return retAddr;
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ boxNonDouble(ValueTypeFromMIRType(type), reg.gpr(), dest);
+ return;
+ }
+
+ ScratchDoubleScope fpscratch(asMasm());
+ FloatRegister scratch = fpscratch;
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ boxDouble(freg, dest, scratch);
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ if (src == dest) {
+ return;
+ }
+ movePtr(src.valueReg(), dest.valueReg());
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ if (!src.isGCThing()) {
+ ma_li(dest.valueReg(), ImmWord(src.asRawBits()));
+ return;
+ }
+
+ writeDataRelocation(src);
+ movWithPatch(ImmWord(src.asRawBits()), dest.valueReg());
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+ ma_and(buffer, ptr, Imm32(int32_t(~gc::ChunkMask)));
+ loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+ Register temp, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(ptr != temp);
+ MOZ_ASSERT(ptr != ScratchRegister &&
+ ptr != SecondScratchReg); // Both may be used internally.
+ MOZ_ASSERT(temp != ScratchRegister && temp != SecondScratchReg);
+ MOZ_ASSERT(temp != InvalidReg);
+
+ ma_and(temp, ptr, Imm32(int32_t(~gc::ChunkMask)));
+ branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
+ zero, label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ branchValueIsNurseryCellImpl(cond, address, temp, label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ branchValueIsNurseryCellImpl(cond, value, temp, label);
+}
+
+template <typename T>
+void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
+ const T& value, Register temp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(temp != InvalidReg);
+ Label done;
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+
+ getGCThingValueChunk(value, temp);
+ loadPtr(Address(temp, gc::ChunkStoreBufferOffset), temp);
+ branchPtr(InvertCondition(cond), temp, zero, label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs.valueReg() != scratch);
+ moveValue(rhs, ValueOperand(scratch));
+ ma_b(lhs.valueReg(), scratch, label, cond);
+}
+
+// ========================================================================
+// Memory access primitives.
+
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ boxDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ if (value.constant()) {
+ storeValue(value.value(), dest);
+ } else {
+ storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
+ dest);
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); }
+
+// ===============================================================
+// WebAssembly
+
+CodeOffset MacroAssembler::wasmTrapInstruction() {
+ CodeOffset offset(currentOffset());
+ as_break(WASM_TRAP); // TODO: as_teq(zero, zero, WASM_TRAP)
+ return offset;
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* ok) {
+ ma_b(index, boundsCheckLimit, ok, cond);
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* ok) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ load32(boundsCheckLimit, scratch2);
+ ma_b(index, Register(scratch2), ok, cond);
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok) {
+ ma_b(index.reg, boundsCheckLimit.reg, ok, cond);
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(boundsCheckLimit, scratch2);
+ ma_b(index.reg, scratch2, ok, cond);
+}
+
+// FTINTRZ behaves as follows:
+//
+// on NaN it produces zero
+// on too large it produces INT_MAX (for appropriate type)
+// on too small it produces INT_MIN (ditto)
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+ if (!isSaturating) {
+ ma_bc_d(input, input, oolEntry, Assembler::DoubleUnordered);
+ }
+ as_ftintrz_l_d(fpscratch, input);
+ moveFromDouble(fpscratch, output);
+ as_srli_d(scratch, output, 32);
+ as_slli_w(output, output, 0);
+ ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+ if (!isSaturating) {
+ ma_bc_s(input, input, oolEntry, Assembler::DoubleUnordered);
+ }
+ as_ftintrz_l_s(fpscratch, input);
+ moveFromDouble(fpscratch, output);
+ as_srli_d(scratch, output, 32);
+ as_slli_w(output, output, 0);
+ ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual);
+}
+
+// Assembler::CauseV is a enum,called FCSRBit. Assembler::CauseV == 16
+void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchFloat32Scope fpscratch(asMasm());
+ as_ftintrz_w_d(fpscratch, input);
+ as_movfcsr2gr(scratch);
+ moveFromFloat32(fpscratch, output);
+ MOZ_ASSERT(Assembler::CauseV < 32);
+ as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchFloat32Scope fpscratch(asMasm());
+ as_ftintrz_w_s(fpscratch, input);
+ as_movfcsr2gr(scratch);
+ moveFromFloat32(fpscratch, output);
+ MOZ_ASSERT(Assembler::CauseV < 32);
+ as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ ma_b(scratch, Imm32(0), oolEntry, Assembler::NotEqual);
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt64(
+ FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_ASSERT(tempDouble.isInvalid());
+ ScratchDoubleScope fpscratch(asMasm());
+ Register output = output_.reg;
+
+ Label done;
+
+ if (!isSaturating) {
+ ma_bc_d(input, input, oolEntry, Assembler::DoubleUnordered);
+ }
+ as_ftintrz_l_d(fpscratch, input);
+ moveFromDouble(fpscratch, output);
+ loadConstantDouble(double(INT64_MAX + 1ULL), fpscratch);
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch2, ImmWord(INT64_MAX));
+ // For numbers in -1.[ : ]INT64_MAX range do nothing more
+ ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump);
+
+ ma_li(scratch2, ImmWord(INT64_MIN));
+ as_fsub_d(fpscratch, input, fpscratch);
+ as_ftintrz_l_d(fpscratch, fpscratch);
+ as_movfcsr2gr(scratch);
+ moveFromDouble(fpscratch, output);
+ as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ as_add_d(output, output, scratch2);
+
+ // Guard against negative values that result in 0 due the precision loss.
+ as_sltui(scratch2, output, 1);
+ as_or(scratch, scratch, scratch2);
+
+ ma_b(scratch, zero, oolEntry, Assembler::NotEqual);
+
+ bind(&done);
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt64(
+ FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempFloat) {
+ MOZ_ASSERT(tempFloat.isInvalid());
+ ScratchDoubleScope fpscratch(asMasm());
+ Register output = output_.reg;
+
+ Label done;
+
+ if (!isSaturating) {
+ ma_bc_s(input, input, oolEntry, Assembler::DoubleUnordered);
+ }
+ as_ftintrz_l_s(fpscratch, input);
+ moveFromDouble(fpscratch, output);
+ loadConstantFloat32(float(INT64_MAX + 1ULL), fpscratch);
+
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch2, ImmWord(INT64_MAX));
+ // For numbers in -1.[ : ]INT64_MAX range do nothing more
+ ma_b(output, Register(scratch2), &done, Assembler::Below, ShortJump);
+
+ ma_li(scratch2, ImmWord(INT64_MIN));
+ as_fsub_s(fpscratch, input, fpscratch);
+ as_ftintrz_l_s(fpscratch, fpscratch);
+ as_movfcsr2gr(scratch);
+ moveFromDouble(fpscratch, output);
+ as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ as_add_d(output, output, scratch2);
+
+ // Guard against negative values that result in 0 due the precision loss.
+ as_sltui(scratch2, output, 1);
+ as_or(scratch, scratch, scratch2);
+
+ ma_b(scratch, zero, oolEntry, Assembler::NotEqual);
+
+ bind(&done);
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_ASSERT(tempDouble.isInvalid());
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+
+ as_ftintrz_l_d(fpscratch, input);
+ as_movfcsr2gr(scratch);
+ moveFromDouble(fpscratch, output.reg);
+ as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ ma_b(scratch, zero, oolEntry, Assembler::NotEqual);
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempFloat) {
+ MOZ_ASSERT(tempFloat.isInvalid());
+ ScratchRegisterScope scratch(asMasm());
+ ScratchDoubleScope fpscratch(asMasm());
+
+ as_ftintrz_l_s(fpscratch, input);
+ as_movfcsr2gr(scratch);
+ moveFromDouble(fpscratch, output.reg);
+ as_bstrpick_d(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ ma_b(scratch, zero, oolEntry, Assembler::NotEqual);
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToInt32Check(input, output, MIRType::Float32, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToInt32Check(input, output, MIRType::Double, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToInt64Check(input, output, MIRType::Float32, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToInt64Check(input, output, MIRType::Double, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, AnyRegister output) {
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output) {
+ wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
+}
+
+void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Register memoryBase,
+ Register ptr, Register ptrScratch) {
+ wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
+}
+
+void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch) {
+ wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
+}
+
+void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
+ ExitFrameType type) {
+ enterFakeExitFrame(cxreg, scratch, type);
+}
+
+// TODO(loong64): widenInt32 should be nop?
+void MacroAssembler::widenInt32(Register r) {
+ move32To64SignExtend(r, Register64(r));
+}
+
+// ========================================================================
+// Convert floating point.
+
+void MacroAssembler::convertUInt64ToFloat32(Register64 src_, FloatRegister dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+
+ Register src = src_.reg;
+ Label positive, done;
+ ma_b(src, src, &positive, NotSigned, ShortJump);
+
+ MOZ_ASSERT(src != scratch);
+ MOZ_ASSERT(src != scratch2);
+
+ ma_and(scratch, src, Imm32(1));
+ as_srli_d(scratch2, src, 1);
+ as_or(scratch, scratch, scratch2);
+ as_movgr2fr_d(dest, scratch);
+ as_ffint_s_l(dest, dest);
+ addFloat32(dest, dest);
+ ma_b(&done, ShortJump);
+
+ bind(&positive);
+ as_movgr2fr_d(dest, src);
+ as_ffint_s_l(dest, dest);
+
+ bind(&done);
+}
+
+void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) {
+ as_movgr2fr_d(dest, src.reg);
+ as_ffint_s_l(dest, dest);
+}
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ MacroAssemblerSpecific::convertUInt64ToDouble(src.reg, dest);
+}
+
+void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) {
+ as_movgr2fr_d(dest, src.reg);
+ as_ffint_d_l(dest, dest);
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ convertInt64ToDouble(Register64(src), dest);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+template <typename T>
+static void CompareExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ const T& mem, Register oldval, Register newval,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again, end;
+
+ masm.computeEffectiveAddress(mem, scratch);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_w(output, scratch, 0);
+ masm.ma_b(output, oldval, &end, Assembler::NotEqual, ShortJump);
+ masm.as_or(scratch2, newval, zero);
+ masm.as_sc_w(scratch2, scratch, 0);
+ masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+ masm.bind(&end);
+
+ return;
+ }
+
+ masm.as_andi(offsetTemp, scratch, 3);
+ masm.subPtr(offsetTemp, scratch);
+ masm.as_slli_w(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sll_w(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, zero, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_w(scratch2, scratch, 0);
+
+ masm.as_srl_w(output, scratch2, offsetTemp);
+
+ switch (nbytes) {
+ case 1:
+ if (signExtend) {
+ masm.as_ext_w_b(valueTemp, oldval);
+ masm.as_ext_w_b(output, output);
+ } else {
+ masm.as_andi(valueTemp, oldval, 0xff);
+ masm.as_andi(output, output, 0xff);
+ }
+ break;
+ case 2:
+ if (signExtend) {
+ masm.as_ext_w_h(valueTemp, oldval);
+ masm.as_ext_w_h(output, output);
+ } else {
+ masm.as_bstrpick_d(valueTemp, oldval, 15, 0);
+ masm.as_bstrpick_d(output, output, 15, 0);
+ }
+ break;
+ }
+
+ masm.ma_b(output, valueTemp, &end, Assembler::NotEqual, ShortJump);
+
+ masm.as_sll_w(valueTemp, newval, offsetTemp);
+ masm.as_and(scratch2, scratch2, maskTemp);
+ masm.as_or(scratch2, scratch2, valueTemp);
+
+ masm.as_sc_w(scratch2, scratch, 0);
+
+ masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ masm.bind(&end);
+}
+
+template <typename T>
+static void CompareExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, const T& mem,
+ Register64 expect, Register64 replace,
+ Register64 output) {
+ MOZ_ASSERT(expect != output && replace != output);
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+ masm.computeEffectiveAddress(mem, scratch);
+
+ Label tryAgain;
+ Label exit;
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&tryAgain);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_d(output.reg, scratch, 0);
+
+ masm.ma_b(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump);
+ masm.movePtr(replace.reg, scratch2);
+ masm.as_sc_d(scratch2, scratch, 0);
+ masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ masm.bind(&exit);
+}
+
+template <typename T>
+static void AtomicExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ const T& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, scratch);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_w(output, scratch, 0);
+ masm.as_or(scratch2, value, zero);
+ masm.as_sc_w(scratch2, scratch, 0);
+ masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+ masm.as_andi(offsetTemp, scratch, 3);
+ masm.subPtr(offsetTemp, scratch);
+ masm.as_slli_w(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sll_w(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, zero, maskTemp);
+ switch (nbytes) {
+ case 1:
+ masm.as_andi(valueTemp, value, 0xff);
+ break;
+ case 2:
+ masm.as_bstrpick_d(valueTemp, value, 15, 0);
+ break;
+ }
+ masm.as_sll_w(valueTemp, valueTemp, offsetTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_w(output, scratch, 0);
+ masm.as_and(scratch2, output, maskTemp);
+ masm.as_or(scratch2, scratch2, valueTemp);
+
+ masm.as_sc_w(scratch2, scratch, 0);
+
+ masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump);
+
+ masm.as_srl_w(output, output, offsetTemp);
+
+ switch (nbytes) {
+ case 1:
+ if (signExtend) {
+ masm.as_ext_w_b(output, output);
+ } else {
+ masm.as_andi(output, output, 0xff);
+ }
+ break;
+ case 2:
+ if (signExtend) {
+ masm.as_ext_w_h(output, output);
+ } else {
+ masm.as_bstrpick_d(output, output, 15, 0);
+ }
+ break;
+ }
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <typename T>
+static void AtomicExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, const T& mem,
+ Register64 value, Register64 output) {
+ MOZ_ASSERT(value != output);
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+ masm.computeEffectiveAddress(mem, scratch);
+
+ Label tryAgain;
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&tryAgain);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_d(output.reg, scratch, 0);
+
+ masm.movePtr(value.reg, scratch2);
+ masm.as_sc_d(scratch2, scratch, 0);
+ masm.ma_b(scratch2, Register(scratch2), &tryAgain, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <typename T>
+static void AtomicFetchOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, const T& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, scratch);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_w(output, scratch, 0);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_add_w(scratch2, output, value);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sub_w(scratch2, output, value);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(scratch2, output, value);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(scratch2, output, value);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(scratch2, output, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.as_sc_w(scratch2, scratch, 0);
+ masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+ masm.as_andi(offsetTemp, scratch, 3);
+ masm.subPtr(offsetTemp, scratch);
+ masm.as_slli_w(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sll_w(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, zero, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_w(scratch2, scratch, 0);
+ masm.as_srl_w(output, scratch2, offsetTemp);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_add_w(valueTemp, output, value);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sub_w(valueTemp, output, value);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(valueTemp, output, value);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(valueTemp, output, value);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(valueTemp, output, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi(valueTemp, valueTemp, 0xff);
+ break;
+ case 2:
+ masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0);
+ break;
+ }
+
+ masm.as_sll_w(valueTemp, valueTemp, offsetTemp);
+
+ masm.as_and(scratch2, scratch2, maskTemp);
+ masm.as_or(scratch2, scratch2, valueTemp);
+
+ masm.as_sc_w(scratch2, scratch, 0);
+
+ masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump);
+
+ switch (nbytes) {
+ case 1:
+ if (signExtend) {
+ masm.as_ext_w_b(output, output);
+ } else {
+ masm.as_andi(output, output, 0xff);
+ }
+ break;
+ case 2:
+ if (signExtend) {
+ masm.as_ext_w_h(output, output);
+ } else {
+ masm.as_bstrpick_d(output, output, 15, 0);
+ }
+ break;
+ }
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <typename T>
+static void AtomicFetchOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, AtomicOp op,
+ Register64 value, const T& mem, Register64 temp,
+ Register64 output) {
+ MOZ_ASSERT(value != output);
+ MOZ_ASSERT(value != temp);
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+ masm.computeEffectiveAddress(mem, scratch);
+
+ Label tryAgain;
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&tryAgain);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_d(output.reg, scratch, 0);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_add_d(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sub_d(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(temp.reg, output.reg, value.reg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.as_sc_d(temp.reg, scratch, 0);
+ masm.ma_b(temp.reg, temp.reg, &tryAgain, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const Address& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization& sync,
+ const Address& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
+ newval, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
+ newval, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
+ output);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
+ output);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const Address& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, output);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, output);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization& sync,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, nullptr, sync, mem, value, output);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, nullptr, sync, mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
+ valueTemp, offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
+ valueTemp, offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type type,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp,
+ offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type type,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp,
+ offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value,
+ valueTemp, offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value,
+ valueTemp, offsetTemp, maskTemp, output);
+}
+
+template <typename T>
+static void AtomicEffectOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, const T& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) {
+ ScratchRegisterScope scratch(masm);
+ SecondScratchRegisterScope scratch2(masm);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, scratch);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_w(scratch2, scratch, 0);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_add_w(scratch2, scratch2, value);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sub_w(scratch2, scratch2, value);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(scratch2, scratch2, value);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(scratch2, scratch2, value);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(scratch2, scratch2, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.as_sc_w(scratch2, scratch, 0);
+ masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+ masm.as_andi(offsetTemp, scratch, 3);
+ masm.subPtr(offsetTemp, scratch);
+ masm.as_slli_w(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sll_w(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, zero, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll_w(scratch2, scratch, 0);
+ masm.as_srl_w(valueTemp, scratch2, offsetTemp);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_add_w(valueTemp, valueTemp, value);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sub_w(valueTemp, valueTemp, value);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(valueTemp, valueTemp, value);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(valueTemp, valueTemp, value);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(valueTemp, valueTemp, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi(valueTemp, valueTemp, 0xff);
+ break;
+ case 2:
+ masm.as_bstrpick_d(valueTemp, valueTemp, 15, 0);
+ break;
+ }
+
+ masm.as_sll_w(valueTemp, valueTemp, offsetTemp);
+
+ masm.as_and(scratch2, scratch2, maskTemp);
+ masm.as_or(scratch2, scratch2, valueTemp);
+
+ masm.as_sc_w(scratch2, scratch, 0);
+
+ masm.ma_b(scratch2, Register(scratch2), &again, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register valueTemp,
+ Register offsetTemp,
+ Register maskTemp) {
+ AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value,
+ valueTemp, offsetTemp, maskTemp);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) {
+ AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value,
+ valueTemp, offsetTemp, maskTemp);
+}
+
+template <typename T>
+static void WasmAtomicExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access,
+ const T& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(masm, &access, access.sync(), mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 src,
+ Register64 output) {
+ WasmAtomicExchange64(*this, access, mem, src, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 src,
+ Register64 output) {
+ WasmAtomicExchange64(*this, access, mem, src, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
+}
+
+// ========================================================================
+// JS atomic operations.
+
+template <typename T>
+static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register oldval, Register newval,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, output.gpr());
+ }
+}
+
+template <typename T>
+static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, output.gpr());
+ }
+}
+
+template <typename T>
+static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const T& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp,
+ maskTemp, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp,
+ maskTemp, output.gpr());
+ }
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, temp, output);
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, temp, output);
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, temp, output);
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, temp, output);
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp,
+ maskTemp, temp, output);
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp,
+ maskTemp, temp, output);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) {
+ AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp,
+ offsetTemp, maskTemp);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) {
+ AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp,
+ offsetTemp, maskTemp);
+}
+
+void MacroAssembler::flexibleQuotient32(Register rhs, Register srcDest,
+ bool isUnsigned,
+ const LiveRegisterSet&) {
+ quotient32(rhs, srcDest, isUnsigned);
+}
+
+void MacroAssembler::flexibleRemainder32(Register rhs, Register srcDest,
+ bool isUnsigned,
+ const LiveRegisterSet&) {
+ remainder32(rhs, srcDest, isUnsigned);
+}
+
+void MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest,
+ Register remOutput, bool isUnsigned,
+ const LiveRegisterSet&) {
+ if (isUnsigned) {
+ as_mod_wu(remOutput, srcDest, rhs);
+ as_div_wu(srcDest, srcDest, rhs);
+ } else {
+ as_mod_w(remOutput, srcDest, rhs);
+ as_div_w(srcDest, srcDest, rhs);
+ }
+}
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ return movWithPatch(ImmPtr(nullptr), dest);
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target) {
+ PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr));
+}
+
+// ========================================================================
+// Spectre Mitigations.
+
+void MacroAssembler::speculationBarrier() { MOZ_CRASH(); }
+
+void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchFloat32Scope fpscratch(asMasm());
+ FloatRegister scratch = fpscratch;
+ Label skipCheck, done;
+
+ // If Nan, 0 or -0 check for bailout
+ loadConstantFloat32(0.0f, scratch);
+ ma_bc_s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump);
+
+ // If high part is not zero, it is NaN or -0, so we bail.
+ {
+ ScratchRegisterScope scratch(asMasm());
+ moveFromDoubleLo(src, scratch);
+ branch32(Assembler::NotEqual, scratch, zero, fail);
+ }
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&done, ShortJump);
+
+ bind(&skipCheck);
+ as_ftintrm_w_s(scratch, src);
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchDoubleScope fpscratch(asMasm());
+ FloatRegister scratch = fpscratch;
+ Label skipCheck, done;
+
+ // If Nan, 0 or -0 check for bailout
+ loadConstantDouble(0.0, scratch);
+ ma_bc_d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump);
+
+ // If high part is not zero, it is NaN or -0, so we bail.
+ {
+ ScratchRegisterScope scratch(asMasm());
+ moveFromDoubleHi(src, scratch);
+ branch32(Assembler::NotEqual, scratch, zero, fail);
+ }
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&done, ShortJump);
+
+ bind(&skipCheck);
+ as_ftintrm_w_d(scratch, src);
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchFloat32Scope fpscratch(asMasm());
+ FloatRegister scratch = fpscratch;
+ Label performCeil, done;
+
+ // If x < -1 or x > 0 then perform ceil.
+ loadConstantFloat32(0.0f, scratch);
+ branchFloat(Assembler::DoubleGreaterThan, src, scratch, &performCeil);
+ loadConstantFloat32(-1.0f, scratch);
+ branchFloat(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil);
+
+ // If binary value is not zero, the input was not 0, so we bail.
+ {
+ ScratchRegisterScope scratch(asMasm());
+ moveFromFloat32(src, scratch);
+ branch32(Assembler::NotEqual, scratch, zero, fail);
+ }
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&done, ShortJump);
+
+ bind(&performCeil);
+ as_ftintrp_w_s(scratch, src);
+ moveFromFloat32(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchDoubleScope fpscratch(asMasm());
+ FloatRegister scratch = fpscratch;
+ Label performCeil, done;
+
+ // If x < -1 or x > 0 then perform ceil.
+ loadConstantDouble(0, scratch);
+ branchDouble(Assembler::DoubleGreaterThan, src, scratch, &performCeil);
+ loadConstantDouble(-1.0, scratch);
+ branchDouble(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil);
+
+ // If binary value is not zero, the input was not 0, so we bail.
+ {
+ ScratchRegisterScope scratch(asMasm());
+ moveFromDoubleHi(src, scratch);
+ branch32(Assembler::NotEqual, scratch, zero, fail);
+ }
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&done, ShortJump);
+
+ bind(&performCeil);
+ as_ftintrp_w_d(scratch, src);
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ ScratchFloat32Scope scratch(*this);
+
+ Label negative, end, skipCheck;
+
+ // Load biggest number less than 0.5 in the temp register.
+ loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp);
+
+ // Branch to a slow path for negative inputs. Doesn't catch NaN or -0.
+ loadConstantFloat32(0.0f, scratch);
+ ma_bc_s(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump);
+
+ // If Nan, 0 or -0 check for bailout
+ ma_bc_s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump);
+
+ // If binary value is not zero, it is NaN or -0, so we bail.
+ {
+ ScratchRegisterScope scratch(asMasm());
+ moveFromFloat32(src, scratch);
+ branch32(Assembler::NotEqual, scratch, zero, fail);
+ }
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&end, ShortJump);
+
+ bind(&skipCheck);
+ as_fadd_s(scratch, src, temp);
+ as_ftintrm_w_s(scratch, scratch);
+
+ moveFromFloat32(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ jump(&end);
+
+ // Input is negative, but isn't -0.
+ bind(&negative);
+
+ // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to
+ // be added the biggest double less than 0.5.
+ Label loadJoin;
+ loadConstantFloat32(-0.5f, scratch);
+ branchFloat(Assembler::DoubleLessThan, src, scratch, &loadJoin);
+ loadConstantFloat32(0.5f, temp);
+ bind(&loadJoin);
+
+ as_fadd_s(temp, src, temp);
+
+ // If input + 0.5 >= 0, input is a negative number >= -0.5 and the
+ // result is -0.
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail);
+
+ // Truncate and round toward zero.
+ // This is off-by-one for everything but integer-valued inputs.
+ as_ftintrm_w_s(scratch, temp);
+ moveFromFloat32(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+
+ bind(&end);
+}
+
+void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ ScratchDoubleScope scratch(*this);
+
+ Label negative, end, skipCheck;
+
+ // Load biggest number less than 0.5 in the temp register.
+ loadConstantDouble(GetBiggestNumberLessThan(0.5), temp);
+
+ // Branch to a slow path for negative inputs. Doesn't catch NaN or -0.
+ loadConstantDouble(0.0, scratch);
+ ma_bc_d(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump);
+
+ // If Nan, 0 or -0 check for bailout
+ ma_bc_d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump);
+
+ // If high part is not zero, it is NaN or -0, so we bail.
+ {
+ ScratchRegisterScope scratch(asMasm());
+ moveFromDoubleHi(src, scratch);
+ branch32(Assembler::NotEqual, scratch, zero, fail);
+ }
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&end, ShortJump);
+
+ bind(&skipCheck);
+ as_fadd_d(scratch, src, temp);
+ as_ftintrm_w_d(scratch, scratch);
+
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ jump(&end);
+
+ // Input is negative, but isn't -0.
+ bind(&negative);
+
+ // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to
+ // be added the biggest double less than 0.5.
+ Label loadJoin;
+ loadConstantDouble(-0.5, scratch);
+ branchDouble(Assembler::DoubleLessThan, src, scratch, &loadJoin);
+ loadConstantDouble(0.5, temp);
+ bind(&loadJoin);
+
+ addDouble(src, temp);
+
+ // If input + 0.5 >= 0, input is a negative number >= -0.5 and the
+ // result is -0.
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail);
+
+ // Truncate and round toward zero.
+ // This is off-by-one for everything but integer-valued inputs.
+ as_ftintrm_w_d(scratch, temp);
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+
+ bind(&end);
+}
+
+void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchFloat32Scope fpscratch(asMasm());
+
+ Label notZero;
+ as_ftintrz_w_s(fpscratch, src);
+ as_movfcsr2gr(scratch);
+ moveFromFloat32(fpscratch, dest);
+ as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
+
+ {
+ // dest == zero
+ SecondScratchRegisterScope scratch2(asMasm());
+ moveFromFloat32(src, scratch2);
+ // Check if input is in ]-1; -0] range by checking the sign bit.
+ as_slt(scratch2, scratch2, zero);
+ as_add_d(scratch, scratch, scratch2);
+ }
+
+ bind(&notZero);
+ branch32(Assembler::NotEqual, Register(scratch), zero, fail);
+}
+
+void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchRegisterScope scratch(asMasm());
+ ScratchFloat32Scope fpscratch(asMasm());
+
+ Label notZero;
+ as_ftintrz_w_d(fpscratch, src);
+ as_movfcsr2gr(scratch);
+ moveFromFloat32(fpscratch, dest);
+ as_bstrpick_w(scratch, scratch, Assembler::CauseV, Assembler::CauseV);
+ ma_b(dest, zero, &notZero, Assembler::NotEqual, ShortJump);
+
+ {
+ // dest == zero
+ SecondScratchRegisterScope scratch2(asMasm());
+ moveFromDoubleHi(src, scratch2);
+ // Check if input is in ]-1; -0] range by checking the sign bit.
+ as_slt(scratch2, scratch2, zero);
+ as_add_d(scratch, scratch, scratch2);
+ }
+
+ bind(&notZero);
+ branch32(Assembler::NotEqual, Register(scratch), zero, fail);
+}
+
+void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssemblerLOONG64Compat::move32(Imm32 imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerLOONG64Compat::move32(Register src, Register dest) {
+ as_slli_w(dest, src, 0);
+}
+
+void MacroAssemblerLOONG64Compat::movePtr(Register src, Register dest) {
+ as_or(dest, src, zero);
+}
+void MacroAssemblerLOONG64Compat::movePtr(ImmWord imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerLOONG64Compat::movePtr(ImmGCPtr imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerLOONG64Compat::movePtr(ImmPtr imm, Register dest) {
+ movePtr(ImmWord(uintptr_t(imm.value)), dest);
+}
+
+void MacroAssemblerLOONG64Compat::movePtr(wasm::SymbolicAddress imm,
+ Register dest) {
+ append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm));
+ ma_liPatchable(dest, ImmWord(-1));
+}
+
+void MacroAssemblerLOONG64Compat::load8ZeroExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeByte, ZeroExtend);
+}
+
+void MacroAssemblerLOONG64Compat::load8ZeroExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeByte, ZeroExtend);
+}
+
+void MacroAssemblerLOONG64Compat::load8SignExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeByte, SignExtend);
+}
+
+void MacroAssemblerLOONG64Compat::load8SignExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeByte, SignExtend);
+}
+
+void MacroAssemblerLOONG64Compat::load16ZeroExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeHalfWord, ZeroExtend);
+}
+
+void MacroAssemblerLOONG64Compat::load16ZeroExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeHalfWord, ZeroExtend);
+}
+
+void MacroAssemblerLOONG64Compat::load16SignExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeHalfWord, SignExtend);
+}
+
+void MacroAssemblerLOONG64Compat::load16SignExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeHalfWord, SignExtend);
+}
+
+void MacroAssemblerLOONG64Compat::load32(const Address& address,
+ Register dest) {
+ ma_ld_w(dest, address);
+}
+
+void MacroAssemblerLOONG64Compat::load32(const BaseIndex& address,
+ Register dest) {
+ Register base = address.base;
+ Register index = address.index;
+ int32_t offset = address.offset;
+ uint32_t shift = Imm32::ShiftOf(address.scale).value;
+
+ if (offset != 0) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, Imm32(offset));
+ if (shift != 0) {
+ MOZ_ASSERT(shift <= 4);
+ as_alsl_d(scratch, index, scratch, shift - 1);
+ } else {
+ as_add_d(scratch, index, scratch);
+ }
+ as_ldx_w(dest, base, scratch);
+ } else if (shift != 0) {
+ ScratchRegisterScope scratch(asMasm());
+ as_slli_d(scratch, index, shift);
+ as_ldx_w(dest, base, scratch);
+ } else {
+ as_ldx_w(dest, base, index);
+ }
+}
+
+void MacroAssemblerLOONG64Compat::load32(AbsoluteAddress address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(ImmPtr(address.addr), scratch);
+ load32(Address(scratch, 0), dest);
+}
+
+void MacroAssemblerLOONG64Compat::load32(wasm::SymbolicAddress address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(address, scratch);
+ load32(Address(scratch, 0), dest);
+}
+
+void MacroAssemblerLOONG64Compat::loadPtr(const Address& address,
+ Register dest) {
+ ma_ld_d(dest, address);
+}
+
+void MacroAssemblerLOONG64Compat::loadPtr(const BaseIndex& src, Register dest) {
+ Register base = src.base;
+ Register index = src.index;
+ int32_t offset = src.offset;
+ uint32_t shift = Imm32::ShiftOf(src.scale).value;
+
+ if (offset != 0) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, Imm32(offset));
+ if (shift != 0) {
+ MOZ_ASSERT(shift <= 4);
+ as_alsl_d(scratch, index, scratch, shift - 1);
+ } else {
+ as_add_d(scratch, index, scratch);
+ }
+ as_ldx_d(dest, base, scratch);
+ } else if (shift != 0) {
+ ScratchRegisterScope scratch(asMasm());
+ as_slli_d(scratch, index, shift);
+ as_ldx_d(dest, base, scratch);
+ } else {
+ as_ldx_d(dest, base, index);
+ }
+}
+
+void MacroAssemblerLOONG64Compat::loadPtr(AbsoluteAddress address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(ImmPtr(address.addr), scratch);
+ loadPtr(Address(scratch, 0), dest);
+}
+
+void MacroAssemblerLOONG64Compat::loadPtr(wasm::SymbolicAddress address,
+ Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(address, scratch);
+ loadPtr(Address(scratch, 0), dest);
+}
+
+void MacroAssemblerLOONG64Compat::loadPrivate(const Address& address,
+ Register dest) {
+ loadPtr(address, dest);
+}
+
+void MacroAssemblerLOONG64Compat::store8(Imm32 imm, const Address& address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch2, imm);
+ ma_store(scratch2, address, SizeByte);
+}
+
+void MacroAssemblerLOONG64Compat::store8(Register src, const Address& address) {
+ ma_store(src, address, SizeByte);
+}
+
+void MacroAssemblerLOONG64Compat::store8(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeByte);
+}
+
+void MacroAssemblerLOONG64Compat::store8(Register src, const BaseIndex& dest) {
+ ma_store(src, dest, SizeByte);
+}
+
+void MacroAssemblerLOONG64Compat::store16(Imm32 imm, const Address& address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch2, imm);
+ ma_store(scratch2, address, SizeHalfWord);
+}
+
+void MacroAssemblerLOONG64Compat::store16(Register src,
+ const Address& address) {
+ ma_store(src, address, SizeHalfWord);
+}
+
+void MacroAssemblerLOONG64Compat::store16(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeHalfWord);
+}
+
+void MacroAssemblerLOONG64Compat::store16(Register src,
+ const BaseIndex& address) {
+ ma_store(src, address, SizeHalfWord);
+}
+
+void MacroAssemblerLOONG64Compat::store32(Register src,
+ AbsoluteAddress address) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(ImmPtr(address.addr), scratch);
+ store32(src, Address(scratch, 0));
+}
+
+void MacroAssemblerLOONG64Compat::store32(Register src,
+ const Address& address) {
+ ma_store(src, address, SizeWord);
+}
+
+void MacroAssemblerLOONG64Compat::store32(Imm32 src, const Address& address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ move32(src, scratch2);
+ ma_store(scratch2, address, SizeWord);
+}
+
+void MacroAssemblerLOONG64Compat::store32(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeWord);
+}
+
+void MacroAssemblerLOONG64Compat::store32(Register src, const BaseIndex& dest) {
+ ma_store(src, dest, SizeWord);
+}
+
+template <typename T>
+void MacroAssemblerLOONG64Compat::storePtr(ImmWord imm, T address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_li(scratch2, imm);
+ ma_store(scratch2, address, SizeDouble);
+}
+
+template void MacroAssemblerLOONG64Compat::storePtr<Address>(ImmWord imm,
+ Address address);
+template void MacroAssemblerLOONG64Compat::storePtr<BaseIndex>(
+ ImmWord imm, BaseIndex address);
+
+template <typename T>
+void MacroAssemblerLOONG64Compat::storePtr(ImmPtr imm, T address) {
+ storePtr(ImmWord(uintptr_t(imm.value)), address);
+}
+
+template void MacroAssemblerLOONG64Compat::storePtr<Address>(ImmPtr imm,
+ Address address);
+template void MacroAssemblerLOONG64Compat::storePtr<BaseIndex>(
+ ImmPtr imm, BaseIndex address);
+
+template <typename T>
+void MacroAssemblerLOONG64Compat::storePtr(ImmGCPtr imm, T address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ movePtr(imm, scratch2);
+ storePtr(scratch2, address);
+}
+
+template void MacroAssemblerLOONG64Compat::storePtr<Address>(ImmGCPtr imm,
+ Address address);
+template void MacroAssemblerLOONG64Compat::storePtr<BaseIndex>(
+ ImmGCPtr imm, BaseIndex address);
+
+void MacroAssemblerLOONG64Compat::storePtr(Register src,
+ const Address& address) {
+ ma_st_d(src, address);
+}
+
+void MacroAssemblerLOONG64Compat::storePtr(Register src,
+ const BaseIndex& address) {
+ Register base = address.base;
+ Register index = address.index;
+ int32_t offset = address.offset;
+ int32_t shift = Imm32::ShiftOf(address.scale).value;
+
+ if ((offset == 0) && (shift == 0)) {
+ as_stx_d(src, base, index);
+ } else if (is_intN(offset, 12)) {
+ ScratchRegisterScope scratch(asMasm());
+ if (shift == 0) {
+ as_add_d(scratch, base, index);
+ } else {
+ as_alsl_d(scratch, index, base, shift - 1);
+ }
+ as_st_d(src, scratch, offset);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, Imm32(offset));
+ if (shift == 0) {
+ as_add_d(scratch, scratch, index);
+ } else {
+ as_alsl_d(scratch, index, scratch, shift - 1);
+ }
+ as_stx_d(src, base, scratch);
+ }
+}
+
+void MacroAssemblerLOONG64Compat::storePtr(Register src, AbsoluteAddress dest) {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(ImmPtr(dest.addr), scratch);
+ storePtr(src, Address(scratch, 0));
+}
+
+void MacroAssemblerLOONG64Compat::testNullSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(asMasm());
+ splitTag(value, scratch2);
+ ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_NULL), cond);
+}
+
+void MacroAssemblerLOONG64Compat::testObjectSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(asMasm());
+ splitTag(value, scratch2);
+ ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_OBJECT), cond);
+}
+
+void MacroAssemblerLOONG64Compat::testUndefinedSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(asMasm());
+ splitTag(value, scratch2);
+ ma_cmp_set(dest, scratch2, ImmTag(JSVAL_TAG_UNDEFINED), cond);
+}
+
+void MacroAssemblerLOONG64Compat::unboxInt32(const ValueOperand& operand,
+ Register dest) {
+ as_slli_w(dest, operand.valueReg(), 0);
+}
+
+void MacroAssemblerLOONG64Compat::unboxInt32(Register src, Register dest) {
+ as_slli_w(dest, src, 0);
+}
+
+void MacroAssemblerLOONG64Compat::unboxInt32(const Address& src,
+ Register dest) {
+ load32(Address(src.base, src.offset), dest);
+}
+
+void MacroAssemblerLOONG64Compat::unboxInt32(const BaseIndex& src,
+ Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ computeScaledAddress(src, scratch2);
+ load32(Address(scratch2, src.offset), dest);
+}
+
+void MacroAssemblerLOONG64Compat::unboxBoolean(const ValueOperand& operand,
+ Register dest) {
+ as_slli_w(dest, operand.valueReg(), 0);
+}
+
+void MacroAssemblerLOONG64Compat::unboxBoolean(Register src, Register dest) {
+ as_slli_w(dest, src, 0);
+}
+
+void MacroAssemblerLOONG64Compat::unboxBoolean(const Address& src,
+ Register dest) {
+ ma_ld_w(dest, src);
+}
+
+void MacroAssemblerLOONG64Compat::unboxBoolean(const BaseIndex& src,
+ Register dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ computeScaledAddress(src, scratch2);
+ ma_ld_w(dest, Address(scratch2, src.offset));
+}
+
+void MacroAssemblerLOONG64Compat::unboxDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ as_movgr2fr_d(dest, operand.valueReg());
+}
+
+void MacroAssemblerLOONG64Compat::unboxDouble(const Address& src,
+ FloatRegister dest) {
+ ma_fld_d(dest, Address(src.base, src.offset));
+}
+
+void MacroAssemblerLOONG64Compat::unboxDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(src, scratch2);
+ unboxDouble(ValueOperand(scratch2), dest);
+}
+
+void MacroAssemblerLOONG64Compat::unboxString(const ValueOperand& operand,
+ Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_STRING);
+}
+
+void MacroAssemblerLOONG64Compat::unboxString(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+}
+
+void MacroAssemblerLOONG64Compat::unboxString(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+}
+
+void MacroAssemblerLOONG64Compat::unboxSymbol(const ValueOperand& operand,
+ Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void MacroAssemblerLOONG64Compat::unboxSymbol(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void MacroAssemblerLOONG64Compat::unboxSymbol(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void MacroAssemblerLOONG64Compat::unboxBigInt(const ValueOperand& operand,
+ Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT);
+}
+
+void MacroAssemblerLOONG64Compat::unboxBigInt(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+}
+
+void MacroAssemblerLOONG64Compat::unboxBigInt(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+}
+
+void MacroAssemblerLOONG64Compat::unboxObject(const ValueOperand& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void MacroAssemblerLOONG64Compat::unboxObject(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void MacroAssemblerLOONG64Compat::unboxObject(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void MacroAssemblerLOONG64Compat::unboxValue(const ValueOperand& src,
+ AnyRegister dest,
+ JSValueType type) {
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.valueReg(), dest.fpu());
+ ma_b(&end, ShortJump);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else {
+ unboxNonDouble(src, dest.gpr(), type);
+ }
+}
+
+void MacroAssemblerLOONG64Compat::boxDouble(FloatRegister src,
+ const ValueOperand& dest,
+ FloatRegister) {
+ as_movfr2gr_d(dest.valueReg(), src);
+}
+
+void MacroAssemblerLOONG64Compat::boxNonDouble(JSValueType type, Register src,
+ const ValueOperand& dest) {
+ boxValue(type, src, dest.valueReg());
+}
+
+void MacroAssemblerLOONG64Compat::boolValueToDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+ convertBoolToInt32(operand.valueReg(), scratch);
+ convertInt32ToDouble(scratch, dest);
+}
+
+void MacroAssemblerLOONG64Compat::int32ValueToDouble(
+ const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+}
+
+void MacroAssemblerLOONG64Compat::boolValueToFloat32(
+ const ValueOperand& operand, FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+ convertBoolToInt32(operand.valueReg(), scratch);
+ convertInt32ToFloat32(scratch, dest);
+}
+
+void MacroAssemblerLOONG64Compat::int32ValueToFloat32(
+ const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+}
+
+void MacroAssemblerLOONG64Compat::loadConstantFloat32(float f,
+ FloatRegister dest) {
+ ma_lis(dest, f);
+}
+
+void MacroAssemblerLOONG64Compat::loadInt32OrDouble(const Address& src,
+ FloatRegister dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ Label end;
+
+ // If it's an int, convert it to double.
+ loadPtr(Address(src.base, src.offset), scratch2);
+ as_movgr2fr_d(dest, scratch2);
+ as_srli_d(scratch2, scratch2, JSVAL_TAG_SHIFT);
+ asMasm().branchTestInt32(Assembler::NotEqual, scratch2, &end);
+ as_ffint_d_w(dest, dest);
+
+ bind(&end);
+}
+
+void MacroAssemblerLOONG64Compat::loadInt32OrDouble(const BaseIndex& addr,
+ FloatRegister dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ Label end;
+
+ // If it's an int, convert it to double.
+ computeScaledAddress(addr, scratch2);
+ // Since we only have one scratch, we need to stomp over it with the tag.
+ loadPtr(Address(scratch2, 0), scratch2);
+ as_movgr2fr_d(dest, scratch2);
+ as_srli_d(scratch2, scratch2, JSVAL_TAG_SHIFT);
+ asMasm().branchTestInt32(Assembler::NotEqual, scratch2, &end);
+ as_ffint_d_w(dest, dest);
+
+ bind(&end);
+}
+
+void MacroAssemblerLOONG64Compat::loadConstantDouble(double dp,
+ FloatRegister dest) {
+ ma_lid(dest, dp);
+}
+
+Register MacroAssemblerLOONG64Compat::extractObject(const Address& address,
+ Register scratch) {
+ loadPtr(Address(address.base, address.offset), scratch);
+ as_bstrpick_d(scratch, scratch, JSVAL_TAG_SHIFT - 1, 0);
+ return scratch;
+}
+
+Register MacroAssemblerLOONG64Compat::extractTag(const Address& address,
+ Register scratch) {
+ loadPtr(Address(address.base, address.offset), scratch);
+ as_bstrpick_d(scratch, scratch, 63, JSVAL_TAG_SHIFT);
+ return scratch;
+}
+
+Register MacroAssemblerLOONG64Compat::extractTag(const BaseIndex& address,
+ Register scratch) {
+ computeScaledAddress(address, scratch);
+ return extractTag(Address(scratch, address.offset), scratch);
+}
+
+/////////////////////////////////////////////////////////////////
+// X86/X64-common/ARM/LoongArch interface.
+/////////////////////////////////////////////////////////////////
+void MacroAssemblerLOONG64Compat::storeValue(ValueOperand val,
+ const Address& dest) {
+ storePtr(val.valueReg(), Address(dest.base, dest.offset));
+}
+
+void MacroAssemblerLOONG64Compat::storeValue(ValueOperand val,
+ const BaseIndex& dest) {
+ storePtr(val.valueReg(), dest);
+}
+
+void MacroAssemblerLOONG64Compat::storeValue(JSValueType type, Register reg,
+ Address dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(dest.base != scratch2);
+
+ tagValue(type, reg, ValueOperand(scratch2));
+ storePtr(scratch2, dest);
+}
+
+void MacroAssemblerLOONG64Compat::storeValue(JSValueType type, Register reg,
+ BaseIndex dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(dest.base != scratch2);
+
+ tagValue(type, reg, ValueOperand(scratch2));
+ storePtr(scratch2, dest);
+}
+
+void MacroAssemblerLOONG64Compat::storeValue(const Value& val, Address dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(dest.base != scratch2);
+
+ if (val.isGCThing()) {
+ writeDataRelocation(val);
+ movWithPatch(ImmWord(val.asRawBits()), scratch2);
+ } else {
+ ma_li(scratch2, ImmWord(val.asRawBits()));
+ }
+ storePtr(scratch2, dest);
+}
+
+void MacroAssemblerLOONG64Compat::storeValue(const Value& val, BaseIndex dest) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(dest.base != scratch2);
+
+ if (val.isGCThing()) {
+ writeDataRelocation(val);
+ movWithPatch(ImmWord(val.asRawBits()), scratch2);
+ } else {
+ ma_li(scratch2, ImmWord(val.asRawBits()));
+ }
+ storePtr(scratch2, dest);
+}
+
+void MacroAssemblerLOONG64Compat::loadValue(Address src, ValueOperand val) {
+ loadPtr(src, val.valueReg());
+}
+
+void MacroAssemblerLOONG64Compat::loadValue(const BaseIndex& src,
+ ValueOperand val) {
+ loadPtr(src, val.valueReg());
+}
+
+void MacroAssemblerLOONG64Compat::tagValue(JSValueType type, Register payload,
+ ValueOperand dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest.valueReg() != scratch);
+
+ if (payload == dest.valueReg()) {
+ as_or(scratch, payload, zero);
+ payload = scratch;
+ }
+ ma_li(dest.valueReg(), ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)));
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ as_bstrins_d(dest.valueReg(), payload, 31, 0);
+ } else {
+ as_bstrins_d(dest.valueReg(), payload, JSVAL_TAG_SHIFT - 1, 0);
+ }
+}
+
+void MacroAssemblerLOONG64Compat::pushValue(ValueOperand val) {
+ push(val.valueReg());
+}
+
+void MacroAssemblerLOONG64Compat::pushValue(const Address& addr) { push(addr); }
+
+void MacroAssemblerLOONG64Compat::popValue(ValueOperand val) {
+ pop(val.valueReg());
+}
+
+void MacroAssemblerLOONG64Compat::breakpoint(uint32_t value) {
+ as_break(value);
+}
+
+void MacroAssemblerLOONG64Compat::handleFailureWithHandlerTail(
+ Label* profilerExitTail, Label* bailoutTail) {
+ // Reserve space for exception information.
+ int size = (sizeof(ResumeFromException) + ABIStackAlignment) &
+ ~(ABIStackAlignment - 1);
+ asMasm().subPtr(Imm32(size), StackPointer);
+ mov(StackPointer, a0); // Use a0 since it is a first function argument
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(a1);
+ asMasm().passABIArg(a0);
+ asMasm().callWithABI<Fn, HandleException>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ // Already clobbered a0, so use it...
+ load32(Address(StackPointer, ResumeFromException::offsetOfKind()), a0);
+ asMasm().branch32(Assembler::Equal, a0,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, a0,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, a0,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::WasmCatch),
+ &wasmCatch);
+
+ breakpoint(); // Invalid kind.
+
+ // No exception handler. Load the error value, restore state and return from
+ // the entry frame.
+ bind(&entryFrame);
+ asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+
+ // We're going to be returning by the ion calling convention
+ ma_pop(ra);
+ as_jirl(zero, ra, BOffImm16(0));
+
+ // If we found a catch handler, this must be a baseline frame. Restore
+ // state and jump to the catch block.
+ bind(&catch_);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), a0);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ jump(a0);
+
+ // If we found a finally block, this must be a baseline frame. Push two
+ // values expected by the finally block: the exception and BooleanValue(true).
+ bind(&finally);
+ ValueOperand exception = ValueOperand(a1);
+ loadValue(Address(sp, ResumeFromException::offsetOfException()), exception);
+
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a0);
+ loadPtr(Address(sp, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp);
+
+ pushValue(exception);
+ pushValue(BooleanValue(true));
+ jump(a0);
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ jump(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(Address(StackPointer, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ // Test if profiler enabled.
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ as_or(StackPointer, FramePointer, zero);
+ pop(FramePointer);
+ ret();
+
+ // If we are bailing out to baseline to handle an exception, jump to
+ // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
+ bind(&bailout);
+ loadPtr(Address(sp, ResumeFromException::offsetOfBailoutInfo()), a2);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ ma_li(ReturnReg, Imm32(1));
+ jump(bailoutTail);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ ret();
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a1);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ jump(a1);
+}
+
+CodeOffset MacroAssemblerLOONG64Compat::toggledJump(Label* label) {
+ CodeOffset ret(nextOffset().getOffset());
+ ma_b(label);
+ return ret;
+}
+
+CodeOffset MacroAssemblerLOONG64Compat::toggledCall(JitCode* target,
+ bool enabled) {
+ ScratchRegisterScope scratch(asMasm());
+ BufferOffset bo = nextOffset();
+ CodeOffset offset(bo.getOffset()); // first instruction location,not changed.
+ addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(scratch, ImmPtr(target->raw()));
+ if (enabled) {
+ as_jirl(ra, scratch, BOffImm16(0));
+ } else {
+ as_nop();
+ }
+ MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() ==
+ ToggledCallSize(nullptr));
+ return offset; // location of first instruction of call instr sequence.
+}
+
+void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
+ Register pointer) {
+ if (IsShiftInScaleRange(shift)) {
+ computeEffectiveAddress(
+ BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer);
+ return;
+ }
+ lshift32(Imm32(shift), indexTemp32);
+ addPtr(indexTemp32, pointer);
+}
+
+//}}} check_macroassembler_style
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/loong64/MacroAssembler-loong64.h b/js/src/jit/loong64/MacroAssembler-loong64.h
new file mode 100644
index 0000000000..72b1e4804b
--- /dev/null
+++ b/js/src/jit/loong64/MacroAssembler-loong64.h
@@ -0,0 +1,1042 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_MacroAssembler_loong64_h
+#define jit_loong64_MacroAssembler_loong64_h
+
+#include "jit/loong64/Assembler-loong64.h"
+#include "jit/MoveResolver.h"
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+enum LoadStoreSize {
+ SizeByte = 8,
+ SizeHalfWord = 16,
+ SizeWord = 32,
+ SizeDouble = 64
+};
+
+enum LoadStoreExtension { ZeroExtend = 0, SignExtend = 1 };
+
+enum JumpKind { LongJump = 0, ShortJump = 1 };
+
+static Register CallReg = t8;
+
+enum LiFlags {
+ Li64 = 0,
+ Li48 = 1,
+};
+
+struct ImmShiftedTag : public ImmWord {
+ explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {}
+
+ explicit ImmShiftedTag(JSValueType type)
+ : ImmWord(uintptr_t(JSValueShiftedTag(JSVAL_TYPE_TO_SHIFTED_TAG(type)))) {
+ }
+};
+
+struct ImmTag : public Imm32 {
+ ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {}
+};
+
+static const int defaultShift = 3;
+static_assert(1 << defaultShift == sizeof(JS::Value),
+ "The defaultShift is wrong");
+
+// See documentation for ScratchTagScope and ScratchTagScopeRelease in
+// MacroAssembler-x64.h.
+
+class ScratchTagScope : public SecondScratchRegisterScope {
+ public:
+ ScratchTagScope(MacroAssembler& masm, const ValueOperand&)
+ : SecondScratchRegisterScope(masm) {}
+};
+
+class ScratchTagScopeRelease {
+ ScratchTagScope* ts_;
+
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
+ ts_->release();
+ }
+ ~ScratchTagScopeRelease() { ts_->reacquire(); }
+};
+
+class MacroAssemblerLOONG64 : public Assembler {
+ protected:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ Condition ma_cmp(Register rd, Register lhs, Register rhs, Condition c);
+ Condition ma_cmp(Register rd, Register lhs, Imm32 imm, Condition c);
+
+ void compareFloatingPoint(FloatFormat fmt, FloatRegister lhs,
+ FloatRegister rhs, DoubleCondition c,
+ FPConditionBit fcc = FCC0);
+
+ public:
+ void ma_li(Register dest, CodeLabel* label);
+ void ma_li(Register dest, ImmWord imm);
+ void ma_liPatchable(Register dest, ImmPtr imm);
+ void ma_liPatchable(Register dest, ImmWord imm, LiFlags flags = Li48);
+
+ // load
+ void ma_ld_b(Register dest, Address address);
+ void ma_ld_h(Register dest, Address address);
+ void ma_ld_w(Register dest, Address address);
+ void ma_ld_d(Register dest, Address address);
+ void ma_ld_bu(Register dest, Address address);
+ void ma_ld_hu(Register dest, Address address);
+ void ma_ld_wu(Register dest, Address address);
+ void ma_load(Register dest, Address address, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // store
+ void ma_st_b(Register src, Address address);
+ void ma_st_h(Register src, Address address);
+ void ma_st_w(Register src, Address address);
+ void ma_st_d(Register src, Address address);
+ void ma_store(Register data, Address address, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // arithmetic based ops
+ // add
+ void ma_add_d(Register rd, Register rj, Imm32 imm);
+ void ma_add32TestOverflow(Register rd, Register rj, Register rk,
+ Label* overflow);
+ void ma_add32TestOverflow(Register rd, Register rj, Imm32 imm,
+ Label* overflow);
+ void ma_addPtrTestOverflow(Register rd, Register rj, Register rk,
+ Label* overflow);
+ void ma_addPtrTestOverflow(Register rd, Register rj, Imm32 imm,
+ Label* overflow);
+ void ma_addPtrTestOverflow(Register rd, Register rj, ImmWord imm,
+ Label* overflow);
+ void ma_addPtrTestCarry(Condition cond, Register rd, Register rj, Register rk,
+ Label* overflow);
+ void ma_addPtrTestCarry(Condition cond, Register rd, Register rj, Imm32 imm,
+ Label* overflow);
+ void ma_addPtrTestCarry(Condition cond, Register rd, Register rj, ImmWord imm,
+ Label* overflow);
+
+ // subtract
+ void ma_sub_d(Register rd, Register rj, Imm32 imm);
+ void ma_sub32TestOverflow(Register rd, Register rj, Register rk,
+ Label* overflow);
+ void ma_subPtrTestOverflow(Register rd, Register rj, Register rk,
+ Label* overflow);
+ void ma_subPtrTestOverflow(Register rd, Register rj, Imm32 imm,
+ Label* overflow);
+
+ // multiplies. For now, there are only few that we care about.
+ void ma_mul_d(Register rd, Register rj, Imm32 imm);
+ void ma_mulh_d(Register rd, Register rj, Imm32 imm);
+ void ma_mulPtrTestOverflow(Register rd, Register rj, Register rk,
+ Label* overflow);
+
+ // stack
+ void ma_pop(Register r);
+ void ma_push(Register r);
+
+ void branchWithCode(InstImm code, Label* label, JumpKind jumpKind);
+ // branches when done from within la-specific code
+ void ma_b(Register lhs, ImmWord imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Register lhs, Address addr, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, Imm32 imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, ImmGCPtr imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, Register rhs, Label* l, Condition c,
+ JumpKind jumpKind = LongJump) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rhs != scratch);
+ ma_ld_d(scratch, addr);
+ ma_b(scratch, rhs, l, c, jumpKind);
+ }
+
+ void ma_bl(Label* l);
+
+ // fp instructions
+ void ma_lid(FloatRegister dest, double value);
+
+ void ma_mv(FloatRegister src, ValueOperand dest);
+ void ma_mv(ValueOperand src, FloatRegister dest);
+
+ void ma_fld_s(FloatRegister ft, Address address);
+ void ma_fld_d(FloatRegister ft, Address address);
+ void ma_fst_d(FloatRegister ft, Address address);
+ void ma_fst_s(FloatRegister ft, Address address);
+
+ void ma_pop(FloatRegister f);
+ void ma_push(FloatRegister f);
+
+ void ma_cmp_set(Register dst, Register lhs, ImmWord imm, Condition c);
+ void ma_cmp_set(Register dst, Register lhs, ImmPtr imm, Condition c);
+ void ma_cmp_set(Register dst, Address address, Imm32 imm, Condition c);
+ void ma_cmp_set(Register dst, Address address, ImmWord imm, Condition c);
+
+ void moveIfZero(Register dst, Register src, Register cond) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dst != scratch && cond != scratch);
+ as_masknez(scratch, src, cond);
+ as_maskeqz(dst, dst, cond);
+ as_or(dst, dst, scratch);
+ }
+ void moveIfNotZero(Register dst, Register src, Register cond) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dst != scratch && cond != scratch);
+ as_maskeqz(scratch, src, cond);
+ as_masknez(dst, dst, cond);
+ as_or(dst, dst, scratch);
+ }
+
+ // These functions abstract the access to high part of the double precision
+ // float register. They are intended to work on both 32 bit and 64 bit
+ // floating point coprocessor.
+ void moveToDoubleHi(Register src, FloatRegister dest) {
+ as_movgr2frh_w(dest, src);
+ }
+ void moveFromDoubleHi(FloatRegister src, Register dest) {
+ as_movfrh2gr_s(dest, src);
+ }
+
+ void moveToDouble(Register src, FloatRegister dest) {
+ as_movgr2fr_d(dest, src);
+ }
+ void moveFromDouble(FloatRegister src, Register dest) {
+ as_movfr2gr_d(dest, src);
+ }
+
+ public:
+ void ma_li(Register dest, ImmGCPtr ptr);
+
+ void ma_li(Register dest, Imm32 imm);
+ void ma_liPatchable(Register dest, Imm32 imm);
+
+ void ma_rotr_w(Register rd, Register rj, Imm32 shift);
+
+ void ma_fmovz(FloatFormat fmt, FloatRegister fd, FloatRegister fj,
+ Register rk);
+ void ma_fmovn(FloatFormat fmt, FloatRegister fd, FloatRegister fj,
+ Register rk);
+
+ void ma_and(Register rd, Register rj, Imm32 imm, bool bit32 = false);
+
+ void ma_or(Register rd, Register rj, Imm32 imm, bool bit32 = false);
+
+ void ma_xor(Register rd, Register rj, Imm32 imm, bool bit32 = false);
+
+ // load
+ void ma_load(Register dest, const BaseIndex& src,
+ LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // store
+ void ma_store(Register data, const BaseIndex& dest,
+ LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+ void ma_store(Imm32 imm, const BaseIndex& dest, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // arithmetic based ops
+ // add
+ void ma_add_w(Register rd, Register rj, Imm32 imm);
+ void ma_add32TestCarry(Condition cond, Register rd, Register rj, Register rk,
+ Label* overflow);
+ void ma_add32TestCarry(Condition cond, Register rd, Register rj, Imm32 imm,
+ Label* overflow);
+
+ // subtract
+ void ma_sub_w(Register rd, Register rj, Imm32 imm);
+ void ma_sub_w(Register rd, Register rj, Register rk);
+ void ma_sub32TestOverflow(Register rd, Register rj, Imm32 imm,
+ Label* overflow);
+
+ // multiplies. For now, there are only few that we care about.
+ void ma_mul(Register rd, Register rj, Imm32 imm);
+ void ma_mul32TestOverflow(Register rd, Register rj, Register rk,
+ Label* overflow);
+ void ma_mul32TestOverflow(Register rd, Register rj, Imm32 imm,
+ Label* overflow);
+
+ // divisions
+ void ma_div_branch_overflow(Register rd, Register rj, Register rk,
+ Label* overflow);
+ void ma_div_branch_overflow(Register rd, Register rj, Imm32 imm,
+ Label* overflow);
+
+ // fast mod, uses scratch registers, and thus needs to be in the assembler
+ // implicitly assumes that we can overwrite dest at the beginning of the
+ // sequence
+ void ma_mod_mask(Register src, Register dest, Register hold, Register remain,
+ int32_t shift, Label* negZero = nullptr);
+
+ // branches when done from within la-specific code
+ void ma_b(Register lhs, Register rhs, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Register lhs, Imm32 imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Register lhs, ImmPtr imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Register lhs, ImmGCPtr imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ ma_li(scratch, imm);
+ ma_b(lhs, scratch, l, c, jumpKind);
+ }
+
+ void ma_b(Label* l, JumpKind jumpKind = LongJump);
+
+ // fp instructions
+ void ma_lis(FloatRegister dest, float value);
+
+ void ma_fst_d(FloatRegister src, BaseIndex address);
+ void ma_fst_s(FloatRegister src, BaseIndex address);
+
+ void ma_fld_d(FloatRegister dest, const BaseIndex& src);
+ void ma_fld_s(FloatRegister dest, const BaseIndex& src);
+
+ // FP branches
+ void ma_bc_s(FloatRegister lhs, FloatRegister rhs, Label* label,
+ DoubleCondition c, JumpKind jumpKind = LongJump,
+ FPConditionBit fcc = FCC0);
+ void ma_bc_d(FloatRegister lhs, FloatRegister rhs, Label* label,
+ DoubleCondition c, JumpKind jumpKind = LongJump,
+ FPConditionBit fcc = FCC0);
+
+ void ma_call(ImmPtr dest);
+
+ void ma_jump(ImmPtr dest);
+
+ void ma_cmp_set(Register dst, Register lhs, Register rhs, Condition c);
+ void ma_cmp_set(Register dst, Register lhs, Imm32 imm, Condition c);
+ void ma_cmp_set_double(Register dst, FloatRegister lhs, FloatRegister rhs,
+ DoubleCondition c);
+ void ma_cmp_set_float32(Register dst, FloatRegister lhs, FloatRegister rhs,
+ DoubleCondition c);
+
+ void moveToDoubleLo(Register src, FloatRegister dest) {
+ as_movgr2fr_w(dest, src);
+ }
+ void moveFromDoubleLo(FloatRegister src, Register dest) {
+ as_movfr2gr_s(dest, src);
+ }
+
+ void moveToFloat32(Register src, FloatRegister dest) {
+ as_movgr2fr_w(dest, src);
+ }
+ void moveFromFloat32(FloatRegister src, Register dest) {
+ as_movfr2gr_s(dest, src);
+ }
+
+ // Evaluate srcDest = minmax<isMax>{Float32,Double}(srcDest, other).
+ // Handle NaN specially if handleNaN is true.
+ void minMaxDouble(FloatRegister srcDest, FloatRegister other, bool handleNaN,
+ bool isMax);
+ void minMaxFloat32(FloatRegister srcDest, FloatRegister other, bool handleNaN,
+ bool isMax);
+
+ void loadDouble(const Address& addr, FloatRegister dest);
+ void loadDouble(const BaseIndex& src, FloatRegister dest);
+
+ // Load a float value into a register, then expand it to a double.
+ void loadFloatAsDouble(const Address& addr, FloatRegister dest);
+ void loadFloatAsDouble(const BaseIndex& src, FloatRegister dest);
+
+ void loadFloat32(const Address& addr, FloatRegister dest);
+ void loadFloat32(const BaseIndex& src, FloatRegister dest);
+
+ void outOfLineWasmTruncateToInt32Check(FloatRegister input, Register output,
+ MIRType fromType, TruncFlags flags,
+ Label* rejoin,
+ wasm::BytecodeOffset trapOffset);
+ void outOfLineWasmTruncateToInt64Check(FloatRegister input, Register64 output,
+ MIRType fromType, TruncFlags flags,
+ Label* rejoin,
+ wasm::BytecodeOffset trapOffset);
+
+ protected:
+ void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, AnyRegister output,
+ Register tmp);
+ void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp);
+};
+
+class MacroAssembler;
+
+class MacroAssemblerLOONG64Compat : public MacroAssemblerLOONG64 {
+ public:
+ using MacroAssemblerLOONG64::call;
+
+ MacroAssemblerLOONG64Compat() {}
+
+ void convertBoolToInt32(Register src, Register dest) {
+ ma_and(dest, src, Imm32(0xff));
+ };
+ void convertInt32ToDouble(Register src, FloatRegister dest) {
+ as_movgr2fr_w(dest, src);
+ as_ffint_d_w(dest, dest);
+ };
+ void convertInt32ToDouble(const Address& src, FloatRegister dest) {
+ ma_fld_s(dest, src);
+ as_ffint_d_w(dest, dest);
+ };
+ void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(scratch != src.base);
+ MOZ_ASSERT(scratch != src.index);
+ computeScaledAddress(src, scratch);
+ convertInt32ToDouble(Address(scratch, src.offset), dest);
+ };
+ void convertUInt32ToDouble(Register src, FloatRegister dest);
+ void convertUInt32ToFloat32(Register src, FloatRegister dest);
+ void convertDoubleToFloat32(FloatRegister src, FloatRegister dest);
+ void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+
+ void convertFloat32ToDouble(FloatRegister src, FloatRegister dest);
+ void convertInt32ToFloat32(Register src, FloatRegister dest);
+ void convertInt32ToFloat32(const Address& src, FloatRegister dest);
+
+ void movq(Register rj, Register rd);
+
+ void computeScaledAddress(const BaseIndex& address, Register dest);
+
+ void computeEffectiveAddress(const Address& address, Register dest) {
+ ma_add_d(dest, address.base, Imm32(address.offset));
+ }
+
+ void computeEffectiveAddress(const BaseIndex& address, Register dest) {
+ computeScaledAddress(address, dest);
+ if (address.offset) {
+ ma_add_d(dest, dest, Imm32(address.offset));
+ }
+ }
+
+ void j(Label* dest) { ma_b(dest); }
+
+ void mov(Register src, Register dest) { as_ori(dest, src, 0); }
+ void mov(ImmWord imm, Register dest) { ma_li(dest, imm); }
+ void mov(ImmPtr imm, Register dest) {
+ mov(ImmWord(uintptr_t(imm.value)), dest);
+ }
+ void mov(CodeLabel* label, Register dest) { ma_li(dest, label); }
+ void mov(Register src, Address dest) { MOZ_CRASH("NYI-IC"); }
+ void mov(Address src, Register dest) { MOZ_CRASH("NYI-IC"); }
+
+ void writeDataRelocation(const Value& val) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // TraceOneDataRelocation.
+ if (val.isGCThing()) {
+ gc::Cell* cell = val.toGCThing();
+ if (cell && gc::IsInsideNursery(cell)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(currentOffset());
+ }
+ }
+
+ void branch(JitCode* c) {
+ ScratchRegisterScope scratch(asMasm());
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(scratch, ImmPtr(c->raw()));
+ as_jirl(zero, scratch, BOffImm16(0));
+ }
+ void branch(const Register reg) { as_jirl(zero, reg, BOffImm16(0)); }
+ void nop() { as_nop(); }
+ void ret() {
+ ma_pop(ra);
+ as_jirl(zero, ra, BOffImm16(0));
+ }
+ inline void retn(Imm32 n);
+ void push(Imm32 imm) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ ma_push(scratch);
+ }
+ void push(ImmWord imm) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ ma_push(scratch);
+ }
+ void push(ImmGCPtr imm) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ ma_push(scratch);
+ }
+ void push(const Address& address) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ loadPtr(address, scratch2);
+ ma_push(scratch2);
+ }
+ void push(Register reg) { ma_push(reg); }
+ void push(FloatRegister reg) { ma_push(reg); }
+ void pop(Register reg) { ma_pop(reg); }
+ void pop(FloatRegister reg) { ma_pop(reg); }
+
+ // Emit a branch that can be toggled to a non-operation. On LOONG64 we use
+ // "andi" instruction to toggle the branch.
+ // See ToggleToJmp(), ToggleToCmp().
+ CodeOffset toggledJump(Label* label);
+
+ // Emit a "jalr" or "nop" instruction. ToggleCall can be used to patch
+ // this instruction.
+ CodeOffset toggledCall(JitCode* target, bool enabled);
+
+ static size_t ToggledCallSize(uint8_t* code) {
+ // Four instructions used in: MacroAssemblerLOONG64Compat::toggledCall
+ return 4 * sizeof(uint32_t);
+ }
+
+ CodeOffset pushWithPatch(ImmWord imm) {
+ ScratchRegisterScope scratch(asMasm());
+ CodeOffset offset = movWithPatch(imm, scratch);
+ ma_push(scratch);
+ return offset;
+ }
+
+ CodeOffset movWithPatch(ImmWord imm, Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ ma_liPatchable(dest, imm, Li64);
+ return offset;
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ ma_liPatchable(dest, imm);
+ return offset;
+ }
+
+ void writeCodePointer(CodeLabel* label) {
+ label->patchAt()->bind(currentOffset());
+ label->setLinkMode(CodeLabel::RawPointer);
+ m_buffer.ensureSpace(sizeof(void*));
+ writeInst(-1);
+ writeInst(-1);
+ }
+
+ void jump(Label* label) { ma_b(label); }
+ void jump(Register reg) { as_jirl(zero, reg, BOffImm16(0)); }
+ void jump(const Address& address) {
+ ScratchRegisterScope scratch(asMasm());
+ loadPtr(address, scratch);
+ as_jirl(zero, scratch, BOffImm16(0));
+ }
+
+ void jump(JitCode* code) { branch(code); }
+
+ void jump(ImmPtr ptr) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ptr, RelocationKind::HARDCODED);
+ ma_jump(ptr);
+ }
+
+ void jump(TrampolinePtr code) { jump(ImmPtr(code.value)); }
+
+ void splitTag(Register src, Register dest) {
+ as_srli_d(dest, src, JSVAL_TAG_SHIFT);
+ }
+
+ void splitTag(const ValueOperand& operand, Register dest) {
+ splitTag(operand.valueReg(), dest);
+ }
+
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ splitTag(value, tag);
+ }
+
+ // unboxing code
+ void unboxNonDouble(const ValueOperand& operand, Register dest,
+ JSValueType type) {
+ unboxNonDouble(operand.valueReg(), dest, type);
+ }
+
+ template <typename T>
+ void unboxNonDouble(T src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ load32(src, dest);
+ return;
+ }
+ loadPtr(src, dest);
+ unboxNonDouble(dest, dest, type);
+ }
+
+ void unboxNonDouble(Register src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ as_slli_w(dest, src, 0);
+ return;
+ }
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(scratch != src);
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ as_xor(dest, src, scratch);
+ }
+
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ static_assert(JS::detail::ValueObjectOrNullBit ==
+ (uint64_t(0x8) << JSVAL_TAG_SHIFT));
+ as_bstrins_d(dest, zero, JSVAL_TAG_SHIFT + 3, JSVAL_TAG_SHIFT + 3);
+ }
+
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ loadPtr(src, dest);
+ as_bstrpick_d(dest, dest, JSVAL_TAG_SHIFT - 1, 0);
+ }
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ as_bstrpick_d(dest, src.valueReg(), JSVAL_TAG_SHIFT - 1, 0);
+ }
+
+ // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
+ void getGCThingValueChunk(const Address& src, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(scratch != dest);
+ loadPtr(src, dest);
+ movePtr(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), scratch);
+ as_and(dest, dest, scratch);
+ }
+ void getGCThingValueChunk(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movePtr(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ as_and(dest, dest, src.valueReg());
+ }
+
+ void unboxInt32(const ValueOperand& operand, Register dest);
+ void unboxInt32(Register src, Register dest);
+ void unboxInt32(const Address& src, Register dest);
+ void unboxInt32(const BaseIndex& src, Register dest);
+ void unboxBoolean(const ValueOperand& operand, Register dest);
+ void unboxBoolean(Register src, Register dest);
+ void unboxBoolean(const Address& src, Register dest);
+ void unboxBoolean(const BaseIndex& src, Register dest);
+ void unboxDouble(const ValueOperand& operand, FloatRegister dest);
+ void unboxDouble(Register src, Register dest);
+ void unboxDouble(const Address& src, FloatRegister dest);
+ void unboxDouble(const BaseIndex& src, FloatRegister dest);
+ void unboxString(const ValueOperand& operand, Register dest);
+ void unboxString(Register src, Register dest);
+ void unboxString(const Address& src, Register dest);
+ void unboxSymbol(const ValueOperand& src, Register dest);
+ void unboxSymbol(Register src, Register dest);
+ void unboxSymbol(const Address& src, Register dest);
+ void unboxBigInt(const ValueOperand& operand, Register dest);
+ void unboxBigInt(Register src, Register dest);
+ void unboxBigInt(const Address& src, Register dest);
+ void unboxObject(const ValueOperand& src, Register dest);
+ void unboxObject(Register src, Register dest);
+ void unboxObject(const Address& src, Register dest);
+ void unboxObject(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType type);
+
+ void notBoolean(const ValueOperand& val) {
+ as_xori(val.valueReg(), val.valueReg(), 1);
+ }
+
+ // boxing code
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister);
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ [[nodiscard]] Register extractObject(const Address& address,
+ Register scratch);
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ unboxObject(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractString(const ValueOperand& value,
+ Register scratch) {
+ unboxString(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ unboxSymbol(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ unboxInt32(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ unboxBoolean(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch);
+ [[nodiscard]] Register extractTag(const BaseIndex& address, Register scratch);
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ splitTag(value, scratch);
+ return scratch;
+ }
+
+ inline void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void loadInt32OrDouble(const Address& src, FloatRegister dest);
+ void loadInt32OrDouble(const BaseIndex& addr, FloatRegister dest);
+ void loadConstantDouble(double dp, FloatRegister dest);
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest);
+
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest);
+
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest);
+
+ // higher level tag testing code
+ Address ToPayload(Address value) { return value; }
+
+ template <typename T>
+ void loadUnboxedValue(const T& address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(address, dest.fpu());
+ } else {
+ unboxNonDouble(address, dest.gpr(), ValueTypeFromMIRType(type));
+ }
+ }
+
+ void storeUnboxedPayload(ValueOperand value, BaseIndex address, size_t nbytes,
+ JSValueType type) {
+ switch (nbytes) {
+ case 8: {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ if (type == JSVAL_TYPE_OBJECT) {
+ unboxObjectOrNull(value, scratch2);
+ } else {
+ unboxNonDouble(value, scratch2, type);
+ }
+ computeEffectiveAddress(address, scratch);
+ as_st_d(scratch2, scratch, 0);
+ return;
+ }
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void storeUnboxedPayload(ValueOperand value, Address address, size_t nbytes,
+ JSValueType type) {
+ switch (nbytes) {
+ case 8: {
+ SecondScratchRegisterScope scratch2(asMasm());
+ if (type == JSVAL_TYPE_OBJECT) {
+ unboxObjectOrNull(value, scratch2);
+ } else {
+ unboxNonDouble(value, scratch2, type);
+ }
+ storePtr(scratch2, address);
+ return;
+ }
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void boxValue(JSValueType type, Register src, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ if (src == dest) {
+ as_ori(scratch, src, 0);
+ src = scratch;
+ }
+#ifdef DEBUG
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ Label upper32BitsSignExtended;
+ as_slli_w(dest, src, 0);
+ ma_b(src, dest, &upper32BitsSignExtended, Equal, ShortJump);
+ breakpoint();
+ bind(&upper32BitsSignExtended);
+ }
+#endif
+ ma_li(dest, ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)));
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ as_bstrins_d(dest, src, 31, 0);
+ } else {
+ as_bstrins_d(dest, src, JSVAL_TAG_SHIFT - 1, 0);
+ }
+ }
+
+ void storeValue(ValueOperand val, const Address& dest);
+ void storeValue(ValueOperand val, const BaseIndex& dest);
+ void storeValue(JSValueType type, Register reg, Address dest);
+ void storeValue(JSValueType type, Register reg, BaseIndex dest);
+ void storeValue(const Value& val, Address dest);
+ void storeValue(const Value& val, BaseIndex dest);
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ loadPtr(src, temp);
+ storePtr(temp, dest);
+ }
+
+ void storePrivateValue(Register src, const Address& dest) {
+ storePtr(src, dest);
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ storePtr(imm, dest);
+ }
+
+ void loadValue(Address src, ValueOperand val);
+ void loadValue(const BaseIndex& src, ValueOperand val);
+
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+
+ void tagValue(JSValueType type, Register payload, ValueOperand dest);
+
+ void pushValue(ValueOperand val);
+ void popValue(ValueOperand val);
+ void pushValue(const Value& val) {
+ if (val.isGCThing()) {
+ ScratchRegisterScope scratch(asMasm());
+ writeDataRelocation(val);
+ movWithPatch(ImmWord(val.asRawBits()), scratch);
+ push(scratch);
+ } else {
+ push(ImmWord(val.asRawBits()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ boxValue(type, reg, scratch2);
+ push(scratch2);
+ }
+ void pushValue(const Address& addr);
+
+ void handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail);
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+ public:
+ // The following functions are exposed for use in platform-shared code.
+
+ inline void incrementInt32Value(const Address& addr);
+
+ void move32(Imm32 imm, Register dest);
+ void move32(Register src, Register dest);
+
+ void movePtr(Register src, Register dest);
+ void movePtr(ImmWord imm, Register dest);
+ void movePtr(ImmPtr imm, Register dest);
+ void movePtr(wasm::SymbolicAddress imm, Register dest);
+ void movePtr(ImmGCPtr imm, Register dest);
+
+ void load8SignExtend(const Address& address, Register dest);
+ void load8SignExtend(const BaseIndex& src, Register dest);
+
+ void load8ZeroExtend(const Address& address, Register dest);
+ void load8ZeroExtend(const BaseIndex& src, Register dest);
+
+ void load16SignExtend(const Address& address, Register dest);
+ void load16SignExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ void load16UnalignedSignExtend(const S& src, Register dest) {
+ load16SignExtend(src, dest);
+ }
+
+ void load16ZeroExtend(const Address& address, Register dest);
+ void load16ZeroExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ void load16UnalignedZeroExtend(const S& src, Register dest) {
+ load16ZeroExtend(src, dest);
+ }
+
+ void load32(const Address& address, Register dest);
+ void load32(const BaseIndex& address, Register dest);
+ void load32(AbsoluteAddress address, Register dest);
+ void load32(wasm::SymbolicAddress address, Register dest);
+
+ template <typename S>
+ void load32Unaligned(const S& src, Register dest) {
+ load32(src, dest);
+ }
+
+ void load64(const Address& address, Register64 dest) {
+ loadPtr(address, dest.reg);
+ }
+ void load64(const BaseIndex& address, Register64 dest) {
+ loadPtr(address, dest.reg);
+ }
+
+ template <typename S>
+ void load64Unaligned(const S& src, Register64 dest) {
+ load64(src, dest);
+ }
+
+ void loadPtr(const Address& address, Register dest);
+ void loadPtr(const BaseIndex& src, Register dest);
+ void loadPtr(AbsoluteAddress address, Register dest);
+ void loadPtr(wasm::SymbolicAddress address, Register dest);
+
+ void loadPrivate(const Address& address, Register dest);
+
+ void store8(Register src, const Address& address);
+ void store8(Imm32 imm, const Address& address);
+ void store8(Register src, const BaseIndex& address);
+ void store8(Imm32 imm, const BaseIndex& address);
+
+ void store16(Register src, const Address& address);
+ void store16(Imm32 imm, const Address& address);
+ void store16(Register src, const BaseIndex& address);
+ void store16(Imm32 imm, const BaseIndex& address);
+
+ template <typename T>
+ void store16Unaligned(Register src, const T& dest) {
+ store16(src, dest);
+ }
+
+ void store32(Register src, AbsoluteAddress address);
+ void store32(Register src, const Address& address);
+ void store32(Register src, const BaseIndex& address);
+ void store32(Imm32 src, const Address& address);
+ void store32(Imm32 src, const BaseIndex& address);
+
+ // NOTE: This will use second scratch on LOONG64. Only ARM needs the
+ // implementation without second scratch.
+ void store32_NoSecondScratch(Imm32 src, const Address& address) {
+ store32(src, address);
+ }
+
+ template <typename T>
+ void store32Unaligned(Register src, const T& dest) {
+ store32(src, dest);
+ }
+
+ void store64(Imm64 imm, Address address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+ void store64(Imm64 imm, const BaseIndex& address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+
+ void store64(Register64 src, Address address) { storePtr(src.reg, address); }
+ void store64(Register64 src, const BaseIndex& address) {
+ storePtr(src.reg, address);
+ }
+
+ template <typename T>
+ void store64Unaligned(Register64 src, const T& dest) {
+ store64(src, dest);
+ }
+
+ template <typename T>
+ void storePtr(ImmWord imm, T address);
+ template <typename T>
+ void storePtr(ImmPtr imm, T address);
+ template <typename T>
+ void storePtr(ImmGCPtr imm, T address);
+ void storePtr(Register src, const Address& address);
+ void storePtr(Register src, const BaseIndex& address);
+ void storePtr(Register src, AbsoluteAddress dest);
+
+ void moveDouble(FloatRegister src, FloatRegister dest) {
+ as_fmov_d(dest, src);
+ }
+
+ void zeroDouble(FloatRegister reg) { moveToDouble(zero, reg); }
+
+ void convertUInt64ToDouble(Register src, FloatRegister dest);
+
+ void breakpoint(uint32_t value = 0);
+
+ void checkStackAlignment() {
+#ifdef DEBUG
+ Label aligned;
+ ScratchRegisterScope scratch(asMasm());
+ as_andi(scratch, sp, ABIStackAlignment - 1);
+ ma_b(scratch, zero, &aligned, Equal, ShortJump);
+ breakpoint();
+ bind(&aligned);
+#endif
+ };
+
+ static void calculateAlignedStackPointer(void** stackPointer);
+
+ void cmpPtrSet(Assembler::Condition cond, Address lhs, ImmPtr rhs,
+ Register dest);
+ void cmpPtrSet(Assembler::Condition cond, Register lhs, Address rhs,
+ Register dest);
+ void cmpPtrSet(Assembler::Condition cond, Address lhs, Register rhs,
+ Register dest);
+
+ void cmp32Set(Assembler::Condition cond, Register lhs, Address rhs,
+ Register dest);
+
+ protected:
+ bool buildOOLFakeExitFrame(void* fakeReturnAddr);
+
+ void wasmLoadI64Impl(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register64 output, Register tmp);
+ void wasmStoreI64Impl(const wasm::MemoryAccessDesc& access, Register64 value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp);
+
+ public:
+ void lea(Operand addr, Register dest) {
+ ma_add_d(dest, addr.baseReg(), Imm32(addr.disp()));
+ }
+
+ void abiret() { as_jirl(zero, ra, BOffImm16(0)); }
+
+ void moveFloat32(FloatRegister src, FloatRegister dest) {
+ as_fmov_s(dest, src);
+ }
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+typedef MacroAssemblerLOONG64Compat MacroAssemblerSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_MacroAssembler_loong64_h */
diff --git a/js/src/jit/loong64/MoveEmitter-loong64.cpp b/js/src/jit/loong64/MoveEmitter-loong64.cpp
new file mode 100644
index 0000000000..a12378be83
--- /dev/null
+++ b/js/src/jit/loong64/MoveEmitter-loong64.cpp
@@ -0,0 +1,326 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/loong64/MoveEmitter-loong64.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void MoveEmitterLOONG64::breakCycle(const MoveOperand& from,
+ const MoveOperand& to, MoveOp::Type type,
+ uint32_t slotId) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (A -> B), which we reach first. We save B, then allow
+ // the original move to continue.
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ ScratchFloat32Scope fpscratch32(masm);
+ masm.loadFloat32(getAdjustedAddress(to), fpscratch32);
+ masm.storeFloat32(fpscratch32, cycleSlot(slotId));
+ } else {
+ masm.storeFloat32(to.floatReg(), cycleSlot(slotId));
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ ScratchDoubleScope fpscratch64(masm);
+ masm.loadDouble(getAdjustedAddress(to), fpscratch64);
+ masm.storeDouble(fpscratch64, cycleSlot(slotId));
+ } else {
+ masm.storeDouble(to.floatReg(), cycleSlot(slotId));
+ }
+ break;
+ case MoveOp::INT32:
+ if (to.isMemory()) {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.load32(getAdjustedAddress(to), scratch2);
+ masm.store32(scratch2, cycleSlot(0));
+ } else {
+ masm.store32(to.reg(), cycleSlot(0));
+ }
+ break;
+ case MoveOp::GENERAL:
+ if (to.isMemory()) {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.loadPtr(getAdjustedAddress(to), scratch2);
+ masm.storePtr(scratch2, cycleSlot(0));
+ } else {
+ masm.storePtr(to.reg(), cycleSlot(0));
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterLOONG64::completeCycle(const MoveOperand& from,
+ const MoveOperand& to, MoveOp::Type type,
+ uint32_t slotId) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (B -> A), which we reach last. We emit a move from the
+ // saved value of B, to A.
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ ScratchFloat32Scope fpscratch32(masm);
+ masm.loadFloat32(cycleSlot(slotId), fpscratch32);
+ masm.storeFloat32(fpscratch32, getAdjustedAddress(to));
+ } else {
+ masm.loadFloat32(cycleSlot(slotId), to.floatReg());
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ ScratchDoubleScope fpscratch64(masm);
+ masm.loadDouble(cycleSlot(slotId), fpscratch64);
+ masm.storeDouble(fpscratch64, getAdjustedAddress(to));
+ } else {
+ masm.loadDouble(cycleSlot(slotId), to.floatReg());
+ }
+ break;
+ case MoveOp::INT32:
+ MOZ_ASSERT(slotId == 0);
+ if (to.isMemory()) {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.load32(cycleSlot(0), scratch2);
+ masm.store32(scratch2, getAdjustedAddress(to));
+ } else {
+ masm.load32(cycleSlot(0), to.reg());
+ }
+ break;
+ case MoveOp::GENERAL:
+ MOZ_ASSERT(slotId == 0);
+ if (to.isMemory()) {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.loadPtr(cycleSlot(0), scratch2);
+ masm.storePtr(scratch2, getAdjustedAddress(to));
+ } else {
+ masm.loadPtr(cycleSlot(0), to.reg());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterLOONG64::emit(const MoveResolver& moves) {
+ if (moves.numCycles()) {
+ // Reserve stack for cycle resolution
+ static_assert(SpillSlotSize == 8);
+ masm.reserveStack(moves.numCycles() * SpillSlotSize);
+ pushedAtCycle_ = masm.framePushed();
+ }
+
+ for (size_t i = 0; i < moves.numMoves(); i++) {
+ emit(moves.getMove(i));
+ }
+}
+
+Address MoveEmitterLOONG64::cycleSlot(uint32_t slot, uint32_t subslot) const {
+ int32_t offset = masm.framePushed() - pushedAtCycle_;
+ MOZ_ASSERT(Imm16::IsInSignedRange(offset));
+ return Address(StackPointer, offset + slot * sizeof(double) + subslot);
+}
+
+int32_t MoveEmitterLOONG64::getAdjustedOffset(const MoveOperand& operand) {
+ MOZ_ASSERT(operand.isMemoryOrEffectiveAddress());
+ if (operand.base() != StackPointer) {
+ return operand.disp();
+ }
+
+ // Adjust offset if stack pointer has been moved.
+ return operand.disp() + masm.framePushed() - pushedAtStart_;
+}
+
+Address MoveEmitterLOONG64::getAdjustedAddress(const MoveOperand& operand) {
+ return Address(operand.base(), getAdjustedOffset(operand));
+}
+
+void MoveEmitterLOONG64::emitMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isGeneralReg()) {
+ if (to.isGeneralReg()) {
+ masm.movePtr(from.reg(), to.reg());
+ } else if (to.isMemory()) {
+ masm.storePtr(from.reg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+ } else if (from.isMemory()) {
+ if (to.isGeneralReg()) {
+ masm.loadPtr(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.loadPtr(getAdjustedAddress(from), scratch2);
+ masm.storePtr(scratch2, getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+ } else if (from.isEffectiveAddress()) {
+ if (to.isGeneralReg()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.computeEffectiveAddress(getAdjustedAddress(from), scratch2);
+ masm.storePtr(scratch2, getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+}
+
+void MoveEmitterLOONG64::emitInt32Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isGeneralReg()) {
+ if (to.isGeneralReg()) {
+ masm.move32(from.reg(), to.reg());
+ } else if (to.isMemory()) {
+ masm.store32(from.reg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+ } else if (from.isMemory()) {
+ if (to.isGeneralReg()) {
+ masm.load32(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.load32(getAdjustedAddress(from), scratch2);
+ masm.store32(scratch2, getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+ } else if (from.isEffectiveAddress()) {
+ if (to.isGeneralReg()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.computeEffectiveAddress(getAdjustedAddress(from), scratch2);
+ masm.store32(scratch2, getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+}
+
+void MoveEmitterLOONG64::emitFloat32Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveFloat32(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ // This should only be used when passing float parameter in a1,a2,a3
+ MOZ_ASSERT(to.reg() == a1 || to.reg() == a2 || to.reg() == a3);
+ masm.moveFromFloat32(from.floatReg(), to.reg());
+ } else {
+ MOZ_ASSERT(to.isMemory());
+ masm.storeFloat32(from.floatReg(), getAdjustedAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ MOZ_ASSERT(from.isMemory());
+ masm.loadFloat32(getAdjustedAddress(from), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ MOZ_ASSERT(from.isMemory());
+ // This should only be used when passing float parameter in a1,a2,a3
+ MOZ_ASSERT(to.reg() == a1 || to.reg() == a2 || to.reg() == a3);
+ masm.loadPtr(getAdjustedAddress(from), to.reg());
+ } else {
+ MOZ_ASSERT(from.isMemory());
+ MOZ_ASSERT(to.isMemory());
+ ScratchFloat32Scope fpscratch32(masm);
+ masm.loadFloat32(getAdjustedAddress(from), fpscratch32);
+ masm.storeFloat32(fpscratch32, getAdjustedAddress(to));
+ }
+}
+
+void MoveEmitterLOONG64::emitDoubleMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveDouble(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ masm.moveFromDouble(from.floatReg(), to.reg());
+ } else {
+ MOZ_ASSERT(to.isMemory());
+ masm.storeDouble(from.floatReg(), getAdjustedAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ if (from.isMemory()) {
+ masm.loadDouble(getAdjustedAddress(from), to.floatReg());
+ } else {
+ masm.moveToDouble(from.reg(), to.floatReg());
+ }
+ } else {
+ MOZ_ASSERT(from.isMemory());
+ MOZ_ASSERT(to.isMemory());
+ ScratchDoubleScope fpscratch64(masm);
+ masm.loadDouble(getAdjustedAddress(from), fpscratch64);
+ masm.storeDouble(fpscratch64, getAdjustedAddress(to));
+ }
+}
+
+void MoveEmitterLOONG64::emit(const MoveOp& move) {
+ const MoveOperand& from = move.from();
+ const MoveOperand& to = move.to();
+
+ if (move.isCycleEnd() && move.isCycleBegin()) {
+ // A fun consequence of aliased registers is you can have multiple
+ // cycles at once, and one can end exactly where another begins.
+ breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot());
+ completeCycle(from, to, move.type(), move.cycleEndSlot());
+ return;
+ }
+
+ if (move.isCycleEnd()) {
+ MOZ_ASSERT(inCycle_);
+ completeCycle(from, to, move.type(), move.cycleEndSlot());
+ MOZ_ASSERT(inCycle_ > 0);
+ inCycle_--;
+ return;
+ }
+
+ if (move.isCycleBegin()) {
+ breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot());
+ inCycle_++;
+ }
+
+ switch (move.type()) {
+ case MoveOp::FLOAT32:
+ emitFloat32Move(from, to);
+ break;
+ case MoveOp::DOUBLE:
+ emitDoubleMove(from, to);
+ break;
+ case MoveOp::INT32:
+ emitInt32Move(from, to);
+ break;
+ case MoveOp::GENERAL:
+ emitMove(from, to);
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterLOONG64::assertDone() { MOZ_ASSERT(inCycle_ == 0); }
+
+void MoveEmitterLOONG64::finish() {
+ assertDone();
+
+ masm.freeStack(masm.framePushed() - pushedAtStart_);
+}
diff --git a/js/src/jit/loong64/MoveEmitter-loong64.h b/js/src/jit/loong64/MoveEmitter-loong64.h
new file mode 100644
index 0000000000..1481c8f973
--- /dev/null
+++ b/js/src/jit/loong64/MoveEmitter-loong64.h
@@ -0,0 +1,76 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_MoveEmitter_loong64_h
+#define jit_loong64_MoveEmitter_loong64_h
+
+#include "jit/MacroAssembler.h"
+#include "jit/MoveResolver.h"
+
+namespace js {
+namespace jit {
+
+class MoveEmitterLOONG64 {
+ void emitDoubleMove(const MoveOperand& from, const MoveOperand& to);
+ void breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+ void completeCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+
+ protected:
+ uint32_t inCycle_;
+ MacroAssembler& masm;
+
+ // Original stack push value.
+ uint32_t pushedAtStart_;
+
+ // These store stack offsets to spill locations, snapshotting
+ // codegen->framePushed_ at the time they were allocated. They are -1 if no
+ // stack space has been allocated for that particular spill.
+ int32_t pushedAtCycle_;
+ int32_t pushedAtSpill_;
+
+ // These are registers that are available for temporary use. They may be
+ // assigned InvalidReg. If no corresponding spill space has been assigned,
+ // then these registers do not need to be spilled.
+ Register spilledReg_;
+ FloatRegister spilledFloatReg_;
+
+ void assertDone();
+ Register tempReg();
+ FloatRegister tempFloatReg();
+ Address cycleSlot(uint32_t slot, uint32_t subslot = 0) const;
+ int32_t getAdjustedOffset(const MoveOperand& operand);
+ Address getAdjustedAddress(const MoveOperand& operand);
+
+ void emitMove(const MoveOperand& from, const MoveOperand& to);
+ void emitInt32Move(const MoveOperand& from, const MoveOperand& to);
+ void emitFloat32Move(const MoveOperand& from, const MoveOperand& to);
+ void emit(const MoveOp& move);
+
+ public:
+ MoveEmitterLOONG64(MacroAssembler& masm)
+ : inCycle_(0),
+ masm(masm),
+ pushedAtStart_(masm.framePushed()),
+ pushedAtCycle_(-1),
+ pushedAtSpill_(-1),
+ spilledReg_(InvalidReg),
+ spilledFloatReg_(InvalidFloatReg) {}
+
+ ~MoveEmitterLOONG64() { assertDone(); }
+
+ void emit(const MoveResolver& moves);
+ void finish();
+ void setScratchRegister(Register reg) {}
+};
+
+typedef MoveEmitterLOONG64 MoveEmitter;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_MoveEmitter_loong64_h */
diff --git a/js/src/jit/loong64/SharedICHelpers-loong64-inl.h b/js/src/jit/loong64/SharedICHelpers-loong64-inl.h
new file mode 100644
index 0000000000..960a583ccf
--- /dev/null
+++ b/js/src/jit/loong64/SharedICHelpers-loong64-inl.h
@@ -0,0 +1,83 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_SharedICHelpers_loong64_inl_h
+#define jit_loong64_SharedICHelpers_loong64_inl_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+ uint32_t argSize) {
+#ifdef DEBUG
+ Register scratch = R2.scratchReg();
+
+ // Compute frame size.
+ masm.movePtr(FramePointer, scratch);
+ masm.subPtr(StackPointer, scratch);
+
+ // Store frame size without VMFunction arguments for debug assertions.
+ masm.subPtr(Imm32(argSize), scratch);
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+ masm.addPtr(Imm32(argSize), scratch);
+#endif
+
+ // Push frame descriptor and perform the tail call.
+ // ICTailCallReg (ra) already contains the return address (as we
+ // keep it there through the stub calls), but the VMWrapper code being
+ // called expects the return address to also be pushed on the stack.
+ MOZ_ASSERT(ICTailCallReg == ra);
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(ra);
+
+ masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) {
+ MOZ_ASSERT(scratch != ICTailCallReg);
+
+#ifdef DEBUG
+ // Compute frame size.
+ masm.movePtr(FramePointer, scratch);
+ masm.subPtr(StackPointer, scratch);
+
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Note: when making changes here, don't forget to update
+ // BaselineStubFrame if needed.
+
+ // Push frame descriptor and return address.
+ masm.PushFrameDescriptor(FrameType::BaselineJS);
+ masm.Push(ICTailCallReg);
+
+ // Save old frame pointer, stack pointer and stub reg.
+ masm.Push(FramePointer);
+ masm.movePtr(StackPointer, FramePointer);
+ masm.Push(ICStubReg);
+
+ // Stack should remain aligned.
+ masm.assertStackAlignment(sizeof(Value), 0);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_SharedICHelpers_loong64_inl_h */
diff --git a/js/src/jit/loong64/SharedICHelpers-loong64.h b/js/src/jit/loong64/SharedICHelpers-loong64.h
new file mode 100644
index 0000000000..e8934e2f82
--- /dev/null
+++ b/js/src/jit/loong64/SharedICHelpers-loong64.h
@@ -0,0 +1,91 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_SharedICHelpers_loong64_h
+#define jit_loong64_SharedICHelpers_loong64_h
+
+#include "jit/BaselineIC.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from sp to the top Value inside an IC stub (no return address on
+// the stack on LoongArch).
+static const size_t ICStackValueOffset = 0;
+
+struct BaselineStubFrame {
+ uintptr_t savedFrame;
+ uintptr_t savedStub;
+ uintptr_t returnAddress;
+ uintptr_t descriptor;
+};
+
+inline void EmitRestoreTailCallReg(MacroAssembler& masm) {
+ // No-op on LA because ra register is always holding the return address.
+}
+
+inline void EmitRepushTailCallReg(MacroAssembler& masm) {
+ // No-op on LA because ra register is always holding the return address.
+}
+
+inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
+ // The stub pointer must already be in ICStubReg.
+ // Load stubcode pointer from the ICStub.
+ // R2 won't be active when we call ICs, so we can use it as scratch.
+ masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg());
+
+ // Call the stubcode via a direct jump-and-link
+ masm.call(R2.scratchReg());
+ *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void EmitReturnFromIC(MacroAssembler& masm) { masm.branch(ra); }
+
+inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
+ masm.loadPtr(
+ Address(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP),
+ ICStubReg);
+
+ masm.movePtr(FramePointer, StackPointer);
+ masm.Pop(FramePointer);
+
+ // Load the return address.
+ masm.Pop(ICTailCallReg);
+
+ // Discard the frame descriptor.
+ {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.Pop(scratch2);
+ }
+
+ masm.checkStackAlignment();
+}
+
+template <typename AddrType>
+inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr,
+ MIRType type) {
+ // On LoongArch, $ra is clobbered by guardedCallPreBarrier. Save it first.
+ masm.push(ra);
+ masm.guardedCallPreBarrier(addr, type);
+ masm.pop(ra);
+}
+
+inline void EmitStubGuardFailure(MacroAssembler& masm) {
+ // Load next stub into ICStubReg
+ masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+ // Return address is already loaded, just jump to the next stubcode.
+ MOZ_ASSERT(ICTailCallReg == ra);
+ masm.jump(Address(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_SharedICHelpers_loong64_h */
diff --git a/js/src/jit/loong64/SharedICRegisters-loong64.h b/js/src/jit/loong64/SharedICRegisters-loong64.h
new file mode 100644
index 0000000000..d51336c1d2
--- /dev/null
+++ b/js/src/jit/loong64/SharedICRegisters-loong64.h
@@ -0,0 +1,42 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_loong64_SharedICRegisters_loong64_h
+#define jit_loong64_SharedICRegisters_loong64_h
+
+#include "jit/loong64/Assembler-loong64.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+// ValueOperands R0, R1, and R2.
+// R0 == JSReturnReg, and R2 uses registers not preserved across calls. R1 value
+// should be preserved across calls.
+static constexpr ValueOperand R0(a2);
+static constexpr ValueOperand R1(s1);
+static constexpr ValueOperand R2(a0);
+
+// ICTailCallReg and ICStubReg
+// These use registers that are not preserved across calls.
+static constexpr Register ICTailCallReg = ra;
+static constexpr Register ICStubReg = t0;
+
+// Note that ICTailCallReg is actually just the link register.
+// In LoongArch code emission, we do not clobber ICTailCallReg since we keep
+// the return address for calls there.
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = f0;
+static constexpr FloatRegister FloatReg1 = f1;
+static constexpr FloatRegister FloatReg2 = f2;
+static constexpr FloatRegister FloatReg3 = f3;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_loong64_SharedICRegisters_loong64_h */
diff --git a/js/src/jit/loong64/Simulator-loong64.cpp b/js/src/jit/loong64/Simulator-loong64.cpp
new file mode 100644
index 0000000000..da1cea9aaa
--- /dev/null
+++ b/js/src/jit/loong64/Simulator-loong64.cpp
@@ -0,0 +1,5220 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80: */
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/loong64/Simulator-loong64.h"
+
+#include <float.h>
+#include <limits>
+
+#include "jit/AtomicOperations.h"
+#include "jit/loong64/Assembler-loong64.h"
+#include "js/Conversions.h"
+#include "threading/LockGuard.h"
+#include "vm/JSContext.h"
+#include "vm/Runtime.h"
+#include "wasm/WasmInstance.h"
+#include "wasm/WasmSignalHandlers.h"
+
+#define I8(v) static_cast<int8_t>(v)
+#define I16(v) static_cast<int16_t>(v)
+#define U16(v) static_cast<uint16_t>(v)
+#define I32(v) static_cast<int32_t>(v)
+#define U32(v) static_cast<uint32_t>(v)
+#define I64(v) static_cast<int64_t>(v)
+#define U64(v) static_cast<uint64_t>(v)
+#define I128(v) static_cast<__int128_t>(v)
+#define U128(v) static_cast<__uint128_t>(v)
+
+#define I32_CHECK(v) \
+ ({ \
+ MOZ_ASSERT(I64(I32(v)) == I64(v)); \
+ I32((v)); \
+ })
+
+namespace js {
+namespace jit {
+
+static int64_t MultiplyHighSigned(int64_t u, int64_t v) {
+ uint64_t u0, v0, w0;
+ int64_t u1, v1, w1, w2, t;
+
+ u0 = u & 0xFFFFFFFFL;
+ u1 = u >> 32;
+ v0 = v & 0xFFFFFFFFL;
+ v1 = v >> 32;
+
+ w0 = u0 * v0;
+ t = u1 * v0 + (w0 >> 32);
+ w1 = t & 0xFFFFFFFFL;
+ w2 = t >> 32;
+ w1 = u0 * v1 + w1;
+
+ return u1 * v1 + w2 + (w1 >> 32);
+}
+
+static uint64_t MultiplyHighUnsigned(uint64_t u, uint64_t v) {
+ uint64_t u0, v0, w0;
+ uint64_t u1, v1, w1, w2, t;
+
+ u0 = u & 0xFFFFFFFFL;
+ u1 = u >> 32;
+ v0 = v & 0xFFFFFFFFL;
+ v1 = v >> 32;
+
+ w0 = u0 * v0;
+ t = u1 * v0 + (w0 >> 32);
+ w1 = t & 0xFFFFFFFFL;
+ w2 = t >> 32;
+ w1 = u0 * v1 + w1;
+
+ return u1 * v1 + w2 + (w1 >> 32);
+}
+
+// Precondition: 0 <= shift < 32
+inline constexpr uint32_t RotateRight32(uint32_t value, uint32_t shift) {
+ return (value >> shift) | (value << ((32 - shift) & 31));
+}
+
+// Precondition: 0 <= shift < 32
+inline constexpr uint32_t RotateLeft32(uint32_t value, uint32_t shift) {
+ return (value << shift) | (value >> ((32 - shift) & 31));
+}
+
+// Precondition: 0 <= shift < 64
+inline constexpr uint64_t RotateRight64(uint64_t value, uint64_t shift) {
+ return (value >> shift) | (value << ((64 - shift) & 63));
+}
+
+// Precondition: 0 <= shift < 64
+inline constexpr uint64_t RotateLeft64(uint64_t value, uint64_t shift) {
+ return (value << shift) | (value >> ((64 - shift) & 63));
+}
+
+// break instr with MAX_BREAK_CODE.
+static const Instr kCallRedirInstr = op_break | CODEMask;
+
+// -----------------------------------------------------------------------------
+// LoongArch64 assembly various constants.
+
+class SimInstruction {
+ public:
+ enum {
+ kInstrSize = 4,
+ // On LoongArch, PC cannot actually be directly accessed. We behave as if PC
+ // was always the value of the current instruction being executed.
+ kPCReadOffset = 0
+ };
+
+ // Get the raw instruction bits.
+ inline Instr instructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void setInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int bit(int nr) const { return (instructionBits() >> nr) & 1; }
+
+ // Read a bit field out of the instruction bits.
+ inline int bits(int hi, int lo) const {
+ return (instructionBits() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Instruction type.
+ enum Type {
+ kUnsupported = -1,
+ kOp6Type,
+ kOp7Type,
+ kOp8Type,
+ kOp10Type,
+ kOp11Type,
+ kOp12Type,
+ kOp14Type,
+ kOp15Type,
+ kOp16Type,
+ kOp17Type,
+ kOp22Type,
+ kOp24Type
+ };
+
+ // Get the encoding type of the instruction.
+ Type instructionType() const;
+
+ inline int rjValue() const { return bits(RJShift + RJBits - 1, RJShift); }
+
+ inline int rkValue() const { return bits(RKShift + RKBits - 1, RKShift); }
+
+ inline int rdValue() const { return bits(RDShift + RDBits - 1, RDShift); }
+
+ inline int sa2Value() const { return bits(SAShift + SA2Bits - 1, SAShift); }
+
+ inline int sa3Value() const { return bits(SAShift + SA3Bits - 1, SAShift); }
+
+ inline int lsbwValue() const {
+ return bits(LSBWShift + LSBWBits - 1, LSBWShift);
+ }
+
+ inline int msbwValue() const {
+ return bits(MSBWShift + MSBWBits - 1, MSBWShift);
+ }
+
+ inline int lsbdValue() const {
+ return bits(LSBDShift + LSBDBits - 1, LSBDShift);
+ }
+
+ inline int msbdValue() const {
+ return bits(MSBDShift + MSBDBits - 1, MSBDShift);
+ }
+
+ inline int fdValue() const { return bits(FDShift + FDBits - 1, FDShift); }
+
+ inline int fjValue() const { return bits(FJShift + FJBits - 1, FJShift); }
+
+ inline int fkValue() const { return bits(FKShift + FKBits - 1, FKShift); }
+
+ inline int faValue() const { return bits(FAShift + FABits - 1, FAShift); }
+
+ inline int cdValue() const { return bits(CDShift + CDBits - 1, CDShift); }
+
+ inline int cjValue() const { return bits(CJShift + CJBits - 1, CJShift); }
+
+ inline int caValue() const { return bits(CAShift + CABits - 1, CAShift); }
+
+ inline int condValue() const {
+ return bits(CONDShift + CONDBits - 1, CONDShift);
+ }
+
+ inline int imm5Value() const {
+ return bits(Imm5Shift + Imm5Bits - 1, Imm5Shift);
+ }
+
+ inline int imm6Value() const {
+ return bits(Imm6Shift + Imm6Bits - 1, Imm6Shift);
+ }
+
+ inline int imm12Value() const {
+ return bits(Imm12Shift + Imm12Bits - 1, Imm12Shift);
+ }
+
+ inline int imm14Value() const {
+ return bits(Imm14Shift + Imm14Bits - 1, Imm14Shift);
+ }
+
+ inline int imm16Value() const {
+ return bits(Imm16Shift + Imm16Bits - 1, Imm16Shift);
+ }
+
+ inline int imm20Value() const {
+ return bits(Imm20Shift + Imm20Bits - 1, Imm20Shift);
+ }
+
+ inline int32_t imm26Value() const {
+ return bits(Imm26Shift + Imm26Bits - 1, Imm26Shift);
+ }
+
+ // Say if the instruction is a debugger break/trap.
+ bool isTrap() const;
+
+ private:
+ SimInstruction() = delete;
+ SimInstruction(const SimInstruction& other) = delete;
+ void operator=(const SimInstruction& other) = delete;
+};
+
+bool SimInstruction::isTrap() const {
+ // is break??
+ switch (bits(31, 15) << 15) {
+ case op_break:
+ return (instructionBits() != kCallRedirInstr) && (bits(15, 0) != 6);
+ default:
+ return false;
+ };
+}
+
+SimInstruction::Type SimInstruction::instructionType() const {
+ SimInstruction::Type kType = kUnsupported;
+
+ // Check for kOp6Type
+ switch (bits(31, 26) << 26) {
+ case op_beqz:
+ case op_bnez:
+ case op_bcz:
+ case op_jirl:
+ case op_b:
+ case op_bl:
+ case op_beq:
+ case op_bne:
+ case op_blt:
+ case op_bge:
+ case op_bltu:
+ case op_bgeu:
+ case op_addu16i_d:
+ kType = kOp6Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp7Type
+ switch (bits(31, 25) << 25) {
+ case op_lu12i_w:
+ case op_lu32i_d:
+ case op_pcaddi:
+ case op_pcalau12i:
+ case op_pcaddu12i:
+ case op_pcaddu18i:
+ kType = kOp7Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp8Type
+ switch (bits(31, 24) << 24) {
+ case op_ll_w:
+ case op_sc_w:
+ case op_ll_d:
+ case op_sc_d:
+ case op_ldptr_w:
+ case op_stptr_w:
+ case op_ldptr_d:
+ case op_stptr_d:
+ kType = kOp8Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp10Type
+ switch (bits(31, 22) << 22) {
+ case op_bstrins_d:
+ case op_bstrpick_d:
+ case op_slti:
+ case op_sltui:
+ case op_addi_w:
+ case op_addi_d:
+ case op_lu52i_d:
+ case op_andi:
+ case op_ori:
+ case op_xori:
+ case op_ld_b:
+ case op_ld_h:
+ case op_ld_w:
+ case op_ld_d:
+ case op_st_b:
+ case op_st_h:
+ case op_st_w:
+ case op_st_d:
+ case op_ld_bu:
+ case op_ld_hu:
+ case op_ld_wu:
+ case op_preld:
+ case op_fld_s:
+ case op_fst_s:
+ case op_fld_d:
+ case op_fst_d:
+ case op_bstr_w: // BSTRINS_W & BSTRPICK_W
+ kType = kOp10Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp11Type
+ switch (bits(31, 21) << 21) {
+ case op_bstr_w:
+ kType = kOp11Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp12Type
+ switch (bits(31, 20) << 20) {
+ case op_fmadd_s:
+ case op_fmadd_d:
+ case op_fmsub_s:
+ case op_fmsub_d:
+ case op_fnmadd_s:
+ case op_fnmadd_d:
+ case op_fnmsub_s:
+ case op_fnmsub_d:
+ case op_fcmp_cond_s:
+ case op_fcmp_cond_d:
+ kType = kOp12Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp14Type
+ switch (bits(31, 18) << 18) {
+ case op_bytepick_d:
+ case op_fsel:
+ kType = kOp14Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp15Type
+ switch (bits(31, 17) << 17) {
+ case op_bytepick_w:
+ case op_alsl_w:
+ case op_alsl_wu:
+ case op_alsl_d:
+ kType = kOp15Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp16Type
+ switch (bits(31, 16) << 16) {
+ case op_slli_d:
+ case op_srli_d:
+ case op_srai_d:
+ case op_rotri_d:
+ kType = kOp16Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp17Type
+ switch (bits(31, 15) << 15) {
+ case op_slli_w:
+ case op_srli_w:
+ case op_srai_w:
+ case op_rotri_w:
+ case op_add_w:
+ case op_add_d:
+ case op_sub_w:
+ case op_sub_d:
+ case op_slt:
+ case op_sltu:
+ case op_maskeqz:
+ case op_masknez:
+ case op_nor:
+ case op_and:
+ case op_or:
+ case op_xor:
+ case op_orn:
+ case op_andn:
+ case op_sll_w:
+ case op_srl_w:
+ case op_sra_w:
+ case op_sll_d:
+ case op_srl_d:
+ case op_sra_d:
+ case op_rotr_w:
+ case op_rotr_d:
+ case op_mul_w:
+ case op_mul_d:
+ case op_mulh_d:
+ case op_mulh_du:
+ case op_mulh_w:
+ case op_mulh_wu:
+ case op_mulw_d_w:
+ case op_mulw_d_wu:
+ case op_div_w:
+ case op_mod_w:
+ case op_div_wu:
+ case op_mod_wu:
+ case op_div_d:
+ case op_mod_d:
+ case op_div_du:
+ case op_mod_du:
+ case op_break:
+ case op_fadd_s:
+ case op_fadd_d:
+ case op_fsub_s:
+ case op_fsub_d:
+ case op_fmul_s:
+ case op_fmul_d:
+ case op_fdiv_s:
+ case op_fdiv_d:
+ case op_fmax_s:
+ case op_fmax_d:
+ case op_fmin_s:
+ case op_fmin_d:
+ case op_fmaxa_s:
+ case op_fmaxa_d:
+ case op_fmina_s:
+ case op_fmina_d:
+ case op_fcopysign_s:
+ case op_fcopysign_d:
+ case op_ldx_b:
+ case op_ldx_h:
+ case op_ldx_w:
+ case op_ldx_d:
+ case op_stx_b:
+ case op_stx_h:
+ case op_stx_w:
+ case op_stx_d:
+ case op_ldx_bu:
+ case op_ldx_hu:
+ case op_ldx_wu:
+ case op_fldx_s:
+ case op_fldx_d:
+ case op_fstx_s:
+ case op_fstx_d:
+ case op_amswap_w:
+ case op_amswap_d:
+ case op_amadd_w:
+ case op_amadd_d:
+ case op_amand_w:
+ case op_amand_d:
+ case op_amor_w:
+ case op_amor_d:
+ case op_amxor_w:
+ case op_amxor_d:
+ case op_ammax_w:
+ case op_ammax_d:
+ case op_ammin_w:
+ case op_ammin_d:
+ case op_ammax_wu:
+ case op_ammax_du:
+ case op_ammin_wu:
+ case op_ammin_du:
+ case op_amswap_db_w:
+ case op_amswap_db_d:
+ case op_amadd_db_w:
+ case op_amadd_db_d:
+ case op_amand_db_w:
+ case op_amand_db_d:
+ case op_amor_db_w:
+ case op_amor_db_d:
+ case op_amxor_db_w:
+ case op_amxor_db_d:
+ case op_ammax_db_w:
+ case op_ammax_db_d:
+ case op_ammin_db_w:
+ case op_ammin_db_d:
+ case op_ammax_db_wu:
+ case op_ammax_db_du:
+ case op_ammin_db_wu:
+ case op_ammin_db_du:
+ case op_dbar:
+ case op_ibar:
+ kType = kOp17Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp22Type
+ switch (bits(31, 10) << 10) {
+ case op_clo_w:
+ case op_clz_w:
+ case op_cto_w:
+ case op_ctz_w:
+ case op_clo_d:
+ case op_clz_d:
+ case op_cto_d:
+ case op_ctz_d:
+ case op_revb_2h:
+ case op_revb_4h:
+ case op_revb_2w:
+ case op_revb_d:
+ case op_revh_2w:
+ case op_revh_d:
+ case op_bitrev_4b:
+ case op_bitrev_8b:
+ case op_bitrev_w:
+ case op_bitrev_d:
+ case op_ext_w_h:
+ case op_ext_w_b:
+ case op_fabs_s:
+ case op_fabs_d:
+ case op_fneg_s:
+ case op_fneg_d:
+ case op_fsqrt_s:
+ case op_fsqrt_d:
+ case op_fmov_s:
+ case op_fmov_d:
+ case op_movgr2fr_w:
+ case op_movgr2fr_d:
+ case op_movgr2frh_w:
+ case op_movfr2gr_s:
+ case op_movfr2gr_d:
+ case op_movfrh2gr_s:
+ case op_movfcsr2gr:
+ case op_movfr2cf:
+ case op_movgr2cf:
+ case op_fcvt_s_d:
+ case op_fcvt_d_s:
+ case op_ftintrm_w_s:
+ case op_ftintrm_w_d:
+ case op_ftintrm_l_s:
+ case op_ftintrm_l_d:
+ case op_ftintrp_w_s:
+ case op_ftintrp_w_d:
+ case op_ftintrp_l_s:
+ case op_ftintrp_l_d:
+ case op_ftintrz_w_s:
+ case op_ftintrz_w_d:
+ case op_ftintrz_l_s:
+ case op_ftintrz_l_d:
+ case op_ftintrne_w_s:
+ case op_ftintrne_w_d:
+ case op_ftintrne_l_s:
+ case op_ftintrne_l_d:
+ case op_ftint_w_s:
+ case op_ftint_w_d:
+ case op_ftint_l_s:
+ case op_ftint_l_d:
+ case op_ffint_s_w:
+ case op_ffint_s_l:
+ case op_ffint_d_w:
+ case op_ffint_d_l:
+ case op_frint_s:
+ case op_frint_d:
+ kType = kOp22Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ if (kType == kUnsupported) {
+ // Check for kOp24Type
+ switch (bits(31, 8) << 8) {
+ case op_movcf2fr:
+ case op_movcf2gr:
+ kType = kOp24Type;
+ break;
+ default:
+ kType = kUnsupported;
+ }
+ }
+
+ return kType;
+}
+
+// C/C++ argument slots size.
+const int kCArgSlotCount = 0;
+const int kCArgsSlotsSize = kCArgSlotCount * sizeof(uintptr_t);
+
+class CachePage {
+ public:
+ static const int LINE_VALID = 0;
+ static const int LINE_INVALID = 1;
+
+ static const int kPageShift = 12;
+ static const int kPageSize = 1 << kPageShift;
+ static const int kPageMask = kPageSize - 1;
+ static const int kLineShift = 2; // The cache line is only 4 bytes right now.
+ static const int kLineLength = 1 << kLineShift;
+ static const int kLineMask = kLineLength - 1;
+
+ CachePage() { memset(&validity_map_, LINE_INVALID, sizeof(validity_map_)); }
+
+ char* validityByte(int offset) {
+ return &validity_map_[offset >> kLineShift];
+ }
+
+ char* cachedData(int offset) { return &data_[offset]; }
+
+ private:
+ char data_[kPageSize]; // The cached data.
+ static const int kValidityMapSize = kPageSize >> kLineShift;
+ char validity_map_[kValidityMapSize]; // One byte per line.
+};
+
+// Protects the icache() and redirection() properties of the
+// Simulator.
+class AutoLockSimulatorCache : public LockGuard<Mutex> {
+ using Base = LockGuard<Mutex>;
+
+ public:
+ explicit AutoLockSimulatorCache()
+ : Base(SimulatorProcess::singleton_->cacheLock_) {}
+};
+
+mozilla::Atomic<size_t, mozilla::ReleaseAcquire>
+ SimulatorProcess::ICacheCheckingDisableCount(
+ 1); // Checking is disabled by default.
+SimulatorProcess* SimulatorProcess::singleton_ = nullptr;
+
+int64_t Simulator::StopSimAt = -1;
+
+Simulator* Simulator::Create() {
+ auto sim = MakeUnique<Simulator>();
+ if (!sim) {
+ return nullptr;
+ }
+
+ if (!sim->init()) {
+ return nullptr;
+ }
+
+ int64_t stopAt;
+ char* stopAtStr = getenv("LOONG64_SIM_STOP_AT");
+ if (stopAtStr && sscanf(stopAtStr, "%" PRIi64, &stopAt) == 1) {
+ fprintf(stderr, "\nStopping simulation at icount %" PRIi64 "\n", stopAt);
+ Simulator::StopSimAt = stopAt;
+ }
+
+ return sim.release();
+}
+
+void Simulator::Destroy(Simulator* sim) { js_delete(sim); }
+
+// The loong64Debugger class is used by the simulator while debugging simulated
+// code.
+class loong64Debugger {
+ public:
+ explicit loong64Debugger(Simulator* sim) : sim_(sim) {}
+
+ void stop(SimInstruction* instr);
+ void debug();
+ // Print all registers with a nice formatting.
+ void printAllRegs();
+ void printAllRegsIncludingFPU();
+
+ private:
+ // We set the breakpoint code to 0x7fff to easily recognize it.
+ static const Instr kBreakpointInstr = op_break | (0x7fff & CODEMask);
+ static const Instr kNopInstr = 0x0;
+
+ Simulator* sim_;
+
+ int64_t getRegisterValue(int regnum);
+ int64_t getFPURegisterValueLong(int regnum);
+ float getFPURegisterValueFloat(int regnum);
+ double getFPURegisterValueDouble(int regnum);
+ bool getValue(const char* desc, int64_t* value);
+
+ // Set or delete a breakpoint. Returns true if successful.
+ bool setBreakpoint(SimInstruction* breakpc);
+ bool deleteBreakpoint(SimInstruction* breakpc);
+
+ // Undo and redo all breakpoints. This is needed to bracket disassembly and
+ // execution to skip past breakpoints when run from the debugger.
+ void undoBreakpoints();
+ void redoBreakpoints();
+};
+
+static void UNIMPLEMENTED() {
+ printf("UNIMPLEMENTED instruction.\n");
+ MOZ_CRASH();
+}
+static void UNREACHABLE() {
+ printf("UNREACHABLE instruction.\n");
+ MOZ_CRASH();
+}
+static void UNSUPPORTED() {
+ printf("Unsupported instruction.\n");
+ MOZ_CRASH();
+}
+
+void loong64Debugger::stop(SimInstruction* instr) {
+ // Get the stop code.
+ uint32_t code = instr->bits(25, 6);
+ // Retrieve the encoded address, which comes just after this stop.
+ char* msg =
+ *reinterpret_cast<char**>(sim_->get_pc() + SimInstruction::kInstrSize);
+ // Update this stop description.
+ if (!sim_->watchedStops_[code].desc_) {
+ sim_->watchedStops_[code].desc_ = msg;
+ }
+ // Print the stop message and code if it is not the default code.
+ if (code != kMaxStopCode) {
+ printf("Simulator hit stop %u: %s\n", code, msg);
+ } else {
+ printf("Simulator hit %s\n", msg);
+ }
+ sim_->set_pc(sim_->get_pc() + 2 * SimInstruction::kInstrSize);
+ debug();
+}
+
+int64_t loong64Debugger::getRegisterValue(int regnum) {
+ if (regnum == kPCRegister) {
+ return sim_->get_pc();
+ }
+ return sim_->getRegister(regnum);
+}
+
+int64_t loong64Debugger::getFPURegisterValueLong(int regnum) {
+ return sim_->getFpuRegister(regnum);
+}
+
+float loong64Debugger::getFPURegisterValueFloat(int regnum) {
+ return sim_->getFpuRegisterFloat(regnum);
+}
+
+double loong64Debugger::getFPURegisterValueDouble(int regnum) {
+ return sim_->getFpuRegisterDouble(regnum);
+}
+
+bool loong64Debugger::getValue(const char* desc, int64_t* value) {
+ Register reg = Register::FromName(desc);
+ if (reg != InvalidReg) {
+ *value = getRegisterValue(reg.code());
+ return true;
+ }
+
+ if (strncmp(desc, "0x", 2) == 0) {
+ return sscanf(desc + 2, "%lx", reinterpret_cast<uint64_t*>(value)) == 1;
+ }
+ return sscanf(desc, "%lu", reinterpret_cast<uint64_t*>(value)) == 1;
+}
+
+bool loong64Debugger::setBreakpoint(SimInstruction* breakpc) {
+ // Check if a breakpoint can be set. If not return without any side-effects.
+ if (sim_->break_pc_ != nullptr) {
+ return false;
+ }
+
+ // Set the breakpoint.
+ sim_->break_pc_ = breakpc;
+ sim_->break_instr_ = breakpc->instructionBits();
+ // Not setting the breakpoint instruction in the code itself. It will be set
+ // when the debugger shell continues.
+ return true;
+}
+
+bool loong64Debugger::deleteBreakpoint(SimInstruction* breakpc) {
+ if (sim_->break_pc_ != nullptr) {
+ sim_->break_pc_->setInstructionBits(sim_->break_instr_);
+ }
+
+ sim_->break_pc_ = nullptr;
+ sim_->break_instr_ = 0;
+ return true;
+}
+
+void loong64Debugger::undoBreakpoints() {
+ if (sim_->break_pc_) {
+ sim_->break_pc_->setInstructionBits(sim_->break_instr_);
+ }
+}
+
+void loong64Debugger::redoBreakpoints() {
+ if (sim_->break_pc_) {
+ sim_->break_pc_->setInstructionBits(kBreakpointInstr);
+ }
+}
+
+void loong64Debugger::printAllRegs() {
+ int64_t value;
+ for (uint32_t i = 0; i < Registers::Total; i++) {
+ value = getRegisterValue(i);
+ printf("%3s: 0x%016" PRIx64 " %20" PRIi64 " ", Registers::GetName(i),
+ value, value);
+
+ if (i % 2) {
+ printf("\n");
+ }
+ }
+ printf("\n");
+
+ value = getRegisterValue(Simulator::pc);
+ printf(" pc: 0x%016" PRIx64 "\n", value);
+}
+
+void loong64Debugger::printAllRegsIncludingFPU() {
+ printAllRegs();
+
+ printf("\n\n");
+ // f0, f1, f2, ... f31.
+ for (uint32_t i = 0; i < FloatRegisters::TotalPhys; i++) {
+ printf("%3s: 0x%016" PRIi64 "\tflt: %-8.4g\tdbl: %-16.4g\n",
+ FloatRegisters::GetName(i), getFPURegisterValueLong(i),
+ getFPURegisterValueFloat(i), getFPURegisterValueDouble(i));
+ }
+}
+
+static char* ReadLine(const char* prompt) {
+ UniqueChars result;
+ char lineBuf[256];
+ int offset = 0;
+ bool keepGoing = true;
+ fprintf(stdout, "%s", prompt);
+ fflush(stdout);
+ while (keepGoing) {
+ if (fgets(lineBuf, sizeof(lineBuf), stdin) == nullptr) {
+ // fgets got an error. Just give up.
+ return nullptr;
+ }
+ int len = strlen(lineBuf);
+ if (len > 0 && lineBuf[len - 1] == '\n') {
+ // Since we read a new line we are done reading the line. This
+ // will exit the loop after copying this buffer into the result.
+ keepGoing = false;
+ }
+ if (!result) {
+ // Allocate the initial result and make room for the terminating '\0'
+ result.reset(js_pod_malloc<char>(len + 1));
+ if (!result) {
+ return nullptr;
+ }
+ } else {
+ // Allocate a new result with enough room for the new addition.
+ int new_len = offset + len + 1;
+ char* new_result = js_pod_malloc<char>(new_len);
+ if (!new_result) {
+ return nullptr;
+ }
+ // Copy the existing input into the new array and set the new
+ // array as the result.
+ memcpy(new_result, result.get(), offset * sizeof(char));
+ result.reset(new_result);
+ }
+ // Copy the newly read line into the result.
+ memcpy(result.get() + offset, lineBuf, len * sizeof(char));
+ offset += len;
+ }
+
+ MOZ_ASSERT(result);
+ result[offset] = '\0';
+ return result.release();
+}
+
+static void DisassembleInstruction(uint64_t pc) {
+ printf("Not supported on loongarch64 yet\n");
+}
+
+void loong64Debugger::debug() {
+ intptr_t lastPC = -1;
+ bool done = false;
+
+#define COMMAND_SIZE 63
+#define ARG_SIZE 255
+
+#define STR(a) #a
+#define XSTR(a) STR(a)
+
+ char cmd[COMMAND_SIZE + 1];
+ char arg1[ARG_SIZE + 1];
+ char arg2[ARG_SIZE + 1];
+ char* argv[3] = {cmd, arg1, arg2};
+
+ // Make sure to have a proper terminating character if reaching the limit.
+ cmd[COMMAND_SIZE] = 0;
+ arg1[ARG_SIZE] = 0;
+ arg2[ARG_SIZE] = 0;
+
+ // Undo all set breakpoints while running in the debugger shell. This will
+ // make them invisible to all commands.
+ undoBreakpoints();
+
+ while (!done && (sim_->get_pc() != Simulator::end_sim_pc)) {
+ if (lastPC != sim_->get_pc()) {
+ DisassembleInstruction(sim_->get_pc());
+ printf(" 0x%016" PRIi64 " \n", sim_->get_pc());
+ lastPC = sim_->get_pc();
+ }
+ char* line = ReadLine("sim> ");
+ if (line == nullptr) {
+ break;
+ } else {
+ char* last_input = sim_->lastDebuggerInput();
+ if (strcmp(line, "\n") == 0 && last_input != nullptr) {
+ line = last_input;
+ } else {
+ // Ownership is transferred to sim_;
+ sim_->setLastDebuggerInput(line);
+ }
+ // Use sscanf to parse the individual parts of the command line. At the
+ // moment no command expects more than two parameters.
+ int argc = sscanf(line,
+ "%" XSTR(COMMAND_SIZE) "s "
+ "%" XSTR(ARG_SIZE) "s "
+ "%" XSTR(ARG_SIZE) "s",
+ cmd, arg1, arg2);
+ if ((strcmp(cmd, "si") == 0) || (strcmp(cmd, "stepi") == 0)) {
+ SimInstruction* instr =
+ reinterpret_cast<SimInstruction*>(sim_->get_pc());
+ if (!instr->isTrap()) {
+ sim_->instructionDecode(
+ reinterpret_cast<SimInstruction*>(sim_->get_pc()));
+ } else {
+ // Allow si to jump over generated breakpoints.
+ printf("/!\\ Jumping over generated breakpoint.\n");
+ sim_->set_pc(sim_->get_pc() + SimInstruction::kInstrSize);
+ }
+ sim_->icount_++;
+ } else if ((strcmp(cmd, "c") == 0) || (strcmp(cmd, "cont") == 0)) {
+ // Execute the one instruction we broke at with breakpoints disabled.
+ sim_->instructionDecode(
+ reinterpret_cast<SimInstruction*>(sim_->get_pc()));
+ sim_->icount_++;
+ // Leave the debugger shell.
+ done = true;
+ } else if ((strcmp(cmd, "p") == 0) || (strcmp(cmd, "print") == 0)) {
+ if (argc == 2) {
+ int64_t value;
+ if (strcmp(arg1, "all") == 0) {
+ printAllRegs();
+ } else if (strcmp(arg1, "allf") == 0) {
+ printAllRegsIncludingFPU();
+ } else {
+ Register reg = Register::FromName(arg1);
+ FloatRegisters::Code fReg = FloatRegisters::FromName(arg1);
+ if (reg != InvalidReg) {
+ value = getRegisterValue(reg.code());
+ printf("%s: 0x%016" PRIi64 " %20" PRIi64 " \n", arg1, value,
+ value);
+ } else if (fReg != FloatRegisters::Invalid) {
+ printf("%3s: 0x%016" PRIi64 "\tflt: %-8.4g\tdbl: %-16.4g\n",
+ FloatRegisters::GetName(fReg),
+ getFPURegisterValueLong(fReg),
+ getFPURegisterValueFloat(fReg),
+ getFPURegisterValueDouble(fReg));
+ } else {
+ printf("%s unrecognized\n", arg1);
+ }
+ }
+ } else {
+ printf("print <register> or print <fpu register> single\n");
+ }
+ } else if (strcmp(cmd, "stack") == 0 || strcmp(cmd, "mem") == 0) {
+ int64_t* cur = nullptr;
+ int64_t* end = nullptr;
+ int next_arg = 1;
+
+ if (strcmp(cmd, "stack") == 0) {
+ cur = reinterpret_cast<int64_t*>(sim_->getRegister(Simulator::sp));
+ } else { // Command "mem".
+ int64_t value;
+ if (!getValue(arg1, &value)) {
+ printf("%s unrecognized\n", arg1);
+ continue;
+ }
+ cur = reinterpret_cast<int64_t*>(value);
+ next_arg++;
+ }
+
+ int64_t words;
+ if (argc == next_arg) {
+ words = 10;
+ } else {
+ if (!getValue(argv[next_arg], &words)) {
+ words = 10;
+ }
+ }
+ end = cur + words;
+
+ while (cur < end) {
+ printf(" %p: 0x%016" PRIx64 " %20" PRIi64, cur, *cur, *cur);
+ printf("\n");
+ cur++;
+ }
+
+ } else if ((strcmp(cmd, "disasm") == 0) || (strcmp(cmd, "dpc") == 0) ||
+ (strcmp(cmd, "di") == 0)) {
+ uint8_t* cur = nullptr;
+ uint8_t* end = nullptr;
+
+ if (argc == 1) {
+ cur = reinterpret_cast<uint8_t*>(sim_->get_pc());
+ end = cur + (10 * SimInstruction::kInstrSize);
+ } else if (argc == 2) {
+ Register reg = Register::FromName(arg1);
+ if (reg != InvalidReg || strncmp(arg1, "0x", 2) == 0) {
+ // The argument is an address or a register name.
+ int64_t value;
+ if (getValue(arg1, &value)) {
+ cur = reinterpret_cast<uint8_t*>(value);
+ // Disassemble 10 instructions at <arg1>.
+ end = cur + (10 * SimInstruction::kInstrSize);
+ }
+ } else {
+ // The argument is the number of instructions.
+ int64_t value;
+ if (getValue(arg1, &value)) {
+ cur = reinterpret_cast<uint8_t*>(sim_->get_pc());
+ // Disassemble <arg1> instructions.
+ end = cur + (value * SimInstruction::kInstrSize);
+ }
+ }
+ } else {
+ int64_t value1;
+ int64_t value2;
+ if (getValue(arg1, &value1) && getValue(arg2, &value2)) {
+ cur = reinterpret_cast<uint8_t*>(value1);
+ end = cur + (value2 * SimInstruction::kInstrSize);
+ }
+ }
+
+ while (cur < end) {
+ DisassembleInstruction(uint64_t(cur));
+ cur += SimInstruction::kInstrSize;
+ }
+ } else if (strcmp(cmd, "gdb") == 0) {
+ printf("relinquishing control to gdb\n");
+ asm("int $3");
+ printf("regaining control from gdb\n");
+ } else if (strcmp(cmd, "break") == 0) {
+ if (argc == 2) {
+ int64_t value;
+ if (getValue(arg1, &value)) {
+ if (!setBreakpoint(reinterpret_cast<SimInstruction*>(value))) {
+ printf("setting breakpoint failed\n");
+ }
+ } else {
+ printf("%s unrecognized\n", arg1);
+ }
+ } else {
+ printf("break <address>\n");
+ }
+ } else if (strcmp(cmd, "del") == 0) {
+ if (!deleteBreakpoint(nullptr)) {
+ printf("deleting breakpoint failed\n");
+ }
+ } else if (strcmp(cmd, "flags") == 0) {
+ printf("No flags on LOONG64 !\n");
+ } else if (strcmp(cmd, "stop") == 0) {
+ int64_t value;
+ intptr_t stop_pc = sim_->get_pc() - 2 * SimInstruction::kInstrSize;
+ SimInstruction* stop_instr = reinterpret_cast<SimInstruction*>(stop_pc);
+ SimInstruction* msg_address = reinterpret_cast<SimInstruction*>(
+ stop_pc + SimInstruction::kInstrSize);
+ if ((argc == 2) && (strcmp(arg1, "unstop") == 0)) {
+ // Remove the current stop.
+ if (sim_->isStopInstruction(stop_instr)) {
+ stop_instr->setInstructionBits(kNopInstr);
+ msg_address->setInstructionBits(kNopInstr);
+ } else {
+ printf("Not at debugger stop.\n");
+ }
+ } else if (argc == 3) {
+ // Print information about all/the specified breakpoint(s).
+ if (strcmp(arg1, "info") == 0) {
+ if (strcmp(arg2, "all") == 0) {
+ printf("Stop information:\n");
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->printStopInfo(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->printStopInfo(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ } else if (strcmp(arg1, "enable") == 0) {
+ // Enable all/the specified breakpoint(s).
+ if (strcmp(arg2, "all") == 0) {
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->enableStop(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->enableStop(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ } else if (strcmp(arg1, "disable") == 0) {
+ // Disable all/the specified breakpoint(s).
+ if (strcmp(arg2, "all") == 0) {
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->disableStop(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->disableStop(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ }
+ } else {
+ printf("Wrong usage. Use help command for more information.\n");
+ }
+ } else if ((strcmp(cmd, "h") == 0) || (strcmp(cmd, "help") == 0)) {
+ printf("cont\n");
+ printf(" continue execution (alias 'c')\n");
+ printf("stepi\n");
+ printf(" step one instruction (alias 'si')\n");
+ printf("print <register>\n");
+ printf(" print register content (alias 'p')\n");
+ printf(" use register name 'all' to print all registers\n");
+ printf("printobject <register>\n");
+ printf(" print an object from a register (alias 'po')\n");
+ printf("stack [<words>]\n");
+ printf(" dump stack content, default dump 10 words)\n");
+ printf("mem <address> [<words>]\n");
+ printf(" dump memory content, default dump 10 words)\n");
+ printf("flags\n");
+ printf(" print flags\n");
+ printf("disasm [<instructions>]\n");
+ printf("disasm [<address/register>]\n");
+ printf("disasm [[<address/register>] <instructions>]\n");
+ printf(" disassemble code, default is 10 instructions\n");
+ printf(" from pc (alias 'di')\n");
+ printf("gdb\n");
+ printf(" enter gdb\n");
+ printf("break <address>\n");
+ printf(" set a break point on the address\n");
+ printf("del\n");
+ printf(" delete the breakpoint\n");
+ printf("stop feature:\n");
+ printf(" Description:\n");
+ printf(" Stops are debug instructions inserted by\n");
+ printf(" the Assembler::stop() function.\n");
+ printf(" When hitting a stop, the Simulator will\n");
+ printf(" stop and and give control to the Debugger.\n");
+ printf(" All stop codes are watched:\n");
+ printf(" - They can be enabled / disabled: the Simulator\n");
+ printf(" will / won't stop when hitting them.\n");
+ printf(" - The Simulator keeps track of how many times they \n");
+ printf(" are met. (See the info command.) Going over a\n");
+ printf(" disabled stop still increases its counter. \n");
+ printf(" Commands:\n");
+ printf(" stop info all/<code> : print infos about number <code>\n");
+ printf(" or all stop(s).\n");
+ printf(" stop enable/disable all/<code> : enables / disables\n");
+ printf(" all or number <code> stop(s)\n");
+ printf(" stop unstop\n");
+ printf(" ignore the stop instruction at the current location\n");
+ printf(" from now on\n");
+ } else {
+ printf("Unknown command: %s\n", cmd);
+ }
+ }
+ }
+
+ // Add all the breakpoints back to stop execution and enter the debugger
+ // shell when hit.
+ redoBreakpoints();
+
+#undef COMMAND_SIZE
+#undef ARG_SIZE
+
+#undef STR
+#undef XSTR
+}
+
+static bool AllOnOnePage(uintptr_t start, int size) {
+ intptr_t start_page = (start & ~CachePage::kPageMask);
+ intptr_t end_page = ((start + size) & ~CachePage::kPageMask);
+ return start_page == end_page;
+}
+
+void Simulator::setLastDebuggerInput(char* input) {
+ js_free(lastDebuggerInput_);
+ lastDebuggerInput_ = input;
+}
+
+static CachePage* GetCachePageLocked(SimulatorProcess::ICacheMap& i_cache,
+ void* page) {
+ SimulatorProcess::ICacheMap::AddPtr p = i_cache.lookupForAdd(page);
+ if (p) {
+ return p->value();
+ }
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ CachePage* new_page = js_new<CachePage>();
+ if (!new_page || !i_cache.add(p, page, new_page)) {
+ oomUnsafe.crash("Simulator CachePage");
+ }
+ return new_page;
+}
+
+// Flush from start up to and not including start + size.
+static void FlushOnePageLocked(SimulatorProcess::ICacheMap& i_cache,
+ intptr_t start, int size) {
+ MOZ_ASSERT(size <= CachePage::kPageSize);
+ MOZ_ASSERT(AllOnOnePage(start, size - 1));
+ MOZ_ASSERT((start & CachePage::kLineMask) == 0);
+ MOZ_ASSERT((size & CachePage::kLineMask) == 0);
+ void* page = reinterpret_cast<void*>(start & (~CachePage::kPageMask));
+ int offset = (start & CachePage::kPageMask);
+ CachePage* cache_page = GetCachePageLocked(i_cache, page);
+ char* valid_bytemap = cache_page->validityByte(offset);
+ memset(valid_bytemap, CachePage::LINE_INVALID, size >> CachePage::kLineShift);
+}
+
+static void FlushICacheLocked(SimulatorProcess::ICacheMap& i_cache,
+ void* start_addr, size_t size) {
+ intptr_t start = reinterpret_cast<intptr_t>(start_addr);
+ int intra_line = (start & CachePage::kLineMask);
+ start -= intra_line;
+ size += intra_line;
+ size = ((size - 1) | CachePage::kLineMask) + 1;
+ int offset = (start & CachePage::kPageMask);
+ while (!AllOnOnePage(start, size - 1)) {
+ int bytes_to_flush = CachePage::kPageSize - offset;
+ FlushOnePageLocked(i_cache, start, bytes_to_flush);
+ start += bytes_to_flush;
+ size -= bytes_to_flush;
+ MOZ_ASSERT((start & CachePage::kPageMask) == 0);
+ offset = 0;
+ }
+ if (size != 0) {
+ FlushOnePageLocked(i_cache, start, size);
+ }
+}
+
+/* static */
+void SimulatorProcess::checkICacheLocked(SimInstruction* instr) {
+ intptr_t address = reinterpret_cast<intptr_t>(instr);
+ void* page = reinterpret_cast<void*>(address & (~CachePage::kPageMask));
+ void* line = reinterpret_cast<void*>(address & (~CachePage::kLineMask));
+ int offset = (address & CachePage::kPageMask);
+ CachePage* cache_page = GetCachePageLocked(icache(), page);
+ char* cache_valid_byte = cache_page->validityByte(offset);
+ bool cache_hit = (*cache_valid_byte == CachePage::LINE_VALID);
+ char* cached_line = cache_page->cachedData(offset & ~CachePage::kLineMask);
+
+ if (cache_hit) {
+ // Check that the data in memory matches the contents of the I-cache.
+ mozilla::DebugOnly<int> cmpret =
+ memcmp(reinterpret_cast<void*>(instr), cache_page->cachedData(offset),
+ SimInstruction::kInstrSize);
+ MOZ_ASSERT(cmpret == 0);
+ } else {
+ // Cache miss. Load memory into the cache.
+ memcpy(cached_line, line, CachePage::kLineLength);
+ *cache_valid_byte = CachePage::LINE_VALID;
+ }
+}
+
+HashNumber SimulatorProcess::ICacheHasher::hash(const Lookup& l) {
+ return U32(reinterpret_cast<uintptr_t>(l)) >> 2;
+}
+
+bool SimulatorProcess::ICacheHasher::match(const Key& k, const Lookup& l) {
+ MOZ_ASSERT((reinterpret_cast<intptr_t>(k) & CachePage::kPageMask) == 0);
+ MOZ_ASSERT((reinterpret_cast<intptr_t>(l) & CachePage::kPageMask) == 0);
+ return k == l;
+}
+
+/* static */
+void SimulatorProcess::FlushICache(void* start_addr, size_t size) {
+ if (!ICacheCheckingDisableCount) {
+ AutoLockSimulatorCache als;
+ js::jit::FlushICacheLocked(icache(), start_addr, size);
+ }
+}
+
+Simulator::Simulator() {
+ // Set up simulator support first. Some of this information is needed to
+ // setup the architecture state.
+
+ // Note, allocation and anything that depends on allocated memory is
+ // deferred until init(), in order to handle OOM properly.
+
+ stack_ = nullptr;
+ stackLimit_ = 0;
+ pc_modified_ = false;
+ icount_ = 0;
+ break_count_ = 0;
+ break_pc_ = nullptr;
+ break_instr_ = 0;
+ single_stepping_ = false;
+ single_step_callback_ = nullptr;
+ single_step_callback_arg_ = nullptr;
+
+ // Set up architecture state.
+ // All registers are initialized to zero to start with.
+ for (int i = 0; i < Register::kNumSimuRegisters; i++) {
+ registers_[i] = 0;
+ }
+ for (int i = 0; i < Simulator::FPURegister::kNumFPURegisters; i++) {
+ FPUregisters_[i] = 0;
+ }
+
+ for (int i = 0; i < kNumCFRegisters; i++) {
+ CFregisters_[i] = 0;
+ }
+
+ FCSR_ = 0;
+ LLBit_ = false;
+ LLAddr_ = 0;
+ lastLLValue_ = 0;
+
+ // The ra and pc are initialized to a known bad value that will cause an
+ // access violation if the simulator ever tries to execute it.
+ registers_[pc] = bad_ra;
+ registers_[ra] = bad_ra;
+
+ for (int i = 0; i < kNumExceptions; i++) {
+ exceptions[i] = 0;
+ }
+
+ lastDebuggerInput_ = nullptr;
+}
+
+bool Simulator::init() {
+ // Allocate 2MB for the stack. Note that we will only use 1MB, see below.
+ static const size_t stackSize = 2 * 1024 * 1024;
+ stack_ = js_pod_malloc<char>(stackSize);
+ if (!stack_) {
+ return false;
+ }
+
+ // Leave a safety margin of 1MB to prevent overrunning the stack when
+ // pushing values (total stack size is 2MB).
+ stackLimit_ = reinterpret_cast<uintptr_t>(stack_) + 1024 * 1024;
+
+ // The sp is initialized to point to the bottom (high address) of the
+ // allocated stack area. To be safe in potential stack underflows we leave
+ // some buffer below.
+ registers_[sp] = reinterpret_cast<int64_t>(stack_) + stackSize - 64;
+
+ return true;
+}
+
+// When the generated code calls an external reference we need to catch that in
+// the simulator. The external reference will be a function compiled for the
+// host architecture. We need to call that function instead of trying to
+// execute it with the simulator. We do that by redirecting the external
+// reference to a swi (software-interrupt) instruction that is handled by
+// the simulator. We write the original destination of the jump just at a known
+// offset from the swi instruction so the simulator knows what to call.
+class Redirection {
+ friend class SimulatorProcess;
+
+ // sim's lock must already be held.
+ Redirection(void* nativeFunction, ABIFunctionType type)
+ : nativeFunction_(nativeFunction),
+ swiInstruction_(kCallRedirInstr),
+ type_(type),
+ next_(nullptr) {
+ next_ = SimulatorProcess::redirection();
+ if (!SimulatorProcess::ICacheCheckingDisableCount) {
+ FlushICacheLocked(SimulatorProcess::icache(), addressOfSwiInstruction(),
+ SimInstruction::kInstrSize);
+ }
+ SimulatorProcess::setRedirection(this);
+ }
+
+ public:
+ void* addressOfSwiInstruction() { return &swiInstruction_; }
+ void* nativeFunction() const { return nativeFunction_; }
+ ABIFunctionType type() const { return type_; }
+
+ static Redirection* Get(void* nativeFunction, ABIFunctionType type) {
+ AutoLockSimulatorCache als;
+
+ Redirection* current = SimulatorProcess::redirection();
+ for (; current != nullptr; current = current->next_) {
+ if (current->nativeFunction_ == nativeFunction) {
+ MOZ_ASSERT(current->type() == type);
+ return current;
+ }
+ }
+
+ // Note: we can't use js_new here because the constructor is private.
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ Redirection* redir = js_pod_malloc<Redirection>(1);
+ if (!redir) {
+ oomUnsafe.crash("Simulator redirection");
+ }
+ new (redir) Redirection(nativeFunction, type);
+ return redir;
+ }
+
+ static Redirection* FromSwiInstruction(SimInstruction* swiInstruction) {
+ uint8_t* addrOfSwi = reinterpret_cast<uint8_t*>(swiInstruction);
+ uint8_t* addrOfRedirection =
+ addrOfSwi - offsetof(Redirection, swiInstruction_);
+ return reinterpret_cast<Redirection*>(addrOfRedirection);
+ }
+
+ private:
+ void* nativeFunction_;
+ uint32_t swiInstruction_;
+ ABIFunctionType type_;
+ Redirection* next_;
+};
+
+Simulator::~Simulator() { js_free(stack_); }
+
+SimulatorProcess::SimulatorProcess()
+ : cacheLock_(mutexid::SimulatorCacheLock), redirection_(nullptr) {
+ if (getenv("LOONG64_SIM_ICACHE_CHECKS")) {
+ ICacheCheckingDisableCount = 0;
+ }
+}
+
+SimulatorProcess::~SimulatorProcess() {
+ Redirection* r = redirection_;
+ while (r) {
+ Redirection* next = r->next_;
+ js_delete(r);
+ r = next;
+ }
+}
+
+/* static */
+void* Simulator::RedirectNativeFunction(void* nativeFunction,
+ ABIFunctionType type) {
+ Redirection* redirection = Redirection::Get(nativeFunction, type);
+ return redirection->addressOfSwiInstruction();
+}
+
+// Get the active Simulator for the current thread.
+Simulator* Simulator::Current() {
+ JSContext* cx = TlsContext.get();
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cx->runtime()));
+ return cx->simulator();
+}
+
+// Sets the register in the architecture state. It will also deal with updating
+// Simulator internal state for special registers such as PC.
+void Simulator::setRegister(int reg, int64_t value) {
+ MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters));
+ if (reg == pc) {
+ pc_modified_ = true;
+ }
+
+ // Zero register always holds 0.
+ registers_[reg] = (reg == 0) ? 0 : value;
+}
+
+void Simulator::setFpuRegister(int fpureg, int64_t value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ FPUregisters_[fpureg] = value;
+}
+
+void Simulator::setFpuRegisterHiWord(int fpureg, int32_t value) {
+ // Set ONLY upper 32-bits, leaving lower bits untouched.
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ int32_t* phiword;
+ phiword = (reinterpret_cast<int32_t*>(&FPUregisters_[fpureg])) + 1;
+
+ *phiword = value;
+}
+
+void Simulator::setFpuRegisterWord(int fpureg, int32_t value) {
+ // Set ONLY lower 32-bits, leaving upper bits untouched.
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ int32_t* pword;
+ pword = reinterpret_cast<int32_t*>(&FPUregisters_[fpureg]);
+
+ *pword = value;
+}
+
+void Simulator::setFpuRegisterWordInvalidResult(float original, float rounded,
+ int fpureg) {
+ double max_int32 = static_cast<double>(INT32_MAX);
+ double min_int32 = static_cast<double>(INT32_MIN);
+
+ if (std::isnan(original)) {
+ setFpuRegisterWord(fpureg, 0);
+ } else if (rounded > max_int32) {
+ setFpuRegister(fpureg, kFPUInvalidResult);
+ } else if (rounded < min_int32) {
+ setFpuRegister(fpureg, kFPUInvalidResultNegative);
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void Simulator::setFpuRegisterWordInvalidResult(double original, double rounded,
+ int fpureg) {
+ double max_int32 = static_cast<double>(INT32_MAX);
+ double min_int32 = static_cast<double>(INT32_MIN);
+
+ if (std::isnan(original)) {
+ setFpuRegisterWord(fpureg, 0);
+ } else if (rounded > max_int32) {
+ setFpuRegisterWord(fpureg, kFPUInvalidResult);
+ } else if (rounded < min_int32) {
+ setFpuRegisterWord(fpureg, kFPUInvalidResultNegative);
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void Simulator::setFpuRegisterInvalidResult(float original, float rounded,
+ int fpureg) {
+ double max_int32 = static_cast<double>(INT32_MAX);
+ double min_int32 = static_cast<double>(INT32_MIN);
+
+ if (std::isnan(original)) {
+ setFpuRegister(fpureg, 0);
+ } else if (rounded > max_int32) {
+ setFpuRegister(fpureg, kFPUInvalidResult);
+ } else if (rounded < min_int32) {
+ setFpuRegister(fpureg, kFPUInvalidResultNegative);
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void Simulator::setFpuRegisterInvalidResult(double original, double rounded,
+ int fpureg) {
+ double max_int32 = static_cast<double>(INT32_MAX);
+ double min_int32 = static_cast<double>(INT32_MIN);
+
+ if (std::isnan(original)) {
+ setFpuRegister(fpureg, 0);
+ } else if (rounded > max_int32) {
+ setFpuRegister(fpureg, kFPUInvalidResult);
+ } else if (rounded < min_int32) {
+ setFpuRegister(fpureg, kFPUInvalidResultNegative);
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void Simulator::setFpuRegisterInvalidResult64(float original, float rounded,
+ int fpureg) {
+ // The value of INT64_MAX (2^63-1) can't be represented as double exactly,
+ // loading the most accurate representation into max_int64, which is 2^63.
+ double max_int64 = static_cast<double>(INT64_MAX);
+ double min_int64 = static_cast<double>(INT64_MIN);
+
+ if (std::isnan(original)) {
+ setFpuRegister(fpureg, 0);
+ } else if (rounded >= max_int64) {
+ setFpuRegister(fpureg, kFPU64InvalidResult);
+ } else if (rounded < min_int64) {
+ setFpuRegister(fpureg, kFPU64InvalidResultNegative);
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void Simulator::setFpuRegisterInvalidResult64(double original, double rounded,
+ int fpureg) {
+ // The value of INT64_MAX (2^63-1) can't be represented as double exactly,
+ // loading the most accurate representation into max_int64, which is 2^63.
+ double max_int64 = static_cast<double>(INT64_MAX);
+ double min_int64 = static_cast<double>(INT64_MIN);
+
+ if (std::isnan(original)) {
+ setFpuRegister(fpureg, 0);
+ } else if (rounded >= max_int64) {
+ setFpuRegister(fpureg, kFPU64InvalidResult);
+ } else if (rounded < min_int64) {
+ setFpuRegister(fpureg, kFPU64InvalidResultNegative);
+ } else {
+ UNREACHABLE();
+ }
+}
+
+void Simulator::setFpuRegisterFloat(int fpureg, float value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ *mozilla::BitwiseCast<float*>(&FPUregisters_[fpureg]) = value;
+}
+
+void Simulator::setFpuRegisterDouble(int fpureg, double value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ *mozilla::BitwiseCast<double*>(&FPUregisters_[fpureg]) = value;
+}
+
+void Simulator::setCFRegister(int cfreg, bool value) {
+ MOZ_ASSERT((cfreg >= 0) && (cfreg < kNumCFRegisters));
+ CFregisters_[cfreg] = value;
+}
+
+bool Simulator::getCFRegister(int cfreg) const {
+ MOZ_ASSERT((cfreg >= 0) && (cfreg < kNumCFRegisters));
+ return CFregisters_[cfreg];
+}
+
+// Get the register from the architecture state. This function does handle
+// the special case of accessing the PC register.
+int64_t Simulator::getRegister(int reg) const {
+ MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters));
+ if (reg == 0) {
+ return 0;
+ }
+ return registers_[reg] + ((reg == pc) ? SimInstruction::kPCReadOffset : 0);
+}
+
+int64_t Simulator::getFpuRegister(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return FPUregisters_[fpureg];
+}
+
+int32_t Simulator::getFpuRegisterWord(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *mozilla::BitwiseCast<int32_t*>(&FPUregisters_[fpureg]);
+}
+
+int32_t Simulator::getFpuRegisterSignedWord(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *mozilla::BitwiseCast<int32_t*>(&FPUregisters_[fpureg]);
+}
+
+int32_t Simulator::getFpuRegisterHiWord(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *((mozilla::BitwiseCast<int32_t*>(&FPUregisters_[fpureg])) + 1);
+}
+
+float Simulator::getFpuRegisterFloat(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *mozilla::BitwiseCast<float*>(&FPUregisters_[fpureg]);
+}
+
+double Simulator::getFpuRegisterDouble(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *mozilla::BitwiseCast<double*>(&FPUregisters_[fpureg]);
+}
+
+void Simulator::setCallResultDouble(double result) {
+ setFpuRegisterDouble(f0, result);
+}
+
+void Simulator::setCallResultFloat(float result) {
+ setFpuRegisterFloat(f0, result);
+}
+
+void Simulator::setCallResult(int64_t res) { setRegister(a0, res); }
+
+void Simulator::setCallResult(__int128_t res) {
+ setRegister(a0, I64(res));
+ setRegister(a1, I64(res >> 64));
+}
+
+// Helper functions for setting and testing the FCSR register's bits.
+void Simulator::setFCSRBit(uint32_t cc, bool value) {
+ if (value) {
+ FCSR_ |= (1 << cc);
+ } else {
+ FCSR_ &= ~(1 << cc);
+ }
+}
+
+bool Simulator::testFCSRBit(uint32_t cc) { return FCSR_ & (1 << cc); }
+
+unsigned int Simulator::getFCSRRoundingMode() {
+ return FCSR_ & kFPURoundingModeMask;
+}
+
+// Sets the rounding error codes in FCSR based on the result of the rounding.
+// Returns true if the operation was invalid.
+template <typename T>
+bool Simulator::setFCSRRoundError(double original, double rounded) {
+ bool ret = false;
+
+ setFCSRBit(kFCSRInexactCauseBit, false);
+ setFCSRBit(kFCSRUnderflowCauseBit, false);
+ setFCSRBit(kFCSROverflowCauseBit, false);
+ setFCSRBit(kFCSRInvalidOpCauseBit, false);
+
+ if (!std::isfinite(original) || !std::isfinite(rounded)) {
+ setFCSRBit(kFCSRInvalidOpFlagBit, true);
+ setFCSRBit(kFCSRInvalidOpCauseBit, true);
+ ret = true;
+ }
+
+ if (original != rounded) {
+ setFCSRBit(kFCSRInexactFlagBit, true);
+ setFCSRBit(kFCSRInexactCauseBit, true);
+ }
+
+ if (rounded < DBL_MIN && rounded > -DBL_MIN && rounded != 0) {
+ setFCSRBit(kFCSRUnderflowFlagBit, true);
+ setFCSRBit(kFCSRUnderflowCauseBit, true);
+ ret = true;
+ }
+
+ if ((long double)rounded > (long double)std::numeric_limits<T>::max() ||
+ (long double)rounded < (long double)std::numeric_limits<T>::min()) {
+ setFCSRBit(kFCSROverflowFlagBit, true);
+ setFCSRBit(kFCSROverflowCauseBit, true);
+ // The reference is not really clear but it seems this is required:
+ setFCSRBit(kFCSRInvalidOpFlagBit, true);
+ setFCSRBit(kFCSRInvalidOpCauseBit, true);
+ ret = true;
+ }
+
+ return ret;
+}
+
+// For cvt instructions only
+template <typename T>
+void Simulator::roundAccordingToFCSR(T toRound, T* rounded,
+ int32_t* rounded_int) {
+ switch ((FCSR_ >> 8) & 3) {
+ case kRoundToNearest:
+ *rounded = std::floor(toRound + 0.5);
+ *rounded_int = static_cast<int32_t>(*rounded);
+ if ((*rounded_int & 1) != 0 && *rounded_int - toRound == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ *rounded_int -= 1;
+ *rounded -= 1.;
+ }
+ break;
+ case kRoundToZero:
+ *rounded = trunc(toRound);
+ *rounded_int = static_cast<int32_t>(*rounded);
+ break;
+ case kRoundToPlusInf:
+ *rounded = std::ceil(toRound);
+ *rounded_int = static_cast<int32_t>(*rounded);
+ break;
+ case kRoundToMinusInf:
+ *rounded = std::floor(toRound);
+ *rounded_int = static_cast<int32_t>(*rounded);
+ break;
+ }
+}
+
+template <typename T>
+void Simulator::round64AccordingToFCSR(T toRound, T* rounded,
+ int64_t* rounded_int) {
+ switch ((FCSR_ >> 8) & 3) {
+ case kRoundToNearest:
+ *rounded = std::floor(toRound + 0.5);
+ *rounded_int = static_cast<int64_t>(*rounded);
+ if ((*rounded_int & 1) != 0 && *rounded_int - toRound == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ *rounded_int -= 1;
+ *rounded -= 1.;
+ }
+ break;
+ case kRoundToZero:
+ *rounded = trunc(toRound);
+ *rounded_int = static_cast<int64_t>(*rounded);
+ break;
+ case kRoundToPlusInf:
+ *rounded = std::ceil(toRound);
+ *rounded_int = static_cast<int64_t>(*rounded);
+ break;
+ case kRoundToMinusInf:
+ *rounded = std::floor(toRound);
+ *rounded_int = static_cast<int64_t>(*rounded);
+ break;
+ }
+}
+
+// Raw access to the PC register.
+void Simulator::set_pc(int64_t value) {
+ pc_modified_ = true;
+ registers_[pc] = value;
+}
+
+bool Simulator::has_bad_pc() const {
+ return ((registers_[pc] == bad_ra) || (registers_[pc] == end_sim_pc));
+}
+
+// Raw access to the PC register without the special adjustment when reading.
+int64_t Simulator::get_pc() const { return registers_[pc]; }
+
+JS::ProfilingFrameIterator::RegisterState Simulator::registerState() {
+ wasm::RegisterState state;
+ state.pc = (void*)get_pc();
+ state.fp = (void*)getRegister(fp);
+ state.sp = (void*)getRegister(sp);
+ state.lr = (void*)getRegister(ra);
+ return state;
+}
+
+uint8_t Simulator::readBU(uint64_t addr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return 0xff;
+ }
+
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(addr);
+ return *ptr;
+}
+
+int8_t Simulator::readB(uint64_t addr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return -1;
+ }
+
+ int8_t* ptr = reinterpret_cast<int8_t*>(addr);
+ return *ptr;
+}
+
+void Simulator::writeB(uint64_t addr, uint8_t value) {
+ if (handleWasmSegFault(addr, 1)) {
+ return;
+ }
+
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(addr);
+ *ptr = value;
+}
+
+void Simulator::writeB(uint64_t addr, int8_t value) {
+ if (handleWasmSegFault(addr, 1)) {
+ return;
+ }
+
+ int8_t* ptr = reinterpret_cast<int8_t*>(addr);
+ *ptr = value;
+}
+
+uint16_t Simulator::readHU(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return 0xffff;
+ }
+
+ uint16_t* ptr = reinterpret_cast<uint16_t*>(addr);
+ return *ptr;
+}
+
+int16_t Simulator::readH(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return -1;
+ }
+
+ int16_t* ptr = reinterpret_cast<int16_t*>(addr);
+ return *ptr;
+}
+
+void Simulator::writeH(uint64_t addr, uint16_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return;
+ }
+
+ uint16_t* ptr = reinterpret_cast<uint16_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+}
+
+void Simulator::writeH(uint64_t addr, int16_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return;
+ }
+
+ int16_t* ptr = reinterpret_cast<int16_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+}
+
+uint32_t Simulator::readWU(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ uint32_t* ptr = reinterpret_cast<uint32_t*>(addr);
+ return *ptr;
+}
+
+int32_t Simulator::readW(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ int32_t* ptr = reinterpret_cast<int32_t*>(addr);
+ return *ptr;
+}
+
+void Simulator::writeW(uint64_t addr, uint32_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return;
+ }
+
+ uint32_t* ptr = reinterpret_cast<uint32_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+}
+
+void Simulator::writeW(uint64_t addr, int32_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return;
+ }
+
+ int32_t* ptr = reinterpret_cast<int32_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+}
+
+int64_t Simulator::readDW(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return -1;
+ }
+
+ intptr_t* ptr = reinterpret_cast<intptr_t*>(addr);
+ return *ptr;
+}
+
+void Simulator::writeDW(uint64_t addr, int64_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return;
+ }
+
+ int64_t* ptr = reinterpret_cast<int64_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+}
+
+double Simulator::readD(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return NAN;
+ }
+
+ double* ptr = reinterpret_cast<double*>(addr);
+ return *ptr;
+}
+
+void Simulator::writeD(uint64_t addr, double value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return;
+ }
+
+ double* ptr = reinterpret_cast<double*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+}
+
+int Simulator::loadLinkedW(uint64_t addr, SimInstruction* instr) {
+ if ((addr & 3) == 0) {
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ volatile int32_t* ptr = reinterpret_cast<volatile int32_t*>(addr);
+ int32_t value = *ptr;
+ lastLLValue_ = value;
+ LLAddr_ = addr;
+ // Note that any memory write or "external" interrupt should reset this
+ // value to false.
+ LLBit_ = true;
+ return value;
+ }
+ printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int Simulator::storeConditionalW(uint64_t addr, int value,
+ SimInstruction* instr) {
+ // Correct behavior in this case, as defined by architecture, is to just
+ // return 0, but there is no point at allowing that. It is certainly an
+ // indicator of a bug.
+ if (addr != LLAddr_) {
+ printf("SC to bad address: 0x%016" PRIx64 ", pc=0x%016" PRIx64
+ ", expected: 0x%016" PRIx64 "\n",
+ addr, reinterpret_cast<intptr_t>(instr), LLAddr_);
+ MOZ_CRASH();
+ }
+
+ if ((addr & 3) == 0) {
+ SharedMem<int32_t*> ptr =
+ SharedMem<int32_t*>::shared(reinterpret_cast<int32_t*>(addr));
+
+ if (!LLBit_) {
+ return 0;
+ }
+
+ LLBit_ = false;
+ LLAddr_ = 0;
+ int32_t expected = int32_t(lastLLValue_);
+ int32_t old =
+ AtomicOperations::compareExchangeSeqCst(ptr, expected, int32_t(value));
+ return (old == expected) ? 1 : 0;
+ }
+ printf("Unaligned SC at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int64_t Simulator::loadLinkedD(uint64_t addr, SimInstruction* instr) {
+ if ((addr & kPointerAlignmentMask) == 0) {
+ if (handleWasmSegFault(addr, 8)) {
+ return -1;
+ }
+
+ volatile int64_t* ptr = reinterpret_cast<volatile int64_t*>(addr);
+ int64_t value = *ptr;
+ lastLLValue_ = value;
+ LLAddr_ = addr;
+ // Note that any memory write or "external" interrupt should reset this
+ // value to false.
+ LLBit_ = true;
+ return value;
+ }
+ printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int Simulator::storeConditionalD(uint64_t addr, int64_t value,
+ SimInstruction* instr) {
+ // Correct behavior in this case, as defined by architecture, is to just
+ // return 0, but there is no point at allowing that. It is certainly an
+ // indicator of a bug.
+ if (addr != LLAddr_) {
+ printf("SC to bad address: 0x%016" PRIx64 ", pc=0x%016" PRIx64
+ ", expected: 0x%016" PRIx64 "\n",
+ addr, reinterpret_cast<intptr_t>(instr), LLAddr_);
+ MOZ_CRASH();
+ }
+
+ if ((addr & kPointerAlignmentMask) == 0) {
+ SharedMem<int64_t*> ptr =
+ SharedMem<int64_t*>::shared(reinterpret_cast<int64_t*>(addr));
+
+ if (!LLBit_) {
+ return 0;
+ }
+
+ LLBit_ = false;
+ LLAddr_ = 0;
+ int64_t expected = lastLLValue_;
+ int64_t old =
+ AtomicOperations::compareExchangeSeqCst(ptr, expected, int64_t(value));
+ return (old == expected) ? 1 : 0;
+ }
+ printf("Unaligned SC at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+uintptr_t Simulator::stackLimit() const { return stackLimit_; }
+
+uintptr_t* Simulator::addressOfStackLimit() { return &stackLimit_; }
+
+bool Simulator::overRecursed(uintptr_t newsp) const {
+ if (newsp == 0) {
+ newsp = getRegister(sp);
+ }
+ return newsp <= stackLimit();
+}
+
+bool Simulator::overRecursedWithExtra(uint32_t extra) const {
+ uintptr_t newsp = getRegister(sp) - extra;
+ return newsp <= stackLimit();
+}
+
+// Unsupported instructions use format to print an error and stop execution.
+void Simulator::format(SimInstruction* instr, const char* format) {
+ printf("Simulator found unsupported instruction:\n 0x%016lx: %s\n",
+ reinterpret_cast<intptr_t>(instr), format);
+ MOZ_CRASH();
+}
+
+// Note: With the code below we assume that all runtime calls return a 64 bits
+// result. If they don't, the a1 result register contains a bogus value, which
+// is fine because it is caller-saved.
+typedef int64_t (*Prototype_General0)();
+typedef int64_t (*Prototype_General1)(int64_t arg0);
+typedef int64_t (*Prototype_General2)(int64_t arg0, int64_t arg1);
+typedef int64_t (*Prototype_General3)(int64_t arg0, int64_t arg1, int64_t arg2);
+typedef int64_t (*Prototype_General4)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3);
+typedef int64_t (*Prototype_General5)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3, int64_t arg4);
+typedef int64_t (*Prototype_General6)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3, int64_t arg4, int64_t arg5);
+typedef int64_t (*Prototype_General7)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3, int64_t arg4, int64_t arg5,
+ int64_t arg6);
+typedef int64_t (*Prototype_General8)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3, int64_t arg4, int64_t arg5,
+ int64_t arg6, int64_t arg7);
+typedef int64_t (*Prototype_GeneralGeneralGeneralInt64)(int64_t arg0,
+ int64_t arg1,
+ int64_t arg2,
+ int64_t arg3);
+typedef int64_t (*Prototype_GeneralGeneralInt64Int64)(int64_t arg0,
+ int64_t arg1,
+ int64_t arg2,
+ int64_t arg3);
+typedef int64_t (*Prototype_Int_Float32)(float arg0);
+typedef int64_t (*Prototype_Int_Double)(double arg0);
+typedef int64_t (*Prototype_Int_IntDouble)(int64_t arg0, double arg1);
+typedef int64_t (*Prototype_Int_DoubleInt)(double arg0, int64_t arg1);
+typedef int64_t (*Prototype_Int_DoubleIntInt)(double arg0, int64_t arg1,
+ int64_t arg2);
+typedef int64_t (*Prototype_Int_IntDoubleIntInt)(int64_t arg0, double arg1,
+ int64_t arg2, int64_t arg3);
+
+typedef float (*Prototype_Float32_Float32)(float arg0);
+typedef float (*Prototype_Float32_Float32Float32)(float arg0, float arg1);
+
+typedef double (*Prototype_Double_None)();
+typedef double (*Prototype_Double_Double)(double arg0);
+typedef double (*Prototype_Double_Int)(int64_t arg0);
+typedef double (*Prototype_Double_DoubleInt)(double arg0, int64_t arg1);
+typedef double (*Prototype_Double_IntDouble)(int64_t arg0, double arg1);
+typedef double (*Prototype_Double_DoubleDouble)(double arg0, double arg1);
+typedef double (*Prototype_Double_DoubleDoubleDouble)(double arg0, double arg1,
+ double arg2);
+typedef double (*Prototype_Double_DoubleDoubleDoubleDouble)(double arg0,
+ double arg1,
+ double arg2,
+ double arg3);
+
+typedef int32_t (*Prototype_Int32_General)(int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32)(int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32)(int64_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32)(int64_t, int32_t,
+ int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32Int32)(
+ int64_t, int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32General)(
+ int64_t, int32_t, int32_t, int32_t, int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32General)(
+ int64_t, int32_t, int32_t, int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int64)(int64_t, int32_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32General)(int64_t, int32_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int64Int64)(int64_t, int32_t,
+ int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32)(int64_t, int32_t,
+ int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32Int32)(
+ int64_t, int32_t, int64_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneral)(int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralGeneral)(int64_t, int64_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralInt32Int32)(int64_t, int64_t,
+ int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int32Int32)(int64_t, int64_t,
+ int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32)(int64_t, int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int64)(int64_t, int64_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int64General)(
+ int64_t, int64_t, int32_t, int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int64Int64)(int64_t, int64_t,
+ int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int64Int64General)(
+ int64_t, int64_t, int64_t, int64_t, int64_t);
+typedef int64_t (*Prototype_General_GeneralInt32)(int64_t, int32_t);
+typedef int64_t (*Prototype_General_GeneralInt32Int32)(int64_t, int32_t,
+ int32_t);
+typedef int64_t (*Prototype_General_GeneralInt32General)(int64_t, int32_t,
+ int64_t);
+typedef int64_t (*Prototype_General_GeneralInt32Int32GeneralInt32)(
+ int64_t, int32_t, int32_t, int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32)(
+ int64_t, int64_t, int32_t, int64_t, int32_t, int32_t, int32_t);
+typedef int64_t (*Prototype_Int64_General)(int64_t);
+typedef int64_t (*Prototype_Int64_GeneralInt64)(int64_t, int64_t);
+
+inline int32_t Simulator::rj_reg(SimInstruction* instr) const {
+ return instr->rjValue();
+}
+
+inline int64_t Simulator::rj(SimInstruction* instr) const {
+ return getRegister(rj_reg(instr));
+}
+
+inline uint64_t Simulator::rj_u(SimInstruction* instr) const {
+ return static_cast<uint64_t>(getRegister(rj_reg(instr)));
+}
+
+inline int32_t Simulator::rk_reg(SimInstruction* instr) const {
+ return instr->rkValue();
+}
+
+inline int64_t Simulator::rk(SimInstruction* instr) const {
+ return getRegister(rk_reg(instr));
+}
+
+inline uint64_t Simulator::rk_u(SimInstruction* instr) const {
+ return static_cast<uint64_t>(getRegister(rk_reg(instr)));
+}
+
+inline int32_t Simulator::rd_reg(SimInstruction* instr) const {
+ return instr->rdValue();
+}
+
+inline int64_t Simulator::rd(SimInstruction* instr) const {
+ return getRegister(rd_reg(instr));
+}
+
+inline uint64_t Simulator::rd_u(SimInstruction* instr) const {
+ return static_cast<uint64_t>(getRegister(rd_reg(instr)));
+}
+
+inline int32_t Simulator::fa_reg(SimInstruction* instr) const {
+ return instr->faValue();
+}
+
+inline float Simulator::fa_float(SimInstruction* instr) const {
+ return getFpuRegisterFloat(fa_reg(instr));
+}
+
+inline double Simulator::fa_double(SimInstruction* instr) const {
+ return getFpuRegisterDouble(fa_reg(instr));
+}
+
+inline int32_t Simulator::fj_reg(SimInstruction* instr) const {
+ return instr->fjValue();
+}
+
+inline float Simulator::fj_float(SimInstruction* instr) const {
+ return getFpuRegisterFloat(fj_reg(instr));
+}
+
+inline double Simulator::fj_double(SimInstruction* instr) const {
+ return getFpuRegisterDouble(fj_reg(instr));
+}
+
+inline int32_t Simulator::fk_reg(SimInstruction* instr) const {
+ return instr->fkValue();
+}
+
+inline float Simulator::fk_float(SimInstruction* instr) const {
+ return getFpuRegisterFloat(fk_reg(instr));
+}
+
+inline double Simulator::fk_double(SimInstruction* instr) const {
+ return getFpuRegisterDouble(fk_reg(instr));
+}
+
+inline int32_t Simulator::fd_reg(SimInstruction* instr) const {
+ return instr->fdValue();
+}
+
+inline float Simulator::fd_float(SimInstruction* instr) const {
+ return getFpuRegisterFloat(fd_reg(instr));
+}
+
+inline double Simulator::fd_double(SimInstruction* instr) const {
+ return getFpuRegisterDouble(fd_reg(instr));
+}
+
+inline int32_t Simulator::cj_reg(SimInstruction* instr) const {
+ return instr->cjValue();
+}
+
+inline bool Simulator::cj(SimInstruction* instr) const {
+ return getCFRegister(cj_reg(instr));
+}
+
+inline int32_t Simulator::cd_reg(SimInstruction* instr) const {
+ return instr->cdValue();
+}
+
+inline bool Simulator::cd(SimInstruction* instr) const {
+ return getCFRegister(cd_reg(instr));
+}
+
+inline int32_t Simulator::ca_reg(SimInstruction* instr) const {
+ return instr->caValue();
+}
+
+inline bool Simulator::ca(SimInstruction* instr) const {
+ return getCFRegister(ca_reg(instr));
+}
+
+inline uint32_t Simulator::sa2(SimInstruction* instr) const {
+ return instr->sa2Value();
+}
+
+inline uint32_t Simulator::sa3(SimInstruction* instr) const {
+ return instr->sa3Value();
+}
+
+inline uint32_t Simulator::ui5(SimInstruction* instr) const {
+ return instr->imm5Value();
+}
+
+inline uint32_t Simulator::ui6(SimInstruction* instr) const {
+ return instr->imm6Value();
+}
+
+inline uint32_t Simulator::lsbw(SimInstruction* instr) const {
+ return instr->lsbwValue();
+}
+
+inline uint32_t Simulator::msbw(SimInstruction* instr) const {
+ return instr->msbwValue();
+}
+
+inline uint32_t Simulator::lsbd(SimInstruction* instr) const {
+ return instr->lsbdValue();
+}
+
+inline uint32_t Simulator::msbd(SimInstruction* instr) const {
+ return instr->msbdValue();
+}
+
+inline uint32_t Simulator::cond(SimInstruction* instr) const {
+ return instr->condValue();
+}
+
+inline int32_t Simulator::si12(SimInstruction* instr) const {
+ return (instr->imm12Value() << 20) >> 20;
+}
+
+inline uint32_t Simulator::ui12(SimInstruction* instr) const {
+ return instr->imm12Value();
+}
+
+inline int32_t Simulator::si14(SimInstruction* instr) const {
+ return (instr->imm14Value() << 18) >> 18;
+}
+
+inline int32_t Simulator::si16(SimInstruction* instr) const {
+ return (instr->imm16Value() << 16) >> 16;
+}
+
+inline int32_t Simulator::si20(SimInstruction* instr) const {
+ return (instr->imm20Value() << 12) >> 12;
+}
+
+// Software interrupt instructions are used by the simulator to call into C++.
+void Simulator::softwareInterrupt(SimInstruction* instr) {
+ // the break_ instruction could get us here.
+ mozilla::DebugOnly<int32_t> opcode_hi15 = instr->bits(31, 17);
+ MOZ_ASSERT(opcode_hi15 == 0x15);
+ uint32_t code = instr->bits(14, 0);
+
+ if (instr->instructionBits() == kCallRedirInstr) {
+ Redirection* redirection = Redirection::FromSwiInstruction(instr);
+ uintptr_t nativeFn =
+ reinterpret_cast<uintptr_t>(redirection->nativeFunction());
+
+ int64_t arg0 = getRegister(a0);
+ int64_t arg1 = getRegister(a1);
+ int64_t arg2 = getRegister(a2);
+ int64_t arg3 = getRegister(a3);
+ int64_t arg4 = getRegister(a4);
+ int64_t arg5 = getRegister(a5);
+
+ // This is dodgy but it works because the C entry stubs are never moved.
+ // See comment in codegen-arm.cc and bug 1242173.
+ int64_t saved_ra = getRegister(ra);
+
+ intptr_t external =
+ reinterpret_cast<intptr_t>(redirection->nativeFunction());
+
+ bool stack_aligned = (getRegister(sp) & (ABIStackAlignment - 1)) == 0;
+ if (!stack_aligned) {
+ fprintf(stderr, "Runtime call with unaligned stack!\n");
+ MOZ_CRASH();
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ switch (redirection->type()) {
+ case Args_General0: {
+ Prototype_General0 target =
+ reinterpret_cast<Prototype_General0>(external);
+ int64_t result = target();
+ setCallResult(result);
+ break;
+ }
+ case Args_General1: {
+ Prototype_General1 target =
+ reinterpret_cast<Prototype_General1>(external);
+ int64_t result = target(arg0);
+ setCallResult(result);
+ break;
+ }
+ case Args_General2: {
+ Prototype_General2 target =
+ reinterpret_cast<Prototype_General2>(external);
+ int64_t result = target(arg0, arg1);
+ setCallResult(result);
+ break;
+ }
+ case Args_General3: {
+ Prototype_General3 target =
+ reinterpret_cast<Prototype_General3>(external);
+ int64_t result = target(arg0, arg1, arg2);
+ if (external == intptr_t(&js::wasm::Instance::wake_m32)) {
+ result = int32_t(result);
+ }
+ setCallResult(result);
+ break;
+ }
+ case Args_General4: {
+ Prototype_General4 target =
+ reinterpret_cast<Prototype_General4>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ setCallResult(result);
+ break;
+ }
+ case Args_General5: {
+ Prototype_General5 target =
+ reinterpret_cast<Prototype_General5>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4);
+ setCallResult(result);
+ break;
+ }
+ case Args_General6: {
+ Prototype_General6 target =
+ reinterpret_cast<Prototype_General6>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5);
+ setCallResult(result);
+ break;
+ }
+ case Args_General7: {
+ Prototype_General7 target =
+ reinterpret_cast<Prototype_General7>(external);
+ int64_t arg6 = getRegister(a6);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6);
+ setCallResult(result);
+ break;
+ }
+ case Args_General8: {
+ Prototype_General8 target =
+ reinterpret_cast<Prototype_General8>(external);
+ int64_t arg6 = getRegister(a6);
+ int64_t arg7 = getRegister(a7);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+ setCallResult(result);
+ break;
+ }
+ case Args_Double_None: {
+ Prototype_Double_None target =
+ reinterpret_cast<Prototype_Double_None>(external);
+ double dresult = target();
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int_Float32: {
+ float fval0;
+ fval0 = getFpuRegisterFloat(0);
+ Prototype_Int_Float32 target =
+ reinterpret_cast<Prototype_Int_Float32>(external);
+ int64_t result = target(fval0);
+ setRegister(a0, result);
+ break;
+ }
+ case Args_Int_Double: {
+ double dval0 = getFpuRegisterDouble(0);
+ Prototype_Int_Double target =
+ reinterpret_cast<Prototype_Int_Double>(external);
+ int64_t result = target(dval0);
+ if (external == intptr_t((int32_t(*)(double))JS::ToInt32)) {
+ result = int32_t(result);
+ }
+ setRegister(a0, result);
+ break;
+ }
+ case Args_Int_GeneralGeneralGeneralInt64: {
+ Prototype_GeneralGeneralGeneralInt64 target =
+ reinterpret_cast<Prototype_GeneralGeneralGeneralInt64>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ if (external == intptr_t(&js::wasm::Instance::wait_i32_m32)) {
+ result = int32_t(result);
+ }
+ setRegister(a0, result);
+ break;
+ }
+ case Args_Int_GeneralGeneralInt64Int64: {
+ Prototype_GeneralGeneralInt64Int64 target =
+ reinterpret_cast<Prototype_GeneralGeneralInt64Int64>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ if (external == intptr_t(&js::wasm::Instance::wait_i64_m32)) {
+ result = int32_t(result);
+ }
+ setRegister(a0, result);
+ break;
+ }
+ case Args_Int_DoubleInt: {
+ double dval = getFpuRegisterDouble(0);
+ Prototype_Int_DoubleInt target =
+ reinterpret_cast<Prototype_Int_DoubleInt>(external);
+ int64_t result = target(dval, arg0);
+ setRegister(a0, result);
+ break;
+ }
+ case Args_Int_DoubleIntInt: {
+ double dval = getFpuRegisterDouble(0);
+ Prototype_Int_DoubleIntInt target =
+ reinterpret_cast<Prototype_Int_DoubleIntInt>(external);
+ int64_t result = target(dval, arg0, arg1);
+ setRegister(a0, result);
+ break;
+ }
+ case Args_Int_IntDoubleIntInt: {
+ double dval = getFpuRegisterDouble(0);
+ Prototype_Int_IntDoubleIntInt target =
+ reinterpret_cast<Prototype_Int_IntDoubleIntInt>(external);
+ int64_t result = target(arg0, dval, arg1, arg2);
+ setRegister(a0, result);
+ break;
+ }
+ case Args_Double_Double: {
+ double dval0 = getFpuRegisterDouble(0);
+ Prototype_Double_Double target =
+ reinterpret_cast<Prototype_Double_Double>(external);
+ double dresult = target(dval0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Float32_Float32: {
+ float fval0;
+ fval0 = getFpuRegisterFloat(0);
+ Prototype_Float32_Float32 target =
+ reinterpret_cast<Prototype_Float32_Float32>(external);
+ float fresult = target(fval0);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Float32_Float32Float32: {
+ float fval0;
+ float fval1;
+ fval0 = getFpuRegisterFloat(0);
+ fval1 = getFpuRegisterFloat(1);
+ Prototype_Float32_Float32Float32 target =
+ reinterpret_cast<Prototype_Float32_Float32Float32>(external);
+ float fresult = target(fval0, fval1);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Double_Int: {
+ Prototype_Double_Int target =
+ reinterpret_cast<Prototype_Double_Int>(external);
+ double dresult = target(arg0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleInt: {
+ double dval0 = getFpuRegisterDouble(0);
+ Prototype_Double_DoubleInt target =
+ reinterpret_cast<Prototype_Double_DoubleInt>(external);
+ double dresult = target(dval0, arg0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleDouble: {
+ double dval0 = getFpuRegisterDouble(0);
+ double dval1 = getFpuRegisterDouble(1);
+ Prototype_Double_DoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDouble>(external);
+ double dresult = target(dval0, dval1);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_IntDouble: {
+ double dval0 = getFpuRegisterDouble(0);
+ Prototype_Double_IntDouble target =
+ reinterpret_cast<Prototype_Double_IntDouble>(external);
+ double dresult = target(arg0, dval0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int_IntDouble: {
+ double dval0 = getFpuRegisterDouble(0);
+ Prototype_Int_IntDouble target =
+ reinterpret_cast<Prototype_Int_IntDouble>(external);
+ int64_t result = target(arg0, dval0);
+ setRegister(a0, result);
+ break;
+ }
+ case Args_Double_DoubleDoubleDouble: {
+ double dval0 = getFpuRegisterDouble(0);
+ double dval1 = getFpuRegisterDouble(1);
+ double dval2 = getFpuRegisterDouble(2);
+ Prototype_Double_DoubleDoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDoubleDouble>(external);
+ double dresult = target(dval0, dval1, dval2);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleDoubleDoubleDouble: {
+ double dval0 = getFpuRegisterDouble(0);
+ double dval1 = getFpuRegisterDouble(1);
+ double dval2 = getFpuRegisterDouble(2);
+ double dval3 = getFpuRegisterDouble(3);
+ Prototype_Double_DoubleDoubleDoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDoubleDoubleDouble>(
+ external);
+ double dresult = target(dval0, dval1, dval2, dval3);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int32_General: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_General>(nativeFn)(arg0);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32>(nativeFn)(
+ arg0, I32(arg1));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), I32(arg4));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32Int32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), I32(arg4),
+ I32(arg5));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32General: {
+ int32_t ret = reinterpret_cast<
+ Prototype_Int32_GeneralInt32Int32Int32Int32General>(nativeFn)(
+ arg0, I32(arg1), I32(arg2), I32(arg3), I32(arg4), arg5);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32General>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), arg4);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int64>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), arg3);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32General>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), arg3);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int64Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int64Int64>(
+ nativeFn)(arg0, I32(arg1), arg2, arg3);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32GeneralInt32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32GeneralInt32>(
+ nativeFn)(arg0, I32(arg1), arg2, I32(arg3));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32GeneralInt32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32GeneralInt32Int32>(
+ nativeFn)(arg0, I32(arg1), arg2, I32(arg3), I32(arg4));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralGeneral: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralGeneral>(
+ nativeFn)(arg0, arg1);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralGeneralGeneral: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralGeneralGeneral>(
+ nativeFn)(arg0, arg1, arg2);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralGeneralInt32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralGeneralInt32Int32>(
+ nativeFn)(arg0, arg1, I32(arg2), I32(arg3));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int32Int32>(
+ nativeFn)(arg0, arg1, I32(arg2), I32(arg3), I32(arg4));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int32>(
+ nativeFn)(arg0, arg1, I32(arg2));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int64>(
+ nativeFn)(arg0, arg1, I32(arg2), arg3);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int64General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int64General>(
+ nativeFn)(arg0, arg1, I32(arg2), arg3, arg4);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int64Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int64Int64>(
+ nativeFn)(arg0, arg1, arg2, arg3);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int64Int64General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int64Int64General>(
+ nativeFn)(arg0, arg1, arg2, arg3, arg4);
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case Args_General_GeneralInt32: {
+ int64_t ret = reinterpret_cast<Prototype_General_GeneralInt32>(
+ nativeFn)(arg0, I32(arg1));
+ setRegister(a0, ret);
+ break;
+ }
+ case Args_General_GeneralInt32Int32: {
+ int64_t ret = reinterpret_cast<Prototype_General_GeneralInt32Int32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2));
+ setRegister(a0, ret);
+ break;
+ }
+ case Args_General_GeneralInt32General: {
+ int64_t ret = reinterpret_cast<Prototype_General_GeneralInt32General>(
+ nativeFn)(arg0, I32(arg1), arg2);
+ setRegister(a0, ret);
+ break;
+ }
+ case js::jit::Args_General_GeneralInt32Int32GeneralInt32: {
+ int64_t ret =
+ reinterpret_cast<Prototype_General_GeneralInt32Int32GeneralInt32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), arg3, I32(arg4));
+ setRegister(a0, ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralGeneralInt32GeneralInt32Int32Int32: {
+ int64_t arg6 = getRegister(a6);
+ int32_t ret = reinterpret_cast<
+ Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32>(
+ nativeFn)(arg0, arg1, I32(arg2), arg3, I32(arg4), I32(arg5),
+ I32(arg6));
+ setRegister(a0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int64_General: {
+ int64_t ret = reinterpret_cast<Prototype_Int64_General>(nativeFn)(arg0);
+ setRegister(a0, ret);
+ break;
+ }
+ case js::jit::Args_Int64_GeneralInt64: {
+ int64_t ret = reinterpret_cast<Prototype_Int64_GeneralInt64>(nativeFn)(
+ arg0, arg1);
+ setRegister(a0, ret);
+ break;
+ }
+ default:
+ MOZ_CRASH("Unknown function type.");
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ setRegister(ra, saved_ra);
+ set_pc(getRegister(ra));
+ } else if ((instr->bits(31, 15) << 15 == op_break) && code == kWasmTrapCode) {
+ uint8_t* newPC;
+ if (wasm::HandleIllegalInstruction(registerState(), &newPC)) {
+ set_pc(int64_t(newPC));
+ return;
+ }
+ } else if ((instr->bits(31, 15) << 15 == op_break) && code <= kMaxStopCode &&
+ code != 6) {
+ if (isWatchpoint(code)) {
+ // printWatchpoint(code);
+ } else {
+ increaseStopCounter(code);
+ handleStop(code, instr);
+ }
+ } else {
+ // All remaining break_ codes, and all traps are handled here.
+ loong64Debugger dbg(this);
+ dbg.debug();
+ }
+}
+
+// Stop helper functions.
+bool Simulator::isWatchpoint(uint32_t code) {
+ return (code <= kMaxWatchpointCode);
+}
+
+void Simulator::printWatchpoint(uint32_t code) {
+ loong64Debugger dbg(this);
+ ++break_count_;
+ printf("\n---- break %d marker: %20" PRIi64 " (instr count: %20" PRIi64
+ ") ----\n",
+ code, break_count_, icount_);
+ dbg.printAllRegs(); // Print registers and continue running.
+}
+
+void Simulator::handleStop(uint32_t code, SimInstruction* instr) {
+ // Stop if it is enabled, otherwise go on jumping over the stop
+ // and the message address.
+ if (isEnabledStop(code)) {
+ loong64Debugger dbg(this);
+ dbg.stop(instr);
+ } else {
+ set_pc(get_pc() + 1 * SimInstruction::kInstrSize);
+ }
+}
+
+bool Simulator::isStopInstruction(SimInstruction* instr) {
+ int32_t opcode_hi15 = instr->bits(31, 17);
+ uint32_t code = static_cast<uint32_t>(instr->bits(14, 0));
+ return (opcode_hi15 == 0x15) && code > kMaxWatchpointCode &&
+ code <= kMaxStopCode;
+}
+
+bool Simulator::isEnabledStop(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ MOZ_ASSERT(code > kMaxWatchpointCode);
+ return !(watchedStops_[code].count_ & kStopDisabledBit);
+}
+
+void Simulator::enableStop(uint32_t code) {
+ if (!isEnabledStop(code)) {
+ watchedStops_[code].count_ &= ~kStopDisabledBit;
+ }
+}
+
+void Simulator::disableStop(uint32_t code) {
+ if (isEnabledStop(code)) {
+ watchedStops_[code].count_ |= kStopDisabledBit;
+ }
+}
+
+void Simulator::increaseStopCounter(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ if ((watchedStops_[code].count_ & ~(1 << 31)) == 0x7fffffff) {
+ printf(
+ "Stop counter for code %i has overflowed.\n"
+ "Enabling this code and reseting the counter to 0.\n",
+ code);
+ watchedStops_[code].count_ = 0;
+ enableStop(code);
+ } else {
+ watchedStops_[code].count_++;
+ }
+}
+
+// Print a stop status.
+void Simulator::printStopInfo(uint32_t code) {
+ if (code <= kMaxWatchpointCode) {
+ printf("That is a watchpoint, not a stop.\n");
+ return;
+ } else if (code > kMaxStopCode) {
+ printf("Code too large, only %u stops can be used\n", kMaxStopCode + 1);
+ return;
+ }
+ const char* state = isEnabledStop(code) ? "Enabled" : "Disabled";
+ int32_t count = watchedStops_[code].count_ & ~kStopDisabledBit;
+ // Don't print the state of unused breakpoints.
+ if (count != 0) {
+ if (watchedStops_[code].desc_) {
+ printf("stop %i - 0x%x: \t%s, \tcounter = %i, \t%s\n", code, code, state,
+ count, watchedStops_[code].desc_);
+ } else {
+ printf("stop %i - 0x%x: \t%s, \tcounter = %i\n", code, code, state,
+ count);
+ }
+ }
+}
+
+void Simulator::signalExceptions() {
+ for (int i = 1; i < kNumExceptions; i++) {
+ if (exceptions[i] != 0) {
+ MOZ_CRASH("Error: Exception raised.");
+ }
+ }
+}
+
+// ReverseBits(value) returns |value| in reverse bit order.
+template <typename T>
+T ReverseBits(T value) {
+ MOZ_ASSERT((sizeof(value) == 1) || (sizeof(value) == 2) ||
+ (sizeof(value) == 4) || (sizeof(value) == 8));
+ T result = 0;
+ for (unsigned i = 0; i < (sizeof(value) * 8); i++) {
+ result = (result << 1) | (value & 1);
+ value >>= 1;
+ }
+ return result;
+}
+
+// Min/Max template functions for Double and Single arguments.
+
+template <typename T>
+static T FPAbs(T a);
+
+template <>
+double FPAbs<double>(double a) {
+ return fabs(a);
+}
+
+template <>
+float FPAbs<float>(float a) {
+ return fabsf(a);
+}
+
+enum class MaxMinKind : int { kMin = 0, kMax = 1 };
+
+template <typename T>
+static bool FPUProcessNaNsAndZeros(T a, T b, MaxMinKind kind, T* result) {
+ if (std::isnan(a) && std::isnan(b)) {
+ *result = a;
+ } else if (std::isnan(a)) {
+ *result = b;
+ } else if (std::isnan(b)) {
+ *result = a;
+ } else if (b == a) {
+ // Handle -0.0 == 0.0 case.
+ // std::signbit() returns int 0 or 1 so subtracting MaxMinKind::kMax
+ // negates the result.
+ *result = std::signbit(b) - static_cast<int>(kind) ? b : a;
+ } else {
+ return false;
+ }
+ return true;
+}
+
+template <typename T>
+static T FPUMin(T a, T b) {
+ T result;
+ if (FPUProcessNaNsAndZeros(a, b, MaxMinKind::kMin, &result)) {
+ return result;
+ } else {
+ return b < a ? b : a;
+ }
+}
+
+template <typename T>
+static T FPUMax(T a, T b) {
+ T result;
+ if (FPUProcessNaNsAndZeros(a, b, MaxMinKind::kMax, &result)) {
+ return result;
+ } else {
+ return b > a ? b : a;
+ }
+}
+
+template <typename T>
+static T FPUMinA(T a, T b) {
+ T result;
+ if (!FPUProcessNaNsAndZeros(a, b, MaxMinKind::kMin, &result)) {
+ if (FPAbs(a) < FPAbs(b)) {
+ result = a;
+ } else if (FPAbs(b) < FPAbs(a)) {
+ result = b;
+ } else {
+ result = a < b ? a : b;
+ }
+ }
+ return result;
+}
+
+template <typename T>
+static T FPUMaxA(T a, T b) {
+ T result;
+ if (!FPUProcessNaNsAndZeros(a, b, MaxMinKind::kMin, &result)) {
+ if (FPAbs(a) > FPAbs(b)) {
+ result = a;
+ } else if (FPAbs(b) > FPAbs(a)) {
+ result = b;
+ } else {
+ result = a > b ? a : b;
+ }
+ }
+ return result;
+}
+
+enum class KeepSign : bool { no = false, yes };
+
+// Handle execution based on instruction types.
+// decodeTypeImmediate
+void Simulator::decodeTypeOp6(SimInstruction* instr) {
+ // Next pc.
+ int64_t next_pc = bad_ra;
+
+ // Used for memory instructions.
+ int64_t alu_out = 0;
+
+ // Branch instructions common part.
+ auto BranchAndLinkHelper = [this, &next_pc](SimInstruction* instr) {
+ int64_t current_pc = get_pc();
+ setRegister(ra, current_pc + SimInstruction::kInstrSize);
+ int32_t offs26_low16 =
+ static_cast<uint32_t>(instr->bits(25, 10) << 16) >> 16;
+ int32_t offs26_high10 = static_cast<int32_t>(instr->bits(9, 0) << 22) >> 6;
+ int32_t offs26 = offs26_low16 | offs26_high10;
+ next_pc = current_pc + (offs26 << 2);
+ set_pc(next_pc);
+ };
+
+ auto BranchOff16Helper = [this, &next_pc](SimInstruction* instr,
+ bool do_branch) {
+ int64_t current_pc = get_pc();
+ int32_t offs16 = static_cast<int32_t>(instr->bits(25, 10) << 16) >> 16;
+ int32_t offs = do_branch ? (offs16 << 2) : SimInstruction::kInstrSize;
+ next_pc = current_pc + offs;
+ set_pc(next_pc);
+ };
+
+ auto BranchOff21Helper = [this, &next_pc](SimInstruction* instr,
+ bool do_branch) {
+ int64_t current_pc = get_pc();
+ int32_t offs21_low16 =
+ static_cast<uint32_t>(instr->bits(25, 10) << 16) >> 16;
+ int32_t offs21_high5 = static_cast<int32_t>(instr->bits(4, 0) << 27) >> 11;
+ int32_t offs = offs21_low16 | offs21_high5;
+ offs = do_branch ? (offs << 2) : SimInstruction::kInstrSize;
+ next_pc = current_pc + offs;
+ set_pc(next_pc);
+ };
+
+ auto BranchOff26Helper = [this, &next_pc](SimInstruction* instr) {
+ int64_t current_pc = get_pc();
+ int32_t offs26_low16 =
+ static_cast<uint32_t>(instr->bits(25, 10) << 16) >> 16;
+ int32_t offs26_high10 = static_cast<int32_t>(instr->bits(9, 0) << 22) >> 6;
+ int32_t offs26 = offs26_low16 | offs26_high10;
+ next_pc = current_pc + (offs26 << 2);
+ set_pc(next_pc);
+ };
+
+ auto JumpOff16Helper = [this, &next_pc](SimInstruction* instr) {
+ int32_t offs16 = static_cast<int32_t>(instr->bits(25, 10) << 16) >> 16;
+ setRegister(rd_reg(instr), get_pc() + SimInstruction::kInstrSize);
+ next_pc = rj(instr) + (offs16 << 2);
+ set_pc(next_pc);
+ };
+
+ switch (instr->bits(31, 26) << 26) {
+ case op_addu16i_d: {
+ int32_t si16_upper = static_cast<int32_t>(si16(instr)) << 16;
+ alu_out = static_cast<int64_t>(si16_upper) + rj(instr);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_beqz: {
+ BranchOff21Helper(instr, rj(instr) == 0);
+ break;
+ }
+ case op_bnez: {
+ BranchOff21Helper(instr, rj(instr) != 0);
+ break;
+ }
+ case op_bcz: {
+ if (instr->bits(9, 8) == 0b00) {
+ // BCEQZ
+ BranchOff21Helper(instr, cj(instr) == false);
+ } else if (instr->bits(9, 8) == 0b01) {
+ // BCNEZ
+ BranchOff21Helper(instr, cj(instr) == true);
+ } else {
+ UNREACHABLE();
+ }
+ break;
+ }
+ case op_jirl: {
+ JumpOff16Helper(instr);
+ break;
+ }
+ case op_b: {
+ BranchOff26Helper(instr);
+ break;
+ }
+ case op_bl: {
+ BranchAndLinkHelper(instr);
+ break;
+ }
+ case op_beq: {
+ BranchOff16Helper(instr, rj(instr) == rd(instr));
+ break;
+ }
+ case op_bne: {
+ BranchOff16Helper(instr, rj(instr) != rd(instr));
+ break;
+ }
+ case op_blt: {
+ BranchOff16Helper(instr, rj(instr) < rd(instr));
+ break;
+ }
+ case op_bge: {
+ BranchOff16Helper(instr, rj(instr) >= rd(instr));
+ break;
+ }
+ case op_bltu: {
+ BranchOff16Helper(instr, rj_u(instr) < rd_u(instr));
+ break;
+ }
+ case op_bgeu: {
+ BranchOff16Helper(instr, rj_u(instr) >= rd_u(instr));
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp7(SimInstruction* instr) {
+ int64_t alu_out;
+
+ switch (instr->bits(31, 25) << 25) {
+ case op_lu12i_w: {
+ int32_t si20_upper = static_cast<int32_t>(si20(instr) << 12);
+ setRegister(rd_reg(instr), static_cast<int64_t>(si20_upper));
+ break;
+ }
+ case op_lu32i_d: {
+ int32_t si20_signExtend = static_cast<int32_t>(si20(instr) << 12) >> 12;
+ int64_t lower_32bit_mask = 0xFFFFFFFF;
+ alu_out = (static_cast<int64_t>(si20_signExtend) << 32) |
+ (rd(instr) & lower_32bit_mask);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_pcaddi: {
+ int32_t si20_signExtend = static_cast<int32_t>(si20(instr) << 12) >> 10;
+ int64_t current_pc = get_pc();
+ alu_out = static_cast<int64_t>(si20_signExtend) + current_pc;
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_pcalau12i: {
+ int32_t si20_signExtend = static_cast<int32_t>(si20(instr) << 12);
+ int64_t current_pc = get_pc();
+ int64_t clear_lower12bit_mask = 0xFFFFFFFFFFFFF000;
+ alu_out = static_cast<int64_t>(si20_signExtend) + current_pc;
+ setRegister(rd_reg(instr), alu_out & clear_lower12bit_mask);
+ break;
+ }
+ case op_pcaddu12i: {
+ int32_t si20_signExtend = static_cast<int32_t>(si20(instr) << 12);
+ int64_t current_pc = get_pc();
+ alu_out = static_cast<int64_t>(si20_signExtend) + current_pc;
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_pcaddu18i: {
+ int64_t si20_signExtend = (static_cast<int64_t>(si20(instr)) << 44) >> 26;
+ int64_t current_pc = get_pc();
+ alu_out = si20_signExtend + current_pc;
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp8(SimInstruction* instr) {
+ int64_t addr = 0x0;
+ int64_t si14_se = (static_cast<int64_t>(si14(instr)) << 50) >> 48;
+
+ switch (instr->bits(31, 24) << 24) {
+ case op_ldptr_w: {
+ setRegister(rd_reg(instr), readW(rj(instr) + si14_se, instr));
+ break;
+ }
+ case op_stptr_w: {
+ writeW(rj(instr) + si14_se, static_cast<int32_t>(rd(instr)), instr);
+ break;
+ }
+ case op_ldptr_d: {
+ setRegister(rd_reg(instr), readDW(rj(instr) + si14_se, instr));
+ break;
+ }
+ case op_stptr_d: {
+ writeDW(rj(instr) + si14_se, rd(instr), instr);
+ break;
+ }
+ case op_ll_w: {
+ addr = si14_se + rj(instr);
+ setRegister(rd_reg(instr), loadLinkedW(addr, instr));
+ break;
+ }
+ case op_sc_w: {
+ addr = si14_se + rj(instr);
+ setRegister(
+ rd_reg(instr),
+ storeConditionalW(addr, static_cast<int32_t>(rd(instr)), instr));
+ break;
+ }
+ case op_ll_d: {
+ addr = si14_se + rj(instr);
+ setRegister(rd_reg(instr), loadLinkedD(addr, instr));
+ break;
+ }
+ case op_sc_d: {
+ addr = si14_se + rj(instr);
+ setRegister(rd_reg(instr), storeConditionalD(addr, rd(instr), instr));
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp10(SimInstruction* instr) {
+ int64_t alu_out = 0x0;
+ int64_t si12_se = (static_cast<int64_t>(si12(instr)) << 52) >> 52;
+ uint64_t si12_ze = (static_cast<uint64_t>(ui12(instr)) << 52) >> 52;
+
+ switch (instr->bits(31, 22) << 22) {
+ case op_bstrins_d: {
+ uint8_t lsbd_ = lsbd(instr);
+ uint8_t msbd_ = msbd(instr);
+ MOZ_ASSERT(lsbd_ <= msbd_);
+ uint8_t size = msbd_ - lsbd_ + 1;
+ if (size < 64) {
+ uint64_t mask = (1ULL << size) - 1;
+ alu_out =
+ (rd_u(instr) & ~(mask << lsbd_)) | ((rj_u(instr) & mask) << lsbd_);
+ setRegister(rd_reg(instr), alu_out);
+ } else if (size == 64) {
+ setRegister(rd_reg(instr), rj(instr));
+ }
+ break;
+ }
+ case op_bstrpick_d: {
+ uint8_t lsbd_ = lsbd(instr);
+ uint8_t msbd_ = msbd(instr);
+ MOZ_ASSERT(lsbd_ <= msbd_);
+ uint8_t size = msbd_ - lsbd_ + 1;
+ if (size < 64) {
+ uint64_t mask = (1ULL << size) - 1;
+ alu_out = (rj_u(instr) & (mask << lsbd_)) >> lsbd_;
+ setRegister(rd_reg(instr), alu_out);
+ } else if (size == 64) {
+ setRegister(rd_reg(instr), rj(instr));
+ }
+ break;
+ }
+ case op_slti: {
+ setRegister(rd_reg(instr), rj(instr) < si12_se ? 1 : 0);
+ break;
+ }
+ case op_sltui: {
+ setRegister(rd_reg(instr),
+ rj_u(instr) < static_cast<uint64_t>(si12_se) ? 1 : 0);
+ break;
+ }
+ case op_addi_w: {
+ int32_t alu32_out =
+ static_cast<int32_t>(rj(instr)) + static_cast<int32_t>(si12_se);
+ setRegister(rd_reg(instr), alu32_out);
+ break;
+ }
+ case op_addi_d: {
+ setRegister(rd_reg(instr), rj(instr) + si12_se);
+ break;
+ }
+ case op_lu52i_d: {
+ int64_t si12_se = static_cast<int64_t>(si12(instr)) << 52;
+ uint64_t mask = (1ULL << 52) - 1;
+ alu_out = si12_se + (rj(instr) & mask);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_andi: {
+ setRegister(rd_reg(instr), rj(instr) & si12_ze);
+ break;
+ }
+ case op_ori: {
+ setRegister(rd_reg(instr), rj_u(instr) | si12_ze);
+ break;
+ }
+ case op_xori: {
+ setRegister(rd_reg(instr), rj_u(instr) ^ si12_ze);
+ break;
+ }
+ case op_ld_b: {
+ setRegister(rd_reg(instr), readB(rj(instr) + si12_se));
+ break;
+ }
+ case op_ld_h: {
+ setRegister(rd_reg(instr), readH(rj(instr) + si12_se, instr));
+ break;
+ }
+ case op_ld_w: {
+ setRegister(rd_reg(instr), readW(rj(instr) + si12_se, instr));
+ break;
+ }
+ case op_ld_d: {
+ setRegister(rd_reg(instr), readDW(rj(instr) + si12_se, instr));
+ break;
+ }
+ case op_st_b: {
+ writeB(rj(instr) + si12_se, static_cast<int8_t>(rd(instr)));
+ break;
+ }
+ case op_st_h: {
+ writeH(rj(instr) + si12_se, static_cast<int16_t>(rd(instr)), instr);
+ break;
+ }
+ case op_st_w: {
+ writeW(rj(instr) + si12_se, static_cast<int32_t>(rd(instr)), instr);
+ break;
+ }
+ case op_st_d: {
+ writeDW(rj(instr) + si12_se, rd(instr), instr);
+ break;
+ }
+ case op_ld_bu: {
+ setRegister(rd_reg(instr), readBU(rj(instr) + si12_se));
+ break;
+ }
+ case op_ld_hu: {
+ setRegister(rd_reg(instr), readHU(rj(instr) + si12_se, instr));
+ break;
+ }
+ case op_ld_wu: {
+ setRegister(rd_reg(instr), readWU(rj(instr) + si12_se, instr));
+ break;
+ }
+ case op_fld_s: {
+ setFpuRegister(fd_reg(instr), kFPUInvalidResult); // Trash upper 32 bits.
+ setFpuRegisterWord(fd_reg(instr), readW(rj(instr) + si12_se, instr));
+ break;
+ }
+ case op_fst_s: {
+ int32_t alu_out_32 = static_cast<int32_t>(getFpuRegister(fd_reg(instr)));
+ writeW(rj(instr) + si12_se, alu_out_32, instr);
+ break;
+ }
+ case op_fld_d: {
+ setFpuRegisterDouble(fd_reg(instr), readD(rj(instr) + si12_se, instr));
+ break;
+ }
+ case op_fst_d: {
+ writeD(rj(instr) + si12_se, getFpuRegisterDouble(fd_reg(instr)), instr);
+ break;
+ }
+ case op_preld:
+ UNIMPLEMENTED();
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp11(SimInstruction* instr) {
+ int64_t alu_out = 0x0;
+
+ switch (instr->bits(31, 21) << 21) {
+ case op_bstr_w: {
+ MOZ_ASSERT(instr->bit(21) == 1);
+ uint8_t lsbw_ = lsbw(instr);
+ uint8_t msbw_ = msbw(instr);
+ MOZ_ASSERT(lsbw_ <= msbw_);
+ uint8_t size = msbw_ - lsbw_ + 1;
+ uint64_t mask = (1ULL << size) - 1;
+ if (instr->bit(15) == 0) {
+ // BSTRINS_W
+ alu_out = static_cast<int32_t>((rd_u(instr) & ~(mask << lsbw_)) |
+ ((rj_u(instr) & mask) << lsbw_));
+ } else {
+ // BSTRPICK_W
+ alu_out =
+ static_cast<int32_t>((rj_u(instr) & (mask << lsbw_)) >> lsbw_);
+ }
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp12(SimInstruction* instr) {
+ switch (instr->bits(31, 20) << 20) {
+ case op_fmadd_s: {
+ setFpuRegisterFloat(
+ fd_reg(instr),
+ std::fma(fj_float(instr), fk_float(instr), fa_float(instr)));
+ break;
+ }
+ case op_fmadd_d: {
+ setFpuRegisterDouble(
+ fd_reg(instr),
+ std::fma(fj_double(instr), fk_double(instr), fa_double(instr)));
+ break;
+ }
+ case op_fmsub_s: {
+ setFpuRegisterFloat(
+ fd_reg(instr),
+ std::fma(-fj_float(instr), fk_float(instr), fa_float(instr)));
+ break;
+ }
+ case op_fmsub_d: {
+ setFpuRegisterDouble(
+ fd_reg(instr),
+ std::fma(-fj_double(instr), fk_double(instr), fa_double(instr)));
+ break;
+ }
+ case op_fnmadd_s: {
+ setFpuRegisterFloat(
+ fd_reg(instr),
+ std::fma(-fj_float(instr), fk_float(instr), -fa_float(instr)));
+ break;
+ }
+ case op_fnmadd_d: {
+ setFpuRegisterDouble(
+ fd_reg(instr),
+ std::fma(-fj_double(instr), fk_double(instr), -fa_double(instr)));
+ break;
+ }
+ case op_fnmsub_s: {
+ setFpuRegisterFloat(
+ fd_reg(instr),
+ std::fma(fj_float(instr), fk_float(instr), -fa_float(instr)));
+ break;
+ }
+ case op_fnmsub_d: {
+ setFpuRegisterDouble(
+ fd_reg(instr),
+ std::fma(fj_double(instr), fk_double(instr), -fa_double(instr)));
+ break;
+ }
+ case op_fcmp_cond_s: {
+ MOZ_ASSERT(instr->bits(4, 3) == 0);
+ float fj = fj_float(instr);
+ float fk = fk_float(instr);
+ switch (cond(instr)) {
+ case AssemblerLOONG64::CAF: {
+ setCFRegister(cd_reg(instr), false);
+ break;
+ }
+ case AssemblerLOONG64::CUN: {
+ setCFRegister(cd_reg(instr), std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CEQ: {
+ setCFRegister(cd_reg(instr), fj == fk);
+ break;
+ }
+ case AssemblerLOONG64::CUEQ: {
+ setCFRegister(cd_reg(instr),
+ (fj == fk) || std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CLT: {
+ setCFRegister(cd_reg(instr), fj < fk);
+ break;
+ }
+ case AssemblerLOONG64::CULT: {
+ setCFRegister(cd_reg(instr),
+ (fj < fk) || std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CLE: {
+ setCFRegister(cd_reg(instr), fj <= fk);
+ break;
+ }
+ case AssemblerLOONG64::CULE: {
+ setCFRegister(cd_reg(instr),
+ (fj <= fk) || std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CNE: {
+ setCFRegister(cd_reg(instr), (fj < fk) || (fj > fk));
+ break;
+ }
+ case AssemblerLOONG64::COR: {
+ setCFRegister(cd_reg(instr), !std::isnan(fj) && !std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CUNE: {
+ setCFRegister(cd_reg(instr), (fj < fk) || (fj > fk) ||
+ std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::SAF:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SUN:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SEQ:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SUEQ:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SLT:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SULT:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SLE:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SULE:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SNE:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SOR:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SUNE:
+ UNIMPLEMENTED();
+ break;
+ default:
+ UNREACHABLE();
+ }
+ break;
+ }
+ case op_fcmp_cond_d: {
+ MOZ_ASSERT(instr->bits(4, 3) == 0);
+ double fj = fj_double(instr);
+ double fk = fk_double(instr);
+ switch (cond(instr)) {
+ case AssemblerLOONG64::CAF: {
+ setCFRegister(cd_reg(instr), false);
+ break;
+ }
+ case AssemblerLOONG64::CUN: {
+ setCFRegister(cd_reg(instr), std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CEQ: {
+ setCFRegister(cd_reg(instr), fj == fk);
+ break;
+ }
+ case AssemblerLOONG64::CUEQ: {
+ setCFRegister(cd_reg(instr),
+ (fj == fk) || std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CLT: {
+ setCFRegister(cd_reg(instr), fj < fk);
+ break;
+ }
+ case AssemblerLOONG64::CULT: {
+ setCFRegister(cd_reg(instr),
+ (fj < fk) || std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CLE: {
+ setCFRegister(cd_reg(instr), fj <= fk);
+ break;
+ }
+ case AssemblerLOONG64::CULE: {
+ setCFRegister(cd_reg(instr),
+ (fj <= fk) || std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CNE: {
+ setCFRegister(cd_reg(instr), (fj < fk) || (fj > fk));
+ break;
+ }
+ case AssemblerLOONG64::COR: {
+ setCFRegister(cd_reg(instr), !std::isnan(fj) && !std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::CUNE: {
+ setCFRegister(cd_reg(instr),
+ (fj != fk) || std::isnan(fj) || std::isnan(fk));
+ break;
+ }
+ case AssemblerLOONG64::SAF:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SUN:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SEQ:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SUEQ:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SLT:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SULT:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SLE:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SULE:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SNE:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SOR:
+ UNIMPLEMENTED();
+ break;
+ case AssemblerLOONG64::SUNE:
+ UNIMPLEMENTED();
+ break;
+ default:
+ UNREACHABLE();
+ }
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp14(SimInstruction* instr) {
+ int64_t alu_out = 0x0;
+
+ switch (instr->bits(31, 18) << 18) {
+ case op_bytepick_d: {
+ uint8_t sa = sa3(instr) * 8;
+ if (sa == 0) {
+ alu_out = rk(instr);
+ } else {
+ int64_t mask = (1ULL << 63) >> (sa - 1);
+ int64_t rk_hi = (rk(instr) & (~mask)) << sa;
+ int64_t rj_lo = (rj(instr) & mask) >> (64 - sa);
+ alu_out = rk_hi | rj_lo;
+ }
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_fsel: {
+ MOZ_ASSERT(instr->bits(19, 18) == 0);
+ if (ca(instr) == 0) {
+ setFpuRegisterDouble(fd_reg(instr), fj_double(instr));
+ } else {
+ setFpuRegisterDouble(fd_reg(instr), fk_double(instr));
+ }
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp15(SimInstruction* instr) {
+ int64_t alu_out = 0x0;
+ int32_t alu32_out = 0x0;
+
+ switch (instr->bits(31, 17) << 17) {
+ case op_bytepick_w: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ uint8_t sa = sa2(instr) * 8;
+ if (sa == 0) {
+ alu32_out = static_cast<int32_t>(rk(instr));
+ } else {
+ int32_t mask = (1 << 31) >> (sa - 1);
+ int32_t rk_hi = (static_cast<int32_t>(rk(instr)) & (~mask)) << sa;
+ int32_t rj_lo = (static_cast<uint32_t>(rj(instr)) & mask) >> (32 - sa);
+ alu32_out = rk_hi | rj_lo;
+ }
+ setRegister(rd_reg(instr), static_cast<int64_t>(alu32_out));
+ break;
+ }
+ case op_alsl_w: {
+ uint8_t sa = sa2(instr) + 1;
+ alu32_out = (static_cast<int32_t>(rj(instr)) << sa) +
+ static_cast<int32_t>(rk(instr));
+ setRegister(rd_reg(instr), alu32_out);
+ break;
+ }
+ case op_alsl_wu: {
+ uint8_t sa = sa2(instr) + 1;
+ alu32_out = (static_cast<int32_t>(rj(instr)) << sa) +
+ static_cast<int32_t>(rk(instr));
+ setRegister(rd_reg(instr), static_cast<uint32_t>(alu32_out));
+ break;
+ }
+ case op_alsl_d: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ uint8_t sa = sa2(instr) + 1;
+ alu_out = (rj(instr) << sa) + rk(instr);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp16(SimInstruction* instr) {
+ int64_t alu_out;
+ switch (instr->bits(31, 16) << 16) {
+ case op_slli_d: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ MOZ_ASSERT(instr->bits(17, 16) == 0b01);
+ setRegister(rd_reg(instr), rj(instr) << ui6(instr));
+ break;
+ }
+ case op_srli_d: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ setRegister(rd_reg(instr), rj_u(instr) >> ui6(instr));
+ break;
+ }
+ case op_srai_d: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ setRegister(rd_reg(instr), rj(instr) >> ui6(instr));
+ break;
+ }
+ case op_rotri_d: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ MOZ_ASSERT(instr->bits(17, 16) == 0b01);
+ alu_out = static_cast<int64_t>(RotateRight64(rj_u(instr), ui6(instr)));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp17(SimInstruction* instr) {
+ int64_t alu_out;
+ int32_t alu32_out;
+
+ switch (instr->bits(31, 15) << 15) {
+ case op_slli_w: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ MOZ_ASSERT(instr->bits(17, 15) == 0b001);
+ alu32_out = static_cast<int32_t>(rj(instr)) << ui5(instr);
+ setRegister(rd_reg(instr), static_cast<int64_t>(alu32_out));
+ break;
+ }
+ case op_srai_w: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ MOZ_ASSERT(instr->bits(17, 15) == 0b001);
+ alu32_out = static_cast<int32_t>(rj(instr)) >> ui5(instr);
+ setRegister(rd_reg(instr), static_cast<int64_t>(alu32_out));
+ break;
+ }
+ case op_rotri_w: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ MOZ_ASSERT(instr->bits(17, 15) == 0b001);
+ alu32_out = static_cast<int32_t>(
+ RotateRight32(static_cast<const uint32_t>(rj_u(instr)),
+ static_cast<const uint32_t>(ui5(instr))));
+ setRegister(rd_reg(instr), static_cast<int64_t>(alu32_out));
+ break;
+ }
+ case op_srli_w: {
+ MOZ_ASSERT(instr->bit(17) == 0);
+ MOZ_ASSERT(instr->bits(17, 15) == 0b001);
+ alu32_out = static_cast<uint32_t>(rj(instr)) >> ui5(instr);
+ setRegister(rd_reg(instr), static_cast<int64_t>(alu32_out));
+ break;
+ }
+ case op_add_w: {
+ int32_t alu32_out = static_cast<int32_t>(rj(instr) + rk(instr));
+ // Sign-extend result of 32bit operation into 64bit register.
+ setRegister(rd_reg(instr), static_cast<int64_t>(alu32_out));
+ break;
+ }
+ case op_add_d:
+ setRegister(rd_reg(instr), rj(instr) + rk(instr));
+ break;
+ case op_sub_w: {
+ int32_t alu32_out = static_cast<int32_t>(rj(instr) - rk(instr));
+ // Sign-extend result of 32bit operation into 64bit register.
+ setRegister(rd_reg(instr), static_cast<int64_t>(alu32_out));
+ break;
+ }
+ case op_sub_d:
+ setRegister(rd_reg(instr), rj(instr) - rk(instr));
+ break;
+ case op_slt:
+ setRegister(rd_reg(instr), rj(instr) < rk(instr) ? 1 : 0);
+ break;
+ case op_sltu:
+ setRegister(rd_reg(instr), rj_u(instr) < rk_u(instr) ? 1 : 0);
+ break;
+ case op_maskeqz:
+ setRegister(rd_reg(instr), rk(instr) == 0 ? 0 : rj(instr));
+ break;
+ case op_masknez:
+ setRegister(rd_reg(instr), rk(instr) != 0 ? 0 : rj(instr));
+ break;
+ case op_nor:
+ setRegister(rd_reg(instr), ~(rj(instr) | rk(instr)));
+ break;
+ case op_and:
+ setRegister(rd_reg(instr), rj(instr) & rk(instr));
+ break;
+ case op_or:
+ setRegister(rd_reg(instr), rj(instr) | rk(instr));
+ break;
+ case op_xor:
+ setRegister(rd_reg(instr), rj(instr) ^ rk(instr));
+ break;
+ case op_orn:
+ setRegister(rd_reg(instr), rj(instr) | (~rk(instr)));
+ break;
+ case op_andn:
+ setRegister(rd_reg(instr), rj(instr) & (~rk(instr)));
+ break;
+ case op_sll_w:
+ setRegister(rd_reg(instr), (int32_t)rj(instr) << (rk_u(instr) % 32));
+ break;
+ case op_srl_w: {
+ alu_out =
+ static_cast<int32_t>((uint32_t)rj_u(instr) >> (rk_u(instr) % 32));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_sra_w:
+ setRegister(rd_reg(instr), (int32_t)rj(instr) >> (rk_u(instr) % 32));
+ break;
+ case op_sll_d:
+ setRegister(rd_reg(instr), rj(instr) << (rk_u(instr) % 64));
+ break;
+ case op_srl_d: {
+ alu_out = static_cast<int64_t>(rj_u(instr) >> (rk_u(instr) % 64));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_sra_d:
+ setRegister(rd_reg(instr), rj(instr) >> (rk_u(instr) % 64));
+ break;
+ case op_rotr_w: {
+ alu_out = static_cast<int32_t>(
+ RotateRight32(static_cast<const uint32_t>(rj_u(instr)),
+ static_cast<const uint32_t>(rk_u(instr) % 32)));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_rotr_d: {
+ alu_out = static_cast<int64_t>(
+ RotateRight64((rj_u(instr)), (rk_u(instr) % 64)));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_mul_w: {
+ alu_out =
+ static_cast<int32_t>(rj(instr)) * static_cast<int32_t>(rk(instr));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_mulh_w: {
+ int32_t rj_lo = static_cast<int32_t>(rj(instr));
+ int32_t rk_lo = static_cast<int32_t>(rk(instr));
+ alu_out = static_cast<int64_t>(rj_lo) * static_cast<int64_t>(rk_lo);
+ setRegister(rd_reg(instr), alu_out >> 32);
+ break;
+ }
+ case op_mulh_wu: {
+ uint32_t rj_lo = static_cast<uint32_t>(rj_u(instr));
+ uint32_t rk_lo = static_cast<uint32_t>(rk_u(instr));
+ alu_out = static_cast<uint64_t>(rj_lo) * static_cast<uint64_t>(rk_lo);
+ setRegister(rd_reg(instr), alu_out >> 32);
+ break;
+ }
+ case op_mul_d:
+ setRegister(rd_reg(instr), rj(instr) * rk(instr));
+ break;
+ case op_mulh_d:
+ setRegister(rd_reg(instr), MultiplyHighSigned(rj(instr), rk(instr)));
+ break;
+ case op_mulh_du:
+ setRegister(rd_reg(instr),
+ MultiplyHighUnsigned(rj_u(instr), rk_u(instr)));
+ break;
+ case op_mulw_d_w: {
+ int64_t rj_i32 = static_cast<int32_t>(rj(instr));
+ int64_t rk_i32 = static_cast<int32_t>(rk(instr));
+ setRegister(rd_reg(instr), rj_i32 * rk_i32);
+ break;
+ }
+ case op_mulw_d_wu: {
+ uint64_t rj_u32 = static_cast<uint32_t>(rj_u(instr));
+ uint64_t rk_u32 = static_cast<uint32_t>(rk_u(instr));
+ setRegister(rd_reg(instr), rj_u32 * rk_u32);
+ break;
+ }
+ case op_div_w: {
+ int32_t rj_i32 = static_cast<int32_t>(rj(instr));
+ int32_t rk_i32 = static_cast<int32_t>(rk(instr));
+ if (rj_i32 == INT_MIN && rk_i32 == -1) {
+ setRegister(rd_reg(instr), INT_MIN);
+ } else if (rk_i32 != 0) {
+ setRegister(rd_reg(instr), rj_i32 / rk_i32);
+ }
+ break;
+ }
+ case op_mod_w: {
+ int32_t rj_i32 = static_cast<int32_t>(rj(instr));
+ int32_t rk_i32 = static_cast<int32_t>(rk(instr));
+ if (rj_i32 == INT_MIN && rk_i32 == -1) {
+ setRegister(rd_reg(instr), 0);
+ } else if (rk_i32 != 0) {
+ setRegister(rd_reg(instr), rj_i32 % rk_i32);
+ }
+ break;
+ }
+ case op_div_wu: {
+ uint32_t rj_u32 = static_cast<uint32_t>(rj(instr));
+ uint32_t rk_u32 = static_cast<uint32_t>(rk(instr));
+ if (rk_u32 != 0) {
+ setRegister(rd_reg(instr), static_cast<int32_t>(rj_u32 / rk_u32));
+ }
+ break;
+ }
+ case op_mod_wu: {
+ uint32_t rj_u32 = static_cast<uint32_t>(rj(instr));
+ uint32_t rk_u32 = static_cast<uint32_t>(rk(instr));
+ if (rk_u32 != 0) {
+ setRegister(rd_reg(instr), static_cast<int32_t>(rj_u32 % rk_u32));
+ }
+ break;
+ }
+ case op_div_d: {
+ if (rj(instr) == INT64_MIN && rk(instr) == -1) {
+ setRegister(rd_reg(instr), INT64_MIN);
+ } else if (rk(instr) != 0) {
+ setRegister(rd_reg(instr), rj(instr) / rk(instr));
+ }
+ break;
+ }
+ case op_mod_d: {
+ if (rj(instr) == LONG_MIN && rk(instr) == -1) {
+ setRegister(rd_reg(instr), 0);
+ } else if (rk(instr) != 0) {
+ setRegister(rd_reg(instr), rj(instr) % rk(instr));
+ }
+ break;
+ }
+ case op_div_du: {
+ if (rk_u(instr) != 0) {
+ setRegister(rd_reg(instr),
+ static_cast<int64_t>(rj_u(instr) / rk_u(instr)));
+ }
+ break;
+ }
+ case op_mod_du: {
+ if (rk_u(instr) != 0) {
+ setRegister(rd_reg(instr),
+ static_cast<int64_t>(rj_u(instr) % rk_u(instr)));
+ }
+ break;
+ }
+ case op_break:
+ softwareInterrupt(instr);
+ break;
+ case op_fadd_s: {
+ setFpuRegisterFloat(fd_reg(instr), fj_float(instr) + fk_float(instr));
+ break;
+ }
+ case op_fadd_d: {
+ setFpuRegisterDouble(fd_reg(instr), fj_double(instr) + fk_double(instr));
+ break;
+ }
+ case op_fsub_s: {
+ setFpuRegisterFloat(fd_reg(instr), fj_float(instr) - fk_float(instr));
+ break;
+ }
+ case op_fsub_d: {
+ setFpuRegisterDouble(fd_reg(instr), fj_double(instr) - fk_double(instr));
+ break;
+ }
+ case op_fmul_s: {
+ setFpuRegisterFloat(fd_reg(instr), fj_float(instr) * fk_float(instr));
+ break;
+ }
+ case op_fmul_d: {
+ setFpuRegisterDouble(fd_reg(instr), fj_double(instr) * fk_double(instr));
+ break;
+ }
+ case op_fdiv_s: {
+ setFpuRegisterFloat(fd_reg(instr), fj_float(instr) / fk_float(instr));
+ break;
+ }
+
+ case op_fdiv_d: {
+ setFpuRegisterDouble(fd_reg(instr), fj_double(instr) / fk_double(instr));
+ break;
+ }
+ case op_fmax_s: {
+ setFpuRegisterFloat(fd_reg(instr),
+ FPUMax(fk_float(instr), fj_float(instr)));
+ break;
+ }
+ case op_fmax_d: {
+ setFpuRegisterDouble(fd_reg(instr),
+ FPUMax(fk_double(instr), fj_double(instr)));
+ break;
+ }
+ case op_fmin_s: {
+ setFpuRegisterFloat(fd_reg(instr),
+ FPUMin(fk_float(instr), fj_float(instr)));
+ break;
+ }
+ case op_fmin_d: {
+ setFpuRegisterDouble(fd_reg(instr),
+ FPUMin(fk_double(instr), fj_double(instr)));
+ break;
+ }
+ case op_fmaxa_s: {
+ setFpuRegisterFloat(fd_reg(instr),
+ FPUMaxA(fk_float(instr), fj_float(instr)));
+ break;
+ }
+ case op_fmaxa_d: {
+ setFpuRegisterDouble(fd_reg(instr),
+ FPUMaxA(fk_double(instr), fj_double(instr)));
+ break;
+ }
+ case op_fmina_s: {
+ setFpuRegisterFloat(fd_reg(instr),
+ FPUMinA(fk_float(instr), fj_float(instr)));
+ break;
+ }
+ case op_fmina_d: {
+ setFpuRegisterDouble(fd_reg(instr),
+ FPUMinA(fk_double(instr), fj_double(instr)));
+ break;
+ }
+ case op_ldx_b:
+ setRegister(rd_reg(instr), readB(rj(instr) + rk(instr)));
+ break;
+ case op_ldx_h:
+ setRegister(rd_reg(instr), readH(rj(instr) + rk(instr), instr));
+ break;
+ case op_ldx_w:
+ setRegister(rd_reg(instr), readW(rj(instr) + rk(instr), instr));
+ break;
+ case op_ldx_d:
+ setRegister(rd_reg(instr), readDW(rj(instr) + rk(instr), instr));
+ break;
+ case op_stx_b:
+ writeB(rj(instr) + rk(instr), static_cast<int8_t>(rd(instr)));
+ break;
+ case op_stx_h:
+ writeH(rj(instr) + rk(instr), static_cast<int16_t>(rd(instr)), instr);
+ break;
+ case op_stx_w:
+ writeW(rj(instr) + rk(instr), static_cast<int32_t>(rd(instr)), instr);
+ break;
+ case op_stx_d:
+ writeDW(rj(instr) + rk(instr), rd(instr), instr);
+ break;
+ case op_ldx_bu:
+ setRegister(rd_reg(instr), readBU(rj(instr) + rk(instr)));
+ break;
+ case op_ldx_hu:
+ setRegister(rd_reg(instr), readHU(rj(instr) + rk(instr), instr));
+ break;
+ case op_ldx_wu:
+ setRegister(rd_reg(instr), readWU(rj(instr) + rk(instr), instr));
+ break;
+ case op_fldx_s:
+ setFpuRegister(fd_reg(instr), kFPUInvalidResult); // Trash upper 32 bits.
+ setFpuRegisterWord(fd_reg(instr), readW(rj(instr) + rk(instr), instr));
+ break;
+ case op_fldx_d:
+ setFpuRegister(fd_reg(instr), kFPUInvalidResult); // Trash upper 32 bits.
+ setFpuRegisterDouble(fd_reg(instr), readD(rj(instr) + rk(instr), instr));
+ break;
+ case op_fstx_s: {
+ int32_t alu_out_32 = static_cast<int32_t>(getFpuRegister(fd_reg(instr)));
+ writeW(rj(instr) + rk(instr), alu_out_32, instr);
+ break;
+ }
+ case op_fstx_d: {
+ writeD(rj(instr) + rk(instr), getFpuRegisterDouble(fd_reg(instr)), instr);
+ break;
+ }
+ case op_amswap_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amswap_d:
+ UNIMPLEMENTED();
+ break;
+ case op_amadd_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amadd_d:
+ UNIMPLEMENTED();
+ break;
+ case op_amand_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amand_d:
+ UNIMPLEMENTED();
+ break;
+ case op_amor_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amor_d:
+ UNIMPLEMENTED();
+ break;
+ case op_amxor_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amxor_d:
+ UNIMPLEMENTED();
+ break;
+ case op_ammax_w:
+ UNIMPLEMENTED();
+ break;
+ case op_ammax_d:
+ UNIMPLEMENTED();
+ break;
+ case op_ammin_w:
+ UNIMPLEMENTED();
+ break;
+ case op_ammin_d:
+ UNIMPLEMENTED();
+ break;
+ case op_ammax_wu:
+ UNIMPLEMENTED();
+ break;
+ case op_ammax_du:
+ UNIMPLEMENTED();
+ break;
+ case op_ammin_wu:
+ UNIMPLEMENTED();
+ break;
+ case op_ammin_du:
+ UNIMPLEMENTED();
+ break;
+ case op_amswap_db_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amswap_db_d:
+ UNIMPLEMENTED();
+ break;
+ case op_amadd_db_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amadd_db_d:
+ UNIMPLEMENTED();
+ break;
+ case op_amand_db_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amand_db_d:
+ UNIMPLEMENTED();
+ break;
+ case op_amor_db_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amor_db_d:
+ UNIMPLEMENTED();
+ break;
+ case op_amxor_db_w:
+ UNIMPLEMENTED();
+ break;
+ case op_amxor_db_d:
+ UNIMPLEMENTED();
+ break;
+ case op_ammax_db_w:
+ UNIMPLEMENTED();
+ break;
+ case op_ammax_db_d:
+ UNIMPLEMENTED();
+ break;
+ case op_ammin_db_w:
+ UNIMPLEMENTED();
+ break;
+ case op_ammin_db_d:
+ UNIMPLEMENTED();
+ break;
+ case op_ammax_db_wu:
+ UNIMPLEMENTED();
+ break;
+ case op_ammax_db_du:
+ UNIMPLEMENTED();
+ break;
+ case op_ammin_db_wu:
+ UNIMPLEMENTED();
+ break;
+ case op_ammin_db_du:
+ UNIMPLEMENTED();
+ break;
+ case op_dbar:
+ // TODO(loong64): dbar simulation
+ break;
+ case op_ibar:
+ UNIMPLEMENTED();
+ break;
+ case op_fcopysign_s:
+ UNIMPLEMENTED();
+ break;
+ case op_fcopysign_d:
+ UNIMPLEMENTED();
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp22(SimInstruction* instr) {
+ int64_t alu_out;
+
+ switch (instr->bits(31, 10) << 10) {
+ case op_clz_w: {
+ alu_out = U32(rj_u(instr)) ? __builtin_clz(U32(rj_u(instr))) : 32;
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_ctz_w: {
+ alu_out = U32(rj_u(instr)) ? __builtin_ctz(U32(rj_u(instr))) : 32;
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_clz_d: {
+ alu_out = U64(rj_u(instr)) ? __builtin_clzll(U64(rj_u(instr))) : 64;
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_ctz_d: {
+ alu_out = U64(rj_u(instr)) ? __builtin_ctzll(U64(rj_u(instr))) : 64;
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_revb_2h: {
+ uint32_t input = static_cast<uint32_t>(rj(instr));
+ uint64_t output = 0;
+
+ uint32_t mask = 0xFF000000;
+ for (int i = 0; i < 4; i++) {
+ uint32_t tmp = mask & input;
+ if (i % 2 == 0) {
+ tmp = tmp >> 8;
+ } else {
+ tmp = tmp << 8;
+ }
+ output = output | tmp;
+ mask = mask >> 8;
+ }
+
+ alu_out = static_cast<int64_t>(static_cast<int32_t>(output));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_revb_4h: {
+ uint64_t input = rj_u(instr);
+ uint64_t output = 0;
+
+ uint64_t mask = 0xFF00000000000000;
+ for (int i = 0; i < 8; i++) {
+ uint64_t tmp = mask & input;
+ if (i % 2 == 0) {
+ tmp = tmp >> 8;
+ } else {
+ tmp = tmp << 8;
+ }
+ output = output | tmp;
+ mask = mask >> 8;
+ }
+
+ alu_out = static_cast<int64_t>(output);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_revb_2w: {
+ uint64_t input = rj_u(instr);
+ uint64_t output = 0;
+
+ uint64_t mask = 0xFF000000FF000000;
+ for (int i = 0; i < 4; i++) {
+ uint64_t tmp = mask & input;
+ if (i <= 1) {
+ tmp = tmp >> (24 - i * 16);
+ } else {
+ tmp = tmp << (i * 16 - 24);
+ }
+ output = output | tmp;
+ mask = mask >> 8;
+ }
+
+ alu_out = static_cast<int64_t>(output);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_revb_d: {
+ uint64_t input = rj_u(instr);
+ uint64_t output = 0;
+
+ uint64_t mask = 0xFF00000000000000;
+ for (int i = 0; i < 8; i++) {
+ uint64_t tmp = mask & input;
+ if (i <= 3) {
+ tmp = tmp >> (56 - i * 16);
+ } else {
+ tmp = tmp << (i * 16 - 56);
+ }
+ output = output | tmp;
+ mask = mask >> 8;
+ }
+
+ alu_out = static_cast<int64_t>(output);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_revh_2w: {
+ uint64_t input = rj_u(instr);
+ uint64_t output = 0;
+
+ uint64_t mask = 0xFFFF000000000000;
+ for (int i = 0; i < 4; i++) {
+ uint64_t tmp = mask & input;
+ if (i % 2 == 0) {
+ tmp = tmp >> 16;
+ } else {
+ tmp = tmp << 16;
+ }
+ output = output | tmp;
+ mask = mask >> 16;
+ }
+
+ alu_out = static_cast<int64_t>(output);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_revh_d: {
+ uint64_t input = rj_u(instr);
+ uint64_t output = 0;
+
+ uint64_t mask = 0xFFFF000000000000;
+ for (int i = 0; i < 4; i++) {
+ uint64_t tmp = mask & input;
+ if (i <= 1) {
+ tmp = tmp >> (48 - i * 32);
+ } else {
+ tmp = tmp << (i * 32 - 48);
+ }
+ output = output | tmp;
+ mask = mask >> 16;
+ }
+
+ alu_out = static_cast<int64_t>(output);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_bitrev_4b: {
+ uint32_t input = static_cast<uint32_t>(rj(instr));
+ uint32_t output = 0;
+ uint8_t i_byte, o_byte;
+
+ // Reverse the bit in byte for each individual byte
+ for (int i = 0; i < 4; i++) {
+ output = output >> 8;
+ i_byte = input & 0xFF;
+
+ // Fast way to reverse bits in byte
+ // Devised by Sean Anderson, July 13, 2001
+ o_byte = static_cast<uint8_t>(((i_byte * 0x0802LU & 0x22110LU) |
+ (i_byte * 0x8020LU & 0x88440LU)) *
+ 0x10101LU >>
+ 16);
+
+ output = output | (static_cast<uint32_t>(o_byte << 24));
+ input = input >> 8;
+ }
+
+ alu_out = static_cast<int64_t>(static_cast<int32_t>(output));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_bitrev_8b: {
+ uint64_t input = rj_u(instr);
+ uint64_t output = 0;
+ uint8_t i_byte, o_byte;
+
+ // Reverse the bit in byte for each individual byte
+ for (int i = 0; i < 8; i++) {
+ output = output >> 8;
+ i_byte = input & 0xFF;
+
+ // Fast way to reverse bits in byte
+ // Devised by Sean Anderson, July 13, 2001
+ o_byte = static_cast<uint8_t>(((i_byte * 0x0802LU & 0x22110LU) |
+ (i_byte * 0x8020LU & 0x88440LU)) *
+ 0x10101LU >>
+ 16);
+
+ output = output | (static_cast<uint64_t>(o_byte) << 56);
+ input = input >> 8;
+ }
+
+ alu_out = static_cast<int64_t>(output);
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_bitrev_w: {
+ uint32_t input = static_cast<uint32_t>(rj(instr));
+ uint32_t output = 0;
+ output = ReverseBits(input);
+ alu_out = static_cast<int64_t>(static_cast<int32_t>(output));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_bitrev_d: {
+ alu_out = static_cast<int64_t>(ReverseBits(rj_u(instr)));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_ext_w_b: {
+ uint8_t input = static_cast<uint8_t>(rj(instr));
+ alu_out = static_cast<int64_t>(static_cast<int8_t>(input));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_ext_w_h: {
+ uint16_t input = static_cast<uint16_t>(rj(instr));
+ alu_out = static_cast<int64_t>(static_cast<int16_t>(input));
+ setRegister(rd_reg(instr), alu_out);
+ break;
+ }
+ case op_fabs_s: {
+ setFpuRegisterFloat(fd_reg(instr), std::abs(fj_float(instr)));
+ break;
+ }
+ case op_fabs_d: {
+ setFpuRegisterDouble(fd_reg(instr), std::abs(fj_double(instr)));
+ break;
+ }
+ case op_fneg_s: {
+ setFpuRegisterFloat(fd_reg(instr), -fj_float(instr));
+ break;
+ }
+ case op_fneg_d: {
+ setFpuRegisterDouble(fd_reg(instr), -fj_double(instr));
+ break;
+ }
+ case op_fsqrt_s: {
+ if (fj_float(instr) >= 0) {
+ setFpuRegisterFloat(fd_reg(instr), std::sqrt(fj_float(instr)));
+ } else {
+ setFpuRegisterFloat(fd_reg(instr), std::sqrt(-1)); // qnan
+ setFCSRBit(kFCSRInvalidOpFlagBit, true);
+ }
+ break;
+ }
+ case op_fsqrt_d: {
+ if (fj_double(instr) >= 0) {
+ setFpuRegisterDouble(fd_reg(instr), std::sqrt(fj_double(instr)));
+ } else {
+ setFpuRegisterDouble(fd_reg(instr), std::sqrt(-1)); // qnan
+ setFCSRBit(kFCSRInvalidOpFlagBit, true);
+ }
+ break;
+ }
+ case op_fmov_s: {
+ setFpuRegisterFloat(fd_reg(instr), fj_float(instr));
+ break;
+ }
+ case op_fmov_d: {
+ setFpuRegisterDouble(fd_reg(instr), fj_double(instr));
+ break;
+ }
+ case op_movgr2fr_w: {
+ setFpuRegisterWord(fd_reg(instr), static_cast<int32_t>(rj(instr)));
+ break;
+ }
+ case op_movgr2fr_d: {
+ setFpuRegister(fd_reg(instr), rj(instr));
+ break;
+ }
+ case op_movgr2frh_w: {
+ setFpuRegisterHiWord(fd_reg(instr), static_cast<int32_t>(rj(instr)));
+ break;
+ }
+ case op_movfr2gr_s: {
+ setRegister(rd_reg(instr),
+ static_cast<int64_t>(getFpuRegisterWord(fj_reg(instr))));
+ break;
+ }
+ case op_movfr2gr_d: {
+ setRegister(rd_reg(instr), getFpuRegister(fj_reg(instr)));
+ break;
+ }
+ case op_movfrh2gr_s: {
+ setRegister(rd_reg(instr), getFpuRegisterHiWord(fj_reg(instr)));
+ break;
+ }
+ case op_movgr2fcsr: {
+ // fcsr could be 0-3
+ MOZ_ASSERT(rd_reg(instr) < 4);
+ FCSR_ = static_cast<uint32_t>(rj(instr));
+ break;
+ }
+ case op_movfcsr2gr: {
+ setRegister(rd_reg(instr), FCSR_);
+ break;
+ }
+ case op_fcvt_s_d: {
+ setFpuRegisterFloat(fd_reg(instr), static_cast<float>(fj_double(instr)));
+ break;
+ }
+ case op_fcvt_d_s: {
+ setFpuRegisterDouble(fd_reg(instr), static_cast<double>(fj_float(instr)));
+ break;
+ }
+ case op_ftintrm_w_s: {
+ float fj = fj_float(instr);
+ float rounded = std::floor(fj);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrm_w_d: {
+ double fj = fj_double(instr);
+ double rounded = std::floor(fj);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrm_l_s: {
+ float fj = fj_float(instr);
+ float rounded = std::floor(fj);
+ int64_t result = static_cast<int64_t>(rounded);
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrm_l_d: {
+ double fj = fj_double(instr);
+ double rounded = std::floor(fj);
+ int64_t result = static_cast<int64_t>(rounded);
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrp_w_s: {
+ float fj = fj_float(instr);
+ float rounded = std::ceil(fj);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrp_w_d: {
+ double fj = fj_double(instr);
+ double rounded = std::ceil(fj);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrp_l_s: {
+ float fj = fj_float(instr);
+ float rounded = std::ceil(fj);
+ int64_t result = static_cast<int64_t>(rounded);
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrp_l_d: {
+ double fj = fj_double(instr);
+ double rounded = std::ceil(fj);
+ int64_t result = static_cast<int64_t>(rounded);
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrz_w_s: {
+ float fj = fj_float(instr);
+ float rounded = std::trunc(fj);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrz_w_d: {
+ double fj = fj_double(instr);
+ double rounded = std::trunc(fj);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrz_l_s: {
+ float fj = fj_float(instr);
+ float rounded = std::trunc(fj);
+ int64_t result = static_cast<int64_t>(rounded);
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrz_l_d: {
+ double fj = fj_double(instr);
+ double rounded = std::trunc(fj);
+ int64_t result = static_cast<int64_t>(rounded);
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrne_w_s: {
+ float fj = fj_float(instr);
+ float rounded = std::floor(fj + 0.5);
+ int32_t result = static_cast<int32_t>(rounded);
+ if ((result & 1) != 0 && result - fj == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ result--;
+ }
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrne_w_d: {
+ double fj = fj_double(instr);
+ double rounded = std::floor(fj + 0.5);
+ int32_t result = static_cast<int32_t>(rounded);
+ if ((result & 1) != 0 && result - fj == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ result--;
+ }
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrne_l_s: {
+ float fj = fj_float(instr);
+ float rounded = std::floor(fj + 0.5);
+ int64_t result = static_cast<int64_t>(rounded);
+ if ((result & 1) != 0 && result - fj == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ result--;
+ }
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftintrne_l_d: {
+ double fj = fj_double(instr);
+ double rounded = std::floor(fj + 0.5);
+ int64_t result = static_cast<int64_t>(rounded);
+ if ((result & 1) != 0 && result - fj == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ result--;
+ }
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftint_w_s: {
+ float fj = fj_float(instr);
+ float rounded;
+ int32_t result;
+ roundAccordingToFCSR<float>(fj, &rounded, &result);
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftint_w_d: {
+ double fj = fj_double(instr);
+ double rounded;
+ int32_t result;
+ roundAccordingToFCSR<double>(fj, &rounded, &result);
+ setFpuRegisterWord(fd_reg(instr), result);
+ if (setFCSRRoundError<int32_t>(fj, rounded)) {
+ setFpuRegisterWordInvalidResult(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftint_l_s: {
+ float fj = fj_float(instr);
+ float rounded;
+ int64_t result;
+ round64AccordingToFCSR<float>(fj, &rounded, &result);
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ftint_l_d: {
+ double fj = fj_double(instr);
+ double rounded;
+ int64_t result;
+ round64AccordingToFCSR<double>(fj, &rounded, &result);
+ setFpuRegister(fd_reg(instr), result);
+ if (setFCSRRoundError<int64_t>(fj, rounded)) {
+ setFpuRegisterInvalidResult64(fj, rounded, fd_reg(instr));
+ }
+ break;
+ }
+ case op_ffint_s_w: {
+ alu_out = getFpuRegisterSignedWord(fj_reg(instr));
+ setFpuRegisterFloat(fd_reg(instr), static_cast<float>(alu_out));
+ break;
+ }
+ case op_ffint_s_l: {
+ alu_out = getFpuRegister(fj_reg(instr));
+ setFpuRegisterFloat(fd_reg(instr), static_cast<float>(alu_out));
+ break;
+ }
+ case op_ffint_d_w: {
+ alu_out = getFpuRegisterSignedWord(fj_reg(instr));
+ setFpuRegisterDouble(fd_reg(instr), static_cast<double>(alu_out));
+ break;
+ }
+ case op_ffint_d_l: {
+ alu_out = getFpuRegister(fj_reg(instr));
+ setFpuRegisterDouble(fd_reg(instr), static_cast<double>(alu_out));
+ break;
+ }
+ case op_frint_s: {
+ float fj = fj_float(instr);
+ float result, temp_result;
+ double temp;
+ float upper = std::ceil(fj);
+ float lower = std::floor(fj);
+ switch (getFCSRRoundingMode()) {
+ case kRoundToNearest:
+ if (upper - fj < fj - lower) {
+ result = upper;
+ } else if (upper - fj > fj - lower) {
+ result = lower;
+ } else {
+ temp_result = upper / 2;
+ float reminder = std::modf(temp_result, &temp);
+ if (reminder == 0) {
+ result = upper;
+ } else {
+ result = lower;
+ }
+ }
+ break;
+ case kRoundToZero:
+ result = (fj > 0 ? lower : upper);
+ break;
+ case kRoundToPlusInf:
+ result = upper;
+ break;
+ case kRoundToMinusInf:
+ result = lower;
+ break;
+ }
+ setFpuRegisterFloat(fd_reg(instr), result);
+ if (result != fj) {
+ setFCSRBit(kFCSRInexactFlagBit, true);
+ }
+ break;
+ }
+ case op_frint_d: {
+ double fj = fj_double(instr);
+ double result, temp, temp_result;
+ double upper = std::ceil(fj);
+ double lower = std::floor(fj);
+ switch (getFCSRRoundingMode()) {
+ case kRoundToNearest:
+ if (upper - fj < fj - lower) {
+ result = upper;
+ } else if (upper - fj > fj - lower) {
+ result = lower;
+ } else {
+ temp_result = upper / 2;
+ double reminder = std::modf(temp_result, &temp);
+ if (reminder == 0) {
+ result = upper;
+ } else {
+ result = lower;
+ }
+ }
+ break;
+ case kRoundToZero:
+ result = (fj > 0 ? lower : upper);
+ break;
+ case kRoundToPlusInf:
+ result = upper;
+ break;
+ case kRoundToMinusInf:
+ result = lower;
+ break;
+ }
+ setFpuRegisterDouble(fd_reg(instr), result);
+ if (result != fj) {
+ setFCSRBit(kFCSRInexactFlagBit, true);
+ }
+ break;
+ }
+ case op_movfr2cf:
+ printf("Sim UNIMPLEMENTED: MOVFR2CF\n");
+ UNIMPLEMENTED();
+ break;
+ case op_movgr2cf:
+ printf("Sim UNIMPLEMENTED: MOVGR2CF\n");
+ UNIMPLEMENTED();
+ break;
+ case op_clo_w:
+ printf("Sim UNIMPLEMENTED: FCO_W\n");
+ UNIMPLEMENTED();
+ break;
+ case op_cto_w:
+ printf("Sim UNIMPLEMENTED: FTO_W\n");
+ UNIMPLEMENTED();
+ break;
+ case op_clo_d:
+ printf("Sim UNIMPLEMENTED: FLO_D\n");
+ UNIMPLEMENTED();
+ break;
+ case op_cto_d:
+ printf("Sim UNIMPLEMENTED: FTO_D\n");
+ UNIMPLEMENTED();
+ break;
+ // Unimplemented opcodes raised an error in the configuration step before,
+ // so we can use the default here to set the destination register in common
+ // cases.
+ default:
+ UNREACHABLE();
+ }
+}
+
+void Simulator::decodeTypeOp24(SimInstruction* instr) {
+ switch (instr->bits(31, 8) << 8) {
+ case op_movcf2fr:
+ UNIMPLEMENTED();
+ break;
+ case op_movcf2gr:
+ setRegister(rd_reg(instr), getCFRegister(cj_reg(instr)));
+ break;
+ UNIMPLEMENTED();
+ break;
+ default:
+ UNREACHABLE();
+ }
+}
+
+// Executes the current instruction.
+void Simulator::instructionDecode(SimInstruction* instr) {
+ if (!SimulatorProcess::ICacheCheckingDisableCount) {
+ AutoLockSimulatorCache als;
+ SimulatorProcess::checkICacheLocked(instr);
+ }
+ pc_modified_ = false;
+
+ switch (instr->instructionType()) {
+ case SimInstruction::kOp6Type:
+ decodeTypeOp6(instr);
+ break;
+ case SimInstruction::kOp7Type:
+ decodeTypeOp7(instr);
+ break;
+ case SimInstruction::kOp8Type:
+ decodeTypeOp8(instr);
+ break;
+ case SimInstruction::kOp10Type:
+ decodeTypeOp10(instr);
+ break;
+ case SimInstruction::kOp11Type:
+ decodeTypeOp11(instr);
+ break;
+ case SimInstruction::kOp12Type:
+ decodeTypeOp12(instr);
+ break;
+ case SimInstruction::kOp14Type:
+ decodeTypeOp14(instr);
+ break;
+ case SimInstruction::kOp15Type:
+ decodeTypeOp15(instr);
+ break;
+ case SimInstruction::kOp16Type:
+ decodeTypeOp16(instr);
+ break;
+ case SimInstruction::kOp17Type:
+ decodeTypeOp17(instr);
+ break;
+ case SimInstruction::kOp22Type:
+ decodeTypeOp22(instr);
+ break;
+ case SimInstruction::kOp24Type:
+ decodeTypeOp24(instr);
+ break;
+ default:
+ UNSUPPORTED();
+ }
+ if (!pc_modified_) {
+ setRegister(pc,
+ reinterpret_cast<int64_t>(instr) + SimInstruction::kInstrSize);
+ }
+}
+
+void Simulator::enable_single_stepping(SingleStepCallback cb, void* arg) {
+ single_stepping_ = true;
+ single_step_callback_ = cb;
+ single_step_callback_arg_ = arg;
+ single_step_callback_(single_step_callback_arg_, this, (void*)get_pc());
+}
+
+void Simulator::disable_single_stepping() {
+ if (!single_stepping_) {
+ return;
+ }
+ single_step_callback_(single_step_callback_arg_, this, (void*)get_pc());
+ single_stepping_ = false;
+ single_step_callback_ = nullptr;
+ single_step_callback_arg_ = nullptr;
+}
+
+template <bool enableStopSimAt>
+void Simulator::execute() {
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ // Get the PC to simulate. Cannot use the accessor here as we need the
+ // raw PC value and not the one used as input to arithmetic instructions.
+ int64_t program_counter = get_pc();
+
+ while (program_counter != end_sim_pc) {
+ if (enableStopSimAt && (icount_ == Simulator::StopSimAt)) {
+ loong64Debugger dbg(this);
+ dbg.debug();
+ } else {
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this,
+ (void*)program_counter);
+ }
+ SimInstruction* instr =
+ reinterpret_cast<SimInstruction*>(program_counter);
+ instructionDecode(instr);
+ icount_++;
+ }
+ program_counter = get_pc();
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+}
+
+void Simulator::callInternal(uint8_t* entry) {
+ // Prepare to execute the code at entry.
+ setRegister(pc, reinterpret_cast<int64_t>(entry));
+ // Put down marker for end of simulation. The simulator will stop simulation
+ // when the PC reaches this value. By saving the "end simulation" value into
+ // the LR the simulation stops when returning to this call point.
+ setRegister(ra, end_sim_pc);
+
+ // Remember the values of callee-saved registers.
+ // The code below assumes that r9 is not used as sb (static base) in
+ // simulator code and therefore is regarded as a callee-saved register.
+ int64_t s0_val = getRegister(s0);
+ int64_t s1_val = getRegister(s1);
+ int64_t s2_val = getRegister(s2);
+ int64_t s3_val = getRegister(s3);
+ int64_t s4_val = getRegister(s4);
+ int64_t s5_val = getRegister(s5);
+ int64_t s6_val = getRegister(s6);
+ int64_t s7_val = getRegister(s7);
+ int64_t s8_val = getRegister(s8);
+ int64_t gp_val = getRegister(gp);
+ int64_t sp_val = getRegister(sp);
+ int64_t tp_val = getRegister(tp);
+ int64_t fp_val = getRegister(fp);
+
+ // Set up the callee-saved registers with a known value. To be able to check
+ // that they are preserved properly across JS execution.
+ int64_t callee_saved_value = icount_;
+ setRegister(s0, callee_saved_value);
+ setRegister(s1, callee_saved_value);
+ setRegister(s2, callee_saved_value);
+ setRegister(s3, callee_saved_value);
+ setRegister(s4, callee_saved_value);
+ setRegister(s5, callee_saved_value);
+ setRegister(s6, callee_saved_value);
+ setRegister(s7, callee_saved_value);
+ setRegister(s8, callee_saved_value);
+ setRegister(gp, callee_saved_value);
+ setRegister(tp, callee_saved_value);
+ setRegister(fp, callee_saved_value);
+
+ // Start the simulation.
+ if (Simulator::StopSimAt != -1) {
+ execute<true>();
+ } else {
+ execute<false>();
+ }
+
+ // Check that the callee-saved registers have been preserved.
+ MOZ_ASSERT(callee_saved_value == getRegister(s0));
+ MOZ_ASSERT(callee_saved_value == getRegister(s1));
+ MOZ_ASSERT(callee_saved_value == getRegister(s2));
+ MOZ_ASSERT(callee_saved_value == getRegister(s3));
+ MOZ_ASSERT(callee_saved_value == getRegister(s4));
+ MOZ_ASSERT(callee_saved_value == getRegister(s5));
+ MOZ_ASSERT(callee_saved_value == getRegister(s6));
+ MOZ_ASSERT(callee_saved_value == getRegister(s7));
+ MOZ_ASSERT(callee_saved_value == getRegister(s8));
+ MOZ_ASSERT(callee_saved_value == getRegister(gp));
+ MOZ_ASSERT(callee_saved_value == getRegister(tp));
+ MOZ_ASSERT(callee_saved_value == getRegister(fp));
+
+ // Restore callee-saved registers with the original value.
+ setRegister(s0, s0_val);
+ setRegister(s1, s1_val);
+ setRegister(s2, s2_val);
+ setRegister(s3, s3_val);
+ setRegister(s4, s4_val);
+ setRegister(s5, s5_val);
+ setRegister(s6, s6_val);
+ setRegister(s7, s7_val);
+ setRegister(s8, s8_val);
+ setRegister(gp, gp_val);
+ setRegister(sp, sp_val);
+ setRegister(tp, tp_val);
+ setRegister(fp, fp_val);
+}
+
+int64_t Simulator::call(uint8_t* entry, int argument_count, ...) {
+ va_list parameters;
+ va_start(parameters, argument_count);
+
+ int64_t original_stack = getRegister(sp);
+ // Compute position of stack on entry to generated code.
+ int64_t entry_stack = original_stack;
+ if (argument_count > kCArgSlotCount) {
+ entry_stack = entry_stack - argument_count * sizeof(int64_t);
+ } else {
+ entry_stack = entry_stack - kCArgsSlotsSize;
+ }
+
+ entry_stack &= ~U64(ABIStackAlignment - 1);
+
+ intptr_t* stack_argument = reinterpret_cast<intptr_t*>(entry_stack);
+
+ // Setup the arguments.
+ for (int i = 0; i < argument_count; i++) {
+ js::jit::Register argReg;
+ if (GetIntArgReg(i, &argReg)) {
+ setRegister(argReg.code(), va_arg(parameters, int64_t));
+ } else {
+ stack_argument[i] = va_arg(parameters, int64_t);
+ }
+ }
+
+ va_end(parameters);
+ setRegister(sp, entry_stack);
+
+ callInternal(entry);
+
+ // Pop stack passed arguments.
+ MOZ_ASSERT(entry_stack == getRegister(sp));
+ setRegister(sp, original_stack);
+
+ int64_t result = getRegister(a0);
+ return result;
+}
+
+uintptr_t Simulator::pushAddress(uintptr_t address) {
+ int new_sp = getRegister(sp) - sizeof(uintptr_t);
+ uintptr_t* stack_slot = reinterpret_cast<uintptr_t*>(new_sp);
+ *stack_slot = address;
+ setRegister(sp, new_sp);
+ return new_sp;
+}
+
+uintptr_t Simulator::popAddress() {
+ int current_sp = getRegister(sp);
+ uintptr_t* stack_slot = reinterpret_cast<uintptr_t*>(current_sp);
+ uintptr_t address = *stack_slot;
+ setRegister(sp, current_sp + sizeof(uintptr_t));
+ return address;
+}
+
+} // namespace jit
+} // namespace js
+
+js::jit::Simulator* JSContext::simulator() const { return simulator_; }
diff --git a/js/src/jit/loong64/Simulator-loong64.h b/js/src/jit/loong64/Simulator-loong64.h
new file mode 100644
index 0000000000..233f218256
--- /dev/null
+++ b/js/src/jit/loong64/Simulator-loong64.h
@@ -0,0 +1,650 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80: */
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef jit_loong64_Simulator_loong64_h
+#define jit_loong64_Simulator_loong64_h
+
+#ifdef JS_SIMULATOR_LOONG64
+
+# include "mozilla/Atomics.h"
+
+# include "jit/IonTypes.h"
+# include "js/ProfilingFrameIterator.h"
+# include "threading/Thread.h"
+# include "vm/MutexIDs.h"
+# include "wasm/WasmSignalHandlers.h"
+
+namespace js {
+
+namespace jit {
+
+class JitActivation;
+
+class Simulator;
+class Redirection;
+class CachePage;
+class AutoLockSimulator;
+
+// When the SingleStepCallback is called, the simulator is about to execute
+// sim->get_pc() and the current machine state represents the completed
+// execution of the previous pc.
+typedef void (*SingleStepCallback)(void* arg, Simulator* sim, void* pc);
+
+const intptr_t kPointerAlignment = 8;
+const intptr_t kPointerAlignmentMask = kPointerAlignment - 1;
+
+const intptr_t kDoubleAlignment = 8;
+const intptr_t kDoubleAlignmentMask = kDoubleAlignment - 1;
+
+// Number of general purpose registers.
+const int kNumRegisters = 32;
+
+// In the simulator, the PC register is simulated as the 34th register.
+const int kPCRegister = 32;
+
+// Number coprocessor registers.
+const int kNumFPURegisters = 32;
+
+// FPU (coprocessor 1) control registers. Currently only FCSR is implemented.
+// TODO fcsr0 fcsr1 fcsr2 fcsr3
+const int kFCSRRegister = 0;
+const int kInvalidFPUControlRegister = -1;
+const uint32_t kFPUInvalidResult = static_cast<uint32_t>(1 << 31) - 1;
+const int32_t kFPUInvalidResultNegative = static_cast<int32_t>(1u << 31);
+const uint64_t kFPU64InvalidResult =
+ static_cast<uint64_t>(static_cast<uint64_t>(1) << 63) - 1;
+const int64_t kFPU64InvalidResultNegative =
+ static_cast<int64_t>(static_cast<uint64_t>(1) << 63);
+
+const uint32_t kFPURoundingModeShift = 8;
+const uint32_t kFPURoundingModeMask = 0b11 << kFPURoundingModeShift;
+
+// FPU rounding modes.
+enum FPURoundingMode {
+ RN = 0b00 << kFPURoundingModeShift, // Round to Nearest.
+ RZ = 0b01 << kFPURoundingModeShift, // Round towards zero.
+ RP = 0b10 << kFPURoundingModeShift, // Round towards Plus Infinity.
+ RM = 0b11 << kFPURoundingModeShift, // Round towards Minus Infinity.
+
+ // Aliases.
+ kRoundToNearest = RN,
+ kRoundToZero = RZ,
+ kRoundToPlusInf = RP,
+ kRoundToMinusInf = RM,
+
+ mode_round = RN,
+ mode_ceil = RP,
+ mode_floor = RM,
+ mode_trunc = RZ
+};
+
+// FCSR constants.
+const uint32_t kFCSRInexactFlagBit = 16;
+const uint32_t kFCSRUnderflowFlagBit = 17;
+const uint32_t kFCSROverflowFlagBit = 18;
+const uint32_t kFCSRDivideByZeroFlagBit = 19;
+const uint32_t kFCSRInvalidOpFlagBit = 20;
+
+const uint32_t kFCSRInexactCauseBit = 24;
+const uint32_t kFCSRUnderflowCauseBit = 25;
+const uint32_t kFCSROverflowCauseBit = 26;
+const uint32_t kFCSRDivideByZeroCauseBit = 27;
+const uint32_t kFCSRInvalidOpCauseBit = 28;
+
+const uint32_t kFCSRInexactFlagMask = 1 << kFCSRInexactFlagBit;
+const uint32_t kFCSRUnderflowFlagMask = 1 << kFCSRUnderflowFlagBit;
+const uint32_t kFCSROverflowFlagMask = 1 << kFCSROverflowFlagBit;
+const uint32_t kFCSRDivideByZeroFlagMask = 1 << kFCSRDivideByZeroFlagBit;
+const uint32_t kFCSRInvalidOpFlagMask = 1 << kFCSRInvalidOpFlagBit;
+
+const uint32_t kFCSRFlagMask =
+ kFCSRInexactFlagMask | kFCSRUnderflowFlagMask | kFCSROverflowFlagMask |
+ kFCSRDivideByZeroFlagMask | kFCSRInvalidOpFlagMask;
+
+const uint32_t kFCSRExceptionFlagMask = kFCSRFlagMask ^ kFCSRInexactFlagMask;
+
+// On LoongArch64 Simulator breakpoints can have different codes:
+// - Breaks between 0 and kMaxWatchpointCode are treated as simple watchpoints,
+// the simulator will run through them and print the registers.
+// - Breaks between kMaxWatchpointCode and kMaxStopCode are treated as stop()
+// instructions (see Assembler::stop()).
+// - Breaks larger than kMaxStopCode are simple breaks, dropping you into the
+// debugger.
+const uint32_t kMaxWatchpointCode = 31;
+const uint32_t kMaxStopCode = 127;
+const uint32_t kWasmTrapCode = 6;
+
+// -----------------------------------------------------------------------------
+// Utility functions
+
+typedef uint32_t Instr;
+class SimInstruction;
+
+// Per thread simulator state.
+class Simulator {
+ friend class loong64Debugger;
+
+ public:
+ // Registers are declared in order.
+ enum Register {
+ no_reg = -1,
+ zero_reg = 0,
+ ra,
+ gp,
+ sp,
+ a0,
+ a1,
+ a2,
+ a3,
+ a4,
+ a5,
+ a6,
+ a7,
+ t0,
+ t1,
+ t2,
+ t3,
+ t4,
+ t5,
+ t6,
+ t7,
+ t8,
+ tp,
+ fp,
+ s0,
+ s1,
+ s2,
+ s3,
+ s4,
+ s5,
+ s6,
+ s7,
+ s8,
+ pc, // pc must be the last register.
+ kNumSimuRegisters,
+ // aliases
+ v0 = a0,
+ v1 = a1,
+ };
+
+ // Condition flag registers.
+ enum CFRegister {
+ fcc0,
+ fcc1,
+ fcc2,
+ fcc3,
+ fcc4,
+ fcc5,
+ fcc6,
+ fcc7,
+ kNumCFRegisters
+ };
+
+ // Floating point registers.
+ enum FPURegister {
+ f0,
+ f1,
+ f2,
+ f3,
+ f4,
+ f5,
+ f6,
+ f7,
+ f8,
+ f9,
+ f10,
+ f11,
+ f12,
+ f13,
+ f14,
+ f15,
+ f16,
+ f17,
+ f18,
+ f19,
+ f20,
+ f21,
+ f22,
+ f23,
+ f24,
+ f25,
+ f26,
+ f27,
+ f28,
+ f29,
+ f30,
+ f31,
+ kNumFPURegisters
+ };
+
+ // Returns nullptr on OOM.
+ static Simulator* Create();
+
+ static void Destroy(Simulator* simulator);
+
+ // Constructor/destructor are for internal use only; use the static methods
+ // above.
+ Simulator();
+ ~Simulator();
+
+ // The currently executing Simulator instance. Potentially there can be one
+ // for each native thread.
+ static Simulator* Current();
+
+ static inline uintptr_t StackLimit() {
+ return Simulator::Current()->stackLimit();
+ }
+
+ uintptr_t* addressOfStackLimit();
+
+ // Accessors for register state. Reading the pc value adheres to the LOONG64
+ // architecture specification and is off by a 8 from the currently executing
+ // instruction.
+ void setRegister(int reg, int64_t value);
+ int64_t getRegister(int reg) const;
+ // Same for FPURegisters.
+ void setFpuRegister(int fpureg, int64_t value);
+ void setFpuRegisterWord(int fpureg, int32_t value);
+ void setFpuRegisterHiWord(int fpureg, int32_t value);
+ void setFpuRegisterFloat(int fpureg, float value);
+ void setFpuRegisterDouble(int fpureg, double value);
+
+ void setFpuRegisterWordInvalidResult(float original, float rounded,
+ int fpureg);
+ void setFpuRegisterWordInvalidResult(double original, double rounded,
+ int fpureg);
+ void setFpuRegisterInvalidResult(float original, float rounded, int fpureg);
+ void setFpuRegisterInvalidResult(double original, double rounded, int fpureg);
+ void setFpuRegisterInvalidResult64(float original, float rounded, int fpureg);
+ void setFpuRegisterInvalidResult64(double original, double rounded,
+ int fpureg);
+
+ int64_t getFpuRegister(int fpureg) const;
+ // int32_t getFpuRegisterLo(int fpureg) const;
+ // int32_t getFpuRegisterHi(int fpureg) const;
+ int32_t getFpuRegisterWord(int fpureg) const;
+ int32_t getFpuRegisterSignedWord(int fpureg) const;
+ int32_t getFpuRegisterHiWord(int fpureg) const;
+ float getFpuRegisterFloat(int fpureg) const;
+ double getFpuRegisterDouble(int fpureg) const;
+
+ void setCFRegister(int cfreg, bool value);
+ bool getCFRegister(int cfreg) const;
+
+ void set_fcsr_rounding_mode(FPURoundingMode mode);
+
+ void setFCSRBit(uint32_t cc, bool value);
+ bool testFCSRBit(uint32_t cc);
+ unsigned int getFCSRRoundingMode();
+ template <typename T>
+ bool setFCSRRoundError(double original, double rounded);
+ bool setFCSRRound64Error(float original, float rounded);
+
+ template <typename T>
+ void roundAccordingToFCSR(T toRound, T* rounded, int32_t* rounded_int);
+
+ template <typename T>
+ void round64AccordingToFCSR(T toRound, T* rounded, int64_t* rounded_int);
+
+ // Special case of set_register and get_register to access the raw PC value.
+ void set_pc(int64_t value);
+ int64_t get_pc() const;
+
+ template <typename T>
+ T get_pc_as() const {
+ return reinterpret_cast<T>(get_pc());
+ }
+
+ void enable_single_stepping(SingleStepCallback cb, void* arg);
+ void disable_single_stepping();
+
+ // Accessor to the internal simulator stack area.
+ uintptr_t stackLimit() const;
+ bool overRecursed(uintptr_t newsp = 0) const;
+ bool overRecursedWithExtra(uint32_t extra) const;
+
+ // Executes LOONG64 instructions until the PC reaches end_sim_pc.
+ template <bool enableStopSimAt>
+ void execute();
+
+ // Sets up the simulator state and grabs the result on return.
+ int64_t call(uint8_t* entry, int argument_count, ...);
+
+ // Push an address onto the JS stack.
+ uintptr_t pushAddress(uintptr_t address);
+
+ // Pop an address from the JS stack.
+ uintptr_t popAddress();
+
+ // Debugger input.
+ void setLastDebuggerInput(char* input);
+ char* lastDebuggerInput() { return lastDebuggerInput_; }
+
+ // Returns true if pc register contains one of the 'SpecialValues' defined
+ // below (bad_ra, end_sim_pc).
+ bool has_bad_pc() const;
+
+ private:
+ enum SpecialValues {
+ // Known bad pc value to ensure that the simulator does not execute
+ // without being properly setup.
+ bad_ra = -1,
+ // A pc value used to signal the simulator to stop execution. Generally
+ // the ra is set to this value on transition from native C code to
+ // simulated execution, so that the simulator can "return" to the native
+ // C code.
+ end_sim_pc = -2,
+ // Unpredictable value.
+ Unpredictable = 0xbadbeaf
+ };
+
+ bool init();
+
+ // Unsupported instructions use Format to print an error and stop execution.
+ void format(SimInstruction* instr, const char* format);
+
+ // Read and write memory.
+ inline uint8_t readBU(uint64_t addr);
+ inline int8_t readB(uint64_t addr);
+ inline void writeB(uint64_t addr, uint8_t value);
+ inline void writeB(uint64_t addr, int8_t value);
+
+ inline uint16_t readHU(uint64_t addr, SimInstruction* instr);
+ inline int16_t readH(uint64_t addr, SimInstruction* instr);
+ inline void writeH(uint64_t addr, uint16_t value, SimInstruction* instr);
+ inline void writeH(uint64_t addr, int16_t value, SimInstruction* instr);
+
+ inline uint32_t readWU(uint64_t addr, SimInstruction* instr);
+ inline int32_t readW(uint64_t addr, SimInstruction* instr);
+ inline void writeW(uint64_t addr, uint32_t value, SimInstruction* instr);
+ inline void writeW(uint64_t addr, int32_t value, SimInstruction* instr);
+
+ inline int64_t readDW(uint64_t addr, SimInstruction* instr);
+ inline void writeDW(uint64_t addr, int64_t value, SimInstruction* instr);
+
+ inline double readD(uint64_t addr, SimInstruction* instr);
+ inline void writeD(uint64_t addr, double value, SimInstruction* instr);
+
+ inline int32_t loadLinkedW(uint64_t addr, SimInstruction* instr);
+ inline int storeConditionalW(uint64_t addr, int32_t value,
+ SimInstruction* instr);
+
+ inline int64_t loadLinkedD(uint64_t addr, SimInstruction* instr);
+ inline int storeConditionalD(uint64_t addr, int64_t value,
+ SimInstruction* instr);
+
+ // Executing is handled based on the instruction type.
+ void decodeTypeOp6(SimInstruction* instr);
+ void decodeTypeOp7(SimInstruction* instr);
+ void decodeTypeOp8(SimInstruction* instr);
+ void decodeTypeOp10(SimInstruction* instr);
+ void decodeTypeOp11(SimInstruction* instr);
+ void decodeTypeOp12(SimInstruction* instr);
+ void decodeTypeOp14(SimInstruction* instr);
+ void decodeTypeOp15(SimInstruction* instr);
+ void decodeTypeOp16(SimInstruction* instr);
+ void decodeTypeOp17(SimInstruction* instr);
+ void decodeTypeOp22(SimInstruction* instr);
+ void decodeTypeOp24(SimInstruction* instr);
+
+ inline int32_t rj_reg(SimInstruction* instr) const;
+ inline int64_t rj(SimInstruction* instr) const;
+ inline uint64_t rj_u(SimInstruction* instr) const;
+ inline int32_t rk_reg(SimInstruction* instr) const;
+ inline int64_t rk(SimInstruction* instr) const;
+ inline uint64_t rk_u(SimInstruction* instr) const;
+ inline int32_t rd_reg(SimInstruction* instr) const;
+ inline int64_t rd(SimInstruction* instr) const;
+ inline uint64_t rd_u(SimInstruction* instr) const;
+ inline int32_t fa_reg(SimInstruction* instr) const;
+ inline float fa_float(SimInstruction* instr) const;
+ inline double fa_double(SimInstruction* instr) const;
+
+ inline int32_t fj_reg(SimInstruction* instr) const;
+ inline float fj_float(SimInstruction* instr) const;
+ inline double fj_double(SimInstruction* instr) const;
+
+ inline int32_t fk_reg(SimInstruction* instr) const;
+ inline float fk_float(SimInstruction* instr) const;
+ inline double fk_double(SimInstruction* instr) const;
+ inline int32_t fd_reg(SimInstruction* instr) const;
+ inline float fd_float(SimInstruction* instr) const;
+ inline double fd_double(SimInstruction* instr) const;
+
+ inline int32_t cj_reg(SimInstruction* instr) const;
+ inline bool cj(SimInstruction* instr) const;
+
+ inline int32_t cd_reg(SimInstruction* instr) const;
+ inline bool cd(SimInstruction* instr) const;
+
+ inline int32_t ca_reg(SimInstruction* instr) const;
+ inline bool ca(SimInstruction* instr) const;
+ inline uint32_t sa2(SimInstruction* instr) const;
+ inline uint32_t sa3(SimInstruction* instr) const;
+ inline uint32_t ui5(SimInstruction* instr) const;
+ inline uint32_t ui6(SimInstruction* instr) const;
+ inline uint32_t lsbw(SimInstruction* instr) const;
+ inline uint32_t msbw(SimInstruction* instr) const;
+ inline uint32_t lsbd(SimInstruction* instr) const;
+ inline uint32_t msbd(SimInstruction* instr) const;
+ inline uint32_t cond(SimInstruction* instr) const;
+ inline int32_t si12(SimInstruction* instr) const;
+ inline uint32_t ui12(SimInstruction* instr) const;
+ inline int32_t si14(SimInstruction* instr) const;
+ inline int32_t si16(SimInstruction* instr) const;
+ inline int32_t si20(SimInstruction* instr) const;
+
+ // Used for breakpoints.
+ void softwareInterrupt(SimInstruction* instr);
+
+ // Stop helper functions.
+ bool isWatchpoint(uint32_t code);
+ void printWatchpoint(uint32_t code);
+ void handleStop(uint32_t code, SimInstruction* instr);
+ bool isStopInstruction(SimInstruction* instr);
+ bool isEnabledStop(uint32_t code);
+ void enableStop(uint32_t code);
+ void disableStop(uint32_t code);
+ void increaseStopCounter(uint32_t code);
+ void printStopInfo(uint32_t code);
+
+ JS::ProfilingFrameIterator::RegisterState registerState();
+
+ // Handle any wasm faults, returning true if the fault was handled.
+ // This method is rather hot so inline the normal (no-wasm) case.
+ bool MOZ_ALWAYS_INLINE handleWasmSegFault(uint64_t addr, unsigned numBytes) {
+ if (MOZ_LIKELY(!js::wasm::CodeExists)) {
+ return false;
+ }
+
+ uint8_t* newPC;
+ if (!js::wasm::MemoryAccessTraps(registerState(), (uint8_t*)addr, numBytes,
+ &newPC)) {
+ return false;
+ }
+
+ LLBit_ = false;
+ set_pc(int64_t(newPC));
+ return true;
+ }
+
+ // Executes one instruction.
+ void instructionDecode(SimInstruction* instr);
+
+ public:
+ static int64_t StopSimAt;
+
+ // Runtime call support.
+ static void* RedirectNativeFunction(void* nativeFunction,
+ ABIFunctionType type);
+
+ private:
+ enum Exception {
+ kNone,
+ kIntegerOverflow,
+ kIntegerUnderflow,
+ kDivideByZero,
+ kNumExceptions
+ };
+ int16_t exceptions[kNumExceptions];
+
+ // Exceptions.
+ void signalExceptions();
+
+ // Handle return value for runtime FP functions.
+ void setCallResultDouble(double result);
+ void setCallResultFloat(float result);
+ void setCallResult(int64_t res);
+ void setCallResult(__int128 res);
+
+ void callInternal(uint8_t* entry);
+
+ // Architecture state.
+ // Registers.
+ int64_t registers_[kNumSimuRegisters];
+ // Floating point Registers.
+ int64_t FPUregisters_[kNumFPURegisters];
+ // Condition flags Registers.
+ bool CFregisters_[kNumCFRegisters];
+ // FPU control register.
+ uint32_t FCSR_;
+
+ bool LLBit_;
+ uintptr_t LLAddr_;
+ int64_t lastLLValue_;
+
+ // Simulator support.
+ char* stack_;
+ uintptr_t stackLimit_;
+ bool pc_modified_;
+ int64_t icount_;
+ int64_t break_count_;
+
+ // Debugger input.
+ char* lastDebuggerInput_;
+
+ // Registered breakpoints.
+ SimInstruction* break_pc_;
+ Instr break_instr_;
+
+ // Single-stepping support
+ bool single_stepping_;
+ SingleStepCallback single_step_callback_;
+ void* single_step_callback_arg_;
+
+ // A stop is watched if its code is less than kNumOfWatchedStops.
+ // Only watched stops support enabling/disabling and the counter feature.
+ static const uint32_t kNumOfWatchedStops = 256;
+
+ // Stop is disabled if bit 31 is set.
+ static const uint32_t kStopDisabledBit = 1U << 31;
+
+ // A stop is enabled, meaning the simulator will stop when meeting the
+ // instruction, if bit 31 of watchedStops_[code].count is unset.
+ // The value watchedStops_[code].count & ~(1 << 31) indicates how many times
+ // the breakpoint was hit or gone through.
+ struct StopCountAndDesc {
+ uint32_t count_;
+ char* desc_;
+ };
+ StopCountAndDesc watchedStops_[kNumOfWatchedStops];
+};
+
+// Process wide simulator state.
+class SimulatorProcess {
+ friend class Redirection;
+ friend class AutoLockSimulatorCache;
+
+ private:
+ // ICache checking.
+ struct ICacheHasher {
+ typedef void* Key;
+ typedef void* Lookup;
+ static HashNumber hash(const Lookup& l);
+ static bool match(const Key& k, const Lookup& l);
+ };
+
+ public:
+ typedef HashMap<void*, CachePage*, ICacheHasher, SystemAllocPolicy> ICacheMap;
+
+ static mozilla::Atomic<size_t, mozilla::ReleaseAcquire>
+ ICacheCheckingDisableCount;
+ static void FlushICache(void* start, size_t size);
+
+ static void checkICacheLocked(SimInstruction* instr);
+
+ static bool initialize() {
+ singleton_ = js_new<SimulatorProcess>();
+ return singleton_;
+ }
+ static void destroy() {
+ js_delete(singleton_);
+ singleton_ = nullptr;
+ }
+
+ SimulatorProcess();
+ ~SimulatorProcess();
+
+ private:
+ static SimulatorProcess* singleton_;
+
+ // This lock creates a critical section around 'redirection_' and
+ // 'icache_', which are referenced both by the execution engine
+ // and by the off-thread compiler (see Redirection::Get in the cpp file).
+ Mutex cacheLock_;
+
+ Redirection* redirection_;
+ ICacheMap icache_;
+
+ public:
+ static ICacheMap& icache() {
+ // Technically we need the lock to access the innards of the
+ // icache, not to take its address, but the latter condition
+ // serves as a useful complement to the former.
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ return singleton_->icache_;
+ }
+
+ static Redirection* redirection() {
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ return singleton_->redirection_;
+ }
+
+ static void setRedirection(js::jit::Redirection* redirection) {
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ singleton_->redirection_ = redirection;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* JS_SIMULATOR_LOONG64 */
+
+#endif /* jit_loong64_Simulator_loong64_h */
diff --git a/js/src/jit/loong64/Trampoline-loong64.cpp b/js/src/jit/loong64/Trampoline-loong64.cpp
new file mode 100644
index 0000000000..1d8b5cd7b3
--- /dev/null
+++ b/js/src/jit/loong64/Trampoline-loong64.cpp
@@ -0,0 +1,833 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/loong64/SharedICHelpers-loong64.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// All registers to save and restore. This includes the stack pointer, since we
+// use the ability to reference register values on the stack by index.
+static const LiveRegisterSet AllRegs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::AllMask),
+ FloatRegisterSet(FloatRegisters::AllMask));
+
+static_assert(sizeof(uintptr_t) == sizeof(uint64_t), "Not 32-bit clean.");
+
+struct EnterJITRegs {
+ double f31;
+ double f30;
+ double f29;
+ double f28;
+ double f27;
+ double f26;
+ double f25;
+ double f24;
+
+ // uintptr_t align;
+
+ // non-volatile registers.
+ uint64_t ra;
+ uint64_t fp;
+ uint64_t s8;
+ uint64_t s7;
+ uint64_t s6;
+ uint64_t s5;
+ uint64_t s4;
+ uint64_t s3;
+ uint64_t s2;
+ uint64_t s1;
+ uint64_t s0;
+ // Save reg_vp(a7) on stack, use it after call jit code.
+ uint64_t a7;
+};
+
+static void GenerateReturn(MacroAssembler& masm, int returnCode) {
+ MOZ_ASSERT(masm.framePushed() == sizeof(EnterJITRegs));
+
+ // Restore non-volatile registers
+ masm.as_ld_d(s0, StackPointer, offsetof(EnterJITRegs, s0));
+ masm.as_ld_d(s1, StackPointer, offsetof(EnterJITRegs, s1));
+ masm.as_ld_d(s2, StackPointer, offsetof(EnterJITRegs, s2));
+ masm.as_ld_d(s3, StackPointer, offsetof(EnterJITRegs, s3));
+ masm.as_ld_d(s4, StackPointer, offsetof(EnterJITRegs, s4));
+ masm.as_ld_d(s5, StackPointer, offsetof(EnterJITRegs, s5));
+ masm.as_ld_d(s6, StackPointer, offsetof(EnterJITRegs, s6));
+ masm.as_ld_d(s7, StackPointer, offsetof(EnterJITRegs, s7));
+ masm.as_ld_d(s8, StackPointer, offsetof(EnterJITRegs, s8));
+ masm.as_ld_d(fp, StackPointer, offsetof(EnterJITRegs, fp));
+ masm.as_ld_d(ra, StackPointer, offsetof(EnterJITRegs, ra));
+
+ // Restore non-volatile floating point registers
+ masm.as_fld_d(f24, StackPointer, offsetof(EnterJITRegs, f24));
+ masm.as_fld_d(f25, StackPointer, offsetof(EnterJITRegs, f25));
+ masm.as_fld_d(f26, StackPointer, offsetof(EnterJITRegs, f26));
+ masm.as_fld_d(f27, StackPointer, offsetof(EnterJITRegs, f27));
+ masm.as_fld_d(f28, StackPointer, offsetof(EnterJITRegs, f28));
+ masm.as_fld_d(f29, StackPointer, offsetof(EnterJITRegs, f29));
+ masm.as_fld_d(f30, StackPointer, offsetof(EnterJITRegs, f30));
+ masm.as_fld_d(f31, StackPointer, offsetof(EnterJITRegs, f31));
+
+ masm.freeStack(sizeof(EnterJITRegs));
+
+ masm.branch(ra);
+}
+
+static void GeneratePrologue(MacroAssembler& masm) {
+ masm.reserveStack(sizeof(EnterJITRegs));
+
+ masm.as_st_d(s0, StackPointer, offsetof(EnterJITRegs, s0));
+ masm.as_st_d(s1, StackPointer, offsetof(EnterJITRegs, s1));
+ masm.as_st_d(s2, StackPointer, offsetof(EnterJITRegs, s2));
+ masm.as_st_d(s3, StackPointer, offsetof(EnterJITRegs, s3));
+ masm.as_st_d(s4, StackPointer, offsetof(EnterJITRegs, s4));
+ masm.as_st_d(s5, StackPointer, offsetof(EnterJITRegs, s5));
+ masm.as_st_d(s6, StackPointer, offsetof(EnterJITRegs, s6));
+ masm.as_st_d(s7, StackPointer, offsetof(EnterJITRegs, s7));
+ masm.as_st_d(s8, StackPointer, offsetof(EnterJITRegs, s8));
+ masm.as_st_d(fp, StackPointer, offsetof(EnterJITRegs, fp));
+ masm.as_st_d(ra, StackPointer, offsetof(EnterJITRegs, ra));
+ masm.as_st_d(a7, StackPointer, offsetof(EnterJITRegs, a7));
+
+ masm.as_fst_d(f24, StackPointer, offsetof(EnterJITRegs, f24));
+ masm.as_fst_d(f25, StackPointer, offsetof(EnterJITRegs, f25));
+ masm.as_fst_d(f26, StackPointer, offsetof(EnterJITRegs, f26));
+ masm.as_fst_d(f27, StackPointer, offsetof(EnterJITRegs, f27));
+ masm.as_fst_d(f28, StackPointer, offsetof(EnterJITRegs, f28));
+ masm.as_fst_d(f29, StackPointer, offsetof(EnterJITRegs, f29));
+ masm.as_fst_d(f30, StackPointer, offsetof(EnterJITRegs, f30));
+ masm.as_fst_d(f31, StackPointer, offsetof(EnterJITRegs, f31));
+}
+
+// Generates a trampoline for calling Jit compiled code from a C++ function.
+// The trampoline use the EnterJitCode signature, with the standard x64 fastcall
+// calling convention.
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ const Register reg_code = IntArgReg0;
+ const Register reg_argc = IntArgReg1;
+ const Register reg_argv = IntArgReg2;
+ const mozilla::DebugOnly<Register> reg_frame = IntArgReg3;
+ const Register reg_token = IntArgReg4;
+ const Register reg_chain = IntArgReg5;
+ const Register reg_values = IntArgReg6;
+ const Register reg_vp = IntArgReg7;
+
+ MOZ_ASSERT(OsrFrameReg == reg_frame);
+
+ GeneratePrologue(masm);
+
+ // Save stack pointer as baseline frame.
+ masm.movePtr(StackPointer, FramePointer);
+
+ // Load the number of actual arguments into s3.
+ masm.unboxInt32(Address(reg_vp, 0), s3);
+
+ /***************************************************************
+ Loop over argv vector, push arguments onto stack in reverse order
+ ***************************************************************/
+
+ // if we are constructing, that also needs to include newTarget
+ {
+ Label noNewTarget;
+ masm.branchTest32(Assembler::Zero, reg_token,
+ Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+ masm.add32(Imm32(1), reg_argc);
+
+ masm.bind(&noNewTarget);
+ }
+
+ // Make stack algined
+ masm.ma_and(s0, reg_argc, Imm32(1));
+ masm.ma_sub_d(s1, zero, Imm32(sizeof(Value)));
+ masm.as_maskeqz(s1, s1, s0);
+ masm.as_add_d(StackPointer, StackPointer, s1);
+
+ masm.as_slli_d(s0, reg_argc, 3); // Value* argv
+ masm.addPtr(reg_argv, s0); // s0 = &argv[argc]
+
+ // Loop over arguments, copying them from an unknown buffer onto the Ion
+ // stack so they can be accessed from JIT'ed code.
+ Label header, footer;
+ // If there aren't any arguments, don't do anything
+ masm.ma_b(s0, reg_argv, &footer, Assembler::BelowOrEqual, ShortJump);
+ {
+ masm.bind(&header);
+
+ masm.subPtr(Imm32(sizeof(Value)), s0);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+
+ ValueOperand value = ValueOperand(s6);
+ masm.loadValue(Address(s0, 0), value);
+ masm.storeValue(value, Address(StackPointer, 0));
+
+ masm.ma_b(s0, reg_argv, &header, Assembler::Above, ShortJump);
+ }
+ masm.bind(&footer);
+
+ masm.push(reg_token);
+ masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, s3, s3);
+
+ CodeLabel returnLabel;
+ Label oomReturnLabel;
+ {
+ // Handle Interpreter -> Baseline OSR.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.take(OsrFrameReg);
+ regs.take(reg_code);
+ MOZ_ASSERT(!regs.has(ReturnReg), "ReturnReg matches reg_code");
+
+ Label notOsr;
+ masm.ma_b(OsrFrameReg, OsrFrameReg, &notOsr, Assembler::Zero, ShortJump);
+
+ Register numStackValues = reg_values;
+ regs.take(numStackValues);
+ Register scratch = regs.takeAny();
+
+ // Push return address.
+ masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ masm.ma_li(scratch, &returnLabel);
+ masm.storePtr(scratch, Address(StackPointer, 0));
+
+ // Push previous frame pointer.
+ masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ masm.storePtr(FramePointer, Address(StackPointer, 0));
+
+ // Reserve frame.
+ Register framePtr = FramePointer;
+ masm.movePtr(StackPointer, framePtr);
+ masm.subPtr(Imm32(BaselineFrame::Size()), StackPointer);
+
+ Register framePtrScratch = regs.takeAny();
+ masm.movePtr(sp, framePtrScratch);
+
+ // Reserve space for locals and stack values.
+ masm.as_slli_d(scratch, numStackValues, 3);
+ masm.subPtr(scratch, StackPointer);
+
+ // Enter exit frame.
+ masm.reserveStack(3 * sizeof(uintptr_t));
+ masm.storePtr(
+ ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)),
+ Address(StackPointer, 2 * sizeof(uintptr_t))); // Frame descriptor
+ masm.storePtr(
+ zero, Address(StackPointer, sizeof(uintptr_t))); // fake return address
+ masm.storePtr(FramePointer, Address(StackPointer, 0));
+
+ // No GC things to mark, push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ masm.reserveStack(2 * sizeof(uintptr_t));
+ masm.storePtr(framePtr,
+ Address(StackPointer, sizeof(uintptr_t))); // BaselineFrame
+ masm.storePtr(reg_code, Address(StackPointer, 0)); // jitcode
+
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(framePtrScratch); // BaselineFrame
+ masm.passABIArg(OsrFrameReg); // InterpreterFrame
+ masm.passABIArg(numStackValues);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ regs.add(OsrFrameReg);
+ Register jitcode = regs.takeAny();
+ masm.loadPtr(Address(StackPointer, 0), jitcode);
+ masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), framePtr);
+ masm.freeStack(2 * sizeof(uintptr_t));
+
+ Label error;
+ masm.freeStack(ExitFrameLayout::SizeWithFooter());
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.profilerEnterFrame(framePtr, scratch);
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+ masm.jump(jitcode);
+
+ // OOM: load error value, discard return address and previous frame
+ // pointer and return.
+ masm.bind(&error);
+ masm.movePtr(framePtr, StackPointer);
+ masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.jump(&oomReturnLabel);
+
+ masm.bind(&notOsr);
+ // Load the scope chain in R1.
+ MOZ_ASSERT(R1.scratchReg() != reg_code);
+ masm.as_or(R1.scratchReg(), reg_chain, zero);
+ }
+
+ // The call will push the return address and frame pointer on the stack, thus
+ // we check that the stack would be aligned once the call is complete.
+ masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+ // Call the function with pushing return address to stack.
+ masm.callJitNoProfiler(reg_code);
+
+ {
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&returnLabel);
+ masm.addCodeLabel(returnLabel);
+ masm.bind(&oomReturnLabel);
+ }
+
+ // Discard arguments and padding. Set sp to the address of the EnterJITRegs
+ // on the stack.
+ masm.mov(FramePointer, StackPointer);
+
+ // Store the returned value into the vp
+ masm.as_ld_d(reg_vp, StackPointer, offsetof(EnterJITRegs, a7));
+ masm.storeValue(JSReturnOperand, Address(reg_vp, 0));
+
+ // Restore non-volatile registers and return.
+ GenerateReturn(masm, ShortJump);
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ // Not supported, or not implemented yet.
+ // TODO: Implement along with the corresponding stack-walker changes, in
+ // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
+ return mozilla::Nothing{};
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // Stack has to be alligned here. If not, we will have to fix it.
+ masm.checkStackAlignment();
+
+ // Push registers such that we can access them from [base + code].
+ masm.PushRegsInMask(AllRegs);
+
+ // Pass pointer to InvalidationBailoutStack structure.
+ masm.movePtr(StackPointer, a0);
+
+ // Reserve place for BailoutInfo pointer. Two words to ensure alignment for
+ // setupAlignedABICall.
+ masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ // Pass pointer to BailoutInfo
+ masm.movePtr(StackPointer, a1);
+
+ using Fn =
+ bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupAlignedABICall();
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.callWithABI<Fn, InvalidationBailout>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(a2);
+
+ // Pop the machine state and the dead frame.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind) {
+ // Do not erase the frame pointer in this function.
+
+ AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+ masm.pushReturnAddress();
+ // Caller:
+ // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- sp
+
+ // Frame prologue.
+ //
+ // NOTE: if this changes, fix the Baseline bailout code too!
+ // See BaselineStackBuilder::calculatePrevFramePtr and
+ // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+ masm.push(FramePointer);
+ masm.mov(StackPointer, FramePointer);
+
+ // Load argc.
+ masm.loadNumActualArgs(FramePointer, s3);
+
+ Register numActArgsReg = a6;
+ Register calleeTokenReg = a7;
+ Register numArgsReg = a5;
+
+ // Load |nformals| into numArgsReg.
+ masm.loadPtr(
+ Address(FramePointer, RectifierFrameLayout::offsetOfCalleeToken()),
+ calleeTokenReg);
+ masm.mov(calleeTokenReg, numArgsReg);
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), numArgsReg);
+ masm.loadFunctionArgCount(numArgsReg, numArgsReg);
+
+ // Stash another copy in t3, since we are going to do destructive operations
+ // on numArgsReg
+ masm.mov(numArgsReg, t3);
+
+ static_assert(
+ CalleeToken_FunctionConstructing == 1,
+ "Ensure that we can use the constructing bit to count the value");
+ masm.mov(calleeTokenReg, t2);
+ masm.ma_and(t2, t2, Imm32(uint32_t(CalleeToken_FunctionConstructing)));
+
+ // Including |this|, and |new.target|, there are (|nformals| + 1 +
+ // isConstructing) arguments to push to the stack. Then we push a
+ // JitFrameLayout. We compute the padding expressed in the number of extra
+ // |undefined| values to push on the stack.
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ static_assert(
+ JitStackAlignment % sizeof(Value) == 0,
+ "Ensure that we can pad the stack by pushing extra UndefinedValue");
+
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(JitStackValueAlignment));
+ masm.add32(
+ Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */),
+ numArgsReg);
+ masm.add32(t2, numArgsReg);
+ masm.and32(Imm32(~(JitStackValueAlignment - 1)), numArgsReg);
+
+ // Load the number of |undefined|s to push into t1. Subtract 1 for |this|.
+ masm.as_sub_d(t1, numArgsReg, s3);
+ masm.sub32(Imm32(1), t1);
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- sp
+ // '--- s3 ----'
+ //
+ // Rectifier frame:
+ // [fp'] [undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee]
+ // [descr] [raddr] ]
+ // '-------- t1 ---------' '--- s3 ----'
+
+ // Copy number of actual arguments into numActArgsReg.
+ masm.mov(s3, numActArgsReg);
+
+ masm.moveValue(UndefinedValue(), ValueOperand(t0));
+
+ // Push undefined. (including the padding)
+ {
+ Label undefLoopTop;
+
+ masm.bind(&undefLoopTop);
+ masm.sub32(Imm32(1), t1);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.storeValue(ValueOperand(t0), Address(StackPointer, 0));
+
+ masm.ma_b(t1, t1, &undefLoopTop, Assembler::NonZero, ShortJump);
+ }
+
+ // Get the topmost argument.
+ static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments");
+
+ // Get the topmost argument.
+ masm.as_slli_d(t0, s3, 3); // t0 <- nargs * 8
+ masm.as_add_d(t1, FramePointer, t0); // t1 <- fp(saved sp) + nargs * 8
+ masm.addPtr(Imm32(sizeof(RectifierFrameLayout)), t1);
+
+ // Push arguments, |nargs| + 1 times (to include |this|).
+ masm.addPtr(Imm32(1), s3);
+ {
+ Label copyLoopTop;
+
+ masm.bind(&copyLoopTop);
+ masm.sub32(Imm32(1), s3);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.loadValue(Address(t1, 0), ValueOperand(t0));
+ masm.storeValue(ValueOperand(t0), Address(StackPointer, 0));
+ masm.subPtr(Imm32(sizeof(Value)), t1);
+
+ masm.ma_b(s3, s3, &copyLoopTop, Assembler::NonZero, ShortJump);
+ }
+
+ // if constructing, copy newTarget
+ {
+ Label notConstructing;
+
+ masm.branchTest32(Assembler::Zero, calleeTokenReg,
+ Imm32(CalleeToken_FunctionConstructing),
+ &notConstructing);
+
+ // thisFrame[numFormals] = prevFrame[argc]
+ ValueOperand newTarget(t0);
+
+ // Load vp[argc]. Add sizeof(Value) for |this|.
+ BaseIndex newTargetSrc(FramePointer, numActArgsReg, TimesEight,
+ sizeof(RectifierFrameLayout) + sizeof(Value));
+ masm.loadValue(newTargetSrc, newTarget);
+
+ // Again, 1 for |this|
+ BaseIndex newTargetDest(StackPointer, t3, TimesEight, sizeof(Value));
+ masm.storeValue(newTarget, newTargetDest);
+
+ masm.bind(&notConstructing);
+ }
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ]
+ //
+ //
+ // Rectifier frame:
+ // [fp'] <- fp [undef] [undef] [undef] [arg2] [arg1] [this] <- sp [ [argc]
+ // [callee] [descr] [raddr] ]
+ //
+
+ // Construct JitFrameLayout.
+ masm.push(calleeTokenReg);
+ masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, numActArgsReg,
+ numActArgsReg);
+
+ // Call the target function.
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), calleeTokenReg);
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(calleeTokenReg, t1);
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(t1);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(calleeTokenReg, t1, &noBaselineScript);
+ masm.callJitNoProfiler(t1);
+ masm.jump(&done);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(calleeTokenReg, t1);
+ masm.callJitNoProfiler(t1);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.mov(FramePointer, StackPointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+/* - When bailout is done via out of line code (lazy bailout).
+ * Frame size is stored in $ra (look at
+ * CodeGeneratorLOONG64::generateOutOfLineCode()) and thunk code should save it
+ * on stack. Other difference is that members snapshotOffset_ and padding_ are
+ * pushed to the stack by CodeGeneratorLOONG64::visitOutOfLineBailout().
+ */
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+ // Push the frameSize_ stored in ra
+ // See: CodeGeneratorLOONG64::generateOutOfLineCode()
+ masm.push(ra);
+
+ // Push registers such that we can access them from [base + code].
+ masm.PushRegsInMask(AllRegs);
+
+ // Put pointer to BailoutStack as first argument to the Bailout()
+ masm.movePtr(StackPointer, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+ PushBailoutFrame(masm, a0);
+
+ // Make space for Bailout's bailoutInfo outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.movePtr(StackPointer, a1);
+
+ // Call the bailout function.
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(a2);
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ // Get the bailoutInfo outparam.
+ masm.pop(a2);
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in a2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+ *wrapperOffset = startTrampolineCode(masm);
+
+ // Avoid conflicts with argument registers while discarding the result after
+ // the function call.
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert(
+ (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set should be a superset of Volatile register set.");
+
+ // The context is the first argument; a0 is the first argument register.
+ Register cxreg = a0;
+ regs.take(cxreg);
+
+ // If it isn't a tail call, then the return address needs to be saved
+ if (f.expectTailCall == NonTailCall) {
+ masm.pushReturnAddress();
+ }
+
+ // Push the frame pointer to finish the exit frame, then link it up.
+ masm.Push(FramePointer);
+ masm.loadJSContext(cxreg);
+ masm.enterExitFrame(cxreg, regs.getAny(), &f);
+
+ // Save the base of the argument set stored on the stack.
+ Register argsBase = InvalidReg;
+ if (f.explicitArgs) {
+ argsBase = t1; // Use temporary register.
+ regs.take(argsBase);
+ masm.ma_add_d(argsBase, StackPointer,
+ Imm32(ExitFrameLayout::SizeWithFooter()));
+ }
+
+ // Reserve space for the outparameter.
+ Register outReg = InvalidReg;
+ switch (f.outParam) {
+ case Type_Value:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(Value));
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ case Type_Handle:
+ outReg = regs.takeAny();
+ masm.PushEmptyRooted(f.outParamRootType);
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ case Type_Bool:
+ case Type_Int32:
+ outReg = regs.takeAny();
+ // Reserve 4-byte space to make stack aligned to 8-byte.
+ masm.reserveStack(2 * sizeof(int32_t));
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ case Type_Pointer:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(uintptr_t));
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ case Type_Double:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(double));
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ masm.setupUnalignedABICall(regs.getAny());
+ masm.passABIArg(cxreg);
+
+ size_t argDisp = 0;
+
+ // Copy any arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ if (f.argPassedInFloatReg(explicitArg)) {
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
+ } else {
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ }
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByValue:
+ case VMFunctionData::DoubleByRef:
+ MOZ_CRASH(
+ "NYI: LOONG64 callVM should not be used with 128bits values.");
+ break;
+ }
+ }
+
+ // Copy the implicit outparam, if any.
+ if (InvalidReg != outReg) {
+ masm.passABIArg(outReg);
+ }
+
+ masm.callWithABI(nativeFun, MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, a0, a0, masm.failureLabel());
+ break;
+ case Type_Bool:
+ // Called functions return bools, which are 0/false and non-zero/true
+ masm.branchIfFalseBool(a0, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ // Load the outparam and free any allocated stack.
+ switch (f.outParam) {
+ case Type_Handle:
+ masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+ break;
+
+ case Type_Value:
+ masm.loadValue(Address(StackPointer, 0), JSReturnOperand);
+ masm.freeStack(sizeof(Value));
+ break;
+
+ case Type_Int32:
+ masm.load32(Address(StackPointer, 0), ReturnReg);
+ masm.freeStack(2 * sizeof(int32_t));
+ break;
+
+ case Type_Pointer:
+ masm.loadPtr(Address(StackPointer, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ case Type_Bool:
+ masm.load8ZeroExtend(Address(StackPointer, 0), ReturnReg);
+ masm.freeStack(2 * sizeof(int32_t));
+ break;
+
+ case Type_Double:
+ masm.as_fld_d(ReturnDoubleReg, StackPointer, 0);
+ masm.freeStack(sizeof(double));
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ // Pop ExitFooterFrame and the frame pointer.
+ masm.leaveExitFrame(sizeof(void*));
+
+ // Return. Subtract sizeof(void*) for the frame pointer.
+ masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+ f.explicitStackSlots() * sizeof(void*) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type) {
+ AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+ uint32_t offset = startTrampolineCode(masm);
+
+ MOZ_ASSERT(PreBarrierReg == a1);
+ Register temp1 = a0;
+ Register temp2 = a2;
+ Register temp3 = a3;
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+ &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ LiveRegisterSet save;
+ save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileMask));
+ masm.push(ra);
+ masm.PushRegsInMask(save);
+
+ masm.movePtr(ImmPtr(cx->runtime()), a0);
+
+ masm.setupUnalignedABICall(a2);
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.callWithABI(JitPreWriteBarrier(type));
+
+ masm.PopRegsInMask(save);
+ masm.ret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.abiret();
+
+ return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+ masm.bind(bailoutTail);
+ masm.generateBailoutTail(a1, a2);
+}
diff --git a/js/src/jit/mips-shared/Architecture-mips-shared.cpp b/js/src/jit/mips-shared/Architecture-mips-shared.cpp
new file mode 100644
index 0000000000..fb28b298ea
--- /dev/null
+++ b/js/src/jit/mips-shared/Architecture-mips-shared.cpp
@@ -0,0 +1,121 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips-shared/Architecture-mips-shared.h"
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "jit/FlushICache.h" // js::jit::FlushICache
+#include "jit/mips32/Simulator-mips32.h"
+#include "jit/mips64/Simulator-mips64.h"
+#include "jit/RegisterSets.h"
+
+#if defined(__linux__) && !defined(JS_SIMULATOR)
+# include <sys/cachectl.h>
+#endif
+
+#define HWCAP_MIPS (1 << 28)
+#define HWCAP_LOONGSON (1 << 27)
+#define HWCAP_R2 (1 << 26)
+#define HWCAP_FPU (1 << 0)
+
+namespace js {
+namespace jit {
+
+static uint32_t get_mips_flags() {
+ uint32_t flags = HWCAP_MIPS;
+
+#if defined(JS_SIMULATOR_MIPS32) || defined(JS_SIMULATOR_MIPS64)
+ flags |= HWCAP_FPU;
+ flags |= HWCAP_R2;
+#else
+# ifdef __linux__
+ FILE* fp = fopen("/proc/cpuinfo", "r");
+ if (!fp) {
+ return flags;
+ }
+
+ char buf[1024] = {};
+ (void)fread(buf, sizeof(char), sizeof(buf) - 1, fp);
+ fclose(fp);
+ if (strstr(buf, "FPU")) {
+ flags |= HWCAP_FPU;
+ }
+ if (strstr(buf, "Loongson")) {
+ flags |= HWCAP_LOONGSON;
+ }
+ if (strstr(buf, "mips32r2") || strstr(buf, "mips64r2")) {
+ flags |= HWCAP_R2;
+ }
+# endif
+#endif // JS_SIMULATOR_MIPS32 || JS_SIMULATOR_MIPS64
+ return flags;
+}
+
+static bool check_fpu() { return mips_private::Flags & HWCAP_FPU; }
+
+static bool check_loongson() { return mips_private::Flags & HWCAP_LOONGSON; }
+
+static bool check_r2() { return mips_private::Flags & HWCAP_R2; }
+
+namespace mips_private {
+// Cache a local copy so we only have to read /proc/cpuinfo once.
+uint32_t Flags = get_mips_flags();
+bool hasFPU = check_fpu();
+;
+bool isLoongson = check_loongson();
+bool hasR2 = check_r2();
+} // namespace mips_private
+
+bool CPUFlagsHaveBeenComputed() {
+ // Flags were computed above.
+ return true;
+}
+
+Registers::Code Registers::FromName(const char* name) {
+ for (size_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(i), name) == 0) {
+ return Code(i);
+ }
+ }
+
+ return Invalid;
+}
+
+void FlushICache(void* code, size_t size) {
+#if defined(JS_SIMULATOR)
+ js::jit::SimulatorProcess::FlushICache(code, size);
+
+#elif defined(_MIPS_ARCH_LOONGSON3A)
+ // On Loongson3-CPUs, The cache flushed automatically
+ // by hardware. Just need to execute an instruction hazard.
+ uintptr_t tmp;
+ asm volatile(
+ ".set push \n"
+ ".set noreorder \n"
+ "move %[tmp], $ra \n"
+ "bal 1f \n"
+ "daddiu $ra, 8 \n"
+ "1: \n"
+ "jr.hb $ra \n"
+ "move $ra, %[tmp] \n"
+ ".set pop\n"
+ : [tmp] "=&r"(tmp));
+
+#elif defined(__GNUC__)
+ intptr_t end = reinterpret_cast<intptr_t>(code) + size;
+ __builtin___clear_cache(reinterpret_cast<char*>(code),
+ reinterpret_cast<char*>(end));
+
+#else
+ _flush_cache(reinterpret_cast<char*>(code), size, BCACHE);
+
+#endif
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/mips-shared/Architecture-mips-shared.h b/js/src/jit/mips-shared/Architecture-mips-shared.h
new file mode 100644
index 0000000000..1749a2fe6c
--- /dev/null
+++ b/js/src/jit/mips-shared/Architecture-mips-shared.h
@@ -0,0 +1,341 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_Architecture_mips_shared_h
+#define jit_mips_shared_Architecture_mips_shared_h
+
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+#include <limits.h>
+#include <stdint.h>
+
+#include "jit/shared/Architecture-shared.h"
+
+#include "js/Utility.h"
+
+// gcc appears to use _mips_hard_float to denote
+// that the target is a hard-float target.
+#ifdef _mips_hard_float
+# define JS_CODEGEN_MIPS_HARDFP
+#endif
+
+#if (defined(_MIPS_SIM) && (_MIPS_SIM == _ABIO32)) || \
+ defined(JS_SIMULATOR_MIPS32)
+# define USES_O32_ABI
+#elif (defined(_MIPS_SIM) && (_MIPS_SIM == _ABI64)) || \
+ defined(JS_SIMULATOR_MIPS64)
+# define USES_N64_ABI
+#else
+# error "Unsupported ABI"
+#endif
+
+#if (defined(__mips_isa_rev) && (__mips_isa_rev >= 6))
+# define MIPSR6
+#endif
+
+namespace js {
+namespace jit {
+
+// How far forward/back can a jump go? Provide a generous buffer for thunks.
+static const uint32_t JumpImmediateRange = UINT32_MAX;
+
+class Registers {
+ public:
+ enum RegisterID {
+ r0 = 0,
+ r1,
+ r2,
+ r3,
+ r4,
+ r5,
+ r6,
+ r7,
+ r8,
+ r9,
+ r10,
+ r11,
+ r12,
+ r13,
+ r14,
+ r15,
+ r16,
+ r17,
+ r18,
+ r19,
+ r20,
+ r21,
+ r22,
+ r23,
+ r24,
+ r25,
+ r26,
+ r27,
+ r28,
+ r29,
+ r30,
+ r31,
+ zero = r0,
+ at = r1,
+ v0 = r2,
+ v1 = r3,
+ a0 = r4,
+ a1 = r5,
+ a2 = r6,
+ a3 = r7,
+#if defined(USES_O32_ABI)
+ t0 = r8,
+ t1 = r9,
+ t2 = r10,
+ t3 = r11,
+ t4 = r12,
+ t5 = r13,
+ t6 = r14,
+ t7 = r15,
+ ta0 = t4,
+ ta1 = t5,
+ ta2 = t6,
+ ta3 = t7,
+#elif defined(USES_N64_ABI)
+ a4 = r8,
+ a5 = r9,
+ a6 = r10,
+ a7 = r11,
+ t0 = r12,
+ t1 = r13,
+ t2 = r14,
+ t3 = r15,
+ ta0 = a4,
+ ta1 = a5,
+ ta2 = a6,
+ ta3 = a7,
+#endif
+ s0 = r16,
+ s1 = r17,
+ s2 = r18,
+ s3 = r19,
+ s4 = r20,
+ s5 = r21,
+ s6 = r22,
+ s7 = r23,
+ t8 = r24,
+ t9 = r25,
+ k0 = r26,
+ k1 = r27,
+ gp = r28,
+ sp = r29,
+ fp = r30,
+ ra = r31,
+ invalid_reg
+ };
+ typedef uint8_t Code;
+ typedef RegisterID Encoding;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ uintptr_t r;
+ };
+
+ static const char* const RegNames[];
+ static const char* GetName(Code code) {
+ MOZ_ASSERT(code < Total);
+ return RegNames[code];
+ }
+ static const char* GetName(Encoding i) { return GetName(Code(i)); }
+
+ static Code FromName(const char* name);
+
+ static const Encoding StackPointer = sp;
+ static const Encoding Invalid = invalid_reg;
+
+ static const uint32_t Total = 32;
+ static const uint32_t Allocatable;
+
+ typedef uint32_t SetType;
+ static const SetType AllMask = 0xffffffff;
+ static const SetType SharedArgRegMask =
+ (1 << a0) | (1 << a1) | (1 << a2) | (1 << a3);
+ static const SetType ArgRegMask;
+
+ static const SetType VolatileMask =
+ (1 << Registers::v0) | (1 << Registers::v1) | (1 << Registers::a0) |
+ (1 << Registers::a1) | (1 << Registers::a2) | (1 << Registers::a3) |
+ (1 << Registers::t0) | (1 << Registers::t1) | (1 << Registers::t2) |
+ (1 << Registers::t3) | (1 << Registers::ta0) | (1 << Registers::ta1) |
+ (1 << Registers::ta2) | (1 << Registers::ta3);
+
+ // We use this constant to save registers when entering functions. This
+ // is why $ra is added here even though it is not "Non Volatile".
+ static const SetType NonVolatileMask =
+ (1 << Registers::s0) | (1 << Registers::s1) | (1 << Registers::s2) |
+ (1 << Registers::s3) | (1 << Registers::s4) | (1 << Registers::s5) |
+ (1 << Registers::s6) | (1 << Registers::s7) | (1 << Registers::fp) |
+ (1 << Registers::ra);
+
+ static const SetType WrapperMask = VolatileMask | // = arguments
+ (1 << Registers::t0) | // = outReg
+ (1 << Registers::t1); // = argBase
+
+ static const SetType NonAllocatableMask =
+ (1 << Registers::zero) | (1 << Registers::at) | // at = scratch
+ (1 << Registers::t8) | // t8 = scratch
+ (1 << Registers::t9) | // t9 = scratch
+ (1 << Registers::k0) | (1 << Registers::k1) | (1 << Registers::gp) |
+ (1 << Registers::sp) | (1 << Registers::ra) | (1 << Registers::fp);
+
+ // Registers returned from a JS -> JS call.
+ static const SetType JSCallMask;
+
+ // Registers returned from a JS -> C call.
+ static const SetType SharedCallMask = (1 << Registers::v0);
+ static const SetType CallMask;
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return mozilla::CountPopulation32(x);
+ }
+ static uint32_t FirstBit(SetType x) {
+ return mozilla::CountTrailingZeroes32(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ return 31 - mozilla::CountLeadingZeroes32(x);
+ }
+};
+
+// Smallest integer type that can hold a register bitmask.
+typedef uint32_t PackedRegisterMask;
+
+class FloatRegistersMIPSShared {
+ public:
+ enum FPRegisterID {
+ f0 = 0,
+ f1,
+ f2,
+ f3,
+ f4,
+ f5,
+ f6,
+ f7,
+ f8,
+ f9,
+ f10,
+ f11,
+ f12,
+ f13,
+ f14,
+ f15,
+ f16,
+ f17,
+ f18,
+ f19,
+ f20,
+ f21,
+ f22,
+ f23,
+ f24,
+ f25,
+ f26,
+ f27,
+ f28,
+ f29,
+ f30,
+ f31,
+ invalid_freg
+ };
+ typedef uint32_t Code;
+ typedef FPRegisterID Encoding;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ double d;
+ };
+
+ static const char* GetName(Encoding code) {
+ static const char* const Names[] = {
+ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7",
+ "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15",
+ "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
+ "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"};
+ return Names[code];
+ }
+
+ static const Encoding Invalid = invalid_freg;
+
+#if defined(JS_CODEGEN_MIPS32)
+ typedef uint32_t SetType;
+#elif defined(JS_CODEGEN_MIPS64)
+ typedef uint64_t SetType;
+#endif
+};
+
+static const uint32_t SpillSlotSize =
+ std::max(sizeof(Registers::RegisterContent),
+ sizeof(FloatRegistersMIPSShared::RegisterContent));
+
+template <typename T>
+class TypedRegisterSet;
+
+class FloatRegisterMIPSShared {
+ public:
+ bool isSimd128() const { return false; }
+
+ typedef FloatRegistersMIPSShared::SetType SetType;
+
+#if defined(JS_CODEGEN_MIPS32)
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return mozilla::CountPopulation32(x);
+ }
+ static uint32_t FirstBit(SetType x) {
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return mozilla::CountTrailingZeroes32(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return 31 - mozilla::CountLeadingZeroes32(x);
+ }
+#elif defined(JS_CODEGEN_MIPS64)
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
+ return mozilla::CountPopulation64(x);
+ }
+ static uint32_t FirstBit(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
+ return mozilla::CountTrailingZeroes64(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
+ return 63 - mozilla::CountLeadingZeroes64(x);
+ }
+#endif
+};
+
+namespace mips_private {
+extern uint32_t Flags;
+extern bool hasFPU;
+extern bool isLoongson;
+extern bool hasR2;
+} // namespace mips_private
+
+inline uint32_t GetMIPSFlags() { return mips_private::Flags; }
+inline bool hasFPU() { return mips_private::hasFPU; }
+inline bool isLoongson() { return mips_private::isLoongson; }
+inline bool hasR2() { return mips_private::hasR2; }
+
+// MIPS doesn't have double registers that can NOT be treated as float32.
+inline bool hasUnaliasedDouble() { return false; }
+
+// MIPS64 doesn't support it and on MIPS32 we don't allocate odd single fp
+// registers thus not exposing multi aliasing to the jit.
+// See comments in Arhitecture-mips32.h.
+inline bool hasMultiAlias() { return false; }
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_Architecture_mips_shared_h */
diff --git a/js/src/jit/mips-shared/Assembler-mips-shared.cpp b/js/src/jit/mips-shared/Assembler-mips-shared.cpp
new file mode 100644
index 0000000000..11f834c3c7
--- /dev/null
+++ b/js/src/jit/mips-shared/Assembler-mips-shared.cpp
@@ -0,0 +1,2094 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips-shared/Assembler-mips-shared.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "gc/Marking.h"
+#include "jit/ExecutableAllocator.h"
+#include "vm/Realm.h"
+
+using mozilla::DebugOnly;
+
+using namespace js;
+using namespace js::jit;
+
+// Encode a standard register when it is being used as rd, the rs, and
+// an extra register(rt). These should never be called with an InvalidReg.
+uint32_t js::jit::RS(Register r) {
+ MOZ_ASSERT((r.code() & ~RegMask) == 0);
+ return r.code() << RSShift;
+}
+
+uint32_t js::jit::RT(Register r) {
+ MOZ_ASSERT((r.code() & ~RegMask) == 0);
+ return r.code() << RTShift;
+}
+
+uint32_t js::jit::RD(Register r) {
+ MOZ_ASSERT((r.code() & ~RegMask) == 0);
+ return r.code() << RDShift;
+}
+
+uint32_t js::jit::RZ(Register r) {
+ MOZ_ASSERT((r.code() & ~RegMask) == 0);
+ return r.code() << RZShift;
+}
+
+uint32_t js::jit::SA(uint32_t value) {
+ MOZ_ASSERT(value < 32);
+ return value << SAShift;
+}
+
+uint32_t js::jit::FS(uint32_t value) {
+ MOZ_ASSERT(value < 32);
+ return value << FSShift;
+}
+
+Register js::jit::toRS(Instruction& i) {
+ return Register::FromCode((i.encode() & RSMask) >> RSShift);
+}
+
+Register js::jit::toRT(Instruction& i) {
+ return Register::FromCode((i.encode() & RTMask) >> RTShift);
+}
+
+Register js::jit::toRD(Instruction& i) {
+ return Register::FromCode((i.encode() & RDMask) >> RDShift);
+}
+
+Register js::jit::toR(Instruction& i) {
+ return Register::FromCode(i.encode() & RegMask);
+}
+
+void InstImm::extractImm16(BOffImm16* dest) { *dest = BOffImm16(*this); }
+
+void AssemblerMIPSShared::finish() {
+ MOZ_ASSERT(!isFinished);
+ isFinished = true;
+}
+
+bool AssemblerMIPSShared::appendRawCode(const uint8_t* code, size_t numBytes) {
+ return m_buffer.appendRawCode(code, numBytes);
+}
+
+bool AssemblerMIPSShared::reserve(size_t size) {
+ // This buffer uses fixed-size chunks so there's no point in reserving
+ // now vs. on-demand.
+ return !oom();
+}
+
+bool AssemblerMIPSShared::swapBuffer(wasm::Bytes& bytes) {
+ // For now, specialize to the one use case. As long as wasm::Bytes is a
+ // Vector, not a linked-list of chunks, there's not much we can do other
+ // than copy.
+ MOZ_ASSERT(bytes.empty());
+ if (!bytes.resize(bytesNeeded())) {
+ return false;
+ }
+ m_buffer.executableCopy(bytes.begin());
+ return true;
+}
+
+void AssemblerMIPSShared::copyJumpRelocationTable(uint8_t* dest) {
+ if (jumpRelocations_.length()) {
+ memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
+ }
+}
+
+void AssemblerMIPSShared::copyDataRelocationTable(uint8_t* dest) {
+ if (dataRelocations_.length()) {
+ memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
+ }
+}
+
+AssemblerMIPSShared::Condition AssemblerMIPSShared::InvertCondition(
+ Condition cond) {
+ switch (cond) {
+ case Equal:
+ return NotEqual;
+ case NotEqual:
+ return Equal;
+ case Zero:
+ return NonZero;
+ case NonZero:
+ return Zero;
+ case LessThan:
+ return GreaterThanOrEqual;
+ case LessThanOrEqual:
+ return GreaterThan;
+ case GreaterThan:
+ return LessThanOrEqual;
+ case GreaterThanOrEqual:
+ return LessThan;
+ case Above:
+ return BelowOrEqual;
+ case AboveOrEqual:
+ return Below;
+ case Below:
+ return AboveOrEqual;
+ case BelowOrEqual:
+ return Above;
+ case Signed:
+ return NotSigned;
+ case NotSigned:
+ return Signed;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+AssemblerMIPSShared::DoubleCondition AssemblerMIPSShared::InvertCondition(
+ DoubleCondition cond) {
+ switch (cond) {
+ case DoubleOrdered:
+ return DoubleUnordered;
+ case DoubleEqual:
+ return DoubleNotEqualOrUnordered;
+ case DoubleNotEqual:
+ return DoubleEqualOrUnordered;
+ case DoubleGreaterThan:
+ return DoubleLessThanOrEqualOrUnordered;
+ case DoubleGreaterThanOrEqual:
+ return DoubleLessThanOrUnordered;
+ case DoubleLessThan:
+ return DoubleGreaterThanOrEqualOrUnordered;
+ case DoubleLessThanOrEqual:
+ return DoubleGreaterThanOrUnordered;
+ case DoubleUnordered:
+ return DoubleOrdered;
+ case DoubleEqualOrUnordered:
+ return DoubleNotEqual;
+ case DoubleNotEqualOrUnordered:
+ return DoubleEqual;
+ case DoubleGreaterThanOrUnordered:
+ return DoubleLessThanOrEqual;
+ case DoubleGreaterThanOrEqualOrUnordered:
+ return DoubleLessThan;
+ case DoubleLessThanOrUnordered:
+ return DoubleGreaterThanOrEqual;
+ case DoubleLessThanOrEqualOrUnordered:
+ return DoubleGreaterThan;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+BOffImm16::BOffImm16(InstImm inst) : data(inst.encode() & Imm16Mask) {}
+
+Instruction* BOffImm16::getDest(Instruction* src) const {
+ return &src[(((int32_t)data << 16) >> 16) + 1];
+}
+
+bool AssemblerMIPSShared::oom() const {
+ return AssemblerShared::oom() || m_buffer.oom() || jumpRelocations_.oom() ||
+ dataRelocations_.oom();
+}
+
+// Size of the instruction stream, in bytes.
+size_t AssemblerMIPSShared::size() const { return m_buffer.size(); }
+
+// Size of the relocation table, in bytes.
+size_t AssemblerMIPSShared::jumpRelocationTableBytes() const {
+ return jumpRelocations_.length();
+}
+
+size_t AssemblerMIPSShared::dataRelocationTableBytes() const {
+ return dataRelocations_.length();
+}
+
+// Size of the data table, in bytes.
+size_t AssemblerMIPSShared::bytesNeeded() const {
+ return size() + jumpRelocationTableBytes() + dataRelocationTableBytes();
+}
+
+// write a blob of binary into the instruction stream
+BufferOffset AssemblerMIPSShared::writeInst(uint32_t x, uint32_t* dest) {
+ MOZ_ASSERT(hasCreator());
+ if (dest == nullptr) {
+ return m_buffer.putInt(x);
+ }
+
+ WriteInstStatic(x, dest);
+ return BufferOffset();
+}
+
+void AssemblerMIPSShared::WriteInstStatic(uint32_t x, uint32_t* dest) {
+ MOZ_ASSERT(dest != nullptr);
+ *dest = x;
+}
+
+BufferOffset AssemblerMIPSShared::haltingAlign(int alignment) {
+ // TODO: Implement a proper halting align.
+ return nopAlign(alignment);
+}
+
+BufferOffset AssemblerMIPSShared::nopAlign(int alignment) {
+ BufferOffset ret;
+ MOZ_ASSERT(m_buffer.isAligned(4));
+ if (alignment == 8) {
+ if (!m_buffer.isAligned(alignment)) {
+ BufferOffset tmp = as_nop();
+ if (!ret.assigned()) {
+ ret = tmp;
+ }
+ }
+ } else {
+ MOZ_ASSERT((alignment & (alignment - 1)) == 0);
+ while (size() & (alignment - 1)) {
+ BufferOffset tmp = as_nop();
+ if (!ret.assigned()) {
+ ret = tmp;
+ }
+ }
+ }
+ return ret;
+}
+
+BufferOffset AssemblerMIPSShared::as_nop() {
+ spew("nop");
+ return writeInst(op_special | ff_sll);
+}
+
+// Logical operations.
+BufferOffset AssemblerMIPSShared::as_and(Register rd, Register rs,
+ Register rt) {
+ spew("and %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_and).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_or(Register rd, Register rs, Register rt) {
+ spew("or %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_or).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_xor(Register rd, Register rs,
+ Register rt) {
+ spew("xor %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_xor).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_nor(Register rd, Register rs,
+ Register rt) {
+ spew("nor %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_nor).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_andi(Register rd, Register rs, int32_t j) {
+ MOZ_ASSERT(Imm16::IsInUnsignedRange(j));
+ spew("andi %3s,%3s,0x%x", rd.name(), rs.name(), j);
+ return writeInst(InstImm(op_andi, rs, rd, Imm16(j)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ori(Register rd, Register rs, int32_t j) {
+ MOZ_ASSERT(Imm16::IsInUnsignedRange(j));
+ spew("ori %3s,%3s,0x%x", rd.name(), rs.name(), j);
+ return writeInst(InstImm(op_ori, rs, rd, Imm16(j)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_xori(Register rd, Register rs, int32_t j) {
+ MOZ_ASSERT(Imm16::IsInUnsignedRange(j));
+ spew("xori %3s,%3s,0x%x", rd.name(), rs.name(), j);
+ return writeInst(InstImm(op_xori, rs, rd, Imm16(j)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lui(Register rd, int32_t j) {
+ MOZ_ASSERT(Imm16::IsInUnsignedRange(j));
+ spew("lui %3s,0x%x", rd.name(), j);
+ return writeInst(InstImm(op_lui, zero, rd, Imm16(j)).encode());
+}
+
+// Branch and jump instructions
+BufferOffset AssemblerMIPSShared::as_bal(BOffImm16 off) {
+ spew("bal %d", off.decode());
+ BufferOffset bo =
+ writeInst(InstImm(op_regimm, zero, rt_bgezal, off).encode());
+ return bo;
+}
+
+BufferOffset AssemblerMIPSShared::as_b(BOffImm16 off) {
+ spew("b %d", off.decode());
+ BufferOffset bo = writeInst(InstImm(op_beq, zero, zero, off).encode());
+ return bo;
+}
+
+InstImm AssemblerMIPSShared::getBranchCode(JumpOrCall jumpOrCall) {
+ if (jumpOrCall == BranchIsCall) {
+ return InstImm(op_regimm, zero, rt_bgezal, BOffImm16(0));
+ }
+
+ return InstImm(op_beq, zero, zero, BOffImm16(0));
+}
+
+InstImm AssemblerMIPSShared::getBranchCode(Register s, Register t,
+ Condition c) {
+ MOZ_ASSERT(c == AssemblerMIPSShared::Equal ||
+ c == AssemblerMIPSShared::NotEqual);
+ return InstImm(c == AssemblerMIPSShared::Equal ? op_beq : op_bne, s, t,
+ BOffImm16(0));
+}
+
+InstImm AssemblerMIPSShared::getBranchCode(Register s, Condition c) {
+ switch (c) {
+ case AssemblerMIPSShared::Equal:
+ case AssemblerMIPSShared::Zero:
+ case AssemblerMIPSShared::BelowOrEqual:
+ return InstImm(op_beq, s, zero, BOffImm16(0));
+ case AssemblerMIPSShared::NotEqual:
+ case AssemblerMIPSShared::NonZero:
+ case AssemblerMIPSShared::Above:
+ return InstImm(op_bne, s, zero, BOffImm16(0));
+ case AssemblerMIPSShared::GreaterThan:
+ return InstImm(op_bgtz, s, zero, BOffImm16(0));
+ case AssemblerMIPSShared::GreaterThanOrEqual:
+ case AssemblerMIPSShared::NotSigned:
+ return InstImm(op_regimm, s, rt_bgez, BOffImm16(0));
+ case AssemblerMIPSShared::LessThan:
+ case AssemblerMIPSShared::Signed:
+ return InstImm(op_regimm, s, rt_bltz, BOffImm16(0));
+ case AssemblerMIPSShared::LessThanOrEqual:
+ return InstImm(op_blez, s, zero, BOffImm16(0));
+ default:
+ MOZ_CRASH("Condition not supported.");
+ }
+}
+
+InstImm AssemblerMIPSShared::getBranchCode(FloatTestKind testKind,
+ FPConditionBit fcc) {
+ MOZ_ASSERT(!(fcc && FccMask));
+#ifdef MIPSR6
+ RSField rsField = ((testKind == TestForTrue ? rs_t : rs_f));
+
+ return InstImm(op_cop1, rsField, FloatRegisters::f24 << 16, BOffImm16(0));
+#else
+ uint32_t rtField = ((testKind == TestForTrue ? 1 : 0) | (fcc << FccShift))
+ << RTShift;
+
+ return InstImm(op_cop1, rs_bc1, rtField, BOffImm16(0));
+#endif
+}
+
+BufferOffset AssemblerMIPSShared::as_j(JOffImm26 off) {
+ spew("j 0x%x", off.decode());
+ BufferOffset bo = writeInst(InstJump(op_j, off).encode());
+ return bo;
+}
+BufferOffset AssemblerMIPSShared::as_jal(JOffImm26 off) {
+ spew("jal 0x%x", off.decode());
+ BufferOffset bo = writeInst(InstJump(op_jal, off).encode());
+ return bo;
+}
+
+BufferOffset AssemblerMIPSShared::as_jr(Register rs) {
+ spew("jr %3s", rs.name());
+#ifdef MIPSR6
+ BufferOffset bo =
+ writeInst(InstReg(op_special, rs, zero, zero, ff_jalr).encode());
+#else
+ BufferOffset bo =
+ writeInst(InstReg(op_special, rs, zero, zero, ff_jr).encode());
+#endif
+ return bo;
+}
+BufferOffset AssemblerMIPSShared::as_jalr(Register rs) {
+ spew("jalr %3s", rs.name());
+ BufferOffset bo =
+ writeInst(InstReg(op_special, rs, zero, ra, ff_jalr).encode());
+ return bo;
+}
+
+// Arithmetic instructions
+BufferOffset AssemblerMIPSShared::as_addu(Register rd, Register rs,
+ Register rt) {
+ spew("addu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_addu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_addiu(Register rd, Register rs,
+ int32_t j) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(j));
+ spew("addiu %3s,%3s,0x%x", rd.name(), rs.name(), j);
+ return writeInst(InstImm(op_addiu, rs, rd, Imm16(j)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_daddu(Register rd, Register rs,
+ Register rt) {
+ spew("daddu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_daddu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_daddiu(Register rd, Register rs,
+ int32_t j) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(j));
+ spew("daddiu %3s,%3s,0x%x", rd.name(), rs.name(), j);
+ return writeInst(InstImm(op_daddiu, rs, rd, Imm16(j)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_subu(Register rd, Register rs,
+ Register rt) {
+ spew("subu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_subu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsubu(Register rd, Register rs,
+ Register rt) {
+ spew("dsubu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_dsubu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mult(Register rs, Register rt) {
+ spew("mult %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, ff_mult).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_multu(Register rs, Register rt) {
+ spew("multu %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, ff_multu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmult(Register rs, Register rt) {
+ spew("dmult %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, ff_dmult).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmultu(Register rs, Register rt) {
+ spew("dmultu %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, ff_dmultu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_div(Register rs, Register rt) {
+ spew("div %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, ff_div).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_div(Register rd, Register rs,
+ Register rt) {
+ spew("div %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x2, ff_div).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_divu(Register rs, Register rt) {
+ spew("divu %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, ff_divu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_divu(Register rd, Register rs,
+ Register rt) {
+ spew("divu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x2, ff_divu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mod(Register rd, Register rs,
+ Register rt) {
+ spew("mod %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x3, ff_mod).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_modu(Register rd, Register rs,
+ Register rt) {
+ spew("modu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x3, ff_modu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ddiv(Register rs, Register rt) {
+ spew("ddiv %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, ff_ddiv).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ddiv(Register rd, Register rs,
+ Register rt) {
+ spew("ddiv %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x2, ff_ddiv).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ddivu(Register rs, Register rt) {
+ spew("ddivu %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, ff_ddivu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ddivu(Register rd, Register rs,
+ Register rt) {
+ spew("ddivu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x2, ff_ddivu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mul(Register rd, Register rs,
+ Register rt) {
+ spew("mul %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+#ifdef MIPSR6
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x2, ff_mul).encode());
+#else
+ return writeInst(InstReg(op_special2, rs, rt, rd, ff_mul).encode());
+#endif
+}
+
+BufferOffset AssemblerMIPSShared::as_muh(Register rd, Register rs,
+ Register rt) {
+ spew("muh %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x3, ff_muh).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mulu(Register rd, Register rs,
+ Register rt) {
+ spew("mulu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x2, ff_mulu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_muhu(Register rd, Register rs,
+ Register rt) {
+ spew("muhu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x3, ff_muhu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmul(Register rd, Register rs,
+ Register rt) {
+ spew("dmul %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x2, ff_dmul).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmuh(Register rd, Register rs,
+ Register rt) {
+ spew("dmuh %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x3, ff_dmuh).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmulu(Register rd, Register rt,
+ Register rs) {
+ spew("dmulu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x2, ff_dmulu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmuhu(Register rd, Register rt,
+ Register rs) {
+ spew("dmuhu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x3, ff_dmuhu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmod(Register rd, Register rs,
+ Register rt) {
+ spew("dmod %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x3, ff_dmod).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmodu(Register rd, Register rs,
+ Register rt) {
+ spew("dmodu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x3, ff_dmodu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_madd(Register rs, Register rt) {
+ spew("madd %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special2, rs, rt, ff_madd).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_maddu(Register rs, Register rt) {
+ spew("maddu %3s,%3s", rs.name(), rt.name());
+ return writeInst(InstReg(op_special2, rs, rt, ff_maddu).encode());
+}
+
+// Shift instructions
+BufferOffset AssemblerMIPSShared::as_sll(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(sa < 32);
+ spew("sll %3s,%3s, 0x%x", rd.name(), rt.name(), sa);
+ return writeInst(InstReg(op_special, rs_zero, rt, rd, sa, ff_sll).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsll(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(sa < 32);
+ spew("dsll %3s,%3s, 0x%x", rd.name(), rt.name(), sa);
+ return writeInst(InstReg(op_special, rs_zero, rt, rd, sa, ff_dsll).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsll32(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(31 < sa && sa < 64);
+ spew("dsll32 %3s,%3s, 0x%x", rd.name(), rt.name(), sa - 32);
+ return writeInst(
+ InstReg(op_special, rs_zero, rt, rd, sa - 32, ff_dsll32).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sllv(Register rd, Register rt,
+ Register rs) {
+ spew("sllv %3s,%3s,%3s", rd.name(), rt.name(), rs.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_sllv).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsllv(Register rd, Register rt,
+ Register rs) {
+ spew("dsllv %3s,%3s,%3s", rd.name(), rt.name(), rs.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_dsllv).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_srl(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(sa < 32);
+ spew("srl %3s,%3s, 0x%x", rd.name(), rt.name(), sa);
+ return writeInst(InstReg(op_special, rs_zero, rt, rd, sa, ff_srl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsrl(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(sa < 32);
+ spew("dsrl %3s,%3s, 0x%x", rd.name(), rt.name(), sa);
+ return writeInst(InstReg(op_special, rs_zero, rt, rd, sa, ff_dsrl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsrl32(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(31 < sa && sa < 64);
+ spew("dsrl32 %3s,%3s, 0x%x", rd.name(), rt.name(), sa - 32);
+ return writeInst(
+ InstReg(op_special, rs_zero, rt, rd, sa - 32, ff_dsrl32).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_srlv(Register rd, Register rt,
+ Register rs) {
+ spew("srlv %3s,%3s,%3s", rd.name(), rt.name(), rs.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_srlv).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsrlv(Register rd, Register rt,
+ Register rs) {
+ spew("dsrlv %3s,%3s,%3s", rd.name(), rt.name(), rs.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_dsrlv).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sra(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(sa < 32);
+ spew("sra %3s,%3s, 0x%x", rd.name(), rt.name(), sa);
+ return writeInst(InstReg(op_special, rs_zero, rt, rd, sa, ff_sra).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsra(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(sa < 32);
+ spew("dsra %3s,%3s, 0x%x", rd.name(), rt.name(), sa);
+ return writeInst(InstReg(op_special, rs_zero, rt, rd, sa, ff_dsra).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsra32(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(31 < sa && sa < 64);
+ spew("dsra32 %3s,%3s, 0x%x", rd.name(), rt.name(), sa - 32);
+ return writeInst(
+ InstReg(op_special, rs_zero, rt, rd, sa - 32, ff_dsra32).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_srav(Register rd, Register rt,
+ Register rs) {
+ spew("srav %3s,%3s,%3s", rd.name(), rt.name(), rs.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_srav).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsrav(Register rd, Register rt,
+ Register rs) {
+ spew("dsrav %3s,%3s,%3s", rd.name(), rt.name(), rs.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_dsrav).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_rotr(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(sa < 32);
+ spew("rotr %3s,%3s, 0x%x", rd.name(), rt.name(), sa);
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special, rs_one, rt, rd, sa, ff_srl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_drotr(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(sa < 32);
+ spew("drotr %3s,%3s, 0x%x", rd.name(), rt.name(), sa);
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special, rs_one, rt, rd, sa, ff_dsrl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_drotr32(Register rd, Register rt,
+ uint16_t sa) {
+ MOZ_ASSERT(31 < sa && sa < 64);
+ spew("drotr32%3s,%3s, 0x%x", rd.name(), rt.name(), sa - 32);
+ MOZ_ASSERT(hasR2());
+ return writeInst(
+ InstReg(op_special, rs_one, rt, rd, sa - 32, ff_dsrl32).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_rotrv(Register rd, Register rt,
+ Register rs) {
+ spew("rotrv %3s,%3s,%3s", rd.name(), rt.name(), rs.name());
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special, rs, rt, rd, 1, ff_srlv).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_drotrv(Register rd, Register rt,
+ Register rs) {
+ spew("drotrv %3s,%3s,%3s", rd.name(), rt.name(), rs.name());
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special, rs, rt, rd, 1, ff_dsrlv).encode());
+}
+
+// Load and store instructions
+BufferOffset AssemblerMIPSShared::as_lb(Register rd, Register rs, int16_t off) {
+ spew("lb %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_lb, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lbu(Register rd, Register rs,
+ int16_t off) {
+ spew("lbu %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_lbu, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lh(Register rd, Register rs, int16_t off) {
+ spew("lh %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_lh, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lhu(Register rd, Register rs,
+ int16_t off) {
+ spew("lhu %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_lhu, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lw(Register rd, Register rs, int16_t off) {
+ spew("lw %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_lw, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lwu(Register rd, Register rs,
+ int16_t off) {
+ spew("lwu %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_lwu, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lwl(Register rd, Register rs,
+ int16_t off) {
+ spew("lwl %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_lwl, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lwr(Register rd, Register rs,
+ int16_t off) {
+ spew("lwr %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_lwr, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ll(Register rd, Register rs, int16_t off) {
+ spew("ll %3s, (0x%x)%2s", rd.name(), off, rs.name());
+#ifdef MIPSR6
+ return writeInst(InstReg(op_special3, rs, rd, ff_ll).encode());
+#else
+ return writeInst(InstImm(op_ll, rs, rd, Imm16(off)).encode());
+#endif
+}
+
+BufferOffset AssemblerMIPSShared::as_lld(Register rd, Register rs,
+ int16_t off) {
+ spew("lld %3s, (0x%x)%2s", rd.name(), off, rs.name());
+#ifdef MIPSR6
+ return writeInst(InstReg(op_special3, rs, rd, ff_lld).encode());
+#else
+ return writeInst(InstImm(op_lld, rs, rd, Imm16(off)).encode());
+#endif
+}
+
+BufferOffset AssemblerMIPSShared::as_ld(Register rd, Register rs, int16_t off) {
+ spew("ld %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_ld, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ldl(Register rd, Register rs,
+ int16_t off) {
+ spew("ldl %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_ldl, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ldr(Register rd, Register rs,
+ int16_t off) {
+ spew("ldr %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_ldr, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sb(Register rd, Register rs, int16_t off) {
+ spew("sb %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_sb, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sh(Register rd, Register rs, int16_t off) {
+ spew("sh %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_sh, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sw(Register rd, Register rs, int16_t off) {
+ spew("sw %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_sw, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_swl(Register rd, Register rs,
+ int16_t off) {
+ spew("swl %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_swl, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_swr(Register rd, Register rs,
+ int16_t off) {
+ spew("swr %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_swr, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sc(Register rd, Register rs, int16_t off) {
+ spew("sc %3s, (0x%x)%2s", rd.name(), off, rs.name());
+#ifdef MIPSR6
+ return writeInst(InstReg(op_special3, rs, rd, ff_sc).encode());
+#else
+ return writeInst(InstImm(op_sc, rs, rd, Imm16(off)).encode());
+#endif
+}
+
+BufferOffset AssemblerMIPSShared::as_scd(Register rd, Register rs,
+ int16_t off) {
+#ifdef MIPSR6
+ return writeInst(InstReg(op_special3, rs, rd, ff_scd).encode());
+#else
+ spew("scd %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_scd, rs, rd, Imm16(off)).encode());
+#endif
+}
+
+BufferOffset AssemblerMIPSShared::as_sd(Register rd, Register rs, int16_t off) {
+ spew("sd %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_sd, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sdl(Register rd, Register rs,
+ int16_t off) {
+ spew("sdl %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_sdl, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sdr(Register rd, Register rs,
+ int16_t off) {
+ spew("sdr %3s, (0x%x)%2s", rd.name(), off, rs.name());
+ return writeInst(InstImm(op_sdr, rs, rd, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_seleqz(Register rd, Register rs,
+ Register rt) {
+ spew("seleqz %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x0, ff_seleqz).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_selnez(Register rd, Register rs,
+ Register rt) {
+ spew("selnez %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, 0x0, ff_selnez).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gslbx(Register rd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gslbx %3s,%3s, (0x%x)%2s", rd.name(), rs.name(), off, ri.name());
+ return writeInst(InstGS(op_ldc2, rs, rd, ri, Imm8(off), ff_gsxbx).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gssbx(Register rd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gssbx %3s,%3s, (0x%x)%2s", rd.name(), rs.name(), off, ri.name());
+ return writeInst(InstGS(op_sdc2, rs, rd, ri, Imm8(off), ff_gsxbx).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gslhx(Register rd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gslhx %3s,%3s, (0x%x)%2s", rd.name(), rs.name(), off, ri.name());
+ return writeInst(InstGS(op_ldc2, rs, rd, ri, Imm8(off), ff_gsxhx).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsshx(Register rd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsshx %3s,%3s, (0x%x)%2s", rd.name(), rs.name(), off, ri.name());
+ return writeInst(InstGS(op_sdc2, rs, rd, ri, Imm8(off), ff_gsxhx).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gslwx(Register rd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gslwx %3s,%3s, (0x%x)%2s", rd.name(), rs.name(), off, ri.name());
+ return writeInst(InstGS(op_ldc2, rs, rd, ri, Imm8(off), ff_gsxwx).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsswx(Register rd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsswx %3s,%3s, (0x%x)%2s", rd.name(), rs.name(), off, ri.name());
+ return writeInst(InstGS(op_sdc2, rs, rd, ri, Imm8(off), ff_gsxwx).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsldx(Register rd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsldx %3s,%3s, (0x%x)%2s", rd.name(), rs.name(), off, ri.name());
+ return writeInst(InstGS(op_ldc2, rs, rd, ri, Imm8(off), ff_gsxdx).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gssdx(Register rd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gssdx %3s,%3s, (0x%x)%2s", rd.name(), rs.name(), off, ri.name());
+ return writeInst(InstGS(op_sdc2, rs, rd, ri, Imm8(off), ff_gsxdx).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gslq(Register rh, Register rl, Register rs,
+ int16_t off) {
+ MOZ_ASSERT(GSImm13::IsInRange(off));
+ spew("gslq %3s,%3s, (0x%x)%2s", rh.name(), rl.name(), off, rs.name());
+ return writeInst(InstGS(op_lwc2, rs, rl, rh, GSImm13(off), ff_gsxq).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gssq(Register rh, Register rl, Register rs,
+ int16_t off) {
+ MOZ_ASSERT(GSImm13::IsInRange(off));
+ spew("gssq %3s,%3s, (0x%x)%2s", rh.name(), rl.name(), off, rs.name());
+ return writeInst(InstGS(op_swc2, rs, rl, rh, GSImm13(off), ff_gsxq).encode());
+}
+
+// Move from HI/LO register.
+BufferOffset AssemblerMIPSShared::as_mfhi(Register rd) {
+ spew("mfhi %3s", rd.name());
+ return writeInst(InstReg(op_special, rd, ff_mfhi).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mflo(Register rd) {
+ spew("mflo %3s", rd.name());
+ return writeInst(InstReg(op_special, rd, ff_mflo).encode());
+}
+
+// Set on less than.
+BufferOffset AssemblerMIPSShared::as_slt(Register rd, Register rs,
+ Register rt) {
+ spew("slt %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_slt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sltu(Register rd, Register rs,
+ Register rt) {
+ spew("sltu %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_sltu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_slti(Register rd, Register rs, int32_t j) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(j));
+ spew("slti %3s,%3s, 0x%x", rd.name(), rs.name(), j);
+ return writeInst(InstImm(op_slti, rs, rd, Imm16(j)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sltiu(Register rd, Register rs,
+ uint32_t j) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(int32_t(j)));
+ spew("sltiu %3s,%3s, 0x%x", rd.name(), rs.name(), j);
+ return writeInst(InstImm(op_sltiu, rs, rd, Imm16(j)).encode());
+}
+
+// Conditional move.
+BufferOffset AssemblerMIPSShared::as_movz(Register rd, Register rs,
+ Register rt) {
+ spew("movz %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_movz).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_movn(Register rd, Register rs,
+ Register rt) {
+ spew("movn %3s,%3s,%3s", rd.name(), rs.name(), rt.name());
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_movn).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_movt(Register rd, Register rs,
+ uint16_t cc) {
+ Register rt;
+ rt = Register::FromCode((cc & 0x7) << 2 | 1);
+ spew("movt %3s,%3s, FCC%d", rd.name(), rs.name(), cc);
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_movci).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_movf(Register rd, Register rs,
+ uint16_t cc) {
+ Register rt;
+ rt = Register::FromCode((cc & 0x7) << 2 | 0);
+ spew("movf %3s,%3s, FCC%d", rd.name(), rs.name(), cc);
+ return writeInst(InstReg(op_special, rs, rt, rd, ff_movci).encode());
+}
+
+// Bit twiddling.
+BufferOffset AssemblerMIPSShared::as_clz(Register rd, Register rs) {
+ spew("clz %3s,%3s", rd.name(), rs.name());
+#ifdef MIPSR6
+ return writeInst(InstReg(op_special, rs, 0x0, rd, 0x1, ff_clz).encode());
+#else
+ return writeInst(InstReg(op_special2, rs, rd, rd, ff_clz).encode());
+#endif
+}
+
+BufferOffset AssemblerMIPSShared::as_dclz(Register rd, Register rs) {
+ spew("dclz %3s,%3s", rd.name(), rs.name());
+#ifdef MIPSR6
+ return writeInst(InstReg(op_special, rs, 0x0, rd, 0x1, ff_dclz).encode());
+#else
+ return writeInst(InstReg(op_special2, rs, rd, rd, ff_dclz).encode());
+#endif
+}
+
+BufferOffset AssemblerMIPSShared::as_wsbh(Register rd, Register rt) {
+ spew("wsbh %3s,%3s", rd.name(), rt.name());
+ return writeInst(InstReg(op_special3, zero, rt, rd, 0x2, ff_bshfl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dsbh(Register rd, Register rt) {
+ spew("dsbh %3s,%3s", rd.name(), rt.name());
+ return writeInst(InstReg(op_special3, zero, rt, rd, 0x2, ff_dbshfl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dshd(Register rd, Register rt) {
+ spew("dshd %3s,%3s", rd.name(), rt.name());
+ return writeInst(InstReg(op_special3, zero, rt, rd, 0x5, ff_dbshfl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ins(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ MOZ_ASSERT(pos < 32 && size != 0 && size <= 32 && pos + size != 0 &&
+ pos + size <= 32);
+ Register rd;
+ rd = Register::FromCode(pos + size - 1);
+ spew("ins %3s,%3s, %d, %d", rt.name(), rs.name(), pos, size);
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special3, rs, rt, rd, pos, ff_ins).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dins(Register rt, Register rs,
+ uint16_t pos, uint16_t size) {
+ MOZ_ASSERT(pos < 32 && size != 0 && size <= 32 && pos + size != 0 &&
+ pos + size <= 32);
+ Register rd;
+ rd = Register::FromCode(pos + size - 1);
+ spew("dins %3s,%3s, %d, %d", rt.name(), rs.name(), pos, size);
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special3, rs, rt, rd, pos, ff_dins).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dinsm(Register rt, Register rs,
+ uint16_t pos, uint16_t size) {
+ MOZ_ASSERT(pos < 32 && size >= 2 && size <= 64 && pos + size > 32 &&
+ pos + size <= 64);
+ Register rd;
+ rd = Register::FromCode(pos + size - 1 - 32);
+ spew("dinsm %3s,%3s, %d, %d", rt.name(), rs.name(), pos, size);
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special3, rs, rt, rd, pos, ff_dinsm).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dinsu(Register rt, Register rs,
+ uint16_t pos, uint16_t size) {
+ MOZ_ASSERT(pos >= 32 && pos < 64 && size >= 1 && size <= 32 &&
+ pos + size > 32 && pos + size <= 64);
+ Register rd;
+ rd = Register::FromCode(pos + size - 1 - 32);
+ spew("dinsu %3s,%3s, %d, %d", rt.name(), rs.name(), pos, size);
+ MOZ_ASSERT(hasR2());
+ return writeInst(
+ InstReg(op_special3, rs, rt, rd, pos - 32, ff_dinsu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ext(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ MOZ_ASSERT(pos < 32 && size != 0 && size <= 32 && pos + size != 0 &&
+ pos + size <= 32);
+ Register rd;
+ rd = Register::FromCode(size - 1);
+ spew("ext %3s,%3s, %d, %d", rt.name(), rs.name(), pos, size);
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special3, rs, rt, rd, pos, ff_ext).encode());
+}
+
+// Sign extend
+BufferOffset AssemblerMIPSShared::as_seb(Register rd, Register rt) {
+ spew("seb %3s,%3s", rd.name(), rt.name());
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special3, zero, rt, rd, 16, ff_bshfl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_seh(Register rd, Register rt) {
+ spew("seh %3s,%3s", rd.name(), rt.name());
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special3, zero, rt, rd, 24, ff_bshfl).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dext(Register rt, Register rs,
+ uint16_t pos, uint16_t size) {
+ MOZ_ASSERT(pos < 32 && size != 0 && size <= 32 && pos + size != 0 &&
+ pos + size <= 63);
+ Register rd;
+ rd = Register::FromCode(size - 1);
+ spew("dext %3s,%3s, %d, %d", rt.name(), rs.name(), pos, size);
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special3, rs, rt, rd, pos, ff_dext).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dextm(Register rt, Register rs,
+ uint16_t pos, uint16_t size) {
+ MOZ_ASSERT(pos < 32 && size > 32 && size <= 64 && pos + size > 32 &&
+ pos + size <= 64);
+ Register rd;
+ rd = Register::FromCode(size - 1 - 32);
+ spew("dextm %3s,%3s, %d, %d", rt.name(), rs.name(), pos, size);
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_special3, rs, rt, rd, pos, ff_dextm).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dextu(Register rt, Register rs,
+ uint16_t pos, uint16_t size) {
+ MOZ_ASSERT(pos >= 32 && pos < 64 && size != 0 && size <= 32 &&
+ pos + size > 32 && pos + size <= 64);
+ Register rd;
+ rd = Register::FromCode(size - 1);
+ spew("dextu %3s,%3s, %d, %d", rt.name(), rs.name(), pos, size);
+ MOZ_ASSERT(hasR2());
+ return writeInst(
+ InstReg(op_special3, rs, rt, rd, pos - 32, ff_dextu).encode());
+}
+
+// FP instructions
+BufferOffset AssemblerMIPSShared::as_ldc1(FloatRegister ft, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(off));
+ spew("ldc1 %3s, (0x%x)%2s", ft.name(), off, base.name());
+ return writeInst(InstImm(op_ldc1, base, ft, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sdc1(FloatRegister ft, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(off));
+ spew("sdc1 %3s, (0x%x)%2s", ft.name(), off, base.name());
+ return writeInst(InstImm(op_sdc1, base, ft, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_lwc1(FloatRegister ft, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(off));
+ spew("lwc1 %3s, (0x%x)%2s", ft.name(), off, base.name());
+ return writeInst(InstImm(op_lwc1, base, ft, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_swc1(FloatRegister ft, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(off));
+ spew("swc1 %3s, (0x%x)%2s", ft.name(), off, base.name());
+ return writeInst(InstImm(op_swc1, base, ft, Imm16(off)).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsldl(FloatRegister fd, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsldl %3s, (0x%x)%2s", fd.name(), off, base.name());
+ return writeInst(InstGS(op_lwc2, base, fd, Imm8(off), ff_gsxdlc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsldr(FloatRegister fd, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsldr %3s, (0x%x)%2s", fd.name(), off, base.name());
+ return writeInst(InstGS(op_lwc2, base, fd, Imm8(off), ff_gsxdrc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gssdl(FloatRegister fd, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gssdl %3s, (0x%x)%2s", fd.name(), off, base.name());
+ return writeInst(InstGS(op_swc2, base, fd, Imm8(off), ff_gsxdlc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gssdr(FloatRegister fd, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gssdr %3s, (0x%x)%2s", fd.name(), off, base.name());
+ return writeInst(InstGS(op_swc2, base, fd, Imm8(off), ff_gsxdrc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gslsl(FloatRegister fd, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gslsl %3s, (0x%x)%2s", fd.name(), off, base.name());
+ return writeInst(InstGS(op_lwc2, base, fd, Imm8(off), ff_gsxwlc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gslsr(FloatRegister fd, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gslsr %3s, (0x%x)%2s", fd.name(), off, base.name());
+ return writeInst(InstGS(op_lwc2, base, fd, Imm8(off), ff_gsxwrc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsssl(FloatRegister fd, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsssl %3s, (0x%x)%2s", fd.name(), off, base.name());
+ return writeInst(InstGS(op_swc2, base, fd, Imm8(off), ff_gsxwlc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsssr(FloatRegister fd, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsssr %3s, (0x%x)%2s", fd.name(), off, base.name());
+ return writeInst(InstGS(op_swc2, base, fd, Imm8(off), ff_gsxwrc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gslsx(FloatRegister fd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gslsx %3s, (%3s,%3s, 0x%x)", fd.name(), rs.name(), ri.name(), off);
+ return writeInst(InstGS(op_ldc2, rs, fd, ri, Imm8(off), ff_gsxwxc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsssx(FloatRegister fd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsssx %3s, (%3s,%3s, 0x%x)", fd.name(), rs.name(), ri.name(), off);
+ return writeInst(InstGS(op_sdc2, rs, fd, ri, Imm8(off), ff_gsxwxc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gsldx(FloatRegister fd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gsldx %3s, (%3s,%3s, 0x%x)", fd.name(), rs.name(), ri.name(), off);
+ return writeInst(InstGS(op_ldc2, rs, fd, ri, Imm8(off), ff_gsxdxc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gssdx(FloatRegister fd, Register rs,
+ Register ri, int16_t off) {
+ MOZ_ASSERT(Imm8::IsInSignedRange(off));
+ spew("gssdx %3s, (%3s,%3s, 0x%x)", fd.name(), rs.name(), ri.name(), off);
+ return writeInst(InstGS(op_sdc2, rs, fd, ri, Imm8(off), ff_gsxdxc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gslq(FloatRegister rh, FloatRegister rl,
+ Register rs, int16_t off) {
+ MOZ_ASSERT(GSImm13::IsInRange(off));
+ spew("gslq %3s,%3s, (0x%x)%2s", rh.name(), rl.name(), off, rs.name());
+ return writeInst(
+ InstGS(op_lwc2, rs, rl, rh, GSImm13(off), ff_gsxqc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_gssq(FloatRegister rh, FloatRegister rl,
+ Register rs, int16_t off) {
+ MOZ_ASSERT(GSImm13::IsInRange(off));
+ spew("gssq %3s,%3s, (0x%x)%2s", rh.name(), rl.name(), off, rs.name());
+ return writeInst(
+ InstGS(op_swc2, rs, rl, rh, GSImm13(off), ff_gsxqc1).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_movs(FloatRegister fd, FloatRegister fs) {
+ spew("mov.s %3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_s, zero, fs, fd, ff_mov_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_movd(FloatRegister fd, FloatRegister fs) {
+ spew("mov.d %3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_d, zero, fs, fd, ff_mov_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ctc1(Register rt, FPControl fc) {
+ spew("ctc1 %3s,%d", rt.name(), fc);
+ return writeInst(InstReg(op_cop1, rs_ctc1, rt, (uint32_t)fc).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cfc1(Register rt, FPControl fc) {
+ spew("cfc1 %3s,%d", rt.name(), fc);
+ return writeInst(InstReg(op_cop1, rs_cfc1, rt, (uint32_t)fc).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mtc1(Register rt, FloatRegister fs) {
+ spew("mtc1 %3s,%3s", rt.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_mtc1, rt, fs).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mfc1(Register rt, FloatRegister fs) {
+ spew("mfc1 %3s,%3s", rt.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_mfc1, rt, fs).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mthc1(Register rt, FloatRegister fs) {
+ spew("mthc1 %3s,%3s", rt.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_mthc1, rt, fs).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_mfhc1(Register rt, FloatRegister fs) {
+ spew("mfhc1 %3s,%3s", rt.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_mfhc1, rt, fs).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmtc1(Register rt, FloatRegister fs) {
+ spew("dmtc1 %3s,%3s", rt.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_dmtc1, rt, fs).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_dmfc1(Register rt, FloatRegister fs) {
+ spew("dmfc1 %3s,%3s", rt.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_dmfc1, rt, fs).encode());
+}
+
+// FP convert instructions
+BufferOffset AssemblerMIPSShared::as_ceilws(FloatRegister fd,
+ FloatRegister fs) {
+ spew("ceil.w.s%3s,%3s", fd.name(), fs.name());
+ return writeInst(
+ InstReg(op_cop1, rs_s, zero, fs, fd, ff_ceil_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_floorws(FloatRegister fd,
+ FloatRegister fs) {
+ spew("floor.w.s%3s,%3s", fd.name(), fs.name());
+ return writeInst(
+ InstReg(op_cop1, rs_s, zero, fs, fd, ff_floor_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_roundws(FloatRegister fd,
+ FloatRegister fs) {
+ spew("round.w.s%3s,%3s", fd.name(), fs.name());
+ return writeInst(
+ InstReg(op_cop1, rs_s, zero, fs, fd, ff_round_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_truncws(FloatRegister fd,
+ FloatRegister fs) {
+ spew("trunc.w.s%3s,%3s", fd.name(), fs.name());
+ return writeInst(
+ InstReg(op_cop1, rs_s, zero, fs, fd, ff_trunc_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_truncls(FloatRegister fd,
+ FloatRegister fs) {
+ spew("trunc.l.s%3s,%3s", fd.name(), fs.name());
+ MOZ_ASSERT(hasR2());
+ return writeInst(
+ InstReg(op_cop1, rs_s, zero, fs, fd, ff_trunc_l_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_ceilwd(FloatRegister fd,
+ FloatRegister fs) {
+ spew("ceil.w.d%3s,%3s", fd.name(), fs.name());
+ return writeInst(
+ InstReg(op_cop1, rs_d, zero, fs, fd, ff_ceil_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_floorwd(FloatRegister fd,
+ FloatRegister fs) {
+ spew("floor.w.d%3s,%3s", fd.name(), fs.name());
+ return writeInst(
+ InstReg(op_cop1, rs_d, zero, fs, fd, ff_floor_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_roundwd(FloatRegister fd,
+ FloatRegister fs) {
+ spew("round.w.d%3s,%3s", fd.name(), fs.name());
+ return writeInst(
+ InstReg(op_cop1, rs_d, zero, fs, fd, ff_round_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_truncwd(FloatRegister fd,
+ FloatRegister fs) {
+ spew("trunc.w.d%3s,%3s", fd.name(), fs.name());
+ return writeInst(
+ InstReg(op_cop1, rs_d, zero, fs, fd, ff_trunc_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_truncld(FloatRegister fd,
+ FloatRegister fs) {
+ spew("trunc.l.d%3s,%3s", fd.name(), fs.name());
+ MOZ_ASSERT(hasR2());
+ return writeInst(
+ InstReg(op_cop1, rs_d, zero, fs, fd, ff_trunc_l_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cvtdl(FloatRegister fd, FloatRegister fs) {
+ spew("cvt.d.l%3s,%3s", fd.name(), fs.name());
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_cop1, rs_l, zero, fs, fd, ff_cvt_d_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cvtds(FloatRegister fd, FloatRegister fs) {
+ spew("cvt.d.s%3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_s, zero, fs, fd, ff_cvt_d_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cvtdw(FloatRegister fd, FloatRegister fs) {
+ spew("cvt.d.w%3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_w, zero, fs, fd, ff_cvt_d_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cvtsd(FloatRegister fd, FloatRegister fs) {
+ spew("cvt.s.d%3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_d, zero, fs, fd, ff_cvt_s_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cvtsl(FloatRegister fd, FloatRegister fs) {
+ spew("cvt.s.l%3s,%3s", fd.name(), fs.name());
+ MOZ_ASSERT(hasR2());
+ return writeInst(InstReg(op_cop1, rs_l, zero, fs, fd, ff_cvt_s_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cvtsw(FloatRegister fd, FloatRegister fs) {
+ spew("cvt.s.w%3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_w, zero, fs, fd, ff_cvt_s_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cvtwd(FloatRegister fd, FloatRegister fs) {
+ spew("cvt.w.d%3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_d, zero, fs, fd, ff_cvt_w_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_cvtws(FloatRegister fd, FloatRegister fs) {
+ spew("cvt.w.s%3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_s, zero, fs, fd, ff_cvt_w_fmt).encode());
+}
+
+// FP arithmetic instructions
+BufferOffset AssemblerMIPSShared::as_adds(FloatRegister fd, FloatRegister fs,
+ FloatRegister ft) {
+ spew("add.s %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_s, ft, fs, fd, ff_add_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_addd(FloatRegister fd, FloatRegister fs,
+ FloatRegister ft) {
+ spew("add.d %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_d, ft, fs, fd, ff_add_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_subs(FloatRegister fd, FloatRegister fs,
+ FloatRegister ft) {
+ spew("sub.s %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_s, ft, fs, fd, ff_sub_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_subd(FloatRegister fd, FloatRegister fs,
+ FloatRegister ft) {
+ spew("sub.d %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_d, ft, fs, fd, ff_sub_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_abss(FloatRegister fd, FloatRegister fs) {
+ spew("abs.s %3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_s, zero, fs, fd, ff_abs_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_absd(FloatRegister fd, FloatRegister fs) {
+ spew("abs.d %3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_d, zero, fs, fd, ff_abs_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_negs(FloatRegister fd, FloatRegister fs) {
+ spew("neg.s %3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_s, zero, fs, fd, ff_neg_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_negd(FloatRegister fd, FloatRegister fs) {
+ spew("neg.d %3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_d, zero, fs, fd, ff_neg_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_muls(FloatRegister fd, FloatRegister fs,
+ FloatRegister ft) {
+ spew("mul.s %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_s, ft, fs, fd, ff_mul_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_muld(FloatRegister fd, FloatRegister fs,
+ FloatRegister ft) {
+ spew("mul.d %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_d, ft, fs, fd, ff_mul_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_divs(FloatRegister fd, FloatRegister fs,
+ FloatRegister ft) {
+ spew("div.s %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_s, ft, fs, fd, ff_div_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_divd(FloatRegister fd, FloatRegister fs,
+ FloatRegister ft) {
+ spew("divd.d %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_d, ft, fs, fd, ff_div_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sqrts(FloatRegister fd, FloatRegister fs) {
+ spew("sqrts %3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_s, zero, fs, fd, ff_sqrt_fmt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_sqrtd(FloatRegister fd, FloatRegister fs) {
+ spew("sqrtd %3s,%3s", fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_d, zero, fs, fd, ff_sqrt_fmt).encode());
+}
+
+// FP compare instructions
+BufferOffset AssemblerMIPSShared::as_cf(FloatFormat fmt, FloatRegister fs,
+ FloatRegister ft, FPConditionBit fcc) {
+ if (fmt == DoubleFloat) {
+ spew("c.f.d FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_d_r6, ft, fs, FloatRegisters::f24, ff_c_f_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_d, ft, fs, fcc << FccShift, ff_c_f_fmt).encode());
+#endif
+ } else {
+ spew("c.f.s FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_s_r6, ft, fs, FloatRegisters::f24, ff_c_f_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_s, ft, fs, fcc << FccShift, ff_c_f_fmt).encode());
+#endif
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_cun(FloatFormat fmt, FloatRegister fs,
+ FloatRegister ft, FPConditionBit fcc) {
+ if (fmt == DoubleFloat) {
+ spew("c.un.d FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_d_r6, ft, fs, FloatRegisters::f24, ff_c_un_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_d, ft, fs, fcc << FccShift, ff_c_un_fmt).encode());
+#endif
+ } else {
+ spew("c.un.s FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_s_r6, ft, fs, FloatRegisters::f24, ff_c_un_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_s, ft, fs, fcc << FccShift, ff_c_un_fmt).encode());
+#endif
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_ceq(FloatFormat fmt, FloatRegister fs,
+ FloatRegister ft, FPConditionBit fcc) {
+ if (fmt == DoubleFloat) {
+ spew("c.eq.d FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_d_r6, ft, fs, FloatRegisters::f24, ff_c_eq_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_d, ft, fs, fcc << FccShift, ff_c_eq_fmt).encode());
+#endif
+ } else {
+ spew("c.eq.s FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_s_r6, ft, fs, FloatRegisters::f24, ff_c_eq_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_s, ft, fs, fcc << FccShift, ff_c_eq_fmt).encode());
+#endif
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_cueq(FloatFormat fmt, FloatRegister fs,
+ FloatRegister ft,
+ FPConditionBit fcc) {
+ if (fmt == DoubleFloat) {
+ spew("c.ueq.d FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_d_r6, ft, fs, FloatRegisters::f24, ff_c_ueq_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_d, ft, fs, fcc << FccShift, ff_c_ueq_fmt).encode());
+#endif
+ } else {
+ spew("c.ueq.s FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_s_r6, ft, fs, FloatRegisters::f24, ff_c_ueq_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_s, ft, fs, fcc << FccShift, ff_c_ueq_fmt).encode());
+#endif
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_colt(FloatFormat fmt, FloatRegister fs,
+ FloatRegister ft,
+ FPConditionBit fcc) {
+ if (fmt == DoubleFloat) {
+ spew("c.olt.d FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_d_r6, ft, fs, FloatRegisters::f24, ff_c_olt_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_d, ft, fs, fcc << FccShift, ff_c_olt_fmt).encode());
+#endif
+ } else {
+ spew("c.olt.s FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_s_r6, ft, fs, FloatRegisters::f24, ff_c_olt_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_s, ft, fs, fcc << FccShift, ff_c_olt_fmt).encode());
+#endif
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_cult(FloatFormat fmt, FloatRegister fs,
+ FloatRegister ft,
+ FPConditionBit fcc) {
+ if (fmt == DoubleFloat) {
+ spew("c.ult.d FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_d_r6, ft, fs, FloatRegisters::f24, ff_c_ult_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_d, ft, fs, fcc << FccShift, ff_c_ult_fmt).encode());
+#endif
+ } else {
+ spew("c.ult.s FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_s_r6, ft, fs, FloatRegisters::f24, ff_c_ult_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_s, ft, fs, fcc << FccShift, ff_c_ult_fmt).encode());
+#endif
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_cole(FloatFormat fmt, FloatRegister fs,
+ FloatRegister ft,
+ FPConditionBit fcc) {
+ if (fmt == DoubleFloat) {
+ spew("c.ole.d FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_d_r6, ft, fs, FloatRegisters::f24, ff_c_ole_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_d, ft, fs, fcc << FccShift, ff_c_ole_fmt).encode());
+#endif
+ } else {
+ spew("c.ole.s FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_s_r6, ft, fs, FloatRegisters::f24, ff_c_ole_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_s, ft, fs, fcc << FccShift, ff_c_ole_fmt).encode());
+#endif
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_cule(FloatFormat fmt, FloatRegister fs,
+ FloatRegister ft,
+ FPConditionBit fcc) {
+ if (fmt == DoubleFloat) {
+ spew("c.ule.d FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_d_r6, ft, fs, FloatRegisters::f24, ff_c_ule_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_d, ft, fs, fcc << FccShift, ff_c_ule_fmt).encode());
+#endif
+ } else {
+ spew("c.ule.s FCC%d,%3s,%3s", fcc, fs.name(), ft.name());
+#ifdef MIPSR6
+ return writeInst(
+ InstReg(op_cop1, rs_s_r6, ft, fs, FloatRegisters::f24, ff_c_ule_fmt)
+ .encode());
+#else
+ return writeInst(
+ InstReg(op_cop1, rs_s, ft, fs, fcc << FccShift, ff_c_ule_fmt).encode());
+#endif
+ }
+}
+
+// FP conditional move.
+BufferOffset AssemblerMIPSShared::as_movt(FloatFormat fmt, FloatRegister fd,
+ FloatRegister fs,
+ FPConditionBit fcc) {
+ Register rt = Register::FromCode(fcc << 2 | 1);
+ if (fmt == DoubleFloat) {
+ spew("movt.d FCC%d,%3s,%3s", fcc, fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_d, rt, fs, fd, ff_movf_fmt).encode());
+ } else {
+ spew("movt.s FCC%d,%3s,%3s", fcc, fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_s, rt, fs, fd, ff_movf_fmt).encode());
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_movf(FloatFormat fmt, FloatRegister fd,
+ FloatRegister fs,
+ FPConditionBit fcc) {
+ Register rt = Register::FromCode(fcc << 2 | 0);
+ if (fmt == DoubleFloat) {
+ spew("movf.d FCC%d,%3s,%3s", fcc, fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_d, rt, fs, fd, ff_movf_fmt).encode());
+ } else {
+ spew("movf.s FCC%d,%3s,%3s", fcc, fd.name(), fs.name());
+ return writeInst(InstReg(op_cop1, rs_s, rt, fs, fd, ff_movf_fmt).encode());
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_movz(FloatFormat fmt, FloatRegister fd,
+ FloatRegister fs, Register rt) {
+ if (fmt == DoubleFloat) {
+ spew("movz.d %3s,%3s,%3s", fd.name(), fs.name(), rt.name());
+ return writeInst(InstReg(op_cop1, rs_d, rt, fs, fd, ff_movz_fmt).encode());
+ } else {
+ spew("movz.s %3s,%3s,%3s", fd.name(), fs.name(), rt.name());
+ return writeInst(InstReg(op_cop1, rs_s, rt, fs, fd, ff_movz_fmt).encode());
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_movn(FloatFormat fmt, FloatRegister fd,
+ FloatRegister fs, Register rt) {
+ if (fmt == DoubleFloat) {
+ spew("movn.d %3s,%3s,%3s", fd.name(), fs.name(), rt.name());
+ return writeInst(InstReg(op_cop1, rs_d, rt, fs, fd, ff_movn_fmt).encode());
+ } else {
+ spew("movn.s %3s,%3s,%3s", fd.name(), fs.name(), rt.name());
+ return writeInst(InstReg(op_cop1, rs_s, rt, fs, fd, ff_movn_fmt).encode());
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_max(FloatFormat fmt, FloatRegister fd,
+ FloatRegister fs, FloatRegister ft) {
+ if (fmt == DoubleFloat) {
+ spew("max %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_d, ft, fs, fd, ff_max).encode());
+ } else {
+ spew("max %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_s, ft, fs, fd, ff_max).encode());
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_min(FloatFormat fmt, FloatRegister fd,
+ FloatRegister fs, FloatRegister ft) {
+ if (fmt == DoubleFloat) {
+ spew("min %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_d, ft, fs, fd, ff_min).encode());
+ } else {
+ spew("min %3s,%3s,%3s", fd.name(), fs.name(), ft.name());
+ return writeInst(InstReg(op_cop1, rs_s, ft, fs, fd, ff_min).encode());
+ }
+}
+
+BufferOffset AssemblerMIPSShared::as_tge(Register rs, Register rt,
+ uint32_t code) {
+ MOZ_ASSERT(code <= MAX_BREAK_CODE);
+ spew("tge %3s,%3s,%d", rs.name(), rt.name(), code);
+ return writeInst(InstReg(op_special, rs, rt, zero, code, ff_tge).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_tgeu(Register rs, Register rt,
+ uint32_t code) {
+ MOZ_ASSERT(code <= MAX_BREAK_CODE);
+ spew("tgeu %3s,%3s,%d", rs.name(), rt.name(), code);
+ return writeInst(InstReg(op_special, rs, rt, zero, code, ff_tgeu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_tlt(Register rs, Register rt,
+ uint32_t code) {
+ MOZ_ASSERT(code <= MAX_BREAK_CODE);
+ spew("tlt %3s,%3s,%d", rs.name(), rt.name(), code);
+ return writeInst(InstReg(op_special, rs, rt, zero, code, ff_tlt).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_tltu(Register rs, Register rt,
+ uint32_t code) {
+ MOZ_ASSERT(code <= MAX_BREAK_CODE);
+ spew("tltu %3s,%3s,%d", rs.name(), rt.name(), code);
+ return writeInst(InstReg(op_special, rs, rt, zero, code, ff_tltu).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_teq(Register rs, Register rt,
+ uint32_t code) {
+ MOZ_ASSERT(code <= MAX_BREAK_CODE);
+ spew("teq %3s,%3s,%d", rs.name(), rt.name(), code);
+ return writeInst(InstReg(op_special, rs, rt, zero, code, ff_teq).encode());
+}
+
+BufferOffset AssemblerMIPSShared::as_tne(Register rs, Register rt,
+ uint32_t code) {
+ MOZ_ASSERT(code <= MAX_BREAK_CODE);
+ spew("tne %3s,%3s,%d", rs.name(), rt.name(), code);
+ return writeInst(InstReg(op_special, rs, rt, zero, code, ff_tne).encode());
+}
+
+void AssemblerMIPSShared::bind(Label* label, BufferOffset boff) {
+ spew(".set Llabel %p", label);
+ // If our caller didn't give us an explicit target to bind to
+ // then we want to bind to the location of the next instruction
+ BufferOffset dest = boff.assigned() ? boff : nextOffset();
+ if (label->used()) {
+ int32_t next;
+
+ // A used label holds a link to branch that uses it.
+ BufferOffset b(label);
+ do {
+ // Even a 0 offset may be invalid if we're out of memory.
+ if (oom()) {
+ return;
+ }
+
+ Instruction* inst = editSrc(b);
+
+ // Second word holds a pointer to the next branch in label's chain.
+ next = inst[1].encode();
+ bind(reinterpret_cast<InstImm*>(inst), b.getOffset(), dest.getOffset());
+
+ b = BufferOffset(next);
+ } while (next != LabelBase::INVALID_OFFSET);
+ }
+ label->bind(dest.getOffset());
+}
+
+void AssemblerMIPSShared::retarget(Label* label, Label* target) {
+ spew("retarget %p -> %p", label, target);
+ if (label->used() && !oom()) {
+ if (target->bound()) {
+ bind(label, BufferOffset(target));
+ } else if (target->used()) {
+ // The target is not bound but used. Prepend label's branch list
+ // onto target's.
+ int32_t next;
+ BufferOffset labelBranchOffset(label);
+
+ // Find the head of the use chain for label.
+ do {
+ Instruction* inst = editSrc(labelBranchOffset);
+
+ // Second word holds a pointer to the next branch in chain.
+ next = inst[1].encode();
+ labelBranchOffset = BufferOffset(next);
+ } while (next != LabelBase::INVALID_OFFSET);
+
+ // Then patch the head of label's use chain to the tail of
+ // target's use chain, prepending the entire use chain of target.
+ Instruction* inst = editSrc(labelBranchOffset);
+ int32_t prev = target->offset();
+ target->use(label->offset());
+ inst[1].setData(prev);
+ } else {
+ // The target is unbound and unused. We can just take the head of
+ // the list hanging off of label, and dump that into target.
+ target->use(label->offset());
+ }
+ }
+ label->reset();
+}
+
+void dbg_break() {}
+void AssemblerMIPSShared::as_break(uint32_t code) {
+ MOZ_ASSERT(code <= MAX_BREAK_CODE);
+ spew("break %d", code);
+ writeInst(op_special | code << FunctionBits | ff_break);
+}
+
+void AssemblerMIPSShared::as_sync(uint32_t stype) {
+ MOZ_ASSERT(stype <= 31);
+ spew("sync %d", stype);
+ writeInst(InstReg(op_special, zero, zero, zero, stype, ff_sync).encode());
+}
+
+// This just stomps over memory with 32 bits of raw data. Its purpose is to
+// overwrite the call of JITed code with 32 bits worth of an offset. This will
+// is only meant to function on code that has been invalidated, so it should
+// be totally safe. Since that instruction will never be executed again, a
+// ICache flush should not be necessary
+void AssemblerMIPSShared::PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm) {
+ // Raw is going to be the return address.
+ uint32_t* raw = (uint32_t*)label.raw();
+ // Overwrite the 4 bytes before the return address, which will
+ // end up being the call instruction.
+ *(raw - 1) = imm.value;
+}
+
+uint8_t* AssemblerMIPSShared::NextInstruction(uint8_t* inst_, uint32_t* count) {
+ Instruction* inst = reinterpret_cast<Instruction*>(inst_);
+ if (count != nullptr) {
+ *count += sizeof(Instruction);
+ }
+ return reinterpret_cast<uint8_t*>(inst->next());
+}
+
+// Since there are no pools in MIPS implementation, this should be simple.
+Instruction* Instruction::next() { return this + 1; }
+
+InstImm AssemblerMIPSShared::invertBranch(InstImm branch,
+ BOffImm16 skipOffset) {
+ uint32_t rt = 0;
+ OpcodeField op = (OpcodeField)(branch.extractOpcode() << OpcodeShift);
+ switch (op) {
+ case op_beq:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_bne);
+ return branch;
+ case op_bne:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_beq);
+ return branch;
+ case op_bgtz:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_blez);
+ return branch;
+ case op_blez:
+ branch.setBOffImm16(skipOffset);
+ branch.setOpcode(op_bgtz);
+ return branch;
+ case op_regimm:
+ branch.setBOffImm16(skipOffset);
+ rt = branch.extractRT();
+ if (rt == (rt_bltz >> RTShift)) {
+ branch.setRT(rt_bgez);
+ return branch;
+ }
+ if (rt == (rt_bgez >> RTShift)) {
+ branch.setRT(rt_bltz);
+ return branch;
+ }
+
+ MOZ_CRASH("Error creating long branch.");
+
+ case op_cop1:
+ MOZ_ASSERT(branch.extractRS() == rs_bc1 >> RSShift);
+
+ branch.setBOffImm16(skipOffset);
+ rt = branch.extractRT();
+ if (rt & 0x1) {
+ branch.setRT((RTField)((rt & ~0x1) << RTShift));
+ } else {
+ branch.setRT((RTField)((rt | 0x1) << RTShift));
+ }
+ return branch;
+ default:
+ MOZ_CRASH("Error creating long branch.");
+ }
+}
+
+void AssemblerMIPSShared::ToggleToJmp(CodeLocationLabel inst_) {
+ InstImm* inst = (InstImm*)inst_.raw();
+
+ MOZ_ASSERT(inst->extractOpcode() == ((uint32_t)op_andi >> OpcodeShift));
+ // We converted beq to andi, so now we restore it.
+ inst->setOpcode(op_beq);
+}
+
+void AssemblerMIPSShared::ToggleToCmp(CodeLocationLabel inst_) {
+ InstImm* inst = (InstImm*)inst_.raw();
+
+ // toggledJump is allways used for short jumps.
+ MOZ_ASSERT(inst->extractOpcode() == ((uint32_t)op_beq >> OpcodeShift));
+ // Replace "beq $zero, $zero, offset" with "andi $zero, $zero, offset"
+ inst->setOpcode(op_andi);
+}
+
+void AssemblerMIPSShared::UpdateLuiOriValue(Instruction* inst0,
+ Instruction* inst1,
+ uint32_t value) {
+ MOZ_ASSERT(inst0->extractOpcode() == ((uint32_t)op_lui >> OpcodeShift));
+ MOZ_ASSERT(inst1->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ ((InstImm*)inst0)->setImm16(Imm16::Upper(Imm32(value)));
+ ((InstImm*)inst1)->setImm16(Imm16::Lower(Imm32(value)));
+}
+
+#ifdef JS_JITSPEW
+void AssemblerMIPSShared::decodeBranchInstAndSpew(InstImm branch) {
+ OpcodeField op = (OpcodeField)(branch.extractOpcode() << OpcodeShift);
+ uint32_t rt_id;
+ uint32_t rs_id;
+ uint32_t immi = branch.extractImm16Value();
+ uint32_t fcc;
+ switch (op) {
+ case op_beq:
+ rt_id = branch.extractRT();
+ rs_id = branch.extractRS();
+ spew("beq %3s,%3s,0x%x", Registers::GetName(rs_id),
+ Registers::GetName(rt_id), (int32_t(immi << 18) >> 16) + 4);
+ break;
+ case op_bne:
+ rt_id = branch.extractRT();
+ rs_id = branch.extractRS();
+ spew("bne %3s,%3s,0x%x", Registers::GetName(rs_id),
+ Registers::GetName(rt_id), (int32_t(immi << 18) >> 16) + 4);
+ break;
+ case op_bgtz:
+ rs_id = branch.extractRS();
+ spew("bgt %3s, 0,0x%x", Registers::GetName(rs_id),
+ (int32_t(immi << 18) >> 16) + 4);
+ break;
+ case op_blez:
+ rs_id = branch.extractRS();
+ spew("ble %3s, 0,0x%x", Registers::GetName(rs_id),
+ (int32_t(immi << 18) >> 16) + 4);
+ break;
+ case op_regimm:
+ rt_id = branch.extractRT();
+ if (rt_id == (rt_bltz >> RTShift)) {
+ rs_id = branch.extractRS();
+ spew("blt %3s, 0,0x%x", Registers::GetName(rs_id),
+ (int32_t(immi << 18) >> 16) + 4);
+ } else if (rt_id == (rt_bgez >> RTShift)) {
+ rs_id = branch.extractRS();
+ spew("bge %3s, 0,0x%x", Registers::GetName(rs_id),
+ (int32_t(immi << 18) >> 16) + 4);
+ } else {
+ MOZ_CRASH("Error disassemble branch.");
+ }
+ break;
+ case op_cop1:
+ MOZ_ASSERT(branch.extractRS() == rs_bc1 >> RSShift);
+ rt_id = branch.extractRT();
+ fcc = branch.extractBitField(FCccShift + FCccBits - 1, FCccShift);
+ if (rt_id & 0x1) {
+ spew("bc1t FCC%d, 0x%x", fcc, (int32_t(immi << 18) >> 16) + 4);
+ } else {
+ spew("bc1f FCC%d, 0x%x", fcc, (int32_t(immi << 18) >> 16) + 4);
+ }
+ break;
+ default:
+ MOZ_CRASH("Error disassemble branch.");
+ }
+}
+#endif
diff --git a/js/src/jit/mips-shared/Assembler-mips-shared.h b/js/src/jit/mips-shared/Assembler-mips-shared.h
new file mode 100644
index 0000000000..fabb963fdc
--- /dev/null
+++ b/js/src/jit/mips-shared/Assembler-mips-shared.h
@@ -0,0 +1,1495 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_Assembler_mips_shared_h
+#define jit_mips_shared_Assembler_mips_shared_h
+
+#include "mozilla/Attributes.h"
+#include "mozilla/MathAlgorithms.h"
+#include "mozilla/Sprintf.h"
+
+#include "jit/CompactBuffer.h"
+#include "jit/JitCode.h"
+#include "jit/JitSpewer.h"
+#include "jit/mips-shared/Architecture-mips-shared.h"
+#include "jit/shared/Assembler-shared.h"
+#include "jit/shared/IonAssemblerBuffer.h"
+#include "wasm/WasmTypeDecls.h"
+
+namespace js {
+namespace jit {
+
+static constexpr Register zero{Registers::zero};
+static constexpr Register at{Registers::at};
+static constexpr Register v0{Registers::v0};
+static constexpr Register v1{Registers::v1};
+static constexpr Register a0{Registers::a0};
+static constexpr Register a1{Registers::a1};
+static constexpr Register a2{Registers::a2};
+static constexpr Register a3{Registers::a3};
+static constexpr Register a4{Registers::ta0};
+static constexpr Register a5{Registers::ta1};
+static constexpr Register a6{Registers::ta2};
+static constexpr Register a7{Registers::ta3};
+static constexpr Register t0{Registers::t0};
+static constexpr Register t1{Registers::t1};
+static constexpr Register t2{Registers::t2};
+static constexpr Register t3{Registers::t3};
+static constexpr Register t4{Registers::ta0};
+static constexpr Register t5{Registers::ta1};
+static constexpr Register t6{Registers::ta2};
+static constexpr Register t7{Registers::ta3};
+static constexpr Register s0{Registers::s0};
+static constexpr Register s1{Registers::s1};
+static constexpr Register s2{Registers::s2};
+static constexpr Register s3{Registers::s3};
+static constexpr Register s4{Registers::s4};
+static constexpr Register s5{Registers::s5};
+static constexpr Register s6{Registers::s6};
+static constexpr Register s7{Registers::s7};
+static constexpr Register t8{Registers::t8};
+static constexpr Register t9{Registers::t9};
+static constexpr Register k0{Registers::k0};
+static constexpr Register k1{Registers::k1};
+static constexpr Register gp{Registers::gp};
+static constexpr Register sp{Registers::sp};
+static constexpr Register fp{Registers::fp};
+static constexpr Register ra{Registers::ra};
+
+static constexpr Register ScratchRegister = at;
+static constexpr Register SecondScratchReg = t8;
+
+// Helper classes for ScratchRegister usage. Asserts that only one piece
+// of code thinks it has exclusive ownership of each scratch register.
+struct ScratchRegisterScope : public AutoRegisterScope {
+ explicit ScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, ScratchRegister) {}
+};
+struct SecondScratchRegisterScope : public AutoRegisterScope {
+ explicit SecondScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, SecondScratchReg) {}
+};
+
+// Use arg reg from EnterJIT function as OsrFrameReg.
+static constexpr Register OsrFrameReg = a3;
+static constexpr Register CallTempReg0 = t0;
+static constexpr Register CallTempReg1 = t1;
+static constexpr Register CallTempReg2 = t2;
+static constexpr Register CallTempReg3 = t3;
+
+static constexpr Register IntArgReg0 = a0;
+static constexpr Register IntArgReg1 = a1;
+static constexpr Register IntArgReg2 = a2;
+static constexpr Register IntArgReg3 = a3;
+static constexpr Register IntArgReg4 = a4;
+static constexpr Register IntArgReg5 = a5;
+static constexpr Register IntArgReg6 = a6;
+static constexpr Register IntArgReg7 = a7;
+static constexpr Register GlobalReg = s6; // used by Odin
+static constexpr Register HeapReg = s7; // used by Odin
+
+static constexpr Register PreBarrierReg = a1;
+
+static constexpr Register InvalidReg{Registers::invalid_reg};
+static constexpr FloatRegister InvalidFloatReg;
+
+static constexpr Register StackPointer = sp;
+static constexpr Register FramePointer = fp;
+static constexpr Register ReturnReg = v0;
+static constexpr FloatRegister ReturnSimd128Reg = InvalidFloatReg;
+static constexpr FloatRegister ScratchSimd128Reg = InvalidFloatReg;
+
+// A bias applied to the GlobalReg to allow the use of instructions with small
+// negative immediate offsets which doubles the range of global data that can be
+// accessed with a single instruction.
+static const int32_t WasmGlobalRegBias = 32768;
+
+// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registerd used in RegExpTester instruction (do not use ReturnReg).
+static constexpr Register RegExpTesterRegExpReg = CallTempReg0;
+static constexpr Register RegExpTesterStringReg = CallTempReg1;
+static constexpr Register RegExpTesterLastIndexReg = CallTempReg2;
+
+static constexpr uint32_t CodeAlignment = 8;
+
+/* clang-format off */
+// MIPS instruction types
+// +---------------------------------------------------------------+
+// | 6 | 5 | 5 | 5 | 5 | 6 |
+// +---------------------------------------------------------------+
+// Register type | Opcode | Rs | Rt | Rd | Sa | Function |
+// +---------------------------------------------------------------+
+// | 6 | 5 | 5 | 16 |
+// +---------------------------------------------------------------+
+// Immediate type | Opcode | Rs | Rt | 2's complement constant |
+// +---------------------------------------------------------------+
+// | 6 | 26 |
+// +---------------------------------------------------------------+
+// Jump type | Opcode | jump_target |
+// +---------------------------------------------------------------+
+// 31 bit bit 0
+/* clang-format on */
+
+// MIPS instruction encoding constants.
+static const uint32_t OpcodeShift = 26;
+static const uint32_t OpcodeBits = 6;
+static const uint32_t RSShift = 21;
+static const uint32_t RSBits = 5;
+static const uint32_t RTShift = 16;
+static const uint32_t RTBits = 5;
+static const uint32_t RDShift = 11;
+static const uint32_t RDBits = 5;
+static const uint32_t RZShift = 0;
+static const uint32_t RZBits = 5;
+static const uint32_t SAShift = 6;
+static const uint32_t SABits = 5;
+static const uint32_t FunctionShift = 0;
+static const uint32_t FunctionBits = 6;
+static const uint32_t Imm16Shift = 0;
+static const uint32_t Imm16Bits = 16;
+static const uint32_t Imm26Shift = 0;
+static const uint32_t Imm26Bits = 26;
+static const uint32_t Imm28Shift = 0;
+static const uint32_t Imm28Bits = 28;
+static const uint32_t ImmFieldShift = 2;
+static const uint32_t FRBits = 5;
+static const uint32_t FRShift = 21;
+static const uint32_t FSShift = 11;
+static const uint32_t FSBits = 5;
+static const uint32_t FTShift = 16;
+static const uint32_t FTBits = 5;
+static const uint32_t FDShift = 6;
+static const uint32_t FDBits = 5;
+static const uint32_t FCccShift = 8;
+static const uint32_t FCccBits = 3;
+static const uint32_t FBccShift = 18;
+static const uint32_t FBccBits = 3;
+static const uint32_t FBtrueShift = 16;
+static const uint32_t FBtrueBits = 1;
+static const uint32_t FccMask = 0x7;
+static const uint32_t FccShift = 2;
+
+// MIPS instruction field bit masks.
+static const uint32_t OpcodeMask = ((1 << OpcodeBits) - 1) << OpcodeShift;
+static const uint32_t Imm16Mask = ((1 << Imm16Bits) - 1) << Imm16Shift;
+static const uint32_t Imm26Mask = ((1 << Imm26Bits) - 1) << Imm26Shift;
+static const uint32_t Imm28Mask = ((1 << Imm28Bits) - 1) << Imm28Shift;
+static const uint32_t RSMask = ((1 << RSBits) - 1) << RSShift;
+static const uint32_t RTMask = ((1 << RTBits) - 1) << RTShift;
+static const uint32_t RDMask = ((1 << RDBits) - 1) << RDShift;
+static const uint32_t SAMask = ((1 << SABits) - 1) << SAShift;
+static const uint32_t FunctionMask = ((1 << FunctionBits) - 1) << FunctionShift;
+static const uint32_t RegMask = Registers::Total - 1;
+
+static const uint32_t BREAK_STACK_UNALIGNED = 1;
+static const uint32_t MAX_BREAK_CODE = 1024 - 1;
+static const uint32_t WASM_TRAP = 6; // BRK_OVERFLOW
+
+class Instruction;
+class InstReg;
+class InstImm;
+class InstJump;
+
+uint32_t RS(Register r);
+uint32_t RT(Register r);
+uint32_t RT(FloatRegister r);
+uint32_t RD(Register r);
+uint32_t RD(FloatRegister r);
+uint32_t RZ(Register r);
+uint32_t RZ(FloatRegister r);
+uint32_t SA(uint32_t value);
+uint32_t SA(FloatRegister r);
+uint32_t FS(uint32_t value);
+
+Register toRS(Instruction& i);
+Register toRT(Instruction& i);
+Register toRD(Instruction& i);
+Register toR(Instruction& i);
+
+// MIPS enums for instruction fields
+enum OpcodeField {
+ op_special = 0 << OpcodeShift,
+ op_regimm = 1 << OpcodeShift,
+
+ op_j = 2 << OpcodeShift,
+ op_jal = 3 << OpcodeShift,
+ op_beq = 4 << OpcodeShift,
+ op_bne = 5 << OpcodeShift,
+ op_blez = 6 << OpcodeShift,
+ op_bgtz = 7 << OpcodeShift,
+
+ op_addi = 8 << OpcodeShift,
+ op_addiu = 9 << OpcodeShift,
+ op_slti = 10 << OpcodeShift,
+ op_sltiu = 11 << OpcodeShift,
+ op_andi = 12 << OpcodeShift,
+ op_ori = 13 << OpcodeShift,
+ op_xori = 14 << OpcodeShift,
+ op_lui = 15 << OpcodeShift,
+
+ op_cop1 = 17 << OpcodeShift,
+ op_cop1x = 19 << OpcodeShift,
+
+ op_beql = 20 << OpcodeShift,
+ op_bnel = 21 << OpcodeShift,
+ op_blezl = 22 << OpcodeShift,
+ op_bgtzl = 23 << OpcodeShift,
+
+ op_daddi = 24 << OpcodeShift,
+ op_daddiu = 25 << OpcodeShift,
+
+ op_ldl = 26 << OpcodeShift,
+ op_ldr = 27 << OpcodeShift,
+
+ op_special2 = 28 << OpcodeShift,
+ op_special3 = 31 << OpcodeShift,
+
+ op_lb = 32 << OpcodeShift,
+ op_lh = 33 << OpcodeShift,
+ op_lwl = 34 << OpcodeShift,
+ op_lw = 35 << OpcodeShift,
+ op_lbu = 36 << OpcodeShift,
+ op_lhu = 37 << OpcodeShift,
+ op_lwr = 38 << OpcodeShift,
+ op_lwu = 39 << OpcodeShift,
+ op_sb = 40 << OpcodeShift,
+ op_sh = 41 << OpcodeShift,
+ op_swl = 42 << OpcodeShift,
+ op_sw = 43 << OpcodeShift,
+ op_sdl = 44 << OpcodeShift,
+ op_sdr = 45 << OpcodeShift,
+ op_swr = 46 << OpcodeShift,
+
+ op_ll = 48 << OpcodeShift,
+ op_lwc1 = 49 << OpcodeShift,
+ op_lwc2 = 50 << OpcodeShift,
+ op_lld = 52 << OpcodeShift,
+ op_ldc1 = 53 << OpcodeShift,
+ op_ldc2 = 54 << OpcodeShift,
+ op_ld = 55 << OpcodeShift,
+
+ op_sc = 56 << OpcodeShift,
+ op_swc1 = 57 << OpcodeShift,
+ op_swc2 = 58 << OpcodeShift,
+ op_scd = 60 << OpcodeShift,
+ op_sdc1 = 61 << OpcodeShift,
+ op_sdc2 = 62 << OpcodeShift,
+ op_sd = 63 << OpcodeShift,
+};
+
+enum RSField {
+ rs_zero = 0 << RSShift,
+ // cop1 encoding of RS field.
+ rs_mfc1 = 0 << RSShift,
+ rs_one = 1 << RSShift,
+ rs_dmfc1 = 1 << RSShift,
+ rs_cfc1 = 2 << RSShift,
+ rs_mfhc1 = 3 << RSShift,
+ rs_mtc1 = 4 << RSShift,
+ rs_dmtc1 = 5 << RSShift,
+ rs_ctc1 = 6 << RSShift,
+ rs_mthc1 = 7 << RSShift,
+ rs_bc1 = 8 << RSShift,
+ rs_f = 0x9 << RSShift,
+ rs_t = 0xd << RSShift,
+ rs_s_r6 = 20 << RSShift,
+ rs_d_r6 = 21 << RSShift,
+ rs_s = 16 << RSShift,
+ rs_d = 17 << RSShift,
+ rs_w = 20 << RSShift,
+ rs_l = 21 << RSShift,
+ rs_ps = 22 << RSShift
+};
+
+enum RTField {
+ rt_zero = 0 << RTShift,
+ // regimm encoding of RT field.
+ rt_bltz = 0 << RTShift,
+ rt_bgez = 1 << RTShift,
+ rt_bltzal = 16 << RTShift,
+ rt_bgezal = 17 << RTShift
+};
+
+enum FunctionField {
+ // special encoding of function field.
+ ff_sll = 0,
+ ff_movci = 1,
+ ff_srl = 2,
+ ff_sra = 3,
+ ff_sllv = 4,
+ ff_srlv = 6,
+ ff_srav = 7,
+
+ ff_jr = 8,
+ ff_jalr = 9,
+ ff_movz = 10,
+ ff_movn = 11,
+ ff_break = 13,
+ ff_sync = 15,
+
+ ff_mfhi = 16,
+ ff_mflo = 18,
+
+ ff_dsllv = 20,
+ ff_dsrlv = 22,
+ ff_dsrav = 23,
+
+ ff_mult = 24,
+ ff_multu = 25,
+
+ ff_mulu = 25,
+ ff_muh = 24,
+ ff_muhu = 25,
+ ff_dmul = 28,
+ ff_dmulu = 29,
+ ff_dmuh = 28,
+ ff_dmuhu = 29,
+
+ ff_div = 26,
+ ff_mod = 26,
+ ff_divu = 27,
+ ff_modu = 27,
+ ff_dmult = 28,
+ ff_dmultu = 29,
+ ff_ddiv = 30,
+ ff_dmod = 30,
+ ff_ddivu = 31,
+ ff_dmodu = 31,
+
+ ff_add = 32,
+ ff_addu = 33,
+ ff_sub = 34,
+ ff_subu = 35,
+ ff_and = 36,
+ ff_or = 37,
+ ff_xor = 38,
+ ff_nor = 39,
+
+ ff_slt = 42,
+ ff_sltu = 43,
+ ff_dadd = 44,
+ ff_daddu = 45,
+ ff_dsub = 46,
+ ff_dsubu = 47,
+
+ ff_tge = 48,
+ ff_tgeu = 49,
+ ff_tlt = 50,
+ ff_tltu = 51,
+ ff_teq = 52,
+ ff_seleqz = 53,
+ ff_tne = 54,
+ ff_selnez = 55,
+ ff_dsll = 56,
+ ff_dsrl = 58,
+ ff_dsra = 59,
+ ff_dsll32 = 60,
+ ff_dsrl32 = 62,
+ ff_dsra32 = 63,
+
+ // special2 encoding of function field.
+ ff_madd = 0,
+ ff_maddu = 1,
+#ifdef MIPSR6
+ ff_clz = 16,
+ ff_dclz = 18,
+ ff_mul = 24,
+#else
+ ff_mul = 2,
+ ff_clz = 32,
+ ff_dclz = 36,
+#endif
+ ff_clo = 33,
+
+ // special3 encoding of function field.
+ ff_ext = 0,
+ ff_dextm = 1,
+ ff_dextu = 2,
+ ff_dext = 3,
+ ff_ins = 4,
+ ff_dinsm = 5,
+ ff_dinsu = 6,
+ ff_dins = 7,
+ ff_bshfl = 32,
+ ff_dbshfl = 36,
+ ff_sc = 38,
+ ff_scd = 39,
+ ff_ll = 54,
+ ff_lld = 55,
+
+ // cop1 encoding of function field.
+ ff_add_fmt = 0,
+ ff_sub_fmt = 1,
+ ff_mul_fmt = 2,
+ ff_div_fmt = 3,
+ ff_sqrt_fmt = 4,
+ ff_abs_fmt = 5,
+ ff_mov_fmt = 6,
+ ff_neg_fmt = 7,
+
+ ff_round_l_fmt = 8,
+ ff_trunc_l_fmt = 9,
+ ff_ceil_l_fmt = 10,
+ ff_floor_l_fmt = 11,
+
+ ff_round_w_fmt = 12,
+ ff_trunc_w_fmt = 13,
+ ff_ceil_w_fmt = 14,
+ ff_floor_w_fmt = 15,
+
+ ff_movf_fmt = 17,
+ ff_movz_fmt = 18,
+ ff_movn_fmt = 19,
+
+ ff_min = 28,
+ ff_max = 30,
+
+ ff_cvt_s_fmt = 32,
+ ff_cvt_d_fmt = 33,
+ ff_cvt_w_fmt = 36,
+ ff_cvt_l_fmt = 37,
+ ff_cvt_ps_s = 38,
+
+#ifdef MIPSR6
+ ff_c_f_fmt = 0,
+ ff_c_un_fmt = 1,
+ ff_c_eq_fmt = 2,
+ ff_c_ueq_fmt = 3,
+ ff_c_olt_fmt = 4,
+ ff_c_ult_fmt = 5,
+ ff_c_ole_fmt = 6,
+ ff_c_ule_fmt = 7,
+#else
+ ff_c_f_fmt = 48,
+ ff_c_un_fmt = 49,
+ ff_c_eq_fmt = 50,
+ ff_c_ueq_fmt = 51,
+ ff_c_olt_fmt = 52,
+ ff_c_ult_fmt = 53,
+ ff_c_ole_fmt = 54,
+ ff_c_ule_fmt = 55,
+#endif
+
+ ff_madd_s = 32,
+ ff_madd_d = 33,
+
+ // Loongson encoding of function field.
+ ff_gsxbx = 0,
+ ff_gsxhx = 1,
+ ff_gsxwx = 2,
+ ff_gsxdx = 3,
+ ff_gsxwlc1 = 4,
+ ff_gsxwrc1 = 5,
+ ff_gsxdlc1 = 6,
+ ff_gsxdrc1 = 7,
+ ff_gsxwxc1 = 6,
+ ff_gsxdxc1 = 7,
+ ff_gsxq = 0x20,
+ ff_gsxqc1 = 0x8020,
+
+ ff_null = 0
+};
+
+class Operand;
+
+// A BOffImm16 is a 16 bit immediate that is used for branches.
+class BOffImm16 {
+ uint32_t data;
+
+ public:
+ uint32_t encode() {
+ MOZ_ASSERT(!isInvalid());
+ return data;
+ }
+ int32_t decode() {
+ MOZ_ASSERT(!isInvalid());
+ return (int32_t(data << 18) >> 16) + 4;
+ }
+
+ explicit BOffImm16(int offset) : data((offset - 4) >> 2 & Imm16Mask) {
+ MOZ_ASSERT((offset & 0x3) == 0);
+ MOZ_ASSERT(IsInRange(offset));
+ }
+ static bool IsInRange(int offset) {
+ if ((offset - 4) < int(unsigned(INT16_MIN) << 2)) {
+ return false;
+ }
+ if ((offset - 4) > (INT16_MAX << 2)) {
+ return false;
+ }
+ return true;
+ }
+ static const uint32_t INVALID = 0x00020000;
+ BOffImm16() : data(INVALID) {}
+
+ bool isInvalid() { return data == INVALID; }
+ Instruction* getDest(Instruction* src) const;
+
+ BOffImm16(InstImm inst);
+};
+
+// A JOffImm26 is a 26 bit immediate that is used for unconditional jumps.
+class JOffImm26 {
+ uint32_t data;
+
+ public:
+ uint32_t encode() {
+ MOZ_ASSERT(!isInvalid());
+ return data;
+ }
+ int32_t decode() {
+ MOZ_ASSERT(!isInvalid());
+ return (int32_t(data << 8) >> 6) + 4;
+ }
+
+ explicit JOffImm26(int offset) : data((offset - 4) >> 2 & Imm26Mask) {
+ MOZ_ASSERT((offset & 0x3) == 0);
+ MOZ_ASSERT(IsInRange(offset));
+ }
+ static bool IsInRange(int offset) {
+ if ((offset - 4) < -536870912) {
+ return false;
+ }
+ if ((offset - 4) > 536870908) {
+ return false;
+ }
+ return true;
+ }
+ static const uint32_t INVALID = 0x20000000;
+ JOffImm26() : data(INVALID) {}
+
+ bool isInvalid() { return data == INVALID; }
+ Instruction* getDest(Instruction* src);
+};
+
+class Imm16 {
+ uint16_t value;
+
+ public:
+ Imm16();
+ Imm16(uint32_t imm) : value(imm) {}
+ uint32_t encode() { return value; }
+ int32_t decodeSigned() { return value; }
+ uint32_t decodeUnsigned() { return value; }
+ static bool IsInSignedRange(int32_t imm) {
+ return imm >= INT16_MIN && imm <= INT16_MAX;
+ }
+ static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT16_MAX; }
+ static Imm16 Lower(Imm32 imm) { return Imm16(imm.value & 0xffff); }
+ static Imm16 Upper(Imm32 imm) { return Imm16((imm.value >> 16) & 0xffff); }
+};
+
+class Imm8 {
+ uint8_t value;
+
+ public:
+ Imm8();
+ Imm8(uint32_t imm) : value(imm) {}
+ uint32_t encode(uint32_t shift) { return value << shift; }
+ int32_t decodeSigned() { return value; }
+ uint32_t decodeUnsigned() { return value; }
+ static bool IsInSignedRange(int32_t imm) {
+ return imm >= INT8_MIN && imm <= INT8_MAX;
+ }
+ static bool IsInUnsignedRange(uint32_t imm) { return imm <= UINT8_MAX; }
+ static Imm8 Lower(Imm16 imm) { return Imm8(imm.decodeSigned() & 0xff); }
+ static Imm8 Upper(Imm16 imm) {
+ return Imm8((imm.decodeSigned() >> 8) & 0xff);
+ }
+};
+
+class GSImm13 {
+ uint16_t value;
+
+ public:
+ GSImm13();
+ GSImm13(uint32_t imm) : value(imm & ~0xf) {}
+ uint32_t encode(uint32_t shift) { return ((value >> 4) & 0x1ff) << shift; }
+ int32_t decodeSigned() { return value; }
+ uint32_t decodeUnsigned() { return value; }
+ static bool IsInRange(int32_t imm) {
+ return imm >= int32_t(uint32_t(-256) << 4) && imm <= (255 << 4);
+ }
+};
+
+class Operand {
+ public:
+ enum Tag { REG, FREG, MEM };
+
+ private:
+ Tag tag : 3;
+ uint32_t reg : 5;
+ int32_t offset;
+
+ public:
+ Operand(Register reg_) : tag(REG), reg(reg_.code()) {}
+
+ Operand(FloatRegister freg) : tag(FREG), reg(freg.code()) {}
+
+ Operand(Register base, Imm32 off)
+ : tag(MEM), reg(base.code()), offset(off.value) {}
+
+ Operand(Register base, int32_t off)
+ : tag(MEM), reg(base.code()), offset(off) {}
+
+ Operand(const Address& addr)
+ : tag(MEM), reg(addr.base.code()), offset(addr.offset) {}
+
+ Tag getTag() const { return tag; }
+
+ Register toReg() const {
+ MOZ_ASSERT(tag == REG);
+ return Register::FromCode(reg);
+ }
+
+ FloatRegister toFReg() const {
+ MOZ_ASSERT(tag == FREG);
+ return FloatRegister::FromCode(reg);
+ }
+
+ void toAddr(Register* r, Imm32* dest) const {
+ MOZ_ASSERT(tag == MEM);
+ *r = Register::FromCode(reg);
+ *dest = Imm32(offset);
+ }
+ Address toAddress() const {
+ MOZ_ASSERT(tag == MEM);
+ return Address(Register::FromCode(reg), offset);
+ }
+ int32_t disp() const {
+ MOZ_ASSERT(tag == MEM);
+ return offset;
+ }
+
+ int32_t base() const {
+ MOZ_ASSERT(tag == MEM);
+ return reg;
+ }
+ Register baseReg() const {
+ MOZ_ASSERT(tag == MEM);
+ return Register::FromCode(reg);
+ }
+};
+
+inline Imm32 Imm64::firstHalf() const { return low(); }
+
+inline Imm32 Imm64::secondHalf() const { return hi(); }
+
+static constexpr int32_t SliceSize = 1024;
+typedef js::jit::AssemblerBuffer<SliceSize, Instruction> MIPSBuffer;
+
+class MIPSBufferWithExecutableCopy : public MIPSBuffer {
+ public:
+ void executableCopy(uint8_t* buffer) {
+ if (this->oom()) {
+ return;
+ }
+
+ for (Slice* cur = head; cur != nullptr; cur = cur->getNext()) {
+ memcpy(buffer, &cur->instructions, cur->length());
+ buffer += cur->length();
+ }
+ }
+
+ bool appendRawCode(const uint8_t* code, size_t numBytes) {
+ if (this->oom()) {
+ return false;
+ }
+ while (numBytes > SliceSize) {
+ this->putBytes(SliceSize, code);
+ numBytes -= SliceSize;
+ code += SliceSize;
+ }
+ this->putBytes(numBytes, code);
+ return !this->oom();
+ }
+};
+
+class AssemblerMIPSShared : public AssemblerShared {
+ public:
+ enum Condition {
+ Equal,
+ NotEqual,
+ Above,
+ AboveOrEqual,
+ Below,
+ BelowOrEqual,
+ GreaterThan,
+ GreaterThanOrEqual,
+ LessThan,
+ LessThanOrEqual,
+ Overflow,
+ CarrySet,
+ CarryClear,
+ Signed,
+ NotSigned,
+ Zero,
+ NonZero,
+ Always,
+ };
+
+ enum DoubleCondition {
+ // These conditions will only evaluate to true if the comparison is ordered
+ // - i.e. neither operand is NaN.
+ DoubleOrdered,
+ DoubleEqual,
+ DoubleNotEqual,
+ DoubleGreaterThan,
+ DoubleGreaterThanOrEqual,
+ DoubleLessThan,
+ DoubleLessThanOrEqual,
+ // If either operand is NaN, these conditions always evaluate to true.
+ DoubleUnordered,
+ DoubleEqualOrUnordered,
+ DoubleNotEqualOrUnordered,
+ DoubleGreaterThanOrUnordered,
+ DoubleGreaterThanOrEqualOrUnordered,
+ DoubleLessThanOrUnordered,
+ DoubleLessThanOrEqualOrUnordered
+ };
+
+ enum FPConditionBit { FCC0 = 0, FCC1, FCC2, FCC3, FCC4, FCC5, FCC6, FCC7 };
+
+ enum FPControl {
+ FIR = 0,
+ UFR,
+ UNFR = 4,
+ FCCR = 25,
+ FEXR,
+ FENR = 28,
+ FCSR = 31
+ };
+
+ enum FCSRBit { CauseI = 12, CauseU, CauseO, CauseZ, CauseV };
+
+ enum FloatFormat { SingleFloat, DoubleFloat };
+
+ enum JumpOrCall { BranchIsJump, BranchIsCall };
+
+ enum FloatTestKind { TestForTrue, TestForFalse };
+
+ // :( this should be protected, but since CodeGenerator
+ // wants to use it, It needs to go out here :(
+
+ BufferOffset nextOffset() { return m_buffer.nextOffset(); }
+
+ protected:
+ Instruction* editSrc(BufferOffset bo) { return m_buffer.getInst(bo); }
+
+ // structure for fixing up pc-relative loads/jumps when a the machine code
+ // gets moved (executable copy, gc, etc.)
+ struct RelativePatch {
+ // the offset within the code buffer where the value is loaded that
+ // we want to fix-up
+ BufferOffset offset;
+ void* target;
+ RelocationKind kind;
+
+ RelativePatch(BufferOffset offset, void* target, RelocationKind kind)
+ : offset(offset), target(target), kind(kind) {}
+ };
+
+ js::Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;
+
+ CompactBufferWriter jumpRelocations_;
+ CompactBufferWriter dataRelocations_;
+
+ MIPSBufferWithExecutableCopy m_buffer;
+
+#ifdef JS_JITSPEW
+ Sprinter* printer;
+#endif
+
+ public:
+ AssemblerMIPSShared()
+ : m_buffer(),
+#ifdef JS_JITSPEW
+ printer(nullptr),
+#endif
+ isFinished(false) {
+ }
+
+ static Condition InvertCondition(Condition cond);
+ static DoubleCondition InvertCondition(DoubleCondition cond);
+
+ // As opposed to x86/x64 version, the data relocation has to be executed
+ // before to recover the pointer, and not after.
+ void writeDataRelocation(ImmGCPtr ptr) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // TraceOneDataRelocation.
+ if (ptr.value) {
+ if (gc::IsInsideNursery(ptr.value)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(nextOffset().getOffset());
+ }
+ }
+
+ void assertNoGCThings() const {
+#ifdef DEBUG
+ MOZ_ASSERT(dataRelocations_.length() == 0);
+ for (auto& j : jumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+#endif
+ }
+
+ public:
+ void setUnlimitedBuffer() { m_buffer.setUnlimited(); }
+ bool oom() const;
+
+ void setPrinter(Sprinter* sp) {
+#ifdef JS_JITSPEW
+ printer = sp;
+#endif
+ }
+
+#ifdef JS_JITSPEW
+ inline void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) {
+ if (MOZ_UNLIKELY(printer || JitSpewEnabled(JitSpew_Codegen))) {
+ va_list va;
+ va_start(va, fmt);
+ spew(fmt, va);
+ va_end(va);
+ }
+ }
+
+ void decodeBranchInstAndSpew(InstImm branch);
+#else
+ MOZ_ALWAYS_INLINE void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) {}
+#endif
+
+#ifdef JS_JITSPEW
+ MOZ_COLD void spew(const char* fmt, va_list va) MOZ_FORMAT_PRINTF(2, 0) {
+ // Buffer to hold the formatted string. Note that this may contain
+ // '%' characters, so do not pass it directly to printf functions.
+ char buf[200];
+
+ int i = VsprintfLiteral(buf, fmt, va);
+ if (i > -1) {
+ if (printer) {
+ printer->printf("%s\n", buf);
+ }
+ js::jit::JitSpew(js::jit::JitSpew_Codegen, "%s", buf);
+ }
+ }
+#endif
+
+ Register getStackPointer() const { return StackPointer; }
+
+ protected:
+ bool isFinished;
+
+ public:
+ void finish();
+ bool appendRawCode(const uint8_t* code, size_t numBytes);
+ bool reserve(size_t size);
+ bool swapBuffer(wasm::Bytes& bytes);
+ void executableCopy(void* buffer);
+ void copyJumpRelocationTable(uint8_t* dest);
+ void copyDataRelocationTable(uint8_t* dest);
+
+ // Size of the instruction stream, in bytes.
+ size_t size() const;
+ // Size of the jump relocation table, in bytes.
+ size_t jumpRelocationTableBytes() const;
+ size_t dataRelocationTableBytes() const;
+
+ // Size of the data table, in bytes.
+ size_t bytesNeeded() const;
+
+ // Write a blob of binary into the instruction stream *OR*
+ // into a destination address. If dest is nullptr (the default), then the
+ // instruction gets written into the instruction stream. If dest is not null
+ // it is interpreted as a pointer to the location that we want the
+ // instruction to be written.
+ BufferOffset writeInst(uint32_t x, uint32_t* dest = nullptr);
+ // A static variant for the cases where we don't want to have an assembler
+ // object at all. Normally, you would use the dummy (nullptr) object.
+ static void WriteInstStatic(uint32_t x, uint32_t* dest);
+
+ public:
+ BufferOffset haltingAlign(int alignment);
+ BufferOffset nopAlign(int alignment);
+ BufferOffset as_nop();
+
+ // Branch and jump instructions
+ BufferOffset as_bal(BOffImm16 off);
+ BufferOffset as_b(BOffImm16 off);
+
+ InstImm getBranchCode(JumpOrCall jumpOrCall);
+ InstImm getBranchCode(Register s, Register t, Condition c);
+ InstImm getBranchCode(Register s, Condition c);
+ InstImm getBranchCode(FloatTestKind testKind, FPConditionBit fcc);
+
+ BufferOffset as_j(JOffImm26 off);
+ BufferOffset as_jal(JOffImm26 off);
+
+ BufferOffset as_jr(Register rs);
+ BufferOffset as_jalr(Register rs);
+
+ // Arithmetic instructions
+ BufferOffset as_addu(Register rd, Register rs, Register rt);
+ BufferOffset as_addiu(Register rd, Register rs, int32_t j);
+ BufferOffset as_daddu(Register rd, Register rs, Register rt);
+ BufferOffset as_daddiu(Register rd, Register rs, int32_t j);
+ BufferOffset as_subu(Register rd, Register rs, Register rt);
+ BufferOffset as_dsubu(Register rd, Register rs, Register rt);
+ BufferOffset as_mult(Register rs, Register rt);
+ BufferOffset as_multu(Register rs, Register rt);
+ BufferOffset as_dmult(Register rs, Register rt);
+ BufferOffset as_dmultu(Register rs, Register rt);
+ BufferOffset as_div(Register rs, Register rt);
+ BufferOffset as_divu(Register rs, Register rt);
+ BufferOffset as_mul(Register rd, Register rs, Register rt);
+ BufferOffset as_madd(Register rs, Register rt);
+ BufferOffset as_maddu(Register rs, Register rt);
+ BufferOffset as_ddiv(Register rs, Register rt);
+ BufferOffset as_ddivu(Register rs, Register rt);
+
+ BufferOffset as_muh(Register rd, Register rs, Register rt);
+ BufferOffset as_muhu(Register rd, Register rs, Register rt);
+ BufferOffset as_mulu(Register rd, Register rs, Register rt);
+ BufferOffset as_dmuh(Register rd, Register rs, Register rt);
+ BufferOffset as_dmuhu(Register rd, Register rs, Register rt);
+ BufferOffset as_dmul(Register rd, Register rs, Register rt);
+ BufferOffset as_dmulu(Register rd, Register rs, Register rt);
+ BufferOffset as_div(Register rd, Register rs, Register rt);
+ BufferOffset as_divu(Register rd, Register rs, Register rt);
+ BufferOffset as_mod(Register rd, Register rs, Register rt);
+ BufferOffset as_modu(Register rd, Register rs, Register rt);
+ BufferOffset as_ddiv(Register rd, Register rs, Register rt);
+ BufferOffset as_ddivu(Register rd, Register rs, Register rt);
+ BufferOffset as_dmod(Register rd, Register rs, Register rt);
+ BufferOffset as_dmodu(Register rd, Register rs, Register rt);
+
+ // Logical instructions
+ BufferOffset as_and(Register rd, Register rs, Register rt);
+ BufferOffset as_or(Register rd, Register rs, Register rt);
+ BufferOffset as_xor(Register rd, Register rs, Register rt);
+ BufferOffset as_nor(Register rd, Register rs, Register rt);
+
+ BufferOffset as_andi(Register rd, Register rs, int32_t j);
+ BufferOffset as_ori(Register rd, Register rs, int32_t j);
+ BufferOffset as_xori(Register rd, Register rs, int32_t j);
+ BufferOffset as_lui(Register rd, int32_t j);
+
+ // Shift instructions
+ // as_sll(zero, zero, x) instructions are reserved as nop
+ BufferOffset as_sll(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_dsll(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_dsll32(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_sllv(Register rd, Register rt, Register rs);
+ BufferOffset as_dsllv(Register rd, Register rt, Register rs);
+ BufferOffset as_srl(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_dsrl(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_dsrl32(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_srlv(Register rd, Register rt, Register rs);
+ BufferOffset as_dsrlv(Register rd, Register rt, Register rs);
+ BufferOffset as_sra(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_dsra(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_dsra32(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_srav(Register rd, Register rt, Register rs);
+ BufferOffset as_rotr(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_rotrv(Register rd, Register rt, Register rs);
+ BufferOffset as_dsrav(Register rd, Register rt, Register rs);
+ BufferOffset as_drotr(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_drotr32(Register rd, Register rt, uint16_t sa);
+ BufferOffset as_drotrv(Register rd, Register rt, Register rs);
+
+ // Load and store instructions
+ BufferOffset as_lb(Register rd, Register rs, int16_t off);
+ BufferOffset as_lbu(Register rd, Register rs, int16_t off);
+ BufferOffset as_lh(Register rd, Register rs, int16_t off);
+ BufferOffset as_lhu(Register rd, Register rs, int16_t off);
+ BufferOffset as_lw(Register rd, Register rs, int16_t off);
+ BufferOffset as_lwu(Register rd, Register rs, int16_t off);
+ BufferOffset as_lwl(Register rd, Register rs, int16_t off);
+ BufferOffset as_lwr(Register rd, Register rs, int16_t off);
+ BufferOffset as_ll(Register rd, Register rs, int16_t off);
+ BufferOffset as_lld(Register rd, Register rs, int16_t off);
+ BufferOffset as_ld(Register rd, Register rs, int16_t off);
+ BufferOffset as_ldl(Register rd, Register rs, int16_t off);
+ BufferOffset as_ldr(Register rd, Register rs, int16_t off);
+ BufferOffset as_sb(Register rd, Register rs, int16_t off);
+ BufferOffset as_sh(Register rd, Register rs, int16_t off);
+ BufferOffset as_sw(Register rd, Register rs, int16_t off);
+ BufferOffset as_swl(Register rd, Register rs, int16_t off);
+ BufferOffset as_swr(Register rd, Register rs, int16_t off);
+ BufferOffset as_sc(Register rd, Register rs, int16_t off);
+ BufferOffset as_scd(Register rd, Register rs, int16_t off);
+ BufferOffset as_sd(Register rd, Register rs, int16_t off);
+ BufferOffset as_sdl(Register rd, Register rs, int16_t off);
+ BufferOffset as_sdr(Register rd, Register rs, int16_t off);
+
+ // Loongson-specific load and store instructions
+ BufferOffset as_gslbx(Register rd, Register rs, Register ri, int16_t off);
+ BufferOffset as_gssbx(Register rd, Register rs, Register ri, int16_t off);
+ BufferOffset as_gslhx(Register rd, Register rs, Register ri, int16_t off);
+ BufferOffset as_gsshx(Register rd, Register rs, Register ri, int16_t off);
+ BufferOffset as_gslwx(Register rd, Register rs, Register ri, int16_t off);
+ BufferOffset as_gsswx(Register rd, Register rs, Register ri, int16_t off);
+ BufferOffset as_gsldx(Register rd, Register rs, Register ri, int16_t off);
+ BufferOffset as_gssdx(Register rd, Register rs, Register ri, int16_t off);
+ BufferOffset as_gslq(Register rh, Register rl, Register rs, int16_t off);
+ BufferOffset as_gssq(Register rh, Register rl, Register rs, int16_t off);
+
+ // Move from HI/LO register.
+ BufferOffset as_mfhi(Register rd);
+ BufferOffset as_mflo(Register rd);
+
+ // Set on less than.
+ BufferOffset as_slt(Register rd, Register rs, Register rt);
+ BufferOffset as_sltu(Register rd, Register rs, Register rt);
+ BufferOffset as_slti(Register rd, Register rs, int32_t j);
+ BufferOffset as_sltiu(Register rd, Register rs, uint32_t j);
+
+ // Conditional move.
+ BufferOffset as_movz(Register rd, Register rs, Register rt);
+ BufferOffset as_movn(Register rd, Register rs, Register rt);
+ BufferOffset as_movt(Register rd, Register rs, uint16_t cc = 0);
+ BufferOffset as_movf(Register rd, Register rs, uint16_t cc = 0);
+ BufferOffset as_seleqz(Register rd, Register rs, Register rt);
+ BufferOffset as_selnez(Register rd, Register rs, Register rt);
+
+ // Bit twiddling.
+ BufferOffset as_clz(Register rd, Register rs);
+ BufferOffset as_dclz(Register rd, Register rs);
+ BufferOffset as_wsbh(Register rd, Register rt);
+ BufferOffset as_dsbh(Register rd, Register rt);
+ BufferOffset as_dshd(Register rd, Register rt);
+ BufferOffset as_ins(Register rt, Register rs, uint16_t pos, uint16_t size);
+ BufferOffset as_dins(Register rt, Register rs, uint16_t pos, uint16_t size);
+ BufferOffset as_dinsm(Register rt, Register rs, uint16_t pos, uint16_t size);
+ BufferOffset as_dinsu(Register rt, Register rs, uint16_t pos, uint16_t size);
+ BufferOffset as_ext(Register rt, Register rs, uint16_t pos, uint16_t size);
+ BufferOffset as_dext(Register rt, Register rs, uint16_t pos, uint16_t size);
+ BufferOffset as_dextm(Register rt, Register rs, uint16_t pos, uint16_t size);
+ BufferOffset as_dextu(Register rt, Register rs, uint16_t pos, uint16_t size);
+
+ // Sign extend
+ BufferOffset as_seb(Register rd, Register rt);
+ BufferOffset as_seh(Register rd, Register rt);
+
+ // FP instructions
+
+ BufferOffset as_ldc1(FloatRegister ft, Register base, int32_t off);
+ BufferOffset as_sdc1(FloatRegister ft, Register base, int32_t off);
+
+ BufferOffset as_lwc1(FloatRegister ft, Register base, int32_t off);
+ BufferOffset as_swc1(FloatRegister ft, Register base, int32_t off);
+
+ // Loongson-specific FP load and store instructions
+ BufferOffset as_gsldl(FloatRegister fd, Register base, int32_t off);
+ BufferOffset as_gsldr(FloatRegister fd, Register base, int32_t off);
+ BufferOffset as_gssdl(FloatRegister fd, Register base, int32_t off);
+ BufferOffset as_gssdr(FloatRegister fd, Register base, int32_t off);
+ BufferOffset as_gslsl(FloatRegister fd, Register base, int32_t off);
+ BufferOffset as_gslsr(FloatRegister fd, Register base, int32_t off);
+ BufferOffset as_gsssl(FloatRegister fd, Register base, int32_t off);
+ BufferOffset as_gsssr(FloatRegister fd, Register base, int32_t off);
+ BufferOffset as_gslsx(FloatRegister fd, Register rs, Register ri,
+ int16_t off);
+ BufferOffset as_gsssx(FloatRegister fd, Register rs, Register ri,
+ int16_t off);
+ BufferOffset as_gsldx(FloatRegister fd, Register rs, Register ri,
+ int16_t off);
+ BufferOffset as_gssdx(FloatRegister fd, Register rs, Register ri,
+ int16_t off);
+ BufferOffset as_gslq(FloatRegister rh, FloatRegister rl, Register rs,
+ int16_t off);
+ BufferOffset as_gssq(FloatRegister rh, FloatRegister rl, Register rs,
+ int16_t off);
+
+ BufferOffset as_movs(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_movd(FloatRegister fd, FloatRegister fs);
+
+ BufferOffset as_ctc1(Register rt, FPControl fc);
+ BufferOffset as_cfc1(Register rt, FPControl fc);
+
+ BufferOffset as_mtc1(Register rt, FloatRegister fs);
+ BufferOffset as_mfc1(Register rt, FloatRegister fs);
+
+ BufferOffset as_mthc1(Register rt, FloatRegister fs);
+ BufferOffset as_mfhc1(Register rt, FloatRegister fs);
+ BufferOffset as_dmtc1(Register rt, FloatRegister fs);
+ BufferOffset as_dmfc1(Register rt, FloatRegister fs);
+
+ public:
+ // FP convert instructions
+ BufferOffset as_ceilws(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_floorws(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_roundws(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_truncws(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_truncls(FloatRegister fd, FloatRegister fs);
+
+ BufferOffset as_ceilwd(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_floorwd(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_roundwd(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_truncwd(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_truncld(FloatRegister fd, FloatRegister fs);
+
+ BufferOffset as_cvtdl(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtds(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtdw(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtld(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtls(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtsd(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtsl(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtsw(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtwd(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_cvtws(FloatRegister fd, FloatRegister fs);
+
+ // FP arithmetic instructions
+ BufferOffset as_adds(FloatRegister fd, FloatRegister fs, FloatRegister ft);
+ BufferOffset as_addd(FloatRegister fd, FloatRegister fs, FloatRegister ft);
+ BufferOffset as_subs(FloatRegister fd, FloatRegister fs, FloatRegister ft);
+ BufferOffset as_subd(FloatRegister fd, FloatRegister fs, FloatRegister ft);
+
+ BufferOffset as_abss(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_absd(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_negs(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_negd(FloatRegister fd, FloatRegister fs);
+
+ BufferOffset as_muls(FloatRegister fd, FloatRegister fs, FloatRegister ft);
+ BufferOffset as_muld(FloatRegister fd, FloatRegister fs, FloatRegister ft);
+ BufferOffset as_divs(FloatRegister fd, FloatRegister fs, FloatRegister ft);
+ BufferOffset as_divd(FloatRegister fd, FloatRegister fs, FloatRegister ft);
+ BufferOffset as_sqrts(FloatRegister fd, FloatRegister fs);
+ BufferOffset as_sqrtd(FloatRegister fd, FloatRegister fs);
+
+ BufferOffset as_max(FloatFormat fmt, FloatRegister fd, FloatRegister fs,
+ FloatRegister ft);
+ BufferOffset as_min(FloatFormat fmt, FloatRegister fd, FloatRegister fs,
+ FloatRegister ft);
+
+ // FP compare instructions
+ BufferOffset as_cf(FloatFormat fmt, FloatRegister fs, FloatRegister ft,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_cun(FloatFormat fmt, FloatRegister fs, FloatRegister ft,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_ceq(FloatFormat fmt, FloatRegister fs, FloatRegister ft,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_cueq(FloatFormat fmt, FloatRegister fs, FloatRegister ft,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_colt(FloatFormat fmt, FloatRegister fs, FloatRegister ft,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_cult(FloatFormat fmt, FloatRegister fs, FloatRegister ft,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_cole(FloatFormat fmt, FloatRegister fs, FloatRegister ft,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_cule(FloatFormat fmt, FloatRegister fs, FloatRegister ft,
+ FPConditionBit fcc = FCC0);
+
+ // FP conditional move.
+ BufferOffset as_movt(FloatFormat fmt, FloatRegister fd, FloatRegister fs,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_movf(FloatFormat fmt, FloatRegister fd, FloatRegister fs,
+ FPConditionBit fcc = FCC0);
+ BufferOffset as_movz(FloatFormat fmt, FloatRegister fd, FloatRegister fs,
+ Register rt);
+ BufferOffset as_movn(FloatFormat fmt, FloatRegister fd, FloatRegister fs,
+ Register rt);
+
+ // Conditional trap operations
+ BufferOffset as_tge(Register rs, Register rt, uint32_t code = 0);
+ BufferOffset as_tgeu(Register rs, Register rt, uint32_t code = 0);
+ BufferOffset as_tlt(Register rs, Register rt, uint32_t code = 0);
+ BufferOffset as_tltu(Register rs, Register rt, uint32_t code = 0);
+ BufferOffset as_teq(Register rs, Register rt, uint32_t code = 0);
+ BufferOffset as_tne(Register rs, Register rt, uint32_t code = 0);
+
+ // label operations
+ void bind(Label* label, BufferOffset boff = BufferOffset());
+ virtual void bind(InstImm* inst, uintptr_t branch, uintptr_t target) = 0;
+ void bind(CodeLabel* label) { label->target()->bind(currentOffset()); }
+ uint32_t currentOffset() { return nextOffset().getOffset(); }
+ void retarget(Label* label, Label* target);
+
+ void call(Label* label);
+ void call(void* target);
+
+ void as_break(uint32_t code);
+ void as_sync(uint32_t stype = 0);
+
+ public:
+ static bool SupportsFloatingPoint() {
+#if (defined(__mips_hard_float) && !defined(__mips_single_float)) || \
+ defined(JS_SIMULATOR_MIPS32) || defined(JS_SIMULATOR_MIPS64)
+ return true;
+#else
+ return false;
+#endif
+ }
+ static bool SupportsUnalignedAccesses() { return true; }
+ static bool SupportsFastUnalignedFPAccesses() { return false; }
+
+ static bool HasRoundInstruction(RoundingMode mode) { return false; }
+
+ protected:
+ InstImm invertBranch(InstImm branch, BOffImm16 skipOffset);
+ void addPendingJump(BufferOffset src, ImmPtr target, RelocationKind kind) {
+ enoughMemory_ &= jumps_.append(RelativePatch(src, target.value, kind));
+ if (kind == RelocationKind::JITCODE) {
+ jumpRelocations_.writeUnsigned(src.getOffset());
+ }
+ }
+
+ void addLongJump(BufferOffset src, BufferOffset dst) {
+ CodeLabel cl;
+ cl.patchAt()->bind(src.getOffset());
+ cl.target()->bind(dst.getOffset());
+ cl.setLinkMode(CodeLabel::JumpImmediate);
+ addCodeLabel(std::move(cl));
+ }
+
+ public:
+ void flushBuffer() {}
+
+ void comment(const char* msg) { spew("; %s", msg); }
+
+ static uint32_t NopSize() { return 4; }
+
+ static void PatchWrite_Imm32(CodeLocationLabel label, Imm32 imm);
+
+ static uint32_t AlignDoubleArg(uint32_t offset) {
+ return (offset + 1U) & ~1U;
+ }
+
+ static uint8_t* NextInstruction(uint8_t* instruction,
+ uint32_t* count = nullptr);
+
+ static void ToggleToJmp(CodeLocationLabel inst_);
+ static void ToggleToCmp(CodeLocationLabel inst_);
+
+ static void UpdateLuiOriValue(Instruction* inst0, Instruction* inst1,
+ uint32_t value);
+
+ void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
+ const Disassembler::HeapAccess& heapAccess) {
+ // Implement this if we implement a disassembler.
+ }
+}; // AssemblerMIPSShared
+
+// sll zero, zero, 0
+const uint32_t NopInst = 0x00000000;
+
+// An Instruction is a structure for both encoding and decoding any and all
+// MIPS instructions.
+class Instruction {
+ protected:
+ uint32_t data;
+
+ // Standard constructor
+ Instruction(uint32_t data_) : data(data_) {}
+
+ // You should never create an instruction directly. You should create a
+ // more specific instruction which will eventually call one of these
+ // constructors for you.
+ public:
+ uint32_t encode() const { return data; }
+
+ void makeNop() { data = NopInst; }
+
+ void setData(uint32_t data) { this->data = data; }
+
+ const Instruction& operator=(const Instruction& src) {
+ data = src.data;
+ return *this;
+ }
+
+ // Extract the one particular bit.
+ uint32_t extractBit(uint32_t bit) { return (encode() >> bit) & 1; }
+ // Extract a bit field out of the instruction
+ uint32_t extractBitField(uint32_t hi, uint32_t lo) {
+ return (encode() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+ // Since all MIPS instructions have opcode, the opcode
+ // extractor resides in the base class.
+ uint32_t extractOpcode() {
+ return extractBitField(OpcodeShift + OpcodeBits - 1, OpcodeShift);
+ }
+ // Return the fields at their original place in the instruction encoding.
+ OpcodeField OpcodeFieldRaw() const {
+ return static_cast<OpcodeField>(encode() & OpcodeMask);
+ }
+
+ // Get the next instruction in the instruction stream.
+ // This does neat things like ignoreconstant pools and their guards.
+ Instruction* next();
+
+ // Sometimes, an api wants a uint32_t (or a pointer to it) rather than
+ // an instruction. raw() just coerces this into a pointer to a uint32_t
+ const uint32_t* raw() const { return &data; }
+ uint32_t size() const { return 4; }
+}; // Instruction
+
+// make sure that it is the right size
+static_assert(sizeof(Instruction) == 4,
+ "Size of Instruction class has to be 4 bytes.");
+
+class InstNOP : public Instruction {
+ public:
+ InstNOP() : Instruction(NopInst) {}
+};
+
+// Class for register type instructions.
+class InstReg : public Instruction {
+ public:
+ InstReg(OpcodeField op, Register rd, FunctionField ff)
+ : Instruction(op | RD(rd) | ff) {}
+ InstReg(OpcodeField op, Register rs, Register rt, FunctionField ff)
+ : Instruction(op | RS(rs) | RT(rt) | ff) {}
+ InstReg(OpcodeField op, Register rs, Register rt, Register rd,
+ FunctionField ff)
+ : Instruction(op | RS(rs) | RT(rt) | RD(rd) | ff) {}
+ InstReg(OpcodeField op, Register rs, Register rt, Register rd, uint32_t sa,
+ FunctionField ff)
+ : Instruction(op | RS(rs) | RT(rt) | RD(rd) | SA(sa) | ff) {}
+ InstReg(OpcodeField op, RSField rs, Register rt, Register rd, uint32_t sa,
+ FunctionField ff)
+ : Instruction(op | rs | RT(rt) | RD(rd) | SA(sa) | ff) {}
+ InstReg(OpcodeField op, Register rs, RTField rt, Register rd, uint32_t sa,
+ FunctionField ff)
+ : Instruction(op | RS(rs) | rt | RD(rd) | SA(sa) | ff) {}
+ InstReg(OpcodeField op, Register rs, uint32_t cc, Register rd, uint32_t sa,
+ FunctionField ff)
+ : Instruction(op | RS(rs) | cc | RD(rd) | SA(sa) | ff) {}
+ InstReg(OpcodeField op, uint32_t code, FunctionField ff)
+ : Instruction(op | code | ff) {}
+ // for float point
+ InstReg(OpcodeField op, RSField rs, Register rt, uint32_t fs)
+ : Instruction(op | rs | RT(rt) | FS(fs)) {}
+ InstReg(OpcodeField op, RSField rs, Register rt, FloatRegister rd)
+ : Instruction(op | rs | RT(rt) | RD(rd)) {}
+ InstReg(OpcodeField op, RSField rs, Register rt, FloatRegister rd,
+ uint32_t sa, FunctionField ff)
+ : Instruction(op | rs | RT(rt) | RD(rd) | SA(sa) | ff) {}
+ InstReg(OpcodeField op, RSField rs, Register rt, FloatRegister fs,
+ FloatRegister fd, FunctionField ff)
+ : Instruction(op | rs | RT(rt) | RD(fs) | SA(fd) | ff) {}
+ InstReg(OpcodeField op, RSField rs, FloatRegister ft, FloatRegister fs,
+ FloatRegister fd, FunctionField ff)
+ : Instruction(op | rs | RT(ft) | RD(fs) | SA(fd) | ff) {}
+ InstReg(OpcodeField op, RSField rs, FloatRegister ft, FloatRegister fd,
+ uint32_t sa, FunctionField ff)
+ : Instruction(op | rs | RT(ft) | RD(fd) | SA(sa) | ff) {}
+
+ uint32_t extractRS() {
+ return extractBitField(RSShift + RSBits - 1, RSShift);
+ }
+ uint32_t extractRT() {
+ return extractBitField(RTShift + RTBits - 1, RTShift);
+ }
+ uint32_t extractRD() {
+ return extractBitField(RDShift + RDBits - 1, RDShift);
+ }
+ uint32_t extractSA() {
+ return extractBitField(SAShift + SABits - 1, SAShift);
+ }
+ uint32_t extractFunctionField() {
+ return extractBitField(FunctionShift + FunctionBits - 1, FunctionShift);
+ }
+};
+
+// Class for branch, load and store instructions with immediate offset.
+class InstImm : public Instruction {
+ public:
+ void extractImm16(BOffImm16* dest);
+
+ InstImm(OpcodeField op, Register rs, Register rt, BOffImm16 off)
+ : Instruction(op | RS(rs) | RT(rt) | off.encode()) {}
+ InstImm(OpcodeField op, Register rs, RTField rt, BOffImm16 off)
+ : Instruction(op | RS(rs) | rt | off.encode()) {}
+ InstImm(OpcodeField op, RSField rs, uint32_t cc, BOffImm16 off)
+ : Instruction(op | rs | cc | off.encode()) {}
+ InstImm(OpcodeField op, Register rs, Register rt, Imm16 off)
+ : Instruction(op | RS(rs) | RT(rt) | off.encode()) {}
+ InstImm(uint32_t raw) : Instruction(raw) {}
+ // For floating-point loads and stores.
+ InstImm(OpcodeField op, Register rs, FloatRegister rt, Imm16 off)
+ : Instruction(op | RS(rs) | RT(rt) | off.encode()) {}
+
+ uint32_t extractOpcode() {
+ return extractBitField(OpcodeShift + OpcodeBits - 1, OpcodeShift);
+ }
+ void setOpcode(OpcodeField op) { data = (data & ~OpcodeMask) | op; }
+ uint32_t extractRS() {
+ return extractBitField(RSShift + RSBits - 1, RSShift);
+ }
+ uint32_t extractRT() {
+ return extractBitField(RTShift + RTBits - 1, RTShift);
+ }
+ void setRT(RTField rt) { data = (data & ~RTMask) | rt; }
+ uint32_t extractImm16Value() {
+ return extractBitField(Imm16Shift + Imm16Bits - 1, Imm16Shift);
+ }
+ void setBOffImm16(BOffImm16 off) {
+ // Reset immediate field and replace it
+ data = (data & ~Imm16Mask) | off.encode();
+ }
+ void setImm16(Imm16 off) {
+ // Reset immediate field and replace it
+ data = (data & ~Imm16Mask) | off.encode();
+ }
+};
+
+// Class for Jump type instructions.
+class InstJump : public Instruction {
+ public:
+ InstJump(OpcodeField op, JOffImm26 off) : Instruction(op | off.encode()) {}
+
+ uint32_t extractImm26Value() {
+ return extractBitField(Imm26Shift + Imm26Bits - 1, Imm26Shift);
+ }
+};
+
+// Class for Loongson-specific instructions
+class InstGS : public Instruction {
+ public:
+ // For indexed loads and stores.
+ InstGS(OpcodeField op, Register rs, Register rt, Register rd, Imm8 off,
+ FunctionField ff)
+ : Instruction(op | RS(rs) | RT(rt) | RD(rd) | off.encode(3) | ff) {}
+ InstGS(OpcodeField op, Register rs, FloatRegister rt, Register rd, Imm8 off,
+ FunctionField ff)
+ : Instruction(op | RS(rs) | RT(rt) | RD(rd) | off.encode(3) | ff) {}
+ // For quad-word loads and stores.
+ InstGS(OpcodeField op, Register rs, Register rt, Register rz, GSImm13 off,
+ FunctionField ff)
+ : Instruction(op | RS(rs) | RT(rt) | RZ(rz) | off.encode(6) | ff) {}
+ InstGS(OpcodeField op, Register rs, FloatRegister rt, FloatRegister rz,
+ GSImm13 off, FunctionField ff)
+ : Instruction(op | RS(rs) | RT(rt) | RZ(rz) | off.encode(6) | ff) {}
+ InstGS(uint32_t raw) : Instruction(raw) {}
+ // For floating-point unaligned loads and stores.
+ InstGS(OpcodeField op, Register rs, FloatRegister rt, Imm8 off,
+ FunctionField ff)
+ : Instruction(op | RS(rs) | RT(rt) | off.encode(6) | ff) {}
+};
+
+inline bool IsUnaligned(const wasm::MemoryAccessDesc& access) {
+ if (!access.align()) {
+ return false;
+ }
+
+#ifdef JS_CODEGEN_MIPS32
+ if (access.type() == Scalar::Int64 && access.align() >= 4) {
+ return false;
+ }
+#endif
+
+ return access.align() < access.byteSize();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_Assembler_mips_shared_h */
diff --git a/js/src/jit/mips-shared/AtomicOperations-mips-shared.h b/js/src/jit/mips-shared/AtomicOperations-mips-shared.h
new file mode 100644
index 0000000000..5ef11fd8c2
--- /dev/null
+++ b/js/src/jit/mips-shared/AtomicOperations-mips-shared.h
@@ -0,0 +1,521 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* For documentation, see jit/AtomicOperations.h */
+
+// NOTE, MIPS32 unlike MIPS64 doesn't provide hardware support for lock-free
+// 64-bit atomics. We lie down below about 8-byte atomics being always lock-
+// free in order to support wasm jit. The 64-bit atomic for MIPS32 do not use
+// __atomic intrinsic and therefore do not relay on -latomic.
+// Access to a aspecific 64-bit variable in memory is protected by an
+// AddressLock whose instance is shared between jit and AtomicOperations.
+
+#ifndef jit_mips_shared_AtomicOperations_mips_shared_h
+#define jit_mips_shared_AtomicOperations_mips_shared_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Types.h"
+
+#include "builtin/AtomicsObject.h"
+#include "vm/Uint8Clamped.h"
+
+#if !defined(__clang__) && !defined(__GNUC__)
+# error "This file only for gcc-compatible compilers"
+#endif
+
+#if defined(JS_SIMULATOR_MIPS32) && !defined(__i386__)
+# error "The MIPS32 simulator atomics assume x86"
+#endif
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+struct AddressLock {
+ public:
+ void acquire();
+ void release();
+
+ private:
+ uint32_t spinlock;
+};
+
+static_assert(sizeof(AddressLock) == sizeof(uint32_t),
+ "AddressLock must be 4 bytes for it to be consumed by jit");
+
+// For now use a single global AddressLock.
+static AddressLock gAtomic64Lock;
+
+struct MOZ_RAII AddressGuard {
+ explicit AddressGuard(void* addr) { gAtomic64Lock.acquire(); }
+
+ ~AddressGuard() { gAtomic64Lock.release(); }
+};
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+inline bool js::jit::AtomicOperations::hasAtomic8() { return true; }
+
+inline bool js::jit::AtomicOperations::isLockfree8() {
+ MOZ_ASSERT(__atomic_always_lock_free(sizeof(int8_t), 0));
+ MOZ_ASSERT(__atomic_always_lock_free(sizeof(int16_t), 0));
+ MOZ_ASSERT(__atomic_always_lock_free(sizeof(int32_t), 0));
+#if defined(JS_64BIT)
+ MOZ_ASSERT(__atomic_always_lock_free(sizeof(int64_t), 0));
+#endif
+ return true;
+}
+
+inline void js::jit::AtomicOperations::fenceSeqCst() {
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+}
+
+template <typename T>
+inline T js::jit::AtomicOperations::loadSeqCst(T* addr) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ T v;
+ __atomic_load(addr, &v, __ATOMIC_SEQ_CST);
+ return v;
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::loadSeqCst(int64_t* addr) {
+ AddressGuard guard(addr);
+ return *addr;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::loadSeqCst(uint64_t* addr) {
+ AddressGuard guard(addr);
+ return *addr;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline void js::jit::AtomicOperations::storeSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ __atomic_store(addr, &val, __ATOMIC_SEQ_CST);
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline void js::jit::AtomicOperations::storeSeqCst(int64_t* addr, int64_t val) {
+ AddressGuard guard(addr);
+ *addr = val;
+}
+
+template <>
+inline void js::jit::AtomicOperations::storeSeqCst(uint64_t* addr,
+ uint64_t val) {
+ AddressGuard guard(addr);
+ *addr = val;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline T js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval,
+ T newval) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST,
+ __ATOMIC_SEQ_CST);
+ return oldval;
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::compareExchangeSeqCst(
+ int64_t* addr, int64_t oldval, int64_t newval) {
+ AddressGuard guard(addr);
+ int64_t val = *addr;
+ if (val == oldval) {
+ *addr = newval;
+ }
+ return val;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::compareExchangeSeqCst(
+ uint64_t* addr, uint64_t oldval, uint64_t newval) {
+ AddressGuard guard(addr);
+ uint64_t val = *addr;
+ if (val == oldval) {
+ *addr = newval;
+ }
+ return val;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::fetchAddSeqCst(int64_t* addr,
+ int64_t val) {
+ AddressGuard guard(addr);
+ int64_t old = *addr;
+ *addr = old + val;
+ return old;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::fetchAddSeqCst(uint64_t* addr,
+ uint64_t val) {
+ AddressGuard guard(addr);
+ uint64_t old = *addr;
+ *addr = old + val;
+ return old;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::fetchSubSeqCst(int64_t* addr,
+ int64_t val) {
+ AddressGuard guard(addr);
+ int64_t old = *addr;
+ *addr = old - val;
+ return old;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::fetchSubSeqCst(uint64_t* addr,
+ uint64_t val) {
+ AddressGuard guard(addr);
+ uint64_t old = *addr;
+ *addr = old - val;
+ return old;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::fetchAndSeqCst(int64_t* addr,
+ int64_t val) {
+ AddressGuard guard(addr);
+ int64_t old = *addr;
+ *addr = old & val;
+ return old;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::fetchAndSeqCst(uint64_t* addr,
+ uint64_t val) {
+ AddressGuard guard(addr);
+ uint64_t old = *addr;
+ *addr = old & val;
+ return old;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::fetchOrSeqCst(int64_t* addr,
+ int64_t val) {
+ AddressGuard guard(addr);
+ int64_t old = *addr;
+ *addr = old | val;
+ return old;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::fetchOrSeqCst(uint64_t* addr,
+ uint64_t val) {
+ AddressGuard guard(addr);
+ uint64_t old = *addr;
+ *addr = old | val;
+ return old;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST);
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::fetchXorSeqCst(int64_t* addr,
+ int64_t val) {
+ AddressGuard guard(addr);
+ int64_t old = *addr;
+ *addr = old ^ val;
+ return old;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::fetchXorSeqCst(uint64_t* addr,
+ uint64_t val) {
+ AddressGuard guard(addr);
+ uint64_t old = *addr;
+ *addr = old ^ val;
+ return old;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ T v;
+ __atomic_load(addr, &v, __ATOMIC_RELAXED);
+ return v;
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::loadSafeWhenRacy(int64_t* addr) {
+ return *addr;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::loadSafeWhenRacy(uint64_t* addr) {
+ return *addr;
+}
+
+#endif
+
+template <>
+inline uint8_clamped js::jit::AtomicOperations::loadSafeWhenRacy(
+ uint8_clamped* addr) {
+ uint8_t v;
+ __atomic_load(&addr->val, &v, __ATOMIC_RELAXED);
+ return uint8_clamped(v);
+}
+
+template <>
+inline float js::jit::AtomicOperations::loadSafeWhenRacy(float* addr) {
+ return *addr;
+}
+
+template <>
+inline double js::jit::AtomicOperations::loadSafeWhenRacy(double* addr) {
+ return *addr;
+}
+
+} // namespace jit
+} // namespace js
+
+template <typename T>
+inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ __atomic_store(addr, &val, __ATOMIC_RELAXED);
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline void js::jit::AtomicOperations::storeSafeWhenRacy(int64_t* addr,
+ int64_t val) {
+ *addr = val;
+}
+
+template <>
+inline void js::jit::AtomicOperations::storeSafeWhenRacy(uint64_t* addr,
+ uint64_t val) {
+ *addr = val;
+}
+
+#endif
+
+template <>
+inline void js::jit::AtomicOperations::storeSafeWhenRacy(uint8_clamped* addr,
+ uint8_clamped val) {
+ __atomic_store(&addr->val, &val.val, __ATOMIC_RELAXED);
+}
+
+template <>
+inline void js::jit::AtomicOperations::storeSafeWhenRacy(float* addr,
+ float val) {
+ *addr = val;
+}
+
+template <>
+inline void js::jit::AtomicOperations::storeSafeWhenRacy(double* addr,
+ double val) {
+ *addr = val;
+}
+
+} // namespace jit
+} // namespace js
+
+inline void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest,
+ const void* src,
+ size_t nbytes) {
+ MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes));
+ MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes));
+ ::memcpy(dest, src, nbytes);
+}
+
+inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest,
+ const void* src,
+ size_t nbytes) {
+ ::memmove(dest, src, nbytes);
+}
+
+template <typename T>
+inline T js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= sizeof(void*),
+ "atomics supported up to pointer size only");
+ T v;
+ __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST);
+ return v;
+}
+
+namespace js {
+namespace jit {
+
+#if !defined(JS_64BIT)
+
+template <>
+inline int64_t js::jit::AtomicOperations::exchangeSeqCst(int64_t* addr,
+ int64_t val) {
+ AddressGuard guard(addr);
+ int64_t old = *addr;
+ *addr = val;
+ return old;
+}
+
+template <>
+inline uint64_t js::jit::AtomicOperations::exchangeSeqCst(uint64_t* addr,
+ uint64_t val) {
+ AddressGuard guard(addr);
+ uint64_t old = *addr;
+ *addr = val;
+ return old;
+}
+
+#endif
+
+} // namespace jit
+} // namespace js
+
+#if !defined(JS_64BIT)
+
+inline void js::jit::AddressLock::acquire() {
+ uint32_t zero = 0;
+ uint32_t one = 1;
+ while (!__atomic_compare_exchange(&spinlock, &zero, &one, true,
+ __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)) {
+ zero = 0;
+ }
+}
+
+inline void js::jit::AddressLock::release() {
+ uint32_t zero = 0;
+ __atomic_store(&spinlock, &zero, __ATOMIC_SEQ_CST);
+}
+
+#endif
+
+#endif // jit_mips_shared_AtomicOperations_mips_shared_h
diff --git a/js/src/jit/mips-shared/BaselineIC-mips-shared.cpp b/js/src/jit/mips-shared/BaselineIC-mips-shared.cpp
new file mode 100644
index 0000000000..6e21edc0ba
--- /dev/null
+++ b/js/src/jit/mips-shared/BaselineIC-mips-shared.cpp
@@ -0,0 +1,37 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/BaselineIC.h"
+#include "jit/SharedICHelpers.h"
+
+using namespace js;
+using namespace js::jit;
+
+namespace js {
+namespace jit {
+
+bool ICCompare_Double::Compiler::generateStubCode(MacroAssembler& masm) {
+ Label failure, isNaN;
+ masm.ensureDouble(R0, FloatReg0, &failure);
+ masm.ensureDouble(R1, FloatReg1, &failure);
+
+ Register dest = R0.scratchReg();
+
+ Assembler::DoubleCondition doubleCond = JSOpToDoubleCondition(op);
+
+ masm.ma_cmp_set_double(dest, FloatReg0, FloatReg1, doubleCond);
+
+ masm.tagValue(JSVAL_TYPE_BOOLEAN, dest, R0);
+ EmitReturnFromIC(masm);
+
+ // Failure case - jump to next stub
+ masm.bind(&failure);
+ EmitStubGuardFailure(masm);
+ return true;
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp b/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp
new file mode 100644
index 0000000000..68f1e5fe34
--- /dev/null
+++ b/js/src/jit/mips-shared/CodeGenerator-mips-shared.cpp
@@ -0,0 +1,2440 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips-shared/CodeGenerator-mips-shared.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jsnum.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitRuntime.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "js/Conversions.h"
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+#include "vm/Shape.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::GenericNaN;
+using JS::ToInt32;
+using mozilla::DebugOnly;
+using mozilla::FloorLog2;
+using mozilla::NegativeInfinity;
+
+// shared
+CodeGeneratorMIPSShared::CodeGeneratorMIPSShared(MIRGenerator* gen,
+ LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorShared(gen, graph, masm) {}
+
+Operand CodeGeneratorMIPSShared::ToOperand(const LAllocation& a) {
+ if (a.isGeneralReg()) {
+ return Operand(a.toGeneralReg()->reg());
+ }
+ if (a.isFloatReg()) {
+ return Operand(a.toFloatReg()->reg());
+ }
+ return Operand(ToAddress(a));
+}
+
+Operand CodeGeneratorMIPSShared::ToOperand(const LAllocation* a) {
+ return ToOperand(*a);
+}
+
+Operand CodeGeneratorMIPSShared::ToOperand(const LDefinition* def) {
+ return ToOperand(def->output());
+}
+
+#ifdef JS_PUNBOX64
+Operand CodeGeneratorMIPSShared::ToOperandOrRegister64(
+ const LInt64Allocation input) {
+ return ToOperand(input.value());
+}
+#else
+Register64 CodeGeneratorMIPSShared::ToOperandOrRegister64(
+ const LInt64Allocation input) {
+ return ToRegister64(input);
+}
+#endif
+
+void CodeGeneratorMIPSShared::branchToBlock(Assembler::FloatFormat fmt,
+ FloatRegister lhs,
+ FloatRegister rhs, MBasicBlock* mir,
+ Assembler::DoubleCondition cond) {
+ // Skip past trivial blocks.
+ Label* label = skipTrivialBlocks(mir)->lir()->label();
+ if (fmt == Assembler::DoubleFloat) {
+ masm.branchDouble(cond, lhs, rhs, label);
+ } else {
+ masm.branchFloat(cond, lhs, rhs, label);
+ }
+}
+
+void OutOfLineBailout::accept(CodeGeneratorMIPSShared* codegen) {
+ codegen->visitOutOfLineBailout(this);
+}
+
+void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
+ const LAllocation* opd = test->getOperand(0);
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ emitBranch(ToRegister(opd), Imm32(0), Assembler::NonZero, ifTrue, ifFalse);
+}
+
+void CodeGenerator::visitCompare(LCompare* comp) {
+ MCompare* mir = comp->mir();
+ Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop());
+ const LAllocation* left = comp->getOperand(0);
+ const LAllocation* right = comp->getOperand(1);
+ const LDefinition* def = comp->getDef(0);
+
+#ifdef JS_CODEGEN_MIPS64
+ if (mir->compareType() == MCompare::Compare_Object ||
+ mir->compareType() == MCompare::Compare_Symbol ||
+ mir->compareType() == MCompare::Compare_UIntPtr ||
+ mir->compareType() == MCompare::Compare_RefOrNull) {
+ if (right->isConstant()) {
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_UIntPtr);
+ masm.cmpPtrSet(cond, ToRegister(left), Imm32(ToInt32(right)),
+ ToRegister(def));
+ } else if (right->isGeneralReg()) {
+ masm.cmpPtrSet(cond, ToRegister(left), ToRegister(right),
+ ToRegister(def));
+ } else {
+ masm.cmpPtrSet(cond, ToRegister(left), ToAddress(right), ToRegister(def));
+ }
+ return;
+ }
+#endif
+
+ if (right->isConstant()) {
+ masm.cmp32Set(cond, ToRegister(left), Imm32(ToInt32(right)),
+ ToRegister(def));
+ } else if (right->isGeneralReg()) {
+ masm.cmp32Set(cond, ToRegister(left), ToRegister(right), ToRegister(def));
+ } else {
+ masm.cmp32Set(cond, ToRegister(left), ToAddress(right), ToRegister(def));
+ }
+}
+
+void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
+ MCompare* mir = comp->cmpMir();
+ Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop());
+
+#ifdef JS_CODEGEN_MIPS64
+ if (mir->compareType() == MCompare::Compare_Object ||
+ mir->compareType() == MCompare::Compare_Symbol ||
+ mir->compareType() == MCompare::Compare_UIntPtr ||
+ mir->compareType() == MCompare::Compare_RefOrNull) {
+ if (comp->right()->isConstant()) {
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_UIntPtr);
+ emitBranch(ToRegister(comp->left()), Imm32(ToInt32(comp->right())), cond,
+ comp->ifTrue(), comp->ifFalse());
+ } else if (comp->right()->isGeneralReg()) {
+ emitBranch(ToRegister(comp->left()), ToRegister(comp->right()), cond,
+ comp->ifTrue(), comp->ifFalse());
+ } else {
+ masm.loadPtr(ToAddress(comp->right()), ScratchRegister);
+ emitBranch(ToRegister(comp->left()), ScratchRegister, cond,
+ comp->ifTrue(), comp->ifFalse());
+ }
+ return;
+ }
+#endif
+
+ if (comp->right()->isConstant()) {
+ emitBranch(ToRegister(comp->left()), Imm32(ToInt32(comp->right())), cond,
+ comp->ifTrue(), comp->ifFalse());
+ } else if (comp->right()->isGeneralReg()) {
+ emitBranch(ToRegister(comp->left()), ToRegister(comp->right()), cond,
+ comp->ifTrue(), comp->ifFalse());
+ } else {
+ masm.load32(ToAddress(comp->right()), ScratchRegister);
+ emitBranch(ToRegister(comp->left()), ScratchRegister, cond, comp->ifTrue(),
+ comp->ifFalse());
+ }
+}
+
+bool CodeGeneratorMIPSShared::generateOutOfLineCode() {
+ if (!CodeGeneratorShared::generateOutOfLineCode()) {
+ return false;
+ }
+
+ if (deoptLabel_.used()) {
+ // All non-table-based bailouts will go here.
+ masm.bind(&deoptLabel_);
+
+ // Push the frame size, so the handler can recover the IonScript.
+ // Frame size is stored in 'ra' and pushed by GenerateBailoutThunk
+ // We have to use 'ra' because generateBailoutTable will implicitly do
+ // the same.
+ masm.move32(Imm32(frameSize()), ra);
+
+ TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
+ masm.jump(handler);
+ }
+
+ return !masm.oom();
+}
+
+void CodeGeneratorMIPSShared::bailoutFrom(Label* label, LSnapshot* snapshot) {
+ MOZ_ASSERT_IF(!masm.oom(), label->used());
+ MOZ_ASSERT_IF(!masm.oom(), !label->bound());
+
+ encode(snapshot);
+
+ InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
+ OutOfLineBailout* ool =
+ new (alloc()) OutOfLineBailout(snapshot, masm.framePushed());
+ addOutOfLineCode(ool,
+ new (alloc()) BytecodeSite(tree, tree->script()->code()));
+
+ masm.retarget(label, ool->entry());
+}
+
+void CodeGeneratorMIPSShared::bailout(LSnapshot* snapshot) {
+ Label label;
+ masm.jump(&label);
+ bailoutFrom(&label, snapshot);
+}
+
+void CodeGenerator::visitMinMaxD(LMinMaxD* ins) {
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+
+ MOZ_ASSERT(first == ToFloatRegister(ins->output()));
+
+ if (ins->mir()->isMax()) {
+ masm.maxDouble(second, first, true);
+ } else {
+ masm.minDouble(second, first, true);
+ }
+}
+
+void CodeGenerator::visitMinMaxF(LMinMaxF* ins) {
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+
+ MOZ_ASSERT(first == ToFloatRegister(ins->output()));
+
+ if (ins->mir()->isMax()) {
+ masm.maxFloat32(second, first, true);
+ } else {
+ masm.minFloat32(second, first, true);
+ }
+}
+
+void CodeGenerator::visitAddI(LAddI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ MOZ_ASSERT(rhs->isConstant() || rhs->isGeneralReg());
+
+ // If there is no snapshot, we don't need to check for overflow
+ if (!ins->snapshot()) {
+ if (rhs->isConstant()) {
+ masm.ma_addu(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ } else {
+ masm.as_addu(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ }
+ return;
+ }
+
+ Label overflow;
+ if (rhs->isConstant()) {
+ masm.ma_add32TestOverflow(ToRegister(dest), ToRegister(lhs),
+ Imm32(ToInt32(rhs)), &overflow);
+ } else {
+ masm.ma_add32TestOverflow(ToRegister(dest), ToRegister(lhs),
+ ToRegister(rhs), &overflow);
+ }
+
+ bailoutFrom(&overflow, ins->snapshot());
+}
+
+void CodeGenerator::visitAddI64(LAddI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void CodeGenerator::visitSubI(LSubI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+
+ MOZ_ASSERT(rhs->isConstant() || rhs->isGeneralReg());
+
+ // If there is no snapshot, we don't need to check for overflow
+ if (!ins->snapshot()) {
+ if (rhs->isConstant()) {
+ masm.ma_subu(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ } else {
+ masm.as_subu(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ }
+ return;
+ }
+
+ Label overflow;
+ if (rhs->isConstant()) {
+ masm.ma_sub32TestOverflow(ToRegister(dest), ToRegister(lhs),
+ Imm32(ToInt32(rhs)), &overflow);
+ } else {
+ masm.ma_sub32TestOverflow(ToRegister(dest), ToRegister(lhs),
+ ToRegister(rhs), &overflow);
+ }
+
+ bailoutFrom(&overflow, ins->snapshot());
+}
+
+void CodeGenerator::visitSubI64(LSubI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void CodeGenerator::visitMulI(LMulI* ins) {
+ const LAllocation* lhs = ins->lhs();
+ const LAllocation* rhs = ins->rhs();
+ Register dest = ToRegister(ins->output());
+ MMul* mul = ins->mir();
+
+ MOZ_ASSERT_IF(mul->mode() == MMul::Integer,
+ !mul->canBeNegativeZero() && !mul->canOverflow());
+
+ if (rhs->isConstant()) {
+ int32_t constant = ToInt32(rhs);
+ Register src = ToRegister(lhs);
+
+ // Bailout on -0.0
+ if (mul->canBeNegativeZero() && constant <= 0) {
+ Assembler::Condition cond =
+ (constant == 0) ? Assembler::LessThan : Assembler::Equal;
+ bailoutCmp32(cond, src, Imm32(0), ins->snapshot());
+ }
+
+ switch (constant) {
+ case -1:
+ if (mul->canOverflow()) {
+ bailoutCmp32(Assembler::Equal, src, Imm32(INT32_MIN),
+ ins->snapshot());
+ }
+
+ masm.ma_negu(dest, src);
+ break;
+ case 0:
+ masm.move32(Imm32(0), dest);
+ break;
+ case 1:
+ masm.move32(src, dest);
+ break;
+ case 2:
+ if (mul->canOverflow()) {
+ Label mulTwoOverflow;
+ masm.ma_add32TestOverflow(dest, src, src, &mulTwoOverflow);
+
+ bailoutFrom(&mulTwoOverflow, ins->snapshot());
+ } else {
+ masm.as_addu(dest, src, src);
+ }
+ break;
+ default:
+ uint32_t shift = FloorLog2(constant);
+
+ if (!mul->canOverflow() && (constant > 0)) {
+ // If it cannot overflow, we can do lots of optimizations.
+ uint32_t rest = constant - (1 << shift);
+
+ // See if the constant has one bit set, meaning it can be
+ // encoded as a bitshift.
+ if ((1 << shift) == constant) {
+ masm.ma_sll(dest, src, Imm32(shift));
+ return;
+ }
+
+ // If the constant cannot be encoded as (1<<C1), see if it can
+ // be encoded as (1<<C1) | (1<<C2), which can be computed
+ // using an add and a shift.
+ uint32_t shift_rest = FloorLog2(rest);
+ if (src != dest && (1u << shift_rest) == rest) {
+ masm.ma_sll(dest, src, Imm32(shift - shift_rest));
+ masm.add32(src, dest);
+ if (shift_rest != 0) {
+ masm.ma_sll(dest, dest, Imm32(shift_rest));
+ }
+ return;
+ }
+ }
+
+ if (mul->canOverflow() && (constant > 0) && (src != dest)) {
+ // To stay on the safe side, only optimize things that are a
+ // power of 2.
+
+ if ((1 << shift) == constant) {
+ // dest = lhs * pow(2, shift)
+ masm.ma_sll(dest, src, Imm32(shift));
+ // At runtime, check (lhs == dest >> shift), if this does
+ // not hold, some bits were lost due to overflow, and the
+ // computation should be resumed as a double.
+ masm.ma_sra(ScratchRegister, dest, Imm32(shift));
+ bailoutCmp32(Assembler::NotEqual, src, ScratchRegister,
+ ins->snapshot());
+ return;
+ }
+ }
+
+ if (mul->canOverflow()) {
+ Label mulConstOverflow;
+ masm.ma_mul32TestOverflow(dest, ToRegister(lhs), Imm32(ToInt32(rhs)),
+ &mulConstOverflow);
+
+ bailoutFrom(&mulConstOverflow, ins->snapshot());
+ } else {
+ masm.ma_mul(dest, src, Imm32(ToInt32(rhs)));
+ }
+ break;
+ }
+ } else {
+ Label multRegOverflow;
+
+ if (mul->canOverflow()) {
+ masm.ma_mul32TestOverflow(dest, ToRegister(lhs), ToRegister(rhs),
+ &multRegOverflow);
+ bailoutFrom(&multRegOverflow, ins->snapshot());
+ } else {
+ masm.as_mul(dest, ToRegister(lhs), ToRegister(rhs));
+ }
+
+ if (mul->canBeNegativeZero()) {
+ Label done;
+ masm.ma_b(dest, dest, &done, Assembler::NonZero, ShortJump);
+
+ // Result is -0 if lhs or rhs is negative.
+ // In that case result must be double value so bailout
+ Register scratch = SecondScratchReg;
+ masm.as_or(scratch, ToRegister(lhs), ToRegister(rhs));
+ bailoutCmp32(Assembler::Signed, scratch, scratch, ins->snapshot());
+
+ masm.bind(&done);
+ }
+ }
+}
+
+void CodeGenerator::visitMulI64(LMulI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
+ const Register64 output = ToOutRegister64(lir);
+
+ if (IsConstant(rhs)) {
+ int64_t constant = ToInt64(rhs);
+ switch (constant) {
+ case -1:
+ masm.neg64(ToRegister64(lhs));
+ return;
+ case 0:
+ masm.xor64(ToRegister64(lhs), ToRegister64(lhs));
+ return;
+ case 1:
+ // nop
+ return;
+ default:
+ if (constant > 0) {
+ if (mozilla::IsPowerOfTwo(static_cast<uint32_t>(constant + 1))) {
+ masm.move64(ToRegister64(lhs), output);
+ masm.lshift64(Imm32(FloorLog2(constant + 1)), output);
+ masm.sub64(ToRegister64(lhs), output);
+ return;
+ } else if (mozilla::IsPowerOfTwo(
+ static_cast<uint32_t>(constant - 1))) {
+ masm.move64(ToRegister64(lhs), output);
+ masm.lshift64(Imm32(FloorLog2(constant - 1u)), output);
+ masm.add64(ToRegister64(lhs), output);
+ return;
+ }
+ // Use shift if constant is power of 2.
+ int32_t shift = mozilla::FloorLog2(constant);
+ if (int64_t(1) << shift == constant) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ return;
+ }
+ }
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
+ }
+ } else {
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp);
+ }
+}
+
+void CodeGenerator::visitDivI(LDivI* ins) {
+ // Extract the registers from this instruction
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register dest = ToRegister(ins->output());
+ Register temp = ToRegister(ins->getTemp(0));
+ MDiv* mir = ins->mir();
+
+ Label done;
+
+ // Handle divide by zero.
+ if (mir->canBeDivideByZero()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else if (mir->canTruncateInfinities()) {
+ // Truncated division by zero is zero (Infinity|0 == 0)
+ Label notzero;
+ masm.ma_b(rhs, rhs, &notzero, Assembler::NonZero, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&notzero);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Zero, rhs, rhs, ins->snapshot());
+ }
+ }
+
+ // Handle an integer overflow exception from -2147483648 / -1.
+ if (mir->canBeNegativeOverflow()) {
+ Label notMinInt;
+ masm.move32(Imm32(INT32_MIN), temp);
+ masm.ma_b(lhs, temp, &notMinInt, Assembler::NotEqual, ShortJump);
+
+ masm.move32(Imm32(-1), temp);
+ if (mir->trapOnError()) {
+ Label ok;
+ masm.ma_b(rhs, temp, &ok, Assembler::NotEqual);
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
+ masm.bind(&ok);
+ } else if (mir->canTruncateOverflow()) {
+ // (-INT32_MIN)|0 == INT32_MIN
+ Label skip;
+ masm.ma_b(rhs, temp, &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(INT32_MIN), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, rhs, temp, ins->snapshot());
+ }
+ masm.bind(&notMinInt);
+ }
+
+ // Handle negative 0. (0/-Y)
+ if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
+ Label nonzero;
+ masm.ma_b(lhs, lhs, &nonzero, Assembler::NonZero, ShortJump);
+ bailoutCmp32(Assembler::LessThan, rhs, Imm32(0), ins->snapshot());
+ masm.bind(&nonzero);
+ }
+ // Note: above safety checks could not be verified as Ion seems to be
+ // smarter and requires double arithmetic in such cases.
+
+ // All regular. Lets call div.
+ if (mir->canTruncateRemainder()) {
+#ifdef MIPSR6
+ masm.as_div(dest, lhs, rhs);
+#else
+ masm.as_div(lhs, rhs);
+ masm.as_mflo(dest);
+#endif
+ } else {
+ MOZ_ASSERT(mir->fallible());
+
+ Label remainderNonZero;
+ masm.ma_div_branch_overflow(dest, lhs, rhs, &remainderNonZero);
+ bailoutFrom(&remainderNonZero, ins->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) {
+ Register lhs = ToRegister(ins->numerator());
+ Register dest = ToRegister(ins->output());
+ Register tmp = ToRegister(ins->getTemp(0));
+ int32_t shift = ins->shift();
+
+ if (shift != 0) {
+ MDiv* mir = ins->mir();
+ if (!mir->isTruncated()) {
+ // If the remainder is going to be != 0, bailout since this must
+ // be a double.
+ masm.ma_sll(tmp, lhs, Imm32(32 - shift));
+ bailoutCmp32(Assembler::NonZero, tmp, tmp, ins->snapshot());
+ }
+
+ if (!mir->canBeNegativeDividend()) {
+ // Numerator is unsigned, so needs no adjusting. Do the shift.
+ masm.ma_sra(dest, lhs, Imm32(shift));
+ return;
+ }
+
+ // Adjust the value so that shifting produces a correctly rounded result
+ // when the numerator is negative. See 10-1 "Signed Division by a Known
+ // Power of 2" in Henry S. Warren, Jr.'s Hacker's Delight.
+ if (shift > 1) {
+ masm.ma_sra(tmp, lhs, Imm32(31));
+ masm.ma_srl(tmp, tmp, Imm32(32 - shift));
+ masm.add32(lhs, tmp);
+ } else {
+ masm.ma_srl(tmp, lhs, Imm32(32 - shift));
+ masm.add32(lhs, tmp);
+ }
+
+ // Do the shift.
+ masm.ma_sra(dest, tmp, Imm32(shift));
+ } else {
+ masm.move32(lhs, dest);
+ }
+}
+
+void CodeGenerator::visitModI(LModI* ins) {
+ // Extract the registers from this instruction
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register dest = ToRegister(ins->output());
+ Register callTemp = ToRegister(ins->callTemp());
+ MMod* mir = ins->mir();
+ Label done, prevent;
+
+ masm.move32(lhs, callTemp);
+
+ // Prevent INT_MIN % -1;
+ // The integer division will give INT_MIN, but we want -(double)INT_MIN.
+ if (mir->canBeNegativeDividend()) {
+ masm.ma_b(lhs, Imm32(INT_MIN), &prevent, Assembler::NotEqual, ShortJump);
+ if (mir->isTruncated()) {
+ // (INT_MIN % -1)|0 == 0
+ Label skip;
+ masm.ma_b(rhs, Imm32(-1), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(-1), ins->snapshot());
+ }
+ masm.bind(&prevent);
+ }
+
+ // 0/X (with X < 0) is bad because both of these values *should* be
+ // doubles, and the result should be -0.0, which cannot be represented in
+ // integers. X/0 is bad because it will give garbage (or abort), when it
+ // should give either \infty, -\infty or NAN.
+
+ // Prevent 0 / X (with X < 0) and X / 0
+ // testing X / Y. Compare Y with 0.
+ // There are three cases: (Y < 0), (Y == 0) and (Y > 0)
+ // If (Y < 0), then we compare X with 0, and bail if X == 0
+ // If (Y == 0), then we simply want to bail.
+ // if (Y > 0), we don't bail.
+
+ if (mir->canBeDivideByZero()) {
+ if (mir->isTruncated()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ Label skip;
+ masm.ma_b(rhs, Imm32(0), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ }
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot());
+ }
+ }
+
+ if (mir->canBeNegativeDividend()) {
+ Label notNegative;
+ masm.ma_b(rhs, Imm32(0), &notNegative, Assembler::GreaterThan, ShortJump);
+ if (mir->isTruncated()) {
+ // NaN|0 == 0 and (0 % -X)|0 == 0
+ Label skip;
+ masm.ma_b(lhs, Imm32(0), &skip, Assembler::NotEqual, ShortJump);
+ masm.move32(Imm32(0), dest);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&skip);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, lhs, Imm32(0), ins->snapshot());
+ }
+ masm.bind(&notNegative);
+ }
+#ifdef MIPSR6
+ masm.as_mod(dest, lhs, rhs);
+#else
+ masm.as_div(lhs, rhs);
+ masm.as_mfhi(dest);
+#endif
+
+ // If X%Y == 0 and X < 0, then we *actually* wanted to return -0.0
+ if (mir->canBeNegativeDividend()) {
+ if (mir->isTruncated()) {
+ // -0.0|0 == 0
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ // See if X < 0
+ masm.ma_b(dest, Imm32(0), &done, Assembler::NotEqual, ShortJump);
+ bailoutCmp32(Assembler::Signed, callTemp, Imm32(0), ins->snapshot());
+ }
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) {
+ Register in = ToRegister(ins->getOperand(0));
+ Register out = ToRegister(ins->getDef(0));
+ MMod* mir = ins->mir();
+ Label negative, done;
+
+ masm.move32(in, out);
+ masm.ma_b(in, in, &done, Assembler::Zero, ShortJump);
+ // Switch based on sign of the lhs.
+ // Positive numbers are just a bitmask
+ masm.ma_b(in, in, &negative, Assembler::Signed, ShortJump);
+ {
+ masm.and32(Imm32((1 << ins->shift()) - 1), out);
+ masm.ma_b(&done, ShortJump);
+ }
+
+ // Negative numbers need a negate, bitmask, negate
+ {
+ masm.bind(&negative);
+ masm.neg32(out);
+ masm.and32(Imm32((1 << ins->shift()) - 1), out);
+ masm.neg32(out);
+ }
+ if (mir->canBeNegativeDividend()) {
+ if (!mir->isTruncated()) {
+ MOZ_ASSERT(mir->fallible());
+ bailoutCmp32(Assembler::Equal, out, zero, ins->snapshot());
+ } else {
+ // -0|0 == 0
+ }
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitModMaskI(LModMaskI* ins) {
+ Register src = ToRegister(ins->getOperand(0));
+ Register dest = ToRegister(ins->getDef(0));
+ Register tmp0 = ToRegister(ins->getTemp(0));
+ Register tmp1 = ToRegister(ins->getTemp(1));
+ MMod* mir = ins->mir();
+
+ if (!mir->isTruncated() && mir->canBeNegativeDividend()) {
+ MOZ_ASSERT(mir->fallible());
+
+ Label bail;
+ masm.ma_mod_mask(src, dest, tmp0, tmp1, ins->shift(), &bail);
+ bailoutFrom(&bail, ins->snapshot());
+ } else {
+ masm.ma_mod_mask(src, dest, tmp0, tmp1, ins->shift(), nullptr);
+ }
+}
+
+void CodeGenerator::visitBitNotI(LBitNotI* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ const LDefinition* dest = ins->getDef(0);
+ MOZ_ASSERT(!input->isConstant());
+
+ masm.ma_not(ToRegister(dest), ToRegister(input));
+}
+
+void CodeGenerator::visitBitOpI(LBitOpI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+ const LDefinition* dest = ins->getDef(0);
+ // all of these bitops should be either imm32's, or integer registers.
+ switch (ins->bitop()) {
+ case JSOp::BitOr:
+ if (rhs->isConstant()) {
+ masm.ma_or(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ } else {
+ masm.as_or(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ }
+ break;
+ case JSOp::BitXor:
+ if (rhs->isConstant()) {
+ masm.ma_xor(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ } else {
+ masm.as_xor(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ }
+ break;
+ case JSOp::BitAnd:
+ if (rhs->isConstant()) {
+ masm.ma_and(ToRegister(dest), ToRegister(lhs), Imm32(ToInt32(rhs)));
+ } else {
+ masm.as_and(ToRegister(dest), ToRegister(lhs), ToRegister(rhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitBitOpI64(LBitOpI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LBitOpI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LBitOpI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ switch (lir->bitop()) {
+ case JSOp::BitOr:
+ if (IsConstant(rhs)) {
+ masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.or64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::BitXor:
+ if (IsConstant(rhs)) {
+ masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.xor64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::BitAnd:
+ if (IsConstant(rhs)) {
+ masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.and64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitShiftI(LShiftI* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ const LAllocation* rhs = ins->rhs();
+ Register dest = ToRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ if (shift) {
+ masm.ma_sll(dest, lhs, Imm32(shift));
+ } else {
+ masm.move32(lhs, dest);
+ }
+ break;
+ case JSOp::Rsh:
+ if (shift) {
+ masm.ma_sra(dest, lhs, Imm32(shift));
+ } else {
+ masm.move32(lhs, dest);
+ }
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.ma_srl(dest, lhs, Imm32(shift));
+ } else {
+ // x >>> 0 can overflow.
+ if (ins->mir()->toUrsh()->fallible()) {
+ bailoutCmp32(Assembler::LessThan, lhs, Imm32(0), ins->snapshot());
+ }
+ masm.move32(lhs, dest);
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ } else {
+ // The shift amounts should be AND'ed into the 0-31 range
+ masm.ma_and(dest, ToRegister(rhs), Imm32(0x1F));
+
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ masm.ma_sll(dest, lhs, dest);
+ break;
+ case JSOp::Rsh:
+ masm.ma_sra(dest, lhs, dest);
+ break;
+ case JSOp::Ursh:
+ masm.ma_srl(dest, lhs, dest);
+ if (ins->mir()->toUrsh()->fallible()) {
+ // x >>> 0 can overflow.
+ bailoutCmp32(Assembler::LessThan, dest, Imm32(0), ins->snapshot());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ }
+}
+
+void CodeGenerator::visitShiftI64(LShiftI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LShiftI64::Lhs);
+ LAllocation* rhs = lir->getOperand(LShiftI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (rhs->isConstant()) {
+ int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ if (shift) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::Rsh:
+ if (shift) {
+ masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.rshift64(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ return;
+ }
+
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ masm.lshift64(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::Rsh:
+ masm.rshift64Arithmetic(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ case JSOp::Ursh:
+ masm.rshift64(ToRegister(rhs), ToRegister64(lhs));
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+}
+
+void CodeGenerator::visitRotateI64(LRotateI64* lir) {
+ MRotate* mir = lir->mir();
+ LAllocation* count = lir->count();
+
+ Register64 input = ToRegister64(lir->input());
+ Register64 output = ToOutRegister64(lir);
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+
+#ifdef JS_CODEGEN_MIPS64
+ MOZ_ASSERT(input == output);
+#endif
+
+ if (count->isConstant()) {
+ int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F);
+ if (!c) {
+#ifdef JS_CODEGEN_MIPS32
+ masm.move64(input, output);
+#endif
+ return;
+ }
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft64(Imm32(c), input, output, temp);
+ } else {
+ masm.rotateRight64(Imm32(c), input, output, temp);
+ }
+ } else {
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft64(ToRegister(count), input, output, temp);
+ } else {
+ masm.rotateRight64(ToRegister(count), input, output, temp);
+ }
+ }
+}
+
+void CodeGenerator::visitUrshD(LUrshD* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register temp = ToRegister(ins->temp());
+
+ const LAllocation* rhs = ins->rhs();
+ FloatRegister out = ToFloatRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ masm.ma_srl(temp, lhs, Imm32(ToInt32(rhs)));
+ } else {
+ masm.ma_srl(temp, lhs, ToRegister(rhs));
+ }
+
+ masm.convertUInt32ToDouble(temp, out);
+}
+
+void CodeGenerator::visitClzI(LClzI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.as_clz(output, input);
+}
+
+void CodeGenerator::visitCtzI(LCtzI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.ma_ctz(output, input);
+}
+
+void CodeGenerator::visitPopcntI(LPopcntI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+ Register tmp = ToRegister(ins->temp0());
+
+ masm.popcnt32(input, output, tmp);
+}
+
+void CodeGenerator::visitPopcntI64(LPopcntI64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ Register64 output = ToOutRegister64(ins);
+ Register tmp = ToRegister(ins->getTemp(0));
+
+ masm.popcnt64(input, output, tmp);
+}
+
+void CodeGenerator::visitPowHalfD(LPowHalfD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ Label done, skip;
+
+ // Masm.pow(-Infinity, 0.5) == Infinity.
+ masm.loadConstantDouble(NegativeInfinity<double>(), ScratchDoubleReg);
+ masm.ma_bc1d(input, ScratchDoubleReg, &skip,
+ Assembler::DoubleNotEqualOrUnordered, ShortJump);
+ masm.as_negd(output, ScratchDoubleReg);
+ masm.ma_b(&done, ShortJump);
+
+ masm.bind(&skip);
+ // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
+ // Adding 0 converts any -0 to 0.
+ masm.loadConstantDouble(0.0, ScratchDoubleReg);
+ masm.as_addd(output, input, ScratchDoubleReg);
+ masm.as_sqrtd(output, output);
+
+ masm.bind(&done);
+}
+
+MoveOperand CodeGeneratorMIPSShared::toMoveOperand(LAllocation a) const {
+ if (a.isGeneralReg()) {
+ return MoveOperand(ToRegister(a));
+ }
+ if (a.isFloatReg()) {
+ return MoveOperand(ToFloatRegister(a));
+ }
+ MoveOperand::Kind kind =
+ a.isStackArea() ? MoveOperand::EFFECTIVE_ADDRESS : MoveOperand::MEMORY;
+ Address address = ToAddress(a);
+ MOZ_ASSERT((address.offset & 3) == 0);
+ return MoveOperand(address, kind);
+}
+
+void CodeGenerator::visitMathD(LMathD* math) {
+ FloatRegister src1 = ToFloatRegister(math->getOperand(0));
+ FloatRegister src2 = ToFloatRegister(math->getOperand(1));
+ FloatRegister output = ToFloatRegister(math->getDef(0));
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.as_addd(output, src1, src2);
+ break;
+ case JSOp::Sub:
+ masm.as_subd(output, src1, src2);
+ break;
+ case JSOp::Mul:
+ masm.as_muld(output, src1, src2);
+ break;
+ case JSOp::Div:
+ masm.as_divd(output, src1, src2);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitMathF(LMathF* math) {
+ FloatRegister src1 = ToFloatRegister(math->getOperand(0));
+ FloatRegister src2 = ToFloatRegister(math->getOperand(1));
+ FloatRegister output = ToFloatRegister(math->getDef(0));
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.as_adds(output, src1, src2);
+ break;
+ case JSOp::Sub:
+ masm.as_subs(output, src1, src2);
+ break;
+ case JSOp::Mul:
+ masm.as_muls(output, src1, src2);
+ break;
+ case JSOp::Div:
+ masm.as_divs(output, src1, src2);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+ emitTruncateDouble(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+ emitTruncateFloat32(ToFloatRegister(ins->input()), ToRegister(ins->output()),
+ ins->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* lir) {
+ emitTruncateDouble(ToFloatRegister(lir->getOperand(0)),
+ ToRegister(lir->getDef(0)), lir->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+ LWasmBuiltinTruncateFToInt32* lir) {
+ emitTruncateFloat32(ToFloatRegister(lir->getOperand(0)),
+ ToRegister(lir->getDef(0)), lir->mir());
+}
+
+void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) {
+ auto input = ToFloatRegister(lir->input());
+ auto output = ToRegister(lir->output());
+
+ MWasmTruncateToInt32* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ if (mir->isUnsigned()) {
+ if (fromType == MIRType::Double) {
+ masm.wasmTruncateDoubleToUInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else if (fromType == MIRType::Float32) {
+ masm.wasmTruncateFloat32ToUInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+
+ masm.bind(ool->rejoin());
+ return;
+ }
+
+ if (fromType == MIRType::Double) {
+ masm.wasmTruncateDoubleToInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else if (fromType == MIRType::Float32) {
+ masm.wasmTruncateFloat32ToInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+
+ masm.bind(ool->rejoin());
+}
+
+void CodeGeneratorMIPSShared::visitOutOfLineBailout(OutOfLineBailout* ool) {
+ // Push snapshotOffset and make sure stack is aligned.
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.storePtr(ImmWord(ool->snapshot()->snapshotOffset()),
+ Address(StackPointer, 0));
+
+ masm.jump(&deoptLabel_);
+}
+
+void CodeGeneratorMIPSShared::visitOutOfLineWasmTruncateCheck(
+ OutOfLineWasmTruncateCheck* ool) {
+ if (ool->toType() == MIRType::Int32) {
+ masm.outOfLineWasmTruncateToInt32Check(
+ ool->input(), ool->output(), ool->fromType(), ool->flags(),
+ ool->rejoin(), ool->bytecodeOffset());
+ } else {
+ MOZ_ASSERT(ool->toType() == MIRType::Int64);
+ masm.outOfLineWasmTruncateToInt64Check(
+ ool->input(), ool->output64(), ool->fromType(), ool->flags(),
+ ool->rejoin(), ool->bytecodeOffset());
+ }
+}
+
+void CodeGenerator::visitCopySignF(LCopySignF* ins) {
+ FloatRegister lhs = ToFloatRegister(ins->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(ins->getOperand(1));
+ FloatRegister output = ToFloatRegister(ins->getDef(0));
+
+ Register lhsi = ToRegister(ins->getTemp(0));
+ Register rhsi = ToRegister(ins->getTemp(1));
+
+ masm.moveFromFloat32(lhs, lhsi);
+ masm.moveFromFloat32(rhs, rhsi);
+
+ // Combine.
+ masm.ma_ins(rhsi, lhsi, 0, 31);
+
+ masm.moveToFloat32(rhsi, output);
+}
+
+void CodeGenerator::visitCopySignD(LCopySignD* ins) {
+ FloatRegister lhs = ToFloatRegister(ins->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(ins->getOperand(1));
+ FloatRegister output = ToFloatRegister(ins->getDef(0));
+
+ Register lhsi = ToRegister(ins->getTemp(0));
+ Register rhsi = ToRegister(ins->getTemp(1));
+
+ // Manipulate high words of double inputs.
+ masm.moveFromDoubleHi(lhs, lhsi);
+ masm.moveFromDoubleHi(rhs, rhsi);
+
+ // Combine.
+ masm.ma_ins(rhsi, lhsi, 0, 31);
+
+ masm.moveToDoubleHi(rhsi, output);
+}
+
+void CodeGenerator::visitValue(LValue* value) {
+ const ValueOperand out = ToOutValue(value);
+
+ masm.moveValue(value->value(), out);
+}
+
+void CodeGenerator::visitDouble(LDouble* ins) {
+ const LDefinition* out = ins->getDef(0);
+
+ masm.loadConstantDouble(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitFloat32(LFloat32* ins) {
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantFloat32(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) {
+ FloatRegister input = ToFloatRegister(test->input());
+
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ masm.loadConstantDouble(0.0, ScratchDoubleReg);
+ // If 0, or NaN, the result is false.
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::DoubleFloat, input, ScratchDoubleReg, ifTrue,
+ Assembler::DoubleNotEqual);
+ } else {
+ branchToBlock(Assembler::DoubleFloat, input, ScratchDoubleReg, ifFalse,
+ Assembler::DoubleEqualOrUnordered);
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) {
+ FloatRegister input = ToFloatRegister(test->input());
+
+ MBasicBlock* ifTrue = test->ifTrue();
+ MBasicBlock* ifFalse = test->ifFalse();
+
+ masm.loadConstantFloat32(0.0f, ScratchFloat32Reg);
+ // If 0, or NaN, the result is false.
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::SingleFloat, input, ScratchFloat32Reg, ifTrue,
+ Assembler::DoubleNotEqual);
+ } else {
+ branchToBlock(Assembler::SingleFloat, input, ScratchFloat32Reg, ifFalse,
+ Assembler::DoubleEqualOrUnordered);
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitCompareD(LCompareD* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+ Register dest = ToRegister(comp->output());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+ masm.ma_cmp_set_double(dest, lhs, rhs, cond);
+}
+
+void CodeGenerator::visitCompareF(LCompareF* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+ Register dest = ToRegister(comp->output());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+ masm.ma_cmp_set_float32(dest, lhs, rhs, cond);
+}
+
+void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ MBasicBlock* ifTrue = comp->ifTrue();
+ MBasicBlock* ifFalse = comp->ifFalse();
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::DoubleFloat, lhs, rhs, ifTrue, cond);
+ } else {
+ branchToBlock(Assembler::DoubleFloat, lhs, rhs, ifFalse,
+ Assembler::InvertCondition(cond));
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+ MBasicBlock* ifTrue = comp->ifTrue();
+ MBasicBlock* ifFalse = comp->ifFalse();
+
+ if (isNextBlock(ifFalse->lir())) {
+ branchToBlock(Assembler::SingleFloat, lhs, rhs, ifTrue, cond);
+ } else {
+ branchToBlock(Assembler::SingleFloat, lhs, rhs, ifFalse,
+ Assembler::InvertCondition(cond));
+ jumpToBlock(ifTrue);
+ }
+}
+
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* lir) {
+ if (lir->right()->isConstant()) {
+ masm.ma_and(ScratchRegister, ToRegister(lir->left()),
+ Imm32(ToInt32(lir->right())));
+ } else {
+ masm.as_and(ScratchRegister, ToRegister(lir->left()),
+ ToRegister(lir->right()));
+ }
+ emitBranch(ScratchRegister, ScratchRegister, lir->cond(), lir->ifTrue(),
+ lir->ifFalse());
+}
+
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+ masm.convertUInt32ToDouble(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+ masm.convertUInt32ToFloat32(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitNotI(LNotI* ins) {
+ masm.cmp32Set(Assembler::Equal, ToRegister(ins->input()), Imm32(0),
+ ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitNotD(LNotD* ins) {
+ // Since this operation is not, we want to set a bit if
+ // the double is falsey, which means 0.0, -0.0 or NaN.
+ FloatRegister in = ToFloatRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+
+ masm.loadConstantDouble(0.0, ScratchDoubleReg);
+ masm.ma_cmp_set_double(dest, in, ScratchDoubleReg,
+ Assembler::DoubleEqualOrUnordered);
+}
+
+void CodeGenerator::visitNotF(LNotF* ins) {
+ // Since this operation is not, we want to set a bit if
+ // the float32 is falsey, which means 0.0, -0.0 or NaN.
+ FloatRegister in = ToFloatRegister(ins->input());
+ Register dest = ToRegister(ins->output());
+
+ masm.loadConstantFloat32(0.0f, ScratchFloat32Reg);
+ masm.ma_cmp_set_float32(dest, in, ScratchFloat32Reg,
+ Assembler::DoubleEqualOrUnordered);
+}
+
+void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) {
+ masm.memoryBarrier(ins->type());
+}
+
+void CodeGeneratorMIPSShared::generateInvalidateEpilogue() {
+ // Ensure that there is enough space in the buffer for the OsiPoint
+ // patching to occur. Otherwise, we could overwrite the invalidation
+ // epilogue.
+ for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) {
+ masm.nop();
+ }
+
+ masm.bind(&invalidate_);
+
+ // Push the return address of the point that we bailed out at to the stack
+ masm.Push(ra);
+
+ // Push the Ion script onto the stack (when we determine what that
+ // pointer is).
+ invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
+
+ // Jump to the invalidator which will replace the current frame.
+ TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
+ masm.jump(thunk);
+}
+
+class js::jit::OutOfLineTableSwitch
+ : public OutOfLineCodeBase<CodeGeneratorMIPSShared> {
+ MTableSwitch* mir_;
+ CodeLabel jumpLabel_;
+
+ void accept(CodeGeneratorMIPSShared* codegen) {
+ codegen->visitOutOfLineTableSwitch(this);
+ }
+
+ public:
+ OutOfLineTableSwitch(MTableSwitch* mir) : mir_(mir) {}
+
+ MTableSwitch* mir() const { return mir_; }
+
+ CodeLabel* jumpLabel() { return &jumpLabel_; }
+};
+
+void CodeGeneratorMIPSShared::visitOutOfLineTableSwitch(
+ OutOfLineTableSwitch* ool) {
+ MTableSwitch* mir = ool->mir();
+
+ masm.haltingAlign(sizeof(void*));
+ masm.bind(ool->jumpLabel());
+ masm.addCodeLabel(*ool->jumpLabel());
+
+ for (size_t i = 0; i < mir->numCases(); i++) {
+ LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir();
+ Label* caseheader = caseblock->label();
+ uint32_t caseoffset = caseheader->offset();
+
+ // The entries of the jump table need to be absolute addresses and thus
+ // must be patched after codegen is finished.
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ cl.target()->bind(caseoffset);
+ masm.addCodeLabel(cl);
+ }
+}
+
+void CodeGeneratorMIPSShared::emitTableSwitchDispatch(MTableSwitch* mir,
+ Register index,
+ Register base) {
+ Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
+
+ // Lower value with low value
+ if (mir->low() != 0) {
+ masm.subPtr(Imm32(mir->low()), index);
+ }
+
+ // Jump to default case if input is out of range
+ int32_t cases = mir->numCases();
+ masm.branchPtr(Assembler::AboveOrEqual, index, ImmWord(cases), defaultcase);
+
+ // To fill in the CodeLabels for the case entries, we need to first
+ // generate the case entries (we don't yet know their offsets in the
+ // instruction stream).
+ OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(mir);
+ addOutOfLineCode(ool, mir);
+
+ // Compute the position where a pointer to the right case stands.
+ masm.ma_li(base, ool->jumpLabel());
+
+ BaseIndex pointer(base, index, ScalePointer);
+
+ // Jump to the right case
+ masm.branchToComputedAddress(pointer);
+}
+
+void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) {
+ MOZ_ASSERT(ins->instance()->isBogus());
+ masm.movePtr(HeapReg, ToRegister(ins->output()));
+}
+
+template <typename T>
+void CodeGeneratorMIPSShared::emitWasmLoad(T* lir) {
+ const MWasmLoad* mir = lir->mir();
+ SecondScratchRegisterScope scratch2(masm);
+
+ Register ptr = ToRegister(lir->ptr());
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (mir->base()->type() == MIRType::Int32) {
+ masm.move32To64ZeroExtend(ptr, Register64(scratch2));
+ ptr = scratch2;
+ ptrScratch = ptrScratch != InvalidReg ? scratch2 : InvalidReg;
+ }
+
+ if (IsUnaligned(mir->access())) {
+ if (IsFloatingPointType(mir->type())) {
+ masm.wasmUnalignedLoadFP(mir->access(), HeapReg, ptr, ptrScratch,
+ ToFloatRegister(lir->output()),
+ ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmUnalignedLoad(mir->access(), HeapReg, ptr, ptrScratch,
+ ToRegister(lir->output()),
+ ToRegister(lir->getTemp(1)));
+ }
+ } else {
+ masm.wasmLoad(mir->access(), HeapReg, ptr, ptrScratch,
+ ToAnyRegister(lir->output()));
+ }
+}
+
+void CodeGenerator::visitWasmLoad(LWasmLoad* lir) { emitWasmLoad(lir); }
+
+void CodeGenerator::visitWasmUnalignedLoad(LWasmUnalignedLoad* lir) {
+ emitWasmLoad(lir);
+}
+
+template <typename T>
+void CodeGeneratorMIPSShared::emitWasmStore(T* lir) {
+ const MWasmStore* mir = lir->mir();
+ SecondScratchRegisterScope scratch2(masm);
+
+ Register ptr = ToRegister(lir->ptr());
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (mir->base()->type() == MIRType::Int32) {
+ masm.move32To64ZeroExtend(ptr, Register64(scratch2));
+ ptr = scratch2;
+ ptrScratch = ptrScratch != InvalidReg ? scratch2 : InvalidReg;
+ }
+
+ if (IsUnaligned(mir->access())) {
+ if (mir->access().type() == Scalar::Float32 ||
+ mir->access().type() == Scalar::Float64) {
+ masm.wasmUnalignedStoreFP(mir->access(), ToFloatRegister(lir->value()),
+ HeapReg, ptr, ptrScratch,
+ ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmUnalignedStore(mir->access(), ToRegister(lir->value()), HeapReg,
+ ptr, ptrScratch, ToRegister(lir->getTemp(1)));
+ }
+ } else {
+ masm.wasmStore(mir->access(), ToAnyRegister(lir->value()), HeapReg, ptr,
+ ptrScratch);
+ }
+}
+
+void CodeGenerator::visitWasmStore(LWasmStore* lir) { emitWasmStore(lir); }
+
+void CodeGenerator::visitWasmUnalignedStore(LWasmUnalignedStore* lir) {
+ emitWasmStore(lir);
+}
+
+void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) {
+ const MAsmJSLoadHeap* mir = ins->mir();
+ const LAllocation* ptr = ins->ptr();
+ const LDefinition* out = ins->output();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ bool isSigned;
+ int size;
+ bool isFloat = false;
+ switch (mir->access().type()) {
+ case Scalar::Int8:
+ isSigned = true;
+ size = 8;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ size = 8;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ size = 16;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ size = 16;
+ break;
+ case Scalar::Int32:
+ isSigned = true;
+ size = 32;
+ break;
+ case Scalar::Uint32:
+ isSigned = false;
+ size = 32;
+ break;
+ case Scalar::Float64:
+ isFloat = true;
+ size = 64;
+ break;
+ case Scalar::Float32:
+ isFloat = true;
+ size = 32;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ if (ptr->isConstant()) {
+ MOZ_ASSERT(!mir->needsBoundsCheck());
+ int32_t ptrImm = ptr->toConstant()->toInt32();
+ MOZ_ASSERT(ptrImm >= 0);
+ if (isFloat) {
+ if (size == 32) {
+ masm.loadFloat32(Address(HeapReg, ptrImm), ToFloatRegister(out));
+ } else {
+ masm.loadDouble(Address(HeapReg, ptrImm), ToFloatRegister(out));
+ }
+ } else {
+ masm.ma_load(ToRegister(out), Address(HeapReg, ptrImm),
+ static_cast<LoadStoreSize>(size),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+
+ Register ptrReg = ToRegister(ptr);
+
+ if (!mir->needsBoundsCheck()) {
+ if (isFloat) {
+ if (size == 32) {
+ masm.loadFloat32(BaseIndex(HeapReg, ptrReg, TimesOne),
+ ToFloatRegister(out));
+ } else {
+ masm.loadDouble(BaseIndex(HeapReg, ptrReg, TimesOne),
+ ToFloatRegister(out));
+ }
+ } else {
+ masm.ma_load(ToRegister(out), BaseIndex(HeapReg, ptrReg, TimesOne),
+ static_cast<LoadStoreSize>(size),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+
+ Label done, outOfRange;
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg,
+ ToRegister(boundsCheckLimit), &outOfRange);
+ // Offset is ok, let's load value.
+ if (isFloat) {
+ if (size == 32) {
+ masm.loadFloat32(BaseIndex(HeapReg, ptrReg, TimesOne),
+ ToFloatRegister(out));
+ } else {
+ masm.loadDouble(BaseIndex(HeapReg, ptrReg, TimesOne),
+ ToFloatRegister(out));
+ }
+ } else {
+ masm.ma_load(ToRegister(out), BaseIndex(HeapReg, ptrReg, TimesOne),
+ static_cast<LoadStoreSize>(size),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&outOfRange);
+ // Offset is out of range. Load default values.
+ if (isFloat) {
+ if (size == 32) {
+ masm.loadConstantFloat32(float(GenericNaN()), ToFloatRegister(out));
+ } else {
+ masm.loadConstantDouble(GenericNaN(), ToFloatRegister(out));
+ }
+ } else {
+ masm.move32(Imm32(0), ToRegister(out));
+ }
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) {
+ const MAsmJSStoreHeap* mir = ins->mir();
+ const LAllocation* value = ins->value();
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ bool isSigned;
+ int size;
+ bool isFloat = false;
+ switch (mir->access().type()) {
+ case Scalar::Int8:
+ isSigned = true;
+ size = 8;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ size = 8;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ size = 16;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ size = 16;
+ break;
+ case Scalar::Int32:
+ isSigned = true;
+ size = 32;
+ break;
+ case Scalar::Uint32:
+ isSigned = false;
+ size = 32;
+ break;
+ case Scalar::Float64:
+ isFloat = true;
+ size = 64;
+ break;
+ case Scalar::Float32:
+ isFloat = true;
+ size = 32;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ if (ptr->isConstant()) {
+ MOZ_ASSERT(!mir->needsBoundsCheck());
+ int32_t ptrImm = ptr->toConstant()->toInt32();
+ MOZ_ASSERT(ptrImm >= 0);
+
+ if (isFloat) {
+ FloatRegister freg = ToFloatRegister(value);
+ Address addr(HeapReg, ptrImm);
+ if (size == 32) {
+ masm.storeFloat32(freg, addr);
+ } else {
+ masm.storeDouble(freg, addr);
+ }
+ } else {
+ masm.ma_store(ToRegister(value), Address(HeapReg, ptrImm),
+ static_cast<LoadStoreSize>(size),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+
+ Register ptrReg = ToRegister(ptr);
+ Address dstAddr(ptrReg, 0);
+
+ if (!mir->needsBoundsCheck()) {
+ if (isFloat) {
+ FloatRegister freg = ToFloatRegister(value);
+ BaseIndex bi(HeapReg, ptrReg, TimesOne);
+ if (size == 32) {
+ masm.storeFloat32(freg, bi);
+ } else {
+ masm.storeDouble(freg, bi);
+ }
+ } else {
+ masm.ma_store(ToRegister(value), BaseIndex(HeapReg, ptrReg, TimesOne),
+ static_cast<LoadStoreSize>(size),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+
+ Label outOfRange;
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ptrReg,
+ ToRegister(boundsCheckLimit), &outOfRange);
+
+ // Offset is ok, let's store value.
+ if (isFloat) {
+ if (size == 32) {
+ masm.storeFloat32(ToFloatRegister(value),
+ BaseIndex(HeapReg, ptrReg, TimesOne));
+ } else
+ masm.storeDouble(ToFloatRegister(value),
+ BaseIndex(HeapReg, ptrReg, TimesOne));
+ } else {
+ masm.ma_store(ToRegister(value), BaseIndex(HeapReg, ptrReg, TimesOne),
+ static_cast<LoadStoreSize>(size),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+
+ masm.bind(&outOfRange);
+}
+
+void CodeGenerator::visitWasmCompareExchangeHeap(
+ LWasmCompareExchangeHeap* ins) {
+ MWasmCompareExchangeHeap* mir = ins->mir();
+ Register ptrReg = ToRegister(ins->ptr());
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Register oldval = ToRegister(ins->oldValue());
+ Register newval = ToRegister(ins->newValue());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+ MWasmAtomicExchangeHeap* mir = ins->mir();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ masm.wasmAtomicExchange(mir->access(), srcAddr, value, valueTemp, offsetTemp,
+ maskTemp, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+ MOZ_ASSERT(ins->mir()->hasUses());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+
+ masm.wasmAtomicFetchOp(mir->access(), mir->operation(),
+ ToRegister(ins->value()), srcAddr, valueTemp,
+ offsetTemp, maskTemp, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+ LWasmAtomicBinopHeapForEffect* ins) {
+ MOZ_ASSERT(!ins->mir()->hasUses());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ Register ptrReg = ToRegister(ins->ptr());
+ Register valueTemp = ToTempRegisterOrInvalid(ins->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(ins->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(ins->maskTemp());
+
+ BaseIndex srcAddr(HeapReg, ptrReg, TimesOne, mir->access().offset());
+ masm.wasmAtomicEffectOp(mir->access(), mir->operation(),
+ ToRegister(ins->value()), srcAddr, valueTemp,
+ offsetTemp, maskTemp);
+}
+
+void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ if (ins->arg()->isConstant()) {
+ masm.storePtr(ImmWord(ToInt32(ins->arg())),
+ Address(StackPointer, mir->spOffset()));
+ } else {
+ if (ins->arg()->isGeneralReg()) {
+ masm.storePtr(ToRegister(ins->arg()),
+ Address(StackPointer, mir->spOffset()));
+ } else if (mir->input()->type() == MIRType::Double) {
+ masm.storeDouble(ToFloatRegister(ins->arg()).doubleOverlay(),
+ Address(StackPointer, mir->spOffset()));
+ } else {
+ masm.storeFloat32(ToFloatRegister(ins->arg()),
+ Address(StackPointer, mir->spOffset()));
+ }
+ }
+}
+
+void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(StackPointer, mir->spOffset());
+ if (IsConstant(ins->arg())) {
+ masm.store64(Imm64(ToInt64(ins->arg())), dst);
+ } else {
+ masm.store64(ToRegister64(ins->arg()), dst);
+ }
+}
+
+void CodeGenerator::visitWasmSelect(LWasmSelect* ins) {
+ MIRType mirType = ins->mir()->type();
+
+ Register cond = ToRegister(ins->condExpr());
+ const LAllocation* falseExpr = ins->falseExpr();
+
+ if (mirType == MIRType::Int32 || mirType == MIRType::RefOrNull) {
+ Register out = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->trueExpr()) == out,
+ "true expr input is reused for output");
+ if (falseExpr->isRegister()) {
+ masm.as_movz(out, ToRegister(falseExpr), cond);
+ } else {
+ masm.cmp32Load32(Assembler::Zero, cond, cond, ToAddress(falseExpr), out);
+ }
+ return;
+ }
+
+ FloatRegister out = ToFloatRegister(ins->output());
+ MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out,
+ "true expr input is reused for output");
+
+ if (falseExpr->isFloatReg()) {
+ if (mirType == MIRType::Float32) {
+ masm.as_movz(Assembler::SingleFloat, out, ToFloatRegister(falseExpr),
+ cond);
+ } else if (mirType == MIRType::Double) {
+ masm.as_movz(Assembler::DoubleFloat, out, ToFloatRegister(falseExpr),
+ cond);
+ } else {
+ MOZ_CRASH("unhandled type in visitWasmSelect!");
+ }
+ } else {
+ Label done;
+ masm.ma_b(cond, cond, &done, Assembler::NonZero, ShortJump);
+
+ if (mirType == MIRType::Float32) {
+ masm.loadFloat32(ToAddress(falseExpr), out);
+ } else if (mirType == MIRType::Double) {
+ masm.loadDouble(ToAddress(falseExpr), out);
+ } else {
+ MOZ_CRASH("unhandled type in visitWasmSelect!");
+ }
+
+ masm.bind(&done);
+ }
+}
+
+// We expect to handle only the case where compare is {U,}Int32 and select is
+// {U,}Int32, and the "true" input is reused for the output.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
+ ins->compareType() == MCompare::Compare_UInt32;
+ bool selIs32bit = ins->mir()->type() == MIRType::Int32;
+
+ MOZ_RELEASE_ASSERT(
+ cmpIs32bit && selIs32bit,
+ "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
+
+ Register trueExprAndDest = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
+ "true expr input is reused for output");
+
+ Assembler::Condition cond = Assembler::InvertCondition(
+ JSOpToCondition(ins->compareType(), ins->jsop()));
+ const LAllocation* rhs = ins->rightExpr();
+ const LAllocation* falseExpr = ins->ifFalseExpr();
+ Register lhs = ToRegister(ins->leftExpr());
+
+ masm.cmp32Move32(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+}
+
+void CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MWasmReinterpret* ins = lir->mir();
+
+ MIRType to = ins->type();
+ DebugOnly<MIRType> from = ins->input()->type();
+
+ switch (to) {
+ case MIRType::Int32:
+ MOZ_ASSERT(from == MIRType::Float32);
+ masm.as_mfc1(ToRegister(lir->output()), ToFloatRegister(lir->input()));
+ break;
+ case MIRType::Float32:
+ MOZ_ASSERT(from == MIRType::Int32);
+ masm.as_mtc1(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+ break;
+ case MIRType::Double:
+ case MIRType::Int64:
+ MOZ_CRASH("not handled by this LIR opcode");
+ default:
+ MOZ_CRASH("unexpected WasmReinterpret");
+ }
+}
+
+void CodeGenerator::visitUDivOrMod(LUDivOrMod* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+ Label done;
+
+ // Prevent divide by zero.
+ if (ins->canBeDivideByZero()) {
+ if (ins->mir()->isTruncated()) {
+ if (ins->trapOnError()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ // Infinity|0 == 0
+ Label notzero;
+ masm.ma_b(rhs, rhs, &notzero, Assembler::NonZero, ShortJump);
+ masm.move32(Imm32(0), output);
+ masm.ma_b(&done, ShortJump);
+ masm.bind(&notzero);
+ }
+ } else {
+ bailoutCmp32(Assembler::Equal, rhs, Imm32(0), ins->snapshot());
+ }
+ }
+
+#ifdef MIPSR6
+ masm.as_modu(output, lhs, rhs);
+#else
+ masm.as_divu(lhs, rhs);
+ masm.as_mfhi(output);
+#endif
+
+ // If the remainder is > 0, bailout since this must be a double.
+ if (ins->mir()->isDiv()) {
+ if (!ins->mir()->toDiv()->canTruncateRemainder()) {
+ bailoutCmp32(Assembler::NonZero, output, output, ins->snapshot());
+ }
+ // Get quotient
+#ifdef MIPSR6
+ masm.as_divu(output, lhs, rhs);
+#else
+ masm.as_mflo(output);
+#endif
+ }
+
+ if (!ins->mir()->isTruncated()) {
+ bailoutCmp32(Assembler::LessThan, output, Imm32(0), ins->snapshot());
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins) {
+ const MEffectiveAddress* mir = ins->mir();
+ Register base = ToRegister(ins->base());
+ Register index = ToRegister(ins->index());
+ Register output = ToRegister(ins->output());
+
+ BaseIndex address(base, index, mir->scale(), mir->displacement());
+ masm.computeEffectiveAddress(address, output);
+}
+
+void CodeGenerator::visitNegI(LNegI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ masm.ma_negu(output, input);
+}
+
+void CodeGenerator::visitNegI64(LNegI64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ MOZ_ASSERT(input == ToOutRegister64(ins));
+ masm.neg64(input);
+}
+
+void CodeGenerator::visitNegD(LNegD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ masm.as_negd(output, input);
+}
+
+void CodeGenerator::visitNegF(LNegF* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ masm.as_negs(output, input);
+}
+
+void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register base = ToRegister(lir->base());
+ Register out = ToRegister(lir->output());
+
+ Label ok;
+ masm.ma_add32TestCarry(Assembler::CarryClear, out, base, Imm32(mir->offset()),
+ &ok);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
+ masm.bind(&ok);
+}
+
+void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register64 base = ToRegister64(lir->base());
+ Register64 out = ToOutRegister64(lir);
+
+ Label ok;
+ masm.ma_addPtrTestCarry(Assembler::CarryClear, out.reg, base.reg,
+ ImmWord(mir->offset()), &ok);
+ masm.wasmTrap(wasm::Trap::OutOfBounds, mir->bytecodeOffset());
+ masm.bind(&ok);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop(
+ LAtomicTypedArrayElementBinop* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register elements = ToRegister(lir->elements());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp2());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+ Register value = ToRegister(lir->value());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, valueTemp,
+ offsetTemp, maskTemp, outTemp, output);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, valueTemp,
+ offsetTemp, maskTemp, outTemp, output);
+ }
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(
+ LAtomicTypedArrayElementBinopForEffect* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+ Register value = ToRegister(lir->value());
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, valueTemp,
+ offsetTemp, maskTemp);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(),
+ lir->mir()->operation(), value, mem, valueTemp,
+ offsetTemp, maskTemp);
+ }
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement(
+ LCompareExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp());
+
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, valueTemp, offsetTemp, maskTemp, outTemp,
+ output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, valueTemp, offsetTemp, maskTemp, outTemp,
+ output);
+ }
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement(
+ LAtomicExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register outTemp = ToTempRegisterOrInvalid(lir->temp());
+
+ Register value = ToRegister(lir->value());
+ Register valueTemp = ToTempRegisterOrInvalid(lir->valueTemp());
+ Register offsetTemp = ToTempRegisterOrInvalid(lir->offsetTemp());
+ Register maskTemp = ToTempRegisterOrInvalid(lir->maskTemp());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value,
+ valueTemp, offsetTemp, maskTemp, outTemp, output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value,
+ valueTemp, offsetTemp, maskTemp, outTemp, output);
+ }
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 tempOut(out);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(oldval, temp1);
+ masm.loadBigInt64(newval, tempOut);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, tempOut,
+ temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = Register64(ToRegister(lir->temp2()));
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_ASSERT(lir->mir()->hasUses());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+ Register64 tempOut = Register64(out);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ tempOut, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ tempOut, temp2);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp2, out, temp1.scratchReg());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_ASSERT(!lir->mir()->hasUses());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ temp2);
+ }
+}
+
+void CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 oldValue = ToRegister64(lir->oldValue());
+ Register64 newValue = ToRegister64(lir->newValue());
+ Register64 output = ToOutRegister64(lir);
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+ masm.wasmCompareExchange64(lir->mir()->access(), addr, oldValue, newValue,
+ output);
+}
+
+void CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 output = ToOutRegister64(lir);
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+ masm.wasmAtomicExchange64(lir->mir()->access(), addr, value, output);
+}
+
+void CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register64 output = ToOutRegister64(lir);
+#ifdef JS_CODEGEN_MIPS32
+ Register64 temp(ToRegister(lir->getTemp(0)), ToRegister(lir->getTemp(1)));
+#else
+ Register64 temp(ToRegister(lir->getTemp(0)));
+#endif
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+
+ masm.wasmAtomicFetchOp64(lir->mir()->access(), lir->mir()->operation(), value,
+ addr, temp, output);
+}
+
+void CodeGenerator::visitNearbyInt(LNearbyInt*) { MOZ_CRASH("NYI"); }
+
+void CodeGenerator::visitNearbyIntF(LNearbyIntF*) { MOZ_CRASH("NYI"); }
+
+void CodeGenerator::visitSimd128(LSimd128* ins) { MOZ_CRASH("No SIMD"); }
+
+void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmBinarySimd128WithConstant(
+ LWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmVariableShiftSimd128(
+ LWasmVariableShiftSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmConstantShiftSimd128(
+ LWasmConstantShiftSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmSignReplicationSimd128(
+ LWasmSignReplicationSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
+ LWasmReplaceInt64LaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceAndBranchSimd128(
+ LWasmReduceAndBranchSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmReduceSimd128ToInt64(
+ LWasmReduceSimd128ToInt64* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
+
+void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH("No SIMD");
+}
diff --git a/js/src/jit/mips-shared/CodeGenerator-mips-shared.h b/js/src/jit/mips-shared/CodeGenerator-mips-shared.h
new file mode 100644
index 0000000000..2452a443be
--- /dev/null
+++ b/js/src/jit/mips-shared/CodeGenerator-mips-shared.h
@@ -0,0 +1,157 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_CodeGenerator_mips_shared_h
+#define jit_mips_shared_CodeGenerator_mips_shared_h
+
+#include "jit/shared/CodeGenerator-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorMIPSShared;
+class OutOfLineBailout;
+class OutOfLineTableSwitch;
+
+using OutOfLineWasmTruncateCheck =
+ OutOfLineWasmTruncateCheckBase<CodeGeneratorMIPSShared>;
+
+class CodeGeneratorMIPSShared : public CodeGeneratorShared {
+ friend class MoveResolverMIPS;
+
+ protected:
+ CodeGeneratorMIPSShared(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm);
+
+ NonAssertingLabel deoptLabel_;
+
+ Operand ToOperand(const LAllocation& a);
+ Operand ToOperand(const LAllocation* a);
+ Operand ToOperand(const LDefinition* def);
+
+#ifdef JS_PUNBOX64
+ Operand ToOperandOrRegister64(const LInt64Allocation input);
+#else
+ Register64 ToOperandOrRegister64(const LInt64Allocation input);
+#endif
+
+ MoveOperand toMoveOperand(LAllocation a) const;
+
+ template <typename T1, typename T2>
+ void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ Label bail;
+ masm.branch32(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutTest32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ Label bail;
+ masm.branchTest32(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ Label bail;
+ masm.branchPtr(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs,
+ LSnapshot* snapshot) {
+ Label bail;
+ masm.branchTestPtr(c, lhs, rhs, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) {
+ Label bail;
+ masm.branchTest32(Assembler::Zero, reg, Imm32(0xFF), &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+
+ void bailoutFrom(Label* label, LSnapshot* snapshot);
+ void bailout(LSnapshot* snapshot);
+
+ bool generateOutOfLineCode();
+
+ template <typename T>
+ void branchToBlock(Register lhs, T rhs, MBasicBlock* mir,
+ Assembler::Condition cond) {
+ masm.ma_b(lhs, rhs, skipTrivialBlocks(mir)->lir()->label(), cond);
+ }
+ void branchToBlock(Assembler::FloatFormat fmt, FloatRegister lhs,
+ FloatRegister rhs, MBasicBlock* mir,
+ Assembler::DoubleCondition cond);
+
+ // Emits a branch that directs control flow to the true block if |cond| is
+ // true, and the false block if |cond| is false.
+ template <typename T>
+ void emitBranch(Register lhs, T rhs, Assembler::Condition cond,
+ MBasicBlock* mirTrue, MBasicBlock* mirFalse) {
+ if (isNextBlock(mirFalse->lir())) {
+ branchToBlock(lhs, rhs, mirTrue, cond);
+ } else {
+ branchToBlock(lhs, rhs, mirFalse, Assembler::InvertCondition(cond));
+ jumpToBlock(mirTrue);
+ }
+ }
+ void testZeroEmitBranch(Assembler::Condition cond, Register reg,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ emitBranch(reg, Imm32(0), cond, ifTrue, ifFalse);
+ }
+
+ void emitTableSwitchDispatch(MTableSwitch* mir, Register index,
+ Register base);
+
+ template <typename T>
+ void emitWasmLoad(T* ins);
+ template <typename T>
+ void emitWasmStore(T* ins);
+
+ void generateInvalidateEpilogue();
+
+ // Generating a result.
+ template <typename S, typename T>
+ void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
+ const S& value, const T& mem,
+ Register flagTemp, Register outTemp,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, AnyRegister output);
+
+ // Generating no result.
+ template <typename S, typename T>
+ void atomicBinopToTypedIntArray(AtomicOp op, Scalar::Type arrayType,
+ const S& value, const T& mem,
+ Register flagTemp, Register valueTemp,
+ Register offsetTemp, Register maskTemp);
+
+ public:
+ // Out of line visitors.
+ void visitOutOfLineBailout(OutOfLineBailout* ool);
+ void visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool);
+ void visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool);
+};
+
+// An out-of-line bailout thunk.
+class OutOfLineBailout : public OutOfLineCodeBase<CodeGeneratorMIPSShared> {
+ protected:
+ LSnapshot* snapshot_;
+ uint32_t frameSize_;
+
+ public:
+ OutOfLineBailout(LSnapshot* snapshot, uint32_t frameSize)
+ : snapshot_(snapshot), frameSize_(frameSize) {}
+
+ void accept(CodeGeneratorMIPSShared* codegen) override;
+
+ LSnapshot* snapshot() const { return snapshot_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_CodeGenerator_mips_shared_h */
diff --git a/js/src/jit/mips-shared/LIR-mips-shared.h b/js/src/jit/mips-shared/LIR-mips-shared.h
new file mode 100644
index 0000000000..624e9eb6a7
--- /dev/null
+++ b/js/src/jit/mips-shared/LIR-mips-shared.h
@@ -0,0 +1,360 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_LIR_mips_shared_h
+#define jit_mips_shared_LIR_mips_shared_h
+
+namespace js {
+namespace jit {
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToDouble)
+
+ LWasmUint32ToDouble(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToFloat32)
+
+ LWasmUint32ToFloat32(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+class LDivI : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivI);
+
+ LDivI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LDivPowTwoI : public LInstructionHelper<1, 1, 1> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(DivPowTwoI)
+
+ LDivPowTwoI(const LAllocation& lhs, int32_t shift, const LDefinition& temp)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+ int32_t shift() const { return shift_; }
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LModI : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(ModI);
+
+ LModI(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& callTemp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, callTemp);
+ }
+
+ const LDefinition* callTemp() { return getTemp(0); }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModPowTwoI : public LInstructionHelper<1, 1, 0> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModPowTwoI);
+
+ LModPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ }
+
+ int32_t shift() const { return shift_; }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModMaskI : public LInstructionHelper<1, 1, 2> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModMaskI);
+
+ LModMaskI(const LAllocation& lhs, const LDefinition& temp0,
+ const LDefinition& temp1, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ }
+
+ int32_t shift() const { return shift_; }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitch : public LInstructionHelper<0, 1, 2> {
+ public:
+ LIR_HEADER(TableSwitch);
+
+ LTableSwitch(const LAllocation& in, const LDefinition& inputCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setTemp(0, inputCopy);
+ setTemp(1, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+ const LAllocation* index() { return getOperand(0); }
+ const LDefinition* tempInt() { return getTemp(0); }
+ // This is added to share the same CodeGenerator prefixes.
+ const LDefinition* tempPointer() { return getTemp(1); }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitchV : public LInstructionHelper<0, BOX_PIECES, 3> {
+ public:
+ LIR_HEADER(TableSwitchV);
+
+ LTableSwitchV(const LBoxAllocation& input, const LDefinition& inputCopy,
+ const LDefinition& floatCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setBoxOperand(InputValue, input);
+ setTemp(0, inputCopy);
+ setTemp(1, floatCopy);
+ setTemp(2, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+
+ static const size_t InputValue = 0;
+
+ const LDefinition* tempInt() { return getTemp(0); }
+ const LDefinition* tempFloat() { return getTemp(1); }
+ const LDefinition* tempPointer() { return getTemp(2); }
+};
+
+class LMulI : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(MulI);
+
+ LMulI() : LBinaryMath(classOpcode) {}
+
+ MMul* mir() { return mir_->toMul(); }
+};
+
+class LUDivOrMod : public LBinaryMath<0> {
+ public:
+ LIR_HEADER(UDivOrMod);
+
+ LUDivOrMod() : LBinaryMath(classOpcode) {}
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+
+ bool trapOnError() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->trapOnError();
+ }
+ return mir_->toDiv()->trapOnError();
+ }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+namespace details {
+
+// Base class for the int64 and non-int64 variants.
+template <size_t NumDefs>
+class LWasmUnalignedLoadBase : public details::LWasmLoadBase<NumDefs, 2> {
+ public:
+ typedef LWasmLoadBase<NumDefs, 2> Base;
+
+ explicit LWasmUnalignedLoadBase(LNode::Opcode opcode, const LAllocation& ptr,
+ const LDefinition& valueHelper)
+ : Base(opcode, ptr, LAllocation()) {
+ Base::setTemp(0, LDefinition::BogusTemp());
+ Base::setTemp(1, valueHelper);
+ }
+
+ const LAllocation* ptr() { return Base::getOperand(0); }
+ const LDefinition* ptrCopy() { return Base::getTemp(0); }
+};
+
+} // namespace details
+
+class LWasmUnalignedLoad : public details::LWasmUnalignedLoadBase<1> {
+ public:
+ LIR_HEADER(WasmUnalignedLoad);
+
+ explicit LWasmUnalignedLoad(const LAllocation& ptr,
+ const LDefinition& valueHelper)
+ : LWasmUnalignedLoadBase(classOpcode, ptr, valueHelper) {}
+};
+
+class LWasmUnalignedLoadI64
+ : public details::LWasmUnalignedLoadBase<INT64_PIECES> {
+ public:
+ LIR_HEADER(WasmUnalignedLoadI64);
+
+ explicit LWasmUnalignedLoadI64(const LAllocation& ptr,
+ const LDefinition& valueHelper)
+ : LWasmUnalignedLoadBase(classOpcode, ptr, valueHelper) {}
+};
+
+namespace details {
+
+// Base class for the int64 and non-int64 variants.
+template <size_t NumOps>
+class LWasmUnalignedStoreBase : public LInstructionHelper<0, NumOps, 2> {
+ public:
+ typedef LInstructionHelper<0, NumOps, 2> Base;
+
+ static const size_t PtrIndex = 0;
+ static const size_t ValueIndex = 1;
+
+ LWasmUnalignedStoreBase(LNode::Opcode opcode, const LAllocation& ptr,
+ const LDefinition& valueHelper)
+ : Base(opcode) {
+ Base::setOperand(0, ptr);
+ Base::setTemp(0, LDefinition::BogusTemp());
+ Base::setTemp(1, valueHelper);
+ }
+
+ MWasmStore* mir() const { return Base::mir_->toWasmStore(); }
+ const LAllocation* ptr() { return Base::getOperand(PtrIndex); }
+ const LDefinition* ptrCopy() { return Base::getTemp(0); }
+};
+
+} // namespace details
+
+class LWasmUnalignedStore : public details::LWasmUnalignedStoreBase<2> {
+ public:
+ LIR_HEADER(WasmUnalignedStore);
+
+ LWasmUnalignedStore(const LAllocation& ptr, const LAllocation& value,
+ const LDefinition& valueHelper)
+ : LWasmUnalignedStoreBase(classOpcode, ptr, valueHelper) {
+ setOperand(1, value);
+ }
+
+ const LAllocation* value() { return Base::getOperand(ValueIndex); }
+};
+
+class LWasmUnalignedStoreI64
+ : public details::LWasmUnalignedStoreBase<1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(WasmUnalignedStoreI64);
+ LWasmUnalignedStoreI64(const LAllocation& ptr, const LInt64Allocation& value,
+ const LDefinition& valueHelper)
+ : LWasmUnalignedStoreBase(classOpcode, ptr, valueHelper) {
+ setInt64Operand(1, value);
+ }
+
+ const LInt64Allocation value() { return getInt64Operand(ValueIndex); }
+};
+
+class LWasmCompareExchangeI64
+ : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES + INT64_PIECES,
+ 0> {
+ public:
+ LIR_HEADER(WasmCompareExchangeI64);
+
+ LWasmCompareExchangeI64(const LAllocation& ptr,
+ const LInt64Allocation& oldValue,
+ const LInt64Allocation& newValue)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, oldValue);
+ setInt64Operand(1 + INT64_PIECES, newValue);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation oldValue() { return getInt64Operand(1); }
+ const LInt64Allocation newValue() {
+ return getInt64Operand(1 + INT64_PIECES);
+ }
+ const MWasmCompareExchangeHeap* mir() const {
+ return mir_->toWasmCompareExchangeHeap();
+ }
+};
+
+class LWasmAtomicExchangeI64
+ : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(WasmAtomicExchangeI64);
+
+ LWasmAtomicExchangeI64(const LAllocation& ptr, const LInt64Allocation& value)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation value() { return getInt64Operand(1); }
+ const MWasmAtomicExchangeHeap* mir() const {
+ return mir_->toWasmAtomicExchangeHeap();
+ }
+};
+
+class LWasmAtomicBinopI64
+ : public LInstructionHelper<INT64_PIECES, 1 + INT64_PIECES, 2> {
+ public:
+ LIR_HEADER(WasmAtomicBinopI64);
+
+ LWasmAtomicBinopI64(const LAllocation& ptr, const LInt64Allocation& value)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation value() { return getInt64Operand(1); }
+ const MWasmAtomicBinopHeap* mir() const {
+ return mir_->toWasmAtomicBinopHeap();
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_LIR_mips_shared_h */
diff --git a/js/src/jit/mips-shared/Lowering-mips-shared.cpp b/js/src/jit/mips-shared/Lowering-mips-shared.cpp
new file mode 100644
index 0000000000..ea9e0ba3dd
--- /dev/null
+++ b/js/src/jit/mips-shared/Lowering-mips-shared.cpp
@@ -0,0 +1,1023 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips-shared/Lowering-mips-shared.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::FloorLog2;
+
+LAllocation LIRGeneratorMIPSShared::useByteOpRegister(MDefinition* mir) {
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorMIPSShared::useByteOpRegisterAtStart(MDefinition* mir) {
+ return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorMIPSShared::useByteOpRegisterOrNonDoubleConstant(
+ MDefinition* mir) {
+ return useRegisterOrNonDoubleConstant(mir);
+}
+
+LDefinition LIRGeneratorMIPSShared::tempByteOpRegister() { return temp(); }
+
+// x = !y
+void LIRGeneratorMIPSShared::lowerForALU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(0, useRegister(input));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+// z = x+y
+void LIRGeneratorMIPSShared::lowerForALU(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegisterOrConstant(rhs));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+void LIRGeneratorMIPSShared::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+ MDefinition* input) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorMIPSShared::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorMIPSShared::lowerForMulInt64(LMulI64* ins, MMul* mir,
+ MDefinition* lhs,
+ MDefinition* rhs) {
+ bool needsTemp = false;
+ bool cannotAliasRhs = false;
+ bool reuseInput = true;
+
+#ifdef JS_CODEGEN_MIPS32
+ needsTemp = true;
+ cannotAliasRhs = true;
+ // See the documentation on willHaveDifferentLIRNodes; that test does not
+ // allow additional constraints.
+ MOZ_CRASH(
+ "cannotAliasRhs cannot be used the way it is used in the guard below");
+ if (rhs->isConstant()) {
+ int64_t constant = rhs->toConstant()->toInt64();
+ int32_t shift = mozilla::FloorLog2(constant);
+ // See special cases in CodeGeneratorMIPSShared::visitMulI64
+ if (constant >= -1 && constant <= 2) {
+ needsTemp = false;
+ }
+ if (int64_t(1) << shift == constant) {
+ needsTemp = false;
+ }
+ if (mozilla::IsPowerOfTwo(static_cast<uint32_t>(constant + 1)) ||
+ mozilla::IsPowerOfTwo(static_cast<uint32_t>(constant - 1)))
+ reuseInput = false;
+ }
+#endif
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES,
+ (willHaveDifferentLIRNodes(lhs, rhs) || cannotAliasRhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+
+ if (needsTemp) {
+ ins->setTemp(0, temp());
+ }
+ if (reuseInput) {
+ defineInt64ReuseInput(ins, mir, 0);
+ } else {
+ defineInt64(ins, mir);
+ }
+}
+
+template <size_t Temps>
+void LIRGeneratorMIPSShared::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+#ifdef JS_CODEGEN_MIPS32
+ if (mir->isRotate()) {
+ if (!rhs->isConstant()) {
+ ins->setTemp(0, temp());
+ }
+ ins->setInt64Operand(0, useInt64Register(lhs));
+ } else {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ }
+#else
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+#endif
+
+ static_assert(LShiftI64::Rhs == INT64_PIECES,
+ "Assume Rhs is located at INT64_PIECES.");
+ static_assert(LRotateI64::Count == INT64_PIECES,
+ "Assume Count is located at INT64_PIECES.");
+
+ ins->setOperand(INT64_PIECES, useRegisterOrConstant(rhs));
+
+#ifdef JS_CODEGEN_MIPS32
+ if (mir->isRotate()) {
+ defineInt64(ins, mir);
+ } else {
+ defineInt64ReuseInput(ins, mir, 0);
+ }
+#else
+ defineInt64ReuseInput(ins, mir, 0);
+#endif
+}
+
+template void LIRGeneratorMIPSShared::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+template void LIRGeneratorMIPSShared::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 1>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+void LIRGeneratorMIPSShared::lowerForCompareI64AndBranch(
+ MTest* mir, MCompare* comp, JSOp op, MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ LCompareI64AndBranch* lir = new (alloc())
+ LCompareI64AndBranch(comp, op, useInt64Register(left),
+ useInt64OrConstant(right), ifTrue, ifFalse);
+ add(lir, mir);
+}
+
+void LIRGeneratorMIPSShared::lowerForFPU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(0, useRegister(input));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template <size_t Temps>
+void LIRGeneratorMIPSShared::lowerForFPU(LInstructionHelper<1, 2, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegister(rhs));
+ define(
+ ins, mir,
+ LDefinition(LDefinition::TypeFrom(mir->type()), LDefinition::REGISTER));
+}
+
+template void LIRGeneratorMIPSShared::lowerForFPU(
+ LInstructionHelper<1, 2, 0>* ins, MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+template void LIRGeneratorMIPSShared::lowerForFPU(
+ LInstructionHelper<1, 2, 1>* ins, MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+
+void LIRGeneratorMIPSShared::lowerForBitAndAndBranch(LBitAndAndBranch* baab,
+ MInstruction* mir,
+ MDefinition* lhs,
+ MDefinition* rhs) {
+ baab->setOperand(0, useRegisterAtStart(lhs));
+ baab->setOperand(1, useRegisterOrConstantAtStart(rhs));
+ add(baab, mir);
+}
+
+void LIRGeneratorMIPSShared::lowerWasmBuiltinTruncateToInt32(
+ MWasmBuiltinTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ if (opd->type() == MIRType::Double) {
+ define(new (alloc()) LWasmBuiltinTruncateDToInt32(
+ useRegister(opd), useFixed(ins->instance(), InstanceReg),
+ LDefinition::BogusTemp()),
+ ins);
+ return;
+ }
+
+ define(new (alloc()) LWasmBuiltinTruncateFToInt32(
+ useRegister(opd), useFixed(ins->instance(), InstanceReg),
+ LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGeneratorMIPSShared::lowerForShift(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegister(lhs));
+ ins->setOperand(1, useRegisterOrConstant(rhs));
+ define(ins, mir);
+}
+
+void LIRGeneratorMIPSShared::lowerDivI(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDiv(div);
+ return;
+ }
+
+ // Division instructions are slow. Division by constant denominators can be
+ // rewritten to use other instructions.
+ if (div->rhs()->isConstant()) {
+ int32_t rhs = div->rhs()->toConstant()->toInt32();
+ // Check for division by a positive power of two, which is an easy and
+ // important case to optimize. Note that other optimizations are also
+ // possible; division by negative powers of two can be optimized in a
+ // similar manner as positive powers of two, and division by other
+ // constants can be optimized by a reciprocal multiplication technique.
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LDivPowTwoI* lir =
+ new (alloc()) LDivPowTwoI(useRegister(div->lhs()), shift, temp());
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+ return;
+ }
+ }
+
+ LDivI* lir = new (alloc())
+ LDivI(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ define(lir, div);
+}
+
+void LIRGeneratorMIPSShared::lowerNegI(MInstruction* ins, MDefinition* input) {
+ define(new (alloc()) LNegI(useRegisterAtStart(input)), ins);
+}
+
+void LIRGeneratorMIPSShared::lowerNegI64(MInstruction* ins,
+ MDefinition* input) {
+ defineInt64ReuseInput(new (alloc()) LNegI64(useInt64RegisterAtStart(input)),
+ ins, 0);
+}
+
+void LIRGenerator::visitAbs(MAbs* ins) {
+ define(allocateAbs(ins, useRegisterAtStart(ins->input())), ins);
+}
+
+void LIRGeneratorMIPSShared::lowerMulI(MMul* mul, MDefinition* lhs,
+ MDefinition* rhs) {
+ LMulI* lir = new (alloc()) LMulI;
+ if (mul->fallible()) {
+ assignSnapshot(lir, mul->bailoutKind());
+ }
+
+ lowerForALU(lir, mul, lhs, rhs);
+}
+
+void LIRGeneratorMIPSShared::lowerModI(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUMod(mod);
+ return;
+ }
+
+ if (mod->rhs()->isConstant()) {
+ int32_t rhs = mod->rhs()->toConstant()->toInt32();
+ int32_t shift = FloorLog2(rhs);
+ if (rhs > 0 && 1 << shift == rhs) {
+ LModPowTwoI* lir =
+ new (alloc()) LModPowTwoI(useRegister(mod->lhs()), shift);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ } else if (shift < 31 && (1 << (shift + 1)) - 1 == rhs) {
+ LModMaskI* lir = new (alloc())
+ LModMaskI(useRegister(mod->lhs()), temp(LDefinition::GENERAL),
+ temp(LDefinition::GENERAL), shift + 1);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+ return;
+ }
+ }
+ LModI* lir =
+ new (alloc()) LModI(useRegister(mod->lhs()), useRegister(mod->rhs()),
+ temp(LDefinition::GENERAL));
+
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ define(lir, mod);
+}
+
+void LIRGenerator::visitPowHalf(MPowHalf* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Double);
+ LPowHalfD* lir = new (alloc()) LPowHalfD(useRegisterAtStart(input));
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGeneratorMIPSShared::lowerWasmSelectI(MWasmSelect* select) {
+ auto* lir = new (alloc())
+ LWasmSelect(useRegisterAtStart(select->trueExpr()),
+ useAny(select->falseExpr()), useRegister(select->condExpr()));
+ defineReuseInput(lir, select, LWasmSelect::TrueExprIndex);
+}
+
+void LIRGeneratorMIPSShared::lowerWasmSelectI64(MWasmSelect* select) {
+ auto* lir = new (alloc()) LWasmSelectI64(
+ useInt64RegisterAtStart(select->trueExpr()),
+ useInt64(select->falseExpr()), useRegister(select->condExpr()));
+ defineInt64ReuseInput(lir, select, LWasmSelectI64::TrueExprIndex);
+}
+
+LTableSwitch* LIRGeneratorMIPSShared::newLTableSwitch(
+ const LAllocation& in, const LDefinition& inputCopy,
+ MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitch(in, inputCopy, temp(), tableswitch);
+}
+
+LTableSwitchV* LIRGeneratorMIPSShared::newLTableSwitchV(
+ MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitchV(useBox(tableswitch->getOperand(0)), temp(),
+ tempDouble(), temp(), tableswitch);
+}
+
+void LIRGeneratorMIPSShared::lowerUrshD(MUrsh* mir) {
+ MDefinition* lhs = mir->lhs();
+ MDefinition* rhs = mir->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+
+ LUrshD* lir = new (alloc())
+ LUrshD(useRegister(lhs), useRegisterOrConstant(rhs), temp());
+ define(lir, mir);
+}
+
+void LIRGeneratorMIPSShared::lowerPowOfTwoI(MPow* mir) {
+ int32_t base = mir->input()->toConstant()->toInt32();
+ MDefinition* power = mir->power();
+
+ auto* lir = new (alloc()) LPowOfTwoI(useRegister(power), base);
+ assignSnapshot(lir, mir->bailoutKind());
+ define(lir, mir);
+}
+
+void LIRGeneratorMIPSShared::lowerBigIntLsh(MBigIntLsh* ins) {
+ auto* lir = new (alloc()) LBigIntLsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorMIPSShared::lowerBigIntRsh(MBigIntRsh* ins) {
+ auto* lir = new (alloc()) LBigIntRsh(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), temp(), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmNeg(MWasmNeg* ins) {
+ if (ins->type() == MIRType::Int32) {
+ define(new (alloc()) LNegI(useRegisterAtStart(ins->input())), ins);
+ } else if (ins->type() == MIRType::Float32) {
+ define(new (alloc()) LNegF(useRegisterAtStart(ins->input())), ins);
+ } else {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ define(new (alloc()) LNegD(useRegisterAtStart(ins->input())), ins);
+ }
+}
+
+void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) {
+ auto* lir = new (alloc()) LWasmHeapBase(LAllocation());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
+ MDefinition* base = ins->base();
+ // 'base' is a GPR but may be of either type. If it is 32-bit, it is
+ // sign-extended on mips64 platform and we should explicitly promote it to
+ // 64-bit when use it as an index register in memory accesses.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ LAllocation ptr;
+#ifdef JS_CODEGEN_MIPS32
+ if (ins->type() == MIRType::Int64) {
+ ptr = useRegister(base);
+ } else {
+ ptr = useRegisterAtStart(base);
+ }
+#else
+ ptr = useRegisterAtStart(base);
+#endif
+
+ if (IsUnaligned(ins->access())) {
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new (alloc()) LWasmUnalignedLoadI64(ptr, temp());
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ defineInt64(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LWasmUnalignedLoad(ptr, temp());
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ define(lir, ins);
+ return;
+ }
+
+ if (ins->type() == MIRType::Int64) {
+#ifdef JS_CODEGEN_MIPS32
+ if (ins->access().isAtomic()) {
+ auto* lir = new (alloc()) LWasmAtomicLoadI64(ptr);
+ defineInt64(lir, ins);
+ return;
+ }
+#endif
+ auto* lir = new (alloc()) LWasmLoadI64(ptr);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ defineInt64(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LWasmLoad(ptr);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmStore(MWasmStore* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ MDefinition* value = ins->value();
+
+ if (IsUnaligned(ins->access())) {
+ LAllocation baseAlloc = useRegisterAtStart(base);
+ if (ins->access().type() == Scalar::Int64) {
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+ auto* lir =
+ new (alloc()) LWasmUnalignedStoreI64(baseAlloc, valueAlloc, temp());
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ auto* lir =
+ new (alloc()) LWasmUnalignedStore(baseAlloc, valueAlloc, temp());
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ add(lir, ins);
+ return;
+ }
+
+ if (ins->access().type() == Scalar::Int64) {
+#ifdef JS_CODEGEN_MIPS32
+ if (ins->access().isAtomic()) {
+ auto* lir = new (alloc()) LWasmAtomicStoreI64(
+ useRegister(base), useInt64Register(value), temp());
+ add(lir, ins);
+ return;
+ }
+#endif
+
+ LAllocation baseAlloc = useRegisterAtStart(base);
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(value);
+ auto* lir = new (alloc()) LWasmStoreI64(baseAlloc, valueAlloc);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation baseAlloc = useRegisterAtStart(base);
+ LAllocation valueAlloc = useRegisterAtStart(value);
+ auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc);
+ if (ins->access().offset()) {
+ lir->setTemp(0, tempCopy(base, 0));
+ }
+
+ add(lir, ins);
+}
+
+void LIRGeneratorMIPSShared::lowerUDiv(MDiv* div) {
+ MDefinition* lhs = div->getOperand(0);
+ MDefinition* rhs = div->getOperand(1);
+
+ LUDivOrMod* lir = new (alloc()) LUDivOrMod;
+ lir->setOperand(0, useRegister(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+
+ define(lir, div);
+}
+
+void LIRGeneratorMIPSShared::lowerUMod(MMod* mod) {
+ MDefinition* lhs = mod->getOperand(0);
+ MDefinition* rhs = mod->getOperand(1);
+
+ LUDivOrMod* lir = new (alloc()) LUDivOrMod;
+ lir->setOperand(0, useRegister(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+
+ define(lir, mod);
+}
+
+void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToDouble* lir =
+ new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToFloat32* lir =
+ new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins) {
+ MOZ_ASSERT(ins->access().offset() == 0);
+
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+ LAllocation baseAlloc;
+ LAllocation limitAlloc;
+ // For MIPS it is best to keep the 'base' in a register if a bounds check
+ // is needed.
+ if (base->isConstant() && !ins->needsBoundsCheck()) {
+ // A bounds check is only skipped for a positive index.
+ MOZ_ASSERT(base->toConstant()->toInt32() >= 0);
+ baseAlloc = LAllocation(base->toConstant());
+ } else {
+ baseAlloc = useRegisterAtStart(base);
+ if (ins->needsBoundsCheck()) {
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT(boundsCheckLimit->type() == MIRType::Int32);
+ limitAlloc = useRegisterAtStart(boundsCheckLimit);
+ }
+ }
+
+ define(new (alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc, LAllocation()),
+ ins);
+}
+
+void LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins) {
+ MOZ_ASSERT(ins->access().offset() == 0);
+
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+ LAllocation baseAlloc;
+ LAllocation limitAlloc;
+ if (base->isConstant() && !ins->needsBoundsCheck()) {
+ MOZ_ASSERT(base->toConstant()->toInt32() >= 0);
+ baseAlloc = LAllocation(base->toConstant());
+ } else {
+ baseAlloc = useRegisterAtStart(base);
+ if (ins->needsBoundsCheck()) {
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT(boundsCheckLimit->type() == MIRType::Int32);
+ limitAlloc = useRegisterAtStart(boundsCheckLimit);
+ }
+ }
+
+ add(new (alloc()) LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()),
+ limitAlloc, LAllocation()),
+ ins);
+}
+
+void LIRGenerator::visitSubstr(MSubstr* ins) {
+ LSubstr* lir = new (alloc())
+ LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
+ useRegister(ins->length()), temp(), temp(), tempByteOpRegister());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation newval = useRegister(ins->newval());
+ const LAllocation oldval = useRegister(ins->oldval());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64();
+
+ auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+ elements, index, oldval, newval, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ // If the target is a floating register then we need a temp at the
+ // CodeGenerator level for creating the result.
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ outTemp = temp();
+ }
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LCompareExchangeTypedArrayElement* lir = new (alloc())
+ LCompareExchangeTypedArrayElement(elements, index, oldval, newval,
+ outTemp, valueTemp, offsetTemp,
+ maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LDefinition temp2 = temp();
+
+ auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64(
+ elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ // If the target is a floating register then we need a temp at the
+ // CodeGenerator level for creating the result.
+
+ MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32);
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32) {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ outTemp = temp();
+ }
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LAtomicExchangeTypedArrayElement* lir =
+ new (alloc()) LAtomicExchangeTypedArrayElement(
+ elements, index, value, outTemp, valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmCompareExchangeI64(
+ useRegister(base), useInt64Register(ins->oldValue()),
+ useInt64Register(ins->newValue()));
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LWasmCompareExchangeHeap* lir = new (alloc()) LWasmCompareExchangeHeap(
+ useRegister(base), useRegister(ins->oldValue()),
+ useRegister(ins->newValue()), valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmAtomicExchangeI64(
+ useRegister(base), useInt64Register(ins->value()));
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ LWasmAtomicExchangeHeap* lir = new (alloc())
+ LWasmAtomicExchangeHeap(useRegister(base), useRegister(ins->value()),
+ valueTemp, offsetTemp, maskTemp);
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc())
+ LWasmAtomicBinopI64(useRegister(base), useInt64Register(ins->value()));
+ lir->setTemp(0, temp());
+#ifdef JS_CODEGEN_MIPS32
+ lir->setTemp(1, temp());
+#endif
+ defineInt64(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (ins->access().byteSize() < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ if (!ins->hasUses()) {
+ LWasmAtomicBinopHeapForEffect* lir = new (alloc())
+ LWasmAtomicBinopHeapForEffect(useRegister(base),
+ useRegister(ins->value()), valueTemp,
+ offsetTemp, maskTemp);
+ add(lir, ins);
+ return;
+ }
+
+ LWasmAtomicBinopHeap* lir = new (alloc())
+ LWasmAtomicBinopHeap(useRegister(base), useRegister(ins->value()),
+ valueTemp, offsetTemp, maskTemp);
+
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ const LAllocation value = useRegister(ins->value());
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2 = tempInt64();
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We can omit allocating the result BigInt.
+
+ if (ins->isForEffect()) {
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+ elements, index, value, temp1, temp2);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+
+ auto* lir = new (alloc())
+ LAtomicTypedArrayElementBinop64(elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ LDefinition valueTemp = LDefinition::BogusTemp();
+ LDefinition offsetTemp = LDefinition::BogusTemp();
+ LDefinition maskTemp = LDefinition::BogusTemp();
+
+ if (Scalar::byteSize(ins->arrayType()) < 4) {
+ valueTemp = temp();
+ offsetTemp = temp();
+ maskTemp = temp();
+ }
+
+ if (ins->isForEffect()) {
+ LAtomicTypedArrayElementBinopForEffect* lir =
+ new (alloc()) LAtomicTypedArrayElementBinopForEffect(
+ elements, index, value, valueTemp, offsetTemp, maskTemp);
+ add(lir, ins);
+ return;
+ }
+
+ // For a Uint32Array with a known double result we need a temp for
+ // the intermediate output.
+
+ LDefinition outTemp = LDefinition::BogusTemp();
+
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ outTemp = temp();
+ }
+
+ LAtomicTypedArrayElementBinop* lir =
+ new (alloc()) LAtomicTypedArrayElementBinop(
+ elements, index, value, outTemp, valueTemp, offsetTemp, maskTemp);
+ define(lir, ins);
+}
+
+void LIRGenerator::visitCopySign(MCopySign* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(IsFloatingPointType(lhs->type()));
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(lhs->type() == ins->type());
+
+ LInstructionHelper<1, 2, 2>* lir;
+ if (lhs->type() == MIRType::Double) {
+ lir = new (alloc()) LCopySignD();
+ } else {
+ lir = new (alloc()) LCopySignF();
+ }
+
+ lir->setTemp(0, temp());
+ lir->setTemp(1, temp());
+
+ lir->setOperand(0, useRegisterAtStart(lhs));
+ lir->setOperand(1, useRegister(rhs));
+ defineReuseInput(lir, ins, 0);
+}
+
+void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
+ defineInt64(
+ new (alloc()) LExtendInt32ToInt64(useRegisterAtStart(ins->input())), ins);
+}
+
+void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
+ defineInt64(new (alloc())
+ LSignExtendInt64(useInt64RegisterAtStart(ins->input())),
+ ins);
+}
+
+// On mips we specialize the only cases where compare is {U,}Int32 and select
+// is {U,}Int32.
+bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
+ MCompare::CompareType compTy, MIRType insTy) {
+ return insTy == MIRType::Int32 && (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32);
+}
+
+void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
+ MDefinition* lhs,
+ MDefinition* rhs,
+ MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+ auto* lir = new (alloc()) LWasmCompareAndSelect(
+ useRegister(lhs), useRegister(rhs), compTy, jsop,
+ useRegisterAtStart(ins->trueExpr()), useRegister(ins->falseExpr()));
+ defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
+}
+
+void LIRGenerator::visitWasmTernarySimd128(MWasmTernarySimd128* ins) {
+ MOZ_CRASH("ternary SIMD NYI");
+}
+
+void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) {
+ MOZ_CRASH("binary SIMD NYI");
+}
+
+#ifdef ENABLE_WASM_SIMD
+bool MWasmTernarySimd128::specializeBitselectConstantMaskAsShuffle(
+ int8_t shuffle[16]) {
+ return false;
+}
+bool MWasmTernarySimd128::canRelaxBitselect() { return false; }
+
+bool MWasmBinarySimd128::canPmaddubsw() { return false; }
+#endif
+
+bool MWasmBinarySimd128::specializeForConstantRhs() {
+ // Probably many we want to do here
+ return false;
+}
+
+void LIRGenerator::visitWasmBinarySimd128WithConstant(
+ MWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH("binary SIMD with constant NYI");
+}
+
+void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) {
+ MOZ_CRASH("shift SIMD NYI");
+}
+
+void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) {
+ MOZ_CRASH("shuffle SIMD NYI");
+}
+
+void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH("replace-lane SIMD NYI");
+}
+
+void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) {
+ MOZ_CRASH("scalar-to-SIMD NYI");
+}
+
+void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) {
+ MOZ_CRASH("unary SIMD NYI");
+}
+
+void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) {
+ MOZ_CRASH("reduce-SIMD NYI");
+}
+
+void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH("load-lane SIMD NYI");
+}
+
+void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH("store-lane SIMD NYI");
+}
diff --git a/js/src/jit/mips-shared/Lowering-mips-shared.h b/js/src/jit/mips-shared/Lowering-mips-shared.h
new file mode 100644
index 0000000000..ca74a7aaf5
--- /dev/null
+++ b/js/src/jit/mips-shared/Lowering-mips-shared.h
@@ -0,0 +1,89 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_Lowering_mips_shared_h
+#define jit_mips_shared_Lowering_mips_shared_h
+
+#include "jit/shared/Lowering-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorMIPSShared : public LIRGeneratorShared {
+ protected:
+ LIRGeneratorMIPSShared(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorShared(gen, graph, lirGraph) {}
+
+ // x86 has constraints on what registers can be formatted for 1-byte
+ // stores and loads; on MIPS all registers are okay.
+ LAllocation useByteOpRegister(MDefinition* mir);
+ LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+ LDefinition tempByteOpRegister();
+
+ bool needTempForPostBarrier() { return false; }
+
+ void lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerUrshD(MUrsh* mir);
+
+ void lowerPowOfTwoI(MPow* mir);
+
+ void lowerForALU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* input);
+ void lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* input);
+ void lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+ void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs,
+ MDefinition* rhs);
+ template <size_t Temps>
+ void lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForCompareI64AndBranch(MTest* mir, MCompare* comp, JSOp op,
+ MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse);
+
+ void lowerForFPU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* src);
+ template <size_t Temps>
+ void lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForBitAndAndBranch(LBitAndAndBranch* baab, MInstruction* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins);
+ void lowerDivI(MDiv* div);
+ void lowerModI(MMod* mod);
+ void lowerNegI(MInstruction* ins, MDefinition* input);
+ void lowerNegI64(MInstruction* ins, MDefinition* input);
+ void lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs);
+ void lowerUDiv(MDiv* div);
+ void lowerUMod(MMod* mod);
+ void lowerWasmSelectI(MWasmSelect* select);
+ void lowerWasmSelectI64(MWasmSelect* select);
+
+ void lowerBigIntLsh(MBigIntLsh* ins);
+ void lowerBigIntRsh(MBigIntRsh* ins);
+
+ LTableSwitch* newLTableSwitch(const LAllocation& in,
+ const LDefinition& inputCopy,
+ MTableSwitch* ins);
+ LTableSwitchV* newLTableSwitchV(MTableSwitch* ins);
+
+ void lowerPhi(MPhi* phi);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_Lowering_mips_shared_h */
diff --git a/js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h b/js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h
new file mode 100644
index 0000000000..69c914f584
--- /dev/null
+++ b/js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h
@@ -0,0 +1,1312 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_MacroAssembler_mips_shared_inl_h
+#define jit_mips_shared_MacroAssembler_mips_shared_inl_h
+
+#include "jit/mips-shared/MacroAssembler-mips-shared.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
+ moveFromFloat32(src, dest);
+}
+
+void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
+ moveToFloat32(src, dest);
+}
+
+void MacroAssembler::move8SignExtend(Register src, Register dest) {
+ ma_seb(dest, src);
+}
+
+void MacroAssembler::move16SignExtend(Register src, Register dest) {
+ ma_seh(dest, src);
+}
+
+void MacroAssembler::loadAbiReturnAddress(Register dest) { movePtr(ra, dest); }
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::not32(Register reg) { ma_not(reg, reg); }
+
+void MacroAssembler::and32(Register src, Register dest) {
+ as_and(dest, dest, src);
+}
+
+void MacroAssembler::and32(Imm32 imm, Register dest) { ma_and(dest, imm); }
+
+void MacroAssembler::and32(Imm32 imm, const Address& dest) {
+ load32(dest, SecondScratchReg);
+ ma_and(SecondScratchReg, imm);
+ store32(SecondScratchReg, dest);
+}
+
+void MacroAssembler::and32(const Address& src, Register dest) {
+ load32(src, SecondScratchReg);
+ ma_and(dest, SecondScratchReg);
+}
+
+void MacroAssembler::or32(Register src, Register dest) { ma_or(dest, src); }
+
+void MacroAssembler::or32(Imm32 imm, Register dest) { ma_or(dest, imm); }
+
+void MacroAssembler::or32(Imm32 imm, const Address& dest) {
+ load32(dest, SecondScratchReg);
+ ma_or(SecondScratchReg, imm);
+ store32(SecondScratchReg, dest);
+}
+
+void MacroAssembler::xor32(Register src, Register dest) { ma_xor(dest, src); }
+
+void MacroAssembler::xor32(Imm32 imm, Register dest) { ma_xor(dest, imm); }
+
+void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
+ load32(dest, SecondScratchReg);
+ ma_xor(SecondScratchReg, imm);
+ store32(SecondScratchReg, dest);
+}
+
+void MacroAssembler::xor32(const Address& src, Register dest) {
+ load32(src, SecondScratchReg);
+ ma_xor(dest, SecondScratchReg);
+}
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap16SignExtend(Register reg) {
+ ma_wsbh(reg, reg);
+ ma_seh(reg, reg);
+}
+
+void MacroAssembler::byteSwap16ZeroExtend(Register reg) {
+ ma_wsbh(reg, reg);
+ ma_and(reg, Imm32(0xFFFF));
+}
+
+void MacroAssembler::byteSwap32(Register reg) {
+ ma_wsbh(reg, reg);
+ as_rotr(reg, reg, 16);
+}
+
+// ===============================================================
+// Arithmetic instructions
+
+void MacroAssembler::add32(Register src, Register dest) {
+ as_addu(dest, dest, src);
+}
+
+void MacroAssembler::add32(Imm32 imm, Register dest) {
+ ma_addu(dest, dest, imm);
+}
+
+void MacroAssembler::add32(Imm32 imm, const Address& dest) {
+ load32(dest, SecondScratchReg);
+ ma_addu(SecondScratchReg, imm);
+ store32(SecondScratchReg, dest);
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
+ loadPtr(dest, ScratchRegister);
+ addPtr(imm, ScratchRegister);
+ storePtr(ScratchRegister, dest);
+}
+
+void MacroAssembler::addPtr(const Address& src, Register dest) {
+ loadPtr(src, ScratchRegister);
+ addPtr(ScratchRegister, dest);
+}
+
+void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
+ as_addd(dest, dest, src);
+}
+
+void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
+ as_adds(dest, dest, src);
+}
+
+void MacroAssembler::sub32(Register src, Register dest) {
+ as_subu(dest, dest, src);
+}
+
+void MacroAssembler::sub32(Imm32 imm, Register dest) {
+ ma_subu(dest, dest, imm);
+}
+
+void MacroAssembler::sub32(const Address& src, Register dest) {
+ load32(src, SecondScratchReg);
+ as_subu(dest, dest, SecondScratchReg);
+}
+
+void MacroAssembler::subPtr(Register src, const Address& dest) {
+ loadPtr(dest, SecondScratchReg);
+ subPtr(src, SecondScratchReg);
+ storePtr(SecondScratchReg, dest);
+}
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) {
+ loadPtr(addr, SecondScratchReg);
+ subPtr(SecondScratchReg, dest);
+}
+
+void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
+ as_subd(dest, dest, src);
+}
+
+void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
+ as_subs(dest, dest, src);
+}
+
+void MacroAssembler::mul32(Register rhs, Register srcDest) {
+ as_mul(srcDest, srcDest, rhs);
+}
+
+void MacroAssembler::mul32(Imm32 imm, Register srcDest) {
+ move32(imm, SecondScratchReg);
+ mul32(SecondScratchReg, srcDest);
+}
+
+void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
+ as_muls(dest, dest, src);
+}
+
+void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
+ as_muld(dest, dest, src);
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+ FloatRegister dest) {
+ movePtr(imm, ScratchRegister);
+ loadDouble(Address(ScratchRegister, 0), ScratchDoubleReg);
+ mulDouble(ScratchDoubleReg, dest);
+}
+
+void MacroAssembler::quotient32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ if (isUnsigned) {
+#ifdef MIPSR6
+ as_divu(srcDest, srcDest, rhs);
+#else
+ as_divu(srcDest, rhs);
+#endif
+ } else {
+#ifdef MIPSR6
+ as_div(srcDest, srcDest, rhs);
+#else
+ as_div(srcDest, rhs);
+#endif
+ }
+#ifndef MIPSR6
+ as_mflo(srcDest);
+#endif
+}
+
+void MacroAssembler::remainder32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ if (isUnsigned) {
+#ifdef MIPSR6
+ as_modu(srcDest, srcDest, rhs);
+#else
+ as_divu(srcDest, rhs);
+#endif
+ } else {
+#ifdef MIPSR6
+ as_mod(srcDest, srcDest, rhs);
+#else
+ as_div(srcDest, rhs);
+#endif
+ }
+#ifndef MIPSR6
+ as_mfhi(srcDest);
+#endif
+}
+
+void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
+ as_divs(dest, dest, src);
+}
+
+void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
+ as_divd(dest, dest, src);
+}
+
+void MacroAssembler::neg32(Register reg) { ma_negu(reg, reg); }
+
+void MacroAssembler::negateDouble(FloatRegister reg) { as_negd(reg, reg); }
+
+void MacroAssembler::negateFloat(FloatRegister reg) { as_negs(reg, reg); }
+
+void MacroAssembler::abs32(Register src, Register dest) {
+ // TODO: There's probably a better way to do this.
+ if (src != dest) {
+ move32(src, dest);
+ }
+ Label positive;
+ branchTest32(Assembler::NotSigned, dest, dest, &positive);
+ neg32(dest);
+ bind(&positive);
+}
+
+void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
+ as_abss(dest, src);
+}
+
+void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
+ as_absd(dest, src);
+}
+
+void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
+ as_sqrts(dest, src);
+}
+
+void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
+ as_sqrtd(dest, src);
+}
+
+void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxFloat32(srcDest, other, handleNaN, false);
+}
+
+void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxDouble(srcDest, other, handleNaN, false);
+}
+
+void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxFloat32(srcDest, other, handleNaN, true);
+}
+
+void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxDouble(srcDest, other, handleNaN, true);
+}
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshift32(Register src, Register dest) {
+ ma_sll(dest, dest, src);
+}
+
+void MacroAssembler::flexibleLshift32(Register src, Register dest) {
+ lshift32(src, dest);
+}
+
+void MacroAssembler::lshift32(Imm32 imm, Register dest) {
+ ma_sll(dest, dest, imm);
+}
+
+void MacroAssembler::rshift32(Register src, Register dest) {
+ ma_srl(dest, dest, src);
+}
+
+void MacroAssembler::flexibleRshift32(Register src, Register dest) {
+ rshift32(src, dest);
+}
+
+void MacroAssembler::rshift32(Imm32 imm, Register dest) {
+ ma_srl(dest, dest, imm);
+}
+
+void MacroAssembler::rshift32Arithmetic(Register src, Register dest) {
+ ma_sra(dest, dest, src);
+}
+
+void MacroAssembler::flexibleRshift32Arithmetic(Register src, Register dest) {
+ rshift32Arithmetic(src, dest);
+}
+
+void MacroAssembler::rshift32Arithmetic(Imm32 imm, Register dest) {
+ ma_sra(dest, dest, imm);
+}
+
+// ===============================================================
+// Rotation functions
+void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
+ if (count.value) {
+ ma_rol(dest, input, count);
+ } else {
+ ma_move(dest, input);
+ }
+}
+void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
+ ma_rol(dest, input, count);
+}
+void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
+ if (count.value) {
+ ma_ror(dest, input, count);
+ } else {
+ ma_move(dest, input);
+ }
+}
+void MacroAssembler::rotateRight(Register count, Register input,
+ Register dest) {
+ ma_ror(dest, input, count);
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
+ as_clz(dest, src);
+}
+
+void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
+ ma_ctz(dest, src);
+}
+
+void MacroAssembler::popcnt32(Register input, Register output, Register tmp) {
+ // Equivalent to GCC output of mozilla::CountPopulation32()
+ ma_move(output, input);
+ ma_sra(tmp, input, Imm32(1));
+ ma_and(tmp, Imm32(0x55555555));
+ ma_subu(output, tmp);
+ ma_sra(tmp, output, Imm32(2));
+ ma_and(output, Imm32(0x33333333));
+ ma_and(tmp, Imm32(0x33333333));
+ ma_addu(output, tmp);
+ ma_srl(tmp, output, Imm32(4));
+ ma_addu(output, tmp);
+ ma_and(output, Imm32(0xF0F0F0F));
+ ma_sll(tmp, output, Imm32(8));
+ ma_addu(output, tmp);
+ ma_sll(tmp, output, Imm32(16));
+ ma_addu(output, tmp);
+ ma_sra(output, output, Imm32(24));
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, scratch2);
+ ma_cmp_set(dest, scratch2, Imm32(uint8_t(rhs.value)), cond);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, scratch2);
+ ma_cmp_set(dest, scratch2, Imm32(int8_t(rhs.value)), cond);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load16ZeroExtend(lhs, scratch2);
+ ma_cmp_set(dest, scratch2, Imm32(uint16_t(rhs.value)), cond);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load16SignExtend(lhs, scratch2);
+ ma_cmp_set(dest, scratch2, Imm32(int16_t(rhs.value)), cond);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(lhs, scratch2);
+ branch32(cond, scratch2, Imm32(uint8_t(rhs.value)), label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(lhs, scratch2);
+ branch32(cond, scratch2, Imm32(int8_t(rhs.value)), label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ computeScaledAddress(lhs, scratch2);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load8ZeroExtend(Address(scratch2, lhs.offset), scratch2);
+ branch32(cond, scratch2, rhs, label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load8SignExtend(Address(scratch2, lhs.offset), scratch2);
+ branch32(cond, scratch2, rhs, label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(scratch2 != lhs.base);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ load16ZeroExtend(lhs, scratch2);
+ branch32(cond, scratch2, Imm32(uint16_t(rhs.value)), label);
+ break;
+
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ load16SignExtend(lhs, scratch2);
+ branch32(cond, scratch2, Imm32(int16_t(rhs.value)), label);
+ break;
+
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 imm,
+ L label) {
+ ma_b(lhs, imm, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
+ Label* label) {
+ load32(lhs, SecondScratchReg);
+ ma_b(SecondScratchReg, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ load32(lhs, SecondScratchReg);
+ ma_b(SecondScratchReg, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ load32(lhs, SecondScratchReg);
+ ma_b(SecondScratchReg, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ load32(lhs, SecondScratchReg);
+ ma_b(SecondScratchReg, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
+ Label* label) {
+ load32(lhs, SecondScratchReg);
+ ma_b(SecondScratchReg, rhs, label, cond);
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress addr,
+ Imm32 imm, Label* label) {
+ load32(addr, SecondScratchReg);
+ ma_b(SecondScratchReg, imm, label, cond);
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
+ Label* label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
+ Label* label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
+ Label* label) {
+ ma_b(lhs, rhs, label, cond);
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
+ L label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
+ Label* label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
+ Label* label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
+ Label* label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ ImmWord rhs, Label* label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+ Register rhs, Label* label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ ImmWord rhs, Label* label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ Register rhs, Label* label) {
+ SecondScratchRegisterScope scratch(*this);
+ loadPtr(lhs, scratch);
+ branchPtr(cond, scratch, rhs, label);
+}
+
+void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ ma_bc1s(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ convertFloat32ToInt32(src, dest, fail, false);
+}
+
+void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ ma_bc1d(lhs, rhs, label, cond);
+}
+
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ convertDoubleToInt32(src, dest, fail, false);
+}
+
+template <typename T>
+void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
+ Label* overflow) {
+ switch (cond) {
+ case Overflow:
+ ma_add32TestOverflow(dest, dest, src, overflow);
+ break;
+ case CarryClear:
+ case CarrySet:
+ ma_add32TestCarry(cond, dest, dest, src, overflow);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <typename T>
+void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
+ Label* overflow) {
+ switch (cond) {
+ case Overflow:
+ ma_sub32TestOverflow(dest, dest, src, overflow);
+ break;
+ case NonZero:
+ case Zero:
+ case Signed:
+ case NotSigned:
+ ma_subu(dest, src);
+ ma_b(dest, dest, overflow, cond);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <typename T>
+void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
+ Label* overflow) {
+ MOZ_ASSERT(cond == Assembler::Overflow);
+ ma_mul32TestOverflow(dest, dest, src, overflow);
+}
+
+template <typename T>
+void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ rshift32(src, dest);
+ branch32(cond == Zero ? Equal : NotEqual, dest, Imm32(0), label);
+}
+
+void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
+ MOZ_ASSERT(cond == Overflow);
+ neg32(reg);
+ branch32(Assembler::Equal, reg, Imm32(INT32_MIN), label);
+}
+
+template <typename T>
+void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ switch (cond) {
+ case Overflow:
+ ma_addPtrTestOverflow(dest, dest, src, label);
+ break;
+ case CarryClear:
+ case CarrySet:
+ ma_addPtrTestCarry(cond, dest, dest, src, label);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+template <typename T>
+void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ switch (cond) {
+ case Overflow:
+ ma_subPtrTestOverflow(dest, dest, src, label);
+ break;
+ case NonZero:
+ case Zero:
+ case Signed:
+ case NotSigned:
+ subPtr(src, dest);
+ ma_b(dest, dest, label, cond);
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+}
+
+void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Overflow);
+ ma_mulPtrTestOverflow(dest, dest, src, label);
+}
+
+void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ subPtr(rhs, lhs);
+ branchPtr(cond, lhs, Imm32(0), label);
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ if (lhs == rhs) {
+ ma_b(lhs, rhs, label, cond);
+ } else {
+ as_and(ScratchRegister, lhs, rhs);
+ ma_b(ScratchRegister, ScratchRegister, label, cond);
+ }
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ ma_and(ScratchRegister, lhs, rhs);
+ ma_b(ScratchRegister, ScratchRegister, label, cond);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ load32(lhs, SecondScratchReg);
+ branchTest32(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ load32(lhs, SecondScratchReg);
+ branchTest32(cond, SecondScratchReg, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ if (lhs == rhs) {
+ ma_b(lhs, rhs, label, cond);
+ } else {
+ as_and(ScratchRegister, lhs, rhs);
+ ma_b(ScratchRegister, ScratchRegister, label, cond);
+ }
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ ma_and(ScratchRegister, lhs, rhs);
+ ma_b(ScratchRegister, ScratchRegister, label, cond);
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
+ Imm32 rhs, Label* label) {
+ loadPtr(lhs, SecondScratchReg);
+ branchTestPtr(cond, SecondScratchReg, rhs, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_UNDEFINED), label, cond);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestUndefined(cond, tag, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestUndefined(cond, tag, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_INT32), label, cond);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestInt32(cond, tag, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestInt32(cond, tag, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestDouble(cond, tag, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestDouble(cond, tag, label);
+}
+
+void MacroAssembler::branchTestDoubleTruthy(bool b, FloatRegister value,
+ Label* label) {
+ ma_lid(ScratchDoubleReg, 0.0);
+ DoubleCondition cond = b ? DoubleNotEqual : DoubleEqualOrUnordered;
+ ma_bc1d(value, ScratchDoubleReg, label, cond);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = cond == Equal ? BelowOrEqual : Above;
+ ma_b(tag, ImmTag(JS::detail::ValueUpperInclNumberTag), label, actual);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_BOOLEAN), label, cond);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestBoolean(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestBoolean(cond, tag, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_STRING), label, cond);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestString(cond, tag, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestString(cond, tag, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_SYMBOL), label, cond);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestSymbol(cond, tag, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestSymbol(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_BIGINT), label, cond);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestBigInt(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_NULL), label, cond);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestNull(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestNull(cond, tag, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_OBJECT), label, cond);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestObject(cond, tag, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestObject(cond, tag, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond,
+ const ValueOperand& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& address,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ ma_b(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag), label,
+ (cond == Equal) ? AboveOrEqual : Below);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag), label,
+ (cond == Equal) ? Below : AboveOrEqual);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(tag, ImmTag(JSVAL_TAG_MAGIC), label, cond);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestMagic(cond, tag, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestMagic(cond, tag, label);
+}
+
+template <typename T>
+void MacroAssembler::testNumberSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JS::detail::ValueUpperInclNumberTag),
+ cond == Equal ? BelowOrEqual : Above);
+}
+
+template <typename T>
+void MacroAssembler::testBooleanSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_BOOLEAN), cond);
+}
+
+template <typename T>
+void MacroAssembler::testStringSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_STRING), cond);
+}
+
+template <typename T>
+void MacroAssembler::testSymbolSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_SYMBOL), cond);
+}
+
+template <typename T>
+void MacroAssembler::testBigIntSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(src, scratch2);
+ ma_cmp_set(dest, tag, ImmTag(JSVAL_TAG_BIGINT), cond);
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
+ loadPtr(addr, ScratchRegister);
+ branch(ScratchRegister);
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ Register scratch = ScratchRegister;
+ MOZ_ASSERT(src != scratch && dest != scratch);
+ cmp32Set(cond, lhs, rhs, scratch);
+#ifdef MIPSR6
+ as_selnez(src, src, scratch);
+ as_seleqz(dest, dest, scratch);
+ as_or(dest, dest, src);
+#else
+ as_movn(dest, src, scratch);
+#endif
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ Register scratch = ScratchRegister;
+ MOZ_ASSERT(src != scratch && dest != scratch);
+ cmp32Set(cond, lhs, rhs, scratch);
+#ifdef MIPSR6
+ as_selnez(src, src, scratch);
+ as_seleqz(dest, dest, scratch);
+ as_or(dest, dest, src);
+#else
+ as_movn(dest, src, scratch);
+#endif
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ SecondScratchRegisterScope scratch2(*this);
+ MOZ_ASSERT(lhs != scratch2 && src != scratch2 && dest != scratch2);
+ load32(rhs, scratch2);
+ cmp32Move32(cond, lhs, scratch2, src, dest);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
+ const Address& rhs, const Address& src,
+ Register dest) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(lhs != scratch && dest != scratch);
+ load32(rhs, scratch);
+ cmp32Load32(cond, lhs, scratch, src, dest);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
+ const Address& src, Register dest) {
+ Label skip;
+ branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
+ load32(src, dest);
+ bind(&skip);
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ Label skip;
+ branch32(Assembler::InvertCondition(cond), lhs, rhs, &skip);
+ loadPtr(src, dest);
+ bind(&skip);
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+ Imm32 mask, const Address& src,
+ Register dest) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreStringMitigations);
+ Label skip;
+ branchTest32(Assembler::InvertCondition(cond), addr, mask, &skip);
+ loadPtr(src, dest);
+ bind(&skip);
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+ Imm32 mask, Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
+ branch32(Assembler::BelowOrEqual, length, index, failure);
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
+ branchPtr(Assembler::BelowOrEqual, length, index, failure);
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+ const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_RELEASE_ASSERT(!JitOptions.spectreIndexMasking);
+ branchPtr(Assembler::BelowOrEqual, length, index, failure);
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+// ========================================================================
+// Memory access primitives.
+
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const Address& addr) {
+ ma_sd(src, addr);
+}
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const BaseIndex& addr) {
+ ma_sd(src, addr);
+}
+
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const Address& addr) {
+ ma_ss(src, addr);
+}
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const BaseIndex& addr) {
+ ma_ss(src, addr);
+}
+
+void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) {
+ if (barrier) {
+ as_sync();
+ }
+}
+
+// ===============================================================
+// Clamping functions.
+
+void MacroAssembler::clampIntToUint8(Register reg) {
+ // If reg is < 0, then we want to clamp to 0.
+ as_slti(ScratchRegister, reg, 0);
+#ifdef MIPSR6
+ as_seleqz(reg, reg, ScratchRegister);
+#else
+ as_movn(reg, zero, ScratchRegister);
+#endif
+ // If reg is >= 255, then we want to clamp to 255.
+ ma_li(SecondScratchReg, Imm32(255));
+ as_slti(ScratchRegister, reg, 255);
+#ifdef MIPSR6
+ as_seleqz(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ as_selnez(reg, reg, ScratchRegister);
+ as_or(reg, reg, SecondScratchReg);
+#else
+ as_movz(reg, SecondScratchReg, ScratchRegister);
+#endif
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_MacroAssembler_mips_shared_inl_h */
diff --git a/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp b/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp
new file mode 100644
index 0000000000..0f52a28e43
--- /dev/null
+++ b/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp
@@ -0,0 +1,3355 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips-shared/MacroAssembler-mips-shared.h"
+
+#include "mozilla/EndianUtils.h"
+
+#include "jsmath.h"
+
+#include "jit/MacroAssembler.h"
+
+using namespace js;
+using namespace jit;
+
+void MacroAssemblerMIPSShared::ma_move(Register rd, Register rs) {
+ as_or(rd, rs, zero);
+}
+
+void MacroAssemblerMIPSShared::ma_li(Register dest, ImmGCPtr ptr) {
+ writeDataRelocation(ptr);
+ asMasm().ma_liPatchable(dest, ImmPtr(ptr.value));
+}
+
+void MacroAssemblerMIPSShared::ma_li(Register dest, Imm32 imm) {
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_addiu(dest, zero, imm.value);
+ } else if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_ori(dest, zero, Imm16::Lower(imm).encode());
+ } else if (Imm16::Lower(imm).encode() == 0) {
+ as_lui(dest, Imm16::Upper(imm).encode());
+ } else {
+ as_lui(dest, Imm16::Upper(imm).encode());
+ as_ori(dest, dest, Imm16::Lower(imm).encode());
+ }
+}
+
+// This method generates lui and ori instruction pair that can be modified by
+// UpdateLuiOriValue, either during compilation (eg. Assembler::bind), or
+// during execution (eg. jit::PatchJump).
+void MacroAssemblerMIPSShared::ma_liPatchable(Register dest, Imm32 imm) {
+ m_buffer.ensureSpace(2 * sizeof(uint32_t));
+ as_lui(dest, Imm16::Upper(imm).encode());
+ as_ori(dest, dest, Imm16::Lower(imm).encode());
+}
+
+// Shifts
+void MacroAssemblerMIPSShared::ma_sll(Register rd, Register rt, Imm32 shift) {
+ as_sll(rd, rt, shift.value % 32);
+}
+void MacroAssemblerMIPSShared::ma_srl(Register rd, Register rt, Imm32 shift) {
+ as_srl(rd, rt, shift.value % 32);
+}
+
+void MacroAssemblerMIPSShared::ma_sra(Register rd, Register rt, Imm32 shift) {
+ as_sra(rd, rt, shift.value % 32);
+}
+
+void MacroAssemblerMIPSShared::ma_ror(Register rd, Register rt, Imm32 shift) {
+ if (hasR2()) {
+ as_rotr(rd, rt, shift.value % 32);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ as_srl(scratch, rt, shift.value % 32);
+ as_sll(rd, rt, (32 - (shift.value % 32)) % 32);
+ as_or(rd, rd, scratch);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_rol(Register rd, Register rt, Imm32 shift) {
+ if (hasR2()) {
+ as_rotr(rd, rt, (32 - (shift.value % 32)) % 32);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ as_srl(scratch, rt, (32 - (shift.value % 32)) % 32);
+ as_sll(rd, rt, shift.value % 32);
+ as_or(rd, rd, scratch);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_sll(Register rd, Register rt,
+ Register shift) {
+ as_sllv(rd, rt, shift);
+}
+
+void MacroAssemblerMIPSShared::ma_srl(Register rd, Register rt,
+ Register shift) {
+ as_srlv(rd, rt, shift);
+}
+
+void MacroAssemblerMIPSShared::ma_sra(Register rd, Register rt,
+ Register shift) {
+ as_srav(rd, rt, shift);
+}
+
+void MacroAssemblerMIPSShared::ma_ror(Register rd, Register rt,
+ Register shift) {
+ if (hasR2()) {
+ as_rotrv(rd, rt, shift);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ ma_negu(scratch, shift);
+ as_sllv(scratch, rt, scratch);
+ as_srlv(rd, rt, shift);
+ as_or(rd, rd, scratch);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_rol(Register rd, Register rt,
+ Register shift) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_negu(scratch, shift);
+ if (hasR2()) {
+ as_rotrv(rd, rt, scratch);
+ } else {
+ as_srlv(rd, rt, scratch);
+ as_sllv(scratch, rt, shift);
+ as_or(rd, rd, scratch);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_negu(Register rd, Register rs) {
+ as_subu(rd, zero, rs);
+}
+
+void MacroAssemblerMIPSShared::ma_not(Register rd, Register rs) {
+ as_nor(rd, rs, zero);
+}
+
+// Bit extract/insert
+void MacroAssemblerMIPSShared::ma_ext(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ MOZ_ASSERT(pos < 32);
+ MOZ_ASSERT(pos + size < 33);
+
+ if (hasR2()) {
+ as_ext(rt, rs, pos, size);
+ } else {
+ int shift_left = 32 - (pos + size);
+ as_sll(rt, rs, shift_left);
+ int shift_right = 32 - size;
+ if (shift_right > 0) {
+ as_srl(rt, rt, shift_right);
+ }
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_ins(Register rt, Register rs, uint16_t pos,
+ uint16_t size) {
+ MOZ_ASSERT(pos < 32);
+ MOZ_ASSERT(pos + size <= 32);
+ MOZ_ASSERT(size != 0);
+
+ if (hasR2()) {
+ as_ins(rt, rs, pos, size);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_subu(scratch, zero, Imm32(1));
+ as_srl(scratch, scratch, 32 - size);
+ as_and(scratch2, rs, scratch);
+ as_sll(scratch2, scratch2, pos);
+ as_sll(scratch, scratch, pos);
+ as_nor(scratch, scratch, zero);
+ as_and(scratch, rt, scratch);
+ as_or(rt, scratch2, scratch);
+ }
+}
+
+// Sign extend
+void MacroAssemblerMIPSShared::ma_seb(Register rd, Register rt) {
+ if (hasR2()) {
+ as_seb(rd, rt);
+ } else {
+ as_sll(rd, rt, 24);
+ as_sra(rd, rd, 24);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_seh(Register rd, Register rt) {
+ if (hasR2()) {
+ as_seh(rd, rt);
+ } else {
+ as_sll(rd, rt, 16);
+ as_sra(rd, rd, 16);
+ }
+}
+
+// And.
+void MacroAssemblerMIPSShared::ma_and(Register rd, Register rs) {
+ as_and(rd, rd, rs);
+}
+
+void MacroAssemblerMIPSShared::ma_and(Register rd, Imm32 imm) {
+ ma_and(rd, rd, imm);
+}
+
+void MacroAssemblerMIPSShared::ma_and(Register rd, Register rs, Imm32 imm) {
+ if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_andi(rd, rs, imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_and(rd, rs, ScratchRegister);
+ }
+}
+
+// Or.
+void MacroAssemblerMIPSShared::ma_or(Register rd, Register rs) {
+ as_or(rd, rd, rs);
+}
+
+void MacroAssemblerMIPSShared::ma_or(Register rd, Imm32 imm) {
+ ma_or(rd, rd, imm);
+}
+
+void MacroAssemblerMIPSShared::ma_or(Register rd, Register rs, Imm32 imm) {
+ if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_ori(rd, rs, imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_or(rd, rs, ScratchRegister);
+ }
+}
+
+// xor
+void MacroAssemblerMIPSShared::ma_xor(Register rd, Register rs) {
+ as_xor(rd, rd, rs);
+}
+
+void MacroAssemblerMIPSShared::ma_xor(Register rd, Imm32 imm) {
+ ma_xor(rd, rd, imm);
+}
+
+void MacroAssemblerMIPSShared::ma_xor(Register rd, Register rs, Imm32 imm) {
+ if (Imm16::IsInUnsignedRange(imm.value)) {
+ as_xori(rd, rs, imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_xor(rd, rs, ScratchRegister);
+ }
+}
+
+// word swap bytes within halfwords
+void MacroAssemblerMIPSShared::ma_wsbh(Register rd, Register rt) {
+ as_wsbh(rd, rt);
+}
+
+void MacroAssemblerMIPSShared::ma_ctz(Register rd, Register rs) {
+ as_addiu(ScratchRegister, rs, -1);
+ as_xor(rd, ScratchRegister, rs);
+ as_and(rd, rd, ScratchRegister);
+ as_clz(rd, rd);
+ ma_li(ScratchRegister, Imm32(0x20));
+ as_subu(rd, ScratchRegister, rd);
+}
+
+// Arithmetic-based ops.
+
+// Add.
+void MacroAssemblerMIPSShared::ma_addu(Register rd, Register rs, Imm32 imm) {
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_addiu(rd, rs, imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_addu(rd, rs, ScratchRegister);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_addu(Register rd, Register rs) {
+ as_addu(rd, rd, rs);
+}
+
+void MacroAssemblerMIPSShared::ma_addu(Register rd, Imm32 imm) {
+ ma_addu(rd, rd, imm);
+}
+
+void MacroAssemblerMIPSShared::ma_add32TestCarry(Condition cond, Register rd,
+ Register rs, Register rt,
+ Label* overflow) {
+ MOZ_ASSERT(cond == Assembler::CarrySet || cond == Assembler::CarryClear);
+ MOZ_ASSERT_IF(rd == rs, rt != rd);
+ as_addu(rd, rs, rt);
+ as_sltu(SecondScratchReg, rd, rd == rs ? rt : rs);
+ ma_b(SecondScratchReg, SecondScratchReg, overflow,
+ cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
+}
+
+void MacroAssemblerMIPSShared::ma_add32TestCarry(Condition cond, Register rd,
+ Register rs, Imm32 imm,
+ Label* overflow) {
+ ma_li(ScratchRegister, imm);
+ ma_add32TestCarry(cond, rd, rs, ScratchRegister, overflow);
+}
+
+// Subtract.
+void MacroAssemblerMIPSShared::ma_subu(Register rd, Register rs, Imm32 imm) {
+ if (Imm16::IsInSignedRange(-imm.value)) {
+ as_addiu(rd, rs, -imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_subu(rd, rs, ScratchRegister);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_subu(Register rd, Imm32 imm) {
+ ma_subu(rd, rd, imm);
+}
+
+void MacroAssemblerMIPSShared::ma_subu(Register rd, Register rs) {
+ as_subu(rd, rd, rs);
+}
+
+void MacroAssemblerMIPSShared::ma_sub32TestOverflow(Register rd, Register rs,
+ Imm32 imm,
+ Label* overflow) {
+ if (imm.value != INT32_MIN) {
+ asMasm().ma_add32TestOverflow(rd, rs, Imm32(-imm.value), overflow);
+ } else {
+ ma_li(ScratchRegister, Imm32(imm.value));
+ asMasm().ma_sub32TestOverflow(rd, rs, ScratchRegister, overflow);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_mul(Register rd, Register rs, Imm32 imm) {
+ ma_li(ScratchRegister, imm);
+ as_mul(rd, rs, ScratchRegister);
+}
+
+void MacroAssemblerMIPSShared::ma_mul32TestOverflow(Register rd, Register rs,
+ Register rt,
+ Label* overflow) {
+#ifdef MIPSR6
+ if (rd == rs) {
+ ma_move(SecondScratchReg, rs);
+ rs = SecondScratchReg;
+ }
+ as_mul(rd, rs, rt);
+ as_muh(SecondScratchReg, rs, rt);
+#else
+ as_mult(rs, rt);
+ as_mflo(rd);
+ as_mfhi(SecondScratchReg);
+#endif
+ as_sra(ScratchRegister, rd, 31);
+ ma_b(ScratchRegister, SecondScratchReg, overflow, Assembler::NotEqual);
+}
+
+void MacroAssemblerMIPSShared::ma_mul32TestOverflow(Register rd, Register rs,
+ Imm32 imm,
+ Label* overflow) {
+ ma_li(ScratchRegister, imm);
+ ma_mul32TestOverflow(rd, rs, ScratchRegister, overflow);
+}
+
+void MacroAssemblerMIPSShared::ma_div_branch_overflow(Register rd, Register rs,
+ Register rt,
+ Label* overflow) {
+#ifdef MIPSR6
+ if (rd == rs) {
+ ma_move(SecondScratchReg, rs);
+ rs = SecondScratchReg;
+ }
+ as_mod(ScratchRegister, rs, rt);
+#else
+ as_div(rs, rt);
+ as_mfhi(ScratchRegister);
+#endif
+ ma_b(ScratchRegister, ScratchRegister, overflow, Assembler::NonZero);
+#ifdef MIPSR6
+ as_div(rd, rs, rt);
+#else
+ as_mflo(rd);
+#endif
+}
+
+void MacroAssemblerMIPSShared::ma_div_branch_overflow(Register rd, Register rs,
+ Imm32 imm,
+ Label* overflow) {
+ ma_li(ScratchRegister, imm);
+ ma_div_branch_overflow(rd, rs, ScratchRegister, overflow);
+}
+
+void MacroAssemblerMIPSShared::ma_mod_mask(Register src, Register dest,
+ Register hold, Register remain,
+ int32_t shift, Label* negZero) {
+ // MATH:
+ // We wish to compute x % (1<<y) - 1 for a known constant, y.
+ // First, let b = (1<<y) and C = (1<<y)-1, then think of the 32 bit
+ // dividend as a number in base b, namely
+ // c_0*1 + c_1*b + c_2*b^2 ... c_n*b^n
+ // now, since both addition and multiplication commute with modulus,
+ // x % C == (c_0 + c_1*b + ... + c_n*b^n) % C ==
+ // (c_0 % C) + (c_1%C) * (b % C) + (c_2 % C) * (b^2 % C)...
+ // now, since b == C + 1, b % C == 1, and b^n % C == 1
+ // this means that the whole thing simplifies to:
+ // c_0 + c_1 + c_2 ... c_n % C
+ // each c_n can easily be computed by a shift/bitextract, and the modulus
+ // can be maintained by simply subtracting by C whenever the number gets
+ // over C.
+ int32_t mask = (1 << shift) - 1;
+ Label head, negative, sumSigned, done;
+
+ // hold holds -1 if the value was negative, 1 otherwise.
+ // remain holds the remaining bits that have not been processed
+ // SecondScratchReg serves as a temporary location to store extracted bits
+ // into as well as holding the trial subtraction as a temp value dest is
+ // the accumulator (and holds the final result)
+
+ // move the whole value into the remain.
+ ma_move(remain, src);
+ // Zero out the dest.
+ ma_li(dest, Imm32(0));
+ // Set the hold appropriately.
+ ma_b(remain, remain, &negative, Signed, ShortJump);
+ ma_li(hold, Imm32(1));
+ ma_b(&head, ShortJump);
+
+ bind(&negative);
+ ma_li(hold, Imm32(-1));
+ ma_negu(remain, remain);
+
+ // Begin the main loop.
+ bind(&head);
+
+ // Extract the bottom bits into SecondScratchReg.
+ ma_and(SecondScratchReg, remain, Imm32(mask));
+ // Add those bits to the accumulator.
+ as_addu(dest, dest, SecondScratchReg);
+ // Do a trial subtraction
+ ma_subu(SecondScratchReg, dest, Imm32(mask));
+ // If (sum - C) > 0, store sum - C back into sum, thus performing a
+ // modulus.
+ ma_b(SecondScratchReg, SecondScratchReg, &sumSigned, Signed, ShortJump);
+ ma_move(dest, SecondScratchReg);
+ bind(&sumSigned);
+ // Get rid of the bits that we extracted before.
+ as_srl(remain, remain, shift);
+ // If the shift produced zero, finish, otherwise, continue in the loop.
+ ma_b(remain, remain, &head, NonZero, ShortJump);
+ // Check the hold to see if we need to negate the result.
+ ma_b(hold, hold, &done, NotSigned, ShortJump);
+
+ // If the hold was non-zero, negate the result to be in line with
+ // what JS wants
+ if (negZero != nullptr) {
+ // Jump out in case of negative zero.
+ ma_b(hold, hold, negZero, Zero);
+ ma_negu(dest, dest);
+ } else {
+ ma_negu(dest, dest);
+ }
+
+ bind(&done);
+}
+
+// Memory.
+
+void MacroAssemblerMIPSShared::ma_load(Register dest, const BaseIndex& src,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ if (isLoongson() && ZeroExtend != extension &&
+ Imm8::IsInSignedRange(src.offset)) {
+ Register index = src.index;
+
+ if (src.scale != TimesOne) {
+ int32_t shift = Imm32::ShiftOf(src.scale).value;
+
+ MOZ_ASSERT(SecondScratchReg != src.base);
+ index = SecondScratchReg;
+#ifdef JS_CODEGEN_MIPS64
+ asMasm().ma_dsll(index, src.index, Imm32(shift));
+#else
+ asMasm().ma_sll(index, src.index, Imm32(shift));
+#endif
+ }
+
+ switch (size) {
+ case SizeByte:
+ as_gslbx(dest, src.base, index, src.offset);
+ break;
+ case SizeHalfWord:
+ as_gslhx(dest, src.base, index, src.offset);
+ break;
+ case SizeWord:
+ as_gslwx(dest, src.base, index, src.offset);
+ break;
+ case SizeDouble:
+ as_gsldx(dest, src.base, index, src.offset);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+ return;
+ }
+
+ asMasm().computeScaledAddress(src, SecondScratchReg);
+ asMasm().ma_load(dest, Address(SecondScratchReg, src.offset), size,
+ extension);
+}
+
+void MacroAssemblerMIPSShared::ma_load_unaligned(Register dest,
+ const BaseIndex& src,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ int16_t lowOffset, hiOffset;
+ SecondScratchRegisterScope base(asMasm());
+ asMasm().computeScaledAddress(src, base);
+ ScratchRegisterScope scratch(asMasm());
+
+ if (Imm16::IsInSignedRange(src.offset) &&
+ Imm16::IsInSignedRange(src.offset + size / 8 - 1)) {
+ lowOffset = Imm16(src.offset).encode();
+ hiOffset = Imm16(src.offset + size / 8 - 1).encode();
+ } else {
+ ma_li(scratch, Imm32(src.offset));
+ asMasm().addPtr(scratch, base);
+ lowOffset = Imm16(0).encode();
+ hiOffset = Imm16(size / 8 - 1).encode();
+ }
+
+ switch (size) {
+ case SizeHalfWord:
+ MOZ_ASSERT(dest != scratch);
+ if (extension == ZeroExtend) {
+ as_lbu(scratch, base, hiOffset);
+ } else {
+ as_lb(scratch, base, hiOffset);
+ }
+ as_lbu(dest, base, lowOffset);
+ ma_ins(dest, scratch, 8, 24);
+ break;
+ case SizeWord:
+ MOZ_ASSERT(dest != base);
+ as_lwl(dest, base, hiOffset);
+ as_lwr(dest, base, lowOffset);
+#ifdef JS_CODEGEN_MIPS64
+ if (extension == ZeroExtend) {
+ as_dext(dest, dest, 0, 32);
+ }
+#endif
+ break;
+#ifdef JS_CODEGEN_MIPS64
+ case SizeDouble:
+ MOZ_ASSERT(dest != base);
+ as_ldl(dest, base, hiOffset);
+ as_ldr(dest, base, lowOffset);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Invalid argument for ma_load_unaligned");
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_load_unaligned(Register dest,
+ const Address& address,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ int16_t lowOffset, hiOffset;
+ ScratchRegisterScope scratch1(asMasm());
+ SecondScratchRegisterScope scratch2(asMasm());
+ Register base;
+
+ if (Imm16::IsInSignedRange(address.offset) &&
+ Imm16::IsInSignedRange(address.offset + size / 8 - 1)) {
+ base = address.base;
+ lowOffset = Imm16(address.offset).encode();
+ hiOffset = Imm16(address.offset + size / 8 - 1).encode();
+ } else {
+ ma_li(scratch1, Imm32(address.offset));
+ asMasm().addPtr(address.base, scratch1);
+ base = scratch1;
+ lowOffset = Imm16(0).encode();
+ hiOffset = Imm16(size / 8 - 1).encode();
+ }
+
+ switch (size) {
+ case SizeHalfWord:
+ MOZ_ASSERT(base != scratch2 && dest != scratch2);
+ if (extension == ZeroExtend) {
+ as_lbu(scratch2, base, hiOffset);
+ } else {
+ as_lb(scratch2, base, hiOffset);
+ }
+ as_lbu(dest, base, lowOffset);
+ ma_ins(dest, scratch2, 8, 24);
+ break;
+ case SizeWord:
+ MOZ_ASSERT(dest != base);
+ as_lwl(dest, base, hiOffset);
+ as_lwr(dest, base, lowOffset);
+#ifdef JS_CODEGEN_MIPS64
+ if (extension == ZeroExtend) {
+ as_dext(dest, dest, 0, 32);
+ }
+#endif
+ break;
+#ifdef JS_CODEGEN_MIPS64
+ case SizeDouble:
+ MOZ_ASSERT(dest != base);
+ as_ldl(dest, base, hiOffset);
+ as_ldr(dest, base, lowOffset);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Invalid argument for ma_load_unaligned");
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_load_unaligned(
+ const wasm::MemoryAccessDesc& access, Register dest, const BaseIndex& src,
+ Register temp, LoadStoreSize size, LoadStoreExtension extension) {
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Wasm-only; wasm is disabled on big-endian.");
+ int16_t lowOffset, hiOffset;
+ Register base;
+
+ asMasm().computeScaledAddress(src, SecondScratchReg);
+
+ if (Imm16::IsInSignedRange(src.offset) &&
+ Imm16::IsInSignedRange(src.offset + size / 8 - 1)) {
+ base = SecondScratchReg;
+ lowOffset = Imm16(src.offset).encode();
+ hiOffset = Imm16(src.offset + size / 8 - 1).encode();
+ } else {
+ ma_li(ScratchRegister, Imm32(src.offset));
+ asMasm().addPtr(SecondScratchReg, ScratchRegister);
+ base = ScratchRegister;
+ lowOffset = Imm16(0).encode();
+ hiOffset = Imm16(size / 8 - 1).encode();
+ }
+
+ BufferOffset load;
+ switch (size) {
+ case SizeHalfWord:
+ if (extension == ZeroExtend) {
+ load = as_lbu(temp, base, hiOffset);
+ } else {
+ load = as_lb(temp, base, hiOffset);
+ }
+ as_lbu(dest, base, lowOffset);
+ ma_ins(dest, temp, 8, 24);
+ break;
+ case SizeWord:
+ load = as_lwl(dest, base, hiOffset);
+ as_lwr(dest, base, lowOffset);
+#ifdef JS_CODEGEN_MIPS64
+ if (extension == ZeroExtend) {
+ as_dext(dest, dest, 0, 32);
+ }
+#endif
+ break;
+#ifdef JS_CODEGEN_MIPS64
+ case SizeDouble:
+ load = as_ldl(dest, base, hiOffset);
+ as_ldr(dest, base, lowOffset);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+
+ append(access, load.getOffset());
+}
+
+void MacroAssemblerMIPSShared::ma_store(Register data, const BaseIndex& dest,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ if (isLoongson() && Imm8::IsInSignedRange(dest.offset)) {
+ Register index = dest.index;
+
+ if (dest.scale != TimesOne) {
+ int32_t shift = Imm32::ShiftOf(dest.scale).value;
+
+ MOZ_ASSERT(SecondScratchReg != dest.base);
+ index = SecondScratchReg;
+#ifdef JS_CODEGEN_MIPS64
+ asMasm().ma_dsll(index, dest.index, Imm32(shift));
+#else
+ asMasm().ma_sll(index, dest.index, Imm32(shift));
+#endif
+ }
+
+ switch (size) {
+ case SizeByte:
+ as_gssbx(data, dest.base, index, dest.offset);
+ break;
+ case SizeHalfWord:
+ as_gsshx(data, dest.base, index, dest.offset);
+ break;
+ case SizeWord:
+ as_gsswx(data, dest.base, index, dest.offset);
+ break;
+ case SizeDouble:
+ as_gssdx(data, dest.base, index, dest.offset);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+ return;
+ }
+
+ asMasm().computeScaledAddress(dest, SecondScratchReg);
+ asMasm().ma_store(data, Address(SecondScratchReg, dest.offset), size,
+ extension);
+}
+
+void MacroAssemblerMIPSShared::ma_store(Imm32 imm, const BaseIndex& dest,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ if (isLoongson() && Imm8::IsInSignedRange(dest.offset)) {
+ Register data = zero;
+ Register index = dest.index;
+
+ if (imm.value) {
+ MOZ_ASSERT(ScratchRegister != dest.base);
+ MOZ_ASSERT(ScratchRegister != dest.index);
+ data = ScratchRegister;
+ ma_li(data, imm);
+ }
+
+ if (dest.scale != TimesOne) {
+ int32_t shift = Imm32::ShiftOf(dest.scale).value;
+
+ MOZ_ASSERT(SecondScratchReg != dest.base);
+ index = SecondScratchReg;
+#ifdef JS_CODEGEN_MIPS64
+ asMasm().ma_dsll(index, dest.index, Imm32(shift));
+#else
+ asMasm().ma_sll(index, dest.index, Imm32(shift));
+#endif
+ }
+
+ switch (size) {
+ case SizeByte:
+ as_gssbx(data, dest.base, index, dest.offset);
+ break;
+ case SizeHalfWord:
+ as_gsshx(data, dest.base, index, dest.offset);
+ break;
+ case SizeWord:
+ as_gsswx(data, dest.base, index, dest.offset);
+ break;
+ case SizeDouble:
+ as_gssdx(data, dest.base, index, dest.offset);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+ return;
+ }
+
+ // Make sure that SecondScratchReg contains absolute address so that
+ // offset is 0.
+ asMasm().computeEffectiveAddress(dest, SecondScratchReg);
+
+ // Scrach register is free now, use it for loading imm value
+ ma_li(ScratchRegister, imm);
+
+ // with offset=0 ScratchRegister will not be used in ma_store()
+ // so we can use it as a parameter here
+ asMasm().ma_store(ScratchRegister, Address(SecondScratchReg, 0), size,
+ extension);
+}
+
+void MacroAssemblerMIPSShared::ma_store_unaligned(Register data,
+ const Address& address,
+ LoadStoreSize size) {
+ int16_t lowOffset, hiOffset;
+ ScratchRegisterScope scratch(asMasm());
+ Register base;
+
+ if (Imm16::IsInSignedRange(address.offset) &&
+ Imm16::IsInSignedRange(address.offset + size / 8 - 1)) {
+ base = address.base;
+ lowOffset = Imm16(address.offset).encode();
+ hiOffset = Imm16(address.offset + size / 8 - 1).encode();
+ } else {
+ ma_li(scratch, Imm32(address.offset));
+ asMasm().addPtr(address.base, scratch);
+ base = scratch;
+ lowOffset = Imm16(0).encode();
+ hiOffset = Imm16(size / 8 - 1).encode();
+ }
+
+ switch (size) {
+ case SizeHalfWord: {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(base != scratch2);
+ as_sb(data, base, lowOffset);
+ ma_ext(scratch2, data, 8, 8);
+ as_sb(scratch2, base, hiOffset);
+ break;
+ }
+ case SizeWord:
+ as_swl(data, base, hiOffset);
+ as_swr(data, base, lowOffset);
+ break;
+#ifdef JS_CODEGEN_MIPS64
+ case SizeDouble:
+ as_sdl(data, base, hiOffset);
+ as_sdr(data, base, lowOffset);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Invalid argument for ma_store_unaligned");
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_store_unaligned(Register data,
+ const BaseIndex& dest,
+ LoadStoreSize size) {
+ int16_t lowOffset, hiOffset;
+ SecondScratchRegisterScope base(asMasm());
+ asMasm().computeScaledAddress(dest, base);
+ ScratchRegisterScope scratch(asMasm());
+
+ if (Imm16::IsInSignedRange(dest.offset) &&
+ Imm16::IsInSignedRange(dest.offset + size / 8 - 1)) {
+ lowOffset = Imm16(dest.offset).encode();
+ hiOffset = Imm16(dest.offset + size / 8 - 1).encode();
+ } else {
+ ma_li(scratch, Imm32(dest.offset));
+ asMasm().addPtr(scratch, base);
+ lowOffset = Imm16(0).encode();
+ hiOffset = Imm16(size / 8 - 1).encode();
+ }
+
+ switch (size) {
+ case SizeHalfWord:
+ MOZ_ASSERT(base != scratch);
+ as_sb(data, base, lowOffset);
+ ma_ext(scratch, data, 8, 8);
+ as_sb(scratch, base, hiOffset);
+ break;
+ case SizeWord:
+ as_swl(data, base, hiOffset);
+ as_swr(data, base, lowOffset);
+ break;
+#ifdef JS_CODEGEN_MIPS64
+ case SizeDouble:
+ as_sdl(data, base, hiOffset);
+ as_sdr(data, base, lowOffset);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Invalid argument for ma_store_unaligned");
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_store_unaligned(
+ const wasm::MemoryAccessDesc& access, Register data, const BaseIndex& dest,
+ Register temp, LoadStoreSize size, LoadStoreExtension extension) {
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Wasm-only; wasm is disabled on big-endian.");
+ int16_t lowOffset, hiOffset;
+ Register base;
+
+ asMasm().computeScaledAddress(dest, SecondScratchReg);
+
+ if (Imm16::IsInSignedRange(dest.offset) &&
+ Imm16::IsInSignedRange(dest.offset + size / 8 - 1)) {
+ base = SecondScratchReg;
+ lowOffset = Imm16(dest.offset).encode();
+ hiOffset = Imm16(dest.offset + size / 8 - 1).encode();
+ } else {
+ ma_li(ScratchRegister, Imm32(dest.offset));
+ asMasm().addPtr(SecondScratchReg, ScratchRegister);
+ base = ScratchRegister;
+ lowOffset = Imm16(0).encode();
+ hiOffset = Imm16(size / 8 - 1).encode();
+ }
+
+ BufferOffset store;
+ switch (size) {
+ case SizeHalfWord:
+ ma_ext(temp, data, 8, 8);
+ store = as_sb(temp, base, hiOffset);
+ as_sb(data, base, lowOffset);
+ break;
+ case SizeWord:
+ store = as_swl(data, base, hiOffset);
+ as_swr(data, base, lowOffset);
+ break;
+#ifdef JS_CODEGEN_MIPS64
+ case SizeDouble:
+ store = as_sdl(data, base, hiOffset);
+ as_sdr(data, base, lowOffset);
+ break;
+#endif
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+ append(access, store.getOffset());
+}
+
+// Branches when done from within mips-specific code.
+void MacroAssemblerMIPSShared::ma_b(Register lhs, Register rhs, Label* label,
+ Condition c, JumpKind jumpKind) {
+ switch (c) {
+ case Equal:
+ case NotEqual:
+ asMasm().branchWithCode(getBranchCode(lhs, rhs, c), label, jumpKind);
+ break;
+ case Always:
+ ma_b(label, jumpKind);
+ break;
+ case Zero:
+ case NonZero:
+ case Signed:
+ case NotSigned:
+ MOZ_ASSERT(lhs == rhs);
+ asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind);
+ break;
+ default:
+ Condition cond = ma_cmp(ScratchRegister, lhs, rhs, c);
+ asMasm().branchWithCode(getBranchCode(ScratchRegister, cond), label,
+ jumpKind);
+ break;
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_b(Register lhs, Imm32 imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ MOZ_ASSERT(c != Overflow);
+ if (imm.value == 0) {
+ if (c == Always || c == AboveOrEqual) {
+ ma_b(label, jumpKind);
+ } else if (c == Below) {
+ ; // This condition is always false. No branch required.
+ } else {
+ asMasm().branchWithCode(getBranchCode(lhs, c), label, jumpKind);
+ }
+ } else {
+ switch (c) {
+ case Equal:
+ case NotEqual:
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_b(lhs, ScratchRegister, label, c, jumpKind);
+ break;
+ default:
+ Condition cond = ma_cmp(ScratchRegister, lhs, imm, c);
+ asMasm().branchWithCode(getBranchCode(ScratchRegister, cond), label,
+ jumpKind);
+ }
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_b(Register lhs, ImmPtr imm, Label* l,
+ Condition c, JumpKind jumpKind) {
+ asMasm().ma_b(lhs, ImmWord(uintptr_t(imm.value)), l, c, jumpKind);
+}
+
+void MacroAssemblerMIPSShared::ma_b(Label* label, JumpKind jumpKind) {
+ asMasm().branchWithCode(getBranchCode(BranchIsJump), label, jumpKind);
+}
+
+Assembler::Condition MacroAssemblerMIPSShared::ma_cmp(Register dest,
+ Register lhs,
+ Register rhs,
+ Condition c) {
+ switch (c) {
+ case Above:
+ // bgtu s,t,label =>
+ // sltu at,t,s
+ // bne at,$zero,offs
+ as_sltu(dest, rhs, lhs);
+ return NotEqual;
+ case AboveOrEqual:
+ // bgeu s,t,label =>
+ // sltu at,s,t
+ // beq at,$zero,offs
+ as_sltu(dest, lhs, rhs);
+ return Equal;
+ case Below:
+ // bltu s,t,label =>
+ // sltu at,s,t
+ // bne at,$zero,offs
+ as_sltu(dest, lhs, rhs);
+ return NotEqual;
+ case BelowOrEqual:
+ // bleu s,t,label =>
+ // sltu at,t,s
+ // beq at,$zero,offs
+ as_sltu(dest, rhs, lhs);
+ return Equal;
+ case GreaterThan:
+ // bgt s,t,label =>
+ // slt at,t,s
+ // bne at,$zero,offs
+ as_slt(dest, rhs, lhs);
+ return NotEqual;
+ case GreaterThanOrEqual:
+ // bge s,t,label =>
+ // slt at,s,t
+ // beq at,$zero,offs
+ as_slt(dest, lhs, rhs);
+ return Equal;
+ case LessThan:
+ // blt s,t,label =>
+ // slt at,s,t
+ // bne at,$zero,offs
+ as_slt(dest, lhs, rhs);
+ return NotEqual;
+ case LessThanOrEqual:
+ // ble s,t,label =>
+ // slt at,t,s
+ // beq at,$zero,offs
+ as_slt(dest, rhs, lhs);
+ return Equal;
+ default:
+ MOZ_CRASH("Invalid condition.");
+ }
+ return Always;
+}
+
+Assembler::Condition MacroAssemblerMIPSShared::ma_cmp(Register dest,
+ Register lhs, Imm32 imm,
+ Condition c) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+
+ switch (c) {
+ case Above:
+ case BelowOrEqual:
+ if (Imm16::IsInSignedRange(imm.value + 1) && imm.value != -1) {
+ // lhs <= rhs via lhs < rhs + 1 if rhs + 1 does not overflow
+ as_sltiu(dest, lhs, imm.value + 1);
+
+ return (c == BelowOrEqual ? NotEqual : Equal);
+ } else {
+ ma_li(scratch, imm);
+ as_sltu(dest, scratch, lhs);
+ return (c == BelowOrEqual ? Equal : NotEqual);
+ }
+ case AboveOrEqual:
+ case Below:
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_sltiu(dest, lhs, imm.value);
+ } else {
+ ma_li(scratch, imm);
+ as_sltu(dest, lhs, scratch);
+ }
+ return (c == AboveOrEqual ? Equal : NotEqual);
+ case GreaterThan:
+ case LessThanOrEqual:
+ if (Imm16::IsInSignedRange(imm.value + 1)) {
+ // lhs <= rhs via lhs < rhs + 1.
+ as_slti(dest, lhs, imm.value + 1);
+ return (c == LessThanOrEqual ? NotEqual : Equal);
+ } else {
+ ma_li(scratch, imm);
+ as_slt(dest, scratch, lhs);
+ return (c == LessThanOrEqual ? Equal : NotEqual);
+ }
+ case GreaterThanOrEqual:
+ case LessThan:
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_slti(dest, lhs, imm.value);
+ } else {
+ ma_li(scratch, imm);
+ as_slt(dest, lhs, scratch);
+ }
+ return (c == GreaterThanOrEqual ? Equal : NotEqual);
+ default:
+ MOZ_CRASH("Invalid condition.");
+ }
+ return Always;
+}
+
+void MacroAssemblerMIPSShared::ma_cmp_set(Register rd, Register rs, Register rt,
+ Condition c) {
+ switch (c) {
+ case Equal:
+ // seq d,s,t =>
+ // xor d,s,t
+ // sltiu d,d,1
+ as_xor(rd, rs, rt);
+ as_sltiu(rd, rd, 1);
+ break;
+ case NotEqual:
+ // sne d,s,t =>
+ // xor d,s,t
+ // sltu d,$zero,d
+ as_xor(rd, rs, rt);
+ as_sltu(rd, zero, rd);
+ break;
+ case Above:
+ // sgtu d,s,t =>
+ // sltu d,t,s
+ as_sltu(rd, rt, rs);
+ break;
+ case AboveOrEqual:
+ // sgeu d,s,t =>
+ // sltu d,s,t
+ // xori d,d,1
+ as_sltu(rd, rs, rt);
+ as_xori(rd, rd, 1);
+ break;
+ case Below:
+ // sltu d,s,t
+ as_sltu(rd, rs, rt);
+ break;
+ case BelowOrEqual:
+ // sleu d,s,t =>
+ // sltu d,t,s
+ // xori d,d,1
+ as_sltu(rd, rt, rs);
+ as_xori(rd, rd, 1);
+ break;
+ case GreaterThan:
+ // sgt d,s,t =>
+ // slt d,t,s
+ as_slt(rd, rt, rs);
+ break;
+ case GreaterThanOrEqual:
+ // sge d,s,t =>
+ // slt d,s,t
+ // xori d,d,1
+ as_slt(rd, rs, rt);
+ as_xori(rd, rd, 1);
+ break;
+ case LessThan:
+ // slt d,s,t
+ as_slt(rd, rs, rt);
+ break;
+ case LessThanOrEqual:
+ // sle d,s,t =>
+ // slt d,t,s
+ // xori d,d,1
+ as_slt(rd, rt, rs);
+ as_xori(rd, rd, 1);
+ break;
+ case Zero:
+ MOZ_ASSERT(rs == rt);
+ // seq d,s,$zero =>
+ // sltiu d,s,1
+ as_sltiu(rd, rs, 1);
+ break;
+ case NonZero:
+ MOZ_ASSERT(rs == rt);
+ // sne d,s,$zero =>
+ // sltu d,$zero,s
+ as_sltu(rd, zero, rs);
+ break;
+ case Signed:
+ MOZ_ASSERT(rs == rt);
+ as_slt(rd, rs, zero);
+ break;
+ case NotSigned:
+ MOZ_ASSERT(rs == rt);
+ // sge d,s,$zero =>
+ // slt d,s,$zero
+ // xori d,d,1
+ as_slt(rd, rs, zero);
+ as_xori(rd, rd, 1);
+ break;
+ default:
+ MOZ_CRASH("Invalid condition.");
+ }
+}
+
+void MacroAssemblerMIPSShared::compareFloatingPoint(
+ FloatFormat fmt, FloatRegister lhs, FloatRegister rhs, DoubleCondition c,
+ FloatTestKind* testKind, FPConditionBit fcc) {
+ switch (c) {
+ case DoubleOrdered:
+ as_cun(fmt, lhs, rhs, fcc);
+ *testKind = TestForFalse;
+ break;
+ case DoubleEqual:
+ as_ceq(fmt, lhs, rhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleNotEqual:
+ as_cueq(fmt, lhs, rhs, fcc);
+ *testKind = TestForFalse;
+ break;
+ case DoubleGreaterThan:
+ as_colt(fmt, rhs, lhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleGreaterThanOrEqual:
+ as_cole(fmt, rhs, lhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleLessThan:
+ as_colt(fmt, lhs, rhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleLessThanOrEqual:
+ as_cole(fmt, lhs, rhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleUnordered:
+ as_cun(fmt, lhs, rhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleEqualOrUnordered:
+ as_cueq(fmt, lhs, rhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleNotEqualOrUnordered:
+ as_ceq(fmt, lhs, rhs, fcc);
+ *testKind = TestForFalse;
+ break;
+ case DoubleGreaterThanOrUnordered:
+ as_cult(fmt, rhs, lhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleGreaterThanOrEqualOrUnordered:
+ as_cule(fmt, rhs, lhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleLessThanOrUnordered:
+ as_cult(fmt, lhs, rhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ case DoubleLessThanOrEqualOrUnordered:
+ as_cule(fmt, lhs, rhs, fcc);
+ *testKind = TestForTrue;
+ break;
+ default:
+ MOZ_CRASH("Invalid DoubleCondition.");
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_cmp_set_double(Register dest,
+ FloatRegister lhs,
+ FloatRegister rhs,
+ DoubleCondition c) {
+ FloatTestKind moveCondition;
+ compareFloatingPoint(DoubleFloat, lhs, rhs, c, &moveCondition);
+
+#ifdef MIPSR6
+ as_mfc1(dest, FloatRegisters::f24);
+ if (moveCondition == TestForTrue) {
+ as_andi(dest, dest, 0x1);
+ } else {
+ as_addiu(dest, dest, 0x1);
+ }
+#else
+ ma_li(dest, Imm32(1));
+
+ if (moveCondition == TestForTrue) {
+ as_movf(dest, zero);
+ } else {
+ as_movt(dest, zero);
+ }
+#endif
+}
+
+void MacroAssemblerMIPSShared::ma_cmp_set_float32(Register dest,
+ FloatRegister lhs,
+ FloatRegister rhs,
+ DoubleCondition c) {
+ FloatTestKind moveCondition;
+ compareFloatingPoint(SingleFloat, lhs, rhs, c, &moveCondition);
+
+#ifdef MIPSR6
+ as_mfc1(dest, FloatRegisters::f24);
+ if (moveCondition == TestForTrue) {
+ as_andi(dest, dest, 0x1);
+ } else {
+ as_addiu(dest, dest, 0x1);
+ }
+#else
+ ma_li(dest, Imm32(1));
+
+ if (moveCondition == TestForTrue) {
+ as_movf(dest, zero);
+ } else {
+ as_movt(dest, zero);
+ }
+#endif
+}
+
+void MacroAssemblerMIPSShared::ma_cmp_set(Register rd, Register rs, Imm32 imm,
+ Condition c) {
+ if (imm.value == 0) {
+ switch (c) {
+ case Equal:
+ case BelowOrEqual:
+ as_sltiu(rd, rs, 1);
+ break;
+ case NotEqual:
+ case Above:
+ as_sltu(rd, zero, rs);
+ break;
+ case AboveOrEqual:
+ case Below:
+ as_ori(rd, zero, c == AboveOrEqual ? 1 : 0);
+ break;
+ case GreaterThan:
+ case LessThanOrEqual:
+ as_slt(rd, zero, rs);
+ if (c == LessThanOrEqual) {
+ as_xori(rd, rd, 1);
+ }
+ break;
+ case LessThan:
+ case GreaterThanOrEqual:
+ as_slt(rd, rs, zero);
+ if (c == GreaterThanOrEqual) {
+ as_xori(rd, rd, 1);
+ }
+ break;
+ case Zero:
+ as_sltiu(rd, rs, 1);
+ break;
+ case NonZero:
+ as_sltu(rd, zero, rs);
+ break;
+ case Signed:
+ as_slt(rd, rs, zero);
+ break;
+ case NotSigned:
+ as_slt(rd, rs, zero);
+ as_xori(rd, rd, 1);
+ break;
+ default:
+ MOZ_CRASH("Invalid condition.");
+ }
+ return;
+ }
+
+ switch (c) {
+ case Equal:
+ case NotEqual:
+ MOZ_ASSERT(rs != ScratchRegister);
+ ma_xor(rd, rs, imm);
+ if (c == Equal) {
+ as_sltiu(rd, rd, 1);
+ } else {
+ as_sltu(rd, zero, rd);
+ }
+ break;
+ case Zero:
+ case NonZero:
+ case Signed:
+ case NotSigned:
+ MOZ_CRASH("Invalid condition.");
+ default:
+ Condition cond = ma_cmp(rd, rs, imm, c);
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+
+ if (cond == Equal) as_xori(rd, rd, 1);
+ }
+}
+
+// fp instructions
+void MacroAssemblerMIPSShared::ma_lis(FloatRegister dest, float value) {
+ Imm32 imm(mozilla::BitwiseCast<uint32_t>(value));
+
+ if (imm.value != 0) {
+ ma_li(ScratchRegister, imm);
+ moveToFloat32(ScratchRegister, dest);
+ } else {
+ moveToFloat32(zero, dest);
+ }
+}
+
+void MacroAssemblerMIPSShared::ma_sd(FloatRegister ft, BaseIndex address) {
+ if (isLoongson() && Imm8::IsInSignedRange(address.offset)) {
+ Register index = address.index;
+
+ if (address.scale != TimesOne) {
+ int32_t shift = Imm32::ShiftOf(address.scale).value;
+
+ MOZ_ASSERT(SecondScratchReg != address.base);
+ index = SecondScratchReg;
+#ifdef JS_CODEGEN_MIPS64
+ asMasm().ma_dsll(index, address.index, Imm32(shift));
+#else
+ asMasm().ma_sll(index, address.index, Imm32(shift));
+#endif
+ }
+
+ as_gssdx(ft, address.base, index, address.offset);
+ return;
+ }
+
+ asMasm().computeScaledAddress(address, SecondScratchReg);
+ asMasm().ma_sd(ft, Address(SecondScratchReg, address.offset));
+}
+
+void MacroAssemblerMIPSShared::ma_ss(FloatRegister ft, BaseIndex address) {
+ if (isLoongson() && Imm8::IsInSignedRange(address.offset)) {
+ Register index = address.index;
+
+ if (address.scale != TimesOne) {
+ int32_t shift = Imm32::ShiftOf(address.scale).value;
+
+ MOZ_ASSERT(SecondScratchReg != address.base);
+ index = SecondScratchReg;
+#ifdef JS_CODEGEN_MIPS64
+ asMasm().ma_dsll(index, address.index, Imm32(shift));
+#else
+ asMasm().ma_sll(index, address.index, Imm32(shift));
+#endif
+ }
+
+ as_gsssx(ft, address.base, index, address.offset);
+ return;
+ }
+
+ asMasm().computeScaledAddress(address, SecondScratchReg);
+ asMasm().ma_ss(ft, Address(SecondScratchReg, address.offset));
+}
+
+void MacroAssemblerMIPSShared::ma_ld(FloatRegister ft, const BaseIndex& src) {
+ asMasm().computeScaledAddress(src, SecondScratchReg);
+ asMasm().ma_ld(ft, Address(SecondScratchReg, src.offset));
+}
+
+void MacroAssemblerMIPSShared::ma_ls(FloatRegister ft, const BaseIndex& src) {
+ asMasm().computeScaledAddress(src, SecondScratchReg);
+ asMasm().ma_ls(ft, Address(SecondScratchReg, src.offset));
+}
+
+void MacroAssemblerMIPSShared::ma_bc1s(FloatRegister lhs, FloatRegister rhs,
+ Label* label, DoubleCondition c,
+ JumpKind jumpKind, FPConditionBit fcc) {
+ FloatTestKind testKind;
+ compareFloatingPoint(SingleFloat, lhs, rhs, c, &testKind, fcc);
+ asMasm().branchWithCode(getBranchCode(testKind, fcc), label, jumpKind);
+}
+
+void MacroAssemblerMIPSShared::ma_bc1d(FloatRegister lhs, FloatRegister rhs,
+ Label* label, DoubleCondition c,
+ JumpKind jumpKind, FPConditionBit fcc) {
+ FloatTestKind testKind;
+ compareFloatingPoint(DoubleFloat, lhs, rhs, c, &testKind, fcc);
+ asMasm().branchWithCode(getBranchCode(testKind, fcc), label, jumpKind);
+}
+
+void MacroAssemblerMIPSShared::minMaxDouble(FloatRegister srcDest,
+ FloatRegister second,
+ bool handleNaN, bool isMax) {
+ FloatRegister first = srcDest;
+
+ Assembler::DoubleCondition cond = isMax ? Assembler::DoubleLessThanOrEqual
+ : Assembler::DoubleGreaterThanOrEqual;
+ Label nan, equal, done;
+ FloatTestKind moveCondition;
+
+ // First or second is NaN, result is NaN.
+ ma_bc1d(first, second, &nan, Assembler::DoubleUnordered, ShortJump);
+#ifdef MIPSR6
+ if (isMax) {
+ as_max(DoubleFloat, srcDest, first, second);
+ } else {
+ as_min(DoubleFloat, srcDest, first, second);
+ }
+#else
+ // Make sure we handle -0 and 0 right.
+ ma_bc1d(first, second, &equal, Assembler::DoubleEqual, ShortJump);
+ compareFloatingPoint(DoubleFloat, first, second, cond, &moveCondition);
+ MOZ_ASSERT(TestForTrue == moveCondition);
+ as_movt(DoubleFloat, first, second);
+ ma_b(&done, ShortJump);
+
+ // Check for zero.
+ bind(&equal);
+ asMasm().loadConstantDouble(0.0, ScratchDoubleReg);
+ compareFloatingPoint(DoubleFloat, first, ScratchDoubleReg,
+ Assembler::DoubleEqual, &moveCondition);
+
+ // So now both operands are either -0 or 0.
+ if (isMax) {
+ // -0 + -0 = -0 and -0 + 0 = 0.
+ as_addd(ScratchDoubleReg, first, second);
+ } else {
+ as_negd(ScratchDoubleReg, first);
+ as_subd(ScratchDoubleReg, ScratchDoubleReg, second);
+ as_negd(ScratchDoubleReg, ScratchDoubleReg);
+ }
+ MOZ_ASSERT(TestForTrue == moveCondition);
+ // First is 0 or -0, move max/min to it, else just return it.
+ as_movt(DoubleFloat, first, ScratchDoubleReg);
+#endif
+ ma_b(&done, ShortJump);
+
+ bind(&nan);
+ asMasm().loadConstantDouble(JS::GenericNaN(), srcDest);
+
+ bind(&done);
+}
+
+void MacroAssemblerMIPSShared::minMaxFloat32(FloatRegister srcDest,
+ FloatRegister second,
+ bool handleNaN, bool isMax) {
+ FloatRegister first = srcDest;
+
+ Assembler::DoubleCondition cond = isMax ? Assembler::DoubleLessThanOrEqual
+ : Assembler::DoubleGreaterThanOrEqual;
+ Label nan, equal, done;
+ FloatTestKind moveCondition;
+
+ // First or second is NaN, result is NaN.
+ ma_bc1s(first, second, &nan, Assembler::DoubleUnordered, ShortJump);
+#ifdef MIPSR6
+ if (isMax) {
+ as_max(SingleFloat, srcDest, first, second);
+ } else {
+ as_min(SingleFloat, srcDest, first, second);
+ }
+#else
+ // Make sure we handle -0 and 0 right.
+ ma_bc1s(first, second, &equal, Assembler::DoubleEqual, ShortJump);
+ compareFloatingPoint(SingleFloat, first, second, cond, &moveCondition);
+ MOZ_ASSERT(TestForTrue == moveCondition);
+ as_movt(SingleFloat, first, second);
+ ma_b(&done, ShortJump);
+
+ // Check for zero.
+ bind(&equal);
+ asMasm().loadConstantFloat32(0.0f, ScratchFloat32Reg);
+ compareFloatingPoint(SingleFloat, first, ScratchFloat32Reg,
+ Assembler::DoubleEqual, &moveCondition);
+
+ // So now both operands are either -0 or 0.
+ if (isMax) {
+ // -0 + -0 = -0 and -0 + 0 = 0.
+ as_adds(ScratchFloat32Reg, first, second);
+ } else {
+ as_negs(ScratchFloat32Reg, first);
+ as_subs(ScratchFloat32Reg, ScratchFloat32Reg, second);
+ as_negs(ScratchFloat32Reg, ScratchFloat32Reg);
+ }
+ MOZ_ASSERT(TestForTrue == moveCondition);
+ // First is 0 or -0, move max/min to it, else just return it.
+ as_movt(SingleFloat, first, ScratchFloat32Reg);
+#endif
+ ma_b(&done, ShortJump);
+
+ bind(&nan);
+ asMasm().loadConstantFloat32(JS::GenericNaN(), srcDest);
+
+ bind(&done);
+}
+
+void MacroAssemblerMIPSShared::loadDouble(const Address& address,
+ FloatRegister dest) {
+ asMasm().ma_ld(dest, address);
+}
+
+void MacroAssemblerMIPSShared::loadDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ asMasm().ma_ld(dest, src);
+}
+
+void MacroAssemblerMIPSShared::loadFloatAsDouble(const Address& address,
+ FloatRegister dest) {
+ asMasm().ma_ls(dest, address);
+ as_cvtds(dest, dest);
+}
+
+void MacroAssemblerMIPSShared::loadFloatAsDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ asMasm().loadFloat32(src, dest);
+ as_cvtds(dest, dest);
+}
+
+void MacroAssemblerMIPSShared::loadFloat32(const Address& address,
+ FloatRegister dest) {
+ asMasm().ma_ls(dest, address);
+}
+
+void MacroAssemblerMIPSShared::loadFloat32(const BaseIndex& src,
+ FloatRegister dest) {
+ asMasm().ma_ls(dest, src);
+}
+
+void MacroAssemblerMIPSShared::ma_call(ImmPtr dest) {
+ asMasm().ma_liPatchable(CallReg, dest);
+ as_jalr(CallReg);
+ as_nop();
+}
+
+void MacroAssemblerMIPSShared::ma_jump(ImmPtr dest) {
+ asMasm().ma_liPatchable(ScratchRegister, dest);
+ as_jr(ScratchRegister);
+ as_nop();
+}
+
+MacroAssembler& MacroAssemblerMIPSShared::asMasm() {
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerMIPSShared::asMasm() const {
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// MacroAssembler high-level usage.
+
+void MacroAssembler::flush() {}
+
+// ===============================================================
+// Stack manipulation functions.
+
+void MacroAssembler::Push(Register reg) {
+ ma_push(reg);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Push(const Imm32 imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Push(const ImmWord imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Push(const ImmPtr imm) {
+ Push(ImmWord(uintptr_t(imm.value)));
+}
+
+void MacroAssembler::Push(const ImmGCPtr ptr) {
+ ma_li(ScratchRegister, ptr);
+ ma_push(ScratchRegister);
+ adjustFrame(int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Push(FloatRegister f) {
+ ma_push(f);
+ adjustFrame(int32_t(f.pushSize()));
+}
+
+void MacroAssembler::Pop(Register reg) {
+ ma_pop(reg);
+ adjustFrame(-int32_t(sizeof(intptr_t)));
+}
+
+void MacroAssembler::Pop(FloatRegister f) {
+ ma_pop(f);
+ adjustFrame(-int32_t(f.pushSize()));
+}
+
+void MacroAssembler::Pop(const ValueOperand& val) {
+ popValue(val);
+ adjustFrame(-int32_t(sizeof(Value)));
+}
+
+void MacroAssembler::PopStackPtr() {
+ loadPtr(Address(StackPointer, 0), StackPointer);
+ adjustFrame(-int32_t(sizeof(intptr_t)));
+}
+
+// ===============================================================
+// Simple call functions.
+
+CodeOffset MacroAssembler::call(Register reg) {
+ as_jalr(reg);
+ as_nop();
+ return CodeOffset(currentOffset());
+}
+
+CodeOffset MacroAssembler::call(Label* label) {
+ ma_bal(label);
+ return CodeOffset(currentOffset());
+}
+
+CodeOffset MacroAssembler::callWithPatch() {
+ as_bal(BOffImm16(3 * sizeof(uint32_t)));
+ addPtr(Imm32(5 * sizeof(uint32_t)), ra);
+ // Allocate space which will be patched by patchCall().
+ spew(".space 32bit initValue 0xffff ffff");
+ writeInst(UINT32_MAX);
+ as_lw(ScratchRegister, ra, -(int32_t)(5 * sizeof(uint32_t)));
+ addPtr(ra, ScratchRegister);
+ as_jr(ScratchRegister);
+ as_nop();
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
+ BufferOffset call(callerOffset - 7 * sizeof(uint32_t));
+
+ BOffImm16 offset = BufferOffset(calleeOffset).diffB<BOffImm16>(call);
+ if (!offset.isInvalid()) {
+ InstImm* bal = (InstImm*)editSrc(call);
+ bal->setBOffImm16(offset);
+ } else {
+ uint32_t u32Offset = callerOffset - 5 * sizeof(uint32_t);
+ uint32_t* u32 =
+ reinterpret_cast<uint32_t*>(editSrc(BufferOffset(u32Offset)));
+ *u32 = calleeOffset - callerOffset;
+ }
+}
+
+CodeOffset MacroAssembler::farJumpWithPatch() {
+ ma_move(SecondScratchReg, ra);
+ as_bal(BOffImm16(3 * sizeof(uint32_t)));
+ as_lw(ScratchRegister, ra, 0);
+ // Allocate space which will be patched by patchFarJump().
+ CodeOffset farJump(currentOffset());
+ spew(".space 32bit initValue 0xffff ffff");
+ writeInst(UINT32_MAX);
+ addPtr(ra, ScratchRegister);
+ as_jr(ScratchRegister);
+ ma_move(ra, SecondScratchReg);
+ return farJump;
+}
+
+void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
+ uint32_t* u32 =
+ reinterpret_cast<uint32_t*>(editSrc(BufferOffset(farJump.offset())));
+ MOZ_ASSERT(*u32 == UINT32_MAX);
+ *u32 = targetOffset - farJump.offset();
+}
+
+CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) {
+ movePtr(target, CallReg);
+ return call(CallReg);
+}
+
+void MacroAssembler::call(const Address& addr) {
+ loadPtr(addr, CallReg);
+ call(CallReg);
+}
+
+void MacroAssembler::call(ImmWord target) { call(ImmPtr((void*)target.value)); }
+
+void MacroAssembler::call(ImmPtr target) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, target, RelocationKind::HARDCODED);
+ ma_call(target);
+}
+
+void MacroAssembler::call(JitCode* c) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(ScratchRegister, ImmPtr(c->raw()));
+ callJitNoProfiler(ScratchRegister);
+}
+
+CodeOffset MacroAssembler::nopPatchableToCall() {
+ // MIPS32 //MIPS64
+ as_nop(); // lui // lui
+ as_nop(); // ori // ori
+ as_nop(); // jalr // drotr32
+ as_nop(); // ori
+#ifdef JS_CODEGEN_MIPS64
+ as_nop(); // jalr
+ as_nop();
+#endif
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
+#ifdef JS_CODEGEN_MIPS64
+ Instruction* inst = (Instruction*)call - 6 /* six nops */;
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)target);
+ inst[4] = InstReg(op_special, ScratchRegister, zero, ra, ff_jalr);
+#else
+ Instruction* inst = (Instruction*)call - 4 /* four nops */;
+ Assembler::WriteLuiOriInstructions(inst, &inst[1], ScratchRegister,
+ (uint32_t)target);
+ inst[2] = InstReg(op_special, ScratchRegister, zero, ra, ff_jalr);
+#endif
+}
+
+void MacroAssembler::patchCallToNop(uint8_t* call) {
+#ifdef JS_CODEGEN_MIPS64
+ Instruction* inst = (Instruction*)call - 6 /* six nops */;
+#else
+ Instruction* inst = (Instruction*)call - 4 /* four nops */;
+#endif
+
+ inst[0].makeNop();
+ inst[1].makeNop();
+ inst[2].makeNop();
+ inst[3].makeNop();
+#ifdef JS_CODEGEN_MIPS64
+ inst[4].makeNop();
+ inst[5].makeNop();
+#endif
+}
+
+void MacroAssembler::pushReturnAddress() { push(ra); }
+
+void MacroAssembler::popReturnAddress() { pop(ra); }
+
+// ===============================================================
+// Jit Frames.
+
+uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
+ CodeLabel cl;
+
+ ma_li(scratch, &cl);
+ Push(scratch);
+ bind(&cl);
+ uint32_t retAddr = currentOffset();
+
+ addCodeLabel(cl);
+ return retAddr;
+}
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+ ma_and(buffer, ptr, Imm32(int32_t(~gc::ChunkMask)));
+ loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+ Register temp, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(ptr != temp);
+ MOZ_ASSERT(ptr != SecondScratchReg);
+
+ ma_and(SecondScratchReg, ptr, Imm32(int32_t(~gc::ChunkMask)));
+ branchPtr(InvertCondition(cond),
+ Address(SecondScratchReg, gc::ChunkStoreBufferOffset), ImmWord(0),
+ label);
+}
+
+void MacroAssembler::comment(const char* msg) { Assembler::comment(msg); }
+
+// ===============================================================
+// WebAssembly
+
+CodeOffset MacroAssembler::wasmTrapInstruction() {
+ CodeOffset offset(currentOffset());
+ as_teq(zero, zero, WASM_TRAP);
+ return offset;
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ as_truncwd(ScratchFloat32Reg, input);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, output);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_b(ScratchRegister, Imm32(0), oolEntry, Assembler::NotEqual);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ as_truncws(ScratchFloat32Reg, input);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, output);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_b(ScratchRegister, Imm32(0), oolEntry, Assembler::NotEqual);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToInt32Check(input, output, MIRType::Float32, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToInt32Check(input, output, MIRType::Double, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToInt64Check(input, output, MIRType::Float32, flags,
+ rejoin, off);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ outOfLineWasmTruncateToInt64Check(input, output, MIRType::Double, flags,
+ rejoin, off);
+}
+
+void MacroAssemblerMIPSShared::outOfLineWasmTruncateToInt32Check(
+ FloatRegister input, Register output, MIRType fromType, TruncFlags flags,
+ Label* rejoin, wasm::BytecodeOffset trapOffset) {
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ if (isSaturating) {
+ if (fromType == MIRType::Double) {
+ asMasm().loadConstantDouble(0.0, ScratchDoubleReg);
+ } else {
+ asMasm().loadConstantFloat32(0.0f, ScratchFloat32Reg);
+ }
+
+ if (isUnsigned) {
+ ma_li(output, Imm32(UINT32_MAX));
+
+ FloatTestKind moveCondition;
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input,
+ fromType == MIRType::Double ? ScratchDoubleReg : ScratchFloat32Reg,
+ Assembler::DoubleLessThanOrUnordered, &moveCondition);
+ MOZ_ASSERT(moveCondition == TestForTrue);
+
+ as_movt(output, zero);
+ } else {
+ // Positive overflow is already saturated to INT32_MAX, so we only have
+ // to handle NaN and negative overflow here.
+
+ FloatTestKind moveCondition;
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, input,
+ Assembler::DoubleUnordered, &moveCondition);
+ MOZ_ASSERT(moveCondition == TestForTrue);
+
+ as_movt(output, zero);
+
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input,
+ fromType == MIRType::Double ? ScratchDoubleReg : ScratchFloat32Reg,
+ Assembler::DoubleLessThan, &moveCondition);
+ MOZ_ASSERT(moveCondition == TestForTrue);
+
+ ma_li(ScratchRegister, Imm32(INT32_MIN));
+ as_movt(output, ScratchRegister);
+ }
+
+ MOZ_ASSERT(rejoin->bound());
+ asMasm().jump(rejoin);
+ return;
+ }
+
+ Label inputIsNaN;
+
+ if (fromType == MIRType::Double) {
+ asMasm().branchDouble(Assembler::DoubleUnordered, input, input,
+ &inputIsNaN);
+ } else if (fromType == MIRType::Float32) {
+ asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN);
+ }
+
+ asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
+ asMasm().bind(&inputIsNaN);
+ asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
+}
+
+void MacroAssemblerMIPSShared::outOfLineWasmTruncateToInt64Check(
+ FloatRegister input, Register64 output_, MIRType fromType, TruncFlags flags,
+ Label* rejoin, wasm::BytecodeOffset trapOffset) {
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ if (isSaturating) {
+#if defined(JS_CODEGEN_MIPS32)
+ // Saturating callouts don't use ool path.
+ return;
+#else
+ Register output = output_.reg;
+
+ if (fromType == MIRType::Double) {
+ asMasm().loadConstantDouble(0.0, ScratchDoubleReg);
+ } else {
+ asMasm().loadConstantFloat32(0.0f, ScratchFloat32Reg);
+ }
+
+ if (isUnsigned) {
+ asMasm().ma_li(output, ImmWord(UINT64_MAX));
+
+ FloatTestKind moveCondition;
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input,
+ fromType == MIRType::Double ? ScratchDoubleReg : ScratchFloat32Reg,
+ Assembler::DoubleLessThanOrUnordered, &moveCondition);
+ MOZ_ASSERT(moveCondition == TestForTrue);
+
+ as_movt(output, zero);
+
+ } else {
+ // Positive overflow is already saturated to INT64_MAX, so we only have
+ // to handle NaN and negative overflow here.
+
+ FloatTestKind moveCondition;
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input, input,
+ Assembler::DoubleUnordered, &moveCondition);
+ MOZ_ASSERT(moveCondition == TestForTrue);
+
+ as_movt(output, zero);
+
+ compareFloatingPoint(
+ fromType == MIRType::Double ? DoubleFloat : SingleFloat, input,
+ fromType == MIRType::Double ? ScratchDoubleReg : ScratchFloat32Reg,
+ Assembler::DoubleLessThan, &moveCondition);
+ MOZ_ASSERT(moveCondition == TestForTrue);
+
+ asMasm().ma_li(ScratchRegister, ImmWord(INT64_MIN));
+ as_movt(output, ScratchRegister);
+ }
+
+ MOZ_ASSERT(rejoin->bound());
+ asMasm().jump(rejoin);
+ return;
+#endif
+ }
+
+ Label inputIsNaN;
+
+ if (fromType == MIRType::Double) {
+ asMasm().branchDouble(Assembler::DoubleUnordered, input, input,
+ &inputIsNaN);
+ } else if (fromType == MIRType::Float32) {
+ asMasm().branchFloat(Assembler::DoubleUnordered, input, input, &inputIsNaN);
+ }
+
+#if defined(JS_CODEGEN_MIPS32)
+
+ // Only possible valid input that produces INT64_MIN result.
+ double validInput =
+ isUnsigned ? double(uint64_t(INT64_MIN)) : double(int64_t(INT64_MIN));
+
+ if (fromType == MIRType::Double) {
+ asMasm().loadConstantDouble(validInput, ScratchDoubleReg);
+ asMasm().branchDouble(Assembler::DoubleEqual, input, ScratchDoubleReg,
+ rejoin);
+ } else {
+ asMasm().loadConstantFloat32(float(validInput), ScratchFloat32Reg);
+ asMasm().branchFloat(Assembler::DoubleEqual, input, ScratchDoubleReg,
+ rejoin);
+ }
+
+#endif
+
+ asMasm().wasmTrap(wasm::Trap::IntegerOverflow, trapOffset);
+ asMasm().bind(&inputIsNaN);
+ asMasm().wasmTrap(wasm::Trap::InvalidConversionToInteger, trapOffset);
+}
+
+void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, AnyRegister output) {
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
+}
+
+void MacroAssembler::wasmUnalignedLoad(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register output,
+ Register tmp) {
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(output), tmp);
+}
+
+void MacroAssembler::wasmUnalignedLoadFP(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch,
+ FloatRegister output, Register tmp1) {
+ wasmLoadImpl(access, memoryBase, ptr, ptrScratch, AnyRegister(output), tmp1);
+}
+
+void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Register memoryBase,
+ Register ptr, Register ptrScratch) {
+ wasmStoreImpl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
+}
+
+void MacroAssembler::wasmUnalignedStore(const wasm::MemoryAccessDesc& access,
+ Register value, Register memoryBase,
+ Register ptr, Register ptrScratch,
+ Register tmp) {
+ wasmStoreImpl(access, AnyRegister(value), memoryBase, ptr, ptrScratch, tmp);
+}
+
+void MacroAssembler::wasmUnalignedStoreFP(const wasm::MemoryAccessDesc& access,
+ FloatRegister floatValue,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register tmp) {
+ wasmStoreImpl(access, AnyRegister(floatValue), memoryBase, ptr, ptrScratch,
+ tmp);
+}
+
+void MacroAssemblerMIPSShared::wasmLoadImpl(
+ const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
+ Register ptrScratch, AnyRegister output, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(ImmWord(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+ bool isFloat = false;
+
+ MOZ_ASSERT(!access.isZeroExtendSimd128Load());
+ MOZ_ASSERT(!access.isSplatSimd128Load());
+ MOZ_ASSERT(!access.isWidenSimd128Load());
+ switch (access.type()) {
+ case Scalar::Int8:
+ isSigned = true;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ break;
+ case Scalar::Int32:
+ isSigned = true;
+ break;
+ case Scalar::Uint32:
+ isSigned = false;
+ break;
+ case Scalar::Float64:
+ isFloat = true;
+ break;
+ case Scalar::Float32:
+ isFloat = true;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ BaseIndex address(memoryBase, ptr, TimesOne);
+ if (IsUnaligned(access)) {
+ MOZ_ASSERT(tmp != InvalidReg);
+ if (isFloat) {
+ if (byteSize == 4) {
+ asMasm().loadUnalignedFloat32(access, address, tmp, output.fpu());
+ } else {
+ asMasm().loadUnalignedDouble(access, address, tmp, output.fpu());
+ }
+ } else {
+ asMasm().ma_load_unaligned(access, output.gpr(), address, tmp,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+ if (isFloat) {
+ if (byteSize == 4) {
+ asMasm().ma_ls(output.fpu(), address);
+ } else {
+ asMasm().ma_ld(output.fpu(), address);
+ }
+ } else {
+ asMasm().ma_load(output.gpr(), address,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerMIPSShared::wasmStoreImpl(
+ const wasm::MemoryAccessDesc& access, AnyRegister value,
+ Register memoryBase, Register ptr, Register ptrScratch, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(ImmWord(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+ bool isFloat = false;
+
+ switch (access.type()) {
+ case Scalar::Int8:
+ isSigned = true;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ break;
+ case Scalar::Int32:
+ isSigned = true;
+ break;
+ case Scalar::Uint32:
+ isSigned = false;
+ break;
+ case Scalar::Int64:
+ isSigned = true;
+ break;
+ case Scalar::Float64:
+ isFloat = true;
+ break;
+ case Scalar::Float32:
+ isFloat = true;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ BaseIndex address(memoryBase, ptr, TimesOne);
+ if (IsUnaligned(access)) {
+ MOZ_ASSERT(tmp != InvalidReg);
+ if (isFloat) {
+ if (byteSize == 4) {
+ asMasm().storeUnalignedFloat32(access, value.fpu(), tmp, address);
+ } else {
+ asMasm().storeUnalignedDouble(access, value.fpu(), tmp, address);
+ }
+ } else {
+ asMasm().ma_store_unaligned(access, value.gpr(), address, tmp,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ return;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+ if (isFloat) {
+ if (byteSize == 4) {
+ asMasm().ma_ss(value.fpu(), address);
+ } else {
+ asMasm().ma_sd(value.fpu(), address);
+ }
+ } else {
+ asMasm().ma_store(value.gpr(), address,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ }
+ // Only the last emitted instruction is a memory access.
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
+ ExitFrameType type) {
+ enterFakeExitFrame(cxreg, scratch, type);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+template <typename T>
+static void CompareExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ const T& mem, Register oldval, Register newval,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again, end;
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll(output, SecondScratchReg, 0);
+ masm.ma_b(output, oldval, &end, Assembler::NotEqual, ShortJump);
+ masm.ma_move(ScratchRegister, newval);
+ masm.as_sc(ScratchRegister, SecondScratchReg, 0);
+ masm.ma_b(ScratchRegister, ScratchRegister, &again, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+ masm.bind(&end);
+
+ return;
+ }
+
+ masm.as_andi(offsetTemp, SecondScratchReg, 3);
+ masm.subPtr(offsetTemp, SecondScratchReg);
+#if !MOZ_LITTLE_ENDIAN()
+ masm.as_xori(offsetTemp, offsetTemp, 3);
+#endif
+ masm.as_sll(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sllv(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, zero, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll(ScratchRegister, SecondScratchReg, 0);
+
+ masm.as_srlv(output, ScratchRegister, offsetTemp);
+
+ switch (nbytes) {
+ case 1:
+ if (signExtend) {
+ masm.ma_seb(valueTemp, oldval);
+ masm.ma_seb(output, output);
+ } else {
+ masm.as_andi(valueTemp, oldval, 0xff);
+ masm.as_andi(output, output, 0xff);
+ }
+ break;
+ case 2:
+ if (signExtend) {
+ masm.ma_seh(valueTemp, oldval);
+ masm.ma_seh(output, output);
+ } else {
+ masm.as_andi(valueTemp, oldval, 0xffff);
+ masm.as_andi(output, output, 0xffff);
+ }
+ break;
+ }
+
+ masm.ma_b(output, valueTemp, &end, Assembler::NotEqual, ShortJump);
+
+ masm.as_sllv(valueTemp, newval, offsetTemp);
+ masm.as_and(ScratchRegister, ScratchRegister, maskTemp);
+ masm.as_or(ScratchRegister, ScratchRegister, valueTemp);
+
+ masm.as_sc(ScratchRegister, SecondScratchReg, 0);
+
+ masm.ma_b(ScratchRegister, ScratchRegister, &again, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ masm.bind(&end);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const Address& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ CompareExchange(*this, nullptr, type, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
+ newval, valueTemp, offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ CompareExchange(*this, &access, access.type(), access.sync(), mem, oldval,
+ newval, valueTemp, offsetTemp, maskTemp, output);
+}
+
+template <typename T>
+static void AtomicExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ const T& mem, Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll(output, SecondScratchReg, 0);
+ masm.ma_move(ScratchRegister, value);
+ masm.as_sc(ScratchRegister, SecondScratchReg, 0);
+ masm.ma_b(ScratchRegister, ScratchRegister, &again, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+ masm.as_andi(offsetTemp, SecondScratchReg, 3);
+ masm.subPtr(offsetTemp, SecondScratchReg);
+#if !MOZ_LITTLE_ENDIAN()
+ masm.as_xori(offsetTemp, offsetTemp, 3);
+#endif
+ masm.as_sll(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sllv(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, zero, maskTemp);
+ switch (nbytes) {
+ case 1:
+ masm.as_andi(valueTemp, value, 0xff);
+ break;
+ case 2:
+ masm.as_andi(valueTemp, value, 0xffff);
+ break;
+ }
+ masm.as_sllv(valueTemp, valueTemp, offsetTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll(output, SecondScratchReg, 0);
+ masm.as_and(ScratchRegister, output, maskTemp);
+ masm.as_or(ScratchRegister, ScratchRegister, valueTemp);
+
+ masm.as_sc(ScratchRegister, SecondScratchReg, 0);
+
+ masm.ma_b(ScratchRegister, ScratchRegister, &again, Assembler::Zero,
+ ShortJump);
+
+ masm.as_srlv(output, output, offsetTemp);
+
+ switch (nbytes) {
+ case 1:
+ if (signExtend) {
+ masm.ma_seb(output, output);
+ } else {
+ masm.as_andi(output, output, 0xff);
+ }
+ break;
+ case 2:
+ if (signExtend) {
+ masm.ma_seh(output, output);
+ } else {
+ masm.as_andi(output, output, 0xffff);
+ }
+ break;
+ }
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const Address& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, output);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicExchange(*this, nullptr, type, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
+ valueTemp, offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicExchange(*this, &access, access.type(), access.sync(), mem, value,
+ valueTemp, offsetTemp, maskTemp, output);
+}
+
+template <typename T>
+static void AtomicFetchOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, const T& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ bool signExtend = Scalar::isSignedIntType(type);
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll(output, SecondScratchReg, 0);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_addu(ScratchRegister, output, value);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_subu(ScratchRegister, output, value);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(ScratchRegister, output, value);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(ScratchRegister, output, value);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(ScratchRegister, output, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.as_sc(ScratchRegister, SecondScratchReg, 0);
+ masm.ma_b(ScratchRegister, ScratchRegister, &again, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+ masm.as_andi(offsetTemp, SecondScratchReg, 3);
+ masm.subPtr(offsetTemp, SecondScratchReg);
+#if !MOZ_LITTLE_ENDIAN()
+ masm.as_xori(offsetTemp, offsetTemp, 3);
+#endif
+ masm.as_sll(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sllv(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, zero, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll(ScratchRegister, SecondScratchReg, 0);
+ masm.as_srlv(output, ScratchRegister, offsetTemp);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_addu(valueTemp, output, value);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_subu(valueTemp, output, value);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(valueTemp, output, value);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(valueTemp, output, value);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(valueTemp, output, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi(valueTemp, valueTemp, 0xff);
+ break;
+ case 2:
+ masm.as_andi(valueTemp, valueTemp, 0xffff);
+ break;
+ }
+
+ masm.as_sllv(valueTemp, valueTemp, offsetTemp);
+
+ masm.as_and(ScratchRegister, ScratchRegister, maskTemp);
+ masm.as_or(ScratchRegister, ScratchRegister, valueTemp);
+
+ masm.as_sc(ScratchRegister, SecondScratchReg, 0);
+
+ masm.ma_b(ScratchRegister, ScratchRegister, &again, Assembler::Zero,
+ ShortJump);
+
+ switch (nbytes) {
+ case 1:
+ if (signExtend) {
+ masm.ma_seb(output, output);
+ } else {
+ masm.as_andi(output, output, 0xff);
+ }
+ break;
+ case 2:
+ if (signExtend) {
+ masm.ma_seh(output, output);
+ } else {
+ masm.as_andi(output, output, 0xffff);
+ }
+ break;
+ }
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type type,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp,
+ offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type type,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register output) {
+ AtomicFetchOp(*this, nullptr, type, sync, op, mem, value, valueTemp,
+ offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value,
+ valueTemp, offsetTemp, maskTemp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), access.sync(), op, mem, value,
+ valueTemp, offsetTemp, maskTemp, output);
+}
+
+template <typename T>
+static void AtomicEffectOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const Synchronization& sync,
+ AtomicOp op, const T& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) {
+ unsigned nbytes = Scalar::byteSize(type);
+
+ switch (nbytes) {
+ case 1:
+ case 2:
+ break;
+ case 4:
+ MOZ_ASSERT(valueTemp == InvalidReg);
+ MOZ_ASSERT(offsetTemp == InvalidReg);
+ MOZ_ASSERT(maskTemp == InvalidReg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ Label again;
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ if (nbytes == 4) {
+ masm.memoryBarrierBefore(sync);
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll(ScratchRegister, SecondScratchReg, 0);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_addu(ScratchRegister, ScratchRegister, value);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_subu(ScratchRegister, ScratchRegister, value);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(ScratchRegister, ScratchRegister, value);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(ScratchRegister, ScratchRegister, value);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(ScratchRegister, ScratchRegister, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.as_sc(ScratchRegister, SecondScratchReg, 0);
+ masm.ma_b(ScratchRegister, ScratchRegister, &again, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ return;
+ }
+
+ masm.as_andi(offsetTemp, SecondScratchReg, 3);
+ masm.subPtr(offsetTemp, SecondScratchReg);
+#if !MOZ_LITTLE_ENDIAN()
+ masm.as_xori(offsetTemp, offsetTemp, 3);
+#endif
+ masm.as_sll(offsetTemp, offsetTemp, 3);
+ masm.ma_li(maskTemp, Imm32(UINT32_MAX >> ((4 - nbytes) * 8)));
+ masm.as_sllv(maskTemp, maskTemp, offsetTemp);
+ masm.as_nor(maskTemp, zero, maskTemp);
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&again);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_ll(ScratchRegister, SecondScratchReg, 0);
+ masm.as_srlv(valueTemp, ScratchRegister, offsetTemp);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_addu(valueTemp, valueTemp, value);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_subu(valueTemp, valueTemp, value);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(valueTemp, valueTemp, value);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(valueTemp, valueTemp, value);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(valueTemp, valueTemp, value);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ switch (nbytes) {
+ case 1:
+ masm.as_andi(valueTemp, valueTemp, 0xff);
+ break;
+ case 2:
+ masm.as_andi(valueTemp, valueTemp, 0xffff);
+ break;
+ }
+
+ masm.as_sllv(valueTemp, valueTemp, offsetTemp);
+
+ masm.as_and(ScratchRegister, ScratchRegister, maskTemp);
+ masm.as_or(ScratchRegister, ScratchRegister, valueTemp);
+
+ masm.as_sc(ScratchRegister, SecondScratchReg, 0);
+
+ masm.ma_b(ScratchRegister, ScratchRegister, &again, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register valueTemp,
+ Register offsetTemp,
+ Register maskTemp) {
+ AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value,
+ valueTemp, offsetTemp, maskTemp);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) {
+ AtomicEffectOp(*this, &access, access.type(), access.sync(), op, mem, value,
+ valueTemp, offsetTemp, maskTemp);
+}
+
+// ========================================================================
+// JS atomic operations.
+
+template <typename T>
+static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register oldval, Register newval,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, output.gpr());
+ }
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, temp, output);
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, valueTemp,
+ offsetTemp, maskTemp, temp, output);
+}
+
+template <typename T>
+static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register value, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicExchange(arrayType, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, temp, output);
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, valueTemp, offsetTemp,
+ maskTemp, temp, output);
+}
+
+template <typename T>
+static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const T& mem, Register valueTemp,
+ Register offsetTemp, Register maskTemp,
+ Register temp, AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp,
+ maskTemp, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, valueTemp, offsetTemp,
+ maskTemp, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp,
+ maskTemp, temp, output);
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp, Register temp,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, valueTemp, offsetTemp,
+ maskTemp, temp, output);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) {
+ AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp,
+ offsetTemp, maskTemp);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register valueTemp, Register offsetTemp,
+ Register maskTemp) {
+ AtomicEffectOp(*this, nullptr, arrayType, sync, op, mem, value, valueTemp,
+ offsetTemp, maskTemp);
+}
+
+void MacroAssembler::flexibleQuotient32(Register rhs, Register srcDest,
+ bool isUnsigned,
+ const LiveRegisterSet&) {
+ quotient32(rhs, srcDest, isUnsigned);
+}
+
+void MacroAssembler::flexibleRemainder32(Register rhs, Register srcDest,
+ bool isUnsigned,
+ const LiveRegisterSet&) {
+ remainder32(rhs, srcDest, isUnsigned);
+}
+
+void MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest,
+ Register remOutput, bool isUnsigned,
+ const LiveRegisterSet&) {
+ if (isUnsigned) {
+#ifdef MIPSR6
+ as_divu(ScratchRegister, srcDest, rhs);
+ as_modu(remOutput, srcDest, rhs);
+ ma_move(srcDest, ScratchRegister);
+#else
+ as_divu(srcDest, rhs);
+#endif
+ } else {
+#ifdef MIPSR6
+ as_div(ScratchRegister, srcDest, rhs);
+ as_mod(remOutput, srcDest, rhs);
+ ma_move(srcDest, ScratchRegister);
+#else
+ as_div(srcDest, rhs);
+#endif
+ }
+#ifndef MIPSR6
+ as_mfhi(remOutput);
+ as_mflo(srcDest);
+#endif
+}
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ return movWithPatch(ImmPtr(nullptr), dest);
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target) {
+ PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr));
+}
+
+// ========================================================================
+// Spectre Mitigations.
+
+void MacroAssembler::speculationBarrier() { MOZ_CRASH(); }
+
+void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchFloat32Scope scratch(*this);
+
+ Label skipCheck, done;
+
+ // If Nan, 0 or -0 check for bailout
+ loadConstantFloat32(0.0f, scratch);
+ ma_bc1s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump);
+
+ // If binary value is not zero, it is NaN or -0, so we bail.
+ moveFromDoubleLo(src, SecondScratchReg);
+ branch32(Assembler::NotEqual, SecondScratchReg, Imm32(0), fail);
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&done, ShortJump);
+
+ bind(&skipCheck);
+ as_floorws(scratch, src);
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchDoubleScope scratch(*this);
+
+ Label skipCheck, done;
+
+ // If Nan, 0 or -0 check for bailout
+ loadConstantDouble(0.0, scratch);
+ ma_bc1d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump);
+
+ // If high part is not zero, it is NaN or -0, so we bail.
+ moveFromDoubleHi(src, SecondScratchReg);
+ branch32(Assembler::NotEqual, SecondScratchReg, Imm32(0), fail);
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&done, ShortJump);
+
+ bind(&skipCheck);
+ as_floorwd(scratch, src);
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchFloat32Scope scratch(*this);
+
+ Label performCeil, done;
+
+ // If x < -1 or x > 0 then perform ceil.
+ loadConstantFloat32(0.0f, scratch);
+ branchFloat(Assembler::DoubleGreaterThan, src, scratch, &performCeil);
+ loadConstantFloat32(-1.0f, scratch);
+ branchFloat(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil);
+
+ // If binary value is not zero, the input was not 0, so we bail.
+ moveFromFloat32(src, SecondScratchReg);
+ branch32(Assembler::NotEqual, SecondScratchReg, Imm32(0), fail);
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&done, ShortJump);
+
+ bind(&performCeil);
+ as_ceilws(scratch, src);
+ moveFromFloat32(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchDoubleScope scratch(*this);
+
+ Label performCeil, done;
+
+ // If x < -1 or x > 0 then perform ceil.
+ loadConstantDouble(0, scratch);
+ branchDouble(Assembler::DoubleGreaterThan, src, scratch, &performCeil);
+ loadConstantDouble(-1, scratch);
+ branchDouble(Assembler::DoubleLessThanOrEqual, src, scratch, &performCeil);
+
+ // If high part is not zero, the input was not 0, so we bail.
+ moveFromDoubleHi(src, SecondScratchReg);
+ branch32(Assembler::NotEqual, SecondScratchReg, Imm32(0), fail);
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&done, ShortJump);
+
+ bind(&performCeil);
+ as_ceilwd(scratch, src);
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ bind(&done);
+}
+
+void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ ScratchFloat32Scope scratch(*this);
+
+ Label negative, end, skipCheck;
+
+ // Load biggest number less than 0.5 in the temp register.
+ loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp);
+
+ // Branch to a slow path for negative inputs. Doesn't catch NaN or -0.
+ loadConstantFloat32(0.0f, scratch);
+ ma_bc1s(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump);
+
+ // If Nan, 0 or -0 check for bailout
+ ma_bc1s(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump);
+
+ // If binary value is not zero, it is NaN or -0, so we bail.
+ moveFromFloat32(src, SecondScratchReg);
+ branch32(Assembler::NotEqual, SecondScratchReg, Imm32(0), fail);
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&end, ShortJump);
+
+ bind(&skipCheck);
+ as_adds(scratch, src, temp);
+ as_floorws(scratch, scratch);
+
+ moveFromFloat32(scratch, dest);
+
+ branchTest32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branchTest32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ jump(&end);
+
+ // Input is negative, but isn't -0.
+ bind(&negative);
+
+ // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to
+ // be added the biggest double less than 0.5.
+ Label loadJoin;
+ loadConstantFloat32(-0.5f, scratch);
+ branchFloat(Assembler::DoubleLessThan, src, scratch, &loadJoin);
+ loadConstantFloat32(0.5f, temp);
+ bind(&loadJoin);
+
+ as_adds(temp, src, temp);
+
+ // If input + 0.5 >= 0, input is a negative number >= -0.5 and the
+ // result is -0.
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail);
+
+ // Truncate and round toward zero.
+ // This is off-by-one for everything but integer-valued inputs.
+ as_floorws(scratch, temp);
+ moveFromFloat32(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+
+ bind(&end);
+}
+
+void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ ScratchDoubleScope scratch(*this);
+
+ Label negative, end, skipCheck;
+
+ // Load biggest number less than 0.5 in the temp register.
+ loadConstantDouble(GetBiggestNumberLessThan(0.5), temp);
+
+ // Branch to a slow path for negative inputs. Doesn't catch NaN or -0.
+ loadConstantDouble(0.0, scratch);
+ ma_bc1d(src, scratch, &negative, Assembler::DoubleLessThan, ShortJump);
+
+ // If Nan, 0 or -0 check for bailout
+ ma_bc1d(src, scratch, &skipCheck, Assembler::DoubleNotEqual, ShortJump);
+
+ // If high part is not zero, it is NaN or -0, so we bail.
+ moveFromDoubleHi(src, SecondScratchReg);
+ branch32(Assembler::NotEqual, SecondScratchReg, Imm32(0), fail);
+
+ // Input was zero, so return zero.
+ move32(Imm32(0), dest);
+ ma_b(&end, ShortJump);
+
+ bind(&skipCheck);
+ as_addd(scratch, src, temp);
+ as_floorwd(scratch, scratch);
+
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+ branch32(Assembler::Equal, dest, Imm32(INT_MAX), fail);
+
+ jump(&end);
+
+ // Input is negative, but isn't -0.
+ bind(&negative);
+
+ // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to
+ // be added the biggest double less than 0.5.
+ Label loadJoin;
+ loadConstantDouble(-0.5, scratch);
+ branchDouble(Assembler::DoubleLessThan, src, scratch, &loadJoin);
+ loadConstantDouble(0.5, temp);
+ bind(&loadJoin);
+
+ addDouble(src, temp);
+
+ // If input + 0.5 >= 0, input is a negative number >= -0.5 and the
+ // result is -0.
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail);
+
+ // Truncate and round toward zero.
+ // This is off-by-one for everything but integer-valued inputs.
+ as_floorwd(scratch, temp);
+ moveFromDoubleLo(scratch, dest);
+
+ branch32(Assembler::Equal, dest, Imm32(INT_MIN), fail);
+
+ bind(&end);
+}
+
+void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ Label notZero;
+ as_truncws(ScratchFloat32Reg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+
+ ma_b(dest, Imm32(0), &notZero, Assembler::NotEqual, ShortJump);
+ moveFromFloat32(src, ScratchRegister);
+ // Check if src is in ]-1; -0] range by checking the sign bit.
+ as_slt(ScratchRegister, ScratchRegister, zero);
+ bind(&notZero);
+
+ branch32(Assembler::NotEqual, ScratchRegister, Imm32(0), fail);
+}
+
+void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ Label notZero;
+ as_truncwd(ScratchFloat32Reg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+
+ ma_b(dest, Imm32(0), &notZero, Assembler::NotEqual, ShortJump);
+ moveFromDoubleHi(src, ScratchRegister);
+ // Check if src is in ]-1; -0] range by checking the sign bit.
+ as_slt(ScratchRegister, ScratchRegister, zero);
+ bind(&notZero);
+
+ branch32(Assembler::NotEqual, ScratchRegister, Imm32(0), fail);
+}
+
+void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) {
+ MOZ_CRASH("not supported on this platform");
+}
+
+void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
+ Register pointer) {
+ if (IsShiftInScaleRange(shift)) {
+ computeEffectiveAddress(
+ BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer);
+ return;
+ }
+ lshift32(Imm32(shift), indexTemp32);
+ addPtr(indexTemp32, pointer);
+}
+
+//}}} check_macroassembler_style
diff --git a/js/src/jit/mips-shared/MacroAssembler-mips-shared.h b/js/src/jit/mips-shared/MacroAssembler-mips-shared.h
new file mode 100644
index 0000000000..88238accbb
--- /dev/null
+++ b/js/src/jit/mips-shared/MacroAssembler-mips-shared.h
@@ -0,0 +1,258 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_MacroAssembler_mips_shared_h
+#define jit_mips_shared_MacroAssembler_mips_shared_h
+
+#if defined(JS_CODEGEN_MIPS32)
+# include "jit/mips32/Assembler-mips32.h"
+#elif defined(JS_CODEGEN_MIPS64)
+# include "jit/mips64/Assembler-mips64.h"
+#endif
+
+#include "jit/AtomicOp.h"
+
+namespace js {
+namespace jit {
+
+enum LoadStoreSize {
+ SizeByte = 8,
+ SizeHalfWord = 16,
+ SizeWord = 32,
+ SizeDouble = 64
+};
+
+enum LoadStoreExtension { ZeroExtend = 0, SignExtend = 1 };
+
+enum JumpKind { LongJump = 0, ShortJump = 1 };
+
+enum DelaySlotFill { DontFillDelaySlot = 0, FillDelaySlot = 1 };
+
+static Register CallReg = t9;
+
+class MacroAssemblerMIPSShared : public Assembler {
+ protected:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ Condition ma_cmp(Register rd, Register lhs, Register rhs, Condition c);
+ Condition ma_cmp(Register rd, Register lhs, Imm32 imm, Condition c);
+
+ void compareFloatingPoint(FloatFormat fmt, FloatRegister lhs,
+ FloatRegister rhs, DoubleCondition c,
+ FloatTestKind* testKind, FPConditionBit fcc = FCC0);
+
+ public:
+ void ma_move(Register rd, Register rs);
+
+ void ma_li(Register dest, ImmGCPtr ptr);
+
+ void ma_li(Register dest, Imm32 imm);
+ void ma_liPatchable(Register dest, Imm32 imm);
+
+ // Shift operations
+ void ma_sll(Register rd, Register rt, Imm32 shift);
+ void ma_srl(Register rd, Register rt, Imm32 shift);
+ void ma_sra(Register rd, Register rt, Imm32 shift);
+ void ma_ror(Register rd, Register rt, Imm32 shift);
+ void ma_rol(Register rd, Register rt, Imm32 shift);
+
+ void ma_sll(Register rd, Register rt, Register shift);
+ void ma_srl(Register rd, Register rt, Register shift);
+ void ma_sra(Register rd, Register rt, Register shift);
+ void ma_ror(Register rd, Register rt, Register shift);
+ void ma_rol(Register rd, Register rt, Register shift);
+
+ // Negate
+ void ma_negu(Register rd, Register rs);
+
+ void ma_not(Register rd, Register rs);
+
+ // Bit extract/insert
+ void ma_ext(Register rt, Register rs, uint16_t pos, uint16_t size);
+ void ma_ins(Register rt, Register rs, uint16_t pos, uint16_t size);
+
+ // Sign extend
+ void ma_seb(Register rd, Register rt);
+ void ma_seh(Register rd, Register rt);
+
+ // and
+ void ma_and(Register rd, Register rs);
+ void ma_and(Register rd, Imm32 imm);
+ void ma_and(Register rd, Register rs, Imm32 imm);
+
+ // or
+ void ma_or(Register rd, Register rs);
+ void ma_or(Register rd, Imm32 imm);
+ void ma_or(Register rd, Register rs, Imm32 imm);
+
+ // xor
+ void ma_xor(Register rd, Register rs);
+ void ma_xor(Register rd, Imm32 imm);
+ void ma_xor(Register rd, Register rs, Imm32 imm);
+
+ // word swap byte within halfwords
+ void ma_wsbh(Register rd, Register rt);
+
+ void ma_ctz(Register rd, Register rs);
+
+ // load
+ void ma_load(Register dest, const BaseIndex& src,
+ LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+ void ma_load_unaligned(Register dest, const BaseIndex& src,
+ LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+ void ma_load_unaligned(Register dest, const Address& address,
+ LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+ void ma_load_unaligned(const wasm::MemoryAccessDesc& access, Register dest,
+ const BaseIndex& src, Register temp,
+ LoadStoreSize size, LoadStoreExtension extension);
+
+ // store
+ void ma_store(Register data, const BaseIndex& dest,
+ LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+ void ma_store(Imm32 imm, const BaseIndex& dest, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+ void ma_store_unaligned(Register data, const Address& dest,
+ LoadStoreSize size = SizeWord);
+ void ma_store_unaligned(Register data, const BaseIndex& dest,
+ LoadStoreSize size = SizeWord);
+ void ma_store_unaligned(const wasm::MemoryAccessDesc& access, Register data,
+ const BaseIndex& dest, Register temp,
+ LoadStoreSize size, LoadStoreExtension extension);
+
+ // arithmetic based ops
+ // add
+ void ma_addu(Register rd, Register rs, Imm32 imm);
+ void ma_addu(Register rd, Register rs);
+ void ma_addu(Register rd, Imm32 imm);
+ void ma_add32TestCarry(Condition cond, Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_add32TestCarry(Condition cond, Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+
+ // subtract
+ void ma_subu(Register rd, Register rs, Imm32 imm);
+ void ma_subu(Register rd, Register rs);
+ void ma_subu(Register rd, Imm32 imm);
+ void ma_sub32TestOverflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+
+ // multiplies. For now, there are only few that we care about.
+ void ma_mul(Register rd, Register rs, Imm32 imm);
+ void ma_mul32TestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_mul32TestOverflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+
+ // divisions
+ void ma_div_branch_overflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_div_branch_overflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+
+ // fast mod, uses scratch registers, and thus needs to be in the assembler
+ // implicitly assumes that we can overwrite dest at the beginning of the
+ // sequence
+ void ma_mod_mask(Register src, Register dest, Register hold, Register remain,
+ int32_t shift, Label* negZero = nullptr);
+
+ // branches when done from within mips-specific code
+ void ma_b(Register lhs, Register rhs, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Register lhs, Imm32 imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Register lhs, ImmPtr imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Register lhs, ImmGCPtr imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump) {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_b(lhs, ScratchRegister, l, c, jumpKind);
+ }
+
+ void ma_b(Label* l, JumpKind jumpKind = LongJump);
+
+ // fp instructions
+ void ma_lis(FloatRegister dest, float value);
+
+ void ma_sd(FloatRegister src, BaseIndex address);
+ void ma_ss(FloatRegister src, BaseIndex address);
+
+ void ma_ld(FloatRegister dest, const BaseIndex& src);
+ void ma_ls(FloatRegister dest, const BaseIndex& src);
+
+ // FP branches
+ void ma_bc1s(FloatRegister lhs, FloatRegister rhs, Label* label,
+ DoubleCondition c, JumpKind jumpKind = LongJump,
+ FPConditionBit fcc = FCC0);
+ void ma_bc1d(FloatRegister lhs, FloatRegister rhs, Label* label,
+ DoubleCondition c, JumpKind jumpKind = LongJump,
+ FPConditionBit fcc = FCC0);
+
+ void ma_call(ImmPtr dest);
+
+ void ma_jump(ImmPtr dest);
+
+ void ma_cmp_set(Register dst, Register lhs, Register rhs, Condition c);
+ void ma_cmp_set(Register dst, Register lhs, Imm32 imm, Condition c);
+ // void ma_cmp_set(Register dst, Address address, Imm32 imm, Condition c);
+ void ma_cmp_set_double(Register dst, FloatRegister lhs, FloatRegister rhs,
+ DoubleCondition c);
+ void ma_cmp_set_float32(Register dst, FloatRegister lhs, FloatRegister rhs,
+ DoubleCondition c);
+
+ void moveToDoubleLo(Register src, FloatRegister dest) { as_mtc1(src, dest); }
+ void moveFromDoubleLo(FloatRegister src, Register dest) {
+ as_mfc1(dest, src);
+ }
+
+ void moveToFloat32(Register src, FloatRegister dest) { as_mtc1(src, dest); }
+ void moveFromFloat32(FloatRegister src, Register dest) { as_mfc1(dest, src); }
+
+ // Evaluate srcDest = minmax<isMax>{Float32,Double}(srcDest, other).
+ // Handle NaN specially if handleNaN is true.
+ void minMaxDouble(FloatRegister srcDest, FloatRegister other, bool handleNaN,
+ bool isMax);
+ void minMaxFloat32(FloatRegister srcDest, FloatRegister other, bool handleNaN,
+ bool isMax);
+
+ void loadDouble(const Address& addr, FloatRegister dest);
+ void loadDouble(const BaseIndex& src, FloatRegister dest);
+
+ // Load a float value into a register, then expand it to a double.
+ void loadFloatAsDouble(const Address& addr, FloatRegister dest);
+ void loadFloatAsDouble(const BaseIndex& src, FloatRegister dest);
+
+ void loadFloat32(const Address& addr, FloatRegister dest);
+ void loadFloat32(const BaseIndex& src, FloatRegister dest);
+
+ void outOfLineWasmTruncateToInt32Check(FloatRegister input, Register output,
+ MIRType fromType, TruncFlags flags,
+ Label* rejoin,
+ wasm::BytecodeOffset trapOffset);
+ void outOfLineWasmTruncateToInt64Check(FloatRegister input, Register64 output,
+ MIRType fromType, TruncFlags flags,
+ Label* rejoin,
+ wasm::BytecodeOffset trapOffset);
+
+ protected:
+ void wasmLoadImpl(const wasm::MemoryAccessDesc& access, Register memoryBase,
+ Register ptr, Register ptrScratch, AnyRegister output,
+ Register tmp);
+ void wasmStoreImpl(const wasm::MemoryAccessDesc& access, AnyRegister value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_MacroAssembler_mips_shared_h */
diff --git a/js/src/jit/mips-shared/MoveEmitter-mips-shared.cpp b/js/src/jit/mips-shared/MoveEmitter-mips-shared.cpp
new file mode 100644
index 0000000000..5ea8f0b8de
--- /dev/null
+++ b/js/src/jit/mips-shared/MoveEmitter-mips-shared.cpp
@@ -0,0 +1,207 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips-shared/MoveEmitter-mips-shared.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void MoveEmitterMIPSShared::emit(const MoveResolver& moves) {
+ if (moves.numCycles()) {
+ // Reserve stack for cycle resolution
+ static_assert(SpillSlotSize == 8);
+ masm.reserveStack(moves.numCycles() * SpillSlotSize);
+ pushedAtCycle_ = masm.framePushed();
+ }
+
+ for (size_t i = 0; i < moves.numMoves(); i++) {
+ emit(moves.getMove(i));
+ }
+}
+
+Address MoveEmitterMIPSShared::cycleSlot(uint32_t slot,
+ uint32_t subslot) const {
+ int32_t offset = masm.framePushed() - pushedAtCycle_;
+ MOZ_ASSERT(Imm16::IsInSignedRange(offset));
+ return Address(StackPointer, offset + slot * sizeof(double) + subslot);
+}
+
+int32_t MoveEmitterMIPSShared::getAdjustedOffset(const MoveOperand& operand) {
+ MOZ_ASSERT(operand.isMemoryOrEffectiveAddress());
+ if (operand.base() != StackPointer) {
+ return operand.disp();
+ }
+
+ // Adjust offset if stack pointer has been moved.
+ return operand.disp() + masm.framePushed() - pushedAtStart_;
+}
+
+Address MoveEmitterMIPSShared::getAdjustedAddress(const MoveOperand& operand) {
+ return Address(operand.base(), getAdjustedOffset(operand));
+}
+
+Register MoveEmitterMIPSShared::tempReg() {
+ spilledReg_ = SecondScratchReg;
+ return SecondScratchReg;
+}
+
+void MoveEmitterMIPSShared::emitMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isGeneralReg()) {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(from.reg() != spilledReg_);
+
+ if (to.isGeneralReg()) {
+ masm.movePtr(from.reg(), to.reg());
+ } else if (to.isMemory()) {
+ masm.storePtr(from.reg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+ } else if (from.isMemory()) {
+ if (to.isGeneralReg()) {
+ masm.loadPtr(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ masm.loadPtr(getAdjustedAddress(from), tempReg());
+ masm.storePtr(tempReg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+ } else if (from.isEffectiveAddress()) {
+ if (to.isGeneralReg()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), tempReg());
+ masm.storePtr(tempReg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+ } else {
+ MOZ_CRASH("Invalid emitMove arguments.");
+ }
+}
+
+void MoveEmitterMIPSShared::emitInt32Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isGeneralReg()) {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(from.reg() != spilledReg_);
+
+ if (to.isGeneralReg()) {
+ masm.move32(from.reg(), to.reg());
+ } else if (to.isMemory()) {
+ masm.store32(from.reg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+ } else if (from.isMemory()) {
+ if (to.isGeneralReg()) {
+ masm.load32(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ masm.load32(getAdjustedAddress(from), tempReg());
+ masm.store32(tempReg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+ } else if (from.isEffectiveAddress()) {
+ if (to.isGeneralReg()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), to.reg());
+ } else if (to.isMemory()) {
+ masm.computeEffectiveAddress(getAdjustedAddress(from), tempReg());
+ masm.store32(tempReg(), getAdjustedAddress(to));
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+ } else {
+ MOZ_CRASH("Invalid emitInt32Move arguments.");
+ }
+}
+
+void MoveEmitterMIPSShared::emitFloat32Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ // Ensure that we can use ScratchFloat32Reg in memory move.
+ MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg() != ScratchFloat32Reg);
+ MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg() != ScratchFloat32Reg);
+
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveFloat32(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ // This should only be used when passing float parameter in a1,a2,a3
+ MOZ_ASSERT(to.reg() == a1 || to.reg() == a2 || to.reg() == a3);
+ masm.moveFromFloat32(from.floatReg(), to.reg());
+ } else {
+ MOZ_ASSERT(to.isMemory());
+ masm.storeFloat32(from.floatReg(), getAdjustedAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ MOZ_ASSERT(from.isMemory());
+ masm.loadFloat32(getAdjustedAddress(from), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ MOZ_ASSERT(from.isMemory());
+ // This should only be used when passing float parameter in a1,a2,a3
+ MOZ_ASSERT(to.reg() == a1 || to.reg() == a2 || to.reg() == a3);
+ masm.loadPtr(getAdjustedAddress(from), to.reg());
+ } else {
+ MOZ_ASSERT(from.isMemory());
+ MOZ_ASSERT(to.isMemory());
+ masm.loadFloat32(getAdjustedAddress(from), ScratchFloat32Reg);
+ masm.storeFloat32(ScratchFloat32Reg, getAdjustedAddress(to));
+ }
+}
+
+void MoveEmitterMIPSShared::emit(const MoveOp& move) {
+ const MoveOperand& from = move.from();
+ const MoveOperand& to = move.to();
+
+ if (move.isCycleEnd() && move.isCycleBegin()) {
+ // A fun consequence of aliased registers is you can have multiple
+ // cycles at once, and one can end exactly where another begins.
+ breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot());
+ completeCycle(from, to, move.type(), move.cycleEndSlot());
+ return;
+ }
+
+ if (move.isCycleEnd()) {
+ MOZ_ASSERT(inCycle_);
+ completeCycle(from, to, move.type(), move.cycleEndSlot());
+ MOZ_ASSERT(inCycle_ > 0);
+ inCycle_--;
+ return;
+ }
+
+ if (move.isCycleBegin()) {
+ breakCycle(from, to, move.endCycleType(), move.cycleBeginSlot());
+ inCycle_++;
+ }
+
+ switch (move.type()) {
+ case MoveOp::FLOAT32:
+ emitFloat32Move(from, to);
+ break;
+ case MoveOp::DOUBLE:
+ emitDoubleMove(from, to);
+ break;
+ case MoveOp::INT32:
+ emitInt32Move(from, to);
+ break;
+ case MoveOp::GENERAL:
+ emitMove(from, to);
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterMIPSShared::assertDone() { MOZ_ASSERT(inCycle_ == 0); }
+
+void MoveEmitterMIPSShared::finish() {
+ assertDone();
+
+ masm.freeStack(masm.framePushed() - pushedAtStart_);
+}
diff --git a/js/src/jit/mips-shared/MoveEmitter-mips-shared.h b/js/src/jit/mips-shared/MoveEmitter-mips-shared.h
new file mode 100644
index 0000000000..81dbaddc45
--- /dev/null
+++ b/js/src/jit/mips-shared/MoveEmitter-mips-shared.h
@@ -0,0 +1,73 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_MoveEmitter_mips_shared_h
+#define jit_mips_shared_MoveEmitter_mips_shared_h
+
+#include "jit/MacroAssembler.h"
+#include "jit/MoveResolver.h"
+
+namespace js {
+namespace jit {
+
+class MoveEmitterMIPSShared {
+ protected:
+ uint32_t inCycle_;
+ MacroAssembler& masm;
+
+ // Original stack push value.
+ uint32_t pushedAtStart_;
+
+ // These store stack offsets to spill locations, snapshotting
+ // codegen->framePushed_ at the time they were allocated. They are -1 if no
+ // stack space has been allocated for that particular spill.
+ int32_t pushedAtCycle_;
+ int32_t pushedAtSpill_;
+
+ // These are registers that are available for temporary use. They may be
+ // assigned InvalidReg. If no corresponding spill space has been assigned,
+ // then these registers do not need to be spilled.
+ Register spilledReg_;
+ FloatRegister spilledFloatReg_;
+
+ void assertDone();
+ Register tempReg();
+ FloatRegister tempFloatReg();
+ Address cycleSlot(uint32_t slot, uint32_t subslot = 0) const;
+ int32_t getAdjustedOffset(const MoveOperand& operand);
+ Address getAdjustedAddress(const MoveOperand& operand);
+
+ void emitMove(const MoveOperand& from, const MoveOperand& to);
+ void emitInt32Move(const MoveOperand& from, const MoveOperand& to);
+ void emitFloat32Move(const MoveOperand& from, const MoveOperand& to);
+ virtual void emitDoubleMove(const MoveOperand& from,
+ const MoveOperand& to) = 0;
+ virtual void breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot) = 0;
+ virtual void completeCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot) = 0;
+ void emit(const MoveOp& move);
+
+ public:
+ MoveEmitterMIPSShared(MacroAssembler& masm)
+ : inCycle_(0),
+ masm(masm),
+ pushedAtStart_(masm.framePushed()),
+ pushedAtCycle_(-1),
+ pushedAtSpill_(-1),
+ spilledReg_(InvalidReg),
+ spilledFloatReg_(InvalidFloatReg) {}
+ ~MoveEmitterMIPSShared() { assertDone(); }
+ void emit(const MoveResolver& moves);
+ void finish();
+
+ void setScratchRegister(Register reg) {}
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_MoveEmitter_mips_shared_h */
diff --git a/js/src/jit/mips-shared/SharedICHelpers-mips-shared-inl.h b/js/src/jit/mips-shared/SharedICHelpers-mips-shared-inl.h
new file mode 100644
index 0000000000..cee021595f
--- /dev/null
+++ b/js/src/jit/mips-shared/SharedICHelpers-mips-shared-inl.h
@@ -0,0 +1,82 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_SharedICHelpers_mips_shared_inl_h
+#define jit_mips_shared_SharedICHelpers_mips_shared_inl_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+ uint32_t argSize) {
+#ifdef DEBUG
+ Register scratch = R2.scratchReg();
+
+ // Compute frame size.
+ masm.movePtr(FramePointer, scratch);
+ masm.subPtr(StackPointer, scratch);
+
+ // Store frame size without VMFunction arguments for debug assertions.
+ masm.subPtr(Imm32(argSize), scratch);
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+ masm.addPtr(Imm32(argSize), scratch);
+#endif
+
+ // Push frame descriptor and perform the tail call.
+ // ICTailCallReg (ra) already contains the return address (as we
+ // keep it there through the stub calls), but the VMWrapper code being
+ // called expects the return address to also be pushed on the stack.
+ MOZ_ASSERT(ICTailCallReg == ra);
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(ra);
+ masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) {
+ MOZ_ASSERT(scratch != ICTailCallReg);
+
+#ifdef DEBUG
+ // Compute frame size.
+ masm.movePtr(FramePointer, scratch);
+ masm.subPtr(StackPointer, scratch);
+
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Note: when making changes here, don't forget to update
+ // BaselineStubFrame if needed.
+
+ // Push frame descriptor and return address.
+ masm.PushFrameDescriptor(FrameType::BaselineJS);
+ masm.Push(ICTailCallReg);
+
+ // Save old frame pointer, stack pointer and stub reg.
+ masm.Push(FramePointer);
+ masm.movePtr(StackPointer, FramePointer);
+ masm.Push(ICStubReg);
+
+ // Stack should remain aligned.
+ masm.assertStackAlignment(sizeof(Value), 0);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_SharedICHelpers_mips_shared_inl_h */
diff --git a/js/src/jit/mips-shared/SharedICHelpers-mips-shared.h b/js/src/jit/mips-shared/SharedICHelpers-mips-shared.h
new file mode 100644
index 0000000000..979e4b0a42
--- /dev/null
+++ b/js/src/jit/mips-shared/SharedICHelpers-mips-shared.h
@@ -0,0 +1,88 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips_shared_SharedICHelpers_mips_shared_h
+#define jit_mips_shared_SharedICHelpers_mips_shared_h
+
+#include "jit/BaselineIC.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from sp to the top Value inside an IC stub (no return address on
+// the stack on MIPS).
+static const size_t ICStackValueOffset = 0;
+
+struct BaselineStubFrame {
+ uintptr_t savedFrame;
+ uintptr_t savedStub;
+ uintptr_t returnAddress;
+ uintptr_t descriptor;
+};
+
+inline void EmitRestoreTailCallReg(MacroAssembler& masm) {
+ // No-op on MIPS because ra register is always holding the return address.
+}
+
+inline void EmitRepushTailCallReg(MacroAssembler& masm) {
+ // No-op on MIPS because ra register is always holding the return address.
+}
+
+inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
+ // The stub pointer must already be in ICStubReg.
+ // Load stubcode pointer from the ICStub.
+ // R2 won't be active when we call ICs, so we can use it as scratch.
+ masm.loadPtr(Address(ICStubReg, ICStub::offsetOfStubCode()), R2.scratchReg());
+
+ // Call the stubcode via a direct jump-and-link
+ masm.call(R2.scratchReg());
+ *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void EmitReturnFromIC(MacroAssembler& masm) { masm.branch(ra); }
+
+inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
+ masm.loadPtr(
+ Address(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP),
+ ICStubReg);
+ masm.movePtr(FramePointer, StackPointer);
+ masm.Pop(FramePointer);
+
+ // Load the return address.
+ masm.Pop(ICTailCallReg);
+
+ // Discard the frame descriptor.
+ {
+ SecondScratchRegisterScope scratch2(masm);
+ masm.Pop(scratch2);
+ }
+}
+
+template <typename AddrType>
+inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr,
+ MIRType type) {
+ // On MIPS, $ra is clobbered by guardedCallPreBarrier. Save it first.
+ masm.push(ra);
+ masm.guardedCallPreBarrier(addr, type);
+ masm.pop(ra);
+}
+
+inline void EmitStubGuardFailure(MacroAssembler& masm) {
+ // Load next stub into ICStubReg
+ masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+ // Return address is already loaded, just jump to the next stubcode.
+ MOZ_ASSERT(ICTailCallReg == ra);
+ masm.jump(Address(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips_shared_SharedICHelpers_mips_shared_h */
diff --git a/js/src/jit/mips32/Architecture-mips32.cpp b/js/src/jit/mips32/Architecture-mips32.cpp
new file mode 100644
index 0000000000..598551eafe
--- /dev/null
+++ b/js/src/jit/mips32/Architecture-mips32.cpp
@@ -0,0 +1,94 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips32/Architecture-mips32.h"
+
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+const char* const Registers::RegNames[] = {
+ "zero", "at", "v0", "v1", "a0", "a1", "a2", "a3", "t0", "t1", "t2",
+ "t3", "t4", "t5", "t6", "t7", "s0", "s1", "s2", "s3", "s4", "s5",
+ "s6", "s7", "t8", "t9", "k0", "k1", "gp", "sp", "fp", "ra"};
+
+const uint32_t Allocatable = 14;
+
+const Registers::SetType Registers::ArgRegMask = Registers::SharedArgRegMask;
+
+const Registers::SetType Registers::JSCallMask =
+ (1 << Registers::a2) | (1 << Registers::a3);
+
+const Registers::SetType Registers::CallMask =
+ (1 << Registers::v0) |
+ (1 << Registers::v1); // used for double-size returns
+
+FloatRegisters::Encoding FloatRegisters::FromName(const char* name) {
+ for (size_t i = 0; i < RegisterIdLimit; i++) {
+ if (strcmp(GetName(i), name) == 0) {
+ return Encoding(i);
+ }
+ }
+
+ return Invalid;
+}
+
+FloatRegister FloatRegister::doubleOverlay() const {
+ MOZ_ASSERT(isNotOdd());
+ if (isSingle()) {
+ return FloatRegister(code_, Double);
+ }
+ return *this;
+}
+
+FloatRegister FloatRegister::singleOverlay() const {
+ MOZ_ASSERT(isNotOdd());
+ if (isDouble()) {
+ return FloatRegister(code_, Single);
+ }
+ return *this;
+}
+
+FloatRegisterSet FloatRegister::ReduceSetForPush(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ LiveFloatRegisterSet mod;
+ for (FloatRegisterIterator iter(s); iter.more(); ++iter) {
+ // Even for single size registers save complete double register.
+ mod.addUnchecked((*iter).doubleOverlay());
+ }
+ return mod.set();
+}
+
+uint32_t FloatRegister::GetPushSizeInBytes(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ FloatRegisterSet ss = s.reduceSetForPush();
+ uint64_t bits = ss.bits();
+ // We are only pushing double registers.
+ MOZ_ASSERT((bits & 0xFFFF) == 0);
+ uint32_t ret = mozilla::CountPopulation32(bits) * sizeof(double);
+
+ // Additional space needed by MacroAssembler::PushRegsInMask to ensure
+ // correct alignment of double values.
+ if (ret) {
+ ret += sizeof(double);
+ }
+
+ return ret;
+}
+uint32_t FloatRegister::getRegisterDumpOffsetInBytes() {
+ MOZ_ASSERT(isNotOdd());
+ return id() * sizeof(float);
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/mips32/Architecture-mips32.h b/js/src/jit/mips32/Architecture-mips32.h
new file mode 100644
index 0000000000..8e186d2c9c
--- /dev/null
+++ b/js/src/jit/mips32/Architecture-mips32.h
@@ -0,0 +1,282 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_Architecture_mips32_h
+#define jit_mips32_Architecture_mips32_h
+
+#include "mozilla/EndianUtils.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <limits.h>
+#include <stdint.h>
+
+#include "jit/mips-shared/Architecture-mips-shared.h"
+
+#include "js/Utility.h"
+
+namespace js {
+namespace jit {
+
+static const uint32_t ShadowStackSpace = 4 * sizeof(uintptr_t);
+
+// These offsets are specific to nunboxing, and capture offsets into the
+// components of a js::Value.
+// Size of MIPS32 general purpose registers is 32 bits.
+#if MOZ_LITTLE_ENDIAN()
+static const int32_t NUNBOX32_TYPE_OFFSET = 4;
+static const int32_t NUNBOX32_PAYLOAD_OFFSET = 0;
+#else
+static const int32_t NUNBOX32_TYPE_OFFSET = 0;
+static const int32_t NUNBOX32_PAYLOAD_OFFSET = 4;
+#endif
+
+// MIPS32 can have two types of floating-point coprocessors modes:
+// - FR=0 mode/ 32-bit FPRs - Historical default, there are 32 single
+// precision registers and pairs of even and odd float registers are used as
+// double precision registers. Example: f0 (double) is composed of
+// f0 and f1 (single). Loongson3A FPU running in this mode doesn't allow
+// use of odd registers for single precision arithmetic.
+// - FR=1 mode/ 64-bit FPRs - In this case, there are 32 double precision
+// register which can also be used as single precision registers. More info
+// https://dmz-portal.imgtec.com/wiki/MIPS_O32_ABI_-_FR0_and_FR1_Interlinking
+
+// Currently we enable 16 even single precision registers which can be also can
+// be used as double precision registers. It enables jit code to run even on
+// Loongson3A. It does not support FR=1 mode because MacroAssembler threats odd
+// single precision registers as high parts of even double precision registers.
+#ifdef __mips_fpr
+static_assert(__mips_fpr == 32, "MIPS32 jit only supports FR=0 fpu mode.");
+#endif
+
+class FloatRegisters : public FloatRegistersMIPSShared {
+ public:
+ static const char* GetName(uint32_t i) {
+ MOZ_ASSERT(i < RegisterIdLimit);
+ return FloatRegistersMIPSShared::GetName(Encoding(i % 32));
+ }
+
+ static Encoding FromName(const char* name);
+
+ static const uint32_t Total = 32;
+ static const uint32_t TotalDouble = 16;
+ static const uint32_t TotalSingle = 16;
+
+ static const uint32_t Allocatable = 30;
+ static const SetType AllSingleMask = (1ULL << TotalSingle) - 1;
+
+ static const SetType AllDoubleMask = ((1ULL << TotalDouble) - 1)
+ << TotalSingle;
+ static const SetType AllMask = AllDoubleMask | AllSingleMask;
+
+ // When saving all registers we only need to do is save double registers.
+ static const uint32_t TotalPhys = 16;
+ static const uint32_t RegisterIdLimit = 32;
+
+ static_assert(sizeof(SetType) * 8 >= Total,
+ "SetType should be large enough to enumerate all registers.");
+
+ static const SetType NonVolatileMask =
+ ((SetType(1) << (FloatRegisters::f20 >> 1)) |
+ (SetType(1) << (FloatRegisters::f22 >> 1)) |
+ (SetType(1) << (FloatRegisters::f24 >> 1)) |
+ (SetType(1) << (FloatRegisters::f26 >> 1)) |
+ (SetType(1) << (FloatRegisters::f28 >> 1)) |
+ (SetType(1) << (FloatRegisters::f30 >> 1))) *
+ ((1 << TotalSingle) + 1);
+
+ static const SetType VolatileMask = AllMask & ~NonVolatileMask;
+
+ static const SetType WrapperMask = VolatileMask;
+
+ static const SetType NonAllocatableMask =
+ (SetType(1) << (FloatRegisters::f18 >> 1)) * ((1 << TotalSingle) + 1);
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+};
+
+class FloatRegister : public FloatRegisterMIPSShared {
+ public:
+ enum RegType {
+ Single = 0x0,
+ Double = 0x1,
+ };
+
+ typedef FloatRegisters Codes;
+ typedef Codes::Code Code;
+ typedef Codes::Encoding Encoding;
+
+ Encoding code_ : 6;
+
+ protected:
+ RegType kind_ : 1;
+
+ public:
+ constexpr FloatRegister(uint32_t code, RegType kind = Double)
+ : code_(Encoding(code)), kind_(kind) {}
+ constexpr FloatRegister()
+ : code_(FloatRegisters::invalid_freg), kind_(Double) {}
+
+ bool operator==(const FloatRegister& other) const {
+ MOZ_ASSERT(!isInvalid());
+ MOZ_ASSERT(!other.isInvalid());
+ return kind_ == other.kind_ && code_ == other.code_;
+ }
+ bool equiv(const FloatRegister& other) const { return other.kind_ == kind_; }
+ size_t size() const { return (kind_ == Double) ? 8 : 4; }
+ size_t pushSize() const { return size(); }
+
+ bool isNotOdd() const { return !isInvalid() && ((code_ & 1) == 0); }
+
+ bool isSingle() const { return kind_ == Single; }
+ bool isDouble() const { return kind_ == Double; }
+ bool isInvalid() const { return code_ == FloatRegisters::invalid_freg; }
+ bool isSimd128() const { return false; }
+
+ FloatRegister doubleOverlay() const;
+ FloatRegister singleOverlay() const;
+
+ FloatRegister asSingle() const { return singleOverlay(); }
+ FloatRegister asDouble() const { return doubleOverlay(); }
+ FloatRegister asSimd128() const { MOZ_CRASH("NYI"); }
+
+ Code code() const {
+ MOZ_ASSERT(isNotOdd());
+ return Code((code_ >> 1) | (kind_ << 4));
+ }
+ Encoding encoding() const {
+ MOZ_ASSERT(!isInvalid());
+ return code_;
+ }
+ uint32_t id() const {
+ MOZ_ASSERT(!isInvalid());
+ return code_;
+ }
+ static FloatRegister FromCode(uint32_t i) {
+ uint32_t code = i & 15;
+ uint32_t kind = i >> 4;
+ return FloatRegister(Encoding(code << 1), RegType(kind));
+ }
+
+ static FloatRegister FromIndex(uint32_t index, RegType kind) {
+ MOZ_ASSERT(index < 16);
+ return FloatRegister(Encoding(index << 1), kind);
+ }
+
+ bool volatile_() const {
+ return !!((SetType(1) << code()) & FloatRegisters::VolatileMask);
+ }
+ const char* name() const { return FloatRegisters::GetName(code_); }
+ bool operator!=(const FloatRegister& other) const {
+ return other.kind_ != kind_ || code_ != other.code_;
+ }
+ bool aliases(const FloatRegister& other) {
+ MOZ_ASSERT(isNotOdd());
+ return code_ == other.code_;
+ }
+ uint32_t numAliased() const {
+ MOZ_ASSERT(isNotOdd());
+ return 2;
+ }
+ FloatRegister aliased(uint32_t aliasIdx) {
+ MOZ_ASSERT(isNotOdd());
+
+ if (aliasIdx == 0) {
+ return *this;
+ }
+ MOZ_ASSERT(aliasIdx == 1);
+ if (isDouble()) {
+ return singleOverlay();
+ }
+ return doubleOverlay();
+ }
+ uint32_t numAlignedAliased() const {
+ MOZ_ASSERT(isNotOdd());
+ return 2;
+ }
+ FloatRegister alignedAliased(uint32_t aliasIdx) {
+ MOZ_ASSERT(isNotOdd());
+
+ if (aliasIdx == 0) {
+ return *this;
+ }
+ MOZ_ASSERT(aliasIdx == 1);
+ if (isDouble()) {
+ return singleOverlay();
+ }
+ return doubleOverlay();
+ }
+
+ SetType alignedOrDominatedAliasedSet() const {
+ MOZ_ASSERT(isNotOdd());
+ return (SetType(1) << (code_ >> 1)) *
+ ((1 << FloatRegisters::TotalSingle) + 1);
+ }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return LiveAsIndexableSet<Name>(s);
+ }
+
+ static Code FromName(const char* name) {
+ return FloatRegisters::FromName(name);
+ }
+ static TypedRegisterSet<FloatRegister> ReduceSetForPush(
+ const TypedRegisterSet<FloatRegister>& s);
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>& s);
+ uint32_t getRegisterDumpOffsetInBytes();
+};
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ return set & FloatRegisters::AllDoubleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Any>(SetType set) {
+ return set;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::AllocatableAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ // Single registers are not dominating any smaller registers, thus masking
+ // is enough to convert an allocatable set into a set of register list all
+ // single register available.
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::AllocatableAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ return set & FloatRegisters::AllDoubleMask;
+}
+
+// In order to handle functions such as int(*)(int, double) where the first
+// argument is a general purpose register, and the second argument is a floating
+// point register, we have to store the double content into 2 general purpose
+// registers, namely a2 and a3.
+#define JS_CODEGEN_REGISTER_PAIR 1
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_Architecture_mips32_h */
diff --git a/js/src/jit/mips32/Assembler-mips32.cpp b/js/src/jit/mips32/Assembler-mips32.cpp
new file mode 100644
index 0000000000..8073b8e4ec
--- /dev/null
+++ b/js/src/jit/mips32/Assembler-mips32.cpp
@@ -0,0 +1,369 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips32/Assembler-mips32.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Maybe.h"
+
+#include "jit/AutoWritableJitCode.h"
+
+using mozilla::DebugOnly;
+
+using namespace js;
+using namespace js::jit;
+
+ABIArgGenerator::ABIArgGenerator()
+ : usedArgSlots_(0),
+ firstArgFloatSize_(0),
+ useGPRForFloats_(false),
+ current_() {}
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+ Register destReg;
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ if (GetIntArgReg(usedArgSlots_, &destReg)) {
+ current_ = ABIArg(destReg);
+ } else {
+ current_ = ABIArg(usedArgSlots_ * sizeof(intptr_t));
+ }
+ usedArgSlots_++;
+ break;
+ case MIRType::Int64:
+ if (!usedArgSlots_) {
+ current_ = ABIArg(a0, a1);
+ usedArgSlots_ = 2;
+ } else if (usedArgSlots_ <= 2) {
+ current_ = ABIArg(a2, a3);
+ usedArgSlots_ = 4;
+ } else {
+ if (usedArgSlots_ < NumIntArgRegs) {
+ usedArgSlots_ = NumIntArgRegs;
+ }
+ usedArgSlots_ += usedArgSlots_ % 2;
+ current_ = ABIArg(usedArgSlots_ * sizeof(intptr_t));
+ usedArgSlots_ += 2;
+ }
+ break;
+ case MIRType::Float32:
+ if (!usedArgSlots_) {
+ current_ = ABIArg(f12.asSingle());
+ firstArgFloatSize_ = 1;
+ } else if (usedArgSlots_ == firstArgFloatSize_) {
+ current_ = ABIArg(f14.asSingle());
+ } else if (useGPRForFloats_ && GetIntArgReg(usedArgSlots_, &destReg)) {
+ current_ = ABIArg(destReg);
+ } else {
+ if (usedArgSlots_ < NumIntArgRegs) {
+ usedArgSlots_ = NumIntArgRegs;
+ }
+ current_ = ABIArg(usedArgSlots_ * sizeof(intptr_t));
+ }
+ usedArgSlots_++;
+ break;
+ case MIRType::Double:
+ if (!usedArgSlots_) {
+ current_ = ABIArg(f12);
+ usedArgSlots_ = 2;
+ firstArgFloatSize_ = 2;
+ } else if (usedArgSlots_ == firstArgFloatSize_) {
+ current_ = ABIArg(f14);
+ usedArgSlots_ = 4;
+ } else if (useGPRForFloats_ && usedArgSlots_ <= 2) {
+ current_ = ABIArg(a2, a3);
+ usedArgSlots_ = 4;
+ } else {
+ if (usedArgSlots_ < NumIntArgRegs) {
+ usedArgSlots_ = NumIntArgRegs;
+ }
+ usedArgSlots_ += usedArgSlots_ % 2;
+ current_ = ABIArg(usedArgSlots_ * sizeof(intptr_t));
+ usedArgSlots_ += 2;
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+}
+
+uint32_t js::jit::RT(FloatRegister r) {
+ MOZ_ASSERT(r.id() < FloatRegisters::RegisterIdLimit);
+ return r.id() << RTShift;
+}
+
+uint32_t js::jit::RD(FloatRegister r) {
+ MOZ_ASSERT(r.id() < FloatRegisters::RegisterIdLimit);
+ return r.id() << RDShift;
+}
+
+uint32_t js::jit::RZ(FloatRegister r) {
+ MOZ_ASSERT(r.id() < FloatRegisters::RegisterIdLimit);
+ return r.id() << RZShift;
+}
+
+uint32_t js::jit::SA(FloatRegister r) {
+ MOZ_ASSERT(r.id() < FloatRegisters::RegisterIdLimit);
+ return r.id() << SAShift;
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ MOZ_ASSERT(isFinished);
+ m_buffer.executableCopy(buffer);
+}
+
+uintptr_t Assembler::GetPointer(uint8_t* instPtr) {
+ Instruction* inst = (Instruction*)instPtr;
+ return Assembler::ExtractLuiOriValue(inst, inst->next());
+}
+
+static JitCode* CodeFromJump(Instruction* jump) {
+ uint8_t* target = (uint8_t*)Assembler::ExtractLuiOriValue(jump, jump->next());
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ while (reader.more()) {
+ JitCode* child =
+ CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned()));
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ }
+}
+
+static void TraceOneDataRelocation(JSTracer* trc,
+ mozilla::Maybe<AutoWritableJitCode>& awjc,
+ JitCode* code, Instruction* inst) {
+ void* ptr = (void*)Assembler::ExtractLuiOriValue(inst, inst->next());
+ void* prior = ptr;
+
+ // No barrier needed since these are constants.
+ TraceManuallyBarrieredGenericPointerEdge(
+ trc, reinterpret_cast<gc::Cell**>(&ptr), "jit-masm-ptr");
+ if (ptr != prior) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ AssemblerMIPSShared::UpdateLuiOriValue(inst, inst->next(), uint32_t(ptr));
+ }
+}
+
+/* static */
+void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ Instruction* inst = (Instruction*)(code->raw() + offset);
+ TraceOneDataRelocation(trc, awjc, code, inst);
+ }
+}
+
+Assembler::Condition Assembler::UnsignedCondition(Condition cond) {
+ switch (cond) {
+ case Zero:
+ case NonZero:
+ return cond;
+ case LessThan:
+ case Below:
+ return Below;
+ case LessThanOrEqual:
+ case BelowOrEqual:
+ return BelowOrEqual;
+ case GreaterThan:
+ case Above:
+ return Above;
+ case AboveOrEqual:
+ case GreaterThanOrEqual:
+ return AboveOrEqual;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+Assembler::Condition Assembler::ConditionWithoutEqual(Condition cond) {
+ switch (cond) {
+ case LessThan:
+ case LessThanOrEqual:
+ return LessThan;
+ case Below:
+ case BelowOrEqual:
+ return Below;
+ case GreaterThan:
+ case GreaterThanOrEqual:
+ return GreaterThan;
+ case Above:
+ case AboveOrEqual:
+ return Above;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) {
+ if (label.patchAt().bound()) {
+ auto mode = label.linkMode();
+ intptr_t offset = label.patchAt().offset();
+ intptr_t target = label.target().offset();
+
+ if (mode == CodeLabel::RawPointer) {
+ *reinterpret_cast<const void**>(rawCode + offset) = rawCode + target;
+ } else {
+ MOZ_ASSERT(mode == CodeLabel::MoveImmediate ||
+ mode == CodeLabel::JumpImmediate);
+ Instruction* inst = (Instruction*)(rawCode + offset);
+ AssemblerMIPSShared::UpdateLuiOriValue(inst, inst->next(),
+ (uint32_t)(rawCode + target));
+ }
+ }
+}
+
+void Assembler::bind(InstImm* inst, uintptr_t branch, uintptr_t target) {
+ int32_t offset = target - branch;
+ InstImm inst_bgezal = InstImm(op_regimm, zero, rt_bgezal, BOffImm16(0));
+ InstImm inst_beq = InstImm(op_beq, zero, zero, BOffImm16(0));
+
+ // If encoded offset is 4, then the jump must be short
+ if (BOffImm16(inst[0]).decode() == 4) {
+ MOZ_ASSERT(BOffImm16::IsInRange(offset));
+ inst[0].setBOffImm16(BOffImm16(offset));
+ inst[1].makeNop();
+ return;
+ }
+
+ // Generate the long jump for calls because return address has to be the
+ // address after the reserved block.
+ if (inst[0].encode() == inst_bgezal.encode()) {
+ addLongJump(BufferOffset(branch), BufferOffset(target));
+ Assembler::WriteLuiOriInstructions(inst, &inst[1], ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+ inst[2] = InstReg(op_special, ScratchRegister, zero, ra, ff_jalr).encode();
+ // There is 1 nop after this.
+ return;
+ }
+
+ if (BOffImm16::IsInRange(offset)) {
+ bool conditional = (inst[0].encode() != inst_bgezal.encode() &&
+ inst[0].encode() != inst_beq.encode());
+
+ inst[0].setBOffImm16(BOffImm16(offset));
+ inst[1].makeNop();
+
+ // Skip the trailing nops in conditional branches.
+ if (conditional) {
+ inst[2] = InstImm(op_regimm, zero, rt_bgez, BOffImm16(3 * sizeof(void*)))
+ .encode();
+ // There are 2 nops after this
+ }
+ return;
+ }
+
+ if (inst[0].encode() == inst_beq.encode()) {
+ // Handle long unconditional jump.
+ addLongJump(BufferOffset(branch), BufferOffset(target));
+ Assembler::WriteLuiOriInstructions(inst, &inst[1], ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+ inst[2] = InstReg(op_special, ScratchRegister, zero, zero, ff_jr).encode();
+ // There is 1 nop after this.
+ } else {
+ // Handle long conditional jump.
+ inst[0] = invertBranch(inst[0], BOffImm16(5 * sizeof(void*)));
+ // No need for a "nop" here because we can clobber scratch.
+ addLongJump(BufferOffset(branch + sizeof(void*)), BufferOffset(target));
+ Assembler::WriteLuiOriInstructions(&inst[1], &inst[2], ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+ inst[3] = InstReg(op_special, ScratchRegister, zero, zero, ff_jr).encode();
+ // There is 1 nop after this.
+ }
+}
+
+void Assembler::processCodeLabels(uint8_t* rawCode) {
+ for (const CodeLabel& label : codeLabels_) {
+ Bind(rawCode, label);
+ }
+}
+
+uint32_t Assembler::PatchWrite_NearCallSize() { return 4 * sizeof(uint32_t); }
+
+void Assembler::PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall) {
+ Instruction* inst = (Instruction*)start.raw();
+ uint8_t* dest = toCall.raw();
+
+ // Overwrite whatever instruction used to be here with a call.
+ // Always use long jump for two reasons:
+ // - Jump has to be the same size because of PatchWrite_NearCallSize.
+ // - Return address has to be at the end of replaced block.
+ // Short jump wouldn't be more efficient.
+ Assembler::WriteLuiOriInstructions(inst, &inst[1], ScratchRegister,
+ (uint32_t)dest);
+ inst[2] = InstReg(op_special, ScratchRegister, zero, ra, ff_jalr);
+ inst[3] = InstNOP();
+}
+
+uint32_t Assembler::ExtractLuiOriValue(Instruction* inst0, Instruction* inst1) {
+ InstImm* i0 = (InstImm*)inst0;
+ InstImm* i1 = (InstImm*)inst1;
+ MOZ_ASSERT(i0->extractOpcode() == ((uint32_t)op_lui >> OpcodeShift));
+ MOZ_ASSERT(i1->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ uint32_t value = i0->extractImm16Value() << 16;
+ value = value | i1->extractImm16Value();
+ return value;
+}
+
+void Assembler::WriteLuiOriInstructions(Instruction* inst0, Instruction* inst1,
+ Register reg, uint32_t value) {
+ *inst0 = InstImm(op_lui, zero, reg, Imm16::Upper(Imm32(value)));
+ *inst1 = InstImm(op_ori, reg, reg, Imm16::Lower(Imm32(value)));
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ ImmPtr newValue, ImmPtr expectedValue) {
+ PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value),
+ PatchedImmPtr(expectedValue.value));
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue) {
+ Instruction* inst = (Instruction*)label.raw();
+
+ // Extract old Value
+ DebugOnly<uint32_t> value = Assembler::ExtractLuiOriValue(&inst[0], &inst[1]);
+ MOZ_ASSERT(value == uint32_t(expectedValue.value));
+
+ // Replace with new value
+ AssemblerMIPSShared::UpdateLuiOriValue(inst, inst->next(),
+ uint32_t(newValue.value));
+}
+
+uint32_t Assembler::ExtractInstructionImmediate(uint8_t* code) {
+ InstImm* inst = (InstImm*)code;
+ return Assembler::ExtractLuiOriValue(inst, inst->next());
+}
+
+void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) {
+ Instruction* inst = (Instruction*)inst_.raw();
+ InstImm* i0 = (InstImm*)inst;
+ InstImm* i1 = (InstImm*)i0->next();
+ Instruction* i2 = (Instruction*)i1->next();
+
+ MOZ_ASSERT(i0->extractOpcode() == ((uint32_t)op_lui >> OpcodeShift));
+ MOZ_ASSERT(i1->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ if (enabled) {
+ InstReg jalr = InstReg(op_special, ScratchRegister, zero, ra, ff_jalr);
+ *i2 = jalr;
+ } else {
+ InstNOP nop;
+ *i2 = nop;
+ }
+}
diff --git a/js/src/jit/mips32/Assembler-mips32.h b/js/src/jit/mips32/Assembler-mips32.h
new file mode 100644
index 0000000000..496c382590
--- /dev/null
+++ b/js/src/jit/mips32/Assembler-mips32.h
@@ -0,0 +1,265 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_Assembler_mips32_h
+#define jit_mips32_Assembler_mips32_h
+
+#include <iterator>
+
+#include "jit/mips-shared/Assembler-mips-shared.h"
+
+#include "jit/mips32/Architecture-mips32.h"
+
+namespace js {
+namespace jit {
+
+static constexpr Register CallTempReg4 = t4;
+static constexpr Register CallTempReg5 = t5;
+
+static constexpr Register CallTempNonArgRegs[] = {t0, t1, t2, t3, t4};
+static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+class ABIArgGenerator {
+ unsigned usedArgSlots_;
+ unsigned firstArgFloatSize_;
+ // Note: This is not compliant with the system ABI. The Lowering phase
+ // expects to lower an MWasmParameter to only one register.
+ bool useGPRForFloats_;
+ ABIArg current_;
+
+ public:
+ ABIArgGenerator();
+ ABIArg next(MIRType argType);
+ ABIArg& current() { return current_; }
+
+ void enforceO32ABI() { useGPRForFloats_ = true; }
+
+ uint32_t stackBytesConsumedSoFar() const {
+ if (usedArgSlots_ <= 4) {
+ return ShadowStackSpace;
+ }
+
+ return usedArgSlots_ * sizeof(intptr_t);
+ }
+
+ void increaseStackOffset(uint32_t bytes) { MOZ_CRASH("NYI"); }
+};
+
+// These registers may be volatile or nonvolatile.
+static constexpr Register ABINonArgReg0 = t0;
+static constexpr Register ABINonArgReg1 = t1;
+static constexpr Register ABINonArgReg2 = t2;
+static constexpr Register ABINonArgReg3 = t3;
+
+// This register may be volatile or nonvolatile. Avoid f18 which is the
+// ScratchDoubleReg.
+static constexpr FloatRegister ABINonArgDoubleReg{FloatRegisters::f16,
+ FloatRegister::Double};
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = t0;
+static constexpr Register ABINonArgReturnReg1 = t1;
+static constexpr Register ABINonVolatileReg = s0;
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = t0;
+
+// TLS pointer argument register for WebAssembly functions. This must not alias
+// any other register used for passing function arguments or return values.
+// Preserved by WebAssembly functions.
+static constexpr Register InstanceReg = s5;
+
+// Registers used for asm.js/wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = t1;
+
+static constexpr Register InterpreterPCReg = t5;
+
+static constexpr Register JSReturnReg_Type = a3;
+static constexpr Register JSReturnReg_Data = a2;
+static constexpr Register64 ReturnReg64(v1, v0);
+static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::f0,
+ FloatRegister::Single};
+static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::f0,
+ FloatRegister::Double};
+static constexpr FloatRegister ScratchFloat32Reg = {FloatRegisters::f18,
+ FloatRegister::Single};
+static constexpr FloatRegister ScratchDoubleReg = {FloatRegisters::f18,
+ FloatRegister::Double};
+
+struct ScratchFloat32Scope : public AutoFloatRegisterScope {
+ explicit ScratchFloat32Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchFloat32Reg) {}
+};
+
+struct ScratchDoubleScope : public AutoFloatRegisterScope {
+ explicit ScratchDoubleScope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchDoubleReg) {}
+};
+
+static constexpr FloatRegister f0 = {FloatRegisters::f0, FloatRegister::Double};
+static constexpr FloatRegister f2 = {FloatRegisters::f2, FloatRegister::Double};
+static constexpr FloatRegister f4 = {FloatRegisters::f4, FloatRegister::Double};
+static constexpr FloatRegister f6 = {FloatRegisters::f6, FloatRegister::Double};
+static constexpr FloatRegister f8 = {FloatRegisters::f8, FloatRegister::Double};
+static constexpr FloatRegister f10 = {FloatRegisters::f10,
+ FloatRegister::Double};
+static constexpr FloatRegister f12 = {FloatRegisters::f12,
+ FloatRegister::Double};
+static constexpr FloatRegister f14 = {FloatRegisters::f14,
+ FloatRegister::Double};
+static constexpr FloatRegister f16 = {FloatRegisters::f16,
+ FloatRegister::Double};
+static constexpr FloatRegister f18 = {FloatRegisters::f18,
+ FloatRegister::Double};
+static constexpr FloatRegister f20 = {FloatRegisters::f20,
+ FloatRegister::Double};
+static constexpr FloatRegister f22 = {FloatRegisters::f22,
+ FloatRegister::Double};
+static constexpr FloatRegister f24 = {FloatRegisters::f24,
+ FloatRegister::Double};
+static constexpr FloatRegister f26 = {FloatRegisters::f26,
+ FloatRegister::Double};
+static constexpr FloatRegister f28 = {FloatRegisters::f28,
+ FloatRegister::Double};
+static constexpr FloatRegister f30 = {FloatRegisters::f30,
+ FloatRegister::Double};
+
+// MIPS CPUs can only load multibyte data that is "naturally"
+// four-byte-aligned, sp register should be eight-byte-aligned.
+static constexpr uint32_t ABIStackAlignment = 8;
+static constexpr uint32_t JitStackAlignment = 8;
+
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+ JitStackValueAlignment >= 1,
+ "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+// TODO this is just a filler to prevent a build failure. The MIPS SIMD
+// alignment requirements still need to be explored.
+// TODO Copy the static_asserts from x64/x86 assembler files.
+static constexpr uint32_t SimdMemoryAlignment = 8;
+static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static const uint32_t WasmTrapInstructionLength = 4;
+
+// See comments in wasm::GenerateFunctionPrologue. The difference between these
+// is the size of the largest callable prologue on the platform.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+static constexpr Scale ScalePointer = TimesFour;
+
+class Assembler : public AssemblerMIPSShared {
+ public:
+ Assembler() : AssemblerMIPSShared() {}
+
+ static Condition UnsignedCondition(Condition cond);
+ static Condition ConditionWithoutEqual(Condition cond);
+
+ static uintptr_t GetPointer(uint8_t*);
+
+ protected:
+ // This is used to access the odd register form the pair of single
+ // precision registers that make one double register.
+ FloatRegister getOddPair(FloatRegister reg) {
+ MOZ_ASSERT(reg.isDouble());
+ MOZ_ASSERT(reg.id() % 2 == 0);
+ FloatRegister odd(reg.id() | 1, FloatRegister::Single);
+ return odd;
+ }
+
+ public:
+ using AssemblerMIPSShared::bind;
+
+ static void Bind(uint8_t* rawCode, const CodeLabel& label);
+
+ void processCodeLabels(uint8_t* rawCode);
+
+ static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+ static void TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ void bind(InstImm* inst, uintptr_t branch, uintptr_t target);
+
+ // Copy the assembly code to the given buffer, and perform any pending
+ // relocations relying on the target address.
+ void executableCopy(uint8_t* buffer);
+
+ static uint32_t PatchWrite_NearCallSize();
+
+ static uint32_t ExtractLuiOriValue(Instruction* inst0, Instruction* inst1);
+ static void WriteLuiOriInstructions(Instruction* inst, Instruction* inst1,
+ Register reg, uint32_t value);
+
+ static void PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall);
+ static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
+ ImmPtr expectedValue);
+ static void PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue);
+
+ static uint32_t ExtractInstructionImmediate(uint8_t* code);
+
+ static void ToggleCall(CodeLocationLabel inst_, bool enabled);
+}; // Assembler
+
+static const uint32_t NumIntArgRegs = 4;
+
+static inline bool GetIntArgReg(uint32_t usedArgSlots, Register* out) {
+ if (usedArgSlots < NumIntArgRegs) {
+ *out = Register::FromCode(a0.code() + usedArgSlots);
+ return true;
+ }
+ return false;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs, Register* out) {
+ // NOTE: We can't properly determine which regs are used if there are
+ // float arguments. If this is needed, we will have to guess.
+ MOZ_ASSERT(usedFloatArgs == 0);
+
+ if (GetIntArgReg(usedIntArgs, out)) {
+ return true;
+ }
+ // Unfortunately, we have to assume things about the point at which
+ // GetIntArgReg returns false, because we need to know how many registers it
+ // can allocate.
+ usedIntArgs -= NumIntArgRegs;
+ if (usedIntArgs >= NumCallTempNonArgRegs) {
+ return false;
+ }
+ *out = CallTempNonArgRegs[usedIntArgs];
+ return true;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_Assembler_mips32_h */
diff --git a/js/src/jit/mips32/CodeGenerator-mips32.cpp b/js/src/jit/mips32/CodeGenerator-mips32.cpp
new file mode 100644
index 0000000000..b5bc0c041f
--- /dev/null
+++ b/js/src/jit/mips32/CodeGenerator-mips32.cpp
@@ -0,0 +1,507 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips32/CodeGenerator-mips32.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "js/Conversions.h"
+#include "vm/Shape.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+ValueOperand CodeGeneratorMIPS::ToValue(LInstruction* ins, size_t pos) {
+ Register typeReg = ToRegister(ins->getOperand(pos + TYPE_INDEX));
+ Register payloadReg = ToRegister(ins->getOperand(pos + PAYLOAD_INDEX));
+ return ValueOperand(typeReg, payloadReg);
+}
+
+ValueOperand CodeGeneratorMIPS::ToTempValue(LInstruction* ins, size_t pos) {
+ Register typeReg = ToRegister(ins->getTemp(pos + TYPE_INDEX));
+ Register payloadReg = ToRegister(ins->getTemp(pos + PAYLOAD_INDEX));
+ return ValueOperand(typeReg, payloadReg);
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+ const LDefinition* type = box->getDef(TYPE_INDEX);
+
+ MOZ_ASSERT(!box->getOperand(0)->isConstant());
+
+ // For NUNBOX32, the input operand and the output payload have the same
+ // virtual register. All that needs to be written is the type tag for
+ // the type definition.
+ masm.move32(Imm32(MIRTypeToTag(box->type())), ToRegister(type));
+}
+
+void CodeGenerator::visitBoxFloatingPoint(LBoxFloatingPoint* box) {
+ const AnyRegister in = ToAnyRegister(box->getOperand(0));
+ const ValueOperand out = ToOutValue(box);
+
+ masm.moveValue(TypedOrValueRegister(box->type(), in), out);
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+ // Note that for unbox, the type and payload indexes are switched on the
+ // inputs.
+ MUnbox* mir = unbox->mir();
+ Register type = ToRegister(unbox->type());
+
+ if (mir->fallible()) {
+ bailoutCmp32(Assembler::NotEqual, type, Imm32(MIRTypeToTag(mir->type())),
+ unbox->snapshot());
+ }
+}
+
+void CodeGeneratorMIPS::splitTagForTest(const ValueOperand& value,
+ ScratchTagScope& tag) {
+ MOZ_ASSERT(value.typeReg() == tag);
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+ MCompare* mir = lir->mir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register64 lhsRegs = ToRegister64(lhs);
+ Register output = ToRegister(lir->output());
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition condition = JSOpToCondition(lir->jsop(), isSigned);
+
+ if (IsConstant(rhs)) {
+ Imm64 imm = Imm64(ToInt64(rhs));
+ masm.cmp64Set(condition, lhsRegs, imm, output);
+ } else {
+ Register64 rhsRegs = ToRegister64(rhs);
+ masm.cmp64Set(condition, lhsRegs, rhsRegs, output);
+ }
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+ MCompare* mir = lir->cmpMir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register64 lhsRegs = ToRegister64(lhs);
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition condition = JSOpToCondition(lir->jsop(), isSigned);
+
+ Label* trueLabel = getJumpLabelForBranch(lir->ifTrue());
+ Label* falseLabel = getJumpLabelForBranch(lir->ifFalse());
+
+ if (isNextBlock(lir->ifFalse()->lir())) {
+ falseLabel = nullptr;
+ } else if (isNextBlock(lir->ifTrue()->lir())) {
+ condition = Assembler::InvertCondition(condition);
+ trueLabel = falseLabel;
+ falseLabel = nullptr;
+ }
+
+ if (IsConstant(rhs)) {
+ Imm64 imm = Imm64(ToInt64(rhs));
+ masm.branch64(condition, lhsRegs, imm, trueLabel, falseLabel);
+ } else {
+ Register64 rhsRegs = ToRegister64(rhs);
+ masm.branch64(condition, lhsRegs, rhsRegs, trueLabel, falseLabel);
+ }
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+ Register64 lhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Lhs));
+ Register64 rhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Rhs));
+ Register64 output = ToOutRegister64(lir);
+
+ MOZ_ASSERT(output == ReturnReg64);
+
+ Label done;
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.branchTest64(Assembler::NonZero, rhs, rhs, InvalidReg, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ // Handle an integer overflow exception from INT64_MIN / -1.
+ if (lir->canBeNegativeOverflow()) {
+ Label notOverflow;
+ masm.branch64(Assembler::NotEqual, lhs, Imm64(INT64_MIN), &notOverflow);
+ masm.branch64(Assembler::NotEqual, rhs, Imm64(-1), &notOverflow);
+ if (lir->mir()->isMod()) {
+ masm.xor64(output, output);
+ } else {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ masm.jump(&done);
+ masm.bind(&notOverflow);
+ }
+
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs.high);
+ masm.passABIArg(lhs.low);
+ masm.passABIArg(rhs.high);
+ masm.passABIArg(rhs.low);
+
+ MOZ_ASSERT(gen->compilingWasm());
+ if (lir->mir()->isMod()) {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::ModI64);
+ } else {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::DivI64);
+ }
+ MOZ_ASSERT(ReturnReg64 == output);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+ Register64 lhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Lhs));
+ Register64 rhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Rhs));
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ReturnReg64);
+
+ // Prevent divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.branchTest64(Assembler::NonZero, rhs, rhs, InvalidReg, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs.high);
+ masm.passABIArg(lhs.low);
+ masm.passABIArg(rhs.high);
+ masm.passABIArg(rhs.low);
+
+ MOZ_ASSERT(gen->compilingWasm());
+ if (lir->mir()->isMod()) {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::UModI64);
+ } else {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::UDivI64);
+ }
+}
+
+void CodeGeneratorMIPS::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+#ifdef MIPSR6
+ masm.as_div(/* result= */ dividend, dividend, divisor);
+#else
+ masm.as_div(dividend, divisor);
+ masm.as_mflo(dividend);
+#endif
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorMIPS::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+#ifdef MIPSR6
+ masm.as_mod(/* result= */ dividend, dividend, divisor);
+#else
+ masm.as_div(dividend, divisor);
+ masm.as_mfhi(dividend);
+#endif
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+template <typename T>
+void CodeGeneratorMIPS::emitWasmLoadI64(T* lir) {
+ const MWasmLoad* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (IsUnaligned(mir->access())) {
+ masm.wasmUnalignedLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()),
+ ptrScratch, ToOutRegister64(lir),
+ ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToOutRegister64(lir));
+ }
+}
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir) {
+ emitWasmLoadI64(lir);
+}
+
+void CodeGenerator::visitWasmUnalignedLoadI64(LWasmUnalignedLoadI64* lir) {
+ emitWasmLoadI64(lir);
+}
+
+template <typename T>
+void CodeGeneratorMIPS::emitWasmStoreI64(T* lir) {
+ const MWasmStore* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (IsUnaligned(mir->access())) {
+ masm.wasmUnalignedStoreI64(mir->access(), ToRegister64(lir->value()),
+ HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmStoreI64(mir->access(), ToRegister64(lir->value()), HeapReg,
+ ToRegister(lir->ptr()), ptrScratch);
+ }
+}
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir) {
+ emitWasmStoreI64(lir);
+}
+
+void CodeGenerator::visitWasmUnalignedStoreI64(LWasmUnalignedStoreI64* lir) {
+ emitWasmStoreI64(lir);
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ Register cond = ToRegister(lir->condExpr());
+ const LInt64Allocation trueExpr = lir->trueExpr();
+ const LInt64Allocation falseExpr = lir->falseExpr();
+
+ Register64 output = ToOutRegister64(lir);
+
+ masm.move64(ToRegister64(trueExpr), output);
+
+ if (falseExpr.low().isRegister()) {
+ masm.as_movz(output.low, ToRegister(falseExpr.low()), cond);
+ masm.as_movz(output.high, ToRegister(falseExpr.high()), cond);
+ } else {
+ Label done;
+ masm.ma_b(cond, cond, &done, Assembler::NonZero, ShortJump);
+ masm.loadPtr(ToAddress(falseExpr.low()), output.low);
+ masm.loadPtr(ToAddress(falseExpr.high()), output.high);
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ masm.moveToDoubleLo(input.low, output);
+ masm.moveToDoubleHi(input.high, output);
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+ FloatRegister input = ToFloatRegister(lir->getOperand(0));
+ Register64 output = ToOutRegister64(lir);
+
+ masm.moveFromDoubleLo(input, output.low);
+ masm.moveFromDoubleHi(input, output.high);
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ Register input = ToRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ if (input != output.low) {
+ masm.move32(input, output.low);
+ }
+ if (lir->mir()->isUnsigned()) {
+ masm.move32(Imm32(0), output.high);
+ } else {
+ masm.ma_sra(output.high, output.low, Imm32(31));
+ }
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ const LInt64Allocation& input = lir->getInt64Operand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ masm.move32(ToRegister(input.low()), output);
+ } else {
+ masm.move32(ToRegister(input.high()), output);
+ }
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ switch (lir->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.move8SignExtend(input.low, output.low);
+ break;
+ case MSignExtendInt64::Half:
+ masm.move16SignExtend(input.low, output.low);
+ break;
+ case MSignExtendInt64::Word:
+ masm.move32(input.low, output.low);
+ break;
+ }
+ masm.ma_sra(output.high, output.low, Imm32(31));
+}
+
+void CodeGenerator::visitClzI64(LClzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.clz64(input, output.low);
+ masm.move32(Imm32(0), output.high);
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.ctz64(input, output.low);
+ masm.move32(Imm32(0), output.high);
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register output = ToRegister(lir->output());
+
+ masm.as_or(output, input.low, input.high);
+ masm.cmp32Set(Assembler::Equal, output, Imm32(0), output);
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister arg = input;
+ Register64 output = ToOutRegister64(lir);
+ MWasmTruncateToInt64* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ auto* ool = new (alloc())
+ OutOfLineWasmTruncateCheck(mir, input, Register64::Invalid());
+ addOutOfLineCode(ool, mir);
+
+ if (fromType == MIRType::Float32) {
+ arg = ScratchDoubleReg;
+ masm.convertFloat32ToDouble(input, arg);
+ }
+
+ if (!lir->mir()->isSaturating()) {
+ masm.Push(input);
+
+ masm.setupWasmABICall();
+ masm.passABIArg(arg, MoveOp::DOUBLE);
+
+ if (lir->mir()->isUnsigned()) {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::TruncateDoubleToUint64);
+ } else {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::TruncateDoubleToInt64);
+ }
+
+ masm.Pop(input);
+
+ masm.ma_xor(ScratchRegister, output.high, Imm32(0x80000000));
+ masm.ma_or(ScratchRegister, output.low);
+ masm.ma_b(ScratchRegister, Imm32(0), ool->entry(), Assembler::Equal);
+
+ masm.bind(ool->rejoin());
+ } else {
+ masm.setupWasmABICall();
+ masm.passABIArg(arg, MoveOp::DOUBLE);
+ if (lir->mir()->isUnsigned()) {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::SaturatingTruncateDoubleToUint64);
+ } else {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::SaturatingTruncateDoubleToInt64);
+ }
+ }
+
+ MOZ_ASSERT(ReturnReg64 == output);
+}
+
+void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ mozilla::DebugOnly<FloatRegister> output = ToFloatRegister(lir->output());
+
+ MInt64ToFloatingPoint* mir = lir->mir();
+ MIRType toType = mir->type();
+
+ masm.setupWasmABICall();
+ masm.passABIArg(input.high);
+ masm.passABIArg(input.low);
+
+ if (lir->mir()->isUnsigned()) {
+ if (toType == MIRType::Double) {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::Uint64ToDouble, MoveOp::DOUBLE);
+ } else {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::Uint64ToFloat32, MoveOp::FLOAT32);
+ }
+ } else {
+ if (toType == MIRType::Double) {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::Int64ToDouble, MoveOp::DOUBLE);
+ } else {
+ masm.callWithABI(mir->bytecodeOffset(),
+ wasm::SymbolicAddress::Int64ToFloat32, MoveOp::FLOAT32);
+ }
+ }
+
+ MOZ_ASSERT_IF(toType == MIRType::Double, *(&output) == ReturnDoubleReg);
+ MOZ_ASSERT_IF(toType == MIRType::Float32, *(&output) == ReturnFloat32Reg);
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+
+ branchToBlock(input.high, Imm32(0), lir->ifTrue(), Assembler::NonZero);
+ emitBranch(input.low, Imm32(0), Assembler::NonZero, lir->ifTrue(),
+ lir->ifFalse());
+}
+
+void CodeGenerator::visitWasmAtomicLoadI64(LWasmAtomicLoadI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 output = ToOutRegister64(lir);
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+
+ masm.wasmAtomicLoad64(lir->mir()->access(), addr, Register64::Invalid(),
+ output);
+}
+
+void CodeGenerator::visitWasmAtomicStoreI64(LWasmAtomicStoreI64* lir) {
+ Register ptr = ToRegister(lir->ptr());
+ Register64 value = ToRegister64(lir->value());
+ Register tmp = ToRegister(lir->tmp());
+ uint32_t offset = lir->mir()->access().offset();
+
+ BaseIndex addr(HeapReg, ptr, TimesOne, offset);
+
+ masm.wasmAtomicStore64(lir->mir()->access(), addr, tmp, value);
+}
diff --git a/js/src/jit/mips32/CodeGenerator-mips32.h b/js/src/jit/mips32/CodeGenerator-mips32.h
new file mode 100644
index 0000000000..161b94326f
--- /dev/null
+++ b/js/src/jit/mips32/CodeGenerator-mips32.h
@@ -0,0 +1,60 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_CodeGenerator_mips32_h
+#define jit_mips32_CodeGenerator_mips32_h
+
+#include "jit/mips-shared/CodeGenerator-mips-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorMIPS : public CodeGeneratorMIPSShared {
+ protected:
+ CodeGeneratorMIPS(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
+ : CodeGeneratorMIPSShared(gen, graph, masm) {}
+
+ void testNullEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ emitBranch(value.typeReg(), (Imm32)ImmType(JSVAL_TYPE_NULL), cond, ifTrue,
+ ifFalse);
+ }
+ void testUndefinedEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ emitBranch(value.typeReg(), (Imm32)ImmType(JSVAL_TYPE_UNDEFINED), cond,
+ ifTrue, ifFalse);
+ }
+ void testObjectEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ emitBranch(value.typeReg(), (Imm32)ImmType(JSVAL_TYPE_OBJECT), cond, ifTrue,
+ ifFalse);
+ }
+
+ void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+
+ template <typename T>
+ void emitWasmLoadI64(T* ins);
+ template <typename T>
+ void emitWasmStoreI64(T* ins);
+
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ // Functions for LTestVAndBranch.
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag);
+};
+
+typedef CodeGeneratorMIPS CodeGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_CodeGenerator_mips32_h */
diff --git a/js/src/jit/mips32/LIR-mips32.h b/js/src/jit/mips32/LIR-mips32.h
new file mode 100644
index 0000000000..da68ad7464
--- /dev/null
+++ b/js/src/jit/mips32/LIR-mips32.h
@@ -0,0 +1,197 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_LIR_mips32_h
+#define jit_mips32_LIR_mips32_h
+
+namespace js {
+namespace jit {
+
+class LBoxFloatingPoint : public LInstructionHelper<2, 1, 1> {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(BoxFloatingPoint);
+
+ LBoxFloatingPoint(const LAllocation& in, const LDefinition& temp,
+ MIRType type)
+ : LInstructionHelper(classOpcode), type_(type) {
+ setOperand(0, in);
+ setTemp(0, temp);
+ }
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+class LUnbox : public LInstructionHelper<1, 2, 0> {
+ public:
+ LIR_HEADER(Unbox);
+
+ LUnbox() : LInstructionHelper(classOpcode) {}
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+ const LAllocation* payload() { return getOperand(0); }
+ const LAllocation* type() { return getOperand(1); }
+ const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LInstructionHelper<1, 2, 0> {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint);
+
+ static const size_t Input = 0;
+
+ LUnboxFloatingPoint(const LBoxAllocation& input, MIRType type)
+ : LInstructionHelper(classOpcode), type_(type) {
+ setBoxOperand(Input, input);
+ }
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+class LDivOrModI64
+ : public LCallInstructionHelper<INT64_PIECES, INT64_PIECES * 2, 0> {
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+
+ LDivOrModI64(const LInt64Allocation& lhs, const LInt64Allocation& rhs)
+ : LCallInstructionHelper(classOpcode) {
+ setInt64Operand(Lhs, lhs);
+ setInt64Operand(Rhs, rhs);
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LUDivOrModI64
+ : public LCallInstructionHelper<INT64_PIECES, INT64_PIECES * 2, 0> {
+ public:
+ LIR_HEADER(UDivOrModI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+
+ LUDivOrModI64(const LInt64Allocation& lhs, const LInt64Allocation& rhs)
+ : LCallInstructionHelper(classOpcode) {
+ setInt64Operand(Lhs, lhs);
+ setInt64Operand(Rhs, rhs);
+ }
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LWasmTruncateToInt64 : public LCallInstructionHelper<INT64_PIECES, 1, 0> {
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ explicit LWasmTruncateToInt64(const LAllocation& in)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ }
+
+ MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); }
+};
+
+class LInt64ToFloatingPoint
+ : public LCallInstructionHelper<1, INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(Int64ToFloatingPoint);
+
+ explicit LInt64ToFloatingPoint(const LInt64Allocation& in)
+ : LCallInstructionHelper(classOpcode) {
+ setInt64Operand(0, in);
+ }
+
+ MInt64ToFloatingPoint* mir() const { return mir_->toInt64ToFloatingPoint(); }
+};
+
+class LWasmAtomicLoadI64 : public LInstructionHelper<INT64_PIECES, 1, 0> {
+ public:
+ LIR_HEADER(WasmAtomicLoadI64);
+
+ LWasmAtomicLoadI64(const LAllocation& ptr) : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const MWasmLoad* mir() const { return mir_->toWasmLoad(); }
+};
+
+class LWasmAtomicStoreI64 : public LInstructionHelper<0, 1 + INT64_PIECES, 1> {
+ public:
+ LIR_HEADER(WasmAtomicStoreI64);
+
+ LWasmAtomicStoreI64(const LAllocation& ptr, const LInt64Allocation& value,
+ const LDefinition& tmp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setInt64Operand(1, value);
+ setTemp(0, tmp);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LInt64Allocation value() { return getInt64Operand(1); }
+ const LDefinition* tmp() { return getTemp(0); }
+ const MWasmStore* mir() const { return mir_->toWasmStore(); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_LIR_mips32_h */
diff --git a/js/src/jit/mips32/Lowering-mips32.cpp b/js/src/jit/mips32/Lowering-mips32.cpp
new file mode 100644
index 0000000000..721491c46b
--- /dev/null
+++ b/js/src/jit/mips32/Lowering-mips32.cpp
@@ -0,0 +1,257 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips32/Lowering-mips32.h"
+
+#include "jit/Lowering.h"
+#include "jit/mips32/Assembler-mips32.h"
+#include "jit/MIR.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+LBoxAllocation LIRGeneratorMIPS::useBoxFixed(MDefinition* mir, Register reg1,
+ Register reg2, bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+ MOZ_ASSERT(reg1 != reg2);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart),
+ LUse(reg2, VirtualRegisterOfPayload(mir), useAtStart));
+}
+
+void LIRGenerator::visitBox(MBox* box) {
+ MDefinition* inner = box->getOperand(0);
+
+ // If the box wrapped a double, it needs a new register.
+ if (IsFloatingPointType(inner->type())) {
+ defineBox(new (alloc()) LBoxFloatingPoint(
+ useRegisterAtStart(inner), tempCopy(inner, 0), inner->type()),
+ box);
+ return;
+ }
+
+ if (box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (inner->isConstant()) {
+ defineBox(new (alloc()) LValue(inner->toConstant()->toJSValue()), box);
+ return;
+ }
+
+ LBox* lir = new (alloc()) LBox(use(inner), inner->type());
+
+ // Otherwise, we should not define a new register for the payload portion
+ // of the output, so bypass defineBox().
+ uint32_t vreg = getVirtualRegister();
+
+ // Note that because we're using BogusTemp(), we do not change the type of
+ // the definition. We also do not define the first output as "TYPE",
+ // because it has no corresponding payload at (vreg + 1). Also note that
+ // although we copy the input's original type for the payload half of the
+ // definition, this is only for clarity. BogusTemp() definitions are
+ // ignored.
+ lir->setDef(0, LDefinition(vreg, LDefinition::GENERAL));
+ lir->setDef(1, LDefinition::BogusTemp());
+ box->setVirtualRegister(vreg);
+ add(lir);
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+ MDefinition* inner = unbox->getOperand(0);
+
+ // An unbox on mips reads in a type tag (either in memory or a register) and
+ // a payload. Unlike most instructions consuming a box, we ask for the type
+ // second, so that the result can re-use the first input.
+ MOZ_ASSERT(inner->type() == MIRType::Value);
+
+ ensureDefined(inner);
+
+ if (IsFloatingPointType(unbox->type())) {
+ LUnboxFloatingPoint* lir =
+ new (alloc()) LUnboxFloatingPoint(useBox(inner), unbox->type());
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+ define(lir, unbox);
+ return;
+ }
+
+ // Swap the order we use the box pieces so we can re-use the payload
+ // register.
+ LUnbox* lir = new (alloc()) LUnbox;
+ lir->setOperand(0, usePayloadInRegisterAtStart(inner));
+ lir->setOperand(1, useType(inner, LUse::REGISTER));
+
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+
+ // Types and payloads form two separate intervals. If the type becomes dead
+ // before the payload, it could be used as a Value without the type being
+ // recoverable. Unbox's purpose is to eagerly kill the definition of a type
+ // tag, so keeping both alive (for the purpose of gcmaps) is unappealing.
+ // Instead, we create a new virtual register.
+ defineReuseInput(lir, unbox, 0);
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new (alloc()) LReturn(isGenerator);
+ ins->setOperand(0, LUse(JSReturnReg_Type));
+ ins->setOperand(1, LUse(JSReturnReg_Data));
+ fillBoxUses(ins, 0, opd);
+ add(ins);
+}
+
+void LIRGeneratorMIPS::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ MDefinition* operand = phi->getOperand(inputPosition);
+ LPhi* type = block->getPhi(lirIndex + VREG_TYPE_OFFSET);
+ LPhi* payload = block->getPhi(lirIndex + VREG_DATA_OFFSET);
+ type->setOperand(
+ inputPosition,
+ LUse(operand->virtualRegister() + VREG_TYPE_OFFSET, LUse::ANY));
+ payload->setOperand(inputPosition,
+ LUse(VirtualRegisterOfPayload(operand), LUse::ANY));
+}
+
+void LIRGeneratorMIPS::defineInt64Phi(MPhi* phi, size_t lirIndex) {
+ LPhi* low = current->getPhi(lirIndex + INT64LOW_INDEX);
+ LPhi* high = current->getPhi(lirIndex + INT64HIGH_INDEX);
+
+ uint32_t lowVreg = getVirtualRegister();
+
+ phi->setVirtualRegister(lowVreg);
+
+ uint32_t highVreg = getVirtualRegister();
+ MOZ_ASSERT(lowVreg + INT64HIGH_INDEX == highVreg + INT64LOW_INDEX);
+
+ low->setDef(0, LDefinition(lowVreg, LDefinition::INT32));
+ high->setDef(0, LDefinition(highVreg, LDefinition::INT32));
+ annotate(high);
+ annotate(low);
+}
+
+void LIRGeneratorMIPS::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ MDefinition* operand = phi->getOperand(inputPosition);
+ LPhi* low = block->getPhi(lirIndex + INT64LOW_INDEX);
+ LPhi* high = block->getPhi(lirIndex + INT64HIGH_INDEX);
+ low->setOperand(inputPosition,
+ LUse(operand->virtualRegister() + INT64LOW_INDEX, LUse::ANY));
+ high->setOperand(
+ inputPosition,
+ LUse(operand->virtualRegister() + INT64HIGH_INDEX, LUse::ANY));
+}
+
+void LIRGeneratorMIPS::lowerTruncateDToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double);
+
+ define(new (alloc())
+ LTruncateDToInt32(useRegister(opd), LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGeneratorMIPS::lowerTruncateFToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Float32);
+
+ define(new (alloc())
+ LTruncateFToInt32(useRegister(opd), LDefinition::BogusTemp()),
+ ins);
+}
+
+void LIRGeneratorMIPS::lowerDivI64(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDivI64(div);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+ useInt64RegisterAtStart(div->lhs()), useInt64RegisterAtStart(div->rhs()));
+
+ defineReturn(lir, div);
+}
+
+void LIRGeneratorMIPS::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ MOZ_CRASH("We don't use runtime div for this architecture");
+}
+
+void LIRGeneratorMIPS::lowerModI64(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUModI64(mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+ useInt64RegisterAtStart(mod->lhs()), useInt64RegisterAtStart(mod->rhs()));
+
+ defineReturn(lir, mod);
+}
+
+void LIRGeneratorMIPS::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ MOZ_CRASH("We don't use runtime mod for this architecture");
+}
+
+void LIRGeneratorMIPS::lowerUDivI64(MDiv* div) {
+ LUDivOrModI64* lir = new (alloc()) LUDivOrModI64(
+ useInt64RegisterAtStart(div->lhs()), useInt64RegisterAtStart(div->rhs()));
+ defineReturn(lir, div);
+}
+
+void LIRGeneratorMIPS::lowerUModI64(MMod* mod) {
+ LUDivOrModI64* lir = new (alloc()) LUDivOrModI64(
+ useInt64RegisterAtStart(mod->lhs()), useInt64RegisterAtStart(mod->rhs()));
+ defineReturn(lir, mod);
+}
+
+void LIRGeneratorMIPS::lowerBigIntDiv(MBigIntDiv* ins) {
+ auto* lir = new (alloc()) LBigIntDiv(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorMIPS::lowerBigIntMod(MBigIntMod* ins) {
+ auto* lir = new (alloc()) LBigIntMod(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ defineReturn(new (alloc()) LWasmTruncateToInt64(useRegisterAtStart(opd)),
+ ins);
+}
+
+void LIRGeneratorMIPS::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Int64);
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+ defineReturn(
+ new (alloc()) LInt64ToFloatingPoint(useInt64RegisterAtStart(opd)), ins);
+}
+
+void LIRGeneratorMIPS::lowerBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
diff --git a/js/src/jit/mips32/Lowering-mips32.h b/js/src/jit/mips32/Lowering-mips32.h
new file mode 100644
index 0000000000..1565c84656
--- /dev/null
+++ b/js/src/jit/mips32/Lowering-mips32.h
@@ -0,0 +1,54 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_Lowering_mips32_h
+#define jit_mips32_Lowering_mips32_h
+
+#include "jit/mips-shared/Lowering-mips-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorMIPS : public LIRGeneratorMIPSShared {
+ protected:
+ LIRGeneratorMIPS(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorMIPSShared(gen, graph, lirGraph) {}
+
+ // Returns a box allocation with type set to reg1 and payload set to reg2.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2,
+ bool useAtStart = false);
+
+ inline LDefinition tempToUnbox() { return LDefinition::BogusTemp(); }
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+
+ void lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void defineInt64Phi(MPhi* phi, size_t lirIndex);
+
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+ void lowerTruncateDToInt32(MTruncateToInt32* ins);
+ void lowerTruncateFToInt32(MTruncateToInt32* ins);
+
+ void lowerDivI64(MDiv* div);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+ void lowerModI64(MMod* mod);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+ void lowerUDivI64(MDiv* div);
+ void lowerUModI64(MMod* mod);
+
+ void lowerBigIntDiv(MBigIntDiv* ins);
+ void lowerBigIntMod(MBigIntMod* ins);
+};
+
+typedef LIRGeneratorMIPS LIRGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_Lowering_mips32_h */
diff --git a/js/src/jit/mips32/MacroAssembler-mips32-inl.h b/js/src/jit/mips32/MacroAssembler-mips32-inl.h
new file mode 100644
index 0000000000..e0e33bdc41
--- /dev/null
+++ b/js/src/jit/mips32/MacroAssembler-mips32-inl.h
@@ -0,0 +1,1022 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_MacroAssembler_mips32_inl_h
+#define jit_mips32_MacroAssembler_mips32_inl_h
+
+#include "jit/mips32/MacroAssembler-mips32.h"
+
+#include "vm/BigIntType.h" // JS::BigInt
+
+#include "jit/mips-shared/MacroAssembler-mips-shared-inl.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+ move32(src.low, dest.low);
+ move32(src.high, dest.high);
+}
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+ move32(Imm32(imm.value & 0xFFFFFFFFL), dest.low);
+ move32(Imm32((imm.value >> 32) & 0xFFFFFFFFL), dest.high);
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ moveFromDoubleHi(src, dest.high);
+ moveFromDoubleLo(src, dest.low);
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ moveToDoubleHi(src.high, dest);
+ moveToDoubleLo(src.low, dest);
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+ if (src.low != dest) {
+ move32(src.low, dest);
+ }
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ if (src != dest.low) {
+ move32(src, dest.low);
+ }
+ move32(Imm32(0), dest.high);
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ move8SignExtend(src, dest.low);
+ move32To64SignExtend(dest.low, dest);
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ move16SignExtend(src, dest.low);
+ move32To64SignExtend(dest.low, dest);
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ if (src != dest.low) {
+ move32(src, dest.low);
+ }
+ ma_sra(dest.high, dest.low, Imm32(31));
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ move32(src, dest);
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ move32(src, dest);
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ load32(src, dest);
+}
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::notPtr(Register reg) { ma_not(reg, reg); }
+
+void MacroAssembler::andPtr(Register src, Register dest) { ma_and(dest, src); }
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) { ma_and(dest, imm); }
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+ if (imm.low().value != int32_t(0xFFFFFFFF)) {
+ and32(imm.low(), dest.low);
+ }
+ if (imm.hi().value != int32_t(0xFFFFFFFF)) {
+ and32(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+ and32(src.low, dest.low);
+ and32(src.high, dest.high);
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+ if (imm.low().value) {
+ or32(imm.low(), dest.low);
+ }
+ if (imm.hi().value) {
+ or32(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+ if (imm.low().value) {
+ xor32(imm.low(), dest.low);
+ }
+ if (imm.hi().value) {
+ xor32(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) { ma_or(dest, src); }
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) { ma_or(dest, imm); }
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+ or32(src.low, dest.low);
+ or32(src.high, dest.high);
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+ ma_xor(dest.low, src.low);
+ ma_xor(dest.high, src.high);
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) { ma_xor(dest, src); }
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) { ma_xor(dest, imm); }
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap64(Register64 reg) {
+ byteSwap32(reg.high);
+ byteSwap32(reg.low);
+
+ // swap reg.high and reg.low.
+ ma_xor(reg.high, reg.low);
+ ma_xor(reg.low, reg.high);
+ ma_xor(reg.high, reg.low);
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(Register src, Register dest) { ma_addu(dest, src); }
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) { ma_addu(dest, imm); }
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+ addPtr(Imm32(imm.value), dest);
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+ if (dest.low == src.low) {
+ as_sltu(ScratchRegister, src.low, zero);
+ as_addu(dest.low, dest.low, src.low);
+ } else {
+ as_addu(dest.low, dest.low, src.low);
+ as_sltu(ScratchRegister, dest.low, src.low);
+ }
+ as_addu(dest.high, dest.high, src.high);
+ as_addu(dest.high, dest.high, ScratchRegister);
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) {
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_addiu(dest.low, dest.low, imm.value);
+ as_sltiu(ScratchRegister, dest.low, imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_addu(dest.low, dest.low, ScratchRegister);
+ as_sltu(ScratchRegister, dest.low, ScratchRegister);
+ }
+ as_addu(dest.high, dest.high, ScratchRegister);
+}
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+ add64(imm.low(), dest);
+ ma_addu(dest.high, dest.high, imm.hi());
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ ma_liPatchable(dest, Imm32(0));
+ as_subu(dest, StackPointer, dest);
+ return offset;
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ Instruction* lui =
+ (Instruction*)m_buffer.getInst(BufferOffset(offset.offset()));
+ MOZ_ASSERT(lui->extractOpcode() == ((uint32_t)op_lui >> OpcodeShift));
+ MOZ_ASSERT(lui->next()->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ UpdateLuiOriValue(lui, lui->next(), imm.value);
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) {
+ as_subu(dest, dest, src);
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) {
+ ma_subu(dest, dest, imm);
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+ MOZ_ASSERT(dest.low != src.high);
+ MOZ_ASSERT(dest.high != src.low);
+ MOZ_ASSERT(dest.high != src.high);
+
+ as_sltu(ScratchRegister, dest.low, src.low);
+ as_subu(dest.high, dest.high, ScratchRegister);
+ as_subu(dest.low, dest.low, src.low);
+ as_subu(dest.high, dest.high, src.high);
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+ if (Imm16::IsInSignedRange(imm.low().value) &&
+ Imm16::IsInSignedRange(-imm.value)) {
+ as_sltiu(ScratchRegister, dest.low, imm.low().value);
+ as_subu(dest.high, dest.high, ScratchRegister);
+ as_addiu(dest.low, dest.low, -imm.value);
+ } else {
+ ma_li(SecondScratchReg, imm.low());
+ as_sltu(ScratchRegister, dest.low, SecondScratchReg);
+ as_subu(dest.high, dest.high, ScratchRegister);
+ as_subu(dest.low, dest.low, SecondScratchReg);
+ }
+ ma_subu(dest.high, dest.high, imm.hi());
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ MOZ_CRASH("NYI");
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+ as_mul(srcDest, srcDest, rhs);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+ // LOW32 = LOW(LOW(dest) * LOW(imm));
+ // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
+ // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
+ // + HIGH(LOW(dest) * LOW(imm)) [carry]
+
+ if (imm.low().value == 5) {
+ // Optimized case for Math.random().
+ as_sll(ScratchRegister, dest.low, 2);
+ as_srl(SecondScratchReg, dest.low, 32 - 2);
+ as_addu(dest.low, ScratchRegister, dest.low);
+ as_sltu(ScratchRegister, dest.low, ScratchRegister);
+ as_addu(ScratchRegister, ScratchRegister, SecondScratchReg);
+ as_sll(SecondScratchReg, dest.high, 2);
+ as_addu(SecondScratchReg, SecondScratchReg, dest.high);
+ as_addu(dest.high, ScratchRegister, SecondScratchReg);
+ } else {
+ // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
+ // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
+ ma_li(ScratchRegister, imm.low());
+ as_mult(dest.high, ScratchRegister);
+ ma_li(ScratchRegister, imm.hi());
+ as_madd(dest.low, ScratchRegister);
+ as_mflo(dest.high);
+ // + HIGH(LOW(dest) * LOW(imm)) [carry]
+ // LOW32 = LOW(LOW(dest) * LOW(imm));
+ ma_li(ScratchRegister, imm.low());
+ as_multu(dest.low, ScratchRegister);
+ as_mfhi(ScratchRegister);
+ as_mflo(dest.low);
+ as_addu(dest.high, dest.high, ScratchRegister);
+ }
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
+ const Register temp) {
+ // LOW32 = LOW(LOW(dest) * LOW(imm));
+ // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
+ // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
+ // + HIGH(LOW(dest) * LOW(imm)) [carry]
+
+ MOZ_ASSERT(temp != dest.high && temp != dest.low);
+
+ // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
+ // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
+ ma_li(ScratchRegister, imm.low());
+ as_mult(dest.high, ScratchRegister);
+ ma_li(temp, imm.hi());
+ as_madd(dest.low, temp);
+ as_mflo(dest.high);
+ // + HIGH(LOW(dest) * LOW(imm)) [carry]
+ // LOW32 = LOW(LOW(dest) * LOW(imm));
+ as_multu(dest.low, ScratchRegister);
+ as_mfhi(ScratchRegister);
+ as_mflo(dest.low);
+ as_addu(dest.high, dest.high, ScratchRegister);
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ // LOW32 = LOW(LOW(dest) * LOW(imm));
+ // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
+ // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
+ // + HIGH(LOW(dest) * LOW(imm)) [carry]
+
+ MOZ_ASSERT(dest != src);
+ MOZ_ASSERT(dest.low != src.high && dest.high != src.low);
+
+ // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits]
+ // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits]
+ as_mult(dest.high, src.low);
+ as_madd(dest.low, src.high);
+ as_mflo(dest.high);
+ // + HIGH(LOW(dest) * LOW(src)) [carry]
+ // LOW32 = LOW(LOW(dest) * LOW(src));
+ as_multu(dest.low, src.low);
+ as_mfhi(ScratchRegister);
+ as_mflo(dest.low);
+ as_addu(dest.high, dest.high, ScratchRegister);
+}
+
+void MacroAssembler::neg64(Register64 reg) {
+ as_subu(ScratchRegister, zero, reg.low);
+ as_sltu(ScratchRegister, reg.low, ScratchRegister);
+ as_subu(reg.high, zero, reg.high);
+ as_subu(reg.high, reg.high, ScratchRegister);
+}
+
+void MacroAssembler::negPtr(Register reg) { as_subu(reg, zero, reg); }
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+ MOZ_ASSERT(src != ScratchRegister);
+ as_addu(ScratchRegister, src, src);
+ as_addu(dest, ScratchRegister, src);
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+ ma_li(ScratchRegister, Imm32((int32_t)dest.addr));
+ as_lw(SecondScratchReg, ScratchRegister, 0);
+
+ as_addiu(SecondScratchReg, SecondScratchReg, 1);
+ as_sw(SecondScratchReg, ScratchRegister, 0);
+
+ as_sltiu(SecondScratchReg, SecondScratchReg, 1);
+ as_lw(ScratchRegister, ScratchRegister, 4);
+
+ as_addu(SecondScratchReg, ScratchRegister, SecondScratchReg);
+
+ ma_li(ScratchRegister, Imm32((int32_t)dest.addr));
+ as_sw(SecondScratchReg, ScratchRegister, 4);
+}
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ ma_sll(dest, dest, imm);
+}
+
+void MacroAssembler::lshiftPtr(Register src, Register dest) {
+ ma_sll(dest, dest, src);
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ScratchRegisterScope scratch(*this);
+
+ if (imm.value == 0) {
+ return;
+ } else if (imm.value < 32) {
+ as_sll(dest.high, dest.high, imm.value);
+ as_srl(scratch, dest.low, (32 - imm.value) % 32);
+ as_or(dest.high, dest.high, scratch);
+ as_sll(dest.low, dest.low, imm.value);
+ } else {
+ as_sll(dest.high, dest.low, imm.value - 32);
+ move32(Imm32(0), dest.low);
+ }
+}
+
+void MacroAssembler::lshift64(Register unmaskedShift, Register64 dest) {
+ Label done;
+ ScratchRegisterScope shift(*this);
+
+ ma_and(shift, unmaskedShift, Imm32(0x3f));
+ ma_b(shift, Imm32(0), &done, Equal);
+
+ mov(dest.low, SecondScratchReg);
+ ma_sll(dest.low, dest.low, shift);
+ as_nor(shift, zero, shift);
+ as_srl(SecondScratchReg, SecondScratchReg, 1);
+ ma_srl(SecondScratchReg, SecondScratchReg, shift);
+ ma_and(shift, unmaskedShift, Imm32(0x3f));
+ ma_sll(dest.high, dest.high, shift);
+ as_or(dest.high, dest.high, SecondScratchReg);
+
+ ma_and(SecondScratchReg, shift, Imm32(0x20));
+ as_movn(dest.high, dest.low, SecondScratchReg);
+ as_movn(dest.low, zero, SecondScratchReg);
+
+ bind(&done);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ ma_srl(dest, dest, imm);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ ma_sra(dest, dest, imm);
+}
+
+void MacroAssembler::rshiftPtr(Register src, Register dest) {
+ ma_srl(dest, dest, src);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ScratchRegisterScope scratch(*this);
+
+ if (imm.value == 0) {
+ return;
+ } else if (imm.value < 32) {
+ as_srl(dest.low, dest.low, imm.value);
+ as_sll(scratch, dest.high, (32 - imm.value) % 32);
+ as_or(dest.low, dest.low, scratch);
+ as_srl(dest.high, dest.high, imm.value);
+ } else if (imm.value == 32) {
+ ma_move(dest.low, dest.high);
+ move32(Imm32(0), dest.high);
+ } else {
+ ma_srl(dest.low, dest.high, Imm32(imm.value - 32));
+ move32(Imm32(0), dest.high);
+ }
+}
+
+void MacroAssembler::rshift64(Register unmaskedShift, Register64 dest) {
+ Label done;
+ ScratchRegisterScope shift(*this);
+
+ ma_and(shift, unmaskedShift, Imm32(0x3f));
+ ma_b(shift, Imm32(0), &done, Equal);
+
+ mov(dest.high, SecondScratchReg);
+ ma_srl(dest.high, dest.high, shift);
+ as_nor(shift, zero, shift);
+ as_sll(SecondScratchReg, SecondScratchReg, 1);
+ ma_sll(SecondScratchReg, SecondScratchReg, shift);
+ ma_and(shift, unmaskedShift, Imm32(0x3f));
+ ma_srl(dest.low, dest.low, shift);
+ as_or(dest.low, dest.low, SecondScratchReg);
+ ma_and(SecondScratchReg, shift, Imm32(0x20));
+ as_movn(dest.low, dest.high, SecondScratchReg);
+ as_movn(dest.high, zero, SecondScratchReg);
+
+ bind(&done);
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ScratchRegisterScope scratch(*this);
+
+ if (imm.value == 0) {
+ return;
+ } else if (imm.value < 32) {
+ as_srl(dest.low, dest.low, imm.value);
+ as_sll(scratch, dest.high, (32 - imm.value) % 32);
+ as_or(dest.low, dest.low, scratch);
+ as_sra(dest.high, dest.high, imm.value);
+ } else if (imm.value == 32) {
+ ma_move(dest.low, dest.high);
+ as_sra(dest.high, dest.high, 31);
+ } else {
+ as_sra(dest.low, dest.high, imm.value - 32);
+ as_sra(dest.high, dest.high, 31);
+ }
+}
+
+void MacroAssembler::rshift64Arithmetic(Register unmaskedShift,
+ Register64 dest) {
+ Label done;
+
+ ScratchRegisterScope shift(*this);
+ ma_and(shift, unmaskedShift, Imm32(0x3f));
+ ma_b(shift, Imm32(0), &done, Equal);
+
+ mov(dest.high, SecondScratchReg);
+ ma_sra(dest.high, dest.high, shift);
+ as_nor(shift, zero, shift);
+ as_sll(SecondScratchReg, SecondScratchReg, 1);
+ ma_sll(SecondScratchReg, SecondScratchReg, shift);
+ ma_and(shift, unmaskedShift, Imm32(0x3f));
+ ma_srl(dest.low, dest.low, shift);
+ as_or(dest.low, dest.low, SecondScratchReg);
+ ma_and(SecondScratchReg, shift, Imm32(0x20));
+ as_sra(shift, dest.high, 31);
+ as_movn(dest.low, dest.high, SecondScratchReg);
+ as_movn(dest.high, shift, SecondScratchReg);
+
+ bind(&done);
+}
+
+// ===============================================================
+// Rotation functions
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ MOZ_ASSERT(input.low != dest.high && input.high != dest.low);
+
+ int32_t amount = count.value & 0x3f;
+ if (amount > 32) {
+ rotateRight64(Imm32(64 - amount), input, dest, temp);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ if (amount == 0) {
+ ma_move(dest.low, input.low);
+ ma_move(dest.high, input.high);
+ } else if (amount == 32) {
+ ma_move(scratch, input.low);
+ ma_move(dest.low, input.high);
+ ma_move(dest.high, scratch);
+ } else {
+ MOZ_ASSERT(0 < amount && amount < 32);
+ ma_move(scratch, input.high);
+ ma_sll(dest.high, input.high, Imm32(amount));
+ ma_srl(SecondScratchReg, input.low, Imm32(32 - amount));
+ as_or(dest.high, dest.high, SecondScratchReg);
+ ma_sll(dest.low, input.low, Imm32(amount));
+ ma_srl(SecondScratchReg, scratch, Imm32(32 - amount));
+ as_or(dest.low, dest.low, SecondScratchReg);
+ }
+ }
+}
+
+void MacroAssembler::rotateLeft64(Register shift, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp != src.low && temp != src.high);
+ MOZ_ASSERT(shift != src.low && shift != src.high);
+ MOZ_ASSERT(temp != InvalidReg);
+
+ ScratchRegisterScope scratch(*this);
+
+ ma_and(scratch, shift, Imm32(0x3f));
+ as_nor(SecondScratchReg, zero, scratch);
+ ma_sll(temp, src.low, scratch);
+ ma_move(scratch, src.low);
+ as_srl(dest.low, src.high, 1);
+ ma_srl(dest.low, dest.low, SecondScratchReg);
+ as_or(dest.low, dest.low, temp);
+ ma_move(SecondScratchReg, src.high);
+ as_srl(dest.high, scratch, 1);
+ ma_and(scratch, shift, Imm32(0x3f));
+ ma_sll(temp, SecondScratchReg, scratch);
+ as_nor(SecondScratchReg, zero, scratch);
+ ma_srl(dest.high, dest.high, SecondScratchReg);
+ as_or(dest.high, dest.high, temp);
+ ma_and(temp, scratch, Imm32(32));
+ as_movn(SecondScratchReg, dest.high, temp);
+ as_movn(dest.high, dest.low, temp);
+ as_movn(dest.low, SecondScratchReg, temp);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ MOZ_ASSERT(input.low != dest.high && input.high != dest.low);
+
+ int32_t amount = count.value & 0x3f;
+ if (amount > 32) {
+ rotateLeft64(Imm32(64 - amount), input, dest, temp);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ if (amount == 0) {
+ ma_move(dest.low, input.low);
+ ma_move(dest.high, input.high);
+ } else if (amount == 32) {
+ ma_move(scratch, input.low);
+ ma_move(dest.low, input.high);
+ ma_move(dest.high, scratch);
+ } else {
+ MOZ_ASSERT(0 < amount && amount < 32);
+ ma_move(scratch, input.high);
+ ma_srl(dest.high, input.high, Imm32(amount));
+ ma_sll(SecondScratchReg, input.low, Imm32(32 - amount));
+ as_or(dest.high, dest.high, SecondScratchReg);
+ ma_srl(dest.low, input.low, Imm32(amount));
+ ma_sll(SecondScratchReg, scratch, Imm32(32 - amount));
+ as_or(dest.low, dest.low, SecondScratchReg);
+ }
+ }
+}
+
+void MacroAssembler::rotateRight64(Register shift, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp != src.low && temp != src.high);
+ MOZ_ASSERT(shift != src.low && shift != src.high);
+ MOZ_ASSERT(temp != InvalidReg);
+
+ ScratchRegisterScope scratch(*this);
+
+ ma_and(scratch, shift, Imm32(0x3f));
+ as_nor(SecondScratchReg, zero, scratch);
+ ma_srl(temp, src.low, scratch);
+ ma_move(scratch, src.low);
+ as_sll(dest.low, src.high, 1);
+ ma_sll(dest.low, dest.low, SecondScratchReg);
+ as_or(dest.low, dest.low, temp);
+ ma_move(SecondScratchReg, src.high);
+ as_sll(dest.high, scratch, 1);
+ ma_and(scratch, shift, Imm32(0x3f));
+ ma_srl(temp, SecondScratchReg, scratch);
+ as_nor(SecondScratchReg, zero, scratch);
+ ma_sll(dest.high, dest.high, SecondScratchReg);
+ as_or(dest.high, dest.high, temp);
+ ma_and(temp, scratch, Imm32(32));
+ as_movn(SecondScratchReg, dest.high, temp);
+ as_movn(dest.high, dest.low, temp);
+ as_movn(dest.low, SecondScratchReg, temp);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ ma_cmp_set(dest, lhs, rhs, cond);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ ma_cmp_set(dest, lhs, rhs, cond);
+}
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ MOZ_CRASH("NYI");
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+ as_clz(ScratchRegister, src.high);
+ as_clz(SecondScratchReg, src.low);
+ as_movn(SecondScratchReg, zero, src.high);
+ as_addu(dest, ScratchRegister, SecondScratchReg);
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+ as_movz(SecondScratchReg, src.high, src.low);
+ as_movn(SecondScratchReg, src.low, src.low);
+ ma_ctz(SecondScratchReg, SecondScratchReg);
+ ma_li(ScratchRegister, Imm32(0x20));
+ as_movn(ScratchRegister, zero, src.low);
+ as_addu(dest, SecondScratchReg, ScratchRegister);
+}
+
+void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register tmp) {
+ MOZ_ASSERT(dest.low != tmp);
+ MOZ_ASSERT(dest.high != tmp);
+ MOZ_ASSERT(dest.low != dest.high);
+
+ as_srl(tmp, src.low, 1);
+ as_srl(SecondScratchReg, src.high, 1);
+ ma_li(ScratchRegister, Imm32(0x55555555));
+ as_and(tmp, tmp, ScratchRegister);
+ as_subu(tmp, src.low, tmp);
+ as_and(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ as_subu(SecondScratchReg, src.high, SecondScratchReg);
+ ma_li(ScratchRegister, Imm32(0x33333333));
+ as_and(dest.low, tmp, ScratchRegister);
+ as_srl(tmp, tmp, 2);
+ as_and(tmp, tmp, ScratchRegister);
+ as_addu(tmp, dest.low, tmp);
+ as_and(dest.high, SecondScratchReg, ScratchRegister);
+ as_srl(SecondScratchReg, SecondScratchReg, 2);
+ as_and(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ as_addu(SecondScratchReg, dest.high, SecondScratchReg);
+ ma_li(ScratchRegister, Imm32(0x0F0F0F0F));
+ as_addu(tmp, SecondScratchReg, tmp);
+ as_srl(dest.low, tmp, 4);
+ as_and(dest.low, dest.low, ScratchRegister);
+ as_and(tmp, tmp, ScratchRegister);
+ as_addu(dest.low, dest.low, tmp);
+ ma_mul(dest.low, dest.low, Imm32(0x01010101));
+ as_srl(dest.low, dest.low, 24);
+ ma_move(dest.high, zero);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ Label done;
+
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, val.firstHalf(), &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, val.firstHalf(), label);
+ }
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)),
+ val.secondHalf(), label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ Label done;
+
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, rhs.low, &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, rhs.low, label);
+ }
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), rhs.high,
+ label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ Label done;
+
+ load32(rhs, scratch);
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, scratch, &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, scratch, label);
+ }
+
+ load32(Address(rhs.base, rhs.offset + sizeof(uint32_t)), scratch);
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), scratch,
+ label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ if (val.value == 0) {
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::BelowOrEqual:
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ as_or(ScratchRegister, lhs.high, lhs.low);
+ ma_b(ScratchRegister, ScratchRegister, success,
+ (cond == Assembler::Equal || cond == Assembler::BelowOrEqual)
+ ? Assembler::Zero
+ : Assembler::NonZero);
+ break;
+ case Assembler::LessThan:
+ case Assembler::GreaterThanOrEqual:
+ ma_b(lhs.high, Imm32(0), success, cond);
+ break;
+ case Assembler::LessThanOrEqual:
+ case Assembler::GreaterThan:
+ as_or(SecondScratchReg, lhs.high, lhs.low);
+ as_sra(ScratchRegister, lhs.high, 31);
+ as_sltu(ScratchRegister, ScratchRegister, SecondScratchReg);
+ ma_b(ScratchRegister, ScratchRegister, success,
+ (cond == Assembler::LessThanOrEqual) ? Assembler::Zero
+ : Assembler::NonZero);
+ break;
+ case Assembler::Below:
+ // This condition is always false. No branch required.
+ break;
+ case Assembler::AboveOrEqual:
+ ma_b(success);
+ break;
+ default:
+ MOZ_CRASH("Condition code not supported");
+ }
+ return;
+ }
+
+ Condition c = ma_cmp64(cond, lhs, val, SecondScratchReg);
+ ma_b(SecondScratchReg, SecondScratchReg, success, c);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ Condition c = ma_cmp64(cond, lhs, rhs, SecondScratchReg);
+ ma_b(SecondScratchReg, SecondScratchReg, success, c);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ branchPtr(cond, lhs, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ if (cond == Assembler::Zero || cond == Assembler::NonZero) {
+ MOZ_ASSERT(lhs.low == rhs.low);
+ MOZ_ASSERT(lhs.high == rhs.high);
+ as_or(ScratchRegister, lhs.low, lhs.high);
+ ma_b(ScratchRegister, ScratchRegister, label, cond);
+ } else if (cond == Assembler::Signed || cond == Assembler::NotSigned) {
+ branchTest32(cond, lhs.high, rhs.high, label);
+ } else {
+ MOZ_CRASH("Unsupported condition");
+ }
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestUndefined(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestInt32(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestInt32Truthy(bool b, const ValueOperand& value,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ as_and(scratch, value.payloadReg(), value.payloadReg());
+ ma_b(scratch, scratch, label, b ? NonZero : Zero);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = (cond == Equal) ? Below : AboveOrEqual;
+ ma_b(tag, ImmTag(JSVAL_TAG_CLEAR), label, actual);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestDouble(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestNumber(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_b(value.typeReg(), ImmType(JSVAL_TYPE_BOOLEAN), label, cond);
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ ma_b(value.payloadReg(), value.payloadReg(), label, b ? NonZero : Zero);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestString(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestStringTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ Register string = value.payloadReg();
+ SecondScratchRegisterScope scratch2(*this);
+ ma_lw(scratch2, Address(string, JSString::offsetOfLength()));
+ ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestSymbol(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ Register tag = extractTag(address, scratch2);
+ branchTestBigInt(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestBigInt(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestBigIntTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ Register bi = value.payloadReg();
+ SecondScratchRegisterScope scratch2(*this);
+ ma_lw(scratch2, Address(bi, BigInt::offsetOfDigitLength()));
+ ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestNull(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestObject(cond, value.typeReg(), label);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestPrimitive(cond, value.typeReg(), label);
+}
+
+template <class L>
+void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
+ L label) {
+ ma_b(value.typeReg(), ImmTag(JSVAL_TAG_MAGIC), label, cond);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label notMagic;
+ if (cond == Assembler::Equal) {
+ branchTestMagic(Assembler::NotEqual, valaddr, &notMagic);
+ } else {
+ branchTestMagic(Assembler::NotEqual, valaddr, label);
+ }
+
+ branch32(cond, ToPayload(valaddr), Imm32(why), label);
+ bind(&notMagic);
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ as_truncwd(ScratchFloat32Reg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ as_truncws(ScratchFloat32Reg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmp32Move32(cond, lhs, rhs, src, dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ cmp32Move32(cond, lhs, rhs, src, dest);
+}
+
+void MacroAssemblerMIPSCompat::incrementInt32Value(const Address& addr) {
+ asMasm().add32(Imm32(1), ToPayload(addr));
+}
+
+void MacroAssemblerMIPSCompat::computeEffectiveAddress(const BaseIndex& address,
+ Register dest) {
+ computeScaledAddress(address, dest);
+ if (address.offset) {
+ asMasm().addPtr(Imm32(address.offset), dest);
+ }
+}
+
+void MacroAssemblerMIPSCompat::retn(Imm32 n) {
+ // pc <- [sp]; sp += n
+ loadPtr(Address(StackPointer, 0), ra);
+ asMasm().addPtr(n, StackPointer);
+ as_jr(ra);
+ as_nop();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_MacroAssembler_mips32_inl_h */
diff --git a/js/src/jit/mips32/MacroAssembler-mips32.cpp b/js/src/jit/mips32/MacroAssembler-mips32.cpp
new file mode 100644
index 0000000000..1a96cc60fd
--- /dev/null
+++ b/js/src/jit/mips32/MacroAssembler-mips32.cpp
@@ -0,0 +1,2824 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips32/MacroAssembler-mips32.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/EndianUtils.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "jit/mips32/Simulator-mips32.h"
+#include "jit/MoveEmitter.h"
+#include "jit/SharedICRegisters.h"
+#include "util/Memory.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace jit;
+
+using mozilla::Abs;
+
+static const int32_t PAYLOAD_OFFSET = NUNBOX32_PAYLOAD_OFFSET;
+static const int32_t TAG_OFFSET = NUNBOX32_TYPE_OFFSET;
+
+static_assert(sizeof(intptr_t) == 4, "Not 64-bit clean.");
+
+void MacroAssemblerMIPSCompat::convertBoolToInt32(Register src, Register dest) {
+ // Note that C++ bool is only 1 byte, so zero extend it to clear the
+ // higher-order bits.
+ ma_and(dest, src, Imm32(0xff));
+}
+
+void MacroAssemblerMIPSCompat::convertInt32ToDouble(Register src,
+ FloatRegister dest) {
+ as_mtc1(src, dest);
+ as_cvtdw(dest, dest);
+}
+
+void MacroAssemblerMIPSCompat::convertInt32ToDouble(const Address& src,
+ FloatRegister dest) {
+ ma_ls(dest, src);
+ as_cvtdw(dest, dest);
+}
+
+void MacroAssemblerMIPSCompat::convertInt32ToDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ computeScaledAddress(src, ScratchRegister);
+ convertInt32ToDouble(Address(ScratchRegister, src.offset), dest);
+}
+
+void MacroAssemblerMIPSCompat::convertUInt32ToDouble(Register src,
+ FloatRegister dest) {
+ Label positive, done;
+ ma_b(src, src, &positive, NotSigned, ShortJump);
+
+ const uint32_t kExponentShift =
+ mozilla::FloatingPoint<double>::kExponentShift - 32;
+ const uint32_t kExponent =
+ (31 + mozilla::FloatingPoint<double>::kExponentBias);
+
+ ma_ext(SecondScratchReg, src, 31 - kExponentShift, kExponentShift);
+ ma_li(ScratchRegister, Imm32(kExponent << kExponentShift));
+ ma_or(SecondScratchReg, ScratchRegister);
+ ma_sll(ScratchRegister, src, Imm32(kExponentShift + 1));
+ moveToDoubleHi(SecondScratchReg, dest);
+ moveToDoubleLo(ScratchRegister, dest);
+
+ ma_b(&done, ShortJump);
+
+ bind(&positive);
+ convertInt32ToDouble(src, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerMIPSCompat::convertUInt32ToFloat32(Register src,
+ FloatRegister dest) {
+ Label positive, done;
+ ma_b(src, src, &positive, NotSigned, ShortJump);
+
+ const uint32_t kExponentShift =
+ mozilla::FloatingPoint<double>::kExponentShift - 32;
+ const uint32_t kExponent =
+ (31 + mozilla::FloatingPoint<double>::kExponentBias);
+
+ ma_ext(SecondScratchReg, src, 31 - kExponentShift, kExponentShift);
+ ma_li(ScratchRegister, Imm32(kExponent << kExponentShift));
+ ma_or(SecondScratchReg, ScratchRegister);
+ ma_sll(ScratchRegister, src, Imm32(kExponentShift + 1));
+ FloatRegister destDouble = dest.asDouble();
+ moveToDoubleHi(SecondScratchReg, destDouble);
+ moveToDoubleLo(ScratchRegister, destDouble);
+
+ convertDoubleToFloat32(destDouble, dest);
+
+ ma_b(&done, ShortJump);
+
+ bind(&positive);
+ convertInt32ToFloat32(src, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerMIPSCompat::convertDoubleToFloat32(FloatRegister src,
+ FloatRegister dest) {
+ as_cvtsd(dest, src);
+}
+
+void MacroAssemblerMIPSCompat::convertDoubleToPtr(FloatRegister src,
+ Register dest, Label* fail,
+ bool negativeZeroCheck) {
+ convertDoubleToInt32(src, dest, fail, negativeZeroCheck);
+}
+
+const int CauseBitPos = int(Assembler::CauseI);
+const int CauseBitCount = 1 + int(Assembler::CauseV) - int(Assembler::CauseI);
+const int CauseIOrVMask = ((1 << int(Assembler::CauseI)) |
+ (1 << int(Assembler::CauseV))) >>
+ int(Assembler::CauseI);
+
+// Checks whether a double is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void MacroAssemblerMIPSCompat::convertDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail,
+ bool negativeZeroCheck) {
+ if (negativeZeroCheck) {
+ moveFromDoubleHi(src, dest);
+ moveFromDoubleLo(src, SecondScratchReg);
+ ma_xor(dest, Imm32(INT32_MIN));
+ ma_or(dest, SecondScratchReg);
+ ma_b(dest, Imm32(0), fail, Assembler::Equal);
+ }
+
+ // Truncate double to int ; if result is inexact or invalid fail.
+ as_truncwd(ScratchFloat32Reg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, CauseBitPos, CauseBitCount);
+ // Here adding the masking andi instruction just for a precaution.
+ // For the instruction of trunc.*.*, the Floating Point Exceptions can be
+ // only Inexact, Invalid Operation, Unimplemented Operation.
+ // Leaving it maybe is also ok.
+ as_andi(ScratchRegister, ScratchRegister, CauseIOrVMask);
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+}
+
+// Checks whether a float32 is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void MacroAssemblerMIPSCompat::convertFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail,
+ bool negativeZeroCheck) {
+ if (negativeZeroCheck) {
+ moveFromFloat32(src, dest);
+ ma_b(dest, Imm32(INT32_MIN), fail, Assembler::Equal);
+ }
+
+ as_truncws(ScratchFloat32Reg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, CauseBitPos, CauseBitCount);
+ as_andi(ScratchRegister, ScratchRegister, CauseIOrVMask);
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+}
+
+void MacroAssemblerMIPSCompat::convertFloat32ToDouble(FloatRegister src,
+ FloatRegister dest) {
+ as_cvtds(dest, src);
+}
+
+void MacroAssemblerMIPSCompat::convertInt32ToFloat32(Register src,
+ FloatRegister dest) {
+ as_mtc1(src, dest);
+ as_cvtsw(dest, dest);
+}
+
+void MacroAssemblerMIPSCompat::convertInt32ToFloat32(const Address& src,
+ FloatRegister dest) {
+ ma_ls(dest, src);
+ as_cvtsw(dest, dest);
+}
+
+void MacroAssemblerMIPS::ma_li(Register dest, CodeLabel* label) {
+ BufferOffset bo = m_buffer.nextOffset();
+ ma_liPatchable(dest, ImmWord(/* placeholder */ 0));
+ label->patchAt()->bind(bo.getOffset());
+ label->setLinkMode(CodeLabel::MoveImmediate);
+}
+
+void MacroAssemblerMIPS::ma_li(Register dest, ImmWord imm) {
+ ma_li(dest, Imm32(uint32_t(imm.value)));
+}
+
+void MacroAssemblerMIPS::ma_liPatchable(Register dest, ImmPtr imm) {
+ ma_liPatchable(dest, ImmWord(uintptr_t(imm.value)));
+}
+
+void MacroAssemblerMIPS::ma_liPatchable(Register dest, ImmWord imm) {
+ ma_liPatchable(dest, Imm32(int32_t(imm.value)));
+}
+
+// Arithmetic-based ops.
+
+// Add.
+void MacroAssemblerMIPS::ma_add32TestOverflow(Register rd, Register rs,
+ Register rt, Label* overflow) {
+ MOZ_ASSERT_IF(rs == rd, rs != rt);
+ MOZ_ASSERT(rs != ScratchRegister);
+ MOZ_ASSERT(rt != ScratchRegister);
+ MOZ_ASSERT(rd != rt);
+ MOZ_ASSERT(rd != ScratchRegister);
+ MOZ_ASSERT(rd != SecondScratchReg);
+
+ if (rs == rt) {
+ as_addu(rd, rs, rs);
+ as_xor(SecondScratchReg, rs, rd);
+ ma_b(SecondScratchReg, Imm32(0), overflow, Assembler::LessThan);
+ return;
+ }
+
+ // If different sign, no overflow
+ as_xor(ScratchRegister, rs, rt);
+
+ as_addu(rd, rs, rt);
+ as_nor(ScratchRegister, ScratchRegister, zero);
+ // If different sign, then overflow
+ as_xor(SecondScratchReg, rt, rd);
+ as_and(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ ma_b(SecondScratchReg, Imm32(0), overflow, Assembler::LessThan);
+}
+
+void MacroAssemblerMIPS::ma_add32TestOverflow(Register rd, Register rs,
+ Imm32 imm, Label* overflow) {
+ MOZ_ASSERT(rs != ScratchRegister);
+ MOZ_ASSERT(rs != SecondScratchReg);
+ MOZ_ASSERT(rd != ScratchRegister);
+ MOZ_ASSERT(rd != SecondScratchReg);
+
+ Register rs_copy = rs;
+
+ if (imm.value > 0) {
+ as_nor(ScratchRegister, rs, zero);
+ } else if (rs == rd) {
+ ma_move(ScratchRegister, rs);
+ rs_copy = ScratchRegister;
+ }
+
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_addiu(rd, rs, imm.value);
+ } else {
+ ma_li(SecondScratchReg, imm);
+ as_addu(rd, rs, SecondScratchReg);
+ }
+
+ if (imm.value > 0) {
+ as_and(ScratchRegister, ScratchRegister, rd);
+ } else {
+ as_nor(SecondScratchReg, rd, zero);
+ as_and(ScratchRegister, rs_copy, SecondScratchReg);
+ }
+
+ ma_b(ScratchRegister, Imm32(0), overflow, Assembler::LessThan);
+}
+
+// Subtract.
+void MacroAssemblerMIPS::ma_sub32TestOverflow(Register rd, Register rs,
+ Register rt, Label* overflow) {
+ // The rs == rt case should probably be folded at MIR stage.
+ // Happens for Number_isInteger*. Not worth specializing here.
+ MOZ_ASSERT_IF(rs == rd, rs != rt);
+ MOZ_ASSERT(rs != SecondScratchReg);
+ MOZ_ASSERT(rt != SecondScratchReg);
+ MOZ_ASSERT(rd != rt);
+ MOZ_ASSERT(rd != ScratchRegister);
+ MOZ_ASSERT(rd != SecondScratchReg);
+
+ Register rs_copy = rs;
+
+ if (rs == rd) {
+ ma_move(SecondScratchReg, rs);
+ rs_copy = SecondScratchReg;
+ }
+
+ as_subu(rd, rs, rt);
+ // If same sign, no overflow
+ as_xor(ScratchRegister, rs_copy, rt);
+ // If different sign, then overflow
+ as_xor(SecondScratchReg, rs_copy, rd);
+ as_and(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ ma_b(SecondScratchReg, Imm32(0), overflow, Assembler::LessThan);
+}
+
+// Memory.
+
+void MacroAssemblerMIPS::ma_load(Register dest, Address address,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ int16_t encodedOffset;
+ Register base;
+
+ if (isLoongson() && ZeroExtend != extension &&
+ !Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ base = address.base;
+
+ switch (size) {
+ case SizeByte:
+ as_gslbx(dest, base, ScratchRegister, 0);
+ break;
+ case SizeHalfWord:
+ as_gslhx(dest, base, ScratchRegister, 0);
+ break;
+ case SizeWord:
+ as_gslwx(dest, base, ScratchRegister, 0);
+ break;
+ case SizeDouble:
+ as_gsldx(dest, base, ScratchRegister, 0);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+ return;
+ }
+
+ if (!Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_addu(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = Imm16(0).encode();
+ } else {
+ encodedOffset = Imm16(address.offset).encode();
+ base = address.base;
+ }
+
+ switch (size) {
+ case SizeByte:
+ if (ZeroExtend == extension) {
+ as_lbu(dest, base, encodedOffset);
+ } else {
+ as_lb(dest, base, encodedOffset);
+ }
+ break;
+ case SizeHalfWord:
+ if (ZeroExtend == extension) {
+ as_lhu(dest, base, encodedOffset);
+ } else {
+ as_lh(dest, base, encodedOffset);
+ }
+ break;
+ case SizeWord:
+ as_lw(dest, base, encodedOffset);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+}
+
+void MacroAssemblerMIPS::ma_store(Register data, Address address,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ int16_t encodedOffset;
+ Register base;
+
+ if (isLoongson() && !Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ base = address.base;
+
+ switch (size) {
+ case SizeByte:
+ as_gssbx(data, base, ScratchRegister, 0);
+ break;
+ case SizeHalfWord:
+ as_gsshx(data, base, ScratchRegister, 0);
+ break;
+ case SizeWord:
+ as_gsswx(data, base, ScratchRegister, 0);
+ break;
+ case SizeDouble:
+ as_gssdx(data, base, ScratchRegister, 0);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+ return;
+ }
+
+ if (!Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_addu(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = Imm16(0).encode();
+ } else {
+ encodedOffset = Imm16(address.offset).encode();
+ base = address.base;
+ }
+
+ switch (size) {
+ case SizeByte:
+ as_sb(data, base, encodedOffset);
+ break;
+ case SizeHalfWord:
+ as_sh(data, base, encodedOffset);
+ break;
+ case SizeWord:
+ as_sw(data, base, encodedOffset);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+}
+
+void MacroAssemblerMIPSCompat::computeScaledAddress(const BaseIndex& address,
+ Register dest) {
+ int32_t shift = Imm32::ShiftOf(address.scale).value;
+ if (shift) {
+ ma_sll(ScratchRegister, address.index, Imm32(shift));
+ as_addu(dest, address.base, ScratchRegister);
+ } else {
+ as_addu(dest, address.base, address.index);
+ }
+}
+
+// Shortcut for when we know we're transferring 32 bits of data.
+void MacroAssemblerMIPS::ma_lw(Register data, Address address) {
+ ma_load(data, address, SizeWord);
+}
+
+void MacroAssemblerMIPS::ma_sw(Register data, Address address) {
+ ma_store(data, address, SizeWord);
+}
+
+void MacroAssemblerMIPS::ma_sw(Imm32 imm, Address address) {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_sw(ScratchRegister, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != SecondScratchReg);
+
+ ma_li(SecondScratchReg, Imm32(address.offset));
+ as_addu(SecondScratchReg, address.base, SecondScratchReg);
+ as_sw(ScratchRegister, SecondScratchReg, 0);
+ }
+}
+
+void MacroAssemblerMIPS::ma_sw(Register data, BaseIndex& address) {
+ ma_store(data, address, SizeWord);
+}
+
+void MacroAssemblerMIPS::ma_pop(Register r) {
+ as_lw(r, StackPointer, 0);
+ as_addiu(StackPointer, StackPointer, sizeof(intptr_t));
+}
+
+void MacroAssemblerMIPS::ma_push(Register r) {
+ if (r == sp) {
+ // Pushing sp requires one more instruction.
+ ma_move(ScratchRegister, sp);
+ r = ScratchRegister;
+ }
+
+ as_addiu(StackPointer, StackPointer, -sizeof(intptr_t));
+ as_sw(r, StackPointer, 0);
+}
+
+// Branches when done from within mips-specific code.
+void MacroAssemblerMIPS::ma_b(Register lhs, Address addr, Label* label,
+ Condition c, JumpKind jumpKind) {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_lw(ScratchRegister, addr);
+ ma_b(lhs, ScratchRegister, label, c, jumpKind);
+}
+
+void MacroAssemblerMIPS::ma_b(Address addr, Imm32 imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ ma_lw(SecondScratchReg, addr);
+ ma_b(SecondScratchReg, imm, label, c, jumpKind);
+}
+
+void MacroAssemblerMIPS::ma_b(Address addr, ImmGCPtr imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ ma_lw(SecondScratchReg, addr);
+ ma_b(SecondScratchReg, imm, label, c, jumpKind);
+}
+
+void MacroAssemblerMIPS::ma_bal(Label* label, DelaySlotFill delaySlotFill) {
+ spew("branch .Llabel %p\n", label);
+ if (label->bound()) {
+ // Generate the long jump for calls because return address has to be
+ // the address after the reserved block.
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ma_liPatchable(ScratchRegister, Imm32(LabelBase::INVALID_OFFSET));
+ as_jalr(ScratchRegister);
+ if (delaySlotFill == FillDelaySlot) {
+ as_nop();
+ }
+ return;
+ }
+
+ // Second word holds a pointer to the next branch in label's chain.
+ uint32_t nextInChain =
+ label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+
+ // Make the whole branch continous in the buffer.
+ m_buffer.ensureSpace(4 * sizeof(uint32_t));
+
+ spew("bal .Llabel %p\n", label);
+ BufferOffset bo = writeInst(getBranchCode(BranchIsCall).encode());
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ // Leave space for long jump.
+ as_nop();
+ if (delaySlotFill == FillDelaySlot) {
+ as_nop();
+ }
+}
+
+void MacroAssemblerMIPS::branchWithCode(InstImm code, Label* label,
+ JumpKind jumpKind) {
+ spew("branch .Llabel %p", label);
+ MOZ_ASSERT(code.encode() !=
+ InstImm(op_regimm, zero, rt_bgezal, BOffImm16(0)).encode());
+ InstImm inst_beq = InstImm(op_beq, zero, zero, BOffImm16(0));
+
+ if (label->bound()) {
+ int32_t offset = label->offset() - m_buffer.nextOffset().getOffset();
+
+ if (BOffImm16::IsInRange(offset)) {
+ jumpKind = ShortJump;
+ }
+
+ if (jumpKind == ShortJump) {
+ MOZ_ASSERT(BOffImm16::IsInRange(offset));
+ code.setBOffImm16(BOffImm16(offset));
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ writeInst(code.encode());
+ as_nop();
+ return;
+ }
+
+ if (code.encode() == inst_beq.encode()) {
+ // Handle long jump
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ma_liPatchable(ScratchRegister, Imm32(LabelBase::INVALID_OFFSET));
+ as_jr(ScratchRegister);
+ as_nop();
+ return;
+ }
+
+ // Handle long conditional branch
+ spew("invert branch .Llabel %p", label);
+ InstImm code_r = invertBranch(code, BOffImm16(5 * sizeof(uint32_t)));
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code_r);
+#endif
+ writeInst(code_r.encode());
+
+ // No need for a "nop" here because we can clobber scratch.
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ma_liPatchable(ScratchRegister, Imm32(LabelBase::INVALID_OFFSET));
+ as_jr(ScratchRegister);
+ as_nop();
+ return;
+ }
+
+ // Generate open jump and link it to a label.
+
+ // Second word holds a pointer to the next branch in label's chain.
+ uint32_t nextInChain =
+ label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+
+ if (jumpKind == ShortJump) {
+ // Make the whole branch continous in the buffer.
+ m_buffer.ensureSpace(2 * sizeof(uint32_t));
+
+ // Indicate that this is short jump with offset 4.
+ code.setBOffImm16(BOffImm16(4));
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ BufferOffset bo = writeInst(code.encode());
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ return;
+ }
+
+ bool conditional = code.encode() != inst_beq.encode();
+
+ // Make the whole branch continous in the buffer.
+ m_buffer.ensureSpace((conditional ? 5 : 4) * sizeof(uint32_t));
+
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ BufferOffset bo = writeInst(code.encode());
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ // Leave space for potential long jump.
+ as_nop();
+ as_nop();
+ if (conditional) {
+ as_nop();
+ }
+}
+
+void MacroAssemblerMIPSCompat::cmp64Set(Condition cond, Register64 lhs,
+ Imm64 val, Register dest) {
+ if (val.value == 0) {
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::BelowOrEqual:
+ as_or(dest, lhs.high, lhs.low);
+ as_sltiu(dest, dest, 1);
+ break;
+ case Assembler::NotEqual:
+ case Assembler::Above:
+ as_or(dest, lhs.high, lhs.low);
+ as_sltu(dest, zero, dest);
+ break;
+ case Assembler::LessThan:
+ case Assembler::GreaterThanOrEqual:
+ as_slt(dest, lhs.high, zero);
+ if (cond == Assembler::GreaterThanOrEqual) {
+ as_xori(dest, dest, 1);
+ }
+ break;
+ case Assembler::GreaterThan:
+ case Assembler::LessThanOrEqual:
+ as_or(SecondScratchReg, lhs.high, lhs.low);
+ as_sra(ScratchRegister, lhs.high, 31);
+ as_sltu(dest, ScratchRegister, SecondScratchReg);
+ if (cond == Assembler::LessThanOrEqual) {
+ as_xori(dest, dest, 1);
+ }
+ break;
+ case Assembler::Below:
+ case Assembler::AboveOrEqual:
+ as_ori(dest, zero, cond == Assembler::AboveOrEqual ? 1 : 0);
+ break;
+ default:
+ MOZ_CRASH("Condition code not supported");
+ break;
+ }
+ return;
+ }
+
+ Condition c = ma_cmp64(cond, lhs, val, dest);
+
+ switch (cond) {
+ // For Equal/NotEqual cond ma_cmp64 dest holds non boolean result.
+ case Assembler::Equal:
+ as_sltiu(dest, dest, 1);
+ break;
+ case Assembler::NotEqual:
+ as_sltu(dest, zero, dest);
+ break;
+ default:
+ if (c == Assembler::Zero) as_xori(dest, dest, 1);
+ break;
+ }
+}
+
+void MacroAssemblerMIPSCompat::cmp64Set(Condition cond, Register64 lhs,
+ Register64 rhs, Register dest) {
+ Condition c = ma_cmp64(cond, lhs, rhs, dest);
+
+ switch (cond) {
+ // For Equal/NotEqual cond ma_cmp64 dest holds non boolean result.
+ case Assembler::Equal:
+ as_sltiu(dest, dest, 1);
+ break;
+ case Assembler::NotEqual:
+ as_sltu(dest, zero, dest);
+ break;
+ default:
+ if (c == Assembler::Zero) as_xori(dest, dest, 1);
+ break;
+ }
+}
+
+Assembler::Condition MacroAssemblerMIPSCompat::ma_cmp64(Condition cond,
+ Register64 lhs,
+ Register64 rhs,
+ Register dest) {
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ as_xor(SecondScratchReg, lhs.high, rhs.high);
+ as_xor(ScratchRegister, lhs.low, rhs.low);
+ as_or(dest, SecondScratchReg, ScratchRegister);
+ return (cond == Assembler::Equal) ? Assembler::Zero : Assembler::NonZero;
+ break;
+ case Assembler::LessThan:
+ case Assembler::GreaterThanOrEqual:
+ as_slt(SecondScratchReg, rhs.high, lhs.high);
+ as_sltu(ScratchRegister, lhs.low, rhs.low);
+ as_slt(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ as_slt(ScratchRegister, lhs.high, rhs.high);
+ as_or(dest, ScratchRegister, SecondScratchReg);
+ return (cond == Assembler::GreaterThanOrEqual) ? Assembler::Zero
+ : Assembler::NonZero;
+ break;
+ case Assembler::GreaterThan:
+ case Assembler::LessThanOrEqual:
+ as_slt(SecondScratchReg, lhs.high, rhs.high);
+ as_sltu(ScratchRegister, rhs.low, lhs.low);
+ as_slt(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ as_slt(ScratchRegister, rhs.high, lhs.high);
+ as_or(dest, ScratchRegister, SecondScratchReg);
+ return (cond == Assembler::LessThanOrEqual) ? Assembler::Zero
+ : Assembler::NonZero;
+ break;
+ case Assembler::Below:
+ case Assembler::AboveOrEqual:
+ as_sltu(SecondScratchReg, rhs.high, lhs.high);
+ as_sltu(ScratchRegister, lhs.low, rhs.low);
+ as_slt(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ as_sltu(ScratchRegister, lhs.high, rhs.high);
+ as_or(dest, ScratchRegister, SecondScratchReg);
+ return (cond == Assembler::AboveOrEqual) ? Assembler::Zero
+ : Assembler::NonZero;
+ break;
+ case Assembler::Above:
+ case Assembler::BelowOrEqual:
+ as_sltu(SecondScratchReg, lhs.high, rhs.high);
+ as_sltu(ScratchRegister, rhs.low, lhs.low);
+ as_slt(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ as_sltu(ScratchRegister, rhs.high, lhs.high);
+ as_or(dest, ScratchRegister, SecondScratchReg);
+ return (cond == Assembler::BelowOrEqual) ? Assembler::Zero
+ : Assembler::NonZero;
+ break;
+ default:
+ MOZ_CRASH("Condition code not supported");
+ break;
+ }
+}
+
+Assembler::Condition MacroAssemblerMIPSCompat::ma_cmp64(Condition cond,
+ Register64 lhs,
+ Imm64 val,
+ Register dest) {
+ MOZ_ASSERT(val.value != 0);
+
+ switch (cond) {
+ case Assembler::Equal:
+ case Assembler::NotEqual:
+ ma_xor(SecondScratchReg, lhs.high, val.hi());
+ ma_xor(ScratchRegister, lhs.low, val.low());
+ as_or(dest, SecondScratchReg, ScratchRegister);
+ return (cond == Assembler::Equal) ? Assembler::Zero : Assembler::NonZero;
+ break;
+ case Assembler::LessThan:
+ case Assembler::GreaterThanOrEqual:
+ ma_li(SecondScratchReg, val.hi());
+ as_slt(ScratchRegister, lhs.high, SecondScratchReg);
+ as_slt(SecondScratchReg, SecondScratchReg, lhs.high);
+ as_subu(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ ma_li(ScratchRegister, val.low());
+ as_sltu(ScratchRegister, lhs.low, ScratchRegister);
+ as_slt(dest, SecondScratchReg, ScratchRegister);
+ return (cond == Assembler::GreaterThanOrEqual) ? Assembler::Zero
+ : Assembler::NonZero;
+ break;
+ case Assembler::GreaterThan:
+ case Assembler::LessThanOrEqual:
+ ma_li(SecondScratchReg, val.hi());
+ as_slt(ScratchRegister, SecondScratchReg, lhs.high);
+ as_slt(SecondScratchReg, lhs.high, SecondScratchReg);
+ as_subu(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ ma_li(ScratchRegister, val.low());
+ as_sltu(ScratchRegister, ScratchRegister, lhs.low);
+ as_slt(dest, SecondScratchReg, ScratchRegister);
+ return (cond == Assembler::LessThanOrEqual) ? Assembler::Zero
+ : Assembler::NonZero;
+ break;
+ case Assembler::Below:
+ case Assembler::AboveOrEqual:
+ ma_li(SecondScratchReg, val.hi());
+ as_sltu(ScratchRegister, lhs.high, SecondScratchReg);
+ as_sltu(SecondScratchReg, SecondScratchReg, lhs.high);
+ as_subu(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ ma_li(ScratchRegister, val.low());
+ as_sltu(ScratchRegister, lhs.low, ScratchRegister);
+ as_slt(dest, SecondScratchReg, ScratchRegister);
+ return (cond == Assembler::AboveOrEqual) ? Assembler::Zero
+ : Assembler::NonZero;
+ break;
+ case Assembler::Above:
+ case Assembler::BelowOrEqual:
+ ma_li(SecondScratchReg, val.hi());
+ as_sltu(ScratchRegister, SecondScratchReg, lhs.high);
+ as_sltu(SecondScratchReg, lhs.high, SecondScratchReg);
+ as_subu(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ ma_li(ScratchRegister, val.low());
+ as_sltu(ScratchRegister, ScratchRegister, lhs.low);
+ as_slt(dest, SecondScratchReg, ScratchRegister);
+ return (cond == Assembler::BelowOrEqual) ? Assembler::Zero
+ : Assembler::NonZero;
+ break;
+ default:
+ MOZ_CRASH("Condition code not supported");
+ break;
+ }
+}
+
+// fp instructions
+void MacroAssemblerMIPS::ma_lid(FloatRegister dest, double value) {
+ struct DoubleStruct {
+ uint32_t lo;
+ uint32_t hi;
+ };
+ DoubleStruct intStruct = mozilla::BitwiseCast<DoubleStruct>(value);
+#if MOZ_BIG_ENDIAN()
+ std::swap(intStruct.hi, intStruct.lo);
+#endif
+
+ // put hi part of 64 bit value into the odd register
+ if (intStruct.hi == 0) {
+ moveToDoubleHi(zero, dest);
+ } else {
+ ma_li(ScratchRegister, Imm32(intStruct.hi));
+ moveToDoubleHi(ScratchRegister, dest);
+ }
+
+ // put low part of 64 bit value into the even register
+ if (intStruct.lo == 0) {
+ moveToDoubleLo(zero, dest);
+ } else {
+ ma_li(ScratchRegister, Imm32(intStruct.lo));
+ moveToDoubleLo(ScratchRegister, dest);
+ }
+}
+
+void MacroAssemblerMIPS::ma_mv(FloatRegister src, ValueOperand dest) {
+ moveFromDoubleLo(src, dest.payloadReg());
+ moveFromDoubleHi(src, dest.typeReg());
+}
+
+void MacroAssemblerMIPS::ma_mv(ValueOperand src, FloatRegister dest) {
+ moveToDoubleLo(src.payloadReg(), dest);
+ moveToDoubleHi(src.typeReg(), dest);
+}
+
+void MacroAssemblerMIPS::ma_ls(FloatRegister ft, Address address) {
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_lwc1(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ if (isLoongson()) {
+ as_gslsx(ft, address.base, ScratchRegister, 0);
+ } else {
+ as_addu(ScratchRegister, address.base, ScratchRegister);
+ as_lwc1(ft, ScratchRegister, 0);
+ }
+ }
+}
+
+void MacroAssemblerMIPS::ma_ld(FloatRegister ft, Address address) {
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_ldc1(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ if (isLoongson()) {
+ as_gsldx(ft, address.base, ScratchRegister, 0);
+ } else {
+ as_addu(ScratchRegister, address.base, ScratchRegister);
+ as_ldc1(ft, ScratchRegister, 0);
+ }
+ }
+}
+
+void MacroAssemblerMIPS::ma_sd(FloatRegister ft, Address address) {
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_sdc1(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ if (isLoongson()) {
+ as_gssdx(ft, address.base, ScratchRegister, 0);
+ } else {
+ as_addu(ScratchRegister, address.base, ScratchRegister);
+ as_sdc1(ft, ScratchRegister, 0);
+ }
+ }
+}
+
+void MacroAssemblerMIPS::ma_ss(FloatRegister ft, Address address) {
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_swc1(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ if (isLoongson()) {
+ as_gsssx(ft, address.base, ScratchRegister, 0);
+ } else {
+ as_addu(ScratchRegister, address.base, ScratchRegister);
+ as_swc1(ft, ScratchRegister, 0);
+ }
+ }
+}
+
+void MacroAssemblerMIPS::ma_ldc1WordAligned(FloatRegister ft, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(off + PAYLOAD_OFFSET) &&
+ Imm16::IsInSignedRange(off + TAG_OFFSET));
+
+ as_lwc1(ft, base, off + PAYLOAD_OFFSET);
+ as_lwc1(getOddPair(ft), base, off + TAG_OFFSET);
+}
+
+void MacroAssemblerMIPS::ma_sdc1WordAligned(FloatRegister ft, Register base,
+ int32_t off) {
+ MOZ_ASSERT(Imm16::IsInSignedRange(off + PAYLOAD_OFFSET) &&
+ Imm16::IsInSignedRange(off + TAG_OFFSET));
+
+ as_swc1(ft, base, off + PAYLOAD_OFFSET);
+ as_swc1(getOddPair(ft), base, off + TAG_OFFSET);
+}
+
+void MacroAssemblerMIPS::ma_pop(FloatRegister f) {
+ if (f.isDouble()) {
+ ma_ldc1WordAligned(f, StackPointer, 0);
+ } else {
+ as_lwc1(f, StackPointer, 0);
+ }
+
+ as_addiu(StackPointer, StackPointer, f.size());
+}
+
+void MacroAssemblerMIPS::ma_push(FloatRegister f) {
+ as_addiu(StackPointer, StackPointer, -f.size());
+
+ if (f.isDouble()) {
+ ma_sdc1WordAligned(f, StackPointer, 0);
+ } else {
+ as_swc1(f, StackPointer, 0);
+ }
+}
+
+bool MacroAssemblerMIPSCompat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
+ uint32_t descriptor = MakeFrameDescriptor(
+ asMasm().framePushed(), FrameType::IonJS, ExitFrameLayout::Size());
+
+ asMasm().Push(Imm32(descriptor)); // descriptor_
+ asMasm().Push(ImmPtr(fakeReturnAddr));
+
+ return true;
+}
+
+void MacroAssemblerMIPSCompat::move32(Imm32 imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerMIPSCompat::move32(Register src, Register dest) {
+ ma_move(dest, src);
+}
+
+void MacroAssemblerMIPSCompat::movePtr(Register src, Register dest) {
+ ma_move(dest, src);
+}
+void MacroAssemblerMIPSCompat::movePtr(ImmWord imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerMIPSCompat::movePtr(ImmGCPtr imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerMIPSCompat::movePtr(ImmPtr imm, Register dest) {
+ movePtr(ImmWord(uintptr_t(imm.value)), dest);
+}
+void MacroAssemblerMIPSCompat::movePtr(wasm::SymbolicAddress imm,
+ Register dest) {
+ append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm));
+ ma_liPatchable(dest, ImmWord(-1));
+}
+
+void MacroAssemblerMIPSCompat::load8ZeroExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeByte, ZeroExtend);
+}
+
+void MacroAssemblerMIPSCompat::load8ZeroExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeByte, ZeroExtend);
+}
+
+void MacroAssemblerMIPSCompat::load8SignExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeByte, SignExtend);
+}
+
+void MacroAssemblerMIPSCompat::load8SignExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeByte, SignExtend);
+}
+
+void MacroAssemblerMIPSCompat::load16ZeroExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeHalfWord, ZeroExtend);
+}
+
+void MacroAssemblerMIPSCompat::load16ZeroExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeHalfWord, ZeroExtend);
+}
+
+void MacroAssemblerMIPSCompat::load16SignExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeHalfWord, SignExtend);
+}
+
+void MacroAssemblerMIPSCompat::load16SignExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeHalfWord, SignExtend);
+}
+
+void MacroAssemblerMIPSCompat::load32(const Address& address, Register dest) {
+ ma_load(dest, address, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::load32(const BaseIndex& address, Register dest) {
+ ma_load(dest, address, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::load32(AbsoluteAddress address, Register dest) {
+ movePtr(ImmPtr(address.addr), ScratchRegister);
+ load32(Address(ScratchRegister, 0), dest);
+}
+
+void MacroAssemblerMIPSCompat::load32(wasm::SymbolicAddress address,
+ Register dest) {
+ movePtr(address, ScratchRegister);
+ load32(Address(ScratchRegister, 0), dest);
+}
+
+void MacroAssemblerMIPSCompat::loadPtr(const Address& address, Register dest) {
+ ma_load(dest, address, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::loadPtr(const BaseIndex& src, Register dest) {
+ ma_load(dest, src, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::loadPtr(AbsoluteAddress address, Register dest) {
+ movePtr(ImmPtr(address.addr), ScratchRegister);
+ loadPtr(Address(ScratchRegister, 0), dest);
+}
+
+void MacroAssemblerMIPSCompat::loadPtr(wasm::SymbolicAddress address,
+ Register dest) {
+ movePtr(address, ScratchRegister);
+ loadPtr(Address(ScratchRegister, 0), dest);
+}
+
+void MacroAssemblerMIPSCompat::loadPrivate(const Address& address,
+ Register dest) {
+ ma_lw(dest, Address(address.base, address.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::loadUnalignedDouble(
+ const wasm::MemoryAccessDesc& access, const BaseIndex& src, Register temp,
+ FloatRegister dest) {
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Wasm-only; wasm is disabled on big-endian.");
+ computeScaledAddress(src, SecondScratchReg);
+
+ BufferOffset load;
+ if (Imm16::IsInSignedRange(src.offset) &&
+ Imm16::IsInSignedRange(src.offset + 7)) {
+ load = as_lwl(temp, SecondScratchReg, src.offset + INT64LOW_OFFSET + 3);
+ as_lwr(temp, SecondScratchReg, src.offset + INT64LOW_OFFSET);
+ append(access, load.getOffset());
+ moveToDoubleLo(temp, dest);
+ load = as_lwl(temp, SecondScratchReg, src.offset + INT64HIGH_OFFSET + 3);
+ as_lwr(temp, SecondScratchReg, src.offset + INT64HIGH_OFFSET);
+ append(access, load.getOffset());
+ moveToDoubleHi(temp, dest);
+ } else {
+ ma_li(ScratchRegister, Imm32(src.offset));
+ as_daddu(ScratchRegister, SecondScratchReg, ScratchRegister);
+ load = as_lwl(temp, ScratchRegister, INT64LOW_OFFSET + 3);
+ as_lwr(temp, ScratchRegister, INT64LOW_OFFSET);
+ append(access, load.getOffset());
+ moveToDoubleLo(temp, dest);
+ load = as_lwl(temp, ScratchRegister, INT64HIGH_OFFSET + 3);
+ as_lwr(temp, ScratchRegister, INT64HIGH_OFFSET);
+ append(access, load.getOffset());
+ moveToDoubleHi(temp, dest);
+ }
+}
+
+void MacroAssemblerMIPSCompat::loadUnalignedFloat32(
+ const wasm::MemoryAccessDesc& access, const BaseIndex& src, Register temp,
+ FloatRegister dest) {
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Wasm-only; wasm is disabled on big-endian.");
+ computeScaledAddress(src, SecondScratchReg);
+ BufferOffset load;
+ if (Imm16::IsInSignedRange(src.offset) &&
+ Imm16::IsInSignedRange(src.offset + 3)) {
+ load = as_lwl(temp, SecondScratchReg, src.offset + 3);
+ as_lwr(temp, SecondScratchReg, src.offset);
+ } else {
+ ma_li(ScratchRegister, Imm32(src.offset));
+ as_daddu(ScratchRegister, SecondScratchReg, ScratchRegister);
+ load = as_lwl(temp, ScratchRegister, 3);
+ as_lwr(temp, ScratchRegister, 0);
+ }
+ append(access, load.getOffset());
+ moveToFloat32(temp, dest);
+}
+
+void MacroAssemblerMIPSCompat::store8(Imm32 imm, const Address& address) {
+ ma_li(SecondScratchReg, imm);
+ ma_store(SecondScratchReg, address, SizeByte);
+}
+
+void MacroAssemblerMIPSCompat::store8(Register src, const Address& address) {
+ ma_store(src, address, SizeByte);
+}
+
+void MacroAssemblerMIPSCompat::store8(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeByte);
+}
+
+void MacroAssemblerMIPSCompat::store8(Register src, const BaseIndex& dest) {
+ ma_store(src, dest, SizeByte);
+}
+
+void MacroAssemblerMIPSCompat::store16(Imm32 imm, const Address& address) {
+ ma_li(SecondScratchReg, imm);
+ ma_store(SecondScratchReg, address, SizeHalfWord);
+}
+
+void MacroAssemblerMIPSCompat::store16(Register src, const Address& address) {
+ ma_store(src, address, SizeHalfWord);
+}
+
+void MacroAssemblerMIPSCompat::store16(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeHalfWord);
+}
+
+void MacroAssemblerMIPSCompat::store16(Register src, const BaseIndex& address) {
+ ma_store(src, address, SizeHalfWord);
+}
+
+void MacroAssemblerMIPSCompat::store32(Register src, AbsoluteAddress address) {
+ movePtr(ImmPtr(address.addr), ScratchRegister);
+ store32(src, Address(ScratchRegister, 0));
+}
+
+void MacroAssemblerMIPSCompat::store32(Register src, const Address& address) {
+ ma_store(src, address, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::store32(Imm32 src, const Address& address) {
+ move32(src, SecondScratchReg);
+ ma_store(SecondScratchReg, address, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::store32(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::store32(Register src, const BaseIndex& dest) {
+ ma_store(src, dest, SizeWord);
+}
+
+template <typename T>
+void MacroAssemblerMIPSCompat::storePtr(ImmWord imm, T address) {
+ ma_li(SecondScratchReg, imm);
+ ma_store(SecondScratchReg, address, SizeWord);
+}
+
+template void MacroAssemblerMIPSCompat::storePtr<Address>(ImmWord imm,
+ Address address);
+template void MacroAssemblerMIPSCompat::storePtr<BaseIndex>(ImmWord imm,
+ BaseIndex address);
+
+template <typename T>
+void MacroAssemblerMIPSCompat::storePtr(ImmPtr imm, T address) {
+ storePtr(ImmWord(uintptr_t(imm.value)), address);
+}
+
+template void MacroAssemblerMIPSCompat::storePtr<Address>(ImmPtr imm,
+ Address address);
+template void MacroAssemblerMIPSCompat::storePtr<BaseIndex>(ImmPtr imm,
+ BaseIndex address);
+
+template <typename T>
+void MacroAssemblerMIPSCompat::storePtr(ImmGCPtr imm, T address) {
+ movePtr(imm, SecondScratchReg);
+ storePtr(SecondScratchReg, address);
+}
+
+template void MacroAssemblerMIPSCompat::storePtr<Address>(ImmGCPtr imm,
+ Address address);
+template void MacroAssemblerMIPSCompat::storePtr<BaseIndex>(ImmGCPtr imm,
+ BaseIndex address);
+
+void MacroAssemblerMIPSCompat::storePtr(Register src, const Address& address) {
+ ma_store(src, address, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::storePtr(Register src,
+ const BaseIndex& address) {
+ ma_store(src, address, SizeWord);
+}
+
+void MacroAssemblerMIPSCompat::storePtr(Register src, AbsoluteAddress dest) {
+ movePtr(ImmPtr(dest.addr), ScratchRegister);
+ storePtr(src, Address(ScratchRegister, 0));
+}
+
+void MacroAssemblerMIPSCompat::storeUnalignedFloat32(
+ const wasm::MemoryAccessDesc& access, FloatRegister src, Register temp,
+ const BaseIndex& dest) {
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Wasm-only; wasm is disabled on big-endian.");
+ computeScaledAddress(dest, SecondScratchReg);
+ moveFromFloat32(src, temp);
+
+ BufferOffset store;
+ if (Imm16::IsInSignedRange(dest.offset) &&
+ Imm16::IsInSignedRange(dest.offset + 3)) {
+ store = as_swl(temp, SecondScratchReg, dest.offset + 3);
+ as_swr(temp, SecondScratchReg, dest.offset);
+ } else {
+ ma_li(ScratchRegister, Imm32(dest.offset));
+ as_daddu(ScratchRegister, SecondScratchReg, ScratchRegister);
+ store = as_swl(temp, ScratchRegister, 3);
+ as_swr(temp, ScratchRegister, 0);
+ }
+ append(access, store.getOffset());
+}
+
+void MacroAssemblerMIPSCompat::storeUnalignedDouble(
+ const wasm::MemoryAccessDesc& access, FloatRegister src, Register temp,
+ const BaseIndex& dest) {
+ MOZ_ASSERT(MOZ_LITTLE_ENDIAN(), "Wasm-only; wasm is disabled on big-endian.");
+ computeScaledAddress(dest, SecondScratchReg);
+
+ BufferOffset store;
+ if (Imm16::IsInSignedRange(dest.offset) &&
+ Imm16::IsInSignedRange(dest.offset + 7)) {
+ moveFromDoubleHi(src, temp);
+ store = as_swl(temp, SecondScratchReg, dest.offset + INT64HIGH_OFFSET + 3);
+ as_swr(temp, SecondScratchReg, dest.offset + INT64HIGH_OFFSET);
+ moveFromDoubleLo(src, temp);
+ as_swl(temp, SecondScratchReg, dest.offset + INT64LOW_OFFSET + 3);
+ as_swr(temp, SecondScratchReg, dest.offset + INT64LOW_OFFSET);
+
+ } else {
+ ma_li(ScratchRegister, Imm32(dest.offset));
+ as_daddu(ScratchRegister, SecondScratchReg, ScratchRegister);
+ moveFromDoubleHi(src, temp);
+ store = as_swl(temp, ScratchRegister, INT64HIGH_OFFSET + 3);
+ as_swr(temp, ScratchRegister, INT64HIGH_OFFSET);
+ moveFromDoubleLo(src, temp);
+ as_swl(temp, ScratchRegister, INT64LOW_OFFSET + 3);
+ as_swr(temp, ScratchRegister, INT64LOW_OFFSET);
+ }
+ append(access, store.getOffset());
+}
+
+void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
+ as_roundwd(ScratchDoubleReg, input);
+ ma_li(ScratchRegister, Imm32(255));
+ as_mfc1(output, ScratchDoubleReg);
+ zeroDouble(ScratchDoubleReg);
+ as_sltiu(SecondScratchReg, output, 255);
+ as_colt(DoubleFloat, ScratchDoubleReg, input);
+ // if res > 255; res = 255;
+ as_movz(output, ScratchRegister, SecondScratchReg);
+ // if !(input > 0); res = 0;
+ as_movf(output, zero);
+}
+
+// higher level tag testing code
+Operand MacroAssemblerMIPSCompat::ToPayload(Operand base) {
+ return Operand(Register::FromCode(base.base()), base.disp() + PAYLOAD_OFFSET);
+}
+
+Operand MacroAssemblerMIPSCompat::ToType(Operand base) {
+ return Operand(Register::FromCode(base.base()), base.disp() + TAG_OFFSET);
+}
+
+void MacroAssemblerMIPSCompat::testNullSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp_set(dest, value.typeReg(), ImmType(JSVAL_TYPE_NULL), cond);
+}
+
+void MacroAssemblerMIPSCompat::testObjectSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp_set(dest, value.typeReg(), ImmType(JSVAL_TYPE_OBJECT), cond);
+}
+
+void MacroAssemblerMIPSCompat::testUndefinedSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ma_cmp_set(dest, value.typeReg(), ImmType(JSVAL_TYPE_UNDEFINED), cond);
+}
+
+// unboxing code
+void MacroAssemblerMIPSCompat::unboxNonDouble(const ValueOperand& operand,
+ Register dest, JSValueType) {
+ if (operand.payloadReg() != dest) {
+ ma_move(dest, operand.payloadReg());
+ }
+}
+
+void MacroAssemblerMIPSCompat::unboxNonDouble(const Address& src, Register dest,
+ JSValueType) {
+ ma_lw(dest, Address(src.base, src.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::unboxNonDouble(const BaseIndex& src,
+ Register dest, JSValueType) {
+ computeScaledAddress(src, SecondScratchReg);
+ ma_lw(dest, Address(SecondScratchReg, src.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::unboxInt32(const ValueOperand& operand,
+ Register dest) {
+ ma_move(dest, operand.payloadReg());
+}
+
+void MacroAssemblerMIPSCompat::unboxInt32(const Address& src, Register dest) {
+ ma_lw(dest, Address(src.base, src.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::unboxBoolean(const ValueOperand& operand,
+ Register dest) {
+ ma_move(dest, operand.payloadReg());
+}
+
+void MacroAssemblerMIPSCompat::unboxBoolean(const Address& src, Register dest) {
+ ma_lw(dest, Address(src.base, src.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::unboxDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ moveToDoubleLo(operand.payloadReg(), dest);
+ moveToDoubleHi(operand.typeReg(), dest);
+}
+
+void MacroAssemblerMIPSCompat::unboxDouble(const Address& src,
+ FloatRegister dest) {
+ ma_lw(ScratchRegister, Address(src.base, src.offset + PAYLOAD_OFFSET));
+ moveToDoubleLo(ScratchRegister, dest);
+ ma_lw(ScratchRegister, Address(src.base, src.offset + TAG_OFFSET));
+ moveToDoubleHi(ScratchRegister, dest);
+}
+
+void MacroAssemblerMIPSCompat::unboxDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ loadDouble(src, dest);
+}
+
+void MacroAssemblerMIPSCompat::unboxString(const ValueOperand& operand,
+ Register dest) {
+ ma_move(dest, operand.payloadReg());
+}
+
+void MacroAssemblerMIPSCompat::unboxString(const Address& src, Register dest) {
+ ma_lw(dest, Address(src.base, src.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::unboxBigInt(const ValueOperand& operand,
+ Register dest) {
+ ma_move(dest, operand.payloadReg());
+}
+
+void MacroAssemblerMIPSCompat::unboxBigInt(const Address& src, Register dest) {
+ ma_lw(dest, Address(src.base, src.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::unboxObject(const ValueOperand& src,
+ Register dest) {
+ ma_move(dest, src.payloadReg());
+}
+
+void MacroAssemblerMIPSCompat::unboxObject(const Address& src, Register dest) {
+ ma_lw(dest, Address(src.base, src.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::unboxObjectOrNull(const Address& src,
+ Register dest) {
+ ma_lw(dest, Address(src.base, src.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::unboxValue(const ValueOperand& src,
+ AnyRegister dest, JSValueType) {
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.payloadReg(), dest.fpu());
+ ma_b(&end, ShortJump);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else if (src.payloadReg() != dest.gpr()) {
+ ma_move(dest.gpr(), src.payloadReg());
+ }
+}
+
+void MacroAssemblerMIPSCompat::boxDouble(FloatRegister src,
+ const ValueOperand& dest,
+ FloatRegister) {
+ moveFromDoubleLo(src, dest.payloadReg());
+ moveFromDoubleHi(src, dest.typeReg());
+}
+
+void MacroAssemblerMIPSCompat::boxNonDouble(JSValueType type, Register src,
+ const ValueOperand& dest) {
+ if (src != dest.payloadReg()) {
+ ma_move(dest.payloadReg(), src);
+ }
+ ma_li(dest.typeReg(), ImmType(type));
+}
+
+void MacroAssemblerMIPSCompat::boolValueToDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ convertBoolToInt32(operand.payloadReg(), ScratchRegister);
+ convertInt32ToDouble(ScratchRegister, dest);
+}
+
+void MacroAssemblerMIPSCompat::int32ValueToDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ convertInt32ToDouble(operand.payloadReg(), dest);
+}
+
+void MacroAssemblerMIPSCompat::boolValueToFloat32(const ValueOperand& operand,
+ FloatRegister dest) {
+ convertBoolToInt32(operand.payloadReg(), ScratchRegister);
+ convertInt32ToFloat32(ScratchRegister, dest);
+}
+
+void MacroAssemblerMIPSCompat::int32ValueToFloat32(const ValueOperand& operand,
+ FloatRegister dest) {
+ convertInt32ToFloat32(operand.payloadReg(), dest);
+}
+
+void MacroAssemblerMIPSCompat::loadConstantFloat32(float f,
+ FloatRegister dest) {
+ ma_lis(dest, f);
+}
+
+void MacroAssemblerMIPSCompat::loadInt32OrDouble(const Address& src,
+ FloatRegister dest) {
+ Label notInt32, end;
+ // If it's an int, convert it to double.
+ ma_lw(SecondScratchReg, Address(src.base, src.offset + TAG_OFFSET));
+ asMasm().branchTestInt32(Assembler::NotEqual, SecondScratchReg, &notInt32);
+ ma_lw(SecondScratchReg, Address(src.base, src.offset + PAYLOAD_OFFSET));
+ convertInt32ToDouble(SecondScratchReg, dest);
+ ma_b(&end, ShortJump);
+
+ // Not an int, just load as double.
+ bind(&notInt32);
+ ma_ld(dest, src);
+ bind(&end);
+}
+
+void MacroAssemblerMIPSCompat::loadInt32OrDouble(Register base, Register index,
+ FloatRegister dest,
+ int32_t shift) {
+ Label notInt32, end;
+
+ // If it's an int, convert it to double.
+
+ computeScaledAddress(BaseIndex(base, index, ShiftToScale(shift)),
+ SecondScratchReg);
+ // Since we only have one scratch, we need to stomp over it with the tag.
+ load32(Address(SecondScratchReg, TAG_OFFSET), SecondScratchReg);
+ asMasm().branchTestInt32(Assembler::NotEqual, SecondScratchReg, &notInt32);
+
+ computeScaledAddress(BaseIndex(base, index, ShiftToScale(shift)),
+ SecondScratchReg);
+ load32(Address(SecondScratchReg, PAYLOAD_OFFSET), SecondScratchReg);
+ convertInt32ToDouble(SecondScratchReg, dest);
+ ma_b(&end, ShortJump);
+
+ // Not an int, just load as double.
+ bind(&notInt32);
+ // First, recompute the offset that had been stored in the scratch register
+ // since the scratch register was overwritten loading in the type.
+ computeScaledAddress(BaseIndex(base, index, ShiftToScale(shift)),
+ SecondScratchReg);
+ loadDouble(Address(SecondScratchReg, 0), dest);
+ bind(&end);
+}
+
+void MacroAssemblerMIPSCompat::loadConstantDouble(double dp,
+ FloatRegister dest) {
+ ma_lid(dest, dp);
+}
+
+Register MacroAssemblerMIPSCompat::extractObject(const Address& address,
+ Register scratch) {
+ ma_lw(scratch, Address(address.base, address.offset + PAYLOAD_OFFSET));
+ return scratch;
+}
+
+Register MacroAssemblerMIPSCompat::extractTag(const Address& address,
+ Register scratch) {
+ ma_lw(scratch, Address(address.base, address.offset + TAG_OFFSET));
+ return scratch;
+}
+
+Register MacroAssemblerMIPSCompat::extractTag(const BaseIndex& address,
+ Register scratch) {
+ computeScaledAddress(address, scratch);
+ return extractTag(Address(scratch, address.offset), scratch);
+}
+
+uint32_t MacroAssemblerMIPSCompat::getType(const Value& val) {
+ return val.toNunboxTag();
+}
+
+void MacroAssemblerMIPSCompat::moveData(const Value& val, Register data) {
+ if (val.isGCThing()) {
+ ma_li(data, ImmGCPtr(val.toGCThing()));
+ } else {
+ ma_li(data, Imm32(val.toNunboxPayload()));
+ }
+}
+
+/////////////////////////////////////////////////////////////////
+// X86/X64-common/ARM/MIPS interface.
+/////////////////////////////////////////////////////////////////
+void MacroAssemblerMIPSCompat::storeValue(ValueOperand val, Operand dst) {
+ storeValue(val, Address(Register::FromCode(dst.base()), dst.disp()));
+}
+
+void MacroAssemblerMIPSCompat::storeValue(ValueOperand val,
+ const BaseIndex& dest) {
+ computeScaledAddress(dest, SecondScratchReg);
+ storeValue(val, Address(SecondScratchReg, dest.offset));
+}
+
+void MacroAssemblerMIPSCompat::storeValue(JSValueType type, Register reg,
+ BaseIndex dest) {
+ computeScaledAddress(dest, ScratchRegister);
+
+ // Make sure that ma_sw doesn't clobber ScratchRegister
+ int32_t offset = dest.offset;
+ if (!Imm16::IsInSignedRange(offset)) {
+ ma_li(SecondScratchReg, Imm32(offset));
+ as_addu(ScratchRegister, ScratchRegister, SecondScratchReg);
+ offset = 0;
+ }
+
+ storeValue(type, reg, Address(ScratchRegister, offset));
+}
+
+void MacroAssemblerMIPSCompat::storeValue(ValueOperand val,
+ const Address& dest) {
+ ma_sw(val.payloadReg(), Address(dest.base, dest.offset + PAYLOAD_OFFSET));
+ ma_sw(val.typeReg(), Address(dest.base, dest.offset + TAG_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::storeValue(JSValueType type, Register reg,
+ Address dest) {
+ MOZ_ASSERT(dest.base != SecondScratchReg);
+
+ ma_sw(reg, Address(dest.base, dest.offset + PAYLOAD_OFFSET));
+ ma_li(SecondScratchReg, ImmTag(JSVAL_TYPE_TO_TAG(type)));
+ ma_sw(SecondScratchReg, Address(dest.base, dest.offset + TAG_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::storeValue(const Value& val, Address dest) {
+ MOZ_ASSERT(dest.base != SecondScratchReg);
+
+ ma_li(SecondScratchReg, Imm32(getType(val)));
+ ma_sw(SecondScratchReg, Address(dest.base, dest.offset + TAG_OFFSET));
+ moveData(val, SecondScratchReg);
+ ma_sw(SecondScratchReg, Address(dest.base, dest.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::storeValue(const Value& val, BaseIndex dest) {
+ computeScaledAddress(dest, ScratchRegister);
+
+ // Make sure that ma_sw doesn't clobber ScratchRegister
+ int32_t offset = dest.offset;
+ if (!Imm16::IsInSignedRange(offset)) {
+ ma_li(SecondScratchReg, Imm32(offset));
+ as_addu(ScratchRegister, ScratchRegister, SecondScratchReg);
+ offset = 0;
+ }
+ storeValue(val, Address(ScratchRegister, offset));
+}
+
+void MacroAssemblerMIPSCompat::loadValue(const BaseIndex& addr,
+ ValueOperand val) {
+ computeScaledAddress(addr, SecondScratchReg);
+ loadValue(Address(SecondScratchReg, addr.offset), val);
+}
+
+void MacroAssemblerMIPSCompat::loadValue(Address src, ValueOperand val) {
+ // Ensure that loading the payload does not erase the pointer to the
+ // Value in memory.
+ if (src.base != val.payloadReg()) {
+ ma_lw(val.payloadReg(), Address(src.base, src.offset + PAYLOAD_OFFSET));
+ ma_lw(val.typeReg(), Address(src.base, src.offset + TAG_OFFSET));
+ } else {
+ ma_lw(val.typeReg(), Address(src.base, src.offset + TAG_OFFSET));
+ ma_lw(val.payloadReg(), Address(src.base, src.offset + PAYLOAD_OFFSET));
+ }
+}
+
+void MacroAssemblerMIPSCompat::tagValue(JSValueType type, Register payload,
+ ValueOperand dest) {
+ MOZ_ASSERT(dest.typeReg() != dest.payloadReg());
+ if (payload != dest.payloadReg()) {
+ ma_move(dest.payloadReg(), payload);
+ }
+ ma_li(dest.typeReg(), ImmType(type));
+}
+
+void MacroAssemblerMIPSCompat::pushValue(ValueOperand val) {
+ // Allocate stack slots for type and payload. One for each.
+ asMasm().subPtr(Imm32(sizeof(Value)), StackPointer);
+ // Store type and payload.
+ storeValue(val, Address(StackPointer, 0));
+}
+
+void MacroAssemblerMIPSCompat::pushValue(const Address& addr) {
+ // Allocate stack slots for type and payload. One for each.
+ ma_subu(StackPointer, StackPointer, Imm32(sizeof(Value)));
+ // If address is based on StackPointer its offset needs to be adjusted
+ // to accommodate for previous stack allocation.
+ int32_t offset =
+ addr.base != StackPointer ? addr.offset : addr.offset + sizeof(Value);
+ // Store type and payload.
+ ma_lw(ScratchRegister, Address(addr.base, offset + TAG_OFFSET));
+ ma_sw(ScratchRegister, Address(StackPointer, TAG_OFFSET));
+ ma_lw(ScratchRegister, Address(addr.base, offset + PAYLOAD_OFFSET));
+ ma_sw(ScratchRegister, Address(StackPointer, PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::popValue(ValueOperand val) {
+ // Load payload and type.
+ as_lw(val.payloadReg(), StackPointer, PAYLOAD_OFFSET);
+ as_lw(val.typeReg(), StackPointer, TAG_OFFSET);
+ // Free stack.
+ as_addiu(StackPointer, StackPointer, sizeof(Value));
+}
+
+void MacroAssemblerMIPSCompat::storePayload(const Value& val, Address dest) {
+ moveData(val, SecondScratchReg);
+ ma_sw(SecondScratchReg, Address(dest.base, dest.offset + PAYLOAD_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::storePayload(Register src, Address dest) {
+ ma_sw(src, Address(dest.base, dest.offset + PAYLOAD_OFFSET));
+ return;
+}
+
+void MacroAssemblerMIPSCompat::storePayload(const Value& val,
+ const BaseIndex& dest) {
+ MOZ_ASSERT(dest.offset == 0);
+
+ computeScaledAddress(dest, SecondScratchReg);
+
+ moveData(val, ScratchRegister);
+
+ as_sw(ScratchRegister, SecondScratchReg, NUNBOX32_PAYLOAD_OFFSET);
+}
+
+void MacroAssemblerMIPSCompat::storePayload(Register src,
+ const BaseIndex& dest) {
+ MOZ_ASSERT(dest.offset == 0);
+
+ computeScaledAddress(dest, SecondScratchReg);
+ as_sw(src, SecondScratchReg, NUNBOX32_PAYLOAD_OFFSET);
+}
+
+void MacroAssemblerMIPSCompat::storeTypeTag(ImmTag tag, Address dest) {
+ ma_li(SecondScratchReg, tag);
+ ma_sw(SecondScratchReg, Address(dest.base, dest.offset + TAG_OFFSET));
+}
+
+void MacroAssemblerMIPSCompat::storeTypeTag(ImmTag tag, const BaseIndex& dest) {
+ MOZ_ASSERT(dest.offset == 0);
+
+ computeScaledAddress(dest, SecondScratchReg);
+ ma_li(ScratchRegister, tag);
+ as_sw(ScratchRegister, SecondScratchReg, TAG_OFFSET);
+}
+
+void MacroAssemblerMIPSCompat::breakpoint() { as_break(0); }
+
+void MacroAssemblerMIPSCompat::ensureDouble(const ValueOperand& source,
+ FloatRegister dest,
+ Label* failure) {
+ Label isDouble, done;
+ asMasm().branchTestDouble(Assembler::Equal, source.typeReg(), &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, source.typeReg(), failure);
+
+ convertInt32ToDouble(source.payloadReg(), dest);
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerMIPSCompat::checkStackAlignment() {
+#ifdef DEBUG
+ Label aligned;
+ as_andi(ScratchRegister, sp, ABIStackAlignment - 1);
+ ma_b(ScratchRegister, zero, &aligned, Equal, ShortJump);
+ as_break(BREAK_STACK_UNALIGNED);
+ bind(&aligned);
+#endif
+}
+
+void MacroAssemblerMIPSCompat::alignStackPointer() {
+ movePtr(StackPointer, SecondScratchReg);
+ asMasm().subPtr(Imm32(sizeof(intptr_t)), StackPointer);
+ asMasm().andPtr(Imm32(~(ABIStackAlignment - 1)), StackPointer);
+ storePtr(SecondScratchReg, Address(StackPointer, 0));
+}
+
+void MacroAssemblerMIPSCompat::restoreStackPointer() {
+ loadPtr(Address(StackPointer, 0), StackPointer);
+}
+
+void MacroAssemblerMIPSCompat::handleFailureWithHandlerTail(
+ Label* profilerExitTail) {
+ // Reserve space for exception information.
+ int size = (sizeof(ResumeFromException) + ABIStackAlignment) &
+ ~(ABIStackAlignment - 1);
+ asMasm().subPtr(Imm32(size), StackPointer);
+ ma_move(a0, StackPointer); // Use a0 since it is a first function argument
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(a1);
+ asMasm().passABIArg(a0);
+ asMasm().callWithABI<Fn, HandleException>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ // Already clobbered a0, so use it...
+ load32(Address(StackPointer, ResumeFromException::offsetOfKind()), a0);
+ asMasm().branch32(Assembler::Equal, a0,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, r0,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, r0,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::WasmCatch),
+ &wasmCatch);
+
+ breakpoint(); // Invalid kind.
+
+ // No exception handler. Load the error value, load the new stack pointer
+ // and return from the entry frame.
+ bind(&entryFrame);
+ asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+
+ // We're going to be returning by the ion calling convention
+ ma_pop(ra);
+ as_jr(ra);
+ as_nop();
+
+ // If we found a catch handler, this must be a baseline frame. Restore
+ // state and jump to the catch block.
+ bind(&catch_);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), a0);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ jump(a0);
+
+ // If we found a finally block, this must be a baseline frame. Push two
+ // values expected by the finally block: the exception and BooleanValue(true).
+ bind(&finally);
+ ValueOperand exception = ValueOperand(a1, a2);
+ loadValue(Address(sp, ResumeFromException::offsetOfException()), exception);
+
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a0);
+ loadPtr(Address(sp, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp);
+
+ pushValue(exception);
+ pushValue(BooleanValue(true));
+ jump(a0);
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ ma_move(StackPointer, FramePointer);
+ pop(FramePointer);
+ jump(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(Address(StackPointer, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ // Test if profiler enabled.
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ ret();
+
+ // If we are bailing out to baseline to handle an exception, jump to
+ // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
+ bind(&bailout);
+ loadPtr(Address(sp, ResumeFromException::offsetOfBailoutInfo()), a2);
+ ma_li(ReturnReg, Imm32(1));
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a1);
+ jump(a1);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ ret();
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a1);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ jump(a1);
+}
+
+CodeOffset MacroAssemblerMIPSCompat::toggledJump(Label* label) {
+ CodeOffset ret(nextOffset().getOffset());
+ ma_b(label);
+ return ret;
+}
+
+CodeOffset MacroAssemblerMIPSCompat::toggledCall(JitCode* target,
+ bool enabled) {
+ BufferOffset bo = nextOffset();
+ CodeOffset offset(bo.getOffset());
+ addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(ScratchRegister, ImmPtr(target->raw()));
+ if (enabled) {
+ as_jalr(ScratchRegister);
+ as_nop();
+ } else {
+ as_nop();
+ as_nop();
+ }
+ MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() ==
+ ToggledCallSize(nullptr));
+ return offset;
+}
+
+void MacroAssemblerMIPSCompat::profilerEnterFrame(Register framePtr,
+ Register scratch) {
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr,
+ Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr),
+ Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerMIPSCompat::profilerExitFrame() {
+ jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) {
+ if (imm32.value) {
+ asMasm().subPtr(imm32, StackPointer);
+ }
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// Stack manipulation functions.
+
+size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
+ return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
+}
+
+void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
+ int32_t diffF = set.fpus().getPushSizeInBytes();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+
+ reserveStack(diffG);
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ storePtr(*iter, Address(StackPointer, diffG));
+ }
+ MOZ_ASSERT(diffG == 0);
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ if (diffF > 0) {
+ // Double values have to be aligned. We reserve extra space so that we can
+ // start writing from the first aligned location.
+ // We reserve a whole extra double so that the buffer has even size.
+ ma_and(SecondScratchReg, sp, Imm32(~(ABIStackAlignment - 1)));
+ reserveStack(diffF);
+
+ diffF -= sizeof(double);
+
+ for (FloatRegisterForwardIterator iter(set.fpus().reduceSetForPush());
+ iter.more(); ++iter) {
+ as_sdc1(*iter, SecondScratchReg, -diffF);
+ diffF -= sizeof(double);
+ }
+
+ MOZ_ASSERT(diffF == 0);
+ }
+}
+
+void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
+ LiveRegisterSet ignore) {
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+ int32_t diffF = set.fpus().getPushSizeInBytes();
+ const int32_t reservedG = diffG;
+ const int32_t reservedF = diffF;
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ if (reservedF > 0) {
+ // Read the buffer form the first aligned location.
+ ma_addu(SecondScratchReg, sp, Imm32(reservedF));
+ ma_and(SecondScratchReg, SecondScratchReg, Imm32(~(ABIStackAlignment - 1)));
+
+ diffF -= sizeof(double);
+
+ LiveFloatRegisterSet fpignore(ignore.fpus().reduceSetForPush());
+ for (FloatRegisterForwardIterator iter(set.fpus().reduceSetForPush());
+ iter.more(); ++iter) {
+ if (!ignore.has(*iter)) {
+ as_ldc1(*iter, SecondScratchReg, -diffF);
+ }
+ diffF -= sizeof(double);
+ }
+ freeStack(reservedF);
+ MOZ_ASSERT(diffF == 0);
+ }
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ if (!ignore.has(*iter)) {
+ loadPtr(Address(StackPointer, diffG), *iter);
+ }
+ }
+ freeStack(reservedG);
+ MOZ_ASSERT(diffG == 0);
+}
+
+void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
+ Register scratch) {
+ int32_t diffF = set.fpus().getPushSizeInBytes();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+
+ MOZ_ASSERT(dest.offset >= diffG + diffF);
+ MOZ_ASSERT(dest.base == StackPointer);
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ dest.offset -= sizeof(intptr_t);
+ storePtr(*iter, dest);
+ }
+ MOZ_ASSERT(diffG == 0);
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ if (diffF > 0) {
+ computeEffectiveAddress(dest, scratch);
+ ma_and(scratch, scratch, Imm32(~(ABIStackAlignment - 1)));
+
+ diffF -= sizeof(double);
+
+ for (FloatRegisterForwardIterator iter(set.fpus().reduceSetForPush());
+ iter.more(); ++iter) {
+ as_sdc1(*iter, scratch, -diffF);
+ diffF -= sizeof(double);
+ }
+ MOZ_ASSERT(diffF == 0);
+ }
+}
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+ MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls");
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ ma_move(scratch, StackPointer);
+
+ // Force sp to be aligned
+ asMasm().subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1)));
+ storePtr(scratch, Address(StackPointer, 0));
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ // Reserve place for $ra.
+ stackForCall += sizeof(intptr_t);
+
+ if (dynamicAlignment_) {
+ stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment);
+ } else {
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(
+ stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+ }
+
+ *stackAdjust = stackForCall;
+ reserveStack(stackForCall);
+
+ // Save $ra because call is going to clobber it. Restore it in
+ // callWithABIPost. NOTE: This is needed for calls from SharedIC.
+ // Maybe we can do this differently.
+ storePtr(ra, Address(StackPointer, stackForCall - sizeof(intptr_t)));
+
+ // Position all arguments.
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_) {
+ return;
+ }
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool callFromWasm) {
+ // Restore ra value (as stored in callWithABIPre()).
+ loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra);
+
+ if (dynamicAlignment_) {
+ // Restore sp value from stack (as stored in setupUnalignedABICall()).
+ loadPtr(Address(StackPointer, stackAdjust), StackPointer);
+ // Use adjustFrame instead of freeStack because we already restored sp.
+ adjustFrame(-stackAdjust);
+ } else {
+ freeStack(stackAdjust);
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ // Load the callee in t9, no instruction between the lw and call
+ // should clobber it. Note that we can't use fun.base because it may
+ // be one of the IntArg registers clobbered before the call.
+ ma_move(t9, fun);
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(t9);
+ callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ // Load the callee in t9, as above.
+ loadPtr(Address(fun.base, fun.offset), t9);
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(t9);
+ callWithABIPost(stackAdjust, result);
+}
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ if (reg.gpr() != dest.payloadReg()) {
+ move32(reg.gpr(), dest.payloadReg());
+ }
+ mov(ImmWord(MIRTypeToTag(type)), dest.typeReg());
+ return;
+ }
+
+ ScratchDoubleScope scratch(*this);
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ boxDouble(freg, dest, scratch);
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ Register s0 = src.typeReg();
+ Register s1 = src.payloadReg();
+ Register d0 = dest.typeReg();
+ Register d1 = dest.payloadReg();
+
+ // Either one or both of the source registers could be the same as a
+ // destination register.
+ if (s1 == d0) {
+ if (s0 == d1) {
+ // If both are, this is just a swap of two registers.
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(d1 != scratch);
+ MOZ_ASSERT(d0 != scratch);
+ move32(d1, scratch);
+ move32(d0, d1);
+ move32(scratch, d0);
+ return;
+ }
+ // If only one is, copy that source first.
+ std::swap(s0, s1);
+ std::swap(d0, d1);
+ }
+
+ if (s0 != d0) {
+ move32(s0, d0);
+ }
+ if (s1 != d1) {
+ move32(s1, d1);
+ }
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ move32(Imm32(src.toNunboxTag()), dest.typeReg());
+ if (src.isGCThing()) {
+ movePtr(ImmGCPtr(src.toGCThing()), dest.payloadReg());
+ } else {
+ move32(Imm32(src.toNunboxPayload()), dest.payloadReg());
+ }
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label done;
+
+ branchTestGCThing(Assembler::NotEqual, address,
+ cond == Assembler::Equal ? &done : label);
+
+ loadPtr(address, temp);
+ branchPtrInNurseryChunk(cond, temp, InvalidReg, label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label done;
+
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+ branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(*this);
+ moveData(rhs, scratch);
+
+ if (cond == Equal) {
+ Label done;
+ ma_b(lhs.payloadReg(), scratch, &done, NotEqual, ShortJump);
+ { ma_b(lhs.typeReg(), Imm32(getType(rhs)), label, Equal); }
+ bind(&done);
+ } else {
+ ma_b(lhs.payloadReg(), scratch, label, NotEqual);
+
+ ma_b(lhs.typeReg(), Imm32(getType(rhs)), label, NotEqual);
+ }
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ storeDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ // Store the type tag.
+ storeTypeTag(ImmType(ValueTypeFromMIRType(valueType)), dest);
+
+ // Store the payload.
+ if (value.constant()) {
+ storePayload(value.value(), dest);
+ } else {
+ storePayload(value.reg().typedReg().gpr(), dest);
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+void MacroAssembler::PushBoxed(FloatRegister reg) { Push(reg); }
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* ok) {
+ ma_b(index, boundsCheckLimit, ok, cond);
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* ok) {
+ SecondScratchRegisterScope scratch2(*this);
+ load32(boundsCheckLimit, SecondScratchReg);
+ ma_b(index, SecondScratchReg, ok, cond);
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ Label done;
+
+ as_truncwd(ScratchFloat32Reg, input);
+ ma_li(ScratchRegister, Imm32(INT32_MAX));
+ moveFromFloat32(ScratchFloat32Reg, output);
+
+ // For numbers in -1.[ : ]INT32_MAX range do nothing more
+ ma_b(output, ScratchRegister, &done, Assembler::Below, ShortJump);
+
+ loadConstantDouble(double(INT32_MAX + 1ULL), ScratchDoubleReg);
+ ma_li(ScratchRegister, Imm32(INT32_MIN));
+ as_subd(ScratchDoubleReg, input, ScratchDoubleReg);
+ as_truncwd(ScratchFloat32Reg, ScratchDoubleReg);
+ as_cfc1(SecondScratchReg, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, output);
+ ma_ext(SecondScratchReg, SecondScratchReg, Assembler::CauseV, 1);
+ ma_addu(output, ScratchRegister);
+
+ ma_b(SecondScratchReg, Imm32(0), oolEntry, Assembler::NotEqual);
+
+ bind(&done);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ Label done;
+
+ as_truncws(ScratchFloat32Reg, input);
+ ma_li(ScratchRegister, Imm32(INT32_MAX));
+ moveFromFloat32(ScratchFloat32Reg, output);
+ // For numbers in -1.[ : ]INT32_MAX range do nothing more
+ ma_b(output, ScratchRegister, &done, Assembler::Below, ShortJump);
+
+ loadConstantFloat32(float(INT32_MAX + 1ULL), ScratchFloat32Reg);
+ ma_li(ScratchRegister, Imm32(INT32_MIN));
+ as_subs(ScratchFloat32Reg, input, ScratchFloat32Reg);
+ as_truncws(ScratchFloat32Reg, ScratchFloat32Reg);
+ as_cfc1(SecondScratchReg, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, output);
+ ma_ext(SecondScratchReg, SecondScratchReg, Assembler::CauseV, 1);
+ ma_addu(output, ScratchRegister);
+
+ // Guard against negative values that result in 0 due the precision loss.
+ as_sltiu(ScratchRegister, output, 1);
+ ma_or(SecondScratchReg, ScratchRegister);
+
+ ma_b(SecondScratchReg, Imm32(0), oolEntry, Assembler::NotEqual);
+
+ bind(&done);
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output) {
+ wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
+}
+
+void MacroAssembler::wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch,
+ Register64 output, Register tmp) {
+ wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, tmp);
+}
+
+void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch) {
+ wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
+}
+
+void MacroAssembler::wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register tmp) {
+ wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, tmp);
+}
+
+void MacroAssemblerMIPSCompat::wasmLoadI64Impl(
+ const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ MOZ_ASSERT(!access.isZeroExtendSimd128Load());
+ MOZ_ASSERT(!access.isSplatSimd128Load());
+ MOZ_ASSERT(!access.isWidenSimd128Load());
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().movePtr(ptr, ptrScratch);
+ asMasm().addPtr(Imm32(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+
+ switch (access.type()) {
+ case Scalar::Int8:
+ isSigned = true;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ break;
+ case Scalar::Int32:
+ isSigned = true;
+ break;
+ case Scalar::Uint32:
+ isSigned = false;
+ break;
+ case Scalar::Int64:
+ isSigned = true;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ BaseIndex address(memoryBase, ptr, TimesOne);
+ MOZ_ASSERT(INT64LOW_OFFSET == 0);
+ if (IsUnaligned(access)) {
+ MOZ_ASSERT(tmp != InvalidReg);
+ if (byteSize <= 4) {
+ asMasm().ma_load_unaligned(access, output.low, address, tmp,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ if (!isSigned) {
+ asMasm().move32(Imm32(0), output.high);
+ } else {
+ asMasm().ma_sra(output.high, output.low, Imm32(31));
+ }
+ } else {
+ MOZ_ASSERT(output.low != ptr);
+ asMasm().ma_load_unaligned(access, output.low, address, tmp, SizeWord,
+ ZeroExtend);
+ asMasm().ma_load_unaligned(
+ access, output.high,
+ BaseIndex(HeapReg, ptr, TimesOne, INT64HIGH_OFFSET), tmp, SizeWord,
+ SignExtend);
+ }
+ return;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+ if (byteSize <= 4) {
+ asMasm().ma_load(output.low, address,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ asMasm().append(access, asMasm().size() - 4);
+ if (!isSigned) {
+ asMasm().move32(Imm32(0), output.high);
+ } else {
+ asMasm().ma_sra(output.high, output.low, Imm32(31));
+ }
+ } else {
+ MOZ_ASSERT(output.low != ptr);
+ asMasm().ma_load(output.low, BaseIndex(HeapReg, ptr, TimesOne), SizeWord);
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().ma_load(output.high,
+ BaseIndex(HeapReg, ptr, TimesOne, INT64HIGH_OFFSET),
+ SizeWord);
+ asMasm().append(access, asMasm().size() - 4);
+ }
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerMIPSCompat::wasmStoreI64Impl(
+ const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(Imm32(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+ switch (access.type()) {
+ case Scalar::Int8:
+ isSigned = true;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ break;
+ case Scalar::Int32:
+ isSigned = true;
+ break;
+ case Scalar::Uint32:
+ isSigned = false;
+ break;
+ case Scalar::Int64:
+ isSigned = true;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ MOZ_ASSERT(INT64LOW_OFFSET == 0);
+ BaseIndex address(memoryBase, ptr, TimesOne);
+ if (IsUnaligned(access)) {
+ MOZ_ASSERT(tmp != InvalidReg);
+ if (byteSize <= 4) {
+ asMasm().ma_store_unaligned(access, value.low, address, tmp,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ } else {
+ asMasm().ma_store_unaligned(
+ access, value.high,
+ BaseIndex(HeapReg, ptr, TimesOne, INT64HIGH_OFFSET), tmp, SizeWord,
+ SignExtend);
+ asMasm().ma_store_unaligned(access, value.low, address, tmp, SizeWord,
+ ZeroExtend);
+ }
+ return;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+ if (byteSize <= 4) {
+ asMasm().ma_store(value.low, address,
+ static_cast<LoadStoreSize>(8 * byteSize));
+ asMasm().append(access, asMasm().size() - 4);
+ } else {
+ asMasm().ma_store(value.high,
+ BaseIndex(HeapReg, ptr, TimesOne, INT64HIGH_OFFSET),
+ SizeWord);
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().ma_store(value.low, address, SizeWord);
+ }
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+static void EnterAtomic64Region(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access,
+ Register addr, Register spinlock,
+ Register scratch) {
+ masm.movePtr(wasm::SymbolicAddress::js_jit_gAtomic64Lock, spinlock);
+
+ masm.append(access, masm.size());
+ masm.as_lbu(
+ zero, addr,
+ 7); // Force memory trap on invalid access before we enter the spinlock.
+
+ Label tryLock;
+
+ masm.memoryBarrier(MembarFull);
+
+ masm.bind(&tryLock);
+
+ masm.as_ll(scratch, spinlock, 0);
+ masm.ma_b(scratch, scratch, &tryLock, Assembler::NonZero, ShortJump);
+ masm.ma_li(scratch, Imm32(1));
+ masm.as_sc(scratch, spinlock, 0);
+ masm.ma_b(scratch, scratch, &tryLock, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrier(MembarFull);
+}
+
+static void ExitAtomic64Region(MacroAssembler& masm, Register spinlock) {
+ masm.memoryBarrier(MembarFull);
+ masm.as_sw(zero, spinlock, 0);
+ masm.memoryBarrier(MembarFull);
+}
+
+template <typename T>
+static void AtomicLoad64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access, const T& mem,
+ Register64 temp, Register64 output) {
+ MOZ_ASSERT(temp.low == InvalidReg && temp.high == InvalidReg);
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ EnterAtomic64Region(masm, access, /* addr= */ SecondScratchReg,
+ /* spinlock= */ ScratchRegister,
+ /* scratch= */ output.low);
+
+ masm.load64(Address(SecondScratchReg, 0), output);
+
+ ExitAtomic64Region(masm, /* spinlock= */ ScratchRegister);
+}
+
+void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicLoad64(*this, access, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicLoad64(*this, access, mem, temp, output);
+}
+
+template <typename T>
+void MacroAssemblerMIPSCompat::wasmAtomicStore64(
+ const wasm::MemoryAccessDesc& access, const T& mem, Register temp,
+ Register64 value) {
+ computeEffectiveAddress(mem, SecondScratchReg);
+
+ EnterAtomic64Region(asMasm(), access, /* addr= */ SecondScratchReg,
+ /* spinlock= */ ScratchRegister, /* scratch= */ temp);
+
+ store64(value, Address(SecondScratchReg, 0));
+
+ ExitAtomic64Region(asMasm(), /* spinlock= */ ScratchRegister);
+}
+
+template void MacroAssemblerMIPSCompat::wasmAtomicStore64(
+ const wasm::MemoryAccessDesc& access, const Address& mem, Register temp,
+ Register64 value);
+template void MacroAssemblerMIPSCompat::wasmAtomicStore64(
+ const wasm::MemoryAccessDesc& access, const BaseIndex& mem, Register temp,
+ Register64 value);
+
+template <typename T>
+static void WasmCompareExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access,
+ const T& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ MOZ_ASSERT(output != expect);
+ MOZ_ASSERT(output != replace);
+
+ Label exit;
+
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+ Address addr(SecondScratchReg, 0);
+
+ EnterAtomic64Region(masm, access, /* addr= */ SecondScratchReg,
+ /* spinlock= */ ScratchRegister,
+ /* scratch= */ output.low);
+
+ masm.load64(addr, output);
+
+ masm.ma_b(output.low, expect.low, &exit, Assembler::NotEqual, ShortJump);
+ masm.ma_b(output.high, expect.high, &exit, Assembler::NotEqual, ShortJump);
+ masm.store64(replace, addr);
+ masm.bind(&exit);
+ ExitAtomic64Region(masm, /* spinlock= */ ScratchRegister);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ WasmCompareExchange64(*this, access, mem, expect, replace, output);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ WasmCompareExchange64(*this, access, mem, expect, replace, output);
+}
+
+template <typename T>
+static void WasmAtomicExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access,
+ const T& mem, Register64 src,
+ Register64 output) {
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+ Address addr(SecondScratchReg, 0);
+
+ EnterAtomic64Region(masm, access, /* addr= */ SecondScratchReg,
+ /* spinlock= */ ScratchRegister,
+ /* scratch= */ output.low);
+
+ masm.load64(addr, output);
+ masm.store64(src, addr);
+
+ ExitAtomic64Region(masm, /* spinlock= */ ScratchRegister);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 src,
+ Register64 output) {
+ WasmAtomicExchange64(*this, access, mem, src, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 src,
+ Register64 output) {
+ WasmAtomicExchange64(*this, access, mem, src, output);
+}
+
+template <typename T>
+static void AtomicFetchOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access, AtomicOp op,
+ Register64 value, const T& mem, Register64 temp,
+ Register64 output) {
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ EnterAtomic64Region(masm, access, /* addr= */ SecondScratchReg,
+ /* spinlock= */ ScratchRegister,
+ /* scratch= */ output.low);
+
+ masm.load64(Address(SecondScratchReg, 0), output);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_addu(temp.low, output.low, value.low);
+ masm.as_sltu(temp.high, temp.low, output.low);
+ masm.as_addu(temp.high, temp.high, output.high);
+ masm.as_addu(temp.high, temp.high, value.high);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_sltu(temp.high, output.low, value.low);
+ masm.as_subu(temp.high, output.high, temp.high);
+ masm.as_subu(temp.low, output.low, value.low);
+ masm.as_subu(temp.high, temp.high, value.high);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(temp.low, output.low, value.low);
+ masm.as_and(temp.high, output.high, value.high);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(temp.low, output.low, value.low);
+ masm.as_or(temp.high, output.high, value.high);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(temp.low, output.low, value.low);
+ masm.as_xor(temp.high, output.high, value.high);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.store64(temp, Address(SecondScratchReg, 0));
+
+ ExitAtomic64Region(masm, /* spinlock= */ ScratchRegister);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, access, op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, access, op, value, mem, temp, output);
+}
+
+// ========================================================================
+// Convert floating point.
+
+static const double TO_DOUBLE_HIGH_SCALE = 0x100000000;
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ convertUInt32ToDouble(src.high, dest);
+ loadConstantDouble(TO_DOUBLE_HIGH_SCALE, ScratchDoubleReg);
+ mulDouble(ScratchDoubleReg, dest);
+ convertUInt32ToDouble(src.low, ScratchDoubleReg);
+ addDouble(ScratchDoubleReg, dest);
+}
+
+void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) {
+ convertInt32ToDouble(src.high, dest);
+ loadConstantDouble(TO_DOUBLE_HIGH_SCALE, ScratchDoubleReg);
+ mulDouble(ScratchDoubleReg, dest);
+ convertUInt32ToDouble(src.low, ScratchDoubleReg);
+ addDouble(ScratchDoubleReg, dest);
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ convertInt32ToDouble(src, dest);
+}
+
+//}}} check_macroassembler_style
diff --git a/js/src/jit/mips32/MacroAssembler-mips32.h b/js/src/jit/mips32/MacroAssembler-mips32.h
new file mode 100644
index 0000000000..c39def4ebd
--- /dev/null
+++ b/js/src/jit/mips32/MacroAssembler-mips32.h
@@ -0,0 +1,829 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_MacroAssembler_mips32_h
+#define jit_mips32_MacroAssembler_mips32_h
+
+#include "mozilla/EndianUtils.h"
+
+#include "jit/mips-shared/MacroAssembler-mips-shared.h"
+#include "jit/MoveResolver.h"
+#include "vm/BytecodeUtil.h"
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+struct ImmTag : public Imm32 {
+ ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {}
+};
+
+struct ImmType : public ImmTag {
+ ImmType(JSValueType type) : ImmTag(JSVAL_TYPE_TO_TAG(type)) {}
+};
+
+static constexpr ValueOperand JSReturnOperand{JSReturnReg_Type,
+ JSReturnReg_Data};
+static const ValueOperand softfpReturnOperand = ValueOperand(v1, v0);
+
+static const int defaultShift = 3;
+static_assert(1 << defaultShift == sizeof(JS::Value),
+ "The defaultShift is wrong");
+
+static const uint32_t LOW_32_MASK = (1LL << 32) - 1;
+#if MOZ_LITTLE_ENDIAN()
+static const int32_t LOW_32_OFFSET = 0;
+static const int32_t HIGH_32_OFFSET = 4;
+#else
+static const int32_t LOW_32_OFFSET = 4;
+static const int32_t HIGH_32_OFFSET = 0;
+#endif
+
+// See documentation for ScratchTagScope and ScratchTagScopeRelease in
+// MacroAssembler-x64.h.
+
+class ScratchTagScope {
+ const ValueOperand& v_;
+
+ public:
+ ScratchTagScope(MacroAssembler&, const ValueOperand& v) : v_(v) {}
+ operator Register() { return v_.typeReg(); }
+ void release() {}
+ void reacquire() {}
+};
+
+class ScratchTagScopeRelease {
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope*) {}
+};
+
+class MacroAssemblerMIPS : public MacroAssemblerMIPSShared {
+ public:
+ using MacroAssemblerMIPSShared::ma_b;
+ using MacroAssemblerMIPSShared::ma_cmp_set;
+ using MacroAssemblerMIPSShared::ma_ld;
+ using MacroAssemblerMIPSShared::ma_li;
+ using MacroAssemblerMIPSShared::ma_liPatchable;
+ using MacroAssemblerMIPSShared::ma_load;
+ using MacroAssemblerMIPSShared::ma_ls;
+ using MacroAssemblerMIPSShared::ma_sd;
+ using MacroAssemblerMIPSShared::ma_ss;
+ using MacroAssemblerMIPSShared::ma_store;
+ using MacroAssemblerMIPSShared::ma_sub32TestOverflow;
+
+ void ma_li(Register dest, CodeLabel* label);
+
+ void ma_li(Register dest, ImmWord imm);
+ void ma_liPatchable(Register dest, ImmPtr imm);
+ void ma_liPatchable(Register dest, ImmWord imm);
+
+ // load
+ void ma_load(Register dest, Address address, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // store
+ void ma_store(Register data, Address address, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // arithmetic based ops
+ // add
+ void ma_add32TestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_add32TestOverflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+
+ void ma_addPtrTestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow) {
+ ma_add32TestOverflow(rd, rs, rt, overflow);
+ }
+
+ void ma_addPtrTestOverflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow) {
+ ma_add32TestOverflow(rd, rs, imm, overflow);
+ }
+
+ void ma_addPtrTestCarry(Condition cond, Register rd, Register rs, Register rt,
+ Label* overflow) {
+ ma_add32TestCarry(cond, rd, rs, rt, overflow);
+ }
+
+ void ma_addPtrTestCarry(Condition cond, Register rd, Register rs, Imm32 imm,
+ Label* overflow) {
+ ma_add32TestCarry(cond, rd, rs, imm, overflow);
+ }
+
+ // subtract
+ void ma_sub32TestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+
+ void ma_subPtrTestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow) {
+ ma_sub32TestOverflow(rd, rs, rt, overflow);
+ }
+
+ void ma_subPtrTestOverflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow) {
+ ma_li(ScratchRegister, imm);
+ ma_sub32TestOverflow(rd, rs, ScratchRegister, overflow);
+ }
+
+ void ma_mulPtrTestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow) {
+ ma_mul32TestOverflow(rd, rs, rt, overflow);
+ }
+
+ // memory
+ // shortcut for when we know we're transferring 32 bits of data
+ void ma_lw(Register data, Address address);
+
+ void ma_sw(Register data, Address address);
+ void ma_sw(Imm32 imm, Address address);
+ void ma_sw(Register data, BaseIndex& address);
+
+ void ma_pop(Register r);
+ void ma_push(Register r);
+
+ void branchWithCode(InstImm code, Label* label, JumpKind jumpKind);
+ // branches when done from within mips-specific code
+ void ma_b(Register lhs, ImmWord imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump) {
+ ma_b(lhs, Imm32(uint32_t(imm.value)), l, c, jumpKind);
+ }
+ void ma_b(Address addr, ImmWord imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump) {
+ ma_b(addr, Imm32(uint32_t(imm.value)), l, c, jumpKind);
+ }
+
+ void ma_b(Register lhs, Address addr, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, Imm32 imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, ImmGCPtr imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, Register rhs, Label* l, Condition c,
+ JumpKind jumpKind = LongJump) {
+ MOZ_ASSERT(rhs != ScratchRegister);
+ ma_lw(ScratchRegister, addr);
+ ma_b(ScratchRegister, rhs, l, c, jumpKind);
+ }
+
+ void ma_bal(Label* l, DelaySlotFill delaySlotFill = FillDelaySlot);
+
+ // fp instructions
+ void ma_lid(FloatRegister dest, double value);
+
+ void ma_mv(FloatRegister src, ValueOperand dest);
+ void ma_mv(ValueOperand src, FloatRegister dest);
+
+ void ma_ls(FloatRegister ft, Address address);
+ void ma_ld(FloatRegister ft, Address address);
+ void ma_sd(FloatRegister ft, Address address);
+ void ma_ss(FloatRegister ft, Address address);
+
+ void ma_ldc1WordAligned(FloatRegister ft, Register base, int32_t off);
+ void ma_sdc1WordAligned(FloatRegister ft, Register base, int32_t off);
+
+ void ma_pop(FloatRegister f);
+ void ma_push(FloatRegister f);
+
+ void ma_cmp_set(Register dst, Register lhs, ImmWord imm, Condition c) {
+ ma_cmp_set(dst, lhs, Imm32(uint32_t(imm.value)), c);
+ }
+ void ma_cmp_set(Register dst, Register lhs, ImmPtr imm, Condition c) {
+ ma_cmp_set(dst, lhs, ImmWord(uintptr_t(imm.value)), c);
+ }
+ void ma_cmp_set(Register dst, Register lhs, Address addr, Condition c) {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_lw(ScratchRegister, addr);
+ ma_cmp_set(dst, lhs, ScratchRegister, c);
+ }
+ void ma_cmp_set(Register dst, Address lhs, Register rhs, Condition c) {
+ MOZ_ASSERT(rhs != ScratchRegister);
+ ma_lw(ScratchRegister, lhs);
+ ma_cmp_set(dst, ScratchRegister, rhs, c);
+ }
+ void ma_cmp_set(Register dst, Address lhs, ImmPtr imm, Condition c) {
+ ma_lw(SecondScratchReg, lhs);
+ ma_cmp_set(dst, SecondScratchReg, imm, c);
+ }
+
+ // These fuctions abstract the access to high part of the double precision
+ // float register. It is intended to work on both 32 bit and 64 bit
+ // floating point coprocessor.
+ // :TODO: (Bug 985881) Modify this for N32 ABI to use mthc1 and mfhc1
+ void moveToDoubleHi(Register src, FloatRegister dest) {
+ as_mtc1(src, getOddPair(dest));
+ }
+ void moveFromDoubleHi(FloatRegister src, Register dest) {
+ as_mfc1(dest, getOddPair(src));
+ }
+};
+
+class MacroAssembler;
+
+class MacroAssemblerMIPSCompat : public MacroAssemblerMIPS {
+ public:
+ using MacroAssemblerMIPS::call;
+
+ MacroAssemblerMIPSCompat() {}
+
+ void convertBoolToInt32(Register source, Register dest);
+ void convertInt32ToDouble(Register src, FloatRegister dest);
+ void convertInt32ToDouble(const Address& src, FloatRegister dest);
+ void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest);
+ void convertUInt32ToDouble(Register src, FloatRegister dest);
+ void convertUInt32ToFloat32(Register src, FloatRegister dest);
+ void convertDoubleToFloat32(FloatRegister src, FloatRegister dest);
+ void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+
+ void convertFloat32ToDouble(FloatRegister src, FloatRegister dest);
+ void convertInt32ToFloat32(Register src, FloatRegister dest);
+ void convertInt32ToFloat32(const Address& src, FloatRegister dest);
+
+ void computeScaledAddress(const BaseIndex& address, Register dest);
+
+ void computeEffectiveAddress(const Address& address, Register dest) {
+ ma_addu(dest, address.base, Imm32(address.offset));
+ }
+
+ inline void computeEffectiveAddress(const BaseIndex& address, Register dest);
+
+ void j(Label* dest) { ma_b(dest); }
+
+ void mov(Register src, Register dest) { as_ori(dest, src, 0); }
+ void mov(ImmWord imm, Register dest) { ma_li(dest, imm); }
+ void mov(ImmPtr imm, Register dest) {
+ mov(ImmWord(uintptr_t(imm.value)), dest);
+ }
+ void mov(CodeLabel* label, Register dest) { ma_li(dest, label); }
+ void mov(Register src, Address dest) { MOZ_CRASH("NYI-IC"); }
+ void mov(Address src, Register dest) { MOZ_CRASH("NYI-IC"); }
+
+ void branch(JitCode* c) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(ScratchRegister, ImmPtr(c->raw()));
+ as_jr(ScratchRegister);
+ as_nop();
+ }
+ void branch(const Register reg) {
+ as_jr(reg);
+ as_nop();
+ }
+ void nop() { as_nop(); }
+ void ret() {
+ ma_pop(ra);
+ as_jr(ra);
+ as_nop();
+ }
+ inline void retn(Imm32 n);
+ void push(Imm32 imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(ImmWord imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(ImmGCPtr imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(const Address& address) {
+ loadPtr(address, ScratchRegister);
+ ma_push(ScratchRegister);
+ }
+ void push(Register reg) { ma_push(reg); }
+ void push(FloatRegister reg) { ma_push(reg); }
+ void pop(Register reg) { ma_pop(reg); }
+ void pop(FloatRegister reg) { ma_pop(reg); }
+
+ // Emit a branch that can be toggled to a non-operation. On MIPS we use
+ // "andi" instruction to toggle the branch.
+ // See ToggleToJmp(), ToggleToCmp().
+ CodeOffset toggledJump(Label* label);
+
+ // Emit a "jalr" or "nop" instruction. ToggleCall can be used to patch
+ // this instruction.
+ CodeOffset toggledCall(JitCode* target, bool enabled);
+
+ static size_t ToggledCallSize(uint8_t* code) {
+ // Four instructions used in: MacroAssemblerMIPSCompat::toggledCall
+ return 4 * sizeof(uint32_t);
+ }
+
+ CodeOffset pushWithPatch(ImmWord imm) {
+ CodeOffset label = movWithPatch(imm, ScratchRegister);
+ ma_push(ScratchRegister);
+ return label;
+ }
+
+ CodeOffset movWithPatch(ImmWord imm, Register dest) {
+ CodeOffset label = CodeOffset(currentOffset());
+ ma_liPatchable(dest, imm);
+ return label;
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ return movWithPatch(ImmWord(uintptr_t(imm.value)), dest);
+ }
+
+ void writeCodePointer(CodeLabel* label) {
+ BufferOffset off = writeInst(-1);
+ label->patchAt()->bind(off.getOffset());
+ label->setLinkMode(CodeLabel::RawPointer);
+ }
+
+ void jump(Label* label) { ma_b(label); }
+ void jump(Register reg) {
+ as_jr(reg);
+ as_nop();
+ }
+ void jump(const Address& address) {
+ loadPtr(address, ScratchRegister);
+ as_jr(ScratchRegister);
+ as_nop();
+ }
+
+ void jump(JitCode* code) { branch(code); }
+
+ void jump(ImmPtr ptr) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ptr, RelocationKind::HARDCODED);
+ ma_jump(ptr);
+ }
+
+ void jump(TrampolinePtr code) { jump(ImmPtr(code.value)); }
+
+ void negl(Register reg) { ma_negu(reg, reg); }
+
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ MOZ_ASSERT(value.typeReg() == tag);
+ }
+
+ // unboxing code
+ void unboxNonDouble(const ValueOperand& operand, Register dest, JSValueType);
+ void unboxNonDouble(const Address& src, Register dest, JSValueType);
+ void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType);
+ void unboxInt32(const ValueOperand& operand, Register dest);
+ void unboxInt32(const Address& src, Register dest);
+ void unboxBoolean(const ValueOperand& operand, Register dest);
+ void unboxBoolean(const Address& src, Register dest);
+ void unboxDouble(const ValueOperand& operand, FloatRegister dest);
+ void unboxDouble(const Address& src, FloatRegister dest);
+ void unboxDouble(const BaseIndex& src, FloatRegister dest);
+ void unboxString(const ValueOperand& operand, Register dest);
+ void unboxString(const Address& src, Register dest);
+ void unboxBigInt(const ValueOperand& operand, Register dest);
+ void unboxBigInt(const Address& src, Register dest);
+ void unboxObject(const ValueOperand& src, Register dest);
+ void unboxObject(const Address& src, Register dest);
+ void unboxObject(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObjectOrNull(const Address& src, Register dest);
+ void unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType);
+
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ unboxObject(src, dest);
+ }
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ unboxObject(src, dest);
+ }
+
+ void notBoolean(const ValueOperand& val) {
+ as_xori(val.payloadReg(), val.payloadReg(), 1);
+ }
+
+ // boxing code
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister);
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ [[nodiscard]] Register extractObject(const Address& address,
+ Register scratch);
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractString(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch);
+ [[nodiscard]] Register extractTag(const BaseIndex& address, Register scratch);
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ return value.typeReg();
+ }
+
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void loadInt32OrDouble(const Address& address, FloatRegister dest);
+ void loadInt32OrDouble(Register base, Register index, FloatRegister dest,
+ int32_t shift = defaultShift);
+ void loadConstantDouble(double dp, FloatRegister dest);
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest);
+
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest);
+
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest);
+
+ // higher level tag testing code
+ Operand ToPayload(Operand base);
+ Address ToPayload(Address base) {
+ return ToPayload(Operand(base)).toAddress();
+ }
+
+ BaseIndex ToPayload(BaseIndex base) {
+ return BaseIndex(base.base, base.index, base.scale,
+ base.offset + NUNBOX32_PAYLOAD_OFFSET);
+ }
+
+ protected:
+ Operand ToType(Operand base);
+ Address ToType(Address base) { return ToType(Operand(base)).toAddress(); }
+
+ uint32_t getType(const Value& val);
+ void moveData(const Value& val, Register data);
+
+ public:
+ void moveValue(const Value& val, Register type, Register data);
+
+ void loadUnboxedValue(Address address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(address, dest.fpu());
+ } else {
+ ma_lw(dest.gpr(), ToPayload(address));
+ }
+ }
+
+ void loadUnboxedValue(BaseIndex address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(address.base, address.index, dest.fpu(), address.scale);
+ } else {
+ load32(ToPayload(address), dest.gpr());
+ }
+ }
+
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes,
+ JSValueType) {
+ switch (nbytes) {
+ case 4:
+ store32(value.payloadReg(), address);
+ return;
+ case 1:
+ store8(value.payloadReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void moveValue(const Value& val, const ValueOperand& dest);
+
+ void moveValue(const ValueOperand& src, const ValueOperand& dest) {
+ Register s0 = src.typeReg(), d0 = dest.typeReg(), s1 = src.payloadReg(),
+ d1 = dest.payloadReg();
+
+ // Either one or both of the source registers could be the same as a
+ // destination register.
+ if (s1 == d0) {
+ if (s0 == d1) {
+ // If both are, this is just a swap of two registers.
+ MOZ_ASSERT(d1 != ScratchRegister);
+ MOZ_ASSERT(d0 != ScratchRegister);
+ move32(d1, ScratchRegister);
+ move32(d0, d1);
+ move32(ScratchRegister, d0);
+ return;
+ }
+ // If only one is, copy that source first.
+ std::swap(s0, s1);
+ std::swap(d0, d1);
+ }
+
+ if (s0 != d0) {
+ move32(s0, d0);
+ }
+ if (s1 != d1) {
+ move32(s1, d1);
+ }
+ }
+
+ void storeValue(ValueOperand val, Operand dst);
+ void storeValue(ValueOperand val, const BaseIndex& dest);
+ void storeValue(JSValueType type, Register reg, BaseIndex dest);
+ void storeValue(ValueOperand val, const Address& dest);
+ void storeValue(JSValueType type, Register reg, Address dest);
+ void storeValue(const Value& val, Address dest);
+ void storeValue(const Value& val, BaseIndex dest);
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ load32(ToType(src), temp);
+ store32(temp, ToType(dest));
+
+ load32(ToPayload(src), temp);
+ store32(temp, ToPayload(dest));
+ }
+
+ void loadValue(Address src, ValueOperand val);
+ void loadValue(Operand dest, ValueOperand val) {
+ loadValue(dest.toAddress(), val);
+ }
+ void loadValue(const BaseIndex& addr, ValueOperand val);
+
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+
+ void tagValue(JSValueType type, Register payload, ValueOperand dest);
+
+ void pushValue(ValueOperand val);
+ void popValue(ValueOperand val);
+#if MOZ_LITTLE_ENDIAN()
+ void pushValue(const Value& val) {
+ push(Imm32(val.toNunboxTag()));
+ if (val.isGCThing()) {
+ push(ImmGCPtr(val.toGCThing()));
+ } else {
+ push(Imm32(val.toNunboxPayload()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ push(ImmTag(JSVAL_TYPE_TO_TAG(type)));
+ ma_push(reg);
+ }
+#else
+ void pushValue(const Value& val) {
+ if (val.isGCThing()) {
+ push(ImmGCPtr(val.toGCThing()));
+ } else {
+ push(Imm32(val.toNunboxPayload()));
+ }
+ push(Imm32(val.toNunboxTag()));
+ }
+ void pushValue(JSValueType type, Register reg) {
+ ma_push(reg);
+ push(ImmTag(JSVAL_TYPE_TO_TAG(type)));
+ }
+#endif
+ void pushValue(const Address& addr);
+
+ void storePayload(const Value& val, Address dest);
+ void storePayload(Register src, Address dest);
+ void storePayload(const Value& val, const BaseIndex& dest);
+ void storePayload(Register src, const BaseIndex& dest);
+ void storeTypeTag(ImmTag tag, Address dest);
+ void storeTypeTag(ImmTag tag, const BaseIndex& dest);
+
+ void handleFailureWithHandlerTail(Label* profilerExitTail);
+
+ template <typename T>
+ void wasmAtomicStore64(const wasm::MemoryAccessDesc& access, const T& mem,
+ Register temp, Register64 value);
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+ public:
+ // The following functions are exposed for use in platform-shared code.
+
+ inline void incrementInt32Value(const Address& addr);
+
+ void move32(Imm32 imm, Register dest);
+ void move32(Register src, Register dest);
+
+ void movePtr(Register src, Register dest);
+ void movePtr(ImmWord imm, Register dest);
+ void movePtr(ImmPtr imm, Register dest);
+ void movePtr(wasm::SymbolicAddress imm, Register dest);
+ void movePtr(ImmGCPtr imm, Register dest);
+
+ void load8SignExtend(const Address& address, Register dest);
+ void load8SignExtend(const BaseIndex& src, Register dest);
+
+ void load8ZeroExtend(const Address& address, Register dest);
+ void load8ZeroExtend(const BaseIndex& src, Register dest);
+
+ void load16SignExtend(const Address& address, Register dest);
+ void load16SignExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ void load16UnalignedSignExtend(const S& src, Register dest) {
+ ma_load_unaligned(dest, src, SizeHalfWord, SignExtend);
+ }
+
+ void load16ZeroExtend(const Address& address, Register dest);
+ void load16ZeroExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ void load16UnalignedZeroExtend(const S& src, Register dest) {
+ ma_load_unaligned(dest, src, SizeHalfWord, ZeroExtend);
+ }
+
+ void load32(const Address& address, Register dest);
+ void load32(const BaseIndex& address, Register dest);
+ void load32(AbsoluteAddress address, Register dest);
+ void load32(wasm::SymbolicAddress address, Register dest);
+
+ template <typename S>
+ void load32Unaligned(const S& src, Register dest) {
+ ma_load_unaligned(dest, src);
+ }
+
+ void load64(const Address& address, Register64 dest) {
+ load32(LowWord(address), dest.low);
+ load32(HighWord(address), dest.high);
+ }
+ void load64(const BaseIndex& address, Register64 dest) {
+ load32(LowWord(address), dest.low);
+ load32(HighWord(address), dest.high);
+ }
+
+ template <typename S>
+ void load64Unaligned(const S& src, Register64 dest) {
+ ma_load_unaligned(dest.low, LowWord(src));
+ ma_load_unaligned(dest.high, HighWord(src));
+ }
+
+ void loadPtr(const Address& address, Register dest);
+ void loadPtr(const BaseIndex& src, Register dest);
+ void loadPtr(AbsoluteAddress address, Register dest);
+ void loadPtr(wasm::SymbolicAddress address, Register dest);
+
+ void loadPrivate(const Address& address, Register dest);
+
+ void loadUnalignedDouble(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& src, Register temp,
+ FloatRegister dest);
+
+ void loadUnalignedFloat32(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& src, Register temp,
+ FloatRegister dest);
+
+ void store8(Register src, const Address& address);
+ void store8(Imm32 imm, const Address& address);
+ void store8(Register src, const BaseIndex& address);
+ void store8(Imm32 imm, const BaseIndex& address);
+
+ void store16(Register src, const Address& address);
+ void store16(Imm32 imm, const Address& address);
+ void store16(Register src, const BaseIndex& address);
+ void store16(Imm32 imm, const BaseIndex& address);
+
+ template <typename T>
+ void store16Unaligned(Register src, const T& dest) {
+ ma_store_unaligned(src, dest, SizeHalfWord);
+ }
+
+ void store32(Register src, AbsoluteAddress address);
+ void store32(Register src, const Address& address);
+ void store32(Register src, const BaseIndex& address);
+ void store32(Imm32 src, const Address& address);
+ void store32(Imm32 src, const BaseIndex& address);
+
+ // NOTE: This will use second scratch on MIPS. Only ARM needs the
+ // implementation without second scratch.
+ void store32_NoSecondScratch(Imm32 src, const Address& address) {
+ store32(src, address);
+ }
+
+ template <typename T>
+ void store32Unaligned(Register src, const T& dest) {
+ ma_store_unaligned(src, dest);
+ }
+
+ void store64(Register64 src, Address address) {
+ store32(src.low, Address(address.base, address.offset + LOW_32_OFFSET));
+ store32(src.high, Address(address.base, address.offset + HIGH_32_OFFSET));
+ }
+ void store64(Register64 src, const BaseIndex& address) {
+ store32(src.low, Address(address.base, address.offset + LOW_32_OFFSET));
+ store32(src.high, Address(address.base, address.offset + HIGH_32_OFFSET));
+ }
+
+ void store64(Imm64 imm, Address address) {
+ store32(imm.low(), Address(address.base, address.offset + LOW_32_OFFSET));
+ store32(imm.hi(), Address(address.base, address.offset + HIGH_32_OFFSET));
+ }
+ void store64(Imm64 imm, const BaseIndex& address) {
+ store32(imm.low(), Address(address.base, address.offset + LOW_32_OFFSET));
+ store32(imm.hi(), Address(address.base, address.offset + HIGH_32_OFFSET));
+ }
+
+ template <typename T>
+ void store64Unaligned(Register64 src, const T& dest) {
+ ma_store_unaligned(src.low, LowWord(dest));
+ ma_store_unaligned(src.high, HighWord(dest));
+ }
+
+ template <typename T>
+ void storePtr(ImmWord imm, T address);
+ template <typename T>
+ void storePtr(ImmPtr imm, T address);
+ template <typename T>
+ void storePtr(ImmGCPtr imm, T address);
+ void storePtr(Register src, const Address& address);
+ void storePtr(Register src, const BaseIndex& address);
+ void storePtr(Register src, AbsoluteAddress dest);
+
+ void storeUnalignedFloat32(const wasm::MemoryAccessDesc& access,
+ FloatRegister src, Register temp,
+ const BaseIndex& dest);
+ void storeUnalignedDouble(const wasm::MemoryAccessDesc& access,
+ FloatRegister src, Register temp,
+ const BaseIndex& dest);
+
+ void moveDouble(FloatRegister src, FloatRegister dest) { as_movd(dest, src); }
+
+ void zeroDouble(FloatRegister reg) {
+ moveToDoubleLo(zero, reg);
+ moveToDoubleHi(zero, reg);
+ }
+
+ void breakpoint();
+
+ void checkStackAlignment();
+
+ void alignStackPointer();
+ void restoreStackPointer();
+ static void calculateAlignedStackPointer(void** stackPointer);
+
+ // If source is a double, load it into dest. If source is int32,
+ // convert it to double. Else, branch to failure.
+ void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ void cmp64Set(Condition cond, Register64 lhs, Register64 rhs, Register dest);
+ void cmp64Set(Condition cond, Register64 lhs, Imm64 val, Register dest);
+
+ protected:
+ bool buildOOLFakeExitFrame(void* fakeReturnAddr);
+
+ void enterAtomic64Region(Register addr, Register spinlock, Register tmp);
+ void exitAtomic64Region(Register spinlock);
+ void wasmLoadI64Impl(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register64 output, Register tmp);
+ void wasmStoreI64Impl(const wasm::MemoryAccessDesc& access, Register64 value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp);
+ Condition ma_cmp64(Condition cond, Register64 lhs, Register64 rhs,
+ Register dest);
+ Condition ma_cmp64(Condition cond, Register64 lhs, Imm64 val, Register dest);
+
+ public:
+ void lea(Operand addr, Register dest) {
+ ma_addu(dest, addr.baseReg(), Imm32(addr.disp()));
+ }
+
+ void abiret() {
+ as_jr(ra);
+ as_nop();
+ }
+
+ void ma_storeImm(Imm32 imm, const Address& addr) { ma_sw(imm, addr); }
+
+ void moveFloat32(FloatRegister src, FloatRegister dest) {
+ as_movs(dest, src);
+ }
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+typedef MacroAssemblerMIPSCompat MacroAssemblerSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_MacroAssembler_mips32_h */
diff --git a/js/src/jit/mips32/MoveEmitter-mips32.cpp b/js/src/jit/mips32/MoveEmitter-mips32.cpp
new file mode 100644
index 0000000000..2f52c73899
--- /dev/null
+++ b/js/src/jit/mips32/MoveEmitter-mips32.cpp
@@ -0,0 +1,152 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips32/MoveEmitter-mips32.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void MoveEmitterMIPS::breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slotId) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (A -> B), which we reach first. We save B, then allow
+ // the original move to continue.
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchFloat32Reg;
+ masm.loadFloat32(getAdjustedAddress(to), temp);
+ // Since it is uncertain if the load will be aligned or not
+ // just fill both of them with the same value.
+ masm.storeFloat32(temp, cycleSlot(slotId, 0));
+ masm.storeFloat32(temp, cycleSlot(slotId, 4));
+ } else {
+ // Just always store the largest possible size.
+ masm.storeDouble(to.floatReg().doubleOverlay(), cycleSlot(slotId, 0));
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchDoubleReg;
+ masm.loadDouble(getAdjustedAddress(to), temp);
+ masm.storeDouble(temp, cycleSlot(slotId, 0));
+ } else {
+ masm.storeDouble(to.floatReg(), cycleSlot(slotId, 0));
+ }
+ break;
+ case MoveOp::INT32:
+ MOZ_ASSERT(sizeof(uintptr_t) == sizeof(int32_t));
+ [[fallthrough]];
+ case MoveOp::GENERAL:
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.loadPtr(getAdjustedAddress(to), temp);
+ masm.storePtr(temp, cycleSlot(0, 0));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.storePtr(to.reg(), cycleSlot(0, 0));
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterMIPS::completeCycle(const MoveOperand& from,
+ const MoveOperand& to, MoveOp::Type type,
+ uint32_t slotId) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (B -> A), which we reach last. We emit a move from the
+ // saved value of B, to A.
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchFloat32Reg;
+ masm.loadFloat32(cycleSlot(slotId, 0), temp);
+ masm.storeFloat32(temp, getAdjustedAddress(to));
+ } else {
+ uint32_t offset = 0;
+ if (from.floatReg().numAlignedAliased() == 1) {
+ offset = sizeof(float);
+ }
+ masm.loadFloat32(cycleSlot(slotId, offset), to.floatReg());
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchDoubleReg;
+ masm.loadDouble(cycleSlot(slotId, 0), temp);
+ masm.storeDouble(temp, getAdjustedAddress(to));
+ } else {
+ masm.loadDouble(cycleSlot(slotId, 0), to.floatReg());
+ }
+ break;
+ case MoveOp::INT32:
+ MOZ_ASSERT(sizeof(uintptr_t) == sizeof(int32_t));
+ [[fallthrough]];
+ case MoveOp::GENERAL:
+ MOZ_ASSERT(slotId == 0);
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.loadPtr(cycleSlot(0, 0), temp);
+ masm.storePtr(temp, getAdjustedAddress(to));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.loadPtr(cycleSlot(0, 0), to.reg());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterMIPS::emitDoubleMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveDouble(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralRegPair()) {
+ // Used for passing double parameter in a2,a3 register pair.
+ // Two moves are added for one double parameter by
+ // MacroAssembler::passABIArg
+ MOZ_ASSERT(to.evenReg() == a2 && to.oddReg() == a3,
+ "Invalid emitDoubleMove arguments.");
+ masm.moveFromDoubleLo(from.floatReg(), a2);
+ masm.moveFromDoubleHi(from.floatReg(), a3);
+ } else {
+ MOZ_ASSERT(to.isMemory());
+ masm.storeDouble(from.floatReg(), getAdjustedAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ MOZ_ASSERT(from.isMemory());
+ masm.loadDouble(getAdjustedAddress(from), to.floatReg());
+ } else if (to.isGeneralRegPair()) {
+ // Used for passing double parameter in a2,a3 register pair.
+ // Two moves are added for one double parameter by
+ // MacroAssembler::passABIArg
+ MOZ_ASSERT(from.isMemory());
+ MOZ_ASSERT(to.evenReg() == a2 && to.oddReg() == a3,
+ "Invalid emitDoubleMove arguments.");
+ masm.loadPtr(getAdjustedAddress(from), a2);
+ masm.loadPtr(
+ Address(from.base(), getAdjustedOffset(from) + sizeof(uint32_t)), a3);
+ } else {
+ MOZ_ASSERT(from.isMemory());
+ MOZ_ASSERT(to.isMemory());
+ masm.loadDouble(getAdjustedAddress(from), ScratchDoubleReg);
+ masm.storeDouble(ScratchDoubleReg, getAdjustedAddress(to));
+ }
+}
diff --git a/js/src/jit/mips32/MoveEmitter-mips32.h b/js/src/jit/mips32/MoveEmitter-mips32.h
new file mode 100644
index 0000000000..4a669e9991
--- /dev/null
+++ b/js/src/jit/mips32/MoveEmitter-mips32.h
@@ -0,0 +1,31 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_MoveEmitter_mips32_h
+#define jit_mips32_MoveEmitter_mips32_h
+
+#include "jit/mips-shared/MoveEmitter-mips-shared.h"
+
+namespace js {
+namespace jit {
+
+class MoveEmitterMIPS : public MoveEmitterMIPSShared {
+ void emitDoubleMove(const MoveOperand& from, const MoveOperand& to);
+ void breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+ void completeCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+
+ public:
+ MoveEmitterMIPS(MacroAssembler& masm) : MoveEmitterMIPSShared(masm) {}
+};
+
+typedef MoveEmitterMIPS MoveEmitter;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_MoveEmitter_mips32_h */
diff --git a/js/src/jit/mips32/SharedICRegisters-mips32.h b/js/src/jit/mips32/SharedICRegisters-mips32.h
new file mode 100644
index 0000000000..a1017c9399
--- /dev/null
+++ b/js/src/jit/mips32/SharedICRegisters-mips32.h
@@ -0,0 +1,42 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips32_SharedICRegisters_mips32_h
+#define jit_mips32_SharedICRegisters_mips32_h
+
+#include "jit/mips32/Assembler-mips32.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+static constexpr ValueOperand R0(a3, a2);
+static constexpr ValueOperand R1(s7, s6);
+static constexpr ValueOperand R2(t7, t6);
+
+// ICTailCallReg and ICStubReg
+// These use registers that are not preserved across calls.
+static constexpr Register ICTailCallReg = ra;
+static constexpr Register ICStubReg = t5;
+
+// Register used internally by MacroAssemblerMIPS.
+static constexpr Register BaselineSecondScratchReg = SecondScratchReg;
+
+// Note that ICTailCallReg is actually just the link register.
+// In MIPS code emission, we do not clobber ICTailCallReg since we keep
+// the return address for calls there.
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = f0;
+static constexpr FloatRegister FloatReg1 = f2;
+static constexpr FloatRegister FloatReg2 = f4;
+static constexpr FloatRegister FloatReg3 = f6;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips32_SharedICRegisters_mips32_h */
diff --git a/js/src/jit/mips32/Simulator-mips32.cpp b/js/src/jit/mips32/Simulator-mips32.cpp
new file mode 100644
index 0000000000..5efc0f20db
--- /dev/null
+++ b/js/src/jit/mips32/Simulator-mips32.cpp
@@ -0,0 +1,3631 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/mips32/Simulator-mips32.h"
+
+#include "mozilla/Casting.h"
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/Likely.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <float.h>
+
+#include "jit/AtomicOperations.h"
+#include "jit/mips32/Assembler-mips32.h"
+#include "js/UniquePtr.h"
+#include "js/Utility.h"
+#include "vm/Runtime.h"
+#include "wasm/WasmInstance.h"
+#include "wasm/WasmSignalHandlers.h"
+
+#define I8(v) static_cast<int8_t>(v)
+#define I16(v) static_cast<int16_t>(v)
+#define U16(v) static_cast<uint16_t>(v)
+#define I32(v) static_cast<int32_t>(v)
+#define U32(v) static_cast<uint32_t>(v)
+
+namespace js {
+namespace jit {
+
+static const Instr kCallRedirInstr =
+ op_special | MAX_BREAK_CODE << FunctionBits | ff_break;
+
+// Utils functions.
+static bool HaveSameSign(int32_t a, int32_t b) { return ((a ^ b) >= 0); }
+
+static uint32_t GetFCSRConditionBit(uint32_t cc) {
+ if (cc == 0) {
+ return 23;
+ } else {
+ return 24 + cc;
+ }
+}
+
+static const int32_t kRegisterskMaxValue = 0x7fffffff;
+static const int32_t kRegisterskMinValue = 0x80000000;
+
+// -----------------------------------------------------------------------------
+// MIPS assembly various constants.
+
+class SimInstruction {
+ public:
+ enum {
+ kInstrSize = 4,
+ // On MIPS PC cannot actually be directly accessed. We behave as if PC was
+ // always the value of the current instruction being executed.
+ kPCReadOffset = 0
+ };
+
+ // Get the raw instruction bits.
+ inline Instr instructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void setInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int bit(int nr) const { return (instructionBits() >> nr) & 1; }
+
+ // Read a bit field out of the instruction bits.
+ inline int bits(int hi, int lo) const {
+ return (instructionBits() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Instruction type.
+ enum Type { kRegisterType, kImmediateType, kJumpType, kUnsupported = -1 };
+
+ // Get the encoding type of the instruction.
+ Type instructionType() const;
+
+ // Accessors for the different named fields used in the MIPS encoding.
+ inline OpcodeField opcodeValue() const {
+ return static_cast<OpcodeField>(
+ bits(OpcodeShift + OpcodeBits - 1, OpcodeShift));
+ }
+
+ inline int rsValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return bits(RSShift + RSBits - 1, RSShift);
+ }
+
+ inline int rtValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return bits(RTShift + RTBits - 1, RTShift);
+ }
+
+ inline int rdValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType);
+ return bits(RDShift + RDBits - 1, RDShift);
+ }
+
+ inline int saValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType);
+ return bits(SAShift + SABits - 1, SAShift);
+ }
+
+ inline int functionValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return bits(FunctionShift + FunctionBits - 1, FunctionShift);
+ }
+
+ inline int fdValue() const { return bits(FDShift + FDBits - 1, FDShift); }
+
+ inline int fsValue() const { return bits(FSShift + FSBits - 1, FSShift); }
+
+ inline int ftValue() const { return bits(FTShift + FTBits - 1, FTShift); }
+
+ inline int frValue() const { return bits(FRShift + FRBits - 1, FRShift); }
+
+ // Float Compare condition code instruction bits.
+ inline int fcccValue() const {
+ return bits(FCccShift + FCccBits - 1, FCccShift);
+ }
+
+ // Float Branch condition code instruction bits.
+ inline int fbccValue() const {
+ return bits(FBccShift + FBccBits - 1, FBccShift);
+ }
+
+ // Float Branch true/false instruction bit.
+ inline int fbtrueValue() const {
+ return bits(FBtrueShift + FBtrueBits - 1, FBtrueShift);
+ }
+
+ // Return the fields at their original place in the instruction encoding.
+ inline OpcodeField opcodeFieldRaw() const {
+ return static_cast<OpcodeField>(instructionBits() & OpcodeMask);
+ }
+
+ inline int rsFieldRaw() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return instructionBits() & RSMask;
+ }
+
+ // Same as above function, but safe to call within instructionType().
+ inline int rsFieldRawNoAssert() const { return instructionBits() & RSMask; }
+
+ inline int rtFieldRaw() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return instructionBits() & RTMask;
+ }
+
+ inline int rdFieldRaw() const {
+ MOZ_ASSERT(instructionType() == kRegisterType);
+ return instructionBits() & RDMask;
+ }
+
+ inline int saFieldRaw() const {
+ MOZ_ASSERT(instructionType() == kRegisterType);
+ return instructionBits() & SAMask;
+ }
+
+ inline int functionFieldRaw() const {
+ return instructionBits() & FunctionMask;
+ }
+
+ // Get the secondary field according to the opcode.
+ inline int secondaryValue() const {
+ OpcodeField op = opcodeFieldRaw();
+ switch (op) {
+ case op_special:
+ case op_special2:
+ return functionValue();
+ case op_cop1:
+ return rsValue();
+ case op_regimm:
+ return rtValue();
+ default:
+ return ff_null;
+ }
+ }
+
+ inline int32_t imm16Value() const {
+ MOZ_ASSERT(instructionType() == kImmediateType);
+ return bits(Imm16Shift + Imm16Bits - 1, Imm16Shift);
+ }
+
+ inline int32_t imm26Value() const {
+ MOZ_ASSERT(instructionType() == kJumpType);
+ return bits(Imm26Shift + Imm26Bits - 1, Imm26Shift);
+ }
+
+ // Say if the instruction should not be used in a branch delay slot.
+ bool isForbiddenInBranchDelay() const;
+ // Say if the instruction 'links'. e.g. jal, bal.
+ bool isLinkingInstruction() const;
+ // Say if the instruction is a debugger break/trap.
+ bool isTrap() const;
+
+ private:
+ SimInstruction() = delete;
+ SimInstruction(const SimInstruction& other) = delete;
+ void operator=(const SimInstruction& other) = delete;
+};
+
+bool SimInstruction::isForbiddenInBranchDelay() const {
+ const int op = opcodeFieldRaw();
+ switch (op) {
+ case op_j:
+ case op_jal:
+ case op_beq:
+ case op_bne:
+ case op_blez:
+ case op_bgtz:
+ case op_beql:
+ case op_bnel:
+ case op_blezl:
+ case op_bgtzl:
+ return true;
+ case op_regimm:
+ switch (rtFieldRaw()) {
+ case rt_bltz:
+ case rt_bgez:
+ case rt_bltzal:
+ case rt_bgezal:
+ return true;
+ default:
+ return false;
+ };
+ break;
+ case op_special:
+ switch (functionFieldRaw()) {
+ case ff_jr:
+ case ff_jalr:
+ return true;
+ default:
+ return false;
+ };
+ break;
+ default:
+ return false;
+ }
+}
+
+bool SimInstruction::isLinkingInstruction() const {
+ const int op = opcodeFieldRaw();
+ switch (op) {
+ case op_jal:
+ return true;
+ case op_regimm:
+ switch (rtFieldRaw()) {
+ case rt_bgezal:
+ case rt_bltzal:
+ return true;
+ default:
+ return false;
+ };
+ case op_special:
+ switch (functionFieldRaw()) {
+ case ff_jalr:
+ return true;
+ default:
+ return false;
+ };
+ default:
+ return false;
+ };
+}
+
+bool SimInstruction::isTrap() const {
+ if (opcodeFieldRaw() != op_special) {
+ return false;
+ } else {
+ switch (functionFieldRaw()) {
+ case ff_break:
+ return instructionBits() != kCallRedirInstr;
+ case ff_tge:
+ case ff_tgeu:
+ case ff_tlt:
+ case ff_tltu:
+ case ff_teq:
+ case ff_tne:
+ return bits(15, 6) != kWasmTrapCode;
+ default:
+ return false;
+ };
+ }
+}
+
+SimInstruction::Type SimInstruction::instructionType() const {
+ switch (opcodeFieldRaw()) {
+ case op_special:
+ switch (functionFieldRaw()) {
+ case ff_jr:
+ case ff_jalr:
+ case ff_break:
+ case ff_sll:
+ case ff_srl:
+ case ff_sra:
+ case ff_sllv:
+ case ff_srlv:
+ case ff_srav:
+ case ff_mfhi:
+ case ff_mflo:
+ case ff_mult:
+ case ff_multu:
+ case ff_div:
+ case ff_divu:
+ case ff_add:
+ case ff_addu:
+ case ff_sub:
+ case ff_subu:
+ case ff_and:
+ case ff_or:
+ case ff_xor:
+ case ff_nor:
+ case ff_slt:
+ case ff_sltu:
+ case ff_tge:
+ case ff_tgeu:
+ case ff_tlt:
+ case ff_tltu:
+ case ff_teq:
+ case ff_tne:
+ case ff_movz:
+ case ff_movn:
+ case ff_movci:
+ case ff_sync:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ };
+ break;
+ case op_special2:
+ switch (functionFieldRaw()) {
+ case ff_mul:
+ case ff_madd:
+ case ff_maddu:
+ case ff_clz:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ };
+ break;
+ case op_special3:
+ switch (functionFieldRaw()) {
+ case ff_ins:
+ case ff_ext:
+ case ff_bshfl:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ };
+ break;
+ case op_cop1: // Coprocessor instructions.
+ switch (rsFieldRawNoAssert()) {
+ case rs_bc1: // Branch on coprocessor condition.
+ return kImmediateType;
+ default:
+ return kRegisterType;
+ };
+ break;
+ case op_cop1x:
+ return kRegisterType;
+ // 16 bits Immediate type instructions. e.g.: addi dest, src, imm16.
+ case op_regimm:
+ case op_beq:
+ case op_bne:
+ case op_blez:
+ case op_bgtz:
+ case op_addi:
+ case op_addiu:
+ case op_slti:
+ case op_sltiu:
+ case op_andi:
+ case op_ori:
+ case op_xori:
+ case op_lui:
+ case op_beql:
+ case op_bnel:
+ case op_blezl:
+ case op_bgtzl:
+ case op_lb:
+ case op_lh:
+ case op_lwl:
+ case op_lw:
+ case op_lbu:
+ case op_lhu:
+ case op_lwr:
+ case op_sb:
+ case op_sh:
+ case op_swl:
+ case op_sw:
+ case op_swr:
+ case op_lwc1:
+ case op_ldc1:
+ case op_swc1:
+ case op_sdc1:
+ case op_ll:
+ case op_sc:
+ return kImmediateType;
+ // 26 bits immediate type instructions. e.g.: j imm26.
+ case op_j:
+ case op_jal:
+ return kJumpType;
+ default:
+ return kUnsupported;
+ }
+ return kUnsupported;
+}
+
+// C/C++ argument slots size.
+const int kCArgSlotCount = 4;
+const int kCArgsSlotsSize = kCArgSlotCount * SimInstruction::kInstrSize;
+const int kBranchReturnOffset = 2 * SimInstruction::kInstrSize;
+
+class CachePage {
+ public:
+ static const int LINE_VALID = 0;
+ static const int LINE_INVALID = 1;
+
+ static const int kPageShift = 12;
+ static const int kPageSize = 1 << kPageShift;
+ static const int kPageMask = kPageSize - 1;
+ static const int kLineShift = 2; // The cache line is only 4 bytes right now.
+ static const int kLineLength = 1 << kLineShift;
+ static const int kLineMask = kLineLength - 1;
+
+ CachePage() { memset(&validity_map_, LINE_INVALID, sizeof(validity_map_)); }
+
+ char* validityByte(int offset) {
+ return &validity_map_[offset >> kLineShift];
+ }
+
+ char* cachedData(int offset) { return &data_[offset]; }
+
+ private:
+ char data_[kPageSize]; // The cached data.
+ static const int kValidityMapSize = kPageSize >> kLineShift;
+ char validity_map_[kValidityMapSize]; // One byte per line.
+};
+
+// Protects the icache() and redirection() properties of the
+// Simulator.
+class AutoLockSimulatorCache : public LockGuard<Mutex> {
+ using Base = LockGuard<Mutex>;
+
+ public:
+ AutoLockSimulatorCache() : Base(SimulatorProcess::singleton_->cacheLock_) {}
+};
+
+mozilla::Atomic<size_t, mozilla::ReleaseAcquire>
+ SimulatorProcess::ICacheCheckingDisableCount(
+ 1); // Checking is disabled by default.
+SimulatorProcess* SimulatorProcess::singleton_ = nullptr;
+
+int Simulator::StopSimAt = -1;
+
+Simulator* Simulator::Create() {
+ auto sim = MakeUnique<Simulator>();
+ if (!sim) {
+ return nullptr;
+ }
+
+ if (!sim->init()) {
+ return nullptr;
+ }
+
+ char* stopAtStr = getenv("MIPS_SIM_STOP_AT");
+ int64_t stopAt;
+ if (stopAtStr && sscanf(stopAtStr, "%lld", &stopAt) == 1) {
+ fprintf(stderr, "\nStopping simulation at icount %lld\n", stopAt);
+ Simulator::StopSimAt = stopAt;
+ }
+
+ return sim.release();
+}
+
+void Simulator::Destroy(Simulator* sim) { js_delete(sim); }
+
+// The MipsDebugger class is used by the simulator while debugging simulated
+// code.
+class MipsDebugger {
+ public:
+ explicit MipsDebugger(Simulator* sim) : sim_(sim) {}
+
+ void stop(SimInstruction* instr);
+ void debug();
+ // Print all registers with a nice formatting.
+ void printAllRegs();
+ void printAllRegsIncludingFPU();
+
+ private:
+ // We set the breakpoint code to 0xfffff to easily recognize it.
+ static const Instr kBreakpointInstr = op_special | ff_break | 0xfffff << 6;
+ static const Instr kNopInstr = op_special | ff_sll;
+
+ Simulator* sim_;
+
+ int32_t getRegisterValue(int regnum);
+ int32_t getFPURegisterValueInt(int regnum);
+ int64_t getFPURegisterValueLong(int regnum);
+ float getFPURegisterValueFloat(int regnum);
+ double getFPURegisterValueDouble(int regnum);
+ bool getValue(const char* desc, int32_t* value);
+
+ // Set or delete a breakpoint. Returns true if successful.
+ bool setBreakpoint(SimInstruction* breakpc);
+ bool deleteBreakpoint(SimInstruction* breakpc);
+
+ // Undo and redo all breakpoints. This is needed to bracket disassembly and
+ // execution to skip past breakpoints when run from the debugger.
+ void undoBreakpoints();
+ void redoBreakpoints();
+};
+
+static void UNSUPPORTED() {
+ printf("Unsupported instruction.\n");
+ MOZ_CRASH();
+}
+
+void MipsDebugger::stop(SimInstruction* instr) {
+ // Get the stop code.
+ uint32_t code = instr->bits(25, 6);
+ // Retrieve the encoded address, which comes just after this stop.
+ char* msg =
+ *reinterpret_cast<char**>(sim_->get_pc() + SimInstruction::kInstrSize);
+ // Update this stop description.
+ if (!sim_->watchedStops_[code].desc_) {
+ sim_->watchedStops_[code].desc_ = msg;
+ }
+ // Print the stop message and code if it is not the default code.
+ if (code != kMaxStopCode) {
+ printf("Simulator hit stop %u: %s\n", code, msg);
+ } else {
+ printf("Simulator hit %s\n", msg);
+ }
+ sim_->set_pc(sim_->get_pc() + 2 * SimInstruction::kInstrSize);
+ debug();
+}
+
+int32_t MipsDebugger::getRegisterValue(int regnum) {
+ if (regnum == kPCRegister) {
+ return sim_->get_pc();
+ }
+ return sim_->getRegister(regnum);
+}
+
+int32_t MipsDebugger::getFPURegisterValueInt(int regnum) {
+ return sim_->getFpuRegister(regnum);
+}
+
+int64_t MipsDebugger::getFPURegisterValueLong(int regnum) {
+ return sim_->getFpuRegisterLong(regnum);
+}
+
+float MipsDebugger::getFPURegisterValueFloat(int regnum) {
+ return sim_->getFpuRegisterFloat(regnum);
+}
+
+double MipsDebugger::getFPURegisterValueDouble(int regnum) {
+ return sim_->getFpuRegisterDouble(regnum);
+}
+
+bool MipsDebugger::getValue(const char* desc, int32_t* value) {
+ Register reg = Register::FromName(desc);
+ if (reg != InvalidReg) {
+ *value = getRegisterValue(reg.code());
+ return true;
+ }
+
+ if (strncmp(desc, "0x", 2) == 0) {
+ return sscanf(desc, "%x", reinterpret_cast<uint32_t*>(value)) == 1;
+ }
+ return sscanf(desc, "%i", value) == 1;
+}
+
+bool MipsDebugger::setBreakpoint(SimInstruction* breakpc) {
+ // Check if a breakpoint can be set. If not return without any side-effects.
+ if (sim_->break_pc_ != nullptr) {
+ return false;
+ }
+
+ // Set the breakpoint.
+ sim_->break_pc_ = breakpc;
+ sim_->break_instr_ = breakpc->instructionBits();
+ // Not setting the breakpoint instruction in the code itself. It will be set
+ // when the debugger shell continues.
+ return true;
+}
+
+bool MipsDebugger::deleteBreakpoint(SimInstruction* breakpc) {
+ if (sim_->break_pc_ != nullptr) {
+ sim_->break_pc_->setInstructionBits(sim_->break_instr_);
+ }
+
+ sim_->break_pc_ = nullptr;
+ sim_->break_instr_ = 0;
+ return true;
+}
+
+void MipsDebugger::undoBreakpoints() {
+ if (sim_->break_pc_) {
+ sim_->break_pc_->setInstructionBits(sim_->break_instr_);
+ }
+}
+
+void MipsDebugger::redoBreakpoints() {
+ if (sim_->break_pc_) {
+ sim_->break_pc_->setInstructionBits(kBreakpointInstr);
+ }
+}
+
+void MipsDebugger::printAllRegs() {
+ int32_t value;
+ for (uint32_t i = 0; i < Registers::Total; i++) {
+ value = getRegisterValue(i);
+ printf("%3s: 0x%08x %10d ", Registers::GetName(i), value, value);
+
+ if (i % 2) {
+ printf("\n");
+ }
+ }
+ printf("\n");
+
+ value = getRegisterValue(Simulator::LO);
+ printf(" LO: 0x%08x %10d ", value, value);
+ value = getRegisterValue(Simulator::HI);
+ printf(" HI: 0x%08x %10d\n", value, value);
+ value = getRegisterValue(Simulator::pc);
+ printf(" pc: 0x%08x\n", value);
+}
+
+void MipsDebugger::printAllRegsIncludingFPU() {
+ printAllRegs();
+
+ printf("\n\n");
+ // f0, f1, f2, ... f31.
+ for (uint32_t i = 0; i < FloatRegisters::RegisterIdLimit; i++) {
+ if (i & 0x1) {
+ printf("%3s: 0x%08x\tflt: %-8.4g\n", FloatRegisters::GetName(i),
+ getFPURegisterValueInt(i), getFPURegisterValueFloat(i));
+ } else {
+ printf("%3s: 0x%08x\tflt: %-8.4g\tdbl: %-16.4g\n",
+ FloatRegisters::GetName(i), getFPURegisterValueInt(i),
+ getFPURegisterValueFloat(i), getFPURegisterValueDouble(i));
+ }
+ }
+}
+
+static char* ReadLine(const char* prompt) {
+ UniqueChars result;
+ char lineBuf[256];
+ int offset = 0;
+ bool keepGoing = true;
+ fprintf(stdout, "%s", prompt);
+ fflush(stdout);
+ while (keepGoing) {
+ if (fgets(lineBuf, sizeof(lineBuf), stdin) == nullptr) {
+ // fgets got an error. Just give up.
+ return nullptr;
+ }
+ int len = strlen(lineBuf);
+ if (len > 0 && lineBuf[len - 1] == '\n') {
+ // Since we read a new line we are done reading the line. This
+ // will exit the loop after copying this buffer into the result.
+ keepGoing = false;
+ }
+ if (!result) {
+ // Allocate the initial result and make room for the terminating '\0'
+ result.reset(js_pod_malloc<char>(len + 1));
+ if (!result) {
+ return nullptr;
+ }
+ } else {
+ // Allocate a new result with enough room for the new addition.
+ int new_len = offset + len + 1;
+ char* new_result = js_pod_malloc<char>(new_len);
+ if (!new_result) {
+ return nullptr;
+ }
+ // Copy the existing input into the new array and set the new
+ // array as the result.
+ memcpy(new_result, result.get(), offset * sizeof(char));
+ result.reset(new_result);
+ }
+ // Copy the newly read line into the result.
+ memcpy(result.get() + offset, lineBuf, len * sizeof(char));
+ offset += len;
+ }
+
+ MOZ_ASSERT(result);
+ result[offset] = '\0';
+ return result.release();
+}
+
+static void DisassembleInstruction(uint32_t pc) {
+ uint8_t* bytes = reinterpret_cast<uint8_t*>(pc);
+ char hexbytes[256];
+ sprintf(hexbytes, "0x%x 0x%x 0x%x 0x%x", bytes[0], bytes[1], bytes[2],
+ bytes[3]);
+ char llvmcmd[1024];
+ sprintf(llvmcmd,
+ "bash -c \"echo -n '%p'; echo '%s' | "
+ "llvm-mc -disassemble -arch=mipsel -mcpu=mips32r2 | "
+ "grep -v pure_instructions | grep -v .text\"",
+ static_cast<void*>(bytes), hexbytes);
+ if (system(llvmcmd)) {
+ printf("Cannot disassemble instruction.\n");
+ }
+}
+
+void MipsDebugger::debug() {
+ intptr_t lastPC = -1;
+ bool done = false;
+
+#define COMMAND_SIZE 63
+#define ARG_SIZE 255
+
+#define STR(a) #a
+#define XSTR(a) STR(a)
+
+ char cmd[COMMAND_SIZE + 1];
+ char arg1[ARG_SIZE + 1];
+ char arg2[ARG_SIZE + 1];
+ char* argv[3] = {cmd, arg1, arg2};
+
+ // Make sure to have a proper terminating character if reaching the limit.
+ cmd[COMMAND_SIZE] = 0;
+ arg1[ARG_SIZE] = 0;
+ arg2[ARG_SIZE] = 0;
+
+ // Undo all set breakpoints while running in the debugger shell. This will
+ // make them invisible to all commands.
+ undoBreakpoints();
+
+ while (!done && (sim_->get_pc() != Simulator::end_sim_pc)) {
+ if (lastPC != sim_->get_pc()) {
+ DisassembleInstruction(sim_->get_pc());
+ lastPC = sim_->get_pc();
+ }
+ char* line = ReadLine("sim> ");
+ if (line == nullptr) {
+ break;
+ } else {
+ char* last_input = sim_->lastDebuggerInput();
+ if (strcmp(line, "\n") == 0 && last_input != nullptr) {
+ line = last_input;
+ } else {
+ // Ownership is transferred to sim_;
+ sim_->setLastDebuggerInput(line);
+ }
+ // Use sscanf to parse the individual parts of the command line. At the
+ // moment no command expects more than two parameters.
+ int argc = sscanf(line,
+ "%" XSTR(COMMAND_SIZE) "s "
+ "%" XSTR(ARG_SIZE) "s "
+ "%" XSTR(ARG_SIZE) "s",
+ cmd, arg1, arg2);
+ if ((strcmp(cmd, "si") == 0) || (strcmp(cmd, "stepi") == 0)) {
+ SimInstruction* instr =
+ reinterpret_cast<SimInstruction*>(sim_->get_pc());
+ if (!instr->isTrap()) {
+ sim_->instructionDecode(
+ reinterpret_cast<SimInstruction*>(sim_->get_pc()));
+ } else {
+ // Allow si to jump over generated breakpoints.
+ printf("/!\\ Jumping over generated breakpoint.\n");
+ sim_->set_pc(sim_->get_pc() + SimInstruction::kInstrSize);
+ }
+ } else if ((strcmp(cmd, "c") == 0) || (strcmp(cmd, "cont") == 0)) {
+ // Execute the one instruction we broke at with breakpoints disabled.
+ sim_->instructionDecode(
+ reinterpret_cast<SimInstruction*>(sim_->get_pc()));
+ // Leave the debugger shell.
+ done = true;
+ } else if ((strcmp(cmd, "p") == 0) || (strcmp(cmd, "print") == 0)) {
+ if (argc == 2) {
+ int32_t value;
+ if (strcmp(arg1, "all") == 0) {
+ printAllRegs();
+ } else if (strcmp(arg1, "allf") == 0) {
+ printAllRegsIncludingFPU();
+ } else {
+ Register reg = Register::FromName(arg1);
+ FloatRegisters::Code fCode = FloatRegister::FromName(arg1);
+ if (reg != InvalidReg) {
+ value = getRegisterValue(reg.code());
+ printf("%s: 0x%08x %d \n", arg1, value, value);
+ } else if (fCode != FloatRegisters::Invalid) {
+ if (fCode & 0x1) {
+ printf("%3s: 0x%08x\tflt: %-8.4g\n",
+ FloatRegisters::GetName(fCode),
+ getFPURegisterValueInt(fCode),
+ getFPURegisterValueFloat(fCode));
+ } else {
+ printf("%3s: 0x%08x\tflt: %-8.4g\tdbl: %-16.4g\n",
+ FloatRegisters::GetName(fCode),
+ getFPURegisterValueInt(fCode),
+ getFPURegisterValueFloat(fCode),
+ getFPURegisterValueDouble(fCode));
+ }
+ } else {
+ printf("%s unrecognized\n", arg1);
+ }
+ }
+ } else {
+ printf("print <register> or print <fpu register> single\n");
+ }
+ } else if (strcmp(cmd, "stack") == 0 || strcmp(cmd, "mem") == 0) {
+ int32_t* cur = nullptr;
+ int32_t* end = nullptr;
+ int next_arg = 1;
+
+ if (strcmp(cmd, "stack") == 0) {
+ cur = reinterpret_cast<int32_t*>(sim_->getRegister(Simulator::sp));
+ } else { // Command "mem".
+ int32_t value;
+ if (!getValue(arg1, &value)) {
+ printf("%s unrecognized\n", arg1);
+ continue;
+ }
+ cur = reinterpret_cast<int32_t*>(value);
+ next_arg++;
+ }
+
+ int32_t words;
+ if (argc == next_arg) {
+ words = 10;
+ } else {
+ if (!getValue(argv[next_arg], &words)) {
+ words = 10;
+ }
+ }
+ end = cur + words;
+
+ while (cur < end) {
+ printf(" %p: 0x%08x %10d", cur, *cur, *cur);
+ printf("\n");
+ cur++;
+ }
+
+ } else if ((strcmp(cmd, "disasm") == 0) || (strcmp(cmd, "dpc") == 0) ||
+ (strcmp(cmd, "di") == 0)) {
+ uint8_t* cur = nullptr;
+ uint8_t* end = nullptr;
+
+ if (argc == 1) {
+ cur = reinterpret_cast<uint8_t*>(sim_->get_pc());
+ end = cur + (10 * SimInstruction::kInstrSize);
+ } else if (argc == 2) {
+ Register reg = Register::FromName(arg1);
+ if (reg != InvalidReg || strncmp(arg1, "0x", 2) == 0) {
+ // The argument is an address or a register name.
+ int32_t value;
+ if (getValue(arg1, &value)) {
+ cur = reinterpret_cast<uint8_t*>(value);
+ // Disassemble 10 instructions at <arg1>.
+ end = cur + (10 * SimInstruction::kInstrSize);
+ }
+ } else {
+ // The argument is the number of instructions.
+ int32_t value;
+ if (getValue(arg1, &value)) {
+ cur = reinterpret_cast<uint8_t*>(sim_->get_pc());
+ // Disassemble <arg1> instructions.
+ end = cur + (value * SimInstruction::kInstrSize);
+ }
+ }
+ } else {
+ int32_t value1;
+ int32_t value2;
+ if (getValue(arg1, &value1) && getValue(arg2, &value2)) {
+ cur = reinterpret_cast<uint8_t*>(value1);
+ end = cur + (value2 * SimInstruction::kInstrSize);
+ }
+ }
+
+ while (cur < end) {
+ DisassembleInstruction(uint32_t(cur));
+ cur += SimInstruction::kInstrSize;
+ }
+ } else if (strcmp(cmd, "gdb") == 0) {
+ printf("relinquishing control to gdb\n");
+ asm("int $3");
+ printf("regaining control from gdb\n");
+ } else if (strcmp(cmd, "break") == 0) {
+ if (argc == 2) {
+ int32_t value;
+ if (getValue(arg1, &value)) {
+ if (!setBreakpoint(reinterpret_cast<SimInstruction*>(value))) {
+ printf("setting breakpoint failed\n");
+ }
+ } else {
+ printf("%s unrecognized\n", arg1);
+ }
+ } else {
+ printf("break <address>\n");
+ }
+ } else if (strcmp(cmd, "del") == 0) {
+ if (!deleteBreakpoint(nullptr)) {
+ printf("deleting breakpoint failed\n");
+ }
+ } else if (strcmp(cmd, "flags") == 0) {
+ printf("No flags on MIPS !\n");
+ } else if (strcmp(cmd, "stop") == 0) {
+ int32_t value;
+ intptr_t stop_pc = sim_->get_pc() - 2 * SimInstruction::kInstrSize;
+ SimInstruction* stop_instr = reinterpret_cast<SimInstruction*>(stop_pc);
+ SimInstruction* msg_address = reinterpret_cast<SimInstruction*>(
+ stop_pc + SimInstruction::kInstrSize);
+ if ((argc == 2) && (strcmp(arg1, "unstop") == 0)) {
+ // Remove the current stop.
+ if (sim_->isStopInstruction(stop_instr)) {
+ stop_instr->setInstructionBits(kNopInstr);
+ msg_address->setInstructionBits(kNopInstr);
+ } else {
+ printf("Not at debugger stop.\n");
+ }
+ } else if (argc == 3) {
+ // Print information about all/the specified breakpoint(s).
+ if (strcmp(arg1, "info") == 0) {
+ if (strcmp(arg2, "all") == 0) {
+ printf("Stop information:\n");
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->printStopInfo(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->printStopInfo(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ } else if (strcmp(arg1, "enable") == 0) {
+ // Enable all/the specified breakpoint(s).
+ if (strcmp(arg2, "all") == 0) {
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->enableStop(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->enableStop(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ } else if (strcmp(arg1, "disable") == 0) {
+ // Disable all/the specified breakpoint(s).
+ if (strcmp(arg2, "all") == 0) {
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->disableStop(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->disableStop(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ }
+ } else {
+ printf("Wrong usage. Use help command for more information.\n");
+ }
+ } else if ((strcmp(cmd, "h") == 0) || (strcmp(cmd, "help") == 0)) {
+ printf("cont\n");
+ printf(" continue execution (alias 'c')\n");
+ printf("stepi\n");
+ printf(" step one instruction (alias 'si')\n");
+ printf("print <register>\n");
+ printf(" print register content (alias 'p')\n");
+ printf(" use register name 'all' to print all registers\n");
+ printf("printobject <register>\n");
+ printf(" print an object from a register (alias 'po')\n");
+ printf("stack [<words>]\n");
+ printf(" dump stack content, default dump 10 words)\n");
+ printf("mem <address> [<words>]\n");
+ printf(" dump memory content, default dump 10 words)\n");
+ printf("flags\n");
+ printf(" print flags\n");
+ printf("disasm [<instructions>]\n");
+ printf("disasm [<address/register>]\n");
+ printf("disasm [[<address/register>] <instructions>]\n");
+ printf(" disassemble code, default is 10 instructions\n");
+ printf(" from pc (alias 'di')\n");
+ printf("gdb\n");
+ printf(" enter gdb\n");
+ printf("break <address>\n");
+ printf(" set a break point on the address\n");
+ printf("del\n");
+ printf(" delete the breakpoint\n");
+ printf("stop feature:\n");
+ printf(" Description:\n");
+ printf(" Stops are debug instructions inserted by\n");
+ printf(" the Assembler::stop() function.\n");
+ printf(" When hitting a stop, the Simulator will\n");
+ printf(" stop and and give control to the Debugger.\n");
+ printf(" All stop codes are watched:\n");
+ printf(" - They can be enabled / disabled: the Simulator\n");
+ printf(" will / won't stop when hitting them.\n");
+ printf(" - The Simulator keeps track of how many times they \n");
+ printf(" are met. (See the info command.) Going over a\n");
+ printf(" disabled stop still increases its counter. \n");
+ printf(" Commands:\n");
+ printf(" stop info all/<code> : print infos about number <code>\n");
+ printf(" or all stop(s).\n");
+ printf(" stop enable/disable all/<code> : enables / disables\n");
+ printf(" all or number <code> stop(s)\n");
+ printf(" stop unstop\n");
+ printf(" ignore the stop instruction at the current location\n");
+ printf(" from now on\n");
+ } else {
+ printf("Unknown command: %s\n", cmd);
+ }
+ }
+ }
+
+ // Add all the breakpoints back to stop execution and enter the debugger
+ // shell when hit.
+ redoBreakpoints();
+
+#undef COMMAND_SIZE
+#undef ARG_SIZE
+
+#undef STR
+#undef XSTR
+}
+
+static bool AllOnOnePage(uintptr_t start, int size) {
+ intptr_t start_page = (start & ~CachePage::kPageMask);
+ intptr_t end_page = ((start + size) & ~CachePage::kPageMask);
+ return start_page == end_page;
+}
+
+void Simulator::setLastDebuggerInput(char* input) {
+ js_free(lastDebuggerInput_);
+ lastDebuggerInput_ = input;
+}
+
+static CachePage* GetCachePageLocked(SimulatorProcess::ICacheMap& i_cache,
+ void* page) {
+ SimulatorProcess::ICacheMap::AddPtr p = i_cache.lookupForAdd(page);
+ if (p) {
+ return p->value();
+ }
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ CachePage* new_page = js_new<CachePage>();
+ if (!new_page || !i_cache.add(p, page, new_page)) {
+ oomUnsafe.crash("Simulator CachePage");
+ }
+ return new_page;
+}
+
+// Flush from start up to and not including start + size.
+static void FlushOnePageLocked(SimulatorProcess::ICacheMap& i_cache,
+ intptr_t start, int size) {
+ MOZ_ASSERT(size <= CachePage::kPageSize);
+ MOZ_ASSERT(AllOnOnePage(start, size - 1));
+ MOZ_ASSERT((start & CachePage::kLineMask) == 0);
+ MOZ_ASSERT((size & CachePage::kLineMask) == 0);
+ void* page = reinterpret_cast<void*>(start & (~CachePage::kPageMask));
+ int offset = (start & CachePage::kPageMask);
+ CachePage* cache_page = GetCachePageLocked(i_cache, page);
+ char* valid_bytemap = cache_page->validityByte(offset);
+ memset(valid_bytemap, CachePage::LINE_INVALID, size >> CachePage::kLineShift);
+}
+
+static void FlushICacheLocked(SimulatorProcess::ICacheMap& i_cache,
+ void* start_addr, size_t size) {
+ intptr_t start = reinterpret_cast<intptr_t>(start_addr);
+ int intra_line = (start & CachePage::kLineMask);
+ start -= intra_line;
+ size += intra_line;
+ size = ((size - 1) | CachePage::kLineMask) + 1;
+ int offset = (start & CachePage::kPageMask);
+ while (!AllOnOnePage(start, size - 1)) {
+ int bytes_to_flush = CachePage::kPageSize - offset;
+ FlushOnePageLocked(i_cache, start, bytes_to_flush);
+ start += bytes_to_flush;
+ size -= bytes_to_flush;
+ MOZ_ASSERT((start & CachePage::kPageMask) == 0);
+ offset = 0;
+ }
+ if (size != 0) {
+ FlushOnePageLocked(i_cache, start, size);
+ }
+}
+
+/* static */
+void SimulatorProcess::checkICacheLocked(SimInstruction* instr) {
+ intptr_t address = reinterpret_cast<intptr_t>(instr);
+ void* page = reinterpret_cast<void*>(address & (~CachePage::kPageMask));
+ void* line = reinterpret_cast<void*>(address & (~CachePage::kLineMask));
+ int offset = (address & CachePage::kPageMask);
+ CachePage* cache_page = GetCachePageLocked(icache(), page);
+ char* cache_valid_byte = cache_page->validityByte(offset);
+ bool cache_hit = (*cache_valid_byte == CachePage::LINE_VALID);
+ char* cached_line = cache_page->cachedData(offset & ~CachePage::kLineMask);
+
+ if (cache_hit) {
+ // Check that the data in memory matches the contents of the I-cache.
+ int cmpret =
+ memcmp(reinterpret_cast<void*>(instr), cache_page->cachedData(offset),
+ SimInstruction::kInstrSize);
+ MOZ_ASSERT(cmpret == 0);
+ } else {
+ // Cache miss. Load memory into the cache.
+ memcpy(cached_line, line, CachePage::kLineLength);
+ *cache_valid_byte = CachePage::LINE_VALID;
+ }
+}
+
+HashNumber SimulatorProcess::ICacheHasher::hash(const Lookup& l) {
+ return static_cast<uint32_t>(reinterpret_cast<uintptr_t>(l)) >> 2;
+}
+
+bool SimulatorProcess::ICacheHasher::match(const Key& k, const Lookup& l) {
+ MOZ_ASSERT((reinterpret_cast<intptr_t>(k) & CachePage::kPageMask) == 0);
+ MOZ_ASSERT((reinterpret_cast<intptr_t>(l) & CachePage::kPageMask) == 0);
+ return k == l;
+}
+
+/* static */
+void SimulatorProcess::FlushICache(void* start_addr, size_t size) {
+ if (!ICacheCheckingDisableCount) {
+ AutoLockSimulatorCache als;
+ js::jit::FlushICacheLocked(icache(), start_addr, size);
+ }
+}
+
+Simulator::Simulator() {
+ // Set up simulator support first. Some of this information is needed to
+ // setup the architecture state.
+
+ // Note, allocation and anything that depends on allocated memory is
+ // deferred until init(), in order to handle OOM properly.
+
+ stack_ = nullptr;
+ stackLimit_ = 0;
+ pc_modified_ = false;
+ icount_ = 0;
+ break_count_ = 0;
+ break_pc_ = nullptr;
+ break_instr_ = 0;
+ single_stepping_ = false;
+ single_step_callback_ = nullptr;
+ single_step_callback_arg_ = nullptr;
+
+ // Set up architecture state.
+ // All registers are initialized to zero to start with.
+ for (int i = 0; i < Register::kNumSimuRegisters; i++) {
+ registers_[i] = 0;
+ }
+ for (int i = 0; i < Simulator::FPURegister::kNumFPURegisters; i++) {
+ FPUregisters_[i] = 0;
+ }
+ FCSR_ = 0;
+ LLBit_ = false;
+ LLAddr_ = 0;
+ lastLLValue_ = 0;
+
+ // The ra and pc are initialized to a known bad value that will cause an
+ // access violation if the simulator ever tries to execute it.
+ registers_[pc] = bad_ra;
+ registers_[ra] = bad_ra;
+
+ for (int i = 0; i < kNumExceptions; i++) {
+ exceptions[i] = 0;
+ }
+
+ lastDebuggerInput_ = nullptr;
+}
+
+bool Simulator::init() {
+ // Allocate 2MB for the stack. Note that we will only use 1MB, see below.
+ static const size_t stackSize = 2 * 1024 * 1024;
+ stack_ = js_pod_malloc<char>(stackSize);
+ if (!stack_) {
+ return false;
+ }
+
+ // Leave a safety margin of 1MB to prevent overrunning the stack when
+ // pushing values (total stack size is 2MB).
+ stackLimit_ = reinterpret_cast<uintptr_t>(stack_) + 1024 * 1024;
+
+ // The sp is initialized to point to the bottom (high address) of the
+ // allocated stack area. To be safe in potential stack underflows we leave
+ // some buffer below.
+ registers_[sp] = reinterpret_cast<int32_t>(stack_) + stackSize - 64;
+
+ return true;
+}
+
+// When the generated code calls an external reference we need to catch that in
+// the simulator. The external reference will be a function compiled for the
+// host architecture. We need to call that function instead of trying to
+// execute it with the simulator. We do that by redirecting the external
+// reference to a swi (software-interrupt) instruction that is handled by
+// the simulator. We write the original destination of the jump just at a known
+// offset from the swi instruction so the simulator knows what to call.
+class Redirection {
+ friend class SimulatorProcess;
+
+ // sim's lock must already be held.
+ Redirection(void* nativeFunction, ABIFunctionType type)
+ : nativeFunction_(nativeFunction),
+ swiInstruction_(kCallRedirInstr),
+ type_(type),
+ next_(nullptr) {
+ next_ = SimulatorProcess::redirection();
+ if (!SimulatorProcess::ICacheCheckingDisableCount) {
+ FlushICacheLocked(SimulatorProcess::icache(), addressOfSwiInstruction(),
+ SimInstruction::kInstrSize);
+ }
+ SimulatorProcess::setRedirection(this);
+ }
+
+ public:
+ void* addressOfSwiInstruction() { return &swiInstruction_; }
+ void* nativeFunction() const { return nativeFunction_; }
+ ABIFunctionType type() const { return type_; }
+
+ static Redirection* Get(void* nativeFunction, ABIFunctionType type) {
+ AutoLockSimulatorCache als;
+
+ Redirection* current = SimulatorProcess::redirection();
+ for (; current != nullptr; current = current->next_) {
+ if (current->nativeFunction_ == nativeFunction) {
+ MOZ_ASSERT(current->type() == type);
+ return current;
+ }
+ }
+
+ // Note: we can't use js_new here because the constructor is private.
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ Redirection* redir = js_pod_malloc<Redirection>(1);
+ if (!redir) {
+ oomUnsafe.crash("Simulator redirection");
+ }
+ new (redir) Redirection(nativeFunction, type);
+ return redir;
+ }
+
+ static Redirection* FromSwiInstruction(SimInstruction* swiInstruction) {
+ uint8_t* addrOfSwi = reinterpret_cast<uint8_t*>(swiInstruction);
+ uint8_t* addrOfRedirection =
+ addrOfSwi - offsetof(Redirection, swiInstruction_);
+ return reinterpret_cast<Redirection*>(addrOfRedirection);
+ }
+
+ private:
+ void* nativeFunction_;
+ uint32_t swiInstruction_;
+ ABIFunctionType type_;
+ Redirection* next_;
+};
+
+Simulator::~Simulator() { js_free(stack_); }
+
+SimulatorProcess::SimulatorProcess()
+ : cacheLock_(mutexid::SimulatorCacheLock), redirection_(nullptr) {
+ if (getenv("MIPS_SIM_ICACHE_CHECKS")) {
+ ICacheCheckingDisableCount = 0;
+ }
+}
+
+SimulatorProcess::~SimulatorProcess() {
+ Redirection* r = redirection_;
+ while (r) {
+ Redirection* next = r->next_;
+ js_delete(r);
+ r = next;
+ }
+}
+
+/* static */
+void* Simulator::RedirectNativeFunction(void* nativeFunction,
+ ABIFunctionType type) {
+ Redirection* redirection = Redirection::Get(nativeFunction, type);
+ return redirection->addressOfSwiInstruction();
+}
+
+// Get the active Simulator for the current thread.
+Simulator* Simulator::Current() {
+ JSContext* cx = TlsContext.get();
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cx->runtime()));
+ return cx->simulator();
+}
+
+// Sets the register in the architecture state. It will also deal with updating
+// Simulator internal state for special registers such as PC.
+void Simulator::setRegister(int reg, int32_t value) {
+ MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters));
+ if (reg == pc) {
+ pc_modified_ = true;
+ }
+
+ // Zero register always holds 0.
+ registers_[reg] = (reg == 0) ? 0 : value;
+}
+
+void Simulator::setFpuRegister(int fpureg, int32_t value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ FPUregisters_[fpureg] = value;
+}
+
+void Simulator::setFpuRegisterFloat(int fpureg, float value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ *mozilla::BitwiseCast<float*>(&FPUregisters_[fpureg]) = value;
+}
+
+void Simulator::setFpuRegisterDouble(int fpureg, double value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters) &&
+ ((fpureg % 2) == 0));
+ *mozilla::BitwiseCast<double*>(&FPUregisters_[fpureg]) = value;
+}
+
+// Get the register from the architecture state. This function does handle
+// the special case of accessing the PC register.
+int32_t Simulator::getRegister(int reg) const {
+ MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters));
+ if (reg == 0) {
+ return 0;
+ }
+ return registers_[reg] + ((reg == pc) ? SimInstruction::kPCReadOffset : 0);
+}
+
+double Simulator::getDoubleFromRegisterPair(int reg) {
+ MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters) &&
+ ((reg % 2) == 0));
+
+ double dm_val = 0.0;
+ // Read the bits from the unsigned integer register_[] array
+ // into the double precision floating point value and return it.
+ memcpy(&dm_val, &registers_[reg], sizeof(dm_val));
+ return (dm_val);
+}
+
+int32_t Simulator::getFpuRegister(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return FPUregisters_[fpureg];
+}
+
+int64_t Simulator::getFpuRegisterLong(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters) &&
+ ((fpureg % 2) == 0));
+ return *mozilla::BitwiseCast<int64_t*>(
+ const_cast<int32_t*>(&FPUregisters_[fpureg]));
+}
+
+float Simulator::getFpuRegisterFloat(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *mozilla::BitwiseCast<float*>(
+ const_cast<int32_t*>(&FPUregisters_[fpureg]));
+}
+
+double Simulator::getFpuRegisterDouble(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters) &&
+ ((fpureg % 2) == 0));
+ return *mozilla::BitwiseCast<double*>(
+ const_cast<int32_t*>(&FPUregisters_[fpureg]));
+}
+
+// Runtime FP routines take up to two double arguments and zero
+// or one integer arguments. All are constructed here,
+// from a0-a3 or f12 and f14.
+void Simulator::getFpArgs(double* x, double* y, int32_t* z) {
+ *x = getFpuRegisterDouble(12);
+ *y = getFpuRegisterDouble(14);
+ *z = getRegister(a2);
+}
+
+void Simulator::getFpFromStack(int32_t* stack, double* x) {
+ MOZ_ASSERT(stack);
+ MOZ_ASSERT(x);
+ memcpy(x, stack, sizeof(double));
+}
+
+void Simulator::setCallResultDouble(double result) {
+ setFpuRegisterDouble(f0, result);
+}
+
+void Simulator::setCallResultFloat(float result) {
+ setFpuRegisterFloat(f0, result);
+}
+
+void Simulator::setCallResult(int64_t res) {
+ setRegister(v0, static_cast<int32_t>(res));
+ setRegister(v1, static_cast<int32_t>(res >> 32));
+}
+
+// Helper functions for setting and testing the FCSR register's bits.
+void Simulator::setFCSRBit(uint32_t cc, bool value) {
+ if (value) {
+ FCSR_ |= (1 << cc);
+ } else {
+ FCSR_ &= ~(1 << cc);
+ }
+}
+
+bool Simulator::testFCSRBit(uint32_t cc) { return FCSR_ & (1 << cc); }
+
+// Sets the rounding error codes in FCSR based on the result of the rounding.
+// Returns true if the operation was invalid.
+bool Simulator::setFCSRRoundError(double original, double rounded) {
+ bool ret = false;
+
+ setFCSRBit(kFCSRInexactCauseBit, false);
+ setFCSRBit(kFCSRUnderflowCauseBit, false);
+ setFCSRBit(kFCSROverflowCauseBit, false);
+ setFCSRBit(kFCSRInvalidOpCauseBit, false);
+
+ if (!std::isfinite(original) || !std::isfinite(rounded)) {
+ setFCSRBit(kFCSRInvalidOpFlagBit, true);
+ setFCSRBit(kFCSRInvalidOpCauseBit, true);
+ ret = true;
+ }
+
+ if (original != rounded) {
+ setFCSRBit(kFCSRInexactFlagBit, true);
+ setFCSRBit(kFCSRInexactCauseBit, true);
+ }
+
+ if (rounded < DBL_MIN && rounded > -DBL_MIN && rounded != 0) {
+ setFCSRBit(kFCSRUnderflowFlagBit, true);
+ setFCSRBit(kFCSRUnderflowCauseBit, true);
+ ret = true;
+ }
+
+ if (rounded > INT_MAX || rounded < INT_MIN) {
+ setFCSRBit(kFCSROverflowFlagBit, true);
+ setFCSRBit(kFCSROverflowCauseBit, true);
+ // The reference is not really clear but it seems this is required:
+ setFCSRBit(kFCSRInvalidOpFlagBit, true);
+ setFCSRBit(kFCSRInvalidOpCauseBit, true);
+ ret = true;
+ }
+
+ return ret;
+}
+
+// Raw access to the PC register.
+void Simulator::set_pc(int32_t value) {
+ pc_modified_ = true;
+ registers_[pc] = value;
+}
+
+bool Simulator::has_bad_pc() const {
+ return ((registers_[pc] == bad_ra) || (registers_[pc] == end_sim_pc));
+}
+
+// Raw access to the PC register without the special adjustment when reading.
+int32_t Simulator::get_pc() const { return registers_[pc]; }
+
+JS::ProfilingFrameIterator::RegisterState Simulator::registerState() {
+ wasm::RegisterState state;
+ state.pc = (void*)get_pc();
+ state.fp = (void*)getRegister(fp);
+ state.sp = (void*)getRegister(sp);
+ state.lr = (void*)getRegister(ra);
+ return state;
+}
+
+// MIPS memory instructions (except lwl/r and swl/r) trap on unaligned memory
+// access enabling the OS to handle them via trap-and-emulate.
+// Note that simulator runs have the runtime system running directly on the host
+// system and only generated code is executed in the simulator.
+// Since the host is typically IA32 it will not trap on unaligned memory access.
+// We assume that that executing correct generated code will not produce
+// unaligned memory access, so we explicitly check for address alignment and
+// trap. Note that trapping does not occur when executing wasm code, which
+// requires that unaligned memory access provides correct result.
+int Simulator::readW(uint32_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ if ((addr & kPointerAlignmentMask) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ intptr_t* ptr = reinterpret_cast<intptr_t*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned read at 0x%08x, pc=0x%08" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+void Simulator::writeW(uint32_t addr, int value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return;
+ }
+
+ if ((addr & kPointerAlignmentMask) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ intptr_t* ptr = reinterpret_cast<intptr_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned write at 0x%08x, pc=0x%08" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+double Simulator::readD(uint32_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return NAN;
+ }
+
+ if ((addr & kDoubleAlignmentMask) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ double* ptr = reinterpret_cast<double*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned (double) read at 0x%08x, pc=0x%08" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+void Simulator::writeD(uint32_t addr, double value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return;
+ }
+
+ if ((addr & kDoubleAlignmentMask) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ double* ptr = reinterpret_cast<double*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned (double) write at 0x%08x, pc=0x%08" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+uint16_t Simulator::readHU(uint32_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return 0xffff;
+ }
+
+ if ((addr & 1) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ uint16_t* ptr = reinterpret_cast<uint16_t*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned unsigned halfword read at 0x%08x, pc=0x%08" PRIxPTR "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int16_t Simulator::readH(uint32_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return -1;
+ }
+
+ if ((addr & 1) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ int16_t* ptr = reinterpret_cast<int16_t*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned signed halfword read at 0x%08x, pc=0x%08" PRIxPTR "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+void Simulator::writeH(uint32_t addr, uint16_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return;
+ }
+
+ if ((addr & 1) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ uint16_t* ptr = reinterpret_cast<uint16_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned unsigned halfword write at 0x%08x, pc=0x%08" PRIxPTR "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+void Simulator::writeH(uint32_t addr, int16_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return;
+ }
+
+ if ((addr & 1) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ int16_t* ptr = reinterpret_cast<int16_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned halfword write at 0x%08x, pc=0x%08" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+uint32_t Simulator::readBU(uint32_t addr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return 0xff;
+ }
+
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(addr);
+ return *ptr;
+}
+
+int32_t Simulator::readB(uint32_t addr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return -1;
+ }
+
+ int8_t* ptr = reinterpret_cast<int8_t*>(addr);
+ return *ptr;
+}
+
+void Simulator::writeB(uint32_t addr, uint8_t value) {
+ if (handleWasmSegFault(addr, 1)) {
+ return;
+ }
+
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+}
+
+void Simulator::writeB(uint32_t addr, int8_t value) {
+ if (handleWasmSegFault(addr, 1)) {
+ return;
+ }
+
+ int8_t* ptr = reinterpret_cast<int8_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+}
+
+int Simulator::loadLinkedW(uint32_t addr, SimInstruction* instr) {
+ if ((addr & kPointerAlignmentMask) == 0) {
+ if (handleWasmSegFault(addr, 1)) {
+ return -1;
+ }
+
+ volatile int32_t* ptr = reinterpret_cast<volatile int32_t*>(addr);
+ int32_t value = *ptr;
+ lastLLValue_ = value;
+ LLAddr_ = addr;
+ // Note that any memory write or "external" interrupt should reset this
+ // value to false.
+ LLBit_ = true;
+ return value;
+ }
+ printf("Unaligned read at 0x%08x, pc=0x%08" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int Simulator::storeConditionalW(uint32_t addr, int value,
+ SimInstruction* instr) {
+ // Correct behavior in this case, as defined by architecture, is to just
+ // return 0, but there is no point at allowing that. It is certainly an
+ // indicator of a bug.
+ if (addr != LLAddr_) {
+ printf("SC to bad address: 0x%08x, pc=0x%08" PRIxPTR ", expected: 0x%08x\n",
+ addr, reinterpret_cast<intptr_t>(instr), LLAddr_);
+ MOZ_CRASH();
+ }
+
+ if ((addr & kPointerAlignmentMask) == 0) {
+ SharedMem<int32_t*> ptr =
+ SharedMem<int32_t*>::shared(reinterpret_cast<int32_t*>(addr));
+
+ if (!LLBit_) {
+ return 0;
+ }
+
+ LLBit_ = false;
+ LLAddr_ = 0;
+ int32_t expected = lastLLValue_;
+ int32_t old =
+ AtomicOperations::compareExchangeSeqCst(ptr, expected, int32_t(value));
+ return (old == expected) ? 1 : 0;
+ }
+ printf("Unaligned SC at 0x%08x, pc=0x%08" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+uintptr_t Simulator::stackLimit() const { return stackLimit_; }
+
+uintptr_t* Simulator::addressOfStackLimit() { return &stackLimit_; }
+
+bool Simulator::overRecursed(uintptr_t newsp) const {
+ if (newsp == 0) {
+ newsp = getRegister(sp);
+ }
+ return newsp <= stackLimit();
+}
+
+bool Simulator::overRecursedWithExtra(uint32_t extra) const {
+ uintptr_t newsp = getRegister(sp) - extra;
+ return newsp <= stackLimit();
+}
+
+// Unsupported instructions use format to print an error and stop execution.
+void Simulator::format(SimInstruction* instr, const char* format) {
+ printf("Simulator found unsupported instruction:\n 0x%08" PRIxPTR ": %s\n",
+ reinterpret_cast<intptr_t>(instr), format);
+ MOZ_CRASH();
+}
+
+// Note: With the code below we assume that all runtime calls return a 64 bits
+// result. If they don't, the v1 result register contains a bogus value, which
+// is fine because it is caller-saved.
+typedef int64_t (*Prototype_General0)();
+typedef int64_t (*Prototype_General1)(int32_t arg0);
+typedef int64_t (*Prototype_General2)(int32_t arg0, int32_t arg1);
+typedef int64_t (*Prototype_General3)(int32_t arg0, int32_t arg1, int32_t arg2);
+typedef int64_t (*Prototype_General4)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3);
+typedef int64_t (*Prototype_General5)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3, int32_t arg4);
+typedef int64_t (*Prototype_General6)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3, int32_t arg4, int32_t arg5);
+typedef int64_t (*Prototype_General7)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3, int32_t arg4, int32_t arg5,
+ int32_t arg6);
+typedef int64_t (*Prototype_General8)(int32_t arg0, int32_t arg1, int32_t arg2,
+ int32_t arg3, int32_t arg4, int32_t arg5,
+ int32_t arg6, int32_t arg7);
+typedef int64_t (*Prototype_GeneralGeneralGeneralInt64)(int32_t arg0,
+ int32_t arg1,
+ int32_t arg2,
+ int64_t arg3);
+typedef int64_t (*Prototype_GeneralGeneralInt64Int64)(int32_t arg0,
+ int32_t arg1,
+ int64_t arg2,
+ int64_t arg3);
+
+typedef double (*Prototype_Double_None)();
+typedef double (*Prototype_Double_Double)(double arg0);
+typedef double (*Prototype_Double_Int)(int32_t arg0);
+typedef int32_t (*Prototype_Int_Double)(double arg0);
+typedef int64_t (*Prototype_Int64_Double)(double arg0);
+typedef int32_t (*Prototype_Int_DoubleIntInt)(double arg0, int32_t arg1,
+ int32_t arg2);
+typedef int32_t (*Prototype_Int_IntDoubleIntInt)(int32_t arg0, double arg1,
+ int32_t arg2, int32_t arg3);
+typedef float (*Prototype_Float32_Float32)(float arg0);
+typedef int32_t (*Prototype_Int_Float32)(float arg0);
+typedef float (*Prototype_Float32_Float32Float32)(float arg0, float arg1);
+typedef float (*Prototype_Float32_IntInt)(int arg0, int arg1);
+
+typedef double (*Prototype_Double_DoubleInt)(double arg0, int32_t arg1);
+typedef double (*Prototype_Double_IntInt)(int32_t arg0, int32_t arg1);
+typedef double (*Prototype_Double_IntDouble)(int32_t arg0, double arg1);
+typedef double (*Prototype_Double_DoubleDouble)(double arg0, double arg1);
+typedef int32_t (*Prototype_Int_IntDouble)(int32_t arg0, double arg1);
+
+typedef double (*Prototype_Double_DoubleDoubleDouble)(double arg0, double arg1,
+ double arg2);
+typedef double (*Prototype_Double_DoubleDoubleDoubleDouble)(double arg0,
+ double arg1,
+ double arg2,
+ double arg3);
+
+static int64_t MakeInt64(int32_t first, int32_t second) {
+ // Little-endian order.
+ return ((int64_t)second << 32) | (uint32_t)first;
+}
+
+// Software interrupt instructions are used by the simulator to call into C++.
+void Simulator::softwareInterrupt(SimInstruction* instr) {
+ int32_t func = instr->functionFieldRaw();
+ uint32_t code = (func == ff_break) ? instr->bits(25, 6) : -1;
+
+ // We first check if we met a call_rt_redirected.
+ if (instr->instructionBits() == kCallRedirInstr) {
+#if !defined(USES_O32_ABI)
+ MOZ_CRASH("Only O32 ABI supported.");
+#else
+ Redirection* redirection = Redirection::FromSwiInstruction(instr);
+ int32_t arg0 = getRegister(a0);
+ int32_t arg1 = getRegister(a1);
+ int32_t arg2 = getRegister(a2);
+ int32_t arg3 = getRegister(a3);
+
+ int32_t* stack_pointer = reinterpret_cast<int32_t*>(getRegister(sp));
+ // Args 4 and 5 are on the stack after the reserved space for args 0..3.
+ int32_t arg4 = stack_pointer[4];
+ int32_t arg5 = stack_pointer[5];
+
+ // This is dodgy but it works because the C entry stubs are never moved.
+ // See comment in codegen-arm.cc and bug 1242173.
+ int32_t saved_ra = getRegister(ra);
+
+ intptr_t external =
+ reinterpret_cast<intptr_t>(redirection->nativeFunction());
+
+ bool stack_aligned = (getRegister(sp) & (ABIStackAlignment - 1)) == 0;
+ if (!stack_aligned) {
+ fprintf(stderr, "Runtime call with unaligned stack!\n");
+ MOZ_CRASH();
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ switch (redirection->type()) {
+ case Args_General0: {
+ Prototype_General0 target =
+ reinterpret_cast<Prototype_General0>(external);
+ int64_t result = target();
+ setCallResult(result);
+ break;
+ }
+ case Args_General1: {
+ Prototype_General1 target =
+ reinterpret_cast<Prototype_General1>(external);
+ int64_t result = target(arg0);
+ setCallResult(result);
+ break;
+ }
+ case Args_General2: {
+ Prototype_General2 target =
+ reinterpret_cast<Prototype_General2>(external);
+ int64_t result = target(arg0, arg1);
+ setCallResult(result);
+ break;
+ }
+ case Args_General3: {
+ Prototype_General3 target =
+ reinterpret_cast<Prototype_General3>(external);
+ int64_t result = target(arg0, arg1, arg2);
+ setCallResult(result);
+ break;
+ }
+ case Args_General4: {
+ Prototype_General4 target =
+ reinterpret_cast<Prototype_General4>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ setCallResult(result);
+ break;
+ }
+ case Args_General5: {
+ Prototype_General5 target =
+ reinterpret_cast<Prototype_General5>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4);
+ setCallResult(result);
+ break;
+ }
+ case Args_General6: {
+ Prototype_General6 target =
+ reinterpret_cast<Prototype_General6>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5);
+ setCallResult(result);
+ break;
+ }
+ case Args_General7: {
+ Prototype_General7 target =
+ reinterpret_cast<Prototype_General7>(external);
+ int32_t arg6 = stack_pointer[6];
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6);
+ setCallResult(result);
+ break;
+ }
+ case Args_General8: {
+ Prototype_General8 target =
+ reinterpret_cast<Prototype_General8>(external);
+ int32_t arg6 = stack_pointer[6];
+ int32_t arg7 = stack_pointer[7];
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+ setCallResult(result);
+ break;
+ }
+ case Args_Double_None: {
+ Prototype_Double_None target =
+ reinterpret_cast<Prototype_Double_None>(external);
+ double dresult = target();
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int_Double: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Int_Double target =
+ reinterpret_cast<Prototype_Int_Double>(external);
+ int32_t res = target(dval0);
+ setRegister(v0, res);
+ break;
+ }
+ case Args_Int_GeneralGeneralGeneralInt64: {
+ Prototype_GeneralGeneralGeneralInt64 target =
+ reinterpret_cast<Prototype_GeneralGeneralGeneralInt64>(external);
+ // The int64 arg is not split across register and stack
+ int64_t result = target(arg0, arg1, arg2, MakeInt64(arg4, arg5));
+ setCallResult(result);
+ break;
+ }
+ case Args_Int_GeneralGeneralInt64Int64: {
+ Prototype_GeneralGeneralInt64Int64 target =
+ reinterpret_cast<Prototype_GeneralGeneralInt64Int64>(external);
+ int64_t result =
+ target(arg0, arg1, MakeInt64(arg2, arg3), MakeInt64(arg4, arg5));
+ setCallResult(result);
+ break;
+ }
+ case Args_Int64_Double: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Int64_Double target =
+ reinterpret_cast<Prototype_Int64_Double>(external);
+ int64_t result = target(dval0);
+ setCallResult(result);
+ break;
+ }
+ case Args_Int_DoubleIntInt: {
+ double dval = getFpuRegisterDouble(12);
+ Prototype_Int_DoubleIntInt target =
+ reinterpret_cast<Prototype_Int_DoubleIntInt>(external);
+ int32_t res = target(dval, arg2, arg3);
+ setRegister(v0, res);
+ break;
+ }
+ case Args_Int_IntDoubleIntInt: {
+ double dval = getDoubleFromRegisterPair(a2);
+ Prototype_Int_IntDoubleIntInt target =
+ reinterpret_cast<Prototype_Int_IntDoubleIntInt>(external);
+ int32_t res = target(arg0, dval, arg4, arg5);
+ setRegister(v0, res);
+ break;
+ }
+ case Args_Double_Double: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Double_Double target =
+ reinterpret_cast<Prototype_Double_Double>(external);
+ double dresult = target(dval0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Float32_Float32: {
+ float fval0;
+ fval0 = getFpuRegisterFloat(12);
+ Prototype_Float32_Float32 target =
+ reinterpret_cast<Prototype_Float32_Float32>(external);
+ float fresult = target(fval0);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Int_Float32: {
+ float fval0;
+ fval0 = getFpuRegisterFloat(12);
+ Prototype_Int_Float32 target =
+ reinterpret_cast<Prototype_Int_Float32>(external);
+ int32_t result = target(fval0);
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Float32_Float32Float32: {
+ float fval0;
+ float fval1;
+ fval0 = getFpuRegisterFloat(12);
+ fval1 = getFpuRegisterFloat(14);
+ Prototype_Float32_Float32Float32 target =
+ reinterpret_cast<Prototype_Float32_Float32Float32>(external);
+ float fresult = target(fval0, fval1);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Float32_IntInt: {
+ Prototype_Float32_IntInt target =
+ reinterpret_cast<Prototype_Float32_IntInt>(external);
+ float fresult = target(arg0, arg1);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Double_Int: {
+ Prototype_Double_Int target =
+ reinterpret_cast<Prototype_Double_Int>(external);
+ double dresult = target(arg0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_IntInt: {
+ Prototype_Double_IntInt target =
+ reinterpret_cast<Prototype_Double_IntInt>(external);
+ double dresult = target(arg0, arg1);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleInt: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Double_DoubleInt target =
+ reinterpret_cast<Prototype_Double_DoubleInt>(external);
+ double dresult = target(dval0, ival);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleDouble: {
+ double dval0, dval1;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ Prototype_Double_DoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDouble>(external);
+ double dresult = target(dval0, dval1);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_IntDouble: {
+ int32_t ival = getRegister(a0);
+ double dval0 = getDoubleFromRegisterPair(a2);
+ Prototype_Double_IntDouble target =
+ reinterpret_cast<Prototype_Double_IntDouble>(external);
+ double dresult = target(ival, dval0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int_IntDouble: {
+ int32_t ival = getRegister(a0);
+ double dval0 = getDoubleFromRegisterPair(a2);
+ Prototype_Int_IntDouble target =
+ reinterpret_cast<Prototype_Int_IntDouble>(external);
+ int32_t result = target(ival, dval0);
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Double_DoubleDoubleDouble: {
+ double dval0, dval1, dval2;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ // the last argument is on stack
+ getFpFromStack(stack_pointer + 4, &dval2);
+ Prototype_Double_DoubleDoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDoubleDouble>(external);
+ double dresult = target(dval0, dval1, dval2);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleDoubleDoubleDouble: {
+ double dval0, dval1, dval2, dval3;
+ int32_t ival;
+ getFpArgs(&dval0, &dval1, &ival);
+ // the two last arguments are on stack
+ getFpFromStack(stack_pointer + 4, &dval2);
+ getFpFromStack(stack_pointer + 6, &dval3);
+ Prototype_Double_DoubleDoubleDoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDoubleDoubleDouble>(
+ external);
+ double dresult = target(dval0, dval1, dval2, dval3);
+ setCallResultDouble(dresult);
+ break;
+ }
+ default:
+ MOZ_CRASH("call");
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ setRegister(ra, saved_ra);
+ set_pc(getRegister(ra));
+#endif
+ } else if (func == ff_break && code <= kMaxStopCode) {
+ if (isWatchpoint(code)) {
+ printWatchpoint(code);
+ } else {
+ increaseStopCounter(code);
+ handleStop(code, instr);
+ }
+ } else {
+ switch (func) {
+ case ff_tge:
+ case ff_tgeu:
+ case ff_tlt:
+ case ff_tltu:
+ case ff_teq:
+ case ff_tne:
+ if (instr->bits(15, 6) == kWasmTrapCode) {
+ uint8_t* newPC;
+ if (wasm::HandleIllegalInstruction(registerState(), &newPC)) {
+ set_pc(int32_t(newPC));
+ return;
+ }
+ }
+ };
+ // All remaining break_ codes, and all traps are handled here.
+ MipsDebugger dbg(this);
+ dbg.debug();
+ }
+}
+
+// Stop helper functions.
+bool Simulator::isWatchpoint(uint32_t code) {
+ return (code <= kMaxWatchpointCode);
+}
+
+void Simulator::printWatchpoint(uint32_t code) {
+ MipsDebugger dbg(this);
+ ++break_count_;
+ printf(
+ "\n---- break %d marker: %3d (instr count: %8d) ----------"
+ "----------------------------------",
+ code, break_count_, icount_);
+ dbg.printAllRegs(); // Print registers and continue running.
+}
+
+void Simulator::handleStop(uint32_t code, SimInstruction* instr) {
+ // Stop if it is enabled, otherwise go on jumping over the stop
+ // and the message address.
+ if (isEnabledStop(code)) {
+ MipsDebugger dbg(this);
+ dbg.stop(instr);
+ } else {
+ set_pc(get_pc() + 2 * SimInstruction::kInstrSize);
+ }
+}
+
+bool Simulator::isStopInstruction(SimInstruction* instr) {
+ int32_t func = instr->functionFieldRaw();
+ uint32_t code = static_cast<uint32_t>(instr->bits(25, 6));
+ return (func == ff_break) && code > kMaxWatchpointCode &&
+ code <= kMaxStopCode;
+}
+
+bool Simulator::isEnabledStop(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ MOZ_ASSERT(code > kMaxWatchpointCode);
+ return !(watchedStops_[code].count_ & kStopDisabledBit);
+}
+
+void Simulator::enableStop(uint32_t code) {
+ if (!isEnabledStop(code)) {
+ watchedStops_[code].count_ &= ~kStopDisabledBit;
+ }
+}
+
+void Simulator::disableStop(uint32_t code) {
+ if (isEnabledStop(code)) {
+ watchedStops_[code].count_ |= kStopDisabledBit;
+ }
+}
+
+void Simulator::increaseStopCounter(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ if ((watchedStops_[code].count_ & ~(1 << 31)) == 0x7fffffff) {
+ printf(
+ "Stop counter for code %i has overflowed.\n"
+ "Enabling this code and reseting the counter to 0.\n",
+ code);
+ watchedStops_[code].count_ = 0;
+ enableStop(code);
+ } else {
+ watchedStops_[code].count_++;
+ }
+}
+
+// Print a stop status.
+void Simulator::printStopInfo(uint32_t code) {
+ if (code <= kMaxWatchpointCode) {
+ printf("That is a watchpoint, not a stop.\n");
+ return;
+ } else if (code > kMaxStopCode) {
+ printf("Code too large, only %u stops can be used\n", kMaxStopCode + 1);
+ return;
+ }
+ const char* state = isEnabledStop(code) ? "Enabled" : "Disabled";
+ int32_t count = watchedStops_[code].count_ & ~kStopDisabledBit;
+ // Don't print the state of unused breakpoints.
+ if (count != 0) {
+ if (watchedStops_[code].desc_) {
+ printf("stop %i - 0x%x: \t%s, \tcounter = %i, \t%s\n", code, code, state,
+ count, watchedStops_[code].desc_);
+ } else {
+ printf("stop %i - 0x%x: \t%s, \tcounter = %i\n", code, code, state,
+ count);
+ }
+ }
+}
+
+void Simulator::signalExceptions() {
+ for (int i = 1; i < kNumExceptions; i++) {
+ if (exceptions[i] != 0) {
+ MOZ_CRASH("Error: Exception raised.");
+ }
+ }
+}
+
+// Handle execution based on instruction types.
+void Simulator::configureTypeRegister(SimInstruction* instr, int32_t& alu_out,
+ int64_t& i64hilo, uint64_t& u64hilo,
+ int32_t& next_pc,
+ int32_t& return_addr_reg,
+ bool& do_interrupt) {
+ // Every local variable declared here needs to be const.
+ // This is to make sure that changed values are sent back to
+ // decodeTypeRegister correctly.
+
+ // Instruction fields.
+ const OpcodeField op = instr->opcodeFieldRaw();
+ const int32_t rs_reg = instr->rsValue();
+ const int32_t rs = getRegister(rs_reg);
+ const uint32_t rs_u = static_cast<uint32_t>(rs);
+ const int32_t rt_reg = instr->rtValue();
+ const int32_t rt = getRegister(rt_reg);
+ const uint32_t rt_u = static_cast<uint32_t>(rt);
+ const int32_t rd_reg = instr->rdValue();
+ const uint32_t sa = instr->saValue();
+
+ const int32_t fs_reg = instr->fsValue();
+
+ // ---------- Configuration.
+ switch (op) {
+ case op_cop1: // Coprocessor instructions.
+ switch (instr->rsFieldRaw()) {
+ case rs_bc1: // Handled in DecodeTypeImmed, should never come here.
+ MOZ_CRASH();
+ break;
+ case rs_cfc1:
+ // At the moment only FCSR is supported.
+ MOZ_ASSERT(fs_reg == kFCSRRegister);
+ alu_out = FCSR_;
+ break;
+ case rs_mfc1:
+ alu_out = getFpuRegister(fs_reg);
+ break;
+ case rs_mfhc1:
+ MOZ_CRASH();
+ break;
+ case rs_ctc1:
+ case rs_mtc1:
+ case rs_mthc1:
+ // Do the store in the execution step.
+ break;
+ case rs_s:
+ case rs_d:
+ case rs_w:
+ case rs_l:
+ case rs_ps:
+ // Do everything in the execution step.
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case op_cop1x:
+ break;
+ case op_special:
+ switch (instr->functionFieldRaw()) {
+ case ff_jr:
+ case ff_jalr:
+ next_pc = getRegister(instr->rsValue());
+ return_addr_reg = instr->rdValue();
+ break;
+ case ff_sll:
+ alu_out = rt << sa;
+ break;
+ case ff_srl:
+ if (rs_reg == 0) {
+ // Regular logical right shift of a word by a fixed number of
+ // bits instruction. RS field is always equal to 0.
+ alu_out = rt_u >> sa;
+ } else {
+ // Logical right-rotate of a word by a fixed number of bits. This
+ // is special case of SRL instruction, added in MIPS32 Release 2.
+ // RS field is equal to 00001.
+ alu_out = (rt_u >> sa) | (rt_u << (32 - sa));
+ }
+ break;
+ case ff_sra:
+ alu_out = rt >> sa;
+ break;
+ case ff_sllv:
+ alu_out = rt << rs;
+ break;
+ case ff_srlv:
+ if (sa == 0) {
+ // Regular logical right-shift of a word by a variable number of
+ // bits instruction. SA field is always equal to 0.
+ alu_out = rt_u >> rs;
+ } else {
+ // Logical right-rotate of a word by a variable number of bits.
+ // This is special case od SRLV instruction, added in MIPS32
+ // Release 2. SA field is equal to 00001.
+ alu_out = (rt_u >> rs_u) | (rt_u << (32 - rs_u));
+ }
+ break;
+ case ff_srav:
+ alu_out = rt >> rs;
+ break;
+ case ff_mfhi:
+ alu_out = getRegister(HI);
+ break;
+ case ff_mflo:
+ alu_out = getRegister(LO);
+ break;
+ case ff_mult:
+ i64hilo = static_cast<int64_t>(rs) * static_cast<int64_t>(rt);
+ break;
+ case ff_multu:
+ u64hilo = static_cast<uint64_t>(rs_u) * static_cast<uint64_t>(rt_u);
+ break;
+ case ff_add:
+ if (HaveSameSign(rs, rt)) {
+ if (rs > 0) {
+ exceptions[kIntegerOverflow] = rs > (kRegisterskMaxValue - rt);
+ } else if (rs < 0) {
+ exceptions[kIntegerUnderflow] = rs < (kRegisterskMinValue - rt);
+ }
+ }
+ alu_out = rs + rt;
+ break;
+ case ff_addu:
+ alu_out = rs + rt;
+ break;
+ case ff_sub:
+ if (!HaveSameSign(rs, rt)) {
+ if (rs > 0) {
+ exceptions[kIntegerOverflow] = rs > (kRegisterskMaxValue + rt);
+ } else if (rs < 0) {
+ exceptions[kIntegerUnderflow] = rs < (kRegisterskMinValue + rt);
+ }
+ }
+ alu_out = rs - rt;
+ break;
+ case ff_subu:
+ alu_out = rs - rt;
+ break;
+ case ff_and:
+ alu_out = rs & rt;
+ break;
+ case ff_or:
+ alu_out = rs | rt;
+ break;
+ case ff_xor:
+ alu_out = rs ^ rt;
+ break;
+ case ff_nor:
+ alu_out = ~(rs | rt);
+ break;
+ case ff_slt:
+ alu_out = rs < rt ? 1 : 0;
+ break;
+ case ff_sltu:
+ alu_out = rs_u < rt_u ? 1 : 0;
+ break;
+ // Break and trap instructions.
+ case ff_break:
+ do_interrupt = true;
+ break;
+ case ff_tge:
+ do_interrupt = rs >= rt;
+ break;
+ case ff_tgeu:
+ do_interrupt = rs_u >= rt_u;
+ break;
+ case ff_tlt:
+ do_interrupt = rs < rt;
+ break;
+ case ff_tltu:
+ do_interrupt = rs_u < rt_u;
+ break;
+ case ff_teq:
+ do_interrupt = rs == rt;
+ break;
+ case ff_tne:
+ do_interrupt = rs != rt;
+ break;
+ case ff_movn:
+ case ff_movz:
+ case ff_movci:
+ case ff_sync:
+ // No action taken on decode.
+ break;
+ case ff_div:
+ case ff_divu:
+ // div and divu never raise exceptions.
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case op_special2:
+ switch (instr->functionFieldRaw()) {
+ case ff_mul:
+ alu_out = rs_u * rt_u; // Only the lower 32 bits are kept.
+ break;
+ case ff_mult:
+ i64hilo = static_cast<int64_t>(rs) * static_cast<int64_t>(rt);
+ break;
+ case ff_multu:
+ u64hilo = static_cast<uint64_t>(rs_u) * static_cast<uint64_t>(rt_u);
+ break;
+ case ff_madd:
+ i64hilo += static_cast<int64_t>(rs) * static_cast<int64_t>(rt);
+ break;
+ case ff_maddu:
+ u64hilo += static_cast<uint64_t>(rs_u) * static_cast<uint64_t>(rt_u);
+ break;
+ case ff_clz:
+ alu_out = rs_u ? __builtin_clz(rs_u) : 32;
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case op_special3:
+ switch (instr->functionFieldRaw()) {
+ case ff_ins: { // Mips32r2 instruction.
+ // Interpret rd field as 5-bit msb of insert.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of insert.
+ uint16_t lsb = sa;
+ uint16_t size = msb - lsb + 1;
+ uint32_t mask = (1 << size) - 1;
+ alu_out = (rt_u & ~(mask << lsb)) | ((rs_u & mask) << lsb);
+ break;
+ }
+ case ff_ext: { // Mips32r2 instruction.
+ // Interpret rd field as 5-bit msb of extract.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of extract.
+ uint16_t lsb = sa;
+ uint16_t size = msb + 1;
+ uint32_t mask = (1 << size) - 1;
+ alu_out = (rs_u & (mask << lsb)) >> lsb;
+ break;
+ }
+ case ff_bshfl: { // Mips32r2 instruction.
+ if (16 == sa) { // seb
+ alu_out = I32(I8(rt));
+ } else if (24 == sa) { // seh
+ alu_out = I32(I16(rt));
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+void Simulator::decodeTypeRegister(SimInstruction* instr) {
+ // Instruction fields.
+ const OpcodeField op = instr->opcodeFieldRaw();
+ const int32_t rs_reg = instr->rsValue();
+ const int32_t rs = getRegister(rs_reg);
+ const uint32_t rs_u = static_cast<uint32_t>(rs);
+ const int32_t rt_reg = instr->rtValue();
+ const int32_t rt = getRegister(rt_reg);
+ const uint32_t rt_u = static_cast<uint32_t>(rt);
+ const int32_t rd_reg = instr->rdValue();
+
+ const int32_t fr_reg = instr->frValue();
+ const int32_t fs_reg = instr->fsValue();
+ const int32_t ft_reg = instr->ftValue();
+ const int32_t fd_reg = instr->fdValue();
+ int64_t i64hilo = 0;
+ uint64_t u64hilo = 0;
+
+ // ALU output.
+ // It should not be used as is. Instructions using it should always
+ // initialize it first.
+ int32_t alu_out = 0x12345678;
+
+ // For break and trap instructions.
+ bool do_interrupt = false;
+
+ // For jr and jalr.
+ // Get current pc.
+ int32_t current_pc = get_pc();
+ // Next pc
+ int32_t next_pc = 0;
+ int32_t return_addr_reg = 31;
+
+ // Set up the variables if needed before executing the instruction.
+ configureTypeRegister(instr, alu_out, i64hilo, u64hilo, next_pc,
+ return_addr_reg, do_interrupt);
+
+ // ---------- Raise exceptions triggered.
+ signalExceptions();
+
+ // ---------- Execution.
+ switch (op) {
+ case op_cop1:
+ switch (instr->rsFieldRaw()) {
+ case rs_bc1: // Branch on coprocessor condition.
+ MOZ_CRASH();
+ break;
+ case rs_cfc1:
+ case rs_mfc1:
+ setRegister(rt_reg, alu_out);
+ break;
+ case rs_mfhc1:
+ MOZ_CRASH();
+ break;
+ case rs_ctc1:
+ // At the moment only FCSR is supported.
+ MOZ_ASSERT(fs_reg == kFCSRRegister);
+ FCSR_ = registers_[rt_reg];
+ break;
+ case rs_mtc1:
+ FPUregisters_[fs_reg] = registers_[rt_reg];
+ break;
+ case rs_mthc1:
+ MOZ_CRASH();
+ break;
+ case rs_s:
+ float f, ft_value, fs_value;
+ uint32_t cc, fcsr_cc;
+ fs_value = getFpuRegisterFloat(fs_reg);
+ ft_value = getFpuRegisterFloat(ft_reg);
+ cc = instr->fcccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ switch (instr->functionFieldRaw()) {
+ case ff_add_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value + ft_value);
+ break;
+ case ff_sub_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value - ft_value);
+ break;
+ case ff_mul_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value * ft_value);
+ break;
+ case ff_div_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value / ft_value);
+ break;
+ case ff_abs_fmt:
+ setFpuRegisterFloat(fd_reg, fabsf(fs_value));
+ break;
+ case ff_mov_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value);
+ break;
+ case ff_neg_fmt:
+ setFpuRegisterFloat(fd_reg, -fs_value);
+ break;
+ case ff_sqrt_fmt:
+ setFpuRegisterFloat(fd_reg, sqrtf(fs_value));
+ break;
+ case ff_c_un_fmt:
+ setFCSRBit(fcsr_cc,
+ mozilla::IsNaN(fs_value) || mozilla::IsNaN(ft_value));
+ break;
+ case ff_c_eq_fmt:
+ setFCSRBit(fcsr_cc, (fs_value == ft_value));
+ break;
+ case ff_c_ueq_fmt:
+ setFCSRBit(fcsr_cc,
+ (fs_value == ft_value) || (mozilla::IsNaN(fs_value) ||
+ mozilla::IsNaN(ft_value)));
+ break;
+ case ff_c_olt_fmt:
+ setFCSRBit(fcsr_cc, (fs_value < ft_value));
+ break;
+ case ff_c_ult_fmt:
+ setFCSRBit(fcsr_cc,
+ (fs_value < ft_value) || (mozilla::IsNaN(fs_value) ||
+ mozilla::IsNaN(ft_value)));
+ break;
+ case ff_c_ole_fmt:
+ setFCSRBit(fcsr_cc, (fs_value <= ft_value));
+ break;
+ case ff_c_ule_fmt:
+ setFCSRBit(fcsr_cc,
+ (fs_value <= ft_value) || (mozilla::IsNaN(fs_value) ||
+ mozilla::IsNaN(ft_value)));
+ break;
+ case ff_cvt_d_fmt:
+ f = getFpuRegisterFloat(fs_reg);
+ setFpuRegisterDouble(fd_reg, static_cast<double>(f));
+ break;
+ case ff_cvt_w_fmt: // Convert float to word.
+ // Rounding modes are not yet supported.
+ MOZ_ASSERT((FCSR_ & 3) == 0);
+ // In rounding mode 0 it should behave like ROUND.
+ [[fallthrough]];
+ case ff_round_w_fmt: { // Round double to word (round half to
+ // even).
+ float rounded = std::floor(fs_value + 0.5);
+ int32_t result = static_cast<int32_t>(rounded);
+ if ((result & 1) != 0 && result - fs_value == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ result--;
+ }
+ setFpuRegister(fd_reg, result);
+ if (setFCSRRoundError(fs_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_trunc_w_fmt: { // Truncate float to word (round towards 0).
+ float rounded = truncf(fs_value);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegister(fd_reg, result);
+ if (setFCSRRoundError(fs_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_floor_w_fmt: { // Round float to word towards negative
+ // infinity.
+ float rounded = std::floor(fs_value);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegister(fd_reg, result);
+ if (setFCSRRoundError(fs_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_ceil_w_fmt: { // Round double to word towards positive
+ // infinity.
+ float rounded = std::ceil(fs_value);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegister(fd_reg, result);
+ if (setFCSRRoundError(fs_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_cvt_l_fmt:
+ case ff_round_l_fmt:
+ case ff_trunc_l_fmt:
+ case ff_floor_l_fmt:
+ case ff_ceil_l_fmt:
+ case ff_cvt_ps_s:
+ case ff_c_f_fmt:
+ MOZ_CRASH();
+ break;
+ case ff_movf_fmt:
+ // location of cc field in MOVF is equal to float branch
+ // instructions
+ cc = instr->fbccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ if (testFCSRBit(fcsr_cc)) {
+ setFpuRegisterFloat(fd_reg, getFpuRegisterFloat(fs_reg));
+ }
+ break;
+ case ff_movz_fmt:
+ if (rt == 0) {
+ setFpuRegisterFloat(fd_reg, getFpuRegisterFloat(fs_reg));
+ }
+ break;
+ case ff_movn_fmt:
+ if (rt != 0) {
+ setFpuRegisterFloat(fd_reg, getFpuRegisterFloat(fs_reg));
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case rs_d:
+ double dt_value, ds_value;
+ ds_value = getFpuRegisterDouble(fs_reg);
+ cc = instr->fcccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ switch (instr->functionFieldRaw()) {
+ case ff_add_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFpuRegisterDouble(fd_reg, ds_value + dt_value);
+ break;
+ case ff_sub_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFpuRegisterDouble(fd_reg, ds_value - dt_value);
+ break;
+ case ff_mul_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFpuRegisterDouble(fd_reg, ds_value * dt_value);
+ break;
+ case ff_div_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFpuRegisterDouble(fd_reg, ds_value / dt_value);
+ break;
+ case ff_abs_fmt:
+ setFpuRegisterDouble(fd_reg, fabs(ds_value));
+ break;
+ case ff_mov_fmt:
+ setFpuRegisterDouble(fd_reg, ds_value);
+ break;
+ case ff_neg_fmt:
+ setFpuRegisterDouble(fd_reg, -ds_value);
+ break;
+ case ff_sqrt_fmt:
+ setFpuRegisterDouble(fd_reg, sqrt(ds_value));
+ break;
+ case ff_c_un_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFCSRBit(fcsr_cc,
+ mozilla::IsNaN(ds_value) || mozilla::IsNaN(dt_value));
+ break;
+ case ff_c_eq_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFCSRBit(fcsr_cc, (ds_value == dt_value));
+ break;
+ case ff_c_ueq_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFCSRBit(fcsr_cc,
+ (ds_value == dt_value) || (mozilla::IsNaN(ds_value) ||
+ mozilla::IsNaN(dt_value)));
+ break;
+ case ff_c_olt_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFCSRBit(fcsr_cc, (ds_value < dt_value));
+ break;
+ case ff_c_ult_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFCSRBit(fcsr_cc,
+ (ds_value < dt_value) || (mozilla::IsNaN(ds_value) ||
+ mozilla::IsNaN(dt_value)));
+ break;
+ case ff_c_ole_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFCSRBit(fcsr_cc, (ds_value <= dt_value));
+ break;
+ case ff_c_ule_fmt:
+ dt_value = getFpuRegisterDouble(ft_reg);
+ setFCSRBit(fcsr_cc,
+ (ds_value <= dt_value) || (mozilla::IsNaN(ds_value) ||
+ mozilla::IsNaN(dt_value)));
+ break;
+ case ff_cvt_w_fmt: // Convert double to word.
+ // Rounding modes are not yet supported.
+ MOZ_ASSERT((FCSR_ & 3) == 0);
+ // In rounding mode 0 it should behave like ROUND.
+ [[fallthrough]];
+ case ff_round_w_fmt: { // Round double to word (round half to
+ // even).
+ double rounded = std::floor(ds_value + 0.5);
+ int32_t result = static_cast<int32_t>(rounded);
+ if ((result & 1) != 0 && result - ds_value == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ result--;
+ }
+ setFpuRegister(fd_reg, result);
+ if (setFCSRRoundError(ds_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_trunc_w_fmt: { // Truncate double to word (round towards
+ // 0).
+ double rounded = trunc(ds_value);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegister(fd_reg, result);
+ if (setFCSRRoundError(ds_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_floor_w_fmt: { // Round double to word towards negative
+ // infinity.
+ double rounded = std::floor(ds_value);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegister(fd_reg, result);
+ if (setFCSRRoundError(ds_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_ceil_w_fmt: { // Round double to word towards positive
+ // infinity.
+ double rounded = std::ceil(ds_value);
+ int32_t result = static_cast<int32_t>(rounded);
+ setFpuRegister(fd_reg, result);
+ if (setFCSRRoundError(ds_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_cvt_s_fmt: // Convert double to float (single).
+ setFpuRegisterFloat(fd_reg, static_cast<float>(ds_value));
+ break;
+ case ff_cvt_l_fmt:
+ case ff_trunc_l_fmt:
+ case ff_round_l_fmt:
+ case ff_floor_l_fmt:
+ case ff_ceil_l_fmt:
+ case ff_c_f_fmt:
+ MOZ_CRASH();
+ break;
+ case ff_movf_fmt:
+ // location of cc field in MOVF is equal to float branch
+ // instructions
+ cc = instr->fbccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ if (testFCSRBit(fcsr_cc)) {
+ setFpuRegisterDouble(fd_reg, getFpuRegisterDouble(fs_reg));
+ }
+ break;
+ case ff_movz_fmt:
+ if (rt == 0) {
+ setFpuRegisterDouble(fd_reg, getFpuRegisterDouble(fs_reg));
+ }
+ break;
+ case ff_movn_fmt:
+ if (rt != 0) {
+ setFpuRegisterDouble(fd_reg, getFpuRegisterDouble(fs_reg));
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case rs_w:
+ switch (instr->functionFieldRaw()) {
+ case ff_cvt_s_fmt: // Convert word to float (single).
+ alu_out = getFpuRegister(fs_reg);
+ setFpuRegisterFloat(fd_reg, static_cast<float>(alu_out));
+ break;
+ case ff_cvt_d_fmt: // Convert word to double.
+ alu_out = getFpuRegister(fs_reg);
+ setFpuRegisterDouble(fd_reg, static_cast<double>(alu_out));
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case rs_l:
+ switch (instr->functionFieldRaw()) {
+ case ff_cvt_d_fmt:
+ case ff_cvt_s_fmt:
+ MOZ_CRASH();
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case rs_ps:
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case op_cop1x:
+ switch (instr->functionFieldRaw()) {
+ case ff_madd_s:
+ float fr, ft, fs;
+ fr = getFpuRegisterFloat(fr_reg);
+ fs = getFpuRegisterFloat(fs_reg);
+ ft = getFpuRegisterFloat(ft_reg);
+ setFpuRegisterFloat(fd_reg, fs * ft + fr);
+ break;
+ case ff_madd_d:
+ double dr, dt, ds;
+ dr = getFpuRegisterDouble(fr_reg);
+ ds = getFpuRegisterDouble(fs_reg);
+ dt = getFpuRegisterDouble(ft_reg);
+ setFpuRegisterDouble(fd_reg, ds * dt + dr);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case op_special:
+ switch (instr->functionFieldRaw()) {
+ case ff_jr: {
+ SimInstruction* branch_delay_instr =
+ reinterpret_cast<SimInstruction*>(current_pc +
+ SimInstruction::kInstrSize);
+ branchDelayInstructionDecode(branch_delay_instr);
+ set_pc(next_pc);
+ pc_modified_ = true;
+ break;
+ }
+ case ff_jalr: {
+ SimInstruction* branch_delay_instr =
+ reinterpret_cast<SimInstruction*>(current_pc +
+ SimInstruction::kInstrSize);
+ setRegister(return_addr_reg,
+ current_pc + 2 * SimInstruction::kInstrSize);
+ branchDelayInstructionDecode(branch_delay_instr);
+ set_pc(next_pc);
+ pc_modified_ = true;
+ break;
+ }
+ // Instructions using HI and LO registers.
+ case ff_mult:
+ setRegister(LO, static_cast<int32_t>(i64hilo & 0xffffffff));
+ setRegister(HI, static_cast<int32_t>(i64hilo >> 32));
+ break;
+ case ff_multu:
+ setRegister(LO, static_cast<int32_t>(u64hilo & 0xffffffff));
+ setRegister(HI, static_cast<int32_t>(u64hilo >> 32));
+ break;
+ case ff_div:
+ // Divide by zero and overflow was not checked in the configuration
+ // step - div and divu do not raise exceptions. On division by 0
+ // the result will be UNPREDICTABLE. On overflow (INT_MIN/-1),
+ // return INT_MIN which is what the hardware does.
+ if (rs == INT_MIN && rt == -1) {
+ setRegister(LO, INT_MIN);
+ setRegister(HI, 0);
+ } else if (rt != 0) {
+ setRegister(LO, rs / rt);
+ setRegister(HI, rs % rt);
+ }
+ break;
+ case ff_divu:
+ if (rt_u != 0) {
+ setRegister(LO, rs_u / rt_u);
+ setRegister(HI, rs_u % rt_u);
+ }
+ break;
+ // Break and trap instructions.
+ case ff_break:
+ case ff_tge:
+ case ff_tgeu:
+ case ff_tlt:
+ case ff_tltu:
+ case ff_teq:
+ case ff_tne:
+ if (do_interrupt) {
+ softwareInterrupt(instr);
+ }
+ break;
+ case ff_sync:
+ switch (instr->bits(10, 6)) {
+ case 0x0:
+ case 0x4:
+ case 0x10:
+ case 0x11:
+ case 0x12:
+ case 0x13:
+ AtomicOperations::fenceSeqCst();
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ // Conditional moves.
+ case ff_movn:
+ if (rt) setRegister(rd_reg, rs);
+ break;
+ case ff_movci: {
+ uint32_t cc = instr->fbccValue();
+ uint32_t fcsr_cc = GetFCSRConditionBit(cc);
+ if (instr->bit(16)) { // Read Tf bit.
+ if (testFCSRBit(fcsr_cc)) setRegister(rd_reg, rs);
+ } else {
+ if (!testFCSRBit(fcsr_cc)) setRegister(rd_reg, rs);
+ }
+ break;
+ }
+ case ff_movz:
+ if (!rt) setRegister(rd_reg, rs);
+ break;
+ default: // For other special opcodes we do the default operation.
+ setRegister(rd_reg, alu_out);
+ }
+ break;
+ case op_special2:
+ switch (instr->functionFieldRaw()) {
+ case ff_mul:
+ setRegister(rd_reg, alu_out);
+ // HI and LO are UNPREDICTABLE after the operation.
+ setRegister(LO, Unpredictable);
+ setRegister(HI, Unpredictable);
+ break;
+ case ff_madd:
+ setRegister(
+ LO, getRegister(LO) + static_cast<int32_t>(i64hilo & 0xffffffff));
+ setRegister(HI,
+ getRegister(HI) + static_cast<int32_t>(i64hilo >> 32));
+ break;
+ case ff_maddu:
+ setRegister(
+ LO, getRegister(LO) + static_cast<int32_t>(u64hilo & 0xffffffff));
+ setRegister(HI,
+ getRegister(HI) + static_cast<int32_t>(u64hilo >> 32));
+ break;
+ default: // For other special2 opcodes we do the default operation.
+ setRegister(rd_reg, alu_out);
+ }
+ break;
+ case op_special3:
+ switch (instr->functionFieldRaw()) {
+ case ff_ins:
+ // Ins instr leaves result in Rt, rather than Rd.
+ setRegister(rt_reg, alu_out);
+ break;
+ case ff_ext:
+ // Ext instr leaves result in Rt, rather than Rd.
+ setRegister(rt_reg, alu_out);
+ break;
+ case ff_bshfl:
+ setRegister(rd_reg, alu_out);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ // Unimplemented opcodes raised an error in the configuration step before,
+ // so we can use the default here to set the destination register in
+ // common cases.
+ default:
+ setRegister(rd_reg, alu_out);
+ }
+}
+
+// Type 2: instructions using a 16 bytes immediate. (e.g. addi, beq).
+void Simulator::decodeTypeImmediate(SimInstruction* instr) {
+ // Instruction fields.
+ OpcodeField op = instr->opcodeFieldRaw();
+ int32_t rs = getRegister(instr->rsValue());
+ uint32_t rs_u = static_cast<uint32_t>(rs);
+ int32_t rt_reg = instr->rtValue(); // Destination register.
+ int32_t rt = getRegister(rt_reg);
+ int16_t imm16 = instr->imm16Value();
+
+ int32_t ft_reg = instr->ftValue(); // Destination register.
+
+ // Zero extended immediate.
+ uint32_t oe_imm16 = 0xffff & imm16;
+ // Sign extended immediate.
+ int32_t se_imm16 = imm16;
+
+ // Get current pc.
+ int32_t current_pc = get_pc();
+ // Next pc.
+ int32_t next_pc = bad_ra;
+
+ // Used for conditional branch instructions.
+ bool do_branch = false;
+ bool execute_branch_delay_instruction = false;
+
+ // Used for arithmetic instructions.
+ int32_t alu_out = 0;
+ // Floating point.
+ double fp_out = 0.0;
+ uint32_t cc, cc_value, fcsr_cc;
+
+ // Used for memory instructions.
+ uint32_t addr = 0x0;
+ // Value to be written in memory.
+ uint32_t mem_value = 0x0;
+
+ // ---------- Configuration (and execution for op_regimm).
+ switch (op) {
+ // ------------- op_cop1. Coprocessor instructions.
+ case op_cop1:
+ switch (instr->rsFieldRaw()) {
+ case rs_bc1: // Branch on coprocessor condition.
+ cc = instr->fbccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ cc_value = testFCSRBit(fcsr_cc);
+ do_branch = (instr->fbtrueValue()) ? cc_value : !cc_value;
+ execute_branch_delay_instruction = true;
+ // Set next_pc.
+ if (do_branch) {
+ next_pc = current_pc + (imm16 << 2) + SimInstruction::kInstrSize;
+ } else {
+ next_pc = current_pc + kBranchReturnOffset;
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ // ------------- op_regimm class.
+ case op_regimm:
+ switch (instr->rtFieldRaw()) {
+ case rt_bltz:
+ do_branch = (rs < 0);
+ break;
+ case rt_bltzal:
+ do_branch = rs < 0;
+ break;
+ case rt_bgez:
+ do_branch = rs >= 0;
+ break;
+ case rt_bgezal:
+ do_branch = rs >= 0;
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ switch (instr->rtFieldRaw()) {
+ case rt_bltz:
+ case rt_bltzal:
+ case rt_bgez:
+ case rt_bgezal:
+ // Branch instructions common part.
+ execute_branch_delay_instruction = true;
+ // Set next_pc.
+ if (do_branch) {
+ next_pc = current_pc + (imm16 << 2) + SimInstruction::kInstrSize;
+ if (instr->isLinkingInstruction()) {
+ setRegister(31, current_pc + kBranchReturnOffset);
+ }
+ } else {
+ next_pc = current_pc + kBranchReturnOffset;
+ }
+ break;
+ default:
+ break;
+ }
+ break; // case op_regimm.
+ // ------------- Branch instructions.
+ // When comparing to zero, the encoding of rt field is always 0, so we
+ // don't need to replace rt with zero.
+ case op_beq:
+ do_branch = (rs == rt);
+ break;
+ case op_bne:
+ do_branch = rs != rt;
+ break;
+ case op_blez:
+ do_branch = rs <= 0;
+ break;
+ case op_bgtz:
+ do_branch = rs > 0;
+ break;
+ // ------------- Arithmetic instructions.
+ case op_addi:
+ if (HaveSameSign(rs, se_imm16)) {
+ if (rs > 0) {
+ exceptions[kIntegerOverflow] = rs > (kRegisterskMaxValue - se_imm16);
+ } else if (rs < 0) {
+ exceptions[kIntegerUnderflow] = rs < (kRegisterskMinValue - se_imm16);
+ }
+ }
+ alu_out = rs + se_imm16;
+ break;
+ case op_addiu:
+ alu_out = rs + se_imm16;
+ break;
+ case op_slti:
+ alu_out = (rs < se_imm16) ? 1 : 0;
+ break;
+ case op_sltiu:
+ alu_out = (rs_u < static_cast<uint32_t>(se_imm16)) ? 1 : 0;
+ break;
+ case op_andi:
+ alu_out = rs & oe_imm16;
+ break;
+ case op_ori:
+ alu_out = rs | oe_imm16;
+ break;
+ case op_xori:
+ alu_out = rs ^ oe_imm16;
+ break;
+ case op_lui:
+ alu_out = (oe_imm16 << 16);
+ break;
+ // ------------- Memory instructions.
+ case op_lb:
+ addr = rs + se_imm16;
+ alu_out = readB(addr);
+ break;
+ case op_lh:
+ addr = rs + se_imm16;
+ alu_out = readH(addr, instr);
+ break;
+ case op_lwl: {
+ // al_offset is offset of the effective address within an aligned word.
+ uint8_t al_offset = (rs + se_imm16) & kPointerAlignmentMask;
+ uint8_t byte_shift = kPointerAlignmentMask - al_offset;
+ uint32_t mask = (1 << byte_shift * 8) - 1;
+ addr = rs + se_imm16 - al_offset;
+ alu_out = readW(addr, instr);
+ alu_out <<= byte_shift * 8;
+ alu_out |= rt & mask;
+ break;
+ }
+ case op_lw:
+ addr = rs + se_imm16;
+ alu_out = readW(addr, instr);
+ break;
+ case op_lbu:
+ addr = rs + se_imm16;
+ alu_out = readBU(addr);
+ break;
+ case op_lhu:
+ addr = rs + se_imm16;
+ alu_out = readHU(addr, instr);
+ break;
+ case op_lwr: {
+ // al_offset is offset of the effective address within an aligned word.
+ uint8_t al_offset = (rs + se_imm16) & kPointerAlignmentMask;
+ uint8_t byte_shift = kPointerAlignmentMask - al_offset;
+ uint32_t mask = al_offset ? (~0 << (byte_shift + 1) * 8) : 0;
+ addr = rs + se_imm16 - al_offset;
+ alu_out = readW(addr, instr);
+ alu_out = static_cast<uint32_t>(alu_out) >> al_offset * 8;
+ alu_out |= rt & mask;
+ break;
+ }
+ case op_sb:
+ addr = rs + se_imm16;
+ break;
+ case op_sh:
+ addr = rs + se_imm16;
+ break;
+ case op_swl: {
+ uint8_t al_offset = (rs + se_imm16) & kPointerAlignmentMask;
+ uint8_t byte_shift = kPointerAlignmentMask - al_offset;
+ uint32_t mask = byte_shift ? (~0 << (al_offset + 1) * 8) : 0;
+ addr = rs + se_imm16 - al_offset;
+ mem_value = readW(addr, instr) & mask;
+ mem_value |= static_cast<uint32_t>(rt) >> byte_shift * 8;
+ break;
+ }
+ case op_sw:
+ addr = rs + se_imm16;
+ break;
+ case op_swr: {
+ uint8_t al_offset = (rs + se_imm16) & kPointerAlignmentMask;
+ uint32_t mask = (1 << al_offset * 8) - 1;
+ addr = rs + se_imm16 - al_offset;
+ mem_value = readW(addr, instr);
+ mem_value = (rt << al_offset * 8) | (mem_value & mask);
+ break;
+ }
+ case op_lwc1:
+ addr = rs + se_imm16;
+ alu_out = readW(addr, instr);
+ break;
+ case op_ldc1:
+ addr = rs + se_imm16;
+ fp_out = readD(addr, instr);
+ break;
+ case op_swc1:
+ case op_sdc1:
+ addr = rs + se_imm16;
+ break;
+ case op_ll:
+ addr = rs + se_imm16;
+ alu_out = loadLinkedW(addr, instr);
+ break;
+ case op_sc:
+ addr = rs + se_imm16;
+ alu_out = storeConditionalW(addr, rt, instr);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ // ---------- Raise exceptions triggered.
+ signalExceptions();
+
+ // ---------- Execution.
+ switch (op) {
+ // ------------- Branch instructions.
+ case op_beq:
+ case op_bne:
+ case op_blez:
+ case op_bgtz:
+ // Branch instructions common part.
+ execute_branch_delay_instruction = true;
+ // Set next_pc.
+ if (do_branch) {
+ next_pc = current_pc + (imm16 << 2) + SimInstruction::kInstrSize;
+ if (instr->isLinkingInstruction()) {
+ setRegister(31, current_pc + 2 * SimInstruction::kInstrSize);
+ }
+ } else {
+ next_pc = current_pc + 2 * SimInstruction::kInstrSize;
+ }
+ break;
+ // ------------- Arithmetic instructions.
+ case op_addi:
+ case op_addiu:
+ case op_slti:
+ case op_sltiu:
+ case op_andi:
+ case op_ori:
+ case op_xori:
+ case op_lui:
+ setRegister(rt_reg, alu_out);
+ break;
+ // ------------- Memory instructions.
+ case op_lb:
+ case op_lh:
+ case op_lwl:
+ case op_lw:
+ case op_lbu:
+ case op_lhu:
+ case op_lwr:
+ case op_ll:
+ case op_sc:
+ setRegister(rt_reg, alu_out);
+ break;
+ case op_sb:
+ writeB(addr, static_cast<int8_t>(rt));
+ break;
+ case op_sh:
+ writeH(addr, static_cast<uint16_t>(rt), instr);
+ break;
+ case op_swl:
+ writeW(addr, mem_value, instr);
+ break;
+ case op_sw:
+ writeW(addr, rt, instr);
+ break;
+ case op_swr:
+ writeW(addr, mem_value, instr);
+ break;
+ case op_lwc1:
+ setFpuRegister(ft_reg, alu_out);
+ break;
+ case op_ldc1:
+ setFpuRegisterDouble(ft_reg, fp_out);
+ break;
+ case op_swc1:
+ addr = rs + se_imm16;
+ writeW(addr, getFpuRegister(ft_reg), instr);
+ break;
+ case op_sdc1:
+ addr = rs + se_imm16;
+ writeD(addr, getFpuRegisterDouble(ft_reg), instr);
+ break;
+ default:
+ break;
+ }
+
+ if (execute_branch_delay_instruction) {
+ // Execute branch delay slot
+ // We don't check for end_sim_pc. First it should not be met as the current
+ // pc is valid. Secondly a jump should always execute its branch delay slot.
+ SimInstruction* branch_delay_instr = reinterpret_cast<SimInstruction*>(
+ current_pc + SimInstruction::kInstrSize);
+ branchDelayInstructionDecode(branch_delay_instr);
+ }
+
+ // If needed update pc after the branch delay execution.
+ if (next_pc != bad_ra) {
+ set_pc(next_pc);
+ }
+}
+
+// Type 3: instructions using a 26 bytes immediate. (e.g. j, jal).
+void Simulator::decodeTypeJump(SimInstruction* instr) {
+ // Get current pc.
+ int32_t current_pc = get_pc();
+ // Get unchanged bits of pc.
+ int32_t pc_high_bits = current_pc & 0xf0000000;
+ // Next pc.
+ int32_t next_pc = pc_high_bits | (instr->imm26Value() << 2);
+
+ // Execute branch delay slot.
+ // We don't check for end_sim_pc. First it should not be met as the current pc
+ // is valid. Secondly a jump should always execute its branch delay slot.
+ SimInstruction* branch_delay_instr = reinterpret_cast<SimInstruction*>(
+ current_pc + SimInstruction::kInstrSize);
+ branchDelayInstructionDecode(branch_delay_instr);
+
+ // Update pc and ra if necessary.
+ // Do this after the branch delay execution.
+ if (instr->isLinkingInstruction()) {
+ setRegister(31, current_pc + 2 * SimInstruction::kInstrSize);
+ }
+ set_pc(next_pc);
+ pc_modified_ = true;
+}
+
+// Executes the current instruction.
+void Simulator::instructionDecode(SimInstruction* instr) {
+ if (!SimulatorProcess::ICacheCheckingDisableCount) {
+ AutoLockSimulatorCache als;
+ SimulatorProcess::checkICacheLocked(instr);
+ }
+ pc_modified_ = false;
+
+ switch (instr->instructionType()) {
+ case SimInstruction::kRegisterType:
+ decodeTypeRegister(instr);
+ break;
+ case SimInstruction::kImmediateType:
+ decodeTypeImmediate(instr);
+ break;
+ case SimInstruction::kJumpType:
+ decodeTypeJump(instr);
+ break;
+ default:
+ UNSUPPORTED();
+ }
+ if (!pc_modified_) {
+ setRegister(pc,
+ reinterpret_cast<int32_t>(instr) + SimInstruction::kInstrSize);
+ }
+}
+
+void Simulator::branchDelayInstructionDecode(SimInstruction* instr) {
+ if (instr->instructionBits() == NopInst) {
+ // Short-cut generic nop instructions. They are always valid and they
+ // never change the simulator state.
+ return;
+ }
+
+ if (instr->isForbiddenInBranchDelay()) {
+ MOZ_CRASH("Eror:Unexpected opcode in a branch delay slot.");
+ }
+ instructionDecode(instr);
+}
+
+void Simulator::enable_single_stepping(SingleStepCallback cb, void* arg) {
+ single_stepping_ = true;
+ single_step_callback_ = cb;
+ single_step_callback_arg_ = arg;
+ single_step_callback_(single_step_callback_arg_, this, (void*)get_pc());
+}
+
+void Simulator::disable_single_stepping() {
+ if (!single_stepping_) {
+ return;
+ }
+ single_step_callback_(single_step_callback_arg_, this, (void*)get_pc());
+ single_stepping_ = false;
+ single_step_callback_ = nullptr;
+ single_step_callback_arg_ = nullptr;
+}
+
+template <bool enableStopSimAt>
+void Simulator::execute() {
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ // Get the PC to simulate. Cannot use the accessor here as we need the
+ // raw PC value and not the one used as input to arithmetic instructions.
+ int program_counter = get_pc();
+
+ while (program_counter != end_sim_pc) {
+ if (enableStopSimAt && (icount_ == Simulator::StopSimAt)) {
+ MipsDebugger dbg(this);
+ dbg.debug();
+ } else {
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this,
+ (void*)program_counter);
+ }
+ SimInstruction* instr =
+ reinterpret_cast<SimInstruction*>(program_counter);
+ instructionDecode(instr);
+ icount_++;
+ }
+ program_counter = get_pc();
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+}
+
+void Simulator::callInternal(uint8_t* entry) {
+ // Prepare to execute the code at entry.
+ setRegister(pc, reinterpret_cast<int32_t>(entry));
+ // Put down marker for end of simulation. The simulator will stop simulation
+ // when the PC reaches this value. By saving the "end simulation" value into
+ // the LR the simulation stops when returning to this call point.
+ setRegister(ra, end_sim_pc);
+
+ // Remember the values of callee-saved registers.
+ // The code below assumes that r9 is not used as sb (static base) in
+ // simulator code and therefore is regarded as a callee-saved register.
+ int32_t s0_val = getRegister(s0);
+ int32_t s1_val = getRegister(s1);
+ int32_t s2_val = getRegister(s2);
+ int32_t s3_val = getRegister(s3);
+ int32_t s4_val = getRegister(s4);
+ int32_t s5_val = getRegister(s5);
+ int32_t s6_val = getRegister(s6);
+ int32_t s7_val = getRegister(s7);
+ int32_t gp_val = getRegister(gp);
+ int32_t sp_val = getRegister(sp);
+ int32_t fp_val = getRegister(fp);
+
+ // Set up the callee-saved registers with a known value. To be able to check
+ // that they are preserved properly across JS execution.
+ int32_t callee_saved_value = icount_;
+ setRegister(s0, callee_saved_value);
+ setRegister(s1, callee_saved_value);
+ setRegister(s2, callee_saved_value);
+ setRegister(s3, callee_saved_value);
+ setRegister(s4, callee_saved_value);
+ setRegister(s5, callee_saved_value);
+ setRegister(s6, callee_saved_value);
+ setRegister(s7, callee_saved_value);
+ setRegister(gp, callee_saved_value);
+ setRegister(fp, callee_saved_value);
+
+ // Start the simulation.
+ if (Simulator::StopSimAt != -1) {
+ execute<true>();
+ } else {
+ execute<false>();
+ }
+
+ // Check that the callee-saved registers have been preserved.
+ MOZ_ASSERT(callee_saved_value == getRegister(s0));
+ MOZ_ASSERT(callee_saved_value == getRegister(s1));
+ MOZ_ASSERT(callee_saved_value == getRegister(s2));
+ MOZ_ASSERT(callee_saved_value == getRegister(s3));
+ MOZ_ASSERT(callee_saved_value == getRegister(s4));
+ MOZ_ASSERT(callee_saved_value == getRegister(s5));
+ MOZ_ASSERT(callee_saved_value == getRegister(s6));
+ MOZ_ASSERT(callee_saved_value == getRegister(s7));
+ MOZ_ASSERT(callee_saved_value == getRegister(gp));
+ MOZ_ASSERT(callee_saved_value == getRegister(fp));
+
+ // Restore callee-saved registers with the original value.
+ setRegister(s0, s0_val);
+ setRegister(s1, s1_val);
+ setRegister(s2, s2_val);
+ setRegister(s3, s3_val);
+ setRegister(s4, s4_val);
+ setRegister(s5, s5_val);
+ setRegister(s6, s6_val);
+ setRegister(s7, s7_val);
+ setRegister(gp, gp_val);
+ setRegister(sp, sp_val);
+ setRegister(fp, fp_val);
+}
+
+int32_t Simulator::call(uint8_t* entry, int argument_count, ...) {
+ va_list parameters;
+ va_start(parameters, argument_count);
+
+ int original_stack = getRegister(sp);
+ // Compute position of stack on entry to generated code.
+ int entry_stack = original_stack;
+ if (argument_count > kCArgSlotCount) {
+ entry_stack = entry_stack - argument_count * sizeof(int32_t);
+ } else {
+ entry_stack = entry_stack - kCArgsSlotsSize;
+ }
+
+ entry_stack &= ~(ABIStackAlignment - 1);
+
+ intptr_t* stack_argument = reinterpret_cast<intptr_t*>(entry_stack);
+
+ // Setup the arguments.
+ for (int i = 0; i < argument_count; i++) {
+ js::jit::Register argReg;
+ if (GetIntArgReg(i, &argReg)) {
+ setRegister(argReg.code(), va_arg(parameters, int32_t));
+ } else {
+ stack_argument[i] = va_arg(parameters, int32_t);
+ }
+ }
+
+ va_end(parameters);
+ setRegister(sp, entry_stack);
+
+ callInternal(entry);
+
+ // Pop stack passed arguments.
+ MOZ_ASSERT(entry_stack == getRegister(sp));
+ setRegister(sp, original_stack);
+
+ int32_t result = getRegister(v0);
+ return result;
+}
+
+uintptr_t Simulator::pushAddress(uintptr_t address) {
+ int new_sp = getRegister(sp) - sizeof(uintptr_t);
+ uintptr_t* stack_slot = reinterpret_cast<uintptr_t*>(new_sp);
+ *stack_slot = address;
+ setRegister(sp, new_sp);
+ return new_sp;
+}
+
+uintptr_t Simulator::popAddress() {
+ int current_sp = getRegister(sp);
+ uintptr_t* stack_slot = reinterpret_cast<uintptr_t*>(current_sp);
+ uintptr_t address = *stack_slot;
+ setRegister(sp, current_sp + sizeof(uintptr_t));
+ return address;
+}
+
+} // namespace jit
+} // namespace js
+
+js::jit::Simulator* JSContext::simulator() const { return simulator_; }
diff --git a/js/src/jit/mips32/Simulator-mips32.h b/js/src/jit/mips32/Simulator-mips32.h
new file mode 100644
index 0000000000..7ab204af54
--- /dev/null
+++ b/js/src/jit/mips32/Simulator-mips32.h
@@ -0,0 +1,526 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef jit_mips32_Simulator_mips32_h
+#define jit_mips32_Simulator_mips32_h
+
+#ifdef JS_SIMULATOR_MIPS32
+
+# include "mozilla/Atomics.h"
+
+# include "jit/IonTypes.h"
+# include "js/ProfilingFrameIterator.h"
+# include "threading/Thread.h"
+# include "vm/MutexIDs.h"
+# include "wasm/WasmSignalHandlers.h"
+
+namespace js {
+
+namespace jit {
+
+class JitActivation;
+
+class Simulator;
+class Redirection;
+class CachePage;
+class AutoLockSimulator;
+
+// When the SingleStepCallback is called, the simulator is about to execute
+// sim->get_pc() and the current machine state represents the completed
+// execution of the previous pc.
+typedef void (*SingleStepCallback)(void* arg, Simulator* sim, void* pc);
+
+const intptr_t kPointerAlignment = 4;
+const intptr_t kPointerAlignmentMask = kPointerAlignment - 1;
+
+const intptr_t kDoubleAlignment = 8;
+const intptr_t kDoubleAlignmentMask = kDoubleAlignment - 1;
+
+// Number of general purpose registers.
+const int kNumRegisters = 32;
+
+// In the simulator, the PC register is simulated as the 34th register.
+const int kPCRegister = 34;
+
+// Number coprocessor registers.
+const int kNumFPURegisters = 32;
+
+// FPU (coprocessor 1) control registers. Currently only FCSR is implemented.
+const int kFCSRRegister = 31;
+const int kInvalidFPUControlRegister = -1;
+const uint32_t kFPUInvalidResult = static_cast<uint32_t>(1 << 31) - 1;
+
+// FCSR constants.
+const uint32_t kFCSRInexactFlagBit = 2;
+const uint32_t kFCSRUnderflowFlagBit = 3;
+const uint32_t kFCSROverflowFlagBit = 4;
+const uint32_t kFCSRDivideByZeroFlagBit = 5;
+const uint32_t kFCSRInvalidOpFlagBit = 6;
+
+const uint32_t kFCSRInexactCauseBit = 12;
+const uint32_t kFCSRUnderflowCauseBit = 13;
+const uint32_t kFCSROverflowCauseBit = 14;
+const uint32_t kFCSRDivideByZeroCauseBit = 15;
+const uint32_t kFCSRInvalidOpCauseBit = 16;
+
+const uint32_t kFCSRInexactFlagMask = 1 << kFCSRInexactFlagBit;
+const uint32_t kFCSRUnderflowFlagMask = 1 << kFCSRUnderflowFlagBit;
+const uint32_t kFCSROverflowFlagMask = 1 << kFCSROverflowFlagBit;
+const uint32_t kFCSRDivideByZeroFlagMask = 1 << kFCSRDivideByZeroFlagBit;
+const uint32_t kFCSRInvalidOpFlagMask = 1 << kFCSRInvalidOpFlagBit;
+
+const uint32_t kFCSRFlagMask =
+ kFCSRInexactFlagMask | kFCSRUnderflowFlagMask | kFCSROverflowFlagMask |
+ kFCSRDivideByZeroFlagMask | kFCSRInvalidOpFlagMask;
+
+const uint32_t kFCSRExceptionFlagMask = kFCSRFlagMask ^ kFCSRInexactFlagMask;
+
+// On MIPS Simulator breakpoints can have different codes:
+// - Breaks between 0 and kMaxWatchpointCode are treated as simple watchpoints,
+// the simulator will run through them and print the registers.
+// - Breaks between kMaxWatchpointCode and kMaxStopCode are treated as stop()
+// instructions (see Assembler::stop()).
+// - Breaks larger than kMaxStopCode are simple breaks, dropping you into the
+// debugger.
+const uint32_t kMaxWatchpointCode = 31;
+const uint32_t kMaxStopCode = 127;
+const uint32_t kWasmTrapCode = 6;
+
+// -----------------------------------------------------------------------------
+// Utility functions
+
+typedef uint32_t Instr;
+class SimInstruction;
+
+// Per thread simulator state.
+class Simulator {
+ friend class MipsDebugger;
+
+ public:
+ // Registers are declared in order. See "See MIPS Run Linux" chapter 2.
+ enum Register {
+ no_reg = -1,
+ zero_reg = 0,
+ at,
+ v0,
+ v1,
+ a0,
+ a1,
+ a2,
+ a3,
+ t0,
+ t1,
+ t2,
+ t3,
+ t4,
+ t5,
+ t6,
+ t7,
+ s0,
+ s1,
+ s2,
+ s3,
+ s4,
+ s5,
+ s6,
+ s7,
+ t8,
+ t9,
+ k0,
+ k1,
+ gp,
+ sp,
+ s8,
+ ra,
+ // LO, HI, and pc.
+ LO,
+ HI,
+ pc, // pc must be the last register.
+ kNumSimuRegisters,
+ // aliases
+ fp = s8
+ };
+
+ // Coprocessor registers.
+ enum FPURegister {
+ f0,
+ f1,
+ f2,
+ f3,
+ f4,
+ f5,
+ f6,
+ f7,
+ f8,
+ f9,
+ f10,
+ f11,
+ f12,
+ f13,
+ f14,
+ f15, // f12 and f14 are arguments FPURegisters.
+ f16,
+ f17,
+ f18,
+ f19,
+ f20,
+ f21,
+ f22,
+ f23,
+ f24,
+ f25,
+ f26,
+ f27,
+ f28,
+ f29,
+ f30,
+ f31,
+ kNumFPURegisters
+ };
+
+ // Returns nullptr on OOM.
+ static Simulator* Create();
+
+ static void Destroy(Simulator* simulator);
+
+ // Constructor/destructor are for internal use only; use the static methods
+ // above.
+ Simulator();
+ ~Simulator();
+
+ // The currently executing Simulator instance. Potentially there can be one
+ // for each native thread.
+ static Simulator* Current();
+
+ static inline uintptr_t StackLimit() {
+ return Simulator::Current()->stackLimit();
+ }
+
+ uintptr_t* addressOfStackLimit();
+
+ // Accessors for register state. Reading the pc value adheres to the MIPS
+ // architecture specification and is off by a 8 from the currently executing
+ // instruction.
+ void setRegister(int reg, int32_t value);
+ int32_t getRegister(int reg) const;
+ double getDoubleFromRegisterPair(int reg);
+ // Same for FPURegisters.
+ void setFpuRegister(int fpureg, int32_t value);
+ void setFpuRegisterFloat(int fpureg, float value);
+ void setFpuRegisterFloat(int fpureg, int64_t value);
+ void setFpuRegisterDouble(int fpureg, double value);
+ void setFpuRegisterDouble(int fpureg, int64_t value);
+ int32_t getFpuRegister(int fpureg) const;
+ int64_t getFpuRegisterLong(int fpureg) const;
+ float getFpuRegisterFloat(int fpureg) const;
+ double getFpuRegisterDouble(int fpureg) const;
+ void setFCSRBit(uint32_t cc, bool value);
+ bool testFCSRBit(uint32_t cc);
+ bool setFCSRRoundError(double original, double rounded);
+
+ // Special case of set_register and get_register to access the raw PC value.
+ void set_pc(int32_t value);
+ int32_t get_pc() const;
+
+ template <typename T>
+ T get_pc_as() const {
+ return reinterpret_cast<T>(get_pc());
+ }
+
+ void enable_single_stepping(SingleStepCallback cb, void* arg);
+ void disable_single_stepping();
+
+ // Accessor to the internal simulator stack area.
+ uintptr_t stackLimit() const;
+ bool overRecursed(uintptr_t newsp = 0) const;
+ bool overRecursedWithExtra(uint32_t extra) const;
+
+ // Executes MIPS instructions until the PC reaches end_sim_pc.
+ template <bool enableStopSimAt>
+ void execute();
+
+ // Sets up the simulator state and grabs the result on return.
+ int32_t call(uint8_t* entry, int argument_count, ...);
+
+ // Push an address onto the JS stack.
+ uintptr_t pushAddress(uintptr_t address);
+
+ // Pop an address from the JS stack.
+ uintptr_t popAddress();
+
+ // Debugger input.
+ void setLastDebuggerInput(char* input);
+ char* lastDebuggerInput() { return lastDebuggerInput_; }
+
+ // Returns true if pc register contains one of the 'SpecialValues' defined
+ // below (bad_ra, end_sim_pc).
+ bool has_bad_pc() const;
+
+ private:
+ enum SpecialValues {
+ // Known bad pc value to ensure that the simulator does not execute
+ // without being properly setup.
+ bad_ra = -1,
+ // A pc value used to signal the simulator to stop execution. Generally
+ // the ra is set to this value on transition from native C code to
+ // simulated execution, so that the simulator can "return" to the native
+ // C code.
+ end_sim_pc = -2,
+ // Unpredictable value.
+ Unpredictable = 0xbadbeaf
+ };
+
+ bool init();
+
+ // Unsupported instructions use Format to print an error and stop execution.
+ void format(SimInstruction* instr, const char* format);
+
+ // Read and write memory.
+ inline uint32_t readBU(uint32_t addr);
+ inline int32_t readB(uint32_t addr);
+ inline void writeB(uint32_t addr, uint8_t value);
+ inline void writeB(uint32_t addr, int8_t value);
+
+ inline uint16_t readHU(uint32_t addr, SimInstruction* instr);
+ inline int16_t readH(uint32_t addr, SimInstruction* instr);
+ // Note: Overloaded on the sign of the value.
+ inline void writeH(uint32_t addr, uint16_t value, SimInstruction* instr);
+ inline void writeH(uint32_t addr, int16_t value, SimInstruction* instr);
+
+ inline int readW(uint32_t addr, SimInstruction* instr);
+ inline void writeW(uint32_t addr, int value, SimInstruction* instr);
+
+ inline double readD(uint32_t addr, SimInstruction* instr);
+ inline void writeD(uint32_t addr, double value, SimInstruction* instr);
+
+ inline int32_t loadLinkedW(uint32_t addr, SimInstruction* instr);
+ inline int32_t storeConditionalW(uint32_t addr, int32_t value,
+ SimInstruction* instr);
+
+ // Executing is handled based on the instruction type.
+ void decodeTypeRegister(SimInstruction* instr);
+
+ // Helper function for decodeTypeRegister.
+ void configureTypeRegister(SimInstruction* instr, int32_t& alu_out,
+ int64_t& i64hilo, uint64_t& u64hilo,
+ int32_t& next_pc, int32_t& return_addr_reg,
+ bool& do_interrupt);
+
+ void decodeTypeImmediate(SimInstruction* instr);
+ void decodeTypeJump(SimInstruction* instr);
+
+ // Used for breakpoints and traps.
+ void softwareInterrupt(SimInstruction* instr);
+
+ // Stop helper functions.
+ bool isWatchpoint(uint32_t code);
+ void printWatchpoint(uint32_t code);
+ void handleStop(uint32_t code, SimInstruction* instr);
+ bool isStopInstruction(SimInstruction* instr);
+ bool isEnabledStop(uint32_t code);
+ void enableStop(uint32_t code);
+ void disableStop(uint32_t code);
+ void increaseStopCounter(uint32_t code);
+ void printStopInfo(uint32_t code);
+
+ JS::ProfilingFrameIterator::RegisterState registerState();
+
+ // Handle any wasm faults, returning true if the fault was handled.
+ // This method is rather hot so inline the normal (no-wasm) case.
+ bool MOZ_ALWAYS_INLINE handleWasmSegFault(int32_t addr, unsigned numBytes) {
+ if (MOZ_LIKELY(!js::wasm::CodeExists)) {
+ return false;
+ }
+
+ uint8_t* newPC;
+ if (!js::wasm::MemoryAccessTraps(registerState(), (uint8_t*)addr, numBytes,
+ &newPC)) {
+ return false;
+ }
+
+ LLBit_ = false;
+ set_pc(int32_t(newPC));
+ return true;
+ }
+
+ // Executes one instruction.
+ void instructionDecode(SimInstruction* instr);
+ // Execute one instruction placed in a branch delay slot.
+ void branchDelayInstructionDecode(SimInstruction* instr);
+
+ public:
+ static int StopSimAt;
+
+ // Runtime call support.
+ static void* RedirectNativeFunction(void* nativeFunction,
+ ABIFunctionType type);
+
+ private:
+ enum Exception {
+ kNone,
+ kIntegerOverflow,
+ kIntegerUnderflow,
+ kDivideByZero,
+ kNumExceptions
+ };
+ int16_t exceptions[kNumExceptions];
+
+ // Exceptions.
+ void signalExceptions();
+
+ // Handle arguments and return value for runtime FP functions.
+ void getFpArgs(double* x, double* y, int32_t* z);
+ void getFpFromStack(int32_t* stack, double* x);
+
+ void setCallResultDouble(double result);
+ void setCallResultFloat(float result);
+ void setCallResult(int64_t res);
+
+ void callInternal(uint8_t* entry);
+
+ // Architecture state.
+ // Registers.
+ int32_t registers_[kNumSimuRegisters];
+ // Coprocessor Registers.
+ int32_t FPUregisters_[kNumFPURegisters];
+ // FPU control register.
+ uint32_t FCSR_;
+
+ bool LLBit_;
+ uint32_t LLAddr_;
+ int32_t lastLLValue_;
+
+ // Simulator support.
+ char* stack_;
+ uintptr_t stackLimit_;
+ bool pc_modified_;
+ int icount_;
+ int break_count_;
+
+ // Debugger input.
+ char* lastDebuggerInput_;
+
+ // Registered breakpoints.
+ SimInstruction* break_pc_;
+ Instr break_instr_;
+
+ // Single-stepping support
+ bool single_stepping_;
+ SingleStepCallback single_step_callback_;
+ void* single_step_callback_arg_;
+
+ // A stop is watched if its code is less than kNumOfWatchedStops.
+ // Only watched stops support enabling/disabling and the counter feature.
+ static const uint32_t kNumOfWatchedStops = 256;
+
+ // Stop is disabled if bit 31 is set.
+ static const uint32_t kStopDisabledBit = 1U << 31;
+
+ // A stop is enabled, meaning the simulator will stop when meeting the
+ // instruction, if bit 31 of watchedStops_[code].count is unset.
+ // The value watchedStops_[code].count & ~(1 << 31) indicates how many times
+ // the breakpoint was hit or gone through.
+ struct StopCountAndDesc {
+ uint32_t count_;
+ char* desc_;
+ };
+ StopCountAndDesc watchedStops_[kNumOfWatchedStops];
+};
+
+// Process wide simulator state.
+class SimulatorProcess {
+ friend class Redirection;
+ friend class AutoLockSimulatorCache;
+
+ private:
+ // ICache checking.
+ struct ICacheHasher {
+ typedef void* Key;
+ typedef void* Lookup;
+ static HashNumber hash(const Lookup& l);
+ static bool match(const Key& k, const Lookup& l);
+ };
+
+ public:
+ typedef HashMap<void*, CachePage*, ICacheHasher, SystemAllocPolicy> ICacheMap;
+
+ static mozilla::Atomic<size_t, mozilla::ReleaseAcquire>
+ ICacheCheckingDisableCount;
+ static void FlushICache(void* start, size_t size);
+
+ static void checkICacheLocked(SimInstruction* instr);
+
+ static bool initialize() {
+ singleton_ = js_new<SimulatorProcess>();
+ return singleton_;
+ }
+ static void destroy() {
+ js_delete(singleton_);
+ singleton_ = nullptr;
+ }
+
+ SimulatorProcess();
+ ~SimulatorProcess();
+
+ private:
+ static SimulatorProcess* singleton_;
+
+ // This lock creates a critical section around 'redirection_' and
+ // 'icache_', which are referenced both by the execution engine
+ // and by the off-thread compiler (see Redirection::Get in the cpp file).
+ Mutex cacheLock_ MOZ_UNANNOTATED;
+
+ Redirection* redirection_;
+ ICacheMap icache_;
+
+ public:
+ static ICacheMap& icache() {
+ // Technically we need the lock to access the innards of the
+ // icache, not to take its address, but the latter condition
+ // serves as a useful complement to the former.
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ return singleton_->icache_;
+ }
+
+ static Redirection* redirection() {
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ return singleton_->redirection_;
+ }
+
+ static void setRedirection(js::jit::Redirection* redirection) {
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ singleton_->redirection_ = redirection;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* JS_SIMULATOR_MIPS32 */
+
+#endif /* jit_mips32_Simulator_mips32_h */
diff --git a/js/src/jit/mips32/Trampoline-mips32.cpp b/js/src/jit/mips32/Trampoline-mips32.cpp
new file mode 100644
index 0000000000..1f0a5f737f
--- /dev/null
+++ b/js/src/jit/mips32/Trampoline-mips32.cpp
@@ -0,0 +1,942 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/DebugOnly.h"
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/mips-shared/SharedICHelpers-mips-shared.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+static_assert(sizeof(uintptr_t) == sizeof(uint32_t), "Not 64-bit clean.");
+
+struct EnterJITRegs {
+ double f30;
+ double f28;
+ double f26;
+ double f24;
+ double f22;
+ double f20;
+
+ // non-volatile registers.
+ uintptr_t ra;
+ uintptr_t fp;
+ uintptr_t s7;
+ uintptr_t s6;
+ uintptr_t s5;
+ uintptr_t s4;
+ uintptr_t s3;
+ uintptr_t s2;
+ uintptr_t s1;
+ uintptr_t s0;
+};
+
+struct EnterJITArgs {
+ // First 4 argumet placeholders
+ void* jitcode; // <- sp points here when function is entered.
+ int maxArgc;
+ Value* maxArgv;
+ InterpreterFrame* fp;
+
+ // Arguments on stack
+ CalleeToken calleeToken;
+ JSObject* scopeChain;
+ size_t numStackValues;
+ Value* vp;
+};
+
+static void GenerateReturn(MacroAssembler& masm, int returnCode) {
+ MOZ_ASSERT(masm.framePushed() == sizeof(EnterJITRegs));
+
+ // Restore non-volatile registers
+ masm.as_lw(s0, StackPointer, offsetof(EnterJITRegs, s0));
+ masm.as_lw(s1, StackPointer, offsetof(EnterJITRegs, s1));
+ masm.as_lw(s2, StackPointer, offsetof(EnterJITRegs, s2));
+ masm.as_lw(s3, StackPointer, offsetof(EnterJITRegs, s3));
+ masm.as_lw(s4, StackPointer, offsetof(EnterJITRegs, s4));
+ masm.as_lw(s5, StackPointer, offsetof(EnterJITRegs, s5));
+ masm.as_lw(s6, StackPointer, offsetof(EnterJITRegs, s6));
+ masm.as_lw(s7, StackPointer, offsetof(EnterJITRegs, s7));
+ masm.as_lw(fp, StackPointer, offsetof(EnterJITRegs, fp));
+ masm.as_lw(ra, StackPointer, offsetof(EnterJITRegs, ra));
+
+ // Restore non-volatile floating point registers
+ masm.as_ldc1(f20, StackPointer, offsetof(EnterJITRegs, f20));
+ masm.as_ldc1(f22, StackPointer, offsetof(EnterJITRegs, f22));
+ masm.as_ldc1(f24, StackPointer, offsetof(EnterJITRegs, f24));
+ masm.as_ldc1(f26, StackPointer, offsetof(EnterJITRegs, f26));
+ masm.as_ldc1(f28, StackPointer, offsetof(EnterJITRegs, f28));
+ masm.as_ldc1(f30, StackPointer, offsetof(EnterJITRegs, f30));
+
+ masm.freeStack(sizeof(EnterJITRegs));
+
+ masm.branch(ra);
+}
+
+static void GeneratePrologue(MacroAssembler& masm) {
+ // Save non-volatile registers. These must be saved by the trampoline,
+ // rather than the JIT'd code, because they are scanned by the conservative
+ // scanner.
+ masm.reserveStack(sizeof(EnterJITRegs));
+ masm.as_sw(s0, StackPointer, offsetof(EnterJITRegs, s0));
+ masm.as_sw(s1, StackPointer, offsetof(EnterJITRegs, s1));
+ masm.as_sw(s2, StackPointer, offsetof(EnterJITRegs, s2));
+ masm.as_sw(s3, StackPointer, offsetof(EnterJITRegs, s3));
+ masm.as_sw(s4, StackPointer, offsetof(EnterJITRegs, s4));
+ masm.as_sw(s5, StackPointer, offsetof(EnterJITRegs, s5));
+ masm.as_sw(s6, StackPointer, offsetof(EnterJITRegs, s6));
+ masm.as_sw(s7, StackPointer, offsetof(EnterJITRegs, s7));
+ masm.as_sw(fp, StackPointer, offsetof(EnterJITRegs, fp));
+ masm.as_sw(ra, StackPointer, offsetof(EnterJITRegs, ra));
+
+ masm.as_sdc1(f20, StackPointer, offsetof(EnterJITRegs, f20));
+ masm.as_sdc1(f22, StackPointer, offsetof(EnterJITRegs, f22));
+ masm.as_sdc1(f24, StackPointer, offsetof(EnterJITRegs, f24));
+ masm.as_sdc1(f26, StackPointer, offsetof(EnterJITRegs, f26));
+ masm.as_sdc1(f28, StackPointer, offsetof(EnterJITRegs, f28));
+ masm.as_sdc1(f30, StackPointer, offsetof(EnterJITRegs, f30));
+}
+
+/*
+ * This method generates a trampoline for a c++ function with the following
+ * signature:
+ * void enter(void* code, int argc, Value* argv, InterpreterFrame* fp,
+ * CalleeToken calleeToken, JSObject* scopeChain, Value* vp)
+ * ...using standard EABI calling convention
+ */
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ const Register reg_code = a0;
+ const Register reg_argc = a1;
+ const Register reg_argv = a2;
+ const mozilla::DebugOnly<Register> reg_frame = a3;
+
+ MOZ_ASSERT(OsrFrameReg == reg_frame);
+
+ GeneratePrologue(masm);
+
+ const Address slotToken(
+ sp, sizeof(EnterJITRegs) + offsetof(EnterJITArgs, calleeToken));
+ const Address slotVp(sp, sizeof(EnterJITRegs) + offsetof(EnterJITArgs, vp));
+
+ // Save stack pointer into s4
+ masm.movePtr(StackPointer, s4);
+
+ // Load calleeToken into s2.
+ masm.loadPtr(slotToken, s2);
+
+ // Save stack pointer as baseline frame.
+ masm.movePtr(StackPointer, FramePointer);
+
+ // Load the number of actual arguments into s3.
+ masm.loadPtr(slotVp, s3);
+ masm.unboxInt32(Address(s3, 0), s3);
+
+ /***************************************************************
+ Loop over argv vector, push arguments onto stack in reverse order
+ ***************************************************************/
+
+ // if we are constructing, that also needs to include newTarget
+ {
+ Label noNewTarget;
+ masm.branchTest32(Assembler::Zero, s2,
+ Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+ masm.add32(Imm32(1), reg_argc);
+
+ masm.bind(&noNewTarget);
+ }
+
+ masm.as_sll(s0, reg_argc, 3); // s0 = argc * 8
+ masm.addPtr(reg_argv, s0); // s0 = argv + argc * 8
+
+ // Loop over arguments, copying them from an unknown buffer onto the Ion
+ // stack so they can be accessed from JIT'ed code.
+ Label header, footer;
+ // If there aren't any arguments, don't do anything
+ masm.ma_b(s0, reg_argv, &footer, Assembler::BelowOrEqual, ShortJump);
+ {
+ masm.bind(&header);
+
+ masm.subPtr(Imm32(2 * sizeof(uintptr_t)), s0);
+ masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+
+ ValueOperand value = ValueOperand(s6, s7);
+ masm.loadValue(Address(s0, 0), value);
+ masm.storeValue(value, Address(StackPointer, 0));
+
+ masm.ma_b(s0, reg_argv, &header, Assembler::Above, ShortJump);
+ }
+ masm.bind(&footer);
+
+ // Create the frame descriptor.
+ masm.subPtr(StackPointer, s4);
+ masm.makeFrameDescriptor(s4, FrameType::CppToJSJit, JitFrameLayout::Size());
+
+ masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ masm.storePtr(s3,
+ Address(StackPointer, sizeof(uintptr_t))); // actual arguments
+ masm.storePtr(s2, Address(StackPointer, 0)); // callee token
+
+ masm.push(s4); // descriptor
+
+ CodeLabel returnLabel;
+ Label oomReturnLabel;
+ {
+ // Handle Interpreter -> Baseline OSR.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.take(OsrFrameReg);
+ regs.take(reg_code);
+ regs.take(ReturnReg);
+
+ const Address slotNumStackValues(
+ FramePointer,
+ sizeof(EnterJITRegs) + offsetof(EnterJITArgs, numStackValues));
+ const Address slotScopeChain(
+ FramePointer,
+ sizeof(EnterJITRegs) + offsetof(EnterJITArgs, scopeChain));
+
+ Label notOsr;
+ masm.ma_b(OsrFrameReg, OsrFrameReg, &notOsr, Assembler::Zero, ShortJump);
+
+ Register scratch = regs.takeAny();
+
+ Register numStackValues = regs.takeAny();
+ masm.load32(slotNumStackValues, numStackValues);
+
+ // Push return address.
+ masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ masm.ma_li(scratch, &returnLabel);
+ masm.storePtr(scratch, Address(StackPointer, 0));
+
+ // Push previous frame pointer.
+ masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ masm.storePtr(FramePointer, Address(StackPointer, 0));
+
+ // Reserve frame.
+ Register framePtr = FramePointer;
+ masm.subPtr(Imm32(BaselineFrame::Size()), StackPointer);
+ masm.movePtr(StackPointer, framePtr);
+
+ // Reserve space for locals and stack values.
+ masm.ma_sll(scratch, numStackValues, Imm32(3));
+ masm.subPtr(scratch, StackPointer);
+
+ // Enter exit frame.
+ masm.addPtr(
+ Imm32(BaselineFrame::Size() + BaselineFrame::FramePointerOffset),
+ scratch);
+ masm.makeFrameDescriptor(scratch, FrameType::BaselineJS,
+ ExitFrameLayout::Size());
+
+ // Push frame descriptor and fake return address.
+ masm.reserveStack(2 * sizeof(uintptr_t));
+ masm.storePtr(
+ scratch, Address(StackPointer, sizeof(uintptr_t))); // Frame descriptor
+ masm.storePtr(zero, Address(StackPointer, 0)); // fake return address
+
+ // No GC things to mark, push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ masm.reserveStack(2 * sizeof(uintptr_t));
+ masm.storePtr(framePtr,
+ Address(StackPointer, sizeof(uintptr_t))); // BaselineFrame
+ masm.storePtr(reg_code, Address(StackPointer, 0)); // jitcode
+
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(FramePointer); // BaselineFrame
+ masm.passABIArg(OsrFrameReg); // InterpreterFrame
+ masm.passABIArg(numStackValues);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ regs.add(OsrFrameReg);
+ regs.take(JSReturnOperand);
+ Register jitcode = regs.takeAny();
+ masm.loadPtr(Address(StackPointer, 0), jitcode);
+ masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), framePtr);
+ masm.freeStack(2 * sizeof(uintptr_t));
+
+ Label error;
+ masm.freeStack(ExitFrameLayout::SizeWithFooter());
+ masm.addPtr(Imm32(BaselineFrame::Size()), framePtr);
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ Register realFramePtr = numStackValues;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.ma_addu(realFramePtr, framePtr, Imm32(sizeof(void*)));
+ masm.profilerEnterFrame(realFramePtr, scratch);
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+ masm.jump(jitcode);
+
+ // OOM: load error value, discard return address and previous frame
+ // pointer and return.
+ masm.bind(&error);
+ masm.movePtr(framePtr, StackPointer);
+ masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.jump(&oomReturnLabel);
+
+ masm.bind(&notOsr);
+ // Load the scope chain in R1.
+ MOZ_ASSERT(R1.scratchReg() != reg_code);
+ masm.loadPtr(slotScopeChain, R1.scratchReg());
+ }
+
+ // The call will push the return address on the stack, thus we check that
+ // the stack would be aligned once the call is complete.
+ masm.assertStackAlignment(JitStackAlignment, sizeof(uintptr_t));
+
+ // Call the function with pushing return address to stack.
+ masm.callJitNoProfiler(reg_code);
+
+ {
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&returnLabel);
+ masm.addCodeLabel(returnLabel);
+ masm.bind(&oomReturnLabel);
+ }
+
+ // s0 <- 8*argc (size of all arguments we pushed on the stack)
+ masm.pop(s0);
+ masm.rshiftPtr(Imm32(FRAMESIZE_SHIFT), s0);
+
+ // Discard calleeToken, numActualArgs.
+ masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+
+ // Pop arguments off the stack.
+ masm.addPtr(s0, StackPointer);
+
+ // Store the returned value into the slotVp
+ masm.loadPtr(slotVp, s1);
+ masm.storeValue(JSReturnOperand, Address(s1, 0));
+
+ // Restore non-volatile registers and return.
+ GenerateReturn(masm, ShortJump);
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ // Not supported, or not implemented yet.
+ // TODO: Implement along with the corresponding stack-walker changes, in
+ // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
+ return mozilla::Nothing{};
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // NOTE: Members ionScript_ and osiPointReturnAddress_ of
+ // InvalidationBailoutStack are already on the stack.
+ static const uint32_t STACK_DATA_SIZE =
+ sizeof(InvalidationBailoutStack) - 2 * sizeof(uintptr_t);
+
+ // Stack has to be alligned here. If not, we will have to fix it.
+ masm.checkStackAlignment();
+
+ // Make room for data on stack.
+ masm.subPtr(Imm32(STACK_DATA_SIZE), StackPointer);
+
+ // Save general purpose registers
+ for (uint32_t i = 0; i < Registers::Total; i++) {
+ Address address =
+ Address(StackPointer, InvalidationBailoutStack::offsetOfRegs() +
+ i * sizeof(uintptr_t));
+ masm.storePtr(Register::FromCode(i), address);
+ }
+
+ // Save floating point registers
+ // We can use as_sd because stack is alligned.
+ for (uint32_t i = 0; i < FloatRegisters::TotalDouble; i++) {
+ masm.as_sdc1(
+ FloatRegister::FromIndex(i, FloatRegister::Double), StackPointer,
+ InvalidationBailoutStack::offsetOfFpRegs() + i * sizeof(double));
+ }
+
+ // Pass pointer to InvalidationBailoutStack structure.
+ masm.movePtr(StackPointer, a0);
+
+ // Reserve place for return value and BailoutInfo pointer
+ masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ // Pass pointer to return value.
+ masm.ma_addu(a1, StackPointer, Imm32(sizeof(uintptr_t)));
+ // Pass pointer to BailoutInfo
+ masm.movePtr(StackPointer, a2);
+
+ using Fn = bool (*)(InvalidationBailoutStack * sp, size_t * frameSizeOut,
+ BaselineBailoutInfo * *info);
+ masm.setupAlignedABICall();
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.passABIArg(a2);
+ masm.callWithABI<Fn, InvalidationBailout>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.loadPtr(Address(StackPointer, 0), a2);
+ masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), a1);
+ // Remove the return address, the IonScript, the register state
+ // (InvaliationBailoutStack) and the space that was allocated for the
+ // return value.
+ masm.addPtr(Imm32(sizeof(InvalidationBailoutStack) + 2 * sizeof(uintptr_t)),
+ StackPointer);
+ // remove the space that this frame was using before the bailout
+ // (computed by InvalidationBailout)
+ masm.addPtr(a1, StackPointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind) {
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+ masm.pushReturnAddress();
+
+#error "Port changes from bug 1772506"
+
+ Register numActArgsReg = t6;
+ Register calleeTokenReg = t7;
+ Register numArgsReg = t5;
+
+ // Load the number of actual arguments into numActArgsReg
+ masm.loadPtr(
+ Address(StackPointer, RectifierFrameLayout::offsetOfNumActualArgs()),
+ numActArgsReg);
+
+ // Load the number of |undefined|s to push into t1.
+ masm.loadPtr(
+ Address(StackPointer, RectifierFrameLayout::offsetOfCalleeToken()),
+ calleeTokenReg);
+
+ // Copy the number of actual arguments into s3.
+ masm.mov(numActArgsReg, s3);
+
+ masm.mov(calleeTokenReg, numArgsReg);
+ masm.andPtr(Imm32(CalleeTokenMask), numArgsReg);
+ masm.loadFunctionArgCount(numArgsReg, numArgsReg);
+
+ masm.as_subu(t1, numArgsReg, s3);
+
+ // Get the topmost argument.
+ masm.ma_sll(t0, s3, Imm32(3)); // t0 <- nargs * 8
+ masm.as_addu(t2, sp, t0); // t2 <- sp + nargs * 8
+ masm.addPtr(Imm32(sizeof(RectifierFrameLayout)), t2);
+
+ {
+ Label notConstructing;
+
+ masm.branchTest32(Assembler::Zero, calleeTokenReg,
+ Imm32(CalleeToken_FunctionConstructing),
+ &notConstructing);
+
+ // Add sizeof(Value) to overcome |this|
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.load32(Address(t2, NUNBOX32_TYPE_OFFSET + sizeof(Value)), t0);
+ masm.store32(t0, Address(StackPointer, NUNBOX32_TYPE_OFFSET));
+ masm.load32(Address(t2, NUNBOX32_PAYLOAD_OFFSET + sizeof(Value)), t0);
+ masm.store32(t0, Address(StackPointer, NUNBOX32_PAYLOAD_OFFSET));
+
+ // Include the newly pushed newTarget value in the frame size
+ // calculated below.
+ masm.add32(Imm32(1), numArgsReg);
+
+ masm.bind(&notConstructing);
+ }
+
+ // Push undefined.
+ masm.moveValue(UndefinedValue(), ValueOperand(t3, t4));
+ {
+ Label undefLoopTop;
+ masm.bind(&undefLoopTop);
+
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.storeValue(ValueOperand(t3, t4), Address(StackPointer, 0));
+ masm.sub32(Imm32(1), t1);
+
+ masm.ma_b(t1, t1, &undefLoopTop, Assembler::NonZero, ShortJump);
+ }
+
+ // Push arguments, |nargs| + 1 times (to include |this|).
+ {
+ Label copyLoopTop, initialSkip;
+
+ masm.ma_b(&initialSkip, ShortJump);
+
+ masm.bind(&copyLoopTop);
+ masm.subPtr(Imm32(sizeof(Value)), t2);
+ masm.sub32(Imm32(1), s3);
+
+ masm.bind(&initialSkip);
+
+ MOZ_ASSERT(sizeof(Value) == 2 * sizeof(uint32_t));
+ // Read argument and push to stack.
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.load32(Address(t2, NUNBOX32_TYPE_OFFSET), t0);
+ masm.store32(t0, Address(StackPointer, NUNBOX32_TYPE_OFFSET));
+ masm.load32(Address(t2, NUNBOX32_PAYLOAD_OFFSET), t0);
+ masm.store32(t0, Address(StackPointer, NUNBOX32_PAYLOAD_OFFSET));
+
+ masm.ma_b(s3, s3, &copyLoopTop, Assembler::NonZero, ShortJump);
+ }
+
+ // translate the framesize from values into bytes
+ masm.ma_addu(t0, numArgsReg, Imm32(1));
+ masm.lshiftPtr(Imm32(3), t0);
+
+ // Construct sizeDescriptor.
+ masm.makeFrameDescriptor(t0, FrameType::Rectifier, JitFrameLayout::Size());
+
+ // Construct JitFrameLayout.
+ masm.subPtr(Imm32(3 * sizeof(uintptr_t)), StackPointer);
+ // Push actual arguments.
+ masm.storePtr(numActArgsReg, Address(StackPointer, 2 * sizeof(uintptr_t)));
+ // Push callee token.
+ masm.storePtr(calleeTokenReg, Address(StackPointer, sizeof(uintptr_t)));
+ // Push frame descriptor.
+ masm.storePtr(t0, Address(StackPointer, 0));
+
+ // Call the target function.
+ masm.andPtr(Imm32(CalleeTokenMask), calleeTokenReg);
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(calleeTokenReg, t1);
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(t1);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(calleeTokenReg, t1, &noBaselineScript);
+ masm.callJitNoProfiler(t1);
+ masm.jump(&done);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(calleeTokenReg, t1);
+ masm.callJitNoProfiler(t1);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.mov(FramePointer, StackPointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+// NOTE: Members snapshotOffset_ and padding_ of BailoutStack
+// are not stored in PushBailoutFrame().
+static const uint32_t bailoutDataSize =
+ sizeof(BailoutStack) - 2 * sizeof(uintptr_t);
+static const uint32_t bailoutInfoOutParamSize = 2 * sizeof(uintptr_t);
+
+/* There are two different stack layouts when doing bailout. They are
+ * represented via class BailoutStack.
+ *
+ * - First case is when bailout is done trough bailout table. In this case
+ * table offset is stored in $ra (look at JitRuntime::generateBailoutTable())
+ * and thunk code should save it on stack. In this case frameClassId_ cannot
+ * be NO_FRAME_SIZE_CLASS_ID. Members snapshotOffset_ and padding_ are not on
+ * the stack.
+ *
+ * - Other case is when bailout is done via out of line code (lazy bailout).
+ * In this case frame size is stored in $ra (look at
+ * CodeGeneratorMIPS::generateOutOfLineCode()) and thunk code should save it
+ * on stack. Other difference is that members snapshotOffset_ and padding_ are
+ * pushed to the stack by CodeGeneratorMIPS::visitOutOfLineBailout().
+ */
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+ // Make sure that alignment is proper.
+ masm.checkStackAlignment();
+
+ // Make room for data.
+ masm.subPtr(Imm32(bailoutDataSize), StackPointer);
+
+ // Save general purpose registers.
+ for (uint32_t i = 0; i < Registers::Total; i++) {
+ uint32_t off = BailoutStack::offsetOfRegs() + i * sizeof(uintptr_t);
+ masm.storePtr(Register::FromCode(i), Address(StackPointer, off));
+ }
+
+#ifdef ENABLE_WASM_SIMD
+ // What to do for SIMD?
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ // Save floating point registers
+ // We can use as_sdc1 because stack is alligned.
+ for (uint32_t i = 0; i < FloatRegisters::TotalDouble; i++) {
+ masm.as_sdc1(FloatRegister::FromIndex(i, FloatRegister::Double),
+ StackPointer,
+ BailoutStack::offsetOfFpRegs() + i * sizeof(double));
+ }
+
+ // Store the frameSize_ stored in ra
+ // See: JitRuntime::generateBailoutTable()
+ // See: CodeGeneratorMIPS::generateOutOfLineCode()
+ masm.storePtr(ra, Address(StackPointer, BailoutStack::offsetOfFrameSize()));
+
+#error "Code needs to be updated"
+
+ // Put pointer to BailoutStack as first argument to the Bailout()
+ masm.movePtr(StackPointer, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+ PushBailoutFrame(masm, a0);
+
+ // Put pointer to BailoutInfo
+ masm.subPtr(Imm32(bailoutInfoOutParamSize), StackPointer);
+ masm.storePtr(ImmPtr(nullptr), Address(StackPointer, 0));
+ masm.movePtr(StackPointer, a1);
+
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupAlignedABICall();
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ // Get BailoutInfo pointer
+ masm.loadPtr(Address(StackPointer, 0), a2);
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+
+ // Load frameSize from stack
+ masm.loadPtr(Address(StackPointer, bailoutInfoOutParamSize +
+ BailoutStack::offsetOfFrameSize()),
+ a1);
+
+ // Remove complete BailoutStack class and data after it
+ masm.addPtr(Imm32(sizeof(BailoutStack) + bailoutInfoOutParamSize),
+ StackPointer);
+ // Remove frame size srom stack
+ masm.addPtr(a1, StackPointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in a2.
+ masm.jump(bailoutTail);
+}
+
+JitRuntime::BailoutTable JitRuntime::generateBailoutTable(MacroAssembler& masm,
+ Label* bailoutTail,
+ uint32_t frameClass) {
+ uint32_t offset = startTrampolineCode(masm);
+
+ Label bailout;
+ for (size_t i = 0; i < BAILOUT_TABLE_SIZE; i++) {
+ // Calculate offset to the end of table
+ int32_t offset = (BAILOUT_TABLE_SIZE - i) * BAILOUT_TABLE_ENTRY_SIZE;
+
+ // We use the 'ra' as table offset later in GenerateBailoutThunk
+ masm.as_bal(BOffImm16(offset));
+ masm.nop();
+ }
+ masm.bind(&bailout);
+
+ GenerateBailoutThunk(masm, frameClass, bailoutTail);
+
+ return BailoutTable(offset, masm.currentOffset() - offset);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+ Label* bailoutTail) {
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, NO_FRAME_SIZE_CLASS_ID, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset) {
+ *wrapperOffset = startTrampolineCode(masm);
+
+ // Avoid conflicts with argument registers while discarding the result after
+ // the function call.
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert(
+ (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set should be a superset of Volatile register set.");
+
+ // The context is the first argument; a0 is the first argument register.
+ Register cxreg = a0;
+ regs.take(cxreg);
+
+ // If it isn't a tail call, then the return address needs to be saved
+ if (f.expectTailCall == NonTailCall) {
+ masm.pushReturnAddress();
+ }
+
+ // We're aligned to an exit frame, so link it up.
+ masm.loadJSContext(cxreg);
+ masm.enterExitFrame(cxreg, regs.getAny(), &f);
+
+ // Save the base of the argument set stored on the stack.
+ Register argsBase = InvalidReg;
+ if (f.explicitArgs) {
+ argsBase = t1; // Use temporary register.
+ regs.take(argsBase);
+ masm.ma_addu(argsBase, StackPointer,
+ Imm32(ExitFrameLayout::SizeWithFooter()));
+ }
+ uint32_t framePushedBeforeAlignStack = masm.framePushed();
+ masm.alignStackPointer();
+ masm.setFramePushed(0);
+
+ // Reserve space for the outparameter. Reserve sizeof(Value) for every
+ // case so that stack stays aligned.
+ uint32_t outParamSize = 0;
+ switch (f.outParam) {
+ case Type_Value:
+ outParamSize = sizeof(Value);
+ masm.reserveStack(outParamSize);
+ break;
+
+ case Type_Handle: {
+ uint32_t pushed = masm.framePushed();
+ masm.PushEmptyRooted(f.outParamRootType);
+ outParamSize = masm.framePushed() - pushed;
+ } break;
+
+ case Type_Bool:
+ case Type_Int32:
+ MOZ_ASSERT(sizeof(uintptr_t) == sizeof(uint32_t));
+ [[fallthrough]];
+ case Type_Pointer:
+ outParamSize = sizeof(uintptr_t);
+ masm.reserveStack(outParamSize);
+ break;
+
+ case Type_Double:
+ outParamSize = sizeof(double);
+ masm.reserveStack(outParamSize);
+ break;
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ uint32_t outParamOffset = 0;
+ if (f.outParam != Type_Void) {
+ // Make sure that stack is double aligned after outParam.
+ MOZ_ASSERT(outParamSize <= sizeof(double));
+ outParamOffset += sizeof(double) - outParamSize;
+ }
+ // Reserve stack for double sized args that are copied to be aligned.
+ outParamOffset += f.doubleByRefArgs() * sizeof(double);
+
+ Register doubleArgs = t0;
+ masm.reserveStack(outParamOffset);
+ masm.movePtr(StackPointer, doubleArgs);
+
+ masm.setupAlignedABICall();
+ masm.passABIArg(cxreg);
+
+ size_t argDisp = 0;
+ size_t doubleArgDisp = 0;
+
+ // Copy any arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ argDisp += sizeof(uint32_t);
+ break;
+ case VMFunctionData::DoubleByValue:
+ // Values should be passed by reference, not by value, so we
+ // assert that the argument is a double-precision float.
+ MOZ_ASSERT(f.argPassedInFloatReg(explicitArg));
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
+ argDisp += sizeof(double);
+ break;
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += sizeof(uint32_t);
+ break;
+ case VMFunctionData::DoubleByRef:
+ // Copy double sized argument to aligned place.
+ masm.ma_ldc1WordAligned(ScratchDoubleReg, argsBase, argDisp);
+ masm.as_sdc1(ScratchDoubleReg, doubleArgs, doubleArgDisp);
+ masm.passABIArg(MoveOperand(doubleArgs, doubleArgDisp,
+ MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ doubleArgDisp += sizeof(double);
+ argDisp += sizeof(double);
+ break;
+ }
+ }
+
+ MOZ_ASSERT_IF(f.outParam != Type_Void, doubleArgDisp + sizeof(double) ==
+ outParamOffset + outParamSize);
+
+ // Copy the implicit outparam, if any.
+ if (f.outParam != Type_Void) {
+ masm.passABIArg(
+ MoveOperand(doubleArgs, outParamOffset, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ }
+
+ masm.callWithABI(nativeFun, MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, v0, v0, masm.failureLabel());
+ break;
+ case Type_Bool:
+ // Called functions return bools, which are 0/false and non-zero/true
+ masm.branchIfFalseBool(v0, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ masm.freeStack(outParamOffset);
+
+ // Load the outparam and free any allocated stack.
+ switch (f.outParam) {
+ case Type_Handle:
+ masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+ break;
+
+ case Type_Value:
+ masm.loadValue(Address(StackPointer, 0), JSReturnOperand);
+ masm.freeStack(sizeof(Value));
+ break;
+
+ case Type_Int32:
+ MOZ_ASSERT(sizeof(uintptr_t) == sizeof(uint32_t));
+ [[fallthrough]];
+ case Type_Pointer:
+ masm.load32(Address(StackPointer, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ case Type_Bool:
+ masm.load8ZeroExtend(Address(StackPointer, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ case Type_Double:
+ masm.as_ldc1(ReturnDoubleReg, StackPointer, 0);
+ masm.freeStack(sizeof(double));
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ masm.restoreStackPointer();
+ masm.setFramePushed(framePushedBeforeAlignStack);
+
+ masm.leaveExitFrame();
+ masm.retn(Imm32(sizeof(ExitFrameLayout) +
+ f.explicitStackSlots() * sizeof(uintptr_t) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type) {
+ uint32_t offset = startTrampolineCode(masm);
+
+ MOZ_ASSERT(PreBarrierReg == a1);
+ Register temp1 = a0;
+ Register temp2 = a2;
+ Register temp3 = a3;
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+ &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ LiveRegisterSet save;
+ save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileMask));
+ save.add(ra);
+ masm.PushRegsInMask(save);
+
+ masm.movePtr(ImmPtr(cx->runtime()), a0);
+
+ masm.setupUnalignedABICall(a2);
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.callWithABI(JitPreWriteBarrier(type));
+
+ save.take(AnyRegister(ra));
+ masm.PopRegsInMask(save);
+ masm.ret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.abiret();
+
+ return offset;
+}
+
+void JitRuntime::generateExceptionTailStub(MacroAssembler& masm,
+ Label* profilerExitTail) {
+ exceptionTailOffset_ = startTrampolineCode(masm);
+
+ masm.bind(masm.failureLabel());
+ masm.handleFailureWithHandlerTail(profilerExitTail);
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+ Label* bailoutTail) {
+ bailoutTailOffset_ = startTrampolineCode(masm);
+ masm.bind(bailoutTail);
+
+ masm.generateBailoutTail(a1, a2);
+}
diff --git a/js/src/jit/mips64/Architecture-mips64.cpp b/js/src/jit/mips64/Architecture-mips64.cpp
new file mode 100644
index 0000000000..54ae127954
--- /dev/null
+++ b/js/src/jit/mips64/Architecture-mips64.cpp
@@ -0,0 +1,88 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips64/Architecture-mips64.h"
+
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+const char* const Registers::RegNames[] = {
+ "zero", "at", "v0", "v1", "a0", "a1", "a2", "a3", "a4", "a5", "a6",
+ "a7", "t0", "t1", "t2", "t3", "s0", "s1", "s2", "s3", "s4", "s5",
+ "s6", "s7", "t8", "t9", "k0", "k1", "gp", "sp", "fp", "ra"};
+
+const uint32_t Allocatable = 22;
+
+const Registers::SetType Registers::ArgRegMask =
+ Registers::SharedArgRegMask | (1 << a4) | (1 << a5) | (1 << a6) | (1 << a7);
+
+const Registers::SetType Registers::JSCallMask = (1 << Registers::v1);
+
+const Registers::SetType Registers::CallMask = (1 << Registers::v0);
+
+FloatRegisters::Encoding FloatRegisters::FromName(const char* name) {
+ for (size_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(Encoding(i)), name) == 0) {
+ return Encoding(i);
+ }
+ }
+
+ return Invalid;
+}
+
+FloatRegister FloatRegister::singleOverlay() const {
+ MOZ_ASSERT(!isInvalid());
+ if (kind_ == Codes::Double) {
+ return FloatRegister(reg_, Codes::Single);
+ }
+ return *this;
+}
+
+FloatRegister FloatRegister::doubleOverlay() const {
+ MOZ_ASSERT(!isInvalid());
+ if (kind_ != Codes::Double) {
+ return FloatRegister(reg_, Codes::Double);
+ }
+ return *this;
+}
+
+FloatRegisterSet FloatRegister::ReduceSetForPush(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ LiveFloatRegisterSet mod;
+ for (FloatRegisterIterator iter(s); iter.more(); ++iter) {
+ if ((*iter).isSingle()) {
+ // Even for single size registers save complete double register.
+ mod.addUnchecked((*iter).doubleOverlay());
+ } else {
+ mod.addUnchecked(*iter);
+ }
+ }
+ return mod.set();
+}
+
+uint32_t FloatRegister::GetPushSizeInBytes(const FloatRegisterSet& s) {
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ FloatRegisterSet ss = s.reduceSetForPush();
+ uint64_t bits = ss.bits();
+ // We are only pushing double registers.
+ MOZ_ASSERT((bits & 0xffffffff) == 0);
+ uint32_t ret = mozilla::CountPopulation32(bits >> 32) * sizeof(double);
+ return ret;
+}
+uint32_t FloatRegister::getRegisterDumpOffsetInBytes() {
+ return id() * sizeof(double);
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/mips64/Architecture-mips64.h b/js/src/jit/mips64/Architecture-mips64.h
new file mode 100644
index 0000000000..d3db37ea2c
--- /dev/null
+++ b/js/src/jit/mips64/Architecture-mips64.h
@@ -0,0 +1,233 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_Architecture_mips64_h
+#define jit_mips64_Architecture_mips64_h
+
+#include "mozilla/MathAlgorithms.h"
+
+#include <limits.h>
+#include <stdint.h>
+
+#include "jit/mips-shared/Architecture-mips-shared.h"
+
+#include "js/Utility.h"
+
+namespace js {
+namespace jit {
+
+// Shadow stack space is not required on MIPS64.
+static const uint32_t ShadowStackSpace = 0;
+
+// MIPS64 have 64 bit floating-point coprocessor. There are 32 double
+// precision register which can also be used as single precision registers.
+class FloatRegisters : public FloatRegistersMIPSShared {
+ public:
+ enum ContentType { Single, Double, NumTypes };
+
+ static const char* GetName(uint32_t i) {
+ MOZ_ASSERT(i < TotalPhys);
+ return FloatRegistersMIPSShared::GetName(Encoding(i));
+ }
+
+ static Encoding FromName(const char* name);
+
+ static const uint32_t Total = 32 * NumTypes;
+#ifdef MIPSR6
+ static const uint32_t Allocatable = 60;
+#else
+ static const uint32_t Allocatable = 62;
+#endif
+ // When saving all registers we only need to do is save double registers.
+ static const uint32_t TotalPhys = 32;
+
+ static_assert(sizeof(SetType) * 8 >= Total,
+ "SetType should be large enough to enumerate all registers.");
+
+ // Magic values which are used to duplicate a mask of physical register for
+ // a specific type of register. A multiplication is used to copy and shift
+ // the bits of the physical register mask.
+ static const SetType SpreadSingle = SetType(1)
+ << (uint32_t(Single) * TotalPhys);
+ static const SetType SpreadDouble = SetType(1)
+ << (uint32_t(Double) * TotalPhys);
+ static const SetType SpreadScalar = SpreadSingle | SpreadDouble;
+ static const SetType SpreadVector = 0;
+ static const SetType Spread = SpreadScalar | SpreadVector;
+
+ static const SetType AllPhysMask = ((SetType(1) << TotalPhys) - 1);
+ static const SetType AllMask = AllPhysMask * Spread;
+ static const SetType AllSingleMask = AllPhysMask * SpreadSingle;
+ static const SetType AllDoubleMask = AllPhysMask * SpreadDouble;
+
+ static const SetType NonVolatileMask =
+ ((1U << FloatRegisters::f24) | (1U << FloatRegisters::f25) |
+ (1U << FloatRegisters::f26) | (1U << FloatRegisters::f27) |
+ (1U << FloatRegisters::f28) | (1U << FloatRegisters::f29) |
+ (1U << FloatRegisters::f30) | (1U << FloatRegisters::f31)) *
+ SpreadScalar |
+ AllPhysMask * SpreadVector;
+
+ static const SetType VolatileMask = AllMask & ~NonVolatileMask;
+
+ static const SetType WrapperMask = VolatileMask;
+
+#ifdef MIPSR6
+ static const SetType NonAllocatableMask =
+ ((1U << FloatRegisters::f23) | (1U << FloatRegisters::f24)) * Spread;
+#else
+ static const SetType NonAllocatableMask =
+ (1U << FloatRegisters::f23) * Spread;
+#endif
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+};
+
+template <typename T>
+class TypedRegisterSet;
+
+class FloatRegister : public FloatRegisterMIPSShared {
+ public:
+ typedef FloatRegisters Codes;
+ typedef size_t Code;
+ typedef Codes::Encoding Encoding;
+ typedef Codes::ContentType ContentType;
+
+ Encoding reg_ : 6;
+
+ private:
+ ContentType kind_ : 3;
+
+ public:
+ constexpr FloatRegister(uint32_t r, ContentType kind = Codes::Double)
+ : reg_(Encoding(r)), kind_(kind) {}
+ constexpr FloatRegister()
+ : reg_(Encoding(FloatRegisters::invalid_freg)), kind_(Codes::Double) {}
+
+ static uint32_t SetSize(SetType x) {
+ // Count the number of non-aliased registers.
+ x |= x >> Codes::TotalPhys;
+ x &= Codes::AllPhysMask;
+ static_assert(Codes::AllPhysMask <= 0xffffffff,
+ "We can safely use CountPopulation32");
+ return mozilla::CountPopulation32(x);
+ }
+
+ bool operator==(const FloatRegister& other) const {
+ MOZ_ASSERT(!isInvalid());
+ MOZ_ASSERT(!other.isInvalid());
+ return kind_ == other.kind_ && reg_ == other.reg_;
+ }
+ bool equiv(const FloatRegister& other) const { return other.kind_ == kind_; }
+ size_t size() const {
+ return (kind_ == Codes::Double) ? sizeof(double) : sizeof(float);
+ }
+ // Always push doubles to maintain 8-byte stack alignment.
+ size_t pushSize() const { return sizeof(double); }
+ bool isInvalid() const { return reg_ == FloatRegisters::invalid_freg; }
+
+ bool isSingle() const { return kind_ == Codes::Single; }
+ bool isDouble() const { return kind_ == Codes::Double; }
+ bool isSimd128() const { return false; }
+
+ FloatRegister singleOverlay() const;
+ FloatRegister doubleOverlay() const;
+
+ FloatRegister asSingle() const { return singleOverlay(); }
+ FloatRegister asDouble() const { return doubleOverlay(); }
+ FloatRegister asSimd128() const { MOZ_CRASH("NYI"); }
+
+ Code code() const {
+ MOZ_ASSERT(!isInvalid());
+ return Code(reg_ | (kind_ << 5));
+ }
+ Encoding encoding() const {
+ MOZ_ASSERT(!isInvalid());
+ MOZ_ASSERT(uint32_t(reg_) < Codes::TotalPhys);
+ return reg_;
+ }
+ uint32_t id() const { return reg_; }
+ static FloatRegister FromCode(uint32_t i) {
+ uint32_t code = i & 0x1f;
+ uint32_t kind = i >> 5;
+ return FloatRegister(Code(code), ContentType(kind));
+ }
+
+ bool volatile_() const {
+ return !!((1 << reg_) & FloatRegisters::VolatileMask);
+ }
+ const char* name() const { return FloatRegisters::GetName(reg_); }
+ bool operator!=(const FloatRegister& other) const {
+ return kind_ != other.kind_ || reg_ != other.reg_;
+ }
+ bool aliases(const FloatRegister& other) { return reg_ == other.reg_; }
+ uint32_t numAliased() const { return 2; }
+ FloatRegister aliased(uint32_t aliasIdx) {
+ if (aliasIdx == 0) {
+ return *this;
+ }
+ MOZ_ASSERT(aliasIdx == 1);
+ if (isDouble()) {
+ return singleOverlay();
+ }
+ return doubleOverlay();
+ }
+ uint32_t numAlignedAliased() const { return 2; }
+ FloatRegister alignedAliased(uint32_t aliasIdx) {
+ MOZ_ASSERT(isDouble());
+ if (aliasIdx == 0) {
+ return *this;
+ }
+ MOZ_ASSERT(aliasIdx == 1);
+ return singleOverlay();
+ }
+
+ SetType alignedOrDominatedAliasedSet() const { return Codes::Spread << reg_; }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return LiveAsIndexableSet<Name>(s);
+ }
+
+ static Code FromName(const char* name) {
+ return FloatRegisters::FromName(name);
+ }
+ static TypedRegisterSet<FloatRegister> ReduceSetForPush(
+ const TypedRegisterSet<FloatRegister>& s);
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>& s);
+ uint32_t getRegisterDumpOffsetInBytes();
+};
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ return set & FloatRegisters::AllDoubleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Any>(SetType set) {
+ return set;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_Architecture_mips64_h */
diff --git a/js/src/jit/mips64/Assembler-mips64.cpp b/js/src/jit/mips64/Assembler-mips64.cpp
new file mode 100644
index 0000000000..bae7c14a69
--- /dev/null
+++ b/js/src/jit/mips64/Assembler-mips64.cpp
@@ -0,0 +1,371 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips64/Assembler-mips64.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/Maybe.h"
+
+#include "jit/AutoWritableJitCode.h"
+
+using mozilla::DebugOnly;
+
+using namespace js;
+using namespace js::jit;
+
+ABIArgGenerator::ABIArgGenerator()
+ : regIndex_(0), stackOffset_(0), current_() {}
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+ static_assert(NumIntArgRegs == NumFloatArgRegs);
+ if (regIndex_ == NumIntArgRegs) {
+ if (type != MIRType::Simd128) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ } else {
+ // Mips platform does not support simd yet.
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+ }
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults: {
+ Register destReg;
+ GetIntArgReg(regIndex_++, &destReg);
+ current_ = ABIArg(destReg);
+ break;
+ }
+ case MIRType::Float32:
+ case MIRType::Double: {
+ FloatRegister::ContentType contentType;
+ contentType = (type == MIRType::Double) ? FloatRegisters::Double
+ : FloatRegisters::Single;
+ FloatRegister destFReg;
+ GetFloatArgReg(regIndex_++, &destFReg);
+ current_ = ABIArg(FloatRegister(destFReg.id(), contentType));
+ break;
+ }
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+}
+
+uint32_t js::jit::RT(FloatRegister r) {
+ MOZ_ASSERT(r.id() < FloatRegisters::TotalPhys);
+ return r.id() << RTShift;
+}
+
+uint32_t js::jit::RD(FloatRegister r) {
+ MOZ_ASSERT(r.id() < FloatRegisters::TotalPhys);
+ return r.id() << RDShift;
+}
+
+uint32_t js::jit::RZ(FloatRegister r) {
+ MOZ_ASSERT(r.id() < FloatRegisters::TotalPhys);
+ return r.id() << RZShift;
+}
+
+uint32_t js::jit::SA(FloatRegister r) {
+ MOZ_ASSERT(r.id() < FloatRegisters::TotalPhys);
+ return r.id() << SAShift;
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ MOZ_ASSERT(isFinished);
+ m_buffer.executableCopy(buffer);
+}
+
+uintptr_t Assembler::GetPointer(uint8_t* instPtr) {
+ Instruction* inst = (Instruction*)instPtr;
+ return Assembler::ExtractLoad64Value(inst);
+}
+
+static JitCode* CodeFromJump(Instruction* jump) {
+ uint8_t* target = (uint8_t*)Assembler::ExtractLoad64Value(jump);
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ while (reader.more()) {
+ JitCode* child =
+ CodeFromJump((Instruction*)(code->raw() + reader.readUnsigned()));
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ }
+}
+
+static void TraceOneDataRelocation(JSTracer* trc,
+ mozilla::Maybe<AutoWritableJitCode>& awjc,
+ JitCode* code, Instruction* inst) {
+ void* ptr = (void*)Assembler::ExtractLoad64Value(inst);
+ void* prior = ptr;
+
+ // Data relocations can be for Values or for raw pointers. If a Value is
+ // zero-tagged, we can trace it as if it were a raw pointer. If a Value
+ // is not zero-tagged, we have to interpret it as a Value to ensure that the
+ // tag bits are masked off to recover the actual pointer.
+ uintptr_t word = reinterpret_cast<uintptr_t>(ptr);
+ if (word >> JSVAL_TAG_SHIFT) {
+ // This relocation is a Value with a non-zero tag.
+ Value v = Value::fromRawBits(word);
+ TraceManuallyBarrieredEdge(trc, &v, "jit-masm-value");
+ ptr = (void*)v.bitsAsPunboxPointer();
+ } else {
+ // This relocation is a raw pointer or a Value with a zero tag.
+ // No barrier needed since these are constants.
+ TraceManuallyBarrieredGenericPointerEdge(
+ trc, reinterpret_cast<gc::Cell**>(&ptr), "jit-masm-ptr");
+ }
+
+ if (ptr != prior) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ Assembler::UpdateLoad64Value(inst, uint64_t(ptr));
+ }
+}
+
+/* static */
+void Assembler::TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ Instruction* inst = (Instruction*)(code->raw() + offset);
+ TraceOneDataRelocation(trc, awjc, code, inst);
+ }
+}
+
+void Assembler::Bind(uint8_t* rawCode, const CodeLabel& label) {
+ if (label.patchAt().bound()) {
+ auto mode = label.linkMode();
+ intptr_t offset = label.patchAt().offset();
+ intptr_t target = label.target().offset();
+
+ if (mode == CodeLabel::RawPointer) {
+ *reinterpret_cast<const void**>(rawCode + offset) = rawCode + target;
+ } else {
+ MOZ_ASSERT(mode == CodeLabel::MoveImmediate ||
+ mode == CodeLabel::JumpImmediate);
+ Instruction* inst = (Instruction*)(rawCode + offset);
+ Assembler::UpdateLoad64Value(inst, (uint64_t)(rawCode + target));
+ }
+ }
+}
+
+void Assembler::bind(InstImm* inst, uintptr_t branch, uintptr_t target) {
+ int64_t offset = target - branch;
+ InstImm inst_bgezal = InstImm(op_regimm, zero, rt_bgezal, BOffImm16(0));
+ InstImm inst_beq = InstImm(op_beq, zero, zero, BOffImm16(0));
+
+ // If encoded offset is 4, then the jump must be short
+ if (BOffImm16(inst[0]).decode() == 4) {
+ MOZ_ASSERT(BOffImm16::IsInRange(offset));
+ inst[0].setBOffImm16(BOffImm16(offset));
+ inst[1].makeNop();
+ return;
+ }
+
+ // Generate the long jump for calls because return address has to be the
+ // address after the reserved block.
+ if (inst[0].encode() == inst_bgezal.encode()) {
+ addLongJump(BufferOffset(branch), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+ inst[4] = InstReg(op_special, ScratchRegister, zero, ra, ff_jalr).encode();
+ // There is 1 nop after this.
+ return;
+ }
+
+ if (BOffImm16::IsInRange(offset)) {
+ // Don't skip trailing nops can improve performance
+ // on Loongson3 platform.
+ bool skipNops =
+ !isLoongson() && (inst[0].encode() != inst_bgezal.encode() &&
+ inst[0].encode() != inst_beq.encode());
+
+ inst[0].setBOffImm16(BOffImm16(offset));
+ inst[1].makeNop();
+
+ if (skipNops) {
+ inst[2] =
+ InstImm(op_regimm, zero, rt_bgez, BOffImm16(5 * sizeof(uint32_t)))
+ .encode();
+ // There are 4 nops after this
+ }
+ return;
+ }
+
+ if (inst[0].encode() == inst_beq.encode()) {
+ // Handle long unconditional jump.
+ addLongJump(BufferOffset(branch), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+#ifdef MIPSR6
+ inst[4] =
+ InstReg(op_special, ScratchRegister, zero, zero, ff_jalr).encode();
+#else
+ inst[4] = InstReg(op_special, ScratchRegister, zero, zero, ff_jr).encode();
+#endif
+ // There is 1 nop after this.
+ } else {
+ // Handle long conditional jump.
+ inst[0] = invertBranch(inst[0], BOffImm16(7 * sizeof(uint32_t)));
+ // No need for a "nop" here because we can clobber scratch.
+ addLongJump(BufferOffset(branch + sizeof(uint32_t)), BufferOffset(target));
+ Assembler::WriteLoad64Instructions(&inst[1], ScratchRegister,
+ LabelBase::INVALID_OFFSET);
+#ifdef MIPSR6
+ inst[5] =
+ InstReg(op_special, ScratchRegister, zero, zero, ff_jalr).encode();
+#else
+ inst[5] = InstReg(op_special, ScratchRegister, zero, zero, ff_jr).encode();
+#endif
+ // There is 1 nop after this.
+ }
+}
+
+void Assembler::processCodeLabels(uint8_t* rawCode) {
+ for (const CodeLabel& label : codeLabels_) {
+ Bind(rawCode, label);
+ }
+}
+
+uint32_t Assembler::PatchWrite_NearCallSize() {
+ // Load an address needs 4 instructions, and a jump with a delay slot.
+ return (4 + 2) * sizeof(uint32_t);
+}
+
+void Assembler::PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall) {
+ Instruction* inst = (Instruction*)start.raw();
+ uint8_t* dest = toCall.raw();
+
+ // Overwrite whatever instruction used to be here with a call.
+ // Always use long jump for two reasons:
+ // - Jump has to be the same size because of PatchWrite_NearCallSize.
+ // - Return address has to be at the end of replaced block.
+ // Short jump wouldn't be more efficient.
+ Assembler::WriteLoad64Instructions(inst, ScratchRegister, (uint64_t)dest);
+ inst[4] = InstReg(op_special, ScratchRegister, zero, ra, ff_jalr);
+ inst[5] = InstNOP();
+}
+
+uint64_t Assembler::ExtractLoad64Value(Instruction* inst0) {
+ InstImm* i0 = (InstImm*)inst0;
+ InstImm* i1 = (InstImm*)i0->next();
+ InstReg* i2 = (InstReg*)i1->next();
+ InstImm* i3 = (InstImm*)i2->next();
+ InstImm* i5 = (InstImm*)i3->next()->next();
+
+ MOZ_ASSERT(i0->extractOpcode() == ((uint32_t)op_lui >> OpcodeShift));
+ MOZ_ASSERT(i1->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+ MOZ_ASSERT(i3->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ if ((i2->extractOpcode() == ((uint32_t)op_special >> OpcodeShift)) &&
+ (i2->extractFunctionField() == ff_dsrl32)) {
+ uint64_t value = (uint64_t(i0->extractImm16Value()) << 32) |
+ (uint64_t(i1->extractImm16Value()) << 16) |
+ uint64_t(i3->extractImm16Value());
+ return uint64_t((int64_t(value) << 16) >> 16);
+ }
+
+ MOZ_ASSERT(i5->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+ uint64_t value = (uint64_t(i0->extractImm16Value()) << 48) |
+ (uint64_t(i1->extractImm16Value()) << 32) |
+ (uint64_t(i3->extractImm16Value()) << 16) |
+ uint64_t(i5->extractImm16Value());
+ return value;
+}
+
+void Assembler::UpdateLoad64Value(Instruction* inst0, uint64_t value) {
+ InstImm* i0 = (InstImm*)inst0;
+ InstImm* i1 = (InstImm*)i0->next();
+ InstReg* i2 = (InstReg*)i1->next();
+ InstImm* i3 = (InstImm*)i2->next();
+ InstImm* i5 = (InstImm*)i3->next()->next();
+
+ MOZ_ASSERT(i0->extractOpcode() == ((uint32_t)op_lui >> OpcodeShift));
+ MOZ_ASSERT(i1->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+ MOZ_ASSERT(i3->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ if ((i2->extractOpcode() == ((uint32_t)op_special >> OpcodeShift)) &&
+ (i2->extractFunctionField() == ff_dsrl32)) {
+ i0->setImm16(Imm16::Lower(Imm32(value >> 32)));
+ i1->setImm16(Imm16::Upper(Imm32(value)));
+ i3->setImm16(Imm16::Lower(Imm32(value)));
+ return;
+ }
+
+ MOZ_ASSERT(i5->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ i0->setImm16(Imm16::Upper(Imm32(value >> 32)));
+ i1->setImm16(Imm16::Lower(Imm32(value >> 32)));
+ i3->setImm16(Imm16::Upper(Imm32(value)));
+ i5->setImm16(Imm16::Lower(Imm32(value)));
+}
+
+void Assembler::WriteLoad64Instructions(Instruction* inst0, Register reg,
+ uint64_t value) {
+ Instruction* inst1 = inst0->next();
+ Instruction* inst2 = inst1->next();
+ Instruction* inst3 = inst2->next();
+
+ *inst0 = InstImm(op_lui, zero, reg, Imm16::Lower(Imm32(value >> 32)));
+ *inst1 = InstImm(op_ori, reg, reg, Imm16::Upper(Imm32(value)));
+ *inst2 = InstReg(op_special, rs_one, reg, reg, 48 - 32, ff_dsrl32);
+ *inst3 = InstImm(op_ori, reg, reg, Imm16::Lower(Imm32(value)));
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ ImmPtr newValue, ImmPtr expectedValue) {
+ PatchDataWithValueCheck(label, PatchedImmPtr(newValue.value),
+ PatchedImmPtr(expectedValue.value));
+}
+
+void Assembler::PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue) {
+ Instruction* inst = (Instruction*)label.raw();
+
+ // Extract old Value
+ DebugOnly<uint64_t> value = Assembler::ExtractLoad64Value(inst);
+ MOZ_ASSERT(value == uint64_t(expectedValue.value));
+
+ // Replace with new value
+ Assembler::UpdateLoad64Value(inst, uint64_t(newValue.value));
+}
+
+uint64_t Assembler::ExtractInstructionImmediate(uint8_t* code) {
+ InstImm* inst = (InstImm*)code;
+ return Assembler::ExtractLoad64Value(inst);
+}
+
+void Assembler::ToggleCall(CodeLocationLabel inst_, bool enabled) {
+ Instruction* inst = (Instruction*)inst_.raw();
+ InstImm* i0 = (InstImm*)inst;
+ InstImm* i1 = (InstImm*)i0->next();
+ InstImm* i3 = (InstImm*)i1->next()->next();
+ Instruction* i4 = (Instruction*)i3->next();
+
+ MOZ_ASSERT(i0->extractOpcode() == ((uint32_t)op_lui >> OpcodeShift));
+ MOZ_ASSERT(i1->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+ MOZ_ASSERT(i3->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ if (enabled) {
+ MOZ_ASSERT(i4->extractOpcode() != ((uint32_t)op_lui >> OpcodeShift));
+ InstReg jalr = InstReg(op_special, ScratchRegister, zero, ra, ff_jalr);
+ *i4 = jalr;
+ } else {
+ InstNOP nop;
+ *i4 = nop;
+ }
+}
diff --git a/js/src/jit/mips64/Assembler-mips64.h b/js/src/jit/mips64/Assembler-mips64.h
new file mode 100644
index 0000000000..7a51f12407
--- /dev/null
+++ b/js/src/jit/mips64/Assembler-mips64.h
@@ -0,0 +1,288 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_Assembler_mips64_h
+#define jit_mips64_Assembler_mips64_h
+
+#include <iterator>
+
+#include "jit/mips-shared/Assembler-mips-shared.h"
+
+#include "jit/mips64/Architecture-mips64.h"
+
+namespace js {
+namespace jit {
+
+static constexpr Register CallTempReg4 = a4;
+static constexpr Register CallTempReg5 = a5;
+
+static constexpr Register CallTempNonArgRegs[] = {t0, t1, t2, t3};
+static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+class ABIArgGenerator {
+ unsigned regIndex_;
+ uint32_t stackOffset_;
+ ABIArg current_;
+
+ public:
+ ABIArgGenerator();
+ ABIArg next(MIRType argType);
+ ABIArg& current() { return current_; }
+
+ uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+ void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+};
+
+// These registers may be volatile or nonvolatile.
+static constexpr Register ABINonArgReg0 = t0;
+static constexpr Register ABINonArgReg1 = t1;
+static constexpr Register ABINonArgReg2 = t2;
+static constexpr Register ABINonArgReg3 = t3;
+
+// This register may be volatile or nonvolatile. Avoid f23 which is the
+// ScratchDoubleReg.
+static constexpr FloatRegister ABINonArgDoubleReg{FloatRegisters::f21,
+ FloatRegisters::Double};
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = t0;
+static constexpr Register ABINonArgReturnReg1 = t1;
+static constexpr Register ABINonVolatileReg = s0;
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = t0;
+
+// TLS pointer argument register for WebAssembly functions. This must not alias
+// any other register used for passing function arguments or return values.
+// Preserved by WebAssembly functions.
+static constexpr Register InstanceReg = s5;
+
+// Registers used for wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = t1;
+
+static constexpr Register InterpreterPCReg = t5;
+
+static constexpr Register JSReturnReg = v1;
+static constexpr Register JSReturnReg_Type = JSReturnReg;
+static constexpr Register JSReturnReg_Data = JSReturnReg;
+static constexpr Register64 ReturnReg64(ReturnReg);
+static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::f0,
+ FloatRegisters::Single};
+static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::f0,
+ FloatRegisters::Double};
+static constexpr FloatRegister ScratchFloat32Reg = {FloatRegisters::f23,
+ FloatRegisters::Single};
+static constexpr FloatRegister ScratchDoubleReg = {FloatRegisters::f23,
+ FloatRegisters::Double};
+
+struct ScratchFloat32Scope : public AutoFloatRegisterScope {
+ explicit ScratchFloat32Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchFloat32Reg) {}
+};
+
+struct ScratchDoubleScope : public AutoFloatRegisterScope {
+ explicit ScratchDoubleScope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchDoubleReg) {}
+};
+
+static constexpr FloatRegister f0 = {FloatRegisters::f0,
+ FloatRegisters::Double};
+static constexpr FloatRegister f1 = {FloatRegisters::f1,
+ FloatRegisters::Double};
+static constexpr FloatRegister f2 = {FloatRegisters::f2,
+ FloatRegisters::Double};
+static constexpr FloatRegister f3 = {FloatRegisters::f3,
+ FloatRegisters::Double};
+static constexpr FloatRegister f4 = {FloatRegisters::f4,
+ FloatRegisters::Double};
+static constexpr FloatRegister f5 = {FloatRegisters::f5,
+ FloatRegisters::Double};
+static constexpr FloatRegister f6 = {FloatRegisters::f6,
+ FloatRegisters::Double};
+static constexpr FloatRegister f7 = {FloatRegisters::f7,
+ FloatRegisters::Double};
+static constexpr FloatRegister f8 = {FloatRegisters::f8,
+ FloatRegisters::Double};
+static constexpr FloatRegister f9 = {FloatRegisters::f9,
+ FloatRegisters::Double};
+static constexpr FloatRegister f10 = {FloatRegisters::f10,
+ FloatRegisters::Double};
+static constexpr FloatRegister f11 = {FloatRegisters::f11,
+ FloatRegisters::Double};
+static constexpr FloatRegister f12 = {FloatRegisters::f12,
+ FloatRegisters::Double};
+static constexpr FloatRegister f13 = {FloatRegisters::f13,
+ FloatRegisters::Double};
+static constexpr FloatRegister f14 = {FloatRegisters::f14,
+ FloatRegisters::Double};
+static constexpr FloatRegister f15 = {FloatRegisters::f15,
+ FloatRegisters::Double};
+static constexpr FloatRegister f16 = {FloatRegisters::f16,
+ FloatRegisters::Double};
+static constexpr FloatRegister f17 = {FloatRegisters::f17,
+ FloatRegisters::Double};
+static constexpr FloatRegister f18 = {FloatRegisters::f18,
+ FloatRegisters::Double};
+static constexpr FloatRegister f19 = {FloatRegisters::f19,
+ FloatRegisters::Double};
+static constexpr FloatRegister f20 = {FloatRegisters::f20,
+ FloatRegisters::Double};
+static constexpr FloatRegister f21 = {FloatRegisters::f21,
+ FloatRegisters::Double};
+static constexpr FloatRegister f22 = {FloatRegisters::f22,
+ FloatRegisters::Double};
+static constexpr FloatRegister f23 = {FloatRegisters::f23,
+ FloatRegisters::Double};
+static constexpr FloatRegister f24 = {FloatRegisters::f24,
+ FloatRegisters::Double};
+static constexpr FloatRegister f25 = {FloatRegisters::f25,
+ FloatRegisters::Double};
+static constexpr FloatRegister f26 = {FloatRegisters::f26,
+ FloatRegisters::Double};
+static constexpr FloatRegister f27 = {FloatRegisters::f27,
+ FloatRegisters::Double};
+static constexpr FloatRegister f28 = {FloatRegisters::f28,
+ FloatRegisters::Double};
+static constexpr FloatRegister f29 = {FloatRegisters::f29,
+ FloatRegisters::Double};
+static constexpr FloatRegister f30 = {FloatRegisters::f30,
+ FloatRegisters::Double};
+static constexpr FloatRegister f31 = {FloatRegisters::f31,
+ FloatRegisters::Double};
+
+// MIPS64 CPUs can only load multibyte data that is "naturally"
+// eight-byte-aligned, sp register should be sixteen-byte-aligned.
+static constexpr uint32_t ABIStackAlignment = 16;
+static constexpr uint32_t JitStackAlignment = 16;
+
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+ JitStackValueAlignment >= 1,
+ "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+// TODO this is just a filler to prevent a build failure. The MIPS SIMD
+// alignment requirements still need to be explored.
+// TODO Copy the static_asserts from x64/x86 assembler files.
+static constexpr uint32_t SimdMemoryAlignment = 16;
+
+static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static const uint32_t WasmTrapInstructionLength = 4;
+
+// See comments in wasm::GenerateFunctionPrologue. The difference between these
+// is the size of the largest callable prologue on the platform.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+static constexpr Scale ScalePointer = TimesEight;
+
+class Assembler : public AssemblerMIPSShared {
+ public:
+ Assembler() : AssemblerMIPSShared() {}
+
+ static uintptr_t GetPointer(uint8_t*);
+
+ using AssemblerMIPSShared::bind;
+
+ static void Bind(uint8_t* rawCode, const CodeLabel& label);
+
+ void processCodeLabels(uint8_t* rawCode);
+
+ static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+ static void TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ void bind(InstImm* inst, uintptr_t branch, uintptr_t target);
+
+ // Copy the assembly code to the given buffer, and perform any pending
+ // relocations relying on the target address.
+ void executableCopy(uint8_t* buffer);
+
+ static uint32_t PatchWrite_NearCallSize();
+
+ static uint64_t ExtractLoad64Value(Instruction* inst0);
+ static void UpdateLoad64Value(Instruction* inst0, uint64_t value);
+ static void WriteLoad64Instructions(Instruction* inst0, Register reg,
+ uint64_t value);
+
+ static void PatchWrite_NearCall(CodeLocationLabel start,
+ CodeLocationLabel toCall);
+ static void PatchDataWithValueCheck(CodeLocationLabel label, ImmPtr newValue,
+ ImmPtr expectedValue);
+ static void PatchDataWithValueCheck(CodeLocationLabel label,
+ PatchedImmPtr newValue,
+ PatchedImmPtr expectedValue);
+
+ static uint64_t ExtractInstructionImmediate(uint8_t* code);
+
+ static void ToggleCall(CodeLocationLabel inst_, bool enabled);
+}; // Assembler
+
+static const uint32_t NumIntArgRegs = 8;
+static const uint32_t NumFloatArgRegs = NumIntArgRegs;
+
+static inline bool GetIntArgReg(uint32_t usedArgSlots, Register* out) {
+ if (usedArgSlots < NumIntArgRegs) {
+ *out = Register::FromCode(a0.code() + usedArgSlots);
+ return true;
+ }
+ return false;
+}
+
+static inline bool GetFloatArgReg(uint32_t usedArgSlots, FloatRegister* out) {
+ if (usedArgSlots < NumFloatArgRegs) {
+ *out = FloatRegister::FromCode(f12.code() + usedArgSlots);
+ return true;
+ }
+ return false;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs, Register* out) {
+ // NOTE: We can't properly determine which regs are used if there are
+ // float arguments. If this is needed, we will have to guess.
+ MOZ_ASSERT(usedFloatArgs == 0);
+
+ if (GetIntArgReg(usedIntArgs, out)) {
+ return true;
+ }
+ // Unfortunately, we have to assume things about the point at which
+ // GetIntArgReg returns false, because we need to know how many registers it
+ // can allocate.
+ usedIntArgs -= NumIntArgRegs;
+ if (usedIntArgs >= NumCallTempNonArgRegs) {
+ return false;
+ }
+ *out = CallTempNonArgRegs[usedIntArgs];
+ return true;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_Assembler_mips64_h */
diff --git a/js/src/jit/mips64/CodeGenerator-mips64.cpp b/js/src/jit/mips64/CodeGenerator-mips64.cpp
new file mode 100644
index 0000000000..052faa36c4
--- /dev/null
+++ b/js/src/jit/mips64/CodeGenerator-mips64.cpp
@@ -0,0 +1,574 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips64/CodeGenerator-mips64.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "js/Conversions.h"
+#include "vm/Shape.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+ValueOperand CodeGeneratorMIPS64::ToValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getOperand(pos)));
+}
+
+ValueOperand CodeGeneratorMIPS64::ToTempValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getTemp(pos)));
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+ const LAllocation* in = box->getOperand(0);
+ ValueOperand result = ToOutValue(box);
+
+ masm.moveValue(TypedOrValueRegister(box->type(), ToAnyRegister(in)), result);
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+ MUnbox* mir = unbox->mir();
+
+ Register result = ToRegister(unbox->output());
+
+ if (mir->fallible()) {
+ const ValueOperand value = ToValue(unbox, LUnbox::Input);
+ Label bail;
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.fallibleUnboxInt32(value, result, &bail);
+ break;
+ case MIRType::Boolean:
+ masm.fallibleUnboxBoolean(value, result, &bail);
+ break;
+ case MIRType::Object:
+ masm.fallibleUnboxObject(value, result, &bail);
+ break;
+ case MIRType::String:
+ masm.fallibleUnboxString(value, result, &bail);
+ break;
+ case MIRType::Symbol:
+ masm.fallibleUnboxSymbol(value, result, &bail);
+ break;
+ case MIRType::BigInt:
+ masm.fallibleUnboxBigInt(value, result, &bail);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ bailoutFrom(&bail, unbox->snapshot());
+ return;
+ }
+
+ LAllocation* input = unbox->getOperand(LUnbox::Input);
+ if (input->isRegister()) {
+ Register inputReg = ToRegister(input);
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(inputReg, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(inputReg, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(inputReg, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(inputReg, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(inputReg, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(inputReg, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ return;
+ }
+
+ Address inputAddr = ToAddress(input);
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(inputAddr, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(inputAddr, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(inputAddr, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(inputAddr, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(inputAddr, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(inputAddr, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+}
+
+void CodeGeneratorMIPS64::splitTagForTest(const ValueOperand& value,
+ ScratchTagScope& tag) {
+ masm.splitTag(value.valueReg(), tag);
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+ MCompare* mir = lir->mir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ Register output = ToRegister(lir->output());
+ Register rhsReg;
+ ScratchRegisterScope scratch(masm);
+
+ if (IsConstant(rhs)) {
+ rhsReg = scratch;
+ masm.ma_li(rhsReg, ImmWord(ToInt64(rhs)));
+ } else if (rhs.value().isGeneralReg()) {
+ rhsReg = ToRegister64(rhs).reg;
+ } else {
+ rhsReg = scratch;
+ masm.loadPtr(ToAddress(rhs.value()), rhsReg);
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ masm.cmpPtrSet(JSOpToCondition(lir->jsop(), isSigned), lhsReg, rhsReg,
+ output);
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+ MCompare* mir = lir->cmpMir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ Register rhsReg;
+ ScratchRegisterScope scratch(masm);
+
+ if (IsConstant(rhs)) {
+ rhsReg = scratch;
+ masm.ma_li(rhsReg, ImmWord(ToInt64(rhs)));
+ } else if (rhs.value().isGeneralReg()) {
+ rhsReg = ToRegister64(rhs).reg;
+ } else {
+ rhsReg = scratch;
+ masm.loadPtr(ToAddress(rhs.value()), rhsReg);
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition cond = JSOpToCondition(lir->jsop(), isSigned);
+ emitBranch(lhsReg, rhsReg, cond, lir->ifTrue(), lir->ifFalse());
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ Label done;
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ // Handle an integer overflow exception from INT64_MIN / -1.
+ if (lir->canBeNegativeOverflow()) {
+ Label notOverflow;
+ masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &notOverflow);
+ masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &notOverflow);
+ if (lir->mir()->isMod()) {
+ masm.ma_xor(output, output);
+ } else {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ masm.jump(&done);
+ masm.bind(&notOverflow);
+ }
+
+#ifdef MIPSR6
+ if (lir->mir()->isMod()) {
+ masm.as_dmod(output, lhs, rhs);
+ } else {
+ masm.as_ddiv(output, lhs, rhs);
+ }
+#else
+ masm.as_ddiv(lhs, rhs);
+ if (lir->mir()->isMod()) {
+ masm.as_mfhi(output);
+ } else {
+ masm.as_mflo(output);
+ }
+#endif
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ Label done;
+
+ // Prevent divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.ma_b(rhs, rhs, &nonZero, Assembler::NonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+#ifdef MIPSR6
+ if (lir->mir()->isMod()) {
+ masm.as_dmodu(output, lhs, rhs);
+ } else {
+ masm.as_ddivu(output, lhs, rhs);
+ }
+#else
+ masm.as_ddivu(lhs, rhs);
+ if (lir->mir()->isMod()) {
+ masm.as_mfhi(output);
+ } else {
+ masm.as_mflo(output);
+ }
+#endif
+ masm.bind(&done);
+}
+
+void CodeGeneratorMIPS64::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+#ifdef MIPSR6
+ masm.as_ddiv(/* result= */ dividend, dividend, divisor);
+#else
+ masm.as_ddiv(dividend, divisor);
+ masm.as_mflo(dividend);
+#endif
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorMIPS64::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+#ifdef MIPSR6
+ masm.as_dmod(/* result= */ dividend, dividend, divisor);
+#else
+ masm.as_ddiv(dividend, divisor);
+ masm.as_mfhi(dividend);
+#endif
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+template <typename T>
+void CodeGeneratorMIPS64::emitWasmLoadI64(T* lir) {
+ const MWasmLoad* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (IsUnaligned(mir->access())) {
+ masm.wasmUnalignedLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()),
+ ptrScratch, ToOutRegister64(lir),
+ ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmLoadI64(mir->access(), HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToOutRegister64(lir));
+ }
+}
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir) {
+ emitWasmLoadI64(lir);
+}
+
+void CodeGenerator::visitWasmUnalignedLoadI64(LWasmUnalignedLoadI64* lir) {
+ emitWasmLoadI64(lir);
+}
+
+template <typename T>
+void CodeGeneratorMIPS64::emitWasmStoreI64(T* lir) {
+ const MWasmStore* mir = lir->mir();
+
+ Register ptrScratch = InvalidReg;
+ if (!lir->ptrCopy()->isBogusTemp()) {
+ ptrScratch = ToRegister(lir->ptrCopy());
+ }
+
+ if (IsUnaligned(mir->access())) {
+ masm.wasmUnalignedStoreI64(mir->access(), ToRegister64(lir->value()),
+ HeapReg, ToRegister(lir->ptr()), ptrScratch,
+ ToRegister(lir->getTemp(1)));
+ } else {
+ masm.wasmStoreI64(mir->access(), ToRegister64(lir->value()), HeapReg,
+ ToRegister(lir->ptr()), ptrScratch);
+ }
+}
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir) {
+ emitWasmStoreI64(lir);
+}
+
+void CodeGenerator::visitWasmUnalignedStoreI64(LWasmUnalignedStoreI64* lir) {
+ emitWasmStoreI64(lir);
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+
+ Register cond = ToRegister(lir->condExpr());
+ const LInt64Allocation falseExpr = lir->falseExpr();
+
+ Register64 out = ToOutRegister64(lir);
+ MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
+ "true expr is reused for input");
+
+ if (falseExpr.value().isRegister()) {
+ masm.as_movz(out.reg, ToRegister(falseExpr.value()), cond);
+ } else {
+ Label done;
+ masm.ma_b(cond, cond, &done, Assembler::NonZero, ShortJump);
+ masm.loadPtr(ToAddress(falseExpr.value()), out.reg);
+ masm.bind(&done);
+ }
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+ masm.as_dmtc1(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+ masm.as_dmfc1(ToRegister(lir->output()), ToFloatRegister(lir->input()));
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->isUnsigned()) {
+ masm.ma_dext(output, ToRegister(input), Imm32(0), Imm32(32));
+ } else {
+ masm.ma_sll(output, ToRegister(input), Imm32(0));
+ }
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ if (input->isMemory()) {
+ masm.load32(ToAddress(input), output);
+ } else {
+ masm.ma_sll(output, ToRegister(input), Imm32(0));
+ }
+ } else {
+ MOZ_CRASH("Not implemented.");
+ }
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ switch (lir->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.move32To64SignExtend(input.reg, output);
+ masm.move8SignExtend(output.reg, output.reg);
+ break;
+ case MSignExtendInt64::Half:
+ masm.move32To64SignExtend(input.reg, output);
+ masm.move16SignExtend(output.reg, output.reg);
+ break;
+ case MSignExtendInt64::Word:
+ masm.move32To64SignExtend(input.reg, output);
+ break;
+ }
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(input == output);
+ masm.move32To64ZeroExtend(input, Register64(output));
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ Register input = ToRegister(lir->input());
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(input == output);
+ masm.move64To32(Register64(input), output);
+}
+
+void CodeGenerator::visitClzI64(LClzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.clz64(input, output.reg);
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.ctz64(input, output.reg);
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register output = ToRegister(lir->output());
+
+ masm.ma_cmp_set(output, input.reg, zero, Assembler::Equal);
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ MOZ_ASSERT(!input->isConstant());
+ Register inputReg = ToRegister(input);
+ MOZ_ASSERT(inputReg == ToRegister(ins->output()));
+ masm.ma_not(inputReg, inputReg);
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ MWasmTruncateToInt64* mir = lir->mir();
+ MIRType fromType = mir->input()->type();
+
+ MOZ_ASSERT(fromType == MIRType::Double || fromType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ Label* oolRejoin = ool->rejoin();
+ bool isSaturating = mir->isSaturating();
+
+ if (fromType == MIRType::Double) {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ } else {
+ masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ }
+ } else {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ } else {
+ masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, InvalidFloatReg);
+ }
+ }
+}
+
+void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ MIRType outputType = lir->mir()->type();
+ MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
+
+ if (outputType == MIRType::Double) {
+ if (lir->mir()->isUnsigned()) {
+ masm.convertUInt64ToDouble(input, output, Register::Invalid());
+ } else {
+ masm.convertInt64ToDouble(input, output);
+ }
+ } else {
+ if (lir->mir()->isUnsigned()) {
+ masm.convertUInt64ToFloat32(input, output, Register::Invalid());
+ } else {
+ masm.convertInt64ToFloat32(input, output);
+ }
+ }
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ MBasicBlock* ifTrue = lir->ifTrue();
+ MBasicBlock* ifFalse = lir->ifFalse();
+
+ emitBranch(input.reg, Imm32(0), Assembler::NonZero, ifTrue, ifFalse);
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ auto sync = Synchronization::Load();
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ }
+ masm.memoryBarrierAfter(sync);
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp1);
+ auto sync = Synchronization::Store();
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.store64(temp1, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.store64(temp1, dest);
+ }
+ masm.memoryBarrierAfter(sync);
+}
diff --git a/js/src/jit/mips64/CodeGenerator-mips64.h b/js/src/jit/mips64/CodeGenerator-mips64.h
new file mode 100644
index 0000000000..81c30c913e
--- /dev/null
+++ b/js/src/jit/mips64/CodeGenerator-mips64.h
@@ -0,0 +1,65 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_CodeGenerator_mips64_h
+#define jit_mips64_CodeGenerator_mips64_h
+
+#include "jit/mips-shared/CodeGenerator-mips-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorMIPS64 : public CodeGeneratorMIPSShared {
+ protected:
+ CodeGeneratorMIPS64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
+ : CodeGeneratorMIPSShared(gen, graph, masm) {}
+
+ void testNullEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_NULL), cond, ifTrue, ifFalse);
+ }
+ void testUndefinedEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_UNDEFINED), cond, ifTrue,
+ ifFalse);
+ }
+ void testObjectEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ MOZ_ASSERT(value.valueReg() != SecondScratchReg);
+ masm.splitTag(value.valueReg(), SecondScratchReg);
+ emitBranch(SecondScratchReg, ImmTag(JSVAL_TAG_OBJECT), cond, ifTrue,
+ ifFalse);
+ }
+
+ void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+
+ template <typename T>
+ void emitWasmLoadI64(T* ins);
+ template <typename T>
+ void emitWasmStoreI64(T* ins);
+
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ // Functions for LTestVAndBranch.
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag);
+};
+
+typedef CodeGeneratorMIPS64 CodeGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_CodeGenerator_mips64_h */
diff --git a/js/src/jit/mips64/LIR-mips64.h b/js/src/jit/mips64/LIR-mips64.h
new file mode 100644
index 0000000000..4d8228418c
--- /dev/null
+++ b/js/src/jit/mips64/LIR-mips64.h
@@ -0,0 +1,147 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_LIR_mips64_h
+#define jit_mips64_LIR_mips64_h
+
+namespace js {
+namespace jit {
+
+class LUnbox : public LInstructionHelper<1, 1, 0> {
+ protected:
+ LUnbox(LNode::Opcode opcode, const LAllocation& input)
+ : LInstructionHelper(opcode) {
+ setOperand(0, input);
+ }
+
+ public:
+ LIR_HEADER(Unbox);
+
+ explicit LUnbox(const LAllocation& input) : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+
+ static const size_t Input = 0;
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+ const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LUnbox {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint);
+
+ LUnboxFloatingPoint(const LAllocation& input, MIRType type)
+ : LUnbox(classOpcode, input), type_(type) {}
+
+ MIRType type() const { return type_; }
+};
+
+class LDivOrModI64 : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LUDivOrModI64 : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(UDivOrModI64);
+
+ LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+ const char* extraName() const {
+ return mir()->isTruncated() ? "Truncated" : nullptr;
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LWasmTruncateToInt64 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ explicit LWasmTruncateToInt64(const LAllocation& in)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ }
+
+ MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); }
+};
+
+class LInt64ToFloatingPoint : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(Int64ToFloatingPoint);
+
+ explicit LInt64ToFloatingPoint(const LInt64Allocation& in)
+ : LInstructionHelper(classOpcode) {
+ setInt64Operand(0, in);
+ }
+
+ MInt64ToFloatingPoint* mir() const { return mir_->toInt64ToFloatingPoint(); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_LIR_mips64_h */
diff --git a/js/src/jit/mips64/Lowering-mips64.cpp b/js/src/jit/mips64/Lowering-mips64.cpp
new file mode 100644
index 0000000000..e9cda9299c
--- /dev/null
+++ b/js/src/jit/mips64/Lowering-mips64.cpp
@@ -0,0 +1,201 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips64/Lowering-mips64.h"
+
+#include "jit/Lowering.h"
+#include "jit/mips64/Assembler-mips64.h"
+#include "jit/MIR.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void LIRGeneratorMIPS64::defineInt64Phi(MPhi* phi, size_t lirIndex) {
+ defineTypedPhi(phi, lirIndex);
+}
+
+void LIRGeneratorMIPS64::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+LBoxAllocation LIRGeneratorMIPS64::useBoxFixed(MDefinition* mir, Register reg1,
+ Register reg2, bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart));
+}
+
+void LIRGeneratorMIPS64::lowerDivI64(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDivI64(div);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc())
+ LDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ defineInt64(lir, div);
+}
+
+void LIRGeneratorMIPS64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ MOZ_CRASH("We don't use runtime div for this architecture");
+}
+
+void LIRGeneratorMIPS64::lowerModI64(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUModI64(mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc())
+ LDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()), temp());
+ defineInt64(lir, mod);
+}
+
+void LIRGeneratorMIPS64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ MOZ_CRASH("We don't use runtime mod for this architecture");
+}
+
+void LIRGeneratorMIPS64::lowerUDivI64(MDiv* div) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useRegister(div->lhs()), useRegister(div->rhs()), temp());
+ defineInt64(lir, div);
+}
+
+void LIRGeneratorMIPS64::lowerUModI64(MMod* mod) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useRegister(mod->lhs()), useRegister(mod->rhs()), temp());
+ defineInt64(lir, mod);
+}
+
+void LIRGeneratorMIPS64::lowerBigIntDiv(MBigIntDiv* ins) {
+ auto* lir = new (alloc()) LBigIntDiv(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorMIPS64::lowerBigIntMod(MBigIntMod* ins) {
+ auto* lir = new (alloc()) LBigIntMod(useRegister(ins->lhs()),
+ useRegister(ins->rhs()), temp(), temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorMIPS64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+ auto* lir = new (alloc()) LAtomicLoad64(elements, index, temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorMIPS64::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value = useRegister(ins->value());
+
+ add(new (alloc()) LAtomicStore64(elements, index, value, tempInt64()), ins);
+}
+
+void LIRGenerator::visitBox(MBox* box) {
+ MDefinition* opd = box->getOperand(0);
+
+ // If the operand is a constant, emit near its uses.
+ if (opd->isConstant() && box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (opd->isConstant()) {
+ define(new (alloc()) LValue(opd->toConstant()->toJSValue()), box,
+ LDefinition(LDefinition::BOX));
+ } else {
+ LBox* ins = new (alloc()) LBox(useRegister(opd), opd->type());
+ define(ins, box, LDefinition(LDefinition::BOX));
+ }
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+ MDefinition* box = unbox->getOperand(0);
+ MOZ_ASSERT(box->type() == MIRType::Value);
+
+ LUnbox* lir;
+ if (IsFloatingPointType(unbox->type())) {
+ lir = new (alloc())
+ LUnboxFloatingPoint(useRegisterAtStart(box), unbox->type());
+ } else if (unbox->fallible()) {
+ // If the unbox is fallible, load the Value in a register first to
+ // avoid multiple loads.
+ lir = new (alloc()) LUnbox(useRegisterAtStart(box));
+ } else {
+ lir = new (alloc()) LUnbox(useAtStart(box));
+ }
+
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+
+ define(lir, unbox);
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new (alloc()) LReturn(isGenerator);
+ ins->setOperand(0, useFixed(opd, JSReturnReg));
+ add(ins);
+}
+
+void LIRGeneratorMIPS64::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void LIRGeneratorMIPS64::lowerTruncateDToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double);
+
+ define(new (alloc()) LTruncateDToInt32(useRegister(opd), tempDouble()), ins);
+}
+
+void LIRGeneratorMIPS64::lowerTruncateFToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Float32);
+
+ define(new (alloc()) LTruncateFToInt32(useRegister(opd), tempFloat32()), ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd)), ins);
+}
+
+void LIRGeneratorMIPS64::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Int64);
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+ define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd)), ins);
+}
+
+void LIRGeneratorMIPS64::lowerBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
diff --git a/js/src/jit/mips64/Lowering-mips64.h b/js/src/jit/mips64/Lowering-mips64.h
new file mode 100644
index 0000000000..b8543de6d2
--- /dev/null
+++ b/js/src/jit/mips64/Lowering-mips64.h
@@ -0,0 +1,56 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_Lowering_mips64_h
+#define jit_mips64_Lowering_mips64_h
+
+#include "jit/mips-shared/Lowering-mips-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorMIPS64 : public LIRGeneratorMIPSShared {
+ protected:
+ LIRGeneratorMIPS64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorMIPSShared(gen, graph, lirGraph) {}
+
+ void lowerInt64PhiInput(MPhi*, uint32_t, LBlock*, size_t);
+ void defineInt64Phi(MPhi*, size_t);
+
+ // Returns a box allocation. reg2 is ignored on 64-bit platforms.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2,
+ bool useAtStart = false);
+
+ inline LDefinition tempToUnbox() { return temp(); }
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+ void lowerTruncateDToInt32(MTruncateToInt32* ins);
+ void lowerTruncateFToInt32(MTruncateToInt32* ins);
+
+ void lowerDivI64(MDiv* div);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+ void lowerModI64(MMod* mod);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+ void lowerUDivI64(MDiv* div);
+ void lowerUModI64(MMod* mod);
+
+ void lowerBigIntDiv(MBigIntDiv* ins);
+ void lowerBigIntMod(MBigIntMod* ins);
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+ void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+};
+
+typedef LIRGeneratorMIPS64 LIRGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_Lowering_mips64_h */
diff --git a/js/src/jit/mips64/MacroAssembler-mips64-inl.h b/js/src/jit/mips64/MacroAssembler-mips64-inl.h
new file mode 100644
index 0000000000..cd899b0e18
--- /dev/null
+++ b/js/src/jit/mips64/MacroAssembler-mips64-inl.h
@@ -0,0 +1,824 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_MacroAssembler_mips64_inl_h
+#define jit_mips64_MacroAssembler_mips64_inl_h
+
+#include "jit/mips64/MacroAssembler-mips64.h"
+
+#include "vm/BigIntType.h" // JS::BigInt
+
+#include "jit/mips-shared/MacroAssembler-mips-shared-inl.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+ movePtr(src.reg, dest.reg);
+}
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+ movePtr(ImmWord(imm.value), dest.reg);
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ moveFromDouble(src, dest.reg);
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ moveToDouble(src.reg, dest);
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+ ma_sll(dest, src.reg, Imm32(0));
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ ma_dext(dest.reg, src, Imm32(0), Imm32(32));
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ move32To64SignExtend(src, dest);
+ move8SignExtend(dest.reg, dest.reg);
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ move32To64SignExtend(src, dest);
+ move16SignExtend(dest.reg, dest.reg);
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ ma_sll(dest.reg, src, Imm32(0));
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ ma_sll(dest, src, Imm32(0));
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ ma_dext(dest, src, Imm32(0), Imm32(32));
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ load32(src, dest);
+}
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::notPtr(Register reg) { ma_not(reg, reg); }
+
+void MacroAssembler::andPtr(Register src, Register dest) { ma_and(dest, src); }
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) { ma_and(dest, imm); }
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+ ma_li(ScratchRegister, ImmWord(imm.value));
+ ma_and(dest.reg, ScratchRegister);
+}
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+ ma_and(dest.reg, src.reg);
+}
+
+void MacroAssembler::and64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ and64(scratch, dest);
+ } else {
+ and64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+ ma_li(ScratchRegister, ImmWord(imm.value));
+ ma_or(dest.reg, ScratchRegister);
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+ ma_li(ScratchRegister, ImmWord(imm.value));
+ ma_xor(dest.reg, ScratchRegister);
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) { ma_or(dest, src); }
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) { ma_or(dest, imm); }
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+ ma_or(dest.reg, src.reg);
+}
+
+void MacroAssembler::or64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ or64(scratch, dest);
+ } else {
+ or64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+ ma_xor(dest.reg, src.reg);
+}
+
+void MacroAssembler::xor64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ xor64(scratch, dest);
+ } else {
+ xor64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) { ma_xor(dest, src); }
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) { ma_xor(dest, imm); }
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap64(Register64 reg64) {
+ Register reg = reg64.reg;
+ ma_dsbh(reg, reg);
+ ma_dshd(reg, reg);
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(Register src, Register dest) {
+ ma_daddu(dest, src);
+}
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) { ma_daddu(dest, imm); }
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+ movePtr(imm, ScratchRegister);
+ addPtr(ScratchRegister, dest);
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+ addPtr(src.reg, dest.reg);
+}
+
+void MacroAssembler::add64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ add64(scratch, dest);
+ } else {
+ add64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) {
+ ma_daddu(dest.reg, imm);
+}
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ mov(ImmWord(imm.value), ScratchRegister);
+ ma_daddu(dest.reg, ScratchRegister);
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ MacroAssemblerMIPSShared::ma_liPatchable(dest, Imm32(0));
+ as_dsubu(dest, StackPointer, dest);
+ return offset;
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ Instruction* lui =
+ (Instruction*)m_buffer.getInst(BufferOffset(offset.offset()));
+ MOZ_ASSERT(lui->extractOpcode() == ((uint32_t)op_lui >> OpcodeShift));
+ MOZ_ASSERT(lui->next()->extractOpcode() == ((uint32_t)op_ori >> OpcodeShift));
+
+ MacroAssemblerMIPSShared::UpdateLuiOriValue(lui, lui->next(), imm.value);
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) {
+ as_dsubu(dest, dest, src);
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) {
+ ma_dsubu(dest, dest, imm);
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+ as_dsubu(dest.reg, dest.reg, src.reg);
+}
+
+void MacroAssembler::sub64(const Operand& src, Register64 dest) {
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ sub64(scratch, dest);
+ } else {
+ sub64(Register64(src.toReg()), dest);
+ }
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ mov(ImmWord(imm.value), ScratchRegister);
+ as_dsubu(dest.reg, dest.reg, ScratchRegister);
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(src != scratch);
+ move32(imm, scratch);
+#ifdef MIPSR6
+ as_muhu(dest, src, scratch);
+#else
+ as_multu(src, scratch);
+ as_mfhi(dest);
+#endif
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+#ifdef MIPSR6
+ as_dmulu(srcDest, srcDest, rhs);
+#else
+ as_dmultu(srcDest, rhs);
+ as_mflo(srcDest);
+#endif
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+ MOZ_ASSERT(dest.reg != ScratchRegister);
+ mov(ImmWord(imm.value), ScratchRegister);
+#ifdef MIPSR6
+ as_dmulu(dest.reg, ScratchRegister, dest.reg);
+#else
+ as_dmultu(dest.reg, ScratchRegister);
+ as_mflo(dest.reg);
+#endif
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ mul64(imm, dest);
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+#ifdef MIPSR6
+ as_dmulu(dest.reg, src.reg, dest.reg);
+#else
+ as_dmultu(dest.reg, src.reg);
+ as_mflo(dest.reg);
+#endif
+}
+
+void MacroAssembler::mul64(const Operand& src, const Register64& dest,
+ const Register temp) {
+ if (src.getTag() == Operand::MEM) {
+ Register64 scratch(ScratchRegister);
+
+ load64(src.toAddress(), scratch);
+ mul64(scratch, dest, temp);
+ } else {
+ mul64(Register64(src.toReg()), dest, temp);
+ }
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+ MOZ_ASSERT(src != ScratchRegister);
+ as_daddu(ScratchRegister, src, src);
+ as_daddu(dest, ScratchRegister, src);
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+ ma_li(ScratchRegister, ImmWord(uintptr_t(dest.addr)));
+ as_ld(SecondScratchReg, ScratchRegister, 0);
+ as_daddiu(SecondScratchReg, SecondScratchReg, 1);
+ as_sd(SecondScratchReg, ScratchRegister, 0);
+}
+
+void MacroAssembler::neg64(Register64 reg) { as_dsubu(reg.reg, zero, reg.reg); }
+
+void MacroAssembler::negPtr(Register reg) { as_dsubu(reg, zero, reg); }
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ma_dsll(dest, dest, imm);
+}
+
+void MacroAssembler::lshiftPtr(Register shift, Register dest) {
+ ma_dsll(dest, dest, shift);
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ma_dsll(dest.reg, dest.reg, imm);
+}
+
+void MacroAssembler::lshift64(Register shift, Register64 dest) {
+ ma_dsll(dest.reg, dest.reg, shift);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ma_dsrl(dest, dest, imm);
+}
+
+void MacroAssembler::rshiftPtr(Register shift, Register dest) {
+ ma_dsrl(dest, dest, shift);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ma_dsrl(dest.reg, dest.reg, imm);
+}
+
+void MacroAssembler::rshift64(Register shift, Register64 dest) {
+ ma_dsrl(dest.reg, dest.reg, shift);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ma_dsra(dest, dest, imm);
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ ma_dsra(dest.reg, dest.reg, imm);
+}
+
+void MacroAssembler::rshift64Arithmetic(Register shift, Register64 dest) {
+ ma_dsra(dest.reg, dest.reg, shift);
+}
+
+// ===============================================================
+// Rotation functions
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+
+ if (count.value) {
+ ma_drol(dest.reg, src.reg, count);
+ } else {
+ ma_move(dest.reg, src.reg);
+ }
+}
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ ma_drol(dest.reg, src.reg, count);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+
+ if (count.value) {
+ ma_dror(dest.reg, src.reg, count);
+ } else {
+ ma_move(dest.reg, src.reg);
+ }
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ ma_dror(dest.reg, src.reg, count);
+}
+
+// ===============================================================
+// Condition functions
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ ma_cmp_set(dest, lhs, rhs, cond);
+}
+
+// Also see below for specializations of cmpPtrSet.
+
+template <typename T1, typename T2>
+void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ ma_cmp_set(dest, lhs, rhs, cond);
+}
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ ma_cmp_set(dest, lhs, ImmWord(uint64_t(rhs.value)), cond);
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+ as_dclz(dest, src.reg);
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+ ma_dctz(dest, src.reg);
+}
+
+void MacroAssembler::popcnt64(Register64 input, Register64 output,
+ Register tmp) {
+ ma_move(output.reg, input.reg);
+ ma_dsra(tmp, input.reg, Imm32(1));
+ ma_li(ScratchRegister, ImmWord(0x5555555555555555UL));
+ ma_and(tmp, ScratchRegister);
+ ma_dsubu(output.reg, tmp);
+ ma_dsra(tmp, output.reg, Imm32(2));
+ ma_li(ScratchRegister, ImmWord(0x3333333333333333UL));
+ ma_and(output.reg, ScratchRegister);
+ ma_and(tmp, ScratchRegister);
+ ma_daddu(output.reg, tmp);
+ ma_dsrl(tmp, output.reg, Imm32(4));
+ ma_daddu(output.reg, tmp);
+ ma_li(ScratchRegister, ImmWord(0xF0F0F0F0F0F0F0FUL));
+ ma_and(output.reg, ScratchRegister);
+ ma_dsll(tmp, output.reg, Imm32(8));
+ ma_daddu(output.reg, tmp);
+ ma_dsll(tmp, output.reg, Imm32(16));
+ ma_daddu(output.reg, tmp);
+ ma_dsll(tmp, output.reg, Imm32(32));
+ ma_daddu(output.reg, tmp);
+ ma_dsra(output.reg, output.reg, Imm32(56));
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan ||
+ cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan ||
+ cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, ImmWord(val.value), success);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan ||
+ cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan ||
+ cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, rhs.reg, success);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, ImmWord(val.value), label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, rhs.reg, label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ loadPtr(rhs, scratch);
+ branchPtr(cond, lhs, scratch, label);
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ branchPtr(cond, lhs, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ branchTestPtr(cond, lhs.reg, rhs.reg, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestUndefined(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestInt32(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestInt32Truthy(bool b, const ValueOperand& value,
+ Label* label) {
+ ScratchRegisterScope scratch(*this);
+ ma_dext(scratch, value.valueReg(), Imm32(0), Imm32(32));
+ ma_b(scratch, scratch, label, b ? NonZero : Zero);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, Register tag,
+ Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = (cond == Equal) ? BelowOrEqual : Above;
+ ma_b(tag, ImmTag(JSVAL_TAG_MAX_DOUBLE), label, actual);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestDouble(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestNumber(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestBoolean(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ unboxBoolean(value, scratch2);
+ ma_b(scratch2, scratch2, label, b ? NonZero : Zero);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestString(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestStringTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ unboxString(value, scratch2);
+ load32(Address(scratch2, JSString::offsetOfLength()), scratch2);
+ ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestSymbol(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ computeEffectiveAddress(address, scratch2);
+ splitTag(scratch2, scratch2);
+ branchTestBigInt(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestBigInt(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestBigIntTruthy(bool b, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ unboxBigInt(value, scratch2);
+ load32(Address(scratch2, BigInt::offsetOfDigitLength()), scratch2);
+ ma_b(scratch2, Imm32(0), label, b ? NotEqual : Equal);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestNull(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestObject(cond, scratch2, label);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ branchTestPrimitive(cond, scratch2, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
+ L label) {
+ SecondScratchRegisterScope scratch2(*this);
+ splitTag(value, scratch2);
+ ma_b(scratch2, ImmTag(JSVAL_TAG_MAGIC), label, cond);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ uint64_t magic = MagicValue(why).asRawBits();
+ SecondScratchRegisterScope scratch(*this);
+ loadPtr(valaddr, scratch);
+ ma_b(scratch, ImmWord(magic), label, cond);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ branchPtr(cond, lhs, rhs.valueReg(), label);
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ as_truncld(ScratchDoubleReg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromDouble(ScratchDoubleReg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+
+ as_sll(dest, dest, 0);
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ as_truncls(ScratchDoubleReg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromDouble(ScratchDoubleReg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+
+ as_sll(dest, dest, 0);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) {
+ MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
+ type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
+ // dest := src XOR mask
+ // scratch := dest >> JSVAL_TAG_SHIFT
+ // fail if scratch != 0
+ //
+ // Note: src and dest can be the same register
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ ma_xor(scratch, src.valueReg());
+ ma_move(dest, scratch);
+ ma_dsrl(scratch, scratch, Imm32(JSVAL_TAG_SHIFT));
+ ma_b(scratch, Imm32(0), fail, Assembler::NotEqual);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) {
+ loadValue(src, ValueOperand(dest));
+ fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) {
+ loadValue(src, ValueOperand(dest));
+ fallibleUnboxPtr(ValueOperand(dest), dest, type, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+// The specializations for cmpPtrSet are outside the braces because
+// check_macroassembler_style can't yet deal with specializations.
+
+template <>
+inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs,
+ ImmPtr rhs, Register dest) {
+ loadPtr(lhs, SecondScratchReg);
+ cmpPtrSet(cond, SecondScratchReg, rhs, dest);
+}
+
+template <>
+inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Register lhs,
+ Address rhs, Register dest) {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ loadPtr(rhs, ScratchRegister);
+ cmpPtrSet(cond, lhs, ScratchRegister, dest);
+}
+
+template <>
+inline void MacroAssembler::cmpPtrSet(Assembler::Condition cond, Address lhs,
+ Register rhs, Register dest) {
+ MOZ_ASSERT(rhs != ScratchRegister);
+ loadPtr(lhs, ScratchRegister);
+ cmpPtrSet(cond, ScratchRegister, rhs, dest);
+}
+
+template <>
+inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Register lhs,
+ Address rhs, Register dest) {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ load32(rhs, ScratchRegister);
+ cmp32Set(cond, lhs, ScratchRegister, dest);
+}
+
+template <>
+inline void MacroAssembler::cmp32Set(Assembler::Condition cond, Address lhs,
+ Register rhs, Register dest) {
+ MOZ_ASSERT(rhs != ScratchRegister);
+ load32(lhs, ScratchRegister);
+ cmp32Set(cond, ScratchRegister, rhs, dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ Register scratch = ScratchRegister;
+ MOZ_ASSERT(src != scratch && dest != scratch);
+ cmpPtrSet(cond, lhs, rhs, scratch);
+#ifdef MIPSR6
+ as_selnez(src, src, scratch);
+ as_seleqz(dest, dest, scratch);
+ as_or(dest, dest, src);
+#else
+ as_movn(dest, src, scratch);
+#endif
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ MOZ_CRASH("NYI");
+}
+
+void MacroAssemblerMIPS64Compat::incrementInt32Value(const Address& addr) {
+ asMasm().add32(Imm32(1), addr);
+}
+
+void MacroAssemblerMIPS64Compat::retn(Imm32 n) {
+ // pc <- [sp]; sp += n
+ loadPtr(Address(StackPointer, 0), ra);
+ asMasm().addPtr(n, StackPointer);
+ as_jr(ra);
+ as_nop();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_MacroAssembler_mips64_inl_h */
diff --git a/js/src/jit/mips64/MacroAssembler-mips64.cpp b/js/src/jit/mips64/MacroAssembler-mips64.cpp
new file mode 100644
index 0000000000..17a24a9a4f
--- /dev/null
+++ b/js/src/jit/mips64/MacroAssembler-mips64.cpp
@@ -0,0 +1,2851 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips64/MacroAssembler-mips64.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "jit/mips64/Simulator-mips64.h"
+#include "jit/MoveEmitter.h"
+#include "jit/SharedICRegisters.h"
+#include "util/Memory.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace jit;
+
+using mozilla::Abs;
+
+static_assert(sizeof(intptr_t) == 8, "Not 32-bit clean.");
+
+void MacroAssemblerMIPS64Compat::convertBoolToInt32(Register src,
+ Register dest) {
+ // Note that C++ bool is only 1 byte, so zero extend it to clear the
+ // higher-order bits.
+ ma_and(dest, src, Imm32(0xff));
+}
+
+void MacroAssemblerMIPS64Compat::convertInt32ToDouble(Register src,
+ FloatRegister dest) {
+ as_mtc1(src, dest);
+ as_cvtdw(dest, dest);
+}
+
+void MacroAssemblerMIPS64Compat::convertInt32ToDouble(const Address& src,
+ FloatRegister dest) {
+ ma_ls(dest, src);
+ as_cvtdw(dest, dest);
+}
+
+void MacroAssemblerMIPS64Compat::convertInt32ToDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ computeScaledAddress(src, ScratchRegister);
+ convertInt32ToDouble(Address(ScratchRegister, src.offset), dest);
+}
+
+void MacroAssemblerMIPS64Compat::convertUInt32ToDouble(Register src,
+ FloatRegister dest) {
+ ma_dext(ScratchRegister, src, Imm32(0), Imm32(32));
+ asMasm().convertInt64ToDouble(Register64(ScratchRegister), dest);
+}
+
+void MacroAssemblerMIPS64Compat::convertUInt64ToDouble(Register src,
+ FloatRegister dest) {
+ Label positive, done;
+ ma_b(src, src, &positive, NotSigned, ShortJump);
+
+ MOZ_ASSERT(src != ScratchRegister);
+ MOZ_ASSERT(src != SecondScratchReg);
+
+ ma_and(ScratchRegister, src, Imm32(1));
+ ma_dsrl(SecondScratchReg, src, Imm32(1));
+ ma_or(ScratchRegister, SecondScratchReg);
+ as_dmtc1(ScratchRegister, dest);
+ as_cvtdl(dest, dest);
+ asMasm().addDouble(dest, dest);
+ ma_b(&done, ShortJump);
+
+ bind(&positive);
+ as_dmtc1(src, dest);
+ as_cvtdl(dest, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerMIPS64Compat::convertUInt32ToFloat32(Register src,
+ FloatRegister dest) {
+ ma_dext(ScratchRegister, src, Imm32(0), Imm32(32));
+ asMasm().convertInt64ToFloat32(Register64(ScratchRegister), dest);
+}
+
+void MacroAssemblerMIPS64Compat::convertDoubleToFloat32(FloatRegister src,
+ FloatRegister dest) {
+ as_cvtsd(dest, src);
+}
+
+const int CauseBitPos = int(Assembler::CauseI);
+const int CauseBitCount = 1 + int(Assembler::CauseV) - int(Assembler::CauseI);
+const int CauseIOrVMask = ((1 << int(Assembler::CauseI)) |
+ (1 << int(Assembler::CauseV))) >>
+ int(Assembler::CauseI);
+
+// Checks whether a double is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void MacroAssemblerMIPS64Compat::convertDoubleToInt32(FloatRegister src,
+ Register dest,
+ Label* fail,
+ bool negativeZeroCheck) {
+ if (negativeZeroCheck) {
+ moveFromDouble(src, dest);
+ ma_drol(dest, dest, Imm32(1));
+ ma_b(dest, Imm32(1), fail, Assembler::Equal);
+ }
+
+ // Truncate double to int ; if result is inexact or invalid fail.
+ as_truncwd(ScratchFloat32Reg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, CauseBitPos, CauseBitCount);
+ as_andi(ScratchRegister, ScratchRegister,
+ CauseIOrVMask); // masking for Inexact and Invalid flag.
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+}
+
+void MacroAssemblerMIPS64Compat::convertDoubleToPtr(FloatRegister src,
+ Register dest, Label* fail,
+ bool negativeZeroCheck) {
+ if (negativeZeroCheck) {
+ moveFromDouble(src, dest);
+ ma_drol(dest, dest, Imm32(1));
+ ma_b(dest, Imm32(1), fail, Assembler::Equal);
+ }
+ as_truncld(ScratchDoubleReg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromDouble(ScratchDoubleReg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, CauseBitPos, CauseBitCount);
+ as_andi(ScratchRegister, ScratchRegister, CauseIOrVMask);
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+}
+
+// Checks whether a float32 is representable as a 32-bit integer. If so, the
+// integer is written to the output register. Otherwise, a bailout is taken to
+// the given snapshot. This function overwrites the scratch float register.
+void MacroAssemblerMIPS64Compat::convertFloat32ToInt32(FloatRegister src,
+ Register dest,
+ Label* fail,
+ bool negativeZeroCheck) {
+ if (negativeZeroCheck) {
+ moveFromFloat32(src, dest);
+ ma_b(dest, Imm32(INT32_MIN), fail, Assembler::Equal);
+ }
+
+ as_truncws(ScratchFloat32Reg, src);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromFloat32(ScratchFloat32Reg, dest);
+ ma_ext(ScratchRegister, ScratchRegister, CauseBitPos, CauseBitCount);
+ as_andi(ScratchRegister, ScratchRegister, CauseIOrVMask);
+ ma_b(ScratchRegister, Imm32(0), fail, Assembler::NotEqual);
+}
+
+void MacroAssemblerMIPS64Compat::convertFloat32ToDouble(FloatRegister src,
+ FloatRegister dest) {
+ as_cvtds(dest, src);
+}
+
+void MacroAssemblerMIPS64Compat::convertInt32ToFloat32(Register src,
+ FloatRegister dest) {
+ as_mtc1(src, dest);
+ as_cvtsw(dest, dest);
+}
+
+void MacroAssemblerMIPS64Compat::convertInt32ToFloat32(const Address& src,
+ FloatRegister dest) {
+ ma_ls(dest, src);
+ as_cvtsw(dest, dest);
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ convertInt64ToDouble(Register64(src), dest);
+}
+
+void MacroAssemblerMIPS64Compat::movq(Register rs, Register rd) {
+ ma_move(rd, rs);
+}
+
+void MacroAssemblerMIPS64::ma_li(Register dest, CodeLabel* label) {
+ BufferOffset bo = m_buffer.nextOffset();
+ ma_liPatchable(dest, ImmWord(/* placeholder */ 0));
+ label->patchAt()->bind(bo.getOffset());
+ label->setLinkMode(CodeLabel::MoveImmediate);
+}
+
+void MacroAssemblerMIPS64::ma_li(Register dest, ImmWord imm) {
+ int64_t value = imm.value;
+
+ if (-1 == (value >> 15) || 0 == (value >> 15)) {
+ as_addiu(dest, zero, value);
+ return;
+ }
+ if (0 == (value >> 16)) {
+ as_ori(dest, zero, value);
+ return;
+ }
+
+ if (-1 == (value >> 31) || 0 == (value >> 31)) {
+ as_lui(dest, uint16_t(value >> 16));
+ } else if (0 == (value >> 32)) {
+ as_lui(dest, uint16_t(value >> 16));
+ as_dinsu(dest, zero, 32, 32);
+ } else if (-1 == (value >> 47) || 0 == (value >> 47)) {
+ as_lui(dest, uint16_t(value >> 32));
+ if (uint16_t(value >> 16)) {
+ as_ori(dest, dest, uint16_t(value >> 16));
+ }
+ as_dsll(dest, dest, 16);
+ } else if (0 == (value >> 48)) {
+ as_lui(dest, uint16_t(value >> 32));
+ as_dinsu(dest, zero, 32, 32);
+ if (uint16_t(value >> 16)) {
+ as_ori(dest, dest, uint16_t(value >> 16));
+ }
+ as_dsll(dest, dest, 16);
+ } else {
+ as_lui(dest, uint16_t(value >> 48));
+ if (uint16_t(value >> 32)) {
+ as_ori(dest, dest, uint16_t(value >> 32));
+ }
+ if (uint16_t(value >> 16)) {
+ as_dsll(dest, dest, 16);
+ as_ori(dest, dest, uint16_t(value >> 16));
+ as_dsll(dest, dest, 16);
+ } else {
+ as_dsll32(dest, dest, 32);
+ }
+ }
+ if (uint16_t(value)) {
+ as_ori(dest, dest, uint16_t(value));
+ }
+}
+
+// This method generates lui, dsll and ori instruction block that can be
+// modified by UpdateLoad64Value, either during compilation (eg.
+// Assembler::bind), or during execution (eg. jit::PatchJump).
+void MacroAssemblerMIPS64::ma_liPatchable(Register dest, ImmPtr imm) {
+ return ma_liPatchable(dest, ImmWord(uintptr_t(imm.value)));
+}
+
+void MacroAssemblerMIPS64::ma_liPatchable(Register dest, ImmWord imm,
+ LiFlags flags) {
+ if (Li64 == flags) {
+ m_buffer.ensureSpace(6 * sizeof(uint32_t));
+ as_lui(dest, Imm16::Upper(Imm32(imm.value >> 32)).encode());
+ as_ori(dest, dest, Imm16::Lower(Imm32(imm.value >> 32)).encode());
+ as_dsll(dest, dest, 16);
+ as_ori(dest, dest, Imm16::Upper(Imm32(imm.value)).encode());
+ as_dsll(dest, dest, 16);
+ as_ori(dest, dest, Imm16::Lower(Imm32(imm.value)).encode());
+ } else {
+ m_buffer.ensureSpace(4 * sizeof(uint32_t));
+ as_lui(dest, Imm16::Lower(Imm32(imm.value >> 32)).encode());
+ as_ori(dest, dest, Imm16::Upper(Imm32(imm.value)).encode());
+ as_drotr32(dest, dest, 48);
+ as_ori(dest, dest, Imm16::Lower(Imm32(imm.value)).encode());
+ }
+}
+
+void MacroAssemblerMIPS64::ma_dnegu(Register rd, Register rs) {
+ as_dsubu(rd, zero, rs);
+}
+
+// Shifts
+void MacroAssemblerMIPS64::ma_dsll(Register rd, Register rt, Imm32 shift) {
+ if (31 < shift.value) {
+ as_dsll32(rd, rt, shift.value);
+ } else {
+ as_dsll(rd, rt, shift.value);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_dsrl(Register rd, Register rt, Imm32 shift) {
+ if (31 < shift.value) {
+ as_dsrl32(rd, rt, shift.value);
+ } else {
+ as_dsrl(rd, rt, shift.value);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_dsra(Register rd, Register rt, Imm32 shift) {
+ if (31 < shift.value) {
+ as_dsra32(rd, rt, shift.value);
+ } else {
+ as_dsra(rd, rt, shift.value);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_dror(Register rd, Register rt, Imm32 shift) {
+ if (31 < shift.value) {
+ as_drotr32(rd, rt, shift.value);
+ } else {
+ as_drotr(rd, rt, shift.value);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_drol(Register rd, Register rt, Imm32 shift) {
+ uint32_t s = 64 - shift.value;
+
+ if (31 < s) {
+ as_drotr32(rd, rt, s);
+ } else {
+ as_drotr(rd, rt, s);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_dsll(Register rd, Register rt, Register shift) {
+ as_dsllv(rd, rt, shift);
+}
+
+void MacroAssemblerMIPS64::ma_dsrl(Register rd, Register rt, Register shift) {
+ as_dsrlv(rd, rt, shift);
+}
+
+void MacroAssemblerMIPS64::ma_dsra(Register rd, Register rt, Register shift) {
+ as_dsrav(rd, rt, shift);
+}
+
+void MacroAssemblerMIPS64::ma_dror(Register rd, Register rt, Register shift) {
+ as_drotrv(rd, rt, shift);
+}
+
+void MacroAssemblerMIPS64::ma_drol(Register rd, Register rt, Register shift) {
+ as_dsubu(ScratchRegister, zero, shift);
+ as_drotrv(rd, rt, ScratchRegister);
+}
+
+void MacroAssemblerMIPS64::ma_dins(Register rt, Register rs, Imm32 pos,
+ Imm32 size) {
+ if (pos.value >= 0 && pos.value < 32) {
+ if (pos.value + size.value > 32) {
+ as_dinsm(rt, rs, pos.value, size.value);
+ } else {
+ as_dins(rt, rs, pos.value, size.value);
+ }
+ } else {
+ as_dinsu(rt, rs, pos.value, size.value);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_dext(Register rt, Register rs, Imm32 pos,
+ Imm32 size) {
+ if (pos.value >= 0 && pos.value < 32) {
+ if (size.value > 32) {
+ as_dextm(rt, rs, pos.value, size.value);
+ } else {
+ as_dext(rt, rs, pos.value, size.value);
+ }
+ } else {
+ as_dextu(rt, rs, pos.value, size.value);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_dsbh(Register rd, Register rt) {
+ as_dsbh(rd, rt);
+}
+
+void MacroAssemblerMIPS64::ma_dshd(Register rd, Register rt) {
+ as_dshd(rd, rt);
+}
+
+void MacroAssemblerMIPS64::ma_dctz(Register rd, Register rs) {
+ ma_dnegu(ScratchRegister, rs);
+ as_and(rd, ScratchRegister, rs);
+ as_dclz(rd, rd);
+ ma_dnegu(SecondScratchReg, rd);
+ ma_daddu(SecondScratchReg, Imm32(0x3f));
+#ifdef MIPS64
+ as_selnez(SecondScratchReg, SecondScratchReg, ScratchRegister);
+ as_seleqz(rd, rd, ScratchRegister);
+ as_or(rd, rd, SecondScratchReg);
+#else
+ as_movn(rd, SecondScratchReg, ScratchRegister);
+#endif
+}
+
+// Arithmetic-based ops.
+
+// Add.
+void MacroAssemblerMIPS64::ma_daddu(Register rd, Register rs, Imm32 imm) {
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_daddiu(rd, rs, imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_daddu(rd, rs, ScratchRegister);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_daddu(Register rd, Register rs) {
+ as_daddu(rd, rd, rs);
+}
+
+void MacroAssemblerMIPS64::ma_daddu(Register rd, Imm32 imm) {
+ ma_daddu(rd, rd, imm);
+}
+
+void MacroAssemblerMIPS64::ma_add32TestOverflow(Register rd, Register rs,
+ Register rt, Label* overflow) {
+ as_daddu(SecondScratchReg, rs, rt);
+ as_addu(rd, rs, rt);
+ ma_b(rd, SecondScratchReg, overflow, Assembler::NotEqual);
+}
+
+void MacroAssemblerMIPS64::ma_add32TestOverflow(Register rd, Register rs,
+ Imm32 imm, Label* overflow) {
+ // Check for signed range because of as_daddiu
+ if (Imm16::IsInSignedRange(imm.value)) {
+ as_daddiu(SecondScratchReg, rs, imm.value);
+ as_addiu(rd, rs, imm.value);
+ ma_b(rd, SecondScratchReg, overflow, Assembler::NotEqual);
+ } else {
+ ma_li(ScratchRegister, imm);
+ ma_add32TestOverflow(rd, rs, ScratchRegister, overflow);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_addPtrTestOverflow(Register rd, Register rs,
+ Register rt, Label* overflow) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT(rd != rt);
+ MOZ_ASSERT(rd != scratch2);
+
+ if (rs == rt) {
+ as_daddu(rd, rs, rs);
+ as_xor(scratch2, rs, rd);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rs != scratch2);
+ MOZ_ASSERT(rt != scratch2);
+
+ // If the sign of rs and rt are different, no overflow
+ as_xor(scratch2, rs, rt);
+ as_nor(scratch2, scratch2, zero);
+
+ as_daddu(rd, rs, rt);
+ as_xor(scratch, rd, rt);
+ as_and(scratch, scratch, scratch2);
+ }
+
+ ma_b(scratch2, zero, overflow, Assembler::LessThan);
+}
+
+void MacroAssemblerMIPS64::ma_addPtrTestOverflow(Register rd, Register rs,
+ Imm32 imm, Label* overflow) {
+ ma_li(ScratchRegister, imm);
+ ma_addPtrTestOverflow(rd, rs, ScratchRegister, overflow);
+}
+
+void MacroAssemblerMIPS64::ma_addPtrTestOverflow(Register rd, Register rs,
+ ImmWord imm, Label* overflow) {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ ma_addPtrTestOverflow(rd, rs, scratch, overflow);
+}
+
+void MacroAssemblerMIPS64::ma_addPtrTestCarry(Condition cond, Register rd,
+ Register rs, Register rt,
+ Label* overflow) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ as_daddu(rd, rs, rt);
+ as_sltu(scratch2, rd, rt);
+ ma_b(scratch2, scratch2, overflow,
+ cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
+}
+
+void MacroAssemblerMIPS64::ma_addPtrTestCarry(Condition cond, Register rd,
+ Register rs, Imm32 imm,
+ Label* overflow) {
+ // Check for signed range because of as_daddiu
+ if (Imm16::IsInSignedRange(imm.value)) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ as_daddiu(rd, rs, imm.value);
+ as_sltiu(scratch2, rd, imm.value);
+ ma_b(scratch2, scratch2, overflow,
+ cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
+ } else {
+ ma_li(ScratchRegister, imm);
+ ma_addPtrTestCarry(cond, rd, rs, ScratchRegister, overflow);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_addPtrTestCarry(Condition cond, Register rd,
+ Register rs, ImmWord imm,
+ Label* overflow) {
+ // Check for signed range because of as_daddiu
+ if (Imm16::IsInSignedRange(imm.value)) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ as_daddiu(rd, rs, imm.value);
+ as_sltiu(scratch2, rd, imm.value);
+ ma_b(scratch2, scratch2, overflow,
+ cond == Assembler::CarrySet ? Assembler::NonZero : Assembler::Zero);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ ma_li(scratch, imm);
+ ma_addPtrTestCarry(cond, rd, rs, scratch, overflow);
+ }
+}
+
+// Subtract.
+void MacroAssemblerMIPS64::ma_dsubu(Register rd, Register rs, Imm32 imm) {
+ if (Imm16::IsInSignedRange(-imm.value)) {
+ as_daddiu(rd, rs, -imm.value);
+ } else {
+ ma_li(ScratchRegister, imm);
+ as_dsubu(rd, rs, ScratchRegister);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_dsubu(Register rd, Register rs) {
+ as_dsubu(rd, rd, rs);
+}
+
+void MacroAssemblerMIPS64::ma_dsubu(Register rd, Imm32 imm) {
+ ma_dsubu(rd, rd, imm);
+}
+
+void MacroAssemblerMIPS64::ma_sub32TestOverflow(Register rd, Register rs,
+ Register rt, Label* overflow) {
+ as_dsubu(SecondScratchReg, rs, rt);
+ as_subu(rd, rs, rt);
+ ma_b(rd, SecondScratchReg, overflow, Assembler::NotEqual);
+}
+
+void MacroAssemblerMIPS64::ma_subPtrTestOverflow(Register rd, Register rs,
+ Register rt, Label* overflow) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ MOZ_ASSERT_IF(rs == rd, rs != rt);
+ MOZ_ASSERT(rd != rt);
+ MOZ_ASSERT(rs != scratch2);
+ MOZ_ASSERT(rt != scratch2);
+ MOZ_ASSERT(rd != scratch2);
+
+ Register rs_copy = rs;
+
+ if (rs == rd) {
+ ma_move(scratch2, rs);
+ rs_copy = scratch2;
+ }
+
+ {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(rd != scratch);
+
+ as_dsubu(rd, rs, rt);
+ // If the sign of rs and rt are the same, no overflow
+ as_xor(scratch, rs_copy, rt);
+ // Check if the sign of rd and rs are the same
+ as_xor(scratch2, rd, rs_copy);
+ as_and(scratch2, scratch2, scratch);
+ }
+
+ ma_b(scratch2, zero, overflow, Assembler::LessThan);
+}
+
+void MacroAssemblerMIPS64::ma_subPtrTestOverflow(Register rd, Register rs,
+ Imm32 imm, Label* overflow) {
+ ma_li(ScratchRegister, imm);
+ ma_subPtrTestOverflow(rd, rs, ScratchRegister, overflow);
+}
+
+void MacroAssemblerMIPS64::ma_dmult(Register rs, Imm32 imm) {
+ ma_li(ScratchRegister, imm);
+#ifdef MIPSR6
+ as_dmul(rs, ScratchRegister, SecondScratchReg);
+ as_dmuh(rs, ScratchRegister, rs);
+ ma_move(rs, SecondScratchReg);
+#else
+ as_dmult(rs, ScratchRegister);
+#endif
+}
+
+void MacroAssemblerMIPS64::ma_mulPtrTestOverflow(Register rd, Register rs,
+ Register rt, Label* overflow) {
+#ifdef MIPSR6
+ if (rd == rs) {
+ ma_move(SecondScratchReg, rs);
+ rs = SecondScratchReg;
+ }
+ as_dmul(rd, rs, rt);
+ as_dmuh(SecondScratchReg, rs, rt);
+#else
+ as_dmult(rs, rt);
+ as_mflo(rd);
+ as_mfhi(SecondScratchReg);
+#endif
+ as_dsra32(ScratchRegister, rd, 63);
+ ma_b(ScratchRegister, SecondScratchReg, overflow, Assembler::NotEqual);
+}
+
+// Memory.
+void MacroAssemblerMIPS64::ma_load(Register dest, Address address,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ int16_t encodedOffset;
+ Register base;
+
+ if (isLoongson() && ZeroExtend != extension &&
+ !Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ base = address.base;
+
+ switch (size) {
+ case SizeByte:
+ as_gslbx(dest, base, ScratchRegister, 0);
+ break;
+ case SizeHalfWord:
+ as_gslhx(dest, base, ScratchRegister, 0);
+ break;
+ case SizeWord:
+ as_gslwx(dest, base, ScratchRegister, 0);
+ break;
+ case SizeDouble:
+ as_gsldx(dest, base, ScratchRegister, 0);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+ return;
+ }
+
+ if (!Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_daddu(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = Imm16(0).encode();
+ } else {
+ encodedOffset = Imm16(address.offset).encode();
+ base = address.base;
+ }
+
+ switch (size) {
+ case SizeByte:
+ if (ZeroExtend == extension) {
+ as_lbu(dest, base, encodedOffset);
+ } else {
+ as_lb(dest, base, encodedOffset);
+ }
+ break;
+ case SizeHalfWord:
+ if (ZeroExtend == extension) {
+ as_lhu(dest, base, encodedOffset);
+ } else {
+ as_lh(dest, base, encodedOffset);
+ }
+ break;
+ case SizeWord:
+ if (ZeroExtend == extension) {
+ as_lwu(dest, base, encodedOffset);
+ } else {
+ as_lw(dest, base, encodedOffset);
+ }
+ break;
+ case SizeDouble:
+ as_ld(dest, base, encodedOffset);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_load");
+ }
+}
+
+void MacroAssemblerMIPS64::ma_store(Register data, Address address,
+ LoadStoreSize size,
+ LoadStoreExtension extension) {
+ int16_t encodedOffset;
+ Register base;
+
+ if (isLoongson() && !Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ base = address.base;
+
+ switch (size) {
+ case SizeByte:
+ as_gssbx(data, base, ScratchRegister, 0);
+ break;
+ case SizeHalfWord:
+ as_gsshx(data, base, ScratchRegister, 0);
+ break;
+ case SizeWord:
+ as_gsswx(data, base, ScratchRegister, 0);
+ break;
+ case SizeDouble:
+ as_gssdx(data, base, ScratchRegister, 0);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+ return;
+ }
+
+ if (!Imm16::IsInSignedRange(address.offset)) {
+ ma_li(ScratchRegister, Imm32(address.offset));
+ as_daddu(ScratchRegister, address.base, ScratchRegister);
+ base = ScratchRegister;
+ encodedOffset = Imm16(0).encode();
+ } else {
+ encodedOffset = Imm16(address.offset).encode();
+ base = address.base;
+ }
+
+ switch (size) {
+ case SizeByte:
+ as_sb(data, base, encodedOffset);
+ break;
+ case SizeHalfWord:
+ as_sh(data, base, encodedOffset);
+ break;
+ case SizeWord:
+ as_sw(data, base, encodedOffset);
+ break;
+ case SizeDouble:
+ as_sd(data, base, encodedOffset);
+ break;
+ default:
+ MOZ_CRASH("Invalid argument for ma_store");
+ }
+}
+
+void MacroAssemblerMIPS64Compat::computeScaledAddress(const BaseIndex& address,
+ Register dest) {
+ int32_t shift = Imm32::ShiftOf(address.scale).value;
+ if (shift) {
+ ma_dsll(ScratchRegister, address.index, Imm32(shift));
+ as_daddu(dest, address.base, ScratchRegister);
+ } else {
+ as_daddu(dest, address.base, address.index);
+ }
+}
+
+void MacroAssemblerMIPS64Compat::computeEffectiveAddress(
+ const BaseIndex& address, Register dest) {
+ computeScaledAddress(address, dest);
+ if (address.offset) {
+ asMasm().addPtr(Imm32(address.offset), dest);
+ }
+}
+
+// Shortcut for when we know we're transferring 32 bits of data.
+void MacroAssemblerMIPS64::ma_pop(Register r) {
+ as_ld(r, StackPointer, 0);
+ as_daddiu(StackPointer, StackPointer, sizeof(intptr_t));
+}
+
+void MacroAssemblerMIPS64::ma_push(Register r) {
+ if (r == sp) {
+ // Pushing sp requires one more instruction.
+ ma_move(ScratchRegister, sp);
+ r = ScratchRegister;
+ }
+
+ as_daddiu(StackPointer, StackPointer, (int32_t) - sizeof(intptr_t));
+ as_sd(r, StackPointer, 0);
+}
+
+// Branches when done from within mips-specific code.
+void MacroAssemblerMIPS64::ma_b(Register lhs, ImmWord imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ if (imm.value <= INT32_MAX) {
+ ma_b(lhs, Imm32(uint32_t(imm.value)), label, c, jumpKind);
+ } else {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_li(ScratchRegister, imm);
+ ma_b(lhs, ScratchRegister, label, c, jumpKind);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_b(Register lhs, Address addr, Label* label,
+ Condition c, JumpKind jumpKind) {
+ MOZ_ASSERT(lhs != ScratchRegister);
+ ma_load(ScratchRegister, addr, SizeDouble);
+ ma_b(lhs, ScratchRegister, label, c, jumpKind);
+}
+
+void MacroAssemblerMIPS64::ma_b(Address addr, Imm32 imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ ma_load(SecondScratchReg, addr, SizeDouble);
+ ma_b(SecondScratchReg, imm, label, c, jumpKind);
+}
+
+void MacroAssemblerMIPS64::ma_b(Address addr, ImmGCPtr imm, Label* label,
+ Condition c, JumpKind jumpKind) {
+ ma_load(SecondScratchReg, addr, SizeDouble);
+ ma_b(SecondScratchReg, imm, label, c, jumpKind);
+}
+
+void MacroAssemblerMIPS64::ma_bal(Label* label, DelaySlotFill delaySlotFill) {
+ spew("branch .Llabel %p\n", label);
+ if (label->bound()) {
+ // Generate the long jump for calls because return address has to be
+ // the address after the reserved block.
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ma_liPatchable(ScratchRegister, ImmWord(LabelBase::INVALID_OFFSET));
+ as_jalr(ScratchRegister);
+ if (delaySlotFill == FillDelaySlot) {
+ as_nop();
+ }
+ return;
+ }
+
+ // Second word holds a pointer to the next branch in label's chain.
+ uint32_t nextInChain =
+ label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+
+ // Make the whole branch continous in the buffer. The '6'
+ // instructions are writing at below (contain delay slot).
+ m_buffer.ensureSpace(6 * sizeof(uint32_t));
+
+ spew("bal .Llabel %p\n", label);
+ BufferOffset bo = writeInst(getBranchCode(BranchIsCall).encode());
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ // Leave space for long jump.
+ as_nop();
+ as_nop();
+ as_nop();
+ if (delaySlotFill == FillDelaySlot) {
+ as_nop();
+ }
+}
+
+void MacroAssemblerMIPS64::branchWithCode(InstImm code, Label* label,
+ JumpKind jumpKind) {
+ // simply output the pointer of one label as its id,
+ // notice that after one label destructor, the pointer will be reused.
+ spew("branch .Llabel %p", label);
+ MOZ_ASSERT(code.encode() !=
+ InstImm(op_regimm, zero, rt_bgezal, BOffImm16(0)).encode());
+ InstImm inst_beq = InstImm(op_beq, zero, zero, BOffImm16(0));
+
+ if (label->bound()) {
+ int32_t offset = label->offset() - m_buffer.nextOffset().getOffset();
+
+ if (BOffImm16::IsInRange(offset)) {
+ jumpKind = ShortJump;
+ }
+
+ if (jumpKind == ShortJump) {
+ MOZ_ASSERT(BOffImm16::IsInRange(offset));
+ code.setBOffImm16(BOffImm16(offset));
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ writeInst(code.encode());
+ as_nop();
+ return;
+ }
+
+ if (code.encode() == inst_beq.encode()) {
+ // Handle long jump
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ma_liPatchable(ScratchRegister, ImmWord(LabelBase::INVALID_OFFSET));
+ as_jr(ScratchRegister);
+ as_nop();
+ return;
+ }
+
+ // Handle long conditional branch, the target offset is based on self,
+ // point to next instruction of nop at below.
+ spew("invert branch .Llabel %p", label);
+ InstImm code_r = invertBranch(code, BOffImm16(7 * sizeof(uint32_t)));
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code_r);
+#endif
+ writeInst(code_r.encode());
+ // No need for a "nop" here because we can clobber scratch.
+ addLongJump(nextOffset(), BufferOffset(label->offset()));
+ ma_liPatchable(ScratchRegister, ImmWord(LabelBase::INVALID_OFFSET));
+ as_jr(ScratchRegister);
+ as_nop();
+ return;
+ }
+
+ // Generate open jump and link it to a label.
+
+ // Second word holds a pointer to the next branch in label's chain.
+ uint32_t nextInChain =
+ label->used() ? label->offset() : LabelBase::INVALID_OFFSET;
+
+ if (jumpKind == ShortJump) {
+ // Make the whole branch continous in the buffer.
+ m_buffer.ensureSpace(2 * sizeof(uint32_t));
+
+ // Indicate that this is short jump with offset 4.
+ code.setBOffImm16(BOffImm16(4));
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ BufferOffset bo = writeInst(code.encode());
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ return;
+ }
+
+ bool conditional = code.encode() != inst_beq.encode();
+
+ // Make the whole branch continous in the buffer. The '7'
+ // instructions are writing at below (contain conditional nop).
+ m_buffer.ensureSpace(7 * sizeof(uint32_t));
+
+#ifdef JS_JITSPEW
+ decodeBranchInstAndSpew(code);
+#endif
+ BufferOffset bo = writeInst(code.encode());
+ writeInst(nextInChain);
+ if (!oom()) {
+ label->use(bo.getOffset());
+ }
+ // Leave space for potential long jump.
+ as_nop();
+ as_nop();
+ as_nop();
+ as_nop();
+ if (conditional) {
+ as_nop();
+ }
+}
+
+void MacroAssemblerMIPS64::ma_cmp_set(Register rd, Register rs, ImmWord imm,
+ Condition c) {
+ if (imm.value <= INT32_MAX) {
+ ma_cmp_set(rd, rs, Imm32(uint32_t(imm.value)), c);
+ } else {
+ ma_li(ScratchRegister, imm);
+ ma_cmp_set(rd, rs, ScratchRegister, c);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_cmp_set(Register rd, Address address, ImmWord imm,
+ Condition c) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_load(scratch2, address, SizeDouble);
+ ma_cmp_set(rd, scratch2, imm, c);
+}
+
+void MacroAssemblerMIPS64::ma_cmp_set(Register rd, Register rs, ImmPtr imm,
+ Condition c) {
+ ma_cmp_set(rd, rs, ImmWord(uintptr_t(imm.value)), c);
+}
+
+void MacroAssemblerMIPS64::ma_cmp_set(Register rd, Address address, Imm32 imm,
+ Condition c) {
+ SecondScratchRegisterScope scratch2(asMasm());
+ ma_load(scratch2, address, SizeWord, SignExtend);
+ ma_cmp_set(rd, scratch2, imm, c);
+}
+
+// fp instructions
+void MacroAssemblerMIPS64::ma_lid(FloatRegister dest, double value) {
+ ImmWord imm(mozilla::BitwiseCast<uint64_t>(value));
+
+ if (imm.value != 0) {
+ ma_li(ScratchRegister, imm);
+ moveToDouble(ScratchRegister, dest);
+ } else {
+ moveToDouble(zero, dest);
+ }
+}
+
+void MacroAssemblerMIPS64::ma_mv(FloatRegister src, ValueOperand dest) {
+ as_dmfc1(dest.valueReg(), src);
+}
+
+void MacroAssemblerMIPS64::ma_mv(ValueOperand src, FloatRegister dest) {
+ as_dmtc1(src.valueReg(), dest);
+}
+
+void MacroAssemblerMIPS64::ma_ls(FloatRegister ft, Address address) {
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_lwc1(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ if (isLoongson()) {
+ as_gslsx(ft, address.base, ScratchRegister, 0);
+ } else {
+ as_daddu(ScratchRegister, address.base, ScratchRegister);
+ as_lwc1(ft, ScratchRegister, 0);
+ }
+ }
+}
+
+void MacroAssemblerMIPS64::ma_ld(FloatRegister ft, Address address) {
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_ldc1(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ if (isLoongson()) {
+ as_gsldx(ft, address.base, ScratchRegister, 0);
+ } else {
+ as_daddu(ScratchRegister, address.base, ScratchRegister);
+ as_ldc1(ft, ScratchRegister, 0);
+ }
+ }
+}
+
+void MacroAssemblerMIPS64::ma_sd(FloatRegister ft, Address address) {
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_sdc1(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ if (isLoongson()) {
+ as_gssdx(ft, address.base, ScratchRegister, 0);
+ } else {
+ as_daddu(ScratchRegister, address.base, ScratchRegister);
+ as_sdc1(ft, ScratchRegister, 0);
+ }
+ }
+}
+
+void MacroAssemblerMIPS64::ma_ss(FloatRegister ft, Address address) {
+ if (Imm16::IsInSignedRange(address.offset)) {
+ as_swc1(ft, address.base, address.offset);
+ } else {
+ MOZ_ASSERT(address.base != ScratchRegister);
+ ma_li(ScratchRegister, Imm32(address.offset));
+ if (isLoongson()) {
+ as_gsssx(ft, address.base, ScratchRegister, 0);
+ } else {
+ as_daddu(ScratchRegister, address.base, ScratchRegister);
+ as_swc1(ft, ScratchRegister, 0);
+ }
+ }
+}
+
+void MacroAssemblerMIPS64::ma_pop(FloatRegister f) {
+ as_ldc1(f, StackPointer, 0);
+ as_daddiu(StackPointer, StackPointer, sizeof(double));
+}
+
+void MacroAssemblerMIPS64::ma_push(FloatRegister f) {
+ as_daddiu(StackPointer, StackPointer, (int32_t) - sizeof(double));
+ as_sdc1(f, StackPointer, 0);
+}
+
+bool MacroAssemblerMIPS64Compat::buildOOLFakeExitFrame(void* fakeReturnAddr) {
+ asMasm().PushFrameDescriptor(FrameType::IonJS); // descriptor_
+ asMasm().Push(ImmPtr(fakeReturnAddr));
+ asMasm().Push(FramePointer);
+ return true;
+}
+
+void MacroAssemblerMIPS64Compat::move32(Imm32 imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerMIPS64Compat::move32(Register src, Register dest) {
+ ma_move(dest, src);
+}
+
+void MacroAssemblerMIPS64Compat::movePtr(Register src, Register dest) {
+ ma_move(dest, src);
+}
+void MacroAssemblerMIPS64Compat::movePtr(ImmWord imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerMIPS64Compat::movePtr(ImmGCPtr imm, Register dest) {
+ ma_li(dest, imm);
+}
+
+void MacroAssemblerMIPS64Compat::movePtr(ImmPtr imm, Register dest) {
+ movePtr(ImmWord(uintptr_t(imm.value)), dest);
+}
+void MacroAssemblerMIPS64Compat::movePtr(wasm::SymbolicAddress imm,
+ Register dest) {
+ append(wasm::SymbolicAccess(CodeOffset(nextOffset().getOffset()), imm));
+ ma_liPatchable(dest, ImmWord(-1));
+}
+
+void MacroAssemblerMIPS64Compat::load8ZeroExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeByte, ZeroExtend);
+}
+
+void MacroAssemblerMIPS64Compat::load8ZeroExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeByte, ZeroExtend);
+}
+
+void MacroAssemblerMIPS64Compat::load8SignExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeByte, SignExtend);
+}
+
+void MacroAssemblerMIPS64Compat::load8SignExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeByte, SignExtend);
+}
+
+void MacroAssemblerMIPS64Compat::load16ZeroExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeHalfWord, ZeroExtend);
+}
+
+void MacroAssemblerMIPS64Compat::load16ZeroExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeHalfWord, ZeroExtend);
+}
+
+void MacroAssemblerMIPS64Compat::load16SignExtend(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeHalfWord, SignExtend);
+}
+
+void MacroAssemblerMIPS64Compat::load16SignExtend(const BaseIndex& src,
+ Register dest) {
+ ma_load(dest, src, SizeHalfWord, SignExtend);
+}
+
+void MacroAssemblerMIPS64Compat::load32(const Address& address, Register dest) {
+ ma_load(dest, address, SizeWord);
+}
+
+void MacroAssemblerMIPS64Compat::load32(const BaseIndex& address,
+ Register dest) {
+ ma_load(dest, address, SizeWord);
+}
+
+void MacroAssemblerMIPS64Compat::load32(AbsoluteAddress address,
+ Register dest) {
+ movePtr(ImmPtr(address.addr), ScratchRegister);
+ load32(Address(ScratchRegister, 0), dest);
+}
+
+void MacroAssemblerMIPS64Compat::load32(wasm::SymbolicAddress address,
+ Register dest) {
+ movePtr(address, ScratchRegister);
+ load32(Address(ScratchRegister, 0), dest);
+}
+
+void MacroAssemblerMIPS64Compat::loadPtr(const Address& address,
+ Register dest) {
+ ma_load(dest, address, SizeDouble);
+}
+
+void MacroAssemblerMIPS64Compat::loadPtr(const BaseIndex& src, Register dest) {
+ ma_load(dest, src, SizeDouble);
+}
+
+void MacroAssemblerMIPS64Compat::loadPtr(AbsoluteAddress address,
+ Register dest) {
+ movePtr(ImmPtr(address.addr), ScratchRegister);
+ loadPtr(Address(ScratchRegister, 0), dest);
+}
+
+void MacroAssemblerMIPS64Compat::loadPtr(wasm::SymbolicAddress address,
+ Register dest) {
+ movePtr(address, ScratchRegister);
+ loadPtr(Address(ScratchRegister, 0), dest);
+}
+
+void MacroAssemblerMIPS64Compat::loadPrivate(const Address& address,
+ Register dest) {
+ loadPtr(address, dest);
+}
+
+void MacroAssemblerMIPS64Compat::loadUnalignedDouble(
+ const wasm::MemoryAccessDesc& access, const BaseIndex& src, Register temp,
+ FloatRegister dest) {
+ computeScaledAddress(src, SecondScratchReg);
+ BufferOffset load;
+ if (Imm16::IsInSignedRange(src.offset) &&
+ Imm16::IsInSignedRange(src.offset + 7)) {
+ load = as_ldl(temp, SecondScratchReg, src.offset + 7);
+ as_ldr(temp, SecondScratchReg, src.offset);
+ } else {
+ ma_li(ScratchRegister, Imm32(src.offset));
+ as_daddu(ScratchRegister, SecondScratchReg, ScratchRegister);
+ load = as_ldl(temp, ScratchRegister, 7);
+ as_ldr(temp, ScratchRegister, 0);
+ }
+ append(access, load.getOffset());
+ moveToDouble(temp, dest);
+}
+
+void MacroAssemblerMIPS64Compat::loadUnalignedFloat32(
+ const wasm::MemoryAccessDesc& access, const BaseIndex& src, Register temp,
+ FloatRegister dest) {
+ computeScaledAddress(src, SecondScratchReg);
+ BufferOffset load;
+ if (Imm16::IsInSignedRange(src.offset) &&
+ Imm16::IsInSignedRange(src.offset + 3)) {
+ load = as_lwl(temp, SecondScratchReg, src.offset + 3);
+ as_lwr(temp, SecondScratchReg, src.offset);
+ } else {
+ ma_li(ScratchRegister, Imm32(src.offset));
+ as_daddu(ScratchRegister, SecondScratchReg, ScratchRegister);
+ load = as_lwl(temp, ScratchRegister, 3);
+ as_lwr(temp, ScratchRegister, 0);
+ }
+ append(access, load.getOffset());
+ moveToFloat32(temp, dest);
+}
+
+void MacroAssemblerMIPS64Compat::store8(Imm32 imm, const Address& address) {
+ ma_li(SecondScratchReg, imm);
+ ma_store(SecondScratchReg, address, SizeByte);
+}
+
+void MacroAssemblerMIPS64Compat::store8(Register src, const Address& address) {
+ ma_store(src, address, SizeByte);
+}
+
+void MacroAssemblerMIPS64Compat::store8(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeByte);
+}
+
+void MacroAssemblerMIPS64Compat::store8(Register src, const BaseIndex& dest) {
+ ma_store(src, dest, SizeByte);
+}
+
+void MacroAssemblerMIPS64Compat::store16(Imm32 imm, const Address& address) {
+ ma_li(SecondScratchReg, imm);
+ ma_store(SecondScratchReg, address, SizeHalfWord);
+}
+
+void MacroAssemblerMIPS64Compat::store16(Register src, const Address& address) {
+ ma_store(src, address, SizeHalfWord);
+}
+
+void MacroAssemblerMIPS64Compat::store16(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeHalfWord);
+}
+
+void MacroAssemblerMIPS64Compat::store16(Register src,
+ const BaseIndex& address) {
+ ma_store(src, address, SizeHalfWord);
+}
+
+void MacroAssemblerMIPS64Compat::store32(Register src,
+ AbsoluteAddress address) {
+ movePtr(ImmPtr(address.addr), ScratchRegister);
+ store32(src, Address(ScratchRegister, 0));
+}
+
+void MacroAssemblerMIPS64Compat::store32(Register src, const Address& address) {
+ ma_store(src, address, SizeWord);
+}
+
+void MacroAssemblerMIPS64Compat::store32(Imm32 src, const Address& address) {
+ move32(src, SecondScratchReg);
+ ma_store(SecondScratchReg, address, SizeWord);
+}
+
+void MacroAssemblerMIPS64Compat::store32(Imm32 imm, const BaseIndex& dest) {
+ ma_store(imm, dest, SizeWord);
+}
+
+void MacroAssemblerMIPS64Compat::store32(Register src, const BaseIndex& dest) {
+ ma_store(src, dest, SizeWord);
+}
+
+template <typename T>
+void MacroAssemblerMIPS64Compat::storePtr(ImmWord imm, T address) {
+ ma_li(SecondScratchReg, imm);
+ ma_store(SecondScratchReg, address, SizeDouble);
+}
+
+template void MacroAssemblerMIPS64Compat::storePtr<Address>(ImmWord imm,
+ Address address);
+template void MacroAssemblerMIPS64Compat::storePtr<BaseIndex>(
+ ImmWord imm, BaseIndex address);
+
+template <typename T>
+void MacroAssemblerMIPS64Compat::storePtr(ImmPtr imm, T address) {
+ storePtr(ImmWord(uintptr_t(imm.value)), address);
+}
+
+template void MacroAssemblerMIPS64Compat::storePtr<Address>(ImmPtr imm,
+ Address address);
+template void MacroAssemblerMIPS64Compat::storePtr<BaseIndex>(
+ ImmPtr imm, BaseIndex address);
+
+template <typename T>
+void MacroAssemblerMIPS64Compat::storePtr(ImmGCPtr imm, T address) {
+ movePtr(imm, SecondScratchReg);
+ storePtr(SecondScratchReg, address);
+}
+
+template void MacroAssemblerMIPS64Compat::storePtr<Address>(ImmGCPtr imm,
+ Address address);
+template void MacroAssemblerMIPS64Compat::storePtr<BaseIndex>(
+ ImmGCPtr imm, BaseIndex address);
+
+void MacroAssemblerMIPS64Compat::storePtr(Register src,
+ const Address& address) {
+ ma_store(src, address, SizeDouble);
+}
+
+void MacroAssemblerMIPS64Compat::storePtr(Register src,
+ const BaseIndex& address) {
+ ma_store(src, address, SizeDouble);
+}
+
+void MacroAssemblerMIPS64Compat::storePtr(Register src, AbsoluteAddress dest) {
+ movePtr(ImmPtr(dest.addr), ScratchRegister);
+ storePtr(src, Address(ScratchRegister, 0));
+}
+
+void MacroAssemblerMIPS64Compat::storeUnalignedFloat32(
+ const wasm::MemoryAccessDesc& access, FloatRegister src, Register temp,
+ const BaseIndex& dest) {
+ computeScaledAddress(dest, SecondScratchReg);
+ moveFromFloat32(src, temp);
+ BufferOffset store;
+ if (Imm16::IsInSignedRange(dest.offset) &&
+ Imm16::IsInSignedRange(dest.offset + 3)) {
+ store = as_swl(temp, SecondScratchReg, dest.offset + 3);
+ as_swr(temp, SecondScratchReg, dest.offset);
+ } else {
+ ma_li(ScratchRegister, Imm32(dest.offset));
+ as_daddu(ScratchRegister, SecondScratchReg, ScratchRegister);
+ store = as_swl(temp, ScratchRegister, 3);
+ as_swr(temp, ScratchRegister, 0);
+ }
+ append(access, store.getOffset());
+}
+
+void MacroAssemblerMIPS64Compat::storeUnalignedDouble(
+ const wasm::MemoryAccessDesc& access, FloatRegister src, Register temp,
+ const BaseIndex& dest) {
+ computeScaledAddress(dest, SecondScratchReg);
+ moveFromDouble(src, temp);
+
+ BufferOffset store;
+ if (Imm16::IsInSignedRange(dest.offset) &&
+ Imm16::IsInSignedRange(dest.offset + 7)) {
+ store = as_sdl(temp, SecondScratchReg, dest.offset + 7);
+ as_sdr(temp, SecondScratchReg, dest.offset);
+ } else {
+ ma_li(ScratchRegister, Imm32(dest.offset));
+ as_daddu(ScratchRegister, SecondScratchReg, ScratchRegister);
+ store = as_sdl(temp, ScratchRegister, 7);
+ as_sdr(temp, ScratchRegister, 0);
+ }
+ append(access, store.getOffset());
+}
+
+void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
+ as_roundwd(ScratchDoubleReg, input);
+ ma_li(ScratchRegister, Imm32(255));
+ as_mfc1(output, ScratchDoubleReg);
+#ifdef MIPSR6
+ as_slti(SecondScratchReg, output, 0);
+ as_seleqz(output, output, SecondScratchReg);
+ as_sltiu(SecondScratchReg, output, 255);
+ as_selnez(output, output, SecondScratchReg);
+ as_seleqz(ScratchRegister, ScratchRegister, SecondScratchReg);
+ as_or(output, output, ScratchRegister);
+#else
+ zeroDouble(ScratchDoubleReg);
+ as_sltiu(SecondScratchReg, output, 255);
+ as_colt(DoubleFloat, ScratchDoubleReg, input);
+ // if res > 255; res = 255;
+ as_movz(output, ScratchRegister, SecondScratchReg);
+ // if !(input > 0); res = 0;
+ as_movf(output, zero);
+#endif
+}
+
+void MacroAssemblerMIPS64Compat::testNullSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_NULL), cond);
+}
+
+void MacroAssemblerMIPS64Compat::testObjectSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_OBJECT), cond);
+}
+
+void MacroAssemblerMIPS64Compat::testUndefinedSet(Condition cond,
+ const ValueOperand& value,
+ Register dest) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ splitTag(value, SecondScratchReg);
+ ma_cmp_set(dest, SecondScratchReg, ImmTag(JSVAL_TAG_UNDEFINED), cond);
+}
+
+void MacroAssemblerMIPS64Compat::unboxInt32(const ValueOperand& operand,
+ Register dest) {
+ ma_sll(dest, operand.valueReg(), Imm32(0));
+}
+
+void MacroAssemblerMIPS64Compat::unboxInt32(Register src, Register dest) {
+ ma_sll(dest, src, Imm32(0));
+}
+
+void MacroAssemblerMIPS64Compat::unboxInt32(const Address& src, Register dest) {
+ load32(Address(src.base, src.offset), dest);
+}
+
+void MacroAssemblerMIPS64Compat::unboxInt32(const BaseIndex& src,
+ Register dest) {
+ computeScaledAddress(src, SecondScratchReg);
+ load32(Address(SecondScratchReg, src.offset), dest);
+}
+
+void MacroAssemblerMIPS64Compat::unboxBoolean(const ValueOperand& operand,
+ Register dest) {
+ ma_dext(dest, operand.valueReg(), Imm32(0), Imm32(32));
+}
+
+void MacroAssemblerMIPS64Compat::unboxBoolean(Register src, Register dest) {
+ ma_dext(dest, src, Imm32(0), Imm32(32));
+}
+
+void MacroAssemblerMIPS64Compat::unboxBoolean(const Address& src,
+ Register dest) {
+ ma_load(dest, Address(src.base, src.offset), SizeWord, ZeroExtend);
+}
+
+void MacroAssemblerMIPS64Compat::unboxBoolean(const BaseIndex& src,
+ Register dest) {
+ computeScaledAddress(src, SecondScratchReg);
+ ma_load(dest, Address(SecondScratchReg, src.offset), SizeWord, ZeroExtend);
+}
+
+void MacroAssemblerMIPS64Compat::unboxDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ as_dmtc1(operand.valueReg(), dest);
+}
+
+void MacroAssemblerMIPS64Compat::unboxDouble(const Address& src,
+ FloatRegister dest) {
+ ma_ld(dest, Address(src.base, src.offset));
+}
+void MacroAssemblerMIPS64Compat::unboxDouble(const BaseIndex& src,
+ FloatRegister dest) {
+ SecondScratchRegisterScope scratch(asMasm());
+ loadPtr(src, scratch);
+ unboxDouble(ValueOperand(scratch), dest);
+}
+
+void MacroAssemblerMIPS64Compat::unboxString(const ValueOperand& operand,
+ Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_STRING);
+}
+
+void MacroAssemblerMIPS64Compat::unboxString(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+}
+
+void MacroAssemblerMIPS64Compat::unboxString(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+}
+
+void MacroAssemblerMIPS64Compat::unboxSymbol(const ValueOperand& operand,
+ Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void MacroAssemblerMIPS64Compat::unboxSymbol(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void MacroAssemblerMIPS64Compat::unboxSymbol(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+}
+
+void MacroAssemblerMIPS64Compat::unboxBigInt(const ValueOperand& operand,
+ Register dest) {
+ unboxNonDouble(operand, dest, JSVAL_TYPE_BIGINT);
+}
+
+void MacroAssemblerMIPS64Compat::unboxBigInt(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+}
+
+void MacroAssemblerMIPS64Compat::unboxBigInt(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+}
+
+void MacroAssemblerMIPS64Compat::unboxObject(const ValueOperand& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void MacroAssemblerMIPS64Compat::unboxObject(Register src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void MacroAssemblerMIPS64Compat::unboxObject(const Address& src,
+ Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+}
+
+void MacroAssemblerMIPS64Compat::unboxValue(const ValueOperand& src,
+ AnyRegister dest,
+ JSValueType type) {
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.valueReg(), dest.fpu());
+ ma_b(&end, ShortJump);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else {
+ unboxNonDouble(src, dest.gpr(), type);
+ }
+}
+
+void MacroAssemblerMIPS64Compat::boxDouble(FloatRegister src,
+ const ValueOperand& dest,
+ FloatRegister) {
+ as_dmfc1(dest.valueReg(), src);
+}
+
+void MacroAssemblerMIPS64Compat::boxNonDouble(JSValueType type, Register src,
+ const ValueOperand& dest) {
+ MOZ_ASSERT(src != dest.valueReg());
+ boxValue(type, src, dest.valueReg());
+}
+
+void MacroAssemblerMIPS64Compat::boolValueToDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ convertBoolToInt32(operand.valueReg(), ScratchRegister);
+ convertInt32ToDouble(ScratchRegister, dest);
+}
+
+void MacroAssemblerMIPS64Compat::int32ValueToDouble(const ValueOperand& operand,
+ FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+}
+
+void MacroAssemblerMIPS64Compat::boolValueToFloat32(const ValueOperand& operand,
+ FloatRegister dest) {
+ convertBoolToInt32(operand.valueReg(), ScratchRegister);
+ convertInt32ToFloat32(ScratchRegister, dest);
+}
+
+void MacroAssemblerMIPS64Compat::int32ValueToFloat32(
+ const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+}
+
+void MacroAssemblerMIPS64Compat::loadConstantFloat32(float f,
+ FloatRegister dest) {
+ ma_lis(dest, f);
+}
+
+void MacroAssemblerMIPS64Compat::loadInt32OrDouble(const Address& src,
+ FloatRegister dest) {
+ Label notInt32, end;
+ // If it's an int, convert it to double.
+ loadPtr(Address(src.base, src.offset), ScratchRegister);
+ ma_dsrl(SecondScratchReg, ScratchRegister, Imm32(JSVAL_TAG_SHIFT));
+ asMasm().branchTestInt32(Assembler::NotEqual, SecondScratchReg, &notInt32);
+ loadPtr(Address(src.base, src.offset), SecondScratchReg);
+ convertInt32ToDouble(SecondScratchReg, dest);
+ ma_b(&end, ShortJump);
+
+ // Not an int, just load as double.
+ bind(&notInt32);
+ unboxDouble(src, dest);
+ bind(&end);
+}
+
+void MacroAssemblerMIPS64Compat::loadInt32OrDouble(const BaseIndex& addr,
+ FloatRegister dest) {
+ Label notInt32, end;
+
+ // If it's an int, convert it to double.
+ computeScaledAddress(addr, SecondScratchReg);
+ // Since we only have one scratch, we need to stomp over it with the tag.
+ loadPtr(Address(SecondScratchReg, 0), ScratchRegister);
+ ma_dsrl(SecondScratchReg, ScratchRegister, Imm32(JSVAL_TAG_SHIFT));
+ asMasm().branchTestInt32(Assembler::NotEqual, SecondScratchReg, &notInt32);
+
+ computeScaledAddress(addr, SecondScratchReg);
+ loadPtr(Address(SecondScratchReg, 0), SecondScratchReg);
+ convertInt32ToDouble(SecondScratchReg, dest);
+ ma_b(&end, ShortJump);
+
+ // Not an int, just load as double.
+ bind(&notInt32);
+ // First, recompute the offset that had been stored in the scratch register
+ // since the scratch register was overwritten loading in the type.
+ computeScaledAddress(addr, SecondScratchReg);
+ unboxDouble(Address(SecondScratchReg, 0), dest);
+ bind(&end);
+}
+
+void MacroAssemblerMIPS64Compat::loadConstantDouble(double dp,
+ FloatRegister dest) {
+ ma_lid(dest, dp);
+}
+
+Register MacroAssemblerMIPS64Compat::extractObject(const Address& address,
+ Register scratch) {
+ loadPtr(Address(address.base, address.offset), scratch);
+ ma_dext(scratch, scratch, Imm32(0), Imm32(JSVAL_TAG_SHIFT));
+ return scratch;
+}
+
+Register MacroAssemblerMIPS64Compat::extractTag(const Address& address,
+ Register scratch) {
+ loadPtr(Address(address.base, address.offset), scratch);
+ ma_dext(scratch, scratch, Imm32(JSVAL_TAG_SHIFT),
+ Imm32(64 - JSVAL_TAG_SHIFT));
+ return scratch;
+}
+
+Register MacroAssemblerMIPS64Compat::extractTag(const BaseIndex& address,
+ Register scratch) {
+ computeScaledAddress(address, scratch);
+ return extractTag(Address(scratch, address.offset), scratch);
+}
+
+/////////////////////////////////////////////////////////////////
+// X86/X64-common/ARM/MIPS interface.
+/////////////////////////////////////////////////////////////////
+void MacroAssemblerMIPS64Compat::storeValue(ValueOperand val, Operand dst) {
+ storeValue(val, Address(Register::FromCode(dst.base()), dst.disp()));
+}
+
+void MacroAssemblerMIPS64Compat::storeValue(ValueOperand val,
+ const BaseIndex& dest) {
+ computeScaledAddress(dest, SecondScratchReg);
+ storeValue(val, Address(SecondScratchReg, dest.offset));
+}
+
+void MacroAssemblerMIPS64Compat::storeValue(JSValueType type, Register reg,
+ BaseIndex dest) {
+ computeScaledAddress(dest, ScratchRegister);
+
+ int32_t offset = dest.offset;
+ if (!Imm16::IsInSignedRange(offset)) {
+ ma_li(SecondScratchReg, Imm32(offset));
+ as_daddu(ScratchRegister, ScratchRegister, SecondScratchReg);
+ offset = 0;
+ }
+
+ storeValue(type, reg, Address(ScratchRegister, offset));
+}
+
+void MacroAssemblerMIPS64Compat::storeValue(ValueOperand val,
+ const Address& dest) {
+ storePtr(val.valueReg(), Address(dest.base, dest.offset));
+}
+
+void MacroAssemblerMIPS64Compat::storeValue(JSValueType type, Register reg,
+ Address dest) {
+ MOZ_ASSERT(dest.base != SecondScratchReg);
+
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ store32(reg, dest);
+ JSValueShiftedTag tag = (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
+ store32(((Imm64(tag)).secondHalf()), Address(dest.base, dest.offset + 4));
+ } else {
+ ma_li(SecondScratchReg, ImmTag(JSVAL_TYPE_TO_TAG(type)));
+ ma_dsll(SecondScratchReg, SecondScratchReg, Imm32(JSVAL_TAG_SHIFT));
+ ma_dins(SecondScratchReg, reg, Imm32(0), Imm32(JSVAL_TAG_SHIFT));
+ storePtr(SecondScratchReg, Address(dest.base, dest.offset));
+ }
+}
+
+void MacroAssemblerMIPS64Compat::storeValue(const Value& val, Address dest) {
+ if (val.isGCThing()) {
+ writeDataRelocation(val);
+ movWithPatch(ImmWord(val.asRawBits()), SecondScratchReg);
+ } else {
+ ma_li(SecondScratchReg, ImmWord(val.asRawBits()));
+ }
+ storePtr(SecondScratchReg, Address(dest.base, dest.offset));
+}
+
+void MacroAssemblerMIPS64Compat::storeValue(const Value& val, BaseIndex dest) {
+ computeScaledAddress(dest, ScratchRegister);
+
+ int32_t offset = dest.offset;
+ if (!Imm16::IsInSignedRange(offset)) {
+ ma_li(SecondScratchReg, Imm32(offset));
+ as_daddu(ScratchRegister, ScratchRegister, SecondScratchReg);
+ offset = 0;
+ }
+ storeValue(val, Address(ScratchRegister, offset));
+}
+
+void MacroAssemblerMIPS64Compat::loadValue(const BaseIndex& addr,
+ ValueOperand val) {
+ computeScaledAddress(addr, SecondScratchReg);
+ loadValue(Address(SecondScratchReg, addr.offset), val);
+}
+
+void MacroAssemblerMIPS64Compat::loadValue(Address src, ValueOperand val) {
+ loadPtr(Address(src.base, src.offset), val.valueReg());
+}
+
+void MacroAssemblerMIPS64Compat::tagValue(JSValueType type, Register payload,
+ ValueOperand dest) {
+ MOZ_ASSERT(dest.valueReg() != ScratchRegister);
+ if (payload != dest.valueReg()) {
+ ma_move(dest.valueReg(), payload);
+ }
+ ma_li(ScratchRegister, ImmTag(JSVAL_TYPE_TO_TAG(type)));
+ ma_dins(dest.valueReg(), ScratchRegister, Imm32(JSVAL_TAG_SHIFT),
+ Imm32(64 - JSVAL_TAG_SHIFT));
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ ma_dins(dest.valueReg(), zero, Imm32(32), Imm32(JSVAL_TAG_SHIFT - 32));
+ }
+}
+
+void MacroAssemblerMIPS64Compat::pushValue(ValueOperand val) {
+ // Allocate stack slots for Value. One for each.
+ asMasm().subPtr(Imm32(sizeof(Value)), StackPointer);
+ // Store Value
+ storeValue(val, Address(StackPointer, 0));
+}
+
+void MacroAssemblerMIPS64Compat::pushValue(const Address& addr) {
+ // Load value before allocate stack, addr.base may be is sp.
+ loadPtr(Address(addr.base, addr.offset), ScratchRegister);
+ ma_dsubu(StackPointer, StackPointer, Imm32(sizeof(Value)));
+ storePtr(ScratchRegister, Address(StackPointer, 0));
+}
+
+void MacroAssemblerMIPS64Compat::popValue(ValueOperand val) {
+ as_ld(val.valueReg(), StackPointer, 0);
+ as_daddiu(StackPointer, StackPointer, sizeof(Value));
+}
+
+void MacroAssemblerMIPS64Compat::breakpoint() { as_break(0); }
+
+void MacroAssemblerMIPS64Compat::ensureDouble(const ValueOperand& source,
+ FloatRegister dest,
+ Label* failure) {
+ Label isDouble, done;
+ {
+ ScratchTagScope tag(asMasm(), source);
+ splitTagForTest(source, tag);
+ asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, tag, failure);
+ }
+
+ unboxInt32(source, ScratchRegister);
+ convertInt32ToDouble(ScratchRegister, dest);
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerMIPS64Compat::checkStackAlignment() {
+#ifdef DEBUG
+ Label aligned;
+ as_andi(ScratchRegister, sp, ABIStackAlignment - 1);
+ ma_b(ScratchRegister, zero, &aligned, Equal, ShortJump);
+ as_break(BREAK_STACK_UNALIGNED);
+ bind(&aligned);
+#endif
+}
+
+void MacroAssemblerMIPS64Compat::handleFailureWithHandlerTail(
+ Label* profilerExitTail, Label* bailoutTail) {
+ // Reserve space for exception information.
+ int size = (sizeof(ResumeFromException) + ABIStackAlignment) &
+ ~(ABIStackAlignment - 1);
+ asMasm().subPtr(Imm32(size), StackPointer);
+ ma_move(a0, StackPointer); // Use a0 since it is a first function argument
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(a1);
+ asMasm().passABIArg(a0);
+ asMasm().callWithABI<Fn, HandleException>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ // Already clobbered a0, so use it...
+ load32(Address(StackPointer, ResumeFromException::offsetOfKind()), a0);
+ asMasm().branch32(Assembler::Equal, a0,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, a0,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, a0,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, a0, Imm32(ExceptionResumeKind::WasmCatch),
+ &wasmCatch);
+
+ breakpoint(); // Invalid kind.
+
+ // No exception handler. Load the error value, restore state and return from
+ // the entry frame.
+ bind(&entryFrame);
+ asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+
+ // We're going to be returning by the ion calling convention
+ ma_pop(ra);
+ as_jr(ra);
+ as_nop();
+
+ // If we found a catch handler, this must be a baseline frame. Restore
+ // state and jump to the catch block.
+ bind(&catch_);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfTarget()), a0);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ jump(a0);
+
+ // If we found a finally block, this must be a baseline frame. Push two
+ // values expected by the finally block: the exception and BooleanValue(true).
+ bind(&finally);
+ ValueOperand exception = ValueOperand(a1);
+ loadValue(Address(sp, ResumeFromException::offsetOfException()), exception);
+
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a0);
+ loadPtr(Address(sp, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(sp, ResumeFromException::offsetOfStackPointer()), sp);
+
+ pushValue(exception);
+ pushValue(BooleanValue(true));
+ jump(a0);
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ loadValue(Address(FramePointer, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ jump(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(Address(StackPointer, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ // Test if profiler enabled.
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ ma_move(StackPointer, FramePointer);
+ pop(FramePointer);
+ ret();
+
+ // If we are bailing out to baseline to handle an exception, jump to
+ // the bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
+ bind(&bailout);
+ loadPtr(Address(sp, ResumeFromException::offsetOfBailoutInfo()), a2);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ ma_li(ReturnReg, Imm32(1));
+ jump(bailoutTail);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ ret();
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+ loadPtr(Address(sp, ResumeFromException::offsetOfTarget()), a1);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfFramePointer()),
+ FramePointer);
+ loadPtr(Address(StackPointer, ResumeFromException::offsetOfStackPointer()),
+ StackPointer);
+ jump(a1);
+}
+
+CodeOffset MacroAssemblerMIPS64Compat::toggledJump(Label* label) {
+ CodeOffset ret(nextOffset().getOffset());
+ ma_b(label);
+ return ret;
+}
+
+CodeOffset MacroAssemblerMIPS64Compat::toggledCall(JitCode* target,
+ bool enabled) {
+ BufferOffset bo = nextOffset();
+ CodeOffset offset(bo.getOffset());
+ addPendingJump(bo, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(ScratchRegister, ImmPtr(target->raw()));
+ if (enabled) {
+ as_jalr(ScratchRegister);
+ as_nop();
+ } else {
+ as_nop();
+ as_nop();
+ }
+ MOZ_ASSERT_IF(!oom(), nextOffset().getOffset() - offset.offset() ==
+ ToggledCallSize(nullptr));
+ return offset;
+}
+
+void MacroAssemblerMIPS64Compat::profilerEnterFrame(Register framePtr,
+ Register scratch) {
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr,
+ Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr),
+ Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerMIPS64Compat::profilerExitFrame() {
+ jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) {
+ if (imm32.value) {
+ asMasm().subPtr(imm32, StackPointer);
+ }
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// Stack manipulation functions.
+
+size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
+ return set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
+}
+
+void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
+ int32_t diff =
+ set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
+ const int32_t reserved = diff;
+
+ reserveStack(reserved);
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diff -= sizeof(intptr_t);
+ storePtr(*iter, Address(StackPointer, diff));
+ }
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush());
+ iter.more(); ++iter) {
+ diff -= sizeof(double);
+ storeDouble(*iter, Address(StackPointer, diff));
+ }
+ MOZ_ASSERT(diff == 0);
+}
+
+void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
+ LiveRegisterSet ignore) {
+ int32_t diff =
+ set.gprs().size() * sizeof(intptr_t) + set.fpus().getPushSizeInBytes();
+ const int32_t reserved = diff;
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diff -= sizeof(intptr_t);
+ if (!ignore.has(*iter)) {
+ loadPtr(Address(StackPointer, diff), *iter);
+ }
+ }
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ for (FloatRegisterBackwardIterator iter(set.fpus().reduceSetForPush());
+ iter.more(); ++iter) {
+ diff -= sizeof(double);
+ if (!ignore.has(*iter)) {
+ loadDouble(Address(StackPointer, diff), *iter);
+ }
+ }
+ MOZ_ASSERT(diff == 0);
+ freeStack(reserved);
+}
+
+void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
+ Register) {
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+ unsigned numFpu = fpuSet.size();
+ int32_t diffF = fpuSet.getPushSizeInBytes();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+
+ MOZ_ASSERT(dest.offset >= diffG + diffF);
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ dest.offset -= sizeof(intptr_t);
+ storePtr(*iter, dest);
+ }
+ MOZ_ASSERT(diffG == 0);
+
+#ifdef ENABLE_WASM_SIMD
+# error "Needs more careful logic if SIMD is enabled"
+#endif
+
+ for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ diffF -= reg.size();
+ numFpu -= 1;
+ dest.offset -= reg.size();
+ if (reg.isDouble()) {
+ storeDouble(reg, dest);
+ } else if (reg.isSingle()) {
+ storeFloat32(reg, dest);
+ } else {
+ MOZ_CRASH("Unknown register type.");
+ }
+ }
+ MOZ_ASSERT(numFpu == 0);
+ diffF -= diffF % sizeof(uintptr_t);
+ MOZ_ASSERT(diffF == 0);
+}
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+ MOZ_ASSERT(!IsCompilingWasm(), "wasm should only use aligned ABI calls");
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ ma_move(scratch, StackPointer);
+
+ // Force sp to be aligned
+ asMasm().subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ ma_and(StackPointer, StackPointer, Imm32(~(ABIStackAlignment - 1)));
+ storePtr(scratch, Address(StackPointer, 0));
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ // Reserve place for $ra.
+ stackForCall += sizeof(intptr_t);
+
+ if (dynamicAlignment_) {
+ stackForCall += ComputeByteAlignment(stackForCall, ABIStackAlignment);
+ } else {
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(
+ stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+ }
+
+ *stackAdjust = stackForCall;
+ reserveStack(stackForCall);
+
+ // Save $ra because call is going to clobber it. Restore it in
+ // callWithABIPost. NOTE: This is needed for calls from SharedIC.
+ // Maybe we can do this differently.
+ storePtr(ra, Address(StackPointer, stackForCall - sizeof(intptr_t)));
+
+ // Position all arguments.
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_) {
+ return;
+ }
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool callFromWasm) {
+ // Restore ra value (as stored in callWithABIPre()).
+ loadPtr(Address(StackPointer, stackAdjust - sizeof(intptr_t)), ra);
+
+ if (dynamicAlignment_) {
+ // Restore sp value from stack (as stored in setupUnalignedABICall()).
+ loadPtr(Address(StackPointer, stackAdjust), StackPointer);
+ // Use adjustFrame instead of freeStack because we already restored sp.
+ adjustFrame(-stackAdjust);
+ } else {
+ freeStack(stackAdjust);
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ // Load the callee in t9, no instruction between the lw and call
+ // should clobber it. Note that we can't use fun.base because it may
+ // be one of the IntArg registers clobbered before the call.
+ ma_move(t9, fun);
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(t9);
+ callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ // Load the callee in t9, as above.
+ loadPtr(Address(fun.base, fun.offset), t9);
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(t9);
+ callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Move
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ boxNonDouble(ValueTypeFromMIRType(type), reg.gpr(), dest);
+ return;
+ }
+
+ FloatRegister scratch = ScratchDoubleReg;
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ boxDouble(freg, dest, scratch);
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ if (src == dest) {
+ return;
+ }
+ movePtr(src.valueReg(), dest.valueReg());
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ if (!src.isGCThing()) {
+ ma_li(dest.valueReg(), ImmWord(src.asRawBits()));
+ return;
+ }
+
+ writeDataRelocation(src);
+ movWithPatch(ImmWord(src.asRawBits()), dest.valueReg());
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ branchValueIsNurseryCellImpl(cond, address, temp, label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ branchValueIsNurseryCellImpl(cond, value, temp, label);
+}
+
+template <typename T>
+void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
+ const T& value, Register temp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ Label done;
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+
+ // temp may be InvalidReg, use scratch2 instead.
+ SecondScratchRegisterScope scratch2(*this);
+
+ getGCThingValueChunk(value, scratch2);
+ loadPtr(Address(scratch2, gc::ChunkStoreBufferOffset), scratch2);
+ branchPtr(InvertCondition(cond), scratch2, ImmWord(0), label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(lhs.valueReg() != scratch);
+ moveValue(rhs, ValueOperand(scratch));
+ ma_b(lhs.valueReg(), scratch, label, cond);
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ boxDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ if (value.constant()) {
+ storeValue(value.value(), dest);
+ } else {
+ storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
+ dest);
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+void MacroAssembler::PushBoxed(FloatRegister reg) {
+ subFromStackPtr(Imm32(sizeof(double)));
+ boxDouble(reg, Address(getStackPointer(), 0));
+ adjustFrame(sizeof(double));
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* ok) {
+ ma_b(index, boundsCheckLimit, ok, cond);
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* ok) {
+ SecondScratchRegisterScope scratch2(*this);
+ load32(boundsCheckLimit, scratch2);
+ ma_b(index, scratch2, ok, cond);
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok) {
+ ma_b(index.reg, boundsCheckLimit.reg, ok, cond);
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok) {
+ SecondScratchRegisterScope scratch2(*this);
+ loadPtr(boundsCheckLimit, scratch2);
+ ma_b(index.reg, scratch2, ok, cond);
+}
+
+void MacroAssembler::widenInt32(Register r) {
+ // I *think* this is correct. It may be redundant.
+ move32To64SignExtend(r, Register64(r));
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ as_truncld(ScratchDoubleReg, input);
+ moveFromDouble(ScratchDoubleReg, output);
+ ma_dsrl(ScratchRegister, output, Imm32(32));
+ as_sll(output, output, 0);
+ ma_b(ScratchRegister, Imm32(0), oolEntry, Assembler::NotEqual);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ as_truncls(ScratchDoubleReg, input);
+ moveFromDouble(ScratchDoubleReg, output);
+ ma_dsrl(ScratchRegister, output, Imm32(32));
+ as_sll(output, output, 0);
+ ma_b(ScratchRegister, Imm32(0), oolEntry, Assembler::NotEqual);
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output) {
+ wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, InvalidReg);
+}
+
+void MacroAssembler::wasmUnalignedLoadI64(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr,
+ Register ptrScratch,
+ Register64 output, Register tmp) {
+ wasmLoadI64Impl(access, memoryBase, ptr, ptrScratch, output, tmp);
+}
+
+void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch) {
+ wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, InvalidReg);
+}
+
+void MacroAssembler::wasmUnalignedStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value,
+ Register memoryBase, Register ptr,
+ Register ptrScratch, Register tmp) {
+ wasmStoreI64Impl(access, value, memoryBase, ptr, ptrScratch, tmp);
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_ASSERT(tempDouble.isInvalid());
+
+ as_truncld(ScratchDoubleReg, input);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromDouble(ScratchDoubleReg, output.reg);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_b(ScratchRegister, Imm32(0), oolEntry, Assembler::NotEqual);
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt64(
+ FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_ASSERT(tempDouble.isInvalid());
+ Register output = output_.reg;
+
+ Label done;
+
+ as_truncld(ScratchDoubleReg, input);
+ // ma_li INT64_MAX
+ ma_li(SecondScratchReg, Imm32(-1));
+ ma_dext(SecondScratchReg, SecondScratchReg, Imm32(0), Imm32(63));
+ moveFromDouble(ScratchDoubleReg, output);
+ // For numbers in -1.[ : ]INT64_MAX range do nothing more
+ ma_b(output, SecondScratchReg, &done, Assembler::Below, ShortJump);
+
+ loadConstantDouble(double(INT64_MAX + 1ULL), ScratchDoubleReg);
+ // ma_li INT64_MIN
+ ma_daddu(SecondScratchReg, Imm32(1));
+ as_subd(ScratchDoubleReg, input, ScratchDoubleReg);
+ as_truncld(ScratchDoubleReg, ScratchDoubleReg);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromDouble(ScratchDoubleReg, output);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_daddu(output, SecondScratchReg);
+
+ // Guard against negative values that result in 0 due the precision loss.
+ as_sltiu(SecondScratchReg, output, 1);
+ ma_or(ScratchRegister, SecondScratchReg);
+
+ ma_b(ScratchRegister, Imm32(0), oolEntry, Assembler::NotEqual);
+
+ bind(&done);
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempFloat) {
+ MOZ_ASSERT(tempFloat.isInvalid());
+
+ as_truncls(ScratchDoubleReg, input);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromDouble(ScratchDoubleReg, output.reg);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_b(ScratchRegister, Imm32(0), oolEntry, Assembler::NotEqual);
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt64(
+ FloatRegister input, Register64 output_, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempFloat) {
+ MOZ_ASSERT(tempFloat.isInvalid());
+ Register output = output_.reg;
+
+ Label done;
+
+ as_truncls(ScratchDoubleReg, input);
+ // ma_li INT64_MAX
+ ma_li(SecondScratchReg, Imm32(-1));
+ ma_dext(SecondScratchReg, SecondScratchReg, Imm32(0), Imm32(63));
+ moveFromDouble(ScratchDoubleReg, output);
+ // For numbers in -1.[ : ]INT64_MAX range do nothing more
+ ma_b(output, SecondScratchReg, &done, Assembler::Below, ShortJump);
+
+ loadConstantFloat32(float(INT64_MAX + 1ULL), ScratchFloat32Reg);
+ // ma_li INT64_MIN
+ ma_daddu(SecondScratchReg, Imm32(1));
+ as_subs(ScratchFloat32Reg, input, ScratchFloat32Reg);
+ as_truncls(ScratchDoubleReg, ScratchFloat32Reg);
+ as_cfc1(ScratchRegister, Assembler::FCSR);
+ moveFromDouble(ScratchDoubleReg, output);
+ ma_ext(ScratchRegister, ScratchRegister, Assembler::CauseV, 1);
+ ma_daddu(output, SecondScratchReg);
+
+ // Guard against negative values that result in 0 due the precision loss.
+ as_sltiu(SecondScratchReg, output, 1);
+ ma_or(ScratchRegister, SecondScratchReg);
+
+ ma_b(ScratchRegister, Imm32(0), oolEntry, Assembler::NotEqual);
+
+ bind(&done);
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssemblerMIPS64Compat::wasmLoadI64Impl(
+ const wasm::MemoryAccessDesc& access, Register memoryBase, Register ptr,
+ Register ptrScratch, Register64 output, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ MOZ_ASSERT(!access.isZeroExtendSimd128Load());
+ MOZ_ASSERT(!access.isSplatSimd128Load());
+ MOZ_ASSERT(!access.isWidenSimd128Load());
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(ImmWord(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+
+ switch (access.type()) {
+ case Scalar::Int8:
+ isSigned = true;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ break;
+ case Scalar::Int32:
+ isSigned = true;
+ break;
+ case Scalar::Uint32:
+ isSigned = false;
+ break;
+ case Scalar::Int64:
+ isSigned = true;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ BaseIndex address(memoryBase, ptr, TimesOne);
+ if (IsUnaligned(access)) {
+ MOZ_ASSERT(tmp != InvalidReg);
+ asMasm().ma_load_unaligned(access, output.reg, address, tmp,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ return;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+ asMasm().ma_load(output.reg, address,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+void MacroAssemblerMIPS64Compat::wasmStoreI64Impl(
+ const wasm::MemoryAccessDesc& access, Register64 value, Register memoryBase,
+ Register ptr, Register ptrScratch, Register tmp) {
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < asMasm().wasmMaxOffsetGuardLimit());
+ MOZ_ASSERT_IF(offset, ptrScratch != InvalidReg);
+
+ // Maybe add the offset.
+ if (offset) {
+ asMasm().addPtr(ImmWord(offset), ptrScratch);
+ ptr = ptrScratch;
+ }
+
+ unsigned byteSize = access.byteSize();
+ bool isSigned;
+ switch (access.type()) {
+ case Scalar::Int8:
+ isSigned = true;
+ break;
+ case Scalar::Uint8:
+ isSigned = false;
+ break;
+ case Scalar::Int16:
+ isSigned = true;
+ break;
+ case Scalar::Uint16:
+ isSigned = false;
+ break;
+ case Scalar::Int32:
+ isSigned = true;
+ break;
+ case Scalar::Uint32:
+ isSigned = false;
+ break;
+ case Scalar::Int64:
+ isSigned = true;
+ break;
+ default:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ BaseIndex address(memoryBase, ptr, TimesOne);
+
+ if (IsUnaligned(access)) {
+ MOZ_ASSERT(tmp != InvalidReg);
+ asMasm().ma_store_unaligned(access, value.reg, address, tmp,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ return;
+ }
+
+ asMasm().memoryBarrierBefore(access.sync());
+ asMasm().ma_store(value.reg, address,
+ static_cast<LoadStoreSize>(8 * byteSize),
+ isSigned ? SignExtend : ZeroExtend);
+ asMasm().append(access, asMasm().size() - 4);
+ asMasm().memoryBarrierAfter(access.sync());
+}
+
+template <typename T>
+static void CompareExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, const T& mem,
+ Register64 expect, Register64 replace,
+ Register64 output) {
+ MOZ_ASSERT(expect != output && replace != output);
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ Label tryAgain;
+ Label exit;
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&tryAgain);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.as_lld(output.reg, SecondScratchReg, 0);
+
+ masm.ma_b(output.reg, expect.reg, &exit, Assembler::NotEqual, ShortJump);
+ masm.movePtr(replace.reg, ScratchRegister);
+ masm.as_scd(ScratchRegister, SecondScratchReg, 0);
+ masm.ma_b(ScratchRegister, ScratchRegister, &tryAgain, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+
+ masm.bind(&exit);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
+ output);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expect,
+ Register64 replace,
+ Register64 output) {
+ CompareExchange64(*this, &access, access.sync(), mem, expect, replace,
+ output);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization& sync,
+ const Address& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 expect,
+ Register64 replace, Register64 output) {
+ CompareExchange64(*this, nullptr, sync, mem, expect, replace, output);
+}
+
+template <typename T>
+static void AtomicExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, const T& mem,
+ Register64 value, Register64 output) {
+ MOZ_ASSERT(value != output);
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ Label tryAgain;
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&tryAgain);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_lld(output.reg, SecondScratchReg, 0);
+ masm.movePtr(value.reg, ScratchRegister);
+ masm.as_scd(ScratchRegister, SecondScratchReg, 0);
+ masm.ma_b(ScratchRegister, ScratchRegister, &tryAgain, Assembler::Zero,
+ ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+template <typename T>
+static void WasmAtomicExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc& access,
+ const T& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(masm, &access, access.sync(), mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 src,
+ Register64 output) {
+ WasmAtomicExchange64(*this, access, mem, src, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 src,
+ Register64 output) {
+ WasmAtomicExchange64(*this, access, mem, src, output);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization& sync,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, nullptr, sync, mem, value, output);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization& sync,
+ const BaseIndex& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, nullptr, sync, mem, value, output);
+}
+
+template <typename T>
+static void AtomicFetchOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const Synchronization& sync, AtomicOp op,
+ Register64 value, const T& mem, Register64 temp,
+ Register64 output) {
+ MOZ_ASSERT(value != output);
+ MOZ_ASSERT(value != temp);
+ masm.computeEffectiveAddress(mem, SecondScratchReg);
+
+ Label tryAgain;
+
+ masm.memoryBarrierBefore(sync);
+
+ masm.bind(&tryAgain);
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ masm.as_lld(output.reg, SecondScratchReg, 0);
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.as_daddu(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicFetchSubOp:
+ masm.as_dsubu(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicFetchAndOp:
+ masm.as_and(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicFetchOrOp:
+ masm.as_or(temp.reg, output.reg, value.reg);
+ break;
+ case AtomicFetchXorOp:
+ masm.as_xor(temp.reg, output.reg, value.reg);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+ masm.as_scd(temp.reg, SecondScratchReg, 0);
+ masm.ma_b(temp.reg, temp.reg, &tryAgain, Assembler::Zero, ShortJump);
+
+ masm.memoryBarrierAfter(sync);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, access.sync(), op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp) {
+ AtomicFetchOp64(*this, nullptr, sync, op, value, mem, temp, temp);
+}
+
+// ========================================================================
+// Convert floating point.
+
+void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) {
+ as_dmtc1(src.reg, dest);
+ as_cvtdl(dest, dest);
+}
+
+void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) {
+ as_dmtc1(src.reg, dest);
+ as_cvtsl(dest, dest);
+}
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return false; }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+ MacroAssemblerSpecific::convertUInt64ToDouble(src.reg, dest);
+}
+
+void MacroAssembler::convertUInt64ToFloat32(Register64 src_, FloatRegister dest,
+ Register temp) {
+ MOZ_ASSERT(temp == Register::Invalid());
+
+ Register src = src_.reg;
+ Label positive, done;
+ ma_b(src, src, &positive, NotSigned, ShortJump);
+
+ MOZ_ASSERT(src != ScratchRegister);
+ MOZ_ASSERT(src != SecondScratchReg);
+
+ ma_and(ScratchRegister, src, Imm32(1));
+ ma_dsrl(SecondScratchReg, src, Imm32(1));
+ ma_or(ScratchRegister, SecondScratchReg);
+ as_dmtc1(ScratchRegister, dest);
+ as_cvtsl(dest, dest);
+ addFloat32(dest, dest);
+ ma_b(&done, ShortJump);
+
+ bind(&positive);
+ as_dmtc1(src, dest);
+ as_cvtsl(dest, dest);
+
+ bind(&done);
+}
+
+//}}} check_macroassembler_style
diff --git a/js/src/jit/mips64/MacroAssembler-mips64.h b/js/src/jit/mips64/MacroAssembler-mips64.h
new file mode 100644
index 0000000000..3efb0b840b
--- /dev/null
+++ b/js/src/jit/mips64/MacroAssembler-mips64.h
@@ -0,0 +1,846 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_MacroAssembler_mips64_h
+#define jit_mips64_MacroAssembler_mips64_h
+
+#include "jit/mips-shared/MacroAssembler-mips-shared.h"
+#include "jit/MoveResolver.h"
+#include "vm/BytecodeUtil.h"
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+enum LiFlags {
+ Li64 = 0,
+ Li48 = 1,
+};
+
+struct ImmShiftedTag : public ImmWord {
+ explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {}
+
+ explicit ImmShiftedTag(JSValueType type)
+ : ImmWord(uintptr_t(JSValueShiftedTag(JSVAL_TYPE_TO_SHIFTED_TAG(type)))) {
+ }
+};
+
+struct ImmTag : public Imm32 {
+ ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {}
+};
+
+static constexpr ValueOperand JSReturnOperand{JSReturnReg};
+
+static const int defaultShift = 3;
+static_assert(1 << defaultShift == sizeof(JS::Value),
+ "The defaultShift is wrong");
+
+// See documentation for ScratchTagScope and ScratchTagScopeRelease in
+// MacroAssembler-x64.h.
+
+class ScratchTagScope : public SecondScratchRegisterScope {
+ public:
+ ScratchTagScope(MacroAssembler& masm, const ValueOperand&)
+ : SecondScratchRegisterScope(masm) {}
+};
+
+class ScratchTagScopeRelease {
+ ScratchTagScope* ts_;
+
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
+ ts_->release();
+ }
+ ~ScratchTagScopeRelease() { ts_->reacquire(); }
+};
+
+class MacroAssemblerMIPS64 : public MacroAssemblerMIPSShared {
+ public:
+ using MacroAssemblerMIPSShared::ma_b;
+ using MacroAssemblerMIPSShared::ma_cmp_set;
+ using MacroAssemblerMIPSShared::ma_ld;
+ using MacroAssemblerMIPSShared::ma_li;
+ using MacroAssemblerMIPSShared::ma_load;
+ using MacroAssemblerMIPSShared::ma_ls;
+ using MacroAssemblerMIPSShared::ma_sd;
+ using MacroAssemblerMIPSShared::ma_ss;
+ using MacroAssemblerMIPSShared::ma_store;
+ using MacroAssemblerMIPSShared::ma_sub32TestOverflow;
+
+ void ma_li(Register dest, CodeLabel* label);
+ void ma_li(Register dest, ImmWord imm);
+ void ma_liPatchable(Register dest, ImmPtr imm);
+ void ma_liPatchable(Register dest, ImmWord imm, LiFlags flags = Li48);
+
+ // Negate
+ void ma_dnegu(Register rd, Register rs);
+
+ // Shift operations
+ void ma_dsll(Register rd, Register rt, Imm32 shift);
+ void ma_dsrl(Register rd, Register rt, Imm32 shift);
+ void ma_dsra(Register rd, Register rt, Imm32 shift);
+ void ma_dror(Register rd, Register rt, Imm32 shift);
+ void ma_drol(Register rd, Register rt, Imm32 shift);
+
+ void ma_dsll(Register rd, Register rt, Register shift);
+ void ma_dsrl(Register rd, Register rt, Register shift);
+ void ma_dsra(Register rd, Register rt, Register shift);
+ void ma_dror(Register rd, Register rt, Register shift);
+ void ma_drol(Register rd, Register rt, Register shift);
+
+ void ma_dins(Register rt, Register rs, Imm32 pos, Imm32 size);
+ void ma_dext(Register rt, Register rs, Imm32 pos, Imm32 size);
+
+ // doubleword swap bytes
+ void ma_dsbh(Register rd, Register rt);
+ void ma_dshd(Register rd, Register rt);
+
+ void ma_dctz(Register rd, Register rs);
+
+ // load
+ void ma_load(Register dest, Address address, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // store
+ void ma_store(Register data, Address address, LoadStoreSize size = SizeWord,
+ LoadStoreExtension extension = SignExtend);
+
+ // arithmetic based ops
+ // add
+ void ma_daddu(Register rd, Register rs, Imm32 imm);
+ void ma_daddu(Register rd, Register rs);
+ void ma_daddu(Register rd, Imm32 imm);
+ void ma_add32TestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_add32TestOverflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+ void ma_addPtrTestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_addPtrTestOverflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+ void ma_addPtrTestOverflow(Register rd, Register rs, ImmWord imm,
+ Label* overflow);
+ void ma_addPtrTestCarry(Condition cond, Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_addPtrTestCarry(Condition cond, Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+ void ma_addPtrTestCarry(Condition cond, Register rd, Register rs, ImmWord imm,
+ Label* overflow);
+ // subtract
+ void ma_dsubu(Register rd, Register rs, Imm32 imm);
+ void ma_dsubu(Register rd, Register rs);
+ void ma_dsubu(Register rd, Imm32 imm);
+ void ma_sub32TestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_subPtrTestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+ void ma_subPtrTestOverflow(Register rd, Register rs, Imm32 imm,
+ Label* overflow);
+
+ // multiplies. For now, there are only few that we care about.
+ void ma_dmult(Register rs, Imm32 imm);
+ void ma_mulPtrTestOverflow(Register rd, Register rs, Register rt,
+ Label* overflow);
+
+ // stack
+ void ma_pop(Register r);
+ void ma_push(Register r);
+
+ void branchWithCode(InstImm code, Label* label, JumpKind jumpKind);
+ // branches when done from within mips-specific code
+ void ma_b(Register lhs, ImmWord imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Register lhs, Address addr, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, Imm32 imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, ImmGCPtr imm, Label* l, Condition c,
+ JumpKind jumpKind = LongJump);
+ void ma_b(Address addr, Register rhs, Label* l, Condition c,
+ JumpKind jumpKind = LongJump) {
+ MOZ_ASSERT(rhs != ScratchRegister);
+ ma_load(ScratchRegister, addr, SizeDouble);
+ ma_b(ScratchRegister, rhs, l, c, jumpKind);
+ }
+
+ void ma_bal(Label* l, DelaySlotFill delaySlotFill = FillDelaySlot);
+
+ // fp instructions
+ void ma_lid(FloatRegister dest, double value);
+
+ void ma_mv(FloatRegister src, ValueOperand dest);
+ void ma_mv(ValueOperand src, FloatRegister dest);
+
+ void ma_ls(FloatRegister ft, Address address);
+ void ma_ld(FloatRegister ft, Address address);
+ void ma_sd(FloatRegister ft, Address address);
+ void ma_ss(FloatRegister ft, Address address);
+
+ void ma_pop(FloatRegister f);
+ void ma_push(FloatRegister f);
+
+ void ma_cmp_set(Register dst, Register lhs, ImmWord imm, Condition c);
+ void ma_cmp_set(Register dst, Address address, ImmWord imm, Condition c);
+ void ma_cmp_set(Register dst, Register lhs, ImmPtr imm, Condition c);
+ void ma_cmp_set(Register dst, Address address, Imm32 imm, Condition c);
+
+ // These functions abstract the access to high part of the double precision
+ // float register. They are intended to work on both 32 bit and 64 bit
+ // floating point coprocessor.
+ void moveToDoubleHi(Register src, FloatRegister dest) { as_mthc1(src, dest); }
+ void moveFromDoubleHi(FloatRegister src, Register dest) {
+ as_mfhc1(dest, src);
+ }
+
+ void moveToDouble(Register src, FloatRegister dest) { as_dmtc1(src, dest); }
+ void moveFromDouble(FloatRegister src, Register dest) { as_dmfc1(dest, src); }
+};
+
+class MacroAssembler;
+
+class MacroAssemblerMIPS64Compat : public MacroAssemblerMIPS64 {
+ public:
+ using MacroAssemblerMIPS64::call;
+
+ MacroAssemblerMIPS64Compat() {}
+
+ void convertBoolToInt32(Register source, Register dest);
+ void convertInt32ToDouble(Register src, FloatRegister dest);
+ void convertInt32ToDouble(const Address& src, FloatRegister dest);
+ void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest);
+ void convertUInt32ToDouble(Register src, FloatRegister dest);
+ void convertUInt32ToFloat32(Register src, FloatRegister dest);
+ void convertDoubleToFloat32(FloatRegister src, FloatRegister dest);
+ void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+ void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+
+ void convertFloat32ToDouble(FloatRegister src, FloatRegister dest);
+ void convertInt32ToFloat32(Register src, FloatRegister dest);
+ void convertInt32ToFloat32(const Address& src, FloatRegister dest);
+
+ void movq(Register rs, Register rd);
+
+ void computeScaledAddress(const BaseIndex& address, Register dest);
+
+ void computeEffectiveAddress(const Address& address, Register dest) {
+ ma_daddu(dest, address.base, Imm32(address.offset));
+ }
+
+ void computeEffectiveAddress(const BaseIndex& address, Register dest);
+
+ void j(Label* dest) { ma_b(dest); }
+
+ void mov(Register src, Register dest) { as_ori(dest, src, 0); }
+ void mov(ImmWord imm, Register dest) { ma_li(dest, imm); }
+ void mov(ImmPtr imm, Register dest) {
+ mov(ImmWord(uintptr_t(imm.value)), dest);
+ }
+ void mov(CodeLabel* label, Register dest) { ma_li(dest, label); }
+ void mov(Register src, Address dest) { MOZ_CRASH("NYI-IC"); }
+ void mov(Address src, Register dest) { MOZ_CRASH("NYI-IC"); }
+
+ void writeDataRelocation(const Value& val) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // TraceOneDataRelocation.
+ if (val.isGCThing()) {
+ gc::Cell* cell = val.toGCThing();
+ if (cell && gc::IsInsideNursery(cell)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(currentOffset());
+ }
+ }
+
+ void branch(JitCode* c) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ImmPtr(c->raw()), RelocationKind::JITCODE);
+ ma_liPatchable(ScratchRegister, ImmPtr(c->raw()));
+ as_jr(ScratchRegister);
+ as_nop();
+ }
+ void branch(const Register reg) {
+ as_jr(reg);
+ as_nop();
+ }
+ void nop() { as_nop(); }
+ void ret() {
+ ma_pop(ra);
+ as_jr(ra);
+ as_nop();
+ }
+ inline void retn(Imm32 n);
+ void push(Imm32 imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(ImmWord imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(ImmGCPtr imm) {
+ ma_li(ScratchRegister, imm);
+ ma_push(ScratchRegister);
+ }
+ void push(const Address& address) {
+ loadPtr(address, ScratchRegister);
+ ma_push(ScratchRegister);
+ }
+ void push(Register reg) { ma_push(reg); }
+ void push(FloatRegister reg) { ma_push(reg); }
+ void pop(Register reg) { ma_pop(reg); }
+ void pop(FloatRegister reg) { ma_pop(reg); }
+
+ // Emit a branch that can be toggled to a non-operation. On MIPS64 we use
+ // "andi" instruction to toggle the branch.
+ // See ToggleToJmp(), ToggleToCmp().
+ CodeOffset toggledJump(Label* label);
+
+ // Emit a "jalr" or "nop" instruction. ToggleCall can be used to patch
+ // this instruction.
+ CodeOffset toggledCall(JitCode* target, bool enabled);
+
+ static size_t ToggledCallSize(uint8_t* code) {
+ // Six instructions used in: MacroAssemblerMIPS64Compat::toggledCall
+ return 6 * sizeof(uint32_t);
+ }
+
+ CodeOffset pushWithPatch(ImmWord imm) {
+ CodeOffset offset = movWithPatch(imm, ScratchRegister);
+ ma_push(ScratchRegister);
+ return offset;
+ }
+
+ CodeOffset movWithPatch(ImmWord imm, Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ ma_liPatchable(dest, imm, Li64);
+ return offset;
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ CodeOffset offset = CodeOffset(currentOffset());
+ ma_liPatchable(dest, imm);
+ return offset;
+ }
+
+ void writeCodePointer(CodeLabel* label) {
+ label->patchAt()->bind(currentOffset());
+ label->setLinkMode(CodeLabel::RawPointer);
+ m_buffer.ensureSpace(sizeof(void*));
+ writeInst(-1);
+ writeInst(-1);
+ }
+
+ void jump(Label* label) { ma_b(label); }
+ void jump(Register reg) {
+ as_jr(reg);
+ as_nop();
+ }
+ void jump(const Address& address) {
+ loadPtr(address, ScratchRegister);
+ as_jr(ScratchRegister);
+ as_nop();
+ }
+
+ void jump(JitCode* code) { branch(code); }
+
+ void jump(ImmPtr ptr) {
+ BufferOffset bo = m_buffer.nextOffset();
+ addPendingJump(bo, ptr, RelocationKind::HARDCODED);
+ ma_jump(ptr);
+ }
+
+ void jump(TrampolinePtr code) { jump(ImmPtr(code.value)); }
+
+ void splitTag(Register src, Register dest) {
+ ma_dsrl(dest, src, Imm32(JSVAL_TAG_SHIFT));
+ }
+
+ void splitTag(const ValueOperand& operand, Register dest) {
+ splitTag(operand.valueReg(), dest);
+ }
+
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ splitTag(value, tag);
+ }
+
+ // unboxing code
+ void unboxNonDouble(const ValueOperand& operand, Register dest,
+ JSValueType type) {
+ unboxNonDouble(operand.valueReg(), dest, type);
+ }
+
+ template <typename T>
+ void unboxNonDouble(T src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ load32(src, dest);
+ return;
+ }
+ loadPtr(src, dest);
+ unboxNonDouble(dest, dest, type);
+ }
+
+ void unboxNonDouble(Register src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ ma_sll(dest, src, Imm32(0));
+ return;
+ }
+ MOZ_ASSERT(ScratchRegister != src);
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), ScratchRegister);
+ as_xor(dest, src, ScratchRegister);
+ }
+
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ static_assert(JS::detail::ValueObjectOrNullBit ==
+ (uint64_t(0x8) << JSVAL_TAG_SHIFT));
+ ma_dins(dest, zero, Imm32(JSVAL_TAG_SHIFT + 3), Imm32(1));
+ }
+
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ loadPtr(src, dest);
+ ma_dext(dest, dest, Imm32(0), Imm32(JSVAL_TAG_SHIFT));
+ }
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ ma_dext(dest, src.valueReg(), Imm32(0), Imm32(JSVAL_TAG_SHIFT));
+ }
+
+ // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
+ void getGCThingValueChunk(const Address& src, Register dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(scratch != dest);
+ loadPtr(src, dest);
+ movePtr(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), scratch);
+ as_and(dest, dest, scratch);
+ }
+ void getGCThingValueChunk(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movePtr(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ as_and(dest, dest, src.valueReg());
+ }
+
+ void unboxInt32(const ValueOperand& operand, Register dest);
+ void unboxInt32(Register src, Register dest);
+ void unboxInt32(const Address& src, Register dest);
+ void unboxInt32(const BaseIndex& src, Register dest);
+ void unboxBoolean(const ValueOperand& operand, Register dest);
+ void unboxBoolean(Register src, Register dest);
+ void unboxBoolean(const Address& src, Register dest);
+ void unboxBoolean(const BaseIndex& src, Register dest);
+ void unboxDouble(const ValueOperand& operand, FloatRegister dest);
+ void unboxDouble(Register src, Register dest);
+ void unboxDouble(const Address& src, FloatRegister dest);
+ void unboxDouble(const BaseIndex& src, FloatRegister dest);
+ void unboxString(const ValueOperand& operand, Register dest);
+ void unboxString(Register src, Register dest);
+ void unboxString(const Address& src, Register dest);
+ void unboxSymbol(const ValueOperand& src, Register dest);
+ void unboxSymbol(Register src, Register dest);
+ void unboxSymbol(const Address& src, Register dest);
+ void unboxBigInt(const ValueOperand& operand, Register dest);
+ void unboxBigInt(Register src, Register dest);
+ void unboxBigInt(const Address& src, Register dest);
+ void unboxObject(const ValueOperand& src, Register dest);
+ void unboxObject(Register src, Register dest);
+ void unboxObject(const Address& src, Register dest);
+ void unboxObject(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxValue(const ValueOperand& src, AnyRegister dest, JSValueType type);
+
+ void notBoolean(const ValueOperand& val) {
+ as_xori(val.valueReg(), val.valueReg(), 1);
+ }
+
+ // boxing code
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister);
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ [[nodiscard]] Register extractObject(const Address& address,
+ Register scratch);
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ unboxObject(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractString(const ValueOperand& value,
+ Register scratch) {
+ unboxString(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ unboxSymbol(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ unboxInt32(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ unboxBoolean(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch);
+ [[nodiscard]] Register extractTag(const BaseIndex& address, Register scratch);
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchRegister);
+ splitTag(value, scratch);
+ return scratch;
+ }
+
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest);
+ void loadInt32OrDouble(const Address& src, FloatRegister dest);
+ void loadInt32OrDouble(const BaseIndex& addr, FloatRegister dest);
+ void loadConstantDouble(double dp, FloatRegister dest);
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest);
+
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest);
+
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest);
+
+ // higher level tag testing code
+ Address ToPayload(Address value) { return value; }
+
+ template <typename T>
+ void loadUnboxedValue(const T& address, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(address, dest.fpu());
+ } else {
+ unboxNonDouble(address, dest.gpr(), ValueTypeFromMIRType(type));
+ }
+ }
+
+ void storeUnboxedPayload(ValueOperand value, BaseIndex address, size_t nbytes,
+ JSValueType type) {
+ switch (nbytes) {
+ case 8:
+ if (type == JSVAL_TYPE_OBJECT) {
+ unboxObjectOrNull(value, SecondScratchReg);
+ } else {
+ unboxNonDouble(value, SecondScratchReg, type);
+ }
+ computeEffectiveAddress(address, ScratchRegister);
+ as_sd(SecondScratchReg, ScratchRegister, 0);
+ return;
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void storeUnboxedPayload(ValueOperand value, Address address, size_t nbytes,
+ JSValueType type) {
+ switch (nbytes) {
+ case 8:
+ if (type == JSVAL_TYPE_OBJECT) {
+ unboxObjectOrNull(value, SecondScratchReg);
+ } else {
+ unboxNonDouble(value, SecondScratchReg, type);
+ }
+ storePtr(SecondScratchReg, address);
+ return;
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void boxValue(JSValueType type, Register src, Register dest) {
+ MOZ_ASSERT(src != dest);
+
+ JSValueTag tag = (JSValueTag)JSVAL_TYPE_TO_TAG(type);
+ ma_li(dest, Imm32(tag));
+ ma_dsll(dest, dest, Imm32(JSVAL_TAG_SHIFT));
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ ma_dins(dest, src, Imm32(0), Imm32(32));
+ } else {
+ ma_dins(dest, src, Imm32(0), Imm32(JSVAL_TAG_SHIFT));
+ }
+ }
+
+ void storeValue(ValueOperand val, Operand dst);
+ void storeValue(ValueOperand val, const BaseIndex& dest);
+ void storeValue(JSValueType type, Register reg, BaseIndex dest);
+ void storeValue(ValueOperand val, const Address& dest);
+ void storeValue(JSValueType type, Register reg, Address dest);
+ void storeValue(const Value& val, Address dest);
+ void storeValue(const Value& val, BaseIndex dest);
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ loadPtr(src, temp);
+ storePtr(temp, dest);
+ }
+
+ void storePrivateValue(Register src, const Address& dest) {
+ storePtr(src, dest);
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ storePtr(imm, dest);
+ }
+
+ void loadValue(Address src, ValueOperand val);
+ void loadValue(Operand dest, ValueOperand val) {
+ loadValue(dest.toAddress(), val);
+ }
+ void loadValue(const BaseIndex& addr, ValueOperand val);
+
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+
+ void tagValue(JSValueType type, Register payload, ValueOperand dest);
+
+ void pushValue(ValueOperand val);
+ void popValue(ValueOperand val);
+ void pushValue(const Value& val) {
+ if (val.isGCThing()) {
+ writeDataRelocation(val);
+ movWithPatch(ImmWord(val.asRawBits()), ScratchRegister);
+ push(ScratchRegister);
+ } else {
+ push(ImmWord(val.asRawBits()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ boxValue(type, reg, ScratchRegister);
+ push(ScratchRegister);
+ }
+ void pushValue(const Address& addr);
+
+ void handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail);
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+ public:
+ // The following functions are exposed for use in platform-shared code.
+
+ inline void incrementInt32Value(const Address& addr);
+
+ void move32(Imm32 imm, Register dest);
+ void move32(Register src, Register dest);
+
+ void movePtr(Register src, Register dest);
+ void movePtr(ImmWord imm, Register dest);
+ void movePtr(ImmPtr imm, Register dest);
+ void movePtr(wasm::SymbolicAddress imm, Register dest);
+ void movePtr(ImmGCPtr imm, Register dest);
+
+ void load8SignExtend(const Address& address, Register dest);
+ void load8SignExtend(const BaseIndex& src, Register dest);
+
+ void load8ZeroExtend(const Address& address, Register dest);
+ void load8ZeroExtend(const BaseIndex& src, Register dest);
+
+ void load16SignExtend(const Address& address, Register dest);
+ void load16SignExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ void load16UnalignedSignExtend(const S& src, Register dest) {
+ ma_load_unaligned(dest, src, SizeHalfWord, SignExtend);
+ }
+
+ void load16ZeroExtend(const Address& address, Register dest);
+ void load16ZeroExtend(const BaseIndex& src, Register dest);
+
+ template <typename S>
+ void load16UnalignedZeroExtend(const S& src, Register dest) {
+ ma_load_unaligned(dest, src, SizeHalfWord, ZeroExtend);
+ }
+
+ void load32(const Address& address, Register dest);
+ void load32(const BaseIndex& address, Register dest);
+ void load32(AbsoluteAddress address, Register dest);
+ void load32(wasm::SymbolicAddress address, Register dest);
+
+ template <typename S>
+ void load32Unaligned(const S& src, Register dest) {
+ ma_load_unaligned(dest, src, SizeWord, SignExtend);
+ }
+
+ void load64(const Address& address, Register64 dest) {
+ loadPtr(address, dest.reg);
+ }
+ void load64(const BaseIndex& address, Register64 dest) {
+ loadPtr(address, dest.reg);
+ }
+
+ template <typename S>
+ void load64Unaligned(const S& src, Register64 dest) {
+ ma_load_unaligned(dest.reg, src, SizeDouble, ZeroExtend);
+ }
+
+ void loadPtr(const Address& address, Register dest);
+ void loadPtr(const BaseIndex& src, Register dest);
+ void loadPtr(AbsoluteAddress address, Register dest);
+ void loadPtr(wasm::SymbolicAddress address, Register dest);
+
+ void loadPrivate(const Address& address, Register dest);
+
+ void loadUnalignedDouble(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& src, Register temp,
+ FloatRegister dest);
+ void loadUnalignedFloat32(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& src, Register temp,
+ FloatRegister dest);
+
+ void store8(Register src, const Address& address);
+ void store8(Imm32 imm, const Address& address);
+ void store8(Register src, const BaseIndex& address);
+ void store8(Imm32 imm, const BaseIndex& address);
+
+ void store16(Register src, const Address& address);
+ void store16(Imm32 imm, const Address& address);
+ void store16(Register src, const BaseIndex& address);
+ void store16(Imm32 imm, const BaseIndex& address);
+
+ template <typename T>
+ void store16Unaligned(Register src, const T& dest) {
+ ma_store_unaligned(src, dest, SizeHalfWord);
+ }
+
+ void store32(Register src, AbsoluteAddress address);
+ void store32(Register src, const Address& address);
+ void store32(Register src, const BaseIndex& address);
+ void store32(Imm32 src, const Address& address);
+ void store32(Imm32 src, const BaseIndex& address);
+
+ // NOTE: This will use second scratch on MIPS64. Only ARM needs the
+ // implementation without second scratch.
+ void store32_NoSecondScratch(Imm32 src, const Address& address) {
+ store32(src, address);
+ }
+
+ template <typename T>
+ void store32Unaligned(Register src, const T& dest) {
+ ma_store_unaligned(src, dest, SizeWord);
+ }
+
+ void store64(Imm64 imm, Address address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+ void store64(Imm64 imm, const BaseIndex& address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+
+ void store64(Register64 src, Address address) { storePtr(src.reg, address); }
+ void store64(Register64 src, const BaseIndex& address) {
+ storePtr(src.reg, address);
+ }
+
+ template <typename T>
+ void store64Unaligned(Register64 src, const T& dest) {
+ ma_store_unaligned(src.reg, dest, SizeDouble);
+ }
+
+ template <typename T>
+ void storePtr(ImmWord imm, T address);
+ template <typename T>
+ void storePtr(ImmPtr imm, T address);
+ template <typename T>
+ void storePtr(ImmGCPtr imm, T address);
+ void storePtr(Register src, const Address& address);
+ void storePtr(Register src, const BaseIndex& address);
+ void storePtr(Register src, AbsoluteAddress dest);
+
+ void storeUnalignedFloat32(const wasm::MemoryAccessDesc& access,
+ FloatRegister src, Register temp,
+ const BaseIndex& dest);
+ void storeUnalignedDouble(const wasm::MemoryAccessDesc& access,
+ FloatRegister src, Register temp,
+ const BaseIndex& dest);
+
+ void moveDouble(FloatRegister src, FloatRegister dest) { as_movd(dest, src); }
+
+ void zeroDouble(FloatRegister reg) { moveToDouble(zero, reg); }
+
+ void convertUInt64ToDouble(Register src, FloatRegister dest);
+
+ void breakpoint();
+
+ void checkStackAlignment();
+
+ static void calculateAlignedStackPointer(void** stackPointer);
+
+ // If source is a double, load it into dest. If source is int32,
+ // convert it to double. Else, branch to failure.
+ void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ void cmpPtrSet(Assembler::Condition cond, Address lhs, ImmPtr rhs,
+ Register dest);
+ void cmpPtrSet(Assembler::Condition cond, Register lhs, Address rhs,
+ Register dest);
+ void cmpPtrSet(Assembler::Condition cond, Address lhs, Register rhs,
+ Register dest);
+
+ void cmp32Set(Assembler::Condition cond, Register lhs, Address rhs,
+ Register dest);
+
+ protected:
+ bool buildOOLFakeExitFrame(void* fakeReturnAddr);
+
+ void wasmLoadI64Impl(const wasm::MemoryAccessDesc& access,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register64 output, Register tmp);
+ void wasmStoreI64Impl(const wasm::MemoryAccessDesc& access, Register64 value,
+ Register memoryBase, Register ptr, Register ptrScratch,
+ Register tmp);
+
+ public:
+ void lea(Operand addr, Register dest) {
+ ma_daddu(dest, addr.baseReg(), Imm32(addr.disp()));
+ }
+
+ void abiret() {
+ as_jr(ra);
+ as_nop();
+ }
+
+ void moveFloat32(FloatRegister src, FloatRegister dest) {
+ as_movs(dest, src);
+ }
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+typedef MacroAssemblerMIPS64Compat MacroAssemblerSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_MacroAssembler_mips64_h */
diff --git a/js/src/jit/mips64/MoveEmitter-mips64.cpp b/js/src/jit/mips64/MoveEmitter-mips64.cpp
new file mode 100644
index 0000000000..70217a37f8
--- /dev/null
+++ b/js/src/jit/mips64/MoveEmitter-mips64.cpp
@@ -0,0 +1,149 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/mips64/MoveEmitter-mips64.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void MoveEmitterMIPS64::breakCycle(const MoveOperand& from,
+ const MoveOperand& to, MoveOp::Type type,
+ uint32_t slotId) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (A -> B), which we reach first. We save B, then allow
+ // the original move to continue.
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchFloat32Reg;
+ masm.loadFloat32(getAdjustedAddress(to), temp);
+ masm.storeFloat32(temp, cycleSlot(slotId));
+ } else {
+ masm.storeFloat32(to.floatReg(), cycleSlot(slotId));
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchDoubleReg;
+ masm.loadDouble(getAdjustedAddress(to), temp);
+ masm.storeDouble(temp, cycleSlot(slotId));
+ } else {
+ masm.storeDouble(to.floatReg(), cycleSlot(slotId));
+ }
+ break;
+ case MoveOp::INT32:
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.load32(getAdjustedAddress(to), temp);
+ masm.store32(temp, cycleSlot(0));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.store32(to.reg(), cycleSlot(0));
+ }
+ break;
+ case MoveOp::GENERAL:
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.loadPtr(getAdjustedAddress(to), temp);
+ masm.storePtr(temp, cycleSlot(0));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.storePtr(to.reg(), cycleSlot(0));
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterMIPS64::completeCycle(const MoveOperand& from,
+ const MoveOperand& to, MoveOp::Type type,
+ uint32_t slotId) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (B -> A), which we reach last. We emit a move from the
+ // saved value of B, to A.
+ switch (type) {
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchFloat32Reg;
+ masm.loadFloat32(cycleSlot(slotId), temp);
+ masm.storeFloat32(temp, getAdjustedAddress(to));
+ } else {
+ masm.loadFloat32(cycleSlot(slotId), to.floatReg());
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ FloatRegister temp = ScratchDoubleReg;
+ masm.loadDouble(cycleSlot(slotId), temp);
+ masm.storeDouble(temp, getAdjustedAddress(to));
+ } else {
+ masm.loadDouble(cycleSlot(slotId), to.floatReg());
+ }
+ break;
+ case MoveOp::INT32:
+ MOZ_ASSERT(slotId == 0);
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.load32(cycleSlot(0), temp);
+ masm.store32(temp, getAdjustedAddress(to));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.load32(cycleSlot(0), to.reg());
+ }
+ break;
+ case MoveOp::GENERAL:
+ MOZ_ASSERT(slotId == 0);
+ if (to.isMemory()) {
+ Register temp = tempReg();
+ masm.loadPtr(cycleSlot(0), temp);
+ masm.storePtr(temp, getAdjustedAddress(to));
+ } else {
+ // Second scratch register should not be moved by MoveEmitter.
+ MOZ_ASSERT(to.reg() != spilledReg_);
+ masm.loadPtr(cycleSlot(0), to.reg());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterMIPS64::emitDoubleMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveDouble(from.floatReg(), to.floatReg());
+ } else if (to.isGeneralReg()) {
+ masm.moveFromDouble(from.floatReg(), to.reg());
+ } else {
+ MOZ_ASSERT(to.isMemory());
+ masm.storeDouble(from.floatReg(), getAdjustedAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ if (from.isMemory()) {
+ masm.loadDouble(getAdjustedAddress(from), to.floatReg());
+ } else {
+ masm.moveToDouble(from.reg(), to.floatReg());
+ }
+ } else {
+ MOZ_ASSERT(from.isMemory());
+ MOZ_ASSERT(to.isMemory());
+ masm.loadDouble(getAdjustedAddress(from), ScratchDoubleReg);
+ masm.storeDouble(ScratchDoubleReg, getAdjustedAddress(to));
+ }
+}
diff --git a/js/src/jit/mips64/MoveEmitter-mips64.h b/js/src/jit/mips64/MoveEmitter-mips64.h
new file mode 100644
index 0000000000..e6dbcd0693
--- /dev/null
+++ b/js/src/jit/mips64/MoveEmitter-mips64.h
@@ -0,0 +1,31 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_MoveEmitter_mips64_h
+#define jit_mips64_MoveEmitter_mips64_h
+
+#include "jit/mips-shared/MoveEmitter-mips-shared.h"
+
+namespace js {
+namespace jit {
+
+class MoveEmitterMIPS64 : public MoveEmitterMIPSShared {
+ void emitDoubleMove(const MoveOperand& from, const MoveOperand& to);
+ void breakCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+ void completeCycle(const MoveOperand& from, const MoveOperand& to,
+ MoveOp::Type type, uint32_t slot);
+
+ public:
+ MoveEmitterMIPS64(MacroAssembler& masm) : MoveEmitterMIPSShared(masm) {}
+};
+
+typedef MoveEmitterMIPS64 MoveEmitter;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_MoveEmitter_mips64_h */
diff --git a/js/src/jit/mips64/SharedICRegisters-mips64.h b/js/src/jit/mips64/SharedICRegisters-mips64.h
new file mode 100644
index 0000000000..99b263ca1e
--- /dev/null
+++ b/js/src/jit/mips64/SharedICRegisters-mips64.h
@@ -0,0 +1,45 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_mips64_SharedICRegisters_mips64_h
+#define jit_mips64_SharedICRegisters_mips64_h
+
+#include "jit/mips64/Assembler-mips64.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+// ValueOperands R0, R1, and R2.
+// R0 == JSReturnReg, and R2 uses registers not preserved across calls. R1 value
+// should be preserved across calls.
+static constexpr ValueOperand R0(v1);
+static constexpr ValueOperand R1(s4);
+static constexpr ValueOperand R2(a6);
+
+// ICTailCallReg and ICStubReg
+// These use registers that are not preserved across calls.
+static constexpr Register ICTailCallReg = ra;
+static constexpr Register ICStubReg = a5;
+
+// Register used internally by MacroAssemblerMIPS.
+static constexpr Register BaselineSecondScratchReg = SecondScratchReg;
+
+// Note that ICTailCallReg is actually just the link register.
+// In MIPS code emission, we do not clobber ICTailCallReg since we keep
+// the return address for calls there.
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = f0;
+static constexpr FloatRegister FloatReg1 = f2;
+static constexpr FloatRegister FloatReg2 = f4;
+static constexpr FloatRegister FloatReg3 = f6;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_mips64_SharedICRegisters_mips64_h */
diff --git a/js/src/jit/mips64/Simulator-mips64.cpp b/js/src/jit/mips64/Simulator-mips64.cpp
new file mode 100644
index 0000000000..481639d6de
--- /dev/null
+++ b/js/src/jit/mips64/Simulator-mips64.cpp
@@ -0,0 +1,4386 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80: */
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "jit/mips64/Simulator-mips64.h"
+
+#include "mozilla/Casting.h"
+#include "mozilla/FloatingPoint.h"
+#include "mozilla/IntegerPrintfMacros.h"
+#include "mozilla/Likely.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <float.h>
+#include <limits>
+
+#include "jit/AtomicOperations.h"
+#include "jit/mips64/Assembler-mips64.h"
+#include "js/Conversions.h"
+#include "js/UniquePtr.h"
+#include "js/Utility.h"
+#include "threading/LockGuard.h"
+#include "vm/JSContext.h"
+#include "vm/Runtime.h"
+#include "wasm/WasmInstance.h"
+#include "wasm/WasmSignalHandlers.h"
+
+#define I8(v) static_cast<int8_t>(v)
+#define I16(v) static_cast<int16_t>(v)
+#define U16(v) static_cast<uint16_t>(v)
+#define I32(v) static_cast<int32_t>(v)
+#define U32(v) static_cast<uint32_t>(v)
+#define I64(v) static_cast<int64_t>(v)
+#define U64(v) static_cast<uint64_t>(v)
+#define I128(v) static_cast<__int128_t>(v)
+#define U128(v) static_cast<__uint128_t>(v)
+
+#define I32_CHECK(v) \
+ ({ \
+ MOZ_ASSERT(I64(I32(v)) == I64(v)); \
+ I32((v)); \
+ })
+
+namespace js {
+namespace jit {
+
+static const Instr kCallRedirInstr =
+ op_special | MAX_BREAK_CODE << FunctionBits | ff_break;
+
+// Utils functions.
+static uint32_t GetFCSRConditionBit(uint32_t cc) {
+ if (cc == 0) {
+ return 23;
+ }
+ return 24 + cc;
+}
+
+// -----------------------------------------------------------------------------
+// MIPS assembly various constants.
+
+class SimInstruction {
+ public:
+ enum {
+ kInstrSize = 4,
+ // On MIPS PC cannot actually be directly accessed. We behave as if PC was
+ // always the value of the current instruction being executed.
+ kPCReadOffset = 0
+ };
+
+ // Get the raw instruction bits.
+ inline Instr instructionBits() const {
+ return *reinterpret_cast<const Instr*>(this);
+ }
+
+ // Set the raw instruction bits to value.
+ inline void setInstructionBits(Instr value) {
+ *reinterpret_cast<Instr*>(this) = value;
+ }
+
+ // Read one particular bit out of the instruction bits.
+ inline int bit(int nr) const { return (instructionBits() >> nr) & 1; }
+
+ // Read a bit field out of the instruction bits.
+ inline int bits(int hi, int lo) const {
+ return (instructionBits() >> lo) & ((2 << (hi - lo)) - 1);
+ }
+
+ // Instruction type.
+ enum Type { kRegisterType, kImmediateType, kJumpType, kUnsupported = -1 };
+
+ // Get the encoding type of the instruction.
+ Type instructionType() const;
+
+ // Accessors for the different named fields used in the MIPS encoding.
+ inline OpcodeField opcodeValue() const {
+ return static_cast<OpcodeField>(
+ bits(OpcodeShift + OpcodeBits - 1, OpcodeShift));
+ }
+
+ inline int rsValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return bits(RSShift + RSBits - 1, RSShift);
+ }
+
+ inline int rtValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return bits(RTShift + RTBits - 1, RTShift);
+ }
+
+ inline int rdValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType);
+ return bits(RDShift + RDBits - 1, RDShift);
+ }
+
+ inline int saValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType);
+ return bits(SAShift + SABits - 1, SAShift);
+ }
+
+ inline int functionValue() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return bits(FunctionShift + FunctionBits - 1, FunctionShift);
+ }
+
+ inline int fdValue() const { return bits(FDShift + FDBits - 1, FDShift); }
+
+ inline int fsValue() const { return bits(FSShift + FSBits - 1, FSShift); }
+
+ inline int ftValue() const { return bits(FTShift + FTBits - 1, FTShift); }
+
+ inline int frValue() const { return bits(FRShift + FRBits - 1, FRShift); }
+
+ // Float Compare condition code instruction bits.
+ inline int fcccValue() const {
+ return bits(FCccShift + FCccBits - 1, FCccShift);
+ }
+
+ // Float Branch condition code instruction bits.
+ inline int fbccValue() const {
+ return bits(FBccShift + FBccBits - 1, FBccShift);
+ }
+
+ // Float Branch true/false instruction bit.
+ inline int fbtrueValue() const {
+ return bits(FBtrueShift + FBtrueBits - 1, FBtrueShift);
+ }
+
+ // Return the fields at their original place in the instruction encoding.
+ inline OpcodeField opcodeFieldRaw() const {
+ return static_cast<OpcodeField>(instructionBits() & OpcodeMask);
+ }
+
+ inline int rsFieldRaw() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return instructionBits() & RSMask;
+ }
+
+ // Same as above function, but safe to call within instructionType().
+ inline int rsFieldRawNoAssert() const { return instructionBits() & RSMask; }
+
+ inline int rtFieldRaw() const {
+ MOZ_ASSERT(instructionType() == kRegisterType ||
+ instructionType() == kImmediateType);
+ return instructionBits() & RTMask;
+ }
+
+ inline int rdFieldRaw() const {
+ MOZ_ASSERT(instructionType() == kRegisterType);
+ return instructionBits() & RDMask;
+ }
+
+ inline int saFieldRaw() const {
+ MOZ_ASSERT(instructionType() == kRegisterType);
+ return instructionBits() & SAMask;
+ }
+
+ inline int functionFieldRaw() const {
+ return instructionBits() & FunctionMask;
+ }
+
+ // Get the secondary field according to the opcode.
+ inline int secondaryValue() const {
+ OpcodeField op = opcodeFieldRaw();
+ switch (op) {
+ case op_special:
+ case op_special2:
+ return functionValue();
+ case op_cop1:
+ return rsValue();
+ case op_regimm:
+ return rtValue();
+ default:
+ return ff_null;
+ }
+ }
+
+ inline int32_t imm16Value() const {
+ MOZ_ASSERT(instructionType() == kImmediateType);
+ return bits(Imm16Shift + Imm16Bits - 1, Imm16Shift);
+ }
+
+ inline int32_t imm26Value() const {
+ MOZ_ASSERT(instructionType() == kJumpType);
+ return bits(Imm26Shift + Imm26Bits - 1, Imm26Shift);
+ }
+
+ // Say if the instruction should not be used in a branch delay slot.
+ bool isForbiddenInBranchDelay() const;
+ // Say if the instruction 'links'. e.g. jal, bal.
+ bool isLinkingInstruction() const;
+ // Say if the instruction is a debugger break/trap.
+ bool isTrap() const;
+
+ private:
+ SimInstruction() = delete;
+ SimInstruction(const SimInstruction& other) = delete;
+ void operator=(const SimInstruction& other) = delete;
+};
+
+bool SimInstruction::isForbiddenInBranchDelay() const {
+ const int op = opcodeFieldRaw();
+ switch (op) {
+ case op_j:
+ case op_jal:
+ case op_beq:
+ case op_bne:
+ case op_blez:
+ case op_bgtz:
+ case op_beql:
+ case op_bnel:
+ case op_blezl:
+ case op_bgtzl:
+ return true;
+ case op_regimm:
+ switch (rtFieldRaw()) {
+ case rt_bltz:
+ case rt_bgez:
+ case rt_bltzal:
+ case rt_bgezal:
+ return true;
+ default:
+ return false;
+ };
+ break;
+ case op_special:
+ switch (functionFieldRaw()) {
+ case ff_jr:
+ case ff_jalr:
+ return true;
+ default:
+ return false;
+ };
+ break;
+ default:
+ return false;
+ };
+}
+
+bool SimInstruction::isLinkingInstruction() const {
+ const int op = opcodeFieldRaw();
+ switch (op) {
+ case op_jal:
+ return true;
+ case op_regimm:
+ switch (rtFieldRaw()) {
+ case rt_bgezal:
+ case rt_bltzal:
+ return true;
+ default:
+ return false;
+ };
+ case op_special:
+ switch (functionFieldRaw()) {
+ case ff_jalr:
+ return true;
+ default:
+ return false;
+ };
+ default:
+ return false;
+ };
+}
+
+bool SimInstruction::isTrap() const {
+ if (opcodeFieldRaw() != op_special) {
+ return false;
+ } else {
+ switch (functionFieldRaw()) {
+ case ff_break:
+ return instructionBits() != kCallRedirInstr;
+ case ff_tge:
+ case ff_tgeu:
+ case ff_tlt:
+ case ff_tltu:
+ case ff_teq:
+ case ff_tne:
+ return bits(15, 6) != kWasmTrapCode;
+ default:
+ return false;
+ };
+ }
+}
+
+SimInstruction::Type SimInstruction::instructionType() const {
+ switch (opcodeFieldRaw()) {
+ case op_special:
+ switch (functionFieldRaw()) {
+ case ff_jr:
+ case ff_jalr:
+ case ff_sync:
+ case ff_break:
+ case ff_sll:
+ case ff_dsll:
+ case ff_dsll32:
+ case ff_srl:
+ case ff_dsrl:
+ case ff_dsrl32:
+ case ff_sra:
+ case ff_dsra:
+ case ff_dsra32:
+ case ff_sllv:
+ case ff_dsllv:
+ case ff_srlv:
+ case ff_dsrlv:
+ case ff_srav:
+ case ff_dsrav:
+ case ff_mfhi:
+ case ff_mflo:
+ case ff_mult:
+ case ff_dmult:
+ case ff_multu:
+ case ff_dmultu:
+ case ff_div:
+ case ff_ddiv:
+ case ff_divu:
+ case ff_ddivu:
+ case ff_add:
+ case ff_dadd:
+ case ff_addu:
+ case ff_daddu:
+ case ff_sub:
+ case ff_dsub:
+ case ff_subu:
+ case ff_dsubu:
+ case ff_and:
+ case ff_or:
+ case ff_xor:
+ case ff_nor:
+ case ff_slt:
+ case ff_sltu:
+ case ff_tge:
+ case ff_tgeu:
+ case ff_tlt:
+ case ff_tltu:
+ case ff_teq:
+ case ff_tne:
+ case ff_movz:
+ case ff_movn:
+ case ff_movci:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ };
+ break;
+ case op_special2:
+ switch (functionFieldRaw()) {
+ case ff_mul:
+ case ff_clz:
+ case ff_dclz:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ };
+ break;
+ case op_special3:
+ switch (functionFieldRaw()) {
+ case ff_ins:
+ case ff_dins:
+ case ff_dinsm:
+ case ff_dinsu:
+ case ff_ext:
+ case ff_dext:
+ case ff_dextm:
+ case ff_dextu:
+ case ff_bshfl:
+ case ff_dbshfl:
+ return kRegisterType;
+ default:
+ return kUnsupported;
+ };
+ break;
+ case op_cop1: // Coprocessor instructions.
+ switch (rsFieldRawNoAssert()) {
+ case rs_bc1: // Branch on coprocessor condition.
+ return kImmediateType;
+ default:
+ return kRegisterType;
+ };
+ break;
+ case op_cop1x:
+ return kRegisterType;
+ // 16 bits Immediate type instructions. e.g.: addi dest, src, imm16.
+ case op_regimm:
+ case op_beq:
+ case op_bne:
+ case op_blez:
+ case op_bgtz:
+ case op_addi:
+ case op_daddi:
+ case op_addiu:
+ case op_daddiu:
+ case op_slti:
+ case op_sltiu:
+ case op_andi:
+ case op_ori:
+ case op_xori:
+ case op_lui:
+ case op_beql:
+ case op_bnel:
+ case op_blezl:
+ case op_bgtzl:
+ case op_lb:
+ case op_lbu:
+ case op_lh:
+ case op_lhu:
+ case op_lw:
+ case op_lwu:
+ case op_lwl:
+ case op_lwr:
+ case op_ll:
+ case op_lld:
+ case op_ld:
+ case op_ldl:
+ case op_ldr:
+ case op_sb:
+ case op_sh:
+ case op_sw:
+ case op_swl:
+ case op_swr:
+ case op_sc:
+ case op_scd:
+ case op_sd:
+ case op_sdl:
+ case op_sdr:
+ case op_lwc1:
+ case op_ldc1:
+ case op_swc1:
+ case op_sdc1:
+ return kImmediateType;
+ // 26 bits immediate type instructions. e.g.: j imm26.
+ case op_j:
+ case op_jal:
+ return kJumpType;
+ default:
+ return kUnsupported;
+ };
+ return kUnsupported;
+}
+
+// C/C++ argument slots size.
+const int kCArgSlotCount = 0;
+const int kCArgsSlotsSize = kCArgSlotCount * sizeof(uintptr_t);
+const int kBranchReturnOffset = 2 * SimInstruction::kInstrSize;
+
+class CachePage {
+ public:
+ static const int LINE_VALID = 0;
+ static const int LINE_INVALID = 1;
+
+ static const int kPageShift = 12;
+ static const int kPageSize = 1 << kPageShift;
+ static const int kPageMask = kPageSize - 1;
+ static const int kLineShift = 2; // The cache line is only 4 bytes right now.
+ static const int kLineLength = 1 << kLineShift;
+ static const int kLineMask = kLineLength - 1;
+
+ CachePage() { memset(&validity_map_, LINE_INVALID, sizeof(validity_map_)); }
+
+ char* validityByte(int offset) {
+ return &validity_map_[offset >> kLineShift];
+ }
+
+ char* cachedData(int offset) { return &data_[offset]; }
+
+ private:
+ char data_[kPageSize]; // The cached data.
+ static const int kValidityMapSize = kPageSize >> kLineShift;
+ char validity_map_[kValidityMapSize]; // One byte per line.
+};
+
+// Protects the icache() and redirection() properties of the
+// Simulator.
+class AutoLockSimulatorCache : public LockGuard<Mutex> {
+ using Base = LockGuard<Mutex>;
+
+ public:
+ explicit AutoLockSimulatorCache()
+ : Base(SimulatorProcess::singleton_->cacheLock_) {}
+};
+
+mozilla::Atomic<size_t, mozilla::ReleaseAcquire>
+ SimulatorProcess::ICacheCheckingDisableCount(
+ 1); // Checking is disabled by default.
+SimulatorProcess* SimulatorProcess::singleton_ = nullptr;
+
+int64_t Simulator::StopSimAt = -1;
+
+Simulator* Simulator::Create() {
+ auto sim = MakeUnique<Simulator>();
+ if (!sim) {
+ return nullptr;
+ }
+
+ if (!sim->init()) {
+ return nullptr;
+ }
+
+ int64_t stopAt;
+ char* stopAtStr = getenv("MIPS_SIM_STOP_AT");
+ if (stopAtStr && sscanf(stopAtStr, "%" PRIi64, &stopAt) == 1) {
+ fprintf(stderr, "\nStopping simulation at icount %" PRIi64 "\n", stopAt);
+ Simulator::StopSimAt = stopAt;
+ }
+
+ return sim.release();
+}
+
+void Simulator::Destroy(Simulator* sim) { js_delete(sim); }
+
+// The MipsDebugger class is used by the simulator while debugging simulated
+// code.
+class MipsDebugger {
+ public:
+ explicit MipsDebugger(Simulator* sim) : sim_(sim) {}
+
+ void stop(SimInstruction* instr);
+ void debug();
+ // Print all registers with a nice formatting.
+ void printAllRegs();
+ void printAllRegsIncludingFPU();
+
+ private:
+ // We set the breakpoint code to 0xfffff to easily recognize it.
+ static const Instr kBreakpointInstr = op_special | ff_break | 0xfffff << 6;
+ static const Instr kNopInstr = op_special | ff_sll;
+
+ Simulator* sim_;
+
+ int64_t getRegisterValue(int regnum);
+ int64_t getFPURegisterValueLong(int regnum);
+ float getFPURegisterValueFloat(int regnum);
+ double getFPURegisterValueDouble(int regnum);
+ bool getValue(const char* desc, int64_t* value);
+
+ // Set or delete a breakpoint. Returns true if successful.
+ bool setBreakpoint(SimInstruction* breakpc);
+ bool deleteBreakpoint(SimInstruction* breakpc);
+
+ // Undo and redo all breakpoints. This is needed to bracket disassembly and
+ // execution to skip past breakpoints when run from the debugger.
+ void undoBreakpoints();
+ void redoBreakpoints();
+};
+
+static void UNSUPPORTED() {
+ printf("Unsupported instruction.\n");
+ MOZ_CRASH();
+}
+
+void MipsDebugger::stop(SimInstruction* instr) {
+ // Get the stop code.
+ uint32_t code = instr->bits(25, 6);
+ // Retrieve the encoded address, which comes just after this stop.
+ char* msg =
+ *reinterpret_cast<char**>(sim_->get_pc() + SimInstruction::kInstrSize);
+ // Update this stop description.
+ if (!sim_->watchedStops_[code].desc_) {
+ sim_->watchedStops_[code].desc_ = msg;
+ }
+ // Print the stop message and code if it is not the default code.
+ if (code != kMaxStopCode) {
+ printf("Simulator hit stop %u: %s\n", code, msg);
+ } else {
+ printf("Simulator hit %s\n", msg);
+ }
+ sim_->set_pc(sim_->get_pc() + 2 * SimInstruction::kInstrSize);
+ debug();
+}
+
+int64_t MipsDebugger::getRegisterValue(int regnum) {
+ if (regnum == kPCRegister) {
+ return sim_->get_pc();
+ }
+ return sim_->getRegister(regnum);
+}
+
+int64_t MipsDebugger::getFPURegisterValueLong(int regnum) {
+ return sim_->getFpuRegister(regnum);
+}
+
+float MipsDebugger::getFPURegisterValueFloat(int regnum) {
+ return sim_->getFpuRegisterFloat(regnum);
+}
+
+double MipsDebugger::getFPURegisterValueDouble(int regnum) {
+ return sim_->getFpuRegisterDouble(regnum);
+}
+
+bool MipsDebugger::getValue(const char* desc, int64_t* value) {
+ Register reg = Register::FromName(desc);
+ if (reg != InvalidReg) {
+ *value = getRegisterValue(reg.code());
+ return true;
+ }
+
+ if (strncmp(desc, "0x", 2) == 0) {
+ return sscanf(desc, "%" PRIu64, reinterpret_cast<uint64_t*>(value)) == 1;
+ }
+ return sscanf(desc, "%" PRIi64, value) == 1;
+}
+
+bool MipsDebugger::setBreakpoint(SimInstruction* breakpc) {
+ // Check if a breakpoint can be set. If not return without any side-effects.
+ if (sim_->break_pc_ != nullptr) {
+ return false;
+ }
+
+ // Set the breakpoint.
+ sim_->break_pc_ = breakpc;
+ sim_->break_instr_ = breakpc->instructionBits();
+ // Not setting the breakpoint instruction in the code itself. It will be set
+ // when the debugger shell continues.
+ return true;
+}
+
+bool MipsDebugger::deleteBreakpoint(SimInstruction* breakpc) {
+ if (sim_->break_pc_ != nullptr) {
+ sim_->break_pc_->setInstructionBits(sim_->break_instr_);
+ }
+
+ sim_->break_pc_ = nullptr;
+ sim_->break_instr_ = 0;
+ return true;
+}
+
+void MipsDebugger::undoBreakpoints() {
+ if (sim_->break_pc_) {
+ sim_->break_pc_->setInstructionBits(sim_->break_instr_);
+ }
+}
+
+void MipsDebugger::redoBreakpoints() {
+ if (sim_->break_pc_) {
+ sim_->break_pc_->setInstructionBits(kBreakpointInstr);
+ }
+}
+
+void MipsDebugger::printAllRegs() {
+ int64_t value;
+ for (uint32_t i = 0; i < Registers::Total; i++) {
+ value = getRegisterValue(i);
+ printf("%3s: 0x%016" PRIx64 " %20" PRIi64 " ", Registers::GetName(i),
+ value, value);
+
+ if (i % 2) {
+ printf("\n");
+ }
+ }
+ printf("\n");
+
+ value = getRegisterValue(Simulator::LO);
+ printf(" LO: 0x%016" PRIx64 " %20" PRIi64 " ", value, value);
+ value = getRegisterValue(Simulator::HI);
+ printf(" HI: 0x%016" PRIx64 " %20" PRIi64 "\n", value, value);
+ value = getRegisterValue(Simulator::pc);
+ printf(" pc: 0x%016" PRIx64 "\n", value);
+}
+
+void MipsDebugger::printAllRegsIncludingFPU() {
+ printAllRegs();
+
+ printf("\n\n");
+ // f0, f1, f2, ... f31.
+ for (uint32_t i = 0; i < FloatRegisters::TotalPhys; i++) {
+ printf("%3s: 0x%016" PRIi64 "\tflt: %-8.4g\tdbl: %-16.4g\n",
+ FloatRegisters::GetName(i), getFPURegisterValueLong(i),
+ getFPURegisterValueFloat(i), getFPURegisterValueDouble(i));
+ }
+}
+
+static char* ReadLine(const char* prompt) {
+ UniqueChars result;
+ char lineBuf[256];
+ int offset = 0;
+ bool keepGoing = true;
+ fprintf(stdout, "%s", prompt);
+ fflush(stdout);
+ while (keepGoing) {
+ if (fgets(lineBuf, sizeof(lineBuf), stdin) == nullptr) {
+ // fgets got an error. Just give up.
+ return nullptr;
+ }
+ int len = strlen(lineBuf);
+ if (len > 0 && lineBuf[len - 1] == '\n') {
+ // Since we read a new line we are done reading the line. This
+ // will exit the loop after copying this buffer into the result.
+ keepGoing = false;
+ }
+ if (!result) {
+ // Allocate the initial result and make room for the terminating '\0'
+ result.reset(js_pod_malloc<char>(len + 1));
+ if (!result) {
+ return nullptr;
+ }
+ } else {
+ // Allocate a new result with enough room for the new addition.
+ int new_len = offset + len + 1;
+ char* new_result = js_pod_malloc<char>(new_len);
+ if (!new_result) {
+ return nullptr;
+ }
+ // Copy the existing input into the new array and set the new
+ // array as the result.
+ memcpy(new_result, result.get(), offset * sizeof(char));
+ result.reset(new_result);
+ }
+ // Copy the newly read line into the result.
+ memcpy(result.get() + offset, lineBuf, len * sizeof(char));
+ offset += len;
+ }
+
+ MOZ_ASSERT(result);
+ result[offset] = '\0';
+ return result.release();
+}
+
+static void DisassembleInstruction(uint64_t pc) {
+ uint8_t* bytes = reinterpret_cast<uint8_t*>(pc);
+ char hexbytes[256];
+ sprintf(hexbytes, "0x%x 0x%x 0x%x 0x%x", bytes[0], bytes[1], bytes[2],
+ bytes[3]);
+ char llvmcmd[1024];
+ sprintf(llvmcmd,
+ "bash -c \"echo -n '%p'; echo '%s' | "
+ "llvm-mc -disassemble -arch=mips64el -mcpu=mips64r2 | "
+ "grep -v pure_instructions | grep -v .text\"",
+ static_cast<void*>(bytes), hexbytes);
+ if (system(llvmcmd)) {
+ printf("Cannot disassemble instruction.\n");
+ }
+}
+
+void MipsDebugger::debug() {
+ intptr_t lastPC = -1;
+ bool done = false;
+
+#define COMMAND_SIZE 63
+#define ARG_SIZE 255
+
+#define STR(a) #a
+#define XSTR(a) STR(a)
+
+ char cmd[COMMAND_SIZE + 1];
+ char arg1[ARG_SIZE + 1];
+ char arg2[ARG_SIZE + 1];
+ char* argv[3] = {cmd, arg1, arg2};
+
+ // Make sure to have a proper terminating character if reaching the limit.
+ cmd[COMMAND_SIZE] = 0;
+ arg1[ARG_SIZE] = 0;
+ arg2[ARG_SIZE] = 0;
+
+ // Undo all set breakpoints while running in the debugger shell. This will
+ // make them invisible to all commands.
+ undoBreakpoints();
+
+ while (!done && (sim_->get_pc() != Simulator::end_sim_pc)) {
+ if (lastPC != sim_->get_pc()) {
+ DisassembleInstruction(sim_->get_pc());
+ lastPC = sim_->get_pc();
+ }
+ char* line = ReadLine("sim> ");
+ if (line == nullptr) {
+ break;
+ } else {
+ char* last_input = sim_->lastDebuggerInput();
+ if (strcmp(line, "\n") == 0 && last_input != nullptr) {
+ line = last_input;
+ } else {
+ // Ownership is transferred to sim_;
+ sim_->setLastDebuggerInput(line);
+ }
+ // Use sscanf to parse the individual parts of the command line. At the
+ // moment no command expects more than two parameters.
+ int argc = sscanf(line,
+ "%" XSTR(COMMAND_SIZE) "s "
+ "%" XSTR(ARG_SIZE) "s "
+ "%" XSTR(ARG_SIZE) "s",
+ cmd, arg1, arg2);
+ if ((strcmp(cmd, "si") == 0) || (strcmp(cmd, "stepi") == 0)) {
+ SimInstruction* instr =
+ reinterpret_cast<SimInstruction*>(sim_->get_pc());
+ if (!instr->isTrap()) {
+ sim_->instructionDecode(
+ reinterpret_cast<SimInstruction*>(sim_->get_pc()));
+ } else {
+ // Allow si to jump over generated breakpoints.
+ printf("/!\\ Jumping over generated breakpoint.\n");
+ sim_->set_pc(sim_->get_pc() + SimInstruction::kInstrSize);
+ }
+ } else if ((strcmp(cmd, "c") == 0) || (strcmp(cmd, "cont") == 0)) {
+ // Execute the one instruction we broke at with breakpoints disabled.
+ sim_->instructionDecode(
+ reinterpret_cast<SimInstruction*>(sim_->get_pc()));
+ // Leave the debugger shell.
+ done = true;
+ } else if ((strcmp(cmd, "p") == 0) || (strcmp(cmd, "print") == 0)) {
+ if (argc == 2) {
+ int64_t value;
+ if (strcmp(arg1, "all") == 0) {
+ printAllRegs();
+ } else if (strcmp(arg1, "allf") == 0) {
+ printAllRegsIncludingFPU();
+ } else {
+ Register reg = Register::FromName(arg1);
+ FloatRegisters::Encoding fReg = FloatRegisters::FromName(arg1);
+ if (reg != InvalidReg) {
+ value = getRegisterValue(reg.code());
+ printf("%s: 0x%016" PRIi64 " %20" PRIi64 " \n", arg1, value,
+ value);
+ } else if (fReg != FloatRegisters::Invalid) {
+ printf("%3s: 0x%016" PRIi64 "\tflt: %-8.4g\tdbl: %-16.4g\n",
+ FloatRegisters::GetName(fReg),
+ getFPURegisterValueLong(fReg),
+ getFPURegisterValueFloat(fReg),
+ getFPURegisterValueDouble(fReg));
+ } else {
+ printf("%s unrecognized\n", arg1);
+ }
+ }
+ } else {
+ printf("print <register> or print <fpu register> single\n");
+ }
+ } else if (strcmp(cmd, "stack") == 0 || strcmp(cmd, "mem") == 0) {
+ int64_t* cur = nullptr;
+ int64_t* end = nullptr;
+ int next_arg = 1;
+
+ if (strcmp(cmd, "stack") == 0) {
+ cur = reinterpret_cast<int64_t*>(sim_->getRegister(Simulator::sp));
+ } else { // Command "mem".
+ int64_t value;
+ if (!getValue(arg1, &value)) {
+ printf("%s unrecognized\n", arg1);
+ continue;
+ }
+ cur = reinterpret_cast<int64_t*>(value);
+ next_arg++;
+ }
+
+ int64_t words;
+ if (argc == next_arg) {
+ words = 10;
+ } else {
+ if (!getValue(argv[next_arg], &words)) {
+ words = 10;
+ }
+ }
+ end = cur + words;
+
+ while (cur < end) {
+ printf(" %p: 0x%016" PRIx64 " %20" PRIi64, cur, *cur, *cur);
+ printf("\n");
+ cur++;
+ }
+
+ } else if ((strcmp(cmd, "disasm") == 0) || (strcmp(cmd, "dpc") == 0) ||
+ (strcmp(cmd, "di") == 0)) {
+ uint8_t* cur = nullptr;
+ uint8_t* end = nullptr;
+
+ if (argc == 1) {
+ cur = reinterpret_cast<uint8_t*>(sim_->get_pc());
+ end = cur + (10 * SimInstruction::kInstrSize);
+ } else if (argc == 2) {
+ Register reg = Register::FromName(arg1);
+ if (reg != InvalidReg || strncmp(arg1, "0x", 2) == 0) {
+ // The argument is an address or a register name.
+ int64_t value;
+ if (getValue(arg1, &value)) {
+ cur = reinterpret_cast<uint8_t*>(value);
+ // Disassemble 10 instructions at <arg1>.
+ end = cur + (10 * SimInstruction::kInstrSize);
+ }
+ } else {
+ // The argument is the number of instructions.
+ int64_t value;
+ if (getValue(arg1, &value)) {
+ cur = reinterpret_cast<uint8_t*>(sim_->get_pc());
+ // Disassemble <arg1> instructions.
+ end = cur + (value * SimInstruction::kInstrSize);
+ }
+ }
+ } else {
+ int64_t value1;
+ int64_t value2;
+ if (getValue(arg1, &value1) && getValue(arg2, &value2)) {
+ cur = reinterpret_cast<uint8_t*>(value1);
+ end = cur + (value2 * SimInstruction::kInstrSize);
+ }
+ }
+
+ while (cur < end) {
+ DisassembleInstruction(uint64_t(cur));
+ cur += SimInstruction::kInstrSize;
+ }
+ } else if (strcmp(cmd, "gdb") == 0) {
+ printf("relinquishing control to gdb\n");
+ asm("int $3");
+ printf("regaining control from gdb\n");
+ } else if (strcmp(cmd, "break") == 0) {
+ if (argc == 2) {
+ int64_t value;
+ if (getValue(arg1, &value)) {
+ if (!setBreakpoint(reinterpret_cast<SimInstruction*>(value))) {
+ printf("setting breakpoint failed\n");
+ }
+ } else {
+ printf("%s unrecognized\n", arg1);
+ }
+ } else {
+ printf("break <address>\n");
+ }
+ } else if (strcmp(cmd, "del") == 0) {
+ if (!deleteBreakpoint(nullptr)) {
+ printf("deleting breakpoint failed\n");
+ }
+ } else if (strcmp(cmd, "flags") == 0) {
+ printf("No flags on MIPS !\n");
+ } else if (strcmp(cmd, "stop") == 0) {
+ int64_t value;
+ intptr_t stop_pc = sim_->get_pc() - 2 * SimInstruction::kInstrSize;
+ SimInstruction* stop_instr = reinterpret_cast<SimInstruction*>(stop_pc);
+ SimInstruction* msg_address = reinterpret_cast<SimInstruction*>(
+ stop_pc + SimInstruction::kInstrSize);
+ if ((argc == 2) && (strcmp(arg1, "unstop") == 0)) {
+ // Remove the current stop.
+ if (sim_->isStopInstruction(stop_instr)) {
+ stop_instr->setInstructionBits(kNopInstr);
+ msg_address->setInstructionBits(kNopInstr);
+ } else {
+ printf("Not at debugger stop.\n");
+ }
+ } else if (argc == 3) {
+ // Print information about all/the specified breakpoint(s).
+ if (strcmp(arg1, "info") == 0) {
+ if (strcmp(arg2, "all") == 0) {
+ printf("Stop information:\n");
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->printStopInfo(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->printStopInfo(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ } else if (strcmp(arg1, "enable") == 0) {
+ // Enable all/the specified breakpoint(s).
+ if (strcmp(arg2, "all") == 0) {
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->enableStop(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->enableStop(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ } else if (strcmp(arg1, "disable") == 0) {
+ // Disable all/the specified breakpoint(s).
+ if (strcmp(arg2, "all") == 0) {
+ for (uint32_t i = kMaxWatchpointCode + 1; i <= kMaxStopCode;
+ i++) {
+ sim_->disableStop(i);
+ }
+ } else if (getValue(arg2, &value)) {
+ sim_->disableStop(value);
+ } else {
+ printf("Unrecognized argument.\n");
+ }
+ }
+ } else {
+ printf("Wrong usage. Use help command for more information.\n");
+ }
+ } else if ((strcmp(cmd, "h") == 0) || (strcmp(cmd, "help") == 0)) {
+ printf("cont\n");
+ printf(" continue execution (alias 'c')\n");
+ printf("stepi\n");
+ printf(" step one instruction (alias 'si')\n");
+ printf("print <register>\n");
+ printf(" print register content (alias 'p')\n");
+ printf(" use register name 'all' to print all registers\n");
+ printf("printobject <register>\n");
+ printf(" print an object from a register (alias 'po')\n");
+ printf("stack [<words>]\n");
+ printf(" dump stack content, default dump 10 words)\n");
+ printf("mem <address> [<words>]\n");
+ printf(" dump memory content, default dump 10 words)\n");
+ printf("flags\n");
+ printf(" print flags\n");
+ printf("disasm [<instructions>]\n");
+ printf("disasm [<address/register>]\n");
+ printf("disasm [[<address/register>] <instructions>]\n");
+ printf(" disassemble code, default is 10 instructions\n");
+ printf(" from pc (alias 'di')\n");
+ printf("gdb\n");
+ printf(" enter gdb\n");
+ printf("break <address>\n");
+ printf(" set a break point on the address\n");
+ printf("del\n");
+ printf(" delete the breakpoint\n");
+ printf("stop feature:\n");
+ printf(" Description:\n");
+ printf(" Stops are debug instructions inserted by\n");
+ printf(" the Assembler::stop() function.\n");
+ printf(" When hitting a stop, the Simulator will\n");
+ printf(" stop and and give control to the Debugger.\n");
+ printf(" All stop codes are watched:\n");
+ printf(" - They can be enabled / disabled: the Simulator\n");
+ printf(" will / won't stop when hitting them.\n");
+ printf(" - The Simulator keeps track of how many times they \n");
+ printf(" are met. (See the info command.) Going over a\n");
+ printf(" disabled stop still increases its counter. \n");
+ printf(" Commands:\n");
+ printf(" stop info all/<code> : print infos about number <code>\n");
+ printf(" or all stop(s).\n");
+ printf(" stop enable/disable all/<code> : enables / disables\n");
+ printf(" all or number <code> stop(s)\n");
+ printf(" stop unstop\n");
+ printf(" ignore the stop instruction at the current location\n");
+ printf(" from now on\n");
+ } else {
+ printf("Unknown command: %s\n", cmd);
+ }
+ }
+ }
+
+ // Add all the breakpoints back to stop execution and enter the debugger
+ // shell when hit.
+ redoBreakpoints();
+
+#undef COMMAND_SIZE
+#undef ARG_SIZE
+
+#undef STR
+#undef XSTR
+}
+
+static bool AllOnOnePage(uintptr_t start, int size) {
+ intptr_t start_page = (start & ~CachePage::kPageMask);
+ intptr_t end_page = ((start + size) & ~CachePage::kPageMask);
+ return start_page == end_page;
+}
+
+void Simulator::setLastDebuggerInput(char* input) {
+ js_free(lastDebuggerInput_);
+ lastDebuggerInput_ = input;
+}
+
+static CachePage* GetCachePageLocked(SimulatorProcess::ICacheMap& i_cache,
+ void* page) {
+ SimulatorProcess::ICacheMap::AddPtr p = i_cache.lookupForAdd(page);
+ if (p) {
+ return p->value();
+ }
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ CachePage* new_page = js_new<CachePage>();
+ if (!new_page || !i_cache.add(p, page, new_page)) {
+ oomUnsafe.crash("Simulator CachePage");
+ }
+ return new_page;
+}
+
+// Flush from start up to and not including start + size.
+static void FlushOnePageLocked(SimulatorProcess::ICacheMap& i_cache,
+ intptr_t start, int size) {
+ MOZ_ASSERT(size <= CachePage::kPageSize);
+ MOZ_ASSERT(AllOnOnePage(start, size - 1));
+ MOZ_ASSERT((start & CachePage::kLineMask) == 0);
+ MOZ_ASSERT((size & CachePage::kLineMask) == 0);
+ void* page = reinterpret_cast<void*>(start & (~CachePage::kPageMask));
+ int offset = (start & CachePage::kPageMask);
+ CachePage* cache_page = GetCachePageLocked(i_cache, page);
+ char* valid_bytemap = cache_page->validityByte(offset);
+ memset(valid_bytemap, CachePage::LINE_INVALID, size >> CachePage::kLineShift);
+}
+
+static void FlushICacheLocked(SimulatorProcess::ICacheMap& i_cache,
+ void* start_addr, size_t size) {
+ intptr_t start = reinterpret_cast<intptr_t>(start_addr);
+ int intra_line = (start & CachePage::kLineMask);
+ start -= intra_line;
+ size += intra_line;
+ size = ((size - 1) | CachePage::kLineMask) + 1;
+ int offset = (start & CachePage::kPageMask);
+ while (!AllOnOnePage(start, size - 1)) {
+ int bytes_to_flush = CachePage::kPageSize - offset;
+ FlushOnePageLocked(i_cache, start, bytes_to_flush);
+ start += bytes_to_flush;
+ size -= bytes_to_flush;
+ MOZ_ASSERT((start & CachePage::kPageMask) == 0);
+ offset = 0;
+ }
+ if (size != 0) {
+ FlushOnePageLocked(i_cache, start, size);
+ }
+}
+
+/* static */
+void SimulatorProcess::checkICacheLocked(SimInstruction* instr) {
+ intptr_t address = reinterpret_cast<intptr_t>(instr);
+ void* page = reinterpret_cast<void*>(address & (~CachePage::kPageMask));
+ void* line = reinterpret_cast<void*>(address & (~CachePage::kLineMask));
+ int offset = (address & CachePage::kPageMask);
+ CachePage* cache_page = GetCachePageLocked(icache(), page);
+ char* cache_valid_byte = cache_page->validityByte(offset);
+ bool cache_hit = (*cache_valid_byte == CachePage::LINE_VALID);
+ char* cached_line = cache_page->cachedData(offset & ~CachePage::kLineMask);
+
+ if (cache_hit) {
+ // Check that the data in memory matches the contents of the I-cache.
+ int cmpret =
+ memcmp(reinterpret_cast<void*>(instr), cache_page->cachedData(offset),
+ SimInstruction::kInstrSize);
+ MOZ_ASSERT(cmpret == 0);
+ } else {
+ // Cache miss. Load memory into the cache.
+ memcpy(cached_line, line, CachePage::kLineLength);
+ *cache_valid_byte = CachePage::LINE_VALID;
+ }
+}
+
+HashNumber SimulatorProcess::ICacheHasher::hash(const Lookup& l) {
+ return U32(reinterpret_cast<uintptr_t>(l)) >> 2;
+}
+
+bool SimulatorProcess::ICacheHasher::match(const Key& k, const Lookup& l) {
+ MOZ_ASSERT((reinterpret_cast<intptr_t>(k) & CachePage::kPageMask) == 0);
+ MOZ_ASSERT((reinterpret_cast<intptr_t>(l) & CachePage::kPageMask) == 0);
+ return k == l;
+}
+
+/* static */
+void SimulatorProcess::FlushICache(void* start_addr, size_t size) {
+ if (!ICacheCheckingDisableCount) {
+ AutoLockSimulatorCache als;
+ js::jit::FlushICacheLocked(icache(), start_addr, size);
+ }
+}
+
+Simulator::Simulator() {
+ // Set up simulator support first. Some of this information is needed to
+ // setup the architecture state.
+
+ // Note, allocation and anything that depends on allocated memory is
+ // deferred until init(), in order to handle OOM properly.
+
+ stack_ = nullptr;
+ stackLimit_ = 0;
+ pc_modified_ = false;
+ icount_ = 0;
+ break_count_ = 0;
+ break_pc_ = nullptr;
+ break_instr_ = 0;
+ single_stepping_ = false;
+ single_step_callback_ = nullptr;
+ single_step_callback_arg_ = nullptr;
+
+ // Set up architecture state.
+ // All registers are initialized to zero to start with.
+ for (int i = 0; i < Register::kNumSimuRegisters; i++) {
+ registers_[i] = 0;
+ }
+ for (int i = 0; i < Simulator::FPURegister::kNumFPURegisters; i++) {
+ FPUregisters_[i] = 0;
+ }
+ FCSR_ = 0;
+ LLBit_ = false;
+ LLAddr_ = 0;
+ lastLLValue_ = 0;
+
+ // The ra and pc are initialized to a known bad value that will cause an
+ // access violation if the simulator ever tries to execute it.
+ registers_[pc] = bad_ra;
+ registers_[ra] = bad_ra;
+
+ for (int i = 0; i < kNumExceptions; i++) {
+ exceptions[i] = 0;
+ }
+
+ lastDebuggerInput_ = nullptr;
+}
+
+bool Simulator::init() {
+ // Allocate 2MB for the stack. Note that we will only use 1MB, see below.
+ static const size_t stackSize = 2 * 1024 * 1024;
+ stack_ = js_pod_malloc<char>(stackSize);
+ if (!stack_) {
+ return false;
+ }
+
+ // Leave a safety margin of 1MB to prevent overrunning the stack when
+ // pushing values (total stack size is 2MB).
+ stackLimit_ = reinterpret_cast<uintptr_t>(stack_) + 1024 * 1024;
+
+ // The sp is initialized to point to the bottom (high address) of the
+ // allocated stack area. To be safe in potential stack underflows we leave
+ // some buffer below.
+ registers_[sp] = reinterpret_cast<int64_t>(stack_) + stackSize - 64;
+
+ return true;
+}
+
+// When the generated code calls an external reference we need to catch that in
+// the simulator. The external reference will be a function compiled for the
+// host architecture. We need to call that function instead of trying to
+// execute it with the simulator. We do that by redirecting the external
+// reference to a swi (software-interrupt) instruction that is handled by
+// the simulator. We write the original destination of the jump just at a known
+// offset from the swi instruction so the simulator knows what to call.
+class Redirection {
+ friend class SimulatorProcess;
+
+ // sim's lock must already be held.
+ Redirection(void* nativeFunction, ABIFunctionType type)
+ : nativeFunction_(nativeFunction),
+ swiInstruction_(kCallRedirInstr),
+ type_(type),
+ next_(nullptr) {
+ next_ = SimulatorProcess::redirection();
+ if (!SimulatorProcess::ICacheCheckingDisableCount) {
+ FlushICacheLocked(SimulatorProcess::icache(), addressOfSwiInstruction(),
+ SimInstruction::kInstrSize);
+ }
+ SimulatorProcess::setRedirection(this);
+ }
+
+ public:
+ void* addressOfSwiInstruction() { return &swiInstruction_; }
+ void* nativeFunction() const { return nativeFunction_; }
+ ABIFunctionType type() const { return type_; }
+
+ static Redirection* Get(void* nativeFunction, ABIFunctionType type) {
+ AutoLockSimulatorCache als;
+
+ Redirection* current = SimulatorProcess::redirection();
+ for (; current != nullptr; current = current->next_) {
+ if (current->nativeFunction_ == nativeFunction) {
+ MOZ_ASSERT(current->type() == type);
+ return current;
+ }
+ }
+
+ // Note: we can't use js_new here because the constructor is private.
+ AutoEnterOOMUnsafeRegion oomUnsafe;
+ Redirection* redir = js_pod_malloc<Redirection>(1);
+ if (!redir) {
+ oomUnsafe.crash("Simulator redirection");
+ }
+ new (redir) Redirection(nativeFunction, type);
+ return redir;
+ }
+
+ static Redirection* FromSwiInstruction(SimInstruction* swiInstruction) {
+ uint8_t* addrOfSwi = reinterpret_cast<uint8_t*>(swiInstruction);
+ uint8_t* addrOfRedirection =
+ addrOfSwi - offsetof(Redirection, swiInstruction_);
+ return reinterpret_cast<Redirection*>(addrOfRedirection);
+ }
+
+ private:
+ void* nativeFunction_;
+ uint32_t swiInstruction_;
+ ABIFunctionType type_;
+ Redirection* next_;
+};
+
+Simulator::~Simulator() { js_free(stack_); }
+
+SimulatorProcess::SimulatorProcess()
+ : cacheLock_(mutexid::SimulatorCacheLock), redirection_(nullptr) {
+ if (getenv("MIPS_SIM_ICACHE_CHECKS")) {
+ ICacheCheckingDisableCount = 0;
+ }
+}
+
+SimulatorProcess::~SimulatorProcess() {
+ Redirection* r = redirection_;
+ while (r) {
+ Redirection* next = r->next_;
+ js_delete(r);
+ r = next;
+ }
+}
+
+/* static */
+void* Simulator::RedirectNativeFunction(void* nativeFunction,
+ ABIFunctionType type) {
+ Redirection* redirection = Redirection::Get(nativeFunction, type);
+ return redirection->addressOfSwiInstruction();
+}
+
+// Get the active Simulator for the current thread.
+Simulator* Simulator::Current() {
+ JSContext* cx = TlsContext.get();
+ MOZ_ASSERT(CurrentThreadCanAccessRuntime(cx->runtime()));
+ return cx->simulator();
+}
+
+// Sets the register in the architecture state. It will also deal with updating
+// Simulator internal state for special registers such as PC.
+void Simulator::setRegister(int reg, int64_t value) {
+ MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters));
+ if (reg == pc) {
+ pc_modified_ = true;
+ }
+
+ // Zero register always holds 0.
+ registers_[reg] = (reg == 0) ? 0 : value;
+}
+
+void Simulator::setFpuRegister(int fpureg, int64_t value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ FPUregisters_[fpureg] = value;
+}
+
+void Simulator::setFpuRegisterLo(int fpureg, int32_t value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ *mozilla::BitwiseCast<int32_t*>(&FPUregisters_[fpureg]) = value;
+}
+
+void Simulator::setFpuRegisterHi(int fpureg, int32_t value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ *((mozilla::BitwiseCast<int32_t*>(&FPUregisters_[fpureg])) + 1) = value;
+}
+
+void Simulator::setFpuRegisterFloat(int fpureg, float value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ *mozilla::BitwiseCast<float*>(&FPUregisters_[fpureg]) = value;
+}
+
+void Simulator::setFpuRegisterDouble(int fpureg, double value) {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ *mozilla::BitwiseCast<double*>(&FPUregisters_[fpureg]) = value;
+}
+
+// Get the register from the architecture state. This function does handle
+// the special case of accessing the PC register.
+int64_t Simulator::getRegister(int reg) const {
+ MOZ_ASSERT((reg >= 0) && (reg < Register::kNumSimuRegisters));
+ if (reg == 0) {
+ return 0;
+ }
+ return registers_[reg] + ((reg == pc) ? SimInstruction::kPCReadOffset : 0);
+}
+
+int64_t Simulator::getFpuRegister(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return FPUregisters_[fpureg];
+}
+
+int32_t Simulator::getFpuRegisterLo(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *mozilla::BitwiseCast<int32_t*>(&FPUregisters_[fpureg]);
+}
+
+int32_t Simulator::getFpuRegisterHi(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *((mozilla::BitwiseCast<int32_t*>(&FPUregisters_[fpureg])) + 1);
+}
+
+float Simulator::getFpuRegisterFloat(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *mozilla::BitwiseCast<float*>(&FPUregisters_[fpureg]);
+}
+
+double Simulator::getFpuRegisterDouble(int fpureg) const {
+ MOZ_ASSERT((fpureg >= 0) &&
+ (fpureg < Simulator::FPURegister::kNumFPURegisters));
+ return *mozilla::BitwiseCast<double*>(&FPUregisters_[fpureg]);
+}
+
+void Simulator::setCallResultDouble(double result) {
+ setFpuRegisterDouble(f0, result);
+}
+
+void Simulator::setCallResultFloat(float result) {
+ setFpuRegisterFloat(f0, result);
+}
+
+void Simulator::setCallResult(int64_t res) { setRegister(v0, res); }
+
+void Simulator::setCallResult(__int128_t res) {
+ setRegister(v0, I64(res));
+ setRegister(v1, I64(res >> 64));
+}
+
+// Helper functions for setting and testing the FCSR register's bits.
+void Simulator::setFCSRBit(uint32_t cc, bool value) {
+ if (value) {
+ FCSR_ |= (1 << cc);
+ } else {
+ FCSR_ &= ~(1 << cc);
+ }
+}
+
+bool Simulator::testFCSRBit(uint32_t cc) { return FCSR_ & (1 << cc); }
+
+// Sets the rounding error codes in FCSR based on the result of the rounding.
+// Returns true if the operation was invalid.
+template <typename T>
+bool Simulator::setFCSRRoundError(double original, double rounded) {
+ bool ret = false;
+
+ setFCSRBit(kFCSRInexactCauseBit, false);
+ setFCSRBit(kFCSRUnderflowCauseBit, false);
+ setFCSRBit(kFCSROverflowCauseBit, false);
+ setFCSRBit(kFCSRInvalidOpCauseBit, false);
+
+ if (!std::isfinite(original) || !std::isfinite(rounded)) {
+ setFCSRBit(kFCSRInvalidOpFlagBit, true);
+ setFCSRBit(kFCSRInvalidOpCauseBit, true);
+ ret = true;
+ }
+
+ if (original != rounded) {
+ setFCSRBit(kFCSRInexactFlagBit, true);
+ setFCSRBit(kFCSRInexactCauseBit, true);
+ }
+
+ if (rounded < DBL_MIN && rounded > -DBL_MIN && rounded != 0) {
+ setFCSRBit(kFCSRUnderflowFlagBit, true);
+ setFCSRBit(kFCSRUnderflowCauseBit, true);
+ ret = true;
+ }
+
+ if ((long double)rounded > (long double)std::numeric_limits<T>::max() ||
+ (long double)rounded < (long double)std::numeric_limits<T>::min()) {
+ setFCSRBit(kFCSROverflowFlagBit, true);
+ setFCSRBit(kFCSROverflowCauseBit, true);
+ // The reference is not really clear but it seems this is required:
+ setFCSRBit(kFCSRInvalidOpFlagBit, true);
+ setFCSRBit(kFCSRInvalidOpCauseBit, true);
+ ret = true;
+ }
+
+ return ret;
+}
+
+// Raw access to the PC register.
+void Simulator::set_pc(int64_t value) {
+ pc_modified_ = true;
+ registers_[pc] = value;
+}
+
+bool Simulator::has_bad_pc() const {
+ return ((registers_[pc] == bad_ra) || (registers_[pc] == end_sim_pc));
+}
+
+// Raw access to the PC register without the special adjustment when reading.
+int64_t Simulator::get_pc() const { return registers_[pc]; }
+
+JS::ProfilingFrameIterator::RegisterState Simulator::registerState() {
+ wasm::RegisterState state;
+ state.pc = (void*)get_pc();
+ state.fp = (void*)getRegister(fp);
+ state.sp = (void*)getRegister(sp);
+ state.lr = (void*)getRegister(ra);
+ return state;
+}
+
+static bool AllowUnaligned() {
+ static bool hasReadFlag = false;
+ static bool unalignedAllowedFlag = false;
+ if (!hasReadFlag) {
+ unalignedAllowedFlag = !!getenv("MIPS_UNALIGNED");
+ hasReadFlag = true;
+ }
+ return unalignedAllowedFlag;
+}
+
+// MIPS memory instructions (except lw(d)l/r , sw(d)l/r) trap on unaligned
+// memory access enabling the OS to handle them via trap-and-emulate. Note that
+// simulator runs have the runtime system running directly on the host system
+// and only generated code is executed in the simulator. Since the host is
+// typically IA32 it will not trap on unaligned memory access. We assume that
+// that executing correct generated code will not produce unaligned memory
+// access, so we explicitly check for address alignment and trap. Note that
+// trapping does not occur when executing wasm code, which requires that
+// unaligned memory access provides correct result.
+
+uint8_t Simulator::readBU(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return 0xff;
+ }
+
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(addr);
+ return *ptr;
+}
+
+int8_t Simulator::readB(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return -1;
+ }
+
+ int8_t* ptr = reinterpret_cast<int8_t*>(addr);
+ return *ptr;
+}
+
+void Simulator::writeB(uint64_t addr, uint8_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return;
+ }
+
+ uint8_t* ptr = reinterpret_cast<uint8_t*>(addr);
+ *ptr = value;
+}
+
+void Simulator::writeB(uint64_t addr, int8_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 1)) {
+ return;
+ }
+
+ int8_t* ptr = reinterpret_cast<int8_t*>(addr);
+ *ptr = value;
+}
+
+uint16_t Simulator::readHU(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return 0xffff;
+ }
+
+ if (AllowUnaligned() || (addr & 1) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ uint16_t* ptr = reinterpret_cast<uint16_t*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned unsigned halfword read at 0x%016" PRIx64
+ ", pc=0x%016" PRIxPTR "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int16_t Simulator::readH(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return -1;
+ }
+
+ if (AllowUnaligned() || (addr & 1) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ int16_t* ptr = reinterpret_cast<int16_t*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned signed halfword read at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR
+ "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+void Simulator::writeH(uint64_t addr, uint16_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return;
+ }
+
+ if (AllowUnaligned() || (addr & 1) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ uint16_t* ptr = reinterpret_cast<uint16_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned unsigned halfword write at 0x%016" PRIx64
+ ", pc=0x%016" PRIxPTR "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+void Simulator::writeH(uint64_t addr, int16_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 2)) {
+ return;
+ }
+
+ if (AllowUnaligned() || (addr & 1) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ int16_t* ptr = reinterpret_cast<int16_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned halfword write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+uint32_t Simulator::readWU(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ if (AllowUnaligned() || (addr & 3) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ uint32_t* ptr = reinterpret_cast<uint32_t*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned read at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int32_t Simulator::readW(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ if (AllowUnaligned() || (addr & 3) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ int32_t* ptr = reinterpret_cast<int32_t*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned read at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+void Simulator::writeW(uint64_t addr, uint32_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return;
+ }
+
+ if (AllowUnaligned() || (addr & 3) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ uint32_t* ptr = reinterpret_cast<uint32_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+void Simulator::writeW(uint64_t addr, int32_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 4)) {
+ return;
+ }
+
+ if (AllowUnaligned() || (addr & 3) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ int32_t* ptr = reinterpret_cast<int32_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+int64_t Simulator::readDW(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return -1;
+ }
+
+ if (AllowUnaligned() || (addr & kPointerAlignmentMask) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ intptr_t* ptr = reinterpret_cast<intptr_t*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned read at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+void Simulator::writeDW(uint64_t addr, int64_t value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return;
+ }
+
+ if (AllowUnaligned() || (addr & kPointerAlignmentMask) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ int64_t* ptr = reinterpret_cast<int64_t*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+double Simulator::readD(uint64_t addr, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return NAN;
+ }
+
+ if (AllowUnaligned() || (addr & kDoubleAlignmentMask) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ double* ptr = reinterpret_cast<double*>(addr);
+ return *ptr;
+ }
+ printf("Unaligned (double) read at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+void Simulator::writeD(uint64_t addr, double value, SimInstruction* instr) {
+ if (handleWasmSegFault(addr, 8)) {
+ return;
+ }
+
+ if (AllowUnaligned() || (addr & kDoubleAlignmentMask) == 0 ||
+ wasm::InCompiledCode(reinterpret_cast<void*>(get_pc()))) {
+ double* ptr = reinterpret_cast<double*>(addr);
+ LLBit_ = false;
+ *ptr = value;
+ return;
+ }
+ printf("Unaligned (double) write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n",
+ addr, reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+}
+
+int Simulator::loadLinkedW(uint64_t addr, SimInstruction* instr) {
+ if ((addr & 3) == 0) {
+ if (handleWasmSegFault(addr, 4)) {
+ return -1;
+ }
+
+ volatile int32_t* ptr = reinterpret_cast<volatile int32_t*>(addr);
+ int32_t value = *ptr;
+ lastLLValue_ = value;
+ LLAddr_ = addr;
+ // Note that any memory write or "external" interrupt should reset this
+ // value to false.
+ LLBit_ = true;
+ return value;
+ }
+ printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int Simulator::storeConditionalW(uint64_t addr, int value,
+ SimInstruction* instr) {
+ // Correct behavior in this case, as defined by architecture, is to just
+ // return 0, but there is no point at allowing that. It is certainly an
+ // indicator of a bug.
+ if (addr != LLAddr_) {
+ printf("SC to bad address: 0x%016" PRIx64 ", pc=0x%016" PRIx64
+ ", expected: 0x%016" PRIx64 "\n",
+ addr, reinterpret_cast<intptr_t>(instr), LLAddr_);
+ MOZ_CRASH();
+ }
+
+ if ((addr & 3) == 0) {
+ SharedMem<int32_t*> ptr =
+ SharedMem<int32_t*>::shared(reinterpret_cast<int32_t*>(addr));
+
+ if (!LLBit_) {
+ return 0;
+ }
+
+ LLBit_ = false;
+ LLAddr_ = 0;
+ int32_t expected = int32_t(lastLLValue_);
+ int32_t old =
+ AtomicOperations::compareExchangeSeqCst(ptr, expected, int32_t(value));
+ return (old == expected) ? 1 : 0;
+ }
+ printf("Unaligned SC at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int64_t Simulator::loadLinkedD(uint64_t addr, SimInstruction* instr) {
+ if ((addr & kPointerAlignmentMask) == 0) {
+ if (handleWasmSegFault(addr, 8)) {
+ return -1;
+ }
+
+ volatile int64_t* ptr = reinterpret_cast<volatile int64_t*>(addr);
+ int64_t value = *ptr;
+ lastLLValue_ = value;
+ LLAddr_ = addr;
+ // Note that any memory write or "external" interrupt should reset this
+ // value to false.
+ LLBit_ = true;
+ return value;
+ }
+ printf("Unaligned write at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+int Simulator::storeConditionalD(uint64_t addr, int64_t value,
+ SimInstruction* instr) {
+ // Correct behavior in this case, as defined by architecture, is to just
+ // return 0, but there is no point at allowing that. It is certainly an
+ // indicator of a bug.
+ if (addr != LLAddr_) {
+ printf("SC to bad address: 0x%016" PRIx64 ", pc=0x%016" PRIx64
+ ", expected: 0x%016" PRIx64 "\n",
+ addr, reinterpret_cast<intptr_t>(instr), LLAddr_);
+ MOZ_CRASH();
+ }
+
+ if ((addr & kPointerAlignmentMask) == 0) {
+ SharedMem<int64_t*> ptr =
+ SharedMem<int64_t*>::shared(reinterpret_cast<int64_t*>(addr));
+
+ if (!LLBit_) {
+ return 0;
+ }
+
+ LLBit_ = false;
+ LLAddr_ = 0;
+ int64_t expected = lastLLValue_;
+ int64_t old =
+ AtomicOperations::compareExchangeSeqCst(ptr, expected, int64_t(value));
+ return (old == expected) ? 1 : 0;
+ }
+ printf("Unaligned SC at 0x%016" PRIx64 ", pc=0x%016" PRIxPTR "\n", addr,
+ reinterpret_cast<intptr_t>(instr));
+ MOZ_CRASH();
+ return 0;
+}
+
+uintptr_t Simulator::stackLimit() const { return stackLimit_; }
+
+uintptr_t* Simulator::addressOfStackLimit() { return &stackLimit_; }
+
+bool Simulator::overRecursed(uintptr_t newsp) const {
+ if (newsp == 0) {
+ newsp = getRegister(sp);
+ }
+ return newsp <= stackLimit();
+}
+
+bool Simulator::overRecursedWithExtra(uint32_t extra) const {
+ uintptr_t newsp = getRegister(sp) - extra;
+ return newsp <= stackLimit();
+}
+
+// Unsupported instructions use format to print an error and stop execution.
+void Simulator::format(SimInstruction* instr, const char* format) {
+ printf("Simulator found unsupported instruction:\n 0x%016lx: %s\n",
+ reinterpret_cast<intptr_t>(instr), format);
+ MOZ_CRASH();
+}
+
+// Note: With the code below we assume that all runtime calls return a 64 bits
+// result. If they don't, the v1 result register contains a bogus value, which
+// is fine because it is caller-saved.
+typedef int64_t (*Prototype_General0)();
+typedef int64_t (*Prototype_General1)(int64_t arg0);
+typedef int64_t (*Prototype_General2)(int64_t arg0, int64_t arg1);
+typedef int64_t (*Prototype_General3)(int64_t arg0, int64_t arg1, int64_t arg2);
+typedef int64_t (*Prototype_General4)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3);
+typedef int64_t (*Prototype_General5)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3, int64_t arg4);
+typedef int64_t (*Prototype_General6)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3, int64_t arg4, int64_t arg5);
+typedef int64_t (*Prototype_General7)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3, int64_t arg4, int64_t arg5,
+ int64_t arg6);
+typedef int64_t (*Prototype_General8)(int64_t arg0, int64_t arg1, int64_t arg2,
+ int64_t arg3, int64_t arg4, int64_t arg5,
+ int64_t arg6, int64_t arg7);
+typedef int64_t (*Prototype_GeneralGeneralGeneralInt64)(int64_t arg0,
+ int64_t arg1,
+ int64_t arg2,
+ int64_t arg3);
+typedef int64_t (*Prototype_GeneralGeneralInt64Int64)(int64_t arg0,
+ int64_t arg1,
+ int64_t arg2,
+ int64_t arg3);
+
+typedef int64_t (*Prototype_Int_Double)(double arg0);
+typedef int64_t (*Prototype_Int_IntDouble)(int64_t arg0, double arg1);
+typedef int64_t (*Prototype_Int_DoubleInt)(double arg0, int64_t arg1);
+typedef int64_t (*Prototype_Int_DoubleIntInt)(double arg0, int64_t arg1,
+ int64_t arg2);
+typedef int64_t (*Prototype_Int_IntDoubleIntInt)(int64_t arg0, double arg1,
+ int64_t arg2, int64_t arg3);
+
+typedef float (*Prototype_Float32_Float32)(float arg0);
+typedef int64_t (*Prototype_Int_Float32)(float arg0);
+typedef float (*Prototype_Float32_Float32Float32)(float arg0, float arg1);
+
+typedef double (*Prototype_Double_None)();
+typedef double (*Prototype_Double_Double)(double arg0);
+typedef double (*Prototype_Double_Int)(int64_t arg0);
+typedef double (*Prototype_Double_DoubleInt)(double arg0, int64_t arg1);
+typedef double (*Prototype_Double_IntDouble)(int64_t arg0, double arg1);
+typedef double (*Prototype_Double_DoubleDouble)(double arg0, double arg1);
+typedef double (*Prototype_Double_DoubleDoubleDouble)(double arg0, double arg1,
+ double arg2);
+typedef double (*Prototype_Double_DoubleDoubleDoubleDouble)(double arg0,
+ double arg1,
+ double arg2,
+ double arg3);
+
+typedef int32_t (*Prototype_Int32_General)(int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32)(int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32)(int64_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32)(int64_t, int32_t,
+ int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32Int32)(
+ int64_t, int32_t, int32_t, int32_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32Int32General)(
+ int64_t, int32_t, int32_t, int32_t, int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int32General)(
+ int64_t, int32_t, int32_t, int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32Int64)(int64_t, int32_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int32General)(int64_t, int32_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32Int64Int64)(int64_t, int32_t,
+ int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32)(int64_t, int32_t,
+ int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt32GeneralInt32Int32)(
+ int64_t, int32_t, int64_t, int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneral)(int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralGeneral)(int64_t, int64_t,
+ int64_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralInt32Int32)(int64_t, int64_t,
+ int32_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int32Int32)(int64_t, int64_t,
+ int32_t, int32_t,
+ int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32)(int64_t, int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int64)(int64_t, int64_t,
+ int32_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int32Int64General)(
+ int64_t, int64_t, int32_t, int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int64Int64)(int64_t, int64_t,
+ int64_t, int64_t);
+typedef int32_t (*Prototype_Int32_GeneralInt64Int64Int64General)(
+ int64_t, int64_t, int64_t, int64_t, int64_t);
+typedef int64_t (*Prototype_General_GeneralInt32)(int64_t, int32_t);
+typedef int64_t (*Prototype_General_GeneralInt32Int32)(int64_t, int32_t,
+ int32_t);
+typedef int64_t (*Prototype_General_GeneralInt32General)(int64_t, int32_t,
+ int64_t);
+typedef int64_t (*Prototype_General_GeneralInt32Int32GeneralInt32)(
+ int64_t, int32_t, int32_t, int64_t, int32_t);
+typedef int32_t (*Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32)(
+ int64_t, int64_t, int32_t, int64_t, int32_t, int32_t, int32_t);
+typedef int64_t (*Prototype_Int64_General)(int64_t);
+typedef int64_t (*Prototype_Int64_GeneralInt64)(int64_t, int64_t);
+
+// Software interrupt instructions are used by the simulator to call into C++.
+void Simulator::softwareInterrupt(SimInstruction* instr) {
+ int32_t func = instr->functionFieldRaw();
+ uint32_t code = (func == ff_break) ? instr->bits(25, 6) : -1;
+
+ // We first check if we met a call_rt_redirected.
+ if (instr->instructionBits() == kCallRedirInstr) {
+#if !defined(USES_N64_ABI)
+ MOZ_CRASH("Only N64 ABI supported.");
+#else
+ Redirection* redirection = Redirection::FromSwiInstruction(instr);
+ uintptr_t nativeFn =
+ reinterpret_cast<uintptr_t>(redirection->nativeFunction());
+
+ int64_t arg0 = getRegister(a0);
+ int64_t arg1 = getRegister(a1);
+ int64_t arg2 = getRegister(a2);
+ int64_t arg3 = getRegister(a3);
+ int64_t arg4 = getRegister(a4);
+ int64_t arg5 = getRegister(a5);
+
+ // This is dodgy but it works because the C entry stubs are never moved.
+ // See comment in codegen-arm.cc and bug 1242173.
+ int64_t saved_ra = getRegister(ra);
+
+ intptr_t external =
+ reinterpret_cast<intptr_t>(redirection->nativeFunction());
+
+ bool stack_aligned = (getRegister(sp) & (ABIStackAlignment - 1)) == 0;
+ if (!stack_aligned) {
+ fprintf(stderr, "Runtime call with unaligned stack!\n");
+ MOZ_CRASH();
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ switch (redirection->type()) {
+ case Args_General0: {
+ Prototype_General0 target =
+ reinterpret_cast<Prototype_General0>(external);
+ int64_t result = target();
+ setCallResult(result);
+ break;
+ }
+ case Args_General1: {
+ Prototype_General1 target =
+ reinterpret_cast<Prototype_General1>(external);
+ int64_t result = target(arg0);
+ setCallResult(result);
+ break;
+ }
+ case Args_General2: {
+ Prototype_General2 target =
+ reinterpret_cast<Prototype_General2>(external);
+ int64_t result = target(arg0, arg1);
+ setCallResult(result);
+ break;
+ }
+ case Args_General3: {
+ Prototype_General3 target =
+ reinterpret_cast<Prototype_General3>(external);
+ int64_t result = target(arg0, arg1, arg2);
+ if (external == intptr_t(&js::wasm::Instance::wake_m32)) {
+ result = int32_t(result);
+ }
+ setCallResult(result);
+ break;
+ }
+ case Args_General4: {
+ Prototype_General4 target =
+ reinterpret_cast<Prototype_General4>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ setCallResult(result);
+ break;
+ }
+ case Args_General5: {
+ Prototype_General5 target =
+ reinterpret_cast<Prototype_General5>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4);
+ setCallResult(result);
+ break;
+ }
+ case Args_General6: {
+ Prototype_General6 target =
+ reinterpret_cast<Prototype_General6>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5);
+ setCallResult(result);
+ break;
+ }
+ case Args_General7: {
+ Prototype_General7 target =
+ reinterpret_cast<Prototype_General7>(external);
+ int64_t arg6 = getRegister(a6);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6);
+ setCallResult(result);
+ break;
+ }
+ case Args_General8: {
+ Prototype_General8 target =
+ reinterpret_cast<Prototype_General8>(external);
+ int64_t arg6 = getRegister(a6);
+ int64_t arg7 = getRegister(a7);
+ int64_t result = target(arg0, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+ setCallResult(result);
+ break;
+ }
+ case Args_Double_None: {
+ Prototype_Double_None target =
+ reinterpret_cast<Prototype_Double_None>(external);
+ double dresult = target();
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int_Double: {
+ double dval0 = getFpuRegisterDouble(12);
+ Prototype_Int_Double target =
+ reinterpret_cast<Prototype_Int_Double>(external);
+ int64_t result = target(dval0);
+ if (external == intptr_t((int32_t(*)(double))JS::ToInt32)) {
+ result = int32_t(result);
+ }
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Int_GeneralGeneralGeneralInt64: {
+ Prototype_GeneralGeneralGeneralInt64 target =
+ reinterpret_cast<Prototype_GeneralGeneralGeneralInt64>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ if (external == intptr_t(&js::wasm::Instance::wait_i32_m32)) {
+ result = int32_t(result);
+ }
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Int_GeneralGeneralInt64Int64: {
+ Prototype_GeneralGeneralInt64Int64 target =
+ reinterpret_cast<Prototype_GeneralGeneralInt64Int64>(external);
+ int64_t result = target(arg0, arg1, arg2, arg3);
+ if (external == intptr_t(&js::wasm::Instance::wait_i64_m32)) {
+ result = int32_t(result);
+ }
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Int_DoubleInt: {
+ double dval = getFpuRegisterDouble(12);
+ Prototype_Int_DoubleInt target =
+ reinterpret_cast<Prototype_Int_DoubleInt>(external);
+ int64_t result = target(dval, arg1);
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Int_DoubleIntInt: {
+ double dval = getFpuRegisterDouble(12);
+ Prototype_Int_DoubleIntInt target =
+ reinterpret_cast<Prototype_Int_DoubleIntInt>(external);
+ int64_t result = target(dval, arg1, arg2);
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Int_IntDoubleIntInt: {
+ double dval = getFpuRegisterDouble(13);
+ Prototype_Int_IntDoubleIntInt target =
+ reinterpret_cast<Prototype_Int_IntDoubleIntInt>(external);
+ int64_t result = target(arg0, dval, arg2, arg3);
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Double_Double: {
+ double dval0 = getFpuRegisterDouble(12);
+ Prototype_Double_Double target =
+ reinterpret_cast<Prototype_Double_Double>(external);
+ double dresult = target(dval0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Float32_Float32: {
+ float fval0;
+ fval0 = getFpuRegisterFloat(12);
+ Prototype_Float32_Float32 target =
+ reinterpret_cast<Prototype_Float32_Float32>(external);
+ float fresult = target(fval0);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Int_Float32: {
+ float fval0;
+ fval0 = getFpuRegisterFloat(12);
+ Prototype_Int_Float32 target =
+ reinterpret_cast<Prototype_Int_Float32>(external);
+ int64_t result = target(fval0);
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Float32_Float32Float32: {
+ float fval0;
+ float fval1;
+ fval0 = getFpuRegisterFloat(12);
+ fval1 = getFpuRegisterFloat(13);
+ Prototype_Float32_Float32Float32 target =
+ reinterpret_cast<Prototype_Float32_Float32Float32>(external);
+ float fresult = target(fval0, fval1);
+ setCallResultFloat(fresult);
+ break;
+ }
+ case Args_Double_Int: {
+ Prototype_Double_Int target =
+ reinterpret_cast<Prototype_Double_Int>(external);
+ double dresult = target(arg0);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleInt: {
+ double dval0 = getFpuRegisterDouble(12);
+ Prototype_Double_DoubleInt target =
+ reinterpret_cast<Prototype_Double_DoubleInt>(external);
+ double dresult = target(dval0, arg1);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleDouble: {
+ double dval0 = getFpuRegisterDouble(12);
+ double dval1 = getFpuRegisterDouble(13);
+ Prototype_Double_DoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDouble>(external);
+ double dresult = target(dval0, dval1);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_IntDouble: {
+ double dval1 = getFpuRegisterDouble(13);
+ Prototype_Double_IntDouble target =
+ reinterpret_cast<Prototype_Double_IntDouble>(external);
+ double dresult = target(arg0, dval1);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int_IntDouble: {
+ double dval1 = getFpuRegisterDouble(13);
+ Prototype_Int_IntDouble target =
+ reinterpret_cast<Prototype_Int_IntDouble>(external);
+ int64_t result = target(arg0, dval1);
+ setRegister(v0, result);
+ break;
+ }
+ case Args_Double_DoubleDoubleDouble: {
+ double dval0 = getFpuRegisterDouble(12);
+ double dval1 = getFpuRegisterDouble(13);
+ double dval2 = getFpuRegisterDouble(14);
+ Prototype_Double_DoubleDoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDoubleDouble>(external);
+ double dresult = target(dval0, dval1, dval2);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Double_DoubleDoubleDoubleDouble: {
+ double dval0 = getFpuRegisterDouble(12);
+ double dval1 = getFpuRegisterDouble(13);
+ double dval2 = getFpuRegisterDouble(14);
+ double dval3 = getFpuRegisterDouble(15);
+ Prototype_Double_DoubleDoubleDoubleDouble target =
+ reinterpret_cast<Prototype_Double_DoubleDoubleDoubleDouble>(
+ external);
+ double dresult = target(dval0, dval1, dval2, dval3);
+ setCallResultDouble(dresult);
+ break;
+ }
+ case Args_Int32_General: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_General>(nativeFn)(arg0);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32>(nativeFn)(
+ arg0, I32(arg1));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), I32(arg4));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32Int32Int32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), I32(arg4),
+ I32(arg5));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32Int32General: {
+ int32_t ret = reinterpret_cast<
+ Prototype_Int32_GeneralInt32Int32Int32Int32General>(nativeFn)(
+ arg0, I32(arg1), I32(arg2), I32(arg3), I32(arg4), arg5);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int32General>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), I32(arg3), arg4);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int32Int64>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), arg3);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int32General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32Int32General>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), arg3);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32Int64Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt32Int64Int64>(
+ nativeFn)(arg0, I32(arg1), arg2, arg3);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32GeneralInt32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32GeneralInt32>(
+ nativeFn)(arg0, I32(arg1), arg2, I32(arg3));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralInt32GeneralInt32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt32GeneralInt32Int32>(
+ nativeFn)(arg0, I32(arg1), arg2, I32(arg3), I32(arg4));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralGeneral: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralGeneral>(
+ nativeFn)(arg0, arg1);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralGeneralGeneral: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralGeneralGeneral>(
+ nativeFn)(arg0, arg1, arg2);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_Int32_GeneralGeneralInt32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralGeneralInt32Int32>(
+ nativeFn)(arg0, arg1, I32(arg2), I32(arg3));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int32Int32: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int32Int32>(
+ nativeFn)(arg0, arg1, I32(arg2), I32(arg3), I32(arg4));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int32>(
+ nativeFn)(arg0, arg1, I32(arg2));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int64>(
+ nativeFn)(arg0, arg1, I32(arg2), arg3);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int32Int64General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int32Int64General>(
+ nativeFn)(arg0, arg1, I32(arg2), arg3, arg4);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int64Int64: {
+ int32_t ret = reinterpret_cast<Prototype_Int32_GeneralInt64Int64Int64>(
+ nativeFn)(arg0, arg1, arg2, arg3);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int32_GeneralInt64Int64Int64General: {
+ int32_t ret =
+ reinterpret_cast<Prototype_Int32_GeneralInt64Int64Int64General>(
+ nativeFn)(arg0, arg1, arg2, arg3, arg4);
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case Args_General_GeneralInt32: {
+ int64_t ret = reinterpret_cast<Prototype_General_GeneralInt32>(
+ nativeFn)(arg0, I32(arg1));
+ setRegister(v0, ret);
+ break;
+ }
+ case Args_General_GeneralInt32Int32: {
+ int64_t ret = reinterpret_cast<Prototype_General_GeneralInt32Int32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2));
+ setRegister(v0, ret);
+ break;
+ }
+ case Args_General_GeneralInt32General: {
+ int64_t ret = reinterpret_cast<Prototype_General_GeneralInt32General>(
+ nativeFn)(arg0, I32(arg1), arg2);
+ setRegister(v0, ret);
+ break;
+ }
+ case js::jit::Args_General_GeneralInt32Int32GeneralInt32: {
+ int64_t ret =
+ reinterpret_cast<Prototype_General_GeneralInt32Int32GeneralInt32>(
+ nativeFn)(arg0, I32(arg1), I32(arg2), arg3, I32(arg4));
+ setRegister(v0, ret);
+ break;
+ }
+ case js::jit::Args_Int32_GeneralGeneralInt32GeneralInt32Int32Int32: {
+ int64_t arg6 = getRegister(a6);
+ int32_t ret = reinterpret_cast<
+ Prototype_Int32_GeneralGeneralInt32GeneralInt32Int32Int32>(
+ nativeFn)(arg0, arg1, I32(arg2), arg3, I32(arg4), I32(arg5),
+ I32(arg6));
+ setRegister(v0, I64(ret));
+ break;
+ }
+ case js::jit::Args_Int64_General: {
+ int64_t ret = reinterpret_cast<Prototype_Int64_General>(nativeFn)(arg0);
+ setRegister(v0, ret);
+ break;
+ }
+ case js::jit::Args_Int64_GeneralInt64: {
+ int64_t ret = reinterpret_cast<Prototype_Int64_GeneralInt64>(nativeFn)(
+ arg0, arg1);
+ setRegister(v0, ret);
+ break;
+ }
+ default:
+ MOZ_CRASH("Unknown function type.");
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ setRegister(ra, saved_ra);
+ set_pc(getRegister(ra));
+#endif
+ } else if (func == ff_break && code <= kMaxStopCode) {
+ if (isWatchpoint(code)) {
+ printWatchpoint(code);
+ } else {
+ increaseStopCounter(code);
+ handleStop(code, instr);
+ }
+ } else {
+ switch (func) {
+ case ff_tge:
+ case ff_tgeu:
+ case ff_tlt:
+ case ff_tltu:
+ case ff_teq:
+ case ff_tne:
+ if (instr->bits(15, 6) == kWasmTrapCode) {
+ uint8_t* newPC;
+ if (wasm::HandleIllegalInstruction(registerState(), &newPC)) {
+ set_pc(int64_t(newPC));
+ return;
+ }
+ }
+ };
+ // All remaining break_ codes, and all traps are handled here.
+ MipsDebugger dbg(this);
+ dbg.debug();
+ }
+}
+
+// Stop helper functions.
+bool Simulator::isWatchpoint(uint32_t code) {
+ return (code <= kMaxWatchpointCode);
+}
+
+void Simulator::printWatchpoint(uint32_t code) {
+ MipsDebugger dbg(this);
+ ++break_count_;
+ printf("\n---- break %d marker: %20" PRIi64 " (instr count: %20" PRIi64
+ ") ----\n",
+ code, break_count_, icount_);
+ dbg.printAllRegs(); // Print registers and continue running.
+}
+
+void Simulator::handleStop(uint32_t code, SimInstruction* instr) {
+ // Stop if it is enabled, otherwise go on jumping over the stop
+ // and the message address.
+ if (isEnabledStop(code)) {
+ MipsDebugger dbg(this);
+ dbg.stop(instr);
+ } else {
+ set_pc(get_pc() + 2 * SimInstruction::kInstrSize);
+ }
+}
+
+bool Simulator::isStopInstruction(SimInstruction* instr) {
+ int32_t func = instr->functionFieldRaw();
+ uint32_t code = U32(instr->bits(25, 6));
+ return (func == ff_break) && code > kMaxWatchpointCode &&
+ code <= kMaxStopCode;
+}
+
+bool Simulator::isEnabledStop(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ MOZ_ASSERT(code > kMaxWatchpointCode);
+ return !(watchedStops_[code].count_ & kStopDisabledBit);
+}
+
+void Simulator::enableStop(uint32_t code) {
+ if (!isEnabledStop(code)) {
+ watchedStops_[code].count_ &= ~kStopDisabledBit;
+ }
+}
+
+void Simulator::disableStop(uint32_t code) {
+ if (isEnabledStop(code)) {
+ watchedStops_[code].count_ |= kStopDisabledBit;
+ }
+}
+
+void Simulator::increaseStopCounter(uint32_t code) {
+ MOZ_ASSERT(code <= kMaxStopCode);
+ if ((watchedStops_[code].count_ & ~(1 << 31)) == 0x7fffffff) {
+ printf(
+ "Stop counter for code %i has overflowed.\n"
+ "Enabling this code and reseting the counter to 0.\n",
+ code);
+ watchedStops_[code].count_ = 0;
+ enableStop(code);
+ } else {
+ watchedStops_[code].count_++;
+ }
+}
+
+// Print a stop status.
+void Simulator::printStopInfo(uint32_t code) {
+ if (code <= kMaxWatchpointCode) {
+ printf("That is a watchpoint, not a stop.\n");
+ return;
+ } else if (code > kMaxStopCode) {
+ printf("Code too large, only %u stops can be used\n", kMaxStopCode + 1);
+ return;
+ }
+ const char* state = isEnabledStop(code) ? "Enabled" : "Disabled";
+ int32_t count = watchedStops_[code].count_ & ~kStopDisabledBit;
+ // Don't print the state of unused breakpoints.
+ if (count != 0) {
+ if (watchedStops_[code].desc_) {
+ printf("stop %i - 0x%x: \t%s, \tcounter = %i, \t%s\n", code, code, state,
+ count, watchedStops_[code].desc_);
+ } else {
+ printf("stop %i - 0x%x: \t%s, \tcounter = %i\n", code, code, state,
+ count);
+ }
+ }
+}
+
+void Simulator::signalExceptions() {
+ for (int i = 1; i < kNumExceptions; i++) {
+ if (exceptions[i] != 0) {
+ MOZ_CRASH("Error: Exception raised.");
+ }
+ }
+}
+
+// Helper function for decodeTypeRegister.
+void Simulator::configureTypeRegister(SimInstruction* instr, int64_t& alu_out,
+ __int128& i128hilo,
+ unsigned __int128& u128hilo,
+ int64_t& next_pc,
+ int32_t& return_addr_reg,
+ bool& do_interrupt) {
+ // Every local variable declared here needs to be const.
+ // This is to make sure that changed values are sent back to
+ // decodeTypeRegister correctly.
+
+ // Instruction fields.
+ const OpcodeField op = instr->opcodeFieldRaw();
+ const int32_t rs_reg = instr->rsValue();
+ const int64_t rs = getRegister(rs_reg);
+ const int32_t rt_reg = instr->rtValue();
+ const int64_t rt = getRegister(rt_reg);
+ const int32_t rd_reg = instr->rdValue();
+ const uint32_t sa = instr->saValue();
+
+ const int32_t fs_reg = instr->fsValue();
+ __int128 temp;
+
+ // ---------- Configuration.
+ switch (op) {
+ case op_cop1: // Coprocessor instructions.
+ switch (instr->rsFieldRaw()) {
+ case rs_bc1: // Handled in DecodeTypeImmed, should never come here.
+ MOZ_CRASH();
+ break;
+ case rs_cfc1:
+ // At the moment only FCSR is supported.
+ MOZ_ASSERT(fs_reg == kFCSRRegister);
+ alu_out = FCSR_;
+ break;
+ case rs_mfc1:
+ alu_out = getFpuRegisterLo(fs_reg);
+ break;
+ case rs_dmfc1:
+ alu_out = getFpuRegister(fs_reg);
+ break;
+ case rs_mfhc1:
+ alu_out = getFpuRegisterHi(fs_reg);
+ break;
+ case rs_ctc1:
+ case rs_mtc1:
+ case rs_dmtc1:
+ case rs_mthc1:
+ // Do the store in the execution step.
+ break;
+ case rs_s:
+ case rs_d:
+ case rs_w:
+ case rs_l:
+ case rs_ps:
+ // Do everything in the execution step.
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ case op_cop1x:
+ break;
+ case op_special:
+ switch (instr->functionFieldRaw()) {
+ case ff_jr:
+ case ff_jalr:
+ next_pc = getRegister(instr->rsValue());
+ return_addr_reg = instr->rdValue();
+ break;
+ case ff_sll:
+ alu_out = I64(I32(rt) << sa);
+ break;
+ case ff_dsll:
+ alu_out = rt << sa;
+ break;
+ case ff_dsll32:
+ alu_out = rt << (sa + 32);
+ break;
+ case ff_srl:
+ if (rs_reg == 0) {
+ // Regular logical right shift of a word by a fixed number of
+ // bits instruction. RS field is always equal to 0.
+ alu_out = I64(I32(U32(I32_CHECK(rt)) >> sa));
+ } else {
+ // Logical right-rotate of a word by a fixed number of bits. This
+ // is special case of SRL instruction, added in MIPS32 Release 2.
+ // RS field is equal to 00001.
+ alu_out = I64(I32((U32(I32_CHECK(rt)) >> sa) |
+ (U32(I32_CHECK(rt)) << (32 - sa))));
+ }
+ break;
+ case ff_dsrl:
+ if (rs_reg == 0) {
+ // Regular logical right shift of a double word by a fixed number of
+ // bits instruction. RS field is always equal to 0.
+ alu_out = U64(rt) >> sa;
+ } else {
+ // Logical right-rotate of a word by a fixed number of bits. This
+ // is special case of DSRL instruction, added in MIPS64 Release 2.
+ // RS field is equal to 00001.
+ alu_out = (U64(rt) >> sa) | (U64(rt) << (64 - sa));
+ }
+ break;
+ case ff_dsrl32:
+ if (rs_reg == 0) {
+ // Regular logical right shift of a double word by a fixed number of
+ // bits instruction. RS field is always equal to 0.
+ alu_out = U64(rt) >> (sa + 32);
+ } else {
+ // Logical right-rotate of a double word by a fixed number of bits.
+ // This is special case of DSRL instruction, added in MIPS64
+ // Release 2. RS field is equal to 00001.
+ alu_out = (U64(rt) >> (sa + 32)) | (U64(rt) << (64 - (sa + 32)));
+ }
+ break;
+ case ff_sra:
+ alu_out = I64(I32_CHECK(rt)) >> sa;
+ break;
+ case ff_dsra:
+ alu_out = rt >> sa;
+ break;
+ case ff_dsra32:
+ alu_out = rt >> (sa + 32);
+ break;
+ case ff_sllv:
+ alu_out = I64(I32(rt) << rs);
+ break;
+ case ff_dsllv:
+ alu_out = rt << rs;
+ break;
+ case ff_srlv:
+ if (sa == 0) {
+ // Regular logical right-shift of a word by a variable number of
+ // bits instruction. SA field is always equal to 0.
+ alu_out = I64(I32(U32(I32_CHECK(rt)) >> rs));
+ } else {
+ // Logical right-rotate of a word by a variable number of bits.
+ // This is special case od SRLV instruction, added in MIPS32
+ // Release 2. SA field is equal to 00001.
+ alu_out = I64(I32((U32(I32_CHECK(rt)) >> rs) |
+ (U32(I32_CHECK(rt)) << (32 - rs))));
+ }
+ break;
+ case ff_dsrlv:
+ if (sa == 0) {
+ // Regular logical right-shift of a double word by a variable number
+ // of bits instruction. SA field is always equal to 0.
+ alu_out = U64(rt) >> rs;
+ } else {
+ // Logical right-rotate of a double word by a variable number of
+ // bits. This is special case od DSRLV instruction, added in MIPS64
+ // Release 2. SA field is equal to 00001.
+ alu_out = (U64(rt) >> rs) | (U64(rt) << (64 - rs));
+ }
+ break;
+ case ff_srav:
+ alu_out = I64(I32_CHECK(rt) >> rs);
+ break;
+ case ff_dsrav:
+ alu_out = rt >> rs;
+ break;
+ case ff_mfhi:
+ alu_out = getRegister(HI);
+ break;
+ case ff_mflo:
+ alu_out = getRegister(LO);
+ break;
+ case ff_mult:
+ i128hilo = I64(U32(I32_CHECK(rs))) * I64(U32(I32_CHECK(rt)));
+ break;
+ case ff_dmult:
+ i128hilo = I128(rs) * I128(rt);
+ break;
+ case ff_multu:
+ u128hilo = U64(U32(I32_CHECK(rs))) * U64(U32(I32_CHECK(rt)));
+ break;
+ case ff_dmultu:
+ u128hilo = U128(rs) * U128(rt);
+ break;
+ case ff_add:
+ alu_out = I32_CHECK(rs) + I32_CHECK(rt);
+ if ((alu_out << 32) != (alu_out << 31)) {
+ exceptions[kIntegerOverflow] = 1;
+ }
+ alu_out = I32(alu_out);
+ break;
+ case ff_dadd:
+ temp = I128(rs) + I128(rt);
+ if ((temp << 64) != (temp << 63)) {
+ exceptions[kIntegerOverflow] = 1;
+ }
+ alu_out = I64(temp);
+ break;
+ case ff_addu:
+ alu_out = I32(I32_CHECK(rs) + I32_CHECK(rt));
+ break;
+ case ff_daddu:
+ alu_out = rs + rt;
+ break;
+ case ff_sub:
+ alu_out = I32_CHECK(rs) - I32_CHECK(rt);
+ if ((alu_out << 32) != (alu_out << 31)) {
+ exceptions[kIntegerUnderflow] = 1;
+ }
+ alu_out = I32(alu_out);
+ break;
+ case ff_dsub:
+ temp = I128(rs) - I128(rt);
+ if ((temp << 64) != (temp << 63)) {
+ exceptions[kIntegerUnderflow] = 1;
+ }
+ alu_out = I64(temp);
+ break;
+ case ff_subu:
+ alu_out = I32(I32_CHECK(rs) - I32_CHECK(rt));
+ break;
+ case ff_dsubu:
+ alu_out = rs - rt;
+ break;
+ case ff_and:
+ alu_out = rs & rt;
+ break;
+ case ff_or:
+ alu_out = rs | rt;
+ break;
+ case ff_xor:
+ alu_out = rs ^ rt;
+ break;
+ case ff_nor:
+ alu_out = ~(rs | rt);
+ break;
+ case ff_slt:
+ alu_out = I64(rs) < I64(rt) ? 1 : 0;
+ break;
+ case ff_sltu:
+ alu_out = U64(rs) < U64(rt) ? 1 : 0;
+ break;
+ case ff_sync:
+ break;
+ // Break and trap instructions.
+ case ff_break:
+ do_interrupt = true;
+ break;
+ case ff_tge:
+ do_interrupt = rs >= rt;
+ break;
+ case ff_tgeu:
+ do_interrupt = U64(rs) >= U64(rt);
+ break;
+ case ff_tlt:
+ do_interrupt = rs < rt;
+ break;
+ case ff_tltu:
+ do_interrupt = U64(rs) < U64(rt);
+ break;
+ case ff_teq:
+ do_interrupt = rs == rt;
+ break;
+ case ff_tne:
+ do_interrupt = rs != rt;
+ break;
+ case ff_movn:
+ case ff_movz:
+ case ff_movci:
+ // No action taken on decode.
+ break;
+ case ff_div:
+ if (I32_CHECK(rs) == INT_MIN && I32_CHECK(rt) == -1) {
+ i128hilo = U32(INT_MIN);
+ } else {
+ uint32_t div = I32_CHECK(rs) / I32_CHECK(rt);
+ uint32_t mod = I32_CHECK(rs) % I32_CHECK(rt);
+ i128hilo = (I64(mod) << 32) | div;
+ }
+ break;
+ case ff_ddiv:
+ if (I64(rs) == INT64_MIN && I64(rt) == -1) {
+ i128hilo = U64(INT64_MIN);
+ } else {
+ uint64_t div = rs / rt;
+ uint64_t mod = rs % rt;
+ i128hilo = (I128(mod) << 64) | div;
+ }
+ break;
+ case ff_divu: {
+ uint32_t div = U32(I32_CHECK(rs)) / U32(I32_CHECK(rt));
+ uint32_t mod = U32(I32_CHECK(rs)) % U32(I32_CHECK(rt));
+ i128hilo = (U64(mod) << 32) | div;
+ } break;
+ case ff_ddivu:
+ if (0 == rt) {
+ i128hilo = (I128(Unpredictable) << 64) | I64(Unpredictable);
+ } else {
+ uint64_t div = U64(rs) / U64(rt);
+ uint64_t mod = U64(rs) % U64(rt);
+ i128hilo = (I128(mod) << 64) | div;
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ case op_special2:
+ switch (instr->functionFieldRaw()) {
+ case ff_mul:
+ alu_out = I32(I32_CHECK(rs) *
+ I32_CHECK(rt)); // Only the lower 32 bits are kept.
+ break;
+ case ff_clz:
+ alu_out = U32(I32_CHECK(rs)) ? __builtin_clz(U32(I32_CHECK(rs))) : 32;
+ break;
+ case ff_dclz:
+ alu_out = U64(rs) ? __builtin_clzl(U64(rs)) : 64;
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ case op_special3:
+ switch (instr->functionFieldRaw()) {
+ case ff_ins: { // Mips64r2 instruction.
+ // Interpret rd field as 5-bit msb of insert.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of insert.
+ uint16_t lsb = sa;
+ uint16_t size = msb - lsb + 1;
+ uint32_t mask = (1 << size) - 1;
+ if (lsb > msb) {
+ alu_out = Unpredictable;
+ } else {
+ alu_out = I32((U32(I32_CHECK(rt)) & ~(mask << lsb)) |
+ ((U32(I32_CHECK(rs)) & mask) << lsb));
+ }
+ break;
+ }
+ case ff_dins: { // Mips64r2 instruction.
+ // Interpret rd field as 5-bit msb of insert.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of insert.
+ uint16_t lsb = sa;
+ uint16_t size = msb - lsb + 1;
+ uint64_t mask = (1ul << size) - 1;
+ if (lsb > msb) {
+ alu_out = Unpredictable;
+ } else {
+ alu_out = (U64(rt) & ~(mask << lsb)) | ((U64(rs) & mask) << lsb);
+ }
+ break;
+ }
+ case ff_dinsm: { // Mips64r2 instruction.
+ // Interpret rd field as 5-bit msb of insert.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of insert.
+ uint16_t lsb = sa;
+ uint16_t size = msb - lsb + 33;
+ uint64_t mask = (1ul << size) - 1;
+ alu_out = (U64(rt) & ~(mask << lsb)) | ((U64(rs) & mask) << lsb);
+ break;
+ }
+ case ff_dinsu: { // Mips64r2 instruction.
+ // Interpret rd field as 5-bit msb of insert.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of insert.
+ uint16_t lsb = sa + 32;
+ uint16_t size = msb - lsb + 33;
+ uint64_t mask = (1ul << size) - 1;
+ if (sa > msb) {
+ alu_out = Unpredictable;
+ } else {
+ alu_out = (U64(rt) & ~(mask << lsb)) | ((U64(rs) & mask) << lsb);
+ }
+ break;
+ }
+ case ff_ext: { // Mips64r2 instruction.
+ // Interpret rd field as 5-bit msb of extract.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of extract.
+ uint16_t lsb = sa;
+ uint16_t size = msb + 1;
+ uint32_t mask = (1 << size) - 1;
+ if ((lsb + msb) > 31) {
+ alu_out = Unpredictable;
+ } else {
+ alu_out = (U32(I32_CHECK(rs)) & (mask << lsb)) >> lsb;
+ }
+ break;
+ }
+ case ff_dext: { // Mips64r2 instruction.
+ // Interpret rd field as 5-bit msb of extract.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of extract.
+ uint16_t lsb = sa;
+ uint16_t size = msb + 1;
+ uint64_t mask = (1ul << size) - 1;
+ alu_out = (U64(rs) & (mask << lsb)) >> lsb;
+ break;
+ }
+ case ff_dextm: { // Mips64r2 instruction.
+ // Interpret rd field as 5-bit msb of extract.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of extract.
+ uint16_t lsb = sa;
+ uint16_t size = msb + 33;
+ uint64_t mask = (1ul << size) - 1;
+ if ((lsb + msb + 32 + 1) > 64) {
+ alu_out = Unpredictable;
+ } else {
+ alu_out = (U64(rs) & (mask << lsb)) >> lsb;
+ }
+ break;
+ }
+ case ff_dextu: { // Mips64r2 instruction.
+ // Interpret rd field as 5-bit msb of extract.
+ uint16_t msb = rd_reg;
+ // Interpret sa field as 5-bit lsb of extract.
+ uint16_t lsb = sa + 32;
+ uint16_t size = msb + 1;
+ uint64_t mask = (1ul << size) - 1;
+ if ((lsb + msb + 1) > 64) {
+ alu_out = Unpredictable;
+ } else {
+ alu_out = (U64(rs) & (mask << lsb)) >> lsb;
+ }
+ break;
+ }
+ case ff_bshfl: { // Mips32r2 instruction.
+ if (16 == sa) { // seb
+ alu_out = I64(I8(I32_CHECK(rt)));
+ } else if (24 == sa) { // seh
+ alu_out = I64(I16(I32_CHECK(rt)));
+ } else if (2 == sa) { // wsbh
+ uint32_t input = U32(I32_CHECK(rt));
+ uint64_t output = 0;
+
+ uint32_t mask = 0xFF000000;
+ for (int i = 0; i < 4; i++) {
+ uint32_t tmp = mask & input;
+ if (i % 2 == 0) {
+ tmp = tmp >> 8;
+ } else {
+ tmp = tmp << 8;
+ }
+ output = output | tmp;
+ mask = mask >> 8;
+ }
+ alu_out = I64(I32(output));
+ } else {
+ MOZ_CRASH();
+ }
+ break;
+ }
+ case ff_dbshfl: { // Mips64r2 instruction.
+ uint64_t input = U64(rt);
+ uint64_t output = 0;
+
+ if (2 == sa) { // dsbh
+ uint64_t mask = 0xFF00000000000000;
+ for (int i = 0; i < 8; i++) {
+ uint64_t tmp = mask & input;
+ if (i % 2 == 0)
+ tmp = tmp >> 8;
+ else
+ tmp = tmp << 8;
+
+ output = output | tmp;
+ mask = mask >> 8;
+ }
+ } else if (5 == sa) { // dshd
+ uint64_t mask = 0xFFFF000000000000;
+ for (int i = 0; i < 4; i++) {
+ uint64_t tmp = mask & input;
+ if (i == 0)
+ tmp = tmp >> 48;
+ else if (i == 1)
+ tmp = tmp >> 16;
+ else if (i == 2)
+ tmp = tmp << 16;
+ else
+ tmp = tmp << 48;
+ output = output | tmp;
+ mask = mask >> 16;
+ }
+ } else {
+ MOZ_CRASH();
+ }
+
+ alu_out = I64(output);
+ break;
+ }
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ default:
+ MOZ_CRASH();
+ };
+}
+
+// Handle execution based on instruction types.
+void Simulator::decodeTypeRegister(SimInstruction* instr) {
+ // Instruction fields.
+ const OpcodeField op = instr->opcodeFieldRaw();
+ const int32_t rs_reg = instr->rsValue();
+ const int64_t rs = getRegister(rs_reg);
+ const int32_t rt_reg = instr->rtValue();
+ const int64_t rt = getRegister(rt_reg);
+ const int32_t rd_reg = instr->rdValue();
+
+ const int32_t fr_reg = instr->frValue();
+ const int32_t fs_reg = instr->fsValue();
+ const int32_t ft_reg = instr->ftValue();
+ const int32_t fd_reg = instr->fdValue();
+ __int128 i128hilo = 0;
+ unsigned __int128 u128hilo = 0;
+
+ // ALU output.
+ // It should not be used as is. Instructions using it should always
+ // initialize it first.
+ int64_t alu_out = 0x12345678;
+
+ // For break and trap instructions.
+ bool do_interrupt = false;
+
+ // For jr and jalr.
+ // Get current pc.
+ int64_t current_pc = get_pc();
+ // Next pc
+ int64_t next_pc = 0;
+ int32_t return_addr_reg = 31;
+
+ // Set up the variables if needed before executing the instruction.
+ configureTypeRegister(instr, alu_out, i128hilo, u128hilo, next_pc,
+ return_addr_reg, do_interrupt);
+
+ // ---------- Raise exceptions triggered.
+ signalExceptions();
+
+ // ---------- Execution.
+ switch (op) {
+ case op_cop1:
+ switch (instr->rsFieldRaw()) {
+ case rs_bc1: // Branch on coprocessor condition.
+ MOZ_CRASH();
+ break;
+ case rs_cfc1:
+ setRegister(rt_reg, alu_out);
+ [[fallthrough]];
+ case rs_mfc1:
+ setRegister(rt_reg, alu_out);
+ break;
+ case rs_dmfc1:
+ setRegister(rt_reg, alu_out);
+ break;
+ case rs_mfhc1:
+ setRegister(rt_reg, alu_out);
+ break;
+ case rs_ctc1:
+ // At the moment only FCSR is supported.
+ MOZ_ASSERT(fs_reg == kFCSRRegister);
+ FCSR_ = registers_[rt_reg];
+ break;
+ case rs_mtc1:
+ setFpuRegisterLo(fs_reg, registers_[rt_reg]);
+ break;
+ case rs_dmtc1:
+ setFpuRegister(fs_reg, registers_[rt_reg]);
+ break;
+ case rs_mthc1:
+ setFpuRegisterHi(fs_reg, registers_[rt_reg]);
+ break;
+ case rs_s:
+ float f, ft_value, fs_value;
+ uint32_t cc, fcsr_cc;
+ int64_t i64;
+ fs_value = getFpuRegisterFloat(fs_reg);
+ ft_value = getFpuRegisterFloat(ft_reg);
+ cc = instr->fcccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ switch (instr->functionFieldRaw()) {
+ case ff_add_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value + ft_value);
+ break;
+ case ff_sub_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value - ft_value);
+ break;
+ case ff_mul_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value * ft_value);
+ break;
+ case ff_div_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value / ft_value);
+ break;
+ case ff_abs_fmt:
+ setFpuRegisterFloat(fd_reg, fabsf(fs_value));
+ break;
+ case ff_mov_fmt:
+ setFpuRegisterFloat(fd_reg, fs_value);
+ break;
+ case ff_neg_fmt:
+ setFpuRegisterFloat(fd_reg, -fs_value);
+ break;
+ case ff_sqrt_fmt:
+ setFpuRegisterFloat(fd_reg, sqrtf(fs_value));
+ break;
+ case ff_c_un_fmt:
+ setFCSRBit(fcsr_cc,
+ mozilla::IsNaN(fs_value) || mozilla::IsNaN(ft_value));
+ break;
+ case ff_c_eq_fmt:
+ setFCSRBit(fcsr_cc, (fs_value == ft_value));
+ break;
+ case ff_c_ueq_fmt:
+ setFCSRBit(fcsr_cc,
+ (fs_value == ft_value) || (mozilla::IsNaN(fs_value) ||
+ mozilla::IsNaN(ft_value)));
+ break;
+ case ff_c_olt_fmt:
+ setFCSRBit(fcsr_cc, (fs_value < ft_value));
+ break;
+ case ff_c_ult_fmt:
+ setFCSRBit(fcsr_cc,
+ (fs_value < ft_value) || (mozilla::IsNaN(fs_value) ||
+ mozilla::IsNaN(ft_value)));
+ break;
+ case ff_c_ole_fmt:
+ setFCSRBit(fcsr_cc, (fs_value <= ft_value));
+ break;
+ case ff_c_ule_fmt:
+ setFCSRBit(fcsr_cc,
+ (fs_value <= ft_value) || (mozilla::IsNaN(fs_value) ||
+ mozilla::IsNaN(ft_value)));
+ break;
+ case ff_cvt_d_fmt:
+ f = getFpuRegisterFloat(fs_reg);
+ setFpuRegisterDouble(fd_reg, static_cast<double>(f));
+ break;
+ case ff_cvt_w_fmt: // Convert float to word.
+ // Rounding modes are not yet supported.
+ MOZ_ASSERT((FCSR_ & 3) == 0);
+ // In rounding mode 0 it should behave like ROUND.
+ [[fallthrough]];
+ case ff_round_w_fmt: { // Round double to word (round half to
+ // even).
+ float rounded = std::floor(fs_value + 0.5);
+ int32_t result = I32(rounded);
+ if ((result & 1) != 0 && result - fs_value == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ result--;
+ }
+ setFpuRegisterLo(fd_reg, result);
+ if (setFCSRRoundError<int32_t>(fs_value, rounded)) {
+ setFpuRegisterLo(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_trunc_w_fmt: { // Truncate float to word (round towards 0).
+ float rounded = truncf(fs_value);
+ int32_t result = I32(rounded);
+ setFpuRegisterLo(fd_reg, result);
+ if (setFCSRRoundError<int32_t>(fs_value, rounded)) {
+ setFpuRegisterLo(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_floor_w_fmt: { // Round float to word towards negative
+ // infinity.
+ float rounded = std::floor(fs_value);
+ int32_t result = I32(rounded);
+ setFpuRegisterLo(fd_reg, result);
+ if (setFCSRRoundError<int32_t>(fs_value, rounded)) {
+ setFpuRegisterLo(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_ceil_w_fmt: { // Round double to word towards positive
+ // infinity.
+ float rounded = std::ceil(fs_value);
+ int32_t result = I32(rounded);
+ setFpuRegisterLo(fd_reg, result);
+ if (setFCSRRoundError<int32_t>(fs_value, rounded)) {
+ setFpuRegisterLo(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_cvt_l_fmt: // Mips64r2: Truncate float to 64-bit long-word.
+ // Rounding modes are not yet supported.
+ MOZ_ASSERT((FCSR_ & 3) == 0);
+ // In rounding mode 0 it should behave like ROUND.
+ [[fallthrough]];
+ case ff_round_l_fmt: { // Mips64r2 instruction.
+ float rounded = fs_value > 0 ? std::floor(fs_value + 0.5)
+ : std::ceil(fs_value - 0.5);
+ i64 = I64(rounded);
+ setFpuRegister(fd_reg, i64);
+ if (setFCSRRoundError<int64_t>(fs_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult64);
+ }
+ break;
+ }
+ case ff_trunc_l_fmt: { // Mips64r2 instruction.
+ float rounded = truncf(fs_value);
+ i64 = I64(rounded);
+ setFpuRegister(fd_reg, i64);
+ if (setFCSRRoundError<int64_t>(fs_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult64);
+ }
+ break;
+ }
+ case ff_floor_l_fmt: { // Mips64r2 instruction.
+ float rounded = std::floor(fs_value);
+ i64 = I64(rounded);
+ setFpuRegister(fd_reg, i64);
+ if (setFCSRRoundError<int64_t>(fs_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult64);
+ }
+ break;
+ }
+ case ff_ceil_l_fmt: { // Mips64r2 instruction.
+ float rounded = std::ceil(fs_value);
+ i64 = I64(rounded);
+ setFpuRegister(fd_reg, i64);
+ if (setFCSRRoundError<int64_t>(fs_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult64);
+ }
+ break;
+ }
+ case ff_cvt_ps_s:
+ case ff_c_f_fmt:
+ MOZ_CRASH();
+ break;
+ case ff_movf_fmt:
+ if (testFCSRBit(fcsr_cc)) {
+ setFpuRegisterFloat(fd_reg, getFpuRegisterFloat(fs_reg));
+ }
+ break;
+ case ff_movz_fmt:
+ if (rt == 0) {
+ setFpuRegisterFloat(fd_reg, getFpuRegisterFloat(fs_reg));
+ }
+ break;
+ case ff_movn_fmt:
+ if (rt != 0) {
+ setFpuRegisterFloat(fd_reg, getFpuRegisterFloat(fs_reg));
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case rs_d:
+ double dt_value, ds_value;
+ ds_value = getFpuRegisterDouble(fs_reg);
+ dt_value = getFpuRegisterDouble(ft_reg);
+ cc = instr->fcccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ switch (instr->functionFieldRaw()) {
+ case ff_add_fmt:
+ setFpuRegisterDouble(fd_reg, ds_value + dt_value);
+ break;
+ case ff_sub_fmt:
+ setFpuRegisterDouble(fd_reg, ds_value - dt_value);
+ break;
+ case ff_mul_fmt:
+ setFpuRegisterDouble(fd_reg, ds_value * dt_value);
+ break;
+ case ff_div_fmt:
+ setFpuRegisterDouble(fd_reg, ds_value / dt_value);
+ break;
+ case ff_abs_fmt:
+ setFpuRegisterDouble(fd_reg, fabs(ds_value));
+ break;
+ case ff_mov_fmt:
+ setFpuRegisterDouble(fd_reg, ds_value);
+ break;
+ case ff_neg_fmt:
+ setFpuRegisterDouble(fd_reg, -ds_value);
+ break;
+ case ff_sqrt_fmt:
+ setFpuRegisterDouble(fd_reg, sqrt(ds_value));
+ break;
+ case ff_c_un_fmt:
+ setFCSRBit(fcsr_cc,
+ mozilla::IsNaN(ds_value) || mozilla::IsNaN(dt_value));
+ break;
+ case ff_c_eq_fmt:
+ setFCSRBit(fcsr_cc, (ds_value == dt_value));
+ break;
+ case ff_c_ueq_fmt:
+ setFCSRBit(fcsr_cc,
+ (ds_value == dt_value) || (mozilla::IsNaN(ds_value) ||
+ mozilla::IsNaN(dt_value)));
+ break;
+ case ff_c_olt_fmt:
+ setFCSRBit(fcsr_cc, (ds_value < dt_value));
+ break;
+ case ff_c_ult_fmt:
+ setFCSRBit(fcsr_cc,
+ (ds_value < dt_value) || (mozilla::IsNaN(ds_value) ||
+ mozilla::IsNaN(dt_value)));
+ break;
+ case ff_c_ole_fmt:
+ setFCSRBit(fcsr_cc, (ds_value <= dt_value));
+ break;
+ case ff_c_ule_fmt:
+ setFCSRBit(fcsr_cc,
+ (ds_value <= dt_value) || (mozilla::IsNaN(ds_value) ||
+ mozilla::IsNaN(dt_value)));
+ break;
+ case ff_cvt_w_fmt: // Convert double to word.
+ // Rounding modes are not yet supported.
+ MOZ_ASSERT((FCSR_ & 3) == 0);
+ // In rounding mode 0 it should behave like ROUND.
+ [[fallthrough]];
+ case ff_round_w_fmt: { // Round double to word (round half to
+ // even).
+ double rounded = std::floor(ds_value + 0.5);
+ int32_t result = I32(rounded);
+ if ((result & 1) != 0 && result - ds_value == 0.5) {
+ // If the number is halfway between two integers,
+ // round to the even one.
+ result--;
+ }
+ setFpuRegisterLo(fd_reg, result);
+ if (setFCSRRoundError<int32_t>(ds_value, rounded)) {
+ setFpuRegisterLo(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_trunc_w_fmt: { // Truncate double to word (round towards
+ // 0).
+ double rounded = trunc(ds_value);
+ int32_t result = I32(rounded);
+ setFpuRegisterLo(fd_reg, result);
+ if (setFCSRRoundError<int32_t>(ds_value, rounded)) {
+ setFpuRegisterLo(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_floor_w_fmt: { // Round double to word towards negative
+ // infinity.
+ double rounded = std::floor(ds_value);
+ int32_t result = I32(rounded);
+ setFpuRegisterLo(fd_reg, result);
+ if (setFCSRRoundError<int32_t>(ds_value, rounded)) {
+ setFpuRegisterLo(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_ceil_w_fmt: { // Round double to word towards positive
+ // infinity.
+ double rounded = std::ceil(ds_value);
+ int32_t result = I32(rounded);
+ setFpuRegisterLo(fd_reg, result);
+ if (setFCSRRoundError<int32_t>(ds_value, rounded)) {
+ setFpuRegisterLo(fd_reg, kFPUInvalidResult);
+ }
+ break;
+ }
+ case ff_cvt_s_fmt: // Convert double to float (single).
+ setFpuRegisterFloat(fd_reg, static_cast<float>(ds_value));
+ break;
+ case ff_cvt_l_fmt: // Mips64r2: Truncate double to 64-bit
+ // long-word.
+ // Rounding modes are not yet supported.
+ MOZ_ASSERT((FCSR_ & 3) == 0);
+ // In rounding mode 0 it should behave like ROUND.
+ [[fallthrough]];
+ case ff_round_l_fmt: { // Mips64r2 instruction.
+ double rounded = ds_value > 0 ? std::floor(ds_value + 0.5)
+ : std::ceil(ds_value - 0.5);
+ i64 = I64(rounded);
+ setFpuRegister(fd_reg, i64);
+ if (setFCSRRoundError<int64_t>(ds_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult64);
+ }
+ break;
+ }
+ case ff_trunc_l_fmt: { // Mips64r2 instruction.
+ double rounded = trunc(ds_value);
+ i64 = I64(rounded);
+ setFpuRegister(fd_reg, i64);
+ if (setFCSRRoundError<int64_t>(ds_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult64);
+ }
+ break;
+ }
+ case ff_floor_l_fmt: { // Mips64r2 instruction.
+ double rounded = std::floor(ds_value);
+ i64 = I64(rounded);
+ setFpuRegister(fd_reg, i64);
+ if (setFCSRRoundError<int64_t>(ds_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult64);
+ }
+ break;
+ }
+ case ff_ceil_l_fmt: { // Mips64r2 instruction.
+ double rounded = std::ceil(ds_value);
+ i64 = I64(rounded);
+ setFpuRegister(fd_reg, i64);
+ if (setFCSRRoundError<int64_t>(ds_value, rounded)) {
+ setFpuRegister(fd_reg, kFPUInvalidResult64);
+ }
+ break;
+ }
+ case ff_c_f_fmt:
+ MOZ_CRASH();
+ break;
+ case ff_movz_fmt:
+ if (rt == 0) {
+ setFpuRegisterDouble(fd_reg, getFpuRegisterDouble(fs_reg));
+ }
+ break;
+ case ff_movn_fmt:
+ if (rt != 0) {
+ setFpuRegisterDouble(fd_reg, getFpuRegisterDouble(fs_reg));
+ }
+ break;
+ case ff_movf_fmt:
+ // location of cc field in MOVF is equal to float branch
+ // instructions
+ cc = instr->fbccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ if (testFCSRBit(fcsr_cc)) {
+ setFpuRegisterDouble(fd_reg, getFpuRegisterDouble(fs_reg));
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case rs_w:
+ switch (instr->functionFieldRaw()) {
+ case ff_cvt_s_fmt: // Convert word to float (single).
+ i64 = getFpuRegisterLo(fs_reg);
+ setFpuRegisterFloat(fd_reg, static_cast<float>(i64));
+ break;
+ case ff_cvt_d_fmt: // Convert word to double.
+ i64 = getFpuRegisterLo(fs_reg);
+ setFpuRegisterDouble(fd_reg, static_cast<double>(i64));
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ case rs_l:
+ switch (instr->functionFieldRaw()) {
+ case ff_cvt_d_fmt: // Mips64r2 instruction.
+ i64 = getFpuRegister(fs_reg);
+ setFpuRegisterDouble(fd_reg, static_cast<double>(i64));
+ break;
+ case ff_cvt_s_fmt:
+ i64 = getFpuRegister(fs_reg);
+ setFpuRegisterFloat(fd_reg, static_cast<float>(i64));
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case rs_ps:
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ case op_cop1x:
+ switch (instr->functionFieldRaw()) {
+ case ff_madd_s:
+ float fr, ft, fs;
+ fr = getFpuRegisterFloat(fr_reg);
+ fs = getFpuRegisterFloat(fs_reg);
+ ft = getFpuRegisterFloat(ft_reg);
+ setFpuRegisterFloat(fd_reg, fs * ft + fr);
+ break;
+ case ff_madd_d:
+ double dr, dt, ds;
+ dr = getFpuRegisterDouble(fr_reg);
+ ds = getFpuRegisterDouble(fs_reg);
+ dt = getFpuRegisterDouble(ft_reg);
+ setFpuRegisterDouble(fd_reg, ds * dt + dr);
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ case op_special:
+ switch (instr->functionFieldRaw()) {
+ case ff_jr: {
+ SimInstruction* branch_delay_instr =
+ reinterpret_cast<SimInstruction*>(current_pc +
+ SimInstruction::kInstrSize);
+ branchDelayInstructionDecode(branch_delay_instr);
+ set_pc(next_pc);
+ pc_modified_ = true;
+ break;
+ }
+ case ff_jalr: {
+ SimInstruction* branch_delay_instr =
+ reinterpret_cast<SimInstruction*>(current_pc +
+ SimInstruction::kInstrSize);
+ setRegister(return_addr_reg,
+ current_pc + 2 * SimInstruction::kInstrSize);
+ branchDelayInstructionDecode(branch_delay_instr);
+ set_pc(next_pc);
+ pc_modified_ = true;
+ break;
+ }
+ // Instructions using HI and LO registers.
+ case ff_mult:
+ setRegister(LO, I32(i128hilo & 0xffffffff));
+ setRegister(HI, I32(i128hilo >> 32));
+ break;
+ case ff_dmult:
+ setRegister(LO, I64(i128hilo & 0xfffffffffffffffful));
+ setRegister(HI, I64(i128hilo >> 64));
+ break;
+ case ff_multu:
+ setRegister(LO, I32(u128hilo & 0xffffffff));
+ setRegister(HI, I32(u128hilo >> 32));
+ break;
+ case ff_dmultu:
+ setRegister(LO, I64(u128hilo & 0xfffffffffffffffful));
+ setRegister(HI, I64(u128hilo >> 64));
+ break;
+ case ff_div:
+ case ff_divu:
+ // Divide by zero and overflow was not checked in the configuration
+ // step - div and divu do not raise exceptions. On division by 0
+ // the result will be UNPREDICTABLE. On overflow (INT_MIN/-1),
+ // return INT_MIN which is what the hardware does.
+ setRegister(LO, I32(i128hilo & 0xffffffff));
+ setRegister(HI, I32(i128hilo >> 32));
+ break;
+ case ff_ddiv:
+ case ff_ddivu:
+ // Divide by zero and overflow was not checked in the configuration
+ // step - div and divu do not raise exceptions. On division by 0
+ // the result will be UNPREDICTABLE. On overflow (INT_MIN/-1),
+ // return INT_MIN which is what the hardware does.
+ setRegister(LO, I64(i128hilo & 0xfffffffffffffffful));
+ setRegister(HI, I64(i128hilo >> 64));
+ break;
+ case ff_sync:
+ break;
+ // Break and trap instructions.
+ case ff_break:
+ case ff_tge:
+ case ff_tgeu:
+ case ff_tlt:
+ case ff_tltu:
+ case ff_teq:
+ case ff_tne:
+ if (do_interrupt) {
+ softwareInterrupt(instr);
+ }
+ break;
+ // Conditional moves.
+ case ff_movn:
+ if (rt) {
+ setRegister(rd_reg, rs);
+ }
+ break;
+ case ff_movci: {
+ uint32_t cc = instr->fbccValue();
+ uint32_t fcsr_cc = GetFCSRConditionBit(cc);
+ if (instr->bit(16)) { // Read Tf bit.
+ if (testFCSRBit(fcsr_cc)) {
+ setRegister(rd_reg, rs);
+ }
+ } else {
+ if (!testFCSRBit(fcsr_cc)) {
+ setRegister(rd_reg, rs);
+ }
+ }
+ break;
+ }
+ case ff_movz:
+ if (!rt) {
+ setRegister(rd_reg, rs);
+ }
+ break;
+ default: // For other special opcodes we do the default operation.
+ setRegister(rd_reg, alu_out);
+ };
+ break;
+ case op_special2:
+ switch (instr->functionFieldRaw()) {
+ case ff_mul:
+ setRegister(rd_reg, alu_out);
+ // HI and LO are UNPREDICTABLE after the operation.
+ setRegister(LO, Unpredictable);
+ setRegister(HI, Unpredictable);
+ break;
+ default: // For other special2 opcodes we do the default operation.
+ setRegister(rd_reg, alu_out);
+ }
+ break;
+ case op_special3:
+ switch (instr->functionFieldRaw()) {
+ case ff_ins:
+ case ff_dins:
+ case ff_dinsm:
+ case ff_dinsu:
+ // Ins instr leaves result in Rt, rather than Rd.
+ setRegister(rt_reg, alu_out);
+ break;
+ case ff_ext:
+ case ff_dext:
+ case ff_dextm:
+ case ff_dextu:
+ // Ext instr leaves result in Rt, rather than Rd.
+ setRegister(rt_reg, alu_out);
+ break;
+ case ff_bshfl:
+ setRegister(rd_reg, alu_out);
+ break;
+ case ff_dbshfl:
+ setRegister(rd_reg, alu_out);
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ // Unimplemented opcodes raised an error in the configuration step before,
+ // so we can use the default here to set the destination register in
+ // common cases.
+ default:
+ setRegister(rd_reg, alu_out);
+ };
+}
+
+// Type 2: instructions using a 16 bits immediate. (e.g. addi, beq).
+void Simulator::decodeTypeImmediate(SimInstruction* instr) {
+ // Instruction fields.
+ OpcodeField op = instr->opcodeFieldRaw();
+ int64_t rs = getRegister(instr->rsValue());
+ int32_t rt_reg = instr->rtValue(); // Destination register.
+ int64_t rt = getRegister(rt_reg);
+ int16_t imm16 = instr->imm16Value();
+
+ int32_t ft_reg = instr->ftValue(); // Destination register.
+
+ // Zero extended immediate.
+ uint32_t oe_imm16 = 0xffff & imm16;
+ // Sign extended immediate.
+ int32_t se_imm16 = imm16;
+
+ // Get current pc.
+ int64_t current_pc = get_pc();
+ // Next pc.
+ int64_t next_pc = bad_ra;
+
+ // Used for conditional branch instructions.
+ bool do_branch = false;
+ bool execute_branch_delay_instruction = false;
+
+ // Used for arithmetic instructions.
+ int64_t alu_out = 0;
+ // Floating point.
+ double fp_out = 0.0;
+ uint32_t cc, cc_value, fcsr_cc;
+
+ // Used for memory instructions.
+ uint64_t addr = 0x0;
+ // Value to be written in memory.
+ uint64_t mem_value = 0x0;
+ __int128 temp;
+
+ // ---------- Configuration (and execution for op_regimm).
+ switch (op) {
+ // ------------- op_cop1. Coprocessor instructions.
+ case op_cop1:
+ switch (instr->rsFieldRaw()) {
+ case rs_bc1: // Branch on coprocessor condition.
+ cc = instr->fbccValue();
+ fcsr_cc = GetFCSRConditionBit(cc);
+ cc_value = testFCSRBit(fcsr_cc);
+ do_branch = (instr->fbtrueValue()) ? cc_value : !cc_value;
+ execute_branch_delay_instruction = true;
+ // Set next_pc.
+ if (do_branch) {
+ next_pc = current_pc + (imm16 << 2) + SimInstruction::kInstrSize;
+ } else {
+ next_pc = current_pc + kBranchReturnOffset;
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ break;
+ // ------------- op_regimm class.
+ case op_regimm:
+ switch (instr->rtFieldRaw()) {
+ case rt_bltz:
+ do_branch = (rs < 0);
+ break;
+ case rt_bltzal:
+ do_branch = rs < 0;
+ break;
+ case rt_bgez:
+ do_branch = rs >= 0;
+ break;
+ case rt_bgezal:
+ do_branch = rs >= 0;
+ break;
+ default:
+ MOZ_CRASH();
+ };
+ switch (instr->rtFieldRaw()) {
+ case rt_bltz:
+ case rt_bltzal:
+ case rt_bgez:
+ case rt_bgezal:
+ // Branch instructions common part.
+ execute_branch_delay_instruction = true;
+ // Set next_pc.
+ if (do_branch) {
+ next_pc = current_pc + (imm16 << 2) + SimInstruction::kInstrSize;
+ if (instr->isLinkingInstruction()) {
+ setRegister(31, current_pc + kBranchReturnOffset);
+ }
+ } else {
+ next_pc = current_pc + kBranchReturnOffset;
+ }
+ break;
+ default:
+ break;
+ };
+ break; // case op_regimm.
+ // ------------- Branch instructions.
+ // When comparing to zero, the encoding of rt field is always 0, so we
+ // don't need to replace rt with zero.
+ case op_beq:
+ do_branch = (rs == rt);
+ break;
+ case op_bne:
+ do_branch = rs != rt;
+ break;
+ case op_blez:
+ do_branch = rs <= 0;
+ break;
+ case op_bgtz:
+ do_branch = rs > 0;
+ break;
+ // ------------- Arithmetic instructions.
+ case op_addi:
+ alu_out = I32_CHECK(rs) + se_imm16;
+ if ((alu_out << 32) != (alu_out << 31)) {
+ exceptions[kIntegerOverflow] = 1;
+ }
+ alu_out = I32_CHECK(alu_out);
+ break;
+ case op_daddi:
+ temp = alu_out = rs + se_imm16;
+ if ((temp << 64) != (temp << 63)) {
+ exceptions[kIntegerOverflow] = 1;
+ }
+ alu_out = I64(temp);
+ break;
+ case op_addiu:
+ alu_out = I32(I32_CHECK(rs) + se_imm16);
+ break;
+ case op_daddiu:
+ alu_out = rs + se_imm16;
+ break;
+ case op_slti:
+ alu_out = (rs < se_imm16) ? 1 : 0;
+ break;
+ case op_sltiu:
+ alu_out = (U64(rs) < U64(se_imm16)) ? 1 : 0;
+ break;
+ case op_andi:
+ alu_out = rs & oe_imm16;
+ break;
+ case op_ori:
+ alu_out = rs | oe_imm16;
+ break;
+ case op_xori:
+ alu_out = rs ^ oe_imm16;
+ break;
+ case op_lui:
+ alu_out = (se_imm16 << 16);
+ break;
+ // ------------- Memory instructions.
+ case op_lbu:
+ addr = rs + se_imm16;
+ alu_out = readBU(addr, instr);
+ break;
+ case op_lb:
+ addr = rs + se_imm16;
+ alu_out = readB(addr, instr);
+ break;
+ case op_lhu:
+ addr = rs + se_imm16;
+ alu_out = readHU(addr, instr);
+ break;
+ case op_lh:
+ addr = rs + se_imm16;
+ alu_out = readH(addr, instr);
+ break;
+ case op_lwu:
+ addr = rs + se_imm16;
+ alu_out = readWU(addr, instr);
+ break;
+ case op_lw:
+ addr = rs + se_imm16;
+ alu_out = readW(addr, instr);
+ break;
+ case op_lwl: {
+ // al_offset is offset of the effective address within an aligned word.
+ uint8_t al_offset = (rs + se_imm16) & 3;
+ uint8_t byte_shift = 3 - al_offset;
+ uint32_t mask = (1 << byte_shift * 8) - 1;
+ addr = rs + se_imm16 - al_offset;
+ alu_out = readW(addr, instr);
+ alu_out <<= byte_shift * 8;
+ alu_out |= rt & mask;
+ break;
+ }
+ case op_lwr: {
+ // al_offset is offset of the effective address within an aligned word.
+ uint8_t al_offset = (rs + se_imm16) & 3;
+ uint8_t byte_shift = 3 - al_offset;
+ uint32_t mask = al_offset ? (~0 << (byte_shift + 1) * 8) : 0;
+ addr = rs + se_imm16 - al_offset;
+ alu_out = readW(addr, instr);
+ alu_out = U32(alu_out) >> al_offset * 8;
+ alu_out |= rt & mask;
+ alu_out = I32(alu_out);
+ break;
+ }
+ case op_ll:
+ addr = rs + se_imm16;
+ alu_out = loadLinkedW(addr, instr);
+ break;
+ case op_lld:
+ addr = rs + se_imm16;
+ alu_out = loadLinkedD(addr, instr);
+ break;
+ case op_ld:
+ addr = rs + se_imm16;
+ alu_out = readDW(addr, instr);
+ break;
+ case op_ldl: {
+ // al_offset is offset of the effective address within an aligned word.
+ uint8_t al_offset = (rs + se_imm16) & 7;
+ uint8_t byte_shift = 7 - al_offset;
+ uint64_t mask = (1ul << byte_shift * 8) - 1;
+ addr = rs + se_imm16 - al_offset;
+ alu_out = readDW(addr, instr);
+ alu_out <<= byte_shift * 8;
+ alu_out |= rt & mask;
+ break;
+ }
+ case op_ldr: {
+ // al_offset is offset of the effective address within an aligned word.
+ uint8_t al_offset = (rs + se_imm16) & 7;
+ uint8_t byte_shift = 7 - al_offset;
+ uint64_t mask = al_offset ? (~0ul << (byte_shift + 1) * 8) : 0;
+ addr = rs + se_imm16 - al_offset;
+ alu_out = readDW(addr, instr);
+ alu_out = U64(alu_out) >> al_offset * 8;
+ alu_out |= rt & mask;
+ break;
+ }
+ case op_sb:
+ addr = rs + se_imm16;
+ break;
+ case op_sh:
+ addr = rs + se_imm16;
+ break;
+ case op_sw:
+ addr = rs + se_imm16;
+ break;
+ case op_swl: {
+ uint8_t al_offset = (rs + se_imm16) & 3;
+ uint8_t byte_shift = 3 - al_offset;
+ uint32_t mask = byte_shift ? (~0 << (al_offset + 1) * 8) : 0;
+ addr = rs + se_imm16 - al_offset;
+ mem_value = readW(addr, instr) & mask;
+ mem_value |= U32(rt) >> byte_shift * 8;
+ break;
+ }
+ case op_swr: {
+ uint8_t al_offset = (rs + se_imm16) & 3;
+ uint32_t mask = (1 << al_offset * 8) - 1;
+ addr = rs + se_imm16 - al_offset;
+ mem_value = readW(addr, instr);
+ mem_value = (rt << al_offset * 8) | (mem_value & mask);
+ break;
+ }
+ case op_sc:
+ addr = rs + se_imm16;
+ break;
+ case op_scd:
+ addr = rs + se_imm16;
+ break;
+ case op_sd:
+ addr = rs + se_imm16;
+ break;
+ case op_sdl: {
+ uint8_t al_offset = (rs + se_imm16) & 7;
+ uint8_t byte_shift = 7 - al_offset;
+ uint64_t mask = byte_shift ? (~0ul << (al_offset + 1) * 8) : 0;
+ addr = rs + se_imm16 - al_offset;
+ mem_value = readW(addr, instr) & mask;
+ mem_value |= U64(rt) >> byte_shift * 8;
+ break;
+ }
+ case op_sdr: {
+ uint8_t al_offset = (rs + se_imm16) & 7;
+ uint64_t mask = (1ul << al_offset * 8) - 1;
+ addr = rs + se_imm16 - al_offset;
+ mem_value = readW(addr, instr);
+ mem_value = (rt << al_offset * 8) | (mem_value & mask);
+ break;
+ }
+ case op_lwc1:
+ addr = rs + se_imm16;
+ alu_out = readW(addr, instr);
+ break;
+ case op_ldc1:
+ addr = rs + se_imm16;
+ fp_out = readD(addr, instr);
+ break;
+ case op_swc1:
+ case op_sdc1:
+ addr = rs + se_imm16;
+ break;
+ default:
+ MOZ_CRASH();
+ };
+
+ // ---------- Raise exceptions triggered.
+ signalExceptions();
+
+ // ---------- Execution.
+ switch (op) {
+ // ------------- Branch instructions.
+ case op_beq:
+ case op_bne:
+ case op_blez:
+ case op_bgtz:
+ // Branch instructions common part.
+ execute_branch_delay_instruction = true;
+ // Set next_pc.
+ if (do_branch) {
+ next_pc = current_pc + (imm16 << 2) + SimInstruction::kInstrSize;
+ if (instr->isLinkingInstruction()) {
+ setRegister(31, current_pc + 2 * SimInstruction::kInstrSize);
+ }
+ } else {
+ next_pc = current_pc + 2 * SimInstruction::kInstrSize;
+ }
+ break;
+ // ------------- Arithmetic instructions.
+ case op_addi:
+ case op_daddi:
+ case op_addiu:
+ case op_daddiu:
+ case op_slti:
+ case op_sltiu:
+ case op_andi:
+ case op_ori:
+ case op_xori:
+ case op_lui:
+ setRegister(rt_reg, alu_out);
+ break;
+ // ------------- Memory instructions.
+ case op_lbu:
+ case op_lb:
+ case op_lhu:
+ case op_lh:
+ case op_lwu:
+ case op_lw:
+ case op_lwl:
+ case op_lwr:
+ case op_ll:
+ case op_lld:
+ case op_ld:
+ case op_ldl:
+ case op_ldr:
+ setRegister(rt_reg, alu_out);
+ break;
+ case op_sb:
+ writeB(addr, I8(rt), instr);
+ break;
+ case op_sh:
+ writeH(addr, U16(rt), instr);
+ break;
+ case op_sw:
+ writeW(addr, I32(rt), instr);
+ break;
+ case op_swl:
+ writeW(addr, I32(mem_value), instr);
+ break;
+ case op_swr:
+ writeW(addr, I32(mem_value), instr);
+ break;
+ case op_sc:
+ setRegister(rt_reg, storeConditionalW(addr, I32(rt), instr));
+ break;
+ case op_scd:
+ setRegister(rt_reg, storeConditionalD(addr, rt, instr));
+ break;
+ case op_sd:
+ writeDW(addr, rt, instr);
+ break;
+ case op_sdl:
+ writeDW(addr, mem_value, instr);
+ break;
+ case op_sdr:
+ writeDW(addr, mem_value, instr);
+ break;
+ case op_lwc1:
+ setFpuRegisterLo(ft_reg, alu_out);
+ break;
+ case op_ldc1:
+ setFpuRegisterDouble(ft_reg, fp_out);
+ break;
+ case op_swc1:
+ writeW(addr, getFpuRegisterLo(ft_reg), instr);
+ break;
+ case op_sdc1:
+ writeD(addr, getFpuRegisterDouble(ft_reg), instr);
+ break;
+ default:
+ break;
+ };
+
+ if (execute_branch_delay_instruction) {
+ // Execute branch delay slot
+ // We don't check for end_sim_pc. First it should not be met as the current
+ // pc is valid. Secondly a jump should always execute its branch delay slot.
+ SimInstruction* branch_delay_instr = reinterpret_cast<SimInstruction*>(
+ current_pc + SimInstruction::kInstrSize);
+ branchDelayInstructionDecode(branch_delay_instr);
+ }
+
+ // If needed update pc after the branch delay execution.
+ if (next_pc != bad_ra) {
+ set_pc(next_pc);
+ }
+}
+
+// Type 3: instructions using a 26 bits immediate. (e.g. j, jal).
+void Simulator::decodeTypeJump(SimInstruction* instr) {
+ // Get current pc.
+ int64_t current_pc = get_pc();
+ // Get unchanged bits of pc.
+ int64_t pc_high_bits = current_pc & 0xfffffffff0000000ul;
+ // Next pc.
+ int64_t next_pc = pc_high_bits | (instr->imm26Value() << 2);
+
+ // Execute branch delay slot.
+ // We don't check for end_sim_pc. First it should not be met as the current pc
+ // is valid. Secondly a jump should always execute its branch delay slot.
+ SimInstruction* branch_delay_instr = reinterpret_cast<SimInstruction*>(
+ current_pc + SimInstruction::kInstrSize);
+ branchDelayInstructionDecode(branch_delay_instr);
+
+ // Update pc and ra if necessary.
+ // Do this after the branch delay execution.
+ if (instr->isLinkingInstruction()) {
+ setRegister(31, current_pc + 2 * SimInstruction::kInstrSize);
+ }
+ set_pc(next_pc);
+ pc_modified_ = true;
+}
+
+// Executes the current instruction.
+void Simulator::instructionDecode(SimInstruction* instr) {
+ if (!SimulatorProcess::ICacheCheckingDisableCount) {
+ AutoLockSimulatorCache als;
+ SimulatorProcess::checkICacheLocked(instr);
+ }
+ pc_modified_ = false;
+
+ switch (instr->instructionType()) {
+ case SimInstruction::kRegisterType:
+ decodeTypeRegister(instr);
+ break;
+ case SimInstruction::kImmediateType:
+ decodeTypeImmediate(instr);
+ break;
+ case SimInstruction::kJumpType:
+ decodeTypeJump(instr);
+ break;
+ default:
+ UNSUPPORTED();
+ }
+ if (!pc_modified_) {
+ setRegister(pc,
+ reinterpret_cast<int64_t>(instr) + SimInstruction::kInstrSize);
+ }
+}
+
+void Simulator::branchDelayInstructionDecode(SimInstruction* instr) {
+ if (instr->instructionBits() == NopInst) {
+ // Short-cut generic nop instructions. They are always valid and they
+ // never change the simulator state.
+ return;
+ }
+
+ if (instr->isForbiddenInBranchDelay()) {
+ MOZ_CRASH("Eror:Unexpected opcode in a branch delay slot.");
+ }
+ instructionDecode(instr);
+}
+
+void Simulator::enable_single_stepping(SingleStepCallback cb, void* arg) {
+ single_stepping_ = true;
+ single_step_callback_ = cb;
+ single_step_callback_arg_ = arg;
+ single_step_callback_(single_step_callback_arg_, this, (void*)get_pc());
+}
+
+void Simulator::disable_single_stepping() {
+ if (!single_stepping_) {
+ return;
+ }
+ single_step_callback_(single_step_callback_arg_, this, (void*)get_pc());
+ single_stepping_ = false;
+ single_step_callback_ = nullptr;
+ single_step_callback_arg_ = nullptr;
+}
+
+template <bool enableStopSimAt>
+void Simulator::execute() {
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+
+ // Get the PC to simulate. Cannot use the accessor here as we need the
+ // raw PC value and not the one used as input to arithmetic instructions.
+ int64_t program_counter = get_pc();
+
+ while (program_counter != end_sim_pc) {
+ if (enableStopSimAt && (icount_ == Simulator::StopSimAt)) {
+ MipsDebugger dbg(this);
+ dbg.debug();
+ } else {
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this,
+ (void*)program_counter);
+ }
+ SimInstruction* instr =
+ reinterpret_cast<SimInstruction*>(program_counter);
+ instructionDecode(instr);
+ icount_++;
+ }
+ program_counter = get_pc();
+ }
+
+ if (single_stepping_) {
+ single_step_callback_(single_step_callback_arg_, this, nullptr);
+ }
+}
+
+void Simulator::callInternal(uint8_t* entry) {
+ // Prepare to execute the code at entry.
+ setRegister(pc, reinterpret_cast<int64_t>(entry));
+ // Put down marker for end of simulation. The simulator will stop simulation
+ // when the PC reaches this value. By saving the "end simulation" value into
+ // the LR the simulation stops when returning to this call point.
+ setRegister(ra, end_sim_pc);
+
+ // Remember the values of callee-saved registers.
+ // The code below assumes that r9 is not used as sb (static base) in
+ // simulator code and therefore is regarded as a callee-saved register.
+ int64_t s0_val = getRegister(s0);
+ int64_t s1_val = getRegister(s1);
+ int64_t s2_val = getRegister(s2);
+ int64_t s3_val = getRegister(s3);
+ int64_t s4_val = getRegister(s4);
+ int64_t s5_val = getRegister(s5);
+ int64_t s6_val = getRegister(s6);
+ int64_t s7_val = getRegister(s7);
+ int64_t gp_val = getRegister(gp);
+ int64_t sp_val = getRegister(sp);
+ int64_t fp_val = getRegister(fp);
+
+ // Set up the callee-saved registers with a known value. To be able to check
+ // that they are preserved properly across JS execution.
+ int64_t callee_saved_value = icount_;
+ setRegister(s0, callee_saved_value);
+ setRegister(s1, callee_saved_value);
+ setRegister(s2, callee_saved_value);
+ setRegister(s3, callee_saved_value);
+ setRegister(s4, callee_saved_value);
+ setRegister(s5, callee_saved_value);
+ setRegister(s6, callee_saved_value);
+ setRegister(s7, callee_saved_value);
+ setRegister(gp, callee_saved_value);
+ setRegister(fp, callee_saved_value);
+
+ // Start the simulation.
+ if (Simulator::StopSimAt != -1) {
+ execute<true>();
+ } else {
+ execute<false>();
+ }
+
+ // Check that the callee-saved registers have been preserved.
+ MOZ_ASSERT(callee_saved_value == getRegister(s0));
+ MOZ_ASSERT(callee_saved_value == getRegister(s1));
+ MOZ_ASSERT(callee_saved_value == getRegister(s2));
+ MOZ_ASSERT(callee_saved_value == getRegister(s3));
+ MOZ_ASSERT(callee_saved_value == getRegister(s4));
+ MOZ_ASSERT(callee_saved_value == getRegister(s5));
+ MOZ_ASSERT(callee_saved_value == getRegister(s6));
+ MOZ_ASSERT(callee_saved_value == getRegister(s7));
+ MOZ_ASSERT(callee_saved_value == getRegister(gp));
+ MOZ_ASSERT(callee_saved_value == getRegister(fp));
+
+ // Restore callee-saved registers with the original value.
+ setRegister(s0, s0_val);
+ setRegister(s1, s1_val);
+ setRegister(s2, s2_val);
+ setRegister(s3, s3_val);
+ setRegister(s4, s4_val);
+ setRegister(s5, s5_val);
+ setRegister(s6, s6_val);
+ setRegister(s7, s7_val);
+ setRegister(gp, gp_val);
+ setRegister(sp, sp_val);
+ setRegister(fp, fp_val);
+}
+
+int64_t Simulator::call(uint8_t* entry, int argument_count, ...) {
+ va_list parameters;
+ va_start(parameters, argument_count);
+
+ int64_t original_stack = getRegister(sp);
+ // Compute position of stack on entry to generated code.
+ int64_t entry_stack = original_stack;
+ if (argument_count > kCArgSlotCount) {
+ entry_stack = entry_stack - argument_count * sizeof(int64_t);
+ } else {
+ entry_stack = entry_stack - kCArgsSlotsSize;
+ }
+
+ entry_stack &= ~U64(ABIStackAlignment - 1);
+
+ intptr_t* stack_argument = reinterpret_cast<intptr_t*>(entry_stack);
+
+ // Setup the arguments.
+ for (int i = 0; i < argument_count; i++) {
+ js::jit::Register argReg;
+ if (GetIntArgReg(i, &argReg)) {
+ setRegister(argReg.code(), va_arg(parameters, int64_t));
+ } else {
+ stack_argument[i] = va_arg(parameters, int64_t);
+ }
+ }
+
+ va_end(parameters);
+ setRegister(sp, entry_stack);
+
+ callInternal(entry);
+
+ // Pop stack passed arguments.
+ MOZ_ASSERT(entry_stack == getRegister(sp));
+ setRegister(sp, original_stack);
+
+ int64_t result = getRegister(v0);
+ return result;
+}
+
+uintptr_t Simulator::pushAddress(uintptr_t address) {
+ int new_sp = getRegister(sp) - sizeof(uintptr_t);
+ uintptr_t* stack_slot = reinterpret_cast<uintptr_t*>(new_sp);
+ *stack_slot = address;
+ setRegister(sp, new_sp);
+ return new_sp;
+}
+
+uintptr_t Simulator::popAddress() {
+ int current_sp = getRegister(sp);
+ uintptr_t* stack_slot = reinterpret_cast<uintptr_t*>(current_sp);
+ uintptr_t address = *stack_slot;
+ setRegister(sp, current_sp + sizeof(uintptr_t));
+ return address;
+}
+
+} // namespace jit
+} // namespace js
+
+js::jit::Simulator* JSContext::simulator() const { return simulator_; }
diff --git a/js/src/jit/mips64/Simulator-mips64.h b/js/src/jit/mips64/Simulator-mips64.h
new file mode 100644
index 0000000000..02b2774f24
--- /dev/null
+++ b/js/src/jit/mips64/Simulator-mips64.h
@@ -0,0 +1,536 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80: */
+// Copyright 2011 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef jit_mips64_Simulator_mips64_h
+#define jit_mips64_Simulator_mips64_h
+
+#ifdef JS_SIMULATOR_MIPS64
+
+# include "mozilla/Atomics.h"
+
+# include "jit/IonTypes.h"
+# include "js/ProfilingFrameIterator.h"
+# include "threading/Thread.h"
+# include "vm/MutexIDs.h"
+# include "wasm/WasmSignalHandlers.h"
+
+namespace js {
+
+namespace jit {
+
+class JitActivation;
+
+class Simulator;
+class Redirection;
+class CachePage;
+class AutoLockSimulator;
+
+// When the SingleStepCallback is called, the simulator is about to execute
+// sim->get_pc() and the current machine state represents the completed
+// execution of the previous pc.
+typedef void (*SingleStepCallback)(void* arg, Simulator* sim, void* pc);
+
+const intptr_t kPointerAlignment = 8;
+const intptr_t kPointerAlignmentMask = kPointerAlignment - 1;
+
+const intptr_t kDoubleAlignment = 8;
+const intptr_t kDoubleAlignmentMask = kDoubleAlignment - 1;
+
+// Number of general purpose registers.
+const int kNumRegisters = 32;
+
+// In the simulator, the PC register is simulated as the 34th register.
+const int kPCRegister = 34;
+
+// Number coprocessor registers.
+const int kNumFPURegisters = 32;
+
+// FPU (coprocessor 1) control registers. Currently only FCSR is implemented.
+const int kFCSRRegister = 31;
+const int kInvalidFPUControlRegister = -1;
+const uint32_t kFPUInvalidResult = static_cast<uint32_t>(1 << 31) - 1;
+const uint64_t kFPUInvalidResult64 = static_cast<uint64_t>(1ULL << 63) - 1;
+
+// FCSR constants.
+const uint32_t kFCSRInexactFlagBit = 2;
+const uint32_t kFCSRUnderflowFlagBit = 3;
+const uint32_t kFCSROverflowFlagBit = 4;
+const uint32_t kFCSRDivideByZeroFlagBit = 5;
+const uint32_t kFCSRInvalidOpFlagBit = 6;
+
+const uint32_t kFCSRInexactCauseBit = 12;
+const uint32_t kFCSRUnderflowCauseBit = 13;
+const uint32_t kFCSROverflowCauseBit = 14;
+const uint32_t kFCSRDivideByZeroCauseBit = 15;
+const uint32_t kFCSRInvalidOpCauseBit = 16;
+
+const uint32_t kFCSRInexactFlagMask = 1 << kFCSRInexactFlagBit;
+const uint32_t kFCSRUnderflowFlagMask = 1 << kFCSRUnderflowFlagBit;
+const uint32_t kFCSROverflowFlagMask = 1 << kFCSROverflowFlagBit;
+const uint32_t kFCSRDivideByZeroFlagMask = 1 << kFCSRDivideByZeroFlagBit;
+const uint32_t kFCSRInvalidOpFlagMask = 1 << kFCSRInvalidOpFlagBit;
+
+const uint32_t kFCSRFlagMask =
+ kFCSRInexactFlagMask | kFCSRUnderflowFlagMask | kFCSROverflowFlagMask |
+ kFCSRDivideByZeroFlagMask | kFCSRInvalidOpFlagMask;
+
+const uint32_t kFCSRExceptionFlagMask = kFCSRFlagMask ^ kFCSRInexactFlagMask;
+
+// On MIPS64 Simulator breakpoints can have different codes:
+// - Breaks between 0 and kMaxWatchpointCode are treated as simple watchpoints,
+// the simulator will run through them and print the registers.
+// - Breaks between kMaxWatchpointCode and kMaxStopCode are treated as stop()
+// instructions (see Assembler::stop()).
+// - Breaks larger than kMaxStopCode are simple breaks, dropping you into the
+// debugger.
+const uint32_t kMaxWatchpointCode = 31;
+const uint32_t kMaxStopCode = 127;
+const uint32_t kWasmTrapCode = 6;
+
+// -----------------------------------------------------------------------------
+// Utility functions
+
+typedef uint32_t Instr;
+class SimInstruction;
+
+// Per thread simulator state.
+class Simulator {
+ friend class MipsDebugger;
+
+ public:
+ // Registers are declared in order. See "See MIPS Run Linux" chapter 2.
+ enum Register {
+ no_reg = -1,
+ zero_reg = 0,
+ at,
+ v0,
+ v1,
+ a0,
+ a1,
+ a2,
+ a3,
+ a4,
+ a5,
+ a6,
+ a7,
+ t0,
+ t1,
+ t2,
+ t3,
+ s0,
+ s1,
+ s2,
+ s3,
+ s4,
+ s5,
+ s6,
+ s7,
+ t8,
+ t9,
+ k0,
+ k1,
+ gp,
+ sp,
+ s8,
+ ra,
+ // LO, HI, and pc.
+ LO,
+ HI,
+ pc, // pc must be the last register.
+ kNumSimuRegisters,
+ // aliases
+ fp = s8
+ };
+
+ // Coprocessor registers.
+ enum FPURegister {
+ f0,
+ f1,
+ f2,
+ f3,
+ f4,
+ f5,
+ f6,
+ f7,
+ f8,
+ f9,
+ f10,
+ f11,
+ f12,
+ f13,
+ f14,
+ f15,
+ f16,
+ f17,
+ f18,
+ f19,
+ f20,
+ f21,
+ f22,
+ f23,
+ f24,
+ f25,
+ f26,
+ f27,
+ f28,
+ f29,
+ f30,
+ f31,
+ kNumFPURegisters
+ };
+
+ // Returns nullptr on OOM.
+ static Simulator* Create();
+
+ static void Destroy(Simulator* simulator);
+
+ // Constructor/destructor are for internal use only; use the static methods
+ // above.
+ Simulator();
+ ~Simulator();
+
+ // The currently executing Simulator instance. Potentially there can be one
+ // for each native thread.
+ static Simulator* Current();
+
+ static inline uintptr_t StackLimit() {
+ return Simulator::Current()->stackLimit();
+ }
+
+ uintptr_t* addressOfStackLimit();
+
+ // Accessors for register state. Reading the pc value adheres to the MIPS
+ // architecture specification and is off by a 8 from the currently executing
+ // instruction.
+ void setRegister(int reg, int64_t value);
+ int64_t getRegister(int reg) const;
+ // Same for FPURegisters.
+ void setFpuRegister(int fpureg, int64_t value);
+ void setFpuRegisterLo(int fpureg, int32_t value);
+ void setFpuRegisterHi(int fpureg, int32_t value);
+ void setFpuRegisterFloat(int fpureg, float value);
+ void setFpuRegisterDouble(int fpureg, double value);
+ int64_t getFpuRegister(int fpureg) const;
+ int32_t getFpuRegisterLo(int fpureg) const;
+ int32_t getFpuRegisterHi(int fpureg) const;
+ float getFpuRegisterFloat(int fpureg) const;
+ double getFpuRegisterDouble(int fpureg) const;
+ void setFCSRBit(uint32_t cc, bool value);
+ bool testFCSRBit(uint32_t cc);
+ template <typename T>
+ bool setFCSRRoundError(double original, double rounded);
+
+ // Special case of set_register and get_register to access the raw PC value.
+ void set_pc(int64_t value);
+ int64_t get_pc() const;
+
+ template <typename T>
+ T get_pc_as() const {
+ return reinterpret_cast<T>(get_pc());
+ }
+
+ void enable_single_stepping(SingleStepCallback cb, void* arg);
+ void disable_single_stepping();
+
+ // Accessor to the internal simulator stack area.
+ uintptr_t stackLimit() const;
+ bool overRecursed(uintptr_t newsp = 0) const;
+ bool overRecursedWithExtra(uint32_t extra) const;
+
+ // Executes MIPS instructions until the PC reaches end_sim_pc.
+ template <bool enableStopSimAt>
+ void execute();
+
+ // Sets up the simulator state and grabs the result on return.
+ int64_t call(uint8_t* entry, int argument_count, ...);
+
+ // Push an address onto the JS stack.
+ uintptr_t pushAddress(uintptr_t address);
+
+ // Pop an address from the JS stack.
+ uintptr_t popAddress();
+
+ // Debugger input.
+ void setLastDebuggerInput(char* input);
+ char* lastDebuggerInput() { return lastDebuggerInput_; }
+
+ // Returns true if pc register contains one of the 'SpecialValues' defined
+ // below (bad_ra, end_sim_pc).
+ bool has_bad_pc() const;
+
+ private:
+ enum SpecialValues {
+ // Known bad pc value to ensure that the simulator does not execute
+ // without being properly setup.
+ bad_ra = -1,
+ // A pc value used to signal the simulator to stop execution. Generally
+ // the ra is set to this value on transition from native C code to
+ // simulated execution, so that the simulator can "return" to the native
+ // C code.
+ end_sim_pc = -2,
+ // Unpredictable value.
+ Unpredictable = 0xbadbeaf
+ };
+
+ bool init();
+
+ // Unsupported instructions use Format to print an error and stop execution.
+ void format(SimInstruction* instr, const char* format);
+
+ // Read and write memory.
+ inline uint8_t readBU(uint64_t addr, SimInstruction* instr);
+ inline int8_t readB(uint64_t addr, SimInstruction* instr);
+ inline void writeB(uint64_t addr, uint8_t value, SimInstruction* instr);
+ inline void writeB(uint64_t addr, int8_t value, SimInstruction* instr);
+
+ inline uint16_t readHU(uint64_t addr, SimInstruction* instr);
+ inline int16_t readH(uint64_t addr, SimInstruction* instr);
+ inline void writeH(uint64_t addr, uint16_t value, SimInstruction* instr);
+ inline void writeH(uint64_t addr, int16_t value, SimInstruction* instr);
+
+ inline uint32_t readWU(uint64_t addr, SimInstruction* instr);
+ inline int32_t readW(uint64_t addr, SimInstruction* instr);
+ inline void writeW(uint64_t addr, uint32_t value, SimInstruction* instr);
+ inline void writeW(uint64_t addr, int32_t value, SimInstruction* instr);
+
+ inline int64_t readDW(uint64_t addr, SimInstruction* instr);
+ inline int64_t readDWL(uint64_t addr, SimInstruction* instr);
+ inline int64_t readDWR(uint64_t addr, SimInstruction* instr);
+ inline void writeDW(uint64_t addr, int64_t value, SimInstruction* instr);
+
+ inline double readD(uint64_t addr, SimInstruction* instr);
+ inline void writeD(uint64_t addr, double value, SimInstruction* instr);
+
+ inline int32_t loadLinkedW(uint64_t addr, SimInstruction* instr);
+ inline int storeConditionalW(uint64_t addr, int32_t value,
+ SimInstruction* instr);
+
+ inline int64_t loadLinkedD(uint64_t addr, SimInstruction* instr);
+ inline int storeConditionalD(uint64_t addr, int64_t value,
+ SimInstruction* instr);
+
+ // Helper function for decodeTypeRegister.
+ void configureTypeRegister(SimInstruction* instr, int64_t& alu_out,
+ __int128& i128hilo, unsigned __int128& u128hilo,
+ int64_t& next_pc, int32_t& return_addr_reg,
+ bool& do_interrupt);
+
+ // Executing is handled based on the instruction type.
+ void decodeTypeRegister(SimInstruction* instr);
+ void decodeTypeImmediate(SimInstruction* instr);
+ void decodeTypeJump(SimInstruction* instr);
+
+ // Used for breakpoints and traps.
+ void softwareInterrupt(SimInstruction* instr);
+
+ // Stop helper functions.
+ bool isWatchpoint(uint32_t code);
+ void printWatchpoint(uint32_t code);
+ void handleStop(uint32_t code, SimInstruction* instr);
+ bool isStopInstruction(SimInstruction* instr);
+ bool isEnabledStop(uint32_t code);
+ void enableStop(uint32_t code);
+ void disableStop(uint32_t code);
+ void increaseStopCounter(uint32_t code);
+ void printStopInfo(uint32_t code);
+
+ JS::ProfilingFrameIterator::RegisterState registerState();
+
+ // Handle any wasm faults, returning true if the fault was handled.
+ // This method is rather hot so inline the normal (no-wasm) case.
+ bool MOZ_ALWAYS_INLINE handleWasmSegFault(uint64_t addr, unsigned numBytes) {
+ if (MOZ_LIKELY(!js::wasm::CodeExists)) {
+ return false;
+ }
+
+ uint8_t* newPC;
+ if (!js::wasm::MemoryAccessTraps(registerState(), (uint8_t*)addr, numBytes,
+ &newPC)) {
+ return false;
+ }
+
+ LLBit_ = false;
+ set_pc(int64_t(newPC));
+ return true;
+ }
+
+ // Executes one instruction.
+ void instructionDecode(SimInstruction* instr);
+ // Execute one instruction placed in a branch delay slot.
+ void branchDelayInstructionDecode(SimInstruction* instr);
+
+ public:
+ static int64_t StopSimAt;
+
+ // Runtime call support.
+ static void* RedirectNativeFunction(void* nativeFunction,
+ ABIFunctionType type);
+
+ private:
+ enum Exception {
+ kNone,
+ kIntegerOverflow,
+ kIntegerUnderflow,
+ kDivideByZero,
+ kNumExceptions
+ };
+ int16_t exceptions[kNumExceptions];
+
+ // Exceptions.
+ void signalExceptions();
+
+ // Handle return value for runtime FP functions.
+ void setCallResultDouble(double result);
+ void setCallResultFloat(float result);
+ void setCallResult(int64_t res);
+ void setCallResult(__int128 res);
+
+ void callInternal(uint8_t* entry);
+
+ // Architecture state.
+ // Registers.
+ int64_t registers_[kNumSimuRegisters];
+ // Coprocessor Registers.
+ int64_t FPUregisters_[kNumFPURegisters];
+ // FPU control register.
+ uint32_t FCSR_;
+
+ bool LLBit_;
+ uintptr_t LLAddr_;
+ int64_t lastLLValue_;
+
+ // Simulator support.
+ char* stack_;
+ uintptr_t stackLimit_;
+ bool pc_modified_;
+ int64_t icount_;
+ int64_t break_count_;
+
+ // Debugger input.
+ char* lastDebuggerInput_;
+
+ // Registered breakpoints.
+ SimInstruction* break_pc_;
+ Instr break_instr_;
+
+ // Single-stepping support
+ bool single_stepping_;
+ SingleStepCallback single_step_callback_;
+ void* single_step_callback_arg_;
+
+ // A stop is watched if its code is less than kNumOfWatchedStops.
+ // Only watched stops support enabling/disabling and the counter feature.
+ static const uint32_t kNumOfWatchedStops = 256;
+
+ // Stop is disabled if bit 31 is set.
+ static const uint32_t kStopDisabledBit = 1U << 31;
+
+ // A stop is enabled, meaning the simulator will stop when meeting the
+ // instruction, if bit 31 of watchedStops_[code].count is unset.
+ // The value watchedStops_[code].count & ~(1 << 31) indicates how many times
+ // the breakpoint was hit or gone through.
+ struct StopCountAndDesc {
+ uint32_t count_;
+ char* desc_;
+ };
+ StopCountAndDesc watchedStops_[kNumOfWatchedStops];
+};
+
+// Process wide simulator state.
+class SimulatorProcess {
+ friend class Redirection;
+ friend class AutoLockSimulatorCache;
+
+ private:
+ // ICache checking.
+ struct ICacheHasher {
+ typedef void* Key;
+ typedef void* Lookup;
+ static HashNumber hash(const Lookup& l);
+ static bool match(const Key& k, const Lookup& l);
+ };
+
+ public:
+ typedef HashMap<void*, CachePage*, ICacheHasher, SystemAllocPolicy> ICacheMap;
+
+ static mozilla::Atomic<size_t, mozilla::ReleaseAcquire>
+ ICacheCheckingDisableCount;
+ static void FlushICache(void* start, size_t size);
+
+ static void checkICacheLocked(SimInstruction* instr);
+
+ static bool initialize() {
+ singleton_ = js_new<SimulatorProcess>();
+ return singleton_;
+ }
+ static void destroy() {
+ js_delete(singleton_);
+ singleton_ = nullptr;
+ }
+
+ SimulatorProcess();
+ ~SimulatorProcess();
+
+ private:
+ static SimulatorProcess* singleton_;
+
+ // This lock creates a critical section around 'redirection_' and
+ // 'icache_', which are referenced both by the execution engine
+ // and by the off-thread compiler (see Redirection::Get in the cpp file).
+ Mutex cacheLock_ MOZ_UNANNOTATED;
+
+ Redirection* redirection_;
+ ICacheMap icache_;
+
+ public:
+ static ICacheMap& icache() {
+ // Technically we need the lock to access the innards of the
+ // icache, not to take its address, but the latter condition
+ // serves as a useful complement to the former.
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ return singleton_->icache_;
+ }
+
+ static Redirection* redirection() {
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ return singleton_->redirection_;
+ }
+
+ static void setRedirection(js::jit::Redirection* redirection) {
+ singleton_->cacheLock_.assertOwnedByCurrentThread();
+ singleton_->redirection_ = redirection;
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* JS_SIMULATOR_MIPS64 */
+
+#endif /* jit_mips64_Simulator_mips64_h */
diff --git a/js/src/jit/mips64/Trampoline-mips64.cpp b/js/src/jit/mips64/Trampoline-mips64.cpp
new file mode 100644
index 0000000000..cdfe0c4741
--- /dev/null
+++ b/js/src/jit/mips64/Trampoline-mips64.cpp
@@ -0,0 +1,870 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/DebugOnly.h"
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/mips-shared/SharedICHelpers-mips-shared.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// All registers to save and restore. This includes the stack pointer, since we
+// use the ability to reference register values on the stack by index.
+static const LiveRegisterSet AllRegs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::AllMask),
+ FloatRegisterSet(FloatRegisters::AllMask));
+
+static_assert(sizeof(uintptr_t) == sizeof(uint64_t), "Not 32-bit clean.");
+
+struct EnterJITRegs {
+ double f31;
+ double f30;
+ double f29;
+ double f28;
+ double f27;
+ double f26;
+ double f25;
+ double f24;
+
+ uintptr_t align;
+
+ // non-volatile registers.
+ uint64_t ra;
+ uint64_t fp;
+ uint64_t s7;
+ uint64_t s6;
+ uint64_t s5;
+ uint64_t s4;
+ uint64_t s3;
+ uint64_t s2;
+ uint64_t s1;
+ uint64_t s0;
+ // Save reg_vp(a7) on stack, use it after call jit code.
+ uint64_t a7;
+};
+
+static void GenerateReturn(MacroAssembler& masm, int returnCode) {
+ MOZ_ASSERT(masm.framePushed() == sizeof(EnterJITRegs));
+
+ if (isLoongson()) {
+ // Restore non-volatile registers
+ masm.as_ld(s0, StackPointer, offsetof(EnterJITRegs, s0));
+ masm.as_gslq(s1, s2, StackPointer, offsetof(EnterJITRegs, s2));
+ masm.as_gslq(s3, s4, StackPointer, offsetof(EnterJITRegs, s4));
+ masm.as_gslq(s5, s6, StackPointer, offsetof(EnterJITRegs, s6));
+ masm.as_gslq(s7, fp, StackPointer, offsetof(EnterJITRegs, fp));
+ masm.as_ld(ra, StackPointer, offsetof(EnterJITRegs, ra));
+
+ // Restore non-volatile floating point registers
+ masm.as_gslq(f24, f25, StackPointer, offsetof(EnterJITRegs, f25));
+ masm.as_gslq(f26, f27, StackPointer, offsetof(EnterJITRegs, f27));
+ masm.as_gslq(f28, f29, StackPointer, offsetof(EnterJITRegs, f29));
+ masm.as_gslq(f30, f31, StackPointer, offsetof(EnterJITRegs, f31));
+ } else {
+ // Restore non-volatile registers
+ masm.as_ld(s0, StackPointer, offsetof(EnterJITRegs, s0));
+ masm.as_ld(s1, StackPointer, offsetof(EnterJITRegs, s1));
+ masm.as_ld(s2, StackPointer, offsetof(EnterJITRegs, s2));
+ masm.as_ld(s3, StackPointer, offsetof(EnterJITRegs, s3));
+ masm.as_ld(s4, StackPointer, offsetof(EnterJITRegs, s4));
+ masm.as_ld(s5, StackPointer, offsetof(EnterJITRegs, s5));
+ masm.as_ld(s6, StackPointer, offsetof(EnterJITRegs, s6));
+ masm.as_ld(s7, StackPointer, offsetof(EnterJITRegs, s7));
+ masm.as_ld(fp, StackPointer, offsetof(EnterJITRegs, fp));
+ masm.as_ld(ra, StackPointer, offsetof(EnterJITRegs, ra));
+
+ // Restore non-volatile floating point registers
+ masm.as_ldc1(f24, StackPointer, offsetof(EnterJITRegs, f24));
+ masm.as_ldc1(f25, StackPointer, offsetof(EnterJITRegs, f25));
+ masm.as_ldc1(f26, StackPointer, offsetof(EnterJITRegs, f26));
+ masm.as_ldc1(f27, StackPointer, offsetof(EnterJITRegs, f27));
+ masm.as_ldc1(f28, StackPointer, offsetof(EnterJITRegs, f28));
+ masm.as_ldc1(f29, StackPointer, offsetof(EnterJITRegs, f29));
+ masm.as_ldc1(f30, StackPointer, offsetof(EnterJITRegs, f30));
+ masm.as_ldc1(f31, StackPointer, offsetof(EnterJITRegs, f31));
+ }
+
+ masm.freeStack(sizeof(EnterJITRegs));
+
+ masm.branch(ra);
+}
+
+static void GeneratePrologue(MacroAssembler& masm) {
+ masm.reserveStack(sizeof(EnterJITRegs));
+
+ if (isLoongson()) {
+ masm.as_gssq(a7, s0, StackPointer, offsetof(EnterJITRegs, s0));
+ masm.as_gssq(s1, s2, StackPointer, offsetof(EnterJITRegs, s2));
+ masm.as_gssq(s3, s4, StackPointer, offsetof(EnterJITRegs, s4));
+ masm.as_gssq(s5, s6, StackPointer, offsetof(EnterJITRegs, s6));
+ masm.as_gssq(s7, fp, StackPointer, offsetof(EnterJITRegs, fp));
+ masm.as_sd(ra, StackPointer, offsetof(EnterJITRegs, ra));
+
+ masm.as_gssq(f24, f25, StackPointer, offsetof(EnterJITRegs, f25));
+ masm.as_gssq(f26, f27, StackPointer, offsetof(EnterJITRegs, f27));
+ masm.as_gssq(f28, f29, StackPointer, offsetof(EnterJITRegs, f29));
+ masm.as_gssq(f30, f31, StackPointer, offsetof(EnterJITRegs, f31));
+ return;
+ }
+
+ masm.as_sd(s0, StackPointer, offsetof(EnterJITRegs, s0));
+ masm.as_sd(s1, StackPointer, offsetof(EnterJITRegs, s1));
+ masm.as_sd(s2, StackPointer, offsetof(EnterJITRegs, s2));
+ masm.as_sd(s3, StackPointer, offsetof(EnterJITRegs, s3));
+ masm.as_sd(s4, StackPointer, offsetof(EnterJITRegs, s4));
+ masm.as_sd(s5, StackPointer, offsetof(EnterJITRegs, s5));
+ masm.as_sd(s6, StackPointer, offsetof(EnterJITRegs, s6));
+ masm.as_sd(s7, StackPointer, offsetof(EnterJITRegs, s7));
+ masm.as_sd(fp, StackPointer, offsetof(EnterJITRegs, fp));
+ masm.as_sd(ra, StackPointer, offsetof(EnterJITRegs, ra));
+ masm.as_sd(a7, StackPointer, offsetof(EnterJITRegs, a7));
+
+ masm.as_sdc1(f24, StackPointer, offsetof(EnterJITRegs, f24));
+ masm.as_sdc1(f25, StackPointer, offsetof(EnterJITRegs, f25));
+ masm.as_sdc1(f26, StackPointer, offsetof(EnterJITRegs, f26));
+ masm.as_sdc1(f27, StackPointer, offsetof(EnterJITRegs, f27));
+ masm.as_sdc1(f28, StackPointer, offsetof(EnterJITRegs, f28));
+ masm.as_sdc1(f29, StackPointer, offsetof(EnterJITRegs, f29));
+ masm.as_sdc1(f30, StackPointer, offsetof(EnterJITRegs, f30));
+ masm.as_sdc1(f31, StackPointer, offsetof(EnterJITRegs, f31));
+}
+
+// Generates a trampoline for calling Jit compiled code from a C++ function.
+// The trampoline use the EnterJitCode signature, with the standard x64 fastcall
+// calling convention.
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ const Register reg_code = IntArgReg0;
+ const Register reg_argc = IntArgReg1;
+ const Register reg_argv = IntArgReg2;
+ const mozilla::DebugOnly<Register> reg_frame = IntArgReg3;
+ const Register reg_token = IntArgReg4;
+ const Register reg_chain = IntArgReg5;
+ const Register reg_values = IntArgReg6;
+ const Register reg_vp = IntArgReg7;
+
+ MOZ_ASSERT(OsrFrameReg == reg_frame);
+
+ GeneratePrologue(masm);
+
+ // Save stack pointer as baseline frame.
+ masm.movePtr(StackPointer, FramePointer);
+
+ // Load the number of actual arguments into s3.
+ masm.unboxInt32(Address(reg_vp, 0), s3);
+
+ /***************************************************************
+ Loop over argv vector, push arguments onto stack in reverse order
+ ***************************************************************/
+
+ // if we are constructing, that also needs to include newTarget
+ {
+ Label noNewTarget;
+ masm.branchTest32(Assembler::Zero, reg_token,
+ Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+ masm.add32(Imm32(1), reg_argc);
+
+ masm.bind(&noNewTarget);
+ }
+
+ // Make stack algined
+ masm.ma_and(s0, reg_argc, Imm32(1));
+ masm.ma_dsubu(s1, StackPointer, Imm32(sizeof(Value)));
+#ifdef MIPSR6
+ masm.as_selnez(s1, s1, s0);
+ masm.as_seleqz(StackPointer, StackPointer, s0);
+ masm.as_or(StackPointer, StackPointer, s1);
+#else
+ masm.as_movn(StackPointer, s1, s0);
+#endif
+
+ masm.as_dsll(s0, reg_argc, 3); // Value* argv
+ masm.addPtr(reg_argv, s0); // s0 = &argv[argc]
+
+ // Loop over arguments, copying them from an unknown buffer onto the Ion
+ // stack so they can be accessed from JIT'ed code.
+ Label header, footer;
+ // If there aren't any arguments, don't do anything
+ masm.ma_b(s0, reg_argv, &footer, Assembler::BelowOrEqual, ShortJump);
+ {
+ masm.bind(&header);
+
+ masm.subPtr(Imm32(sizeof(Value)), s0);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+
+ ValueOperand value = ValueOperand(s6);
+ masm.loadValue(Address(s0, 0), value);
+ masm.storeValue(value, Address(StackPointer, 0));
+
+ masm.ma_b(s0, reg_argv, &header, Assembler::Above, ShortJump);
+ }
+ masm.bind(&footer);
+
+ masm.push(reg_token);
+ masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, s3, s3);
+
+ CodeLabel returnLabel;
+ Label oomReturnLabel;
+ {
+ // Handle Interpreter -> Baseline OSR.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(FramePointer));
+ regs.take(OsrFrameReg);
+ regs.take(reg_code);
+
+ Label notOsr;
+ masm.ma_b(OsrFrameReg, OsrFrameReg, &notOsr, Assembler::Zero, ShortJump);
+
+ Register numStackValues = reg_values;
+ regs.take(numStackValues);
+ Register scratch = regs.takeAny();
+
+ // Push return address.
+ masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ masm.ma_li(scratch, &returnLabel);
+ masm.storePtr(scratch, Address(StackPointer, 0));
+
+ // Push previous frame pointer.
+ masm.subPtr(Imm32(sizeof(uintptr_t)), StackPointer);
+ masm.storePtr(FramePointer, Address(StackPointer, 0));
+
+ // Reserve frame.
+ Register framePtr = FramePointer;
+ masm.movePtr(StackPointer, framePtr);
+ masm.subPtr(Imm32(BaselineFrame::Size()), StackPointer);
+
+ Register framePtrScratch = regs.takeAny();
+ masm.movePtr(sp, framePtrScratch);
+
+ // Reserve space for locals and stack values.
+ masm.ma_dsll(scratch, numStackValues, Imm32(3));
+ masm.subPtr(scratch, StackPointer);
+
+ // Enter exit frame.
+ masm.reserveStack(3 * sizeof(uintptr_t));
+ masm.storePtr(
+ ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)),
+ Address(StackPointer, 2 * sizeof(uintptr_t))); // Frame descriptor
+ masm.storePtr(
+ zero, Address(StackPointer, sizeof(uintptr_t))); // fake return address
+ masm.storePtr(FramePointer, Address(StackPointer, 0));
+
+ // No GC things to mark, push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ masm.reserveStack(2 * sizeof(uintptr_t));
+ masm.storePtr(framePtr,
+ Address(StackPointer, sizeof(uintptr_t))); // BaselineFrame
+ masm.storePtr(reg_code, Address(StackPointer, 0)); // jitcode
+
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(framePtrScratch); // BaselineFrame
+ masm.passABIArg(OsrFrameReg); // InterpreterFrame
+ masm.passABIArg(numStackValues);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ regs.add(OsrFrameReg);
+ Register jitcode = regs.takeAny();
+ masm.loadPtr(Address(StackPointer, 0), jitcode);
+ masm.loadPtr(Address(StackPointer, sizeof(uintptr_t)), framePtr);
+ masm.freeStack(2 * sizeof(uintptr_t));
+
+ Label error;
+ masm.freeStack(ExitFrameLayout::SizeWithFooter());
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.profilerEnterFrame(framePtr, scratch);
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+ masm.jump(jitcode);
+
+ // OOM: load error value, discard return address and previous frame
+ // pointer and return.
+ masm.bind(&error);
+ masm.movePtr(framePtr, StackPointer);
+ masm.addPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.jump(&oomReturnLabel);
+
+ masm.bind(&notOsr);
+ // Load the scope chain in R1.
+ MOZ_ASSERT(R1.scratchReg() != reg_code);
+ masm.ma_move(R1.scratchReg(), reg_chain);
+ }
+
+ // The call will push the return address on the stack, thus we check that
+ // the stack would be aligned once the call is complete.
+ masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+ // Call the function with pushing return address to stack.
+ masm.callJitNoProfiler(reg_code);
+
+ {
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&returnLabel);
+ masm.addCodeLabel(returnLabel);
+ masm.bind(&oomReturnLabel);
+ }
+
+ // Discard arguments and padding. Set sp to the address of the EnterJITRegs
+ // on the stack.
+ masm.mov(FramePointer, StackPointer);
+
+ // Store the returned value into the vp
+ masm.as_ld(reg_vp, StackPointer, offsetof(EnterJITRegs, a7));
+ masm.storeValue(JSReturnOperand, Address(reg_vp, 0));
+
+ // Restore non-volatile registers and return.
+ GenerateReturn(masm, ShortJump);
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ // Not supported, or not implemented yet.
+ // TODO: Implement along with the corresponding stack-walker changes, in
+ // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
+ return mozilla::Nothing{};
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // Stack has to be alligned here. If not, we will have to fix it.
+ masm.checkStackAlignment();
+
+ // Push registers such that we can access them from [base + code].
+ masm.PushRegsInMask(AllRegs);
+
+ // Pass pointer to InvalidationBailoutStack structure.
+ masm.movePtr(StackPointer, a0);
+
+ // Reserve place for BailoutInfo pointer. Two words to ensure alignment for
+ // setupAlignedABICall.
+ masm.subPtr(Imm32(2 * sizeof(uintptr_t)), StackPointer);
+ // Pass pointer to BailoutInfo
+ masm.movePtr(StackPointer, a1);
+
+ using Fn =
+ bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupAlignedABICall();
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.callWithABI<Fn, InvalidationBailout>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(a2);
+
+ // Pop the machine state and the dead frame.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind) {
+ // Do not erase the frame pointer in this function.
+
+ AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+ masm.pushReturnAddress();
+
+ // Caller:
+ // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- sp
+
+ // Frame prologue.
+ //
+ // NOTE: if this changes, fix the Baseline bailout code too!
+ // See BaselineStackBuilder::calculatePrevFramePtr and
+ // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+ masm.push(FramePointer);
+ masm.mov(StackPointer, FramePointer);
+
+ // Load argc.
+ masm.loadNumActualArgs(FramePointer, s3);
+
+ Register numActArgsReg = a6;
+ Register calleeTokenReg = a7;
+ Register numArgsReg = a5;
+
+ // Load |nformals| into numArgsReg.
+ masm.loadPtr(
+ Address(FramePointer, RectifierFrameLayout::offsetOfCalleeToken()),
+ calleeTokenReg);
+ masm.mov(calleeTokenReg, numArgsReg);
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), numArgsReg);
+ masm.loadFunctionArgCount(numArgsReg, numArgsReg);
+
+ // Stash another copy in t3, since we are going to do destructive operations
+ // on numArgsReg
+ masm.mov(numArgsReg, t3);
+
+ static_assert(
+ CalleeToken_FunctionConstructing == 1,
+ "Ensure that we can use the constructing bit to count the value");
+ masm.mov(calleeTokenReg, t2);
+ masm.ma_and(t2, Imm32(uint32_t(CalleeToken_FunctionConstructing)));
+
+ // Including |this|, and |new.target|, there are (|nformals| + 1 +
+ // isConstructing) arguments to push to the stack. Then we push a
+ // JitFrameLayout. We compute the padding expressed in the number of extra
+ // |undefined| values to push on the stack.
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ static_assert(
+ JitStackAlignment % sizeof(Value) == 0,
+ "Ensure that we can pad the stack by pushing extra UndefinedValue");
+
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(JitStackValueAlignment));
+ masm.add32(
+ Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */),
+ numArgsReg);
+ masm.add32(t2, numArgsReg);
+ masm.and32(Imm32(~(JitStackValueAlignment - 1)), numArgsReg);
+
+ // Load the number of |undefined|s to push into t1. Subtract 1 for |this|.
+ masm.as_dsubu(t1, numArgsReg, s3);
+ masm.sub32(Imm32(1), t1);
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- sp
+ // '--- s3 ----'
+ //
+ // Rectifier frame:
+ // [fp'][undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee]
+ // [descr] [raddr] ]
+ // '-------- t1 ---------' '--- s3 ----'
+
+ // Copy number of actual arguments into numActArgsReg
+ masm.mov(s3, numActArgsReg); // Save %sp.
+
+ masm.moveValue(UndefinedValue(), ValueOperand(t0));
+
+ // Push undefined. (including the padding)
+ {
+ Label undefLoopTop;
+
+ masm.bind(&undefLoopTop);
+ masm.sub32(Imm32(1), t1);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.storeValue(ValueOperand(t0), Address(StackPointer, 0));
+
+ masm.ma_b(t1, t1, &undefLoopTop, Assembler::NonZero, ShortJump);
+ }
+
+ // Get the topmost argument.
+ static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments");
+
+ // Get the topmost argument.
+ masm.ma_dsll(t0, s3, Imm32(3)); // t0 <- nargs * 8
+ masm.as_daddu(t1, FramePointer, t0); // t1 <- fp(saved sp) + nargs * 8
+ masm.addPtr(Imm32(sizeof(RectifierFrameLayout)), t1);
+
+ // Push arguments, |nargs| + 1 times (to include |this|).
+
+ masm.addPtr(Imm32(1), s3);
+ {
+ Label copyLoopTop;
+
+ masm.bind(&copyLoopTop);
+ masm.sub32(Imm32(1), s3);
+ masm.subPtr(Imm32(sizeof(Value)), StackPointer);
+ masm.loadValue(Address(t1, 0), ValueOperand(t0));
+ masm.storeValue(ValueOperand(t0), Address(StackPointer, 0));
+ masm.subPtr(Imm32(sizeof(Value)), t1);
+
+ masm.ma_b(s3, s3, &copyLoopTop, Assembler::NonZero, ShortJump);
+ }
+
+ // if constructing, copy newTarget
+ {
+ Label notConstructing;
+
+ masm.branchTest32(Assembler::Zero, calleeTokenReg,
+ Imm32(CalleeToken_FunctionConstructing),
+ &notConstructing);
+
+ // thisFrame[numFormals] = prevFrame[argc]
+ ValueOperand newTarget(t0);
+
+ // Load vp[argc]. Add sizeof(Value) for |this|.
+ BaseIndex newTargetSrc(FramePointer, numActArgsReg, TimesEight,
+ sizeof(RectifierFrameLayout) + sizeof(Value));
+ masm.loadValue(newTargetSrc, newTarget);
+
+ // Again, 1 for |this|
+ BaseIndex newTargetDest(StackPointer, t3, TimesEight, sizeof(Value));
+ masm.storeValue(newTarget, newTargetDest);
+
+ masm.bind(&notConstructing);
+ }
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ]
+ //
+ //
+ // Rectifier frame:
+ // [fp'] <- fp [undef] [undef] [undef] [arg2] [arg1] [this] <- sp [ [argc]
+ // [callee] [descr] [raddr] ]
+
+ // Construct JitFrameLayout.
+ masm.push(calleeTokenReg);
+ masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, numActArgsReg,
+ numActArgsReg);
+
+ // Call the target function.
+ masm.andPtr(Imm32(uint32_t(CalleeTokenMask)), calleeTokenReg);
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(calleeTokenReg, t1);
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(t1);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(calleeTokenReg, t1, &noBaselineScript);
+ masm.callJitNoProfiler(t1);
+ masm.jump(&done);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(calleeTokenReg, t1);
+ masm.callJitNoProfiler(t1);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.mov(FramePointer, StackPointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+/* - When bailout is done via out of line code (lazy bailout).
+ * Frame size is stored in $ra (look at
+ * CodeGeneratorMIPS64::generateOutOfLineCode()) and thunk code should save it
+ * on stack. Other difference is that members snapshotOffset_ and padding_ are
+ * pushed to the stack by CodeGeneratorMIPS64::visitOutOfLineBailout().
+ */
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+ // Push the frameSize_ stored in ra
+ // See: CodeGeneratorMIPS64::generateOutOfLineCode()
+ masm.push(ra);
+
+ // Push registers such that we can access them from [base + code].
+ masm.PushRegsInMask(AllRegs);
+
+ // Put pointer to BailoutStack as first argument to the Bailout()
+ masm.movePtr(StackPointer, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+ PushBailoutFrame(masm, a0);
+
+ // Put pointer to BailoutInfo
+ static const uint32_t sizeOfBailoutInfo = sizeof(uintptr_t) * 2;
+ masm.subPtr(Imm32(sizeOfBailoutInfo), StackPointer);
+ masm.movePtr(StackPointer, a1);
+
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupAlignedABICall();
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ // Get BailoutInfo pointer
+ masm.loadPtr(Address(StackPointer, 0), a2);
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in a2.
+ masm.jump(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+ *wrapperOffset = startTrampolineCode(masm);
+
+ // Avoid conflicts with argument registers while discarding the result after
+ // the function call.
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert(
+ (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set should be a superset of Volatile register set.");
+
+ // The context is the first argument; a0 is the first argument register.
+ Register cxreg = a0;
+ regs.take(cxreg);
+
+ // If it isn't a tail call, then the return address needs to be saved
+ if (f.expectTailCall == NonTailCall) {
+ masm.pushReturnAddress();
+ }
+
+ // Push the frame pointer to finish the exit frame, then link it up.
+ masm.Push(FramePointer);
+ masm.loadJSContext(cxreg);
+ masm.enterExitFrame(cxreg, regs.getAny(), &f);
+
+ // Save the base of the argument set stored on the stack.
+ Register argsBase = InvalidReg;
+ if (f.explicitArgs) {
+ argsBase = t1; // Use temporary register.
+ regs.take(argsBase);
+ masm.ma_daddu(argsBase, StackPointer,
+ Imm32(ExitFrameLayout::SizeWithFooter()));
+ }
+
+ // Reserve space for the outparameter.
+ Register outReg = InvalidReg;
+ switch (f.outParam) {
+ case Type_Value:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(Value));
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ case Type_Handle:
+ outReg = regs.takeAny();
+ masm.PushEmptyRooted(f.outParamRootType);
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ case Type_Bool:
+ case Type_Int32:
+ outReg = regs.takeAny();
+ // Reserve 4-byte space to make stack aligned to 8-byte.
+ masm.reserveStack(2 * sizeof(int32_t));
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ case Type_Pointer:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(uintptr_t));
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ case Type_Double:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(double));
+ masm.movePtr(StackPointer, outReg);
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ masm.setupUnalignedABICall(regs.getAny());
+ masm.passABIArg(cxreg);
+
+ size_t argDisp = 0;
+
+ // Copy any arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ if (f.argPassedInFloatReg(explicitArg)) {
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
+ } else {
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ }
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByValue:
+ case VMFunctionData::DoubleByRef:
+ MOZ_CRASH("NYI: MIPS64 callVM should not be used with 128bits values.");
+ break;
+ }
+ }
+
+ // Copy the implicit outparam, if any.
+ if (InvalidReg != outReg) {
+ masm.passABIArg(outReg);
+ }
+
+ masm.callWithABI(nativeFun, MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, v0, v0, masm.failureLabel());
+ break;
+ case Type_Bool:
+ // Called functions return bools, which are 0/false and non-zero/true
+ masm.branchIfFalseBool(v0, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ // Load the outparam and free any allocated stack.
+ switch (f.outParam) {
+ case Type_Handle:
+ masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+ break;
+
+ case Type_Value:
+ masm.loadValue(Address(StackPointer, 0), JSReturnOperand);
+ masm.freeStack(sizeof(Value));
+ break;
+
+ case Type_Int32:
+ masm.load32(Address(StackPointer, 0), ReturnReg);
+ masm.freeStack(2 * sizeof(int32_t));
+ break;
+
+ case Type_Pointer:
+ masm.loadPtr(Address(StackPointer, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ case Type_Bool:
+ masm.load8ZeroExtend(Address(StackPointer, 0), ReturnReg);
+ masm.freeStack(2 * sizeof(int32_t));
+ break;
+
+ case Type_Double:
+ masm.as_ldc1(ReturnDoubleReg, StackPointer, 0);
+ masm.freeStack(sizeof(double));
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ // Pop ExitFooterFrame and the frame pointer.
+ masm.leaveExitFrame(sizeof(void*));
+
+ // Return. Subtract sizeof(void*) for the frame pointer.
+ masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+ f.explicitStackSlots() * sizeof(void*) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type) {
+ AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+ uint32_t offset = startTrampolineCode(masm);
+
+ MOZ_ASSERT(PreBarrierReg == a1);
+ Register temp1 = a0;
+ Register temp2 = a2;
+ Register temp3 = a3;
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+ &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ LiveRegisterSet save;
+ save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileMask));
+ save.add(ra);
+ masm.PushRegsInMask(save);
+
+ masm.movePtr(ImmPtr(cx->runtime()), a0);
+
+ masm.setupUnalignedABICall(a2);
+ masm.passABIArg(a0);
+ masm.passABIArg(a1);
+ masm.callWithABI(JitPreWriteBarrier(type));
+
+ save.take(AnyRegister(ra));
+ masm.PopRegsInMask(save);
+ masm.ret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.abiret();
+
+ return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+ masm.bind(bailoutTail);
+ masm.generateBailoutTail(a1, a2);
+}
diff --git a/js/src/jit/moz.build b/js/src/jit/moz.build
new file mode 100644
index 0000000000..bccb96e225
--- /dev/null
+++ b/js/src/jit/moz.build
@@ -0,0 +1,269 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+FINAL_LIBRARY = "js"
+
+# Includes should be relative to parent path
+LOCAL_INCLUDES += ["!..", ".."]
+
+include("../js-config.mozbuild")
+include("../js-cxxflags.mozbuild")
+
+UNIFIED_SOURCES += [
+ "AliasAnalysis.cpp",
+ "AlignmentMaskAnalysis.cpp",
+ "BacktrackingAllocator.cpp",
+ "Bailouts.cpp",
+ "BaselineBailouts.cpp",
+ "BaselineCacheIRCompiler.cpp",
+ "BaselineCodeGen.cpp",
+ "BaselineDebugModeOSR.cpp",
+ "BaselineFrame.cpp",
+ "BaselineFrameInfo.cpp",
+ "BaselineIC.cpp",
+ "BaselineJIT.cpp",
+ "BitSet.cpp",
+ "BytecodeAnalysis.cpp",
+ "CacheIR.cpp",
+ "CacheIRCompiler.cpp",
+ "CacheIRHealth.cpp",
+ "CacheIRSpewer.cpp",
+ "CodeGenerator.cpp",
+ "CompileWrappers.cpp",
+ "Disassemble.cpp",
+ "EdgeCaseAnalysis.cpp",
+ "EffectiveAddressAnalysis.cpp",
+ "ExecutableAllocator.cpp",
+ "FlushICache.cpp",
+ "FoldLinearArithConstants.cpp",
+ "InlinableNatives.cpp",
+ "InstructionReordering.cpp",
+ "Ion.cpp",
+ "IonAnalysis.cpp",
+ "IonCacheIRCompiler.cpp",
+ "IonCompileTask.cpp",
+ "IonIC.cpp",
+ "IonOptimizationLevels.cpp",
+ "Jit.cpp",
+ "JitcodeMap.cpp",
+ "JitContext.cpp",
+ "JitFrames.cpp",
+ "JitOptions.cpp",
+ "JitScript.cpp",
+ "JitSpewer.cpp",
+ "JSJitFrameIter.cpp",
+ "JSONSpewer.cpp",
+ "KnownClass.cpp",
+ "Label.cpp",
+ "LICM.cpp",
+ "Linker.cpp",
+ "LIR.cpp",
+ "Lowering.cpp",
+ "MacroAssembler.cpp",
+ "MIR.cpp",
+ "MIRGraph.cpp",
+ "MoveResolver.cpp",
+ "PerfSpewer.cpp",
+ "ProcessExecutableMemory.cpp",
+ "RangeAnalysis.cpp",
+ "ReciprocalMulConstants.cpp",
+ "Recover.cpp",
+ "RegisterAllocator.cpp",
+ "RematerializedFrame.cpp",
+ "SafepointIndex.cpp",
+ "Safepoints.cpp",
+ "ScalarReplacement.cpp",
+ "shared/Assembler-shared.cpp",
+ "shared/AtomicOperations-shared-jit.cpp",
+ "shared/CodeGenerator-shared.cpp",
+ "shared/Disassembler-shared.cpp",
+ "shared/Lowering-shared.cpp",
+ "ShuffleAnalysis.cpp",
+ "Sink.cpp",
+ "Snapshots.cpp",
+ "Trampoline.cpp",
+ "TrialInlining.cpp",
+ "TypePolicy.cpp",
+ "ValueNumbering.cpp",
+ "VMFunctions.cpp",
+ "WarpBuilder.cpp",
+ "WarpBuilderShared.cpp",
+ "WarpCacheIRTranspiler.cpp",
+ "WarpOracle.cpp",
+ "WarpSnapshot.cpp",
+ "WasmBCE.cpp",
+ "XrayJitInfo.cpp",
+]
+
+if CONFIG["JS_CODEGEN_NONE"]:
+ UNIFIED_SOURCES += ["none/Trampoline-none.cpp"]
+elif CONFIG["JS_CODEGEN_X86"] or CONFIG["JS_CODEGEN_X64"]:
+ UNIFIED_SOURCES += [
+ "x86-shared/Architecture-x86-shared.cpp",
+ "x86-shared/Assembler-x86-shared.cpp",
+ "x86-shared/AssemblerBuffer-x86-shared.cpp",
+ "x86-shared/CodeGenerator-x86-shared.cpp",
+ "x86-shared/Lowering-x86-shared.cpp",
+ "x86-shared/MacroAssembler-x86-shared-SIMD.cpp",
+ "x86-shared/MacroAssembler-x86-shared.cpp",
+ "x86-shared/MoveEmitter-x86-shared.cpp",
+ ]
+ if CONFIG["JS_CODEGEN_X64"]:
+ UNIFIED_SOURCES += [
+ "x64/Assembler-x64.cpp",
+ "x64/CodeGenerator-x64.cpp",
+ "x64/Lowering-x64.cpp",
+ "x64/MacroAssembler-x64.cpp",
+ "x64/Trampoline-x64.cpp",
+ ]
+ else:
+ UNIFIED_SOURCES += [
+ "x86/Assembler-x86.cpp",
+ "x86/CodeGenerator-x86.cpp",
+ "x86/Lowering-x86.cpp",
+ "x86/MacroAssembler-x86.cpp",
+ "x86/Trampoline-x86.cpp",
+ ]
+elif CONFIG["JS_CODEGEN_ARM"]:
+ UNIFIED_SOURCES += [
+ "arm/Architecture-arm.cpp",
+ "arm/Assembler-arm.cpp",
+ "arm/CodeGenerator-arm.cpp",
+ "arm/disasm/Constants-arm.cpp",
+ "arm/disasm/Disasm-arm.cpp",
+ "arm/Lowering-arm.cpp",
+ "arm/MacroAssembler-arm.cpp",
+ "arm/MoveEmitter-arm.cpp",
+ "arm/Trampoline-arm.cpp",
+ ]
+ if CONFIG["JS_SIMULATOR_ARM"]:
+ UNIFIED_SOURCES += ["arm/Simulator-arm.cpp"]
+ elif CONFIG["OS_ARCH"] == "Darwin":
+ SOURCES += [
+ "arm/llvm-compiler-rt/arm/aeabi_idivmod.S",
+ "arm/llvm-compiler-rt/arm/aeabi_uidivmod.S",
+ ]
+elif CONFIG["JS_CODEGEN_ARM64"]:
+ UNIFIED_SOURCES += [
+ "arm64/Architecture-arm64.cpp",
+ "arm64/Assembler-arm64.cpp",
+ "arm64/CodeGenerator-arm64.cpp",
+ "arm64/Lowering-arm64.cpp",
+ "arm64/MacroAssembler-arm64.cpp",
+ "arm64/MoveEmitter-arm64.cpp",
+ "arm64/Trampoline-arm64.cpp",
+ "arm64/vixl/Assembler-vixl.cpp",
+ "arm64/vixl/Cpu-Features-vixl.cpp",
+ "arm64/vixl/Cpu-vixl.cpp",
+ "arm64/vixl/Decoder-vixl.cpp",
+ "arm64/vixl/Instructions-vixl.cpp",
+ "arm64/vixl/MacroAssembler-vixl.cpp",
+ "arm64/vixl/MozAssembler-vixl.cpp",
+ "arm64/vixl/MozCpu-vixl.cpp",
+ "arm64/vixl/MozInstructions-vixl.cpp",
+ "arm64/vixl/Utils-vixl.cpp",
+ ]
+ vixl_werror_sources = [
+ "arm64/vixl/Disasm-vixl.cpp",
+ "arm64/vixl/Instrument-vixl.cpp",
+ ]
+ SOURCES += vixl_werror_sources
+ if CONFIG["CC_TYPE"] == "clang-cl":
+ for f in vixl_werror_sources:
+ SOURCES[f].flags += ["-Wno-c++11-narrowing"]
+ if CONFIG["JS_SIMULATOR_ARM64"]:
+ UNIFIED_SOURCES += [
+ "arm64/vixl/Debugger-vixl.cpp",
+ "arm64/vixl/Logic-vixl.cpp",
+ "arm64/vixl/MozSimulator-vixl.cpp",
+ "arm64/vixl/Simulator-vixl.cpp",
+ ]
+elif CONFIG["JS_CODEGEN_MIPS32"] or CONFIG["JS_CODEGEN_MIPS64"]:
+ UNIFIED_SOURCES += [
+ "mips-shared/Architecture-mips-shared.cpp",
+ "mips-shared/Assembler-mips-shared.cpp",
+ "mips-shared/CodeGenerator-mips-shared.cpp",
+ "mips-shared/Lowering-mips-shared.cpp",
+ "mips-shared/MacroAssembler-mips-shared.cpp",
+ "mips-shared/MoveEmitter-mips-shared.cpp",
+ ]
+ if CONFIG["JS_CODEGEN_MIPS32"]:
+ UNIFIED_SOURCES += [
+ "mips32/Architecture-mips32.cpp",
+ "mips32/Assembler-mips32.cpp",
+ "mips32/CodeGenerator-mips32.cpp",
+ "mips32/Lowering-mips32.cpp",
+ "mips32/MacroAssembler-mips32.cpp",
+ "mips32/MoveEmitter-mips32.cpp",
+ "mips32/Trampoline-mips32.cpp",
+ ]
+ if CONFIG["JS_SIMULATOR_MIPS32"]:
+ UNIFIED_SOURCES += ["mips32/Simulator-mips32.cpp"]
+ elif CONFIG["JS_CODEGEN_MIPS64"]:
+ UNIFIED_SOURCES += [
+ "mips64/Architecture-mips64.cpp",
+ "mips64/Assembler-mips64.cpp",
+ "mips64/CodeGenerator-mips64.cpp",
+ "mips64/Lowering-mips64.cpp",
+ "mips64/MacroAssembler-mips64.cpp",
+ "mips64/MoveEmitter-mips64.cpp",
+ "mips64/Trampoline-mips64.cpp",
+ ]
+ if CONFIG["JS_SIMULATOR_MIPS64"]:
+ UNIFIED_SOURCES += ["mips64/Simulator-mips64.cpp"]
+elif CONFIG["JS_CODEGEN_LOONG64"]:
+ UNIFIED_SOURCES += [
+ "loong64/Architecture-loong64.cpp",
+ "loong64/Assembler-loong64.cpp",
+ "loong64/CodeGenerator-loong64.cpp",
+ "loong64/Lowering-loong64.cpp",
+ "loong64/MacroAssembler-loong64.cpp",
+ "loong64/MoveEmitter-loong64.cpp",
+ "loong64/Trampoline-loong64.cpp",
+ ]
+ if CONFIG["JS_SIMULATOR_LOONG64"]:
+ UNIFIED_SOURCES += ["loong64/Simulator-loong64.cpp"]
+elif CONFIG["JS_CODEGEN_WASM32"]:
+ UNIFIED_SOURCES += [
+ "wasm32/CodeGenerator-wasm32.cpp",
+ "wasm32/MacroAssembler-wasm32.cpp",
+ "wasm32/Trampoline-wasm32.cpp",
+ ]
+
+# Generate jit/MIROpsGenerated.h from jit/MIROps.yaml
+GeneratedFile(
+ "MIROpsGenerated.h",
+ script="GenerateMIRFiles.py",
+ entry_point="generate_mir_header",
+ inputs=["MIROps.yaml"],
+)
+
+# Generate jit/LIROpsGenerated.h from jit/LIR.h, jit/shared/LIR-shared.h, and
+# platform-specific LIR files.
+GeneratedFile(
+ "LIROpsGenerated.h",
+ script="GenerateLIRFiles.py",
+ entry_point="generate_lir_header",
+ inputs=["LIROps.yaml"],
+)
+
+# Generate jit/CacheIROpsGenerated.h from jit/CacheIROps.yaml
+GeneratedFile(
+ "CacheIROpsGenerated.h",
+ script="GenerateCacheIRFiles.py",
+ entry_point="generate_cacheirops_header",
+ inputs=["CacheIROps.yaml"],
+)
+
+GeneratedFile(
+ "AtomicOperationsGenerated.h",
+ script="GenerateAtomicOperations.py",
+ entry_point="generate_atomics_header",
+ inputs=[],
+)
+
+if CONFIG["FUZZING_INTERFACES"] or CONFIG["FUZZING_JS_FUZZILLI"]:
+ include("/tools/fuzzing/libfuzzer-config.mozbuild")
diff --git a/js/src/jit/none/Architecture-none.h b/js/src/jit/none/Architecture-none.h
new file mode 100644
index 0000000000..2433234fbf
--- /dev/null
+++ b/js/src/jit/none/Architecture-none.h
@@ -0,0 +1,171 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_Architecture_none_h
+#define jit_none_Architecture_none_h
+
+// JitSpewer.h is included through MacroAssembler implementations for other
+// platforms, so include it here to avoid inadvertent build bustage.
+#include "jit/JitSpewer.h"
+
+#include "jit/shared/Architecture-shared.h"
+
+namespace js {
+namespace jit {
+
+static const uint32_t SimdMemoryAlignment =
+ 4; // Make it 4 to avoid a bunch of div-by-zero warnings
+static const uint32_t WasmStackAlignment = 8;
+static const uint32_t WasmTrapInstructionLength = 0;
+
+// See comments in wasm::GenerateFunctionPrologue.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+class Registers {
+ public:
+ enum RegisterID {
+ r0 = 0,
+ invalid_reg,
+ invalid_reg2, // To avoid silly static_assert failures.
+ };
+ typedef uint8_t Code;
+ typedef RegisterID Encoding;
+ union RegisterContent {
+ uintptr_t r;
+ };
+
+ typedef uint8_t SetType;
+
+ static uint32_t SetSize(SetType) { MOZ_CRASH(); }
+ static uint32_t FirstBit(SetType) { MOZ_CRASH(); }
+ static uint32_t LastBit(SetType) { MOZ_CRASH(); }
+ static const char* GetName(Code) { MOZ_CRASH(); }
+ static Code FromName(const char*) { MOZ_CRASH(); }
+
+ static const Encoding StackPointer = invalid_reg;
+ static const Encoding Invalid = invalid_reg;
+ static const uint32_t Total = 1;
+ static const uint32_t TotalPhys = 0;
+ static const uint32_t Allocatable = 0;
+ static const SetType AllMask = 0;
+ static const SetType ArgRegMask = 0;
+ static const SetType VolatileMask = 0;
+ static const SetType NonVolatileMask = 0;
+ static const SetType NonAllocatableMask = 0;
+ static const SetType AllocatableMask = 0;
+ static const SetType JSCallMask = 0;
+ static const SetType CallMask = 0;
+};
+
+typedef uint8_t PackedRegisterMask;
+
+class FloatRegisters {
+ public:
+ enum FPRegisterID { f0 = 0, invalid_reg };
+ typedef FPRegisterID Code;
+ typedef FPRegisterID Encoding;
+ union RegisterContent {
+ float s;
+ double d;
+ };
+
+ typedef uint32_t SetType;
+
+ static const char* GetName(Code) { MOZ_CRASH(); }
+ static Code FromName(const char*) { MOZ_CRASH(); }
+
+ static const Code Invalid = invalid_reg;
+ static const uint32_t Total = 0;
+ static const uint32_t TotalPhys = 0;
+ static const uint32_t Allocatable = 0;
+ static const SetType AllMask = 0;
+ static const SetType AllDoubleMask = 0;
+ static const SetType AllSingleMask = 0;
+ static const SetType VolatileMask = 0;
+ static const SetType NonVolatileMask = 0;
+ static const SetType NonAllocatableMask = 0;
+ static const SetType AllocatableMask = 0;
+};
+
+template <typename T>
+class TypedRegisterSet;
+
+struct FloatRegister {
+ typedef FloatRegisters Codes;
+ typedef Codes::Code Code;
+ typedef Codes::Encoding Encoding;
+ typedef Codes::SetType SetType;
+
+ Code _;
+
+ static uint32_t FirstBit(SetType) { MOZ_CRASH(); }
+ static uint32_t LastBit(SetType) { MOZ_CRASH(); }
+ static FloatRegister FromCode(uint32_t) { MOZ_CRASH(); }
+ bool isSingle() const { MOZ_CRASH(); }
+ bool isDouble() const { MOZ_CRASH(); }
+ bool isSimd128() const { MOZ_CRASH(); }
+ bool isInvalid() const { MOZ_CRASH(); }
+ FloatRegister asSingle() const { MOZ_CRASH(); }
+ FloatRegister asDouble() const { MOZ_CRASH(); }
+ FloatRegister asSimd128() const { MOZ_CRASH(); }
+ Code code() const { MOZ_CRASH(); }
+ Encoding encoding() const { MOZ_CRASH(); }
+ const char* name() const { MOZ_CRASH(); }
+ bool volatile_() const { MOZ_CRASH(); }
+ bool operator!=(FloatRegister) const { MOZ_CRASH(); }
+ bool operator==(FloatRegister) const { MOZ_CRASH(); }
+ bool aliases(FloatRegister) const { MOZ_CRASH(); }
+ uint32_t numAliased() const { MOZ_CRASH(); }
+ FloatRegister aliased(uint32_t) { MOZ_CRASH(); }
+ bool equiv(FloatRegister) const { MOZ_CRASH(); }
+ uint32_t size() const { MOZ_CRASH(); }
+ uint32_t numAlignedAliased() const { MOZ_CRASH(); }
+ FloatRegister alignedAliased(uint32_t) { MOZ_CRASH(); }
+ SetType alignedOrDominatedAliasedSet() const { MOZ_CRASH(); }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return SetType(0);
+ }
+
+ template <typename T>
+ static T ReduceSetForPush(T) {
+ MOZ_CRASH();
+ }
+ uint32_t getRegisterDumpOffsetInBytes() { MOZ_CRASH(); }
+ static uint32_t SetSize(SetType x) { MOZ_CRASH(); }
+ static Code FromName(const char* name) { MOZ_CRASH(); }
+
+ // This is used in static initializers, so produce a bogus value instead of
+ // crashing.
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>&) {
+ return 0;
+ }
+};
+
+inline bool hasUnaliasedDouble() { MOZ_CRASH(); }
+inline bool hasMultiAlias() { MOZ_CRASH(); }
+
+static const uint32_t ShadowStackSpace = 0;
+static const uint32_t JumpImmediateRange = INT32_MAX;
+
+#ifdef JS_NUNBOX32
+static const int32_t NUNBOX32_TYPE_OFFSET = 4;
+static const int32_t NUNBOX32_PAYLOAD_OFFSET = 0;
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_Architecture_none_h */
diff --git a/js/src/jit/none/Assembler-none.h b/js/src/jit/none/Assembler-none.h
new file mode 100644
index 0000000000..a149bba1c5
--- /dev/null
+++ b/js/src/jit/none/Assembler-none.h
@@ -0,0 +1,210 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_Assembler_none_h
+#define jit_none_Assembler_none_h
+
+#include "mozilla/Assertions.h"
+
+#include <stdint.h>
+
+#include "jit/none/Architecture-none.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "jit/shared/Assembler-shared.h"
+
+namespace js {
+namespace jit {
+
+class MacroAssembler;
+
+static constexpr Register StackPointer{Registers::invalid_reg};
+static constexpr Register FramePointer{Registers::invalid_reg};
+static constexpr Register ReturnReg{Registers::invalid_reg2};
+static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister ReturnSimd128Reg = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister ScratchSimd128Reg = {
+ FloatRegisters::invalid_reg};
+static constexpr FloatRegister InvalidFloatReg = {FloatRegisters::invalid_reg};
+
+struct ScratchFloat32Scope : FloatRegister {
+ explicit ScratchFloat32Scope(MacroAssembler& masm) {}
+};
+
+struct ScratchDoubleScope : FloatRegister {
+ explicit ScratchDoubleScope(MacroAssembler& masm) {}
+};
+
+static constexpr Register OsrFrameReg{Registers::invalid_reg};
+static constexpr Register PreBarrierReg{Registers::invalid_reg};
+static constexpr Register InterpreterPCReg{Registers::invalid_reg};
+static constexpr Register CallTempReg0{Registers::invalid_reg};
+static constexpr Register CallTempReg1{Registers::invalid_reg};
+static constexpr Register CallTempReg2{Registers::invalid_reg};
+static constexpr Register CallTempReg3{Registers::invalid_reg};
+static constexpr Register CallTempReg4{Registers::invalid_reg};
+static constexpr Register CallTempReg5{Registers::invalid_reg};
+static constexpr Register InvalidReg{Registers::invalid_reg};
+static constexpr Register CallTempNonArgRegs[] = {InvalidReg, InvalidReg};
+static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr Register IntArgReg0{Registers::invalid_reg};
+static constexpr Register IntArgReg1{Registers::invalid_reg};
+static constexpr Register IntArgReg2{Registers::invalid_reg};
+static constexpr Register IntArgReg3{Registers::invalid_reg};
+static constexpr Register HeapReg{Registers::invalid_reg};
+
+static constexpr Register RegExpTesterRegExpReg{Registers::invalid_reg};
+static constexpr Register RegExpTesterStringReg{Registers::invalid_reg};
+static constexpr Register RegExpTesterLastIndexReg{Registers::invalid_reg};
+static constexpr Register RegExpTesterStickyReg{Registers::invalid_reg};
+
+static constexpr Register RegExpMatcherRegExpReg{Registers::invalid_reg};
+static constexpr Register RegExpMatcherStringReg{Registers::invalid_reg};
+static constexpr Register RegExpMatcherLastIndexReg{Registers::invalid_reg};
+static constexpr Register RegExpMatcherStickyReg{Registers::invalid_reg};
+
+// Uses |invalid_reg2| to avoid static_assert failures.
+static constexpr Register JSReturnReg_Type{Registers::invalid_reg2};
+static constexpr Register JSReturnReg_Data{Registers::invalid_reg2};
+static constexpr Register JSReturnReg{Registers::invalid_reg2};
+
+#if defined(JS_NUNBOX32)
+static constexpr ValueOperand JSReturnOperand(InvalidReg, InvalidReg);
+static constexpr Register64 ReturnReg64(InvalidReg, InvalidReg);
+#elif defined(JS_PUNBOX64)
+static constexpr ValueOperand JSReturnOperand(InvalidReg);
+static constexpr Register64 ReturnReg64(InvalidReg);
+#else
+# error "Bad architecture"
+#endif
+
+static constexpr Register ABINonArgReg0{Registers::invalid_reg};
+static constexpr Register ABINonArgReg1{Registers::invalid_reg};
+static constexpr Register ABINonArgReg2{Registers::invalid_reg};
+static constexpr Register ABINonArgReg3{Registers::invalid_reg};
+static constexpr Register ABINonArgReturnReg0{Registers::invalid_reg};
+static constexpr Register ABINonArgReturnReg1{Registers::invalid_reg};
+static constexpr Register ABINonVolatileReg{Registers::invalid_reg};
+static constexpr Register ABINonArgReturnVolatileReg{Registers::invalid_reg};
+
+static constexpr FloatRegister ABINonArgDoubleReg = {
+ FloatRegisters::invalid_reg};
+
+static constexpr Register WasmTableCallScratchReg0{Registers::invalid_reg};
+static constexpr Register WasmTableCallScratchReg1{Registers::invalid_reg};
+static constexpr Register WasmTableCallSigReg{Registers::invalid_reg};
+static constexpr Register WasmTableCallIndexReg{Registers::invalid_reg};
+static constexpr Register InstanceReg{Registers::invalid_reg};
+static constexpr Register WasmJitEntryReturnScratch{Registers::invalid_reg};
+static constexpr Register WasmCallRefCallScratchReg0{Registers::invalid_reg};
+static constexpr Register WasmCallRefCallScratchReg1{Registers::invalid_reg};
+static constexpr Register WasmCallRefReg{Registers::invalid_reg};
+
+static constexpr uint32_t ABIStackAlignment = 4;
+static constexpr uint32_t CodeAlignment = 16;
+static constexpr uint32_t JitStackAlignment = 8;
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+
+static const Scale ScalePointer = TimesOne;
+
+class Assembler : public AssemblerShared {
+ public:
+ enum Condition {
+ Equal,
+ NotEqual,
+ Above,
+ AboveOrEqual,
+ Below,
+ BelowOrEqual,
+ GreaterThan,
+ GreaterThanOrEqual,
+ LessThan,
+ LessThanOrEqual,
+ Overflow,
+ CarrySet,
+ CarryClear,
+ Signed,
+ NotSigned,
+ Zero,
+ NonZero,
+ Always,
+ };
+
+ enum DoubleCondition {
+ DoubleOrdered,
+ DoubleEqual,
+ DoubleNotEqual,
+ DoubleGreaterThan,
+ DoubleGreaterThanOrEqual,
+ DoubleLessThan,
+ DoubleLessThanOrEqual,
+ DoubleUnordered,
+ DoubleEqualOrUnordered,
+ DoubleNotEqualOrUnordered,
+ DoubleGreaterThanOrUnordered,
+ DoubleGreaterThanOrEqualOrUnordered,
+ DoubleLessThanOrUnordered,
+ DoubleLessThanOrEqualOrUnordered
+ };
+
+ static Condition InvertCondition(Condition) { MOZ_CRASH(); }
+
+ static DoubleCondition InvertCondition(DoubleCondition) { MOZ_CRASH(); }
+
+ template <typename T, typename S>
+ static void PatchDataWithValueCheck(CodeLocationLabel, T, S) {
+ MOZ_CRASH();
+ }
+ static void PatchWrite_Imm32(CodeLocationLabel, Imm32) { MOZ_CRASH(); }
+
+ static void PatchWrite_NearCall(CodeLocationLabel, CodeLocationLabel) {
+ MOZ_CRASH();
+ }
+ static uint32_t PatchWrite_NearCallSize() { MOZ_CRASH(); }
+
+ static void ToggleToJmp(CodeLocationLabel) { MOZ_CRASH(); }
+ static void ToggleToCmp(CodeLocationLabel) { MOZ_CRASH(); }
+ static void ToggleCall(CodeLocationLabel, bool) { MOZ_CRASH(); }
+
+ static void Bind(uint8_t*, const CodeLabel&) { MOZ_CRASH(); }
+
+ static uintptr_t GetPointer(uint8_t*) { MOZ_CRASH(); }
+
+ static bool HasRoundInstruction(RoundingMode) { return false; }
+
+ void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
+ const Disassembler::HeapAccess& heapAccess) {
+ MOZ_CRASH();
+ }
+
+ void setUnlimitedBuffer() { MOZ_CRASH(); }
+};
+
+class Operand {
+ public:
+ explicit Operand(const Address&) { MOZ_CRASH(); }
+ explicit Operand(const Register) { MOZ_CRASH(); }
+ explicit Operand(const FloatRegister) { MOZ_CRASH(); }
+ explicit Operand(Register, Imm32) { MOZ_CRASH(); }
+ explicit Operand(Register, int32_t) { MOZ_CRASH(); }
+};
+
+class ABIArgGenerator {
+ public:
+ ABIArgGenerator() { MOZ_CRASH(); }
+ ABIArg next(MIRType) { MOZ_CRASH(); }
+ ABIArg& current() { MOZ_CRASH(); }
+ uint32_t stackBytesConsumedSoFar() const { MOZ_CRASH(); }
+ void increaseStackOffset(uint32_t) { MOZ_CRASH(); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_Assembler_none_h */
diff --git a/js/src/jit/none/CodeGenerator-none.h b/js/src/jit/none/CodeGenerator-none.h
new file mode 100644
index 0000000000..6efd71555e
--- /dev/null
+++ b/js/src/jit/none/CodeGenerator-none.h
@@ -0,0 +1,78 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_CodeGenerator_none_h
+#define jit_none_CodeGenerator_none_h
+
+#include "jit/shared/CodeGenerator-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorNone : public CodeGeneratorShared {
+ protected:
+ CodeGeneratorNone(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
+ : CodeGeneratorShared(gen, graph, masm) {
+ MOZ_CRASH();
+ }
+
+ MoveOperand toMoveOperand(LAllocation) const { MOZ_CRASH(); }
+ template <typename T1, typename T2>
+ void bailoutCmp32(Assembler::Condition, T1, T2, LSnapshot*) {
+ MOZ_CRASH();
+ }
+ template <typename T1, typename T2>
+ void bailoutTest32(Assembler::Condition, T1, T2, LSnapshot*) {
+ MOZ_CRASH();
+ }
+ template <typename T1, typename T2>
+ void bailoutCmpPtr(Assembler::Condition, T1, T2, LSnapshot*) {
+ MOZ_CRASH();
+ }
+ void bailoutTestPtr(Assembler::Condition, Register, Register, LSnapshot*) {
+ MOZ_CRASH();
+ }
+ void bailoutIfFalseBool(Register, LSnapshot*) { MOZ_CRASH(); }
+ void bailoutFrom(Label*, LSnapshot*) { MOZ_CRASH(); }
+ void bailout(LSnapshot*) { MOZ_CRASH(); }
+ void bailoutIf(Assembler::Condition, LSnapshot*) { MOZ_CRASH(); }
+ bool generateOutOfLineCode() { MOZ_CRASH(); }
+ void testNullEmitBranch(Assembler::Condition, ValueOperand, MBasicBlock*,
+ MBasicBlock*) {
+ MOZ_CRASH();
+ }
+ void testUndefinedEmitBranch(Assembler::Condition, ValueOperand, MBasicBlock*,
+ MBasicBlock*) {
+ MOZ_CRASH();
+ }
+ void testObjectEmitBranch(Assembler::Condition, ValueOperand, MBasicBlock*,
+ MBasicBlock*) {
+ MOZ_CRASH();
+ }
+ void testZeroEmitBranch(Assembler::Condition, Register, MBasicBlock*,
+ MBasicBlock*) {
+ MOZ_CRASH();
+ }
+ void emitTableSwitchDispatch(MTableSwitch*, Register, Register) {
+ MOZ_CRASH();
+ }
+ void emitBigIntDiv(LBigIntDiv*, Register, Register, Register, Label*) {
+ MOZ_CRASH();
+ }
+ void emitBigIntMod(LBigIntMod*, Register, Register, Register, Label*) {
+ MOZ_CRASH();
+ }
+ ValueOperand ToValue(LInstruction*, size_t) { MOZ_CRASH(); }
+ ValueOperand ToTempValue(LInstruction*, size_t) { MOZ_CRASH(); }
+ void generateInvalidateEpilogue() { MOZ_CRASH(); }
+};
+
+typedef CodeGeneratorNone CodeGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_CodeGenerator_none_h */
diff --git a/js/src/jit/none/LIR-none.h b/js/src/jit/none/LIR-none.h
new file mode 100644
index 0000000000..c04578630e
--- /dev/null
+++ b/js/src/jit/none/LIR-none.h
@@ -0,0 +1,111 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_LIR_none_h
+#define jit_none_LIR_none_h
+
+namespace js {
+namespace jit {
+
+class LUnboxFloatingPoint : public LInstruction {
+ public:
+ LIR_HEADER(UnboxFloatingPoint)
+ static const size_t Input = 0;
+
+ MUnbox* mir() const { MOZ_CRASH(); }
+
+ const LDefinition* output() const { MOZ_CRASH(); }
+ MIRType type() const { MOZ_CRASH(); }
+};
+
+class LTableSwitch : public LInstruction {
+ public:
+ LIR_HEADER(TableSwitch)
+ MTableSwitch* mir() { MOZ_CRASH(); }
+
+ const LAllocation* index() { MOZ_CRASH(); }
+ const LDefinition* tempInt() { MOZ_CRASH(); }
+ const LDefinition* tempPointer() { MOZ_CRASH(); }
+};
+
+class LTableSwitchV : public LInstruction {
+ public:
+ LIR_HEADER(TableSwitchV)
+ MTableSwitch* mir() { MOZ_CRASH(); }
+
+ const LDefinition* tempInt() { MOZ_CRASH(); }
+ const LDefinition* tempFloat() { MOZ_CRASH(); }
+ const LDefinition* tempPointer() { MOZ_CRASH(); }
+
+ static const size_t InputValue = 0;
+};
+
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+ explicit LWasmUint32ToFloat32(const LAllocation&)
+ : LInstructionHelper(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+};
+
+class LUnbox : public LInstructionHelper<1, 2, 0> {
+ public:
+ MUnbox* mir() const { MOZ_CRASH(); }
+ const LAllocation* payload() { MOZ_CRASH(); }
+ const LAllocation* type() { MOZ_CRASH(); }
+ const char* extraName() const { MOZ_CRASH(); }
+};
+class LDivI : public LBinaryMath<1> {
+ public:
+ LDivI(const LAllocation&, const LAllocation&, const LDefinition&)
+ : LBinaryMath(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+ MDiv* mir() const { MOZ_CRASH(); }
+};
+class LDivPowTwoI : public LInstructionHelper<1, 1, 0> {
+ public:
+ LDivPowTwoI(const LAllocation&, int32_t)
+ : LInstructionHelper(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+ const LAllocation* numerator() { MOZ_CRASH(); }
+ int32_t shift() { MOZ_CRASH(); }
+ MDiv* mir() const { MOZ_CRASH(); }
+};
+class LModI : public LBinaryMath<1> {
+ public:
+ LModI(const LAllocation&, const LAllocation&, const LDefinition&)
+ : LBinaryMath(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+
+ const LDefinition* callTemp() { MOZ_CRASH(); }
+ MMod* mir() const { MOZ_CRASH(); }
+};
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+ explicit LWasmUint32ToDouble(const LAllocation&)
+ : LInstructionHelper(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+};
+class LModPowTwoI : public LInstructionHelper<1, 1, 0> {
+ public:
+ int32_t shift() { MOZ_CRASH(); }
+ LModPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+ MMod* mir() const { MOZ_CRASH(); }
+};
+
+class LMulI : public LInstruction {};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_LIR_none_h */
diff --git a/js/src/jit/none/Lowering-none.h b/js/src/jit/none/Lowering-none.h
new file mode 100644
index 0000000000..ad804b970b
--- /dev/null
+++ b/js/src/jit/none/Lowering-none.h
@@ -0,0 +1,130 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_Lowering_none_h
+#define jit_none_Lowering_none_h
+
+#include "jit/shared/Lowering-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorNone : public LIRGeneratorShared {
+ protected:
+ LIRGeneratorNone(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorShared(gen, graph, lirGraph) {
+ MOZ_CRASH();
+ }
+
+ LBoxAllocation useBoxFixed(MDefinition*, Register, Register,
+ bool useAtStart = false) {
+ MOZ_CRASH();
+ }
+
+ LAllocation useByteOpRegister(MDefinition*) { MOZ_CRASH(); }
+ LAllocation useByteOpRegisterAtStart(MDefinition*) { MOZ_CRASH(); }
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition*) {
+ MOZ_CRASH();
+ }
+ LDefinition tempByteOpRegister() { MOZ_CRASH(); }
+ LDefinition tempToUnbox() { MOZ_CRASH(); }
+ bool needTempForPostBarrier() { MOZ_CRASH(); }
+ void lowerUntypedPhiInput(MPhi*, uint32_t, LBlock*, size_t) { MOZ_CRASH(); }
+ void lowerInt64PhiInput(MPhi*, uint32_t, LBlock*, size_t) { MOZ_CRASH(); }
+ void defineInt64Phi(MPhi*, size_t) { MOZ_CRASH(); }
+ void lowerForShift(LInstructionHelper<1, 2, 0>*, MDefinition*, MDefinition*,
+ MDefinition*) {
+ MOZ_CRASH();
+ }
+ void lowerUrshD(MUrsh*) { MOZ_CRASH(); }
+ void lowerPowOfTwoI(MPow*) { MOZ_CRASH(); }
+ template <typename T>
+ void lowerForALU(T, MDefinition*, MDefinition*, MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void lowerForFPU(T, MDefinition*, MDefinition*, MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void lowerForALUInt64(T, MDefinition*, MDefinition*,
+ MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ void lowerForMulInt64(LMulI64*, MMul*, MDefinition*,
+ MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void lowerForShiftInt64(T, MDefinition*, MDefinition*,
+ MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ void lowerForBitAndAndBranch(LBitAndAndBranch*, MInstruction*, MDefinition*,
+ MDefinition*) {
+ MOZ_CRASH();
+ }
+ void lowerForCompareI64AndBranch(MTest*, MCompare*, JSOp, MDefinition*,
+ MDefinition*, MBasicBlock*, MBasicBlock*) {
+ MOZ_CRASH();
+ }
+
+ void lowerConstantDouble(double, MInstruction*) { MOZ_CRASH(); }
+ void lowerConstantFloat32(float, MInstruction*) { MOZ_CRASH(); }
+ void lowerTruncateDToInt32(MTruncateToInt32*) { MOZ_CRASH(); }
+ void lowerTruncateFToInt32(MTruncateToInt32*) { MOZ_CRASH(); }
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH();
+ }
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH();
+ }
+ void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins) {
+ MOZ_CRASH();
+ }
+ void lowerDivI(MDiv*) { MOZ_CRASH(); }
+ void lowerModI(MMod*) { MOZ_CRASH(); }
+ void lowerDivI64(MDiv*) { MOZ_CRASH(); }
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) { MOZ_CRASH(); }
+ void lowerModI64(MMod*) { MOZ_CRASH(); }
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) { MOZ_CRASH(); }
+ void lowerNegI(MInstruction*, MDefinition*) { MOZ_CRASH(); }
+ void lowerNegI64(MInstruction*, MDefinition*) { MOZ_CRASH(); }
+ void lowerMulI(MMul*, MDefinition*, MDefinition*) { MOZ_CRASH(); }
+ void lowerUDiv(MDiv*) { MOZ_CRASH(); }
+ void lowerUMod(MMod*) { MOZ_CRASH(); }
+ void lowerWasmSelectI(MWasmSelect* select) { MOZ_CRASH(); }
+ void lowerWasmSelectI64(MWasmSelect* select) { MOZ_CRASH(); }
+ void lowerWasmCompareAndSelect(MWasmSelect* ins, MDefinition* lhs,
+ MDefinition* rhs, MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_CRASH();
+ }
+ bool canSpecializeWasmCompareAndSelect(MCompare::CompareType compTy,
+ MIRType insTy) {
+ MOZ_CRASH();
+ }
+
+ void lowerBigIntLsh(MBigIntLsh*) { MOZ_CRASH(); }
+ void lowerBigIntRsh(MBigIntRsh*) { MOZ_CRASH(); }
+ void lowerBigIntDiv(MBigIntDiv*) { MOZ_CRASH(); }
+ void lowerBigIntMod(MBigIntMod*) { MOZ_CRASH(); }
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar*) { MOZ_CRASH(); }
+ void lowerAtomicStore64(MStoreUnboxedScalar*) { MOZ_CRASH(); }
+
+ LTableSwitch* newLTableSwitch(LAllocation, LDefinition, MTableSwitch*) {
+ MOZ_CRASH();
+ }
+ LTableSwitchV* newLTableSwitchV(MTableSwitch*) { MOZ_CRASH(); }
+};
+
+typedef LIRGeneratorNone LIRGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_Lowering_none_h */
diff --git a/js/src/jit/none/MacroAssembler-none.h b/js/src/jit/none/MacroAssembler-none.h
new file mode 100644
index 0000000000..c1514b8fd3
--- /dev/null
+++ b/js/src/jit/none/MacroAssembler-none.h
@@ -0,0 +1,458 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_MacroAssembler_none_h
+#define jit_none_MacroAssembler_none_h
+
+#include <iterator>
+
+#include "jit/MoveResolver.h"
+#include "jit/none/Assembler-none.h"
+#include "wasm/WasmTypeDecls.h"
+
+namespace js {
+namespace jit {
+
+class CompactBufferReader;
+
+class ScratchTagScope {
+ public:
+ ScratchTagScope(MacroAssembler&, const ValueOperand) {}
+ operator Register() { MOZ_CRASH(); }
+ void release() { MOZ_CRASH(); }
+ void reacquire() { MOZ_CRASH(); }
+};
+
+class ScratchTagScopeRelease {
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope*) {}
+};
+
+class MacroAssemblerNone : public Assembler {
+ public:
+ MacroAssemblerNone() { MOZ_CRASH(); }
+
+ MoveResolver moveResolver_;
+
+ size_t size() const { MOZ_CRASH(); }
+ size_t bytesNeeded() const { MOZ_CRASH(); }
+ size_t jumpRelocationTableBytes() const { MOZ_CRASH(); }
+ size_t dataRelocationTableBytes() const { MOZ_CRASH(); }
+ size_t preBarrierTableBytes() const { MOZ_CRASH(); }
+
+ size_t numCodeLabels() const { MOZ_CRASH(); }
+ CodeLabel codeLabel(size_t) { MOZ_CRASH(); }
+
+ bool reserve(size_t size) { MOZ_CRASH(); }
+ bool appendRawCode(const uint8_t* code, size_t numBytes) { MOZ_CRASH(); }
+ bool swapBuffer(wasm::Bytes& bytes) { MOZ_CRASH(); }
+
+ void assertNoGCThings() const { MOZ_CRASH(); }
+
+ static void TraceJumpRelocations(JSTracer*, JitCode*, CompactBufferReader&) {
+ MOZ_CRASH();
+ }
+ static void TraceDataRelocations(JSTracer*, JitCode*, CompactBufferReader&) {
+ MOZ_CRASH();
+ }
+
+ static bool SupportsFloatingPoint() { return false; }
+ static bool SupportsUnalignedAccesses() { return false; }
+ static bool SupportsFastUnalignedFPAccesses() { return false; }
+
+ void executableCopy(void*, bool = true) { MOZ_CRASH(); }
+ void copyJumpRelocationTable(uint8_t*) { MOZ_CRASH(); }
+ void copyDataRelocationTable(uint8_t*) { MOZ_CRASH(); }
+ void copyPreBarrierTable(uint8_t*) { MOZ_CRASH(); }
+ void processCodeLabels(uint8_t*) { MOZ_CRASH(); }
+
+ void flushBuffer() { MOZ_CRASH(); }
+
+ template <typename T>
+ void bind(T) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void j(Condition, T) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void jump(T) {
+ MOZ_CRASH();
+ }
+ void writeCodePointer(CodeLabel* label) { MOZ_CRASH(); }
+ void haltingAlign(size_t) { MOZ_CRASH(); }
+ void nopAlign(size_t) { MOZ_CRASH(); }
+ void checkStackAlignment() { MOZ_CRASH(); }
+ uint32_t currentOffset() { MOZ_CRASH(); }
+
+ void nop() { MOZ_CRASH(); }
+ void breakpoint() { MOZ_CRASH(); }
+ void abiret() { MOZ_CRASH(); }
+ void ret() { MOZ_CRASH(); }
+
+ CodeOffset toggledJump(Label*) { MOZ_CRASH(); }
+ CodeOffset toggledCall(JitCode*, bool) { MOZ_CRASH(); }
+ static size_t ToggledCallSize(uint8_t*) { MOZ_CRASH(); }
+
+ void finish() { MOZ_CRASH(); }
+
+ template <typename T, typename S>
+ void moveValue(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S, typename U>
+ void moveValue(T, S, U) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void storeValue(const T&, const S&) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S, typename U>
+ void storeValue(T, S, U) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void storePrivateValue(const T&, const S&) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void loadValue(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void loadUnalignedValue(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void pushValue(const T&) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void pushValue(T, S) {
+ MOZ_CRASH();
+ }
+ void popValue(ValueOperand) { MOZ_CRASH(); }
+ void tagValue(JSValueType, Register, ValueOperand) { MOZ_CRASH(); }
+ void retn(Imm32 n) { MOZ_CRASH(); }
+ template <typename T>
+ void push(const T&) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void Push(T) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void pop(T) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void Pop(T) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ CodeOffset pushWithPatch(T) {
+ MOZ_CRASH();
+ }
+
+ void testNullSet(Condition, ValueOperand, Register) { MOZ_CRASH(); }
+ void testObjectSet(Condition, ValueOperand, Register) { MOZ_CRASH(); }
+ void testUndefinedSet(Condition, ValueOperand, Register) { MOZ_CRASH(); }
+
+ template <typename T, typename S>
+ void cmpPtrSet(Condition, T, S, Register) {
+ MOZ_CRASH();
+ }
+ void cmp8Set(Condition, Address, Imm32, Register) { MOZ_CRASH(); }
+ void cmp16Set(Condition, Address, Imm32, Register) { MOZ_CRASH(); }
+ template <typename T, typename S>
+ void cmp32Set(Condition, T, S, Register) {
+ MOZ_CRASH();
+ }
+ void cmp64Set(Condition, Address, Imm64, Register) { MOZ_CRASH(); }
+
+ template <typename T>
+ void mov(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void movePtr(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void move32(const T&, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void movq(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void moveFloat32(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void moveDouble(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void move64(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ CodeOffset movWithPatch(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void loadPtr(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load32(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load32Unaligned(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void loadFloat32(T, FloatRegister) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void loadDouble(T, FloatRegister) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void loadPrivate(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load8SignExtend(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load8ZeroExtend(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load16SignExtend(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load16UnalignedSignExtend(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load16ZeroExtend(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load16UnalignedZeroExtend(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load64(T, Register64) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void load64Unaligned(T, Register64) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void storePtr(const T&, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void store32(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void store32_NoSecondScratch(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void store32Unaligned(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void storeFloat32(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void storeDouble(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void store8(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void store16(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void store16Unaligned(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void store64(T, S) {
+ MOZ_CRASH();
+ }
+ template <typename T, typename S>
+ void store64Unaligned(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void computeEffectiveAddress(T, Register) {
+ MOZ_CRASH();
+ }
+
+ void splitTagForTest(ValueOperand, ScratchTagScope&) { MOZ_CRASH(); }
+
+ void boxDouble(FloatRegister, ValueOperand, FloatRegister) { MOZ_CRASH(); }
+ void boxNonDouble(JSValueType, Register, ValueOperand) { MOZ_CRASH(); }
+ template <typename T>
+ void boxDouble(FloatRegister src, const T& dest) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void unboxInt32(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void unboxBoolean(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void unboxString(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void unboxSymbol(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void unboxBigInt(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void unboxObject(T, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void unboxDouble(T, FloatRegister) {
+ MOZ_CRASH();
+ }
+ void unboxValue(const ValueOperand&, AnyRegister, JSValueType) {
+ MOZ_CRASH();
+ }
+ void unboxNonDouble(const ValueOperand&, Register, JSValueType) {
+ MOZ_CRASH();
+ }
+ void unboxNonDouble(const Address&, Register, JSValueType) { MOZ_CRASH(); }
+ template <typename T>
+ void unboxGCThingForGCBarrier(const T&, Register) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ MOZ_CRASH();
+ }
+ void notBoolean(ValueOperand) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractObject(Address, Register) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractObject(ValueOperand, Register) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractSymbol(ValueOperand, Register) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractInt32(ValueOperand, Register) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractBoolean(ValueOperand, Register) { MOZ_CRASH(); }
+ template <typename T>
+ [[nodiscard]] Register extractTag(T, Register) {
+ MOZ_CRASH();
+ }
+
+ void convertFloat32ToInt32(FloatRegister, Register, Label*, bool v = true) {
+ MOZ_CRASH();
+ }
+ void convertDoubleToInt32(FloatRegister, Register, Label*, bool v = true) {
+ MOZ_CRASH();
+ }
+ void convertDoubleToPtr(FloatRegister, Register, Label*, bool v = true) {
+ MOZ_CRASH();
+ }
+ void convertBoolToInt32(Register, Register) { MOZ_CRASH(); }
+
+ void convertDoubleToFloat32(FloatRegister, FloatRegister) { MOZ_CRASH(); }
+ void convertInt32ToFloat32(Register, FloatRegister) { MOZ_CRASH(); }
+
+ template <typename T>
+ void convertInt32ToDouble(T, FloatRegister) {
+ MOZ_CRASH();
+ }
+ void convertFloat32ToDouble(FloatRegister, FloatRegister) { MOZ_CRASH(); }
+
+ void boolValueToDouble(ValueOperand, FloatRegister) { MOZ_CRASH(); }
+ void boolValueToFloat32(ValueOperand, FloatRegister) { MOZ_CRASH(); }
+ void int32ValueToDouble(ValueOperand, FloatRegister) { MOZ_CRASH(); }
+ void int32ValueToFloat32(ValueOperand, FloatRegister) { MOZ_CRASH(); }
+
+ void loadConstantDouble(double, FloatRegister) { MOZ_CRASH(); }
+ void loadConstantFloat32(float, FloatRegister) { MOZ_CRASH(); }
+ Condition testInt32Truthy(bool, ValueOperand) { MOZ_CRASH(); }
+ Condition testStringTruthy(bool, ValueOperand) { MOZ_CRASH(); }
+ Condition testBigIntTruthy(bool, ValueOperand) { MOZ_CRASH(); }
+
+ template <typename T>
+ void loadUnboxedValue(T, MIRType, AnyRegister) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void storeUnboxedValue(const ConstantOrRegister&, MIRType, T) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T, size_t, JSValueType) {
+ MOZ_CRASH();
+ }
+
+ void convertUInt32ToDouble(Register, FloatRegister) { MOZ_CRASH(); }
+ void convertUInt32ToFloat32(Register, FloatRegister) { MOZ_CRASH(); }
+ void incrementInt32Value(Address) { MOZ_CRASH(); }
+ void ensureDouble(ValueOperand, FloatRegister, Label*) { MOZ_CRASH(); }
+ void handleFailureWithHandlerTail(Label*, Label*) { MOZ_CRASH(); }
+
+ void buildFakeExitFrame(Register, uint32_t*) { MOZ_CRASH(); }
+ bool buildOOLFakeExitFrame(void*) { MOZ_CRASH(); }
+
+ void setPrinter(Sprinter*) { MOZ_CRASH(); }
+ Operand ToPayload(Operand base) { MOZ_CRASH(); }
+ Address ToPayload(Address) { MOZ_CRASH(); }
+
+ Register getStackPointer() const { MOZ_CRASH(); }
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register, Register) { MOZ_CRASH(); }
+ void profilerExitFrame() { MOZ_CRASH(); }
+
+#ifdef JS_NUNBOX32
+ Address ToType(Address) { MOZ_CRASH(); }
+#endif
+};
+
+typedef MacroAssemblerNone MacroAssemblerSpecific;
+
+static inline bool GetTempRegForIntArg(uint32_t, uint32_t, Register*) {
+ MOZ_CRASH();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_MacroAssembler_none_h */
diff --git a/js/src/jit/none/MoveEmitter-none.h b/js/src/jit/none/MoveEmitter-none.h
new file mode 100644
index 0000000000..39e60bfaee
--- /dev/null
+++ b/js/src/jit/none/MoveEmitter-none.h
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_MoveEmitter_none_h
+#define jit_none_MoveEmitter_none_h
+
+#include "mozilla/Assertions.h"
+
+namespace js {
+namespace jit {
+
+class MacroAssemblerNone;
+class MoveResolver;
+struct Register;
+
+class MoveEmitterNone {
+ public:
+ explicit MoveEmitterNone(MacroAssemblerNone&) { MOZ_CRASH(); }
+ void emit(const MoveResolver&) { MOZ_CRASH(); }
+ void finish() { MOZ_CRASH(); }
+ void setScratchRegister(Register) { MOZ_CRASH(); }
+};
+
+typedef MoveEmitterNone MoveEmitter;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_MoveEmitter_none_h */
diff --git a/js/src/jit/none/SharedICHelpers-none-inl.h b/js/src/jit/none/SharedICHelpers-none-inl.h
new file mode 100644
index 0000000000..2d63956ba7
--- /dev/null
+++ b/js/src/jit/none/SharedICHelpers-none-inl.h
@@ -0,0 +1,31 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_SharedICHelpers_none_inl_h
+#define jit_none_SharedICHelpers_none_inl_h
+
+#include "jit/SharedICHelpers.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr, MacroAssembler&, uint32_t) {
+ MOZ_CRASH();
+}
+inline void EmitBaselineCreateStubFrameDescriptor(MacroAssembler&, Register,
+ uint32_t) {
+ MOZ_CRASH();
+}
+inline void EmitBaselineCallVM(TrampolinePtr, MacroAssembler&) { MOZ_CRASH(); }
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler&, Register) {
+ MOZ_CRASH();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_SharedICHelpers_none_inl_h */
diff --git a/js/src/jit/none/SharedICHelpers-none.h b/js/src/jit/none/SharedICHelpers-none.h
new file mode 100644
index 0000000000..8c2b4ee396
--- /dev/null
+++ b/js/src/jit/none/SharedICHelpers-none.h
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_SharedICHelpers_none_h
+#define jit_none_SharedICHelpers_none_h
+
+namespace js {
+namespace jit {
+
+static const size_t ICStackValueOffset = 0;
+
+inline void EmitRestoreTailCallReg(MacroAssembler&) { MOZ_CRASH(); }
+inline void EmitRepushTailCallReg(MacroAssembler&) { MOZ_CRASH(); }
+inline void EmitCallIC(MacroAssembler&, CodeOffset*) { MOZ_CRASH(); }
+inline void EmitReturnFromIC(MacroAssembler&) { MOZ_CRASH(); }
+inline void EmitBaselineLeaveStubFrame(MacroAssembler&, bool v = false) {
+ MOZ_CRASH();
+}
+inline void EmitStubGuardFailure(MacroAssembler&) { MOZ_CRASH(); }
+
+template <typename T>
+inline void EmitPreBarrier(MacroAssembler&, T, MIRType) {
+ MOZ_CRASH();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_SharedICHelpers_none_h */
diff --git a/js/src/jit/none/SharedICRegisters-none.h b/js/src/jit/none/SharedICRegisters-none.h
new file mode 100644
index 0000000000..170e5058a9
--- /dev/null
+++ b/js/src/jit/none/SharedICRegisters-none.h
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_none_SharedICRegisters_none_h
+#define jit_none_SharedICRegisters_none_h
+
+#include "jit/none/MacroAssembler-none.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+
+namespace js {
+namespace jit {
+
+static constexpr ValueOperand R0 = JSReturnOperand;
+static constexpr ValueOperand R1 = JSReturnOperand;
+static constexpr ValueOperand R2 = JSReturnOperand;
+
+static constexpr Register ICTailCallReg{Registers::invalid_reg};
+static constexpr Register ICStubReg{Registers::invalid_reg};
+
+static constexpr FloatRegister FloatReg0 = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister FloatReg1 = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister FloatReg2 = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister FloatReg3 = {FloatRegisters::invalid_reg};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_none_SharedICRegisters_none_h */
diff --git a/js/src/jit/none/Trampoline-none.cpp b/js/src/jit/none/Trampoline-none.cpp
new file mode 100644
index 0000000000..d2ee3a9a05
--- /dev/null
+++ b/js/src/jit/none/Trampoline-none.cpp
@@ -0,0 +1,43 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineIC.h"
+#include "jit/JitRuntime.h"
+#include "vm/Realm.h"
+
+using namespace js;
+using namespace js::jit;
+
+// This file includes stubs for generating the JIT trampolines when there is no
+// JIT backend, and also includes implementations for assorted random things
+// which can't be implemented in headers.
+
+void JitRuntime::generateEnterJIT(JSContext*, MacroAssembler&) { MOZ_CRASH(); }
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ return mozilla::Nothing{};
+}
+void JitRuntime::generateInvalidator(MacroAssembler&, Label*) { MOZ_CRASH(); }
+void JitRuntime::generateArgumentsRectifier(MacroAssembler&,
+ ArgumentsRectifierKind kind) {
+ MOZ_CRASH();
+}
+void JitRuntime::generateBailoutHandler(MacroAssembler&, Label*) {
+ MOZ_CRASH();
+}
+uint32_t JitRuntime::generatePreBarrier(JSContext*, MacroAssembler&, MIRType) {
+ MOZ_CRASH();
+}
+void JitRuntime::generateBailoutTailStub(MacroAssembler&, Label*) {
+ MOZ_CRASH();
+}
+
+bool JitRuntime::generateVMWrapper(JSContext*, MacroAssembler&,
+ const VMFunctionData&, DynFn, uint32_t*) {
+ MOZ_CRASH();
+}
diff --git a/js/src/jit/shared/Architecture-shared.h b/js/src/jit/shared/Architecture-shared.h
new file mode 100644
index 0000000000..33085a6bdb
--- /dev/null
+++ b/js/src/jit/shared/Architecture-shared.h
@@ -0,0 +1,18 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_Architecture_shared_h
+#define jit_shared_Architecture_shared_h
+
+namespace js {
+namespace jit {
+
+enum class RegTypeName { GPR, Float32, Float64, Vector128, Any };
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_shared_Architecture_shared_h */
diff --git a/js/src/jit/shared/Assembler-shared.cpp b/js/src/jit/shared/Assembler-shared.cpp
new file mode 100644
index 0000000000..96dcf69a72
--- /dev/null
+++ b/js/src/jit/shared/Assembler-shared.cpp
@@ -0,0 +1,74 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/shared/Assembler-shared.h"
+
+#include "jit/JitSpewer.h"
+#include "vm/NativeObject.h"
+
+namespace js::jit {
+
+void BaseObjectElementIndex::staticAssertions() {
+ NativeObject::elementsSizeMustNotOverflow();
+}
+
+void BaseObjectSlotIndex::staticAssertions() {
+ NativeObject::slotsSizeMustNotOverflow();
+}
+
+AssemblerShared::~AssemblerShared() {
+#ifdef DEBUG
+ while (hasCreator()) {
+ popCreator();
+ }
+#endif
+}
+
+#ifdef DEBUG
+void AssemblerShared::pushCreator(const char* who) {
+ (void)creators_.append(who);
+ JitSpewStart(JitSpew_Codegen, "# BEGIN creators: ");
+ bool first = true;
+ for (const char* str : creators_) {
+ JitSpewCont(JitSpew_Codegen, "%s%s", first ? "" : "/", str);
+ first = false;
+ }
+ JitSpewCont(JitSpew_Codegen, "\n");
+}
+
+void AssemblerShared::popCreator() {
+ JitSpewStart(JitSpew_Codegen, "# END creators: ");
+ bool first = true;
+ for (const char* str : creators_) {
+ JitSpewCont(JitSpew_Codegen, "%s%s", first ? "" : "/", str);
+ first = false;
+ }
+ JitSpewCont(JitSpew_Codegen, "\n");
+ if (creators_.empty()) {
+ JitSpew(JitSpew_Codegen, " ");
+ }
+ MOZ_ASSERT(!creators_.empty());
+ creators_.popBack();
+}
+
+bool AssemblerShared::hasCreator() const {
+ // If you get failures of assertions of the form `MOZ_ASSERT(hasCreator())`,
+ // what this means is that a `MacroAssembler` (or, really, anything that
+ // inherits from `js::jit::AssemblerShared`) has emitted code or data from a
+ // place, in the SM C++ hierarchy, that is not nested within an
+ // `AutoCreatedBy` RAII scope. Consequently the emitted instructions/data
+ // won't have any owner that is identifiable in the `IONFLAGS=codegen`
+ // output.
+ //
+ // Fixing this is easy: work back up the crash stack and decide on a place
+ // to put an `AutoCreatedBy` call. A bit of grepping for `AutoCreatedBy`
+ // should make it obvious what to do. If in doubt, add `AutoCreatedBy`
+ // calls liberally; "extra" ones are harmless.
+ return !creators_.empty();
+}
+#endif
+
+} // namespace js::jit
diff --git a/js/src/jit/shared/Assembler-shared.h b/js/src/jit/shared/Assembler-shared.h
new file mode 100644
index 0000000000..5a478f1e16
--- /dev/null
+++ b/js/src/jit/shared/Assembler-shared.h
@@ -0,0 +1,709 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_Assembler_shared_h
+#define jit_shared_Assembler_shared_h
+
+#include "mozilla/CheckedInt.h"
+
+#include <limits.h>
+
+#include "gc/Barrier.h"
+#include "jit/AtomicOp.h"
+#include "jit/JitAllocPolicy.h"
+#include "jit/JitCode.h"
+#include "jit/JitContext.h"
+#include "jit/Label.h"
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "vm/HelperThreads.h"
+#include "wasm/WasmCodegenTypes.h"
+#include "wasm/WasmConstants.h"
+
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \
+ defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_WASM32)
+// Push return addresses callee-side.
+# define JS_USE_LINK_REGISTER
+#endif
+
+#if defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_ARM64) || defined(JS_CODEGEN_LOONG64)
+// JS_CODELABEL_LINKMODE gives labels additional metadata
+// describing how Bind() should patch them.
+# define JS_CODELABEL_LINKMODE
+#endif
+
+namespace js {
+namespace jit {
+
+enum class FrameType;
+enum class ExceptionResumeKind : int32_t;
+
+namespace Disassembler {
+class HeapAccess;
+} // namespace Disassembler
+
+static constexpr uint32_t Simd128DataSize = 4 * sizeof(int32_t);
+static_assert(Simd128DataSize == 4 * sizeof(int32_t),
+ "SIMD data should be able to contain int32x4");
+static_assert(Simd128DataSize == 4 * sizeof(float),
+ "SIMD data should be able to contain float32x4");
+static_assert(Simd128DataSize == 2 * sizeof(double),
+ "SIMD data should be able to contain float64x2");
+
+enum Scale { TimesOne = 0, TimesTwo = 1, TimesFour = 2, TimesEight = 3 };
+
+static_assert(sizeof(JS::Value) == 8,
+ "required for TimesEight and 3 below to be correct");
+static const Scale ValueScale = TimesEight;
+static const size_t ValueShift = 3;
+
+static inline unsigned ScaleToShift(Scale scale) { return unsigned(scale); }
+
+static inline bool IsShiftInScaleRange(int i) {
+ return i >= TimesOne && i <= TimesEight;
+}
+
+static inline Scale ShiftToScale(int i) {
+ MOZ_ASSERT(IsShiftInScaleRange(i));
+ return Scale(i);
+}
+
+static inline Scale ScaleFromElemWidth(int shift) {
+ switch (shift) {
+ case 1:
+ return TimesOne;
+ case 2:
+ return TimesTwo;
+ case 4:
+ return TimesFour;
+ case 8:
+ return TimesEight;
+ }
+
+ MOZ_CRASH("Invalid scale");
+}
+
+static inline Scale ScaleFromScalarType(Scalar::Type type) {
+ return ScaleFromElemWidth(Scalar::byteSize(type));
+}
+
+// Used for 32-bit immediates which do not require relocation.
+struct Imm32 {
+ int32_t value;
+
+ explicit Imm32(int32_t value) : value(value) {}
+ explicit Imm32(FrameType type) : Imm32(int32_t(type)) {}
+ explicit Imm32(ExceptionResumeKind kind) : Imm32(int32_t(kind)) {}
+
+ static inline Imm32 ShiftOf(enum Scale s) {
+ switch (s) {
+ case TimesOne:
+ return Imm32(0);
+ case TimesTwo:
+ return Imm32(1);
+ case TimesFour:
+ return Imm32(2);
+ case TimesEight:
+ return Imm32(3);
+ };
+ MOZ_CRASH("Invalid scale");
+ }
+
+ static inline Imm32 FactorOf(enum Scale s) {
+ return Imm32(1 << ShiftOf(s).value);
+ }
+};
+
+// Pointer-sized integer to be embedded as an immediate in an instruction.
+struct ImmWord {
+ uintptr_t value;
+
+ explicit ImmWord(uintptr_t value) : value(value) {}
+};
+
+// Used for 64-bit immediates which do not require relocation.
+struct Imm64 {
+ uint64_t value;
+
+ explicit Imm64(int64_t value) : value(value) {}
+
+ Imm32 low() const { return Imm32(int32_t(value)); }
+
+ Imm32 hi() const { return Imm32(int32_t(value >> 32)); }
+
+ inline Imm32 firstHalf() const;
+ inline Imm32 secondHalf() const;
+};
+
+#ifdef DEBUG
+static inline bool IsCompilingWasm() {
+ return GetJitContext()->isCompilingWasm();
+}
+#endif
+
+// Pointer to be embedded as an immediate in an instruction.
+struct ImmPtr {
+ void* value;
+
+ struct NoCheckToken {};
+
+ explicit ImmPtr(void* value, NoCheckToken) : value(value) {
+ // A special unchecked variant for contexts where we know it is safe to
+ // use an immptr. This is assuming the caller knows what they're doing.
+ }
+
+ explicit ImmPtr(const void* value) : value(const_cast<void*>(value)) {
+ // To make code serialization-safe, wasm compilation should only
+ // compile pointer immediates using a SymbolicAddress.
+ MOZ_ASSERT(!IsCompilingWasm());
+ }
+
+ template <class R>
+ explicit ImmPtr(R (*pf)()) : value(JS_FUNC_TO_DATA_PTR(void*, pf)) {
+ MOZ_ASSERT(!IsCompilingWasm());
+ }
+
+ template <class R, class A1>
+ explicit ImmPtr(R (*pf)(A1)) : value(JS_FUNC_TO_DATA_PTR(void*, pf)) {
+ MOZ_ASSERT(!IsCompilingWasm());
+ }
+
+ template <class R, class A1, class A2>
+ explicit ImmPtr(R (*pf)(A1, A2)) : value(JS_FUNC_TO_DATA_PTR(void*, pf)) {
+ MOZ_ASSERT(!IsCompilingWasm());
+ }
+
+ template <class R, class A1, class A2, class A3>
+ explicit ImmPtr(R (*pf)(A1, A2, A3)) : value(JS_FUNC_TO_DATA_PTR(void*, pf)) {
+ MOZ_ASSERT(!IsCompilingWasm());
+ }
+
+ template <class R, class A1, class A2, class A3, class A4>
+ explicit ImmPtr(R (*pf)(A1, A2, A3, A4))
+ : value(JS_FUNC_TO_DATA_PTR(void*, pf)) {
+ MOZ_ASSERT(!IsCompilingWasm());
+ }
+};
+
+// The same as ImmPtr except that the intention is to patch this
+// instruction. The initial value of the immediate is 'addr' and this value is
+// either clobbered or used in the patching process.
+struct PatchedImmPtr {
+ void* value;
+
+ explicit PatchedImmPtr() : value(nullptr) {}
+ explicit PatchedImmPtr(const void* value) : value(const_cast<void*>(value)) {}
+};
+
+class AssemblerShared;
+class ImmGCPtr;
+
+// Used for immediates which require relocation.
+class ImmGCPtr {
+ public:
+ const gc::Cell* value;
+
+ explicit ImmGCPtr(const gc::Cell* ptr) : value(ptr) {
+ // Nursery pointers can't be used if the main thread might be currently
+ // performing a minor GC.
+ MOZ_ASSERT_IF(ptr && !ptr->isTenured(), !CurrentThreadIsIonCompiling());
+
+ // wasm shouldn't be creating GC things
+ MOZ_ASSERT(!IsCompilingWasm());
+ }
+
+ private:
+ ImmGCPtr() : value(0) {}
+};
+
+// Pointer to trampoline code. Trampoline code is kept alive until the runtime
+// is destroyed, so does not need to be traced.
+struct TrampolinePtr {
+ uint8_t* value;
+
+ TrampolinePtr() : value(nullptr) {}
+ explicit TrampolinePtr(uint8_t* value) : value(value) { MOZ_ASSERT(value); }
+};
+
+// Pointer to be embedded as an immediate that is loaded/stored from by an
+// instruction.
+struct AbsoluteAddress {
+ void* addr;
+
+ explicit AbsoluteAddress(const void* addr) : addr(const_cast<void*>(addr)) {
+ MOZ_ASSERT(!IsCompilingWasm());
+ }
+
+ AbsoluteAddress offset(ptrdiff_t delta) {
+ return AbsoluteAddress(((uint8_t*)addr) + delta);
+ }
+};
+
+// The same as AbsoluteAddress except that the intention is to patch this
+// instruction. The initial value of the immediate is 'addr' and this value is
+// either clobbered or used in the patching process.
+struct PatchedAbsoluteAddress {
+ void* addr;
+
+ explicit PatchedAbsoluteAddress() : addr(nullptr) {}
+ explicit PatchedAbsoluteAddress(const void* addr)
+ : addr(const_cast<void*>(addr)) {}
+ explicit PatchedAbsoluteAddress(uintptr_t addr)
+ : addr(reinterpret_cast<void*>(addr)) {}
+};
+
+// Specifies an address computed in the form of a register base and a constant,
+// 32-bit offset.
+struct Address {
+ RegisterOrSP base;
+ int32_t offset;
+
+ Address(Register base, int32_t offset)
+ : base(RegisterOrSP(base)), offset(offset) {}
+
+#ifdef JS_HAS_HIDDEN_SP
+ Address(RegisterOrSP base, int32_t offset) : base(base), offset(offset) {}
+#endif
+
+ Address() = delete;
+};
+
+#if JS_BITS_PER_WORD == 32
+
+static inline Address LowWord(const Address& address) {
+ using mozilla::CheckedInt;
+
+ CheckedInt<int32_t> offset =
+ CheckedInt<int32_t>(address.offset) + INT64LOW_OFFSET;
+ MOZ_ALWAYS_TRUE(offset.isValid());
+ return Address(address.base, offset.value());
+}
+
+static inline Address HighWord(const Address& address) {
+ using mozilla::CheckedInt;
+
+ CheckedInt<int32_t> offset =
+ CheckedInt<int32_t>(address.offset) + INT64HIGH_OFFSET;
+ MOZ_ALWAYS_TRUE(offset.isValid());
+ return Address(address.base, offset.value());
+}
+
+#endif
+
+// Specifies an address computed in the form of a register base, a register
+// index with a scale, and a constant, 32-bit offset.
+struct BaseIndex {
+ RegisterOrSP base;
+ Register index;
+ Scale scale;
+ int32_t offset;
+
+ BaseIndex(Register base, Register index, Scale scale, int32_t offset = 0)
+ : base(RegisterOrSP(base)), index(index), scale(scale), offset(offset) {}
+
+#ifdef JS_HAS_HIDDEN_SP
+ BaseIndex(RegisterOrSP base, Register index, Scale scale, int32_t offset = 0)
+ : base(base), index(index), scale(scale), offset(offset) {}
+#endif
+
+ BaseIndex() = delete;
+};
+
+#if JS_BITS_PER_WORD == 32
+
+static inline BaseIndex LowWord(const BaseIndex& address) {
+ using mozilla::CheckedInt;
+
+ CheckedInt<int32_t> offset =
+ CheckedInt<int32_t>(address.offset) + INT64LOW_OFFSET;
+ MOZ_ALWAYS_TRUE(offset.isValid());
+ return BaseIndex(address.base, address.index, address.scale, offset.value());
+}
+
+static inline BaseIndex HighWord(const BaseIndex& address) {
+ using mozilla::CheckedInt;
+
+ CheckedInt<int32_t> offset =
+ CheckedInt<int32_t>(address.offset) + INT64HIGH_OFFSET;
+ MOZ_ALWAYS_TRUE(offset.isValid());
+ return BaseIndex(address.base, address.index, address.scale, offset.value());
+}
+
+#endif
+
+// A BaseIndex used to access Values. Note that |offset| is *not* scaled by
+// sizeof(Value). Use this *only* if you're indexing into a series of Values
+// that aren't object elements or object slots (for example, values on the
+// stack, values in an arguments object, &c.). If you're indexing into an
+// object's elements or slots, don't use this directly! Use
+// BaseObject{Element,Slot}Index instead.
+struct BaseValueIndex : BaseIndex {
+ BaseValueIndex(Register base, Register index, int32_t offset = 0)
+ : BaseIndex(RegisterOrSP(base), index, ValueScale, offset) {}
+
+#ifdef JS_HAS_HIDDEN_SP
+ BaseValueIndex(RegisterOrSP base, Register index, int32_t offset = 0)
+ : BaseIndex(base, index, ValueScale, offset) {}
+#endif
+};
+
+// Specifies the address of an indexed Value within object elements from a
+// base. The index must not already be scaled by sizeof(Value)!
+struct BaseObjectElementIndex : BaseValueIndex {
+ BaseObjectElementIndex(Register base, Register index, int32_t offset = 0)
+ : BaseValueIndex(base, index, offset) {}
+
+#ifdef JS_HAS_HIDDEN_SP
+ BaseObjectElementIndex(RegisterOrSP base, Register index, int32_t offset = 0)
+ : BaseValueIndex(base, index, offset) {}
+#endif
+
+ static void staticAssertions();
+};
+
+// Like BaseObjectElementIndex, except for object slots.
+struct BaseObjectSlotIndex : BaseValueIndex {
+ BaseObjectSlotIndex(Register base, Register index)
+ : BaseValueIndex(base, index) {}
+
+#ifdef JS_HAS_HIDDEN_SP
+ BaseObjectSlotIndex(RegisterOrSP base, Register index)
+ : BaseValueIndex(base, index) {}
+#endif
+
+ static void staticAssertions();
+};
+
+enum class RelocationKind {
+ // The target is immovable, so patching is only needed if the source
+ // buffer is relocated and the reference is relative.
+ HARDCODED,
+
+ // The target is the start of a JitCode buffer, which must be traced
+ // during garbage collection. Relocations and patching may be needed.
+ JITCODE
+};
+
+class CodeOffset {
+ size_t offset_;
+
+ static const size_t NOT_BOUND = size_t(-1);
+
+ public:
+ explicit CodeOffset(size_t offset) : offset_(offset) {}
+ CodeOffset() : offset_(NOT_BOUND) {}
+
+ size_t offset() const {
+ MOZ_ASSERT(bound());
+ return offset_;
+ }
+
+ void bind(size_t offset) {
+ MOZ_ASSERT(!bound());
+ offset_ = offset;
+ MOZ_ASSERT(bound());
+ }
+ bool bound() const { return offset_ != NOT_BOUND; }
+
+ void offsetBy(size_t delta) {
+ MOZ_ASSERT(bound());
+ MOZ_ASSERT(offset_ + delta >= offset_, "no overflow");
+ offset_ += delta;
+ }
+};
+
+// A code label contains an absolute reference to a point in the code. Thus, it
+// cannot be patched until after linking.
+// When the source label is resolved into a memory address, this address is
+// patched into the destination address.
+// Some need to distinguish between multiple ways of patching that address.
+// See JS_CODELABEL_LINKMODE.
+class CodeLabel {
+ // The destination position, where the absolute reference should get
+ // patched into.
+ CodeOffset patchAt_;
+
+ // The source label (relative) in the code to where the destination should
+ // get patched to.
+ CodeOffset target_;
+
+#ifdef JS_CODELABEL_LINKMODE
+ public:
+ enum LinkMode { Uninitialized = 0, RawPointer, MoveImmediate, JumpImmediate };
+
+ private:
+ LinkMode linkMode_ = Uninitialized;
+#endif
+
+ public:
+ CodeLabel() = default;
+ explicit CodeLabel(const CodeOffset& patchAt) : patchAt_(patchAt) {}
+ CodeLabel(const CodeOffset& patchAt, const CodeOffset& target)
+ : patchAt_(patchAt), target_(target) {}
+ CodeOffset* patchAt() { return &patchAt_; }
+ CodeOffset* target() { return &target_; }
+ CodeOffset patchAt() const { return patchAt_; }
+ CodeOffset target() const { return target_; }
+#ifdef JS_CODELABEL_LINKMODE
+ LinkMode linkMode() const { return linkMode_; }
+ void setLinkMode(LinkMode value) { linkMode_ = value; }
+#endif
+};
+
+typedef Vector<CodeLabel, 0, SystemAllocPolicy> CodeLabelVector;
+
+class CodeLocationLabel {
+ uint8_t* raw_ = nullptr;
+
+ public:
+ CodeLocationLabel(JitCode* code, CodeOffset base) {
+ MOZ_ASSERT(base.offset() < code->instructionsSize());
+ raw_ = code->raw() + base.offset();
+ }
+ explicit CodeLocationLabel(JitCode* code) { raw_ = code->raw(); }
+ explicit CodeLocationLabel(uint8_t* raw) {
+ MOZ_ASSERT(raw);
+ raw_ = raw;
+ }
+
+ ptrdiff_t operator-(const CodeLocationLabel& other) {
+ return raw_ - other.raw_;
+ }
+
+ uint8_t* raw() const { return raw_; }
+};
+
+} // namespace jit
+
+namespace wasm {
+
+// Represents an instruction to be patched and the intended pointee. These
+// links are accumulated in the MacroAssembler, but patching is done outside
+// the MacroAssembler (in Module::staticallyLink).
+
+struct SymbolicAccess {
+ SymbolicAccess(jit::CodeOffset patchAt, SymbolicAddress target)
+ : patchAt(patchAt), target(target) {}
+
+ jit::CodeOffset patchAt;
+ SymbolicAddress target;
+};
+
+typedef Vector<SymbolicAccess, 0, SystemAllocPolicy> SymbolicAccessVector;
+
+// Describes a single wasm or asm.js memory access for the purpose of generating
+// code and metadata.
+
+class MemoryAccessDesc {
+ uint64_t offset64_;
+ uint32_t align_;
+ Scalar::Type type_;
+ jit::Synchronization sync_;
+ wasm::BytecodeOffset trapOffset_;
+ wasm::SimdOp widenOp_;
+ enum { Plain, ZeroExtend, Splat, Widen } loadOp_;
+
+ public:
+ explicit MemoryAccessDesc(
+ Scalar::Type type, uint32_t align, uint64_t offset,
+ BytecodeOffset trapOffset,
+ const jit::Synchronization& sync = jit::Synchronization::None())
+ : offset64_(offset),
+ align_(align),
+ type_(type),
+ sync_(sync),
+ trapOffset_(trapOffset),
+ widenOp_(wasm::SimdOp::Limit),
+ loadOp_(Plain) {
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(align));
+ }
+
+ // The offset is a 64-bit value because of memory64. Almost always, it will
+ // fit in 32 bits, and hence offset() checks that it will, this method is used
+ // almost everywhere in the engine. The compiler front-ends must use
+ // offset64() to bypass the check performed by offset(), and must resolve
+ // offsets that don't fit in 32 bits early in the compilation pipeline so that
+ // no large offsets are observed later.
+ uint32_t offset() const {
+ MOZ_ASSERT(offset64_ <= UINT32_MAX);
+ return uint32_t(offset64_);
+ }
+ uint64_t offset64() const { return offset64_; }
+
+ // The offset can be cleared without worrying about its magnitude.
+ void clearOffset() { offset64_ = 0; }
+
+ // The offset can be set (after compile-time evaluation) but only to values
+ // that fit in 32 bits.
+ void setOffset32(uint32_t offset) { offset64_ = offset; }
+
+ uint32_t align() const { return align_; }
+ Scalar::Type type() const { return type_; }
+ unsigned byteSize() const { return Scalar::byteSize(type()); }
+ const jit::Synchronization& sync() const { return sync_; }
+ BytecodeOffset trapOffset() const { return trapOffset_; }
+ wasm::SimdOp widenSimdOp() const {
+ MOZ_ASSERT(isWidenSimd128Load());
+ return widenOp_;
+ }
+ bool isAtomic() const { return !sync_.isNone(); }
+ bool isZeroExtendSimd128Load() const { return loadOp_ == ZeroExtend; }
+ bool isSplatSimd128Load() const { return loadOp_ == Splat; }
+ bool isWidenSimd128Load() const { return loadOp_ == Widen; }
+
+ void setZeroExtendSimd128Load() {
+ MOZ_ASSERT(type() == Scalar::Float32 || type() == Scalar::Float64);
+ MOZ_ASSERT(!isAtomic());
+ MOZ_ASSERT(loadOp_ == Plain);
+ loadOp_ = ZeroExtend;
+ }
+
+ void setSplatSimd128Load() {
+ MOZ_ASSERT(type() == Scalar::Uint8 || type() == Scalar::Uint16 ||
+ type() == Scalar::Float32 || type() == Scalar::Float64);
+ MOZ_ASSERT(!isAtomic());
+ MOZ_ASSERT(loadOp_ == Plain);
+ loadOp_ = Splat;
+ }
+
+ void setWidenSimd128Load(wasm::SimdOp op) {
+ MOZ_ASSERT(type() == Scalar::Float64);
+ MOZ_ASSERT(!isAtomic());
+ MOZ_ASSERT(loadOp_ == Plain);
+ widenOp_ = op;
+ loadOp_ = Widen;
+ }
+};
+
+} // namespace wasm
+
+namespace jit {
+
+// The base class of all Assemblers for all archs.
+class AssemblerShared {
+ wasm::CallSiteVector callSites_;
+ wasm::CallSiteTargetVector callSiteTargets_;
+ wasm::TrapSiteVectorArray trapSites_;
+ wasm::SymbolicAccessVector symbolicAccesses_;
+ wasm::TryNoteVector tryNotes_;
+#ifdef DEBUG
+ // To facilitate figuring out which part of SM created each instruction as
+ // shown by IONFLAGS=codegen, this maintains a stack of (notionally)
+ // code-creating routines, which is printed in the log output every time an
+ // entry is pushed or popped. Do not push/pop entries directly; instead use
+ // `class AutoCreatedBy`.
+ mozilla::Vector<const char*> creators_;
+#endif
+
+ protected:
+ CodeLabelVector codeLabels_;
+
+ bool enoughMemory_;
+ bool embedsNurseryPointers_;
+
+ public:
+ AssemblerShared() : enoughMemory_(true), embedsNurseryPointers_(false) {}
+
+ ~AssemblerShared();
+
+#ifdef DEBUG
+ // Do not use these directly; instead use `class AutoCreatedBy`.
+ void pushCreator(const char*);
+ void popCreator();
+ // See comment on the implementation of `hasCreator` for guidance on what to
+ // do if you get failures of the assertion `MOZ_ASSERT(hasCreator())`,
+ bool hasCreator() const;
+#endif
+
+ void propagateOOM(bool success) { enoughMemory_ &= success; }
+
+ void setOOM() { enoughMemory_ = false; }
+
+ bool oom() const { return !enoughMemory_; }
+
+ bool embedsNurseryPointers() const { return embedsNurseryPointers_; }
+
+ void addCodeLabel(CodeLabel label) {
+ propagateOOM(codeLabels_.append(label));
+ }
+ size_t numCodeLabels() const { return codeLabels_.length(); }
+ CodeLabel codeLabel(size_t i) { return codeLabels_[i]; }
+ CodeLabelVector& codeLabels() { return codeLabels_; }
+
+ // WebAssembly metadata emitted by masm operations accumulated on the
+ // MacroAssembler, and swapped into a wasm::CompiledCode after finish().
+
+ template <typename... Args>
+ void append(const wasm::CallSiteDesc& desc, CodeOffset retAddr,
+ Args&&... args) {
+ enoughMemory_ &= callSites_.emplaceBack(desc, retAddr.offset());
+ enoughMemory_ &= callSiteTargets_.emplaceBack(std::forward<Args>(args)...);
+ }
+ void append(wasm::Trap trap, wasm::TrapSite site) {
+ enoughMemory_ &= trapSites_[trap].append(site);
+ }
+ void append(const wasm::MemoryAccessDesc& access, uint32_t pcOffset) {
+ appendOutOfBoundsTrap(access.trapOffset(), pcOffset);
+ }
+ void appendOutOfBoundsTrap(wasm::BytecodeOffset trapOffset,
+ uint32_t pcOffset) {
+ append(wasm::Trap::OutOfBounds, wasm::TrapSite(pcOffset, trapOffset));
+ }
+ void append(wasm::SymbolicAccess access) {
+ enoughMemory_ &= symbolicAccesses_.append(access);
+ }
+ // This one returns an index as the try note so that it can be looked up
+ // later to add the end point and stack position of the try block.
+ [[nodiscard]] bool append(wasm::TryNote tryNote, size_t* tryNoteIndex) {
+ if (!tryNotes_.append(tryNote)) {
+ enoughMemory_ = false;
+ return false;
+ }
+ *tryNoteIndex = tryNotes_.length() - 1;
+ return true;
+ }
+
+ wasm::CallSiteVector& callSites() { return callSites_; }
+ wasm::CallSiteTargetVector& callSiteTargets() { return callSiteTargets_; }
+ wasm::TrapSiteVectorArray& trapSites() { return trapSites_; }
+ wasm::SymbolicAccessVector& symbolicAccesses() { return symbolicAccesses_; }
+ wasm::TryNoteVector& tryNotes() { return tryNotes_; }
+};
+
+// AutoCreatedBy pushes and later pops a who-created-these-insns? tag into the
+// JitSpew_Codegen output. These could be created fairly frequently, so a
+// dummy inlineable-out version is provided for non-debug builds. The tag
+// text can be completely arbitrary -- it serves only to help readers of the
+// output text to relate instructions back to the part(s) of SM that created
+// them.
+#ifdef DEBUG
+class MOZ_RAII AutoCreatedBy {
+ private:
+ AssemblerShared& ash_;
+
+ public:
+ AutoCreatedBy(AssemblerShared& ash, const char* who) : ash_(ash) {
+ ash_.pushCreator(who);
+ }
+ ~AutoCreatedBy() { ash_.popCreator(); }
+};
+#else
+class MOZ_RAII AutoCreatedBy {
+ public:
+ inline AutoCreatedBy(AssemblerShared& ash, const char* who) {}
+ // A user-defined constructor is necessary to stop some compilers from
+ // complaining about unused variables.
+ inline ~AutoCreatedBy() {}
+};
+#endif
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_shared_Assembler_shared_h */
diff --git a/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h b/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h
new file mode 100644
index 0000000000..8f700aaab3
--- /dev/null
+++ b/js/src/jit/shared/AtomicOperations-feeling-lucky-gcc.h
@@ -0,0 +1,453 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* For documentation, see jit/AtomicOperations.h, both the comment block at the
+ * beginning and the #ifdef nest near the end.
+ *
+ * This is a common file for tier-3 platforms (including simulators for our
+ * tier-1 platforms) that are not providing hardware-specific implementations of
+ * the atomic operations. Please keep it reasonably platform-independent by
+ * adding #ifdefs at the beginning as much as possible, not throughout the file.
+ *
+ *
+ * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ * !!!! NOTE !!!!
+ * !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ *
+ * The implementations in this file are NOT SAFE and cannot be safe even in
+ * principle because they rely on C++ undefined behavior. However, they are
+ * frequently good enough for tier-3 platforms.
+ */
+
+#ifndef jit_shared_AtomicOperations_feeling_lucky_gcc_h
+#define jit_shared_AtomicOperations_feeling_lucky_gcc_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Types.h"
+
+// Explicitly exclude tier-1 platforms.
+
+#if (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \
+ defined(_M_IX86) || (defined(__arm__) && __ARM_ARCH >= 7) || \
+ defined(__aarch64__))
+# error "Do not use on a tier-1 platform where inline assembly is available"
+#endif
+
+#if !(defined(__clang__) || defined(__GNUC__))
+# error "This file only for gcc/Clang"
+#endif
+
+// 64-bit atomics are not required by the JS spec, and you can compile
+// SpiderMonkey without them. 64-bit atomics are required for BigInt
+// support.
+//
+// 64-bit lock-free atomics are required for WebAssembly, but gating in the
+// WebAssembly subsystem ensures that no WebAssembly-supporting platforms need
+// code in this file.
+
+#if defined(JS_SIMULATOR_ARM64) || defined(JS_SIMULATOR_ARM) || \
+ defined(JS_SIMULATOR_MIPS64) || defined(JS_SIMULATOR_LOONG64)
+// On some x86 (32-bit) systems this will not work because the compiler does not
+// open-code 64-bit atomics. If so, try linking with -latomic. If that doesn't
+// work, you're mostly on your own.
+# define HAS_64BIT_ATOMICS
+# define HAS_64BIT_LOCKFREE
+#endif
+
+#if defined(__arm__)
+# define HAS_64BIT_ATOMICS
+#endif
+
+#if defined(__ppc64__) || defined(__PPC64__) || defined(__ppc64le__) || \
+ defined(__PPC64LE__)
+# define HAS_64BIT_ATOMICS
+# define HAS_64BIT_LOCKFREE
+#endif
+
+#if defined(__riscv) && __riscv_xlen == 64
+# define HAS_64BIT_ATOMICS
+# define HAS_64BIT_LOCKFREE
+#endif
+
+#if defined(__loongarch64)
+# define HAS_64BIT_ATOMICS
+# define HAS_64BIT_LOCKFREE
+#endif
+
+#ifdef __sparc__
+# ifdef __LP64__
+# define HAS_64BIT_ATOMICS
+# define HAS_64BIT_LOCKFREE
+# endif
+#endif
+
+#ifdef JS_CODEGEN_NONE
+# ifdef JS_64BIT
+# define HAS_64BIT_ATOMICS
+# define HAS_64BIT_LOCKFREE
+# endif
+#endif
+
+// The default implementation tactic for gcc/clang is to use the newer __atomic
+// intrinsics added for use in C++11 <atomic>. Where that isn't available, we
+// use GCC's older __sync functions instead.
+//
+// ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS is kept as a backward compatible
+// option for older compilers: enable this to use GCC's old __sync functions
+// instead of the newer __atomic functions. This will be required for GCC 4.6.x
+// and earlier, and probably for Clang 3.1, should we need to use those
+// versions. Firefox no longer supports compilers that old.
+
+//#define ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+
+// Sanity check.
+
+#if defined(HAS_64BIT_LOCKFREE) && !defined(HAS_64BIT_ATOMICS)
+# error "This combination of features is senseless, please fix"
+#endif
+
+// Try to avoid platform #ifdefs below this point.
+
+// When compiling with Clang on 32-bit linux it will be necessary to link with
+// -latomic to get the proper 64-bit intrinsics.
+
+inline bool js::jit::AtomicOperations::hasAtomic8() {
+#if defined(HAS_64BIT_ATOMICS)
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline bool js::jit::AtomicOperations::isLockfree8() {
+#if defined(HAS_64BIT_LOCKFREE)
+ return true;
+#else
+ return false;
+#endif
+}
+
+inline void js::jit::AtomicOperations::fenceSeqCst() {
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ __sync_synchronize();
+#else
+ __atomic_thread_fence(__ATOMIC_SEQ_CST);
+#endif
+}
+
+template <typename T>
+inline T js::jit::AtomicOperations::loadSeqCst(T* addr) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ __sync_synchronize();
+ T v = *addr;
+ __sync_synchronize();
+#else
+ T v;
+ __atomic_load(addr, &v, __ATOMIC_SEQ_CST);
+#endif
+ return v;
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline int64_t AtomicOperations::loadSeqCst(int64_t* addr) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline uint64_t AtomicOperations::loadSeqCst(uint64_t* addr) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline void js::jit::AtomicOperations::storeSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ __sync_synchronize();
+ *addr = val;
+ __sync_synchronize();
+#else
+ __atomic_store(addr, &val, __ATOMIC_SEQ_CST);
+#endif
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline void AtomicOperations::storeSeqCst(int64_t* addr, int64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline void AtomicOperations::storeSeqCst(uint64_t* addr, uint64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline T js::jit::AtomicOperations::exchangeSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ T v;
+ __sync_synchronize();
+ do {
+ v = *addr;
+ } while (__sync_val_compare_and_swap(addr, v, val) != v);
+ return v;
+#else
+ T v;
+ __atomic_exchange(addr, &val, &v, __ATOMIC_SEQ_CST);
+ return v;
+#endif
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline int64_t AtomicOperations::exchangeSeqCst(int64_t* addr, int64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline uint64_t AtomicOperations::exchangeSeqCst(uint64_t* addr, uint64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline T js::jit::AtomicOperations::compareExchangeSeqCst(T* addr, T oldval,
+ T newval) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ return __sync_val_compare_and_swap(addr, oldval, newval);
+#else
+ __atomic_compare_exchange(addr, &oldval, &newval, false, __ATOMIC_SEQ_CST,
+ __ATOMIC_SEQ_CST);
+ return oldval;
+#endif
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline int64_t AtomicOperations::compareExchangeSeqCst(int64_t* addr,
+ int64_t oldval,
+ int64_t newval) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline uint64_t AtomicOperations::compareExchangeSeqCst(uint64_t* addr,
+ uint64_t oldval,
+ uint64_t newval) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchAddSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ return __sync_fetch_and_add(addr, val);
+#else
+ return __atomic_fetch_add(addr, val, __ATOMIC_SEQ_CST);
+#endif
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline int64_t AtomicOperations::fetchAddSeqCst(int64_t* addr, int64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline uint64_t AtomicOperations::fetchAddSeqCst(uint64_t* addr, uint64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchSubSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ return __sync_fetch_and_sub(addr, val);
+#else
+ return __atomic_fetch_sub(addr, val, __ATOMIC_SEQ_CST);
+#endif
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline int64_t AtomicOperations::fetchSubSeqCst(int64_t* addr, int64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline uint64_t AtomicOperations::fetchSubSeqCst(uint64_t* addr, uint64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchAndSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ return __sync_fetch_and_and(addr, val);
+#else
+ return __atomic_fetch_and(addr, val, __ATOMIC_SEQ_CST);
+#endif
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline int64_t AtomicOperations::fetchAndSeqCst(int64_t* addr, int64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline uint64_t AtomicOperations::fetchAndSeqCst(uint64_t* addr, uint64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchOrSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ return __sync_fetch_and_or(addr, val);
+#else
+ return __atomic_fetch_or(addr, val, __ATOMIC_SEQ_CST);
+#endif
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline int64_t AtomicOperations::fetchOrSeqCst(int64_t* addr, int64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline uint64_t AtomicOperations::fetchOrSeqCst(uint64_t* addr, uint64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline T js::jit::AtomicOperations::fetchXorSeqCst(T* addr, T val) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+#ifdef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+ return __sync_fetch_and_xor(addr, val);
+#else
+ return __atomic_fetch_xor(addr, val, __ATOMIC_SEQ_CST);
+#endif
+}
+
+#ifndef HAS_64BIT_ATOMICS
+namespace js {
+namespace jit {
+
+template <>
+inline int64_t AtomicOperations::fetchXorSeqCst(int64_t* addr, int64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+template <>
+inline uint64_t AtomicOperations::fetchXorSeqCst(uint64_t* addr, uint64_t val) {
+ MOZ_CRASH("No 64-bit atomics");
+}
+
+} // namespace jit
+} // namespace js
+#endif
+
+template <typename T>
+inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+ // This is actually roughly right even on 32-bit platforms since in that
+ // case, double, int64, and uint64 loads need not be access-atomic.
+ //
+ // We could use __atomic_load, but it would be needlessly expensive on
+ // 32-bit platforms that could support it and just plain wrong on others.
+ return *addr;
+}
+
+template <typename T>
+inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) {
+ static_assert(sizeof(T) <= 8, "atomics supported up to 8 bytes only");
+ // This is actually roughly right even on 32-bit platforms since in that
+ // case, double, int64, and uint64 loads need not be access-atomic.
+ //
+ // We could use __atomic_store, but it would be needlessly expensive on
+ // 32-bit platforms that could support it and just plain wrong on others.
+ *addr = val;
+}
+
+inline void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest,
+ const void* src,
+ size_t nbytes) {
+ MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes));
+ MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes));
+ ::memcpy(dest, src, nbytes);
+}
+
+inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest,
+ const void* src,
+ size_t nbytes) {
+ ::memmove(dest, src, nbytes);
+}
+
+#undef ATOMICS_IMPLEMENTED_WITH_SYNC_INTRINSICS
+#undef HAS_64BIT_ATOMICS
+#undef HAS_64BIT_LOCKFREE
+
+#endif // jit_shared_AtomicOperations_feeling_lucky_gcc_h
diff --git a/js/src/jit/shared/AtomicOperations-feeling-lucky.h b/js/src/jit/shared/AtomicOperations-feeling-lucky.h
new file mode 100644
index 0000000000..4aa7883fd4
--- /dev/null
+++ b/js/src/jit/shared/AtomicOperations-feeling-lucky.h
@@ -0,0 +1,16 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_AtomicOperations_feeling_lucky_h
+#define jit_shared_AtomicOperations_feeling_lucky_h
+
+#if defined(__clang__) || defined(__GNUC__)
+# include "jit/shared/AtomicOperations-feeling-lucky-gcc.h"
+#else
+# error "No AtomicOperations support for this platform+compiler combination"
+#endif
+
+#endif // jit_shared_AtomicOperations_feeling_lucky_h
diff --git a/js/src/jit/shared/AtomicOperations-shared-jit.cpp b/js/src/jit/shared/AtomicOperations-shared-jit.cpp
new file mode 100644
index 0000000000..cc760b8ea7
--- /dev/null
+++ b/js/src/jit/shared/AtomicOperations-shared-jit.cpp
@@ -0,0 +1,176 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/AtomicOperations.h"
+
+#ifdef JS_HAVE_GENERATED_ATOMIC_OPS
+
+# include <atomic>
+
+# include "js/GCAPI.h"
+
+using namespace js;
+using namespace js::jit;
+
+// A "block" is a sequence of bytes that is a reasonable quantum to copy to
+// amortize call overhead when implementing memcpy and memmove. A block will
+// not fit in registers on all platforms and copying it without using
+// intermediate memory will therefore be sensitive to overlap.
+//
+// A "word" is an item that we can copy using only register intermediate storage
+// on all platforms; words can be individually copied without worrying about
+// overlap.
+//
+// Blocks and words can be aligned or unaligned; specific (generated) copying
+// functions handle this in platform-specific ways.
+
+static constexpr size_t WORDSIZE = sizeof(uintptr_t);
+static constexpr size_t BLOCKSIZE = 8 * WORDSIZE; // Must be a power of 2
+
+static_assert(BLOCKSIZE % WORDSIZE == 0,
+ "A block is an integral number of words");
+
+// Constants must match the ones in GenerateAtomicOperations.py
+static_assert(JS_GENERATED_ATOMICS_BLOCKSIZE == BLOCKSIZE);
+static_assert(JS_GENERATED_ATOMICS_WORDSIZE == WORDSIZE);
+
+static constexpr size_t WORDMASK = WORDSIZE - 1;
+static constexpr size_t BLOCKMASK = BLOCKSIZE - 1;
+
+namespace js {
+namespace jit {
+
+static bool UnalignedAccessesAreOK() {
+# ifdef DEBUG
+ const char* flag = getenv("JS_NO_UNALIGNED_MEMCPY");
+ if (flag && *flag == '1') return false;
+# endif
+# if defined(__x86_64__) || defined(__i386__)
+ return true;
+# elif defined(__arm__)
+ return !HasAlignmentFault();
+# elif defined(__aarch64__)
+ // This is not necessarily true but it's the best guess right now.
+ return true;
+# else
+# error "Unsupported platform"
+# endif
+}
+
+# ifndef JS_64BIT
+void AtomicCompilerFence() {
+ std::atomic_signal_fence(std::memory_order_acq_rel);
+}
+# endif
+
+void AtomicMemcpyDownUnsynchronized(uint8_t* dest, const uint8_t* src,
+ size_t nbytes) {
+ JS::AutoSuppressGCAnalysis nogc;
+
+ const uint8_t* lim = src + nbytes;
+
+ // Set up bulk copying. The cases are ordered the way they are on the
+ // assumption that if we can achieve aligned copies even with a little
+ // preprocessing then that is better than unaligned copying on a platform
+ // that supports it.
+
+ if (nbytes >= WORDSIZE) {
+ void (*copyBlock)(uint8_t * dest, const uint8_t* src);
+ void (*copyWord)(uint8_t * dest, const uint8_t* src);
+
+ if (((uintptr_t(dest) ^ uintptr_t(src)) & WORDMASK) == 0) {
+ const uint8_t* cutoff = (const uint8_t*)RoundUp(uintptr_t(src), WORDSIZE);
+ MOZ_ASSERT(cutoff <= lim); // because nbytes >= WORDSIZE
+ while (src < cutoff) {
+ AtomicCopyByteUnsynchronized(dest++, src++);
+ }
+ copyBlock = AtomicCopyBlockDownUnsynchronized;
+ copyWord = AtomicCopyWordUnsynchronized;
+ } else if (UnalignedAccessesAreOK()) {
+ copyBlock = AtomicCopyBlockDownUnsynchronized;
+ copyWord = AtomicCopyWordUnsynchronized;
+ } else {
+ copyBlock = AtomicCopyUnalignedBlockDownUnsynchronized;
+ copyWord = AtomicCopyUnalignedWordDownUnsynchronized;
+ }
+
+ // Bulk copy, first larger blocks and then individual words.
+
+ const uint8_t* blocklim = src + ((lim - src) & ~BLOCKMASK);
+ while (src < blocklim) {
+ copyBlock(dest, src);
+ dest += BLOCKSIZE;
+ src += BLOCKSIZE;
+ }
+
+ const uint8_t* wordlim = src + ((lim - src) & ~WORDMASK);
+ while (src < wordlim) {
+ copyWord(dest, src);
+ dest += WORDSIZE;
+ src += WORDSIZE;
+ }
+ }
+
+ // Byte copy any remaining tail.
+
+ while (src < lim) {
+ AtomicCopyByteUnsynchronized(dest++, src++);
+ }
+}
+
+void AtomicMemcpyUpUnsynchronized(uint8_t* dest, const uint8_t* src,
+ size_t nbytes) {
+ JS::AutoSuppressGCAnalysis nogc;
+
+ const uint8_t* lim = src;
+
+ src += nbytes;
+ dest += nbytes;
+
+ if (nbytes >= WORDSIZE) {
+ void (*copyBlock)(uint8_t * dest, const uint8_t* src);
+ void (*copyWord)(uint8_t * dest, const uint8_t* src);
+
+ if (((uintptr_t(dest) ^ uintptr_t(src)) & WORDMASK) == 0) {
+ const uint8_t* cutoff = (const uint8_t*)(uintptr_t(src) & ~WORDMASK);
+ MOZ_ASSERT(cutoff >= lim); // Because nbytes >= WORDSIZE
+ while (src > cutoff) {
+ AtomicCopyByteUnsynchronized(--dest, --src);
+ }
+ copyBlock = AtomicCopyBlockUpUnsynchronized;
+ copyWord = AtomicCopyWordUnsynchronized;
+ } else if (UnalignedAccessesAreOK()) {
+ copyBlock = AtomicCopyBlockUpUnsynchronized;
+ copyWord = AtomicCopyWordUnsynchronized;
+ } else {
+ copyBlock = AtomicCopyUnalignedBlockUpUnsynchronized;
+ copyWord = AtomicCopyUnalignedWordUpUnsynchronized;
+ }
+
+ const uint8_t* blocklim = src - ((src - lim) & ~BLOCKMASK);
+ while (src > blocklim) {
+ dest -= BLOCKSIZE;
+ src -= BLOCKSIZE;
+ copyBlock(dest, src);
+ }
+
+ const uint8_t* wordlim = src - ((src - lim) & ~WORDMASK);
+ while (src > wordlim) {
+ dest -= WORDSIZE;
+ src -= WORDSIZE;
+ copyWord(dest, src);
+ }
+ }
+
+ while (src > lim) {
+ AtomicCopyByteUnsynchronized(--dest, --src);
+ }
+}
+
+} // namespace jit
+} // namespace js
+
+#endif // JS_HAVE_GENERATED_ATOMIC_OPS
diff --git a/js/src/jit/shared/AtomicOperations-shared-jit.h b/js/src/jit/shared/AtomicOperations-shared-jit.h
new file mode 100644
index 0000000000..ca66a6f9b9
--- /dev/null
+++ b/js/src/jit/shared/AtomicOperations-shared-jit.h
@@ -0,0 +1,490 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=4 et sw=4 tw=99:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* For overall documentation, see jit/AtomicOperations.h.
+ *
+ * NOTE CAREFULLY: This file is only applicable when we have configured a JIT
+ * and the JIT is for the same architecture that we're compiling the shell for.
+ * Simulators must use a different mechanism.
+ *
+ * See comments before the include nest near the end of jit/AtomicOperations.h
+ * if you didn't understand that.
+ */
+
+#ifndef jit_shared_AtomicOperations_shared_jit_h
+#define jit_shared_AtomicOperations_shared_jit_h
+
+#include "mozilla/Assertions.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/AtomicOperationsGenerated.h"
+#include "vm/Uint8Clamped.h"
+
+namespace js {
+namespace jit {
+
+#ifndef JS_64BIT
+// `AtomicCompilerFence` erects a reordering boundary for operations on the
+// current thread. We use it to prevent the compiler from reordering loads and
+// stores inside larger primitives that are synthesized from cmpxchg.
+extern void AtomicCompilerFence();
+#endif
+
+// `...MemcpyDown` moves bytes toward lower addresses in memory: dest <= src.
+// `...MemcpyUp` moves bytes toward higher addresses in memory: dest >= src.
+extern void AtomicMemcpyDownUnsynchronized(uint8_t* dest, const uint8_t* src,
+ size_t nbytes);
+extern void AtomicMemcpyUpUnsynchronized(uint8_t* dest, const uint8_t* src,
+ size_t nbytes);
+
+} // namespace jit
+} // namespace js
+
+inline bool js::jit::AtomicOperations::hasAtomic8() { return true; }
+
+inline bool js::jit::AtomicOperations::isLockfree8() { return true; }
+
+inline void js::jit::AtomicOperations::fenceSeqCst() { AtomicFenceSeqCst(); }
+
+#define JIT_LOADOP(T, U, loadop) \
+ template <> \
+ inline T AtomicOperations::loadSeqCst(T* addr) { \
+ return (T)loadop((U*)addr); \
+ }
+
+#ifndef JS_64BIT
+# define JIT_LOADOP_CAS(T) \
+ template <> \
+ inline T AtomicOperations::loadSeqCst(T* addr) { \
+ AtomicCompilerFence(); \
+ return (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, 0, 0); \
+ }
+#endif // !JS_64BIT
+
+namespace js {
+namespace jit {
+
+JIT_LOADOP(int8_t, uint8_t, AtomicLoad8SeqCst)
+JIT_LOADOP(uint8_t, uint8_t, AtomicLoad8SeqCst)
+JIT_LOADOP(int16_t, uint16_t, AtomicLoad16SeqCst)
+JIT_LOADOP(uint16_t, uint16_t, AtomicLoad16SeqCst)
+JIT_LOADOP(int32_t, uint32_t, AtomicLoad32SeqCst)
+JIT_LOADOP(uint32_t, uint32_t, AtomicLoad32SeqCst)
+
+#ifdef JIT_LOADOP_CAS
+JIT_LOADOP_CAS(int64_t)
+JIT_LOADOP_CAS(uint64_t)
+#else
+JIT_LOADOP(int64_t, uint64_t, AtomicLoad64SeqCst)
+JIT_LOADOP(uint64_t, uint64_t, AtomicLoad64SeqCst)
+#endif
+
+} // namespace jit
+} // namespace js
+
+#undef JIT_LOADOP
+#undef JIT_LOADOP_CAS
+
+#define JIT_STOREOP(T, U, storeop) \
+ template <> \
+ inline void AtomicOperations::storeSeqCst(T* addr, T val) { \
+ storeop((U*)addr, val); \
+ }
+
+#ifndef JS_64BIT
+# define JIT_STOREOP_CAS(T) \
+ template <> \
+ inline void AtomicOperations::storeSeqCst(T* addr, T val) { \
+ AtomicCompilerFence(); \
+ T oldval = *addr; /* good initial approximation */ \
+ for (;;) { \
+ T nextval = (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, \
+ (uint64_t)oldval, (uint64_t)val); \
+ if (nextval == oldval) { \
+ break; \
+ } \
+ oldval = nextval; \
+ } \
+ AtomicCompilerFence(); \
+ }
+#endif // !JS_64BIT
+
+namespace js {
+namespace jit {
+
+JIT_STOREOP(int8_t, uint8_t, AtomicStore8SeqCst)
+JIT_STOREOP(uint8_t, uint8_t, AtomicStore8SeqCst)
+JIT_STOREOP(int16_t, uint16_t, AtomicStore16SeqCst)
+JIT_STOREOP(uint16_t, uint16_t, AtomicStore16SeqCst)
+JIT_STOREOP(int32_t, uint32_t, AtomicStore32SeqCst)
+JIT_STOREOP(uint32_t, uint32_t, AtomicStore32SeqCst)
+
+#ifdef JIT_STOREOP_CAS
+JIT_STOREOP_CAS(int64_t)
+JIT_STOREOP_CAS(uint64_t)
+#else
+JIT_STOREOP(int64_t, uint64_t, AtomicStore64SeqCst)
+JIT_STOREOP(uint64_t, uint64_t, AtomicStore64SeqCst)
+#endif
+
+} // namespace jit
+} // namespace js
+
+#undef JIT_STOREOP
+#undef JIT_STOREOP_CAS
+
+#define JIT_EXCHANGEOP(T, U, xchgop) \
+ template <> \
+ inline T AtomicOperations::exchangeSeqCst(T* addr, T val) { \
+ return (T)xchgop((U*)addr, (U)val); \
+ }
+
+#ifndef JS_64BIT
+# define JIT_EXCHANGEOP_CAS(T) \
+ template <> \
+ inline T AtomicOperations::exchangeSeqCst(T* addr, T val) { \
+ AtomicCompilerFence(); \
+ T oldval = *addr; \
+ for (;;) { \
+ T nextval = (T)AtomicCmpXchg64SeqCst((uint64_t*)addr, \
+ (uint64_t)oldval, (uint64_t)val); \
+ if (nextval == oldval) { \
+ break; \
+ } \
+ oldval = nextval; \
+ } \
+ AtomicCompilerFence(); \
+ return oldval; \
+ }
+#endif // !JS_64BIT
+
+namespace js {
+namespace jit {
+
+JIT_EXCHANGEOP(int8_t, uint8_t, AtomicExchange8SeqCst)
+JIT_EXCHANGEOP(uint8_t, uint8_t, AtomicExchange8SeqCst)
+JIT_EXCHANGEOP(int16_t, uint16_t, AtomicExchange16SeqCst)
+JIT_EXCHANGEOP(uint16_t, uint16_t, AtomicExchange16SeqCst)
+JIT_EXCHANGEOP(int32_t, uint32_t, AtomicExchange32SeqCst)
+JIT_EXCHANGEOP(uint32_t, uint32_t, AtomicExchange32SeqCst)
+
+#ifdef JIT_EXCHANGEOP_CAS
+JIT_EXCHANGEOP_CAS(int64_t)
+JIT_EXCHANGEOP_CAS(uint64_t)
+#else
+JIT_EXCHANGEOP(int64_t, uint64_t, AtomicExchange64SeqCst)
+JIT_EXCHANGEOP(uint64_t, uint64_t, AtomicExchange64SeqCst)
+#endif
+
+} // namespace jit
+} // namespace js
+
+#undef JIT_EXCHANGEOP
+#undef JIT_EXCHANGEOP_CAS
+
+#define JIT_CAS(T, U, cmpxchg) \
+ template <> \
+ inline T AtomicOperations::compareExchangeSeqCst(T* addr, T oldval, \
+ T newval) { \
+ return (T)cmpxchg((U*)addr, (U)oldval, (U)newval); \
+ }
+
+namespace js {
+namespace jit {
+
+JIT_CAS(int8_t, uint8_t, AtomicCmpXchg8SeqCst)
+JIT_CAS(uint8_t, uint8_t, AtomicCmpXchg8SeqCst)
+JIT_CAS(int16_t, uint16_t, AtomicCmpXchg16SeqCst)
+JIT_CAS(uint16_t, uint16_t, AtomicCmpXchg16SeqCst)
+JIT_CAS(int32_t, uint32_t, AtomicCmpXchg32SeqCst)
+JIT_CAS(uint32_t, uint32_t, AtomicCmpXchg32SeqCst)
+JIT_CAS(int64_t, uint64_t, AtomicCmpXchg64SeqCst)
+JIT_CAS(uint64_t, uint64_t, AtomicCmpXchg64SeqCst)
+
+} // namespace jit
+} // namespace js
+
+#undef JIT_CAS
+
+#define JIT_FETCHADDOP(T, U, xadd) \
+ template <> \
+ inline T AtomicOperations::fetchAddSeqCst(T* addr, T val) { \
+ return (T)xadd((U*)addr, (U)val); \
+ }
+
+#define JIT_FETCHSUBOP(T) \
+ template <> \
+ inline T AtomicOperations::fetchSubSeqCst(T* addr, T val) { \
+ return fetchAddSeqCst(addr, (T)(0 - val)); \
+ }
+
+#ifndef JS_64BIT
+# define JIT_FETCHADDOP_CAS(T) \
+ template <> \
+ inline T AtomicOperations::fetchAddSeqCst(T* addr, T val) { \
+ AtomicCompilerFence(); \
+ T oldval = *addr; /* Good initial approximation */ \
+ for (;;) { \
+ T nextval = (T)AtomicCmpXchg64SeqCst( \
+ (uint64_t*)addr, (uint64_t)oldval, (uint64_t)(oldval + val)); \
+ if (nextval == oldval) { \
+ break; \
+ } \
+ oldval = nextval; \
+ } \
+ AtomicCompilerFence(); \
+ return oldval; \
+ }
+#endif // !JS_64BIT
+
+namespace js {
+namespace jit {
+
+JIT_FETCHADDOP(int8_t, uint8_t, AtomicAdd8SeqCst)
+JIT_FETCHADDOP(uint8_t, uint8_t, AtomicAdd8SeqCst)
+JIT_FETCHADDOP(int16_t, uint16_t, AtomicAdd16SeqCst)
+JIT_FETCHADDOP(uint16_t, uint16_t, AtomicAdd16SeqCst)
+JIT_FETCHADDOP(int32_t, uint32_t, AtomicAdd32SeqCst)
+JIT_FETCHADDOP(uint32_t, uint32_t, AtomicAdd32SeqCst)
+
+#ifdef JIT_FETCHADDOP_CAS
+JIT_FETCHADDOP_CAS(int64_t)
+JIT_FETCHADDOP_CAS(uint64_t)
+#else
+JIT_FETCHADDOP(int64_t, uint64_t, AtomicAdd64SeqCst)
+JIT_FETCHADDOP(uint64_t, uint64_t, AtomicAdd64SeqCst)
+#endif
+
+JIT_FETCHSUBOP(int8_t)
+JIT_FETCHSUBOP(uint8_t)
+JIT_FETCHSUBOP(int16_t)
+JIT_FETCHSUBOP(uint16_t)
+JIT_FETCHSUBOP(int32_t)
+JIT_FETCHSUBOP(uint32_t)
+JIT_FETCHSUBOP(int64_t)
+JIT_FETCHSUBOP(uint64_t)
+
+} // namespace jit
+} // namespace js
+
+#undef JIT_FETCHADDOP
+#undef JIT_FETCHADDOP_CAS
+#undef JIT_FETCHSUBOP
+
+#define JIT_FETCHBITOPX(T, U, name, op) \
+ template <> \
+ inline T AtomicOperations::name(T* addr, T val) { \
+ return (T)op((U*)addr, (U)val); \
+ }
+
+#define JIT_FETCHBITOP(T, U, andop, orop, xorop) \
+ JIT_FETCHBITOPX(T, U, fetchAndSeqCst, andop) \
+ JIT_FETCHBITOPX(T, U, fetchOrSeqCst, orop) \
+ JIT_FETCHBITOPX(T, U, fetchXorSeqCst, xorop)
+
+#ifndef JS_64BIT
+
+# define AND_OP &
+# define OR_OP |
+# define XOR_OP ^
+
+# define JIT_FETCHBITOPX_CAS(T, name, OP) \
+ template <> \
+ inline T AtomicOperations::name(T* addr, T val) { \
+ AtomicCompilerFence(); \
+ T oldval = *addr; \
+ for (;;) { \
+ T nextval = (T)AtomicCmpXchg64SeqCst( \
+ (uint64_t*)addr, (uint64_t)oldval, (uint64_t)(oldval OP val)); \
+ if (nextval == oldval) { \
+ break; \
+ } \
+ oldval = nextval; \
+ } \
+ AtomicCompilerFence(); \
+ return oldval; \
+ }
+
+# define JIT_FETCHBITOP_CAS(T) \
+ JIT_FETCHBITOPX_CAS(T, fetchAndSeqCst, AND_OP) \
+ JIT_FETCHBITOPX_CAS(T, fetchOrSeqCst, OR_OP) \
+ JIT_FETCHBITOPX_CAS(T, fetchXorSeqCst, XOR_OP)
+
+#endif // !JS_64BIT
+
+namespace js {
+namespace jit {
+
+JIT_FETCHBITOP(int8_t, uint8_t, AtomicAnd8SeqCst, AtomicOr8SeqCst,
+ AtomicXor8SeqCst)
+JIT_FETCHBITOP(uint8_t, uint8_t, AtomicAnd8SeqCst, AtomicOr8SeqCst,
+ AtomicXor8SeqCst)
+JIT_FETCHBITOP(int16_t, uint16_t, AtomicAnd16SeqCst, AtomicOr16SeqCst,
+ AtomicXor16SeqCst)
+JIT_FETCHBITOP(uint16_t, uint16_t, AtomicAnd16SeqCst, AtomicOr16SeqCst,
+ AtomicXor16SeqCst)
+JIT_FETCHBITOP(int32_t, uint32_t, AtomicAnd32SeqCst, AtomicOr32SeqCst,
+ AtomicXor32SeqCst)
+JIT_FETCHBITOP(uint32_t, uint32_t, AtomicAnd32SeqCst, AtomicOr32SeqCst,
+ AtomicXor32SeqCst)
+
+#ifdef JIT_FETCHBITOP_CAS
+JIT_FETCHBITOP_CAS(int64_t)
+JIT_FETCHBITOP_CAS(uint64_t)
+#else
+JIT_FETCHBITOP(int64_t, uint64_t, AtomicAnd64SeqCst, AtomicOr64SeqCst,
+ AtomicXor64SeqCst)
+JIT_FETCHBITOP(uint64_t, uint64_t, AtomicAnd64SeqCst, AtomicOr64SeqCst,
+ AtomicXor64SeqCst)
+#endif
+
+} // namespace jit
+} // namespace js
+
+#undef JIT_FETCHBITOPX_CAS
+#undef JIT_FETCHBITOPX
+#undef JIT_FETCHBITOP_CAS
+#undef JIT_FETCHBITOP
+
+#define JIT_LOADSAFE(T, U, loadop) \
+ template <> \
+ inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { \
+ union { \
+ U u; \
+ T t; \
+ }; \
+ u = loadop((U*)addr); \
+ return t; \
+ }
+
+#ifndef JS_64BIT
+# define JIT_LOADSAFE_TEARING(T) \
+ template <> \
+ inline T js::jit::AtomicOperations::loadSafeWhenRacy(T* addr) { \
+ MOZ_ASSERT(sizeof(T) == 8); \
+ union { \
+ uint32_t u[2]; \
+ T t; \
+ }; \
+ uint32_t* ptr = (uint32_t*)addr; \
+ u[0] = AtomicLoad32Unsynchronized(ptr); \
+ u[1] = AtomicLoad32Unsynchronized(ptr + 1); \
+ return t; \
+ }
+#endif // !JS_64BIT
+
+namespace js {
+namespace jit {
+
+JIT_LOADSAFE(int8_t, uint8_t, AtomicLoad8Unsynchronized)
+JIT_LOADSAFE(uint8_t, uint8_t, AtomicLoad8Unsynchronized)
+JIT_LOADSAFE(int16_t, uint16_t, AtomicLoad16Unsynchronized)
+JIT_LOADSAFE(uint16_t, uint16_t, AtomicLoad16Unsynchronized)
+JIT_LOADSAFE(int32_t, uint32_t, AtomicLoad32Unsynchronized)
+JIT_LOADSAFE(uint32_t, uint32_t, AtomicLoad32Unsynchronized)
+#ifdef JIT_LOADSAFE_TEARING
+JIT_LOADSAFE_TEARING(int64_t)
+JIT_LOADSAFE_TEARING(uint64_t)
+JIT_LOADSAFE_TEARING(double)
+#else
+JIT_LOADSAFE(int64_t, uint64_t, AtomicLoad64Unsynchronized)
+JIT_LOADSAFE(uint64_t, uint64_t, AtomicLoad64Unsynchronized)
+JIT_LOADSAFE(double, uint64_t, AtomicLoad64Unsynchronized)
+#endif
+JIT_LOADSAFE(float, uint32_t, AtomicLoad32Unsynchronized)
+
+// Clang requires a specialization for uint8_clamped.
+template <>
+inline uint8_clamped js::jit::AtomicOperations::loadSafeWhenRacy(
+ uint8_clamped* addr) {
+ return uint8_clamped(loadSafeWhenRacy((uint8_t*)addr));
+}
+
+} // namespace jit
+} // namespace js
+
+#undef JIT_LOADSAFE
+#undef JIT_LOADSAFE_TEARING
+
+#define JIT_STORESAFE(T, U, storeop) \
+ template <> \
+ inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { \
+ union { \
+ U u; \
+ T t; \
+ }; \
+ t = val; \
+ storeop((U*)addr, u); \
+ }
+
+#ifndef JS_64BIT
+# define JIT_STORESAFE_TEARING(T) \
+ template <> \
+ inline void js::jit::AtomicOperations::storeSafeWhenRacy(T* addr, T val) { \
+ union { \
+ uint32_t u[2]; \
+ T t; \
+ }; \
+ t = val; \
+ uint32_t* ptr = (uint32_t*)addr; \
+ AtomicStore32Unsynchronized(ptr, u[0]); \
+ AtomicStore32Unsynchronized(ptr + 1, u[1]); \
+ }
+#endif // !JS_64BIT
+
+namespace js {
+namespace jit {
+
+JIT_STORESAFE(int8_t, uint8_t, AtomicStore8Unsynchronized)
+JIT_STORESAFE(uint8_t, uint8_t, AtomicStore8Unsynchronized)
+JIT_STORESAFE(int16_t, uint16_t, AtomicStore16Unsynchronized)
+JIT_STORESAFE(uint16_t, uint16_t, AtomicStore16Unsynchronized)
+JIT_STORESAFE(int32_t, uint32_t, AtomicStore32Unsynchronized)
+JIT_STORESAFE(uint32_t, uint32_t, AtomicStore32Unsynchronized)
+#ifdef JIT_STORESAFE_TEARING
+JIT_STORESAFE_TEARING(int64_t)
+JIT_STORESAFE_TEARING(uint64_t)
+JIT_STORESAFE_TEARING(double)
+#else
+JIT_STORESAFE(int64_t, uint64_t, AtomicStore64Unsynchronized)
+JIT_STORESAFE(uint64_t, uint64_t, AtomicStore64Unsynchronized)
+JIT_STORESAFE(double, uint64_t, AtomicStore64Unsynchronized)
+#endif
+JIT_STORESAFE(float, uint32_t, AtomicStore32Unsynchronized)
+
+// Clang requires a specialization for uint8_clamped.
+template <>
+inline void js::jit::AtomicOperations::storeSafeWhenRacy(uint8_clamped* addr,
+ uint8_clamped val) {
+ storeSafeWhenRacy((uint8_t*)addr, (uint8_t)val);
+}
+
+} // namespace jit
+} // namespace js
+
+#undef JIT_STORESAFE
+#undef JIT_STORESAFE_TEARING
+
+void js::jit::AtomicOperations::memcpySafeWhenRacy(void* dest, const void* src,
+ size_t nbytes) {
+ MOZ_ASSERT(!((char*)dest <= (char*)src && (char*)src < (char*)dest + nbytes));
+ MOZ_ASSERT(!((char*)src <= (char*)dest && (char*)dest < (char*)src + nbytes));
+ AtomicMemcpyDownUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes);
+}
+
+inline void js::jit::AtomicOperations::memmoveSafeWhenRacy(void* dest,
+ const void* src,
+ size_t nbytes) {
+ if ((char*)dest <= (char*)src) {
+ AtomicMemcpyDownUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes);
+ } else {
+ AtomicMemcpyUpUnsynchronized((uint8_t*)dest, (const uint8_t*)src, nbytes);
+ }
+}
+
+#endif // jit_shared_AtomicOperations_shared_jit_h
diff --git a/js/src/jit/shared/CodeGenerator-shared-inl.h b/js/src/jit/shared/CodeGenerator-shared-inl.h
new file mode 100644
index 0000000000..1c9880f59f
--- /dev/null
+++ b/js/src/jit/shared/CodeGenerator-shared-inl.h
@@ -0,0 +1,342 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_CodeGenerator_shared_inl_h
+#define jit_shared_CodeGenerator_shared_inl_h
+
+#include "jit/shared/CodeGenerator-shared.h"
+
+#include "jit/JitFrames.h"
+#include "jit/ScalarTypeUtils.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+static inline bool IsConstant(const LInt64Allocation& a) {
+#if JS_BITS_PER_WORD == 32
+ if (a.high().isConstantValue()) {
+ return true;
+ }
+ if (a.high().isConstantIndex()) {
+ return true;
+ }
+#else
+ if (a.value().isConstantValue()) {
+ return true;
+ }
+ if (a.value().isConstantIndex()) {
+ return true;
+ }
+#endif
+ return false;
+}
+
+static inline int32_t ToInt32(const LAllocation* a) {
+ if (a->isConstantValue()) {
+ const MConstant* cst = a->toConstant();
+ if (cst->type() == MIRType::Int32) {
+ return cst->toInt32();
+ }
+ intptr_t val = cst->toIntPtr();
+ MOZ_ASSERT(INT32_MIN <= val && val <= INT32_MAX);
+ return int32_t(val);
+ }
+ if (a->isConstantIndex()) {
+ return a->toConstantIndex()->index();
+ }
+ MOZ_CRASH("this is not a constant!");
+}
+
+static inline int64_t ToInt64(const LAllocation* a) {
+ if (a->isConstantValue()) {
+ return a->toConstant()->toInt64();
+ }
+ if (a->isConstantIndex()) {
+ return a->toConstantIndex()->index();
+ }
+ MOZ_CRASH("this is not a constant!");
+}
+
+static inline int64_t ToInt64(const LInt64Allocation& a) {
+#if JS_BITS_PER_WORD == 32
+ if (a.high().isConstantValue()) {
+ return a.high().toConstant()->toInt64();
+ }
+ if (a.high().isConstantIndex()) {
+ return a.high().toConstantIndex()->index();
+ }
+#else
+ if (a.value().isConstantValue()) {
+ return a.value().toConstant()->toInt64();
+ }
+ if (a.value().isConstantIndex()) {
+ return a.value().toConstantIndex()->index();
+ }
+#endif
+ MOZ_CRASH("this is not a constant!");
+}
+
+static inline double ToDouble(const LAllocation* a) {
+ return a->toConstant()->numberToDouble();
+}
+
+static inline bool ToBoolean(const LAllocation* a) {
+ return a->toConstant()->toBoolean();
+}
+
+static inline Register ToRegister(const LAllocation& a) {
+ MOZ_ASSERT(a.isGeneralReg());
+ return a.toGeneralReg()->reg();
+}
+
+static inline Register ToRegister(const LAllocation* a) {
+ return ToRegister(*a);
+}
+
+static inline Register ToRegister(const LDefinition* def) {
+ return ToRegister(*def->output());
+}
+
+static inline Register64 ToOutRegister64(LInstruction* ins) {
+#if JS_BITS_PER_WORD == 32
+ Register loReg = ToRegister(ins->getDef(INT64LOW_INDEX));
+ Register hiReg = ToRegister(ins->getDef(INT64HIGH_INDEX));
+ return Register64(hiReg, loReg);
+#else
+ return Register64(ToRegister(ins->getDef(0)));
+#endif
+}
+
+static inline Register64 ToRegister64(const LInt64Allocation& a) {
+#if JS_BITS_PER_WORD == 32
+ return Register64(ToRegister(a.high()), ToRegister(a.low()));
+#else
+ return Register64(ToRegister(a.value()));
+#endif
+}
+
+static inline Register64 ToRegister64(const LInt64Definition& a) {
+#if JS_BITS_PER_WORD == 32
+ return Register64(ToRegister(a.pointerHigh()), ToRegister(a.pointerLow()));
+#else
+ return Register64(ToRegister(a.pointer()));
+#endif
+}
+
+static inline Register ToTempRegisterOrInvalid(const LDefinition* def) {
+ if (def->isBogusTemp()) {
+ return InvalidReg;
+ }
+ return ToRegister(def);
+}
+
+static inline Register64 ToTempRegister64OrInvalid(
+ const LInt64Definition& def) {
+ if (def.isBogusTemp()) {
+ return Register64::Invalid();
+ }
+ return ToRegister64(def);
+}
+
+static inline Register ToTempUnboxRegister(const LDefinition* def) {
+ return ToTempRegisterOrInvalid(def);
+}
+
+static inline Register ToRegisterOrInvalid(const LDefinition* a) {
+ return a ? ToRegister(a) : InvalidReg;
+}
+
+static inline FloatRegister ToFloatRegister(const LAllocation& a) {
+ MOZ_ASSERT(a.isFloatReg());
+ return a.toFloatReg()->reg();
+}
+
+static inline FloatRegister ToFloatRegister(const LAllocation* a) {
+ return ToFloatRegister(*a);
+}
+
+static inline FloatRegister ToFloatRegister(const LDefinition* def) {
+ return ToFloatRegister(*def->output());
+}
+
+static inline FloatRegister ToTempFloatRegisterOrInvalid(
+ const LDefinition* def) {
+ if (def->isBogusTemp()) {
+ return InvalidFloatReg;
+ }
+ return ToFloatRegister(def);
+}
+
+static inline AnyRegister ToAnyRegister(const LAllocation& a) {
+ MOZ_ASSERT(a.isGeneralReg() || a.isFloatReg());
+ if (a.isGeneralReg()) {
+ return AnyRegister(ToRegister(a));
+ }
+ return AnyRegister(ToFloatRegister(a));
+}
+
+static inline AnyRegister ToAnyRegister(const LAllocation* a) {
+ return ToAnyRegister(*a);
+}
+
+static inline AnyRegister ToAnyRegister(const LDefinition* def) {
+ return ToAnyRegister(def->output());
+}
+
+static inline ValueOperand ToOutValue(LInstruction* ins) {
+#if defined(JS_NUNBOX32)
+ return ValueOperand(ToRegister(ins->getDef(TYPE_INDEX)),
+ ToRegister(ins->getDef(PAYLOAD_INDEX)));
+#elif defined(JS_PUNBOX64)
+ return ValueOperand(ToRegister(ins->getDef(0)));
+#else
+# error "Unknown"
+#endif
+}
+
+static inline ValueOperand GetTempValue(Register type, Register payload) {
+#if defined(JS_NUNBOX32)
+ return ValueOperand(type, payload);
+#elif defined(JS_PUNBOX64)
+ (void)type;
+ return ValueOperand(payload);
+#else
+# error "Unknown"
+#endif
+}
+
+// For argument construction for calls. Argslots are Value-sized.
+Address CodeGeneratorShared::AddressOfPassedArg(uint32_t slot) const {
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+
+ MOZ_ASSERT(slot > 0);
+ MOZ_ASSERT(slot <= graph.argumentSlotCount());
+
+ uint32_t offsetFromBase = offsetOfPassedArgSlots_ + slot * sizeof(Value);
+ MOZ_ASSERT(offsetFromBase <= frameSize());
+
+ // Space for passed arguments is reserved below a function's local stack
+ // storage. Note that passedArgSlotsOffset_ is aligned to at least
+ // sizeof(Value) to ensure proper alignment.
+ MOZ_ASSERT((offsetFromBase % sizeof(Value)) == 0);
+
+ if (JitOptions.baseRegForLocals == BaseRegForAddress::SP) {
+ return Address(masm.getStackPointer(), frameSize() - offsetFromBase);
+ }
+ MOZ_ASSERT(JitOptions.baseRegForLocals == BaseRegForAddress::FP);
+ return Address(FramePointer, -int32_t(offsetFromBase));
+}
+
+uint32_t CodeGeneratorShared::UnusedStackBytesForCall(
+ uint32_t numArgSlots) const {
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+ MOZ_ASSERT(numArgSlots <= graph.argumentSlotCount());
+ uint32_t unusedArgSlots = graph.argumentSlotCount() - numArgSlots;
+ return unusedArgSlots * sizeof(Value);
+}
+
+template <BaseRegForAddress Base>
+Address CodeGeneratorShared::ToAddress(const LAllocation& a) const {
+ MOZ_ASSERT(a.isMemory() || a.isStackArea());
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+
+ if (a.isArgument()) {
+ // Use the frame pointer, unless the caller explicitly requested a
+ // stack-pointer-relative address.
+ uint32_t offsetFromFP = offsetOfArgsFromFP_ + a.toArgument()->index();
+ if constexpr (Base == BaseRegForAddress::SP) {
+ return Address(masm.getStackPointer(), frameSize() + offsetFromFP);
+ } else {
+ static_assert(Base == BaseRegForAddress::Default ||
+ Base == BaseRegForAddress::FP);
+ return Address(FramePointer, offsetFromFP);
+ }
+ }
+
+ uint32_t slot =
+ a.isStackSlot() ? a.toStackSlot()->slot() : a.toStackArea()->base();
+ MOZ_ASSERT(slot > 0 && slot <= graph.localSlotsSize());
+ MOZ_ASSERT(slot <= frameSize());
+
+ BaseRegForAddress base = Base;
+ if constexpr (Base == BaseRegForAddress::Default) {
+ base = JitOptions.baseRegForLocals;
+ }
+
+ if (base == BaseRegForAddress::FP) {
+ return Address(FramePointer, -int32_t(slot));
+ }
+ MOZ_ASSERT(base == BaseRegForAddress::SP);
+ return Address(masm.getStackPointer(), frameSize() - slot);
+}
+
+template <BaseRegForAddress Base>
+Address CodeGeneratorShared::ToAddress(const LAllocation* a) const {
+ return ToAddress<Base>(*a);
+}
+
+// static
+Address CodeGeneratorShared::ToAddress(Register elements,
+ const LAllocation* index,
+ Scalar::Type type,
+ int32_t offsetAdjustment) {
+ int32_t idx = ToInt32(index);
+ int32_t offset;
+ MOZ_ALWAYS_TRUE(ArrayOffsetFitsInInt32(idx, type, offsetAdjustment, &offset));
+ return Address(elements, offset);
+}
+
+void CodeGeneratorShared::saveLive(LInstruction* ins) {
+ MOZ_ASSERT(!ins->isCall());
+ LSafepoint* safepoint = ins->safepoint();
+ masm.PushRegsInMask(safepoint->liveRegs());
+}
+
+void CodeGeneratorShared::restoreLive(LInstruction* ins) {
+ MOZ_ASSERT(!ins->isCall());
+ LSafepoint* safepoint = ins->safepoint();
+ masm.PopRegsInMask(safepoint->liveRegs());
+}
+
+void CodeGeneratorShared::restoreLiveIgnore(LInstruction* ins,
+ LiveRegisterSet ignore) {
+ MOZ_ASSERT(!ins->isCall());
+ LSafepoint* safepoint = ins->safepoint();
+ masm.PopRegsInMaskIgnore(safepoint->liveRegs(), ignore);
+}
+
+LiveRegisterSet CodeGeneratorShared::liveVolatileRegs(LInstruction* ins) {
+ MOZ_ASSERT(!ins->isCall());
+ LSafepoint* safepoint = ins->safepoint();
+ LiveRegisterSet regs;
+ regs.set() = RegisterSet::Intersect(safepoint->liveRegs().set(),
+ RegisterSet::Volatile());
+ return regs;
+}
+
+void CodeGeneratorShared::saveLiveVolatile(LInstruction* ins) {
+ LiveRegisterSet regs = liveVolatileRegs(ins);
+ masm.PushRegsInMask(regs);
+}
+
+void CodeGeneratorShared::restoreLiveVolatile(LInstruction* ins) {
+ LiveRegisterSet regs = liveVolatileRegs(ins);
+ masm.PopRegsInMask(regs);
+}
+
+inline bool CodeGeneratorShared::isGlobalObject(JSObject* object) {
+ // Calling object->is<GlobalObject>() is racy because this relies on
+ // checking the group and this can be changed while we are compiling off the
+ // main thread. Note that we only check for the script realm's global here.
+ return object == gen->realm->maybeGlobal();
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_shared_CodeGenerator_shared_inl_h */
diff --git a/js/src/jit/shared/CodeGenerator-shared.cpp b/js/src/jit/shared/CodeGenerator-shared.cpp
new file mode 100644
index 0000000000..84ee2aeb26
--- /dev/null
+++ b/js/src/jit/shared/CodeGenerator-shared.cpp
@@ -0,0 +1,983 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/shared/CodeGenerator-shared-inl.h"
+
+#include "mozilla/DebugOnly.h"
+
+#include <utility>
+
+#include "jit/CodeGenerator.h"
+#include "jit/CompactBuffer.h"
+#include "jit/CompileInfo.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitcodeMap.h"
+#include "jit/JitFrames.h"
+#include "jit/JitSpewer.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+#include "jit/SafepointIndex.h"
+#include "js/Conversions.h"
+#include "util/Memory.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::BitwiseCast;
+using mozilla::DebugOnly;
+
+namespace js {
+namespace jit {
+
+MacroAssembler& CodeGeneratorShared::ensureMasm(MacroAssembler* masmArg,
+ TempAllocator& alloc,
+ CompileRealm* realm) {
+ if (masmArg) {
+ return *masmArg;
+ }
+ maybeMasm_.emplace(alloc, realm);
+ return *maybeMasm_;
+}
+
+CodeGeneratorShared::CodeGeneratorShared(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masmArg)
+ : maybeMasm_(),
+ masm(ensureMasm(masmArg, gen->alloc(), gen->realm)),
+ gen(gen),
+ graph(*graph),
+ current(nullptr),
+ snapshots_(),
+ recovers_(),
+#ifdef DEBUG
+ pushedArgs_(0),
+#endif
+ lastOsiPointOffset_(0),
+ safepoints_(graph->localSlotsSize(),
+ (gen->outerInfo().nargs() + 1) * sizeof(Value)),
+ returnLabel_(),
+ nativeToBytecodeMap_(nullptr),
+ nativeToBytecodeMapSize_(0),
+ nativeToBytecodeTableOffset_(0),
+#ifdef CHECK_OSIPOINT_REGISTERS
+ checkOsiPointRegisters(JitOptions.checkOsiPointRegisters),
+#endif
+ frameDepth_(0) {
+ if (gen->isProfilerInstrumentationEnabled()) {
+ masm.enableProfilingInstrumentation();
+ }
+
+ if (gen->compilingWasm()) {
+ offsetOfArgsFromFP_ = sizeof(wasm::Frame);
+
+#ifdef JS_CODEGEN_ARM64
+ // Ensure SP is aligned to 16 bytes.
+ frameDepth_ = AlignBytes(graph->localSlotsSize(), WasmStackAlignment);
+#else
+ frameDepth_ = AlignBytes(graph->localSlotsSize(), sizeof(uintptr_t));
+#endif
+
+#ifdef ENABLE_WASM_SIMD
+# if defined(JS_CODEGEN_X64) || defined(JS_CODEGEN_X86) || \
+ defined(JS_CODEGEN_ARM64)
+ // On X64/x86 and ARM64, we don't need alignment for Wasm SIMD at this time.
+# else
+# error \
+ "we may need padding so that local slots are SIMD-aligned and the stack must be kept SIMD-aligned too."
+# endif
+#endif
+
+ if (gen->needsStaticStackAlignment()) {
+ // Since wasm uses the system ABI which does not necessarily use a
+ // regular array where all slots are sizeof(Value), it maintains the max
+ // argument stack depth separately.
+ MOZ_ASSERT(graph->argumentSlotCount() == 0);
+ frameDepth_ += gen->wasmMaxStackArgBytes();
+
+ // An MWasmCall does not align the stack pointer at calls sites but
+ // instead relies on the a priori stack adjustment. This must be the
+ // last adjustment of frameDepth_.
+ frameDepth_ += ComputeByteAlignment(sizeof(wasm::Frame) + frameDepth_,
+ WasmStackAlignment);
+ }
+
+#ifdef JS_CODEGEN_ARM64
+ MOZ_ASSERT((frameDepth_ % WasmStackAlignment) == 0,
+ "Trap exit stub needs 16-byte aligned stack pointer");
+#endif
+ } else {
+ offsetOfArgsFromFP_ = sizeof(JitFrameLayout);
+
+ // Allocate space for local slots (register allocator spills). Round to
+ // JitStackAlignment, and implicitly to sizeof(Value) as JitStackAlignment
+ // is a multiple of sizeof(Value). This was originally implemented for
+ // SIMD.js, but now lets us use faster ABI calls via setupAlignedABICall.
+ frameDepth_ = AlignBytes(graph->localSlotsSize(), JitStackAlignment);
+
+ // Allocate space for argument Values passed to callee functions.
+ offsetOfPassedArgSlots_ = frameDepth_;
+ MOZ_ASSERT((offsetOfPassedArgSlots_ % sizeof(JS::Value)) == 0);
+ frameDepth_ += graph->argumentSlotCount() * sizeof(JS::Value);
+
+ MOZ_ASSERT((frameDepth_ % JitStackAlignment) == 0);
+ }
+}
+
+bool CodeGeneratorShared::generatePrologue() {
+ MOZ_ASSERT(masm.framePushed() == 0);
+ MOZ_ASSERT(!gen->compilingWasm());
+
+#ifdef JS_USE_LINK_REGISTER
+ masm.pushReturnAddress();
+#endif
+
+ // Frame prologue.
+ masm.push(FramePointer);
+ masm.moveStackPtrTo(FramePointer);
+
+ // Ensure that the Ion frame is properly aligned.
+ masm.assertStackAlignment(JitStackAlignment, 0);
+
+ // If profiling, save the current frame pointer to a per-thread global field.
+ if (isProfilerInstrumentationEnabled()) {
+ masm.profilerEnterFrame(FramePointer, CallTempReg0);
+ }
+
+ // Note that this automatically sets MacroAssembler::framePushed().
+ masm.reserveStack(frameSize());
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+ masm.checkStackAlignment();
+
+ return true;
+}
+
+bool CodeGeneratorShared::generateEpilogue() {
+ MOZ_ASSERT(!gen->compilingWasm());
+ masm.bind(&returnLabel_);
+
+ // If profiling, jump to a trampoline to reset the JitActivation's
+ // lastProfilingFrame to point to the previous frame and return to the caller.
+ if (isProfilerInstrumentationEnabled()) {
+ masm.profilerExitFrame();
+ }
+
+ MOZ_ASSERT(masm.framePushed() == frameSize());
+ masm.moveToStackPtr(FramePointer);
+ masm.pop(FramePointer);
+ masm.setFramePushed(0);
+
+ masm.ret();
+
+ // On systems that use a constant pool, this is a good time to emit.
+ masm.flushBuffer();
+ return true;
+}
+
+bool CodeGeneratorShared::generateOutOfLineCode() {
+ AutoCreatedBy acb(masm, "CodeGeneratorShared::generateOutOfLineCode");
+
+ // OOL paths should not attempt to use |current| as it's the last block
+ // instead of the block corresponding to the OOL path.
+ current = nullptr;
+
+ for (size_t i = 0; i < outOfLineCode_.length(); i++) {
+ // Add native => bytecode mapping entries for OOL sites.
+ // Not enabled on wasm yet since it doesn't contain bytecode mappings.
+ if (!gen->compilingWasm()) {
+ if (!addNativeToBytecodeEntry(outOfLineCode_[i]->bytecodeSite())) {
+ return false;
+ }
+ }
+
+ if (!gen->alloc().ensureBallast()) {
+ return false;
+ }
+
+ JitSpew(JitSpew_Codegen, "# Emitting out of line code");
+
+ masm.setFramePushed(outOfLineCode_[i]->framePushed());
+ outOfLineCode_[i]->bind(&masm);
+
+ outOfLineCode_[i]->generate(this);
+ }
+
+ return !masm.oom();
+}
+
+void CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code,
+ const MInstruction* mir) {
+ MOZ_ASSERT(mir);
+ addOutOfLineCode(code, mir->trackedSite());
+}
+
+void CodeGeneratorShared::addOutOfLineCode(OutOfLineCode* code,
+ const BytecodeSite* site) {
+ MOZ_ASSERT_IF(!gen->compilingWasm(), site->script()->containsPC(site->pc()));
+ code->setFramePushed(masm.framePushed());
+ code->setBytecodeSite(site);
+ masm.propagateOOM(outOfLineCode_.append(code));
+}
+
+bool CodeGeneratorShared::addNativeToBytecodeEntry(const BytecodeSite* site) {
+ MOZ_ASSERT(site);
+ MOZ_ASSERT(site->tree());
+ MOZ_ASSERT(site->pc());
+
+ // Skip the table entirely if profiling is not enabled.
+ if (!isProfilerInstrumentationEnabled()) {
+ return true;
+ }
+
+ // Fails early if the last added instruction caused the macro assembler to
+ // run out of memory as continuity assumption below do not hold.
+ if (masm.oom()) {
+ return false;
+ }
+
+ InlineScriptTree* tree = site->tree();
+ jsbytecode* pc = site->pc();
+ uint32_t nativeOffset = masm.currentOffset();
+
+ MOZ_ASSERT_IF(nativeToBytecodeList_.empty(), nativeOffset == 0);
+
+ if (!nativeToBytecodeList_.empty()) {
+ size_t lastIdx = nativeToBytecodeList_.length() - 1;
+ NativeToBytecode& lastEntry = nativeToBytecodeList_[lastIdx];
+
+ MOZ_ASSERT(nativeOffset >= lastEntry.nativeOffset.offset());
+
+ // If the new entry is for the same inlineScriptTree and same
+ // bytecodeOffset, but the nativeOffset has changed, do nothing.
+ // The same site just generated some more code.
+ if (lastEntry.tree == tree && lastEntry.pc == pc) {
+ JitSpew(JitSpew_Profiling, " => In-place update [%zu-%" PRIu32 "]",
+ lastEntry.nativeOffset.offset(), nativeOffset);
+ return true;
+ }
+
+ // If the new entry is for the same native offset, then update the
+ // previous entry with the new bytecode site, since the previous
+ // bytecode site did not generate any native code.
+ if (lastEntry.nativeOffset.offset() == nativeOffset) {
+ lastEntry.tree = tree;
+ lastEntry.pc = pc;
+ JitSpew(JitSpew_Profiling, " => Overwriting zero-length native region.");
+
+ // This overwrite might have made the entry merge-able with a
+ // previous one. If so, merge it.
+ if (lastIdx > 0) {
+ NativeToBytecode& nextToLastEntry = nativeToBytecodeList_[lastIdx - 1];
+ if (nextToLastEntry.tree == lastEntry.tree &&
+ nextToLastEntry.pc == lastEntry.pc) {
+ JitSpew(JitSpew_Profiling, " => Merging with previous region");
+ nativeToBytecodeList_.erase(&lastEntry);
+ }
+ }
+
+ dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1);
+ return true;
+ }
+ }
+
+ // Otherwise, some native code was generated for the previous bytecode site.
+ // Add a new entry for code that is about to be generated.
+ NativeToBytecode entry;
+ entry.nativeOffset = CodeOffset(nativeOffset);
+ entry.tree = tree;
+ entry.pc = pc;
+ if (!nativeToBytecodeList_.append(entry)) {
+ return false;
+ }
+
+ JitSpew(JitSpew_Profiling, " => Push new entry.");
+ dumpNativeToBytecodeEntry(nativeToBytecodeList_.length() - 1);
+ return true;
+}
+
+void CodeGeneratorShared::dumpNativeToBytecodeEntries() {
+#ifdef JS_JITSPEW
+ InlineScriptTree* topTree = gen->outerInfo().inlineScriptTree();
+ JitSpewStart(JitSpew_Profiling, "Native To Bytecode Entries for %s:%u:%u\n",
+ topTree->script()->filename(), topTree->script()->lineno(),
+ topTree->script()->column());
+ for (unsigned i = 0; i < nativeToBytecodeList_.length(); i++) {
+ dumpNativeToBytecodeEntry(i);
+ }
+#endif
+}
+
+void CodeGeneratorShared::dumpNativeToBytecodeEntry(uint32_t idx) {
+#ifdef JS_JITSPEW
+ NativeToBytecode& ref = nativeToBytecodeList_[idx];
+ InlineScriptTree* tree = ref.tree;
+ JSScript* script = tree->script();
+ uint32_t nativeOffset = ref.nativeOffset.offset();
+ unsigned nativeDelta = 0;
+ unsigned pcDelta = 0;
+ if (idx + 1 < nativeToBytecodeList_.length()) {
+ NativeToBytecode* nextRef = &ref + 1;
+ nativeDelta = nextRef->nativeOffset.offset() - nativeOffset;
+ if (nextRef->tree == ref.tree) {
+ pcDelta = nextRef->pc - ref.pc;
+ }
+ }
+ JitSpewStart(
+ JitSpew_Profiling, " %08zx [+%-6u] => %-6ld [%-4u] {%-10s} (%s:%u:%u",
+ ref.nativeOffset.offset(), nativeDelta, (long)(ref.pc - script->code()),
+ pcDelta, CodeName(JSOp(*ref.pc)), script->filename(), script->lineno(),
+ script->column());
+
+ for (tree = tree->caller(); tree; tree = tree->caller()) {
+ JitSpewCont(JitSpew_Profiling, " <= %s:%u:%u", tree->script()->filename(),
+ tree->script()->lineno(), tree->script()->column());
+ }
+ JitSpewCont(JitSpew_Profiling, ")");
+ JitSpewFin(JitSpew_Profiling);
+#endif
+}
+
+// see OffsetOfFrameSlot
+static inline int32_t ToStackIndex(LAllocation* a) {
+ if (a->isStackSlot()) {
+ MOZ_ASSERT(a->toStackSlot()->slot() >= 1);
+ return a->toStackSlot()->slot();
+ }
+ return -int32_t(sizeof(JitFrameLayout) + a->toArgument()->index());
+}
+
+void CodeGeneratorShared::encodeAllocation(LSnapshot* snapshot,
+ MDefinition* mir,
+ uint32_t* allocIndex) {
+ if (mir->isBox()) {
+ mir = mir->toBox()->getOperand(0);
+ }
+
+ MIRType type = mir->isRecoveredOnBailout() ? MIRType::None
+ : mir->isUnused() ? MIRType::MagicOptimizedOut
+ : mir->type();
+
+ RValueAllocation alloc;
+
+ switch (type) {
+ case MIRType::None: {
+ MOZ_ASSERT(mir->isRecoveredOnBailout());
+ uint32_t index = 0;
+ LRecoverInfo* recoverInfo = snapshot->recoverInfo();
+ MNode** it = recoverInfo->begin();
+ MNode** end = recoverInfo->end();
+ while (it != end && mir != *it) {
+ ++it;
+ ++index;
+ }
+
+ // This MDefinition is recovered, thus it should be listed in the
+ // LRecoverInfo.
+ MOZ_ASSERT(it != end && mir == *it);
+
+ // Lambda should have a default value readable for iterating over the
+ // inner frames.
+ MConstant* functionOperand = nullptr;
+ if (mir->isLambda()) {
+ functionOperand = mir->toLambda()->functionOperand();
+ } else if (mir->isFunctionWithProto()) {
+ functionOperand = mir->toFunctionWithProto()->functionOperand();
+ }
+ if (functionOperand) {
+ uint32_t cstIndex;
+ masm.propagateOOM(
+ graph.addConstantToPool(functionOperand->toJSValue(), &cstIndex));
+ alloc = RValueAllocation::RecoverInstruction(index, cstIndex);
+ break;
+ }
+
+ alloc = RValueAllocation::RecoverInstruction(index);
+ break;
+ }
+ case MIRType::Undefined:
+ alloc = RValueAllocation::Undefined();
+ break;
+ case MIRType::Null:
+ alloc = RValueAllocation::Null();
+ break;
+ case MIRType::Int32:
+ case MIRType::String:
+ case MIRType::Symbol:
+ case MIRType::BigInt:
+ case MIRType::Object:
+ case MIRType::Shape:
+ case MIRType::Boolean:
+ case MIRType::Double: {
+ LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
+ if (payload->isConstant()) {
+ MConstant* constant = mir->toConstant();
+ uint32_t index;
+ masm.propagateOOM(
+ graph.addConstantToPool(constant->toJSValue(), &index));
+ alloc = RValueAllocation::ConstantPool(index);
+ break;
+ }
+
+ JSValueType valueType = ValueTypeFromMIRType(type);
+
+ MOZ_DIAGNOSTIC_ASSERT(payload->isMemory() || payload->isRegister());
+ if (payload->isMemory()) {
+ alloc = RValueAllocation::Typed(valueType, ToStackIndex(payload));
+ } else if (payload->isGeneralReg()) {
+ alloc = RValueAllocation::Typed(valueType, ToRegister(payload));
+ } else if (payload->isFloatReg()) {
+ alloc = RValueAllocation::Double(ToFloatRegister(payload));
+ } else {
+ MOZ_CRASH("Unexpected payload type.");
+ }
+ break;
+ }
+ case MIRType::Float32:
+ case MIRType::Simd128: {
+ LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
+ if (payload->isConstant()) {
+ MConstant* constant = mir->toConstant();
+ uint32_t index;
+ masm.propagateOOM(
+ graph.addConstantToPool(constant->toJSValue(), &index));
+ alloc = RValueAllocation::ConstantPool(index);
+ break;
+ }
+
+ MOZ_ASSERT(payload->isMemory() || payload->isFloatReg());
+ if (payload->isFloatReg()) {
+ alloc = RValueAllocation::AnyFloat(ToFloatRegister(payload));
+ } else {
+ alloc = RValueAllocation::AnyFloat(ToStackIndex(payload));
+ }
+ break;
+ }
+ case MIRType::MagicOptimizedOut:
+ case MIRType::MagicUninitializedLexical:
+ case MIRType::MagicIsConstructing: {
+ uint32_t index;
+ JSWhyMagic why = JS_GENERIC_MAGIC;
+ switch (type) {
+ case MIRType::MagicOptimizedOut:
+ why = JS_OPTIMIZED_OUT;
+ break;
+ case MIRType::MagicUninitializedLexical:
+ why = JS_UNINITIALIZED_LEXICAL;
+ break;
+ case MIRType::MagicIsConstructing:
+ why = JS_IS_CONSTRUCTING;
+ break;
+ default:
+ MOZ_CRASH("Invalid Magic MIRType");
+ }
+
+ Value v = MagicValue(why);
+ masm.propagateOOM(graph.addConstantToPool(v, &index));
+ alloc = RValueAllocation::ConstantPool(index);
+ break;
+ }
+ default: {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+ LAllocation* payload = snapshot->payloadOfSlot(*allocIndex);
+#ifdef JS_NUNBOX32
+ LAllocation* type = snapshot->typeOfSlot(*allocIndex);
+ if (type->isRegister()) {
+ if (payload->isRegister()) {
+ alloc =
+ RValueAllocation::Untyped(ToRegister(type), ToRegister(payload));
+ } else {
+ alloc = RValueAllocation::Untyped(ToRegister(type),
+ ToStackIndex(payload));
+ }
+ } else {
+ if (payload->isRegister()) {
+ alloc = RValueAllocation::Untyped(ToStackIndex(type),
+ ToRegister(payload));
+ } else {
+ alloc = RValueAllocation::Untyped(ToStackIndex(type),
+ ToStackIndex(payload));
+ }
+ }
+#elif JS_PUNBOX64
+ if (payload->isRegister()) {
+ alloc = RValueAllocation::Untyped(ToRegister(payload));
+ } else {
+ alloc = RValueAllocation::Untyped(ToStackIndex(payload));
+ }
+#endif
+ break;
+ }
+ }
+ MOZ_DIAGNOSTIC_ASSERT(alloc.valid());
+
+ // This set an extra bit as part of the RValueAllocation, such that we know
+ // that recover instruction have to be executed without wrapping the
+ // instruction in a no-op recover instruction.
+ if (mir->isIncompleteObject()) {
+ alloc.setNeedSideEffect();
+ }
+
+ masm.propagateOOM(snapshots_.add(alloc));
+
+ *allocIndex += mir->isRecoveredOnBailout() ? 0 : 1;
+}
+
+void CodeGeneratorShared::encode(LRecoverInfo* recover) {
+ if (recover->recoverOffset() != INVALID_RECOVER_OFFSET) {
+ return;
+ }
+
+ uint32_t numInstructions = recover->numInstructions();
+ JitSpew(JitSpew_IonSnapshots,
+ "Encoding LRecoverInfo %p (frameCount %u, instructions %u)",
+ (void*)recover, recover->mir()->frameCount(), numInstructions);
+
+ RecoverOffset offset = recovers_.startRecover(numInstructions);
+
+ for (MNode* insn : *recover) {
+ recovers_.writeInstruction(insn);
+ }
+
+ recovers_.endRecover();
+ recover->setRecoverOffset(offset);
+ masm.propagateOOM(!recovers_.oom());
+}
+
+void CodeGeneratorShared::encode(LSnapshot* snapshot) {
+ if (snapshot->snapshotOffset() != INVALID_SNAPSHOT_OFFSET) {
+ return;
+ }
+
+ LRecoverInfo* recoverInfo = snapshot->recoverInfo();
+ encode(recoverInfo);
+
+ RecoverOffset recoverOffset = recoverInfo->recoverOffset();
+ MOZ_ASSERT(recoverOffset != INVALID_RECOVER_OFFSET);
+
+ JitSpew(JitSpew_IonSnapshots, "Encoding LSnapshot %p (LRecover %p)",
+ (void*)snapshot, (void*)recoverInfo);
+
+ SnapshotOffset offset =
+ snapshots_.startSnapshot(recoverOffset, snapshot->bailoutKind());
+
+#ifdef TRACK_SNAPSHOTS
+ uint32_t pcOpcode = 0;
+ uint32_t lirOpcode = 0;
+ uint32_t lirId = 0;
+ uint32_t mirOpcode = 0;
+ uint32_t mirId = 0;
+
+ if (LInstruction* ins = instruction()) {
+ lirOpcode = uint32_t(ins->op());
+ lirId = ins->id();
+ if (MDefinition* mir = ins->mirRaw()) {
+ mirOpcode = uint32_t(mir->op());
+ mirId = mir->id();
+ if (jsbytecode* pc = mir->trackedSite()->pc()) {
+ pcOpcode = *pc;
+ }
+ }
+ }
+ snapshots_.trackSnapshot(pcOpcode, mirOpcode, mirId, lirOpcode, lirId);
+#endif
+
+ uint32_t allocIndex = 0;
+ for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) {
+ DebugOnly<uint32_t> allocWritten = snapshots_.allocWritten();
+ encodeAllocation(snapshot, *it, &allocIndex);
+ MOZ_ASSERT_IF(!snapshots_.oom(),
+ allocWritten + 1 == snapshots_.allocWritten());
+ }
+
+ MOZ_ASSERT(allocIndex == snapshot->numSlots());
+ snapshots_.endSnapshot();
+ snapshot->setSnapshotOffset(offset);
+ masm.propagateOOM(!snapshots_.oom());
+}
+
+bool CodeGeneratorShared::encodeSafepoints() {
+ for (CodegenSafepointIndex& index : safepointIndices_) {
+ LSafepoint* safepoint = index.safepoint();
+
+ if (!safepoint->encoded()) {
+ safepoints_.encode(safepoint);
+ }
+ }
+
+ return !safepoints_.oom();
+}
+
+bool CodeGeneratorShared::createNativeToBytecodeScriptList(
+ JSContext* cx, IonEntry::ScriptList& scripts) {
+ MOZ_ASSERT(scripts.empty());
+
+ InlineScriptTree* tree = gen->outerInfo().inlineScriptTree();
+ for (;;) {
+ // Add script from current tree.
+ bool found = false;
+ for (uint32_t i = 0; i < scripts.length(); i++) {
+ if (scripts[i].script == tree->script()) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ UniqueChars str =
+ GeckoProfilerRuntime::allocProfileString(cx, tree->script());
+ if (!str) {
+ return false;
+ }
+ if (!scripts.emplaceBack(tree->script(), std::move(str))) {
+ return false;
+ }
+ }
+
+ // Process rest of tree
+
+ // If children exist, emit children.
+ if (tree->hasChildren()) {
+ tree = tree->firstChild();
+ continue;
+ }
+
+ // Otherwise, find the first tree up the chain (including this one)
+ // that contains a next sibling.
+ while (!tree->hasNextCallee() && tree->hasCaller()) {
+ tree = tree->caller();
+ }
+
+ // If we found a sibling, use it.
+ if (tree->hasNextCallee()) {
+ tree = tree->nextCallee();
+ continue;
+ }
+
+ // Otherwise, we must have reached the top without finding any siblings.
+ MOZ_ASSERT(tree->isOutermostCaller());
+ break;
+ }
+
+ return true;
+}
+
+bool CodeGeneratorShared::generateCompactNativeToBytecodeMap(
+ JSContext* cx, JitCode* code, IonEntry::ScriptList& scripts) {
+ MOZ_ASSERT(nativeToBytecodeMap_ == nullptr);
+ MOZ_ASSERT(nativeToBytecodeMapSize_ == 0);
+ MOZ_ASSERT(nativeToBytecodeTableOffset_ == 0);
+
+ if (!createNativeToBytecodeScriptList(cx, scripts)) {
+ return false;
+ }
+
+ CompactBufferWriter writer;
+ uint32_t tableOffset = 0;
+ uint32_t numRegions = 0;
+
+ if (!JitcodeIonTable::WriteIonTable(
+ writer, scripts, &nativeToBytecodeList_[0],
+ &nativeToBytecodeList_[0] + nativeToBytecodeList_.length(),
+ &tableOffset, &numRegions)) {
+ return false;
+ }
+
+ MOZ_ASSERT(tableOffset > 0);
+ MOZ_ASSERT(numRegions > 0);
+
+ // Writer is done, copy it to sized buffer.
+ uint8_t* data = cx->pod_malloc<uint8_t>(writer.length());
+ if (!data) {
+ return false;
+ }
+
+ memcpy(data, writer.buffer(), writer.length());
+ nativeToBytecodeMap_.reset(data);
+ nativeToBytecodeMapSize_ = writer.length();
+ nativeToBytecodeTableOffset_ = tableOffset;
+
+ verifyCompactNativeToBytecodeMap(code, scripts, numRegions);
+
+ JitSpew(JitSpew_Profiling, "Compact Native To Bytecode Map [%p-%p]", data,
+ data + nativeToBytecodeMapSize_);
+
+ return true;
+}
+
+void CodeGeneratorShared::verifyCompactNativeToBytecodeMap(
+ JitCode* code, const IonEntry::ScriptList& scripts, uint32_t numRegions) {
+#ifdef DEBUG
+ MOZ_ASSERT(nativeToBytecodeMap_ != nullptr);
+ MOZ_ASSERT(nativeToBytecodeMapSize_ > 0);
+ MOZ_ASSERT(nativeToBytecodeTableOffset_ > 0);
+ MOZ_ASSERT(numRegions > 0);
+
+ // The pointer to the table must be 4-byte aligned
+ const uint8_t* tablePtr =
+ nativeToBytecodeMap_.get() + nativeToBytecodeTableOffset_;
+ MOZ_ASSERT(uintptr_t(tablePtr) % sizeof(uint32_t) == 0);
+
+ // Verify that numRegions was encoded correctly.
+ const JitcodeIonTable* ionTable =
+ reinterpret_cast<const JitcodeIonTable*>(tablePtr);
+ MOZ_ASSERT(ionTable->numRegions() == numRegions);
+
+ // Region offset for first region should be at the start of the payload
+ // region. Since the offsets are backward from the start of the table, the
+ // first entry backoffset should be equal to the forward table offset from the
+ // start of the allocated data.
+ MOZ_ASSERT(ionTable->regionOffset(0) == nativeToBytecodeTableOffset_);
+
+ // Verify each region.
+ for (uint32_t i = 0; i < ionTable->numRegions(); i++) {
+ // Back-offset must point into the payload region preceding the table, not
+ // before it.
+ MOZ_ASSERT(ionTable->regionOffset(i) <= nativeToBytecodeTableOffset_);
+
+ // Back-offset must point to a later area in the payload region than
+ // previous back-offset. This means that back-offsets decrease
+ // monotonically.
+ MOZ_ASSERT_IF(i > 0,
+ ionTable->regionOffset(i) < ionTable->regionOffset(i - 1));
+
+ JitcodeRegionEntry entry = ionTable->regionEntry(i);
+
+ // Ensure native code offset for region falls within jitcode.
+ MOZ_ASSERT(entry.nativeOffset() <= code->instructionsSize());
+
+ // Read out script/pc stack and verify.
+ JitcodeRegionEntry::ScriptPcIterator scriptPcIter =
+ entry.scriptPcIterator();
+ while (scriptPcIter.hasMore()) {
+ uint32_t scriptIdx = 0, pcOffset = 0;
+ scriptPcIter.readNext(&scriptIdx, &pcOffset);
+
+ // Ensure scriptIdx refers to a valid script in the list.
+ JSScript* script = scripts[scriptIdx].script;
+
+ // Ensure pcOffset falls within the script.
+ MOZ_ASSERT(pcOffset < script->length());
+ }
+
+ // Obtain the original nativeOffset and pcOffset and script.
+ uint32_t curNativeOffset = entry.nativeOffset();
+ JSScript* script = nullptr;
+ uint32_t curPcOffset = 0;
+ {
+ uint32_t scriptIdx = 0;
+ scriptPcIter.reset();
+ scriptPcIter.readNext(&scriptIdx, &curPcOffset);
+ script = scripts[scriptIdx].script;
+ }
+
+ // Read out nativeDeltas and pcDeltas and verify.
+ JitcodeRegionEntry::DeltaIterator deltaIter = entry.deltaIterator();
+ while (deltaIter.hasMore()) {
+ uint32_t nativeDelta = 0;
+ int32_t pcDelta = 0;
+ deltaIter.readNext(&nativeDelta, &pcDelta);
+
+ curNativeOffset += nativeDelta;
+ curPcOffset = uint32_t(int32_t(curPcOffset) + pcDelta);
+
+ // Ensure that nativeOffset still falls within jitcode after delta.
+ MOZ_ASSERT(curNativeOffset <= code->instructionsSize());
+
+ // Ensure that pcOffset still falls within bytecode after delta.
+ MOZ_ASSERT(curPcOffset < script->length());
+ }
+ }
+#endif // DEBUG
+}
+
+void CodeGeneratorShared::markSafepoint(LInstruction* ins) {
+ markSafepointAt(masm.currentOffset(), ins);
+}
+
+void CodeGeneratorShared::markSafepointAt(uint32_t offset, LInstruction* ins) {
+ MOZ_ASSERT_IF(
+ !safepointIndices_.empty() && !masm.oom(),
+ offset - safepointIndices_.back().displacement() >= sizeof(uint32_t));
+ masm.propagateOOM(safepointIndices_.append(
+ CodegenSafepointIndex(offset, ins->safepoint())));
+}
+
+void CodeGeneratorShared::ensureOsiSpace() {
+ // For a refresher, an invalidation point is of the form:
+ // 1: call <target>
+ // 2: ...
+ // 3: <osipoint>
+ //
+ // The four bytes *before* instruction 2 are overwritten with an offset.
+ // Callers must ensure that the instruction itself has enough bytes to
+ // support this.
+ //
+ // The bytes *at* instruction 3 are overwritten with an invalidation jump.
+ // jump. These bytes may be in a completely different IR sequence, but
+ // represent the join point of the call out of the function.
+ //
+ // At points where we want to ensure that invalidation won't corrupt an
+ // important instruction, we make sure to pad with nops.
+ if (masm.currentOffset() - lastOsiPointOffset_ <
+ Assembler::PatchWrite_NearCallSize()) {
+ int32_t paddingSize = Assembler::PatchWrite_NearCallSize();
+ paddingSize -= masm.currentOffset() - lastOsiPointOffset_;
+ for (int32_t i = 0; i < paddingSize; ++i) {
+ masm.nop();
+ }
+ }
+ MOZ_ASSERT_IF(!masm.oom(), masm.currentOffset() - lastOsiPointOffset_ >=
+ Assembler::PatchWrite_NearCallSize());
+ lastOsiPointOffset_ = masm.currentOffset();
+}
+
+uint32_t CodeGeneratorShared::markOsiPoint(LOsiPoint* ins) {
+ encode(ins->snapshot());
+ ensureOsiSpace();
+
+ uint32_t offset = masm.currentOffset();
+ SnapshotOffset so = ins->snapshot()->snapshotOffset();
+ masm.propagateOOM(osiIndices_.append(OsiIndex(offset, so)));
+
+ return offset;
+}
+
+class OutOfLineTruncateSlow : public OutOfLineCodeBase<CodeGeneratorShared> {
+ FloatRegister src_;
+ Register dest_;
+ bool widenFloatToDouble_;
+ wasm::BytecodeOffset bytecodeOffset_;
+ bool preserveInstance_;
+
+ public:
+ OutOfLineTruncateSlow(
+ FloatRegister src, Register dest, bool widenFloatToDouble = false,
+ wasm::BytecodeOffset bytecodeOffset = wasm::BytecodeOffset(),
+ bool preserveInstance = false)
+ : src_(src),
+ dest_(dest),
+ widenFloatToDouble_(widenFloatToDouble),
+ bytecodeOffset_(bytecodeOffset),
+ preserveInstance_(preserveInstance) {}
+
+ void accept(CodeGeneratorShared* codegen) override {
+ codegen->visitOutOfLineTruncateSlow(this);
+ }
+ FloatRegister src() const { return src_; }
+ Register dest() const { return dest_; }
+ bool widenFloatToDouble() const { return widenFloatToDouble_; }
+ bool preserveInstance() const { return preserveInstance_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+};
+
+OutOfLineCode* CodeGeneratorShared::oolTruncateDouble(
+ FloatRegister src, Register dest, MInstruction* mir,
+ wasm::BytecodeOffset bytecodeOffset, bool preserveInstance) {
+ MOZ_ASSERT_IF(IsCompilingWasm(), bytecodeOffset.isValid());
+
+ OutOfLineTruncateSlow* ool = new (alloc()) OutOfLineTruncateSlow(
+ src, dest, /* float32 */ false, bytecodeOffset, preserveInstance);
+ addOutOfLineCode(ool, mir);
+ return ool;
+}
+
+void CodeGeneratorShared::emitTruncateDouble(FloatRegister src, Register dest,
+ MInstruction* mir) {
+ MOZ_ASSERT(mir->isTruncateToInt32() || mir->isWasmBuiltinTruncateToInt32());
+ wasm::BytecodeOffset bytecodeOffset =
+ mir->isTruncateToInt32()
+ ? mir->toTruncateToInt32()->bytecodeOffset()
+ : mir->toWasmBuiltinTruncateToInt32()->bytecodeOffset();
+ OutOfLineCode* ool = oolTruncateDouble(src, dest, mir, bytecodeOffset);
+
+ masm.branchTruncateDoubleMaybeModUint32(src, dest, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGeneratorShared::emitTruncateFloat32(FloatRegister src, Register dest,
+ MInstruction* mir) {
+ MOZ_ASSERT(mir->isTruncateToInt32() || mir->isWasmBuiltinTruncateToInt32());
+ wasm::BytecodeOffset bytecodeOffset =
+ mir->isTruncateToInt32()
+ ? mir->toTruncateToInt32()->bytecodeOffset()
+ : mir->toWasmBuiltinTruncateToInt32()->bytecodeOffset();
+ OutOfLineTruncateSlow* ool = new (alloc())
+ OutOfLineTruncateSlow(src, dest, /* float32 */ true, bytecodeOffset);
+ addOutOfLineCode(ool, mir);
+
+ masm.branchTruncateFloat32MaybeModUint32(src, dest, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGeneratorShared::visitOutOfLineTruncateSlow(
+ OutOfLineTruncateSlow* ool) {
+ FloatRegister src = ool->src();
+ Register dest = ool->dest();
+
+ saveVolatile(dest);
+ masm.outOfLineTruncateSlow(src, dest, ool->widenFloatToDouble(),
+ gen->compilingWasm(), ool->bytecodeOffset());
+ restoreVolatile(dest);
+
+ masm.jump(ool->rejoin());
+}
+
+bool CodeGeneratorShared::omitOverRecursedCheck() const {
+ // If the current function makes no calls (which means it isn't recursive)
+ // and it uses only a small amount of stack space, it doesn't need a
+ // stack overflow check. Note that the actual number here is somewhat
+ // arbitrary, and codegen actually uses small bounded amounts of
+ // additional stack space in some cases too.
+ return frameSize() < MAX_UNCHECKED_LEAF_FRAME_SIZE &&
+ !gen->needsOverrecursedCheck();
+}
+
+void CodeGeneratorShared::emitPreBarrier(Register elements,
+ const LAllocation* index) {
+ if (index->isConstant()) {
+ Address address(elements, ToInt32(index) * sizeof(Value));
+ masm.guardedCallPreBarrier(address, MIRType::Value);
+ } else {
+ BaseObjectElementIndex address(elements, ToRegister(index));
+ masm.guardedCallPreBarrier(address, MIRType::Value);
+ }
+}
+
+void CodeGeneratorShared::emitPreBarrier(Address address) {
+ masm.guardedCallPreBarrier(address, MIRType::Value);
+}
+
+void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir) {
+ // Skip past trivial blocks.
+ mir = skipTrivialBlocks(mir);
+
+ // No jump necessary if we can fall through to the next block.
+ if (isNextBlock(mir->lir())) {
+ return;
+ }
+
+ masm.jump(mir->lir()->label());
+}
+
+Label* CodeGeneratorShared::getJumpLabelForBranch(MBasicBlock* block) {
+ // Skip past trivial blocks.
+ return skipTrivialBlocks(block)->lir()->label();
+}
+
+// This function is not used for MIPS/MIPS64/LOONG64. They have
+// branchToBlock.
+#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64) && \
+ !defined(JS_CODEGEN_LOONG64)
+void CodeGeneratorShared::jumpToBlock(MBasicBlock* mir,
+ Assembler::Condition cond) {
+ // Skip past trivial blocks.
+ masm.j(cond, skipTrivialBlocks(mir)->lir()->label());
+}
+#endif
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/shared/CodeGenerator-shared.h b/js/src/jit/shared/CodeGenerator-shared.h
new file mode 100644
index 0000000000..b956fd7e84
--- /dev/null
+++ b/js/src/jit/shared/CodeGenerator-shared.h
@@ -0,0 +1,488 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_CodeGenerator_shared_h
+#define jit_shared_CodeGenerator_shared_h
+
+#include "mozilla/Alignment.h"
+
+#include <utility>
+
+#include "jit/InlineScriptTree.h"
+#include "jit/JitcodeMap.h"
+#include "jit/LIR.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MIRGenerator.h"
+#include "jit/MIRGraph.h"
+#include "jit/SafepointIndex.h"
+#include "jit/Safepoints.h"
+#include "jit/Snapshots.h"
+
+namespace js {
+namespace jit {
+
+class OutOfLineCode;
+class CodeGenerator;
+class MacroAssembler;
+class IonIC;
+
+class OutOfLineTruncateSlow;
+
+class CodeGeneratorShared : public LElementVisitor {
+ js::Vector<OutOfLineCode*, 0, SystemAllocPolicy> outOfLineCode_;
+
+ MacroAssembler& ensureMasm(MacroAssembler* masm, TempAllocator& alloc,
+ CompileRealm* realm);
+ mozilla::Maybe<IonHeapMacroAssembler> maybeMasm_;
+
+ public:
+ MacroAssembler& masm;
+
+ protected:
+ MIRGenerator* gen;
+ LIRGraph& graph;
+ LBlock* current;
+ SnapshotWriter snapshots_;
+ RecoverWriter recovers_;
+#ifdef DEBUG
+ uint32_t pushedArgs_;
+#endif
+ uint32_t lastOsiPointOffset_;
+ SafepointWriter safepoints_;
+ Label invalidate_;
+ CodeOffset invalidateEpilogueData_;
+
+ // Label for the common return path.
+ NonAssertingLabel returnLabel_;
+
+ js::Vector<CodegenSafepointIndex, 0, SystemAllocPolicy> safepointIndices_;
+ js::Vector<OsiIndex, 0, SystemAllocPolicy> osiIndices_;
+
+ // Allocated data space needed at runtime.
+ js::Vector<uint8_t, 0, SystemAllocPolicy> runtimeData_;
+
+ // Vector mapping each IC index to its offset in runtimeData_.
+ js::Vector<uint32_t, 0, SystemAllocPolicy> icList_;
+
+ // IC data we need at compile-time. Discarded after creating the IonScript.
+ struct CompileTimeICInfo {
+ CodeOffset icOffsetForJump;
+ CodeOffset icOffsetForPush;
+ };
+ js::Vector<CompileTimeICInfo, 0, SystemAllocPolicy> icInfo_;
+
+ protected:
+ js::Vector<NativeToBytecode, 0, SystemAllocPolicy> nativeToBytecodeList_;
+ UniquePtr<uint8_t> nativeToBytecodeMap_;
+ uint32_t nativeToBytecodeMapSize_;
+ uint32_t nativeToBytecodeTableOffset_;
+
+ bool isProfilerInstrumentationEnabled() {
+ return gen->isProfilerInstrumentationEnabled();
+ }
+
+ gc::InitialHeap initialStringHeap() const { return gen->initialStringHeap(); }
+ gc::InitialHeap initialBigIntHeap() const { return gen->initialBigIntHeap(); }
+
+ protected:
+ // The offset of the first instruction of the OSR entry block from the
+ // beginning of the code buffer.
+ mozilla::Maybe<size_t> osrEntryOffset_ = {};
+
+ TempAllocator& alloc() const { return graph.mir().alloc(); }
+
+ void setOsrEntryOffset(size_t offset) { osrEntryOffset_.emplace(offset); }
+
+ size_t getOsrEntryOffset() const {
+ MOZ_RELEASE_ASSERT(osrEntryOffset_.isSome());
+ return *osrEntryOffset_;
+ }
+
+ typedef js::Vector<CodegenSafepointIndex, 8, SystemAllocPolicy>
+ SafepointIndices;
+
+ protected:
+#ifdef CHECK_OSIPOINT_REGISTERS
+ // See JitOptions.checkOsiPointRegisters. We set this here to avoid
+ // races when enableOsiPointRegisterChecks is called while we're generating
+ // code off-thread.
+ bool checkOsiPointRegisters;
+#endif
+
+ // The initial size of the frame in bytes. These are bytes beyond the
+ // constant header present for every Ion frame, used for pre-determined
+ // spills.
+ uint32_t frameDepth_;
+
+ // Offset in bytes to the incoming arguments, relative to the frame pointer.
+ uint32_t offsetOfArgsFromFP_ = 0;
+
+ // Offset in bytes of the stack region reserved for passed argument Values.
+ uint32_t offsetOfPassedArgSlots_ = 0;
+
+ // For argument construction for calls. Argslots are Value-sized.
+ inline Address AddressOfPassedArg(uint32_t slot) const;
+ inline uint32_t UnusedStackBytesForCall(uint32_t numArgSlots) const;
+
+ template <BaseRegForAddress Base = BaseRegForAddress::Default>
+ inline Address ToAddress(const LAllocation& a) const;
+
+ template <BaseRegForAddress Base = BaseRegForAddress::Default>
+ inline Address ToAddress(const LAllocation* a) const;
+
+ static inline Address ToAddress(Register elements, const LAllocation* index,
+ Scalar::Type type,
+ int32_t offsetAdjustment = 0);
+
+ uint32_t frameSize() const { return frameDepth_; }
+
+ protected:
+ bool addNativeToBytecodeEntry(const BytecodeSite* site);
+ void dumpNativeToBytecodeEntries();
+ void dumpNativeToBytecodeEntry(uint32_t idx);
+
+ public:
+ MIRGenerator& mirGen() const { return *gen; }
+
+ // When appending to runtimeData_, the vector might realloc, leaving pointers
+ // int the origianl vector stale and unusable. DataPtr acts like a pointer,
+ // but allows safety in the face of potentially realloc'ing vector appends.
+ friend class DataPtr;
+ template <typename T>
+ class DataPtr {
+ CodeGeneratorShared* cg_;
+ size_t index_;
+
+ T* lookup() { return reinterpret_cast<T*>(&cg_->runtimeData_[index_]); }
+
+ public:
+ DataPtr(CodeGeneratorShared* cg, size_t index) : cg_(cg), index_(index) {}
+
+ T* operator->() { return lookup(); }
+ T* operator*() { return lookup(); }
+ };
+
+ protected:
+ [[nodiscard]] bool allocateData(size_t size, size_t* offset) {
+ MOZ_ASSERT(size % sizeof(void*) == 0);
+ *offset = runtimeData_.length();
+ masm.propagateOOM(runtimeData_.appendN(0, size));
+ return !masm.oom();
+ }
+
+ template <typename T>
+ inline size_t allocateIC(const T& cache) {
+ static_assert(std::is_base_of_v<IonIC, T>, "T must inherit from IonIC");
+ size_t index;
+ masm.propagateOOM(
+ allocateData(sizeof(mozilla::AlignedStorage2<T>), &index));
+ masm.propagateOOM(icList_.append(index));
+ masm.propagateOOM(icInfo_.append(CompileTimeICInfo()));
+ if (masm.oom()) {
+ return SIZE_MAX;
+ }
+ // Use the copy constructor on the allocated space.
+ MOZ_ASSERT(index == icList_.back());
+ new (&runtimeData_[index]) T(cache);
+ return index;
+ }
+
+ protected:
+ // Encodes an LSnapshot into the compressed snapshot buffer.
+ void encode(LRecoverInfo* recover);
+ void encode(LSnapshot* snapshot);
+ void encodeAllocation(LSnapshot* snapshot, MDefinition* def,
+ uint32_t* startIndex);
+
+ // Encode all encountered safepoints in CG-order, and resolve |indices| for
+ // safepoint offsets.
+ bool encodeSafepoints();
+
+ // Fixup offsets of native-to-bytecode map.
+ bool createNativeToBytecodeScriptList(JSContext* cx,
+ IonEntry::ScriptList& scripts);
+ bool generateCompactNativeToBytecodeMap(JSContext* cx, JitCode* code,
+ IonEntry::ScriptList& scripts);
+ void verifyCompactNativeToBytecodeMap(JitCode* code,
+ const IonEntry::ScriptList& scripts,
+ uint32_t numRegions);
+
+ // Mark the safepoint on |ins| as corresponding to the current assembler
+ // location. The location should be just after a call.
+ void markSafepoint(LInstruction* ins);
+ void markSafepointAt(uint32_t offset, LInstruction* ins);
+
+ // Mark the OSI point |ins| as corresponding to the current
+ // assembler location inside the |osiIndices_|. Return the assembler
+ // location for the OSI point return location.
+ uint32_t markOsiPoint(LOsiPoint* ins);
+
+ // Ensure that there is enough room between the last OSI point and the
+ // current instruction, such that:
+ // (1) Invalidation will not overwrite the current instruction, and
+ // (2) Overwriting the current instruction will not overwrite
+ // an invalidation marker.
+ void ensureOsiSpace();
+
+ OutOfLineCode* oolTruncateDouble(
+ FloatRegister src, Register dest, MInstruction* mir,
+ wasm::BytecodeOffset callOffset = wasm::BytecodeOffset(),
+ bool preserveInstance = false);
+ void emitTruncateDouble(FloatRegister src, Register dest, MInstruction* mir);
+ void emitTruncateFloat32(FloatRegister src, Register dest, MInstruction* mir);
+
+ void emitPreBarrier(Register elements, const LAllocation* index);
+ void emitPreBarrier(Address address);
+
+ // We don't emit code for trivial blocks, so if we want to branch to the
+ // given block, and it's trivial, return the ultimate block we should
+ // actually branch directly to.
+ MBasicBlock* skipTrivialBlocks(MBasicBlock* block) {
+ while (block->lir()->isTrivial()) {
+ LGoto* ins = block->lir()->rbegin()->toGoto();
+ MOZ_ASSERT(ins->numSuccessors() == 1);
+ block = ins->getSuccessor(0);
+ }
+ return block;
+ }
+
+ // Test whether the given block can be reached via fallthrough from the
+ // current block.
+ inline bool isNextBlock(LBlock* block) {
+ uint32_t target = skipTrivialBlocks(block->mir())->id();
+ uint32_t i = current->mir()->id() + 1;
+ if (target < i) {
+ return false;
+ }
+ // Trivial blocks can be crossed via fallthrough.
+ for (; i != target; ++i) {
+ if (!graph.getBlock(i)->isTrivial()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ protected:
+ // Save and restore all volatile registers to/from the stack, excluding the
+ // specified register(s), before a function call made using callWithABI and
+ // after storing the function call's return value to an output register.
+ // (The only registers that don't need to be saved/restored are 1) the
+ // temporary register used to store the return value of the function call,
+ // if there is one [otherwise that stored value would be overwritten]; and
+ // 2) temporary registers whose values aren't needed in the rest of the LIR
+ // instruction [this is purely an optimization]. All other volatiles must
+ // be saved and restored in case future LIR instructions need those values.)
+ void saveVolatile(Register output) {
+ LiveRegisterSet regs(RegisterSet::Volatile());
+ regs.takeUnchecked(output);
+ masm.PushRegsInMask(regs);
+ }
+ void restoreVolatile(Register output) {
+ LiveRegisterSet regs(RegisterSet::Volatile());
+ regs.takeUnchecked(output);
+ masm.PopRegsInMask(regs);
+ }
+ void saveVolatile(FloatRegister output) {
+ LiveRegisterSet regs(RegisterSet::Volatile());
+ regs.takeUnchecked(output);
+ masm.PushRegsInMask(regs);
+ }
+ void restoreVolatile(FloatRegister output) {
+ LiveRegisterSet regs(RegisterSet::Volatile());
+ regs.takeUnchecked(output);
+ masm.PopRegsInMask(regs);
+ }
+ void saveVolatile(LiveRegisterSet temps) {
+ masm.PushRegsInMask(LiveRegisterSet(RegisterSet::VolatileNot(temps.set())));
+ }
+ void restoreVolatile(LiveRegisterSet temps) {
+ masm.PopRegsInMask(LiveRegisterSet(RegisterSet::VolatileNot(temps.set())));
+ }
+ void saveVolatile() {
+ masm.PushRegsInMask(LiveRegisterSet(RegisterSet::Volatile()));
+ }
+ void restoreVolatile() {
+ masm.PopRegsInMask(LiveRegisterSet(RegisterSet::Volatile()));
+ }
+
+ // These functions have to be called before and after any callVM and before
+ // any modifications of the stack. Modification of the stack made after
+ // these calls should update the framePushed variable, needed by the exit
+ // frame produced by callVM.
+ inline void saveLive(LInstruction* ins);
+ inline void restoreLive(LInstruction* ins);
+ inline void restoreLiveIgnore(LInstruction* ins, LiveRegisterSet reg);
+
+ // Get/save/restore all registers that are both live and volatile.
+ inline LiveRegisterSet liveVolatileRegs(LInstruction* ins);
+ inline void saveLiveVolatile(LInstruction* ins);
+ inline void restoreLiveVolatile(LInstruction* ins);
+
+ public:
+ template <typename T>
+ void pushArg(const T& t) {
+ masm.Push(t);
+#ifdef DEBUG
+ pushedArgs_++;
+#endif
+ }
+
+ void pushArg(jsid id, Register temp) {
+ masm.Push(id, temp);
+#ifdef DEBUG
+ pushedArgs_++;
+#endif
+ }
+
+ template <typename T>
+ CodeOffset pushArgWithPatch(const T& t) {
+#ifdef DEBUG
+ pushedArgs_++;
+#endif
+ return masm.PushWithPatch(t);
+ }
+
+ void storePointerResultTo(Register reg) { masm.storeCallPointerResult(reg); }
+
+ void storeFloatResultTo(FloatRegister reg) { masm.storeCallFloatResult(reg); }
+
+ template <typename T>
+ void storeResultValueTo(const T& t) {
+ masm.storeCallResultValue(t);
+ }
+
+ protected:
+ void addIC(LInstruction* lir, size_t cacheIndex);
+
+ protected:
+ bool generatePrologue();
+ bool generateEpilogue();
+
+ void addOutOfLineCode(OutOfLineCode* code, const MInstruction* mir);
+ void addOutOfLineCode(OutOfLineCode* code, const BytecodeSite* site);
+ bool generateOutOfLineCode();
+
+ Label* getJumpLabelForBranch(MBasicBlock* block);
+
+ // Generate a jump to the start of the specified block. Use this in place of
+ // jumping directly to mir->lir()->label(), or use getJumpLabelForBranch()
+ // if a label to use directly is needed.
+ void jumpToBlock(MBasicBlock* mir);
+
+// This function is not used for MIPS. MIPS has branchToBlock.
+#if !defined(JS_CODEGEN_MIPS32) && !defined(JS_CODEGEN_MIPS64)
+ void jumpToBlock(MBasicBlock* mir, Assembler::Condition cond);
+#endif
+
+ private:
+ void generateInvalidateEpilogue();
+
+ public:
+ CodeGeneratorShared(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
+
+ public:
+ void visitOutOfLineTruncateSlow(OutOfLineTruncateSlow* ool);
+
+ bool omitOverRecursedCheck() const;
+
+ public:
+ bool isGlobalObject(JSObject* object);
+};
+
+// An out-of-line path is generated at the end of the function.
+class OutOfLineCode : public TempObject {
+ Label entry_;
+ Label rejoin_;
+ uint32_t framePushed_;
+ const BytecodeSite* site_;
+
+ public:
+ OutOfLineCode() : framePushed_(0), site_() {}
+
+ virtual void generate(CodeGeneratorShared* codegen) = 0;
+
+ Label* entry() { return &entry_; }
+ virtual void bind(MacroAssembler* masm) { masm->bind(entry()); }
+ Label* rejoin() { return &rejoin_; }
+ void setFramePushed(uint32_t framePushed) { framePushed_ = framePushed; }
+ uint32_t framePushed() const { return framePushed_; }
+ void setBytecodeSite(const BytecodeSite* site) { site_ = site; }
+ const BytecodeSite* bytecodeSite() const { return site_; }
+};
+
+// For OOL paths that want a specific-typed code generator.
+template <typename T>
+class OutOfLineCodeBase : public OutOfLineCode {
+ public:
+ virtual void generate(CodeGeneratorShared* codegen) override {
+ accept(static_cast<T*>(codegen));
+ }
+
+ public:
+ virtual void accept(T* codegen) = 0;
+};
+
+template <class CodeGen>
+class OutOfLineWasmTruncateCheckBase : public OutOfLineCodeBase<CodeGen> {
+ MIRType fromType_;
+ MIRType toType_;
+ FloatRegister input_;
+ Register output_;
+ Register64 output64_;
+ TruncFlags flags_;
+ wasm::BytecodeOffset bytecodeOffset_;
+
+ public:
+ OutOfLineWasmTruncateCheckBase(MWasmTruncateToInt32* mir, FloatRegister input,
+ Register output)
+ : fromType_(mir->input()->type()),
+ toType_(MIRType::Int32),
+ input_(input),
+ output_(output),
+ output64_(Register64::Invalid()),
+ flags_(mir->flags()),
+ bytecodeOffset_(mir->bytecodeOffset()) {}
+
+ OutOfLineWasmTruncateCheckBase(MWasmBuiltinTruncateToInt64* mir,
+ FloatRegister input, Register64 output)
+ : fromType_(mir->input()->type()),
+ toType_(MIRType::Int64),
+ input_(input),
+ output_(Register::Invalid()),
+ output64_(output),
+ flags_(mir->flags()),
+ bytecodeOffset_(mir->bytecodeOffset()) {}
+
+ OutOfLineWasmTruncateCheckBase(MWasmTruncateToInt64* mir, FloatRegister input,
+ Register64 output)
+ : fromType_(mir->input()->type()),
+ toType_(MIRType::Int64),
+ input_(input),
+ output_(Register::Invalid()),
+ output64_(output),
+ flags_(mir->flags()),
+ bytecodeOffset_(mir->bytecodeOffset()) {}
+
+ void accept(CodeGen* codegen) override {
+ codegen->visitOutOfLineWasmTruncateCheck(this);
+ }
+
+ FloatRegister input() const { return input_; }
+ Register output() const { return output_; }
+ Register64 output64() const { return output64_; }
+ MIRType toType() const { return toType_; }
+ MIRType fromType() const { return fromType_; }
+ bool isUnsigned() const { return flags_ & TRUNC_UNSIGNED; }
+ bool isSaturating() const { return flags_ & TRUNC_SATURATING; }
+ TruncFlags flags() const { return flags_; }
+ wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_shared_CodeGenerator_shared_h */
diff --git a/js/src/jit/shared/Disassembler-shared.cpp b/js/src/jit/shared/Disassembler-shared.cpp
new file mode 100644
index 0000000000..11d062dd53
--- /dev/null
+++ b/js/src/jit/shared/Disassembler-shared.cpp
@@ -0,0 +1,248 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/shared/Disassembler-shared.h"
+
+#include "jit/JitSpewer.h"
+#include "jit/Label.h"
+#include "vm/Printer.h"
+
+using namespace js::jit;
+
+using js::Sprinter;
+
+#ifdef JS_DISASM_SUPPORTED
+// Concurrent assemblers are disambiguated by prefixing every disassembly with a
+// tag that is quasi-unique, and certainly unique enough in realistic cases
+// where we are debugging and looking at disassembler output. The tag is a
+// letter or digit between brackets prefixing the disassembly, eg, [X]. This
+// wraps around every 62 assemblers.
+//
+// When running with --no-threads we can still have concurrent assemblers in the
+// form of nested assemblers, as when an IC stub is created by one assembler
+// while a JS compilation is going on and producing output in another assembler.
+//
+// We generate the tag for an assembler by incrementing a global mod-2^32
+// counter every time a new disassembler is created.
+
+mozilla::Atomic<uint32_t> DisassemblerSpew::counter_(0);
+#endif
+
+DisassemblerSpew::DisassemblerSpew()
+ : printer_(nullptr)
+#ifdef JS_DISASM_SUPPORTED
+ ,
+ labelIndent_(""),
+ targetIndent_(""),
+ spewNext_(1000),
+ nodes_(nullptr),
+ tag_(0)
+#endif
+{
+#ifdef JS_DISASM_SUPPORTED
+ tag_ = counter_++;
+#endif
+}
+
+DisassemblerSpew::~DisassemblerSpew() {
+#ifdef JS_DISASM_SUPPORTED
+ Node* p = nodes_;
+ while (p) {
+ Node* victim = p;
+ p = p->next;
+ js_free(victim);
+ }
+#endif
+}
+
+void DisassemblerSpew::setPrinter(Sprinter* printer) { printer_ = printer; }
+
+bool DisassemblerSpew::isDisabled() {
+ return !(JitSpewEnabled(JitSpew_Codegen) || printer_);
+}
+
+void DisassemblerSpew::spew(const char* fmt, ...) {
+#ifdef JS_DISASM_SUPPORTED
+ static const char prefix_chars[] =
+ "0123456789"
+ "abcdefghijklmnopqrstuvwxyz"
+ "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+ static const char prefix_fmt[] = "[%c] ";
+
+ char fmt2[1024];
+ if (sizeof(fmt2) >= strlen(fmt) + sizeof(prefix_fmt)) {
+ snprintf(fmt2, sizeof(prefix_fmt), prefix_fmt,
+ prefix_chars[tag_ % (sizeof(prefix_chars) - 1)]);
+ strcat(fmt2, fmt);
+ fmt = fmt2;
+ }
+#endif
+
+ va_list args;
+ va_start(args, fmt);
+ spewVA(fmt, args);
+ va_end(args);
+}
+
+void DisassemblerSpew::spewVA(const char* fmt, va_list va) {
+ if (printer_) {
+ printer_->vprintf(fmt, va);
+ printer_->put("\n");
+ }
+ js::jit::JitSpewVA(js::jit::JitSpew_Codegen, fmt, va);
+}
+
+#ifdef JS_DISASM_SUPPORTED
+
+void DisassemblerSpew::setLabelIndent(const char* s) { labelIndent_ = s; }
+
+void DisassemblerSpew::setTargetIndent(const char* s) { targetIndent_ = s; }
+
+DisassemblerSpew::LabelDoc DisassemblerSpew::refLabel(const Label* l) {
+ return l ? LabelDoc(internalResolve(l), l->bound()) : LabelDoc();
+}
+
+void DisassemblerSpew::spewRef(const LabelDoc& target) {
+ if (isDisabled()) {
+ return;
+ }
+ if (!target.valid) {
+ return;
+ }
+ spew("%s-> %d%s", targetIndent_, target.doc, !target.bound ? "f" : "");
+}
+
+void DisassemblerSpew::spewBind(const Label* label) {
+ if (isDisabled()) {
+ return;
+ }
+ uint32_t v = internalResolve(label);
+ Node* probe = lookup(label);
+ if (probe) {
+ probe->bound = true;
+ }
+ spew("%s%d:", labelIndent_, v);
+}
+
+void DisassemblerSpew::spewRetarget(const Label* label, const Label* target) {
+ if (isDisabled()) {
+ return;
+ }
+ LabelDoc labelDoc = LabelDoc(internalResolve(label), label->bound());
+ LabelDoc targetDoc = LabelDoc(internalResolve(target), target->bound());
+ Node* probe = lookup(label);
+ if (probe) {
+ probe->bound = true;
+ }
+ spew("%s%d: .retarget -> %d%s", labelIndent_, labelDoc.doc, targetDoc.doc,
+ !targetDoc.bound ? "f" : "");
+}
+
+void DisassemblerSpew::formatLiteral(const LiteralDoc& doc, char* buffer,
+ size_t bufsize) {
+ switch (doc.type) {
+ case LiteralDoc::Type::Patchable:
+ snprintf(buffer, bufsize, "patchable");
+ break;
+ case LiteralDoc::Type::I32:
+ snprintf(buffer, bufsize, "%d", doc.value.i32);
+ break;
+ case LiteralDoc::Type::U32:
+ snprintf(buffer, bufsize, "%u", doc.value.u32);
+ break;
+ case LiteralDoc::Type::I64:
+ snprintf(buffer, bufsize, "%" PRIi64, doc.value.i64);
+ break;
+ case LiteralDoc::Type::U64:
+ snprintf(buffer, bufsize, "%" PRIu64, doc.value.u64);
+ break;
+ case LiteralDoc::Type::F32:
+ snprintf(buffer, bufsize, "%g", doc.value.f32);
+ break;
+ case LiteralDoc::Type::F64:
+ snprintf(buffer, bufsize, "%g", doc.value.f64);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+void DisassemblerSpew::spewOrphans() {
+ for (Node* p = nodes_; p; p = p->next) {
+ if (!p->bound) {
+ spew("%s%d: ; .orphan", labelIndent_, p->value);
+ }
+ }
+}
+
+uint32_t DisassemblerSpew::internalResolve(const Label* l) {
+ // Note, internalResolve will sometimes return 0 when it is triggered by the
+ // profiler and not by a full disassembly, since in that case a label can be
+ // used or bound but not previously have been defined. In that case,
+ // internalResolve(l) will not necessarily create a binding for l!
+ // Consequently a subsequent lookup(l) may still return null.
+ return l->used() || l->bound() ? probe(l) : define(l);
+}
+
+uint32_t DisassemblerSpew::probe(const Label* l) {
+ Node* n = lookup(l);
+ return n ? n->value : 0;
+}
+
+uint32_t DisassemblerSpew::define(const Label* l) {
+ remove(l);
+ uint32_t value = spewNext_++;
+ if (!add(l, value)) {
+ return 0;
+ }
+ return value;
+}
+
+DisassemblerSpew::Node* DisassemblerSpew::lookup(const Label* key) {
+ Node* p;
+ for (p = nodes_; p && p->key != key; p = p->next) {
+ ;
+ }
+ return p;
+}
+
+DisassemblerSpew::Node* DisassemblerSpew::add(const Label* key,
+ uint32_t value) {
+ MOZ_ASSERT(!lookup(key));
+ Node* node = js_new<Node>();
+ if (node) {
+ node->key = key;
+ node->value = value;
+ node->bound = false;
+ node->next = nodes_;
+ nodes_ = node;
+ }
+ return node;
+}
+
+bool DisassemblerSpew::remove(const Label* key) {
+ // We do not require that there is a node matching the key.
+ for (Node *p = nodes_, *pp = nullptr; p; pp = p, p = p->next) {
+ if (p->key == key) {
+ if (pp) {
+ pp->next = p->next;
+ } else {
+ nodes_ = p->next;
+ }
+ js_free(p);
+ return true;
+ }
+ }
+ return false;
+}
+
+#else
+
+DisassemblerSpew::LabelDoc DisassemblerSpew::refLabel(const Label* l) {
+ return LabelDoc();
+}
+
+#endif
diff --git a/js/src/jit/shared/Disassembler-shared.h b/js/src/jit/shared/Disassembler-shared.h
new file mode 100644
index 0000000000..6bd82c48f5
--- /dev/null
+++ b/js/src/jit/shared/Disassembler-shared.h
@@ -0,0 +1,182 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_Disassembler_shared_h
+#define jit_shared_Disassembler_shared_h
+
+#include "mozilla/Atomics.h"
+#include "mozilla/Attributes.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(JS_DISASM_ARM) || defined(JS_DISASM_ARM64)
+# define JS_DISASM_SUPPORTED
+#endif
+
+namespace js {
+
+class Sprinter;
+
+namespace jit {
+
+class Label;
+
+// A wrapper around spew/disassembly functionality. The disassembler is built
+// on a per-instruction disassembler (as in our ARM, ARM64 back-ends) and
+// formats labels with meaningful names and literals with meaningful values, if
+// the assembler creates documentation (with provided helpers) at appropriate
+// points.
+
+class DisassemblerSpew {
+#ifdef JS_DISASM_SUPPORTED
+ struct Node {
+ const Label* key; // Never dereferenced, only used for its value
+ uint32_t value; // The printable label value
+ bool bound; // If the label has been seen by spewBind()
+ Node* next;
+ };
+
+ Node* lookup(const Label* key);
+ Node* add(const Label* key, uint32_t value);
+ bool remove(const Label* key);
+
+ uint32_t probe(const Label* l);
+ uint32_t define(const Label* l);
+ uint32_t internalResolve(const Label* l);
+#endif
+
+ void spewVA(const char* fmt, va_list args) MOZ_FORMAT_PRINTF(2, 0);
+
+ public:
+ DisassemblerSpew();
+ ~DisassemblerSpew();
+
+#ifdef JS_DISASM_SUPPORTED
+ // Set indentation strings. The spewer retains a reference to s.
+ void setLabelIndent(const char* s);
+ void setTargetIndent(const char* s);
+#endif
+
+ // Set the spew printer, which will always be used if it is set, regardless
+ // of whether the system spew channel is enabled or not. The spewer retains
+ // a reference to sp.
+ void setPrinter(Sprinter* sp);
+
+ // Return true if disassembly spew is disabled and no additional printer is
+ // set.
+ bool isDisabled();
+
+ // Format and print text on the spew channel; output is suppressed if spew
+ // is disabled. The output is not indented, and is terminated by a newline.
+ void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3);
+
+ // Documentation for a label reference.
+ struct LabelDoc {
+#ifdef JS_DISASM_SUPPORTED
+ LabelDoc() : doc(0), bound(false), valid(false) {}
+ LabelDoc(uint32_t doc, bool bound) : doc(doc), bound(bound), valid(true) {}
+ const uint32_t doc;
+ const bool bound;
+ const bool valid;
+#else
+ LabelDoc() = default;
+ LabelDoc(uint32_t, bool) {}
+#endif
+ };
+
+ // Documentation for a literal load.
+ struct LiteralDoc {
+#ifdef JS_DISASM_SUPPORTED
+ enum class Type { Patchable, I32, U32, I64, U64, F32, F64 };
+ const Type type;
+ union {
+ int32_t i32;
+ uint32_t u32;
+ int64_t i64;
+ uint64_t u64;
+ float f32;
+ double f64;
+ } value;
+ LiteralDoc() : type(Type::Patchable) {}
+ explicit LiteralDoc(int32_t v) : type(Type::I32) { value.i32 = v; }
+ explicit LiteralDoc(uint32_t v) : type(Type::U32) { value.u32 = v; }
+ explicit LiteralDoc(int64_t v) : type(Type::I64) { value.i64 = v; }
+ explicit LiteralDoc(uint64_t v) : type(Type::U64) { value.u64 = v; }
+ explicit LiteralDoc(float v) : type(Type::F32) { value.f32 = v; }
+ explicit LiteralDoc(double v) : type(Type::F64) { value.f64 = v; }
+#else
+ LiteralDoc() = default;
+ explicit LiteralDoc(int32_t) {}
+ explicit LiteralDoc(uint32_t) {}
+ explicit LiteralDoc(int64_t) {}
+ explicit LiteralDoc(uint64_t) {}
+ explicit LiteralDoc(float) {}
+ explicit LiteralDoc(double) {}
+#endif
+ };
+
+ // Reference a label, resolving it to a printable representation.
+ //
+ // NOTE: The printable representation depends on the state of the label, so
+ // if we call resolve() when emitting & disassembling a branch instruction
+ // then it should be called before the label becomes Used, if emitting the
+ // branch can change the label's state.
+ //
+ // If the disassembler is not defined this returns a structure that is
+ // marked not valid.
+ LabelDoc refLabel(const Label* l);
+
+#ifdef JS_DISASM_SUPPORTED
+ // Spew the label information previously gathered by refLabel(), at a point
+ // where the label is referenced. The output is indented by targetIndent_
+ // and terminated by a newline.
+ void spewRef(const LabelDoc& target);
+
+ // Spew the label at the point where the label is bound. The output is
+ // indented by labelIndent_ and terminated by a newline.
+ void spewBind(const Label* label);
+
+ // Spew a retarget directive at the point where the retarget is recorded.
+ // The output is indented by labelIndent_ and terminated by a newline.
+ void spewRetarget(const Label* label, const Label* target);
+
+ // Format a literal value into the buffer. The buffer is always
+ // NUL-terminated even if this chops the formatted value.
+ void formatLiteral(const LiteralDoc& doc, char* buffer, size_t bufsize);
+
+ // Print any unbound labels, one per line, with normal label indent and with
+ // a comment indicating the label is not defined. Labels can be referenced
+ // but unbound in some legitimate cases, normally for traps. Printing them
+ // reduces confusion.
+ void spewOrphans();
+#endif
+
+ private:
+ Sprinter* printer_;
+#ifdef JS_DISASM_SUPPORTED
+ const char* labelIndent_;
+ const char* targetIndent_;
+ uint32_t spewNext_;
+ Node* nodes_;
+ uint32_t tag_;
+
+ // This global is used to disambiguate concurrently live assemblers, see
+ // comments in Disassembler-shared.cpp for why this is desirable.
+ //
+ // The variable is atomic to avoid any kind of complaint from thread
+ // sanitizers etc. However, trying to look at disassembly without using
+ // --no-threads is basically insane, so you can ignore the multi-threading
+ // implications here.
+ static mozilla::Atomic<uint32_t> counter_;
+#endif
+};
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_shared_Disassembler_shared_h
diff --git a/js/src/jit/shared/IonAssemblerBuffer.h b/js/src/jit/shared/IonAssemblerBuffer.h
new file mode 100644
index 0000000000..170f098707
--- /dev/null
+++ b/js/src/jit/shared/IonAssemblerBuffer.h
@@ -0,0 +1,438 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_IonAssemblerBuffer_h
+#define jit_shared_IonAssemblerBuffer_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+
+#include "jit/ProcessExecutableMemory.h"
+#include "jit/shared/Assembler-shared.h"
+
+namespace js {
+namespace jit {
+
+// The offset into a buffer, in bytes.
+class BufferOffset {
+ int offset;
+
+ public:
+ friend BufferOffset nextOffset();
+
+ BufferOffset() : offset(INT_MIN) {}
+
+ explicit BufferOffset(int offset_) : offset(offset_) {
+ MOZ_ASSERT(offset >= 0);
+ }
+
+ explicit BufferOffset(Label* l) : offset(l->offset()) {
+ MOZ_ASSERT(offset >= 0);
+ }
+
+ int getOffset() const { return offset; }
+ bool assigned() const { return offset != INT_MIN; }
+
+ // A BOffImm is a Branch Offset Immediate. It is an architecture-specific
+ // structure that holds the immediate for a pc relative branch. diffB takes
+ // the label for the destination of the branch, and encodes the immediate
+ // for the branch. This will need to be fixed up later, since A pool may be
+ // inserted between the branch and its destination.
+ template <class BOffImm>
+ BOffImm diffB(BufferOffset other) const {
+ if (!BOffImm::IsInRange(offset - other.offset)) {
+ return BOffImm();
+ }
+ return BOffImm(offset - other.offset);
+ }
+
+ template <class BOffImm>
+ BOffImm diffB(Label* other) const {
+ MOZ_ASSERT(other->bound());
+ if (!BOffImm::IsInRange(offset - other->offset())) {
+ return BOffImm();
+ }
+ return BOffImm(offset - other->offset());
+ }
+};
+
+inline bool operator<(BufferOffset a, BufferOffset b) {
+ return a.getOffset() < b.getOffset();
+}
+
+inline bool operator>(BufferOffset a, BufferOffset b) {
+ return a.getOffset() > b.getOffset();
+}
+
+inline bool operator<=(BufferOffset a, BufferOffset b) {
+ return a.getOffset() <= b.getOffset();
+}
+
+inline bool operator>=(BufferOffset a, BufferOffset b) {
+ return a.getOffset() >= b.getOffset();
+}
+
+inline bool operator==(BufferOffset a, BufferOffset b) {
+ return a.getOffset() == b.getOffset();
+}
+
+inline bool operator!=(BufferOffset a, BufferOffset b) {
+ return a.getOffset() != b.getOffset();
+}
+
+template <int SliceSize>
+class BufferSlice {
+ protected:
+ BufferSlice<SliceSize>* prev_;
+ BufferSlice<SliceSize>* next_;
+
+ size_t bytelength_;
+
+ public:
+ mozilla::Array<uint8_t, SliceSize> instructions;
+
+ public:
+ explicit BufferSlice() : prev_(nullptr), next_(nullptr), bytelength_(0) {}
+
+ size_t length() const { return bytelength_; }
+ static inline size_t Capacity() { return SliceSize; }
+
+ BufferSlice* getNext() const { return next_; }
+ BufferSlice* getPrev() const { return prev_; }
+
+ void setNext(BufferSlice<SliceSize>* next) {
+ MOZ_ASSERT(next_ == nullptr);
+ MOZ_ASSERT(next->prev_ == nullptr);
+ next_ = next;
+ next->prev_ = this;
+ }
+
+ void putBytes(size_t numBytes, const void* source) {
+ MOZ_ASSERT(bytelength_ + numBytes <= SliceSize);
+ if (source) {
+ memcpy(&instructions[length()], source, numBytes);
+ }
+ bytelength_ += numBytes;
+ }
+
+ MOZ_ALWAYS_INLINE
+ void putU32Aligned(uint32_t value) {
+ MOZ_ASSERT(bytelength_ + 4 <= SliceSize);
+ MOZ_ASSERT((bytelength_ & 3) == 0);
+ MOZ_ASSERT((uintptr_t(&instructions[0]) & 3) == 0);
+ *reinterpret_cast<uint32_t*>(&instructions[bytelength_]) = value;
+ bytelength_ += 4;
+ }
+};
+
+template <int SliceSize, class Inst>
+class AssemblerBuffer {
+ protected:
+ typedef BufferSlice<SliceSize> Slice;
+
+ // Doubly-linked list of BufferSlices, with the most recent in tail position.
+ Slice* head;
+ Slice* tail;
+
+ bool m_oom;
+
+ // How many bytes has been committed to the buffer thus far.
+ // Does not include tail.
+ uint32_t bufferSize;
+
+ // How many bytes can be in the buffer. Normally this is
+ // MaxCodeBytesPerBuffer, but for pasteup buffers where we handle far jumps
+ // explicitly it can be larger.
+ uint32_t maxSize;
+
+ // Finger for speeding up accesses.
+ Slice* finger;
+ int finger_offset;
+
+ LifoAlloc lifoAlloc_;
+
+ public:
+ explicit AssemblerBuffer()
+ : head(nullptr),
+ tail(nullptr),
+ m_oom(false),
+ bufferSize(0),
+ maxSize(MaxCodeBytesPerBuffer),
+ finger(nullptr),
+ finger_offset(0),
+ lifoAlloc_(8192) {}
+
+ public:
+ bool isAligned(size_t alignment) const {
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment));
+ return !(size() & (alignment - 1));
+ }
+
+ void setUnlimited() { maxSize = MaxCodeBytesPerProcess; }
+
+ private:
+ Slice* newSlice(LifoAlloc& a) {
+ if (size() > maxSize - sizeof(Slice)) {
+ fail_oom();
+ return nullptr;
+ }
+ Slice* tmp = static_cast<Slice*>(a.alloc(sizeof(Slice)));
+ if (!tmp) {
+ fail_oom();
+ return nullptr;
+ }
+ return new (tmp) Slice;
+ }
+
+ public:
+ bool ensureSpace(size_t size) {
+ // Space can exist in the most recent Slice.
+ if (tail && tail->length() + size <= tail->Capacity()) {
+ // Simulate allocation failure even when we don't need a new slice.
+ if (js::oom::ShouldFailWithOOM()) {
+ return fail_oom();
+ }
+
+ return true;
+ }
+
+ // Otherwise, a new Slice must be added.
+ Slice* slice = newSlice(lifoAlloc_);
+ if (slice == nullptr) {
+ return fail_oom();
+ }
+
+ // If this is the first Slice in the buffer, add to head position.
+ if (!head) {
+ head = slice;
+ finger = slice;
+ finger_offset = 0;
+ }
+
+ // Finish the last Slice and add the new Slice to the linked list.
+ if (tail) {
+ bufferSize += tail->length();
+ tail->setNext(slice);
+ }
+ tail = slice;
+
+ return true;
+ }
+
+ BufferOffset putByte(uint8_t value) {
+ return putBytes(sizeof(value), &value);
+ }
+
+ BufferOffset putShort(uint16_t value) {
+ return putBytes(sizeof(value), &value);
+ }
+
+ BufferOffset putInt(uint32_t value) {
+ return putBytes(sizeof(value), &value);
+ }
+
+ MOZ_ALWAYS_INLINE
+ BufferOffset putU32Aligned(uint32_t value) {
+ if (!ensureSpace(sizeof(value))) {
+ return BufferOffset();
+ }
+
+ BufferOffset ret = nextOffset();
+ tail->putU32Aligned(value);
+ return ret;
+ }
+
+ // Add numBytes bytes to this buffer.
+ // The data must fit in a single slice.
+ BufferOffset putBytes(size_t numBytes, const void* inst) {
+ if (!ensureSpace(numBytes)) {
+ return BufferOffset();
+ }
+
+ BufferOffset ret = nextOffset();
+ tail->putBytes(numBytes, inst);
+ return ret;
+ }
+
+ // Add a potentially large amount of data to this buffer.
+ // The data may be distrubuted across multiple slices.
+ // Return the buffer offset of the first added byte.
+ BufferOffset putBytesLarge(size_t numBytes, const void* data) {
+ BufferOffset ret = nextOffset();
+ while (numBytes > 0) {
+ if (!ensureSpace(1)) {
+ return BufferOffset();
+ }
+ size_t avail = tail->Capacity() - tail->length();
+ size_t xfer = numBytes < avail ? numBytes : avail;
+ MOZ_ASSERT(xfer > 0, "ensureSpace should have allocated a slice");
+ tail->putBytes(xfer, data);
+ data = (const uint8_t*)data + xfer;
+ numBytes -= xfer;
+ }
+ return ret;
+ }
+
+ unsigned int size() const {
+ if (tail) {
+ return bufferSize + tail->length();
+ }
+ return bufferSize;
+ }
+ BufferOffset nextOffset() const { return BufferOffset(size()); }
+
+ bool oom() const { return m_oom; }
+
+ bool fail_oom() {
+ m_oom = true;
+#ifdef DEBUG
+ JitContext* context = MaybeGetJitContext();
+ if (context) {
+ context->setOOM();
+ }
+#endif
+ return false;
+ }
+
+ private:
+ void update_finger(Slice* finger_, int fingerOffset_) {
+ finger = finger_;
+ finger_offset = fingerOffset_;
+ }
+
+ static const unsigned SliceDistanceRequiringFingerUpdate = 3;
+
+ Inst* getInstForwards(BufferOffset off, Slice* start, int startOffset,
+ bool updateFinger = false) {
+ const int offset = off.getOffset();
+
+ int cursor = startOffset;
+ unsigned slicesSkipped = 0;
+
+ MOZ_ASSERT(offset >= cursor);
+
+ for (Slice* slice = start; slice != nullptr; slice = slice->getNext()) {
+ const int slicelen = slice->length();
+
+ // Is the offset within the bounds of this slice?
+ if (offset < cursor + slicelen) {
+ if (updateFinger ||
+ slicesSkipped >= SliceDistanceRequiringFingerUpdate) {
+ update_finger(slice, cursor);
+ }
+
+ MOZ_ASSERT(offset - cursor < (int)slice->length());
+ return (Inst*)&slice->instructions[offset - cursor];
+ }
+
+ cursor += slicelen;
+ slicesSkipped++;
+ }
+
+ MOZ_CRASH("Invalid instruction cursor.");
+ }
+
+ Inst* getInstBackwards(BufferOffset off, Slice* start, int startOffset,
+ bool updateFinger = false) {
+ const int offset = off.getOffset();
+
+ int cursor = startOffset; // First (lowest) offset in the start Slice.
+ unsigned slicesSkipped = 0;
+
+ MOZ_ASSERT(offset < int(cursor + start->length()));
+
+ for (Slice* slice = start; slice != nullptr;) {
+ // Is the offset within the bounds of this slice?
+ if (offset >= cursor) {
+ if (updateFinger ||
+ slicesSkipped >= SliceDistanceRequiringFingerUpdate) {
+ update_finger(slice, cursor);
+ }
+
+ MOZ_ASSERT(offset - cursor < (int)slice->length());
+ return (Inst*)&slice->instructions[offset - cursor];
+ }
+
+ // Move the cursor to the start of the previous slice.
+ Slice* prev = slice->getPrev();
+ cursor -= prev->length();
+
+ slice = prev;
+ slicesSkipped++;
+ }
+
+ MOZ_CRASH("Invalid instruction cursor.");
+ }
+
+ public:
+ Inst* getInstOrNull(BufferOffset off) {
+ if (!off.assigned()) {
+ return nullptr;
+ }
+ return getInst(off);
+ }
+
+ // Get a pointer to the instruction at offset |off| which must be within the
+ // bounds of the buffer. Use |getInstOrNull()| if |off| may be unassigned.
+ Inst* getInst(BufferOffset off) {
+ const int offset = off.getOffset();
+ // This function is hot, do not make the next line a RELEASE_ASSERT.
+ MOZ_ASSERT(off.assigned() && offset >= 0 && unsigned(offset) < size());
+
+ // Is the instruction in the last slice?
+ if (offset >= int(bufferSize)) {
+ return (Inst*)&tail->instructions[offset - bufferSize];
+ }
+
+ // How close is this offset to the previous one we looked up?
+ // If it is sufficiently far from the start and end of the buffer,
+ // use the finger to start midway through the list.
+ int finger_dist = abs(offset - finger_offset);
+ if (finger_dist < std::min(offset, int(bufferSize - offset))) {
+ if (finger_offset < offset) {
+ return getInstForwards(off, finger, finger_offset, true);
+ }
+ return getInstBackwards(off, finger, finger_offset, true);
+ }
+
+ // Is the instruction closer to the start or to the end?
+ if (offset < int(bufferSize - offset)) {
+ return getInstForwards(off, head, 0);
+ }
+
+ // The last slice was already checked above, so start at the
+ // second-to-last.
+ Slice* prev = tail->getPrev();
+ return getInstBackwards(off, prev, bufferSize - prev->length());
+ }
+
+ typedef AssemblerBuffer<SliceSize, Inst> ThisClass;
+
+ class AssemblerBufferInstIterator {
+ BufferOffset bo_;
+ ThisClass* buffer_;
+
+ public:
+ explicit AssemblerBufferInstIterator(BufferOffset bo, ThisClass* buffer)
+ : bo_(bo), buffer_(buffer) {}
+ void advance(int offset) { bo_ = BufferOffset(bo_.getOffset() + offset); }
+ Inst* next() {
+ advance(cur()->size());
+ return cur();
+ }
+ Inst* peek() {
+ return buffer_->getInst(BufferOffset(bo_.getOffset() + cur()->size()));
+ }
+ Inst* cur() const { return buffer_->getInst(bo_); }
+ };
+};
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_shared_IonAssemblerBuffer_h
diff --git a/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h b/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h
new file mode 100644
index 0000000000..545a0c714b
--- /dev/null
+++ b/js/src/jit/shared/IonAssemblerBufferWithConstantPools.h
@@ -0,0 +1,1197 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_IonAssemblerBufferWithConstantPools_h
+#define jit_shared_IonAssemblerBufferWithConstantPools_h
+
+#include "mozilla/CheckedInt.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+
+#include "jit/JitSpewer.h"
+#include "jit/shared/IonAssemblerBuffer.h"
+
+// [SMDOC] JIT AssemblerBuffer constant pooling (ARM/ARM64/MIPS)
+//
+// This code extends the AssemblerBuffer to support the pooling of values loaded
+// using program-counter relative addressing modes. This is necessary with the
+// ARM instruction set because it has a fixed instruction size that can not
+// encode all values as immediate arguments in instructions. Pooling the values
+// allows the values to be placed in large chunks which minimizes the number of
+// forced branches around them in the code. This is used for loading floating
+// point constants, for loading 32 bit constants on the ARMv6, for absolute
+// branch targets, and in future will be needed for large branches on the ARMv6.
+//
+// For simplicity of the implementation, the constant pools are always placed
+// after the loads referencing them. When a new constant pool load is added to
+// the assembler buffer, a corresponding pool entry is added to the current
+// pending pool. The finishPool() method copies the current pending pool entries
+// into the assembler buffer at the current offset and patches the pending
+// constant pool load instructions.
+//
+// Before inserting instructions or pool entries, it is necessary to determine
+// if doing so would place a pending pool entry out of reach of an instruction,
+// and if so then the pool must firstly be dumped. With the allocation algorithm
+// used below, the recalculation of all the distances between instructions and
+// their pool entries can be avoided by noting that there will be a limiting
+// instruction and pool entry pair that does not change when inserting more
+// instructions. Adding more instructions makes the same increase to the
+// distance, between instructions and their pool entries, for all such
+// pairs. This pair is recorded as the limiter, and it is updated when new pool
+// entries are added, see updateLimiter()
+//
+// The pools consist of: a guard instruction that branches around the pool, a
+// header word that helps identify a pool in the instruction stream, and then
+// the pool entries allocated in units of words. The guard instruction could be
+// omitted if control does not reach the pool, and this is referred to as a
+// natural guard below, but for simplicity the guard branch is always
+// emitted. The pool header is an identifiable word that in combination with the
+// guard uniquely identifies a pool in the instruction stream. The header also
+// encodes the pool size and a flag indicating if the guard is natural. It is
+// possible to iterate through the code instructions skipping or examining the
+// pools. E.g. it might be necessary to skip pools when search for, or patching,
+// an instruction sequence.
+//
+// It is often required to keep a reference to a pool entry, to patch it after
+// the buffer is finished. Each pool entry is assigned a unique index, counting
+// up from zero (see the poolEntryCount slot below). These can be mapped back to
+// the offset of the pool entry in the finished buffer, see poolEntryOffset().
+//
+// The code supports no-pool regions, and for these the size of the region, in
+// instructions, must be supplied. This size is used to determine if inserting
+// the instructions would place a pool entry out of range, and if so then a pool
+// is firstly flushed. The DEBUG code checks that the emitted code is within the
+// supplied size to detect programming errors. See enterNoPool() and
+// leaveNoPool().
+
+// The only planned instruction sets that require inline constant pools are the
+// ARM, ARM64, and MIPS, and these all have fixed 32-bit sized instructions so
+// for simplicity the code below is specialized for fixed 32-bit sized
+// instructions and makes no attempt to support variable length
+// instructions. The base assembler buffer which supports variable width
+// instruction is used by the x86 and x64 backends.
+
+// The AssemblerBufferWithConstantPools template class uses static callbacks to
+// the provided Asm template argument class:
+//
+// void Asm::InsertIndexIntoTag(uint8_t* load_, uint32_t index)
+//
+// When allocEntry() is called to add a constant pool load with an associated
+// constant pool entry, this callback is called to encode the index of the
+// allocated constant pool entry into the load instruction.
+//
+// After the constant pool has been placed, PatchConstantPoolLoad() is called
+// to update the load instruction with the right load offset.
+//
+// void Asm::WritePoolGuard(BufferOffset branch,
+// Instruction* dest,
+// BufferOffset afterPool)
+//
+// Write out the constant pool guard branch before emitting the pool.
+//
+// branch
+// Offset of the guard branch in the buffer.
+//
+// dest
+// Pointer into the buffer where the guard branch should be emitted. (Same
+// as getInst(branch)). Space for guardSize_ instructions has been reserved.
+//
+// afterPool
+// Offset of the first instruction after the constant pool. This includes
+// both pool entries and branch veneers added after the pool data.
+//
+// void Asm::WritePoolHeader(uint8_t* start, Pool* p, bool isNatural)
+//
+// Write out the pool header which follows the guard branch.
+//
+// void Asm::PatchConstantPoolLoad(void* loadAddr, void* constPoolAddr)
+//
+// Re-encode a load of a constant pool entry after the location of the
+// constant pool is known.
+//
+// The load instruction at loadAddr was previously passed to
+// InsertIndexIntoTag(). The constPoolAddr is the final address of the
+// constant pool in the assembler buffer.
+//
+// void Asm::PatchShortRangeBranchToVeneer(AssemblerBufferWithConstantPools*,
+// unsigned rangeIdx,
+// BufferOffset deadline,
+// BufferOffset veneer)
+//
+// Patch a short-range branch to jump through a veneer before it goes out of
+// range.
+//
+// rangeIdx, deadline
+// These arguments were previously passed to registerBranchDeadline(). It is
+// assumed that PatchShortRangeBranchToVeneer() knows how to compute the
+// offset of the short-range branch from this information.
+//
+// veneer
+// Space for a branch veneer, guaranteed to be <= deadline. At this
+// position, guardSize_ * InstSize bytes are allocated. They should be
+// initialized to the proper unconditional branch instruction.
+//
+// Unbound branches to the same unbound label are organized as a linked list:
+//
+// Label::offset -> Branch1 -> Branch2 -> Branch3 -> nil
+//
+// This callback should insert a new veneer branch into the list:
+//
+// Label::offset -> Branch1 -> Branch2 -> Veneer -> Branch3 -> nil
+//
+// When Assembler::bind() rewrites the branches with the real label offset, it
+// probably has to bind Branch2 to target the veneer branch instead of jumping
+// straight to the label.
+
+namespace js {
+namespace jit {
+
+// BranchDeadlineSet - Keep track of pending branch deadlines.
+//
+// Some architectures like arm and arm64 have branch instructions with limited
+// range. When assembling a forward branch, it is not always known if the final
+// target label will be in range of the branch instruction.
+//
+// The BranchDeadlineSet data structure is used to keep track of the set of
+// pending forward branches. It supports the following fast operations:
+//
+// 1. Get the earliest deadline in the set.
+// 2. Add a new branch deadline.
+// 3. Remove a branch deadline.
+//
+// Architectures may have different branch encodings with different ranges. Each
+// supported range is assigned a small integer starting at 0. This data
+// structure does not care about the actual range of branch instructions, just
+// the latest buffer offset that can be reached - the deadline offset.
+//
+// Branched are stored as (rangeIdx, deadline) tuples. The target-specific code
+// can compute the location of the branch itself from this information. This
+// data structure does not need to know.
+//
+template <unsigned NumRanges>
+class BranchDeadlineSet {
+ // Maintain a list of pending deadlines for each range separately.
+ //
+ // The offsets in each vector are always kept in ascending order.
+ //
+ // Because we have a separate vector for different ranges, as forward
+ // branches are added to the assembler buffer, their deadlines will
+ // always be appended to the vector corresponding to their range.
+ //
+ // When binding labels, we expect a more-or-less LIFO order of branch
+ // resolutions. This would always hold if we had strictly structured control
+ // flow.
+ //
+ // We allow branch deadlines to be added and removed in any order, but
+ // performance is best in the expected case of near LIFO order.
+ //
+ typedef Vector<BufferOffset, 8, LifoAllocPolicy<Fallible>> RangeVector;
+
+ // We really just want "RangeVector deadline_[NumRanges];", but each vector
+ // needs to be initialized with a LifoAlloc, and C++ doesn't bend that way.
+ //
+ // Use raw aligned storage instead and explicitly construct NumRanges
+ // vectors in our constructor.
+ mozilla::AlignedStorage2<RangeVector[NumRanges]> deadlineStorage_;
+
+ // Always access the range vectors through this method.
+ RangeVector& vectorForRange(unsigned rangeIdx) {
+ MOZ_ASSERT(rangeIdx < NumRanges, "Invalid branch range index");
+ return (*deadlineStorage_.addr())[rangeIdx];
+ }
+
+ const RangeVector& vectorForRange(unsigned rangeIdx) const {
+ MOZ_ASSERT(rangeIdx < NumRanges, "Invalid branch range index");
+ return (*deadlineStorage_.addr())[rangeIdx];
+ }
+
+ // Maintain a precomputed earliest deadline at all times.
+ // This is unassigned only when all deadline vectors are empty.
+ BufferOffset earliest_;
+
+ // The range vector owning earliest_. Uninitialized when empty.
+ unsigned earliestRange_;
+
+ // Recompute the earliest deadline after it's been invalidated.
+ void recomputeEarliest() {
+ earliest_ = BufferOffset();
+ for (unsigned r = 0; r < NumRanges; r++) {
+ auto& vec = vectorForRange(r);
+ if (!vec.empty() && (!earliest_.assigned() || vec[0] < earliest_)) {
+ earliest_ = vec[0];
+ earliestRange_ = r;
+ }
+ }
+ }
+
+ // Update the earliest deadline if needed after inserting (rangeIdx,
+ // deadline). Always return true for convenience:
+ // return insert() && updateEarliest().
+ bool updateEarliest(unsigned rangeIdx, BufferOffset deadline) {
+ if (!earliest_.assigned() || deadline < earliest_) {
+ earliest_ = deadline;
+ earliestRange_ = rangeIdx;
+ }
+ return true;
+ }
+
+ public:
+ explicit BranchDeadlineSet(LifoAlloc& alloc) : earliestRange_(0) {
+ // Manually construct vectors in the uninitialized aligned storage.
+ // This is because C++ arrays can otherwise only be constructed with
+ // the default constructor.
+ for (unsigned r = 0; r < NumRanges; r++) {
+ new (&vectorForRange(r)) RangeVector(alloc);
+ }
+ }
+
+ ~BranchDeadlineSet() {
+ // Aligned storage doesn't destruct its contents automatically.
+ for (unsigned r = 0; r < NumRanges; r++) {
+ vectorForRange(r).~RangeVector();
+ }
+ }
+
+ // Is this set completely empty?
+ bool empty() const { return !earliest_.assigned(); }
+
+ // Get the total number of deadlines in the set.
+ size_t size() const {
+ size_t count = 0;
+ for (unsigned r = 0; r < NumRanges; r++) {
+ count += vectorForRange(r).length();
+ }
+ return count;
+ }
+
+ // Get the number of deadlines for the range with the most elements.
+ size_t maxRangeSize() const {
+ size_t count = 0;
+ for (unsigned r = 0; r < NumRanges; r++) {
+ count = std::max(count, vectorForRange(r).length());
+ }
+ return count;
+ }
+
+ // Get the first deadline that is still in the set.
+ BufferOffset earliestDeadline() const {
+ MOZ_ASSERT(!empty());
+ return earliest_;
+ }
+
+ // Get the range index corresponding to earliestDeadlineRange().
+ unsigned earliestDeadlineRange() const {
+ MOZ_ASSERT(!empty());
+ return earliestRange_;
+ }
+
+ // Add a (rangeIdx, deadline) tuple to the set.
+ //
+ // It is assumed that this tuple is not already in the set.
+ // This function performs best id the added deadline is later than any
+ // existing deadline for the same range index.
+ //
+ // Return true if the tuple was added, false if the tuple could not be added
+ // because of an OOM error.
+ bool addDeadline(unsigned rangeIdx, BufferOffset deadline) {
+ MOZ_ASSERT(deadline.assigned(), "Can only store assigned buffer offsets");
+ // This is the vector where deadline should be saved.
+ auto& vec = vectorForRange(rangeIdx);
+
+ // Fast case: Simple append to the relevant array. This never affects
+ // the earliest deadline.
+ if (!vec.empty() && vec.back() < deadline) {
+ return vec.append(deadline);
+ }
+
+ // Fast case: First entry to the vector. We need to update earliest_.
+ if (vec.empty()) {
+ return vec.append(deadline) && updateEarliest(rangeIdx, deadline);
+ }
+
+ return addDeadlineSlow(rangeIdx, deadline);
+ }
+
+ private:
+ // General case of addDeadline. This is split into two functions such that
+ // the common case in addDeadline can be inlined while this part probably
+ // won't inline.
+ bool addDeadlineSlow(unsigned rangeIdx, BufferOffset deadline) {
+ auto& vec = vectorForRange(rangeIdx);
+
+ // Inserting into the middle of the vector. Use a log time binary search
+ // and a linear time insert().
+ // Is it worthwhile special-casing the empty vector?
+ auto at = std::lower_bound(vec.begin(), vec.end(), deadline);
+ MOZ_ASSERT(at == vec.end() || *at != deadline,
+ "Cannot insert duplicate deadlines");
+ return vec.insert(at, deadline) && updateEarliest(rangeIdx, deadline);
+ }
+
+ public:
+ // Remove a deadline from the set.
+ // If (rangeIdx, deadline) is not in the set, nothing happens.
+ void removeDeadline(unsigned rangeIdx, BufferOffset deadline) {
+ auto& vec = vectorForRange(rangeIdx);
+
+ if (vec.empty()) {
+ return;
+ }
+
+ if (deadline == vec.back()) {
+ // Expected fast case: Structured control flow causes forward
+ // branches to be bound in reverse order.
+ vec.popBack();
+ } else {
+ // Slow case: Binary search + linear erase.
+ auto where = std::lower_bound(vec.begin(), vec.end(), deadline);
+ if (where == vec.end() || *where != deadline) {
+ return;
+ }
+ vec.erase(where);
+ }
+ if (deadline == earliest_) {
+ recomputeEarliest();
+ }
+ }
+};
+
+// Specialization for architectures that don't need to track short-range
+// branches.
+template <>
+class BranchDeadlineSet<0u> {
+ public:
+ explicit BranchDeadlineSet(LifoAlloc& alloc) {}
+ bool empty() const { return true; }
+ size_t size() const { return 0; }
+ size_t maxRangeSize() const { return 0; }
+ BufferOffset earliestDeadline() const { MOZ_CRASH(); }
+ unsigned earliestDeadlineRange() const { MOZ_CRASH(); }
+ bool addDeadline(unsigned rangeIdx, BufferOffset deadline) { MOZ_CRASH(); }
+ void removeDeadline(unsigned rangeIdx, BufferOffset deadline) { MOZ_CRASH(); }
+};
+
+// The allocation unit size for pools.
+typedef int32_t PoolAllocUnit;
+
+// Hysteresis given to short-range branches.
+//
+// If any short-range branches will go out of range in the next N bytes,
+// generate a veneer for them in the current pool. The hysteresis prevents the
+// creation of many tiny constant pools for branch veneers.
+const size_t ShortRangeBranchHysteresis = 128;
+
+struct Pool {
+ private:
+ // The maximum program-counter relative offset below which the instruction
+ // set can encode. Different classes of intructions might support different
+ // ranges but for simplicity the minimum is used here, and for the ARM this
+ // is constrained to 1024 by the float load instructions.
+ const size_t maxOffset_;
+ // An offset to apply to program-counter relative offsets. The ARM has a
+ // bias of 8.
+ const unsigned bias_;
+
+ // The content of the pool entries.
+ Vector<PoolAllocUnit, 8, LifoAllocPolicy<Fallible>> poolData_;
+
+ // Flag that tracks OOM conditions. This is set after any append failed.
+ bool oom_;
+
+ // The limiting instruction and pool-entry pair. The instruction program
+ // counter relative offset of this limiting instruction will go out of range
+ // first as the pool position moves forward. It is more efficient to track
+ // just this limiting pair than to recheck all offsets when testing if the
+ // pool needs to be dumped.
+ //
+ // 1. The actual offset of the limiting instruction referencing the limiting
+ // pool entry.
+ BufferOffset limitingUser;
+ // 2. The pool entry index of the limiting pool entry.
+ unsigned limitingUsee;
+
+ public:
+ // A record of the code offset of instructions that reference pool
+ // entries. These instructions need to be patched when the actual position
+ // of the instructions and pools are known, and for the code below this
+ // occurs when each pool is finished, see finishPool().
+ Vector<BufferOffset, 8, LifoAllocPolicy<Fallible>> loadOffsets;
+
+ // Create a Pool. Don't allocate anything from lifoAloc, just capture its
+ // reference.
+ explicit Pool(size_t maxOffset, unsigned bias, LifoAlloc& lifoAlloc)
+ : maxOffset_(maxOffset),
+ bias_(bias),
+ poolData_(lifoAlloc),
+ oom_(false),
+ limitingUser(),
+ limitingUsee(INT_MIN),
+ loadOffsets(lifoAlloc) {}
+
+ // If poolData() returns nullptr then oom_ will also be true.
+ const PoolAllocUnit* poolData() const { return poolData_.begin(); }
+
+ unsigned numEntries() const { return poolData_.length(); }
+
+ size_t getPoolSize() const { return numEntries() * sizeof(PoolAllocUnit); }
+
+ bool oom() const { return oom_; }
+
+ // Update the instruction/pool-entry pair that limits the position of the
+ // pool. The nextInst is the actual offset of the new instruction being
+ // allocated.
+ //
+ // This is comparing the offsets, see checkFull() below for the equation,
+ // but common expressions on both sides have been canceled from the ranges
+ // being compared. Notably, the poolOffset cancels out, so the limiting pair
+ // does not depend on where the pool is placed.
+ void updateLimiter(BufferOffset nextInst) {
+ ptrdiff_t oldRange =
+ limitingUsee * sizeof(PoolAllocUnit) - limitingUser.getOffset();
+ ptrdiff_t newRange = getPoolSize() - nextInst.getOffset();
+ if (!limitingUser.assigned() || newRange > oldRange) {
+ // We have a new largest range!
+ limitingUser = nextInst;
+ limitingUsee = numEntries();
+ }
+ }
+
+ // Check if inserting a pool at the actual offset poolOffset would place
+ // pool entries out of reach. This is called before inserting instructions
+ // to check that doing so would not push pool entries out of reach, and if
+ // so then the pool would need to be firstly dumped. The poolOffset is the
+ // first word of the pool, after the guard and header and alignment fill.
+ bool checkFull(size_t poolOffset) const {
+ // Not full if there are no uses.
+ if (!limitingUser.assigned()) {
+ return false;
+ }
+ size_t offset = poolOffset + limitingUsee * sizeof(PoolAllocUnit) -
+ (limitingUser.getOffset() + bias_);
+ return offset >= maxOffset_;
+ }
+
+ static const unsigned OOM_FAIL = unsigned(-1);
+
+ unsigned insertEntry(unsigned num, uint8_t* data, BufferOffset off,
+ LifoAlloc& lifoAlloc) {
+ if (oom_) {
+ return OOM_FAIL;
+ }
+ unsigned ret = numEntries();
+ if (!poolData_.append((PoolAllocUnit*)data, num) ||
+ !loadOffsets.append(off)) {
+ oom_ = true;
+ return OOM_FAIL;
+ }
+ return ret;
+ }
+
+ void reset() {
+ poolData_.clear();
+ loadOffsets.clear();
+
+ limitingUser = BufferOffset();
+ limitingUsee = -1;
+ }
+};
+
+// Template arguments:
+//
+// SliceSize
+// Number of bytes in each allocated BufferSlice. See
+// AssemblerBuffer::SliceSize.
+//
+// InstSize
+// Size in bytes of the fixed-size instructions. This should be equal to
+// sizeof(Inst). This is only needed here because the buffer is defined before
+// the Instruction.
+//
+// Inst
+// The actual type used to represent instructions. This is only really used as
+// the return type of the getInst() method.
+//
+// Asm
+// Class defining the needed static callback functions. See documentation of
+// the Asm::* callbacks above.
+//
+// NumShortBranchRanges
+// The number of short branch ranges to support. This can be 0 if no support
+// for tracking short range branches is needed. The
+// AssemblerBufferWithConstantPools class does not need to know what the range
+// of branches is - it deals in branch 'deadlines' which is the last buffer
+// position that a short-range forward branch can reach. It is assumed that
+// the Asm class is able to find the actual branch instruction given a
+// (range-index, deadline) pair.
+//
+//
+template <size_t SliceSize, size_t InstSize, class Inst, class Asm,
+ unsigned NumShortBranchRanges = 0>
+struct AssemblerBufferWithConstantPools
+ : public AssemblerBuffer<SliceSize, Inst> {
+ private:
+ // The PoolEntry index counter. Each PoolEntry is given a unique index,
+ // counting up from zero, and these can be mapped back to the actual pool
+ // entry offset after finishing the buffer, see poolEntryOffset().
+ size_t poolEntryCount;
+
+ public:
+ class PoolEntry {
+ size_t index_;
+
+ public:
+ explicit PoolEntry(size_t index) : index_(index) {}
+
+ PoolEntry() : index_(-1) {}
+
+ size_t index() const { return index_; }
+ };
+
+ private:
+ typedef AssemblerBuffer<SliceSize, Inst> Parent;
+ using typename Parent::Slice;
+
+ // The size of a pool guard, in instructions. A branch around the pool.
+ const unsigned guardSize_;
+ // The size of the header that is put at the beginning of a full pool, in
+ // instruction sized units.
+ const unsigned headerSize_;
+
+ // The maximum pc relative offset encoded in instructions that reference
+ // pool entries. This is generally set to the maximum offset that can be
+ // encoded by the instructions, but for testing can be lowered to affect the
+ // pool placement and frequency of pool placement.
+ const size_t poolMaxOffset_;
+
+ // The bias on pc relative addressing mode offsets, in units of bytes. The
+ // ARM has a bias of 8 bytes.
+ const unsigned pcBias_;
+
+ // The current working pool. Copied out as needed before resetting.
+ Pool pool_;
+
+ // The buffer should be aligned to this address.
+ const size_t instBufferAlign_;
+
+ struct PoolInfo {
+ // The index of the first entry in this pool.
+ // Pool entries are numbered uniquely across all pools, starting from 0.
+ unsigned firstEntryIndex;
+
+ // The location of this pool's first entry in the main assembler buffer.
+ // Note that the pool guard and header come before this offset which
+ // points directly at the data.
+ BufferOffset offset;
+
+ explicit PoolInfo(unsigned index, BufferOffset data)
+ : firstEntryIndex(index), offset(data) {}
+ };
+
+ // Info for each pool that has already been dumped. This does not include
+ // any entries in pool_.
+ Vector<PoolInfo, 8, LifoAllocPolicy<Fallible>> poolInfo_;
+
+ // Set of short-range forward branches that have not yet been bound.
+ // We may need to insert veneers if the final label turns out to be out of
+ // range.
+ //
+ // This set stores (rangeIdx, deadline) pairs instead of the actual branch
+ // locations.
+ BranchDeadlineSet<NumShortBranchRanges> branchDeadlines_;
+
+ // When true dumping pools is inhibited.
+ bool canNotPlacePool_;
+
+#ifdef DEBUG
+ // State for validating the 'maxInst' argument to enterNoPool().
+ // The buffer offset when entering the no-pool region.
+ size_t canNotPlacePoolStartOffset_;
+ // The maximum number of word sized instructions declared for the no-pool
+ // region.
+ size_t canNotPlacePoolMaxInst_;
+#endif
+
+ // Instruction to use for alignment fill.
+ const uint32_t alignFillInst_;
+
+ // Insert a number of NOP instructions between each requested instruction at
+ // all locations at which a pool can potentially spill. This is useful for
+ // checking that instruction locations are correctly referenced and/or
+ // followed.
+ const uint32_t nopFillInst_;
+ const unsigned nopFill_;
+
+ // For inhibiting the insertion of fill NOPs in the dynamic context in which
+ // they are being inserted.
+ bool inhibitNops_;
+
+ private:
+ // The buffer slices are in a double linked list.
+ Slice* getHead() const { return this->head; }
+ Slice* getTail() const { return this->tail; }
+
+ public:
+ AssemblerBufferWithConstantPools(unsigned guardSize, unsigned headerSize,
+ size_t instBufferAlign, size_t poolMaxOffset,
+ unsigned pcBias, uint32_t alignFillInst,
+ uint32_t nopFillInst, unsigned nopFill = 0)
+ : poolEntryCount(0),
+ guardSize_(guardSize),
+ headerSize_(headerSize),
+ poolMaxOffset_(poolMaxOffset),
+ pcBias_(pcBias),
+ pool_(poolMaxOffset, pcBias, this->lifoAlloc_),
+ instBufferAlign_(instBufferAlign),
+ poolInfo_(this->lifoAlloc_),
+ branchDeadlines_(this->lifoAlloc_),
+ canNotPlacePool_(false),
+#ifdef DEBUG
+ canNotPlacePoolStartOffset_(0),
+ canNotPlacePoolMaxInst_(0),
+#endif
+ alignFillInst_(alignFillInst),
+ nopFillInst_(nopFillInst),
+ nopFill_(nopFill),
+ inhibitNops_(false) {
+ }
+
+ private:
+ size_t sizeExcludingCurrentPool() const {
+ // Return the actual size of the buffer, excluding the current pending
+ // pool.
+ return this->nextOffset().getOffset();
+ }
+
+ public:
+ size_t size() const {
+ // Return the current actual size of the buffer. This is only accurate
+ // if there are no pending pool entries to dump, check.
+ MOZ_ASSERT_IF(!this->oom(), pool_.numEntries() == 0);
+ return sizeExcludingCurrentPool();
+ }
+
+ private:
+ void insertNopFill() {
+ // Insert fill for testing.
+ if (nopFill_ > 0 && !inhibitNops_ && !canNotPlacePool_) {
+ inhibitNops_ = true;
+
+ // Fill using a branch-nop rather than a NOP so this can be
+ // distinguished and skipped.
+ for (size_t i = 0; i < nopFill_; i++) {
+ putInt(nopFillInst_);
+ }
+
+ inhibitNops_ = false;
+ }
+ }
+
+ static const unsigned OOM_FAIL = unsigned(-1);
+ static const unsigned DUMMY_INDEX = unsigned(-2);
+
+ // Check if it is possible to add numInst instructions and numPoolEntries
+ // constant pool entries without needing to flush the current pool.
+ bool hasSpaceForInsts(unsigned numInsts, unsigned numPoolEntries) const {
+ size_t nextOffset = sizeExcludingCurrentPool();
+ // Earliest starting offset for the current pool after adding numInsts.
+ // This is the beginning of the pool entries proper, after inserting a
+ // guard branch + pool header.
+ size_t poolOffset =
+ nextOffset + (numInsts + guardSize_ + headerSize_) * InstSize;
+
+ // Any constant pool loads that would go out of range?
+ if (pool_.checkFull(poolOffset)) {
+ return false;
+ }
+
+ // Any short-range branch that would go out of range?
+ if (!branchDeadlines_.empty()) {
+ size_t deadline = branchDeadlines_.earliestDeadline().getOffset();
+ size_t poolEnd = poolOffset + pool_.getPoolSize() +
+ numPoolEntries * sizeof(PoolAllocUnit);
+
+ // When NumShortBranchRanges > 1, is is possible for branch deadlines to
+ // expire faster than we can insert veneers. Suppose branches are 4 bytes
+ // each, we could have the following deadline set:
+ //
+ // Range 0: 40, 44, 48
+ // Range 1: 44, 48
+ //
+ // It is not good enough to start inserting veneers at the 40 deadline; we
+ // would not be able to create veneers for the second 44 deadline.
+ // Instead, we need to start at 32:
+ //
+ // 32: veneer(40)
+ // 36: veneer(44)
+ // 40: veneer(44)
+ // 44: veneer(48)
+ // 48: veneer(48)
+ //
+ // This is a pretty conservative solution to the problem: If we begin at
+ // the earliest deadline, we can always emit all veneers for the range
+ // that currently has the most pending deadlines. That may not leave room
+ // for veneers for the remaining ranges, so reserve space for those
+ // secondary range veneers assuming the worst case deadlines.
+
+ // Total pending secondary range veneer size.
+ size_t secondaryVeneers = guardSize_ * (branchDeadlines_.size() -
+ branchDeadlines_.maxRangeSize());
+
+ if (deadline < poolEnd + secondaryVeneers) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ unsigned insertEntryForwards(unsigned numInst, unsigned numPoolEntries,
+ uint8_t* inst, uint8_t* data) {
+ // If inserting pool entries then find a new limiter before we do the
+ // range check.
+ if (numPoolEntries) {
+ pool_.updateLimiter(BufferOffset(sizeExcludingCurrentPool()));
+ }
+
+ if (!hasSpaceForInsts(numInst, numPoolEntries)) {
+ if (numPoolEntries) {
+ JitSpew(JitSpew_Pools, "Inserting pool entry caused a spill");
+ } else {
+ JitSpew(JitSpew_Pools, "Inserting instruction(%zu) caused a spill",
+ sizeExcludingCurrentPool());
+ }
+
+ finishPool(numInst * InstSize);
+ if (this->oom()) {
+ return OOM_FAIL;
+ }
+ return insertEntryForwards(numInst, numPoolEntries, inst, data);
+ }
+ if (numPoolEntries) {
+ unsigned result = pool_.insertEntry(numPoolEntries, data,
+ this->nextOffset(), this->lifoAlloc_);
+ if (result == Pool::OOM_FAIL) {
+ this->fail_oom();
+ return OOM_FAIL;
+ }
+ return result;
+ }
+
+ // The pool entry index is returned above when allocating an entry, but
+ // when not allocating an entry a dummy value is returned - it is not
+ // expected to be used by the caller.
+ return DUMMY_INDEX;
+ }
+
+ public:
+ // Get the next buffer offset where an instruction would be inserted.
+ // This may flush the current constant pool before returning nextOffset().
+ BufferOffset nextInstrOffset() {
+ if (!hasSpaceForInsts(/* numInsts= */ 1, /* numPoolEntries= */ 0)) {
+ JitSpew(JitSpew_Pools,
+ "nextInstrOffset @ %d caused a constant pool spill",
+ this->nextOffset().getOffset());
+ finishPool(ShortRangeBranchHysteresis);
+ }
+ return this->nextOffset();
+ }
+
+ MOZ_NEVER_INLINE
+ BufferOffset allocEntry(size_t numInst, unsigned numPoolEntries,
+ uint8_t* inst, uint8_t* data,
+ PoolEntry* pe = nullptr) {
+ // The allocation of pool entries is not supported in a no-pool region,
+ // check.
+ MOZ_ASSERT_IF(numPoolEntries, !canNotPlacePool_);
+
+ if (this->oom()) {
+ return BufferOffset();
+ }
+
+ insertNopFill();
+
+#ifdef JS_JITSPEW
+ if (numPoolEntries && JitSpewEnabled(JitSpew_Pools)) {
+ JitSpew(JitSpew_Pools, "Inserting %d entries into pool", numPoolEntries);
+ JitSpewStart(JitSpew_Pools, "data is: 0x");
+ size_t length = numPoolEntries * sizeof(PoolAllocUnit);
+ for (unsigned idx = 0; idx < length; idx++) {
+ JitSpewCont(JitSpew_Pools, "%02x", data[length - idx - 1]);
+ if (((idx & 3) == 3) && (idx + 1 != length)) {
+ JitSpewCont(JitSpew_Pools, "_");
+ }
+ }
+ JitSpewFin(JitSpew_Pools);
+ }
+#endif
+
+ // Insert the pool value.
+ unsigned index = insertEntryForwards(numInst, numPoolEntries, inst, data);
+ if (this->oom()) {
+ return BufferOffset();
+ }
+
+ // Now to get an instruction to write.
+ PoolEntry retPE;
+ if (numPoolEntries) {
+ JitSpew(JitSpew_Pools, "Entry has index %u, offset %zu", index,
+ sizeExcludingCurrentPool());
+ Asm::InsertIndexIntoTag(inst, index);
+ // Figure out the offset within the pool entries.
+ retPE = PoolEntry(poolEntryCount);
+ poolEntryCount += numPoolEntries;
+ }
+ // Now inst is a valid thing to insert into the instruction stream.
+ if (pe != nullptr) {
+ *pe = retPE;
+ }
+ return this->putBytes(numInst * InstSize, inst);
+ }
+
+ // putInt is the workhorse for the assembler and higher-level buffer
+ // abstractions: it places one instruction into the instruction stream.
+ // Under normal circumstances putInt should just check that the constant
+ // pool does not need to be flushed, that there is space for the single word
+ // of the instruction, and write that word and update the buffer pointer.
+ //
+ // To do better here we need a status variable that handles both nopFill_
+ // and capacity, so that we can quickly know whether to go the slow path.
+ // That could be a variable that has the remaining number of simple
+ // instructions that can be inserted before a more expensive check,
+ // which is set to zero when nopFill_ is set.
+ //
+ // We assume that we don't have to check this->oom() if there is space to
+ // insert a plain instruction; there will always come a later time when it
+ // will be checked anyway.
+
+ MOZ_ALWAYS_INLINE
+ BufferOffset putInt(uint32_t value) {
+ if (nopFill_ ||
+ !hasSpaceForInsts(/* numInsts= */ 1, /* numPoolEntries= */ 0)) {
+ return allocEntry(1, 0, (uint8_t*)&value, nullptr, nullptr);
+ }
+
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \
+ defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) || \
+ defined(JS_CODEGEN_LOONG64)
+ return this->putU32Aligned(value);
+#else
+ return this->AssemblerBuffer<SliceSize, Inst>::putInt(value);
+#endif
+ }
+
+ // Register a short-range branch deadline.
+ //
+ // After inserting a short-range forward branch, call this method to
+ // register the branch 'deadline' which is the last buffer offset that the
+ // branch instruction can reach.
+ //
+ // When the branch is bound to a destination label, call
+ // unregisterBranchDeadline() to stop tracking this branch,
+ //
+ // If the assembled code is about to exceed the registered branch deadline,
+ // and unregisterBranchDeadline() has not yet been called, an
+ // instruction-sized constant pool entry is allocated before the branch
+ // deadline.
+ //
+ // rangeIdx
+ // A number < NumShortBranchRanges identifying the range of the branch.
+ //
+ // deadline
+ // The highest buffer offset the the short-range branch can reach
+ // directly.
+ //
+ void registerBranchDeadline(unsigned rangeIdx, BufferOffset deadline) {
+ if (!this->oom() && !branchDeadlines_.addDeadline(rangeIdx, deadline)) {
+ this->fail_oom();
+ }
+ }
+
+ // Un-register a short-range branch deadline.
+ //
+ // When a short-range branch has been successfully bound to its destination
+ // label, call this function to stop traching the branch.
+ //
+ // The (rangeIdx, deadline) pair must be previously registered.
+ //
+ void unregisterBranchDeadline(unsigned rangeIdx, BufferOffset deadline) {
+ if (!this->oom()) {
+ branchDeadlines_.removeDeadline(rangeIdx, deadline);
+ }
+ }
+
+ private:
+ // Are any short-range branches about to expire?
+ bool hasExpirableShortRangeBranches(size_t reservedBytes) const {
+ if (branchDeadlines_.empty()) {
+ return false;
+ }
+
+ // Include branches that would expire in the next N bytes. The reservedBytes
+ // argument avoids the needless creation of many tiny constant pools.
+ //
+ // As the reservedBytes could be of any sizes such as SIZE_MAX, in the case
+ // of flushPool, we have to check for overflow when comparing the deadline
+ // with our expected reserved bytes.
+ size_t deadline = branchDeadlines_.earliestDeadline().getOffset();
+ using CheckedSize = mozilla::CheckedInt<size_t>;
+ CheckedSize current(this->nextOffset().getOffset());
+ CheckedSize poolFreeSpace(reservedBytes);
+ auto future = current + poolFreeSpace;
+ return !future.isValid() || deadline < future.value();
+ }
+
+ bool isPoolEmptyFor(size_t bytes) const {
+ return pool_.numEntries() == 0 && !hasExpirableShortRangeBranches(bytes);
+ }
+ void finishPool(size_t reservedBytes) {
+ JitSpew(JitSpew_Pools, "Attempting to finish pool %zu with %u entries.",
+ poolInfo_.length(), pool_.numEntries());
+
+ if (reservedBytes < ShortRangeBranchHysteresis) {
+ reservedBytes = ShortRangeBranchHysteresis;
+ }
+
+ if (isPoolEmptyFor(reservedBytes)) {
+ // If there is no data in the pool being dumped, don't dump anything.
+ JitSpew(JitSpew_Pools, "Aborting because the pool is empty");
+ return;
+ }
+
+ // Should not be placing a pool in a no-pool region, check.
+ MOZ_ASSERT(!canNotPlacePool_);
+
+ // Dump the pool with a guard branch around the pool.
+ BufferOffset guard = this->putBytes(guardSize_ * InstSize, nullptr);
+ BufferOffset header = this->putBytes(headerSize_ * InstSize, nullptr);
+ BufferOffset data = this->putBytesLarge(pool_.getPoolSize(),
+ (const uint8_t*)pool_.poolData());
+ if (this->oom()) {
+ return;
+ }
+
+ // Now generate branch veneers for any short-range branches that are
+ // about to expire.
+ while (hasExpirableShortRangeBranches(reservedBytes)) {
+ unsigned rangeIdx = branchDeadlines_.earliestDeadlineRange();
+ BufferOffset deadline = branchDeadlines_.earliestDeadline();
+
+ // Stop tracking this branch. The Asm callback below may register
+ // new branches to track.
+ branchDeadlines_.removeDeadline(rangeIdx, deadline);
+
+ // Make room for the veneer. Same as a pool guard branch.
+ BufferOffset veneer = this->putBytes(guardSize_ * InstSize, nullptr);
+ if (this->oom()) {
+ return;
+ }
+
+ // Fix the branch so it targets the veneer.
+ // The Asm class knows how to find the original branch given the
+ // (rangeIdx, deadline) pair.
+ Asm::PatchShortRangeBranchToVeneer(this, rangeIdx, deadline, veneer);
+ }
+
+ // We only reserved space for the guard branch and pool header.
+ // Fill them in.
+ BufferOffset afterPool = this->nextOffset();
+ Asm::WritePoolGuard(guard, this->getInst(guard), afterPool);
+ Asm::WritePoolHeader((uint8_t*)this->getInst(header), &pool_, false);
+
+ // With the pool's final position determined it is now possible to patch
+ // the instructions that reference entries in this pool, and this is
+ // done incrementally as each pool is finished.
+ size_t poolOffset = data.getOffset();
+
+ unsigned idx = 0;
+ for (BufferOffset* iter = pool_.loadOffsets.begin();
+ iter != pool_.loadOffsets.end(); ++iter, ++idx) {
+ // All entries should be before the pool.
+ MOZ_ASSERT(iter->getOffset() < guard.getOffset());
+
+ // Everything here is known so we can safely do the necessary
+ // substitutions.
+ Inst* inst = this->getInst(*iter);
+ size_t codeOffset = poolOffset - iter->getOffset();
+
+ // That is, PatchConstantPoolLoad wants to be handed the address of
+ // the pool entry that is being loaded. We need to do a non-trivial
+ // amount of math here, since the pool that we've made does not
+ // actually reside there in memory.
+ JitSpew(JitSpew_Pools, "Fixing entry %d offset to %zu", idx, codeOffset);
+ Asm::PatchConstantPoolLoad(inst, (uint8_t*)inst + codeOffset);
+ }
+
+ // Record the pool info.
+ unsigned firstEntry = poolEntryCount - pool_.numEntries();
+ if (!poolInfo_.append(PoolInfo(firstEntry, data))) {
+ this->fail_oom();
+ return;
+ }
+
+ // Reset everything to the state that it was in when we started.
+ pool_.reset();
+ }
+
+ public:
+ void flushPool() {
+ if (this->oom()) {
+ return;
+ }
+ JitSpew(JitSpew_Pools, "Requesting a pool flush");
+ finishPool(SIZE_MAX);
+ }
+
+ void enterNoPool(size_t maxInst) {
+ if (this->oom()) {
+ return;
+ }
+ // Don't allow re-entry.
+ MOZ_ASSERT(!canNotPlacePool_);
+ insertNopFill();
+
+ // Check if the pool will spill by adding maxInst instructions, and if
+ // so then finish the pool before entering the no-pool region. It is
+ // assumed that no pool entries are allocated in a no-pool region and
+ // this is asserted when allocating entries.
+ if (!hasSpaceForInsts(maxInst, 0)) {
+ JitSpew(JitSpew_Pools, "No-Pool instruction(%zu) caused a spill.",
+ sizeExcludingCurrentPool());
+ finishPool(maxInst * InstSize);
+ if (this->oom()) {
+ return;
+ }
+ MOZ_ASSERT(hasSpaceForInsts(maxInst, 0));
+ }
+
+#ifdef DEBUG
+ // Record the buffer position to allow validating maxInst when leaving
+ // the region.
+ canNotPlacePoolStartOffset_ = this->nextOffset().getOffset();
+ canNotPlacePoolMaxInst_ = maxInst;
+#endif
+
+ canNotPlacePool_ = true;
+ }
+
+ void leaveNoPool() {
+ if (this->oom()) {
+ canNotPlacePool_ = false;
+ return;
+ }
+ MOZ_ASSERT(canNotPlacePool_);
+ canNotPlacePool_ = false;
+
+ // Validate the maxInst argument supplied to enterNoPool().
+ MOZ_ASSERT(this->nextOffset().getOffset() - canNotPlacePoolStartOffset_ <=
+ canNotPlacePoolMaxInst_ * InstSize);
+ }
+
+ void enterNoNops() {
+ MOZ_ASSERT(!inhibitNops_);
+ inhibitNops_ = true;
+ }
+ void leaveNoNops() {
+ MOZ_ASSERT(inhibitNops_);
+ inhibitNops_ = false;
+ }
+ void assertNoPoolAndNoNops() {
+ MOZ_ASSERT(inhibitNops_);
+ MOZ_ASSERT_IF(!this->oom(), isPoolEmptyFor(InstSize) || canNotPlacePool_);
+ }
+
+ void align(unsigned alignment) { align(alignment, alignFillInst_); }
+
+ void align(unsigned alignment, uint32_t pattern) {
+ MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment));
+ MOZ_ASSERT(alignment >= InstSize);
+
+ // A pool many need to be dumped at this point, so insert NOP fill here.
+ insertNopFill();
+
+ // Check if the code position can be aligned without dumping a pool.
+ unsigned requiredFill = sizeExcludingCurrentPool() & (alignment - 1);
+ if (requiredFill == 0) {
+ return;
+ }
+ requiredFill = alignment - requiredFill;
+
+ // Add an InstSize because it is probably not useful for a pool to be
+ // dumped at the aligned code position.
+ if (!hasSpaceForInsts(requiredFill / InstSize + 1, 0)) {
+ // Alignment would cause a pool dump, so dump the pool now.
+ JitSpew(JitSpew_Pools, "Alignment of %d at %zu caused a spill.",
+ alignment, sizeExcludingCurrentPool());
+ finishPool(requiredFill);
+ }
+
+ bool prevInhibitNops = inhibitNops_;
+ inhibitNops_ = true;
+ while ((sizeExcludingCurrentPool() & (alignment - 1)) && !this->oom()) {
+ putInt(pattern);
+ }
+ inhibitNops_ = prevInhibitNops;
+ }
+
+ public:
+ void executableCopy(uint8_t* dest) {
+ if (this->oom()) {
+ return;
+ }
+ // The pools should have all been flushed, check.
+ MOZ_ASSERT(pool_.numEntries() == 0);
+ for (Slice* cur = getHead(); cur != nullptr; cur = cur->getNext()) {
+ memcpy(dest, &cur->instructions[0], cur->length());
+ dest += cur->length();
+ }
+ }
+
+ bool appendRawCode(const uint8_t* code, size_t numBytes) {
+ if (this->oom()) {
+ return false;
+ }
+ // The pools should have all been flushed, check.
+ MOZ_ASSERT(pool_.numEntries() == 0);
+ while (numBytes > SliceSize) {
+ this->putBytes(SliceSize, code);
+ numBytes -= SliceSize;
+ code += SliceSize;
+ }
+ this->putBytes(numBytes, code);
+ return !this->oom();
+ }
+
+ public:
+ size_t poolEntryOffset(PoolEntry pe) const {
+ MOZ_ASSERT(pe.index() < poolEntryCount - pool_.numEntries(),
+ "Invalid pool entry, or not flushed yet.");
+ // Find the pool containing pe.index().
+ // The array is sorted, so we can use a binary search.
+ auto b = poolInfo_.begin(), e = poolInfo_.end();
+ // A note on asymmetric types in the upper_bound comparator:
+ // http://permalink.gmane.org/gmane.comp.compilers.clang.devel/10101
+ auto i = std::upper_bound(b, e, pe.index(),
+ [](size_t value, const PoolInfo& entry) {
+ return value < entry.firstEntryIndex;
+ });
+ // Since upper_bound finds the first pool greater than pe,
+ // we want the previous one which is the last one less than or equal.
+ MOZ_ASSERT(i != b, "PoolInfo not sorted or empty?");
+ --i;
+ // The i iterator now points to the pool containing pe.index.
+ MOZ_ASSERT(i->firstEntryIndex <= pe.index() &&
+ (i + 1 == e || (i + 1)->firstEntryIndex > pe.index()));
+ // Compute the byte offset into the pool.
+ unsigned relativeIndex = pe.index() - i->firstEntryIndex;
+ return i->offset.getOffset() + relativeIndex * sizeof(PoolAllocUnit);
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif // jit_shared_IonAssemblerBufferWithConstantPools_h
diff --git a/js/src/jit/shared/LIR-shared.h b/js/src/jit/shared/LIR-shared.h
new file mode 100644
index 0000000000..9bfdb1c3a0
--- /dev/null
+++ b/js/src/jit/shared/LIR-shared.h
@@ -0,0 +1,4183 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_LIR_shared_h
+#define jit_shared_LIR_shared_h
+
+#include "mozilla/Maybe.h"
+#include "jit/AtomicOp.h"
+#include "jit/shared/Assembler-shared.h"
+#include "util/Memory.h"
+
+// This file declares LIR instructions that are common to every platform.
+
+namespace js {
+namespace jit {
+
+LIR_OPCODE_CLASS_GENERATED
+
+#ifdef FUZZING_JS_FUZZILLI
+class LFuzzilliHashT : public LInstructionHelper<1, 1, 2> {
+ public:
+ LIR_HEADER(FuzzilliHashT);
+
+ LFuzzilliHashT(const LAllocation& value, const LDefinition& temp,
+ const LDefinition& tempFloat)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, value);
+ setTemp(0, temp);
+ setTemp(1, tempFloat);
+ }
+
+ const LAllocation* value() { return getOperand(0); }
+
+ MFuzzilliHash* mir() const { return mir_->toFuzzilliHash(); }
+};
+
+class LFuzzilliHashV : public LInstructionHelper<1, BOX_PIECES, 2> {
+ public:
+ LIR_HEADER(FuzzilliHashV);
+
+ LFuzzilliHashV(const LBoxAllocation& value, const LDefinition& temp,
+ const LDefinition& tempFloat)
+ : LInstructionHelper(classOpcode) {
+ setBoxOperand(0, value);
+ setTemp(0, temp);
+ setTemp(1, tempFloat);
+ }
+
+ MFuzzilliHash* mir() const { return mir_->toFuzzilliHash(); }
+};
+
+class LFuzzilliHashStore : public LInstructionHelper<0, 1, 2> {
+ public:
+ LIR_HEADER(FuzzilliHashStore);
+
+ LFuzzilliHashStore(const LAllocation& value, const LDefinition& temp1,
+ const LDefinition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, value);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LAllocation* value() { return getOperand(0); }
+
+ MFuzzilliHashStore* mir() const { return mir_->toFuzzilliHashStore(); }
+};
+#endif
+
+class LBox : public LInstructionHelper<BOX_PIECES, 1, 0> {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(Box);
+
+ LBox(const LAllocation& payload, MIRType type)
+ : LInstructionHelper(classOpcode), type_(type) {
+ setOperand(0, payload);
+ }
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+template <size_t Temps, size_t ExtraUses = 0>
+class LBinaryMath : public LInstructionHelper<1, 2 + ExtraUses, Temps> {
+ protected:
+ explicit LBinaryMath(LNode::Opcode opcode)
+ : LInstructionHelper<1, 2 + ExtraUses, Temps>(opcode) {}
+
+ public:
+ const LAllocation* lhs() { return this->getOperand(0); }
+ const LAllocation* rhs() { return this->getOperand(1); }
+};
+
+template <size_t Temps, size_t ExtraUses = 0>
+class LUnaryMath : public LInstructionHelper<1, 1 + ExtraUses, Temps> {
+ protected:
+ explicit LUnaryMath(LNode::Opcode opcode)
+ : LInstructionHelper<1, 1 + ExtraUses, Temps>(opcode) {}
+
+ public:
+ const LAllocation* input() { return this->getOperand(0); }
+};
+
+// An LOsiPoint captures a snapshot after a call and ensures enough space to
+// patch in a call to the invalidation mechanism.
+//
+// Note: LSafepoints are 1:1 with LOsiPoints, so it holds a reference to the
+// corresponding LSafepoint to inform it of the LOsiPoint's masm offset when it
+// gets GC'd.
+class LOsiPoint : public LInstructionHelper<0, 0, 0> {
+ LSafepoint* safepoint_;
+
+ public:
+ LOsiPoint(LSafepoint* safepoint, LSnapshot* snapshot)
+ : LInstructionHelper(classOpcode), safepoint_(safepoint) {
+ MOZ_ASSERT(safepoint && snapshot);
+ assignSnapshot(snapshot);
+ }
+
+ LSafepoint* associatedSafepoint() { return safepoint_; }
+
+ LIR_HEADER(OsiPoint)
+};
+
+class LMove {
+ LAllocation from_;
+ LAllocation to_;
+ LDefinition::Type type_;
+
+ public:
+ LMove(LAllocation from, LAllocation to, LDefinition::Type type)
+ : from_(from), to_(to), type_(type) {}
+
+ LAllocation from() const { return from_; }
+ LAllocation to() const { return to_; }
+ LDefinition::Type type() const { return type_; }
+};
+
+class LMoveGroup : public LInstructionHelper<0, 0, 0> {
+ js::Vector<LMove, 2, JitAllocPolicy> moves_;
+
+#ifdef JS_CODEGEN_X86
+ // Optional general register available for use when executing moves.
+ LAllocation scratchRegister_;
+#endif
+
+ explicit LMoveGroup(TempAllocator& alloc)
+ : LInstructionHelper(classOpcode), moves_(alloc) {}
+
+ public:
+ LIR_HEADER(MoveGroup)
+
+ static LMoveGroup* New(TempAllocator& alloc) {
+ return new (alloc) LMoveGroup(alloc);
+ }
+
+ void printOperands(GenericPrinter& out);
+
+ // Add a move which takes place simultaneously with all others in the group.
+ bool add(LAllocation from, LAllocation to, LDefinition::Type type);
+
+ // Add a move which takes place after existing moves in the group.
+ bool addAfter(LAllocation from, LAllocation to, LDefinition::Type type);
+
+ size_t numMoves() const { return moves_.length(); }
+ const LMove& getMove(size_t i) const { return moves_[i]; }
+
+#ifdef JS_CODEGEN_X86
+ void setScratchRegister(Register reg) { scratchRegister_ = LGeneralReg(reg); }
+ LAllocation maybeScratchRegister() { return scratchRegister_; }
+#endif
+
+ bool uses(Register reg) {
+ for (size_t i = 0; i < numMoves(); i++) {
+ LMove move = getMove(i);
+ if (move.from() == LGeneralReg(reg) || move.to() == LGeneralReg(reg)) {
+ return true;
+ }
+ }
+ return false;
+ }
+};
+
+// A constant Value.
+class LValue : public LInstructionHelper<BOX_PIECES, 0, 0> {
+ Value v_;
+
+ public:
+ LIR_HEADER(Value)
+
+ explicit LValue(const Value& v) : LInstructionHelper(classOpcode), v_(v) {}
+
+ Value value() const { return v_; }
+};
+
+// Base class for control instructions (goto, branch, etc.)
+template <size_t Succs, size_t Operands, size_t Temps>
+class LControlInstructionHelper
+ : public LInstructionHelper<0, Operands, Temps> {
+ mozilla::Array<MBasicBlock*, Succs> successors_;
+
+ protected:
+ explicit LControlInstructionHelper(LNode::Opcode opcode)
+ : LInstructionHelper<0, Operands, Temps>(opcode) {}
+
+ public:
+ size_t numSuccessors() const { return Succs; }
+ MBasicBlock* getSuccessor(size_t i) const { return successors_[i]; }
+
+ void setSuccessor(size_t i, MBasicBlock* successor) {
+ successors_[i] = successor;
+ }
+};
+
+// Jumps to the start of a basic block.
+class LGoto : public LControlInstructionHelper<1, 0, 0> {
+ public:
+ LIR_HEADER(Goto)
+
+ explicit LGoto(MBasicBlock* block) : LControlInstructionHelper(classOpcode) {
+ setSuccessor(0, block);
+ }
+
+ MBasicBlock* target() const { return getSuccessor(0); }
+};
+
+class LNewArray : public LInstructionHelper<1, 0, 1> {
+ public:
+ LIR_HEADER(NewArray)
+
+ explicit LNewArray(const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setTemp(0, temp);
+ }
+
+ const char* extraName() const {
+ return mir()->isVMCall() ? "VMCall" : nullptr;
+ }
+
+ const LDefinition* temp() { return getTemp(0); }
+
+ MNewArray* mir() const { return mir_->toNewArray(); }
+};
+
+class LNewObject : public LInstructionHelper<1, 0, 1> {
+ public:
+ LIR_HEADER(NewObject)
+
+ explicit LNewObject(const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setTemp(0, temp);
+ }
+
+ const char* extraName() const {
+ return mir()->isVMCall() ? "VMCall" : nullptr;
+ }
+
+ const LDefinition* temp() { return getTemp(0); }
+
+ MNewObject* mir() const { return mir_->toNewObject(); }
+};
+
+template <size_t Defs, size_t Ops>
+class LWasmReinterpretBase : public LInstructionHelper<Defs, Ops, 0> {
+ typedef LInstructionHelper<Defs, Ops, 0> Base;
+
+ protected:
+ explicit LWasmReinterpretBase(LNode::Opcode opcode) : Base(opcode) {}
+
+ public:
+ const LAllocation* input() { return Base::getOperand(0); }
+ MWasmReinterpret* mir() const { return Base::mir_->toWasmReinterpret(); }
+};
+
+class LWasmReinterpret : public LWasmReinterpretBase<1, 1> {
+ public:
+ LIR_HEADER(WasmReinterpret);
+ explicit LWasmReinterpret(const LAllocation& input)
+ : LWasmReinterpretBase(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+class LWasmReinterpretFromI64 : public LWasmReinterpretBase<1, INT64_PIECES> {
+ public:
+ static const size_t Input = 0;
+
+ LIR_HEADER(WasmReinterpretFromI64);
+ explicit LWasmReinterpretFromI64(const LInt64Allocation& input)
+ : LWasmReinterpretBase(classOpcode) {
+ setInt64Operand(Input, input);
+ }
+};
+
+class LWasmReinterpretToI64 : public LWasmReinterpretBase<INT64_PIECES, 1> {
+ public:
+ LIR_HEADER(WasmReinterpretToI64);
+ explicit LWasmReinterpretToI64(const LAllocation& input)
+ : LWasmReinterpretBase(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+namespace details {
+template <size_t Defs, size_t Ops, size_t Temps>
+class RotateBase : public LInstructionHelper<Defs, Ops, Temps> {
+ typedef LInstructionHelper<Defs, Ops, Temps> Base;
+
+ protected:
+ explicit RotateBase(LNode::Opcode opcode) : Base(opcode) {}
+
+ public:
+ MRotate* mir() { return Base::mir_->toRotate(); }
+};
+} // namespace details
+
+class LRotate : public details::RotateBase<1, 2, 0> {
+ public:
+ LIR_HEADER(Rotate);
+
+ LRotate() : RotateBase(classOpcode) {}
+
+ const LAllocation* input() { return getOperand(0); }
+ LAllocation* count() { return getOperand(1); }
+};
+
+class LRotateI64
+ : public details::RotateBase<INT64_PIECES, INT64_PIECES + 1, 1> {
+ public:
+ LIR_HEADER(RotateI64);
+
+ LRotateI64() : RotateBase(classOpcode) {
+ setTemp(0, LDefinition::BogusTemp());
+ }
+
+ static const size_t Input = 0;
+ static const size_t Count = INT64_PIECES;
+
+ const LInt64Allocation input() { return getInt64Operand(Input); }
+ const LDefinition* temp() { return getTemp(0); }
+ LAllocation* count() { return getOperand(Count); }
+};
+
+// Allocate a new arguments object for an inlined frame.
+class LCreateInlinedArgumentsObject : public LVariadicInstruction<1, 2> {
+ public:
+ LIR_HEADER(CreateInlinedArgumentsObject)
+
+ static const size_t CallObj = 0;
+ static const size_t Callee = 1;
+ static const size_t NumNonArgumentOperands = 2;
+ static size_t ArgIndex(size_t i) {
+ return NumNonArgumentOperands + BOX_PIECES * i;
+ }
+
+ LCreateInlinedArgumentsObject(uint32_t numOperands, const LDefinition& temp1,
+ const LDefinition& temp2)
+ : LVariadicInstruction(classOpcode, numOperands) {
+ setIsCall();
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LAllocation* getCallObject() { return getOperand(CallObj); }
+ const LAllocation* getCallee() { return getOperand(Callee); }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+
+ MCreateInlinedArgumentsObject* mir() const {
+ return mir_->toCreateInlinedArgumentsObject();
+ }
+};
+
+class LGetInlinedArgument : public LVariadicInstruction<BOX_PIECES, 0> {
+ public:
+ LIR_HEADER(GetInlinedArgument)
+
+ static const size_t Index = 0;
+ static const size_t NumNonArgumentOperands = 1;
+ static size_t ArgIndex(size_t i) {
+ return NumNonArgumentOperands + BOX_PIECES * i;
+ }
+
+ explicit LGetInlinedArgument(uint32_t numOperands)
+ : LVariadicInstruction(classOpcode, numOperands) {}
+
+ const LAllocation* getIndex() { return getOperand(Index); }
+
+ MGetInlinedArgument* mir() const { return mir_->toGetInlinedArgument(); }
+};
+
+class LGetInlinedArgumentHole : public LVariadicInstruction<BOX_PIECES, 0> {
+ public:
+ LIR_HEADER(GetInlinedArgumentHole)
+
+ static const size_t Index = 0;
+ static const size_t NumNonArgumentOperands = 1;
+ static size_t ArgIndex(size_t i) {
+ return NumNonArgumentOperands + BOX_PIECES * i;
+ }
+
+ explicit LGetInlinedArgumentHole(uint32_t numOperands)
+ : LVariadicInstruction(classOpcode, numOperands) {}
+
+ const LAllocation* getIndex() { return getOperand(Index); }
+
+ MGetInlinedArgumentHole* mir() const {
+ return mir_->toGetInlinedArgumentHole();
+ }
+};
+
+class LInlineArgumentsSlice : public LVariadicInstruction<1, 1> {
+ public:
+ LIR_HEADER(InlineArgumentsSlice)
+
+ static const size_t Begin = 0;
+ static const size_t Count = 1;
+ static const size_t NumNonArgumentOperands = 2;
+ static size_t ArgIndex(size_t i) {
+ return NumNonArgumentOperands + BOX_PIECES * i;
+ }
+
+ explicit LInlineArgumentsSlice(uint32_t numOperands, const LDefinition& temp)
+ : LVariadicInstruction(classOpcode, numOperands) {
+ setTemp(0, temp);
+ }
+
+ const LAllocation* begin() { return getOperand(Begin); }
+ const LAllocation* count() { return getOperand(Count); }
+ const LDefinition* temp() { return getTemp(0); }
+
+ MInlineArgumentsSlice* mir() const { return mir_->toInlineArgumentsSlice(); }
+};
+
+// Common code for LIR descended from MCall.
+template <size_t Defs, size_t Operands, size_t Temps>
+class LJSCallInstructionHelper
+ : public LCallInstructionHelper<Defs, Operands, Temps> {
+ protected:
+ explicit LJSCallInstructionHelper(LNode::Opcode opcode)
+ : LCallInstructionHelper<Defs, Operands, Temps>(opcode) {}
+
+ public:
+ uint32_t paddedNumStackArgs() const {
+ if (JitStackValueAlignment > 1) {
+ return AlignBytes(mir()->numStackArgs(), JitStackValueAlignment);
+ }
+ return mir()->numStackArgs();
+ }
+ MCall* mir() const { return this->mir_->toCall(); }
+
+ bool hasSingleTarget() const { return getSingleTarget() != nullptr; }
+ WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); }
+
+ // Does not include |this|.
+ uint32_t numActualArgs() const { return mir()->numActualArgs(); }
+
+ bool isConstructing() const { return mir()->isConstructing(); }
+ bool ignoresReturnValue() const { return mir()->ignoresReturnValue(); }
+};
+
+// Generates a polymorphic callsite, wherein the function being called is
+// unknown and anticipated to vary.
+class LCallGeneric : public LJSCallInstructionHelper<BOX_PIECES, 1, 2> {
+ public:
+ LIR_HEADER(CallGeneric)
+
+ LCallGeneric(const LAllocation& func, const LDefinition& nargsreg,
+ const LDefinition& tmpobjreg)
+ : LJSCallInstructionHelper(classOpcode) {
+ setOperand(0, func);
+ setTemp(0, nargsreg);
+ setTemp(1, tmpobjreg);
+ }
+
+ const LAllocation* getFunction() { return getOperand(0); }
+ const LDefinition* getNargsReg() { return getTemp(0); }
+ const LDefinition* getTempObject() { return getTemp(1); }
+};
+
+// Generates a hardcoded callsite for a known, non-native target.
+class LCallKnown : public LJSCallInstructionHelper<BOX_PIECES, 1, 1> {
+ public:
+ LIR_HEADER(CallKnown)
+
+ LCallKnown(const LAllocation& func, const LDefinition& tmpobjreg)
+ : LJSCallInstructionHelper(classOpcode) {
+ setOperand(0, func);
+ setTemp(0, tmpobjreg);
+ }
+
+ const LAllocation* getFunction() { return getOperand(0); }
+ const LDefinition* getTempObject() { return getTemp(0); }
+};
+
+// Generates a hardcoded callsite for a known, native target.
+class LCallNative : public LJSCallInstructionHelper<BOX_PIECES, 0, 4> {
+ public:
+ LIR_HEADER(CallNative)
+
+ LCallNative(const LDefinition& argContext, const LDefinition& argUintN,
+ const LDefinition& argVp, const LDefinition& tmpreg)
+ : LJSCallInstructionHelper(classOpcode) {
+ // Registers used for callWithABI().
+ setTemp(0, argContext);
+ setTemp(1, argUintN);
+ setTemp(2, argVp);
+
+ // Temporary registers.
+ setTemp(3, tmpreg);
+ }
+
+ const LDefinition* getArgContextReg() { return getTemp(0); }
+ const LDefinition* getArgUintNReg() { return getTemp(1); }
+ const LDefinition* getArgVpReg() { return getTemp(2); }
+ const LDefinition* getTempReg() { return getTemp(3); }
+};
+
+// Generates a hardcoded callsite for a known, DOM-native target.
+class LCallDOMNative : public LJSCallInstructionHelper<BOX_PIECES, 0, 4> {
+ public:
+ LIR_HEADER(CallDOMNative)
+
+ LCallDOMNative(const LDefinition& argJSContext, const LDefinition& argObj,
+ const LDefinition& argPrivate, const LDefinition& argArgs)
+ : LJSCallInstructionHelper(classOpcode) {
+ setTemp(0, argJSContext);
+ setTemp(1, argObj);
+ setTemp(2, argPrivate);
+ setTemp(3, argArgs);
+ }
+
+ const LDefinition* getArgJSContext() { return getTemp(0); }
+ const LDefinition* getArgObj() { return getTemp(1); }
+ const LDefinition* getArgPrivate() { return getTemp(2); }
+ const LDefinition* getArgArgs() { return getTemp(3); }
+};
+
+class LUnreachable : public LControlInstructionHelper<0, 0, 0> {
+ public:
+ LIR_HEADER(Unreachable)
+
+ LUnreachable() : LControlInstructionHelper(classOpcode) {}
+};
+
+class LUnreachableResultV : public LInstructionHelper<BOX_PIECES, 0, 0> {
+ public:
+ LIR_HEADER(UnreachableResultV)
+
+ LUnreachableResultV() : LInstructionHelper(classOpcode) {}
+};
+
+template <size_t defs, size_t ops>
+class LDOMPropertyInstructionHelper
+ : public LCallInstructionHelper<defs, 1 + ops, 3> {
+ protected:
+ LDOMPropertyInstructionHelper(LNode::Opcode opcode,
+ const LDefinition& JSContextReg,
+ const LAllocation& ObjectReg,
+ const LDefinition& PrivReg,
+ const LDefinition& ValueReg)
+ : LCallInstructionHelper<defs, 1 + ops, 3>(opcode) {
+ this->setOperand(0, ObjectReg);
+ this->setTemp(0, JSContextReg);
+ this->setTemp(1, PrivReg);
+ this->setTemp(2, ValueReg);
+ }
+
+ public:
+ const LDefinition* getJSContextReg() { return this->getTemp(0); }
+ const LAllocation* getObjectReg() { return this->getOperand(0); }
+ const LDefinition* getPrivReg() { return this->getTemp(1); }
+ const LDefinition* getValueReg() { return this->getTemp(2); }
+};
+
+class LGetDOMProperty : public LDOMPropertyInstructionHelper<BOX_PIECES, 0> {
+ public:
+ LIR_HEADER(GetDOMProperty)
+
+ LGetDOMProperty(const LDefinition& JSContextReg, const LAllocation& ObjectReg,
+ const LDefinition& PrivReg, const LDefinition& ValueReg)
+ : LDOMPropertyInstructionHelper<BOX_PIECES, 0>(
+ classOpcode, JSContextReg, ObjectReg, PrivReg, ValueReg) {}
+
+ MGetDOMProperty* mir() const { return mir_->toGetDOMProperty(); }
+};
+
+class LGetDOMMemberV : public LInstructionHelper<BOX_PIECES, 1, 0> {
+ public:
+ LIR_HEADER(GetDOMMemberV);
+ explicit LGetDOMMemberV(const LAllocation& object)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, object);
+ }
+
+ const LAllocation* object() { return getOperand(0); }
+
+ MGetDOMMember* mir() const { return mir_->toGetDOMMember(); }
+};
+
+class LSetDOMProperty : public LDOMPropertyInstructionHelper<0, BOX_PIECES> {
+ public:
+ LIR_HEADER(SetDOMProperty)
+
+ LSetDOMProperty(const LDefinition& JSContextReg, const LAllocation& ObjectReg,
+ const LBoxAllocation& value, const LDefinition& PrivReg,
+ const LDefinition& ValueReg)
+ : LDOMPropertyInstructionHelper<0, BOX_PIECES>(
+ classOpcode, JSContextReg, ObjectReg, PrivReg, ValueReg) {
+ setBoxOperand(Value, value);
+ }
+
+ static const size_t Value = 1;
+
+ MSetDOMProperty* mir() const { return mir_->toSetDOMProperty(); }
+};
+
+// Generates a polymorphic callsite, wherein the function being called is
+// unknown and anticipated to vary.
+class LApplyArgsGeneric
+ : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 2, 2> {
+ public:
+ LIR_HEADER(ApplyArgsGeneric)
+
+ LApplyArgsGeneric(const LAllocation& func, const LAllocation& argc,
+ const LBoxAllocation& thisv, const LDefinition& tmpobjreg,
+ const LDefinition& tmpcopy)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, func);
+ setOperand(1, argc);
+ setBoxOperand(ThisIndex, thisv);
+ setTemp(0, tmpobjreg);
+ setTemp(1, tmpcopy);
+ }
+
+ MApplyArgs* mir() const { return mir_->toApplyArgs(); }
+
+ bool hasSingleTarget() const { return getSingleTarget() != nullptr; }
+ WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); }
+
+ uint32_t numExtraFormals() const { return mir()->numExtraFormals(); }
+
+ const LAllocation* getFunction() { return getOperand(0); }
+ const LAllocation* getArgc() { return getOperand(1); }
+ static const size_t ThisIndex = 2;
+
+ const LDefinition* getTempObject() { return getTemp(0); }
+ const LDefinition* getTempForArgCopy() { return getTemp(1); }
+};
+
+class LApplyArgsObj
+ : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 2, 2> {
+ public:
+ LIR_HEADER(ApplyArgsObj)
+
+ LApplyArgsObj(const LAllocation& func, const LAllocation& argsObj,
+ const LBoxAllocation& thisv, const LDefinition& tmpObjReg,
+ const LDefinition& tmpCopy)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, func);
+ setOperand(1, argsObj);
+ setBoxOperand(ThisIndex, thisv);
+ setTemp(0, tmpObjReg);
+ setTemp(1, tmpCopy);
+ }
+
+ MApplyArgsObj* mir() const { return mir_->toApplyArgsObj(); }
+
+ bool hasSingleTarget() const { return getSingleTarget() != nullptr; }
+ WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); }
+
+ const LAllocation* getFunction() { return getOperand(0); }
+ const LAllocation* getArgsObj() { return getOperand(1); }
+ // All registers are calltemps. argc is mapped to the same register as
+ // ArgsObj. argc becomes live as ArgsObj is dying.
+ const LAllocation* getArgc() { return getOperand(1); }
+ static const size_t ThisIndex = 2;
+
+ const LDefinition* getTempObject() { return getTemp(0); }
+ const LDefinition* getTempForArgCopy() { return getTemp(1); }
+};
+
+class LApplyArrayGeneric
+ : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 2, 2> {
+ public:
+ LIR_HEADER(ApplyArrayGeneric)
+
+ LApplyArrayGeneric(const LAllocation& func, const LAllocation& elements,
+ const LBoxAllocation& thisv, const LDefinition& tmpobjreg,
+ const LDefinition& tmpcopy)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, func);
+ setOperand(1, elements);
+ setBoxOperand(ThisIndex, thisv);
+ setTemp(0, tmpobjreg);
+ setTemp(1, tmpcopy);
+ }
+
+ MApplyArray* mir() const { return mir_->toApplyArray(); }
+
+ bool hasSingleTarget() const { return getSingleTarget() != nullptr; }
+ WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); }
+
+ const LAllocation* getFunction() { return getOperand(0); }
+ const LAllocation* getElements() { return getOperand(1); }
+ // argc is mapped to the same register as elements: argc becomes
+ // live as elements is dying, all registers are calltemps.
+ const LAllocation* getArgc() { return getOperand(1); }
+ static const size_t ThisIndex = 2;
+
+ const LDefinition* getTempObject() { return getTemp(0); }
+ const LDefinition* getTempForArgCopy() { return getTemp(1); }
+};
+
+class LConstructArgsGeneric
+ : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 3, 1> {
+ public:
+ LIR_HEADER(ConstructArgsGeneric)
+
+ LConstructArgsGeneric(const LAllocation& func, const LAllocation& argc,
+ const LAllocation& newTarget,
+ const LBoxAllocation& thisv,
+ const LDefinition& tmpobjreg)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, func);
+ setOperand(1, argc);
+ setOperand(2, newTarget);
+ setBoxOperand(ThisIndex, thisv);
+ setTemp(0, tmpobjreg);
+ }
+
+ MConstructArgs* mir() const { return mir_->toConstructArgs(); }
+
+ bool hasSingleTarget() const { return getSingleTarget() != nullptr; }
+ WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); }
+
+ uint32_t numExtraFormals() const { return mir()->numExtraFormals(); }
+
+ const LAllocation* getFunction() { return getOperand(0); }
+ const LAllocation* getArgc() { return getOperand(1); }
+ const LAllocation* getNewTarget() { return getOperand(2); }
+
+ static const size_t ThisIndex = 3;
+
+ const LDefinition* getTempObject() { return getTemp(0); }
+
+ // tempForArgCopy is mapped to the same register as newTarget:
+ // tempForArgCopy becomes live as newTarget is dying, all registers are
+ // calltemps.
+ const LAllocation* getTempForArgCopy() { return getOperand(2); }
+};
+
+class LConstructArrayGeneric
+ : public LCallInstructionHelper<BOX_PIECES, BOX_PIECES + 3, 1> {
+ public:
+ LIR_HEADER(ConstructArrayGeneric)
+
+ LConstructArrayGeneric(const LAllocation& func, const LAllocation& elements,
+ const LAllocation& newTarget,
+ const LBoxAllocation& thisv,
+ const LDefinition& tmpobjreg)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, func);
+ setOperand(1, elements);
+ setOperand(2, newTarget);
+ setBoxOperand(ThisIndex, thisv);
+ setTemp(0, tmpobjreg);
+ }
+
+ MConstructArray* mir() const { return mir_->toConstructArray(); }
+
+ bool hasSingleTarget() const { return getSingleTarget() != nullptr; }
+ WrappedFunction* getSingleTarget() const { return mir()->getSingleTarget(); }
+
+ const LAllocation* getFunction() { return getOperand(0); }
+ const LAllocation* getElements() { return getOperand(1); }
+ const LAllocation* getNewTarget() { return getOperand(2); }
+
+ static const size_t ThisIndex = 3;
+
+ const LDefinition* getTempObject() { return getTemp(0); }
+
+ // argc is mapped to the same register as elements: argc becomes
+ // live as elements is dying, all registers are calltemps.
+ const LAllocation* getArgc() { return getOperand(1); }
+
+ // tempForArgCopy is mapped to the same register as newTarget:
+ // tempForArgCopy becomes live as newTarget is dying, all registers are
+ // calltemps.
+ const LAllocation* getTempForArgCopy() { return getOperand(2); }
+};
+
+// Takes in either an integer or boolean input and tests it for truthiness.
+class LTestIAndBranch : public LControlInstructionHelper<2, 1, 0> {
+ public:
+ LIR_HEADER(TestIAndBranch)
+
+ LTestIAndBranch(const LAllocation& in, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+};
+
+// Takes in an int64 input and tests it for truthiness.
+class LTestI64AndBranch : public LControlInstructionHelper<2, INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(TestI64AndBranch)
+
+ LTestI64AndBranch(const LInt64Allocation& in, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode) {
+ setInt64Operand(0, in);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+};
+
+// Takes in a double input and tests it for truthiness.
+class LTestDAndBranch : public LControlInstructionHelper<2, 1, 0> {
+ public:
+ LIR_HEADER(TestDAndBranch)
+
+ LTestDAndBranch(const LAllocation& in, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+};
+
+// Takes in a float32 input and tests it for truthiness.
+class LTestFAndBranch : public LControlInstructionHelper<2, 1, 0> {
+ public:
+ LIR_HEADER(TestFAndBranch)
+
+ LTestFAndBranch(const LAllocation& in, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+};
+
+// Takes in a bigint input and tests it for truthiness.
+class LTestBIAndBranch : public LControlInstructionHelper<2, 1, 0> {
+ public:
+ LIR_HEADER(TestBIAndBranch)
+
+ LTestBIAndBranch(const LAllocation& in, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ MBasicBlock* ifTrue() { return getSuccessor(0); }
+ MBasicBlock* ifFalse() { return getSuccessor(1); }
+};
+
+// Takes an object and tests it for truthiness. An object is falsy iff it
+// emulates |undefined|; see js::EmulatesUndefined.
+class LTestOAndBranch : public LControlInstructionHelper<2, 1, 1> {
+ public:
+ LIR_HEADER(TestOAndBranch)
+
+ LTestOAndBranch(const LAllocation& input, MBasicBlock* ifTruthy,
+ MBasicBlock* ifFalsy, const LDefinition& temp)
+ : LControlInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setSuccessor(0, ifTruthy);
+ setSuccessor(1, ifFalsy);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* temp() { return getTemp(0); }
+
+ MBasicBlock* ifTruthy() { return getSuccessor(0); }
+ MBasicBlock* ifFalsy() { return getSuccessor(1); }
+
+ MTest* mir() { return mir_->toTest(); }
+};
+
+// Takes in a boxed value and tests it for truthiness.
+class LTestVAndBranch : public LControlInstructionHelper<2, BOX_PIECES, 3> {
+ public:
+ LIR_HEADER(TestVAndBranch)
+
+ LTestVAndBranch(MBasicBlock* ifTruthy, MBasicBlock* ifFalsy,
+ const LBoxAllocation& input, const LDefinition& temp0,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LControlInstructionHelper(classOpcode) {
+ setSuccessor(0, ifTruthy);
+ setSuccessor(1, ifFalsy);
+ setBoxOperand(Input, input);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ setTemp(2, temp2);
+ }
+
+ static const size_t Input = 0;
+
+ const LDefinition* tempFloat() { return getTemp(0); }
+
+ const LDefinition* temp1() { return getTemp(1); }
+
+ const LDefinition* temp2() { return getTemp(2); }
+
+ MBasicBlock* ifTruthy() { return getSuccessor(0); }
+ MBasicBlock* ifFalsy() { return getSuccessor(1); }
+
+ MTest* mir() const { return mir_->toTest(); }
+};
+
+// Compares two integral values of the same JS type, either integer or object.
+// For objects, both operands are in registers.
+class LCompare : public LInstructionHelper<1, 2, 0> {
+ JSOp jsop_;
+
+ public:
+ LIR_HEADER(Compare)
+ LCompare(JSOp jsop, const LAllocation& left, const LAllocation& right)
+ : LInstructionHelper(classOpcode), jsop_(jsop) {
+ setOperand(0, left);
+ setOperand(1, right);
+ }
+
+ JSOp jsop() const { return jsop_; }
+ const LAllocation* left() { return getOperand(0); }
+ const LAllocation* right() { return getOperand(1); }
+ MCompare* mir() { return mir_->toCompare(); }
+ const char* extraName() const { return CodeName(jsop_); }
+};
+
+class LCompareI64 : public LInstructionHelper<1, 2 * INT64_PIECES, 0> {
+ JSOp jsop_;
+
+ public:
+ LIR_HEADER(CompareI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+
+ LCompareI64(JSOp jsop, const LInt64Allocation& left,
+ const LInt64Allocation& right)
+ : LInstructionHelper(classOpcode), jsop_(jsop) {
+ setInt64Operand(Lhs, left);
+ setInt64Operand(Rhs, right);
+ }
+
+ JSOp jsop() const { return jsop_; }
+ MCompare* mir() { return mir_->toCompare(); }
+ const char* extraName() const { return CodeName(jsop_); }
+};
+
+class LCompareI64AndBranch
+ : public LControlInstructionHelper<2, 2 * INT64_PIECES, 0> {
+ MCompare* cmpMir_;
+ JSOp jsop_;
+
+ public:
+ LIR_HEADER(CompareI64AndBranch)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+
+ LCompareI64AndBranch(MCompare* cmpMir, JSOp jsop,
+ const LInt64Allocation& left,
+ const LInt64Allocation& right, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir), jsop_(jsop) {
+ setInt64Operand(Lhs, left);
+ setInt64Operand(Rhs, right);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ JSOp jsop() const { return jsop_; }
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ MTest* mir() const { return mir_->toTest(); }
+ MCompare* cmpMir() const { return cmpMir_; }
+ const char* extraName() const { return CodeName(jsop_); }
+};
+
+// Compares two integral values of the same JS type, either integer or object.
+// For objects, both operands are in registers.
+class LCompareAndBranch : public LControlInstructionHelper<2, 2, 0> {
+ MCompare* cmpMir_;
+ JSOp jsop_;
+
+ public:
+ LIR_HEADER(CompareAndBranch)
+ LCompareAndBranch(MCompare* cmpMir, JSOp jsop, const LAllocation& left,
+ const LAllocation& right, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir), jsop_(jsop) {
+ setOperand(0, left);
+ setOperand(1, right);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ JSOp jsop() const { return jsop_; }
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ const LAllocation* left() { return getOperand(0); }
+ const LAllocation* right() { return getOperand(1); }
+ MTest* mir() const { return mir_->toTest(); }
+ MCompare* cmpMir() const { return cmpMir_; }
+ const char* extraName() const { return CodeName(jsop_); }
+};
+
+class LCompareDAndBranch : public LControlInstructionHelper<2, 2, 0> {
+ MCompare* cmpMir_;
+
+ public:
+ LIR_HEADER(CompareDAndBranch)
+
+ LCompareDAndBranch(MCompare* cmpMir, const LAllocation& left,
+ const LAllocation& right, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) {
+ setOperand(0, left);
+ setOperand(1, right);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ const LAllocation* left() { return getOperand(0); }
+ const LAllocation* right() { return getOperand(1); }
+ MTest* mir() const { return mir_->toTest(); }
+ MCompare* cmpMir() const { return cmpMir_; }
+};
+
+class LCompareFAndBranch : public LControlInstructionHelper<2, 2, 0> {
+ MCompare* cmpMir_;
+
+ public:
+ LIR_HEADER(CompareFAndBranch)
+ LCompareFAndBranch(MCompare* cmpMir, const LAllocation& left,
+ const LAllocation& right, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) {
+ setOperand(0, left);
+ setOperand(1, right);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ const LAllocation* left() { return getOperand(0); }
+ const LAllocation* right() { return getOperand(1); }
+ MTest* mir() const { return mir_->toTest(); }
+ MCompare* cmpMir() const { return cmpMir_; }
+};
+
+class LBitAndAndBranch : public LControlInstructionHelper<2, 2, 0> {
+ // This denotes only a single-word AND on the target. Hence `is64_` is
+ // required to be `false` on a 32-bit target.
+ bool is64_;
+ Assembler::Condition cond_;
+
+ public:
+ LIR_HEADER(BitAndAndBranch)
+ LBitAndAndBranch(MBasicBlock* ifTrue, MBasicBlock* ifFalse, bool is64,
+ Assembler::Condition cond = Assembler::NonZero)
+ : LControlInstructionHelper(classOpcode), is64_(is64), cond_(cond) {
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ const LAllocation* left() { return getOperand(0); }
+ const LAllocation* right() { return getOperand(1); }
+ bool is64() const { return is64_; }
+ Assembler::Condition cond() const {
+ MOZ_ASSERT(cond_ == Assembler::Zero || cond_ == Assembler::NonZero);
+ return cond_;
+ }
+};
+
+class LIsNullOrLikeUndefinedAndBranchV
+ : public LControlInstructionHelper<2, BOX_PIECES, 2> {
+ MCompare* cmpMir_;
+
+ public:
+ LIR_HEADER(IsNullOrLikeUndefinedAndBranchV)
+
+ LIsNullOrLikeUndefinedAndBranchV(MCompare* cmpMir, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse,
+ const LBoxAllocation& value,
+ const LDefinition& temp,
+ const LDefinition& tempToUnbox)
+ : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) {
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ setBoxOperand(Value, value);
+ setTemp(0, temp);
+ setTemp(1, tempToUnbox);
+ }
+
+ static const size_t Value = 0;
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ MTest* mir() const { return mir_->toTest(); }
+ MCompare* cmpMir() const { return cmpMir_; }
+ const LDefinition* temp() { return getTemp(0); }
+ const LDefinition* tempToUnbox() { return getTemp(1); }
+};
+
+class LIsNullOrLikeUndefinedAndBranchT
+ : public LControlInstructionHelper<2, 1, 1> {
+ MCompare* cmpMir_;
+
+ public:
+ LIR_HEADER(IsNullOrLikeUndefinedAndBranchT)
+
+ LIsNullOrLikeUndefinedAndBranchT(MCompare* cmpMir, const LAllocation& input,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse,
+ const LDefinition& temp)
+ : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) {
+ setOperand(0, input);
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ setTemp(0, temp);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ MTest* mir() const { return mir_->toTest(); }
+ MCompare* cmpMir() const { return cmpMir_; }
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+class LIsNullAndBranch : public LControlInstructionHelper<2, BOX_PIECES, 0> {
+ MCompare* cmpMir_;
+
+ public:
+ LIR_HEADER(IsNullAndBranch)
+
+ LIsNullAndBranch(MCompare* cmpMir, MBasicBlock* ifTrue, MBasicBlock* ifFalse,
+ const LBoxAllocation& value)
+ : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) {
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ setBoxOperand(Value, value);
+ }
+
+ static const size_t Value = 0;
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ MTest* mir() const { return mir_->toTest(); }
+ MCompare* cmpMir() const { return cmpMir_; }
+};
+
+class LIsUndefinedAndBranch
+ : public LControlInstructionHelper<2, BOX_PIECES, 0> {
+ MCompare* cmpMir_;
+
+ public:
+ LIR_HEADER(IsUndefinedAndBranch)
+
+ LIsUndefinedAndBranch(MCompare* cmpMir, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse, const LBoxAllocation& value)
+ : LControlInstructionHelper(classOpcode), cmpMir_(cmpMir) {
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ setBoxOperand(Value, value);
+ }
+
+ static const size_t Value = 0;
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+ MTest* mir() const { return mir_->toTest(); }
+ MCompare* cmpMir() const { return cmpMir_; }
+};
+
+// Bitwise not operation, takes a 32-bit integer as input and returning
+// a 32-bit integer result as an output.
+class LBitNotI : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(BitNotI)
+
+ LBitNotI() : LInstructionHelper(classOpcode) {}
+};
+
+class LBitNotI64 : public LInstructionHelper<INT64_PIECES, INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(BitNotI64)
+
+ LBitNotI64() : LInstructionHelper(classOpcode) {}
+};
+
+// Binary bitwise operation, taking two 32-bit integers as inputs and returning
+// a 32-bit integer result as an output.
+class LBitOpI : public LInstructionHelper<1, 2, 0> {
+ JSOp op_;
+
+ public:
+ LIR_HEADER(BitOpI)
+
+ explicit LBitOpI(JSOp op) : LInstructionHelper(classOpcode), op_(op) {}
+
+ const char* extraName() const {
+ if (bitop() == JSOp::Ursh && mir_->toUrsh()->bailoutsDisabled()) {
+ return "ursh:BailoutsDisabled";
+ }
+ return CodeName(op_);
+ }
+
+ JSOp bitop() const { return op_; }
+};
+
+class LBitOpI64 : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0> {
+ JSOp op_;
+
+ public:
+ LIR_HEADER(BitOpI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+
+ explicit LBitOpI64(JSOp op) : LInstructionHelper(classOpcode), op_(op) {}
+
+ const char* extraName() const { return CodeName(op_); }
+
+ JSOp bitop() const { return op_; }
+};
+
+// Shift operation, taking two 32-bit integers as inputs and returning
+// a 32-bit integer result as an output.
+class LShiftI : public LBinaryMath<0> {
+ JSOp op_;
+
+ public:
+ LIR_HEADER(ShiftI)
+
+ explicit LShiftI(JSOp op) : LBinaryMath(classOpcode), op_(op) {}
+
+ JSOp bitop() { return op_; }
+
+ MInstruction* mir() { return mir_->toInstruction(); }
+
+ const char* extraName() const { return CodeName(op_); }
+};
+
+class LShiftI64 : public LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 0> {
+ JSOp op_;
+
+ public:
+ LIR_HEADER(ShiftI64)
+
+ explicit LShiftI64(JSOp op) : LInstructionHelper(classOpcode), op_(op) {}
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+
+ JSOp bitop() { return op_; }
+
+ MInstruction* mir() { return mir_->toInstruction(); }
+
+ const char* extraName() const { return CodeName(op_); }
+};
+
+class LSignExtendInt64
+ : public LInstructionHelper<INT64_PIECES, INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(SignExtendInt64)
+
+ explicit LSignExtendInt64(const LInt64Allocation& input)
+ : LInstructionHelper(classOpcode) {
+ setInt64Operand(0, input);
+ }
+
+ const MSignExtendInt64* mir() const { return mir_->toSignExtendInt64(); }
+
+ MSignExtendInt64::Mode mode() const { return mir()->mode(); }
+};
+
+class LUrshD : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(UrshD)
+
+ LUrshD(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+// Returns from the function being compiled (not used in inlined frames). The
+// input must be a box.
+class LReturn : public LInstructionHelper<0, BOX_PIECES, 0> {
+ bool isGenerator_;
+
+ public:
+ LIR_HEADER(Return)
+
+ explicit LReturn(bool isGenerator)
+ : LInstructionHelper(classOpcode), isGenerator_(isGenerator) {}
+
+ bool isGenerator() { return isGenerator_; }
+};
+
+class LMinMaxBase : public LInstructionHelper<1, 2, 0> {
+ protected:
+ LMinMaxBase(LNode::Opcode opcode, const LAllocation& first,
+ const LAllocation& second)
+ : LInstructionHelper(opcode) {
+ setOperand(0, first);
+ setOperand(1, second);
+ }
+
+ public:
+ const LAllocation* first() { return this->getOperand(0); }
+ const LAllocation* second() { return this->getOperand(1); }
+ const LDefinition* output() { return this->getDef(0); }
+ MMinMax* mir() const { return mir_->toMinMax(); }
+ const char* extraName() const { return mir()->isMax() ? "Max" : "Min"; }
+};
+
+class LMinMaxI : public LMinMaxBase {
+ public:
+ LIR_HEADER(MinMaxI)
+ LMinMaxI(const LAllocation& first, const LAllocation& second)
+ : LMinMaxBase(classOpcode, first, second) {}
+};
+
+class LMinMaxD : public LMinMaxBase {
+ public:
+ LIR_HEADER(MinMaxD)
+ LMinMaxD(const LAllocation& first, const LAllocation& second)
+ : LMinMaxBase(classOpcode, first, second) {}
+};
+
+class LMinMaxF : public LMinMaxBase {
+ public:
+ LIR_HEADER(MinMaxF)
+ LMinMaxF(const LAllocation& first, const LAllocation& second)
+ : LMinMaxBase(classOpcode, first, second) {}
+};
+
+class LMinMaxArrayI : public LInstructionHelper<1, 1, 3> {
+ public:
+ LIR_HEADER(MinMaxArrayI);
+ LMinMaxArrayI(const LAllocation& array, const LDefinition& temp0,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, array);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ setTemp(2, temp2);
+ }
+
+ const LAllocation* array() { return getOperand(0); }
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+ const LDefinition* temp3() { return getTemp(2); }
+
+ bool isMax() const { return mir_->toMinMaxArray()->isMax(); }
+};
+
+class LMinMaxArrayD : public LInstructionHelper<1, 1, 3> {
+ public:
+ LIR_HEADER(MinMaxArrayD);
+ LMinMaxArrayD(const LAllocation& array, const LDefinition& floatTemp,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, array);
+ setTemp(0, floatTemp);
+ setTemp(1, temp1);
+ setTemp(2, temp2);
+ }
+
+ const LAllocation* array() { return getOperand(0); }
+ const LDefinition* floatTemp() { return getTemp(0); }
+ const LDefinition* temp1() { return getTemp(1); }
+ const LDefinition* temp2() { return getTemp(2); }
+
+ bool isMax() const { return mir_->toMinMaxArray()->isMax(); }
+};
+
+// Copysign for doubles.
+class LCopySignD : public LInstructionHelper<1, 2, 2> {
+ public:
+ LIR_HEADER(CopySignD)
+ explicit LCopySignD() : LInstructionHelper(classOpcode) {}
+};
+
+// Copysign for float32.
+class LCopySignF : public LInstructionHelper<1, 2, 2> {
+ public:
+ LIR_HEADER(CopySignF)
+ explicit LCopySignF() : LInstructionHelper(classOpcode) {}
+};
+
+class LAtan2D : public LCallInstructionHelper<1, 2, 0> {
+ public:
+ LIR_HEADER(Atan2D)
+ LAtan2D(const LAllocation& y, const LAllocation& x)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, y);
+ setOperand(1, x);
+ }
+
+ const LAllocation* y() { return getOperand(0); }
+
+ const LAllocation* x() { return getOperand(1); }
+
+ const LDefinition* output() { return getDef(0); }
+};
+
+class LHypot : public LCallInstructionHelper<1, 4, 0> {
+ uint32_t numOperands_;
+
+ public:
+ LIR_HEADER(Hypot)
+ LHypot(const LAllocation& x, const LAllocation& y)
+ : LCallInstructionHelper(classOpcode), numOperands_(2) {
+ setOperand(0, x);
+ setOperand(1, y);
+ }
+
+ LHypot(const LAllocation& x, const LAllocation& y, const LAllocation& z)
+ : LCallInstructionHelper(classOpcode), numOperands_(3) {
+ setOperand(0, x);
+ setOperand(1, y);
+ setOperand(2, z);
+ }
+
+ LHypot(const LAllocation& x, const LAllocation& y, const LAllocation& z,
+ const LAllocation& w)
+ : LCallInstructionHelper(classOpcode), numOperands_(4) {
+ setOperand(0, x);
+ setOperand(1, y);
+ setOperand(2, z);
+ setOperand(3, w);
+ }
+
+ uint32_t numArgs() const { return numOperands_; }
+
+ const LAllocation* x() { return getOperand(0); }
+
+ const LAllocation* y() { return getOperand(1); }
+
+ const LDefinition* output() { return getDef(0); }
+};
+
+class LMathFunctionD : public LCallInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(MathFunctionD)
+ explicit LMathFunctionD(const LAllocation& input)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+
+ MMathFunction* mir() const { return mir_->toMathFunction(); }
+ const char* extraName() const {
+ return MMathFunction::FunctionName(mir()->function());
+ }
+};
+
+class LMathFunctionF : public LCallInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(MathFunctionF)
+ explicit LMathFunctionF(const LAllocation& input)
+ : LCallInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+
+ MMathFunction* mir() const { return mir_->toMathFunction(); }
+ const char* extraName() const {
+ return MMathFunction::FunctionName(mir()->function());
+ }
+};
+
+// Adds two integers, returning an integer value.
+class LAddI : public LBinaryMath<0> {
+ bool recoversInput_;
+
+ public:
+ LIR_HEADER(AddI)
+
+ LAddI() : LBinaryMath(classOpcode), recoversInput_(false) {}
+
+ const char* extraName() const {
+ return snapshot() ? "OverflowCheck" : nullptr;
+ }
+
+ bool recoversInput() const { return recoversInput_; }
+ void setRecoversInput() { recoversInput_ = true; }
+
+ MAdd* mir() const { return mir_->toAdd(); }
+};
+
+class LAddI64 : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(AddI64)
+
+ LAddI64() : LInstructionHelper(classOpcode) {}
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+};
+
+// Subtracts two integers, returning an integer value.
+class LSubI : public LBinaryMath<0> {
+ bool recoversInput_;
+
+ public:
+ LIR_HEADER(SubI)
+
+ LSubI() : LBinaryMath(classOpcode), recoversInput_(false) {}
+
+ const char* extraName() const {
+ return snapshot() ? "OverflowCheck" : nullptr;
+ }
+
+ bool recoversInput() const { return recoversInput_; }
+ void setRecoversInput() { recoversInput_ = true; }
+ MSub* mir() const { return mir_->toSub(); }
+};
+
+inline bool LNode::recoversInput() const {
+ switch (op()) {
+ case Opcode::AddI:
+ return toAddI()->recoversInput();
+ case Opcode::SubI:
+ return toSubI()->recoversInput();
+ default:
+ return false;
+ }
+}
+
+class LSubI64 : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(SubI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+
+ LSubI64() : LInstructionHelper(classOpcode) {}
+};
+
+class LMulI64 : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 1> {
+ public:
+ LIR_HEADER(MulI64)
+
+ explicit LMulI64() : LInstructionHelper(classOpcode) {
+ setTemp(0, LDefinition());
+ }
+
+ const LDefinition* temp() { return getTemp(0); }
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+};
+
+// Performs an add, sub, mul, or div on two double values.
+class LMathD : public LBinaryMath<0> {
+ JSOp jsop_;
+
+ public:
+ LIR_HEADER(MathD)
+
+ explicit LMathD(JSOp jsop) : LBinaryMath(classOpcode), jsop_(jsop) {}
+
+ JSOp jsop() const { return jsop_; }
+
+ const char* extraName() const { return CodeName(jsop_); }
+};
+
+// Performs an add, sub, mul, or div on two double values.
+class LMathF : public LBinaryMath<0> {
+ JSOp jsop_;
+
+ public:
+ LIR_HEADER(MathF)
+
+ explicit LMathF(JSOp jsop) : LBinaryMath(classOpcode), jsop_(jsop) {}
+
+ JSOp jsop() const { return jsop_; }
+
+ const char* extraName() const { return CodeName(jsop_); }
+};
+
+class LModD : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(ModD)
+
+ LModD(const LAllocation& lhs, const LAllocation& rhs)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setIsCall();
+ }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LModPowTwoD : public LInstructionHelper<1, 1, 0> {
+ const uint32_t divisor_;
+
+ public:
+ LIR_HEADER(ModPowTwoD)
+
+ LModPowTwoD(const LAllocation& lhs, uint32_t divisor)
+ : LInstructionHelper(classOpcode), divisor_(divisor) {
+ setOperand(0, lhs);
+ }
+
+ uint32_t divisor() const { return divisor_; }
+ const LAllocation* lhs() { return getOperand(0); }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+class LBigIntAdd : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntAdd)
+
+ LBigIntAdd(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+class LBigIntSub : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntSub)
+
+ LBigIntSub(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+class LBigIntMul : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntMul)
+
+ LBigIntMul(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+class LBigIntDiv : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntDiv)
+
+ LBigIntDiv(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+
+ const MBigIntDiv* mir() const { return mirRaw()->toBigIntDiv(); }
+};
+
+class LBigIntMod : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntMod)
+
+ LBigIntMod(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+
+ const MBigIntMod* mir() const { return mirRaw()->toBigIntMod(); }
+};
+
+class LBigIntPow : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntPow)
+
+ LBigIntPow(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+
+ const MBigIntPow* mir() const { return mirRaw()->toBigIntPow(); }
+};
+
+class LBigIntBitAnd : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntBitAnd)
+
+ LBigIntBitAnd(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+class LBigIntBitOr : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntBitOr)
+
+ LBigIntBitOr(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+class LBigIntBitXor : public LBinaryMath<2> {
+ public:
+ LIR_HEADER(BigIntBitXor)
+
+ LBigIntBitXor(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+class LBigIntLsh : public LBinaryMath<3> {
+ public:
+ LIR_HEADER(BigIntLsh)
+
+ LBigIntLsh(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2,
+ const LDefinition& temp3)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ setTemp(2, temp3);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+ const LDefinition* temp3() { return getTemp(2); }
+};
+
+class LBigIntRsh : public LBinaryMath<3> {
+ public:
+ LIR_HEADER(BigIntRsh)
+
+ LBigIntRsh(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp1, const LDefinition& temp2,
+ const LDefinition& temp3)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ setTemp(2, temp3);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+ const LDefinition* temp3() { return getTemp(2); }
+};
+
+class LBigIntIncrement : public LUnaryMath<2> {
+ public:
+ LIR_HEADER(BigIntIncrement)
+
+ LBigIntIncrement(const LAllocation& input, const LDefinition& temp1,
+ const LDefinition& temp2)
+ : LUnaryMath(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+class LBigIntDecrement : public LUnaryMath<2> {
+ public:
+ LIR_HEADER(BigIntDecrement)
+
+ LBigIntDecrement(const LAllocation& input, const LDefinition& temp1,
+ const LDefinition& temp2)
+ : LUnaryMath(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+class LBigIntNegate : public LUnaryMath<1> {
+ public:
+ LIR_HEADER(BigIntNegate)
+
+ LBigIntNegate(const LAllocation& input, const LDefinition& temp)
+ : LUnaryMath(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+class LBigIntBitNot : public LUnaryMath<2> {
+ public:
+ LIR_HEADER(BigIntBitNot)
+
+ LBigIntBitNot(const LAllocation& input, const LDefinition& temp1,
+ const LDefinition& temp2)
+ : LUnaryMath(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+};
+
+// Convert a value to an int32.
+// Input: components of a Value
+// Output: 32-bit integer
+// Bailout: undefined, string, object, or non-int32 double
+// Temps: one float register, one GP register
+//
+// This instruction requires a temporary float register.
+class LValueToInt32 : public LInstructionHelper<1, BOX_PIECES, 2> {
+ public:
+ enum Mode { NORMAL, TRUNCATE, TRUNCATE_NOWRAP };
+
+ private:
+ Mode mode_;
+
+ public:
+ LIR_HEADER(ValueToInt32)
+
+ LValueToInt32(const LBoxAllocation& input, const LDefinition& temp0,
+ const LDefinition& temp1, Mode mode)
+ : LInstructionHelper(classOpcode), mode_(mode) {
+ setBoxOperand(Input, input);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ }
+
+ const char* extraName() const {
+ return mode() == NORMAL ? "Normal"
+ : mode() == TRUNCATE ? "Truncate"
+ : "TruncateNoWrap";
+ }
+
+ static const size_t Input = 0;
+
+ Mode mode() const { return mode_; }
+ const LDefinition* tempFloat() { return getTemp(0); }
+ const LDefinition* temp() { return getTemp(1); }
+ MToNumberInt32* mirNormal() const {
+ MOZ_ASSERT(mode_ == NORMAL);
+ return mir_->toToNumberInt32();
+ }
+ MTruncateToInt32* mirTruncate() const {
+ MOZ_ASSERT(mode_ == TRUNCATE);
+ return mir_->toTruncateToInt32();
+ }
+ MToIntegerInt32* mirTruncateNoWrap() const {
+ MOZ_ASSERT(mode_ == TRUNCATE_NOWRAP);
+ return mir_->toToIntegerInt32();
+ }
+ MInstruction* mir() const { return mir_->toInstruction(); }
+};
+
+// Double raised to a half power.
+class LPowHalfD : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(PowHalfD);
+ explicit LPowHalfD(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+
+ const LAllocation* input() { return getOperand(0); }
+ const LDefinition* output() { return getDef(0); }
+ MPowHalf* mir() const { return mir_->toPowHalf(); }
+};
+
+// Passed the BaselineFrame address in the OsrFrameReg via the IonOsrTempData
+// populated by PrepareOsrTempData.
+//
+// Forwards this object to the LOsrValues for Value materialization.
+class LOsrEntry : public LInstructionHelper<1, 0, 1> {
+ protected:
+ Label label_;
+ uint32_t frameDepth_;
+
+ public:
+ LIR_HEADER(OsrEntry)
+
+ explicit LOsrEntry(const LDefinition& temp)
+ : LInstructionHelper(classOpcode), frameDepth_(0) {
+ setTemp(0, temp);
+ }
+
+ void setFrameDepth(uint32_t depth) { frameDepth_ = depth; }
+ uint32_t getFrameDepth() { return frameDepth_; }
+ Label* label() { return &label_; }
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+// Bailout if index + minimum < 0 or index + maximum >= length.
+class LBoundsCheckRange : public LInstructionHelper<0, 2, 1> {
+ public:
+ LIR_HEADER(BoundsCheckRange)
+
+ LBoundsCheckRange(const LAllocation& index, const LAllocation& length,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, index);
+ setOperand(1, length);
+ setTemp(0, temp);
+ }
+ const MBoundsCheck* mir() const { return mir_->toBoundsCheck(); }
+ const LAllocation* index() { return getOperand(0); }
+ const LAllocation* length() { return getOperand(1); }
+};
+
+// Load a value from a dense array's elements vector. Bail out if it's the hole
+// value.
+class LLoadElementV : public LInstructionHelper<BOX_PIECES, 2, 0> {
+ public:
+ LIR_HEADER(LoadElementV)
+
+ LLoadElementV(const LAllocation& elements, const LAllocation& index)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ }
+
+ const MLoadElement* mir() const { return mir_->toLoadElement(); }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+};
+
+// Load a value from an array's elements vector, loading |undefined| if we hit a
+// hole. Bail out if we get a negative index.
+class LLoadElementHole : public LInstructionHelper<BOX_PIECES, 3, 0> {
+ public:
+ LIR_HEADER(LoadElementHole)
+
+ LLoadElementHole(const LAllocation& elements, const LAllocation& index,
+ const LAllocation& initLength)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, initLength);
+ }
+
+ const MLoadElementHole* mir() const { return mir_->toLoadElementHole(); }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* initLength() { return getOperand(2); }
+};
+
+// Store a boxed value to a dense array's element vector.
+class LStoreElementV : public LInstructionHelper<0, 2 + BOX_PIECES, 0> {
+ public:
+ LIR_HEADER(StoreElementV)
+
+ LStoreElementV(const LAllocation& elements, const LAllocation& index,
+ const LBoxAllocation& value)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setBoxOperand(Value, value);
+ }
+
+ const char* extraName() const {
+ return mir()->needsHoleCheck() ? "HoleCheck" : nullptr;
+ }
+
+ static const size_t Value = 2;
+
+ const MStoreElement* mir() const { return mir_->toStoreElement(); }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+};
+
+// Store a typed value to a dense array's elements vector. Compared to
+// LStoreElementV, this instruction can store doubles and constants directly,
+// and does not store the type tag if the array is monomorphic and known to
+// be packed.
+class LStoreElementT : public LInstructionHelper<0, 3, 0> {
+ public:
+ LIR_HEADER(StoreElementT)
+
+ LStoreElementT(const LAllocation& elements, const LAllocation& index,
+ const LAllocation& value)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ }
+
+ const char* extraName() const {
+ return mir()->needsHoleCheck() ? "HoleCheck" : nullptr;
+ }
+
+ const MStoreElement* mir() const { return mir_->toStoreElement(); }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+};
+
+class LArrayPopShift : public LInstructionHelper<BOX_PIECES, 1, 2> {
+ public:
+ LIR_HEADER(ArrayPopShift)
+
+ LArrayPopShift(const LAllocation& object, const LDefinition& temp0,
+ const LDefinition& temp1)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, object);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ }
+
+ const char* extraName() const {
+ return mir()->mode() == MArrayPopShift::Pop ? "Pop" : "Shift";
+ }
+
+ const MArrayPopShift* mir() const { return mir_->toArrayPopShift(); }
+ const LAllocation* object() { return getOperand(0); }
+ const LDefinition* temp0() { return getTemp(0); }
+ const LDefinition* temp1() { return getTemp(1); }
+};
+
+class LLoadUnboxedBigInt : public LInstructionHelper<1, 2, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(LoadUnboxedBigInt)
+
+ LLoadUnboxedBigInt(const LAllocation& elements, const LAllocation& index,
+ const LDefinition& temp, const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+ const MLoadUnboxedScalar* mir() const { return mir_->toLoadUnboxedScalar(); }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LDefinition* temp() { return getTemp(0); }
+ const LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+class LLoadDataViewElement : public LInstructionHelper<1, 3, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(LoadDataViewElement)
+
+ LLoadDataViewElement(const LAllocation& elements, const LAllocation& index,
+ const LAllocation& littleEndian, const LDefinition& temp,
+ const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, littleEndian);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+ const MLoadDataViewElement* mir() const {
+ return mir_->toLoadDataViewElement();
+ }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* littleEndian() { return getOperand(2); }
+ const LDefinition* temp() { return getTemp(0); }
+ const LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+class LLoadTypedArrayElementHoleBigInt
+ : public LInstructionHelper<BOX_PIECES, 2, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(LoadTypedArrayElementHoleBigInt)
+
+ LLoadTypedArrayElementHoleBigInt(const LAllocation& object,
+ const LAllocation& index,
+ const LDefinition& temp,
+ const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, object);
+ setOperand(1, index);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+ const MLoadTypedArrayElementHole* mir() const {
+ return mir_->toLoadTypedArrayElementHole();
+ }
+ const LAllocation* object() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LDefinition* temp() { return getTemp(0); }
+ const LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+class LStoreUnboxedBigInt : public LInstructionHelper<0, 3, INT64_PIECES> {
+ public:
+ LIR_HEADER(StoreUnboxedBigInt)
+
+ LStoreUnboxedBigInt(const LAllocation& elements, const LAllocation& index,
+ const LAllocation& value, const LInt64Definition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp);
+ }
+
+ const MStoreUnboxedScalar* mir() const {
+ return mir_->toStoreUnboxedScalar();
+ }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+ LInt64Definition temp() { return getInt64Temp(0); }
+};
+
+class LStoreDataViewElement
+ : public LInstructionHelper<0, 4, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(StoreDataViewElement)
+
+ LStoreDataViewElement(const LAllocation& elements, const LAllocation& index,
+ const LAllocation& value,
+ const LAllocation& littleEndian,
+ const LDefinition& temp, const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setOperand(3, littleEndian);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+
+ const MStoreDataViewElement* mir() const {
+ return mir_->toStoreDataViewElement();
+ }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+ const LAllocation* littleEndian() { return getOperand(3); }
+ const LDefinition* temp() { return getTemp(0); }
+ const LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+class LStoreTypedArrayElementHoleBigInt
+ : public LInstructionHelper<0, 4, INT64_PIECES> {
+ public:
+ LIR_HEADER(StoreTypedArrayElementHoleBigInt)
+
+ LStoreTypedArrayElementHoleBigInt(const LAllocation& elements,
+ const LAllocation& length,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, length);
+ setOperand(2, index);
+ setOperand(3, value);
+ setInt64Temp(0, temp);
+ }
+
+ const MStoreTypedArrayElementHole* mir() const {
+ return mir_->toStoreTypedArrayElementHole();
+ }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* length() { return getOperand(1); }
+ const LAllocation* index() { return getOperand(2); }
+ const LAllocation* value() { return getOperand(3); }
+ LInt64Definition temp() { return getInt64Temp(0); }
+};
+
+class LCompareExchangeTypedArrayElement : public LInstructionHelper<1, 4, 4> {
+ public:
+ LIR_HEADER(CompareExchangeTypedArrayElement)
+
+ // ARM, ARM64, x86, x64
+ LCompareExchangeTypedArrayElement(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& oldval,
+ const LAllocation& newval,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, oldval);
+ setOperand(3, newval);
+ setTemp(0, temp);
+ }
+ // MIPS32, MIPS64
+ LCompareExchangeTypedArrayElement(
+ const LAllocation& elements, const LAllocation& index,
+ const LAllocation& oldval, const LAllocation& newval,
+ const LDefinition& temp, const LDefinition& valueTemp,
+ const LDefinition& offsetTemp, const LDefinition& maskTemp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, oldval);
+ setOperand(3, newval);
+ setTemp(0, temp);
+ setTemp(1, valueTemp);
+ setTemp(2, offsetTemp);
+ setTemp(3, maskTemp);
+ }
+
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* oldval() { return getOperand(2); }
+ const LAllocation* newval() { return getOperand(3); }
+ const LDefinition* temp() { return getTemp(0); }
+
+ // Temp that may be used on LL/SC platforms for extract/insert bits of word.
+ const LDefinition* valueTemp() { return getTemp(1); }
+ const LDefinition* offsetTemp() { return getTemp(2); }
+ const LDefinition* maskTemp() { return getTemp(3); }
+
+ const MCompareExchangeTypedArrayElement* mir() const {
+ return mir_->toCompareExchangeTypedArrayElement();
+ }
+};
+
+class LAtomicExchangeTypedArrayElement : public LInstructionHelper<1, 3, 4> {
+ public:
+ LIR_HEADER(AtomicExchangeTypedArrayElement)
+
+ // ARM, ARM64, x86, x64
+ LAtomicExchangeTypedArrayElement(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setTemp(0, temp);
+ }
+ // MIPS32, MIPS64
+ LAtomicExchangeTypedArrayElement(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LDefinition& temp,
+ const LDefinition& valueTemp,
+ const LDefinition& offsetTemp,
+ const LDefinition& maskTemp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setTemp(0, temp);
+ setTemp(1, valueTemp);
+ setTemp(2, offsetTemp);
+ setTemp(3, maskTemp);
+ }
+
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+ const LDefinition* temp() { return getTemp(0); }
+
+ // Temp that may be used on LL/SC platforms for extract/insert bits of word.
+ const LDefinition* valueTemp() { return getTemp(1); }
+ const LDefinition* offsetTemp() { return getTemp(2); }
+ const LDefinition* maskTemp() { return getTemp(3); }
+
+ const MAtomicExchangeTypedArrayElement* mir() const {
+ return mir_->toAtomicExchangeTypedArrayElement();
+ }
+};
+
+class LAtomicTypedArrayElementBinop : public LInstructionHelper<1, 3, 5> {
+ public:
+ LIR_HEADER(AtomicTypedArrayElementBinop)
+
+ static const int32_t valueOp = 2;
+
+ // ARM, ARM64, x86, x64
+ LAtomicTypedArrayElementBinop(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LDefinition& temp1,
+ const LDefinition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setTemp(0, temp1);
+ setTemp(1, temp2);
+ }
+ // MIPS32, MIPS64
+ LAtomicTypedArrayElementBinop(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LDefinition& temp2,
+ const LDefinition& valueTemp,
+ const LDefinition& offsetTemp,
+ const LDefinition& maskTemp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setTemp(0, LDefinition::BogusTemp());
+ setTemp(1, temp2);
+ setTemp(2, valueTemp);
+ setTemp(3, offsetTemp);
+ setTemp(4, maskTemp);
+ }
+
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() {
+ MOZ_ASSERT(valueOp == 2);
+ return getOperand(2);
+ }
+ const LDefinition* temp1() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+
+ // Temp that may be used on LL/SC platforms for extract/insert bits of word.
+ const LDefinition* valueTemp() { return getTemp(2); }
+ const LDefinition* offsetTemp() { return getTemp(3); }
+ const LDefinition* maskTemp() { return getTemp(4); }
+
+ const MAtomicTypedArrayElementBinop* mir() const {
+ return mir_->toAtomicTypedArrayElementBinop();
+ }
+};
+
+// Atomic binary operation where the result is discarded.
+class LAtomicTypedArrayElementBinopForEffect
+ : public LInstructionHelper<0, 3, 4> {
+ public:
+ LIR_HEADER(AtomicTypedArrayElementBinopForEffect)
+
+ // ARM, ARM64, x86, x64
+ LAtomicTypedArrayElementBinopForEffect(
+ const LAllocation& elements, const LAllocation& index,
+ const LAllocation& value,
+ const LDefinition& flagTemp = LDefinition::BogusTemp())
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setTemp(0, flagTemp);
+ }
+ // MIPS32, MIPS64
+ LAtomicTypedArrayElementBinopForEffect(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LDefinition& valueTemp,
+ const LDefinition& offsetTemp,
+ const LDefinition& maskTemp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setTemp(0, LDefinition::BogusTemp());
+ setTemp(1, valueTemp);
+ setTemp(2, offsetTemp);
+ setTemp(3, maskTemp);
+ }
+
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+
+ // Temp that may be used on LL/SC platforms for the flag result of the store.
+ const LDefinition* flagTemp() { return getTemp(0); }
+ // Temp that may be used on LL/SC platforms for extract/insert bits of word.
+ const LDefinition* valueTemp() { return getTemp(1); }
+ const LDefinition* offsetTemp() { return getTemp(2); }
+ const LDefinition* maskTemp() { return getTemp(3); }
+
+ const MAtomicTypedArrayElementBinop* mir() const {
+ return mir_->toAtomicTypedArrayElementBinop();
+ }
+};
+
+class LAtomicLoad64 : public LInstructionHelper<1, 2, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(AtomicLoad64)
+
+ LAtomicLoad64(const LAllocation& elements, const LAllocation& index,
+ const LDefinition& temp, const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+ const MLoadUnboxedScalar* mir() const { return mir_->toLoadUnboxedScalar(); }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LDefinition* temp() { return getTemp(0); }
+ LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+class LAtomicStore64 : public LInstructionHelper<0, 3, 2 * INT64_PIECES + 1> {
+ public:
+ LIR_HEADER(AtomicStore64)
+
+ // x64, ARM64
+ LAtomicStore64(const LAllocation& elements, const LAllocation& index,
+ const LAllocation& value, const LInt64Definition& temp1)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp());
+ setTemp(2 * INT64_PIECES, LDefinition::BogusTemp());
+ }
+
+ // ARM32
+ LAtomicStore64(const LAllocation& elements, const LAllocation& index,
+ const LAllocation& value, const LInt64Definition& temp1,
+ const LInt64Definition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, temp2);
+ setTemp(2 * INT64_PIECES, LDefinition::BogusTemp());
+ }
+
+ // x86
+ LAtomicStore64(const LAllocation& elements, const LAllocation& index,
+ const LAllocation& value, const LInt64Definition& temp1,
+ const LDefinition& tempLow)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp());
+ setTemp(2 * INT64_PIECES, tempLow);
+ }
+
+ const MStoreUnboxedScalar* mir() const {
+ return mir_->toStoreUnboxedScalar();
+ }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+ LInt64Definition temp1() { return getInt64Temp(0); }
+ LInt64Definition temp2() { return getInt64Temp(INT64_PIECES); }
+ const LDefinition* tempLow() { return getTemp(2 * INT64_PIECES); }
+};
+
+class LCompareExchangeTypedArrayElement64
+ : public LInstructionHelper<1, 4, 3 * INT64_PIECES + 1> {
+ public:
+ LIR_HEADER(CompareExchangeTypedArrayElement64)
+
+ // x64, ARM64
+ LCompareExchangeTypedArrayElement64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& oldval,
+ const LAllocation& newval,
+ const LInt64Definition& temp1,
+ const LInt64Definition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, oldval);
+ setOperand(3, newval);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, temp2);
+ setInt64Temp(2 * INT64_PIECES, LInt64Definition::BogusTemp());
+ setTemp(3 * INT64_PIECES, LDefinition::BogusTemp());
+ }
+
+ // x86
+ LCompareExchangeTypedArrayElement64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& oldval,
+ const LAllocation& newval,
+ const LDefinition& tempLow)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, oldval);
+ setOperand(3, newval);
+ setInt64Temp(0, LInt64Definition::BogusTemp());
+ setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp());
+ setInt64Temp(2 * INT64_PIECES, LInt64Definition::BogusTemp());
+ setTemp(3 * INT64_PIECES, tempLow);
+ }
+
+ // ARM
+ LCompareExchangeTypedArrayElement64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& oldval,
+ const LAllocation& newval,
+ const LInt64Definition& temp1,
+ const LInt64Definition& temp2,
+ const LInt64Definition& temp3)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, oldval);
+ setOperand(3, newval);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, temp2);
+ setInt64Temp(2 * INT64_PIECES, temp3);
+ setTemp(3 * INT64_PIECES, LDefinition::BogusTemp());
+ }
+
+ const MCompareExchangeTypedArrayElement* mir() const {
+ return mir_->toCompareExchangeTypedArrayElement();
+ }
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* oldval() { return getOperand(2); }
+ const LAllocation* newval() { return getOperand(3); }
+ LInt64Definition temp1() { return getInt64Temp(0); }
+ LInt64Definition temp2() { return getInt64Temp(INT64_PIECES); }
+ LInt64Definition temp3() { return getInt64Temp(2 * INT64_PIECES); }
+ const LDefinition* tempLow() { return getTemp(3 * INT64_PIECES); }
+};
+
+class LAtomicExchangeTypedArrayElement64
+ : public LInstructionHelper<1, 3, INT64_PIECES + 1> {
+ public:
+ LIR_HEADER(AtomicExchangeTypedArrayElement64)
+
+ // ARM, ARM64, x64
+ LAtomicExchangeTypedArrayElement64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp1,
+ const LDefinition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp1);
+ setTemp(INT64_PIECES, temp2);
+ }
+
+ // x86
+ LAtomicExchangeTypedArrayElement64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp);
+ setTemp(INT64_PIECES, LDefinition::BogusTemp());
+ }
+
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+ LInt64Definition temp1() { return getInt64Temp(0); }
+ const LDefinition* temp2() { return getTemp(INT64_PIECES); }
+
+ const MAtomicExchangeTypedArrayElement* mir() const {
+ return mir_->toAtomicExchangeTypedArrayElement();
+ }
+};
+
+class LAtomicTypedArrayElementBinop64
+ : public LInstructionHelper<1, 3, 3 * INT64_PIECES> {
+ public:
+ LIR_HEADER(AtomicTypedArrayElementBinop64)
+
+ // x86
+ LAtomicTypedArrayElementBinop64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp1)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp());
+ setInt64Temp(2 * INT64_PIECES, LInt64Definition::BogusTemp());
+ }
+
+ // ARM64, x64
+ LAtomicTypedArrayElementBinop64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp1,
+ const LInt64Definition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, temp2);
+ setInt64Temp(2 * INT64_PIECES, LInt64Definition::BogusTemp());
+ }
+
+ // ARM
+ LAtomicTypedArrayElementBinop64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp1,
+ const LInt64Definition& temp2,
+ const LInt64Definition& temp3)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, temp2);
+ setInt64Temp(2 * INT64_PIECES, temp3);
+ }
+
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+ LInt64Definition temp1() { return getInt64Temp(0); }
+ LInt64Definition temp2() { return getInt64Temp(INT64_PIECES); }
+ LInt64Definition temp3() { return getInt64Temp(2 * INT64_PIECES); }
+
+ const MAtomicTypedArrayElementBinop* mir() const {
+ return mir_->toAtomicTypedArrayElementBinop();
+ }
+};
+
+// Atomic binary operation where the result is discarded.
+class LAtomicTypedArrayElementBinopForEffect64
+ : public LInstructionHelper<0, 3, 2 * INT64_PIECES + 1> {
+ public:
+ LIR_HEADER(AtomicTypedArrayElementBinopForEffect64)
+
+ // x86
+ LAtomicTypedArrayElementBinopForEffect64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp,
+ const LDefinition& tempLow)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp);
+ setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp());
+ setTemp(2 * INT64_PIECES, tempLow);
+ }
+
+ // x64
+ LAtomicTypedArrayElementBinopForEffect64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp);
+ setInt64Temp(INT64_PIECES, LInt64Definition::BogusTemp());
+ setTemp(2 * INT64_PIECES, LDefinition::BogusTemp());
+ }
+
+ // ARM64
+ LAtomicTypedArrayElementBinopForEffect64(const LAllocation& elements,
+ const LAllocation& index,
+ const LAllocation& value,
+ const LInt64Definition& temp1,
+ const LInt64Definition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, elements);
+ setOperand(1, index);
+ setOperand(2, value);
+ setInt64Temp(0, temp1);
+ setInt64Temp(INT64_PIECES, temp2);
+ setTemp(2 * INT64_PIECES, LDefinition::BogusTemp());
+ }
+
+ const LAllocation* elements() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ const LAllocation* value() { return getOperand(2); }
+ LInt64Definition temp1() { return getInt64Temp(0); }
+ LInt64Definition temp2() { return getInt64Temp(INT64_PIECES); }
+ const LDefinition* tempLow() { return getTemp(2 * INT64_PIECES); }
+
+ const MAtomicTypedArrayElementBinop* mir() const {
+ return mir_->toAtomicTypedArrayElementBinop();
+ }
+};
+
+class LIsNoIterAndBranch : public LControlInstructionHelper<2, BOX_PIECES, 0> {
+ public:
+ LIR_HEADER(IsNoIterAndBranch)
+
+ LIsNoIterAndBranch(MBasicBlock* ifTrue, MBasicBlock* ifFalse,
+ const LBoxAllocation& input)
+ : LControlInstructionHelper(classOpcode) {
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ setBoxOperand(Input, input);
+ }
+
+ static const size_t Input = 0;
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+};
+
+class LInstanceOfCache : public LInstructionHelper<1, BOX_PIECES + 1, 0> {
+ public:
+ LIR_HEADER(InstanceOfCache)
+ LInstanceOfCache(const LBoxAllocation& lhs, const LAllocation& rhs)
+ : LInstructionHelper(classOpcode) {
+ setBoxOperand(LHS, lhs);
+ setOperand(RHS, rhs);
+ }
+
+ const LDefinition* output() { return this->getDef(0); }
+ const LAllocation* rhs() { return getOperand(RHS); }
+
+ static const size_t LHS = 0;
+ static const size_t RHS = BOX_PIECES;
+};
+
+class LIsObjectAndBranch : public LControlInstructionHelper<2, BOX_PIECES, 0> {
+ public:
+ LIR_HEADER(IsObjectAndBranch)
+
+ LIsObjectAndBranch(MBasicBlock* ifTrue, MBasicBlock* ifFalse,
+ const LBoxAllocation& input)
+ : LControlInstructionHelper(classOpcode) {
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ setBoxOperand(Input, input);
+ }
+
+ static const size_t Input = 0;
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+};
+
+class LIsNullOrUndefinedAndBranch
+ : public LControlInstructionHelper<2, BOX_PIECES, 0> {
+ MIsNullOrUndefined* isNullOrUndefined_;
+
+ public:
+ LIR_HEADER(IsNullOrUndefinedAndBranch)
+ static const size_t Input = 0;
+
+ LIsNullOrUndefinedAndBranch(MIsNullOrUndefined* isNullOrUndefined,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse,
+ const LBoxAllocation& input)
+ : LControlInstructionHelper(classOpcode),
+ isNullOrUndefined_(isNullOrUndefined) {
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ setBoxOperand(Input, input);
+ }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+
+ MIsNullOrUndefined* isNullOrUndefinedMir() const {
+ return isNullOrUndefined_;
+ }
+};
+
+template <size_t Defs, size_t Ops>
+class LWasmSelectBase : public LInstructionHelper<Defs, Ops, 0> {
+ typedef LInstructionHelper<Defs, Ops, 0> Base;
+
+ protected:
+ explicit LWasmSelectBase(LNode::Opcode opcode) : Base(opcode) {}
+
+ public:
+ MWasmSelect* mir() const { return Base::mir_->toWasmSelect(); }
+};
+
+class LWasmSelect : public LWasmSelectBase<1, 3> {
+ public:
+ LIR_HEADER(WasmSelect);
+
+ static const size_t TrueExprIndex = 0;
+ static const size_t FalseExprIndex = 1;
+ static const size_t CondIndex = 2;
+
+ LWasmSelect(const LAllocation& trueExpr, const LAllocation& falseExpr,
+ const LAllocation& cond)
+ : LWasmSelectBase(classOpcode) {
+ setOperand(TrueExprIndex, trueExpr);
+ setOperand(FalseExprIndex, falseExpr);
+ setOperand(CondIndex, cond);
+ }
+
+ const LAllocation* trueExpr() { return getOperand(TrueExprIndex); }
+ const LAllocation* falseExpr() { return getOperand(FalseExprIndex); }
+ const LAllocation* condExpr() { return getOperand(CondIndex); }
+};
+
+class LWasmSelectI64
+ : public LWasmSelectBase<INT64_PIECES, 2 * INT64_PIECES + 1> {
+ public:
+ LIR_HEADER(WasmSelectI64);
+
+ static const size_t TrueExprIndex = 0;
+ static const size_t FalseExprIndex = INT64_PIECES;
+ static const size_t CondIndex = INT64_PIECES * 2;
+
+ LWasmSelectI64(const LInt64Allocation& trueExpr,
+ const LInt64Allocation& falseExpr, const LAllocation& cond)
+ : LWasmSelectBase(classOpcode) {
+ setInt64Operand(TrueExprIndex, trueExpr);
+ setInt64Operand(FalseExprIndex, falseExpr);
+ setOperand(CondIndex, cond);
+ }
+
+ const LInt64Allocation trueExpr() { return getInt64Operand(TrueExprIndex); }
+ const LInt64Allocation falseExpr() { return getInt64Operand(FalseExprIndex); }
+ const LAllocation* condExpr() { return getOperand(CondIndex); }
+};
+
+class LWasmCompareAndSelect : public LWasmSelectBase<1, 4> {
+ MCompare::CompareType compareType_;
+ JSOp jsop_;
+
+ public:
+ LIR_HEADER(WasmCompareAndSelect);
+
+ static const size_t LeftExprIndex = 0;
+ static const size_t RightExprIndex = 1;
+ static const size_t IfTrueExprIndex = 2;
+ static const size_t IfFalseExprIndex = 3;
+
+ LWasmCompareAndSelect(const LAllocation& leftExpr,
+ const LAllocation& rightExpr,
+ MCompare::CompareType compareType, JSOp jsop,
+ const LAllocation& ifTrueExpr,
+ const LAllocation& ifFalseExpr)
+ : LWasmSelectBase(classOpcode), compareType_(compareType), jsop_(jsop) {
+ setOperand(LeftExprIndex, leftExpr);
+ setOperand(RightExprIndex, rightExpr);
+ setOperand(IfTrueExprIndex, ifTrueExpr);
+ setOperand(IfFalseExprIndex, ifFalseExpr);
+ }
+
+ const LAllocation* leftExpr() { return getOperand(LeftExprIndex); }
+ const LAllocation* rightExpr() { return getOperand(RightExprIndex); }
+ const LAllocation* ifTrueExpr() { return getOperand(IfTrueExprIndex); }
+ const LAllocation* ifFalseExpr() { return getOperand(IfFalseExprIndex); }
+
+ MCompare::CompareType compareType() { return compareType_; }
+ JSOp jsop() { return jsop_; }
+};
+
+class LWasmBoundsCheck64
+ : public LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(WasmBoundsCheck64);
+ explicit LWasmBoundsCheck64(const LInt64Allocation& ptr,
+ const LInt64Allocation& boundsCheckLimit)
+ : LInstructionHelper(classOpcode) {
+ setInt64Operand(0, ptr);
+ setInt64Operand(INT64_PIECES, boundsCheckLimit);
+ }
+ MWasmBoundsCheck* mir() const { return mir_->toWasmBoundsCheck(); }
+ LInt64Allocation ptr() { return getInt64Operand(0); }
+ LInt64Allocation boundsCheckLimit() { return getInt64Operand(INT64_PIECES); }
+};
+
+namespace details {
+
+// This is a base class for LWasmLoad/LWasmLoadI64.
+template <size_t Defs, size_t Temp>
+class LWasmLoadBase : public LInstructionHelper<Defs, 2, Temp> {
+ public:
+ typedef LInstructionHelper<Defs, 2, Temp> Base;
+ explicit LWasmLoadBase(LNode::Opcode opcode, const LAllocation& ptr,
+ const LAllocation& memoryBase)
+ : Base(opcode) {
+ Base::setOperand(0, ptr);
+ Base::setOperand(1, memoryBase);
+ }
+ MWasmLoad* mir() const { return Base::mir_->toWasmLoad(); }
+ const LAllocation* ptr() { return Base::getOperand(0); }
+ const LAllocation* memoryBase() { return Base::getOperand(1); }
+};
+
+} // namespace details
+
+class LWasmLoad : public details::LWasmLoadBase<1, 1> {
+ public:
+ explicit LWasmLoad(const LAllocation& ptr,
+ const LAllocation& memoryBase = LAllocation())
+ : LWasmLoadBase(classOpcode, ptr, memoryBase) {
+ setTemp(0, LDefinition::BogusTemp());
+ }
+
+ const LDefinition* ptrCopy() { return Base::getTemp(0); }
+
+ LIR_HEADER(WasmLoad);
+};
+
+class LWasmLoadI64 : public details::LWasmLoadBase<INT64_PIECES, 1> {
+ public:
+ explicit LWasmLoadI64(const LAllocation& ptr,
+ const LAllocation& memoryBase = LAllocation())
+ : LWasmLoadBase(classOpcode, ptr, memoryBase) {
+ setTemp(0, LDefinition::BogusTemp());
+ }
+
+ const LDefinition* ptrCopy() { return Base::getTemp(0); }
+
+ LIR_HEADER(WasmLoadI64);
+};
+
+class LWasmStore : public LInstructionHelper<0, 3, 1> {
+ public:
+ LIR_HEADER(WasmStore);
+
+ static const size_t PtrIndex = 0;
+ static const size_t ValueIndex = 1;
+ static const size_t MemoryBaseIndex = 2;
+
+ LWasmStore(const LAllocation& ptr, const LAllocation& value,
+ const LAllocation& memoryBase = LAllocation())
+ : LInstructionHelper(classOpcode) {
+ setOperand(PtrIndex, ptr);
+ setOperand(ValueIndex, value);
+ setOperand(MemoryBaseIndex, memoryBase);
+ setTemp(0, LDefinition::BogusTemp());
+ }
+ MWasmStore* mir() const { return mir_->toWasmStore(); }
+ const LAllocation* ptr() { return getOperand(PtrIndex); }
+ const LDefinition* ptrCopy() { return getTemp(0); }
+ const LAllocation* value() { return getOperand(ValueIndex); }
+ const LAllocation* memoryBase() { return getOperand(MemoryBaseIndex); }
+};
+
+class LWasmStoreI64 : public LInstructionHelper<0, INT64_PIECES + 2, 1> {
+ public:
+ LIR_HEADER(WasmStoreI64);
+
+ static const size_t PtrIndex = 0;
+ static const size_t MemoryBaseIndex = 1;
+ static const size_t ValueIndex = 2;
+
+ LWasmStoreI64(const LAllocation& ptr, const LInt64Allocation& value,
+ const LAllocation& memoryBase = LAllocation())
+ : LInstructionHelper(classOpcode) {
+ setOperand(PtrIndex, ptr);
+ setOperand(MemoryBaseIndex, memoryBase);
+ setInt64Operand(ValueIndex, value);
+ setTemp(0, LDefinition::BogusTemp());
+ }
+ MWasmStore* mir() const { return mir_->toWasmStore(); }
+ const LAllocation* ptr() { return getOperand(PtrIndex); }
+ const LAllocation* memoryBase() { return getOperand(MemoryBaseIndex); }
+ const LDefinition* ptrCopy() { return getTemp(0); }
+ const LInt64Allocation value() { return getInt64Operand(ValueIndex); }
+};
+
+class LWasmCompareExchangeHeap : public LInstructionHelper<1, 4, 4> {
+ public:
+ LIR_HEADER(WasmCompareExchangeHeap);
+
+ // ARM, ARM64, x86, x64
+ LWasmCompareExchangeHeap(const LAllocation& ptr, const LAllocation& oldValue,
+ const LAllocation& newValue,
+ const LAllocation& memoryBase = LAllocation())
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, oldValue);
+ setOperand(2, newValue);
+ setOperand(3, memoryBase);
+ setTemp(0, LDefinition::BogusTemp());
+ }
+ // MIPS32, MIPS64
+ LWasmCompareExchangeHeap(const LAllocation& ptr, const LAllocation& oldValue,
+ const LAllocation& newValue,
+ const LDefinition& valueTemp,
+ const LDefinition& offsetTemp,
+ const LDefinition& maskTemp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, oldValue);
+ setOperand(2, newValue);
+ setOperand(3, LAllocation());
+ setTemp(0, LDefinition::BogusTemp());
+ setTemp(1, valueTemp);
+ setTemp(2, offsetTemp);
+ setTemp(3, maskTemp);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LAllocation* oldValue() { return getOperand(1); }
+ const LAllocation* newValue() { return getOperand(2); }
+ const LAllocation* memoryBase() { return getOperand(3); }
+ const LDefinition* addrTemp() { return getTemp(0); }
+
+ void setAddrTemp(const LDefinition& addrTemp) { setTemp(0, addrTemp); }
+
+ // Temp that may be used on LL/SC platforms for extract/insert bits of word.
+ const LDefinition* valueTemp() { return getTemp(1); }
+ const LDefinition* offsetTemp() { return getTemp(2); }
+ const LDefinition* maskTemp() { return getTemp(3); }
+
+ MWasmCompareExchangeHeap* mir() const {
+ return mir_->toWasmCompareExchangeHeap();
+ }
+};
+
+class LWasmAtomicExchangeHeap : public LInstructionHelper<1, 3, 4> {
+ public:
+ LIR_HEADER(WasmAtomicExchangeHeap);
+
+ // ARM, ARM64, x86, x64
+ LWasmAtomicExchangeHeap(const LAllocation& ptr, const LAllocation& value,
+ const LAllocation& memoryBase = LAllocation())
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, value);
+ setOperand(2, memoryBase);
+ setTemp(0, LDefinition::BogusTemp());
+ }
+ // MIPS32, MIPS64
+ LWasmAtomicExchangeHeap(const LAllocation& ptr, const LAllocation& value,
+ const LDefinition& valueTemp,
+ const LDefinition& offsetTemp,
+ const LDefinition& maskTemp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, value);
+ setOperand(2, LAllocation());
+ setTemp(0, LDefinition::BogusTemp());
+ setTemp(1, valueTemp);
+ setTemp(2, offsetTemp);
+ setTemp(3, maskTemp);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LAllocation* value() { return getOperand(1); }
+ const LAllocation* memoryBase() { return getOperand(2); }
+ const LDefinition* addrTemp() { return getTemp(0); }
+
+ void setAddrTemp(const LDefinition& addrTemp) { setTemp(0, addrTemp); }
+
+ // Temp that may be used on LL/SC platforms for extract/insert bits of word.
+ const LDefinition* valueTemp() { return getTemp(1); }
+ const LDefinition* offsetTemp() { return getTemp(2); }
+ const LDefinition* maskTemp() { return getTemp(3); }
+
+ MWasmAtomicExchangeHeap* mir() const {
+ return mir_->toWasmAtomicExchangeHeap();
+ }
+};
+
+class LWasmAtomicBinopHeap : public LInstructionHelper<1, 3, 6> {
+ public:
+ LIR_HEADER(WasmAtomicBinopHeap);
+
+ static const int32_t valueOp = 1;
+
+ // ARM, ARM64, x86, x64
+ LWasmAtomicBinopHeap(const LAllocation& ptr, const LAllocation& value,
+ const LDefinition& temp,
+ const LDefinition& flagTemp = LDefinition::BogusTemp(),
+ const LAllocation& memoryBase = LAllocation())
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, value);
+ setOperand(2, memoryBase);
+ setTemp(0, temp);
+ setTemp(1, LDefinition::BogusTemp());
+ setTemp(2, flagTemp);
+ }
+ // MIPS32, MIPS64
+ LWasmAtomicBinopHeap(const LAllocation& ptr, const LAllocation& value,
+ const LDefinition& valueTemp,
+ const LDefinition& offsetTemp,
+ const LDefinition& maskTemp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, value);
+ setOperand(2, LAllocation());
+ setTemp(0, LDefinition::BogusTemp());
+ setTemp(1, LDefinition::BogusTemp());
+ setTemp(2, LDefinition::BogusTemp());
+ setTemp(3, valueTemp);
+ setTemp(4, offsetTemp);
+ setTemp(5, maskTemp);
+ }
+ const LAllocation* ptr() { return getOperand(0); }
+ const LAllocation* value() {
+ MOZ_ASSERT(valueOp == 1);
+ return getOperand(1);
+ }
+ const LAllocation* memoryBase() { return getOperand(2); }
+ const LDefinition* temp() { return getTemp(0); }
+
+ // Temp that may be used on some platforms to hold a computed address.
+ const LDefinition* addrTemp() { return getTemp(1); }
+ void setAddrTemp(const LDefinition& addrTemp) { setTemp(1, addrTemp); }
+
+ // Temp that may be used on LL/SC platforms for the flag result of the store.
+ const LDefinition* flagTemp() { return getTemp(2); }
+ // Temp that may be used on LL/SC platforms for extract/insert bits of word.
+ const LDefinition* valueTemp() { return getTemp(3); }
+ const LDefinition* offsetTemp() { return getTemp(4); }
+ const LDefinition* maskTemp() { return getTemp(5); }
+
+ MWasmAtomicBinopHeap* mir() const { return mir_->toWasmAtomicBinopHeap(); }
+};
+
+// Atomic binary operation where the result is discarded.
+class LWasmAtomicBinopHeapForEffect : public LInstructionHelper<0, 3, 5> {
+ public:
+ LIR_HEADER(WasmAtomicBinopHeapForEffect);
+ // ARM, ARM64, x86, x64
+ LWasmAtomicBinopHeapForEffect(
+ const LAllocation& ptr, const LAllocation& value,
+ const LDefinition& flagTemp = LDefinition::BogusTemp(),
+ const LAllocation& memoryBase = LAllocation())
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, value);
+ setOperand(2, memoryBase);
+ setTemp(0, LDefinition::BogusTemp());
+ setTemp(1, flagTemp);
+ }
+ // MIPS32, MIPS64
+ LWasmAtomicBinopHeapForEffect(const LAllocation& ptr,
+ const LAllocation& value,
+ const LDefinition& valueTemp,
+ const LDefinition& offsetTemp,
+ const LDefinition& maskTemp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, value);
+ setOperand(2, LAllocation());
+ setTemp(0, LDefinition::BogusTemp());
+ setTemp(1, LDefinition::BogusTemp());
+ setTemp(2, valueTemp);
+ setTemp(3, offsetTemp);
+ setTemp(4, maskTemp);
+ }
+ const LAllocation* ptr() { return getOperand(0); }
+ const LAllocation* value() { return getOperand(1); }
+ const LAllocation* memoryBase() { return getOperand(2); }
+
+ // Temp that may be used on some platforms to hold a computed address.
+ const LDefinition* addrTemp() { return getTemp(0); }
+ void setAddrTemp(const LDefinition& addrTemp) { setTemp(0, addrTemp); }
+
+ // Temp that may be used on LL/SC platforms for the flag result of the store.
+ const LDefinition* flagTemp() { return getTemp(1); }
+ // Temp that may be used on LL/SC platforms for extract/insert bits of word.
+ const LDefinition* valueTemp() { return getTemp(2); }
+ const LDefinition* offsetTemp() { return getTemp(3); }
+ const LDefinition* maskTemp() { return getTemp(4); }
+
+ MWasmAtomicBinopHeap* mir() const { return mir_->toWasmAtomicBinopHeap(); }
+};
+
+class LWasmDerivedPointer : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmDerivedPointer);
+ explicit LWasmDerivedPointer(const LAllocation& base)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, base);
+ }
+ const LAllocation* base() { return getOperand(0); }
+ size_t offset() { return mirRaw()->toWasmDerivedPointer()->offset(); }
+};
+
+class LWasmDerivedIndexPointer : public LInstructionHelper<1, 2, 0> {
+ public:
+ LIR_HEADER(WasmDerivedIndexPointer);
+ explicit LWasmDerivedIndexPointer(const LAllocation& base,
+ const LAllocation& index)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, base);
+ setOperand(1, index);
+ }
+ const LAllocation* base() { return getOperand(0); }
+ const LAllocation* index() { return getOperand(1); }
+ Scale scale() { return mirRaw()->toWasmDerivedIndexPointer()->scale(); }
+};
+
+class LWasmParameterI64 : public LInstructionHelper<INT64_PIECES, 0, 0> {
+ public:
+ LIR_HEADER(WasmParameterI64);
+
+ LWasmParameterI64() : LInstructionHelper(classOpcode) {}
+};
+
+// This is used only with LWasmCall.
+class LWasmCallIndirectAdjunctSafepoint : public LInstructionHelper<0, 0, 0> {
+ CodeOffset offs_;
+ uint32_t framePushedAtStackMapBase_;
+
+ public:
+ LIR_HEADER(WasmCallIndirectAdjunctSafepoint);
+
+ LWasmCallIndirectAdjunctSafepoint()
+ : LInstructionHelper(classOpcode),
+ offs_(0),
+ framePushedAtStackMapBase_(0) {}
+
+ CodeOffset safepointLocation() const {
+ MOZ_ASSERT(offs_.offset() != 0);
+ return offs_;
+ }
+ uint32_t framePushedAtStackMapBase() const {
+ MOZ_ASSERT(offs_.offset() != 0);
+ return framePushedAtStackMapBase_;
+ }
+ void recordSafepointInfo(CodeOffset offs, uint32_t framePushed) {
+ offs_ = offs;
+ framePushedAtStackMapBase_ = framePushed;
+ }
+};
+
+// LWasmCall may be generated into two function calls in the case of
+// call_indirect, one for the fast path and one for the slow path. In that
+// case, the node carries a pointer to a companion node, the "adjunct
+// safepoint", representing the safepoint for the second of the two calls. The
+// dual-call construction is only meaningful for wasm because wasm has no
+// invalidation of code; this is not a pattern to be used generally.
+class LWasmCall : public LVariadicInstruction<0, 0> {
+ bool needsBoundsCheck_;
+ mozilla::Maybe<uint32_t> tableSize_;
+ LWasmCallIndirectAdjunctSafepoint* adjunctSafepoint_;
+
+ public:
+ LIR_HEADER(WasmCall);
+
+ LWasmCall(uint32_t numOperands, bool needsBoundsCheck,
+ mozilla::Maybe<uint32_t> tableSize = mozilla::Nothing())
+ : LVariadicInstruction(classOpcode, numOperands),
+ needsBoundsCheck_(needsBoundsCheck),
+ tableSize_(tableSize),
+ adjunctSafepoint_(nullptr) {
+ this->setIsCall();
+ }
+
+ MWasmCallBase* callBase() const {
+ if (isCatchable()) {
+ return static_cast<MWasmCallBase*>(mirCatchable());
+ }
+ return static_cast<MWasmCallBase*>(mirUncatchable());
+ }
+ bool isCatchable() const { return mir_->isWasmCallCatchable(); }
+ MWasmCallCatchable* mirCatchable() const {
+ return mir_->toWasmCallCatchable();
+ }
+ MWasmCallUncatchable* mirUncatchable() const {
+ return mir_->toWasmCallUncatchable();
+ }
+
+ static bool isCallPreserved(AnyRegister reg) {
+ // All MWasmCalls preserve the TLS register:
+ // - internal/indirect calls do by the internal wasm ABI
+ // - import calls do by explicitly saving/restoring at the callsite
+ // - builtin calls do because the TLS reg is non-volatile
+ // See also CodeGeneratorShared::emitWasmCall.
+ //
+ // All other registers are not preserved. This is is relied upon by
+ // MWasmCallCatchable which needs all live registers to be spilled before
+ // a call.
+ return !reg.isFloat() && reg.gpr() == InstanceReg;
+ }
+
+ bool needsBoundsCheck() const { return needsBoundsCheck_; }
+ mozilla::Maybe<uint32_t> tableSize() const { return tableSize_; }
+ LWasmCallIndirectAdjunctSafepoint* adjunctSafepoint() const {
+ MOZ_ASSERT(adjunctSafepoint_ != nullptr);
+ return adjunctSafepoint_;
+ }
+ void setAdjunctSafepoint(LWasmCallIndirectAdjunctSafepoint* asp) {
+ adjunctSafepoint_ = asp;
+ }
+};
+
+class LWasmRegisterResult : public LInstructionHelper<1, 0, 0> {
+ public:
+ LIR_HEADER(WasmRegisterResult);
+
+ LWasmRegisterResult() : LInstructionHelper(classOpcode) {}
+
+ MWasmRegisterResult* mir() const {
+ if (!mir_->isWasmRegisterResult()) {
+ return nullptr;
+ }
+ return mir_->toWasmRegisterResult();
+ }
+};
+
+class LWasmRegisterPairResult : public LInstructionHelper<2, 0, 0> {
+ public:
+ LIR_HEADER(WasmRegisterPairResult);
+
+ LWasmRegisterPairResult() : LInstructionHelper(classOpcode) {}
+
+ MDefinition* mir() const { return mirRaw(); }
+};
+
+inline uint32_t LStackArea::base() const {
+ return ins()->toWasmStackResultArea()->mir()->base();
+}
+inline void LStackArea::setBase(uint32_t base) {
+ ins()->toWasmStackResultArea()->mir()->setBase(base);
+}
+inline uint32_t LStackArea::size() const {
+ return ins()->toWasmStackResultArea()->mir()->byteSize();
+}
+
+inline bool LStackArea::ResultIterator::done() const {
+ return idx_ == alloc_.ins()->toWasmStackResultArea()->mir()->resultCount();
+}
+inline void LStackArea::ResultIterator::next() {
+ MOZ_ASSERT(!done());
+ idx_++;
+}
+inline LAllocation LStackArea::ResultIterator::alloc() const {
+ MOZ_ASSERT(!done());
+ MWasmStackResultArea* area = alloc_.ins()->toWasmStackResultArea()->mir();
+ return LStackSlot(area->base() - area->result(idx_).offset());
+}
+inline bool LStackArea::ResultIterator::isGcPointer() const {
+ MOZ_ASSERT(!done());
+ MWasmStackResultArea* area = alloc_.ins()->toWasmStackResultArea()->mir();
+ MIRType type = area->result(idx_).type();
+#ifndef JS_PUNBOX64
+ // LDefinition::TypeFrom isn't defined for MIRType::Int64 values on
+ // this platform, so here we have a special case.
+ if (type == MIRType::Int64) {
+ return false;
+ }
+#endif
+ return LDefinition::TypeFrom(type) == LDefinition::OBJECT;
+}
+
+class LWasmStackResult : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmStackResult);
+
+ LWasmStackResult() : LInstructionHelper(classOpcode) {}
+
+ MWasmStackResult* mir() const { return mir_->toWasmStackResult(); }
+ LStackSlot result(uint32_t base) const {
+ return LStackSlot(base - mir()->result().offset());
+ }
+};
+
+class LWasmStackResult64 : public LInstructionHelper<INT64_PIECES, 1, 0> {
+ public:
+ LIR_HEADER(WasmStackResult64);
+
+ LWasmStackResult64() : LInstructionHelper(classOpcode) {}
+
+ MWasmStackResult* mir() const { return mir_->toWasmStackResult(); }
+ LStackSlot result(uint32_t base, LDefinition* def) {
+ uint32_t offset = base - mir()->result().offset();
+#if defined(JS_NUNBOX32)
+ if (def == getDef(INT64LOW_INDEX)) {
+ offset -= INT64LOW_OFFSET;
+ } else {
+ MOZ_ASSERT(def == getDef(INT64HIGH_INDEX));
+ offset -= INT64HIGH_OFFSET;
+ }
+#else
+ MOZ_ASSERT(def == getDef(0));
+#endif
+ return LStackSlot(offset);
+ }
+};
+
+inline LStackSlot LStackArea::resultAlloc(LInstruction* lir,
+ LDefinition* def) const {
+ if (lir->isWasmStackResult64()) {
+ return lir->toWasmStackResult64()->result(base(), def);
+ }
+ MOZ_ASSERT(def == lir->getDef(0));
+ return lir->toWasmStackResult()->result(base());
+}
+
+inline bool LNode::isCallPreserved(AnyRegister reg) const {
+ return isWasmCall() && LWasmCall::isCallPreserved(reg);
+}
+
+class LAssertRangeI : public LInstructionHelper<0, 1, 0> {
+ public:
+ LIR_HEADER(AssertRangeI)
+
+ explicit LAssertRangeI(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+
+ const LAllocation* input() { return getOperand(0); }
+
+ MAssertRange* mir() { return mir_->toAssertRange(); }
+ const Range* range() { return mir()->assertedRange(); }
+};
+
+class LAssertRangeD : public LInstructionHelper<0, 1, 1> {
+ public:
+ LIR_HEADER(AssertRangeD)
+
+ LAssertRangeD(const LAllocation& input, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* input() { return getOperand(0); }
+
+ const LDefinition* temp() { return getTemp(0); }
+
+ MAssertRange* mir() { return mir_->toAssertRange(); }
+ const Range* range() { return mir()->assertedRange(); }
+};
+
+class LAssertRangeF : public LInstructionHelper<0, 1, 2> {
+ public:
+ LIR_HEADER(AssertRangeF)
+ LAssertRangeF(const LAllocation& input, const LDefinition& temp,
+ const LDefinition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ setTemp(1, temp2);
+ }
+
+ const LAllocation* input() { return getOperand(0); }
+ const LDefinition* temp() { return getTemp(0); }
+ const LDefinition* temp2() { return getTemp(1); }
+
+ MAssertRange* mir() { return mir_->toAssertRange(); }
+ const Range* range() { return mir()->assertedRange(); }
+};
+
+class LAssertRangeV : public LInstructionHelper<0, BOX_PIECES, 3> {
+ public:
+ LIR_HEADER(AssertRangeV)
+
+ LAssertRangeV(const LBoxAllocation& input, const LDefinition& temp,
+ const LDefinition& floatTemp1, const LDefinition& floatTemp2)
+ : LInstructionHelper(classOpcode) {
+ setBoxOperand(Input, input);
+ setTemp(0, temp);
+ setTemp(1, floatTemp1);
+ setTemp(2, floatTemp2);
+ }
+
+ static const size_t Input = 0;
+
+ const LDefinition* temp() { return getTemp(0); }
+ const LDefinition* floatTemp1() { return getTemp(1); }
+ const LDefinition* floatTemp2() { return getTemp(2); }
+
+ MAssertRange* mir() { return mir_->toAssertRange(); }
+ const Range* range() { return mir()->assertedRange(); }
+};
+
+class LMemoryBarrier : public LInstructionHelper<0, 0, 0> {
+ private:
+ const MemoryBarrierBits type_;
+
+ public:
+ LIR_HEADER(MemoryBarrier)
+
+ // The parameter 'type' is a bitwise 'or' of the barrier types needed,
+ // see AtomicOp.h.
+ explicit LMemoryBarrier(MemoryBarrierBits type)
+ : LInstructionHelper(classOpcode), type_(type) {
+ MOZ_ASSERT((type_ & ~MembarAllbits) == MembarNobits);
+ }
+
+ MemoryBarrierBits type() const { return type_; }
+};
+
+// Math.random().
+class LRandom : public LInstructionHelper<1, 0, 1 + 2 * INT64_PIECES> {
+ public:
+ LIR_HEADER(Random)
+ LRandom(const LDefinition& temp0, const LInt64Definition& temp1,
+ const LInt64Definition& temp2)
+ : LInstructionHelper(classOpcode) {
+ setTemp(0, temp0);
+ setInt64Temp(1, temp1);
+ setInt64Temp(1 + INT64_PIECES, temp2);
+ }
+ const LDefinition* temp0() { return getTemp(0); }
+ LInt64Definition temp1() { return getInt64Temp(1); }
+ LInt64Definition temp2() { return getInt64Temp(1 + INT64_PIECES); }
+
+ MRandom* mir() const { return mir_->toRandom(); }
+};
+
+class LBigIntAsIntN64 : public LInstructionHelper<1, 1, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(BigIntAsIntN64)
+
+ LBigIntAsIntN64(const LAllocation& input, const LDefinition& temp,
+ const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+
+ const LAllocation* input() { return getOperand(0); }
+ const LDefinition* temp() { return getTemp(0); }
+ LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+class LBigIntAsIntN32 : public LInstructionHelper<1, 1, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(BigIntAsIntN32)
+
+ LBigIntAsIntN32(const LAllocation& input, const LDefinition& temp,
+ const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+
+ const LAllocation* input() { return getOperand(0); }
+ const LDefinition* temp() { return getTemp(0); }
+ LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+class LBigIntAsUintN64 : public LInstructionHelper<1, 1, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(BigIntAsUintN64)
+
+ LBigIntAsUintN64(const LAllocation& input, const LDefinition& temp,
+ const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+
+ const LAllocation* input() { return getOperand(0); }
+ const LDefinition* temp() { return getTemp(0); }
+ LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+class LBigIntAsUintN32 : public LInstructionHelper<1, 1, 1 + INT64_PIECES> {
+ public:
+ LIR_HEADER(BigIntAsUintN32)
+
+ LBigIntAsUintN32(const LAllocation& input, const LDefinition& temp,
+ const LInt64Definition& temp64)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ setInt64Temp(1, temp64);
+ }
+
+ const LAllocation* input() { return getOperand(0); }
+ const LDefinition* temp() { return getTemp(0); }
+ LInt64Definition temp64() { return getInt64Temp(1); }
+};
+
+template <size_t NumDefs>
+class LIonToWasmCallBase : public LVariadicInstruction<NumDefs, 1> {
+ using Base = LVariadicInstruction<NumDefs, 1>;
+
+ public:
+ explicit LIonToWasmCallBase(LNode::Opcode classOpcode, uint32_t numOperands,
+ const LDefinition& temp)
+ : Base(classOpcode, numOperands) {
+ this->setIsCall();
+ this->setTemp(0, temp);
+ }
+ MIonToWasmCall* mir() const { return this->mir_->toIonToWasmCall(); }
+ const LDefinition* temp() { return this->getTemp(0); }
+};
+
+class LIonToWasmCall : public LIonToWasmCallBase<1> {
+ public:
+ LIR_HEADER(IonToWasmCall);
+ LIonToWasmCall(uint32_t numOperands, const LDefinition& temp)
+ : LIonToWasmCallBase<1>(classOpcode, numOperands, temp) {}
+};
+
+class LIonToWasmCallV : public LIonToWasmCallBase<BOX_PIECES> {
+ public:
+ LIR_HEADER(IonToWasmCallV);
+ LIonToWasmCallV(uint32_t numOperands, const LDefinition& temp)
+ : LIonToWasmCallBase<BOX_PIECES>(classOpcode, numOperands, temp) {}
+};
+
+class LIonToWasmCallI64 : public LIonToWasmCallBase<INT64_PIECES> {
+ public:
+ LIR_HEADER(IonToWasmCallI64);
+ LIonToWasmCallI64(uint32_t numOperands, const LDefinition& temp)
+ : LIonToWasmCallBase<INT64_PIECES>(classOpcode, numOperands, temp) {}
+};
+
+class LWasmGcObjectIsSubtypeOfAndBranch
+ : public LControlInstructionHelper<2, 2, 2> {
+ uint32_t subTypingDepth_;
+
+ public:
+ LIR_HEADER(WasmGcObjectIsSubtypeOfAndBranch)
+
+ static constexpr uint32_t Object = 0;
+ static constexpr uint32_t SuperTypeDef = 1;
+
+ LWasmGcObjectIsSubtypeOfAndBranch(MBasicBlock* ifTrue, MBasicBlock* ifFalse,
+ const LAllocation& object,
+ const LAllocation& superTypeDef,
+ uint32_t subTypingDepth,
+ const LDefinition& temp0,
+ const LDefinition& temp1)
+ : LControlInstructionHelper(classOpcode),
+ subTypingDepth_(subTypingDepth) {
+ setSuccessor(0, ifTrue);
+ setSuccessor(1, ifFalse);
+ setOperand(Object, object);
+ setOperand(SuperTypeDef, superTypeDef);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ }
+
+ uint32_t subTypingDepth() const { return subTypingDepth_; }
+
+ MBasicBlock* ifTrue() const { return getSuccessor(0); }
+ MBasicBlock* ifFalse() const { return getSuccessor(1); }
+
+ const LAllocation* object() { return getOperand(Object); }
+ const LAllocation* superTypeDef() { return getOperand(SuperTypeDef); }
+ const LDefinition* temp0() { return getTemp(0); }
+ const LDefinition* temp1() { return getTemp(1); }
+};
+
+// Wasm SIMD.
+
+// (v128, v128, v128) -> v128 effect-free operation.
+// temp is FPR.
+class LWasmTernarySimd128 : public LInstructionHelper<1, 3, 1> {
+ wasm::SimdOp op_;
+
+ public:
+ LIR_HEADER(WasmTernarySimd128)
+
+ static constexpr uint32_t V0 = 0;
+ static constexpr uint32_t V1 = 1;
+ static constexpr uint32_t V2 = 2;
+
+ LWasmTernarySimd128(wasm::SimdOp op, const LAllocation& v0,
+ const LAllocation& v1, const LAllocation& v2)
+ : LInstructionHelper(classOpcode), op_(op) {
+ setOperand(V0, v0);
+ setOperand(V1, v1);
+ setOperand(V2, v2);
+ }
+
+ LWasmTernarySimd128(wasm::SimdOp op, const LAllocation& v0,
+ const LAllocation& v1, const LAllocation& v2,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode), op_(op) {
+ setOperand(V0, v0);
+ setOperand(V1, v1);
+ setOperand(V2, v2);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* v0() { return getOperand(V0); }
+ const LAllocation* v1() { return getOperand(V1); }
+ const LAllocation* v2() { return getOperand(V2); }
+ const LDefinition* temp() { return getTemp(0); }
+
+ wasm::SimdOp simdOp() const { return op_; }
+};
+
+// (v128, v128) -> v128 effect-free operations
+// lhs and dest are the same.
+// temps (if in use) are FPR.
+// The op may differ from the MIR node's op.
+class LWasmBinarySimd128 : public LInstructionHelper<1, 2, 2> {
+ wasm::SimdOp op_;
+
+ public:
+ LIR_HEADER(WasmBinarySimd128)
+
+ static constexpr uint32_t Lhs = 0;
+ static constexpr uint32_t LhsDest = 0;
+ static constexpr uint32_t Rhs = 1;
+
+ LWasmBinarySimd128(wasm::SimdOp op, const LAllocation& lhs,
+ const LAllocation& rhs, const LDefinition& temp0,
+ const LDefinition& temp1)
+ : LInstructionHelper(classOpcode), op_(op) {
+ setOperand(Lhs, lhs);
+ setOperand(Rhs, rhs);
+ setTemp(0, temp0);
+ setTemp(1, temp1);
+ }
+
+ const LAllocation* lhs() { return getOperand(Lhs); }
+ const LAllocation* lhsDest() { return getOperand(LhsDest); }
+ const LAllocation* rhs() { return getOperand(Rhs); }
+ wasm::SimdOp simdOp() const { return op_; }
+
+ static bool SpecializeForConstantRhs(wasm::SimdOp op);
+};
+
+class LWasmBinarySimd128WithConstant : public LInstructionHelper<1, 1, 1> {
+ SimdConstant rhs_;
+
+ public:
+ LIR_HEADER(WasmBinarySimd128WithConstant)
+
+ static constexpr uint32_t Lhs = 0;
+ static constexpr uint32_t LhsDest = 0;
+
+ LWasmBinarySimd128WithConstant(const LAllocation& lhs,
+ const SimdConstant& rhs,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode), rhs_(rhs) {
+ setOperand(Lhs, lhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* lhs() { return getOperand(Lhs); }
+ const LAllocation* lhsDest() { return getOperand(LhsDest); }
+ const SimdConstant& rhs() { return rhs_; }
+ wasm::SimdOp simdOp() const {
+ return mir_->toWasmBinarySimd128WithConstant()->simdOp();
+ }
+};
+
+// (v128, i32) -> v128 effect-free variable-width shift operations
+// lhs and dest are the same.
+// temp is an FPR (if in use).
+class LWasmVariableShiftSimd128 : public LInstructionHelper<1, 2, 1> {
+ public:
+ LIR_HEADER(WasmVariableShiftSimd128)
+
+ static constexpr uint32_t Lhs = 0;
+ static constexpr uint32_t LhsDest = 0;
+ static constexpr uint32_t Rhs = 1;
+
+ LWasmVariableShiftSimd128(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(Lhs, lhs);
+ setOperand(Rhs, rhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* lhs() { return getOperand(Lhs); }
+ const LAllocation* lhsDest() { return getOperand(LhsDest); }
+ const LAllocation* rhs() { return getOperand(Rhs); }
+ wasm::SimdOp simdOp() const { return mir_->toWasmShiftSimd128()->simdOp(); }
+};
+
+// (v128, i32) -> v128 effect-free constant-width shift operations
+class LWasmConstantShiftSimd128 : public LInstructionHelper<1, 1, 0> {
+ int32_t shift_;
+
+ public:
+ LIR_HEADER(WasmConstantShiftSimd128)
+
+ static constexpr uint32_t Src = 0;
+
+ LWasmConstantShiftSimd128(const LAllocation& src, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(Src, src);
+ }
+
+ const LAllocation* src() { return getOperand(Src); }
+ int32_t shift() { return shift_; }
+ wasm::SimdOp simdOp() const { return mir_->toWasmShiftSimd128()->simdOp(); }
+};
+
+// (v128) -> v128 sign replication operation.
+class LWasmSignReplicationSimd128 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmSignReplicationSimd128)
+
+ static constexpr uint32_t Src = 0;
+
+ explicit LWasmSignReplicationSimd128(const LAllocation& src)
+ : LInstructionHelper(classOpcode) {
+ setOperand(Src, src);
+ }
+
+ const LAllocation* src() { return getOperand(Src); }
+ wasm::SimdOp simdOp() const { return mir_->toWasmShiftSimd128()->simdOp(); }
+};
+
+// (v128, v128, imm_simd) -> v128 effect-free operation.
+// temp is FPR (and always in use).
+class LWasmShuffleSimd128 : public LInstructionHelper<1, 2, 1> {
+ private:
+ SimdShuffleOp op_;
+ SimdConstant control_;
+
+ public:
+ LIR_HEADER(WasmShuffleSimd128)
+
+ static constexpr uint32_t Lhs = 0;
+ static constexpr uint32_t LhsDest = 0;
+ static constexpr uint32_t Rhs = 1;
+
+ LWasmShuffleSimd128(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp, SimdShuffleOp op,
+ SimdConstant control)
+ : LInstructionHelper(classOpcode), op_(op), control_(control) {
+ setOperand(Lhs, lhs);
+ setOperand(Rhs, rhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* lhs() { return getOperand(Lhs); }
+ const LAllocation* lhsDest() { return getOperand(LhsDest); }
+ const LAllocation* rhs() { return getOperand(Rhs); }
+ const LDefinition* temp() { return getTemp(0); }
+ SimdShuffleOp op() { return op_; }
+ SimdConstant control() { return control_; }
+};
+
+// (v128, imm_simd) -> v128 effect-free operation.
+class LWasmPermuteSimd128 : public LInstructionHelper<1, 1, 0> {
+ private:
+ SimdPermuteOp op_;
+ SimdConstant control_;
+
+ public:
+ LIR_HEADER(WasmPermuteSimd128)
+
+ static constexpr uint32_t Src = 0;
+
+ LWasmPermuteSimd128(const LAllocation& src, SimdPermuteOp op,
+ SimdConstant control)
+ : LInstructionHelper(classOpcode), op_(op), control_(control) {
+ setOperand(Src, src);
+ }
+
+ const LAllocation* src() { return getOperand(Src); }
+ SimdPermuteOp op() { return op_; }
+ SimdConstant control() { return control_; }
+};
+
+class LWasmReplaceLaneSimd128 : public LInstructionHelper<1, 2, 0> {
+ public:
+ LIR_HEADER(WasmReplaceLaneSimd128)
+
+ static constexpr uint32_t Lhs = 0;
+ static constexpr uint32_t LhsDest = 0;
+ static constexpr uint32_t Rhs = 1;
+
+ LWasmReplaceLaneSimd128(const LAllocation& lhs, const LAllocation& rhs)
+ : LInstructionHelper(classOpcode) {
+ setOperand(Lhs, lhs);
+ setOperand(Rhs, rhs);
+ }
+
+ const LAllocation* lhs() { return getOperand(Lhs); }
+ const LAllocation* lhsDest() { return getOperand(LhsDest); }
+ const LAllocation* rhs() { return getOperand(Rhs); }
+ uint32_t laneIndex() const {
+ return mir_->toWasmReplaceLaneSimd128()->laneIndex();
+ }
+ wasm::SimdOp simdOp() const {
+ return mir_->toWasmReplaceLaneSimd128()->simdOp();
+ }
+};
+
+class LWasmReplaceInt64LaneSimd128
+ : public LInstructionHelper<1, INT64_PIECES + 1, 0> {
+ public:
+ LIR_HEADER(WasmReplaceInt64LaneSimd128)
+
+ static constexpr uint32_t Lhs = 0;
+ static constexpr uint32_t LhsDest = 0;
+ static constexpr uint32_t Rhs = 1;
+
+ LWasmReplaceInt64LaneSimd128(const LAllocation& lhs,
+ const LInt64Allocation& rhs)
+ : LInstructionHelper(classOpcode) {
+ setOperand(Lhs, lhs);
+ setInt64Operand(Rhs, rhs);
+ }
+
+ const LAllocation* lhs() { return getOperand(Lhs); }
+ const LAllocation* lhsDest() { return getOperand(LhsDest); }
+ const LInt64Allocation rhs() { return getInt64Operand(Rhs); }
+ const LDefinition* output() { return this->getDef(0); }
+ uint32_t laneIndex() const {
+ return mir_->toWasmReplaceLaneSimd128()->laneIndex();
+ }
+ wasm::SimdOp simdOp() const {
+ return mir_->toWasmReplaceLaneSimd128()->simdOp();
+ }
+};
+
+// (scalar) -> v128 effect-free operations, scalar != int64
+class LWasmScalarToSimd128 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmScalarToSimd128)
+
+ static constexpr uint32_t Src = 0;
+
+ explicit LWasmScalarToSimd128(const LAllocation& src)
+ : LInstructionHelper(classOpcode) {
+ setOperand(Src, src);
+ }
+
+ const LAllocation* src() { return getOperand(Src); }
+ wasm::SimdOp simdOp() const {
+ return mir_->toWasmScalarToSimd128()->simdOp();
+ }
+};
+
+// (int64) -> v128 effect-free operations
+class LWasmInt64ToSimd128 : public LInstructionHelper<1, INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(WasmInt64ToSimd128)
+
+ static constexpr uint32_t Src = 0;
+
+ explicit LWasmInt64ToSimd128(const LInt64Allocation& src)
+ : LInstructionHelper(classOpcode) {
+ setInt64Operand(Src, src);
+ }
+
+ const LInt64Allocation src() { return getInt64Operand(Src); }
+ wasm::SimdOp simdOp() const {
+ return mir_->toWasmScalarToSimd128()->simdOp();
+ }
+};
+
+// (v128) -> v128 effect-free operations
+// temp is FPR (if in use).
+class LWasmUnarySimd128 : public LInstructionHelper<1, 1, 1> {
+ public:
+ LIR_HEADER(WasmUnarySimd128)
+
+ static constexpr uint32_t Src = 0;
+
+ LWasmUnarySimd128(const LAllocation& src, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(Src, src);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* src() { return getOperand(Src); }
+ const LDefinition* temp() { return getTemp(0); }
+ wasm::SimdOp simdOp() const { return mir_->toWasmUnarySimd128()->simdOp(); }
+};
+
+// (v128, imm) -> scalar effect-free operations.
+// temp is FPR (if in use).
+class LWasmReduceSimd128 : public LInstructionHelper<1, 1, 1> {
+ public:
+ LIR_HEADER(WasmReduceSimd128)
+
+ static constexpr uint32_t Src = 0;
+
+ explicit LWasmReduceSimd128(const LAllocation& src, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(Src, src);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* src() { return getOperand(Src); }
+ uint32_t imm() const { return mir_->toWasmReduceSimd128()->imm(); }
+ wasm::SimdOp simdOp() const { return mir_->toWasmReduceSimd128()->simdOp(); }
+};
+
+// (v128, onTrue, onFalse) test-and-branch operations.
+class LWasmReduceAndBranchSimd128 : public LControlInstructionHelper<2, 1, 0> {
+ wasm::SimdOp op_;
+
+ public:
+ LIR_HEADER(WasmReduceAndBranchSimd128)
+
+ static constexpr uint32_t Src = 0;
+ static constexpr uint32_t IfTrue = 0;
+ static constexpr uint32_t IfFalse = 1;
+
+ LWasmReduceAndBranchSimd128(const LAllocation& src, wasm::SimdOp op,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse)
+ : LControlInstructionHelper(classOpcode), op_(op) {
+ setOperand(Src, src);
+ setSuccessor(IfTrue, ifTrue);
+ setSuccessor(IfFalse, ifFalse);
+ }
+
+ const LAllocation* src() { return getOperand(Src); }
+ wasm::SimdOp simdOp() const { return op_; }
+ MBasicBlock* ifTrue() const { return getSuccessor(IfTrue); }
+ MBasicBlock* ifFalse() const { return getSuccessor(IfFalse); }
+};
+
+// (v128, imm) -> i64 effect-free operations
+class LWasmReduceSimd128ToInt64
+ : public LInstructionHelper<INT64_PIECES, 1, 0> {
+ public:
+ LIR_HEADER(WasmReduceSimd128ToInt64)
+
+ static constexpr uint32_t Src = 0;
+
+ explicit LWasmReduceSimd128ToInt64(const LAllocation& src)
+ : LInstructionHelper(classOpcode) {
+ setOperand(Src, src);
+ }
+
+ const LAllocation* src() { return getOperand(Src); }
+ uint32_t imm() const { return mir_->toWasmReduceSimd128()->imm(); }
+ wasm::SimdOp simdOp() const { return mir_->toWasmReduceSimd128()->simdOp(); }
+};
+
+class LWasmLoadLaneSimd128 : public LInstructionHelper<1, 3, 1> {
+ public:
+ LIR_HEADER(WasmLoadLaneSimd128);
+
+ static constexpr uint32_t Src = 2;
+
+ explicit LWasmLoadLaneSimd128(const LAllocation& ptr, const LAllocation& src,
+ const LDefinition& temp,
+ const LAllocation& memoryBase = LAllocation())
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, memoryBase);
+ setOperand(Src, src);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LAllocation* memoryBase() { return getOperand(1); }
+ const LAllocation* src() { return getOperand(Src); }
+ const LDefinition* temp() { return getTemp(0); }
+ MWasmLoadLaneSimd128* mir() const { return mir_->toWasmLoadLaneSimd128(); }
+ uint32_t laneSize() const {
+ return mir_->toWasmLoadLaneSimd128()->laneSize();
+ }
+ uint32_t laneIndex() const {
+ return mir_->toWasmLoadLaneSimd128()->laneIndex();
+ }
+};
+
+class LWasmStoreLaneSimd128 : public LInstructionHelper<1, 3, 1> {
+ public:
+ LIR_HEADER(WasmStoreLaneSimd128);
+
+ static constexpr uint32_t Src = 2;
+
+ explicit LWasmStoreLaneSimd128(const LAllocation& ptr, const LAllocation& src,
+ const LDefinition& temp,
+ const LAllocation& memoryBase = LAllocation())
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, ptr);
+ setOperand(1, memoryBase);
+ setOperand(Src, src);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* ptr() { return getOperand(0); }
+ const LAllocation* memoryBase() { return getOperand(1); }
+ const LAllocation* src() { return getOperand(Src); }
+ const LDefinition* temp() { return getTemp(0); }
+ MWasmStoreLaneSimd128* mir() const { return mir_->toWasmStoreLaneSimd128(); }
+ uint32_t laneSize() const {
+ return mir_->toWasmStoreLaneSimd128()->laneSize();
+ }
+ uint32_t laneIndex() const {
+ return mir_->toWasmStoreLaneSimd128()->laneIndex();
+ }
+};
+
+// End Wasm SIMD
+
+// End Wasm Exception Handling
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_shared_LIR_shared_h */
diff --git a/js/src/jit/shared/Lowering-shared-inl.h b/js/src/jit/shared/Lowering-shared-inl.h
new file mode 100644
index 0000000000..30f00601cd
--- /dev/null
+++ b/js/src/jit/shared/Lowering-shared-inl.h
@@ -0,0 +1,890 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_Lowering_shared_inl_h
+#define jit_shared_Lowering_shared_inl_h
+
+#include "jit/shared/Lowering-shared.h"
+
+#include "jit/MIR.h"
+#include "jit/MIRGenerator.h"
+
+namespace js {
+namespace jit {
+
+void LIRGeneratorShared::emitAtUses(MInstruction* mir) {
+ MOZ_ASSERT(mir->canEmitAtUses());
+ mir->setEmittedAtUses();
+ mir->setVirtualRegister(0);
+}
+
+LUse LIRGeneratorShared::use(MDefinition* mir, LUse policy) {
+ // It is illegal to call use() on an instruction with two defs.
+#if BOX_PIECES > 1
+ MOZ_ASSERT(mir->type() != MIRType::Value);
+#endif
+#if INT64_PIECES > 1
+ MOZ_ASSERT(mir->type() != MIRType::Int64);
+#endif
+ ensureDefined(mir);
+ policy.setVirtualRegister(mir->virtualRegister());
+ return policy;
+}
+
+template <size_t X>
+void LIRGeneratorShared::define(
+ details::LInstructionFixedDefsTempsHelper<1, X>* lir, MDefinition* mir,
+ LDefinition::Policy policy) {
+ LDefinition::Type type = LDefinition::TypeFrom(mir->type());
+ define(lir, mir, LDefinition(type, policy));
+}
+
+template <size_t X>
+void LIRGeneratorShared::define(
+ details::LInstructionFixedDefsTempsHelper<1, X>* lir, MDefinition* mir,
+ const LDefinition& def) {
+ // Call instructions should use defineReturn.
+ MOZ_ASSERT(!lir->isCall());
+
+ uint32_t vreg = getVirtualRegister();
+
+ // Assign the definition and a virtual register. Then, propagate this
+ // virtual register to the MIR, so we can map MIR to LIR during lowering.
+ lir->setDef(0, def);
+ lir->getDef(0)->setVirtualRegister(vreg);
+ lir->setMir(mir);
+ mir->setVirtualRegister(vreg);
+ add(lir);
+}
+
+template <size_t X, size_t Y>
+void LIRGeneratorShared::defineFixed(LInstructionHelper<1, X, Y>* lir,
+ MDefinition* mir,
+ const LAllocation& output) {
+ LDefinition::Type type = LDefinition::TypeFrom(mir->type());
+
+ LDefinition def(type, LDefinition::FIXED);
+ def.setOutput(output);
+
+ define(lir, mir, def);
+}
+
+template <size_t Ops, size_t Temps>
+void LIRGeneratorShared::defineInt64Fixed(
+ LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir,
+ const LInt64Allocation& output) {
+ uint32_t vreg = getVirtualRegister();
+
+#if JS_BITS_PER_WORD == 64
+ LDefinition def(LDefinition::GENERAL, LDefinition::FIXED);
+ def.setOutput(output.value());
+ lir->setDef(0, def);
+ lir->getDef(0)->setVirtualRegister(vreg);
+#else
+ LDefinition def0(LDefinition::GENERAL, LDefinition::FIXED);
+ def0.setOutput(output.low());
+ lir->setDef(0, def0);
+ lir->getDef(0)->setVirtualRegister(vreg);
+
+ getVirtualRegister();
+ LDefinition def1(LDefinition::GENERAL, LDefinition::FIXED);
+ def1.setOutput(output.high());
+ lir->setDef(1, def1);
+ lir->getDef(1)->setVirtualRegister(vreg + 1);
+#endif
+
+ lir->setMir(mir);
+ mir->setVirtualRegister(vreg);
+ add(lir);
+}
+
+template <size_t Ops, size_t Temps>
+void LIRGeneratorShared::defineReuseInput(
+ LInstructionHelper<1, Ops, Temps>* lir, MDefinition* mir,
+ uint32_t operand) {
+ // Note: Any other operand that is not the same as this operand should be
+ // marked as not being "atStart". The regalloc cannot handle those and can
+ // overwrite the inputs!
+
+ // The input should be used at the start of the instruction, to avoid moves.
+ MOZ_ASSERT(lir->getOperand(operand)->toUse()->usedAtStart());
+
+ LDefinition::Type type = LDefinition::TypeFrom(mir->type());
+
+ LDefinition def(type, LDefinition::MUST_REUSE_INPUT);
+ def.setReusedInput(operand);
+
+ define(lir, mir, def);
+}
+
+template <size_t Ops, size_t Temps>
+void LIRGeneratorShared::defineInt64ReuseInput(
+ LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir,
+ uint32_t operand) {
+ // Note: Any other operand that is not the same as this operand should be
+ // marked as not being "atStart". The regalloc cannot handle those and can
+ // overwrite the inputs!
+
+ // The input should be used at the start of the instruction, to avoid moves.
+ MOZ_ASSERT(lir->getOperand(operand)->toUse()->usedAtStart());
+#if JS_BITS_PER_WORD == 32
+ MOZ_ASSERT(lir->getOperand(operand + 1)->toUse()->usedAtStart());
+#endif
+ MOZ_ASSERT(!lir->isCall());
+
+ uint32_t vreg = getVirtualRegister();
+
+ LDefinition def1(LDefinition::GENERAL, LDefinition::MUST_REUSE_INPUT);
+ def1.setReusedInput(operand);
+ lir->setDef(0, def1);
+ lir->getDef(0)->setVirtualRegister(vreg);
+
+#if JS_BITS_PER_WORD == 32
+ getVirtualRegister();
+ LDefinition def2(LDefinition::GENERAL, LDefinition::MUST_REUSE_INPUT);
+ def2.setReusedInput(operand + 1);
+ lir->setDef(1, def2);
+ lir->getDef(1)->setVirtualRegister(vreg + 1);
+#endif
+
+ lir->setMir(mir);
+ mir->setVirtualRegister(vreg);
+ add(lir);
+}
+
+template <size_t Ops, size_t Temps>
+void LIRGeneratorShared::defineBoxReuseInput(
+ LInstructionHelper<BOX_PIECES, Ops, Temps>* lir, MDefinition* mir,
+ uint32_t operand) {
+ // The input should be used at the start of the instruction, to avoid moves.
+ MOZ_ASSERT(lir->getOperand(operand)->toUse()->usedAtStart());
+#ifdef JS_NUNBOX32
+ MOZ_ASSERT(lir->getOperand(operand + 1)->toUse()->usedAtStart());
+#endif
+ MOZ_ASSERT(!lir->isCall());
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ uint32_t vreg = getVirtualRegister();
+
+#ifdef JS_NUNBOX32
+ static_assert(VREG_TYPE_OFFSET == 0,
+ "Code below assumes VREG_TYPE_OFFSET == 0");
+ static_assert(VREG_DATA_OFFSET == 1,
+ "Code below assumes VREG_DATA_OFFSET == 1");
+
+ LDefinition def1(LDefinition::TYPE, LDefinition::MUST_REUSE_INPUT);
+ def1.setReusedInput(operand);
+ def1.setVirtualRegister(vreg);
+ lir->setDef(0, def1);
+
+ getVirtualRegister();
+ LDefinition def2(LDefinition::PAYLOAD, LDefinition::MUST_REUSE_INPUT);
+ def2.setReusedInput(operand + 1);
+ def2.setVirtualRegister(vreg + 1);
+ lir->setDef(1, def2);
+#else
+ LDefinition def(LDefinition::BOX, LDefinition::MUST_REUSE_INPUT);
+ def.setReusedInput(operand);
+ def.setVirtualRegister(vreg);
+ lir->setDef(0, def);
+#endif
+
+ lir->setMir(mir);
+ mir->setVirtualRegister(vreg);
+ add(lir);
+}
+
+template <size_t Temps>
+void LIRGeneratorShared::defineBox(
+ details::LInstructionFixedDefsTempsHelper<BOX_PIECES, Temps>* lir,
+ MDefinition* mir, LDefinition::Policy policy) {
+ // Call instructions should use defineReturn.
+ MOZ_ASSERT(!lir->isCall());
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ uint32_t vreg = getVirtualRegister();
+
+#if defined(JS_NUNBOX32)
+ lir->setDef(0,
+ LDefinition(vreg + VREG_TYPE_OFFSET, LDefinition::TYPE, policy));
+ lir->setDef(
+ 1, LDefinition(vreg + VREG_DATA_OFFSET, LDefinition::PAYLOAD, policy));
+ getVirtualRegister();
+#elif defined(JS_PUNBOX64)
+ lir->setDef(0, LDefinition(vreg, LDefinition::BOX, policy));
+#endif
+ lir->setMir(mir);
+
+ mir->setVirtualRegister(vreg);
+ add(lir);
+}
+
+template <size_t Ops, size_t Temps>
+void LIRGeneratorShared::defineInt64(
+ LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir,
+ LDefinition::Policy policy) {
+ // Call instructions should use defineReturn.
+ MOZ_ASSERT(!lir->isCall());
+
+#ifdef JS_64BIT
+ MOZ_ASSERT(mir->type() == MIRType::Int64 || mir->type() == MIRType::IntPtr);
+#else
+ MOZ_ASSERT(mir->type() == MIRType::Int64);
+#endif
+
+ uint32_t vreg = getVirtualRegister();
+
+#if JS_BITS_PER_WORD == 32
+ lir->setDef(0,
+ LDefinition(vreg + INT64LOW_INDEX, LDefinition::GENERAL, policy));
+ lir->setDef(
+ 1, LDefinition(vreg + INT64HIGH_INDEX, LDefinition::GENERAL, policy));
+ getVirtualRegister();
+#else
+ lir->setDef(0, LDefinition(vreg, LDefinition::GENERAL, policy));
+#endif
+ lir->setMir(mir);
+
+ mir->setVirtualRegister(vreg);
+ add(lir);
+}
+
+void LIRGeneratorShared::defineReturn(LInstruction* lir, MDefinition* mir) {
+ lir->setMir(mir);
+
+ MOZ_ASSERT(lir->isCall());
+
+ uint32_t vreg = getVirtualRegister();
+
+ switch (mir->type()) {
+ case MIRType::Value:
+#if defined(JS_NUNBOX32)
+ lir->setDef(TYPE_INDEX,
+ LDefinition(vreg + VREG_TYPE_OFFSET, LDefinition::TYPE,
+ LGeneralReg(JSReturnReg_Type)));
+ lir->setDef(PAYLOAD_INDEX,
+ LDefinition(vreg + VREG_DATA_OFFSET, LDefinition::PAYLOAD,
+ LGeneralReg(JSReturnReg_Data)));
+ getVirtualRegister();
+#elif defined(JS_PUNBOX64)
+ lir->setDef(
+ 0, LDefinition(vreg, LDefinition::BOX, LGeneralReg(JSReturnReg)));
+#endif
+ break;
+ case MIRType::Int64:
+#if defined(JS_NUNBOX32)
+ lir->setDef(INT64LOW_INDEX,
+ LDefinition(vreg + INT64LOW_INDEX, LDefinition::GENERAL,
+ LGeneralReg(ReturnReg64.low)));
+ lir->setDef(INT64HIGH_INDEX,
+ LDefinition(vreg + INT64HIGH_INDEX, LDefinition::GENERAL,
+ LGeneralReg(ReturnReg64.high)));
+ getVirtualRegister();
+#elif defined(JS_PUNBOX64)
+ lir->setDef(
+ 0, LDefinition(vreg, LDefinition::GENERAL, LGeneralReg(ReturnReg)));
+#endif
+ break;
+ case MIRType::Float32:
+ lir->setDef(0, LDefinition(vreg, LDefinition::FLOAT32,
+ LFloatReg(ReturnFloat32Reg)));
+ break;
+ case MIRType::Double:
+ lir->setDef(0, LDefinition(vreg, LDefinition::DOUBLE,
+ LFloatReg(ReturnDoubleReg)));
+ break;
+ case MIRType::Simd128:
+#ifdef ENABLE_WASM_SIMD
+ lir->setDef(0, LDefinition(vreg, LDefinition::SIMD128,
+ LFloatReg(ReturnSimd128Reg)));
+ break;
+#else
+ MOZ_CRASH("No SIMD support");
+#endif
+ default:
+ LDefinition::Type type = LDefinition::TypeFrom(mir->type());
+ switch (type) {
+ case LDefinition::GENERAL:
+ case LDefinition::INT32:
+ case LDefinition::OBJECT:
+ case LDefinition::SLOTS:
+ case LDefinition::STACKRESULTS:
+ lir->setDef(0, LDefinition(vreg, type, LGeneralReg(ReturnReg)));
+ break;
+ case LDefinition::DOUBLE:
+ case LDefinition::FLOAT32:
+ case LDefinition::SIMD128:
+ MOZ_CRASH("Float cases must have been handled earlier");
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+ break;
+ }
+
+ mir->setVirtualRegister(vreg);
+ add(lir);
+}
+
+#ifdef DEBUG
+// This function checks that when making redefinitions, we don't accidentally
+// coerce two incompatible types.
+static inline bool IsCompatibleLIRCoercion(MIRType to, MIRType from) {
+ if (to == from) {
+ return true;
+ }
+ // In LIR, we treat boolean and int32 as the same low-level type (INTEGER).
+ // When snapshotting, we recover the actual JS type from MIR.
+ if ((to == MIRType::Int32 || to == MIRType::Boolean) &&
+ (from == MIRType::Int32 || from == MIRType::Boolean)) {
+ return true;
+ }
+# ifndef JS_64BIT
+ // On 32-bit platforms Int32 can be redefined as IntPtr and vice versa.
+ if ((to == MIRType::Int32 || to == MIRType::IntPtr) &&
+ (from == MIRType::IntPtr || from == MIRType::Int32)) {
+ return true;
+ }
+# endif
+ return false;
+}
+#endif
+
+void LIRGeneratorShared::redefine(MDefinition* def, MDefinition* as) {
+ MOZ_ASSERT(IsCompatibleLIRCoercion(def->type(), as->type()));
+
+ // Try to emit MIR marked as emitted-at-uses at, well, uses. For
+ // snapshotting reasons we delay the MIRTypes match, or when we are
+ // coercing between bool and int32 constants.
+ if (as->isEmittedAtUses() &&
+ (def->type() == as->type() ||
+ (as->isConstant() &&
+ (def->type() == MIRType::Int32 || def->type() == MIRType::Boolean) &&
+ (as->type() == MIRType::Int32 || as->type() == MIRType::Boolean)))) {
+ MInstruction* replacement;
+ if (def->type() != as->type()) {
+ if (as->type() == MIRType::Int32) {
+ replacement =
+ MConstant::New(alloc(), BooleanValue(as->toConstant()->toInt32()));
+ } else {
+ replacement =
+ MConstant::New(alloc(), Int32Value(as->toConstant()->toBoolean()));
+ }
+ def->block()->insertBefore(def->toInstruction(), replacement);
+ emitAtUses(replacement->toInstruction());
+ } else {
+ replacement = as->toInstruction();
+ }
+ def->replaceAllUsesWith(replacement);
+ } else {
+ ensureDefined(as);
+ def->setVirtualRegister(as->virtualRegister());
+ }
+}
+
+void LIRGeneratorShared::ensureDefined(MDefinition* mir) {
+ if (mir->isEmittedAtUses()) {
+ visitEmittedAtUses(mir->toInstruction());
+ MOZ_ASSERT(mir->isLowered());
+ }
+}
+
+bool LIRGeneratorShared::willHaveDifferentLIRNodes(MDefinition* mir1,
+ MDefinition* mir2) {
+ if (mir1 != mir2) {
+ return true;
+ }
+ if (mir1->isEmittedAtUses()) {
+ return true;
+ }
+ return false;
+}
+
+template <typename LClass, typename... Args>
+LClass* LIRGeneratorShared::allocateVariadic(uint32_t numOperands,
+ Args&&... args) {
+ size_t numBytes = sizeof(LClass) + numOperands * sizeof(LAllocation);
+ void* buf = alloc().allocate(numBytes);
+ if (!buf) {
+ return nullptr;
+ }
+
+ LClass* ins = static_cast<LClass*>(buf);
+ new (ins) LClass(numOperands, std::forward<Args>(args)...);
+
+ ins->initOperandsOffset(sizeof(LClass));
+
+ for (uint32_t i = 0; i < numOperands; i++) {
+ ins->setOperand(i, LAllocation());
+ }
+
+ return ins;
+}
+
+LUse LIRGeneratorShared::useRegister(MDefinition* mir) {
+ return use(mir, LUse(LUse::REGISTER));
+}
+
+LUse LIRGeneratorShared::useRegisterAtStart(MDefinition* mir) {
+ return use(mir, LUse(LUse::REGISTER, true));
+}
+
+LUse LIRGeneratorShared::use(MDefinition* mir) {
+ return use(mir, LUse(LUse::ANY));
+}
+
+LUse LIRGeneratorShared::useAtStart(MDefinition* mir) {
+ return use(mir, LUse(LUse::ANY, true));
+}
+
+LAllocation LIRGeneratorShared::useOrConstant(MDefinition* mir) {
+ if (mir->isConstant()) {
+ return LAllocation(mir->toConstant());
+ }
+ return use(mir);
+}
+
+LAllocation LIRGeneratorShared::useOrConstantAtStart(MDefinition* mir) {
+ if (mir->isConstant()) {
+ return LAllocation(mir->toConstant());
+ }
+ return useAtStart(mir);
+}
+
+LAllocation LIRGeneratorShared::useRegisterOrConstant(MDefinition* mir) {
+ if (mir->isConstant()) {
+ return LAllocation(mir->toConstant());
+ }
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorShared::useRegisterOrConstantAtStart(MDefinition* mir) {
+ if (mir->isConstant()) {
+ return LAllocation(mir->toConstant());
+ }
+ return useRegisterAtStart(mir);
+}
+
+inline bool CanUseInt32Constant(MDefinition* mir) {
+ if (!mir->isConstant()) {
+ return false;
+ }
+ MConstant* cst = mir->toConstant();
+ if (cst->type() == MIRType::IntPtr) {
+ return INT32_MIN <= cst->toIntPtr() && cst->toIntPtr() <= INT32_MAX;
+ }
+ MOZ_ASSERT(cst->type() == MIRType::Int32);
+ return true;
+}
+
+LAllocation LIRGeneratorShared::useRegisterOrInt32Constant(MDefinition* mir) {
+ if (CanUseInt32Constant(mir)) {
+ return LAllocation(mir->toConstant());
+ }
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorShared::useAnyOrInt32Constant(MDefinition* mir) {
+ if (CanUseInt32Constant(mir)) {
+ return LAllocation(mir->toConstant());
+ }
+ return useAny(mir);
+}
+
+LAllocation LIRGeneratorShared::useRegisterOrZero(MDefinition* mir) {
+ if (mir->isConstant() &&
+ (mir->toConstant()->isInt32(0) || mir->toConstant()->isInt64(0))) {
+ return LAllocation();
+ }
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorShared::useRegisterOrZeroAtStart(MDefinition* mir) {
+ if (mir->isConstant() &&
+ (mir->toConstant()->isInt32(0) || mir->toConstant()->isInt64(0))) {
+ return LAllocation();
+ }
+ return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorShared::useRegisterOrNonDoubleConstant(
+ MDefinition* mir) {
+ if (mir->isConstant() && mir->type() != MIRType::Double &&
+ mir->type() != MIRType::Float32) {
+ return LAllocation(mir->toConstant());
+ }
+ return useRegister(mir);
+}
+
+#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \
+ defined(JS_CODEGEN_LOONG64) || defined(JS_CODEGEN_MIPS64)
+LAllocation LIRGeneratorShared::useAnyOrConstant(MDefinition* mir) {
+ return useRegisterOrConstant(mir);
+}
+LAllocation LIRGeneratorShared::useStorable(MDefinition* mir) {
+ return useRegister(mir);
+}
+LAllocation LIRGeneratorShared::useStorableAtStart(MDefinition* mir) {
+ return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorShared::useAny(MDefinition* mir) {
+ return useRegister(mir);
+}
+LAllocation LIRGeneratorShared::useAnyAtStart(MDefinition* mir) {
+ return useRegisterAtStart(mir);
+}
+#else
+LAllocation LIRGeneratorShared::useAnyOrConstant(MDefinition* mir) {
+ return useOrConstant(mir);
+}
+
+LAllocation LIRGeneratorShared::useAny(MDefinition* mir) { return use(mir); }
+LAllocation LIRGeneratorShared::useAnyAtStart(MDefinition* mir) {
+ return useAtStart(mir);
+}
+LAllocation LIRGeneratorShared::useStorable(MDefinition* mir) {
+ return useRegisterOrConstant(mir);
+}
+LAllocation LIRGeneratorShared::useStorableAtStart(MDefinition* mir) {
+ return useRegisterOrConstantAtStart(mir);
+}
+
+#endif
+
+LAllocation LIRGeneratorShared::useKeepalive(MDefinition* mir) {
+ return use(mir, LUse(LUse::KEEPALIVE));
+}
+
+LAllocation LIRGeneratorShared::useKeepaliveOrConstant(MDefinition* mir) {
+ if (mir->isConstant()) {
+ return LAllocation(mir->toConstant());
+ }
+ return useKeepalive(mir);
+}
+
+LUse LIRGeneratorShared::useFixed(MDefinition* mir, Register reg) {
+ return use(mir, LUse(reg));
+}
+
+LUse LIRGeneratorShared::useFixedAtStart(MDefinition* mir, Register reg) {
+ return use(mir, LUse(reg, true));
+}
+
+LUse LIRGeneratorShared::useFixed(MDefinition* mir, FloatRegister reg) {
+ return use(mir, LUse(reg));
+}
+
+LUse LIRGeneratorShared::useFixed(MDefinition* mir, AnyRegister reg) {
+ return reg.isFloat() ? use(mir, LUse(reg.fpu())) : use(mir, LUse(reg.gpr()));
+}
+
+LUse LIRGeneratorShared::useFixedAtStart(MDefinition* mir, AnyRegister reg) {
+ return reg.isFloat() ? use(mir, LUse(reg.fpu(), true))
+ : use(mir, LUse(reg.gpr(), true));
+}
+
+LDefinition LIRGeneratorShared::temp(LDefinition::Type type,
+ LDefinition::Policy policy) {
+ return LDefinition(getVirtualRegister(), type, policy);
+}
+
+LInt64Definition LIRGeneratorShared::tempInt64(LDefinition::Policy policy) {
+#if JS_BITS_PER_WORD == 32
+ LDefinition high = temp(LDefinition::GENERAL, policy);
+ LDefinition low = temp(LDefinition::GENERAL, policy);
+ return LInt64Definition(high, low);
+#else
+ return LInt64Definition(temp(LDefinition::GENERAL, policy));
+#endif
+}
+
+LDefinition LIRGeneratorShared::tempFixed(Register reg) {
+ LDefinition t = temp(LDefinition::GENERAL);
+ t.setOutput(LGeneralReg(reg));
+ return t;
+}
+
+LInt64Definition LIRGeneratorShared::tempInt64Fixed(Register64 reg) {
+#if JS_BITS_PER_WORD == 32
+ LDefinition high = temp(LDefinition::GENERAL);
+ LDefinition low = temp(LDefinition::GENERAL);
+ high.setOutput(LGeneralReg(reg.high));
+ low.setOutput(LGeneralReg(reg.low));
+ return LInt64Definition(high, low);
+#else
+ LDefinition t = temp(LDefinition::GENERAL);
+ t.setOutput(LGeneralReg(reg.reg));
+ return LInt64Definition(t);
+#endif
+}
+
+LDefinition LIRGeneratorShared::tempFixed(FloatRegister reg) {
+ LDefinition t = temp(LDefinition::DOUBLE);
+ t.setOutput(LFloatReg(reg));
+ return t;
+}
+
+LDefinition LIRGeneratorShared::tempFloat32() {
+ return temp(LDefinition::FLOAT32);
+}
+
+LDefinition LIRGeneratorShared::tempDouble() {
+ return temp(LDefinition::DOUBLE);
+}
+
+#ifdef ENABLE_WASM_SIMD
+LDefinition LIRGeneratorShared::tempSimd128() {
+ return temp(LDefinition::SIMD128);
+}
+#endif
+
+LDefinition LIRGeneratorShared::tempCopy(MDefinition* input,
+ uint32_t reusedInput) {
+ MOZ_ASSERT(input->virtualRegister());
+ LDefinition t =
+ temp(LDefinition::TypeFrom(input->type()), LDefinition::MUST_REUSE_INPUT);
+ t.setReusedInput(reusedInput);
+ return t;
+}
+
+template <typename T>
+void LIRGeneratorShared::annotate(T* ins) {
+ ins->setId(lirGraph_.getInstructionId());
+}
+
+template <typename T>
+void LIRGeneratorShared::add(T* ins, MInstruction* mir) {
+ MOZ_ASSERT(!ins->isPhi());
+ current->add(ins);
+ if (mir) {
+ MOZ_ASSERT(current == mir->block()->lir());
+ ins->setMir(mir);
+ }
+ annotate(ins);
+ if (ins->isCall()) {
+ gen->setNeedsOverrecursedCheck();
+ gen->setNeedsStaticStackAlignment();
+ }
+}
+
+#ifdef JS_NUNBOX32
+// Returns the virtual register of a js::Value-defining instruction. This is
+// abstracted because MBox is a special value-returning instruction that
+// redefines its input payload if its input is not constant. Therefore, it is
+// illegal to request a box's payload by adding VREG_DATA_OFFSET to its raw id.
+static inline uint32_t VirtualRegisterOfPayload(MDefinition* mir) {
+ if (mir->isBox()) {
+ MDefinition* inner = mir->toBox()->getOperand(0);
+ if (!inner->isConstant() && inner->type() != MIRType::Double &&
+ inner->type() != MIRType::Float32) {
+ return inner->virtualRegister();
+ }
+ }
+ return mir->virtualRegister() + VREG_DATA_OFFSET;
+}
+
+// Note: always call ensureDefined before calling useType/usePayload,
+// so that emitted-at-use operands are handled correctly.
+LUse LIRGeneratorShared::useType(MDefinition* mir, LUse::Policy policy) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ return LUse(mir->virtualRegister() + VREG_TYPE_OFFSET, policy);
+}
+
+LUse LIRGeneratorShared::usePayload(MDefinition* mir, LUse::Policy policy) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ return LUse(VirtualRegisterOfPayload(mir), policy);
+}
+
+LUse LIRGeneratorShared::usePayloadAtStart(MDefinition* mir,
+ LUse::Policy policy) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ return LUse(VirtualRegisterOfPayload(mir), policy, true);
+}
+
+LUse LIRGeneratorShared::usePayloadInRegisterAtStart(MDefinition* mir) {
+ return usePayloadAtStart(mir, LUse::REGISTER);
+}
+
+void LIRGeneratorShared::fillBoxUses(LInstruction* lir, size_t n,
+ MDefinition* mir) {
+ ensureDefined(mir);
+ lir->getOperand(n)->toUse()->setVirtualRegister(mir->virtualRegister() +
+ VREG_TYPE_OFFSET);
+ lir->getOperand(n + 1)->toUse()->setVirtualRegister(
+ VirtualRegisterOfPayload(mir));
+}
+#endif
+
+LUse LIRGeneratorShared::useRegisterForTypedLoad(MDefinition* mir,
+ MIRType type) {
+ MOZ_ASSERT(type != MIRType::Value && type != MIRType::None);
+ MOZ_ASSERT(mir->type() == MIRType::Object || mir->type() == MIRType::Slots);
+
+#ifdef JS_PUNBOX64
+ // On x64, masm.loadUnboxedValue emits slightly less efficient code when
+ // the input and output use the same register and we're not loading an
+ // int32/bool/double, so we just call useRegister in this case.
+ if (type != MIRType::Int32 && type != MIRType::Boolean &&
+ type != MIRType::Double) {
+ return useRegister(mir);
+ }
+#endif
+
+ return useRegisterAtStart(mir);
+}
+
+LBoxAllocation LIRGeneratorShared::useBox(MDefinition* mir, LUse::Policy policy,
+ bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ ensureDefined(mir);
+
+#if defined(JS_NUNBOX32)
+ return LBoxAllocation(
+ LUse(mir->virtualRegister(), policy, useAtStart),
+ LUse(VirtualRegisterOfPayload(mir), policy, useAtStart));
+#else
+ return LBoxAllocation(LUse(mir->virtualRegister(), policy, useAtStart));
+#endif
+}
+
+LBoxAllocation LIRGeneratorShared::useBoxOrTyped(MDefinition* mir,
+ bool useAtStart) {
+ if (mir->type() == MIRType::Value) {
+ return useBox(mir, LUse::REGISTER, useAtStart);
+ }
+
+#if defined(JS_NUNBOX32)
+ return LBoxAllocation(useAtStart ? useRegisterAtStart(mir) : useRegister(mir),
+ LAllocation());
+#else
+ return LBoxAllocation(useAtStart ? useRegisterAtStart(mir)
+ : useRegister(mir));
+#endif
+}
+
+LBoxAllocation LIRGeneratorShared::useBoxOrTypedOrConstant(MDefinition* mir,
+ bool useConstant,
+ bool useAtStart) {
+ if (useConstant && mir->isConstant()) {
+#if defined(JS_NUNBOX32)
+ return LBoxAllocation(LAllocation(mir->toConstant()), LAllocation());
+#else
+ return LBoxAllocation(LAllocation(mir->toConstant()));
+#endif
+ }
+
+ return useBoxOrTyped(mir, useAtStart);
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64(MDefinition* mir,
+ LUse::Policy policy,
+ bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Int64);
+
+ ensureDefined(mir);
+
+ uint32_t vreg = mir->virtualRegister();
+#if JS_BITS_PER_WORD == 32
+ return LInt64Allocation(LUse(vreg + INT64HIGH_INDEX, policy, useAtStart),
+ LUse(vreg + INT64LOW_INDEX, policy, useAtStart));
+#else
+ return LInt64Allocation(LUse(vreg, policy, useAtStart));
+#endif
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64Fixed(MDefinition* mir,
+ Register64 regs,
+ bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Int64);
+
+ ensureDefined(mir);
+
+ uint32_t vreg = mir->virtualRegister();
+#if JS_BITS_PER_WORD == 32
+ return LInt64Allocation(LUse(regs.high, vreg + INT64HIGH_INDEX, useAtStart),
+ LUse(regs.low, vreg + INT64LOW_INDEX, useAtStart));
+#else
+ return LInt64Allocation(LUse(regs.reg, vreg, useAtStart));
+#endif
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64FixedAtStart(MDefinition* mir,
+ Register64 regs) {
+ return useInt64Fixed(mir, regs, true);
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64(MDefinition* mir,
+ bool useAtStart) {
+ // On 32-bit platforms, always load the value in registers.
+#if JS_BITS_PER_WORD == 32
+ return useInt64(mir, LUse::REGISTER, useAtStart);
+#else
+ return useInt64(mir, LUse::ANY, useAtStart);
+#endif
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64AtStart(MDefinition* mir) {
+ return useInt64(mir, /* useAtStart = */ true);
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64Register(MDefinition* mir,
+ bool useAtStart) {
+ return useInt64(mir, LUse::REGISTER, useAtStart);
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64OrConstant(MDefinition* mir,
+ bool useAtStart) {
+ if (mir->isConstant()) {
+#if defined(JS_NUNBOX32)
+ return LInt64Allocation(LAllocation(mir->toConstant()), LAllocation());
+#else
+ return LInt64Allocation(LAllocation(mir->toConstant()));
+#endif
+ }
+ return useInt64(mir, useAtStart);
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64RegisterOrConstant(
+ MDefinition* mir, bool useAtStart) {
+ if (mir->isConstant()) {
+#if defined(JS_NUNBOX32)
+ return LInt64Allocation(LAllocation(mir->toConstant()), LAllocation());
+#else
+ return LInt64Allocation(LAllocation(mir->toConstant()));
+#endif
+ }
+ return useInt64Register(mir, useAtStart);
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64RegisterAtStart(MDefinition* mir) {
+ return useInt64Register(mir, /* useAtStart = */ true);
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64RegisterOrConstantAtStart(
+ MDefinition* mir) {
+ return useInt64RegisterOrConstant(mir, /* useAtStart = */ true);
+}
+
+LInt64Allocation LIRGeneratorShared::useInt64OrConstantAtStart(
+ MDefinition* mir) {
+ return useInt64OrConstant(mir, /* useAtStart = */ true);
+}
+
+void LIRGeneratorShared::lowerConstantDouble(double d, MInstruction* mir) {
+ define(new (alloc()) LDouble(d), mir);
+}
+void LIRGeneratorShared::lowerConstantFloat32(float f, MInstruction* mir) {
+ define(new (alloc()) LFloat32(f), mir);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_shared_Lowering_shared_inl_h */
diff --git a/js/src/jit/shared/Lowering-shared.cpp b/js/src/jit/shared/Lowering-shared.cpp
new file mode 100644
index 0000000000..f910c34d67
--- /dev/null
+++ b/js/src/jit/shared/Lowering-shared.cpp
@@ -0,0 +1,319 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+#include "jit/LIR.h"
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/ScalarTypeUtils.h"
+
+#include "vm/SymbolType.h"
+
+using namespace js;
+using namespace jit;
+
+using mozilla::Maybe;
+using mozilla::Nothing;
+using mozilla::Some;
+
+bool LIRGeneratorShared::ShouldReorderCommutative(MDefinition* lhs,
+ MDefinition* rhs,
+ MInstruction* ins) {
+ // lhs and rhs are used by the commutative operator.
+ MOZ_ASSERT(lhs->hasDefUses());
+ MOZ_ASSERT(rhs->hasDefUses());
+
+ // Ensure that if there is a constant, then it is in rhs.
+ if (rhs->isConstant()) {
+ return false;
+ }
+ if (lhs->isConstant()) {
+ return true;
+ }
+
+ // Since clobbering binary operations clobber the left operand, prefer a
+ // non-constant lhs operand with no further uses. To be fully precise, we
+ // should check whether this is the *last* use, but checking hasOneDefUse()
+ // is a decent approximation which doesn't require any extra analysis.
+ bool rhsSingleUse = rhs->hasOneDefUse();
+ bool lhsSingleUse = lhs->hasOneDefUse();
+ if (rhsSingleUse) {
+ if (!lhsSingleUse) {
+ return true;
+ }
+ } else {
+ if (lhsSingleUse) {
+ return false;
+ }
+ }
+
+ // If this is a reduction-style computation, such as
+ //
+ // sum = 0;
+ // for (...)
+ // sum += ...;
+ //
+ // put the phi on the left to promote coalescing. This is fairly specific.
+ if (rhsSingleUse && rhs->isPhi() && rhs->block()->isLoopHeader() &&
+ ins == rhs->toPhi()->getLoopBackedgeOperand()) {
+ return true;
+ }
+
+ return false;
+}
+
+void LIRGeneratorShared::ReorderCommutative(MDefinition** lhsp,
+ MDefinition** rhsp,
+ MInstruction* ins) {
+ MDefinition* lhs = *lhsp;
+ MDefinition* rhs = *rhsp;
+
+ if (ShouldReorderCommutative(lhs, rhs, ins)) {
+ *rhsp = lhs;
+ *lhsp = rhs;
+ }
+}
+
+void LIRGeneratorShared::definePhiOneRegister(MPhi* phi, size_t lirIndex) {
+ LPhi* lir = current->getPhi(lirIndex);
+
+ uint32_t vreg = getVirtualRegister();
+
+ phi->setVirtualRegister(vreg);
+ lir->setDef(0, LDefinition(vreg, LDefinition::TypeFrom(phi->type())));
+ annotate(lir);
+}
+
+#ifdef JS_NUNBOX32
+void LIRGeneratorShared::definePhiTwoRegisters(MPhi* phi, size_t lirIndex) {
+ LPhi* type = current->getPhi(lirIndex + VREG_TYPE_OFFSET);
+ LPhi* payload = current->getPhi(lirIndex + VREG_DATA_OFFSET);
+
+ uint32_t typeVreg = getVirtualRegister();
+ phi->setVirtualRegister(typeVreg);
+
+ uint32_t payloadVreg = getVirtualRegister();
+ MOZ_ASSERT(typeVreg + 1 == payloadVreg);
+
+ type->setDef(0, LDefinition(typeVreg, LDefinition::TYPE));
+ payload->setDef(0, LDefinition(payloadVreg, LDefinition::PAYLOAD));
+ annotate(type);
+ annotate(payload);
+}
+#endif
+
+void LIRGeneratorShared::lowerTypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ MDefinition* operand = phi->getOperand(inputPosition);
+ LPhi* lir = block->getPhi(lirIndex);
+ lir->setOperand(inputPosition, LUse(operand->virtualRegister(), LUse::ANY));
+}
+
+LRecoverInfo* LIRGeneratorShared::getRecoverInfo(MResumePoint* rp) {
+ if (cachedRecoverInfo_ && cachedRecoverInfo_->mir() == rp) {
+ return cachedRecoverInfo_;
+ }
+
+ LRecoverInfo* recoverInfo = LRecoverInfo::New(gen, rp);
+ if (!recoverInfo) {
+ return nullptr;
+ }
+
+ cachedRecoverInfo_ = recoverInfo;
+ return recoverInfo;
+}
+
+#ifdef DEBUG
+bool LRecoverInfo::OperandIter::canOptimizeOutIfUnused() {
+ MDefinition* ins = **this;
+
+ // We check ins->type() in addition to ins->isUnused() because
+ // EliminateDeadResumePointOperands may replace nodes with the constant
+ // MagicValue(JS_OPTIMIZED_OUT).
+ if ((ins->isUnused() || ins->type() == MIRType::MagicOptimizedOut) &&
+ (*it_)->isResumePoint()) {
+ return !(*it_)->toResumePoint()->isObservableOperand(op_);
+ }
+
+ return true;
+}
+#endif
+
+LAllocation LIRGeneratorShared::useRegisterOrIndexConstant(
+ MDefinition* mir, Scalar::Type type, int32_t offsetAdjustment) {
+ if (CanUseInt32Constant(mir)) {
+ MConstant* cst = mir->toConstant();
+ int32_t val =
+ cst->type() == MIRType::Int32 ? cst->toInt32() : cst->toIntPtr();
+ int32_t offset;
+ if (ArrayOffsetFitsInInt32(val, type, offsetAdjustment, &offset)) {
+ return LAllocation(mir->toConstant());
+ }
+ }
+ return useRegister(mir);
+}
+
+#ifdef JS_NUNBOX32
+LSnapshot* LIRGeneratorShared::buildSnapshot(MResumePoint* rp,
+ BailoutKind kind) {
+ LRecoverInfo* recoverInfo = getRecoverInfo(rp);
+ if (!recoverInfo) {
+ return nullptr;
+ }
+
+ LSnapshot* snapshot = LSnapshot::New(gen, recoverInfo, kind);
+ if (!snapshot) {
+ return nullptr;
+ }
+
+ size_t index = 0;
+ for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) {
+ // Check that optimized out operands are in eliminable slots.
+ MOZ_ASSERT(it.canOptimizeOutIfUnused());
+
+ MDefinition* ins = *it;
+
+ if (ins->isRecoveredOnBailout()) {
+ continue;
+ }
+
+ LAllocation* type = snapshot->typeOfSlot(index);
+ LAllocation* payload = snapshot->payloadOfSlot(index);
+ ++index;
+
+ if (ins->isBox()) {
+ ins = ins->toBox()->getOperand(0);
+ }
+
+ // Guards should never be eliminated.
+ MOZ_ASSERT_IF(ins->isUnused(), !ins->isGuard());
+
+ // Snapshot operands other than constants should never be
+ // emitted-at-uses. Try-catch support depends on there being no
+ // code between an instruction and the LOsiPoint that follows it.
+ MOZ_ASSERT_IF(!ins->isConstant(), !ins->isEmittedAtUses());
+
+ // The register allocation will fill these fields in with actual
+ // register/stack assignments. During code generation, we can restore
+ // interpreter state with the given information. Note that for
+ // constants, including known types, we record a dummy placeholder,
+ // since we can recover the same information, much cleaner, from MIR.
+ if (ins->isConstant() || ins->isUnused()) {
+ *type = LAllocation();
+ *payload = LAllocation();
+ } else if (ins->type() != MIRType::Value) {
+ *type = LAllocation();
+ *payload = use(ins, LUse(LUse::KEEPALIVE));
+ } else {
+ *type = useType(ins, LUse::KEEPALIVE);
+ *payload = usePayload(ins, LUse::KEEPALIVE);
+ }
+ }
+
+ return snapshot;
+}
+
+#elif JS_PUNBOX64
+
+LSnapshot* LIRGeneratorShared::buildSnapshot(MResumePoint* rp,
+ BailoutKind kind) {
+ LRecoverInfo* recoverInfo = getRecoverInfo(rp);
+ if (!recoverInfo) {
+ return nullptr;
+ }
+
+ LSnapshot* snapshot = LSnapshot::New(gen, recoverInfo, kind);
+ if (!snapshot) {
+ return nullptr;
+ }
+
+ size_t index = 0;
+ for (LRecoverInfo::OperandIter it(recoverInfo); !it; ++it) {
+ // Check that optimized out operands are in eliminable slots.
+ MOZ_ASSERT(it.canOptimizeOutIfUnused());
+
+ MDefinition* def = *it;
+
+ if (def->isRecoveredOnBailout()) {
+ continue;
+ }
+
+ if (def->isBox()) {
+ def = def->toBox()->getOperand(0);
+ }
+
+ // Guards should never be eliminated.
+ MOZ_ASSERT_IF(def->isUnused(), !def->isGuard());
+
+ // Snapshot operands other than constants should never be
+ // emitted-at-uses. Try-catch support depends on there being no
+ // code between an instruction and the LOsiPoint that follows it.
+ MOZ_ASSERT_IF(!def->isConstant(), !def->isEmittedAtUses());
+
+ LAllocation* a = snapshot->getEntry(index++);
+
+ if (def->isUnused()) {
+ *a = LAllocation();
+ continue;
+ }
+
+ *a = useKeepaliveOrConstant(def);
+ }
+
+ return snapshot;
+}
+#endif
+
+void LIRGeneratorShared::assignSnapshot(LInstruction* ins, BailoutKind kind) {
+ // assignSnapshot must be called before define/add, since
+ // it may add new instructions for emitted-at-use operands.
+ MOZ_ASSERT(ins->id() == 0);
+ MOZ_ASSERT(kind != BailoutKind::Unknown);
+
+ LSnapshot* snapshot = buildSnapshot(lastResumePoint_, kind);
+ if (!snapshot) {
+ abort(AbortReason::Alloc, "buildSnapshot failed");
+ return;
+ }
+
+ ins->assignSnapshot(snapshot);
+}
+
+void LIRGeneratorShared::assignSafepoint(LInstruction* ins, MInstruction* mir,
+ BailoutKind kind) {
+ MOZ_ASSERT(!osiPoint_);
+ MOZ_ASSERT(!ins->safepoint());
+
+ ins->initSafepoint(alloc());
+
+ MResumePoint* mrp =
+ mir->resumePoint() ? mir->resumePoint() : lastResumePoint_;
+ LSnapshot* postSnapshot = buildSnapshot(mrp, kind);
+ if (!postSnapshot) {
+ abort(AbortReason::Alloc, "buildSnapshot failed");
+ return;
+ }
+
+ osiPoint_ = new (alloc()) LOsiPoint(ins->safepoint(), postSnapshot);
+
+ if (!lirGraph_.noteNeedsSafepoint(ins)) {
+ abort(AbortReason::Alloc, "noteNeedsSafepoint failed");
+ return;
+ }
+}
+
+void LIRGeneratorShared::assignWasmSafepoint(LInstruction* ins) {
+ MOZ_ASSERT(!osiPoint_);
+ MOZ_ASSERT(!ins->safepoint());
+
+ ins->initSafepoint(alloc());
+
+ if (!lirGraph_.noteNeedsSafepoint(ins)) {
+ abort(AbortReason::Alloc, "noteNeedsSafepoint failed");
+ return;
+ }
+}
diff --git a/js/src/jit/shared/Lowering-shared.h b/js/src/jit/shared/Lowering-shared.h
new file mode 100644
index 0000000000..d26e349d8c
--- /dev/null
+++ b/js/src/jit/shared/Lowering-shared.h
@@ -0,0 +1,371 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_shared_Lowering_shared_h
+#define jit_shared_Lowering_shared_h
+
+// This file declares the structures that are used for attaching LIR to a
+// MIRGraph.
+
+#include "jit/LIR.h"
+#include "jit/MIRGenerator.h"
+
+namespace js {
+namespace jit {
+
+class MIRGenerator;
+class MIRGraph;
+class MDefinition;
+class MInstruction;
+class LOsiPoint;
+
+class LIRGeneratorShared {
+ protected:
+ MIRGenerator* gen;
+ MIRGraph& graph;
+ LIRGraph& lirGraph_;
+ LBlock* current;
+ MResumePoint* lastResumePoint_;
+ LRecoverInfo* cachedRecoverInfo_;
+ LOsiPoint* osiPoint_;
+
+ LIRGeneratorShared(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : gen(gen),
+ graph(graph),
+ lirGraph_(lirGraph),
+ current(nullptr),
+ lastResumePoint_(nullptr),
+ cachedRecoverInfo_(nullptr),
+ osiPoint_(nullptr) {}
+
+ MIRGenerator* mir() { return gen; }
+
+ // Abort errors are caught at end of visitInstruction. It is possible for
+ // multiple errors to be detected before the end of visitInstruction. In
+ // this case, we only report the first back to the MIRGenerator.
+ bool errored() { return gen->getOffThreadStatus().isErr(); }
+ void abort(AbortReason r, const char* message, ...) MOZ_FORMAT_PRINTF(3, 4) {
+ if (errored()) {
+ return;
+ }
+
+ va_list ap;
+ va_start(ap, message);
+ auto reason_ = gen->abortFmt(r, message, ap);
+ va_end(ap);
+ gen->setOffThreadStatus(reason_);
+ }
+ void abort(AbortReason r) {
+ if (errored()) {
+ return;
+ }
+
+ auto reason_ = gen->abort(r);
+ gen->setOffThreadStatus(reason_);
+ }
+
+ static void ReorderCommutative(MDefinition** lhsp, MDefinition** rhsp,
+ MInstruction* ins);
+ static bool ShouldReorderCommutative(MDefinition* lhs, MDefinition* rhs,
+ MInstruction* ins);
+
+ // A backend can decide that an instruction should be emitted at its uses,
+ // rather than at its definition. To communicate this, set the
+ // instruction's virtual register set to 0. When using the instruction,
+ // its virtual register is temporarily reassigned. To know to clear it
+ // after constructing the use information, the worklist bit is temporarily
+ // unset.
+ //
+ // The backend can use the worklist bit to determine whether or not a
+ // definition should be created.
+ inline void emitAtUses(MInstruction* mir);
+
+ // The lowest-level calls to use, those that do not wrap another call to
+ // use(), must prefix grabbing virtual register IDs by these calls.
+ inline void ensureDefined(MDefinition* mir);
+
+ void visitEmittedAtUses(MInstruction* ins);
+
+ // These all create a use of a virtual register, with an optional
+ // allocation policy.
+ //
+ // Some of these use functions have atStart variants.
+ // - non-atStart variants will tell the register allocator that the input
+ // allocation must be different from any Temp or Definition also needed for
+ // this LInstruction.
+ // - atStart variants relax that restriction and allow the input to be in
+ // the same register as any output Definition (but not Temps) used by the
+ // LInstruction. Note that it doesn't *imply* this will actually happen,
+ // but gives a hint to the register allocator that it can do it.
+ //
+ // TL;DR: Use non-atStart variants only if you need the input value after
+ // writing to any definitions (excluding temps), during code generation of
+ // this LInstruction. Otherwise, use atStart variants, which will lower
+ // register pressure.
+ //
+ // There is an additional constraint. Consider a MIR node with two
+ // MDefinition* operands, op1 and op2. If the node reuses the register of op1
+ // for its output then op1 must be used as atStart. Then, if op1 and op2
+ // represent the same LIR node then op2 must be an atStart use too; otherwise
+ // op2 must be a non-atStart use. There is however not always a 1-1 mapping
+ // from MDefinition* to LNode*, so to determine whether two MDefinition* map
+ // to the same LNode*, ALWAYS go via the willHaveDifferentLIRNodes()
+ // predicate. Do not use pointer equality on the MIR nodes.
+ //
+ // Do not add other conditions when using willHaveDifferentLIRNodes(). The
+ // predicate is the source of truth about whether to use atStart or not, no
+ // other conditions may apply in contexts when it is appropriate to use it.
+ inline LUse use(MDefinition* mir, LUse policy);
+ inline LUse use(MDefinition* mir);
+ inline LUse useAtStart(MDefinition* mir);
+ inline LUse useRegister(MDefinition* mir);
+ inline LUse useRegisterAtStart(MDefinition* mir);
+ inline LUse useFixed(MDefinition* mir, Register reg);
+ inline LUse useFixed(MDefinition* mir, FloatRegister reg);
+ inline LUse useFixed(MDefinition* mir, AnyRegister reg);
+ inline LUse useFixedAtStart(MDefinition* mir, Register reg);
+ inline LUse useFixedAtStart(MDefinition* mir, AnyRegister reg);
+ inline LAllocation useOrConstant(MDefinition* mir);
+ inline LAllocation useOrConstantAtStart(MDefinition* mir);
+ // "Any" is architecture dependent, and will include registers and stack
+ // slots on X86, and only registers on ARM.
+ inline LAllocation useAny(MDefinition* mir);
+ inline LAllocation useAnyAtStart(MDefinition* mir);
+ inline LAllocation useAnyOrConstant(MDefinition* mir);
+ // "Storable" is architecture dependend, and will include registers and
+ // constants on X86 and only registers on ARM. This is a generic "things
+ // we can expect to write into memory in 1 instruction".
+ inline LAllocation useStorable(MDefinition* mir);
+ inline LAllocation useStorableAtStart(MDefinition* mir);
+ inline LAllocation useKeepalive(MDefinition* mir);
+ inline LAllocation useKeepaliveOrConstant(MDefinition* mir);
+ inline LAllocation useRegisterOrConstant(MDefinition* mir);
+ inline LAllocation useRegisterOrConstantAtStart(MDefinition* mir);
+ inline LAllocation useRegisterOrZeroAtStart(MDefinition* mir);
+ inline LAllocation useRegisterOrZero(MDefinition* mir);
+ inline LAllocation useRegisterOrNonDoubleConstant(MDefinition* mir);
+
+ // These methods accept either an Int32 or IntPtr value. A constant is used if
+ // the value fits in an int32.
+ inline LAllocation useRegisterOrInt32Constant(MDefinition* mir);
+ inline LAllocation useAnyOrInt32Constant(MDefinition* mir);
+
+ // Like useRegisterOrInt32Constant, but uses a constant only if
+ // |int32val * Scalar::byteSize(type) + offsetAdjustment| doesn't overflow
+ // int32.
+ LAllocation useRegisterOrIndexConstant(MDefinition* mir, Scalar::Type type,
+ int32_t offsetAdjustment = 0);
+
+ inline LUse useRegisterForTypedLoad(MDefinition* mir, MIRType type);
+
+#ifdef JS_NUNBOX32
+ inline LUse useType(MDefinition* mir, LUse::Policy policy);
+ inline LUse usePayload(MDefinition* mir, LUse::Policy policy);
+ inline LUse usePayloadAtStart(MDefinition* mir, LUse::Policy policy);
+ inline LUse usePayloadInRegisterAtStart(MDefinition* mir);
+
+ // Adds a box input to an instruction, setting operand |n| to the type and
+ // |n+1| to the payload. Does not modify the operands, instead expecting a
+ // policy to already be set.
+ inline void fillBoxUses(LInstruction* lir, size_t n, MDefinition* mir);
+#endif
+
+ // Test whether mir1 and mir2 may give rise to different LIR nodes even if
+ // mir1 == mir2; use it to guide the selection of the use directive for one of
+ // the nodes in the context of a reused input. See comments above about why
+ // it's important to use this predicate and not pointer equality.
+ //
+ // This predicate may be called before or after the application of a use
+ // directive to the first of the nodes, but it is meaningless to call it after
+ // the application of a directive to the second node.
+ inline bool willHaveDifferentLIRNodes(MDefinition* mir1, MDefinition* mir2);
+
+ // These create temporary register requests.
+ inline LDefinition temp(LDefinition::Type type = LDefinition::GENERAL,
+ LDefinition::Policy policy = LDefinition::REGISTER);
+ inline LInt64Definition tempInt64(
+ LDefinition::Policy policy = LDefinition::REGISTER);
+ inline LDefinition tempFloat32();
+ inline LDefinition tempDouble();
+#ifdef ENABLE_WASM_SIMD
+ inline LDefinition tempSimd128();
+#endif
+ inline LDefinition tempCopy(MDefinition* input, uint32_t reusedInput);
+
+ // Note that the fixed register has a GENERAL type,
+ // unless the arg is of FloatRegister type
+ inline LDefinition tempFixed(Register reg);
+ inline LDefinition tempFixed(FloatRegister reg);
+ inline LInt64Definition tempInt64Fixed(Register64 reg);
+
+ template <size_t Ops, size_t Temps>
+ inline void defineFixed(LInstructionHelper<1, Ops, Temps>* lir,
+ MDefinition* mir, const LAllocation& output);
+
+ template <size_t Temps>
+ inline void defineBox(
+ details::LInstructionFixedDefsTempsHelper<BOX_PIECES, Temps>* lir,
+ MDefinition* mir, LDefinition::Policy policy = LDefinition::REGISTER);
+
+ template <size_t Ops, size_t Temps>
+ inline void defineInt64(LInstructionHelper<INT64_PIECES, Ops, Temps>* lir,
+ MDefinition* mir,
+ LDefinition::Policy policy = LDefinition::REGISTER);
+
+ template <size_t Ops, size_t Temps>
+ inline void defineInt64Fixed(
+ LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir,
+ const LInt64Allocation& output);
+
+ inline void defineReturn(LInstruction* lir, MDefinition* mir);
+
+ template <size_t X>
+ inline void define(details::LInstructionFixedDefsTempsHelper<1, X>* lir,
+ MDefinition* mir,
+ LDefinition::Policy policy = LDefinition::REGISTER);
+ template <size_t X>
+ inline void define(details::LInstructionFixedDefsTempsHelper<1, X>* lir,
+ MDefinition* mir, const LDefinition& def);
+
+ template <size_t Ops, size_t Temps>
+ inline void defineReuseInput(LInstructionHelper<1, Ops, Temps>* lir,
+ MDefinition* mir, uint32_t operand);
+
+ template <size_t Ops, size_t Temps>
+ inline void defineBoxReuseInput(
+ LInstructionHelper<BOX_PIECES, Ops, Temps>* lir, MDefinition* mir,
+ uint32_t operand);
+
+ template <size_t Ops, size_t Temps>
+ inline void defineInt64ReuseInput(
+ LInstructionHelper<INT64_PIECES, Ops, Temps>* lir, MDefinition* mir,
+ uint32_t operand);
+
+ // Returns a box allocation for a Value-typed instruction.
+ inline LBoxAllocation useBox(MDefinition* mir,
+ LUse::Policy policy = LUse::REGISTER,
+ bool useAtStart = false);
+
+ // Returns a box allocation. The use is either typed, a Value, or
+ // a constant (if useConstant is true).
+ inline LBoxAllocation useBoxOrTypedOrConstant(MDefinition* mir,
+ bool useConstant,
+ bool useAtStart = false);
+ inline LBoxAllocation useBoxOrTyped(MDefinition* mir,
+ bool useAtStart = false);
+
+ // Returns an int64 allocation for an Int64-typed instruction.
+ inline LInt64Allocation useInt64(MDefinition* mir, LUse::Policy policy,
+ bool useAtStart);
+ inline LInt64Allocation useInt64(MDefinition* mir, bool useAtStart = false);
+ inline LInt64Allocation useInt64AtStart(MDefinition* mir);
+ inline LInt64Allocation useInt64OrConstant(MDefinition* mir,
+ bool useAtStart = false);
+ inline LInt64Allocation useInt64Register(MDefinition* mir,
+ bool useAtStart = false);
+ inline LInt64Allocation useInt64RegisterOrConstant(MDefinition* mir,
+ bool useAtStart = false);
+ inline LInt64Allocation useInt64Fixed(MDefinition* mir, Register64 regs,
+ bool useAtStart = false);
+ inline LInt64Allocation useInt64FixedAtStart(MDefinition* mir,
+ Register64 regs);
+
+ inline LInt64Allocation useInt64RegisterAtStart(MDefinition* mir);
+ inline LInt64Allocation useInt64RegisterOrConstantAtStart(MDefinition* mir);
+ inline LInt64Allocation useInt64OrConstantAtStart(MDefinition* mir);
+
+ // Rather than defining a new virtual register, sets |ins| to have the same
+ // virtual register as |as|.
+ inline void redefine(MDefinition* ins, MDefinition* as);
+
+ template <typename LClass, typename... Args>
+ inline LClass* allocateVariadic(uint32_t numOperands, Args&&... args);
+
+ TempAllocator& alloc() const { return graph.alloc(); }
+
+ uint32_t getVirtualRegister() {
+ uint32_t vreg = lirGraph_.getVirtualRegister();
+
+ // If we run out of virtual registers, mark code generation as having
+ // failed and return a dummy vreg. Include a + 1 here for NUNBOX32
+ // platforms that expect Value vregs to be adjacent.
+ if (vreg + 1 >= MAX_VIRTUAL_REGISTERS) {
+ abort(AbortReason::Alloc, "max virtual registers");
+ return 1;
+ }
+ return vreg;
+ }
+
+ template <typename T>
+ void annotate(T* ins);
+ template <typename T>
+ void add(T* ins, MInstruction* mir = nullptr);
+
+ void lowerTypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+
+ void definePhiOneRegister(MPhi* phi, size_t lirIndex);
+#ifdef JS_NUNBOX32
+ void definePhiTwoRegisters(MPhi* phi, size_t lirIndex);
+#endif
+
+ void defineTypedPhi(MPhi* phi, size_t lirIndex) {
+ // One register containing the payload.
+ definePhiOneRegister(phi, lirIndex);
+ }
+ void defineUntypedPhi(MPhi* phi, size_t lirIndex) {
+#ifdef JS_NUNBOX32
+ // Two registers: one for the type, one for the payload.
+ definePhiTwoRegisters(phi, lirIndex);
+#else
+ // One register containing the full Value.
+ definePhiOneRegister(phi, lirIndex);
+#endif
+ }
+
+ LOsiPoint* popOsiPoint() {
+ LOsiPoint* tmp = osiPoint_;
+ osiPoint_ = nullptr;
+ return tmp;
+ }
+
+ LRecoverInfo* getRecoverInfo(MResumePoint* rp);
+ LSnapshot* buildSnapshot(MResumePoint* rp, BailoutKind kind);
+ bool assignPostSnapshot(MInstruction* mir, LInstruction* ins);
+
+ // Marks this instruction as fallible, meaning that before it performs
+ // effects (if any), it may check pre-conditions and bailout if they do not
+ // hold. This function informs the register allocator that it will need to
+ // capture appropriate state.
+ void assignSnapshot(LInstruction* ins, BailoutKind kind);
+
+ // Marks this instruction as needing to call into either the VM or GC. This
+ // function may build a snapshot that captures the result of its own
+ // instruction, and as such, should generally be called after define*().
+ void assignSafepoint(LInstruction* ins, MInstruction* mir,
+ BailoutKind kind = BailoutKind::DuringVMCall);
+
+ // Marks this instruction as needing a wasm safepoint.
+ void assignWasmSafepoint(LInstruction* ins);
+
+ inline void lowerConstantDouble(double d, MInstruction* mir);
+ inline void lowerConstantFloat32(float f, MInstruction* mir);
+
+ bool canSpecializeWasmCompareAndSelect(MCompare::CompareType compTy,
+ MIRType insTy);
+ void lowerWasmCompareAndSelect(MWasmSelect* ins, MDefinition* lhs,
+ MDefinition* rhs, MCompare::CompareType compTy,
+ JSOp jsop);
+
+ public:
+ // Whether to generate typed reads for element accesses with hole checks.
+ static bool allowTypedElementHoleCheck() { return false; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_shared_Lowering_shared_h */
diff --git a/js/src/jit/wasm32/Architecture-wasm32.h b/js/src/jit/wasm32/Architecture-wasm32.h
new file mode 100644
index 0000000000..d7726eaa5f
--- /dev/null
+++ b/js/src/jit/wasm32/Architecture-wasm32.h
@@ -0,0 +1,174 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_Architecture_wasm32_h
+#define jit_wasm32_Architecture_wasm32_h
+
+// JitSpewer.h is included through MacroAssembler implementations for other
+// platforms, so include it here to avoid inadvertent build bustage.
+#include "jit/JitSpewer.h"
+
+#include "jit/shared/Architecture-shared.h"
+
+namespace js::jit {
+
+static const uint32_t SimdMemoryAlignment =
+ 4; // Make it 4 to avoid a bunch of div-by-zero warnings
+static const uint32_t WasmStackAlignment = 8;
+static const uint32_t WasmTrapInstructionLength = 0;
+
+// See comments in wasm::GenerateFunctionPrologue.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+class Registers {
+ public:
+ enum RegisterID {
+ sp = 0, // corresponds to global __stack_pointer which is mapped into
+ // global[0]
+ fp = 1,
+ r2 = 2,
+ r3 = 3,
+ invalid_reg,
+ invalid_reg2, // To avoid silly static_assert failures.
+ };
+ typedef uint8_t Code;
+ typedef RegisterID Encoding;
+ union RegisterContent {
+ uintptr_t r;
+ };
+
+ typedef uint8_t SetType;
+
+ static uint32_t SetSize(SetType) { MOZ_CRASH(); }
+ static uint32_t FirstBit(SetType) { MOZ_CRASH(); }
+ static uint32_t LastBit(SetType) { MOZ_CRASH(); }
+ static const char* GetName(Code) { MOZ_CRASH(); }
+ static Code FromName(const char*) { MOZ_CRASH(); }
+
+ static const Encoding StackPointer = RegisterID::sp;
+ static const Encoding FramePointer = RegisterID::fp;
+ static const Encoding Invalid = invalid_reg;
+ static const uint32_t Total = 5;
+ static const uint32_t TotalPhys = 0;
+ static const uint32_t Allocatable = 0;
+ static const SetType AllMask = 0;
+ static const SetType ArgRegMask = 0;
+ static const SetType VolatileMask = 0;
+ static const SetType NonVolatileMask = 0;
+ static const SetType NonAllocatableMask = 0;
+ static const SetType AllocatableMask = 0;
+ static const SetType JSCallMask = 0;
+ static const SetType CallMask = 0;
+};
+
+typedef uint8_t PackedRegisterMask;
+
+class FloatRegisters {
+ public:
+ enum FPRegisterID { f0 = 0, invalid_reg };
+ typedef FPRegisterID Code;
+ typedef FPRegisterID Encoding;
+ union RegisterContent {
+ float s;
+ double d;
+ };
+
+ typedef uint32_t SetType;
+
+ static const char* GetName(Code) { MOZ_CRASH(); }
+ static Code FromName(const char*) { MOZ_CRASH(); }
+
+ static const Code Invalid = invalid_reg;
+ static const uint32_t Total = 0;
+ static const uint32_t TotalPhys = 0;
+ static const uint32_t Allocatable = 0;
+ static const SetType AllMask = 0;
+ static const SetType AllDoubleMask = 0;
+ static const SetType AllSingleMask = 0;
+ static const SetType VolatileMask = 0;
+ static const SetType NonVolatileMask = 0;
+ static const SetType NonAllocatableMask = 0;
+ static const SetType AllocatableMask = 0;
+};
+
+template <typename T>
+class TypedRegisterSet;
+
+struct FloatRegister {
+ typedef FloatRegisters Codes;
+ typedef Codes::Code Code;
+ typedef Codes::Encoding Encoding;
+ typedef Codes::SetType SetType;
+
+ Code _;
+
+ static uint32_t FirstBit(SetType) { MOZ_CRASH(); }
+ static uint32_t LastBit(SetType) { MOZ_CRASH(); }
+ static FloatRegister FromCode(uint32_t) { MOZ_CRASH(); }
+ bool isSingle() const { MOZ_CRASH(); }
+ bool isDouble() const { MOZ_CRASH(); }
+ bool isSimd128() const { MOZ_CRASH(); }
+ bool isInvalid() const { MOZ_CRASH(); }
+ FloatRegister asSingle() const { MOZ_CRASH(); }
+ FloatRegister asDouble() const { MOZ_CRASH(); }
+ FloatRegister asSimd128() const { MOZ_CRASH(); }
+ Code code() const { MOZ_CRASH(); }
+ Encoding encoding() const { MOZ_CRASH(); }
+ const char* name() const { MOZ_CRASH(); }
+ bool volatile_() const { MOZ_CRASH(); }
+ bool operator!=(FloatRegister) const { MOZ_CRASH(); }
+ bool operator==(FloatRegister) const { MOZ_CRASH(); }
+ bool aliases(FloatRegister) const { MOZ_CRASH(); }
+ uint32_t numAliased() const { MOZ_CRASH(); }
+ FloatRegister aliased(uint32_t) { MOZ_CRASH(); }
+ bool equiv(FloatRegister) const { MOZ_CRASH(); }
+ uint32_t size() const { MOZ_CRASH(); }
+ uint32_t numAlignedAliased() const { MOZ_CRASH(); }
+ FloatRegister alignedAliased(uint32_t) { MOZ_CRASH(); }
+ SetType alignedOrDominatedAliasedSet() const { MOZ_CRASH(); }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return SetType(0);
+ }
+
+ template <typename T>
+ static T ReduceSetForPush(T) {
+ MOZ_CRASH();
+ }
+ uint32_t getRegisterDumpOffsetInBytes() { MOZ_CRASH(); }
+ static uint32_t SetSize(SetType x) { MOZ_CRASH(); }
+ static Code FromName(const char* name) { MOZ_CRASH(); }
+
+ // This is used in static initializers, so produce a bogus value instead of
+ // crashing.
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>&) {
+ return 0;
+ }
+};
+
+inline bool hasUnaliasedDouble() { MOZ_CRASH(); }
+inline bool hasMultiAlias() { MOZ_CRASH(); }
+
+static const uint32_t ShadowStackSpace = 0;
+static const uint32_t JumpImmediateRange = INT32_MAX;
+
+#ifdef JS_NUNBOX32
+static const int32_t NUNBOX32_TYPE_OFFSET = 4;
+static const int32_t NUNBOX32_PAYLOAD_OFFSET = 0;
+#endif
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_Architecture_wasm32_h */
diff --git a/js/src/jit/wasm32/Assembler-wasm32.h b/js/src/jit/wasm32/Assembler-wasm32.h
new file mode 100644
index 0000000000..cdf832d1b2
--- /dev/null
+++ b/js/src/jit/wasm32/Assembler-wasm32.h
@@ -0,0 +1,228 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_Assembler_wasm32_h
+#define jit_wasm32_Assembler_wasm32_h
+
+#include "mozilla/Assertions.h"
+
+#include <cstdint>
+
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "jit/shared/Assembler-shared.h"
+#include "jit/wasm32/Architecture-wasm32.h"
+#include "js/Value.h"
+
+namespace js::jit {
+
+struct ImmTag : public Imm32 {
+ explicit ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {}
+};
+
+struct ImmType : public ImmTag {
+ explicit ImmType(JSValueType type) : ImmTag(JSVAL_TYPE_TO_TAG(type)) {}
+};
+
+class MacroAssembler;
+
+static constexpr Register StackPointer{Registers::StackPointer};
+static constexpr Register FramePointer{Registers::FramePointer};
+
+static constexpr Register ReturnReg{Registers::invalid_reg2};
+static constexpr FloatRegister ReturnFloat32Reg = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister ReturnDoubleReg = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister ReturnSimd128Reg = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister ScratchSimd128Reg = {
+ FloatRegisters::invalid_reg};
+static constexpr FloatRegister InvalidFloatReg = {FloatRegisters::invalid_reg};
+
+struct ScratchFloat32Scope : FloatRegister {
+ explicit ScratchFloat32Scope(MacroAssembler& masm) {}
+};
+
+struct ScratchDoubleScope : FloatRegister {
+ explicit ScratchDoubleScope(MacroAssembler& masm) {}
+};
+
+static constexpr Register OsrFrameReg{Registers::invalid_reg};
+static constexpr Register PreBarrierReg{Registers::invalid_reg};
+static constexpr Register InterpreterPCReg{Registers::invalid_reg};
+static constexpr Register CallTempReg0{Registers::invalid_reg};
+static constexpr Register CallTempReg1{Registers::invalid_reg};
+static constexpr Register CallTempReg2{Registers::invalid_reg};
+static constexpr Register CallTempReg3{Registers::invalid_reg};
+static constexpr Register CallTempReg4{Registers::invalid_reg};
+static constexpr Register CallTempReg5{Registers::invalid_reg};
+static constexpr Register InvalidReg{Registers::invalid_reg};
+static constexpr Register CallTempNonArgRegs[] = {InvalidReg, InvalidReg};
+static const uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr Register IntArgReg0{Registers::invalid_reg};
+static constexpr Register IntArgReg1{Registers::invalid_reg};
+static constexpr Register IntArgReg2{Registers::invalid_reg};
+static constexpr Register IntArgReg3{Registers::invalid_reg};
+static constexpr Register HeapReg{Registers::invalid_reg};
+
+static constexpr Register RegExpTesterRegExpReg{Registers::invalid_reg};
+static constexpr Register RegExpTesterStringReg{Registers::invalid_reg};
+static constexpr Register RegExpTesterLastIndexReg{Registers::invalid_reg};
+static constexpr Register RegExpTesterStickyReg{Registers::invalid_reg};
+
+static constexpr Register RegExpMatcherRegExpReg{Registers::invalid_reg};
+static constexpr Register RegExpMatcherStringReg{Registers::invalid_reg};
+static constexpr Register RegExpMatcherLastIndexReg{Registers::invalid_reg};
+static constexpr Register RegExpMatcherStickyReg{Registers::invalid_reg};
+
+// Uses |invalid_reg2| to avoid static_assert failures.
+static constexpr Register JSReturnReg_Type{Registers::invalid_reg2};
+static constexpr Register JSReturnReg_Data{Registers::invalid_reg2};
+static constexpr Register JSReturnReg{Registers::invalid_reg2};
+
+#if defined(JS_NUNBOX32)
+static constexpr ValueOperand JSReturnOperand(Register{Registers::r2},
+ Register{Registers::r3});
+static constexpr Register64 ReturnReg64(InvalidReg, InvalidReg);
+#elif defined(JS_PUNBOX64)
+static constexpr ValueOperand JSReturnOperand(InvalidReg);
+static constexpr Register64 ReturnReg64(InvalidReg);
+#else
+# error "Bad architecture"
+#endif
+
+static constexpr Register ABINonArgReg0{Registers::invalid_reg};
+static constexpr Register ABINonArgReg1{Registers::invalid_reg};
+static constexpr Register ABINonArgReg2{Registers::invalid_reg};
+static constexpr Register ABINonArgReg3{Registers::invalid_reg};
+static constexpr Register ABINonArgReturnReg0{Registers::invalid_reg};
+static constexpr Register ABINonArgReturnReg1{Registers::invalid_reg};
+static constexpr Register ABINonVolatileReg{Registers::invalid_reg};
+static constexpr Register ABINonArgReturnVolatileReg{Registers::invalid_reg};
+
+static constexpr FloatRegister ABINonArgDoubleReg = {
+ FloatRegisters::invalid_reg};
+
+static constexpr Register WasmTableCallScratchReg0{Registers::invalid_reg};
+static constexpr Register WasmTableCallScratchReg1{Registers::invalid_reg};
+static constexpr Register WasmTableCallSigReg{Registers::invalid_reg};
+static constexpr Register WasmTableCallIndexReg{Registers::invalid_reg};
+static constexpr Register InstanceReg{Registers::invalid_reg};
+static constexpr Register WasmJitEntryReturnScratch{Registers::invalid_reg};
+static constexpr Register WasmCallRefCallScratchReg0{Registers::invalid_reg};
+static constexpr Register WasmCallRefCallScratchReg1{Registers::invalid_reg};
+static constexpr Register WasmCallRefReg{Registers::invalid_reg};
+
+static constexpr uint32_t ABIStackAlignment = 4;
+static constexpr uint32_t CodeAlignment = 16;
+static constexpr uint32_t JitStackAlignment = 8;
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+
+static const Scale ScalePointer = TimesOne;
+
+static constexpr uint32_t Int32SizeLog2 = 2;
+
+struct MemoryArgument {
+ uint32_t align;
+ uint32_t offset;
+};
+
+class AssemblerWasm32 : public AssemblerShared {};
+
+class Assembler : public AssemblerWasm32 {
+ public:
+ enum Condition {
+ Equal,
+ NotEqual,
+ Above,
+ AboveOrEqual,
+ Below,
+ BelowOrEqual,
+ GreaterThan,
+ GreaterThanOrEqual,
+ LessThan,
+ LessThanOrEqual,
+ Overflow,
+ CarrySet,
+ CarryClear,
+ Signed,
+ NotSigned,
+ Zero,
+ NonZero,
+ Always,
+ };
+
+ enum DoubleCondition {
+ DoubleOrdered,
+ DoubleEqual,
+ DoubleNotEqual,
+ DoubleGreaterThan,
+ DoubleGreaterThanOrEqual,
+ DoubleLessThan,
+ DoubleLessThanOrEqual,
+ DoubleUnordered,
+ DoubleEqualOrUnordered,
+ DoubleNotEqualOrUnordered,
+ DoubleGreaterThanOrUnordered,
+ DoubleGreaterThanOrEqualOrUnordered,
+ DoubleLessThanOrUnordered,
+ DoubleLessThanOrEqualOrUnordered
+ };
+
+ static Condition InvertCondition(Condition) { MOZ_CRASH(); }
+
+ static DoubleCondition InvertCondition(DoubleCondition) { MOZ_CRASH(); }
+
+ template <typename T, typename S>
+ static void PatchDataWithValueCheck(CodeLocationLabel, T, S) {
+ MOZ_CRASH();
+ }
+ static void PatchWrite_Imm32(CodeLocationLabel, Imm32) { MOZ_CRASH(); }
+
+ static void PatchWrite_NearCall(CodeLocationLabel, CodeLocationLabel) {
+ MOZ_CRASH();
+ }
+ static uint32_t PatchWrite_NearCallSize() { MOZ_CRASH(); }
+
+ static void ToggleToJmp(CodeLocationLabel) { MOZ_CRASH(); }
+ static void ToggleToCmp(CodeLocationLabel) { MOZ_CRASH(); }
+ static void ToggleCall(CodeLocationLabel, bool) { MOZ_CRASH(); }
+
+ static void Bind(uint8_t*, const CodeLabel&) { MOZ_CRASH(); }
+
+ static uintptr_t GetPointer(uint8_t*) { MOZ_CRASH(); }
+
+ static bool HasRoundInstruction(RoundingMode) { return false; }
+
+ void verifyHeapAccessDisassembly(uint32_t begin, uint32_t end,
+ const Disassembler::HeapAccess& heapAccess) {
+ MOZ_CRASH();
+ }
+
+ void setUnlimitedBuffer() { MOZ_CRASH(); }
+};
+
+class Operand {
+ public:
+ explicit Operand(const Address&) { MOZ_CRASH(); }
+ explicit Operand(const Register) { MOZ_CRASH(); }
+ explicit Operand(const FloatRegister) { MOZ_CRASH(); }
+ explicit Operand(Register, Imm32) { MOZ_CRASH(); }
+ explicit Operand(Register, int32_t) { MOZ_CRASH(); }
+};
+
+class ABIArgGenerator {
+ public:
+ ABIArgGenerator() = default;
+ ABIArg next(MIRType) { MOZ_CRASH(); }
+ ABIArg& current() { MOZ_CRASH(); }
+ uint32_t stackBytesConsumedSoFar() const { MOZ_CRASH(); }
+ void increaseStackOffset(uint32_t) { MOZ_CRASH(); }
+};
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_Assembler_wasm32_h */
diff --git a/js/src/jit/wasm32/CodeGenerator-wasm32.cpp b/js/src/jit/wasm32/CodeGenerator-wasm32.cpp
new file mode 100644
index 0000000000..5535eed21d
--- /dev/null
+++ b/js/src/jit/wasm32/CodeGenerator-wasm32.cpp
@@ -0,0 +1,254 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/wasm32/CodeGenerator-wasm32.h"
+
+#include "jit/CodeGenerator.h"
+
+using namespace js::jit;
+
+void CodeGenerator::visitDouble(LDouble*) { MOZ_CRASH(); }
+void CodeGenerator::visitFloat32(LFloat32* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitValue(LValue* value) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitRotateI64(LRotateI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) { MOZ_CRASH(); }
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) { MOZ_CRASH(); }
+void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) { MOZ_CRASH(); }
+void CodeGenerator::visitCompare(LCompare* comp) { MOZ_CRASH(); }
+void CodeGenerator::visitCompareI64(LCompareI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitCompareD(LCompareD* comp) { MOZ_CRASH(); }
+void CodeGenerator::visitCompareF(LCompareF* comp) { MOZ_CRASH(); }
+void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitNotI(LNotI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitNotI64(LNotI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitNotD(LNotD* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitNotF(LNotF* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitBitNotI(LBitNotI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitBitOpI(LBitOpI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitBitOpI64(LBitOpI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitShiftI(LShiftI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitShiftI64(LShiftI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitUrshD(LUrshD* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitMinMaxD(LMinMaxD* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitMinMaxF(LMinMaxF* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitNegI(LNegI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitNegI64(LNegI64* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitNegD(LNegD* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitNegF(LNegF* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitCopySignD(LCopySignD* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitCopySignF(LCopySignF* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitClzI(LClzI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitClzI64(LClzI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitCtzI(LCtzI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitCtzI64(LCtzI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitPopcntI(LPopcntI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitPopcntI64(LPopcntI64* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitAddI(LAddI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitAddI64(LAddI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitSubI(LSubI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitSubI64(LSubI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitMulI64(LMulI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitMathD(LMathD* math) { MOZ_CRASH(); }
+void CodeGenerator::visitMathF(LMathF* math) { MOZ_CRASH(); }
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+ LWasmBuiltinTruncateFToInt32* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitPowHalfD(LPowHalfD* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitCompareExchangeTypedArrayElement(
+ LCompareExchangeTypedArrayElement* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitAtomicExchangeTypedArrayElement(
+ LAtomicExchangeTypedArrayElement* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitNearbyInt(LNearbyInt*) { MOZ_CRASH(); }
+void CodeGenerator::visitNearbyIntF(LNearbyIntF*) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitAtomicTypedArrayElementBinop(
+ LAtomicTypedArrayElementBinop* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(
+ LAtomicTypedArrayElementBinopForEffect* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmSelect(LWasmSelect* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmLoad(LWasmLoad* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmStore(LWasmStore* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* lir) { MOZ_CRASH(); }
+void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmCompareExchangeHeap(
+ LWasmCompareExchangeHeap* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+ LWasmAtomicBinopHeapForEffect* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitSimd128(LSimd128* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmBinarySimd128WithConstant(
+ LWasmBinarySimd128WithConstant* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmVariableShiftSimd128(
+ LWasmVariableShiftSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmConstantShiftSimd128(
+ LWasmConstantShiftSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmSignReplicationSimd128(
+ LWasmSignReplicationSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
+ LWasmReplaceInt64LaneSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmReduceAndBranchSimd128(
+ LWasmReduceAndBranchSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmReduceSimd128ToInt64(
+ LWasmReduceSimd128ToInt64* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitUnbox(LUnbox* unbox) { MOZ_CRASH(); }
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+ MOZ_CRASH();
+}
+void CodeGenerator::visitDivI(LDivI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitModI(LModI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitMulI(LMulI* ins) { MOZ_CRASH(); }
+void CodeGenerator::visitBox(LBox* box) { MOZ_CRASH(); }
diff --git a/js/src/jit/wasm32/CodeGenerator-wasm32.h b/js/src/jit/wasm32/CodeGenerator-wasm32.h
new file mode 100644
index 0000000000..26d4adf982
--- /dev/null
+++ b/js/src/jit/wasm32/CodeGenerator-wasm32.h
@@ -0,0 +1,76 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_CodeGenerator_wasm32_h
+#define jit_wasm32_CodeGenerator_wasm32_h
+
+#include "jit/shared/CodeGenerator-shared.h"
+
+namespace js::jit {
+
+class CodeGeneratorWasm32 : public CodeGeneratorShared {
+ protected:
+ CodeGeneratorWasm32(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm)
+ : CodeGeneratorShared(gen, graph, masm) {
+ MOZ_CRASH();
+ }
+
+ MoveOperand toMoveOperand(LAllocation) const { MOZ_CRASH(); }
+ template <typename T1, typename T2>
+ void bailoutCmp32(Assembler::Condition, T1, T2, LSnapshot*) {
+ MOZ_CRASH();
+ }
+ template <typename T1, typename T2>
+ void bailoutTest32(Assembler::Condition, T1, T2, LSnapshot*) {
+ MOZ_CRASH();
+ }
+ template <typename T1, typename T2>
+ void bailoutCmpPtr(Assembler::Condition, T1, T2, LSnapshot*) {
+ MOZ_CRASH();
+ }
+ void bailoutTestPtr(Assembler::Condition, Register, Register, LSnapshot*) {
+ MOZ_CRASH();
+ }
+ void bailoutIfFalseBool(Register, LSnapshot*) { MOZ_CRASH(); }
+ void bailoutFrom(Label*, LSnapshot*) { MOZ_CRASH(); }
+ void bailout(LSnapshot*) { MOZ_CRASH(); }
+ void bailoutIf(Assembler::Condition, LSnapshot*) { MOZ_CRASH(); }
+ bool generateOutOfLineCode() { MOZ_CRASH(); }
+ void testNullEmitBranch(Assembler::Condition, ValueOperand, MBasicBlock*,
+ MBasicBlock*) {
+ MOZ_CRASH();
+ }
+ void testUndefinedEmitBranch(Assembler::Condition, ValueOperand, MBasicBlock*,
+ MBasicBlock*) {
+ MOZ_CRASH();
+ }
+ void testObjectEmitBranch(Assembler::Condition, ValueOperand, MBasicBlock*,
+ MBasicBlock*) {
+ MOZ_CRASH();
+ }
+ void testZeroEmitBranch(Assembler::Condition, Register, MBasicBlock*,
+ MBasicBlock*) {
+ MOZ_CRASH();
+ }
+ void emitTableSwitchDispatch(MTableSwitch*, Register, Register) {
+ MOZ_CRASH();
+ }
+ void emitBigIntDiv(LBigIntDiv*, Register, Register, Register, Label*) {
+ MOZ_CRASH();
+ }
+ void emitBigIntMod(LBigIntMod*, Register, Register, Register, Label*) {
+ MOZ_CRASH();
+ }
+ ValueOperand ToValue(LInstruction*, size_t) { MOZ_CRASH(); }
+ ValueOperand ToTempValue(LInstruction*, size_t) { MOZ_CRASH(); }
+ void generateInvalidateEpilogue() { MOZ_CRASH(); }
+};
+
+typedef CodeGeneratorWasm32 CodeGeneratorSpecific;
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_CodeGenerator_wasm32_h */
diff --git a/js/src/jit/wasm32/LIR-wasm32.h b/js/src/jit/wasm32/LIR-wasm32.h
new file mode 100644
index 0000000000..8943d89143
--- /dev/null
+++ b/js/src/jit/wasm32/LIR-wasm32.h
@@ -0,0 +1,109 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_LIR_wasm32_h
+#define jit_wasm32_LIR_wasm32_h
+
+namespace js::jit {
+
+class LUnboxFloatingPoint : public LInstruction {
+ public:
+ LIR_HEADER(UnboxFloatingPoint)
+ static const size_t Input = 0;
+
+ MUnbox* mir() const { MOZ_CRASH(); }
+
+ const LDefinition* output() const { MOZ_CRASH(); }
+ MIRType type() const { MOZ_CRASH(); }
+};
+
+class LTableSwitch : public LInstruction {
+ public:
+ LIR_HEADER(TableSwitch)
+ MTableSwitch* mir() { MOZ_CRASH(); }
+
+ const LAllocation* index() { MOZ_CRASH(); }
+ const LDefinition* tempInt() { MOZ_CRASH(); }
+ const LDefinition* tempPointer() { MOZ_CRASH(); }
+};
+
+class LTableSwitchV : public LInstruction {
+ public:
+ LIR_HEADER(TableSwitchV)
+ MTableSwitch* mir() { MOZ_CRASH(); }
+
+ const LDefinition* tempInt() { MOZ_CRASH(); }
+ const LDefinition* tempFloat() { MOZ_CRASH(); }
+ const LDefinition* tempPointer() { MOZ_CRASH(); }
+
+ static const size_t InputValue = 0;
+};
+
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+ explicit LWasmUint32ToFloat32(const LAllocation&)
+ : LInstructionHelper(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+};
+
+class LUnbox : public LInstructionHelper<1, 2, 0> {
+ public:
+ MUnbox* mir() const { MOZ_CRASH(); }
+ const LAllocation* payload() { MOZ_CRASH(); }
+ const LAllocation* type() { MOZ_CRASH(); }
+ const char* extraName() const { MOZ_CRASH(); }
+};
+class LDivI : public LBinaryMath<1> {
+ public:
+ LDivI(const LAllocation&, const LAllocation&, const LDefinition&)
+ : LBinaryMath(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+ MDiv* mir() const { MOZ_CRASH(); }
+};
+class LDivPowTwoI : public LInstructionHelper<1, 1, 0> {
+ public:
+ LDivPowTwoI(const LAllocation&, int32_t)
+ : LInstructionHelper(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+ const LAllocation* numerator() { MOZ_CRASH(); }
+ int32_t shift() { MOZ_CRASH(); }
+ MDiv* mir() const { MOZ_CRASH(); }
+};
+class LModI : public LBinaryMath<1> {
+ public:
+ LModI(const LAllocation&, const LAllocation&, const LDefinition&)
+ : LBinaryMath(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+
+ const LDefinition* callTemp() { MOZ_CRASH(); }
+ MMod* mir() const { MOZ_CRASH(); }
+};
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+ explicit LWasmUint32ToDouble(const LAllocation&)
+ : LInstructionHelper(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+};
+class LModPowTwoI : public LInstructionHelper<1, 1, 0> {
+ public:
+ int32_t shift() { MOZ_CRASH(); }
+ LModPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(Opcode::Invalid) {
+ MOZ_CRASH();
+ }
+ MMod* mir() const { MOZ_CRASH(); }
+};
+
+class LMulI : public LInstruction {};
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_LIR_wasm32_h */
diff --git a/js/src/jit/wasm32/Lowering-wasm32.h b/js/src/jit/wasm32/Lowering-wasm32.h
new file mode 100644
index 0000000000..3a0aab364a
--- /dev/null
+++ b/js/src/jit/wasm32/Lowering-wasm32.h
@@ -0,0 +1,128 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_Lowering_wasm32_h
+#define jit_wasm32_Lowering_wasm32_h
+
+#include "jit/shared/Lowering-shared.h"
+
+namespace js::jit {
+
+class LIRGeneratorWasm32 : public LIRGeneratorShared {
+ protected:
+ LIRGeneratorWasm32(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorShared(gen, graph, lirGraph) {
+ MOZ_CRASH();
+ }
+
+ LBoxAllocation useBoxFixed(MDefinition*, Register, Register,
+ bool useAtStart = false) {
+ MOZ_CRASH();
+ }
+
+ LAllocation useByteOpRegister(MDefinition*) { MOZ_CRASH(); }
+ LAllocation useByteOpRegisterAtStart(MDefinition*) { MOZ_CRASH(); }
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition*) {
+ MOZ_CRASH();
+ }
+ LDefinition tempByteOpRegister() { MOZ_CRASH(); }
+ LDefinition tempToUnbox() { MOZ_CRASH(); }
+ bool needTempForPostBarrier() { MOZ_CRASH(); }
+ void lowerUntypedPhiInput(MPhi*, uint32_t, LBlock*, size_t) { MOZ_CRASH(); }
+ void lowerInt64PhiInput(MPhi*, uint32_t, LBlock*, size_t) { MOZ_CRASH(); }
+ void defineInt64Phi(MPhi*, size_t) { MOZ_CRASH(); }
+ void lowerForShift(LInstructionHelper<1, 2, 0>*, MDefinition*, MDefinition*,
+ MDefinition*) {
+ MOZ_CRASH();
+ }
+ void lowerUrshD(MUrsh*) { MOZ_CRASH(); }
+ void lowerPowOfTwoI(MPow*) { MOZ_CRASH(); }
+ template <typename T>
+ void lowerForALU(T, MDefinition*, MDefinition*, MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void lowerForFPU(T, MDefinition*, MDefinition*, MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void lowerForALUInt64(T, MDefinition*, MDefinition*,
+ MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ void lowerForMulInt64(LMulI64*, MMul*, MDefinition*,
+ MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ template <typename T>
+ void lowerForShiftInt64(T, MDefinition*, MDefinition*,
+ MDefinition* v = nullptr) {
+ MOZ_CRASH();
+ }
+ void lowerForBitAndAndBranch(LBitAndAndBranch*, MInstruction*, MDefinition*,
+ MDefinition*) {
+ MOZ_CRASH();
+ }
+ void lowerForCompareI64AndBranch(MTest*, MCompare*, JSOp, MDefinition*,
+ MDefinition*, MBasicBlock*, MBasicBlock*) {
+ MOZ_CRASH();
+ }
+
+ void lowerConstantDouble(double, MInstruction*) { MOZ_CRASH(); }
+ void lowerConstantFloat32(float, MInstruction*) { MOZ_CRASH(); }
+ void lowerTruncateDToInt32(MTruncateToInt32*) { MOZ_CRASH(); }
+ void lowerTruncateFToInt32(MTruncateToInt32*) { MOZ_CRASH(); }
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH();
+ }
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH();
+ }
+ void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins) {
+ MOZ_CRASH();
+ }
+ void lowerDivI(MDiv*) { MOZ_CRASH(); }
+ void lowerModI(MMod*) { MOZ_CRASH(); }
+ void lowerDivI64(MDiv*) { MOZ_CRASH(); }
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) { MOZ_CRASH(); }
+ void lowerModI64(MMod*) { MOZ_CRASH(); }
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) { MOZ_CRASH(); }
+ void lowerNegI(MInstruction*, MDefinition*) { MOZ_CRASH(); }
+ void lowerNegI64(MInstruction*, MDefinition*) { MOZ_CRASH(); }
+ void lowerMulI(MMul*, MDefinition*, MDefinition*) { MOZ_CRASH(); }
+ void lowerUDiv(MDiv*) { MOZ_CRASH(); }
+ void lowerUMod(MMod*) { MOZ_CRASH(); }
+ void lowerWasmSelectI(MWasmSelect* select) { MOZ_CRASH(); }
+ void lowerWasmSelectI64(MWasmSelect* select) { MOZ_CRASH(); }
+ void lowerWasmCompareAndSelect(MWasmSelect* ins, MDefinition* lhs,
+ MDefinition* rhs, MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_CRASH();
+ }
+ bool canSpecializeWasmCompareAndSelect(MCompare::CompareType compTy,
+ MIRType insTy) {
+ MOZ_CRASH();
+ }
+
+ void lowerBigIntLsh(MBigIntLsh*) { MOZ_CRASH(); }
+ void lowerBigIntRsh(MBigIntRsh*) { MOZ_CRASH(); }
+ void lowerBigIntDiv(MBigIntDiv*) { MOZ_CRASH(); }
+ void lowerBigIntMod(MBigIntMod*) { MOZ_CRASH(); }
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar*) { MOZ_CRASH(); }
+ void lowerAtomicStore64(MStoreUnboxedScalar*) { MOZ_CRASH(); }
+
+ LTableSwitch* newLTableSwitch(LAllocation, LDefinition, MTableSwitch*) {
+ MOZ_CRASH();
+ }
+ LTableSwitchV* newLTableSwitchV(MTableSwitch*) { MOZ_CRASH(); }
+};
+
+typedef LIRGeneratorWasm32 LIRGeneratorSpecific;
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_Lowering_wasm32_h */
diff --git a/js/src/jit/wasm32/MacroAssembler-wasm32-inl.h b/js/src/jit/wasm32/MacroAssembler-wasm32-inl.h
new file mode 100644
index 0000000000..eca2a07a65
--- /dev/null
+++ b/js/src/jit/wasm32/MacroAssembler-wasm32-inl.h
@@ -0,0 +1,1176 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_MacroAssembler_wasm32_inl_h
+#define jit_wasm32_MacroAssembler_wasm32_inl_h
+
+#include "jit/wasm32/MacroAssembler-wasm32.h"
+
+namespace js::jit {
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::move64(Register64 src, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::notPtr(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::andPtr(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::orPtr(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::and64(Register64 src, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::or64(Register64 src, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::xorPtr(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::byteSwap64(Register64 reg) { MOZ_CRASH(); }
+
+void MacroAssembler::addPtr(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::add64(Register64 src, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) { MOZ_CRASH(); }
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) { MOZ_CRASH(); }
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) { MOZ_CRASH(); }
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::inc64(AbsoluteAddress dest) { MOZ_CRASH(); }
+
+void MacroAssembler::neg64(Register64 reg) { MOZ_CRASH(); }
+
+void MacroAssembler::negPtr(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) { MOZ_CRASH(); }
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::lshiftPtr(Register shift, Register srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rshiftPtr(Register shift, Register srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::clz64(Register64 src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::ctz64(Register64 src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register temp) {
+ MOZ_CRASH();
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& address) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::move8SignExtend(Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::move16SignExtend(Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::loadAbiReturnAddress(Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::not32(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::and32(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::and32(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::and32(Imm32 imm, const Address& dest) { MOZ_CRASH(); }
+
+void MacroAssembler::and32(const Address& src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::or32(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::or32(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::or32(Imm32 imm, const Address& dest) { MOZ_CRASH(); }
+
+void MacroAssembler::xor32(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::xor32(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::xor32(Imm32 imm, const Address& dest) { MOZ_CRASH(); }
+
+void MacroAssembler::xor32(const Address& src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::byteSwap16SignExtend(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::byteSwap16ZeroExtend(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::byteSwap32(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::add32(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::add32(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::add32(Imm32 imm, const Address& dest) { MOZ_CRASH(); }
+
+void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::sub32(const Address& src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::sub32(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::sub32(Imm32 imm, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::mul32(Register rhs, Register srcDest) { MOZ_CRASH(); }
+
+void MacroAssembler::mul32(Imm32 imm, Register srcDest) { MOZ_CRASH(); }
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::neg32(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::negateFloat(FloatRegister reg) { MOZ_CRASH(); }
+
+void MacroAssembler::negateDouble(FloatRegister reg) { MOZ_CRASH(); }
+
+void MacroAssembler::abs32(Register src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::lshift32(Imm32 shift, Register srcDest) { MOZ_CRASH(); }
+
+void MacroAssembler::rshift32(Imm32 shift, Register srcDest) { MOZ_CRASH(); }
+
+void MacroAssembler::rshift32Arithmetic(Imm32 shift, Register srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rshift32Arithmetic(Register shift, Register srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::lshift32(Register shift, Register srcDest) { MOZ_CRASH(); }
+
+void MacroAssembler::rshift32(Register shift, Register srcDest) { MOZ_CRASH(); }
+
+void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) { MOZ_CRASH(); }
+
+void MacroAssembler::clampIntToUint8(Register reg) { MOZ_CRASH(); }
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_CRASH();
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhh,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+template <class L>
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_CRASH();
+}
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
+ Imm32 rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestString(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestNull(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestObject(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, Register tag,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestString(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestDoubleTruthy(bool truthy, FloatRegister reg,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestStringTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBigIntTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_CRASH();
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
+ Imm32 rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
+ L label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ ImmWord rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ Register rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ ImmWord rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+ Register rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const Address& dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const BaseIndex& dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const Address& dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const BaseIndex& dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) { MOZ_CRASH(); }
+
+void MacroAssembler::addPtr(const Address& src, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::subPtr(Register src, const Address& dest) { MOZ_CRASH(); }
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) { MOZ_CRASH(); }
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+ Imm32 mask, Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+ Imm32 mask, const Address& src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+ const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+ FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestInt32Truthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+template <class L>
+void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
+ L label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchAdd64(Condition cond, Imm64 imm, Register64 dest,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::quotient32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::remainder32(Register rhs, Register srcDest,
+ bool isUnsigned) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rotateLeft64(Register count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rotateRight(Register count, Register input,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 input,
+ Register64 dest, Register temp) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::flexibleLshift32(Register shift, Register srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::flexibleRshift32(Register shift, Register srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::flexibleRshift32Arithmetic(Register shift,
+ Register srcDest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::popcnt32(Register src, Register dest, Register temp) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::testNumberSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::testBooleanSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::testStringSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::testSymbolSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::testBigIntSet(Condition cond, const T& src,
+ Register dest) {
+ MOZ_CRASH();
+}
+
+template <typename T>
+void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& t,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+//}}} check_macroassembler_style
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_MacroAssembler_wasm32_inl_h */
diff --git a/js/src/jit/wasm32/MacroAssembler-wasm32.cpp b/js/src/jit/wasm32/MacroAssembler-wasm32.cpp
new file mode 100644
index 0000000000..fe82793499
--- /dev/null
+++ b/js/src/jit/wasm32/MacroAssembler-wasm32.cpp
@@ -0,0 +1,502 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/wasm32/MacroAssembler-wasm32.h"
+
+namespace js::jit {
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) { MOZ_CRASH(); }
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::PushBoxed(FloatRegister reg) { MOZ_CRASH(); }
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+ Register temp, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::pushReturnAddress() { MOZ_CRASH(); }
+
+void MacroAssembler::popReturnAddress() { MOZ_CRASH(); }
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target) {
+ MOZ_CRASH();
+}
+
+size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
+ MOZ_CRASH();
+ return 0;
+}
+
+void MacroAssembler::PushRegsInMask(LiveRegisterSet set) { MOZ_CRASH(); }
+
+void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
+ LiveRegisterSet ignore) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::PopStackPtr() { MOZ_CRASH(); }
+
+void MacroAssembler::flexibleDivMod32(Register rhs, Register srcDest,
+ Register remOutput, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::flexibleRemainder32(
+ Register rhs, Register srcDest, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
+ Register scratch) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* ok) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* ok) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempDouble) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::convertUInt64ToFloat32(Register64 src, FloatRegister dest,
+ Register temp) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::convertInt64ToFloat32(Register64 src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { MOZ_CRASH(); }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
+ Register temp) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::convertInt64ToDouble(Register64 src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::patchNopToCall(uint8_t* call, uint8_t* target) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::patchCallToNop(uint8_t* call) { MOZ_CRASH(); }
+
+void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
+ MOZ_CRASH();
+}
+
+CodeOffset MacroAssembler::farJumpWithPatch() {
+ MOZ_CRASH();
+ return CodeOffset(0);
+}
+
+void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
+ MOZ_CRASH();
+}
+
+CodeOffset MacroAssembler::call(Register reg) {
+ MOZ_CRASH();
+ return CodeOffset(0);
+}
+
+CodeOffset MacroAssembler::call(Label* label) {
+ MOZ_CRASH();
+ return CodeOffset(0);
+}
+
+CodeOffset MacroAssembler::call(wasm::SymbolicAddress imm) {
+ MOZ_CRASH();
+ return CodeOffset(0);
+}
+
+CodeOffset MacroAssembler::callWithPatch() {
+ MOZ_CRASH();
+ return CodeOffset(0);
+}
+
+CodeOffset MacroAssembler::nopPatchableToCall() {
+ MOZ_CRASH();
+ return CodeOffset(0);
+}
+
+CodeOffset MacroAssembler::wasmTrapInstruction() {
+ MOZ_CRASH();
+ return CodeOffset(0);
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_CRASH();
+}
+
+uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::Pop(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::Pop(FloatRegister t) { MOZ_CRASH(); }
+
+void MacroAssembler::Pop(const ValueOperand& val) { MOZ_CRASH(); }
+
+void MacroAssembler::Push(Register reg) { MOZ_CRASH(); }
+
+void MacroAssembler::Push(const Imm32 imm) { MOZ_CRASH(); }
+
+void MacroAssembler::Push(const ImmWord imm) { MOZ_CRASH(); }
+
+void MacroAssembler::Push(const ImmPtr imm) { MOZ_CRASH(); }
+
+void MacroAssembler::Push(const ImmGCPtr ptr) { MOZ_CRASH(); }
+
+void MacroAssembler::Push(FloatRegister reg) { MOZ_CRASH(); }
+
+void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 value, Register64 output) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::speculationBarrier() { MOZ_CRASH(); }
+
+void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
+ Register pointer) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) { MOZ_CRASH(); }
+
+void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
+ ExitFrameType type) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::call(const Address& addr) { MOZ_CRASH(); }
+
+void MacroAssembler::call(ImmWord imm) { MOZ_CRASH(); }
+
+void MacroAssembler::call(ImmPtr imm) { MOZ_CRASH(); }
+
+void MacroAssembler::call(JitCode* c) { MOZ_CRASH(); }
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool callFromWasm) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::comment(const char* msg) { MOZ_CRASH(); }
+
+void MacroAssembler::flush() { MOZ_CRASH(); }
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ MOZ_CRASH();
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ MOZ_CRASH();
+}
+
+//}}} check_macroassembler_style
+
+void MacroAssemblerWasm32::executableCopy(void* buffer) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::jump(Label* label) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::writeCodePointer(CodeLabel* label) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::haltingAlign(size_t) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::nopAlign(size_t) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::checkStackAlignment() { MOZ_CRASH(); }
+
+uint32_t MacroAssemblerWasm32::currentOffset() {
+ MOZ_CRASH();
+ return 0;
+}
+
+void MacroAssemblerWasm32::nop() { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::breakpoint() { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::abiret() { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::ret() { MOZ_CRASH(); }
+
+CodeOffset MacroAssemblerWasm32::toggledJump(Label*) { MOZ_CRASH(); }
+
+CodeOffset MacroAssemblerWasm32::toggledCall(JitCode*, bool) { MOZ_CRASH(); }
+
+size_t MacroAssemblerWasm32::ToggledCallSize(uint8_t*) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::finish() { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::pushValue(ValueOperand val) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::popValue(ValueOperand) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::tagValue(JSValueType, Register, ValueOperand) {
+ MOZ_CRASH();
+}
+
+void MacroAssemblerWasm32::retn(Imm32 n) { MOZ_CRASH(); }
+
+void MacroAssemblerWasm32::push(Register reg) { MOZ_CRASH(); }
+
+Address MacroAssemblerWasm32::ToType(const Address& address) { MOZ_CRASH(); }
+
+} // namespace js::jit
diff --git a/js/src/jit/wasm32/MacroAssembler-wasm32.h b/js/src/jit/wasm32/MacroAssembler-wasm32.h
new file mode 100644
index 0000000000..1ee794624f
--- /dev/null
+++ b/js/src/jit/wasm32/MacroAssembler-wasm32.h
@@ -0,0 +1,533 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_MacroAssembler_wasm32_h
+#define jit_wasm32_MacroAssembler_wasm32_h
+
+#include "jit/wasm32/Assembler-wasm32.h"
+
+namespace js::jit {
+
+class CompactBufferReader;
+
+class ScratchTagScope {
+ public:
+ ScratchTagScope(MacroAssembler&, const ValueOperand) {}
+ operator Register() { MOZ_CRASH(); }
+ void release() { MOZ_CRASH(); }
+ void reacquire() { MOZ_CRASH(); }
+};
+
+class ScratchTagScopeRelease {
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope*) {}
+};
+
+class MacroAssemblerWasm32 : public Assembler {
+ public:
+ size_t size() const { return bytesNeeded(); }
+
+ size_t bytesNeeded() const { MOZ_CRASH(); }
+
+ size_t jumpRelocationTableBytes() const { MOZ_CRASH(); }
+
+ size_t dataRelocationTableBytes() const { MOZ_CRASH(); }
+
+ size_t preBarrierTableBytes() const { MOZ_CRASH(); }
+
+ size_t numCodeLabels() const { MOZ_CRASH(); }
+ CodeLabel codeLabel(size_t) { MOZ_CRASH(); }
+
+ bool reserve(size_t size) { MOZ_CRASH(); }
+ bool appendRawCode(const uint8_t* code, size_t numBytes) { MOZ_CRASH(); }
+ bool swapBuffer(wasm::Bytes& bytes) { MOZ_CRASH(); }
+
+ void assertNoGCThings() const { MOZ_CRASH(); }
+
+ static void TraceJumpRelocations(JSTracer*, JitCode*, CompactBufferReader&) {
+ MOZ_CRASH();
+ }
+ static void TraceDataRelocations(JSTracer*, JitCode*, CompactBufferReader&) {
+ MOZ_CRASH();
+ }
+
+ static bool SupportsFloatingPoint() { return true; }
+ static bool SupportsUnalignedAccesses() { return false; }
+ static bool SupportsFastUnalignedFPAccesses() { return false; }
+
+ void executableCopy(void* buffer);
+
+ void copyJumpRelocationTable(uint8_t*) { MOZ_CRASH(); }
+
+ void copyDataRelocationTable(uint8_t*) { MOZ_CRASH(); }
+
+ void copyPreBarrierTable(uint8_t*) { MOZ_CRASH(); }
+
+ void processCodeLabels(uint8_t*) { MOZ_CRASH(); }
+
+ void flushBuffer() { MOZ_CRASH(); }
+
+ void bind(Label* label) { MOZ_CRASH(); }
+
+ void bind(CodeLabel* label) { MOZ_CRASH(); }
+
+ template <typename T>
+ void j(Condition, T) {
+ MOZ_CRASH();
+ }
+
+ void jump(Label* label);
+
+ void jump(JitCode* code) { MOZ_CRASH(); }
+
+ void jump(Register reg) { MOZ_CRASH(); }
+
+ void jump(const Address& address) { MOZ_CRASH(); }
+
+ void jump(ImmPtr ptr) { MOZ_CRASH(); }
+
+ void jump(TrampolinePtr code) { MOZ_CRASH(); }
+
+ void writeCodePointer(CodeLabel* label);
+
+ void haltingAlign(size_t);
+
+ void nopAlign(size_t);
+ void checkStackAlignment();
+
+ uint32_t currentOffset();
+
+ void nop();
+
+ void breakpoint();
+
+ void abiret();
+ void ret();
+
+ CodeOffset toggledJump(Label*);
+ CodeOffset toggledCall(JitCode*, bool);
+ static size_t ToggledCallSize(uint8_t*);
+
+ void finish();
+
+ template <typename T, typename S>
+ void moveValue(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S, typename U>
+ void moveValue(T, S, U) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void storeValue(const T&, const S&) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S, typename U>
+ void storeValue(T, S, U) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void storePrivateValue(const T&, const S&) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void loadValue(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void loadUnalignedValue(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void pushValue(const T&) {
+ MOZ_CRASH();
+ }
+
+ void pushValue(ValueOperand val);
+
+ template <typename T, typename S>
+ void pushValue(T, S) {
+ MOZ_CRASH();
+ }
+
+ void popValue(ValueOperand);
+ void tagValue(JSValueType, Register, ValueOperand);
+ void retn(Imm32 n);
+
+ template <typename T>
+ void push(const T&) {
+ MOZ_CRASH();
+ }
+
+ void push(Register reg);
+
+ template <typename T>
+ void Push(T) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void pop(T) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ CodeOffset pushWithPatch(T) {
+ MOZ_CRASH();
+ }
+
+ void testNullSet(Condition, ValueOperand, Register) { MOZ_CRASH(); }
+ void testObjectSet(Condition, ValueOperand, Register) { MOZ_CRASH(); }
+ void testUndefinedSet(Condition, ValueOperand, Register) { MOZ_CRASH(); }
+
+ template <typename T, typename S>
+ void cmpPtrSet(Condition, T, S, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void mov(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void movePtr(T, Register) {
+ MOZ_CRASH();
+ }
+
+ void movePtr(Register src, Register dst) { MOZ_CRASH(); }
+
+ template <typename T>
+ void move32(const T&, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void movq(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void moveFloat32(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void moveDouble(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void move64(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ CodeOffset movWithPatch(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void loadPtr(T, Register) {
+ MOZ_CRASH();
+ }
+
+ void loadPtr(const Address& address, Register dest) { MOZ_CRASH(); }
+
+ template <typename T>
+ void load32(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load32Unaligned(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void loadFloat32(T, FloatRegister) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void loadDouble(T, FloatRegister) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void loadPrivate(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load8SignExtend(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load8ZeroExtend(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load16SignExtend(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load16UnalignedSignExtend(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load16ZeroExtend(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load16UnalignedZeroExtend(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load64(T, Register64) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void load64Unaligned(T, Register64) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void storePtr(const T&, S) {
+ MOZ_CRASH();
+ }
+
+ void storePtr(Register src, const Address& address) { MOZ_CRASH(); }
+ void storePtr(ImmPtr src, const Address& address) { MOZ_CRASH(); }
+
+ template <typename T, typename S>
+ void store32(T, S) {
+ MOZ_CRASH();
+ }
+
+ void store32(Imm32 src, const Address& address) { MOZ_CRASH(); }
+
+ template <typename T, typename S>
+ void store32_NoSecondScratch(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void store32Unaligned(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void storeFloat32(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void storeDouble(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void store8(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void store16(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void store16Unaligned(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void store64(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T, typename S>
+ void store64Unaligned(T, S) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void computeEffectiveAddress(T, Register) {
+ MOZ_CRASH();
+ }
+
+ void splitTagForTest(ValueOperand, ScratchTagScope&) { MOZ_CRASH(); }
+
+ void boxDouble(FloatRegister, ValueOperand, FloatRegister) { MOZ_CRASH(); }
+ void boxNonDouble(JSValueType, Register, ValueOperand) { MOZ_CRASH(); }
+
+ template <typename T>
+ void boxDouble(FloatRegister src, const T& dest) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void unboxInt32(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void unboxBoolean(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void unboxString(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void unboxSymbol(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void unboxBigInt(T, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void unboxObject(T, Register) {
+ MOZ_CRASH();
+ }
+
+ void unboxObject(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+
+ template <typename T>
+ void unboxDouble(T, FloatRegister) {
+ MOZ_CRASH();
+ }
+
+ void unboxValue(const ValueOperand&, AnyRegister, JSValueType) {
+ MOZ_CRASH();
+ }
+
+ void unboxNonDouble(const ValueOperand&, Register, JSValueType) {
+ MOZ_CRASH();
+ }
+
+ void unboxNonDouble(const Address& address, Register dest, JSValueType type) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void unboxGCThingForGCBarrier(const T&, Register) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ MOZ_CRASH();
+ }
+
+ void notBoolean(ValueOperand) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractObject(Address, Register) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractObject(ValueOperand, Register) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractSymbol(ValueOperand, Register) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractInt32(ValueOperand, Register) { MOZ_CRASH(); }
+ [[nodiscard]] Register extractBoolean(ValueOperand, Register) { MOZ_CRASH(); }
+
+ template <typename T>
+ [[nodiscard]] Register extractTag(T, Register) {
+ MOZ_CRASH();
+ }
+
+ void convertFloat32ToInt32(FloatRegister, Register, Label*, bool v = true) {
+ MOZ_CRASH();
+ }
+ void convertDoubleToInt32(FloatRegister, Register, Label*, bool v = true) {
+ MOZ_CRASH();
+ }
+ void convertDoubleToPtr(FloatRegister, Register, Label*, bool v = true) {
+ MOZ_CRASH();
+ }
+ void convertBoolToInt32(Register, Register) { MOZ_CRASH(); }
+ void convertDoubleToFloat32(FloatRegister, FloatRegister) { MOZ_CRASH(); }
+ void convertInt32ToFloat32(Register, FloatRegister) { MOZ_CRASH(); }
+
+ template <typename T>
+ void convertInt32ToDouble(T, FloatRegister) {
+ MOZ_CRASH();
+ }
+
+ void convertFloat32ToDouble(FloatRegister, FloatRegister) { MOZ_CRASH(); }
+
+ void boolValueToDouble(ValueOperand, FloatRegister) { MOZ_CRASH(); }
+ void boolValueToFloat32(ValueOperand, FloatRegister) { MOZ_CRASH(); }
+ void int32ValueToDouble(ValueOperand, FloatRegister) { MOZ_CRASH(); }
+ void int32ValueToFloat32(ValueOperand, FloatRegister) { MOZ_CRASH(); }
+
+ void loadConstantDouble(double, FloatRegister) { MOZ_CRASH(); }
+ void loadConstantFloat32(float, FloatRegister) { MOZ_CRASH(); }
+ Condition testInt32Truthy(bool, ValueOperand) { MOZ_CRASH(); }
+ Condition testStringTruthy(bool, ValueOperand) { MOZ_CRASH(); }
+ Condition testBigIntTruthy(bool, ValueOperand) { MOZ_CRASH(); }
+
+ template <typename T>
+ void loadUnboxedValue(T, MIRType, AnyRegister) {
+ MOZ_CRASH();
+ }
+
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T, size_t, JSValueType) {
+ MOZ_CRASH();
+ }
+
+ void convertUInt32ToDouble(Register, FloatRegister) { MOZ_CRASH(); }
+ void convertUInt32ToFloat32(Register, FloatRegister) { MOZ_CRASH(); }
+ void incrementInt32Value(Address) { MOZ_CRASH(); }
+ void ensureDouble(ValueOperand, FloatRegister, Label*) { MOZ_CRASH(); }
+ void handleFailureWithHandlerTail(Label*, Label*) { MOZ_CRASH(); }
+
+ void buildFakeExitFrame(Register, uint32_t*) { MOZ_CRASH(); }
+ bool buildOOLFakeExitFrame(void*) { MOZ_CRASH(); }
+
+ void setPrinter(Sprinter*) { MOZ_CRASH(); }
+ Operand ToPayload(Operand base) { MOZ_CRASH(); }
+ Address ToPayload(const Address& base) const { return base; }
+
+ Register getStackPointer() const { return StackPointer; }
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register, Register) { MOZ_CRASH(); }
+ void profilerExitFrame() { MOZ_CRASH(); }
+
+#ifdef JS_NUNBOX32
+ Address ToType(const Address& address);
+#endif
+};
+
+typedef MacroAssemblerWasm32 MacroAssemblerSpecific;
+
+static inline bool GetTempRegForIntArg(uint32_t, uint32_t, Register*) {
+ MOZ_CRASH();
+}
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_MacroAssembler_wasm32_h */
diff --git a/js/src/jit/wasm32/MoveEmitter-wasm32.h b/js/src/jit/wasm32/MoveEmitter-wasm32.h
new file mode 100644
index 0000000000..01fc494dfd
--- /dev/null
+++ b/js/src/jit/wasm32/MoveEmitter-wasm32.h
@@ -0,0 +1,30 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_MoveEmitter_wasm32_h
+#define jit_wasm32_MoveEmitter_wasm32_h
+
+#include "mozilla/Assertions.h"
+
+namespace js::jit {
+
+class MacroAssemblerWasm32;
+class MoveResolver;
+struct Register;
+
+class MoveEmitterWasm32 {
+ public:
+ explicit MoveEmitterWasm32(MacroAssemblerWasm32&) { MOZ_CRASH(); }
+ void emit(const MoveResolver&) { MOZ_CRASH(); }
+ void finish() { MOZ_CRASH(); }
+ void setScratchRegister(Register) { MOZ_CRASH(); }
+};
+
+typedef MoveEmitterWasm32 MoveEmitter;
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_MoveEmitter_wasm32_h */
diff --git a/js/src/jit/wasm32/SharedICHelpers-wasm32-inl.h b/js/src/jit/wasm32/SharedICHelpers-wasm32-inl.h
new file mode 100644
index 0000000000..d4629dc93f
--- /dev/null
+++ b/js/src/jit/wasm32/SharedICHelpers-wasm32-inl.h
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_SharedICHelpers_wasm32_inl_h
+#define jit_wasm32_SharedICHelpers_wasm32_inl_h
+
+#include "jit/SharedICHelpers.h"
+
+namespace js::jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr, MacroAssembler&, uint32_t) {
+ MOZ_CRASH();
+}
+inline void EmitBaselineCreateStubFrameDescriptor(MacroAssembler&, Register,
+ uint32_t) {
+ MOZ_CRASH();
+}
+inline void EmitBaselineCallVM(TrampolinePtr, MacroAssembler&) { MOZ_CRASH(); }
+
+static const uint32_t STUB_FRAME_SIZE = 0;
+static const uint32_t STUB_FRAME_SAVED_STUB_OFFSET = 0;
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler&, Register) {
+ MOZ_CRASH();
+}
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_SharedICHelpers_wasm32_inl_h */
diff --git a/js/src/jit/wasm32/SharedICHelpers-wasm32.h b/js/src/jit/wasm32/SharedICHelpers-wasm32.h
new file mode 100644
index 0000000000..f45b085838
--- /dev/null
+++ b/js/src/jit/wasm32/SharedICHelpers-wasm32.h
@@ -0,0 +1,30 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_SharedICHelpers_wasm32_h
+#define jit_wasm32_SharedICHelpers_wasm32_h
+
+namespace js::jit {
+
+static const size_t ICStackValueOffset = 0;
+
+inline void EmitRestoreTailCallReg(MacroAssembler&) { MOZ_CRASH(); }
+inline void EmitRepushTailCallReg(MacroAssembler&) { MOZ_CRASH(); }
+inline void EmitCallIC(MacroAssembler&, CodeOffset*) { MOZ_CRASH(); }
+inline void EmitReturnFromIC(MacroAssembler&) { MOZ_CRASH(); }
+inline void EmitBaselineLeaveStubFrame(MacroAssembler&, bool v = false) {
+ MOZ_CRASH();
+}
+inline void EmitStubGuardFailure(MacroAssembler&) { MOZ_CRASH(); }
+
+template <typename T>
+inline void EmitPreBarrier(MacroAssembler&, T, MIRType) {
+ MOZ_CRASH();
+}
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_SharedICHelpers_wasm32_h */
diff --git a/js/src/jit/wasm32/SharedICRegisters-wasm32.h b/js/src/jit/wasm32/SharedICRegisters-wasm32.h
new file mode 100644
index 0000000000..23e43b2239
--- /dev/null
+++ b/js/src/jit/wasm32/SharedICRegisters-wasm32.h
@@ -0,0 +1,36 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_wasm32_SharedICRegisters_wasm32_h
+#define jit_wasm32_SharedICRegisters_wasm32_h
+
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "jit/wasm32/MacroAssembler-wasm32.h"
+
+namespace js::jit {
+
+static constexpr Register BaselineStackReg = StackPointer;
+static constexpr Register BaselineFrameReg = FramePointer;
+
+static constexpr ValueOperand R0 = JSReturnOperand;
+static constexpr ValueOperand R1 = JSReturnOperand;
+static constexpr ValueOperand R2 = JSReturnOperand;
+
+static constexpr Register ICTailCallReg{Registers::invalid_reg};
+static constexpr Register ICStubReg{Registers::invalid_reg};
+
+static constexpr Register ExtractTemp0{Registers::invalid_reg};
+static constexpr Register ExtractTemp1{Registers::invalid_reg};
+
+static constexpr FloatRegister FloatReg0 = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister FloatReg1 = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister FloatReg2 = {FloatRegisters::invalid_reg};
+static constexpr FloatRegister FloatReg3 = {FloatRegisters::invalid_reg};
+
+} // namespace js::jit
+
+#endif /* jit_wasm32_SharedICRegisters_wasm32_h */
diff --git a/js/src/jit/wasm32/Trampoline-wasm32.cpp b/js/src/jit/wasm32/Trampoline-wasm32.cpp
new file mode 100644
index 0000000000..7c4c4db348
--- /dev/null
+++ b/js/src/jit/wasm32/Trampoline-wasm32.cpp
@@ -0,0 +1,46 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineIC.h"
+#include "jit/JitRuntime.h"
+#include "vm/Realm.h"
+
+using namespace js;
+using namespace js::jit;
+
+void JitRuntime::generateEnterJIT(JSContext*, MacroAssembler&) { MOZ_CRASH(); }
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ return mozilla::Nothing{};
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler&, Label*) { MOZ_CRASH(); }
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler&,
+ ArgumentsRectifierKind kind) {
+ MOZ_CRASH();
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler&, Label*) {
+ MOZ_CRASH();
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext*, MacroAssembler&, MIRType) {
+ MOZ_CRASH();
+ return 0;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler&, Label*) {
+ MOZ_CRASH();
+}
+
+bool JitRuntime::generateVMWrapper(JSContext*, MacroAssembler&,
+ const VMFunctionData&, DynFn, uint32_t*) {
+ MOZ_CRASH();
+}
diff --git a/js/src/jit/x64/Assembler-x64.cpp b/js/src/jit/x64/Assembler-x64.cpp
new file mode 100644
index 0000000000..e00e34e830
--- /dev/null
+++ b/js/src/jit/x64/Assembler-x64.cpp
@@ -0,0 +1,246 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/Assembler-x64.h"
+
+#include "gc/Tracer.h"
+#include "util/Memory.h"
+
+using namespace js;
+using namespace js::jit;
+
+ABIArgGenerator::ABIArgGenerator()
+ :
+#if defined(XP_WIN)
+ regIndex_(0),
+ stackOffset_(ShadowStackSpace),
+#else
+ intRegIndex_(0),
+ floatRegIndex_(0),
+ stackOffset_(0),
+#endif
+ current_() {
+}
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+#if defined(XP_WIN)
+ static_assert(NumIntArgRegs == NumFloatArgRegs);
+ if (regIndex_ == NumIntArgRegs) {
+ if (type == MIRType::Simd128) {
+ // On Win64, >64 bit args need to be passed by reference. However, wasm
+ // doesn't allow passing SIMD values to JS, so the only way to reach this
+ // is wasm to wasm calls. Ergo we can break the native ABI here and use
+ // the Wasm ABI instead.
+ stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment);
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += Simd128DataSize;
+ } else {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ }
+ return current_;
+ }
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ current_ = ABIArg(IntArgRegs[regIndex_++]);
+ break;
+ case MIRType::Float32:
+ current_ = ABIArg(FloatArgRegs[regIndex_++].asSingle());
+ break;
+ case MIRType::Double:
+ current_ = ABIArg(FloatArgRegs[regIndex_++]);
+ break;
+ case MIRType::Simd128:
+ // On Win64, >64 bit args need to be passed by reference, but wasm
+ // doesn't allow passing SIMD values to FFIs. The only way to reach
+ // here is asm to asm calls, so we can break the ABI here.
+ current_ = ABIArg(FloatArgRegs[regIndex_++].asSimd128());
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+#else
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Int64:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ if (intRegIndex_ == NumIntArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ break;
+ }
+ current_ = ABIArg(IntArgRegs[intRegIndex_++]);
+ break;
+ case MIRType::Double:
+ case MIRType::Float32:
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ break;
+ }
+ if (type == MIRType::Float32) {
+ current_ = ABIArg(FloatArgRegs[floatRegIndex_++].asSingle());
+ } else {
+ current_ = ABIArg(FloatArgRegs[floatRegIndex_++]);
+ }
+ break;
+ case MIRType::Simd128:
+ if (floatRegIndex_ == NumFloatArgRegs) {
+ stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment);
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += Simd128DataSize;
+ break;
+ }
+ current_ = ABIArg(FloatArgRegs[floatRegIndex_++].asSimd128());
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+#endif
+}
+
+void Assembler::addPendingJump(JmpSrc src, ImmPtr target,
+ RelocationKind reloc) {
+ MOZ_ASSERT(target.value != nullptr);
+
+ // Emit reloc before modifying the jump table, since it computes a 0-based
+ // index. This jump is not patchable at runtime.
+ if (reloc == RelocationKind::JITCODE) {
+ jumpRelocations_.writeUnsigned(src.offset());
+ }
+
+ static_assert(MaxCodeBytesPerProcess <= uint64_t(2) * 1024 * 1024 * 1024,
+ "Code depends on using int32_t for cross-JitCode jump offsets");
+
+ MOZ_ASSERT_IF(reloc == RelocationKind::JITCODE,
+ AddressIsInExecutableMemory(target.value));
+
+ RelativePatch patch(src.offset(), target.value, reloc);
+ if (reloc == RelocationKind::JITCODE ||
+ AddressIsInExecutableMemory(target.value)) {
+ enoughMemory_ &= codeJumps_.append(patch);
+ } else {
+ enoughMemory_ &= extendedJumps_.append(patch);
+ }
+}
+
+void Assembler::finish() {
+ if (oom()) {
+ return;
+ }
+
+ AutoCreatedBy acb(*this, "Assembler::finish");
+
+ if (!extendedJumps_.length()) {
+ // Since we may be folowed by non-executable data, eagerly insert an
+ // undefined instruction byte to prevent processors from decoding
+ // gibberish into their pipelines. See Intel performance guides.
+ masm.ud2();
+ return;
+ }
+
+ // Emit the jump table.
+ masm.haltingAlign(SizeOfJumpTableEntry);
+ extendedJumpTable_ = masm.size();
+
+ // Zero the extended jumps table.
+ for (size_t i = 0; i < extendedJumps_.length(); i++) {
+#ifdef DEBUG
+ size_t oldSize = masm.size();
+#endif
+ MOZ_ASSERT(hasCreator());
+ masm.jmp_rip(2);
+ MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == 6);
+ // Following an indirect branch with ud2 hints to the hardware that
+ // there's no fall-through. This also aligns the 64-bit immediate.
+ masm.ud2();
+ MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == 8);
+ masm.immediate64(0);
+ MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == SizeOfExtendedJump);
+ MOZ_ASSERT_IF(!masm.oom(), masm.size() - oldSize == SizeOfJumpTableEntry);
+ }
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ AssemblerX86Shared::executableCopy(buffer);
+
+ for (RelativePatch& rp : codeJumps_) {
+ uint8_t* src = buffer + rp.offset;
+ MOZ_ASSERT(rp.target);
+
+ MOZ_RELEASE_ASSERT(X86Encoding::CanRelinkJump(src, rp.target));
+ X86Encoding::SetRel32(src, rp.target);
+ }
+
+ for (size_t i = 0; i < extendedJumps_.length(); i++) {
+ RelativePatch& rp = extendedJumps_[i];
+ uint8_t* src = buffer + rp.offset;
+ MOZ_ASSERT(rp.target);
+
+ if (X86Encoding::CanRelinkJump(src, rp.target)) {
+ X86Encoding::SetRel32(src, rp.target);
+ } else {
+ // An extended jump table must exist, and its offset must be in
+ // range.
+ MOZ_ASSERT(extendedJumpTable_);
+ MOZ_ASSERT((extendedJumpTable_ + i * SizeOfJumpTableEntry) <=
+ size() - SizeOfJumpTableEntry);
+
+ // Patch the jump to go to the extended jump entry.
+ uint8_t* entry = buffer + extendedJumpTable_ + i * SizeOfJumpTableEntry;
+ X86Encoding::SetRel32(src, entry);
+
+ // Now patch the pointer, note that we need to align it to
+ // *after* the extended jump, i.e. after the 64-bit immedate.
+ X86Encoding::SetPointer(entry + SizeOfExtendedJump, rp.target);
+ }
+ }
+}
+
+class RelocationIterator {
+ CompactBufferReader reader_;
+ uint32_t offset_ = 0;
+
+ public:
+ explicit RelocationIterator(CompactBufferReader& reader) : reader_(reader) {}
+
+ bool read() {
+ if (!reader_.more()) {
+ return false;
+ }
+ offset_ = reader_.readUnsigned();
+ return true;
+ }
+
+ uint32_t offset() const { return offset_; }
+};
+
+JitCode* Assembler::CodeFromJump(JitCode* code, uint8_t* jump) {
+ uint8_t* target = (uint8_t*)X86Encoding::GetRel32Target(jump);
+
+ MOZ_ASSERT(!code->containsNativePC(target),
+ "Extended jump table not used for cross-JitCode jumps");
+
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ RelocationIterator iter(reader);
+ while (iter.read()) {
+ JitCode* child = CodeFromJump(code, code->raw() + iter.offset());
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ MOZ_ASSERT(child == CodeFromJump(code, code->raw() + iter.offset()));
+ }
+}
diff --git a/js/src/jit/x64/Assembler-x64.h b/js/src/jit/x64/Assembler-x64.h
new file mode 100644
index 0000000000..a58059d10c
--- /dev/null
+++ b/js/src/jit/x64/Assembler-x64.h
@@ -0,0 +1,1244 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_Assembler_x64_h
+#define jit_x64_Assembler_x64_h
+
+#include <iterator>
+
+#include "jit/JitCode.h"
+#include "jit/shared/Assembler-shared.h"
+
+namespace js {
+namespace jit {
+
+static constexpr Register rax{X86Encoding::rax};
+static constexpr Register rbx{X86Encoding::rbx};
+static constexpr Register rcx{X86Encoding::rcx};
+static constexpr Register rdx{X86Encoding::rdx};
+static constexpr Register rsi{X86Encoding::rsi};
+static constexpr Register rdi{X86Encoding::rdi};
+static constexpr Register rbp{X86Encoding::rbp};
+static constexpr Register r8{X86Encoding::r8};
+static constexpr Register r9{X86Encoding::r9};
+static constexpr Register r10{X86Encoding::r10};
+static constexpr Register r11{X86Encoding::r11};
+static constexpr Register r12{X86Encoding::r12};
+static constexpr Register r13{X86Encoding::r13};
+static constexpr Register r14{X86Encoding::r14};
+static constexpr Register r15{X86Encoding::r15};
+static constexpr Register rsp{X86Encoding::rsp};
+
+static constexpr FloatRegister xmm0 =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+static constexpr FloatRegister xmm1 =
+ FloatRegister(X86Encoding::xmm1, FloatRegisters::Double);
+static constexpr FloatRegister xmm2 =
+ FloatRegister(X86Encoding::xmm2, FloatRegisters::Double);
+static constexpr FloatRegister xmm3 =
+ FloatRegister(X86Encoding::xmm3, FloatRegisters::Double);
+static constexpr FloatRegister xmm4 =
+ FloatRegister(X86Encoding::xmm4, FloatRegisters::Double);
+static constexpr FloatRegister xmm5 =
+ FloatRegister(X86Encoding::xmm5, FloatRegisters::Double);
+static constexpr FloatRegister xmm6 =
+ FloatRegister(X86Encoding::xmm6, FloatRegisters::Double);
+static constexpr FloatRegister xmm7 =
+ FloatRegister(X86Encoding::xmm7, FloatRegisters::Double);
+static constexpr FloatRegister xmm8 =
+ FloatRegister(X86Encoding::xmm8, FloatRegisters::Double);
+static constexpr FloatRegister xmm9 =
+ FloatRegister(X86Encoding::xmm9, FloatRegisters::Double);
+static constexpr FloatRegister xmm10 =
+ FloatRegister(X86Encoding::xmm10, FloatRegisters::Double);
+static constexpr FloatRegister xmm11 =
+ FloatRegister(X86Encoding::xmm11, FloatRegisters::Double);
+static constexpr FloatRegister xmm12 =
+ FloatRegister(X86Encoding::xmm12, FloatRegisters::Double);
+static constexpr FloatRegister xmm13 =
+ FloatRegister(X86Encoding::xmm13, FloatRegisters::Double);
+static constexpr FloatRegister xmm14 =
+ FloatRegister(X86Encoding::xmm14, FloatRegisters::Double);
+static constexpr FloatRegister xmm15 =
+ FloatRegister(X86Encoding::xmm15, FloatRegisters::Double);
+
+// Vector registers fixed for use with some instructions, e.g. PBLENDVB.
+static constexpr FloatRegister vmm0 =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128);
+
+// X86-common synonyms.
+static constexpr Register eax = rax;
+static constexpr Register ebx = rbx;
+static constexpr Register ecx = rcx;
+static constexpr Register edx = rdx;
+static constexpr Register esi = rsi;
+static constexpr Register edi = rdi;
+static constexpr Register ebp = rbp;
+static constexpr Register esp = rsp;
+
+static constexpr Register InvalidReg{X86Encoding::invalid_reg};
+static constexpr FloatRegister InvalidFloatReg = FloatRegister();
+
+static constexpr Register StackPointer = rsp;
+static constexpr Register FramePointer = rbp;
+static constexpr Register JSReturnReg = rcx;
+// Avoid, except for assertions.
+static constexpr Register JSReturnReg_Type = JSReturnReg;
+static constexpr Register JSReturnReg_Data = JSReturnReg;
+
+static constexpr Register ScratchReg = r11;
+
+// Helper class for ScratchRegister usage. Asserts that only one piece
+// of code thinks it has exclusive ownership of the scratch register.
+struct ScratchRegisterScope : public AutoRegisterScope {
+ explicit ScratchRegisterScope(MacroAssembler& masm)
+ : AutoRegisterScope(masm, ScratchReg) {}
+};
+
+static constexpr Register ReturnReg = rax;
+static constexpr Register HeapReg = r15;
+static constexpr Register64 ReturnReg64(rax);
+static constexpr FloatRegister ReturnFloat32Reg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Single);
+static constexpr FloatRegister ReturnDoubleReg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+static constexpr FloatRegister ReturnSimd128Reg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128);
+static constexpr FloatRegister ScratchFloat32Reg_ =
+ FloatRegister(X86Encoding::xmm15, FloatRegisters::Single);
+static constexpr FloatRegister ScratchDoubleReg_ =
+ FloatRegister(X86Encoding::xmm15, FloatRegisters::Double);
+static constexpr FloatRegister ScratchSimd128Reg =
+ FloatRegister(X86Encoding::xmm15, FloatRegisters::Simd128);
+
+// Avoid rbp, which is the FramePointer, which is unavailable in some modes.
+static constexpr Register CallTempReg0 = rax;
+static constexpr Register CallTempReg1 = rdi;
+static constexpr Register CallTempReg2 = rbx;
+static constexpr Register CallTempReg3 = rcx;
+static constexpr Register CallTempReg4 = rsi;
+static constexpr Register CallTempReg5 = rdx;
+
+// Different argument registers for WIN64
+#if defined(_WIN64)
+static constexpr Register IntArgReg0 = rcx;
+static constexpr Register IntArgReg1 = rdx;
+static constexpr Register IntArgReg2 = r8;
+static constexpr Register IntArgReg3 = r9;
+static constexpr uint32_t NumIntArgRegs = 4;
+static constexpr Register IntArgRegs[NumIntArgRegs] = {rcx, rdx, r8, r9};
+
+static constexpr Register CallTempNonArgRegs[] = {rax, rdi, rbx, rsi};
+static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr FloatRegister FloatArgReg0 = xmm0;
+static constexpr FloatRegister FloatArgReg1 = xmm1;
+static constexpr FloatRegister FloatArgReg2 = xmm2;
+static constexpr FloatRegister FloatArgReg3 = xmm3;
+static constexpr uint32_t NumFloatArgRegs = 4;
+static constexpr FloatRegister FloatArgRegs[NumFloatArgRegs] = {xmm0, xmm1,
+ xmm2, xmm3};
+#else
+static constexpr Register IntArgReg0 = rdi;
+static constexpr Register IntArgReg1 = rsi;
+static constexpr Register IntArgReg2 = rdx;
+static constexpr Register IntArgReg3 = rcx;
+static constexpr Register IntArgReg4 = r8;
+static constexpr Register IntArgReg5 = r9;
+static constexpr uint32_t NumIntArgRegs = 6;
+static constexpr Register IntArgRegs[NumIntArgRegs] = {rdi, rsi, rdx,
+ rcx, r8, r9};
+
+static constexpr Register CallTempNonArgRegs[] = {rax, rbx};
+static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+static constexpr FloatRegister FloatArgReg0 = xmm0;
+static constexpr FloatRegister FloatArgReg1 = xmm1;
+static constexpr FloatRegister FloatArgReg2 = xmm2;
+static constexpr FloatRegister FloatArgReg3 = xmm3;
+static constexpr FloatRegister FloatArgReg4 = xmm4;
+static constexpr FloatRegister FloatArgReg5 = xmm5;
+static constexpr FloatRegister FloatArgReg6 = xmm6;
+static constexpr FloatRegister FloatArgReg7 = xmm7;
+static constexpr uint32_t NumFloatArgRegs = 8;
+static constexpr FloatRegister FloatArgRegs[NumFloatArgRegs] = {
+ xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7};
+#endif
+
+// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registerd used in RegExpTester instruction (do not use ReturnReg).
+static constexpr Register RegExpTesterRegExpReg = CallTempReg1;
+static constexpr Register RegExpTesterStringReg = CallTempReg2;
+static constexpr Register RegExpTesterLastIndexReg = CallTempReg3;
+
+class ABIArgGenerator {
+#if defined(XP_WIN)
+ unsigned regIndex_;
+#else
+ unsigned intRegIndex_;
+ unsigned floatRegIndex_;
+#endif
+ uint32_t stackOffset_;
+ ABIArg current_;
+
+ public:
+ ABIArgGenerator();
+ ABIArg next(MIRType argType);
+ ABIArg& current() { return current_; }
+ uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+ void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+};
+
+// These registers may be volatile or nonvolatile.
+// Avoid r11, which is the MacroAssembler's ScratchReg.
+static constexpr Register ABINonArgReg0 = rax;
+static constexpr Register ABINonArgReg1 = rbx;
+static constexpr Register ABINonArgReg2 = r10;
+static constexpr Register ABINonArgReg3 = r12;
+
+// This register may be volatile or nonvolatile. Avoid xmm15 which is the
+// ScratchDoubleReg.
+static constexpr FloatRegister ABINonArgDoubleReg =
+ FloatRegister(X86Encoding::xmm8, FloatRegisters::Double);
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = r10;
+static constexpr Register ABINonArgReturnReg1 = r12;
+static constexpr Register ABINonVolatileReg = r13;
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = r10;
+
+// Instance pointer argument register for WebAssembly functions. This must not
+// alias any other register used for passing function arguments or return
+// values. Preserved by WebAssembly functions.
+static constexpr Register InstanceReg = r14;
+
+// Registers used for asm.js/wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = rbx;
+
+static constexpr Register OsrFrameReg = IntArgReg3;
+
+static constexpr Register PreBarrierReg = rdx;
+
+static constexpr Register InterpreterPCReg = r14;
+
+static constexpr uint32_t ABIStackAlignment = 16;
+static constexpr uint32_t CodeAlignment = 16;
+static constexpr uint32_t JitStackAlignment = 16;
+
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+ JitStackValueAlignment >= 1,
+ "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+static constexpr uint32_t SimdMemoryAlignment = 16;
+
+static_assert(CodeAlignment % SimdMemoryAlignment == 0,
+ "Code alignment should be larger than any of the alignments "
+ "which are used for "
+ "the constant sections of the code buffer. Thus it should be "
+ "larger than the "
+ "alignment for SIMD constants.");
+
+static_assert(JitStackAlignment % SimdMemoryAlignment == 0,
+ "Stack alignment should be larger than any of the alignments "
+ "which are used for "
+ "spilled values. Thus it should be larger than the alignment "
+ "for SIMD accesses.");
+
+static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static constexpr uint32_t WasmTrapInstructionLength = 2;
+
+// See comments in wasm::GenerateFunctionPrologue. The difference between these
+// is the size of the largest callable prologue on the platform.
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+static constexpr Scale ScalePointer = TimesEight;
+
+} // namespace jit
+} // namespace js
+
+#include "jit/x86-shared/Assembler-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+// Return operand from a JS -> JS call.
+static constexpr ValueOperand JSReturnOperand = ValueOperand(JSReturnReg);
+
+class Assembler : public AssemblerX86Shared {
+ // x64 jumps may need extra bits of relocation, because a jump may extend
+ // beyond the signed 32-bit range. To account for this we add an extended
+ // jump table at the bottom of the instruction stream, and if a jump
+ // overflows its range, it will redirect here.
+ //
+ // Each entry in this table is a jmp [rip], followed by a ud2 to hint to the
+ // hardware branch predictor that there is no fallthrough, followed by the
+ // eight bytes containing an immediate address. This comes out to 16 bytes.
+ // +1 byte for opcode
+ // +1 byte for mod r/m
+ // +4 bytes for rip-relative offset (2)
+ // +2 bytes for ud2 instruction
+ // +8 bytes for 64-bit address
+ //
+ static const uint32_t SizeOfExtendedJump = 1 + 1 + 4 + 2 + 8;
+ static const uint32_t SizeOfJumpTableEntry = 16;
+
+ // Two kinds of jumps on x64:
+ //
+ // * codeJumps_ tracks jumps with target within the executable code region
+ // for the process. These jumps don't need entries in the extended jump
+ // table because source and target must be within 2 GB of each other.
+ //
+ // * extendedJumps_ tracks jumps with target outside the executable code
+ // region. These jumps need entries in the extended jump table described
+ // above.
+ using PendingJumpVector = Vector<RelativePatch, 8, SystemAllocPolicy>;
+ PendingJumpVector codeJumps_;
+ PendingJumpVector extendedJumps_;
+
+ uint32_t extendedJumpTable_;
+
+ static JitCode* CodeFromJump(JitCode* code, uint8_t* jump);
+
+ private:
+ void addPendingJump(JmpSrc src, ImmPtr target, RelocationKind reloc);
+
+ public:
+ using AssemblerX86Shared::j;
+ using AssemblerX86Shared::jmp;
+ using AssemblerX86Shared::pop;
+ using AssemblerX86Shared::push;
+ using AssemblerX86Shared::vmovq;
+
+ Assembler() : extendedJumpTable_(0) {}
+
+ static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ // The buffer is about to be linked, make sure any constant pools or excess
+ // bookkeeping has been flushed to the instruction stream.
+ void finish();
+
+ // Copy the assembly code to the given buffer, and perform any pending
+ // relocations relying on the target address.
+ void executableCopy(uint8_t* buffer);
+
+ void assertNoGCThings() const {
+#ifdef DEBUG
+ MOZ_ASSERT(dataRelocations_.length() == 0);
+ for (auto& j : codeJumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+ for (auto& j : extendedJumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+#endif
+ }
+
+ // Actual assembly emitting functions.
+
+ void push(const ImmGCPtr ptr) {
+ movq(ptr, ScratchReg);
+ push(ScratchReg);
+ }
+ void push(const ImmWord ptr) {
+ // We often end up with ImmWords that actually fit into int32.
+ // Be aware of the sign extension behavior.
+ if (ptr.value <= INT32_MAX) {
+ push(Imm32(ptr.value));
+ } else {
+ movq(ptr, ScratchReg);
+ push(ScratchReg);
+ }
+ }
+ void push(ImmPtr imm) { push(ImmWord(uintptr_t(imm.value))); }
+ void push(FloatRegister src) {
+ subq(Imm32(sizeof(double)), StackPointer);
+ vmovsd(src, Address(StackPointer, 0));
+ }
+ CodeOffset pushWithPatch(ImmWord word) {
+ CodeOffset label = movWithPatch(word, ScratchReg);
+ push(ScratchReg);
+ return label;
+ }
+
+ void pop(FloatRegister src) {
+ vmovsd(Address(StackPointer, 0), src);
+ addq(Imm32(sizeof(double)), StackPointer);
+ }
+
+ CodeOffset movWithPatch(ImmWord word, Register dest) {
+ masm.movq_i64r(word.value, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ return movWithPatch(ImmWord(uintptr_t(imm.value)), dest);
+ }
+
+ // This is for patching during code generation, not after.
+ void patchAddq(CodeOffset offset, int32_t n) {
+ unsigned char* code = masm.data();
+ X86Encoding::SetInt32(code + offset.offset(), n);
+ }
+
+ // Load an ImmWord value into a register. Note that this instruction will
+ // attempt to optimize its immediate field size. When a full 64-bit
+ // immediate is needed for a relocation, use movWithPatch.
+ void movq(ImmWord word, Register dest) {
+ // Load a 64-bit immediate into a register. If the value falls into
+ // certain ranges, we can use specialized instructions which have
+ // smaller encodings.
+ if (word.value <= UINT32_MAX) {
+ // movl has a 32-bit unsigned (effectively) immediate field.
+ masm.movl_i32r((uint32_t)word.value, dest.encoding());
+ } else if ((intptr_t)word.value >= INT32_MIN &&
+ (intptr_t)word.value <= INT32_MAX) {
+ // movq has a 32-bit signed immediate field.
+ masm.movq_i32r((int32_t)(intptr_t)word.value, dest.encoding());
+ } else {
+ // Otherwise use movabs.
+ masm.movq_i64r(word.value, dest.encoding());
+ }
+ }
+ void movq(ImmPtr imm, Register dest) {
+ movq(ImmWord(uintptr_t(imm.value)), dest);
+ }
+ void movq(ImmGCPtr ptr, Register dest) {
+ masm.movq_i64r(uintptr_t(ptr.value), dest.encoding());
+ writeDataRelocation(ptr);
+ }
+ void movq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.movq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movq_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movq(Imm32 imm32, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.movl_i32r(imm32.value, dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movq_i32m(imm32.value, dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movq_i32m(imm32.value, dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movq_i32m(imm32.value, dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovq(Register src, FloatRegister dest) {
+ masm.vmovq_rr(src.encoding(), dest.encoding());
+ }
+ void vmovq(FloatRegister src, Register dest) {
+ masm.vmovq_rr(src.encoding(), dest.encoding());
+ }
+ void movq(Register src, Register dest) {
+ masm.movq_rr(src.encoding(), dest.encoding());
+ }
+
+ void cmovCCq(Condition cond, const Operand& src, Register dest) {
+ X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.cmovCCq_rr(cc, src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmovCCq_mr(cc, src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmovCCq_mr(cc, src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmovCCq(Condition cond, Register src, Register dest) {
+ X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
+ masm.cmovCCq_rr(cc, src.encoding(), dest.encoding());
+ }
+
+ void cmovzq(const Operand& src, Register dest) {
+ cmovCCq(Condition::Zero, src, dest);
+ }
+ void cmovnzq(const Operand& src, Register dest) {
+ cmovCCq(Condition::NonZero, src, dest);
+ }
+
+ template <typename T>
+ void lock_addq(T src, const Operand& op) {
+ masm.prefix_lock();
+ addq(src, op);
+ }
+ template <typename T>
+ void lock_subq(T src, const Operand& op) {
+ masm.prefix_lock();
+ subq(src, op);
+ }
+ template <typename T>
+ void lock_andq(T src, const Operand& op) {
+ masm.prefix_lock();
+ andq(src, op);
+ }
+ template <typename T>
+ void lock_orq(T src, const Operand& op) {
+ masm.prefix_lock();
+ orq(src, op);
+ }
+ template <typename T>
+ void lock_xorq(T src, const Operand& op) {
+ masm.prefix_lock();
+ xorq(src, op);
+ }
+
+ void lock_cmpxchgq(Register src, const Operand& mem) {
+ masm.prefix_lock();
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.cmpxchgq(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpxchgq(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void xchgq(Register src, Register dest) {
+ masm.xchgq_rr(src.encoding(), dest.encoding());
+ }
+
+ void xchgq(Register src, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.xchgq_rm(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xchgq_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void lock_xaddq(Register srcdest, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.lock_xaddq_rm(srcdest.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.lock_xaddq_rm(srcdest.encoding(), mem.disp(), mem.base(),
+ mem.index(), mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void movsbq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movsbq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movsbq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movsbq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void movzbq(const Operand& src, Register dest) {
+ // movzbl zero-extends to 64 bits and is one byte smaller, so use that
+ // instead.
+ movzbl(src, dest);
+ }
+
+ void movswq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movswq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movswq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movswq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void movzwq(const Operand& src, Register dest) {
+ // movzwl zero-extends to 64 bits and is one byte smaller, so use that
+ // instead.
+ movzwl(src, dest);
+ }
+
+ void movslq(Register src, Register dest) {
+ masm.movslq_rr(src.encoding(), dest.encoding());
+ }
+ void movslq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movslq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movslq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movslq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void andq(Register src, Register dest) {
+ masm.andq_rr(src.encoding(), dest.encoding());
+ }
+ void andq(Imm32 imm, Register dest) {
+ masm.andq_ir(imm.value, dest.encoding());
+ }
+ void andq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.andq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.andq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void andq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.andq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void addq(Imm32 imm, Register dest) {
+ masm.addq_ir(imm.value, dest.encoding());
+ }
+ CodeOffset addqWithPatch(Imm32 imm, Register dest) {
+ masm.addq_i32r(imm.value, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ void addq(Imm32 imm, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.addq_ir(imm.value, dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addq_im(imm.value, dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.addq_im(imm.value, dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void addq(Register src, Register dest) {
+ masm.addq_rr(src.encoding(), dest.encoding());
+ }
+ void addq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.addq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.addq_mr(src.address(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void addq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.addq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void subq(Imm32 imm, Register dest) {
+ masm.subq_ir(imm.value, dest.encoding());
+ }
+ void subq(Register src, Register dest) {
+ masm.subq_rr(src.encoding(), dest.encoding());
+ }
+ void subq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.subq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.subq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void subq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.subq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.subq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void shlq(Imm32 imm, Register dest) {
+ masm.shlq_ir(imm.value, dest.encoding());
+ }
+ void shrq(Imm32 imm, Register dest) {
+ masm.shrq_ir(imm.value, dest.encoding());
+ }
+ void sarq(Imm32 imm, Register dest) {
+ masm.sarq_ir(imm.value, dest.encoding());
+ }
+ void shlq_cl(Register dest) { masm.shlq_CLr(dest.encoding()); }
+ void shrq_cl(Register dest) { masm.shrq_CLr(dest.encoding()); }
+ void sarq_cl(Register dest) { masm.sarq_CLr(dest.encoding()); }
+ void sarxq(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.sarxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+ void shlxq(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.shlxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+ void shrxq(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.shrxq_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+ void rolq(Imm32 imm, Register dest) {
+ masm.rolq_ir(imm.value, dest.encoding());
+ }
+ void rolq_cl(Register dest) { masm.rolq_CLr(dest.encoding()); }
+ void rorq(Imm32 imm, Register dest) {
+ masm.rorq_ir(imm.value, dest.encoding());
+ }
+ void rorq_cl(Register dest) { masm.rorq_CLr(dest.encoding()); }
+ void orq(Imm32 imm, Register dest) {
+ masm.orq_ir(imm.value, dest.encoding());
+ }
+ void orq(Register src, Register dest) {
+ masm.orq_rr(src.encoding(), dest.encoding());
+ }
+ void orq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.orq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.orq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void orq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.orq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.orq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorq(Register src, Register dest) {
+ masm.xorq_rr(src.encoding(), dest.encoding());
+ }
+ void xorq(Imm32 imm, Register dest) {
+ masm.xorq_ir(imm.value, dest.encoding());
+ }
+ void xorq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.xorq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.xorq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorq(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.xorq_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void bsrq(const Register& src, const Register& dest) {
+ masm.bsrq_rr(src.encoding(), dest.encoding());
+ }
+ void bsfq(const Register& src, const Register& dest) {
+ masm.bsfq_rr(src.encoding(), dest.encoding());
+ }
+ void bswapq(const Register& reg) { masm.bswapq_r(reg.encoding()); }
+ void lzcntq(const Register& src, const Register& dest) {
+ masm.lzcntq_rr(src.encoding(), dest.encoding());
+ }
+ void tzcntq(const Register& src, const Register& dest) {
+ masm.tzcntq_rr(src.encoding(), dest.encoding());
+ }
+ void popcntq(const Register& src, const Register& dest) {
+ masm.popcntq_rr(src.encoding(), dest.encoding());
+ }
+
+ void imulq(Imm32 imm, Register src, Register dest) {
+ masm.imulq_ir(imm.value, src.encoding(), dest.encoding());
+ }
+ void imulq(Register src, Register dest) {
+ masm.imulq_rr(src.encoding(), dest.encoding());
+ }
+ void imulq(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.imulq_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.imulq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ MOZ_CRASH("NYI");
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void cqo() { masm.cqo(); }
+ void idivq(Register divisor) { masm.idivq_r(divisor.encoding()); }
+ void udivq(Register divisor) { masm.divq_r(divisor.encoding()); }
+
+ void vcvtsi2sdq(Register src, FloatRegister dest) {
+ masm.vcvtsi2sdq_rr(src.encoding(), dest.encoding());
+ }
+
+ void vpextrq(unsigned lane, FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpextrq_irr(lane, src.encoding(), dest.encoding());
+ }
+
+ void vpinsrq(unsigned lane, Register src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpinsrq_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
+ }
+
+ void negq(Register reg) { masm.negq_r(reg.encoding()); }
+
+ void notq(Register reg) { masm.notq_r(reg.encoding()); }
+
+ void mov(ImmWord word, Register dest) {
+ // Use xor for setting registers to zero, as it is specially optimized
+ // for this purpose on modern hardware. Note that it does clobber FLAGS
+ // though. Use xorl instead of xorq since they are functionally
+ // equivalent (32-bit instructions zero-extend their results to 64 bits)
+ // and xorl has a smaller encoding.
+ if (word.value == 0) {
+ xorl(dest, dest);
+ } else {
+ movq(word, dest);
+ }
+ }
+ void mov(ImmPtr imm, Register dest) { movq(imm, dest); }
+ void mov(wasm::SymbolicAddress imm, Register dest) {
+ masm.movq_i64r(-1, dest.encoding());
+ append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), imm));
+ }
+ void mov(const Operand& src, Register dest) { movq(src, dest); }
+ void mov(Register src, const Operand& dest) { movq(src, dest); }
+ void mov(Imm32 imm32, const Operand& dest) { movq(imm32, dest); }
+ void mov(Register src, Register dest) { movq(src, dest); }
+ void mov(CodeLabel* label, Register dest) {
+ masm.movq_i64r(/* placeholder */ 0, dest.encoding());
+ label->patchAt()->bind(masm.size());
+ }
+ void xchg(Register src, Register dest) { xchgq(src, dest); }
+
+ void lea(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.leaq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.leaq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexepcted operand kind");
+ }
+ }
+
+ void cmovz32(const Operand& src, Register dest) { return cmovzl(src, dest); }
+ void cmovzPtr(const Operand& src, Register dest) { return cmovzq(src, dest); }
+
+ CodeOffset loadRipRelativeInt32(Register dest) {
+ return CodeOffset(masm.movl_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeInt64(Register dest) {
+ return CodeOffset(masm.movq_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeDouble(FloatRegister dest) {
+ return CodeOffset(masm.vmovsd_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeFloat32(FloatRegister dest) {
+ return CodeOffset(masm.vmovss_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeInt32x4(FloatRegister dest) {
+ return CodeOffset(masm.vmovdqa_ripr(dest.encoding()).offset());
+ }
+ CodeOffset loadRipRelativeFloat32x4(FloatRegister dest) {
+ return CodeOffset(masm.vmovaps_ripr(dest.encoding()).offset());
+ }
+ CodeOffset leaRipRelative(Register dest) {
+ return CodeOffset(masm.leaq_rip(dest.encoding()).offset());
+ }
+
+ void cmpq(Register rhs, Register lhs) {
+ masm.cmpq_rr(rhs.encoding(), lhs.encoding());
+ }
+ void cmpq(Register rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpq_rr(rhs.encoding(), lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpq_rm(rhs.encoding(), lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpq_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpq_rm(rhs.encoding(), lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpq(Imm32 rhs, Register lhs) {
+ masm.cmpq_ir(rhs.value, lhs.encoding());
+ }
+ void cmpq(Imm32 rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpq_ir(rhs.value, lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpq_im(rhs.value, lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpq_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpq_im(rhs.value, lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpq(const Operand& rhs, Register lhs) {
+ switch (rhs.kind()) {
+ case Operand::REG:
+ masm.cmpq_rr(rhs.reg(), lhs.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpq_mr(rhs.disp(), rhs.base(), lhs.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void testq(Imm32 rhs, Register lhs) {
+ masm.testq_ir(rhs.value, lhs.encoding());
+ }
+ void testq(Register rhs, Register lhs) {
+ masm.testq_rr(rhs.encoding(), lhs.encoding());
+ }
+ void testq(Imm32 rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.testq_ir(rhs.value, lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.testq_i32m(rhs.value, lhs.disp(), lhs.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+
+ void jmp(ImmPtr target, RelocationKind reloc = RelocationKind::HARDCODED) {
+ MOZ_ASSERT(hasCreator());
+ JmpSrc src = masm.jmp();
+ addPendingJump(src, target, reloc);
+ }
+ void j(Condition cond, ImmPtr target,
+ RelocationKind reloc = RelocationKind::HARDCODED) {
+ JmpSrc src = masm.jCC(static_cast<X86Encoding::Condition>(cond));
+ addPendingJump(src, target, reloc);
+ }
+
+ void jmp(JitCode* target) {
+ jmp(ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void j(Condition cond, JitCode* target) {
+ j(cond, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void call(JitCode* target) {
+ JmpSrc src = masm.call();
+ addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void call(ImmWord target) { call(ImmPtr((void*)target.value)); }
+ void call(ImmPtr target) {
+ JmpSrc src = masm.call();
+ addPendingJump(src, target, RelocationKind::HARDCODED);
+ }
+
+ // Emit a CALL or CMP (nop) instruction. ToggleCall can be used to patch
+ // this instruction.
+ CodeOffset toggledCall(JitCode* target, bool enabled) {
+ CodeOffset offset(size());
+ JmpSrc src = enabled ? masm.call() : masm.cmp_eax();
+ addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ MOZ_ASSERT_IF(!oom(), size() - offset.offset() == ToggledCallSize(nullptr));
+ return offset;
+ }
+
+ static size_t ToggledCallSize(uint8_t* code) {
+ // Size of a call instruction.
+ return 5;
+ }
+
+ // Do not mask shared implementations.
+ using AssemblerX86Shared::call;
+
+ void vcvttsd2sq(FloatRegister src, Register dest) {
+ masm.vcvttsd2sq_rr(src.encoding(), dest.encoding());
+ }
+ void vcvttss2sq(FloatRegister src, Register dest) {
+ masm.vcvttss2sq_rr(src.encoding(), dest.encoding());
+ }
+ void vcvtsq2sd(Register src1, FloatRegister src0, FloatRegister dest) {
+ masm.vcvtsq2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vcvtsq2ss(Register src1, FloatRegister src0, FloatRegister dest) {
+ masm.vcvtsq2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+};
+
+static inline bool GetIntArgReg(uint32_t intArg, uint32_t floatArg,
+ Register* out) {
+#if defined(_WIN64)
+ uint32_t arg = intArg + floatArg;
+#else
+ uint32_t arg = intArg;
+#endif
+ if (arg >= NumIntArgRegs) {
+ return false;
+ }
+ *out = IntArgRegs[arg];
+ return true;
+}
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs, Register* out) {
+ if (GetIntArgReg(usedIntArgs, usedFloatArgs, out)) {
+ return true;
+ }
+ // Unfortunately, we have to assume things about the point at which
+ // GetIntArgReg returns false, because we need to know how many registers it
+ // can allocate.
+#if defined(_WIN64)
+ uint32_t arg = usedIntArgs + usedFloatArgs;
+#else
+ uint32_t arg = usedIntArgs;
+#endif
+ arg -= NumIntArgRegs;
+ if (arg >= NumCallTempNonArgRegs) {
+ return false;
+ }
+ *out = CallTempNonArgRegs[arg];
+ return true;
+}
+
+static inline bool GetFloatArgReg(uint32_t intArg, uint32_t floatArg,
+ FloatRegister* out) {
+#if defined(_WIN64)
+ uint32_t arg = intArg + floatArg;
+#else
+ uint32_t arg = floatArg;
+#endif
+ if (floatArg >= NumFloatArgRegs) {
+ return false;
+ }
+ *out = FloatArgRegs[arg];
+ return true;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_Assembler_x64_h */
diff --git a/js/src/jit/x64/BaseAssembler-x64.h b/js/src/jit/x64/BaseAssembler-x64.h
new file mode 100644
index 0000000000..85cac32ad7
--- /dev/null
+++ b/js/src/jit/x64/BaseAssembler-x64.h
@@ -0,0 +1,1373 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_BaseAssembler_x64_h
+#define jit_x64_BaseAssembler_x64_h
+
+#include "jit/x86-shared/BaseAssembler-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+namespace X86Encoding {
+
+class BaseAssemblerX64 : public BaseAssembler {
+ public:
+ // Arithmetic operations:
+
+ void addq_rr(RegisterID src, RegisterID dst) {
+ spew("addq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_ADD_GvEv, src, dst);
+ }
+
+ void addq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("addq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_ADD_GvEv, offset, base, dst);
+ }
+
+ void addq_mr(const void* addr, RegisterID dst) {
+ spew("addq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_ADD_GvEv, addr, dst);
+ }
+
+ void addq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("addq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_ADD_GvEv, offset, base, index, scale, dst);
+ }
+
+ void addq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("addq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_ADD_EvGv, offset, base, src);
+ }
+
+ void addq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("addq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_ADD_EvGv, offset, base, index, scale, src);
+ }
+
+ void addq_ir(int32_t imm, RegisterID dst) {
+ spew("addq $%d, %s", imm, GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_ADD_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void addq_i32r(int32_t imm, RegisterID dst) {
+ // 32-bit immediate always, for patching.
+ spew("addq $0x%04x, %s", uint32_t(imm), GPReg64Name(dst));
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_ADD_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+ }
+ m_formatter.immediate32(imm);
+ }
+
+ void addq_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("addq $%d, " MEM_ob, imm, ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, GROUP1_OP_ADD);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void addq_im(int32_t imm, const void* addr) {
+ spew("addq $%d, %p", imm, addr);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, addr, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, addr, GROUP1_OP_ADD);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void andq_rr(RegisterID src, RegisterID dst) {
+ spew("andq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_AND_GvEv, src, dst);
+ }
+
+ void andq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("andq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_AND_GvEv, offset, base, dst);
+ }
+
+ void andq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("andq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_AND_GvEv, offset, base, index, scale, dst);
+ }
+
+ void andq_mr(const void* addr, RegisterID dst) {
+ spew("andq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_AND_GvEv, addr, dst);
+ }
+
+ void andq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("andq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_AND_EvGv, offset, base, src);
+ }
+
+ void andq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("andq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_AND_EvGv, offset, base, index, scale, src);
+ }
+
+ void orq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("orq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_OR_GvEv, offset, base, dst);
+ }
+
+ void orq_mr(const void* addr, RegisterID dst) {
+ spew("orq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_OR_GvEv, addr, dst);
+ }
+
+ void orq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("orq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_OR_EvGv, offset, base, src);
+ }
+
+ void orq_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("orq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_OR_EvGv, offset, base, index, scale, src);
+ }
+
+ void xorq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("xorq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XOR_GvEv, offset, base, dst);
+ }
+
+ void xorq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("xorq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XOR_GvEv, offset, base, index, scale, dst);
+ }
+
+ void xorq_mr(const void* addr, RegisterID dst) {
+ spew("xorq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XOR_GvEv, addr, dst);
+ }
+
+ void xorq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xorq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_XOR_EvGv, offset, base, src);
+ }
+
+ void xorq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xorq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_XOR_EvGv, offset, base, index, scale, src);
+ }
+
+ void bswapq_r(RegisterID dst) {
+ spew("bswapq %s", GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_BSWAP, dst);
+ }
+
+ void bsrq_rr(RegisterID src, RegisterID dst) {
+ spew("bsrq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_BSR_GvEv, src, dst);
+ }
+
+ void bsfq_rr(RegisterID src, RegisterID dst) {
+ spew("bsfq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_BSF_GvEv, src, dst);
+ }
+
+ void lzcntq_rr(RegisterID src, RegisterID dst) {
+ spew("lzcntq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp64(OP2_LZCNT_GvEv, src, dst);
+ }
+
+ void tzcntq_rr(RegisterID src, RegisterID dst) {
+ spew("tzcntq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp64(OP2_TZCNT_GvEv, src, dst);
+ }
+
+ void popcntq_rr(RegisterID src, RegisterID dst) {
+ spew("popcntq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp64(OP2_POPCNT_GvEv, src, dst);
+ }
+
+ void andq_ir(int32_t imm, RegisterID dst) {
+ spew("andq $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_AND_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_AND);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void negq_r(RegisterID dst) {
+ spew("negq %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP3_Ev, dst, GROUP3_OP_NEG);
+ }
+
+ void orq_rr(RegisterID src, RegisterID dst) {
+ spew("orq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_OR_GvEv, src, dst);
+ }
+
+ void orq_ir(int32_t imm, RegisterID dst) {
+ spew("orq $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_OR_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_OR);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void notq_r(RegisterID dst) {
+ spew("notq %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP3_Ev, dst, GROUP3_OP_NOT);
+ }
+
+ void subq_rr(RegisterID src, RegisterID dst) {
+ spew("subq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_SUB_GvEv, src, dst);
+ }
+
+ void subq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("subq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_SUB_EvGv, offset, base, src);
+ }
+
+ void subq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("subq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_SUB_EvGv, offset, base, index, scale, src);
+ }
+
+ void subq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("subq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_SUB_GvEv, offset, base, dst);
+ }
+
+ void subq_mr(const void* addr, RegisterID dst) {
+ spew("subq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_SUB_GvEv, addr, dst);
+ }
+
+ void subq_ir(int32_t imm, RegisterID dst) {
+ spew("subq $%d, %s", imm, GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_SUB_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_SUB);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void xorq_rr(RegisterID src, RegisterID dst) {
+ spew("xorq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XOR_GvEv, src, dst);
+ }
+
+ void xorq_ir(int32_t imm, RegisterID dst) {
+ spew("xorq $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, dst, GROUP1_OP_XOR);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp64(OP_XOR_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, dst, GROUP1_OP_XOR);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void sarq_CLr(RegisterID dst) {
+ spew("sarq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SAR);
+ }
+
+ void shlq_CLr(RegisterID dst) {
+ spew("shlq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SHL);
+ }
+
+ void shrq_CLr(RegisterID dst) {
+ spew("shrq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_SHR);
+ }
+
+ void sarq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("sarq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SAR);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SAR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void shlq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("shlq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SHL);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SHL);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void shrq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("shrq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_SHR);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_SHR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void rolq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("rolq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_ROL);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_ROL);
+ m_formatter.immediate8u(imm);
+ }
+ }
+ void rolq_CLr(RegisterID dst) {
+ spew("rolq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_ROL);
+ }
+
+ void rorq_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 64);
+ spew("rorq $%d, %s", imm, GPReg64Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp64(OP_GROUP2_Ev1, dst, GROUP2_OP_ROR);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP2_EvIb, dst, GROUP2_OP_ROR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+ void rorq_CLr(RegisterID dst) {
+ spew("rorq %%cl, %s", GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_GROUP2_EvCL, dst, GROUP2_OP_ROR);
+ }
+
+ void imulq_rr(RegisterID src, RegisterID dst) {
+ spew("imulq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_IMUL_GvEv, src, dst);
+ }
+
+ void imulq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("imulq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_IMUL_GvEv, offset, base, dst);
+ }
+
+ void imulq_ir(int32_t value, RegisterID src, RegisterID dst) {
+ spew("imulq $%d, %s, %s", value, GPReg64Name(src), GPReg64Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(value)) {
+ m_formatter.oneByteOp64(OP_IMUL_GvEvIb, src, dst);
+ m_formatter.immediate8s(value);
+ } else {
+ m_formatter.oneByteOp64(OP_IMUL_GvEvIz, src, dst);
+ m_formatter.immediate32(value);
+ }
+ }
+
+ void cqo() {
+ spew("cqo ");
+ m_formatter.oneByteOp64(OP_CDQ);
+ }
+
+ void idivq_r(RegisterID divisor) {
+ spew("idivq %s", GPReg64Name(divisor));
+ m_formatter.oneByteOp64(OP_GROUP3_Ev, divisor, GROUP3_OP_IDIV);
+ }
+
+ void divq_r(RegisterID divisor) {
+ spew("divq %s", GPReg64Name(divisor));
+ m_formatter.oneByteOp64(OP_GROUP3_Ev, divisor, GROUP3_OP_DIV);
+ }
+
+ // Comparisons:
+
+ void cmpq_rr(RegisterID rhs, RegisterID lhs) {
+ spew("cmpq %s, %s", GPReg64Name(rhs), GPReg64Name(lhs));
+ m_formatter.oneByteOp64(OP_CMP_GvEv, rhs, lhs);
+ }
+
+ void cmpq_rm(RegisterID rhs, int32_t offset, RegisterID base) {
+ spew("cmpq %s, " MEM_ob, GPReg64Name(rhs), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_CMP_EvGv, offset, base, rhs);
+ }
+
+ void cmpq_rm(RegisterID rhs, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpq %s, " MEM_obs, GPReg64Name(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_CMP_EvGv, offset, base, index, scale, rhs);
+ }
+
+ void cmpq_mr(int32_t offset, RegisterID base, RegisterID lhs) {
+ spew("cmpq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(lhs));
+ m_formatter.oneByteOp64(OP_CMP_GvEv, offset, base, lhs);
+ }
+
+ void cmpq_ir(int32_t rhs, RegisterID lhs) {
+ if (rhs == 0) {
+ testq_rr(lhs, lhs);
+ return;
+ }
+
+ spew("cmpq $0x%" PRIx64 ", %s", uint64_t(rhs), GPReg64Name(lhs));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, lhs, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ if (lhs == rax) {
+ m_formatter.oneByteOp64(OP_CMP_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, lhs, GROUP1_OP_CMP);
+ }
+ m_formatter.immediate32(rhs);
+ }
+ }
+
+ void cmpq_im(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("cmpq $0x%" PRIx64 ", " MEM_ob, uint64_t(rhs),
+ ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+
+ void cmpq_im(int32_t rhs, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("cmpq $0x%x, " MEM_obs, uint32_t(rhs),
+ ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+ void cmpq_im(int32_t rhs, const void* addr) {
+ spew("cmpq $0x%" PRIx64 ", %p", uint64_t(rhs), addr);
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIb, addr, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP1_EvIz, addr, GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+ void cmpq_rm(RegisterID rhs, const void* addr) {
+ spew("cmpq %s, %p", GPReg64Name(rhs), addr);
+ m_formatter.oneByteOp64(OP_CMP_EvGv, addr, rhs);
+ }
+
+ void testq_rr(RegisterID rhs, RegisterID lhs) {
+ spew("testq %s, %s", GPReg64Name(rhs), GPReg64Name(lhs));
+ m_formatter.oneByteOp64(OP_TEST_EvGv, lhs, rhs);
+ }
+
+ void testq_ir(int32_t rhs, RegisterID lhs) {
+ // If the mask fits in a 32-bit immediate, we can use testl with a
+ // 32-bit subreg.
+ if (CAN_ZERO_EXTEND_32_64(rhs)) {
+ testl_ir(rhs, lhs);
+ return;
+ }
+ spew("testq $0x%" PRIx64 ", %s", uint64_t(rhs), GPReg64Name(lhs));
+ if (lhs == rax) {
+ m_formatter.oneByteOp64(OP_TEST_EAXIv);
+ } else {
+ m_formatter.oneByteOp64(OP_GROUP3_EvIz, lhs, GROUP3_OP_TEST);
+ }
+ m_formatter.immediate32(rhs);
+ }
+
+ void testq_i32m(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("testq $0x%" PRIx64 ", " MEM_ob, uint64_t(rhs),
+ ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_GROUP3_EvIz, offset, base, GROUP3_OP_TEST);
+ m_formatter.immediate32(rhs);
+ }
+
+ void testq_i32m(int32_t rhs, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("testq $0x%4x, " MEM_obs, uint32_t(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_GROUP3_EvIz, offset, base, index, scale,
+ GROUP3_OP_TEST);
+ m_formatter.immediate32(rhs);
+ }
+
+ // Various move ops:
+
+ void cmovCCq_rr(Condition cond, RegisterID src, RegisterID dst) {
+ spew("cmov%s %s, %s", CCName(cond), GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(cmovccOpcode(cond), src, dst);
+ }
+ void cmovCCq_mr(Condition cond, int32_t offset, RegisterID base,
+ RegisterID dst) {
+ spew("cmov%s " MEM_ob ", %s", CCName(cond), ADDR_ob(offset, base),
+ GPReg64Name(dst));
+ m_formatter.twoByteOp64(cmovccOpcode(cond), offset, base, dst);
+ }
+ void cmovCCq_mr(Condition cond, int32_t offset, RegisterID base,
+ RegisterID index, int scale, RegisterID dst) {
+ spew("cmov%s " MEM_obs ", %s", CCName(cond),
+ ADDR_obs(offset, base, index, scale), GPReg64Name(dst));
+ m_formatter.twoByteOp64(cmovccOpcode(cond), offset, base, index, scale,
+ dst);
+ }
+
+ void cmpxchgq(RegisterID src, int32_t offset, RegisterID base) {
+ spew("cmpxchgq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.twoByteOp64(OP2_CMPXCHG_GvEw, offset, base, src);
+ }
+
+ void cmpxchgq(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpxchgq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.twoByteOp64(OP2_CMPXCHG_GvEw, offset, base, index, scale, src);
+ }
+
+ void lock_xaddq_rm(RegisterID srcdest, int32_t offset, RegisterID base) {
+ spew("lock xaddq %s, " MEM_ob, GPReg64Name(srcdest), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(PRE_LOCK);
+ m_formatter.twoByteOp64(OP2_XADD_EvGv, offset, base, srcdest);
+ }
+
+ void lock_xaddq_rm(RegisterID srcdest, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("lock xaddq %s, " MEM_obs, GPReg64Name(srcdest),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(PRE_LOCK);
+ m_formatter.twoByteOp64(OP2_XADD_EvGv, offset, base, index, scale, srcdest);
+ }
+
+ void xchgq_rr(RegisterID src, RegisterID dst) {
+ spew("xchgq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_XCHG_GvEv, src, dst);
+ }
+ void xchgq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xchgq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_XCHG_GvEv, offset, base, src);
+ }
+ void xchgq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xchgq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_XCHG_GvEv, offset, base, index, scale, src);
+ }
+
+ void movq_rr(RegisterID src, RegisterID dst) {
+ spew("movq %s, %s", GPReg64Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_GvEv, src, dst);
+ }
+
+ void movq_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movq %s, " MEM_ob, GPReg64Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_MOV_EvGv, offset, base, src);
+ }
+
+ void movq_rm_disp32(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movq %s, " MEM_o32b, GPReg64Name(src), ADDR_o32b(offset, base));
+ m_formatter.oneByteOp64_disp32(OP_MOV_EvGv, offset, base, src);
+ }
+
+ void movq_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("movq %s, " MEM_obs, GPReg64Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_MOV_EvGv, offset, base, index, scale, src);
+ }
+
+ void movq_rm(RegisterID src, const void* addr) {
+ if (src == rax && !IsAddressImmediate(addr)) {
+ movq_EAXm(addr);
+ return;
+ }
+
+ spew("movq %s, %p", GPReg64Name(src), addr);
+ m_formatter.oneByteOp64(OP_MOV_EvGv, addr, src);
+ }
+
+ void movq_mEAX(const void* addr) {
+ if (IsAddressImmediate(addr)) {
+ movq_mr(addr, rax);
+ return;
+ }
+
+ spew("movq %p, %%rax", addr);
+ m_formatter.oneByteOp64(OP_MOV_EAXOv);
+ m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+ }
+
+ void movq_EAXm(const void* addr) {
+ if (IsAddressImmediate(addr)) {
+ movq_rm(rax, addr);
+ return;
+ }
+
+ spew("movq %%rax, %p", addr);
+ m_formatter.oneByteOp64(OP_MOV_OvEAX);
+ m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+ }
+
+ void movq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_GvEv, offset, base, dst);
+ }
+
+ void movq_mr_disp32(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movq " MEM_o32b ", %s", ADDR_o32b(offset, base),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64_disp32(OP_MOV_GvEv, offset, base, dst);
+ }
+
+ void movq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_GvEv, offset, base, index, scale, dst);
+ }
+
+ void movq_mr(const void* addr, RegisterID dst) {
+ if (dst == rax && !IsAddressImmediate(addr)) {
+ movq_mEAX(addr);
+ return;
+ }
+
+ spew("movq %p, %s", addr, GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_GvEv, addr, dst);
+ }
+
+ void leaq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("leaq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_LEA, offset, base, index, scale, dst);
+ }
+
+ void movq_i32m(int32_t imm, int32_t offset, RegisterID base) {
+ spew("movq $%d, " MEM_ob, imm, ADDR_ob(offset, base));
+ m_formatter.oneByteOp64(OP_GROUP11_EvIz, offset, base, GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+ void movq_i32m(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("movq $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp64(OP_GROUP11_EvIz, offset, base, index, scale,
+ GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+ void movq_i32m(int32_t imm, const void* addr) {
+ spew("movq $%d, %p", imm, addr);
+ m_formatter.oneByteOp64(OP_GROUP11_EvIz, addr, GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+
+ // Note that this instruction sign-extends its 32-bit immediate field to 64
+ // bits and loads the 64-bit value into a 64-bit register.
+ //
+ // Note also that this is similar to the movl_i32r instruction, except that
+ // movl_i32r *zero*-extends its 32-bit immediate, and it has smaller code
+ // size, so it's preferred for values which could use either.
+ void movq_i32r(int32_t imm, RegisterID dst) {
+ spew("movq $%d, %s", imm, GPRegName(dst));
+ m_formatter.oneByteOp64(OP_GROUP11_EvIz, dst, GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+
+ void movq_i64r(int64_t imm, RegisterID dst) {
+ spew("movabsq $0x%" PRIx64 ", %s", uint64_t(imm), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOV_EAXIv, dst);
+ m_formatter.immediate64(imm);
+ }
+
+ void movsbq_rr(RegisterID src, RegisterID dst) {
+ spew("movsbq %s, %s", GPReg32Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEb, src, dst);
+ }
+ void movsbq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movsbq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEb, offset, base, dst);
+ }
+ void movsbq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movsbq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEb, offset, base, index, scale, dst);
+ }
+
+ void movswq_rr(RegisterID src, RegisterID dst) {
+ spew("movswq %s, %s", GPReg32Name(src), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEw, src, dst);
+ }
+ void movswq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movswq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEw, offset, base, dst);
+ }
+ void movswq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movswq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.twoByteOp64(OP2_MOVSX_GvEw, offset, base, index, scale, dst);
+ }
+
+ void movslq_rr(RegisterID src, RegisterID dst) {
+ spew("movslq %s, %s", GPReg32Name(src), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOVSXD_GvEv, src, dst);
+ }
+ void movslq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movslq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOVSXD_GvEv, offset, base, dst);
+ }
+ void movslq_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movslq " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_MOVSXD_GvEv, offset, base, index, scale, dst);
+ }
+
+ [[nodiscard]] JmpSrc movl_ripr(RegisterID dst) {
+ m_formatter.oneByteRipOp(OP_MOV_GvEv, 0, (RegisterID)dst);
+ JmpSrc label(m_formatter.size());
+ spew("movl " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+ GPReg32Name(dst));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc movl_rrip(RegisterID src) {
+ m_formatter.oneByteRipOp(OP_MOV_EvGv, 0, (RegisterID)src);
+ JmpSrc label(m_formatter.size());
+ spew("movl %s, " MEM_o32r "", GPReg32Name(src),
+ ADDR_o32r(label.offset()));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc movq_ripr(RegisterID dst) {
+ m_formatter.oneByteRipOp64(OP_MOV_GvEv, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("movq " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+ GPRegName(dst));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc movq_rrip(RegisterID src) {
+ m_formatter.oneByteRipOp64(OP_MOV_EvGv, 0, (RegisterID)src);
+ JmpSrc label(m_formatter.size());
+ spew("movq %s, " MEM_o32r "", GPRegName(src),
+ ADDR_o32r(label.offset()));
+ return label;
+ }
+
+ void leaq_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("leaq " MEM_ob ", %s", ADDR_ob(offset, base), GPReg64Name(dst));
+ m_formatter.oneByteOp64(OP_LEA, offset, base, dst);
+ }
+
+ [[nodiscard]] JmpSrc leaq_rip(RegisterID dst) {
+ m_formatter.oneByteRipOp64(OP_LEA, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("leaq " MEM_o32r ", %s", ADDR_o32r(label.offset()),
+ GPRegName(dst));
+ return label;
+ }
+
+ // Flow control:
+
+ void jmp_rip(int ripOffset) {
+ // rip-relative addressing.
+ spew("jmp *%d(%%rip)", ripOffset);
+ m_formatter.oneByteRipOp(OP_GROUP5_Ev, ripOffset, GROUP5_OP_JMPN);
+ }
+
+ void immediate64(int64_t imm) {
+ spew(".quad %lld", (long long)imm);
+ m_formatter.immediate64(imm);
+ }
+
+ // SSE operations:
+
+ void vcvtsq2sd_rr(RegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpInt64Simd("vcvtsi2sd", VEX_SD, OP2_CVTSI2SD_VsdEd, src1, src0,
+ dst);
+ }
+ void vcvtsq2ss_rr(RegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpInt64Simd("vcvtsi2ss", VEX_SS, OP2_CVTSI2SD_VsdEd, src1, src0,
+ dst);
+ }
+
+ void vcvtsi2sdq_rr(RegisterID src, XMMRegisterID dst) {
+ twoByteOpInt64Simd("vcvtsi2sdq", VEX_SD, OP2_CVTSI2SD_VsdEd, src,
+ invalid_xmm, dst);
+ }
+
+ void vcvttsd2sq_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt64("vcvttsd2si", VEX_SD, OP2_CVTTSD2SI_GdWsd, src, dst);
+ }
+
+ void vcvttss2sq_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt64("vcvttss2si", VEX_SS, OP2_CVTTSD2SI_GdWsd, src, dst);
+ }
+
+ void vmovq_rr(XMMRegisterID src, RegisterID dst) {
+ // While this is called "vmovq", it actually uses the vmovd encoding
+ // with a REX prefix modifying it to be 64-bit.
+ twoByteOpSimdInt64("vmovq", VEX_PD, OP2_MOVD_EdVd, (XMMRegisterID)dst,
+ (RegisterID)src);
+ }
+
+ void vpextrq_irr(unsigned lane, XMMRegisterID src, RegisterID dst) {
+ MOZ_ASSERT(lane < 2);
+ threeByteOpImmSimdInt64("vpextrq", VEX_PD, OP3_PEXTRQ_EvVdqIb, ESCAPE_3A,
+ lane, src, dst);
+ }
+
+ void vpinsrq_irr(unsigned lane, RegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(lane < 2);
+ threeByteOpImmInt64Simd("vpinsrq", VEX_PD, OP3_PINSRQ_VdqEvIb, ESCAPE_3A,
+ lane, src1, src0, dst);
+ }
+
+ void vmovq_rr(RegisterID src, XMMRegisterID dst) {
+ // While this is called "vmovq", it actually uses the vmovd encoding
+ // with a REX prefix modifying it to be 64-bit.
+ twoByteOpInt64Simd("vmovq", VEX_PD, OP2_MOVD_VdEd, src, invalid_xmm, dst);
+ }
+
+ [[nodiscard]] JmpSrc vmovsd_ripr(XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmovsd", VEX_SD, OP2_MOVSD_VsdWsd, dst);
+ }
+ [[nodiscard]] JmpSrc vmovss_ripr(XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmovss", VEX_SS, OP2_MOVSD_VsdWsd, dst);
+ }
+ [[nodiscard]] JmpSrc vmovaps_ripr(XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_VsdWsd, dst);
+ }
+ [[nodiscard]] JmpSrc vmovdqa_ripr(XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_VdqWdq, dst);
+ }
+
+ [[nodiscard]] JmpSrc vpaddb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddb", VEX_PD, OP2_PADDB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddw", VEX_PD, OP2_PADDW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddd", VEX_PD, OP2_PADDD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddq", VEX_PD, OP2_PADDQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubb", VEX_PD, OP2_PSUBB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubw", VEX_PD, OP2_PSUBW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubd", VEX_PD, OP2_PSUBD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubq", VEX_PD, OP2_PSUBQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmullw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmullw", VEX_PD, OP2_PMULLW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmulld_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmulld", VEX_PD, OP3_PMULLD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddsb", VEX_PD, OP2_PADDSB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddusb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddusb", VEX_PD, OP2_PADDUSB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddsw", VEX_PD, OP2_PADDSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpaddusw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpaddusw", VEX_PD, OP2_PADDUSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubsb", VEX_PD, OP2_PSUBSB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubusb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubusb", VEX_PD, OP2_PSUBUSB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubsw", VEX_PD, OP2_PSUBSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpsubusw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpsubusw", VEX_PD, OP2_PSUBUSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpminsb", VEX_PD, OP3_PMINSB_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminub_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpminub", VEX_PD, OP2_PMINUB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpminsw", VEX_PD, OP2_PMINSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminuw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpminuw", VEX_PD, OP3_PMINUW_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminsd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpminsd", VEX_PD, OP3_PMINSD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpminud_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpminud", VEX_PD, OP3_PMINUD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxsb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaxsb", VEX_PD, OP3_PMAXSB_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxub_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmaxub", VEX_PD, OP2_PMAXUB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmaxsw", VEX_PD, OP2_PMAXSW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxuw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaxuw", VEX_PD, OP3_PMAXUW_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxsd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaxsd", VEX_PD, OP3_PMAXSD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaxud_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaxud", VEX_PD, OP3_PMAXUD_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpand_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpand", VEX_PD, OP2_PANDDQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpxor_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpxor", VEX_PD, OP2_PXORDQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpor_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpor", VEX_PD, OP2_PORDQ_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vaddps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vaddps", VEX_PS, OP2_ADDPS_VpsWps, src, dst);
+ }
+ [[nodiscard]] JmpSrc vaddpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vaddpd", VEX_PD, OP2_ADDPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vsubps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vsubps", VEX_PS, OP2_SUBPS_VpsWps, src, dst);
+ }
+ [[nodiscard]] JmpSrc vsubpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vsubpd", VEX_PD, OP2_SUBPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vdivps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vdivps", VEX_PS, OP2_DIVPS_VpsWps, src, dst);
+ }
+ [[nodiscard]] JmpSrc vdivpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vdivpd", VEX_PD, OP2_DIVPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vmulps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmulps", VEX_PS, OP2_MULPS_VpsWps, src, dst);
+ }
+ [[nodiscard]] JmpSrc vmulpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vmulpd", VEX_PD, OP2_MULPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vandpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vandpd", VEX_PD, OP2_ANDPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vminpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vminpd", VEX_PD, OP2_MINPD_VpdWpd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpacksswb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpacksswb", VEX_PD, OP2_PACKSSWB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpackuswb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpackuswb", VEX_PD, OP2_PACKUSWB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpackssdw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpackssdw", VEX_PD, OP2_PACKSSDW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpackusdw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpackusdw", VEX_PD, OP3_PACKUSDW_VdqWdq,
+ ESCAPE_38, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpunpckldq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpunpckldq", VEX_PD, OP2_PUNPCKLDQ_VdqWdq, src,
+ dst);
+ }
+ [[nodiscard]] JmpSrc vunpcklps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vunpcklps", VEX_PS, OP2_UNPCKLPS_VsdWsd, src, dst);
+ }
+ [[nodiscard]] JmpSrc vptest_ripr(XMMRegisterID lhs) {
+ return threeByteRipOpSimd("vptest", VEX_PD, OP3_PTEST_VdVd, ESCAPE_38, lhs);
+ }
+ [[nodiscard]] JmpSrc vpshufb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpshufb", VEX_PD, OP3_PSHUFB_VdqWdq, ESCAPE_38,
+ src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaddwd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmaddwd", VEX_PD, OP2_PMADDWD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpeqb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpeqb", VEX_PD, OP2_PCMPEQB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpgtb_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpgtb", VEX_PD, OP2_PCMPGTB_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpeqw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpeqw", VEX_PD, OP2_PCMPEQW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpgtw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpgtw", VEX_PD, OP2_PCMPGTW_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpeqd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpeqd", VEX_PD, OP2_PCMPEQD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpcmpgtd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpcmpgtd", VEX_PD, OP2_PCMPGTD_VdqWdq, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpeqps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_EQ, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpneqps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_NEQ, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpltps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_LT, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpleps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_LE, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpgeps_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps,
+ X86Encoding::ConditionCmp_GE, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpeqpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+ X86Encoding::ConditionCmp_EQ, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpneqpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+ X86Encoding::ConditionCmp_NEQ, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmpltpd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+ X86Encoding::ConditionCmp_LT, src, dst);
+ }
+ [[nodiscard]] JmpSrc vcmplepd_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd,
+ X86Encoding::ConditionCmp_LE, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmaddubsw_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return threeByteRipOpSimd("vpmaddubsw", VEX_PD, OP3_PMADDUBSW_VdqWdq,
+ ESCAPE_38, src, dst);
+ }
+ [[nodiscard]] JmpSrc vpmuludq_ripr(XMMRegisterID src, XMMRegisterID dst) {
+ return twoByteRipOpSimd("vpmuludq", VEX_PD, OP2_PMULUDQ_VdqWdq, src, dst);
+ }
+
+ // BMI instructions:
+
+ void sarxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("sarxq %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+ GPReg64Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex64(VEX_SS /* = F3 */, OP3_SARX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ void shlxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("shlxq %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+ GPReg64Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex64(VEX_PD /* = 66 */, OP3_SHLX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ void shrxq_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("shrxq %s, %s, %s", GPReg64Name(src), GPReg64Name(shift),
+ GPReg64Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex64(VEX_SD /* = F2 */, OP3_SHRX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ private:
+ [[nodiscard]] JmpSrc twoByteRipOpSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode,
+ XMMRegisterID reg) {
+ MOZ_ASSERT(!IsXMMReversedOperands(opcode));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteRipOp(opcode, 0, reg);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s " MEM_o32r ", %s", legacySSEOpName(name),
+ ADDR_o32r(label.offset()), XMMRegName(reg));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc twoByteRipOpSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ MOZ_ASSERT(src0 != invalid_xmm && !IsXMMReversedOperands(opcode));
+ if (useLegacySSEEncoding(src0, dst)) {
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteRipOp(opcode, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+ ADDR_o32r(label.offset()), XMMRegName(dst));
+ return label;
+ }
+
+ m_formatter.twoByteRipOpVex(ty, opcode, 0, src0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s, " MEM_o32r ", %s, %s", name, ADDR_o32r(label.offset()),
+ XMMRegName(src0), XMMRegName(dst));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc twoByteRipOpImmSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm,
+ XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(src0 != invalid_xmm && !IsXMMReversedOperands(opcode));
+ if (useLegacySSEEncoding(src0, dst)) {
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteRipOp(opcode, 0, dst);
+ m_formatter.immediate8u(imm);
+ JmpSrc label(m_formatter.size(),
+ /* bytes trailing the patch field = */ 1);
+ spew("%-11s$0x%x, " MEM_o32r ", %s", legacySSEOpName(name), imm,
+ ADDR_o32r(label.offset()), XMMRegName(dst));
+ return label;
+ }
+
+ m_formatter.twoByteRipOpVex(ty, opcode, 0, src0, dst);
+ m_formatter.immediate8u(imm);
+ JmpSrc label(m_formatter.size(),
+ /* bytes trailing the patch field = */ 1);
+ spew("%-11s$0x%x, " MEM_o32r ", %s, %s", name, imm,
+ ADDR_o32r(label.offset()), XMMRegName(src0), XMMRegName(dst));
+ return label;
+ }
+
+ void twoByteOpInt64Simd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, RegisterID rm,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(dst),
+ GPRegName(rm));
+ } else {
+ spew("%-11s%s, %s", legacySSEOpName(name), GPRegName(rm),
+ XMMRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp64(opcode, rm, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", name, XMMRegName(dst), GPRegName(rm));
+ } else {
+ spew("%-11s%s, %s", name, GPRegName(rm), XMMRegName(dst));
+ }
+ } else {
+ spew("%-11s%s, %s, %s", name, GPRegName(rm), XMMRegName(src0),
+ XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex64(ty, opcode, rm, src0, dst);
+ }
+
+ void twoByteOpSimdInt64(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, XMMRegisterID rm,
+ RegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), GPRegName(dst),
+ XMMRegName(rm));
+ } else if (opcode == OP2_MOVD_EdVd) {
+ spew("%-11s%s, %s", legacySSEOpName(name),
+ XMMRegName((XMMRegisterID)dst), GPRegName((RegisterID)rm));
+ } else {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(rm),
+ GPRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp64(opcode, (RegisterID)rm, dst);
+ return;
+ }
+
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", name, GPRegName(dst), XMMRegName(rm));
+ } else if (opcode == OP2_MOVD_EdVd) {
+ spew("%-11s%s, %s", name, XMMRegName((XMMRegisterID)dst),
+ GPRegName((RegisterID)rm));
+ } else {
+ spew("%-11s%s, %s", name, XMMRegName(rm), GPRegName(dst));
+ }
+ m_formatter.twoByteOpVex64(ty, opcode, (RegisterID)rm, invalid_xmm,
+ (XMMRegisterID)dst);
+ }
+
+ [[nodiscard]] JmpSrc threeByteRipOpSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape,
+ XMMRegisterID dst) {
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteRipOp(opcode, escape, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+ ADDR_o32r(label.offset()), XMMRegName(dst));
+ return label;
+ }
+
+ [[nodiscard]] JmpSrc threeByteRipOpSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape,
+ XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(src0 != invalid_xmm);
+ if (useLegacySSEEncoding(src0, dst)) {
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteRipOp(opcode, escape, 0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s" MEM_o32r ", %s", legacySSEOpName(name),
+ ADDR_o32r(label.offset()), XMMRegName(dst));
+ return label;
+ }
+
+ m_formatter.threeByteRipOpVex(ty, opcode, escape, 0, src0, dst);
+ JmpSrc label(m_formatter.size());
+ spew("%-11s" MEM_o32r ", %s, %s", name, ADDR_o32r(label.offset()),
+ XMMRegName(src0), XMMRegName(dst));
+ return label;
+ }
+
+ void threeByteOpImmSimdInt64(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, XMMRegisterID src,
+ RegisterID dst) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, GPReg64Name(dst),
+ XMMRegName(src));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp64(opcode, escape, dst, (RegisterID)src);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpImmInt64Simd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, RegisterID src1,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, GPReg64Name(src1),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp64(opcode, escape, src1, (RegisterID)dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ MOZ_ASSERT(src0 != invalid_xmm);
+ spew("%-11s$0x%x, %s, %s, %s", name, imm, GPReg64Name(src1),
+ XMMRegName(src0), XMMRegName(dst));
+ m_formatter.threeByteOpVex64(ty, opcode, escape, src1, src0,
+ (RegisterID)dst);
+ m_formatter.immediate8u(imm);
+ }
+};
+
+using BaseAssemblerSpecific = BaseAssemblerX64;
+
+} // namespace X86Encoding
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_BaseAssembler_x64_h */
diff --git a/js/src/jit/x64/CodeGenerator-x64.cpp b/js/src/jit/x64/CodeGenerator-x64.cpp
new file mode 100644
index 0000000000..9bd7e9b253
--- /dev/null
+++ b/js/src/jit/x64/CodeGenerator-x64.cpp
@@ -0,0 +1,984 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/CodeGenerator-x64.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/MIR.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+
+CodeGeneratorX64::CodeGeneratorX64(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorX86Shared(gen, graph, masm) {}
+
+ValueOperand CodeGeneratorX64::ToValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getOperand(pos)));
+}
+
+ValueOperand CodeGeneratorX64::ToTempValue(LInstruction* ins, size_t pos) {
+ return ValueOperand(ToRegister(ins->getTemp(pos)));
+}
+
+Operand CodeGeneratorX64::ToOperand64(const LInt64Allocation& a64) {
+ const LAllocation& a = a64.value();
+ MOZ_ASSERT(!a.isFloatReg());
+ if (a.isGeneralReg()) {
+ return Operand(a.toGeneralReg()->reg());
+ }
+ return Operand(ToAddress(a));
+}
+
+void CodeGenerator::visitValue(LValue* value) {
+ ValueOperand result = ToOutValue(value);
+ masm.moveValue(value->value(), result);
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+ const LAllocation* in = box->getOperand(0);
+ ValueOperand result = ToOutValue(box);
+
+ masm.moveValue(TypedOrValueRegister(box->type(), ToAnyRegister(in)), result);
+
+ if (JitOptions.spectreValueMasking && IsFloatingPointType(box->type())) {
+ ScratchRegisterScope scratch(masm);
+ masm.movePtr(ImmWord(JSVAL_SHIFTED_TAG_MAX_DOUBLE), scratch);
+ masm.cmpPtrMovePtr(Assembler::Below, scratch, result.valueReg(), scratch,
+ result.valueReg());
+ }
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+ MUnbox* mir = unbox->mir();
+
+ Register result = ToRegister(unbox->output());
+
+ if (mir->fallible()) {
+ const ValueOperand value = ToValue(unbox, LUnbox::Input);
+ Label bail;
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.fallibleUnboxInt32(value, result, &bail);
+ break;
+ case MIRType::Boolean:
+ masm.fallibleUnboxBoolean(value, result, &bail);
+ break;
+ case MIRType::Object:
+ masm.fallibleUnboxObject(value, result, &bail);
+ break;
+ case MIRType::String:
+ masm.fallibleUnboxString(value, result, &bail);
+ break;
+ case MIRType::Symbol:
+ masm.fallibleUnboxSymbol(value, result, &bail);
+ break;
+ case MIRType::BigInt:
+ masm.fallibleUnboxBigInt(value, result, &bail);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+ bailoutFrom(&bail, unbox->snapshot());
+ return;
+ }
+
+ // Infallible unbox.
+
+ Operand input = ToOperand(unbox->getOperand(LUnbox::Input));
+
+#ifdef DEBUG
+ // Assert the types match.
+ JSValueTag tag = MIRTypeToTag(mir->type());
+ Label ok;
+ masm.splitTag(input, ScratchReg);
+ masm.branch32(Assembler::Equal, ScratchReg, Imm32(tag), &ok);
+ masm.assumeUnreachable("Infallible unbox type mismatch");
+ masm.bind(&ok);
+#endif
+
+ switch (mir->type()) {
+ case MIRType::Int32:
+ masm.unboxInt32(input, result);
+ break;
+ case MIRType::Boolean:
+ masm.unboxBoolean(input, result);
+ break;
+ case MIRType::Object:
+ masm.unboxObject(input, result);
+ break;
+ case MIRType::String:
+ masm.unboxString(input, result);
+ break;
+ case MIRType::Symbol:
+ masm.unboxSymbol(input, result);
+ break;
+ case MIRType::BigInt:
+ masm.unboxBigInt(input, result);
+ break;
+ default:
+ MOZ_CRASH("Given MIRType cannot be unboxed.");
+ }
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+ MCompare* mir = lir->mir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+ Register output = ToRegister(lir->output());
+
+ if (IsConstant(rhs)) {
+ masm.cmpPtr(lhsReg, ImmWord(ToInt64(rhs)));
+ } else {
+ masm.cmpPtr(lhsReg, ToOperand64(rhs));
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ masm.emitSet(JSOpToCondition(lir->jsop(), isSigned), output);
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+ MCompare* mir = lir->cmpMir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register lhsReg = ToRegister64(lhs).reg;
+
+ if (IsConstant(rhs)) {
+ masm.cmpPtr(lhsReg, ImmWord(ToInt64(rhs)));
+ } else {
+ masm.cmpPtr(lhsReg, ToOperand64(rhs));
+ }
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ emitBranch(JSOpToCondition(lir->jsop(), isSigned), lir->ifTrue(),
+ lir->ifFalse());
+}
+
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
+ Register regL = ToRegister(baab->left());
+ if (baab->is64()) {
+ if (baab->right()->isConstant()) {
+ masm.test64(regL, Imm64(ToInt64(baab->right())));
+ } else {
+ masm.test64(regL, ToRegister(baab->right()));
+ }
+ } else {
+ if (baab->right()->isConstant()) {
+ masm.test32(regL, Imm32(ToInt32(baab->right())));
+ } else {
+ masm.test32(regL, ToRegister(baab->right()));
+ }
+ }
+ emitBranch(baab->cond(), baab->ifTrue(), baab->ifFalse());
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+ Register output = ToRegister(lir->output());
+
+ MOZ_ASSERT_IF(lhs != rhs, rhs != rax);
+ MOZ_ASSERT(rhs != rdx);
+ MOZ_ASSERT_IF(output == rax, ToRegister(lir->remainder()) == rdx);
+ MOZ_ASSERT_IF(output == rdx, ToRegister(lir->remainder()) == rax);
+
+ Label done;
+
+ // Put the lhs in rax.
+ if (lhs != rax) {
+ masm.mov(lhs, rax);
+ }
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.branchTestPtr(Assembler::NonZero, rhs, rhs, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ // Handle an integer overflow exception from INT64_MIN / -1.
+ if (lir->canBeNegativeOverflow()) {
+ Label notOverflow;
+ masm.branchPtr(Assembler::NotEqual, lhs, ImmWord(INT64_MIN), &notOverflow);
+ masm.branchPtr(Assembler::NotEqual, rhs, ImmWord(-1), &notOverflow);
+ if (lir->mir()->isMod()) {
+ masm.xorl(output, output);
+ } else {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ masm.jump(&done);
+ masm.bind(&notOverflow);
+ }
+
+ // Sign extend the lhs into rdx to make rdx:rax.
+ masm.cqo();
+ masm.idivq(rhs);
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+ Register lhs = ToRegister(lir->lhs());
+ Register rhs = ToRegister(lir->rhs());
+
+ DebugOnly<Register> output = ToRegister(lir->output());
+ MOZ_ASSERT_IF(lhs != rhs, rhs != rax);
+ MOZ_ASSERT(rhs != rdx);
+ MOZ_ASSERT_IF(output.value == rax, ToRegister(lir->remainder()) == rdx);
+ MOZ_ASSERT_IF(output.value == rdx, ToRegister(lir->remainder()) == rax);
+
+ // Put the lhs in rax.
+ if (lhs != rax) {
+ masm.mov(lhs, rax);
+ }
+
+ Label done;
+
+ // Prevent divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ masm.branchTestPtr(Assembler::NonZero, rhs, rhs, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ // Zero extend the lhs into rdx to make (rdx:rax).
+ masm.xorl(rdx, rdx);
+ masm.udivq(rhs);
+
+ masm.bind(&done);
+}
+
+void CodeGeneratorX64::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ MOZ_ASSERT(dividend == rax);
+ MOZ_ASSERT(output == rdx);
+
+ // Sign extend the lhs into rdx to make rdx:rax.
+ masm.cqo();
+
+ masm.idivq(divisor);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorX64::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ MOZ_ASSERT(dividend == rax);
+ MOZ_ASSERT(output == rdx);
+
+ // Sign extend the lhs into rdx to make rdx:rax.
+ masm.cqo();
+
+ masm.idivq(divisor);
+
+ // Move the remainder from rdx.
+ masm.movq(output, dividend);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ auto sync = Synchronization::Load();
+
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.load64(source, temp64);
+ }
+ masm.memoryBarrierAfter(sync);
+
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp1);
+
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ auto sync = Synchronization::Store();
+
+ masm.memoryBarrierBefore(sync);
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.store64(temp1, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.store64(temp1, dest);
+ }
+ masm.memoryBarrierAfter(sync);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+
+ MOZ_ASSERT(temp1.reg == rax);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(oldval, temp1);
+ masm.loadBigInt64(newval, temp2);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, temp2, temp1);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchange64(Synchronization::Full(), dest, temp1, temp2, temp1);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp1, out, temp2.reg);
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register temp2 = ToRegister(lir->temp2());
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp1);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp1);
+ }
+
+ emitCreateBigInt(lir, arrayType, temp1, out, temp2);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = ToRegister64(lir->temp2());
+ Register out = ToRegister(lir->output());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ Register64 fetchTemp = Register64(out);
+ Register64 fetchOut = temp2;
+ Register createTemp = temp1.reg;
+
+ // Add and Sub don't need |fetchTemp| and can save a `mov` when the value and
+ // output register are equal to each other.
+ if (atomicOp == AtomicFetchAddOp || atomicOp == AtomicFetchSubOp) {
+ fetchTemp = Register64::Invalid();
+ fetchOut = temp1;
+ createTemp = temp2.reg;
+ } else {
+ MOZ_ASSERT(temp2.reg == rax);
+ }
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ fetchTemp, fetchOut);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, temp1, dest,
+ fetchTemp, fetchOut);
+ }
+
+ emitCreateBigInt(lir, arrayType, fetchOut, out, createTemp);
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicEffectOp64(Synchronization::Full(), atomicOp, temp1, dest);
+ }
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+
+ Register cond = ToRegister(lir->condExpr());
+
+ Operand falseExpr = ToOperandOrRegister64(lir->falseExpr());
+
+ Register64 out = ToOutRegister64(lir);
+ MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
+ "true expr is reused for input");
+
+ masm.test32(cond, cond);
+ masm.cmovzq(falseExpr, out.reg);
+}
+
+// We expect to handle only the cases: compare is {U,}Int{32,64}, and select
+// is {U,}Int{32,64}, independently. Some values may be stack allocated, and
+// the "true" input is reused for the output.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
+ ins->compareType() == MCompare::Compare_UInt32;
+ bool cmpIs64bit = ins->compareType() == MCompare::Compare_Int64 ||
+ ins->compareType() == MCompare::Compare_UInt64;
+ bool selIs32bit = ins->mir()->type() == MIRType::Int32;
+ bool selIs64bit = ins->mir()->type() == MIRType::Int64;
+
+ // Throw out unhandled cases
+ MOZ_RELEASE_ASSERT(
+ cmpIs32bit != cmpIs64bit && selIs32bit != selIs64bit,
+ "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
+
+ using C = Assembler::Condition;
+ using R = Register;
+ using A = const Address&;
+
+ // Identify macroassembler methods to generate instructions, based on the
+ // type of the comparison and the select. This avoids having to duplicate
+ // the code-generation tree below 4 times. These assignments to
+ // `cmpMove_CRRRR` et al are unambiguous as a result of the combination of
+ // the template parameters and the 5 argument types ((C, R, R, R, R) etc).
+ void (MacroAssembler::*cmpMove_CRRRR)(C, R, R, R, R) = nullptr;
+ void (MacroAssembler::*cmpMove_CRARR)(C, R, A, R, R) = nullptr;
+ void (MacroAssembler::*cmpLoad_CRRAR)(C, R, R, A, R) = nullptr;
+ void (MacroAssembler::*cmpLoad_CRAAR)(C, R, A, A, R) = nullptr;
+
+ if (cmpIs32bit) {
+ if (selIs32bit) {
+ cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<32, 32>;
+ cmpMove_CRARR = &MacroAssemblerX64::cmpMove<32, 32>;
+ cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<32, 32>;
+ cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<32, 32>;
+ } else {
+ cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<32, 64>;
+ cmpMove_CRARR = &MacroAssemblerX64::cmpMove<32, 64>;
+ cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<32, 64>;
+ cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<32, 64>;
+ }
+ } else {
+ if (selIs32bit) {
+ cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<64, 32>;
+ cmpMove_CRARR = &MacroAssemblerX64::cmpMove<64, 32>;
+ cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<64, 32>;
+ cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<64, 32>;
+ } else {
+ cmpMove_CRRRR = &MacroAssemblerX64::cmpMove<64, 64>;
+ cmpMove_CRARR = &MacroAssemblerX64::cmpMove<64, 64>;
+ cmpLoad_CRRAR = &MacroAssemblerX64::cmpLoad<64, 64>;
+ cmpLoad_CRAAR = &MacroAssemblerX64::cmpLoad<64, 64>;
+ }
+ }
+
+ Register trueExprAndDest = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
+ "true expr input is reused for output");
+
+ Assembler::Condition cond = Assembler::InvertCondition(
+ JSOpToCondition(ins->compareType(), ins->jsop()));
+ const LAllocation* rhs = ins->rightExpr();
+ const LAllocation* falseExpr = ins->ifFalseExpr();
+ Register lhs = ToRegister(ins->leftExpr());
+
+ // We generate one of four cmp+cmov pairings, depending on whether one of
+ // the cmp args and one of the cmov args is in memory or a register.
+ if (rhs->isRegister()) {
+ if (falseExpr->isRegister()) {
+ (masm.*cmpMove_CRRRR)(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+ } else {
+ (masm.*cmpLoad_CRRAR)(cond, lhs, ToRegister(rhs), ToAddress(falseExpr),
+ trueExprAndDest);
+ }
+ } else {
+ if (falseExpr->isRegister()) {
+ (masm.*cmpMove_CRARR)(cond, lhs, ToAddress(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+ } else {
+ (masm.*cmpLoad_CRAAR)(cond, lhs, ToAddress(rhs), ToAddress(falseExpr),
+ trueExprAndDest);
+ }
+ }
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+ masm.vmovq(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+ masm.vmovq(ToFloatRegister(lir->input()), ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+ masm.convertUInt32ToDouble(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+ masm.convertUInt32ToFloat32(ToRegister(lir->input()),
+ ToFloatRegister(lir->output()));
+}
+
+void CodeGeneratorX64::wasmStore(const wasm::MemoryAccessDesc& access,
+ const LAllocation* value, Operand dstAddr) {
+ if (value->isConstant()) {
+ masm.memoryBarrierBefore(access.sync());
+
+ const MConstant* mir = value->toConstant();
+ Imm32 cst =
+ Imm32(mir->type() == MIRType::Int32 ? mir->toInt32() : mir->toInt64());
+
+ masm.append(access, masm.size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ masm.movb(cst, dstAddr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ masm.movw(cst, dstAddr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ masm.movl(cst, dstAddr);
+ break;
+ case Scalar::Int64:
+ case Scalar::Simd128:
+ case Scalar::Float32:
+ case Scalar::Float64:
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ masm.memoryBarrierAfter(access.sync());
+ } else {
+ masm.wasmStore(access, ToAnyRegister(value), dstAddr);
+ }
+}
+
+void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) {
+ MOZ_ASSERT(ins->instance()->isBogus());
+ masm.movePtr(HeapReg, ToRegister(ins->output()));
+}
+
+template <typename T>
+void CodeGeneratorX64::emitWasmLoad(T* ins) {
+ const MWasmLoad* mir = ins->mir();
+
+ uint32_t offset = mir->access().offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ // ptr is a GPR and is either a 32-bit value zero-extended to 64-bit, or a
+ // true 64-bit value.
+ const LAllocation* ptr = ins->ptr();
+ Operand srcAddr = ptr->isBogus()
+ ? Operand(HeapReg, offset)
+ : Operand(HeapReg, ToRegister(ptr), TimesOne, offset);
+
+ if (mir->type() == MIRType::Int64) {
+ masm.wasmLoadI64(mir->access(), srcAddr, ToOutRegister64(ins));
+ } else {
+ masm.wasmLoad(mir->access(), srcAddr, ToAnyRegister(ins->output()));
+ }
+}
+
+void CodeGenerator::visitWasmLoad(LWasmLoad* ins) { emitWasmLoad(ins); }
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* ins) { emitWasmLoad(ins); }
+
+template <typename T>
+void CodeGeneratorX64::emitWasmStore(T* ins) {
+ const MWasmStore* mir = ins->mir();
+ const wasm::MemoryAccessDesc& access = mir->access();
+
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* value = ins->getOperand(ins->ValueIndex);
+ const LAllocation* ptr = ins->ptr();
+ Operand dstAddr = ptr->isBogus()
+ ? Operand(HeapReg, offset)
+ : Operand(HeapReg, ToRegister(ptr), TimesOne, offset);
+
+ wasmStore(access, value, dstAddr);
+}
+
+void CodeGenerator::visitWasmStore(LWasmStore* ins) { emitWasmStore(ins); }
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* ins) {
+ MOZ_CRASH("Unused on this platform");
+}
+
+void CodeGenerator::visitWasmCompareExchangeHeap(
+ LWasmCompareExchangeHeap* ins) {
+ MWasmCompareExchangeHeap* mir = ins->mir();
+
+ Register ptr = ToRegister(ins->ptr());
+ Register oldval = ToRegister(ins->oldValue());
+ Register newval = ToRegister(ins->newValue());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Scalar::Type accessType = mir->access().type();
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (accessType == Scalar::Int64) {
+ masm.wasmCompareExchange64(mir->access(), srcAddr, Register64(oldval),
+ Register64(newval), ToOutRegister64(ins));
+ } else {
+ masm.wasmCompareExchange(mir->access(), srcAddr, oldval, newval,
+ ToRegister(ins->output()));
+ }
+}
+
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+ MWasmAtomicExchangeHeap* mir = ins->mir();
+
+ Register ptr = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Scalar::Type accessType = mir->access().type();
+
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (accessType == Scalar::Int64) {
+ masm.wasmAtomicExchange64(mir->access(), srcAddr, Register64(value),
+ ToOutRegister64(ins));
+ } else {
+ masm.wasmAtomicExchange(mir->access(), srcAddr, value,
+ ToRegister(ins->output()));
+ }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ MOZ_ASSERT(mir->hasUses());
+
+ Register ptr = ToRegister(ins->ptr());
+ const LAllocation* value = ins->value();
+ Register temp =
+ ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp());
+ Register output = ToRegister(ins->output());
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Scalar::Type accessType = mir->access().type();
+ if (accessType == Scalar::Uint32) {
+ accessType = Scalar::Int32;
+ }
+
+ AtomicOp op = mir->operation();
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (accessType == Scalar::Int64) {
+ Register64 val = Register64(ToRegister(value));
+ Register64 out = Register64(output);
+ Register64 tmp = Register64(temp);
+ masm.wasmAtomicFetchOp64(mir->access(), op, val, srcAddr, tmp, out);
+ } else if (value->isConstant()) {
+ masm.wasmAtomicFetchOp(mir->access(), op, Imm32(ToInt32(value)), srcAddr,
+ temp, output);
+ } else {
+ masm.wasmAtomicFetchOp(mir->access(), op, ToRegister(value), srcAddr, temp,
+ output);
+ }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+ LWasmAtomicBinopHeapForEffect* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ MOZ_ASSERT(!mir->hasUses());
+
+ Register ptr = ToRegister(ins->ptr());
+ const LAllocation* value = ins->value();
+ MOZ_ASSERT(ins->addrTemp()->isBogusTemp());
+
+ Scalar::Type accessType = mir->access().type();
+ AtomicOp op = mir->operation();
+
+ BaseIndex srcAddr(HeapReg, ptr, TimesOne, mir->access().offset());
+
+ if (accessType == Scalar::Int64) {
+ Register64 val = Register64(ToRegister(value));
+ masm.wasmAtomicEffectOp64(mir->access(), op, val, srcAddr);
+ } else if (value->isConstant()) {
+ Imm32 c(0);
+ if (value->toConstant()->type() == MIRType::Int64) {
+ c = Imm32(ToInt64(value));
+ } else {
+ c = Imm32(ToInt32(value));
+ }
+ masm.wasmAtomicEffectOp(mir->access(), op, c, srcAddr, InvalidReg);
+ } else {
+ masm.wasmAtomicEffectOp(mir->access(), op, ToRegister(value), srcAddr,
+ InvalidReg);
+ }
+}
+
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ // On x64, branchTruncateDouble uses vcvttsd2sq. Unlike the x86
+ // implementation, this should handle most doubles and we can just
+ // call a stub if it fails.
+ emitTruncateDouble(input, output, ins->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->getOperand(0));
+ Register output = ToRegister(lir->getDef(0));
+
+ emitTruncateDouble(input, output, lir->mir());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+ LWasmBuiltinTruncateFToInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->getOperand(0));
+ Register output = ToRegister(lir->getDef(0));
+
+ emitTruncateFloat32(input, output, lir->mir());
+}
+
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ // On x64, branchTruncateFloat32 uses vcvttss2sq. Unlike the x86
+ // implementation, this should handle most floats and we can just
+ // call a stub if it fails.
+ emitTruncateFloat32(input, output, ins->mir());
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ masm.movl(ToOperand(input), output);
+ } else {
+ MOZ_CRASH("Not implemented.");
+ }
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ const LAllocation* input = lir->getOperand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->isUnsigned()) {
+ masm.movl(ToOperand(input), output);
+ } else {
+ masm.movslq(ToOperand(input), output);
+ }
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index* lir) {
+ // Generates no code on this platform because the input is assumed to have
+ // canonical form.
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+ masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ // Generates no code on this platform because the input is assumed to have
+ // canonical form.
+ Register output = ToRegister(lir->output());
+ MOZ_ASSERT(ToRegister(lir->input()) == output);
+ masm.debugAssertCanonicalInt32(output);
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ Register64 output = ToOutRegister64(ins);
+ switch (ins->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.movsbq(Operand(input.reg), output.reg);
+ break;
+ case MSignExtendInt64::Half:
+ masm.movswq(Operand(input.reg), output.reg);
+ break;
+ case MSignExtendInt64::Word:
+ masm.movslq(Operand(input.reg), output.reg);
+ break;
+ }
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ MWasmTruncateToInt64* mir = lir->mir();
+ MIRType inputType = mir->input()->type();
+
+ MOZ_ASSERT(inputType == MIRType::Double || inputType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ FloatRegister temp =
+ mir->isUnsigned() ? ToFloatRegister(lir->temp()) : InvalidFloatReg;
+
+ Label* oolEntry = ool->entry();
+ Label* oolRejoin = ool->rejoin();
+ bool isSaturating = mir->isSaturating();
+ if (inputType == MIRType::Double) {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, temp);
+ } else {
+ masm.wasmTruncateDoubleToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, temp);
+ }
+ } else {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, temp);
+ } else {
+ masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, oolEntry,
+ oolRejoin, temp);
+ }
+ }
+}
+
+void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ MInt64ToFloatingPoint* mir = lir->mir();
+ bool isUnsigned = mir->isUnsigned();
+
+ MIRType outputType = mir->type();
+ MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
+ MOZ_ASSERT(isUnsigned == !lir->getTemp(0)->isBogusTemp());
+
+ if (outputType == MIRType::Double) {
+ if (isUnsigned) {
+ masm.convertUInt64ToDouble(input, output, ToRegister(lir->getTemp(0)));
+ } else {
+ masm.convertInt64ToDouble(input, output);
+ }
+ } else {
+ if (isUnsigned) {
+ masm.convertUInt64ToFloat32(input, output, ToRegister(lir->getTemp(0)));
+ } else {
+ masm.convertInt64ToFloat32(input, output);
+ }
+ }
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+ masm.cmpq(Imm32(0), ToRegister(lir->input()));
+ masm.emitSet(Assembler::Equal, ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitClzI64(LClzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.clz64(input, output.reg);
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ masm.ctz64(input, output.reg);
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ MOZ_ASSERT(!input->isConstant());
+ Register inputR = ToRegister(input);
+ MOZ_ASSERT(inputR == ToRegister(ins->output()));
+ masm.notq(inputR);
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ Register input = ToRegister(lir->input());
+ masm.testq(input, input);
+ emitBranch(Assembler::NonZero, lir->ifTrue(), lir->ifFalse());
+}
diff --git a/js/src/jit/x64/CodeGenerator-x64.h b/js/src/jit/x64/CodeGenerator-x64.h
new file mode 100644
index 0000000000..c3359d0190
--- /dev/null
+++ b/js/src/jit/x64/CodeGenerator-x64.h
@@ -0,0 +1,41 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_CodeGenerator_x64_h
+#define jit_x64_CodeGenerator_x64_h
+
+#include "jit/x86-shared/CodeGenerator-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorX64 : public CodeGeneratorX86Shared {
+ protected:
+ CodeGeneratorX64(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
+
+ Operand ToOperand64(const LInt64Allocation& a);
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+
+ void wasmStore(const wasm::MemoryAccessDesc& access, const LAllocation* value,
+ Operand dstAddr);
+ template <typename T>
+ void emitWasmLoad(T* ins);
+ template <typename T>
+ void emitWasmStore(T* ins);
+};
+
+using CodeGeneratorSpecific = CodeGeneratorX64;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_CodeGenerator_x64_h */
diff --git a/js/src/jit/x64/LIR-x64.h b/js/src/jit/x64/LIR-x64.h
new file mode 100644
index 0000000000..efaedc4499
--- /dev/null
+++ b/js/src/jit/x64/LIR-x64.h
@@ -0,0 +1,170 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_LIR_x64_h
+#define jit_x64_LIR_x64_h
+
+namespace js {
+namespace jit {
+
+// Given an untyped input, guards on whether it's a specific type and returns
+// the unboxed payload.
+class LUnboxBase : public LInstructionHelper<1, 1, 0> {
+ public:
+ LUnboxBase(LNode::Opcode op, const LAllocation& input)
+ : LInstructionHelper<1, 1, 0>(op) {
+ setOperand(0, input);
+ }
+
+ static const size_t Input = 0;
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+};
+
+class LUnbox : public LUnboxBase {
+ public:
+ LIR_HEADER(Unbox)
+
+ explicit LUnbox(const LAllocation& input) : LUnboxBase(classOpcode, input) {}
+
+ const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LUnboxBase {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint)
+
+ LUnboxFloatingPoint(const LAllocation& input, MIRType type)
+ : LUnboxBase(classOpcode, input), type_(type) {}
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToDouble)
+
+ explicit LWasmUint32ToDouble(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 0> {
+ public:
+ LIR_HEADER(WasmUint32ToFloat32)
+
+ explicit LWasmUint32ToFloat32(const LAllocation& input)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ }
+};
+
+class LDivOrModI64 : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ LDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+// This class performs a simple x86 'div', yielding either a quotient or
+// remainder depending on whether this instruction is defined to output
+// rax (quotient) or rdx (remainder).
+class LUDivOrModI64 : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(UDivOrModI64);
+
+ LUDivOrModI64(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+
+ const char* extraName() const {
+ return mir()->isTruncated() ? "Truncated" : nullptr;
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LWasmTruncateToInt64 : public LInstructionHelper<1, 1, 1> {
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ LWasmTruncateToInt64(const LAllocation& in, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setTemp(0, temp);
+ }
+
+ MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); }
+
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_LIR_x64_h */
diff --git a/js/src/jit/x64/Lowering-x64.cpp b/js/src/jit/x64/Lowering-x64.cpp
new file mode 100644
index 0000000000..d2e328930c
--- /dev/null
+++ b/js/src/jit/x64/Lowering-x64.cpp
@@ -0,0 +1,565 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/Lowering-x64.h"
+
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/x64/Assembler-x64.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+LBoxAllocation LIRGeneratorX64::useBoxFixed(MDefinition* mir, Register reg1,
+ Register, bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart));
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegister(MDefinition* mir) {
+ return useRegister(mir);
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegisterAtStart(MDefinition* mir) {
+ return useRegisterAtStart(mir);
+}
+
+LAllocation LIRGeneratorX64::useByteOpRegisterOrNonDoubleConstant(
+ MDefinition* mir) {
+ return useRegisterOrNonDoubleConstant(mir);
+}
+
+LDefinition LIRGeneratorX64::tempByteOpRegister() { return temp(); }
+
+LDefinition LIRGeneratorX64::tempToUnbox() { return temp(); }
+
+void LIRGeneratorX64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+ MDefinition* input) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX64::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX64::lowerForMulInt64(LMulI64* ins, MMul* mir,
+ MDefinition* lhs, MDefinition* rhs) {
+ // X64 doesn't need a temp for 64bit multiplication.
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useInt64OrConstant(rhs)
+ : useInt64OrConstantAtStart(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGenerator::visitBox(MBox* box) {
+ MDefinition* opd = box->getOperand(0);
+
+ // If the operand is a constant, emit near its uses.
+ if (opd->isConstant() && box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (opd->isConstant()) {
+ define(new (alloc()) LValue(opd->toConstant()->toJSValue()), box,
+ LDefinition(LDefinition::BOX));
+ } else {
+ LBox* ins = new (alloc()) LBox(useRegister(opd), opd->type());
+ define(ins, box, LDefinition(LDefinition::BOX));
+ }
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+ MDefinition* box = unbox->getOperand(0);
+ MOZ_ASSERT(box->type() == MIRType::Value);
+
+ LUnboxBase* lir;
+ if (IsFloatingPointType(unbox->type())) {
+ lir = new (alloc())
+ LUnboxFloatingPoint(useRegisterAtStart(box), unbox->type());
+ } else if (unbox->fallible()) {
+ // If the unbox is fallible, load the Value in a register first to
+ // avoid multiple loads.
+ lir = new (alloc()) LUnbox(useRegisterAtStart(box));
+ } else {
+ lir = new (alloc()) LUnbox(useAtStart(box));
+ }
+
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+
+ define(lir, unbox);
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new (alloc()) LReturn(isGenerator);
+ ins->setOperand(0, useFixed(opd, JSReturnReg));
+ add(ins);
+}
+
+void LIRGeneratorX64::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void LIRGeneratorX64::defineInt64Phi(MPhi* phi, size_t lirIndex) {
+ defineTypedPhi(phi, lirIndex);
+}
+
+void LIRGeneratorX64::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ lowerTypedPhiInput(phi, inputPosition, block, lirIndex);
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LUse oldval = useRegister(ins->oldval());
+ LUse newval = useRegister(ins->newval());
+ LInt64Definition temp1 = tempInt64Fixed(Register64(rax));
+ LInt64Definition temp2 = tempInt64();
+
+ auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+ elements, index, oldval, newval, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerCompareExchangeTypedArrayElement(ins,
+ /* useI386ByteRegisters = */ false);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LAllocation value = useRegister(ins->value());
+ LInt64Definition temp1 = tempInt64();
+ LDefinition temp2 = temp();
+
+ auto* lir = new (alloc()) LAtomicExchangeTypedArrayElement64(
+ elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerAtomicExchangeTypedArrayElement(ins, /* useI386ByteRegisters = */ false);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LAllocation value = useRegister(ins->value());
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We can omit allocating the result BigInt.
+
+ if (ins->isForEffect()) {
+ LInt64Definition temp = tempInt64();
+
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+ elements, index, value, temp);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+ //
+ // For ADD and SUB we'll use XADD.
+ //
+ // For AND/OR/XOR we need to use a CMPXCHG loop with rax as a temp register.
+
+ bool bitOp = !(ins->operation() == AtomicFetchAddOp ||
+ ins->operation() == AtomicFetchSubOp);
+
+ LInt64Definition temp1 = tempInt64();
+ LInt64Definition temp2;
+ if (bitOp) {
+ temp2 = tempInt64Fixed(Register64(rax));
+ } else {
+ temp2 = tempInt64();
+ }
+
+ auto* lir = new (alloc())
+ LAtomicTypedArrayElementBinop64(elements, index, value, temp1, temp2);
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerAtomicTypedArrayElementBinop(ins, /* useI386ByteRegisters = */ false);
+}
+
+void LIRGeneratorX64::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+ auto* lir = new (alloc()) LAtomicLoad64(elements, index, temp(), tempInt64());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value = useRegister(ins->value());
+
+ add(new (alloc()) LAtomicStore64(elements, index, value, tempInt64()), ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToDouble* lir =
+ new (alloc()) LWasmUint32ToDouble(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToFloat32* lir =
+ new (alloc()) LWasmUint32ToFloat32(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) {
+ auto* lir = new (alloc()) LWasmHeapBase(LAllocation());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
+ MDefinition* base = ins->base();
+ // 'base' is a GPR but may be of either type. If it is 32-bit it is
+ // zero-extended and can act as 64-bit.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ if (ins->type() != MIRType::Int64) {
+ auto* lir = new (alloc()) LWasmLoad(useRegisterOrZeroAtStart(base));
+ define(lir, ins);
+ return;
+ }
+
+ auto* lir = new (alloc()) LWasmLoadI64(useRegisterOrZeroAtStart(base));
+ defineInt64(lir, ins);
+}
+
+void LIRGenerator::visitWasmStore(MWasmStore* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ MDefinition* value = ins->value();
+ LAllocation valueAlloc;
+ switch (ins->access().type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ valueAlloc = useRegisterOrConstantAtStart(value);
+ break;
+ case Scalar::Int64:
+ // No way to encode an int64-to-memory move on x64.
+ if (value->isConstant() && value->type() != MIRType::Int64) {
+ valueAlloc = useOrConstantAtStart(value);
+ } else {
+ valueAlloc = useRegisterAtStart(value);
+ }
+ break;
+ case Scalar::Float32:
+ case Scalar::Float64:
+ valueAlloc = useRegisterAtStart(value);
+ break;
+ case Scalar::Simd128:
+#ifdef ENABLE_WASM_SIMD
+ valueAlloc = useRegisterAtStart(value);
+ break;
+#else
+ MOZ_CRASH("unexpected array type");
+#endif
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::Uint8Clamped:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ LAllocation baseAlloc = useRegisterOrZeroAtStart(base);
+ auto* lir = new (alloc()) LWasmStore(baseAlloc, valueAlloc);
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ // The output may not be used but will be clobbered regardless, so
+ // pin the output to eax.
+ //
+ // The input values must both be in registers.
+
+ const LAllocation oldval = useRegister(ins->oldValue());
+ const LAllocation newval = useRegister(ins->newValue());
+
+ LWasmCompareExchangeHeap* lir =
+ new (alloc()) LWasmCompareExchangeHeap(useRegister(base), oldval, newval);
+
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+}
+
+void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(ins->base()->type() == MIRType::Int32 ||
+ ins->base()->type() == MIRType::Int64);
+
+ const LAllocation base = useRegister(ins->base());
+ const LAllocation value = useRegister(ins->value());
+
+ // The output may not be used but will be clobbered regardless,
+ // so ignore the case where we're not using the value and just
+ // use the output register as a temp.
+
+ LWasmAtomicExchangeHeap* lir =
+ new (alloc()) LWasmAtomicExchangeHeap(base, value);
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
+ MDefinition* base = ins->base();
+ // See comment in visitWasmLoad re the type of 'base'.
+ MOZ_ASSERT(base->type() == MIRType::Int32 || base->type() == MIRType::Int64);
+
+ // No support for 64-bit operations with constants at the masm level.
+
+ bool canTakeConstant = ins->access().type() != Scalar::Int64;
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We'll emit a single instruction: LOCK ADD, LOCK SUB, LOCK AND,
+ // LOCK OR, or LOCK XOR.
+
+ if (!ins->hasUses()) {
+ LAllocation value = canTakeConstant ? useRegisterOrConstant(ins->value())
+ : useRegister(ins->value());
+ LWasmAtomicBinopHeapForEffect* lir =
+ new (alloc()) LWasmAtomicBinopHeapForEffect(useRegister(base), value);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+ //
+ // For ADD and SUB we'll use XADD with word and byte ops as
+ // appropriate. Any output register can be used and if value is a
+ // register it's best if it's the same as output:
+ //
+ // movl value, output ; if value != output
+ // lock xaddl output, mem
+ //
+ // For AND/OR/XOR we need to use a CMPXCHG loop, and the output is
+ // always in rax:
+ //
+ // movl *mem, rax
+ // L: mov rax, temp
+ // andl value, temp
+ // lock cmpxchg temp, mem ; reads rax also
+ // jnz L
+ // ; result in rax
+ //
+ // Note the placement of L, cmpxchg will update rax with *mem if
+ // *mem does not have the expected value, so reloading it at the
+ // top of the loop would be redundant.
+
+ bool bitOp = !(ins->operation() == AtomicFetchAddOp ||
+ ins->operation() == AtomicFetchSubOp);
+ bool reuseInput = false;
+ LAllocation value;
+
+ if (bitOp || ins->value()->isConstant()) {
+ value = canTakeConstant ? useRegisterOrConstant(ins->value())
+ : useRegister(ins->value());
+ } else {
+ reuseInput = true;
+ value = useRegisterAtStart(ins->value());
+ }
+
+ auto* lir = new (alloc()) LWasmAtomicBinopHeap(
+ useRegister(base), value, bitOp ? temp() : LDefinition::BogusTemp());
+
+ if (reuseInput) {
+ defineReuseInput(lir, ins, LWasmAtomicBinopHeap::valueOp);
+ } else if (bitOp) {
+ defineFixed(lir, ins, LAllocation(AnyRegister(rax)));
+ } else {
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitSubstr(MSubstr* ins) {
+ LSubstr* lir = new (alloc())
+ LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
+ useRegister(ins->length()), temp(), temp(), tempByteOpRegister());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerDivI64(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDivI64(div);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+ useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(rdx));
+ defineInt64Fixed(lir, div, LInt64Allocation(LAllocation(AnyRegister(rax))));
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ MOZ_CRASH("We don't use runtime div for this architecture");
+}
+
+void LIRGeneratorX64::lowerModI64(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUModI64(mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc()) LDivOrModI64(
+ useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(rax));
+ defineInt64Fixed(lir, mod, LInt64Allocation(LAllocation(AnyRegister(rdx))));
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ MOZ_CRASH("We don't use runtime mod for this architecture");
+}
+
+void LIRGeneratorX64::lowerUDivI64(MDiv* div) {
+ LUDivOrModI64* lir = new (alloc()) LUDivOrModI64(
+ useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(rdx));
+ defineInt64Fixed(lir, div, LInt64Allocation(LAllocation(AnyRegister(rax))));
+}
+
+void LIRGeneratorX64::lowerUModI64(MMod* mod) {
+ LUDivOrModI64* lir = new (alloc()) LUDivOrModI64(
+ useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(rax));
+ defineInt64Fixed(lir, mod, LInt64Allocation(LAllocation(AnyRegister(rdx))));
+}
+
+void LIRGeneratorX64::lowerBigIntDiv(MBigIntDiv* ins) {
+ auto* lir = new (alloc()) LBigIntDiv(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(rax), temp());
+ defineFixed(lir, ins, LAllocation(AnyRegister(rdx)));
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX64::lowerBigIntMod(MBigIntMod* ins) {
+ auto* lir = new (alloc()) LBigIntMod(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(rax), temp());
+ defineFixed(lir, ins, LAllocation(AnyRegister(rdx)));
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ LDefinition maybeTemp =
+ ins->isUnsigned() ? tempDouble() : LDefinition::BogusTemp();
+ defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd), maybeTemp),
+ ins);
+}
+
+void LIRGeneratorX64::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Int64);
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+ LDefinition maybeTemp = ins->isUnsigned() ? temp() : LDefinition::BogusTemp();
+ define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd), maybeTemp),
+ ins);
+}
+
+void LIRGeneratorX64::lowerBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
+ defineInt64(new (alloc()) LExtendInt32ToInt64(useAtStart(ins->input())), ins);
+}
+
+void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
+ defineInt64(new (alloc())
+ LSignExtendInt64(useInt64RegisterAtStart(ins->input())),
+ ins);
+}
+
+// On x64 we specialize the cases: compare is {U,}Int{32,64}, and select is
+// {U,}Int{32,64}, independently.
+bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
+ MCompare::CompareType compTy, MIRType insTy) {
+ return (insTy == MIRType::Int32 || insTy == MIRType::Int64) &&
+ (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32 ||
+ compTy == MCompare::Compare_Int64 ||
+ compTy == MCompare::Compare_UInt64);
+}
+
+void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
+ MDefinition* lhs,
+ MDefinition* rhs,
+ MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+ auto* lir = new (alloc()) LWasmCompareAndSelect(
+ useRegister(lhs), useAny(rhs), compTy, jsop,
+ useRegisterAtStart(ins->trueExpr()), useAny(ins->falseExpr()));
+ defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
+}
diff --git a/js/src/jit/x64/Lowering-x64.h b/js/src/jit/x64/Lowering-x64.h
new file mode 100644
index 0000000000..1c34ea8693
--- /dev/null
+++ b/js/src/jit/x64/Lowering-x64.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_Lowering_x64_h
+#define jit_x64_Lowering_x64_h
+
+#include "jit/x86-shared/Lowering-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorX64 : public LIRGeneratorX86Shared {
+ protected:
+ LIRGeneratorX64(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorX86Shared(gen, graph, lirGraph) {}
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void defineInt64Phi(MPhi* phi, size_t lirIndex);
+
+ void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* input);
+ void lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+ void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs,
+ MDefinition* rhs);
+
+ // Returns a box allocation. reg2 is ignored on 64-bit platforms.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register,
+ bool useAtStart = false);
+
+ // x86 has constraints on what registers can be formatted for 1-byte
+ // stores and loads; on x64 all registers are okay.
+ LAllocation useByteOpRegister(MDefinition* mir);
+ LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+ LDefinition tempByteOpRegister();
+
+ LDefinition tempToUnbox();
+
+ bool needTempForPostBarrier() { return true; }
+
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+ void lowerDivI64(MDiv* div);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+ void lowerModI64(MMod* mod);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+ void lowerUDivI64(MDiv* div);
+ void lowerUModI64(MMod* mod);
+
+ void lowerBigIntDiv(MBigIntDiv* ins);
+ void lowerBigIntMod(MBigIntMod* ins);
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+ void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+};
+
+using LIRGeneratorSpecific = LIRGeneratorX64;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_Lowering_x64_h */
diff --git a/js/src/jit/x64/MacroAssembler-x64-inl.h b/js/src/jit/x64/MacroAssembler-x64-inl.h
new file mode 100644
index 0000000000..6869e6c4b6
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64-inl.h
@@ -0,0 +1,1099 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_MacroAssembler_x64_inl_h
+#define jit_x64_MacroAssembler_x64_inl_h
+
+#include "jit/x64/MacroAssembler-x64.h"
+
+#include "jit/x86-shared/MacroAssembler-x86-shared-inl.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+// ===============================================================
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+ // Use mov instead of movq because it has special optimizations for imm == 0.
+ mov(ImmWord(imm.value), dest.reg);
+}
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+ movq(src.reg, dest.reg);
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ vmovq(src, dest.reg);
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ vmovq(src.reg, dest);
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+ movl(src.reg, dest);
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ movl(src, dest.reg);
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ movsbq(Operand(src), dest.reg);
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ movswq(Operand(src), dest.reg);
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ movslq(src, dest.reg);
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ movslq(src, dest);
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ movl(src, dest);
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ movslq(Operand(src), dest);
+}
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::notPtr(Register reg) { notq(reg); }
+
+void MacroAssembler::andPtr(Register src, Register dest) { andq(src, dest); }
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) { andq(imm, dest); }
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+ if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+ andq(Imm32(imm.value), dest.reg);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ movq(ImmWord(uintptr_t(imm.value)), scratch);
+ andq(scratch, dest.reg);
+ }
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+ if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+ orq(Imm32(imm.value), dest.reg);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ movq(ImmWord(uintptr_t(imm.value)), scratch);
+ orq(scratch, dest.reg);
+ }
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+ if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+ xorq(Imm32(imm.value), dest.reg);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ movq(ImmWord(uintptr_t(imm.value)), scratch);
+ xorq(scratch, dest.reg);
+ }
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) { orq(src, dest); }
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) { orq(imm, dest); }
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+ andq(src.reg, dest.reg);
+}
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+ orq(src.reg, dest.reg);
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+ xorq(src.reg, dest.reg);
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) { xorq(src, dest); }
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) { xorq(imm, dest); }
+
+void MacroAssembler::and64(const Operand& src, Register64 dest) {
+ andq(src, dest.reg);
+}
+
+void MacroAssembler::or64(const Operand& src, Register64 dest) {
+ orq(src, dest.reg);
+}
+
+void MacroAssembler::xor64(const Operand& src, Register64 dest) {
+ xorq(src, dest.reg);
+}
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap64(Register64 reg) { bswapq(reg.reg); }
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(Register src, Register dest) { addq(src, dest); }
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) { addq(imm, dest); }
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(dest != scratch);
+ if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+ addq(Imm32((int32_t)imm.value), dest);
+ } else {
+ mov(imm, scratch);
+ addq(scratch, dest);
+ }
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
+ addq(imm, Operand(dest));
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const AbsoluteAddress& dest) {
+ addq(imm, Operand(dest));
+}
+
+void MacroAssembler::addPtr(const Address& src, Register dest) {
+ addq(Operand(src), dest);
+}
+
+void MacroAssembler::add64(const Operand& src, Register64 dest) {
+ addq(src, dest.reg);
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+ addq(src.reg, dest.reg);
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) { addq(imm, dest.reg); }
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+ addPtr(ImmWord(imm.value), dest.reg);
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ moveStackPtrTo(dest);
+ addqWithPatch(Imm32(0), dest);
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ patchAddq(offset, -imm.value);
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) { subq(src, dest); }
+
+void MacroAssembler::subPtr(Register src, const Address& dest) {
+ subq(src, Operand(dest));
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) { subq(imm, dest); }
+
+void MacroAssembler::subPtr(ImmWord imm, Register dest) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(dest != scratch);
+ if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+ subq(Imm32((int32_t)imm.value), dest);
+ } else {
+ mov(imm, scratch);
+ subq(scratch, dest);
+ }
+}
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) {
+ subq(Operand(addr), dest);
+}
+
+void MacroAssembler::sub64(const Operand& src, Register64 dest) {
+ subq(src, dest.reg);
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+ subq(src.reg, dest.reg);
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+ subPtr(ImmWord(imm.value), dest.reg);
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ // To compute the unsigned multiplication using imulq, we have to ensure both
+ // operands don't have any bits set in the high word.
+
+ if (imm.value >= 0) {
+ // Clear the high word of |src|.
+ movl(src, src);
+
+ // |imm| and |src| are both positive, so directly perform imulq.
+ imulq(imm, src, dest);
+ } else {
+ // Store the low word of |src| into |dest|.
+ movl(src, dest);
+
+ // Compute the unsigned value of |imm| before performing imulq.
+ movl(imm, ScratchReg);
+ imulq(ScratchReg, dest);
+ }
+
+ // Move the high word into |dest|.
+ shrq(Imm32(32), dest);
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+ imulq(rhs, srcDest);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ mul64(imm, dest);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+ if (INT32_MIN <= int64_t(imm.value) && int64_t(imm.value) <= INT32_MAX) {
+ imulq(Imm32((int32_t)imm.value), dest.reg, dest.reg);
+ } else {
+ movq(ImmWord(uintptr_t(imm.value)), ScratchReg);
+ imulq(ScratchReg, dest.reg);
+ }
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ mul64(Operand(src.reg), dest);
+}
+
+void MacroAssembler::mul64(const Operand& src, const Register64& dest) {
+ imulq(src, dest.reg);
+}
+
+void MacroAssembler::mul64(const Operand& src, const Register64& dest,
+ const Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ mul64(src, dest);
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+ lea(Operand(src, src, TimesTwo), dest);
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+ FloatRegister dest) {
+ movq(imm, ScratchReg);
+ vmulsd(Operand(ScratchReg, 0), dest, dest);
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+ if (X86Encoding::IsAddressImmediate(dest.addr)) {
+ addPtr(Imm32(1), dest);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(dest.addr), scratch);
+ addPtr(Imm32(1), Address(scratch, 0));
+ }
+}
+
+void MacroAssembler::neg64(Register64 reg) { negq(reg.reg); }
+
+void MacroAssembler::negPtr(Register reg) { negq(reg); }
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ shlq(imm, dest);
+}
+
+void MacroAssembler::lshiftPtr(Register shift, Register srcDest) {
+ if (Assembler::HasBMI2()) {
+ shlxq(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ shlq_cl(srcDest);
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ lshiftPtr(imm, dest.reg);
+}
+
+void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
+ if (Assembler::HasBMI2()) {
+ shlxq(srcDest.reg, shift, srcDest.reg);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ shlq_cl(srcDest.reg);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ shrq(imm, dest);
+}
+
+void MacroAssembler::rshiftPtr(Register shift, Register srcDest) {
+ if (Assembler::HasBMI2()) {
+ shrxq(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ shrq_cl(srcDest);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+ rshiftPtr(imm, dest.reg);
+}
+
+void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
+ if (Assembler::HasBMI2()) {
+ shrxq(srcDest.reg, shift, srcDest.reg);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ shrq_cl(srcDest.reg);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ sarq(imm, dest);
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ rshiftPtrArithmetic(imm, dest.reg);
+}
+
+void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
+ if (Assembler::HasBMI2()) {
+ sarxq(srcDest.reg, shift, srcDest.reg);
+ return;
+ }
+ MOZ_ASSERT(shift == rcx);
+ sarq_cl(srcDest.reg);
+}
+
+// ===============================================================
+// Rotation functions
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+ Register64 dest) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+
+ rolq_cl(dest.reg);
+}
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ rotateLeft64(count, src, dest);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+ Register64 dest) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+
+ rorq_cl(dest.reg);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ rotateRight64(count, src, dest);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src,
+ Register64 dest) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ rolq(count, dest.reg);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ rotateLeft64(count, src, dest);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src,
+ Register64 dest) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ rorq(count, dest.reg);
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ rotateRight64(count, src, dest);
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ cmpPtrSet(cond, lhs, ImmWord(static_cast<uintptr_t>(rhs.value)), dest);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ cmpPtr(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+ if (AssemblerX86Shared::HasLZCNT()) {
+ lzcntq(src.reg, dest);
+ return;
+ }
+
+ Label nonzero;
+ bsrq(src.reg, dest);
+ j(Assembler::NonZero, &nonzero);
+ movq(ImmWord(0x7F), dest);
+ bind(&nonzero);
+ xorq(Imm32(0x3F), dest);
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+ if (AssemblerX86Shared::HasBMI1()) {
+ tzcntq(src.reg, dest);
+ return;
+ }
+
+ Label nonzero;
+ bsfq(src.reg, dest);
+ j(Assembler::NonZero, &nonzero);
+ movq(ImmWord(64), dest);
+ bind(&nonzero);
+}
+
+void MacroAssembler::popcnt64(Register64 src64, Register64 dest64,
+ Register tmp) {
+ Register src = src64.reg;
+ Register dest = dest64.reg;
+
+ if (AssemblerX86Shared::HasPOPCNT()) {
+ MOZ_ASSERT(tmp == InvalidReg);
+ popcntq(src, dest);
+ return;
+ }
+
+ if (src != dest) {
+ movq(src, dest);
+ }
+
+ MOZ_ASSERT(tmp != dest);
+
+ ScratchRegisterScope scratch(*this);
+
+ // Equivalent to mozilla::CountPopulation32, adapted for 64 bits.
+ // x -= (x >> 1) & m1;
+ movq(src, tmp);
+ movq(ImmWord(0x5555555555555555), scratch);
+ shrq(Imm32(1), tmp);
+ andq(scratch, tmp);
+ subq(tmp, dest);
+
+ // x = (x & m2) + ((x >> 2) & m2);
+ movq(dest, tmp);
+ movq(ImmWord(0x3333333333333333), scratch);
+ andq(scratch, dest);
+ shrq(Imm32(2), tmp);
+ andq(scratch, tmp);
+ addq(tmp, dest);
+
+ // x = (x + (x >> 4)) & m4;
+ movq(dest, tmp);
+ movq(ImmWord(0x0f0f0f0f0f0f0f0f), scratch);
+ shrq(Imm32(4), tmp);
+ addq(tmp, dest);
+ andq(scratch, dest);
+
+ // (x * h01) >> 56
+ movq(ImmWord(0x0101010101010101), scratch);
+ imulq(scratch, dest);
+ shrq(Imm32(56), dest);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ branch32(cond, Operand(lhs), rhs, label);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(lhs.addr), scratch);
+ branch32(cond, Address(scratch, 0), rhs, label);
+ }
+}
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ branch32(cond, Operand(lhs), rhs, label);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(lhs.addr), scratch);
+ branch32(cond, Address(scratch, 0), rhs, label);
+ }
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
+ Imm32 rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+ mov(lhs, scratch);
+ branch32(cond, Address(scratch, 0), rhs, label);
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan ||
+ cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan ||
+ cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, ImmWord(val.value), success);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal ||
+ cond == Assembler::LessThan ||
+ cond == Assembler::LessThanOrEqual ||
+ cond == Assembler::GreaterThan ||
+ cond == Assembler::GreaterThanOrEqual ||
+ cond == Assembler::Below || cond == Assembler::BelowOrEqual ||
+ cond == Assembler::Above || cond == Assembler::AboveOrEqual,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs.reg, rhs.reg, success);
+ if (fail) {
+ jump(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, ImmWord(val.value), label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ branchPtr(cond, lhs, rhs.reg, label);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ loadPtr(rhs, scratch);
+ branchPtr(cond, lhs, scratch, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(rhs != scratch);
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ branchPtrImpl(cond, Operand(lhs), rhs, label);
+ } else {
+ mov(ImmPtr(lhs.addr), scratch);
+ branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+ }
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ ImmWord rhs, Label* label) {
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ branchPtrImpl(cond, Operand(lhs), rhs, label);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(lhs.addr), scratch);
+ branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+ }
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+ Register rhs, Label* label) {
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(rhs != scratch);
+ mov(lhs, scratch);
+ branchPtrImpl(cond, Operand(scratch, 0x0), rhs, label);
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ branchPtr(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchTruncateFloat32ToPtr(FloatRegister src,
+ Register dest, Label* fail) {
+ vcvttss2sq(src, dest);
+
+ // Same trick as for Doubles
+ cmpPtr(dest, Imm32(1));
+ j(Assembler::Overflow, fail);
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ branchTruncateFloat32ToPtr(src, dest, fail);
+ movl(dest, dest); // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ branchTruncateFloat32ToPtr(src, dest, fail);
+
+ // Check that the result is in the int32_t range.
+ ScratchRegisterScope scratch(*this);
+ move32To64SignExtend(dest, Register64(scratch));
+ cmpPtr(dest, scratch);
+ j(Assembler::NotEqual, fail);
+
+ movl(dest, dest); // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateDoubleToPtr(FloatRegister src, Register dest,
+ Label* fail) {
+ vcvttsd2sq(src, dest);
+
+ // vcvttsd2sq returns 0x8000000000000000 on failure. Test for it by
+ // subtracting 1 and testing overflow (this avoids the need to
+ // materialize that value in a register).
+ cmpPtr(dest, Imm32(1));
+ j(Assembler::Overflow, fail);
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ branchTruncateDoubleToPtr(src, dest, fail);
+ movl(dest, dest); // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ branchTruncateDoubleToPtr(src, dest, fail);
+
+ // Check that the result is in the int32_t range.
+ ScratchRegisterScope scratch(*this);
+ move32To64SignExtend(dest, Register64(scratch));
+ cmpPtr(dest, scratch);
+ j(Assembler::NotEqual, fail);
+
+ movl(dest, dest); // Zero upper 32-bits.
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ if (X86Encoding::IsAddressImmediate(lhs.addr)) {
+ test32(Operand(lhs), rhs);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ mov(ImmPtr(lhs.addr), scratch);
+ test32(Operand(scratch, 0), rhs);
+ }
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ branchTestPtr(cond, lhs.reg, rhs.reg, label);
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ test32(value.valueReg(), value.valueReg());
+ j(truthy ? NonZero : Zero, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ uint64_t magic = MagicValue(why).asRawBits();
+ cmpPtr(valaddr, ImmWord(magic));
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ branchPtr(cond, lhs, rhs.valueReg(), label);
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& address) {
+ jmp(Operand(address));
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmpPtr(lhs, rhs);
+ cmovCCq(cond, src, dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ cmpPtr(lhs, Operand(rhs));
+ cmovCCq(cond, src, dest);
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ cmp32(lhs, rhs);
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ cmp32(lhs, rhs);
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+ Imm32 mask, const Address& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+ Imm32 mask, Register src, Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+ Register dest) {
+ cmovCCq(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(length != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(index != scratch);
+ MOZ_ASSERT(length != scratch);
+
+ if (JitOptions.spectreIndexMasking) {
+ move32(Imm32(0), scratch);
+ }
+
+ cmp32(index, length);
+ j(Assembler::AboveOrEqual, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCl(Assembler::AboveOrEqual, scratch, index);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(index != scratch);
+ MOZ_ASSERT(length.base != scratch);
+
+ if (JitOptions.spectreIndexMasking) {
+ move32(Imm32(0), scratch);
+ }
+
+ cmp32(index, Operand(length));
+ j(Assembler::AboveOrEqual, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCl(Assembler::AboveOrEqual, scratch, index);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(length != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(index != scratch);
+ MOZ_ASSERT(length != scratch);
+
+ if (JitOptions.spectreIndexMasking) {
+ movePtr(ImmWord(0), scratch);
+ }
+
+ cmpPtr(index, length);
+ j(Assembler::AboveOrEqual, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCq(Assembler::AboveOrEqual, scratch, index);
+ }
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+ const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(index != scratch);
+ MOZ_ASSERT(length.base != scratch);
+
+ if (JitOptions.spectreIndexMasking) {
+ movePtr(ImmWord(0), scratch);
+ }
+
+ cmpPtr(index, Operand(length));
+ j(Assembler::AboveOrEqual, failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCq(Assembler::AboveOrEqual, scratch, index);
+ }
+}
+
+// ========================================================================
+// SIMD.
+
+// Extract lane as scalar
+
+void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src,
+ Register64 dest) {
+ if (lane == 0) {
+ vmovq(src, dest.reg);
+ } else {
+ vpextrq(lane, src, dest.reg);
+ }
+}
+
+// Replace lane value
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs,
+ FloatRegister lhsDest) {
+ vpinsrq(lane, rhs.reg, lhsDest, lhsDest);
+}
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, FloatRegister lhs,
+ Register64 rhs, FloatRegister dest) {
+ vpinsrq(lane, rhs.reg, lhs, dest);
+}
+
+// Splat
+
+void MacroAssembler::splatX2(Register64 src, FloatRegister dest) {
+ vmovq(src.reg, dest);
+ if (HasAVX2()) {
+ vbroadcastq(Operand(dest), dest);
+ } else {
+ vpunpcklqdq(dest, dest, dest);
+ }
+}
+
+// ========================================================================
+// Truncate floating point.
+
+void MacroAssembler::truncateFloat32ToUInt64(Address src, Address dest,
+ Register temp,
+ FloatRegister floatTemp) {
+ Label done;
+
+ loadFloat32(src, floatTemp);
+
+ truncateFloat32ToInt64(src, dest, temp);
+
+ // For unsigned conversion the case of [INT64, UINT64] needs to get handled
+ // separately.
+ loadPtr(dest, temp);
+ branchPtr(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
+
+ // Move the value inside INT64 range.
+ storeFloat32(floatTemp, dest);
+ loadConstantFloat32(double(int64_t(0x8000000000000000)), floatTemp);
+ vaddss(Operand(dest), floatTemp, floatTemp);
+ storeFloat32(floatTemp, dest);
+ truncateFloat32ToInt64(dest, dest, temp);
+
+ loadPtr(dest, temp);
+ or64(Imm64(0x8000000000000000), Register64(temp));
+ storePtr(temp, dest);
+
+ bind(&done);
+}
+
+void MacroAssembler::truncateDoubleToUInt64(Address src, Address dest,
+ Register temp,
+ FloatRegister floatTemp) {
+ Label done;
+
+ loadDouble(src, floatTemp);
+
+ truncateDoubleToInt64(src, dest, temp);
+
+ // For unsigned conversion the case of [INT64, UINT64] needs to get handle
+ // seperately.
+ loadPtr(dest, temp);
+ branchPtr(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
+
+ // Move the value inside INT64 range.
+ storeDouble(floatTemp, dest);
+ loadConstantDouble(double(int64_t(0x8000000000000000)), floatTemp);
+ vaddsd(Operand(dest), floatTemp, floatTemp);
+ storeDouble(floatTemp, dest);
+ truncateDoubleToInt64(dest, dest, temp);
+
+ loadPtr(dest, temp);
+ or64(Imm64(0x8000000000000000), Register64(temp));
+ storePtr(temp, dest);
+
+ bind(&done);
+}
+
+void MacroAssemblerX64::fallibleUnboxPtrImpl(const Operand& src, Register dest,
+ JSValueType type, Label* fail) {
+ MOZ_ASSERT(type == JSVAL_TYPE_OBJECT || type == JSVAL_TYPE_STRING ||
+ type == JSVAL_TYPE_SYMBOL || type == JSVAL_TYPE_BIGINT);
+ // dest := src XOR mask
+ // scratch := dest >> JSVAL_TAG_SHIFT
+ // fail if scratch != 0
+ //
+ // Note: src and dest can be the same register.
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ xorq(src, scratch);
+ mov(scratch, dest);
+ shrq(Imm32(JSVAL_TAG_SHIFT), scratch);
+ j(Assembler::NonZero, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(Operand(src.valueReg()), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(Operand(src), dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(Operand(src), dest, type, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+void MacroAssemblerX64::incrementInt32Value(const Address& addr) {
+ asMasm().addPtr(Imm32(1), addr);
+}
+
+void MacroAssemblerX64::unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType type) {
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.valueReg(), dest.fpu());
+ jump(&end);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else {
+ unboxNonDouble(src, dest.gpr(), type);
+ }
+}
+
+template <typename T>
+void MacroAssemblerX64::loadInt32OrDouble(const T& src, FloatRegister dest) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src, dest);
+ jump(&end);
+ bind(&notInt32);
+ unboxDouble(src, dest);
+ bind(&end);
+}
+
+// If source is a double, load it into dest. If source is int32,
+// convert it to double. Else, branch to failure.
+void MacroAssemblerX64::ensureDouble(const ValueOperand& source,
+ FloatRegister dest, Label* failure) {
+ Label isDouble, done;
+ {
+ ScratchTagScope tag(asMasm(), source);
+ splitTagForTest(source, tag);
+ asMasm().branchTestDouble(Assembler::Equal, tag, &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, tag, failure);
+ }
+
+ {
+ ScratchRegisterScope scratch(asMasm());
+ unboxInt32(source, scratch);
+ convertInt32ToDouble(scratch, dest);
+ }
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_MacroAssembler_x64_inl_h */
diff --git a/js/src/jit/x64/MacroAssembler-x64.cpp b/js/src/jit/x64/MacroAssembler-x64.cpp
new file mode 100644
index 0000000000..3513f93e33
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64.cpp
@@ -0,0 +1,1746 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x64/MacroAssembler-x64.h"
+
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MoveEmitter.h"
+#include "util/Memory.h"
+#include "vm/BigIntType.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/StringType.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void MacroAssemblerX64::loadConstantDouble(double d, FloatRegister dest) {
+ if (maybeInlineDouble(d, dest)) {
+ return;
+ }
+ Double* dbl = getDouble(d);
+ if (!dbl) {
+ return;
+ }
+ // The constants will be stored in a pool appended to the text (see
+ // finish()), so they will always be a fixed distance from the
+ // instructions which reference them. This allows the instructions to use
+ // PC-relative addressing. Use "jump" label support code, because we need
+ // the same PC-relative address patching that jumps use.
+ JmpSrc j = masm.vmovsd_ripr(dest.encoding());
+ propagateOOM(dbl->uses.append(j));
+}
+
+void MacroAssemblerX64::loadConstantFloat32(float f, FloatRegister dest) {
+ if (maybeInlineFloat(f, dest)) {
+ return;
+ }
+ Float* flt = getFloat(f);
+ if (!flt) {
+ return;
+ }
+ // See comment in loadConstantDouble
+ JmpSrc j = masm.vmovss_ripr(dest.encoding());
+ propagateOOM(flt->uses.append(j));
+}
+
+void MacroAssemblerX64::vpRiprOpSimd128(
+ const SimdConstant& v, FloatRegister reg,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID id)) {
+ SimdData* val = getSimdData(v);
+ if (!val) {
+ return;
+ }
+ JmpSrc j = (masm.*op)(reg.encoding());
+ propagateOOM(val->uses.append(j));
+}
+
+void MacroAssemblerX64::vpRiprOpSimd128(
+ const SimdConstant& v, FloatRegister src, FloatRegister dest,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID srcId, X86Encoding::XMMRegisterID destId)) {
+ SimdData* val = getSimdData(v);
+ if (!val) {
+ return;
+ }
+ JmpSrc j = (masm.*op)(src.encoding(), dest.encoding());
+ propagateOOM(val->uses.append(j));
+}
+
+void MacroAssemblerX64::loadConstantSimd128Int(const SimdConstant& v,
+ FloatRegister dest) {
+ if (maybeInlineSimd128Int(v, dest)) {
+ return;
+ }
+ vpRiprOpSimd128(v, dest, &X86Encoding::BaseAssemblerX64::vmovdqa_ripr);
+}
+
+void MacroAssemblerX64::loadConstantSimd128Float(const SimdConstant& v,
+ FloatRegister dest) {
+ if (maybeInlineSimd128Float(v, dest)) {
+ return;
+ }
+ vpRiprOpSimd128(v, dest, &X86Encoding::BaseAssemblerX64::vmovaps_ripr);
+}
+
+void MacroAssemblerX64::vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddb_ripr);
+}
+
+void MacroAssemblerX64::vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddw_ripr);
+}
+
+void MacroAssemblerX64::vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddd_ripr);
+}
+
+void MacroAssemblerX64::vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddq_ripr);
+}
+
+void MacroAssemblerX64::vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubb_ripr);
+}
+
+void MacroAssemblerX64::vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubw_ripr);
+}
+
+void MacroAssemblerX64::vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubd_ripr);
+}
+
+void MacroAssemblerX64::vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubq_ripr);
+}
+
+void MacroAssemblerX64::vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmullw_ripr);
+}
+
+void MacroAssemblerX64::vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmulld_ripr);
+}
+
+void MacroAssemblerX64::vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddsb_ripr);
+}
+
+void MacroAssemblerX64::vpaddusbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddusb_ripr);
+}
+
+void MacroAssemblerX64::vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddsw_ripr);
+}
+
+void MacroAssemblerX64::vpadduswSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpaddusw_ripr);
+}
+
+void MacroAssemblerX64::vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubsb_ripr);
+}
+
+void MacroAssemblerX64::vpsubusbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubusb_ripr);
+}
+
+void MacroAssemblerX64::vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubsw_ripr);
+}
+
+void MacroAssemblerX64::vpsubuswSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpsubusw_ripr);
+}
+
+void MacroAssemblerX64::vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsb_ripr);
+}
+
+void MacroAssemblerX64::vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminub_ripr);
+}
+
+void MacroAssemblerX64::vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsw_ripr);
+}
+
+void MacroAssemblerX64::vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminuw_ripr);
+}
+
+void MacroAssemblerX64::vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminsd_ripr);
+}
+
+void MacroAssemblerX64::vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpminud_ripr);
+}
+
+void MacroAssemblerX64::vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsb_ripr);
+}
+
+void MacroAssemblerX64::vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxub_ripr);
+}
+
+void MacroAssemblerX64::vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsw_ripr);
+}
+
+void MacroAssemblerX64::vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxuw_ripr);
+}
+
+void MacroAssemblerX64::vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxsd_ripr);
+}
+
+void MacroAssemblerX64::vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaxud_ripr);
+}
+
+void MacroAssemblerX64::vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpand_ripr);
+}
+
+void MacroAssemblerX64::vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpxor_ripr);
+}
+
+void MacroAssemblerX64::vporSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpor_ripr);
+}
+
+void MacroAssemblerX64::vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vaddps_ripr);
+}
+
+void MacroAssemblerX64::vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vaddpd_ripr);
+}
+
+void MacroAssemblerX64::vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vsubps_ripr);
+}
+
+void MacroAssemblerX64::vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vsubpd_ripr);
+}
+
+void MacroAssemblerX64::vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vdivps_ripr);
+}
+
+void MacroAssemblerX64::vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vdivpd_ripr);
+}
+
+void MacroAssemblerX64::vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vmulps_ripr);
+}
+
+void MacroAssemblerX64::vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vmulpd_ripr);
+}
+
+void MacroAssemblerX64::vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vandpd_ripr);
+}
+
+void MacroAssemblerX64::vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vminpd_ripr);
+}
+
+void MacroAssemblerX64::vpacksswbSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpacksswb_ripr);
+}
+
+void MacroAssemblerX64::vpackuswbSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackuswb_ripr);
+}
+
+void MacroAssemblerX64::vpackssdwSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackssdw_ripr);
+}
+
+void MacroAssemblerX64::vpackusdwSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpackusdw_ripr);
+}
+
+void MacroAssemblerX64::vpunpckldqSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest,
+ &X86Encoding::BaseAssemblerX64::vpunpckldq_ripr);
+}
+
+void MacroAssemblerX64::vunpcklpsSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vunpcklps_ripr);
+}
+
+void MacroAssemblerX64::vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpshufb_ripr);
+}
+
+void MacroAssemblerX64::vptestSimd128(const SimdConstant& v,
+ FloatRegister lhs) {
+ vpRiprOpSimd128(v, lhs, &X86Encoding::BaseAssemblerX64::vptest_ripr);
+}
+
+void MacroAssemblerX64::vpmaddwdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmaddwd_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqb_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtb_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqwSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqw_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtwSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtw_ripr);
+}
+
+void MacroAssemblerX64::vpcmpeqdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpeqd_ripr);
+}
+
+void MacroAssemblerX64::vpcmpgtdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpcmpgtd_ripr);
+}
+
+void MacroAssemblerX64::vcmpeqpsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpeqps_ripr);
+}
+
+void MacroAssemblerX64::vcmpneqpsSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpneqps_ripr);
+}
+
+void MacroAssemblerX64::vcmpltpsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpltps_ripr);
+}
+
+void MacroAssemblerX64::vcmplepsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpleps_ripr);
+}
+
+void MacroAssemblerX64::vcmpgepsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpgeps_ripr);
+}
+
+void MacroAssemblerX64::vcmpeqpdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpeqpd_ripr);
+}
+
+void MacroAssemblerX64::vcmpneqpdSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpneqpd_ripr);
+}
+
+void MacroAssemblerX64::vcmpltpdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmpltpd_ripr);
+}
+
+void MacroAssemblerX64::vcmplepdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vcmplepd_ripr);
+}
+
+void MacroAssemblerX64::vpmaddubswSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest,
+ &X86Encoding::BaseAssemblerX64::vpmaddubsw_ripr);
+}
+
+void MacroAssemblerX64::vpmuludqSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpRiprOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX64::vpmuludq_ripr);
+}
+
+void MacroAssemblerX64::bindOffsets(
+ const MacroAssemblerX86Shared::UsesVector& uses) {
+ for (JmpSrc src : uses) {
+ JmpDst dst(currentOffset());
+ // Using linkJump here is safe, as explained in the comment in
+ // loadConstantDouble.
+ masm.linkJump(src, dst);
+ }
+}
+
+void MacroAssemblerX64::finish() {
+ if (!doubles_.empty()) {
+ masm.haltingAlign(sizeof(double));
+ }
+ for (const Double& d : doubles_) {
+ bindOffsets(d.uses);
+ masm.doubleConstant(d.value);
+ }
+
+ if (!floats_.empty()) {
+ masm.haltingAlign(sizeof(float));
+ }
+ for (const Float& f : floats_) {
+ bindOffsets(f.uses);
+ masm.floatConstant(f.value);
+ }
+
+ // SIMD memory values must be suitably aligned.
+ if (!simds_.empty()) {
+ masm.haltingAlign(SimdMemoryAlignment);
+ }
+ for (const SimdData& v : simds_) {
+ bindOffsets(v.uses);
+ masm.simd128Constant(v.value.bytes());
+ }
+
+ MacroAssemblerX86Shared::finish();
+}
+
+void MacroAssemblerX64::boxValue(JSValueType type, Register src,
+ Register dest) {
+ MOZ_ASSERT(src != dest);
+
+ JSValueShiftedTag tag = (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
+#ifdef DEBUG
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ Label upper32BitsZeroed;
+ movePtr(ImmWord(UINT32_MAX), dest);
+ asMasm().branchPtr(Assembler::BelowOrEqual, src, dest, &upper32BitsZeroed);
+ breakpoint();
+ bind(&upper32BitsZeroed);
+ }
+#endif
+ mov(ImmShiftedTag(tag), dest);
+ orq(src, dest);
+}
+
+void MacroAssemblerX64::handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail) {
+ // Reserve space for exception information.
+ subq(Imm32(sizeof(ResumeFromException)), rsp);
+ movq(rsp, rax);
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(rcx);
+ asMasm().passABIArg(rax);
+ asMasm().callWithABI<Fn, HandleException>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ load32(Address(rsp, ResumeFromException::offsetOfKind()), rax);
+ asMasm().branch32(Assembler::Equal, rax,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, rax,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, rax,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, rax, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, rax,
+ Imm32(ExceptionResumeKind::WasmCatch), &wasmCatch);
+
+ breakpoint(); // Invalid kind.
+
+ // No exception handler. Load the error value, restore state and return from
+ // the entry frame.
+ bind(&entryFrame);
+ asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ ret();
+
+ // If we found a catch handler, this must be a baseline frame. Restore state
+ // and jump to the catch block.
+ bind(&catch_);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ jmp(Operand(rax));
+
+ // If we found a finally block, this must be a baseline frame. Push two
+ // values expected by the finally block: the exception and BooleanValue(true).
+ bind(&finally);
+ ValueOperand exception = ValueOperand(rcx);
+ loadValue(Address(esp, ResumeFromException::offsetOfException()), exception);
+
+ loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+
+ pushValue(exception);
+ pushValue(BooleanValue(true));
+ jmp(Operand(rax));
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ loadValue(Address(rbp, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ jmp(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(Address(rsp, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ movq(rbp, rsp);
+ pop(rbp);
+ ret();
+
+ // If we are bailing out to baseline to handle an exception, jump to the
+ // bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
+ bind(&bailout);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfBailoutInfo()), r9);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ move32(Imm32(1), ReturnReg);
+ jump(bailoutTail);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ masm.ret();
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfTarget()), rax);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfFramePointer()), rbp);
+ loadPtr(Address(rsp, ResumeFromException::offsetOfStackPointer()), rsp);
+ jmp(Operand(rax));
+}
+
+void MacroAssemblerX64::profilerEnterFrame(Register framePtr,
+ Register scratch) {
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr,
+ Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr),
+ Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerX64::profilerExitFrame() {
+ jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+Assembler::Condition MacroAssemblerX64::testStringTruthy(
+ bool truthy, const ValueOperand& value) {
+ ScratchRegisterScope scratch(asMasm());
+ unboxString(value, scratch);
+ cmp32(Operand(scratch, JSString::offsetOfLength()), Imm32(0));
+ return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+Assembler::Condition MacroAssemblerX64::testBigIntTruthy(
+ bool truthy, const ValueOperand& value) {
+ ScratchRegisterScope scratch(asMasm());
+ unboxBigInt(value, scratch);
+ cmp32(Operand(scratch, JS::BigInt::offsetOfDigitLength()), Imm32(0));
+ return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+MacroAssembler& MacroAssemblerX64::asMasm() {
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerX64::asMasm() const {
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) {
+ if (imm32.value) {
+ // On windows, we cannot skip very far down the stack without touching the
+ // memory pages in-between. This is a corner-case code for situations where
+ // the Ion frame data for a piece of code is very large. To handle this
+ // special case, for frames over 4k in size we allocate memory on the stack
+ // incrementally, touching it as we go.
+ //
+ // When the amount is quite large, which it can be, we emit an actual loop,
+ // in order to keep the function prologue compact. Compactness is a
+ // requirement for eg Wasm's CodeRange data structure, which can encode only
+ // 8-bit offsets.
+ uint32_t amountLeft = imm32.value;
+ uint32_t fullPages = amountLeft / 4096;
+ if (fullPages <= 8) {
+ while (amountLeft > 4096) {
+ subq(Imm32(4096), StackPointer);
+ store32(Imm32(0), Address(StackPointer, 0));
+ amountLeft -= 4096;
+ }
+ subq(Imm32(amountLeft), StackPointer);
+ } else {
+ ScratchRegisterScope scratch(*this);
+ Label top;
+ move32(Imm32(fullPages), scratch);
+ bind(&top);
+ subq(Imm32(4096), StackPointer);
+ store32(Imm32(0), Address(StackPointer, 0));
+ subl(Imm32(1), scratch);
+ j(Assembler::NonZero, &top);
+ amountLeft -= fullPages * 4096;
+ if (amountLeft) {
+ subq(Imm32(amountLeft), StackPointer);
+ }
+ }
+ }
+}
+
+void MacroAssemblerX64::convertDoubleToPtr(FloatRegister src, Register dest,
+ Label* fail,
+ bool negativeZeroCheck) {
+ // Check for -0.0
+ if (negativeZeroCheck) {
+ branchNegativeZero(src, dest, fail);
+ }
+
+ ScratchDoubleScope scratch(asMasm());
+ vcvttsd2sq(src, dest);
+ asMasm().convertInt64ToDouble(Register64(dest), scratch);
+ vucomisd(scratch, src);
+ j(Assembler::Parity, fail);
+ j(Assembler::NotEqual, fail);
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ movq(rsp, scratch);
+ andq(Imm32(~(ABIStackAlignment - 1)), rsp);
+ push(scratch);
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ if (dynamicAlignment_) {
+ // sizeof(intptr_t) accounts for the saved stack pointer pushed by
+ // setupUnalignedABICall.
+ stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t),
+ ABIStackAlignment);
+ } else {
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(
+ stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+ }
+
+ *stackAdjust = stackForCall;
+ reserveStack(stackForCall);
+
+ // Position all arguments.
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_) {
+ return;
+ }
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool cleanupArg) {
+ freeStack(stackAdjust);
+ if (dynamicAlignment_) {
+ pop(rsp);
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+static bool IsIntArgReg(Register reg) {
+ for (uint32_t i = 0; i < NumIntArgRegs; i++) {
+ if (IntArgRegs[i] == reg) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ if (IsIntArgReg(fun)) {
+ // Callee register may be clobbered for an argument. Move the callee to
+ // r10, a volatile, non-argument register.
+ propagateOOM(moveResolver_.addMove(MoveOperand(fun), MoveOperand(r10),
+ MoveOp::GENERAL));
+ fun = r10;
+ }
+
+ MOZ_ASSERT(!IsIntArgReg(fun));
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(fun);
+ callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ Address safeFun = fun;
+ if (IsIntArgReg(safeFun.base)) {
+ // Callee register may be clobbered for an argument. Move the callee to
+ // r10, a volatile, non-argument register.
+ propagateOOM(moveResolver_.addMove(MoveOperand(fun.base), MoveOperand(r10),
+ MoveOp::GENERAL));
+ safeFun.base = r10;
+ }
+
+ MOZ_ASSERT(!IsIntArgReg(safeFun.base));
+
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(safeFun);
+ callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ boxValue(ValueTypeFromMIRType(type), reg.gpr(), dest.valueReg());
+ return;
+ }
+
+ ScratchDoubleScope scratch(*this);
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ boxDouble(freg, dest, freg);
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ if (src == dest) {
+ return;
+ }
+ movq(src.valueReg(), dest.valueReg());
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ movWithPatch(ImmWord(src.asRawBits()), dest.valueReg());
+ writeDataRelocation(src);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+ if (ptr != buffer) {
+ movePtr(ptr, buffer);
+ }
+ andPtr(Imm32(int32_t(~gc::ChunkMask)), buffer);
+ loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+ Register temp, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(ptr != temp);
+ MOZ_ASSERT(ptr != scratch);
+
+ movePtr(ptr, scratch);
+ andPtr(Imm32(int32_t(~gc::ChunkMask)), scratch);
+ branchPtr(InvertCondition(cond), Address(scratch, gc::ChunkStoreBufferOffset),
+ ImmWord(0), label);
+}
+
+template <typename T>
+void MacroAssembler::branchValueIsNurseryCellImpl(Condition cond,
+ const T& value, Register temp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ MOZ_ASSERT(temp != InvalidReg);
+
+ Label done;
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+
+ getGCThingValueChunk(value, temp);
+ branchPtr(InvertCondition(cond), Address(temp, gc::ChunkStoreBufferOffset),
+ ImmWord(0), label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ branchValueIsNurseryCellImpl(cond, address, temp, label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ branchValueIsNurseryCellImpl(cond, value, temp, label);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ ScratchRegisterScope scratch(*this);
+ MOZ_ASSERT(lhs.valueReg() != scratch);
+ moveValue(rhs, ValueOperand(scratch));
+ cmpPtr(lhs.valueReg(), scratch);
+ j(cond, label);
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ boxDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ if (value.constant()) {
+ storeValue(value.value(), dest);
+ } else {
+ storeValue(ValueTypeFromMIRType(valueType), value.reg().typedReg().gpr(),
+ dest);
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+void MacroAssembler::PushBoxed(FloatRegister reg) {
+ subq(Imm32(sizeof(double)), StackPointer);
+ boxDouble(reg, Address(StackPointer, 0));
+ adjustFrame(sizeof(double));
+}
+
+// ========================================================================
+// wasm support
+
+void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
+ Operand srcAddr, AnyRegister out) {
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ memoryBarrierBefore(access.sync());
+
+ MOZ_ASSERT_IF(
+ access.isZeroExtendSimd128Load(),
+ access.type() == Scalar::Float32 || access.type() == Scalar::Float64);
+ MOZ_ASSERT_IF(
+ access.isSplatSimd128Load(),
+ access.type() == Scalar::Uint8 || access.type() == Scalar::Uint16 ||
+ access.type() == Scalar::Float32 || access.type() == Scalar::Float64);
+ MOZ_ASSERT_IF(access.isWidenSimd128Load(), access.type() == Scalar::Float64);
+
+ append(access, size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ movsbl(srcAddr, out.gpr());
+ break;
+ case Scalar::Uint8:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastb(srcAddr, out.fpu());
+ } else {
+ movzbl(srcAddr, out.gpr());
+ }
+ break;
+ case Scalar::Int16:
+ movswl(srcAddr, out.gpr());
+ break;
+ case Scalar::Uint16:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastw(srcAddr, out.fpu());
+ } else {
+ movzwl(srcAddr, out.gpr());
+ }
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ movl(srcAddr, out.gpr());
+ break;
+ case Scalar::Float32:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastss(srcAddr, out.fpu());
+ } else {
+ // vmovss does the right thing also for access.isZeroExtendSimd128Load()
+ vmovss(srcAddr, out.fpu());
+ }
+ break;
+ case Scalar::Float64:
+ if (access.isSplatSimd128Load()) {
+ vmovddup(srcAddr, out.fpu());
+ } else if (access.isWidenSimd128Load()) {
+ switch (access.widenSimdOp()) {
+ case wasm::SimdOp::V128Load8x8S:
+ vpmovsxbw(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load8x8U:
+ vpmovzxbw(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load16x4S:
+ vpmovsxwd(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load16x4U:
+ vpmovzxwd(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load32x2S:
+ vpmovsxdq(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load32x2U:
+ vpmovzxdq(srcAddr, out.fpu());
+ break;
+ default:
+ MOZ_CRASH("Unexpected widening op for wasmLoad");
+ }
+ } else {
+ // vmovsd does the right thing also for access.isZeroExtendSimd128Load()
+ vmovsd(srcAddr, out.fpu());
+ }
+ break;
+ case Scalar::Simd128:
+ MacroAssemblerX64::loadUnalignedSimd128(srcAddr, out.fpu());
+ break;
+ case Scalar::Int64:
+ MOZ_CRASH("int64 loads must use load64");
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::Uint8Clamped:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected scalar type for wasmLoad");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+ Operand srcAddr, Register64 out) {
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ memoryBarrierBefore(access.sync());
+
+ append(access, size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ movsbq(srcAddr, out.reg);
+ break;
+ case Scalar::Uint8:
+ movzbq(srcAddr, out.reg);
+ break;
+ case Scalar::Int16:
+ movswq(srcAddr, out.reg);
+ break;
+ case Scalar::Uint16:
+ movzwq(srcAddr, out.reg);
+ break;
+ case Scalar::Int32:
+ movslq(srcAddr, out.reg);
+ break;
+ // Int32 to int64 moves zero-extend by default.
+ case Scalar::Uint32:
+ movl(srcAddr, out.reg);
+ break;
+ case Scalar::Int64:
+ movq(srcAddr, out.reg);
+ break;
+ case Scalar::Float32:
+ case Scalar::Float64:
+ case Scalar::Simd128:
+ MOZ_CRASH("float loads must use wasmLoad");
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected scalar type for wasmLoadI64");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Operand dstAddr) {
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ memoryBarrierBefore(access.sync());
+
+ append(access, masm.size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ movb(value.gpr(), dstAddr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ movw(value.gpr(), dstAddr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ movl(value.gpr(), dstAddr);
+ break;
+ case Scalar::Int64:
+ movq(value.gpr(), dstAddr);
+ break;
+ case Scalar::Float32:
+ storeUncanonicalizedFloat32(value.fpu(), dstAddr);
+ break;
+ case Scalar::Float64:
+ storeUncanonicalizedDouble(value.fpu(), dstAddr);
+ break;
+ case Scalar::Simd128:
+ MacroAssemblerX64::storeUnalignedSimd128(value.fpu(), dstAddr);
+ break;
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ vcvttsd2sq(input, output);
+
+ // Check that the result is in the uint32_t range.
+ ScratchRegisterScope scratch(*this);
+ move32(Imm32(0xffffffff), scratch);
+ cmpq(scratch, output);
+ j(Assembler::Above, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ vcvttss2sq(input, output);
+
+ // Check that the result is in the uint32_t range.
+ ScratchRegisterScope scratch(*this);
+ move32(Imm32(0xffffffff), scratch);
+ cmpq(scratch, output);
+ j(Assembler::Above, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ vcvttsd2sq(input, output.reg);
+ cmpq(Imm32(1), output.reg);
+ j(Assembler::Overflow, oolEntry);
+ bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ vcvttss2sq(input, output.reg);
+ cmpq(Imm32(1), output.reg);
+ j(Assembler::Overflow, oolEntry);
+ bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ // If the input < INT64_MAX, vcvttsd2sq will do the right thing, so
+ // we use it directly. Else, we subtract INT64_MAX, convert to int64,
+ // and then add INT64_MAX to the result.
+
+ Label isLarge;
+
+ ScratchDoubleScope scratch(*this);
+ loadConstantDouble(double(0x8000000000000000), scratch);
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, input, scratch, &isLarge);
+ vcvttsd2sq(input, output.reg);
+ testq(output.reg, output.reg);
+ j(Assembler::Signed, oolEntry);
+ jump(oolRejoin);
+
+ bind(&isLarge);
+
+ moveDouble(input, tempReg);
+ vsubsd(scratch, tempReg, tempReg);
+ vcvttsd2sq(tempReg, output.reg);
+ testq(output.reg, output.reg);
+ j(Assembler::Signed, oolEntry);
+ or64(Imm64(0x8000000000000000), output);
+
+ bind(oolRejoin);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ // If the input < INT64_MAX, vcvttss2sq will do the right thing, so
+ // we use it directly. Else, we subtract INT64_MAX, convert to int64,
+ // and then add INT64_MAX to the result.
+
+ Label isLarge;
+
+ ScratchFloat32Scope scratch(*this);
+ loadConstantFloat32(float(0x8000000000000000), scratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, scratch, &isLarge);
+ vcvttss2sq(input, output.reg);
+ testq(output.reg, output.reg);
+ j(Assembler::Signed, oolEntry);
+ jump(oolRejoin);
+
+ bind(&isLarge);
+
+ moveFloat32(input, tempReg);
+ vsubss(scratch, tempReg, tempReg);
+ vcvttss2sq(tempReg, output.reg);
+ testq(output.reg, output.reg);
+ j(Assembler::Signed, oolEntry);
+ or64(Imm64(0x8000000000000000), output);
+
+ bind(oolRejoin);
+}
+
+void MacroAssembler::widenInt32(Register r) {
+ move32To64ZeroExtend(r, Register64(r));
+}
+
+// ========================================================================
+// Convert floating point.
+
+void MacroAssembler::convertInt64ToDouble(Register64 input,
+ FloatRegister output) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(output);
+
+ vcvtsq2sd(input.reg, output, output);
+}
+
+void MacroAssembler::convertInt64ToFloat32(Register64 input,
+ FloatRegister output) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroFloat32(output);
+
+ vcvtsq2ss(input.reg, output, output);
+}
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return true; }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 input,
+ FloatRegister output,
+ Register temp) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(output);
+
+ // If the input's sign bit is not set we use vcvtsq2sd directly.
+ // Else, we divide by 2 and keep the LSB, convert to double, and multiply
+ // the result by 2.
+ Label done;
+ Label isSigned;
+
+ testq(input.reg, input.reg);
+ j(Assembler::Signed, &isSigned);
+ vcvtsq2sd(input.reg, output, output);
+ jump(&done);
+
+ bind(&isSigned);
+
+ ScratchRegisterScope scratch(*this);
+ mov(input.reg, scratch);
+ mov(input.reg, temp);
+ shrq(Imm32(1), scratch);
+ andq(Imm32(1), temp);
+ orq(temp, scratch);
+
+ vcvtsq2sd(scratch, output, output);
+ vaddsd(output, output, output);
+
+ bind(&done);
+}
+
+void MacroAssembler::convertUInt64ToFloat32(Register64 input,
+ FloatRegister output,
+ Register temp) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroFloat32(output);
+
+ // See comment in convertUInt64ToDouble.
+ Label done;
+ Label isSigned;
+
+ testq(input.reg, input.reg);
+ j(Assembler::Signed, &isSigned);
+ vcvtsq2ss(input.reg, output, output);
+ jump(&done);
+
+ bind(&isSigned);
+
+ ScratchRegisterScope scratch(*this);
+ mov(input.reg, scratch);
+ mov(input.reg, temp);
+ shrq(Imm32(1), scratch);
+ andq(Imm32(1), temp);
+ orq(temp, scratch);
+
+ vcvtsq2ss(scratch, output, output);
+ vaddss(output, output, output);
+
+ bind(&done);
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ convertInt64ToDouble(Register64(src), dest);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ MOZ_ASSERT(output.reg == rax);
+ if (expected != output) {
+ movq(expected.reg, output.reg);
+ }
+ append(access, size());
+ lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ MOZ_ASSERT(output.reg == rax);
+ if (expected != output) {
+ movq(expected.reg, output.reg);
+ }
+ append(access, size());
+ lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ if (value != output) {
+ movq(value.reg, output.reg);
+ }
+ append(access, masm.size());
+ xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 value, Register64 output) {
+ if (value != output) {
+ movq(value.reg, output.reg);
+ }
+ append(access, masm.size());
+ xchgq(output.reg, Operand(mem));
+}
+
+template <typename T>
+static void AtomicFetchOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access, AtomicOp op,
+ Register value, const T& mem, Register temp,
+ Register output) {
+ // NOTE: the generated code must match the assembly code in gen_fetchop in
+ // GenerateAtomicOperations.py
+ if (op == AtomicFetchAddOp) {
+ if (value != output) {
+ masm.movq(value, output);
+ }
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.lock_xaddq(output, Operand(mem));
+ } else if (op == AtomicFetchSubOp) {
+ if (value != output) {
+ masm.movq(value, output);
+ }
+ masm.negq(output);
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.lock_xaddq(output, Operand(mem));
+ } else {
+ Label again;
+ MOZ_ASSERT(output == rax);
+ MOZ_ASSERT(value != output);
+ MOZ_ASSERT(value != temp);
+ MOZ_ASSERT(temp != output);
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.movq(Operand(mem), rax);
+ masm.bind(&again);
+ masm.movq(rax, temp);
+ switch (op) {
+ case AtomicFetchAndOp:
+ masm.andq(value, temp);
+ break;
+ case AtomicFetchOrOp:
+ masm.orq(value, temp);
+ break;
+ case AtomicFetchXorOp:
+ masm.xorq(value, temp);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ masm.lock_cmpxchgq(temp, Operand(mem));
+ masm.j(MacroAssembler::NonZero, &again);
+ }
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, op, value.reg, mem, temp.reg, output.reg);
+}
+
+template <typename T>
+static void AtomicEffectOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access, AtomicOp op,
+ Register value, const T& mem) {
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.lock_addq(value, Operand(mem));
+ break;
+ case AtomicFetchSubOp:
+ masm.lock_subq(value, Operand(mem));
+ break;
+ case AtomicFetchAndOp:
+ masm.lock_andq(value, Operand(mem));
+ break;
+ case AtomicFetchOrOp:
+ masm.lock_orq(value, Operand(mem));
+ break;
+ case AtomicFetchXorOp:
+ masm.lock_xorq(value, Operand(mem));
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+void MacroAssembler::wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register64 value,
+ const BaseIndex& mem) {
+ AtomicEffectOp64(*this, &access, op, value.reg, mem);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization&,
+ const Address& mem, Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ // NOTE: the generated code must match the assembly code in gen_cmpxchg in
+ // GenerateAtomicOperations.py
+ MOZ_ASSERT(output.reg == rax);
+ if (expected != output) {
+ movq(expected.reg, output.reg);
+ }
+ lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::compareExchange64(const Synchronization&,
+ const BaseIndex& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ MOZ_ASSERT(output.reg == rax);
+ if (expected != output) {
+ movq(expected.reg, output.reg);
+ }
+ lock_cmpxchgq(replacement.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization&,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ // NOTE: the generated code must match the assembly code in gen_exchange in
+ // GenerateAtomicOperations.py
+ if (value != output) {
+ movq(value.reg, output.reg);
+ }
+ xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization&,
+ const BaseIndex& mem, Register64 value,
+ Register64 output) {
+ if (value != output) {
+ movq(value.reg, output.reg);
+ }
+ xchgq(output.reg, Operand(mem));
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, op, value.reg, mem, temp.reg, output.reg);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const Address& mem) {
+ AtomicEffectOp64(*this, nullptr, op, value.reg, mem);
+}
+
+void MacroAssembler::atomicEffectOp64(const Synchronization& sync, AtomicOp op,
+ Register64 value, const BaseIndex& mem) {
+ AtomicEffectOp64(*this, nullptr, op, value.reg, mem);
+}
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ return leaRipRelative(dest);
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target) {
+ ptrdiff_t off = target - loc;
+ MOZ_ASSERT(off > ptrdiff_t(INT32_MIN));
+ MOZ_ASSERT(off < ptrdiff_t(INT32_MAX));
+ PatchWrite_Imm32(loc, Imm32(off));
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok) {
+ cmpPtr(index.reg, boundsCheckLimit.reg);
+ j(cond, ok);
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCq(cond, Operand(boundsCheckLimit.reg), index.reg);
+ }
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok) {
+ cmpPtr(index.reg, Operand(boundsCheckLimit));
+ j(cond, ok);
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCq(cond, Operand(boundsCheckLimit), index.reg);
+ }
+}
+
+// ========================================================================
+// Integer compare-then-conditionally-load/move operations.
+
+// cmpMove, Cond-Reg-Reg-Reg-Reg cases
+
+template <size_t CmpSize, size_t MoveSize>
+void MacroAssemblerX64::cmpMove(Condition cond, Register lhs, Register rhs,
+ Register falseVal, Register trueValAndDest) {
+ if constexpr (CmpSize == 32) {
+ cmp32(lhs, rhs);
+ } else {
+ static_assert(CmpSize == 64);
+ cmpPtr(lhs, rhs);
+ }
+ if constexpr (MoveSize == 32) {
+ cmovCCl(cond, Operand(falseVal), trueValAndDest);
+ } else {
+ static_assert(MoveSize == 64);
+ cmovCCq(cond, Operand(falseVal), trueValAndDest);
+ }
+}
+template void MacroAssemblerX64::cmpMove<32, 32>(Condition cond, Register lhs,
+ Register rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<32, 64>(Condition cond, Register lhs,
+ Register rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 32>(Condition cond, Register lhs,
+ Register rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 64>(Condition cond, Register lhs,
+ Register rhs,
+ Register falseVal,
+ Register trueValAndDest);
+
+// cmpMove, Cond-Reg-Addr-Reg-Reg cases
+
+template <size_t CmpSize, size_t MoveSize>
+void MacroAssemblerX64::cmpMove(Condition cond, Register lhs,
+ const Address& rhs, Register falseVal,
+ Register trueValAndDest) {
+ if constexpr (CmpSize == 32) {
+ cmp32(lhs, Operand(rhs));
+ } else {
+ static_assert(CmpSize == 64);
+ cmpPtr(lhs, Operand(rhs));
+ }
+ if constexpr (MoveSize == 32) {
+ cmovCCl(cond, Operand(falseVal), trueValAndDest);
+ } else {
+ static_assert(MoveSize == 64);
+ cmovCCq(cond, Operand(falseVal), trueValAndDest);
+ }
+}
+template void MacroAssemblerX64::cmpMove<32, 32>(Condition cond, Register lhs,
+ const Address& rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<32, 64>(Condition cond, Register lhs,
+ const Address& rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 32>(Condition cond, Register lhs,
+ const Address& rhs,
+ Register falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpMove<64, 64>(Condition cond, Register lhs,
+ const Address& rhs,
+ Register falseVal,
+ Register trueValAndDest);
+
+// cmpLoad, Cond-Reg-Reg-Addr-Reg cases
+
+template <size_t CmpSize, size_t LoadSize>
+void MacroAssemblerX64::cmpLoad(Condition cond, Register lhs, Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest) {
+ if constexpr (CmpSize == 32) {
+ cmp32(lhs, rhs);
+ } else {
+ static_assert(CmpSize == 64);
+ cmpPtr(lhs, rhs);
+ }
+ if constexpr (LoadSize == 32) {
+ cmovCCl(cond, Operand(falseVal), trueValAndDest);
+ } else {
+ static_assert(LoadSize == 64);
+ cmovCCq(cond, Operand(falseVal), trueValAndDest);
+ }
+}
+template void MacroAssemblerX64::cmpLoad<32, 32>(Condition cond, Register lhs,
+ Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<32, 64>(Condition cond, Register lhs,
+ Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 32>(Condition cond, Register lhs,
+ Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 64>(Condition cond, Register lhs,
+ Register rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+
+// cmpLoad, Cond-Reg-Addr-Addr-Reg cases
+
+template <size_t CmpSize, size_t LoadSize>
+void MacroAssemblerX64::cmpLoad(Condition cond, Register lhs,
+ const Address& rhs, const Address& falseVal,
+ Register trueValAndDest) {
+ if constexpr (CmpSize == 32) {
+ cmp32(lhs, Operand(rhs));
+ } else {
+ static_assert(CmpSize == 64);
+ cmpPtr(lhs, Operand(rhs));
+ }
+ if constexpr (LoadSize == 32) {
+ cmovCCl(cond, Operand(falseVal), trueValAndDest);
+ } else {
+ static_assert(LoadSize == 64);
+ cmovCCq(cond, Operand(falseVal), trueValAndDest);
+ }
+}
+template void MacroAssemblerX64::cmpLoad<32, 32>(Condition cond, Register lhs,
+ const Address& rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<32, 64>(Condition cond, Register lhs,
+ const Address& rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 32>(Condition cond, Register lhs,
+ const Address& rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+template void MacroAssemblerX64::cmpLoad<64, 64>(Condition cond, Register lhs,
+ const Address& rhs,
+ const Address& falseVal,
+ Register trueValAndDest);
+
+//}}} check_macroassembler_style
diff --git a/js/src/jit/x64/MacroAssembler-x64.h b/js/src/jit/x64/MacroAssembler-x64.h
new file mode 100644
index 0000000000..c222cde5b1
--- /dev/null
+++ b/js/src/jit/x64/MacroAssembler-x64.h
@@ -0,0 +1,1219 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_MacroAssembler_x64_h
+#define jit_x64_MacroAssembler_x64_h
+
+#include "jit/x86-shared/MacroAssembler-x86-shared.h"
+#include "js/HeapAPI.h"
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+struct ImmShiftedTag : public ImmWord {
+ explicit ImmShiftedTag(JSValueShiftedTag shtag) : ImmWord((uintptr_t)shtag) {}
+
+ explicit ImmShiftedTag(JSValueType type)
+ : ImmWord(uintptr_t(JSVAL_TYPE_TO_SHIFTED_TAG(type))) {}
+};
+
+struct ImmTag : public Imm32 {
+ explicit ImmTag(JSValueTag tag) : Imm32(tag) {}
+};
+
+// ScratchTagScope and ScratchTagScopeRelease are used to manage the tag
+// register for splitTagForTest(), which has different register management on
+// different platforms. On 64-bit platforms it requires a scratch register that
+// does not interfere with other operations; on 32-bit platforms it uses a
+// register that is already part of the Value.
+//
+// The ScratchTagScope RAII type acquires the appropriate register; a reference
+// to a variable of this type is then passed to splitTagForTest().
+//
+// On 64-bit platforms ScratchTagScopeRelease makes the owned scratch register
+// available in a dynamic scope during compilation. However it is important to
+// remember that that does not preserve the register value in any way, so this
+// RAII type should only be used along paths that eventually branch past further
+// uses of the extracted tag value.
+//
+// On 32-bit platforms ScratchTagScopeRelease has no effect, since it does not
+// manage a register, it only aliases a register in the ValueOperand.
+
+class ScratchTagScope : public ScratchRegisterScope {
+ public:
+ ScratchTagScope(MacroAssembler& masm, const ValueOperand&)
+ : ScratchRegisterScope(masm) {}
+};
+
+class ScratchTagScopeRelease {
+ ScratchTagScope* ts_;
+
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope* ts) : ts_(ts) {
+ ts_->release();
+ }
+ ~ScratchTagScopeRelease() { ts_->reacquire(); }
+};
+
+class MacroAssemblerX64 : public MacroAssemblerX86Shared {
+ private:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ void bindOffsets(const MacroAssemblerX86Shared::UsesVector&);
+
+ void vpRiprOpSimd128(const SimdConstant& v, FloatRegister reg,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID id));
+
+ void vpRiprOpSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest,
+ JmpSrc (X86Encoding::BaseAssemblerX64::*op)(
+ X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId));
+
+ public:
+ using MacroAssemblerX86Shared::load32;
+ using MacroAssemblerX86Shared::store16;
+ using MacroAssemblerX86Shared::store32;
+
+ MacroAssemblerX64() = default;
+
+ // The buffer is about to be linked, make sure any constant pools or excess
+ // bookkeeping has been flushed to the instruction stream.
+ void finish();
+
+ /////////////////////////////////////////////////////////////////
+ // X64 helpers.
+ /////////////////////////////////////////////////////////////////
+ void writeDataRelocation(const Value& val) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (val.isGCThing()) {
+ gc::Cell* cell = val.toGCThing();
+ if (cell && gc::IsInsideNursery(cell)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(masm.currentOffset());
+ }
+ }
+
+ // Refers to the upper 32 bits of a 64-bit Value operand.
+ // On x86_64, the upper 32 bits do not necessarily only contain the type.
+ Operand ToUpper32(Operand base) {
+ switch (base.kind()) {
+ case Operand::MEM_REG_DISP:
+ return Operand(Register::FromCode(base.base()), base.disp() + 4);
+
+ case Operand::MEM_SCALE:
+ return Operand(Register::FromCode(base.base()),
+ Register::FromCode(base.index()), base.scale(),
+ base.disp() + 4);
+
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ static inline Operand ToUpper32(const Address& address) {
+ return Operand(address.base, address.offset + 4);
+ }
+ static inline Operand ToUpper32(const BaseIndex& address) {
+ return Operand(address.base, address.index, address.scale,
+ address.offset + 4);
+ }
+
+ uint32_t Upper32Of(JSValueShiftedTag tag) { return uint32_t(tag >> 32); }
+
+ JSValueShiftedTag GetShiftedTag(JSValueType type) {
+ return (JSValueShiftedTag)JSVAL_TYPE_TO_SHIFTED_TAG(type);
+ }
+
+ /////////////////////////////////////////////////////////////////
+ // X86/X64-common interface.
+ /////////////////////////////////////////////////////////////////
+ Address ToPayload(Address value) { return value; }
+
+ void storeValue(ValueOperand val, Operand dest) {
+ movq(val.valueReg(), dest);
+ }
+ void storeValue(ValueOperand val, const Address& dest) {
+ storeValue(val, Operand(dest));
+ }
+ template <typename T>
+ void storeValue(JSValueType type, Register reg, const T& dest) {
+ // Value types with 32-bit payloads can be emitted as two 32-bit moves.
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(reg, Operand(dest));
+ movl(Imm32(Upper32Of(GetShiftedTag(type))), ToUpper32(Operand(dest)));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ boxValue(type, reg, scratch);
+ movq(scratch, Operand(dest));
+ }
+ }
+ template <typename T>
+ void storeValue(const Value& val, const T& dest) {
+ ScratchRegisterScope scratch(asMasm());
+ if (val.isGCThing()) {
+ movWithPatch(ImmWord(val.asRawBits()), scratch);
+ writeDataRelocation(val);
+ } else {
+ mov(ImmWord(val.asRawBits()), scratch);
+ }
+ movq(scratch, Operand(dest));
+ }
+ void storeValue(ValueOperand val, BaseIndex dest) {
+ storeValue(val, Operand(dest));
+ }
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ loadPtr(src, temp);
+ storePtr(temp, dest);
+ }
+ void storePrivateValue(Register src, const Address& dest) {
+ storePtr(src, dest);
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ storePtr(imm, dest);
+ }
+ void loadValue(Operand src, ValueOperand val) { movq(src, val.valueReg()); }
+ void loadValue(Address src, ValueOperand val) {
+ loadValue(Operand(src), val);
+ }
+ void loadValue(const BaseIndex& src, ValueOperand val) {
+ loadValue(Operand(src), val);
+ }
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+ void tagValue(JSValueType type, Register payload, ValueOperand dest) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest.valueReg() != scratch);
+ if (payload != dest.valueReg()) {
+ movq(payload, dest.valueReg());
+ }
+ mov(ImmShiftedTag(type), scratch);
+ orq(scratch, dest.valueReg());
+ }
+ void pushValue(ValueOperand val) { push(val.valueReg()); }
+ void popValue(ValueOperand val) { pop(val.valueReg()); }
+ void pushValue(const Value& val) {
+ if (val.isGCThing()) {
+ ScratchRegisterScope scratch(asMasm());
+ movWithPatch(ImmWord(val.asRawBits()), scratch);
+ writeDataRelocation(val);
+ push(scratch);
+ } else {
+ push(ImmWord(val.asRawBits()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ ScratchRegisterScope scratch(asMasm());
+ boxValue(type, reg, scratch);
+ push(scratch);
+ }
+ void pushValue(const Address& addr) { push(Operand(addr)); }
+
+ void boxValue(JSValueType type, Register src, Register dest);
+
+ Condition testUndefined(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+ }
+ Condition testInt32(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_INT32));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+ }
+ Condition testNull(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_NULL));
+ return cond;
+ }
+ Condition testString(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_STRING));
+ return cond;
+ }
+ Condition testSymbol(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+ }
+ Condition testBigInt(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+ }
+ Condition testObject(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+ }
+ Condition testDouble(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JSVAL_TAG_MAX_DOUBLE));
+ return cond == Equal ? BelowOrEqual : Above;
+ }
+ Condition testNumber(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JS::detail::ValueUpperInclNumberTag));
+ return cond == Equal ? BelowOrEqual : Above;
+ }
+ Condition testGCThing(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, Imm32(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+ }
+
+ Condition testMagic(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+ }
+ Condition testError(Condition cond, Register tag) {
+ return testMagic(cond, tag);
+ }
+ Condition testPrimitive(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
+ return cond == Equal ? Below : AboveOrEqual;
+ }
+
+ Condition testUndefined(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testInt32(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testNumber(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNumber(cond, scratch);
+ }
+ Condition testNull(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testString(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testObject(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+ Condition testPrimitive(Condition cond, const ValueOperand& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testPrimitive(cond, scratch);
+ }
+
+ Condition testUndefined(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src),
+ Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_UNDEFINED))));
+ return cond;
+ }
+ Condition testInt32(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_INT32))));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_BOOLEAN))));
+ return cond;
+ }
+ Condition testDouble(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testNumber(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNumber(cond, scratch);
+ }
+ Condition testNull(Condition cond, const Address& src) {
+ cmp32(ToUpper32(src), Imm32(Upper32Of(GetShiftedTag(JSVAL_TYPE_NULL))));
+ return cond;
+ }
+ Condition testString(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testObject(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testPrimitive(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testPrimitive(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const Address& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testMagic(cond, scratch);
+ }
+
+ Condition testUndefined(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testUndefined(cond, scratch);
+ }
+ Condition testNull(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testNull(cond, scratch);
+ }
+ Condition testBoolean(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBoolean(cond, scratch);
+ }
+ Condition testString(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testString(cond, scratch);
+ }
+ Condition testSymbol(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testSymbol(cond, scratch);
+ }
+ Condition testBigInt(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testBigInt(cond, scratch);
+ }
+ Condition testInt32(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testInt32(cond, scratch);
+ }
+ Condition testObject(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testObject(cond, scratch);
+ }
+ Condition testDouble(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testDouble(cond, scratch);
+ }
+ Condition testMagic(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testGCThing(Condition cond, const BaseIndex& src) {
+ ScratchRegisterScope scratch(asMasm());
+ splitTag(src, scratch);
+ return testGCThing(cond, scratch);
+ }
+
+ Condition isMagic(Condition cond, const ValueOperand& src, JSWhyMagic why) {
+ uint64_t magic = MagicValue(why).asRawBits();
+ cmpPtr(src.valueReg(), ImmWord(magic));
+ return cond;
+ }
+
+ void cmpPtr(Register lhs, const ImmWord rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ if (intptr_t(rhs.value) <= INT32_MAX && intptr_t(rhs.value) >= INT32_MIN) {
+ cmpPtr(lhs, Imm32(int32_t(rhs.value)));
+ } else {
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ }
+ void cmpPtr(Register lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(Register lhs, const ImmGCPtr rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(lhs != scratch);
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ void cmpPtr(Register lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) {
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(!lhs.containsReg(scratch));
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ void cmpPtr(const Operand& lhs, const ImmWord rhs) {
+ if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
+ cmpPtr(lhs, Imm32((int32_t)rhs.value));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ movePtr(rhs, scratch);
+ cmpPtr(lhs, scratch);
+ }
+ }
+ void cmpPtr(const Operand& lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(const Address& lhs, const ImmGCPtr rhs) {
+ cmpPtr(Operand(lhs), rhs);
+ }
+ void cmpPtr(const Address& lhs, const ImmWord rhs) {
+ cmpPtr(Operand(lhs), rhs);
+ }
+ void cmpPtr(const Address& lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(const Operand& lhs, Register rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(Register lhs, const Operand& rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Operand& lhs, const Imm32 rhs) { cmpq(rhs, lhs); }
+ void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); }
+ void cmpPtr(Register lhs, Register rhs) { cmpq(rhs, lhs); }
+ void testPtr(Register lhs, Register rhs) { testq(rhs, lhs); }
+ void testPtr(Register lhs, Imm32 rhs) { testq(rhs, lhs); }
+ void testPtr(const Operand& lhs, Imm32 rhs) { testq(rhs, lhs); }
+ void test64(Register lhs, Register rhs) { testq(rhs, lhs); }
+ void test64(Register lhs, const Imm64 rhs) {
+ if ((intptr_t)rhs.value <= INT32_MAX && (intptr_t)rhs.value >= INT32_MIN) {
+ testq(Imm32((int32_t)rhs.value), lhs);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ movq(ImmWord(rhs.value), scratch);
+ testq(scratch, lhs);
+ }
+ }
+
+ // Compare-then-conditionally-move/load, for integer types
+ template <size_t CmpSize, size_t MoveSize>
+ void cmpMove(Condition cond, Register lhs, Register rhs, Register falseVal,
+ Register trueValAndDest);
+
+ template <size_t CmpSize, size_t MoveSize>
+ void cmpMove(Condition cond, Register lhs, const Address& rhs,
+ Register falseVal, Register trueValAndDest);
+
+ template <size_t CmpSize, size_t LoadSize>
+ void cmpLoad(Condition cond, Register lhs, Register rhs,
+ const Address& falseVal, Register trueValAndDest);
+
+ template <size_t CmpSize, size_t LoadSize>
+ void cmpLoad(Condition cond, Register lhs, const Address& rhs,
+ const Address& falseVal, Register trueValAndDest);
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+
+ void movePtr(Register src, Register dest) { movq(src, dest); }
+ void movePtr(Register src, const Operand& dest) { movq(src, dest); }
+ void movePtr(ImmWord imm, Register dest) { mov(imm, dest); }
+ void movePtr(ImmPtr imm, Register dest) { mov(imm, dest); }
+ void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); }
+ void movePtr(ImmGCPtr imm, Register dest) { movq(imm, dest); }
+ void loadPtr(AbsoluteAddress address, Register dest) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movq(Operand(address), dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ loadPtr(Address(scratch, 0x0), dest);
+ }
+ }
+ void loadPtr(const Address& address, Register dest) {
+ movq(Operand(address), dest);
+ }
+ void load64(const Address& address, Register dest) {
+ movq(Operand(address), dest);
+ }
+ void loadPtr(const Operand& src, Register dest) { movq(src, dest); }
+ void loadPtr(const BaseIndex& src, Register dest) {
+ movq(Operand(src), dest);
+ }
+ void loadPrivate(const Address& src, Register dest) { loadPtr(src, dest); }
+ void load32(AbsoluteAddress address, Register dest) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movl(Operand(address), dest);
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ load32(Address(scratch, 0x0), dest);
+ }
+ }
+ void load64(const Operand& address, Register64 dest) {
+ movq(address, dest.reg);
+ }
+ void load64(const Address& address, Register64 dest) {
+ movq(Operand(address), dest.reg);
+ }
+ void load64(const BaseIndex& address, Register64 dest) {
+ movq(Operand(address), dest.reg);
+ }
+ template <typename S>
+ void load64Unaligned(const S& src, Register64 dest) {
+ load64(src, dest);
+ }
+ template <typename T>
+ void storePtr(ImmWord imm, T address) {
+ if ((intptr_t)imm.value <= INT32_MAX && (intptr_t)imm.value >= INT32_MIN) {
+ movq(Imm32((int32_t)imm.value), Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(imm, scratch);
+ movq(scratch, Operand(address));
+ }
+ }
+ template <typename T>
+ void storePtr(ImmPtr imm, T address) {
+ storePtr(ImmWord(uintptr_t(imm.value)), address);
+ }
+ template <typename T>
+ void storePtr(ImmGCPtr imm, T address) {
+ ScratchRegisterScope scratch(asMasm());
+ movq(imm, scratch);
+ movq(scratch, Operand(address));
+ }
+ void storePtr(Register src, const Address& address) {
+ movq(src, Operand(address));
+ }
+ void store64(Register src, const Address& address) {
+ movq(src, Operand(address));
+ }
+ void store64(Register64 src, const Operand& address) {
+ movq(src.reg, address);
+ }
+ void storePtr(Register src, const BaseIndex& address) {
+ movq(src, Operand(address));
+ }
+ void storePtr(Register src, const Operand& dest) { movq(src, dest); }
+ void storePtr(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movq(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ storePtr(src, Address(scratch, 0x0));
+ }
+ }
+ void store32(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movl(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ store32(src, Address(scratch, 0x0));
+ }
+ }
+ void store16(Register src, AbsoluteAddress address) {
+ if (X86Encoding::IsAddressImmediate(address.addr)) {
+ movw(src, Operand(address));
+ } else {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmPtr(address.addr), scratch);
+ store16(src, Address(scratch, 0x0));
+ }
+ }
+ void store64(Register64 src, Address address) { storePtr(src.reg, address); }
+ void store64(Register64 src, const BaseIndex& address) {
+ storePtr(src.reg, address);
+ }
+ void store64(Imm64 imm, Address address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+ void store64(Imm64 imm, const BaseIndex& address) {
+ storePtr(ImmWord(imm.value), address);
+ }
+ template <typename S, typename T>
+ void store64Unaligned(const S& src, const T& dest) {
+ store64(src, dest);
+ }
+
+ void splitTag(Register src, Register dest) {
+ if (src != dest) {
+ movq(src, dest);
+ }
+ shrq(Imm32(JSVAL_TAG_SHIFT), dest);
+ }
+ void splitTag(const ValueOperand& operand, Register dest) {
+ splitTag(operand.valueReg(), dest);
+ }
+ void splitTag(const Operand& operand, Register dest) {
+ movq(operand, dest);
+ shrq(Imm32(JSVAL_TAG_SHIFT), dest);
+ }
+ void splitTag(const Address& operand, Register dest) {
+ splitTag(Operand(operand), dest);
+ }
+ void splitTag(const BaseIndex& operand, Register dest) {
+ splitTag(Operand(operand), dest);
+ }
+
+ // Extracts the tag of a value and places it in tag.
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ splitTag(value, tag);
+ }
+ void cmpTag(const ValueOperand& operand, ImmTag tag) {
+ ScratchTagScope reg(asMasm(), operand);
+ splitTagForTest(operand, reg);
+ cmp32(reg, tag);
+ }
+
+ Condition testMagic(Condition cond, const ValueOperand& src) {
+ ScratchTagScope scratch(asMasm(), src);
+ splitTagForTest(src, scratch);
+ return testMagic(cond, scratch);
+ }
+ Condition testError(Condition cond, const ValueOperand& src) {
+ return testMagic(cond, src);
+ }
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testNull(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testObject(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest) {
+ cond = testUndefined(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void boxDouble(FloatRegister src, const ValueOperand& dest, FloatRegister) {
+ vmovq(src, dest.valueReg());
+ }
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) {
+ MOZ_ASSERT(src != dest.valueReg());
+ boxValue(type, src, dest.valueReg());
+ }
+
+ // Note that the |dest| register here may be ScratchReg, so we shouldn't
+ // use it.
+ void unboxInt32(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+ void unboxInt32(const Operand& src, Register dest) { movl(src, dest); }
+ void unboxInt32(const Address& src, Register dest) {
+ unboxInt32(Operand(src), dest);
+ }
+ void unboxInt32(const BaseIndex& src, Register dest) {
+ unboxInt32(Operand(src), dest);
+ }
+ template <typename T>
+ void unboxDouble(const T& src, FloatRegister dest) {
+ loadDouble(Operand(src), dest);
+ }
+
+ void unboxArgObjMagic(const ValueOperand& src, Register dest) {
+ unboxArgObjMagic(Operand(src.valueReg()), dest);
+ }
+ void unboxArgObjMagic(const Operand& src, Register dest) {
+ mov(ImmWord(0), dest);
+ }
+ void unboxArgObjMagic(const Address& src, Register dest) {
+ unboxArgObjMagic(Operand(src), dest);
+ }
+
+ void unboxBoolean(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+ void unboxBoolean(const Operand& src, Register dest) { movl(src, dest); }
+ void unboxBoolean(const Address& src, Register dest) {
+ unboxBoolean(Operand(src), dest);
+ }
+ void unboxBoolean(const BaseIndex& src, Register dest) {
+ unboxBoolean(Operand(src), dest);
+ }
+
+ void unboxMagic(const ValueOperand& src, Register dest) {
+ movl(src.valueReg(), dest);
+ }
+
+ void unboxDouble(const ValueOperand& src, FloatRegister dest) {
+ vmovq(src.valueReg(), dest);
+ }
+
+ void notBoolean(const ValueOperand& val) { xorq(Imm32(1), val.valueReg()); }
+
+ void unboxNonDouble(const ValueOperand& src, Register dest,
+ JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(src.valueReg(), dest);
+ return;
+ }
+ if (src.valueReg() == dest) {
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ xorq(scratch, dest);
+ } else {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest);
+ xorq(src.valueReg(), dest);
+ }
+ }
+ void unboxNonDouble(const Operand& src, Register dest, JSValueType type) {
+ MOZ_ASSERT(type != JSVAL_TYPE_DOUBLE);
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movl(src, dest);
+ return;
+ }
+ // Explicitly permits |dest| to be used in |src|.
+ ScratchRegisterScope scratch(asMasm());
+ MOZ_ASSERT(dest != scratch);
+ if (src.containsReg(dest)) {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), scratch);
+ // If src is already a register, then src and dest are the same
+ // thing and we don't need to move anything into dest.
+ if (src.kind() != Operand::REG) {
+ movq(src, dest);
+ }
+ xorq(scratch, dest);
+ } else {
+ mov(ImmWord(JSVAL_TYPE_TO_SHIFTED_TAG(type)), dest);
+ xorq(src, dest);
+ }
+ }
+ void unboxNonDouble(const Address& src, Register dest, JSValueType type) {
+ unboxNonDouble(Operand(src), dest, type);
+ }
+ void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) {
+ unboxNonDouble(Operand(src), dest, type);
+ }
+
+ void unboxString(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+
+ void unboxSymbol(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxSymbol(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+
+ void unboxBigInt(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+
+ void unboxObject(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Operand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Address& src, Register dest) {
+ unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const BaseIndex& src, Register dest) {
+ unboxNonDouble(Operand(src), dest, JSVAL_TYPE_OBJECT);
+ }
+
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ ScratchRegisterScope scratch(asMasm());
+ mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch);
+ andq(scratch, dest);
+ }
+
+ // This should only be used for GC barrier code, to unbox a GC thing Value.
+ // It's fine there because we don't depend on the actual Value type (all Cells
+ // are treated the same way). In almost all other cases this would be
+ // Spectre-unsafe - use unboxNonDouble and friends instead.
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
+ andq(Operand(src), dest);
+ }
+ void unboxGCThingForGCBarrier(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movq(ImmWord(JS::detail::ValueGCThingPayloadMask), dest);
+ andq(src.valueReg(), dest);
+ }
+
+ // Like unboxGCThingForGCBarrier, but loads the GC thing's chunk base.
+ void getGCThingValueChunk(const Address& src, Register dest) {
+ movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ andq(Operand(src), dest);
+ }
+ void getGCThingValueChunk(const ValueOperand& src, Register dest) {
+ MOZ_ASSERT(src.valueReg() != dest);
+ movq(ImmWord(JS::detail::ValueGCThingPayloadChunkMask), dest);
+ andq(src.valueReg(), dest);
+ }
+
+ inline void fallibleUnboxPtrImpl(const Operand& src, Register dest,
+ JSValueType type, Label* fail);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ [[nodiscard]] Register extractObject(const Address& address,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxObject(address, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxObject(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxSymbol(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxInt32(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ unboxBoolean(value, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ loadPtr(address, scratch);
+ splitTag(scratch, scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ MOZ_ASSERT(scratch != ScratchReg);
+ splitTag(value, scratch);
+ return scratch;
+ }
+
+ inline void unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType type);
+
+ // These two functions use the low 32-bits of the full value register.
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+ }
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.valueReg(), dest);
+ }
+
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+ }
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.valueReg(), dest);
+ }
+
+ void loadConstantDouble(double d, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest);
+ void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest);
+ void vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddusbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpadduswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubusbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubuswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vporSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpacksswbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackuswbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackssdwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackusdwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpunpckldqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vunpcklpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vptestSimd128(const SimdConstant& v, FloatRegister lhs);
+ void vpmaddwdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpeqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpneqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpltpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmplepsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpgepsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpeqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpneqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpltpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmplepdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaddubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmuludqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+
+ public:
+ Condition testInt32Truthy(bool truthy, const ValueOperand& operand) {
+ test32(operand.valueReg(), operand.valueReg());
+ return truthy ? NonZero : Zero;
+ }
+ Condition testStringTruthy(bool truthy, const ValueOperand& value);
+ Condition testBigIntTruthy(bool truthy, const ValueOperand& value);
+
+ template <typename T>
+ inline void loadInt32OrDouble(const T& src, FloatRegister dest);
+
+ template <typename T>
+ void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(src, dest.fpu());
+ } else {
+ unboxNonDouble(Operand(src), dest.gpr(), ValueTypeFromMIRType(type));
+ }
+ }
+
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes,
+ JSValueType type) {
+ switch (nbytes) {
+ case 8: {
+ ScratchRegisterScope scratch(asMasm());
+ unboxNonDouble(value, scratch, type);
+ storePtr(scratch, address);
+ if (type == JSVAL_TYPE_OBJECT) {
+ // Ideally we would call unboxObjectOrNull, but we need an extra
+ // scratch register for that. So unbox as object, then clear the
+ // object-or-null bit.
+ mov(ImmWord(~JS::detail::ValueObjectOrNullBit), scratch);
+ andq(scratch, Operand(address));
+ }
+ return;
+ }
+ case 4:
+ store32(value.valueReg(), address);
+ return;
+ case 1:
+ store8(value.valueReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void loadInstructionPointerAfterCall(Register dest) {
+ loadPtr(Address(StackPointer, 0x0), dest);
+ }
+
+ // Checks whether a double is representable as a 64-bit integer. If so, the
+ // integer is written to the output register. Otherwise, a bailout is taken to
+ // the given snapshot. This function overwrites the scratch float register.
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true);
+
+ void convertUInt32ToDouble(Register src, FloatRegister dest) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(dest);
+
+ vcvtsq2sd(src, dest, dest);
+ }
+
+ void convertUInt32ToFloat32(Register src, FloatRegister dest) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(dest);
+
+ vcvtsq2ss(src, dest, dest);
+ }
+
+ inline void incrementInt32Value(const Address& addr);
+
+ inline void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ public:
+ void handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail);
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+using MacroAssemblerSpecific = MacroAssemblerX64;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_MacroAssembler_x64_h */
diff --git a/js/src/jit/x64/SharedICHelpers-x64-inl.h b/js/src/jit/x64/SharedICHelpers-x64-inl.h
new file mode 100644
index 0000000000..c40eb45c74
--- /dev/null
+++ b/js/src/jit/x64/SharedICHelpers-x64-inl.h
@@ -0,0 +1,80 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICHelpers_x64_inl_h
+#define jit_x64_SharedICHelpers_x64_inl_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+ uint32_t argSize) {
+#ifdef DEBUG
+ ScratchRegisterScope scratch(masm);
+
+ // We can assume during this that R0 and R1 have been pushed.
+ // Store frame size without VMFunction arguments for debug assertions.
+ masm.movq(FramePointer, scratch);
+ masm.subq(StackPointer, scratch);
+ masm.subq(Imm32(argSize), scratch);
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Push frame descriptor and perform the tail call.
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(ICTailCallReg);
+ masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register) {
+#ifdef DEBUG
+ // Compute frame size. Because the return address is still on the stack,
+ // this is:
+ //
+ // FramePointer
+ // - StackPointer
+ // - sizeof(return address)
+
+ ScratchRegisterScope scratch(masm);
+ masm.movq(FramePointer, scratch);
+ masm.subq(StackPointer, scratch);
+ masm.subq(Imm32(sizeof(void*)), scratch); // Return address.
+
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Push the return address that's currently on top of the stack.
+ masm.Push(Operand(StackPointer, 0));
+
+ // Replace the original return address with the frame descriptor.
+ masm.storePtr(ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)),
+ Address(StackPointer, sizeof(uintptr_t)));
+
+ // Save old frame pointer, stack pointer and stub reg.
+ masm.Push(FramePointer);
+ masm.mov(StackPointer, FramePointer);
+
+ masm.Push(ICStubReg);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_SharedICHelpers_x64_inl_h */
diff --git a/js/src/jit/x64/SharedICHelpers-x64.h b/js/src/jit/x64/SharedICHelpers-x64.h
new file mode 100644
index 0000000000..8233db5735
--- /dev/null
+++ b/js/src/jit/x64/SharedICHelpers-x64.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICHelpers_x64_h
+#define jit_x64_SharedICHelpers_x64_h
+
+#include "jit/BaselineIC.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from Stack top to the top Value inside an IC stub (this is the
+// return address).
+static const size_t ICStackValueOffset = sizeof(void*);
+
+inline void EmitRestoreTailCallReg(MacroAssembler& masm) {
+ masm.Pop(ICTailCallReg);
+}
+
+inline void EmitRepushTailCallReg(MacroAssembler& masm) {
+ masm.Push(ICTailCallReg);
+}
+
+inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
+ // The stub pointer must already be in ICStubReg.
+ // Call the stubcode.
+ masm.call(Address(ICStubReg, ICStub::offsetOfStubCode()));
+ *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void EmitReturnFromIC(MacroAssembler& masm) { masm.ret(); }
+
+inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
+ Address stubAddr(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP);
+ masm.loadPtr(stubAddr, ICStubReg);
+
+ masm.mov(FramePointer, StackPointer);
+ masm.Pop(FramePointer);
+
+ // The return address is on top of the stack, followed by the frame
+ // descriptor. Use a pop instruction to overwrite the frame descriptor
+ // with the return address. Note that pop increments the stack pointer
+ // before computing the address.
+ masm.Pop(Operand(StackPointer, 0));
+}
+
+template <typename AddrType>
+inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr,
+ MIRType type) {
+ masm.guardedCallPreBarrier(addr, type);
+}
+
+inline void EmitStubGuardFailure(MacroAssembler& masm) {
+ // Load next stub into ICStubReg
+ masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+ // Return address is already loaded, just jump to the next stubcode.
+ masm.jmp(Operand(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_SharedICHelpers_x64_h */
diff --git a/js/src/jit/x64/SharedICRegisters-x64.h b/js/src/jit/x64/SharedICRegisters-x64.h
new file mode 100644
index 0000000000..8e52e5f3c9
--- /dev/null
+++ b/js/src/jit/x64/SharedICRegisters-x64.h
@@ -0,0 +1,33 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x64_SharedICRegisters_x64_h
+#define jit_x64_SharedICRegisters_x64_h
+
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "jit/x64/Assembler-x64.h"
+
+namespace js {
+namespace jit {
+
+static constexpr ValueOperand R0(rcx);
+static constexpr ValueOperand R1(rbx);
+static constexpr ValueOperand R2(rax);
+
+static constexpr Register ICTailCallReg = rsi;
+static constexpr Register ICStubReg = rdi;
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = xmm0;
+static constexpr FloatRegister FloatReg1 = xmm1;
+static constexpr FloatRegister FloatReg2 = xmm2;
+static constexpr FloatRegister FloatReg3 = xmm3;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x64_SharedICRegisters_x64_h */
diff --git a/js/src/jit/x64/Trampoline-x64.cpp b/js/src/jit/x64/Trampoline-x64.cpp
new file mode 100644
index 0000000000..946b21db77
--- /dev/null
+++ b/js/src/jit/x64/Trampoline-x64.cpp
@@ -0,0 +1,886 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "jit/x64/SharedICRegisters-x64.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::IsPowerOfTwo;
+
+// This struct reflects the contents of the stack entry.
+// Given a `CommonFrameLayout* frame`:
+// - `frame->prevType()` should be `FrameType::CppToJSJit`.
+// - Then EnterJITStackEntry starts at:
+// frame->callerFramePtr() + EnterJITStackEntry::offsetFromFP()
+// (the offset is negative, so this subtracts from the frame pointer)
+struct EnterJITStackEntry {
+ // Offset from frame pointer to EnterJITStackEntry*.
+ static constexpr int32_t offsetFromFP() {
+ return -int32_t(offsetof(EnterJITStackEntry, rbp));
+ }
+
+ void* result;
+
+#if defined(_WIN64)
+ struct XMM {
+ using XMM128 = char[16];
+ XMM128 xmm6;
+ XMM128 xmm7;
+ XMM128 xmm8;
+ XMM128 xmm9;
+ XMM128 xmm10;
+ XMM128 xmm11;
+ XMM128 xmm12;
+ XMM128 xmm13;
+ XMM128 xmm14;
+ XMM128 xmm15;
+ } xmm;
+
+ // 16-byte aligment for xmm registers above.
+ uint64_t xmmPadding;
+
+ void* rsi;
+ void* rdi;
+#endif
+
+ void* r15;
+ void* r14;
+ void* r13;
+ void* r12;
+ void* rbx;
+ void* rbp;
+
+ // Pushed by CALL.
+ void* rip;
+};
+
+// All registers to save and restore. This includes the stack pointer, since we
+// use the ability to reference register values on the stack by index.
+static const LiveRegisterSet AllRegs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::AllMask),
+ FloatRegisterSet(FloatRegisters::AllMask));
+
+// Generates a trampoline for calling Jit compiled code from a C++ function.
+// The trampoline use the EnterJitCode signature, with the standard x64 fastcall
+// calling convention.
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ masm.assertStackAlignment(ABIStackAlignment,
+ -int32_t(sizeof(uintptr_t)) /* return address */);
+
+ const Register reg_code = IntArgReg0;
+ const Register reg_argc = IntArgReg1;
+ const Register reg_argv = IntArgReg2;
+ static_assert(OsrFrameReg == IntArgReg3);
+
+#if defined(_WIN64)
+ const Address token = Address(rbp, 16 + ShadowStackSpace);
+ const Operand scopeChain = Operand(rbp, 24 + ShadowStackSpace);
+ const Operand numStackValuesAddr = Operand(rbp, 32 + ShadowStackSpace);
+ const Operand result = Operand(rbp, 40 + ShadowStackSpace);
+#else
+ const Register token = IntArgReg4;
+ const Register scopeChain = IntArgReg5;
+ const Operand numStackValuesAddr = Operand(rbp, 16 + ShadowStackSpace);
+ const Operand result = Operand(rbp, 24 + ShadowStackSpace);
+#endif
+
+ // Note: the stack pushes below must match the fields in EnterJITStackEntry.
+
+ // Save old stack frame pointer, set new stack frame pointer.
+ masm.push(rbp);
+ masm.mov(rsp, rbp);
+
+ // Save non-volatile registers. These must be saved by the trampoline, rather
+ // than by the JIT'd code, because they are scanned by the conservative
+ // scanner.
+ masm.push(rbx);
+ masm.push(r12);
+ masm.push(r13);
+ masm.push(r14);
+ masm.push(r15);
+#if defined(_WIN64)
+ masm.push(rdi);
+ masm.push(rsi);
+
+ // 16-byte aligment for vmovdqa
+ masm.subq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp);
+
+ masm.vmovdqa(xmm6, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)));
+ masm.vmovdqa(xmm7, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)));
+ masm.vmovdqa(xmm8, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)));
+ masm.vmovdqa(xmm9, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)));
+ masm.vmovdqa(xmm10, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)));
+ masm.vmovdqa(xmm11, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)));
+ masm.vmovdqa(xmm12, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)));
+ masm.vmovdqa(xmm13, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)));
+ masm.vmovdqa(xmm14, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)));
+ masm.vmovdqa(xmm15, Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)));
+#endif
+
+ // Save arguments passed in registers needed after function call.
+ masm.push(result);
+
+ // End of pushes reflected in EnterJITStackEntry, i.e. EnterJITStackEntry
+ // starts at this rsp.
+
+ // Remember number of bytes occupied by argument vector
+ masm.mov(reg_argc, r13);
+
+ // if we are constructing, that also needs to include newTarget
+ {
+ Label noNewTarget;
+ masm.branchTest32(Assembler::Zero, token,
+ Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+ masm.addq(Imm32(1), r13);
+
+ masm.bind(&noNewTarget);
+ }
+
+ masm.shll(Imm32(3), r13); // r13 = argc * sizeof(Value)
+ static_assert(sizeof(Value) == 1 << 3, "Constant is baked in assembly code");
+
+ // Guarantee stack alignment of Jit frames.
+ //
+ // This code compensates for the offset created by the copy of the vector of
+ // arguments, such that the jit frame will be aligned once the return
+ // address is pushed on the stack.
+ //
+ // In the computation of the offset, we omit the size of the JitFrameLayout
+ // which is pushed on the stack, as the JitFrameLayout size is a multiple of
+ // the JitStackAlignment.
+ masm.mov(rsp, r12);
+ masm.subq(r13, r12);
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ masm.andl(Imm32(JitStackAlignment - 1), r12);
+ masm.subq(r12, rsp);
+
+ /***************************************************************
+ Loop over argv vector, push arguments onto stack in reverse order
+ ***************************************************************/
+
+ // r13 still stores the number of bytes in the argument vector.
+ masm.addq(reg_argv, r13); // r13 points above last argument or newTarget
+
+ // while r13 > rdx, push arguments.
+ {
+ Label header, footer;
+ masm.bind(&header);
+
+ masm.cmpPtr(r13, reg_argv);
+ masm.j(AssemblerX86Shared::BelowOrEqual, &footer);
+
+ masm.subq(Imm32(8), r13);
+ masm.push(Operand(r13, 0));
+ masm.jmp(&header);
+
+ masm.bind(&footer);
+ }
+
+ // Load the number of actual arguments. |result| is used to store the
+ // actual number of arguments without adding an extra argument to the enter
+ // JIT.
+ masm.movq(result, reg_argc);
+ masm.unboxInt32(Operand(reg_argc, 0), reg_argc);
+
+ // Push the callee token.
+ masm.push(token);
+
+ // Push the descriptor.
+ masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, reg_argc, reg_argc);
+
+ CodeLabel returnLabel;
+ Label oomReturnLabel;
+ {
+ // Handle Interpreter -> Baseline OSR.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(rbp));
+ regs.take(OsrFrameReg);
+ regs.take(reg_code);
+
+ Register scratch = regs.takeAny();
+
+ Label notOsr;
+ masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, &notOsr);
+
+ Register numStackValues = regs.takeAny();
+ masm.movq(numStackValuesAddr, numStackValues);
+
+ // Push return address
+ masm.mov(&returnLabel, scratch);
+ masm.push(scratch);
+
+ // Frame prologue.
+ masm.push(rbp);
+ masm.mov(rsp, rbp);
+
+ // Reserve frame.
+ masm.subPtr(Imm32(BaselineFrame::Size()), rsp);
+
+ Register framePtrScratch = regs.takeAny();
+ masm.touchFrameValues(numStackValues, scratch, framePtrScratch);
+ masm.mov(rsp, framePtrScratch);
+
+ // Reserve space for locals and stack values.
+ Register valuesSize = regs.takeAny();
+ masm.mov(numStackValues, valuesSize);
+ masm.shll(Imm32(3), valuesSize);
+ masm.subPtr(valuesSize, rsp);
+
+ // Enter exit frame.
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(Imm32(0)); // Fake return address.
+ masm.push(FramePointer);
+ // No GC things to mark, push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ regs.add(valuesSize);
+
+ masm.push(reg_code);
+
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(framePtrScratch); // BaselineFrame
+ masm.passABIArg(OsrFrameReg); // InterpreterFrame
+ masm.passABIArg(numStackValues);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ masm.pop(reg_code);
+
+ MOZ_ASSERT(reg_code != ReturnReg);
+
+ Label error;
+ masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), rsp);
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.profilerEnterFrame(rbp, scratch);
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+ masm.jump(reg_code);
+
+ // OOM: frame epilogue, load error value, discard return address and return.
+ masm.bind(&error);
+ masm.mov(rbp, rsp);
+ masm.pop(rbp);
+ masm.addPtr(Imm32(sizeof(uintptr_t)), rsp); // Return address.
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.jump(&oomReturnLabel);
+
+ masm.bind(&notOsr);
+ masm.movq(scopeChain, R1.scratchReg());
+ }
+
+ // The call will push the return address and frame pointer on the stack, thus
+ // we check that the stack would be aligned once the call is complete.
+ masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+ // Call function.
+ masm.callJitNoProfiler(reg_code);
+
+ {
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&returnLabel);
+ masm.addCodeLabel(returnLabel);
+ masm.bind(&oomReturnLabel);
+ }
+
+ // Discard arguments and padding. Set rsp to the address of the
+ // EnterJITStackEntry on the stack.
+ masm.lea(Operand(rbp, EnterJITStackEntry::offsetFromFP()), rsp);
+
+ /*****************************************************************
+ Place return value where it belongs, pop all saved registers
+ *****************************************************************/
+ masm.pop(r12); // vp
+ masm.storeValue(JSReturnOperand, Operand(r12, 0));
+
+ // Restore non-volatile registers.
+#if defined(_WIN64)
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm6)), xmm6);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm7)), xmm7);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm8)), xmm8);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm9)), xmm9);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm10)), xmm10);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm11)), xmm11);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm12)), xmm12);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm13)), xmm13);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm14)), xmm14);
+ masm.vmovdqa(Operand(rsp, offsetof(EnterJITStackEntry::XMM, xmm15)), xmm15);
+
+ masm.addq(Imm32(sizeof(EnterJITStackEntry::XMM) + 8), rsp);
+
+ masm.pop(rsi);
+ masm.pop(rdi);
+#endif
+ masm.pop(r15);
+ masm.pop(r14);
+ masm.pop(r13);
+ masm.pop(r12);
+ masm.pop(rbx);
+
+ // Restore frame pointer and return.
+ masm.pop(rbp);
+ masm.ret();
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ if (frameStackAddress->prevType() != FrameType::CppToJSJit) {
+ // This is not a CppToJSJit frame, there are no C++ registers here.
+ return mozilla::Nothing{};
+ }
+
+ // Compute pointer to start of EnterJITStackEntry on the stack.
+ uint8_t* fp = frameStackAddress->callerFramePtr();
+ auto* enterJITStackEntry = reinterpret_cast<EnterJITStackEntry*>(
+ fp + EnterJITStackEntry::offsetFromFP());
+
+ // Extract native function call registers.
+ ::JS::ProfilingFrameIterator::RegisterState registerState;
+ registerState.fp = enterJITStackEntry->rbp;
+ registerState.pc = enterJITStackEntry->rip;
+ // sp should be inside the caller's frame, so set sp to the value of the stack
+ // pointer before the call to the EnterJit trampoline.
+ registerState.sp = &enterJITStackEntry->rip + 1;
+ // No lr in this world.
+ registerState.lr = nullptr;
+ return mozilla::Some(registerState);
+}
+
+// Push AllRegs in a way that is compatible with RegisterDump, regardless of
+// what PushRegsInMask might do to reduce the set size.
+static void DumpAllRegs(MacroAssembler& masm) {
+#ifdef ENABLE_WASM_SIMD
+ masm.PushRegsInMask(AllRegs);
+#else
+ // When SIMD isn't supported, PushRegsInMask reduces the set of float
+ // registers to be double-sized, while the RegisterDump expects each of
+ // the float registers to have the maximal possible size
+ // (Simd128DataSize). To work around this, we just spill the double
+ // registers by hand here, using the register dump offset directly.
+ for (GeneralRegisterBackwardIterator iter(AllRegs.gprs()); iter.more();
+ ++iter) {
+ masm.Push(*iter);
+ }
+
+ masm.reserveStack(sizeof(RegisterDump::FPUArray));
+ for (FloatRegisterBackwardIterator iter(AllRegs.fpus()); iter.more();
+ ++iter) {
+ FloatRegister reg = *iter;
+ Address spillAddress(StackPointer, reg.getRegisterDumpOffsetInBytes());
+ masm.storeDouble(reg, spillAddress);
+ }
+#endif
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+ // See explanatory comment in x86's JitRuntime::generateInvalidator.
+
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // Push registers such that we can access them from [base + code].
+ DumpAllRegs(masm);
+
+ masm.movq(rsp, rax); // Argument to jit::InvalidationBailout.
+
+ // Make space for InvalidationBailout's bailoutInfo outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.movq(rsp, rbx);
+
+ using Fn =
+ bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(rdx);
+ masm.passABIArg(rax);
+ masm.passABIArg(rbx);
+ masm.callWithABI<Fn, InvalidationBailout>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(r9); // Get the bailoutInfo outparam.
+
+ // Pop the machine state and the dead frame.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
+ masm.jmp(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind) {
+ // Do not erase the frame pointer in this function.
+
+ AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+
+ // Caller:
+ // [arg2] [arg1] [this] [[argc] [callee] [descr] [raddr]] <- rsp
+
+ // Frame prologue.
+ //
+ // NOTE: if this changes, fix the Baseline bailout code too!
+ // See BaselineStackBuilder::calculatePrevFramePtr and
+ // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+ masm.push(FramePointer);
+ masm.movq(rsp, FramePointer);
+
+ // Load argc.
+ masm.loadNumActualArgs(FramePointer, r8);
+
+ // Load |nformals| into %rcx.
+ masm.loadPtr(Address(rbp, RectifierFrameLayout::offsetOfCalleeToken()), rax);
+ masm.mov(rax, rcx);
+ masm.andq(Imm32(uint32_t(CalleeTokenMask)), rcx);
+ masm.loadFunctionArgCount(rcx, rcx);
+
+ // Stash another copy in r11, since we are going to do destructive operations
+ // on rcx
+ masm.mov(rcx, r11);
+
+ static_assert(
+ CalleeToken_FunctionConstructing == 1,
+ "Ensure that we can use the constructing bit to count the value");
+ masm.mov(rax, rdx);
+ masm.andq(Imm32(uint32_t(CalleeToken_FunctionConstructing)), rdx);
+
+ // Including |this|, and |new.target|, there are (|nformals| + 1 +
+ // isConstructing) arguments to push to the stack. Then we push a
+ // JitFrameLayout. We compute the padding expressed in the number of extra
+ // |undefined| values to push on the stack.
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ static_assert(
+ JitStackAlignment % sizeof(Value) == 0,
+ "Ensure that we can pad the stack by pushing extra UndefinedValue");
+ static_assert(IsPowerOfTwo(JitStackValueAlignment),
+ "must have power of two for masm.andl to do its job");
+
+ masm.addl(
+ Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */),
+ rcx);
+ masm.addl(rdx, rcx);
+ masm.andl(Imm32(~(JitStackValueAlignment - 1)), rcx);
+
+ // Load the number of |undefined|s to push into %rcx. Subtract 1 for |this|.
+ masm.subl(r8, rcx);
+ masm.subl(Imm32(1), rcx);
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- rsp
+ // '--- #r8 ---'
+ //
+ // Rectifier frame:
+ // [rbp'] [undef] [undef] [undef] [arg2] [arg1] [this] [ [argc] [callee]
+ // [descr] [raddr] ]
+ // '------- #rcx --------' '--- #r8 ---'
+
+ // Copy the number of actual arguments into rdx.
+ masm.mov(r8, rdx);
+
+ masm.moveValue(UndefinedValue(), ValueOperand(r10));
+
+ // Push undefined. (including the padding)
+ {
+ Label undefLoopTop;
+ masm.bind(&undefLoopTop);
+
+ masm.push(r10);
+ masm.subl(Imm32(1), rcx);
+ masm.j(Assembler::NonZero, &undefLoopTop);
+ }
+
+ // Get the topmost argument.
+ static_assert(sizeof(Value) == 8, "TimesEight is used to skip arguments");
+
+ // Get the topmost argument.
+ BaseIndex b(FramePointer, r8, TimesEight, sizeof(RectifierFrameLayout));
+ masm.lea(Operand(b), rcx);
+
+ // Push arguments, |nargs| + 1 times (to include |this|).
+ masm.addl(Imm32(1), r8);
+ {
+ Label copyLoopTop;
+
+ masm.bind(&copyLoopTop);
+ masm.push(Operand(rcx, 0x0));
+ masm.subq(Imm32(sizeof(Value)), rcx);
+ masm.subl(Imm32(1), r8);
+ masm.j(Assembler::NonZero, &copyLoopTop);
+ }
+
+ // if constructing, copy newTarget
+ {
+ Label notConstructing;
+
+ masm.branchTest32(Assembler::Zero, rax,
+ Imm32(CalleeToken_FunctionConstructing),
+ &notConstructing);
+
+ // thisFrame[numFormals] = prevFrame[argc]
+ ValueOperand newTarget(r10);
+
+ // Load vp[argc]. Add sizeof(Value) for |this|.
+ BaseIndex newTargetSrc(FramePointer, rdx, TimesEight,
+ sizeof(RectifierFrameLayout) + sizeof(Value));
+ masm.loadValue(newTargetSrc, newTarget);
+
+ // Again, 1 for |this|
+ BaseIndex newTargetDest(rsp, r11, TimesEight, sizeof(Value));
+ masm.storeValue(newTarget, newTargetDest);
+
+ masm.bind(&notConstructing);
+ }
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ]
+ //
+ //
+ // Rectifier frame:
+ // [rbp'] <- rbp [undef] [undef] [undef] [arg2] [arg1] [this] <- rsp [ [argc]
+ // [callee] [descr] [raddr] ]
+ //
+
+ // Construct JitFrameLayout.
+ masm.push(rax); // callee token
+ masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, rdx, rdx);
+
+ // Call the target function.
+ masm.andq(Imm32(uint32_t(CalleeTokenMask)), rax);
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(rax, rax);
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(rax);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(rax, rbx, &noBaselineScript);
+ masm.callJitNoProfiler(rbx);
+ masm.jump(&done);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(rax, rax);
+ masm.callJitNoProfiler(rax);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.mov(FramePointer, StackPointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+ // Push registers such that we can access them from [base + code].
+ DumpAllRegs(masm);
+
+ // Get the stack pointer into a register, pre-alignment.
+ masm.movq(rsp, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+ PushBailoutFrame(masm, r8);
+
+ // Make space for Bailout's bailoutInfo outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.movq(rsp, r9);
+
+ // Call the bailout function.
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(rax);
+ masm.passABIArg(r8);
+ masm.passABIArg(r9);
+ masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(r9); // Get the bailoutInfo outparam.
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in r9.
+ masm.jmp(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+ *wrapperOffset = startTrampolineCode(masm);
+
+ // Avoid conflicts with argument registers while discarding the result after
+ // the function call.
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert(
+ (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set must be a superset of Volatile register set");
+
+ // The context is the first argument.
+ Register cxreg = IntArgReg0;
+ regs.take(cxreg);
+
+ // Stack is:
+ // ... frame ...
+ // +12 [args]
+ // +8 descriptor
+ // +0 returnAddress
+ //
+ // Push the frame pointer to finish the exit frame, then link it up.
+ masm.Push(FramePointer);
+ masm.loadJSContext(cxreg);
+ masm.enterExitFrame(cxreg, regs.getAny(), &f);
+
+ // Save the current stack pointer as the base for copying arguments.
+ Register argsBase = InvalidReg;
+ if (f.explicitArgs) {
+ argsBase = r10;
+ regs.take(argsBase);
+ masm.lea(Operand(rsp, ExitFrameLayout::SizeWithFooter()), argsBase);
+ }
+
+ // Reserve space for the outparameter.
+ Register outReg = InvalidReg;
+ switch (f.outParam) {
+ case Type_Value:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(Value));
+ masm.movq(esp, outReg);
+ break;
+
+ case Type_Handle:
+ outReg = regs.takeAny();
+ masm.PushEmptyRooted(f.outParamRootType);
+ masm.movq(esp, outReg);
+ break;
+
+ case Type_Int32:
+ case Type_Bool:
+ case Type_Pointer:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(uintptr_t));
+ masm.movq(esp, outReg);
+ break;
+
+ case Type_Double:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(double));
+ masm.movq(esp, outReg);
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ masm.setupUnalignedABICall(regs.getAny());
+ masm.passABIArg(cxreg);
+
+ size_t argDisp = 0;
+
+ // Copy arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ if (f.argPassedInFloatReg(explicitArg)) {
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::DOUBLE);
+ } else {
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ }
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByValue:
+ case VMFunctionData::DoubleByRef:
+ MOZ_CRASH("NYI: x64 callVM should not be used with 128bits values.");
+ }
+ }
+
+ // Copy the implicit outparam, if any.
+ if (outReg != InvalidReg) {
+ masm.passABIArg(outReg);
+ }
+
+ masm.callWithABI(nativeFun, MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, rax, rax, masm.failureLabel());
+ break;
+ case Type_Bool:
+ masm.testb(rax, rax);
+ masm.j(Assembler::Zero, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ // Load the outparam and free any allocated stack.
+ switch (f.outParam) {
+ case Type_Handle:
+ masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+ break;
+
+ case Type_Value:
+ masm.loadValue(Address(esp, 0), JSReturnOperand);
+ masm.freeStack(sizeof(Value));
+ break;
+
+ case Type_Int32:
+ masm.load32(Address(esp, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ case Type_Bool:
+ masm.load8ZeroExtend(Address(esp, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ case Type_Double:
+ masm.loadDouble(Address(esp, 0), ReturnDoubleReg);
+ masm.freeStack(sizeof(double));
+ break;
+
+ case Type_Pointer:
+ masm.loadPtr(Address(esp, 0), ReturnReg);
+ masm.freeStack(sizeof(uintptr_t));
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ // Until C++ code is instrumented against Spectre, prevent speculative
+ // execution from returning any private data.
+ if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
+ masm.speculationBarrier();
+ }
+
+ // Pop ExitFooterFrame and the frame pointer.
+ masm.leaveExitFrame(sizeof(void*));
+
+ // Return. Subtract sizeof(void*) for the frame pointer.
+ masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+ f.explicitStackSlots() * sizeof(void*) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type) {
+ AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+ uint32_t offset = startTrampolineCode(masm);
+
+ static_assert(PreBarrierReg == rdx);
+ Register temp1 = rax;
+ Register temp2 = rbx;
+ Register temp3 = rcx;
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+ &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ LiveRegisterSet regs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileMask));
+ masm.PushRegsInMask(regs);
+
+ masm.mov(ImmPtr(cx->runtime()), rcx);
+
+ masm.setupUnalignedABICall(rax);
+ masm.passABIArg(rcx);
+ masm.passABIArg(rdx);
+ masm.callWithABI(JitPreWriteBarrier(type));
+
+ masm.PopRegsInMask(regs);
+ masm.ret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.ret();
+
+ return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+ masm.bind(bailoutTail);
+ masm.generateBailoutTail(rdx, r9);
+}
diff --git a/js/src/jit/x86-shared/Architecture-x86-shared.cpp b/js/src/jit/x86-shared/Architecture-x86-shared.cpp
new file mode 100644
index 0000000000..f000b09c77
--- /dev/null
+++ b/js/src/jit/x86-shared/Architecture-x86-shared.cpp
@@ -0,0 +1,93 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86-shared/Architecture-x86-shared.h"
+#if !defined(JS_CODEGEN_X86) && !defined(JS_CODEGEN_X64)
+# error "Wrong architecture. Only x86 and x64 should build this file!"
+#endif
+
+#include <iterator>
+
+#include "jit/RegisterSets.h"
+
+const char* js::jit::FloatRegister::name() const {
+ static const char* const names[] = {
+
+#ifdef JS_CODEGEN_X64
+# define FLOAT_REGS_(TYPE) \
+ "%xmm0" TYPE, "%xmm1" TYPE, "%xmm2" TYPE, "%xmm3" TYPE, "%xmm4" TYPE, \
+ "%xmm5" TYPE, "%xmm6" TYPE, "%xmm7" TYPE, "%xmm8" TYPE, "%xmm9" TYPE, \
+ "%xmm10" TYPE, "%xmm11" TYPE, "%xmm12" TYPE, "%xmm13" TYPE, \
+ "%xmm14" TYPE, "%xmm15" TYPE
+#else
+# define FLOAT_REGS_(TYPE) \
+ "%xmm0" TYPE, "%xmm1" TYPE, "%xmm2" TYPE, "%xmm3" TYPE, "%xmm4" TYPE, \
+ "%xmm5" TYPE, "%xmm6" TYPE, "%xmm7" TYPE
+#endif
+
+ // These should be enumerated in the same order as in
+ // FloatRegisters::ContentType.
+ FLOAT_REGS_(".s"), FLOAT_REGS_(".d"), FLOAT_REGS_(".i4"),
+ FLOAT_REGS_(".s4")
+#undef FLOAT_REGS_
+
+ };
+ MOZ_ASSERT(size_t(code()) < std::size(names));
+ return names[size_t(code())];
+}
+
+js::jit::FloatRegisterSet js::jit::FloatRegister::ReduceSetForPush(
+ const FloatRegisterSet& s) {
+ SetType bits = s.bits();
+
+ // Ignore all SIMD register, if not supported.
+#ifndef ENABLE_WASM_SIMD
+ bits &= Codes::AllPhysMask * Codes::SpreadScalar;
+#endif
+
+ // Exclude registers which are already pushed with a larger type. High bits
+ // are associated with larger register types. Thus we keep the set of
+ // registers which are not included in larger type.
+ bits &= ~(bits >> (1 * Codes::TotalPhys));
+ bits &= ~(bits >> (2 * Codes::TotalPhys));
+ bits &= ~(bits >> (3 * Codes::TotalPhys));
+
+ return FloatRegisterSet(bits);
+}
+
+uint32_t js::jit::FloatRegister::GetPushSizeInBytes(const FloatRegisterSet& s) {
+ SetType all = s.bits();
+ SetType set128b = (all >> (uint32_t(Codes::Simd128) * Codes::TotalPhys)) &
+ Codes::AllPhysMask;
+ SetType doubleSet = (all >> (uint32_t(Codes::Double) * Codes::TotalPhys)) &
+ Codes::AllPhysMask;
+ SetType singleSet = (all >> (uint32_t(Codes::Single) * Codes::TotalPhys)) &
+ Codes::AllPhysMask;
+
+ // PushRegsInMask pushes the largest register first, and thus avoids pushing
+ // aliased registers. So we have to filter out the physical registers which
+ // are already pushed as part of larger registers.
+ SetType set64b = doubleSet & ~set128b;
+ SetType set32b = singleSet & ~set64b & ~set128b;
+
+ static_assert(Codes::AllPhysMask <= 0xffff,
+ "We can safely use CountPopulation32");
+ uint32_t count32b = mozilla::CountPopulation32(set32b);
+
+#if defined(JS_CODEGEN_X64)
+ // If we have an odd number of 32 bits values, then we increase the size to
+ // keep the stack aligned on 8 bytes. Note: Keep in sync with
+ // PushRegsInMask, and PopRegsInMaskIgnore.
+ count32b += count32b & 1;
+#endif
+
+ return mozilla::CountPopulation32(set128b) * (4 * sizeof(int32_t)) +
+ mozilla::CountPopulation32(set64b) * sizeof(double) +
+ count32b * sizeof(float);
+}
+uint32_t js::jit::FloatRegister::getRegisterDumpOffsetInBytes() {
+ return uint32_t(encoding()) * sizeof(FloatRegisters::RegisterContent);
+}
diff --git a/js/src/jit/x86-shared/Architecture-x86-shared.h b/js/src/jit/x86-shared/Architecture-x86-shared.h
new file mode 100644
index 0000000000..6b767c48c6
--- /dev/null
+++ b/js/src/jit/x86-shared/Architecture-x86-shared.h
@@ -0,0 +1,468 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_Architecture_x86_h
+#define jit_x86_shared_Architecture_x86_h
+
+#if !defined(JS_CODEGEN_X86) && !defined(JS_CODEGEN_X64)
+# error "Unsupported architecture!"
+#endif
+
+#include "mozilla/MathAlgorithms.h"
+
+#include <algorithm>
+#include <string.h>
+
+#include "jit/shared/Architecture-shared.h"
+
+#include "jit/x86-shared/Constants-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+#if defined(JS_CODEGEN_X86)
+// These offsets are specific to nunboxing, and capture offsets into the
+// components of a js::Value.
+static const int32_t NUNBOX32_TYPE_OFFSET = 4;
+static const int32_t NUNBOX32_PAYLOAD_OFFSET = 0;
+#endif
+
+#if defined(JS_CODEGEN_X64) && defined(_WIN64)
+static const uint32_t ShadowStackSpace = 32;
+#else
+static const uint32_t ShadowStackSpace = 0;
+#endif
+
+static const uint32_t JumpImmediateRange = INT32_MAX;
+
+class Registers {
+ public:
+ using Code = uint8_t;
+ using Encoding = X86Encoding::RegisterID;
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ uintptr_t r;
+ };
+
+#if defined(JS_CODEGEN_X86)
+ using SetType = uint8_t;
+
+ static const char* GetName(Code code) {
+ return X86Encoding::GPRegName(Encoding(code));
+ }
+
+ static const uint32_t Total = 8;
+ static const uint32_t TotalPhys = 8;
+ static const uint32_t Allocatable = 7;
+
+#elif defined(JS_CODEGEN_X64)
+ using SetType = uint16_t;
+
+ static const char* GetName(Code code) {
+ static const char* const Names[] = {
+ "rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi",
+ "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"};
+ return Names[code];
+ }
+
+ static const uint32_t Total = 16;
+ static const uint32_t TotalPhys = 16;
+ static const uint32_t Allocatable = 14;
+#endif
+
+ static uint32_t SetSize(SetType x) {
+ static_assert(sizeof(SetType) <= 4, "SetType must be, at most, 32 bits");
+ return mozilla::CountPopulation32(x);
+ }
+ static uint32_t FirstBit(SetType x) {
+ return mozilla::CountTrailingZeroes32(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ return 31 - mozilla::CountLeadingZeroes32(x);
+ }
+
+ static Code FromName(const char* name) {
+ for (size_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(Code(i)), name) == 0) {
+ return Code(i);
+ }
+ }
+ return Invalid;
+ }
+
+ static const Encoding StackPointer = X86Encoding::rsp;
+ static const Encoding Invalid = X86Encoding::invalid_reg;
+
+ static const SetType AllMask = (1 << Total) - 1;
+
+#if defined(JS_CODEGEN_X86)
+ static const SetType ArgRegMask = 0;
+
+ static const SetType VolatileMask = (1 << X86Encoding::rax) |
+ (1 << X86Encoding::rcx) |
+ (1 << X86Encoding::rdx);
+
+ static const SetType WrapperMask = VolatileMask | (1 << X86Encoding::rbx);
+
+ static const SetType SingleByteRegs =
+ (1 << X86Encoding::rax) | (1 << X86Encoding::rcx) |
+ (1 << X86Encoding::rdx) | (1 << X86Encoding::rbx);
+
+ static const SetType NonAllocatableMask =
+ (1 << X86Encoding::rsp) | (1 << X86Encoding::rbp);
+
+ // Registers returned from a JS -> JS call.
+ static const SetType JSCallMask =
+ (1 << X86Encoding::rcx) | (1 << X86Encoding::rdx);
+
+ // Registers returned from a JS -> C call.
+ static const SetType CallMask = (1 << X86Encoding::rax);
+
+#elif defined(JS_CODEGEN_X64)
+ static const SetType ArgRegMask =
+# if !defined(_WIN64)
+ (1 << X86Encoding::rdi) | (1 << X86Encoding::rsi) |
+# endif
+ (1 << X86Encoding::rdx) | (1 << X86Encoding::rcx) |
+ (1 << X86Encoding::r8) | (1 << X86Encoding::r9);
+
+ static const SetType VolatileMask = (1 << X86Encoding::rax) | ArgRegMask |
+ (1 << X86Encoding::r10) |
+ (1 << X86Encoding::r11);
+
+ static const SetType WrapperMask = VolatileMask;
+
+ static const SetType SingleByteRegs = AllMask & ~(1 << X86Encoding::rsp);
+
+ static const SetType NonAllocatableMask =
+ (1 << X86Encoding::rsp) | (1 << X86Encoding::rbp) |
+ (1 << X86Encoding::r11); // This is ScratchReg.
+
+ // Registers returned from a JS -> JS call.
+ static const SetType JSCallMask = (1 << X86Encoding::rcx);
+
+ // Registers returned from a JS -> C call.
+ static const SetType CallMask = (1 << X86Encoding::rax);
+
+#endif
+
+ static const SetType NonVolatileMask =
+ AllMask & ~VolatileMask & ~(1 << X86Encoding::rsp);
+
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+};
+
+using PackedRegisterMask = Registers::SetType;
+
+class FloatRegisters {
+ public:
+ using Encoding = X86Encoding::XMMRegisterID;
+
+ // Observe that there is a Simd128 type on both x86 and x64 whether SIMD is
+ // implemented/enabled or not, and that the RegisterContent union is large
+ // enough for a V128 datum always. Producers and consumers of a register dump
+ // must be aware of this even if they don't need to save/restore values in the
+ // high lanes of the SIMD registers. See the DumpAllRegs() implementations,
+ // for example.
+
+ enum ContentType {
+ Single, // 32-bit float.
+ Double, // 64-bit double.
+ Simd128, // 128-bit Wasm SIMD type.
+ NumTypes
+ };
+
+ // Content spilled during bailouts.
+ union RegisterContent {
+ float s;
+ double d;
+ uint8_t v128[16];
+ };
+
+ static const char* GetName(Encoding code) {
+ return X86Encoding::XMMRegName(code);
+ }
+
+ static Encoding FromName(const char* name) {
+ for (size_t i = 0; i < Total; i++) {
+ if (strcmp(GetName(Encoding(i)), name) == 0) {
+ return Encoding(i);
+ }
+ }
+ return Invalid;
+ }
+
+ static const Encoding Invalid = X86Encoding::invalid_xmm;
+
+#if defined(JS_CODEGEN_X86)
+ static const uint32_t Total = 8 * NumTypes;
+ static const uint32_t TotalPhys = 8;
+ static const uint32_t Allocatable = 7;
+ using SetType = uint32_t;
+#elif defined(JS_CODEGEN_X64)
+ static const uint32_t Total = 16 * NumTypes;
+ static const uint32_t TotalPhys = 16;
+ static const uint32_t Allocatable = 15;
+ using SetType = uint64_t;
+#endif
+
+ static_assert(sizeof(SetType) * 8 >= Total,
+ "SetType should be large enough to enumerate all registers.");
+
+ // Magic values which are used to duplicate a mask of physical register for
+ // a specific type of register. A multiplication is used to copy and shift
+ // the bits of the physical register mask.
+ static const SetType SpreadSingle = SetType(1)
+ << (uint32_t(Single) * TotalPhys);
+ static const SetType SpreadDouble = SetType(1)
+ << (uint32_t(Double) * TotalPhys);
+ static const SetType SpreadSimd128 = SetType(1)
+ << (uint32_t(Simd128) * TotalPhys);
+ static const SetType SpreadScalar = SpreadSingle | SpreadDouble;
+ static const SetType SpreadVector = SpreadSimd128;
+ static const SetType Spread = SpreadScalar | SpreadVector;
+
+ static const SetType AllPhysMask = ((1 << TotalPhys) - 1);
+ static const SetType AllMask = AllPhysMask * Spread;
+ static const SetType AllDoubleMask = AllPhysMask * SpreadDouble;
+ static const SetType AllSingleMask = AllPhysMask * SpreadSingle;
+ static const SetType AllVector128Mask = AllPhysMask * SpreadSimd128;
+
+#if defined(JS_CODEGEN_X86)
+ static const SetType NonAllocatableMask =
+ Spread * (1 << X86Encoding::xmm7); // This is ScratchDoubleReg.
+
+#elif defined(JS_CODEGEN_X64)
+ static const SetType NonAllocatableMask =
+ Spread * (1 << X86Encoding::xmm15); // This is ScratchDoubleReg.
+#endif
+
+#if defined(JS_CODEGEN_X64) && defined(_WIN64)
+ static const SetType VolatileMask =
+ ((1 << X86Encoding::xmm0) | (1 << X86Encoding::xmm1) |
+ (1 << X86Encoding::xmm2) | (1 << X86Encoding::xmm3) |
+ (1 << X86Encoding::xmm4) | (1 << X86Encoding::xmm5)) *
+ SpreadScalar |
+ AllPhysMask * SpreadVector;
+#else
+ static const SetType VolatileMask = AllMask;
+#endif
+
+ static const SetType NonVolatileMask = AllMask & ~VolatileMask;
+ static const SetType WrapperMask = VolatileMask;
+ static const SetType AllocatableMask = AllMask & ~NonAllocatableMask;
+};
+
+static const uint32_t SpillSlotSize =
+ std::max(sizeof(Registers::RegisterContent),
+ sizeof(FloatRegisters::RegisterContent));
+
+template <typename T>
+class TypedRegisterSet;
+
+struct FloatRegister {
+ using Codes = FloatRegisters;
+ using Code = size_t;
+ using Encoding = Codes::Encoding;
+ using SetType = Codes::SetType;
+ static uint32_t SetSize(SetType x) {
+ // Count the number of non-aliased registers, for the moment.
+ //
+ // Copy the set bits of each typed register to the low part of the of
+ // the Set, and count the number of registers. This is made to avoid
+ // registers which are allocated twice with different types (such as in
+ // AllMask).
+ x |= x >> (2 * Codes::TotalPhys);
+ x |= x >> Codes::TotalPhys;
+ x &= Codes::AllPhysMask;
+ static_assert(Codes::AllPhysMask <= 0xffff,
+ "We can safely use CountPopulation32");
+ return mozilla::CountPopulation32(x);
+ }
+
+#if defined(JS_CODEGEN_X86)
+ static uint32_t FirstBit(SetType x) {
+ static_assert(sizeof(SetType) == 4, "SetType must be 32 bits");
+ return mozilla::CountTrailingZeroes32(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ return 31 - mozilla::CountLeadingZeroes32(x);
+ }
+
+#elif defined(JS_CODEGEN_X64)
+ static uint32_t FirstBit(SetType x) {
+ static_assert(sizeof(SetType) == 8, "SetType must be 64 bits");
+ return mozilla::CountTrailingZeroes64(x);
+ }
+ static uint32_t LastBit(SetType x) {
+ return 63 - mozilla::CountLeadingZeroes64(x);
+ }
+#endif
+
+ private:
+ // Note: These fields are using one extra bit to make the invalid enumerated
+ // values fit, and thus prevent a warning.
+ Codes::Encoding reg_ : 5;
+ Codes::ContentType type_ : 3;
+ bool isInvalid_ : 1;
+
+ // Constants used for exporting/importing the float register code.
+#if defined(JS_CODEGEN_X86)
+ static const size_t RegSize = 3;
+#elif defined(JS_CODEGEN_X64)
+ static const size_t RegSize = 4;
+#endif
+ static const size_t RegMask = (1 << RegSize) - 1;
+
+ public:
+ constexpr FloatRegister()
+ : reg_(Codes::Encoding(0)), type_(Codes::Single), isInvalid_(true) {}
+ constexpr FloatRegister(uint32_t r, Codes::ContentType k)
+ : reg_(Codes::Encoding(r)), type_(k), isInvalid_(false) {}
+ constexpr FloatRegister(Codes::Encoding r, Codes::ContentType k)
+ : reg_(r), type_(k), isInvalid_(false) {}
+
+ static FloatRegister FromCode(uint32_t i) {
+ MOZ_ASSERT(i < Codes::Total);
+ return FloatRegister(i & RegMask, Codes::ContentType(i >> RegSize));
+ }
+
+ bool isSingle() const {
+ MOZ_ASSERT(!isInvalid());
+ return type_ == Codes::Single;
+ }
+ bool isDouble() const {
+ MOZ_ASSERT(!isInvalid());
+ return type_ == Codes::Double;
+ }
+ bool isSimd128() const {
+ MOZ_ASSERT(!isInvalid());
+ return type_ == Codes::Simd128;
+ }
+ bool isInvalid() const { return isInvalid_; }
+
+ FloatRegister asSingle() const {
+ MOZ_ASSERT(!isInvalid());
+ return FloatRegister(reg_, Codes::Single);
+ }
+ FloatRegister asDouble() const {
+ MOZ_ASSERT(!isInvalid());
+ return FloatRegister(reg_, Codes::Double);
+ }
+ FloatRegister asSimd128() const {
+ MOZ_ASSERT(!isInvalid());
+ return FloatRegister(reg_, Codes::Simd128);
+ }
+
+ uint32_t size() const {
+ MOZ_ASSERT(!isInvalid());
+ if (isSingle()) {
+ return sizeof(float);
+ }
+ if (isDouble()) {
+ return sizeof(double);
+ }
+ MOZ_ASSERT(isSimd128());
+ return 4 * sizeof(int32_t);
+ }
+
+ Code code() const {
+ MOZ_ASSERT(!isInvalid());
+ MOZ_ASSERT(uint32_t(reg_) < Codes::TotalPhys);
+ // :TODO: ARM is doing the same thing, but we should avoid this, except
+ // that the RegisterSets depends on this.
+ return Code(reg_ | (type_ << RegSize));
+ }
+ Encoding encoding() const {
+ MOZ_ASSERT(!isInvalid());
+ MOZ_ASSERT(uint32_t(reg_) < Codes::TotalPhys);
+ return reg_;
+ }
+ // defined in Assembler-x86-shared.cpp
+ const char* name() const;
+ bool volatile_() const {
+ return !!((SetType(1) << code()) & FloatRegisters::VolatileMask);
+ }
+ bool operator!=(FloatRegister other) const {
+ return other.reg_ != reg_ || other.type_ != type_;
+ }
+ bool operator==(FloatRegister other) const {
+ return other.reg_ == reg_ && other.type_ == type_;
+ }
+ bool aliases(FloatRegister other) const { return other.reg_ == reg_; }
+ // Check if two floating point registers have the same type.
+ bool equiv(FloatRegister other) const { return other.type_ == type_; }
+
+ uint32_t numAliased() const { return Codes::NumTypes; }
+ uint32_t numAlignedAliased() const { return numAliased(); }
+
+ FloatRegister aliased(uint32_t aliasIdx) const {
+ MOZ_ASSERT(aliasIdx < Codes::NumTypes);
+ return FloatRegister(
+ reg_, Codes::ContentType((aliasIdx + type_) % Codes::NumTypes));
+ }
+ FloatRegister alignedAliased(uint32_t aliasIdx) const {
+ return aliased(aliasIdx);
+ }
+
+ SetType alignedOrDominatedAliasedSet() const { return Codes::Spread << reg_; }
+
+ static constexpr RegTypeName DefaultType = RegTypeName::Float64;
+
+ template <RegTypeName = DefaultType>
+ static SetType LiveAsIndexableSet(SetType s) {
+ return SetType(0);
+ }
+
+ template <RegTypeName Name = DefaultType>
+ static SetType AllocatableAsIndexableSet(SetType s) {
+ static_assert(Name != RegTypeName::Any, "Allocatable set are not iterable");
+ return LiveAsIndexableSet<Name>(s);
+ }
+
+ static TypedRegisterSet<FloatRegister> ReduceSetForPush(
+ const TypedRegisterSet<FloatRegister>& s);
+ static uint32_t GetPushSizeInBytes(const TypedRegisterSet<FloatRegister>& s);
+ uint32_t getRegisterDumpOffsetInBytes();
+};
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float32>(SetType set) {
+ return set & FloatRegisters::AllSingleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Float64>(SetType set) {
+ return set & FloatRegisters::AllDoubleMask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Vector128>(SetType set) {
+ return set & FloatRegisters::AllVector128Mask;
+}
+
+template <>
+inline FloatRegister::SetType
+FloatRegister::LiveAsIndexableSet<RegTypeName::Any>(SetType set) {
+ return set;
+}
+
+// Arm/D32 has double registers that can NOT be treated as float32
+// and this requires some dances in lowering.
+inline bool hasUnaliasedDouble() { return false; }
+
+// On ARM, Dn aliases both S2n and S2n+1, so if you need to convert a float32
+// to a double as a temporary, you need a temporary double register.
+inline bool hasMultiAlias() { return false; }
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_Architecture_x86_h */
diff --git a/js/src/jit/x86-shared/Assembler-x86-shared.cpp b/js/src/jit/x86-shared/Assembler-x86-shared.cpp
new file mode 100644
index 0000000000..65fd124cf8
--- /dev/null
+++ b/js/src/jit/x86-shared/Assembler-x86-shared.cpp
@@ -0,0 +1,355 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/Maybe.h"
+
+#include <algorithm>
+
+#include "jit/AutoWritableJitCode.h"
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/MacroAssembler-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/MacroAssembler-x64.h"
+#else
+# error "Wrong architecture. Only x86 and x64 should build this file!"
+#endif
+
+#ifdef _MSC_VER
+# include <intrin.h> // for __cpuid
+# if defined(_M_X64) && (_MSC_FULL_VER >= 160040219)
+# include <immintrin.h> // for _xgetbv
+# endif
+#endif
+
+using namespace js;
+using namespace js::jit;
+
+void AssemblerX86Shared::copyJumpRelocationTable(uint8_t* dest) {
+ if (jumpRelocations_.length()) {
+ memcpy(dest, jumpRelocations_.buffer(), jumpRelocations_.length());
+ }
+}
+
+void AssemblerX86Shared::copyDataRelocationTable(uint8_t* dest) {
+ if (dataRelocations_.length()) {
+ memcpy(dest, dataRelocations_.buffer(), dataRelocations_.length());
+ }
+}
+
+/* static */
+void AssemblerX86Shared::TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ mozilla::Maybe<AutoWritableJitCode> awjc;
+
+ while (reader.more()) {
+ size_t offset = reader.readUnsigned();
+ MOZ_ASSERT(offset >= sizeof(void*) && offset <= code->instructionsSize());
+
+ uint8_t* src = code->raw() + offset;
+ void* data = X86Encoding::GetPointer(src);
+
+#ifdef JS_PUNBOX64
+ // Data relocations can be for Values or for raw pointers. If a Value is
+ // zero-tagged, we can trace it as if it were a raw pointer. If a Value
+ // is not zero-tagged, we have to interpret it as a Value to ensure that the
+ // tag bits are masked off to recover the actual pointer.
+
+ uintptr_t word = reinterpret_cast<uintptr_t>(data);
+ if (word >> JSVAL_TAG_SHIFT) {
+ // This relocation is a Value with a non-zero tag.
+ Value value = Value::fromRawBits(word);
+ MOZ_ASSERT_IF(value.isGCThing(),
+ gc::IsCellPointerValid(value.toGCThing()));
+ TraceManuallyBarrieredEdge(trc, &value, "jit-masm-value");
+ if (word != value.asRawBits()) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ X86Encoding::SetPointer(src, value.bitsAsPunboxPointer());
+ }
+ continue;
+ }
+#endif
+
+ // This relocation is a raw pointer or a Value with a zero tag.
+ gc::Cell* cell = static_cast<gc::Cell*>(data);
+ MOZ_ASSERT(gc::IsCellPointerValid(cell));
+ TraceManuallyBarrieredGenericPointerEdge(trc, &cell, "jit-masm-ptr");
+ if (cell != data) {
+ if (awjc.isNothing()) {
+ awjc.emplace(code);
+ }
+ X86Encoding::SetPointer(src, cell);
+ }
+ }
+}
+
+void AssemblerX86Shared::executableCopy(void* buffer) {
+ masm.executableCopy(buffer);
+}
+
+void AssemblerX86Shared::processCodeLabels(uint8_t* rawCode) {
+ for (const CodeLabel& label : codeLabels_) {
+ Bind(rawCode, label);
+ }
+}
+
+AssemblerX86Shared::Condition AssemblerX86Shared::InvertCondition(
+ Condition cond) {
+ switch (cond) {
+ case Zero:
+ return NonZero;
+ case NonZero:
+ return Zero;
+ case LessThan:
+ return GreaterThanOrEqual;
+ case LessThanOrEqual:
+ return GreaterThan;
+ case GreaterThan:
+ return LessThanOrEqual;
+ case GreaterThanOrEqual:
+ return LessThan;
+ case Above:
+ return BelowOrEqual;
+ case AboveOrEqual:
+ return Below;
+ case Below:
+ return AboveOrEqual;
+ case BelowOrEqual:
+ return Above;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+AssemblerX86Shared::Condition AssemblerX86Shared::UnsignedCondition(
+ Condition cond) {
+ switch (cond) {
+ case Zero:
+ case NonZero:
+ return cond;
+ case LessThan:
+ case Below:
+ return Below;
+ case LessThanOrEqual:
+ case BelowOrEqual:
+ return BelowOrEqual;
+ case GreaterThan:
+ case Above:
+ return Above;
+ case AboveOrEqual:
+ case GreaterThanOrEqual:
+ return AboveOrEqual;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+AssemblerX86Shared::Condition AssemblerX86Shared::ConditionWithoutEqual(
+ Condition cond) {
+ switch (cond) {
+ case LessThan:
+ case LessThanOrEqual:
+ return LessThan;
+ case Below:
+ case BelowOrEqual:
+ return Below;
+ case GreaterThan:
+ case GreaterThanOrEqual:
+ return GreaterThan;
+ case Above:
+ case AboveOrEqual:
+ return Above;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+AssemblerX86Shared::DoubleCondition AssemblerX86Shared::InvertCondition(
+ DoubleCondition cond) {
+ switch (cond) {
+ case DoubleEqual:
+ return DoubleNotEqualOrUnordered;
+ case DoubleEqualOrUnordered:
+ return DoubleNotEqual;
+ case DoubleNotEqualOrUnordered:
+ return DoubleEqual;
+ case DoubleNotEqual:
+ return DoubleEqualOrUnordered;
+ case DoubleLessThan:
+ return DoubleGreaterThanOrEqualOrUnordered;
+ case DoubleLessThanOrUnordered:
+ return DoubleGreaterThanOrEqual;
+ case DoubleLessThanOrEqual:
+ return DoubleGreaterThanOrUnordered;
+ case DoubleLessThanOrEqualOrUnordered:
+ return DoubleGreaterThan;
+ case DoubleGreaterThan:
+ return DoubleLessThanOrEqualOrUnordered;
+ case DoubleGreaterThanOrUnordered:
+ return DoubleLessThanOrEqual;
+ case DoubleGreaterThanOrEqual:
+ return DoubleLessThanOrUnordered;
+ case DoubleGreaterThanOrEqualOrUnordered:
+ return DoubleLessThan;
+ default:
+ MOZ_CRASH("unexpected condition");
+ }
+}
+
+CPUInfo::SSEVersion CPUInfo::maxSSEVersion = UnknownSSE;
+CPUInfo::SSEVersion CPUInfo::maxEnabledSSEVersion = UnknownSSE;
+bool CPUInfo::avxPresent = false;
+#ifdef ENABLE_WASM_AVX
+bool CPUInfo::avxEnabled = true;
+#else
+bool CPUInfo::avxEnabled = false;
+#endif
+bool CPUInfo::popcntPresent = false;
+bool CPUInfo::bmi1Present = false;
+bool CPUInfo::bmi2Present = false;
+bool CPUInfo::lzcntPresent = false;
+bool CPUInfo::avx2Present = false;
+bool CPUInfo::fmaPresent = false;
+
+namespace js {
+namespace jit {
+bool CPUFlagsHaveBeenComputed() { return CPUInfo::FlagsHaveBeenComputed(); }
+} // namespace jit
+} // namespace js
+
+static uintptr_t ReadXGETBV() {
+ // We use a variety of low-level mechanisms to get at the xgetbv
+ // instruction, including spelling out the xgetbv instruction as bytes,
+ // because older compilers and assemblers may not recognize the instruction
+ // by name.
+ size_t xcr0EAX = 0;
+#if defined(_XCR_XFEATURE_ENABLED_MASK)
+ xcr0EAX = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
+#elif defined(__GNUC__)
+ // xgetbv returns its results in %eax and %edx, and for our purposes here,
+ // we're only interested in the %eax value.
+ asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0EAX) : "c"(0) : "%edx");
+#elif defined(_MSC_VER) && defined(_M_IX86)
+ __asm {
+ xor ecx, ecx
+ _asm _emit 0x0f _asm _emit 0x01 _asm _emit 0xd0
+ mov xcr0EAX, eax
+ }
+#endif
+ return xcr0EAX;
+}
+
+static void ReadCPUInfo(int* flagsEax, int* flagsEbx, int* flagsEcx,
+ int* flagsEdx) {
+#ifdef _MSC_VER
+ int cpuinfo[4];
+ __cpuid(cpuinfo, *flagsEax);
+ *flagsEax = cpuinfo[0];
+ *flagsEbx = cpuinfo[1];
+ *flagsEcx = cpuinfo[2];
+ *flagsEdx = cpuinfo[3];
+#elif defined(__GNUC__)
+ // Some older 32-bits processors don't fill the ecx register with cpuid, so
+ // clobber it before calling cpuid, so that there's no risk of picking
+ // random bits indicating SSE3/SSE4 are present. Also make sure that it's
+ // set to 0 as an input for BMI detection on all platforms.
+ *flagsEcx = 0;
+# ifdef JS_CODEGEN_X64
+ asm("cpuid;"
+ : "+a"(*flagsEax), "=b"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
+# else
+ // On x86, preserve ebx. The compiler needs it for PIC mode.
+ asm("mov %%ebx, %%edi;"
+ "cpuid;"
+ "xchg %%edi, %%ebx;"
+ : "+a"(*flagsEax), "=D"(*flagsEbx), "+c"(*flagsEcx), "=d"(*flagsEdx));
+# endif
+#else
+# error "Unsupported compiler"
+#endif
+}
+
+void CPUInfo::ComputeFlags() {
+ MOZ_ASSERT(!FlagsHaveBeenComputed());
+
+ int flagsEax = 1;
+ int flagsEbx = 0;
+ int flagsEcx = 0;
+ int flagsEdx = 0;
+ ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
+
+ static constexpr int SSEBit = 1 << 25;
+ static constexpr int SSE2Bit = 1 << 26;
+ static constexpr int SSE3Bit = 1 << 0;
+ static constexpr int SSSE3Bit = 1 << 9;
+ static constexpr int SSE41Bit = 1 << 19;
+ static constexpr int SSE42Bit = 1 << 20;
+
+ if (flagsEcx & SSE42Bit) {
+ maxSSEVersion = SSE4_2;
+ } else if (flagsEcx & SSE41Bit) {
+ maxSSEVersion = SSE4_1;
+ } else if (flagsEcx & SSSE3Bit) {
+ maxSSEVersion = SSSE3;
+ } else if (flagsEcx & SSE3Bit) {
+ maxSSEVersion = SSE3;
+ } else if (flagsEdx & SSE2Bit) {
+ maxSSEVersion = SSE2;
+ } else if (flagsEdx & SSEBit) {
+ maxSSEVersion = SSE;
+ } else {
+ maxSSEVersion = NoSSE;
+ }
+
+ if (maxEnabledSSEVersion != UnknownSSE) {
+ maxSSEVersion = std::min(maxSSEVersion, maxEnabledSSEVersion);
+ }
+
+ static constexpr int AVXBit = 1 << 28;
+ static constexpr int XSAVEBit = 1 << 27;
+ avxPresent = (flagsEcx & AVXBit) && (flagsEcx & XSAVEBit) && avxEnabled;
+
+ // If the hardware supports AVX, check whether the OS supports it too.
+ if (avxPresent) {
+ size_t xcr0EAX = ReadXGETBV();
+ static constexpr int xcr0SSEBit = 1 << 1;
+ static constexpr int xcr0AVXBit = 1 << 2;
+ avxPresent = (xcr0EAX & xcr0SSEBit) && (xcr0EAX & xcr0AVXBit);
+ }
+
+ // CMOV instruction are supposed to be supported by all CPU which have SSE2
+ // enabled. While this might be true, this is not guaranteed by any
+ // documentation, nor AMD, nor Intel.
+ static constexpr int CMOVBit = 1 << 15;
+ MOZ_RELEASE_ASSERT(flagsEdx & CMOVBit,
+ "CMOVcc instruction is not recognized by this CPU.");
+
+ static constexpr int POPCNTBit = 1 << 23;
+ popcntPresent = (flagsEcx & POPCNTBit);
+
+ // Use the avxEnabled flag to enable/disable FMA.
+ static constexpr int FMABit = 1 << 12;
+ fmaPresent = (flagsEcx & FMABit) && avxEnabled;
+
+ flagsEax = 0x80000001;
+ ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
+
+ static constexpr int LZCNTBit = 1 << 5;
+ lzcntPresent = (flagsEcx & LZCNTBit);
+
+ flagsEax = 0x7;
+ ReadCPUInfo(&flagsEax, &flagsEbx, &flagsEcx, &flagsEdx);
+
+ static constexpr int BMI1Bit = 1 << 3;
+ static constexpr int BMI2Bit = 1 << 8;
+ static constexpr int AVX2Bit = 1 << 5;
+ bmi1Present = (flagsEbx & BMI1Bit);
+ bmi2Present = bmi1Present && (flagsEbx & BMI2Bit);
+ avx2Present = avxPresent && (flagsEbx & AVX2Bit);
+
+ MOZ_ASSERT(FlagsHaveBeenComputed());
+}
diff --git a/js/src/jit/x86-shared/Assembler-x86-shared.h b/js/src/jit/x86-shared/Assembler-x86-shared.h
new file mode 100644
index 0000000000..922a2de029
--- /dev/null
+++ b/js/src/jit/x86-shared/Assembler-x86-shared.h
@@ -0,0 +1,4887 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_Assembler_x86_shared_h
+#define jit_x86_shared_Assembler_x86_shared_h
+
+#include <cstddef>
+
+#include "jit/shared/Assembler-shared.h"
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/BaseAssembler-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/BaseAssembler-x64.h"
+#else
+# error "Unknown architecture!"
+#endif
+#include "jit/CompactBuffer.h"
+#include "wasm/WasmTypeDecls.h"
+
+namespace js {
+namespace jit {
+
+// Do not reference ScratchFloat32Reg_ directly, use ScratchFloat32Scope
+// instead.
+struct ScratchFloat32Scope : public AutoFloatRegisterScope {
+ explicit ScratchFloat32Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchFloat32Reg_) {}
+};
+
+// Do not reference ScratchDoubleReg_ directly, use ScratchDoubleScope instead.
+struct ScratchDoubleScope : public AutoFloatRegisterScope {
+ explicit ScratchDoubleScope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchDoubleReg_) {}
+};
+
+struct ScratchSimd128Scope : public AutoFloatRegisterScope {
+ explicit ScratchSimd128Scope(MacroAssembler& masm)
+ : AutoFloatRegisterScope(masm, ScratchSimd128Reg) {}
+};
+
+class Operand {
+ public:
+ enum Kind { REG, MEM_REG_DISP, FPREG, MEM_SCALE, MEM_ADDRESS32 };
+
+ private:
+ Kind kind_ : 4;
+ // Used as a Register::Encoding and a FloatRegister::Encoding.
+ uint32_t base_ : 5;
+ Scale scale_ : 3;
+ // We don't use all 8 bits, of course, but GCC complains if the size of
+ // this field is smaller than the size of Register::Encoding.
+ Register::Encoding index_ : 8;
+ int32_t disp_;
+
+ public:
+ explicit Operand(Register reg)
+ : kind_(REG),
+ base_(reg.encoding()),
+ scale_(TimesOne),
+ index_(Registers::Invalid),
+ disp_(0) {}
+ explicit Operand(FloatRegister reg)
+ : kind_(FPREG),
+ base_(reg.encoding()),
+ scale_(TimesOne),
+ index_(Registers::Invalid),
+ disp_(0) {}
+ explicit Operand(const Address& address)
+ : kind_(MEM_REG_DISP),
+ base_(address.base.encoding()),
+ scale_(TimesOne),
+ index_(Registers::Invalid),
+ disp_(address.offset) {}
+ explicit Operand(const BaseIndex& address)
+ : kind_(MEM_SCALE),
+ base_(address.base.encoding()),
+ scale_(address.scale),
+ index_(address.index.encoding()),
+ disp_(address.offset) {}
+ Operand(Register base, Register index, Scale scale, int32_t disp = 0)
+ : kind_(MEM_SCALE),
+ base_(base.encoding()),
+ scale_(scale),
+ index_(index.encoding()),
+ disp_(disp) {}
+ Operand(Register reg, int32_t disp)
+ : kind_(MEM_REG_DISP),
+ base_(reg.encoding()),
+ scale_(TimesOne),
+ index_(Registers::Invalid),
+ disp_(disp) {}
+ explicit Operand(AbsoluteAddress address)
+ : kind_(MEM_ADDRESS32),
+ base_(Registers::Invalid),
+ scale_(TimesOne),
+ index_(Registers::Invalid),
+ disp_(X86Encoding::AddressImmediate(address.addr)) {}
+ explicit Operand(PatchedAbsoluteAddress address)
+ : kind_(MEM_ADDRESS32),
+ base_(Registers::Invalid),
+ scale_(TimesOne),
+ index_(Registers::Invalid),
+ disp_(X86Encoding::AddressImmediate(address.addr)) {}
+
+ Address toAddress() const {
+ MOZ_ASSERT(kind() == MEM_REG_DISP);
+ return Address(Register::FromCode(base()), disp());
+ }
+
+ BaseIndex toBaseIndex() const {
+ MOZ_ASSERT(kind() == MEM_SCALE);
+ return BaseIndex(Register::FromCode(base()), Register::FromCode(index()),
+ scale(), disp());
+ }
+
+ Kind kind() const { return kind_; }
+ Register::Encoding reg() const {
+ MOZ_ASSERT(kind() == REG);
+ return Register::Encoding(base_);
+ }
+ Register::Encoding base() const {
+ MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
+ return Register::Encoding(base_);
+ }
+ Register::Encoding index() const {
+ MOZ_ASSERT(kind() == MEM_SCALE);
+ return index_;
+ }
+ Scale scale() const {
+ MOZ_ASSERT(kind() == MEM_SCALE);
+ return scale_;
+ }
+ FloatRegister::Encoding fpu() const {
+ MOZ_ASSERT(kind() == FPREG);
+ return FloatRegister::Encoding(base_);
+ }
+ int32_t disp() const {
+ MOZ_ASSERT(kind() == MEM_REG_DISP || kind() == MEM_SCALE);
+ return disp_;
+ }
+ void* address() const {
+ MOZ_ASSERT(kind() == MEM_ADDRESS32);
+ return reinterpret_cast<void*>(disp_);
+ }
+
+ bool containsReg(Register r) const {
+ switch (kind()) {
+ case REG:
+ return r.encoding() == reg();
+ case MEM_REG_DISP:
+ return r.encoding() == base();
+ case MEM_SCALE:
+ return r.encoding() == base() || r.encoding() == index();
+ default:
+ return false;
+ }
+ }
+};
+
+inline Imm32 Imm64::firstHalf() const { return low(); }
+
+inline Imm32 Imm64::secondHalf() const { return hi(); }
+
+class CPUInfo {
+ public:
+ // As the SSE's were introduced in order, the presence of a later SSE implies
+ // the presence of an earlier SSE. For example, SSE4_2 support implies SSE2
+ // support.
+ enum SSEVersion {
+ UnknownSSE = 0,
+ NoSSE = 1,
+ SSE = 2,
+ SSE2 = 3,
+ SSE3 = 4,
+ SSSE3 = 5,
+ SSE4_1 = 6,
+ SSE4_2 = 7
+ };
+ static const int AVX_PRESENT_BIT = 8;
+
+ static SSEVersion GetSSEVersion() {
+ MOZ_ASSERT(FlagsHaveBeenComputed());
+ MOZ_ASSERT_IF(maxEnabledSSEVersion != UnknownSSE,
+ maxSSEVersion <= maxEnabledSSEVersion);
+ return maxSSEVersion;
+ }
+
+ static bool IsAVXPresent() {
+ MOZ_ASSERT(FlagsHaveBeenComputed());
+ MOZ_ASSERT_IF(!avxEnabled, !avxPresent);
+ return avxPresent;
+ }
+
+ static inline uint32_t GetFingerprint() {
+ return GetSSEVersion() | (IsAVXPresent() ? AVX_PRESENT_BIT : 0);
+ }
+
+ private:
+ static SSEVersion maxSSEVersion;
+ static SSEVersion maxEnabledSSEVersion;
+ static bool avxPresent;
+ static bool avxEnabled;
+ static bool popcntPresent;
+ static bool bmi1Present;
+ static bool bmi2Present;
+ static bool lzcntPresent;
+ static bool fmaPresent;
+ static bool avx2Present;
+
+ static void SetMaxEnabledSSEVersion(SSEVersion v) {
+ if (maxEnabledSSEVersion == UnknownSSE) {
+ maxEnabledSSEVersion = v;
+ } else {
+ maxEnabledSSEVersion = std::min(v, maxEnabledSSEVersion);
+ }
+ }
+
+ public:
+ static bool IsSSE2Present() {
+#ifdef JS_CODEGEN_X64
+ return true;
+#else
+ return GetSSEVersion() >= SSE2;
+#endif
+ }
+ static bool IsSSE3Present() { return GetSSEVersion() >= SSE3; }
+ static bool IsSSSE3Present() { return GetSSEVersion() >= SSSE3; }
+ static bool IsSSE41Present() { return GetSSEVersion() >= SSE4_1; }
+ static bool IsSSE42Present() { return GetSSEVersion() >= SSE4_2; }
+ static bool IsPOPCNTPresent() { return popcntPresent; }
+ static bool IsBMI1Present() { return bmi1Present; }
+ static bool IsBMI2Present() { return bmi2Present; }
+ static bool IsLZCNTPresent() { return lzcntPresent; }
+ static bool IsFMAPresent() { return fmaPresent; }
+ static bool IsAVX2Present() { return avx2Present; }
+
+ static bool FlagsHaveBeenComputed() { return maxSSEVersion != UnknownSSE; }
+
+ static void ComputeFlags();
+
+ // The following should be called only before JS_Init (where the flags are
+ // computed). If several are called, the most restrictive setting is kept.
+
+ static void SetSSE3Disabled() {
+ MOZ_ASSERT(!FlagsHaveBeenComputed());
+ SetMaxEnabledSSEVersion(SSE2);
+ avxEnabled = false;
+ }
+ static void SetSSSE3Disabled() {
+ MOZ_ASSERT(!FlagsHaveBeenComputed());
+ SetMaxEnabledSSEVersion(SSE3);
+ avxEnabled = false;
+ }
+ static void SetSSE41Disabled() {
+ MOZ_ASSERT(!FlagsHaveBeenComputed());
+ SetMaxEnabledSSEVersion(SSSE3);
+ avxEnabled = false;
+ }
+ static void SetSSE42Disabled() {
+ MOZ_ASSERT(!FlagsHaveBeenComputed());
+ SetMaxEnabledSSEVersion(SSE4_1);
+ avxEnabled = false;
+ }
+ static void SetAVXDisabled() {
+ MOZ_ASSERT(!FlagsHaveBeenComputed());
+ avxEnabled = false;
+ }
+ static void SetAVXEnabled() {
+ MOZ_ASSERT(!FlagsHaveBeenComputed());
+ avxEnabled = true;
+ }
+};
+
+class AssemblerX86Shared : public AssemblerShared {
+ protected:
+ struct RelativePatch {
+ int32_t offset;
+ void* target;
+ RelocationKind kind;
+
+ RelativePatch(int32_t offset, void* target, RelocationKind kind)
+ : offset(offset), target(target), kind(kind) {}
+ };
+
+ CompactBufferWriter jumpRelocations_;
+ CompactBufferWriter dataRelocations_;
+
+ void writeDataRelocation(ImmGCPtr ptr) {
+ // Raw GC pointer relocations and Value relocations both end up in
+ // Assembler::TraceDataRelocations.
+ if (ptr.value) {
+ if (gc::IsInsideNursery(ptr.value)) {
+ embedsNurseryPointers_ = true;
+ }
+ dataRelocations_.writeUnsigned(masm.currentOffset());
+ }
+ }
+
+ protected:
+ X86Encoding::BaseAssemblerSpecific masm;
+
+ using JmpSrc = X86Encoding::JmpSrc;
+ using JmpDst = X86Encoding::JmpDst;
+
+ public:
+ AssemblerX86Shared() {
+ if (!HasAVX()) {
+ masm.disableVEX();
+ }
+ }
+
+ enum Condition {
+ Equal = X86Encoding::ConditionE,
+ NotEqual = X86Encoding::ConditionNE,
+ Above = X86Encoding::ConditionA,
+ AboveOrEqual = X86Encoding::ConditionAE,
+ Below = X86Encoding::ConditionB,
+ BelowOrEqual = X86Encoding::ConditionBE,
+ GreaterThan = X86Encoding::ConditionG,
+ GreaterThanOrEqual = X86Encoding::ConditionGE,
+ LessThan = X86Encoding::ConditionL,
+ LessThanOrEqual = X86Encoding::ConditionLE,
+ Overflow = X86Encoding::ConditionO,
+ NoOverflow = X86Encoding::ConditionNO,
+ CarrySet = X86Encoding::ConditionC,
+ CarryClear = X86Encoding::ConditionNC,
+ Signed = X86Encoding::ConditionS,
+ NotSigned = X86Encoding::ConditionNS,
+ Zero = X86Encoding::ConditionE,
+ NonZero = X86Encoding::ConditionNE,
+ Parity = X86Encoding::ConditionP,
+ NoParity = X86Encoding::ConditionNP
+ };
+
+ enum class SSERoundingMode {
+ Nearest = int(X86Encoding::SSERoundingMode::RoundToNearest),
+ Floor = int(X86Encoding::SSERoundingMode::RoundDown),
+ Ceil = int(X86Encoding::SSERoundingMode::RoundUp),
+ Trunc = int(X86Encoding::SSERoundingMode::RoundToZero)
+ };
+
+ // If this bit is set, the vucomisd operands have to be inverted.
+ static const int DoubleConditionBitInvert = 0x10;
+
+ // Bit set when a DoubleCondition does not map to a single x86 condition.
+ // The macro assembler has to special-case these conditions.
+ static const int DoubleConditionBitSpecial = 0x20;
+ static const int DoubleConditionBits =
+ DoubleConditionBitInvert | DoubleConditionBitSpecial;
+
+ enum DoubleCondition {
+ // These conditions will only evaluate to true if the comparison is ordered
+ // - i.e. neither operand is NaN.
+ DoubleOrdered = NoParity,
+ DoubleEqual = Equal | DoubleConditionBitSpecial,
+ DoubleNotEqual = NotEqual,
+ DoubleGreaterThan = Above,
+ DoubleGreaterThanOrEqual = AboveOrEqual,
+ DoubleLessThan = Above | DoubleConditionBitInvert,
+ DoubleLessThanOrEqual = AboveOrEqual | DoubleConditionBitInvert,
+ // If either operand is NaN, these conditions always evaluate to true.
+ DoubleUnordered = Parity,
+ DoubleEqualOrUnordered = Equal,
+ DoubleNotEqualOrUnordered = NotEqual | DoubleConditionBitSpecial,
+ DoubleGreaterThanOrUnordered = Below | DoubleConditionBitInvert,
+ DoubleGreaterThanOrEqualOrUnordered =
+ BelowOrEqual | DoubleConditionBitInvert,
+ DoubleLessThanOrUnordered = Below,
+ DoubleLessThanOrEqualOrUnordered = BelowOrEqual
+ };
+
+ enum NaNCond { NaN_HandledByCond, NaN_IsTrue, NaN_IsFalse };
+
+ // If the primary condition returned by ConditionFromDoubleCondition doesn't
+ // handle NaNs properly, return NaN_IsFalse if the comparison should be
+ // overridden to return false on NaN, NaN_IsTrue if it should be overridden
+ // to return true on NaN, or NaN_HandledByCond if no secondary check is
+ // needed.
+ static inline NaNCond NaNCondFromDoubleCondition(DoubleCondition cond) {
+ switch (cond) {
+ case DoubleOrdered:
+ case DoubleNotEqual:
+ case DoubleGreaterThan:
+ case DoubleGreaterThanOrEqual:
+ case DoubleLessThan:
+ case DoubleLessThanOrEqual:
+ case DoubleUnordered:
+ case DoubleEqualOrUnordered:
+ case DoubleGreaterThanOrUnordered:
+ case DoubleGreaterThanOrEqualOrUnordered:
+ case DoubleLessThanOrUnordered:
+ case DoubleLessThanOrEqualOrUnordered:
+ return NaN_HandledByCond;
+ case DoubleEqual:
+ return NaN_IsFalse;
+ case DoubleNotEqualOrUnordered:
+ return NaN_IsTrue;
+ }
+
+ MOZ_CRASH("Unknown double condition");
+ }
+
+ static void StaticAsserts() {
+ // DoubleConditionBits should not interfere with x86 condition codes.
+ static_assert(!((Equal | NotEqual | Above | AboveOrEqual | Below |
+ BelowOrEqual | Parity | NoParity) &
+ DoubleConditionBits));
+ }
+
+ static Condition InvertCondition(Condition cond);
+ static Condition UnsignedCondition(Condition cond);
+ static Condition ConditionWithoutEqual(Condition cond);
+
+ static DoubleCondition InvertCondition(DoubleCondition cond);
+
+ // Return the primary condition to test. Some primary conditions may not
+ // handle NaNs properly and may therefore require a secondary condition.
+ // Use NaNCondFromDoubleCondition to determine what else is needed.
+ static inline Condition ConditionFromDoubleCondition(DoubleCondition cond) {
+ return static_cast<Condition>(cond & ~DoubleConditionBits);
+ }
+
+ static void TraceDataRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ void setUnlimitedBuffer() {
+ // No-op on this platform
+ }
+ bool oom() const {
+ return AssemblerShared::oom() || masm.oom() || jumpRelocations_.oom() ||
+ dataRelocations_.oom();
+ }
+ bool reserve(size_t size) { return masm.reserve(size); }
+ bool swapBuffer(wasm::Bytes& other) { return masm.swapBuffer(other); }
+
+ void setPrinter(Sprinter* sp) { masm.setPrinter(sp); }
+
+ Register getStackPointer() const { return StackPointer; }
+
+ void executableCopy(void* buffer);
+ void processCodeLabels(uint8_t* rawCode);
+ void copyJumpRelocationTable(uint8_t* dest);
+ void copyDataRelocationTable(uint8_t* dest);
+
+ // Size of the instruction stream, in bytes.
+ size_t size() const { return masm.size(); }
+ // Size of the jump relocation table, in bytes.
+ size_t jumpRelocationTableBytes() const { return jumpRelocations_.length(); }
+ size_t dataRelocationTableBytes() const { return dataRelocations_.length(); }
+ // Size of the data table, in bytes.
+ size_t bytesNeeded() const {
+ return size() + jumpRelocationTableBytes() + dataRelocationTableBytes();
+ }
+
+ public:
+ void haltingAlign(int alignment) {
+ MOZ_ASSERT(hasCreator());
+ masm.haltingAlign(alignment);
+ }
+ void nopAlign(int alignment) {
+ MOZ_ASSERT(hasCreator());
+ masm.nopAlign(alignment);
+ }
+ void writeCodePointer(CodeLabel* label) {
+ MOZ_ASSERT(hasCreator());
+ // Use -1 as dummy value. This will be patched after codegen.
+ masm.jumpTablePointer(-1);
+ label->patchAt()->bind(masm.size());
+ }
+ void cmovCCl(Condition cond, const Operand& src, Register dest) {
+ X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.cmovCCl_rr(cc, src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmovCCl_mr(cc, src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmovCCl_mr(cc, src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmovCCl(Condition cond, Register src, Register dest) {
+ X86Encoding::Condition cc = static_cast<X86Encoding::Condition>(cond);
+ masm.cmovCCl_rr(cc, src.encoding(), dest.encoding());
+ }
+ void cmovzl(const Operand& src, Register dest) {
+ cmovCCl(Condition::Zero, src, dest);
+ }
+ void cmovnzl(const Operand& src, Register dest) {
+ cmovCCl(Condition::NonZero, src, dest);
+ }
+ void movl(Imm32 imm32, Register dest) {
+ MOZ_ASSERT(hasCreator());
+ masm.movl_i32r(imm32.value, dest.encoding());
+ }
+ void movl(Register src, Register dest) {
+ MOZ_ASSERT(hasCreator());
+ masm.movl_rr(src.encoding(), dest.encoding());
+ }
+ void movl(const Operand& src, Register dest) {
+ MOZ_ASSERT(hasCreator());
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movl_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movl_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movl_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movl(Register src, const Operand& dest) {
+ MOZ_ASSERT(hasCreator());
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.movl_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movl_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movl_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movl(Imm32 imm32, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.movl_i32r(imm32.value, dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movl_i32m(imm32.value, dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movl_i32m(imm32.value, dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movl_i32m(imm32.value, dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void xchgl(Register src, Register dest) {
+ masm.xchgl_rr(src.encoding(), dest.encoding());
+ }
+
+ void vmovapd(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovapd_rr(src.encoding(), dest.encoding());
+ }
+ // Eventually vmovapd should be overloaded to support loads and
+ // stores too.
+ void vmovapd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vmovapd_rr(src.fpu(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vmovaps(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovaps_rr(src.encoding(), dest.encoding());
+ }
+ void vmovaps(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovaps_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovaps_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::FPREG:
+ masm.vmovaps_rr(src.fpu(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovaps(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovaps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovups(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovups_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovups_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovups(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovups_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovups_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vmovsd(const Address& src, FloatRegister dest) {
+ masm.vmovsd_mr(src.offset, src.base.encoding(), dest.encoding());
+ }
+ void vmovsd(const BaseIndex& src, FloatRegister dest) {
+ masm.vmovsd_mr(src.offset, src.base.encoding(), src.index.encoding(),
+ src.scale, dest.encoding());
+ }
+ void vmovsd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(hasCreator());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ vmovsd(src.toAddress(), dest);
+ break;
+ case Operand::MEM_SCALE:
+ vmovsd(src.toBaseIndex(), dest);
+ break;
+ default:
+ MOZ_CRASH("Unknown operand for vmovsd");
+ }
+ }
+ void vmovsd(FloatRegister src, const Address& dest) {
+ masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding());
+ }
+ void vmovsd(FloatRegister src, const BaseIndex& dest) {
+ masm.vmovsd_rm(src.encoding(), dest.offset, dest.base.encoding(),
+ dest.index.encoding(), dest.scale);
+ }
+ // Note special semantics of this - does not clobber high bits of destination.
+ void vmovsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ masm.vmovsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vmovss(const Address& src, FloatRegister dest) {
+ masm.vmovss_mr(src.offset, src.base.encoding(), dest.encoding());
+ }
+ void vmovss(const BaseIndex& src, FloatRegister dest) {
+ masm.vmovss_mr(src.offset, src.base.encoding(), src.index.encoding(),
+ src.scale, dest.encoding());
+ }
+ void vmovss(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(hasCreator());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ vmovss(src.toAddress(), dest);
+ break;
+ case Operand::MEM_SCALE:
+ vmovss(src.toBaseIndex(), dest);
+ break;
+ default:
+ MOZ_CRASH("Unknown operand for vmovss");
+ }
+ }
+ void vmovss(FloatRegister src, const Address& dest) {
+ masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding());
+ }
+ void vmovss(FloatRegister src, const BaseIndex& dest) {
+ masm.vmovss_rm(src.encoding(), dest.offset, dest.base.encoding(),
+ dest.index.encoding(), dest.scale);
+ }
+ void vmovss(FloatRegister src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ vmovss(src, dest.toAddress());
+ break;
+ case Operand::MEM_SCALE:
+ vmovss(src, dest.toBaseIndex());
+ break;
+ default:
+ MOZ_CRASH("Unknown operand for vmovss");
+ }
+ }
+ // Note special semantics of this - does not clobber high bits of destination.
+ void vmovss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ masm.vmovss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vmovdqu(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(hasCreator());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovdqu_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovdqu_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovdqu(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(hasCreator());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovdqu_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovdqa(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vmovdqa_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmovdqa_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovdqa_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovdqa(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovdqa_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovdqa(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovdqa_rr(src.encoding(), dest.encoding());
+ }
+ void vcvtss2sd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vcvtss2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vcvtsd2ss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vcvtsd2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void movzbl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movzbl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movzbl_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movsbl(Register src, Register dest) {
+ masm.movsbl_rr(src.encoding(), dest.encoding());
+ }
+ void movsbl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movsbl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movsbl_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movb(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movb_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movb_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movb(Imm32 src, Register dest) {
+ masm.movb_ir(src.value & 255, dest.encoding());
+ }
+ void movb(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movb_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movb_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movb(Imm32 src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movb_im(src.value, dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movb_im(src.value, dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movzwl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.movzwl_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movzwl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movzwl_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movzwl(Register src, Register dest) {
+ masm.movzwl_rr(src.encoding(), dest.encoding());
+ }
+ void movw(const Operand& src, Register dest) {
+ masm.prefix_16_for_32();
+ movl(src, dest);
+ }
+ void movw(Imm32 src, Register dest) {
+ masm.prefix_16_for_32();
+ movl(src, dest);
+ }
+ void movw(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movw_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movw(Imm32 src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movw_im(src.value, dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movw_im(src.value, dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movswl(Register src, Register dest) {
+ masm.movswl_rr(src.encoding(), dest.encoding());
+ }
+ void movswl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movswl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.movswl_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void leal(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.leal_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.leal_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ protected:
+ void jSrc(Condition cond, Label* label) {
+ if (label->bound()) {
+ // The jump can be immediately encoded to the correct destination.
+ masm.jCC_i(static_cast<X86Encoding::Condition>(cond),
+ JmpDst(label->offset()));
+ } else {
+ // Thread the jump list through the unpatched jump targets.
+ JmpSrc j = masm.jCC(static_cast<X86Encoding::Condition>(cond));
+ JmpSrc prev;
+ if (label->used()) {
+ prev = JmpSrc(label->offset());
+ }
+ label->use(j.offset());
+ masm.setNextJump(j, prev);
+ }
+ }
+ void jmpSrc(Label* label) {
+ if (label->bound()) {
+ // The jump can be immediately encoded to the correct destination.
+ masm.jmp_i(JmpDst(label->offset()));
+ } else {
+ // Thread the jump list through the unpatched jump targets.
+ JmpSrc j = masm.jmp();
+ JmpSrc prev;
+ if (label->used()) {
+ prev = JmpSrc(label->offset());
+ }
+ label->use(j.offset());
+ masm.setNextJump(j, prev);
+ }
+ }
+
+ // Comparison of EAX against the address given by a Label.
+ JmpSrc cmpSrc(Label* label) {
+ JmpSrc j = masm.cmp_eax();
+ if (label->bound()) {
+ // The jump can be immediately patched to the correct destination.
+ masm.linkJump(j, JmpDst(label->offset()));
+ } else {
+ // Thread the jump list through the unpatched jump targets.
+ JmpSrc prev;
+ if (label->used()) {
+ prev = JmpSrc(label->offset());
+ }
+ label->use(j.offset());
+ masm.setNextJump(j, prev);
+ }
+ return j;
+ }
+
+ public:
+ void nop() {
+ MOZ_ASSERT(hasCreator());
+ masm.nop();
+ }
+ void nop(size_t n) {
+ MOZ_ASSERT(hasCreator());
+ masm.insert_nop(n);
+ }
+ void j(Condition cond, Label* label) {
+ MOZ_ASSERT(hasCreator());
+ jSrc(cond, label);
+ }
+ void jmp(Label* label) {
+ MOZ_ASSERT(hasCreator());
+ jmpSrc(label);
+ }
+
+ void jmp(const Operand& op) {
+ MOZ_ASSERT(hasCreator());
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.jmp_m(op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.jmp_m(op.disp(), op.base(), op.index(), op.scale());
+ break;
+ case Operand::REG:
+ masm.jmp_r(op.reg());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpEAX(Label* label) { cmpSrc(label); }
+ void bind(Label* label) {
+ JmpDst dst(masm.label());
+ if (label->used()) {
+ bool more;
+ JmpSrc jmp(label->offset());
+ do {
+ JmpSrc next;
+ more = masm.nextJump(jmp, &next);
+ masm.linkJump(jmp, dst);
+ jmp = next;
+ } while (more);
+ }
+ label->bind(dst.offset());
+ }
+ void bind(CodeLabel* label) { label->target()->bind(currentOffset()); }
+ uint32_t currentOffset() { return masm.label().offset(); }
+
+ // Re-routes pending jumps to a new label.
+ void retarget(Label* label, Label* target) {
+ if (!label->used()) {
+ return;
+ }
+ bool more;
+ JmpSrc jmp(label->offset());
+ do {
+ JmpSrc next;
+ more = masm.nextJump(jmp, &next);
+ if (target->bound()) {
+ // The jump can be immediately patched to the correct destination.
+ masm.linkJump(jmp, JmpDst(target->offset()));
+ } else {
+ // Thread the jump list through the unpatched jump targets.
+ JmpSrc prev;
+ if (target->used()) {
+ prev = JmpSrc(target->offset());
+ }
+ target->use(jmp.offset());
+ masm.setNextJump(jmp, prev);
+ }
+ jmp = JmpSrc(next.offset());
+ } while (more);
+ label->reset();
+ }
+
+ static void Bind(uint8_t* raw, const CodeLabel& label) {
+ if (label.patchAt().bound()) {
+ intptr_t offset = label.patchAt().offset();
+ intptr_t target = label.target().offset();
+ X86Encoding::SetPointer(raw + offset, raw + target);
+ }
+ }
+
+ void ret() {
+ MOZ_ASSERT(hasCreator());
+ masm.ret();
+ }
+ void retn(Imm32 n) {
+ MOZ_ASSERT(hasCreator());
+ // Remove the size of the return address which is included in the frame.
+ masm.ret_i(n.value - sizeof(void*));
+ }
+ CodeOffset call(Label* label) {
+ JmpSrc j = masm.call();
+ if (label->bound()) {
+ masm.linkJump(j, JmpDst(label->offset()));
+ } else {
+ JmpSrc prev;
+ if (label->used()) {
+ prev = JmpSrc(label->offset());
+ }
+ label->use(j.offset());
+ masm.setNextJump(j, prev);
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset call(Register reg) {
+ masm.call_r(reg.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ void call(const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.call_r(op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.call_m(op.disp(), op.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ CodeOffset callWithPatch() { return CodeOffset(masm.call().offset()); }
+
+ void patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
+ unsigned char* code = masm.data();
+ X86Encoding::SetRel32(code + callerOffset, code + calleeOffset);
+ }
+ CodeOffset farJumpWithPatch() { return CodeOffset(masm.jmp().offset()); }
+ void patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
+ unsigned char* code = masm.data();
+ X86Encoding::SetRel32(code + farJump.offset(), code + targetOffset);
+ }
+
+ // This is for patching during code generation, not after.
+ void patchAddl(CodeOffset offset, int32_t n) {
+ unsigned char* code = masm.data();
+ X86Encoding::SetInt32(code + offset.offset(), n);
+ }
+
+ static void patchFiveByteNopToCall(uint8_t* callsite, uint8_t* target) {
+ X86Encoding::BaseAssembler::patchFiveByteNopToCall(callsite, target);
+ }
+ static void patchCallToFiveByteNop(uint8_t* callsite) {
+ X86Encoding::BaseAssembler::patchCallToFiveByteNop(callsite);
+ }
+
+ void breakpoint() { masm.int3(); }
+ CodeOffset ud2() {
+ MOZ_ASSERT(hasCreator());
+ CodeOffset off(masm.currentOffset());
+ masm.ud2();
+ return off;
+ }
+
+ static bool HasSSE2() { return CPUInfo::IsSSE2Present(); }
+ static bool HasSSE3() { return CPUInfo::IsSSE3Present(); }
+ static bool HasSSSE3() { return CPUInfo::IsSSSE3Present(); }
+ static bool HasSSE41() { return CPUInfo::IsSSE41Present(); }
+ static bool HasSSE42() { return CPUInfo::IsSSE42Present(); }
+ static bool HasPOPCNT() { return CPUInfo::IsPOPCNTPresent(); }
+ static bool HasBMI1() { return CPUInfo::IsBMI1Present(); }
+ static bool HasBMI2() { return CPUInfo::IsBMI2Present(); }
+ static bool HasLZCNT() { return CPUInfo::IsLZCNTPresent(); }
+ static bool SupportsFloatingPoint() { return CPUInfo::IsSSE2Present(); }
+ static bool SupportsUnalignedAccesses() { return true; }
+ static bool SupportsFastUnalignedFPAccesses() { return true; }
+ static bool SupportsWasmSimd() { return CPUInfo::IsSSE41Present(); }
+ static bool HasAVX() { return CPUInfo::IsAVXPresent(); }
+ static bool HasAVX2() { return CPUInfo::IsAVX2Present(); }
+ static bool HasFMA() { return CPUInfo::IsFMAPresent(); }
+
+ static bool HasRoundInstruction(RoundingMode mode) {
+ switch (mode) {
+ case RoundingMode::Up:
+ case RoundingMode::Down:
+ case RoundingMode::NearestTiesToEven:
+ case RoundingMode::TowardsZero:
+ return CPUInfo::IsSSE41Present();
+ }
+ MOZ_CRASH("unexpected mode");
+ }
+
+ void cmpl(Register rhs, Register lhs) {
+ masm.cmpl_rr(rhs.encoding(), lhs.encoding());
+ }
+ void cmpl(const Operand& rhs, Register lhs) {
+ switch (rhs.kind()) {
+ case Operand::REG:
+ masm.cmpl_rr(rhs.reg(), lhs.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpl_mr(rhs.disp(), rhs.base(), lhs.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpl_mr(rhs.address(), lhs.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpl(Register rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpl_rr(rhs.encoding(), lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpl_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpl_rm(rhs.encoding(), lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpl(Imm32 rhs, Register lhs) {
+ masm.cmpl_ir(rhs.value, lhs.encoding());
+ }
+ void cmpl(Imm32 rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpl_ir(rhs.value, lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpl_im(rhs.value, lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpl_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpl_im(rhs.value, lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpw(Register rhs, Register lhs) {
+ masm.cmpw_rr(rhs.encoding(), lhs.encoding());
+ }
+ void cmpw(Imm32 rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpw_ir(rhs.value, lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpw_im(rhs.value, lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpw_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpw_im(rhs.value, lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpb(Register rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpb_rr(rhs.encoding(), lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpb_rm(rhs.encoding(), lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpb_rm(rhs.encoding(), lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpb_rm(rhs.encoding(), lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpb(Imm32 rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpb_ir(rhs.value, lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpb_im(rhs.value, lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpb_im(rhs.value, lhs.disp(), lhs.base(), lhs.index(),
+ lhs.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpb_im(rhs.value, lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void setCC(Condition cond, Register r) {
+ masm.setCC_r(static_cast<X86Encoding::Condition>(cond), r.encoding());
+ }
+ void testb(Register rhs, Register lhs) {
+ MOZ_ASSERT(
+ AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(rhs));
+ MOZ_ASSERT(
+ AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(lhs));
+ masm.testb_rr(rhs.encoding(), lhs.encoding());
+ }
+ void testw(Register rhs, Register lhs) {
+ masm.testw_rr(lhs.encoding(), rhs.encoding());
+ }
+ void testl(Register rhs, Register lhs) {
+ masm.testl_rr(lhs.encoding(), rhs.encoding());
+ }
+ void testl(Imm32 rhs, Register lhs) {
+ masm.testl_ir(rhs.value, lhs.encoding());
+ }
+ void testl(Imm32 rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.testl_ir(rhs.value, lhs.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.testl_i32m(rhs.value, lhs.disp(), lhs.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.testl_i32m(rhs.value, lhs.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+
+ void addl(Imm32 imm, Register dest) {
+ masm.addl_ir(imm.value, dest.encoding());
+ }
+ CodeOffset addlWithPatch(Imm32 imm, Register dest) {
+ masm.addl_i32r(imm.value, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ void addl(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.addl_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addl_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.addl_im(imm.value, op.address());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void addw(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.addw_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addw_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.addw_im(imm.value, op.address());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void subl(Imm32 imm, Register dest) {
+ masm.subl_ir(imm.value, dest.encoding());
+ }
+ void subl(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.subl_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subl_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.subl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void subw(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.subw_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subw_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.subw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void addl(Register src, Register dest) {
+ masm.addl_rr(src.encoding(), dest.encoding());
+ }
+ void addl(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.addl_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addl_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void addw(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.addw_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addw_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void sbbl(Register src, Register dest) {
+ masm.sbbl_rr(src.encoding(), dest.encoding());
+ }
+ void subl(Register src, Register dest) {
+ masm.subl_rr(src.encoding(), dest.encoding());
+ }
+ void subl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.subl_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void subl(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.subl_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subl_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.subl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void subw(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.subw_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.subw_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.subw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void orl(Register reg, Register dest) {
+ masm.orl_rr(reg.encoding(), dest.encoding());
+ }
+ void orl(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.orl_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orl_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.orl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void orw(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.orw_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orw_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.orw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void orl(Imm32 imm, Register reg) { masm.orl_ir(imm.value, reg.encoding()); }
+ void orl(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.orl_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orl_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.orl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void orw(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.orw_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orw_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.orw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorl(Register src, Register dest) {
+ masm.xorl_rr(src.encoding(), dest.encoding());
+ }
+ void xorl(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.xorl_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorl_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorw(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.xorw_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorw_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorl(Imm32 imm, Register reg) {
+ masm.xorl_ir(imm.value, reg.encoding());
+ }
+ void xorl(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.xorl_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorl_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorw(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.xorw_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorw_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void andl(Register src, Register dest) {
+ masm.andl_rr(src.encoding(), dest.encoding());
+ }
+ void andl(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.andl_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andl_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andl_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void andw(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.andw_rr(src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andw_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andw_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void andl(Imm32 imm, Register dest) {
+ masm.andl_ir(imm.value, dest.encoding());
+ }
+ void andl(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.andl_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andl_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andl_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void andw(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::REG:
+ masm.andw_ir(imm.value, op.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andw_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andw_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void addl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.addl_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.addl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void orl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.orl_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.orl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xorl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.xorl_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.xorl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void andl(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.andl_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.andl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andl_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void bsrl(const Register& src, const Register& dest) {
+ masm.bsrl_rr(src.encoding(), dest.encoding());
+ }
+ void bsfl(const Register& src, const Register& dest) {
+ masm.bsfl_rr(src.encoding(), dest.encoding());
+ }
+ void bswapl(Register reg) { masm.bswapl_r(reg.encoding()); }
+ void lzcntl(const Register& src, const Register& dest) {
+ masm.lzcntl_rr(src.encoding(), dest.encoding());
+ }
+ void tzcntl(const Register& src, const Register& dest) {
+ masm.tzcntl_rr(src.encoding(), dest.encoding());
+ }
+ void popcntl(const Register& src, const Register& dest) {
+ masm.popcntl_rr(src.encoding(), dest.encoding());
+ }
+ void imull(Register multiplier) {
+ // Consumes eax as the other argument
+ // and clobbers edx, as result is in edx:eax
+ masm.imull_r(multiplier.encoding());
+ }
+ void umull(Register multiplier) { masm.mull_r(multiplier.encoding()); }
+ void imull(Imm32 imm, Register dest) {
+ masm.imull_ir(imm.value, dest.encoding(), dest.encoding());
+ }
+ void imull(Register src, Register dest) {
+ masm.imull_rr(src.encoding(), dest.encoding());
+ }
+ void imull(Imm32 imm, Register src, Register dest) {
+ masm.imull_ir(imm.value, src.encoding(), dest.encoding());
+ }
+ void imull(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.imull_rr(src.reg(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.imull_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void negl(const Operand& src) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.negl_r(src.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.negl_m(src.disp(), src.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void negl(Register reg) { masm.negl_r(reg.encoding()); }
+ void notl(const Operand& src) {
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.notl_r(src.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.notl_m(src.disp(), src.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void notl(Register reg) { masm.notl_r(reg.encoding()); }
+ void shrl(const Imm32 imm, Register dest) {
+ masm.shrl_ir(imm.value, dest.encoding());
+ }
+ void shll(const Imm32 imm, Register dest) {
+ masm.shll_ir(imm.value, dest.encoding());
+ }
+ void sarl(const Imm32 imm, Register dest) {
+ masm.sarl_ir(imm.value, dest.encoding());
+ }
+ void shrl_cl(Register dest) { masm.shrl_CLr(dest.encoding()); }
+ void shll_cl(Register dest) { masm.shll_CLr(dest.encoding()); }
+ void sarl_cl(Register dest) { masm.sarl_CLr(dest.encoding()); }
+ void shrdl_cl(Register src, Register dest) {
+ masm.shrdl_CLr(src.encoding(), dest.encoding());
+ }
+ void shldl_cl(Register src, Register dest) {
+ masm.shldl_CLr(src.encoding(), dest.encoding());
+ }
+
+ void sarxl(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.sarxl_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+ void shlxl(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.shlxl_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+ void shrxl(Register src, Register shift, Register dest) {
+ MOZ_ASSERT(HasBMI2());
+ masm.shrxl_rrr(src.encoding(), shift.encoding(), dest.encoding());
+ }
+
+ void roll(const Imm32 imm, Register dest) {
+ masm.roll_ir(imm.value, dest.encoding());
+ }
+ void roll_cl(Register dest) { masm.roll_CLr(dest.encoding()); }
+ void rolw(const Imm32 imm, Register dest) {
+ masm.rolw_ir(imm.value, dest.encoding());
+ }
+ void rorl(const Imm32 imm, Register dest) {
+ masm.rorl_ir(imm.value, dest.encoding());
+ }
+ void rorl_cl(Register dest) { masm.rorl_CLr(dest.encoding()); }
+
+ void incl(const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.incl_m32(op.disp(), op.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void lock_incl(const Operand& op) {
+ masm.prefix_lock();
+ incl(op);
+ }
+
+ void decl(const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.decl_m32(op.disp(), op.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void lock_decl(const Operand& op) {
+ masm.prefix_lock();
+ decl(op);
+ }
+
+ void addb(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.addb_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+ void addb(Register src, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.addb_rm(src.encoding(), op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.addb_rm(src.encoding(), op.disp(), op.base(), op.index(),
+ op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+
+ void subb(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.subb_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.subb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+ void subb(Register src, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.subb_rm(src.encoding(), op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.subb_rm(src.encoding(), op.disp(), op.base(), op.index(),
+ op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+
+ void andb(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.andb_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+ void andb(Register src, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.andb_rm(src.encoding(), op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.andb_rm(src.encoding(), op.disp(), op.base(), op.index(),
+ op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+
+ void orb(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.orb_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.orb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+ void orb(Register src, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.orb_rm(src.encoding(), op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.orb_rm(src.encoding(), op.disp(), op.base(), op.index(),
+ op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+
+ void xorb(Imm32 imm, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.xorb_im(imm.value, op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorb_im(imm.value, op.disp(), op.base(), op.index(), op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+ void xorb(Register src, const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.xorb_rm(src.encoding(), op.disp(), op.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xorb_rm(src.encoding(), op.disp(), op.base(), op.index(),
+ op.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ break;
+ }
+ }
+
+ template <typename T>
+ void lock_addb(T src, const Operand& op) {
+ masm.prefix_lock();
+ addb(src, op);
+ }
+ template <typename T>
+ void lock_subb(T src, const Operand& op) {
+ masm.prefix_lock();
+ subb(src, op);
+ }
+ template <typename T>
+ void lock_andb(T src, const Operand& op) {
+ masm.prefix_lock();
+ andb(src, op);
+ }
+ template <typename T>
+ void lock_orb(T src, const Operand& op) {
+ masm.prefix_lock();
+ orb(src, op);
+ }
+ template <typename T>
+ void lock_xorb(T src, const Operand& op) {
+ masm.prefix_lock();
+ xorb(src, op);
+ }
+
+ template <typename T>
+ void lock_addw(T src, const Operand& op) {
+ masm.prefix_lock();
+ addw(src, op);
+ }
+ template <typename T>
+ void lock_subw(T src, const Operand& op) {
+ masm.prefix_lock();
+ subw(src, op);
+ }
+ template <typename T>
+ void lock_andw(T src, const Operand& op) {
+ masm.prefix_lock();
+ andw(src, op);
+ }
+ template <typename T>
+ void lock_orw(T src, const Operand& op) {
+ masm.prefix_lock();
+ orw(src, op);
+ }
+ template <typename T>
+ void lock_xorw(T src, const Operand& op) {
+ masm.prefix_lock();
+ xorw(src, op);
+ }
+
+ // Note, lock_addl(imm, op) is used for a memory barrier on non-SSE2 systems,
+ // among other things. Do not optimize, replace by XADDL, or similar.
+ template <typename T>
+ void lock_addl(T src, const Operand& op) {
+ masm.prefix_lock();
+ addl(src, op);
+ }
+ template <typename T>
+ void lock_subl(T src, const Operand& op) {
+ masm.prefix_lock();
+ subl(src, op);
+ }
+ template <typename T>
+ void lock_andl(T src, const Operand& op) {
+ masm.prefix_lock();
+ andl(src, op);
+ }
+ template <typename T>
+ void lock_orl(T src, const Operand& op) {
+ masm.prefix_lock();
+ orl(src, op);
+ }
+ template <typename T>
+ void lock_xorl(T src, const Operand& op) {
+ masm.prefix_lock();
+ xorl(src, op);
+ }
+
+ void lock_cmpxchgb(Register src, const Operand& mem) {
+ masm.prefix_lock();
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.cmpxchgb(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpxchgb(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void lock_cmpxchgw(Register src, const Operand& mem) {
+ masm.prefix_lock();
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.cmpxchgw(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpxchgw(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void lock_cmpxchgl(Register src, const Operand& mem) {
+ masm.prefix_lock();
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.cmpxchgl(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpxchgl(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void lock_cmpxchg8b(Register srcHi, Register srcLo, Register newHi,
+ Register newLo, const Operand& mem) {
+ masm.prefix_lock();
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.cmpxchg8b(srcHi.encoding(), srcLo.encoding(), newHi.encoding(),
+ newLo.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.cmpxchg8b(srcHi.encoding(), srcLo.encoding(), newHi.encoding(),
+ newLo.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void xchgb(Register src, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.xchgb_rm(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xchgb_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xchgw(Register src, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.xchgw_rm(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xchgw_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void xchgl(Register src, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.xchgl_rm(src.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.xchgl_rm(src.encoding(), mem.disp(), mem.base(), mem.index(),
+ mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void lock_xaddb(Register srcdest, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.lock_xaddb_rm(srcdest.encoding(), mem.disp(), mem.base(),
+ mem.index(), mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void lock_xaddw(Register srcdest, const Operand& mem) {
+ masm.prefix_16_for_32();
+ lock_xaddl(srcdest, mem);
+ }
+ void lock_xaddl(Register srcdest, const Operand& mem) {
+ switch (mem.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.lock_xaddl_rm(srcdest.encoding(), mem.disp(), mem.base(),
+ mem.index(), mem.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void push(const Imm32 imm) { masm.push_i(imm.value); }
+
+ void push(const Operand& src) {
+ MOZ_ASSERT(hasCreator());
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.push_r(src.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.push_m(src.disp(), src.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.push_m(src.disp(), src.base(), src.index(), src.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void push(Register src) {
+ MOZ_ASSERT(hasCreator());
+ masm.push_r(src.encoding());
+ }
+ void push(const Address& src) {
+ masm.push_m(src.offset, src.base.encoding());
+ }
+
+ void pop(const Operand& src) {
+ MOZ_ASSERT(hasCreator());
+ switch (src.kind()) {
+ case Operand::REG:
+ masm.pop_r(src.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.pop_m(src.disp(), src.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void pop(Register src) {
+ MOZ_ASSERT(hasCreator());
+ masm.pop_r(src.encoding());
+ }
+ void pop(const Address& src) { masm.pop_m(src.offset, src.base.encoding()); }
+
+ void pushFlags() { masm.push_flags(); }
+ void popFlags() { masm.pop_flags(); }
+
+#ifdef JS_CODEGEN_X86
+ void pushAllRegs() { masm.pusha(); }
+ void popAllRegs() { masm.popa(); }
+#endif
+
+ // Zero-extend byte to 32-bit integer.
+ void movzbl(Register src, Register dest) {
+ masm.movzbl_rr(src.encoding(), dest.encoding());
+ }
+
+ void cdq() { masm.cdq(); }
+ void idiv(Register divisor) { masm.idivl_r(divisor.encoding()); }
+ void udiv(Register divisor) { masm.divl_r(divisor.encoding()); }
+
+ void vpblendw(uint32_t mask, FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpblendw_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
+ }
+
+ void vpblendvb(FloatRegister mask, FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpblendvb_rr(mask.encoding(), src1.encoding(), src0.encoding(),
+ dest.encoding());
+ }
+
+ void vpinsrb(unsigned lane, const Operand& src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::REG:
+ masm.vpinsrb_irr(lane, src1.reg(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpinsrb_imr(lane, src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpinsrb_imr(lane, src1.disp(), src1.base(), src1.index(),
+ src1.scale(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpinsrw(unsigned lane, const Operand& src1, FloatRegister src0,
+ FloatRegister dest) {
+ switch (src1.kind()) {
+ case Operand::REG:
+ masm.vpinsrw_irr(lane, src1.reg(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpinsrw_imr(lane, src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpinsrw_imr(lane, src1.disp(), src1.base(), src1.index(),
+ src1.scale(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vpinsrd(unsigned lane, Register src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpinsrd_irr(lane, src1.encoding(), src0.encoding(), dest.encoding());
+ }
+
+ void vpextrb(unsigned lane, FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.vpextrb_irr(lane, src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpextrb_irm(lane, src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpextrb_irm(lane, src.encoding(), dest.disp(), dest.base(),
+ dest.index(), dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpextrw(unsigned lane, FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.vpextrw_irr(lane, src.encoding(), dest.reg());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpextrw_irm(lane, src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpextrw_irm(lane, src.encoding(), dest.disp(), dest.base(),
+ dest.index(), dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpextrd(unsigned lane, FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpextrd_irr(lane, src.encoding(), dest.encoding());
+ }
+ void vpsrldq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrldq_ir(shift.value, src0.encoding(), dest.encoding());
+ }
+ void vpslldq(Imm32 shift, FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpslldq_ir(shift.value, src.encoding(), dest.encoding());
+ }
+ void vpsllq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsllq_ir(shift.value, src0.encoding(), dest.encoding());
+ }
+ void vpsllq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsllq_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpsrlq(Imm32 shift, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrlq_ir(shift.value, src0.encoding(), dest.encoding());
+ }
+ void vpsrlq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrlq_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpslld(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpslld_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpslld(Imm32 count, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpslld_ir(count.value, src0.encoding(), dest.encoding());
+ }
+ void vpsrad(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrad_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpsrad(Imm32 count, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrad_ir(count.value, src0.encoding(), dest.encoding());
+ }
+ void vpsrld(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrld_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpsrld(Imm32 count, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrld_ir(count.value, src0.encoding(), dest.encoding());
+ }
+
+ void vpsllw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsllw_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpsllw(Imm32 count, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsllw_ir(count.value, src0.encoding(), dest.encoding());
+ }
+ void vpsraw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsraw_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpsraw(Imm32 count, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsraw_ir(count.value, src0.encoding(), dest.encoding());
+ }
+ void vpsrlw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrlw_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpsrlw(Imm32 count, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpsrlw_ir(count.value, src0.encoding(), dest.encoding());
+ }
+
+ void vcvtsi2sd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::REG:
+ masm.vcvtsi2sd_rr(src1.reg(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vcvtsi2sd_mr(src1.disp(), src1.base(), src1.index(), src1.scale(),
+ src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vcvttsd2si(FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vcvttsd2si_rr(src.encoding(), dest.encoding());
+ }
+ void vcvttss2si(FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vcvttss2si_rr(src.encoding(), dest.encoding());
+ }
+ void vcvtsi2ss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::REG:
+ masm.vcvtsi2ss_rr(src1.reg(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vcvtsi2ss_mr(src1.disp(), src1.base(), src1.index(), src1.scale(),
+ src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vcvtsi2ss(Register src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vcvtsi2ss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vcvtsi2sd(Register src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vcvtsi2sd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vcvttps2dq(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vcvttps2dq_rr(src.encoding(), dest.encoding());
+ }
+ void vcvttpd2dq(FloatRegister src, FloatRegister dest) {
+ masm.vcvttpd2dq_rr(src.encoding(), dest.encoding());
+ }
+ void vcvtdq2ps(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vcvtdq2ps_rr(src.encoding(), dest.encoding());
+ }
+ void vcvtdq2pd(FloatRegister src, FloatRegister dest) {
+ masm.vcvtdq2pd_rr(src.encoding(), dest.encoding());
+ }
+ void vcvtps2pd(FloatRegister src, FloatRegister dest) {
+ masm.vcvtps2pd_rr(src.encoding(), dest.encoding());
+ }
+ void vcvtpd2ps(FloatRegister src, FloatRegister dest) {
+ masm.vcvtpd2ps_rr(src.encoding(), dest.encoding());
+ }
+ void vmovmskpd(FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovmskpd_rr(src.encoding(), dest.encoding());
+ }
+ void vmovmskps(FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovmskps_rr(src.encoding(), dest.encoding());
+ }
+ void vpmovmskb(FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpmovmskb_rr(src.encoding(), dest.encoding());
+ }
+ void vptest(FloatRegister rhs, FloatRegister lhs) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vptest_rr(rhs.encoding(), lhs.encoding());
+ }
+ void vucomisd(FloatRegister rhs, FloatRegister lhs) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vucomisd_rr(rhs.encoding(), lhs.encoding());
+ }
+ void vucomiss(FloatRegister rhs, FloatRegister lhs) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vucomiss_rr(rhs.encoding(), lhs.encoding());
+ }
+
+ void vpcmpeqb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vpcmpeqb_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpcmpeqb_mr(rhs.disp(), rhs.base(), lhs.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpcmpeqb_mr(rhs.address(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpcmpgtb(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vpcmpgtb_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpcmpgtb_mr(rhs.disp(), rhs.base(), lhs.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpcmpgtb_mr(rhs.address(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vpcmpeqw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vpcmpeqw_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpcmpeqw_mr(rhs.disp(), rhs.base(), lhs.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpcmpeqw_mr(rhs.address(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpcmpgtw(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vpcmpgtw_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpcmpgtw_mr(rhs.disp(), rhs.base(), lhs.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpcmpgtw_mr(rhs.address(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vpcmpeqd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vpcmpeqd_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpcmpeqd_mr(rhs.disp(), rhs.base(), lhs.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpcmpeqd_mr(rhs.address(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpcmpgtd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vpcmpgtd_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpcmpgtd_mr(rhs.disp(), rhs.base(), lhs.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpcmpgtd_mr(rhs.address(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpcmpgtq(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE42());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vpcmpgtq_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vpcmpeqq(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vpcmpeqq_rr(rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpcmpeqq_mr(rhs.disp(), rhs.base(), lhs.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpcmpeqq_mr(rhs.address(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vcmpps(uint8_t order, Operand rhs, FloatRegister lhs,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vcmpps_rr(order, rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vcmpps_mr(order, rhs.disp(), rhs.base(), lhs.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vcmpps_mr(order, rhs.address(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vcmpeqps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmpps(X86Encoding::ConditionCmp_EQ, rhs, lhs, dest);
+ }
+ void vcmpltps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmpps(X86Encoding::ConditionCmp_LT, rhs, lhs, dest);
+ }
+ void vcmpleps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmpps(X86Encoding::ConditionCmp_LE, rhs, lhs, dest);
+ }
+ void vcmpunordps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmpps(X86Encoding::ConditionCmp_UNORD, rhs, lhs, dest);
+ }
+ void vcmpneqps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmpps(X86Encoding::ConditionCmp_NEQ, rhs, lhs, dest);
+ }
+ void vcmpordps(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmpps(X86Encoding::ConditionCmp_ORD, rhs, lhs, dest);
+ }
+ void vcmppd(uint8_t order, Operand rhs, FloatRegister lhs,
+ FloatRegister dest) {
+ switch (rhs.kind()) {
+ case Operand::FPREG:
+ masm.vcmppd_rr(order, rhs.fpu(), lhs.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+ }
+ void vcmpeqpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmppd(X86Encoding::ConditionCmp_EQ, rhs, lhs, dest);
+ }
+ void vcmpltpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmppd(X86Encoding::ConditionCmp_LT, rhs, lhs, dest);
+ }
+ void vcmplepd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmppd(X86Encoding::ConditionCmp_LE, rhs, lhs, dest);
+ }
+ void vcmpneqpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmppd(X86Encoding::ConditionCmp_NEQ, rhs, lhs, dest);
+ }
+ void vcmpordpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmppd(X86Encoding::ConditionCmp_ORD, rhs, lhs, dest);
+ }
+ void vcmpunordpd(const Operand& rhs, FloatRegister lhs, FloatRegister dest) {
+ vcmppd(X86Encoding::ConditionCmp_UNORD, rhs, lhs, dest);
+ }
+ void vrcpps(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vrcpps_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vrcpps_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vrcpps_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vsqrtps(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vsqrtps_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vsqrtps_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vsqrtps_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vrsqrtps(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vrsqrtps_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vrsqrtps_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vrsqrtps_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vsqrtpd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vsqrtpd_rr(src.fpu(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovd(Register src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovd_rr(src.encoding(), dest.encoding());
+ }
+ void vmovd(FloatRegister src, Register dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovd_rr(src.encoding(), dest.encoding());
+ }
+ void vmovd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovd_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovd_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovd(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovd_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovq_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovq(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovq(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovq_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovq_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmaddubsw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSSE3());
+ masm.vpmaddubsw_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpaddb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpaddb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpaddb_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpaddb_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpsubb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpsubb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpsubb_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpsubb_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpaddsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpaddsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpaddsb_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpaddsb_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpaddusb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpaddusb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpaddusb_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpaddusb_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpsubsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpsubsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpsubsb_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpsubsb_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpsubusb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpsubusb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpsubusb_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpsubusb_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpaddw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpaddw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpaddw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpaddw_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpsubw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpsubw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpsubw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpsubw_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpaddsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpaddsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpaddsw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpaddsw_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpaddusw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpaddusw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpaddusw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpaddusw_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpsubsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpsubsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpsubsw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpsubsw_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpsubusw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpsubusw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpsubusw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpsubusw_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpaddd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpaddd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpaddd_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpsubd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpsubd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpsubd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpsubd_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmuldq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vpmuldq_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpmuludq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpmuludq_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpmuludq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmuludq_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmuludq_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmullw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmullw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmullw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmulhw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmulhw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmulhw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmulhuw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmulhuw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmulhuw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmulhrsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmulhrsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmulhrsw_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmulld(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmulld_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmulld_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpmulld_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmaddwd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmaddwd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpaddq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpaddq_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpsubq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpsubq_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vaddps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vaddps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vaddps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vaddps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vsubps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vsubps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vsubps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vsubps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmulps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vmulps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmulps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmulps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vdivps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vdivps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vdivps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vdivps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmaxps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vmaxps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmaxps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmaxps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vminps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vminps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vminps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vminps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vminpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vminpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmaxpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vmaxpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vaddpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vaddpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vsubpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vsubpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmulpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vmulpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vdivpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vdivpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpavgb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpavgb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpavgw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpavgw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpminsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpminsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpminub(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpminub_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmaxsb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmaxsb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmaxub(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmaxub_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpminsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpminsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpminuw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpminuw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmaxsw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmaxsw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmaxuw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmaxuw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpminsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpminud(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpminud_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmaxud(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpmaxud_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpacksswb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpacksswb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpackuswb(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpackuswb_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpackssdw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpackssdw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpackusdw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpackusdw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpabsb(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE3());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpabsb_rr(src.fpu(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpabsw(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE3());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpabsw_rr(src.fpu(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpabsd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE3());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpabsd_rr(src.fpu(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmovsxbw(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpmovsxbw_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmovsxbw_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpmovsxbw_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmovzxbw(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpmovzxbw_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmovzxbw_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpmovzxbw_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmovsxwd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpmovsxwd_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmovsxwd_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpmovsxwd_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmovzxwd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpmovzxwd_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmovzxwd_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpmovzxwd_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmovsxdq(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpmovsxdq_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmovsxdq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpmovsxdq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpmovzxdq(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vpmovzxdq_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpmovzxdq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vpmovzxdq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vphaddd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vphaddd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpalignr(const Operand& src1, FloatRegister src0, FloatRegister dest,
+ uint8_t shift) {
+ MOZ_ASSERT(HasSSE3());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpalignr_irr(shift, src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpunpcklbw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(src1.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ masm.vpunpcklbw_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpunpckhbw(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(src1.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ masm.vpunpckhbw_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpunpcklbw(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpunpcklbw_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpunpckldq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(src1.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ masm.vpunpckldq_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpunpckldq(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ switch (src1.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vpunpckldq_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpunpckldq_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpunpcklqdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(src1.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ masm.vpunpcklqdq_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpunpcklqdq(const Operand& src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ switch (src1.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vpunpcklqdq_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpunpcklqdq_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpunpckhdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(src1.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ masm.vpunpckhdq_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpunpckhqdq(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(src1.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ masm.vpunpckhqdq_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpunpcklwd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(src1.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ masm.vpunpcklwd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpunpckhwd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ MOZ_ASSERT(src0.size() == 16);
+ MOZ_ASSERT(src1.size() == 16);
+ MOZ_ASSERT(dest.size() == 16);
+ masm.vpunpckhwd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+
+ void vandps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vandps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vandps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vandps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vandnps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ // Negates bits of dest and then applies AND
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vandnps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vandnps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vandnps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vorps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vorps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vorps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vorps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vxorps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vxorps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vxorps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vxorps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vandpd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vandpd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vpand(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpand_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpand(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpand_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpand_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpand_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpor(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpor_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpor(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpor_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpor_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpor_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpxor(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpxor_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpxor(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpxor_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpxor_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpxor_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vpandn(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpandn_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vpandn(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpandn_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpandn_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpandn_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vpshufd(uint32_t mask, FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpshufd_irr(mask, src.encoding(), dest.encoding());
+ }
+ void vpshufd(uint32_t mask, const Operand& src1, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vpshufd_irr(mask, src1.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vpshufd_imr(mask, src1.disp(), src1.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vpshufd_imr(mask, src1.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vpshuflw(uint32_t mask, FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpshuflw_irr(mask, src.encoding(), dest.encoding());
+ }
+ void vpshufhw(uint32_t mask, FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vpshufhw_irr(mask, src.encoding(), dest.encoding());
+ }
+ void vpshufb(FloatRegister mask, FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSSE3());
+ masm.vpshufb_rr(mask.encoding(), src.encoding(), dest.encoding());
+ }
+ void vmovddup(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE3());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vmovddup_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmovddup_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovddup_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovhlps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovhlps_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vmovlhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovlhps_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vunpcklps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vunpcklps_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vunpcklps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vunpcklps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vunpcklps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vunpcklps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vunpckhps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vunpckhps_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vunpckhps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vunpckhps_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vunpckhps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vunpckhps_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vshufps(uint32_t mask, FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vshufps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vshufps(uint32_t mask, const Operand& src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vshufps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vshufps_imr(mask, src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vshufps_imr(mask, src1.address(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vshufpd(uint32_t mask, FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vshufpd_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vaddsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vaddsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vaddss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vaddss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vaddsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vaddsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vaddsd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vaddsd_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vaddss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vaddss_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vaddss_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vaddss_mr(src1.address(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vsubsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vsubsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vsubss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vsubss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vsubsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vsubsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vsubsd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vsubss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vsubss_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vsubss_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmulsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmulsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vmulsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vmulsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmulsd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmulss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vmulss_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmulss_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmulss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmulss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vdivsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vdivsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vdivss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vdivss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vdivsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vdivsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vdivsd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vdivss(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vdivss_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vdivss_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vxorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vxorpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vxorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vxorps_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vorpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vorpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vorps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vorps_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vandpd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vandpd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vandps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vandps_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vsqrtsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vsqrtsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vsqrtss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vsqrtss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vroundps(SSERoundingMode mode, const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vroundps_irr((X86Encoding::SSERoundingMode)mode, src.fpu(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vroundpd(SSERoundingMode mode, const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vroundpd_irr((X86Encoding::SSERoundingMode)mode, src.fpu(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ static X86Encoding::RoundingMode ToX86RoundingMode(RoundingMode mode) {
+ switch (mode) {
+ case RoundingMode::Up:
+ return X86Encoding::RoundUp;
+ case RoundingMode::Down:
+ return X86Encoding::RoundDown;
+ case RoundingMode::NearestTiesToEven:
+ return X86Encoding::RoundToNearest;
+ case RoundingMode::TowardsZero:
+ return X86Encoding::RoundToZero;
+ }
+ MOZ_CRASH("unexpected mode");
+ }
+ void vroundsd(X86Encoding::RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vroundsd_irr(mode, src.encoding(), dest.encoding());
+ }
+ void vroundss(X86Encoding::RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vroundss_irr(mode, src.encoding(), dest.encoding());
+ }
+
+ unsigned vinsertpsMask(unsigned sourceLane, unsigned destLane,
+ unsigned zeroMask = 0) {
+ // Note that the sourceLane bits are ignored in the case of a source
+ // memory operand, and the source is the given 32-bits memory location.
+ MOZ_ASSERT(zeroMask < 16);
+ unsigned ret = zeroMask;
+ ret |= destLane << 4;
+ ret |= sourceLane << 6;
+ MOZ_ASSERT(ret < 256);
+ return ret;
+ }
+ void vinsertps(uint32_t mask, FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vinsertps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vinsertps(uint32_t mask, const Operand& src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vinsertps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vinsertps_imr(mask, src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vinsertps_imr(mask, src1.disp(), src1.base(), src1.index(),
+ src1.scale(), src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovlps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ switch (src1.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovlps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovlps_mr(src1.disp(), src1.base(), src1.index(), src1.scale(),
+ src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovlps(FloatRegister src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovlps_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovlps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovhps(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ switch (src1.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovhps_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovhps_mr(src1.disp(), src1.base(), src1.index(), src1.scale(),
+ src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovhps(FloatRegister src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovhps_rm(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovhps_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vextractps(unsigned lane, FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE41());
+ MOZ_ASSERT(lane < 4);
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vextractps_rm(lane, src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vextractps_rm(lane, src.encoding(), dest.disp(), dest.base(),
+ dest.index(), dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ unsigned blendpsMask(bool x, bool y, bool z, bool w) {
+ return (x << 0) | (y << 1) | (z << 2) | (w << 3);
+ }
+ void vblendps(unsigned mask, FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vblendps_irr(mask, src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vblendps(unsigned mask, const Operand& src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vblendps_irr(mask, src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vblendps_imr(mask, src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vblendvps(FloatRegister mask, FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vblendvps_rr(mask.encoding(), src1.encoding(), src0.encoding(),
+ dest.encoding());
+ }
+ void vblendvps(FloatRegister mask, const Operand& src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vblendvps_rr(mask.encoding(), src1.fpu(), src0.encoding(),
+ dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vblendvps_mr(mask.encoding(), src1.disp(), src1.base(),
+ src0.encoding(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vblendvpd(FloatRegister mask, FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasSSE41());
+ masm.vblendvpd_rr(mask.encoding(), src1.encoding(), src0.encoding(),
+ dest.encoding());
+ }
+ void vmovsldup(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE3());
+ masm.vmovsldup_rr(src.encoding(), dest.encoding());
+ }
+ void vmovsldup(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE3());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vmovsldup_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmovsldup_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmovshdup(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE3());
+ masm.vmovshdup_rr(src.encoding(), dest.encoding());
+ }
+ void vmovshdup(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE3());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vmovshdup_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmovshdup_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vminsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vminsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vminsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vminsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vminsd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vminss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vminss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vmaxsd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmaxsd_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vmaxsd(const Operand& src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src1.kind()) {
+ case Operand::FPREG:
+ masm.vmaxsd_rr(src1.fpu(), src0.encoding(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vmaxsd_mr(src1.disp(), src1.base(), src0.encoding(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vmaxss(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmaxss_rr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void fisttp(const Operand& dest) {
+ MOZ_ASSERT(HasSSE3());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fisttp_m(dest.disp(), dest.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void fistp(const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fistp_m(dest.disp(), dest.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void fnstcw(const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fnstcw_m(dest.disp(), dest.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void fldcw(const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fldcw_m(dest.disp(), dest.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void fnstsw(const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fnstsw_m(dest.disp(), dest.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void fld(const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fld_m(dest.disp(), dest.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void fld32(const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fld32_m(dest.disp(), dest.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void fstp(const Operand& src) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fstp_m(src.disp(), src.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void fstp32(const Operand& src) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fstp32_m(src.disp(), src.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void vbroadcastb(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasAVX2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vbroadcastb_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vbroadcastb_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vbroadcastb_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vbroadcastw(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasAVX2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vbroadcastw_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vbroadcastw_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vbroadcastw_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vbroadcastd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasAVX2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vbroadcastd_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vbroadcastd_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vbroadcastd_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vbroadcastq(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasAVX2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vbroadcastq_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vbroadcastq_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vbroadcastq_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vbroadcastss(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasAVX2());
+ switch (src.kind()) {
+ case Operand::FPREG:
+ masm.vbroadcastss_rr(src.fpu(), dest.encoding());
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.vbroadcastss_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vbroadcastss_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void vfmadd231ps(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasFMA());
+ masm.vfmadd231ps_rrr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vfnmadd231ps(FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasFMA());
+ masm.vfnmadd231ps_rrr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vfmadd231pd(FloatRegister src1, FloatRegister src0, FloatRegister dest) {
+ MOZ_ASSERT(HasFMA());
+ masm.vfmadd231pd_rrr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+ void vfnmadd231pd(FloatRegister src1, FloatRegister src0,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasFMA());
+ masm.vfnmadd231pd_rrr(src1.encoding(), src0.encoding(), dest.encoding());
+ }
+
+ void flushBuffer() {}
+
+ // Patching.
+
+ static size_t PatchWrite_NearCallSize() { return 5; }
+ static uintptr_t GetPointer(uint8_t* instPtr) {
+ uint8_t* ptr = instPtr - sizeof(uintptr_t);
+ return mozilla::LittleEndian::readUintptr(ptr);
+ }
+ // Write a relative call at the start location |dataLabel|.
+ // Note that this DOES NOT patch data that comes before |label|.
+ static void PatchWrite_NearCall(CodeLocationLabel startLabel,
+ CodeLocationLabel target) {
+ uint8_t* start = startLabel.raw();
+ *start = 0xE8; // <CALL> rel32
+ ptrdiff_t offset = target - startLabel - PatchWrite_NearCallSize();
+ MOZ_ASSERT(int32_t(offset) == offset);
+ mozilla::LittleEndian::writeInt32(start + 1, offset); // CALL <rel32>
+ }
+
+ static void PatchWrite_Imm32(CodeLocationLabel dataLabel, Imm32 toWrite) {
+ // dataLabel is a code location which targets the end of an instruction
+ // which has a 32 bits immediate. Thus writting a value requires shifting
+ // back to the address of the 32 bits immediate within the instruction.
+ uint8_t* ptr = dataLabel.raw();
+ mozilla::LittleEndian::writeInt32(ptr - sizeof(int32_t), toWrite.value);
+ }
+
+ static void PatchDataWithValueCheck(CodeLocationLabel data,
+ PatchedImmPtr newData,
+ PatchedImmPtr expectedData) {
+ // The pointer given is a pointer to *after* the data.
+ uint8_t* ptr = data.raw() - sizeof(uintptr_t);
+ MOZ_ASSERT(mozilla::LittleEndian::readUintptr(ptr) ==
+ uintptr_t(expectedData.value));
+ mozilla::LittleEndian::writeUintptr(ptr, uintptr_t(newData.value));
+ }
+ static void PatchDataWithValueCheck(CodeLocationLabel data, ImmPtr newData,
+ ImmPtr expectedData) {
+ PatchDataWithValueCheck(data, PatchedImmPtr(newData.value),
+ PatchedImmPtr(expectedData.value));
+ }
+
+ static uint32_t NopSize() { return 1; }
+ static uint8_t* NextInstruction(uint8_t* cur, uint32_t* count) {
+ MOZ_CRASH("nextInstruction NYI on x86");
+ }
+
+ // Toggle a jmp or cmp emitted by toggledJump().
+ static void ToggleToJmp(CodeLocationLabel inst) {
+ uint8_t* ptr = (uint8_t*)inst.raw();
+ MOZ_ASSERT(*ptr == 0x3D); // <CMP> eax, imm32
+ *ptr = 0xE9; // <JMP> rel32
+ }
+ static void ToggleToCmp(CodeLocationLabel inst) {
+ uint8_t* ptr = (uint8_t*)inst.raw();
+ MOZ_ASSERT(*ptr == 0xE9); // <JMP> rel32
+ *ptr = 0x3D; // <CMP> eax, imm32
+ }
+ static void ToggleCall(CodeLocationLabel inst, bool enabled) {
+ uint8_t* ptr = (uint8_t*)inst.raw();
+ MOZ_ASSERT(*ptr == 0x3D || // <CMP> eax, imm32
+ *ptr == 0xE8); // <CALL> rel32
+ *ptr = enabled ? 0xE8 : 0x3D;
+ }
+
+ MOZ_COLD void verifyHeapAccessDisassembly(
+ uint32_t begin, uint32_t end, const Disassembler::HeapAccess& heapAccess);
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_Assembler_x86_shared_h */
diff --git a/js/src/jit/x86-shared/AssemblerBuffer-x86-shared.cpp b/js/src/jit/x86-shared/AssemblerBuffer-x86-shared.cpp
new file mode 100644
index 0000000000..835d7755c9
--- /dev/null
+++ b/js/src/jit/x86-shared/AssemblerBuffer-x86-shared.cpp
@@ -0,0 +1,57 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86-shared/AssemblerBuffer-x86-shared.h"
+
+#include "mozilla/Sprintf.h"
+
+using namespace js;
+using namespace jit;
+
+bool AssemblerBuffer::swap(Vector<uint8_t, 0, SystemAllocPolicy>& bytes) {
+ // For now, specialize to the one use case.
+ MOZ_ASSERT(bytes.empty());
+
+ if (m_buffer.empty()) {
+ if (bytes.capacity() > m_buffer.capacity()) {
+ size_t newCapacity = bytes.capacity();
+ uint8_t* newBuffer = bytes.extractRawBuffer();
+ MOZ_ASSERT(newBuffer);
+ m_buffer.replaceRawBuffer((unsigned char*)newBuffer, 0, newCapacity);
+ }
+ return true;
+ }
+
+ size_t newLength = m_buffer.length();
+ size_t newCapacity = m_buffer.capacity();
+ unsigned char* newBuffer = m_buffer.extractRawBuffer();
+
+ // NB: extractRawBuffer() only returns null if the Vector is using
+ // inline storage and thus a malloc would be needed. In this case,
+ // just make a simple copy.
+ if (!newBuffer) {
+ return bytes.append(m_buffer.begin(), m_buffer.end());
+ }
+
+ bytes.replaceRawBuffer((uint8_t*)newBuffer, newLength, newCapacity);
+ return true;
+}
+
+#ifdef JS_JITSPEW
+void js::jit::GenericAssembler::spew(const char* fmt, va_list va) {
+ // Buffer to hold the formatted string. Note that this may contain
+ // '%' characters, so do not pass it directly to printf functions.
+ char buf[200];
+
+ int i = VsprintfLiteral(buf, fmt, va);
+ if (i > -1) {
+ if (printer) {
+ printer->printf("%s\n", buf);
+ }
+ js::jit::JitSpew(js::jit::JitSpew_Codegen, "%s", buf);
+ }
+}
+#endif
diff --git a/js/src/jit/x86-shared/AssemblerBuffer-x86-shared.h b/js/src/jit/x86-shared/AssemblerBuffer-x86-shared.h
new file mode 100644
index 0000000000..a10cffc08e
--- /dev/null
+++ b/js/src/jit/x86-shared/AssemblerBuffer-x86-shared.h
@@ -0,0 +1,256 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ *
+ * ***** BEGIN LICENSE BLOCK *****
+ * Copyright (C) 2008 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef jit_x86_shared_AssemblerBuffer_x86_shared_h
+#define jit_x86_shared_AssemblerBuffer_x86_shared_h
+
+#include "mozilla/Assertions.h"
+#include "mozilla/Attributes.h"
+#include "mozilla/Likely.h"
+#include "mozilla/Vector.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/JitContext.h"
+#include "jit/JitSpewer.h"
+#include "jit/ProcessExecutableMemory.h"
+#include "js/AllocPolicy.h"
+#include "js/Vector.h"
+
+// Spew formatting helpers.
+#define PRETTYHEX(x) \
+ (((x) < 0) ? "-" : ""), \
+ ((unsigned)((x) ^ ((x) >> 31)) + ((unsigned)(x) >> 31))
+
+#define MEM_o "%s0x%x"
+#define MEM_os MEM_o "(,%s,%d)"
+#define MEM_ob MEM_o "(%s)"
+#define MEM_obs MEM_o "(%s,%s,%d)"
+
+#define MEM_o32 "%s0x%04x"
+#define MEM_o32s MEM_o32 "(,%s,%d)"
+#define MEM_o32b MEM_o32 "(%s)"
+#define MEM_o32bs MEM_o32 "(%s,%s,%d)"
+#define MEM_o32r ".Lfrom%d(%%rip)"
+
+#define ADDR_o(offset) PRETTYHEX(offset)
+#define ADDR_os(offset, index, scale) \
+ ADDR_o(offset), GPRegName((index)), (1 << (scale))
+#define ADDR_ob(offset, base) ADDR_o(offset), GPRegName((base))
+#define ADDR_obs(offset, base, index, scale) \
+ ADDR_ob(offset, base), GPRegName((index)), (1 << (scale))
+
+#define ADDR_o32(offset) ADDR_o(offset)
+#define ADDR_o32s(offset, index, scale) ADDR_os(offset, index, scale)
+#define ADDR_o32b(offset, base) ADDR_ob(offset, base)
+#define ADDR_o32bs(offset, base, index, scale) \
+ ADDR_obs(offset, base, index, scale)
+#define ADDR_o32r(offset) (offset)
+
+namespace js {
+
+class Sprinter;
+
+namespace jit {
+
+// AllocPolicy for AssemblerBuffer. OOMs when trying to allocate more than
+// MaxCodeBytesPerProcess bytes. Use private inheritance to make sure we
+// explicitly have to expose SystemAllocPolicy methods.
+class AssemblerBufferAllocPolicy : private SystemAllocPolicy {
+ public:
+ using SystemAllocPolicy::checkSimulatedOOM;
+ using SystemAllocPolicy::free_;
+ using SystemAllocPolicy::reportAllocOverflow;
+
+ template <typename T>
+ T* pod_realloc(T* p, size_t oldSize, size_t newSize) {
+ static_assert(
+ sizeof(T) == 1,
+ "AssemblerBufferAllocPolicy should only be used with byte vectors");
+ MOZ_ASSERT(oldSize <= MaxCodeBytesPerProcess);
+ if (MOZ_UNLIKELY(newSize > MaxCodeBytesPerProcess)) {
+ return nullptr;
+ }
+ return SystemAllocPolicy::pod_realloc<T>(p, oldSize, newSize);
+ }
+ template <typename T>
+ T* pod_malloc(size_t numElems) {
+ static_assert(
+ sizeof(T) == 1,
+ "AssemblerBufferAllocPolicy should only be used with byte vectors");
+ if (MOZ_UNLIKELY(numElems > MaxCodeBytesPerProcess)) {
+ return nullptr;
+ }
+ return SystemAllocPolicy::pod_malloc<T>(numElems);
+ }
+};
+
+class AssemblerBuffer {
+ template <size_t size, typename T>
+ MOZ_ALWAYS_INLINE void sizedAppendUnchecked(T value) {
+ m_buffer.infallibleAppend(reinterpret_cast<unsigned char*>(&value), size);
+ }
+
+ template <size_t size, typename T>
+ MOZ_ALWAYS_INLINE void sizedAppend(T value) {
+ if (MOZ_UNLIKELY(
+ !m_buffer.append(reinterpret_cast<unsigned char*>(&value), size))) {
+ oomDetected();
+ }
+ }
+
+ public:
+ AssemblerBuffer() : m_oom(false) {}
+
+ void ensureSpace(size_t space) {
+ // This should only be called with small |space| values to ensure
+ // we don't overflow below.
+ MOZ_ASSERT(space <= 16);
+ if (MOZ_UNLIKELY(!m_buffer.reserve(m_buffer.length() + space))) {
+ oomDetected();
+ }
+ }
+
+ bool isAligned(size_t alignment) const {
+ return !(m_buffer.length() & (alignment - 1));
+ }
+
+ MOZ_ALWAYS_INLINE void putByteUnchecked(int value) {
+ sizedAppendUnchecked<1>(value);
+ }
+ MOZ_ALWAYS_INLINE void putShortUnchecked(int value) {
+ sizedAppendUnchecked<2>(value);
+ }
+ MOZ_ALWAYS_INLINE void putIntUnchecked(int value) {
+ sizedAppendUnchecked<4>(value);
+ }
+ MOZ_ALWAYS_INLINE void putInt64Unchecked(int64_t value) {
+ sizedAppendUnchecked<8>(value);
+ }
+
+ MOZ_ALWAYS_INLINE void putByte(int value) { sizedAppend<1>(value); }
+ MOZ_ALWAYS_INLINE void putShort(int value) { sizedAppend<2>(value); }
+ MOZ_ALWAYS_INLINE void putInt(int value) { sizedAppend<4>(value); }
+ MOZ_ALWAYS_INLINE void putInt64(int64_t value) { sizedAppend<8>(value); }
+
+ [[nodiscard]] bool append(const unsigned char* values, size_t size) {
+ if (MOZ_UNLIKELY(!m_buffer.append(values, size))) {
+ oomDetected();
+ return false;
+ }
+ return true;
+ }
+
+ size_t size() const { return m_buffer.length(); }
+
+ bool oom() const { return m_oom; }
+
+ bool reserve(size_t size) { return !m_oom && m_buffer.reserve(size); }
+
+ bool swap(Vector<uint8_t, 0, SystemAllocPolicy>& bytes);
+
+ const unsigned char* buffer() const {
+ MOZ_RELEASE_ASSERT(!m_oom);
+ return m_buffer.begin();
+ }
+
+ unsigned char* data() { return m_buffer.begin(); }
+
+ protected:
+ /*
+ * OOM handling: This class can OOM in the ensureSpace() method trying
+ * to allocate a new buffer. In response to an OOM, we need to avoid
+ * crashing and report the error. We also want to make it so that
+ * users of this class need to check for OOM only at certain points
+ * and not after every operation.
+ *
+ * Our strategy for handling an OOM is to set m_oom, and then clear (but
+ * not free) m_buffer, preserving the current buffer. This way, the user
+ * can continue assembling into the buffer, deferring OOM checking
+ * until the user wants to read code out of the buffer.
+ *
+ * See also the |buffer| method.
+ */
+ void oomDetected() {
+ m_oom = true;
+ m_buffer.clear();
+#ifdef DEBUG
+ JitContext* context = MaybeGetJitContext();
+ if (context) {
+ context->setOOM();
+ }
+#endif
+ }
+
+ mozilla::Vector<unsigned char, 256, AssemblerBufferAllocPolicy> m_buffer;
+ bool m_oom;
+};
+
+class GenericAssembler {
+#ifdef JS_JITSPEW
+ Sprinter* printer;
+#endif
+ public:
+ GenericAssembler()
+#ifdef JS_JITSPEW
+ : printer(nullptr)
+#endif
+ {
+ }
+
+ void setPrinter(Sprinter* sp) {
+#ifdef JS_JITSPEW
+ printer = sp;
+#endif
+ }
+
+#ifdef JS_JITSPEW
+ inline void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) {
+ if (MOZ_UNLIKELY(printer || JitSpewEnabled(JitSpew_Codegen))) {
+ va_list va;
+ va_start(va, fmt);
+ spew(fmt, va);
+ va_end(va);
+ }
+ }
+#else
+ MOZ_ALWAYS_INLINE void spew(const char* fmt, ...) MOZ_FORMAT_PRINTF(2, 3) {}
+#endif
+
+#ifdef JS_JITSPEW
+ MOZ_COLD void spew(const char* fmt, va_list va) MOZ_FORMAT_PRINTF(2, 0);
+#endif
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_AssemblerBuffer_x86_shared_h */
diff --git a/js/src/jit/x86-shared/BaseAssembler-x86-shared.h b/js/src/jit/x86-shared/BaseAssembler-x86-shared.h
new file mode 100644
index 0000000000..b7a5031757
--- /dev/null
+++ b/js/src/jit/x86-shared/BaseAssembler-x86-shared.h
@@ -0,0 +1,6460 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ *
+ * ***** BEGIN LICENSE BLOCK *****
+ * Copyright (C) 2008 Apple Inc. All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+ * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+#ifndef jit_x86_shared_BaseAssembler_x86_shared_h
+#define jit_x86_shared_BaseAssembler_x86_shared_h
+
+#include "mozilla/IntegerPrintfMacros.h"
+
+#include "jit/x86-shared/AssemblerBuffer-x86-shared.h"
+#include "jit/x86-shared/Encoding-x86-shared.h"
+#include "jit/x86-shared/Patching-x86-shared.h"
+#include "wasm/WasmTypeDecls.h"
+
+namespace js {
+namespace jit {
+
+namespace X86Encoding {
+
+class BaseAssembler;
+
+class BaseAssembler : public GenericAssembler {
+ public:
+ BaseAssembler() : useVEX_(true) {}
+
+ void disableVEX() { useVEX_ = false; }
+
+ size_t size() const { return m_formatter.size(); }
+ const unsigned char* buffer() const { return m_formatter.buffer(); }
+ unsigned char* data() { return m_formatter.data(); }
+ bool oom() const { return m_formatter.oom(); }
+ bool reserve(size_t size) { return m_formatter.reserve(size); }
+ bool swapBuffer(wasm::Bytes& other) { return m_formatter.swapBuffer(other); }
+
+ void nop() {
+ spew("nop");
+ m_formatter.oneByteOp(OP_NOP);
+ }
+
+ void comment(const char* msg) { spew("; %s", msg); }
+
+ static void patchFiveByteNopToCall(uint8_t* callsite, uint8_t* target) {
+ // Note: the offset is relative to the address of the instruction after
+ // the call which is five bytes.
+ uint8_t* inst = callsite - sizeof(int32_t) - 1;
+ // The nop can be already patched as call, overriding the call.
+ // See also nop_five.
+ MOZ_ASSERT(inst[0] == OP_NOP_0F || inst[0] == OP_CALL_rel32);
+ MOZ_ASSERT_IF(inst[0] == OP_NOP_0F,
+ inst[1] == OP_NOP_1F || inst[2] == OP_NOP_44 ||
+ inst[3] == OP_NOP_00 || inst[4] == OP_NOP_00);
+ inst[0] = OP_CALL_rel32;
+ SetRel32(callsite, target);
+ }
+
+ static void patchCallToFiveByteNop(uint8_t* callsite) {
+ // See also patchFiveByteNopToCall and nop_five.
+ uint8_t* inst = callsite - sizeof(int32_t) - 1;
+ // The call can be already patched as nop.
+ if (inst[0] == OP_NOP_0F) {
+ MOZ_ASSERT(inst[1] == OP_NOP_1F || inst[2] == OP_NOP_44 ||
+ inst[3] == OP_NOP_00 || inst[4] == OP_NOP_00);
+ return;
+ }
+ MOZ_ASSERT(inst[0] == OP_CALL_rel32);
+ inst[0] = OP_NOP_0F;
+ inst[1] = OP_NOP_1F;
+ inst[2] = OP_NOP_44;
+ inst[3] = OP_NOP_00;
+ inst[4] = OP_NOP_00;
+ }
+
+ /*
+ * The nop multibytes sequences are directly taken from the Intel's
+ * architecture software developer manual.
+ * They are defined for sequences of sizes from 1 to 9 included.
+ */
+ void nop_one() { m_formatter.oneByteOp(OP_NOP); }
+
+ void nop_two() {
+ m_formatter.oneByteOp(OP_NOP_66);
+ m_formatter.oneByteOp(OP_NOP);
+ }
+
+ void nop_three() {
+ m_formatter.oneByteOp(OP_NOP_0F);
+ m_formatter.oneByteOp(OP_NOP_1F);
+ m_formatter.oneByteOp(OP_NOP_00);
+ }
+
+ void nop_four() {
+ m_formatter.oneByteOp(OP_NOP_0F);
+ m_formatter.oneByteOp(OP_NOP_1F);
+ m_formatter.oneByteOp(OP_NOP_40);
+ m_formatter.oneByteOp(OP_NOP_00);
+ }
+
+ void nop_five() {
+ m_formatter.oneByteOp(OP_NOP_0F);
+ m_formatter.oneByteOp(OP_NOP_1F);
+ m_formatter.oneByteOp(OP_NOP_44);
+ m_formatter.oneByteOp(OP_NOP_00);
+ m_formatter.oneByteOp(OP_NOP_00);
+ }
+
+ void nop_six() {
+ m_formatter.oneByteOp(OP_NOP_66);
+ nop_five();
+ }
+
+ void nop_seven() {
+ m_formatter.oneByteOp(OP_NOP_0F);
+ m_formatter.oneByteOp(OP_NOP_1F);
+ m_formatter.oneByteOp(OP_NOP_80);
+ for (int i = 0; i < 4; ++i) {
+ m_formatter.oneByteOp(OP_NOP_00);
+ }
+ }
+
+ void nop_eight() {
+ m_formatter.oneByteOp(OP_NOP_0F);
+ m_formatter.oneByteOp(OP_NOP_1F);
+ m_formatter.oneByteOp(OP_NOP_84);
+ for (int i = 0; i < 5; ++i) {
+ m_formatter.oneByteOp(OP_NOP_00);
+ }
+ }
+
+ void nop_nine() {
+ m_formatter.oneByteOp(OP_NOP_66);
+ nop_eight();
+ }
+
+ void insert_nop(int size) {
+ switch (size) {
+ case 1:
+ nop_one();
+ break;
+ case 2:
+ nop_two();
+ break;
+ case 3:
+ nop_three();
+ break;
+ case 4:
+ nop_four();
+ break;
+ case 5:
+ nop_five();
+ break;
+ case 6:
+ nop_six();
+ break;
+ case 7:
+ nop_seven();
+ break;
+ case 8:
+ nop_eight();
+ break;
+ case 9:
+ nop_nine();
+ break;
+ case 10:
+ nop_three();
+ nop_seven();
+ break;
+ case 11:
+ nop_four();
+ nop_seven();
+ break;
+ case 12:
+ nop_six();
+ nop_six();
+ break;
+ case 13:
+ nop_six();
+ nop_seven();
+ break;
+ case 14:
+ nop_seven();
+ nop_seven();
+ break;
+ case 15:
+ nop_one();
+ nop_seven();
+ nop_seven();
+ break;
+ default:
+ MOZ_CRASH("Unhandled alignment");
+ }
+ }
+
+ // Stack operations:
+
+ void push_r(RegisterID reg) {
+ spew("push %s", GPRegName(reg));
+ m_formatter.oneByteOp(OP_PUSH_EAX, reg);
+ }
+
+ void pop_r(RegisterID reg) {
+ spew("pop %s", GPRegName(reg));
+ m_formatter.oneByteOp(OP_POP_EAX, reg);
+ }
+
+ void push_i(int32_t imm) {
+ spew("push $%s0x%x", PRETTYHEX(imm));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_PUSH_Ib);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_PUSH_Iz);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void push_i32(int32_t imm) {
+ spew("push $%s0x%04x", PRETTYHEX(imm));
+ m_formatter.oneByteOp(OP_PUSH_Iz);
+ m_formatter.immediate32(imm);
+ }
+
+ void push_m(int32_t offset, RegisterID base) {
+ spew("push " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP5_Ev, offset, base, GROUP5_OP_PUSH);
+ }
+ void push_m(int32_t offset, RegisterID base, RegisterID index, int scale) {
+ spew("push " MEM_obs, ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP5_Ev, offset, base, index, scale,
+ GROUP5_OP_PUSH);
+ }
+
+ void pop_m(int32_t offset, RegisterID base) {
+ spew("pop " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP1A_Ev, offset, base, GROUP1A_OP_POP);
+ }
+
+ void push_flags() {
+ spew("pushf");
+ m_formatter.oneByteOp(OP_PUSHFLAGS);
+ }
+
+ void pop_flags() {
+ spew("popf");
+ m_formatter.oneByteOp(OP_POPFLAGS);
+ }
+
+ // Arithmetic operations:
+
+ void addl_rr(RegisterID src, RegisterID dst) {
+ spew("addl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_ADD_GvEv, src, dst);
+ }
+
+ void addw_rr(RegisterID src, RegisterID dst) {
+ spew("addw %s, %s", GPReg16Name(src), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_ADD_GvEv, src, dst);
+ }
+
+ void addl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("addl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_ADD_GvEv, offset, base, dst);
+ }
+
+ void addl_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("addl %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_ADD_EvGv, offset, base, src);
+ }
+
+ void addl_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("addl %s, " MEM_obs, GPReg32Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_ADD_EvGv, offset, base, index, scale, src);
+ }
+
+ void addl_ir(int32_t imm, RegisterID dst) {
+ spew("addl $%d, %s", imm, GPReg32Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_ADD_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void addw_ir(int32_t imm, RegisterID dst) {
+ spew("addw $%d, %s", int16_t(imm), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+ m_formatter.immediate16(imm);
+ }
+
+ void addl_i32r(int32_t imm, RegisterID dst) {
+ // 32-bit immediate always, for patching.
+ spew("addl $0x%04x, %s", uint32_t(imm), GPReg32Name(dst));
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_ADD_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_ADD);
+ }
+ m_formatter.immediate32(imm);
+ }
+
+ void addl_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("addl $%d, " MEM_ob, imm, ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_ADD);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void addl_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("addl $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_ADD);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void addl_im(int32_t imm, const void* addr) {
+ spew("addl $%d, %p", imm, addr);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_ADD);
+ m_formatter.immediate32(imm);
+ }
+ }
+ void addw_im(int32_t imm, const void* addr) {
+ spew("addw $%d, %p", int16_t(imm), addr);
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_ADD);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_ADD);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void addw_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("addw $%d, " MEM_ob, int16_t(imm), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_ADD);
+ m_formatter.immediate16(imm);
+ }
+
+ void addw_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("addw $%d, " MEM_obs, int16_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_ADD);
+ m_formatter.immediate16(imm);
+ }
+
+ void addw_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("addw %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_ADD_EvGv, offset, base, src);
+ }
+
+ void addw_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("addw %s, " MEM_obs, GPReg16Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_ADD_EvGv, offset, base, index, scale, src);
+ }
+
+ void addb_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("addb $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_ADD);
+ m_formatter.immediate8(imm);
+ }
+
+ void addb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("addb $%d, " MEM_obs, int8_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale,
+ GROUP1_OP_ADD);
+ m_formatter.immediate8(imm);
+ }
+
+ void addb_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("addb %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp8(OP_ADD_EbGb, offset, base, src);
+ }
+
+ void addb_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("addb %s, " MEM_obs, GPReg8Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp8(OP_ADD_EbGb, offset, base, index, scale, src);
+ }
+
+ void subb_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("subb $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_SUB);
+ m_formatter.immediate8(imm);
+ }
+
+ void subb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("subb $%d, " MEM_obs, int8_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale,
+ GROUP1_OP_SUB);
+ m_formatter.immediate8(imm);
+ }
+
+ void subb_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("subb %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp8(OP_SUB_EbGb, offset, base, src);
+ }
+
+ void subb_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("subb %s, " MEM_obs, GPReg8Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp8(OP_SUB_EbGb, offset, base, index, scale, src);
+ }
+
+ void andb_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("andb $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_AND);
+ m_formatter.immediate8(imm);
+ }
+
+ void andb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("andb $%d, " MEM_obs, int8_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale,
+ GROUP1_OP_AND);
+ m_formatter.immediate8(imm);
+ }
+
+ void andb_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("andb %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp8(OP_AND_EbGb, offset, base, src);
+ }
+
+ void andb_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("andb %s, " MEM_obs, GPReg8Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp8(OP_AND_EbGb, offset, base, index, scale, src);
+ }
+
+ void orb_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("orb $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_OR);
+ m_formatter.immediate8(imm);
+ }
+
+ void orb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("orb $%d, " MEM_obs, int8_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale,
+ GROUP1_OP_OR);
+ m_formatter.immediate8(imm);
+ }
+
+ void orb_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("orb %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp8(OP_OR_EbGb, offset, base, src);
+ }
+
+ void orb_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("orb %s, " MEM_obs, GPReg8Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp8(OP_OR_EbGb, offset, base, index, scale, src);
+ }
+
+ void xorb_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("xorb $%d, " MEM_ob, int8_t(imm), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_XOR);
+ m_formatter.immediate8(imm);
+ }
+
+ void xorb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("xorb $%d, " MEM_obs, int8_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale,
+ GROUP1_OP_XOR);
+ m_formatter.immediate8(imm);
+ }
+
+ void xorb_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xorb %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp8(OP_XOR_EbGb, offset, base, src);
+ }
+
+ void xorb_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xorb %s, " MEM_obs, GPReg8Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp8(OP_XOR_EbGb, offset, base, index, scale, src);
+ }
+
+ void lock_xaddb_rm(RegisterID srcdest, int32_t offset, RegisterID base) {
+ spew("lock xaddb %s, " MEM_ob, GPReg8Name(srcdest), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(PRE_LOCK);
+ m_formatter.twoByteOp8(OP2_XADD_EbGb, offset, base, srcdest);
+ }
+
+ void lock_xaddb_rm(RegisterID srcdest, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("lock xaddb %s, " MEM_obs, GPReg8Name(srcdest),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(PRE_LOCK);
+ m_formatter.twoByteOp8(OP2_XADD_EbGb, offset, base, index, scale, srcdest);
+ }
+
+ void lock_xaddl_rm(RegisterID srcdest, int32_t offset, RegisterID base) {
+ spew("lock xaddl %s, " MEM_ob, GPReg32Name(srcdest), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(PRE_LOCK);
+ m_formatter.twoByteOp(OP2_XADD_EvGv, offset, base, srcdest);
+ }
+
+ void lock_xaddl_rm(RegisterID srcdest, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("lock xaddl %s, " MEM_obs, GPReg32Name(srcdest),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(PRE_LOCK);
+ m_formatter.twoByteOp(OP2_XADD_EvGv, offset, base, index, scale, srcdest);
+ }
+
+ void vpmaddubsw_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ threeByteOpSimd("vpmaddubsw", VEX_PD, OP3_PMADDUBSW_VdqWdq, ESCAPE_38, src1,
+ src0, dst);
+ }
+ void vpmaddubsw_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ threeByteOpSimd("vpmaddubsw", VEX_PD, OP3_PMADDUBSW_VdqWdq, ESCAPE_38,
+ address, src0, dst);
+ }
+
+ void vpaddb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddb", VEX_PD, OP2_PADDB_VdqWdq, src1, src0, dst);
+ }
+ void vpaddb_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpaddb", VEX_PD, OP2_PADDB_VdqWdq, offset, base, src0, dst);
+ }
+ void vpaddb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddb", VEX_PD, OP2_PADDB_VdqWdq, address, src0, dst);
+ }
+
+ void vpaddsb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddsb", VEX_PD, OP2_PADDSB_VdqWdq, src1, src0, dst);
+ }
+ void vpaddsb_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpaddsb", VEX_PD, OP2_PADDSB_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpaddsb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddsb", VEX_PD, OP2_PADDSB_VdqWdq, address, src0, dst);
+ }
+
+ void vpaddusb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddusb", VEX_PD, OP2_PADDUSB_VdqWdq, src1, src0, dst);
+ }
+ void vpaddusb_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpaddusb", VEX_PD, OP2_PADDUSB_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpaddusb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddusb", VEX_PD, OP2_PADDUSB_VdqWdq, address, src0, dst);
+ }
+
+ void vpaddw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddw", VEX_PD, OP2_PADDW_VdqWdq, src1, src0, dst);
+ }
+ void vpaddw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpaddw", VEX_PD, OP2_PADDW_VdqWdq, offset, base, src0, dst);
+ }
+ void vpaddw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddw", VEX_PD, OP2_PADDW_VdqWdq, address, src0, dst);
+ }
+
+ void vpaddsw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddsw", VEX_PD, OP2_PADDSW_VdqWdq, src1, src0, dst);
+ }
+ void vpaddsw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpaddsw", VEX_PD, OP2_PADDSW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpaddsw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddsw", VEX_PD, OP2_PADDSW_VdqWdq, address, src0, dst);
+ }
+
+ void vpaddusw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddusw", VEX_PD, OP2_PADDUSW_VdqWdq, src1, src0, dst);
+ }
+ void vpaddusw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpaddusw", VEX_PD, OP2_PADDUSW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpaddusw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddusw", VEX_PD, OP2_PADDUSW_VdqWdq, address, src0, dst);
+ }
+
+ void vpaddd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddd", VEX_PD, OP2_PADDD_VdqWdq, src1, src0, dst);
+ }
+ void vpaddd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpaddd", VEX_PD, OP2_PADDD_VdqWdq, offset, base, src0, dst);
+ }
+ void vpaddd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddd", VEX_PD, OP2_PADDD_VdqWdq, address, src0, dst);
+ }
+
+ void vpaddq_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddq", VEX_PD, OP2_PADDQ_VdqWdq, address, src0, dst);
+ }
+
+ void vpsubb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubb", VEX_PD, OP2_PSUBB_VdqWdq, src1, src0, dst);
+ }
+ void vpsubb_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpsubb", VEX_PD, OP2_PSUBB_VdqWdq, offset, base, src0, dst);
+ }
+ void vpsubb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubb", VEX_PD, OP2_PSUBB_VdqWdq, address, src0, dst);
+ }
+
+ void vpsubsb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubsb", VEX_PD, OP2_PSUBSB_VdqWdq, src1, src0, dst);
+ }
+ void vpsubsb_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpsubsb", VEX_PD, OP2_PSUBSB_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpsubsb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubsb", VEX_PD, OP2_PSUBSB_VdqWdq, address, src0, dst);
+ }
+
+ void vpsubusb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubusb", VEX_PD, OP2_PSUBUSB_VdqWdq, src1, src0, dst);
+ }
+ void vpsubusb_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpsubusb", VEX_PD, OP2_PSUBUSB_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpsubusb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubusb", VEX_PD, OP2_PSUBUSB_VdqWdq, address, src0, dst);
+ }
+
+ void vpsubw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubw", VEX_PD, OP2_PSUBW_VdqWdq, src1, src0, dst);
+ }
+ void vpsubw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpsubw", VEX_PD, OP2_PSUBW_VdqWdq, offset, base, src0, dst);
+ }
+ void vpsubw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubw", VEX_PD, OP2_PSUBW_VdqWdq, address, src0, dst);
+ }
+
+ void vpsubsw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubsw", VEX_PD, OP2_PSUBSW_VdqWdq, src1, src0, dst);
+ }
+ void vpsubsw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpsubsw", VEX_PD, OP2_PSUBSW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpsubsw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubsw", VEX_PD, OP2_PSUBSW_VdqWdq, address, src0, dst);
+ }
+
+ void vpsubusw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubusw", VEX_PD, OP2_PSUBUSW_VdqWdq, src1, src0, dst);
+ }
+ void vpsubusw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpsubusw", VEX_PD, OP2_PSUBUSW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpsubusw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubusw", VEX_PD, OP2_PSUBUSW_VdqWdq, address, src0, dst);
+ }
+
+ void vpsubd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubd", VEX_PD, OP2_PSUBD_VdqWdq, src1, src0, dst);
+ }
+ void vpsubd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpsubd", VEX_PD, OP2_PSUBD_VdqWdq, offset, base, src0, dst);
+ }
+ void vpsubd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubd", VEX_PD, OP2_PSUBD_VdqWdq, address, src0, dst);
+ }
+
+ void vpsubq_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubq", VEX_PD, OP2_PSUBQ_VdqWdq, address, src0, dst);
+ }
+
+ void vpmuldq_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmuldq", VEX_PD, OP3_PMULDQ_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+
+ void vpmuludq_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmuludq", VEX_PD, OP2_PMULUDQ_VdqWdq, src1, src0, dst);
+ }
+ void vpmuludq_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpmuludq", VEX_PD, OP2_PMULUDQ_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpmuludq_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmuludq", VEX_PD, OP2_PMULUDQ_VdqWdq, address, src0, dst);
+ }
+
+ void vpmaddwd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmaddwd", VEX_PD, OP2_PMADDWD_VdqWdq, src1, src0, dst);
+ }
+ void vpmaddwd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmaddwd", VEX_PD, OP2_PMADDWD_VdqWdq, address, src0, dst);
+ }
+
+ void vpmullw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmullw", VEX_PD, OP2_PMULLW_VdqWdq, src1, src0, dst);
+ }
+ void vpmulhw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmulhw", VEX_PD, OP2_PMULHW_VdqWdq, src1, src0, dst);
+ }
+ void vpmulhuw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmulhuw", VEX_PD, OP2_PMULHUW_VdqWdq, src1, src0, dst);
+ }
+ void vpmullw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpmullw", VEX_PD, OP2_PMULLW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpmulhw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpmulhw", VEX_PD, OP2_PMULHW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpmulhuw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpmulhuw", VEX_PD, OP2_PMULHUW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpmullw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmullw", VEX_PD, OP2_PMULLW_VdqWdq, address, src0, dst);
+ }
+
+ void vpmulld_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmulld", VEX_PD, OP3_PMULLD_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpmulld_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ threeByteOpSimd("vpmulld", VEX_PD, OP3_PMULLD_VdqWdq, ESCAPE_38, offset,
+ base, src0, dst);
+ }
+ void vpmulld_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmulld", VEX_PD, OP3_PMULLD_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+ void vpmulhrsw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmulhrsw", VEX_PD, OP3_PMULHRSW_VdqWdq, ESCAPE_38, src1,
+ src0, dst);
+ }
+ void vpmulhrsw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ threeByteOpSimd("vpmulhrsw", VEX_PD, OP3_PMULHRSW_VdqWdq, ESCAPE_38, offset,
+ base, src0, dst);
+ }
+
+ void vaddps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vaddps", VEX_PS, OP2_ADDPS_VpsWps, src1, src0, dst);
+ }
+ void vaddps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vaddps", VEX_PS, OP2_ADDPS_VpsWps, offset, base, src0, dst);
+ }
+ void vaddps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vaddps", VEX_PS, OP2_ADDPS_VpsWps, address, src0, dst);
+ }
+
+ void vsubps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsubps", VEX_PS, OP2_SUBPS_VpsWps, src1, src0, dst);
+ }
+ void vsubps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vsubps", VEX_PS, OP2_SUBPS_VpsWps, offset, base, src0, dst);
+ }
+ void vsubps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsubps", VEX_PS, OP2_SUBPS_VpsWps, address, src0, dst);
+ }
+
+ void vmulps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmulps", VEX_PS, OP2_MULPS_VpsWps, src1, src0, dst);
+ }
+ void vmulps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmulps", VEX_PS, OP2_MULPS_VpsWps, offset, base, src0, dst);
+ }
+ void vmulps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmulps", VEX_PS, OP2_MULPS_VpsWps, address, src0, dst);
+ }
+
+ void vdivps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vdivps", VEX_PS, OP2_DIVPS_VpsWps, src1, src0, dst);
+ }
+ void vdivps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vdivps", VEX_PS, OP2_DIVPS_VpsWps, offset, base, src0, dst);
+ }
+ void vdivps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vdivps", VEX_PS, OP2_DIVPS_VpsWps, address, src0, dst);
+ }
+
+ void vmaxps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmaxps", VEX_PS, OP2_MAXPS_VpsWps, src1, src0, dst);
+ }
+ void vmaxps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmaxps", VEX_PS, OP2_MAXPS_VpsWps, offset, base, src0, dst);
+ }
+ void vmaxps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmaxps", VEX_PS, OP2_MAXPS_VpsWps, address, src0, dst);
+ }
+
+ void vmaxpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmaxpd", VEX_PD, OP2_MAXPD_VpdWpd, src1, src0, dst);
+ }
+
+ void vminps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vminps", VEX_PS, OP2_MINPS_VpsWps, src1, src0, dst);
+ }
+ void vminps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vminps", VEX_PS, OP2_MINPS_VpsWps, offset, base, src0, dst);
+ }
+ void vminps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vminps", VEX_PS, OP2_MINPS_VpsWps, address, src0, dst);
+ }
+
+ void vminpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vminpd", VEX_PD, OP2_MINPD_VpdWpd, src1, src0, dst);
+ }
+ void vminpd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vminpd", VEX_PD, OP2_MINPD_VpdWpd, address, src0, dst);
+ }
+
+ void vaddpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vaddpd", VEX_PD, OP2_ADDPD_VpdWpd, src1, src0, dst);
+ }
+ void vaddpd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vaddpd", VEX_PD, OP2_ADDPD_VpdWpd, address, src0, dst);
+ }
+
+ void vsubpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsubpd", VEX_PD, OP2_SUBPD_VpdWpd, src1, src0, dst);
+ }
+ void vsubpd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsubpd", VEX_PD, OP2_SUBPD_VpdWpd, address, src0, dst);
+ }
+
+ void vmulpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmulpd", VEX_PD, OP2_MULPD_VpdWpd, src1, src0, dst);
+ }
+ void vmulpd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmulpd", VEX_PD, OP2_MULPD_VpdWpd, address, src0, dst);
+ }
+
+ void vdivpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vdivpd", VEX_PD, OP2_DIVPD_VpdWpd, src1, src0, dst);
+ }
+ void vdivpd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vdivpd", VEX_PD, OP2_DIVPD_VpdWpd, address, src0, dst);
+ }
+
+ void andl_rr(RegisterID src, RegisterID dst) {
+ spew("andl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_AND_GvEv, src, dst);
+ }
+
+ void andw_rr(RegisterID src, RegisterID dst) {
+ spew("andw %s, %s", GPReg16Name(src), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_AND_GvEv, src, dst);
+ }
+
+ void andl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("andl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_AND_GvEv, offset, base, dst);
+ }
+
+ void andl_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("andl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_AND_GvEv, offset, base, index, scale, dst);
+ }
+
+ void andl_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("andl %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_AND_EvGv, offset, base, src);
+ }
+
+ void andw_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("andw %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_AND_EvGv, offset, base, src);
+ }
+
+ void andl_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("andl %s, " MEM_obs, GPReg32Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_AND_EvGv, offset, base, index, scale, src);
+ }
+
+ void andw_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("andw %s, " MEM_obs, GPReg16Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_AND_EvGv, offset, base, index, scale, src);
+ }
+
+ void andl_ir(int32_t imm, RegisterID dst) {
+ spew("andl $0x%x, %s", uint32_t(imm), GPReg32Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_AND_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_AND);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void andw_ir(int32_t imm, RegisterID dst) {
+ spew("andw $0x%x, %s", uint16_t(imm), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_AND_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_AND);
+ }
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void andl_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("andl $0x%x, " MEM_ob, uint32_t(imm), ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_AND);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void andw_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("andw $0x%x, " MEM_ob, uint16_t(imm), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_AND);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void andl_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("andl $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_AND);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void andw_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("andw $%d, " MEM_obs, int16_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_AND);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void fld_m(int32_t offset, RegisterID base) {
+ spew("fld " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FPU6, offset, base, FPU6_OP_FLD);
+ }
+ void fld32_m(int32_t offset, RegisterID base) {
+ spew("fld " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FPU6_F32, offset, base, FPU6_OP_FLD);
+ }
+ void faddp() {
+ spew("addp ");
+ m_formatter.oneByteOp(OP_FPU6_ADDP);
+ m_formatter.oneByteOp(OP_ADDP_ST0_ST1);
+ }
+ void fisttp_m(int32_t offset, RegisterID base) {
+ spew("fisttp " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FPU6, offset, base, FPU6_OP_FISTTP);
+ }
+ void fistp_m(int32_t offset, RegisterID base) {
+ spew("fistp " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FILD, offset, base, FPU6_OP_FISTP);
+ }
+ void fstp_m(int32_t offset, RegisterID base) {
+ spew("fstp " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FPU6, offset, base, FPU6_OP_FSTP);
+ }
+ void fstp32_m(int32_t offset, RegisterID base) {
+ spew("fstp32 " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FPU6_F32, offset, base, FPU6_OP_FSTP);
+ }
+ void fnstcw_m(int32_t offset, RegisterID base) {
+ spew("fnstcw " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FPU6_F32, offset, base, FPU6_OP_FISTP);
+ }
+ void fldcw_m(int32_t offset, RegisterID base) {
+ spew("fldcw " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FPU6_F32, offset, base, FPU6_OP_FLDCW);
+ }
+ void fnstsw_m(int32_t offset, RegisterID base) {
+ spew("fnstsw " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_FPU6, offset, base, FPU6_OP_FISTP);
+ }
+
+ void negl_r(RegisterID dst) {
+ spew("negl %s", GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_GROUP3_Ev, dst, GROUP3_OP_NEG);
+ }
+
+ void negl_m(int32_t offset, RegisterID base) {
+ spew("negl " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP3_Ev, offset, base, GROUP3_OP_NEG);
+ }
+
+ void notl_r(RegisterID dst) {
+ spew("notl %s", GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_GROUP3_Ev, dst, GROUP3_OP_NOT);
+ }
+
+ void notl_m(int32_t offset, RegisterID base) {
+ spew("notl " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP3_Ev, offset, base, GROUP3_OP_NOT);
+ }
+
+ void orl_rr(RegisterID src, RegisterID dst) {
+ spew("orl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_OR_GvEv, src, dst);
+ }
+
+ void orw_rr(RegisterID src, RegisterID dst) {
+ spew("orw %s, %s", GPReg16Name(src), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_OR_GvEv, src, dst);
+ }
+
+ void orl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("orl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_OR_GvEv, offset, base, dst);
+ }
+
+ void orl_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("orl %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_OR_EvGv, offset, base, src);
+ }
+
+ void orw_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("orw %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_OR_EvGv, offset, base, src);
+ }
+
+ void orl_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("orl %s, " MEM_obs, GPReg32Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_OR_EvGv, offset, base, index, scale, src);
+ }
+
+ void orw_rm(RegisterID src, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("orw %s, " MEM_obs, GPReg16Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_OR_EvGv, offset, base, index, scale, src);
+ }
+
+ void orl_ir(int32_t imm, RegisterID dst) {
+ spew("orl $0x%x, %s", uint32_t(imm), GPReg32Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_OR_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_OR);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void orw_ir(int32_t imm, RegisterID dst) {
+ spew("orw $0x%x, %s", uint16_t(imm), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_OR_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_OR);
+ }
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void orl_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("orl $0x%x, " MEM_ob, uint32_t(imm), ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_OR);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void orw_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("orw $0x%x, " MEM_ob, uint16_t(imm), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_OR);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void orl_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("orl $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_OR);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void orw_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("orw $%d, " MEM_obs, int16_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_OR);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void sbbl_rr(RegisterID src, RegisterID dst) {
+ spew("sbbl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_SBB_GvEv, src, dst);
+ }
+
+ void subl_rr(RegisterID src, RegisterID dst) {
+ spew("subl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_SUB_GvEv, src, dst);
+ }
+
+ void subw_rr(RegisterID src, RegisterID dst) {
+ spew("subw %s, %s", GPReg16Name(src), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_SUB_GvEv, src, dst);
+ }
+
+ void subl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("subl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_SUB_GvEv, offset, base, dst);
+ }
+
+ void subl_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("subl %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_SUB_EvGv, offset, base, src);
+ }
+
+ void subw_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("subw %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_SUB_EvGv, offset, base, src);
+ }
+
+ void subl_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("subl %s, " MEM_obs, GPReg32Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_SUB_EvGv, offset, base, index, scale, src);
+ }
+
+ void subw_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("subw %s, " MEM_obs, GPReg16Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_SUB_EvGv, offset, base, index, scale, src);
+ }
+
+ void subl_ir(int32_t imm, RegisterID dst) {
+ spew("subl $%d, %s", imm, GPReg32Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_SUB_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_SUB);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void subw_ir(int32_t imm, RegisterID dst) {
+ spew("subw $%d, %s", int16_t(imm), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_SUB_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_SUB);
+ }
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void subl_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("subl $%d, " MEM_ob, imm, ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_SUB);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void subw_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("subw $%d, " MEM_ob, int16_t(imm), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_SUB);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void subl_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("subl $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_SUB);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void subw_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("subw $%d, " MEM_obs, int16_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_SUB);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void xorl_rr(RegisterID src, RegisterID dst) {
+ spew("xorl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_XOR_GvEv, src, dst);
+ }
+
+ void xorw_rr(RegisterID src, RegisterID dst) {
+ spew("xorw %s, %s", GPReg16Name(src), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_XOR_GvEv, src, dst);
+ }
+
+ void xorl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("xorl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_XOR_GvEv, offset, base, dst);
+ }
+
+ void xorl_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xorl %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_XOR_EvGv, offset, base, src);
+ }
+
+ void xorw_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xorw %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_XOR_EvGv, offset, base, src);
+ }
+
+ void xorl_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xorl %s, " MEM_obs, GPReg32Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_XOR_EvGv, offset, base, index, scale, src);
+ }
+
+ void xorw_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xorw %s, " MEM_obs, GPReg16Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_XOR_EvGv, offset, base, index, scale, src);
+ }
+
+ void xorl_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("xorl $0x%x, " MEM_ob, uint32_t(imm), ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_XOR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_XOR);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void xorw_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("xorw $0x%x, " MEM_ob, uint16_t(imm), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_XOR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_XOR);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void xorl_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("xorl $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_XOR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_XOR);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void xorw_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("xorw $%d, " MEM_obs, int16_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_XOR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_XOR);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void xorl_ir(int32_t imm, RegisterID dst) {
+ spew("xorl $%d, %s", imm, GPReg32Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_XOR);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_XOR_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_XOR);
+ }
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void xorw_ir(int32_t imm, RegisterID dst) {
+ spew("xorw $%d, %s", int16_t(imm), GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_XOR);
+ m_formatter.immediate8s(imm);
+ } else {
+ if (dst == rax) {
+ m_formatter.oneByteOp(OP_XOR_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_XOR);
+ }
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void bswapl_r(RegisterID dst) {
+ spew("bswap %s", GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_BSWAP, dst);
+ }
+
+ void sarl_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ spew("sarl $%d, %s", imm, GPReg32Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp(OP_GROUP2_Ev1, dst, GROUP2_OP_SAR);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP2_EvIb, dst, GROUP2_OP_SAR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void sarl_CLr(RegisterID dst) {
+ spew("sarl %%cl, %s", GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_GROUP2_EvCL, dst, GROUP2_OP_SAR);
+ }
+
+ void shrl_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ spew("shrl $%d, %s", imm, GPReg32Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp(OP_GROUP2_Ev1, dst, GROUP2_OP_SHR);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP2_EvIb, dst, GROUP2_OP_SHR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void shrl_CLr(RegisterID dst) {
+ spew("shrl %%cl, %s", GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_GROUP2_EvCL, dst, GROUP2_OP_SHR);
+ }
+
+ void shrdl_CLr(RegisterID src, RegisterID dst) {
+ spew("shrdl %%cl, %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_SHRD_GvEv, dst, src);
+ }
+
+ void shldl_CLr(RegisterID src, RegisterID dst) {
+ spew("shldl %%cl, %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_SHLD_GvEv, dst, src);
+ }
+
+ void shll_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ spew("shll $%d, %s", imm, GPReg32Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp(OP_GROUP2_Ev1, dst, GROUP2_OP_SHL);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP2_EvIb, dst, GROUP2_OP_SHL);
+ m_formatter.immediate8u(imm);
+ }
+ }
+
+ void shll_CLr(RegisterID dst) {
+ spew("shll %%cl, %s", GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_GROUP2_EvCL, dst, GROUP2_OP_SHL);
+ }
+
+ void roll_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ spew("roll $%d, %s", imm, GPReg32Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp(OP_GROUP2_Ev1, dst, GROUP2_OP_ROL);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP2_EvIb, dst, GROUP2_OP_ROL);
+ m_formatter.immediate8u(imm);
+ }
+ }
+ void rolw_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ spew("roll $%d, %s", imm, GPReg16Name(dst));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ if (imm == 1) {
+ m_formatter.oneByteOp(OP_GROUP2_Ev1, dst, GROUP2_OP_ROL);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP2_EvIb, dst, GROUP2_OP_ROL);
+ m_formatter.immediate8u(imm);
+ }
+ }
+ void roll_CLr(RegisterID dst) {
+ spew("roll %%cl, %s", GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_GROUP2_EvCL, dst, GROUP2_OP_ROL);
+ }
+
+ void rorl_ir(int32_t imm, RegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ spew("rorl $%d, %s", imm, GPReg32Name(dst));
+ if (imm == 1) {
+ m_formatter.oneByteOp(OP_GROUP2_Ev1, dst, GROUP2_OP_ROR);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP2_EvIb, dst, GROUP2_OP_ROR);
+ m_formatter.immediate8u(imm);
+ }
+ }
+ void rorl_CLr(RegisterID dst) {
+ spew("rorl %%cl, %s", GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_GROUP2_EvCL, dst, GROUP2_OP_ROR);
+ }
+
+ void bsrl_rr(RegisterID src, RegisterID dst) {
+ spew("bsrl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_BSR_GvEv, src, dst);
+ }
+
+ void bsfl_rr(RegisterID src, RegisterID dst) {
+ spew("bsfl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_BSF_GvEv, src, dst);
+ }
+
+ void lzcntl_rr(RegisterID src, RegisterID dst) {
+ spew("lzcntl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp(OP2_LZCNT_GvEv, src, dst);
+ }
+
+ void tzcntl_rr(RegisterID src, RegisterID dst) {
+ spew("tzcntl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp(OP2_TZCNT_GvEv, src, dst);
+ }
+
+ void popcntl_rr(RegisterID src, RegisterID dst) {
+ spew("popcntl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.legacySSEPrefix(VEX_SS);
+ m_formatter.twoByteOp(OP2_POPCNT_GvEv, src, dst);
+ }
+
+ void imull_rr(RegisterID src, RegisterID dst) {
+ spew("imull %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_IMUL_GvEv, src, dst);
+ }
+
+ void imull_r(RegisterID multiplier) {
+ spew("imull %s", GPReg32Name(multiplier));
+ m_formatter.oneByteOp(OP_GROUP3_Ev, multiplier, GROUP3_OP_IMUL);
+ }
+
+ void imull_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("imull " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_IMUL_GvEv, offset, base, dst);
+ }
+
+ void imull_ir(int32_t value, RegisterID src, RegisterID dst) {
+ spew("imull $%d, %s, %s", value, GPReg32Name(src), GPReg32Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(value)) {
+ m_formatter.oneByteOp(OP_IMUL_GvEvIb, src, dst);
+ m_formatter.immediate8s(value);
+ } else {
+ m_formatter.oneByteOp(OP_IMUL_GvEvIz, src, dst);
+ m_formatter.immediate32(value);
+ }
+ }
+
+ void mull_r(RegisterID multiplier) {
+ spew("mull %s", GPReg32Name(multiplier));
+ m_formatter.oneByteOp(OP_GROUP3_Ev, multiplier, GROUP3_OP_MUL);
+ }
+
+ void idivl_r(RegisterID divisor) {
+ spew("idivl %s", GPReg32Name(divisor));
+ m_formatter.oneByteOp(OP_GROUP3_Ev, divisor, GROUP3_OP_IDIV);
+ }
+
+ void divl_r(RegisterID divisor) {
+ spew("div %s", GPReg32Name(divisor));
+ m_formatter.oneByteOp(OP_GROUP3_Ev, divisor, GROUP3_OP_DIV);
+ }
+
+ void prefix_lock() {
+ spew("lock");
+ m_formatter.oneByteOp(PRE_LOCK);
+ }
+
+ void prefix_16_for_32() {
+ spew("[16-bit operands next]");
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ }
+
+ void incl_m32(int32_t offset, RegisterID base) {
+ spew("incl " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP5_Ev, offset, base, GROUP5_OP_INC);
+ }
+
+ void decl_m32(int32_t offset, RegisterID base) {
+ spew("decl " MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP5_Ev, offset, base, GROUP5_OP_DEC);
+ }
+
+ // Note that CMPXCHG performs comparison against REG = %al/%ax/%eax/%rax.
+ // If %REG == [%base+offset], then %src -> [%base+offset].
+ // Otherwise, [%base+offset] -> %REG.
+ // For the 8-bit operations src must also be an 8-bit register.
+
+ void cmpxchgb(RegisterID src, int32_t offset, RegisterID base) {
+ spew("cmpxchgb %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+ m_formatter.twoByteOp8(OP2_CMPXCHG_GvEb, offset, base, src);
+ }
+ void cmpxchgb(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpxchgb %s, " MEM_obs, GPReg8Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.twoByteOp8(OP2_CMPXCHG_GvEb, offset, base, index, scale, src);
+ }
+ void cmpxchgw(RegisterID src, int32_t offset, RegisterID base) {
+ spew("cmpxchgw %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.twoByteOp(OP2_CMPXCHG_GvEw, offset, base, src);
+ }
+ void cmpxchgw(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpxchgw %s, " MEM_obs, GPReg16Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.twoByteOp(OP2_CMPXCHG_GvEw, offset, base, index, scale, src);
+ }
+ void cmpxchgl(RegisterID src, int32_t offset, RegisterID base) {
+ spew("cmpxchgl %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
+ m_formatter.twoByteOp(OP2_CMPXCHG_GvEw, offset, base, src);
+ }
+ void cmpxchgl(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpxchgl %s, " MEM_obs, GPReg32Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.twoByteOp(OP2_CMPXCHG_GvEw, offset, base, index, scale, src);
+ }
+
+ void cmpxchg8b(RegisterID srcHi, RegisterID srcLo, RegisterID newHi,
+ RegisterID newLo, int32_t offset, RegisterID base) {
+ MOZ_ASSERT(srcHi == edx.code() && srcLo == eax.code());
+ MOZ_ASSERT(newHi == ecx.code() && newLo == ebx.code());
+ spew("cmpxchg8b %s, " MEM_ob, "edx:eax", ADDR_ob(offset, base));
+ m_formatter.twoByteOp(OP2_CMPXCHGNB, offset, base, 1);
+ }
+ void cmpxchg8b(RegisterID srcHi, RegisterID srcLo, RegisterID newHi,
+ RegisterID newLo, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ MOZ_ASSERT(srcHi == edx.code() && srcLo == eax.code());
+ MOZ_ASSERT(newHi == ecx.code() && newLo == ebx.code());
+ spew("cmpxchg8b %s, " MEM_obs, "edx:eax",
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.twoByteOp(OP2_CMPXCHGNB, offset, base, index, scale, 1);
+ }
+
+ // Comparisons:
+
+ void cmpl_rr(RegisterID rhs, RegisterID lhs) {
+ spew("cmpl %s, %s", GPReg32Name(rhs), GPReg32Name(lhs));
+ m_formatter.oneByteOp(OP_CMP_GvEv, rhs, lhs);
+ }
+
+ void cmpl_rm(RegisterID rhs, int32_t offset, RegisterID base) {
+ spew("cmpl %s, " MEM_ob, GPReg32Name(rhs), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_CMP_EvGv, offset, base, rhs);
+ }
+
+ void cmpl_rm(RegisterID rhs, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpl %s, " MEM_obs, GPReg32Name(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_CMP_EvGv, offset, base, index, scale, rhs);
+ }
+
+ void cmpl_mr(int32_t offset, RegisterID base, RegisterID lhs) {
+ spew("cmpl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(lhs));
+ m_formatter.oneByteOp(OP_CMP_GvEv, offset, base, lhs);
+ }
+
+ void cmpl_mr(const void* address, RegisterID lhs) {
+ spew("cmpl %p, %s", address, GPReg32Name(lhs));
+ m_formatter.oneByteOp(OP_CMP_GvEv, address, lhs);
+ }
+
+ void cmpl_ir(int32_t rhs, RegisterID lhs) {
+ if (rhs == 0) {
+ testl_rr(lhs, lhs);
+ return;
+ }
+
+ spew("cmpl $0x%x, %s", uint32_t(rhs), GPReg32Name(lhs));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, lhs, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ if (lhs == rax) {
+ m_formatter.oneByteOp(OP_CMP_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, lhs, GROUP1_OP_CMP);
+ }
+ m_formatter.immediate32(rhs);
+ }
+ }
+
+ void cmpl_i32r(int32_t rhs, RegisterID lhs) {
+ spew("cmpl $0x%04x, %s", uint32_t(rhs), GPReg32Name(lhs));
+ if (lhs == rax) {
+ m_formatter.oneByteOp(OP_CMP_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, lhs, GROUP1_OP_CMP);
+ }
+ m_formatter.immediate32(rhs);
+ }
+
+ void cmpl_im(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("cmpl $0x%x, " MEM_ob, uint32_t(rhs), ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+
+ void cmpb_rr(RegisterID rhs, RegisterID lhs) {
+ spew("cmpb %s, %s", GPReg8Name(rhs), GPReg8Name(lhs));
+ m_formatter.oneByteOp(OP_CMP_GbEb, rhs, lhs);
+ }
+
+ void cmpb_rm(RegisterID rhs, int32_t offset, RegisterID base) {
+ spew("cmpb %s, " MEM_ob, GPReg8Name(rhs), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_CMP_EbGb, offset, base, rhs);
+ }
+
+ void cmpb_rm(RegisterID rhs, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpb %s, " MEM_obs, GPReg8Name(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_CMP_EbGb, offset, base, index, scale, rhs);
+ }
+
+ void cmpb_rm(RegisterID rhs, const void* addr) {
+ spew("cmpb %s, %p", GPReg8Name(rhs), addr);
+ m_formatter.oneByteOp(OP_CMP_EbGb, addr, rhs);
+ }
+
+ void cmpb_ir(int32_t rhs, RegisterID lhs) {
+ if (rhs == 0) {
+ testb_rr(lhs, lhs);
+ return;
+ }
+
+ spew("cmpb $0x%x, %s", uint32_t(rhs), GPReg8Name(lhs));
+ if (lhs == rax) {
+ m_formatter.oneByteOp(OP_CMP_EAXIb);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, lhs, GROUP1_OP_CMP);
+ }
+ m_formatter.immediate8(rhs);
+ }
+
+ void cmpb_im(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("cmpb $0x%x, " MEM_ob, uint32_t(rhs), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate8(rhs);
+ }
+
+ void cmpb_im(int32_t rhs, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("cmpb $0x%x, " MEM_obs, uint32_t(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate8(rhs);
+ }
+
+ void cmpb_im(int32_t rhs, const void* addr) {
+ spew("cmpb $0x%x, %p", uint32_t(rhs), addr);
+ m_formatter.oneByteOp(OP_GROUP1_EbIb, addr, GROUP1_OP_CMP);
+ m_formatter.immediate8(rhs);
+ }
+
+ void cmpl_im(int32_t rhs, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("cmpl $0x%x, " MEM_obs, uint32_t(rhs),
+ ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+
+ [[nodiscard]] JmpSrc cmpl_im_disp32(int32_t rhs, int32_t offset,
+ RegisterID base) {
+ spew("cmpl $0x%x, " MEM_o32b, uint32_t(rhs), ADDR_o32b(offset, base));
+ JmpSrc r;
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp_disp32(OP_GROUP1_EvIb, offset, base, GROUP1_OP_CMP);
+ r = JmpSrc(m_formatter.size());
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp_disp32(OP_GROUP1_EvIz, offset, base, GROUP1_OP_CMP);
+ r = JmpSrc(m_formatter.size());
+ m_formatter.immediate32(rhs);
+ }
+ return r;
+ }
+
+ [[nodiscard]] JmpSrc cmpl_im_disp32(int32_t rhs, const void* addr) {
+ spew("cmpl $0x%x, %p", uint32_t(rhs), addr);
+ JmpSrc r;
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp_disp32(OP_GROUP1_EvIb, addr, GROUP1_OP_CMP);
+ r = JmpSrc(m_formatter.size());
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp_disp32(OP_GROUP1_EvIz, addr, GROUP1_OP_CMP);
+ r = JmpSrc(m_formatter.size());
+ m_formatter.immediate32(rhs);
+ }
+ return r;
+ }
+
+ void cmpl_i32m(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("cmpl $0x%04x, " MEM_ob, uint32_t(rhs), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+
+ void cmpl_i32m(int32_t rhs, const void* addr) {
+ spew("cmpl $0x%04x, %p", uint32_t(rhs), addr);
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+
+ void cmpl_rm(RegisterID rhs, const void* addr) {
+ spew("cmpl %s, %p", GPReg32Name(rhs), addr);
+ m_formatter.oneByteOp(OP_CMP_EvGv, addr, rhs);
+ }
+
+ void cmpl_rm_disp32(RegisterID rhs, const void* addr) {
+ spew("cmpl %s, %p", GPReg32Name(rhs), addr);
+ m_formatter.oneByteOp_disp32(OP_CMP_EvGv, addr, rhs);
+ }
+
+ void cmpl_im(int32_t rhs, const void* addr) {
+ spew("cmpl $0x%x, %p", uint32_t(rhs), addr);
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_CMP);
+ m_formatter.immediate32(rhs);
+ }
+ }
+
+ void cmpw_rr(RegisterID rhs, RegisterID lhs) {
+ spew("cmpw %s, %s", GPReg16Name(rhs), GPReg16Name(lhs));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_CMP_GvEv, rhs, lhs);
+ }
+
+ void cmpw_rm(RegisterID rhs, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("cmpw %s, " MEM_obs, GPReg16Name(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_CMP_EvGv, offset, base, index, scale, rhs);
+ }
+
+ void cmpw_ir(int32_t rhs, RegisterID lhs) {
+ if (rhs == 0) {
+ testw_rr(lhs, lhs);
+ return;
+ }
+
+ spew("cmpw $0x%x, %s", uint32_t(rhs), GPReg16Name(lhs));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, lhs, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, lhs, GROUP1_OP_CMP);
+ m_formatter.immediate16(rhs);
+ }
+ }
+
+ void cmpw_im(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("cmpw $0x%x, " MEM_ob, uint32_t(rhs), ADDR_ob(offset, base));
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, GROUP1_OP_CMP);
+ m_formatter.immediate16(rhs);
+ }
+ }
+
+ void cmpw_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("cmpw $%d, " MEM_obs, imm, ADDR_obs(offset, base, index, scale));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, offset, base, index, scale,
+ GROUP1_OP_CMP);
+ m_formatter.immediate16(imm);
+ }
+ }
+
+ void cmpw_im(int32_t rhs, const void* addr) {
+ spew("cmpw $0x%x, %p", uint32_t(rhs), addr);
+ if (CAN_SIGN_EXTEND_8_32(rhs)) {
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_CMP);
+ m_formatter.immediate8s(rhs);
+ } else {
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_CMP);
+ m_formatter.immediate16(rhs);
+ }
+ }
+
+ void testl_rr(RegisterID rhs, RegisterID lhs) {
+ spew("testl %s, %s", GPReg32Name(rhs), GPReg32Name(lhs));
+ m_formatter.oneByteOp(OP_TEST_EvGv, lhs, rhs);
+ }
+
+ void testb_rr(RegisterID rhs, RegisterID lhs) {
+ spew("testb %s, %s", GPReg8Name(rhs), GPReg8Name(lhs));
+ m_formatter.oneByteOp(OP_TEST_EbGb, lhs, rhs);
+ }
+
+ void testl_ir(int32_t rhs, RegisterID lhs) {
+ // If the mask fits in an 8-bit immediate, we can use testb with an
+ // 8-bit subreg.
+ if (CAN_ZERO_EXTEND_8_32(rhs) && HasSubregL(lhs)) {
+ testb_ir(rhs, lhs);
+ return;
+ }
+ // If the mask is a subset of 0xff00, we can use testb with an h reg, if
+ // one happens to be available.
+ if (CAN_ZERO_EXTEND_8H_32(rhs) && HasSubregH(lhs)) {
+ testb_ir_norex(rhs >> 8, GetSubregH(lhs));
+ return;
+ }
+ spew("testl $0x%x, %s", uint32_t(rhs), GPReg32Name(lhs));
+ if (lhs == rax) {
+ m_formatter.oneByteOp(OP_TEST_EAXIv);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP3_EvIz, lhs, GROUP3_OP_TEST);
+ }
+ m_formatter.immediate32(rhs);
+ }
+
+ void testl_i32m(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("testl $0x%x, " MEM_ob, uint32_t(rhs), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP3_EvIz, offset, base, GROUP3_OP_TEST);
+ m_formatter.immediate32(rhs);
+ }
+
+ void testl_i32m(int32_t rhs, const void* addr) {
+ spew("testl $0x%x, %p", uint32_t(rhs), addr);
+ m_formatter.oneByteOp(OP_GROUP3_EvIz, addr, GROUP3_OP_TEST);
+ m_formatter.immediate32(rhs);
+ }
+
+ void testb_im(int32_t rhs, int32_t offset, RegisterID base) {
+ spew("testb $0x%x, " MEM_ob, uint32_t(rhs), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP3_EbIb, offset, base, GROUP3_OP_TEST);
+ m_formatter.immediate8(rhs);
+ }
+
+ void testb_im(int32_t rhs, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("testb $0x%x, " MEM_obs, uint32_t(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP3_EbIb, offset, base, index, scale,
+ GROUP3_OP_TEST);
+ m_formatter.immediate8(rhs);
+ }
+
+ void testl_i32m(int32_t rhs, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("testl $0x%4x, " MEM_obs, uint32_t(rhs),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP3_EvIz, offset, base, index, scale,
+ GROUP3_OP_TEST);
+ m_formatter.immediate32(rhs);
+ }
+
+ void testw_rr(RegisterID rhs, RegisterID lhs) {
+ spew("testw %s, %s", GPReg16Name(rhs), GPReg16Name(lhs));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_TEST_EvGv, lhs, rhs);
+ }
+
+ void testb_ir(int32_t rhs, RegisterID lhs) {
+ spew("testb $0x%x, %s", uint32_t(rhs), GPReg8Name(lhs));
+ if (lhs == rax) {
+ m_formatter.oneByteOp8(OP_TEST_EAXIb);
+ } else {
+ m_formatter.oneByteOp8(OP_GROUP3_EbIb, lhs, GROUP3_OP_TEST);
+ }
+ m_formatter.immediate8(rhs);
+ }
+
+ // Like testb_ir, but never emits a REX prefix. This may be used to
+ // reference ah..bh.
+ void testb_ir_norex(int32_t rhs, HRegisterID lhs) {
+ spew("testb $0x%x, %s", uint32_t(rhs), HRegName8(lhs));
+ m_formatter.oneByteOp8_norex(OP_GROUP3_EbIb, lhs, GROUP3_OP_TEST);
+ m_formatter.immediate8(rhs);
+ }
+
+ void setCC_r(Condition cond, RegisterID lhs) {
+ spew("set%s %s", CCName(cond), GPReg8Name(lhs));
+ m_formatter.twoByteOp8(setccOpcode(cond), lhs, (GroupOpcodeID)0);
+ }
+
+ void sete_r(RegisterID dst) { setCC_r(ConditionE, dst); }
+
+ void setz_r(RegisterID dst) { sete_r(dst); }
+
+ void setne_r(RegisterID dst) { setCC_r(ConditionNE, dst); }
+
+ void setnz_r(RegisterID dst) { setne_r(dst); }
+
+ // Various move ops:
+
+ void cdq() {
+ spew("cdq ");
+ m_formatter.oneByteOp(OP_CDQ);
+ }
+
+ void xchgb_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xchgb %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp8(OP_XCHG_GbEb, offset, base, src);
+ }
+ void xchgb_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xchgb %s, " MEM_obs, GPReg8Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp8(OP_XCHG_GbEb, offset, base, index, scale, src);
+ }
+
+ void xchgw_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xchgw %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_XCHG_GvEv, offset, base, src);
+ }
+ void xchgw_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xchgw %s, " MEM_obs, GPReg16Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_XCHG_GvEv, offset, base, index, scale, src);
+ }
+
+ void xchgl_rr(RegisterID src, RegisterID dst) {
+ spew("xchgl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_XCHG_GvEv, src, dst);
+ }
+ void xchgl_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("xchgl %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_XCHG_GvEv, offset, base, src);
+ }
+ void xchgl_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("xchgl %s, " MEM_obs, GPReg32Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_XCHG_GvEv, offset, base, index, scale, src);
+ }
+
+ void cmovCCl_rr(Condition cond, RegisterID src, RegisterID dst) {
+ spew("cmov%s %s, %s", CCName(cond), GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp(cmovccOpcode(cond), src, dst);
+ }
+ void cmovCCl_mr(Condition cond, int32_t offset, RegisterID base,
+ RegisterID dst) {
+ spew("cmov%s " MEM_ob ", %s", CCName(cond), ADDR_ob(offset, base),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp(cmovccOpcode(cond), offset, base, dst);
+ }
+ void cmovCCl_mr(Condition cond, int32_t offset, RegisterID base,
+ RegisterID index, int scale, RegisterID dst) {
+ spew("cmov%s " MEM_obs ", %s", CCName(cond),
+ ADDR_obs(offset, base, index, scale), GPReg32Name(dst));
+ m_formatter.twoByteOp(cmovccOpcode(cond), offset, base, index, scale, dst);
+ }
+
+ void movl_rr(RegisterID src, RegisterID dst) {
+ spew("movl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_MOV_GvEv, src, dst);
+ }
+
+ void movw_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movw %s, " MEM_ob, GPReg16Name(src), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_MOV_EvGv, offset, base, src);
+ }
+
+ void movw_rm_disp32(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movw %s, " MEM_o32b, GPReg16Name(src), ADDR_o32b(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp_disp32(OP_MOV_EvGv, offset, base, src);
+ }
+
+ void movw_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("movw %s, " MEM_obs, GPReg16Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_MOV_EvGv, offset, base, index, scale, src);
+ }
+
+ void movw_rm(RegisterID src, const void* addr) {
+ spew("movw %s, %p", GPReg16Name(src), addr);
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp_disp32(OP_MOV_EvGv, addr, src);
+ }
+
+ void movl_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movl %s, " MEM_ob, GPReg32Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_MOV_EvGv, offset, base, src);
+ }
+
+ void movl_rm_disp32(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movl %s, " MEM_o32b, GPReg32Name(src), ADDR_o32b(offset, base));
+ m_formatter.oneByteOp_disp32(OP_MOV_EvGv, offset, base, src);
+ }
+
+ void movl_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("movl %s, " MEM_obs, GPReg32Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_MOV_EvGv, offset, base, index, scale, src);
+ }
+
+ void movl_mEAX(const void* addr) {
+#ifdef JS_CODEGEN_X64
+ if (IsAddressImmediate(addr)) {
+ movl_mr(addr, rax);
+ return;
+ }
+#endif
+
+#ifdef JS_CODEGEN_X64
+ spew("movabs %p, %%eax", addr);
+#else
+ spew("movl %p, %%eax", addr);
+#endif
+ m_formatter.oneByteOp(OP_MOV_EAXOv);
+#ifdef JS_CODEGEN_X64
+ m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+#else
+ m_formatter.immediate32(reinterpret_cast<int32_t>(addr));
+#endif
+ }
+
+ void movl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_MOV_GvEv, offset, base, dst);
+ }
+
+ void movl_mr_disp32(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movl " MEM_o32b ", %s", ADDR_o32b(offset, base),
+ GPReg32Name(dst));
+ m_formatter.oneByteOp_disp32(OP_MOV_GvEv, offset, base, dst);
+ }
+
+ void movl_mr(const void* base, RegisterID index, int scale, RegisterID dst) {
+ int32_t disp = AddressImmediate(base);
+
+ spew("movl " MEM_os ", %s", ADDR_os(disp, index, scale),
+ GPReg32Name(dst));
+ m_formatter.oneByteOp_disp32(OP_MOV_GvEv, disp, index, scale, dst);
+ }
+
+ void movl_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_MOV_GvEv, offset, base, index, scale, dst);
+ }
+
+ void movl_mr(const void* addr, RegisterID dst) {
+ if (dst == rax
+#ifdef JS_CODEGEN_X64
+ && !IsAddressImmediate(addr)
+#endif
+ ) {
+ movl_mEAX(addr);
+ return;
+ }
+
+ spew("movl %p, %s", addr, GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_MOV_GvEv, addr, dst);
+ }
+
+ void movl_i32r(int32_t imm, RegisterID dst) {
+ spew("movl $0x%x, %s", uint32_t(imm), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_MOV_EAXIv, dst);
+ m_formatter.immediate32(imm);
+ }
+
+ void movb_ir(int32_t imm, RegisterID reg) {
+ spew("movb $0x%x, %s", uint32_t(imm), GPReg8Name(reg));
+ m_formatter.oneByteOp8(OP_MOV_EbIb, reg);
+ m_formatter.immediate8(imm);
+ }
+
+ void movb_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("movb $0x%x, " MEM_ob, uint32_t(imm), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP11_EvIb, offset, base, GROUP11_MOV);
+ m_formatter.immediate8(imm);
+ }
+
+ void movb_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("movb $0x%x, " MEM_obs, uint32_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP11_EvIb, offset, base, index, scale,
+ GROUP11_MOV);
+ m_formatter.immediate8(imm);
+ }
+
+ void movb_im(int32_t imm, const void* addr) {
+ spew("movb $%d, %p", imm, addr);
+ m_formatter.oneByteOp_disp32(OP_GROUP11_EvIb, addr, GROUP11_MOV);
+ m_formatter.immediate8(imm);
+ }
+
+ void movw_im(int32_t imm, int32_t offset, RegisterID base) {
+ spew("movw $0x%x, " MEM_ob, uint32_t(imm), ADDR_ob(offset, base));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP11_EvIz, offset, base, GROUP11_MOV);
+ m_formatter.immediate16(imm);
+ }
+
+ void movw_im(int32_t imm, const void* addr) {
+ spew("movw $%d, %p", imm, addr);
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp_disp32(OP_GROUP11_EvIz, addr, GROUP11_MOV);
+ m_formatter.immediate16(imm);
+ }
+
+ void movl_i32m(int32_t imm, int32_t offset, RegisterID base) {
+ spew("movl $0x%x, " MEM_ob, uint32_t(imm), ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP11_EvIz, offset, base, GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+
+ void movw_im(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("movw $0x%x, " MEM_obs, uint32_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.prefix(PRE_OPERAND_SIZE);
+ m_formatter.oneByteOp(OP_GROUP11_EvIz, offset, base, index, scale,
+ GROUP11_MOV);
+ m_formatter.immediate16(imm);
+ }
+
+ void movl_i32m(int32_t imm, int32_t offset, RegisterID base, RegisterID index,
+ int scale) {
+ spew("movl $0x%x, " MEM_obs, uint32_t(imm),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP11_EvIz, offset, base, index, scale,
+ GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+
+ void movl_EAXm(const void* addr) {
+#ifdef JS_CODEGEN_X64
+ if (IsAddressImmediate(addr)) {
+ movl_rm(rax, addr);
+ return;
+ }
+#endif
+
+ spew("movl %%eax, %p", addr);
+ m_formatter.oneByteOp(OP_MOV_OvEAX);
+#ifdef JS_CODEGEN_X64
+ m_formatter.immediate64(reinterpret_cast<int64_t>(addr));
+#else
+ m_formatter.immediate32(reinterpret_cast<int32_t>(addr));
+#endif
+ }
+
+ void vmovq_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ // vmovq_rm can be encoded either as a true vmovq or as a vmovd with a
+ // REX prefix modifying it to be 64-bit. We choose the vmovq encoding
+ // because it's smaller (when it doesn't need a REX prefix for other
+ // reasons) and because it works on 32-bit x86 too.
+ twoByteOpSimd("vmovq", VEX_PD, OP2_MOVQ_WdVd, offset, base, invalid_xmm,
+ src);
+ }
+
+ void vmovq_rm_disp32(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd_disp32("vmovq", VEX_PD, OP2_MOVQ_WdVd, offset, base,
+ invalid_xmm, src);
+ }
+
+ void vmovq_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovq", VEX_PD, OP2_MOVQ_WdVd, offset, base, index, scale,
+ invalid_xmm, src);
+ }
+
+ void vmovq_rm(XMMRegisterID src, const void* addr) {
+ twoByteOpSimd("vmovq", VEX_PD, OP2_MOVQ_WdVd, addr, invalid_xmm, src);
+ }
+
+ void vmovq_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ // vmovq_mr can be encoded either as a true vmovq or as a vmovd with a
+ // REX prefix modifying it to be 64-bit. We choose the vmovq encoding
+ // because it's smaller (when it doesn't need a REX prefix for other
+ // reasons) and because it works on 32-bit x86 too.
+ twoByteOpSimd("vmovq", VEX_SS, OP2_MOVQ_VdWd, offset, base, invalid_xmm,
+ dst);
+ }
+
+ void vmovq_mr_disp32(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd_disp32("vmovq", VEX_SS, OP2_MOVQ_VdWd, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vmovq_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ twoByteOpSimd("vmovq", VEX_SS, OP2_MOVQ_VdWd, offset, base, index, scale,
+ invalid_xmm, dst);
+ }
+
+ void vmovq_mr(const void* addr, XMMRegisterID dst) {
+ twoByteOpSimd("vmovq", VEX_SS, OP2_MOVQ_VdWd, addr, invalid_xmm, dst);
+ }
+
+ void movl_rm(RegisterID src, const void* addr) {
+ if (src == rax
+#ifdef JS_CODEGEN_X64
+ && !IsAddressImmediate(addr)
+#endif
+ ) {
+ movl_EAXm(addr);
+ return;
+ }
+
+ spew("movl %s, %p", GPReg32Name(src), addr);
+ m_formatter.oneByteOp(OP_MOV_EvGv, addr, src);
+ }
+
+ void movl_i32m(int32_t imm, const void* addr) {
+ spew("movl $%d, %p", imm, addr);
+ m_formatter.oneByteOp(OP_GROUP11_EvIz, addr, GROUP11_MOV);
+ m_formatter.immediate32(imm);
+ }
+
+ void movb_rm(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movb %s, " MEM_ob, GPReg8Name(src), ADDR_ob(offset, base));
+ m_formatter.oneByteOp8(OP_MOV_EbGv, offset, base, src);
+ }
+
+ void movb_rm_disp32(RegisterID src, int32_t offset, RegisterID base) {
+ spew("movb %s, " MEM_o32b, GPReg8Name(src), ADDR_o32b(offset, base));
+ m_formatter.oneByteOp8_disp32(OP_MOV_EbGv, offset, base, src);
+ }
+
+ void movb_rm(RegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ spew("movb %s, " MEM_obs, GPReg8Name(src),
+ ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp8(OP_MOV_EbGv, offset, base, index, scale, src);
+ }
+
+ void movb_rm(RegisterID src, const void* addr) {
+ spew("movb %s, %p", GPReg8Name(src), addr);
+ m_formatter.oneByteOp8(OP_MOV_EbGv, addr, src);
+ }
+
+ void movb_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movb " MEM_ob ", %s", ADDR_ob(offset, base), GPReg8Name(dst));
+ m_formatter.oneByteOp(OP_MOV_GvEb, offset, base, dst);
+ }
+
+ void movb_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movb " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg8Name(dst));
+ m_formatter.oneByteOp(OP_MOV_GvEb, offset, base, index, scale, dst);
+ }
+
+ void movzbl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movzbl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVZX_GvEb, offset, base, dst);
+ }
+
+ void movzbl_mr_disp32(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movzbl " MEM_o32b ", %s", ADDR_o32b(offset, base),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp_disp32(OP2_MOVZX_GvEb, offset, base, dst);
+ }
+
+ void movzbl_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movzbl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVZX_GvEb, offset, base, index, scale, dst);
+ }
+
+ void movzbl_mr(const void* addr, RegisterID dst) {
+ spew("movzbl %p, %s", addr, GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVZX_GvEb, addr, dst);
+ }
+
+ void movsbl_rr(RegisterID src, RegisterID dst) {
+ spew("movsbl %s, %s", GPReg8Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp8_movx(OP2_MOVSX_GvEb, src, dst);
+ }
+
+ void movsbl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movsbl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVSX_GvEb, offset, base, dst);
+ }
+
+ void movsbl_mr_disp32(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movsbl " MEM_o32b ", %s", ADDR_o32b(offset, base),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp_disp32(OP2_MOVSX_GvEb, offset, base, dst);
+ }
+
+ void movsbl_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movsbl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVSX_GvEb, offset, base, index, scale, dst);
+ }
+
+ void movsbl_mr(const void* addr, RegisterID dst) {
+ spew("movsbl %p, %s", addr, GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVSX_GvEb, addr, dst);
+ }
+
+ void movzwl_rr(RegisterID src, RegisterID dst) {
+ spew("movzwl %s, %s", GPReg16Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVZX_GvEw, src, dst);
+ }
+
+ void movzwl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movzwl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVZX_GvEw, offset, base, dst);
+ }
+
+ void movzwl_mr_disp32(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movzwl " MEM_o32b ", %s", ADDR_o32b(offset, base),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp_disp32(OP2_MOVZX_GvEw, offset, base, dst);
+ }
+
+ void movzwl_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movzwl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVZX_GvEw, offset, base, index, scale, dst);
+ }
+
+ void movzwl_mr(const void* addr, RegisterID dst) {
+ spew("movzwl %p, %s", addr, GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVZX_GvEw, addr, dst);
+ }
+
+ void movswl_rr(RegisterID src, RegisterID dst) {
+ spew("movswl %s, %s", GPReg16Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVSX_GvEw, src, dst);
+ }
+
+ void movswl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movswl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVSX_GvEw, offset, base, dst);
+ }
+
+ void movswl_mr_disp32(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("movswl " MEM_o32b ", %s", ADDR_o32b(offset, base),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp_disp32(OP2_MOVSX_GvEw, offset, base, dst);
+ }
+
+ void movswl_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("movswl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVSX_GvEw, offset, base, index, scale, dst);
+ }
+
+ void movswl_mr(const void* addr, RegisterID dst) {
+ spew("movswl %p, %s", addr, GPReg32Name(dst));
+ m_formatter.twoByteOp(OP2_MOVSX_GvEw, addr, dst);
+ }
+
+ void movzbl_rr(RegisterID src, RegisterID dst) {
+ spew("movzbl %s, %s", GPReg8Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp8_movx(OP2_MOVZX_GvEb, src, dst);
+ }
+
+ void leal_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("leal " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_LEA, offset, base, index, scale, dst);
+ }
+
+ void leal_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("leal " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_LEA, offset, base, dst);
+ }
+
+ // Flow control:
+
+ [[nodiscard]] JmpSrc call() {
+ m_formatter.oneByteOp(OP_CALL_rel32);
+ JmpSrc r = m_formatter.immediateRel32();
+ spew("call .Lfrom%d", r.offset());
+ return r;
+ }
+
+ void call_r(RegisterID dst) {
+ m_formatter.oneByteOp(OP_GROUP5_Ev, dst, GROUP5_OP_CALLN);
+ spew("call *%s", GPRegName(dst));
+ }
+
+ void call_m(int32_t offset, RegisterID base) {
+ spew("call *" MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP5_Ev, offset, base, GROUP5_OP_CALLN);
+ }
+
+ // Comparison of EAX against a 32-bit immediate. The immediate is patched
+ // in as if it were a jump target. The intention is to toggle the first
+ // byte of the instruction between a CMP and a JMP to produce a pseudo-NOP.
+ [[nodiscard]] JmpSrc cmp_eax() {
+ m_formatter.oneByteOp(OP_CMP_EAXIv);
+ JmpSrc r = m_formatter.immediateRel32();
+ spew("cmpl %%eax, .Lfrom%d", r.offset());
+ return r;
+ }
+
+ void jmp_i(JmpDst dst) {
+ int32_t diff = dst.offset() - m_formatter.size();
+ spew("jmp .Llabel%d", dst.offset());
+
+ // The jump immediate is an offset from the end of the jump instruction.
+ // A jump instruction is either 1 byte opcode and 1 byte offset, or 1
+ // byte opcode and 4 bytes offset.
+ if (CAN_SIGN_EXTEND_8_32(diff - 2)) {
+ m_formatter.oneByteOp(OP_JMP_rel8);
+ m_formatter.immediate8s(diff - 2);
+ } else {
+ m_formatter.oneByteOp(OP_JMP_rel32);
+ m_formatter.immediate32(diff - 5);
+ }
+ }
+ [[nodiscard]] JmpSrc jmp() {
+ m_formatter.oneByteOp(OP_JMP_rel32);
+ JmpSrc r = m_formatter.immediateRel32();
+ spew("jmp .Lfrom%d", r.offset());
+ return r;
+ }
+
+ void jmp_r(RegisterID dst) {
+ spew("jmp *%s", GPRegName(dst));
+ m_formatter.oneByteOp(OP_GROUP5_Ev, dst, GROUP5_OP_JMPN);
+ }
+
+ void jmp_m(int32_t offset, RegisterID base) {
+ spew("jmp *" MEM_ob, ADDR_ob(offset, base));
+ m_formatter.oneByteOp(OP_GROUP5_Ev, offset, base, GROUP5_OP_JMPN);
+ }
+
+ void jmp_m(int32_t offset, RegisterID base, RegisterID index, int scale) {
+ spew("jmp *" MEM_obs, ADDR_obs(offset, base, index, scale));
+ m_formatter.oneByteOp(OP_GROUP5_Ev, offset, base, index, scale,
+ GROUP5_OP_JMPN);
+ }
+
+ void jCC_i(Condition cond, JmpDst dst) {
+ int32_t diff = dst.offset() - m_formatter.size();
+ spew("j%s .Llabel%d", CCName(cond), dst.offset());
+
+ // The jump immediate is an offset from the end of the jump instruction.
+ // A conditional jump instruction is either 1 byte opcode and 1 byte
+ // offset, or 2 bytes opcode and 4 bytes offset.
+ if (CAN_SIGN_EXTEND_8_32(diff - 2)) {
+ m_formatter.oneByteOp(jccRel8(cond));
+ m_formatter.immediate8s(diff - 2);
+ } else {
+ m_formatter.twoByteOp(jccRel32(cond));
+ m_formatter.immediate32(diff - 6);
+ }
+ }
+
+ [[nodiscard]] JmpSrc jCC(Condition cond) {
+ m_formatter.twoByteOp(jccRel32(cond));
+ JmpSrc r = m_formatter.immediateRel32();
+ spew("j%s .Lfrom%d", CCName(cond), r.offset());
+ return r;
+ }
+
+ // SSE operations:
+
+ void vpcmpeqb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqb", VEX_PD, OP2_PCMPEQB_VdqWdq, src1, src0, dst);
+ }
+ void vpcmpeqb_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqb", VEX_PD, OP2_PCMPEQB_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpcmpeqb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqb", VEX_PD, OP2_PCMPEQB_VdqWdq, address, src0, dst);
+ }
+
+ void vpcmpgtb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtb", VEX_PD, OP2_PCMPGTB_VdqWdq, src1, src0, dst);
+ }
+ void vpcmpgtb_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtb", VEX_PD, OP2_PCMPGTB_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpcmpgtb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtb", VEX_PD, OP2_PCMPGTB_VdqWdq, address, src0, dst);
+ }
+
+ void vpcmpeqw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqw", VEX_PD, OP2_PCMPEQW_VdqWdq, src1, src0, dst);
+ }
+ void vpcmpeqw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqw", VEX_PD, OP2_PCMPEQW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpcmpeqw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqw", VEX_PD, OP2_PCMPEQW_VdqWdq, address, src0, dst);
+ }
+
+ void vpcmpgtw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtw", VEX_PD, OP2_PCMPGTW_VdqWdq, src1, src0, dst);
+ }
+ void vpcmpgtw_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtw", VEX_PD, OP2_PCMPGTW_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpcmpgtw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtw", VEX_PD, OP2_PCMPGTW_VdqWdq, address, src0, dst);
+ }
+
+ void vpcmpeqd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqd", VEX_PD, OP2_PCMPEQD_VdqWdq, src1, src0, dst);
+ }
+ void vpcmpeqd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqd", VEX_PD, OP2_PCMPEQD_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpcmpeqd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpeqd", VEX_PD, OP2_PCMPEQD_VdqWdq, address, src0, dst);
+ }
+
+ void vpcmpgtd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtd", VEX_PD, OP2_PCMPGTD_VdqWdq, src1, src0, dst);
+ }
+ void vpcmpgtd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtd", VEX_PD, OP2_PCMPGTD_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpcmpgtd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpcmpgtd", VEX_PD, OP2_PCMPGTD_VdqWdq, address, src0, dst);
+ }
+
+ void vpcmpgtq_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpcmpgtq", VEX_PD, OP3_PCMPGTQ_VdqWdq, ESCAPE_38, src1,
+ src0, dst);
+ }
+
+ void vpcmpeqq_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpcmpeqq", VEX_PD, OP3_PCMPEQQ_VdqWdq, ESCAPE_38, src1,
+ src0, dst);
+ }
+ void vpcmpeqq_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ threeByteOpSimd("vpcmpeqq", VEX_PD, OP3_PCMPEQQ_VdqWdq, ESCAPE_38, offset,
+ base, src0, dst);
+ }
+ void vpcmpeqq_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpcmpeqq", VEX_PD, OP3_PCMPEQQ_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vcmpps_rr(uint8_t order, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT_IF(!useVEX_,
+ order < uint8_t(X86Encoding::ConditionCmp_AVX_Enabled));
+ twoByteOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps, order, src1, src0,
+ dst);
+ }
+ void vcmpps_mr(uint8_t order, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ MOZ_ASSERT_IF(!useVEX_,
+ order < uint8_t(X86Encoding::ConditionCmp_AVX_Enabled));
+ twoByteOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps, order, offset, base,
+ src0, dst);
+ }
+ void vcmpps_mr(uint8_t order, const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT_IF(!useVEX_,
+ order < uint8_t(X86Encoding::ConditionCmp_AVX_Enabled));
+ twoByteOpImmSimd("vcmpps", VEX_PS, OP2_CMPPS_VpsWps, order, address, src0,
+ dst);
+ }
+
+ static constexpr size_t CMPPS_MR_PATCH_OFFSET = 1;
+
+ size_t vcmpeqps_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmpps_mr(X86Encoding::ConditionCmp_EQ, address, src0, dst);
+ return CMPPS_MR_PATCH_OFFSET;
+ }
+ size_t vcmpneqps_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmpps_mr(X86Encoding::ConditionCmp_NEQ, address, src0, dst);
+ return CMPPS_MR_PATCH_OFFSET;
+ }
+ size_t vcmpltps_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmpps_mr(X86Encoding::ConditionCmp_LT, address, src0, dst);
+ return CMPPS_MR_PATCH_OFFSET;
+ }
+ size_t vcmpleps_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmpps_mr(X86Encoding::ConditionCmp_LE, address, src0, dst);
+ return CMPPS_MR_PATCH_OFFSET;
+ }
+ size_t vcmpgeps_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmpps_mr(X86Encoding::ConditionCmp_GE, address, src0, dst);
+ return CMPPS_MR_PATCH_OFFSET;
+ }
+
+ void vcmppd_rr(uint8_t order, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd, order, src1, src0,
+ dst);
+ }
+ void vcmppd_mr(uint8_t order, const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpImmSimd("vcmppd", VEX_PD, OP2_CMPPD_VpdWpd, order, address, src0,
+ dst);
+ }
+
+ static constexpr size_t CMPPD_MR_PATCH_OFFSET = 1;
+
+ size_t vcmpeqpd_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmppd_mr(X86Encoding::ConditionCmp_EQ, address, src0, dst);
+ return CMPPD_MR_PATCH_OFFSET;
+ }
+ size_t vcmpneqpd_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmppd_mr(X86Encoding::ConditionCmp_NEQ, address, src0, dst);
+ return CMPPD_MR_PATCH_OFFSET;
+ }
+ size_t vcmpltpd_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmppd_mr(X86Encoding::ConditionCmp_LT, address, src0, dst);
+ return CMPPD_MR_PATCH_OFFSET;
+ }
+ size_t vcmplepd_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vcmppd_mr(X86Encoding::ConditionCmp_LE, address, src0, dst);
+ return CMPPD_MR_PATCH_OFFSET;
+ }
+
+ void vrcpps_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vrcpps", VEX_PS, OP2_RCPPS_VpsWps, src, invalid_xmm, dst);
+ }
+ void vrcpps_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vrcpps", VEX_PS, OP2_RCPPS_VpsWps, offset, base, invalid_xmm,
+ dst);
+ }
+ void vrcpps_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vrcpps", VEX_PS, OP2_RCPPS_VpsWps, address, invalid_xmm,
+ dst);
+ }
+
+ void vrsqrtps_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vrsqrtps", VEX_PS, OP2_RSQRTPS_VpsWps, src, invalid_xmm,
+ dst);
+ }
+ void vrsqrtps_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vrsqrtps", VEX_PS, OP2_RSQRTPS_VpsWps, offset, base,
+ invalid_xmm, dst);
+ }
+ void vrsqrtps_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vrsqrtps", VEX_PS, OP2_RSQRTPS_VpsWps, address, invalid_xmm,
+ dst);
+ }
+
+ void vsqrtps_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vsqrtps", VEX_PS, OP2_SQRTPS_VpsWps, src, invalid_xmm, dst);
+ }
+ void vsqrtps_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vsqrtps", VEX_PS, OP2_SQRTPS_VpsWps, offset, base,
+ invalid_xmm, dst);
+ }
+ void vsqrtps_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vsqrtps", VEX_PS, OP2_SQRTPS_VpsWps, address, invalid_xmm,
+ dst);
+ }
+ void vsqrtpd_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vsqrtpd", VEX_PD, OP2_SQRTPD_VpdWpd, src, invalid_xmm, dst);
+ }
+
+ void vaddsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vaddsd", VEX_SD, OP2_ADDSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vaddss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vaddss", VEX_SS, OP2_ADDSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vaddsd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vaddsd", VEX_SD, OP2_ADDSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vaddss_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vaddss", VEX_SS, OP2_ADDSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vaddsd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vaddsd", VEX_SD, OP2_ADDSD_VsdWsd, address, src0, dst);
+ }
+ void vaddss_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vaddss", VEX_SS, OP2_ADDSD_VsdWsd, address, src0, dst);
+ }
+
+ void vcvtss2sd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vcvtss2sd", VEX_SS, OP2_CVTSS2SD_VsdEd, src1, src0, dst);
+ }
+
+ void vcvtsd2ss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vcvtsd2ss", VEX_SD, OP2_CVTSD2SS_VsdEd, src1, src0, dst);
+ }
+
+ void vcvtsi2ss_rr(RegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpInt32Simd("vcvtsi2ss", VEX_SS, OP2_CVTSI2SD_VsdEd, src1, src0,
+ dst);
+ }
+
+ void vcvtsi2sd_rr(RegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpInt32Simd("vcvtsi2sd", VEX_SD, OP2_CVTSI2SD_VsdEd, src1, src0,
+ dst);
+ }
+
+ void vcvttps2dq_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vcvttps2dq", VEX_SS, OP2_CVTTPS2DQ_VdqWps, src, invalid_xmm,
+ dst);
+ }
+
+ void vcvttpd2dq_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vcvttpd2dq", VEX_PD, OP2_CVTTPD2DQ_VdqWpd, src, invalid_xmm,
+ dst);
+ }
+
+ void vcvtdq2ps_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vcvtdq2ps", VEX_PS, OP2_CVTDQ2PS_VpsWdq, src, invalid_xmm,
+ dst);
+ }
+
+ void vcvtdq2pd_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vcvtdq2pd", VEX_SS, OP2_CVTDQ2PD_VpdWdq, src, invalid_xmm,
+ dst);
+ }
+
+ void vcvtpd2ps_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vcvtpd2ps", VEX_PD, OP2_CVTPD2PS_VpsWpd, src, invalid_xmm,
+ dst);
+ }
+
+ void vcvtps2pd_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vcvtps2pd", VEX_PS, OP2_CVTPS2PD_VpdWps, src, invalid_xmm,
+ dst);
+ }
+
+ void vcvtsi2sd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vcvtsi2sd", VEX_SD, OP2_CVTSI2SD_VsdEd, offset, base, src0,
+ dst);
+ }
+
+ void vcvtsi2sd_mr(int32_t offset, RegisterID base, RegisterID index,
+ int scale, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vcvtsi2sd", VEX_SD, OP2_CVTSI2SD_VsdEd, offset, base, index,
+ scale, src0, dst);
+ }
+
+ void vcvtsi2ss_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vcvtsi2ss", VEX_SS, OP2_CVTSI2SD_VsdEd, offset, base, src0,
+ dst);
+ }
+
+ void vcvtsi2ss_mr(int32_t offset, RegisterID base, RegisterID index,
+ int scale, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vcvtsi2ss", VEX_SS, OP2_CVTSI2SD_VsdEd, offset, base, index,
+ scale, src0, dst);
+ }
+
+ void vcvttsd2si_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt32("vcvttsd2si", VEX_SD, OP2_CVTTSD2SI_GdWsd, src, dst);
+ }
+
+ void vcvttss2si_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt32("vcvttss2si", VEX_SS, OP2_CVTTSD2SI_GdWsd, src, dst);
+ }
+
+ void vunpcklps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vunpcklps", VEX_PS, OP2_UNPCKLPS_VsdWsd, src1, src0, dst);
+ }
+ void vunpcklps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vunpcklps", VEX_PS, OP2_UNPCKLPS_VsdWsd, offset, base, src0,
+ dst);
+ }
+ void vunpcklps_mr(const void* addr, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vunpcklps", VEX_PS, OP2_UNPCKLPS_VsdWsd, addr, src0, dst);
+ }
+
+ void vunpckhps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vunpckhps", VEX_PS, OP2_UNPCKHPS_VsdWsd, src1, src0, dst);
+ }
+ void vunpckhps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vunpckhps", VEX_PS, OP2_UNPCKHPS_VsdWsd, offset, base, src0,
+ dst);
+ }
+ void vunpckhps_mr(const void* addr, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vunpckhps", VEX_PS, OP2_UNPCKHPS_VsdWsd, addr, src0, dst);
+ }
+
+ void vpand_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpand", VEX_PD, OP2_PANDDQ_VdqWdq, src1, src0, dst);
+ }
+ void vpand_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpand", VEX_PD, OP2_PANDDQ_VdqWdq, offset, base, src0, dst);
+ }
+ void vpand_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpand", VEX_PD, OP2_PANDDQ_VdqWdq, address, src0, dst);
+ }
+ void vpor_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpor", VEX_PD, OP2_PORDQ_VdqWdq, src1, src0, dst);
+ }
+ void vpor_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpor", VEX_PD, OP2_PORDQ_VdqWdq, offset, base, src0, dst);
+ }
+ void vpor_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpor", VEX_PD, OP2_PORDQ_VdqWdq, address, src0, dst);
+ }
+ void vpxor_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpxor", VEX_PD, OP2_PXORDQ_VdqWdq, src1, src0, dst);
+ }
+ void vpxor_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpxor", VEX_PD, OP2_PXORDQ_VdqWdq, offset, base, src0, dst);
+ }
+ void vpxor_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpxor", VEX_PD, OP2_PXORDQ_VdqWdq, address, src0, dst);
+ }
+ void vpandn_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpandn", VEX_PD, OP2_PANDNDQ_VdqWdq, src1, src0, dst);
+ }
+ void vpandn_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpandn", VEX_PD, OP2_PANDNDQ_VdqWdq, offset, base, src0,
+ dst);
+ }
+ void vpandn_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpandn", VEX_PD, OP2_PANDNDQ_VdqWdq, address, src0, dst);
+ }
+ void vptest_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vptest", VEX_PD, OP3_PTEST_VdVd, ESCAPE_38, address, src0,
+ dst);
+ }
+
+ void vpshufd_irr(uint32_t mask, XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpImmSimd("vpshufd", VEX_PD, OP2_PSHUFD_VdqWdqIb, mask, src,
+ invalid_xmm, dst);
+ }
+ void vpshufd_imr(uint32_t mask, int32_t offset, RegisterID base,
+ XMMRegisterID dst) {
+ twoByteOpImmSimd("vpshufd", VEX_PD, OP2_PSHUFD_VdqWdqIb, mask, offset, base,
+ invalid_xmm, dst);
+ }
+ void vpshufd_imr(uint32_t mask, const void* address, XMMRegisterID dst) {
+ twoByteOpImmSimd("vpshufd", VEX_PD, OP2_PSHUFD_VdqWdqIb, mask, address,
+ invalid_xmm, dst);
+ }
+
+ void vpshuflw_irr(uint32_t mask, XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpImmSimd("vpshuflw", VEX_SD, OP2_PSHUFLW_VdqWdqIb, mask, src,
+ invalid_xmm, dst);
+ }
+
+ void vpshufhw_irr(uint32_t mask, XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpImmSimd("vpshufhw", VEX_SS, OP2_PSHUFHW_VdqWdqIb, mask, src,
+ invalid_xmm, dst);
+ }
+
+ void vpshufb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpshufb", VEX_PD, OP3_PSHUFB_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpshufb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpshufb", VEX_PD, OP3_PSHUFB_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vshufps_irr(uint32_t mask, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpImmSimd("vshufps", VEX_PS, OP2_SHUFPS_VpsWpsIb, mask, src1, src0,
+ dst);
+ }
+ void vshufps_imr(uint32_t mask, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpImmSimd("vshufps", VEX_PS, OP2_SHUFPS_VpsWpsIb, mask, offset, base,
+ src0, dst);
+ }
+ void vshufps_imr(uint32_t mask, const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpImmSimd("vshufps", VEX_PS, OP2_SHUFPS_VpsWpsIb, mask, address,
+ src0, dst);
+ }
+ void vshufpd_irr(uint32_t mask, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpImmSimd("vshufpd", VEX_PD, OP2_SHUFPD_VpdWpdIb, mask, src1, src0,
+ dst);
+ }
+
+ void vmovddup_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vmovddup", VEX_SD, OP2_MOVDDUP_VqWq, src, invalid_xmm, dst);
+ }
+ void vmovddup_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovddup", VEX_SD, OP2_MOVDDUP_VqWq, offset, base,
+ invalid_xmm, dst);
+ }
+ void vmovddup_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ twoByteOpSimd("vmovddup", VEX_SD, OP2_MOVDDUP_VqWq, offset, base, index,
+ scale, invalid_xmm, dst);
+ }
+
+ void vmovhlps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmovhlps", VEX_PS, OP2_MOVHLPS_VqUq, src1, src0, dst);
+ }
+
+ void vmovlhps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmovlhps", VEX_PS, OP2_MOVLHPS_VqUq, src1, src0, dst);
+ }
+
+ void vpsrldq_ir(uint32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 16);
+ shiftOpImmSimd("vpsrldq", OP2_PSRLDQ_Vd, ShiftID::vpsrldq, count, src, dst);
+ }
+
+ void vpslldq_ir(uint32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 16);
+ shiftOpImmSimd("vpslldq", OP2_PSRLDQ_Vd, ShiftID::vpslldq, count, src, dst);
+ }
+
+ void vpsllq_ir(uint32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 64);
+ shiftOpImmSimd("vpsllq", OP2_PSRLDQ_Vd, ShiftID::vpsllx, count, src, dst);
+ }
+
+ void vpsllq_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsllq", VEX_PD, OP2_PSLLQ_VdqWdq, src1, src0, dst);
+ }
+
+ void vpsrlq_ir(uint32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 64);
+ shiftOpImmSimd("vpsrlq", OP2_PSRLDQ_Vd, ShiftID::vpsrlx, count, src, dst);
+ }
+
+ void vpsrlq_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsrlq", VEX_PD, OP2_PSRLQ_VdqWdq, src1, src0, dst);
+ }
+
+ void vpslld_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpslld", VEX_PD, OP2_PSLLD_VdqWdq, src1, src0, dst);
+ }
+
+ void vpslld_ir(uint32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 32);
+ shiftOpImmSimd("vpslld", OP2_PSLLD_UdqIb, ShiftID::vpsllx, count, src, dst);
+ }
+
+ void vpsrad_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsrad", VEX_PD, OP2_PSRAD_VdqWdq, src1, src0, dst);
+ }
+
+ void vpsrad_ir(int32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 32);
+ shiftOpImmSimd("vpsrad", OP2_PSRAD_UdqIb, ShiftID::vpsrad, count, src, dst);
+ }
+
+ void vpsrld_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsrld", VEX_PD, OP2_PSRLD_VdqWdq, src1, src0, dst);
+ }
+
+ void vpsrld_ir(uint32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 32);
+ shiftOpImmSimd("vpsrld", OP2_PSRLD_UdqIb, ShiftID::vpsrlx, count, src, dst);
+ }
+
+ void vpsllw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsllw", VEX_PD, OP2_PSLLW_VdqWdq, src1, src0, dst);
+ }
+
+ void vpsllw_ir(uint32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 16);
+ shiftOpImmSimd("vpsllw", OP2_PSLLW_UdqIb, ShiftID::vpsllx, count, src, dst);
+ }
+
+ void vpsraw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsraw", VEX_PD, OP2_PSRAW_VdqWdq, src1, src0, dst);
+ }
+
+ void vpsraw_ir(int32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 16);
+ shiftOpImmSimd("vpsraw", OP2_PSRAW_UdqIb, ShiftID::vpsrad, count, src, dst);
+ }
+
+ void vpsrlw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsrlw", VEX_PD, OP2_PSRLW_VdqWdq, src1, src0, dst);
+ }
+
+ void vpsrlw_ir(uint32_t count, XMMRegisterID src, XMMRegisterID dst) {
+ MOZ_ASSERT(count < 16);
+ shiftOpImmSimd("vpsrlw", OP2_PSRLW_UdqIb, ShiftID::vpsrlx, count, src, dst);
+ }
+
+ void vmovmskpd_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt32("vmovmskpd", VEX_PD, OP2_MOVMSKPD_EdVd, src, dst);
+ }
+
+ void vmovmskps_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt32("vmovmskps", VEX_PS, OP2_MOVMSKPD_EdVd, src, dst);
+ }
+
+ void vpmovmskb_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt32("vpmovmskb", VEX_PD, OP2_PMOVMSKB_EdVd, src, dst);
+ }
+
+ void vptest_rr(XMMRegisterID rhs, XMMRegisterID lhs) {
+ threeByteOpSimd("vptest", VEX_PD, OP3_PTEST_VdVd, ESCAPE_38, rhs,
+ invalid_xmm, lhs);
+ }
+
+ void vmovd_rr(XMMRegisterID src, RegisterID dst) {
+ twoByteOpSimdInt32("vmovd", VEX_PD, OP2_MOVD_EdVd, (XMMRegisterID)dst,
+ (RegisterID)src);
+ }
+
+ void vmovd_rr(RegisterID src, XMMRegisterID dst) {
+ twoByteOpInt32Simd("vmovd", VEX_PD, OP2_MOVD_VdEd, src, invalid_xmm, dst);
+ }
+
+ void vmovd_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovd", VEX_PD, OP2_MOVD_VdEd, offset, base, invalid_xmm,
+ dst);
+ }
+
+ void vmovd_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ twoByteOpSimd("vmovd", VEX_PD, OP2_MOVD_VdEd, offset, base, index, scale,
+ invalid_xmm, dst);
+ }
+
+ void vmovd_mr_disp32(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd_disp32("vmovd", VEX_PD, OP2_MOVD_VdEd, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vmovd_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vmovd", VEX_PD, OP2_MOVD_VdEd, address, invalid_xmm, dst);
+ }
+
+ void vmovd_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovd", VEX_PD, OP2_MOVD_EdVd, offset, base, invalid_xmm,
+ src);
+ }
+
+ void vmovd_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovd", VEX_PD, OP2_MOVD_EdVd, offset, base, index, scale,
+ invalid_xmm, src);
+ }
+
+ void vmovd_rm_disp32(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd_disp32("vmovd", VEX_PD, OP2_MOVD_EdVd, offset, base,
+ invalid_xmm, src);
+ }
+
+ void vmovd_rm(XMMRegisterID src, const void* address) {
+ twoByteOpSimd("vmovd", VEX_PD, OP2_MOVD_EdVd, address, invalid_xmm, src);
+ }
+
+ void vmovsd_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovsd", VEX_SD, OP2_MOVSD_WsdVsd, offset, base, invalid_xmm,
+ src);
+ }
+
+ void vmovsd_rm_disp32(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd_disp32("vmovsd", VEX_SD, OP2_MOVSD_WsdVsd, offset, base,
+ invalid_xmm, src);
+ }
+
+ void vmovss_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovss", VEX_SS, OP2_MOVSD_WsdVsd, offset, base, invalid_xmm,
+ src);
+ }
+
+ void vmovss_rm_disp32(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd_disp32("vmovss", VEX_SS, OP2_MOVSD_WsdVsd, offset, base,
+ invalid_xmm, src);
+ }
+
+ void vmovss_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovss", VEX_SS, OP2_MOVSD_VsdWsd, offset, base, invalid_xmm,
+ dst);
+ }
+
+ void vmovss_mr_disp32(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd_disp32("vmovss", VEX_SS, OP2_MOVSD_VsdWsd, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vmovsd_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovsd", VEX_SD, OP2_MOVSD_WsdVsd, offset, base, index,
+ scale, invalid_xmm, src);
+ }
+
+ void vmovss_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovss", VEX_SS, OP2_MOVSD_WsdVsd, offset, base, index,
+ scale, invalid_xmm, src);
+ }
+
+ void vmovss_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmovss", VEX_SS, OP2_MOVSD_VsdWsd, offset, base, index,
+ scale, invalid_xmm, dst);
+ }
+
+ void vmovsd_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovsd", VEX_SD, OP2_MOVSD_VsdWsd, offset, base, invalid_xmm,
+ dst);
+ }
+
+ void vmovsd_mr_disp32(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd_disp32("vmovsd", VEX_SD, OP2_MOVSD_VsdWsd, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vmovsd_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmovsd", VEX_SD, OP2_MOVSD_VsdWsd, offset, base, index,
+ scale, invalid_xmm, dst);
+ }
+
+ // Note that the register-to-register form of vmovsd does not write to the
+ // entire output register. For general-purpose register-to-register moves,
+ // use vmovapd instead.
+ void vmovsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmovsd", VEX_SD, OP2_MOVSD_VsdWsd, src1, src0, dst);
+ }
+
+ // The register-to-register form of vmovss has the same problem as vmovsd
+ // above. Prefer vmovaps for register-to-register moves.
+ void vmovss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmovss", VEX_SS, OP2_MOVSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vmovsd_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vmovsd", VEX_SD, OP2_MOVSD_VsdWsd, address, invalid_xmm,
+ dst);
+ }
+
+ void vmovss_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vmovss", VEX_SS, OP2_MOVSD_VsdWsd, address, invalid_xmm,
+ dst);
+ }
+
+ void vmovups_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vmovups", VEX_PS, OP2_MOVPS_VpsWps, address, invalid_xmm,
+ dst);
+ }
+
+ void vmovdqu_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vmovdqu", VEX_SS, OP2_MOVDQ_VdqWdq, address, invalid_xmm,
+ dst);
+ }
+
+ void vmovsd_rm(XMMRegisterID src, const void* address) {
+ twoByteOpSimd("vmovsd", VEX_SD, OP2_MOVSD_WsdVsd, address, invalid_xmm,
+ src);
+ }
+
+ void vmovss_rm(XMMRegisterID src, const void* address) {
+ twoByteOpSimd("vmovss", VEX_SS, OP2_MOVSD_WsdVsd, address, invalid_xmm,
+ src);
+ }
+
+ void vmovdqa_rm(XMMRegisterID src, const void* address) {
+ twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_WdqVdq, address, invalid_xmm,
+ src);
+ }
+
+ void vmovaps_rm(XMMRegisterID src, const void* address) {
+ twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_WsdVsd, address, invalid_xmm,
+ src);
+ }
+
+ void vmovdqu_rm(XMMRegisterID src, const void* address) {
+ twoByteOpSimd("vmovdqu", VEX_SS, OP2_MOVDQ_WdqVdq, address, invalid_xmm,
+ src);
+ }
+
+ void vmovups_rm(XMMRegisterID src, const void* address) {
+ twoByteOpSimd("vmovups", VEX_PS, OP2_MOVPS_WpsVps, address, invalid_xmm,
+ src);
+ }
+
+ void vmovaps_rr(XMMRegisterID src, XMMRegisterID dst) {
+#ifdef JS_CODEGEN_X64
+ // There are two opcodes that can encode this instruction. If we have
+ // one register in [xmm8,xmm15] and one in [xmm0,xmm7], use the
+ // opcode which swaps the operands, as that way we can get a two-byte
+ // VEX in that case.
+ if (src >= xmm8 && dst < xmm8) {
+ twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_WsdVsd, dst, invalid_xmm,
+ src);
+ return;
+ }
+#endif
+ twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_VsdWsd, src, invalid_xmm, dst);
+ }
+ void vmovaps_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_WsdVsd, offset, base,
+ invalid_xmm, src);
+ }
+ void vmovaps_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_WsdVsd, offset, base, index,
+ scale, invalid_xmm, src);
+ }
+ void vmovaps_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_VsdWsd, offset, base,
+ invalid_xmm, dst);
+ }
+ void vmovaps_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_VsdWsd, offset, base, index,
+ scale, invalid_xmm, dst);
+ }
+
+ void vmovups_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovups", VEX_PS, OP2_MOVPS_WpsVps, offset, base,
+ invalid_xmm, src);
+ }
+ void vmovups_rm_disp32(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd_disp32("vmovups", VEX_PS, OP2_MOVPS_WpsVps, offset, base,
+ invalid_xmm, src);
+ }
+ void vmovups_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovups", VEX_PS, OP2_MOVPS_WpsVps, offset, base, index,
+ scale, invalid_xmm, src);
+ }
+ void vmovups_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovups", VEX_PS, OP2_MOVPS_VpsWps, offset, base,
+ invalid_xmm, dst);
+ }
+ void vmovups_mr_disp32(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd_disp32("vmovups", VEX_PS, OP2_MOVPS_VpsWps, offset, base,
+ invalid_xmm, dst);
+ }
+ void vmovups_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmovups", VEX_PS, OP2_MOVPS_VpsWps, offset, base, index,
+ scale, invalid_xmm, dst);
+ }
+
+ void vmovapd_rr(XMMRegisterID src, XMMRegisterID dst) {
+#ifdef JS_CODEGEN_X64
+ // There are two opcodes that can encode this instruction. If we have
+ // one register in [xmm8,xmm15] and one in [xmm0,xmm7], use the
+ // opcode which swaps the operands, as that way we can get a two-byte
+ // VEX in that case.
+ if (src >= xmm8 && dst < xmm8) {
+ twoByteOpSimd("vmovapd", VEX_PD, OP2_MOVAPS_WsdVsd, dst, invalid_xmm,
+ src);
+ return;
+ }
+#endif
+ twoByteOpSimd("vmovapd", VEX_PD, OP2_MOVAPD_VsdWsd, src, invalid_xmm, dst);
+ }
+
+ void vmovdqu_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovdqu", VEX_SS, OP2_MOVDQ_WdqVdq, offset, base,
+ invalid_xmm, src);
+ }
+
+ void vmovdqu_rm_disp32(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd_disp32("vmovdqu", VEX_SS, OP2_MOVDQ_WdqVdq, offset, base,
+ invalid_xmm, src);
+ }
+
+ void vmovdqu_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovdqu", VEX_SS, OP2_MOVDQ_WdqVdq, offset, base, index,
+ scale, invalid_xmm, src);
+ }
+
+ void vmovdqu_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovdqu", VEX_SS, OP2_MOVDQ_VdqWdq, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vmovdqu_mr_disp32(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd_disp32("vmovdqu", VEX_SS, OP2_MOVDQ_VdqWdq, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vmovdqu_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmovdqu", VEX_SS, OP2_MOVDQ_VdqWdq, offset, base, index,
+ scale, invalid_xmm, dst);
+ }
+
+ void vmovdqa_rr(XMMRegisterID src, XMMRegisterID dst) {
+#ifdef JS_CODEGEN_X64
+ // There are two opcodes that can encode this instruction. If we have
+ // one register in [xmm8,xmm15] and one in [xmm0,xmm7], use the
+ // opcode which swaps the operands, as that way we can get a two-byte
+ // VEX in that case.
+ if (src >= xmm8 && dst < xmm8) {
+ twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_WdqVdq, dst, invalid_xmm, src);
+ return;
+ }
+#endif
+ twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_VdqWdq, src, invalid_xmm, dst);
+ }
+
+ void vmovdqa_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_WdqVdq, offset, base,
+ invalid_xmm, src);
+ }
+
+ void vmovdqa_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_WdqVdq, offset, base, index,
+ scale, invalid_xmm, src);
+ }
+
+ void vmovdqa_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_VdqWdq, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vmovdqa_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_VdqWdq, offset, base, index,
+ scale, invalid_xmm, dst);
+ }
+
+ void vmulsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmulsd", VEX_SD, OP2_MULSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vmulss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmulss", VEX_SS, OP2_MULSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vmulsd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmulsd", VEX_SD, OP2_MULSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vmulss_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmulss", VEX_SS, OP2_MULSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vpinsrw_irr(uint32_t whichWord, RegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(whichWord < 8);
+ twoByteOpImmInt32Simd("vpinsrw", VEX_PD, OP2_PINSRW, whichWord, src1, src0,
+ dst);
+ }
+ void vpinsrw_imr(unsigned lane, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ MOZ_ASSERT(lane < 16);
+ twoByteOpImmInt32Simd("vpinsrw", VEX_PD, OP2_PINSRW, lane, offset, base,
+ src0, dst);
+ }
+ void vpinsrw_imr(unsigned lane, int32_t offset, RegisterID base,
+ RegisterID index, int32_t scale, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(lane < 16);
+ twoByteOpImmInt32Simd("vpinsrw", VEX_PD, OP2_PINSRW, lane, offset, base,
+ index, scale, src0, dst);
+ }
+
+ void vpextrw_irr(uint32_t whichWord, XMMRegisterID src, RegisterID dst) {
+ MOZ_ASSERT(whichWord < 8);
+ twoByteOpImmSimdInt32("vpextrw", VEX_PD, OP2_PEXTRW_GdUdIb, whichWord, src,
+ dst);
+ }
+
+ void vpextrw_irm(unsigned lane, XMMRegisterID src, int32_t offset,
+ RegisterID base) {
+ MOZ_ASSERT(lane < 8);
+ threeByteOpImmSimdInt32("vpextrw", VEX_PD, OP3_PEXTRW_EwVdqIb, ESCAPE_3A,
+ lane, offset, base, (RegisterID)src);
+ }
+
+ void vpextrw_irm(unsigned lane, XMMRegisterID src, int32_t offset,
+ RegisterID base, RegisterID index, int scale) {
+ MOZ_ASSERT(lane < 8);
+ threeByteOpImmSimdInt32("vpextrw", VEX_PD, OP3_PEXTRW_EwVdqIb, ESCAPE_3A,
+ lane, offset, base, index, scale, (RegisterID)src);
+ }
+
+ void vsubsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsubsd", VEX_SD, OP2_SUBSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vsubss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsubss", VEX_SS, OP2_SUBSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vsubsd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vsubsd", VEX_SD, OP2_SUBSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vsubss_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vsubss", VEX_SS, OP2_SUBSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vucomiss_rr(XMMRegisterID rhs, XMMRegisterID lhs) {
+ twoByteOpSimdFlags("vucomiss", VEX_PS, OP2_UCOMISD_VsdWsd, rhs, lhs);
+ }
+
+ void vucomisd_rr(XMMRegisterID rhs, XMMRegisterID lhs) {
+ twoByteOpSimdFlags("vucomisd", VEX_PD, OP2_UCOMISD_VsdWsd, rhs, lhs);
+ }
+
+ void vucomisd_mr(int32_t offset, RegisterID base, XMMRegisterID lhs) {
+ twoByteOpSimdFlags("vucomisd", VEX_PD, OP2_UCOMISD_VsdWsd, offset, base,
+ lhs);
+ }
+
+ void vdivsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vdivsd", VEX_SD, OP2_DIVSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vdivss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vdivss", VEX_SS, OP2_DIVSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vdivsd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vdivsd", VEX_SD, OP2_DIVSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vdivss_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vdivss", VEX_SS, OP2_DIVSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vxorpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vxorpd", VEX_PD, OP2_XORPD_VpdWpd, src1, src0, dst);
+ }
+
+ void vorpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vorpd", VEX_PD, OP2_ORPD_VpdWpd, src1, src0, dst);
+ }
+
+ void vandpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vandpd", VEX_PD, OP2_ANDPD_VpdWpd, src1, src0, dst);
+ }
+ void vandpd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vandpd", VEX_PD, OP2_ANDPD_VpdWpd, address, src0, dst);
+ }
+
+ void vandps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vandps", VEX_PS, OP2_ANDPS_VpsWps, src1, src0, dst);
+ }
+
+ void vandps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vandps", VEX_PS, OP2_ANDPS_VpsWps, offset, base, src0, dst);
+ }
+
+ void vandps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vandps", VEX_PS, OP2_ANDPS_VpsWps, address, src0, dst);
+ }
+
+ void vandnps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vandnps", VEX_PS, OP2_ANDNPS_VpsWps, src1, src0, dst);
+ }
+
+ void vandnps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vandnps", VEX_PS, OP2_ANDNPS_VpsWps, offset, base, src0,
+ dst);
+ }
+
+ void vandnps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vandnps", VEX_PS, OP2_ANDNPS_VpsWps, address, src0, dst);
+ }
+
+ void vorps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vorps", VEX_PS, OP2_ORPS_VpsWps, src1, src0, dst);
+ }
+
+ void vorps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vorps", VEX_PS, OP2_ORPS_VpsWps, offset, base, src0, dst);
+ }
+
+ void vorps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vorps", VEX_PS, OP2_ORPS_VpsWps, address, src0, dst);
+ }
+
+ void vxorps_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vxorps", VEX_PS, OP2_XORPS_VpsWps, src1, src0, dst);
+ }
+
+ void vxorps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vxorps", VEX_PS, OP2_XORPS_VpsWps, offset, base, src0, dst);
+ }
+
+ void vxorps_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vxorps", VEX_PS, OP2_XORPS_VpsWps, address, src0, dst);
+ }
+
+ void vsqrtsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsqrtsd", VEX_SD, OP2_SQRTSD_VsdWsd, src1, src0, dst);
+ }
+
+ void vsqrtss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsqrtss", VEX_SS, OP2_SQRTSS_VssWss, src1, src0, dst);
+ }
+
+ void vroundsd_irr(RoundingMode mode, XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpImmSimd("vroundsd", VEX_PD, OP3_ROUNDSD_VsdWsd, ESCAPE_3A, mode,
+ src, invalid_xmm, dst);
+ }
+
+ void vroundss_irr(RoundingMode mode, XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpImmSimd("vroundss", VEX_PD, OP3_ROUNDSS_VsdWsd, ESCAPE_3A, mode,
+ src, invalid_xmm, dst);
+ }
+ void vroundps_irr(SSERoundingMode mode, XMMRegisterID src,
+ XMMRegisterID dst) {
+ threeByteOpImmSimd("vroundps", VEX_PD, OP3_ROUNDPS_VpsWps, ESCAPE_3A,
+ int(mode), src, invalid_xmm, dst);
+ }
+ void vroundpd_irr(SSERoundingMode mode, XMMRegisterID src,
+ XMMRegisterID dst) {
+ threeByteOpImmSimd("vroundpd", VEX_PD, OP3_ROUNDPD_VpdWpd, ESCAPE_3A,
+ int(mode), src, invalid_xmm, dst);
+ }
+
+ void vinsertps_irr(uint32_t mask, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ threeByteOpImmSimd("vinsertps", VEX_PD, OP3_INSERTPS_VpsUps, ESCAPE_3A,
+ mask, src1, src0, dst);
+ }
+ void vinsertps_imr(uint32_t mask, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpImmSimd("vinsertps", VEX_PD, OP3_INSERTPS_VpsUps, ESCAPE_3A,
+ mask, offset, base, src0, dst);
+ }
+ void vinsertps_imr(uint32_t mask, int32_t offset, RegisterID base,
+ RegisterID index, int scale, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ threeByteOpImmSimd("vinsertps", VEX_PD, OP3_INSERTPS_VpsUps, ESCAPE_3A,
+ mask, offset, base, index, scale, src0, dst);
+ }
+
+ void vmovlps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmovlps", VEX_PS, OP2_MOVLPS_VqEq, offset, base, src0, dst);
+ }
+ void vmovlps_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmovlps", VEX_PS, OP2_MOVLPS_VqEq, offset, base, index,
+ scale, src0, dst);
+ }
+ void vmovlps_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovlps", VEX_PS, OP2_MOVLPS_EqVq, offset, base, invalid_xmm,
+ src);
+ }
+ void vmovlps_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovlps", VEX_PS, OP2_MOVLPS_EqVq, offset, base, index,
+ scale, invalid_xmm, src);
+ }
+
+ void vmovhps_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmovhps", VEX_PS, OP2_MOVHPS_VqEq, offset, base, src0, dst);
+ }
+ void vmovhps_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmovhps", VEX_PS, OP2_MOVHPS_VqEq, offset, base, index,
+ scale, src0, dst);
+ }
+
+ void vmovhps_rm(XMMRegisterID src, int32_t offset, RegisterID base) {
+ twoByteOpSimd("vmovhps", VEX_PS, OP2_MOVHPS_EqVq, offset, base, invalid_xmm,
+ src);
+ }
+ void vmovhps_rm(XMMRegisterID src, int32_t offset, RegisterID base,
+ RegisterID index, int scale) {
+ twoByteOpSimd("vmovhps", VEX_PS, OP2_MOVHPS_EqVq, offset, base, index,
+ scale, invalid_xmm, src);
+ }
+
+ void vextractps_rm(unsigned lane, XMMRegisterID src, int32_t offset,
+ RegisterID base) {
+ threeByteOpImmSimd("vextractps", VEX_PD, OP3_EXTRACTPS_EdVdqIb, ESCAPE_3A,
+ lane, offset, base, invalid_xmm, src);
+ }
+ void vextractps_rm(unsigned lane, XMMRegisterID src, int32_t offset,
+ RegisterID base, RegisterID index, int scale) {
+ threeByteOpImmSimd("vextractps", VEX_PD, OP3_EXTRACTPS_EdVdqIb, ESCAPE_3A,
+ lane, offset, base, index, scale, invalid_xmm, src);
+ }
+
+ void vpblendw_irr(unsigned mask, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(mask < 256);
+ threeByteOpImmSimd("vpblendw", VEX_PD, OP3_PBLENDW_VdqWdqIb, ESCAPE_3A,
+ mask, src1, src0, dst);
+ }
+
+ void vpblendvb_rr(XMMRegisterID mask, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vblendvOpSimd("vpblendvb", OP3_PBLENDVB_VdqWdq, OP3_VPBLENDVB_VdqWdq, mask,
+ src1, src0, dst);
+ }
+
+ void vpinsrb_irr(unsigned lane, RegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(lane < 16);
+ threeByteOpImmInt32Simd("vpinsrb", VEX_PD, OP3_PINSRB_VdqEvIb, ESCAPE_3A,
+ lane, src1, src0, dst);
+ }
+ void vpinsrb_imr(unsigned lane, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ MOZ_ASSERT(lane < 16);
+ threeByteOpImmInt32Simd("vpinsrb", VEX_PD, OP3_PINSRB_VdqEvIb, ESCAPE_3A,
+ lane, offset, base, src0, dst);
+ }
+ void vpinsrb_imr(unsigned lane, int32_t offset, RegisterID base,
+ RegisterID index, int32_t scale, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(lane < 16);
+ threeByteOpImmInt32Simd("vpinsrb", VEX_PD, OP3_PINSRB_VdqEvIb, ESCAPE_3A,
+ lane, offset, base, index, scale, src0, dst);
+ }
+
+ void vpinsrd_irr(unsigned lane, RegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(lane < 4);
+ threeByteOpImmInt32Simd("vpinsrd", VEX_PD, OP3_PINSRD_VdqEvIb, ESCAPE_3A,
+ lane, src1, src0, dst);
+ }
+
+ void vpextrb_irr(unsigned lane, XMMRegisterID src, RegisterID dst) {
+ MOZ_ASSERT(lane < 16);
+ threeByteOpImmSimdInt32("vpextrb", VEX_PD, OP3_PEXTRB_EvVdqIb, ESCAPE_3A,
+ lane, (XMMRegisterID)dst, (RegisterID)src);
+ }
+
+ void vpextrb_irm(unsigned lane, XMMRegisterID src, int32_t offset,
+ RegisterID base) {
+ MOZ_ASSERT(lane < 16);
+ threeByteOpImmSimdInt32("vpextrb", VEX_PD, OP3_PEXTRB_EvVdqIb, ESCAPE_3A,
+ lane, offset, base, (RegisterID)src);
+ }
+
+ void vpextrb_irm(unsigned lane, XMMRegisterID src, int32_t offset,
+ RegisterID base, RegisterID index, int scale) {
+ MOZ_ASSERT(lane < 16);
+ threeByteOpImmSimdInt32("vpextrb", VEX_PD, OP3_PEXTRB_EvVdqIb, ESCAPE_3A,
+ lane, offset, base, index, scale, (RegisterID)src);
+ }
+
+ void vpextrd_irr(unsigned lane, XMMRegisterID src, RegisterID dst) {
+ MOZ_ASSERT(lane < 4);
+ threeByteOpImmSimdInt32("vpextrd", VEX_PD, OP3_PEXTRD_EvVdqIb, ESCAPE_3A,
+ lane, (XMMRegisterID)dst, (RegisterID)src);
+ }
+
+ void vblendps_irr(unsigned imm, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(imm < 16);
+ // Despite being a "ps" instruction, vblendps is encoded with the "pd"
+ // prefix.
+ threeByteOpImmSimd("vblendps", VEX_PD, OP3_BLENDPS_VpsWpsIb, ESCAPE_3A, imm,
+ src1, src0, dst);
+ }
+
+ void vblendps_imr(unsigned imm, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ MOZ_ASSERT(imm < 16);
+ // Despite being a "ps" instruction, vblendps is encoded with the "pd"
+ // prefix.
+ threeByteOpImmSimd("vblendps", VEX_PD, OP3_BLENDPS_VpsWpsIb, ESCAPE_3A, imm,
+ offset, base, src0, dst);
+ }
+
+ void vblendvps_rr(XMMRegisterID mask, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vblendvOpSimd("vblendvps", OP3_BLENDVPS_VdqWdq, OP3_VBLENDVPS_VdqWdq, mask,
+ src1, src0, dst);
+ }
+ void vblendvps_mr(XMMRegisterID mask, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ vblendvOpSimd("vblendvps", OP3_BLENDVPS_VdqWdq, OP3_VBLENDVPS_VdqWdq, mask,
+ offset, base, src0, dst);
+ }
+ void vblendvpd_rr(XMMRegisterID mask, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ vblendvOpSimd("vblendvpd", OP3_BLENDVPD_VdqWdq, OP3_VBLENDVPD_VdqWdq, mask,
+ src1, src0, dst);
+ }
+
+ void vmovsldup_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vmovsldup", VEX_SS, OP2_MOVSLDUP_VpsWps, src, invalid_xmm,
+ dst);
+ }
+ void vmovsldup_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovsldup", VEX_SS, OP2_MOVSLDUP_VpsWps, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vmovshdup_rr(XMMRegisterID src, XMMRegisterID dst) {
+ twoByteOpSimd("vmovshdup", VEX_SS, OP2_MOVSHDUP_VpsWps, src, invalid_xmm,
+ dst);
+ }
+ void vmovshdup_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ twoByteOpSimd("vmovshdup", VEX_SS, OP2_MOVSHDUP_VpsWps, offset, base,
+ invalid_xmm, dst);
+ }
+
+ void vminsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vminsd", VEX_SD, OP2_MINSD_VsdWsd, src1, src0, dst);
+ }
+ void vminsd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vminsd", VEX_SD, OP2_MINSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vminss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vminss", VEX_SS, OP2_MINSS_VssWss, src1, src0, dst);
+ }
+
+ void vmaxsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmaxsd", VEX_SD, OP2_MAXSD_VsdWsd, src1, src0, dst);
+ }
+ void vmaxsd_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vmaxsd", VEX_SD, OP2_MAXSD_VsdWsd, offset, base, src0, dst);
+ }
+
+ void vmaxss_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vmaxss", VEX_SS, OP2_MAXSS_VssWss, src1, src0, dst);
+ }
+
+ void vpavgb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpavgb", VEX_PD, OP2_PAVGB_VdqWdq, src1, src0, dst);
+ }
+
+ void vpavgw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpavgw", VEX_PD, OP2_PAVGW_VdqWdq, src1, src0, dst);
+ }
+
+ void vpminsb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpminsb", VEX_PD, OP3_PMINSB_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpminsb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpminsb", VEX_PD, OP3_PMINSB_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vpmaxsb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmaxsb", VEX_PD, OP3_PMAXSB_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpmaxsb_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmaxsb", VEX_PD, OP3_PMAXSB_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vpminub_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpminub", VEX_PD, OP2_PMINUB_VdqWdq, src1, src0, dst);
+ }
+ void vpminub_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpminub", VEX_PD, OP2_PMINUB_VdqWdq, address, src0, dst);
+ }
+
+ void vpmaxub_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmaxub", VEX_PD, OP2_PMAXUB_VdqWdq, src1, src0, dst);
+ }
+ void vpmaxub_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmaxub", VEX_PD, OP2_PMAXUB_VdqWdq, address, src0, dst);
+ }
+
+ void vpminsw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpminsw", VEX_PD, OP2_PMINSW_VdqWdq, src1, src0, dst);
+ }
+ void vpminsw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpminsw", VEX_PD, OP2_PMINSW_VdqWdq, address, src0, dst);
+ }
+
+ void vpmaxsw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmaxsw", VEX_PD, OP2_PMAXSW_VdqWdq, src1, src0, dst);
+ }
+ void vpmaxsw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpmaxsw", VEX_PD, OP2_PMAXSW_VdqWdq, address, src0, dst);
+ }
+
+ void vpminuw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpminuw", VEX_PD, OP3_PMINUW_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpminuw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpminuw", VEX_PD, OP3_PMINUW_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vpmaxuw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmaxuw", VEX_PD, OP3_PMAXUW_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpmaxuw_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmaxuw", VEX_PD, OP3_PMAXUW_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vpminsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpminsd", VEX_PD, OP3_PMINSD_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpminsd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpminsd", VEX_PD, OP3_PMINSD_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vpmaxsd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmaxsd", VEX_PD, OP3_PMAXSD_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpmaxsd_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmaxsd", VEX_PD, OP3_PMAXSD_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vpminud_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpminud", VEX_PD, OP3_PMINUD_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpminud_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpminud", VEX_PD, OP3_PMINUD_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vpmaxud_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmaxud", VEX_PD, OP3_PMAXUD_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+ void vpmaxud_mr(const void* address, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpmaxud", VEX_PD, OP3_PMAXUD_VdqWdq, ESCAPE_38, address,
+ src0, dst);
+ }
+
+ void vpacksswb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpacksswb", VEX_PD, OP2_PACKSSWB_VdqWdq, src1, src0, dst);
+ }
+ void vpacksswb_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpacksswb", VEX_PD, OP2_PACKSSWB_VdqWdq, address, src0, dst);
+ }
+
+ void vpackuswb_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpackuswb", VEX_PD, OP2_PACKUSWB_VdqWdq, src1, src0, dst);
+ }
+ void vpackuswb_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpackuswb", VEX_PD, OP2_PACKUSWB_VdqWdq, address, src0, dst);
+ }
+
+ void vpackssdw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpackssdw", VEX_PD, OP2_PACKSSDW_VdqWdq, src1, src0, dst);
+ }
+ void vpackssdw_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpackssdw", VEX_PD, OP2_PACKSSDW_VdqWdq, address, src0, dst);
+ }
+
+ void vpackusdw_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vpackusdw", VEX_PD, OP3_PACKUSDW_VdqWdq, ESCAPE_38, src1,
+ src0, dst);
+ }
+ void vpackusdw_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ threeByteOpSimd("vpackusdw", VEX_PD, OP3_PACKUSDW_VdqWdq, ESCAPE_38,
+ address, src0, dst);
+ }
+
+ void vpabsb_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpabsb", VEX_PD, OP3_PABSB_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+
+ void vpabsw_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpabsw", VEX_PD, OP3_PABSW_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+
+ void vpabsd_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpabsd", VEX_PD, OP3_PABSD_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+
+ void vpmovsxbw_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxbw", VEX_PD, OP3_PMOVSXBW_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vpmovsxbw_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxbw", VEX_PD, OP3_PMOVSXBW_VdqWdq, ESCAPE_38, offset,
+ base, invalid_xmm, dst);
+ }
+ void vpmovsxbw_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxbw", VEX_PD, OP3_PMOVSXBW_VdqWdq, ESCAPE_38, offset,
+ base, index, scale, invalid_xmm, dst);
+ }
+
+ void vpmovzxbw_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxbw", VEX_PD, OP3_PMOVZXBW_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vpmovzxbw_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxbw", VEX_PD, OP3_PMOVZXBW_VdqWdq, ESCAPE_38, offset,
+ base, invalid_xmm, dst);
+ }
+ void vpmovzxbw_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxbw", VEX_PD, OP3_PMOVZXBW_VdqWdq, ESCAPE_38, offset,
+ base, index, scale, invalid_xmm, dst);
+ }
+
+ void vpmovsxwd_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxwd", VEX_PD, OP3_PMOVSXWD_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vpmovsxwd_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxwd", VEX_PD, OP3_PMOVSXWD_VdqWdq, ESCAPE_38, offset,
+ base, invalid_xmm, dst);
+ }
+ void vpmovsxwd_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxwd", VEX_PD, OP3_PMOVSXWD_VdqWdq, ESCAPE_38, offset,
+ base, index, scale, invalid_xmm, dst);
+ }
+
+ void vpmovzxwd_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxwd", VEX_PD, OP3_PMOVZXWD_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vpmovzxwd_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxwd", VEX_PD, OP3_PMOVZXWD_VdqWdq, ESCAPE_38, offset,
+ base, invalid_xmm, dst);
+ }
+ void vpmovzxwd_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxwd", VEX_PD, OP3_PMOVZXWD_VdqWdq, ESCAPE_38, offset,
+ base, index, scale, invalid_xmm, dst);
+ }
+
+ void vpmovsxdq_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxwd", VEX_PD, OP3_PMOVSXDQ_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vpmovsxdq_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxdq", VEX_PD, OP3_PMOVSXDQ_VdqWdq, ESCAPE_38, offset,
+ base, invalid_xmm, dst);
+ }
+ void vpmovsxdq_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovsxdq", VEX_PD, OP3_PMOVSXDQ_VdqWdq, ESCAPE_38, offset,
+ base, index, scale, invalid_xmm, dst);
+ }
+
+ void vpmovzxdq_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxwd", VEX_PD, OP3_PMOVZXDQ_VdqWdq, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vpmovzxdq_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxdq", VEX_PD, OP3_PMOVZXDQ_VdqWdq, ESCAPE_38, offset,
+ base, invalid_xmm, dst);
+ }
+ void vpmovzxdq_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vpmovzxdq", VEX_PD, OP3_PMOVZXDQ_VdqWdq, ESCAPE_38, offset,
+ base, index, scale, invalid_xmm, dst);
+ }
+
+ void vphaddd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ threeByteOpSimd("vphaddd", VEX_PD, OP3_PHADDD_VdqWdq, ESCAPE_38, src1, src0,
+ dst);
+ }
+
+ void vpalignr_irr(unsigned imm, XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ threeByteOpImmSimd("vpalignr", VEX_PD, OP3_PALIGNR_VdqWdqIb, ESCAPE_3A, imm,
+ src1, src0, dst);
+ }
+
+ void vpunpcklbw_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpcklbw", VEX_PD, OP2_PUNPCKLBW_VdqWdq, src1, src0, dst);
+ }
+ void vpunpckhbw_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpckhbw", VEX_PD, OP2_PUNPCKHBW_VdqWdq, src1, src0, dst);
+ }
+
+ void vpunpckldq_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpckldq", VEX_PD, OP2_PUNPCKLDQ_VdqWdq, src1, src0, dst);
+ }
+ void vpunpckldq_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpckldq", VEX_PD, OP2_PUNPCKLDQ_VdqWdq, offset, base,
+ src0, dst);
+ }
+ void vpunpckldq_mr(const void* addr, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpunpckldq", VEX_PD, OP2_PUNPCKLDQ_VdqWdq, addr, src0, dst);
+ }
+ void vpunpcklqdq_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpcklqdq", VEX_PD, OP2_PUNPCKLQDQ_VdqWdq, src1, src0,
+ dst);
+ }
+ void vpunpcklqdq_mr(int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpcklqdq", VEX_PD, OP2_PUNPCKLQDQ_VdqWdq, offset, base,
+ src0, dst);
+ }
+ void vpunpcklqdq_mr(const void* addr, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpunpcklqdq", VEX_PD, OP2_PUNPCKLQDQ_VdqWdq, addr, src0,
+ dst);
+ }
+ void vpunpckhdq_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpckhdq", VEX_PD, OP2_PUNPCKHDQ_VdqWdq, src1, src0, dst);
+ }
+ void vpunpckhqdq_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpckhqdq", VEX_PD, OP2_PUNPCKHQDQ_VdqWdq, src1, src0,
+ dst);
+ }
+ void vpunpcklwd_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpcklwd", VEX_PD, OP2_PUNPCKLWD_VdqWdq, src1, src0, dst);
+ }
+ void vpunpckhwd_rr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vpunpckhwd", VEX_PD, OP2_PUNPCKHWD_VdqWdq, src1, src0, dst);
+ }
+
+ void vpaddq_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpaddq", VEX_PD, OP2_PADDQ_VdqWdq, src1, src0, dst);
+ }
+ void vpsubq_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vpsubq", VEX_PD, OP2_PSUBQ_VdqWdq, src1, src0, dst);
+ }
+
+ void vbroadcastb_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastb", VEX_PD, OP3_VBROADCASTB_VxWx, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vbroadcastb_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastb", VEX_PD, OP3_VBROADCASTB_VxWx, ESCAPE_38,
+ offset, base, invalid_xmm, dst);
+ }
+ void vbroadcastb_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastb", VEX_PD, OP3_VBROADCASTB_VxWx, ESCAPE_38,
+ offset, base, index, scale, invalid_xmm, dst);
+ }
+ void vbroadcastw_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastw", VEX_PD, OP3_VBROADCASTW_VxWx, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vbroadcastw_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastw", VEX_PD, OP3_VBROADCASTW_VxWx, ESCAPE_38,
+ offset, base, invalid_xmm, dst);
+ }
+ void vbroadcastw_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastw", VEX_PD, OP3_VBROADCASTW_VxWx, ESCAPE_38,
+ offset, base, index, scale, invalid_xmm, dst);
+ }
+ void vbroadcastd_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastd", VEX_PD, OP3_VBROADCASTD_VxWx, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vbroadcastd_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastd", VEX_PD, OP3_VBROADCASTD_VxWx, ESCAPE_38,
+ offset, base, invalid_xmm, dst);
+ }
+ void vbroadcastd_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastd", VEX_PD, OP3_VBROADCASTD_VxWx, ESCAPE_38,
+ offset, base, index, scale, invalid_xmm, dst);
+ }
+ void vbroadcastq_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastq", VEX_PD, OP3_VBROADCASTQ_VxWx, ESCAPE_38, src,
+ invalid_xmm, dst);
+ }
+ void vbroadcastq_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastq", VEX_PD, OP3_VBROADCASTQ_VxWx, ESCAPE_38,
+ offset, base, invalid_xmm, dst);
+ }
+ void vbroadcastq_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastq", VEX_PD, OP3_VBROADCASTQ_VxWx, ESCAPE_38,
+ offset, base, index, scale, invalid_xmm, dst);
+ }
+ void vbroadcastss_rr(XMMRegisterID src, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastss", VEX_PD, OP3_VBROADCASTSS_VxWd, ESCAPE_38,
+ src, invalid_xmm, dst);
+ }
+ void vbroadcastss_mr(int32_t offset, RegisterID base, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastss", VEX_PD, OP3_VBROADCASTSS_VxWd, ESCAPE_38,
+ offset, base, invalid_xmm, dst);
+ }
+ void vbroadcastss_mr(int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID dst) {
+ threeByteOpSimd("vbroadcastss", VEX_PD, OP3_VBROADCASTSS_VxWd, ESCAPE_38,
+ offset, base, index, scale, invalid_xmm, dst);
+ }
+
+ // BMI instructions:
+
+ void sarxl_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("sarxl %s, %s, %s", GPReg32Name(src), GPReg32Name(shift),
+ GPReg32Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex(VEX_SS /* = F3 */, OP3_SARX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ void shlxl_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("shlxl %s, %s, %s", GPReg32Name(src), GPReg32Name(shift),
+ GPReg32Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex(VEX_PD /* = 66 */, OP3_SHLX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ void shrxl_rrr(RegisterID src, RegisterID shift, RegisterID dst) {
+ spew("shrxl %s, %s, %s", GPReg32Name(src), GPReg32Name(shift),
+ GPReg32Name(dst));
+
+ RegisterID rm = src;
+ XMMRegisterID src0 = static_cast<XMMRegisterID>(shift);
+ int reg = dst;
+ m_formatter.threeByteOpVex(VEX_SD /* = F2 */, OP3_SHRX_GyEyBy, ESCAPE_38,
+ rm, src0, reg);
+ }
+
+ // FMA instructions:
+
+ void vfmadd231ps_rrr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ spew("vfmadd213ps %s, %s, %s", XMMRegName(src1), XMMRegName(src0),
+ XMMRegName(dst));
+
+ m_formatter.threeByteOpVex(VEX_PD, OP3_VFMADD231PS_VxHxWx, ESCAPE_38,
+ (RegisterID)src1, src0, (RegisterID)dst);
+ }
+
+ void vfnmadd231ps_rrr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ spew("vfnmadd213ps %s, %s, %s", XMMRegName(src1), XMMRegName(src0),
+ XMMRegName(dst));
+
+ m_formatter.threeByteOpVex(VEX_PD, OP3_VFNMADD231PS_VxHxWx, ESCAPE_38,
+ (RegisterID)src1, src0, (RegisterID)dst);
+ }
+
+ void vfmadd231pd_rrr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ spew("vfmadd213pd %s, %s, %s", XMMRegName(src1), XMMRegName(src0),
+ XMMRegName(dst));
+
+ m_formatter.threeByteOpVex64(VEX_PD, OP3_VFMADD231PD_VxHxWx, ESCAPE_38,
+ (RegisterID)src1, src0, (RegisterID)dst);
+ }
+
+ void vfnmadd231pd_rrr(XMMRegisterID src1, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ spew("vfnmadd213pd %s, %s, %s", XMMRegName(src1), XMMRegName(src0),
+ XMMRegName(dst));
+
+ m_formatter.threeByteOpVex64(VEX_PD, OP3_VFNMADD231PD_VxHxWx, ESCAPE_38,
+ (RegisterID)src1, src0, (RegisterID)dst);
+ }
+
+ // Misc instructions:
+
+ void int3() {
+ spew("int3");
+ m_formatter.oneByteOp(OP_INT3);
+ }
+
+ void ud2() {
+ spew("ud2");
+ m_formatter.twoByteOp(OP2_UD2);
+ }
+
+ void ret() {
+ spew("ret");
+ m_formatter.oneByteOp(OP_RET);
+ }
+
+ void ret_i(int32_t imm) {
+ spew("ret $%d", imm);
+ m_formatter.oneByteOp(OP_RET_Iz);
+ m_formatter.immediate16u(imm);
+ }
+
+ void lfence() {
+ spew("lfence");
+ m_formatter.twoByteOp(OP_FENCE, (RegisterID)0, 0b101);
+ }
+ void mfence() {
+ spew("mfence");
+ m_formatter.twoByteOp(OP_FENCE, (RegisterID)0, 0b110);
+ }
+
+ // Assembler admin methods:
+
+ JmpDst label() {
+ JmpDst r = JmpDst(m_formatter.size());
+ spew(".set .Llabel%d, .", r.offset());
+ return r;
+ }
+
+ size_t currentOffset() const { return m_formatter.size(); }
+
+ static JmpDst labelFor(JmpSrc jump, intptr_t offset = 0) {
+ return JmpDst(jump.offset() + offset);
+ }
+
+ void haltingAlign(int alignment) {
+ spew(".balign %d, 0x%x # hlt", alignment, unsigned(OP_HLT));
+ while (!m_formatter.isAligned(alignment)) {
+ m_formatter.oneByteOp(OP_HLT);
+ }
+ }
+
+ void nopAlign(int alignment) {
+ spew(".balign %d", alignment);
+
+ int remainder = m_formatter.size() % alignment;
+ if (remainder > 0) {
+ insert_nop(alignment - remainder);
+ }
+ }
+
+ void jumpTablePointer(uintptr_t ptr) {
+#ifdef JS_CODEGEN_X64
+ spew(".quad 0x%" PRIxPTR, ptr);
+#else
+ spew(".int 0x%" PRIxPTR, ptr);
+#endif
+ m_formatter.jumpTablePointer(ptr);
+ }
+
+ void doubleConstant(double d) {
+ spew(".double %.16g", d);
+ m_formatter.doubleConstant(d);
+ }
+ void floatConstant(float f) {
+ spew(".float %.16g", f);
+ m_formatter.floatConstant(f);
+ }
+
+ void simd128Constant(const void* data) {
+ const uint32_t* dw = reinterpret_cast<const uint32_t*>(data);
+ spew(".int 0x%08x,0x%08x,0x%08x,0x%08x", dw[0], dw[1], dw[2], dw[3]);
+ MOZ_ASSERT(m_formatter.isAligned(16));
+ m_formatter.simd128Constant(data);
+ }
+
+ void int32Constant(int32_t i) {
+ spew(".int %d", i);
+ m_formatter.int32Constant(i);
+ }
+ void int64Constant(int64_t i) {
+ spew(".quad %lld", (long long)i);
+ m_formatter.int64Constant(i);
+ }
+
+ // Linking & patching:
+
+ void assertValidJmpSrc(JmpSrc src) {
+ // The target offset is stored at offset - 4.
+ MOZ_RELEASE_ASSERT(src.offset() > int32_t(sizeof(int32_t)));
+ MOZ_RELEASE_ASSERT(size_t(src.offset()) <= size());
+ }
+
+ bool nextJump(const JmpSrc& from, JmpSrc* next) {
+ // Sanity check - if the assembler has OOM'd, it will start overwriting
+ // its internal buffer and thus our links could be garbage.
+ if (oom()) {
+ return false;
+ }
+
+ assertValidJmpSrc(from);
+ MOZ_ASSERT(from.trailing() == 0);
+
+ const unsigned char* code = m_formatter.data();
+ int32_t offset = GetInt32(code + from.offset());
+ if (offset == -1) {
+ return false;
+ }
+
+ MOZ_RELEASE_ASSERT(size_t(offset) < size(), "nextJump bogus offset");
+
+ *next = JmpSrc(offset);
+ return true;
+ }
+ void setNextJump(const JmpSrc& from, const JmpSrc& to) {
+ // Sanity check - if the assembler has OOM'd, it will start overwriting
+ // its internal buffer and thus our links could be garbage.
+ if (oom()) {
+ return;
+ }
+
+ assertValidJmpSrc(from);
+ MOZ_ASSERT(from.trailing() == 0);
+ MOZ_RELEASE_ASSERT(to.offset() == -1 || size_t(to.offset()) <= size());
+
+ unsigned char* code = m_formatter.data();
+ SetInt32(code + from.offset(), to.offset());
+ }
+
+ void linkJump(JmpSrc from, JmpDst to) {
+ MOZ_ASSERT(from.offset() != -1);
+ MOZ_ASSERT(to.offset() != -1);
+
+ // Sanity check - if the assembler has OOM'd, it will start overwriting
+ // its internal buffer and thus our links could be garbage.
+ if (oom()) {
+ return;
+ }
+
+ assertValidJmpSrc(from);
+ MOZ_RELEASE_ASSERT(size_t(to.offset()) <= size());
+
+ spew(".set .Lfrom%d, .Llabel%d", from.offset(), to.offset());
+ unsigned char* code = m_formatter.data();
+ SetRel32(code + from.offset(), code + to.offset(), from.trailing());
+ }
+
+ void executableCopy(void* dst) {
+ const unsigned char* src = m_formatter.buffer();
+ memcpy(dst, src, size());
+ }
+ [[nodiscard]] bool appendRawCode(const uint8_t* code, size_t numBytes) {
+ return m_formatter.append(code, numBytes);
+ }
+
+ // `offset` is the instruction offset at the end of the instruction.
+ void addToPCRel4(uint32_t offset, int32_t bias) {
+ unsigned char* code = m_formatter.data();
+ SetInt32(code + offset, GetInt32(code + offset) + bias);
+ }
+
+ protected:
+ static bool CAN_SIGN_EXTEND_8_32(int32_t value) {
+ return value == (int32_t)(int8_t)value;
+ }
+ static bool CAN_SIGN_EXTEND_16_32(int32_t value) {
+ return value == (int32_t)(int16_t)value;
+ }
+ static bool CAN_ZERO_EXTEND_8_32(int32_t value) {
+ return value == (int32_t)(uint8_t)value;
+ }
+ static bool CAN_ZERO_EXTEND_8H_32(int32_t value) {
+ return value == (value & 0xff00);
+ }
+ static bool CAN_ZERO_EXTEND_16_32(int32_t value) {
+ return value == (int32_t)(uint16_t)value;
+ }
+ static bool CAN_ZERO_EXTEND_32_64(int32_t value) { return value >= 0; }
+
+ // Methods for encoding SIMD instructions via either legacy SSE encoding or
+ // VEX encoding.
+
+ bool useLegacySSEEncoding(XMMRegisterID src0, XMMRegisterID dst) {
+ // If we don't have AVX or it's disabled, use the legacy SSE encoding.
+ if (!useVEX_) {
+ MOZ_ASSERT(
+ src0 == invalid_xmm || src0 == dst,
+ "Legacy SSE (pre-AVX) encoding requires the output register to be "
+ "the same as the src0 input register");
+ return true;
+ }
+
+ // If src0 is the same as the output register, we might as well use
+ // the legacy SSE encoding, since it is smaller. However, this is only
+ // beneficial as long as we're not using ymm registers anywhere.
+ return src0 == dst;
+ }
+
+ bool useLegacySSEEncodingForVblendv(XMMRegisterID mask, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ // Similar to useLegacySSEEncoding, but for vblendv the Legacy SSE
+ // encoding also requires the mask to be in xmm0.
+
+ if (!useVEX_) {
+ MOZ_ASSERT(
+ src0 == dst,
+ "Legacy SSE (pre-AVX) encoding requires the output register to be "
+ "the same as the src0 input register");
+ MOZ_ASSERT(
+ mask == xmm0,
+ "Legacy SSE (pre-AVX) encoding for blendv requires the mask to be "
+ "in xmm0");
+ return true;
+ }
+
+ return src0 == dst && mask == xmm0;
+ }
+
+ bool useLegacySSEEncodingAlways() { return !useVEX_; }
+
+ const char* legacySSEOpName(const char* name) {
+ MOZ_ASSERT(name[0] == 'v');
+ return name + 1;
+ }
+
+ void twoByteOpSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, XMMRegisterID rm,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(dst),
+ XMMRegName(rm));
+ } else {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(rm),
+ XMMRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, (RegisterID)rm, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", name, XMMRegName(dst), XMMRegName(rm));
+ } else {
+ spew("%-11s%s, %s", name, XMMRegName(rm), XMMRegName(dst));
+ }
+ } else {
+ spew("%-11s%s, %s, %s", name, XMMRegName(rm), XMMRegName(src0),
+ XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex(ty, opcode, (RegisterID)rm, src0, dst);
+ }
+
+ void twoByteOpImmSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm, XMMRegisterID rm,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, XMMRegName(rm),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, (RegisterID)rm, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ spew("%-11s$0x%x, %s, %s", name, imm, XMMRegName(rm), XMMRegName(dst));
+ } else {
+ spew("%-11s$0x%x, %s, %s, %s", name, imm, XMMRegName(rm),
+ XMMRegName(src0), XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex(ty, opcode, (RegisterID)rm, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void twoByteOpSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, " MEM_ob, legacySSEOpName(name), XMMRegName(dst),
+ ADDR_ob(offset, base));
+ } else {
+ spew("%-11s" MEM_ob ", %s", legacySSEOpName(name),
+ ADDR_ob(offset, base), XMMRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, offset, base, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, " MEM_ob, name, XMMRegName(dst), ADDR_ob(offset, base));
+ } else {
+ spew("%-11s" MEM_ob ", %s", name, ADDR_ob(offset, base),
+ XMMRegName(dst));
+ }
+ } else {
+ spew("%-11s" MEM_ob ", %s, %s", name, ADDR_ob(offset, base),
+ XMMRegName(src0), XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex(ty, opcode, offset, base, src0, dst);
+ }
+
+ void twoByteOpSimd_disp32(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, int32_t offset,
+ RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, " MEM_o32b, legacySSEOpName(name), XMMRegName(dst),
+ ADDR_o32b(offset, base));
+ } else {
+ spew("%-11s" MEM_o32b ", %s", legacySSEOpName(name),
+ ADDR_o32b(offset, base), XMMRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp_disp32(opcode, offset, base, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, " MEM_o32b, name, XMMRegName(dst),
+ ADDR_o32b(offset, base));
+ } else {
+ spew("%-11s" MEM_o32b ", %s", name, ADDR_o32b(offset, base),
+ XMMRegName(dst));
+ }
+ } else {
+ spew("%-11s" MEM_o32b ", %s, %s", name, ADDR_o32b(offset, base),
+ XMMRegName(src0), XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex_disp32(ty, opcode, offset, base, src0, dst);
+ }
+
+ void twoByteOpImmSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm, int32_t offset,
+ RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, " MEM_ob ", %s", legacySSEOpName(name), imm,
+ ADDR_ob(offset, base), XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, offset, base, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, " MEM_ob ", %s, %s", name, imm, ADDR_ob(offset, base),
+ XMMRegName(src0), XMMRegName(dst));
+ m_formatter.twoByteOpVex(ty, opcode, offset, base, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void twoByteOpSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, int32_t offset, RegisterID base,
+ RegisterID index, int scale, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, " MEM_obs, legacySSEOpName(name), XMMRegName(dst),
+ ADDR_obs(offset, base, index, scale));
+ } else {
+ spew("%-11s" MEM_obs ", %s", legacySSEOpName(name),
+ ADDR_obs(offset, base, index, scale), XMMRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, offset, base, index, scale, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, " MEM_obs, name, XMMRegName(dst),
+ ADDR_obs(offset, base, index, scale));
+ } else {
+ spew("%-11s" MEM_obs ", %s", name, ADDR_obs(offset, base, index, scale),
+ XMMRegName(dst));
+ }
+ } else {
+ spew("%-11s" MEM_obs ", %s, %s", name,
+ ADDR_obs(offset, base, index, scale), XMMRegName(src0),
+ XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex(ty, opcode, offset, base, index, scale, src0, dst);
+ }
+
+ void twoByteOpSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, const void* address,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %p", legacySSEOpName(name), XMMRegName(dst), address);
+ } else {
+ spew("%-11s%p, %s", legacySSEOpName(name), address, XMMRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, address, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %p", name, XMMRegName(dst), address);
+ } else {
+ spew("%-11s%p, %s", name, address, XMMRegName(dst));
+ }
+ } else {
+ spew("%-11s%p, %s, %s", name, address, XMMRegName(src0), XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex(ty, opcode, address, src0, dst);
+ }
+
+ void twoByteOpImmSimd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm,
+ const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, %p, %s", legacySSEOpName(name), imm, address,
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, address, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, %p, %s, %s", name, imm, address, XMMRegName(src0),
+ XMMRegName(dst));
+ m_formatter.twoByteOpVex(ty, opcode, address, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void twoByteOpInt32Simd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, RegisterID rm,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(dst),
+ GPReg32Name(rm));
+ } else {
+ spew("%-11s%s, %s", legacySSEOpName(name), GPReg32Name(rm),
+ XMMRegName(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, rm, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", name, XMMRegName(dst), GPReg32Name(rm));
+ } else {
+ spew("%-11s%s, %s", name, GPReg32Name(rm), XMMRegName(dst));
+ }
+ } else {
+ spew("%-11s%s, %s, %s", name, GPReg32Name(rm), XMMRegName(src0),
+ XMMRegName(dst));
+ }
+ m_formatter.twoByteOpVex(ty, opcode, rm, src0, dst);
+ }
+
+ void twoByteOpSimdInt32(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, XMMRegisterID rm,
+ RegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), GPReg32Name(dst),
+ XMMRegName(rm));
+ } else if (opcode == OP2_MOVD_EdVd) {
+ spew("%-11s%s, %s", legacySSEOpName(name),
+ XMMRegName((XMMRegisterID)dst), GPReg32Name((RegisterID)rm));
+ } else {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(rm),
+ GPReg32Name(dst));
+ }
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, (RegisterID)rm, dst);
+ return;
+ }
+
+ if (IsXMMReversedOperands(opcode)) {
+ spew("%-11s%s, %s", name, GPReg32Name(dst), XMMRegName(rm));
+ } else if (opcode == OP2_MOVD_EdVd) {
+ spew("%-11s%s, %s", name, XMMRegName((XMMRegisterID)dst),
+ GPReg32Name((RegisterID)rm));
+ } else {
+ spew("%-11s%s, %s", name, XMMRegName(rm), GPReg32Name(dst));
+ }
+ m_formatter.twoByteOpVex(ty, opcode, (RegisterID)rm, invalid_xmm, dst);
+ }
+
+ void twoByteOpImmSimdInt32(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm,
+ XMMRegisterID rm, RegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, XMMRegName(rm),
+ GPReg32Name(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, (RegisterID)rm, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, %s, %s", name, imm, XMMRegName(rm), GPReg32Name(dst));
+ m_formatter.twoByteOpVex(ty, opcode, (RegisterID)rm, invalid_xmm, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void twoByteOpImmInt32Simd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm,
+ RegisterID rm, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, GPReg32Name(rm),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, rm, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, %s, %s", name, imm, GPReg32Name(rm), XMMRegName(dst));
+ m_formatter.twoByteOpVex(ty, opcode, rm, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void twoByteOpImmInt32Simd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm,
+ int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s$0x%x, " MEM_ob ", %s", legacySSEOpName(name), imm,
+ ADDR_ob(offset, base), XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, offset, base, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, " MEM_ob ", %s, %s", name, imm, ADDR_ob(offset, base),
+ XMMRegName(src0), XMMRegName(dst));
+ m_formatter.twoByteOpVex(ty, opcode, offset, base, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void twoByteOpImmInt32Simd(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, uint32_t imm,
+ int32_t offset, RegisterID base, RegisterID index,
+ int scale, XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s$0x%x, " MEM_obs ", %s", legacySSEOpName(name), imm,
+ ADDR_obs(offset, base, index, scale), XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, offset, base, index, scale, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, " MEM_obs ", %s, %s", name, imm,
+ ADDR_obs(offset, base, index, scale), XMMRegName(src0),
+ XMMRegName(dst));
+ m_formatter.twoByteOpVex(ty, opcode, offset, base, index, scale, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void twoByteOpSimdFlags(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, XMMRegisterID rm,
+ XMMRegisterID reg) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(rm),
+ XMMRegName(reg));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, (RegisterID)rm, reg);
+ return;
+ }
+
+ spew("%-11s%s, %s", name, XMMRegName(rm), XMMRegName(reg));
+ m_formatter.twoByteOpVex(ty, opcode, (RegisterID)rm, invalid_xmm,
+ (XMMRegisterID)reg);
+ }
+
+ void twoByteOpSimdFlags(const char* name, VexOperandType ty,
+ TwoByteOpcodeID opcode, int32_t offset,
+ RegisterID base, XMMRegisterID reg) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s" MEM_ob ", %s", legacySSEOpName(name), ADDR_ob(offset, base),
+ XMMRegName(reg));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.twoByteOp(opcode, offset, base, reg);
+ return;
+ }
+
+ spew("%-11s" MEM_ob ", %s", name, ADDR_ob(offset, base), XMMRegName(reg));
+ m_formatter.twoByteOpVex(ty, opcode, offset, base, invalid_xmm,
+ (XMMRegisterID)reg);
+ }
+
+ void threeByteOpSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ XMMRegisterID rm, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(rm),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, (RegisterID)rm, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ spew("%-11s%s, %s", name, XMMRegName(rm), XMMRegName(dst));
+ } else {
+ spew("%-11s%s, %s, %s", name, XMMRegName(rm), XMMRegName(src0),
+ XMMRegName(dst));
+ }
+ m_formatter.threeByteOpVex(ty, opcode, escape, (RegisterID)rm, src0, dst);
+ }
+
+ void threeByteOpImmSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, XMMRegisterID rm, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, XMMRegName(rm),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, (RegisterID)rm, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ spew("%-11s$0x%x, %s, %s", name, imm, XMMRegName(rm), XMMRegName(dst));
+ } else {
+ spew("%-11s$0x%x, %s, %s, %s", name, imm, XMMRegName(rm),
+ XMMRegName(src0), XMMRegName(dst));
+ }
+ m_formatter.threeByteOpVex(ty, opcode, escape, (RegisterID)rm, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s" MEM_ob ", %s", legacySSEOpName(name), ADDR_ob(offset, base),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, offset, base, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ spew("%-11s" MEM_ob ", %s", name, ADDR_ob(offset, base), XMMRegName(dst));
+ } else {
+ spew("%-11s" MEM_ob ", %s, %s", name, ADDR_ob(offset, base),
+ XMMRegName(src0), XMMRegName(dst));
+ }
+ m_formatter.threeByteOpVex(ty, opcode, escape, offset, base, src0, dst);
+ }
+
+ void threeByteOpSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s" MEM_obs ", %s", legacySSEOpName(name),
+ ADDR_obs(offset, base, index, scale), XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, offset, base, index, scale, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ spew("%-11s" MEM_obs ", %s", name, ADDR_obs(offset, base, index, scale),
+ XMMRegName(dst));
+ } else {
+ spew("%-11s" MEM_obs ", %s, %s", name,
+ ADDR_obs(offset, base, index, scale), XMMRegName(src0),
+ XMMRegName(dst));
+ }
+ m_formatter.threeByteOpVex(ty, opcode, escape, offset, base, index, scale,
+ src0, dst);
+ }
+
+ void threeByteOpImmSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, " MEM_ob ", %s", legacySSEOpName(name), imm,
+ ADDR_ob(offset, base), XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, offset, base, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ spew("%-11s$0x%x, " MEM_ob ", %s", name, imm, ADDR_ob(offset, base),
+ XMMRegName(dst));
+ } else {
+ spew("%-11s$0x%x, " MEM_ob ", %s, %s", name, imm, ADDR_ob(offset, base),
+ XMMRegName(src0), XMMRegName(dst));
+ }
+ m_formatter.threeByteOpVex(ty, opcode, escape, offset, base, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpImmSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, int32_t offset, RegisterID base,
+ RegisterID index, int scale, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, " MEM_obs ", %s", legacySSEOpName(name), imm,
+ ADDR_obs(offset, base, index, scale), XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, offset, base, index, scale, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ spew("%-11s$0x%x, " MEM_obs ", %s", name, imm,
+ ADDR_obs(offset, base, index, scale), XMMRegName(dst));
+ } else {
+ spew("%-11s$0x%x, " MEM_obs ", %s, %s", name, imm,
+ ADDR_obs(offset, base, index, scale), XMMRegName(src0),
+ XMMRegName(dst));
+ }
+ m_formatter.threeByteOpVex(ty, opcode, escape, offset, base, index, scale,
+ src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpSimd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s%p, %s", legacySSEOpName(name), address, XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, address, dst);
+ return;
+ }
+
+ if (src0 == invalid_xmm) {
+ spew("%-11s%p, %s", name, address, XMMRegName(dst));
+ } else {
+ spew("%-11s%p, %s, %s", name, address, XMMRegName(src0), XMMRegName(dst));
+ }
+ m_formatter.threeByteOpVex(ty, opcode, escape, address, src0, dst);
+ }
+
+ void threeByteOpImmInt32Simd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, RegisterID src1,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, GPReg32Name(src1),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, src1, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, %s, %s, %s", name, imm, GPReg32Name(src1),
+ XMMRegName(src0), XMMRegName(dst));
+ m_formatter.threeByteOpVex(ty, opcode, escape, src1, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpImmInt32Simd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, int32_t offset, RegisterID base,
+ XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, " MEM_ob ", %s", legacySSEOpName(name), imm,
+ ADDR_ob(offset, base), XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, offset, base, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, " MEM_ob ", %s, %s", name, imm, ADDR_ob(offset, base),
+ XMMRegName(src0), XMMRegName(dst));
+ m_formatter.threeByteOpVex(ty, opcode, escape, offset, base, src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpImmInt32Simd(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, int32_t offset, RegisterID base,
+ RegisterID index, int scale, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src0, dst)) {
+ spew("%-11s$0x%x, " MEM_obs ", %s", legacySSEOpName(name), imm,
+ ADDR_obs(offset, base, index, scale), XMMRegName(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, offset, base, index, scale, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, " MEM_obs ", %s, %s", name, imm,
+ ADDR_obs(offset, base, index, scale), XMMRegName(src0),
+ XMMRegName(dst));
+ m_formatter.threeByteOpVex(ty, opcode, escape, offset, base, index, scale,
+ src0, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpImmSimdInt32(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, XMMRegisterID src,
+ RegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s$0x%x, %s, %s", legacySSEOpName(name), imm, XMMRegName(src),
+ GPReg32Name(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, (RegisterID)src, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ if (opcode == OP3_PEXTRD_EvVdqIb) {
+ spew("%-11s$0x%x, %s, %s", name, imm, XMMRegName((XMMRegisterID)dst),
+ GPReg32Name((RegisterID)src));
+ } else {
+ spew("%-11s$0x%x, %s, %s", name, imm, XMMRegName(src), GPReg32Name(dst));
+ }
+ m_formatter.threeByteOpVex(ty, opcode, escape, (RegisterID)src, invalid_xmm,
+ dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpImmSimdInt32(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, int32_t offset, RegisterID base,
+ RegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s$0x%x, " MEM_ob ", %s", legacySSEOpName(name), imm,
+ ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, offset, base, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, " MEM_ob ", %s", name, imm, ADDR_ob(offset, base),
+ GPReg32Name(dst));
+ m_formatter.threeByteOpVex(ty, opcode, escape, offset, base, invalid_xmm,
+ dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ void threeByteOpImmSimdInt32(const char* name, VexOperandType ty,
+ ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ uint32_t imm, int32_t offset, RegisterID base,
+ RegisterID index, int scale, RegisterID dst) {
+ if (useLegacySSEEncodingAlways()) {
+ spew("%-11s$0x%x, " MEM_obs ", %s", legacySSEOpName(name), imm,
+ ADDR_obs(offset, base, index, scale), GPReg32Name(dst));
+ m_formatter.legacySSEPrefix(ty);
+ m_formatter.threeByteOp(opcode, escape, offset, base, index, scale, dst);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$0x%x, " MEM_obs ", %s", name, imm,
+ ADDR_obs(offset, base, index, scale), GPReg32Name(dst));
+ m_formatter.threeByteOpVex(ty, opcode, escape, offset, base, index, scale,
+ invalid_xmm, dst);
+ m_formatter.immediate8u(imm);
+ }
+
+ // Blendv is a three-byte op, but the VEX encoding has a different opcode
+ // than the SSE encoding, so we handle it specially.
+ void vblendvOpSimd(const char* name, ThreeByteOpcodeID opcode,
+ ThreeByteOpcodeID vexOpcode, XMMRegisterID mask,
+ XMMRegisterID rm, XMMRegisterID src0, XMMRegisterID dst) {
+ if (useLegacySSEEncodingForVblendv(mask, src0, dst)) {
+ spew("%-11s%s, %s", legacySSEOpName(name), XMMRegName(rm),
+ XMMRegName(dst));
+ // Even though a "ps" instruction, vblendv is encoded with the "pd"
+ // prefix.
+ m_formatter.legacySSEPrefix(VEX_PD);
+ m_formatter.threeByteOp(opcode, ESCAPE_38, (RegisterID)rm, dst);
+ return;
+ }
+
+ spew("%-11s%s, %s, %s, %s", name, XMMRegName(mask), XMMRegName(rm),
+ XMMRegName(src0), XMMRegName(dst));
+ // Even though a "ps" instruction, vblendv is encoded with the "pd" prefix.
+ m_formatter.vblendvOpVex(VEX_PD, vexOpcode, ESCAPE_3A, mask, (RegisterID)rm,
+ src0, dst);
+ }
+
+ void vblendvOpSimd(const char* name, ThreeByteOpcodeID opcode,
+ ThreeByteOpcodeID vexOpcode, XMMRegisterID mask,
+ int32_t offset, RegisterID base, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncodingForVblendv(mask, src0, dst)) {
+ spew("%-11s" MEM_ob ", %s", legacySSEOpName(name), ADDR_ob(offset, base),
+ XMMRegName(dst));
+ // Even though a "ps" instruction, vblendv is encoded with the "pd"
+ // prefix.
+ m_formatter.legacySSEPrefix(VEX_PD);
+ m_formatter.threeByteOp(opcode, ESCAPE_38, offset, base, dst);
+ return;
+ }
+
+ spew("%-11s%s, " MEM_ob ", %s, %s", name, XMMRegName(mask),
+ ADDR_ob(offset, base), XMMRegName(src0), XMMRegName(dst));
+ // Even though a "ps" instruction, vblendv is encoded with the "pd" prefix.
+ m_formatter.vblendvOpVex(VEX_PD, vexOpcode, ESCAPE_3A, mask, offset, base,
+ src0, dst);
+ }
+
+ void shiftOpImmSimd(const char* name, TwoByteOpcodeID opcode,
+ ShiftID shiftKind, uint32_t imm, XMMRegisterID src,
+ XMMRegisterID dst) {
+ if (useLegacySSEEncoding(src, dst)) {
+ spew("%-11s$%d, %s", legacySSEOpName(name), int32_t(imm),
+ XMMRegName(dst));
+ m_formatter.legacySSEPrefix(VEX_PD);
+ m_formatter.twoByteOp(opcode, (RegisterID)dst, (int)shiftKind);
+ m_formatter.immediate8u(imm);
+ return;
+ }
+
+ spew("%-11s$%d, %s, %s", name, int32_t(imm), XMMRegName(src),
+ XMMRegName(dst));
+ // For shift instructions, destination is stored in vvvv field.
+ m_formatter.twoByteOpVex(VEX_PD, opcode, (RegisterID)src, dst,
+ (int)shiftKind);
+ m_formatter.immediate8u(imm);
+ }
+
+ class X86InstructionFormatter {
+ public:
+ // Legacy prefix bytes:
+ //
+ // These are emmitted prior to the instruction.
+
+ void prefix(OneByteOpcodeID pre) { m_buffer.putByte(pre); }
+
+ void legacySSEPrefix(VexOperandType ty) {
+ switch (ty) {
+ case VEX_PS:
+ break;
+ case VEX_PD:
+ prefix(PRE_SSE_66);
+ break;
+ case VEX_SS:
+ prefix(PRE_SSE_F3);
+ break;
+ case VEX_SD:
+ prefix(PRE_SSE_F2);
+ break;
+ }
+ }
+
+ /* clang-format off */
+ //
+ // Word-sized operands / no operand instruction formatters.
+ //
+ // In addition to the opcode, the following operand permutations are supported:
+ // * None - instruction takes no operands.
+ // * One register - the low three bits of the RegisterID are added into the opcode.
+ // * Two registers - encode a register form ModRm (for all ModRm formats, the reg field is passed first, and a GroupOpcodeID may be passed in its place).
+ // * Three argument ModRM - a register, and a register and an offset describing a memory operand.
+ // * Five argument ModRM - a register, and a base register, an index, scale, and offset describing a memory operand.
+ //
+ // For 32-bit x86 targets, the address operand may also be provided as a
+ // void*. On 64-bit targets REX prefixes will be planted as necessary,
+ // where high numbered registers are used.
+ //
+ // The twoByteOp methods plant two-byte Intel instructions sequences
+ // (first opcode byte 0x0F).
+ //
+ /* clang-format on */
+
+ void oneByteOp(OneByteOpcodeID opcode) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ m_buffer.putByteUnchecked(opcode);
+ }
+
+ void oneByteOp(OneByteOpcodeID opcode, RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(0, 0, reg);
+ m_buffer.putByteUnchecked(opcode + (reg & 7));
+ }
+
+ void oneByteOp(OneByteOpcodeID opcode, RegisterID rm, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, rm);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, reg);
+ }
+
+ void oneByteOp(OneByteOpcodeID opcode, int32_t offset, RegisterID base,
+ int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void oneByteOp_disp32(OneByteOpcodeID opcode, int32_t offset,
+ RegisterID base, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM_disp32(offset, base, reg);
+ }
+
+ void oneByteOp(OneByteOpcodeID opcode, int32_t offset, RegisterID base,
+ RegisterID index, int scale, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, index, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ void oneByteOp_disp32(OneByteOpcodeID opcode, int32_t offset,
+ RegisterID index, int scale, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, index, 0);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM_disp32(offset, index, scale, reg);
+ }
+
+ void oneByteOp(OneByteOpcodeID opcode, const void* address, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, 0);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM_disp32(address, reg);
+ }
+
+ void oneByteOp_disp32(OneByteOpcodeID opcode, const void* address,
+ int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, 0);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM_disp32(address, reg);
+ }
+#ifdef JS_CODEGEN_X64
+ void oneByteRipOp(OneByteOpcodeID opcode, int ripOffset, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, 0);
+ m_buffer.putByteUnchecked(opcode);
+ putModRm(ModRmMemoryNoDisp, noBase, reg);
+ m_buffer.putIntUnchecked(ripOffset);
+ }
+
+ void oneByteRipOp64(OneByteOpcodeID opcode, int ripOffset, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, 0);
+ m_buffer.putByteUnchecked(opcode);
+ putModRm(ModRmMemoryNoDisp, noBase, reg);
+ m_buffer.putIntUnchecked(ripOffset);
+ }
+
+ void twoByteRipOp(TwoByteOpcodeID opcode, int ripOffset, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, 0);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ putModRm(ModRmMemoryNoDisp, noBase, reg);
+ m_buffer.putIntUnchecked(ripOffset);
+ }
+
+ void twoByteRipOpVex(VexOperandType ty, TwoByteOpcodeID opcode,
+ int ripOffset, XMMRegisterID src0, XMMRegisterID reg) {
+ int r = (reg >> 3), x = 0, b = 0;
+ int m = 1; // 0x0F
+ int w = 0, v = src0, l = 0;
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ putModRm(ModRmMemoryNoDisp, noBase, reg);
+ m_buffer.putIntUnchecked(ripOffset);
+ }
+#endif
+
+ void twoByteOp(TwoByteOpcodeID opcode) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ }
+
+ void twoByteOp(TwoByteOpcodeID opcode, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(0, 0, reg);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode + (reg & 7));
+ }
+
+ void twoByteOp(TwoByteOpcodeID opcode, RegisterID rm, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, rm);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, reg);
+ }
+
+ void twoByteOpVex(VexOperandType ty, TwoByteOpcodeID opcode, RegisterID rm,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = (rm >> 3);
+ int m = 1; // 0x0F
+ int w = 0, v = src0, l = 0;
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ registerModRM(rm, reg);
+ }
+
+ void twoByteOp(TwoByteOpcodeID opcode, int32_t offset, RegisterID base,
+ int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void twoByteOpVex(VexOperandType ty, TwoByteOpcodeID opcode, int32_t offset,
+ RegisterID base, XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = (base >> 3);
+ int m = 1; // 0x0F
+ int w = 0, v = src0, l = 0;
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void twoByteOp_disp32(TwoByteOpcodeID opcode, int32_t offset,
+ RegisterID base, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM_disp32(offset, base, reg);
+ }
+
+ void twoByteOpVex_disp32(VexOperandType ty, TwoByteOpcodeID opcode,
+ int32_t offset, RegisterID base,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = (base >> 3);
+ int m = 1; // 0x0F
+ int w = 0, v = src0, l = 0;
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ memoryModRM_disp32(offset, base, reg);
+ }
+
+ void twoByteOp(TwoByteOpcodeID opcode, int32_t offset, RegisterID base,
+ RegisterID index, int scale, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, index, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ void twoByteOpVex(VexOperandType ty, TwoByteOpcodeID opcode, int32_t offset,
+ RegisterID base, RegisterID index, int scale,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = (index >> 3), b = (base >> 3);
+ int m = 1; // 0x0F
+ int w = 0, v = src0, l = 0;
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ void twoByteOp(TwoByteOpcodeID opcode, const void* address, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, 0);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(address, reg);
+ }
+
+ void twoByteOpVex(VexOperandType ty, TwoByteOpcodeID opcode,
+ const void* address, XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = 0;
+ int m = 1; // 0x0F
+ int w = 0, v = src0, l = 0;
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ memoryModRM(address, reg);
+ }
+
+ void threeByteOp(ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ RegisterID rm, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, rm);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(escape);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, reg);
+ }
+
+ void threeByteOpVex(VexOperandType ty, ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape, RegisterID rm,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = (rm >> 3);
+ int m = 0, w = 0, v = src0, l = 0;
+ switch (escape) {
+ case ESCAPE_38:
+ m = 2;
+ break;
+ case ESCAPE_3A:
+ m = 3;
+ break;
+ default:
+ MOZ_CRASH("unexpected escape");
+ }
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ registerModRM(rm, reg);
+ }
+
+ void threeByteOp(ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ int32_t offset, RegisterID base, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(escape);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void threeByteOp(ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ int32_t offset, RegisterID base, RegisterID index,
+ int32_t scale, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, index, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(escape);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ void threeByteOpVex(VexOperandType ty, ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape, int32_t offset, RegisterID base,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = (base >> 3);
+ int m = 0, w = 0, v = src0, l = 0;
+ switch (escape) {
+ case ESCAPE_38:
+ m = 2;
+ break;
+ case ESCAPE_3A:
+ m = 3;
+ break;
+ default:
+ MOZ_CRASH("unexpected escape");
+ }
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void threeByteOpVex(VexOperandType ty, ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape, int32_t offset, RegisterID base,
+ RegisterID index, int scale, XMMRegisterID src0,
+ int reg) {
+ int r = (reg >> 3), x = (index >> 3), b = (base >> 3);
+ int m = 0, w = 0, v = src0, l = 0;
+ switch (escape) {
+ case ESCAPE_38:
+ m = 2;
+ break;
+ case ESCAPE_3A:
+ m = 3;
+ break;
+ default:
+ MOZ_CRASH("unexpected escape");
+ }
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ void threeByteOp(ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ const void* address, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, 0);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(escape);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(address, reg);
+ }
+
+ void threeByteRipOp(ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ int ripOffset, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIfNeeded(reg, 0, 0);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(escape);
+ m_buffer.putByteUnchecked(opcode);
+ putModRm(ModRmMemoryNoDisp, noBase, reg);
+ m_buffer.putIntUnchecked(ripOffset);
+ }
+
+ void threeByteOpVex(VexOperandType ty, ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape, const void* address,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = 0;
+ int m = 0, w = 0, v = src0, l = 0;
+ switch (escape) {
+ case ESCAPE_38:
+ m = 2;
+ break;
+ case ESCAPE_3A:
+ m = 3;
+ break;
+ default:
+ MOZ_CRASH("unexpected escape");
+ }
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ memoryModRM(address, reg);
+ }
+
+ void threeByteRipOpVex(VexOperandType ty, ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape, int ripOffset,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = 0;
+ int m = 0;
+ switch (escape) {
+ case ESCAPE_38:
+ m = 2;
+ break;
+ case ESCAPE_3A:
+ m = 3;
+ break;
+ default:
+ MOZ_CRASH("unexpected escape");
+ }
+ int w = 0, v = src0, l = 0;
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ putModRm(ModRmMemoryNoDisp, noBase, reg);
+ m_buffer.putIntUnchecked(ripOffset);
+ }
+
+ void vblendvOpVex(VexOperandType ty, ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape, XMMRegisterID mask, RegisterID rm,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = (rm >> 3);
+ int m = 0, w = 0, v = src0, l = 0;
+ switch (escape) {
+ case ESCAPE_38:
+ m = 2;
+ break;
+ case ESCAPE_3A:
+ m = 3;
+ break;
+ default:
+ MOZ_CRASH("unexpected escape");
+ }
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ registerModRM(rm, reg);
+ immediate8u(mask << 4);
+ }
+
+ void vblendvOpVex(VexOperandType ty, ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape, XMMRegisterID mask,
+ int32_t offset, RegisterID base, XMMRegisterID src0,
+ int reg) {
+ int r = (reg >> 3), x = 0, b = (base >> 3);
+ int m = 0, w = 0, v = src0, l = 0;
+ switch (escape) {
+ case ESCAPE_38:
+ m = 2;
+ break;
+ case ESCAPE_3A:
+ m = 3;
+ break;
+ default:
+ MOZ_CRASH("unexpected escape");
+ }
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ memoryModRM(offset, base, reg);
+ immediate8u(mask << 4);
+ }
+
+#ifdef JS_CODEGEN_X64
+ // Quad-word-sized operands:
+ //
+ // Used to format 64-bit operantions, planting a REX.w prefix. When
+ // planting d64 or f64 instructions, not requiring a REX.w prefix, the
+ // normal (non-'64'-postfixed) formatters should be used.
+
+ void oneByteOp64(OneByteOpcodeID opcode) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(0, 0, 0);
+ m_buffer.putByteUnchecked(opcode);
+ }
+
+ void oneByteOp64(OneByteOpcodeID opcode, RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(0, 0, reg);
+ m_buffer.putByteUnchecked(opcode + (reg & 7));
+ }
+
+ void oneByteOp64(OneByteOpcodeID opcode, RegisterID rm, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, rm);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, reg);
+ }
+
+ void oneByteOp64(OneByteOpcodeID opcode, int32_t offset, RegisterID base,
+ int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void oneByteOp64_disp32(OneByteOpcodeID opcode, int32_t offset,
+ RegisterID base, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM_disp32(offset, base, reg);
+ }
+
+ void oneByteOp64(OneByteOpcodeID opcode, int32_t offset, RegisterID base,
+ RegisterID index, int scale, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, index, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ void oneByteOp64(OneByteOpcodeID opcode, const void* address, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, 0);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(address, reg);
+ }
+
+ void twoByteOp64(TwoByteOpcodeID opcode, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(0, 0, reg);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode + (reg & 7));
+ }
+
+ void twoByteOp64(TwoByteOpcodeID opcode, RegisterID rm, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, rm);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, reg);
+ }
+
+ void twoByteOp64(TwoByteOpcodeID opcode, int offset, RegisterID base,
+ int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void twoByteOp64(TwoByteOpcodeID opcode, int offset, RegisterID base,
+ RegisterID index, int scale, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, index, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ void twoByteOp64(TwoByteOpcodeID opcode, const void* address, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, 0);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(address, reg);
+ }
+
+ void twoByteOpVex64(VexOperandType ty, TwoByteOpcodeID opcode,
+ RegisterID rm, XMMRegisterID src0, XMMRegisterID reg) {
+ int r = (reg >> 3), x = 0, b = (rm >> 3);
+ int m = 1; // 0x0F
+ int w = 1, v = src0, l = 0;
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ registerModRM(rm, reg);
+ }
+
+ void threeByteOp64(ThreeByteOpcodeID opcode, ThreeByteEscape escape,
+ RegisterID rm, int reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexW(reg, 0, rm);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(escape);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, reg);
+ }
+#endif // JS_CODEGEN_X64
+
+ void threeByteOpVex64(VexOperandType ty, ThreeByteOpcodeID opcode,
+ ThreeByteEscape escape, RegisterID rm,
+ XMMRegisterID src0, int reg) {
+ int r = (reg >> 3), x = 0, b = (rm >> 3);
+ int m = 0, w = 1, v = src0, l = 0;
+ switch (escape) {
+ case ESCAPE_38:
+ m = 2;
+ break;
+ case ESCAPE_3A:
+ m = 3;
+ break;
+ default:
+ MOZ_CRASH("unexpected escape");
+ }
+ threeOpVex(ty, r, x, b, m, w, v, l, opcode);
+ registerModRM(rm, reg);
+ }
+
+ // Byte-operands:
+ //
+ // These methods format byte operations. Byte operations differ from
+ // the normal formatters in the circumstances under which they will
+ // decide to emit REX prefixes. These should be used where any register
+ // operand signifies a byte register.
+ //
+ // The disctinction is due to the handling of register numbers in the
+ // range 4..7 on x86-64. These register numbers may either represent
+ // the second byte of the first four registers (ah..bh) or the first
+ // byte of the second four registers (spl..dil).
+ //
+ // Address operands should still be checked using regRequiresRex(),
+ // while byteRegRequiresRex() is provided to check byte register
+ // operands.
+
+ void oneByteOp8(OneByteOpcodeID opcode) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ m_buffer.putByteUnchecked(opcode);
+ }
+
+ void oneByteOp8(OneByteOpcodeID opcode, RegisterID r) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(r), 0, 0, r);
+ m_buffer.putByteUnchecked(opcode + (r & 7));
+ }
+
+ void oneByteOp8(OneByteOpcodeID opcode, RegisterID rm,
+ GroupOpcodeID groupOp) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(rm), 0, 0, rm);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, groupOp);
+ }
+
+ // Like oneByteOp8, but never emits a REX prefix.
+ void oneByteOp8_norex(OneByteOpcodeID opcode, HRegisterID rm,
+ GroupOpcodeID groupOp) {
+ MOZ_ASSERT(!regRequiresRex(RegisterID(rm)));
+ m_buffer.ensureSpace(MaxInstructionSize);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(RegisterID(rm), groupOp);
+ }
+
+ void oneByteOp8(OneByteOpcodeID opcode, int32_t offset, RegisterID base,
+ RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(reg), reg, 0, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void oneByteOp8_disp32(OneByteOpcodeID opcode, int32_t offset,
+ RegisterID base, RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(reg), reg, 0, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM_disp32(offset, base, reg);
+ }
+
+ void oneByteOp8(OneByteOpcodeID opcode, int32_t offset, RegisterID base,
+ RegisterID index, int scale, RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(reg), reg, index, base);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ void oneByteOp8(OneByteOpcodeID opcode, const void* address,
+ RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(reg), reg, 0, 0);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM_disp32(address, reg);
+ }
+
+ void twoByteOp8(TwoByteOpcodeID opcode, RegisterID rm, RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(reg) || byteRegRequiresRex(rm), reg, 0, rm);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, reg);
+ }
+
+ void twoByteOp8(TwoByteOpcodeID opcode, int32_t offset, RegisterID base,
+ RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(reg) || regRequiresRex(base), reg, 0, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, reg);
+ }
+
+ void twoByteOp8(TwoByteOpcodeID opcode, int32_t offset, RegisterID base,
+ RegisterID index, int scale, RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(reg) || regRequiresRex(base) ||
+ regRequiresRex(index),
+ reg, index, base);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ memoryModRM(offset, base, index, scale, reg);
+ }
+
+ // Like twoByteOp8 but doesn't add a REX prefix if the destination reg
+ // is in esp..edi. This may be used when the destination is not an 8-bit
+ // register (as in a movzbl instruction), so it doesn't need a REX
+ // prefix to disambiguate it from ah..bh.
+ void twoByteOp8_movx(TwoByteOpcodeID opcode, RegisterID rm,
+ RegisterID reg) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(regRequiresRex(reg) || byteRegRequiresRex(rm), reg, 0, rm);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, reg);
+ }
+
+ void twoByteOp8(TwoByteOpcodeID opcode, RegisterID rm,
+ GroupOpcodeID groupOp) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+ emitRexIf(byteRegRequiresRex(rm), 0, 0, rm);
+ m_buffer.putByteUnchecked(OP_2BYTE_ESCAPE);
+ m_buffer.putByteUnchecked(opcode);
+ registerModRM(rm, groupOp);
+ }
+
+ // Immediates:
+ //
+ // An immedaite should be appended where appropriate after an op has
+ // been emitted. The writes are unchecked since the opcode formatters
+ // above will have ensured space.
+
+ // A signed 8-bit immediate.
+ MOZ_ALWAYS_INLINE void immediate8s(int32_t imm) {
+ MOZ_ASSERT(CAN_SIGN_EXTEND_8_32(imm));
+ m_buffer.putByteUnchecked(imm);
+ }
+
+ // An unsigned 8-bit immediate.
+ MOZ_ALWAYS_INLINE void immediate8u(uint32_t imm) {
+ MOZ_ASSERT(CAN_ZERO_EXTEND_8_32(imm));
+ m_buffer.putByteUnchecked(int32_t(imm));
+ }
+
+ // An 8-bit immediate with is either signed or unsigned, for use in
+ // instructions which actually only operate on 8 bits.
+ MOZ_ALWAYS_INLINE void immediate8(int32_t imm) {
+ m_buffer.putByteUnchecked(imm);
+ }
+
+ // A signed 16-bit immediate.
+ MOZ_ALWAYS_INLINE void immediate16s(int32_t imm) {
+ MOZ_ASSERT(CAN_SIGN_EXTEND_16_32(imm));
+ m_buffer.putShortUnchecked(imm);
+ }
+
+ // An unsigned 16-bit immediate.
+ MOZ_ALWAYS_INLINE void immediate16u(int32_t imm) {
+ MOZ_ASSERT(CAN_ZERO_EXTEND_16_32(imm));
+ m_buffer.putShortUnchecked(imm);
+ }
+
+ // A 16-bit immediate with is either signed or unsigned, for use in
+ // instructions which actually only operate on 16 bits.
+ MOZ_ALWAYS_INLINE void immediate16(int32_t imm) {
+ m_buffer.putShortUnchecked(imm);
+ }
+
+ MOZ_ALWAYS_INLINE void immediate32(int32_t imm) {
+ m_buffer.putIntUnchecked(imm);
+ }
+
+ MOZ_ALWAYS_INLINE void immediate64(int64_t imm) {
+ m_buffer.putInt64Unchecked(imm);
+ }
+
+ [[nodiscard]] MOZ_ALWAYS_INLINE JmpSrc immediateRel32() {
+ m_buffer.putIntUnchecked(0);
+ return JmpSrc(m_buffer.size());
+ }
+
+ // Data:
+
+ void jumpTablePointer(uintptr_t ptr) {
+ m_buffer.ensureSpace(sizeof(uintptr_t));
+#ifdef JS_CODEGEN_X64
+ m_buffer.putInt64Unchecked(ptr);
+#else
+ m_buffer.putIntUnchecked(ptr);
+#endif
+ }
+
+ void doubleConstant(double d) {
+ m_buffer.ensureSpace(sizeof(double));
+ m_buffer.putInt64Unchecked(mozilla::BitwiseCast<uint64_t>(d));
+ }
+
+ void floatConstant(float f) {
+ m_buffer.ensureSpace(sizeof(float));
+ m_buffer.putIntUnchecked(mozilla::BitwiseCast<uint32_t>(f));
+ }
+
+ void simd128Constant(const void* data) {
+ const uint8_t* bytes = reinterpret_cast<const uint8_t*>(data);
+ m_buffer.ensureSpace(16);
+ for (size_t i = 0; i < 16; ++i) {
+ m_buffer.putByteUnchecked(bytes[i]);
+ }
+ }
+
+ void int64Constant(int64_t i) {
+ m_buffer.ensureSpace(sizeof(int64_t));
+ m_buffer.putInt64Unchecked(i);
+ }
+
+ void int32Constant(int32_t i) {
+ m_buffer.ensureSpace(sizeof(int32_t));
+ m_buffer.putIntUnchecked(i);
+ }
+
+ // Administrative methods:
+
+ size_t size() const { return m_buffer.size(); }
+ const unsigned char* buffer() const { return m_buffer.buffer(); }
+ unsigned char* data() { return m_buffer.data(); }
+ bool oom() const { return m_buffer.oom(); }
+ bool reserve(size_t size) { return m_buffer.reserve(size); }
+ bool swapBuffer(wasm::Bytes& other) { return m_buffer.swap(other); }
+ bool isAligned(int alignment) const {
+ return m_buffer.isAligned(alignment);
+ }
+
+ [[nodiscard]] bool append(const unsigned char* values, size_t size) {
+ return m_buffer.append(values, size);
+ }
+
+ private:
+ // Internals; ModRm and REX formatters.
+
+ // Byte operand register spl & above requir a REX prefix, which precludes
+ // use of the h registers in the same instruction.
+ static bool byteRegRequiresRex(RegisterID reg) {
+#ifdef JS_CODEGEN_X64
+ return reg >= rsp;
+#else
+ return false;
+#endif
+ }
+
+ // For non-byte sizes, registers r8 & above always require a REX prefix.
+ static bool regRequiresRex(RegisterID reg) {
+#ifdef JS_CODEGEN_X64
+ return reg >= r8;
+#else
+ return false;
+#endif
+ }
+
+#ifdef JS_CODEGEN_X64
+ // Format a REX prefix byte.
+ void emitRex(bool w, int r, int x, int b) {
+ m_buffer.putByteUnchecked(PRE_REX | ((int)w << 3) | ((r >> 3) << 2) |
+ ((x >> 3) << 1) | (b >> 3));
+ }
+
+ // Used to plant a REX byte with REX.w set (for 64-bit operations).
+ void emitRexW(int r, int x, int b) { emitRex(true, r, x, b); }
+
+ // Used for operations with byte operands - use byteRegRequiresRex() to
+ // check register operands, regRequiresRex() to check other registers
+ // (i.e. address base & index).
+ //
+ // NB: WebKit's use of emitRexIf() is limited such that the
+ // reqRequiresRex() checks are not needed. SpiderMonkey extends
+ // oneByteOp8 and twoByteOp8 functionality such that r, x, and b
+ // can all be used.
+ void emitRexIf(bool condition, int r, int x, int b) {
+ if (condition || regRequiresRex(RegisterID(r)) ||
+ regRequiresRex(RegisterID(x)) || regRequiresRex(RegisterID(b))) {
+ emitRex(false, r, x, b);
+ }
+ }
+
+ // Used for word sized operations, will plant a REX prefix if necessary
+ // (if any register is r8 or above).
+ void emitRexIfNeeded(int r, int x, int b) { emitRexIf(false, r, x, b); }
+#else
+ // No REX prefix bytes on 32-bit x86.
+ void emitRexIf(bool condition, int, int, int) {
+ MOZ_ASSERT(!condition, "32-bit x86 should never use a REX prefix");
+ }
+ void emitRexIfNeeded(int, int, int) {}
+#endif
+
+ void putModRm(ModRmMode mode, RegisterID rm, int reg) {
+ m_buffer.putByteUnchecked((mode << 6) | ((reg & 7) << 3) | (rm & 7));
+ }
+
+ void putModRmSib(ModRmMode mode, RegisterID base, RegisterID index,
+ int scale, int reg) {
+ MOZ_ASSERT(mode != ModRmRegister);
+
+ putModRm(mode, hasSib, reg);
+ m_buffer.putByteUnchecked((scale << 6) | ((index & 7) << 3) | (base & 7));
+ }
+
+ void registerModRM(RegisterID rm, int reg) {
+ putModRm(ModRmRegister, rm, reg);
+ }
+
+ void memoryModRM(int32_t offset, RegisterID base, int reg) {
+ // A base of esp or r12 would be interpreted as a sib, so force a
+ // sib with no index & put the base in there.
+#ifdef JS_CODEGEN_X64
+ if ((base == hasSib) || (base == hasSib2)) {
+#else
+ if (base == hasSib) {
+#endif
+ if (!offset) { // No need to check if the base is noBase, since we know
+ // it is hasSib!
+ putModRmSib(ModRmMemoryNoDisp, base, noIndex, 0, reg);
+ } else if (CAN_SIGN_EXTEND_8_32(offset)) {
+ putModRmSib(ModRmMemoryDisp8, base, noIndex, 0, reg);
+ m_buffer.putByteUnchecked(offset);
+ } else {
+ putModRmSib(ModRmMemoryDisp32, base, noIndex, 0, reg);
+ m_buffer.putIntUnchecked(offset);
+ }
+ } else {
+#ifdef JS_CODEGEN_X64
+ if (!offset && (base != noBase) && (base != noBase2)) {
+#else
+ if (!offset && (base != noBase)) {
+#endif
+ putModRm(ModRmMemoryNoDisp, base, reg);
+ } else if (CAN_SIGN_EXTEND_8_32(offset)) {
+ putModRm(ModRmMemoryDisp8, base, reg);
+ m_buffer.putByteUnchecked(offset);
+ } else {
+ putModRm(ModRmMemoryDisp32, base, reg);
+ m_buffer.putIntUnchecked(offset);
+ }
+ }
+ }
+
+ void memoryModRM_disp32(int32_t offset, RegisterID base, int reg) {
+ // A base of esp or r12 would be interpreted as a sib, so force a
+ // sib with no index & put the base in there.
+#ifdef JS_CODEGEN_X64
+ if ((base == hasSib) || (base == hasSib2)) {
+#else
+ if (base == hasSib) {
+#endif
+ putModRmSib(ModRmMemoryDisp32, base, noIndex, 0, reg);
+ m_buffer.putIntUnchecked(offset);
+ } else {
+ putModRm(ModRmMemoryDisp32, base, reg);
+ m_buffer.putIntUnchecked(offset);
+ }
+ }
+
+ void memoryModRM(int32_t offset, RegisterID base, RegisterID index,
+ int scale, int reg) {
+ MOZ_ASSERT(index != noIndex);
+
+#ifdef JS_CODEGEN_X64
+ if (!offset && (base != noBase) && (base != noBase2)) {
+#else
+ if (!offset && (base != noBase)) {
+#endif
+ putModRmSib(ModRmMemoryNoDisp, base, index, scale, reg);
+ } else if (CAN_SIGN_EXTEND_8_32(offset)) {
+ putModRmSib(ModRmMemoryDisp8, base, index, scale, reg);
+ m_buffer.putByteUnchecked(offset);
+ } else {
+ putModRmSib(ModRmMemoryDisp32, base, index, scale, reg);
+ m_buffer.putIntUnchecked(offset);
+ }
+ }
+
+ void memoryModRM_disp32(int32_t offset, RegisterID index, int scale,
+ int reg) {
+ MOZ_ASSERT(index != noIndex);
+
+ // NB: the base-less memoryModRM overloads generate different code
+ // then the base-full memoryModRM overloads in the base == noBase
+ // case. The base-less overloads assume that the desired effective
+ // address is:
+ //
+ // reg := [scaled index] + disp32
+ //
+ // which means the mod needs to be ModRmMemoryNoDisp. The base-full
+ // overloads pass ModRmMemoryDisp32 in all cases and thus, when
+ // base == noBase (== ebp), the effective address is:
+ //
+ // reg := [scaled index] + disp32 + [ebp]
+ //
+ // See Intel developer manual, Vol 2, 2.1.5, Table 2-3.
+ putModRmSib(ModRmMemoryNoDisp, noBase, index, scale, reg);
+ m_buffer.putIntUnchecked(offset);
+ }
+
+ void memoryModRM_disp32(const void* address, int reg) {
+ int32_t disp = AddressImmediate(address);
+
+#ifdef JS_CODEGEN_X64
+ // On x64-64, non-RIP-relative absolute mode requires a SIB.
+ putModRmSib(ModRmMemoryNoDisp, noBase, noIndex, 0, reg);
+#else
+ // noBase + ModRmMemoryNoDisp means noBase + ModRmMemoryDisp32!
+ putModRm(ModRmMemoryNoDisp, noBase, reg);
+#endif
+ m_buffer.putIntUnchecked(disp);
+ }
+
+ void memoryModRM(const void* address, int reg) {
+ memoryModRM_disp32(address, reg);
+ }
+
+ void threeOpVex(VexOperandType p, int r, int x, int b, int m, int w, int v,
+ int l, int opcode) {
+ m_buffer.ensureSpace(MaxInstructionSize);
+
+ if (v == invalid_xmm) {
+ v = XMMRegisterID(0);
+ }
+
+ if (x == 0 && b == 0 && m == 1 && w == 0) {
+ // Two byte VEX.
+ m_buffer.putByteUnchecked(PRE_VEX_C5);
+ m_buffer.putByteUnchecked(((r << 7) | (v << 3) | (l << 2) | p) ^ 0xf8);
+ } else {
+ // Three byte VEX.
+ m_buffer.putByteUnchecked(PRE_VEX_C4);
+ m_buffer.putByteUnchecked(((r << 7) | (x << 6) | (b << 5) | m) ^ 0xe0);
+ m_buffer.putByteUnchecked(((w << 7) | (v << 3) | (l << 2) | p) ^ 0x78);
+ }
+
+ m_buffer.putByteUnchecked(opcode);
+ }
+
+ AssemblerBuffer m_buffer;
+ } m_formatter;
+
+ bool useVEX_;
+};
+
+} // namespace X86Encoding
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_BaseAssembler_x86_shared_h */
diff --git a/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp b/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp
new file mode 100644
index 0000000000..07849ccd49
--- /dev/null
+++ b/js/src/jit/x86-shared/CodeGenerator-x86-shared.cpp
@@ -0,0 +1,3888 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86-shared/CodeGenerator-x86-shared.h"
+
+#include "mozilla/DebugOnly.h"
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/InlineScriptTree.h"
+#include "jit/JitRuntime.h"
+#include "jit/RangeAnalysis.h"
+#include "jit/ReciprocalMulConstants.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+#include "util/DifferentialTesting.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::Abs;
+using mozilla::DebugOnly;
+using mozilla::FloorLog2;
+using mozilla::NegativeInfinity;
+
+using JS::GenericNaN;
+
+namespace js {
+namespace jit {
+
+CodeGeneratorX86Shared::CodeGeneratorX86Shared(MIRGenerator* gen,
+ LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorShared(gen, graph, masm) {}
+
+#ifdef JS_PUNBOX64
+Operand CodeGeneratorX86Shared::ToOperandOrRegister64(
+ const LInt64Allocation input) {
+ return ToOperand(input.value());
+}
+#else
+Register64 CodeGeneratorX86Shared::ToOperandOrRegister64(
+ const LInt64Allocation input) {
+ return ToRegister64(input);
+}
+#endif
+
+void OutOfLineBailout::accept(CodeGeneratorX86Shared* codegen) {
+ codegen->visitOutOfLineBailout(this);
+}
+
+void CodeGeneratorX86Shared::emitBranch(Assembler::Condition cond,
+ MBasicBlock* mirTrue,
+ MBasicBlock* mirFalse,
+ Assembler::NaNCond ifNaN) {
+ if (ifNaN == Assembler::NaN_IsFalse) {
+ jumpToBlock(mirFalse, Assembler::Parity);
+ } else if (ifNaN == Assembler::NaN_IsTrue) {
+ jumpToBlock(mirTrue, Assembler::Parity);
+ }
+
+ if (isNextBlock(mirFalse->lir())) {
+ jumpToBlock(mirTrue, cond);
+ } else {
+ jumpToBlock(mirFalse, Assembler::InvertCondition(cond));
+ jumpToBlock(mirTrue);
+ }
+}
+
+void CodeGenerator::visitDouble(LDouble* ins) {
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantDouble(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitFloat32(LFloat32* ins) {
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantFloat32(ins->value(), ToFloatRegister(out));
+}
+
+void CodeGenerator::visitTestIAndBranch(LTestIAndBranch* test) {
+ Register input = ToRegister(test->input());
+ masm.test32(input, input);
+ emitBranch(Assembler::NonZero, test->ifTrue(), test->ifFalse());
+}
+
+void CodeGenerator::visitTestDAndBranch(LTestDAndBranch* test) {
+ const LAllocation* opd = test->input();
+
+ // vucomisd flags:
+ // Z P C
+ // ---------
+ // NaN 1 1 1
+ // > 0 0 0
+ // < 0 0 1
+ // = 1 0 0
+ //
+ // NaN is falsey, so comparing against 0 and then using the Z flag is
+ // enough to determine which branch to take.
+ ScratchDoubleScope scratch(masm);
+ masm.zeroDouble(scratch);
+ masm.vucomisd(scratch, ToFloatRegister(opd));
+ emitBranch(Assembler::NotEqual, test->ifTrue(), test->ifFalse());
+}
+
+void CodeGenerator::visitTestFAndBranch(LTestFAndBranch* test) {
+ const LAllocation* opd = test->input();
+ // vucomiss flags are the same as doubles; see comment above
+ {
+ ScratchFloat32Scope scratch(masm);
+ masm.zeroFloat32(scratch);
+ masm.vucomiss(scratch, ToFloatRegister(opd));
+ }
+ emitBranch(Assembler::NotEqual, test->ifTrue(), test->ifFalse());
+}
+
+void CodeGeneratorX86Shared::emitCompare(MCompare::CompareType type,
+ const LAllocation* left,
+ const LAllocation* right) {
+#ifdef JS_CODEGEN_X64
+ if (type == MCompare::Compare_Object || type == MCompare::Compare_Symbol ||
+ type == MCompare::Compare_UIntPtr ||
+ type == MCompare::Compare_RefOrNull) {
+ if (right->isConstant()) {
+ MOZ_ASSERT(type == MCompare::Compare_UIntPtr);
+ masm.cmpPtr(ToRegister(left), Imm32(ToInt32(right)));
+ } else {
+ masm.cmpPtr(ToRegister(left), ToOperand(right));
+ }
+ return;
+ }
+#endif
+
+ if (right->isConstant()) {
+ masm.cmp32(ToRegister(left), Imm32(ToInt32(right)));
+ } else {
+ masm.cmp32(ToRegister(left), ToOperand(right));
+ }
+}
+
+void CodeGenerator::visitCompare(LCompare* comp) {
+ MCompare* mir = comp->mir();
+ emitCompare(mir->compareType(), comp->left(), comp->right());
+ masm.emitSet(JSOpToCondition(mir->compareType(), comp->jsop()),
+ ToRegister(comp->output()));
+}
+
+void CodeGenerator::visitCompareAndBranch(LCompareAndBranch* comp) {
+ MCompare* mir = comp->cmpMir();
+ emitCompare(mir->compareType(), comp->left(), comp->right());
+ Assembler::Condition cond = JSOpToCondition(mir->compareType(), comp->jsop());
+ emitBranch(cond, comp->ifTrue(), comp->ifFalse());
+}
+
+void CodeGenerator::visitCompareD(LCompareD* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+
+ Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
+ if (comp->mir()->operandsAreNeverNaN()) {
+ nanCond = Assembler::NaN_HandledByCond;
+ }
+
+ masm.compareDouble(cond, lhs, rhs);
+ masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
+ ToRegister(comp->output()), nanCond);
+}
+
+void CodeGenerator::visitCompareF(LCompareF* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond = JSOpToDoubleCondition(comp->mir()->jsop());
+
+ Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
+ if (comp->mir()->operandsAreNeverNaN()) {
+ nanCond = Assembler::NaN_HandledByCond;
+ }
+
+ masm.compareFloat(cond, lhs, rhs);
+ masm.emitSet(Assembler::ConditionFromDoubleCondition(cond),
+ ToRegister(comp->output()), nanCond);
+}
+
+void CodeGenerator::visitNotI(LNotI* ins) {
+ masm.cmp32(ToRegister(ins->input()), Imm32(0));
+ masm.emitSet(Assembler::Equal, ToRegister(ins->output()));
+}
+
+void CodeGenerator::visitNotD(LNotD* ins) {
+ FloatRegister opd = ToFloatRegister(ins->input());
+
+ // Not returns true if the input is a NaN. We don't have to worry about
+ // it if we know the input is never NaN though.
+ Assembler::NaNCond nanCond = Assembler::NaN_IsTrue;
+ if (ins->mir()->operandIsNeverNaN()) {
+ nanCond = Assembler::NaN_HandledByCond;
+ }
+
+ ScratchDoubleScope scratch(masm);
+ masm.zeroDouble(scratch);
+ masm.compareDouble(Assembler::DoubleEqualOrUnordered, opd, scratch);
+ masm.emitSet(Assembler::Equal, ToRegister(ins->output()), nanCond);
+}
+
+void CodeGenerator::visitNotF(LNotF* ins) {
+ FloatRegister opd = ToFloatRegister(ins->input());
+
+ // Not returns true if the input is a NaN. We don't have to worry about
+ // it if we know the input is never NaN though.
+ Assembler::NaNCond nanCond = Assembler::NaN_IsTrue;
+ if (ins->mir()->operandIsNeverNaN()) {
+ nanCond = Assembler::NaN_HandledByCond;
+ }
+
+ ScratchFloat32Scope scratch(masm);
+ masm.zeroFloat32(scratch);
+ masm.compareFloat(Assembler::DoubleEqualOrUnordered, opd, scratch);
+ masm.emitSet(Assembler::Equal, ToRegister(ins->output()), nanCond);
+}
+
+void CodeGenerator::visitCompareDAndBranch(LCompareDAndBranch* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+
+ Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
+ if (comp->cmpMir()->operandsAreNeverNaN()) {
+ nanCond = Assembler::NaN_HandledByCond;
+ }
+
+ masm.compareDouble(cond, lhs, rhs);
+ emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
+ comp->ifFalse(), nanCond);
+}
+
+void CodeGenerator::visitCompareFAndBranch(LCompareFAndBranch* comp) {
+ FloatRegister lhs = ToFloatRegister(comp->left());
+ FloatRegister rhs = ToFloatRegister(comp->right());
+
+ Assembler::DoubleCondition cond =
+ JSOpToDoubleCondition(comp->cmpMir()->jsop());
+
+ Assembler::NaNCond nanCond = Assembler::NaNCondFromDoubleCondition(cond);
+ if (comp->cmpMir()->operandsAreNeverNaN()) {
+ nanCond = Assembler::NaN_HandledByCond;
+ }
+
+ masm.compareFloat(cond, lhs, rhs);
+ emitBranch(Assembler::ConditionFromDoubleCondition(cond), comp->ifTrue(),
+ comp->ifFalse(), nanCond);
+}
+
+void CodeGenerator::visitWasmStackArg(LWasmStackArg* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(StackPointer, mir->spOffset());
+ if (ins->arg()->isConstant()) {
+ masm.storePtr(ImmWord(ToInt32(ins->arg())), dst);
+ } else if (ins->arg()->isGeneralReg()) {
+ masm.storePtr(ToRegister(ins->arg()), dst);
+ } else {
+ switch (mir->input()->type()) {
+ case MIRType::Double:
+ masm.storeDouble(ToFloatRegister(ins->arg()), dst);
+ return;
+ case MIRType::Float32:
+ masm.storeFloat32(ToFloatRegister(ins->arg()), dst);
+ return;
+#ifdef ENABLE_WASM_SIMD
+ case MIRType::Simd128:
+ masm.storeUnalignedSimd128(ToFloatRegister(ins->arg()), dst);
+ return;
+#endif
+ default:
+ break;
+ }
+ MOZ_CRASH("unexpected mir type in WasmStackArg");
+ }
+}
+
+void CodeGenerator::visitWasmStackArgI64(LWasmStackArgI64* ins) {
+ const MWasmStackArg* mir = ins->mir();
+ Address dst(StackPointer, mir->spOffset());
+ if (IsConstant(ins->arg())) {
+ masm.store64(Imm64(ToInt64(ins->arg())), dst);
+ } else {
+ masm.store64(ToRegister64(ins->arg()), dst);
+ }
+}
+
+void CodeGenerator::visitWasmSelect(LWasmSelect* ins) {
+ MIRType mirType = ins->mir()->type();
+
+ Register cond = ToRegister(ins->condExpr());
+ Operand falseExpr = ToOperand(ins->falseExpr());
+
+ masm.test32(cond, cond);
+
+ if (mirType == MIRType::Int32 || mirType == MIRType::RefOrNull) {
+ Register out = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->trueExpr()) == out,
+ "true expr input is reused for output");
+ if (mirType == MIRType::Int32) {
+ masm.cmovz32(falseExpr, out);
+ } else {
+ masm.cmovzPtr(falseExpr, out);
+ }
+ return;
+ }
+
+ FloatRegister out = ToFloatRegister(ins->output());
+ MOZ_ASSERT(ToFloatRegister(ins->trueExpr()) == out,
+ "true expr input is reused for output");
+
+ Label done;
+ masm.j(Assembler::NonZero, &done);
+
+ if (mirType == MIRType::Float32) {
+ if (falseExpr.kind() == Operand::FPREG) {
+ masm.moveFloat32(ToFloatRegister(ins->falseExpr()), out);
+ } else {
+ masm.loadFloat32(falseExpr, out);
+ }
+ } else if (mirType == MIRType::Double) {
+ if (falseExpr.kind() == Operand::FPREG) {
+ masm.moveDouble(ToFloatRegister(ins->falseExpr()), out);
+ } else {
+ masm.loadDouble(falseExpr, out);
+ }
+ } else if (mirType == MIRType::Simd128) {
+ if (falseExpr.kind() == Operand::FPREG) {
+ masm.moveSimd128(ToFloatRegister(ins->falseExpr()), out);
+ } else {
+ masm.loadUnalignedSimd128(falseExpr, out);
+ }
+ } else {
+ MOZ_CRASH("unhandled type in visitWasmSelect!");
+ }
+
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitWasmReinterpret(LWasmReinterpret* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MWasmReinterpret* ins = lir->mir();
+
+ MIRType to = ins->type();
+#ifdef DEBUG
+ MIRType from = ins->input()->type();
+#endif
+
+ switch (to) {
+ case MIRType::Int32:
+ MOZ_ASSERT(from == MIRType::Float32);
+ masm.vmovd(ToFloatRegister(lir->input()), ToRegister(lir->output()));
+ break;
+ case MIRType::Float32:
+ MOZ_ASSERT(from == MIRType::Int32);
+ masm.vmovd(ToRegister(lir->input()), ToFloatRegister(lir->output()));
+ break;
+ case MIRType::Double:
+ case MIRType::Int64:
+ MOZ_CRASH("not handled by this LIR opcode");
+ default:
+ MOZ_CRASH("unexpected WasmReinterpret");
+ }
+}
+
+void CodeGenerator::visitAsmJSLoadHeap(LAsmJSLoadHeap* ins) {
+ const MAsmJSLoadHeap* mir = ins->mir();
+ MOZ_ASSERT(mir->access().offset() == 0);
+
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+ AnyRegister out = ToAnyRegister(ins->output());
+
+ Scalar::Type accessType = mir->accessType();
+
+ OutOfLineLoadTypedArrayOutOfBounds* ool = nullptr;
+ if (mir->needsBoundsCheck()) {
+ ool = new (alloc()) OutOfLineLoadTypedArrayOutOfBounds(out, accessType);
+ addOutOfLineCode(ool, mir);
+
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ToRegister(ptr),
+ ToRegister(boundsCheckLimit), ool->entry());
+ }
+
+ Operand srcAddr = toMemoryAccessOperand(ins, 0);
+ masm.wasmLoad(mir->access(), srcAddr, out);
+
+ if (ool) {
+ masm.bind(ool->rejoin());
+ }
+}
+
+void CodeGeneratorX86Shared::visitOutOfLineLoadTypedArrayOutOfBounds(
+ OutOfLineLoadTypedArrayOutOfBounds* ool) {
+ switch (ool->viewType()) {
+ case Scalar::Int64:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::Simd128:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ case Scalar::Float32:
+ masm.loadConstantFloat32(float(GenericNaN()), ool->dest().fpu());
+ break;
+ case Scalar::Float64:
+ masm.loadConstantDouble(GenericNaN(), ool->dest().fpu());
+ break;
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Uint8Clamped:
+ Register destReg = ool->dest().gpr();
+ masm.mov(ImmWord(0), destReg);
+ break;
+ }
+ masm.jmp(ool->rejoin());
+}
+
+void CodeGenerator::visitAsmJSStoreHeap(LAsmJSStoreHeap* ins) {
+ const MAsmJSStoreHeap* mir = ins->mir();
+
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* value = ins->value();
+ const LAllocation* boundsCheckLimit = ins->boundsCheckLimit();
+
+ Scalar::Type accessType = mir->accessType();
+ canonicalizeIfDeterministic(accessType, value);
+
+ Label rejoin;
+ if (mir->needsBoundsCheck()) {
+ masm.wasmBoundsCheck32(Assembler::AboveOrEqual, ToRegister(ptr),
+ ToRegister(boundsCheckLimit), &rejoin);
+ }
+
+ Operand dstAddr = toMemoryAccessOperand(ins, 0);
+ masm.wasmStore(mir->access(), ToAnyRegister(value), dstAddr);
+
+ if (rejoin.used()) {
+ masm.bind(&rejoin);
+ }
+}
+
+void CodeGenerator::visitWasmAddOffset(LWasmAddOffset* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register base = ToRegister(lir->base());
+ Register out = ToRegister(lir->output());
+
+ if (base != out) {
+ masm.move32(base, out);
+ }
+ masm.add32(Imm32(mir->offset()), out);
+ OutOfLineAbortingWasmTrap* ool = new (alloc())
+ OutOfLineAbortingWasmTrap(mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
+ addOutOfLineCode(ool, mir);
+ masm.j(Assembler::CarrySet, ool->entry());
+}
+
+void CodeGenerator::visitWasmAddOffset64(LWasmAddOffset64* lir) {
+ MWasmAddOffset* mir = lir->mir();
+ Register64 base = ToRegister64(lir->base());
+ Register64 out = ToOutRegister64(lir);
+
+ if (base != out) {
+ masm.move64(base, out);
+ }
+ masm.add64(Imm64(mir->offset()), out);
+ OutOfLineAbortingWasmTrap* ool = new (alloc())
+ OutOfLineAbortingWasmTrap(mir->bytecodeOffset(), wasm::Trap::OutOfBounds);
+ addOutOfLineCode(ool, mir);
+ masm.j(Assembler::CarrySet, ool->entry());
+}
+
+void CodeGenerator::visitWasmTruncateToInt32(LWasmTruncateToInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register output = ToRegister(lir->output());
+
+ MWasmTruncateToInt32* mir = lir->mir();
+ MIRType inputType = mir->input()->type();
+
+ MOZ_ASSERT(inputType == MIRType::Double || inputType == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ Label* oolEntry = ool->entry();
+ if (mir->isUnsigned()) {
+ if (inputType == MIRType::Double) {
+ masm.wasmTruncateDoubleToUInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else if (inputType == MIRType::Float32) {
+ masm.wasmTruncateFloat32ToUInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+ if (mir->isSaturating()) {
+ masm.bind(ool->rejoin());
+ }
+ return;
+ }
+
+ if (inputType == MIRType::Double) {
+ masm.wasmTruncateDoubleToInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else if (inputType == MIRType::Float32) {
+ masm.wasmTruncateFloat32ToInt32(input, output, mir->isSaturating(),
+ oolEntry);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+
+ masm.bind(ool->rejoin());
+}
+
+bool CodeGeneratorX86Shared::generateOutOfLineCode() {
+ if (!CodeGeneratorShared::generateOutOfLineCode()) {
+ return false;
+ }
+
+ if (deoptLabel_.used()) {
+ // All non-table-based bailouts will go here.
+ masm.bind(&deoptLabel_);
+
+ // Push the frame size, so the handler can recover the IonScript.
+ masm.push(Imm32(frameSize()));
+
+ TrampolinePtr handler = gen->jitRuntime()->getGenericBailoutHandler();
+ masm.jump(handler);
+ }
+
+ return !masm.oom();
+}
+
+class BailoutJump {
+ Assembler::Condition cond_;
+
+ public:
+ explicit BailoutJump(Assembler::Condition cond) : cond_(cond) {}
+#ifdef JS_CODEGEN_X86
+ void operator()(MacroAssembler& masm, uint8_t* code) const {
+ masm.j(cond_, ImmPtr(code), RelocationKind::HARDCODED);
+ }
+#endif
+ void operator()(MacroAssembler& masm, Label* label) const {
+ masm.j(cond_, label);
+ }
+};
+
+class BailoutLabel {
+ Label* label_;
+
+ public:
+ explicit BailoutLabel(Label* label) : label_(label) {}
+#ifdef JS_CODEGEN_X86
+ void operator()(MacroAssembler& masm, uint8_t* code) const {
+ masm.retarget(label_, ImmPtr(code), RelocationKind::HARDCODED);
+ }
+#endif
+ void operator()(MacroAssembler& masm, Label* label) const {
+ masm.retarget(label_, label);
+ }
+};
+
+template <typename T>
+void CodeGeneratorX86Shared::bailout(const T& binder, LSnapshot* snapshot) {
+ encode(snapshot);
+
+ // All bailout code is associated with the bytecodeSite of the block we are
+ // bailing out from.
+ InlineScriptTree* tree = snapshot->mir()->block()->trackedTree();
+ OutOfLineBailout* ool = new (alloc()) OutOfLineBailout(snapshot);
+ addOutOfLineCode(ool,
+ new (alloc()) BytecodeSite(tree, tree->script()->code()));
+
+ binder(masm, ool->entry());
+}
+
+void CodeGeneratorX86Shared::bailoutIf(Assembler::Condition condition,
+ LSnapshot* snapshot) {
+ bailout(BailoutJump(condition), snapshot);
+}
+
+void CodeGeneratorX86Shared::bailoutIf(Assembler::DoubleCondition condition,
+ LSnapshot* snapshot) {
+ MOZ_ASSERT(Assembler::NaNCondFromDoubleCondition(condition) ==
+ Assembler::NaN_HandledByCond);
+ bailoutIf(Assembler::ConditionFromDoubleCondition(condition), snapshot);
+}
+
+void CodeGeneratorX86Shared::bailoutFrom(Label* label, LSnapshot* snapshot) {
+ MOZ_ASSERT_IF(!masm.oom(), label->used() && !label->bound());
+ bailout(BailoutLabel(label), snapshot);
+}
+
+void CodeGeneratorX86Shared::bailout(LSnapshot* snapshot) {
+ Label label;
+ masm.jump(&label);
+ bailoutFrom(&label, snapshot);
+}
+
+void CodeGeneratorX86Shared::visitOutOfLineBailout(OutOfLineBailout* ool) {
+ masm.push(Imm32(ool->snapshot()->snapshotOffset()));
+ masm.jmp(&deoptLabel_);
+}
+
+void CodeGenerator::visitMinMaxD(LMinMaxD* ins) {
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+#ifdef DEBUG
+ FloatRegister output = ToFloatRegister(ins->output());
+ MOZ_ASSERT(first == output);
+#endif
+
+ bool handleNaN = !ins->mir()->range() || ins->mir()->range()->canBeNaN();
+
+ if (ins->mir()->isMax()) {
+ masm.maxDouble(second, first, handleNaN);
+ } else {
+ masm.minDouble(second, first, handleNaN);
+ }
+}
+
+void CodeGenerator::visitMinMaxF(LMinMaxF* ins) {
+ FloatRegister first = ToFloatRegister(ins->first());
+ FloatRegister second = ToFloatRegister(ins->second());
+#ifdef DEBUG
+ FloatRegister output = ToFloatRegister(ins->output());
+ MOZ_ASSERT(first == output);
+#endif
+
+ bool handleNaN = !ins->mir()->range() || ins->mir()->range()->canBeNaN();
+
+ if (ins->mir()->isMax()) {
+ masm.maxFloat32(second, first, handleNaN);
+ } else {
+ masm.minFloat32(second, first, handleNaN);
+ }
+}
+
+void CodeGenerator::visitClzI(LClzI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+ bool knownNotZero = ins->mir()->operandIsNeverZero();
+
+ masm.clz32(input, output, knownNotZero);
+}
+
+void CodeGenerator::visitCtzI(LCtzI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+ bool knownNotZero = ins->mir()->operandIsNeverZero();
+
+ masm.ctz32(input, output, knownNotZero);
+}
+
+void CodeGenerator::visitPopcntI(LPopcntI* ins) {
+ Register input = ToRegister(ins->input());
+ Register output = ToRegister(ins->output());
+ Register temp =
+ ins->temp0()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp0());
+
+ masm.popcnt32(input, output, temp);
+}
+
+void CodeGenerator::visitPowHalfD(LPowHalfD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ FloatRegister output = ToFloatRegister(ins->output());
+
+ ScratchDoubleScope scratch(masm);
+
+ Label done, sqrt;
+
+ if (!ins->mir()->operandIsNeverNegativeInfinity()) {
+ // Branch if not -Infinity.
+ masm.loadConstantDouble(NegativeInfinity<double>(), scratch);
+
+ Assembler::DoubleCondition cond = Assembler::DoubleNotEqualOrUnordered;
+ if (ins->mir()->operandIsNeverNaN()) {
+ cond = Assembler::DoubleNotEqual;
+ }
+ masm.branchDouble(cond, input, scratch, &sqrt);
+
+ // Math.pow(-Infinity, 0.5) == Infinity.
+ masm.zeroDouble(output);
+ masm.subDouble(scratch, output);
+ masm.jump(&done);
+
+ masm.bind(&sqrt);
+ }
+
+ if (!ins->mir()->operandIsNeverNegativeZero()) {
+ // Math.pow(-0, 0.5) == 0 == Math.pow(0, 0.5).
+ // Adding 0 converts any -0 to 0.
+ masm.zeroDouble(scratch);
+ masm.addDouble(input, scratch);
+ masm.vsqrtsd(scratch, output, output);
+ } else {
+ masm.vsqrtsd(input, output, output);
+ }
+
+ masm.bind(&done);
+}
+
+class OutOfLineUndoALUOperation
+ : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
+ LInstruction* ins_;
+
+ public:
+ explicit OutOfLineUndoALUOperation(LInstruction* ins) : ins_(ins) {}
+
+ virtual void accept(CodeGeneratorX86Shared* codegen) override {
+ codegen->visitOutOfLineUndoALUOperation(this);
+ }
+ LInstruction* ins() const { return ins_; }
+};
+
+void CodeGenerator::visitAddI(LAddI* ins) {
+ if (ins->rhs()->isConstant()) {
+ masm.addl(Imm32(ToInt32(ins->rhs())), ToOperand(ins->lhs()));
+ } else {
+ masm.addl(ToOperand(ins->rhs()), ToRegister(ins->lhs()));
+ }
+
+ if (ins->snapshot()) {
+ if (ins->recoversInput()) {
+ OutOfLineUndoALUOperation* ool =
+ new (alloc()) OutOfLineUndoALUOperation(ins);
+ addOutOfLineCode(ool, ins->mir());
+ masm.j(Assembler::Overflow, ool->entry());
+ } else {
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ }
+ }
+}
+
+void CodeGenerator::visitAddI64(LAddI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LAddI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LAddI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.add64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.add64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void CodeGenerator::visitSubI(LSubI* ins) {
+ if (ins->rhs()->isConstant()) {
+ masm.subl(Imm32(ToInt32(ins->rhs())), ToOperand(ins->lhs()));
+ } else {
+ masm.subl(ToOperand(ins->rhs()), ToRegister(ins->lhs()));
+ }
+
+ if (ins->snapshot()) {
+ if (ins->recoversInput()) {
+ OutOfLineUndoALUOperation* ool =
+ new (alloc()) OutOfLineUndoALUOperation(ins);
+ addOutOfLineCode(ool, ins->mir());
+ masm.j(Assembler::Overflow, ool->entry());
+ } else {
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ }
+ }
+}
+
+void CodeGenerator::visitSubI64(LSubI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LSubI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LSubI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (IsConstant(rhs)) {
+ masm.sub64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ return;
+ }
+
+ masm.sub64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+}
+
+void CodeGeneratorX86Shared::visitOutOfLineUndoALUOperation(
+ OutOfLineUndoALUOperation* ool) {
+ LInstruction* ins = ool->ins();
+ Register reg = ToRegister(ins->getDef(0));
+
+ DebugOnly<LAllocation*> lhs = ins->getOperand(0);
+ LAllocation* rhs = ins->getOperand(1);
+
+ MOZ_ASSERT(reg == ToRegister(lhs));
+ MOZ_ASSERT_IF(rhs->isGeneralReg(), reg != ToRegister(rhs));
+
+ // Undo the effect of the ALU operation, which was performed on the output
+ // register and overflowed. Writing to the output register clobbered an
+ // input reg, and the original value of the input needs to be recovered
+ // to satisfy the constraint imposed by any RECOVERED_INPUT operands to
+ // the bailout snapshot.
+
+ if (rhs->isConstant()) {
+ Imm32 constant(ToInt32(rhs));
+ if (ins->isAddI()) {
+ masm.subl(constant, reg);
+ } else {
+ masm.addl(constant, reg);
+ }
+ } else {
+ if (ins->isAddI()) {
+ masm.subl(ToOperand(rhs), reg);
+ } else {
+ masm.addl(ToOperand(rhs), reg);
+ }
+ }
+
+ bailout(ool->ins()->snapshot());
+}
+
+class MulNegativeZeroCheck : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
+ LMulI* ins_;
+
+ public:
+ explicit MulNegativeZeroCheck(LMulI* ins) : ins_(ins) {}
+
+ virtual void accept(CodeGeneratorX86Shared* codegen) override {
+ codegen->visitMulNegativeZeroCheck(this);
+ }
+ LMulI* ins() const { return ins_; }
+};
+
+void CodeGenerator::visitMulI(LMulI* ins) {
+ const LAllocation* lhs = ins->lhs();
+ const LAllocation* rhs = ins->rhs();
+ MMul* mul = ins->mir();
+ MOZ_ASSERT_IF(mul->mode() == MMul::Integer,
+ !mul->canBeNegativeZero() && !mul->canOverflow());
+
+ if (rhs->isConstant()) {
+ // Bailout on -0.0
+ int32_t constant = ToInt32(rhs);
+ if (mul->canBeNegativeZero() && constant <= 0) {
+ Assembler::Condition bailoutCond =
+ (constant == 0) ? Assembler::Signed : Assembler::Equal;
+ masm.test32(ToRegister(lhs), ToRegister(lhs));
+ bailoutIf(bailoutCond, ins->snapshot());
+ }
+
+ switch (constant) {
+ case -1:
+ masm.negl(ToOperand(lhs));
+ break;
+ case 0:
+ masm.xorl(ToOperand(lhs), ToRegister(lhs));
+ return; // escape overflow check;
+ case 1:
+ // nop
+ return; // escape overflow check;
+ case 2:
+ masm.addl(ToOperand(lhs), ToRegister(lhs));
+ break;
+ default:
+ if (!mul->canOverflow() && constant > 0) {
+ // Use shift if cannot overflow and constant is power of 2
+ int32_t shift = FloorLog2(constant);
+ if ((1 << shift) == constant) {
+ masm.shll(Imm32(shift), ToRegister(lhs));
+ return;
+ }
+ }
+ masm.imull(Imm32(ToInt32(rhs)), ToRegister(lhs));
+ }
+
+ // Bailout on overflow
+ if (mul->canOverflow()) {
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ }
+ } else {
+ masm.imull(ToOperand(rhs), ToRegister(lhs));
+
+ // Bailout on overflow
+ if (mul->canOverflow()) {
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ }
+
+ if (mul->canBeNegativeZero()) {
+ // Jump to an OOL path if the result is 0.
+ MulNegativeZeroCheck* ool = new (alloc()) MulNegativeZeroCheck(ins);
+ addOutOfLineCode(ool, mul);
+
+ masm.test32(ToRegister(lhs), ToRegister(lhs));
+ masm.j(Assembler::Zero, ool->entry());
+ masm.bind(ool->rejoin());
+ }
+ }
+}
+
+void CodeGenerator::visitMulI64(LMulI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LMulI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LMulI64::Rhs);
+
+ MOZ_ASSERT(ToRegister64(lhs) == ToOutRegister64(lir));
+
+ if (IsConstant(rhs)) {
+ int64_t constant = ToInt64(rhs);
+ switch (constant) {
+ case -1:
+ masm.neg64(ToRegister64(lhs));
+ return;
+ case 0:
+ masm.xor64(ToRegister64(lhs), ToRegister64(lhs));
+ return;
+ case 1:
+ // nop
+ return;
+ case 2:
+ masm.add64(ToRegister64(lhs), ToRegister64(lhs));
+ return;
+ default:
+ if (constant > 0) {
+ // Use shift if constant is power of 2.
+ int32_t shift = mozilla::FloorLog2(constant);
+ if (int64_t(1) << shift == constant) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ return;
+ }
+ }
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(Imm64(constant), ToRegister64(lhs), temp);
+ }
+ } else {
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+ masm.mul64(ToOperandOrRegister64(rhs), ToRegister64(lhs), temp);
+ }
+}
+
+class ReturnZero : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
+ Register reg_;
+
+ public:
+ explicit ReturnZero(Register reg) : reg_(reg) {}
+
+ virtual void accept(CodeGeneratorX86Shared* codegen) override {
+ codegen->visitReturnZero(this);
+ }
+ Register reg() const { return reg_; }
+};
+
+void CodeGeneratorX86Shared::visitReturnZero(ReturnZero* ool) {
+ masm.mov(ImmWord(0), ool->reg());
+ masm.jmp(ool->rejoin());
+}
+
+void CodeGenerator::visitUDivOrMod(LUDivOrMod* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+
+ MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
+ MOZ_ASSERT(rhs != edx);
+ MOZ_ASSERT_IF(output == eax, ToRegister(ins->remainder()) == edx);
+
+ ReturnZero* ool = nullptr;
+
+ // Put the lhs in eax.
+ if (lhs != eax) {
+ masm.mov(lhs, eax);
+ }
+
+ // Prevent divide by zero.
+ if (ins->canBeDivideByZero()) {
+ masm.test32(rhs, rhs);
+ if (ins->mir()->isTruncated()) {
+ if (ins->trapOnError()) {
+ Label nonZero;
+ masm.j(Assembler::NonZero, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ ool = new (alloc()) ReturnZero(output);
+ masm.j(Assembler::Zero, ool->entry());
+ }
+ } else {
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ }
+ }
+
+ // Zero extend the lhs into edx to make (edx:eax), since udiv is 64-bit.
+ masm.mov(ImmWord(0), edx);
+ masm.udiv(rhs);
+
+ // If the remainder is > 0, bailout since this must be a double.
+ if (ins->mir()->isDiv() && !ins->mir()->toDiv()->canTruncateRemainder()) {
+ Register remainder = ToRegister(ins->remainder());
+ masm.test32(remainder, remainder);
+ bailoutIf(Assembler::NonZero, ins->snapshot());
+ }
+
+ // Unsigned div or mod can return a value that's not a signed int32.
+ // If our users aren't expecting that, bail.
+ if (!ins->mir()->isTruncated()) {
+ masm.test32(output, output);
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ }
+
+ if (ool) {
+ addOutOfLineCode(ool, ins->mir());
+ masm.bind(ool->rejoin());
+ }
+}
+
+void CodeGenerator::visitUDivOrModConstant(LUDivOrModConstant* ins) {
+ Register lhs = ToRegister(ins->numerator());
+ Register output = ToRegister(ins->output());
+ uint32_t d = ins->denominator();
+
+ // This emits the division answer into edx or the modulus answer into eax.
+ MOZ_ASSERT(output == eax || output == edx);
+ MOZ_ASSERT(lhs != eax && lhs != edx);
+ bool isDiv = (output == edx);
+
+ if (d == 0) {
+ if (ins->mir()->isTruncated()) {
+ if (ins->trapOnError()) {
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, ins->bytecodeOffset());
+ } else {
+ masm.xorl(output, output);
+ }
+ } else {
+ bailout(ins->snapshot());
+ }
+ return;
+ }
+
+ // The denominator isn't a power of 2 (see LDivPowTwoI and LModPowTwoI).
+ MOZ_ASSERT((d & (d - 1)) != 0);
+
+ auto rmc = ReciprocalMulConstants::computeUnsignedDivisionConstants(d);
+
+ // We first compute (M * n) >> 32, where M = rmc.multiplier.
+ masm.movl(Imm32(rmc.multiplier), eax);
+ masm.umull(lhs);
+ if (rmc.multiplier > UINT32_MAX) {
+ // M >= 2^32 and shift == 0 is impossible, as d >= 2 implies that
+ // ((M * n) >> (32 + shift)) >= n > floor(n/d) whenever n >= d,
+ // contradicting the proof of correctness in computeDivisionConstants.
+ MOZ_ASSERT(rmc.shiftAmount > 0);
+ MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 33));
+
+ // We actually computed edx = ((uint32_t(M) * n) >> 32) instead. Since
+ // (M * n) >> (32 + shift) is the same as (edx + n) >> shift, we can
+ // correct for the overflow. This case is a bit trickier than the signed
+ // case, though, as the (edx + n) addition itself can overflow; however,
+ // note that (edx + n) >> shift == (((n - edx) >> 1) + edx) >> (shift - 1),
+ // which is overflow-free. See Hacker's Delight, section 10-8 for details.
+
+ // Compute (n - edx) >> 1 into eax.
+ masm.movl(lhs, eax);
+ masm.subl(edx, eax);
+ masm.shrl(Imm32(1), eax);
+
+ // Finish the computation.
+ masm.addl(eax, edx);
+ masm.shrl(Imm32(rmc.shiftAmount - 1), edx);
+ } else {
+ masm.shrl(Imm32(rmc.shiftAmount), edx);
+ }
+
+ // We now have the truncated division value in edx. If we're
+ // computing a modulus or checking whether the division resulted
+ // in an integer, we need to multiply the obtained value by d and
+ // finish the computation/check.
+ if (!isDiv) {
+ masm.imull(Imm32(d), edx, edx);
+ masm.movl(lhs, eax);
+ masm.subl(edx, eax);
+
+ // The final result of the modulus op, just computed above by the
+ // sub instruction, can be a number in the range [2^31, 2^32). If
+ // this is the case and the modulus is not truncated, we must bail
+ // out.
+ if (!ins->mir()->isTruncated()) {
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ }
+ } else if (!ins->mir()->isTruncated()) {
+ masm.imull(Imm32(d), edx, eax);
+ masm.cmpl(lhs, eax);
+ bailoutIf(Assembler::NotEqual, ins->snapshot());
+ }
+}
+
+void CodeGeneratorX86Shared::visitMulNegativeZeroCheck(
+ MulNegativeZeroCheck* ool) {
+ LMulI* ins = ool->ins();
+ Register result = ToRegister(ins->output());
+ Operand lhsCopy = ToOperand(ins->lhsCopy());
+ Operand rhs = ToOperand(ins->rhs());
+ MOZ_ASSERT_IF(lhsCopy.kind() == Operand::REG, lhsCopy.reg() != result.code());
+
+ // Result is -0 if lhs or rhs is negative.
+ masm.movl(lhsCopy, result);
+ masm.orl(rhs, result);
+ bailoutIf(Assembler::Signed, ins->snapshot());
+
+ masm.mov(ImmWord(0), result);
+ masm.jmp(ool->rejoin());
+}
+
+void CodeGenerator::visitDivPowTwoI(LDivPowTwoI* ins) {
+ Register lhs = ToRegister(ins->numerator());
+ DebugOnly<Register> output = ToRegister(ins->output());
+
+ int32_t shift = ins->shift();
+ bool negativeDivisor = ins->negativeDivisor();
+ MDiv* mir = ins->mir();
+
+ // We use defineReuseInput so these should always be the same, which is
+ // convenient since all of our instructions here are two-address.
+ MOZ_ASSERT(lhs == output);
+
+ if (!mir->isTruncated() && negativeDivisor) {
+ // 0 divided by a negative number must return a double.
+ masm.test32(lhs, lhs);
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ }
+
+ if (shift) {
+ if (!mir->isTruncated()) {
+ // If the remainder is != 0, bailout since this must be a double.
+ masm.test32(lhs, Imm32(UINT32_MAX >> (32 - shift)));
+ bailoutIf(Assembler::NonZero, ins->snapshot());
+ }
+
+ if (mir->isUnsigned()) {
+ masm.shrl(Imm32(shift), lhs);
+ } else {
+ // Adjust the value so that shifting produces a correctly
+ // rounded result when the numerator is negative. See 10-1
+ // "Signed Division by a Known Power of 2" in Henry
+ // S. Warren, Jr.'s Hacker's Delight.
+ if (mir->canBeNegativeDividend() && mir->isTruncated()) {
+ // Note: There is no need to execute this code, which handles how to
+ // round the signed integer division towards 0, if we previously bailed
+ // due to a non-zero remainder.
+ Register lhsCopy = ToRegister(ins->numeratorCopy());
+ MOZ_ASSERT(lhsCopy != lhs);
+ if (shift > 1) {
+ // Copy the sign bit of the numerator. (= (2^32 - 1) or 0)
+ masm.sarl(Imm32(31), lhs);
+ }
+ // Divide by 2^(32 - shift)
+ // i.e. (= (2^32 - 1) / 2^(32 - shift) or 0)
+ // i.e. (= (2^shift - 1) or 0)
+ masm.shrl(Imm32(32 - shift), lhs);
+ // If signed, make any 1 bit below the shifted bits to bubble up, such
+ // that once shifted the value would be rounded towards 0.
+ masm.addl(lhsCopy, lhs);
+ }
+ masm.sarl(Imm32(shift), lhs);
+
+ if (negativeDivisor) {
+ masm.negl(lhs);
+ }
+ }
+ return;
+ }
+
+ if (negativeDivisor) {
+ // INT32_MIN / -1 overflows.
+ masm.negl(lhs);
+ if (!mir->isTruncated()) {
+ bailoutIf(Assembler::Overflow, ins->snapshot());
+ } else if (mir->trapOnError()) {
+ Label ok;
+ masm.j(Assembler::NoOverflow, &ok);
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
+ masm.bind(&ok);
+ }
+ } else if (mir->isUnsigned() && !mir->isTruncated()) {
+ // Unsigned division by 1 can overflow if output is not
+ // truncated.
+ masm.test32(lhs, lhs);
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ }
+}
+
+void CodeGenerator::visitDivOrModConstantI(LDivOrModConstantI* ins) {
+ Register lhs = ToRegister(ins->numerator());
+ Register output = ToRegister(ins->output());
+ int32_t d = ins->denominator();
+
+ // This emits the division answer into edx or the modulus answer into eax.
+ MOZ_ASSERT(output == eax || output == edx);
+ MOZ_ASSERT(lhs != eax && lhs != edx);
+ bool isDiv = (output == edx);
+
+ // The absolute value of the denominator isn't a power of 2 (see LDivPowTwoI
+ // and LModPowTwoI).
+ MOZ_ASSERT((Abs(d) & (Abs(d) - 1)) != 0);
+
+ // We will first divide by Abs(d), and negate the answer if d is negative.
+ // If desired, this can be avoided by generalizing computeDivisionConstants.
+ auto rmc = ReciprocalMulConstants::computeSignedDivisionConstants(Abs(d));
+
+ // We first compute (M * n) >> 32, where M = rmc.multiplier.
+ masm.movl(Imm32(rmc.multiplier), eax);
+ masm.imull(lhs);
+ if (rmc.multiplier > INT32_MAX) {
+ MOZ_ASSERT(rmc.multiplier < (int64_t(1) << 32));
+
+ // We actually computed edx = ((int32_t(M) * n) >> 32) instead. Since
+ // (M * n) >> 32 is the same as (edx + n), we can correct for the overflow.
+ // (edx + n) can't overflow, as n and edx have opposite signs because
+ // int32_t(M) is negative.
+ masm.addl(lhs, edx);
+ }
+ // (M * n) >> (32 + shift) is the truncated division answer if n is
+ // non-negative, as proved in the comments of computeDivisionConstants. We
+ // must add 1 later if n is negative to get the right answer in all cases.
+ masm.sarl(Imm32(rmc.shiftAmount), edx);
+
+ // We'll subtract -1 instead of adding 1, because (n < 0 ? -1 : 0) can be
+ // computed with just a sign-extending shift of 31 bits.
+ if (ins->canBeNegativeDividend()) {
+ masm.movl(lhs, eax);
+ masm.sarl(Imm32(31), eax);
+ masm.subl(eax, edx);
+ }
+
+ // After this, edx contains the correct truncated division result.
+ if (d < 0) {
+ masm.negl(edx);
+ }
+
+ if (!isDiv) {
+ masm.imull(Imm32(-d), edx, eax);
+ masm.addl(lhs, eax);
+ }
+
+ if (!ins->mir()->isTruncated()) {
+ if (isDiv) {
+ // This is a division op. Multiply the obtained value by d to check if
+ // the correct answer is an integer. This cannot overflow, since |d| > 1.
+ masm.imull(Imm32(d), edx, eax);
+ masm.cmp32(lhs, eax);
+ bailoutIf(Assembler::NotEqual, ins->snapshot());
+
+ // If lhs is zero and the divisor is negative, the answer should have
+ // been -0.
+ if (d < 0) {
+ masm.test32(lhs, lhs);
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ }
+ } else if (ins->canBeNegativeDividend()) {
+ // This is a mod op. If the computed value is zero and lhs
+ // is negative, the answer should have been -0.
+ Label done;
+
+ masm.cmp32(lhs, Imm32(0));
+ masm.j(Assembler::GreaterThanOrEqual, &done);
+
+ masm.test32(eax, eax);
+ bailoutIf(Assembler::Zero, ins->snapshot());
+
+ masm.bind(&done);
+ }
+ }
+}
+
+void CodeGenerator::visitDivI(LDivI* ins) {
+ Register remainder = ToRegister(ins->remainder());
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+ Register output = ToRegister(ins->output());
+
+ MDiv* mir = ins->mir();
+
+ MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
+ MOZ_ASSERT(rhs != edx);
+ MOZ_ASSERT(remainder == edx);
+ MOZ_ASSERT(output == eax);
+
+ Label done;
+ ReturnZero* ool = nullptr;
+
+ // Put the lhs in eax, for either the negative overflow case or the regular
+ // divide case.
+ if (lhs != eax) {
+ masm.mov(lhs, eax);
+ }
+
+ // Handle divide by zero.
+ if (mir->canBeDivideByZero()) {
+ masm.test32(rhs, rhs);
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.j(Assembler::NonZero, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else if (mir->canTruncateInfinities()) {
+ // Truncated division by zero is zero (Infinity|0 == 0)
+ if (!ool) {
+ ool = new (alloc()) ReturnZero(output);
+ }
+ masm.j(Assembler::Zero, ool->entry());
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ }
+ }
+
+ // Handle an integer overflow exception from -2147483648 / -1.
+ if (mir->canBeNegativeOverflow()) {
+ Label notOverflow;
+ masm.cmp32(lhs, Imm32(INT32_MIN));
+ masm.j(Assembler::NotEqual, &notOverflow);
+ masm.cmp32(rhs, Imm32(-1));
+ if (mir->trapOnError()) {
+ masm.j(Assembler::NotEqual, &notOverflow);
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, mir->bytecodeOffset());
+ } else if (mir->canTruncateOverflow()) {
+ // (-INT32_MIN)|0 == INT32_MIN and INT32_MIN is already in the
+ // output register (lhs == eax).
+ masm.j(Assembler::Equal, &done);
+ } else {
+ MOZ_ASSERT(mir->fallible());
+ bailoutIf(Assembler::Equal, ins->snapshot());
+ }
+ masm.bind(&notOverflow);
+ }
+
+ // Handle negative 0.
+ if (!mir->canTruncateNegativeZero() && mir->canBeNegativeZero()) {
+ Label nonzero;
+ masm.test32(lhs, lhs);
+ masm.j(Assembler::NonZero, &nonzero);
+ masm.cmp32(rhs, Imm32(0));
+ bailoutIf(Assembler::LessThan, ins->snapshot());
+ masm.bind(&nonzero);
+ }
+
+ // Sign extend the lhs into edx to make (edx:eax), since idiv is 64-bit.
+ if (lhs != eax) {
+ masm.mov(lhs, eax);
+ }
+ masm.cdq();
+ masm.idiv(rhs);
+
+ if (!mir->canTruncateRemainder()) {
+ // If the remainder is > 0, bailout since this must be a double.
+ masm.test32(remainder, remainder);
+ bailoutIf(Assembler::NonZero, ins->snapshot());
+ }
+
+ masm.bind(&done);
+
+ if (ool) {
+ addOutOfLineCode(ool, mir);
+ masm.bind(ool->rejoin());
+ }
+}
+
+void CodeGenerator::visitModPowTwoI(LModPowTwoI* ins) {
+ Register lhs = ToRegister(ins->getOperand(0));
+ int32_t shift = ins->shift();
+
+ Label negative;
+
+ if (!ins->mir()->isUnsigned() && ins->mir()->canBeNegativeDividend()) {
+ // Switch based on sign of the lhs.
+ // Positive numbers are just a bitmask
+ masm.branchTest32(Assembler::Signed, lhs, lhs, &negative);
+ }
+
+ masm.andl(Imm32((uint32_t(1) << shift) - 1), lhs);
+
+ if (!ins->mir()->isUnsigned() && ins->mir()->canBeNegativeDividend()) {
+ Label done;
+ masm.jump(&done);
+
+ // Negative numbers need a negate, bitmask, negate
+ masm.bind(&negative);
+
+ // Unlike in the visitModI case, we are not computing the mod by means of a
+ // division. Therefore, the divisor = -1 case isn't problematic (the andl
+ // always returns 0, which is what we expect).
+ //
+ // The negl instruction overflows if lhs == INT32_MIN, but this is also not
+ // a problem: shift is at most 31, and so the andl also always returns 0.
+ masm.negl(lhs);
+ masm.andl(Imm32((uint32_t(1) << shift) - 1), lhs);
+ masm.negl(lhs);
+
+ // Since a%b has the same sign as b, and a is negative in this branch,
+ // an answer of 0 means the correct result is actually -0. Bail out.
+ if (!ins->mir()->isTruncated()) {
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ }
+ masm.bind(&done);
+ }
+}
+
+class ModOverflowCheck : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
+ Label done_;
+ LModI* ins_;
+ Register rhs_;
+
+ public:
+ explicit ModOverflowCheck(LModI* ins, Register rhs) : ins_(ins), rhs_(rhs) {}
+
+ virtual void accept(CodeGeneratorX86Shared* codegen) override {
+ codegen->visitModOverflowCheck(this);
+ }
+ Label* done() { return &done_; }
+ LModI* ins() const { return ins_; }
+ Register rhs() const { return rhs_; }
+};
+
+void CodeGeneratorX86Shared::visitModOverflowCheck(ModOverflowCheck* ool) {
+ masm.cmp32(ool->rhs(), Imm32(-1));
+ if (ool->ins()->mir()->isTruncated()) {
+ masm.j(Assembler::NotEqual, ool->rejoin());
+ masm.mov(ImmWord(0), edx);
+ masm.jmp(ool->done());
+ } else {
+ bailoutIf(Assembler::Equal, ool->ins()->snapshot());
+ masm.jmp(ool->rejoin());
+ }
+}
+
+void CodeGenerator::visitModI(LModI* ins) {
+ Register remainder = ToRegister(ins->remainder());
+ Register lhs = ToRegister(ins->lhs());
+ Register rhs = ToRegister(ins->rhs());
+
+ // Required to use idiv.
+ MOZ_ASSERT_IF(lhs != rhs, rhs != eax);
+ MOZ_ASSERT(rhs != edx);
+ MOZ_ASSERT(remainder == edx);
+ MOZ_ASSERT(ToRegister(ins->getTemp(0)) == eax);
+
+ Label done;
+ ReturnZero* ool = nullptr;
+ ModOverflowCheck* overflow = nullptr;
+
+ // Set up eax in preparation for doing a div.
+ if (lhs != eax) {
+ masm.mov(lhs, eax);
+ }
+
+ MMod* mir = ins->mir();
+
+ // Prevent divide by zero.
+ if (mir->canBeDivideByZero()) {
+ masm.test32(rhs, rhs);
+ if (mir->isTruncated()) {
+ if (mir->trapOnError()) {
+ Label nonZero;
+ masm.j(Assembler::NonZero, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, mir->bytecodeOffset());
+ masm.bind(&nonZero);
+ } else {
+ if (!ool) {
+ ool = new (alloc()) ReturnZero(edx);
+ }
+ masm.j(Assembler::Zero, ool->entry());
+ }
+ } else {
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ }
+ }
+
+ Label negative;
+
+ // Switch based on sign of the lhs.
+ if (mir->canBeNegativeDividend()) {
+ masm.branchTest32(Assembler::Signed, lhs, lhs, &negative);
+ }
+
+ // If lhs >= 0 then remainder = lhs % rhs. The remainder must be positive.
+ {
+ // Check if rhs is a power-of-two.
+ if (mir->canBePowerOfTwoDivisor()) {
+ MOZ_ASSERT(rhs != remainder);
+
+ // Rhs y is a power-of-two if (y & (y-1)) == 0. Note that if
+ // y is any negative number other than INT32_MIN, both y and
+ // y-1 will have the sign bit set so these are never optimized
+ // as powers-of-two. If y is INT32_MIN, y-1 will be INT32_MAX
+ // and because lhs >= 0 at this point, lhs & INT32_MAX returns
+ // the correct value.
+ Label notPowerOfTwo;
+ masm.mov(rhs, remainder);
+ masm.subl(Imm32(1), remainder);
+ masm.branchTest32(Assembler::NonZero, remainder, rhs, &notPowerOfTwo);
+ {
+ masm.andl(lhs, remainder);
+ masm.jmp(&done);
+ }
+ masm.bind(&notPowerOfTwo);
+ }
+
+ // Since lhs >= 0, the sign-extension will be 0
+ masm.mov(ImmWord(0), edx);
+ masm.idiv(rhs);
+ }
+
+ // Otherwise, we have to beware of two special cases:
+ if (mir->canBeNegativeDividend()) {
+ masm.jump(&done);
+
+ masm.bind(&negative);
+
+ // Prevent an integer overflow exception from -2147483648 % -1
+ masm.cmp32(lhs, Imm32(INT32_MIN));
+ overflow = new (alloc()) ModOverflowCheck(ins, rhs);
+ masm.j(Assembler::Equal, overflow->entry());
+ masm.bind(overflow->rejoin());
+ masm.cdq();
+ masm.idiv(rhs);
+
+ if (!mir->isTruncated()) {
+ // A remainder of 0 means that the rval must be -0, which is a double.
+ masm.test32(remainder, remainder);
+ bailoutIf(Assembler::Zero, ins->snapshot());
+ }
+ }
+
+ masm.bind(&done);
+
+ if (overflow) {
+ addOutOfLineCode(overflow, mir);
+ masm.bind(overflow->done());
+ }
+
+ if (ool) {
+ addOutOfLineCode(ool, mir);
+ masm.bind(ool->rejoin());
+ }
+}
+
+void CodeGenerator::visitBitNotI(LBitNotI* ins) {
+ const LAllocation* input = ins->getOperand(0);
+ MOZ_ASSERT(!input->isConstant());
+
+ masm.notl(ToOperand(input));
+}
+
+void CodeGenerator::visitBitOpI(LBitOpI* ins) {
+ const LAllocation* lhs = ins->getOperand(0);
+ const LAllocation* rhs = ins->getOperand(1);
+
+ switch (ins->bitop()) {
+ case JSOp::BitOr:
+ if (rhs->isConstant()) {
+ masm.orl(Imm32(ToInt32(rhs)), ToOperand(lhs));
+ } else {
+ masm.orl(ToOperand(rhs), ToRegister(lhs));
+ }
+ break;
+ case JSOp::BitXor:
+ if (rhs->isConstant()) {
+ masm.xorl(Imm32(ToInt32(rhs)), ToOperand(lhs));
+ } else {
+ masm.xorl(ToOperand(rhs), ToRegister(lhs));
+ }
+ break;
+ case JSOp::BitAnd:
+ if (rhs->isConstant()) {
+ masm.andl(Imm32(ToInt32(rhs)), ToOperand(lhs));
+ } else {
+ masm.andl(ToOperand(rhs), ToRegister(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitBitOpI64(LBitOpI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LBitOpI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LBitOpI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ switch (lir->bitop()) {
+ case JSOp::BitOr:
+ if (IsConstant(rhs)) {
+ masm.or64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.or64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::BitXor:
+ if (IsConstant(rhs)) {
+ masm.xor64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.xor64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::BitAnd:
+ if (IsConstant(rhs)) {
+ masm.and64(Imm64(ToInt64(rhs)), ToRegister64(lhs));
+ } else {
+ masm.and64(ToOperandOrRegister64(rhs), ToRegister64(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected binary opcode");
+ }
+}
+
+void CodeGenerator::visitShiftI(LShiftI* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ const LAllocation* rhs = ins->rhs();
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ if (shift) {
+ masm.lshift32(Imm32(shift), lhs);
+ }
+ break;
+ case JSOp::Rsh:
+ if (shift) {
+ masm.rshift32Arithmetic(Imm32(shift), lhs);
+ }
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.rshift32(Imm32(shift), lhs);
+ } else if (ins->mir()->toUrsh()->fallible()) {
+ // x >>> 0 can overflow.
+ masm.test32(lhs, lhs);
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ } else {
+ Register shift = ToRegister(rhs);
+ switch (ins->bitop()) {
+ case JSOp::Lsh:
+ masm.lshift32(shift, lhs);
+ break;
+ case JSOp::Rsh:
+ masm.rshift32Arithmetic(shift, lhs);
+ break;
+ case JSOp::Ursh:
+ masm.rshift32(shift, lhs);
+ if (ins->mir()->toUrsh()->fallible()) {
+ // x >>> 0 can overflow.
+ masm.test32(lhs, lhs);
+ bailoutIf(Assembler::Signed, ins->snapshot());
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ }
+}
+
+void CodeGenerator::visitShiftI64(LShiftI64* lir) {
+ const LInt64Allocation lhs = lir->getInt64Operand(LShiftI64::Lhs);
+ LAllocation* rhs = lir->getOperand(LShiftI64::Rhs);
+
+ MOZ_ASSERT(ToOutRegister64(lir) == ToRegister64(lhs));
+
+ if (rhs->isConstant()) {
+ int32_t shift = int32_t(rhs->toConstant()->toInt64() & 0x3F);
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ if (shift) {
+ masm.lshift64(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::Rsh:
+ if (shift) {
+ masm.rshift64Arithmetic(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ case JSOp::Ursh:
+ if (shift) {
+ masm.rshift64(Imm32(shift), ToRegister64(lhs));
+ }
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+ return;
+ }
+
+ Register shift = ToRegister(rhs);
+#ifdef JS_CODEGEN_X86
+ MOZ_ASSERT(shift == ecx);
+#endif
+ switch (lir->bitop()) {
+ case JSOp::Lsh:
+ masm.lshift64(shift, ToRegister64(lhs));
+ break;
+ case JSOp::Rsh:
+ masm.rshift64Arithmetic(shift, ToRegister64(lhs));
+ break;
+ case JSOp::Ursh:
+ masm.rshift64(shift, ToRegister64(lhs));
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift op");
+ }
+}
+
+void CodeGenerator::visitUrshD(LUrshD* ins) {
+ Register lhs = ToRegister(ins->lhs());
+ MOZ_ASSERT(ToRegister(ins->temp()) == lhs);
+
+ const LAllocation* rhs = ins->rhs();
+ FloatRegister out = ToFloatRegister(ins->output());
+
+ if (rhs->isConstant()) {
+ int32_t shift = ToInt32(rhs) & 0x1F;
+ if (shift) {
+ masm.shrl(Imm32(shift), lhs);
+ }
+ } else {
+ Register shift = ToRegister(rhs);
+ masm.rshift32(shift, lhs);
+ }
+
+ masm.convertUInt32ToDouble(lhs, out);
+}
+
+Operand CodeGeneratorX86Shared::ToOperand(const LAllocation& a) {
+ if (a.isGeneralReg()) {
+ return Operand(a.toGeneralReg()->reg());
+ }
+ if (a.isFloatReg()) {
+ return Operand(a.toFloatReg()->reg());
+ }
+ return Operand(ToAddress(a));
+}
+
+Operand CodeGeneratorX86Shared::ToOperand(const LAllocation* a) {
+ return ToOperand(*a);
+}
+
+Operand CodeGeneratorX86Shared::ToOperand(const LDefinition* def) {
+ return ToOperand(def->output());
+}
+
+MoveOperand CodeGeneratorX86Shared::toMoveOperand(LAllocation a) const {
+ if (a.isGeneralReg()) {
+ return MoveOperand(ToRegister(a));
+ }
+ if (a.isFloatReg()) {
+ return MoveOperand(ToFloatRegister(a));
+ }
+ MoveOperand::Kind kind =
+ a.isStackArea() ? MoveOperand::EFFECTIVE_ADDRESS : MoveOperand::MEMORY;
+ return MoveOperand(ToAddress(a), kind);
+}
+
+class OutOfLineTableSwitch : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
+ MTableSwitch* mir_;
+ CodeLabel jumpLabel_;
+
+ void accept(CodeGeneratorX86Shared* codegen) override {
+ codegen->visitOutOfLineTableSwitch(this);
+ }
+
+ public:
+ explicit OutOfLineTableSwitch(MTableSwitch* mir) : mir_(mir) {}
+
+ MTableSwitch* mir() const { return mir_; }
+
+ CodeLabel* jumpLabel() { return &jumpLabel_; }
+};
+
+void CodeGeneratorX86Shared::visitOutOfLineTableSwitch(
+ OutOfLineTableSwitch* ool) {
+ MTableSwitch* mir = ool->mir();
+
+ masm.haltingAlign(sizeof(void*));
+ masm.bind(ool->jumpLabel());
+ masm.addCodeLabel(*ool->jumpLabel());
+
+ for (size_t i = 0; i < mir->numCases(); i++) {
+ LBlock* caseblock = skipTrivialBlocks(mir->getCase(i))->lir();
+ Label* caseheader = caseblock->label();
+ uint32_t caseoffset = caseheader->offset();
+
+ // The entries of the jump table need to be absolute addresses and thus
+ // must be patched after codegen is finished.
+ CodeLabel cl;
+ masm.writeCodePointer(&cl);
+ cl.target()->bind(caseoffset);
+ masm.addCodeLabel(cl);
+ }
+}
+
+void CodeGeneratorX86Shared::emitTableSwitchDispatch(MTableSwitch* mir,
+ Register index,
+ Register base) {
+ Label* defaultcase = skipTrivialBlocks(mir->getDefault())->lir()->label();
+
+ // Lower value with low value
+ if (mir->low() != 0) {
+ masm.subl(Imm32(mir->low()), index);
+ }
+
+ // Jump to default case if input is out of range
+ int32_t cases = mir->numCases();
+ masm.cmp32(index, Imm32(cases));
+ masm.j(AssemblerX86Shared::AboveOrEqual, defaultcase);
+
+ // To fill in the CodeLabels for the case entries, we need to first
+ // generate the case entries (we don't yet know their offsets in the
+ // instruction stream).
+ OutOfLineTableSwitch* ool = new (alloc()) OutOfLineTableSwitch(mir);
+ addOutOfLineCode(ool, mir);
+
+ // Compute the position where a pointer to the right case stands.
+ masm.mov(ool->jumpLabel(), base);
+ BaseIndex pointer(base, index, ScalePointer);
+
+ // Jump to the right case
+ masm.branchToComputedAddress(pointer);
+}
+
+void CodeGenerator::visitMathD(LMathD* math) {
+ FloatRegister lhs = ToFloatRegister(math->lhs());
+ Operand rhs = ToOperand(math->rhs());
+ FloatRegister output = ToFloatRegister(math->output());
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.vaddsd(rhs, lhs, output);
+ break;
+ case JSOp::Sub:
+ masm.vsubsd(rhs, lhs, output);
+ break;
+ case JSOp::Mul:
+ masm.vmulsd(rhs, lhs, output);
+ break;
+ case JSOp::Div:
+ masm.vdivsd(rhs, lhs, output);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitMathF(LMathF* math) {
+ FloatRegister lhs = ToFloatRegister(math->lhs());
+ Operand rhs = ToOperand(math->rhs());
+ FloatRegister output = ToFloatRegister(math->output());
+
+ switch (math->jsop()) {
+ case JSOp::Add:
+ masm.vaddss(rhs, lhs, output);
+ break;
+ case JSOp::Sub:
+ masm.vsubss(rhs, lhs, output);
+ break;
+ case JSOp::Mul:
+ masm.vmulss(rhs, lhs, output);
+ break;
+ case JSOp::Div:
+ masm.vdivss(rhs, lhs, output);
+ break;
+ default:
+ MOZ_CRASH("unexpected opcode");
+ }
+}
+
+void CodeGenerator::visitNearbyInt(LNearbyInt* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ RoundingMode roundingMode = lir->mir()->roundingMode();
+ masm.nearbyIntDouble(roundingMode, input, output);
+}
+
+void CodeGenerator::visitNearbyIntF(LNearbyIntF* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ RoundingMode roundingMode = lir->mir()->roundingMode();
+ masm.nearbyIntFloat32(roundingMode, input, output);
+}
+
+void CodeGenerator::visitEffectiveAddress(LEffectiveAddress* ins) {
+ const MEffectiveAddress* mir = ins->mir();
+ Register base = ToRegister(ins->base());
+ Register index = ToRegister(ins->index());
+ Register output = ToRegister(ins->output());
+ masm.leal(Operand(base, index, mir->scale(), mir->displacement()), output);
+}
+
+void CodeGeneratorX86Shared::generateInvalidateEpilogue() {
+ // Ensure that there is enough space in the buffer for the OsiPoint
+ // patching to occur. Otherwise, we could overwrite the invalidation
+ // epilogue.
+ for (size_t i = 0; i < sizeof(void*); i += Assembler::NopSize()) {
+ masm.nop();
+ }
+
+ masm.bind(&invalidate_);
+
+ // Push the Ion script onto the stack (when we determine what that pointer
+ // is).
+ invalidateEpilogueData_ = masm.pushWithPatch(ImmWord(uintptr_t(-1)));
+
+ // Jump to the invalidator which will replace the current frame.
+ TrampolinePtr thunk = gen->jitRuntime()->getInvalidationThunk();
+ masm.jump(thunk);
+}
+
+void CodeGenerator::visitNegI(LNegI* ins) {
+ Register input = ToRegister(ins->input());
+ MOZ_ASSERT(input == ToRegister(ins->output()));
+
+ masm.neg32(input);
+}
+
+void CodeGenerator::visitNegI64(LNegI64* ins) {
+ Register64 input = ToRegister64(ins->getInt64Operand(0));
+ MOZ_ASSERT(input == ToOutRegister64(ins));
+ masm.neg64(input);
+}
+
+void CodeGenerator::visitNegD(LNegD* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ MOZ_ASSERT(input == ToFloatRegister(ins->output()));
+
+ masm.negateDouble(input);
+}
+
+void CodeGenerator::visitNegF(LNegF* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ MOZ_ASSERT(input == ToFloatRegister(ins->output()));
+
+ masm.negateFloat(input);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement(
+ LCompareExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register temp =
+ lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
+
+ Register oldval = ToRegister(lir->oldval());
+ Register newval = ToRegister(lir->newval());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, temp, output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchangeJS(arrayType, Synchronization::Full(), dest, oldval,
+ newval, temp, output);
+ }
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement(
+ LAtomicExchangeTypedArrayElement* lir) {
+ Register elements = ToRegister(lir->elements());
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register temp =
+ lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
+
+ Register value = ToRegister(lir->value());
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
+ output);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchangeJS(arrayType, Synchronization::Full(), dest, value, temp,
+ output);
+ }
+}
+
+template <typename T>
+static inline void AtomicBinopToTypedArray(MacroAssembler& masm, AtomicOp op,
+ Scalar::Type arrayType,
+ const LAllocation* value,
+ const T& mem, Register temp1,
+ Register temp2, AnyRegister output) {
+ if (value->isConstant()) {
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(), op,
+ Imm32(ToInt32(value)), mem, temp1, temp2, output);
+ } else {
+ masm.atomicFetchOpJS(arrayType, Synchronization::Full(), op,
+ ToRegister(value), mem, temp1, temp2, output);
+ }
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop(
+ LAtomicTypedArrayElementBinop* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ AnyRegister output = ToAnyRegister(lir->output());
+ Register elements = ToRegister(lir->elements());
+ Register temp1 =
+ lir->temp1()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp1());
+ Register temp2 =
+ lir->temp2()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp2());
+ const LAllocation* value = lir->value();
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ AtomicBinopToTypedArray(masm, lir->mir()->operation(), arrayType, value,
+ mem, temp1, temp2, output);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ AtomicBinopToTypedArray(masm, lir->mir()->operation(), arrayType, value,
+ mem, temp1, temp2, output);
+ }
+}
+
+template <typename T>
+static inline void AtomicBinopToTypedArray(MacroAssembler& masm,
+ Scalar::Type arrayType, AtomicOp op,
+ const LAllocation* value,
+ const T& mem) {
+ if (value->isConstant()) {
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(), op,
+ Imm32(ToInt32(value)), mem, InvalidReg);
+ } else {
+ masm.atomicEffectOpJS(arrayType, Synchronization::Full(), op,
+ ToRegister(value), mem, InvalidReg);
+ }
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect(
+ LAtomicTypedArrayElementBinopForEffect* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ const LAllocation* value = lir->value();
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ if (lir->index()->isConstant()) {
+ Address mem = ToAddress(elements, lir->index(), arrayType);
+ AtomicBinopToTypedArray(masm, arrayType, lir->mir()->operation(), value,
+ mem);
+ } else {
+ BaseIndex mem(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ AtomicBinopToTypedArray(masm, arrayType, lir->mir()->operation(), value,
+ mem);
+ }
+}
+
+void CodeGenerator::visitMemoryBarrier(LMemoryBarrier* ins) {
+ if (ins->type() & MembarStoreLoad) {
+ masm.storeLoadFence();
+ }
+}
+
+void CodeGeneratorX86Shared::visitOutOfLineWasmTruncateCheck(
+ OutOfLineWasmTruncateCheck* ool) {
+ FloatRegister input = ool->input();
+ Register output = ool->output();
+ Register64 output64 = ool->output64();
+ MIRType fromType = ool->fromType();
+ MIRType toType = ool->toType();
+ Label* oolRejoin = ool->rejoin();
+ TruncFlags flags = ool->flags();
+ wasm::BytecodeOffset off = ool->bytecodeOffset();
+
+ if (fromType == MIRType::Float32) {
+ if (toType == MIRType::Int32) {
+ masm.oolWasmTruncateCheckF32ToI32(input, output, flags, off, oolRejoin);
+ } else if (toType == MIRType::Int64) {
+ masm.oolWasmTruncateCheckF32ToI64(input, output64, flags, off, oolRejoin);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+ } else if (fromType == MIRType::Double) {
+ if (toType == MIRType::Int32) {
+ masm.oolWasmTruncateCheckF64ToI32(input, output, flags, off, oolRejoin);
+ } else if (toType == MIRType::Int64) {
+ masm.oolWasmTruncateCheckF64ToI64(input, output64, flags, off, oolRejoin);
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+ } else {
+ MOZ_CRASH("unexpected type");
+ }
+}
+
+void CodeGeneratorX86Shared::canonicalizeIfDeterministic(
+ Scalar::Type type, const LAllocation* value) {
+#ifdef DEBUG
+ if (!js::SupportDifferentialTesting()) {
+ return;
+ }
+
+ switch (type) {
+ case Scalar::Float32: {
+ FloatRegister in = ToFloatRegister(value);
+ masm.canonicalizeFloatIfDeterministic(in);
+ break;
+ }
+ case Scalar::Float64: {
+ FloatRegister in = ToFloatRegister(value);
+ masm.canonicalizeDoubleIfDeterministic(in);
+ break;
+ }
+ default: {
+ // Other types don't need canonicalization.
+ break;
+ }
+ }
+#endif // DEBUG
+}
+
+template <typename T>
+Operand CodeGeneratorX86Shared::toMemoryAccessOperand(T* lir, int32_t disp) {
+ const LAllocation* ptr = lir->ptr();
+#ifdef JS_CODEGEN_X86
+ const LAllocation* memoryBase = lir->memoryBase();
+ Operand destAddr = ptr->isBogus() ? Operand(ToRegister(memoryBase), disp)
+ : Operand(ToRegister(memoryBase),
+ ToRegister(ptr), TimesOne, disp);
+#else
+ Operand destAddr = ptr->isBogus()
+ ? Operand(HeapReg, disp)
+ : Operand(HeapReg, ToRegister(ptr), TimesOne, disp);
+#endif
+ return destAddr;
+}
+
+void CodeGenerator::visitCopySignF(LCopySignF* lir) {
+ FloatRegister lhs = ToFloatRegister(lir->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(lir->getOperand(1));
+
+ FloatRegister out = ToFloatRegister(lir->output());
+
+ if (lhs == rhs) {
+ if (lhs != out) {
+ masm.moveFloat32(lhs, out);
+ }
+ return;
+ }
+
+ masm.copySignFloat32(lhs, rhs, out);
+}
+
+void CodeGenerator::visitCopySignD(LCopySignD* lir) {
+ FloatRegister lhs = ToFloatRegister(lir->getOperand(0));
+ FloatRegister rhs = ToFloatRegister(lir->getOperand(1));
+
+ FloatRegister out = ToFloatRegister(lir->output());
+
+ if (lhs == rhs) {
+ if (lhs != out) {
+ masm.moveDouble(lhs, out);
+ }
+ return;
+ }
+
+ masm.copySignDouble(lhs, rhs, out);
+}
+
+void CodeGenerator::visitRotateI64(LRotateI64* lir) {
+ MRotate* mir = lir->mir();
+ LAllocation* count = lir->count();
+
+ Register64 input = ToRegister64(lir->input());
+ Register64 output = ToOutRegister64(lir);
+ Register temp = ToTempRegisterOrInvalid(lir->temp());
+
+ MOZ_ASSERT(input == output);
+
+ if (count->isConstant()) {
+ int32_t c = int32_t(count->toConstant()->toInt64() & 0x3F);
+ if (!c) {
+ return;
+ }
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft64(Imm32(c), input, output, temp);
+ } else {
+ masm.rotateRight64(Imm32(c), input, output, temp);
+ }
+ } else {
+ if (mir->isLeftRotate()) {
+ masm.rotateLeft64(ToRegister(count), input, output, temp);
+ } else {
+ masm.rotateRight64(ToRegister(count), input, output, temp);
+ }
+ }
+}
+
+void CodeGenerator::visitPopcntI64(LPopcntI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ Register temp = InvalidReg;
+ if (!AssemblerX86Shared::HasPOPCNT()) {
+ temp = ToRegister(lir->getTemp(0));
+ }
+
+ masm.popcnt64(input, output, temp);
+}
+
+void CodeGenerator::visitSimd128(LSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ const LDefinition* out = ins->getDef(0);
+ masm.loadConstantSimd128(ins->simd128(), ToFloatRegister(out));
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmTernarySimd128(LWasmTernarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128Bitselect: {
+ FloatRegister lhsDest = ToFloatRegister(ins->v0());
+ FloatRegister rhs = ToFloatRegister(ins->v1());
+ FloatRegister control = ToFloatRegister(ins->v2());
+ FloatRegister temp = ToFloatRegister(ins->temp());
+ masm.bitwiseSelectSimd128(control, lhsDest, rhs, lhsDest, temp);
+ break;
+ }
+ case wasm::SimdOp::F32x4RelaxedFma:
+ masm.fmaFloat32x4(ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::F32x4RelaxedFnma:
+ masm.fnmaFloat32x4(ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::F64x2RelaxedFma:
+ masm.fmaFloat64x2(ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::F64x2RelaxedFnma:
+ masm.fnmaFloat64x2(ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::I8x16RelaxedLaneSelect:
+ case wasm::SimdOp::I16x8RelaxedLaneSelect:
+ case wasm::SimdOp::I32x4RelaxedLaneSelect:
+ case wasm::SimdOp::I64x2RelaxedLaneSelect: {
+ FloatRegister lhs = ToFloatRegister(ins->v0());
+ FloatRegister rhs = ToFloatRegister(ins->v1());
+ FloatRegister mask = ToFloatRegister(ins->v2());
+ FloatRegister dest = ToFloatRegister(ins->output());
+ masm.laneSelectSimd128(mask, lhs, rhs, dest);
+ break;
+ }
+ case wasm::SimdOp::I32x4DotI8x16I7x16AddS:
+ masm.dotInt8x16Int7x16ThenAdd(ToFloatRegister(ins->v0()),
+ ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()));
+ break;
+ case wasm::SimdOp::F32x4RelaxedDotBF16x8AddF32x4:
+ masm.dotBFloat16x8ThenAdd(
+ ToFloatRegister(ins->v0()), ToFloatRegister(ins->v1()),
+ ToFloatRegister(ins->v2()), ToFloatRegister(ins->temp()));
+ break;
+ default:
+ MOZ_CRASH("NYI");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmBinarySimd128(LWasmBinarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister lhs = ToFloatRegister(ins->lhsDest());
+ FloatRegister rhs = ToFloatRegister(ins->rhs());
+ FloatRegister temp1 = ToTempFloatRegisterOrInvalid(ins->getTemp(0));
+ FloatRegister temp2 = ToTempFloatRegisterOrInvalid(ins->getTemp(1));
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128And:
+ masm.bitwiseAndSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128Or:
+ masm.bitwiseOrSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128Xor:
+ masm.bitwiseXorSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128AndNot:
+ // x86/x64 specific: The CPU provides ~A & B. The operands were swapped
+ // during lowering, and we'll compute A & ~B here as desired.
+ masm.bitwiseNotAndSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16AvgrU:
+ masm.unsignedAverageInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8AvgrU:
+ masm.unsignedAverageInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Add:
+ masm.addInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16AddSatS:
+ masm.addSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16AddSatU:
+ masm.unsignedAddSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Sub:
+ masm.subInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16SubSatS:
+ masm.subSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16SubSatU:
+ masm.unsignedSubSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MinS:
+ masm.minInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MinU:
+ masm.unsignedMinInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MaxS:
+ masm.maxInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MaxU:
+ masm.unsignedMaxInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Add:
+ masm.addInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8AddSatS:
+ masm.addSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8AddSatU:
+ masm.unsignedAddSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Sub:
+ masm.subInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8SubSatS:
+ masm.subSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8SubSatU:
+ masm.unsignedSubSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Mul:
+ masm.mulInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MinS:
+ masm.minInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MinU:
+ masm.unsignedMinInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MaxS:
+ masm.maxInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MaxU:
+ masm.unsignedMaxInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Add:
+ masm.addInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Sub:
+ masm.subInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Mul:
+ masm.mulInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MinS:
+ masm.minInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MinU:
+ masm.unsignedMinInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MaxS:
+ masm.maxInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MaxU:
+ masm.unsignedMaxInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Add:
+ masm.addInt64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Sub:
+ masm.subInt64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Mul:
+ masm.mulInt64x2(lhs, rhs, dest, temp1);
+ break;
+ case wasm::SimdOp::F32x4Add:
+ masm.addFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Sub:
+ masm.subFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Mul:
+ masm.mulFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Div:
+ masm.divFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Min:
+ masm.minFloat32x4(lhs, rhs, dest, temp1, temp2);
+ break;
+ case wasm::SimdOp::F32x4Max:
+ masm.maxFloat32x4(lhs, rhs, dest, temp1, temp2);
+ break;
+ case wasm::SimdOp::F64x2Add:
+ masm.addFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Sub:
+ masm.subFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Mul:
+ masm.mulFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Div:
+ masm.divFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Min:
+ masm.minFloat64x2(lhs, rhs, dest, temp1, temp2);
+ break;
+ case wasm::SimdOp::F64x2Max:
+ masm.maxFloat64x2(lhs, rhs, dest, temp1, temp2);
+ break;
+ case wasm::SimdOp::I8x16Swizzle:
+ masm.swizzleInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16RelaxedSwizzle:
+ masm.swizzleInt8x16Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16NarrowI16x8S:
+ masm.narrowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16NarrowI16x8U:
+ masm.unsignedNarrowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8NarrowI32x4S:
+ masm.narrowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8NarrowI32x4U:
+ masm.unsignedNarrowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Eq:
+ masm.compareInt8x16(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Ne:
+ masm.compareInt8x16(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LtS:
+ masm.compareInt8x16(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GtS:
+ masm.compareInt8x16(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LeS:
+ masm.compareInt8x16(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GeS:
+ masm.compareInt8x16(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LtU:
+ masm.compareInt8x16(Assembler::Below, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GtU:
+ masm.compareInt8x16(Assembler::Above, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LeU:
+ masm.compareInt8x16(Assembler::BelowOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GeU:
+ masm.compareInt8x16(Assembler::AboveOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Eq:
+ masm.compareInt16x8(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Ne:
+ masm.compareInt16x8(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LtS:
+ masm.compareInt16x8(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GtS:
+ masm.compareInt16x8(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LeS:
+ masm.compareInt16x8(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GeS:
+ masm.compareInt16x8(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LtU:
+ masm.compareInt16x8(Assembler::Below, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GtU:
+ masm.compareInt16x8(Assembler::Above, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LeU:
+ masm.compareInt16x8(Assembler::BelowOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GeU:
+ masm.compareInt16x8(Assembler::AboveOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Eq:
+ masm.compareInt32x4(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Ne:
+ masm.compareInt32x4(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LtS:
+ masm.compareInt32x4(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GtS:
+ masm.compareInt32x4(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LeS:
+ masm.compareInt32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GeS:
+ masm.compareInt32x4(Assembler::GreaterThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LtU:
+ masm.compareInt32x4(Assembler::Below, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GtU:
+ masm.compareInt32x4(Assembler::Above, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LeU:
+ masm.compareInt32x4(Assembler::BelowOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GeU:
+ masm.compareInt32x4(Assembler::AboveOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Eq:
+ masm.compareForEqualityInt64x2(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Ne:
+ masm.compareForEqualityInt64x2(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2LtS:
+ masm.compareForOrderingInt64x2(Assembler::LessThan, lhs, rhs, dest, temp1,
+ temp2);
+ break;
+ case wasm::SimdOp::I64x2GtS:
+ masm.compareForOrderingInt64x2(Assembler::GreaterThan, lhs, rhs, dest,
+ temp1, temp2);
+ break;
+ case wasm::SimdOp::I64x2LeS:
+ masm.compareForOrderingInt64x2(Assembler::LessThanOrEqual, lhs, rhs, dest,
+ temp1, temp2);
+ break;
+ case wasm::SimdOp::I64x2GeS:
+ masm.compareForOrderingInt64x2(Assembler::GreaterThanOrEqual, lhs, rhs,
+ dest, temp1, temp2);
+ break;
+ case wasm::SimdOp::F32x4Eq:
+ masm.compareFloat32x4(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Ne:
+ masm.compareFloat32x4(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Lt:
+ masm.compareFloat32x4(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Le:
+ masm.compareFloat32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Eq:
+ masm.compareFloat64x2(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Ne:
+ masm.compareFloat64x2(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Lt:
+ masm.compareFloat64x2(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Le:
+ masm.compareFloat64x2(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4PMax:
+ // `lhs` and `rhs` are swapped, for non-VEX platforms the output is rhs.
+ masm.pseudoMaxFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4PMin:
+ // `lhs` and `rhs` are swapped, for non-VEX platforms the output is rhs.
+ masm.pseudoMinFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2PMax:
+ // `lhs` and `rhs` are swapped, for non-VEX platforms the output is rhs.
+ masm.pseudoMaxFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2PMin:
+ // `lhs` and `rhs` are swapped, for non-VEX platforms the output is rhs.
+ masm.pseudoMinFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4DotI16x8S:
+ masm.widenDotInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtmulLowI8x16S:
+ masm.extMulLowInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtmulHighI8x16S:
+ masm.extMulHighInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtmulLowI8x16U:
+ masm.unsignedExtMulLowInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtmulHighI8x16U:
+ masm.unsignedExtMulHighInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtmulLowI16x8S:
+ masm.extMulLowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtmulHighI16x8S:
+ masm.extMulHighInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtmulLowI16x8U:
+ masm.unsignedExtMulLowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtmulHighI16x8U:
+ masm.unsignedExtMulHighInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtmulLowI32x4S:
+ masm.extMulLowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtmulHighI32x4S:
+ masm.extMulHighInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtmulLowI32x4U:
+ masm.unsignedExtMulLowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtmulHighI32x4U:
+ masm.unsignedExtMulHighInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Q15MulrSatS:
+ masm.q15MulrSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4RelaxedMin:
+ masm.minFloat32x4Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4RelaxedMax:
+ masm.maxFloat32x4Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2RelaxedMin:
+ masm.minFloat64x2Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2RelaxedMax:
+ masm.maxFloat64x2Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8RelaxedQ15MulrS:
+ masm.q15MulrInt16x8Relaxed(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8DotI8x16I7x16S:
+ masm.dotInt8x16Int7x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::MozPMADDUBSW:
+ masm.vpmaddubsw(rhs, lhs, dest);
+ break;
+ default:
+ MOZ_CRASH("Binary SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmBinarySimd128WithConstant(
+ LWasmBinarySimd128WithConstant* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister lhs = ToFloatRegister(ins->lhsDest());
+ const SimdConstant& rhs = ins->rhs();
+ FloatRegister dest = ToFloatRegister(ins->output());
+ FloatRegister temp = ToTempFloatRegisterOrInvalid(ins->getTemp(0));
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Add:
+ masm.addInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Add:
+ masm.addInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Add:
+ masm.addInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Add:
+ masm.addInt64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Sub:
+ masm.subInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Sub:
+ masm.subInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Sub:
+ masm.subInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Sub:
+ masm.subInt64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Mul:
+ masm.mulInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Mul:
+ masm.mulInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16AddSatS:
+ masm.addSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16AddSatU:
+ masm.unsignedAddSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8AddSatS:
+ masm.addSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8AddSatU:
+ masm.unsignedAddSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16SubSatS:
+ masm.subSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16SubSatU:
+ masm.unsignedSubSatInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8SubSatS:
+ masm.subSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8SubSatU:
+ masm.unsignedSubSatInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MinS:
+ masm.minInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MinU:
+ masm.unsignedMinInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MinS:
+ masm.minInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MinU:
+ masm.unsignedMinInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MinS:
+ masm.minInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MinU:
+ masm.unsignedMinInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MaxS:
+ masm.maxInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16MaxU:
+ masm.unsignedMaxInt8x16(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MaxS:
+ masm.maxInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8MaxU:
+ masm.unsignedMaxInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MaxS:
+ masm.maxInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4MaxU:
+ masm.unsignedMaxInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128And:
+ masm.bitwiseAndSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128Or:
+ masm.bitwiseOrSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::V128Xor:
+ masm.bitwiseXorSimd128(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Eq:
+ masm.compareInt8x16(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16Ne:
+ masm.compareInt8x16(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16GtS:
+ masm.compareInt8x16(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16LeS:
+ masm.compareInt8x16(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Eq:
+ masm.compareInt16x8(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8Ne:
+ masm.compareInt16x8(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8GtS:
+ masm.compareInt16x8(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8LeS:
+ masm.compareInt16x8(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Eq:
+ masm.compareInt32x4(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4Ne:
+ masm.compareInt32x4(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4GtS:
+ masm.compareInt32x4(Assembler::GreaterThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4LeS:
+ masm.compareInt32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I64x2Mul:
+ masm.mulInt64x2(lhs, rhs, dest, temp);
+ break;
+ case wasm::SimdOp::F32x4Eq:
+ masm.compareFloat32x4(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Ne:
+ masm.compareFloat32x4(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Lt:
+ masm.compareFloat32x4(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Le:
+ masm.compareFloat32x4(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Eq:
+ masm.compareFloat64x2(Assembler::Equal, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Ne:
+ masm.compareFloat64x2(Assembler::NotEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Lt:
+ masm.compareFloat64x2(Assembler::LessThan, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Le:
+ masm.compareFloat64x2(Assembler::LessThanOrEqual, lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I32x4DotI16x8S:
+ masm.widenDotInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Add:
+ masm.addFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Add:
+ masm.addFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Sub:
+ masm.subFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Sub:
+ masm.subFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Div:
+ masm.divFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Div:
+ masm.divFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F32x4Mul:
+ masm.mulFloat32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::F64x2Mul:
+ masm.mulFloat64x2(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16NarrowI16x8S:
+ masm.narrowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I8x16NarrowI16x8U:
+ masm.unsignedNarrowInt16x8(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8NarrowI32x4S:
+ masm.narrowInt32x4(lhs, rhs, dest);
+ break;
+ case wasm::SimdOp::I16x8NarrowI32x4U:
+ masm.unsignedNarrowInt32x4(lhs, rhs, dest);
+ break;
+ default:
+ MOZ_CRASH("Binary SimdOp with constant not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmVariableShiftSimd128(
+ LWasmVariableShiftSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister lhsDest = ToFloatRegister(ins->lhsDest());
+ Register rhs = ToRegister(ins->rhs());
+ FloatRegister temp = ToTempFloatRegisterOrInvalid(ins->getTemp(0));
+
+ MOZ_ASSERT(ToFloatRegister(ins->output()) == lhsDest);
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Shl:
+ masm.leftShiftInt8x16(rhs, lhsDest, temp);
+ break;
+ case wasm::SimdOp::I8x16ShrS:
+ masm.rightShiftInt8x16(rhs, lhsDest, temp);
+ break;
+ case wasm::SimdOp::I8x16ShrU:
+ masm.unsignedRightShiftInt8x16(rhs, lhsDest, temp);
+ break;
+ case wasm::SimdOp::I16x8Shl:
+ masm.leftShiftInt16x8(rhs, lhsDest);
+ break;
+ case wasm::SimdOp::I16x8ShrS:
+ masm.rightShiftInt16x8(rhs, lhsDest);
+ break;
+ case wasm::SimdOp::I16x8ShrU:
+ masm.unsignedRightShiftInt16x8(rhs, lhsDest);
+ break;
+ case wasm::SimdOp::I32x4Shl:
+ masm.leftShiftInt32x4(rhs, lhsDest);
+ break;
+ case wasm::SimdOp::I32x4ShrS:
+ masm.rightShiftInt32x4(rhs, lhsDest);
+ break;
+ case wasm::SimdOp::I32x4ShrU:
+ masm.unsignedRightShiftInt32x4(rhs, lhsDest);
+ break;
+ case wasm::SimdOp::I64x2Shl:
+ masm.leftShiftInt64x2(rhs, lhsDest);
+ break;
+ case wasm::SimdOp::I64x2ShrS:
+ masm.rightShiftInt64x2(rhs, lhsDest, temp);
+ break;
+ case wasm::SimdOp::I64x2ShrU:
+ masm.unsignedRightShiftInt64x2(rhs, lhsDest);
+ break;
+ default:
+ MOZ_CRASH("Shift SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmConstantShiftSimd128(
+ LWasmConstantShiftSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+ int32_t shift = ins->shift();
+
+ if (shift == 0) {
+ masm.moveSimd128(src, dest);
+ return;
+ }
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Shl:
+ masm.leftShiftInt8x16(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I8x16ShrS:
+ masm.rightShiftInt8x16(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I8x16ShrU:
+ masm.unsignedRightShiftInt8x16(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I16x8Shl:
+ masm.leftShiftInt16x8(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I16x8ShrS:
+ masm.rightShiftInt16x8(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I16x8ShrU:
+ masm.unsignedRightShiftInt16x8(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I32x4Shl:
+ masm.leftShiftInt32x4(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I32x4ShrS:
+ masm.rightShiftInt32x4(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I32x4ShrU:
+ masm.unsignedRightShiftInt32x4(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I64x2Shl:
+ masm.leftShiftInt64x2(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I64x2ShrS:
+ masm.rightShiftInt64x2(Imm32(shift), src, dest);
+ break;
+ case wasm::SimdOp::I64x2ShrU:
+ masm.unsignedRightShiftInt64x2(Imm32(shift), src, dest);
+ break;
+ default:
+ MOZ_CRASH("Shift SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmSignReplicationSimd128(
+ LWasmSignReplicationSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16ShrS:
+ masm.signReplicationInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ShrS:
+ masm.signReplicationInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ShrS:
+ masm.signReplicationInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ShrS:
+ masm.signReplicationInt64x2(src, dest);
+ break;
+ default:
+ MOZ_CRASH("Shift SimdOp unsupported sign replication optimization");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmShuffleSimd128(LWasmShuffleSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister lhsDest = ToFloatRegister(ins->lhsDest());
+ FloatRegister rhs = ToFloatRegister(ins->rhs());
+ SimdConstant control = ins->control();
+ FloatRegister output = ToFloatRegister(ins->output());
+ switch (ins->op()) {
+ case SimdShuffleOp::BLEND_8x16: {
+ masm.blendInt8x16(reinterpret_cast<const uint8_t*>(control.asInt8x16()),
+ lhsDest, rhs, output, ToFloatRegister(ins->temp()));
+ break;
+ }
+ case SimdShuffleOp::BLEND_16x8: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.blendInt16x8(reinterpret_cast<const uint16_t*>(control.asInt16x8()),
+ lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::CONCAT_RIGHT_SHIFT_8x16: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ int8_t count = 16 - control.asInt8x16()[0];
+ MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
+ masm.concatAndRightShiftSimd128(lhsDest, rhs, output, count);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_HIGH_8x16: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.interleaveHighInt8x16(lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_HIGH_16x8: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.interleaveHighInt16x8(lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_HIGH_32x4: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.interleaveHighInt32x4(lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_HIGH_64x2: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.interleaveHighInt64x2(lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_LOW_8x16: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.interleaveLowInt8x16(lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_LOW_16x8: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.interleaveLowInt16x8(lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_LOW_32x4: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.interleaveLowInt32x4(lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::INTERLEAVE_LOW_64x2: {
+ MOZ_ASSERT(ins->temp()->isBogusTemp());
+ masm.interleaveLowInt64x2(lhsDest, rhs, output);
+ break;
+ }
+ case SimdShuffleOp::SHUFFLE_BLEND_8x16: {
+ masm.shuffleInt8x16(reinterpret_cast<const uint8_t*>(control.asInt8x16()),
+ lhsDest, rhs, output);
+ break;
+ }
+ default: {
+ MOZ_CRASH("Unsupported SIMD shuffle operation");
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+#ifdef ENABLE_WASM_SIMD
+
+enum PermuteX64I16x8Action : uint16_t {
+ UNAVAILABLE = 0,
+ SWAP_QWORDS = 1, // Swap qwords first
+ PERM_LOW = 2, // Permute low qword by control_[0..3]
+ PERM_HIGH = 4 // Permute high qword by control_[4..7]
+};
+
+// Skip lanes that equal v starting at i, returning the index just beyond the
+// last of those. There is no requirement that the initial lanes[i] == v.
+template <typename T>
+static int ScanConstant(const T* lanes, int v, int i) {
+ int len = int(16 / sizeof(T));
+ MOZ_ASSERT(i <= len);
+ while (i < len && lanes[i] == v) {
+ i++;
+ }
+ return i;
+}
+
+// Apply a transformation to each lane value.
+template <typename T>
+static void MapLanes(T* result, const T* input, int (*f)(int)) {
+ int len = int(16 / sizeof(T));
+ for (int i = 0; i < len; i++) {
+ result[i] = f(input[i]);
+ }
+}
+
+// Recognize part of an identity permutation starting at start, with
+// the first value of the permutation expected to be bias.
+template <typename T>
+static bool IsIdentity(const T* lanes, int start, int len, int bias) {
+ if (lanes[start] != bias) {
+ return false;
+ }
+ for (int i = start + 1; i < start + len; i++) {
+ if (lanes[i] != lanes[i - 1] + 1) {
+ return false;
+ }
+ }
+ return true;
+}
+
+// We can permute by words if the mask is reducible to a word mask, but the x64
+// lowering is only efficient if we can permute the high and low quadwords
+// separately, possibly after swapping quadwords.
+static PermuteX64I16x8Action CalculateX64Permute16x8(SimdConstant* control) {
+ const SimdConstant::I16x8& lanes = control->asInt16x8();
+ SimdConstant::I16x8 mapped;
+ MapLanes(mapped, lanes, [](int x) -> int { return x < 4 ? 0 : 1; });
+ int i = ScanConstant(mapped, mapped[0], 0);
+ if (i != 4) {
+ return PermuteX64I16x8Action::UNAVAILABLE;
+ }
+ i = ScanConstant(mapped, mapped[4], 4);
+ if (i != 8) {
+ return PermuteX64I16x8Action::UNAVAILABLE;
+ }
+ // Now compute the operation bits. `mapped` holds the adjusted lane mask.
+ memcpy(mapped, lanes, sizeof(mapped));
+ uint16_t op = 0;
+ if (mapped[0] > mapped[4]) {
+ op |= PermuteX64I16x8Action::SWAP_QWORDS;
+ }
+ for (auto& m : mapped) {
+ m &= 3;
+ }
+ if (!IsIdentity(mapped, 0, 4, 0)) {
+ op |= PermuteX64I16x8Action::PERM_LOW;
+ }
+ if (!IsIdentity(mapped, 4, 4, 0)) {
+ op |= PermuteX64I16x8Action::PERM_HIGH;
+ }
+ MOZ_ASSERT(op != PermuteX64I16x8Action::UNAVAILABLE);
+ *control = SimdConstant::CreateX8(mapped);
+ return (PermuteX64I16x8Action)op;
+}
+
+#endif
+
+void CodeGenerator::visitWasmPermuteSimd128(LWasmPermuteSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+ SimdConstant control = ins->control();
+ switch (ins->op()) {
+ // For broadcast, would MOVDDUP be better than PSHUFD for the last step?
+ case SimdPermuteOp::BROADCAST_8x16: {
+ const SimdConstant::I8x16& mask = control.asInt8x16();
+ int8_t source = mask[0];
+ if (source == 0 && Assembler::HasAVX2()) {
+ masm.vbroadcastb(Operand(src), dest);
+ break;
+ }
+ MOZ_ASSERT_IF(!Assembler::HasAVX(), src == dest);
+ if (source < 8) {
+ masm.interleaveLowInt8x16(src, src, dest);
+ } else {
+ masm.interleaveHighInt8x16(src, src, dest);
+ source -= 8;
+ }
+ uint16_t v = uint16_t(source & 3);
+ uint16_t wordMask[4] = {v, v, v, v};
+ if (source < 4) {
+ masm.permuteLowInt16x8(wordMask, dest, dest);
+ uint32_t dwordMask[4] = {0, 0, 0, 0};
+ masm.permuteInt32x4(dwordMask, dest, dest);
+ } else {
+ masm.permuteHighInt16x8(wordMask, dest, dest);
+ uint32_t dwordMask[4] = {2, 2, 2, 2};
+ masm.permuteInt32x4(dwordMask, dest, dest);
+ }
+ break;
+ }
+ case SimdPermuteOp::BROADCAST_16x8: {
+ const SimdConstant::I16x8& mask = control.asInt16x8();
+ int16_t source = mask[0];
+ if (source == 0 && Assembler::HasAVX2()) {
+ masm.vbroadcastw(Operand(src), dest);
+ break;
+ }
+ uint16_t v = uint16_t(source & 3);
+ uint16_t wordMask[4] = {v, v, v, v};
+ if (source < 4) {
+ masm.permuteLowInt16x8(wordMask, src, dest);
+ uint32_t dwordMask[4] = {0, 0, 0, 0};
+ masm.permuteInt32x4(dwordMask, dest, dest);
+ } else {
+ masm.permuteHighInt16x8(wordMask, src, dest);
+ uint32_t dwordMask[4] = {2, 2, 2, 2};
+ masm.permuteInt32x4(dwordMask, dest, dest);
+ }
+ break;
+ }
+ case SimdPermuteOp::MOVE: {
+ masm.moveSimd128(src, dest);
+ break;
+ }
+ case SimdPermuteOp::PERMUTE_8x16: {
+ const SimdConstant::I8x16& mask = control.asInt8x16();
+# ifdef DEBUG
+ DebugOnly<int> i;
+ for (i = 0; i < 16 && mask[i] == i; i++) {
+ }
+ MOZ_ASSERT(i < 16, "Should have been a MOVE operation");
+# endif
+ masm.permuteInt8x16(reinterpret_cast<const uint8_t*>(mask), src, dest);
+ break;
+ }
+ case SimdPermuteOp::PERMUTE_16x8: {
+# ifdef DEBUG
+ const SimdConstant::I16x8& mask = control.asInt16x8();
+ DebugOnly<int> i;
+ for (i = 0; i < 8 && mask[i] == i; i++) {
+ }
+ MOZ_ASSERT(i < 8, "Should have been a MOVE operation");
+# endif
+ PermuteX64I16x8Action op = CalculateX64Permute16x8(&control);
+ if (op != PermuteX64I16x8Action::UNAVAILABLE) {
+ const SimdConstant::I16x8& mask = control.asInt16x8();
+ if (op & PermuteX64I16x8Action::SWAP_QWORDS) {
+ uint32_t dwordMask[4] = {2, 3, 0, 1};
+ masm.permuteInt32x4(dwordMask, src, dest);
+ src = dest;
+ }
+ if (op & PermuteX64I16x8Action::PERM_LOW) {
+ masm.permuteLowInt16x8(reinterpret_cast<const uint16_t*>(mask) + 0,
+ src, dest);
+ src = dest;
+ }
+ if (op & PermuteX64I16x8Action::PERM_HIGH) {
+ masm.permuteHighInt16x8(reinterpret_cast<const uint16_t*>(mask) + 4,
+ src, dest);
+ src = dest;
+ }
+ } else {
+ const SimdConstant::I16x8& wmask = control.asInt16x8();
+ uint8_t mask[16];
+ for (unsigned i = 0; i < 16; i += 2) {
+ mask[i] = wmask[i / 2] * 2;
+ mask[i + 1] = wmask[i / 2] * 2 + 1;
+ }
+ masm.permuteInt8x16(mask, src, dest);
+ }
+ break;
+ }
+ case SimdPermuteOp::PERMUTE_32x4: {
+ const SimdConstant::I32x4& mask = control.asInt32x4();
+ if (Assembler::HasAVX2() && mask[0] == 0 && mask[1] == 0 &&
+ mask[2] == 0 && mask[3] == 0) {
+ masm.vbroadcastd(Operand(src), dest);
+ break;
+ }
+# ifdef DEBUG
+ DebugOnly<int> i;
+ for (i = 0; i < 4 && mask[i] == i; i++) {
+ }
+ MOZ_ASSERT(i < 4, "Should have been a MOVE operation");
+# endif
+ masm.permuteInt32x4(reinterpret_cast<const uint32_t*>(mask), src, dest);
+ break;
+ }
+ case SimdPermuteOp::ROTATE_RIGHT_8x16: {
+ MOZ_ASSERT_IF(!Assembler::HasAVX(), src == dest);
+ int8_t count = control.asInt8x16()[0];
+ MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
+ masm.concatAndRightShiftSimd128(src, src, dest, count);
+ break;
+ }
+ case SimdPermuteOp::SHIFT_LEFT_8x16: {
+ int8_t count = control.asInt8x16()[0];
+ MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
+ masm.leftShiftSimd128(Imm32(count), src, dest);
+ break;
+ }
+ case SimdPermuteOp::SHIFT_RIGHT_8x16: {
+ int8_t count = control.asInt8x16()[0];
+ MOZ_ASSERT(count > 0, "Should have been a MOVE operation");
+ masm.rightShiftSimd128(Imm32(count), src, dest);
+ break;
+ }
+ case SimdPermuteOp::REVERSE_16x8:
+ masm.reverseInt16x8(src, dest);
+ break;
+ case SimdPermuteOp::REVERSE_32x4:
+ masm.reverseInt32x4(src, dest);
+ break;
+ case SimdPermuteOp::REVERSE_64x2:
+ masm.reverseInt64x2(src, dest);
+ break;
+ default: {
+ MOZ_CRASH("Unsupported SIMD permutation operation");
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReplaceLaneSimd128(LWasmReplaceLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister lhs = ToFloatRegister(ins->lhsDest());
+ FloatRegister dest = ToFloatRegister(ins->output());
+ const LAllocation* rhs = ins->rhs();
+ uint32_t laneIndex = ins->laneIndex();
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16ReplaceLane:
+ masm.replaceLaneInt8x16(laneIndex, lhs, ToRegister(rhs), dest);
+ break;
+ case wasm::SimdOp::I16x8ReplaceLane:
+ masm.replaceLaneInt16x8(laneIndex, lhs, ToRegister(rhs), dest);
+ break;
+ case wasm::SimdOp::I32x4ReplaceLane:
+ masm.replaceLaneInt32x4(laneIndex, lhs, ToRegister(rhs), dest);
+ break;
+ case wasm::SimdOp::F32x4ReplaceLane:
+ masm.replaceLaneFloat32x4(laneIndex, lhs, ToFloatRegister(rhs), dest);
+ break;
+ case wasm::SimdOp::F64x2ReplaceLane:
+ masm.replaceLaneFloat64x2(laneIndex, lhs, ToFloatRegister(rhs), dest);
+ break;
+ default:
+ MOZ_CRASH("ReplaceLane SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReplaceInt64LaneSimd128(
+ LWasmReplaceInt64LaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_RELEASE_ASSERT(ins->simdOp() == wasm::SimdOp::I64x2ReplaceLane);
+ masm.replaceLaneInt64x2(ins->laneIndex(), ToFloatRegister(ins->lhs()),
+ ToRegister64(ins->rhs()),
+ ToFloatRegister(ins->output()));
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmScalarToSimd128(LWasmScalarToSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Splat:
+ masm.splatX16(ToRegister(ins->src()), dest);
+ break;
+ case wasm::SimdOp::I16x8Splat:
+ masm.splatX8(ToRegister(ins->src()), dest);
+ break;
+ case wasm::SimdOp::I32x4Splat:
+ masm.splatX4(ToRegister(ins->src()), dest);
+ break;
+ case wasm::SimdOp::F32x4Splat:
+ masm.splatX4(ToFloatRegister(ins->src()), dest);
+ break;
+ case wasm::SimdOp::F64x2Splat:
+ masm.splatX2(ToFloatRegister(ins->src()), dest);
+ break;
+ default:
+ MOZ_CRASH("ScalarToSimd128 SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmInt64ToSimd128(LWasmInt64ToSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ Register64 src = ToRegister64(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I64x2Splat:
+ masm.splatX2(src, dest);
+ break;
+ case wasm::SimdOp::V128Load8x8S:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.widenLowInt8x16(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load8x8U:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.unsignedWidenLowInt8x16(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load16x4S:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.widenLowInt16x8(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load16x4U:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.unsignedWidenLowInt16x8(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load32x2S:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.widenLowInt32x4(dest, dest);
+ break;
+ case wasm::SimdOp::V128Load32x2U:
+ masm.moveGPR64ToDouble(src, dest);
+ masm.unsignedWidenLowInt32x4(dest, dest);
+ break;
+ default:
+ MOZ_CRASH("Int64ToSimd128 SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmUnarySimd128(LWasmUnarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ FloatRegister dest = ToFloatRegister(ins->output());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Neg:
+ masm.negInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8Neg:
+ masm.negInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtendLowI8x16S:
+ masm.widenLowInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtendHighI8x16S:
+ masm.widenHighInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtendLowI8x16U:
+ masm.unsignedWidenLowInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtendHighI8x16U:
+ masm.unsignedWidenHighInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I32x4Neg:
+ masm.negInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtendLowI16x8S:
+ masm.widenLowInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtendHighI16x8S:
+ masm.widenHighInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtendLowI16x8U:
+ masm.unsignedWidenLowInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtendHighI16x8U:
+ masm.unsignedWidenHighInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4TruncSatF32x4S:
+ masm.truncSatFloat32x4ToInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I32x4TruncSatF32x4U:
+ masm.unsignedTruncSatFloat32x4ToInt32x4(src, dest,
+ ToFloatRegister(ins->temp()));
+ break;
+ case wasm::SimdOp::I64x2Neg:
+ masm.negInt64x2(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtendLowI32x4S:
+ masm.widenLowInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtendHighI32x4S:
+ masm.widenHighInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtendLowI32x4U:
+ masm.unsignedWidenLowInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2ExtendHighI32x4U:
+ masm.unsignedWidenHighInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Abs:
+ masm.absFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Neg:
+ masm.negFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Sqrt:
+ masm.sqrtFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4ConvertI32x4S:
+ masm.convertInt32x4ToFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4ConvertI32x4U:
+ masm.unsignedConvertInt32x4ToFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Abs:
+ masm.absFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Neg:
+ masm.negFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Sqrt:
+ masm.sqrtFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::V128Not:
+ masm.bitwiseNotSimd128(src, dest);
+ break;
+ case wasm::SimdOp::I8x16Popcnt:
+ masm.popcntInt8x16(src, dest, ToFloatRegister(ins->temp()));
+ break;
+ case wasm::SimdOp::I8x16Abs:
+ masm.absInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8Abs:
+ masm.absInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4Abs:
+ masm.absInt32x4(src, dest);
+ break;
+ case wasm::SimdOp::I64x2Abs:
+ masm.absInt64x2(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Ceil:
+ masm.ceilFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Floor:
+ masm.floorFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Trunc:
+ masm.truncFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F32x4Nearest:
+ masm.nearestFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Ceil:
+ masm.ceilFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Floor:
+ masm.floorFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Trunc:
+ masm.truncFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2Nearest:
+ masm.nearestFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F32x4DemoteF64x2Zero:
+ masm.convertFloat64x2ToFloat32x4(src, dest);
+ break;
+ case wasm::SimdOp::F64x2PromoteLowF32x4:
+ masm.convertFloat32x4ToFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2ConvertLowI32x4S:
+ masm.convertInt32x4ToFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::F64x2ConvertLowI32x4U:
+ masm.unsignedConvertInt32x4ToFloat64x2(src, dest);
+ break;
+ case wasm::SimdOp::I32x4TruncSatF64x2SZero:
+ masm.truncSatFloat64x2ToInt32x4(src, dest, ToFloatRegister(ins->temp()));
+ break;
+ case wasm::SimdOp::I32x4TruncSatF64x2UZero:
+ masm.unsignedTruncSatFloat64x2ToInt32x4(src, dest,
+ ToFloatRegister(ins->temp()));
+ break;
+ case wasm::SimdOp::I16x8ExtaddPairwiseI8x16S:
+ masm.extAddPairwiseInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I16x8ExtaddPairwiseI8x16U:
+ masm.unsignedExtAddPairwiseInt8x16(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtaddPairwiseI16x8S:
+ masm.extAddPairwiseInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4ExtaddPairwiseI16x8U:
+ masm.unsignedExtAddPairwiseInt16x8(src, dest);
+ break;
+ case wasm::SimdOp::I32x4RelaxedTruncSSatF32x4:
+ masm.truncSatFloat32x4ToInt32x4Relaxed(src, dest);
+ break;
+ case wasm::SimdOp::I32x4RelaxedTruncUSatF32x4:
+ masm.unsignedTruncSatFloat32x4ToInt32x4Relaxed(src, dest);
+ break;
+ case wasm::SimdOp::I32x4RelaxedTruncSatF64x2SZero:
+ masm.truncSatFloat64x2ToInt32x4Relaxed(src, dest);
+ break;
+ case wasm::SimdOp::I32x4RelaxedTruncSatF64x2UZero:
+ masm.unsignedTruncSatFloat64x2ToInt32x4Relaxed(src, dest);
+ break;
+ default:
+ MOZ_CRASH("Unary SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReduceSimd128(LWasmReduceSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ const LDefinition* dest = ins->output();
+ uint32_t imm = ins->imm();
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128AnyTrue:
+ masm.anyTrueSimd128(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I8x16AllTrue:
+ masm.allTrueInt8x16(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I16x8AllTrue:
+ masm.allTrueInt16x8(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I32x4AllTrue:
+ masm.allTrueInt32x4(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I64x2AllTrue:
+ masm.allTrueInt64x2(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I8x16Bitmask:
+ masm.bitmaskInt8x16(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I16x8Bitmask:
+ masm.bitmaskInt16x8(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I32x4Bitmask:
+ masm.bitmaskInt32x4(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I64x2Bitmask:
+ masm.bitmaskInt64x2(src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I8x16ExtractLaneS:
+ masm.extractLaneInt8x16(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I8x16ExtractLaneU:
+ masm.unsignedExtractLaneInt8x16(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I16x8ExtractLaneS:
+ masm.extractLaneInt16x8(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I16x8ExtractLaneU:
+ masm.unsignedExtractLaneInt16x8(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::I32x4ExtractLane:
+ masm.extractLaneInt32x4(imm, src, ToRegister(dest));
+ break;
+ case wasm::SimdOp::F32x4ExtractLane:
+ masm.extractLaneFloat32x4(imm, src, ToFloatRegister(dest));
+ break;
+ case wasm::SimdOp::F64x2ExtractLane:
+ masm.extractLaneFloat64x2(imm, src, ToFloatRegister(dest));
+ break;
+ default:
+ MOZ_CRASH("Reduce SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReduceAndBranchSimd128(
+ LWasmReduceAndBranchSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128AnyTrue:
+ // Set the zero flag if all of the lanes are zero, and branch on that.
+ masm.vptest(src, src);
+ emitBranch(Assembler::NotEqual, ins->ifTrue(), ins->ifFalse());
+ break;
+ case wasm::SimdOp::I8x16AllTrue:
+ case wasm::SimdOp::I16x8AllTrue:
+ case wasm::SimdOp::I32x4AllTrue:
+ case wasm::SimdOp::I64x2AllTrue: {
+ // Compare all lanes to zero, set the zero flag if none of the lanes are
+ // zero, and branch on that.
+ ScratchSimd128Scope tmp(masm);
+ masm.vpxor(tmp, tmp, tmp);
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16AllTrue:
+ masm.vpcmpeqb(Operand(src), tmp, tmp);
+ break;
+ case wasm::SimdOp::I16x8AllTrue:
+ masm.vpcmpeqw(Operand(src), tmp, tmp);
+ break;
+ case wasm::SimdOp::I32x4AllTrue:
+ masm.vpcmpeqd(Operand(src), tmp, tmp);
+ break;
+ case wasm::SimdOp::I64x2AllTrue:
+ masm.vpcmpeqq(Operand(src), tmp, tmp);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ masm.vptest(tmp, tmp);
+ emitBranch(Assembler::Equal, ins->ifTrue(), ins->ifFalse());
+ break;
+ }
+ case wasm::SimdOp::I16x8Bitmask: {
+ masm.bitwiseTestSimd128(SimdConstant::SplatX8(0x8000), src);
+ emitBranch(Assembler::NotEqual, ins->ifTrue(), ins->ifFalse());
+ break;
+ }
+ default:
+ MOZ_CRASH("Reduce-and-branch SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmReduceSimd128ToInt64(
+ LWasmReduceSimd128ToInt64* ins) {
+#ifdef ENABLE_WASM_SIMD
+ FloatRegister src = ToFloatRegister(ins->src());
+ Register64 dest = ToOutRegister64(ins);
+ uint32_t imm = ins->imm();
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I64x2ExtractLane:
+ masm.extractLaneInt64x2(imm, src, dest);
+ break;
+ default:
+ MOZ_CRASH("Reduce SimdOp not implemented");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmLoadLaneSimd128(LWasmLoadLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ const MWasmLoadLaneSimd128* mir = ins->mir();
+ const wasm::MemoryAccessDesc& access = mir->access();
+
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* value = ins->src();
+ Operand srcAddr = toMemoryAccessOperand(ins, offset);
+
+ masm.append(access, masm.size());
+ switch (ins->laneSize()) {
+ case 1: {
+ masm.vpinsrb(ins->laneIndex(), srcAddr, ToFloatRegister(value),
+ ToFloatRegister(value));
+ break;
+ }
+ case 2: {
+ masm.vpinsrw(ins->laneIndex(), srcAddr, ToFloatRegister(value),
+ ToFloatRegister(value));
+ break;
+ }
+ case 4: {
+ masm.vinsertps(ins->laneIndex() << 4, srcAddr, ToFloatRegister(value),
+ ToFloatRegister(value));
+ break;
+ }
+ case 8: {
+ if (ins->laneIndex() == 0) {
+ masm.vmovlps(srcAddr, ToFloatRegister(value), ToFloatRegister(value));
+ } else {
+ masm.vmovhps(srcAddr, ToFloatRegister(value), ToFloatRegister(value));
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Unsupported load lane size");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void CodeGenerator::visitWasmStoreLaneSimd128(LWasmStoreLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ const MWasmStoreLaneSimd128* mir = ins->mir();
+ const wasm::MemoryAccessDesc& access = mir->access();
+
+ uint32_t offset = access.offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* src = ins->src();
+ Operand destAddr = toMemoryAccessOperand(ins, offset);
+
+ masm.append(access, masm.size());
+ switch (ins->laneSize()) {
+ case 1: {
+ masm.vpextrb(ins->laneIndex(), ToFloatRegister(src), destAddr);
+ break;
+ }
+ case 2: {
+ masm.vpextrw(ins->laneIndex(), ToFloatRegister(src), destAddr);
+ break;
+ }
+ case 4: {
+ unsigned lane = ins->laneIndex();
+ if (lane == 0) {
+ masm.vmovss(ToFloatRegister(src), destAddr);
+ } else {
+ masm.vextractps(lane, ToFloatRegister(src), destAddr);
+ }
+ break;
+ }
+ case 8: {
+ if (ins->laneIndex() == 0) {
+ masm.vmovlps(ToFloatRegister(src), destAddr);
+ } else {
+ masm.vmovhps(ToFloatRegister(src), destAddr);
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Unsupported store lane size");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+} // namespace jit
+} // namespace js
diff --git a/js/src/jit/x86-shared/CodeGenerator-x86-shared.h b/js/src/jit/x86-shared/CodeGenerator-x86-shared.h
new file mode 100644
index 0000000000..74bcb91149
--- /dev/null
+++ b/js/src/jit/x86-shared/CodeGenerator-x86-shared.h
@@ -0,0 +1,189 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_CodeGenerator_x86_shared_h
+#define jit_x86_shared_CodeGenerator_x86_shared_h
+
+#include "jit/shared/CodeGenerator-shared.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+namespace js {
+namespace jit {
+
+class CodeGeneratorX86Shared;
+class OutOfLineBailout;
+class OutOfLineUndoALUOperation;
+class OutOfLineLoadTypedArrayOutOfBounds;
+class MulNegativeZeroCheck;
+class ModOverflowCheck;
+class ReturnZero;
+class OutOfLineTableSwitch;
+
+using OutOfLineWasmTruncateCheck =
+ OutOfLineWasmTruncateCheckBase<CodeGeneratorX86Shared>;
+
+class CodeGeneratorX86Shared : public CodeGeneratorShared {
+ friend class MoveResolverX86;
+
+ template <typename T>
+ void bailout(const T& t, LSnapshot* snapshot);
+
+ protected:
+ CodeGeneratorX86Shared(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm);
+
+ // Load a NaN or zero into a register for an out of bounds AsmJS or static
+ // typed array load.
+ class OutOfLineLoadTypedArrayOutOfBounds
+ : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
+ AnyRegister dest_;
+ Scalar::Type viewType_;
+
+ public:
+ OutOfLineLoadTypedArrayOutOfBounds(AnyRegister dest, Scalar::Type viewType)
+ : dest_(dest), viewType_(viewType) {}
+
+ AnyRegister dest() const { return dest_; }
+ Scalar::Type viewType() const { return viewType_; }
+ void accept(CodeGeneratorX86Shared* codegen) override {
+ codegen->visitOutOfLineLoadTypedArrayOutOfBounds(this);
+ }
+ };
+
+ NonAssertingLabel deoptLabel_;
+
+ Operand ToOperand(const LAllocation& a);
+ Operand ToOperand(const LAllocation* a);
+ Operand ToOperand(const LDefinition* def);
+
+#ifdef JS_PUNBOX64
+ Operand ToOperandOrRegister64(const LInt64Allocation input);
+#else
+ Register64 ToOperandOrRegister64(const LInt64Allocation input);
+#endif
+
+ MoveOperand toMoveOperand(LAllocation a) const;
+
+ void bailoutIf(Assembler::Condition condition, LSnapshot* snapshot);
+ void bailoutIf(Assembler::DoubleCondition condition, LSnapshot* snapshot);
+ void bailoutFrom(Label* label, LSnapshot* snapshot);
+ void bailout(LSnapshot* snapshot);
+
+ template <typename T1, typename T2>
+ void bailoutCmpPtr(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.cmpPtr(lhs, rhs);
+ bailoutIf(c, snapshot);
+ }
+ void bailoutTestPtr(Assembler::Condition c, Register lhs, Register rhs,
+ LSnapshot* snapshot) {
+ masm.testPtr(lhs, rhs);
+ bailoutIf(c, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutCmp32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.cmp32(lhs, rhs);
+ bailoutIf(c, snapshot);
+ }
+ template <typename T1, typename T2>
+ void bailoutTest32(Assembler::Condition c, T1 lhs, T2 rhs,
+ LSnapshot* snapshot) {
+ masm.test32(lhs, rhs);
+ bailoutIf(c, snapshot);
+ }
+ void bailoutIfFalseBool(Register reg, LSnapshot* snapshot) {
+ masm.test32(reg, Imm32(0xFF));
+ bailoutIf(Assembler::Zero, snapshot);
+ }
+ void bailoutCvttsd2si(FloatRegister src, Register dest, LSnapshot* snapshot) {
+ Label bail;
+ masm.truncateDoubleToInt32(src, dest, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+ void bailoutCvttss2si(FloatRegister src, Register dest, LSnapshot* snapshot) {
+ Label bail;
+ masm.truncateFloat32ToInt32(src, dest, &bail);
+ bailoutFrom(&bail, snapshot);
+ }
+
+ bool generateOutOfLineCode();
+
+ void emitCompare(MCompare::CompareType type, const LAllocation* left,
+ const LAllocation* right);
+
+ // Emits a branch that directs control flow to the true block if |cond| is
+ // true, and the false block if |cond| is false.
+ void emitBranch(Assembler::Condition cond, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse,
+ Assembler::NaNCond ifNaN = Assembler::NaN_HandledByCond);
+ void emitBranch(Assembler::DoubleCondition cond, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse);
+
+ void testNullEmitBranch(Assembler::Condition cond, const ValueOperand& value,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ cond = masm.testNull(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+ void testUndefinedEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ cond = masm.testUndefined(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+ void testObjectEmitBranch(Assembler::Condition cond,
+ const ValueOperand& value, MBasicBlock* ifTrue,
+ MBasicBlock* ifFalse) {
+ cond = masm.testObject(cond, value);
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+
+ void testZeroEmitBranch(Assembler::Condition cond, Register reg,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ masm.cmpPtr(reg, ImmWord(0));
+ emitBranch(cond, ifTrue, ifFalse);
+ }
+
+ void emitTableSwitchDispatch(MTableSwitch* mir, Register index,
+ Register base);
+
+ void generateInvalidateEpilogue();
+
+ void canonicalizeIfDeterministic(Scalar::Type type, const LAllocation* value);
+
+ template <typename T>
+ Operand toMemoryAccessOperand(T* lir, int32_t disp);
+
+ public:
+ // Out of line visitors.
+ void visitOutOfLineBailout(OutOfLineBailout* ool);
+ void visitOutOfLineUndoALUOperation(OutOfLineUndoALUOperation* ool);
+ void visitMulNegativeZeroCheck(MulNegativeZeroCheck* ool);
+ void visitModOverflowCheck(ModOverflowCheck* ool);
+ void visitReturnZero(ReturnZero* ool);
+ void visitOutOfLineTableSwitch(OutOfLineTableSwitch* ool);
+ void visitOutOfLineLoadTypedArrayOutOfBounds(
+ OutOfLineLoadTypedArrayOutOfBounds* ool);
+ void visitOutOfLineWasmTruncateCheck(OutOfLineWasmTruncateCheck* ool);
+};
+
+// An out-of-line bailout thunk.
+class OutOfLineBailout : public OutOfLineCodeBase<CodeGeneratorX86Shared> {
+ LSnapshot* snapshot_;
+
+ public:
+ explicit OutOfLineBailout(LSnapshot* snapshot) : snapshot_(snapshot) {}
+
+ void accept(CodeGeneratorX86Shared* codegen) override;
+
+ LSnapshot* snapshot() const { return snapshot_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_CodeGenerator_x86_shared_h */
diff --git a/js/src/jit/x86-shared/Constants-x86-shared.h b/js/src/jit/x86-shared/Constants-x86-shared.h
new file mode 100644
index 0000000000..6c59515b21
--- /dev/null
+++ b/js/src/jit/x86-shared/Constants-x86-shared.h
@@ -0,0 +1,326 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_Constants_x86_shared_h
+#define jit_x86_shared_Constants_x86_shared_h
+
+#include "mozilla/Assertions.h"
+
+#include <iterator>
+#include <stddef.h>
+#include <stdint.h>
+
+namespace js {
+namespace jit {
+
+namespace X86Encoding {
+
+enum RegisterID : uint8_t {
+ rax,
+ rcx,
+ rdx,
+ rbx,
+ rsp,
+ rbp,
+ rsi,
+ rdi
+#ifdef JS_CODEGEN_X64
+ ,
+ r8,
+ r9,
+ r10,
+ r11,
+ r12,
+ r13,
+ r14,
+ r15
+#endif
+ ,
+ invalid_reg
+};
+
+enum HRegisterID { ah = rsp, ch = rbp, dh = rsi, bh = rdi };
+
+enum XMMRegisterID
+// GCC < 8.0 has a bug with bitfields of enums with an underlying type.
+#if defined(__clang__) || __GNUC__ > 7
+ : uint8_t
+#endif
+{
+ xmm0 = 0,
+ xmm1,
+ xmm2,
+ xmm3,
+ xmm4,
+ xmm5,
+ xmm6,
+ xmm7
+#ifdef JS_CODEGEN_X64
+ ,
+ xmm8,
+ xmm9,
+ xmm10,
+ xmm11,
+ xmm12,
+ xmm13,
+ xmm14,
+ xmm15
+#endif
+ ,
+ invalid_xmm
+};
+
+inline const char* XMMRegName(XMMRegisterID reg) {
+ static const char* const names[] = {"%xmm0",
+ "%xmm1",
+ "%xmm2",
+ "%xmm3",
+ "%xmm4",
+ "%xmm5",
+ "%xmm6",
+ "%xmm7"
+#ifdef JS_CODEGEN_X64
+ ,
+ "%xmm8",
+ "%xmm9",
+ "%xmm10",
+ "%xmm11",
+ "%xmm12",
+ "%xmm13",
+ "%xmm14",
+ "%xmm15"
+#endif
+ };
+ MOZ_ASSERT(size_t(reg) < std::size(names));
+ return names[reg];
+}
+
+#ifdef JS_CODEGEN_X64
+inline const char* GPReg64Name(RegisterID reg) {
+ static const char* const names[] = {"%rax",
+ "%rcx",
+ "%rdx",
+ "%rbx",
+ "%rsp",
+ "%rbp",
+ "%rsi",
+ "%rdi"
+# ifdef JS_CODEGEN_X64
+ ,
+ "%r8",
+ "%r9",
+ "%r10",
+ "%r11",
+ "%r12",
+ "%r13",
+ "%r14",
+ "%r15"
+# endif
+ };
+ MOZ_ASSERT(size_t(reg) < std::size(names));
+ return names[reg];
+}
+#endif
+
+inline const char* GPReg32Name(RegisterID reg) {
+ static const char* const names[] = {"%eax",
+ "%ecx",
+ "%edx",
+ "%ebx",
+ "%esp",
+ "%ebp",
+ "%esi",
+ "%edi"
+#ifdef JS_CODEGEN_X64
+ ,
+ "%r8d",
+ "%r9d",
+ "%r10d",
+ "%r11d",
+ "%r12d",
+ "%r13d",
+ "%r14d",
+ "%r15d"
+#endif
+ };
+ MOZ_ASSERT(size_t(reg) < std::size(names));
+ return names[reg];
+}
+
+inline const char* GPReg16Name(RegisterID reg) {
+ static const char* const names[] = {"%ax",
+ "%cx",
+ "%dx",
+ "%bx",
+ "%sp",
+ "%bp",
+ "%si",
+ "%di"
+#ifdef JS_CODEGEN_X64
+ ,
+ "%r8w",
+ "%r9w",
+ "%r10w",
+ "%r11w",
+ "%r12w",
+ "%r13w",
+ "%r14w",
+ "%r15w"
+#endif
+ };
+ MOZ_ASSERT(size_t(reg) < std::size(names));
+ return names[reg];
+}
+
+inline const char* GPReg8Name(RegisterID reg) {
+ static const char* const names[] = {"%al",
+ "%cl",
+ "%dl",
+ "%bl"
+#ifdef JS_CODEGEN_X64
+ ,
+ "%spl",
+ "%bpl",
+ "%sil",
+ "%dil",
+ "%r8b",
+ "%r9b",
+ "%r10b",
+ "%r11b",
+ "%r12b",
+ "%r13b",
+ "%r14b",
+ "%r15b"
+#endif
+ };
+ MOZ_ASSERT(size_t(reg) < std::size(names));
+ return names[reg];
+}
+
+inline const char* GPRegName(RegisterID reg) {
+#ifdef JS_CODEGEN_X64
+ return GPReg64Name(reg);
+#else
+ return GPReg32Name(reg);
+#endif
+}
+
+inline bool HasSubregL(RegisterID reg) {
+#ifdef JS_CODEGEN_X64
+ // In 64-bit mode, all registers have an 8-bit lo subreg.
+ return true;
+#else
+ // In 32-bit mode, only the first four registers do.
+ return reg <= rbx;
+#endif
+}
+
+inline bool HasSubregH(RegisterID reg) {
+ // The first four registers always have h registers. However, note that
+ // on x64, h registers may not be used in instructions using REX
+ // prefixes. Also note that this may depend on what other registers are
+ // used!
+ return reg <= rbx;
+}
+
+inline HRegisterID GetSubregH(RegisterID reg) {
+ MOZ_ASSERT(HasSubregH(reg));
+ return HRegisterID(reg + 4);
+}
+
+inline const char* HRegName8(HRegisterID reg) {
+ static const char* const names[] = {"%ah", "%ch", "%dh", "%bh"};
+ size_t index = reg - GetSubregH(rax);
+ MOZ_ASSERT(index < std::size(names));
+ return names[index];
+}
+
+enum Condition {
+ ConditionO,
+ ConditionNO,
+ ConditionB,
+ ConditionAE,
+ ConditionE,
+ ConditionNE,
+ ConditionBE,
+ ConditionA,
+ ConditionS,
+ ConditionNS,
+ ConditionP,
+ ConditionNP,
+ ConditionL,
+ ConditionGE,
+ ConditionLE,
+ ConditionG,
+
+ ConditionC = ConditionB,
+ ConditionNC = ConditionAE
+};
+
+inline const char* CCName(Condition cc) {
+ static const char* const names[] = {"o ", "no", "b ", "ae", "e ", "ne",
+ "be", "a ", "s ", "ns", "p ", "np",
+ "l ", "ge", "le", "g "};
+ MOZ_ASSERT(size_t(cc) < std::size(names));
+ return names[cc];
+}
+
+// Conditions for CMP instructions (CMPSS, CMPSD, CMPPS, CMPPD, etc).
+enum ConditionCmp {
+ ConditionCmp_EQ = 0x0,
+ ConditionCmp_LT = 0x1,
+ ConditionCmp_LE = 0x2,
+ ConditionCmp_UNORD = 0x3,
+ ConditionCmp_NEQ = 0x4,
+ ConditionCmp_NLT = 0x5,
+ ConditionCmp_NLE = 0x6,
+ ConditionCmp_ORD = 0x7,
+ ConditionCmp_AVX_Enabled = 0x8,
+ ConditionCmp_GE = 0xD,
+};
+
+// Rounding modes for ROUNDSS / ROUNDSD.
+enum RoundingMode {
+ RoundToNearest = 0x0,
+ RoundDown = 0x1,
+ RoundUp = 0x2,
+ RoundToZero = 0x3
+};
+
+// Rounding modes for ROUNDPS / ROUNDPD. Note these are the same as for
+// RoundingMode above but incorporate the 'inexact' bit which says not to signal
+// exceptions for lost precision. It's not obvious that this bit is needed; it
+// was however suggested in the wasm SIMD proposal that led to these encodings.
+enum class SSERoundingMode {
+ RoundToNearest = 0x08,
+ RoundDown = 0x09,
+ RoundUp = 0x0A,
+ RoundToZero = 0x0B
+};
+
+// Test whether the given address will fit in an address immediate field.
+// This is always true on x86, but on x64 it's only true for addreses which
+// fit in the 32-bit immediate field.
+inline bool IsAddressImmediate(const void* address) {
+ intptr_t value = reinterpret_cast<intptr_t>(address);
+ int32_t immediate = static_cast<int32_t>(value);
+ return value == immediate;
+}
+
+// Convert the given address to a 32-bit immediate field value. This is a
+// no-op on x86, but on x64 it asserts that the address is actually a valid
+// address immediate.
+inline int32_t AddressImmediate(const void* address) {
+ MOZ_ASSERT(IsAddressImmediate(address));
+ return static_cast<int32_t>(reinterpret_cast<intptr_t>(address));
+}
+
+} // namespace X86Encoding
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_Constants_x86_shared_h */
diff --git a/js/src/jit/x86-shared/Encoding-x86-shared.h b/js/src/jit/x86-shared/Encoding-x86-shared.h
new file mode 100644
index 0000000000..d3d78f4e5e
--- /dev/null
+++ b/js/src/jit/x86-shared/Encoding-x86-shared.h
@@ -0,0 +1,508 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_Encoding_x86_shared_h
+#define jit_x86_shared_Encoding_x86_shared_h
+
+#include <type_traits>
+
+#include "jit/x86-shared/Constants-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+namespace X86Encoding {
+
+static const size_t MaxInstructionSize = 16;
+
+// These enumerated values are following the Intel documentation Volume 2C [1],
+// Appendix A.2 and Appendix A.3.
+//
+// Operand size/types as listed in the Appendix A.2. Tables of the instructions
+// and their operands can be found in the Appendix A.3.
+//
+// B = reg (VEX.vvvv of VEX prefix)
+// E = reg/mem
+// G = reg (reg field of ModR/M)
+// U = xmm (R/M field of ModR/M)
+// V = xmm (reg field of ModR/M)
+// W = xmm/mem64
+// I = immediate
+// O = offset
+//
+// b = byte (8-bit)
+// w = word (16-bit)
+// v = register size
+// d = double (32-bit)
+// dq = double-quad (128-bit) (xmm)
+// ss = scalar float 32 (xmm)
+// ps = packed float 32 (xmm)
+// sd = scalar double (xmm)
+// pd = packed double (xmm)
+// y = 32/64-bit
+// z = 16/32/64-bit
+// vqp = (*)
+//
+// (*) Some website [2] provides a convenient list of all instructions, but be
+// aware that they do not follow the Intel documentation naming, as the
+// following enumeration does. Do not use these names as a reference for adding
+// new instructions.
+//
+// [1]
+// http://www.intel.com/content/www/us/en/architecture-and-technology/64-ia-32-architectures-software-developer-manual-325462.html
+// [2] http://ref.x86asm.net/geek.html
+//
+// OPn_NAME_DstSrc
+enum OneByteOpcodeID {
+ OP_NOP_00 = 0x00,
+ OP_ADD_EbGb = 0x00,
+ OP_ADD_EvGv = 0x01,
+ OP_ADD_GvEv = 0x03,
+ OP_ADD_EAXIv = 0x05,
+ OP_OR_EbGb = 0x08,
+ OP_OR_EvGv = 0x09,
+ OP_OR_GvEv = 0x0B,
+ OP_OR_EAXIv = 0x0D,
+ OP_2BYTE_ESCAPE = 0x0F,
+ OP_NOP_0F = 0x0F,
+ OP_ADC_GvEv = 0x13,
+ OP_SBB_GvEv = 0x1B,
+ OP_NOP_1F = 0x1F,
+ OP_AND_EbGb = 0x20,
+ OP_AND_EvGv = 0x21,
+ OP_AND_GvEv = 0x23,
+ OP_AND_EAXIv = 0x25,
+ OP_SUB_EbGb = 0x28,
+ OP_SUB_EvGv = 0x29,
+ OP_SUB_GvEv = 0x2B,
+ OP_SUB_EAXIv = 0x2D,
+ PRE_PREDICT_BRANCH_NOT_TAKEN = 0x2E,
+ OP_XOR_EbGb = 0x30,
+ OP_XOR_EvGv = 0x31,
+ OP_XOR_GvEv = 0x33,
+ OP_XOR_EAXIv = 0x35,
+ OP_CMP_EbGb = 0x38,
+ OP_CMP_EvGv = 0x39,
+ OP_CMP_GbEb = 0x3A,
+ OP_CMP_GvEv = 0x3B,
+ OP_CMP_EAXIb = 0x3C,
+ OP_CMP_EAXIv = 0x3D,
+#ifdef JS_CODEGEN_X64
+ PRE_REX = 0x40,
+#endif
+ OP_NOP_40 = 0x40,
+ OP_NOP_44 = 0x44,
+ OP_PUSH_EAX = 0x50,
+ OP_POP_EAX = 0x58,
+#ifdef JS_CODEGEN_X86
+ OP_PUSHA = 0x60,
+ OP_POPA = 0x61,
+#endif
+#ifdef JS_CODEGEN_X64
+ OP_MOVSXD_GvEv = 0x63,
+#endif
+ PRE_OPERAND_SIZE = 0x66,
+ PRE_SSE_66 = 0x66,
+ OP_NOP_66 = 0x66,
+ OP_PUSH_Iz = 0x68,
+ OP_IMUL_GvEvIz = 0x69,
+ OP_PUSH_Ib = 0x6a,
+ OP_IMUL_GvEvIb = 0x6b,
+ OP_JCC_rel8 = 0x70,
+ OP_GROUP1_EbIb = 0x80,
+ OP_NOP_80 = 0x80,
+ OP_GROUP1_EvIz = 0x81,
+ OP_GROUP1_EvIb = 0x83,
+ OP_TEST_EbGb = 0x84,
+ OP_NOP_84 = 0x84,
+ OP_TEST_EvGv = 0x85,
+ OP_XCHG_GbEb = 0x86,
+ OP_XCHG_GvEv = 0x87,
+ OP_MOV_EbGv = 0x88,
+ OP_MOV_EvGv = 0x89,
+ OP_MOV_GvEb = 0x8A,
+ OP_MOV_GvEv = 0x8B,
+ OP_LEA = 0x8D,
+ OP_GROUP1A_Ev = 0x8F,
+ OP_NOP = 0x90,
+ OP_PUSHFLAGS = 0x9C,
+ OP_POPFLAGS = 0x9D,
+ OP_CDQ = 0x99,
+ OP_MOV_EAXOv = 0xA1,
+ OP_MOV_OvEAX = 0xA3,
+ OP_TEST_EAXIb = 0xA8,
+ OP_TEST_EAXIv = 0xA9,
+ OP_MOV_EbIb = 0xB0,
+ OP_MOV_EAXIv = 0xB8,
+ OP_GROUP2_EvIb = 0xC1,
+ OP_ADDP_ST0_ST1 = 0xC1,
+ OP_RET_Iz = 0xC2,
+ PRE_VEX_C4 = 0xC4,
+ PRE_VEX_C5 = 0xC5,
+ OP_RET = 0xC3,
+ OP_GROUP11_EvIb = 0xC6,
+ OP_GROUP11_EvIz = 0xC7,
+ OP_INT3 = 0xCC,
+ OP_GROUP2_Ev1 = 0xD1,
+ OP_GROUP2_EvCL = 0xD3,
+ OP_FPU6 = 0xDD,
+ OP_FPU6_F32 = 0xD9,
+ OP_FPU6_ADDP = 0xDE,
+ OP_FILD = 0xDF,
+ OP_CALL_rel32 = 0xE8,
+ OP_JMP_rel32 = 0xE9,
+ OP_JMP_rel8 = 0xEB,
+ PRE_LOCK = 0xF0,
+ PRE_SSE_F2 = 0xF2,
+ PRE_SSE_F3 = 0xF3,
+ OP_HLT = 0xF4,
+ OP_GROUP3_EbIb = 0xF6,
+ OP_GROUP3_Ev = 0xF7,
+ OP_GROUP3_EvIz =
+ 0xF7, // OP_GROUP3_Ev has an immediate, when instruction is a test.
+ OP_GROUP5_Ev = 0xFF
+};
+
+enum class ShiftID {
+ vpsrlx = 2,
+ vpsrldq = 3,
+ vpsrad = 4,
+ vpsllx = 6,
+ vpslldq = 7
+};
+
+enum TwoByteOpcodeID {
+ OP2_UD2 = 0x0B,
+ OP2_MOVSD_VsdWsd = 0x10,
+ OP2_MOVPS_VpsWps = 0x10,
+ OP2_MOVSD_WsdVsd = 0x11,
+ OP2_MOVPS_WpsVps = 0x11,
+ OP2_MOVDDUP_VqWq = 0x12,
+ OP2_MOVHLPS_VqUq = 0x12,
+ OP2_MOVSLDUP_VpsWps = 0x12,
+ OP2_MOVLPS_VqEq = 0x12,
+ OP2_MOVLPS_EqVq = 0x13,
+ OP2_UNPCKLPS_VsdWsd = 0x14,
+ OP2_UNPCKHPS_VsdWsd = 0x15,
+ OP2_MOVLHPS_VqUq = 0x16,
+ OP2_MOVSHDUP_VpsWps = 0x16,
+ OP2_MOVHPS_VqEq = 0x16,
+ OP2_MOVHPS_EqVq = 0x17,
+ OP2_MOVAPD_VsdWsd = 0x28,
+ OP2_MOVAPS_VsdWsd = 0x28,
+ OP2_MOVAPS_WsdVsd = 0x29,
+ OP2_CVTSI2SD_VsdEd = 0x2A,
+ OP2_CVTTSD2SI_GdWsd = 0x2C,
+ OP2_UCOMISD_VsdWsd = 0x2E,
+ OP2_CMOVCC_GvEv = 0x40,
+ OP2_MOVMSKPD_EdVd = 0x50,
+ OP2_ANDPS_VpsWps = 0x54,
+ OP2_ANDNPS_VpsWps = 0x55,
+ OP2_ORPS_VpsWps = 0x56,
+ OP2_XORPS_VpsWps = 0x57,
+ OP2_ADDSD_VsdWsd = 0x58,
+ OP2_ADDPS_VpsWps = 0x58,
+ OP2_ADDPD_VpdWpd = 0x58,
+ OP2_MULSD_VsdWsd = 0x59,
+ OP2_MULPD_VpdWpd = 0x59,
+ OP2_MULPS_VpsWps = 0x59,
+ OP2_CVTSS2SD_VsdEd = 0x5A,
+ OP2_CVTSD2SS_VsdEd = 0x5A,
+ OP2_CVTPS2PD_VpdWps = 0x5A,
+ OP2_CVTPD2PS_VpsWpd = 0x5A,
+ OP2_CVTTPS2DQ_VdqWps = 0x5B,
+ OP2_CVTDQ2PS_VpsWdq = 0x5B,
+ OP2_SUBSD_VsdWsd = 0x5C,
+ OP2_SUBPS_VpsWps = 0x5C,
+ OP2_SUBPD_VpdWpd = 0x5C,
+ OP2_MINSD_VsdWsd = 0x5D,
+ OP2_MINSS_VssWss = 0x5D,
+ OP2_MINPS_VpsWps = 0x5D,
+ OP2_MINPD_VpdWpd = 0x5D,
+ OP2_DIVSD_VsdWsd = 0x5E,
+ OP2_DIVPS_VpsWps = 0x5E,
+ OP2_DIVPD_VpdWpd = 0x5E,
+ OP2_MAXSD_VsdWsd = 0x5F,
+ OP2_MAXSS_VssWss = 0x5F,
+ OP2_MAXPS_VpsWps = 0x5F,
+ OP2_MAXPD_VpdWpd = 0x5F,
+ OP2_SQRTSD_VsdWsd = 0x51,
+ OP2_SQRTSS_VssWss = 0x51,
+ OP2_SQRTPS_VpsWps = 0x51,
+ OP2_SQRTPD_VpdWpd = 0x51,
+ OP2_RSQRTPS_VpsWps = 0x52,
+ OP2_RCPPS_VpsWps = 0x53,
+ OP2_ANDPD_VpdWpd = 0x54,
+ OP2_ORPD_VpdWpd = 0x56,
+ OP2_XORPD_VpdWpd = 0x57,
+ OP2_PUNPCKLBW_VdqWdq = 0x60,
+ OP2_PUNPCKLWD_VdqWdq = 0x61,
+ OP2_PUNPCKLDQ_VdqWdq = 0x62,
+ OP2_PACKSSWB_VdqWdq = 0x63,
+ OP2_PCMPGTB_VdqWdq = 0x64,
+ OP2_PCMPGTW_VdqWdq = 0x65,
+ OP2_PCMPGTD_VdqWdq = 0x66,
+ OP2_PACKUSWB_VdqWdq = 0x67,
+ OP2_PUNPCKHBW_VdqWdq = 0x68,
+ OP2_PUNPCKHWD_VdqWdq = 0x69,
+ OP2_PUNPCKHDQ_VdqWdq = 0x6A,
+ OP2_PACKSSDW_VdqWdq = 0x6B,
+ OP2_PUNPCKLQDQ_VdqWdq = 0x6C,
+ OP2_PUNPCKHQDQ_VdqWdq = 0x6D,
+ OP2_MOVD_VdEd = 0x6E,
+ OP2_MOVDQ_VsdWsd = 0x6F,
+ OP2_MOVDQ_VdqWdq = 0x6F,
+ OP2_PSHUFD_VdqWdqIb = 0x70,
+ OP2_PSHUFLW_VdqWdqIb = 0x70,
+ OP2_PSHUFHW_VdqWdqIb = 0x70,
+ OP2_PSLLW_UdqIb = 0x71,
+ OP2_PSRAW_UdqIb = 0x71,
+ OP2_PSRLW_UdqIb = 0x71,
+ OP2_PSLLD_UdqIb = 0x72,
+ OP2_PSRAD_UdqIb = 0x72,
+ OP2_PSRLD_UdqIb = 0x72,
+ OP2_PSRLDQ_Vd = 0x73,
+ OP2_PCMPEQB_VdqWdq = 0x74,
+ OP2_PCMPEQW_VdqWdq = 0x75,
+ OP2_PCMPEQD_VdqWdq = 0x76,
+ OP2_HADDPD = 0x7C,
+ OP2_MOVD_EdVd = 0x7E,
+ OP2_MOVQ_VdWd = 0x7E,
+ OP2_MOVDQ_WdqVdq = 0x7F,
+ OP2_JCC_rel32 = 0x80,
+ OP_SETCC = 0x90,
+ OP2_SHLD = 0xA4,
+ OP2_SHLD_GvEv = 0xA5,
+ OP2_SHRD = 0xAC,
+ OP2_SHRD_GvEv = 0xAD,
+ OP_FENCE = 0xAE,
+ OP2_IMUL_GvEv = 0xAF,
+ OP2_CMPXCHG_GvEb = 0xB0,
+ OP2_CMPXCHG_GvEw = 0xB1,
+ OP2_POPCNT_GvEv = 0xB8,
+ OP2_BSF_GvEv = 0xBC,
+ OP2_TZCNT_GvEv = 0xBC,
+ OP2_BSR_GvEv = 0xBD,
+ OP2_LZCNT_GvEv = 0xBD,
+ OP2_MOVSX_GvEb = 0xBE,
+ OP2_MOVSX_GvEw = 0xBF,
+ OP2_MOVZX_GvEb = 0xB6,
+ OP2_MOVZX_GvEw = 0xB7,
+ OP2_XADD_EbGb = 0xC0,
+ OP2_XADD_EvGv = 0xC1,
+ OP2_CMPPS_VpsWps = 0xC2,
+ OP2_CMPPD_VpdWpd = 0xC2,
+ OP2_PINSRW = 0xC4,
+ OP2_PEXTRW_GdUdIb = 0xC5,
+ OP2_SHUFPS_VpsWpsIb = 0xC6,
+ OP2_SHUFPD_VpdWpdIb = 0xC6,
+ OP2_CMPXCHGNB = 0xC7, // CMPXCHG8B; CMPXCHG16B with REX
+ OP2_BSWAP = 0xC8,
+ OP2_PSRLW_VdqWdq = 0xD1,
+ OP2_PSRLD_VdqWdq = 0xD2,
+ OP2_PSRLQ_VdqWdq = 0xD3,
+ OP2_PADDQ_VdqWdq = 0xD4,
+ OP2_PMULLW_VdqWdq = 0xD5,
+ OP2_MOVQ_WdVd = 0xD6,
+ OP2_PMOVMSKB_EdVd = 0xD7,
+ OP2_PSUBUSB_VdqWdq = 0xD8,
+ OP2_PSUBUSW_VdqWdq = 0xD9,
+ OP2_PMINUB_VdqWdq = 0xDA,
+ OP2_PANDDQ_VdqWdq = 0xDB,
+ OP2_PADDUSB_VdqWdq = 0xDC,
+ OP2_PADDUSW_VdqWdq = 0xDD,
+ OP2_PMAXUB_VdqWdq = 0xDE,
+ OP2_PANDNDQ_VdqWdq = 0xDF,
+ OP2_PAVGB_VdqWdq = 0xE0,
+ OP2_PSRAW_VdqWdq = 0xE1,
+ OP2_PSRAD_VdqWdq = 0xE2,
+ OP2_PAVGW_VdqWdq = 0xE3,
+ OP2_PMULHUW_VdqWdq = 0xE4,
+ OP2_PMULHW_VdqWdq = 0xE5,
+ OP2_CVTDQ2PD_VpdWdq = 0xE6,
+ OP2_CVTTPD2DQ_VdqWpd = 0xE6,
+ OP2_PSUBSB_VdqWdq = 0xE8,
+ OP2_PSUBSW_VdqWdq = 0xE9,
+ OP2_PMINSW_VdqWdq = 0xEA,
+ OP2_PORDQ_VdqWdq = 0xEB,
+ OP2_PADDSB_VdqWdq = 0xEC,
+ OP2_PADDSW_VdqWdq = 0xED,
+ OP2_PMAXSW_VdqWdq = 0xEE,
+ OP2_PXORDQ_VdqWdq = 0xEF,
+ OP2_PSLLW_VdqWdq = 0xF1,
+ OP2_PSLLD_VdqWdq = 0xF2,
+ OP2_PSLLQ_VdqWdq = 0xF3,
+ OP2_PMULUDQ_VdqWdq = 0xF4,
+ OP2_PMADDWD_VdqWdq = 0xF5,
+ OP2_PSUBB_VdqWdq = 0xF8,
+ OP2_PSUBW_VdqWdq = 0xF9,
+ OP2_PSUBD_VdqWdq = 0xFA,
+ OP2_PSUBQ_VdqWdq = 0xFB,
+ OP2_PADDB_VdqWdq = 0xFC,
+ OP2_PADDW_VdqWdq = 0xFD,
+ OP2_PADDD_VdqWdq = 0xFE
+};
+
+enum ThreeByteOpcodeID {
+ OP3_PSHUFB_VdqWdq = 0x00,
+ OP3_PHADDD_VdqWdq = 0x02,
+ OP3_PMADDUBSW_VdqWdq = 0x04,
+ OP3_ROUNDPS_VpsWps = 0x08,
+ OP3_ROUNDPD_VpdWpd = 0x09,
+ OP3_ROUNDSS_VsdWsd = 0x0A,
+ OP3_ROUNDSD_VsdWsd = 0x0B,
+ OP3_PMULHRSW_VdqWdq = 0x0B,
+ OP3_BLENDPS_VpsWpsIb = 0x0C,
+ OP3_PBLENDW_VdqWdqIb = 0x0E,
+ OP3_PALIGNR_VdqWdqIb = 0x0F,
+ OP3_PBLENDVB_VdqWdq = 0x10,
+ OP3_BLENDVPS_VdqWdq = 0x14,
+ OP3_PEXTRB_EvVdqIb = 0x14,
+ OP3_PEXTRW_EwVdqIb = 0x15,
+ OP3_BLENDVPD_VdqWdq = 0x15,
+ OP3_PEXTRD_EvVdqIb = 0x16,
+ OP3_PEXTRQ_EvVdqIb = 0x16,
+ OP3_PTEST_VdVd = 0x17,
+ OP3_EXTRACTPS_EdVdqIb = 0x17,
+ OP3_VBROADCASTSS_VxWd = 0x18,
+ OP3_PABSB_VdqWdq = 0x1C,
+ OP3_PABSW_VdqWdq = 0x1D,
+ OP3_PABSD_VdqWdq = 0x1E,
+ OP3_PINSRB_VdqEvIb = 0x20,
+ OP3_PMOVSXBW_VdqWdq = 0x20,
+ OP3_INSERTPS_VpsUps = 0x21,
+ OP3_PINSRD_VdqEvIb = 0x22,
+ OP3_PINSRQ_VdqEvIb = 0x22,
+ OP3_PMOVSXWD_VdqWdq = 0x23,
+ OP3_PMOVSXDQ_VdqWdq = 0x25,
+ OP3_PMULDQ_VdqWdq = 0x28,
+ OP3_PCMPEQQ_VdqWdq = 0x29,
+ OP3_PACKUSDW_VdqWdq = 0x2B,
+ OP3_PMOVZXBW_VdqWdq = 0x30,
+ OP3_PMOVZXWD_VdqWdq = 0x33,
+ OP3_PMOVZXDQ_VdqWdq = 0x35,
+ OP3_PCMPGTQ_VdqWdq = 0x37,
+ OP3_PMINSB_VdqWdq = 0x38,
+ OP3_PMINSD_VdqWdq = 0x39,
+ OP3_PMINUW_VdqWdq = 0x3A,
+ OP3_PMINUD_VdqWdq = 0x3B,
+ OP3_PMAXSB_VdqWdq = 0x3C,
+ OP3_PMAXSD_VdqWdq = 0x3D,
+ OP3_PMAXUW_VdqWdq = 0x3E,
+ OP3_PMAXUD_VdqWdq = 0x3F,
+ OP3_PMULLD_VdqWdq = 0x40,
+ OP3_VBLENDVPS_VdqWdq = 0x4A,
+ OP3_VBLENDVPD_VdqWdq = 0x4B,
+ OP3_VPBLENDVB_VdqWdq = 0x4C,
+ OP3_VBROADCASTD_VxWx = 0x58,
+ OP3_VBROADCASTQ_VxWx = 0x59,
+ OP3_VBROADCASTB_VxWx = 0x78,
+ OP3_VBROADCASTW_VxWx = 0x79,
+ OP3_VFMADD231PS_VxHxWx = 0xB8,
+ OP3_VFMADD231PD_VxHxWx = 0xB8,
+ OP3_VFNMADD231PS_VxHxWx = 0xBC,
+ OP3_VFNMADD231PD_VxHxWx = 0xBC,
+ OP3_SHLX_GyEyBy = 0xF7,
+ OP3_SARX_GyEyBy = 0xF7,
+ OP3_SHRX_GyEyBy = 0xF7,
+};
+
+// Test whether the given opcode should be printed with its operands reversed.
+inline bool IsXMMReversedOperands(TwoByteOpcodeID opcode) {
+ switch (opcode) {
+ case OP2_MOVSD_WsdVsd: // also OP2_MOVPS_WpsVps
+ case OP2_MOVAPS_WsdVsd:
+ case OP2_MOVDQ_WdqVdq:
+ return true;
+ default:
+ break;
+ }
+ return false;
+}
+
+enum ThreeByteEscape { ESCAPE_38 = 0x38, ESCAPE_3A = 0x3A };
+
+enum VexOperandType { VEX_PS = 0, VEX_PD = 1, VEX_SS = 2, VEX_SD = 3 };
+
+inline OneByteOpcodeID jccRel8(Condition cond) {
+ return OneByteOpcodeID(OP_JCC_rel8 + std::underlying_type_t<Condition>(cond));
+}
+inline TwoByteOpcodeID jccRel32(Condition cond) {
+ return TwoByteOpcodeID(OP2_JCC_rel32 +
+ std::underlying_type_t<Condition>(cond));
+}
+inline TwoByteOpcodeID setccOpcode(Condition cond) {
+ return TwoByteOpcodeID(OP_SETCC + std::underlying_type_t<Condition>(cond));
+}
+inline TwoByteOpcodeID cmovccOpcode(Condition cond) {
+ return TwoByteOpcodeID(OP2_CMOVCC_GvEv +
+ std::underlying_type_t<Condition>(cond));
+}
+
+enum GroupOpcodeID {
+ GROUP1_OP_ADD = 0,
+ GROUP1_OP_OR = 1,
+ GROUP1_OP_ADC = 2,
+ GROUP1_OP_SBB = 3,
+ GROUP1_OP_AND = 4,
+ GROUP1_OP_SUB = 5,
+ GROUP1_OP_XOR = 6,
+ GROUP1_OP_CMP = 7,
+
+ GROUP1A_OP_POP = 0,
+
+ GROUP2_OP_ROL = 0,
+ GROUP2_OP_ROR = 1,
+ GROUP2_OP_SHL = 4,
+ GROUP2_OP_SHR = 5,
+ GROUP2_OP_SAR = 7,
+
+ GROUP3_OP_TEST = 0,
+ GROUP3_OP_NOT = 2,
+ GROUP3_OP_NEG = 3,
+ GROUP3_OP_MUL = 4,
+ GROUP3_OP_IMUL = 5,
+ GROUP3_OP_DIV = 6,
+ GROUP3_OP_IDIV = 7,
+
+ GROUP5_OP_INC = 0,
+ GROUP5_OP_DEC = 1,
+ GROUP5_OP_CALLN = 2,
+ GROUP5_OP_JMPN = 4,
+ GROUP5_OP_PUSH = 6,
+
+ FILD_OP_64 = 5,
+
+ FPU6_OP_FLD = 0,
+ FPU6_OP_FISTTP = 1,
+ FPU6_OP_FSTP = 3,
+ FPU6_OP_FLDCW = 5,
+ FPU6_OP_FISTP = 7,
+
+ GROUP11_MOV = 0
+};
+
+static const RegisterID noBase = rbp;
+static const RegisterID hasSib = rsp;
+static const RegisterID noIndex = rsp;
+#ifdef JS_CODEGEN_X64
+static const RegisterID noBase2 = r13;
+static const RegisterID hasSib2 = r12;
+#endif
+
+enum ModRmMode {
+ ModRmMemoryNoDisp,
+ ModRmMemoryDisp8,
+ ModRmMemoryDisp32,
+ ModRmRegister
+};
+
+} // namespace X86Encoding
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_Encoding_x86_shared_h */
diff --git a/js/src/jit/x86-shared/LIR-x86-shared.h b/js/src/jit/x86-shared/LIR-x86-shared.h
new file mode 100644
index 0000000000..27f9f86468
--- /dev/null
+++ b/js/src/jit/x86-shared/LIR-x86-shared.h
@@ -0,0 +1,304 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_LIR_x86_shared_h
+#define jit_x86_shared_LIR_x86_shared_h
+
+namespace js {
+namespace jit {
+
+class LDivI : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(DivI)
+
+ LDivI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const char* extraName() const {
+ if (mir()->isTruncated()) {
+ if (mir()->canBeNegativeZero()) {
+ return mir()->canBeNegativeOverflow()
+ ? "Truncate_NegativeZero_NegativeOverflow"
+ : "Truncate_NegativeZero";
+ }
+ return mir()->canBeNegativeOverflow() ? "Truncate_NegativeOverflow"
+ : "Truncate";
+ }
+ if (mir()->canBeNegativeZero()) {
+ return mir()->canBeNegativeOverflow() ? "NegativeZero_NegativeOverflow"
+ : "NegativeZero";
+ }
+ return mir()->canBeNegativeOverflow() ? "NegativeOverflow" : nullptr;
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+// Signed division by a power-of-two constant.
+class LDivPowTwoI : public LBinaryMath<0> {
+ const int32_t shift_;
+ const bool negativeDivisor_;
+
+ public:
+ LIR_HEADER(DivPowTwoI)
+
+ LDivPowTwoI(const LAllocation& lhs, const LAllocation& lhsCopy, int32_t shift,
+ bool negativeDivisor)
+ : LBinaryMath(classOpcode),
+ shift_(shift),
+ negativeDivisor_(negativeDivisor) {
+ setOperand(0, lhs);
+ setOperand(1, lhsCopy);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+ const LAllocation* numeratorCopy() { return getOperand(1); }
+ int32_t shift() const { return shift_; }
+ bool negativeDivisor() const { return negativeDivisor_; }
+ MDiv* mir() const { return mir_->toDiv(); }
+};
+
+class LDivOrModConstantI : public LInstructionHelper<1, 1, 1> {
+ const int32_t denominator_;
+
+ public:
+ LIR_HEADER(DivOrModConstantI)
+
+ LDivOrModConstantI(const LAllocation& lhs, int32_t denominator,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode), denominator_(denominator) {
+ setOperand(0, lhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+ int32_t denominator() const { return denominator_; }
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+ bool canBeNegativeDividend() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeDividend();
+ }
+};
+
+class LModI : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(ModI)
+
+ LModI(const LAllocation& lhs, const LAllocation& rhs, const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const char* extraName() const {
+ return mir()->isTruncated() ? "Truncated" : nullptr;
+ }
+
+ const LDefinition* remainder() { return getDef(0); }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+// This class performs a simple x86 'div', yielding either a quotient or
+// remainder depending on whether this instruction is defined to output eax
+// (quotient) or edx (remainder).
+class LUDivOrMod : public LBinaryMath<1> {
+ public:
+ LIR_HEADER(UDivOrMod);
+
+ LUDivOrMod(const LAllocation& lhs, const LAllocation& rhs,
+ const LDefinition& temp)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setTemp(0, temp);
+ }
+
+ const LDefinition* remainder() { return getTemp(0); }
+
+ const char* extraName() const {
+ return mir()->isTruncated() ? "Truncated" : nullptr;
+ }
+
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+
+ bool canBeDivideByZero() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeDivideByZero();
+ }
+ return mir_->toDiv()->canBeDivideByZero();
+ }
+
+ bool trapOnError() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->trapOnError();
+ }
+ return mir_->toDiv()->trapOnError();
+ }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LUDivOrModConstant : public LInstructionHelper<1, 1, 1> {
+ const uint32_t denominator_;
+
+ public:
+ LIR_HEADER(UDivOrModConstant)
+
+ LUDivOrModConstant(const LAllocation& lhs, uint32_t denominator,
+ const LDefinition& temp)
+ : LInstructionHelper(classOpcode), denominator_(denominator) {
+ setOperand(0, lhs);
+ setTemp(0, temp);
+ }
+
+ const LAllocation* numerator() { return getOperand(0); }
+ uint32_t denominator() const { return denominator_; }
+ MBinaryArithInstruction* mir() const {
+ MOZ_ASSERT(mir_->isDiv() || mir_->isMod());
+ return static_cast<MBinaryArithInstruction*>(mir_);
+ }
+ bool canBeNegativeDividend() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->canBeNegativeDividend();
+ }
+ return mir_->toDiv()->canBeNegativeDividend();
+ }
+ bool trapOnError() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->trapOnError();
+ }
+ return mir_->toDiv()->trapOnError();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ if (mir_->isMod()) {
+ return mir_->toMod()->bytecodeOffset();
+ }
+ return mir_->toDiv()->bytecodeOffset();
+ }
+};
+
+class LModPowTwoI : public LInstructionHelper<1, 1, 0> {
+ const int32_t shift_;
+
+ public:
+ LIR_HEADER(ModPowTwoI)
+
+ LModPowTwoI(const LAllocation& lhs, int32_t shift)
+ : LInstructionHelper(classOpcode), shift_(shift) {
+ setOperand(0, lhs);
+ }
+
+ int32_t shift() const { return shift_; }
+ const LDefinition* remainder() { return getDef(0); }
+ MMod* mir() const { return mir_->toMod(); }
+};
+
+// Takes a tableswitch with an integer to decide
+class LTableSwitch : public LInstructionHelper<0, 1, 2> {
+ public:
+ LIR_HEADER(TableSwitch)
+
+ LTableSwitch(const LAllocation& in, const LDefinition& inputCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setTemp(0, inputCopy);
+ setTemp(1, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+
+ const LAllocation* index() { return getOperand(0); }
+ const LDefinition* tempInt() { return getTemp(0); }
+ const LDefinition* tempPointer() { return getTemp(1); }
+};
+
+// Takes a tableswitch with a value to decide
+class LTableSwitchV : public LInstructionHelper<0, BOX_PIECES, 3> {
+ public:
+ LIR_HEADER(TableSwitchV)
+
+ LTableSwitchV(const LBoxAllocation& input, const LDefinition& inputCopy,
+ const LDefinition& floatCopy,
+ const LDefinition& jumpTablePointer, MTableSwitch* ins)
+ : LInstructionHelper(classOpcode) {
+ setBoxOperand(InputValue, input);
+ setTemp(0, inputCopy);
+ setTemp(1, floatCopy);
+ setTemp(2, jumpTablePointer);
+ setMir(ins);
+ }
+
+ MTableSwitch* mir() const { return mir_->toTableSwitch(); }
+
+ static const size_t InputValue = 0;
+
+ const LDefinition* tempInt() { return getTemp(0); }
+ const LDefinition* tempFloat() { return getTemp(1); }
+ const LDefinition* tempPointer() { return getTemp(2); }
+};
+
+class LMulI : public LBinaryMath<0, 1> {
+ public:
+ LIR_HEADER(MulI)
+
+ LMulI(const LAllocation& lhs, const LAllocation& rhs,
+ const LAllocation& lhsCopy)
+ : LBinaryMath(classOpcode) {
+ setOperand(0, lhs);
+ setOperand(1, rhs);
+ setOperand(2, lhsCopy);
+ }
+
+ const char* extraName() const {
+ return (mir()->mode() == MMul::Integer)
+ ? "Integer"
+ : (mir()->canBeNegativeZero() ? "CanBeNegativeZero" : nullptr);
+ }
+
+ MMul* mir() const { return mir_->toMul(); }
+ const LAllocation* lhsCopy() { return this->getOperand(2); }
+};
+
+class LInt64ToFloatingPoint : public LInstructionHelper<1, INT64_PIECES, 1> {
+ public:
+ LIR_HEADER(Int64ToFloatingPoint);
+
+ LInt64ToFloatingPoint(const LInt64Allocation& in, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setInt64Operand(0, in);
+ setTemp(0, temp);
+ }
+
+ MInt64ToFloatingPoint* mir() const { return mir_->toInt64ToFloatingPoint(); }
+
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_LIR_x86_shared_h */
diff --git a/js/src/jit/x86-shared/Lowering-x86-shared.cpp b/js/src/jit/x86-shared/Lowering-x86-shared.cpp
new file mode 100644
index 0000000000..3b69e06876
--- /dev/null
+++ b/js/src/jit/x86-shared/Lowering-x86-shared.cpp
@@ -0,0 +1,1869 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86-shared/Lowering-x86-shared.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::Abs;
+using mozilla::FloorLog2;
+using mozilla::Maybe;
+using mozilla::Nothing;
+using mozilla::Some;
+
+LTableSwitch* LIRGeneratorX86Shared::newLTableSwitch(
+ const LAllocation& in, const LDefinition& inputCopy,
+ MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitch(in, inputCopy, temp(), tableswitch);
+}
+
+LTableSwitchV* LIRGeneratorX86Shared::newLTableSwitchV(
+ MTableSwitch* tableswitch) {
+ return new (alloc()) LTableSwitchV(useBox(tableswitch->getOperand(0)), temp(),
+ tempDouble(), temp(), tableswitch);
+}
+
+void LIRGenerator::visitPowHalf(MPowHalf* ins) {
+ MDefinition* input = ins->input();
+ MOZ_ASSERT(input->type() == MIRType::Double);
+ LPowHalfD* lir = new (alloc()) LPowHalfD(useRegisterAtStart(input));
+ define(lir, ins);
+}
+
+void LIRGeneratorX86Shared::lowerForShift(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegisterAtStart(lhs));
+
+ // Shift operand should be constant or, unless BMI2 is available, in register
+ // ecx. x86 can't shift a non-ecx register.
+ if (rhs->isConstant()) {
+ ins->setOperand(1, useOrConstantAtStart(rhs));
+ } else if (Assembler::HasBMI2() && !mir->isRotate()) {
+ ins->setOperand(1, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useRegister(rhs)
+ : useRegisterAtStart(rhs));
+ } else {
+ ins->setOperand(1, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useFixed(rhs, ecx)
+ : useFixedAtStart(rhs, ecx));
+ }
+
+ defineReuseInput(ins, mir, 0);
+}
+
+template <size_t Temps>
+void LIRGeneratorX86Shared::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+#if defined(JS_NUNBOX32)
+ if (mir->isRotate()) {
+ ins->setTemp(0, temp());
+ }
+#endif
+
+ static_assert(LShiftI64::Rhs == INT64_PIECES,
+ "Assume Rhs is located at INT64_PIECES.");
+ static_assert(LRotateI64::Count == INT64_PIECES,
+ "Assume Count is located at INT64_PIECES.");
+
+ // Shift operand should be constant or, unless BMI2 is available, in register
+ // ecx. x86 can't shift a non-ecx register.
+ if (rhs->isConstant()) {
+ ins->setOperand(INT64_PIECES, useOrConstantAtStart(rhs));
+#ifdef JS_CODEGEN_X64
+ } else if (Assembler::HasBMI2() && !mir->isRotate()) {
+ ins->setOperand(INT64_PIECES, useRegister(rhs));
+#endif
+ } else {
+ // The operands are int64, but we only care about the lower 32 bits of
+ // the RHS. On 32-bit, the code below will load that part in ecx and
+ // will discard the upper half.
+ ensureDefined(rhs);
+ LUse use(ecx);
+ use.setVirtualRegister(rhs->virtualRegister());
+ ins->setOperand(INT64_PIECES, use);
+ }
+
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+template void LIRGeneratorX86Shared::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+template void LIRGeneratorX86Shared::lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, 1>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+void LIRGeneratorX86Shared::lowerForCompareI64AndBranch(
+ MTest* mir, MCompare* comp, JSOp op, MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse) {
+ auto* lir = new (alloc())
+ LCompareI64AndBranch(comp, op, useInt64Register(left),
+ useInt64OrConstant(right), ifTrue, ifFalse);
+ add(lir, mir);
+}
+
+void LIRGeneratorX86Shared::lowerForALU(LInstructionHelper<1, 1, 0>* ins,
+ MDefinition* mir, MDefinition* input) {
+ ins->setOperand(0, useRegisterAtStart(input));
+ defineReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX86Shared::lowerForALU(LInstructionHelper<1, 2, 0>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ ins->setOperand(0, useRegisterAtStart(lhs));
+ ins->setOperand(1, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useOrConstant(rhs)
+ : useOrConstantAtStart(rhs));
+ defineReuseInput(ins, mir, 0);
+}
+
+template <size_t Temps>
+void LIRGeneratorX86Shared::lowerForFPU(LInstructionHelper<1, 2, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs) {
+ // Without AVX, we'll need to use the x86 encodings where one of the
+ // inputs must be the same location as the output.
+ if (!Assembler::HasAVX()) {
+ ins->setOperand(0, useRegisterAtStart(lhs));
+ ins->setOperand(
+ 1, willHaveDifferentLIRNodes(lhs, rhs) ? use(rhs) : useAtStart(rhs));
+ defineReuseInput(ins, mir, 0);
+ } else {
+ ins->setOperand(0, useRegisterAtStart(lhs));
+ ins->setOperand(1, useAtStart(rhs));
+ define(ins, mir);
+ }
+}
+
+template void LIRGeneratorX86Shared::lowerForFPU(
+ LInstructionHelper<1, 2, 0>* ins, MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+template void LIRGeneratorX86Shared::lowerForFPU(
+ LInstructionHelper<1, 2, 1>* ins, MDefinition* mir, MDefinition* lhs,
+ MDefinition* rhs);
+
+void LIRGeneratorX86Shared::lowerForBitAndAndBranch(LBitAndAndBranch* baab,
+ MInstruction* mir,
+ MDefinition* lhs,
+ MDefinition* rhs) {
+ baab->setOperand(0, useRegisterAtStart(lhs));
+ baab->setOperand(1, useRegisterOrConstantAtStart(rhs));
+ add(baab, mir);
+}
+
+void LIRGeneratorX86Shared::lowerNegI(MInstruction* ins, MDefinition* input) {
+ defineReuseInput(new (alloc()) LNegI(useRegisterAtStart(input)), ins, 0);
+}
+
+void LIRGeneratorX86Shared::lowerNegI64(MInstruction* ins, MDefinition* input) {
+ defineInt64ReuseInput(new (alloc()) LNegI64(useInt64RegisterAtStart(input)),
+ ins, 0);
+}
+
+void LIRGenerator::visitAbs(MAbs* ins) {
+ defineReuseInput(allocateAbs(ins, useRegisterAtStart(ins->input())), ins, 0);
+}
+
+void LIRGeneratorX86Shared::lowerMulI(MMul* mul, MDefinition* lhs,
+ MDefinition* rhs) {
+ // Note: If we need a negative zero check, lhs is used twice.
+ LAllocation lhsCopy = mul->canBeNegativeZero() ? use(lhs) : LAllocation();
+ LMulI* lir = new (alloc())
+ LMulI(useRegisterAtStart(lhs),
+ willHaveDifferentLIRNodes(lhs, rhs) ? useOrConstant(rhs)
+ : useOrConstantAtStart(rhs),
+ lhsCopy);
+ if (mul->fallible()) {
+ assignSnapshot(lir, mul->bailoutKind());
+ }
+ defineReuseInput(lir, mul, 0);
+}
+
+void LIRGeneratorX86Shared::lowerDivI(MDiv* div) {
+ if (div->isUnsigned()) {
+ lowerUDiv(div);
+ return;
+ }
+
+ // Division instructions are slow. Division by constant denominators can be
+ // rewritten to use other instructions.
+ if (div->rhs()->isConstant()) {
+ int32_t rhs = div->rhs()->toConstant()->toInt32();
+
+ // Division by powers of two can be done by shifting, and division by
+ // other numbers can be done by a reciprocal multiplication technique.
+ int32_t shift = FloorLog2(Abs(rhs));
+ if (rhs != 0 && uint32_t(1) << shift == Abs(rhs)) {
+ LAllocation lhs = useRegisterAtStart(div->lhs());
+ LDivPowTwoI* lir;
+ // When truncated with maybe a non-zero remainder, we have to round the
+ // result toward 0. This requires an extra register to round up/down
+ // whether the left-hand-side is signed.
+ bool needRoundNeg = div->canBeNegativeDividend() && div->isTruncated();
+ if (!needRoundNeg) {
+ // Numerator is unsigned, so does not need adjusting.
+ lir = new (alloc()) LDivPowTwoI(lhs, lhs, shift, rhs < 0);
+ } else {
+ // Numerator might be signed, and needs adjusting, and an extra lhs copy
+ // is needed to round the result of the integer division towards zero.
+ lir = new (alloc())
+ LDivPowTwoI(lhs, useRegister(div->lhs()), shift, rhs < 0);
+ }
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ defineReuseInput(lir, div, 0);
+ return;
+ }
+ if (rhs != 0) {
+ LDivOrModConstantI* lir;
+ lir = new (alloc())
+ LDivOrModConstantI(useRegister(div->lhs()), rhs, tempFixed(eax));
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ defineFixed(lir, div, LAllocation(AnyRegister(edx)));
+ return;
+ }
+ }
+
+ LDivI* lir = new (alloc())
+ LDivI(useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(edx));
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ defineFixed(lir, div, LAllocation(AnyRegister(eax)));
+}
+
+void LIRGeneratorX86Shared::lowerModI(MMod* mod) {
+ if (mod->isUnsigned()) {
+ lowerUMod(mod);
+ return;
+ }
+
+ if (mod->rhs()->isConstant()) {
+ int32_t rhs = mod->rhs()->toConstant()->toInt32();
+ int32_t shift = FloorLog2(Abs(rhs));
+ if (rhs != 0 && uint32_t(1) << shift == Abs(rhs)) {
+ LModPowTwoI* lir =
+ new (alloc()) LModPowTwoI(useRegisterAtStart(mod->lhs()), shift);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ defineReuseInput(lir, mod, 0);
+ return;
+ }
+ if (rhs != 0) {
+ LDivOrModConstantI* lir;
+ lir = new (alloc())
+ LDivOrModConstantI(useRegister(mod->lhs()), rhs, tempFixed(edx));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ defineFixed(lir, mod, LAllocation(AnyRegister(eax)));
+ return;
+ }
+ }
+
+ LModI* lir = new (alloc())
+ LModI(useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(eax));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ defineFixed(lir, mod, LAllocation(AnyRegister(edx)));
+}
+
+void LIRGenerator::visitWasmNeg(MWasmNeg* ins) {
+ switch (ins->type()) {
+ case MIRType::Int32:
+ defineReuseInput(new (alloc()) LNegI(useRegisterAtStart(ins->input())),
+ ins, 0);
+ break;
+ case MIRType::Float32:
+ defineReuseInput(new (alloc()) LNegF(useRegisterAtStart(ins->input())),
+ ins, 0);
+ break;
+ case MIRType::Double:
+ defineReuseInput(new (alloc()) LNegD(useRegisterAtStart(ins->input())),
+ ins, 0);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+void LIRGeneratorX86Shared::lowerWasmSelectI(MWasmSelect* select) {
+ auto* lir = new (alloc())
+ LWasmSelect(useRegisterAtStart(select->trueExpr()),
+ useAny(select->falseExpr()), useRegister(select->condExpr()));
+ defineReuseInput(lir, select, LWasmSelect::TrueExprIndex);
+}
+
+void LIRGeneratorX86Shared::lowerWasmSelectI64(MWasmSelect* select) {
+ auto* lir = new (alloc()) LWasmSelectI64(
+ useInt64RegisterAtStart(select->trueExpr()),
+ useInt64(select->falseExpr()), useRegister(select->condExpr()));
+ defineInt64ReuseInput(lir, select, LWasmSelectI64::TrueExprIndex);
+}
+
+void LIRGenerator::visitAsmJSLoadHeap(MAsmJSLoadHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT_IF(ins->needsBoundsCheck(),
+ boundsCheckLimit->type() == MIRType::Int32);
+
+ // For simplicity, require a register if we're going to emit a bounds-check
+ // branch, so that we don't have special cases for constants. This should
+ // only happen in rare constant-folding cases since asm.js sets the minimum
+ // heap size based when accessed via constant.
+ LAllocation baseAlloc = ins->needsBoundsCheck()
+ ? useRegisterAtStart(base)
+ : useRegisterOrZeroAtStart(base);
+
+ LAllocation limitAlloc = ins->needsBoundsCheck()
+ ? useRegisterAtStart(boundsCheckLimit)
+ : LAllocation();
+ LAllocation memoryBaseAlloc = ins->hasMemoryBase()
+ ? useRegisterAtStart(ins->memoryBase())
+ : LAllocation();
+
+ auto* lir =
+ new (alloc()) LAsmJSLoadHeap(baseAlloc, limitAlloc, memoryBaseAlloc);
+ define(lir, ins);
+}
+
+void LIRGenerator::visitAsmJSStoreHeap(MAsmJSStoreHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* boundsCheckLimit = ins->boundsCheckLimit();
+ MOZ_ASSERT_IF(ins->needsBoundsCheck(),
+ boundsCheckLimit->type() == MIRType::Int32);
+
+ // For simplicity, require a register if we're going to emit a bounds-check
+ // branch, so that we don't have special cases for constants. This should
+ // only happen in rare constant-folding cases since asm.js sets the minimum
+ // heap size based when accessed via constant.
+ LAllocation baseAlloc = ins->needsBoundsCheck()
+ ? useRegisterAtStart(base)
+ : useRegisterOrZeroAtStart(base);
+
+ LAllocation limitAlloc = ins->needsBoundsCheck()
+ ? useRegisterAtStart(boundsCheckLimit)
+ : LAllocation();
+ LAllocation memoryBaseAlloc = ins->hasMemoryBase()
+ ? useRegisterAtStart(ins->memoryBase())
+ : LAllocation();
+
+ LAsmJSStoreHeap* lir = nullptr;
+ switch (ins->access().type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+#ifdef JS_CODEGEN_X86
+ // See comment for LIRGeneratorX86::useByteOpRegister.
+ lir = new (alloc()) LAsmJSStoreHeap(
+ baseAlloc, useFixed(ins->value(), eax), limitAlloc, memoryBaseAlloc);
+ break;
+#endif
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ case Scalar::Float64:
+ // For now, don't allow constant values. The immediate operand affects
+ // instruction layout which affects patching.
+ lir = new (alloc())
+ LAsmJSStoreHeap(baseAlloc, useRegisterAtStart(ins->value()),
+ limitAlloc, memoryBaseAlloc);
+ break;
+ case Scalar::Int64:
+ case Scalar::Simd128:
+ MOZ_CRASH("NYI");
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+ add(lir, ins);
+}
+
+void LIRGeneratorX86Shared::lowerUDiv(MDiv* div) {
+ if (div->rhs()->isConstant()) {
+ uint32_t rhs = div->rhs()->toConstant()->toInt32();
+ int32_t shift = FloorLog2(rhs);
+
+ LAllocation lhs = useRegisterAtStart(div->lhs());
+ if (rhs != 0 && uint32_t(1) << shift == rhs) {
+ LDivPowTwoI* lir = new (alloc()) LDivPowTwoI(lhs, lhs, shift, false);
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ defineReuseInput(lir, div, 0);
+ } else {
+ LUDivOrModConstant* lir = new (alloc())
+ LUDivOrModConstant(useRegister(div->lhs()), rhs, tempFixed(eax));
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ defineFixed(lir, div, LAllocation(AnyRegister(edx)));
+ }
+ return;
+ }
+
+ LUDivOrMod* lir = new (alloc()) LUDivOrMod(
+ useRegister(div->lhs()), useRegister(div->rhs()), tempFixed(edx));
+ if (div->fallible()) {
+ assignSnapshot(lir, div->bailoutKind());
+ }
+ defineFixed(lir, div, LAllocation(AnyRegister(eax)));
+}
+
+void LIRGeneratorX86Shared::lowerUMod(MMod* mod) {
+ if (mod->rhs()->isConstant()) {
+ uint32_t rhs = mod->rhs()->toConstant()->toInt32();
+ int32_t shift = FloorLog2(rhs);
+
+ if (rhs != 0 && uint32_t(1) << shift == rhs) {
+ LModPowTwoI* lir =
+ new (alloc()) LModPowTwoI(useRegisterAtStart(mod->lhs()), shift);
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ defineReuseInput(lir, mod, 0);
+ } else {
+ LUDivOrModConstant* lir = new (alloc())
+ LUDivOrModConstant(useRegister(mod->lhs()), rhs, tempFixed(edx));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ defineFixed(lir, mod, LAllocation(AnyRegister(eax)));
+ }
+ return;
+ }
+
+ LUDivOrMod* lir = new (alloc()) LUDivOrMod(
+ useRegister(mod->lhs()), useRegister(mod->rhs()), tempFixed(eax));
+ if (mod->fallible()) {
+ assignSnapshot(lir, mod->bailoutKind());
+ }
+ defineFixed(lir, mod, LAllocation(AnyRegister(edx)));
+}
+
+void LIRGeneratorX86Shared::lowerUrshD(MUrsh* mir) {
+ MDefinition* lhs = mir->lhs();
+ MDefinition* rhs = mir->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Int32);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+ MOZ_ASSERT(mir->type() == MIRType::Double);
+
+#ifdef JS_CODEGEN_X64
+ static_assert(ecx == rcx);
+#endif
+
+ // Without BMI2, x86 can only shift by ecx.
+ LUse lhsUse = useRegisterAtStart(lhs);
+ LAllocation rhsAlloc;
+ if (rhs->isConstant()) {
+ rhsAlloc = useOrConstant(rhs);
+ } else if (Assembler::HasBMI2()) {
+ rhsAlloc = useRegister(rhs);
+ } else {
+ rhsAlloc = useFixed(rhs, ecx);
+ }
+
+ LUrshD* lir = new (alloc()) LUrshD(lhsUse, rhsAlloc, tempCopy(lhs, 0));
+ define(lir, mir);
+}
+
+void LIRGeneratorX86Shared::lowerPowOfTwoI(MPow* mir) {
+ int32_t base = mir->input()->toConstant()->toInt32();
+ MDefinition* power = mir->power();
+
+ // Shift operand should be in register ecx, unless BMI2 is available.
+ // x86 can't shift a non-ecx register.
+ LAllocation powerAlloc =
+ Assembler::HasBMI2() ? useRegister(power) : useFixed(power, ecx);
+ auto* lir = new (alloc()) LPowOfTwoI(powerAlloc, base);
+ assignSnapshot(lir, mir->bailoutKind());
+ define(lir, mir);
+}
+
+void LIRGeneratorX86Shared::lowerBigIntLsh(MBigIntLsh* ins) {
+ // Shift operand should be in register ecx, unless BMI2 is available.
+ // x86 can't shift a non-ecx register.
+ LDefinition shiftAlloc = Assembler::HasBMI2() ? temp() : tempFixed(ecx);
+ auto* lir =
+ new (alloc()) LBigIntLsh(useRegister(ins->lhs()), useRegister(ins->rhs()),
+ temp(), shiftAlloc, temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX86Shared::lowerBigIntRsh(MBigIntRsh* ins) {
+ // Shift operand should be in register ecx, unless BMI2 is available.
+ // x86 can't shift a non-ecx register.
+ LDefinition shiftAlloc = Assembler::HasBMI2() ? temp() : tempFixed(ecx);
+ auto* lir =
+ new (alloc()) LBigIntRsh(useRegister(ins->lhs()), useRegister(ins->rhs()),
+ temp(), shiftAlloc, temp());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX86Shared::lowerWasmBuiltinTruncateToInt32(
+ MWasmBuiltinTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ LDefinition maybeTemp =
+ Assembler::HasSSE3() ? LDefinition::BogusTemp() : tempDouble();
+ if (opd->type() == MIRType::Double) {
+ define(new (alloc()) LWasmBuiltinTruncateDToInt32(
+ useRegister(opd), useFixed(ins->instance(), InstanceReg),
+ maybeTemp),
+ ins);
+ return;
+ }
+
+ define(
+ new (alloc()) LWasmBuiltinTruncateFToInt32(
+ useRegister(opd), useFixed(ins->instance(), InstanceReg), maybeTemp),
+ ins);
+}
+
+void LIRGeneratorX86Shared::lowerTruncateDToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double);
+
+ LDefinition maybeTemp =
+ Assembler::HasSSE3() ? LDefinition::BogusTemp() : tempDouble();
+ define(new (alloc()) LTruncateDToInt32(useRegister(opd), maybeTemp), ins);
+}
+
+void LIRGeneratorX86Shared::lowerTruncateFToInt32(MTruncateToInt32* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Float32);
+
+ LDefinition maybeTemp =
+ Assembler::HasSSE3() ? LDefinition::BogusTemp() : tempFloat32();
+ define(new (alloc()) LTruncateFToInt32(useRegister(opd), maybeTemp), ins);
+}
+
+void LIRGeneratorX86Shared::lowerCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins, bool useI386ByteRegisters) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ // If the target is a floating register then we need a temp at the
+ // lower level; that temp must be eax.
+ //
+ // Otherwise the target (if used) is an integer register, which
+ // must be eax. If the target is not used the machine code will
+ // still clobber eax, so just pretend it's used.
+ //
+ // oldval must be in a register.
+ //
+ // newval must be in a register. If the source is a byte array
+ // then newval must be a register that has a byte size: on x86
+ // this must be ebx, ecx, or edx (eax is taken for the output).
+ //
+ // Bug #1077036 describes some further optimization opportunities.
+
+ bool fixedOutput = false;
+ LDefinition tempDef = LDefinition::BogusTemp();
+ LAllocation newval;
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ tempDef = tempFixed(eax);
+ newval = useRegister(ins->newval());
+ } else {
+ fixedOutput = true;
+ if (useI386ByteRegisters && ins->isByteArray()) {
+ newval = useFixed(ins->newval(), ebx);
+ } else {
+ newval = useRegister(ins->newval());
+ }
+ }
+
+ const LAllocation oldval = useRegister(ins->oldval());
+
+ LCompareExchangeTypedArrayElement* lir =
+ new (alloc()) LCompareExchangeTypedArrayElement(elements, index, oldval,
+ newval, tempDef);
+
+ if (fixedOutput) {
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+ } else {
+ define(lir, ins);
+ }
+}
+
+void LIRGeneratorX86Shared::lowerAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins, bool useI386ByteRegisters) {
+ MOZ_ASSERT(ins->arrayType() <= Scalar::Uint32);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ const LAllocation value = useRegister(ins->value());
+
+ // The underlying instruction is XCHG, which can operate on any
+ // register.
+ //
+ // If the target is a floating register (for Uint32) then we need
+ // a temp into which to exchange.
+ //
+ // If the source is a byte array then we need a register that has
+ // a byte size; in this case -- on x86 only -- pin the output to
+ // an appropriate register and use that as a temp in the back-end.
+
+ LDefinition tempDef = LDefinition::BogusTemp();
+ if (ins->arrayType() == Scalar::Uint32) {
+ MOZ_ASSERT(ins->type() == MIRType::Double);
+ tempDef = temp();
+ }
+
+ LAtomicExchangeTypedArrayElement* lir = new (alloc())
+ LAtomicExchangeTypedArrayElement(elements, index, value, tempDef);
+
+ if (useI386ByteRegisters && ins->isByteArray()) {
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+ } else {
+ define(lir, ins);
+ }
+}
+
+void LIRGeneratorX86Shared::lowerAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins, bool useI386ByteRegisters) {
+ MOZ_ASSERT(ins->arrayType() != Scalar::Uint8Clamped);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float32);
+ MOZ_ASSERT(ins->arrayType() != Scalar::Float64);
+
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We'll emit a single instruction: LOCK ADD, LOCK SUB, LOCK AND,
+ // LOCK OR, or LOCK XOR. We can do this even for the Uint32 case.
+
+ if (ins->isForEffect()) {
+ LAllocation value;
+ if (useI386ByteRegisters && ins->isByteArray() &&
+ !ins->value()->isConstant()) {
+ value = useFixed(ins->value(), ebx);
+ } else {
+ value = useRegisterOrConstant(ins->value());
+ }
+
+ LAtomicTypedArrayElementBinopForEffect* lir = new (alloc())
+ LAtomicTypedArrayElementBinopForEffect(elements, index, value);
+
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+ //
+ // For ADD and SUB we'll use XADD:
+ //
+ // movl src, output
+ // lock xaddl output, mem
+ //
+ // For the 8-bit variants XADD needs a byte register for the output.
+ //
+ // For AND/OR/XOR we need to use a CMPXCHG loop:
+ //
+ // movl *mem, eax
+ // L: mov eax, temp
+ // andl src, temp
+ // lock cmpxchg temp, mem ; reads eax also
+ // jnz L
+ // ; result in eax
+ //
+ // Note the placement of L, cmpxchg will update eax with *mem if
+ // *mem does not have the expected value, so reloading it at the
+ // top of the loop would be redundant.
+ //
+ // If the array is not a uint32 array then:
+ // - eax should be the output (one result of the cmpxchg)
+ // - there is a temp, which must have a byte register if
+ // the array has 1-byte elements elements
+ //
+ // If the array is a uint32 array then:
+ // - eax is the first temp
+ // - we also need a second temp
+ //
+ // There are optimization opportunities:
+ // - better register allocation in the x86 8-bit case, Bug #1077036.
+
+ bool bitOp = !(ins->operation() == AtomicFetchAddOp ||
+ ins->operation() == AtomicFetchSubOp);
+ bool fixedOutput = true;
+ bool reuseInput = false;
+ LDefinition tempDef1 = LDefinition::BogusTemp();
+ LDefinition tempDef2 = LDefinition::BogusTemp();
+ LAllocation value;
+
+ if (ins->arrayType() == Scalar::Uint32 && IsFloatingPointType(ins->type())) {
+ value = useRegisterOrConstant(ins->value());
+ fixedOutput = false;
+ if (bitOp) {
+ tempDef1 = tempFixed(eax);
+ tempDef2 = temp();
+ } else {
+ tempDef1 = temp();
+ }
+ } else if (useI386ByteRegisters && ins->isByteArray()) {
+ if (ins->value()->isConstant()) {
+ value = useRegisterOrConstant(ins->value());
+ } else {
+ value = useFixed(ins->value(), ebx);
+ }
+ if (bitOp) {
+ tempDef1 = tempFixed(ecx);
+ }
+ } else if (bitOp) {
+ value = useRegisterOrConstant(ins->value());
+ tempDef1 = temp();
+ } else if (ins->value()->isConstant()) {
+ fixedOutput = false;
+ value = useRegisterOrConstant(ins->value());
+ } else {
+ fixedOutput = false;
+ reuseInput = true;
+ value = useRegisterAtStart(ins->value());
+ }
+
+ LAtomicTypedArrayElementBinop* lir = new (alloc())
+ LAtomicTypedArrayElementBinop(elements, index, value, tempDef1, tempDef2);
+
+ if (fixedOutput) {
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+ } else if (reuseInput) {
+ defineReuseInput(lir, ins, LAtomicTypedArrayElementBinop::valueOp);
+ } else {
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitCopySign(MCopySign* ins) {
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(IsFloatingPointType(lhs->type()));
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(lhs->type() == ins->type());
+
+ LInstructionHelper<1, 2, 2>* lir;
+ if (lhs->type() == MIRType::Double) {
+ lir = new (alloc()) LCopySignD();
+ } else {
+ lir = new (alloc()) LCopySignF();
+ }
+
+ // As lowerForFPU, but we want rhs to be in a FP register too.
+ lir->setOperand(0, useRegisterAtStart(lhs));
+ if (!Assembler::HasAVX()) {
+ lir->setOperand(1, willHaveDifferentLIRNodes(lhs, rhs)
+ ? useRegister(rhs)
+ : useRegisterAtStart(rhs));
+ defineReuseInput(lir, ins, 0);
+ } else {
+ lir->setOperand(1, useRegisterAtStart(rhs));
+ define(lir, ins);
+ }
+}
+
+// These lowerings are really x86-shared but some Masm APIs are not yet
+// available on x86.
+
+// Ternary and binary operators require the dest register to be the same as
+// their first input register, leading to a pattern of useRegisterAtStart +
+// defineReuseInput.
+
+void LIRGenerator::visitWasmTernarySimd128(MWasmTernarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->v0()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->v1()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->v2()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::V128Bitselect: {
+ // Enforcing lhs == output avoids one setup move. We would like to also
+ // enforce merging the control with the temp (with
+ // usRegisterAtStart(control) and tempCopy()), but the register allocator
+ // ignores those constraints at present.
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegisterAtStart(ins->v0()), useRegister(ins->v1()),
+ useRegister(ins->v2()), tempSimd128());
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V0);
+ break;
+ }
+ case wasm::SimdOp::F32x4RelaxedFma:
+ case wasm::SimdOp::F32x4RelaxedFnma:
+ case wasm::SimdOp::F64x2RelaxedFma:
+ case wasm::SimdOp::F64x2RelaxedFnma: {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()), useRegister(ins->v1()),
+ useRegisterAtStart(ins->v2()));
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V2);
+ break;
+ }
+ case wasm::SimdOp::I32x4DotI8x16I7x16AddS: {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()), useRegister(ins->v1()),
+ useRegisterAtStart(ins->v2()));
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V2);
+ break;
+ }
+ case wasm::SimdOp::I8x16RelaxedLaneSelect:
+ case wasm::SimdOp::I16x8RelaxedLaneSelect:
+ case wasm::SimdOp::I32x4RelaxedLaneSelect:
+ case wasm::SimdOp::I64x2RelaxedLaneSelect: {
+ if (Assembler::HasAVX()) {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegisterAtStart(ins->v0()),
+ useRegisterAtStart(ins->v1()), useRegisterAtStart(ins->v2()));
+ define(lir, ins);
+ } else {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()),
+ useRegisterAtStart(ins->v1()), useFixed(ins->v2(), vmm0));
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V1);
+ }
+ break;
+ }
+ case wasm::SimdOp::F32x4RelaxedDotBF16x8AddF32x4: {
+ auto* lir = new (alloc()) LWasmTernarySimd128(
+ ins->simdOp(), useRegister(ins->v0()), useRegister(ins->v1()),
+ useRegisterAtStart(ins->v2()), tempSimd128());
+ defineReuseInput(lir, ins, LWasmTernarySimd128::V2);
+ break;
+ }
+ default:
+ MOZ_CRASH("NYI");
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmBinarySimd128(MWasmBinarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+ wasm::SimdOp op = ins->simdOp();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Simd128);
+ MOZ_ASSERT(rhs->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ // Note MWasmBinarySimd128::foldsTo has already specialized operations that
+ // have a constant operand, so this takes care of more general cases of
+ // reordering, see ReorderCommutative.
+ if (ins->isCommutative()) {
+ ReorderCommutative(&lhs, &rhs, ins);
+ }
+
+ // Swap operands and change operation if necessary, these are all x86/x64
+ // dependent transformations. Except where noted, this is about avoiding
+ // unnecessary moves and fixups in the code generator macros.
+ bool swap = false;
+ switch (op) {
+ case wasm::SimdOp::V128AndNot: {
+ // Code generation requires the operands to be reversed.
+ swap = true;
+ break;
+ }
+ case wasm::SimdOp::I8x16LtS: {
+ swap = true;
+ op = wasm::SimdOp::I8x16GtS;
+ break;
+ }
+ case wasm::SimdOp::I8x16GeS: {
+ swap = true;
+ op = wasm::SimdOp::I8x16LeS;
+ break;
+ }
+ case wasm::SimdOp::I16x8LtS: {
+ swap = true;
+ op = wasm::SimdOp::I16x8GtS;
+ break;
+ }
+ case wasm::SimdOp::I16x8GeS: {
+ swap = true;
+ op = wasm::SimdOp::I16x8LeS;
+ break;
+ }
+ case wasm::SimdOp::I32x4LtS: {
+ swap = true;
+ op = wasm::SimdOp::I32x4GtS;
+ break;
+ }
+ case wasm::SimdOp::I32x4GeS: {
+ swap = true;
+ op = wasm::SimdOp::I32x4LeS;
+ break;
+ }
+ case wasm::SimdOp::F32x4Gt: {
+ swap = true;
+ op = wasm::SimdOp::F32x4Lt;
+ break;
+ }
+ case wasm::SimdOp::F32x4Ge: {
+ swap = true;
+ op = wasm::SimdOp::F32x4Le;
+ break;
+ }
+ case wasm::SimdOp::F64x2Gt: {
+ swap = true;
+ op = wasm::SimdOp::F64x2Lt;
+ break;
+ }
+ case wasm::SimdOp::F64x2Ge: {
+ swap = true;
+ op = wasm::SimdOp::F64x2Le;
+ break;
+ }
+ case wasm::SimdOp::F32x4PMin:
+ case wasm::SimdOp::F32x4PMax:
+ case wasm::SimdOp::F64x2PMin:
+ case wasm::SimdOp::F64x2PMax: {
+ // Code generation requires the operations to be reversed (the rhs is the
+ // output register).
+ swap = true;
+ break;
+ }
+ default:
+ break;
+ }
+ if (swap) {
+ MDefinition* tmp = lhs;
+ lhs = rhs;
+ rhs = tmp;
+ }
+
+ // Allocate temp registers
+ LDefinition tempReg0 = LDefinition::BogusTemp();
+ LDefinition tempReg1 = LDefinition::BogusTemp();
+ switch (op) {
+ case wasm::SimdOp::I64x2Mul:
+ tempReg0 = tempSimd128();
+ break;
+ case wasm::SimdOp::F32x4Min:
+ case wasm::SimdOp::F32x4Max:
+ case wasm::SimdOp::F64x2Min:
+ case wasm::SimdOp::F64x2Max:
+ tempReg0 = tempSimd128();
+ tempReg1 = tempSimd128();
+ break;
+ case wasm::SimdOp::I64x2LtS:
+ case wasm::SimdOp::I64x2GtS:
+ case wasm::SimdOp::I64x2LeS:
+ case wasm::SimdOp::I64x2GeS:
+ // The compareForOrderingInt64x2AVX implementation does not require
+ // temps but needs SSE4.2 support. Checking if both AVX and SSE4.2
+ // are enabled.
+ if (!(Assembler::HasAVX() && Assembler::HasSSE42())) {
+ tempReg0 = tempSimd128();
+ tempReg1 = tempSimd128();
+ }
+ break;
+ default:
+ break;
+ }
+
+ // For binary ops, without AVX support, the Masm API always is usually
+ // (rhs, lhsDest) and requires AtStart+ReuseInput for the lhs.
+ //
+ // For a few ops, the API is actually (rhsDest, lhs) and the rules are the
+ // same but the reversed. We swapped operands above; they will be swapped
+ // again in the code generator to emit the right code.
+ //
+ // If AVX support is enabled, some binary ops can use output as destination,
+ // useRegisterAtStart is applied for both operands and no need for ReuseInput.
+
+ switch (op) {
+ case wasm::SimdOp::I8x16AvgrU:
+ case wasm::SimdOp::I16x8AvgrU:
+ case wasm::SimdOp::I8x16Add:
+ case wasm::SimdOp::I8x16AddSatS:
+ case wasm::SimdOp::I8x16AddSatU:
+ case wasm::SimdOp::I8x16Sub:
+ case wasm::SimdOp::I8x16SubSatS:
+ case wasm::SimdOp::I8x16SubSatU:
+ case wasm::SimdOp::I16x8Mul:
+ case wasm::SimdOp::I16x8MinS:
+ case wasm::SimdOp::I16x8MinU:
+ case wasm::SimdOp::I16x8MaxS:
+ case wasm::SimdOp::I16x8MaxU:
+ case wasm::SimdOp::I32x4Add:
+ case wasm::SimdOp::I32x4Sub:
+ case wasm::SimdOp::I32x4Mul:
+ case wasm::SimdOp::I32x4MinS:
+ case wasm::SimdOp::I32x4MinU:
+ case wasm::SimdOp::I32x4MaxS:
+ case wasm::SimdOp::I32x4MaxU:
+ case wasm::SimdOp::I64x2Add:
+ case wasm::SimdOp::I64x2Sub:
+ case wasm::SimdOp::I64x2Mul:
+ case wasm::SimdOp::F32x4Add:
+ case wasm::SimdOp::F32x4Sub:
+ case wasm::SimdOp::F32x4Mul:
+ case wasm::SimdOp::F32x4Div:
+ case wasm::SimdOp::F64x2Add:
+ case wasm::SimdOp::F64x2Sub:
+ case wasm::SimdOp::F64x2Mul:
+ case wasm::SimdOp::F64x2Div:
+ case wasm::SimdOp::F32x4Eq:
+ case wasm::SimdOp::F32x4Ne:
+ case wasm::SimdOp::F32x4Lt:
+ case wasm::SimdOp::F32x4Le:
+ case wasm::SimdOp::F64x2Eq:
+ case wasm::SimdOp::F64x2Ne:
+ case wasm::SimdOp::F64x2Lt:
+ case wasm::SimdOp::F64x2Le:
+ case wasm::SimdOp::F32x4PMin:
+ case wasm::SimdOp::F32x4PMax:
+ case wasm::SimdOp::F64x2PMin:
+ case wasm::SimdOp::F64x2PMax:
+ case wasm::SimdOp::I8x16Swizzle:
+ case wasm::SimdOp::I8x16RelaxedSwizzle:
+ case wasm::SimdOp::I8x16Eq:
+ case wasm::SimdOp::I8x16Ne:
+ case wasm::SimdOp::I8x16GtS:
+ case wasm::SimdOp::I8x16LeS:
+ case wasm::SimdOp::I8x16LtU:
+ case wasm::SimdOp::I8x16GtU:
+ case wasm::SimdOp::I8x16LeU:
+ case wasm::SimdOp::I8x16GeU:
+ case wasm::SimdOp::I16x8Eq:
+ case wasm::SimdOp::I16x8Ne:
+ case wasm::SimdOp::I16x8GtS:
+ case wasm::SimdOp::I16x8LeS:
+ case wasm::SimdOp::I16x8LtU:
+ case wasm::SimdOp::I16x8GtU:
+ case wasm::SimdOp::I16x8LeU:
+ case wasm::SimdOp::I16x8GeU:
+ case wasm::SimdOp::I32x4Eq:
+ case wasm::SimdOp::I32x4Ne:
+ case wasm::SimdOp::I32x4GtS:
+ case wasm::SimdOp::I32x4LeS:
+ case wasm::SimdOp::I32x4LtU:
+ case wasm::SimdOp::I32x4GtU:
+ case wasm::SimdOp::I32x4LeU:
+ case wasm::SimdOp::I32x4GeU:
+ case wasm::SimdOp::I64x2Eq:
+ case wasm::SimdOp::I64x2Ne:
+ case wasm::SimdOp::I64x2LtS:
+ case wasm::SimdOp::I64x2GtS:
+ case wasm::SimdOp::I64x2LeS:
+ case wasm::SimdOp::I64x2GeS:
+ case wasm::SimdOp::V128And:
+ case wasm::SimdOp::V128Or:
+ case wasm::SimdOp::V128Xor:
+ case wasm::SimdOp::V128AndNot:
+ case wasm::SimdOp::F32x4Min:
+ case wasm::SimdOp::F32x4Max:
+ case wasm::SimdOp::F64x2Min:
+ case wasm::SimdOp::F64x2Max:
+ case wasm::SimdOp::I8x16NarrowI16x8S:
+ case wasm::SimdOp::I8x16NarrowI16x8U:
+ case wasm::SimdOp::I16x8NarrowI32x4S:
+ case wasm::SimdOp::I16x8NarrowI32x4U:
+ case wasm::SimdOp::I32x4DotI16x8S:
+ case wasm::SimdOp::I16x8ExtmulLowI8x16S:
+ case wasm::SimdOp::I16x8ExtmulHighI8x16S:
+ case wasm::SimdOp::I16x8ExtmulLowI8x16U:
+ case wasm::SimdOp::I16x8ExtmulHighI8x16U:
+ case wasm::SimdOp::I32x4ExtmulLowI16x8S:
+ case wasm::SimdOp::I32x4ExtmulHighI16x8S:
+ case wasm::SimdOp::I32x4ExtmulLowI16x8U:
+ case wasm::SimdOp::I32x4ExtmulHighI16x8U:
+ case wasm::SimdOp::I64x2ExtmulLowI32x4S:
+ case wasm::SimdOp::I64x2ExtmulHighI32x4S:
+ case wasm::SimdOp::I64x2ExtmulLowI32x4U:
+ case wasm::SimdOp::I64x2ExtmulHighI32x4U:
+ case wasm::SimdOp::I16x8Q15MulrSatS:
+ case wasm::SimdOp::F32x4RelaxedMin:
+ case wasm::SimdOp::F32x4RelaxedMax:
+ case wasm::SimdOp::F64x2RelaxedMin:
+ case wasm::SimdOp::F64x2RelaxedMax:
+ case wasm::SimdOp::I16x8RelaxedQ15MulrS:
+ case wasm::SimdOp::I16x8DotI8x16I7x16S:
+ case wasm::SimdOp::MozPMADDUBSW:
+ if (isThreeOpAllowed()) {
+ auto* lir = new (alloc())
+ LWasmBinarySimd128(op, useRegisterAtStart(lhs),
+ useRegisterAtStart(rhs), tempReg0, tempReg1);
+ define(lir, ins);
+ break;
+ }
+ [[fallthrough]];
+ default: {
+ LAllocation lhsDestAlloc = useRegisterAtStart(lhs);
+ LAllocation rhsAlloc = willHaveDifferentLIRNodes(lhs, rhs)
+ ? useRegister(rhs)
+ : useRegisterAtStart(rhs);
+ auto* lir = new (alloc())
+ LWasmBinarySimd128(op, lhsDestAlloc, rhsAlloc, tempReg0, tempReg1);
+ defineReuseInput(lir, ins, LWasmBinarySimd128::LhsDest);
+ break;
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+#ifdef ENABLE_WASM_SIMD
+bool MWasmTernarySimd128::specializeBitselectConstantMaskAsShuffle(
+ int8_t shuffle[16]) {
+ if (simdOp() != wasm::SimdOp::V128Bitselect) {
+ return false;
+ }
+
+ // Optimization when control vector is a mask with all 0 or all 1 per lane.
+ // On x86, there is no bitselect, blend operations will be a win,
+ // e.g. via PBLENDVB or PBLENDW.
+ SimdConstant constant = static_cast<MWasmFloatConstant*>(v2())->toSimd128();
+ const SimdConstant::I8x16& bytes = constant.asInt8x16();
+ for (int8_t i = 0; i < 16; i++) {
+ if (bytes[i] == -1) {
+ shuffle[i] = i + 16;
+ } else if (bytes[i] == 0) {
+ shuffle[i] = i;
+ } else {
+ return false;
+ }
+ }
+ return true;
+}
+bool MWasmTernarySimd128::canRelaxBitselect() {
+ wasm::SimdOp simdOp;
+ if (v2()->isWasmBinarySimd128()) {
+ simdOp = v2()->toWasmBinarySimd128()->simdOp();
+ } else if (v2()->isWasmBinarySimd128WithConstant()) {
+ simdOp = v2()->toWasmBinarySimd128WithConstant()->simdOp();
+ } else {
+ return false;
+ }
+ switch (simdOp) {
+ case wasm::SimdOp::I8x16Eq:
+ case wasm::SimdOp::I8x16Ne:
+ case wasm::SimdOp::I8x16GtS:
+ case wasm::SimdOp::I8x16GeS:
+ case wasm::SimdOp::I8x16LtS:
+ case wasm::SimdOp::I8x16LeS:
+ case wasm::SimdOp::I8x16GtU:
+ case wasm::SimdOp::I8x16GeU:
+ case wasm::SimdOp::I8x16LtU:
+ case wasm::SimdOp::I8x16LeU:
+ case wasm::SimdOp::I16x8Eq:
+ case wasm::SimdOp::I16x8Ne:
+ case wasm::SimdOp::I16x8GtS:
+ case wasm::SimdOp::I16x8GeS:
+ case wasm::SimdOp::I16x8LtS:
+ case wasm::SimdOp::I16x8LeS:
+ case wasm::SimdOp::I16x8GtU:
+ case wasm::SimdOp::I16x8GeU:
+ case wasm::SimdOp::I16x8LtU:
+ case wasm::SimdOp::I16x8LeU:
+ case wasm::SimdOp::I32x4Eq:
+ case wasm::SimdOp::I32x4Ne:
+ case wasm::SimdOp::I32x4GtS:
+ case wasm::SimdOp::I32x4GeS:
+ case wasm::SimdOp::I32x4LtS:
+ case wasm::SimdOp::I32x4LeS:
+ case wasm::SimdOp::I32x4GtU:
+ case wasm::SimdOp::I32x4GeU:
+ case wasm::SimdOp::I32x4LtU:
+ case wasm::SimdOp::I32x4LeU:
+ case wasm::SimdOp::I64x2Eq:
+ case wasm::SimdOp::I64x2Ne:
+ case wasm::SimdOp::I64x2GtS:
+ case wasm::SimdOp::I64x2GeS:
+ case wasm::SimdOp::I64x2LtS:
+ case wasm::SimdOp::I64x2LeS:
+ case wasm::SimdOp::F32x4Eq:
+ case wasm::SimdOp::F32x4Ne:
+ case wasm::SimdOp::F32x4Gt:
+ case wasm::SimdOp::F32x4Ge:
+ case wasm::SimdOp::F32x4Lt:
+ case wasm::SimdOp::F32x4Le:
+ case wasm::SimdOp::F64x2Eq:
+ case wasm::SimdOp::F64x2Ne:
+ case wasm::SimdOp::F64x2Gt:
+ case wasm::SimdOp::F64x2Ge:
+ case wasm::SimdOp::F64x2Lt:
+ case wasm::SimdOp::F64x2Le:
+ return true;
+ default:
+ break;
+ }
+ return false;
+}
+
+bool MWasmBinarySimd128::canPmaddubsw() {
+ MOZ_ASSERT(Assembler::HasSSE3());
+ return true;
+}
+#endif
+
+bool MWasmBinarySimd128::specializeForConstantRhs() {
+ // The order follows MacroAssembler.h, generally
+ switch (simdOp()) {
+ // Operations implemented by a single native instruction where it is
+ // plausible that the rhs (after commutation if available) could be a
+ // constant.
+ //
+ // Swizzle is not here because it was handled earlier in the pipeline.
+ //
+ // Integer compares >= and < are not here because they are not supported in
+ // the hardware.
+ //
+ // Floating compares are not here because our patching machinery can't
+ // handle them yet.
+ //
+ // Floating-point min and max (including pmin and pmax) are not here because
+ // they are not straightforward to implement.
+ case wasm::SimdOp::I8x16Add:
+ case wasm::SimdOp::I16x8Add:
+ case wasm::SimdOp::I32x4Add:
+ case wasm::SimdOp::I64x2Add:
+ case wasm::SimdOp::I8x16Sub:
+ case wasm::SimdOp::I16x8Sub:
+ case wasm::SimdOp::I32x4Sub:
+ case wasm::SimdOp::I64x2Sub:
+ case wasm::SimdOp::I16x8Mul:
+ case wasm::SimdOp::I32x4Mul:
+ case wasm::SimdOp::I8x16AddSatS:
+ case wasm::SimdOp::I8x16AddSatU:
+ case wasm::SimdOp::I16x8AddSatS:
+ case wasm::SimdOp::I16x8AddSatU:
+ case wasm::SimdOp::I8x16SubSatS:
+ case wasm::SimdOp::I8x16SubSatU:
+ case wasm::SimdOp::I16x8SubSatS:
+ case wasm::SimdOp::I16x8SubSatU:
+ case wasm::SimdOp::I8x16MinS:
+ case wasm::SimdOp::I8x16MinU:
+ case wasm::SimdOp::I16x8MinS:
+ case wasm::SimdOp::I16x8MinU:
+ case wasm::SimdOp::I32x4MinS:
+ case wasm::SimdOp::I32x4MinU:
+ case wasm::SimdOp::I8x16MaxS:
+ case wasm::SimdOp::I8x16MaxU:
+ case wasm::SimdOp::I16x8MaxS:
+ case wasm::SimdOp::I16x8MaxU:
+ case wasm::SimdOp::I32x4MaxS:
+ case wasm::SimdOp::I32x4MaxU:
+ case wasm::SimdOp::V128And:
+ case wasm::SimdOp::V128Or:
+ case wasm::SimdOp::V128Xor:
+ case wasm::SimdOp::I8x16Eq:
+ case wasm::SimdOp::I8x16Ne:
+ case wasm::SimdOp::I8x16GtS:
+ case wasm::SimdOp::I8x16LeS:
+ case wasm::SimdOp::I16x8Eq:
+ case wasm::SimdOp::I16x8Ne:
+ case wasm::SimdOp::I16x8GtS:
+ case wasm::SimdOp::I16x8LeS:
+ case wasm::SimdOp::I32x4Eq:
+ case wasm::SimdOp::I32x4Ne:
+ case wasm::SimdOp::I32x4GtS:
+ case wasm::SimdOp::I32x4LeS:
+ case wasm::SimdOp::I64x2Mul:
+ case wasm::SimdOp::F32x4Eq:
+ case wasm::SimdOp::F32x4Ne:
+ case wasm::SimdOp::F32x4Lt:
+ case wasm::SimdOp::F32x4Le:
+ case wasm::SimdOp::F64x2Eq:
+ case wasm::SimdOp::F64x2Ne:
+ case wasm::SimdOp::F64x2Lt:
+ case wasm::SimdOp::F64x2Le:
+ case wasm::SimdOp::I32x4DotI16x8S:
+ case wasm::SimdOp::F32x4Add:
+ case wasm::SimdOp::F64x2Add:
+ case wasm::SimdOp::F32x4Sub:
+ case wasm::SimdOp::F64x2Sub:
+ case wasm::SimdOp::F32x4Div:
+ case wasm::SimdOp::F64x2Div:
+ case wasm::SimdOp::F32x4Mul:
+ case wasm::SimdOp::F64x2Mul:
+ case wasm::SimdOp::I8x16NarrowI16x8S:
+ case wasm::SimdOp::I8x16NarrowI16x8U:
+ case wasm::SimdOp::I16x8NarrowI32x4S:
+ case wasm::SimdOp::I16x8NarrowI32x4U:
+ return true;
+ default:
+ return false;
+ }
+}
+
+void LIRGenerator::visitWasmBinarySimd128WithConstant(
+ MWasmBinarySimd128WithConstant* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* lhs = ins->lhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ // Allocate temp registers
+ LDefinition tempReg = LDefinition::BogusTemp();
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I64x2Mul:
+ tempReg = tempSimd128();
+ break;
+ default:
+ break;
+ }
+
+ if (isThreeOpAllowed()) {
+ // The non-destructive versions of instructions will be available
+ // when AVX is enabled.
+ LAllocation lhsAlloc = useRegisterAtStart(lhs);
+ auto* lir = new (alloc())
+ LWasmBinarySimd128WithConstant(lhsAlloc, ins->rhs(), tempReg);
+ define(lir, ins);
+ } else {
+ // Always beneficial to reuse the lhs register here, see discussion in
+ // visitWasmBinarySimd128() and also code in specializeForConstantRhs().
+ LAllocation lhsDestAlloc = useRegisterAtStart(lhs);
+ auto* lir = new (alloc())
+ LWasmBinarySimd128WithConstant(lhsDestAlloc, ins->rhs(), tempReg);
+ defineReuseInput(lir, ins, LWasmBinarySimd128WithConstant::LhsDest);
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmShiftSimd128(MWasmShiftSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MDefinition* lhs = ins->lhs();
+ MDefinition* rhs = ins->rhs();
+
+ MOZ_ASSERT(lhs->type() == MIRType::Simd128);
+ MOZ_ASSERT(rhs->type() == MIRType::Int32);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ if (rhs->isConstant()) {
+ int32_t shiftCountMask;
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Shl:
+ case wasm::SimdOp::I8x16ShrU:
+ case wasm::SimdOp::I8x16ShrS:
+ shiftCountMask = 7;
+ break;
+ case wasm::SimdOp::I16x8Shl:
+ case wasm::SimdOp::I16x8ShrU:
+ case wasm::SimdOp::I16x8ShrS:
+ shiftCountMask = 15;
+ break;
+ case wasm::SimdOp::I32x4Shl:
+ case wasm::SimdOp::I32x4ShrU:
+ case wasm::SimdOp::I32x4ShrS:
+ shiftCountMask = 31;
+ break;
+ case wasm::SimdOp::I64x2Shl:
+ case wasm::SimdOp::I64x2ShrU:
+ case wasm::SimdOp::I64x2ShrS:
+ shiftCountMask = 63;
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift operation");
+ }
+
+ int32_t shiftCount = rhs->toConstant()->toInt32() & shiftCountMask;
+ if (shiftCount == shiftCountMask) {
+ // Check if possible to apply sign replication optimization.
+ // For some ops the input shall be reused.
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16ShrS: {
+ auto* lir =
+ new (alloc()) LWasmSignReplicationSimd128(useRegister(lhs));
+ define(lir, ins);
+ return;
+ }
+ case wasm::SimdOp::I16x8ShrS:
+ case wasm::SimdOp::I32x4ShrS:
+ case wasm::SimdOp::I64x2ShrS: {
+ auto* lir = new (alloc())
+ LWasmSignReplicationSimd128(useRegisterAtStart(lhs));
+ if (isThreeOpAllowed()) {
+ define(lir, ins);
+ } else {
+ // For non-AVX, it is always beneficial to reuse the input.
+ defineReuseInput(lir, ins, LWasmConstantShiftSimd128::Src);
+ }
+ return;
+ }
+ default:
+ break;
+ }
+ }
+
+# ifdef DEBUG
+ js::wasm::ReportSimdAnalysis("shift -> constant shift");
+# endif
+ auto* lir = new (alloc())
+ LWasmConstantShiftSimd128(useRegisterAtStart(lhs), shiftCount);
+ if (isThreeOpAllowed()) {
+ define(lir, ins);
+ } else {
+ // For non-AVX, it is always beneficial to reuse the input.
+ defineReuseInput(lir, ins, LWasmConstantShiftSimd128::Src);
+ }
+ return;
+ }
+
+# ifdef DEBUG
+ js::wasm::ReportSimdAnalysis("shift -> variable shift");
+# endif
+
+ LDefinition tempReg = LDefinition::BogusTemp();
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Shl:
+ case wasm::SimdOp::I8x16ShrS:
+ case wasm::SimdOp::I8x16ShrU:
+ case wasm::SimdOp::I64x2ShrS:
+ tempReg = tempSimd128();
+ break;
+ default:
+ break;
+ }
+
+ // Reusing the input if possible is never detrimental.
+ LAllocation lhsDestAlloc = useRegisterAtStart(lhs);
+ LAllocation rhsAlloc = useRegisterAtStart(rhs);
+ auto* lir =
+ new (alloc()) LWasmVariableShiftSimd128(lhsDestAlloc, rhsAlloc, tempReg);
+ defineReuseInput(lir, ins, LWasmVariableShiftSimd128::LhsDest);
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmShuffleSimd128(MWasmShuffleSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->lhs()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->rhs()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ SimdShuffle s = ins->shuffle();
+ switch (s.opd) {
+ case SimdShuffle::Operand::LEFT:
+ case SimdShuffle::Operand::RIGHT: {
+ LAllocation src;
+ bool reuse = false;
+ switch (*s.permuteOp) {
+ case SimdPermuteOp::MOVE:
+ reuse = true;
+ break;
+ case SimdPermuteOp::BROADCAST_8x16:
+ case SimdPermuteOp::BROADCAST_16x8:
+ case SimdPermuteOp::PERMUTE_8x16:
+ case SimdPermuteOp::PERMUTE_16x8:
+ case SimdPermuteOp::PERMUTE_32x4:
+ case SimdPermuteOp::ROTATE_RIGHT_8x16:
+ case SimdPermuteOp::SHIFT_LEFT_8x16:
+ case SimdPermuteOp::SHIFT_RIGHT_8x16:
+ case SimdPermuteOp::REVERSE_16x8:
+ case SimdPermuteOp::REVERSE_32x4:
+ case SimdPermuteOp::REVERSE_64x2:
+ // No need to reuse registers when VEX instructions are enabled.
+ reuse = !Assembler::HasAVX();
+ break;
+ default:
+ MOZ_CRASH("Unexpected operator");
+ }
+ if (s.opd == SimdShuffle::Operand::LEFT) {
+ src = useRegisterAtStart(ins->lhs());
+ } else {
+ src = useRegisterAtStart(ins->rhs());
+ }
+ auto* lir =
+ new (alloc()) LWasmPermuteSimd128(src, *s.permuteOp, s.control);
+ if (reuse) {
+ defineReuseInput(lir, ins, LWasmPermuteSimd128::Src);
+ } else {
+ define(lir, ins);
+ }
+ break;
+ }
+ case SimdShuffle::Operand::BOTH:
+ case SimdShuffle::Operand::BOTH_SWAPPED: {
+ LDefinition temp = LDefinition::BogusTemp();
+ switch (*s.shuffleOp) {
+ case SimdShuffleOp::BLEND_8x16:
+ temp = Assembler::HasAVX() ? tempSimd128() : tempFixed(xmm0);
+ break;
+ default:
+ break;
+ }
+ if (isThreeOpAllowed()) {
+ LAllocation lhs;
+ LAllocation rhs;
+ if (s.opd == SimdShuffle::Operand::BOTH) {
+ lhs = useRegisterAtStart(ins->lhs());
+ rhs = useRegisterAtStart(ins->rhs());
+ } else {
+ lhs = useRegisterAtStart(ins->rhs());
+ rhs = useRegisterAtStart(ins->lhs());
+ }
+ auto* lir = new (alloc())
+ LWasmShuffleSimd128(lhs, rhs, temp, *s.shuffleOp, s.control);
+ define(lir, ins);
+ } else {
+ LAllocation lhs;
+ LAllocation rhs;
+ if (s.opd == SimdShuffle::Operand::BOTH) {
+ lhs = useRegisterAtStart(ins->lhs());
+ rhs = useRegister(ins->rhs());
+ } else {
+ lhs = useRegisterAtStart(ins->rhs());
+ rhs = useRegister(ins->lhs());
+ }
+ auto* lir = new (alloc())
+ LWasmShuffleSimd128(lhs, rhs, temp, *s.shuffleOp, s.control);
+ defineReuseInput(lir, ins, LWasmShuffleSimd128::LhsDest);
+ }
+ break;
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmReplaceLaneSimd128(MWasmReplaceLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->lhs()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ // If AVX support is disabled, the Masm API is (rhs, lhsDest) and requires
+ // AtStart+ReuseInput for the lhs. For type reasons, the rhs will never be
+ // the same as the lhs and is therefore a plain Use.
+ //
+ // If AVX support is enabled, useRegisterAtStart is preferred.
+
+ if (ins->rhs()->type() == MIRType::Int64) {
+ if (isThreeOpAllowed()) {
+ auto* lir = new (alloc()) LWasmReplaceInt64LaneSimd128(
+ useRegisterAtStart(ins->lhs()), useInt64RegisterAtStart(ins->rhs()));
+ define(lir, ins);
+ } else {
+ auto* lir = new (alloc()) LWasmReplaceInt64LaneSimd128(
+ useRegisterAtStart(ins->lhs()), useInt64Register(ins->rhs()));
+ defineReuseInput(lir, ins, LWasmReplaceInt64LaneSimd128::LhsDest);
+ }
+ } else {
+ if (isThreeOpAllowed()) {
+ auto* lir = new (alloc()) LWasmReplaceLaneSimd128(
+ useRegisterAtStart(ins->lhs()), useRegisterAtStart(ins->rhs()));
+ define(lir, ins);
+ } else {
+ auto* lir = new (alloc()) LWasmReplaceLaneSimd128(
+ useRegisterAtStart(ins->lhs()), useRegister(ins->rhs()));
+ defineReuseInput(lir, ins, LWasmReplaceLaneSimd128::LhsDest);
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmScalarToSimd128(MWasmScalarToSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ switch (ins->input()->type()) {
+ case MIRType::Int64: {
+ // 64-bit integer splats.
+ // Load-and-(sign|zero)extend.
+ auto* lir = new (alloc())
+ LWasmInt64ToSimd128(useInt64RegisterAtStart(ins->input()));
+ define(lir, ins);
+ break;
+ }
+ case MIRType::Float32:
+ case MIRType::Double: {
+ // Floating-point splats.
+ // Ideally we save a move on SSE systems by reusing the input register,
+ // but since the input and output register types differ, we can't.
+ auto* lir =
+ new (alloc()) LWasmScalarToSimd128(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+ break;
+ }
+ default: {
+ // 32-bit integer splats.
+ auto* lir =
+ new (alloc()) LWasmScalarToSimd128(useRegisterAtStart(ins->input()));
+ define(lir, ins);
+ break;
+ }
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmUnarySimd128(MWasmUnarySimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ MOZ_ASSERT(ins->input()->type() == MIRType::Simd128);
+ MOZ_ASSERT(ins->type() == MIRType::Simd128);
+
+ bool useAtStart = false;
+ bool reuseInput = false;
+ LDefinition tempReg = LDefinition::BogusTemp();
+ switch (ins->simdOp()) {
+ case wasm::SimdOp::I8x16Neg:
+ case wasm::SimdOp::I16x8Neg:
+ case wasm::SimdOp::I32x4Neg:
+ case wasm::SimdOp::I64x2Neg:
+ case wasm::SimdOp::I16x8ExtaddPairwiseI8x16S:
+ // Prefer src != dest to avoid an unconditional src->temp move.
+ MOZ_ASSERT(!reuseInput);
+ // If AVX is enabled, we prefer useRegisterAtStart.
+ useAtStart = isThreeOpAllowed();
+ break;
+ case wasm::SimdOp::F32x4Neg:
+ case wasm::SimdOp::F64x2Neg:
+ case wasm::SimdOp::F32x4Abs:
+ case wasm::SimdOp::F64x2Abs:
+ case wasm::SimdOp::V128Not:
+ case wasm::SimdOp::F32x4Sqrt:
+ case wasm::SimdOp::F64x2Sqrt:
+ case wasm::SimdOp::I8x16Abs:
+ case wasm::SimdOp::I16x8Abs:
+ case wasm::SimdOp::I32x4Abs:
+ case wasm::SimdOp::I64x2Abs:
+ case wasm::SimdOp::I32x4TruncSatF32x4S:
+ case wasm::SimdOp::F32x4ConvertI32x4U:
+ case wasm::SimdOp::I16x8ExtaddPairwiseI8x16U:
+ case wasm::SimdOp::I32x4ExtaddPairwiseI16x8S:
+ case wasm::SimdOp::I32x4ExtaddPairwiseI16x8U:
+ case wasm::SimdOp::I32x4RelaxedTruncSSatF32x4:
+ case wasm::SimdOp::I32x4RelaxedTruncUSatF32x4:
+ case wasm::SimdOp::I32x4RelaxedTruncSatF64x2SZero:
+ case wasm::SimdOp::I32x4RelaxedTruncSatF64x2UZero:
+ case wasm::SimdOp::I64x2ExtendHighI32x4S:
+ case wasm::SimdOp::I64x2ExtendHighI32x4U:
+ // Prefer src == dest to avoid an unconditional src->dest move
+ // for better performance in non-AVX mode (e.g. non-PSHUFD use).
+ useAtStart = true;
+ reuseInput = !isThreeOpAllowed();
+ break;
+ case wasm::SimdOp::I32x4TruncSatF32x4U:
+ case wasm::SimdOp::I32x4TruncSatF64x2SZero:
+ case wasm::SimdOp::I32x4TruncSatF64x2UZero:
+ case wasm::SimdOp::I8x16Popcnt:
+ tempReg = tempSimd128();
+ // Prefer src == dest to avoid an unconditional src->dest move
+ // in non-AVX mode.
+ useAtStart = true;
+ reuseInput = !isThreeOpAllowed();
+ break;
+ case wasm::SimdOp::I16x8ExtendLowI8x16S:
+ case wasm::SimdOp::I16x8ExtendHighI8x16S:
+ case wasm::SimdOp::I16x8ExtendLowI8x16U:
+ case wasm::SimdOp::I16x8ExtendHighI8x16U:
+ case wasm::SimdOp::I32x4ExtendLowI16x8S:
+ case wasm::SimdOp::I32x4ExtendHighI16x8S:
+ case wasm::SimdOp::I32x4ExtendLowI16x8U:
+ case wasm::SimdOp::I32x4ExtendHighI16x8U:
+ case wasm::SimdOp::I64x2ExtendLowI32x4S:
+ case wasm::SimdOp::I64x2ExtendLowI32x4U:
+ case wasm::SimdOp::F32x4ConvertI32x4S:
+ case wasm::SimdOp::F32x4Ceil:
+ case wasm::SimdOp::F32x4Floor:
+ case wasm::SimdOp::F32x4Trunc:
+ case wasm::SimdOp::F32x4Nearest:
+ case wasm::SimdOp::F64x2Ceil:
+ case wasm::SimdOp::F64x2Floor:
+ case wasm::SimdOp::F64x2Trunc:
+ case wasm::SimdOp::F64x2Nearest:
+ case wasm::SimdOp::F32x4DemoteF64x2Zero:
+ case wasm::SimdOp::F64x2PromoteLowF32x4:
+ case wasm::SimdOp::F64x2ConvertLowI32x4S:
+ case wasm::SimdOp::F64x2ConvertLowI32x4U:
+ // Prefer src == dest to exert the lowest register pressure on the
+ // surrounding code.
+ useAtStart = true;
+ MOZ_ASSERT(!reuseInput);
+ break;
+ default:
+ MOZ_CRASH("Unary SimdOp not implemented");
+ }
+
+ LUse inputUse =
+ useAtStart ? useRegisterAtStart(ins->input()) : useRegister(ins->input());
+ LWasmUnarySimd128* lir = new (alloc()) LWasmUnarySimd128(inputUse, tempReg);
+ if (reuseInput) {
+ defineReuseInput(lir, ins, LWasmUnarySimd128::Src);
+ } else {
+ define(lir, ins);
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmLoadLaneSimd128(MWasmLoadLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ // A trick: On 32-bit systems, the base pointer is 32 bits (it was bounds
+ // checked and then chopped). On 64-bit systems, it can be 32 bits or 64
+ // bits. Either way, it fits in a GPR so we can ignore the
+ // Register/Register64 distinction here.
+# ifndef JS_64BIT
+ MOZ_ASSERT(ins->base()->type() == MIRType::Int32);
+# endif
+ LUse base = useRegisterAtStart(ins->base());
+ LUse inputUse = useRegisterAtStart(ins->value());
+ LAllocation memoryBase = ins->hasMemoryBase()
+ ? useRegisterAtStart(ins->memoryBase())
+ : LAllocation();
+ LWasmLoadLaneSimd128* lir = new (alloc()) LWasmLoadLaneSimd128(
+ base, inputUse, LDefinition::BogusTemp(), memoryBase);
+ defineReuseInput(lir, ins, LWasmLoadLaneSimd128::Src);
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+void LIRGenerator::visitWasmStoreLaneSimd128(MWasmStoreLaneSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ // See comment above.
+# ifndef JS_64BIT
+ MOZ_ASSERT(ins->base()->type() == MIRType::Int32);
+# endif
+ LUse base = useRegisterAtStart(ins->base());
+ LUse input = useRegisterAtStart(ins->value());
+ LAllocation memoryBase = ins->hasMemoryBase()
+ ? useRegisterAtStart(ins->memoryBase())
+ : LAllocation();
+ LWasmStoreLaneSimd128* lir = new (alloc())
+ LWasmStoreLaneSimd128(base, input, LDefinition::BogusTemp(), memoryBase);
+ add(lir, ins);
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
+
+#ifdef ENABLE_WASM_SIMD
+
+bool LIRGeneratorX86Shared::canFoldReduceSimd128AndBranch(wasm::SimdOp op) {
+ switch (op) {
+ case wasm::SimdOp::V128AnyTrue:
+ case wasm::SimdOp::I8x16AllTrue:
+ case wasm::SimdOp::I16x8AllTrue:
+ case wasm::SimdOp::I32x4AllTrue:
+ case wasm::SimdOp::I64x2AllTrue:
+ case wasm::SimdOp::I16x8Bitmask:
+ return true;
+ default:
+ return false;
+ }
+}
+
+bool LIRGeneratorX86Shared::canEmitWasmReduceSimd128AtUses(
+ MWasmReduceSimd128* ins) {
+ if (!ins->canEmitAtUses()) {
+ return false;
+ }
+ // Only specific ops generating int32.
+ if (ins->type() != MIRType::Int32) {
+ return false;
+ }
+ if (!canFoldReduceSimd128AndBranch(ins->simdOp())) {
+ return false;
+ }
+ // If never used then defer (it will be removed).
+ MUseIterator iter(ins->usesBegin());
+ if (iter == ins->usesEnd()) {
+ return true;
+ }
+ // We require an MTest consumer.
+ MNode* node = iter->consumer();
+ if (!node->isDefinition() || !node->toDefinition()->isTest()) {
+ return false;
+ }
+ // Defer only if there's only one use.
+ iter++;
+ return iter == ins->usesEnd();
+}
+
+#endif // ENABLE_WASM_SIMD
+
+void LIRGenerator::visitWasmReduceSimd128(MWasmReduceSimd128* ins) {
+#ifdef ENABLE_WASM_SIMD
+ if (canEmitWasmReduceSimd128AtUses(ins)) {
+ emitAtUses(ins);
+ return;
+ }
+
+ // Reductions (any_true, all_true, bitmask, extract_lane) uniformly prefer
+ // useRegisterAtStart:
+ //
+ // - In most cases, the input type differs from the output type, so there's no
+ // conflict and it doesn't really matter.
+ //
+ // - For extract_lane(0) on F32x4 and F64x2, input == output results in zero
+ // code being generated.
+ //
+ // - For extract_lane(k > 0) on F32x4 and F64x2, allowing the input register
+ // to be targeted lowers register pressure if it's the last use of the
+ // input.
+
+ if (ins->type() == MIRType::Int64) {
+ auto* lir = new (alloc())
+ LWasmReduceSimd128ToInt64(useRegisterAtStart(ins->input()));
+ defineInt64(lir, ins);
+ } else {
+ // Ideally we would reuse the input register for floating extract_lane if
+ // the lane is zero, but constraints in the register allocator require the
+ // input and output register types to be the same.
+ auto* lir = new (alloc()) LWasmReduceSimd128(
+ useRegisterAtStart(ins->input()), LDefinition::BogusTemp());
+ define(lir, ins);
+ }
+#else
+ MOZ_CRASH("No SIMD");
+#endif
+}
diff --git a/js/src/jit/x86-shared/Lowering-x86-shared.h b/js/src/jit/x86-shared/Lowering-x86-shared.h
new file mode 100644
index 0000000000..69d367270a
--- /dev/null
+++ b/js/src/jit/x86-shared/Lowering-x86-shared.h
@@ -0,0 +1,78 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_Lowering_x86_shared_h
+#define jit_x86_shared_Lowering_x86_shared_h
+
+#include "jit/shared/Lowering-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorX86Shared : public LIRGeneratorShared {
+ protected:
+ LIRGeneratorX86Shared(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorShared(gen, graph, lirGraph) {}
+
+ LTableSwitch* newLTableSwitch(const LAllocation& in,
+ const LDefinition& inputCopy,
+ MTableSwitch* ins);
+ LTableSwitchV* newLTableSwitchV(MTableSwitch* ins);
+
+ void lowerForShift(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerForALU(LInstructionHelper<1, 1, 0>* ins, MDefinition* mir,
+ MDefinition* input);
+ void lowerForALU(LInstructionHelper<1, 2, 0>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+
+ template <size_t Temps>
+ void lowerForShiftInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES + 1, Temps>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+
+ void lowerForCompareI64AndBranch(MTest* mir, MCompare* comp, JSOp op,
+ MDefinition* left, MDefinition* right,
+ MBasicBlock* ifTrue, MBasicBlock* ifFalse);
+ template <size_t Temps>
+ void lowerForFPU(LInstructionHelper<1, 2, Temps>* ins, MDefinition* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerForBitAndAndBranch(LBitAndAndBranch* baab, MInstruction* mir,
+ MDefinition* lhs, MDefinition* rhs);
+ void lowerNegI(MInstruction* ins, MDefinition* input);
+ void lowerNegI64(MInstruction* ins, MDefinition* input);
+ void lowerMulI(MMul* mul, MDefinition* lhs, MDefinition* rhs);
+ void lowerDivI(MDiv* div);
+ void lowerModI(MMod* mod);
+ void lowerUDiv(MDiv* div);
+ void lowerUMod(MMod* mod);
+ void lowerUrshD(MUrsh* mir);
+ void lowerPowOfTwoI(MPow* mir);
+ void lowerWasmSelectI(MWasmSelect* select);
+ void lowerWasmSelectI64(MWasmSelect* select);
+ void lowerBigIntLsh(MBigIntLsh* ins);
+ void lowerBigIntRsh(MBigIntRsh* ins);
+ void lowerWasmBuiltinTruncateToInt32(MWasmBuiltinTruncateToInt32* ins);
+ void lowerTruncateDToInt32(MTruncateToInt32* ins);
+ void lowerTruncateFToInt32(MTruncateToInt32* ins);
+ void lowerCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins, bool useI386ByteRegisters);
+ void lowerAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins, bool useI386ByteRegisters);
+ void lowerAtomicTypedArrayElementBinop(MAtomicTypedArrayElementBinop* ins,
+ bool useI386ByteRegisters);
+
+#ifdef ENABLE_WASM_SIMD
+ bool isThreeOpAllowed() { return Assembler::HasAVX(); }
+ bool canFoldReduceSimd128AndBranch(wasm::SimdOp op);
+ bool canEmitWasmReduceSimd128AtUses(MWasmReduceSimd128* ins);
+#endif
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_Lowering_x86_shared_h */
diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared-SIMD.cpp b/js/src/jit/x86-shared/MacroAssembler-x86-shared-SIMD.cpp
new file mode 100644
index 0000000000..80ee3ae888
--- /dev/null
+++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared-SIMD.cpp
@@ -0,0 +1,1511 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/MacroAssembler.h"
+#include "jit/x86-shared/MacroAssembler-x86-shared.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::DebugOnly;
+using mozilla::FloatingPoint;
+using mozilla::Maybe;
+using mozilla::SpecificNaN;
+
+void MacroAssemblerX86Shared::splatX16(Register input, FloatRegister output) {
+ ScratchSimd128Scope scratch(asMasm());
+
+ vmovd(input, output);
+ if (HasAVX2()) {
+ vbroadcastb(Operand(output), output);
+ return;
+ }
+ vpxor(scratch, scratch, scratch);
+ vpshufb(scratch, output, output);
+}
+
+void MacroAssemblerX86Shared::splatX8(Register input, FloatRegister output) {
+ vmovd(input, output);
+ if (HasAVX2()) {
+ vbroadcastw(Operand(output), output);
+ return;
+ }
+ vpshuflw(0, output, output);
+ vpshufd(0, output, output);
+}
+
+void MacroAssemblerX86Shared::splatX4(Register input, FloatRegister output) {
+ vmovd(input, output);
+ if (HasAVX2()) {
+ vbroadcastd(Operand(output), output);
+ return;
+ }
+ vpshufd(0, output, output);
+}
+
+void MacroAssemblerX86Shared::splatX4(FloatRegister input,
+ FloatRegister output) {
+ MOZ_ASSERT(input.isSingle() && output.isSimd128());
+ if (HasAVX2()) {
+ vbroadcastss(Operand(input), output);
+ return;
+ }
+ input = asMasm().moveSimd128FloatIfNotAVX(input.asSimd128(), output);
+ vshufps(0, input, input, output);
+}
+
+void MacroAssemblerX86Shared::splatX2(FloatRegister input,
+ FloatRegister output) {
+ MOZ_ASSERT(input.isDouble() && output.isSimd128());
+ vmovddup(Operand(input.asSimd128()), output);
+}
+
+void MacroAssemblerX86Shared::extractLaneInt32x4(FloatRegister input,
+ Register output,
+ unsigned lane) {
+ if (lane == 0) {
+ // The value we want to extract is in the low double-word
+ moveLowInt32(input, output);
+ } else {
+ vpextrd(lane, input, output);
+ }
+}
+
+void MacroAssemblerX86Shared::extractLaneFloat32x4(FloatRegister input,
+ FloatRegister output,
+ unsigned lane) {
+ MOZ_ASSERT(input.isSimd128() && output.isSingle());
+ if (lane == 0) {
+ // The value we want to extract is in the low double-word
+ if (input.asSingle() != output) {
+ moveFloat32(input, output);
+ }
+ } else if (lane == 2) {
+ moveHighPairToLowPairFloat32(input, output);
+ } else {
+ uint32_t mask = MacroAssembler::ComputeShuffleMask(lane);
+ FloatRegister dest = output.asSimd128();
+ input = moveSimd128FloatIfNotAVX(input, dest);
+ vshufps(mask, input, input, dest);
+ }
+}
+
+void MacroAssemblerX86Shared::extractLaneFloat64x2(FloatRegister input,
+ FloatRegister output,
+ unsigned lane) {
+ MOZ_ASSERT(input.isSimd128() && output.isDouble());
+ if (lane == 0) {
+ // The value we want to extract is in the low quadword
+ if (input.asDouble() != output) {
+ moveDouble(input, output);
+ }
+ } else {
+ vpalignr(Operand(input), output, output, 8);
+ }
+}
+
+void MacroAssemblerX86Shared::extractLaneInt16x8(FloatRegister input,
+ Register output, unsigned lane,
+ SimdSign sign) {
+ vpextrw(lane, input, Operand(output));
+ if (sign == SimdSign::Signed) {
+ movswl(output, output);
+ }
+}
+
+void MacroAssemblerX86Shared::extractLaneInt8x16(FloatRegister input,
+ Register output, unsigned lane,
+ SimdSign sign) {
+ vpextrb(lane, input, Operand(output));
+ if (sign == SimdSign::Signed) {
+ if (!AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(output)) {
+ xchgl(eax, output);
+ movsbl(eax, eax);
+ xchgl(eax, output);
+ } else {
+ movsbl(output, output);
+ }
+ }
+}
+
+void MacroAssemblerX86Shared::replaceLaneFloat32x4(unsigned lane,
+ FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ MOZ_ASSERT(lhs.isSimd128() && rhs.isSingle());
+
+ if (lane == 0) {
+ if (rhs.asSimd128() == lhs) {
+ // no-op, although this should not normally happen for type checking
+ // reasons higher up in the stack.
+ moveSimd128Float(lhs, dest);
+ } else {
+ // move low dword of value into low dword of output
+ vmovss(rhs, lhs, dest);
+ }
+ } else {
+ vinsertps(vinsertpsMask(0, lane), rhs, lhs, dest);
+ }
+}
+
+void MacroAssemblerX86Shared::replaceLaneFloat64x2(unsigned lane,
+ FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ MOZ_ASSERT(lhs.isSimd128() && rhs.isDouble());
+
+ if (lane == 0) {
+ if (rhs.asSimd128() == lhs) {
+ // no-op, although this should not normally happen for type checking
+ // reasons higher up in the stack.
+ moveSimd128Float(lhs, dest);
+ } else {
+ // move low qword of value into low qword of output
+ vmovsd(rhs, lhs, dest);
+ }
+ } else {
+ // move low qword of value into high qword of output
+ vshufpd(0, rhs, lhs, dest);
+ }
+}
+
+void MacroAssemblerX86Shared::blendInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output,
+ FloatRegister temp,
+ const uint8_t lanes[16]) {
+ asMasm().loadConstantSimd128Int(
+ SimdConstant::CreateX16(reinterpret_cast<const int8_t*>(lanes)), temp);
+ vpblendvb(temp, rhs, lhs, output);
+}
+
+void MacroAssemblerX86Shared::blendInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output,
+ const uint16_t lanes[8]) {
+ uint32_t mask = 0;
+ for (unsigned i = 0; i < 8; i++) {
+ if (lanes[i]) {
+ mask |= (1 << i);
+ }
+ }
+ vpblendw(mask, rhs, lhs, output);
+}
+
+void MacroAssemblerX86Shared::laneSelectSimd128(FloatRegister mask,
+ FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister output) {
+ vpblendvb(mask, lhs, rhs, output);
+}
+
+void MacroAssemblerX86Shared::shuffleInt8x16(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister output,
+ const uint8_t lanes[16]) {
+ ScratchSimd128Scope scratch(asMasm());
+
+ // Use pshufb instructions to gather the lanes from each source vector.
+ // A negative index creates a zero lane, so the two vectors can be combined.
+
+ // Set scratch = lanes from rhs.
+ int8_t idx[16];
+ for (unsigned i = 0; i < 16; i++) {
+ idx[i] = lanes[i] >= 16 ? lanes[i] - 16 : -1;
+ }
+ rhs = moveSimd128IntIfNotAVX(rhs, scratch);
+ asMasm().vpshufbSimd128(SimdConstant::CreateX16(idx), rhs, scratch);
+
+ // Set output = lanes from lhs.
+ for (unsigned i = 0; i < 16; i++) {
+ idx[i] = lanes[i] < 16 ? lanes[i] : -1;
+ }
+ lhs = moveSimd128IntIfNotAVX(lhs, output);
+ asMasm().vpshufbSimd128(SimdConstant::CreateX16(idx), lhs, output);
+
+ // Combine.
+ vpor(scratch, output, output);
+}
+
+static inline FloatRegister ToSimdFloatRegister(const Operand& op) {
+ return FloatRegister(op.fpu(), FloatRegister::Codes::ContentType::Simd128);
+}
+
+void MacroAssemblerX86Shared::compareInt8x16(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond,
+ FloatRegister output) {
+ switch (cond) {
+ case Assembler::Condition::GreaterThan:
+ vpcmpgtb(rhs, lhs, output);
+ break;
+ case Assembler::Condition::Equal:
+ vpcmpeqb(rhs, lhs, output);
+ break;
+ case Assembler::Condition::LessThan: {
+ ScratchSimd128Scope scratch(asMasm());
+ if (lhs == output) {
+ moveSimd128Int(lhs, scratch);
+ lhs = scratch;
+ }
+ if (rhs.kind() == Operand::FPREG) {
+ moveSimd128Int(ToSimdFloatRegister(rhs), output);
+ } else {
+ loadAlignedSimd128Int(rhs, output);
+ }
+ vpcmpgtb(Operand(lhs), output, output);
+ break;
+ }
+ case Assembler::Condition::NotEqual:
+ vpcmpeqb(rhs, lhs, output);
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Condition::GreaterThanOrEqual: {
+ ScratchSimd128Scope scratch(asMasm());
+ if (lhs == output) {
+ moveSimd128Int(lhs, scratch);
+ lhs = scratch;
+ }
+ if (rhs.kind() == Operand::FPREG) {
+ moveSimd128Int(ToSimdFloatRegister(rhs), output);
+ } else {
+ loadAlignedSimd128Int(rhs, output);
+ }
+ vpcmpgtb(Operand(lhs), output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ // lhs <= rhs is equivalent to !(rhs < lhs), which we compute here.
+ vpcmpgtb(rhs, lhs, output);
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Above:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpminub(rhs, lhs, output);
+ vpcmpeqb(Operand(lhs), output, output);
+ } else {
+ vpmaxub(rhs, lhs, output);
+ vpcmpeqb(rhs, output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::BelowOrEqual:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpminub(rhs, lhs, output);
+ vpcmpeqb(Operand(lhs), output, output);
+ } else {
+ vpmaxub(rhs, lhs, output);
+ vpcmpeqb(rhs, output, output);
+ }
+ break;
+ case Assembler::Below:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpmaxub(rhs, lhs, output);
+ vpcmpeqb(Operand(lhs), output, output);
+ } else {
+ vpminub(rhs, lhs, output);
+ vpcmpeqb(rhs, output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::AboveOrEqual:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpmaxub(rhs, lhs, output);
+ vpcmpeqb(Operand(lhs), output, output);
+ } else {
+ vpminub(rhs, lhs, output);
+ vpcmpeqb(rhs, output, output);
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareInt8x16(Assembler::Condition cond,
+ FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ bool complement = false;
+ switch (cond) {
+ case Assembler::Condition::NotEqual:
+ complement = true;
+ [[fallthrough]];
+ case Assembler::Condition::Equal:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpcmpeqb,
+ &MacroAssembler::vpcmpeqbSimd128);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ complement = true;
+ [[fallthrough]];
+ case Assembler::Condition::GreaterThan:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpcmpgtb,
+ &MacroAssembler::vpcmpgtbSimd128);
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+ if (complement) {
+ asMasm().bitwiseXorSimd128(dest, SimdConstant::SplatX16(-1), dest);
+ }
+}
+
+void MacroAssemblerX86Shared::compareInt16x8(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond,
+ FloatRegister output) {
+ switch (cond) {
+ case Assembler::Condition::GreaterThan:
+ vpcmpgtw(rhs, lhs, output);
+ break;
+ case Assembler::Condition::Equal:
+ vpcmpeqw(rhs, lhs, output);
+ break;
+ case Assembler::Condition::LessThan: {
+ ScratchSimd128Scope scratch(asMasm());
+ if (lhs == output) {
+ moveSimd128Int(lhs, scratch);
+ lhs = scratch;
+ }
+ if (rhs.kind() == Operand::FPREG) {
+ moveSimd128Int(ToSimdFloatRegister(rhs), output);
+ } else {
+ loadAlignedSimd128Int(rhs, output);
+ }
+ vpcmpgtw(Operand(lhs), output, output);
+ break;
+ }
+ case Assembler::Condition::NotEqual:
+ vpcmpeqw(rhs, lhs, output);
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Condition::GreaterThanOrEqual: {
+ ScratchSimd128Scope scratch(asMasm());
+ if (lhs == output) {
+ moveSimd128Int(lhs, scratch);
+ lhs = scratch;
+ }
+ if (rhs.kind() == Operand::FPREG) {
+ moveSimd128Int(ToSimdFloatRegister(rhs), output);
+ } else {
+ loadAlignedSimd128Int(rhs, output);
+ }
+ vpcmpgtw(Operand(lhs), output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ // lhs <= rhs is equivalent to !(rhs < lhs), which we compute here.
+ vpcmpgtw(rhs, lhs, output);
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Above:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpminuw(rhs, lhs, output);
+ vpcmpeqw(Operand(lhs), output, output);
+ } else {
+ vpmaxuw(rhs, lhs, output);
+ vpcmpeqw(rhs, output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::BelowOrEqual:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpminuw(rhs, lhs, output);
+ vpcmpeqw(Operand(lhs), output, output);
+ } else {
+ vpmaxuw(rhs, lhs, output);
+ vpcmpeqw(rhs, output, output);
+ }
+ break;
+ case Assembler::Below:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpmaxuw(rhs, lhs, output);
+ vpcmpeqw(Operand(lhs), output, output);
+ } else {
+ vpminuw(rhs, lhs, output);
+ vpcmpeqw(rhs, output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::AboveOrEqual:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpmaxuw(rhs, lhs, output);
+ vpcmpeqw(Operand(lhs), output, output);
+ } else {
+ vpminuw(rhs, lhs, output);
+ vpcmpeqw(rhs, output, output);
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareInt16x8(Assembler::Condition cond,
+ FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ bool complement = false;
+ switch (cond) {
+ case Assembler::Condition::NotEqual:
+ complement = true;
+ [[fallthrough]];
+ case Assembler::Condition::Equal:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpcmpeqw,
+ &MacroAssembler::vpcmpeqwSimd128);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ complement = true;
+ [[fallthrough]];
+ case Assembler::Condition::GreaterThan:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpcmpgtw,
+ &MacroAssembler::vpcmpgtwSimd128);
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+ if (complement) {
+ asMasm().bitwiseXorSimd128(dest, SimdConstant::SplatX16(-1), dest);
+ }
+}
+
+void MacroAssemblerX86Shared::compareInt32x4(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond,
+ FloatRegister output) {
+ switch (cond) {
+ case Assembler::Condition::GreaterThan:
+ vpcmpgtd(rhs, lhs, output);
+ break;
+ case Assembler::Condition::Equal:
+ vpcmpeqd(rhs, lhs, output);
+ break;
+ case Assembler::Condition::LessThan: {
+ ScratchSimd128Scope scratch(asMasm());
+ if (lhs == output) {
+ moveSimd128Int(lhs, scratch);
+ lhs = scratch;
+ }
+ if (rhs.kind() == Operand::FPREG) {
+ moveSimd128Int(ToSimdFloatRegister(rhs), output);
+ } else {
+ loadAlignedSimd128Int(rhs, output);
+ }
+ vpcmpgtd(Operand(lhs), output, output);
+ break;
+ }
+ case Assembler::Condition::NotEqual:
+ vpcmpeqd(rhs, lhs, output);
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Condition::GreaterThanOrEqual: {
+ ScratchSimd128Scope scratch(asMasm());
+ if (lhs == output) {
+ moveSimd128Int(lhs, scratch);
+ lhs = scratch;
+ }
+ if (rhs.kind() == Operand::FPREG) {
+ moveSimd128Int(ToSimdFloatRegister(rhs), output);
+ } else {
+ loadAlignedSimd128Int(rhs, output);
+ }
+ vpcmpgtd(Operand(lhs), output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ // lhs <= rhs is equivalent to !(rhs < lhs), which we compute here.
+ vpcmpgtd(rhs, lhs, output);
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::Above:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpminud(rhs, lhs, output);
+ vpcmpeqd(Operand(lhs), output, output);
+ } else {
+ vpmaxud(rhs, lhs, output);
+ vpcmpeqd(rhs, output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::BelowOrEqual:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpminud(rhs, lhs, output);
+ vpcmpeqd(Operand(lhs), output, output);
+ } else {
+ vpmaxud(rhs, lhs, output);
+ vpcmpeqd(rhs, output, output);
+ }
+ break;
+ case Assembler::Below:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpmaxud(rhs, lhs, output);
+ vpcmpeqd(Operand(lhs), output, output);
+ } else {
+ vpminud(rhs, lhs, output);
+ vpcmpeqd(rhs, output, output);
+ }
+ asMasm().bitwiseNotSimd128(output, output);
+ break;
+ case Assembler::AboveOrEqual:
+ if (rhs.kind() == Operand::FPREG && ToSimdFloatRegister(rhs) == output) {
+ vpmaxud(rhs, lhs, output);
+ vpcmpeqd(Operand(lhs), output, output);
+ } else {
+ vpminud(rhs, lhs, output);
+ vpcmpeqd(rhs, output, output);
+ }
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareInt32x4(Assembler::Condition cond,
+ FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ bool complement = false;
+ switch (cond) {
+ case Assembler::Condition::NotEqual:
+ complement = true;
+ [[fallthrough]];
+ case Assembler::Condition::Equal:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpcmpeqd,
+ &MacroAssembler::vpcmpeqdSimd128);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ complement = true;
+ [[fallthrough]];
+ case Assembler::Condition::GreaterThan:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpcmpgtd,
+ &MacroAssembler::vpcmpgtdSimd128);
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+ if (complement) {
+ asMasm().bitwiseXorSimd128(dest, SimdConstant::SplatX16(-1), dest);
+ }
+}
+
+void MacroAssemblerX86Shared::compareForEqualityInt64x2(
+ FloatRegister lhs, Operand rhs, Assembler::Condition cond,
+ FloatRegister output) {
+ static const SimdConstant allOnes = SimdConstant::SplatX4(-1);
+ switch (cond) {
+ case Assembler::Condition::Equal:
+ vpcmpeqq(rhs, lhs, output);
+ break;
+ case Assembler::Condition::NotEqual:
+ vpcmpeqq(rhs, lhs, output);
+ asMasm().bitwiseXorSimd128(output, allOnes, output);
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareForOrderingInt64x2(
+ FloatRegister lhs, Operand rhs, Assembler::Condition cond,
+ FloatRegister temp1, FloatRegister temp2, FloatRegister output) {
+ static const SimdConstant allOnes = SimdConstant::SplatX4(-1);
+ // The pseudo code is for (e.g. > comparison):
+ // __m128i pcmpgtq_sse2 (__m128i a, __m128i b) {
+ // __m128i r = _mm_and_si128(_mm_cmpeq_epi32(a, b), _mm_sub_epi64(b, a));
+ // r = _mm_or_si128(r, _mm_cmpgt_epi32(a, b));
+ // return _mm_shuffle_epi32(r, _MM_SHUFFLE(3,3,1,1));
+ // }
+ // Credits to https://stackoverflow.com/a/65175746
+ switch (cond) {
+ case Assembler::Condition::GreaterThan:
+ vmovdqa(rhs, temp1);
+ vmovdqa(Operand(lhs), temp2);
+ vpsubq(Operand(lhs), temp1, temp1);
+ vpcmpeqd(rhs, temp2, temp2);
+ vandpd(temp2, temp1, temp1);
+ lhs = asMasm().moveSimd128IntIfNotAVX(lhs, output);
+ vpcmpgtd(rhs, lhs, output);
+ vpor(Operand(temp1), output, output);
+ vpshufd(MacroAssembler::ComputeShuffleMask(1, 1, 3, 3), output, output);
+ break;
+ case Assembler::Condition::LessThan:
+ vmovdqa(rhs, temp1);
+ vmovdqa(Operand(lhs), temp2);
+ vpcmpgtd(Operand(lhs), temp1, temp1);
+ vpcmpeqd(Operand(rhs), temp2, temp2);
+ lhs = asMasm().moveSimd128IntIfNotAVX(lhs, output);
+ vpsubq(rhs, lhs, output);
+ vandpd(temp2, output, output);
+ vpor(Operand(temp1), output, output);
+ vpshufd(MacroAssembler::ComputeShuffleMask(1, 1, 3, 3), output, output);
+ break;
+ case Assembler::Condition::GreaterThanOrEqual:
+ vmovdqa(rhs, temp1);
+ vmovdqa(Operand(lhs), temp2);
+ vpcmpgtd(Operand(lhs), temp1, temp1);
+ vpcmpeqd(Operand(rhs), temp2, temp2);
+ lhs = asMasm().moveSimd128IntIfNotAVX(lhs, output);
+ vpsubq(rhs, lhs, output);
+ vandpd(temp2, output, output);
+ vpor(Operand(temp1), output, output);
+ vpshufd(MacroAssembler::ComputeShuffleMask(1, 1, 3, 3), output, output);
+ asMasm().bitwiseXorSimd128(output, allOnes, output);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ vmovdqa(rhs, temp1);
+ vmovdqa(Operand(lhs), temp2);
+ vpsubq(Operand(lhs), temp1, temp1);
+ vpcmpeqd(rhs, temp2, temp2);
+ vandpd(temp2, temp1, temp1);
+ lhs = asMasm().moveSimd128IntIfNotAVX(lhs, output);
+ vpcmpgtd(rhs, lhs, output);
+ vpor(Operand(temp1), output, output);
+ vpshufd(MacroAssembler::ComputeShuffleMask(1, 1, 3, 3), output, output);
+ asMasm().bitwiseXorSimd128(output, allOnes, output);
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareForOrderingInt64x2AVX(
+ FloatRegister lhs, FloatRegister rhs, Assembler::Condition cond,
+ FloatRegister output) {
+ MOZ_ASSERT(HasSSE42());
+ static const SimdConstant allOnes = SimdConstant::SplatX4(-1);
+ switch (cond) {
+ case Assembler::Condition::GreaterThan:
+ vpcmpgtq(Operand(rhs), lhs, output);
+ break;
+ case Assembler::Condition::LessThan:
+ vpcmpgtq(Operand(lhs), rhs, output);
+ break;
+ case Assembler::Condition::GreaterThanOrEqual:
+ vpcmpgtq(Operand(lhs), rhs, output);
+ asMasm().bitwiseXorSimd128(output, allOnes, output);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ vpcmpgtq(Operand(rhs), lhs, output);
+ asMasm().bitwiseXorSimd128(output, allOnes, output);
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareFloat32x4(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond,
+ FloatRegister output) {
+ // TODO Can do better here with three-address compares
+
+ // Move lhs to output if lhs!=output; move rhs out of the way if rhs==output.
+ // This is bad, but Ion does not need this fixup.
+ ScratchSimd128Scope scratch(asMasm());
+ if (!HasAVX() && !lhs.aliases(output)) {
+ if (rhs.kind() == Operand::FPREG &&
+ output.aliases(FloatRegister::FromCode(rhs.fpu()))) {
+ vmovaps(rhs, scratch);
+ rhs = Operand(scratch);
+ }
+ vmovaps(lhs, output);
+ lhs = output;
+ }
+
+ switch (cond) {
+ case Assembler::Condition::Equal:
+ vcmpeqps(rhs, lhs, output);
+ break;
+ case Assembler::Condition::LessThan:
+ vcmpltps(rhs, lhs, output);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ vcmpleps(rhs, lhs, output);
+ break;
+ case Assembler::Condition::NotEqual:
+ vcmpneqps(rhs, lhs, output);
+ break;
+ case Assembler::Condition::GreaterThanOrEqual:
+ case Assembler::Condition::GreaterThan:
+ // We reverse these operations in the -inl.h file so that we don't have to
+ // copy into and out of temporaries after codegen.
+ MOZ_CRASH("should have reversed this");
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareFloat32x4(Assembler::Condition cond,
+ FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ switch (cond) {
+ case Assembler::Condition::Equal:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vcmpeqps,
+ &MacroAssembler::vcmpeqpsSimd128);
+ break;
+ case Assembler::Condition::LessThan:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vcmpltps,
+ &MacroAssembler::vcmpltpsSimd128);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vcmpleps,
+ &MacroAssembler::vcmplepsSimd128);
+ break;
+ case Assembler::Condition::NotEqual:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vcmpneqps,
+ &MacroAssembler::vcmpneqpsSimd128);
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareFloat64x2(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond,
+ FloatRegister output) {
+ // TODO Can do better here with three-address compares
+
+ // Move lhs to output if lhs!=output; move rhs out of the way if rhs==output.
+ // This is bad, but Ion does not need this fixup.
+ ScratchSimd128Scope scratch(asMasm());
+ if (!HasAVX() && !lhs.aliases(output)) {
+ if (rhs.kind() == Operand::FPREG &&
+ output.aliases(FloatRegister::FromCode(rhs.fpu()))) {
+ vmovapd(rhs, scratch);
+ rhs = Operand(scratch);
+ }
+ vmovapd(lhs, output);
+ lhs = output;
+ }
+
+ switch (cond) {
+ case Assembler::Condition::Equal:
+ vcmpeqpd(rhs, lhs, output);
+ break;
+ case Assembler::Condition::LessThan:
+ vcmpltpd(rhs, lhs, output);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ vcmplepd(rhs, lhs, output);
+ break;
+ case Assembler::Condition::NotEqual:
+ vcmpneqpd(rhs, lhs, output);
+ break;
+ case Assembler::Condition::GreaterThanOrEqual:
+ case Assembler::Condition::GreaterThan:
+ // We reverse these operations in the -inl.h file so that we don't have to
+ // copy into and out of temporaries after codegen.
+ MOZ_CRASH("should have reversed this");
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+void MacroAssemblerX86Shared::compareFloat64x2(Assembler::Condition cond,
+ FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ switch (cond) {
+ case Assembler::Condition::Equal:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vcmpeqpd,
+ &MacroAssembler::vcmpeqpdSimd128);
+ break;
+ case Assembler::Condition::LessThan:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vcmpltpd,
+ &MacroAssembler::vcmpltpdSimd128);
+ break;
+ case Assembler::Condition::LessThanOrEqual:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vcmplepd,
+ &MacroAssembler::vcmplepdSimd128);
+ break;
+ case Assembler::Condition::NotEqual:
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vcmpneqpd,
+ &MacroAssembler::vcmpneqpdSimd128);
+ break;
+ default:
+ MOZ_CRASH("unexpected condition op");
+ }
+}
+
+// Semantics of wasm max and min.
+//
+// * -0 < 0
+// * If one input is NaN then that NaN is the output
+// * If both inputs are NaN then the output is selected nondeterministically
+// * Any returned NaN is always made quiet
+// * The MVP spec 2.2.3 says "No distinction is made between signalling and
+// quiet NaNs", suggesting SNaN inputs are allowed and should not fault
+//
+// Semantics of maxps/minps/maxpd/minpd:
+//
+// * If the values are both +/-0 the rhs is returned
+// * If the rhs is SNaN then the rhs is returned
+// * If either value is NaN then the rhs is returned
+// * An SNaN operand does not appear to give rise to an exception, at least
+// not in the JS shell on Linux, though the Intel spec lists Invalid
+// as one of the possible exceptions
+
+// Various unaddressed considerations:
+//
+// It's pretty insane for this to take an Operand rhs - it really needs to be
+// a register, given the number of times we access it.
+//
+// Constant load can be folded into the ANDPS. Do we care? It won't save us
+// any registers, since output/temp1/temp2/scratch are all live at the same time
+// after the first instruction of the slow path.
+//
+// Can we use blend for the NaN extraction/insertion? We'd need xmm0 for the
+// mask, which is no fun. But it would be lhs UNORD lhs -> mask, blend;
+// rhs UNORD rhs -> mask; blend. Better than the mess we have below. But
+// we'd still need to setup the QNaN bits, unless we can blend those too
+// with the lhs UNORD rhs mask?
+//
+// If we could determine that both input lanes are NaN then the result of the
+// fast path should be fine modulo the QNaN bits, but it's not obvious this is
+// much of an advantage.
+
+void MacroAssemblerX86Shared::minMaxFloat32x4(bool isMin, FloatRegister lhs,
+ Operand rhs, FloatRegister temp1,
+ FloatRegister temp2,
+ FloatRegister output) {
+ ScratchSimd128Scope scratch(asMasm());
+ Label l;
+ SimdConstant quietBits(SimdConstant::SplatX4(int32_t(0x00400000)));
+
+ /* clang-format off */ /* leave my comments alone */
+ lhs = moveSimd128FloatIfNotAVXOrOther(lhs, scratch, output);
+ if (isMin) {
+ vmovaps(lhs, output); // compute
+ vminps(rhs, output, output); // min lhs, rhs
+ vmovaps(rhs, temp1); // compute
+ vminps(Operand(lhs), temp1, temp1); // min rhs, lhs
+ vorps(temp1, output, output); // fix min(-0, 0) with OR
+ } else {
+ vmovaps(lhs, output); // compute
+ vmaxps(rhs, output, output); // max lhs, rhs
+ vmovaps(rhs, temp1); // compute
+ vmaxps(Operand(lhs), temp1, temp1); // max rhs, lhs
+ vandps(temp1, output, output); // fix max(-0, 0) with AND
+ }
+ vmovaps(lhs, temp1); // compute
+ vcmpunordps(rhs, temp1, temp1); // lhs UNORD rhs
+ vptest(temp1, temp1); // check if any unordered
+ j(Assembler::Equal, &l); // and exit if not
+
+ // Slow path.
+ // output has result for non-NaN lanes, garbage in NaN lanes.
+ // temp1 has lhs UNORD rhs.
+ // temp2 is dead.
+
+ vmovaps(temp1, temp2); // clear NaN lanes of result
+ vpandn(output, temp2, temp2); // result now in temp2
+ asMasm().vpandSimd128(quietBits, temp1, temp1); // setup QNaN bits in NaN lanes
+ vorps(temp1, temp2, temp2); // and OR into result
+ vmovaps(lhs, temp1); // find NaN lanes
+ vcmpunordps(Operand(temp1), temp1, temp1); // in lhs
+ vmovaps(temp1, output); // (and save them for later)
+ vandps(lhs, temp1, temp1); // and extract the NaNs
+ vorps(temp1, temp2, temp2); // and add to the result
+ vmovaps(rhs, temp1); // find NaN lanes
+ vcmpunordps(Operand(temp1), temp1, temp1); // in rhs
+ vpandn(temp1, output, output); // except if they were in lhs
+ vandps(rhs, output, output); // and extract the NaNs
+ vorps(temp2, output, output); // and add to the result
+
+ bind(&l);
+ /* clang-format on */
+}
+
+void MacroAssemblerX86Shared::minMaxFloat32x4AVX(bool isMin, FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister temp1,
+ FloatRegister temp2,
+ FloatRegister output) {
+ ScratchSimd128Scope scratch(asMasm());
+ Label l;
+ SimdConstant quietBits(SimdConstant::SplatX4(int32_t(0x00400000)));
+
+ /* clang-format off */ /* leave my comments alone */
+ FloatRegister lhsCopy = moveSimd128FloatIfEqual(lhs, scratch, output);
+ // Allow rhs be assigned to scratch when rhs == lhs and == output --
+ // don't make a special case since the semantics require setup QNaN bits.
+ FloatRegister rhsCopy = moveSimd128FloatIfEqual(rhs, scratch, output);
+ if (isMin) {
+ vminps(Operand(rhs), lhs, temp2); // min lhs, rhs
+ vminps(Operand(lhs), rhs, temp1); // min rhs, lhs
+ vorps(temp1, temp2, output); // fix min(-0, 0) with OR
+ } else {
+ vmaxps(Operand(rhs), lhs, temp2); // max lhs, rhs
+ vmaxps(Operand(lhs), rhs, temp1); // max rhs, lhs
+ vandps(temp1, temp2, output); // fix max(-0, 0) with AND
+ }
+ vcmpunordps(Operand(rhsCopy), lhsCopy, temp1); // lhs UNORD rhs
+ vptest(temp1, temp1); // check if any unordered
+ j(Assembler::Equal, &l); // and exit if not
+
+ // Slow path.
+ // output has result for non-NaN lanes, garbage in NaN lanes.
+ // temp1 has lhs UNORD rhs.
+ // temp2 is dead.
+ vcmpunordps(Operand(lhsCopy), lhsCopy, temp2); // find NaN lanes in lhs
+ vblendvps(temp2, lhsCopy, rhsCopy, temp2); // add other lines from rhs
+ asMasm().vporSimd128(quietBits, temp2, temp2); // setup QNaN bits in NaN lanes
+ vblendvps(temp1, temp2, output, output); // replace NaN lines from temp2
+
+ bind(&l);
+ /* clang-format on */
+}
+
+// Exactly as above.
+void MacroAssemblerX86Shared::minMaxFloat64x2(bool isMin, FloatRegister lhs,
+ Operand rhs, FloatRegister temp1,
+ FloatRegister temp2,
+ FloatRegister output) {
+ ScratchSimd128Scope scratch(asMasm());
+ Label l;
+ SimdConstant quietBits(SimdConstant::SplatX2(int64_t(0x0008000000000000ull)));
+
+ /* clang-format off */ /* leave my comments alone */
+ lhs = moveSimd128FloatIfNotAVXOrOther(lhs, scratch, output);
+ if (isMin) {
+ vmovapd(lhs, output); // compute
+ vminpd(rhs, output, output); // min lhs, rhs
+ vmovapd(rhs, temp1); // compute
+ vminpd(Operand(lhs), temp1, temp1); // min rhs, lhs
+ vorpd(temp1, output, output); // fix min(-0, 0) with OR
+ } else {
+ vmovapd(lhs, output); // compute
+ vmaxpd(rhs, output, output); // max lhs, rhs
+ vmovapd(rhs, temp1); // compute
+ vmaxpd(Operand(lhs), temp1, temp1); // max rhs, lhs
+ vandpd(temp1, output, output); // fix max(-0, 0) with AND
+ }
+ vmovapd(lhs, temp1); // compute
+ vcmpunordpd(rhs, temp1, temp1); // lhs UNORD rhs
+ vptest(temp1, temp1); // check if any unordered
+ j(Assembler::Equal, &l); // and exit if not
+
+ // Slow path.
+ // output has result for non-NaN lanes, garbage in NaN lanes.
+ // temp1 has lhs UNORD rhs.
+ // temp2 is dead.
+
+ vmovapd(temp1, temp2); // clear NaN lanes of result
+ vpandn(output, temp2, temp2); // result now in temp2
+ asMasm().vpandSimd128(quietBits, temp1, temp1); // setup QNaN bits in NaN lanes
+ vorpd(temp1, temp2, temp2); // and OR into result
+ vmovapd(lhs, temp1); // find NaN lanes
+ vcmpunordpd(Operand(temp1), temp1, temp1); // in lhs
+ vmovapd(temp1, output); // (and save them for later)
+ vandpd(lhs, temp1, temp1); // and extract the NaNs
+ vorpd(temp1, temp2, temp2); // and add to the result
+ vmovapd(rhs, temp1); // find NaN lanes
+ vcmpunordpd(Operand(temp1), temp1, temp1); // in rhs
+ vpandn(temp1, output, output); // except if they were in lhs
+ vandpd(rhs, output, output); // and extract the NaNs
+ vorpd(temp2, output, output); // and add to the result
+
+ bind(&l);
+ /* clang-format on */
+}
+
+void MacroAssemblerX86Shared::minMaxFloat64x2AVX(bool isMin, FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister temp1,
+ FloatRegister temp2,
+ FloatRegister output) {
+ ScratchSimd128Scope scratch(asMasm());
+ Label l;
+ SimdConstant quietBits(SimdConstant::SplatX2(int64_t(0x0008000000000000ull)));
+
+ /* clang-format off */ /* leave my comments alone */
+ FloatRegister lhsCopy = moveSimd128FloatIfEqual(lhs, scratch, output);
+ // Allow rhs be assigned to scratch when rhs == lhs and == output --
+ // don't make a special case since the semantics require setup QNaN bits.
+ FloatRegister rhsCopy = moveSimd128FloatIfEqual(rhs, scratch, output);
+ if (isMin) {
+ vminpd(Operand(rhs), lhs, temp2); // min lhs, rhs
+ vminpd(Operand(lhs), rhs, temp1); // min rhs, lhs
+ vorpd(temp1, temp2, output); // fix min(-0, 0) with OR
+ } else {
+ vmaxpd(Operand(rhs), lhs, temp2); // max lhs, rhs
+ vmaxpd(Operand(lhs), rhs, temp1); // max rhs, lhs
+ vandpd(temp1, temp2, output); // fix max(-0, 0) with AND
+ }
+ vcmpunordpd(Operand(rhsCopy), lhsCopy, temp1); // lhs UNORD rhs
+ vptest(temp1, temp1); // check if any unordered
+ j(Assembler::Equal, &l); // and exit if not
+
+ // Slow path.
+ // output has result for non-NaN lanes, garbage in NaN lanes.
+ // temp1 has lhs UNORD rhs.
+ // temp2 is dead.
+ vcmpunordpd(Operand(lhsCopy), lhsCopy, temp2); // find NaN lanes in lhs
+ vblendvpd(temp2, lhsCopy, rhsCopy, temp2); // add other lines from rhs
+ asMasm().vporSimd128(quietBits, temp2, temp2); // setup QNaN bits in NaN lanes
+ vblendvpd(temp1, temp2, output, output); // replace NaN lines from temp2
+
+ bind(&l);
+ /* clang-format on */
+}
+
+void MacroAssemblerX86Shared::minFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister temp1,
+ FloatRegister temp2,
+ FloatRegister output) {
+ if (HasAVX()) {
+ minMaxFloat32x4AVX(/*isMin=*/true, lhs, rhs, temp1, temp2, output);
+ return;
+ }
+ minMaxFloat32x4(/*isMin=*/true, lhs, Operand(rhs), temp1, temp2, output);
+}
+
+void MacroAssemblerX86Shared::maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister temp1,
+ FloatRegister temp2,
+ FloatRegister output) {
+ if (HasAVX()) {
+ minMaxFloat32x4AVX(/*isMin=*/false, lhs, rhs, temp1, temp2, output);
+ return;
+ }
+ minMaxFloat32x4(/*isMin=*/false, lhs, Operand(rhs), temp1, temp2, output);
+}
+
+void MacroAssemblerX86Shared::minFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister temp1,
+ FloatRegister temp2,
+ FloatRegister output) {
+ if (HasAVX()) {
+ minMaxFloat64x2AVX(/*isMin=*/true, lhs, rhs, temp1, temp2, output);
+ return;
+ }
+ minMaxFloat64x2(/*isMin=*/true, lhs, Operand(rhs), temp1, temp2, output);
+}
+
+void MacroAssemblerX86Shared::maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister temp1,
+ FloatRegister temp2,
+ FloatRegister output) {
+ if (HasAVX()) {
+ minMaxFloat64x2AVX(/*isMin=*/false, lhs, rhs, temp1, temp2, output);
+ return;
+ }
+ minMaxFloat64x2(/*isMin=*/false, lhs, Operand(rhs), temp1, temp2, output);
+}
+
+void MacroAssemblerX86Shared::packedShiftByScalarInt8x16(
+ FloatRegister in, Register count, FloatRegister xtmp, FloatRegister dest,
+ void (MacroAssemblerX86Shared::*shift)(FloatRegister, FloatRegister,
+ FloatRegister),
+ void (MacroAssemblerX86Shared::*extend)(const Operand&, FloatRegister)) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+
+ // High bytes
+ vpalignr(Operand(in), xtmp, xtmp, 8);
+ (this->*extend)(Operand(xtmp), xtmp);
+ (this->*shift)(scratch, xtmp, xtmp);
+
+ // Low bytes
+ (this->*extend)(Operand(dest), dest);
+ (this->*shift)(scratch, dest, dest);
+
+ // Mask off garbage to avoid saturation during packing
+ asMasm().loadConstantSimd128Int(SimdConstant::SplatX4(int32_t(0x00FF00FF)),
+ scratch);
+ vpand(Operand(scratch), xtmp, xtmp);
+ vpand(Operand(scratch), dest, dest);
+
+ vpackuswb(Operand(xtmp), dest, dest);
+}
+
+void MacroAssemblerX86Shared::packedLeftShiftByScalarInt8x16(
+ FloatRegister in, Register count, FloatRegister xtmp, FloatRegister dest) {
+ packedShiftByScalarInt8x16(in, count, xtmp, dest,
+ &MacroAssemblerX86Shared::vpsllw,
+ &MacroAssemblerX86Shared::vpmovzxbw);
+}
+
+void MacroAssemblerX86Shared::packedLeftShiftByScalarInt8x16(
+ Imm32 count, FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(count.value <= 7);
+ if (MOZ_UNLIKELY(count.value == 0)) {
+ moveSimd128Int(src, dest);
+ return;
+ }
+ src = asMasm().moveSimd128IntIfNotAVX(src, dest);
+ // Use the doubling trick for low shift counts, otherwise mask off the bits
+ // that are shifted out of the low byte of each word and use word shifts. The
+ // optimal cutoff remains to be explored.
+ if (count.value <= 3) {
+ vpaddb(Operand(src), src, dest);
+ for (int32_t shift = count.value - 1; shift > 0; --shift) {
+ vpaddb(Operand(dest), dest, dest);
+ }
+ } else {
+ asMasm().bitwiseAndSimd128(src, SimdConstant::SplatX16(0xFF >> count.value),
+ dest);
+ vpsllw(count, dest, dest);
+ }
+}
+
+void MacroAssemblerX86Shared::packedRightShiftByScalarInt8x16(
+ FloatRegister in, Register count, FloatRegister xtmp, FloatRegister dest) {
+ packedShiftByScalarInt8x16(in, count, xtmp, dest,
+ &MacroAssemblerX86Shared::vpsraw,
+ &MacroAssemblerX86Shared::vpmovsxbw);
+}
+
+void MacroAssemblerX86Shared::packedRightShiftByScalarInt8x16(
+ Imm32 count, FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(count.value <= 7);
+ ScratchSimd128Scope scratch(asMasm());
+
+ vpunpckhbw(src, scratch, scratch);
+ vpunpcklbw(src, dest, dest);
+ vpsraw(Imm32(count.value + 8), scratch, scratch);
+ vpsraw(Imm32(count.value + 8), dest, dest);
+ vpacksswb(Operand(scratch), dest, dest);
+}
+
+void MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt8x16(
+ FloatRegister in, Register count, FloatRegister xtmp, FloatRegister dest) {
+ packedShiftByScalarInt8x16(in, count, xtmp, dest,
+ &MacroAssemblerX86Shared::vpsrlw,
+ &MacroAssemblerX86Shared::vpmovzxbw);
+}
+
+void MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt8x16(
+ Imm32 count, FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(count.value <= 7);
+ src = asMasm().moveSimd128IntIfNotAVX(src, dest);
+ asMasm().bitwiseAndSimd128(
+ src, SimdConstant::SplatX16((0xFF << count.value) & 0xFF), dest);
+ vpsrlw(count, dest, dest);
+}
+
+void MacroAssemblerX86Shared::packedLeftShiftByScalarInt16x8(
+ FloatRegister in, Register count, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+ vpsllw(scratch, in, dest);
+}
+
+void MacroAssemblerX86Shared::packedRightShiftByScalarInt16x8(
+ FloatRegister in, Register count, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+ vpsraw(scratch, in, dest);
+}
+
+void MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt16x8(
+ FloatRegister in, Register count, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+ vpsrlw(scratch, in, dest);
+}
+
+void MacroAssemblerX86Shared::packedLeftShiftByScalarInt32x4(
+ FloatRegister in, Register count, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+ vpslld(scratch, in, dest);
+}
+
+void MacroAssemblerX86Shared::packedRightShiftByScalarInt32x4(
+ FloatRegister in, Register count, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+ vpsrad(scratch, in, dest);
+}
+
+void MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt32x4(
+ FloatRegister in, Register count, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+ vpsrld(scratch, in, dest);
+}
+
+void MacroAssemblerX86Shared::packedLeftShiftByScalarInt64x2(
+ FloatRegister in, Register count, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+ vpsllq(scratch, in, dest);
+}
+
+void MacroAssemblerX86Shared::packedRightShiftByScalarInt64x2(
+ FloatRegister in, Register count, FloatRegister temp, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, temp);
+ asMasm().signReplicationInt64x2(in, scratch);
+ in = asMasm().moveSimd128FloatIfNotAVX(in, dest);
+ // Invert if negative, shift all, invert back if negative.
+ vpxor(Operand(scratch), in, dest);
+ vpsrlq(temp, dest, dest);
+ vpxor(Operand(scratch), dest, dest);
+}
+
+void MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt64x2(
+ FloatRegister in, Register count, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ vmovd(count, scratch);
+ vpsrlq(scratch, in, dest);
+}
+
+void MacroAssemblerX86Shared::packedRightShiftByScalarInt64x2(
+ Imm32 count, FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ asMasm().signReplicationInt64x2(src, scratch);
+ // Invert if negative, shift all, invert back if negative.
+ src = asMasm().moveSimd128FloatIfNotAVX(src, dest);
+ vpxor(Operand(scratch), src, dest);
+ vpsrlq(Imm32(count.value & 63), dest, dest);
+ vpxor(Operand(scratch), dest, dest);
+}
+
+void MacroAssemblerX86Shared::selectSimd128(FloatRegister mask,
+ FloatRegister onTrue,
+ FloatRegister onFalse,
+ FloatRegister temp,
+ FloatRegister output) {
+ // Normally the codegen will attempt to enforce these register assignments so
+ // that the moves are avoided.
+
+ onTrue = asMasm().moveSimd128IntIfNotAVX(onTrue, output);
+ if (MOZ_UNLIKELY(mask == onTrue)) {
+ vpor(Operand(onFalse), onTrue, output);
+ return;
+ }
+
+ mask = asMasm().moveSimd128IntIfNotAVX(mask, temp);
+
+ vpand(Operand(mask), onTrue, output);
+ vpandn(Operand(onFalse), mask, temp);
+ vpor(Operand(temp), output, output);
+}
+
+// Code sequences for int32x4<->float32x4 culled from v8; commentary added.
+
+void MacroAssemblerX86Shared::unsignedConvertInt32x4ToFloat32x4(
+ FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ src = asMasm().moveSimd128IntIfNotAVX(src, dest);
+ vpxor(Operand(scratch), scratch, scratch); // extract low bits
+ vpblendw(0x55, src, scratch, scratch); // into scratch
+ vpsubd(Operand(scratch), src, dest); // and high bits into dest
+ vcvtdq2ps(scratch, scratch); // convert low bits
+ vpsrld(Imm32(1), dest, dest); // get high into unsigned range
+ vcvtdq2ps(dest, dest); // convert
+ vaddps(Operand(dest), dest, dest); // and back into signed
+ vaddps(Operand(scratch), dest, dest); // combine high+low: may round
+}
+
+void MacroAssemblerX86Shared::truncSatFloat32x4ToInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+
+ // The cvttps2dq instruction is the workhorse but does not handle NaN or out
+ // of range values as we need it to. We want to saturate too-large positive
+ // values to 7FFFFFFFh and too-large negative values to 80000000h. NaN and -0
+ // become 0.
+
+ // Convert NaN to 0 by masking away values that compare unordered to itself.
+ if (HasAVX()) {
+ vcmpeqps(Operand(src), src, scratch);
+ vpand(Operand(scratch), src, dest);
+ } else {
+ vmovaps(src, scratch);
+ vcmpeqps(Operand(scratch), scratch, scratch);
+ moveSimd128Float(src, dest);
+ vpand(Operand(scratch), dest, dest);
+ }
+
+ // Make lanes in scratch == 0xFFFFFFFFh, if dest overflows during cvttps2dq,
+ // otherwise 0.
+ static const SimdConstant minOverflowedInt =
+ SimdConstant::SplatX4(2147483648.f);
+ if (HasAVX()) {
+ asMasm().vcmpgepsSimd128(minOverflowedInt, dest, scratch);
+ } else {
+ asMasm().loadConstantSimd128Float(minOverflowedInt, scratch);
+ vcmpleps(Operand(dest), scratch, scratch);
+ }
+
+ // Convert. This will make the output 80000000h if the input is out of range.
+ vcvttps2dq(dest, dest);
+
+ // Convert overflow lanes to 0x7FFFFFFF.
+ vpxor(Operand(scratch), dest, dest);
+}
+
+void MacroAssemblerX86Shared::unsignedTruncSatFloat32x4ToInt32x4(
+ FloatRegister src, FloatRegister temp, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ src = asMasm().moveSimd128FloatIfNotAVX(src, dest);
+
+ // The cvttps2dq instruction is the workhorse but does not handle NaN or out
+ // of range values as we need it to. We want to saturate too-large positive
+ // values to FFFFFFFFh and negative values to zero. NaN and -0 become 0.
+
+ // Convert NaN and negative values to zeroes in dest.
+ vxorps(Operand(scratch), scratch, scratch);
+ vmaxps(Operand(scratch), src, dest);
+
+ // Place the largest positive signed integer in all lanes in scratch.
+ // We use it to bias the conversion to handle edge cases.
+ asMasm().loadConstantSimd128Float(SimdConstant::SplatX4(2147483647.f),
+ scratch);
+
+ // temp = dest - 7FFFFFFFh (as floating), this brings integers in the unsigned
+ // range but above the signed range into the signed range; 0 => -7FFFFFFFh.
+ vmovaps(dest, temp);
+ vsubps(Operand(scratch), temp, temp);
+
+ // scratch = mask of biased values that are greater than 7FFFFFFFh.
+ vcmpleps(Operand(temp), scratch, scratch);
+
+ // Convert the biased values to integer. Positive values above 7FFFFFFFh will
+ // have been converted to 80000000h, all others become the expected integer.
+ vcvttps2dq(temp, temp);
+
+ // As lanes of scratch are ~0 where the result overflows, this computes
+ // 7FFFFFFF in lanes of temp that are 80000000h, and leaves other lanes
+ // untouched as the biased integer.
+ vpxor(Operand(scratch), temp, temp);
+
+ // Convert negative biased lanes in temp to zero. After this, temp will be
+ // zero where the result should be zero or is less than 80000000h, 7FFFFFFF
+ // where the result overflows, and will have the converted biased result in
+ // other lanes (for input values >= 80000000h).
+ vpxor(Operand(scratch), scratch, scratch);
+ vpmaxsd(Operand(scratch), temp, temp);
+
+ // Convert. Overflow lanes above 7FFFFFFFh will be 80000000h, other lanes will
+ // be what they should be.
+ vcvttps2dq(dest, dest);
+
+ // Add temp to the result. Overflow lanes with 80000000h becomes FFFFFFFFh,
+ // biased high-value unsigned lanes become unbiased, everything else is left
+ // unchanged.
+ vpaddd(Operand(temp), dest, dest);
+}
+
+void MacroAssemblerX86Shared::unsignedTruncSatFloat32x4ToInt32x4Relaxed(
+ FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+ src = asMasm().moveSimd128FloatIfNotAVX(src, dest);
+
+ // Place lanes below 80000000h into dest, otherwise into scratch.
+ // Keep dest or scratch 0 as default.
+ asMasm().loadConstantSimd128Float(SimdConstant::SplatX4(0x4f000000), scratch);
+ vcmpltps(Operand(src), scratch, scratch);
+ vpand(Operand(src), scratch, scratch);
+ vpxor(Operand(scratch), src, dest);
+
+ // Convert lanes below 80000000h into unsigned int without issues.
+ vcvttps2dq(dest, dest);
+ // Knowing IEEE-754 number representation: convert lanes above 7FFFFFFFh,
+ // mutiply by 2 (to add 1 in exponent) and shift to the left by 8 bits.
+ vaddps(Operand(scratch), scratch, scratch);
+ vpslld(Imm32(8), scratch, scratch);
+
+ // Combine the results.
+ vpaddd(Operand(scratch), dest, dest);
+}
+
+void MacroAssemblerX86Shared::unsignedConvertInt32x4ToFloat64x2(
+ FloatRegister src, FloatRegister dest) {
+ src = asMasm().moveSimd128FloatIfNotAVX(src, dest);
+ asMasm().vunpcklpsSimd128(SimdConstant::SplatX4(0x43300000), src, dest);
+ asMasm().vsubpdSimd128(SimdConstant::SplatX2(4503599627370496.0), dest, dest);
+}
+
+void MacroAssemblerX86Shared::truncSatFloat64x2ToInt32x4(FloatRegister src,
+ FloatRegister temp,
+ FloatRegister dest) {
+ FloatRegister srcForTemp = asMasm().moveSimd128FloatIfNotAVX(src, temp);
+ vcmpeqpd(Operand(srcForTemp), srcForTemp, temp);
+
+ src = asMasm().moveSimd128FloatIfNotAVX(src, dest);
+ asMasm().vandpdSimd128(SimdConstant::SplatX2(2147483647.0), temp, temp);
+ vminpd(Operand(temp), src, dest);
+ vcvttpd2dq(dest, dest);
+}
+
+void MacroAssemblerX86Shared::unsignedTruncSatFloat64x2ToInt32x4(
+ FloatRegister src, FloatRegister temp, FloatRegister dest) {
+ src = asMasm().moveSimd128FloatIfNotAVX(src, dest);
+
+ vxorpd(temp, temp, temp);
+ vmaxpd(Operand(temp), src, dest);
+
+ asMasm().vminpdSimd128(SimdConstant::SplatX2(4294967295.0), dest, dest);
+ vroundpd(SSERoundingMode::Trunc, Operand(dest), dest);
+ asMasm().vaddpdSimd128(SimdConstant::SplatX2(4503599627370496.0), dest, dest);
+
+ // temp == 0
+ vshufps(0x88, temp, dest, dest);
+}
+
+void MacroAssemblerX86Shared::unsignedTruncSatFloat64x2ToInt32x4Relaxed(
+ FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(asMasm());
+
+ // The same as unsignedConvertInt32x4ToFloat64x2, but without NaN
+ // and out-of-bounds checks.
+ vroundpd(SSERoundingMode::Trunc, Operand(src), dest);
+ asMasm().loadConstantSimd128Float(SimdConstant::SplatX2(4503599627370496.0),
+ scratch);
+ vaddpd(Operand(scratch), dest, dest);
+ // The scratch has zeros in f32x4 lanes with index 0 and 2. The in-memory
+ // representantation of the splatted double contantant contains zero in its
+ // low bits.
+ vshufps(0x88, scratch, dest, dest);
+}
+
+void MacroAssemblerX86Shared::popcntInt8x16(FloatRegister src,
+ FloatRegister temp,
+ FloatRegister output) {
+ ScratchSimd128Scope scratch(asMasm());
+ asMasm().loadConstantSimd128Int(SimdConstant::SplatX16(0x0f), scratch);
+ FloatRegister srcForTemp = asMasm().moveSimd128IntIfNotAVX(src, temp);
+ vpand(scratch, srcForTemp, temp);
+ vpandn(src, scratch, scratch);
+ int8_t counts[] = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4};
+ asMasm().loadConstantSimd128(SimdConstant::CreateX16(counts), output);
+ vpsrlw(Imm32(4), scratch, scratch);
+ vpshufb(temp, output, output);
+ asMasm().loadConstantSimd128(SimdConstant::CreateX16(counts), temp);
+ vpshufb(scratch, temp, temp);
+ vpaddb(Operand(temp), output, output);
+}
+
+void MacroAssemblerX86Shared::dotBFloat16x8ThenAdd(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest,
+ FloatRegister temp) {
+ ScratchSimd128Scope scratch(asMasm());
+ FloatRegister lhsCopy = asMasm().moveSimd128IntIfNotAVX(lhs, scratch);
+ FloatRegister rhsCopy = asMasm().moveSimd128IntIfNotAVX(rhs, temp);
+ vpslld(Imm32(16), lhsCopy, scratch);
+ vpslld(Imm32(16), rhsCopy, temp);
+ if (HasFMA()) {
+ vfmadd231ps(temp, scratch, dest);
+ } else {
+ asMasm().mulFloat32x4(scratch, temp, scratch);
+ asMasm().addFloat32x4(dest, scratch, dest);
+ }
+ // The temp has 0 in low half-word. Use pblendw instead of `& 0xFFFF0000`.
+ FloatRegister tempCopy = asMasm().moveSimd128IntIfNotAVX(temp, scratch);
+ vpblendw(0xAA, lhs, tempCopy, scratch);
+ vpblendw(0xAA, rhs, temp, temp);
+ if (HasFMA()) {
+ vfmadd231ps(temp, scratch, dest);
+ } else {
+ asMasm().mulFloat32x4(scratch, temp, scratch);
+ asMasm().addFloat32x4(dest, scratch, dest);
+ }
+}
diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h b/js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h
new file mode 100644
index 0000000000..91d0327d5f
--- /dev/null
+++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h
@@ -0,0 +1,3403 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_MacroAssembler_x86_shared_inl_h
+#define jit_x86_shared_MacroAssembler_x86_shared_inl_h
+
+#include "jit/x86-shared/MacroAssembler-x86-shared.h"
+
+#include "mozilla/MathAlgorithms.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveFloat32ToGPR(FloatRegister src, Register dest) {
+ vmovd(src, dest);
+}
+
+void MacroAssembler::moveGPRToFloat32(Register src, FloatRegister dest) {
+ vmovd(src, dest);
+}
+
+void MacroAssembler::move8SignExtend(Register src, Register dest) {
+ movsbl(src, dest);
+}
+
+void MacroAssembler::move16SignExtend(Register src, Register dest) {
+ movswl(src, dest);
+}
+
+void MacroAssembler::loadAbiReturnAddress(Register dest) {
+ loadPtr(Address(getStackPointer(), 0), dest);
+}
+
+// ===============================================================
+// Logical instructions
+
+void MacroAssembler::not32(Register reg) { notl(reg); }
+
+void MacroAssembler::and32(Register src, Register dest) { andl(src, dest); }
+
+void MacroAssembler::and32(Imm32 imm, Register dest) { andl(imm, dest); }
+
+void MacroAssembler::and32(Imm32 imm, const Address& dest) {
+ andl(imm, Operand(dest));
+}
+
+void MacroAssembler::and32(const Address& src, Register dest) {
+ andl(Operand(src), dest);
+}
+
+void MacroAssembler::or32(Register src, Register dest) { orl(src, dest); }
+
+void MacroAssembler::or32(Imm32 imm, Register dest) { orl(imm, dest); }
+
+void MacroAssembler::or32(Imm32 imm, const Address& dest) {
+ orl(imm, Operand(dest));
+}
+
+void MacroAssembler::xor32(Register src, Register dest) { xorl(src, dest); }
+
+void MacroAssembler::xor32(Imm32 imm, Register dest) { xorl(imm, dest); }
+
+void MacroAssembler::xor32(Imm32 imm, const Address& dest) {
+ xorl(imm, Operand(dest));
+}
+
+void MacroAssembler::xor32(const Address& src, Register dest) {
+ xorl(Operand(src), dest);
+}
+
+void MacroAssembler::clz32(Register src, Register dest, bool knownNotZero) {
+ if (AssemblerX86Shared::HasLZCNT()) {
+ lzcntl(src, dest);
+ return;
+ }
+
+ bsrl(src, dest);
+ if (!knownNotZero) {
+ // If the source is zero then bsrl leaves garbage in the destination.
+ Label nonzero;
+ j(Assembler::NonZero, &nonzero);
+ movl(Imm32(0x3F), dest);
+ bind(&nonzero);
+ }
+ xorl(Imm32(0x1F), dest);
+}
+
+void MacroAssembler::ctz32(Register src, Register dest, bool knownNotZero) {
+ if (AssemblerX86Shared::HasBMI1()) {
+ tzcntl(src, dest);
+ return;
+ }
+
+ bsfl(src, dest);
+ if (!knownNotZero) {
+ Label nonzero;
+ j(Assembler::NonZero, &nonzero);
+ movl(Imm32(32), dest);
+ bind(&nonzero);
+ }
+}
+
+void MacroAssembler::popcnt32(Register input, Register output, Register tmp) {
+ if (AssemblerX86Shared::HasPOPCNT()) {
+ popcntl(input, output);
+ return;
+ }
+
+ MOZ_ASSERT(tmp != InvalidReg);
+
+ // Equivalent to mozilla::CountPopulation32()
+
+ movl(input, tmp);
+ if (input != output) {
+ movl(input, output);
+ }
+ shrl(Imm32(1), output);
+ andl(Imm32(0x55555555), output);
+ subl(output, tmp);
+ movl(tmp, output);
+ andl(Imm32(0x33333333), output);
+ shrl(Imm32(2), tmp);
+ andl(Imm32(0x33333333), tmp);
+ addl(output, tmp);
+ movl(tmp, output);
+ shrl(Imm32(4), output);
+ addl(tmp, output);
+ andl(Imm32(0xF0F0F0F), output);
+ imull(Imm32(0x1010101), output, output);
+ shrl(Imm32(24), output);
+}
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap16SignExtend(Register reg) {
+ rolw(Imm32(8), reg);
+ movswl(reg, reg);
+}
+
+void MacroAssembler::byteSwap16ZeroExtend(Register reg) {
+ rolw(Imm32(8), reg);
+ movzwl(reg, reg);
+}
+
+void MacroAssembler::byteSwap32(Register reg) { bswapl(reg); }
+
+// ===============================================================
+// Arithmetic instructions
+
+void MacroAssembler::add32(Register src, Register dest) { addl(src, dest); }
+
+void MacroAssembler::add32(Imm32 imm, Register dest) { addl(imm, dest); }
+
+void MacroAssembler::add32(Imm32 imm, const Address& dest) {
+ addl(imm, Operand(dest));
+}
+
+void MacroAssembler::add32(Imm32 imm, const AbsoluteAddress& dest) {
+ addl(imm, Operand(dest));
+}
+
+void MacroAssembler::addFloat32(FloatRegister src, FloatRegister dest) {
+ vaddss(src, dest, dest);
+}
+
+void MacroAssembler::addDouble(FloatRegister src, FloatRegister dest) {
+ vaddsd(src, dest, dest);
+}
+
+void MacroAssembler::sub32(Register src, Register dest) { subl(src, dest); }
+
+void MacroAssembler::sub32(Imm32 imm, Register dest) { subl(imm, dest); }
+
+void MacroAssembler::sub32(const Address& src, Register dest) {
+ subl(Operand(src), dest);
+}
+
+void MacroAssembler::subDouble(FloatRegister src, FloatRegister dest) {
+ vsubsd(src, dest, dest);
+}
+
+void MacroAssembler::subFloat32(FloatRegister src, FloatRegister dest) {
+ vsubss(src, dest, dest);
+}
+
+void MacroAssembler::mul32(Register rhs, Register srcDest) {
+ imull(rhs, srcDest);
+}
+
+void MacroAssembler::mul32(Imm32 imm, Register srcDest) { imull(imm, srcDest); }
+
+void MacroAssembler::mulFloat32(FloatRegister src, FloatRegister dest) {
+ vmulss(src, dest, dest);
+}
+
+void MacroAssembler::mulDouble(FloatRegister src, FloatRegister dest) {
+ vmulsd(src, dest, dest);
+}
+
+void MacroAssembler::quotient32(Register rhs, Register srcDest,
+ Register tempEdx, bool isUnsigned) {
+ MOZ_ASSERT(srcDest == eax && tempEdx == edx);
+
+ // Sign extend eax into edx to make (edx:eax): idiv/udiv are 64-bit.
+ if (isUnsigned) {
+ mov(ImmWord(0), edx);
+ udiv(rhs);
+ } else {
+ cdq();
+ idiv(rhs);
+ }
+}
+
+void MacroAssembler::remainder32(Register rhs, Register srcDest,
+ Register tempEdx, bool isUnsigned) {
+ MOZ_ASSERT(srcDest == eax && tempEdx == edx);
+
+ // Sign extend eax into edx to make (edx:eax): idiv/udiv are 64-bit.
+ if (isUnsigned) {
+ mov(ImmWord(0), edx);
+ udiv(rhs);
+ } else {
+ cdq();
+ idiv(rhs);
+ }
+ mov(edx, eax);
+}
+
+void MacroAssembler::divFloat32(FloatRegister src, FloatRegister dest) {
+ vdivss(src, dest, dest);
+}
+
+void MacroAssembler::divDouble(FloatRegister src, FloatRegister dest) {
+ vdivsd(src, dest, dest);
+}
+
+void MacroAssembler::neg32(Register reg) { negl(reg); }
+
+void MacroAssembler::negateFloat(FloatRegister reg) {
+ ScratchFloat32Scope scratch(*this);
+ vpcmpeqw(Operand(scratch), scratch, scratch);
+ vpsllq(Imm32(31), scratch, scratch);
+
+ // XOR the float in a float register with -0.0.
+ vxorps(scratch, reg, reg); // s ^ 0x80000000
+}
+
+void MacroAssembler::negateDouble(FloatRegister reg) {
+ // From MacroAssemblerX86Shared::maybeInlineDouble
+ ScratchDoubleScope scratch(*this);
+ vpcmpeqw(Operand(scratch), scratch, scratch);
+ vpsllq(Imm32(63), scratch, scratch);
+
+ // XOR the float in a float register with -0.0.
+ vxorpd(scratch, reg, reg); // s ^ 0x80000000000000
+}
+
+void MacroAssembler::abs32(Register src, Register dest) {
+ if (src != dest) {
+ move32(src, dest);
+ }
+ Label positive;
+ branchTest32(Assembler::NotSigned, dest, dest, &positive);
+ neg32(dest);
+ bind(&positive);
+}
+
+void MacroAssembler::absFloat32(FloatRegister src, FloatRegister dest) {
+ ScratchFloat32Scope scratch(*this);
+ loadConstantFloat32(mozilla::SpecificNaN<float>(
+ 0, mozilla::FloatingPoint<float>::kSignificandBits),
+ scratch);
+ vandps(scratch, src, dest);
+}
+
+void MacroAssembler::absDouble(FloatRegister src, FloatRegister dest) {
+ ScratchDoubleScope scratch(*this);
+ loadConstantDouble(mozilla::SpecificNaN<double>(
+ 0, mozilla::FloatingPoint<double>::kSignificandBits),
+ scratch);
+ vandpd(scratch, src, dest);
+}
+
+void MacroAssembler::sqrtFloat32(FloatRegister src, FloatRegister dest) {
+ vsqrtss(src, dest, dest);
+}
+
+void MacroAssembler::sqrtDouble(FloatRegister src, FloatRegister dest) {
+ vsqrtsd(src, dest, dest);
+}
+
+void MacroAssembler::minFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxFloat32(srcDest, other, handleNaN, false);
+}
+
+void MacroAssembler::minDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxDouble(srcDest, other, handleNaN, false);
+}
+
+void MacroAssembler::maxFloat32(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxFloat32(srcDest, other, handleNaN, true);
+}
+
+void MacroAssembler::maxDouble(FloatRegister other, FloatRegister srcDest,
+ bool handleNaN) {
+ minMaxDouble(srcDest, other, handleNaN, true);
+}
+
+// ===============================================================
+// Rotation instructions
+void MacroAssembler::rotateLeft(Imm32 count, Register input, Register dest) {
+ MOZ_ASSERT(input == dest, "defineReuseInput");
+ count.value &= 0x1f;
+ if (count.value) {
+ roll(count, input);
+ }
+}
+
+void MacroAssembler::rotateLeft(Register count, Register input, Register dest) {
+ MOZ_ASSERT(input == dest, "defineReuseInput");
+ MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+ roll_cl(input);
+}
+
+void MacroAssembler::rotateRight(Imm32 count, Register input, Register dest) {
+ MOZ_ASSERT(input == dest, "defineReuseInput");
+ count.value &= 0x1f;
+ if (count.value) {
+ rorl(count, input);
+ }
+}
+
+void MacroAssembler::rotateRight(Register count, Register input,
+ Register dest) {
+ MOZ_ASSERT(input == dest, "defineReuseInput");
+ MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+ rorl_cl(input);
+}
+
+// ===============================================================
+// Shift instructions
+
+void MacroAssembler::lshift32(Register shift, Register srcDest) {
+ if (HasBMI2()) {
+ shlxl(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == ecx);
+ shll_cl(srcDest);
+}
+
+void MacroAssembler::flexibleLshift32(Register shift, Register srcDest) {
+ if (HasBMI2()) {
+ shlxl(srcDest, shift, srcDest);
+ return;
+ }
+ if (shift == ecx) {
+ shll_cl(srcDest);
+ } else {
+ // Shift amount must be in ecx.
+ xchg(shift, ecx);
+ shll_cl(shift == srcDest ? ecx : srcDest == ecx ? shift : srcDest);
+ xchg(shift, ecx);
+ }
+}
+
+void MacroAssembler::rshift32(Register shift, Register srcDest) {
+ if (HasBMI2()) {
+ shrxl(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == ecx);
+ shrl_cl(srcDest);
+}
+
+void MacroAssembler::flexibleRshift32(Register shift, Register srcDest) {
+ if (HasBMI2()) {
+ shrxl(srcDest, shift, srcDest);
+ return;
+ }
+ if (shift == ecx) {
+ shrl_cl(srcDest);
+ } else {
+ // Shift amount must be in ecx.
+ xchg(shift, ecx);
+ shrl_cl(shift == srcDest ? ecx : srcDest == ecx ? shift : srcDest);
+ xchg(shift, ecx);
+ }
+}
+
+void MacroAssembler::rshift32Arithmetic(Register shift, Register srcDest) {
+ if (HasBMI2()) {
+ sarxl(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == ecx);
+ sarl_cl(srcDest);
+}
+
+void MacroAssembler::flexibleRshift32Arithmetic(Register shift,
+ Register srcDest) {
+ if (HasBMI2()) {
+ sarxl(srcDest, shift, srcDest);
+ return;
+ }
+ if (shift == ecx) {
+ sarl_cl(srcDest);
+ } else {
+ // Shift amount must be in ecx.
+ xchg(shift, ecx);
+ sarl_cl(shift == srcDest ? ecx : srcDest == ecx ? shift : srcDest);
+ xchg(shift, ecx);
+ }
+}
+
+void MacroAssembler::lshift32(Imm32 shift, Register srcDest) {
+ shll(shift, srcDest);
+}
+
+void MacroAssembler::rshift32(Imm32 shift, Register srcDest) {
+ shrl(shift, srcDest);
+}
+
+void MacroAssembler::rshift32Arithmetic(Imm32 shift, Register srcDest) {
+ sarl(shift, srcDest);
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp8Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ cmp8(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+void MacroAssembler::cmp16Set(Condition cond, Address lhs, Imm32 rhs,
+ Register dest) {
+ cmp16(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmp32Set(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ cmp32(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+// ===============================================================
+// Branch instructions
+
+void MacroAssembler::branch8(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ cmp8(lhs, rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch8(Condition cond, const BaseIndex& lhs, Register rhs,
+ Label* label) {
+ cmp8(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch16(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ cmp16(lhs, rhs);
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ cmp32(lhs, rhs);
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branch32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ cmp32(lhs, rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Register rhs,
+ Label* label) {
+ cmp32(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ cmp32(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs,
+ Register rhs, Label* label) {
+ cmp32(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const BaseIndex& lhs, Imm32 rhs,
+ Label* label) {
+ cmp32(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const Operand& lhs, Register rhs,
+ Label* label) {
+ cmp32(lhs, rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const Operand& lhs, Imm32 rhs,
+ Label* label) {
+ cmp32(lhs, rhs);
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ cmpPtr(lhs, rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmPtr rhs,
+ Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmGCPtr rhs,
+ Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, Register lhs, ImmWord rhs,
+ Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+template <class L>
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, Register rhs,
+ L label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmPtr rhs,
+ Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmGCPtr rhs,
+ Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const Address& lhs, ImmWord rhs,
+ Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ ImmWord rhs, Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const BaseIndex& lhs,
+ Register rhs, Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+template <typename T, typename S, typename L>
+void MacroAssembler::branchPtrImpl(Condition cond, const T& lhs, const S& rhs,
+ L label) {
+ cmpPtr(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branchFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ compareFloat(cond, lhs, rhs);
+
+ if (cond == DoubleEqual) {
+ Label unordered;
+ j(Parity, &unordered);
+ j(Equal, label);
+ bind(&unordered);
+ return;
+ }
+
+ if (cond == DoubleNotEqualOrUnordered) {
+ j(NotEqual, label);
+ j(Parity, label);
+ return;
+ }
+
+ MOZ_ASSERT(!(cond & DoubleConditionBitSpecial));
+ j(ConditionFromDoubleCondition(cond), label);
+}
+
+void MacroAssembler::branchDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs, Label* label) {
+ compareDouble(cond, lhs, rhs);
+
+ if (cond == DoubleEqual) {
+ Label unordered;
+ j(Parity, &unordered);
+ j(Equal, label);
+ bind(&unordered);
+ return;
+ }
+ if (cond == DoubleNotEqualOrUnordered) {
+ j(NotEqual, label);
+ j(Parity, label);
+ return;
+ }
+
+ MOZ_ASSERT(!(cond & DoubleConditionBitSpecial));
+ j(ConditionFromDoubleCondition(cond), label);
+}
+
+template <typename T>
+void MacroAssembler::branchAdd32(Condition cond, T src, Register dest,
+ Label* label) {
+ addl(src, dest);
+ j(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchSub32(Condition cond, T src, Register dest,
+ Label* label) {
+ subl(src, dest);
+ j(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchMul32(Condition cond, T src, Register dest,
+ Label* label) {
+ mul32(src, dest);
+ j(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchRshift32(Condition cond, T src, Register dest,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero);
+ rshift32(src, dest);
+ j(cond, label);
+}
+
+void MacroAssembler::branchNeg32(Condition cond, Register reg, Label* label) {
+ MOZ_ASSERT(cond == Overflow);
+ neg32(reg);
+ j(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchAddPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ addPtr(src, dest);
+ j(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::branchSubPtr(Condition cond, T src, Register dest,
+ Label* label) {
+ subPtr(src, dest);
+ j(cond, label);
+}
+
+void MacroAssembler::branchMulPtr(Condition cond, Register src, Register dest,
+ Label* label) {
+ mulPtr(src, dest);
+ j(cond, label);
+}
+
+void MacroAssembler::decBranchPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ subPtr(rhs, lhs);
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Register rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ test32(lhs, rhs);
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest32(Condition cond, Register lhs, Imm32 rhs,
+ L label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ test32(lhs, rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const Address& lhs, Imm32 rhs,
+ Label* label) {
+ MOZ_ASSERT(cond == Zero || cond == NonZero || cond == Signed ||
+ cond == NotSigned);
+ test32(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Register rhs,
+ L label) {
+ testPtr(lhs, rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, Register lhs, Imm32 rhs,
+ Label* label) {
+ testPtr(lhs, rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestPtr(Condition cond, const Address& lhs,
+ Imm32 rhs, Label* label) {
+ testPtr(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, Register tag,
+ Label* label) {
+ branchTestUndefinedImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond, const Address& address,
+ Label* label) {
+ branchTestUndefinedImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const BaseIndex& address,
+ Label* label) {
+ branchTestUndefinedImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestUndefined(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestUndefinedImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestUndefinedImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testUndefined(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, Register tag,
+ Label* label) {
+ branchTestInt32Impl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const Address& address,
+ Label* label) {
+ branchTestInt32Impl(cond, address, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestInt32Impl(cond, address, label);
+}
+
+void MacroAssembler::branchTestInt32(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestInt32Impl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestInt32Impl(Condition cond, const T& t,
+ Label* label) {
+ cond = testInt32(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestInt32Truthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition cond = testInt32Truthy(truthy, value);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, Register tag,
+ Label* label) {
+ branchTestDoubleImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const Address& address,
+ Label* label) {
+ branchTestDoubleImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestDoubleImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestDouble(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestDoubleImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestDoubleImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testDouble(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestDoubleTruthy(bool truthy, FloatRegister reg,
+ Label* label) {
+ Condition cond = testDoubleTruthy(truthy, reg);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, Register tag,
+ Label* label) {
+ branchTestNumberImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNumber(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestNumberImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestNumberImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testNumber(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, Register tag,
+ Label* label) {
+ branchTestBooleanImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const Address& address,
+ Label* label) {
+ branchTestBooleanImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestBooleanImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBoolean(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestBooleanImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestBooleanImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testBoolean(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, Register tag,
+ Label* label) {
+ branchTestStringImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const Address& address,
+ Label* label) {
+ branchTestStringImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestStringImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestString(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestStringImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestStringImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testString(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestStringTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition cond = testStringTruthy(truthy, value);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, Register tag,
+ Label* label) {
+ branchTestSymbolImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const Address& address,
+ Label* label) {
+ branchTestSymbolImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestSymbolImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestSymbol(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestSymbolImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestSymbolImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testSymbol(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, Register tag,
+ Label* label) {
+ branchTestBigIntImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const Address& address,
+ Label* label) {
+ branchTestBigIntImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestBigIntImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestBigInt(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestBigIntImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestBigIntImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testBigInt(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestBigIntTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ Condition cond = testBigIntTruthy(truthy, value);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, Register tag,
+ Label* label) {
+ branchTestNullImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const Address& address,
+ Label* label) {
+ branchTestNullImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestNullImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestNull(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestNullImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestNullImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testNull(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, Register tag,
+ Label* label) {
+ branchTestObjectImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const Address& address,
+ Label* label) {
+ branchTestObjectImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestObjectImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestObject(Condition cond, const ValueOperand& value,
+ Label* label) {
+ branchTestObjectImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestObjectImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testObject(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const Address& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestGCThingImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestGCThing(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestGCThingImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestGCThingImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testGCThing(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond, Register tag,
+ Label* label) {
+ branchTestPrimitiveImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestPrimitive(Condition cond,
+ const ValueOperand& value,
+ Label* label) {
+ branchTestPrimitiveImpl(cond, value, label);
+}
+
+template <typename T>
+void MacroAssembler::branchTestPrimitiveImpl(Condition cond, const T& t,
+ Label* label) {
+ cond = testPrimitive(cond, t);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, Register tag,
+ Label* label) {
+ branchTestMagicImpl(cond, tag, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& address,
+ Label* label) {
+ branchTestMagicImpl(cond, address, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const BaseIndex& address,
+ Label* label) {
+ branchTestMagicImpl(cond, address, label);
+}
+
+template <class L>
+void MacroAssembler::branchTestMagic(Condition cond, const ValueOperand& value,
+ L label) {
+ branchTestMagicImpl(cond, value, label);
+}
+
+template <typename T, class L>
+void MacroAssembler::branchTestMagicImpl(Condition cond, const T& t, L label) {
+ cond = testMagic(cond, t);
+ j(cond, label);
+}
+
+template <typename T>
+void MacroAssembler::testNumberSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testNumber(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testBooleanSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testBoolean(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testStringSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testString(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testSymbolSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testSymbol(cond, src);
+ emitSet(cond, dest);
+}
+
+template <typename T>
+void MacroAssembler::testBigIntSet(Condition cond, const T& src,
+ Register dest) {
+ cond = testBigInt(cond, src);
+ emitSet(cond, dest);
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmp32(lhs, rhs);
+ cmovCCl(cond, src, dest);
+}
+
+void MacroAssembler::cmp32Move32(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ cmp32(lhs, Operand(rhs));
+ cmovCCl(cond, src, dest);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs,
+ const Address& rhs, const Address& src,
+ Register dest) {
+ cmp32(lhs, Operand(rhs));
+ cmovCCl(cond, Operand(src), dest);
+}
+
+void MacroAssembler::cmp32Load32(Condition cond, Register lhs, Register rhs,
+ const Address& src, Register dest) {
+ cmp32(lhs, rhs);
+ cmovCCl(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreZeroRegister(Condition cond, Register scratch,
+ Register dest) {
+ // Note: use movl instead of move32/xorl to ensure flags are not clobbered.
+ movl(Imm32(0), scratch);
+ spectreMovePtr(cond, scratch, dest);
+}
+
+// ========================================================================
+// Memory access primitives.
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const Address& dest) {
+ vmovsd(src, dest);
+}
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const BaseIndex& dest) {
+ vmovsd(src, dest);
+}
+void MacroAssembler::storeUncanonicalizedDouble(FloatRegister src,
+ const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ storeUncanonicalizedDouble(src, dest.toAddress());
+ break;
+ case Operand::MEM_SCALE:
+ storeUncanonicalizedDouble(src, dest.toBaseIndex());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+}
+
+template void MacroAssembler::storeDouble(FloatRegister src,
+ const Operand& dest);
+
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const Address& dest) {
+ vmovss(src, dest);
+}
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const BaseIndex& dest) {
+ vmovss(src, dest);
+}
+void MacroAssembler::storeUncanonicalizedFloat32(FloatRegister src,
+ const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ storeUncanonicalizedFloat32(src, dest.toAddress());
+ break;
+ case Operand::MEM_SCALE:
+ storeUncanonicalizedFloat32(src, dest.toBaseIndex());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+}
+
+template void MacroAssembler::storeFloat32(FloatRegister src,
+ const Operand& dest);
+
+void MacroAssembler::memoryBarrier(MemoryBarrierBits barrier) {
+ if (barrier & MembarStoreLoad) {
+ storeLoadFence();
+ }
+}
+
+// ========================================================================
+// Wasm SIMD
+//
+// Some parts of the masm API are currently agnostic as to the data's
+// interpretation as int or float, despite the Intel architecture having
+// separate functional units and sometimes penalizing type-specific instructions
+// that operate on data in the "wrong" unit.
+//
+// For the time being, we always choose the integer interpretation when we are
+// forced to choose blind, but whether that is right or wrong depends on the
+// application. This applies to moveSimd128, loadConstantSimd128,
+// loadUnalignedSimd128, and storeUnalignedSimd128, at least.
+//
+// SSE4.1 or better is assumed.
+//
+// The order of operations here follows the header file.
+
+// Moves. See comments above regarding integer operation.
+
+void MacroAssembler::moveSimd128(FloatRegister src, FloatRegister dest) {
+ MacroAssemblerX86Shared::moveSimd128Int(src, dest);
+}
+
+// Constants. See comments above regarding integer operation.
+
+void MacroAssembler::loadConstantSimd128(const SimdConstant& v,
+ FloatRegister dest) {
+ if (v.isFloatingType()) {
+ loadConstantSimd128Float(v, dest);
+ } else {
+ loadConstantSimd128Int(v, dest);
+ }
+}
+
+// Splat
+
+void MacroAssembler::splatX16(Register src, FloatRegister dest) {
+ MacroAssemblerX86Shared::splatX16(src, dest);
+}
+
+void MacroAssembler::splatX8(Register src, FloatRegister dest) {
+ MacroAssemblerX86Shared::splatX8(src, dest);
+}
+
+void MacroAssembler::splatX4(Register src, FloatRegister dest) {
+ MacroAssemblerX86Shared::splatX4(src, dest);
+}
+
+void MacroAssembler::splatX4(FloatRegister src, FloatRegister dest) {
+ MacroAssemblerX86Shared::splatX4(src, dest);
+}
+
+void MacroAssembler::splatX2(FloatRegister src, FloatRegister dest) {
+ MacroAssemblerX86Shared::splatX2(src, dest);
+}
+
+// Extract lane as scalar
+
+void MacroAssembler::extractLaneInt8x16(uint32_t lane, FloatRegister src,
+ Register dest) {
+ MacroAssemblerX86Shared::extractLaneInt8x16(src, dest, lane,
+ SimdSign::Signed);
+}
+
+void MacroAssembler::unsignedExtractLaneInt8x16(uint32_t lane,
+ FloatRegister src,
+ Register dest) {
+ MacroAssemblerX86Shared::extractLaneInt8x16(src, dest, lane,
+ SimdSign::Unsigned);
+}
+
+void MacroAssembler::extractLaneInt16x8(uint32_t lane, FloatRegister src,
+ Register dest) {
+ MacroAssemblerX86Shared::extractLaneInt16x8(src, dest, lane,
+ SimdSign::Signed);
+}
+
+void MacroAssembler::unsignedExtractLaneInt16x8(uint32_t lane,
+ FloatRegister src,
+ Register dest) {
+ MacroAssemblerX86Shared::extractLaneInt16x8(src, dest, lane,
+ SimdSign::Unsigned);
+}
+
+void MacroAssembler::extractLaneInt32x4(uint32_t lane, FloatRegister src,
+ Register dest) {
+ MacroAssemblerX86Shared::extractLaneInt32x4(src, dest, lane);
+}
+
+void MacroAssembler::extractLaneFloat32x4(uint32_t lane, FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::extractLaneFloat32x4(src, dest, lane);
+}
+
+void MacroAssembler::extractLaneFloat64x2(uint32_t lane, FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::extractLaneFloat64x2(src, dest, lane);
+}
+
+// Replace lane value
+
+void MacroAssembler::replaceLaneInt8x16(unsigned lane, FloatRegister lhs,
+ Register rhs, FloatRegister dest) {
+ vpinsrb(lane, Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::replaceLaneInt8x16(unsigned lane, Register rhs,
+ FloatRegister lhsDest) {
+ vpinsrb(lane, Operand(rhs), lhsDest, lhsDest);
+}
+
+void MacroAssembler::replaceLaneInt16x8(unsigned lane, FloatRegister lhs,
+ Register rhs, FloatRegister dest) {
+ vpinsrw(lane, Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::replaceLaneInt16x8(unsigned lane, Register rhs,
+ FloatRegister lhsDest) {
+ vpinsrw(lane, Operand(rhs), lhsDest, lhsDest);
+}
+
+void MacroAssembler::replaceLaneInt32x4(unsigned lane, FloatRegister lhs,
+ Register rhs, FloatRegister dest) {
+ vpinsrd(lane, rhs, lhs, dest);
+}
+
+void MacroAssembler::replaceLaneInt32x4(unsigned lane, Register rhs,
+ FloatRegister lhsDest) {
+ vpinsrd(lane, rhs, lhsDest, lhsDest);
+}
+
+void MacroAssembler::replaceLaneFloat32x4(unsigned lane, FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::replaceLaneFloat32x4(lane, lhs, rhs, dest);
+}
+
+void MacroAssembler::replaceLaneFloat32x4(unsigned lane, FloatRegister rhs,
+ FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::replaceLaneFloat32x4(lane, lhsDest, rhs, lhsDest);
+}
+
+void MacroAssembler::replaceLaneFloat64x2(unsigned lane, FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::replaceLaneFloat64x2(lane, lhs, rhs, dest);
+}
+
+void MacroAssembler::replaceLaneFloat64x2(unsigned lane, FloatRegister rhs,
+ FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::replaceLaneFloat64x2(lane, lhsDest, rhs, lhsDest);
+}
+
+// Shuffle - permute with immediate indices
+
+void MacroAssembler::shuffleInt8x16(const uint8_t lanes[16], FloatRegister rhs,
+ FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::shuffleInt8x16(lhsDest, rhs, lhsDest, lanes);
+}
+
+void MacroAssembler::shuffleInt8x16(const uint8_t lanes[16], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest) {
+ MacroAssemblerX86Shared::shuffleInt8x16(lhs, rhs, dest, lanes);
+}
+
+void MacroAssembler::blendInt8x16(const uint8_t lanes[16], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::blendInt8x16(lhs, rhs, dest, temp, lanes);
+}
+
+void MacroAssembler::blendInt16x8(const uint16_t lanes[8], FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest) {
+ MacroAssemblerX86Shared::blendInt16x8(lhs, rhs, dest, lanes);
+}
+
+void MacroAssembler::laneSelectSimd128(FloatRegister mask, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister dest) {
+ MacroAssemblerX86Shared::laneSelectSimd128(mask, lhs, rhs, dest);
+}
+
+void MacroAssembler::interleaveHighInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpunpckhwd(rhs, lhs, dest);
+}
+
+void MacroAssembler::interleaveHighInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpunpckhdq(rhs, lhs, dest);
+}
+
+void MacroAssembler::interleaveHighInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpunpckhqdq(rhs, lhs, dest);
+}
+
+void MacroAssembler::interleaveHighInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpunpckhbw(rhs, lhs, dest);
+}
+
+void MacroAssembler::interleaveLowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpunpcklwd(rhs, lhs, dest);
+}
+
+void MacroAssembler::interleaveLowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpunpckldq(rhs, lhs, dest);
+}
+
+void MacroAssembler::interleaveLowInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpunpcklqdq(rhs, lhs, dest);
+}
+
+void MacroAssembler::interleaveLowInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpunpcklbw(rhs, lhs, dest);
+}
+
+void MacroAssembler::permuteInt8x16(const uint8_t lanes[16], FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpshufbSimd128(SimdConstant::CreateX16((const int8_t*)lanes), src, dest);
+}
+
+void MacroAssembler::permuteLowInt16x8(const uint16_t lanes[4],
+ FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(lanes[0] < 4 && lanes[1] < 4 && lanes[2] < 4 && lanes[3] < 4);
+ vpshuflw(ComputeShuffleMask(lanes[0], lanes[1], lanes[2], lanes[3]), src,
+ dest);
+}
+
+void MacroAssembler::permuteHighInt16x8(const uint16_t lanes[4],
+ FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(lanes[0] < 4 && lanes[1] < 4 && lanes[2] < 4 && lanes[3] < 4);
+ vpshufhw(ComputeShuffleMask(lanes[0], lanes[1], lanes[2], lanes[3]), src,
+ dest);
+}
+
+void MacroAssembler::permuteInt32x4(const uint32_t lanes[4], FloatRegister src,
+ FloatRegister dest) {
+ vpshufd(ComputeShuffleMask(lanes[0], lanes[1], lanes[2], lanes[3]), src,
+ dest);
+}
+
+void MacroAssembler::concatAndRightShiftSimd128(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest,
+ uint32_t shift) {
+ vpalignr(Operand(rhs), lhs, dest, shift);
+}
+
+void MacroAssembler::leftShiftSimd128(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpslldq(count, src, dest);
+}
+
+void MacroAssembler::rightShiftSimd128(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsrldq(count, src, dest);
+}
+
+// Reverse bytes in lanes.
+
+void MacroAssembler::reverseInt16x8(FloatRegister src, FloatRegister dest) {
+ // Byteswap is MOV + PSLLW + PSRLW + POR, a small win over PSHUFB.
+ ScratchSimd128Scope scratch(*this);
+ FloatRegister srcForScratch = moveSimd128IntIfNotAVX(src, scratch);
+ vpsrlw(Imm32(8), srcForScratch, scratch);
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsllw(Imm32(8), src, dest);
+ vpor(scratch, dest, dest);
+}
+
+void MacroAssembler::reverseInt32x4(FloatRegister src, FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ int8_t lanes[] = {3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12};
+ vpshufbSimd128(SimdConstant::CreateX16((const int8_t*)lanes), src, dest);
+}
+
+void MacroAssembler::reverseInt64x2(FloatRegister src, FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ int8_t lanes[] = {7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8};
+ vpshufbSimd128(SimdConstant::CreateX16((const int8_t*)lanes), src, dest);
+}
+
+// Any lane true, ie any bit set
+
+void MacroAssembler::anyTrueSimd128(FloatRegister src, Register dest) {
+ vptest(src, src);
+ emitSetRegisterIf(Condition::NonZero, dest);
+}
+
+// All lanes true
+
+void MacroAssembler::allTrueInt8x16(FloatRegister src, Register dest) {
+ ScratchSimd128Scope xtmp(*this);
+ // xtmp is all-00h
+ vpxor(xtmp, xtmp, xtmp);
+ // Set FFh if byte==0 otherwise 00h
+ // Operand ordering constraint: lhs==output
+ vpcmpeqb(Operand(src), xtmp, xtmp);
+ // Check if xtmp is 0.
+ vptest(xtmp, xtmp);
+ emitSetRegisterIf(Condition::Zero, dest);
+}
+
+void MacroAssembler::allTrueInt16x8(FloatRegister src, Register dest) {
+ ScratchSimd128Scope xtmp(*this);
+ // xtmp is all-00h
+ vpxor(xtmp, xtmp, xtmp);
+ // Set FFFFh if word==0 otherwise 0000h
+ // Operand ordering constraint: lhs==output
+ vpcmpeqw(Operand(src), xtmp, xtmp);
+ // Check if xtmp is 0.
+ vptest(xtmp, xtmp);
+ emitSetRegisterIf(Condition::Zero, dest);
+}
+
+void MacroAssembler::allTrueInt32x4(FloatRegister src, Register dest) {
+ ScratchSimd128Scope xtmp(*this);
+ // xtmp is all-00h
+ vpxor(xtmp, xtmp, xtmp);
+ // Set FFFFFFFFh if doubleword==0 otherwise 00000000h
+ // Operand ordering constraint: lhs==output
+ vpcmpeqd(Operand(src), xtmp, xtmp);
+ // Check if xtmp is 0.
+ vptest(xtmp, xtmp);
+ emitSetRegisterIf(Condition::Zero, dest);
+}
+
+void MacroAssembler::allTrueInt64x2(FloatRegister src, Register dest) {
+ ScratchSimd128Scope xtmp(*this);
+ // xtmp is all-00h
+ vpxor(xtmp, xtmp, xtmp);
+ // Set FFFFFFFFFFFFFFFFh if quadword==0 otherwise 0000000000000000h
+ // Operand ordering constraint: lhs==output
+ vpcmpeqq(Operand(src), xtmp, xtmp);
+ // Check if xtmp is 0.
+ vptest(xtmp, xtmp);
+ emitSetRegisterIf(Condition::Zero, dest);
+}
+
+// Bitmask
+
+void MacroAssembler::bitmaskInt8x16(FloatRegister src, Register dest) {
+ vpmovmskb(src, dest);
+}
+
+void MacroAssembler::bitmaskInt16x8(FloatRegister src, Register dest) {
+ ScratchSimd128Scope scratch(*this);
+ // A three-instruction sequence is possible by using scratch as a don't-care
+ // input and shifting rather than masking at the end, but creates a false
+ // dependency on the old value of scratch. The better fix is to allow src to
+ // be clobbered.
+ src = moveSimd128IntIfNotAVX(src, scratch);
+ vpacksswb(Operand(src), src, scratch);
+ vpmovmskb(scratch, dest);
+ andl(Imm32(0xFF), dest);
+}
+
+void MacroAssembler::bitmaskInt32x4(FloatRegister src, Register dest) {
+ vmovmskps(src, dest);
+}
+
+void MacroAssembler::bitmaskInt64x2(FloatRegister src, Register dest) {
+ vmovmskpd(src, dest);
+}
+
+// Swizzle - permute with variable indices
+
+void MacroAssembler::swizzleInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ rhs = moveSimd128IntIfNotAVX(rhs, scratch);
+ // Set high bit to 1 for values > 15 via adding with saturation.
+ vpaddusbSimd128(SimdConstant::SplatX16(0x70), rhs, scratch);
+ vpshufb(scratch, lhs, dest); // permute
+}
+
+void MacroAssembler::swizzleInt8x16Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpshufb(rhs, lhs, dest);
+}
+
+// Integer Add
+
+void MacroAssembler::addInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpaddb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::addInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpaddb,
+ &MacroAssembler::vpaddbSimd128);
+}
+
+void MacroAssembler::addInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpaddw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::addInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpaddw,
+ &MacroAssembler::vpaddwSimd128);
+}
+
+void MacroAssembler::addInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpaddd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::addInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpaddd,
+ &MacroAssembler::vpadddSimd128);
+}
+
+void MacroAssembler::addInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpaddq(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::addInt64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpaddq,
+ &MacroAssembler::vpaddqSimd128);
+}
+
+// Integer subtract
+
+void MacroAssembler::subInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpsubb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::subInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpsubb,
+ &MacroAssembler::vpsubbSimd128);
+}
+
+void MacroAssembler::subInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpsubw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::subInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpsubw,
+ &MacroAssembler::vpsubwSimd128);
+}
+
+void MacroAssembler::subInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpsubd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::subInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpsubd,
+ &MacroAssembler::vpsubdSimd128);
+}
+
+void MacroAssembler::subInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpsubq(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::subInt64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpsubq,
+ &MacroAssembler::vpsubqSimd128);
+}
+
+// Integer multiply
+
+void MacroAssembler::mulInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmullw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::mulInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmullw,
+ &MacroAssembler::vpmullwSimd128);
+}
+
+void MacroAssembler::mulInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmulld(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::mulInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmulld,
+ &MacroAssembler::vpmulldSimd128);
+}
+
+void MacroAssembler::mulInt64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp) {
+ ScratchSimd128Scope temp2(*this);
+ // lhs = <D C> <B A>
+ // rhs = <H G> <F E>
+ // result = <(DG+CH)_low+CG_high CG_low> <(BE+AF)_low+AE_high AE_low>
+ FloatRegister lhsForTemp =
+ moveSimd128IntIfNotAVX(lhs, temp); // temp = <D C> <B A>
+ vpsrlq(Imm32(32), lhsForTemp, temp); // temp = <0 D> <0 B>
+ vpmuludq(rhs, temp, temp); // temp = <DG> <BE>
+ FloatRegister rhsForTemp =
+ moveSimd128IntIfNotAVX(rhs, temp2); // temp2 = <H G> <F E>
+ vpsrlq(Imm32(32), rhsForTemp, temp2); // temp2 = <0 H> <0 F>
+ vpmuludq(lhs, temp2, temp2); // temp2 = <CH> <AF>
+ vpaddq(Operand(temp), temp2, temp2); // temp2 = <DG+CH> <BE+AF>
+ vpsllq(Imm32(32), temp2, temp2); // temp2 = <(DG+CH)_low 0>
+ // <(BE+AF)_low 0>
+ vpmuludq(rhs, lhs, dest); // dest = <CG_high CG_low>
+ // <AE_high AE_low>
+ vpaddq(Operand(temp2), dest, dest); // dest =
+ // <(DG+CH)_low+CG_high CG_low>
+ // <(BE+AF)_low+AE_high AE_low>
+}
+
+void MacroAssembler::mulInt64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest, FloatRegister temp) {
+ // Check if we can specialize that to less than eight instructions
+ // (in comparison with the above mulInt64x2 version).
+ const int64_t* c = static_cast<const int64_t*>(rhs.bytes());
+ const int64_t val = c[0];
+ if (val == c[1]) {
+ switch (mozilla::CountPopulation64(val)) {
+ case 0: // val == 0
+ vpxor(Operand(dest), dest, dest);
+ return;
+ case 64: // val == -1
+ negInt64x2(lhs, dest);
+ return;
+ case 1: // val == power of 2
+ if (val == 1) {
+ moveSimd128Int(lhs, dest);
+ } else {
+ lhs = moveSimd128IntIfNotAVX(lhs, dest);
+ vpsllq(Imm32(mozilla::CountTrailingZeroes64(val)), lhs, dest);
+ }
+ return;
+ case 2: {
+ // Constants with 2 bits set, such as 3, 5, 10, etc.
+ int i0 = mozilla::CountTrailingZeroes64(val);
+ int i1 = mozilla::CountTrailingZeroes64(val & (val - 1));
+ FloatRegister lhsForTemp = moveSimd128IntIfNotAVX(lhs, temp);
+ vpsllq(Imm32(i1), lhsForTemp, temp);
+ lhs = moveSimd128IntIfNotAVX(lhs, dest);
+ if (i0 > 0) {
+ vpsllq(Imm32(i0), lhs, dest);
+ lhs = dest;
+ }
+ vpaddq(Operand(temp), lhs, dest);
+ return;
+ }
+ case 63: {
+ // Some constants with 1 bit unset, such as -2, -3, -5, etc.
+ FloatRegister lhsForTemp = moveSimd128IntIfNotAVX(lhs, temp);
+ vpsllq(Imm32(mozilla::CountTrailingZeroes64(~val)), lhsForTemp, temp);
+ negInt64x2(lhs, dest);
+ vpsubq(Operand(temp), dest, dest);
+ return;
+ }
+ }
+ }
+
+ // lhs = <D C> <B A>
+ // rhs = <H G> <F E>
+ // result = <(DG+CH)_low+CG_high CG_low> <(BE+AF)_low+AE_high AE_low>
+
+ if ((c[0] >> 32) == 0 && (c[1] >> 32) == 0) {
+ // If the H and F == 0, simplify calculations:
+ // result = <DG_low+CG_high CG_low> <BE_low+AE_high AE_low>
+ const int64_t rhsShifted[2] = {c[0] << 32, c[1] << 32};
+ FloatRegister lhsForTemp = moveSimd128IntIfNotAVX(lhs, temp);
+ vpmulldSimd128(SimdConstant::CreateSimd128(rhsShifted), lhsForTemp, temp);
+ vpmuludqSimd128(rhs, lhs, dest);
+ vpaddq(Operand(temp), dest, dest);
+ return;
+ }
+
+ const int64_t rhsSwapped[2] = {
+ static_cast<int64_t>(static_cast<uint64_t>(c[0]) >> 32) | (c[0] << 32),
+ static_cast<int64_t>(static_cast<uint64_t>(c[1]) >> 32) | (c[1] << 32),
+ }; // rhsSwapped = <G H> <E F>
+ FloatRegister lhsForTemp = moveSimd128IntIfNotAVX(lhs, temp);
+ vpmulldSimd128(SimdConstant::CreateSimd128(rhsSwapped), lhsForTemp,
+ temp); // temp = <DG CH> <BE AF>
+ vphaddd(Operand(temp), temp, temp); // temp = <xx xx> <DG+CH BE+AF>
+ vpmovzxdq(Operand(temp), temp); // temp = <0 DG+CG> <0 BE+AF>
+ vpmuludqSimd128(rhs, lhs, dest); // dest = <CG_high CG_low>
+ // <AE_high AE_low>
+ vpsllq(Imm32(32), temp, temp); // temp = <(DG+CH)_low 0>
+ // <(BE+AF)_low 0>
+ vpaddq(Operand(temp), dest, dest);
+}
+
+// Code generation from the PR: https://github.com/WebAssembly/simd/pull/376.
+// The double PSHUFD for the 32->64 case is not great, and there's some
+// discussion on the PR (scroll down far enough) on how to avoid one of them,
+// but we need benchmarking + correctness proofs.
+
+void MacroAssembler::extMulLowInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ widenLowInt8x16(rhs, scratch);
+ widenLowInt8x16(lhs, dest);
+ mulInt16x8(dest, scratch, dest);
+}
+
+void MacroAssembler::extMulHighInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ widenHighInt8x16(rhs, scratch);
+ widenHighInt8x16(lhs, dest);
+ mulInt16x8(dest, scratch, dest);
+}
+
+void MacroAssembler::unsignedExtMulLowInt8x16(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ unsignedWidenLowInt8x16(rhs, scratch);
+ unsignedWidenLowInt8x16(lhs, dest);
+ mulInt16x8(dest, scratch, dest);
+}
+
+void MacroAssembler::unsignedExtMulHighInt8x16(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ unsignedWidenHighInt8x16(rhs, scratch);
+ unsignedWidenHighInt8x16(lhs, dest);
+ mulInt16x8(dest, scratch, dest);
+}
+
+void MacroAssembler::extMulLowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ FloatRegister lhsCopy = moveSimd128IntIfNotAVX(lhs, scratch);
+ vpmulhw(Operand(rhs), lhsCopy, scratch);
+ vpmullw(Operand(rhs), lhs, dest);
+ vpunpcklwd(scratch, dest, dest);
+}
+
+void MacroAssembler::extMulHighInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ FloatRegister lhsCopy = moveSimd128IntIfNotAVX(lhs, scratch);
+ vpmulhw(Operand(rhs), lhsCopy, scratch);
+ vpmullw(Operand(rhs), lhs, dest);
+ vpunpckhwd(scratch, dest, dest);
+}
+
+void MacroAssembler::unsignedExtMulLowInt16x8(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ FloatRegister lhsCopy = moveSimd128IntIfNotAVX(lhs, scratch);
+ vpmulhuw(Operand(rhs), lhsCopy, scratch);
+ vpmullw(Operand(rhs), lhs, dest);
+ vpunpcklwd(scratch, dest, dest);
+}
+
+void MacroAssembler::unsignedExtMulHighInt16x8(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ FloatRegister lhsCopy = moveSimd128IntIfNotAVX(lhs, scratch);
+ vpmulhuw(Operand(rhs), lhsCopy, scratch);
+ vpmullw(Operand(rhs), lhs, dest);
+ vpunpckhwd(scratch, dest, dest);
+}
+
+void MacroAssembler::extMulLowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ vpshufd(ComputeShuffleMask(0, 0, 1, 0), lhs, scratch);
+ vpshufd(ComputeShuffleMask(0, 0, 1, 0), rhs, dest);
+ vpmuldq(scratch, dest, dest);
+}
+
+void MacroAssembler::extMulHighInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ vpshufd(ComputeShuffleMask(2, 0, 3, 0), lhs, scratch);
+ vpshufd(ComputeShuffleMask(2, 0, 3, 0), rhs, dest);
+ vpmuldq(scratch, dest, dest);
+}
+
+void MacroAssembler::unsignedExtMulLowInt32x4(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ vpshufd(ComputeShuffleMask(0, 0, 1, 0), lhs, scratch);
+ vpshufd(ComputeShuffleMask(0, 0, 1, 0), rhs, dest);
+ vpmuludq(Operand(scratch), dest, dest);
+}
+
+void MacroAssembler::unsignedExtMulHighInt32x4(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ vpshufd(ComputeShuffleMask(2, 0, 3, 0), lhs, scratch);
+ vpshufd(ComputeShuffleMask(2, 0, 3, 0), rhs, dest);
+ vpmuludq(Operand(scratch), dest, dest);
+}
+
+void MacroAssembler::q15MulrSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ vpmulhrsw(Operand(rhs), lhs, dest);
+ FloatRegister destCopy = moveSimd128IntIfNotAVX(dest, scratch);
+ vpcmpeqwSimd128(SimdConstant::SplatX8(0x8000), destCopy, scratch);
+ vpxor(scratch, dest, dest);
+}
+
+void MacroAssembler::q15MulrInt16x8Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmulhrsw(Operand(rhs), lhs, dest);
+}
+
+// Integer negate
+
+void MacroAssembler::negInt8x16(FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (src == dest) {
+ moveSimd128Int(src, scratch);
+ src = scratch;
+ }
+ vpxor(Operand(dest), dest, dest);
+ vpsubb(Operand(src), dest, dest);
+}
+
+void MacroAssembler::negInt16x8(FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (src == dest) {
+ moveSimd128Int(src, scratch);
+ src = scratch;
+ }
+ vpxor(Operand(dest), dest, dest);
+ vpsubw(Operand(src), dest, dest);
+}
+
+void MacroAssembler::negInt32x4(FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (src == dest) {
+ moveSimd128Int(src, scratch);
+ src = scratch;
+ }
+ vpxor(Operand(dest), dest, dest);
+ vpsubd(Operand(src), dest, dest);
+}
+
+void MacroAssembler::negInt64x2(FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (src == dest) {
+ moveSimd128Int(src, scratch);
+ src = scratch;
+ }
+ vpxor(Operand(dest), dest, dest);
+ vpsubq(Operand(src), dest, dest);
+}
+
+// Saturating integer add
+
+void MacroAssembler::addSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpaddsb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::addSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpaddsb,
+ &MacroAssembler::vpaddsbSimd128);
+}
+
+void MacroAssembler::unsignedAddSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpaddusb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedAddSatInt8x16(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpaddusb,
+ &MacroAssembler::vpaddusbSimd128);
+}
+
+void MacroAssembler::addSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpaddsw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::addSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpaddsw,
+ &MacroAssembler::vpaddswSimd128);
+}
+
+void MacroAssembler::unsignedAddSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpaddusw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedAddSatInt16x8(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpaddusw,
+ &MacroAssembler::vpadduswSimd128);
+}
+
+// Saturating integer subtract
+
+void MacroAssembler::subSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpsubsb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::subSatInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpsubsb,
+ &MacroAssembler::vpsubsbSimd128);
+}
+
+void MacroAssembler::unsignedSubSatInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpsubusb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedSubSatInt8x16(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpsubusb,
+ &MacroAssembler::vpsubusbSimd128);
+}
+
+void MacroAssembler::subSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpsubsw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::subSatInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpsubsw,
+ &MacroAssembler::vpsubswSimd128);
+}
+
+void MacroAssembler::unsignedSubSatInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpsubusw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedSubSatInt16x8(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpsubusw,
+ &MacroAssembler::vpsubuswSimd128);
+}
+
+// Lane-wise integer minimum
+
+void MacroAssembler::minInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpminsb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::minInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpminsb,
+ &MacroAssembler::vpminsbSimd128);
+}
+
+void MacroAssembler::unsignedMinInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpminub(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedMinInt8x16(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpminub,
+ &MacroAssembler::vpminubSimd128);
+}
+
+void MacroAssembler::minInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpminsw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::minInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpminsw,
+ &MacroAssembler::vpminswSimd128);
+}
+
+void MacroAssembler::unsignedMinInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpminuw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedMinInt16x8(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpminuw,
+ &MacroAssembler::vpminuwSimd128);
+}
+
+void MacroAssembler::minInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpminsd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::minInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpminsd,
+ &MacroAssembler::vpminsdSimd128);
+}
+
+void MacroAssembler::unsignedMinInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpminud(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedMinInt32x4(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpminud,
+ &MacroAssembler::vpminudSimd128);
+}
+
+// Lane-wise integer maximum
+
+void MacroAssembler::maxInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmaxsb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::maxInt8x16(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmaxsb,
+ &MacroAssembler::vpmaxsbSimd128);
+}
+
+void MacroAssembler::unsignedMaxInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmaxub(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedMaxInt8x16(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmaxub,
+ &MacroAssembler::vpmaxubSimd128);
+}
+
+void MacroAssembler::maxInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmaxsw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::maxInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmaxsw,
+ &MacroAssembler::vpmaxswSimd128);
+}
+
+void MacroAssembler::unsignedMaxInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmaxuw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedMaxInt16x8(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmaxuw,
+ &MacroAssembler::vpmaxuwSimd128);
+}
+
+void MacroAssembler::maxInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmaxsd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::maxInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmaxsd,
+ &MacroAssembler::vpmaxsdSimd128);
+}
+
+void MacroAssembler::unsignedMaxInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmaxud(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedMaxInt32x4(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmaxud,
+ &MacroAssembler::vpmaxudSimd128);
+}
+
+// Lane-wise integer rounding average
+
+void MacroAssembler::unsignedAverageInt8x16(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ vpavgb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedAverageInt16x8(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ vpavgw(Operand(rhs), lhs, dest);
+}
+
+// Lane-wise integer absolute value
+
+void MacroAssembler::absInt8x16(FloatRegister src, FloatRegister dest) {
+ vpabsb(Operand(src), dest);
+}
+
+void MacroAssembler::absInt16x8(FloatRegister src, FloatRegister dest) {
+ vpabsw(Operand(src), dest);
+}
+
+void MacroAssembler::absInt32x4(FloatRegister src, FloatRegister dest) {
+ vpabsd(Operand(src), dest);
+}
+
+void MacroAssembler::absInt64x2(FloatRegister src, FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ signReplicationInt64x2(src, scratch);
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpxor(Operand(scratch), src, dest);
+ vpsubq(Operand(scratch), dest, dest);
+}
+
+// Left shift by scalar
+
+void MacroAssembler::leftShiftInt8x16(Register rhs, FloatRegister lhsDest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::packedLeftShiftByScalarInt8x16(lhsDest, rhs, temp,
+ lhsDest);
+}
+
+void MacroAssembler::leftShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::packedLeftShiftByScalarInt8x16(count, src, dest);
+}
+
+void MacroAssembler::leftShiftInt16x8(Register rhs, FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::packedLeftShiftByScalarInt16x8(lhsDest, rhs,
+ lhsDest);
+}
+
+void MacroAssembler::leftShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsllw(count, src, dest);
+}
+
+void MacroAssembler::leftShiftInt32x4(Register rhs, FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::packedLeftShiftByScalarInt32x4(lhsDest, rhs,
+ lhsDest);
+}
+
+void MacroAssembler::leftShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpslld(count, src, dest);
+}
+
+void MacroAssembler::leftShiftInt64x2(Register rhs, FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::packedLeftShiftByScalarInt64x2(lhsDest, rhs,
+ lhsDest);
+}
+
+void MacroAssembler::leftShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsllq(count, src, dest);
+}
+
+// Right shift by scalar
+
+void MacroAssembler::rightShiftInt8x16(Register rhs, FloatRegister lhsDest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::packedRightShiftByScalarInt8x16(lhsDest, rhs, temp,
+ lhsDest);
+}
+
+void MacroAssembler::rightShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::packedRightShiftByScalarInt8x16(count, src, dest);
+}
+
+void MacroAssembler::unsignedRightShiftInt8x16(Register rhs,
+ FloatRegister lhsDest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt8x16(
+ lhsDest, rhs, temp, lhsDest);
+}
+
+void MacroAssembler::unsignedRightShiftInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt8x16(count, src,
+ dest);
+}
+
+void MacroAssembler::rightShiftInt16x8(Register rhs, FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::packedRightShiftByScalarInt16x8(lhsDest, rhs,
+ lhsDest);
+}
+
+void MacroAssembler::rightShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsraw(count, src, dest);
+}
+
+void MacroAssembler::unsignedRightShiftInt16x8(Register rhs,
+ FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt16x8(lhsDest, rhs,
+ lhsDest);
+}
+
+void MacroAssembler::unsignedRightShiftInt16x8(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsrlw(count, src, dest);
+}
+
+void MacroAssembler::rightShiftInt32x4(Register rhs, FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::packedRightShiftByScalarInt32x4(lhsDest, rhs,
+ lhsDest);
+}
+
+void MacroAssembler::rightShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsrad(count, src, dest);
+}
+
+void MacroAssembler::unsignedRightShiftInt32x4(Register rhs,
+ FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt32x4(lhsDest, rhs,
+ lhsDest);
+}
+
+void MacroAssembler::unsignedRightShiftInt32x4(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsrld(count, src, dest);
+}
+
+void MacroAssembler::rightShiftInt64x2(Register rhs, FloatRegister lhsDest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::packedRightShiftByScalarInt64x2(lhsDest, rhs, temp,
+ lhsDest);
+}
+
+void MacroAssembler::rightShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::packedRightShiftByScalarInt64x2(count, src, dest);
+}
+
+void MacroAssembler::unsignedRightShiftInt64x2(Register rhs,
+ FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::packedUnsignedRightShiftByScalarInt64x2(lhsDest, rhs,
+ lhsDest);
+}
+
+void MacroAssembler::unsignedRightShiftInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsrlq(count, src, dest);
+}
+
+// Sign replication operation
+
+void MacroAssembler::signReplicationInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(src != dest);
+ vpxor(Operand(dest), dest, dest);
+ vpcmpgtb(Operand(src), dest, dest);
+}
+
+void MacroAssembler::signReplicationInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsraw(Imm32(15), src, dest);
+}
+
+void MacroAssembler::signReplicationInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpsrad(Imm32(31), src, dest);
+}
+
+void MacroAssembler::signReplicationInt64x2(FloatRegister src,
+ FloatRegister dest) {
+ vpshufd(ComputeShuffleMask(1, 1, 3, 3), src, dest);
+ vpsrad(Imm32(31), dest, dest);
+}
+
+// Bitwise and, or, xor, not
+
+void MacroAssembler::bitwiseAndSimd128(FloatRegister rhs,
+ FloatRegister lhsDest) {
+ vpand(Operand(rhs), lhsDest, lhsDest);
+}
+
+void MacroAssembler::bitwiseAndSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpand(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::bitwiseAndSimd128(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpand,
+ &MacroAssembler::vpandSimd128);
+}
+
+void MacroAssembler::bitwiseOrSimd128(FloatRegister rhs,
+ FloatRegister lhsDest) {
+ vpor(Operand(rhs), lhsDest, lhsDest);
+}
+
+void MacroAssembler::bitwiseOrSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpor(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::bitwiseOrSimd128(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpor,
+ &MacroAssembler::vporSimd128);
+}
+
+void MacroAssembler::bitwiseXorSimd128(FloatRegister rhs,
+ FloatRegister lhsDest) {
+ vpxor(Operand(rhs), lhsDest, lhsDest);
+}
+
+void MacroAssembler::bitwiseXorSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpxor(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::bitwiseXorSimd128(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpxor,
+ &MacroAssembler::vpxorSimd128);
+}
+
+void MacroAssembler::bitwiseNotSimd128(FloatRegister src, FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ bitwiseXorSimd128(src, SimdConstant::SplatX16(-1), dest);
+}
+
+// Bitwise and-not
+
+void MacroAssembler::bitwiseNotAndSimd128(FloatRegister rhs,
+ FloatRegister lhsDest) {
+ vpandn(Operand(rhs), lhsDest, lhsDest);
+}
+
+void MacroAssembler::bitwiseNotAndSimd128(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpandn(Operand(rhs), lhs, dest);
+}
+
+// Bitwise select
+
+void MacroAssembler::bitwiseSelectSimd128(FloatRegister mask,
+ FloatRegister onTrue,
+ FloatRegister onFalse,
+ FloatRegister dest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::selectSimd128(mask, onTrue, onFalse, temp, dest);
+}
+
+// Population count
+
+void MacroAssembler::popcntInt8x16(FloatRegister src, FloatRegister dest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::popcntInt8x16(src, temp, dest);
+}
+
+// Comparisons (integer and floating-point)
+
+void MacroAssembler::compareInt8x16(Assembler::Condition cond,
+ FloatRegister rhs, FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::compareInt8x16(lhsDest, Operand(rhs), cond, lhsDest);
+}
+
+void MacroAssembler::compareInt8x16(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::compareInt8x16(lhs, Operand(rhs), cond, dest);
+}
+
+void MacroAssembler::compareInt8x16(Assembler::Condition cond,
+ FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ MOZ_ASSERT(cond != Assembler::Condition::LessThan &&
+ cond != Assembler::Condition::GreaterThanOrEqual);
+ MacroAssemblerX86Shared::compareInt8x16(cond, lhs, rhs, dest);
+}
+
+void MacroAssembler::compareInt16x8(Assembler::Condition cond,
+ FloatRegister rhs, FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::compareInt16x8(lhsDest, Operand(rhs), cond, lhsDest);
+}
+
+void MacroAssembler::compareInt16x8(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::compareInt16x8(lhs, Operand(rhs), cond, dest);
+}
+
+void MacroAssembler::compareInt16x8(Assembler::Condition cond,
+ FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ MOZ_ASSERT(cond != Assembler::Condition::LessThan &&
+ cond != Assembler::Condition::GreaterThanOrEqual);
+ MacroAssemblerX86Shared::compareInt16x8(cond, lhs, rhs, dest);
+}
+
+void MacroAssembler::compareInt32x4(Assembler::Condition cond,
+ FloatRegister rhs, FloatRegister lhsDest) {
+ MacroAssemblerX86Shared::compareInt32x4(lhsDest, Operand(rhs), cond, lhsDest);
+}
+
+void MacroAssembler::compareInt32x4(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::compareInt32x4(lhs, Operand(rhs), cond, dest);
+}
+
+void MacroAssembler::compareInt32x4(Assembler::Condition cond,
+ FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ MOZ_ASSERT(cond != Assembler::Condition::LessThan &&
+ cond != Assembler::Condition::GreaterThanOrEqual);
+ MacroAssemblerX86Shared::compareInt32x4(cond, lhs, rhs, dest);
+}
+
+void MacroAssembler::compareForEqualityInt64x2(Assembler::Condition cond,
+ FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::compareForEqualityInt64x2(lhs, Operand(rhs), cond,
+ dest);
+}
+
+void MacroAssembler::compareForOrderingInt64x2(
+ Assembler::Condition cond, FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1, FloatRegister temp2) {
+ if (HasAVX() && HasSSE42()) {
+ MacroAssemblerX86Shared::compareForOrderingInt64x2AVX(lhs, rhs, cond, dest);
+ } else {
+ MacroAssemblerX86Shared::compareForOrderingInt64x2(lhs, Operand(rhs), cond,
+ temp1, temp2, dest);
+ }
+}
+
+void MacroAssembler::compareFloat32x4(Assembler::Condition cond,
+ FloatRegister rhs,
+ FloatRegister lhsDest) {
+ // Code in the SIMD implementation allows operands to be reversed like this,
+ // this benefits the baseline compiler. Ion takes care of the reversing
+ // itself and never generates GT/GE.
+ if (cond == Assembler::GreaterThan) {
+ MacroAssemblerX86Shared::compareFloat32x4(rhs, Operand(lhsDest),
+ Assembler::LessThan, lhsDest);
+ } else if (cond == Assembler::GreaterThanOrEqual) {
+ MacroAssemblerX86Shared::compareFloat32x4(
+ rhs, Operand(lhsDest), Assembler::LessThanOrEqual, lhsDest);
+ } else {
+ MacroAssemblerX86Shared::compareFloat32x4(lhsDest, Operand(rhs), cond,
+ lhsDest);
+ }
+}
+
+void MacroAssembler::compareFloat32x4(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::compareFloat32x4(lhs, Operand(rhs), cond, dest);
+}
+
+void MacroAssembler::compareFloat32x4(Assembler::Condition cond,
+ FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ MOZ_ASSERT(cond != Assembler::Condition::GreaterThan &&
+ cond != Assembler::Condition::GreaterThanOrEqual);
+ MacroAssemblerX86Shared::compareFloat32x4(cond, lhs, rhs, dest);
+}
+
+void MacroAssembler::compareFloat64x2(Assembler::Condition cond,
+ FloatRegister rhs,
+ FloatRegister lhsDest) {
+ compareFloat64x2(cond, lhsDest, rhs, lhsDest);
+}
+
+void MacroAssembler::compareFloat64x2(Assembler::Condition cond,
+ FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ // Code in the SIMD implementation allows operands to be reversed like this,
+ // this benefits the baseline compiler. Ion takes care of the reversing
+ // itself and never generates GT/GE.
+ if (cond == Assembler::GreaterThan) {
+ MacroAssemblerX86Shared::compareFloat64x2(rhs, Operand(lhs),
+ Assembler::LessThan, dest);
+ } else if (cond == Assembler::GreaterThanOrEqual) {
+ MacroAssemblerX86Shared::compareFloat64x2(rhs, Operand(lhs),
+ Assembler::LessThanOrEqual, dest);
+ } else {
+ MacroAssemblerX86Shared::compareFloat64x2(lhs, Operand(rhs), cond, dest);
+ }
+}
+
+void MacroAssembler::compareFloat64x2(Assembler::Condition cond,
+ FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ MOZ_ASSERT(cond != Assembler::Condition::GreaterThan &&
+ cond != Assembler::Condition::GreaterThanOrEqual);
+ MacroAssemblerX86Shared::compareFloat64x2(cond, lhs, rhs, dest);
+}
+
+// Load. See comments above regarding integer operation.
+
+void MacroAssembler::loadUnalignedSimd128(const Operand& src,
+ FloatRegister dest) {
+ loadUnalignedSimd128Int(src, dest);
+}
+
+void MacroAssembler::loadUnalignedSimd128(const Address& src,
+ FloatRegister dest) {
+ loadUnalignedSimd128Int(src, dest);
+}
+
+void MacroAssembler::loadUnalignedSimd128(const BaseIndex& src,
+ FloatRegister dest) {
+ loadUnalignedSimd128Int(src, dest);
+}
+
+// Store. See comments above regarding integer operation.
+
+void MacroAssembler::storeUnalignedSimd128(FloatRegister src,
+ const Address& dest) {
+ storeUnalignedSimd128Int(src, dest);
+}
+
+void MacroAssembler::storeUnalignedSimd128(FloatRegister src,
+ const BaseIndex& dest) {
+ storeUnalignedSimd128Int(src, dest);
+}
+
+// Floating point negation
+
+void MacroAssembler::negFloat32x4(FloatRegister src, FloatRegister dest) {
+ src = moveSimd128FloatIfNotAVX(src, dest);
+ bitwiseXorSimd128(src, SimdConstant::SplatX4(-0.f), dest);
+}
+
+void MacroAssembler::negFloat64x2(FloatRegister src, FloatRegister dest) {
+ src = moveSimd128FloatIfNotAVX(src, dest);
+ bitwiseXorSimd128(src, SimdConstant::SplatX2(-0.0), dest);
+}
+
+// Floating point absolute value
+
+void MacroAssembler::absFloat32x4(FloatRegister src, FloatRegister dest) {
+ src = moveSimd128FloatIfNotAVX(src, dest);
+ bitwiseAndSimd128(src, SimdConstant::SplatX4(0x7FFFFFFF), dest);
+}
+
+void MacroAssembler::absFloat64x2(FloatRegister src, FloatRegister dest) {
+ src = moveSimd128FloatIfNotAVX(src, dest);
+ bitwiseAndSimd128(src, SimdConstant::SplatX2(int64_t(0x7FFFFFFFFFFFFFFFll)),
+ dest);
+}
+
+// NaN-propagating minimum
+
+void MacroAssembler::minFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) {
+ MacroAssemblerX86Shared::minFloat32x4(lhs, rhs, temp1, temp2, dest);
+}
+
+void MacroAssembler::minFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) {
+ MacroAssemblerX86Shared::minFloat64x2(lhs, rhs, temp1, temp2, dest);
+}
+
+// NaN-propagating maximum
+
+void MacroAssembler::maxFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) {
+ MacroAssemblerX86Shared::maxFloat32x4(lhs, rhs, temp1, temp2, dest);
+}
+
+void MacroAssembler::maxFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp1,
+ FloatRegister temp2) {
+ MacroAssemblerX86Shared::maxFloat64x2(lhs, rhs, temp1, temp2, dest);
+}
+
+// Compare-based minimum
+
+void MacroAssembler::pseudoMinFloat32x4(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest) {
+ // Shut up the linter by using the same names as in the declaration, then
+ // aliasing here.
+ FloatRegister rhsDest = rhsOrRhsDest;
+ FloatRegister lhs = lhsOrLhsDest;
+ vminps(Operand(lhs), rhsDest, rhsDest);
+}
+
+void MacroAssembler::pseudoMinFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vminps(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::pseudoMinFloat64x2(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest) {
+ FloatRegister rhsDest = rhsOrRhsDest;
+ FloatRegister lhs = lhsOrLhsDest;
+ vminpd(Operand(lhs), rhsDest, rhsDest);
+}
+
+void MacroAssembler::pseudoMinFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vminpd(Operand(rhs), lhs, dest);
+}
+
+// Compare-based maximum
+
+void MacroAssembler::pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest) {
+ FloatRegister rhsDest = rhsOrRhsDest;
+ FloatRegister lhs = lhsOrLhsDest;
+ vmaxps(Operand(lhs), rhsDest, rhsDest);
+}
+
+void MacroAssembler::pseudoMaxFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vmaxps(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest,
+ FloatRegister lhsOrLhsDest) {
+ FloatRegister rhsDest = rhsOrRhsDest;
+ FloatRegister lhs = lhsOrLhsDest;
+ vmaxpd(Operand(lhs), rhsDest, rhsDest);
+}
+
+void MacroAssembler::pseudoMaxFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vmaxpd(Operand(rhs), lhs, dest);
+}
+
+// Widening/pairwise integer dot product
+
+void MacroAssembler::widenDotInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpmaddwd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::widenDotInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpmaddwd,
+ &MacroAssembler::vpmaddwdSimd128);
+}
+
+void MacroAssembler::dotInt8x16Int7x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (lhs == dest && !HasAVX()) {
+ moveSimd128Int(lhs, scratch);
+ lhs = scratch;
+ }
+ rhs = moveSimd128IntIfNotAVX(rhs, dest);
+ vpmaddubsw(lhs, rhs, dest);
+}
+
+void MacroAssembler::dotInt8x16Int7x16ThenAdd(FloatRegister lhs,
+ FloatRegister rhs,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ rhs = moveSimd128IntIfNotAVX(rhs, scratch);
+ vpmaddubsw(lhs, rhs, scratch);
+ vpmaddwdSimd128(SimdConstant::SplatX8(1), scratch, scratch);
+ vpaddd(Operand(scratch), dest, dest);
+}
+
+void MacroAssembler::dotBFloat16x8ThenAdd(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest,
+ FloatRegister temp) {
+ MOZ_ASSERT(lhs != dest && rhs != dest);
+ MacroAssemblerX86Shared::dotBFloat16x8ThenAdd(lhs, rhs, dest, temp);
+}
+
+// Rounding
+
+void MacroAssembler::ceilFloat32x4(FloatRegister src, FloatRegister dest) {
+ vroundps(Assembler::SSERoundingMode::Ceil, Operand(src), dest);
+}
+
+void MacroAssembler::ceilFloat64x2(FloatRegister src, FloatRegister dest) {
+ vroundpd(Assembler::SSERoundingMode::Ceil, Operand(src), dest);
+}
+
+void MacroAssembler::floorFloat32x4(FloatRegister src, FloatRegister dest) {
+ vroundps(Assembler::SSERoundingMode::Floor, Operand(src), dest);
+}
+
+void MacroAssembler::floorFloat64x2(FloatRegister src, FloatRegister dest) {
+ vroundpd(Assembler::SSERoundingMode::Floor, Operand(src), dest);
+}
+
+void MacroAssembler::truncFloat32x4(FloatRegister src, FloatRegister dest) {
+ vroundps(Assembler::SSERoundingMode::Trunc, Operand(src), dest);
+}
+
+void MacroAssembler::truncFloat64x2(FloatRegister src, FloatRegister dest) {
+ vroundpd(Assembler::SSERoundingMode::Trunc, Operand(src), dest);
+}
+
+void MacroAssembler::nearestFloat32x4(FloatRegister src, FloatRegister dest) {
+ vroundps(Assembler::SSERoundingMode::Nearest, Operand(src), dest);
+}
+
+void MacroAssembler::nearestFloat64x2(FloatRegister src, FloatRegister dest) {
+ vroundpd(Assembler::SSERoundingMode::Nearest, Operand(src), dest);
+}
+
+// Floating add
+
+void MacroAssembler::addFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vaddps(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::addFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vaddps,
+ &MacroAssembler::vaddpsSimd128);
+}
+
+void MacroAssembler::addFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vaddpd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::addFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vaddpd,
+ &MacroAssembler::vaddpdSimd128);
+}
+
+// Floating subtract
+
+void MacroAssembler::subFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vsubps(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::subFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vsubps,
+ &MacroAssembler::vsubpsSimd128);
+}
+
+void MacroAssembler::subFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ AssemblerX86Shared::vsubpd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::subFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vsubpd,
+ &MacroAssembler::vsubpdSimd128);
+}
+
+// Floating division
+
+void MacroAssembler::divFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vdivps(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::divFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vdivps,
+ &MacroAssembler::vdivpsSimd128);
+}
+
+void MacroAssembler::divFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vdivpd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::divFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vdivpd,
+ &MacroAssembler::vdivpdSimd128);
+}
+
+// Floating Multiply
+
+void MacroAssembler::mulFloat32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vmulps(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::mulFloat32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vmulps,
+ &MacroAssembler::vmulpsSimd128);
+}
+
+void MacroAssembler::mulFloat64x2(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vmulpd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::mulFloat64x2(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vmulpd,
+ &MacroAssembler::vmulpdSimd128);
+}
+
+// Pairwise add
+
+void MacroAssembler::extAddPairwiseInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ if (dest == src) {
+ moveSimd128(src, scratch);
+ src = scratch;
+ }
+ loadConstantSimd128Int(SimdConstant::SplatX16(1), dest);
+ vpmaddubsw(src, dest, dest);
+}
+
+void MacroAssembler::unsignedExtAddPairwiseInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpmaddubswSimd128(SimdConstant::SplatX16(1), src, dest);
+}
+
+void MacroAssembler::extAddPairwiseInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpmaddwdSimd128(SimdConstant::SplatX8(1), src, dest);
+}
+
+void MacroAssembler::unsignedExtAddPairwiseInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpxorSimd128(SimdConstant::SplatX8(-0x8000), src, dest);
+ vpmaddwdSimd128(SimdConstant::SplatX8(1), dest, dest);
+ vpadddSimd128(SimdConstant::SplatX4(0x00010000), dest, dest);
+}
+
+// Floating square root
+
+void MacroAssembler::sqrtFloat32x4(FloatRegister src, FloatRegister dest) {
+ vsqrtps(Operand(src), dest);
+}
+
+void MacroAssembler::sqrtFloat64x2(FloatRegister src, FloatRegister dest) {
+ vsqrtpd(Operand(src), dest);
+}
+
+// Integer to floating point with rounding
+
+void MacroAssembler::convertInt32x4ToFloat32x4(FloatRegister src,
+ FloatRegister dest) {
+ vcvtdq2ps(src, dest);
+}
+
+void MacroAssembler::unsignedConvertInt32x4ToFloat32x4(FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::unsignedConvertInt32x4ToFloat32x4(src, dest);
+}
+
+void MacroAssembler::convertInt32x4ToFloat64x2(FloatRegister src,
+ FloatRegister dest) {
+ vcvtdq2pd(src, dest);
+}
+
+void MacroAssembler::unsignedConvertInt32x4ToFloat64x2(FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::unsignedConvertInt32x4ToFloat64x2(src, dest);
+}
+
+// Floating point to integer with saturation
+
+void MacroAssembler::truncSatFloat32x4ToInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ MacroAssemblerX86Shared::truncSatFloat32x4ToInt32x4(src, dest);
+}
+
+void MacroAssembler::unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src,
+ FloatRegister dest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::unsignedTruncSatFloat32x4ToInt32x4(src, temp, dest);
+}
+
+void MacroAssembler::truncSatFloat64x2ToInt32x4(FloatRegister src,
+ FloatRegister dest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::truncSatFloat64x2ToInt32x4(src, temp, dest);
+}
+
+void MacroAssembler::unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src,
+ FloatRegister dest,
+ FloatRegister temp) {
+ MacroAssemblerX86Shared::unsignedTruncSatFloat64x2ToInt32x4(src, temp, dest);
+}
+
+void MacroAssembler::truncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest) {
+ vcvttps2dq(src, dest);
+}
+
+void MacroAssembler::unsignedTruncSatFloat32x4ToInt32x4Relaxed(
+ FloatRegister src, FloatRegister dest) {
+ MacroAssemblerX86Shared::unsignedTruncSatFloat32x4ToInt32x4Relaxed(src, dest);
+}
+
+void MacroAssembler::truncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest) {
+ vcvttpd2dq(src, dest);
+}
+
+void MacroAssembler::unsignedTruncSatFloat64x2ToInt32x4Relaxed(
+ FloatRegister src, FloatRegister dest) {
+ MacroAssemblerX86Shared::unsignedTruncSatFloat64x2ToInt32x4Relaxed(src, dest);
+}
+
+// Floating point widening
+
+void MacroAssembler::convertFloat64x2ToFloat32x4(FloatRegister src,
+ FloatRegister dest) {
+ vcvtpd2ps(src, dest);
+}
+
+void MacroAssembler::convertFloat32x4ToFloat64x2(FloatRegister src,
+ FloatRegister dest) {
+ vcvtps2pd(src, dest);
+}
+
+// Integer to integer narrowing
+
+void MacroAssembler::narrowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpacksswb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::narrowInt16x8(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpacksswb,
+ &MacroAssembler::vpacksswbSimd128);
+}
+
+void MacroAssembler::unsignedNarrowInt16x8(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpackuswb(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedNarrowInt16x8(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpackuswb,
+ &MacroAssembler::vpackuswbSimd128);
+}
+
+void MacroAssembler::narrowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpackssdw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::narrowInt32x4(FloatRegister lhs, const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpackssdw,
+ &MacroAssembler::vpackssdwSimd128);
+}
+
+void MacroAssembler::unsignedNarrowInt32x4(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vpackusdw(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::unsignedNarrowInt32x4(FloatRegister lhs,
+ const SimdConstant& rhs,
+ FloatRegister dest) {
+ binarySimd128(lhs, rhs, dest, &MacroAssembler::vpackusdw,
+ &MacroAssembler::vpackusdwSimd128);
+}
+
+// Integer to integer widening
+
+void MacroAssembler::widenLowInt8x16(FloatRegister src, FloatRegister dest) {
+ vpmovsxbw(Operand(src), dest);
+}
+
+void MacroAssembler::widenHighInt8x16(FloatRegister src, FloatRegister dest) {
+ vpalignr(Operand(src), dest, dest, 8);
+ vpmovsxbw(Operand(dest), dest);
+}
+
+void MacroAssembler::unsignedWidenLowInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ vpmovzxbw(Operand(src), dest);
+}
+
+void MacroAssembler::unsignedWidenHighInt8x16(FloatRegister src,
+ FloatRegister dest) {
+ vpalignr(Operand(src), dest, dest, 8);
+ vpmovzxbw(Operand(dest), dest);
+}
+
+void MacroAssembler::widenLowInt16x8(FloatRegister src, FloatRegister dest) {
+ vpmovsxwd(Operand(src), dest);
+}
+
+void MacroAssembler::widenHighInt16x8(FloatRegister src, FloatRegister dest) {
+ vpalignr(Operand(src), dest, dest, 8);
+ vpmovsxwd(Operand(dest), dest);
+}
+
+void MacroAssembler::unsignedWidenLowInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ vpmovzxwd(Operand(src), dest);
+}
+
+void MacroAssembler::unsignedWidenHighInt16x8(FloatRegister src,
+ FloatRegister dest) {
+ vpalignr(Operand(src), dest, dest, 8);
+ vpmovzxwd(Operand(dest), dest);
+}
+
+void MacroAssembler::widenLowInt32x4(FloatRegister src, FloatRegister dest) {
+ vpmovsxdq(Operand(src), dest);
+}
+
+void MacroAssembler::unsignedWidenLowInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ vpmovzxdq(Operand(src), dest);
+}
+
+void MacroAssembler::widenHighInt32x4(FloatRegister src, FloatRegister dest) {
+ if (src == dest || HasAVX()) {
+ vmovhlps(src, src, dest);
+ } else {
+ vpshufd(ComputeShuffleMask(2, 3, 2, 3), src, dest);
+ }
+ vpmovsxdq(Operand(dest), dest);
+}
+
+void MacroAssembler::unsignedWidenHighInt32x4(FloatRegister src,
+ FloatRegister dest) {
+ ScratchSimd128Scope scratch(*this);
+ src = moveSimd128IntIfNotAVX(src, dest);
+ vpxor(scratch, scratch, scratch);
+ vpunpckhdq(scratch, src, dest);
+}
+
+// Floating multiply-accumulate: srcDest [+-]= src1 * src2
+// The Intel FMA feature is some AVX* special sauce, no support yet.
+
+void MacroAssembler::fmaFloat32x4(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) {
+ if (HasFMA()) {
+ vfmadd231ps(src2, src1, srcDest);
+ return;
+ }
+ ScratchSimd128Scope scratch(*this);
+ src1 = moveSimd128FloatIfNotAVX(src1, scratch);
+ mulFloat32x4(src1, src2, scratch);
+ addFloat32x4(srcDest, scratch, srcDest);
+}
+
+void MacroAssembler::fnmaFloat32x4(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) {
+ if (HasFMA()) {
+ vfnmadd231ps(src2, src1, srcDest);
+ return;
+ }
+ ScratchSimd128Scope scratch(*this);
+ src1 = moveSimd128FloatIfNotAVX(src1, scratch);
+ mulFloat32x4(src1, src2, scratch);
+ subFloat32x4(srcDest, scratch, srcDest);
+}
+
+void MacroAssembler::fmaFloat64x2(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) {
+ if (HasFMA()) {
+ vfmadd231pd(src2, src1, srcDest);
+ return;
+ }
+ ScratchSimd128Scope scratch(*this);
+ src1 = moveSimd128FloatIfNotAVX(src1, scratch);
+ mulFloat64x2(src1, src2, scratch);
+ addFloat64x2(srcDest, scratch, srcDest);
+}
+
+void MacroAssembler::fnmaFloat64x2(FloatRegister src1, FloatRegister src2,
+ FloatRegister srcDest) {
+ if (HasFMA()) {
+ vfnmadd231pd(src2, src1, srcDest);
+ return;
+ }
+ ScratchSimd128Scope scratch(*this);
+ src1 = moveSimd128FloatIfNotAVX(src1, scratch);
+ mulFloat64x2(src1, src2, scratch);
+ subFloat64x2(srcDest, scratch, srcDest);
+}
+
+void MacroAssembler::minFloat32x4Relaxed(FloatRegister src,
+ FloatRegister srcDest) {
+ vminps(Operand(src), srcDest, srcDest);
+}
+
+void MacroAssembler::minFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vminps(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::maxFloat32x4Relaxed(FloatRegister src,
+ FloatRegister srcDest) {
+ vmaxps(Operand(src), srcDest, srcDest);
+}
+
+void MacroAssembler::maxFloat32x4Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vmaxps(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::minFloat64x2Relaxed(FloatRegister src,
+ FloatRegister srcDest) {
+ vminpd(Operand(src), srcDest, srcDest);
+}
+
+void MacroAssembler::minFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vminpd(Operand(rhs), lhs, dest);
+}
+
+void MacroAssembler::maxFloat64x2Relaxed(FloatRegister src,
+ FloatRegister srcDest) {
+ vmaxpd(Operand(src), srcDest, srcDest);
+}
+
+void MacroAssembler::maxFloat64x2Relaxed(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest) {
+ vmaxpd(Operand(rhs), lhs, dest);
+}
+
+// ========================================================================
+// Truncate floating point.
+
+void MacroAssembler::truncateFloat32ToInt64(Address src, Address dest,
+ Register temp) {
+ if (Assembler::HasSSE3()) {
+ fld32(Operand(src));
+ fisttp(Operand(dest));
+ return;
+ }
+
+ if (src.base == esp) {
+ src.offset += 2 * sizeof(int32_t);
+ }
+ if (dest.base == esp) {
+ dest.offset += 2 * sizeof(int32_t);
+ }
+
+ reserveStack(2 * sizeof(int32_t));
+
+ // Set conversion to truncation.
+ fnstcw(Operand(esp, 0));
+ load32(Operand(esp, 0), temp);
+ andl(Imm32(~0xFF00), temp);
+ orl(Imm32(0xCFF), temp);
+ store32(temp, Address(esp, sizeof(int32_t)));
+ fldcw(Operand(esp, sizeof(int32_t)));
+
+ // Load double on fp stack, convert and load regular stack.
+ fld32(Operand(src));
+ fistp(Operand(dest));
+
+ // Reset the conversion flag.
+ fldcw(Operand(esp, 0));
+
+ freeStack(2 * sizeof(int32_t));
+}
+void MacroAssembler::truncateDoubleToInt64(Address src, Address dest,
+ Register temp) {
+ if (Assembler::HasSSE3()) {
+ fld(Operand(src));
+ fisttp(Operand(dest));
+ return;
+ }
+
+ if (src.base == esp) {
+ src.offset += 2 * sizeof(int32_t);
+ }
+ if (dest.base == esp) {
+ dest.offset += 2 * sizeof(int32_t);
+ }
+
+ reserveStack(2 * sizeof(int32_t));
+
+ // Set conversion to truncation.
+ fnstcw(Operand(esp, 0));
+ load32(Operand(esp, 0), temp);
+ andl(Imm32(~0xFF00), temp);
+ orl(Imm32(0xCFF), temp);
+ store32(temp, Address(esp, 1 * sizeof(int32_t)));
+ fldcw(Operand(esp, 1 * sizeof(int32_t)));
+
+ // Load double on fp stack, convert and load regular stack.
+ fld(Operand(src));
+ fistp(Operand(dest));
+
+ // Reset the conversion flag.
+ fldcw(Operand(esp, 0));
+
+ freeStack(2 * sizeof(int32_t));
+}
+
+// ===============================================================
+// Clamping functions.
+
+void MacroAssembler::clampIntToUint8(Register reg) {
+ Label inRange;
+ branchTest32(Assembler::Zero, reg, Imm32(0xffffff00), &inRange);
+ {
+ sarl(Imm32(31), reg);
+ notl(reg);
+ andl(Imm32(255), reg);
+ }
+ bind(&inRange);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_MacroAssembler_x86_shared_inl_h */
diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp
new file mode 100644
index 0000000000..7e2f7ea29e
--- /dev/null
+++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp
@@ -0,0 +1,2154 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86-shared/MacroAssembler-x86-shared.h"
+
+#include "mozilla/Casting.h"
+
+#include "jsmath.h"
+
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "js/ScalarType.h" // js::Scalar::Type
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+// Note: this function clobbers the input register.
+void MacroAssembler::clampDoubleToUint8(FloatRegister input, Register output) {
+ ScratchDoubleScope scratch(*this);
+ MOZ_ASSERT(input != scratch);
+ Label positive, done;
+
+ // <= 0 or NaN --> 0
+ zeroDouble(scratch);
+ branchDouble(DoubleGreaterThan, input, scratch, &positive);
+ {
+ move32(Imm32(0), output);
+ jump(&done);
+ }
+
+ bind(&positive);
+
+ // Add 0.5 and truncate.
+ loadConstantDouble(0.5, scratch);
+ addDouble(scratch, input);
+
+ Label outOfRange;
+
+ // Truncate to int32 and ensure the result <= 255. This relies on the
+ // processor setting output to a value > 255 for doubles outside the int32
+ // range (for instance 0x80000000).
+ vcvttsd2si(input, output);
+ branch32(Assembler::Above, output, Imm32(255), &outOfRange);
+ {
+ // Check if we had a tie.
+ convertInt32ToDouble(output, scratch);
+ branchDouble(DoubleNotEqual, input, scratch, &done);
+
+ // It was a tie. Mask out the ones bit to get an even value.
+ // See also js_TypedArray_uint8_clamp_double.
+ and32(Imm32(~1), output);
+ jump(&done);
+ }
+
+ // > 255 --> 255
+ bind(&outOfRange);
+ { move32(Imm32(255), output); }
+
+ bind(&done);
+}
+
+bool MacroAssemblerX86Shared::buildOOLFakeExitFrame(void* fakeReturnAddr) {
+ asMasm().PushFrameDescriptor(FrameType::IonJS);
+ asMasm().Push(ImmPtr(fakeReturnAddr));
+ asMasm().Push(FramePointer);
+ return true;
+}
+
+void MacroAssemblerX86Shared::branchNegativeZero(FloatRegister reg,
+ Register scratch, Label* label,
+ bool maybeNonZero) {
+ // Determines whether the low double contained in the XMM register reg
+ // is equal to -0.0.
+
+#if defined(JS_CODEGEN_X86)
+ Label nonZero;
+
+ // if not already compared to zero
+ if (maybeNonZero) {
+ ScratchDoubleScope scratchDouble(asMasm());
+
+ // Compare to zero. Lets through {0, -0}.
+ zeroDouble(scratchDouble);
+
+ // If reg is non-zero, jump to nonZero.
+ asMasm().branchDouble(DoubleNotEqual, reg, scratchDouble, &nonZero);
+ }
+ // Input register is either zero or negative zero. Retrieve sign of input.
+ vmovmskpd(reg, scratch);
+
+ // If reg is 1 or 3, input is negative zero.
+ // If reg is 0 or 2, input is a normal zero.
+ asMasm().branchTest32(NonZero, scratch, Imm32(1), label);
+
+ bind(&nonZero);
+#elif defined(JS_CODEGEN_X64)
+ vmovq(reg, scratch);
+ cmpq(Imm32(1), scratch);
+ j(Overflow, label);
+#endif
+}
+
+void MacroAssemblerX86Shared::branchNegativeZeroFloat32(FloatRegister reg,
+ Register scratch,
+ Label* label) {
+ vmovd(reg, scratch);
+ cmp32(scratch, Imm32(1));
+ j(Overflow, label);
+}
+
+MacroAssembler& MacroAssemblerX86Shared::asMasm() {
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerX86Shared::asMasm() const {
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+template <class T, class Map>
+T* MacroAssemblerX86Shared::getConstant(const typename T::Pod& value, Map& map,
+ Vector<T, 0, SystemAllocPolicy>& vec) {
+ using AddPtr = typename Map::AddPtr;
+ size_t index;
+ if (AddPtr p = map.lookupForAdd(value)) {
+ index = p->value();
+ } else {
+ index = vec.length();
+ enoughMemory_ &= vec.append(T(value));
+ if (!enoughMemory_) {
+ return nullptr;
+ }
+ enoughMemory_ &= map.add(p, value, index);
+ if (!enoughMemory_) {
+ return nullptr;
+ }
+ }
+ return &vec[index];
+}
+
+MacroAssemblerX86Shared::Float* MacroAssemblerX86Shared::getFloat(float f) {
+ return getConstant<Float, FloatMap>(f, floatMap_, floats_);
+}
+
+MacroAssemblerX86Shared::Double* MacroAssemblerX86Shared::getDouble(double d) {
+ return getConstant<Double, DoubleMap>(d, doubleMap_, doubles_);
+}
+
+MacroAssemblerX86Shared::SimdData* MacroAssemblerX86Shared::getSimdData(
+ const SimdConstant& v) {
+ return getConstant<SimdData, SimdMap>(v, simdMap_, simds_);
+}
+
+void MacroAssemblerX86Shared::binarySimd128(
+ const SimdConstant& rhs, FloatRegister lhsDest,
+ void (MacroAssembler::*regOp)(const Operand&, FloatRegister, FloatRegister),
+ void (MacroAssembler::*constOp)(const SimdConstant&, FloatRegister)) {
+ ScratchSimd128Scope scratch(asMasm());
+ if (maybeInlineSimd128Int(rhs, scratch)) {
+ (asMasm().*regOp)(Operand(scratch), lhsDest, lhsDest);
+ } else {
+ (asMasm().*constOp)(rhs, lhsDest);
+ }
+}
+
+void MacroAssemblerX86Shared::binarySimd128(
+ FloatRegister lhs, const SimdConstant& rhs, FloatRegister dest,
+ void (MacroAssembler::*regOp)(const Operand&, FloatRegister, FloatRegister),
+ void (MacroAssembler::*constOp)(const SimdConstant&, FloatRegister,
+ FloatRegister)) {
+ ScratchSimd128Scope scratch(asMasm());
+ if (maybeInlineSimd128Int(rhs, scratch)) {
+ (asMasm().*regOp)(Operand(scratch), lhs, dest);
+ } else {
+ (asMasm().*constOp)(rhs, lhs, dest);
+ }
+}
+
+void MacroAssemblerX86Shared::binarySimd128(
+ const SimdConstant& rhs, FloatRegister lhs,
+ void (MacroAssembler::*regOp)(const Operand&, FloatRegister),
+ void (MacroAssembler::*constOp)(const SimdConstant&, FloatRegister)) {
+ ScratchSimd128Scope scratch(asMasm());
+ if (maybeInlineSimd128Int(rhs, scratch)) {
+ (asMasm().*regOp)(Operand(scratch), lhs);
+ } else {
+ (asMasm().*constOp)(rhs, lhs);
+ }
+}
+
+void MacroAssemblerX86Shared::bitwiseTestSimd128(const SimdConstant& rhs,
+ FloatRegister lhs) {
+ ScratchSimd128Scope scratch(asMasm());
+ if (maybeInlineSimd128Int(rhs, scratch)) {
+ vptest(scratch, lhs);
+ } else {
+ asMasm().vptestSimd128(rhs, lhs);
+ }
+}
+
+void MacroAssemblerX86Shared::minMaxDouble(FloatRegister first,
+ FloatRegister second, bool canBeNaN,
+ bool isMax) {
+ Label done, nan, minMaxInst;
+
+ // Do a vucomisd to catch equality and NaNs, which both require special
+ // handling. If the operands are ordered and inequal, we branch straight to
+ // the min/max instruction. If we wanted, we could also branch for less-than
+ // or greater-than here instead of using min/max, however these conditions
+ // will sometimes be hard on the branch predictor.
+ vucomisd(second, first);
+ j(Assembler::NotEqual, &minMaxInst);
+ if (canBeNaN) {
+ j(Assembler::Parity, &nan);
+ }
+
+ // Ordered and equal. The operands are bit-identical unless they are zero
+ // and negative zero. These instructions merge the sign bits in that
+ // case, and are no-ops otherwise.
+ if (isMax) {
+ vandpd(second, first, first);
+ } else {
+ vorpd(second, first, first);
+ }
+ jump(&done);
+
+ // x86's min/max are not symmetric; if either operand is a NaN, they return
+ // the read-only operand. We need to return a NaN if either operand is a
+ // NaN, so we explicitly check for a NaN in the read-write operand.
+ if (canBeNaN) {
+ bind(&nan);
+ vucomisd(first, first);
+ j(Assembler::Parity, &done);
+ }
+
+ // When the values are inequal, or second is NaN, x86's min and max will
+ // return the value we need.
+ bind(&minMaxInst);
+ if (isMax) {
+ vmaxsd(second, first, first);
+ } else {
+ vminsd(second, first, first);
+ }
+
+ bind(&done);
+}
+
+void MacroAssemblerX86Shared::minMaxFloat32(FloatRegister first,
+ FloatRegister second, bool canBeNaN,
+ bool isMax) {
+ Label done, nan, minMaxInst;
+
+ // Do a vucomiss to catch equality and NaNs, which both require special
+ // handling. If the operands are ordered and inequal, we branch straight to
+ // the min/max instruction. If we wanted, we could also branch for less-than
+ // or greater-than here instead of using min/max, however these conditions
+ // will sometimes be hard on the branch predictor.
+ vucomiss(second, first);
+ j(Assembler::NotEqual, &minMaxInst);
+ if (canBeNaN) {
+ j(Assembler::Parity, &nan);
+ }
+
+ // Ordered and equal. The operands are bit-identical unless they are zero
+ // and negative zero. These instructions merge the sign bits in that
+ // case, and are no-ops otherwise.
+ if (isMax) {
+ vandps(second, first, first);
+ } else {
+ vorps(second, first, first);
+ }
+ jump(&done);
+
+ // x86's min/max are not symmetric; if either operand is a NaN, they return
+ // the read-only operand. We need to return a NaN if either operand is a
+ // NaN, so we explicitly check for a NaN in the read-write operand.
+ if (canBeNaN) {
+ bind(&nan);
+ vucomiss(first, first);
+ j(Assembler::Parity, &done);
+ }
+
+ // When the values are inequal, or second is NaN, x86's min and max will
+ // return the value we need.
+ bind(&minMaxInst);
+ if (isMax) {
+ vmaxss(second, first, first);
+ } else {
+ vminss(second, first, first);
+ }
+
+ bind(&done);
+}
+
+#ifdef ENABLE_WASM_SIMD
+bool MacroAssembler::MustMaskShiftCountSimd128(wasm::SimdOp op, int32_t* mask) {
+ switch (op) {
+ case wasm::SimdOp::I8x16Shl:
+ case wasm::SimdOp::I8x16ShrU:
+ case wasm::SimdOp::I8x16ShrS:
+ *mask = 7;
+ break;
+ case wasm::SimdOp::I16x8Shl:
+ case wasm::SimdOp::I16x8ShrU:
+ case wasm::SimdOp::I16x8ShrS:
+ *mask = 15;
+ break;
+ case wasm::SimdOp::I32x4Shl:
+ case wasm::SimdOp::I32x4ShrU:
+ case wasm::SimdOp::I32x4ShrS:
+ *mask = 31;
+ break;
+ case wasm::SimdOp::I64x2Shl:
+ case wasm::SimdOp::I64x2ShrU:
+ case wasm::SimdOp::I64x2ShrS:
+ *mask = 63;
+ break;
+ default:
+ MOZ_CRASH("Unexpected shift operation");
+ }
+ return true;
+}
+#endif
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// MacroAssembler high-level usage.
+
+void MacroAssembler::flush() {}
+
+void MacroAssembler::comment(const char* msg) { masm.comment(msg); }
+
+// This operation really consists of five phases, in order to enforce the
+// restriction that on x86_shared, srcDest must be eax and edx will be
+// clobbered.
+//
+// Input: { rhs, lhsOutput }
+//
+// [PUSH] Preserve registers
+// [MOVE] Generate moves to specific registers
+//
+// [DIV] Input: { regForRhs, EAX }
+// [DIV] extend EAX into EDX
+// [DIV] x86 Division operator
+// [DIV] Ouptut: { EAX, EDX }
+//
+// [MOVE] Move specific registers to outputs
+// [POP] Restore registers
+//
+// Output: { lhsOutput, remainderOutput }
+void MacroAssembler::flexibleDivMod32(Register rhs, Register lhsOutput,
+ Register remOutput, bool isUnsigned,
+ const LiveRegisterSet&) {
+ // Currently this helper can't handle this situation.
+ MOZ_ASSERT(lhsOutput != rhs);
+ MOZ_ASSERT(lhsOutput != remOutput);
+
+ // Choose a register that is not edx, or eax to hold the rhs;
+ // ebx is chosen arbitrarily, and will be preserved if necessary.
+ Register regForRhs = (rhs == eax || rhs == edx) ? ebx : rhs;
+
+ // Add registers we will be clobbering as live, but
+ // also remove the set we do not restore.
+ LiveRegisterSet preserve;
+ preserve.add(edx);
+ preserve.add(eax);
+ preserve.add(regForRhs);
+
+ preserve.takeUnchecked(lhsOutput);
+ preserve.takeUnchecked(remOutput);
+
+ PushRegsInMask(preserve);
+
+ // Shuffle input into place.
+ moveRegPair(lhsOutput, rhs, eax, regForRhs);
+ if (oom()) {
+ return;
+ }
+
+ // Sign extend eax into edx to make (edx:eax): idiv/udiv are 64-bit.
+ if (isUnsigned) {
+ mov(ImmWord(0), edx);
+ udiv(regForRhs);
+ } else {
+ cdq();
+ idiv(regForRhs);
+ }
+
+ moveRegPair(eax, edx, lhsOutput, remOutput);
+ if (oom()) {
+ return;
+ }
+
+ PopRegsInMask(preserve);
+}
+
+void MacroAssembler::flexibleQuotient32(
+ Register rhs, Register srcDest, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs) {
+ // Choose an arbitrary register that isn't eax, edx, rhs or srcDest;
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.takeUnchecked(eax);
+ regs.takeUnchecked(edx);
+ regs.takeUnchecked(rhs);
+ regs.takeUnchecked(srcDest);
+
+ Register remOut = regs.takeAny();
+ push(remOut);
+ flexibleDivMod32(rhs, srcDest, remOut, isUnsigned, volatileLiveRegs);
+ pop(remOut);
+}
+
+void MacroAssembler::flexibleRemainder32(
+ Register rhs, Register srcDest, bool isUnsigned,
+ const LiveRegisterSet& volatileLiveRegs) {
+ // Choose an arbitrary register that isn't eax, edx, rhs or srcDest
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ regs.takeUnchecked(eax);
+ regs.takeUnchecked(edx);
+ regs.takeUnchecked(rhs);
+ regs.takeUnchecked(srcDest);
+
+ Register remOut = regs.takeAny();
+ push(remOut);
+ flexibleDivMod32(rhs, srcDest, remOut, isUnsigned, volatileLiveRegs);
+ mov(remOut, srcDest);
+ pop(remOut);
+}
+
+// ===============================================================
+// Stack manipulation functions.
+
+size_t MacroAssembler::PushRegsInMaskSizeInBytes(LiveRegisterSet set) {
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+ return set.gprs().size() * sizeof(intptr_t) + fpuSet.getPushSizeInBytes();
+}
+
+void MacroAssembler::PushRegsInMask(LiveRegisterSet set) {
+ mozilla::DebugOnly<size_t> framePushedInitial = framePushed();
+
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+ unsigned numFpu = fpuSet.size();
+ int32_t diffF = fpuSet.getPushSizeInBytes();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+
+ // On x86, always use push to push the integer registers, as it's fast
+ // on modern hardware and it's a small instruction.
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ Push(*iter);
+ }
+ MOZ_ASSERT(diffG == 0);
+ (void)diffG;
+
+ reserveStack(diffF);
+ for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ diffF -= reg.size();
+ numFpu -= 1;
+ Address spillAddress(StackPointer, diffF);
+ if (reg.isDouble()) {
+ storeDouble(reg, spillAddress);
+ } else if (reg.isSingle()) {
+ storeFloat32(reg, spillAddress);
+ } else if (reg.isSimd128()) {
+ storeUnalignedSimd128(reg, spillAddress);
+ } else {
+ MOZ_CRASH("Unknown register type.");
+ }
+ }
+ MOZ_ASSERT(numFpu == 0);
+ (void)numFpu;
+
+ // x64 padding to keep the stack aligned on uintptr_t. Keep in sync with
+ // GetPushSizeInBytes.
+ size_t alignExtra = ((size_t)diffF) % sizeof(uintptr_t);
+ MOZ_ASSERT_IF(sizeof(uintptr_t) == 8, alignExtra == 0 || alignExtra == 4);
+ MOZ_ASSERT_IF(sizeof(uintptr_t) == 4, alignExtra == 0);
+ diffF -= alignExtra;
+ MOZ_ASSERT(diffF == 0);
+
+ // The macroassembler will keep the stack sizeof(uintptr_t)-aligned, so
+ // we don't need to take into account `alignExtra` here.
+ MOZ_ASSERT(framePushed() - framePushedInitial ==
+ PushRegsInMaskSizeInBytes(set));
+}
+
+void MacroAssembler::storeRegsInMask(LiveRegisterSet set, Address dest,
+ Register) {
+ mozilla::DebugOnly<size_t> offsetInitial = dest.offset;
+
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+ unsigned numFpu = fpuSet.size();
+ int32_t diffF = fpuSet.getPushSizeInBytes();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+
+ MOZ_ASSERT(dest.offset >= diffG + diffF);
+
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ dest.offset -= sizeof(intptr_t);
+ storePtr(*iter, dest);
+ }
+ MOZ_ASSERT(diffG == 0);
+ (void)diffG;
+
+ for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ diffF -= reg.size();
+ numFpu -= 1;
+ dest.offset -= reg.size();
+ if (reg.isDouble()) {
+ storeDouble(reg, dest);
+ } else if (reg.isSingle()) {
+ storeFloat32(reg, dest);
+ } else if (reg.isSimd128()) {
+ storeUnalignedSimd128(reg, dest);
+ } else {
+ MOZ_CRASH("Unknown register type.");
+ }
+ }
+ MOZ_ASSERT(numFpu == 0);
+ (void)numFpu;
+
+ // x64 padding to keep the stack aligned on uintptr_t. Keep in sync with
+ // GetPushSizeInBytes.
+ size_t alignExtra = ((size_t)diffF) % sizeof(uintptr_t);
+ MOZ_ASSERT_IF(sizeof(uintptr_t) == 8, alignExtra == 0 || alignExtra == 4);
+ MOZ_ASSERT_IF(sizeof(uintptr_t) == 4, alignExtra == 0);
+ diffF -= alignExtra;
+ MOZ_ASSERT(diffF == 0);
+
+ // What this means is: if `alignExtra` is nonzero, then the save area size
+ // actually used is `alignExtra` bytes smaller than what
+ // PushRegsInMaskSizeInBytes claims. Hence we need to compensate for that.
+ MOZ_ASSERT(alignExtra + offsetInitial - dest.offset ==
+ PushRegsInMaskSizeInBytes(set));
+}
+
+void MacroAssembler::PopRegsInMaskIgnore(LiveRegisterSet set,
+ LiveRegisterSet ignore) {
+ mozilla::DebugOnly<size_t> framePushedInitial = framePushed();
+
+ FloatRegisterSet fpuSet(set.fpus().reduceSetForPush());
+ unsigned numFpu = fpuSet.size();
+ int32_t diffG = set.gprs().size() * sizeof(intptr_t);
+ int32_t diffF = fpuSet.getPushSizeInBytes();
+ const int32_t reservedG = diffG;
+ const int32_t reservedF = diffF;
+
+ for (FloatRegisterBackwardIterator iter(fpuSet); iter.more(); ++iter) {
+ FloatRegister reg = *iter;
+ diffF -= reg.size();
+ numFpu -= 1;
+ if (ignore.has(reg)) {
+ continue;
+ }
+
+ Address spillAddress(StackPointer, diffF);
+ if (reg.isDouble()) {
+ loadDouble(spillAddress, reg);
+ } else if (reg.isSingle()) {
+ loadFloat32(spillAddress, reg);
+ } else if (reg.isSimd128()) {
+ loadUnalignedSimd128(spillAddress, reg);
+ } else {
+ MOZ_CRASH("Unknown register type.");
+ }
+ }
+ freeStack(reservedF);
+ MOZ_ASSERT(numFpu == 0);
+ (void)numFpu;
+ // x64 padding to keep the stack aligned on uintptr_t. Keep in sync with
+ // GetPushBytesInSize.
+ diffF -= diffF % sizeof(uintptr_t);
+ MOZ_ASSERT(diffF == 0);
+
+ // On x86, use pop to pop the integer registers, if we're not going to
+ // ignore any slots, as it's fast on modern hardware and it's a small
+ // instruction.
+ if (ignore.emptyGeneral()) {
+ for (GeneralRegisterForwardIterator iter(set.gprs()); iter.more(); ++iter) {
+ diffG -= sizeof(intptr_t);
+ Pop(*iter);
+ }
+ } else {
+ for (GeneralRegisterBackwardIterator iter(set.gprs()); iter.more();
+ ++iter) {
+ diffG -= sizeof(intptr_t);
+ if (!ignore.has(*iter)) {
+ loadPtr(Address(StackPointer, diffG), *iter);
+ }
+ }
+ freeStack(reservedG);
+ }
+ MOZ_ASSERT(diffG == 0);
+
+ MOZ_ASSERT(framePushedInitial - framePushed() ==
+ PushRegsInMaskSizeInBytes(set));
+}
+
+void MacroAssembler::Push(const Operand op) {
+ push(op);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(Register reg) {
+ push(reg);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const Imm32 imm) {
+ push(imm);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const ImmWord imm) {
+ push(imm);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(const ImmPtr imm) {
+ Push(ImmWord(uintptr_t(imm.value)));
+}
+
+void MacroAssembler::Push(const ImmGCPtr ptr) {
+ push(ptr);
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Push(FloatRegister t) {
+ push(t);
+ adjustFrame(sizeof(double));
+}
+
+void MacroAssembler::PushFlags() {
+ pushFlags();
+ adjustFrame(sizeof(intptr_t));
+}
+
+void MacroAssembler::Pop(const Operand op) {
+ pop(op);
+ implicitPop(sizeof(intptr_t));
+}
+
+void MacroAssembler::Pop(Register reg) {
+ pop(reg);
+ implicitPop(sizeof(intptr_t));
+}
+
+void MacroAssembler::Pop(FloatRegister reg) {
+ pop(reg);
+ implicitPop(sizeof(double));
+}
+
+void MacroAssembler::Pop(const ValueOperand& val) {
+ popValue(val);
+ implicitPop(sizeof(Value));
+}
+
+void MacroAssembler::PopFlags() {
+ popFlags();
+ implicitPop(sizeof(intptr_t));
+}
+
+void MacroAssembler::PopStackPtr() { Pop(StackPointer); }
+
+// ===============================================================
+// Simple call functions.
+
+CodeOffset MacroAssembler::call(Register reg) { return Assembler::call(reg); }
+
+CodeOffset MacroAssembler::call(Label* label) { return Assembler::call(label); }
+
+void MacroAssembler::call(const Address& addr) {
+ Assembler::call(Operand(addr.base, addr.offset));
+}
+
+CodeOffset MacroAssembler::call(wasm::SymbolicAddress target) {
+ mov(target, eax);
+ return Assembler::call(eax);
+}
+
+void MacroAssembler::call(ImmWord target) { Assembler::call(target); }
+
+void MacroAssembler::call(ImmPtr target) { Assembler::call(target); }
+
+void MacroAssembler::call(JitCode* target) { Assembler::call(target); }
+
+CodeOffset MacroAssembler::callWithPatch() {
+ return Assembler::callWithPatch();
+}
+void MacroAssembler::patchCall(uint32_t callerOffset, uint32_t calleeOffset) {
+ Assembler::patchCall(callerOffset, calleeOffset);
+}
+
+void MacroAssembler::callAndPushReturnAddress(Register reg) { call(reg); }
+
+void MacroAssembler::callAndPushReturnAddress(Label* label) { call(label); }
+
+// ===============================================================
+// Patchable near/far jumps.
+
+CodeOffset MacroAssembler::farJumpWithPatch() {
+ return Assembler::farJumpWithPatch();
+}
+
+void MacroAssembler::patchFarJump(CodeOffset farJump, uint32_t targetOffset) {
+ Assembler::patchFarJump(farJump, targetOffset);
+}
+
+CodeOffset MacroAssembler::nopPatchableToCall() {
+ masm.nop_five();
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchNopToCall(uint8_t* callsite, uint8_t* target) {
+ Assembler::patchFiveByteNopToCall(callsite, target);
+}
+
+void MacroAssembler::patchCallToNop(uint8_t* callsite) {
+ Assembler::patchCallToFiveByteNop(callsite);
+}
+
+// ===============================================================
+// Jit Frames.
+
+uint32_t MacroAssembler::pushFakeReturnAddress(Register scratch) {
+ CodeLabel cl;
+
+ mov(&cl, scratch);
+ Push(scratch);
+ bind(&cl);
+ uint32_t retAddr = currentOffset();
+
+ addCodeLabel(cl);
+ return retAddr;
+}
+
+// ===============================================================
+// WebAssembly
+
+CodeOffset MacroAssembler::wasmTrapInstruction() { return ud2(); }
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Register boundsCheckLimit, Label* ok) {
+ cmp32(index, boundsCheckLimit);
+ j(cond, ok);
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCl(cond, Operand(boundsCheckLimit), index);
+ }
+}
+
+void MacroAssembler::wasmBoundsCheck32(Condition cond, Register index,
+ Address boundsCheckLimit, Label* ok) {
+ cmp32(index, Operand(boundsCheckLimit));
+ j(cond, ok);
+ if (JitOptions.spectreIndexMasking) {
+ cmovCCl(cond, Operand(boundsCheckLimit), index);
+ }
+}
+
+// RAII class that generates the jumps to traps when it's destructed, to
+// prevent some code duplication in the outOfLineWasmTruncateXtoY methods.
+struct MOZ_RAII AutoHandleWasmTruncateToIntErrors {
+ MacroAssembler& masm;
+ Label inputIsNaN;
+ Label intOverflow;
+ wasm::BytecodeOffset off;
+
+ explicit AutoHandleWasmTruncateToIntErrors(MacroAssembler& masm,
+ wasm::BytecodeOffset off)
+ : masm(masm), off(off) {}
+
+ ~AutoHandleWasmTruncateToIntErrors() {
+ // Handle errors. These cases are not in arbitrary order: code will
+ // fall through to intOverflow.
+ masm.bind(&intOverflow);
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, off);
+
+ masm.bind(&inputIsNaN);
+ masm.wasmTrap(wasm::Trap::InvalidConversionToInteger, off);
+ }
+};
+
+void MacroAssembler::wasmTruncateDoubleToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ vcvttsd2si(input, output);
+ cmp32(output, Imm32(1));
+ j(Assembler::Overflow, oolEntry);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ vcvttss2si(input, output);
+ cmp32(output, Imm32(1));
+ j(Assembler::Overflow, oolEntry);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ if (isSaturating) {
+ if (isUnsigned) {
+ // Negative overflow and NaN both are converted to 0, and the only
+ // other case is positive overflow which is converted to
+ // UINT32_MAX.
+ Label nonNegative;
+ ScratchDoubleScope fpscratch(*this);
+ loadConstantDouble(0.0, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &nonNegative);
+ move32(Imm32(0), output);
+ jump(rejoin);
+
+ bind(&nonNegative);
+ move32(Imm32(UINT32_MAX), output);
+ } else {
+ // Negative overflow is already saturated to INT32_MIN, so we only
+ // have to handle NaN and positive overflow here.
+ Label notNaN;
+ branchDouble(Assembler::DoubleOrdered, input, input, &notNaN);
+ move32(Imm32(0), output);
+ jump(rejoin);
+
+ bind(&notNaN);
+ ScratchDoubleScope fpscratch(*this);
+ loadConstantDouble(0.0, fpscratch);
+ branchDouble(Assembler::DoubleLessThan, input, fpscratch, rejoin);
+ sub32(Imm32(1), output);
+ }
+ jump(rejoin);
+ return;
+ }
+
+ AutoHandleWasmTruncateToIntErrors traps(*this, off);
+
+ // Eagerly take care of NaNs.
+ branchDouble(Assembler::DoubleUnordered, input, input, &traps.inputIsNaN);
+
+ // For unsigned, fall through to intOverflow failure case.
+ if (isUnsigned) {
+ return;
+ }
+
+ // Handle special values.
+
+ // We've used vcvttsd2si. The only valid double values that can
+ // truncate to INT32_MIN are in ]INT32_MIN - 1; INT32_MIN].
+ ScratchDoubleScope fpscratch(*this);
+ loadConstantDouble(double(INT32_MIN) - 1.0, fpscratch);
+ branchDouble(Assembler::DoubleLessThanOrEqual, input, fpscratch,
+ &traps.intOverflow);
+
+ loadConstantDouble(0.0, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThan, input, fpscratch,
+ &traps.intOverflow);
+ jump(rejoin);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI32(FloatRegister input,
+ Register output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ if (isSaturating) {
+ if (isUnsigned) {
+ // Negative overflow and NaN both are converted to 0, and the only
+ // other case is positive overflow which is converted to
+ // UINT32_MAX.
+ Label nonNegative;
+ ScratchFloat32Scope fpscratch(*this);
+ loadConstantFloat32(0.0f, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, input, fpscratch,
+ &nonNegative);
+ move32(Imm32(0), output);
+ jump(rejoin);
+
+ bind(&nonNegative);
+ move32(Imm32(UINT32_MAX), output);
+ } else {
+ // Negative overflow is already saturated to INT32_MIN, so we only
+ // have to handle NaN and positive overflow here.
+ Label notNaN;
+ branchFloat(Assembler::DoubleOrdered, input, input, &notNaN);
+ move32(Imm32(0), output);
+ jump(rejoin);
+
+ bind(&notNaN);
+ ScratchFloat32Scope fpscratch(*this);
+ loadConstantFloat32(0.0f, fpscratch);
+ branchFloat(Assembler::DoubleLessThan, input, fpscratch, rejoin);
+ sub32(Imm32(1), output);
+ }
+ jump(rejoin);
+ return;
+ }
+
+ AutoHandleWasmTruncateToIntErrors traps(*this, off);
+
+ // Eagerly take care of NaNs.
+ branchFloat(Assembler::DoubleUnordered, input, input, &traps.inputIsNaN);
+
+ // For unsigned, fall through to intOverflow failure case.
+ if (isUnsigned) {
+ return;
+ }
+
+ // Handle special values.
+
+ // We've used vcvttss2si. Check that the input wasn't
+ // float(INT32_MIN), which is the only legimitate input that
+ // would truncate to INT32_MIN.
+ ScratchFloat32Scope fpscratch(*this);
+ loadConstantFloat32(float(INT32_MIN), fpscratch);
+ branchFloat(Assembler::DoubleNotEqual, input, fpscratch, &traps.intOverflow);
+ jump(rejoin);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF64ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ if (isSaturating) {
+ if (isUnsigned) {
+ // Negative overflow and NaN both are converted to 0, and the only
+ // other case is positive overflow which is converted to
+ // UINT64_MAX.
+ Label positive;
+ ScratchDoubleScope fpscratch(*this);
+ loadConstantDouble(0.0, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThan, input, fpscratch, &positive);
+ move64(Imm64(0), output);
+ jump(rejoin);
+
+ bind(&positive);
+ move64(Imm64(UINT64_MAX), output);
+ } else {
+ // Negative overflow is already saturated to INT64_MIN, so we only
+ // have to handle NaN and positive overflow here.
+ Label notNaN;
+ branchDouble(Assembler::DoubleOrdered, input, input, &notNaN);
+ move64(Imm64(0), output);
+ jump(rejoin);
+
+ bind(&notNaN);
+ ScratchDoubleScope fpscratch(*this);
+ loadConstantDouble(0.0, fpscratch);
+ branchDouble(Assembler::DoubleLessThan, input, fpscratch, rejoin);
+ sub64(Imm64(1), output);
+ }
+ jump(rejoin);
+ return;
+ }
+
+ AutoHandleWasmTruncateToIntErrors traps(*this, off);
+
+ // Eagerly take care of NaNs.
+ branchDouble(Assembler::DoubleUnordered, input, input, &traps.inputIsNaN);
+
+ // Handle special values.
+ if (isUnsigned) {
+ ScratchDoubleScope fpscratch(*this);
+ loadConstantDouble(0.0, fpscratch);
+ branchDouble(Assembler::DoubleGreaterThan, input, fpscratch,
+ &traps.intOverflow);
+ loadConstantDouble(-1.0, fpscratch);
+ branchDouble(Assembler::DoubleLessThanOrEqual, input, fpscratch,
+ &traps.intOverflow);
+ jump(rejoin);
+ return;
+ }
+
+ // We've used vcvtsd2sq. The only legit value whose i64
+ // truncation is INT64_MIN is double(INT64_MIN): exponent is so
+ // high that the highest resolution around is much more than 1.
+ ScratchDoubleScope fpscratch(*this);
+ loadConstantDouble(double(int64_t(INT64_MIN)), fpscratch);
+ branchDouble(Assembler::DoubleNotEqual, input, fpscratch, &traps.intOverflow);
+ jump(rejoin);
+}
+
+void MacroAssembler::oolWasmTruncateCheckF32ToI64(FloatRegister input,
+ Register64 output,
+ TruncFlags flags,
+ wasm::BytecodeOffset off,
+ Label* rejoin) {
+ bool isUnsigned = flags & TRUNC_UNSIGNED;
+ bool isSaturating = flags & TRUNC_SATURATING;
+
+ if (isSaturating) {
+ if (isUnsigned) {
+ // Negative overflow and NaN both are converted to 0, and the only
+ // other case is positive overflow which is converted to
+ // UINT64_MAX.
+ Label positive;
+ ScratchFloat32Scope fpscratch(*this);
+ loadConstantFloat32(0.0f, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThan, input, fpscratch, &positive);
+ move64(Imm64(0), output);
+ jump(rejoin);
+
+ bind(&positive);
+ move64(Imm64(UINT64_MAX), output);
+ } else {
+ // Negative overflow is already saturated to INT64_MIN, so we only
+ // have to handle NaN and positive overflow here.
+ Label notNaN;
+ branchFloat(Assembler::DoubleOrdered, input, input, &notNaN);
+ move64(Imm64(0), output);
+ jump(rejoin);
+
+ bind(&notNaN);
+ ScratchFloat32Scope fpscratch(*this);
+ loadConstantFloat32(0.0f, fpscratch);
+ branchFloat(Assembler::DoubleLessThan, input, fpscratch, rejoin);
+ sub64(Imm64(1), output);
+ }
+ jump(rejoin);
+ return;
+ }
+
+ AutoHandleWasmTruncateToIntErrors traps(*this, off);
+
+ // Eagerly take care of NaNs.
+ branchFloat(Assembler::DoubleUnordered, input, input, &traps.inputIsNaN);
+
+ // Handle special values.
+ if (isUnsigned) {
+ ScratchFloat32Scope fpscratch(*this);
+ loadConstantFloat32(0.0f, fpscratch);
+ branchFloat(Assembler::DoubleGreaterThan, input, fpscratch,
+ &traps.intOverflow);
+ loadConstantFloat32(-1.0f, fpscratch);
+ branchFloat(Assembler::DoubleLessThanOrEqual, input, fpscratch,
+ &traps.intOverflow);
+ jump(rejoin);
+ return;
+ }
+
+ // We've used vcvtss2sq. See comment in outOfLineWasmTruncateDoubleToInt64.
+ ScratchFloat32Scope fpscratch(*this);
+ loadConstantFloat32(float(int64_t(INT64_MIN)), fpscratch);
+ branchFloat(Assembler::DoubleNotEqual, input, fpscratch, &traps.intOverflow);
+ jump(rejoin);
+}
+
+void MacroAssembler::enterFakeExitFrameForWasm(Register cxreg, Register scratch,
+ ExitFrameType type) {
+ enterFakeExitFrame(cxreg, scratch, type);
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+static void ExtendTo32(MacroAssembler& masm, Scalar::Type type, Register r) {
+ switch (Scalar::byteSize(type)) {
+ case 1:
+ if (Scalar::isSignedIntType(type)) {
+ masm.movsbl(r, r);
+ } else {
+ masm.movzbl(r, r);
+ }
+ break;
+ case 2:
+ if (Scalar::isSignedIntType(type)) {
+ masm.movswl(r, r);
+ } else {
+ masm.movzwl(r, r);
+ }
+ break;
+ default:
+ break;
+ }
+}
+
+static inline void CheckBytereg(Register r) {
+#ifdef DEBUG
+ AllocatableGeneralRegisterSet byteRegs(Registers::SingleByteRegs);
+ MOZ_ASSERT(byteRegs.has(r));
+#endif
+}
+
+static inline void CheckBytereg(Imm32 r) {
+ // Nothing
+}
+
+template <typename T>
+static void CompareExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const T& mem, Register oldval,
+ Register newval, Register output) {
+ MOZ_ASSERT(output == eax);
+
+ if (oldval != output) {
+ masm.movl(oldval, output);
+ }
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ // NOTE: the generated code must match the assembly code in gen_cmpxchg in
+ // GenerateAtomicOperations.py
+ switch (Scalar::byteSize(type)) {
+ case 1:
+ CheckBytereg(newval);
+ masm.lock_cmpxchgb(newval, Operand(mem));
+ break;
+ case 2:
+ masm.lock_cmpxchgw(newval, Operand(mem));
+ break;
+ case 4:
+ masm.lock_cmpxchgl(newval, Operand(mem));
+ break;
+ }
+
+ ExtendTo32(masm, type, output);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type, const Synchronization&,
+ const Address& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, nullptr, type, mem, oldval, newval, output);
+}
+
+void MacroAssembler::compareExchange(Scalar::Type type, const Synchronization&,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, nullptr, type, mem, oldval, newval, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, &access, access.type(), mem, oldval, newval, output);
+}
+
+void MacroAssembler::wasmCompareExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register output) {
+ CompareExchange(*this, &access, access.type(), mem, oldval, newval, output);
+}
+
+template <typename T>
+static void AtomicExchange(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type type, const T& mem, Register value,
+ Register output)
+// NOTE: the generated code must match the assembly code in gen_exchange in
+// GenerateAtomicOperations.py
+{
+ if (value != output) {
+ masm.movl(value, output);
+ }
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ switch (Scalar::byteSize(type)) {
+ case 1:
+ CheckBytereg(output);
+ masm.xchgb(output, Operand(mem));
+ break;
+ case 2:
+ masm.xchgw(output, Operand(mem));
+ break;
+ case 4:
+ masm.xchgl(output, Operand(mem));
+ break;
+ default:
+ MOZ_CRASH("Invalid");
+ }
+ ExtendTo32(masm, type, output);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type, const Synchronization&,
+ const Address& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, nullptr, type, mem, value, output);
+}
+
+void MacroAssembler::atomicExchange(Scalar::Type type, const Synchronization&,
+ const BaseIndex& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, nullptr, type, mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, &access, access.type(), mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register value,
+ Register output) {
+ AtomicExchange(*this, &access, access.type(), mem, value, output);
+}
+
+static void SetupValue(MacroAssembler& masm, AtomicOp op, Imm32 src,
+ Register output) {
+ if (op == AtomicFetchSubOp) {
+ masm.movl(Imm32(-src.value), output);
+ } else {
+ masm.movl(src, output);
+ }
+}
+
+static void SetupValue(MacroAssembler& masm, AtomicOp op, Register src,
+ Register output) {
+ if (src != output) {
+ masm.movl(src, output);
+ }
+ if (op == AtomicFetchSubOp) {
+ masm.negl(output);
+ }
+}
+
+template <typename T, typename V>
+static void AtomicFetchOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type arrayType, AtomicOp op, V value,
+ const T& mem, Register temp, Register output) {
+ // Note value can be an Imm or a Register.
+
+ // NOTE: the generated code must match the assembly code in gen_fetchop in
+ // GenerateAtomicOperations.py
+#define ATOMIC_BITOP_BODY(LOAD, OP, LOCK_CMPXCHG) \
+ do { \
+ MOZ_ASSERT(output != temp); \
+ MOZ_ASSERT(output == eax); \
+ if (access) masm.append(*access, masm.size()); \
+ masm.LOAD(Operand(mem), eax); \
+ Label again; \
+ masm.bind(&again); \
+ masm.movl(eax, temp); \
+ masm.OP(value, temp); \
+ masm.LOCK_CMPXCHG(temp, Operand(mem)); \
+ masm.j(MacroAssembler::NonZero, &again); \
+ } while (0)
+
+ MOZ_ASSERT_IF(op == AtomicFetchAddOp || op == AtomicFetchSubOp,
+ temp == InvalidReg);
+
+ switch (Scalar::byteSize(arrayType)) {
+ case 1:
+ CheckBytereg(output);
+ switch (op) {
+ case AtomicFetchAddOp:
+ case AtomicFetchSubOp:
+ CheckBytereg(value); // But not for the bitwise ops
+ SetupValue(masm, op, value, output);
+ if (access) masm.append(*access, masm.size());
+ masm.lock_xaddb(output, Operand(mem));
+ break;
+ case AtomicFetchAndOp:
+ CheckBytereg(temp);
+ ATOMIC_BITOP_BODY(movb, andl, lock_cmpxchgb);
+ break;
+ case AtomicFetchOrOp:
+ CheckBytereg(temp);
+ ATOMIC_BITOP_BODY(movb, orl, lock_cmpxchgb);
+ break;
+ case AtomicFetchXorOp:
+ CheckBytereg(temp);
+ ATOMIC_BITOP_BODY(movb, xorl, lock_cmpxchgb);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case 2:
+ switch (op) {
+ case AtomicFetchAddOp:
+ case AtomicFetchSubOp:
+ SetupValue(masm, op, value, output);
+ if (access) masm.append(*access, masm.size());
+ masm.lock_xaddw(output, Operand(mem));
+ break;
+ case AtomicFetchAndOp:
+ ATOMIC_BITOP_BODY(movw, andl, lock_cmpxchgw);
+ break;
+ case AtomicFetchOrOp:
+ ATOMIC_BITOP_BODY(movw, orl, lock_cmpxchgw);
+ break;
+ case AtomicFetchXorOp:
+ ATOMIC_BITOP_BODY(movw, xorl, lock_cmpxchgw);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case 4:
+ switch (op) {
+ case AtomicFetchAddOp:
+ case AtomicFetchSubOp:
+ SetupValue(masm, op, value, output);
+ if (access) masm.append(*access, masm.size());
+ masm.lock_xaddl(output, Operand(mem));
+ break;
+ case AtomicFetchAndOp:
+ ATOMIC_BITOP_BODY(movl, andl, lock_cmpxchgl);
+ break;
+ case AtomicFetchOrOp:
+ ATOMIC_BITOP_BODY(movl, orl, lock_cmpxchgl);
+ break;
+ case AtomicFetchXorOp:
+ ATOMIC_BITOP_BODY(movl, xorl, lock_cmpxchgl);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ }
+ ExtendTo32(masm, arrayType, output);
+
+#undef ATOMIC_BITOP_BODY
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type arrayType,
+ const Synchronization&, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp, Register output) {
+ AtomicFetchOp(*this, nullptr, arrayType, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type arrayType,
+ const Synchronization&, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp, Register output) {
+ AtomicFetchOp(*this, nullptr, arrayType, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type arrayType,
+ const Synchronization&, AtomicOp op,
+ Imm32 value, const BaseIndex& mem,
+ Register temp, Register output) {
+ AtomicFetchOp(*this, nullptr, arrayType, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp(Scalar::Type arrayType,
+ const Synchronization&, AtomicOp op,
+ Imm32 value, const Address& mem,
+ Register temp, Register output) {
+ AtomicFetchOp(*this, nullptr, arrayType, op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register temp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Imm32 value,
+ const Address& mem, Register temp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem, Register temp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Imm32 value,
+ const BaseIndex& mem, Register temp,
+ Register output) {
+ AtomicFetchOp(*this, &access, access.type(), op, value, mem, temp, output);
+}
+
+template <typename T, typename V>
+static void AtomicEffectOp(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ Scalar::Type arrayType, AtomicOp op, V value,
+ const T& mem) {
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+
+ switch (Scalar::byteSize(arrayType)) {
+ case 1:
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.lock_addb(value, Operand(mem));
+ break;
+ case AtomicFetchSubOp:
+ masm.lock_subb(value, Operand(mem));
+ break;
+ case AtomicFetchAndOp:
+ masm.lock_andb(value, Operand(mem));
+ break;
+ case AtomicFetchOrOp:
+ masm.lock_orb(value, Operand(mem));
+ break;
+ case AtomicFetchXorOp:
+ masm.lock_xorb(value, Operand(mem));
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case 2:
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.lock_addw(value, Operand(mem));
+ break;
+ case AtomicFetchSubOp:
+ masm.lock_subw(value, Operand(mem));
+ break;
+ case AtomicFetchAndOp:
+ masm.lock_andw(value, Operand(mem));
+ break;
+ case AtomicFetchOrOp:
+ masm.lock_orw(value, Operand(mem));
+ break;
+ case AtomicFetchXorOp:
+ masm.lock_xorw(value, Operand(mem));
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ case 4:
+ switch (op) {
+ case AtomicFetchAddOp:
+ masm.lock_addl(value, Operand(mem));
+ break;
+ case AtomicFetchSubOp:
+ masm.lock_subl(value, Operand(mem));
+ break;
+ case AtomicFetchAndOp:
+ masm.lock_andl(value, Operand(mem));
+ break;
+ case AtomicFetchOrOp:
+ masm.lock_orl(value, Operand(mem));
+ break;
+ case AtomicFetchXorOp:
+ masm.lock_xorl(value, Operand(mem));
+ break;
+ default:
+ MOZ_CRASH();
+ }
+ break;
+ default:
+ MOZ_CRASH();
+ }
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const Address& mem, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ AtomicEffectOp(*this, &access, access.type(), op, value, mem);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Imm32 value,
+ const Address& mem, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ AtomicEffectOp(*this, &access, access.type(), op, value, mem);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Register value,
+ const BaseIndex& mem, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ AtomicEffectOp(*this, &access, access.type(), op, value, mem);
+}
+
+void MacroAssembler::wasmAtomicEffectOp(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, Imm32 value,
+ const BaseIndex& mem, Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ AtomicEffectOp(*this, &access, access.type(), op, value, mem);
+}
+
+// ========================================================================
+// JS atomic operations.
+
+template <typename T>
+static void CompareExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register oldval, Register newval, Register temp,
+ AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.compareExchange(arrayType, sync, mem, oldval, newval, output.gpr());
+ }
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register oldval,
+ Register newval, Register temp,
+ AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
+}
+
+void MacroAssembler::compareExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register oldval,
+ Register newval, Register temp,
+ AnyRegister output) {
+ CompareExchangeJS(*this, arrayType, sync, mem, oldval, newval, temp, output);
+}
+
+template <typename T>
+static void AtomicExchangeJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, const T& mem,
+ Register value, Register temp,
+ AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicExchange(arrayType, sync, mem, value, temp);
+ masm.convertUInt32ToDouble(temp, output.fpu());
+ } else {
+ masm.atomicExchange(arrayType, sync, mem, value, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const Address& mem, Register value,
+ Register temp, AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
+}
+
+void MacroAssembler::atomicExchangeJS(Scalar::Type arrayType,
+ const Synchronization& sync,
+ const BaseIndex& mem, Register value,
+ Register temp, AnyRegister output) {
+ AtomicExchangeJS(*this, arrayType, sync, mem, value, temp, output);
+}
+
+template <typename T>
+static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const T& mem, Register temp1,
+ Register temp2, AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, temp2, temp1);
+ masm.convertUInt32ToDouble(temp1, output.fpu());
+ } else {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, temp1, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp1, Register temp2,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp1, Register temp2,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization&, AtomicOp op,
+ Register value, const BaseIndex& mem,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ AtomicEffectOp(*this, nullptr, arrayType, op, value, mem);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization&, AtomicOp op,
+ Register value, const Address& mem,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ AtomicEffectOp(*this, nullptr, arrayType, op, value, mem);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization&, AtomicOp op,
+ Imm32 value, const Address& mem,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ AtomicEffectOp(*this, nullptr, arrayType, op, value, mem);
+}
+
+void MacroAssembler::atomicEffectOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Imm32 value, const BaseIndex& mem,
+ Register temp) {
+ MOZ_ASSERT(temp == InvalidReg);
+ AtomicEffectOp(*this, nullptr, arrayType, op, value, mem);
+}
+
+template <typename T>
+static void AtomicFetchOpJS(MacroAssembler& masm, Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Imm32 value, const T& mem, Register temp1,
+ Register temp2, AnyRegister output) {
+ if (arrayType == Scalar::Uint32) {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, temp2, temp1);
+ masm.convertUInt32ToDouble(temp1, output.fpu());
+ } else {
+ masm.atomicFetchOp(arrayType, sync, op, value, mem, temp1, output.gpr());
+ }
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Imm32 value, const Address& mem,
+ Register temp1, Register temp2,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
+}
+
+void MacroAssembler::atomicFetchOpJS(Scalar::Type arrayType,
+ const Synchronization& sync, AtomicOp op,
+ Imm32 value, const BaseIndex& mem,
+ Register temp1, Register temp2,
+ AnyRegister output) {
+ AtomicFetchOpJS(*this, arrayType, sync, op, value, mem, temp1, temp2, output);
+}
+
+// ========================================================================
+// Spectre Mitigations.
+
+void MacroAssembler::speculationBarrier() {
+ // Spectre mitigation recommended by Intel and AMD suggest to use lfence as
+ // a way to force all speculative execution of instructions to end.
+ MOZ_ASSERT(HasSSE2());
+ masm.lfence();
+}
+
+void MacroAssembler::floorFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ if (HasSSE41()) {
+ // Fail on negative-zero.
+ branchNegativeZeroFloat32(src, dest, fail);
+
+ // Round toward -Infinity.
+ {
+ ScratchFloat32Scope scratch(*this);
+ vroundss(X86Encoding::RoundDown, src, scratch);
+ truncateFloat32ToInt32(scratch, dest, fail);
+ }
+ } else {
+ Label negative, end;
+
+ // Branch to a slow path for negative inputs. Doesn't catch NaN or -0.
+ {
+ ScratchFloat32Scope scratch(*this);
+ zeroFloat32(scratch);
+ branchFloat(Assembler::DoubleLessThan, src, scratch, &negative);
+ }
+
+ // Fail on negative-zero.
+ branchNegativeZeroFloat32(src, dest, fail);
+
+ // Input is non-negative, so truncation correctly rounds.
+ truncateFloat32ToInt32(src, dest, fail);
+ jump(&end);
+
+ // Input is negative, but isn't -0.
+ // Negative values go on a comparatively expensive path, since no
+ // native rounding mode matches JS semantics. Still better than callVM.
+ bind(&negative);
+ {
+ // Truncate and round toward zero.
+ // This is off-by-one for everything but integer-valued inputs.
+ truncateFloat32ToInt32(src, dest, fail);
+
+ // Test whether the input double was integer-valued.
+ {
+ ScratchFloat32Scope scratch(*this);
+ convertInt32ToFloat32(dest, scratch);
+ branchFloat(Assembler::DoubleEqualOrUnordered, src, scratch, &end);
+ }
+
+ // Input is not integer-valued, so we rounded off-by-one in the
+ // wrong direction. Correct by subtraction.
+ subl(Imm32(1), dest);
+ // Cannot overflow: output was already checked against INT_MIN.
+ }
+
+ bind(&end);
+ }
+}
+
+void MacroAssembler::floorDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ if (HasSSE41()) {
+ // Fail on negative-zero.
+ branchNegativeZero(src, dest, fail);
+
+ // Round toward -Infinity.
+ {
+ ScratchDoubleScope scratch(*this);
+ vroundsd(X86Encoding::RoundDown, src, scratch);
+ truncateDoubleToInt32(scratch, dest, fail);
+ }
+ } else {
+ Label negative, end;
+
+ // Branch to a slow path for negative inputs. Doesn't catch NaN or -0.
+ {
+ ScratchDoubleScope scratch(*this);
+ zeroDouble(scratch);
+ branchDouble(Assembler::DoubleLessThan, src, scratch, &negative);
+ }
+
+ // Fail on negative-zero.
+ branchNegativeZero(src, dest, fail);
+
+ // Input is non-negative, so truncation correctly rounds.
+ truncateDoubleToInt32(src, dest, fail);
+ jump(&end);
+
+ // Input is negative, but isn't -0.
+ // Negative values go on a comparatively expensive path, since no
+ // native rounding mode matches JS semantics. Still better than callVM.
+ bind(&negative);
+ {
+ // Truncate and round toward zero.
+ // This is off-by-one for everything but integer-valued inputs.
+ truncateDoubleToInt32(src, dest, fail);
+
+ // Test whether the input double was integer-valued.
+ {
+ ScratchDoubleScope scratch(*this);
+ convertInt32ToDouble(dest, scratch);
+ branchDouble(Assembler::DoubleEqualOrUnordered, src, scratch, &end);
+ }
+
+ // Input is not integer-valued, so we rounded off-by-one in the
+ // wrong direction. Correct by subtraction.
+ subl(Imm32(1), dest);
+ // Cannot overflow: output was already checked against INT_MIN.
+ }
+
+ bind(&end);
+ }
+}
+
+void MacroAssembler::ceilFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchFloat32Scope scratch(*this);
+
+ Label lessThanOrEqualMinusOne;
+
+ // If x is in ]-1,0], ceil(x) is -0, which cannot be represented as an int32.
+ // Fail if x > -1 and the sign bit is set.
+ loadConstantFloat32(-1.f, scratch);
+ branchFloat(Assembler::DoubleLessThanOrEqualOrUnordered, src, scratch,
+ &lessThanOrEqualMinusOne);
+ vmovmskps(src, dest);
+ branchTest32(Assembler::NonZero, dest, Imm32(1), fail);
+
+ if (HasSSE41()) {
+ // x <= -1 or x > -0
+ bind(&lessThanOrEqualMinusOne);
+ // Round toward +Infinity.
+ vroundss(X86Encoding::RoundUp, src, scratch);
+ truncateFloat32ToInt32(scratch, dest, fail);
+ return;
+ }
+
+ // No SSE4.1
+ Label end;
+
+ // x >= 0 and x is not -0.0. We can truncate integer values, and truncate and
+ // add 1 to non-integer values. This will also work for values >= INT_MAX + 1,
+ // as the truncate operation will return INT_MIN and we'll fail.
+ truncateFloat32ToInt32(src, dest, fail);
+ convertInt32ToFloat32(dest, scratch);
+ branchFloat(Assembler::DoubleEqualOrUnordered, src, scratch, &end);
+
+ // Input is not integer-valued, add 1 to obtain the ceiling value.
+ // If input > INT_MAX, output == INT_MAX so adding 1 will overflow.
+ branchAdd32(Assembler::Overflow, Imm32(1), dest, fail);
+ jump(&end);
+
+ // x <= -1, truncation is the way to go.
+ bind(&lessThanOrEqualMinusOne);
+ truncateFloat32ToInt32(src, dest, fail);
+
+ bind(&end);
+}
+
+void MacroAssembler::ceilDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ ScratchDoubleScope scratch(*this);
+
+ Label lessThanOrEqualMinusOne;
+
+ // If x is in ]-1,0], ceil(x) is -0, which cannot be represented as an int32.
+ // Fail if x > -1 and the sign bit is set.
+ loadConstantDouble(-1.0, scratch);
+ branchDouble(Assembler::DoubleLessThanOrEqualOrUnordered, src, scratch,
+ &lessThanOrEqualMinusOne);
+ vmovmskpd(src, dest);
+ branchTest32(Assembler::NonZero, dest, Imm32(1), fail);
+
+ if (HasSSE41()) {
+ // x <= -1 or x > -0
+ bind(&lessThanOrEqualMinusOne);
+ // Round toward +Infinity.
+ vroundsd(X86Encoding::RoundUp, src, scratch);
+ truncateDoubleToInt32(scratch, dest, fail);
+ return;
+ }
+
+ // No SSE4.1
+ Label end;
+
+ // x >= 0 and x is not -0.0. We can truncate integer values, and truncate and
+ // add 1 to non-integer values. This will also work for values >= INT_MAX + 1,
+ // as the truncate operation will return INT_MIN and we'll fail.
+ truncateDoubleToInt32(src, dest, fail);
+ convertInt32ToDouble(dest, scratch);
+ branchDouble(Assembler::DoubleEqualOrUnordered, src, scratch, &end);
+
+ // Input is not integer-valued, add 1 to obtain the ceiling value.
+ // If input > INT_MAX, output == INT_MAX so adding 1 will overflow.
+ branchAdd32(Assembler::Overflow, Imm32(1), dest, fail);
+ jump(&end);
+
+ // x <= -1, truncation is the way to go.
+ bind(&lessThanOrEqualMinusOne);
+ truncateDoubleToInt32(src, dest, fail);
+
+ bind(&end);
+}
+
+void MacroAssembler::truncDoubleToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ Label lessThanOrEqualMinusOne;
+
+ // Bail on ]-1; -0] range
+ {
+ ScratchDoubleScope scratch(*this);
+ loadConstantDouble(-1, scratch);
+ branchDouble(Assembler::DoubleLessThanOrEqualOrUnordered, src, scratch,
+ &lessThanOrEqualMinusOne);
+ }
+
+ // Test for remaining values with the sign bit set, i.e. ]-1; -0]
+ vmovmskpd(src, dest);
+ branchTest32(Assembler::NonZero, dest, Imm32(1), fail);
+
+ // x <= -1 or x >= +0, truncation is the way to go.
+ bind(&lessThanOrEqualMinusOne);
+ truncateDoubleToInt32(src, dest, fail);
+}
+
+void MacroAssembler::truncFloat32ToInt32(FloatRegister src, Register dest,
+ Label* fail) {
+ Label lessThanOrEqualMinusOne;
+
+ // Bail on ]-1; -0] range
+ {
+ ScratchFloat32Scope scratch(*this);
+ loadConstantFloat32(-1.f, scratch);
+ branchFloat(Assembler::DoubleLessThanOrEqualOrUnordered, src, scratch,
+ &lessThanOrEqualMinusOne);
+ }
+
+ // Test for remaining values with the sign bit set, i.e. ]-1; -0]
+ vmovmskps(src, dest);
+ branchTest32(Assembler::NonZero, dest, Imm32(1), fail);
+
+ // x <= -1 or x >= +0, truncation is the way to go.
+ bind(&lessThanOrEqualMinusOne);
+ truncateFloat32ToInt32(src, dest, fail);
+}
+
+void MacroAssembler::roundFloat32ToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ ScratchFloat32Scope scratch(*this);
+
+ Label negativeOrZero, negative, end;
+
+ // Branch to a slow path for non-positive inputs. Doesn't catch NaN.
+ zeroFloat32(scratch);
+ loadConstantFloat32(GetBiggestNumberLessThan(0.5f), temp);
+ branchFloat(Assembler::DoubleLessThanOrEqual, src, scratch, &negativeOrZero);
+ {
+ // Input is non-negative. Add the biggest float less than 0.5 and truncate,
+ // rounding down (because if the input is the biggest float less than 0.5,
+ // adding 0.5 would undesirably round up to 1). Note that we have to add the
+ // input to the temp register because we're not allowed to modify the input
+ // register.
+ addFloat32(src, temp);
+ truncateFloat32ToInt32(temp, dest, fail);
+ jump(&end);
+ }
+
+ // Input is negative, +0 or -0.
+ bind(&negativeOrZero);
+ {
+ // Branch on negative input.
+ j(Assembler::NotEqual, &negative);
+
+ // Fail on negative-zero.
+ branchNegativeZeroFloat32(src, dest, fail);
+
+ // Input is +0.
+ xor32(dest, dest);
+ jump(&end);
+ }
+
+ // Input is negative.
+ bind(&negative);
+ {
+ // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to
+ // be added the biggest double less than 0.5.
+ Label loadJoin;
+ loadConstantFloat32(-0.5f, scratch);
+ branchFloat(Assembler::DoubleLessThan, src, scratch, &loadJoin);
+ loadConstantFloat32(0.5f, temp);
+ bind(&loadJoin);
+
+ if (HasSSE41()) {
+ // Add 0.5 and round toward -Infinity. The result is stored in the temp
+ // register (currently contains 0.5).
+ addFloat32(src, temp);
+ vroundss(X86Encoding::RoundDown, temp, scratch);
+
+ // Truncate.
+ truncateFloat32ToInt32(scratch, dest, fail);
+
+ // If the result is positive zero, then the actual result is -0. Fail.
+ // Otherwise, the truncation will have produced the correct negative
+ // integer.
+ branchTest32(Assembler::Zero, dest, dest, fail);
+ } else {
+ addFloat32(src, temp);
+ // Round toward -Infinity without the benefit of ROUNDSS.
+ {
+ // If input + 0.5 >= 0, input is a negative number >= -0.5 and the
+ // result is -0.
+ branchFloat(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail);
+
+ // Truncate and round toward zero.
+ // This is off-by-one for everything but integer-valued inputs.
+ truncateFloat32ToInt32(temp, dest, fail);
+
+ // Test whether the truncated double was integer-valued.
+ convertInt32ToFloat32(dest, scratch);
+ branchFloat(Assembler::DoubleEqualOrUnordered, temp, scratch, &end);
+
+ // Input is not integer-valued, so we rounded off-by-one in the
+ // wrong direction. Correct by subtraction.
+ subl(Imm32(1), dest);
+ // Cannot overflow: output was already checked against INT_MIN.
+ }
+ }
+ }
+
+ bind(&end);
+}
+
+void MacroAssembler::roundDoubleToInt32(FloatRegister src, Register dest,
+ FloatRegister temp, Label* fail) {
+ ScratchDoubleScope scratch(*this);
+
+ Label negativeOrZero, negative, end;
+
+ // Branch to a slow path for non-positive inputs. Doesn't catch NaN.
+ zeroDouble(scratch);
+ loadConstantDouble(GetBiggestNumberLessThan(0.5), temp);
+ branchDouble(Assembler::DoubleLessThanOrEqual, src, scratch, &negativeOrZero);
+ {
+ // Input is positive. Add the biggest double less than 0.5 and truncate,
+ // rounding down (because if the input is the biggest double less than 0.5,
+ // adding 0.5 would undesirably round up to 1). Note that we have to add the
+ // input to the temp register because we're not allowed to modify the input
+ // register.
+ addDouble(src, temp);
+ truncateDoubleToInt32(temp, dest, fail);
+ jump(&end);
+ }
+
+ // Input is negative, +0 or -0.
+ bind(&negativeOrZero);
+ {
+ // Branch on negative input.
+ j(Assembler::NotEqual, &negative);
+
+ // Fail on negative-zero.
+ branchNegativeZero(src, dest, fail, /* maybeNonZero = */ false);
+
+ // Input is +0
+ xor32(dest, dest);
+ jump(&end);
+ }
+
+ // Input is negative.
+ bind(&negative);
+ {
+ // Inputs in ]-0.5; 0] need to be added 0.5, other negative inputs need to
+ // be added the biggest double less than 0.5.
+ Label loadJoin;
+ loadConstantDouble(-0.5, scratch);
+ branchDouble(Assembler::DoubleLessThan, src, scratch, &loadJoin);
+ loadConstantDouble(0.5, temp);
+ bind(&loadJoin);
+
+ if (HasSSE41()) {
+ // Add 0.5 and round toward -Infinity. The result is stored in the temp
+ // register (currently contains 0.5).
+ addDouble(src, temp);
+ vroundsd(X86Encoding::RoundDown, temp, scratch);
+
+ // Truncate.
+ truncateDoubleToInt32(scratch, dest, fail);
+
+ // If the result is positive zero, then the actual result is -0. Fail.
+ // Otherwise, the truncation will have produced the correct negative
+ // integer.
+ branchTest32(Assembler::Zero, dest, dest, fail);
+ } else {
+ addDouble(src, temp);
+ // Round toward -Infinity without the benefit of ROUNDSD.
+ {
+ // If input + 0.5 >= 0, input is a negative number >= -0.5 and the
+ // result is -0.
+ branchDouble(Assembler::DoubleGreaterThanOrEqual, temp, scratch, fail);
+
+ // Truncate and round toward zero.
+ // This is off-by-one for everything but integer-valued inputs.
+ truncateDoubleToInt32(temp, dest, fail);
+
+ // Test whether the truncated double was integer-valued.
+ convertInt32ToDouble(dest, scratch);
+ branchDouble(Assembler::DoubleEqualOrUnordered, temp, scratch, &end);
+
+ // Input is not integer-valued, so we rounded off-by-one in the
+ // wrong direction. Correct by subtraction.
+ subl(Imm32(1), dest);
+ // Cannot overflow: output was already checked against INT_MIN.
+ }
+ }
+ }
+
+ bind(&end);
+}
+
+void MacroAssembler::nearbyIntDouble(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasRoundInstruction(mode));
+ vroundsd(Assembler::ToX86RoundingMode(mode), src, dest);
+}
+
+void MacroAssembler::nearbyIntFloat32(RoundingMode mode, FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(HasRoundInstruction(mode));
+ vroundss(Assembler::ToX86RoundingMode(mode), src, dest);
+}
+
+void MacroAssembler::copySignDouble(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) {
+ ScratchDoubleScope scratch(*this);
+
+ // TODO Support AVX2
+ if (rhs == output) {
+ MOZ_ASSERT(lhs != rhs);
+ double keepSignMask = mozilla::BitwiseCast<double>(INT64_MIN);
+ loadConstantDouble(keepSignMask, scratch);
+ vandpd(scratch, rhs, output);
+
+ double clearSignMask = mozilla::BitwiseCast<double>(INT64_MAX);
+ loadConstantDouble(clearSignMask, scratch);
+ vandpd(lhs, scratch, scratch);
+ } else {
+ double clearSignMask = mozilla::BitwiseCast<double>(INT64_MAX);
+ loadConstantDouble(clearSignMask, scratch);
+ vandpd(scratch, lhs, output);
+
+ double keepSignMask = mozilla::BitwiseCast<double>(INT64_MIN);
+ loadConstantDouble(keepSignMask, scratch);
+ vandpd(rhs, scratch, scratch);
+ }
+
+ vorpd(scratch, output, output);
+}
+
+void MacroAssembler::copySignFloat32(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output) {
+ ScratchFloat32Scope scratch(*this);
+
+ // TODO Support AVX2
+ if (rhs == output) {
+ MOZ_ASSERT(lhs != rhs);
+ float keepSignMask = mozilla::BitwiseCast<float>(INT32_MIN);
+ loadConstantFloat32(keepSignMask, scratch);
+ vandps(scratch, output, output);
+
+ float clearSignMask = mozilla::BitwiseCast<float>(INT32_MAX);
+ loadConstantFloat32(clearSignMask, scratch);
+ vandps(lhs, scratch, scratch);
+ } else {
+ float clearSignMask = mozilla::BitwiseCast<float>(INT32_MAX);
+ loadConstantFloat32(clearSignMask, scratch);
+ vandps(scratch, lhs, output);
+
+ float keepSignMask = mozilla::BitwiseCast<float>(INT32_MIN);
+ loadConstantFloat32(keepSignMask, scratch);
+ vandps(rhs, scratch, scratch);
+ }
+
+ vorps(scratch, output, output);
+}
+
+void MacroAssembler::shiftIndex32AndAdd(Register indexTemp32, int shift,
+ Register pointer) {
+ if (IsShiftInScaleRange(shift)) {
+ computeEffectiveAddress(
+ BaseIndex(pointer, indexTemp32, ShiftToScale(shift)), pointer);
+ return;
+ }
+ lshift32(Imm32(shift), indexTemp32);
+ addPtr(indexTemp32, pointer);
+}
+
+//}}} check_macroassembler_style
diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared.h b/js/src/jit/x86-shared/MacroAssembler-x86-shared.h
new file mode 100644
index 0000000000..ab31a25806
--- /dev/null
+++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared.h
@@ -0,0 +1,1004 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_MacroAssembler_x86_shared_h
+#define jit_x86_shared_MacroAssembler_x86_shared_h
+
+#if defined(JS_CODEGEN_X86)
+# include "jit/x86/Assembler-x86.h"
+#elif defined(JS_CODEGEN_X64)
+# include "jit/x64/Assembler-x64.h"
+#endif
+
+namespace js {
+namespace jit {
+
+class MacroAssembler;
+
+class MacroAssemblerX86Shared : public Assembler {
+ private:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ public:
+#ifdef JS_CODEGEN_X64
+ typedef X86Encoding::JmpSrc UsesItem;
+#else
+ typedef CodeOffset UsesItem;
+#endif
+
+ typedef Vector<UsesItem, 0, SystemAllocPolicy> UsesVector;
+ static_assert(sizeof(UsesItem) == 4);
+
+ protected:
+ // For Double, Float and SimdData, make the move ctors explicit so that MSVC
+ // knows what to use instead of copying these data structures.
+ template <class T>
+ struct Constant {
+ using Pod = T;
+
+ T value;
+ UsesVector uses;
+
+ explicit Constant(const T& value) : value(value) {}
+ Constant(Constant<T>&& other)
+ : value(other.value), uses(std::move(other.uses)) {}
+ explicit Constant(const Constant<T>&) = delete;
+ };
+
+ // Containers use SystemAllocPolicy since wasm releases memory after each
+ // function is compiled, and these need to live until after all functions
+ // are compiled.
+ using Double = Constant<double>;
+ Vector<Double, 0, SystemAllocPolicy> doubles_;
+ typedef HashMap<double, size_t, DefaultHasher<double>, SystemAllocPolicy>
+ DoubleMap;
+ DoubleMap doubleMap_;
+
+ using Float = Constant<float>;
+ Vector<Float, 0, SystemAllocPolicy> floats_;
+ typedef HashMap<float, size_t, DefaultHasher<float>, SystemAllocPolicy>
+ FloatMap;
+ FloatMap floatMap_;
+
+ struct SimdData : public Constant<SimdConstant> {
+ explicit SimdData(SimdConstant d) : Constant<SimdConstant>(d) {}
+ SimdData(SimdData&& d) : Constant<SimdConstant>(std::move(d)) {}
+ explicit SimdData(const SimdData&) = delete;
+ SimdConstant::Type type() const { return value.type(); }
+ };
+
+ Vector<SimdData, 0, SystemAllocPolicy> simds_;
+ typedef HashMap<SimdConstant, size_t, SimdConstant, SystemAllocPolicy>
+ SimdMap;
+ SimdMap simdMap_;
+
+ template <class T, class Map>
+ T* getConstant(const typename T::Pod& value, Map& map,
+ Vector<T, 0, SystemAllocPolicy>& vec);
+
+ Float* getFloat(float f);
+ Double* getDouble(double d);
+ SimdData* getSimdData(const SimdConstant& v);
+
+ public:
+ using Assembler::call;
+
+ MacroAssemblerX86Shared() = default;
+
+ bool appendRawCode(const uint8_t* code, size_t numBytes) {
+ return masm.appendRawCode(code, numBytes);
+ }
+
+ void addToPCRel4(uint32_t offset, int32_t bias) {
+ return masm.addToPCRel4(offset, bias);
+ }
+
+ // Evaluate srcDest = minmax<isMax>{Float32,Double}(srcDest, second).
+ // Checks for NaN if canBeNaN is true.
+ void minMaxDouble(FloatRegister srcDest, FloatRegister second, bool canBeNaN,
+ bool isMax);
+ void minMaxFloat32(FloatRegister srcDest, FloatRegister second, bool canBeNaN,
+ bool isMax);
+
+ void compareDouble(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs) {
+ if (cond & DoubleConditionBitInvert) {
+ vucomisd(lhs, rhs);
+ } else {
+ vucomisd(rhs, lhs);
+ }
+ }
+
+ void compareFloat(DoubleCondition cond, FloatRegister lhs,
+ FloatRegister rhs) {
+ if (cond & DoubleConditionBitInvert) {
+ vucomiss(lhs, rhs);
+ } else {
+ vucomiss(rhs, lhs);
+ }
+ }
+
+ void branchNegativeZero(FloatRegister reg, Register scratch, Label* label,
+ bool maybeNonZero = true);
+ void branchNegativeZeroFloat32(FloatRegister reg, Register scratch,
+ Label* label);
+
+ void move32(Imm32 imm, Register dest) {
+ // Use the ImmWord version of mov to register, which has special
+ // optimizations. Casting to uint32_t here ensures that the value
+ // is zero-extended.
+ mov(ImmWord(uint32_t(imm.value)), dest);
+ }
+ void move32(Imm32 imm, const Operand& dest) { movl(imm, dest); }
+ void move32(Register src, Register dest) { movl(src, dest); }
+ void move32(Register src, const Operand& dest) { movl(src, dest); }
+ void test32(Register lhs, Register rhs) { testl(rhs, lhs); }
+ void test32(const Address& addr, Imm32 imm) { testl(imm, Operand(addr)); }
+ void test32(const Operand lhs, Imm32 imm) { testl(imm, lhs); }
+ void test32(Register lhs, Imm32 rhs) { testl(rhs, lhs); }
+ void cmp32(Register lhs, Imm32 rhs) { cmpl(rhs, lhs); }
+ void cmp32(Register lhs, Register rhs) { cmpl(rhs, lhs); }
+ void cmp32(const Address& lhs, Register rhs) { cmp32(Operand(lhs), rhs); }
+ void cmp32(const Address& lhs, Imm32 rhs) { cmp32(Operand(lhs), rhs); }
+ void cmp32(const Operand& lhs, Imm32 rhs) { cmpl(rhs, lhs); }
+ void cmp32(const Operand& lhs, Register rhs) { cmpl(rhs, lhs); }
+ void cmp32(Register lhs, const Operand& rhs) { cmpl(rhs, lhs); }
+
+ void cmp16(const Address& lhs, Imm32 rhs) { cmp16(Operand(lhs), rhs); }
+ void cmp16(const Operand& lhs, Imm32 rhs) { cmpw(rhs, lhs); }
+
+ void cmp8(const Address& lhs, Imm32 rhs) { cmp8(Operand(lhs), rhs); }
+ void cmp8(const Operand& lhs, Imm32 rhs) { cmpb(rhs, lhs); }
+ void cmp8(const Operand& lhs, Register rhs) { cmpb(rhs, lhs); }
+
+ void atomic_inc32(const Operand& addr) { lock_incl(addr); }
+ void atomic_dec32(const Operand& addr) { lock_decl(addr); }
+
+ void storeLoadFence() {
+ // This implementation follows Linux.
+ if (HasSSE2()) {
+ masm.mfence();
+ } else {
+ lock_addl(Imm32(0), Operand(Address(esp, 0)));
+ }
+ }
+
+ void branch16(Condition cond, Register lhs, Register rhs, Label* label) {
+ cmpw(rhs, lhs);
+ j(cond, label);
+ }
+ void branchTest16(Condition cond, Register lhs, Register rhs, Label* label) {
+ testw(rhs, lhs);
+ j(cond, label);
+ }
+
+ void jump(Label* label) { jmp(label); }
+ void jump(JitCode* code) { jmp(code); }
+ void jump(TrampolinePtr code) { jmp(ImmPtr(code.value)); }
+ void jump(ImmPtr ptr) { jmp(ptr); }
+ void jump(Register reg) { jmp(Operand(reg)); }
+ void jump(const Address& addr) { jmp(Operand(addr)); }
+
+ void convertInt32ToDouble(Register src, FloatRegister dest) {
+ // vcvtsi2sd and friends write only part of their output register, which
+ // causes slowdowns on out-of-order processors. Explicitly break
+ // dependencies with vxorpd (and vxorps elsewhere), which are handled
+ // specially in modern CPUs, for this purpose. See sections 8.14, 9.8,
+ // 10.8, 12.9, 13.16, 14.14, and 15.8 of Agner's Microarchitecture
+ // document.
+ zeroDouble(dest);
+ vcvtsi2sd(src, dest, dest);
+ }
+ void convertInt32ToDouble(const Address& src, FloatRegister dest) {
+ convertInt32ToDouble(Operand(src), dest);
+ }
+ void convertInt32ToDouble(const BaseIndex& src, FloatRegister dest) {
+ convertInt32ToDouble(Operand(src), dest);
+ }
+ void convertInt32ToDouble(const Operand& src, FloatRegister dest) {
+ // Clear the output register first to break dependencies; see above;
+ zeroDouble(dest);
+ vcvtsi2sd(Operand(src), dest, dest);
+ }
+ void convertInt32ToFloat32(Register src, FloatRegister dest) {
+ // Clear the output register first to break dependencies; see above;
+ zeroFloat32(dest);
+ vcvtsi2ss(src, dest, dest);
+ }
+ void convertInt32ToFloat32(const Address& src, FloatRegister dest) {
+ convertInt32ToFloat32(Operand(src), dest);
+ }
+ void convertInt32ToFloat32(const Operand& src, FloatRegister dest) {
+ // Clear the output register first to break dependencies; see above;
+ zeroFloat32(dest);
+ vcvtsi2ss(src, dest, dest);
+ }
+ Condition testDoubleTruthy(bool truthy, FloatRegister reg) {
+ ScratchDoubleScope scratch(asMasm());
+ zeroDouble(scratch);
+ vucomisd(reg, scratch);
+ return truthy ? NonZero : Zero;
+ }
+
+ // Class which ensures that registers used in byte ops are compatible with
+ // such instructions, even if the original register passed in wasn't. This
+ // only applies to x86, as on x64 all registers are valid single byte regs.
+ // This doesn't lead to great code but helps to simplify code generation.
+ //
+ // Note that this can currently only be used in cases where the register is
+ // read from by the guarded instruction, not written to.
+ class AutoEnsureByteRegister {
+ MacroAssemblerX86Shared* masm;
+ Register original_;
+ Register substitute_;
+
+ public:
+ template <typename T>
+ AutoEnsureByteRegister(MacroAssemblerX86Shared* masm, T address,
+ Register reg)
+ : masm(masm), original_(reg) {
+ AllocatableGeneralRegisterSet singleByteRegs(Registers::SingleByteRegs);
+ if (singleByteRegs.has(reg)) {
+ substitute_ = reg;
+ } else {
+ MOZ_ASSERT(address.base != StackPointer);
+ do {
+ substitute_ = singleByteRegs.takeAny();
+ } while (Operand(address).containsReg(substitute_));
+
+ masm->push(substitute_);
+ masm->mov(reg, substitute_);
+ }
+ }
+
+ ~AutoEnsureByteRegister() {
+ if (original_ != substitute_) {
+ masm->pop(substitute_);
+ }
+ }
+
+ Register reg() { return substitute_; }
+ };
+
+ void load8ZeroExtend(const Operand& src, Register dest) { movzbl(src, dest); }
+ void load8ZeroExtend(const Address& src, Register dest) {
+ movzbl(Operand(src), dest);
+ }
+ void load8ZeroExtend(const BaseIndex& src, Register dest) {
+ movzbl(Operand(src), dest);
+ }
+ void load8SignExtend(const Operand& src, Register dest) { movsbl(src, dest); }
+ void load8SignExtend(const Address& src, Register dest) {
+ movsbl(Operand(src), dest);
+ }
+ void load8SignExtend(const BaseIndex& src, Register dest) {
+ movsbl(Operand(src), dest);
+ }
+ template <typename T>
+ void store8(Imm32 src, const T& dest) {
+ movb(src, Operand(dest));
+ }
+ template <typename T>
+ void store8(Register src, const T& dest) {
+ AutoEnsureByteRegister ensure(this, dest, src);
+ movb(ensure.reg(), Operand(dest));
+ }
+ void load16ZeroExtend(const Operand& src, Register dest) {
+ movzwl(src, dest);
+ }
+ void load16ZeroExtend(const Address& src, Register dest) {
+ movzwl(Operand(src), dest);
+ }
+ void load16ZeroExtend(const BaseIndex& src, Register dest) {
+ movzwl(Operand(src), dest);
+ }
+ template <typename S>
+ void load16UnalignedZeroExtend(const S& src, Register dest) {
+ load16ZeroExtend(src, dest);
+ }
+ template <typename S, typename T>
+ void store16(const S& src, const T& dest) {
+ movw(src, Operand(dest));
+ }
+ template <typename S, typename T>
+ void store16Unaligned(const S& src, const T& dest) {
+ store16(src, dest);
+ }
+ void load16SignExtend(const Operand& src, Register dest) {
+ movswl(src, dest);
+ }
+ void load16SignExtend(const Address& src, Register dest) {
+ movswl(Operand(src), dest);
+ }
+ void load16SignExtend(const BaseIndex& src, Register dest) {
+ movswl(Operand(src), dest);
+ }
+ template <typename S>
+ void load16UnalignedSignExtend(const S& src, Register dest) {
+ load16SignExtend(src, dest);
+ }
+ void load32(const Address& address, Register dest) {
+ movl(Operand(address), dest);
+ }
+ void load32(const BaseIndex& src, Register dest) { movl(Operand(src), dest); }
+ void load32(const Operand& src, Register dest) { movl(src, dest); }
+ template <typename S>
+ void load32Unaligned(const S& src, Register dest) {
+ load32(src, dest);
+ }
+ template <typename S, typename T>
+ void store32(const S& src, const T& dest) {
+ movl(src, Operand(dest));
+ }
+ template <typename S, typename T>
+ void store32_NoSecondScratch(const S& src, const T& dest) {
+ store32(src, dest);
+ }
+ template <typename S, typename T>
+ void store32Unaligned(const S& src, const T& dest) {
+ store32(src, dest);
+ }
+ void loadDouble(const Address& src, FloatRegister dest) { vmovsd(src, dest); }
+ void loadDouble(const BaseIndex& src, FloatRegister dest) {
+ vmovsd(src, dest);
+ }
+ void loadDouble(const Operand& src, FloatRegister dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ loadDouble(src.toAddress(), dest);
+ break;
+ case Operand::MEM_SCALE:
+ loadDouble(src.toBaseIndex(), dest);
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void moveDouble(FloatRegister src, FloatRegister dest) {
+ // Use vmovapd instead of vmovsd to avoid dependencies.
+ vmovapd(src, dest);
+ }
+ void zeroDouble(FloatRegister reg) { vxorpd(reg, reg, reg); }
+ void zeroFloat32(FloatRegister reg) { vxorps(reg, reg, reg); }
+ void convertFloat32ToDouble(FloatRegister src, FloatRegister dest) {
+ vcvtss2sd(src, dest, dest);
+ }
+ void convertDoubleToFloat32(FloatRegister src, FloatRegister dest) {
+ vcvtsd2ss(src, dest, dest);
+ }
+
+ void loadInt32x4(const Address& addr, FloatRegister dest) {
+ vmovdqa(Operand(addr), dest);
+ }
+ void loadFloat32x4(const Address& addr, FloatRegister dest) {
+ vmovaps(Operand(addr), dest);
+ }
+ void storeInt32x4(FloatRegister src, const Address& addr) {
+ vmovdqa(src, Operand(addr));
+ }
+ void storeFloat32x4(FloatRegister src, const Address& addr) {
+ vmovaps(src, Operand(addr));
+ }
+
+ void convertFloat32x4ToInt32x4(FloatRegister src, FloatRegister dest) {
+ // Note that if the conversion failed (because the converted
+ // result is larger than the maximum signed int32, or less than the
+ // least signed int32, or NaN), this will return the undefined integer
+ // value (0x8000000).
+ vcvttps2dq(src, dest);
+ }
+ void convertInt32x4ToFloat32x4(FloatRegister src, FloatRegister dest) {
+ vcvtdq2ps(src, dest);
+ }
+
+ void binarySimd128(const SimdConstant& rhs, FloatRegister lhsDest,
+ void (MacroAssembler::*regOp)(const Operand&,
+ FloatRegister,
+ FloatRegister),
+ void (MacroAssembler::*constOp)(const SimdConstant&,
+ FloatRegister));
+ void binarySimd128(
+ FloatRegister lhs, const SimdConstant& rhs, FloatRegister dest,
+ void (MacroAssembler::*regOp)(const Operand&, FloatRegister,
+ FloatRegister),
+ void (MacroAssembler::*constOp)(const SimdConstant&, FloatRegister,
+ FloatRegister));
+ void binarySimd128(const SimdConstant& rhs, FloatRegister lhsDest,
+ void (MacroAssembler::*regOp)(const Operand&,
+ FloatRegister),
+ void (MacroAssembler::*constOp)(const SimdConstant&,
+ FloatRegister));
+
+ // SIMD methods, defined in MacroAssembler-x86-shared-SIMD.cpp.
+
+ void unsignedConvertInt32x4ToFloat32x4(FloatRegister src, FloatRegister dest);
+ void unsignedConvertInt32x4ToFloat64x2(FloatRegister src, FloatRegister dest);
+ void bitwiseTestSimd128(const SimdConstant& rhs, FloatRegister lhs);
+
+ void truncSatFloat32x4ToInt32x4(FloatRegister src, FloatRegister dest);
+ void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src, FloatRegister temp,
+ FloatRegister dest);
+ void unsignedTruncSatFloat32x4ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest);
+ void truncSatFloat64x2ToInt32x4(FloatRegister src, FloatRegister temp,
+ FloatRegister dest);
+ void unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src, FloatRegister temp,
+ FloatRegister dest);
+ void unsignedTruncSatFloat64x2ToInt32x4Relaxed(FloatRegister src,
+ FloatRegister dest);
+
+ void splatX16(Register input, FloatRegister output);
+ void splatX8(Register input, FloatRegister output);
+ void splatX4(Register input, FloatRegister output);
+ void splatX4(FloatRegister input, FloatRegister output);
+ void splatX2(FloatRegister input, FloatRegister output);
+
+ void extractLaneInt32x4(FloatRegister input, Register output, unsigned lane);
+ void extractLaneFloat32x4(FloatRegister input, FloatRegister output,
+ unsigned lane);
+ void extractLaneFloat64x2(FloatRegister input, FloatRegister output,
+ unsigned lane);
+ void extractLaneInt16x8(FloatRegister input, Register output, unsigned lane,
+ SimdSign sign);
+ void extractLaneInt8x16(FloatRegister input, Register output, unsigned lane,
+ SimdSign sign);
+
+ void replaceLaneFloat32x4(unsigned lane, FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest);
+ void replaceLaneFloat64x2(unsigned lane, FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest);
+
+ void shuffleInt8x16(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister output, const uint8_t lanes[16]);
+ void blendInt8x16(FloatRegister lhs, FloatRegister rhs, FloatRegister output,
+ FloatRegister temp, const uint8_t lanes[16]);
+ void blendInt16x8(FloatRegister lhs, FloatRegister rhs, FloatRegister output,
+ const uint16_t lanes[8]);
+ void laneSelectSimd128(FloatRegister mask, FloatRegister lhs,
+ FloatRegister rhs, FloatRegister output);
+
+ void compareInt8x16(FloatRegister lhs, Operand rhs, Assembler::Condition cond,
+ FloatRegister output);
+ void compareInt8x16(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest);
+ void compareInt16x8(FloatRegister lhs, Operand rhs, Assembler::Condition cond,
+ FloatRegister output);
+ void compareInt16x8(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest);
+ void compareInt32x4(FloatRegister lhs, Operand rhs, Assembler::Condition cond,
+ FloatRegister output);
+ void compareInt32x4(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest);
+ void compareForEqualityInt64x2(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond,
+ FloatRegister output);
+ void compareForOrderingInt64x2(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond, FloatRegister temp1,
+ FloatRegister temp2, FloatRegister output);
+ void compareForOrderingInt64x2AVX(FloatRegister lhs, FloatRegister rhs,
+ Assembler::Condition cond,
+ FloatRegister output);
+ void compareFloat32x4(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond, FloatRegister output);
+ void compareFloat32x4(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest);
+ void compareFloat64x2(FloatRegister lhs, Operand rhs,
+ Assembler::Condition cond, FloatRegister output);
+ void compareFloat64x2(Assembler::Condition cond, FloatRegister lhs,
+ const SimdConstant& rhs, FloatRegister dest);
+
+ void minMaxFloat32x4(bool isMin, FloatRegister lhs, Operand rhs,
+ FloatRegister temp1, FloatRegister temp2,
+ FloatRegister output);
+ void minMaxFloat32x4AVX(bool isMin, FloatRegister lhs, FloatRegister rhs,
+ FloatRegister temp1, FloatRegister temp2,
+ FloatRegister output);
+ void minMaxFloat64x2(bool isMin, FloatRegister lhs, Operand rhs,
+ FloatRegister temp1, FloatRegister temp2,
+ FloatRegister output);
+ void minMaxFloat64x2AVX(bool isMin, FloatRegister lhs, FloatRegister rhs,
+ FloatRegister temp1, FloatRegister temp2,
+ FloatRegister output);
+ void minFloat32x4(FloatRegister lhs, FloatRegister rhs, FloatRegister temp1,
+ FloatRegister temp2, FloatRegister output);
+ void maxFloat32x4(FloatRegister lhs, FloatRegister rhs, FloatRegister temp1,
+ FloatRegister temp2, FloatRegister output);
+
+ void minFloat64x2(FloatRegister lhs, FloatRegister rhs, FloatRegister temp1,
+ FloatRegister temp2, FloatRegister output);
+ void maxFloat64x2(FloatRegister lhs, FloatRegister rhs, FloatRegister temp1,
+ FloatRegister temp2, FloatRegister output);
+
+ void packedShiftByScalarInt8x16(
+ FloatRegister in, Register count, FloatRegister xtmp, FloatRegister dest,
+ void (MacroAssemblerX86Shared::*shift)(FloatRegister, FloatRegister,
+ FloatRegister),
+ void (MacroAssemblerX86Shared::*extend)(const Operand&, FloatRegister));
+
+ void packedLeftShiftByScalarInt8x16(FloatRegister in, Register count,
+ FloatRegister xtmp, FloatRegister dest);
+ void packedLeftShiftByScalarInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest);
+ void packedRightShiftByScalarInt8x16(FloatRegister in, Register count,
+ FloatRegister xtmp, FloatRegister dest);
+ void packedRightShiftByScalarInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest);
+ void packedUnsignedRightShiftByScalarInt8x16(FloatRegister in, Register count,
+ FloatRegister xtmp,
+ FloatRegister dest);
+ void packedUnsignedRightShiftByScalarInt8x16(Imm32 count, FloatRegister src,
+ FloatRegister dest);
+
+ void packedLeftShiftByScalarInt16x8(FloatRegister in, Register count,
+ FloatRegister dest);
+ void packedRightShiftByScalarInt16x8(FloatRegister in, Register count,
+ FloatRegister dest);
+ void packedUnsignedRightShiftByScalarInt16x8(FloatRegister in, Register count,
+ FloatRegister dest);
+
+ void packedLeftShiftByScalarInt32x4(FloatRegister in, Register count,
+ FloatRegister dest);
+ void packedRightShiftByScalarInt32x4(FloatRegister in, Register count,
+ FloatRegister dest);
+ void packedUnsignedRightShiftByScalarInt32x4(FloatRegister in, Register count,
+ FloatRegister dest);
+ void packedLeftShiftByScalarInt64x2(FloatRegister in, Register count,
+ FloatRegister dest);
+ void packedRightShiftByScalarInt64x2(FloatRegister in, Register count,
+ FloatRegister temp, FloatRegister dest);
+ void packedRightShiftByScalarInt64x2(Imm32 count, FloatRegister src,
+ FloatRegister dest);
+ void packedUnsignedRightShiftByScalarInt64x2(FloatRegister in, Register count,
+ FloatRegister dest);
+ void selectSimd128(FloatRegister mask, FloatRegister onTrue,
+ FloatRegister onFalse, FloatRegister temp,
+ FloatRegister output);
+ void popcntInt8x16(FloatRegister src, FloatRegister temp,
+ FloatRegister output);
+ void dotBFloat16x8ThenAdd(FloatRegister lhs, FloatRegister rhs,
+ FloatRegister dest, FloatRegister temp);
+
+ // SIMD inline methods private to the implementation, that appear to be used.
+
+ template <class T, class Reg>
+ inline void loadScalar(const Operand& src, Reg dest);
+ template <class T, class Reg>
+ inline void storeScalar(Reg src, const Address& dest);
+ template <class T>
+ inline void loadAlignedVector(const Address& src, FloatRegister dest);
+ template <class T>
+ inline void storeAlignedVector(FloatRegister src, const Address& dest);
+
+ void loadAlignedSimd128Int(const Address& src, FloatRegister dest) {
+ vmovdqa(Operand(src), dest);
+ }
+ void loadAlignedSimd128Int(const Operand& src, FloatRegister dest) {
+ vmovdqa(src, dest);
+ }
+ void storeAlignedSimd128Int(FloatRegister src, const Address& dest) {
+ vmovdqa(src, Operand(dest));
+ }
+ void moveSimd128Int(FloatRegister src, FloatRegister dest) {
+ if (src != dest) {
+ vmovdqa(src, dest);
+ }
+ }
+ FloatRegister moveSimd128IntIfNotAVX(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(src.isSimd128() && dest.isSimd128());
+ if (HasAVX()) {
+ return src;
+ }
+ moveSimd128Int(src, dest);
+ return dest;
+ }
+ FloatRegister selectDestIfAVX(FloatRegister src, FloatRegister dest) {
+ MOZ_ASSERT(src.isSimd128() && dest.isSimd128());
+ return HasAVX() ? dest : src;
+ }
+ void loadUnalignedSimd128Int(const Address& src, FloatRegister dest) {
+ vmovdqu(Operand(src), dest);
+ }
+ void loadUnalignedSimd128Int(const BaseIndex& src, FloatRegister dest) {
+ vmovdqu(Operand(src), dest);
+ }
+ void loadUnalignedSimd128Int(const Operand& src, FloatRegister dest) {
+ vmovdqu(src, dest);
+ }
+ void storeUnalignedSimd128Int(FloatRegister src, const Address& dest) {
+ vmovdqu(src, Operand(dest));
+ }
+ void storeUnalignedSimd128Int(FloatRegister src, const BaseIndex& dest) {
+ vmovdqu(src, Operand(dest));
+ }
+ void storeUnalignedSimd128Int(FloatRegister src, const Operand& dest) {
+ vmovdqu(src, dest);
+ }
+ void packedLeftShiftByScalarInt16x8(Imm32 count, FloatRegister dest) {
+ count.value &= 15;
+ vpsllw(count, dest, dest);
+ }
+ void packedRightShiftByScalarInt16x8(Imm32 count, FloatRegister dest) {
+ count.value &= 15;
+ vpsraw(count, dest, dest);
+ }
+ void packedUnsignedRightShiftByScalarInt16x8(Imm32 count,
+ FloatRegister dest) {
+ count.value &= 15;
+ vpsrlw(count, dest, dest);
+ }
+ void packedLeftShiftByScalarInt32x4(Imm32 count, FloatRegister dest) {
+ count.value &= 31;
+ vpslld(count, dest, dest);
+ }
+ void packedRightShiftByScalarInt32x4(Imm32 count, FloatRegister dest) {
+ count.value &= 31;
+ vpsrad(count, dest, dest);
+ }
+ void packedUnsignedRightShiftByScalarInt32x4(Imm32 count,
+ FloatRegister dest) {
+ count.value &= 31;
+ vpsrld(count, dest, dest);
+ }
+ void loadAlignedSimd128Float(const Address& src, FloatRegister dest) {
+ vmovaps(Operand(src), dest);
+ }
+ void loadAlignedSimd128Float(const Operand& src, FloatRegister dest) {
+ vmovaps(src, dest);
+ }
+ void storeAlignedSimd128Float(FloatRegister src, const Address& dest) {
+ vmovaps(src, Operand(dest));
+ }
+ void moveSimd128Float(FloatRegister src, FloatRegister dest) {
+ if (src != dest) {
+ vmovaps(src, dest);
+ }
+ }
+ FloatRegister moveSimd128FloatIfNotAVX(FloatRegister src,
+ FloatRegister dest) {
+ MOZ_ASSERT(src.isSimd128() && dest.isSimd128());
+ if (HasAVX()) {
+ return src;
+ }
+ moveSimd128Float(src, dest);
+ return dest;
+ }
+ FloatRegister moveSimd128FloatIfEqual(FloatRegister src, FloatRegister dest,
+ FloatRegister other) {
+ MOZ_ASSERT(src.isSimd128() && dest.isSimd128());
+ if (src != other) {
+ return src;
+ }
+ moveSimd128Float(src, dest);
+ return dest;
+ }
+ FloatRegister moveSimd128FloatIfNotAVXOrOther(FloatRegister src,
+ FloatRegister dest,
+ FloatRegister other) {
+ MOZ_ASSERT(src.isSimd128() && dest.isSimd128());
+ if (HasAVX() && src != other) {
+ return src;
+ }
+ moveSimd128Float(src, dest);
+ return dest;
+ }
+
+ void loadUnalignedSimd128(const Operand& src, FloatRegister dest) {
+ vmovups(src, dest);
+ }
+ void storeUnalignedSimd128(FloatRegister src, const Operand& dest) {
+ vmovups(src, dest);
+ }
+
+ static uint32_t ComputeShuffleMask(uint32_t x = 0, uint32_t y = 1,
+ uint32_t z = 2, uint32_t w = 3) {
+ MOZ_ASSERT(x < 4 && y < 4 && z < 4 && w < 4);
+ uint32_t r = (w << 6) | (z << 4) | (y << 2) | (x << 0);
+ MOZ_ASSERT(r < 256);
+ return r;
+ }
+
+ void shuffleInt32(uint32_t mask, FloatRegister src, FloatRegister dest) {
+ vpshufd(mask, src, dest);
+ }
+ void moveLowInt32(FloatRegister src, Register dest) { vmovd(src, dest); }
+
+ void moveHighPairToLowPairFloat32(FloatRegister src, FloatRegister dest) {
+ vmovhlps(src, dest, dest);
+ }
+ void moveFloatAsDouble(Register src, FloatRegister dest) {
+ vmovd(src, dest);
+ vcvtss2sd(dest, dest, dest);
+ }
+ void loadFloatAsDouble(const Address& src, FloatRegister dest) {
+ vmovss(src, dest);
+ vcvtss2sd(dest, dest, dest);
+ }
+ void loadFloatAsDouble(const BaseIndex& src, FloatRegister dest) {
+ vmovss(src, dest);
+ vcvtss2sd(dest, dest, dest);
+ }
+ void loadFloatAsDouble(const Operand& src, FloatRegister dest) {
+ loadFloat32(src, dest);
+ vcvtss2sd(dest, dest, dest);
+ }
+ void loadFloat32(const Address& src, FloatRegister dest) {
+ vmovss(src, dest);
+ }
+ void loadFloat32(const BaseIndex& src, FloatRegister dest) {
+ vmovss(src, dest);
+ }
+ void loadFloat32(const Operand& src, FloatRegister dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ loadFloat32(src.toAddress(), dest);
+ break;
+ case Operand::MEM_SCALE:
+ loadFloat32(src.toBaseIndex(), dest);
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void moveFloat32(FloatRegister src, FloatRegister dest) {
+ // Use vmovaps instead of vmovss to avoid dependencies.
+ vmovaps(src, dest);
+ }
+
+ // Checks whether a double is representable as a 32-bit integer. If so, the
+ // integer is written to the output register. Otherwise, a bailout is taken to
+ // the given snapshot. This function overwrites the scratch float register.
+ void convertDoubleToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true) {
+ // Check for -0.0
+ if (negativeZeroCheck) {
+ branchNegativeZero(src, dest, fail);
+ }
+
+ ScratchDoubleScope scratch(asMasm());
+ vcvttsd2si(src, dest);
+ convertInt32ToDouble(dest, scratch);
+ vucomisd(scratch, src);
+ j(Assembler::Parity, fail);
+ j(Assembler::NotEqual, fail);
+ }
+
+ // Checks whether a float32 is representable as a 32-bit integer. If so, the
+ // integer is written to the output register. Otherwise, a bailout is taken to
+ // the given snapshot. This function overwrites the scratch float register.
+ void convertFloat32ToInt32(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true) {
+ // Check for -0.0
+ if (negativeZeroCheck) {
+ branchNegativeZeroFloat32(src, dest, fail);
+ }
+
+ ScratchFloat32Scope scratch(asMasm());
+ vcvttss2si(src, dest);
+ convertInt32ToFloat32(dest, scratch);
+ vucomiss(scratch, src);
+ j(Assembler::Parity, fail);
+ j(Assembler::NotEqual, fail);
+ }
+
+ void truncateDoubleToInt32(FloatRegister src, Register dest, Label* fail) {
+ // vcvttsd2si returns 0x80000000 on failure. Test for it by
+ // subtracting 1 and testing overflow. The other possibility is to test
+ // equality for INT_MIN after a comparison, but 1 costs fewer bytes to
+ // materialize.
+ vcvttsd2si(src, dest);
+ cmp32(dest, Imm32(1));
+ j(Assembler::Overflow, fail);
+ }
+ void truncateFloat32ToInt32(FloatRegister src, Register dest, Label* fail) {
+ // Same trick as explained in the above comment.
+ vcvttss2si(src, dest);
+ cmp32(dest, Imm32(1));
+ j(Assembler::Overflow, fail);
+ }
+
+ inline void clampIntToUint8(Register reg);
+
+ bool maybeInlineDouble(double d, FloatRegister dest) {
+ // Loading zero with xor is specially optimized in hardware.
+ if (mozilla::IsPositiveZero(d)) {
+ zeroDouble(dest);
+ return true;
+ }
+
+ // It is also possible to load several common constants using vpcmpeqw
+ // to get all ones and then vpsllq and vpsrlq to get zeros at the ends,
+ // as described in "13.4 Generating constants" of
+ // "2. Optimizing subroutines in assembly language" by Agner Fog, and as
+ // previously implemented here. However, with x86 and x64 both using
+ // constant pool loads for double constants, this is probably only
+ // worthwhile in cases where a load is likely to be delayed.
+
+ return false;
+ }
+
+ bool maybeInlineFloat(float f, FloatRegister dest) {
+ // See comment above
+ if (mozilla::IsPositiveZero(f)) {
+ zeroFloat32(dest);
+ return true;
+ }
+ return false;
+ }
+
+ bool maybeInlineSimd128Int(const SimdConstant& v, const FloatRegister& dest) {
+ if (v.isZeroBits()) {
+ vpxor(dest, dest, dest);
+ return true;
+ }
+ if (v.isOneBits()) {
+ vpcmpeqw(Operand(dest), dest, dest);
+ return true;
+ }
+ return false;
+ }
+ bool maybeInlineSimd128Float(const SimdConstant& v,
+ const FloatRegister& dest) {
+ if (v.isZeroBits()) {
+ vxorps(dest, dest, dest);
+ return true;
+ }
+ return false;
+ }
+
+ void convertBoolToInt32(Register source, Register dest) {
+ // Note that C++ bool is only 1 byte, so zero extend it to clear the
+ // higher-order bits.
+ movzbl(source, dest);
+ }
+
+ void emitSet(Assembler::Condition cond, Register dest,
+ Assembler::NaNCond ifNaN = Assembler::NaN_HandledByCond) {
+ if (AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(dest)) {
+ // If the register we're defining is a single byte register,
+ // take advantage of the setCC instruction
+ setCC(cond, dest);
+ movzbl(dest, dest);
+
+ if (ifNaN != Assembler::NaN_HandledByCond) {
+ Label noNaN;
+ j(Assembler::NoParity, &noNaN);
+ mov(ImmWord(ifNaN == Assembler::NaN_IsTrue), dest);
+ bind(&noNaN);
+ }
+ } else {
+ Label end;
+ Label ifFalse;
+
+ if (ifNaN == Assembler::NaN_IsFalse) {
+ j(Assembler::Parity, &ifFalse);
+ }
+ // Note a subtlety here: FLAGS is live at this point, and the
+ // mov interface doesn't guarantee to preserve FLAGS. Use
+ // movl instead of mov, because the movl instruction
+ // preserves FLAGS.
+ movl(Imm32(1), dest);
+ j(cond, &end);
+ if (ifNaN == Assembler::NaN_IsTrue) {
+ j(Assembler::Parity, &end);
+ }
+ bind(&ifFalse);
+ mov(ImmWord(0), dest);
+
+ bind(&end);
+ }
+ }
+
+ void emitSetRegisterIf(AssemblerX86Shared::Condition cond, Register dest) {
+ if (AllocatableGeneralRegisterSet(Registers::SingleByteRegs).has(dest)) {
+ // If the register we're defining is a single byte register,
+ // take advantage of the setCC instruction
+ setCC(cond, dest);
+ movzbl(dest, dest);
+ } else {
+ Label end;
+ movl(Imm32(1), dest);
+ j(cond, &end);
+ mov(ImmWord(0), dest);
+ bind(&end);
+ }
+ }
+
+ // Emit a JMP that can be toggled to a CMP. See ToggleToJmp(), ToggleToCmp().
+ CodeOffset toggledJump(Label* label) {
+ CodeOffset offset(size());
+ jump(label);
+ return offset;
+ }
+
+ template <typename T>
+ void computeEffectiveAddress(const T& address, Register dest) {
+ lea(Operand(address), dest);
+ }
+
+ void checkStackAlignment() {
+ // Exists for ARM compatibility.
+ }
+
+ void abiret() { ret(); }
+
+ protected:
+ bool buildOOLFakeExitFrame(void* fakeReturnAddr);
+};
+
+// Specialize for float to use movaps. Use movdqa for everything else.
+template <>
+inline void MacroAssemblerX86Shared::loadAlignedVector<float>(
+ const Address& src, FloatRegister dest) {
+ loadAlignedSimd128Float(src, dest);
+}
+
+template <typename T>
+inline void MacroAssemblerX86Shared::loadAlignedVector(const Address& src,
+ FloatRegister dest) {
+ loadAlignedSimd128Int(src, dest);
+}
+
+// Specialize for float to use movaps. Use movdqa for everything else.
+template <>
+inline void MacroAssemblerX86Shared::storeAlignedVector<float>(
+ FloatRegister src, const Address& dest) {
+ storeAlignedSimd128Float(src, dest);
+}
+
+template <typename T>
+inline void MacroAssemblerX86Shared::storeAlignedVector(FloatRegister src,
+ const Address& dest) {
+ storeAlignedSimd128Int(src, dest);
+}
+
+template <>
+inline void MacroAssemblerX86Shared::loadScalar<int8_t>(const Operand& src,
+ Register dest) {
+ load8ZeroExtend(src, dest);
+}
+template <>
+inline void MacroAssemblerX86Shared::loadScalar<int16_t>(const Operand& src,
+ Register dest) {
+ load16ZeroExtend(src, dest);
+}
+template <>
+inline void MacroAssemblerX86Shared::loadScalar<int32_t>(const Operand& src,
+ Register dest) {
+ load32(src, dest);
+}
+template <>
+inline void MacroAssemblerX86Shared::loadScalar<float>(const Operand& src,
+ FloatRegister dest) {
+ loadFloat32(src, dest);
+}
+
+template <>
+inline void MacroAssemblerX86Shared::storeScalar<int8_t>(Register src,
+ const Address& dest) {
+ store8(src, dest);
+}
+template <>
+inline void MacroAssemblerX86Shared::storeScalar<int16_t>(Register src,
+ const Address& dest) {
+ store16(src, dest);
+}
+template <>
+inline void MacroAssemblerX86Shared::storeScalar<int32_t>(Register src,
+ const Address& dest) {
+ store32(src, dest);
+}
+template <>
+inline void MacroAssemblerX86Shared::storeScalar<float>(FloatRegister src,
+ const Address& dest) {
+ vmovss(src, dest);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_MacroAssembler_x86_shared_h */
diff --git a/js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp b/js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp
new file mode 100644
index 0000000000..590bd90a37
--- /dev/null
+++ b/js/src/jit/x86-shared/MoveEmitter-x86-shared.cpp
@@ -0,0 +1,528 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86-shared/MoveEmitter-x86-shared.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using mozilla::Maybe;
+
+MoveEmitterX86::MoveEmitterX86(MacroAssembler& masm)
+ : inCycle_(false), masm(masm), pushedAtCycle_(-1) {
+ pushedAtStart_ = masm.framePushed();
+}
+
+// Examine the cycle in moves starting at position i. Determine if it's a
+// simple cycle consisting of all register-to-register moves in a single class,
+// and whether it can be implemented entirely by swaps.
+size_t MoveEmitterX86::characterizeCycle(const MoveResolver& moves, size_t i,
+ bool* allGeneralRegs,
+ bool* allFloatRegs) {
+ size_t swapCount = 0;
+
+ for (size_t j = i;; j++) {
+ const MoveOp& move = moves.getMove(j);
+
+ // If it isn't a cycle of registers of the same kind, we won't be able
+ // to optimize it.
+ if (!move.to().isGeneralReg()) {
+ *allGeneralRegs = false;
+ }
+ if (!move.to().isFloatReg()) {
+ *allFloatRegs = false;
+ }
+ if (!*allGeneralRegs && !*allFloatRegs) {
+ return -1;
+ }
+
+ // Stop iterating when we see the last one.
+ if (j != i && move.isCycleEnd()) {
+ break;
+ }
+
+ // Check that this move is actually part of the cycle. This is
+ // over-conservative when there are multiple reads from the same source,
+ // but that's expected to be rare.
+ if (move.from() != moves.getMove(j + 1).to()) {
+ *allGeneralRegs = false;
+ *allFloatRegs = false;
+ return -1;
+ }
+
+ swapCount++;
+ }
+
+ // Check that the last move cycles back to the first move.
+ const MoveOp& move = moves.getMove(i + swapCount);
+ if (move.from() != moves.getMove(i).to()) {
+ *allGeneralRegs = false;
+ *allFloatRegs = false;
+ return -1;
+ }
+
+ return swapCount;
+}
+
+// If we can emit optimized code for the cycle in moves starting at position i,
+// do so, and return true.
+bool MoveEmitterX86::maybeEmitOptimizedCycle(const MoveResolver& moves,
+ size_t i, bool allGeneralRegs,
+ bool allFloatRegs,
+ size_t swapCount) {
+ if (allGeneralRegs && swapCount <= 2) {
+ // Use x86's swap-integer-registers instruction if we only have a few
+ // swaps. (x86 also has a swap between registers and memory but it's
+ // slow.)
+ for (size_t k = 0; k < swapCount; k++) {
+ masm.xchg(moves.getMove(i + k).to().reg(),
+ moves.getMove(i + k + 1).to().reg());
+ }
+ return true;
+ }
+
+ if (allFloatRegs && swapCount == 1) {
+ // There's no xchg for xmm registers, but if we only need a single swap,
+ // it's cheap to do an XOR swap.
+ FloatRegister a = moves.getMove(i).to().floatReg();
+ FloatRegister b = moves.getMove(i + 1).to().floatReg();
+ masm.vxorpd(a, b, b);
+ masm.vxorpd(b, a, a);
+ masm.vxorpd(a, b, b);
+ return true;
+ }
+
+ return false;
+}
+
+void MoveEmitterX86::emit(const MoveResolver& moves) {
+#if defined(JS_CODEGEN_X86) && defined(DEBUG)
+ // Clobber any scratch register we have, to make regalloc bugs more visible.
+ if (scratchRegister_.isSome()) {
+ masm.mov(ImmWord(0xdeadbeef), scratchRegister_.value());
+ }
+#endif
+
+ for (size_t i = 0; i < moves.numMoves(); i++) {
+#if defined(JS_CODEGEN_X86) && defined(DEBUG)
+ if (!scratchRegister_.isSome()) {
+ Maybe<Register> reg = findScratchRegister(moves, i);
+ if (reg.isSome()) {
+ masm.mov(ImmWord(0xdeadbeef), reg.value());
+ }
+ }
+#endif
+
+ const MoveOp& move = moves.getMove(i);
+ const MoveOperand& from = move.from();
+ const MoveOperand& to = move.to();
+
+ if (move.isCycleEnd()) {
+ MOZ_ASSERT(inCycle_);
+ completeCycle(to, move.type());
+ inCycle_ = false;
+ continue;
+ }
+
+ if (move.isCycleBegin()) {
+ MOZ_ASSERT(!inCycle_);
+
+ // Characterize the cycle.
+ bool allGeneralRegs = true, allFloatRegs = true;
+ size_t swapCount =
+ characterizeCycle(moves, i, &allGeneralRegs, &allFloatRegs);
+
+ // Attempt to optimize it to avoid using the stack.
+ if (maybeEmitOptimizedCycle(moves, i, allGeneralRegs, allFloatRegs,
+ swapCount)) {
+ i += swapCount;
+ continue;
+ }
+
+ // Otherwise use the stack.
+ breakCycle(to, move.endCycleType());
+ inCycle_ = true;
+ }
+
+ // A normal move which is not part of a cycle.
+ switch (move.type()) {
+ case MoveOp::FLOAT32:
+ emitFloat32Move(from, to);
+ break;
+ case MoveOp::DOUBLE:
+ emitDoubleMove(from, to);
+ break;
+ case MoveOp::INT32:
+ emitInt32Move(from, to, moves, i);
+ break;
+ case MoveOp::GENERAL:
+ emitGeneralMove(from, to, moves, i);
+ break;
+ case MoveOp::SIMD128:
+ emitSimd128Move(from, to);
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+ }
+}
+
+MoveEmitterX86::~MoveEmitterX86() { assertDone(); }
+
+Address MoveEmitterX86::cycleSlot() {
+ if (pushedAtCycle_ == -1) {
+ // Reserve stack for cycle resolution
+ static_assert(SpillSlotSize == 16);
+ masm.reserveStack(SpillSlotSize);
+ pushedAtCycle_ = masm.framePushed();
+ }
+
+ return Address(StackPointer, masm.framePushed() - pushedAtCycle_);
+}
+
+Address MoveEmitterX86::toAddress(const MoveOperand& operand) const {
+ if (operand.base() != StackPointer) {
+ return Address(operand.base(), operand.disp());
+ }
+
+ MOZ_ASSERT(operand.disp() >= 0);
+
+ // Otherwise, the stack offset may need to be adjusted.
+ return Address(StackPointer,
+ operand.disp() + (masm.framePushed() - pushedAtStart_));
+}
+
+// Warning, do not use the resulting operand with pop instructions, since they
+// compute the effective destination address after altering the stack pointer.
+// Use toPopOperand if an Operand is needed for a pop.
+Operand MoveEmitterX86::toOperand(const MoveOperand& operand) const {
+ if (operand.isMemoryOrEffectiveAddress()) {
+ return Operand(toAddress(operand));
+ }
+ if (operand.isGeneralReg()) {
+ return Operand(operand.reg());
+ }
+
+ MOZ_ASSERT(operand.isFloatReg());
+ return Operand(operand.floatReg());
+}
+
+// This is the same as toOperand except that it computes an Operand suitable for
+// use in a pop.
+Operand MoveEmitterX86::toPopOperand(const MoveOperand& operand) const {
+ if (operand.isMemory()) {
+ if (operand.base() != StackPointer) {
+ return Operand(operand.base(), operand.disp());
+ }
+
+ MOZ_ASSERT(operand.disp() >= 0);
+
+ // Otherwise, the stack offset may need to be adjusted.
+ // Note the adjustment by the stack slot here, to offset for the fact that
+ // pop computes its effective address after incrementing the stack pointer.
+ return Operand(
+ StackPointer,
+ operand.disp() + (masm.framePushed() - sizeof(void*) - pushedAtStart_));
+ }
+ if (operand.isGeneralReg()) {
+ return Operand(operand.reg());
+ }
+
+ MOZ_ASSERT(operand.isFloatReg());
+ return Operand(operand.floatReg());
+}
+
+void MoveEmitterX86::breakCycle(const MoveOperand& to, MoveOp::Type type) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (A -> B), which we reach first. We save B, then allow
+ // the original move to continue.
+ switch (type) {
+ case MoveOp::SIMD128:
+ if (to.isMemory()) {
+ ScratchSimd128Scope scratch(masm);
+ masm.loadUnalignedSimd128(toAddress(to), scratch);
+ masm.storeUnalignedSimd128(scratch, cycleSlot());
+ } else {
+ masm.storeUnalignedSimd128(to.floatReg(), cycleSlot());
+ }
+ break;
+ case MoveOp::FLOAT32:
+ if (to.isMemory()) {
+ ScratchFloat32Scope scratch(masm);
+ masm.loadFloat32(toAddress(to), scratch);
+ masm.storeFloat32(scratch, cycleSlot());
+ } else {
+ masm.storeFloat32(to.floatReg(), cycleSlot());
+ }
+ break;
+ case MoveOp::DOUBLE:
+ if (to.isMemory()) {
+ ScratchDoubleScope scratch(masm);
+ masm.loadDouble(toAddress(to), scratch);
+ masm.storeDouble(scratch, cycleSlot());
+ } else {
+ masm.storeDouble(to.floatReg(), cycleSlot());
+ }
+ break;
+ case MoveOp::INT32:
+#ifdef JS_CODEGEN_X64
+ // x64 can't pop to a 32-bit destination, so don't push.
+ if (to.isMemory()) {
+ masm.load32(toAddress(to), ScratchReg);
+ masm.store32(ScratchReg, cycleSlot());
+ } else {
+ masm.store32(to.reg(), cycleSlot());
+ }
+ break;
+#endif
+ case MoveOp::GENERAL:
+ masm.Push(toOperand(to));
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterX86::completeCycle(const MoveOperand& to, MoveOp::Type type) {
+ // There is some pattern:
+ // (A -> B)
+ // (B -> A)
+ //
+ // This case handles (B -> A), which we reach last. We emit a move from the
+ // saved value of B, to A.
+ switch (type) {
+ case MoveOp::SIMD128:
+ MOZ_ASSERT(pushedAtCycle_ != -1);
+ MOZ_ASSERT(pushedAtCycle_ - pushedAtStart_ >= Simd128DataSize);
+ if (to.isMemory()) {
+ ScratchSimd128Scope scratch(masm);
+ masm.loadUnalignedSimd128(cycleSlot(), scratch);
+ masm.storeUnalignedSimd128(scratch, toAddress(to));
+ } else {
+ masm.loadUnalignedSimd128(cycleSlot(), to.floatReg());
+ }
+ break;
+ case MoveOp::FLOAT32:
+ MOZ_ASSERT(pushedAtCycle_ != -1);
+ MOZ_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(float));
+ if (to.isMemory()) {
+ ScratchFloat32Scope scratch(masm);
+ masm.loadFloat32(cycleSlot(), scratch);
+ masm.storeFloat32(scratch, toAddress(to));
+ } else {
+ masm.loadFloat32(cycleSlot(), to.floatReg());
+ }
+ break;
+ case MoveOp::DOUBLE:
+ MOZ_ASSERT(pushedAtCycle_ != -1);
+ MOZ_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(double));
+ if (to.isMemory()) {
+ ScratchDoubleScope scratch(masm);
+ masm.loadDouble(cycleSlot(), scratch);
+ masm.storeDouble(scratch, toAddress(to));
+ } else {
+ masm.loadDouble(cycleSlot(), to.floatReg());
+ }
+ break;
+ case MoveOp::INT32:
+#ifdef JS_CODEGEN_X64
+ MOZ_ASSERT(pushedAtCycle_ != -1);
+ MOZ_ASSERT(pushedAtCycle_ - pushedAtStart_ >= sizeof(int32_t));
+ // x64 can't pop to a 32-bit destination.
+ if (to.isMemory()) {
+ masm.load32(cycleSlot(), ScratchReg);
+ masm.store32(ScratchReg, toAddress(to));
+ } else {
+ masm.load32(cycleSlot(), to.reg());
+ }
+ break;
+#endif
+ case MoveOp::GENERAL:
+ MOZ_ASSERT(masm.framePushed() - pushedAtStart_ >= sizeof(intptr_t));
+ masm.Pop(toPopOperand(to));
+ break;
+ default:
+ MOZ_CRASH("Unexpected move type");
+ }
+}
+
+void MoveEmitterX86::emitInt32Move(const MoveOperand& from,
+ const MoveOperand& to,
+ const MoveResolver& moves, size_t i) {
+ if (from.isGeneralReg()) {
+ masm.move32(from.reg(), toOperand(to));
+ } else if (to.isGeneralReg()) {
+ MOZ_ASSERT(from.isMemory());
+ masm.load32(toAddress(from), to.reg());
+ } else {
+ // Memory to memory gpr move.
+ MOZ_ASSERT(from.isMemory());
+ Maybe<Register> reg = findScratchRegister(moves, i);
+ if (reg.isSome()) {
+ masm.load32(toAddress(from), reg.value());
+ masm.move32(reg.value(), toOperand(to));
+ } else {
+ // No scratch register available; bounce it off the stack.
+ masm.Push(toOperand(from));
+ masm.Pop(toPopOperand(to));
+ }
+ }
+}
+
+void MoveEmitterX86::emitGeneralMove(const MoveOperand& from,
+ const MoveOperand& to,
+ const MoveResolver& moves, size_t i) {
+ if (from.isGeneralReg()) {
+ masm.mov(from.reg(), toOperand(to));
+ } else if (to.isGeneralReg()) {
+ MOZ_ASSERT(from.isMemoryOrEffectiveAddress());
+ if (from.isMemory()) {
+ masm.loadPtr(toAddress(from), to.reg());
+ } else {
+ masm.lea(toOperand(from), to.reg());
+ }
+ } else if (from.isMemory()) {
+ // Memory to memory gpr move.
+ Maybe<Register> reg = findScratchRegister(moves, i);
+ if (reg.isSome()) {
+ masm.loadPtr(toAddress(from), reg.value());
+ masm.mov(reg.value(), toOperand(to));
+ } else {
+ // No scratch register available; bounce it off the stack.
+ masm.Push(toOperand(from));
+ masm.Pop(toPopOperand(to));
+ }
+ } else {
+ // Effective address to memory move.
+ MOZ_ASSERT(from.isEffectiveAddress());
+ Maybe<Register> reg = findScratchRegister(moves, i);
+ if (reg.isSome()) {
+ masm.lea(toOperand(from), reg.value());
+ masm.mov(reg.value(), toOperand(to));
+ } else {
+ // This is tricky without a scratch reg. We can't do an lea. Bounce the
+ // base register off the stack, then add the offset in place. Note that
+ // this clobbers FLAGS!
+ masm.Push(from.base());
+ masm.Pop(toPopOperand(to));
+ MOZ_ASSERT(to.isMemoryOrEffectiveAddress());
+ masm.addPtr(Imm32(from.disp()), toAddress(to));
+ }
+ }
+}
+
+void MoveEmitterX86::emitFloat32Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg().isSingle());
+ MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg().isSingle());
+
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveFloat32(from.floatReg(), to.floatReg());
+ } else {
+ masm.storeFloat32(from.floatReg(), toAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ masm.loadFloat32(toAddress(from), to.floatReg());
+ } else {
+ // Memory to memory move.
+ MOZ_ASSERT(from.isMemory());
+ ScratchFloat32Scope scratch(masm);
+ masm.loadFloat32(toAddress(from), scratch);
+ masm.storeFloat32(scratch, toAddress(to));
+ }
+}
+
+void MoveEmitterX86::emitDoubleMove(const MoveOperand& from,
+ const MoveOperand& to) {
+ MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg().isDouble());
+ MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg().isDouble());
+
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveDouble(from.floatReg(), to.floatReg());
+ } else {
+ masm.storeDouble(from.floatReg(), toAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ masm.loadDouble(toAddress(from), to.floatReg());
+ } else {
+ // Memory to memory move.
+ MOZ_ASSERT(from.isMemory());
+ ScratchDoubleScope scratch(masm);
+ masm.loadDouble(toAddress(from), scratch);
+ masm.storeDouble(scratch, toAddress(to));
+ }
+}
+
+void MoveEmitterX86::emitSimd128Move(const MoveOperand& from,
+ const MoveOperand& to) {
+ MOZ_ASSERT_IF(from.isFloatReg(), from.floatReg().isSimd128());
+ MOZ_ASSERT_IF(to.isFloatReg(), to.floatReg().isSimd128());
+
+ if (from.isFloatReg()) {
+ if (to.isFloatReg()) {
+ masm.moveSimd128(from.floatReg(), to.floatReg());
+ } else {
+ masm.storeUnalignedSimd128(from.floatReg(), toAddress(to));
+ }
+ } else if (to.isFloatReg()) {
+ masm.loadUnalignedSimd128(toAddress(from), to.floatReg());
+ } else {
+ // Memory to memory move.
+ MOZ_ASSERT(from.isMemory());
+ ScratchSimd128Scope scratch(masm);
+ masm.loadUnalignedSimd128(toAddress(from), scratch);
+ masm.storeUnalignedSimd128(scratch, toAddress(to));
+ }
+}
+
+void MoveEmitterX86::assertDone() { MOZ_ASSERT(!inCycle_); }
+
+void MoveEmitterX86::finish() {
+ assertDone();
+
+ masm.freeStack(masm.framePushed() - pushedAtStart_);
+}
+
+Maybe<Register> MoveEmitterX86::findScratchRegister(const MoveResolver& moves,
+ size_t initial) {
+#ifdef JS_CODEGEN_X86
+ if (scratchRegister_.isSome()) {
+ return scratchRegister_;
+ }
+
+ // All registers are either in use by this move group or are live
+ // afterwards. Look through the remaining moves for a register which is
+ // clobbered before it is used, and is thus dead at this point.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ for (size_t i = initial; i < moves.numMoves(); i++) {
+ const MoveOp& move = moves.getMove(i);
+ if (move.from().isGeneralReg()) {
+ regs.takeUnchecked(move.from().reg());
+ } else if (move.from().isMemoryOrEffectiveAddress()) {
+ regs.takeUnchecked(move.from().base());
+ }
+ if (move.to().isGeneralReg()) {
+ if (i != initial && !move.isCycleBegin() && regs.has(move.to().reg())) {
+ return mozilla::Some(move.to().reg());
+ }
+ regs.takeUnchecked(move.to().reg());
+ } else if (move.to().isMemoryOrEffectiveAddress()) {
+ regs.takeUnchecked(move.to().base());
+ }
+ }
+
+ return mozilla::Nothing();
+#else
+ return mozilla::Some(ScratchReg);
+#endif
+}
diff --git a/js/src/jit/x86-shared/MoveEmitter-x86-shared.h b/js/src/jit/x86-shared/MoveEmitter-x86-shared.h
new file mode 100644
index 0000000000..15a1680c9a
--- /dev/null
+++ b/js/src/jit/x86-shared/MoveEmitter-x86-shared.h
@@ -0,0 +1,83 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_MoveEmitter_x86_shared_h
+#define jit_MoveEmitter_x86_shared_h
+
+#include "mozilla/Maybe.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jit/MoveResolver.h"
+#include "jit/Registers.h"
+
+namespace js {
+namespace jit {
+
+struct Address;
+class MacroAssembler;
+class Operand;
+
+class MoveEmitterX86 {
+ bool inCycle_;
+ MacroAssembler& masm;
+
+ // Original stack push value.
+ uint32_t pushedAtStart_;
+
+ // This is a store stack offset for the cycle-break spill slot, snapshotting
+ // codegen->framePushed_ at the time it is allocated. -1 if not allocated.
+ int32_t pushedAtCycle_;
+
+#ifdef JS_CODEGEN_X86
+ // Optional scratch register for performing moves.
+ mozilla::Maybe<Register> scratchRegister_;
+#endif
+
+ void assertDone();
+ Address cycleSlot();
+ Address toAddress(const MoveOperand& operand) const;
+ Operand toOperand(const MoveOperand& operand) const;
+ Operand toPopOperand(const MoveOperand& operand) const;
+
+ size_t characterizeCycle(const MoveResolver& moves, size_t i,
+ bool* allGeneralRegs, bool* allFloatRegs);
+ bool maybeEmitOptimizedCycle(const MoveResolver& moves, size_t i,
+ bool allGeneralRegs, bool allFloatRegs,
+ size_t swapCount);
+ void emitInt32Move(const MoveOperand& from, const MoveOperand& to,
+ const MoveResolver& moves, size_t i);
+ void emitGeneralMove(const MoveOperand& from, const MoveOperand& to,
+ const MoveResolver& moves, size_t i);
+ void emitFloat32Move(const MoveOperand& from, const MoveOperand& to);
+ void emitDoubleMove(const MoveOperand& from, const MoveOperand& to);
+ void emitSimd128Move(const MoveOperand& from, const MoveOperand& to);
+ void breakCycle(const MoveOperand& to, MoveOp::Type type);
+ void completeCycle(const MoveOperand& to, MoveOp::Type type);
+
+ public:
+ explicit MoveEmitterX86(MacroAssembler& masm);
+ ~MoveEmitterX86();
+ void emit(const MoveResolver& moves);
+ void finish();
+
+ void setScratchRegister(Register reg) {
+#ifdef JS_CODEGEN_X86
+ scratchRegister_.emplace(reg);
+#endif
+ }
+
+ mozilla::Maybe<Register> findScratchRegister(const MoveResolver& moves,
+ size_t i);
+};
+
+using MoveEmitter = MoveEmitterX86;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_MoveEmitter_x86_shared_h */
diff --git a/js/src/jit/x86-shared/Patching-x86-shared.h b/js/src/jit/x86-shared/Patching-x86-shared.h
new file mode 100644
index 0000000000..85c523cd15
--- /dev/null
+++ b/js/src/jit/x86-shared/Patching-x86-shared.h
@@ -0,0 +1,113 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_shared_Patching_x86_shared_h
+#define jit_x86_shared_Patching_x86_shared_h
+
+namespace js {
+namespace jit {
+
+namespace X86Encoding {
+
+inline void* GetPointer(const void* where) {
+ void* res;
+ memcpy(&res, (const char*)where - sizeof(void*), sizeof(void*));
+ return res;
+}
+
+inline void SetPointer(void* where, const void* value) {
+ memcpy((char*)where - sizeof(void*), &value, sizeof(void*));
+}
+
+inline int32_t GetInt32(const void* where) {
+ int32_t res;
+ memcpy(&res, (const char*)where - sizeof(int32_t), sizeof(int32_t));
+ return res;
+}
+
+inline void SetInt32(void* where, int32_t value, uint32_t trailing = 0) {
+ memcpy((char*)where - trailing - sizeof(int32_t), &value, sizeof(int32_t));
+}
+
+inline void SetRel32(void* from, void* to, uint32_t trailing = 0) {
+ intptr_t offset =
+ reinterpret_cast<intptr_t>(to) - reinterpret_cast<intptr_t>(from);
+ MOZ_ASSERT(offset == static_cast<int32_t>(offset),
+ "offset is too great for a 32-bit relocation");
+ if (offset != static_cast<int32_t>(offset)) {
+ MOZ_CRASH("offset is too great for a 32-bit relocation");
+ }
+
+ SetInt32(from, offset, trailing);
+}
+
+inline void* GetRel32Target(void* where) {
+ int32_t rel = GetInt32(where);
+ return (char*)where + rel;
+}
+
+// JmpSrc represents a positive offset within a code buffer, or an uninitialized
+// value. Lots of code depends on uninitialized JmpSrc holding the value -1, on
+// -1 being a legal value of JmpSrc, and on being able to initialize a JmpSrc
+// with the value -1.
+//
+// The value of the `offset` is always positive and <= MaxCodeBytesPerProcess,
+// see ProcessExecutableMemory.h. The latter quantity in turn must fit in an
+// i32. But we further require that the value is not precisely INT32_MAX, so as
+// to allow the JmpSrc value -1 to mean "uninitialized" without ambiguity.
+//
+// The quantity `trailing` denotes the number of bytes of data that follow the
+// patch field in the instruction. The offset points to the end of the
+// instruction as per normal. The information about trailing bytes is needed
+// separately from the offset to correctly patch instructions that have
+// immediates trailing the patch field (eg CMPSS and CMPSD). Currently the only
+// allowed values for `trailing` are 0 and 1.
+
+static_assert(MaxCodeBytesPerProcess < size_t(INT32_MAX), "Invariant");
+
+class JmpSrc {
+ public:
+ JmpSrc() : offset_(INT32_MAX), trailing_(0) {}
+ explicit JmpSrc(int32_t offset) : offset_(offset), trailing_(0) {
+ // offset -1 is stored as INT32_MAX
+ MOZ_ASSERT(offset == -1 || (offset >= 0 && offset < INT32_MAX));
+ }
+ JmpSrc(int32_t offset, uint32_t trailing)
+ : offset_(offset), trailing_(trailing) {
+ // Disallow offset -1 in this situation, it does not apply.
+ MOZ_ASSERT(offset >= 0 && offset < INT32_MAX);
+ MOZ_ASSERT(trailing <= 1);
+ }
+ int32_t offset() const {
+ return offset_ == INT32_MAX ? -1 : int32_t(offset_);
+ }
+ uint32_t trailing() const { return trailing_; }
+
+ private:
+ uint32_t offset_ : 31;
+ uint32_t trailing_ : 1;
+};
+
+class JmpDst {
+ public:
+ explicit JmpDst(int32_t offset) : offset_(offset) {}
+ int32_t offset() const { return offset_; }
+
+ private:
+ int32_t offset_;
+};
+
+inline bool CanRelinkJump(void* from, void* to) {
+ intptr_t offset = static_cast<char*>(to) - static_cast<char*>(from);
+ return (offset == static_cast<int32_t>(offset));
+}
+
+} // namespace X86Encoding
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_shared_Patching_x86_shared_h */
diff --git a/js/src/jit/x86/Assembler-x86.cpp b/js/src/jit/x86/Assembler-x86.cpp
new file mode 100644
index 0000000000..7296f77291
--- /dev/null
+++ b/js/src/jit/x86/Assembler-x86.cpp
@@ -0,0 +1,85 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86/Assembler-x86.h"
+
+#include "gc/Marking.h"
+#include "util/Memory.h"
+
+using namespace js;
+using namespace js::jit;
+
+ABIArgGenerator::ABIArgGenerator() : stackOffset_(0), current_() {}
+
+ABIArg ABIArgGenerator::next(MIRType type) {
+ switch (type) {
+ case MIRType::Int32:
+ case MIRType::Float32:
+ case MIRType::Pointer:
+ case MIRType::RefOrNull:
+ case MIRType::StackResults:
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint32_t);
+ break;
+ case MIRType::Double:
+ case MIRType::Int64:
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += sizeof(uint64_t);
+ break;
+ case MIRType::Simd128:
+ // On Win64, >64 bit args need to be passed by reference. However, wasm
+ // doesn't allow passing SIMD values to JS, so the only way to reach this
+ // is wasm to wasm calls. Ergo we can break the native ABI here and use
+ // the Wasm ABI instead.
+ stackOffset_ = AlignBytes(stackOffset_, SimdMemoryAlignment);
+ current_ = ABIArg(stackOffset_);
+ stackOffset_ += Simd128DataSize;
+ break;
+ default:
+ MOZ_CRASH("Unexpected argument type");
+ }
+ return current_;
+}
+
+void Assembler::executableCopy(uint8_t* buffer) {
+ AssemblerX86Shared::executableCopy(buffer);
+ for (RelativePatch& rp : jumps_) {
+ X86Encoding::SetRel32(buffer + rp.offset, rp.target);
+ }
+}
+
+class RelocationIterator {
+ CompactBufferReader reader_;
+ uint32_t offset_;
+
+ public:
+ explicit RelocationIterator(CompactBufferReader& reader) : reader_(reader) {}
+
+ bool read() {
+ if (!reader_.more()) {
+ return false;
+ }
+ offset_ = reader_.readUnsigned();
+ return true;
+ }
+
+ uint32_t offset() const { return offset_; }
+};
+
+static inline JitCode* CodeFromJump(uint8_t* jump) {
+ uint8_t* target = (uint8_t*)X86Encoding::GetRel32Target(jump);
+ return JitCode::FromExecutable(target);
+}
+
+void Assembler::TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader) {
+ RelocationIterator iter(reader);
+ while (iter.read()) {
+ JitCode* child = CodeFromJump(code->raw() + iter.offset());
+ TraceManuallyBarrieredEdge(trc, &child, "rel32");
+ MOZ_ASSERT(child == CodeFromJump(code->raw() + iter.offset()));
+ }
+}
diff --git a/js/src/jit/x86/Assembler-x86.h b/js/src/jit/x86/Assembler-x86.h
new file mode 100644
index 0000000000..7befad1b55
--- /dev/null
+++ b/js/src/jit/x86/Assembler-x86.h
@@ -0,0 +1,1074 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_Assembler_x86_h
+#define jit_x86_Assembler_x86_h
+
+#include <iterator>
+
+#include "jit/CompactBuffer.h"
+#include "jit/JitCode.h"
+#include "jit/shared/Assembler-shared.h"
+#include "jit/x86-shared/Constants-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+static constexpr Register eax{X86Encoding::rax};
+static constexpr Register ecx{X86Encoding::rcx};
+static constexpr Register edx{X86Encoding::rdx};
+static constexpr Register ebx{X86Encoding::rbx};
+static constexpr Register esp{X86Encoding::rsp};
+static constexpr Register ebp{X86Encoding::rbp};
+static constexpr Register esi{X86Encoding::rsi};
+static constexpr Register edi{X86Encoding::rdi};
+
+static constexpr FloatRegister xmm0 =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+static constexpr FloatRegister xmm1 =
+ FloatRegister(X86Encoding::xmm1, FloatRegisters::Double);
+static constexpr FloatRegister xmm2 =
+ FloatRegister(X86Encoding::xmm2, FloatRegisters::Double);
+static constexpr FloatRegister xmm3 =
+ FloatRegister(X86Encoding::xmm3, FloatRegisters::Double);
+static constexpr FloatRegister xmm4 =
+ FloatRegister(X86Encoding::xmm4, FloatRegisters::Double);
+static constexpr FloatRegister xmm5 =
+ FloatRegister(X86Encoding::xmm5, FloatRegisters::Double);
+static constexpr FloatRegister xmm6 =
+ FloatRegister(X86Encoding::xmm6, FloatRegisters::Double);
+static constexpr FloatRegister xmm7 =
+ FloatRegister(X86Encoding::xmm7, FloatRegisters::Double);
+
+// Vector registers fixed for use with some instructions, e.g. PBLENDVB.
+static constexpr FloatRegister vmm0 =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128);
+
+static constexpr Register InvalidReg{X86Encoding::invalid_reg};
+static constexpr FloatRegister InvalidFloatReg = FloatRegister();
+
+static constexpr Register JSReturnReg_Type = ecx;
+static constexpr Register JSReturnReg_Data = edx;
+static constexpr Register StackPointer = esp;
+static constexpr Register FramePointer = ebp;
+static constexpr Register ReturnReg = eax;
+static constexpr FloatRegister ReturnFloat32Reg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Single);
+static constexpr FloatRegister ReturnDoubleReg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+static constexpr FloatRegister ReturnSimd128Reg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Simd128);
+static constexpr FloatRegister ScratchFloat32Reg_ =
+ FloatRegister(X86Encoding::xmm7, FloatRegisters::Single);
+static constexpr FloatRegister ScratchDoubleReg_ =
+ FloatRegister(X86Encoding::xmm7, FloatRegisters::Double);
+static constexpr FloatRegister ScratchSimd128Reg =
+ FloatRegister(X86Encoding::xmm7, FloatRegisters::Simd128);
+
+// Note, EDX:EAX is the system ABI 64-bit return register, and it is to our
+// advantage to keep the SpiderMonkey ABI in sync with the system ABI.
+//
+// However, using EDX here means that we have to use a register that does not
+// have a word or byte part (eg DX/DH/DL) in some other places; notably,
+// ABINonArgReturnReg1 is EDI. If this becomes a problem and ReturnReg64 has to
+// be something other than EDX:EAX, then jitted code that calls directly to C++
+// will need to shuffle the return value from EDX:EAX into ReturnReg64 directly
+// after the call. See bug 1730161 for discussion and a patch that does that.
+static constexpr Register64 ReturnReg64(edx, eax);
+
+// Avoid ebp, which is the FramePointer, which is unavailable in some modes.
+static constexpr Register CallTempReg0 = edi;
+static constexpr Register CallTempReg1 = eax;
+static constexpr Register CallTempReg2 = ebx;
+static constexpr Register CallTempReg3 = ecx;
+static constexpr Register CallTempReg4 = esi;
+static constexpr Register CallTempReg5 = edx;
+
+// We have no arg regs, so our NonArgRegs are just our CallTempReg*
+static constexpr Register CallTempNonArgRegs[] = {edi, eax, ebx, ecx, esi, edx};
+static constexpr uint32_t NumCallTempNonArgRegs = std::size(CallTempNonArgRegs);
+
+class ABIArgGenerator {
+ uint32_t stackOffset_;
+ ABIArg current_;
+
+ public:
+ ABIArgGenerator();
+ ABIArg next(MIRType argType);
+ ABIArg& current() { return current_; }
+ uint32_t stackBytesConsumedSoFar() const { return stackOffset_; }
+ void increaseStackOffset(uint32_t bytes) { stackOffset_ += bytes; }
+};
+
+// These registers may be volatile or nonvolatile.
+static constexpr Register ABINonArgReg0 = eax;
+static constexpr Register ABINonArgReg1 = ebx;
+static constexpr Register ABINonArgReg2 = ecx;
+static constexpr Register ABINonArgReg3 = edx;
+
+// This register may be volatile or nonvolatile. Avoid xmm7 which is the
+// ScratchDoubleReg_.
+static constexpr FloatRegister ABINonArgDoubleReg =
+ FloatRegister(X86Encoding::xmm0, FloatRegisters::Double);
+
+// These registers may be volatile or nonvolatile.
+// Note: these three registers are all guaranteed to be different
+static constexpr Register ABINonArgReturnReg0 = ecx;
+static constexpr Register ABINonArgReturnReg1 = edi;
+static constexpr Register ABINonVolatileReg = ebx;
+
+// This register is guaranteed to be clobberable during the prologue and
+// epilogue of an ABI call which must preserve both ABI argument, return
+// and non-volatile registers.
+static constexpr Register ABINonArgReturnVolatileReg = ecx;
+
+// Instance pointer argument register for WebAssembly functions. This must not
+// alias any other register used for passing function arguments or return
+// values. Preserved by WebAssembly functions.
+static constexpr Register InstanceReg = esi;
+
+// Registers used for asm.js/wasm table calls. These registers must be disjoint
+// from the ABI argument registers, InstanceReg and each other.
+static constexpr Register WasmTableCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmTableCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmTableCallSigReg = ABINonArgReg2;
+static constexpr Register WasmTableCallIndexReg = ABINonArgReg3;
+
+// Registers used for ref calls.
+static constexpr Register WasmCallRefCallScratchReg0 = ABINonArgReg0;
+static constexpr Register WasmCallRefCallScratchReg1 = ABINonArgReg1;
+static constexpr Register WasmCallRefReg = ABINonArgReg3;
+
+// Register used as a scratch along the return path in the fast js -> wasm stub
+// code. This must not overlap ReturnReg, JSReturnOperand, or InstanceReg.
+// It must be a volatile register.
+static constexpr Register WasmJitEntryReturnScratch = ebx;
+
+static constexpr Register OsrFrameReg = edx;
+static constexpr Register PreBarrierReg = edx;
+
+// Not enough registers for a PC register (R0-R2 use 2 registers each).
+static constexpr Register InterpreterPCReg = InvalidReg;
+
+// Registerd used in RegExpMatcher instruction (do not use JSReturnOperand).
+static constexpr Register RegExpMatcherRegExpReg = CallTempReg0;
+static constexpr Register RegExpMatcherStringReg = CallTempReg1;
+static constexpr Register RegExpMatcherLastIndexReg = CallTempReg2;
+
+// Registerd used in RegExpTester instruction (do not use ReturnReg).
+static constexpr Register RegExpTesterRegExpReg = CallTempReg0;
+static constexpr Register RegExpTesterStringReg = CallTempReg2;
+static constexpr Register RegExpTesterLastIndexReg = CallTempReg3;
+
+// GCC stack is aligned on 16 bytes. Ion does not maintain this for internal
+// calls. wasm code does.
+#if defined(__GNUC__) && !defined(__MINGW32__)
+static constexpr uint32_t ABIStackAlignment = 16;
+#else
+static constexpr uint32_t ABIStackAlignment = 4;
+#endif
+static constexpr uint32_t CodeAlignment = 16;
+static constexpr uint32_t JitStackAlignment = 16;
+
+static constexpr uint32_t JitStackValueAlignment =
+ JitStackAlignment / sizeof(Value);
+static_assert(JitStackAlignment % sizeof(Value) == 0 &&
+ JitStackValueAlignment >= 1,
+ "Stack alignment should be a non-zero multiple of sizeof(Value)");
+
+static constexpr uint32_t SimdMemoryAlignment = 16;
+
+static_assert(CodeAlignment % SimdMemoryAlignment == 0,
+ "Code alignment should be larger than any of the alignments "
+ "which are used for "
+ "the constant sections of the code buffer. Thus it should be "
+ "larger than the "
+ "alignment for SIMD constants.");
+
+static_assert(JitStackAlignment % SimdMemoryAlignment == 0,
+ "Stack alignment should be larger than any of the alignments "
+ "which are used for "
+ "spilled values. Thus it should be larger than the alignment "
+ "for SIMD accesses.");
+
+static constexpr uint32_t WasmStackAlignment = SimdMemoryAlignment;
+static constexpr uint32_t WasmTrapInstructionLength = 2;
+
+// See comments in wasm::GenerateFunctionPrologue. The difference between these
+// is the size of the largest callable prologue on the platform. (We could make
+// the tail offset 3, but I have opted for 4 as that results in a better-aligned
+// branch target.)
+static constexpr uint32_t WasmCheckedCallEntryOffset = 0u;
+
+struct ImmTag : public Imm32 {
+ explicit ImmTag(JSValueTag mask) : Imm32(int32_t(mask)) {}
+};
+
+struct ImmType : public ImmTag {
+ explicit ImmType(JSValueType type) : ImmTag(JSVAL_TYPE_TO_TAG(type)) {}
+};
+
+static constexpr Scale ScalePointer = TimesFour;
+
+} // namespace jit
+} // namespace js
+
+#include "jit/x86-shared/Assembler-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+static inline Operand LowWord(const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ return Operand(LowWord(op.toAddress()));
+ case Operand::MEM_SCALE:
+ return Operand(LowWord(op.toBaseIndex()));
+ default:
+ MOZ_CRASH("Invalid operand type");
+ }
+}
+
+static inline Operand HighWord(const Operand& op) {
+ switch (op.kind()) {
+ case Operand::MEM_REG_DISP:
+ return Operand(HighWord(op.toAddress()));
+ case Operand::MEM_SCALE:
+ return Operand(HighWord(op.toBaseIndex()));
+ default:
+ MOZ_CRASH("Invalid operand type");
+ }
+}
+
+// Return operand from a JS -> JS call.
+static constexpr ValueOperand JSReturnOperand{JSReturnReg_Type,
+ JSReturnReg_Data};
+
+class Assembler : public AssemblerX86Shared {
+ Vector<RelativePatch, 8, SystemAllocPolicy> jumps_;
+
+ void addPendingJump(JmpSrc src, ImmPtr target, RelocationKind kind) {
+ enoughMemory_ &=
+ jumps_.append(RelativePatch(src.offset(), target.value, kind));
+ if (kind == RelocationKind::JITCODE) {
+ jumpRelocations_.writeUnsigned(src.offset());
+ }
+ }
+
+ public:
+ using AssemblerX86Shared::call;
+ using AssemblerX86Shared::cmpl;
+ using AssemblerX86Shared::j;
+ using AssemblerX86Shared::jmp;
+ using AssemblerX86Shared::movl;
+ using AssemblerX86Shared::pop;
+ using AssemblerX86Shared::push;
+ using AssemblerX86Shared::retarget;
+ using AssemblerX86Shared::vmovsd;
+ using AssemblerX86Shared::vmovss;
+
+ static void TraceJumpRelocations(JSTracer* trc, JitCode* code,
+ CompactBufferReader& reader);
+
+ // Copy the assembly code to the given buffer, and perform any pending
+ // relocations relying on the target address.
+ void executableCopy(uint8_t* buffer);
+
+ void assertNoGCThings() const {
+#ifdef DEBUG
+ MOZ_ASSERT(dataRelocations_.length() == 0);
+ for (auto& j : jumps_) {
+ MOZ_ASSERT(j.kind == RelocationKind::HARDCODED);
+ }
+#endif
+ }
+
+ // Actual assembly emitting functions.
+
+ void push(ImmGCPtr ptr) {
+ masm.push_i32(int32_t(ptr.value));
+ writeDataRelocation(ptr);
+ }
+ void push(const ImmWord imm) { push(Imm32(imm.value)); }
+ void push(const ImmPtr imm) { push(ImmWord(uintptr_t(imm.value))); }
+ void push(FloatRegister src) {
+ subl(Imm32(sizeof(double)), StackPointer);
+ vmovsd(src, Address(StackPointer, 0));
+ }
+
+ CodeOffset pushWithPatch(ImmWord word) {
+ masm.push_i32(int32_t(word.value));
+ return CodeOffset(masm.currentOffset());
+ }
+
+ void pop(FloatRegister src) {
+ vmovsd(Address(StackPointer, 0), src);
+ addl(Imm32(sizeof(double)), StackPointer);
+ }
+
+ CodeOffset movWithPatch(ImmWord word, Register dest) {
+ movl(Imm32(word.value), dest);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movWithPatch(ImmPtr imm, Register dest) {
+ return movWithPatch(ImmWord(uintptr_t(imm.value)), dest);
+ }
+
+ void movl(ImmGCPtr ptr, Register dest) {
+ masm.movl_i32r(uintptr_t(ptr.value), dest.encoding());
+ writeDataRelocation(ptr);
+ }
+ void movl(ImmGCPtr ptr, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::REG:
+ masm.movl_i32r(uintptr_t(ptr.value), dest.reg());
+ writeDataRelocation(ptr);
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.movl_i32m(uintptr_t(ptr.value), dest.disp(), dest.base());
+ writeDataRelocation(ptr);
+ break;
+ case Operand::MEM_SCALE:
+ masm.movl_i32m(uintptr_t(ptr.value), dest.disp(), dest.base(),
+ dest.index(), dest.scale());
+ writeDataRelocation(ptr);
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void movl(ImmWord imm, Register dest) {
+ masm.movl_i32r(imm.value, dest.encoding());
+ }
+ void movl(ImmPtr imm, Register dest) {
+ movl(ImmWord(uintptr_t(imm.value)), dest);
+ }
+ void mov(ImmWord imm, Register dest) {
+ // Use xor for setting registers to zero, as it is specially optimized
+ // for this purpose on modern hardware. Note that it does clobber FLAGS
+ // though.
+ if (imm.value == 0) {
+ xorl(dest, dest);
+ } else {
+ movl(imm, dest);
+ }
+ }
+ void mov(ImmPtr imm, Register dest) {
+ mov(ImmWord(uintptr_t(imm.value)), dest);
+ }
+ void mov(wasm::SymbolicAddress imm, Register dest) {
+ masm.movl_i32r(-1, dest.encoding());
+ append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), imm));
+ }
+ void mov(const Operand& src, Register dest) { movl(src, dest); }
+ void mov(Register src, const Operand& dest) { movl(src, dest); }
+ void mov(Imm32 imm, const Operand& dest) { movl(imm, dest); }
+ void mov(CodeLabel* label, Register dest) {
+ // Put a placeholder value in the instruction stream.
+ masm.movl_i32r(0, dest.encoding());
+ label->patchAt()->bind(masm.size());
+ }
+ void mov(Register src, Register dest) { movl(src, dest); }
+ void xchg(Register src, Register dest) { xchgl(src, dest); }
+ void lea(const Operand& src, Register dest) { return leal(src, dest); }
+ void cmovz32(const Operand& src, Register dest) { return cmovzl(src, dest); }
+ void cmovzPtr(const Operand& src, Register dest) { return cmovzl(src, dest); }
+
+ void fstp32(const Operand& src) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fstp32_m(src.disp(), src.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void faddp() { masm.faddp(); }
+
+ void cmpl(ImmWord rhs, Register lhs) {
+ masm.cmpl_ir(rhs.value, lhs.encoding());
+ }
+ void cmpl(ImmPtr rhs, Register lhs) {
+ cmpl(ImmWord(uintptr_t(rhs.value)), lhs);
+ }
+ void cmpl(ImmGCPtr rhs, Register lhs) {
+ masm.cmpl_i32r(uintptr_t(rhs.value), lhs.encoding());
+ writeDataRelocation(rhs);
+ }
+ void cmpl(Register rhs, Register lhs) {
+ masm.cmpl_rr(rhs.encoding(), lhs.encoding());
+ }
+ void cmpl(ImmGCPtr rhs, const Operand& lhs) {
+ switch (lhs.kind()) {
+ case Operand::REG:
+ masm.cmpl_i32r(uintptr_t(rhs.value), lhs.reg());
+ writeDataRelocation(rhs);
+ break;
+ case Operand::MEM_REG_DISP:
+ masm.cmpl_i32m(uintptr_t(rhs.value), lhs.disp(), lhs.base());
+ writeDataRelocation(rhs);
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.cmpl_i32m(uintptr_t(rhs.value), lhs.address());
+ writeDataRelocation(rhs);
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ void cmpl(Register rhs, wasm::SymbolicAddress lhs) {
+ masm.cmpl_rm_disp32(rhs.encoding(), (void*)-1);
+ append(wasm::SymbolicAccess(CodeOffset(masm.currentOffset()), lhs));
+ }
+ void cmpl(Imm32 rhs, wasm::SymbolicAddress lhs) {
+ JmpSrc src = masm.cmpl_im_disp32(rhs.value, (void*)-1);
+ append(wasm::SymbolicAccess(CodeOffset(src.offset()), lhs));
+ }
+
+ void adcl(Imm32 imm, Register dest) {
+ masm.adcl_ir(imm.value, dest.encoding());
+ }
+ void adcl(Register src, Register dest) {
+ masm.adcl_rr(src.encoding(), dest.encoding());
+ }
+ void adcl(Operand src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.adcl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.adcl_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void sbbl(Imm32 imm, Register dest) {
+ masm.sbbl_ir(imm.value, dest.encoding());
+ }
+ void sbbl(Register src, Register dest) {
+ masm.sbbl_rr(src.encoding(), dest.encoding());
+ }
+ void sbbl(Operand src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.sbbl_mr(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.sbbl_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void mull(Register multiplier) { masm.mull_r(multiplier.encoding()); }
+
+ void shldl(const Imm32 imm, Register src, Register dest) {
+ masm.shldl_irr(imm.value, src.encoding(), dest.encoding());
+ }
+ void shrdl(const Imm32 imm, Register src, Register dest) {
+ masm.shrdl_irr(imm.value, src.encoding(), dest.encoding());
+ }
+
+ void vhaddpd(FloatRegister rhs, FloatRegister lhsDest) {
+ MOZ_ASSERT(HasSSE3());
+ MOZ_ASSERT(rhs.size() == 16);
+ MOZ_ASSERT(lhsDest.size() == 16);
+ masm.vhaddpd_rr(rhs.encoding(), lhsDest.encoding(), lhsDest.encoding());
+ }
+
+ void fild(const Operand& src) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.fild_m(src.disp(), src.base());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+
+ void jmp(ImmPtr target, RelocationKind reloc = RelocationKind::HARDCODED) {
+ JmpSrc src = masm.jmp();
+ addPendingJump(src, target, reloc);
+ }
+ void j(Condition cond, ImmPtr target,
+ RelocationKind reloc = RelocationKind::HARDCODED) {
+ JmpSrc src = masm.jCC(static_cast<X86Encoding::Condition>(cond));
+ addPendingJump(src, target, reloc);
+ }
+
+ void jmp(JitCode* target) {
+ jmp(ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void j(Condition cond, JitCode* target) {
+ j(cond, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void call(JitCode* target) {
+ JmpSrc src = masm.call();
+ addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ }
+ void call(ImmWord target) { call(ImmPtr((void*)target.value)); }
+ void call(ImmPtr target) {
+ JmpSrc src = masm.call();
+ addPendingJump(src, target, RelocationKind::HARDCODED);
+ }
+
+ // Emit a CALL or CMP (nop) instruction. ToggleCall can be used to patch
+ // this instruction.
+ CodeOffset toggledCall(JitCode* target, bool enabled) {
+ CodeOffset offset(size());
+ JmpSrc src = enabled ? masm.call() : masm.cmp_eax();
+ addPendingJump(src, ImmPtr(target->raw()), RelocationKind::JITCODE);
+ MOZ_ASSERT_IF(!oom(), size() - offset.offset() == ToggledCallSize(nullptr));
+ return offset;
+ }
+
+ static size_t ToggledCallSize(uint8_t* code) {
+ // Size of a call instruction.
+ return 5;
+ }
+
+ // Re-routes pending jumps to an external target, flushing the label in the
+ // process.
+ void retarget(Label* label, ImmPtr target, RelocationKind reloc) {
+ if (label->used()) {
+ bool more;
+ X86Encoding::JmpSrc jmp(label->offset());
+ do {
+ X86Encoding::JmpSrc next;
+ more = masm.nextJump(jmp, &next);
+ addPendingJump(jmp, target, reloc);
+ jmp = next;
+ } while (more);
+ }
+ label->reset();
+ }
+
+ // Move a 32-bit immediate into a register where the immediate can be
+ // patched.
+ CodeOffset movlWithPatch(Imm32 imm, Register dest) {
+ masm.movl_i32r(imm.value, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+
+ // Load from *(base + disp32) where disp32 can be patched.
+ CodeOffset movsblWithPatch(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movsbl_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movsbl_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movzblWithPatch(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movzbl_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movzbl_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movswlWithPatch(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movswl_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movswl_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movzwlWithPatch(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movzwl_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movzwl_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movlWithPatch(const Operand& src, Register dest) {
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movl_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movl_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovssWithPatch(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovss_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovss_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ void vmovss(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovss_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovss_mr(src.address(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovss_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ CodeOffset vmovdWithPatch(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovd_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovd_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovqWithPatch(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovq_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovq_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovsdWithPatch(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovsd_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovsd_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ void vmovsd(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovsd_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovsd_mr(src.address(), dest.encoding());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovsd_mr(src.disp(), src.base(), src.index(), src.scale(),
+ dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ CodeOffset vmovupsWithPatch(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovups_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovups_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovdquWithPatch(const Operand& src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (src.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovdqu_mr_disp32(src.disp(), src.base(), dest.encoding());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovdqu_mr(src.address(), dest.encoding());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+
+ // Store to *(base + disp32) where disp32 can be patched.
+ CodeOffset movbWithPatch(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movb_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movb_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movwWithPatch(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movw_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movw_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movlWithPatch(Register src, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.movl_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.movl_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movlWithPatchLow(Register regLow, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP: {
+ return movlWithPatch(regLow, LowWord(dest));
+ }
+ case Operand::MEM_ADDRESS32: {
+ Operand low(
+ PatchedAbsoluteAddress(uint32_t(dest.address()) + INT64LOW_OFFSET));
+ return movlWithPatch(regLow, low);
+ }
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ CodeOffset movlWithPatchHigh(Register regHigh, const Operand& dest) {
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP: {
+ return movlWithPatch(regHigh, HighWord(dest));
+ }
+ case Operand::MEM_ADDRESS32: {
+ Operand high(PatchedAbsoluteAddress(uint32_t(dest.address()) +
+ INT64HIGH_OFFSET));
+ return movlWithPatch(regHigh, high);
+ }
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ CodeOffset vmovdWithPatch(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovd_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovd_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovqWithPatch(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovq_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovq_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovssWithPatch(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovss_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovss_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ void vmovss(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovss_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovss_rm(src.encoding(), dest.address());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovss_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ CodeOffset vmovsdWithPatch(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovsd_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovsd_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ void vmovsd(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovsd_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovsd_rm(src.encoding(), dest.address());
+ break;
+ case Operand::MEM_SCALE:
+ masm.vmovsd_rm(src.encoding(), dest.disp(), dest.base(), dest.index(),
+ dest.scale());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ CodeOffset vmovupsWithPatch(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovups_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovups_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovdquWithPatch(FloatRegister src, const Operand& dest) {
+ MOZ_ASSERT(HasSSE2());
+ switch (dest.kind()) {
+ case Operand::MEM_REG_DISP:
+ masm.vmovdqu_rm_disp32(src.encoding(), dest.disp(), dest.base());
+ break;
+ case Operand::MEM_ADDRESS32:
+ masm.vmovdqu_rm(src.encoding(), dest.address());
+ break;
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ return CodeOffset(masm.currentOffset());
+ }
+
+ // Load from *(addr + index*scale) where addr can be patched.
+ CodeOffset movlWithPatch(PatchedAbsoluteAddress addr, Register index,
+ Scale scale, Register dest) {
+ masm.movl_mr(addr.addr, index.encoding(), scale, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+
+ // Load from *src where src can be patched.
+ CodeOffset movsblWithPatch(PatchedAbsoluteAddress src, Register dest) {
+ masm.movsbl_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movzblWithPatch(PatchedAbsoluteAddress src, Register dest) {
+ masm.movzbl_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movswlWithPatch(PatchedAbsoluteAddress src, Register dest) {
+ masm.movswl_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movzwlWithPatch(PatchedAbsoluteAddress src, Register dest) {
+ masm.movzwl_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movlWithPatch(PatchedAbsoluteAddress src, Register dest) {
+ masm.movl_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovssWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovss_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovdWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovd_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovqWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovq_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovsdWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovsd_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovdqaWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovdqa_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovdquWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovdqu_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovapsWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovaps_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovupsWithPatch(PatchedAbsoluteAddress src, FloatRegister dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovups_mr(src.addr, dest.encoding());
+ return CodeOffset(masm.currentOffset());
+ }
+
+ // Store to *dest where dest can be patched.
+ CodeOffset movbWithPatch(Register src, PatchedAbsoluteAddress dest) {
+ masm.movb_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movwWithPatch(Register src, PatchedAbsoluteAddress dest) {
+ masm.movw_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset movlWithPatch(Register src, PatchedAbsoluteAddress dest) {
+ masm.movl_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovssWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovss_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovdWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovd_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovqWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovq_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovsdWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovsd_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovdqaWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovdqa_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovapsWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovaps_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovdquWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovdqu_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+ CodeOffset vmovupsWithPatch(FloatRegister src, PatchedAbsoluteAddress dest) {
+ MOZ_ASSERT(HasSSE2());
+ masm.vmovups_rm(src.encoding(), dest.addr);
+ return CodeOffset(masm.currentOffset());
+ }
+};
+
+// Get a register in which we plan to put a quantity that will be used as an
+// integer argument. This differs from GetIntArgReg in that if we have no more
+// actual argument registers to use we will fall back on using whatever
+// CallTempReg* don't overlap the argument registers, and only fail once those
+// run out too.
+static inline bool GetTempRegForIntArg(uint32_t usedIntArgs,
+ uint32_t usedFloatArgs, Register* out) {
+ if (usedIntArgs >= NumCallTempNonArgRegs) {
+ return false;
+ }
+ *out = CallTempNonArgRegs[usedIntArgs];
+ return true;
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_Assembler_x86_h */
diff --git a/js/src/jit/x86/BaseAssembler-x86.h b/js/src/jit/x86/BaseAssembler-x86.h
new file mode 100644
index 0000000000..a5a5f67bf2
--- /dev/null
+++ b/js/src/jit/x86/BaseAssembler-x86.h
@@ -0,0 +1,190 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_BaseAssembler_x86_h
+#define jit_x86_BaseAssembler_x86_h
+
+#include "jit/x86-shared/BaseAssembler-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+namespace X86Encoding {
+
+class BaseAssemblerX86 : public BaseAssembler {
+ public:
+ // Arithmetic operations:
+
+ void adcl_ir(int32_t imm, RegisterID dst) {
+ spew("adcl $%d, %s", imm, GPReg32Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_ADC);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_ADC);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void adcl_im(int32_t imm, const void* addr) {
+ spew("adcl %d, %p", imm, addr);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_ADC);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_ADC);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void adcl_rr(RegisterID src, RegisterID dst) {
+ spew("adcl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_ADC_GvEv, src, dst);
+ }
+
+ void adcl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("adcl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_ADC_GvEv, offset, base, dst);
+ }
+
+ void adcl_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("adcl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_ADC_GvEv, offset, base, index, scale, dst);
+ }
+
+ void sbbl_ir(int32_t imm, RegisterID dst) {
+ spew("sbbl $%d, %s", imm, GPReg32Name(dst));
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, dst, GROUP1_OP_SBB);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, dst, GROUP1_OP_SBB);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void sbbl_rr(RegisterID src, RegisterID dst) {
+ spew("sbbl %s, %s", GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_SBB_GvEv, src, dst);
+ }
+
+ void sbbl_mr(int32_t offset, RegisterID base, RegisterID dst) {
+ spew("sbbl " MEM_ob ", %s", ADDR_ob(offset, base), GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_SBB_GvEv, offset, base, dst);
+ }
+
+ void sbbl_mr(int32_t offset, RegisterID base, RegisterID index, int scale,
+ RegisterID dst) {
+ spew("sbbl " MEM_obs ", %s", ADDR_obs(offset, base, index, scale),
+ GPReg32Name(dst));
+ m_formatter.oneByteOp(OP_SBB_GvEv, offset, base, index, scale, dst);
+ }
+
+ using BaseAssembler::andl_im;
+ void andl_im(int32_t imm, const void* addr) {
+ spew("andl $0x%x, %p", imm, addr);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_AND);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_AND);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ using BaseAssembler::orl_im;
+ void orl_im(int32_t imm, const void* addr) {
+ spew("orl $0x%x, %p", imm, addr);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_OR);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_OR);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ using BaseAssembler::subl_im;
+ void subl_im(int32_t imm, const void* addr) {
+ spew("subl $%d, %p", imm, addr);
+ if (CAN_SIGN_EXTEND_8_32(imm)) {
+ m_formatter.oneByteOp(OP_GROUP1_EvIb, addr, GROUP1_OP_SUB);
+ m_formatter.immediate8s(imm);
+ } else {
+ m_formatter.oneByteOp(OP_GROUP1_EvIz, addr, GROUP1_OP_SUB);
+ m_formatter.immediate32(imm);
+ }
+ }
+
+ void shldl_irr(int32_t imm, RegisterID src, RegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ spew("shldl $%d, %s, %s", imm, GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp8(OP2_SHLD, dst, src);
+ m_formatter.immediate8u(imm);
+ }
+
+ void shrdl_irr(int32_t imm, RegisterID src, RegisterID dst) {
+ MOZ_ASSERT(imm < 32);
+ spew("shrdl $%d, %s, %s", imm, GPReg32Name(src), GPReg32Name(dst));
+ m_formatter.twoByteOp8(OP2_SHRD, dst, src);
+ m_formatter.immediate8u(imm);
+ }
+
+ // SSE operations:
+
+ using BaseAssembler::vcvtsi2sd_mr;
+ void vcvtsi2sd_mr(const void* address, XMMRegisterID src0,
+ XMMRegisterID dst) {
+ twoByteOpSimd("vcvtsi2sd", VEX_SD, OP2_CVTSI2SD_VsdEd, address, src0, dst);
+ }
+
+ using BaseAssembler::vmovaps_mr;
+ void vmovaps_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vmovaps", VEX_PS, OP2_MOVAPS_VsdWsd, address, invalid_xmm,
+ dst);
+ }
+
+ using BaseAssembler::vmovdqa_mr;
+ void vmovdqa_mr(const void* address, XMMRegisterID dst) {
+ twoByteOpSimd("vmovdqa", VEX_PD, OP2_MOVDQ_VdqWdq, address, invalid_xmm,
+ dst);
+ }
+
+ void vhaddpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vhaddpd", VEX_PD, OP2_HADDPD, src1, src0, dst);
+ }
+
+ void vsubpd_rr(XMMRegisterID src1, XMMRegisterID src0, XMMRegisterID dst) {
+ twoByteOpSimd("vsubpd", VEX_PD, OP2_SUBPS_VpsWps, src1, src0, dst);
+ }
+
+ void fild_m(int32_t offset, RegisterID base) {
+ m_formatter.oneByteOp(OP_FILD, offset, base, FILD_OP_64);
+ }
+
+ // Misc instructions:
+
+ void pusha() {
+ spew("pusha");
+ m_formatter.oneByteOp(OP_PUSHA);
+ }
+
+ void popa() {
+ spew("popa");
+ m_formatter.oneByteOp(OP_POPA);
+ }
+};
+
+typedef BaseAssemblerX86 BaseAssemblerSpecific;
+
+} // namespace X86Encoding
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_BaseAssembler_x86_h */
diff --git a/js/src/jit/x86/CodeGenerator-x86.cpp b/js/src/jit/x86/CodeGenerator-x86.cpp
new file mode 100644
index 0000000000..ce0fb01d9e
--- /dev/null
+++ b/js/src/jit/x86/CodeGenerator-x86.cpp
@@ -0,0 +1,1509 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86/CodeGenerator-x86.h"
+
+#include "mozilla/Casting.h"
+#include "mozilla/DebugOnly.h"
+
+#include <iterator>
+
+#include "jsnum.h"
+
+#include "jit/CodeGenerator.h"
+#include "jit/MIR.h"
+#include "jit/MIRGraph.h"
+#include "js/Conversions.h"
+#include "vm/Shape.h"
+#include "wasm/WasmBuiltins.h"
+#include "wasm/WasmCodegenTypes.h"
+#include "wasm/WasmInstanceData.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "jit/shared/CodeGenerator-shared-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+using JS::GenericNaN;
+using mozilla::BitwiseCast;
+using mozilla::DebugOnly;
+using mozilla::FloatingPoint;
+
+CodeGeneratorX86::CodeGeneratorX86(MIRGenerator* gen, LIRGraph* graph,
+ MacroAssembler* masm)
+ : CodeGeneratorX86Shared(gen, graph, masm) {}
+
+ValueOperand CodeGeneratorX86::ToValue(LInstruction* ins, size_t pos) {
+ Register typeReg = ToRegister(ins->getOperand(pos + TYPE_INDEX));
+ Register payloadReg = ToRegister(ins->getOperand(pos + PAYLOAD_INDEX));
+ return ValueOperand(typeReg, payloadReg);
+}
+
+ValueOperand CodeGeneratorX86::ToTempValue(LInstruction* ins, size_t pos) {
+ Register typeReg = ToRegister(ins->getTemp(pos + TYPE_INDEX));
+ Register payloadReg = ToRegister(ins->getTemp(pos + PAYLOAD_INDEX));
+ return ValueOperand(typeReg, payloadReg);
+}
+
+void CodeGenerator::visitValue(LValue* value) {
+ const ValueOperand out = ToOutValue(value);
+ masm.moveValue(value->value(), out);
+}
+
+void CodeGenerator::visitBox(LBox* box) {
+ const LDefinition* type = box->getDef(TYPE_INDEX);
+
+ DebugOnly<const LAllocation*> a = box->getOperand(0);
+ MOZ_ASSERT(!a->isConstant());
+
+ // On x86, the input operand and the output payload have the same
+ // virtual register. All that needs to be written is the type tag for
+ // the type definition.
+ masm.mov(ImmWord(MIRTypeToTag(box->type())), ToRegister(type));
+}
+
+void CodeGenerator::visitBoxFloatingPoint(LBoxFloatingPoint* box) {
+ const AnyRegister in = ToAnyRegister(box->getOperand(0));
+ const ValueOperand out = ToOutValue(box);
+
+ masm.moveValue(TypedOrValueRegister(box->type(), in), out);
+
+ if (JitOptions.spectreValueMasking) {
+ Register scratch = ToRegister(box->spectreTemp());
+ masm.move32(Imm32(JSVAL_TAG_CLEAR), scratch);
+ masm.cmp32Move32(Assembler::Below, scratch, out.typeReg(), scratch,
+ out.typeReg());
+ }
+}
+
+void CodeGenerator::visitUnbox(LUnbox* unbox) {
+ // Note that for unbox, the type and payload indexes are switched on the
+ // inputs.
+ Operand type = ToOperand(unbox->type());
+ Operand payload = ToOperand(unbox->payload());
+ Register output = ToRegister(unbox->output());
+ MUnbox* mir = unbox->mir();
+
+ JSValueTag tag = MIRTypeToTag(mir->type());
+ if (mir->fallible()) {
+ masm.cmp32(type, Imm32(tag));
+ bailoutIf(Assembler::NotEqual, unbox->snapshot());
+ } else {
+#ifdef DEBUG
+ Label ok;
+ masm.branch32(Assembler::Equal, type, Imm32(tag), &ok);
+ masm.assumeUnreachable("Infallible unbox type mismatch");
+ masm.bind(&ok);
+#endif
+ }
+
+ // Note: If spectreValueMasking is disabled, then this instruction will
+ // default to a no-op as long as the lowering allocate the same register for
+ // the output and the payload.
+ masm.unboxNonDouble(type, payload, output, ValueTypeFromMIRType(mir->type()));
+}
+
+void CodeGenerator::visitAtomicLoad64(LAtomicLoad64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register temp = ToRegister(lir->temp());
+ Register64 temp64 = ToRegister64(lir->temp64());
+ Register out = ToRegister(lir->output());
+
+ MOZ_ASSERT(out == ecx);
+ MOZ_ASSERT(temp == ebx);
+ MOZ_ASSERT(temp64 == Register64(edx, eax));
+
+ const MLoadUnboxedScalar* mir = lir->mir();
+
+ Scalar::Type storageType = mir->storageType();
+
+ if (lir->index()->isConstant()) {
+ Address source =
+ ToAddress(elements, lir->index(), storageType, mir->offsetAdjustment());
+ masm.atomicLoad64(Synchronization::Load(), source, Register64(ecx, ebx),
+ Register64(edx, eax));
+ } else {
+ BaseIndex source(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(storageType), mir->offsetAdjustment());
+ masm.atomicLoad64(Synchronization::Load(), source, Register64(ecx, ebx),
+ Register64(edx, eax));
+ }
+
+ emitCreateBigInt(lir, storageType, temp64, out, temp);
+}
+
+void CodeGenerator::visitAtomicStore64(LAtomicStore64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register64 temp2 = Register64(value, ToRegister(lir->tempLow()));
+
+ MOZ_ASSERT(temp1 == Register64(ecx, ebx));
+ MOZ_ASSERT(temp2 == Register64(edx, eax));
+
+ Scalar::Type writeType = lir->mir()->writeType();
+
+ masm.loadBigInt64(value, temp1);
+
+ masm.push(value);
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), writeType);
+ masm.atomicStore64(Synchronization::Store(), dest, temp1, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(writeType));
+ masm.atomicStore64(Synchronization::Store(), dest, temp1, temp2);
+ }
+ masm.pop(value);
+}
+
+void CodeGenerator::visitCompareExchangeTypedArrayElement64(
+ LCompareExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register oldval = ToRegister(lir->oldval());
+ DebugOnly<Register> newval = ToRegister(lir->newval());
+ DebugOnly<Register> temp = ToRegister(lir->tempLow());
+ Register out = ToRegister(lir->output());
+
+ MOZ_ASSERT(elements == esi);
+ MOZ_ASSERT(oldval == eax);
+ MOZ_ASSERT(newval.inspect() == edx);
+ MOZ_ASSERT(temp.inspect() == ebx);
+ MOZ_ASSERT(out == ecx);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ DebugOnly<uint32_t> framePushed = masm.framePushed();
+
+ // Save eax and edx before they're clobbered below.
+ masm.push(eax);
+ masm.push(edx);
+
+ auto restoreSavedRegisters = [&]() {
+ masm.pop(edx);
+ masm.pop(eax);
+ };
+
+ Register64 expected = Register64(edx, eax);
+ Register64 replacement = Register64(ecx, ebx);
+
+ // Load |oldval| and |newval| into |expected| resp. |replacement|.
+ {
+ // Use `esi` as a temp register.
+ Register bigInt = esi;
+ masm.push(bigInt);
+
+ masm.mov(oldval, bigInt);
+ masm.loadBigInt64(bigInt, expected);
+
+ // |newval| is stored in `edx`, which is already pushed onto the stack.
+ masm.loadPtr(Address(masm.getStackPointer(), sizeof(uintptr_t)), bigInt);
+ masm.loadBigInt64(bigInt, replacement);
+
+ masm.pop(bigInt);
+ }
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.compareExchange64(Synchronization::Full(), dest, expected, replacement,
+ expected);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.compareExchange64(Synchronization::Full(), dest, expected, replacement,
+ expected);
+ }
+
+ // Move the result from `edx:eax` to `ecx:ebx`.
+ masm.move64(expected, replacement);
+
+ // OutOfLineCallVM tracks the currently pushed stack entries as reported by
+ // |masm.framePushed()|. We mustn't have any additional entries on the stack
+ // which weren't previously recorded by the safepoint, otherwise the GC
+ // complains when tracing the Ion frames, because the stack frames don't
+ // have their expected layout.
+ MOZ_ASSERT(framePushed == masm.framePushed());
+
+ OutOfLineCode* ool = createBigIntOutOfLine(lir, arrayType, replacement, out);
+
+ // Use `edx:eax`, which are both already on the stack, as temp registers.
+ Register bigInt = eax;
+ Register temp2 = edx;
+
+ Label fail;
+ masm.newGCBigInt(bigInt, temp2, initialBigIntHeap(), &fail);
+ masm.initializeBigInt64(arrayType, bigInt, replacement);
+ masm.mov(bigInt, out);
+ restoreSavedRegisters();
+ masm.jump(ool->rejoin());
+
+ // Couldn't create the BigInt. Restore `edx:eax` and call into the VM.
+ masm.bind(&fail);
+ restoreSavedRegisters();
+ masm.jump(ool->entry());
+
+ // At this point `edx:eax` must have been restored to their original values.
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitAtomicExchangeTypedArrayElement64(
+ LAtomicExchangeTypedArrayElement64* lir) {
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register out = ToRegister(lir->output());
+ Register64 temp2 = Register64(value, out);
+
+ MOZ_ASSERT(value == edx);
+ MOZ_ASSERT(temp1 == Register64(ecx, ebx));
+ MOZ_ASSERT(temp2 == Register64(edx, eax));
+ MOZ_ASSERT(out == eax);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+
+ DebugOnly<uint32_t> framePushed = masm.framePushed();
+
+ // Save edx before it's clobbered below.
+ masm.push(edx);
+
+ auto restoreSavedRegisters = [&]() { masm.pop(edx); };
+
+ masm.loadBigInt64(value, temp1);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicExchange64(Synchronization::Full(), dest, temp1, temp2);
+ }
+
+ // Move the result from `edx:eax` to `ecx:ebx`.
+ masm.move64(temp2, temp1);
+
+ // OutOfLineCallVM tracks the currently pushed stack entries as reported by
+ // |masm.framePushed()|. We mustn't have any additional entries on the stack
+ // which weren't previously recorded by the safepoint, otherwise the GC
+ // complains when tracing the Ion frames, because the stack frames don't
+ // have their expected layout.
+ MOZ_ASSERT(framePushed == masm.framePushed());
+
+ OutOfLineCode* ool = createBigIntOutOfLine(lir, arrayType, temp1, out);
+
+ // Use `edx`, which is already on the stack, as a temp register.
+ Register temp = edx;
+
+ Label fail;
+ masm.newGCBigInt(out, temp, initialBigIntHeap(), &fail);
+ masm.initializeBigInt64(arrayType, out, temp1);
+ restoreSavedRegisters();
+ masm.jump(ool->rejoin());
+
+ // Couldn't create the BigInt. Restore `edx` and call into the VM.
+ masm.bind(&fail);
+ restoreSavedRegisters();
+ masm.jump(ool->entry());
+
+ // At this point `edx` must have been restored to its original value.
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinop64(
+ LAtomicTypedArrayElementBinop64* lir) {
+ MOZ_ASSERT(!lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register out = ToRegister(lir->output());
+ Register64 temp2 = Register64(value, out);
+
+ MOZ_ASSERT(value == edx);
+ MOZ_ASSERT(temp1 == Register64(ecx, ebx));
+ MOZ_ASSERT(temp2 == Register64(edx, eax));
+ MOZ_ASSERT(out == eax);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ DebugOnly<uint32_t> framePushed = masm.framePushed();
+
+ // Save edx before it's clobbered below.
+ masm.push(edx);
+
+ auto restoreSavedRegisters = [&]() { masm.pop(edx); };
+
+ masm.loadBigInt64(value, temp1);
+
+ masm.Push(temp1);
+
+ Address addr(masm.getStackPointer(), 0);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, addr, dest, temp1,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, addr, dest, temp1,
+ temp2);
+ }
+
+ masm.freeStack(sizeof(uint64_t));
+
+ // Move the result from `edx:eax` to `ecx:ebx`.
+ masm.move64(temp2, temp1);
+
+ // OutOfLineCallVM tracks the currently pushed stack entries as reported by
+ // |masm.framePushed()|. We mustn't have any additional entries on the stack
+ // which weren't previously recorded by the safepoint, otherwise the GC
+ // complains when tracing the Ion frames, because the stack frames don't
+ // have their expected layout.
+ MOZ_ASSERT(framePushed == masm.framePushed());
+
+ OutOfLineCode* ool = createBigIntOutOfLine(lir, arrayType, temp1, out);
+
+ // Use `edx`, which is already on the stack, as a temp register.
+ Register temp = edx;
+
+ Label fail;
+ masm.newGCBigInt(out, temp, initialBigIntHeap(), &fail);
+ masm.initializeBigInt64(arrayType, out, temp1);
+ restoreSavedRegisters();
+ masm.jump(ool->rejoin());
+
+ // Couldn't create the BigInt. Restore `edx` and call into the VM.
+ masm.bind(&fail);
+ restoreSavedRegisters();
+ masm.jump(ool->entry());
+
+ // At this point `edx` must have been restored to its original value.
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitAtomicTypedArrayElementBinopForEffect64(
+ LAtomicTypedArrayElementBinopForEffect64* lir) {
+ MOZ_ASSERT(lir->mir()->isForEffect());
+
+ Register elements = ToRegister(lir->elements());
+ Register value = ToRegister(lir->value());
+ Register64 temp1 = ToRegister64(lir->temp1());
+ Register tempLow = ToRegister(lir->tempLow());
+ Register64 temp2 = Register64(value, tempLow);
+
+ MOZ_ASSERT(value == edx);
+ MOZ_ASSERT(temp1 == Register64(ecx, ebx));
+ MOZ_ASSERT(temp2 == Register64(edx, eax));
+ MOZ_ASSERT(tempLow == eax);
+
+ Scalar::Type arrayType = lir->mir()->arrayType();
+ AtomicOp atomicOp = lir->mir()->operation();
+
+ // Save edx before it's clobbered below.
+ masm.push(edx);
+
+ masm.loadBigInt64(value, temp1);
+
+ masm.Push(temp1);
+
+ Address addr(masm.getStackPointer(), 0);
+
+ if (lir->index()->isConstant()) {
+ Address dest = ToAddress(elements, lir->index(), arrayType);
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, addr, dest, temp1,
+ temp2);
+ } else {
+ BaseIndex dest(elements, ToRegister(lir->index()),
+ ScaleFromScalarType(arrayType));
+ masm.atomicFetchOp64(Synchronization::Full(), atomicOp, addr, dest, temp1,
+ temp2);
+ }
+
+ masm.freeStack(sizeof(uint64_t));
+
+ masm.pop(edx);
+}
+
+void CodeGenerator::visitWasmUint32ToDouble(LWasmUint32ToDouble* lir) {
+ Register input = ToRegister(lir->input());
+ Register temp = ToRegister(lir->temp());
+
+ if (input != temp) {
+ masm.mov(input, temp);
+ }
+
+ // Beware: convertUInt32ToDouble clobbers input.
+ masm.convertUInt32ToDouble(temp, ToFloatRegister(lir->output()));
+}
+
+void CodeGenerator::visitWasmUint32ToFloat32(LWasmUint32ToFloat32* lir) {
+ Register input = ToRegister(lir->input());
+ Register temp = ToRegister(lir->temp());
+ FloatRegister output = ToFloatRegister(lir->output());
+
+ if (input != temp) {
+ masm.mov(input, temp);
+ }
+
+ // Beware: convertUInt32ToFloat32 clobbers input.
+ masm.convertUInt32ToFloat32(temp, output);
+}
+
+void CodeGenerator::visitWasmHeapBase(LWasmHeapBase* ins) {
+ masm.loadPtr(Address(ToRegister(ins->instance()),
+ wasm::Instance::offsetOfMemoryBase()),
+ ToRegister(ins->output()));
+}
+
+template <typename T>
+void CodeGeneratorX86::emitWasmLoad(T* ins) {
+ const MWasmLoad* mir = ins->mir();
+
+ uint32_t offset = mir->access().offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* memoryBase = ins->memoryBase();
+
+ // Lowering has set things up so that we can use a BaseIndex form if the
+ // pointer is constant and the offset is zero, or if the pointer is zero.
+
+ Operand srcAddr =
+ ptr->isBogus()
+ ? Operand(ToRegister(memoryBase),
+ offset ? offset : mir->base()->toConstant()->toInt32())
+ : Operand(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset);
+
+ if (mir->type() == MIRType::Int64) {
+ MOZ_ASSERT_IF(mir->access().isAtomic(),
+ mir->access().type() != Scalar::Int64);
+ masm.wasmLoadI64(mir->access(), srcAddr, ToOutRegister64(ins));
+ } else {
+ masm.wasmLoad(mir->access(), srcAddr, ToAnyRegister(ins->output()));
+ }
+}
+
+void CodeGenerator::visitWasmLoad(LWasmLoad* ins) { emitWasmLoad(ins); }
+
+void CodeGenerator::visitWasmLoadI64(LWasmLoadI64* ins) { emitWasmLoad(ins); }
+
+template <typename T>
+void CodeGeneratorX86::emitWasmStore(T* ins) {
+ const MWasmStore* mir = ins->mir();
+
+ uint32_t offset = mir->access().offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* ptr = ins->ptr();
+ const LAllocation* memoryBase = ins->memoryBase();
+
+ // Lowering has set things up so that we can use a BaseIndex form if the
+ // pointer is constant and the offset is zero, or if the pointer is zero.
+
+ Operand dstAddr =
+ ptr->isBogus()
+ ? Operand(ToRegister(memoryBase),
+ offset ? offset : mir->base()->toConstant()->toInt32())
+ : Operand(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset);
+
+ if (mir->access().type() == Scalar::Int64) {
+ Register64 value =
+ ToRegister64(ins->getInt64Operand(LWasmStoreI64::ValueIndex));
+ masm.wasmStoreI64(mir->access(), value, dstAddr);
+ } else {
+ AnyRegister value = ToAnyRegister(ins->getOperand(LWasmStore::ValueIndex));
+ masm.wasmStore(mir->access(), value, dstAddr);
+ }
+}
+
+void CodeGenerator::visitWasmStore(LWasmStore* ins) { emitWasmStore(ins); }
+
+void CodeGenerator::visitWasmStoreI64(LWasmStoreI64* ins) {
+ emitWasmStore(ins);
+}
+
+void CodeGenerator::visitWasmCompareExchangeHeap(
+ LWasmCompareExchangeHeap* ins) {
+ MWasmCompareExchangeHeap* mir = ins->mir();
+
+ Register ptrReg = ToRegister(ins->ptr());
+ Register oldval = ToRegister(ins->oldValue());
+ Register newval = ToRegister(ins->newValue());
+ Register addrTemp = ToRegister(ins->addrTemp());
+ Register memoryBase = ToRegister(ins->memoryBase());
+ Register output = ToRegister(ins->output());
+
+ masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()),
+ addrTemp);
+
+ Address memAddr(addrTemp, 0);
+ masm.wasmCompareExchange(mir->access(), memAddr, oldval, newval, output);
+}
+
+void CodeGenerator::visitWasmAtomicExchangeHeap(LWasmAtomicExchangeHeap* ins) {
+ MWasmAtomicExchangeHeap* mir = ins->mir();
+
+ Register ptrReg = ToRegister(ins->ptr());
+ Register value = ToRegister(ins->value());
+ Register addrTemp = ToRegister(ins->addrTemp());
+ Register memoryBase = ToRegister(ins->memoryBase());
+ Register output = ToRegister(ins->output());
+
+ masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()),
+ addrTemp);
+
+ Address memAddr(addrTemp, 0);
+ masm.wasmAtomicExchange(mir->access(), memAddr, value, output);
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeap(LWasmAtomicBinopHeap* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+
+ Register ptrReg = ToRegister(ins->ptr());
+ Register temp =
+ ins->temp()->isBogusTemp() ? InvalidReg : ToRegister(ins->temp());
+ Register addrTemp = ToRegister(ins->addrTemp());
+ Register out = ToRegister(ins->output());
+ const LAllocation* value = ins->value();
+ AtomicOp op = mir->operation();
+ Register memoryBase = ToRegister(ins->memoryBase());
+
+ masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()),
+ addrTemp);
+
+ Address memAddr(addrTemp, 0);
+ if (value->isConstant()) {
+ masm.wasmAtomicFetchOp(mir->access(), op, Imm32(ToInt32(value)), memAddr,
+ temp, out);
+ } else {
+ masm.wasmAtomicFetchOp(mir->access(), op, ToRegister(value), memAddr, temp,
+ out);
+ }
+}
+
+void CodeGenerator::visitWasmAtomicBinopHeapForEffect(
+ LWasmAtomicBinopHeapForEffect* ins) {
+ MWasmAtomicBinopHeap* mir = ins->mir();
+ MOZ_ASSERT(!mir->hasUses());
+
+ Register ptrReg = ToRegister(ins->ptr());
+ Register addrTemp = ToRegister(ins->addrTemp());
+ const LAllocation* value = ins->value();
+ AtomicOp op = mir->operation();
+ Register memoryBase = ToRegister(ins->memoryBase());
+
+ masm.leal(Operand(memoryBase, ptrReg, TimesOne, mir->access().offset()),
+ addrTemp);
+
+ Address memAddr(addrTemp, 0);
+ if (value->isConstant()) {
+ masm.wasmAtomicEffectOp(mir->access(), op, Imm32(ToInt32(value)), memAddr,
+ InvalidReg);
+ } else {
+ masm.wasmAtomicEffectOp(mir->access(), op, ToRegister(value), memAddr,
+ InvalidReg);
+ }
+}
+
+void CodeGenerator::visitWasmAtomicLoadI64(LWasmAtomicLoadI64* ins) {
+ uint32_t offset = ins->mir()->access().offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* memoryBase = ins->memoryBase();
+ const LAllocation* ptr = ins->ptr();
+ BaseIndex srcAddr(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset);
+
+ MOZ_ASSERT(ToRegister(ins->t1()) == ecx);
+ MOZ_ASSERT(ToRegister(ins->t2()) == ebx);
+ MOZ_ASSERT(ToOutRegister64(ins).high == edx);
+ MOZ_ASSERT(ToOutRegister64(ins).low == eax);
+
+ masm.wasmAtomicLoad64(ins->mir()->access(), srcAddr, Register64(ecx, ebx),
+ Register64(edx, eax));
+}
+
+void CodeGenerator::visitWasmCompareExchangeI64(LWasmCompareExchangeI64* ins) {
+ uint32_t offset = ins->mir()->access().offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* memoryBase = ins->memoryBase();
+ const LAllocation* ptr = ins->ptr();
+ Operand srcAddr(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset);
+
+ MOZ_ASSERT(ToRegister64(ins->expected()).low == eax);
+ MOZ_ASSERT(ToRegister64(ins->expected()).high == edx);
+ MOZ_ASSERT(ToRegister64(ins->replacement()).low == ebx);
+ MOZ_ASSERT(ToRegister64(ins->replacement()).high == ecx);
+ MOZ_ASSERT(ToOutRegister64(ins).low == eax);
+ MOZ_ASSERT(ToOutRegister64(ins).high == edx);
+
+ masm.append(ins->mir()->access(), masm.size());
+ masm.lock_cmpxchg8b(edx, eax, ecx, ebx, srcAddr);
+}
+
+template <typename T>
+void CodeGeneratorX86::emitWasmStoreOrExchangeAtomicI64(
+ T* ins, const wasm::MemoryAccessDesc& access) {
+ MOZ_ASSERT(access.offset() < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* memoryBase = ins->memoryBase();
+ const LAllocation* ptr = ins->ptr();
+ Operand srcAddr(ToRegister(memoryBase), ToRegister(ptr), TimesOne,
+ access.offset());
+
+ DebugOnly<const LInt64Allocation> value = ins->value();
+ MOZ_ASSERT(ToRegister64(value).low == ebx);
+ MOZ_ASSERT(ToRegister64(value).high == ecx);
+
+ // eax and edx will be overwritten every time through the loop but
+ // memoryBase and ptr must remain live for a possible second iteration.
+
+ MOZ_ASSERT(ToRegister(memoryBase) != edx && ToRegister(memoryBase) != eax);
+ MOZ_ASSERT(ToRegister(ptr) != edx && ToRegister(ptr) != eax);
+
+ Label again;
+ masm.bind(&again);
+ masm.append(access, masm.size());
+ masm.lock_cmpxchg8b(edx, eax, ecx, ebx, srcAddr);
+ masm.j(Assembler::Condition::NonZero, &again);
+}
+
+void CodeGenerator::visitWasmAtomicStoreI64(LWasmAtomicStoreI64* ins) {
+ MOZ_ASSERT(ToRegister(ins->t1()) == edx);
+ MOZ_ASSERT(ToRegister(ins->t2()) == eax);
+
+ emitWasmStoreOrExchangeAtomicI64(ins, ins->mir()->access());
+}
+
+void CodeGenerator::visitWasmAtomicExchangeI64(LWasmAtomicExchangeI64* ins) {
+ MOZ_ASSERT(ToOutRegister64(ins).high == edx);
+ MOZ_ASSERT(ToOutRegister64(ins).low == eax);
+
+ emitWasmStoreOrExchangeAtomicI64(ins, ins->access());
+}
+
+void CodeGenerator::visitWasmAtomicBinopI64(LWasmAtomicBinopI64* ins) {
+ uint32_t offset = ins->access().offset();
+ MOZ_ASSERT(offset < masm.wasmMaxOffsetGuardLimit());
+
+ const LAllocation* memoryBase = ins->memoryBase();
+ const LAllocation* ptr = ins->ptr();
+
+ BaseIndex srcAddr(ToRegister(memoryBase), ToRegister(ptr), TimesOne, offset);
+
+ MOZ_ASSERT(ToRegister(memoryBase) == esi || ToRegister(memoryBase) == edi);
+ MOZ_ASSERT(ToRegister(ptr) == esi || ToRegister(ptr) == edi);
+
+ Register64 value = ToRegister64(ins->value());
+
+ MOZ_ASSERT(value.low == ebx);
+ MOZ_ASSERT(value.high == ecx);
+
+ Register64 output = ToOutRegister64(ins);
+
+ MOZ_ASSERT(output.low == eax);
+ MOZ_ASSERT(output.high == edx);
+
+ masm.Push(ecx);
+ masm.Push(ebx);
+
+ Address valueAddr(esp, 0);
+
+ // Here the `value` register acts as a temp, we'll restore it below.
+ masm.wasmAtomicFetchOp64(ins->access(), ins->operation(), valueAddr, srcAddr,
+ value, output);
+
+ masm.Pop(ebx);
+ masm.Pop(ecx);
+}
+
+namespace js {
+namespace jit {
+
+class OutOfLineTruncate : public OutOfLineCodeBase<CodeGeneratorX86> {
+ LInstruction* ins_;
+
+ public:
+ explicit OutOfLineTruncate(LInstruction* ins) : ins_(ins) {
+ MOZ_ASSERT(ins_->isTruncateDToInt32() ||
+ ins_->isWasmBuiltinTruncateDToInt32());
+ }
+
+ void accept(CodeGeneratorX86* codegen) override {
+ codegen->visitOutOfLineTruncate(this);
+ }
+
+ LAllocation* input() { return ins_->getOperand(0); }
+ LDefinition* output() { return ins_->getDef(0); }
+ LDefinition* tempFloat() { return ins_->getTemp(0); }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ if (ins_->isTruncateDToInt32()) {
+ return ins_->toTruncateDToInt32()->mir()->bytecodeOffset();
+ }
+
+ return ins_->toWasmBuiltinTruncateDToInt32()->mir()->bytecodeOffset();
+ }
+};
+
+class OutOfLineTruncateFloat32 : public OutOfLineCodeBase<CodeGeneratorX86> {
+ LInstruction* ins_;
+
+ public:
+ explicit OutOfLineTruncateFloat32(LInstruction* ins) : ins_(ins) {
+ MOZ_ASSERT(ins_->isTruncateFToInt32() ||
+ ins_->isWasmBuiltinTruncateFToInt32());
+ }
+
+ void accept(CodeGeneratorX86* codegen) override {
+ codegen->visitOutOfLineTruncateFloat32(this);
+ }
+
+ LAllocation* input() { return ins_->getOperand(0); }
+ LDefinition* output() { return ins_->getDef(0); }
+ LDefinition* tempFloat() { return ins_->getTemp(0); }
+
+ wasm::BytecodeOffset bytecodeOffset() const {
+ if (ins_->isTruncateFToInt32()) {
+ return ins_->toTruncateDToInt32()->mir()->bytecodeOffset();
+ }
+
+ return ins_->toWasmBuiltinTruncateFToInt32()->mir()->bytecodeOffset();
+ }
+};
+
+} // namespace jit
+} // namespace js
+
+void CodeGenerator::visitTruncateDToInt32(LTruncateDToInt32* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ OutOfLineTruncate* ool = new (alloc()) OutOfLineTruncate(ins);
+ addOutOfLineCode(ool, ins->mir());
+
+ masm.branchTruncateDoubleMaybeModUint32(input, output, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateDToInt32(
+ LWasmBuiltinTruncateDToInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->getOperand(0));
+ Register output = ToRegister(lir->getDef(0));
+
+ OutOfLineTruncate* ool = new (alloc()) OutOfLineTruncate(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ masm.branchTruncateDoubleMaybeModUint32(input, output, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitTruncateFToInt32(LTruncateFToInt32* ins) {
+ FloatRegister input = ToFloatRegister(ins->input());
+ Register output = ToRegister(ins->output());
+
+ OutOfLineTruncateFloat32* ool = new (alloc()) OutOfLineTruncateFloat32(ins);
+ addOutOfLineCode(ool, ins->mir());
+
+ masm.branchTruncateFloat32MaybeModUint32(input, output, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGenerator::visitWasmBuiltinTruncateFToInt32(
+ LWasmBuiltinTruncateFToInt32* lir) {
+ FloatRegister input = ToFloatRegister(lir->getOperand(0));
+ Register output = ToRegister(lir->getDef(0));
+
+ OutOfLineTruncateFloat32* ool = new (alloc()) OutOfLineTruncateFloat32(lir);
+ addOutOfLineCode(ool, lir->mir());
+
+ masm.branchTruncateFloat32MaybeModUint32(input, output, ool->entry());
+ masm.bind(ool->rejoin());
+}
+
+void CodeGeneratorX86::visitOutOfLineTruncate(OutOfLineTruncate* ool) {
+ FloatRegister input = ToFloatRegister(ool->input());
+ Register output = ToRegister(ool->output());
+
+ Label fail;
+
+ if (Assembler::HasSSE3()) {
+ Label failPopDouble;
+ // Push double.
+ masm.subl(Imm32(sizeof(double)), esp);
+ masm.storeDouble(input, Operand(esp, 0));
+
+ // Check exponent to avoid fp exceptions.
+ masm.branchDoubleNotInInt64Range(Address(esp, 0), output, &failPopDouble);
+
+ // Load double, perform 64-bit truncation.
+ masm.truncateDoubleToInt64(Address(esp, 0), Address(esp, 0), output);
+
+ // Load low word, pop double and jump back.
+ masm.load32(Address(esp, 0), output);
+ masm.addl(Imm32(sizeof(double)), esp);
+ masm.jump(ool->rejoin());
+
+ masm.bind(&failPopDouble);
+ masm.addl(Imm32(sizeof(double)), esp);
+ masm.jump(&fail);
+ } else {
+ FloatRegister temp = ToFloatRegister(ool->tempFloat());
+
+ // Try to convert doubles representing integers within 2^32 of a signed
+ // integer, by adding/subtracting 2^32 and then trying to convert to int32.
+ // This has to be an exact conversion, as otherwise the truncation works
+ // incorrectly on the modified value.
+ {
+ ScratchDoubleScope fpscratch(masm);
+ masm.zeroDouble(fpscratch);
+ masm.vucomisd(fpscratch, input);
+ masm.j(Assembler::Parity, &fail);
+ }
+
+ {
+ Label positive;
+ masm.j(Assembler::Above, &positive);
+
+ masm.loadConstantDouble(4294967296.0, temp);
+ Label skip;
+ masm.jmp(&skip);
+
+ masm.bind(&positive);
+ masm.loadConstantDouble(-4294967296.0, temp);
+ masm.bind(&skip);
+ }
+
+ masm.addDouble(input, temp);
+ masm.vcvttsd2si(temp, output);
+ ScratchDoubleScope fpscratch(masm);
+ masm.vcvtsi2sd(output, fpscratch, fpscratch);
+
+ masm.vucomisd(fpscratch, temp);
+ masm.j(Assembler::Parity, &fail);
+ masm.j(Assembler::Equal, ool->rejoin());
+ }
+
+ masm.bind(&fail);
+ {
+ if (gen->compilingWasm()) {
+ masm.Push(InstanceReg);
+ }
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ saveVolatile(output);
+
+ if (gen->compilingWasm()) {
+ masm.setupWasmABICall();
+ masm.passABIArg(input, MoveOp::DOUBLE);
+
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ masm.callWithABI(ool->bytecodeOffset(), wasm::SymbolicAddress::ToInt32,
+ mozilla::Some(instanceOffset));
+ } else {
+ using Fn = int32_t (*)(double);
+ masm.setupUnalignedABICall(output);
+ masm.passABIArg(input, MoveOp::DOUBLE);
+ masm.callWithABI<Fn, JS::ToInt32>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+ }
+ masm.storeCallInt32Result(output);
+
+ restoreVolatile(output);
+
+ if (gen->compilingWasm()) {
+ masm.Pop(InstanceReg);
+ }
+ }
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGeneratorX86::visitOutOfLineTruncateFloat32(
+ OutOfLineTruncateFloat32* ool) {
+ FloatRegister input = ToFloatRegister(ool->input());
+ Register output = ToRegister(ool->output());
+
+ Label fail;
+
+ if (Assembler::HasSSE3()) {
+ Label failPopFloat;
+
+ // Push float32, but subtracts 64 bits so that the value popped by fisttp
+ // fits
+ masm.subl(Imm32(sizeof(uint64_t)), esp);
+ masm.storeFloat32(input, Operand(esp, 0));
+
+ // Check exponent to avoid fp exceptions.
+ masm.branchFloat32NotInInt64Range(Address(esp, 0), output, &failPopFloat);
+
+ // Load float, perform 32-bit truncation.
+ masm.truncateFloat32ToInt64(Address(esp, 0), Address(esp, 0), output);
+
+ // Load low word, pop 64bits and jump back.
+ masm.load32(Address(esp, 0), output);
+ masm.addl(Imm32(sizeof(uint64_t)), esp);
+ masm.jump(ool->rejoin());
+
+ masm.bind(&failPopFloat);
+ masm.addl(Imm32(sizeof(uint64_t)), esp);
+ masm.jump(&fail);
+ } else {
+ FloatRegister temp = ToFloatRegister(ool->tempFloat());
+
+ // Try to convert float32 representing integers within 2^32 of a signed
+ // integer, by adding/subtracting 2^32 and then trying to convert to int32.
+ // This has to be an exact conversion, as otherwise the truncation works
+ // incorrectly on the modified value.
+ {
+ ScratchFloat32Scope fpscratch(masm);
+ masm.zeroFloat32(fpscratch);
+ masm.vucomiss(fpscratch, input);
+ masm.j(Assembler::Parity, &fail);
+ }
+
+ {
+ Label positive;
+ masm.j(Assembler::Above, &positive);
+
+ masm.loadConstantFloat32(4294967296.f, temp);
+ Label skip;
+ masm.jmp(&skip);
+
+ masm.bind(&positive);
+ masm.loadConstantFloat32(-4294967296.f, temp);
+ masm.bind(&skip);
+ }
+
+ masm.addFloat32(input, temp);
+ masm.vcvttss2si(temp, output);
+ ScratchFloat32Scope fpscratch(masm);
+ masm.vcvtsi2ss(output, fpscratch, fpscratch);
+
+ masm.vucomiss(fpscratch, temp);
+ masm.j(Assembler::Parity, &fail);
+ masm.j(Assembler::Equal, ool->rejoin());
+ }
+
+ masm.bind(&fail);
+ {
+ if (gen->compilingWasm()) {
+ masm.Push(InstanceReg);
+ }
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ saveVolatile(output);
+
+ masm.Push(input);
+
+ if (gen->compilingWasm()) {
+ masm.setupWasmABICall();
+ } else {
+ masm.setupUnalignedABICall(output);
+ }
+
+ masm.vcvtss2sd(input, input, input);
+ masm.passABIArg(input.asDouble(), MoveOp::DOUBLE);
+
+ if (gen->compilingWasm()) {
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ masm.callWithABI(ool->bytecodeOffset(), wasm::SymbolicAddress::ToInt32,
+ mozilla::Some(instanceOffset));
+ } else {
+ using Fn = int32_t (*)(double);
+ masm.callWithABI<Fn, JS::ToInt32>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+ }
+
+ masm.storeCallInt32Result(output);
+ masm.Pop(input);
+
+ restoreVolatile(output);
+
+ if (gen->compilingWasm()) {
+ masm.Pop(InstanceReg);
+ }
+ }
+
+ masm.jump(ool->rejoin());
+}
+
+void CodeGenerator::visitCompareI64(LCompareI64* lir) {
+ MCompare* mir = lir->mir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register64 lhsRegs = ToRegister64(lhs);
+ Register output = ToRegister(lir->output());
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition condition = JSOpToCondition(lir->jsop(), isSigned);
+ Label done;
+
+ masm.move32(Imm32(1), output);
+
+ if (IsConstant(rhs)) {
+ Imm64 imm = Imm64(ToInt64(rhs));
+ masm.branch64(condition, lhsRegs, imm, &done);
+ } else {
+ Register64 rhsRegs = ToRegister64(rhs);
+ masm.branch64(condition, lhsRegs, rhsRegs, &done);
+ }
+
+ masm.xorl(output, output);
+ masm.bind(&done);
+}
+
+void CodeGenerator::visitCompareI64AndBranch(LCompareI64AndBranch* lir) {
+ MCompare* mir = lir->cmpMir();
+ MOZ_ASSERT(mir->compareType() == MCompare::Compare_Int64 ||
+ mir->compareType() == MCompare::Compare_UInt64);
+
+ const LInt64Allocation lhs = lir->getInt64Operand(LCompareI64::Lhs);
+ const LInt64Allocation rhs = lir->getInt64Operand(LCompareI64::Rhs);
+ Register64 lhsRegs = ToRegister64(lhs);
+
+ bool isSigned = mir->compareType() == MCompare::Compare_Int64;
+ Assembler::Condition condition = JSOpToCondition(lir->jsop(), isSigned);
+
+ Label* trueLabel = getJumpLabelForBranch(lir->ifTrue());
+ Label* falseLabel = getJumpLabelForBranch(lir->ifFalse());
+
+ if (isNextBlock(lir->ifFalse()->lir())) {
+ falseLabel = nullptr;
+ } else if (isNextBlock(lir->ifTrue()->lir())) {
+ condition = Assembler::InvertCondition(condition);
+ trueLabel = falseLabel;
+ falseLabel = nullptr;
+ }
+
+ if (IsConstant(rhs)) {
+ Imm64 imm = Imm64(ToInt64(rhs));
+ masm.branch64(condition, lhsRegs, imm, trueLabel, falseLabel);
+ } else {
+ Register64 rhsRegs = ToRegister64(rhs);
+ masm.branch64(condition, lhsRegs, rhsRegs, trueLabel, falseLabel);
+ }
+}
+
+void CodeGenerator::visitDivOrModI64(LDivOrModI64* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MOZ_ASSERT(ToRegister(lir->getOperand(LDivOrModI64::Instance)) ==
+ InstanceReg);
+
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ Register64 lhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Lhs));
+ Register64 rhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Rhs));
+ Register64 output = ToOutRegister64(lir);
+
+ MOZ_ASSERT(output == ReturnReg64);
+
+ Label done;
+
+ // Handle divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ // We can use InstanceReg as temp register because we preserved it
+ // before.
+ masm.branchTest64(Assembler::NonZero, rhs, rhs, InstanceReg, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ MDefinition* mir = lir->mir();
+
+ // Handle an integer overflow exception from INT64_MIN / -1.
+ if (lir->canBeNegativeOverflow()) {
+ Label notOverflow;
+ masm.branch64(Assembler::NotEqual, lhs, Imm64(INT64_MIN), &notOverflow);
+ masm.branch64(Assembler::NotEqual, rhs, Imm64(-1), &notOverflow);
+ if (mir->isWasmBuiltinModI64()) {
+ masm.xor64(output, output);
+ } else {
+ masm.wasmTrap(wasm::Trap::IntegerOverflow, lir->bytecodeOffset());
+ }
+ masm.jump(&done);
+ masm.bind(&notOverflow);
+ }
+
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs.high);
+ masm.passABIArg(lhs.low);
+ masm.passABIArg(rhs.high);
+ masm.passABIArg(rhs.low);
+
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ if (mir->isWasmBuiltinModI64()) {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::ModI64,
+ mozilla::Some(instanceOffset));
+ } else {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::DivI64,
+ mozilla::Some(instanceOffset));
+ }
+
+ // output in edx:eax, move to output register.
+ masm.movl(edx, output.high);
+ MOZ_ASSERT(eax == output.low);
+
+ masm.bind(&done);
+ masm.Pop(InstanceReg);
+}
+
+void CodeGenerator::visitUDivOrModI64(LUDivOrModI64* lir) {
+ MOZ_ASSERT(gen->compilingWasm());
+ MOZ_ASSERT(ToRegister(lir->getOperand(LDivOrModI64::Instance)) ==
+ InstanceReg);
+
+ masm.Push(InstanceReg);
+ int32_t framePushedAfterInstance = masm.framePushed();
+
+ Register64 lhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Lhs));
+ Register64 rhs = ToRegister64(lir->getInt64Operand(LDivOrModI64::Rhs));
+ Register64 output = ToOutRegister64(lir);
+
+ MOZ_ASSERT(output == ReturnReg64);
+
+ // Prevent divide by zero.
+ if (lir->canBeDivideByZero()) {
+ Label nonZero;
+ // We can use InstanceReg as temp register because we preserved it
+ // before.
+ masm.branchTest64(Assembler::NonZero, rhs, rhs, InstanceReg, &nonZero);
+ masm.wasmTrap(wasm::Trap::IntegerDivideByZero, lir->bytecodeOffset());
+ masm.bind(&nonZero);
+ }
+
+ masm.setupWasmABICall();
+ masm.passABIArg(lhs.high);
+ masm.passABIArg(lhs.low);
+ masm.passABIArg(rhs.high);
+ masm.passABIArg(rhs.low);
+
+ MDefinition* mir = lir->mir();
+ int32_t instanceOffset = masm.framePushed() - framePushedAfterInstance;
+ if (mir->isWasmBuiltinModI64()) {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::UModI64,
+ mozilla::Some(instanceOffset));
+ } else {
+ masm.callWithABI(lir->bytecodeOffset(), wasm::SymbolicAddress::UDivI64,
+ mozilla::Some(instanceOffset));
+ }
+
+ // output in edx:eax, move to output register.
+ masm.movl(edx, output.high);
+ MOZ_ASSERT(eax == output.low);
+
+ masm.Pop(InstanceReg);
+}
+
+void CodeGeneratorX86::emitBigIntDiv(LBigIntDiv* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ MOZ_ASSERT(dividend == eax);
+ MOZ_ASSERT(output == edx);
+
+ // Sign extend the lhs into rdx to make rdx:rax.
+ masm.cdq();
+
+ masm.idiv(divisor);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGeneratorX86::emitBigIntMod(LBigIntMod* ins, Register dividend,
+ Register divisor, Register output,
+ Label* fail) {
+ // Callers handle division by zero and integer overflow.
+
+ MOZ_ASSERT(dividend == eax);
+ MOZ_ASSERT(output == edx);
+
+ // Sign extend the lhs into rdx to make edx:eax.
+ masm.cdq();
+
+ masm.idiv(divisor);
+
+ // Move the remainder from edx.
+ masm.movl(output, dividend);
+
+ // Create and return the result.
+ masm.newGCBigInt(output, divisor, initialBigIntHeap(), fail);
+ masm.initializeBigInt(output, dividend);
+}
+
+void CodeGenerator::visitWasmSelectI64(LWasmSelectI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+
+ Register cond = ToRegister(lir->condExpr());
+ Register64 falseExpr = ToRegister64(lir->falseExpr());
+ Register64 out = ToOutRegister64(lir);
+
+ MOZ_ASSERT(ToRegister64(lir->trueExpr()) == out,
+ "true expr is reused for input");
+
+ Label done;
+ masm.branchTest32(Assembler::NonZero, cond, cond, &done);
+ masm.movl(falseExpr.low, out.low);
+ masm.movl(falseExpr.high, out.high);
+ masm.bind(&done);
+}
+
+// We expect to handle only the case where compare is {U,}Int32 and select is
+// {U,}Int32. Some values may be stack allocated, and the "true" input is
+// reused for the output.
+void CodeGenerator::visitWasmCompareAndSelect(LWasmCompareAndSelect* ins) {
+ bool cmpIs32bit = ins->compareType() == MCompare::Compare_Int32 ||
+ ins->compareType() == MCompare::Compare_UInt32;
+ bool selIs32bit = ins->mir()->type() == MIRType::Int32;
+
+ MOZ_RELEASE_ASSERT(
+ cmpIs32bit && selIs32bit,
+ "CodeGenerator::visitWasmCompareAndSelect: unexpected types");
+
+ Register trueExprAndDest = ToRegister(ins->output());
+ MOZ_ASSERT(ToRegister(ins->ifTrueExpr()) == trueExprAndDest,
+ "true expr input is reused for output");
+
+ Assembler::Condition cond = Assembler::InvertCondition(
+ JSOpToCondition(ins->compareType(), ins->jsop()));
+ const LAllocation* rhs = ins->rightExpr();
+ const LAllocation* falseExpr = ins->ifFalseExpr();
+ Register lhs = ToRegister(ins->leftExpr());
+
+ if (rhs->isRegister()) {
+ if (falseExpr->isRegister()) {
+ masm.cmp32Move32(cond, lhs, ToRegister(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+ } else {
+ masm.cmp32Load32(cond, lhs, ToRegister(rhs), ToAddress(falseExpr),
+ trueExprAndDest);
+ }
+ } else {
+ if (falseExpr->isRegister()) {
+ masm.cmp32Move32(cond, lhs, ToAddress(rhs), ToRegister(falseExpr),
+ trueExprAndDest);
+ } else {
+ masm.cmp32Load32(cond, lhs, ToAddress(rhs), ToAddress(falseExpr),
+ trueExprAndDest);
+ }
+ }
+}
+
+void CodeGenerator::visitWasmReinterpretFromI64(LWasmReinterpretFromI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Double);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Int64);
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+
+ masm.Push(input.high);
+ masm.Push(input.low);
+ masm.vmovq(Operand(esp, 0), ToFloatRegister(lir->output()));
+ masm.freeStack(sizeof(uint64_t));
+}
+
+void CodeGenerator::visitWasmReinterpretToI64(LWasmReinterpretToI64* lir) {
+ MOZ_ASSERT(lir->mir()->type() == MIRType::Int64);
+ MOZ_ASSERT(lir->mir()->input()->type() == MIRType::Double);
+ Register64 output = ToOutRegister64(lir);
+
+ masm.reserveStack(sizeof(uint64_t));
+ masm.vmovq(ToFloatRegister(lir->input()), Operand(esp, 0));
+ masm.Pop(output.low);
+ masm.Pop(output.high);
+}
+
+void CodeGenerator::visitExtendInt32ToInt64(LExtendInt32ToInt64* lir) {
+ Register64 output = ToOutRegister64(lir);
+ Register input = ToRegister(lir->input());
+
+ if (lir->mir()->isUnsigned()) {
+ if (output.low != input) {
+ masm.movl(input, output.low);
+ }
+ masm.xorl(output.high, output.high);
+ } else {
+ MOZ_ASSERT(output.low == input);
+ MOZ_ASSERT(output.low == eax);
+ MOZ_ASSERT(output.high == edx);
+ masm.cdq();
+ }
+}
+
+void CodeGenerator::visitSignExtendInt64(LSignExtendInt64* lir) {
+#ifdef DEBUG
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+ MOZ_ASSERT(input.low == eax);
+ MOZ_ASSERT(output.low == eax);
+ MOZ_ASSERT(input.high == edx);
+ MOZ_ASSERT(output.high == edx);
+#endif
+ switch (lir->mode()) {
+ case MSignExtendInt64::Byte:
+ masm.move8SignExtend(eax, eax);
+ break;
+ case MSignExtendInt64::Half:
+ masm.move16SignExtend(eax, eax);
+ break;
+ case MSignExtendInt64::Word:
+ break;
+ }
+ masm.cdq();
+}
+
+void CodeGenerator::visitWrapInt64ToInt32(LWrapInt64ToInt32* lir) {
+ const LInt64Allocation& input = lir->getInt64Operand(0);
+ Register output = ToRegister(lir->output());
+
+ if (lir->mir()->bottomHalf()) {
+ masm.movl(ToRegister(input.low()), output);
+ } else {
+ masm.movl(ToRegister(input.high()), output);
+ }
+}
+
+void CodeGenerator::visitWasmExtendU32Index(LWasmExtendU32Index*) {
+ MOZ_CRASH("64-bit only");
+}
+
+void CodeGenerator::visitWasmWrapU32Index(LWasmWrapU32Index* lir) {
+ // Generates no code on this platform because we just return the low part of
+ // the input register pair.
+ MOZ_ASSERT(ToRegister(lir->input()) == ToRegister(lir->output()));
+}
+
+void CodeGenerator::visitClzI64(LClzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+
+ masm.clz64(input, output.low);
+ masm.xorl(output.high, output.high);
+}
+
+void CodeGenerator::visitCtzI64(LCtzI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register64 output = ToOutRegister64(lir);
+
+ masm.ctz64(input, output.low);
+ masm.xorl(output.high, output.high);
+}
+
+void CodeGenerator::visitNotI64(LNotI64* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ Register output = ToRegister(lir->output());
+
+ if (input.high == output) {
+ masm.orl(input.low, output);
+ } else if (input.low == output) {
+ masm.orl(input.high, output);
+ } else {
+ masm.movl(input.high, output);
+ masm.orl(input.low, output);
+ }
+
+ masm.cmpl(Imm32(0), output);
+ masm.emitSet(Assembler::Equal, output);
+}
+
+void CodeGenerator::visitWasmTruncateToInt64(LWasmTruncateToInt64* lir) {
+ FloatRegister input = ToFloatRegister(lir->input());
+ Register64 output = ToOutRegister64(lir);
+
+ MWasmTruncateToInt64* mir = lir->mir();
+ FloatRegister floatTemp = ToFloatRegister(lir->temp());
+
+ Label fail, convert;
+
+ MOZ_ASSERT(mir->input()->type() == MIRType::Double ||
+ mir->input()->type() == MIRType::Float32);
+
+ auto* ool = new (alloc()) OutOfLineWasmTruncateCheck(mir, input, output);
+ addOutOfLineCode(ool, mir);
+
+ bool isSaturating = mir->isSaturating();
+ if (mir->input()->type() == MIRType::Float32) {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateFloat32ToUInt64(input, output, isSaturating,
+ ool->entry(), ool->rejoin(), floatTemp);
+ } else {
+ masm.wasmTruncateFloat32ToInt64(input, output, isSaturating, ool->entry(),
+ ool->rejoin(), floatTemp);
+ }
+ } else {
+ if (mir->isUnsigned()) {
+ masm.wasmTruncateDoubleToUInt64(input, output, isSaturating, ool->entry(),
+ ool->rejoin(), floatTemp);
+ } else {
+ masm.wasmTruncateDoubleToInt64(input, output, isSaturating, ool->entry(),
+ ool->rejoin(), floatTemp);
+ }
+ }
+}
+
+void CodeGenerator::visitInt64ToFloatingPoint(LInt64ToFloatingPoint* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+ FloatRegister output = ToFloatRegister(lir->output());
+ Register temp =
+ lir->temp()->isBogusTemp() ? InvalidReg : ToRegister(lir->temp());
+
+ MIRType outputType = lir->mir()->type();
+ MOZ_ASSERT(outputType == MIRType::Double || outputType == MIRType::Float32);
+
+ if (outputType == MIRType::Double) {
+ if (lir->mir()->isUnsigned()) {
+ masm.convertUInt64ToDouble(input, output, temp);
+ } else {
+ masm.convertInt64ToDouble(input, output);
+ }
+ } else {
+ if (lir->mir()->isUnsigned()) {
+ masm.convertUInt64ToFloat32(input, output, temp);
+ } else {
+ masm.convertInt64ToFloat32(input, output);
+ }
+ }
+}
+
+void CodeGenerator::visitBitNotI64(LBitNotI64* ins) {
+ const LInt64Allocation input = ins->getInt64Operand(0);
+ Register64 inputR = ToRegister64(input);
+ MOZ_ASSERT(inputR == ToOutRegister64(ins));
+ masm.notl(inputR.high);
+ masm.notl(inputR.low);
+}
+
+void CodeGenerator::visitTestI64AndBranch(LTestI64AndBranch* lir) {
+ Register64 input = ToRegister64(lir->getInt64Operand(0));
+
+ masm.testl(input.high, input.high);
+ jumpToBlock(lir->ifTrue(), Assembler::NonZero);
+ masm.testl(input.low, input.low);
+ emitBranch(Assembler::NonZero, lir->ifTrue(), lir->ifFalse());
+}
+
+void CodeGenerator::visitBitAndAndBranch(LBitAndAndBranch* baab) {
+ // LBitAndAndBranch only represents single-word ANDs, hence it can't be
+ // 64-bit here.
+ MOZ_ASSERT(!baab->is64());
+ Register regL = ToRegister(baab->left());
+ if (baab->right()->isConstant()) {
+ masm.test32(regL, Imm32(ToInt32(baab->right())));
+ } else {
+ masm.test32(regL, ToRegister(baab->right()));
+ }
+ emitBranch(baab->cond(), baab->ifTrue(), baab->ifFalse());
+}
diff --git a/js/src/jit/x86/CodeGenerator-x86.h b/js/src/jit/x86/CodeGenerator-x86.h
new file mode 100644
index 0000000000..4f92bf615f
--- /dev/null
+++ b/js/src/jit/x86/CodeGenerator-x86.h
@@ -0,0 +1,49 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_CodeGenerator_x86_h
+#define jit_x86_CodeGenerator_x86_h
+
+#include "jit/x86-shared/CodeGenerator-x86-shared.h"
+#include "jit/x86/Assembler-x86.h"
+
+namespace js {
+namespace jit {
+
+class OutOfLineTruncate;
+class OutOfLineTruncateFloat32;
+
+class CodeGeneratorX86 : public CodeGeneratorX86Shared {
+ protected:
+ CodeGeneratorX86(MIRGenerator* gen, LIRGraph* graph, MacroAssembler* masm);
+
+ ValueOperand ToValue(LInstruction* ins, size_t pos);
+ ValueOperand ToTempValue(LInstruction* ins, size_t pos);
+
+ void emitBigIntDiv(LBigIntDiv* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+ void emitBigIntMod(LBigIntMod* ins, Register dividend, Register divisor,
+ Register output, Label* fail);
+
+ template <typename T>
+ void emitWasmLoad(T* ins);
+ template <typename T>
+ void emitWasmStore(T* ins);
+ template <typename T>
+ void emitWasmStoreOrExchangeAtomicI64(T* ins,
+ const wasm::MemoryAccessDesc& access);
+
+ public:
+ void visitOutOfLineTruncate(OutOfLineTruncate* ool);
+ void visitOutOfLineTruncateFloat32(OutOfLineTruncateFloat32* ool);
+};
+
+typedef CodeGeneratorX86 CodeGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_CodeGenerator_x86_h */
diff --git a/js/src/jit/x86/LIR-x86.h b/js/src/jit/x86/LIR-x86.h
new file mode 100644
index 0000000000..c7c9587e20
--- /dev/null
+++ b/js/src/jit/x86/LIR-x86.h
@@ -0,0 +1,308 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_LIR_x86_h
+#define jit_x86_LIR_x86_h
+
+namespace js {
+namespace jit {
+
+class LBoxFloatingPoint : public LInstructionHelper<2, 1, 2> {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(BoxFloatingPoint);
+
+ LBoxFloatingPoint(const LAllocation& in, const LDefinition& temp,
+ const LDefinition& spectreTemp, MIRType type)
+ : LInstructionHelper(classOpcode), type_(type) {
+ MOZ_ASSERT(IsFloatingPointType(type));
+ setOperand(0, in);
+ setTemp(0, temp);
+ setTemp(1, spectreTemp);
+ }
+
+ const LDefinition* spectreTemp() { return getTemp(1); }
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+class LUnbox : public LInstructionHelper<1, 2, 0> {
+ public:
+ LIR_HEADER(Unbox);
+
+ LUnbox() : LInstructionHelper(classOpcode) {}
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+ const LAllocation* payload() { return getOperand(0); }
+ const LAllocation* type() { return getOperand(1); }
+ const char* extraName() const { return StringFromMIRType(mir()->type()); }
+};
+
+class LUnboxFloatingPoint : public LInstructionHelper<1, 2, 0> {
+ MIRType type_;
+
+ public:
+ LIR_HEADER(UnboxFloatingPoint);
+
+ static const size_t Input = 0;
+
+ LUnboxFloatingPoint(const LBoxAllocation& input, MIRType type)
+ : LInstructionHelper(classOpcode), type_(type) {
+ setBoxOperand(Input, input);
+ }
+
+ MUnbox* mir() const { return mir_->toUnbox(); }
+
+ MIRType type() const { return type_; }
+ const char* extraName() const { return StringFromMIRType(type_); }
+};
+
+// Convert a 32-bit unsigned integer to a double.
+class LWasmUint32ToDouble : public LInstructionHelper<1, 1, 1> {
+ public:
+ LIR_HEADER(WasmUint32ToDouble)
+
+ LWasmUint32ToDouble(const LAllocation& input, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ }
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+// Convert a 32-bit unsigned integer to a float32.
+class LWasmUint32ToFloat32 : public LInstructionHelper<1, 1, 1> {
+ public:
+ LIR_HEADER(WasmUint32ToFloat32)
+
+ LWasmUint32ToFloat32(const LAllocation& input, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, input);
+ setTemp(0, temp);
+ }
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+class LDivOrModI64
+ : public LCallInstructionHelper<INT64_PIECES, INT64_PIECES * 2 + 1, 0> {
+ public:
+ LIR_HEADER(DivOrModI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+ static const size_t Instance = 2 * INT64_PIECES;
+
+ LDivOrModI64(const LInt64Allocation& lhs, const LInt64Allocation& rhs,
+ const LAllocation& instance)
+ : LCallInstructionHelper(classOpcode) {
+ setInt64Operand(Lhs, lhs);
+ setInt64Operand(Rhs, rhs);
+ setOperand(Instance, instance);
+ }
+
+ MDefinition* mir() const {
+ MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64());
+ return mir_;
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->canBeDivideByZero();
+ }
+ return mir_->toWasmBuiltinDivI64()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->canBeNegativeDividend();
+ }
+ return mir_->toWasmBuiltinDivI64()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64());
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->bytecodeOffset();
+ }
+ return mir_->toWasmBuiltinDivI64()->bytecodeOffset();
+ }
+};
+
+class LUDivOrModI64
+ : public LCallInstructionHelper<INT64_PIECES, INT64_PIECES * 2 + 1, 0> {
+ public:
+ LIR_HEADER(UDivOrModI64)
+
+ static const size_t Lhs = 0;
+ static const size_t Rhs = INT64_PIECES;
+ static const size_t Instance = 2 * INT64_PIECES;
+
+ LUDivOrModI64(const LInt64Allocation& lhs, const LInt64Allocation& rhs,
+ const LAllocation& instance)
+ : LCallInstructionHelper(classOpcode) {
+ setInt64Operand(Lhs, lhs);
+ setInt64Operand(Rhs, rhs);
+ setOperand(Instance, instance);
+ }
+
+ MDefinition* mir() const {
+ MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64());
+ return mir_;
+ }
+ bool canBeDivideByZero() const {
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->canBeDivideByZero();
+ }
+ return mir_->toWasmBuiltinDivI64()->canBeDivideByZero();
+ }
+ bool canBeNegativeOverflow() const {
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->canBeNegativeDividend();
+ }
+ return mir_->toWasmBuiltinDivI64()->canBeNegativeOverflow();
+ }
+ wasm::BytecodeOffset bytecodeOffset() const {
+ MOZ_ASSERT(mir_->isWasmBuiltinDivI64() || mir_->isWasmBuiltinModI64());
+ if (mir_->isWasmBuiltinModI64()) {
+ return mir_->toWasmBuiltinModI64()->bytecodeOffset();
+ }
+ return mir_->toWasmBuiltinDivI64()->bytecodeOffset();
+ }
+};
+
+class LWasmTruncateToInt64 : public LInstructionHelper<INT64_PIECES, 1, 1> {
+ public:
+ LIR_HEADER(WasmTruncateToInt64);
+
+ LWasmTruncateToInt64(const LAllocation& in, const LDefinition& temp)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, in);
+ setTemp(0, temp);
+ }
+
+ MWasmTruncateToInt64* mir() const { return mir_->toWasmTruncateToInt64(); }
+
+ const LDefinition* temp() { return getTemp(0); }
+};
+
+class LWasmAtomicLoadI64 : public LInstructionHelper<INT64_PIECES, 2, 2> {
+ public:
+ LIR_HEADER(WasmAtomicLoadI64);
+
+ LWasmAtomicLoadI64(const LAllocation& memoryBase, const LAllocation& ptr,
+ const LDefinition& t1, const LDefinition& t2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, memoryBase);
+ setOperand(1, ptr);
+ setTemp(0, t1);
+ setTemp(1, t2);
+ }
+
+ MWasmLoad* mir() const { return mir_->toWasmLoad(); }
+ const LAllocation* memoryBase() { return getOperand(0); }
+ const LAllocation* ptr() { return getOperand(1); }
+ const LDefinition* t1() { return getTemp(0); }
+ const LDefinition* t2() { return getTemp(1); }
+};
+
+class LWasmAtomicStoreI64 : public LInstructionHelper<0, 2 + INT64_PIECES, 2> {
+ public:
+ LIR_HEADER(WasmAtomicStoreI64);
+
+ LWasmAtomicStoreI64(const LAllocation& memoryBase, const LAllocation& ptr,
+ const LInt64Allocation& value, const LDefinition& t1,
+ const LDefinition& t2)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, memoryBase);
+ setOperand(1, ptr);
+ setInt64Operand(2, value);
+ setTemp(0, t1);
+ setTemp(1, t2);
+ }
+
+ MWasmStore* mir() const { return mir_->toWasmStore(); }
+ const LAllocation* memoryBase() { return getOperand(0); }
+ const LAllocation* ptr() { return getOperand(1); }
+ const LInt64Allocation value() { return getInt64Operand(2); }
+ const LDefinition* t1() { return getTemp(0); }
+ const LDefinition* t2() { return getTemp(1); }
+};
+
+class LWasmCompareExchangeI64
+ : public LInstructionHelper<INT64_PIECES, 2 + 2 * INT64_PIECES, 0> {
+ public:
+ LIR_HEADER(WasmCompareExchangeI64);
+
+ LWasmCompareExchangeI64(const LAllocation& memoryBase, const LAllocation& ptr,
+ const LInt64Allocation& expected,
+ const LInt64Allocation& replacement)
+ : LInstructionHelper(classOpcode) {
+ setOperand(0, memoryBase);
+ setOperand(1, ptr);
+ setInt64Operand(2, expected);
+ setInt64Operand(2 + INT64_PIECES, replacement);
+ }
+
+ MWasmCompareExchangeHeap* mir() const {
+ return mir_->toWasmCompareExchangeHeap();
+ }
+ const LAllocation* memoryBase() { return getOperand(0); }
+ const LAllocation* ptr() { return getOperand(1); }
+ const LInt64Allocation expected() { return getInt64Operand(2); }
+ const LInt64Allocation replacement() {
+ return getInt64Operand(2 + INT64_PIECES);
+ }
+};
+
+class LWasmAtomicExchangeI64
+ : public LInstructionHelper<INT64_PIECES, 2 + INT64_PIECES, 0> {
+ const wasm::MemoryAccessDesc& access_;
+
+ public:
+ LIR_HEADER(WasmAtomicExchangeI64);
+
+ LWasmAtomicExchangeI64(const LAllocation& memoryBase, const LAllocation& ptr,
+ const LInt64Allocation& value,
+ const wasm::MemoryAccessDesc& access)
+ : LInstructionHelper(classOpcode), access_(access) {
+ setOperand(0, memoryBase);
+ setOperand(1, ptr);
+ setInt64Operand(2, value);
+ }
+
+ const LAllocation* memoryBase() { return getOperand(0); }
+ const LAllocation* ptr() { return getOperand(1); }
+ const LInt64Allocation value() { return getInt64Operand(2); }
+ const wasm::MemoryAccessDesc& access() { return access_; }
+};
+
+class LWasmAtomicBinopI64
+ : public LInstructionHelper<INT64_PIECES, 2 + INT64_PIECES, 0> {
+ const wasm::MemoryAccessDesc& access_;
+ AtomicOp op_;
+
+ public:
+ LIR_HEADER(WasmAtomicBinopI64);
+
+ LWasmAtomicBinopI64(const LAllocation& memoryBase, const LAllocation& ptr,
+ const LInt64Allocation& value,
+ const wasm::MemoryAccessDesc& access, AtomicOp op)
+ : LInstructionHelper(classOpcode), access_(access), op_(op) {
+ setOperand(0, memoryBase);
+ setOperand(1, ptr);
+ setInt64Operand(2, value);
+ }
+
+ const LAllocation* memoryBase() { return getOperand(0); }
+ const LAllocation* ptr() { return getOperand(1); }
+ const LInt64Allocation value() { return getInt64Operand(2); }
+ const wasm::MemoryAccessDesc& access() { return access_; }
+ AtomicOp operation() const { return op_; }
+};
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_LIR_x86_h */
diff --git a/js/src/jit/x86/Lowering-x86.cpp b/js/src/jit/x86/Lowering-x86.cpp
new file mode 100644
index 0000000000..968b5baf14
--- /dev/null
+++ b/js/src/jit/x86/Lowering-x86.cpp
@@ -0,0 +1,840 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86/Lowering-x86.h"
+
+#include "jit/Lowering.h"
+#include "jit/MIR.h"
+#include "jit/x86/Assembler-x86.h"
+
+#include "jit/shared/Lowering-shared-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+LBoxAllocation LIRGeneratorX86::useBoxFixed(MDefinition* mir, Register reg1,
+ Register reg2, bool useAtStart) {
+ MOZ_ASSERT(mir->type() == MIRType::Value);
+ MOZ_ASSERT(reg1 != reg2);
+
+ ensureDefined(mir);
+ return LBoxAllocation(LUse(reg1, mir->virtualRegister(), useAtStart),
+ LUse(reg2, VirtualRegisterOfPayload(mir), useAtStart));
+}
+
+LAllocation LIRGeneratorX86::useByteOpRegister(MDefinition* mir) {
+ return useFixed(mir, eax);
+}
+
+LAllocation LIRGeneratorX86::useByteOpRegisterAtStart(MDefinition* mir) {
+ return useFixedAtStart(mir, eax);
+}
+
+LAllocation LIRGeneratorX86::useByteOpRegisterOrNonDoubleConstant(
+ MDefinition* mir) {
+ return useFixed(mir, eax);
+}
+
+LDefinition LIRGeneratorX86::tempByteOpRegister() { return tempFixed(eax); }
+
+void LIRGenerator::visitBox(MBox* box) {
+ MDefinition* inner = box->getOperand(0);
+
+ // If the box wrapped a double, it needs a new register.
+ if (IsFloatingPointType(inner->type())) {
+ LDefinition spectreTemp =
+ JitOptions.spectreValueMasking ? temp() : LDefinition::BogusTemp();
+ defineBox(new (alloc()) LBoxFloatingPoint(useRegisterAtStart(inner),
+ tempCopy(inner, 0), spectreTemp,
+ inner->type()),
+ box);
+ return;
+ }
+
+ if (box->canEmitAtUses()) {
+ emitAtUses(box);
+ return;
+ }
+
+ if (inner->isConstant()) {
+ defineBox(new (alloc()) LValue(inner->toConstant()->toJSValue()), box);
+ return;
+ }
+
+ LBox* lir = new (alloc()) LBox(use(inner), inner->type());
+
+ // Otherwise, we should not define a new register for the payload portion
+ // of the output, so bypass defineBox().
+ uint32_t vreg = getVirtualRegister();
+
+ // Note that because we're using BogusTemp(), we do not change the type of
+ // the definition. We also do not define the first output as "TYPE",
+ // because it has no corresponding payload at (vreg + 1). Also note that
+ // although we copy the input's original type for the payload half of the
+ // definition, this is only for clarity. BogusTemp() definitions are
+ // ignored.
+ lir->setDef(0, LDefinition(vreg, LDefinition::GENERAL));
+ lir->setDef(1, LDefinition::BogusTemp());
+ box->setVirtualRegister(vreg);
+ add(lir);
+}
+
+void LIRGenerator::visitUnbox(MUnbox* unbox) {
+ MDefinition* inner = unbox->getOperand(0);
+
+ // An unbox on x86 reads in a type tag (either in memory or a register) and
+ // a payload. Unlike most instructions consuming a box, we ask for the type
+ // second, so that the result can re-use the first input.
+ MOZ_ASSERT(inner->type() == MIRType::Value);
+
+ ensureDefined(inner);
+
+ if (IsFloatingPointType(unbox->type())) {
+ LUnboxFloatingPoint* lir =
+ new (alloc()) LUnboxFloatingPoint(useBox(inner), unbox->type());
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+ define(lir, unbox);
+ return;
+ }
+
+ // Swap the order we use the box pieces so we can re-use the payload register.
+ LUnbox* lir = new (alloc()) LUnbox;
+ bool reusePayloadReg = !JitOptions.spectreValueMasking ||
+ unbox->type() == MIRType::Int32 ||
+ unbox->type() == MIRType::Boolean;
+ if (reusePayloadReg) {
+ lir->setOperand(0, usePayloadInRegisterAtStart(inner));
+ lir->setOperand(1, useType(inner, LUse::ANY));
+ } else {
+ lir->setOperand(0, usePayload(inner, LUse::REGISTER));
+ lir->setOperand(1, useType(inner, LUse::ANY));
+ }
+
+ if (unbox->fallible()) {
+ assignSnapshot(lir, unbox->bailoutKind());
+ }
+
+ // Types and payloads form two separate intervals. If the type becomes dead
+ // before the payload, it could be used as a Value without the type being
+ // recoverable. Unbox's purpose is to eagerly kill the definition of a type
+ // tag, so keeping both alive (for the purpose of gcmaps) is unappealing.
+ // Instead, we create a new virtual register.
+ if (reusePayloadReg) {
+ defineReuseInput(lir, unbox, 0);
+ } else {
+ define(lir, unbox);
+ }
+}
+
+void LIRGenerator::visitReturnImpl(MDefinition* opd, bool isGenerator) {
+ MOZ_ASSERT(opd->type() == MIRType::Value);
+
+ LReturn* ins = new (alloc()) LReturn(isGenerator);
+ ins->setOperand(0, LUse(JSReturnReg_Type));
+ ins->setOperand(1, LUse(JSReturnReg_Data));
+ fillBoxUses(ins, 0, opd);
+ add(ins);
+}
+
+void LIRGeneratorX86::lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ MDefinition* operand = phi->getOperand(inputPosition);
+ LPhi* type = block->getPhi(lirIndex + VREG_TYPE_OFFSET);
+ LPhi* payload = block->getPhi(lirIndex + VREG_DATA_OFFSET);
+ type->setOperand(
+ inputPosition,
+ LUse(operand->virtualRegister() + VREG_TYPE_OFFSET, LUse::ANY));
+ payload->setOperand(inputPosition,
+ LUse(VirtualRegisterOfPayload(operand), LUse::ANY));
+}
+
+void LIRGeneratorX86::defineInt64Phi(MPhi* phi, size_t lirIndex) {
+ LPhi* low = current->getPhi(lirIndex + INT64LOW_INDEX);
+ LPhi* high = current->getPhi(lirIndex + INT64HIGH_INDEX);
+
+ uint32_t lowVreg = getVirtualRegister();
+
+ phi->setVirtualRegister(lowVreg);
+
+ uint32_t highVreg = getVirtualRegister();
+ MOZ_ASSERT(lowVreg + INT64HIGH_INDEX == highVreg + INT64LOW_INDEX);
+
+ low->setDef(0, LDefinition(lowVreg, LDefinition::INT32));
+ high->setDef(0, LDefinition(highVreg, LDefinition::INT32));
+ annotate(high);
+ annotate(low);
+}
+
+void LIRGeneratorX86::lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition,
+ LBlock* block, size_t lirIndex) {
+ MDefinition* operand = phi->getOperand(inputPosition);
+ LPhi* low = block->getPhi(lirIndex + INT64LOW_INDEX);
+ LPhi* high = block->getPhi(lirIndex + INT64HIGH_INDEX);
+ low->setOperand(inputPosition,
+ LUse(operand->virtualRegister() + INT64LOW_INDEX, LUse::ANY));
+ high->setOperand(
+ inputPosition,
+ LUse(operand->virtualRegister() + INT64HIGH_INDEX, LUse::ANY));
+}
+
+void LIRGeneratorX86::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins, MDefinition* mir,
+ MDefinition* input) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(input));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX86::lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs) {
+ ins->setInt64Operand(0, useInt64RegisterAtStart(lhs));
+ ins->setInt64Operand(INT64_PIECES, useInt64OrConstant(rhs));
+ defineInt64ReuseInput(ins, mir, 0);
+}
+
+void LIRGeneratorX86::lowerForMulInt64(LMulI64* ins, MMul* mir,
+ MDefinition* lhs, MDefinition* rhs) {
+ bool needsTemp = true;
+
+ if (rhs->isConstant()) {
+ int64_t constant = rhs->toConstant()->toInt64();
+ int32_t shift = mozilla::FloorLog2(constant);
+ // See special cases in CodeGeneratorX86Shared::visitMulI64.
+ if (constant >= -1 && constant <= 2) {
+ needsTemp = false;
+ }
+ if (constant > 0 && int64_t(1) << shift == constant) {
+ needsTemp = false;
+ }
+ }
+
+ // MulI64 on x86 needs output to be in edx, eax;
+ ins->setInt64Operand(
+ 0, useInt64Fixed(lhs, Register64(edx, eax), /*useAtStart = */ true));
+ ins->setInt64Operand(INT64_PIECES, useInt64OrConstant(rhs));
+ if (needsTemp) {
+ ins->setTemp(0, temp());
+ }
+
+ defineInt64Fixed(ins, mir,
+ LInt64Allocation(LAllocation(AnyRegister(edx)),
+ LAllocation(AnyRegister(eax))));
+}
+
+void LIRGenerator::visitCompareExchangeTypedArrayElement(
+ MCompareExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useFixed(ins->elements(), esi);
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LUse oldval = useFixed(ins->oldval(), eax);
+ LUse newval = useFixed(ins->newval(), edx);
+ LDefinition temp = tempFixed(ebx);
+
+ auto* lir = new (alloc()) LCompareExchangeTypedArrayElement64(
+ elements, index, oldval, newval, temp);
+ defineFixed(lir, ins, LAllocation(AnyRegister(ecx)));
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerCompareExchangeTypedArrayElement(ins, /* useI386ByteRegisters = */ true);
+}
+
+void LIRGenerator::visitAtomicExchangeTypedArrayElement(
+ MAtomicExchangeTypedArrayElement* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LAllocation value = useFixed(ins->value(), edx);
+ LInt64Definition temp = tempInt64Fixed(Register64(ecx, ebx));
+
+ auto* lir = new (alloc())
+ LAtomicExchangeTypedArrayElement64(elements, index, value, temp);
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerAtomicExchangeTypedArrayElement(ins, /*useI386ByteRegisters=*/true);
+}
+
+void LIRGenerator::visitAtomicTypedArrayElementBinop(
+ MAtomicTypedArrayElementBinop* ins) {
+ MOZ_ASSERT(ins->elements()->type() == MIRType::Elements);
+ MOZ_ASSERT(ins->index()->type() == MIRType::IntPtr);
+
+ if (Scalar::isBigIntType(ins->arrayType())) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->arrayType());
+ LAllocation value = useFixed(ins->value(), edx);
+ LInt64Definition temp = tempInt64Fixed(Register64(ecx, ebx));
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We can omit allocating the result BigInt.
+
+ if (ins->isForEffect()) {
+ LDefinition tempLow = tempFixed(eax);
+
+ auto* lir = new (alloc()) LAtomicTypedArrayElementBinopForEffect64(
+ elements, index, value, temp, tempLow);
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+
+ auto* lir = new (alloc())
+ LAtomicTypedArrayElementBinop64(elements, index, value, temp);
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+ assignSafepoint(lir, ins);
+ return;
+ }
+
+ lowerAtomicTypedArrayElementBinop(ins, /* useI386ByteRegisters = */ true);
+}
+
+void LIRGeneratorX86::lowerAtomicLoad64(MLoadUnboxedScalar* ins) {
+ const LUse elements = useRegister(ins->elements());
+ const LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->storageType());
+
+ auto* lir = new (alloc()) LAtomicLoad64(elements, index, tempFixed(ebx),
+ tempInt64Fixed(Register64(edx, eax)));
+ defineFixed(lir, ins, LAllocation(AnyRegister(ecx)));
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX86::lowerAtomicStore64(MStoreUnboxedScalar* ins) {
+ LUse elements = useRegister(ins->elements());
+ LAllocation index =
+ useRegisterOrIndexConstant(ins->index(), ins->writeType());
+ LAllocation value = useFixed(ins->value(), edx);
+ LInt64Definition temp1 = tempInt64Fixed(Register64(ecx, ebx));
+ LDefinition temp2 = tempFixed(eax);
+
+ add(new (alloc()) LAtomicStore64(elements, index, value, temp1, temp2), ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToDouble(MWasmUnsignedToDouble* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToDouble* lir = new (alloc())
+ LWasmUint32ToDouble(useRegisterAtStart(ins->input()), temp());
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmUnsignedToFloat32(MWasmUnsignedToFloat32* ins) {
+ MOZ_ASSERT(ins->input()->type() == MIRType::Int32);
+ LWasmUint32ToFloat32* lir = new (alloc())
+ LWasmUint32ToFloat32(useRegisterAtStart(ins->input()), temp());
+ define(lir, ins);
+}
+
+// If the base is a constant, and it is zero or its offset is zero, then
+// code generation will fold the values into the access. Allocate the
+// pointer to a register only if that can't happen.
+
+static bool OptimizableConstantAccess(MDefinition* base,
+ const wasm::MemoryAccessDesc& access) {
+ MOZ_ASSERT(base->isConstant());
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ if (!(base->toConstant()->isInt32(0) || access.offset() == 0)) {
+ return false;
+ }
+ if (access.type() == Scalar::Int64) {
+ // For int64 accesses on 32-bit systems we will need to add another offset
+ // of 4 to access the high part of the value; make sure this does not
+ // overflow the value.
+ int32_t v;
+ if (base->toConstant()->isInt32(0)) {
+ v = access.offset();
+ } else {
+ v = base->toConstant()->toInt32();
+ }
+ return v <= int32_t(INT32_MAX - INT64HIGH_OFFSET);
+ }
+ return true;
+}
+
+void LIRGenerator::visitWasmHeapBase(MWasmHeapBase* ins) {
+ auto* lir = new (alloc()) LWasmHeapBase(useRegisterAtStart(ins->instance()));
+ define(lir, ins);
+}
+
+void LIRGenerator::visitWasmLoad(MWasmLoad* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* memoryBase = ins->memoryBase();
+ MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
+
+ if (ins->access().type() == Scalar::Int64 && ins->access().isAtomic()) {
+ auto* lir = new (alloc())
+ LWasmAtomicLoadI64(useRegister(memoryBase), useRegister(base),
+ tempFixed(ecx), tempFixed(ebx));
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(edx)),
+ LAllocation(AnyRegister(eax))));
+ return;
+ }
+
+ LAllocation baseAlloc;
+ if (!base->isConstant() || !OptimizableConstantAccess(base, ins->access())) {
+ baseAlloc = ins->type() == MIRType::Int64 ? useRegister(base)
+ : useRegisterAtStart(base);
+ }
+
+ if (ins->type() != MIRType::Int64) {
+ auto* lir =
+ new (alloc()) LWasmLoad(baseAlloc, useRegisterAtStart(memoryBase));
+ define(lir, ins);
+ return;
+ }
+
+ // "AtStart" register usage does not work for the 64-bit case because we
+ // clobber two registers for the result and may need two registers for a
+ // scaled address; we can't guarantee non-interference.
+
+ auto* lir = new (alloc()) LWasmLoadI64(baseAlloc, useRegister(memoryBase));
+
+ Scalar::Type accessType = ins->access().type();
+ if (accessType == Scalar::Int8 || accessType == Scalar::Int16 ||
+ accessType == Scalar::Int32) {
+ // We use cdq to sign-extend the result and cdq demands these registers.
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(edx)),
+ LAllocation(AnyRegister(eax))));
+ return;
+ }
+
+ defineInt64(lir, ins);
+}
+
+void LIRGenerator::visitWasmStore(MWasmStore* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* memoryBase = ins->memoryBase();
+ MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
+
+ if (ins->access().type() == Scalar::Int64 && ins->access().isAtomic()) {
+ auto* lir = new (alloc())
+ LWasmAtomicStoreI64(useRegister(memoryBase), useRegister(base),
+ useInt64Fixed(ins->value(), Register64(ecx, ebx)),
+ tempFixed(edx), tempFixed(eax));
+ add(lir, ins);
+ return;
+ }
+
+ LAllocation baseAlloc;
+ if (!base->isConstant() || !OptimizableConstantAccess(base, ins->access())) {
+ baseAlloc = useRegisterAtStart(base);
+ }
+
+ LAllocation valueAlloc;
+ switch (ins->access().type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8:
+ // See comment for LIRGeneratorX86::useByteOpRegister.
+ valueAlloc = useFixed(ins->value(), eax);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ case Scalar::Float32:
+ case Scalar::Float64:
+ // For now, don't allow constant values. The immediate operand affects
+ // instruction layout which affects patching.
+ valueAlloc = useRegisterAtStart(ins->value());
+ break;
+ case Scalar::Simd128:
+#ifdef ENABLE_WASM_SIMD
+ valueAlloc = useRegisterAtStart(ins->value());
+ break;
+#else
+ MOZ_CRASH("unexpected array type");
+#endif
+ case Scalar::Int64: {
+ LInt64Allocation valueAlloc = useInt64RegisterAtStart(ins->value());
+ auto* lir = new (alloc())
+ LWasmStoreI64(baseAlloc, valueAlloc, useRegisterAtStart(memoryBase));
+ add(lir, ins);
+ return;
+ }
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ auto* lir = new (alloc())
+ LWasmStore(baseAlloc, valueAlloc, useRegisterAtStart(memoryBase));
+ add(lir, ins);
+}
+
+void LIRGenerator::visitWasmCompareExchangeHeap(MWasmCompareExchangeHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* memoryBase = ins->memoryBase();
+ MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc()) LWasmCompareExchangeI64(
+ useRegisterAtStart(memoryBase), useRegisterAtStart(base),
+ useInt64FixedAtStart(ins->oldValue(), Register64(edx, eax)),
+ useInt64FixedAtStart(ins->newValue(), Register64(ecx, ebx)));
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(edx)),
+ LAllocation(AnyRegister(eax))));
+ return;
+ }
+
+ MOZ_ASSERT(ins->access().type() < Scalar::Float32);
+
+ bool byteArray = byteSize(ins->access().type()) == 1;
+
+ // Register allocation:
+ //
+ // The output may not be used, but eax will be clobbered regardless
+ // so pin the output to eax.
+ //
+ // oldval must be in a register.
+ //
+ // newval must be in a register. If the source is a byte array
+ // then newval must be a register that has a byte size: this must
+ // be ebx, ecx, or edx (eax is taken).
+ //
+ // Bug #1077036 describes some optimization opportunities.
+
+ const LAllocation oldval = useRegister(ins->oldValue());
+ const LAllocation newval =
+ byteArray ? useFixed(ins->newValue(), ebx) : useRegister(ins->newValue());
+
+ LWasmCompareExchangeHeap* lir = new (alloc()) LWasmCompareExchangeHeap(
+ useRegister(base), oldval, newval, useRegister(memoryBase));
+
+ lir->setAddrTemp(temp());
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+}
+
+void LIRGenerator::visitWasmAtomicExchangeHeap(MWasmAtomicExchangeHeap* ins) {
+ MDefinition* memoryBase = ins->memoryBase();
+ MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
+
+ if (ins->access().type() == Scalar::Int64) {
+ MDefinition* base = ins->base();
+ auto* lir = new (alloc()) LWasmAtomicExchangeI64(
+ useRegister(memoryBase), useRegister(base),
+ useInt64Fixed(ins->value(), Register64(ecx, ebx)), ins->access());
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(edx)),
+ LAllocation(AnyRegister(eax))));
+ return;
+ }
+
+ const LAllocation base = useRegister(ins->base());
+ const LAllocation value = useRegister(ins->value());
+
+ LWasmAtomicExchangeHeap* lir = new (alloc())
+ LWasmAtomicExchangeHeap(base, value, useRegister(memoryBase));
+
+ lir->setAddrTemp(temp());
+ if (byteSize(ins->access().type()) == 1) {
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+ } else {
+ define(lir, ins);
+ }
+}
+
+void LIRGenerator::visitWasmAtomicBinopHeap(MWasmAtomicBinopHeap* ins) {
+ MDefinition* base = ins->base();
+ MOZ_ASSERT(base->type() == MIRType::Int32);
+
+ MDefinition* memoryBase = ins->memoryBase();
+ MOZ_ASSERT(memoryBase->type() == MIRType::Pointer);
+
+ if (ins->access().type() == Scalar::Int64) {
+ auto* lir = new (alloc())
+ LWasmAtomicBinopI64(useRegister(memoryBase), useRegister(base),
+ useInt64Fixed(ins->value(), Register64(ecx, ebx)),
+ ins->access(), ins->operation());
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(edx)),
+ LAllocation(AnyRegister(eax))));
+ return;
+ }
+
+ MOZ_ASSERT(ins->access().type() < Scalar::Float32);
+
+ bool byteArray = byteSize(ins->access().type()) == 1;
+
+ // Case 1: the result of the operation is not used.
+ //
+ // We'll emit a single instruction: LOCK ADD, LOCK SUB, LOCK AND,
+ // LOCK OR, or LOCK XOR. These can all take an immediate.
+
+ if (!ins->hasUses()) {
+ LAllocation value;
+ if (byteArray && !ins->value()->isConstant()) {
+ value = useFixed(ins->value(), ebx);
+ } else {
+ value = useRegisterOrConstant(ins->value());
+ }
+ LWasmAtomicBinopHeapForEffect* lir =
+ new (alloc()) LWasmAtomicBinopHeapForEffect(useRegister(base), value,
+ LDefinition::BogusTemp(),
+ useRegister(memoryBase));
+ lir->setAddrTemp(temp());
+ add(lir, ins);
+ return;
+ }
+
+ // Case 2: the result of the operation is used.
+ //
+ // For ADD and SUB we'll use XADD:
+ //
+ // movl value, output
+ // lock xaddl output, mem
+ //
+ // For the 8-bit variants XADD needs a byte register for the
+ // output only, we can still set up with movl; just pin the output
+ // to eax (or ebx / ecx / edx).
+ //
+ // For AND/OR/XOR we need to use a CMPXCHG loop:
+ //
+ // movl *mem, eax
+ // L: mov eax, temp
+ // andl value, temp
+ // lock cmpxchg temp, mem ; reads eax also
+ // jnz L
+ // ; result in eax
+ //
+ // Note the placement of L, cmpxchg will update eax with *mem if
+ // *mem does not have the expected value, so reloading it at the
+ // top of the loop would be redundant.
+ //
+ // We want to fix eax as the output. We also need a temp for
+ // the intermediate value.
+ //
+ // For the 8-bit variants the temp must have a byte register.
+ //
+ // There are optimization opportunities:
+ // - better 8-bit register allocation and instruction selection, Bug
+ // #1077036.
+
+ bool bitOp = !(ins->operation() == AtomicFetchAddOp ||
+ ins->operation() == AtomicFetchSubOp);
+ LDefinition tempDef = LDefinition::BogusTemp();
+ LAllocation value;
+
+ if (byteArray) {
+ value = useFixed(ins->value(), ebx);
+ if (bitOp) {
+ tempDef = tempFixed(ecx);
+ }
+ } else if (bitOp || ins->value()->isConstant()) {
+ value = useRegisterOrConstant(ins->value());
+ if (bitOp) {
+ tempDef = temp();
+ }
+ } else {
+ value = useRegisterAtStart(ins->value());
+ }
+
+ LWasmAtomicBinopHeap* lir = new (alloc())
+ LWasmAtomicBinopHeap(useRegister(base), value, tempDef,
+ LDefinition::BogusTemp(), useRegister(memoryBase));
+
+ lir->setAddrTemp(temp());
+ if (byteArray || bitOp) {
+ defineFixed(lir, ins, LAllocation(AnyRegister(eax)));
+ } else if (ins->value()->isConstant()) {
+ define(lir, ins);
+ } else {
+ defineReuseInput(lir, ins, LWasmAtomicBinopHeap::valueOp);
+ }
+}
+
+void LIRGeneratorX86::lowerDivI64(MDiv* div) {
+ MOZ_CRASH("We use MWasmBuiltinModI64 instead.");
+}
+
+void LIRGeneratorX86::lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div) {
+ MOZ_ASSERT(div->lhs()->type() == div->rhs()->type());
+ MOZ_ASSERT(IsNumberType(div->type()));
+
+ MOZ_ASSERT(div->type() == MIRType::Int64);
+
+ if (div->isUnsigned()) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useInt64FixedAtStart(div->lhs(), Register64(eax, ebx)),
+ useInt64FixedAtStart(div->rhs(), Register64(ecx, edx)),
+ useFixedAtStart(div->instance(), InstanceReg));
+ defineReturn(lir, div);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc())
+ LDivOrModI64(useInt64FixedAtStart(div->lhs(), Register64(eax, ebx)),
+ useInt64FixedAtStart(div->rhs(), Register64(ecx, edx)),
+ useFixedAtStart(div->instance(), InstanceReg));
+ defineReturn(lir, div);
+}
+
+void LIRGeneratorX86::lowerModI64(MMod* mod) {
+ MOZ_CRASH("We use MWasmBuiltinModI64 instead.");
+}
+
+void LIRGeneratorX86::lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod) {
+ MDefinition* lhs = mod->lhs();
+ MDefinition* rhs = mod->rhs();
+ MOZ_ASSERT(lhs->type() == rhs->type());
+ MOZ_ASSERT(IsNumberType(mod->type()));
+
+ MOZ_ASSERT(mod->type() == MIRType::Int64);
+ MOZ_ASSERT(mod->type() == MIRType::Int64);
+
+ if (mod->isUnsigned()) {
+ LUDivOrModI64* lir = new (alloc())
+ LUDivOrModI64(useInt64FixedAtStart(lhs, Register64(eax, ebx)),
+ useInt64FixedAtStart(rhs, Register64(ecx, edx)),
+ useFixedAtStart(mod->instance(), InstanceReg));
+ defineReturn(lir, mod);
+ return;
+ }
+
+ LDivOrModI64* lir = new (alloc())
+ LDivOrModI64(useInt64FixedAtStart(lhs, Register64(eax, ebx)),
+ useInt64FixedAtStart(rhs, Register64(ecx, edx)),
+ useFixedAtStart(mod->instance(), InstanceReg));
+ defineReturn(lir, mod);
+}
+
+void LIRGeneratorX86::lowerUDivI64(MDiv* div) {
+ MOZ_CRASH("We use MWasmBuiltinDivI64 instead.");
+}
+
+void LIRGeneratorX86::lowerUModI64(MMod* mod) {
+ MOZ_CRASH("We use MWasmBuiltinModI64 instead.");
+}
+
+void LIRGeneratorX86::lowerBigIntDiv(MBigIntDiv* ins) {
+ auto* lir = new (alloc()) LBigIntDiv(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(eax), temp());
+ defineFixed(lir, ins, LAllocation(AnyRegister(edx)));
+ assignSafepoint(lir, ins);
+}
+
+void LIRGeneratorX86::lowerBigIntMod(MBigIntMod* ins) {
+ auto* lir = new (alloc()) LBigIntMod(
+ useRegister(ins->lhs()), useRegister(ins->rhs()), tempFixed(eax), temp());
+ defineFixed(lir, ins, LAllocation(AnyRegister(edx)));
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitSubstr(MSubstr* ins) {
+ // Due to lack of registers on x86, we reuse the string register as
+ // temporary. As a result we only need two temporary registers and take a
+ // bogus temporary as fifth argument.
+ LSubstr* lir = new (alloc())
+ LSubstr(useRegister(ins->string()), useRegister(ins->begin()),
+ useRegister(ins->length()), temp(), LDefinition::BogusTemp(),
+ tempByteOpRegister());
+ define(lir, ins);
+ assignSafepoint(lir, ins);
+}
+
+void LIRGenerator::visitWasmTruncateToInt64(MWasmTruncateToInt64* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Double || opd->type() == MIRType::Float32);
+
+ LDefinition temp = tempDouble();
+ defineInt64(new (alloc()) LWasmTruncateToInt64(useRegister(opd), temp), ins);
+}
+
+void LIRGeneratorX86::lowerWasmBuiltinTruncateToInt64(
+ MWasmBuiltinTruncateToInt64* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitInt64ToFloatingPoint(MInt64ToFloatingPoint* ins) {
+ MDefinition* opd = ins->input();
+ MOZ_ASSERT(opd->type() == MIRType::Int64);
+ MOZ_ASSERT(IsFloatingPointType(ins->type()));
+
+ LDefinition maybeTemp =
+ (ins->isUnsigned() &&
+ ((ins->type() == MIRType::Double && AssemblerX86Shared::HasSSE3()) ||
+ ins->type() == MIRType::Float32))
+ ? temp()
+ : LDefinition::BogusTemp();
+
+ define(new (alloc()) LInt64ToFloatingPoint(useInt64Register(opd), maybeTemp),
+ ins);
+}
+
+void LIRGeneratorX86::lowerBuiltinInt64ToFloatingPoint(
+ MBuiltinInt64ToFloatingPoint* ins) {
+ MOZ_CRASH("We don't use it for this architecture");
+}
+
+void LIRGenerator::visitExtendInt32ToInt64(MExtendInt32ToInt64* ins) {
+ if (ins->isUnsigned()) {
+ defineInt64(new (alloc())
+ LExtendInt32ToInt64(useRegisterAtStart(ins->input())),
+ ins);
+ } else {
+ LExtendInt32ToInt64* lir =
+ new (alloc()) LExtendInt32ToInt64(useFixedAtStart(ins->input(), eax));
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(edx)),
+ LAllocation(AnyRegister(eax))));
+ }
+}
+
+void LIRGenerator::visitSignExtendInt64(MSignExtendInt64* ins) {
+ // Here we'll end up using cdq which requires input and output in (edx,eax).
+ LSignExtendInt64* lir = new (alloc()) LSignExtendInt64(
+ useInt64FixedAtStart(ins->input(), Register64(edx, eax)));
+ defineInt64Fixed(lir, ins,
+ LInt64Allocation(LAllocation(AnyRegister(edx)),
+ LAllocation(AnyRegister(eax))));
+}
+
+// On x86 we specialize the only cases where compare is {U,}Int32 and select
+// is {U,}Int32.
+bool LIRGeneratorShared::canSpecializeWasmCompareAndSelect(
+ MCompare::CompareType compTy, MIRType insTy) {
+ return insTy == MIRType::Int32 && (compTy == MCompare::Compare_Int32 ||
+ compTy == MCompare::Compare_UInt32);
+}
+
+void LIRGeneratorShared::lowerWasmCompareAndSelect(MWasmSelect* ins,
+ MDefinition* lhs,
+ MDefinition* rhs,
+ MCompare::CompareType compTy,
+ JSOp jsop) {
+ MOZ_ASSERT(canSpecializeWasmCompareAndSelect(compTy, ins->type()));
+ auto* lir = new (alloc()) LWasmCompareAndSelect(
+ useRegister(lhs), useAny(rhs), compTy, jsop,
+ useRegisterAtStart(ins->trueExpr()), useAny(ins->falseExpr()));
+ defineReuseInput(lir, ins, LWasmCompareAndSelect::IfTrueExprIndex);
+}
diff --git a/js/src/jit/x86/Lowering-x86.h b/js/src/jit/x86/Lowering-x86.h
new file mode 100644
index 0000000000..b82109981e
--- /dev/null
+++ b/js/src/jit/x86/Lowering-x86.h
@@ -0,0 +1,79 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_Lowering_x86_h
+#define jit_x86_Lowering_x86_h
+
+#include "jit/x86-shared/Lowering-x86-shared.h"
+
+namespace js {
+namespace jit {
+
+class LIRGeneratorX86 : public LIRGeneratorX86Shared {
+ protected:
+ LIRGeneratorX86(MIRGenerator* gen, MIRGraph& graph, LIRGraph& lirGraph)
+ : LIRGeneratorX86Shared(gen, graph, lirGraph) {}
+
+ // Returns a box allocation with type set to reg1 and payload set to reg2.
+ LBoxAllocation useBoxFixed(MDefinition* mir, Register reg1, Register reg2,
+ bool useAtStart = false);
+
+ // It's a trap! On x86, the 1-byte store can only use one of
+ // {al,bl,cl,dl,ah,bh,ch,dh}. That means if the register allocator
+ // gives us one of {edi,esi,ebp,esp}, we're out of luck. (The formatter
+ // will assert on us.) Ideally, we'd just ask the register allocator to
+ // give us one of {al,bl,cl,dl}. For now, just useFixed(al).
+ LAllocation useByteOpRegister(MDefinition* mir);
+ LAllocation useByteOpRegisterAtStart(MDefinition* mir);
+ LAllocation useByteOpRegisterOrNonDoubleConstant(MDefinition* mir);
+ LDefinition tempByteOpRegister();
+
+ inline LDefinition tempToUnbox() { return LDefinition::BogusTemp(); }
+
+ bool needTempForPostBarrier() { return true; }
+
+ void lowerUntypedPhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+
+ void lowerInt64PhiInput(MPhi* phi, uint32_t inputPosition, LBlock* block,
+ size_t lirIndex);
+ void defineInt64Phi(MPhi* phi, size_t lirIndex);
+
+ void lowerForALUInt64(LInstructionHelper<INT64_PIECES, INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* input);
+ void lowerForALUInt64(
+ LInstructionHelper<INT64_PIECES, 2 * INT64_PIECES, 0>* ins,
+ MDefinition* mir, MDefinition* lhs, MDefinition* rhs);
+ void lowerForMulInt64(LMulI64* ins, MMul* mir, MDefinition* lhs,
+ MDefinition* rhs);
+
+ void lowerBuiltinInt64ToFloatingPoint(MBuiltinInt64ToFloatingPoint* ins);
+ void lowerWasmBuiltinTruncateToInt64(MWasmBuiltinTruncateToInt64* ins);
+ void lowerDivI64(MDiv* div);
+ void lowerWasmBuiltinDivI64(MWasmBuiltinDivI64* div);
+ void lowerModI64(MMod* mod);
+ void lowerWasmBuiltinModI64(MWasmBuiltinModI64* mod);
+ void lowerUDivI64(MDiv* div);
+ void lowerUModI64(MMod* mod);
+
+ void lowerBigIntDiv(MBigIntDiv* ins);
+ void lowerBigIntMod(MBigIntMod* ins);
+
+ void lowerAtomicLoad64(MLoadUnboxedScalar* ins);
+ void lowerAtomicStore64(MStoreUnboxedScalar* ins);
+
+ void lowerPhi(MPhi* phi);
+
+ public:
+ static bool allowTypedElementHoleCheck() { return true; }
+};
+
+typedef LIRGeneratorX86 LIRGeneratorSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_Lowering_x86_h */
diff --git a/js/src/jit/x86/MacroAssembler-x86-inl.h b/js/src/jit/x86/MacroAssembler-x86-inl.h
new file mode 100644
index 0000000000..66050cc1b5
--- /dev/null
+++ b/js/src/jit/x86/MacroAssembler-x86-inl.h
@@ -0,0 +1,1386 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_MacroAssembler_x86_inl_h
+#define jit_x86_MacroAssembler_x86_inl_h
+
+#include "jit/x86/MacroAssembler-x86.h"
+
+#include "jit/x86-shared/MacroAssembler-x86-shared-inl.h"
+
+namespace js {
+namespace jit {
+
+//{{{ check_macroassembler_style
+
+void MacroAssembler::move64(Imm64 imm, Register64 dest) {
+ move32(Imm32(imm.value & 0xFFFFFFFFL), dest.low);
+ move32(Imm32((imm.value >> 32) & 0xFFFFFFFFL), dest.high);
+}
+
+void MacroAssembler::move64(Register64 src, Register64 dest) {
+ movl(src.low, dest.low);
+ movl(src.high, dest.high);
+}
+
+void MacroAssembler::moveDoubleToGPR64(FloatRegister src, Register64 dest) {
+ ScratchDoubleScope scratch(*this);
+
+ if (Assembler::HasSSE41()) {
+ vmovd(src, dest.low);
+ vpextrd(1, src, dest.high);
+ } else {
+ vmovd(src, dest.low);
+ moveDouble(src, scratch);
+ vpsrldq(Imm32(4), scratch, scratch);
+ vmovd(scratch, dest.high);
+ }
+}
+
+void MacroAssembler::moveGPR64ToDouble(Register64 src, FloatRegister dest) {
+ if (Assembler::HasSSE41()) {
+ vmovd(src.low, dest);
+ vpinsrd(1, src.high, dest, dest);
+ } else {
+ ScratchDoubleScope fpscratch(*this);
+ vmovd(src.low, dest);
+ vmovd(src.high, fpscratch);
+ vunpcklps(fpscratch, dest, dest);
+ }
+}
+
+void MacroAssembler::move64To32(Register64 src, Register dest) {
+ if (src.low != dest) {
+ movl(src.low, dest);
+ }
+}
+
+void MacroAssembler::move32To64ZeroExtend(Register src, Register64 dest) {
+ if (src != dest.low) {
+ movl(src, dest.low);
+ }
+ movl(Imm32(0), dest.high);
+}
+
+void MacroAssembler::move8To64SignExtend(Register src, Register64 dest) {
+ move8SignExtend(src, dest.low);
+ if (dest.low == eax && dest.high == edx) {
+ masm.cdq();
+ } else {
+ movl(dest.low, dest.high);
+ sarl(Imm32(31), dest.high);
+ }
+}
+
+void MacroAssembler::move16To64SignExtend(Register src, Register64 dest) {
+ move16SignExtend(src, dest.low);
+ if (dest.low == eax && dest.high == edx) {
+ masm.cdq();
+ } else {
+ movl(dest.low, dest.high);
+ sarl(Imm32(31), dest.high);
+ }
+}
+
+void MacroAssembler::move32To64SignExtend(Register src, Register64 dest) {
+ if (src != dest.low) {
+ movl(src, dest.low);
+ }
+ if (dest.low == eax && dest.high == edx) {
+ masm.cdq();
+ } else {
+ movl(dest.low, dest.high);
+ sarl(Imm32(31), dest.high);
+ }
+}
+
+void MacroAssembler::move32SignExtendToPtr(Register src, Register dest) {
+ movl(src, dest);
+}
+
+void MacroAssembler::move32ZeroExtendToPtr(Register src, Register dest) {
+ movl(src, dest);
+}
+
+// ===============================================================
+// Load instructions
+
+void MacroAssembler::load32SignExtendToPtr(const Address& src, Register dest) {
+ load32(src, dest);
+}
+
+// ===============================================================
+// Logical functions
+
+void MacroAssembler::notPtr(Register reg) { notl(reg); }
+
+void MacroAssembler::andPtr(Register src, Register dest) { andl(src, dest); }
+
+void MacroAssembler::andPtr(Imm32 imm, Register dest) { andl(imm, dest); }
+
+void MacroAssembler::and64(Imm64 imm, Register64 dest) {
+ if (imm.low().value != int32_t(0xFFFFFFFF)) {
+ andl(imm.low(), dest.low);
+ }
+ if (imm.hi().value != int32_t(0xFFFFFFFF)) {
+ andl(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::or64(Imm64 imm, Register64 dest) {
+ if (imm.low().value != 0) {
+ orl(imm.low(), dest.low);
+ }
+ if (imm.hi().value != 0) {
+ orl(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::xor64(Imm64 imm, Register64 dest) {
+ if (imm.low().value != 0) {
+ xorl(imm.low(), dest.low);
+ }
+ if (imm.hi().value != 0) {
+ xorl(imm.hi(), dest.high);
+ }
+}
+
+void MacroAssembler::orPtr(Register src, Register dest) { orl(src, dest); }
+
+void MacroAssembler::orPtr(Imm32 imm, Register dest) { orl(imm, dest); }
+
+void MacroAssembler::and64(Register64 src, Register64 dest) {
+ andl(src.low, dest.low);
+ andl(src.high, dest.high);
+}
+
+void MacroAssembler::or64(Register64 src, Register64 dest) {
+ orl(src.low, dest.low);
+ orl(src.high, dest.high);
+}
+
+void MacroAssembler::xor64(Register64 src, Register64 dest) {
+ xorl(src.low, dest.low);
+ xorl(src.high, dest.high);
+}
+
+void MacroAssembler::xorPtr(Register src, Register dest) { xorl(src, dest); }
+
+void MacroAssembler::xorPtr(Imm32 imm, Register dest) { xorl(imm, dest); }
+
+// ===============================================================
+// Swap instructions
+
+void MacroAssembler::byteSwap64(Register64 reg) {
+ bswapl(reg.low);
+ bswapl(reg.high);
+ xchgl(reg.low, reg.high);
+}
+
+// ===============================================================
+// Arithmetic functions
+
+void MacroAssembler::addPtr(Register src, Register dest) { addl(src, dest); }
+
+void MacroAssembler::addPtr(Imm32 imm, Register dest) { addl(imm, dest); }
+
+void MacroAssembler::addPtr(ImmWord imm, Register dest) {
+ addl(Imm32(imm.value), dest);
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const Address& dest) {
+ addl(imm, Operand(dest));
+}
+
+void MacroAssembler::addPtr(Imm32 imm, const AbsoluteAddress& dest) {
+ addl(imm, Operand(dest));
+}
+
+void MacroAssembler::addPtr(const Address& src, Register dest) {
+ addl(Operand(src), dest);
+}
+
+void MacroAssembler::add64(Register64 src, Register64 dest) {
+ addl(src.low, dest.low);
+ adcl(src.high, dest.high);
+}
+
+void MacroAssembler::add64(Imm32 imm, Register64 dest) {
+ addl(imm, dest.low);
+ adcl(Imm32(0), dest.high);
+}
+
+void MacroAssembler::add64(Imm64 imm, Register64 dest) {
+ if (imm.low().value == 0) {
+ addl(imm.hi(), dest.high);
+ return;
+ }
+ addl(imm.low(), dest.low);
+ adcl(imm.hi(), dest.high);
+}
+
+void MacroAssembler::addConstantDouble(double d, FloatRegister dest) {
+ Double* dbl = getDouble(d);
+ if (!dbl) {
+ return;
+ }
+ masm.vaddsd_mr(nullptr, dest.encoding(), dest.encoding());
+ propagateOOM(dbl->uses.append(CodeOffset(masm.size())));
+}
+
+CodeOffset MacroAssembler::sub32FromStackPtrWithPatch(Register dest) {
+ moveStackPtrTo(dest);
+ addlWithPatch(Imm32(0), dest);
+ return CodeOffset(currentOffset());
+}
+
+void MacroAssembler::patchSub32FromStackPtr(CodeOffset offset, Imm32 imm) {
+ patchAddl(offset, -imm.value);
+}
+
+void MacroAssembler::subPtr(Register src, Register dest) { subl(src, dest); }
+
+void MacroAssembler::subPtr(Register src, const Address& dest) {
+ subl(src, Operand(dest));
+}
+
+void MacroAssembler::subPtr(Imm32 imm, Register dest) { subl(imm, dest); }
+
+void MacroAssembler::subPtr(const Address& addr, Register dest) {
+ subl(Operand(addr), dest);
+}
+
+void MacroAssembler::sub64(Register64 src, Register64 dest) {
+ subl(src.low, dest.low);
+ sbbl(src.high, dest.high);
+}
+
+void MacroAssembler::sub64(Imm64 imm, Register64 dest) {
+ if (imm.low().value == 0) {
+ subl(imm.hi(), dest.high);
+ return;
+ }
+ subl(imm.low(), dest.low);
+ sbbl(imm.hi(), dest.high);
+}
+
+void MacroAssembler::mulHighUnsigned32(Imm32 imm, Register src, Register dest) {
+ // Preserve edx:eax, unless they're the destination register.
+ if (edx != dest) {
+ push(edx);
+ }
+ if (eax != dest) {
+ push(eax);
+ }
+
+ if (src != eax) {
+ // Compute edx:eax := eax ∗ src
+ movl(imm, eax);
+ mull(src);
+ } else {
+ // Compute edx:eax := eax ∗ edx
+ movl(imm, edx);
+ mull(edx);
+ }
+
+ // Move the high word from edx into |dest|.
+ if (edx != dest) {
+ movl(edx, dest);
+ }
+
+ // Restore edx:eax.
+ if (eax != dest) {
+ pop(eax);
+ }
+ if (edx != dest) {
+ pop(edx);
+ }
+}
+
+void MacroAssembler::mulPtr(Register rhs, Register srcDest) {
+ imull(rhs, srcDest);
+}
+
+// Note: this function clobbers eax and edx.
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest) {
+ // LOW32 = LOW(LOW(dest) * LOW(imm));
+ // HIGH32 = LOW(HIGH(dest) * LOW(imm)) [multiply imm into upper bits]
+ // + LOW(LOW(dest) * HIGH(imm)) [multiply dest into upper bits]
+ // + HIGH(LOW(dest) * LOW(imm)) [carry]
+
+ MOZ_ASSERT(dest.low != eax && dest.low != edx);
+ MOZ_ASSERT(dest.high != eax && dest.high != edx);
+
+ // HIGH(dest) = LOW(HIGH(dest) * LOW(imm));
+ movl(Imm32(imm.value & 0xFFFFFFFFL), edx);
+ imull(edx, dest.high);
+
+ // edx:eax = LOW(dest) * LOW(imm);
+ movl(Imm32(imm.value & 0xFFFFFFFFL), edx);
+ movl(dest.low, eax);
+ mull(edx);
+
+ // HIGH(dest) += edx;
+ addl(edx, dest.high);
+
+ // HIGH(dest) += LOW(LOW(dest) * HIGH(imm));
+ if (((imm.value >> 32) & 0xFFFFFFFFL) == 5) {
+ leal(Operand(dest.low, dest.low, TimesFour), edx);
+ } else {
+ MOZ_CRASH("Unsupported imm");
+ }
+ addl(edx, dest.high);
+
+ // LOW(dest) = eax;
+ movl(eax, dest.low);
+}
+
+void MacroAssembler::mul64(Imm64 imm, const Register64& dest,
+ const Register temp) {
+ // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
+ // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
+ // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
+ // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
+
+ MOZ_ASSERT(dest == Register64(edx, eax));
+ MOZ_ASSERT(temp != edx && temp != eax);
+
+ movl(dest.low, temp);
+
+ // Compute mul64
+ imull(imm.low(), dest.high); // (2)
+ imull(imm.hi(), temp); // (3)
+ addl(dest.high, temp);
+ movl(imm.low(), dest.high);
+ mull(dest.high /*, dest.low*/); // (4) + (1) output in edx:eax
+ // (dest_hi:dest_lo)
+ addl(temp, dest.high);
+}
+
+void MacroAssembler::mul64(const Register64& src, const Register64& dest,
+ const Register temp) {
+ // LOW32 = LOW(LOW(dest) * LOW(src)); (1)
+ // HIGH32 = LOW(HIGH(dest) * LOW(src)) [multiply src into upper bits] (2)
+ // + LOW(LOW(dest) * HIGH(src)) [multiply dest into upper bits] (3)
+ // + HIGH(LOW(dest) * LOW(src)) [carry] (4)
+
+ MOZ_ASSERT(dest == Register64(edx, eax));
+ MOZ_ASSERT(src != Register64(edx, eax) && src != Register64(eax, edx));
+
+ // Make sure the rhs.high isn't the dest.high register anymore.
+ // This saves us from doing other register moves.
+ movl(dest.low, temp);
+
+ // Compute mul64
+ imull(src.low, dest.high); // (2)
+ imull(src.high, temp); // (3)
+ addl(dest.high, temp);
+ movl(src.low, dest.high);
+ mull(dest.high /*, dest.low*/); // (4) + (1) output in edx:eax
+ // (dest_hi:dest_lo)
+ addl(temp, dest.high);
+}
+
+void MacroAssembler::mulBy3(Register src, Register dest) {
+ lea(Operand(src, src, TimesTwo), dest);
+}
+
+void MacroAssembler::mulDoublePtr(ImmPtr imm, Register temp,
+ FloatRegister dest) {
+ movl(imm, temp);
+ vmulsd(Operand(temp, 0), dest, dest);
+}
+
+void MacroAssembler::inc64(AbsoluteAddress dest) {
+ addl(Imm32(1), Operand(dest));
+ Label noOverflow;
+ j(NonZero, &noOverflow);
+ addl(Imm32(1), Operand(dest.offset(4)));
+ bind(&noOverflow);
+}
+
+void MacroAssembler::neg64(Register64 reg) {
+ negl(reg.low);
+ adcl(Imm32(0), reg.high);
+ negl(reg.high);
+}
+
+void MacroAssembler::negPtr(Register reg) { negl(reg); }
+
+// ===============================================================
+// Shift functions
+
+void MacroAssembler::lshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ shll(imm, dest);
+}
+
+void MacroAssembler::lshiftPtr(Register shift, Register srcDest) {
+ if (HasBMI2()) {
+ shlxl(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == ecx);
+ shll_cl(srcDest);
+}
+
+void MacroAssembler::lshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ if (imm.value < 32) {
+ shldl(imm, dest.low, dest.high);
+ shll(imm, dest.low);
+ return;
+ }
+
+ mov(dest.low, dest.high);
+ shll(Imm32(imm.value & 0x1f), dest.high);
+ xorl(dest.low, dest.low);
+}
+
+void MacroAssembler::lshift64(Register shift, Register64 srcDest) {
+ MOZ_ASSERT(shift == ecx);
+ MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
+
+ Label done;
+
+ shldl_cl(srcDest.low, srcDest.high);
+ shll_cl(srcDest.low);
+
+ testl(Imm32(0x20), ecx);
+ j(Condition::Equal, &done);
+
+ // 32 - 63 bit shift
+ movl(srcDest.low, srcDest.high);
+ xorl(srcDest.low, srcDest.low);
+
+ bind(&done);
+}
+
+void MacroAssembler::rshiftPtr(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ shrl(imm, dest);
+}
+
+void MacroAssembler::rshiftPtr(Register shift, Register srcDest) {
+ if (HasBMI2()) {
+ shrxl(srcDest, shift, srcDest);
+ return;
+ }
+ MOZ_ASSERT(shift == ecx);
+ shrl_cl(srcDest);
+}
+
+void MacroAssembler::rshift64(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ if (imm.value < 32) {
+ shrdl(imm, dest.high, dest.low);
+ shrl(imm, dest.high);
+ return;
+ }
+
+ movl(dest.high, dest.low);
+ shrl(Imm32(imm.value & 0x1f), dest.low);
+ xorl(dest.high, dest.high);
+}
+
+void MacroAssembler::rshift64(Register shift, Register64 srcDest) {
+ MOZ_ASSERT(shift == ecx);
+ MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
+
+ Label done;
+
+ shrdl_cl(srcDest.high, srcDest.low);
+ shrl_cl(srcDest.high);
+
+ testl(Imm32(0x20), ecx);
+ j(Condition::Equal, &done);
+
+ // 32 - 63 bit shift
+ movl(srcDest.high, srcDest.low);
+ xorl(srcDest.high, srcDest.high);
+
+ bind(&done);
+}
+
+void MacroAssembler::rshiftPtrArithmetic(Imm32 imm, Register dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 32);
+ sarl(imm, dest);
+}
+
+void MacroAssembler::rshift64Arithmetic(Imm32 imm, Register64 dest) {
+ MOZ_ASSERT(0 <= imm.value && imm.value < 64);
+ if (imm.value < 32) {
+ shrdl(imm, dest.high, dest.low);
+ sarl(imm, dest.high);
+ return;
+ }
+
+ movl(dest.high, dest.low);
+ sarl(Imm32(imm.value & 0x1f), dest.low);
+ sarl(Imm32(0x1f), dest.high);
+}
+
+void MacroAssembler::rshift64Arithmetic(Register shift, Register64 srcDest) {
+ MOZ_ASSERT(shift == ecx);
+ MOZ_ASSERT(srcDest.low != ecx && srcDest.high != ecx);
+
+ Label done;
+
+ shrdl_cl(srcDest.high, srcDest.low);
+ sarl_cl(srcDest.high);
+
+ testl(Imm32(0x20), ecx);
+ j(Condition::Equal, &done);
+
+ // 32 - 63 bit shift
+ movl(srcDest.high, srcDest.low);
+ sarl(Imm32(0x1f), srcDest.high);
+
+ bind(&done);
+}
+
+// ===============================================================
+// Rotation functions
+
+void MacroAssembler::rotateLeft64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+
+ Label done;
+
+ movl(dest.high, temp);
+ shldl_cl(dest.low, dest.high);
+ shldl_cl(temp, dest.low);
+
+ testl(Imm32(0x20), count);
+ j(Condition::Equal, &done);
+ xchgl(dest.high, dest.low);
+
+ bind(&done);
+}
+
+void MacroAssembler::rotateRight64(Register count, Register64 src,
+ Register64 dest, Register temp) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+ MOZ_ASSERT(count == ecx, "defineFixed(ecx)");
+
+ Label done;
+
+ movl(dest.high, temp);
+ shrdl_cl(dest.low, dest.high);
+ shrdl_cl(temp, dest.low);
+
+ testl(Imm32(0x20), count);
+ j(Condition::Equal, &done);
+ xchgl(dest.high, dest.low);
+
+ bind(&done);
+}
+
+void MacroAssembler::rotateLeft64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+
+ int32_t amount = count.value & 0x3f;
+ if ((amount & 0x1f) != 0) {
+ movl(dest.high, temp);
+ shldl(Imm32(amount & 0x1f), dest.low, dest.high);
+ shldl(Imm32(amount & 0x1f), temp, dest.low);
+ }
+
+ if (!!(amount & 0x20)) {
+ xchgl(dest.high, dest.low);
+ }
+}
+
+void MacroAssembler::rotateRight64(Imm32 count, Register64 src, Register64 dest,
+ Register temp) {
+ MOZ_ASSERT(src == dest, "defineReuseInput");
+
+ int32_t amount = count.value & 0x3f;
+ if ((amount & 0x1f) != 0) {
+ movl(dest.high, temp);
+ shrdl(Imm32(amount & 0x1f), dest.low, dest.high);
+ shrdl(Imm32(amount & 0x1f), temp, dest.low);
+ }
+
+ if (!!(amount & 0x20)) {
+ xchgl(dest.high, dest.low);
+ }
+}
+
+// ===============================================================
+// Bit counting functions
+
+void MacroAssembler::clz64(Register64 src, Register dest) {
+ if (AssemblerX86Shared::HasLZCNT()) {
+ Label nonzero, zero;
+
+ testl(src.high, src.high);
+ j(Assembler::Zero, &zero);
+
+ lzcntl(src.high, dest);
+ jump(&nonzero);
+
+ bind(&zero);
+ lzcntl(src.low, dest);
+ addl(Imm32(32), dest);
+
+ bind(&nonzero);
+ return;
+ }
+
+ // Because |dest| may be equal to |src.low|, we rely on BSR not modifying its
+ // output when the input is zero. AMD ISA documents BSR not modifying the
+ // output and current Intel CPUs follow AMD.
+
+ Label nonzero, zero;
+
+ bsrl(src.high, dest);
+ j(Assembler::Zero, &zero);
+ orl(Imm32(32), dest);
+ jump(&nonzero);
+
+ bind(&zero);
+ bsrl(src.low, dest);
+ j(Assembler::NonZero, &nonzero);
+ movl(Imm32(0x7F), dest);
+
+ bind(&nonzero);
+ xorl(Imm32(0x3F), dest);
+}
+
+void MacroAssembler::ctz64(Register64 src, Register dest) {
+ if (AssemblerX86Shared::HasBMI1()) {
+ Label nonzero, zero;
+
+ testl(src.low, src.low);
+ j(Assembler::Zero, &zero);
+
+ tzcntl(src.low, dest);
+ jump(&nonzero);
+
+ bind(&zero);
+ tzcntl(src.high, dest);
+ addl(Imm32(32), dest);
+
+ bind(&nonzero);
+ return;
+ }
+
+ // Because |dest| may be equal to |src.low|, we rely on BSF not modifying its
+ // output when the input is zero. AMD ISA documents BSF not modifying the
+ // output and current Intel CPUs follow AMD.
+
+ Label done, nonzero;
+
+ bsfl(src.low, dest);
+ j(Assembler::NonZero, &done);
+ bsfl(src.high, dest);
+ j(Assembler::NonZero, &nonzero);
+ movl(Imm32(64), dest);
+ jump(&done);
+
+ bind(&nonzero);
+ orl(Imm32(32), dest);
+
+ bind(&done);
+}
+
+void MacroAssembler::popcnt64(Register64 src, Register64 dest, Register tmp) {
+ // The tmp register is only needed if there is no native POPCNT.
+
+ MOZ_ASSERT(src.low != tmp && src.high != tmp);
+ MOZ_ASSERT(dest.low != tmp && dest.high != tmp);
+
+ if (dest.low != src.high) {
+ popcnt32(src.low, dest.low, tmp);
+ popcnt32(src.high, dest.high, tmp);
+ } else {
+ MOZ_ASSERT(dest.high != src.high);
+ popcnt32(src.low, dest.high, tmp);
+ popcnt32(src.high, dest.low, tmp);
+ }
+ addl(dest.high, dest.low);
+ xorl(dest.high, dest.high);
+}
+
+// ===============================================================
+// Condition functions
+
+void MacroAssembler::cmp64Set(Condition cond, Address lhs, Imm64 rhs,
+ Register dest) {
+ Label success, done;
+
+ branch64(cond, lhs, rhs, &success);
+ move32(Imm32(0), dest);
+ jump(&done);
+ bind(&success);
+ move32(Imm32(1), dest);
+ bind(&done);
+}
+
+template <typename T1, typename T2>
+void MacroAssembler::cmpPtrSet(Condition cond, T1 lhs, T2 rhs, Register dest) {
+ cmpPtr(lhs, rhs);
+ emitSet(cond, dest);
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ cmp32(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ cmp32(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch32(Condition cond, wasm::SymbolicAddress lhs,
+ Imm32 rhs, Label* label) {
+ cmpl(rhs, lhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Imm64 val,
+ Label* success, Label* fail) {
+ bool fallthrough = false;
+ Label fallthroughLabel;
+
+ if (!fail) {
+ fail = &fallthroughLabel;
+ fallthrough = true;
+ }
+
+ switch (cond) {
+ case Assembler::Equal:
+ branch32(Assembler::NotEqual, lhs.low, val.low(), fail);
+ branch32(Assembler::Equal, lhs.high, val.hi(), success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ case Assembler::NotEqual:
+ branch32(Assembler::NotEqual, lhs.low, val.low(), success);
+ branch32(Assembler::NotEqual, lhs.high, val.hi(), success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual: {
+ Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
+ Assembler::Condition cond2 =
+ Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
+ Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
+
+ cmp32(lhs.high, val.hi());
+ j(cond1, success);
+ j(cond2, fail);
+ cmp32(lhs.low, val.low());
+ j(cond3, success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Condition code not supported");
+ break;
+ }
+
+ if (fallthrough) {
+ bind(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, Register64 lhs, Register64 rhs,
+ Label* success, Label* fail) {
+ bool fallthrough = false;
+ Label fallthroughLabel;
+
+ if (!fail) {
+ fail = &fallthroughLabel;
+ fallthrough = true;
+ }
+
+ switch (cond) {
+ case Assembler::Equal:
+ branch32(Assembler::NotEqual, lhs.low, rhs.low, fail);
+ branch32(Assembler::Equal, lhs.high, rhs.high, success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ case Assembler::NotEqual:
+ branch32(Assembler::NotEqual, lhs.low, rhs.low, success);
+ branch32(Assembler::NotEqual, lhs.high, rhs.high, success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ case Assembler::LessThan:
+ case Assembler::LessThanOrEqual:
+ case Assembler::GreaterThan:
+ case Assembler::GreaterThanOrEqual:
+ case Assembler::Below:
+ case Assembler::BelowOrEqual:
+ case Assembler::Above:
+ case Assembler::AboveOrEqual: {
+ Assembler::Condition cond1 = Assembler::ConditionWithoutEqual(cond);
+ Assembler::Condition cond2 =
+ Assembler::ConditionWithoutEqual(Assembler::InvertCondition(cond));
+ Assembler::Condition cond3 = Assembler::UnsignedCondition(cond);
+
+ cmp32(lhs.high, rhs.high);
+ j(cond1, success);
+ j(cond2, fail);
+ cmp32(lhs.low, rhs.low);
+ j(cond3, success);
+ if (!fallthrough) {
+ jump(fail);
+ }
+ break;
+ }
+ default:
+ MOZ_CRASH("Condition code not supported");
+ break;
+ }
+
+ if (fallthrough) {
+ bind(fail);
+ }
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs, Imm64 val,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ Label done;
+
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, val.firstHalf(), &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, val.firstHalf(), label);
+ }
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)),
+ val.secondHalf(), label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ Register64 rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+
+ Label done;
+
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, rhs.low, &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, rhs.low, label);
+ }
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), rhs.high,
+ label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branch64(Condition cond, const Address& lhs,
+ const Address& rhs, Register scratch,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::NotEqual || cond == Assembler::Equal,
+ "other condition codes not supported");
+ MOZ_ASSERT(lhs.base != scratch);
+ MOZ_ASSERT(rhs.base != scratch);
+
+ Label done;
+
+ load32(rhs, scratch);
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, lhs, scratch, &done);
+ } else {
+ branch32(Assembler::NotEqual, lhs, scratch, label);
+ }
+
+ load32(Address(rhs.base, rhs.offset + sizeof(uint32_t)), scratch);
+ branch32(cond, Address(lhs.base, lhs.offset + sizeof(uint32_t)), scratch,
+ label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ Register rhs, Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, const AbsoluteAddress& lhs,
+ ImmWord rhs, Label* label) {
+ branchPtrImpl(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchPtr(Condition cond, wasm::SymbolicAddress lhs,
+ Register rhs, Label* label) {
+ cmpl(rhs, lhs);
+ j(cond, label);
+}
+
+void MacroAssembler::branchPrivatePtr(Condition cond, const Address& lhs,
+ Register rhs, Label* label) {
+ branchPtr(cond, lhs, rhs, label);
+}
+
+void MacroAssembler::branchTruncateFloat32ToPtr(FloatRegister src,
+ Register dest, Label* fail) {
+ branchTruncateFloat32ToInt32(src, dest, fail);
+}
+
+void MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ branchTruncateFloat32ToInt32(src, dest, fail);
+}
+
+void MacroAssembler::branchTruncateFloat32ToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ vcvttss2si(src, dest);
+
+ // vcvttss2si returns 0x80000000 on failure. Test for it by
+ // subtracting 1 and testing overflow (this permits the use of a
+ // smaller immediate field).
+ cmp32(dest, Imm32(1));
+ j(Assembler::Overflow, fail);
+}
+
+void MacroAssembler::branchTruncateDoubleToPtr(FloatRegister src, Register dest,
+ Label* fail) {
+ branchTruncateDoubleToInt32(src, dest, fail);
+}
+
+void MacroAssembler::branchTruncateDoubleMaybeModUint32(FloatRegister src,
+ Register dest,
+ Label* fail) {
+ // TODO: X64 supports supports integers up till 64bits. Here we only support
+ // 32bits, before failing. Implementing this for x86 might give a x86 kraken
+ // win.
+ branchTruncateDoubleToInt32(src, dest, fail);
+}
+
+void MacroAssembler::branchTruncateDoubleToInt32(FloatRegister src,
+ Register dest, Label* fail) {
+ vcvttsd2si(src, dest);
+
+ // vcvttsd2si returns 0x80000000 on failure. Test for it by
+ // subtracting 1 and testing overflow (this permits the use of a
+ // smaller immediate field).
+ cmp32(dest, Imm32(1));
+ j(Assembler::Overflow, fail);
+}
+
+void MacroAssembler::branchAdd64(Condition cond, Imm64 imm, Register64 dest,
+ Label* label) {
+ add64(imm, dest);
+ j(cond, label);
+}
+
+void MacroAssembler::branchTest32(Condition cond, const AbsoluteAddress& lhs,
+ Imm32 rhs, Label* label) {
+ test32(Operand(lhs), rhs);
+ j(cond, label);
+}
+
+template <class L>
+void MacroAssembler::branchTest64(Condition cond, Register64 lhs,
+ Register64 rhs, Register temp, L label) {
+ if (cond == Assembler::Zero || cond == Assembler::NonZero) {
+ MOZ_ASSERT(lhs.low == rhs.low);
+ MOZ_ASSERT(lhs.high == rhs.high);
+ movl(lhs.low, temp);
+ orl(lhs.high, temp);
+ branchTestPtr(cond, temp, temp, label);
+ } else if (cond == Assembler::Signed || cond == Assembler::NotSigned) {
+ branchTest32(cond, lhs.high, rhs.high, label);
+ } else {
+ MOZ_CRASH("Unsupported condition");
+ }
+}
+
+void MacroAssembler::branchTestBooleanTruthy(bool truthy,
+ const ValueOperand& value,
+ Label* label) {
+ test32(value.payloadReg(), value.payloadReg());
+ j(truthy ? NonZero : Zero, label);
+}
+
+void MacroAssembler::branchTestMagic(Condition cond, const Address& valaddr,
+ JSWhyMagic why, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label notMagic;
+ if (cond == Assembler::Equal) {
+ branchTestMagic(Assembler::NotEqual, valaddr, &notMagic);
+ } else {
+ branchTestMagic(Assembler::NotEqual, valaddr, label);
+ }
+
+ branch32(cond, ToPayload(valaddr), Imm32(why), label);
+ bind(&notMagic);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const BaseIndex& lhs,
+ const ValueOperand& rhs, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label notSameValue;
+ if (cond == Assembler::Equal) {
+ branch32(Assembler::NotEqual, ToType(lhs), rhs.typeReg(), &notSameValue);
+ } else {
+ branch32(Assembler::NotEqual, ToType(lhs), rhs.typeReg(), label);
+ }
+
+ branch32(cond, ToPayload(lhs), rhs.payloadReg(), label);
+ bind(&notSameValue);
+}
+
+void MacroAssembler::branchToComputedAddress(const BaseIndex& addr) {
+ jmp(Operand(addr));
+}
+
+void MacroAssembler::cmp32MovePtr(Condition cond, Register lhs, Imm32 rhs,
+ Register src, Register dest) {
+ cmp32(lhs, rhs);
+ cmovCCl(cond, Operand(src), dest);
+}
+
+void MacroAssembler::cmp32LoadPtr(Condition cond, const Address& lhs, Imm32 rhs,
+ const Address& src, Register dest) {
+ cmp32(lhs, rhs);
+ cmovCCl(cond, Operand(src), dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs, Register rhs,
+ Register src, Register dest) {
+ cmp32Move32(cond, lhs, rhs, src, dest);
+}
+
+void MacroAssembler::cmpPtrMovePtr(Condition cond, Register lhs,
+ const Address& rhs, Register src,
+ Register dest) {
+ cmp32Move32(cond, lhs, rhs, src, dest);
+}
+
+void MacroAssembler::test32LoadPtr(Condition cond, const Address& addr,
+ Imm32 mask, const Address& src,
+ Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ cmovCCl(cond, Operand(src), dest);
+}
+
+void MacroAssembler::test32MovePtr(Condition cond, const Address& addr,
+ Imm32 mask, Register src, Register dest) {
+ MOZ_ASSERT(cond == Assembler::Zero || cond == Assembler::NonZero);
+ test32(addr, mask);
+ cmovCCl(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreMovePtr(Condition cond, Register src,
+ Register dest) {
+ cmovCCl(cond, Operand(src), dest);
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Operand& length,
+ Register maybeScratch,
+ Label* failure) {
+ Label failurePopValue;
+ bool pushedValue = false;
+ if (JitOptions.spectreIndexMasking) {
+ if (maybeScratch == InvalidReg) {
+ push(Imm32(0));
+ pushedValue = true;
+ } else {
+ move32(Imm32(0), maybeScratch);
+ }
+ }
+
+ cmp32(index, length);
+ j(Assembler::AboveOrEqual, pushedValue ? &failurePopValue : failure);
+
+ if (JitOptions.spectreIndexMasking) {
+ if (maybeScratch == InvalidReg) {
+ Label done;
+ cmovCCl(Assembler::AboveOrEqual, Operand(StackPointer, 0), index);
+ lea(Operand(StackPointer, sizeof(void*)), StackPointer);
+ jump(&done);
+
+ bind(&failurePopValue);
+ lea(Operand(StackPointer, sizeof(void*)), StackPointer);
+ jump(failure);
+
+ bind(&done);
+ } else {
+ cmovCCl(Assembler::AboveOrEqual, maybeScratch, index);
+ }
+ }
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(length != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ spectreBoundsCheck32(index, Operand(length), maybeScratch, failure);
+}
+
+void MacroAssembler::spectreBoundsCheck32(Register index, const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ MOZ_ASSERT(index != length.base);
+ MOZ_ASSERT(length.base != maybeScratch);
+ MOZ_ASSERT(index != maybeScratch);
+
+ spectreBoundsCheck32(index, Operand(length), maybeScratch, failure);
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index, Register length,
+ Register maybeScratch,
+ Label* failure) {
+ spectreBoundsCheck32(index, length, maybeScratch, failure);
+}
+
+void MacroAssembler::spectreBoundsCheckPtr(Register index,
+ const Address& length,
+ Register maybeScratch,
+ Label* failure) {
+ spectreBoundsCheck32(index, length, maybeScratch, failure);
+}
+
+// ========================================================================
+// SIMD
+
+void MacroAssembler::extractLaneInt64x2(uint32_t lane, FloatRegister src,
+ Register64 dest) {
+ if (lane == 0) {
+ vmovd(src, dest.low);
+ } else {
+ vpextrd(2 * lane, src, dest.low);
+ }
+ vpextrd(2 * lane + 1, src, dest.high);
+}
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, Register64 rhs,
+ FloatRegister lhsDest) {
+ vpinsrd(2 * lane, rhs.low, lhsDest, lhsDest);
+ vpinsrd(2 * lane + 1, rhs.high, lhsDest, lhsDest);
+}
+
+void MacroAssembler::replaceLaneInt64x2(unsigned lane, FloatRegister lhs,
+ Register64 rhs, FloatRegister dest) {
+ vpinsrd(2 * lane, rhs.low, lhs, dest);
+ vpinsrd(2 * lane + 1, rhs.high, dest, dest);
+}
+
+void MacroAssembler::splatX2(Register64 src, FloatRegister dest) {
+ vmovd(src.low, dest);
+ vpinsrd(1, src.high, dest, dest);
+ vpunpcklqdq(dest, dest, dest);
+}
+
+// ========================================================================
+// Truncate floating point.
+
+void MacroAssembler::truncateFloat32ToUInt64(Address src, Address dest,
+ Register temp,
+ FloatRegister floatTemp) {
+ Label done;
+
+ loadFloat32(src, floatTemp);
+
+ truncateFloat32ToInt64(src, dest, temp);
+
+ // For unsigned conversion the case of [INT64, UINT64] needs to get handle
+ // seperately.
+ load32(HighWord(dest), temp);
+ branch32(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
+
+ // Move the value inside INT64 range.
+ storeFloat32(floatTemp, dest);
+ loadConstantFloat32(double(int64_t(0x8000000000000000)), floatTemp);
+ vaddss(Operand(dest), floatTemp, floatTemp);
+ storeFloat32(floatTemp, dest);
+ truncateFloat32ToInt64(dest, dest, temp);
+
+ load32(HighWord(dest), temp);
+ orl(Imm32(0x80000000), temp);
+ store32(temp, HighWord(dest));
+
+ bind(&done);
+}
+
+void MacroAssembler::truncateDoubleToUInt64(Address src, Address dest,
+ Register temp,
+ FloatRegister floatTemp) {
+ Label done;
+
+ loadDouble(src, floatTemp);
+
+ truncateDoubleToInt64(src, dest, temp);
+
+ // For unsigned conversion the case of [INT64, UINT64] needs to get handle
+ // seperately.
+ load32(HighWord(dest), temp);
+ branch32(Assembler::Condition::NotSigned, temp, Imm32(0), &done);
+
+ // Move the value inside INT64 range.
+ storeDouble(floatTemp, dest);
+ loadConstantDouble(double(int64_t(0x8000000000000000)), floatTemp);
+ vaddsd(Operand(dest), floatTemp, floatTemp);
+ storeDouble(floatTemp, dest);
+ truncateDoubleToInt64(dest, dest, temp);
+
+ load32(HighWord(dest), temp);
+ orl(Imm32(0x80000000), temp);
+ store32(temp, HighWord(dest));
+
+ bind(&done);
+}
+
+template <typename T>
+void MacroAssemblerX86::fallibleUnboxPtrImpl(const T& src, Register dest,
+ JSValueType type, Label* fail) {
+ switch (type) {
+ case JSVAL_TYPE_OBJECT:
+ asMasm().branchTestObject(Assembler::NotEqual, src, fail);
+ break;
+ case JSVAL_TYPE_STRING:
+ asMasm().branchTestString(Assembler::NotEqual, src, fail);
+ break;
+ case JSVAL_TYPE_SYMBOL:
+ asMasm().branchTestSymbol(Assembler::NotEqual, src, fail);
+ break;
+ case JSVAL_TYPE_BIGINT:
+ asMasm().branchTestBigInt(Assembler::NotEqual, src, fail);
+ break;
+ default:
+ MOZ_CRASH("Unexpected type");
+ }
+ unboxNonDouble(src, dest, type);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const ValueOperand& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(src, dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const Address& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(src, dest, type, fail);
+}
+
+void MacroAssembler::fallibleUnboxPtr(const BaseIndex& src, Register dest,
+ JSValueType type, Label* fail) {
+ fallibleUnboxPtrImpl(src, dest, type, fail);
+}
+
+//}}} check_macroassembler_style
+// ===============================================================
+
+// Note: this function clobbers the source register.
+void MacroAssemblerX86::convertUInt32ToDouble(Register src,
+ FloatRegister dest) {
+ // src is [0, 2^32-1]
+ subl(Imm32(0x80000000), src);
+
+ // Now src is [-2^31, 2^31-1] - int range, but not the same value.
+ convertInt32ToDouble(src, dest);
+
+ // dest is now a double with the int range.
+ // correct the double value by adding 0x80000000.
+ asMasm().addConstantDouble(2147483648.0, dest);
+}
+
+// Note: this function clobbers the source register.
+void MacroAssemblerX86::convertUInt32ToFloat32(Register src,
+ FloatRegister dest) {
+ convertUInt32ToDouble(src, dest);
+ convertDoubleToFloat32(dest, dest);
+}
+
+void MacroAssemblerX86::unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType) {
+ if (dest.isFloat()) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(src.payloadReg(), dest.fpu());
+ jump(&end);
+ bind(&notInt32);
+ unboxDouble(src, dest.fpu());
+ bind(&end);
+ } else {
+ if (src.payloadReg() != dest.gpr()) {
+ movl(src.payloadReg(), dest.gpr());
+ }
+ }
+}
+
+template <typename T>
+void MacroAssemblerX86::loadInt32OrDouble(const T& src, FloatRegister dest) {
+ Label notInt32, end;
+ asMasm().branchTestInt32(Assembler::NotEqual, src, &notInt32);
+ convertInt32ToDouble(ToPayload(src), dest);
+ jump(&end);
+ bind(&notInt32);
+ loadDouble(src, dest);
+ bind(&end);
+}
+
+template <typename T>
+void MacroAssemblerX86::loadUnboxedValue(const T& src, MIRType type,
+ AnyRegister dest) {
+ if (dest.isFloat()) {
+ loadInt32OrDouble(src, dest.fpu());
+ } else {
+ movl(Operand(src), dest.gpr());
+ }
+}
+
+// If source is a double, load it into dest. If source is int32,
+// convert it to double. Else, branch to failure.
+void MacroAssemblerX86::ensureDouble(const ValueOperand& source,
+ FloatRegister dest, Label* failure) {
+ Label isDouble, done;
+ asMasm().branchTestDouble(Assembler::Equal, source.typeReg(), &isDouble);
+ asMasm().branchTestInt32(Assembler::NotEqual, source.typeReg(), failure);
+
+ convertInt32ToDouble(source.payloadReg(), dest);
+ jump(&done);
+
+ bind(&isDouble);
+ unboxDouble(source, dest);
+
+ bind(&done);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_MacroAssembler_x86_inl_h */
diff --git a/js/src/jit/x86/MacroAssembler-x86.cpp b/js/src/jit/x86/MacroAssembler-x86.cpp
new file mode 100644
index 0000000000..4aeea75788
--- /dev/null
+++ b/js/src/jit/x86/MacroAssembler-x86.cpp
@@ -0,0 +1,1828 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "jit/x86/MacroAssembler-x86.h"
+
+#include "mozilla/Alignment.h"
+#include "mozilla/Casting.h"
+
+#include "jit/AtomicOp.h"
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/MacroAssembler.h"
+#include "jit/MoveEmitter.h"
+#include "util/Memory.h"
+#include "vm/BigIntType.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/StringType.h"
+
+#include "jit/MacroAssembler-inl.h"
+#include "vm/JSScript-inl.h"
+
+using namespace js;
+using namespace js::jit;
+
+void MacroAssemblerX86::loadConstantDouble(double d, FloatRegister dest) {
+ if (maybeInlineDouble(d, dest)) {
+ return;
+ }
+ Double* dbl = getDouble(d);
+ if (!dbl) {
+ return;
+ }
+ masm.vmovsd_mr(nullptr, dest.encoding());
+ propagateOOM(dbl->uses.append(CodeOffset(masm.size())));
+}
+
+void MacroAssemblerX86::loadConstantFloat32(float f, FloatRegister dest) {
+ if (maybeInlineFloat(f, dest)) {
+ return;
+ }
+ Float* flt = getFloat(f);
+ if (!flt) {
+ return;
+ }
+ masm.vmovss_mr(nullptr, dest.encoding());
+ propagateOOM(flt->uses.append(CodeOffset(masm.size())));
+}
+
+void MacroAssemblerX86::loadConstantSimd128Int(const SimdConstant& v,
+ FloatRegister dest) {
+ if (maybeInlineSimd128Int(v, dest)) {
+ return;
+ }
+ SimdData* i4 = getSimdData(v);
+ if (!i4) {
+ return;
+ }
+ masm.vmovdqa_mr(nullptr, dest.encoding());
+ propagateOOM(i4->uses.append(CodeOffset(masm.size())));
+}
+
+void MacroAssemblerX86::loadConstantSimd128Float(const SimdConstant& v,
+ FloatRegister dest) {
+ if (maybeInlineSimd128Float(v, dest)) {
+ return;
+ }
+ SimdData* f4 = getSimdData(v);
+ if (!f4) {
+ return;
+ }
+ masm.vmovaps_mr(nullptr, dest.encoding());
+ propagateOOM(f4->uses.append(CodeOffset(masm.size())));
+}
+
+void MacroAssemblerX86::vpPatchOpSimd128(
+ const SimdConstant& v, FloatRegister src, FloatRegister dest,
+ void (X86Encoding::BaseAssemblerX86::*op)(
+ const void* address, X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId)) {
+ SimdData* val = getSimdData(v);
+ if (!val) {
+ return;
+ }
+ (masm.*op)(nullptr, src.encoding(), dest.encoding());
+ propagateOOM(val->uses.append(CodeOffset(masm.size())));
+}
+
+void MacroAssemblerX86::vpPatchOpSimd128(
+ const SimdConstant& v, FloatRegister src, FloatRegister dest,
+ size_t (X86Encoding::BaseAssemblerX86::*op)(
+ const void* address, X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId)) {
+ SimdData* val = getSimdData(v);
+ if (!val) {
+ return;
+ }
+ size_t patchOffsetFromEnd =
+ (masm.*op)(nullptr, src.encoding(), dest.encoding());
+ propagateOOM(val->uses.append(CodeOffset(masm.size() - patchOffsetFromEnd)));
+}
+
+void MacroAssemblerX86::vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddb_mr);
+}
+
+void MacroAssemblerX86::vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddw_mr);
+}
+
+void MacroAssemblerX86::vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddd_mr);
+}
+
+void MacroAssemblerX86::vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddq_mr);
+}
+
+void MacroAssemblerX86::vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubb_mr);
+}
+
+void MacroAssemblerX86::vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubw_mr);
+}
+
+void MacroAssemblerX86::vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubd_mr);
+}
+
+void MacroAssemblerX86::vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubq_mr);
+}
+
+void MacroAssemblerX86::vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmullw_mr);
+}
+
+void MacroAssemblerX86::vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmulld_mr);
+}
+
+void MacroAssemblerX86::vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddsb_mr);
+}
+
+void MacroAssemblerX86::vpaddusbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddusb_mr);
+}
+
+void MacroAssemblerX86::vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddsw_mr);
+}
+
+void MacroAssemblerX86::vpadduswSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpaddusw_mr);
+}
+
+void MacroAssemblerX86::vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubsb_mr);
+}
+
+void MacroAssemblerX86::vpsubusbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubusb_mr);
+}
+
+void MacroAssemblerX86::vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubsw_mr);
+}
+
+void MacroAssemblerX86::vpsubuswSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpsubusw_mr);
+}
+
+void MacroAssemblerX86::vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminsb_mr);
+}
+
+void MacroAssemblerX86::vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminub_mr);
+}
+
+void MacroAssemblerX86::vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminsw_mr);
+}
+
+void MacroAssemblerX86::vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminuw_mr);
+}
+
+void MacroAssemblerX86::vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminsd_mr);
+}
+
+void MacroAssemblerX86::vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpminud_mr);
+}
+
+void MacroAssemblerX86::vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxsb_mr);
+}
+
+void MacroAssemblerX86::vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxub_mr);
+}
+
+void MacroAssemblerX86::vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxsw_mr);
+}
+
+void MacroAssemblerX86::vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxuw_mr);
+}
+
+void MacroAssemblerX86::vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxsd_mr);
+}
+
+void MacroAssemblerX86::vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaxud_mr);
+}
+
+void MacroAssemblerX86::vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpand_mr);
+}
+
+void MacroAssemblerX86::vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpxor_mr);
+}
+
+void MacroAssemblerX86::vporSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpor_mr);
+}
+
+void MacroAssemblerX86::vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vaddps_mr);
+}
+
+void MacroAssemblerX86::vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vaddpd_mr);
+}
+
+void MacroAssemblerX86::vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vsubps_mr);
+}
+
+void MacroAssemblerX86::vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vsubpd_mr);
+}
+
+void MacroAssemblerX86::vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vdivps_mr);
+}
+
+void MacroAssemblerX86::vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vdivpd_mr);
+}
+
+void MacroAssemblerX86::vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vmulps_mr);
+}
+
+void MacroAssemblerX86::vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vmulpd_mr);
+}
+
+void MacroAssemblerX86::vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vandpd_mr);
+}
+
+void MacroAssemblerX86::vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vminpd_mr);
+}
+
+void MacroAssemblerX86::vpacksswbSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpacksswb_mr);
+}
+
+void MacroAssemblerX86::vpackuswbSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpackuswb_mr);
+}
+
+void MacroAssemblerX86::vpackssdwSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpackssdw_mr);
+}
+
+void MacroAssemblerX86::vpackusdwSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpackusdw_mr);
+}
+
+void MacroAssemblerX86::vpunpckldqSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpunpckldq_mr);
+}
+
+void MacroAssemblerX86::vunpcklpsSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vunpcklps_mr);
+}
+
+void MacroAssemblerX86::vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpshufb_mr);
+}
+
+void MacroAssemblerX86::vptestSimd128(const SimdConstant& v,
+ FloatRegister lhs) {
+ vpPatchOpSimd128(v, lhs, &X86Encoding::BaseAssemblerX86::vptest_mr);
+}
+
+void MacroAssemblerX86::vpmaddwdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaddwd_mr);
+}
+
+void MacroAssemblerX86::vpcmpeqbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpeqb_mr);
+}
+
+void MacroAssemblerX86::vpcmpgtbSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpgtb_mr);
+}
+
+void MacroAssemblerX86::vpcmpeqwSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpeqw_mr);
+}
+
+void MacroAssemblerX86::vpcmpgtwSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpgtw_mr);
+}
+
+void MacroAssemblerX86::vpcmpeqdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpeqd_mr);
+}
+
+void MacroAssemblerX86::vpcmpgtdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpcmpgtd_mr);
+}
+
+void MacroAssemblerX86::vcmpeqpsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpeqps_mr);
+}
+
+void MacroAssemblerX86::vcmpneqpsSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpneqps_mr);
+}
+
+void MacroAssemblerX86::vcmpltpsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpltps_mr);
+}
+
+void MacroAssemblerX86::vcmplepsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpleps_mr);
+}
+
+void MacroAssemblerX86::vcmpgepsSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpgeps_mr);
+}
+
+void MacroAssemblerX86::vcmpeqpdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpeqpd_mr);
+}
+
+void MacroAssemblerX86::vcmpneqpdSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpneqpd_mr);
+}
+
+void MacroAssemblerX86::vcmpltpdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmpltpd_mr);
+}
+
+void MacroAssemblerX86::vcmplepdSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vcmplepd_mr);
+}
+
+void MacroAssemblerX86::vpmaddubswSimd128(const SimdConstant& v,
+ FloatRegister lhs,
+ FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmaddubsw_mr);
+}
+
+void MacroAssemblerX86::vpmuludqSimd128(const SimdConstant& v,
+ FloatRegister lhs, FloatRegister dest) {
+ vpPatchOpSimd128(v, lhs, dest, &X86Encoding::BaseAssemblerX86::vpmuludq_mr);
+}
+
+void MacroAssemblerX86::finish() {
+ // Last instruction may be an indirect jump so eagerly insert an undefined
+ // instruction byte to prevent processors from decoding data values into
+ // their pipelines. See Intel performance guides.
+ masm.ud2();
+
+ if (!doubles_.empty()) {
+ masm.haltingAlign(sizeof(double));
+ }
+ for (const Double& d : doubles_) {
+ CodeOffset cst(masm.currentOffset());
+ for (CodeOffset use : d.uses) {
+ addCodeLabel(CodeLabel(use, cst));
+ }
+ masm.doubleConstant(d.value);
+ if (!enoughMemory_) {
+ return;
+ }
+ }
+
+ if (!floats_.empty()) {
+ masm.haltingAlign(sizeof(float));
+ }
+ for (const Float& f : floats_) {
+ CodeOffset cst(masm.currentOffset());
+ for (CodeOffset use : f.uses) {
+ addCodeLabel(CodeLabel(use, cst));
+ }
+ masm.floatConstant(f.value);
+ if (!enoughMemory_) {
+ return;
+ }
+ }
+
+ // SIMD memory values must be suitably aligned.
+ if (!simds_.empty()) {
+ masm.haltingAlign(SimdMemoryAlignment);
+ }
+ for (const SimdData& v : simds_) {
+ CodeOffset cst(masm.currentOffset());
+ for (CodeOffset use : v.uses) {
+ addCodeLabel(CodeLabel(use, cst));
+ }
+ masm.simd128Constant(v.value.bytes());
+ if (!enoughMemory_) {
+ return;
+ }
+ }
+}
+
+void MacroAssemblerX86::handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail) {
+ // Reserve space for exception information.
+ subl(Imm32(sizeof(ResumeFromException)), esp);
+ movl(esp, eax);
+
+ // Call the handler.
+ using Fn = void (*)(ResumeFromException * rfe);
+ asMasm().setupUnalignedABICall(ecx);
+ asMasm().passABIArg(eax);
+ asMasm().callWithABI<Fn, HandleException>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ Label entryFrame;
+ Label catch_;
+ Label finally;
+ Label returnBaseline;
+ Label returnIon;
+ Label bailout;
+ Label wasm;
+ Label wasmCatch;
+
+ loadPtr(Address(esp, ResumeFromException::offsetOfKind()), eax);
+ asMasm().branch32(Assembler::Equal, eax,
+ Imm32(ExceptionResumeKind::EntryFrame), &entryFrame);
+ asMasm().branch32(Assembler::Equal, eax, Imm32(ExceptionResumeKind::Catch),
+ &catch_);
+ asMasm().branch32(Assembler::Equal, eax, Imm32(ExceptionResumeKind::Finally),
+ &finally);
+ asMasm().branch32(Assembler::Equal, eax,
+ Imm32(ExceptionResumeKind::ForcedReturnBaseline),
+ &returnBaseline);
+ asMasm().branch32(Assembler::Equal, eax,
+ Imm32(ExceptionResumeKind::ForcedReturnIon), &returnIon);
+ asMasm().branch32(Assembler::Equal, eax, Imm32(ExceptionResumeKind::Bailout),
+ &bailout);
+ asMasm().branch32(Assembler::Equal, eax, Imm32(ExceptionResumeKind::Wasm),
+ &wasm);
+ asMasm().branch32(Assembler::Equal, eax,
+ Imm32(ExceptionResumeKind::WasmCatch), &wasmCatch);
+
+ breakpoint(); // Invalid kind.
+
+ // No exception handler. Load the error value, restore state and return from
+ // the entry frame.
+ bind(&entryFrame);
+ asMasm().moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp);
+ loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+ ret();
+
+ // If we found a catch handler, this must be a baseline frame. Restore state
+ // and jump to the catch block.
+ bind(&catch_);
+ loadPtr(Address(esp, ResumeFromException::offsetOfTarget()), eax);
+ loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp);
+ loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+ jmp(Operand(eax));
+
+ // If we found a finally block, this must be a baseline frame. Push two
+ // values expected by the finally block: the exception and BooleanValue(true).
+ bind(&finally);
+ ValueOperand exception = ValueOperand(ecx, edx);
+ loadValue(Address(esp, ResumeFromException::offsetOfException()), exception);
+
+ loadPtr(Address(esp, ResumeFromException::offsetOfTarget()), eax);
+ loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp);
+ loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+
+ pushValue(exception);
+ pushValue(BooleanValue(true));
+ jmp(Operand(eax));
+
+ // Return BaselineFrame->returnValue() to the caller.
+ // Used in debug mode and for GeneratorReturn.
+ Label profilingInstrumentation;
+ bind(&returnBaseline);
+ loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp);
+ loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+ loadValue(Address(ebp, BaselineFrame::reverseOffsetOfReturnValue()),
+ JSReturnOperand);
+ jump(&profilingInstrumentation);
+
+ // Return the given value to the caller.
+ bind(&returnIon);
+ loadValue(Address(esp, ResumeFromException::offsetOfException()),
+ JSReturnOperand);
+ loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp);
+ loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+
+ // If profiling is enabled, then update the lastProfilingFrame to refer to
+ // caller frame before returning. This code is shared by ForcedReturnIon
+ // and ForcedReturnBaseline.
+ bind(&profilingInstrumentation);
+ {
+ Label skipProfilingInstrumentation;
+ // Test if profiler enabled.
+ AbsoluteAddress addressOfEnabled(
+ asMasm().runtime()->geckoProfiler().addressOfEnabled());
+ asMasm().branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ jump(profilerExitTail);
+ bind(&skipProfilingInstrumentation);
+ }
+
+ movl(ebp, esp);
+ pop(ebp);
+ ret();
+
+ // If we are bailing out to baseline to handle an exception, jump to the
+ // bailout tail stub. Load 1 (true) in ReturnReg to indicate success.
+ bind(&bailout);
+ loadPtr(Address(esp, ResumeFromException::offsetOfBailoutInfo()), ecx);
+ loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+ move32(Imm32(1), ReturnReg);
+ jump(bailoutTail);
+
+ // If we are throwing and the innermost frame was a wasm frame, reset SP and
+ // FP; SP is pointing to the unwound return address to the wasm entry, so
+ // we can just ret().
+ bind(&wasm);
+ loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp);
+ loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+ masm.ret();
+
+ // Found a wasm catch handler, restore state and jump to it.
+ bind(&wasmCatch);
+ loadPtr(Address(esp, ResumeFromException::offsetOfTarget()), eax);
+ loadPtr(Address(esp, ResumeFromException::offsetOfFramePointer()), ebp);
+ loadPtr(Address(esp, ResumeFromException::offsetOfStackPointer()), esp);
+ jmp(Operand(eax));
+}
+
+void MacroAssemblerX86::profilerEnterFrame(Register framePtr,
+ Register scratch) {
+ asMasm().loadJSContext(scratch);
+ loadPtr(Address(scratch, offsetof(JSContext, profilingActivation_)), scratch);
+ storePtr(framePtr,
+ Address(scratch, JitActivation::offsetOfLastProfilingFrame()));
+ storePtr(ImmPtr(nullptr),
+ Address(scratch, JitActivation::offsetOfLastProfilingCallSite()));
+}
+
+void MacroAssemblerX86::profilerExitFrame() {
+ jump(asMasm().runtime()->jitRuntime()->getProfilerExitFrameTail());
+}
+
+Assembler::Condition MacroAssemblerX86::testStringTruthy(
+ bool truthy, const ValueOperand& value) {
+ Register string = value.payloadReg();
+ cmp32(Operand(string, JSString::offsetOfLength()), Imm32(0));
+ return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+Assembler::Condition MacroAssemblerX86::testBigIntTruthy(
+ bool truthy, const ValueOperand& value) {
+ Register bi = value.payloadReg();
+ cmp32(Operand(bi, JS::BigInt::offsetOfDigitLength()), Imm32(0));
+ return truthy ? Assembler::NotEqual : Assembler::Equal;
+}
+
+MacroAssembler& MacroAssemblerX86::asMasm() {
+ return *static_cast<MacroAssembler*>(this);
+}
+
+const MacroAssembler& MacroAssemblerX86::asMasm() const {
+ return *static_cast<const MacroAssembler*>(this);
+}
+
+void MacroAssembler::subFromStackPtr(Imm32 imm32) {
+ if (imm32.value) {
+ // On windows, we cannot skip very far down the stack without touching the
+ // memory pages in-between. This is a corner-case code for situations where
+ // the Ion frame data for a piece of code is very large. To handle this
+ // special case, for frames over 4k in size we allocate memory on the stack
+ // incrementally, touching it as we go.
+ //
+ // When the amount is quite large, which it can be, we emit an actual loop,
+ // in order to keep the function prologue compact. Compactness is a
+ // requirement for eg Wasm's CodeRange data structure, which can encode only
+ // 8-bit offsets.
+ uint32_t amountLeft = imm32.value;
+ uint32_t fullPages = amountLeft / 4096;
+ if (fullPages <= 8) {
+ while (amountLeft > 4096) {
+ subl(Imm32(4096), StackPointer);
+ store32(Imm32(0), Address(StackPointer, 0));
+ amountLeft -= 4096;
+ }
+ subl(Imm32(amountLeft), StackPointer);
+ } else {
+ // Save scratch register.
+ push(eax);
+ amountLeft -= 4;
+ fullPages = amountLeft / 4096;
+
+ Label top;
+ move32(Imm32(fullPages), eax);
+ bind(&top);
+ subl(Imm32(4096), StackPointer);
+ store32(Imm32(0), Address(StackPointer, 0));
+ subl(Imm32(1), eax);
+ j(Assembler::NonZero, &top);
+ amountLeft -= fullPages * 4096;
+ if (amountLeft) {
+ subl(Imm32(amountLeft), StackPointer);
+ }
+
+ // Restore scratch register.
+ movl(Operand(StackPointer, uint32_t(imm32.value) - 4), eax);
+ }
+ }
+}
+
+//{{{ check_macroassembler_style
+// ===============================================================
+// ABI function calls.
+
+void MacroAssembler::setupUnalignedABICall(Register scratch) {
+ setupNativeABICall();
+ dynamicAlignment_ = true;
+
+ movl(esp, scratch);
+ andl(Imm32(~(ABIStackAlignment - 1)), esp);
+ push(scratch);
+}
+
+void MacroAssembler::callWithABIPre(uint32_t* stackAdjust, bool callFromWasm) {
+ MOZ_ASSERT(inCall_);
+ uint32_t stackForCall = abiArgs_.stackBytesConsumedSoFar();
+
+ if (dynamicAlignment_) {
+ // sizeof(intptr_t) accounts for the saved stack pointer pushed by
+ // setupUnalignedABICall.
+ stackForCall += ComputeByteAlignment(stackForCall + sizeof(intptr_t),
+ ABIStackAlignment);
+ } else {
+ uint32_t alignmentAtPrologue = callFromWasm ? sizeof(wasm::Frame) : 0;
+ stackForCall += ComputeByteAlignment(
+ stackForCall + framePushed() + alignmentAtPrologue, ABIStackAlignment);
+ }
+
+ *stackAdjust = stackForCall;
+ reserveStack(stackForCall);
+
+ // Position all arguments.
+ {
+ enoughMemory_ &= moveResolver_.resolve();
+ if (!enoughMemory_) {
+ return;
+ }
+
+ MoveEmitter emitter(*this);
+ emitter.emit(moveResolver_);
+ emitter.finish();
+ }
+
+ assertStackAlignment(ABIStackAlignment);
+}
+
+void MacroAssembler::callWithABIPost(uint32_t stackAdjust, MoveOp::Type result,
+ bool callFromWasm) {
+ freeStack(stackAdjust);
+
+ // Calls to native functions in wasm pass through a thunk which already
+ // fixes up the return value for us.
+ if (!callFromWasm) {
+ if (result == MoveOp::DOUBLE) {
+ reserveStack(sizeof(double));
+ fstp(Operand(esp, 0));
+ loadDouble(Operand(esp, 0), ReturnDoubleReg);
+ freeStack(sizeof(double));
+ } else if (result == MoveOp::FLOAT32) {
+ reserveStack(sizeof(float));
+ fstp32(Operand(esp, 0));
+ loadFloat32(Operand(esp, 0), ReturnFloat32Reg);
+ freeStack(sizeof(float));
+ }
+ }
+
+ if (dynamicAlignment_) {
+ pop(esp);
+ }
+
+#ifdef DEBUG
+ MOZ_ASSERT(inCall_);
+ inCall_ = false;
+#endif
+}
+
+void MacroAssembler::callWithABINoProfiler(Register fun, MoveOp::Type result) {
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(fun);
+ callWithABIPost(stackAdjust, result);
+}
+
+void MacroAssembler::callWithABINoProfiler(const Address& fun,
+ MoveOp::Type result) {
+ uint32_t stackAdjust;
+ callWithABIPre(&stackAdjust);
+ call(fun);
+ callWithABIPost(stackAdjust, result);
+}
+
+// ===============================================================
+// Move instructions
+
+void MacroAssembler::moveValue(const TypedOrValueRegister& src,
+ const ValueOperand& dest) {
+ if (src.hasValue()) {
+ moveValue(src.valueReg(), dest);
+ return;
+ }
+
+ MIRType type = src.type();
+ AnyRegister reg = src.typedReg();
+
+ if (!IsFloatingPointType(type)) {
+ if (reg.gpr() != dest.payloadReg()) {
+ movl(reg.gpr(), dest.payloadReg());
+ }
+ mov(ImmWord(MIRTypeToTag(type)), dest.typeReg());
+ return;
+ }
+
+ ScratchDoubleScope scratch(*this);
+ FloatRegister freg = reg.fpu();
+ if (type == MIRType::Float32) {
+ convertFloat32ToDouble(freg, scratch);
+ freg = scratch;
+ }
+ boxDouble(freg, dest, scratch);
+}
+
+void MacroAssembler::moveValue(const ValueOperand& src,
+ const ValueOperand& dest) {
+ Register s0 = src.typeReg();
+ Register s1 = src.payloadReg();
+ Register d0 = dest.typeReg();
+ Register d1 = dest.payloadReg();
+
+ // Either one or both of the source registers could be the same as a
+ // destination register.
+ if (s1 == d0) {
+ if (s0 == d1) {
+ // If both are, this is just a swap of two registers.
+ xchgl(d0, d1);
+ return;
+ }
+ // If only one is, copy that source first.
+ std::swap(s0, s1);
+ std::swap(d0, d1);
+ }
+
+ if (s0 != d0) {
+ movl(s0, d0);
+ }
+ if (s1 != d1) {
+ movl(s1, d1);
+ }
+}
+
+void MacroAssembler::moveValue(const Value& src, const ValueOperand& dest) {
+ movl(Imm32(src.toNunboxTag()), dest.typeReg());
+ if (src.isGCThing()) {
+ movl(ImmGCPtr(src.toGCThing()), dest.payloadReg());
+ } else {
+ movl(Imm32(src.toNunboxPayload()), dest.payloadReg());
+ }
+}
+
+// ===============================================================
+// Branch functions
+
+void MacroAssembler::loadStoreBuffer(Register ptr, Register buffer) {
+ if (ptr != buffer) {
+ movePtr(ptr, buffer);
+ }
+ andPtr(Imm32(~gc::ChunkMask), buffer);
+ loadPtr(Address(buffer, gc::ChunkStoreBufferOffset), buffer);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond, Register ptr,
+ Register temp, Label* label) {
+ MOZ_ASSERT(temp != InvalidReg); // A temp register is required for x86.
+ MOZ_ASSERT(ptr != temp);
+ movePtr(ptr, temp);
+ branchPtrInNurseryChunkImpl(cond, temp, label);
+}
+
+void MacroAssembler::branchPtrInNurseryChunk(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ MOZ_ASSERT(temp != InvalidReg); // A temp register is required for x86.
+ loadPtr(address, temp);
+ branchPtrInNurseryChunkImpl(cond, temp, label);
+}
+
+void MacroAssembler::branchPtrInNurseryChunkImpl(Condition cond, Register ptr,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ andPtr(Imm32(~gc::ChunkMask), ptr);
+ branchPtr(InvertCondition(cond), Address(ptr, gc::ChunkStoreBufferOffset),
+ ImmWord(0), label);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ const Address& address,
+ Register temp, Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label done;
+
+ branchTestGCThing(Assembler::NotEqual, address,
+ cond == Assembler::Equal ? &done : label);
+ branchPtrInNurseryChunk(cond, ToPayload(address), temp, label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchValueIsNurseryCell(Condition cond,
+ ValueOperand value, Register temp,
+ Label* label) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+
+ Label done;
+
+ branchTestGCThing(Assembler::NotEqual, value,
+ cond == Assembler::Equal ? &done : label);
+ branchPtrInNurseryChunk(cond, value.payloadReg(), temp, label);
+
+ bind(&done);
+}
+
+void MacroAssembler::branchTestValue(Condition cond, const ValueOperand& lhs,
+ const Value& rhs, Label* label) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ if (rhs.isGCThing()) {
+ cmpPtr(lhs.payloadReg(), ImmGCPtr(rhs.toGCThing()));
+ } else {
+ cmpPtr(lhs.payloadReg(), ImmWord(rhs.toNunboxPayload()));
+ }
+
+ if (cond == Equal) {
+ Label done;
+ j(NotEqual, &done);
+ {
+ cmp32(lhs.typeReg(), Imm32(rhs.toNunboxTag()));
+ j(Equal, label);
+ }
+ bind(&done);
+ } else {
+ j(NotEqual, label);
+
+ cmp32(lhs.typeReg(), Imm32(rhs.toNunboxTag()));
+ j(NotEqual, label);
+ }
+}
+
+// ========================================================================
+// Memory access primitives.
+template <typename T>
+void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType, const T& dest) {
+ MOZ_ASSERT(valueType < MIRType::Value);
+
+ if (valueType == MIRType::Double) {
+ storeDouble(value.reg().typedReg().fpu(), dest);
+ return;
+ }
+
+ // Store the type tag.
+ storeTypeTag(ImmType(ValueTypeFromMIRType(valueType)), Operand(dest));
+
+ // Store the payload.
+ if (value.constant()) {
+ storePayload(value.value(), Operand(dest));
+ } else {
+ storePayload(value.reg().typedReg().gpr(), Operand(dest));
+ }
+}
+
+template void MacroAssembler::storeUnboxedValue(const ConstantOrRegister& value,
+ MIRType valueType,
+ const Address& dest);
+template void MacroAssembler::storeUnboxedValue(
+ const ConstantOrRegister& value, MIRType valueType,
+ const BaseObjectElementIndex& dest);
+
+// wasm specific methods, used in both the wasm baseline compiler and ion.
+
+void MacroAssembler::wasmLoad(const wasm::MemoryAccessDesc& access,
+ Operand srcAddr, AnyRegister out) {
+ MOZ_ASSERT(srcAddr.kind() == Operand::MEM_REG_DISP ||
+ srcAddr.kind() == Operand::MEM_SCALE);
+
+ MOZ_ASSERT_IF(
+ access.isZeroExtendSimd128Load(),
+ access.type() == Scalar::Float32 || access.type() == Scalar::Float64);
+ MOZ_ASSERT_IF(
+ access.isSplatSimd128Load(),
+ access.type() == Scalar::Uint8 || access.type() == Scalar::Uint16 ||
+ access.type() == Scalar::Float32 || access.type() == Scalar::Float64);
+ MOZ_ASSERT_IF(access.isWidenSimd128Load(), access.type() == Scalar::Float64);
+
+ // NOTE: the generated code must match the assembly code in gen_load in
+ // GenerateAtomicOperations.py
+ memoryBarrierBefore(access.sync());
+
+ append(access, size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ movsbl(srcAddr, out.gpr());
+ break;
+ case Scalar::Uint8:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastb(srcAddr, out.fpu());
+ } else {
+ movzbl(srcAddr, out.gpr());
+ }
+ break;
+ case Scalar::Int16:
+ movswl(srcAddr, out.gpr());
+ break;
+ case Scalar::Uint16:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastw(srcAddr, out.fpu());
+ } else {
+ movzwl(srcAddr, out.gpr());
+ }
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ movl(srcAddr, out.gpr());
+ break;
+ case Scalar::Float32:
+ if (access.isSplatSimd128Load()) {
+ vbroadcastss(srcAddr, out.fpu());
+ } else {
+ // vmovss does the right thing also for access.isZeroExtendSimd128Load()
+ vmovss(srcAddr, out.fpu());
+ }
+ break;
+ case Scalar::Float64:
+ if (access.isSplatSimd128Load()) {
+ vmovddup(srcAddr, out.fpu());
+ } else if (access.isWidenSimd128Load()) {
+ switch (access.widenSimdOp()) {
+ case wasm::SimdOp::V128Load8x8S:
+ vpmovsxbw(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load8x8U:
+ vpmovzxbw(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load16x4S:
+ vpmovsxwd(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load16x4U:
+ vpmovzxwd(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load32x2S:
+ vpmovsxdq(srcAddr, out.fpu());
+ break;
+ case wasm::SimdOp::V128Load32x2U:
+ vpmovzxdq(srcAddr, out.fpu());
+ break;
+ default:
+ MOZ_CRASH("Unexpected widening op for wasmLoad");
+ }
+ } else {
+ // vmovsd does the right thing also for access.isZeroExtendSimd128Load()
+ vmovsd(srcAddr, out.fpu());
+ }
+ break;
+ case Scalar::Simd128:
+ vmovups(srcAddr, out.fpu());
+ break;
+ case Scalar::Int64:
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected type");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmLoadI64(const wasm::MemoryAccessDesc& access,
+ Operand srcAddr, Register64 out) {
+ // Atomic i64 load must use lock_cmpxchg8b.
+ MOZ_ASSERT_IF(access.isAtomic(), access.byteSize() <= 4);
+ MOZ_ASSERT(srcAddr.kind() == Operand::MEM_REG_DISP ||
+ srcAddr.kind() == Operand::MEM_SCALE);
+ MOZ_ASSERT(!access.isZeroExtendSimd128Load()); // Use wasmLoad()
+ MOZ_ASSERT(!access.isSplatSimd128Load()); // Use wasmLoad()
+ MOZ_ASSERT(!access.isWidenSimd128Load()); // Use wasmLoad()
+
+ memoryBarrierBefore(access.sync());
+
+ append(access, size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ MOZ_ASSERT(out == Register64(edx, eax));
+ movsbl(srcAddr, out.low);
+
+ cdq();
+ break;
+ case Scalar::Uint8:
+ movzbl(srcAddr, out.low);
+
+ xorl(out.high, out.high);
+ break;
+ case Scalar::Int16:
+ MOZ_ASSERT(out == Register64(edx, eax));
+ movswl(srcAddr, out.low);
+
+ cdq();
+ break;
+ case Scalar::Uint16:
+ movzwl(srcAddr, out.low);
+
+ xorl(out.high, out.high);
+ break;
+ case Scalar::Int32:
+ MOZ_ASSERT(out == Register64(edx, eax));
+ movl(srcAddr, out.low);
+
+ cdq();
+ break;
+ case Scalar::Uint32:
+ movl(srcAddr, out.low);
+
+ xorl(out.high, out.high);
+ break;
+ case Scalar::Int64: {
+ if (srcAddr.kind() == Operand::MEM_SCALE) {
+ MOZ_RELEASE_ASSERT(srcAddr.toBaseIndex().base != out.low &&
+ srcAddr.toBaseIndex().index != out.low);
+ }
+ if (srcAddr.kind() == Operand::MEM_REG_DISP) {
+ MOZ_RELEASE_ASSERT(srcAddr.toAddress().base != out.low);
+ }
+
+ movl(LowWord(srcAddr), out.low);
+
+ append(access, size());
+ movl(HighWord(srcAddr), out.high);
+
+ break;
+ }
+ case Scalar::Float32:
+ case Scalar::Float64:
+ MOZ_CRASH("non-int64 loads should use load()");
+ case Scalar::Simd128:
+ case Scalar::Uint8Clamped:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ case Scalar::MaxTypedArrayViewType:
+ MOZ_CRASH("unexpected array type");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmStore(const wasm::MemoryAccessDesc& access,
+ AnyRegister value, Operand dstAddr) {
+ MOZ_ASSERT(dstAddr.kind() == Operand::MEM_REG_DISP ||
+ dstAddr.kind() == Operand::MEM_SCALE);
+
+ // NOTE: the generated code must match the assembly code in gen_store in
+ // GenerateAtomicOperations.py
+ memoryBarrierBefore(access.sync());
+
+ append(access, size());
+ switch (access.type()) {
+ case Scalar::Int8:
+ case Scalar::Uint8Clamped:
+ case Scalar::Uint8:
+ movb(value.gpr(), dstAddr);
+ break;
+ case Scalar::Int16:
+ case Scalar::Uint16:
+ movw(value.gpr(), dstAddr);
+ break;
+ case Scalar::Int32:
+ case Scalar::Uint32:
+ movl(value.gpr(), dstAddr);
+ break;
+ case Scalar::Float32:
+ vmovss(value.fpu(), dstAddr);
+ break;
+ case Scalar::Float64:
+ vmovsd(value.fpu(), dstAddr);
+ break;
+ case Scalar::Simd128:
+ vmovups(value.fpu(), dstAddr);
+ break;
+ case Scalar::Int64:
+ MOZ_CRASH("Should be handled in storeI64.");
+ case Scalar::MaxTypedArrayViewType:
+ case Scalar::BigInt64:
+ case Scalar::BigUint64:
+ MOZ_CRASH("unexpected type");
+ }
+
+ memoryBarrierAfter(access.sync());
+}
+
+void MacroAssembler::wasmStoreI64(const wasm::MemoryAccessDesc& access,
+ Register64 value, Operand dstAddr) {
+ // Atomic i64 store must use lock_cmpxchg8b.
+ MOZ_ASSERT(!access.isAtomic());
+ MOZ_ASSERT(dstAddr.kind() == Operand::MEM_REG_DISP ||
+ dstAddr.kind() == Operand::MEM_SCALE);
+
+ // Store the high word first so as to hit guard-page-based OOB checks without
+ // writing partial data.
+ append(access, size());
+ movl(value.high, HighWord(dstAddr));
+
+ append(access, size());
+ movl(value.low, LowWord(dstAddr));
+}
+
+template <typename T>
+static void AtomicLoad64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access, const T& address,
+ Register64 temp, Register64 output) {
+ MOZ_ASSERT(temp.low == ebx);
+ MOZ_ASSERT(temp.high == ecx);
+ MOZ_ASSERT(output.high == edx);
+ MOZ_ASSERT(output.low == eax);
+
+ // In the event edx:eax matches what's in memory, ecx:ebx will be
+ // stored. The two pairs must therefore have the same values.
+ masm.movl(edx, ecx);
+ masm.movl(eax, ebx);
+
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(address));
+}
+
+void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicLoad64(*this, &access, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicLoad64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicLoad64(*this, &access, mem, temp, output);
+}
+
+template <typename T>
+static void CompareExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access,
+ const T& mem, Register64 expected,
+ Register64 replacement, Register64 output) {
+ MOZ_ASSERT(expected == output);
+ MOZ_ASSERT(expected.high == edx);
+ MOZ_ASSERT(expected.low == eax);
+ MOZ_ASSERT(replacement.high == ecx);
+ MOZ_ASSERT(replacement.low == ebx);
+
+ // NOTE: the generated code must match the assembly code in gen_cmpxchg in
+ // GenerateAtomicOperations.py
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem));
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ CompareExchange64(*this, &access, mem, expected, replacement, output);
+}
+
+void MacroAssembler::wasmCompareExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ CompareExchange64(*this, &access, mem, expected, replacement, output);
+}
+
+template <typename T>
+static void AtomicExchange64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access, const T& mem,
+ Register64 value, Register64 output) {
+ MOZ_ASSERT(value.low == ebx);
+ MOZ_ASSERT(value.high == ecx);
+ MOZ_ASSERT(output.high == edx);
+ MOZ_ASSERT(output.low == eax);
+
+ // edx:eax has garbage initially, and that is the best we can do unless
+ // we can guess with high probability what's in memory.
+
+ MOZ_ASSERT(mem.base != edx && mem.base != eax);
+ if constexpr (std::is_same_v<T, BaseIndex>) {
+ MOZ_ASSERT(mem.index != edx && mem.index != eax);
+ } else {
+ static_assert(std::is_same_v<T, Address>);
+ }
+
+ Label again;
+ masm.bind(&again);
+ if (access) {
+ masm.append(*access, masm.size());
+ }
+ masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem));
+ masm.j(MacroAssembler::NonZero, &again);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, &access, mem, value, output);
+}
+
+void MacroAssembler::wasmAtomicExchange64(const wasm::MemoryAccessDesc& access,
+ const BaseIndex& mem,
+ Register64 value, Register64 output) {
+ AtomicExchange64(*this, &access, mem, value, output);
+}
+
+template <typename T>
+static void AtomicFetchOp64(MacroAssembler& masm,
+ const wasm::MemoryAccessDesc* access, AtomicOp op,
+ const Address& value, const T& mem, Register64 temp,
+ Register64 output) {
+ // We don't have enough registers for all the operands on x86, so the rhs
+ // operand is in memory.
+
+#define ATOMIC_OP_BODY(OPERATE) \
+ do { \
+ MOZ_ASSERT(output.low == eax); \
+ MOZ_ASSERT(output.high == edx); \
+ MOZ_ASSERT(temp.low == ebx); \
+ MOZ_ASSERT(temp.high == ecx); \
+ if (access) { \
+ masm.append(*access, masm.size()); \
+ } \
+ masm.load64(mem, output); \
+ Label again; \
+ masm.bind(&again); \
+ masm.move64(output, temp); \
+ masm.OPERATE(Operand(value), temp); \
+ masm.lock_cmpxchg8b(edx, eax, ecx, ebx, Operand(mem)); \
+ masm.j(MacroAssembler::NonZero, &again); \
+ } while (0)
+
+ switch (op) {
+ case AtomicFetchAddOp:
+ ATOMIC_OP_BODY(add64FromMemory);
+ break;
+ case AtomicFetchSubOp:
+ ATOMIC_OP_BODY(sub64FromMemory);
+ break;
+ case AtomicFetchAndOp:
+ ATOMIC_OP_BODY(and64FromMemory);
+ break;
+ case AtomicFetchOrOp:
+ ATOMIC_OP_BODY(or64FromMemory);
+ break;
+ case AtomicFetchXorOp:
+ ATOMIC_OP_BODY(xor64FromMemory);
+ break;
+ default:
+ MOZ_CRASH();
+ }
+
+#undef ATOMIC_OP_BODY
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, const Address& value,
+ const Address& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmAtomicFetchOp64(const wasm::MemoryAccessDesc& access,
+ AtomicOp op, const Address& value,
+ const BaseIndex& mem, Register64 temp,
+ Register64 output) {
+ AtomicFetchOp64(*this, &access, op, value, mem, temp, output);
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ Label done;
+ vcvttsd2si(input, output);
+ branch32(Assembler::Condition::NotSigned, output, Imm32(0), &done);
+
+ ScratchDoubleScope fpscratch(*this);
+ loadConstantDouble(double(int32_t(0x80000000)), fpscratch);
+ addDouble(input, fpscratch);
+ vcvttsd2si(fpscratch, output);
+
+ branch32(Assembler::Condition::Signed, output, Imm32(0), oolEntry);
+ or32(Imm32(0x80000000), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt32(FloatRegister input,
+ Register output,
+ bool isSaturating,
+ Label* oolEntry) {
+ Label done;
+ vcvttss2si(input, output);
+ branch32(Assembler::Condition::NotSigned, output, Imm32(0), &done);
+
+ ScratchFloat32Scope fpscratch(*this);
+ loadConstantFloat32(float(int32_t(0x80000000)), fpscratch);
+ addFloat32(input, fpscratch);
+ vcvttss2si(fpscratch, output);
+
+ branch32(Assembler::Condition::Signed, output, Imm32(0), oolEntry);
+ or32(Imm32(0x80000000), output);
+
+ bind(&done);
+}
+
+void MacroAssembler::wasmTruncateDoubleToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ Label ok;
+ Register temp = output.high;
+
+ reserveStack(2 * sizeof(int32_t));
+ storeDouble(input, Operand(esp, 0));
+
+ truncateDoubleToInt64(Address(esp, 0), Address(esp, 0), temp);
+ load64(Address(esp, 0), output);
+
+ cmpl(Imm32(0), Operand(esp, 0));
+ j(Assembler::NotEqual, &ok);
+
+ cmpl(Imm32(1), Operand(esp, 4));
+ j(Assembler::Overflow, oolEntry);
+
+ bind(&ok);
+ bind(oolRejoin);
+
+ freeStack(2 * sizeof(int32_t));
+}
+
+void MacroAssembler::wasmTruncateFloat32ToInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ Label ok;
+ Register temp = output.high;
+
+ reserveStack(2 * sizeof(int32_t));
+ storeFloat32(input, Operand(esp, 0));
+
+ truncateFloat32ToInt64(Address(esp, 0), Address(esp, 0), temp);
+ load64(Address(esp, 0), output);
+
+ cmpl(Imm32(0), Operand(esp, 0));
+ j(Assembler::NotEqual, &ok);
+
+ cmpl(Imm32(1), Operand(esp, 4));
+ j(Assembler::Overflow, oolEntry);
+
+ bind(&ok);
+ bind(oolRejoin);
+
+ freeStack(2 * sizeof(int32_t));
+}
+
+void MacroAssembler::wasmTruncateDoubleToUInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ Label fail, convert;
+ Register temp = output.high;
+
+ // Make sure input fits in uint64.
+ reserveStack(2 * sizeof(int32_t));
+ storeDouble(input, Operand(esp, 0));
+ branchDoubleNotInUInt64Range(Address(esp, 0), temp, &fail);
+ size_t stackBeforeBranch = framePushed();
+ jump(&convert);
+
+ bind(&fail);
+ freeStack(2 * sizeof(int32_t));
+ jump(oolEntry);
+ if (isSaturating) {
+ // The OOL path computes the right values.
+ setFramePushed(stackBeforeBranch);
+ } else {
+ // The OOL path just checks the input values.
+ bind(oolRejoin);
+ reserveStack(2 * sizeof(int32_t));
+ storeDouble(input, Operand(esp, 0));
+ }
+
+ // Convert the double/float to uint64.
+ bind(&convert);
+ truncateDoubleToUInt64(Address(esp, 0), Address(esp, 0), temp, tempReg);
+
+ // Load value into int64 register.
+ load64(Address(esp, 0), output);
+ freeStack(2 * sizeof(int32_t));
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+void MacroAssembler::wasmTruncateFloat32ToUInt64(
+ FloatRegister input, Register64 output, bool isSaturating, Label* oolEntry,
+ Label* oolRejoin, FloatRegister tempReg) {
+ Label fail, convert;
+ Register temp = output.high;
+
+ // Make sure input fits in uint64.
+ reserveStack(2 * sizeof(int32_t));
+ storeFloat32(input, Operand(esp, 0));
+ branchFloat32NotInUInt64Range(Address(esp, 0), temp, &fail);
+ size_t stackBeforeBranch = framePushed();
+ jump(&convert);
+
+ bind(&fail);
+ freeStack(2 * sizeof(int32_t));
+ jump(oolEntry);
+ if (isSaturating) {
+ // The OOL path computes the right values.
+ setFramePushed(stackBeforeBranch);
+ } else {
+ // The OOL path just checks the input values.
+ bind(oolRejoin);
+ reserveStack(2 * sizeof(int32_t));
+ storeFloat32(input, Operand(esp, 0));
+ }
+
+ // Convert the float to uint64.
+ bind(&convert);
+ truncateFloat32ToUInt64(Address(esp, 0), Address(esp, 0), temp, tempReg);
+
+ // Load value into int64 register.
+ load64(Address(esp, 0), output);
+ freeStack(2 * sizeof(int32_t));
+
+ if (isSaturating) {
+ bind(oolRejoin);
+ }
+}
+
+// ========================================================================
+// Primitive atomic operations.
+
+void MacroAssembler::atomicLoad64(const Synchronization&, const Address& mem,
+ Register64 temp, Register64 output) {
+ AtomicLoad64(*this, nullptr, mem, temp, output);
+}
+
+void MacroAssembler::atomicLoad64(const Synchronization&, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicLoad64(*this, nullptr, mem, temp, output);
+}
+
+void MacroAssembler::atomicStore64(const Synchronization&, const Address& mem,
+ Register64 value, Register64 temp) {
+ AtomicExchange64(*this, nullptr, mem, value, temp);
+}
+
+void MacroAssembler::atomicStore64(const Synchronization&, const BaseIndex& mem,
+ Register64 value, Register64 temp) {
+ AtomicExchange64(*this, nullptr, mem, value, temp);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization&,
+ const Address& mem, Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ CompareExchange64(*this, nullptr, mem, expected, replacement, output);
+}
+
+void MacroAssembler::compareExchange64(const Synchronization&,
+ const BaseIndex& mem,
+ Register64 expected,
+ Register64 replacement,
+ Register64 output) {
+ CompareExchange64(*this, nullptr, mem, expected, replacement, output);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization&,
+ const Address& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, nullptr, mem, value, output);
+}
+
+void MacroAssembler::atomicExchange64(const Synchronization&,
+ const BaseIndex& mem, Register64 value,
+ Register64 output) {
+ AtomicExchange64(*this, nullptr, mem, value, output);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization&, AtomicOp op,
+ const Address& value, const Address& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, op, value, mem, temp, output);
+}
+
+void MacroAssembler::atomicFetchOp64(const Synchronization&, AtomicOp op,
+ const Address& value, const BaseIndex& mem,
+ Register64 temp, Register64 output) {
+ AtomicFetchOp64(*this, nullptr, op, value, mem, temp, output);
+}
+
+// ========================================================================
+// Convert floating point.
+
+bool MacroAssembler::convertUInt64ToDoubleNeedsTemp() { return HasSSE3(); }
+
+void MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest,
+ Register temp) {
+ // SUBPD needs SSE2, HADDPD needs SSE3.
+ if (!HasSSE3()) {
+ MOZ_ASSERT(temp == Register::Invalid());
+
+ // Zero the dest register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(dest);
+
+ Push(src.high);
+ Push(src.low);
+ fild(Operand(esp, 0));
+
+ Label notNegative;
+ branch32(Assembler::NotSigned, src.high, Imm32(0), &notNegative);
+ double add_constant = 18446744073709551616.0; // 2^64
+ store64(Imm64(mozilla::BitwiseCast<uint64_t>(add_constant)),
+ Address(esp, 0));
+ fld(Operand(esp, 0));
+ faddp();
+ bind(&notNegative);
+
+ fstp(Operand(esp, 0));
+ vmovsd(Address(esp, 0), dest);
+ freeStack(2 * sizeof(intptr_t));
+ return;
+ }
+
+ // Following operation uses entire 128-bit of dest XMM register.
+ // Currently higher 64-bit is free when we have access to lower 64-bit.
+ MOZ_ASSERT(dest.size() == 8);
+ FloatRegister dest128 =
+ FloatRegister(dest.encoding(), FloatRegisters::Simd128);
+
+ // Assume that src is represented as following:
+ // src = 0x HHHHHHHH LLLLLLLL
+
+ {
+ // Move src to dest (=dest128) and ScratchInt32x4Reg (=scratch):
+ // dest = 0x 00000000 00000000 00000000 LLLLLLLL
+ // scratch = 0x 00000000 00000000 00000000 HHHHHHHH
+ ScratchSimd128Scope scratch(*this);
+ vmovd(src.low, dest128);
+ vmovd(src.high, scratch);
+
+ // Unpack and interleave dest and scratch to dest:
+ // dest = 0x 00000000 00000000 HHHHHHHH LLLLLLLL
+ vpunpckldq(scratch, dest128, dest128);
+ }
+
+ // Unpack and interleave dest and a constant C1 to dest:
+ // C1 = 0x 00000000 00000000 45300000 43300000
+ // dest = 0x 45300000 HHHHHHHH 43300000 LLLLLLLL
+ // here, each 64-bit part of dest represents following double:
+ // HI(dest) = 0x 1.00000HHHHHHHH * 2**84 == 2**84 + 0x HHHHHHHH 00000000
+ // LO(dest) = 0x 1.00000LLLLLLLL * 2**52 == 2**52 + 0x 00000000 LLLLLLLL
+ // See convertUInt64ToDouble for the details.
+ static const int32_t CST1[4] = {
+ 0x43300000,
+ 0x45300000,
+ 0x0,
+ 0x0,
+ };
+
+ vpunpckldqSimd128(SimdConstant::CreateX4(CST1), dest128, dest128);
+
+ // Subtract a constant C2 from dest, for each 64-bit part:
+ // C2 = 0x 45300000 00000000 43300000 00000000
+ // here, each 64-bit part of C2 represents following double:
+ // HI(C2) = 0x 1.0000000000000 * 2**84 == 2**84
+ // LO(C2) = 0x 1.0000000000000 * 2**52 == 2**52
+ // after the operation each 64-bit part of dest represents following:
+ // HI(dest) = double(0x HHHHHHHH 00000000)
+ // LO(dest) = double(0x 00000000 LLLLLLLL)
+ static const int32_t CST2[4] = {
+ 0x0,
+ 0x43300000,
+ 0x0,
+ 0x45300000,
+ };
+
+ vsubpdSimd128(SimdConstant::CreateX4(CST2), dest128, dest128);
+
+ // Add HI(dest) and LO(dest) in double and store it into LO(dest),
+ // LO(dest) = double(0x HHHHHHHH 00000000) + double(0x 00000000 LLLLLLLL)
+ // = double(0x HHHHHHHH LLLLLLLL)
+ // = double(src)
+ vhaddpd(dest128, dest128);
+}
+
+void MacroAssembler::convertInt64ToDouble(Register64 input,
+ FloatRegister output) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(output);
+
+ Push(input.high);
+ Push(input.low);
+ fild(Operand(esp, 0));
+
+ fstp(Operand(esp, 0));
+ vmovsd(Address(esp, 0), output);
+ freeStack(2 * sizeof(intptr_t));
+}
+
+void MacroAssembler::convertUInt64ToFloat32(Register64 input,
+ FloatRegister output,
+ Register temp) {
+ // Zero the dest register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(output);
+
+ // Set the FPU precision to 80 bits.
+ reserveStack(2 * sizeof(intptr_t));
+ fnstcw(Operand(esp, 0));
+ load32(Operand(esp, 0), temp);
+ orl(Imm32(0x300), temp);
+ store32(temp, Operand(esp, sizeof(intptr_t)));
+ fldcw(Operand(esp, sizeof(intptr_t)));
+
+ Push(input.high);
+ Push(input.low);
+ fild(Operand(esp, 0));
+
+ Label notNegative;
+ branch32(Assembler::NotSigned, input.high, Imm32(0), &notNegative);
+ double add_constant = 18446744073709551616.0; // 2^64
+ uint64_t add_constant_u64 = mozilla::BitwiseCast<uint64_t>(add_constant);
+ store64(Imm64(add_constant_u64), Address(esp, 0));
+
+ fld(Operand(esp, 0));
+ faddp();
+ bind(&notNegative);
+
+ fstp32(Operand(esp, 0));
+ vmovss(Address(esp, 0), output);
+ freeStack(2 * sizeof(intptr_t));
+
+ // Restore FPU precision to the initial value.
+ fldcw(Operand(esp, 0));
+ freeStack(2 * sizeof(intptr_t));
+}
+
+void MacroAssembler::convertInt64ToFloat32(Register64 input,
+ FloatRegister output) {
+ // Zero the output register to break dependencies, see convertInt32ToDouble.
+ zeroDouble(output);
+
+ Push(input.high);
+ Push(input.low);
+ fild(Operand(esp, 0));
+
+ fstp32(Operand(esp, 0));
+ vmovss(Address(esp, 0), output);
+ freeStack(2 * sizeof(intptr_t));
+}
+
+void MacroAssembler::convertIntPtrToDouble(Register src, FloatRegister dest) {
+ convertInt32ToDouble(src, dest);
+}
+
+void MacroAssembler::PushBoxed(FloatRegister reg) { Push(reg); }
+
+CodeOffset MacroAssembler::moveNearAddressWithPatch(Register dest) {
+ return movWithPatch(ImmPtr(nullptr), dest);
+}
+
+void MacroAssembler::patchNearAddressMove(CodeLocationLabel loc,
+ CodeLocationLabel target) {
+ PatchDataWithValueCheck(loc, ImmPtr(target.raw()), ImmPtr(nullptr));
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Register64 boundsCheckLimit, Label* ok) {
+ Label notOk;
+ cmp32(index.high, Imm32(0));
+ j(Assembler::NonZero, &notOk);
+ wasmBoundsCheck32(cond, index.low, boundsCheckLimit.low, ok);
+ bind(&notOk);
+}
+
+void MacroAssembler::wasmBoundsCheck64(Condition cond, Register64 index,
+ Address boundsCheckLimit, Label* ok) {
+ Label notOk;
+ cmp32(index.high, Imm32(0));
+ j(Assembler::NonZero, &notOk);
+ wasmBoundsCheck32(cond, index.low, boundsCheckLimit, ok);
+ bind(&notOk);
+}
+
+//}}} check_macroassembler_style
diff --git a/js/src/jit/x86/MacroAssembler-x86.h b/js/src/jit/x86/MacroAssembler-x86.h
new file mode 100644
index 0000000000..3b60bd750f
--- /dev/null
+++ b/js/src/jit/x86/MacroAssembler-x86.h
@@ -0,0 +1,1140 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_MacroAssembler_x86_h
+#define jit_x86_MacroAssembler_x86_h
+
+#include "jit/JitOptions.h"
+#include "jit/MoveResolver.h"
+#include "jit/x86-shared/MacroAssembler-x86-shared.h"
+#include "js/HeapAPI.h"
+#include "wasm/WasmBuiltins.h"
+
+namespace js {
+namespace jit {
+
+// See documentation for ScratchTagScope and ScratchTagScopeRelease in
+// MacroAssembler-x64.h.
+
+class ScratchTagScope {
+ const ValueOperand& v_;
+
+ public:
+ ScratchTagScope(MacroAssembler&, const ValueOperand& v) : v_(v) {}
+ operator Register() { return v_.typeReg(); }
+ void release() {}
+ void reacquire() {}
+};
+
+class ScratchTagScopeRelease {
+ public:
+ explicit ScratchTagScopeRelease(ScratchTagScope*) {}
+};
+
+class MacroAssemblerX86 : public MacroAssemblerX86Shared {
+ private:
+ // Perform a downcast. Should be removed by Bug 996602.
+ MacroAssembler& asMasm();
+ const MacroAssembler& asMasm() const;
+
+ protected:
+ MoveResolver moveResolver_;
+
+ private:
+ Operand payloadOfAfterStackPush(const Address& address) {
+ // If we are basing off %esp, the address will be invalid after the
+ // first push.
+ if (address.base == StackPointer) {
+ return Operand(address.base, address.offset + 4);
+ }
+ return payloadOf(address);
+ }
+ Operand payloadOf(const Address& address) {
+ return Operand(address.base, address.offset);
+ }
+ Operand payloadOf(const BaseIndex& address) {
+ return Operand(address.base, address.index, address.scale, address.offset);
+ }
+ Operand tagOf(const Address& address) {
+ return Operand(address.base, address.offset + 4);
+ }
+ Operand tagOf(const BaseIndex& address) {
+ return Operand(address.base, address.index, address.scale,
+ address.offset + 4);
+ }
+
+ void setupABICall(uint32_t args);
+
+ void vpPatchOpSimd128(const SimdConstant& v, FloatRegister reg,
+ void (X86Encoding::BaseAssemblerX86::*op)(
+ const void* address,
+ X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId)) {
+ vpPatchOpSimd128(v, reg, reg, op);
+ }
+
+ void vpPatchOpSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest,
+ void (X86Encoding::BaseAssemblerX86::*op)(
+ const void* address,
+ X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId));
+
+ void vpPatchOpSimd128(const SimdConstant& v, FloatRegister reg,
+ size_t (X86Encoding::BaseAssemblerX86::*op)(
+ const void* address,
+ X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId)) {
+ vpPatchOpSimd128(v, reg, reg, op);
+ }
+
+ void vpPatchOpSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest,
+ size_t (X86Encoding::BaseAssemblerX86::*op)(
+ const void* address,
+ X86Encoding::XMMRegisterID srcId,
+ X86Encoding::XMMRegisterID destId));
+
+ public:
+ using MacroAssemblerX86Shared::call;
+ using MacroAssemblerX86Shared::load32;
+ using MacroAssemblerX86Shared::store16;
+ using MacroAssemblerX86Shared::store32;
+
+ MacroAssemblerX86() {}
+
+ // The buffer is about to be linked, make sure any constant pools or excess
+ // bookkeeping has been flushed to the instruction stream.
+ void finish();
+
+ /////////////////////////////////////////////////////////////////
+ // X86-specific interface.
+ /////////////////////////////////////////////////////////////////
+
+ Operand ToPayload(Operand base) { return base; }
+ Address ToPayload(Address base) { return base; }
+ BaseIndex ToPayload(BaseIndex base) { return base; }
+ Operand ToType(Operand base) {
+ switch (base.kind()) {
+ case Operand::MEM_REG_DISP:
+ return Operand(Register::FromCode(base.base()),
+ base.disp() + sizeof(void*));
+
+ case Operand::MEM_SCALE:
+ return Operand(Register::FromCode(base.base()),
+ Register::FromCode(base.index()), base.scale(),
+ base.disp() + sizeof(void*));
+
+ default:
+ MOZ_CRASH("unexpected operand kind");
+ }
+ }
+ Address ToType(Address base) { return ToType(Operand(base)).toAddress(); }
+ BaseIndex ToType(BaseIndex base) {
+ return ToType(Operand(base)).toBaseIndex();
+ }
+
+ template <typename T>
+ void add64FromMemory(const T& address, Register64 dest) {
+ addl(Operand(LowWord(address)), dest.low);
+ adcl(Operand(HighWord(address)), dest.high);
+ }
+ template <typename T>
+ void sub64FromMemory(const T& address, Register64 dest) {
+ subl(Operand(LowWord(address)), dest.low);
+ sbbl(Operand(HighWord(address)), dest.high);
+ }
+ template <typename T>
+ void and64FromMemory(const T& address, Register64 dest) {
+ andl(Operand(LowWord(address)), dest.low);
+ andl(Operand(HighWord(address)), dest.high);
+ }
+ template <typename T>
+ void or64FromMemory(const T& address, Register64 dest) {
+ orl(Operand(LowWord(address)), dest.low);
+ orl(Operand(HighWord(address)), dest.high);
+ }
+ template <typename T>
+ void xor64FromMemory(const T& address, Register64 dest) {
+ xorl(Operand(LowWord(address)), dest.low);
+ xorl(Operand(HighWord(address)), dest.high);
+ }
+
+ /////////////////////////////////////////////////////////////////
+ // X86/X64-common interface.
+ /////////////////////////////////////////////////////////////////
+ void storeValue(ValueOperand val, Operand dest) {
+ movl(val.payloadReg(), ToPayload(dest));
+ movl(val.typeReg(), ToType(dest));
+ }
+ void storeValue(ValueOperand val, const Address& dest) {
+ storeValue(val, Operand(dest));
+ }
+ template <typename T>
+ void storeValue(JSValueType type, Register reg, const T& dest) {
+ storeTypeTag(ImmTag(JSVAL_TYPE_TO_TAG(type)), Operand(dest));
+ storePayload(reg, Operand(dest));
+ }
+ template <typename T>
+ void storeValue(const Value& val, const T& dest) {
+ storeTypeTag(ImmTag(val.toNunboxTag()), Operand(dest));
+ storePayload(val, Operand(dest));
+ }
+ void storeValue(ValueOperand val, BaseIndex dest) {
+ storeValue(val, Operand(dest));
+ }
+ void storeValue(const Address& src, const Address& dest, Register temp) {
+ MOZ_ASSERT(src.base != temp);
+ MOZ_ASSERT(dest.base != temp);
+
+ load32(ToType(src), temp);
+ store32(temp, ToType(dest));
+
+ load32(ToPayload(src), temp);
+ store32(temp, ToPayload(dest));
+ }
+ void storePrivateValue(Register src, const Address& dest) {
+ store32(Imm32(0), ToType(dest));
+ store32(src, ToPayload(dest));
+ }
+ void storePrivateValue(ImmGCPtr imm, const Address& dest) {
+ store32(Imm32(0), ToType(dest));
+ movl(imm, Operand(ToPayload(dest)));
+ }
+ void loadValue(Operand src, ValueOperand val) {
+ Operand payload = ToPayload(src);
+ Operand type = ToType(src);
+
+ // Ensure that loading the payload does not erase the pointer to the
+ // Value in memory or the index.
+ Register baseReg = Register::FromCode(src.base());
+ Register indexReg = (src.kind() == Operand::MEM_SCALE)
+ ? Register::FromCode(src.index())
+ : InvalidReg;
+
+ // If we have a BaseIndex that uses both result registers, first compute
+ // the address and then load the Value from there.
+ if ((baseReg == val.payloadReg() && indexReg == val.typeReg()) ||
+ (baseReg == val.typeReg() && indexReg == val.payloadReg())) {
+ computeEffectiveAddress(src, val.scratchReg());
+ loadValue(Address(val.scratchReg(), 0), val);
+ return;
+ }
+
+ if (baseReg == val.payloadReg() || indexReg == val.payloadReg()) {
+ MOZ_ASSERT(baseReg != val.typeReg());
+ MOZ_ASSERT(indexReg != val.typeReg());
+
+ movl(type, val.typeReg());
+ movl(payload, val.payloadReg());
+ } else {
+ MOZ_ASSERT(baseReg != val.payloadReg());
+ MOZ_ASSERT(indexReg != val.payloadReg());
+
+ movl(payload, val.payloadReg());
+ movl(type, val.typeReg());
+ }
+ }
+ void loadValue(Address src, ValueOperand val) {
+ loadValue(Operand(src), val);
+ }
+ void loadValue(const BaseIndex& src, ValueOperand val) {
+ loadValue(Operand(src), val);
+ }
+ void loadUnalignedValue(const Address& src, ValueOperand dest) {
+ loadValue(src, dest);
+ }
+ void tagValue(JSValueType type, Register payload, ValueOperand dest) {
+ MOZ_ASSERT(dest.typeReg() != dest.payloadReg());
+ if (payload != dest.payloadReg()) {
+ movl(payload, dest.payloadReg());
+ }
+ movl(ImmType(type), dest.typeReg());
+ }
+ void pushValue(ValueOperand val) {
+ push(val.typeReg());
+ push(val.payloadReg());
+ }
+ void popValue(ValueOperand val) {
+ pop(val.payloadReg());
+ pop(val.typeReg());
+ }
+ void pushValue(const Value& val) {
+ push(Imm32(val.toNunboxTag()));
+ if (val.isGCThing()) {
+ push(ImmGCPtr(val.toGCThing()));
+ } else {
+ push(Imm32(val.toNunboxPayload()));
+ }
+ }
+ void pushValue(JSValueType type, Register reg) {
+ push(ImmTag(JSVAL_TYPE_TO_TAG(type)));
+ push(reg);
+ }
+ void pushValue(const Address& addr) {
+ push(tagOf(addr));
+ push(payloadOfAfterStackPush(addr));
+ }
+ void push64(Register64 src) {
+ push(src.high);
+ push(src.low);
+ }
+ void pop64(Register64 dest) {
+ pop(dest.low);
+ pop(dest.high);
+ }
+ void storePayload(const Value& val, Operand dest) {
+ if (val.isGCThing()) {
+ movl(ImmGCPtr(val.toGCThing()), ToPayload(dest));
+ } else {
+ movl(Imm32(val.toNunboxPayload()), ToPayload(dest));
+ }
+ }
+ void storePayload(Register src, Operand dest) { movl(src, ToPayload(dest)); }
+ void storeTypeTag(ImmTag tag, Operand dest) { movl(tag, ToType(dest)); }
+
+ void movePtr(Register src, Register dest) { movl(src, dest); }
+ void movePtr(Register src, const Operand& dest) { movl(src, dest); }
+
+ void splitTagForTest(const ValueOperand& value, ScratchTagScope& tag) {
+ MOZ_ASSERT(value.typeReg() == tag);
+ }
+
+ Condition testUndefined(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+ }
+ Condition testInt32(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_INT32));
+ return cond;
+ }
+ Condition testDouble(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
+ Condition actual = (cond == Equal) ? Below : AboveOrEqual;
+ cmp32(tag, ImmTag(JSVAL_TAG_CLEAR));
+ return actual;
+ }
+ Condition testNull(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_NULL));
+ return cond;
+ }
+ Condition testString(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_STRING));
+ return cond;
+ }
+ Condition testSymbol(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+ }
+ Condition testBigInt(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+ }
+ Condition testObject(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+ }
+ Condition testNumber(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JS::detail::ValueUpperInclNumberTag));
+ return cond == Equal ? BelowOrEqual : Above;
+ }
+ Condition testGCThing(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+ }
+ Condition testGCThing(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+ }
+ Condition testMagic(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+ }
+ Condition testMagic(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+ }
+ Condition testMagic(Condition cond, const Operand& operand) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(ToType(operand), ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+ }
+ Condition testPrimitive(Condition cond, Register tag) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tag, ImmTag(JS::detail::ValueUpperExclPrimitiveTag));
+ return cond == Equal ? Below : AboveOrEqual;
+ }
+ Condition testError(Condition cond, Register tag) {
+ return testMagic(cond, tag);
+ }
+ Condition testBoolean(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(Operand(ToType(address)), ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+ }
+ Condition testInt32(Condition cond, const Operand& operand) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(ToType(operand), ImmTag(JSVAL_TAG_INT32));
+ return cond;
+ }
+ Condition testInt32(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ return testInt32(cond, Operand(address));
+ }
+ Condition testObject(Condition cond, const Operand& operand) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(ToType(operand), ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+ }
+ Condition testObject(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ return testObject(cond, Operand(address));
+ }
+ Condition testDouble(Condition cond, const Operand& operand) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = (cond == Equal) ? Below : AboveOrEqual;
+ cmp32(ToType(operand), ImmTag(JSVAL_TAG_CLEAR));
+ return actual;
+ }
+ Condition testDouble(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ return testDouble(cond, Operand(address));
+ }
+
+ Condition testUndefined(Condition cond, const Operand& operand) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(ToType(operand), ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+ }
+ Condition testUndefined(Condition cond, const Address& addr) {
+ return testUndefined(cond, Operand(addr));
+ }
+ Condition testNull(Condition cond, const Operand& operand) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(ToType(operand), ImmTag(JSVAL_TAG_NULL));
+ return cond;
+ }
+ Condition testNull(Condition cond, const Address& addr) {
+ return testNull(cond, Operand(addr));
+ }
+
+ Condition testUndefined(Condition cond, const ValueOperand& value) {
+ return testUndefined(cond, value.typeReg());
+ }
+ Condition testBoolean(Condition cond, const ValueOperand& value) {
+ return testBoolean(cond, value.typeReg());
+ }
+ Condition testInt32(Condition cond, const ValueOperand& value) {
+ return testInt32(cond, value.typeReg());
+ }
+ Condition testDouble(Condition cond, const ValueOperand& value) {
+ return testDouble(cond, value.typeReg());
+ }
+ Condition testNull(Condition cond, const ValueOperand& value) {
+ return testNull(cond, value.typeReg());
+ }
+ Condition testString(Condition cond, const ValueOperand& value) {
+ return testString(cond, value.typeReg());
+ }
+ Condition testSymbol(Condition cond, const ValueOperand& value) {
+ return testSymbol(cond, value.typeReg());
+ }
+ Condition testBigInt(Condition cond, const ValueOperand& value) {
+ return testBigInt(cond, value.typeReg());
+ }
+ Condition testObject(Condition cond, const ValueOperand& value) {
+ return testObject(cond, value.typeReg());
+ }
+ Condition testMagic(Condition cond, const ValueOperand& value) {
+ return testMagic(cond, value.typeReg());
+ }
+ Condition testError(Condition cond, const ValueOperand& value) {
+ return testMagic(cond, value);
+ }
+ Condition testNumber(Condition cond, const ValueOperand& value) {
+ return testNumber(cond, value.typeReg());
+ }
+ Condition testGCThing(Condition cond, const ValueOperand& value) {
+ return testGCThing(cond, value.typeReg());
+ }
+ Condition testPrimitive(Condition cond, const ValueOperand& value) {
+ return testPrimitive(cond, value.typeReg());
+ }
+
+ Condition testUndefined(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_UNDEFINED));
+ return cond;
+ }
+ Condition testNull(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_NULL));
+ return cond;
+ }
+ Condition testBoolean(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_BOOLEAN));
+ return cond;
+ }
+ Condition testString(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_STRING));
+ return cond;
+ }
+ Condition testString(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_STRING));
+ return cond;
+ }
+ Condition testSymbol(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+ }
+ Condition testSymbol(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_SYMBOL));
+ return cond;
+ }
+ Condition testBigInt(Condition cond, const Address& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+ }
+ Condition testBigInt(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_BIGINT));
+ return cond;
+ }
+ Condition testInt32(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_INT32));
+ return cond;
+ }
+ Condition testObject(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_OBJECT));
+ return cond;
+ }
+ Condition testDouble(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ Condition actual = (cond == Equal) ? Below : AboveOrEqual;
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_CLEAR));
+ return actual;
+ }
+ Condition testMagic(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JSVAL_TAG_MAGIC));
+ return cond;
+ }
+ Condition testGCThing(Condition cond, const BaseIndex& address) {
+ MOZ_ASSERT(cond == Equal || cond == NotEqual);
+ cmp32(tagOf(address), ImmTag(JS::detail::ValueLowerInclGCThingTag));
+ return cond == Equal ? AboveOrEqual : Below;
+ }
+
+ void testNullSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testNull(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testObjectSet(Condition cond, const ValueOperand& value, Register dest) {
+ cond = testObject(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void testUndefinedSet(Condition cond, const ValueOperand& value,
+ Register dest) {
+ cond = testUndefined(cond, value);
+ emitSet(cond, dest);
+ }
+
+ void cmpPtr(Register lhs, const ImmWord rhs) { cmpl(Imm32(rhs.value), lhs); }
+ void cmpPtr(Register lhs, const ImmPtr imm) {
+ cmpPtr(lhs, ImmWord(uintptr_t(imm.value)));
+ }
+ void cmpPtr(Register lhs, const ImmGCPtr rhs) { cmpl(rhs, lhs); }
+ void cmpPtr(const Operand& lhs, Imm32 rhs) { cmp32(lhs, rhs); }
+ void cmpPtr(const Operand& lhs, const ImmWord rhs) {
+ cmp32(lhs, Imm32(rhs.value));
+ }
+ void cmpPtr(const Operand& lhs, const ImmPtr imm) {
+ cmpPtr(lhs, ImmWord(uintptr_t(imm.value)));
+ }
+ void cmpPtr(const Operand& lhs, const ImmGCPtr rhs) { cmpl(rhs, lhs); }
+ void cmpPtr(const Address& lhs, Register rhs) { cmpPtr(Operand(lhs), rhs); }
+ void cmpPtr(const Operand& lhs, Register rhs) { cmp32(lhs, rhs); }
+ void cmpPtr(const Address& lhs, const ImmWord rhs) {
+ cmpPtr(Operand(lhs), rhs);
+ }
+ void cmpPtr(const Address& lhs, const ImmPtr rhs) {
+ cmpPtr(lhs, ImmWord(uintptr_t(rhs.value)));
+ }
+ void cmpPtr(const Address& lhs, const ImmGCPtr rhs) {
+ cmpPtr(Operand(lhs), rhs);
+ }
+ void cmpPtr(Register lhs, Register rhs) { cmp32(lhs, rhs); }
+ void testPtr(Register lhs, Register rhs) { test32(lhs, rhs); }
+ void testPtr(Register lhs, Imm32 rhs) { test32(lhs, rhs); }
+ void testPtr(Register lhs, ImmWord rhs) { test32(lhs, Imm32(rhs.value)); }
+ void testPtr(const Operand& lhs, Imm32 rhs) { test32(lhs, rhs); }
+ void testPtr(const Operand& lhs, ImmWord rhs) {
+ test32(lhs, Imm32(rhs.value));
+ }
+
+ /////////////////////////////////////////////////////////////////
+ // Common interface.
+ /////////////////////////////////////////////////////////////////
+
+ void movePtr(ImmWord imm, Register dest) { movl(Imm32(imm.value), dest); }
+ void movePtr(ImmPtr imm, Register dest) { movl(imm, dest); }
+ void movePtr(wasm::SymbolicAddress imm, Register dest) { mov(imm, dest); }
+ void movePtr(ImmGCPtr imm, Register dest) { movl(imm, dest); }
+ void loadPtr(const Address& address, Register dest) {
+ movl(Operand(address), dest);
+ }
+ void loadPtr(const Operand& src, Register dest) { movl(src, dest); }
+ void loadPtr(const BaseIndex& src, Register dest) {
+ movl(Operand(src), dest);
+ }
+ void loadPtr(AbsoluteAddress address, Register dest) {
+ movl(Operand(address), dest);
+ }
+ void loadPrivate(const Address& src, Register dest) {
+ movl(payloadOf(src), dest);
+ }
+ void load32(AbsoluteAddress address, Register dest) {
+ movl(Operand(address), dest);
+ }
+ void load64(const Address& address, Register64 dest) {
+ bool highBeforeLow = address.base == dest.low;
+ if (highBeforeLow) {
+ movl(Operand(HighWord(address)), dest.high);
+ movl(Operand(LowWord(address)), dest.low);
+ } else {
+ movl(Operand(LowWord(address)), dest.low);
+ movl(Operand(HighWord(address)), dest.high);
+ }
+ }
+ void load64(const BaseIndex& address, Register64 dest) {
+ // If you run into this, relax your register allocation constraints.
+ MOZ_RELEASE_ASSERT(
+ !((address.base == dest.low || address.base == dest.high) &&
+ (address.index == dest.low || address.index == dest.high)));
+ bool highBeforeLow = address.base == dest.low || address.index == dest.low;
+ if (highBeforeLow) {
+ movl(Operand(HighWord(address)), dest.high);
+ movl(Operand(LowWord(address)), dest.low);
+ } else {
+ movl(Operand(LowWord(address)), dest.low);
+ movl(Operand(HighWord(address)), dest.high);
+ }
+ }
+ template <typename T>
+ void load64Unaligned(const T& address, Register64 dest) {
+ load64(address, dest);
+ }
+ template <typename T>
+ void storePtr(ImmWord imm, T address) {
+ movl(Imm32(imm.value), Operand(address));
+ }
+ template <typename T>
+ void storePtr(ImmPtr imm, T address) {
+ storePtr(ImmWord(uintptr_t(imm.value)), address);
+ }
+ template <typename T>
+ void storePtr(ImmGCPtr imm, T address) {
+ movl(imm, Operand(address));
+ }
+ void storePtr(Register src, const Address& address) {
+ movl(src, Operand(address));
+ }
+ void storePtr(Register src, const BaseIndex& address) {
+ movl(src, Operand(address));
+ }
+ void storePtr(Register src, const Operand& dest) { movl(src, dest); }
+ void storePtr(Register src, AbsoluteAddress address) {
+ movl(src, Operand(address));
+ }
+ void store32(Register src, AbsoluteAddress address) {
+ movl(src, Operand(address));
+ }
+ void store16(Register src, AbsoluteAddress address) {
+ movw(src, Operand(address));
+ }
+ template <typename T>
+ void store64(Register64 src, const T& address) {
+ movl(src.low, Operand(LowWord(address)));
+ movl(src.high, Operand(HighWord(address)));
+ }
+ void store64(Imm64 imm, Address address) {
+ movl(imm.low(), Operand(LowWord(address)));
+ movl(imm.hi(), Operand(HighWord(address)));
+ }
+ template <typename S, typename T>
+ void store64Unaligned(const S& src, const T& dest) {
+ store64(src, dest);
+ }
+
+ void setStackArg(Register reg, uint32_t arg) {
+ movl(reg, Operand(esp, arg * sizeof(intptr_t)));
+ }
+
+ void boxDouble(FloatRegister src, const ValueOperand& dest,
+ FloatRegister temp) {
+ if (Assembler::HasSSE41()) {
+ vmovd(src, dest.payloadReg());
+ vpextrd(1, src, dest.typeReg());
+ } else {
+ vmovd(src, dest.payloadReg());
+ if (src != temp) {
+ moveDouble(src, temp);
+ }
+ vpsrldq(Imm32(4), temp, temp);
+ vmovd(temp, dest.typeReg());
+ }
+ }
+ void boxNonDouble(JSValueType type, Register src, const ValueOperand& dest) {
+ if (src != dest.payloadReg()) {
+ movl(src, dest.payloadReg());
+ }
+ movl(ImmType(type), dest.typeReg());
+ }
+
+ void unboxNonDouble(const ValueOperand& src, Register dest, JSValueType type,
+ Register scratch = InvalidReg) {
+ unboxNonDouble(Operand(src.typeReg()), Operand(src.payloadReg()), dest,
+ type, scratch);
+ }
+ void unboxNonDouble(const Operand& tag, const Operand& payload, Register dest,
+ JSValueType type, Register scratch = InvalidReg) {
+ auto movPayloadToDest = [&]() {
+ if (payload.kind() != Operand::REG || !payload.containsReg(dest)) {
+ movl(payload, dest);
+ }
+ };
+ if (!JitOptions.spectreValueMasking) {
+ movPayloadToDest();
+ return;
+ }
+
+ // Spectre mitigation: We zero the payload if the tag does not match the
+ // expected type and if this is a pointer type.
+ if (type == JSVAL_TYPE_INT32 || type == JSVAL_TYPE_BOOLEAN) {
+ movPayloadToDest();
+ return;
+ }
+
+ if (!tag.containsReg(dest) && !payload.containsReg(dest)) {
+ // We zero the destination register and move the payload into it if
+ // the tag corresponds to the given type.
+ xorl(dest, dest);
+ cmpl(Imm32(JSVAL_TYPE_TO_TAG(type)), tag);
+ cmovCCl(Condition::Equal, payload, dest);
+ return;
+ }
+
+ if (scratch == InvalidReg || scratch == dest || tag.containsReg(scratch) ||
+ payload.containsReg(scratch)) {
+ // UnboxedLayout::makeConstructorCode calls extractObject with a
+ // scratch register which aliases the tag register, thus we cannot
+ // assert the above condition.
+ scratch = InvalidReg;
+ }
+
+ // The destination register aliases one of the operands. We create a
+ // zero value either in a scratch register or on the stack and use it
+ // to reset the destination register after reading both the tag and the
+ // payload.
+ Operand zero(Address(esp, 0));
+ if (scratch == InvalidReg) {
+ push(Imm32(0));
+ } else {
+ xorl(scratch, scratch);
+ zero = Operand(scratch);
+ }
+ cmpl(Imm32(JSVAL_TYPE_TO_TAG(type)), tag);
+ movPayloadToDest();
+ cmovCCl(Condition::NotEqual, zero, dest);
+ if (scratch == InvalidReg) {
+ addl(Imm32(sizeof(void*)), esp);
+ }
+ }
+ void unboxNonDouble(const Address& src, Register dest, JSValueType type) {
+ unboxNonDouble(tagOf(src), payloadOf(src), dest, type);
+ }
+ void unboxNonDouble(const BaseIndex& src, Register dest, JSValueType type) {
+ unboxNonDouble(tagOf(src), payloadOf(src), dest, type);
+ }
+ void unboxInt32(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_INT32);
+ }
+ void unboxInt32(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_INT32);
+ }
+ void unboxInt32(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_INT32);
+ }
+ void unboxBoolean(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN);
+ }
+ void unboxBoolean(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN);
+ }
+ void unboxBoolean(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BOOLEAN);
+ }
+ void unboxString(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxString(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_STRING);
+ }
+ void unboxSymbol(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxSymbol(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_SYMBOL);
+ }
+ void unboxBigInt(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxBigInt(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_BIGINT);
+ }
+ void unboxObject(const ValueOperand& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const Address& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ void unboxObject(const BaseIndex& src, Register dest) {
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ template <typename T>
+ void unboxObjectOrNull(const T& src, Register dest) {
+ // Due to Spectre mitigation logic (see Value.h), if the value is an Object
+ // then this yields the object; otherwise it yields zero (null), as desired.
+ unboxNonDouble(src, dest, JSVAL_TYPE_OBJECT);
+ }
+ template <typename T>
+ void unboxDouble(const T& src, FloatRegister dest) {
+ loadDouble(Operand(src), dest);
+ }
+ void unboxDouble(const ValueOperand& src, FloatRegister dest) {
+ if (Assembler::HasSSE41()) {
+ vmovd(src.payloadReg(), dest);
+ vpinsrd(1, src.typeReg(), dest, dest);
+ } else {
+ ScratchDoubleScope fpscratch(asMasm());
+ vmovd(src.payloadReg(), dest);
+ vmovd(src.typeReg(), fpscratch);
+ vunpcklps(fpscratch, dest, dest);
+ }
+ }
+ void unboxDouble(const Operand& payload, const Operand& type,
+ Register scratch, FloatRegister dest) {
+ if (Assembler::HasSSE41()) {
+ movl(payload, scratch);
+ vmovd(scratch, dest);
+ movl(type, scratch);
+ vpinsrd(1, scratch, dest, dest);
+ } else {
+ ScratchDoubleScope fpscratch(asMasm());
+ movl(payload, scratch);
+ vmovd(scratch, dest);
+ movl(type, scratch);
+ vmovd(scratch, fpscratch);
+ vunpcklps(fpscratch, dest, dest);
+ }
+ }
+ inline void unboxValue(const ValueOperand& src, AnyRegister dest,
+ JSValueType type);
+
+ // See comment in MacroAssembler-x64.h.
+ void unboxGCThingForGCBarrier(const Address& src, Register dest) {
+ movl(payloadOf(src), dest);
+ }
+
+ void notBoolean(const ValueOperand& val) { xorl(Imm32(1), val.payloadReg()); }
+
+ template <typename T>
+ void fallibleUnboxPtrImpl(const T& src, Register dest, JSValueType type,
+ Label* fail);
+
+ // Extended unboxing API. If the payload is already in a register, returns
+ // that register. Otherwise, provides a move to the given scratch register,
+ // and returns that.
+ [[nodiscard]] Register extractObject(const Address& address, Register dest) {
+ unboxObject(address, dest);
+ return dest;
+ }
+ [[nodiscard]] Register extractObject(const ValueOperand& value,
+ Register scratch) {
+ unboxNonDouble(value, value.payloadReg(), JSVAL_TYPE_OBJECT, scratch);
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractSymbol(const ValueOperand& value,
+ Register scratch) {
+ unboxNonDouble(value, value.payloadReg(), JSVAL_TYPE_SYMBOL, scratch);
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractInt32(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractBoolean(const ValueOperand& value,
+ Register scratch) {
+ return value.payloadReg();
+ }
+ [[nodiscard]] Register extractTag(const Address& address, Register scratch) {
+ movl(tagOf(address), scratch);
+ return scratch;
+ }
+ [[nodiscard]] Register extractTag(const ValueOperand& value,
+ Register scratch) {
+ return value.typeReg();
+ }
+
+ void convertDoubleToPtr(FloatRegister src, Register dest, Label* fail,
+ bool negativeZeroCheck = true) {
+ convertDoubleToInt32(src, dest, fail, negativeZeroCheck);
+ }
+
+ void boolValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.payloadReg(), dest);
+ }
+ void boolValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.payloadReg(), dest);
+ }
+ void int32ValueToDouble(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToDouble(operand.payloadReg(), dest);
+ }
+ void int32ValueToFloat32(const ValueOperand& operand, FloatRegister dest) {
+ convertInt32ToFloat32(operand.payloadReg(), dest);
+ }
+
+ void loadConstantDouble(double d, FloatRegister dest);
+ void loadConstantFloat32(float f, FloatRegister dest);
+
+ void loadConstantSimd128Int(const SimdConstant& v, FloatRegister dest);
+ void loadConstantSimd128Float(const SimdConstant& v, FloatRegister dest);
+ void vpaddbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpadddSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmullwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmulldSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddusbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpaddswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpadduswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubusbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpsubuswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpminudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxsbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxubSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxuwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxsdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaxudSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpandSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpxorSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vporSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vaddpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vaddpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vsubpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vsubpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vdivpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vdivpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vmulpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vmulpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vandpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vminpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpacksswbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackuswbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackssdwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpackusdwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpunpckldqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vunpcklpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpshufbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vptestSimd128(const SimdConstant& v, FloatRegister lhs);
+ void vpmaddwdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtbSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtwSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpeqdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpcmpgtdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpeqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpneqpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpltpsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmplepsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpgepsSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpeqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpneqpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmpltpdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vcmplepdSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmaddubswSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+ void vpmuludqSimd128(const SimdConstant& v, FloatRegister lhs,
+ FloatRegister dest);
+
+ Condition testInt32Truthy(bool truthy, const ValueOperand& operand) {
+ test32(operand.payloadReg(), operand.payloadReg());
+ return truthy ? NonZero : Zero;
+ }
+ Condition testStringTruthy(bool truthy, const ValueOperand& value);
+ Condition testBigIntTruthy(bool truthy, const ValueOperand& value);
+
+ template <typename T>
+ inline void loadInt32OrDouble(const T& src, FloatRegister dest);
+
+ template <typename T>
+ inline void loadUnboxedValue(const T& src, MIRType type, AnyRegister dest);
+
+ template <typename T>
+ void storeUnboxedPayload(ValueOperand value, T address, size_t nbytes,
+ JSValueType) {
+ switch (nbytes) {
+ case 4:
+ storePtr(value.payloadReg(), address);
+ return;
+ case 1:
+ store8(value.payloadReg(), address);
+ return;
+ default:
+ MOZ_CRASH("Bad payload width");
+ }
+ }
+
+ void loadInstructionPointerAfterCall(Register dest) {
+ movl(Operand(StackPointer, 0x0), dest);
+ }
+
+ // Note: this function clobbers the source register.
+ inline void convertUInt32ToDouble(Register src, FloatRegister dest);
+
+ // Note: this function clobbers the source register.
+ inline void convertUInt32ToFloat32(Register src, FloatRegister dest);
+
+ void incrementInt32Value(const Address& addr) {
+ addl(Imm32(1), payloadOf(addr));
+ }
+
+ inline void ensureDouble(const ValueOperand& source, FloatRegister dest,
+ Label* failure);
+
+ public:
+ // Used from within an Exit frame to handle a pending exception.
+ void handleFailureWithHandlerTail(Label* profilerExitTail,
+ Label* bailoutTail);
+
+ // Instrumentation for entering and leaving the profiler.
+ void profilerEnterFrame(Register framePtr, Register scratch);
+ void profilerExitFrame();
+};
+
+typedef MacroAssemblerX86 MacroAssemblerSpecific;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_MacroAssembler_x86_h */
diff --git a/js/src/jit/x86/SharedICHelpers-x86-inl.h b/js/src/jit/x86/SharedICHelpers-x86-inl.h
new file mode 100644
index 0000000000..2ab41c287a
--- /dev/null
+++ b/js/src/jit/x86/SharedICHelpers-x86-inl.h
@@ -0,0 +1,77 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_SharedICHelpers_x86_inl_h
+#define jit_x86_SharedICHelpers_x86_inl_h
+
+#include "jit/BaselineFrame.h"
+#include "jit/SharedICHelpers.h"
+
+#include "jit/MacroAssembler-inl.h"
+
+namespace js {
+namespace jit {
+
+inline void EmitBaselineTailCallVM(TrampolinePtr target, MacroAssembler& masm,
+ uint32_t argSize) {
+#ifdef DEBUG
+ // We assume during this that R0 and R1 have been pushed.
+ // Store frame size without VMFunction arguments for debug assertions.
+ masm.movl(FramePointer, eax);
+ masm.subl(StackPointer, eax);
+ masm.subl(Imm32(argSize), eax);
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(eax, frameSizeAddr);
+#endif
+
+ // Push frame descriptor and perform the tail call.
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(ICTailCallReg);
+ masm.jump(target);
+}
+
+inline void EmitBaselineCallVM(TrampolinePtr target, MacroAssembler& masm) {
+ masm.pushFrameDescriptor(FrameType::BaselineStub);
+ masm.call(target);
+}
+
+inline void EmitBaselineEnterStubFrame(MacroAssembler& masm, Register scratch) {
+#ifdef DEBUG
+ // Compute frame size. Because the return address is still on the stack,
+ // this is:
+ //
+ // FramePointer
+ // - StackPointer
+ // - sizeof(return address)
+
+ masm.movl(FramePointer, scratch);
+ masm.subl(StackPointer, scratch);
+ masm.subl(Imm32(sizeof(void*)), scratch); // Return address.
+
+ Address frameSizeAddr(FramePointer,
+ BaselineFrame::reverseOffsetOfDebugFrameSize());
+ masm.store32(scratch, frameSizeAddr);
+#endif
+
+ // Push the return address that's currently on top of the stack.
+ masm.Push(Operand(StackPointer, 0));
+
+ // Replace the original return address with the frame descriptor.
+ masm.storePtr(ImmWord(MakeFrameDescriptor(FrameType::BaselineJS)),
+ Address(StackPointer, sizeof(uintptr_t)));
+
+ // Save old frame pointer, stack pointer and stub reg.
+ masm.Push(FramePointer);
+ masm.mov(StackPointer, FramePointer);
+
+ masm.Push(ICStubReg);
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_SharedICHelpers_x86_inl_h */
diff --git a/js/src/jit/x86/SharedICHelpers-x86.h b/js/src/jit/x86/SharedICHelpers-x86.h
new file mode 100644
index 0000000000..07d2ae8a6f
--- /dev/null
+++ b/js/src/jit/x86/SharedICHelpers-x86.h
@@ -0,0 +1,70 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_SharedICHelpers_x86_h
+#define jit_x86_SharedICHelpers_x86_h
+
+#include "jit/BaselineIC.h"
+#include "jit/JitFrames.h"
+#include "jit/MacroAssembler.h"
+#include "jit/SharedICRegisters.h"
+
+namespace js {
+namespace jit {
+
+// Distance from stack top to the top Value inside an IC stub (this is the
+// return address).
+static const size_t ICStackValueOffset = sizeof(void*);
+
+inline void EmitRestoreTailCallReg(MacroAssembler& masm) {
+ masm.Pop(ICTailCallReg);
+}
+
+inline void EmitRepushTailCallReg(MacroAssembler& masm) {
+ masm.Push(ICTailCallReg);
+}
+
+inline void EmitCallIC(MacroAssembler& masm, CodeOffset* callOffset) {
+ // The stub pointer must already be in ICStubReg.
+ // Call the stubcode.
+ masm.call(Address(ICStubReg, ICStub::offsetOfStubCode()));
+ *callOffset = CodeOffset(masm.currentOffset());
+}
+
+inline void EmitReturnFromIC(MacroAssembler& masm) { masm.ret(); }
+
+inline void EmitBaselineLeaveStubFrame(MacroAssembler& masm) {
+ Address stubAddr(FramePointer, BaselineStubFrameLayout::ICStubOffsetFromFP);
+ masm.loadPtr(stubAddr, ICStubReg);
+
+ masm.mov(FramePointer, StackPointer);
+ masm.Pop(FramePointer);
+
+ // The return address is on top of the stack, followed by the frame
+ // descriptor. Use a pop instruction to overwrite the frame descriptor
+ // with the return address. Note that pop increments the stack pointer
+ // before computing the address.
+ masm.Pop(Operand(StackPointer, 0));
+}
+
+template <typename AddrType>
+inline void EmitPreBarrier(MacroAssembler& masm, const AddrType& addr,
+ MIRType type) {
+ masm.guardedCallPreBarrier(addr, type);
+}
+
+inline void EmitStubGuardFailure(MacroAssembler& masm) {
+ // Load next stub into ICStubReg
+ masm.loadPtr(Address(ICStubReg, ICCacheIRStub::offsetOfNext()), ICStubReg);
+
+ // Return address is already loaded, just jump to the next stubcode.
+ masm.jmp(Operand(ICStubReg, ICStub::offsetOfStubCode()));
+}
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_SharedICHelpers_x86_h */
diff --git a/js/src/jit/x86/SharedICRegisters-x86.h b/js/src/jit/x86/SharedICRegisters-x86.h
new file mode 100644
index 0000000000..44edb5288f
--- /dev/null
+++ b/js/src/jit/x86/SharedICRegisters-x86.h
@@ -0,0 +1,36 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef jit_x86_SharedICRegisters_x86_h
+#define jit_x86_SharedICRegisters_x86_h
+
+#include "jit/Registers.h"
+#include "jit/RegisterSets.h"
+#include "jit/x86/Assembler-x86.h"
+
+namespace js {
+namespace jit {
+
+// ValueOperands R0, R1, and R2
+static constexpr ValueOperand R0(ecx, edx);
+static constexpr ValueOperand R1(eax, ebx);
+static constexpr ValueOperand R2(esi, edi);
+
+// ICTailCallReg and ICStubReg reuse
+// registers from R2.
+static constexpr Register ICTailCallReg = esi;
+static constexpr Register ICStubReg = edi;
+
+// FloatReg0 must be equal to ReturnFloatReg.
+static constexpr FloatRegister FloatReg0 = xmm0;
+static constexpr FloatRegister FloatReg1 = xmm1;
+static constexpr FloatRegister FloatReg2 = xmm2;
+static constexpr FloatRegister FloatReg3 = xmm3;
+
+} // namespace jit
+} // namespace js
+
+#endif /* jit_x86_SharedICRegisters_x86_h */
diff --git a/js/src/jit/x86/Trampoline-x86.cpp b/js/src/jit/x86/Trampoline-x86.cpp
new file mode 100644
index 0000000000..c1a93da09c
--- /dev/null
+++ b/js/src/jit/x86/Trampoline-x86.cpp
@@ -0,0 +1,794 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * vim: set ts=8 sts=2 et sw=2 tw=80:
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "mozilla/MathAlgorithms.h"
+
+#include "jit/Bailouts.h"
+#include "jit/BaselineFrame.h"
+#include "jit/BaselineJIT.h"
+#include "jit/CalleeToken.h"
+#include "jit/JitFrames.h"
+#include "jit/JitRuntime.h"
+#include "jit/JitSpewer.h"
+#include "jit/PerfSpewer.h"
+#include "jit/VMFunctions.h"
+#include "jit/x86/SharedICHelpers-x86.h"
+#include "vm/JitActivation.h" // js::jit::JitActivation
+#include "vm/JSContext.h"
+#include "vm/Realm.h"
+#ifdef MOZ_VTUNE
+# include "vtune/VTuneWrapper.h"
+#endif
+
+#include "jit/MacroAssembler-inl.h"
+#include "vm/JSScript-inl.h"
+
+using mozilla::IsPowerOfTwo;
+
+using namespace js;
+using namespace js::jit;
+
+// All registers to save and restore. This includes the stack pointer, since we
+// use the ability to reference register values on the stack by index.
+static const LiveRegisterSet AllRegs =
+ LiveRegisterSet(GeneralRegisterSet(Registers::AllMask),
+ FloatRegisterSet(FloatRegisters::AllMask));
+
+enum EnterJitEbpArgumentOffset {
+ ARG_JITCODE = 2 * sizeof(void*),
+ ARG_ARGC = 3 * sizeof(void*),
+ ARG_ARGV = 4 * sizeof(void*),
+ ARG_STACKFRAME = 5 * sizeof(void*),
+ ARG_CALLEETOKEN = 6 * sizeof(void*),
+ ARG_SCOPECHAIN = 7 * sizeof(void*),
+ ARG_STACKVALUES = 8 * sizeof(void*),
+ ARG_RESULT = 9 * sizeof(void*)
+};
+
+// Generates a trampoline for calling Jit compiled code from a C++ function.
+// The trampoline use the EnterJitCode signature, with the standard cdecl
+// calling convention.
+void JitRuntime::generateEnterJIT(JSContext* cx, MacroAssembler& masm) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateEnterJIT");
+
+ enterJITOffset_ = startTrampolineCode(masm);
+
+ masm.assertStackAlignment(ABIStackAlignment,
+ -int32_t(sizeof(uintptr_t)) /* return address */);
+
+ // Save old stack frame pointer, set new stack frame pointer.
+ masm.push(ebp);
+ masm.movl(esp, ebp);
+
+ // Save non-volatile registers. These must be saved by the trampoline,
+ // rather than the JIT'd code, because they are scanned by the conservative
+ // scanner.
+ masm.push(ebx);
+ masm.push(esi);
+ masm.push(edi);
+
+ // Load the number of values to be copied (argc) into eax
+ masm.loadPtr(Address(ebp, ARG_ARGC), eax);
+
+ // If we are constructing, that also needs to include newTarget
+ {
+ Label noNewTarget;
+ masm.loadPtr(Address(ebp, ARG_CALLEETOKEN), edx);
+ masm.branchTest32(Assembler::Zero, edx,
+ Imm32(CalleeToken_FunctionConstructing), &noNewTarget);
+
+ masm.addl(Imm32(1), eax);
+
+ masm.bind(&noNewTarget);
+ }
+
+ // eax <- 8*numValues, eax is now the offset betwen argv and the last value.
+ masm.shll(Imm32(3), eax);
+
+ // Guarantee stack alignment of Jit frames.
+ //
+ // This code compensates for the offset created by the copy of the vector of
+ // arguments, such that the jit frame will be aligned once the return
+ // address is pushed on the stack.
+ //
+ // In the computation of the offset, we omit the size of the JitFrameLayout
+ // which is pushed on the stack, as the JitFrameLayout size is a multiple of
+ // the JitStackAlignment.
+ masm.movl(esp, ecx);
+ masm.subl(eax, ecx);
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+
+ // ecx = ecx & 15, holds alignment.
+ masm.andl(Imm32(JitStackAlignment - 1), ecx);
+ masm.subl(ecx, esp);
+
+ /***************************************************************
+ Loop over argv vector, push arguments onto stack in reverse order
+ ***************************************************************/
+
+ // ebx = argv --argv pointer is in ebp + 16
+ masm.loadPtr(Address(ebp, ARG_ARGV), ebx);
+
+ // eax = argv[8(argc)] --eax now points one value past the last argument
+ masm.addl(ebx, eax);
+
+ // while (eax > ebx) --while still looping through arguments
+ {
+ Label header, footer;
+ masm.bind(&header);
+
+ masm.cmp32(eax, ebx);
+ masm.j(Assembler::BelowOrEqual, &footer);
+
+ // eax -= 8 --move to previous argument
+ masm.subl(Imm32(8), eax);
+
+ // Push what eax points to on stack, a Value is 2 words
+ masm.push(Operand(eax, 4));
+ masm.push(Operand(eax, 0));
+
+ masm.jmp(&header);
+ masm.bind(&footer);
+ }
+
+ // Load the number of actual arguments. |result| is used to store the
+ // actual number of arguments without adding an extra argument to the enter
+ // JIT.
+ masm.mov(Operand(ebp, ARG_RESULT), eax);
+ masm.unboxInt32(Address(eax, 0x0), eax);
+
+ // Push the callee token.
+ masm.push(Operand(ebp, ARG_CALLEETOKEN));
+
+ // Load the InterpreterFrame address into the OsrFrameReg.
+ // This address is also used for setting the constructing bit on all paths.
+ masm.loadPtr(Address(ebp, ARG_STACKFRAME), OsrFrameReg);
+
+ // Push the descriptor.
+ masm.pushFrameDescriptorForJitCall(FrameType::CppToJSJit, eax, eax);
+
+ CodeLabel returnLabel;
+ Label oomReturnLabel;
+ {
+ // Handle Interpreter -> Baseline OSR.
+ AllocatableGeneralRegisterSet regs(GeneralRegisterSet::All());
+ MOZ_ASSERT(!regs.has(ebp));
+ regs.take(OsrFrameReg);
+ regs.take(ReturnReg);
+
+ Register scratch = regs.takeAny();
+
+ Label notOsr;
+ masm.branchTestPtr(Assembler::Zero, OsrFrameReg, OsrFrameReg, &notOsr);
+
+ Register numStackValues = regs.takeAny();
+ masm.loadPtr(Address(ebp, ARG_STACKVALUES), numStackValues);
+
+ Register jitcode = regs.takeAny();
+ masm.loadPtr(Address(ebp, ARG_JITCODE), jitcode);
+
+ // Push return address.
+ masm.mov(&returnLabel, scratch);
+ masm.push(scratch);
+
+ // Frame prologue.
+ masm.push(ebp);
+ masm.mov(esp, ebp);
+
+ // Reserve frame.
+ masm.subPtr(Imm32(BaselineFrame::Size()), esp);
+
+ Register framePtrScratch = regs.takeAny();
+ masm.touchFrameValues(numStackValues, scratch, framePtrScratch);
+ masm.mov(esp, framePtrScratch);
+
+ // Reserve space for locals and stack values.
+ masm.mov(numStackValues, scratch);
+ masm.shll(Imm32(3), scratch);
+ masm.subPtr(scratch, esp);
+
+ // Enter exit frame.
+ masm.pushFrameDescriptor(FrameType::BaselineJS);
+ masm.push(Imm32(0)); // Fake return address.
+ masm.push(FramePointer);
+ // No GC things to mark on the stack, push a bare token.
+ masm.loadJSContext(scratch);
+ masm.enterFakeExitFrame(scratch, scratch, ExitFrameType::Bare);
+
+ masm.push(jitcode);
+
+ using Fn = bool (*)(BaselineFrame * frame, InterpreterFrame * interpFrame,
+ uint32_t numStackValues);
+ masm.setupUnalignedABICall(scratch);
+ masm.passABIArg(framePtrScratch); // BaselineFrame
+ masm.passABIArg(OsrFrameReg); // InterpreterFrame
+ masm.passABIArg(numStackValues);
+ masm.callWithABI<Fn, jit::InitBaselineFrameForOsr>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ masm.pop(jitcode);
+
+ MOZ_ASSERT(jitcode != ReturnReg);
+
+ Label error;
+ masm.addPtr(Imm32(ExitFrameLayout::SizeWithFooter()), esp);
+ masm.branchIfFalseBool(ReturnReg, &error);
+
+ // If OSR-ing, then emit instrumentation for setting lastProfilerFrame
+ // if profiler instrumentation is enabled.
+ {
+ Label skipProfilingInstrumentation;
+ AbsoluteAddress addressOfEnabled(
+ cx->runtime()->geckoProfiler().addressOfEnabled());
+ masm.branch32(Assembler::Equal, addressOfEnabled, Imm32(0),
+ &skipProfilingInstrumentation);
+ masm.profilerEnterFrame(ebp, scratch);
+ masm.bind(&skipProfilingInstrumentation);
+ }
+
+ masm.jump(jitcode);
+
+ // OOM: frame epilogue, load error value, discard return address and return.
+ masm.bind(&error);
+ masm.mov(ebp, esp);
+ masm.pop(ebp);
+ masm.addPtr(Imm32(sizeof(uintptr_t)), esp); // Return address.
+ masm.moveValue(MagicValue(JS_ION_ERROR), JSReturnOperand);
+ masm.jump(&oomReturnLabel);
+
+ masm.bind(&notOsr);
+ masm.loadPtr(Address(ebp, ARG_SCOPECHAIN), R1.scratchReg());
+ }
+
+ // The call will push the return address and frame pointer on the stack, thus
+ // we check that the stack would be aligned once the call is complete.
+ masm.assertStackAlignment(JitStackAlignment, 2 * sizeof(uintptr_t));
+
+ /***************************************************************
+ Call passed-in code, get return value and fill in the
+ passed in return value pointer
+ ***************************************************************/
+ masm.call(Address(ebp, ARG_JITCODE));
+
+ {
+ // Interpreter -> Baseline OSR will return here.
+ masm.bind(&returnLabel);
+ masm.addCodeLabel(returnLabel);
+ masm.bind(&oomReturnLabel);
+ }
+
+ // Restore the stack pointer so the stack looks like this:
+ // +20 ... arguments ...
+ // +16 <return>
+ // +12 ebp <- %ebp pointing here.
+ // +8 ebx
+ // +4 esi
+ // +0 edi <- %esp pointing here.
+ masm.lea(Operand(ebp, -int32_t(3 * sizeof(void*))), esp);
+
+ // Store the return value.
+ masm.loadPtr(Address(ebp, ARG_RESULT), eax);
+ masm.storeValue(JSReturnOperand, Operand(eax, 0));
+
+ /**************************************************************
+ Return stack and registers to correct state
+ **************************************************************/
+
+ // Restore non-volatile registers
+ masm.pop(edi);
+ masm.pop(esi);
+ masm.pop(ebx);
+
+ // Restore old stack frame pointer
+ masm.pop(ebp);
+ masm.ret();
+}
+
+// static
+mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
+JitRuntime::getCppEntryRegisters(JitFrameLayout* frameStackAddress) {
+ // Not supported, or not implemented yet.
+ // TODO: Implement along with the corresponding stack-walker changes, in
+ // coordination with the Gecko Profiler, see bug 1635987 and follow-ups.
+ return mozilla::Nothing{};
+}
+
+// Push AllRegs in a way that is compatible with RegisterDump, regardless of
+// what PushRegsInMask might do to reduce the set size.
+static void DumpAllRegs(MacroAssembler& masm) {
+#ifdef ENABLE_WASM_SIMD
+ masm.PushRegsInMask(AllRegs);
+#else
+ // When SIMD isn't supported, PushRegsInMask reduces the set of float
+ // registers to be double-sized, while the RegisterDump expects each of
+ // the float registers to have the maximal possible size
+ // (Simd128DataSize). To work around this, we just spill the double
+ // registers by hand here, using the register dump offset directly.
+ for (GeneralRegisterBackwardIterator iter(AllRegs.gprs()); iter.more();
+ ++iter) {
+ masm.Push(*iter);
+ }
+
+ masm.reserveStack(sizeof(RegisterDump::FPUArray));
+ for (FloatRegisterBackwardIterator iter(AllRegs.fpus()); iter.more();
+ ++iter) {
+ FloatRegister reg = *iter;
+ Address spillAddress(StackPointer, reg.getRegisterDumpOffsetInBytes());
+ masm.storeDouble(reg, spillAddress);
+ }
+#endif
+}
+
+void JitRuntime::generateInvalidator(MacroAssembler& masm, Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateInvalidator");
+
+ invalidatorOffset_ = startTrampolineCode(masm);
+
+ // We do the minimum amount of work in assembly and shunt the rest
+ // off to InvalidationBailout. Assembly does:
+ //
+ // - Push the machine state onto the stack.
+ // - Call the InvalidationBailout routine with the stack pointer.
+ // - Now that the frame has been bailed out, convert the invalidated
+ // frame into an exit frame.
+ // - Do the normal check-return-code-and-thunk-to-the-interpreter dance.
+
+ // Push registers such that we can access them from [base + code].
+ DumpAllRegs(masm);
+
+ masm.movl(esp, eax); // Argument to jit::InvalidationBailout.
+
+ // Make space for InvalidationBailout's bailoutInfo outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.movl(esp, ebx);
+
+ using Fn =
+ bool (*)(InvalidationBailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(edx);
+ masm.passABIArg(eax);
+ masm.passABIArg(ebx);
+ masm.callWithABI<Fn, InvalidationBailout>(
+ MoveOp::GENERAL, CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(ecx); // Get bailoutInfo outparam.
+
+ // Pop the machine state and the dead frame.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in ecx.
+ masm.jmp(bailoutTail);
+}
+
+void JitRuntime::generateArgumentsRectifier(MacroAssembler& masm,
+ ArgumentsRectifierKind kind) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateArgumentsRectifier");
+
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ argumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ trialInliningArgumentsRectifierOffset_ = startTrampolineCode(masm);
+ break;
+ }
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ] <- esp
+
+ // Frame prologue.
+ //
+ // NOTE: if this changes, fix the Baseline bailout code too!
+ // See BaselineStackBuilder::calculatePrevFramePtr and
+ // BaselineStackBuilder::buildRectifierFrame (in BaselineBailouts.cpp).
+ masm.push(FramePointer);
+ masm.movl(esp, FramePointer); // Save %esp.
+
+ // Load argc.
+ masm.loadNumActualArgs(FramePointer, esi);
+
+ // Load the number of |undefined|s to push into %ecx.
+ masm.loadPtr(Address(ebp, RectifierFrameLayout::offsetOfCalleeToken()), eax);
+ masm.mov(eax, ecx);
+ masm.andl(Imm32(CalleeTokenMask), ecx);
+ masm.loadFunctionArgCount(ecx, ecx);
+
+ // The frame pointer and its padding are pushed on the stack.
+ // Including |this|, there are (|nformals| + 1) arguments to push to the
+ // stack. Then we push a JitFrameLayout. We compute the padding expressed
+ // in the number of extra |undefined| values to push on the stack.
+ static_assert(
+ sizeof(JitFrameLayout) % JitStackAlignment == 0,
+ "No need to consider the JitFrameLayout for aligning the stack");
+ static_assert(
+ JitStackAlignment % sizeof(Value) == 0,
+ "Ensure that we can pad the stack by pushing extra UndefinedValue");
+ static_assert(IsPowerOfTwo(JitStackValueAlignment),
+ "must have power of two for masm.andl to do its job");
+
+ masm.addl(
+ Imm32(JitStackValueAlignment - 1 /* for padding */ + 1 /* for |this| */),
+ ecx);
+
+ // Account for newTarget, if necessary.
+ static_assert(
+ CalleeToken_FunctionConstructing == 1,
+ "Ensure that we can use the constructing bit to count an extra push");
+ masm.mov(eax, edx);
+ masm.andl(Imm32(CalleeToken_FunctionConstructing), edx);
+ masm.addl(edx, ecx);
+
+ masm.andl(Imm32(~(JitStackValueAlignment - 1)), ecx);
+ masm.subl(esi, ecx);
+ masm.subl(Imm32(1), ecx); // For |this|.
+
+ // Copy the number of actual arguments into edx.
+ masm.mov(esi, edx);
+
+ masm.moveValue(UndefinedValue(), ValueOperand(ebx, edi));
+
+ // Caller:
+ // [arg2] [arg1] [this] [ [argc] [callee] [descr] [raddr] ]
+ // '-- #esi ---'
+ //
+ // Rectifier frame:
+ // [ebp'] <- ebp [padding] <- esp [undef] [undef] [arg2] [arg1] [this]
+ // '--- #ecx ----' '-- #esi ---'
+ //
+ // [ [argc] [callee] [descr] [raddr] ]
+
+ // Push undefined.
+ {
+ Label undefLoopTop;
+ masm.bind(&undefLoopTop);
+
+ masm.push(ebx); // type(undefined);
+ masm.push(edi); // payload(undefined);
+ masm.subl(Imm32(1), ecx);
+ masm.j(Assembler::NonZero, &undefLoopTop);
+ }
+
+ // Get the topmost argument.
+ BaseIndex b(FramePointer, esi, TimesEight, sizeof(RectifierFrameLayout));
+ masm.lea(Operand(b), ecx);
+
+ // Push arguments, |nargs| + 1 times (to include |this|).
+ masm.addl(Imm32(1), esi);
+ {
+ Label copyLoopTop;
+
+ masm.bind(&copyLoopTop);
+ masm.push(Operand(ecx, sizeof(Value) / 2));
+ masm.push(Operand(ecx, 0x0));
+ masm.subl(Imm32(sizeof(Value)), ecx);
+ masm.subl(Imm32(1), esi);
+ masm.j(Assembler::NonZero, &copyLoopTop);
+ }
+
+ {
+ Label notConstructing;
+
+ masm.mov(eax, ebx);
+ masm.branchTest32(Assembler::Zero, ebx,
+ Imm32(CalleeToken_FunctionConstructing),
+ &notConstructing);
+
+ BaseValueIndex src(FramePointer, edx,
+ sizeof(RectifierFrameLayout) + sizeof(Value));
+
+ masm.andl(Imm32(CalleeTokenMask), ebx);
+ masm.loadFunctionArgCount(ebx, ebx);
+
+ BaseValueIndex dst(esp, ebx, sizeof(Value));
+
+ ValueOperand newTarget(ecx, edi);
+
+ masm.loadValue(src, newTarget);
+ masm.storeValue(newTarget, dst);
+
+ masm.bind(&notConstructing);
+ }
+
+ // Construct JitFrameLayout.
+ masm.push(eax); // callee token
+ masm.pushFrameDescriptorForJitCall(FrameType::Rectifier, edx, edx);
+
+ // Call the target function.
+ masm.andl(Imm32(CalleeTokenMask), eax);
+ switch (kind) {
+ case ArgumentsRectifierKind::Normal:
+ masm.loadJitCodeRaw(eax, eax);
+ argumentsRectifierReturnOffset_ = masm.callJitNoProfiler(eax);
+ break;
+ case ArgumentsRectifierKind::TrialInlining:
+ Label noBaselineScript, done;
+ masm.loadBaselineJitCodeRaw(eax, ebx, &noBaselineScript);
+ masm.callJitNoProfiler(ebx);
+ masm.jump(&done);
+
+ // See BaselineCacheIRCompiler::emitCallInlinedFunction.
+ masm.bind(&noBaselineScript);
+ masm.loadJitCodeRaw(eax, eax);
+ masm.callJitNoProfiler(eax);
+ masm.bind(&done);
+ break;
+ }
+
+ masm.mov(FramePointer, StackPointer);
+ masm.pop(FramePointer);
+ masm.ret();
+}
+
+static void PushBailoutFrame(MacroAssembler& masm, Register spArg) {
+ // Push registers such that we can access them from [base + code].
+ DumpAllRegs(masm);
+
+ // The current stack pointer is the first argument to jit::Bailout.
+ masm.movl(esp, spArg);
+}
+
+static void GenerateBailoutThunk(MacroAssembler& masm, Label* bailoutTail) {
+ PushBailoutFrame(masm, eax);
+
+ // Make space for Bailout's bailoutInfo outparam.
+ masm.reserveStack(sizeof(void*));
+ masm.movl(esp, ebx);
+
+ // Call the bailout function.
+ using Fn = bool (*)(BailoutStack * sp, BaselineBailoutInfo * *info);
+ masm.setupUnalignedABICall(ecx);
+ masm.passABIArg(eax);
+ masm.passABIArg(ebx);
+ masm.callWithABI<Fn, Bailout>(MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckOther);
+
+ masm.pop(ecx); // Get the bailoutInfo outparam.
+
+ // Remove both the bailout frame and the topmost Ion frame's stack.
+ masm.moveToStackPtr(FramePointer);
+
+ // Jump to shared bailout tail. The BailoutInfo pointer has to be in ecx.
+ masm.jmp(bailoutTail);
+}
+
+void JitRuntime::generateBailoutHandler(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutHandler");
+
+ bailoutHandlerOffset_ = startTrampolineCode(masm);
+
+ GenerateBailoutThunk(masm, bailoutTail);
+}
+
+bool JitRuntime::generateVMWrapper(JSContext* cx, MacroAssembler& masm,
+ const VMFunctionData& f, DynFn nativeFun,
+ uint32_t* wrapperOffset) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateVMWrapper");
+
+ *wrapperOffset = startTrampolineCode(masm);
+
+ // Avoid conflicts with argument registers while discarding the result after
+ // the function call.
+ AllocatableGeneralRegisterSet regs(Register::Codes::WrapperMask);
+
+ static_assert(
+ (Register::Codes::VolatileMask & ~Register::Codes::WrapperMask) == 0,
+ "Wrapper register set must be a superset of Volatile register set.");
+
+ // The context is the first argument.
+ Register cxreg = regs.takeAny();
+
+ // Stack is:
+ // ... frame ...
+ // +8 [args]
+ // +4 descriptor
+ // +0 returnAddress
+ //
+ // Push the frame pointer to finish the exit frame, then link it up.
+ masm.Push(FramePointer);
+ masm.loadJSContext(cxreg);
+ masm.enterExitFrame(cxreg, regs.getAny(), &f);
+
+ // Save the current stack pointer as the base for copying arguments.
+ Register argsBase = InvalidReg;
+ if (f.explicitArgs) {
+ argsBase = regs.takeAny();
+ masm.lea(Operand(esp, ExitFrameLayout::SizeWithFooter()), argsBase);
+ }
+
+ // Reserve space for the outparameter.
+ Register outReg = InvalidReg;
+ switch (f.outParam) {
+ case Type_Value:
+ outReg = regs.takeAny();
+ masm.Push(UndefinedValue());
+ masm.movl(esp, outReg);
+ break;
+
+ case Type_Handle:
+ outReg = regs.takeAny();
+ masm.PushEmptyRooted(f.outParamRootType);
+ masm.movl(esp, outReg);
+ break;
+
+ case Type_Int32:
+ case Type_Pointer:
+ case Type_Bool:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(int32_t));
+ masm.movl(esp, outReg);
+ break;
+
+ case Type_Double:
+ outReg = regs.takeAny();
+ masm.reserveStack(sizeof(double));
+ masm.movl(esp, outReg);
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ masm.setupUnalignedABICall(regs.getAny());
+ masm.passABIArg(cxreg);
+
+ size_t argDisp = 0;
+
+ // Copy arguments.
+ for (uint32_t explicitArg = 0; explicitArg < f.explicitArgs; explicitArg++) {
+ switch (f.argProperties(explicitArg)) {
+ case VMFunctionData::WordByValue:
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByValue:
+ // We don't pass doubles in float registers on x86, so no need
+ // to check for argPassedInFloatReg.
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ masm.passABIArg(MoveOperand(argsBase, argDisp), MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::WordByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += sizeof(void*);
+ break;
+ case VMFunctionData::DoubleByRef:
+ masm.passABIArg(
+ MoveOperand(argsBase, argDisp, MoveOperand::EFFECTIVE_ADDRESS),
+ MoveOp::GENERAL);
+ argDisp += 2 * sizeof(void*);
+ break;
+ }
+ }
+
+ // Copy the implicit outparam, if any.
+ if (outReg != InvalidReg) {
+ masm.passABIArg(outReg);
+ }
+
+ masm.callWithABI(nativeFun, MoveOp::GENERAL,
+ CheckUnsafeCallWithABI::DontCheckHasExitFrame);
+
+ // Test for failure.
+ switch (f.failType()) {
+ case Type_Cell:
+ masm.branchTestPtr(Assembler::Zero, eax, eax, masm.failureLabel());
+ break;
+ case Type_Bool:
+ masm.testb(eax, eax);
+ masm.j(Assembler::Zero, masm.failureLabel());
+ break;
+ case Type_Void:
+ break;
+ default:
+ MOZ_CRASH("unknown failure kind");
+ }
+
+ // Load the outparam and free any allocated stack.
+ switch (f.outParam) {
+ case Type_Handle:
+ masm.popRooted(f.outParamRootType, ReturnReg, JSReturnOperand);
+ break;
+
+ case Type_Value:
+ masm.Pop(JSReturnOperand);
+ break;
+
+ case Type_Int32:
+ case Type_Pointer:
+ masm.Pop(ReturnReg);
+ break;
+
+ case Type_Bool:
+ masm.Pop(ReturnReg);
+ masm.movzbl(ReturnReg, ReturnReg);
+ break;
+
+ case Type_Double:
+ masm.Pop(ReturnDoubleReg);
+ break;
+
+ default:
+ MOZ_ASSERT(f.outParam == Type_Void);
+ break;
+ }
+
+ // Until C++ code is instrumented against Spectre, prevent speculative
+ // execution from returning any private data.
+ if (f.returnsData() && JitOptions.spectreJitToCxxCalls) {
+ masm.speculationBarrier();
+ }
+
+ // Pop ExitFooterFrame and the frame pointer.
+ masm.leaveExitFrame(sizeof(void*));
+
+ // Return. Subtract sizeof(void*) for the frame pointer.
+ masm.retn(Imm32(sizeof(ExitFrameLayout) - sizeof(void*) +
+ f.explicitStackSlots() * sizeof(void*) +
+ f.extraValuesToPop * sizeof(Value)));
+
+ return true;
+}
+
+uint32_t JitRuntime::generatePreBarrier(JSContext* cx, MacroAssembler& masm,
+ MIRType type) {
+ AutoCreatedBy acb(masm, "JitRuntime::generatePreBarrier");
+
+ uint32_t offset = startTrampolineCode(masm);
+
+ static_assert(PreBarrierReg == edx);
+ Register temp1 = eax;
+ Register temp2 = ebx;
+ Register temp3 = ecx;
+ masm.push(temp1);
+ masm.push(temp2);
+ masm.push(temp3);
+
+ Label noBarrier;
+ masm.emitPreBarrierFastPath(cx->runtime(), type, temp1, temp2, temp3,
+ &noBarrier);
+
+ // Call into C++ to mark this GC thing.
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+
+ LiveRegisterSet save;
+ save.set() = RegisterSet(GeneralRegisterSet(Registers::VolatileMask),
+ FloatRegisterSet(FloatRegisters::VolatileMask));
+ masm.PushRegsInMask(save);
+
+ masm.movl(ImmPtr(cx->runtime()), ecx);
+
+ masm.setupUnalignedABICall(eax);
+ masm.passABIArg(ecx);
+ masm.passABIArg(edx);
+ masm.callWithABI(JitPreWriteBarrier(type));
+
+ masm.PopRegsInMask(save);
+ masm.ret();
+
+ masm.bind(&noBarrier);
+ masm.pop(temp3);
+ masm.pop(temp2);
+ masm.pop(temp1);
+ masm.ret();
+
+ return offset;
+}
+
+void JitRuntime::generateBailoutTailStub(MacroAssembler& masm,
+ Label* bailoutTail) {
+ AutoCreatedBy acb(masm, "JitRuntime::generateBailoutTailStub");
+
+ masm.bind(bailoutTail);
+ masm.generateBailoutTail(edx, ecx);
+}